From 23f5fa85fe5c212ae90e06a3add9545227100b29 Mon Sep 17 00:00:00 2001 From: Arkadiy Kukarkin Date: Wed, 18 Mar 2026 13:18:49 +0100 Subject: [PATCH 1/6] add export-keys command --- cmd/app.go | 1 + cmd/wallet/export_keys.go | 118 ++++++++++ docs/en/SUMMARY.md | 1 + docs/en/cli-reference/wallet/README.md | 9 +- docs/en/cli-reference/wallet/export-keys.md | 31 +++ handler/wallet/export_keys.go | 111 ++++++++++ handler/wallet/export_keys_test.go | 234 ++++++++++++++++++++ model/replication.go | 2 +- 8 files changed, 502 insertions(+), 5 deletions(-) create mode 100644 cmd/wallet/export_keys.go create mode 100644 docs/en/cli-reference/wallet/export-keys.md create mode 100644 handler/wallet/export_keys.go create mode 100644 handler/wallet/export_keys_test.go diff --git a/cmd/app.go b/cmd/app.go index d21b3338..1600bf63 100644 --- a/cmd/app.go +++ b/cmd/app.go @@ -168,6 +168,7 @@ Upgrading: wallet.ImportCmd, wallet.ListCmd, wallet.RemoveCmd, + wallet.ExportKeysCmd, }, }, { diff --git a/cmd/wallet/export_keys.go b/cmd/wallet/export_keys.go new file mode 100644 index 00000000..0c3b747c --- /dev/null +++ b/cmd/wallet/export_keys.go @@ -0,0 +1,118 @@ +package wallet + +import ( + "bufio" + "fmt" + "os" + "strings" + + "github.com/cockroachdb/errors" + "github.com/data-preservation-programs/singularity/database" + "github.com/data-preservation-programs/singularity/handler/wallet" + "github.com/data-preservation-programs/singularity/util/keystore" + "github.com/urfave/cli/v2" +) + +var ExportKeysCmd = &cli.Command{ + Name: "export-keys", + Usage: "Migrate private keys from database (legacy Actor.PrivateKey) to the filesystem keystore", + Description: `Reads private keys stored in the legacy actors table and saves them to +the filesystem keystore (~/.singularity/keystore or SINGULARITY_KEYSTORE). +Creates Wallet records for each exported key and links them to the +corresponding Actor. + +This command is idempotent — actors whose address already has a Wallet +record are skipped. Keys that fail to parse are reported but do not +abort the migration. + +After exporting, prompts to drop the orphaned private_key column from +the actors table. This is irreversible — verify keys are in the keystore +before confirming. For scripted use, pass --drop-db-keys --i-am-really-sure +to skip the prompt.`, + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "drop-db-keys", + Usage: "drop the private_key column from the actors table after export", + }, + &cli.BoolFlag{ + Name: "i-am-really-sure", + Usage: "confirm column drop (required with --drop-db-keys)", + }, + }, + Action: func(c *cli.Context) error { + db, closer, err := database.OpenFromCLI(c) + if err != nil { + return errors.WithStack(err) + } + defer closer.Close() + + keystoreDir := wallet.GetKeystoreDir() + ks, err := keystore.NewLocalKeyStore(keystoreDir) + if err != nil { + return errors.Wrap(err, "failed to init keystore") + } + + result, err := wallet.ExportKeysHandler(c.Context, db, ks) + if err != nil { + return errors.WithStack(err) + } + + fmt.Printf("exported: %d\n", result.Exported) + fmt.Printf("skipped: %d (wallet already exists)\n", result.Skipped) + if len(result.Errors) > 0 { + fmt.Printf("errors: %d\n", len(result.Errors)) + for _, e := range result.Errors { + fmt.Printf(" - %s\n", e) + } + fmt.Println("\nfix the errors above and re-run before dropping the column") + return nil + } + + // list keys in keystore so user can verify + keys, err := ks.List() + if err != nil { + return errors.Wrap(err, "failed to list keystore") + } + fmt.Printf("\nkeystore: %s (%d keys)\n", keystoreDir, len(keys)) + + // determine whether to drop the column + dropFlags := c.Bool("drop-db-keys") + sureFlag := c.Bool("i-am-really-sure") + + if dropFlags && !sureFlag { + return errors.New("--drop-db-keys requires --i-am-really-sure") + } + + shouldDrop := dropFlags && sureFlag + if !shouldDrop { + // interactive prompt + fmt.Printf("\n" + + "WARNING: the next step will DROP the private_key column from the\n" + + "actors table. This is IRREVERSIBLE. All key material in the database\n" + + "will be permanently deleted.\n" + + "\n" + + "Verify that your keys are present in the keystore directory above\n" + + "before continuing. For scripted use, pass:\n" + + " --drop-db-keys --i-am-really-sure\n\n") + fmt.Printf("Drop private_key column? [y/N] ") + + scanner := bufio.NewScanner(os.Stdin) + if !scanner.Scan() { + return nil + } + answer := strings.TrimSpace(strings.ToLower(scanner.Text())) + if answer != "y" && answer != "yes" { + fmt.Println("aborted -- keys exported but column retained") + return nil + } + shouldDrop = true + } + + if err := wallet.DropPrivateKeyColumn(db); err != nil { + return errors.Wrap(err, "failed to drop private_key column") + } + fmt.Println("dropped private_key column from actors table") + + return nil + }, +} diff --git a/docs/en/SUMMARY.md b/docs/en/SUMMARY.md index f02be6ee..39816bf4 100644 --- a/docs/en/SUMMARY.md +++ b/docs/en/SUMMARY.md @@ -66,6 +66,7 @@ * [Import](cli-reference/wallet/import.md) * [List](cli-reference/wallet/list.md) * [Remove](cli-reference/wallet/remove.md) + * [Export Keys](cli-reference/wallet/export-keys.md) * [Storage](cli-reference/storage/README.md) * [Create](cli-reference/storage/create/README.md) * [Azureblob](cli-reference/storage/create/azureblob.md) diff --git a/docs/en/cli-reference/wallet/README.md b/docs/en/cli-reference/wallet/README.md index 0e4e3b18..981094f7 100644 --- a/docs/en/cli-reference/wallet/README.md +++ b/docs/en/cli-reference/wallet/README.md @@ -9,10 +9,11 @@ USAGE: singularity wallet command [command options] COMMANDS: - import Import a wallet from a private key file into the keystore - list List all imported wallets - remove Remove a wallet - help, h Shows a list of commands or help for one command + import Import a wallet from a private key file into the keystore + list List all imported wallets + remove Remove a wallet + export-keys Migrate private keys from database (legacy Actor.PrivateKey) to the filesystem keystore + help, h Shows a list of commands or help for one command OPTIONS: --help, -h show help diff --git a/docs/en/cli-reference/wallet/export-keys.md b/docs/en/cli-reference/wallet/export-keys.md new file mode 100644 index 00000000..d8a23cd8 --- /dev/null +++ b/docs/en/cli-reference/wallet/export-keys.md @@ -0,0 +1,31 @@ +# Migrate private keys from database (legacy Actor.PrivateKey) to the filesystem keystore + +{% code fullWidth="true" %} +``` +NAME: + singularity wallet export-keys - Migrate private keys from database (legacy Actor.PrivateKey) to the filesystem keystore + +USAGE: + singularity wallet export-keys [command options] + +DESCRIPTION: + Reads private keys stored in the legacy actors table and saves them to + the filesystem keystore (~/.singularity/keystore or SINGULARITY_KEYSTORE). + Creates Wallet records for each exported key and links them to the + corresponding Actor. + + This command is idempotent — actors whose address already has a Wallet + record are skipped. Keys that fail to parse are reported but do not + abort the migration. + + After exporting, prompts to drop the orphaned private_key column from + the actors table. This is irreversible — verify keys are in the keystore + before confirming. For scripted use, pass --drop-db-keys --i-am-really-sure + to skip the prompt. + +OPTIONS: + --drop-db-keys drop the private_key column from the actors table after export (default: false) + --i-am-really-sure confirm column drop (required with --drop-db-keys) (default: false) + --help, -h show help +``` +{% endcode %} diff --git a/handler/wallet/export_keys.go b/handler/wallet/export_keys.go new file mode 100644 index 00000000..6c5f702f --- /dev/null +++ b/handler/wallet/export_keys.go @@ -0,0 +1,111 @@ +package wallet + +import ( + "context" + "fmt" + + "github.com/cockroachdb/errors" + "github.com/data-preservation-programs/singularity/model" + "github.com/data-preservation-programs/singularity/util/keystore" + "gorm.io/gorm" +) + +// ExportKeysResult reports what export-keys did for each actor +type ExportKeysResult struct { + Exported int // actors whose keys were saved to keystore + wallet record created + Skipped int // actors whose wallet already exists (by address match) + Errors []string // actors that failed (logged but don't abort) +} + +// exports private keys from the legacy Actor.PrivateKey column into the +// filesystem keystore, creating Wallet records where needed. +// idempotent -- skips actors whose address already has a Wallet record. +// does not delete Actor.PrivateKey; caller decides when to drop the column. +func ExportKeysHandler( + ctx context.Context, + db *gorm.DB, + ks keystore.KeyStore, +) (*ExportKeysResult, error) { + db = db.WithContext(ctx) + + var actors []model.Actor + err := db.Where("private_key != '' AND private_key IS NOT NULL").Find(&actors).Error + if err != nil { + return nil, errors.Wrap(err, "failed to query actors with private keys") + } + + result := &ExportKeysResult{} + + for _, actor := range actors { + exported, errMsg := exportOneKey(db, ks, actor) + if errMsg != "" { + result.Errors = append(result.Errors, errMsg) + continue + } + if exported { + result.Exported++ + } else { + result.Skipped++ + } + } + + return result, nil +} + +// exports a single actor's key to keystore, returns (true, "") on success, +// (false, "") on skip, ("", errMsg) on failure +func exportOneKey(db *gorm.DB, ks keystore.KeyStore, actor model.Actor) (exported bool, errMsg string) { + // derive address to check for existing wallet + addr, err := keystore.AddressFromExport(actor.PrivateKey) + if err != nil { + return false, fmt.Sprintf("actor %s: invalid key format: %v", actor.ID, err) + } + + // check if wallet already exists for this address + var existing model.Wallet + err = db.Where("address = ?", addr.String()).First(&existing).Error + if err == nil { + // wallet exists, link actor if not already linked + if existing.ActorID == nil { + existing.ActorID = &actor.ID + db.Save(&existing) + } + logger.Debugw("wallet already exists for actor", "actorID", actor.ID, "address", addr.String()) + return false, "" + } + if !errors.Is(err, gorm.ErrRecordNotFound) { + return false, fmt.Sprintf("actor %s: db query failed: %v", actor.ID, err) + } + + // save key to keystore + keyPath, _, err := ks.Put(actor.PrivateKey) + if err != nil { + return false, fmt.Sprintf("actor %s: keystore write failed: %v", actor.ID, err) + } + + // create wallet record + w := model.Wallet{ + KeyPath: keyPath, + KeyStore: "local", + Address: addr.String(), + ActorID: &actor.ID, + } + if err := db.Create(&w).Error; err != nil { + // cleanup keystore file on db failure + ks.Delete(keyPath) + return false, fmt.Sprintf("actor %s: wallet create failed: %v", actor.ID, err) + } + + logger.Infow("exported actor key to keystore", + "actorID", actor.ID, "address", addr.String(), "walletID", w.ID) + return true, "" +} + +// drops the private_key column from the actors table. +// caller is responsible for confirming this is desired. +func DropPrivateKeyColumn(db *gorm.DB) error { + if !db.Migrator().HasColumn(&model.Actor{}, "private_key") { + return nil // already dropped + } + return db.Exec("ALTER TABLE actors DROP COLUMN private_key").Error +} diff --git a/handler/wallet/export_keys_test.go b/handler/wallet/export_keys_test.go new file mode 100644 index 00000000..330a1e3c --- /dev/null +++ b/handler/wallet/export_keys_test.go @@ -0,0 +1,234 @@ +package wallet + +import ( + "context" + "testing" + + "github.com/data-preservation-programs/singularity/model" + "github.com/data-preservation-programs/singularity/util/keystore" + "github.com/data-preservation-programs/singularity/util/testutil" + "github.com/stretchr/testify/require" + "gorm.io/gorm" +) + +func TestExportKeysHandler_ExportsActorKey(t *testing.T) { + testutil.All(t, func(ctx context.Context, t *testing.T, db *gorm.DB) { + ks, err := keystore.NewLocalKeyStore(t.TempDir()) + require.NoError(t, err) + + // create an actor with a legacy private key + actor := model.Actor{ + ID: "f01234", + Address: testutil.TestWalletAddr, + PrivateKey: testutil.TestPrivateKeyHex, + } + require.NoError(t, db.Create(&actor).Error) + + result, err := ExportKeysHandler(ctx, db, ks) + require.NoError(t, err) + require.Equal(t, 1, result.Exported) + require.Equal(t, 0, result.Skipped) + require.Empty(t, result.Errors) + + // verify wallet was created and linked to actor + var w model.Wallet + require.NoError(t, db.Where("address = ?", testutil.TestWalletAddr).First(&w).Error) + require.Equal(t, "local", w.KeyStore) + require.NotNil(t, w.ActorID) + require.Equal(t, "f01234", *w.ActorID) + + // verify key is readable from keystore + key, err := ks.Get(w.KeyPath) + require.NoError(t, err) + require.Equal(t, testutil.TestPrivateKeyHex, key) + }) +} + +func TestExportKeysHandler_SkipsExistingWallet(t *testing.T) { + testutil.All(t, func(ctx context.Context, t *testing.T, db *gorm.DB) { + ks, err := keystore.NewLocalKeyStore(t.TempDir()) + require.NoError(t, err) + + // create actor with legacy key + actor := model.Actor{ + ID: "f01234", + Address: testutil.TestWalletAddr, + PrivateKey: testutil.TestPrivateKeyHex, + } + require.NoError(t, db.Create(&actor).Error) + + // pre-import the wallet via normal import path + h := DefaultHandler{} + _, err = h.ImportKeystoreHandler(ctx, db, ks, ImportKeystoreRequest{ + PrivateKey: testutil.TestPrivateKeyHex, + }) + require.NoError(t, err) + + result, err := ExportKeysHandler(ctx, db, ks) + require.NoError(t, err) + require.Equal(t, 0, result.Exported) + require.Equal(t, 1, result.Skipped) + require.Empty(t, result.Errors) + + // verify still only one wallet record + var count int64 + db.Model(&model.Wallet{}).Count(&count) + require.Equal(t, int64(1), count) + }) +} + +func TestExportKeysHandler_SkipsExistingWalletLinksActor(t *testing.T) { + testutil.All(t, func(ctx context.Context, t *testing.T, db *gorm.DB) { + ks, err := keystore.NewLocalKeyStore(t.TempDir()) + require.NoError(t, err) + + // create actor with legacy key + actor := model.Actor{ + ID: "f01234", + Address: testutil.TestWalletAddr, + PrivateKey: testutil.TestPrivateKeyHex, + } + require.NoError(t, db.Create(&actor).Error) + + // pre-import the wallet WITHOUT actor linkage + h := DefaultHandler{} + _, err = h.ImportKeystoreHandler(ctx, db, ks, ImportKeystoreRequest{ + PrivateKey: testutil.TestPrivateKeyHex, + }) + require.NoError(t, err) + + result, err := ExportKeysHandler(ctx, db, ks) + require.NoError(t, err) + require.Equal(t, 0, result.Exported) + require.Equal(t, 1, result.Skipped) + + // verify actor was linked + var w model.Wallet + require.NoError(t, db.Where("address = ?", testutil.TestWalletAddr).First(&w).Error) + require.NotNil(t, w.ActorID) + require.Equal(t, "f01234", *w.ActorID) + }) +} + +func TestExportKeysHandler_Idempotent(t *testing.T) { + testutil.All(t, func(ctx context.Context, t *testing.T, db *gorm.DB) { + ks, err := keystore.NewLocalKeyStore(t.TempDir()) + require.NoError(t, err) + + actor := model.Actor{ + ID: "f01234", + Address: testutil.TestWalletAddr, + PrivateKey: testutil.TestPrivateKeyHex, + } + require.NoError(t, db.Create(&actor).Error) + + // first run exports + r1, err := ExportKeysHandler(ctx, db, ks) + require.NoError(t, err) + require.Equal(t, 1, r1.Exported) + + // second run skips + r2, err := ExportKeysHandler(ctx, db, ks) + require.NoError(t, err) + require.Equal(t, 0, r2.Exported) + require.Equal(t, 1, r2.Skipped) + + // still only one wallet + var count int64 + db.Model(&model.Wallet{}).Count(&count) + require.Equal(t, int64(1), count) + }) +} + +func TestExportKeysHandler_NoActorsWithKeys(t *testing.T) { + testutil.All(t, func(ctx context.Context, t *testing.T, db *gorm.DB) { + ks, err := keystore.NewLocalKeyStore(t.TempDir()) + require.NoError(t, err) + + // actor with no private key + actor := model.Actor{ + ID: "f09999", + Address: "f1abc", + } + require.NoError(t, db.Create(&actor).Error) + + result, err := ExportKeysHandler(ctx, db, ks) + require.NoError(t, err) + require.Equal(t, 0, result.Exported) + require.Equal(t, 0, result.Skipped) + require.Empty(t, result.Errors) + }) +} + +func TestExportKeysHandler_InvalidKeyRecordsError(t *testing.T) { + testutil.All(t, func(ctx context.Context, t *testing.T, db *gorm.DB) { + ks, err := keystore.NewLocalKeyStore(t.TempDir()) + require.NoError(t, err) + + // actor with garbage key + actor := model.Actor{ + ID: "f05555", + Address: "f1bad", + PrivateKey: "not-a-valid-key", + } + require.NoError(t, db.Create(&actor).Error) + + result, err := ExportKeysHandler(ctx, db, ks) + require.NoError(t, err) // overall operation succeeds + require.Equal(t, 0, result.Exported) + require.Equal(t, 0, result.Skipped) + require.Len(t, result.Errors, 1) + require.Contains(t, result.Errors[0], "f05555") + require.Contains(t, result.Errors[0], "invalid key format") + }) +} + +func TestDropPrivateKeyColumn(t *testing.T) { + testutil.All(t, func(ctx context.Context, t *testing.T, db *gorm.DB) { + // column exists after AutoMigrate + require.True(t, db.Migrator().HasColumn(&model.Actor{}, "private_key")) + + // drop it + require.NoError(t, DropPrivateKeyColumn(db)) + require.False(t, db.Migrator().HasColumn(&model.Actor{}, "private_key")) + + // idempotent -- second call is a no-op + require.NoError(t, DropPrivateKeyColumn(db)) + }) +} + +func TestDropPrivateKeyColumn_ExportThenDrop(t *testing.T) { + testutil.All(t, func(ctx context.Context, t *testing.T, db *gorm.DB) { + ks, err := keystore.NewLocalKeyStore(t.TempDir()) + require.NoError(t, err) + + // create actor with legacy key + actor := model.Actor{ + ID: "f01234", + Address: testutil.TestWalletAddr, + PrivateKey: testutil.TestPrivateKeyHex, + } + require.NoError(t, db.Create(&actor).Error) + + // export + result, err := ExportKeysHandler(ctx, db, ks) + require.NoError(t, err) + require.Equal(t, 1, result.Exported) + require.Empty(t, result.Errors) + + // drop column + require.NoError(t, DropPrivateKeyColumn(db)) + + // key is still in keystore + var w model.Wallet + require.NoError(t, db.Where("address = ?", testutil.TestWalletAddr).First(&w).Error) + key, err := ks.Get(w.KeyPath) + require.NoError(t, err) + require.Equal(t, testutil.TestPrivateKeyHex, key) + + // actor record still exists, just without the key column + var a model.Actor + require.NoError(t, db.First(&a, "id = ?", "f01234").Error) + require.Equal(t, testutil.TestWalletAddr, a.Address) + }) +} diff --git a/model/replication.go b/model/replication.go index 8d98e858..11181679 100644 --- a/model/replication.go +++ b/model/replication.go @@ -181,7 +181,7 @@ type Schedule struct { type Actor struct { ID string `gorm:"primaryKey;size:15" json:"id"` // actor ID (f0...) Address string `gorm:"index" json:"address"` // filecoin address - PrivateKey string `json:"privateKey,omitempty" table:"-"` // TODO: orphaned column, will be dropped by export-keys command + PrivateKey string `json:"privateKey,omitempty" table:"-"` // orphaned: run `singularity wallet export-keys` to migrate, then drop column } // GORM will rename "wallets" table to "actors" on AutoMigrate From 6ace2283ac5a30b8c41c0e3492142503f8614474 Mon Sep 17 00:00:00 2001 From: Arkadiy Kukarkin Date: Wed, 18 Mar 2026 13:38:14 +0100 Subject: [PATCH 2/6] prompt user to drop legacy pkey column on export --- cmd/wallet/export_keys.go | 17 ++++++++++++----- handler/wallet/export_keys.go | 6 +++++- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/cmd/wallet/export_keys.go b/cmd/wallet/export_keys.go index 0c3b747c..e8313d5e 100644 --- a/cmd/wallet/export_keys.go +++ b/cmd/wallet/export_keys.go @@ -46,6 +46,12 @@ to skip the prompt.`, } defer closer.Close() + // check if column still exists -- if not, nothing to do + if !wallet.HasPrivateKeyColumn(db) { + fmt.Println("nothing to do -- private_key column already dropped") + return nil + } + keystoreDir := wallet.GetKeystoreDir() ks, err := keystore.NewLocalKeyStore(keystoreDir) if err != nil { @@ -58,7 +64,9 @@ to skip the prompt.`, } fmt.Printf("exported: %d\n", result.Exported) - fmt.Printf("skipped: %d (wallet already exists)\n", result.Skipped) + if result.Skipped > 0 { + fmt.Printf("skipped: %d (wallet already exists)\n", result.Skipped) + } if len(result.Errors) > 0 { fmt.Printf("errors: %d\n", len(result.Errors)) for _, e := range result.Errors { @@ -76,14 +84,14 @@ to skip the prompt.`, fmt.Printf("\nkeystore: %s (%d keys)\n", keystoreDir, len(keys)) // determine whether to drop the column - dropFlags := c.Bool("drop-db-keys") + dropFlag := c.Bool("drop-db-keys") sureFlag := c.Bool("i-am-really-sure") - if dropFlags && !sureFlag { + if dropFlag && !sureFlag { return errors.New("--drop-db-keys requires --i-am-really-sure") } - shouldDrop := dropFlags && sureFlag + shouldDrop := dropFlag && sureFlag if !shouldDrop { // interactive prompt fmt.Printf("\n" + @@ -105,7 +113,6 @@ to skip the prompt.`, fmt.Println("aborted -- keys exported but column retained") return nil } - shouldDrop = true } if err := wallet.DropPrivateKeyColumn(db); err != nil { diff --git a/handler/wallet/export_keys.go b/handler/wallet/export_keys.go index 6c5f702f..ec2d6dab 100644 --- a/handler/wallet/export_keys.go +++ b/handler/wallet/export_keys.go @@ -101,10 +101,14 @@ func exportOneKey(db *gorm.DB, ks keystore.KeyStore, actor model.Actor) (exporte return true, "" } +func HasPrivateKeyColumn(db *gorm.DB) bool { + return db.Migrator().HasColumn(&model.Actor{}, "private_key") +} + // drops the private_key column from the actors table. // caller is responsible for confirming this is desired. func DropPrivateKeyColumn(db *gorm.DB) error { - if !db.Migrator().HasColumn(&model.Actor{}, "private_key") { + if !HasPrivateKeyColumn(db) { return nil // already dropped } return db.Exec("ALTER TABLE actors DROP COLUMN private_key").Error From 751deffc4415f8db1883e3820773b1e3f88eb1c5 Mon Sep 17 00:00:00 2001 From: Arkadiy Kukarkin Date: Wed, 18 Mar 2026 13:43:14 +0100 Subject: [PATCH 3/6] warn about legacy keys on admin init --- cmd/admin/init.go | 7 +++++++ handler/wallet/export_keys.go | 11 +++++++++++ 2 files changed, 18 insertions(+) diff --git a/cmd/admin/init.go b/cmd/admin/init.go index 59cf0eaf..6c49d3a8 100644 --- a/cmd/admin/init.go +++ b/cmd/admin/init.go @@ -4,6 +4,7 @@ import ( "github.com/cockroachdb/errors" "github.com/data-preservation-programs/singularity/database" "github.com/data-preservation-programs/singularity/handler/admin" + "github.com/data-preservation-programs/singularity/handler/wallet" "github.com/urfave/cli/v2" ) @@ -27,6 +28,12 @@ var InitCmd = &cli.Command{ if err != nil { return errors.WithStack(err) } + if n := wallet.CountLegacyKeys(db); n > 0 { + return errors.Errorf( + "%d actor(s) have private keys in the database that are not usable by current code.\n"+ + "Run 'singularity wallet export-keys' to migrate them to the filesystem keystore", + n) + } if c.IsSet("identity") { err = admin.Default.SetIdentityHandler(c.Context, db, admin.SetIdentityRequest{ Identity: c.String("identity"), diff --git a/handler/wallet/export_keys.go b/handler/wallet/export_keys.go index ec2d6dab..1fe90703 100644 --- a/handler/wallet/export_keys.go +++ b/handler/wallet/export_keys.go @@ -105,6 +105,17 @@ func HasPrivateKeyColumn(db *gorm.DB) bool { return db.Migrator().HasColumn(&model.Actor{}, "private_key") } +// counts actors that still have a non-empty private_key in the database. +// returns 0 if the column has been dropped. +func CountLegacyKeys(db *gorm.DB) int64 { + if !HasPrivateKeyColumn(db) { + return 0 + } + var count int64 + db.Raw("SELECT COUNT(*) FROM actors WHERE private_key != '' AND private_key IS NOT NULL").Scan(&count) + return count +} + // drops the private_key column from the actors table. // caller is responsible for confirming this is desired. func DropPrivateKeyColumn(db *gorm.DB) error { From 72d90f2865526c0b8416e4f3c6f4c08a4884f9c8 Mon Sep 17 00:00:00 2001 From: Arkadiy Kukarkin Date: Wed, 18 Mar 2026 13:53:29 +0100 Subject: [PATCH 4/6] automigrate on daemon startup, check for legacy keys --- cmd/run/api.go | 14 +++++- cmd/run/automigrate.go | 43 +++++++++++++++++++ cmd/run/contentprovider.go | 4 +- cmd/run/datasetworker.go | 4 +- cmd/run/dealpusher.go | 4 +- cmd/run/dealtracker.go | 4 +- cmd/run/pdptracker.go | 4 +- docs/en/cli-reference/run/api.md | 5 ++- docs/en/cli-reference/run/content-provider.md | 3 +- docs/en/cli-reference/run/dataset-worker.md | 1 + docs/en/cli-reference/run/deal-pusher.md | 1 + docs/en/cli-reference/run/deal-tracker.md | 1 + docs/en/cli-reference/run/pdp-tracker.md | 1 + handler/wallet/export_keys.go | 4 ++ handler/wallet/export_keys_test.go | 29 +++++++++++++ 15 files changed, 108 insertions(+), 14 deletions(-) create mode 100644 cmd/run/automigrate.go diff --git a/cmd/run/api.go b/cmd/run/api.go index a7c84c9e..49359c38 100644 --- a/cmd/run/api.go +++ b/cmd/run/api.go @@ -9,11 +9,23 @@ var APICmd = &cli.Command{ Name: "api", Usage: "Run the singularity API", Flags: []cli.Flag{ + NoAutoMigrateFlag, &cli.StringFlag{ Name: "bind", Usage: "Bind address for the API server", Value: ":9090", }, }, - Action: api.Run, + Action: func(c *cli.Context) error { + // run automigrate + legacy key check before handing off to api.Run, + // which opens its own db connection internally + db, closer, err := openAndMigrate(c) + if err != nil { + return err + } + closer.Close() + _ = db + + return api.Run(c) + }, } diff --git a/cmd/run/automigrate.go b/cmd/run/automigrate.go new file mode 100644 index 00000000..faf30ab7 --- /dev/null +++ b/cmd/run/automigrate.go @@ -0,0 +1,43 @@ +package run + +import ( + "io" + + "github.com/cockroachdb/errors" + "github.com/data-preservation-programs/singularity/database" + "github.com/data-preservation-programs/singularity/handler/wallet" + "github.com/data-preservation-programs/singularity/model" + "github.com/urfave/cli/v2" + "gorm.io/gorm" +) + +var NoAutoMigrateFlag = &cli.BoolFlag{ + Name: "no-automigrate", + Usage: "skip automatic database migration and correctness checks on startup; only use if you run 'admin init' on every upgrade or manually before starting daemons", +} + +// opens the database, runs AutoMigrate (unless --no-automigrate), and checks +// for legacy keys that need export. returns db and closer as usual. +func openAndMigrate(c *cli.Context) (*gorm.DB, io.Closer, error) { + db, closer, err := database.OpenFromCLI(c) + if err != nil { + return nil, nil, errors.WithStack(err) + } + + if !c.Bool("no-automigrate") { + if err := model.AutoMigrate(db); err != nil { + closer.Close() + return nil, nil, errors.Wrap(err, "automigrate failed") + } + + if n := wallet.CountLegacyKeys(db); n > 0 { + closer.Close() + return nil, nil, errors.Errorf( + "%d actor(s) have private keys in the database that are not usable by current code.\n"+ + "Run 'singularity wallet export-keys' to migrate them to the filesystem keystore", + n) + } + } + + return db, closer, nil +} diff --git a/cmd/run/contentprovider.go b/cmd/run/contentprovider.go index f6026ba1..2a9e025b 100644 --- a/cmd/run/contentprovider.go +++ b/cmd/run/contentprovider.go @@ -2,7 +2,6 @@ package run import ( "github.com/cockroachdb/errors" - "github.com/data-preservation-programs/singularity/database" "github.com/data-preservation-programs/singularity/service/contentprovider" "github.com/urfave/cli/v2" ) @@ -11,6 +10,7 @@ var ContentProviderCmd = &cli.Command{ Name: "content-provider", Usage: "Start a content provider that serves retrieval requests", Flags: []cli.Flag{ + NoAutoMigrateFlag, &cli.StringFlag{ Category: "HTTP Retrieval", Name: "http-bind", @@ -50,7 +50,7 @@ var ContentProviderCmd = &cli.Command{ }, }, Action: func(c *cli.Context) error { - db, closer, err := database.OpenFromCLI(c) + db, closer, err := openAndMigrate(c) if err != nil { return errors.WithStack(err) } diff --git a/cmd/run/datasetworker.go b/cmd/run/datasetworker.go index 4348a237..01acecd8 100644 --- a/cmd/run/datasetworker.go +++ b/cmd/run/datasetworker.go @@ -4,7 +4,6 @@ import ( "time" "github.com/cockroachdb/errors" - "github.com/data-preservation-programs/singularity/database" "github.com/data-preservation-programs/singularity/service/datasetworker" "github.com/urfave/cli/v2" ) @@ -13,6 +12,7 @@ var DatasetWorkerCmd = &cli.Command{ Name: "dataset-worker", Usage: "Start a dataset preparation worker to process dataset scanning and preparation tasks", Flags: []cli.Flag{ + NoAutoMigrateFlag, &cli.IntFlag{ Name: "concurrency", Usage: "Number of concurrent workers to run", @@ -55,7 +55,7 @@ var DatasetWorkerCmd = &cli.Command{ }, }, Action: func(c *cli.Context) error { - db, closer, err := database.OpenFromCLI(c) + db, closer, err := openAndMigrate(c) if err != nil { return errors.WithStack(err) } diff --git a/cmd/run/dealpusher.go b/cmd/run/dealpusher.go index 455c0ac7..d7a11651 100644 --- a/cmd/run/dealpusher.go +++ b/cmd/run/dealpusher.go @@ -4,7 +4,6 @@ import ( "time" "github.com/cockroachdb/errors" - "github.com/data-preservation-programs/singularity/database" "github.com/data-preservation-programs/singularity/service" "github.com/data-preservation-programs/singularity/service/dealpusher" "github.com/data-preservation-programs/singularity/service/epochutil" @@ -15,6 +14,7 @@ var DealPusherCmd = &cli.Command{ Name: "deal-pusher", Usage: "Start a deal pusher that monitors deal schedules and pushes deals to storage providers", Flags: []cli.Flag{ + NoAutoMigrateFlag, &cli.UintFlag{ Name: "deal-attempts", Usage: "Number of times to attempt a deal before giving up", @@ -54,7 +54,7 @@ var DealPusherCmd = &cli.Command{ }, }, Action: func(c *cli.Context) error { - db, closer, err := database.OpenFromCLI(c) + db, closer, err := openAndMigrate(c) if err != nil { return errors.WithStack(err) } diff --git a/cmd/run/dealtracker.go b/cmd/run/dealtracker.go index 9b8eff50..fafd7fa9 100644 --- a/cmd/run/dealtracker.go +++ b/cmd/run/dealtracker.go @@ -4,7 +4,6 @@ import ( "time" "github.com/cockroachdb/errors" - "github.com/data-preservation-programs/singularity/database" "github.com/data-preservation-programs/singularity/service" "github.com/data-preservation-programs/singularity/service/dealtracker" "github.com/data-preservation-programs/singularity/service/epochutil" @@ -15,6 +14,7 @@ var DealTrackerCmd = &cli.Command{ Name: "deal-tracker", Usage: "Start a deal tracker that tracks the deal for all relevant wallets", Flags: []cli.Flag{ + NoAutoMigrateFlag, &cli.StringFlag{ Name: "market-deal-url", Usage: "The URL for ZST compressed state market deals json. Set to empty to use Lotus API.", @@ -35,7 +35,7 @@ var DealTrackerCmd = &cli.Command{ }, }, Action: func(c *cli.Context) error { - db, closer, err := database.OpenFromCLI(c) + db, closer, err := openAndMigrate(c) if err != nil { return errors.WithStack(err) } diff --git a/cmd/run/pdptracker.go b/cmd/run/pdptracker.go index 8c0de08f..4f49c165 100644 --- a/cmd/run/pdptracker.go +++ b/cmd/run/pdptracker.go @@ -8,7 +8,6 @@ import ( "github.com/cockroachdb/errors" "github.com/data-preservation-programs/go-synapse" "github.com/data-preservation-programs/go-synapse/constants" - "github.com/data-preservation-programs/singularity/database" "github.com/data-preservation-programs/singularity/service" "github.com/data-preservation-programs/singularity/service/pdptracker" "github.com/ethereum/go-ethereum/common" @@ -20,6 +19,7 @@ var PDPTrackerCmd = &cli.Command{ Name: "pdp-tracker", Usage: "Track PDP deals via Shovel event indexing (requires PostgreSQL)", Flags: []cli.Flag{ + NoAutoMigrateFlag, &cli.StringFlag{ Name: "eth-rpc", Usage: "Ethereum RPC endpoint for FEVM", @@ -43,7 +43,7 @@ var PDPTrackerCmd = &cli.Command{ return errors.New("PDP tracking requires PostgreSQL (Shovel is Postgres-only)") } - db, closer, err := database.OpenFromCLI(c) + db, closer, err := openAndMigrate(c) if err != nil { return errors.WithStack(err) } diff --git a/docs/en/cli-reference/run/api.md b/docs/en/cli-reference/run/api.md index 50adc1f7..2db5aebb 100644 --- a/docs/en/cli-reference/run/api.md +++ b/docs/en/cli-reference/run/api.md @@ -9,7 +9,8 @@ USAGE: singularity run api [command options] OPTIONS: - --bind value Bind address for the API server (default: ":9090") - --help, -h show help + --no-automigrate skip automatic database migration and correctness checks on startup; only use if you run 'admin init' on every upgrade or manually before starting daemons (default: false) + --bind value Bind address for the API server (default: ":9090") + --help, -h show help ``` {% endcode %} diff --git a/docs/en/cli-reference/run/content-provider.md b/docs/en/cli-reference/run/content-provider.md index 1a4ef8c6..8d88d819 100644 --- a/docs/en/cli-reference/run/content-provider.md +++ b/docs/en/cli-reference/run/content-provider.md @@ -9,7 +9,8 @@ USAGE: singularity run content-provider [command options] OPTIONS: - --help, -h show help + --help, -h show help + --no-automigrate skip automatic database migration and correctness checks on startup; only use if you run 'admin init' on every upgrade or manually before starting daemons (default: false) Bitswap Retrieval diff --git a/docs/en/cli-reference/run/dataset-worker.md b/docs/en/cli-reference/run/dataset-worker.md index ae3954d6..363b3c1e 100644 --- a/docs/en/cli-reference/run/dataset-worker.md +++ b/docs/en/cli-reference/run/dataset-worker.md @@ -9,6 +9,7 @@ USAGE: singularity run dataset-worker [command options] OPTIONS: + --no-automigrate skip automatic database migration and correctness checks on startup; only use if you run 'admin init' on every upgrade or manually before starting daemons (default: false) --concurrency value Number of concurrent workers to run (default: 1) --enable-scan Enable scanning of datasets (default: true) --enable-pack Enable packing of datasets that calculates CIDs and packs them into CAR files (default: true) diff --git a/docs/en/cli-reference/run/deal-pusher.md b/docs/en/cli-reference/run/deal-pusher.md index 1c839a93..791631fe 100644 --- a/docs/en/cli-reference/run/deal-pusher.md +++ b/docs/en/cli-reference/run/deal-pusher.md @@ -9,6 +9,7 @@ USAGE: singularity run deal-pusher [command options] OPTIONS: + --no-automigrate skip automatic database migration and correctness checks on startup; only use if you run 'admin init' on every upgrade or manually before starting daemons (default: false) --deal-attempts value, -d value Number of times to attempt a deal before giving up (default: 3) --max-replication-factor value, -M value Max number of replicas for each individual PieceCID across all clients and providers (default: Unlimited) --pdp-batch-size value Number of roots to include in each PDP add-roots transaction (default: 128) diff --git a/docs/en/cli-reference/run/deal-tracker.md b/docs/en/cli-reference/run/deal-tracker.md index 51757adb..b4c9d46a 100644 --- a/docs/en/cli-reference/run/deal-tracker.md +++ b/docs/en/cli-reference/run/deal-tracker.md @@ -9,6 +9,7 @@ USAGE: singularity run deal-tracker [command options] OPTIONS: + --no-automigrate skip automatic database migration and correctness checks on startup; only use if you run 'admin init' on every upgrade or manually before starting daemons (default: false) --market-deal-url value, -m value The URL for ZST compressed state market deals json. Set to empty to use Lotus API. (default: "https://marketdeals.s3.amazonaws.com/StateMarketDeals.json.zst") [$MARKET_DEAL_URL] --interval value, -i value How often to check for new deals (default: 1h0m0s) --once Run once and exit (default: false) diff --git a/docs/en/cli-reference/run/pdp-tracker.md b/docs/en/cli-reference/run/pdp-tracker.md index 0771b094..a1de8c66 100644 --- a/docs/en/cli-reference/run/pdp-tracker.md +++ b/docs/en/cli-reference/run/pdp-tracker.md @@ -9,6 +9,7 @@ USAGE: singularity run pdp-tracker [command options] OPTIONS: + --no-automigrate skip automatic database migration and correctness checks on startup; only use if you run 'admin init' on every upgrade or manually before starting daemons (default: false) --eth-rpc value Ethereum RPC endpoint for FEVM (default: "https://api.node.glif.io/rpc/v1") [$ETH_RPC_URL] --pdp-poll-interval value How often to check for new events in Shovel tables (default: 30s) --full-sync Re-index events from contract deployment by resetting the Shovel cursor. Derived PDP state (proof sets, deals) is preserved and updated via upserts. Requires an archival RPC node. Involves one RPC call per historical proof set. (default: false) diff --git a/handler/wallet/export_keys.go b/handler/wallet/export_keys.go index 1fe90703..cf60fb29 100644 --- a/handler/wallet/export_keys.go +++ b/handler/wallet/export_keys.go @@ -70,6 +70,10 @@ func exportOneKey(db *gorm.DB, ks keystore.KeyStore, actor model.Actor) (exporte existing.ActorID = &actor.ID db.Save(&existing) } + // verify the key file actually exists on disk + if !ks.Has(existing.KeyPath) { + return false, fmt.Sprintf("actor %s: wallet record exists but key file missing at %s", actor.ID, existing.KeyPath) + } logger.Debugw("wallet already exists for actor", "actorID", actor.ID, "address", addr.String()) return false, "" } diff --git a/handler/wallet/export_keys_test.go b/handler/wallet/export_keys_test.go index 330a1e3c..73652562 100644 --- a/handler/wallet/export_keys_test.go +++ b/handler/wallet/export_keys_test.go @@ -183,6 +183,35 @@ func TestExportKeysHandler_InvalidKeyRecordsError(t *testing.T) { }) } +func TestExportKeysHandler_MissingKeyFile(t *testing.T) { + testutil.All(t, func(ctx context.Context, t *testing.T, db *gorm.DB) { + ks, err := keystore.NewLocalKeyStore(t.TempDir()) + require.NoError(t, err) + + // create actor with legacy key + actor := model.Actor{ + ID: "f01234", + Address: testutil.TestWalletAddr, + PrivateKey: testutil.TestPrivateKeyHex, + } + require.NoError(t, db.Create(&actor).Error) + + // create wallet record pointing to a nonexistent key file + require.NoError(t, db.Create(&model.Wallet{ + KeyPath: "/nonexistent/key", + KeyStore: "local", + Address: testutil.TestWalletAddr, + }).Error) + + result, err := ExportKeysHandler(ctx, db, ks) + require.NoError(t, err) + require.Equal(t, 0, result.Exported) + require.Equal(t, 0, result.Skipped) + require.Len(t, result.Errors, 1) + require.Contains(t, result.Errors[0], "key file missing") + }) +} + func TestDropPrivateKeyColumn(t *testing.T) { testutil.All(t, func(ctx context.Context, t *testing.T, db *gorm.DB) { // column exists after AutoMigrate From fb7b9cc89b4bbe5ee133eed41eccb6f185cf4470 Mon Sep 17 00:00:00 2001 From: Arkadiy Kukarkin Date: Fri, 20 Mar 2026 00:16:26 +0100 Subject: [PATCH 5/6] more robust existing key material check --- handler/wallet/export_keys.go | 14 +++++++++--- handler/wallet/export_keys_test.go | 35 +++++++++++++++++++++++++++++- handler/wallet/sign.go | 16 -------------- model/replication.go | 6 ++--- 4 files changed, 48 insertions(+), 23 deletions(-) diff --git a/handler/wallet/export_keys.go b/handler/wallet/export_keys.go index cf60fb29..7d4991b1 100644 --- a/handler/wallet/export_keys.go +++ b/handler/wallet/export_keys.go @@ -70,9 +70,17 @@ func exportOneKey(db *gorm.DB, ks keystore.KeyStore, actor model.Actor) (exporte existing.ActorID = &actor.ID db.Save(&existing) } - // verify the key file actually exists on disk - if !ks.Has(existing.KeyPath) { - return false, fmt.Sprintf("actor %s: wallet record exists but key file missing at %s", actor.ID, existing.KeyPath) + // verify the key file exists and contains a valid key for this address + stored, err := ks.Get(existing.KeyPath) + if err != nil { + return false, fmt.Sprintf("actor %s: wallet record exists but key file unreadable at %s: %v", actor.ID, existing.KeyPath, err) + } + storedAddr, err := keystore.AddressFromExport(stored) + if err != nil { + return false, fmt.Sprintf("actor %s: key file at %s is corrupt: %v", actor.ID, existing.KeyPath, err) + } + if storedAddr != addr { + return false, fmt.Sprintf("actor %s: key file at %s contains wrong address %s (expected %s)", actor.ID, existing.KeyPath, storedAddr, addr) } logger.Debugw("wallet already exists for actor", "actorID", actor.ID, "address", addr.String()) return false, "" diff --git a/handler/wallet/export_keys_test.go b/handler/wallet/export_keys_test.go index 73652562..b90347ba 100644 --- a/handler/wallet/export_keys_test.go +++ b/handler/wallet/export_keys_test.go @@ -2,6 +2,7 @@ package wallet import ( "context" + "os" "testing" "github.com/data-preservation-programs/singularity/model" @@ -208,7 +209,39 @@ func TestExportKeysHandler_MissingKeyFile(t *testing.T) { require.Equal(t, 0, result.Exported) require.Equal(t, 0, result.Skipped) require.Len(t, result.Errors, 1) - require.Contains(t, result.Errors[0], "key file missing") + require.Contains(t, result.Errors[0], "key file unreadable") + }) +} + +func TestExportKeysHandler_CorruptKeyFile(t *testing.T) { + testutil.All(t, func(ctx context.Context, t *testing.T, db *gorm.DB) { + dir := t.TempDir() + ks, err := keystore.NewLocalKeyStore(dir) + require.NoError(t, err) + + actor := model.Actor{ + ID: "f01234", + Address: testutil.TestWalletAddr, + PrivateKey: testutil.TestPrivateKeyHex, + } + require.NoError(t, db.Create(&actor).Error) + + // write garbage to the key file path + corruptPath := dir + "/corrupt" + require.NoError(t, os.WriteFile(corruptPath, []byte("garbage"), 0600)) + + require.NoError(t, db.Create(&model.Wallet{ + KeyPath: corruptPath, + KeyStore: "local", + Address: testutil.TestWalletAddr, + }).Error) + + result, err := ExportKeysHandler(ctx, db, ks) + require.NoError(t, err) + require.Equal(t, 0, result.Exported) + require.Equal(t, 0, result.Skipped) + require.Len(t, result.Errors, 1) + require.Contains(t, result.Errors[0], "corrupt") }) } diff --git a/handler/wallet/sign.go b/handler/wallet/sign.go index 1d3a7bdf..616af240 100644 --- a/handler/wallet/sign.go +++ b/handler/wallet/sign.go @@ -112,19 +112,3 @@ func GetOrCreateActor( return &newActor, nil } - -// loads wallet by actor ID for signing operations -func LoadWalletByActorID(ctx context.Context, db *gorm.DB, actorID string) (*model.Wallet, error) { - db = db.WithContext(ctx) - - var wallet model.Wallet - err := db.Where("actor_id = ?", actorID).First(&wallet).Error - if err != nil { - if errors.Is(err, gorm.ErrRecordNotFound) { - return nil, errors.Errorf("no wallet found for actor %s - actor may not be controlled by this instance", actorID) - } - return nil, errors.Wrap(err, "failed to query wallet by actor ID") - } - - return &wallet, nil -} diff --git a/model/replication.go b/model/replication.go index 11181679..c6685fb9 100644 --- a/model/replication.go +++ b/model/replication.go @@ -175,9 +175,9 @@ type Schedule struct { Preparation *Preparation `gorm:"foreignKey:PreparationID;constraint:OnDelete:CASCADE" json:"preparation,omitempty" swaggerignore:"true" table:"expand"` } -// on-chain actor identity tracked by singularity -// actor may or may not be controlled by us (linked via optional WalletID) -// TODO: after migration, add WalletID field linking to new Wallet model +// on-chain actor identity tracked by singularity. +// actors we control have a Wallet with ActorID pointing here. +// actors we don't control (counterparties) exist as bare records. type Actor struct { ID string `gorm:"primaryKey;size:15" json:"id"` // actor ID (f0...) Address string `gorm:"index" json:"address"` // filecoin address From 55c29c821eeb2eb4e420a336125a41589df5a48d Mon Sep 17 00:00:00 2001 From: Arkadiy Kukarkin Date: Fri, 20 Mar 2026 09:53:04 +0100 Subject: [PATCH 6/6] exclude Actor.PrivateKey from GORM, read via raw SQL in export-keys --- handler/wallet/export_keys.go | 26 ++++++- handler/wallet/export_keys_test.go | 120 ++++++++++++++--------------- model/migrate.go | 33 ++++++++ model/replication.go | 9 ++- 4 files changed, 117 insertions(+), 71 deletions(-) diff --git a/handler/wallet/export_keys.go b/handler/wallet/export_keys.go index 7d4991b1..0c4a6026 100644 --- a/handler/wallet/export_keys.go +++ b/handler/wallet/export_keys.go @@ -17,6 +17,14 @@ type ExportKeysResult struct { Errors []string // actors that failed (logged but don't abort) } +// actor row with the legacy private_key column, used only by export-keys. +// Actor.PrivateKey is gorm:"-" in the model so we need a local type for raw SQL. +type legacyActorRow struct { + ID string + Address string + PrivateKey string +} + // exports private keys from the legacy Actor.PrivateKey column into the // filesystem keystore, creating Wallet records where needed. // idempotent -- skips actors whose address already has a Wallet record. @@ -28,8 +36,8 @@ func ExportKeysHandler( ) (*ExportKeysResult, error) { db = db.WithContext(ctx) - var actors []model.Actor - err := db.Where("private_key != '' AND private_key IS NOT NULL").Find(&actors).Error + var actors []legacyActorRow + err := db.Raw("SELECT id, address, private_key FROM actors WHERE private_key != '' AND private_key IS NOT NULL").Scan(&actors).Error if err != nil { return nil, errors.Wrap(err, "failed to query actors with private keys") } @@ -54,7 +62,7 @@ func ExportKeysHandler( // exports a single actor's key to keystore, returns (true, "") on success, // (false, "") on skip, ("", errMsg) on failure -func exportOneKey(db *gorm.DB, ks keystore.KeyStore, actor model.Actor) (exported bool, errMsg string) { +func exportOneKey(db *gorm.DB, ks keystore.KeyStore, actor legacyActorRow) (exported bool, errMsg string) { // derive address to check for existing wallet addr, err := keystore.AddressFromExport(actor.PrivateKey) if err != nil { @@ -114,7 +122,17 @@ func exportOneKey(db *gorm.DB, ks keystore.KeyStore, actor model.Actor) (exporte } func HasPrivateKeyColumn(db *gorm.DB) bool { - return db.Migrator().HasColumn(&model.Actor{}, "private_key") + // can't use db.Migrator().HasColumn(&model.Actor{}, "private_key") because + // the field is gorm:"-" -- GORM won't resolve the column name. query directly. + dialect := db.Dialector.Name() + var count int64 + switch dialect { + case "sqlite": + db.Raw("SELECT COUNT(*) FROM pragma_table_info('actors') WHERE name = 'private_key'").Scan(&count) + default: + db.Raw("SELECT COUNT(*) FROM information_schema.columns WHERE table_name = 'actors' AND column_name = 'private_key'").Scan(&count) + } + return count > 0 } // counts actors that still have a non-empty private_key in the database. diff --git a/handler/wallet/export_keys_test.go b/handler/wallet/export_keys_test.go index b90347ba..137a16a3 100644 --- a/handler/wallet/export_keys_test.go +++ b/handler/wallet/export_keys_test.go @@ -12,18 +12,38 @@ import ( "gorm.io/gorm" ) +// model without -:migration, used to simulate a legacy database where +// AutoMigrate created the private_key column +type legacyActor struct { + ID string `gorm:"primaryKey;size:15"` + Address string `gorm:"index"` + PrivateKey string +} + +func (legacyActor) TableName() string { return "actors" } + +// simulates a legacy database by running AutoMigrate with the old model +// that includes private_key as a normal column +func addLegacyColumn(t *testing.T, db *gorm.DB) { + t.Helper() + require.NoError(t, db.AutoMigrate(&legacyActor{})) +} + +// inserts an actor with a legacy private_key via the legacy model +func createLegacyActor(t *testing.T, db *gorm.DB, id, address, privateKey string) { + t.Helper() + require.NoError(t, db.Create(&legacyActor{ + ID: id, Address: address, PrivateKey: privateKey, + }).Error) +} + func TestExportKeysHandler_ExportsActorKey(t *testing.T) { testutil.All(t, func(ctx context.Context, t *testing.T, db *gorm.DB) { ks, err := keystore.NewLocalKeyStore(t.TempDir()) require.NoError(t, err) - // create an actor with a legacy private key - actor := model.Actor{ - ID: "f01234", - Address: testutil.TestWalletAddr, - PrivateKey: testutil.TestPrivateKeyHex, - } - require.NoError(t, db.Create(&actor).Error) + addLegacyColumn(t, db) + createLegacyActor(t, db, "f01234", testutil.TestWalletAddr, testutil.TestPrivateKeyHex) result, err := ExportKeysHandler(ctx, db, ks) require.NoError(t, err) @@ -50,13 +70,8 @@ func TestExportKeysHandler_SkipsExistingWallet(t *testing.T) { ks, err := keystore.NewLocalKeyStore(t.TempDir()) require.NoError(t, err) - // create actor with legacy key - actor := model.Actor{ - ID: "f01234", - Address: testutil.TestWalletAddr, - PrivateKey: testutil.TestPrivateKeyHex, - } - require.NoError(t, db.Create(&actor).Error) + addLegacyColumn(t, db) + createLegacyActor(t, db, "f01234", testutil.TestWalletAddr, testutil.TestPrivateKeyHex) // pre-import the wallet via normal import path h := DefaultHandler{} @@ -83,13 +98,8 @@ func TestExportKeysHandler_SkipsExistingWalletLinksActor(t *testing.T) { ks, err := keystore.NewLocalKeyStore(t.TempDir()) require.NoError(t, err) - // create actor with legacy key - actor := model.Actor{ - ID: "f01234", - Address: testutil.TestWalletAddr, - PrivateKey: testutil.TestPrivateKeyHex, - } - require.NoError(t, db.Create(&actor).Error) + addLegacyColumn(t, db) + createLegacyActor(t, db, "f01234", testutil.TestWalletAddr, testutil.TestPrivateKeyHex) // pre-import the wallet WITHOUT actor linkage h := DefaultHandler{} @@ -116,12 +126,8 @@ func TestExportKeysHandler_Idempotent(t *testing.T) { ks, err := keystore.NewLocalKeyStore(t.TempDir()) require.NoError(t, err) - actor := model.Actor{ - ID: "f01234", - Address: testutil.TestWalletAddr, - PrivateKey: testutil.TestPrivateKeyHex, - } - require.NoError(t, db.Create(&actor).Error) + addLegacyColumn(t, db) + createLegacyActor(t, db, "f01234", testutil.TestWalletAddr, testutil.TestPrivateKeyHex) // first run exports r1, err := ExportKeysHandler(ctx, db, ks) @@ -146,12 +152,12 @@ func TestExportKeysHandler_NoActorsWithKeys(t *testing.T) { ks, err := keystore.NewLocalKeyStore(t.TempDir()) require.NoError(t, err) + addLegacyColumn(t, db) // actor with no private key - actor := model.Actor{ + require.NoError(t, db.Create(&model.Actor{ ID: "f09999", Address: "f1abc", - } - require.NoError(t, db.Create(&actor).Error) + }).Error) result, err := ExportKeysHandler(ctx, db, ks) require.NoError(t, err) @@ -166,13 +172,8 @@ func TestExportKeysHandler_InvalidKeyRecordsError(t *testing.T) { ks, err := keystore.NewLocalKeyStore(t.TempDir()) require.NoError(t, err) - // actor with garbage key - actor := model.Actor{ - ID: "f05555", - Address: "f1bad", - PrivateKey: "not-a-valid-key", - } - require.NoError(t, db.Create(&actor).Error) + addLegacyColumn(t, db) + createLegacyActor(t, db, "f05555", "f1bad", "not-a-valid-key") result, err := ExportKeysHandler(ctx, db, ks) require.NoError(t, err) // overall operation succeeds @@ -189,13 +190,8 @@ func TestExportKeysHandler_MissingKeyFile(t *testing.T) { ks, err := keystore.NewLocalKeyStore(t.TempDir()) require.NoError(t, err) - // create actor with legacy key - actor := model.Actor{ - ID: "f01234", - Address: testutil.TestWalletAddr, - PrivateKey: testutil.TestPrivateKeyHex, - } - require.NoError(t, db.Create(&actor).Error) + addLegacyColumn(t, db) + createLegacyActor(t, db, "f01234", testutil.TestWalletAddr, testutil.TestPrivateKeyHex) // create wallet record pointing to a nonexistent key file require.NoError(t, db.Create(&model.Wallet{ @@ -219,12 +215,8 @@ func TestExportKeysHandler_CorruptKeyFile(t *testing.T) { ks, err := keystore.NewLocalKeyStore(dir) require.NoError(t, err) - actor := model.Actor{ - ID: "f01234", - Address: testutil.TestWalletAddr, - PrivateKey: testutil.TestPrivateKeyHex, - } - require.NoError(t, db.Create(&actor).Error) + addLegacyColumn(t, db) + createLegacyActor(t, db, "f01234", testutil.TestWalletAddr, testutil.TestPrivateKeyHex) // write garbage to the key file path corruptPath := dir + "/corrupt" @@ -245,17 +237,22 @@ func TestExportKeysHandler_CorruptKeyFile(t *testing.T) { }) } -func TestDropPrivateKeyColumn(t *testing.T) { +func TestExportKeysHandler_AutoMigrateDoesNotRecreateColumn(t *testing.T) { testutil.All(t, func(ctx context.Context, t *testing.T, db *gorm.DB) { - // column exists after AutoMigrate - require.True(t, db.Migrator().HasColumn(&model.Actor{}, "private_key")) + // AutoMigrate already ran (via testutil.All) -- column should NOT exist + require.False(t, db.Migrator().HasColumn(&model.Actor{}, "private_key"), + "AutoMigrate must not create private_key column (gorm:-:migration)") - // drop it + // simulate legacy db, export, drop + addLegacyColumn(t, db) + require.True(t, HasPrivateKeyColumn(db)) require.NoError(t, DropPrivateKeyColumn(db)) - require.False(t, db.Migrator().HasColumn(&model.Actor{}, "private_key")) + require.False(t, HasPrivateKeyColumn(db)) - // idempotent -- second call is a no-op - require.NoError(t, DropPrivateKeyColumn(db)) + // re-run AutoMigrate -- must NOT re-add the column + require.NoError(t, model.AutoMigrate(db)) + require.False(t, HasPrivateKeyColumn(db), + "AutoMigrate must not re-add private_key column after drop") }) } @@ -264,13 +261,8 @@ func TestDropPrivateKeyColumn_ExportThenDrop(t *testing.T) { ks, err := keystore.NewLocalKeyStore(t.TempDir()) require.NoError(t, err) - // create actor with legacy key - actor := model.Actor{ - ID: "f01234", - Address: testutil.TestWalletAddr, - PrivateKey: testutil.TestPrivateKeyHex, - } - require.NoError(t, db.Create(&actor).Error) + addLegacyColumn(t, db) + createLegacyActor(t, db, "f01234", testutil.TestWalletAddr, testutil.TestPrivateKeyHex) // export result, err := ExportKeysHandler(ctx, db, ks) diff --git a/model/migrate.go b/model/migrate.go index a48f1d2d..92d6c8a0 100644 --- a/model/migrate.go +++ b/model/migrate.go @@ -70,6 +70,12 @@ var fkMigrations = []fkMigration{ // Returns: // - An error if any issues arise during the process, otherwise nil. func AutoMigrate(db *gorm.DB) error { + // pre-migration: rename legacy wallets table to actors before GORM + // tries to reconcile the new Wallet model with the old table schema + if err := renameLegacyWalletsTable(db); err != nil { + return errors.Wrap(err, "failed to rename legacy wallets table") + } + logger.Info("Auto migrating tables") err := db.AutoMigrate(Tables...) if err != nil { @@ -487,6 +493,33 @@ func migrateWalletAssignments(db *gorm.DB) error { return nil } +// renameLegacyWalletsTable detects the pre-v0.8 schema where "wallets" was the +// actor identity table (id=f0..., private_key) and renames it to "actors" so +// AutoMigrate can create the new "wallets" table (keystore-backed model). +// idempotent -- skips if wallets already has key_path (new schema) or if +// actors already exists. +func renameLegacyWalletsTable(db *gorm.DB) error { + if !db.Migrator().HasTable("wallets") { + return nil // fresh install + } + if db.Migrator().HasColumn(&Wallet{}, "key_path") { + return nil // already migrated + } + // old wallets table has private_key but no key_path -- it's the actor table + if db.Migrator().HasTable("actors") { + return nil // actors table already exists, can't rename + } + + logger.Info("renaming legacy wallets table to actors") + if err := db.Exec("ALTER TABLE wallets RENAME TO actors").Error; err != nil { + return err + } + // drop old indexes that followed the rename -- they'd conflict with + // indexes AutoMigrate creates on the new wallets table + db.Exec("DROP INDEX IF EXISTS idx_wallets_address") + return nil +} + // DropAll removes all tables specified in the Tables slice from the database. // // This function is typically used during development or testing where a clean database diff --git a/model/replication.go b/model/replication.go index c6685fb9..b8ce4f1e 100644 --- a/model/replication.go +++ b/model/replication.go @@ -179,9 +179,12 @@ type Schedule struct { // actors we control have a Wallet with ActorID pointing here. // actors we don't control (counterparties) exist as bare records. type Actor struct { - ID string `gorm:"primaryKey;size:15" json:"id"` // actor ID (f0...) - Address string `gorm:"index" json:"address"` // filecoin address - PrivateKey string `json:"privateKey,omitempty" table:"-"` // orphaned: run `singularity wallet export-keys` to migrate, then drop column + ID string `gorm:"primaryKey;size:15" json:"id"` // actor ID (f0...) + Address string `gorm:"index" json:"address"` // filecoin address + // PrivateKey is excluded from GORM entirely so AutoMigrate won't + // create the column and SELECTs won't reference it. The export-keys + // handler reads it via raw SQL for legacy databases that still have it. + } // GORM will rename "wallets" table to "actors" on AutoMigrate