complete key migration code

cmd/admin/init.go

@@ -4,6 +4,7 @@ import (
 	"github.com/cockroachdb/errors"
 	"github.com/data-preservation-programs/singularity/database"
 	"github.com/data-preservation-programs/singularity/handler/admin"
+	"github.com/data-preservation-programs/singularity/handler/wallet"
 	"github.com/urfave/cli/v2"
 )
 
@@ -27,6 +28,12 @@ var InitCmd = &cli.Command{
 		if err != nil {
 			return errors.WithStack(err)
 		}
+		if n := wallet.CountLegacyKeys(db); n > 0 {
+			return errors.Errorf(
+				"%d actor(s) have private keys in the database that are not usable by current code.\n"+
+					"Run 'singularity wallet export-keys' to migrate them to the filesystem keystore",
+				n)
+		}
 		if c.IsSet("identity") {
 			err = admin.Default.SetIdentityHandler(c.Context, db, admin.SetIdentityRequest{
 				Identity: c.String("identity"),

cmd/app.go

@@ -168,6 +168,7 @@ Upgrading:
 				wallet.ImportCmd,
 				wallet.ListCmd,
 				wallet.RemoveCmd,
+				wallet.ExportKeysCmd,
 			},
 		},
 		{

cmd/run/api.go

@@ -9,11 +9,23 @@ var APICmd = &cli.Command{
 	Name:  "api",
 	Usage: "Run the singularity API",
 	Flags: []cli.Flag{
+		NoAutoMigrateFlag,
 		&cli.StringFlag{
 			Name:  "bind",
 			Usage: "Bind address for the API server",
 			Value: ":9090",
 		},
 	},
-	Action: api.Run,
+	Action: func(c *cli.Context) error {
+		// run automigrate + legacy key check before handing off to api.Run,
+		// which opens its own db connection internally
+		db, closer, err := openAndMigrate(c)
+		if err != nil {
+			return err
+		}
+		closer.Close()
+		_ = db
+
+		return api.Run(c)
+	},
 }

cmd/run/automigrate.go

@@ -0,0 +1,43 @@
+package run
+
+import (
+	"io"
+
+	"github.com/cockroachdb/errors"
+	"github.com/data-preservation-programs/singularity/database"
+	"github.com/data-preservation-programs/singularity/handler/wallet"
+	"github.com/data-preservation-programs/singularity/model"
+	"github.com/urfave/cli/v2"
+	"gorm.io/gorm"
+)
+
+var NoAutoMigrateFlag = &cli.BoolFlag{
+	Name:  "no-automigrate",
+	Usage: "skip automatic database migration and correctness checks on startup; only use if you run 'admin init' on every upgrade or manually before starting daemons",
+}
+
+// opens the database, runs AutoMigrate (unless --no-automigrate), and checks
+// for legacy keys that need export. returns db and closer as usual.
+func openAndMigrate(c *cli.Context) (*gorm.DB, io.Closer, error) {
+	db, closer, err := database.OpenFromCLI(c)
+	if err != nil {
+		return nil, nil, errors.WithStack(err)
+	}
+
+	if !c.Bool("no-automigrate") {
+		if err := model.AutoMigrate(db); err != nil {
+			closer.Close()
+			return nil, nil, errors.Wrap(err, "automigrate failed")
+		}
+
+		if n := wallet.CountLegacyKeys(db); n > 0 {
+			closer.Close()
+			return nil, nil, errors.Errorf(
+				"%d actor(s) have private keys in the database that are not usable by current code.\n"+
+					"Run 'singularity wallet export-keys' to migrate them to the filesystem keystore",
+				n)
+		}
+	}
+
+	return db, closer, nil
+}

cmd/run/contentprovider.go

@@ -2,7 +2,6 @@ package run
 
 import (
 	"github.com/cockroachdb/errors"
-	"github.com/data-preservation-programs/singularity/database"
 	"github.com/data-preservation-programs/singularity/service/contentprovider"
 	"github.com/urfave/cli/v2"
 )
@@ -11,6 +10,7 @@ var ContentProviderCmd = &cli.Command{
 	Name:  "content-provider",
 	Usage: "Start a content provider that serves retrieval requests",
 	Flags: []cli.Flag{
+		NoAutoMigrateFlag,
 		&cli.StringFlag{
 			Category: "HTTP Retrieval",
 			Name:     "http-bind",
@@ -50,7 +50,7 @@ var ContentProviderCmd = &cli.Command{
 		},
 	},
 	Action: func(c *cli.Context) error {
-		db, closer, err := database.OpenFromCLI(c)
+		db, closer, err := openAndMigrate(c)
 		if err != nil {
 			return errors.WithStack(err)
 		}

cmd/run/datasetworker.go

@@ -4,7 +4,6 @@ import (
 	"time"
 
 	"github.com/cockroachdb/errors"
-	"github.com/data-preservation-programs/singularity/database"
 	"github.com/data-preservation-programs/singularity/service/datasetworker"
 	"github.com/urfave/cli/v2"
 )
@@ -13,6 +12,7 @@ var DatasetWorkerCmd = &cli.Command{
 	Name:  "dataset-worker",
 	Usage: "Start a dataset preparation worker to process dataset scanning and preparation tasks",
 	Flags: []cli.Flag{
+		NoAutoMigrateFlag,
 		&cli.IntFlag{
 			Name:  "concurrency",
 			Usage: "Number of concurrent workers to run",
@@ -55,7 +55,7 @@ var DatasetWorkerCmd = &cli.Command{
 		},
 	},
 	Action: func(c *cli.Context) error {
-		db, closer, err := database.OpenFromCLI(c)
+		db, closer, err := openAndMigrate(c)
 		if err != nil {
 			return errors.WithStack(err)
 		}

cmd/run/dealpusher.go

@@ -4,7 +4,6 @@ import (
 	"time"
 
 	"github.com/cockroachdb/errors"
-	"github.com/data-preservation-programs/singularity/database"
 	"github.com/data-preservation-programs/singularity/service"
 	"github.com/data-preservation-programs/singularity/service/dealpusher"
 	"github.com/data-preservation-programs/singularity/service/epochutil"
@@ -15,6 +14,7 @@ var DealPusherCmd = &cli.Command{
 	Name:  "deal-pusher",
 	Usage: "Start a deal pusher that monitors deal schedules and pushes deals to storage providers",
 	Flags: []cli.Flag{
+		NoAutoMigrateFlag,
 		&cli.UintFlag{
 			Name:    "deal-attempts",
 			Usage:   "Number of times to attempt a deal before giving up",
@@ -54,7 +54,7 @@ var DealPusherCmd = &cli.Command{
 		},
 	},
 	Action: func(c *cli.Context) error {
-		db, closer, err := database.OpenFromCLI(c)
+		db, closer, err := openAndMigrate(c)
 		if err != nil {
 			return errors.WithStack(err)
 		}

cmd/run/dealtracker.go

@@ -4,7 +4,6 @@ import (
 	"time"
 
 	"github.com/cockroachdb/errors"
-	"github.com/data-preservation-programs/singularity/database"
 	"github.com/data-preservation-programs/singularity/service"
 	"github.com/data-preservation-programs/singularity/service/dealtracker"
 	"github.com/data-preservation-programs/singularity/service/epochutil"
@@ -15,6 +14,7 @@ var DealTrackerCmd = &cli.Command{
 	Name:  "deal-tracker",
 	Usage: "Start a deal tracker that tracks the deal for all relevant wallets",
 	Flags: []cli.Flag{
+		NoAutoMigrateFlag,
 		&cli.StringFlag{
 			Name:    "market-deal-url",
 			Usage:   "The URL for ZST compressed state market deals json. Set to empty to use Lotus API.",
@@ -35,7 +35,7 @@ var DealTrackerCmd = &cli.Command{
 		},
 	},
 	Action: func(c *cli.Context) error {
-		db, closer, err := database.OpenFromCLI(c)
+		db, closer, err := openAndMigrate(c)
 		if err != nil {
 			return errors.WithStack(err)
 		}

cmd/run/pdptracker.go

@@ -8,7 +8,6 @@ import (
 	"github.com/cockroachdb/errors"
 	"github.com/data-preservation-programs/go-synapse"
 	"github.com/data-preservation-programs/go-synapse/constants"
-	"github.com/data-preservation-programs/singularity/database"
 	"github.com/data-preservation-programs/singularity/service"
 	"github.com/data-preservation-programs/singularity/service/pdptracker"
 	"github.com/ethereum/go-ethereum/common"
@@ -20,6 +19,7 @@ var PDPTrackerCmd = &cli.Command{
 	Name:  "pdp-tracker",
 	Usage: "Track PDP deals via Shovel event indexing (requires PostgreSQL)",
 	Flags: []cli.Flag{
+		NoAutoMigrateFlag,
 		&cli.StringFlag{
 			Name:    "eth-rpc",
 			Usage:   "Ethereum RPC endpoint for FEVM",
@@ -43,7 +43,7 @@ var PDPTrackerCmd = &cli.Command{
 			return errors.New("PDP tracking requires PostgreSQL (Shovel is Postgres-only)")
 		}
 
-		db, closer, err := database.OpenFromCLI(c)
+		db, closer, err := openAndMigrate(c)
 		if err != nil {
 			return errors.WithStack(err)
 		}

cmd/wallet/export_keys.go

@@ -0,0 +1,125 @@
+package wallet
+
+import (
+	"bufio"
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/cockroachdb/errors"
+	"github.com/data-preservation-programs/singularity/database"
+	"github.com/data-preservation-programs/singularity/handler/wallet"
+	"github.com/data-preservation-programs/singularity/util/keystore"
+	"github.com/urfave/cli/v2"
+)
+
+var ExportKeysCmd = &cli.Command{
+	Name:  "export-keys",
+	Usage: "Migrate private keys from database (legacy Actor.PrivateKey) to the filesystem keystore",
+	Description: `Reads private keys stored in the legacy actors table and saves them to
+the filesystem keystore (~/.singularity/keystore or SINGULARITY_KEYSTORE).
+Creates Wallet records for each exported key and links them to the
+corresponding Actor.
+
+This command is idempotent — actors whose address already has a Wallet
+record are skipped. Keys that fail to parse are reported but do not
+abort the migration.
+
+After exporting, prompts to drop the orphaned private_key column from
+the actors table. This is irreversible — verify keys are in the keystore
+before confirming. For scripted use, pass --drop-db-keys --i-am-really-sure
+to skip the prompt.`,
+	Flags: []cli.Flag{
+		&cli.BoolFlag{
+			Name:  "drop-db-keys",
+			Usage: "drop the private_key column from the actors table after export",
+		},
+		&cli.BoolFlag{
+			Name:  "i-am-really-sure",
+			Usage: "confirm column drop (required with --drop-db-keys)",
+		},
+	},
+	Action: func(c *cli.Context) error {
+		db, closer, err := database.OpenFromCLI(c)
+		if err != nil {
+			return errors.WithStack(err)
+		}
+		defer closer.Close()
+
+		// check if column still exists -- if not, nothing to do
+		if !wallet.HasPrivateKeyColumn(db) {
+			fmt.Println("nothing to do -- private_key column already dropped")
+			return nil
+		}
+
+		keystoreDir := wallet.GetKeystoreDir()
+		ks, err := keystore.NewLocalKeyStore(keystoreDir)
+		if err != nil {
+			return errors.Wrap(err, "failed to init keystore")
+		}
+
+		result, err := wallet.ExportKeysHandler(c.Context, db, ks)
+		if err != nil {
+			return errors.WithStack(err)
+		}
+
+		fmt.Printf("exported: %d\n", result.Exported)
+		if result.Skipped > 0 {
+			fmt.Printf("skipped:  %d (wallet already exists)\n", result.Skipped)
+		}
+		if len(result.Errors) > 0 {
+			fmt.Printf("errors:   %d\n", len(result.Errors))
+			for _, e := range result.Errors {
+				fmt.Printf("  - %s\n", e)
+			}
+			fmt.Println("\nfix the errors above and re-run before dropping the column")
+			return nil
+		}
+
+		// list keys in keystore so user can verify
+		keys, err := ks.List()
+		if err != nil {
+			return errors.Wrap(err, "failed to list keystore")
+		}
+		fmt.Printf("\nkeystore: %s (%d keys)\n", keystoreDir, len(keys))
+
+		// determine whether to drop the column
+		dropFlag := c.Bool("drop-db-keys")
+		sureFlag := c.Bool("i-am-really-sure")
+
+		if dropFlag && !sureFlag {
+			return errors.New("--drop-db-keys requires --i-am-really-sure")
+		}
+
+		shouldDrop := dropFlag && sureFlag
+		if !shouldDrop {
+			// interactive prompt
+			fmt.Printf("\n" +
+				"WARNING: the next step will DROP the private_key column from the\n" +
+				"actors table. This is IRREVERSIBLE. All key material in the database\n" +
+				"will be permanently deleted.\n" +
+				"\n" +
+				"Verify that your keys are present in the keystore directory above\n" +
+				"before continuing. For scripted use, pass:\n" +
+				"  --drop-db-keys --i-am-really-sure\n\n")
+			fmt.Printf("Drop private_key column? [y/N] ")
+
+			scanner := bufio.NewScanner(os.Stdin)
+			if !scanner.Scan() {
+				return nil
+			}
+			answer := strings.TrimSpace(strings.ToLower(scanner.Text()))
+			if answer != "y" && answer != "yes" {
+				fmt.Println("aborted -- keys exported but column retained")
+				return nil
+			}
+		}
+
+		if err := wallet.DropPrivateKeyColumn(db); err != nil {
+			return errors.Wrap(err, "failed to drop private_key column")
+		}
+		fmt.Println("dropped private_key column from actors table")
+
+		return nil
+	},
+}

docs/en/SUMMARY.md

@@ -66,6 +66,7 @@
   * [Import](cli-reference/wallet/import.md)
   * [List](cli-reference/wallet/list.md)
   * [Remove](cli-reference/wallet/remove.md)
+  * [Export Keys](cli-reference/wallet/export-keys.md)
 * [Storage](cli-reference/storage/README.md)
   * [Create](cli-reference/storage/create/README.md)
     * [Azureblob](cli-reference/storage/create/azureblob.md)