Add local wallet management to Motion

Add functionality to create and configure a local wallet with local disk
key storage. The changes introduce two new flags:
 * `--localWalletDir` - defaulting to `~/.motion/wallet` indicates where
    the wallet keys are stored.
 * `--localWalletGenerateIfNotExist` - defaulting to `true`
   automatically instantiates a FileCoin wallet if none exists.

The wallet functionality would also automatically set the default wallet
address to the first address found in the key store if no default is
specified.

Upgrade to the latest ribs dependency which introduces opotions to set
the previously hardcoded local wallet, and connect it to the new Motion
wallet.

Other blob store implementations that wish to store keys in database
should implement an instance of `types.KeyStore` and set it to Motion
wallet via `WithKeyStoreOpener` option.

To avoid indirect dependency to `filecoin-ffi`, the code forks the local
wallet implementation from Lotus, clearly marked as HACK. TODOs are left
to revisit this in future PRs in order to either embrace ffi change
build to accommodate C-bindings, or fork code in a separate repo/go
module and depend on that instead. For now, to reduce changes and
iteratively deliver functional code the code is simply forked.

README.md

@@ -16,8 +16,11 @@ COMMANDS:
    help, h  Shows a list of commands or help for one command
 
 GLOBAL OPTIONS:
-   --storeDir value  The path at which to store Motion data (default: OS Temporary directory) [$MOTION_STORE_DIR]
-   --help, -h        show help
+   --storeDir value                 The path at which to store Motion data (default: OS Temporary directory) [$MOTION_STORE_DIR]
+   --experimentalRibsStore          Whether to use experimental RIBS as the storage and deal making (default: Local storage is used)
+   --localWalletDir value           The path to the local wallet directory. (default: Defaults to '<user-home-directory>/.motion/wallet' with wallet key auto-generated if not present. Note that the directory permissions must be at most 0600.) [$MOTION_LOCAL_WALLET_DIR]
+   --localWalletGenerateIfNotExist  Whether to generate the local wallet key if none is found (default: true)
+   --help, -h
 ```
 
 ## Run Server Locally

blob/ribs_store.go

@@ -11,12 +11,14 @@ import (
 	"path"
 	"time"
 
+	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/google/uuid"
 	"github.com/ipfs/boxo/chunker"
 	blocks "github.com/ipfs/go-block-format"
 	"github.com/ipfs/go-cid"
 	"github.com/lotus-web3/ribs"
 	"github.com/lotus-web3/ribs/rbdeal"
+	"github.com/lotus-web3/ribs/ributil"
 	"github.com/multiformats/go-multihash"
 )
 
@@ -52,20 +54,22 @@ type (
 )
 
 // NewRibsStore instantiates a new experimental RIBS store.
-func NewRibsStore(dir string) (*RibsStore, error) {
-	rbdealDir := path.Join(path.Clean(dir), "rbdeal")
+func NewRibsStore(dir string, ks types.KeyStore) (*RibsStore, error) {
+	dir = path.Clean(dir)
+	rbdealDir := path.Join(dir, "rbdeal")
 	if err := os.Mkdir(rbdealDir, 0750); err != nil && !errors.Is(err, os.ErrExist) {
-		return nil, fmt.Errorf("failed to create internal directories: %w", err)
+		return nil, fmt.Errorf("failed to create RIBS deal directory: %w", err)
 	}
-	indexDir := path.Join(path.Clean(dir), "index")
+	indexDir := path.Join(dir, "index")
 	if err := os.Mkdir(indexDir, 0750); err != nil && !errors.Is(err, os.ErrExist) {
-		return nil, fmt.Errorf("failed to create internal directories: %w", err)
+		return nil, fmt.Errorf("failed to create RIBS internal directory: %w", err)
 	}
 
-	// TODO Path to wallet is hardcoded in RIBS. Parameterise it and allow user to configure
-	//      See: https://github.com/FILCAT/ribs/blob/7c8766206ec1e5ec30c613cde2b3a49d0ccf25d0/rbdeal/ribs.go#L156
+	rbs, err := rbdeal.Open(rbdealDir,
+		rbdeal.WithLocalWalletOpener(func(string) (*ributil.LocalWallet, error) {
+			return ributil.NewWallet(ks)
+		}))
 
-	rbs, err := rbdeal.Open(rbdealDir)
 	if err != nil {
 		return nil, err
 	}

cmd/motion/main.go

@@ -4,8 +4,10 @@ import (
 	"os"
 	"os/signal"
 
+	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/motion"
 	"github.com/filecoin-project/motion/blob"
+	"github.com/filecoin-project/motion/wallet"
 	"github.com/ipfs/go-log/v2"
 	"github.com/urfave/cli/v2"
 )
@@ -32,12 +34,39 @@ func main() {
 				Usage:       "Whether to use experimental RIBS as the storage and deal making",
 				DefaultText: "Local storage is used",
 			},
+			&cli.StringFlag{
+				Name:        "localWalletDir",
+				Usage:       "The path to the local wallet directory.",
+				DefaultText: "Defaults to '<user-home-directory>/.motion/wallet' with wallet key auto-generated if not present. Note that the directory permissions must be at most 0600.",
+				EnvVars:     []string{"MOTION_LOCAL_WALLET_DIR"},
+			},
+			&cli.BoolFlag{
+				Name:  "localWalletGenerateIfNotExist",
+				Usage: "Whether to generate the local wallet key if none is found",
+				Value: true,
+			},
 		},
 		Action: func(cctx *cli.Context) error {
+
+			localWalletDir := cctx.String("localWalletDir")
+			localWalletGenIfNotExist := cctx.Bool("localWalletGenerateIfNotExist")
+			ks, err := wallet.DefaultDiskKeyStoreOpener(localWalletDir, localWalletGenIfNotExist)()
+			if err != nil {
+				logger.Errorw("Failed to instantiate local wallet keystore", "err", err)
+				return err
+			}
+			wallet, err := wallet.New(
+				wallet.WithKeyStoreOpener(func() (types.KeyStore, error) { return ks, nil }),
+				wallet.WithGenerateKeyIfNotExist(localWalletGenIfNotExist))
+			if err != nil {
+				logger.Errorw("Failed to instantiate local wallet", "err", err)
+				return err
+			}
+
 			storeDir := cctx.String("storeDir")
 			var store blob.Store
 			if cctx.Bool("experimentalRibsStore") {
-				rbstore, err := blob.NewRibsStore(storeDir)
+				rbstore, err := blob.NewRibsStore(storeDir, ks)
 				if err != nil {
 					return err
 				}
@@ -52,7 +81,7 @@ func main() {
 				logger.Infow("Using local blob store", "storeDir", storeDir)
 			}
 
-			m, err := motion.New(motion.WithBlobStore(store))
+			m, err := motion.New(motion.WithBlobStore(store), motion.WithWallet(wallet))
 			if err != nil {
 				logger.Fatalw("Failed to instantiate Motion", "err", err)
 			}

go.mod

@@ -3,14 +3,19 @@ module github.com/filecoin-project/motion
 go 1.20
 
 require (
+	github.com/filecoin-project/go-address v1.1.0
+	github.com/filecoin-project/go-state-types v0.11.1
+	github.com/filecoin-project/lotus v1.23.2-0.20230628201333-30a9f631653f
 	github.com/google/uuid v1.3.0
 	github.com/ipfs/boxo v0.10.2
 	github.com/ipfs/go-block-format v0.1.2
 	github.com/ipfs/go-cid v0.4.1
 	github.com/ipfs/go-log/v2 v2.5.1
-	github.com/lotus-web3/ribs v0.0.0-20230717152524-50fead08f629
+	github.com/labstack/gommon v0.3.0
+	github.com/lotus-web3/ribs v0.0.0-20230720185531-31964053b4be
 	github.com/multiformats/go-multihash v0.2.3
 	github.com/urfave/cli/v2 v2.25.7
+	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2
 )
 
 require (
@@ -41,7 +46,6 @@ require (
 	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/elastic/gosigar v0.14.2 // indirect
 	github.com/fatih/color v1.13.0 // indirect
-	github.com/filecoin-project/go-address v1.1.0 // indirect
 	github.com/filecoin-project/go-amt-ipld/v2 v2.1.0 // indirect
 	github.com/filecoin-project/go-amt-ipld/v3 v3.1.0 // indirect
 	github.com/filecoin-project/go-amt-ipld/v4 v4.1.0 // indirect
@@ -62,12 +66,10 @@ require (
 	github.com/filecoin-project/go-legs v0.4.9 // indirect
 	github.com/filecoin-project/go-padreader v0.0.1 // indirect
 	github.com/filecoin-project/go-retrieval-types v1.2.0 // indirect
-	github.com/filecoin-project/go-state-types v0.11.1 // indirect
 	github.com/filecoin-project/go-statemachine v1.0.3 // indirect
 	github.com/filecoin-project/go-statestore v0.2.0 // indirect
 	github.com/filecoin-project/kubo-api-client v0.0.1 // indirect
 	github.com/filecoin-project/lassie v0.13.0 // indirect
-	github.com/filecoin-project/lotus v1.23.2-0.20230628201333-30a9f631653f // indirect
 	github.com/filecoin-project/specs-actors v0.9.15 // indirect
 	github.com/filecoin-project/specs-actors/v2 v2.3.6 // indirect
 	github.com/filecoin-project/specs-actors/v3 v3.1.2 // indirect
@@ -198,9 +200,11 @@ require (
 	github.com/rogpeppe/go-internal v1.9.0 // indirect
 	github.com/rs/cors v1.7.0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
-	github.com/shirou/gopsutil v2.18.12+incompatible // indirect
+	github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible // indirect
 	github.com/spaolacci/murmur3 v1.1.0 // indirect
 	github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect
+	github.com/tklauser/go-sysconf v0.3.5 // indirect
+	github.com/tklauser/numcpus v0.2.2 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	github.com/valyala/fasttemplate v1.0.1 // indirect
 	github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc // indirect
@@ -226,7 +230,6 @@ require (
 	golang.org/x/sys v0.9.0 // indirect
 	golang.org/x/text v0.10.0 // indirect
 	golang.org/x/tools v0.9.1 // indirect
-	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
 	google.golang.org/protobuf v1.30.0 // indirect
 	lukechampine.com/blake3 v1.2.1 // indirect
 	nhooyr.io/websocket v1.8.7 // indirect

go.sum

@@ -656,6 +656,7 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/labstack/echo/v4 v4.1.11/go.mod h1:i541M3Fj6f76NZtHSj7TXnyM8n2gaodfvfxNnFqi74g=
+github.com/labstack/gommon v0.3.0 h1:JEeO0bvc78PKdyHxloTKiF8BD5iGrH8T6MSeGvSgob0=
 github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k=
 github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y=
 github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
@@ -738,8 +739,8 @@ github.com/libp2p/go-yamux v1.2.2/go.mod h1:FGTiPvoV/3DVdgWpX+tM0OW3tsM+W5bSE3gZ
 github.com/libp2p/go-yamux v1.2.3/go.mod h1:FGTiPvoV/3DVdgWpX+tM0OW3tsM+W5bSE3gZwqQTcow=
 github.com/libp2p/go-yamux/v4 v4.0.0 h1:+Y80dV2Yx/kv7Y7JKu0LECyVdMXm1VUoko+VQ9rBfZQ=
 github.com/libp2p/go-yamux/v4 v4.0.0/go.mod h1:NWjl8ZTLOGlozrXSOZ/HlfG++39iKNnM5wwmtQP1YB4=
-github.com/lotus-web3/ribs v0.0.0-20230717152524-50fead08f629 h1:g8sJJ12xM4SJgttT+rGA9HjmhUwnCBqYiuQlIOyZxVc=
-github.com/lotus-web3/ribs v0.0.0-20230717152524-50fead08f629/go.mod h1:IzE+ozbGNAj5rRJIxsW1DBkhzDGKOuBSIaGPFyilptM=
+github.com/lotus-web3/ribs v0.0.0-20230720185531-31964053b4be h1:uSTfw63JG29LwJzv4LkiXYh1kNNR6hNicmMgTUbA7uo=
+github.com/lotus-web3/ribs v0.0.0-20230720185531-31964053b4be/go.mod h1:fmHaCL/KCC5STh4ou0/m1lpfKCqGBNrRjqnr94QHe7U=
 github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI=
 github.com/magefile/mage v1.9.0 h1:t3AU2wNwehMCW97vuqQLtw6puppWXHO+O2MHo5a50XE=
 github.com/magefile/mage v1.9.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A=
@@ -948,8 +949,9 @@ github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFo
 github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
 github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
-github.com/shirou/gopsutil v2.18.12+incompatible h1:1eaJvGomDnH74/5cF4CTmTbLHAriGFsTZppLXDX93OM=
 github.com/shirou/gopsutil v2.18.12+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
+github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible h1:Bn1aCHHRnjv4Bl16T8rcaFjYSrGrIZvpiGO6P3Q4GpU=
+github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
 github.com/shurcooL/component v0.0.0-20170202220835-f88ec8f54cc4/go.mod h1:XhFIlyj5a1fBNx5aJTbKoIq0mNaPvOagO+HjB3EtxrY=
 github.com/shurcooL/events v0.0.0-20181021180414-410e4ca65f48/go.mod h1:5u70Mqkb5O5cxEA8nxTsgrgLehJeAw6Oc4Ab1c/P1HM=
 github.com/shurcooL/github_flavored_markdown v0.0.0-20181002035957-2122de532470/go.mod h1:2dOwnU2uBioM+SGy2aZoq1f/Sd1l9OkAeAUvjSyvgU0=
@@ -1014,6 +1016,10 @@ github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70
 github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc=
 github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA=
 github.com/tj/go-spin v1.1.0/go.mod h1:Mg1mzmePZm4dva8Qz60H2lHwmJ2loum4VIrLgVnKwh4=
+github.com/tklauser/go-sysconf v0.3.5 h1:uu3Xl4nkLzQfXNsWn15rPc/HQCJKObbt1dKJeWp3vU4=
+github.com/tklauser/go-sysconf v0.3.5/go.mod h1:MkWzOF4RMCshBAMXuhXJs64Rte09mITnppBXY/rYEFI=
+github.com/tklauser/numcpus v0.2.2 h1:oyhllyrScuYI6g+h/zUvNXNp1wy7x8qQy3t/piefldA=
+github.com/tklauser/numcpus v0.2.2/go.mod h1:x3qojaO3uyYt0i56EW/VUYs7uBvdl2fkfZFu0T9wgjM=
 github.com/twmb/murmur3 v1.1.6 h1:mqrRot1BRxm+Yct+vavLMou2/iJt0tNVTTC0QoIjaZg=
 github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
 github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo=
@@ -1295,6 +1301,7 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210316164454-77fc1eacc6aa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

options.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/filecoin-project/motion/api/server"
 	"github.com/filecoin-project/motion/blob"
+	"github.com/filecoin-project/motion/wallet"
 )
 
 type (
@@ -13,6 +14,7 @@ type (
 	options struct {
 		serverOptions []server.Option
 		blobStore     blob.Store
+		wallet        *wallet.Wallet
 	}
 )
 
@@ -49,3 +51,12 @@ func WithBlobStore(s blob.Store) Option {
 		return nil
 	}
 }
+
+// WithWallet sets the wallet used by Motion to interact with FileCoin network.
+// Defaults to wallet.New.
+func WithWallet(w *wallet.Wallet) Option {
+	return func(o *options) error {
+		o.wallet = w
+		return nil
+	}
+}