Add local wallet management to Motion

Add functionality to create and configure a local wallet with local disk
key storage. The changes introduce two new flags:
 * `--localWalletDir` - defaulting to `~/.motion/wallet` indicates where
    the wallet keys are stored.
 * `--localWalletGenerateIfNotExist` - defaulting to `true`
   automatically instantiates a FileCoin wallet if none exists.

The wallet functionality would also automatically set the default wallet
address to the first address found in the key store if no default is
specified.

Upgrade to the latest ribs dependency which introduces opotions to set
the previously hardcoded local wallet, and connect it to the new Motion
wallet.

Other blob store implementations that wish to store keys in database
should implement an instance of `types.KeyStore` and set it to Motion
wallet via `WithKeyStoreOpener` option.

The build is adjusted to accommodate filecoin-ffi dependency with
replace directive and adjustments are made to the unified CI to work
around issues in filecoin-ffi which otherwise get picked up by the CI
runs in Motion. The builds on MacOS and Window

.dockerignore

@@ -0,0 +1,3 @@
+.github
+extern
+*.md

.github/actions/go-check-setup/action.yml

@@ -0,0 +1,20 @@
+name: Go Check Setup
+description: Set up the environment for go check
+runs:
+  using: "composite"
+  steps:
+    - name: Install filecoin-ffi dependencies
+      shell: bash
+      if: ${{ runner.os == 'Linux' }}
+      run: sudo apt-get install -y libhwloc-dev ocl-icd-opencl-dev
+    - name: Install filecoin-ffi
+      shell: bash
+      run: |
+        make extern/filecoin-ffi
+        rm -rf extern/filecoin-ffi/.git
+    # Fix what we can and silence filecoin-ffi staticcheck since there is no way of instructing the unified CI to skip a directory.
+    - name: Silence checks
+      shell: bash
+      run: |
+        echo 'checks = ["none"]' > extern/filecoin-ffi/staticcheck.conf
+        cd extern/filecoin-ffi && go mod tidy && gofmt -w .

.github/actions/go-test-setup/action.yml

@@ -0,0 +1,19 @@
+name: Go Test Setup
+description: Set up the environment for go test
+runs:
+  using: "composite"
+  steps:
+    - name: Install filecoin-ffi dependencies
+      shell: bash
+      if: ${{ runner.os == 'Linux' }}
+      run: sudo apt-get install -y libhwloc-dev ocl-icd-opencl-dev
+    - name: Install filecoin-ffi
+      shell: bash
+      if: ${{ runner.os == 'Linux' }}
+      run: make extern/filecoin-ffi
+    # Removes all go test files to stop the unified CI from running ffi tests as part of the build.
+    # See: https://github.com/protocol/multiple-go-modules/issues/11
+    - name: Remove filecoin-ffi tests
+      shell: bash
+      if: ${{ runner.os == 'Linux' }}
+      run: find extern/filecoin-ffi -name '*_test.go' -exec rm {} \;

.github/workflows/go-test-config.json

@@ -1,3 +1,4 @@
 {
-  "skip32bit": true
+  "skip32bit": true,
+  "skipOSes": ["windows", "macos"]
 }

.gitignore

@@ -0,0 +1 @@
+extern

Dockerfile

@@ -1,13 +1,23 @@
 FROM golang:1.20-bullseye as build
 
 WORKDIR /go/src/motion
-COPY go.* ./
+
+RUN apt-get update && apt-get install -y libhwloc-dev ocl-icd-opencl-dev jq
+COPY Makefile .
+RUN make extern/filecoin-ffi
+
+COPY go.* .
 RUN go mod download
 
 COPY . .
 RUN CGO_ENABLED=1 go build -o /go/bin/motion ./cmd/motion
 
 FROM gcr.io/distroless/base-debian11
 COPY --from=build /go/bin/motion /usr/bin/
+COPY --from=build /go/src/motion/extern/filecoin-ffi/filcrypto.h \
+                  /go/src/motion/extern/filecoin-ffi/filcrypto.pc \
+                  /go/src/motion/extern/filecoin-ffi/libfilcrypto.a \
+                  /usr/lib/*/libhwloc.so.15 \
+                  /usr/lib/
 
 ENTRYPOINT ["/usr/bin/motion"]

Makefile

@@ -0,0 +1,21 @@
+SHELL=/usr/bin/env bash
+.DEFAULT_GOAL := build
+
+FILECOIN_FFI_VERSION=de34caff946d598e
+FILECOIN_FFI_HOME=extern/filecoin-ffi
+
+.PHONY: build
+build: extern/filecoin-ffi
+	go build ./...
+
+.PHONY: clean
+clean:
+	rm -rf ./$(FILECOIN_FFI_HOME)
+
+extern/filecoin-ffi:
+	git clone --depth 1 https://github.com/filecoin-project/filecoin-ffi.git $(FILECOIN_FFI_HOME) && \
+	cd $(FILECOIN_FFI_HOME) && \
+	git fetch origin $(FILECOIN_FFI_VERSION) && \
+	git checkout $(FILECOIN_FFI_VERSION) && \
+	$(MAKE)
+

README.md

@@ -16,8 +16,11 @@ COMMANDS:
    help, h  Shows a list of commands or help for one command
 
 GLOBAL OPTIONS:
-   --storeDir value  The path at which to store Motion data (default: OS Temporary directory) [$MOTION_STORE_DIR]
-   --help, -h        show help
+   --storeDir value                 The path at which to store Motion data (default: OS Temporary directory) [$MOTION_STORE_DIR]
+   --experimentalRibsStore          Whether to use experimental RIBS as the storage and deal making (default: Local storage is used)
+   --localWalletDir value           The path to the local wallet directory. (default: Defaults to '<user-home-directory>/.motion/wallet' with wallet key auto-generated if not present. Note that the directory permissions must be at most 0600.) [$MOTION_LOCAL_WALLET_DIR]
+   --localWalletGenerateIfNotExist  Whether to generate the local wallet key if none is found (default: true)
+   --help, -h
 ```
 
 ## Run Server Locally
@@ -84,8 +87,19 @@ Not yet implemented.
 See the [Motion OpenAPI specification](openapi.yaml).
 
 ## Status
+
 :construction: This project is currently under active development.
 
+## Local Development
+
+To set up `filecoin-ffi` dependencies, run:
+
+```shell
+make build
+```
+
+This is only necessary to run once. After that you can use the regular `go build` command to build Motion from source.
+
 ## License
 
 [SPDX-License-Identifier: Apache-2.0 OR MIT](LICENSE.md)

blob/ribs_store.go

@@ -11,12 +11,14 @@ import (
 	"path"
 	"time"
 
+	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/google/uuid"
 	"github.com/ipfs/boxo/chunker"
 	blocks "github.com/ipfs/go-block-format"
 	"github.com/ipfs/go-cid"
 	"github.com/lotus-web3/ribs"
 	"github.com/lotus-web3/ribs/rbdeal"
+	"github.com/lotus-web3/ribs/ributil"
 	"github.com/multiformats/go-multihash"
 )
 
@@ -52,20 +54,22 @@ type (
 )
 
 // NewRibsStore instantiates a new experimental RIBS store.
-func NewRibsStore(dir string) (*RibsStore, error) {
-	rbdealDir := path.Join(path.Clean(dir), "rbdeal")
+func NewRibsStore(dir string, ks types.KeyStore) (*RibsStore, error) {
+	dir = path.Clean(dir)
+	rbdealDir := path.Join(dir, "rbdeal")
 	if err := os.Mkdir(rbdealDir, 0750); err != nil && !errors.Is(err, os.ErrExist) {
-		return nil, fmt.Errorf("failed to create internal directories: %w", err)
+		return nil, fmt.Errorf("failed to create RIBS deal directory: %w", err)
 	}
-	indexDir := path.Join(path.Clean(dir), "index")
+	indexDir := path.Join(dir, "index")
 	if err := os.Mkdir(indexDir, 0750); err != nil && !errors.Is(err, os.ErrExist) {
-		return nil, fmt.Errorf("failed to create internal directories: %w", err)
+		return nil, fmt.Errorf("failed to create RIBS internal directory: %w", err)
 	}
 
-	// TODO Path to wallet is hardcoded in RIBS. Parameterise it and allow user to configure
-	//      See: https://github.com/FILCAT/ribs/blob/7c8766206ec1e5ec30c613cde2b3a49d0ccf25d0/rbdeal/ribs.go#L156
+	rbs, err := rbdeal.Open(rbdealDir,
+		rbdeal.WithLocalWalletOpener(func(string) (*ributil.LocalWallet, error) {
+			return ributil.NewWallet(ks)
+		}))
 
-	rbs, err := rbdeal.Open(rbdealDir)
 	if err != nil {
 		return nil, err
 	}

cmd/motion/main.go

@@ -4,8 +4,10 @@ import (
 	"os"
 	"os/signal"
 
+	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/motion"
 	"github.com/filecoin-project/motion/blob"
+	"github.com/filecoin-project/motion/wallet"
 	"github.com/ipfs/go-log/v2"
 	"github.com/urfave/cli/v2"
 )
@@ -32,12 +34,39 @@ func main() {
 				Usage:       "Whether to use experimental RIBS as the storage and deal making",
 				DefaultText: "Local storage is used",
 			},
+			&cli.StringFlag{
+				Name:        "localWalletDir",
+				Usage:       "The path to the local wallet directory.",
+				DefaultText: "Defaults to '<user-home-directory>/.motion/wallet' with wallet key auto-generated if not present. Note that the directory permissions must be at most 0600.",
+				EnvVars:     []string{"MOTION_LOCAL_WALLET_DIR"},
+			},
+			&cli.BoolFlag{
+				Name:  "localWalletGenerateIfNotExist",
+				Usage: "Whether to generate the local wallet key if none is found",
+				Value: true,
+			},
 		},
 		Action: func(cctx *cli.Context) error {
+
+			localWalletDir := cctx.String("localWalletDir")
+			localWalletGenIfNotExist := cctx.Bool("localWalletGenerateIfNotExist")
+			ks, err := wallet.DefaultDiskKeyStoreOpener(localWalletDir, localWalletGenIfNotExist)()
+			if err != nil {
+				logger.Errorw("Failed to instantiate local wallet keystore", "err", err)
+				return err
+			}
+			wallet, err := wallet.New(
+				wallet.WithKeyStoreOpener(func() (types.KeyStore, error) { return ks, nil }),
+				wallet.WithGenerateKeyIfNotExist(localWalletGenIfNotExist))
+			if err != nil {
+				logger.Errorw("Failed to instantiate local wallet", "err", err)
+				return err
+			}
+
 			storeDir := cctx.String("storeDir")
 			var store blob.Store
 			if cctx.Bool("experimentalRibsStore") {
-				rbstore, err := blob.NewRibsStore(storeDir)
+				rbstore, err := blob.NewRibsStore(storeDir, ks)
 				if err != nil {
 					return err
 				}
@@ -52,7 +81,7 @@ func main() {
 				logger.Infow("Using local blob store", "storeDir", storeDir)
 			}
 
-			m, err := motion.New(motion.WithBlobStore(store))
+			m, err := motion.New(motion.WithBlobStore(store), motion.WithWallet(wallet))
 			if err != nil {
 				logger.Fatalw("Failed to instantiate Motion", "err", err)
 			}

go.mod

@@ -2,18 +2,24 @@ module github.com/filecoin-project/motion
 
 go 1.20
 
+replace github.com/filecoin-project/filecoin-ffi => ./extern/filecoin-ffi
+
 require (
+	github.com/filecoin-project/go-address v1.1.0
+	github.com/filecoin-project/go-state-types v0.11.1
+	github.com/filecoin-project/lotus v1.23.2-0.20230628201333-30a9f631653f
 	github.com/google/uuid v1.3.0
 	github.com/ipfs/boxo v0.10.2
 	github.com/ipfs/go-block-format v0.1.2
 	github.com/ipfs/go-cid v0.4.1
 	github.com/ipfs/go-log/v2 v2.5.1
-	github.com/lotus-web3/ribs v0.0.0-20230717152524-50fead08f629
+	github.com/lotus-web3/ribs v0.0.0-20230720185531-31964053b4be
 	github.com/multiformats/go-multihash v0.2.3
 	github.com/urfave/cli/v2 v2.25.7
 )
 
 require (
+	github.com/BurntSushi/toml v1.3.2 // indirect
 	github.com/DataDog/zstd v1.4.5 // indirect
 	github.com/GeertJohan/go.incremental v1.0.0 // indirect
 	github.com/GeertJohan/go.rice v1.0.3 // indirect
@@ -23,6 +29,7 @@ require (
 	github.com/benbjohnson/clock v1.3.5 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bep/debounce v1.2.1 // indirect
+	github.com/cespare/xxhash v1.1.0 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/cockroachdb/errors v1.8.1 // indirect
 	github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f // indirect
@@ -35,13 +42,17 @@ require (
 	github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 // indirect
 	github.com/cskr/pubsub v1.0.2 // indirect
 	github.com/daaku/go.zipexe v1.0.2 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect
 	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect
+	github.com/dgraph-io/badger/v2 v2.2007.4 // indirect
+	github.com/dgraph-io/ristretto v0.1.0 // indirect
+	github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect
 	github.com/docker/go-units v0.5.0 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/elastic/gosigar v0.14.2 // indirect
 	github.com/fatih/color v1.13.0 // indirect
-	github.com/filecoin-project/go-address v1.1.0 // indirect
+	github.com/filecoin-project/filecoin-ffi v0.30.4-0.20220519234331-bfd1f5f9fe38 // indirect
 	github.com/filecoin-project/go-amt-ipld/v2 v2.1.0 // indirect
 	github.com/filecoin-project/go-amt-ipld/v3 v3.1.0 // indirect
 	github.com/filecoin-project/go-amt-ipld/v4 v4.1.0 // indirect
@@ -62,12 +73,10 @@ require (
 	github.com/filecoin-project/go-legs v0.4.9 // indirect
 	github.com/filecoin-project/go-padreader v0.0.1 // indirect
 	github.com/filecoin-project/go-retrieval-types v1.2.0 // indirect
-	github.com/filecoin-project/go-state-types v0.11.1 // indirect
 	github.com/filecoin-project/go-statemachine v1.0.3 // indirect
 	github.com/filecoin-project/go-statestore v0.2.0 // indirect
 	github.com/filecoin-project/kubo-api-client v0.0.1 // indirect
 	github.com/filecoin-project/lassie v0.13.0 // indirect
-	github.com/filecoin-project/lotus v1.23.2-0.20230628201333-30a9f631653f // indirect
 	github.com/filecoin-project/specs-actors v0.9.15 // indirect
 	github.com/filecoin-project/specs-actors/v2 v2.3.6 // indirect
 	github.com/filecoin-project/specs-actors/v3 v3.1.2 // indirect
@@ -85,13 +94,15 @@ require (
 	github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
 	github.com/godbus/dbus/v5 v5.1.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/glog v1.1.0 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/mock v1.6.0 // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/google/gopacket v1.1.19 // indirect
 	github.com/google/pprof v0.0.0-20230602150820-91b7bce49751 // indirect
 	github.com/gorilla/websocket v1.5.0 // indirect
+	github.com/hako/durafmt v0.0.0-20200710122514-c0fb7b4da026 // indirect
 	github.com/hannahhoward/cbor-gen-for v0.0.0-20230214144701-5d17c9d5243c // indirect
 	github.com/hannahhoward/go-pubsub v1.0.0 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
@@ -104,6 +115,10 @@ require (
 	github.com/ipfs/go-bitfield v1.1.0 // indirect
 	github.com/ipfs/go-blockservice v0.5.1 // indirect
 	github.com/ipfs/go-datastore v0.6.0 // indirect
+	github.com/ipfs/go-ds-badger2 v0.1.3 // indirect
+	github.com/ipfs/go-ds-leveldb v0.5.0 // indirect
+	github.com/ipfs/go-ds-measure v0.2.0 // indirect
+	github.com/ipfs/go-fs-lock v0.0.7 // indirect
 	github.com/ipfs/go-graphsync v0.14.6 // indirect
 	github.com/ipfs/go-ipfs-blockstore v1.3.0 // indirect
 	github.com/ipfs/go-ipfs-cmds v0.9.0 // indirect
@@ -134,6 +149,7 @@ require (
 	github.com/jessevdk/go-flags v1.4.0 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/jpillora/backoff v1.0.0 // indirect
+	github.com/kelseyhightower/envconfig v1.4.0 // indirect
 	github.com/klauspost/compress v1.16.5 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.5 // indirect
 	github.com/koron/go-ssdp v0.0.4 // indirect
@@ -183,6 +199,7 @@ require (
 	github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect
 	github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/polydawn/refmt v0.89.0 // indirect
 	github.com/prometheus/client_golang v1.14.0 // indirect
 	github.com/prometheus/client_model v0.4.0 // indirect
@@ -198,17 +215,23 @@ require (
 	github.com/rogpeppe/go-internal v1.9.0 // indirect
 	github.com/rs/cors v1.7.0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
-	github.com/shirou/gopsutil v2.18.12+incompatible // indirect
+	github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible // indirect
 	github.com/spaolacci/murmur3 v1.1.0 // indirect
+	github.com/stretchr/testify v1.8.4 // indirect
 	github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect
+	github.com/tklauser/go-sysconf v0.3.5 // indirect
+	github.com/tklauser/numcpus v0.2.2 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	github.com/valyala/fasttemplate v1.0.1 // indirect
 	github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc // indirect
 	github.com/whyrusleeping/bencher v0.0.0-20190829221104-bb6607aa8bba // indirect
 	github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11 // indirect
 	github.com/whyrusleeping/cbor-gen v0.0.0-20230126041949-52956bd4c9aa // indirect
 	github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f // indirect
+	github.com/whyrusleeping/ledger-filecoin-go v0.9.1-0.20201010031517-c3dcc1bddce4 // indirect
 	github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
+	github.com/zondax/hid v0.9.1 // indirect
+	github.com/zondax/ledger-go v0.12.1 // indirect
 	go.opencensus.io v0.24.0 // indirect
 	go.opentelemetry.io/otel v1.16.0 // indirect
 	go.opentelemetry.io/otel/metric v1.16.0 // indirect
@@ -218,6 +241,7 @@ require (
 	go.uber.org/fx v1.19.3 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.24.0 // indirect
+	go4.org v0.0.0-20230225012048-214862532bf5 // indirect
 	golang.org/x/crypto v0.10.0 // indirect
 	golang.org/x/exp v0.0.0-20230321023759-10a507213a29 // indirect
 	golang.org/x/mod v0.10.0 // indirect
@@ -227,7 +251,10 @@ require (
 	golang.org/x/text v0.10.0 // indirect
 	golang.org/x/tools v0.9.1 // indirect
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
+	google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4 // indirect
+	google.golang.org/grpc v1.55.0 // indirect
 	google.golang.org/protobuf v1.30.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 	lukechampine.com/blake3 v1.2.1 // indirect
 	nhooyr.io/websocket v1.8.7 // indirect
 )