From f8707839b881638014610e12ac74b4af4d6a5ea0 Mon Sep 17 00:00:00 2001 From: Hannah Howard Date: Thu, 5 Oct 2023 01:14:29 -0700 Subject: [PATCH] Add integration test for full deal flow, fix retrieval bugs (#118) # Goals Write an integration test that verifies data is actually sent into singularity and deals are proposed to boost. Test retrieval including: - Test the retrieval process from both the datasource and Filecoin. - Verify the correct functioning of the read seeker. - Test the HTTP endpoint for file retrieval. - Verify Lassie integration and CAR file retrieval. fixes #103 # Implementation - Various bug fixes already shipped and merged to singularity discovered in testing - Move most execution of devnet integration test out of github action into a Makefile. See `integration/test/README.md` for more information. This enables running tests and devnets locally with ease - Switch from SINGULARITY_TAG to SINGULARITY_REF, removing the automatically inserted colon. This allows us to run from direct SHA tags as needed. It also means the code will run is no ref is specified, as opposed to before where it would fail if no SINGULARITY_TAG was present - Fix bug with go-routine execution of singularity preparation introduced in cleanup data PR - Fix bug with blob descriptors when local file is deleted - Fix bug where local data was deleted before the deal status was published or active - Add several hook scripts to enable acceptance of boost deals (such as pricing deals to zero) - Various configuration fixes to make dealmaking in test run smoothly - Integration test that now does a MUCH more full cycle -- pushes, retrieves, waits for full storage on filecoin, then retrieves with Lassie # For discussion - I had to add a custom entrypoint boost in order to properly configure boost to advertise booster-http. We should probably upstream this. - There is a bug fix for singularity that should get in before we merge this. --------- Co-authored-by: Masih H. Derkani Co-authored-by: Elijah --- .env.example | 2 +- .github/workflows/devnet-test.yml | 47 +------ .gitignore | 4 + docker-compose.yml | 14 +- go.work.sum | 3 + integration/singularity/store.go | 10 +- integration/test/Makefile | 47 +++++++ integration/test/README.md | 23 ++++ integration/test/boost-setup.sh | 7 + integration/test/buildsingularity.sh | 8 ++ integration/test/devnet/docker-compose.yaml | 4 + .../devnet/entrypoints/boost/entrypoint.sh | 87 ++++++++++++ integration/test/integration-setup.sh | 17 +++ integration/test/integration_test.go | 127 +++++++++++++++++- integration/test/motionlarity/.env | 11 +- .../test/motionlarity/docker-compose.yaml | 20 +-- integration/test/waitmotion.sh | 7 + 17 files changed, 363 insertions(+), 75 deletions(-) create mode 100755 integration/test/Makefile create mode 100644 integration/test/README.md create mode 100755 integration/test/boost-setup.sh create mode 100755 integration/test/buildsingularity.sh create mode 100755 integration/test/devnet/entrypoints/boost/entrypoint.sh create mode 100755 integration/test/integration-setup.sh create mode 100755 integration/test/waitmotion.sh diff --git a/.env.example b/.env.example index ed3b708..87c6c5d 100644 --- a/.env.example +++ b/.env.example @@ -1,5 +1,5 @@ # The version of singularity you will use to make deals. Generally, this should not change -SINGULARITY_TAG=v0.4.1 +SINGULARITY_REF=:v0.4.1 # Comma seperated list of storage providers motion should make storage deals with # You must set this value to contain at least one storage provider for motion to diff --git a/.github/workflows/devnet-test.yml b/.github/workflows/devnet-test.yml index 2f0dde2..fdc59ec 100644 --- a/.github/workflows/devnet-test.yml +++ b/.github/workflows/devnet-test.yml @@ -9,7 +9,7 @@ on: jobs: build: name: Boost Devnet - runs-on: ubuntu-latest + runs-on: ubuntu-latest-4-cores steps: - name: Checkout uses: actions/checkout@v3 @@ -30,49 +30,10 @@ jobs: ${{ runner.os }}-build-${{ env.cache-name }}- ${{ runner.os }}-build- ${{ runner.os }}- - - name: Run Boost Devnet from pre-built images + - name: Run Motion Integration Tests run: | - cd integration/test/devnet - docker compose up -d - - name: Await Lotus full node startup - run: | - cd integration/test/devnet - docker compose exec lotus lotus wait-api --timeout=20m - - name: Set up Motion Wallet and API endpoints - run: | - cd integration/test/devnet - # Setup Lotus API token - export `docker compose exec lotus lotus auth api-info --perm=admin` - IFS=: read -r token path <<< "${FULLNODE_API_INFO}" - echo "LOTUS_TOKEN=${token}" >> $GITHUB_ENV - - # Setup Motion Wallet - MOTION_WALLET_ADDR=`docker compose exec lotus lotus wallet new` - MOTION_WALLET_KEY=`docker compose exec lotus lotus wallet export ${MOTION_WALLET_ADDR}` - LOTUS_WALLET_DEFAULT_ADDR=`docker compose exec lotus lotus wallet default` - docker compose exec lotus lotus send --from=${LOTUS_WALLET_DEFAULT_ADDR} ${MOTION_WALLET_ADDR} 10 - echo "MOTION_WALLET_ADDR=${MOTION_WALLET_ADDR}" >> $GITHUB_ENV - echo "MOTION_WALLET_KEY=${MOTION_WALLET_KEY}" >> $GITHUB_ENV - echo "MOTION_STORAGE_PROVIDERS=t01000" >> $GITHUB_ENV - echo "MOTION_API_ENDPOINT=http://localhost:40080" >> $GITHUB_ENV - echo "SINGULARITY_API_ENDPOINT=http://localhost:9091" >> $GITHUB_ENV - - name: Run Motionlarity - run: | - cd integration/test/motionlarity - docker compose up -d - - name: Wait until motion is running - uses: nick-fields/retry@v2 - with: - timeout_seconds: 5 - max_attempts: 10 - retry_on: error - command: | - cd integration/test/motionlarity - docker compose ps --services --filter "status=running" | grep motion - - name: Run Motion integration tests - env: - MOTION_INTEGRATION_TEST: 'true' - run: go test ./integration/test -v + cd integration/test + make test - name: Start S3 Connector run: | cd integration/test/s3-connector diff --git a/.gitignore b/.gitignore index d00ca62..4d549f1 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,7 @@ /.env.local integration/test/devnet/data motion +integration/test/motionlarity/.env.local +integration/test/motionlarity/.up +integration/test/devnet/.up +integration/test/devnet/.boostready \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index b1b2525..3f2404c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -19,12 +19,10 @@ services: - 5432:5432 singularity_admin_init: - image: ghcr.io/data-preservation-programs/singularity:${SINGULARITY_TAG} + image: ghcr.io/data-preservation-programs/singularity${SINGULARITY_REF} command: admin init volumes: - motion-singularity-volume:/usr/src/app/storage - ports: - - 9090:9090 environment: DATABASE_CONNECTION_STRING: postgres://${SINGULARITY_DB_USER:-postgres}:${SINGULARITY_DB_PASSWORD:-postgres}@db:5432/${SINGULARITY_DB_NAME:-singularity} LOTUS_TEST: @@ -35,7 +33,7 @@ services: condition: service_healthy singularity_api: - image: ghcr.io/data-preservation-programs/singularity:${SINGULARITY_TAG} + image: ghcr.io/data-preservation-programs/singularity${SINGULARITY_REF} command: run api --bind :9090 volumes: - motion-singularity-volume:/usr/src/app/storage @@ -51,7 +49,7 @@ services: condition: service_completed_successfully singularity_dataset_worker: - image: ghcr.io/data-preservation-programs/singularity:${SINGULARITY_TAG} + image: ghcr.io/data-preservation-programs/singularity${SINGULARITY_REF} volumes: - motion-singularity-volume:/usr/src/app/storage command: run dataset-worker @@ -65,7 +63,7 @@ services: condition: service_completed_successfully singularity_deal_pusher: - image: ghcr.io/data-preservation-programs/singularity:${SINGULARITY_TAG} + image: ghcr.io/data-preservation-programs/singularity${SINGULARITY_REF} volumes: - motion-singularity-volume:/usr/src/app/storage command: run deal-pusher @@ -79,7 +77,7 @@ services: condition: service_completed_successfully singularity_deal_tracker: - image: ghcr.io/data-preservation-programs/singularity:${SINGULARITY_TAG} + image: ghcr.io/data-preservation-programs/singularity${SINGULARITY_REF} volumes: - motion-singularity-volume:/usr/src/app/storage command: run deal-tracker @@ -93,7 +91,7 @@ services: condition: service_completed_successfully singularity_content_provider: - image: ghcr.io/data-preservation-programs/singularity:${SINGULARITY_TAG} + image: ghcr.io/data-preservation-programs/singularity${SINGULARITY_REF} command: run content-provider --http-bind :7778 volumes: - motion-singularity-volume:/usr/src/app/storage diff --git a/go.work.sum b/go.work.sum index 4bae78f..89767e4 100644 --- a/go.work.sum +++ b/go.work.sum @@ -656,6 +656,7 @@ github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5m github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= +github.com/elazarl/goproxy v0.0.0-20221015165544-a0805db90819 h1:RIB4cRk+lBqKK3Oy0r2gRX4ui7tuhiZq2SuTtTCi0/0= github.com/elazarl/goproxy v0.0.0-20221015165544-a0805db90819/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM= github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2/go.mod h1:gNh8nYJoAm43RfaxurUnxr+N1PwuFV3ZMl/efxlIlY8= github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g= @@ -1255,6 +1256,8 @@ github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7V github.com/xhit/go-str2duration/v2 v2.1.0/go.mod h1:ohY8p+0f07DiV6Em5LKB0s2YpLtXVyJfNt1+BlmyAsU= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xorcare/golden v0.6.1-0.20191112154924-b87f686d7542/go.mod h1:7T39/ZMvaSEZlBPoYfVFmsBLmUl3uz9IuzWj/U6FtvQ= +go.dedis.ch/kyber/v3 v3.0.9 h1:i0ZbOQocHUjfFasBiUql5zVeC7u/vahFd96DFA8UOWk= +go.dedis.ch/protobuf v1.0.11 h1:FTYVIEzY/bfl37lu3pR4lIj+F9Vp1jE8oh91VmxKgLo= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= diff --git a/integration/singularity/store.go b/integration/singularity/store.go index cd4add7..9d5e61a 100644 --- a/integration/singularity/store.go +++ b/integration/singularity/store.go @@ -444,9 +444,10 @@ func (s *SingularityStore) Describe(ctx context.Context, id blob.ID) (*blob.Desc if err != nil { return nil, err } - descriptor, err := s.local.Describe(ctx, decoded) - if err != nil { - return nil, err + descriptor := &blob.Descriptor{ + ID: id, + Size: uint64(getFileRes.Payload.Size), + ModificationTime: time.Unix(0, getFileRes.Payload.LastModifiedNano), } getFileDealsRes, err := s.singularityClient.File.GetFileDeals(&file.GetFileDealsParams{ Context: ctx, @@ -545,7 +546,7 @@ binIteration: for _, sp := range s.storageProviders { var foundDealForSP bool for _, deal := range getFileDealsRes.Payload { - if deal.Provider == sp.String() { + if deal.Provider == sp.String() && (deal.State == models.ModelDealStatePublished || deal.State == models.ModelDealStateActive) { foundDealForSP = true break } @@ -560,6 +561,7 @@ binIteration: // If deals have been made for all SPs, the local bin file can be // deleted + logger.Infof("deleting local copy for deal %s, file %s", id, binFileName) binsToDelete = append(binsToDelete, binFileName) } diff --git a/integration/test/Makefile b/integration/test/Makefile new file mode 100755 index 0000000..fa3105d --- /dev/null +++ b/integration/test/Makefile @@ -0,0 +1,47 @@ +.PHONY: devnet/up devnet/down motionlarity/up motionlarity/down test + +./devnet/.up: + echo "Run Boost Devnet from pre-built images" + rm -rf ./devnet/data && docker compose -f ./devnet/docker-compose.yaml up -d + echo "Await Lotus full node startup" + docker compose -f ./devnet/docker-compose.yaml exec lotus lotus wait-api --timeout=20m + echo "Await Lotus-miner full node startup" + docker compose -f ./devnet/docker-compose.yaml exec lotus-miner lotus-miner wait-api --timeout=20m + touch ./devnet/.up + +devnet/up: ./devnet/.up + +./devnet/.boostready: ./devnet/.up + ./boost-setup.sh + touch ./devnet/.boostready + +./motionlarity/.env.local: ./devnet/.up + echo "Set up Motion Wallet and API endpoints" + cat ./motionlarity/.env > ./motionlarity/.env.local + echo "" >> ./motionlarity/.env.local + ./integration-setup.sh ./motionlarity/.env.local + +motionlarity/setup: ./motionlarity/.env.local ./devnet/.boostready + +devnet/down: motionlarity/down + docker compose -f ./devnet/docker-compose.yaml down && sleep 2 && rm -rf ./devnet/data + rm ./devnet/.up || true + +./motionlarity/.up: ./motionlarity/.env.local ./devnet/.boostready + ./buildsingularity.sh + echo "Run Motionlarity" + docker compose -f ./motionlarity/docker-compose.yaml --env-file ./motionlarity/.env.local up -d + ./waitmotion.sh + touch ./motionlarity/.up + +motionlarity/up: ./motionlarity/.up + +motionlarity/down: + docker compose -f ./motionlarity/docker-compose.yaml down --rmi=all --volumes + rm ./motionlarity/.up || true + rm ./motionlarity/.env.local || true + rm ./devnet/.boostready || true + +test: motionlarity/up + echo "Run Motion integration tests" + MOTION_INTEGRATION_TEST='true' go test . -v --count=1 diff --git a/integration/test/README.md b/integration/test/README.md new file mode 100644 index 0000000..8a9a594 --- /dev/null +++ b/integration/test/README.md @@ -0,0 +1,23 @@ +# Motion Devnet Integration Test Suite + +This integration allows you to spin up a complete devnet including Lotus, Boost, Singularity, and Motion, in order to test round trip deal flow. It includes an integration test that covers all steps of a motion deal workflow, including storing to a filecoin provider and retrieving data back via trustless HTTP retrieval. + +It's intended both as an integration test to run in CI, and a platform for developers to spin up test nets so they can try out new Motion features they are working on. + +### Command reference + +All devnets are spun up with `make`. Key commands: + +- `make devnet/up` - will spin up a boost / lotus devnet +- `make motionlarity/up` - will spin up motion and singularity pre-configured to make deals with a lotus/boost devnet. if a lotus / boost devnet is not already spun up, this will spin one up +- `make test` - this will run the integration test on top of the motion/singularity/boost/lotus devnets. It will spin up all required networks that are not already running +- `make motionalirity/down` -- this will shut down motion and singularity processes +- `make devnet/down` -- this will shut down the boost/lotus devnet. If singularity and motion processes running, it will shut them down as well + +### Using local singularity + +Motion processes that are spun up with this test suite automatically use the code that is in the local repository on the current branch. + +However, Singularity processes are by default spun up with a remote image. If you want to spin up Singularity using a local code repository as the base, you'll want to specify `SINGULARITY_LOCAL_DOCKERFILE` as the path to your local singularity repo. For example, if you store your go repositories on the traditional go src folder hierarchy, you can use: + +`SINGULARITY_LOCAL_DOCKERFILE=../../../../data-preservation-programs/singularity make motionlarity/up` \ No newline at end of file diff --git a/integration/test/boost-setup.sh b/integration/test/boost-setup.sh new file mode 100755 index 0000000..9523ba4 --- /dev/null +++ b/integration/test/boost-setup.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +echo "setup boost pricing" +for i in {1..10} +do + curl -X POST -d '{"operationName":"AppStorageAskUpdateMutation","variables":{"update":{"Price":"0", "VerifiedPrice": 0}},"query":"mutation AppStorageAskUpdateMutation($update: StorageAskUpdate!) {\n storageAskUpdate(update: $update)\n}\n"}' http://localhost:8080/graphql/query && break || sleep 5 +done \ No newline at end of file diff --git a/integration/test/buildsingularity.sh b/integration/test/buildsingularity.sh new file mode 100755 index 0000000..1f3cfbb --- /dev/null +++ b/integration/test/buildsingularity.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +if [ -n "$SINGULARITY_LOCAL_DOCKERFILE" ] +then + echo "Building singularity from local source" + source ./motionlarity/.env + docker build -t ghcr.io/data-preservation-programs/singularity${SINGULARITY_REF} ${SINGULARITY_LOCAL_DOCKERFILE} +fi \ No newline at end of file diff --git a/integration/test/devnet/docker-compose.yaml b/integration/test/devnet/docker-compose.yaml index f77c738..43e3ca6 100644 --- a/integration/test/devnet/docker-compose.yaml +++ b/integration/test/devnet/docker-compose.yaml @@ -61,6 +61,7 @@ services: boost: container_name: boost + platform: linux/amd64 image: ${BOOST_IMAGE} init: true ports: @@ -81,8 +82,10 @@ services: - ./data/lotus:/var/lib/lotus:ro - ./data/lotus-miner:/var/lib/lotus-miner:ro - ./data/sample:/app/public:rw + - ./entrypoints/boost:/app:rw booster-http: + platform: linux/amd64 container_name: booster-http image: ${BOOSTER_HTTP_IMAGE} init: true @@ -101,6 +104,7 @@ services: - ./data/lotus-miner:/var/lib/lotus-miner:ro booster-bitswap: + platform: linux/amd64 container_name: booster-bitswap image: ${BOOSTER_BITSWAP_IMAGE} init: true diff --git a/integration/test/devnet/entrypoints/boost/entrypoint.sh b/integration/test/devnet/entrypoints/boost/entrypoint.sh new file mode 100755 index 0000000..6784509 --- /dev/null +++ b/integration/test/devnet/entrypoints/boost/entrypoint.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash +set -e + +echo Wait for lotus is ready ... +lotus wait-api +echo Wait for lotus-miner is ready ... +lotus-miner wait-api +echo BOOST_PATH=$BOOST_PATH +export DEFAULT_WALLET=`lotus wallet default` +export FULLNODE_API_INFO=`lotus auth api-info --perm=admin | cut -f2 -d=` +export MINER_API_INFO=`lotus-miner auth api-info --perm=admin | cut -f2 -d=` + +if [ ! -f $BOOST_PATH/.init.boost ]; then + echo Init wallets ... + export COLLAT_WALLET=`lotus wallet new bls` + export PUBMSG_WALLET=`lotus wallet new bls` + export CLIENT_WALLET=`lotus wallet new bls` + echo MINER_API_INFO=$MINER_API_INFO + echo FULLNODE_API_INFO=$FULLNODE_API_INFO + echo PUBMSG_WALLET=$PUBMSG_WALLET + echo COLLAT_WALLET=$COLLAT_WALLET + + lotus send --from $DEFAULT_WALLET $COLLAT_WALLET 10 + lotus send --from $DEFAULT_WALLET $PUBMSG_WALLET 10 + lotus send --from $DEFAULT_WALLET $CLIENT_WALLET 10 + lotus wallet market add --from $DEFAULT_WALLET --address $CLIENT_WALLET 5 + lotus wallet market add --address $COLLAT_WALLET 5 + + until lotus-miner actor control set --really-do-it ${PUBMSG_WALLET}; do echo Waiting for storage miner API ready ...; sleep 1; done + + echo Init boost on first run ... + + boostd -vv --boost-repo $BOOST_PATH init --api-sealer=$MINER_API_INFO \ + --api-sector-index=$MINER_API_INFO \ + --wallet-publish-storage-deals=$PUBMSG_WALLET \ + --wallet-deal-collateral=$COLLAT_WALLET \ + --max-staging-deals-bytes=2000000000 + + # echo exit code: $? + + echo Setting port in boost config... + sed 's|ip4/0.0.0.0/tcp/0|ip4/0.0.0.0/tcp/50000|g' $BOOST_PATH/config.toml > $BOOST_PATH/config.toml.tmp; cp $BOOST_PATH/config.toml.tmp $BOOST_PATH/config.toml; rm $BOOST_PATH/config.toml.tmp + sed 's|127.0.0.1|0.0.0.0|g' $BOOST_PATH/config.toml > $BOOST_PATH/config.toml.tmp; cp $BOOST_PATH/config.toml.tmp $BOOST_PATH/config.toml; rm $BOOST_PATH/config.toml.tmp + + echo Done + touch $BOOST_PATH/.init.boost +fi + +# TODO(anteva): fixme: hack as boostd fails to start without this dir +mkdir -p /var/lib/boost/deal-staging + +if [ ! -f $BOOST_PATH/.register.boost ]; then + echo Temporary starting boost to get maddr... + + boostd -vv run &> $BOOST_PATH/boostd.log & + BOOST_PID=`echo $!` + echo Got boost PID = $BOOST_PID + + until cat $BOOST_PATH/boostd.log | grep maddr; do echo "Waiting for boost..."; sleep 1; done + echo Looks like boost started and initialized... + + echo Registering to lotus-miner... + MADDR=`cat $BOOST_PATH/boostd.log | grep maddr | cut -f3 -d"{" | cut -f1 -d:` + echo Got maddr=${MADDR} + + lotus-miner actor set-peer-id ${MADDR} + lotus-miner actor set-addrs /dns/boost/tcp/50000 + echo Registered + + touch $BOOST_PATH/.register.boost + echo Try to stop boost... + kill -15 $BOOST_PID || kill -9 $BOOST_PID + rm -f $BOOST_PATH/boostd.log + echo Super. DONE! Boostd is now configured and will be started soon +fi + +## Override config options +echo Updating config values +sed 's|ServiceApiInfo = ""|ServiceApiInfo = "ws://localhost:8042"|g' $BOOST_PATH/config.toml > $BOOST_PATH/config.toml.tmp; cp $BOOST_PATH/config.toml.tmp $BOOST_PATH/config.toml; rm $BOOST_PATH/config.toml.tmp +sed 's|ExpectedSealDuration = "24h0m0s"|ExpectedSealDuration = "0h0m10s"|g' $BOOST_PATH/config.toml > $BOOST_PATH/config.toml.tmp; cp $BOOST_PATH/config.toml.tmp $BOOST_PATH/config.toml; rm $BOOST_PATH/config.toml.tmp + +echo Setting HTTP addr +sed 's|HTTPRetrievalMultiaddr = ""|HTTPRetrievalMultiaddr = "/dns4/booster-http/tcp/7777/http"|g' $BOOST_PATH/config.toml > $BOOST_PATH/config.toml.tmp; cp $BOOST_PATH/config.toml.tmp $BOOST_PATH/config.toml; rm $BOOST_PATH/config.toml.tmp + +echo Starting LID service and boost in dev mode... +trap 'kill %1' SIGINT +exec boostd-data run leveldb --addr=0.0.0.0:8042 & boostd -vv run --nosync=true diff --git a/integration/test/integration-setup.sh b/integration/test/integration-setup.sh new file mode 100755 index 0000000..45ddbaf --- /dev/null +++ b/integration/test/integration-setup.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +echo "build config file and wallets for motion" +# Setup Lotus API token +export `docker compose -f ./devnet/docker-compose.yaml exec lotus lotus auth api-info --perm=admin` +IFS=: read -r token path <<< "${FULLNODE_API_INFO}" +# Setup Motion Wallet +MOTION_WALLET_ADDR=`docker compose -f ./devnet/docker-compose.yaml exec lotus lotus wallet new` +MOTION_WALLET_KEY=`docker compose -f ./devnet/docker-compose.yaml exec lotus lotus wallet export ${MOTION_WALLET_ADDR}` +LOTUS_WALLET_DEFAULT_ADDR=`docker compose -f ./devnet/docker-compose.yaml exec lotus lotus wallet default` +docker compose -f ./devnet/docker-compose.yaml exec lotus lotus send --from=${LOTUS_WALLET_DEFAULT_ADDR} ${MOTION_WALLET_ADDR} 10 +echo "LOTUS_TOKEN=${token}" >> $1 +echo "MOTION_WALLET_ADDR=${MOTION_WALLET_ADDR}" >> $1 +echo "MOTION_WALLET_KEY=${MOTION_WALLET_KEY}" >> $1 +echo "MOTION_STORAGE_PROVIDERS=t01000" >> $1 +echo "MOTION_API_ENDPOINT=http://localhost:40080" >> $1 +echo "SINGULARITY_API_ENDPOINT=http://localhost:9091" >> $1 \ No newline at end of file diff --git a/integration/test/integration_test.go b/integration/test/integration_test.go index 81cdec7..1f3ebb8 100644 --- a/integration/test/integration_test.go +++ b/integration/test/integration_test.go @@ -7,24 +7,65 @@ import ( "io" "net/http" "net/url" + "strings" "testing" + "time" "github.com/filecoin-project/motion/api" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -// TODO: Expand test cases to assert singularity store start-up operations, e.g.: -// * motion dataset exists -// * schedules are created +type BoostPublishQueryDeal struct { + ID string +} + +type BoostPublishQueryDealPublish struct { + Deals []BoostPublishQueryDeal +} + +type BoostPublishQueryData struct { + DealPublish BoostPublishQueryDealPublish `json:"dealPublish"` +} +type BoostPublishQuery struct { + Data BoostPublishQueryData `json:"data"` +} -func TestRoundTripPutAndGet(t *testing.T) { +func TestRoundTripPutStatusAndFullStorage(t *testing.T) { env := NewEnvironment(t) - wantBlob, err := io.ReadAll(io.LimitReader(rand.Reader, 10<<20)) + // make an 8MB random file -- to trigger at least one car generation + wantBlob, err := io.ReadAll(io.LimitReader(rand.Reader, 8<<20)) require.NoError(t, err) buf := bytes.NewBuffer(wantBlob) + // find out how many deals we're starting with + t.Log("check existing deals") + var existingDeals int + { + req, err := http.NewRequest("GET", "http://localhost:8080/graphql/query", strings.NewReader("{\"operationName\":\"AppDealPublishQuery\",\"variables\":{},\"query\":\"query AppDealPublishQuery { dealPublish { Deals { ID __typename } __typename }}\"}")) + require.NoError(t, err) + getResp, err := http.DefaultClient.Do(req) + require.NoError(t, err) + require.EqualValues(t, http.StatusOK, getResp.StatusCode) + decoder := json.NewDecoder(getResp.Body) + var bpq BoostPublishQuery + err = decoder.Decode(&bpq) + require.NoError(t, err) + existingDeals = len(bpq.Data.DealPublish.Deals) + } + + // force boost to clear any publishable deals from singularity + t.Log("clearing boost publish queue") + { + postResp, err := http.Post("http://localhost:8080/graphql/query", "application/json", strings.NewReader("{\"operationName\":\"AppDealPublishNowMutation\",\"variables\":{},\"query\":\"mutation AppDealPublishNowMutation { dealPublishNow }\"}")) + require.NoError(t, err) + require.EqualValues(t, http.StatusOK, postResp.StatusCode) + require.NoError(t, postResp.Body.Close()) + } + var postBlobResp api.PostBlobResponse + t.Log("posting 8MB data into motion") { postResp, err := http.Post(requireJoinUrlPath(t, env.MotionAPIEndpoint, "v0", "blob"), "application/octet-stream", buf) require.NoError(t, err) @@ -36,6 +77,82 @@ func TestRoundTripPutAndGet(t *testing.T) { } var gotBlob []byte + t.Log("requesting data back while still in source storage") + { + getResp, err := http.Get(requireJoinUrlPath(t, env.MotionAPIEndpoint, "v0", "blob", postBlobResp.ID)) + require.NoError(t, err) + defer func() { require.NoError(t, getResp.Body.Close()) }() + require.EqualValues(t, http.StatusOK, getResp.StatusCode) + gotBlob, err = io.ReadAll(getResp.Body) + require.NoError(t, err) + } + require.Equal(t, wantBlob, gotBlob) + + // get the status, and continue to check until we verify a deal was at least proposed through boost + t.Log("waiting for singularity to make deals with boost") + { + require.EventuallyWithT(t, func(c *assert.CollectT) { + getResp, err := http.Get(requireJoinUrlPath(t, env.MotionAPIEndpoint, "v0", "blob", postBlobResp.ID, "status")) + assert.NoError(c, err) + defer func() { assert.NoError(c, getResp.Body.Close()) }() + assert.EqualValues(c, http.StatusOK, getResp.StatusCode) + jsonResp := json.NewDecoder(getResp.Body) + var decoded api.GetStatusResponse + err = jsonResp.Decode(&decoded) + assert.NoError(c, err) + assert.Equal(c, len(decoded.Replicas), 2) + }, 2*time.Minute, 5*time.Second, "never initiated deal making") + } + + // wait for deals to transfer to boost + t.Log("waiting for singularity to transfer data to boost") + { + require.EventuallyWithT(t, func(c *assert.CollectT) { + req, err := http.NewRequest("GET", "http://localhost:8080/graphql/query", strings.NewReader("{\"operationName\":\"AppDealPublishQuery\",\"variables\":{},\"query\":\"query AppDealPublishQuery { dealPublish { Deals { ID __typename } __typename }}\"}")) + assert.NoError(c, err) + getResp, err := http.DefaultClient.Do(req) + assert.NoError(c, err) + assert.EqualValues(c, http.StatusOK, getResp.StatusCode) + decoder := json.NewDecoder(getResp.Body) + var bpq BoostPublishQuery + decoder.Decode(&bpq) + assert.Len(c, bpq.Data.DealPublish.Deals, existingDeals+2) + }, 2*time.Minute, 5*time.Second, "never finished data transfer") + } + + // force boost to publish the deals from singularity + t.Log("triggering data publish in boost") + { + + postResp, err := http.Post("http://localhost:8080/graphql/query", "application/json", strings.NewReader("{\"operationName\":\"AppDealPublishNowMutation\",\"variables\":{},\"query\":\"mutation AppDealPublishNowMutation { dealPublishNow}\"}")) + require.NoError(t, err) + require.EqualValues(t, http.StatusOK, postResp.StatusCode) + require.NoError(t, postResp.Body.Close()) + } + + // await publishing + t.Log("awaiting successful deal publishing") + { + require.EventuallyWithT(t, func(c *assert.CollectT) { + getResp, err := http.Get(requireJoinUrlPath(t, env.MotionAPIEndpoint, "v0", "blob", postBlobResp.ID, "status")) + assert.NoError(c, err) + defer func() { assert.NoError(c, getResp.Body.Close()) }() + assert.EqualValues(c, http.StatusOK, getResp.StatusCode) + jsonResp := json.NewDecoder(getResp.Body) + var decoded api.GetStatusResponse + err = jsonResp.Decode(&decoded) + assert.NoError(c, err) + assert.Equal(c, len(decoded.Replicas), 2) + for _, replica := range decoded.Replicas { + assert.Contains(c, []string{"published", "active"}, replica.Status) + } + }, 2*time.Minute, 5*time.Second, "published deals") + } + + // this is just to allow the cleanup worker to run + t.Log("sleeping for 1 minute to allow cleanup worker to run, and indexing to complete") + time.Sleep(1 * time.Minute) + { getResp, err := http.Get(requireJoinUrlPath(t, env.MotionAPIEndpoint, "v0", "blob", postBlobResp.ID)) require.NoError(t, err) diff --git a/integration/test/motionlarity/.env b/integration/test/motionlarity/.env index 6de0825..92963cd 100644 --- a/integration/test/motionlarity/.env +++ b/integration/test/motionlarity/.env @@ -1,8 +1,11 @@ -SINGULARITY_TAG=main +SINGULARITY_REF=:dummy LOTUS_TEST='true' LOTUS_API=http://lotus:1234/rpc/v1 -MOTION_PRICE_PER_GIB_EPOCH=0.000000000000002 +MOTION_PRICE_PER_GIB_EPOCH=0 MOTION_SINGULARITY_MAX_CAR_SIZE=7MiB MOTION_SINGULARITY_PACK_THRESHOLD=4096 -MOTION_SINGULARITY_SCHEDULE_CRON='* * * * *' -MOTION_VERIFIED_DEAL=true \ No newline at end of file +MOTION_SINGULARITY_SCHEDULE_CRON='* * * * * *' +MOTION_VERIFIED_DEAL=false +MOTION_SINGULARITY_SCHEDULE_DEAL_NUMBER=25 +MARKET_DEAL_URL= +MOTION_SINGULARITY_LOCAL_CLEANUP_INTERVAL=10s \ No newline at end of file diff --git a/integration/test/motionlarity/docker-compose.yaml b/integration/test/motionlarity/docker-compose.yaml index 0e4dc81..4e0c3f4 100644 --- a/integration/test/motionlarity/docker-compose.yaml +++ b/integration/test/motionlarity/docker-compose.yaml @@ -27,12 +27,10 @@ services: logging: *default-logging singularity_admin_init: - image: ghcr.io/data-preservation-programs/singularity:${SINGULARITY_TAG} + image: ghcr.io/data-preservation-programs/singularity${SINGULARITY_REF} command: admin init volumes: - motion-singularity-volume:/usr/src/app/storage - ports: - - 9091:9091 environment: DATABASE_CONNECTION_STRING: postgres://${SINGULARITY_DB_USER:-postgres}:${SINGULARITY_DB_PASSWORD:-postgres}@db:5432/${SINGULARITY_DB_NAME:-singularity} LOTUS_TEST: @@ -43,7 +41,7 @@ services: condition: service_healthy singularity_api: - image: ghcr.io/data-preservation-programs/singularity:${SINGULARITY_TAG} + image: ghcr.io/data-preservation-programs/singularity${SINGULARITY_REF} command: run api --bind :9091 volumes: - motion-singularity-volume:/usr/src/app/storage @@ -61,10 +59,10 @@ services: condition: service_completed_successfully singularity_dataset_worker: - image: ghcr.io/data-preservation-programs/singularity:${SINGULARITY_TAG} + image: ghcr.io/data-preservation-programs/singularity${SINGULARITY_REF} volumes: - motion-singularity-volume:/usr/src/app/storage - command: run dataset-worker + command: run dataset-worker --min-interval 1s --max-interval 5s restart: unless-stopped environment: DATABASE_CONNECTION_STRING: postgres://${SINGULARITY_DB_USER:-postgres}:${SINGULARITY_DB_PASSWORD:-postgres}@db:5432/${SINGULARITY_DB_NAME:-singularity} @@ -77,7 +75,7 @@ services: condition: service_completed_successfully singularity_deal_pusher: - image: ghcr.io/data-preservation-programs/singularity:${SINGULARITY_TAG} + image: ghcr.io/data-preservation-programs/singularity${SINGULARITY_REF} volumes: - motion-singularity-volume:/usr/src/app/storage command: run deal-pusher @@ -93,23 +91,24 @@ services: condition: service_completed_successfully singularity_deal_tracker: - image: ghcr.io/data-preservation-programs/singularity:${SINGULARITY_TAG} + image: ghcr.io/data-preservation-programs/singularity${SINGULARITY_REF} volumes: - motion-singularity-volume:/usr/src/app/storage - command: run deal-tracker + command: run deal-tracker -i 1s restart: unless-stopped environment: DATABASE_CONNECTION_STRING: postgres://${SINGULARITY_DB_USER:-postgres}:${SINGULARITY_DB_PASSWORD:-postgres}@db:5432/${SINGULARITY_DB_NAME:-singularity} LOTUS_TEST: LOTUS_API: LOTUS_TOKEN: + MARKET_DEAL_URL: logging: *default-logging depends_on: singularity_admin_init: condition: service_completed_successfully singularity_content_provider: - image: ghcr.io/data-preservation-programs/singularity:${SINGULARITY_TAG} + image: ghcr.io/data-preservation-programs/singularity${SINGULARITY_REF} command: run content-provider --http-bind :7778 volumes: - motion-singularity-volume:/usr/src/app/storage @@ -150,6 +149,7 @@ services: - MOTION_SINGULARITY_SCHEDULE_DEAL_NUMBER - MOTION_WALLET_KEY - MOTION_VERIFIED_DEAL + - MOTION_SINGULARITY_LOCAL_CLEANUP_INTERVAL volumes: - motion-singularity-volume:/usr/src/app/storage logging: *default-logging diff --git a/integration/test/waitmotion.sh b/integration/test/waitmotion.sh new file mode 100755 index 0000000..0a17372 --- /dev/null +++ b/integration/test/waitmotion.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +echo "Wait until motion is running" +for i in {1..10} +do + docker compose -f ./motionlarity/docker-compose.yaml ps --services --filter "status=running" | grep motion && break || sleep 5 +done \ No newline at end of file