Merge remote-tracking branch 'origin' into xinaxu/deal-last-verified

.dockerignore

@@ -1,3 +1,4 @@
 .github
 *.md
-integration/test
+integration/test
+docker-compose*.yml

README.md

@@ -105,7 +105,7 @@ Motion will begin saving data to Filecoin when it's holding at least 16GB of dat
 If you want to test storing an actual filecoin deal, the following simple script will put about 20GB of random data into motion:
 
 ```shell
-for i in {0..20}; do; head -c 1000000000 /dev/urandom | curl -X POST --data-binary @- -H "Content-Type: application/octet-stream" http://localhost:40080/v0/blob; done
+for i in {0..20}; do head -c 1000000000 /dev/urandom | curl -X POST --data-binary @- -H "Content-Type: application/octet-stream" http://localhost:40080/v0/blob; done
 ```
 
 This should be enough to trigger at least 1 Filecoin deal being made from Motion

cmd/motion/main.go

@@ -76,13 +76,6 @@ func main() {
 				Name:     "lotus-test",
 				Category: "Lotus",
 				EnvVars:  []string{"LOTUS_TEST"},
-				Action: func(context *cli.Context, lotusTest bool) error {
-					if lotusTest {
-						logger.Info("Current network is set to Testnet")
-						address.CurrentNetwork = address.Testnet
-					}
-					return nil
-				},
 			},
 			&cli.UintFlag{
 				Name:        "replicationFactor",
@@ -165,6 +158,12 @@ func main() {
 			},
 		},
 		Action: func(cctx *cli.Context) error {
+			if cctx.Bool("lotus-test") {
+				logger.Info("Current network is set to Testnet")
+				address.CurrentNetwork = address.Testnet
+			} else {
+				address.CurrentNetwork = address.Mainnet
+			}
 			storeDir := cctx.String("storeDir")
 			var store blob.Store
 			if cctx.Bool("experimentalSingularityStore") {

go.mod

@@ -12,6 +12,7 @@ require (
 	github.com/stretchr/testify v1.8.4
 	github.com/urfave/cli/v2 v2.25.7
 	go.uber.org/goleak v1.2.0
+	gopkg.in/yaml.v3 v3.0.1
 )
 
 require (
@@ -164,7 +165,6 @@ require (
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
 	google.golang.org/protobuf v1.30.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
-	gopkg.in/yaml.v3 v3.0.1 // indirect
 	gorm.io/gorm v1.25.2-0.20230530020048-26663ab9bf55 // indirect
 	lukechampine.com/blake3 v1.2.1 // indirect
 )

integration/singularity/store.go

@@ -2,6 +2,7 @@ package singularity
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"io"
 	"math/big"
@@ -416,10 +417,12 @@ func (s *SingularityStore) Get(ctx context.Context, id blob.ID) (io.ReadSeekClos
 }
 
 func (s *SingularityStore) Describe(ctx context.Context, id blob.ID) (*blob.Descriptor, error) {
-	// this is largely artificial -- we're verifying the singularity item, but just reading from
-	// the local store
 	idStream, err := os.Open(path.Join(s.local.Dir(), id.String()+".id"))
 	if err != nil {
+		if errors.Is(err, os.ErrNotExist) {
+			return nil, blob.ErrBlobNotFound
+		}
+
 		return nil, err
 	}
 	fileIDString, err := io.ReadAll(idStream)

integration/test/api_test.go

@@ -0,0 +1,242 @@
+package test
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"strconv"
+	"strings"
+	"testing"
+
+	"github.com/filecoin-project/motion/api"
+	"github.com/stretchr/testify/require"
+	"gopkg.in/yaml.v3"
+)
+
+type testCase struct {
+	name          string
+	onMethod      string
+	onPath        string
+	onBody        string
+	onContentType string
+	expectStatus  int
+	expectBody    string // may be regex. optional (empty = not tested)
+
+	// Silences error for this schema case
+	skip bool
+}
+
+type schemaCase struct {
+	method  string
+	path    string
+	status  string
+	covered bool
+}
+
+func (s schemaCase) String() string {
+	return fmt.Sprintf("%s %s -> %s (covered: %v)\n", s.method, s.path, s.status, s.covered)
+}
+
+func TestApi(t *testing.T) {
+	env := NewEnvironment(t)
+
+	// Prereq: post 1 piece of data to test on
+	var testBlobResp api.PostBlobResponse
+	{
+		resp, err := http.Post(
+			requireJoinUrlPath(t, env.MotionAPIEndpoint, "v0", "blob"),
+			"application/octet-stream",
+			bytes.NewReader([]byte("a")),
+		)
+		require.NoError(t, err)
+
+		require.NoError(t, json.NewDecoder(resp.Body).Decode(&testBlobResp))
+
+		resp.Body.Close()
+	}
+
+	// ---- Add test cases here ----
+	tests := []testCase{
+		{
+			name:          "POST /v0/blob is 201",
+			onMethod:      http.MethodPost,
+			onPath:        "/v0/blob",
+			onBody:        "fish",
+			onContentType: "application/octet-stream",
+			expectBody:    "{\"id\":\".*\"}",
+			expectStatus:  201,
+		},
+		{
+			// not reliably testable
+			onMethod:     http.MethodPost,
+			onPath:       "/v0/blob",
+			expectStatus: 500,
+			skip:         true,
+		},
+		{
+			// not reliably testable
+			onMethod:     http.MethodPost,
+			onPath:       "/v0/blob",
+			expectStatus: 503,
+			skip:         true,
+		},
+		{
+			name:         "GET /v0/blob/{id} is 200",
+			onMethod:     http.MethodGet,
+			onPath:       "/v0/blob/" + testBlobResp.ID,
+			expectStatus: 200,
+		},
+		{
+			name:         "GET /v0/blob/{id} for unknown ID is 404",
+			onMethod:     http.MethodGet,
+			onPath:       "/v0/blob/00000000-0000-0000-0000-000000000000",
+			expectStatus: 404,
+		},
+		{
+			// not reliably testable
+			onMethod:     http.MethodGet,
+			onPath:       "/v0/blob/00000000-0000-0000-0000-000000000000",
+			expectStatus: 500,
+			skip:         true,
+		},
+		{
+			// not reliably testable
+			onMethod:     http.MethodGet,
+			onPath:       "/v0/blob/00000000-0000-0000-0000-000000000000",
+			expectStatus: 503,
+			skip:         true,
+		},
+		{
+			name:         "GET /v0/blob/{id}/status is 200",
+			onMethod:     http.MethodGet,
+			onPath:       "/v0/blob/" + testBlobResp.ID + "/status",
+			expectStatus: 200,
+		},
+		{
+			name:         "GET /v0/blob/{id}/status for unknown ID is 404",
+			onMethod:     http.MethodGet,
+			onPath:       "/v0/blob/00000000-0000-0000-0000-000000000000/status",
+			expectStatus: 404,
+		},
+		{
+			// not reliably testable
+			onMethod:     http.MethodGet,
+			onPath:       "/v0/blob/00000000-0000-0000-0000-000000000000/status",
+			expectStatus: 500,
+			skip:         true,
+		},
+		{
+			// not reliably testable
+			onMethod:     http.MethodGet,
+			onPath:       "/v0/blob/00000000-0000-0000-0000-000000000000/status",
+			expectStatus: 503,
+			skip:         true,
+		},
+	}
+
+	// Read and parse openapi.yaml for ensuring all paths, methods, and status
+	// codes are covered
+
+	schemaString, err := os.ReadFile("../../openapi.yaml")
+	require.NoError(t, err, "could not find openapi.yaml")
+
+	schemaMap := make(map[string]interface{})
+	err = yaml.Unmarshal(schemaString, schemaMap)
+	require.NoError(t, err)
+
+	var schemaCases []schemaCase
+
+	type kvmap = map[string]interface{}
+	for pathName, path := range schemaMap["paths"].(kvmap) {
+		for methodName, method := range path.(kvmap) {
+			for statusCode := range method.(kvmap)["responses"].(kvmap) {
+				schemaCases = append(schemaCases, schemaCase{
+					method:  methodName,
+					path:    pathName,
+					status:  statusCode,
+					covered: false,
+				})
+			}
+		}
+	}
+
+	// Run all tests
+	for _, test := range tests {
+		if !test.skip {
+			req, err := http.NewRequest(
+				test.onMethod,
+				requireJoinUrlPath(t, env.MotionAPIEndpoint, test.onPath),
+				bytes.NewReader([]byte(test.expectBody)),
+			)
+			req.Header.Set("Content-Type", test.onContentType)
+			require.NoError(t, err)
+
+			resp, err := http.DefaultClient.Do(req)
+			require.NoError(t, err)
+
+			// Body must be as expected
+			var body string
+			if test.expectBody != "" {
+				bodyBytes, err := io.ReadAll(resp.Body)
+				require.NoError(t, err)
+
+				body = string(bodyBytes)
+				require.Regexp(t, test.expectBody, string(body))
+			}
+
+			resp.Body.Close()
+
+			// Status code must be as expected
+			require.Equal(t, test.expectStatus, resp.StatusCode, "Incorrect status code for test %#v (resp body: %v)", test, body)
+		}
+
+		// Find matching schema case and mark as covered
+		for i := range schemaCases {
+			methodsMatch := strings.EqualFold(schemaCases[i].method, test.onMethod)
+			pathsMatch := schemaPathFitsTest(schemaCases[i].path, test.onPath)
+			statusesMatch := schemaCases[i].status == strconv.Itoa(test.expectStatus)
+
+			if methodsMatch && pathsMatch && statusesMatch {
+				schemaCases[i].covered = true
+				break
+			}
+		}
+	}
+
+	// Make sure all schema cases are covered
+	var notCovered []schemaCase
+	for _, schemaCase := range schemaCases {
+		if !schemaCase.covered {
+			notCovered = append(notCovered, schemaCase)
+		}
+	}
+
+	require.Empty(t, notCovered, "all schema cases must be covered")
+}
+
+// Checks whether a test's path fits into a schema path listed in openapi.yaml,
+// where the schema path may have variable parts (example, schema /foo/bar/{x}
+// == test /foo/bar/5).
+func schemaPathFitsTest(schemaPath string, testPath string) bool {
+	schemaParts := strings.Split(schemaPath, "/")
+	testParts := strings.Split(testPath, "/")
+
+	if len(schemaParts) != len(testParts) {
+		return false
+	}
+
+	for i := range schemaParts {
+		if schemaParts[i] == testParts[i] {
+			continue
+		} else if schemaParts[i][0] == '{' {
+			continue
+		} else {
+			return false
+		}
+	}
+
+	return true
+}