From be713a0ed83cd5c269aab2123432aa39871f3c26 Mon Sep 17 00:00:00 2001 From: Elijah Date: Fri, 15 Sep 2023 13:47:05 -1000 Subject: [PATCH] feat: singularity retrieve (#122) retrieve from singularity instead of local store closes #111 --- go.mod | 2 +- go.sum | 4 +- go.work.sum | 6 --- integration/singularity/reader.go | 76 ++++++++++++++++++++++++++++ integration/singularity/store.go | 15 +++--- integration/test/integration_test.go | 8 ++- 6 files changed, 92 insertions(+), 19 deletions(-) create mode 100644 integration/singularity/reader.go diff --git a/go.mod b/go.mod index 342dbb9..ad230e0 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/filecoin-project/motion go 1.20 require ( - github.com/data-preservation-programs/singularity v0.4.2-0.20230913002504-a1477cf53bf8 + github.com/data-preservation-programs/singularity v0.4.2-0.20230913015246-e92fc9e0958b github.com/filecoin-project/go-address v1.1.0 github.com/filecoin-project/go-state-types v0.12.0 github.com/google/uuid v1.3.1 diff --git a/go.sum b/go.sum index d0329fa..818583f 100644 --- a/go.sum +++ b/go.sum @@ -49,8 +49,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHH github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cskr/pubsub v1.0.2 h1:vlOzMhl6PFn60gRlTQQsIfVwaPB/B/8MziK8FhEPt/0= -github.com/data-preservation-programs/singularity v0.4.2-0.20230913002504-a1477cf53bf8 h1:vaKApEfEcHibc+rb8Pp9Dhf3nnhlCweNA2du0jeh4nQ= -github.com/data-preservation-programs/singularity v0.4.2-0.20230913002504-a1477cf53bf8/go.mod h1:ccWiS8hr2W0a2mD3SS2YwzQ21DsHVwUefp51HB7tT+U= +github.com/data-preservation-programs/singularity v0.4.2-0.20230913015246-e92fc9e0958b h1:9r/C+EUC3tFIW5ImF7FKAiCZqopuruprUkB4GABplP0= +github.com/data-preservation-programs/singularity v0.4.2-0.20230913015246-e92fc9e0958b/go.mod h1:ccWiS8hr2W0a2mD3SS2YwzQ21DsHVwUefp51HB7tT+U= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/go.work.sum b/go.work.sum index 50a512c..e78f043 100644 --- a/go.work.sum +++ b/go.work.sum @@ -639,7 +639,6 @@ github.com/cyphar/filepath-securejoin v0.2.2 h1:jCwT2GTP+PY5nBz3c/YL5PAIbusElVrP github.com/data-preservation-programs/singularity v0.3.1-0.20230907003343-42ad7e9fdbb4 h1:TdfffzGPvVIwR70hHWQd5ywbIAa1S1eAPKB4SyzvJ7E= github.com/data-preservation-programs/singularity v0.3.1-0.20230907003343-42ad7e9fdbb4/go.mod h1:ccWiS8hr2W0a2mD3SS2YwzQ21DsHVwUefp51HB7tT+U= github.com/data-preservation-programs/singularity v0.3.1-0.20230908045144-b48f89bbe21e/go.mod h1:ccWiS8hr2W0a2mD3SS2YwzQ21DsHVwUefp51HB7tT+U= -github.com/data-preservation-programs/singularity v0.4.2-0.20230912002758-c337aaa01329/go.mod h1:ccWiS8hr2W0a2mD3SS2YwzQ21DsHVwUefp51HB7tT+U= github.com/data-preservation-programs/table v0.0.3/go.mod h1:sRGP/IuuqFc/y9QfmDyb5h6Q2wrnhhnBofEOj9aDRJg= github.com/davidlazar/go-crypto v0.0.0-20170701192655-dcfb0a7ac018/go.mod h1:rQYf4tfk5sSwFsnDg3qYaBxSjsD9S8+59vW0dKUgme4= github.com/decred/dcrd/crypto/blake256 v1.0.1/go.mod h1:2OfgNZ5wDpcsFmHmCK5gZTPcCXqlm2ArzUIkw9czNJo= @@ -657,9 +656,7 @@ github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5m github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= -github.com/elazarl/goproxy v0.0.0-20221015165544-a0805db90819 h1:RIB4cRk+lBqKK3Oy0r2gRX4ui7tuhiZq2SuTtTCi0/0= github.com/elazarl/goproxy v0.0.0-20221015165544-a0805db90819/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM= -github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2 h1:dWB6v3RcOy03t/bUadywsbyrQwCqZeNIEX6M1OtSZOM= github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2/go.mod h1:gNh8nYJoAm43RfaxurUnxr+N1PwuFV3ZMl/efxlIlY8= github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g= github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= @@ -748,7 +745,6 @@ github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.2.1/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= -github.com/googleapis/gax-go v2.0.0+incompatible h1:j0GKcs05QVmm7yesiZq2+9cxHkNK9YM6zKx4D2qucQU= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= @@ -1258,8 +1254,6 @@ github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7V github.com/xhit/go-str2duration/v2 v2.1.0/go.mod h1:ohY8p+0f07DiV6Em5LKB0s2YpLtXVyJfNt1+BlmyAsU= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xorcare/golden v0.6.1-0.20191112154924-b87f686d7542/go.mod h1:7T39/ZMvaSEZlBPoYfVFmsBLmUl3uz9IuzWj/U6FtvQ= -go.dedis.ch/kyber/v3 v3.0.9 h1:i0ZbOQocHUjfFasBiUql5zVeC7u/vahFd96DFA8UOWk= -go.dedis.ch/protobuf v1.0.11 h1:FTYVIEzY/bfl37lu3pR4lIj+F9Vp1jE8oh91VmxKgLo= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= diff --git a/integration/singularity/reader.go b/integration/singularity/reader.go new file mode 100644 index 0000000..e242e78 --- /dev/null +++ b/integration/singularity/reader.go @@ -0,0 +1,76 @@ +package singularity + +import ( + "bytes" + "context" + "fmt" + "io" + + singularityclient "github.com/data-preservation-programs/singularity/client/swagger/http" + "github.com/data-preservation-programs/singularity/client/swagger/http/file" + "github.com/gotidy/ptr" +) + +// io.ReadSeekCloser implementation that reads from remote singularity +type SingularityReader struct { + client *singularityclient.SingularityAPI + fileID uint64 + offset int64 + size int64 +} + +func (r *SingularityReader) Read(p []byte) (int, error) { + logger.Infof("buffer size: %v", len(p)) + + buf := bytes.NewBuffer(p) + buf.Reset() + + if r.offset >= r.size { + return 0, io.EOF + } + + // Figure out how many bytes to read + readLen := int64(len(p)) + remainingBytes := r.size - r.offset + if readLen > remainingBytes { + readLen = remainingBytes + } + + _, _, err := r.client.File.RetrieveFile(&file.RetrieveFileParams{ + Context: context.Background(), + ID: int64(r.fileID), + Range: ptr.String(fmt.Sprintf("bytes=%d-%d", r.offset, r.offset+readLen-1)), + }, buf) + if err != nil { + return 0, fmt.Errorf("failed to retrieve file slice: %w", err) + } + + r.offset += readLen + + return int(readLen), nil +} + +func (r *SingularityReader) Seek(offset int64, whence int) (int64, error) { + var newOffset int64 + + switch whence { + case io.SeekStart: + newOffset = offset + case io.SeekCurrent: + newOffset = r.offset + offset + case io.SeekEnd: + newOffset = r.size + offset + } + + if newOffset > r.size { + return 0, fmt.Errorf("byte offset would exceed file size") + } + + r.offset = newOffset + + return r.offset, nil +} + +func (r *SingularityReader) Close() error { + return nil +} diff --git a/integration/singularity/store.go b/integration/singularity/store.go index e749ed9..8cf04b9 100644 --- a/integration/singularity/store.go +++ b/integration/singularity/store.go @@ -369,8 +369,6 @@ func (s *SingularityStore) Put(ctx context.Context, reader io.ReadCloser) (*blob } func (s *SingularityStore) Get(ctx context.Context, id blob.ID) (io.ReadSeekCloser, error) { - // this is largely artificial -- we're verifying the singularity item, but just reading from - // the local store idStream, err := os.Open(path.Join(s.local.Dir(), id.String()+".id")) if err != nil { return nil, err @@ -394,12 +392,13 @@ func (s *SingularityStore) Get(ctx context.Context, id blob.ID) (io.ReadSeekClos return nil, fmt.Errorf("error loading singularity entry: %w", err) } - var decoded blob.ID - err = decoded.Decode(strings.TrimSuffix(path.Base(getFileRes.Payload.Path), path.Ext(getFileRes.Payload.Path))) - if err != nil { - return nil, err - } - return s.local.Get(ctx, decoded) + + return &SingularityReader{ + client: s.singularityClient, + fileID: fileID, + offset: 0, + size: getFileRes.Payload.Size, + }, nil } func (s *SingularityStore) Describe(ctx context.Context, id blob.ID) (*blob.Descriptor, error) { diff --git a/integration/test/integration_test.go b/integration/test/integration_test.go index 104c1a8..a1c5901 100644 --- a/integration/test/integration_test.go +++ b/integration/test/integration_test.go @@ -19,8 +19,12 @@ import ( func TestRoundTripPutAndGet(t *testing.T) { env := NewEnvironment(t) - wantBlob := []byte("fish") - buf := bytes.NewBuffer(wantBlob) + buf := new(bytes.Buffer) + for i := 0; i < 10000000; i++ { + _, err := buf.Write([]byte("1234567890")) + require.NoError(t, err) + } + wantBlob := buf.Bytes() var postBlobResp api.PostBlobResponse {