Skip to content
This repository has been archived by the owner on May 15, 2024. It is now read-only.

Add replica status when using singularity store #67

Merged
merged 6 commits into from
Aug 26, 2023
Merged

Add replica status when using singularity store #67

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
45e2565
feat(blobstore): add replicas on singularity
hannahhoward Aug 24, 2023
f42d172
fix(deps): update kubo-api-client
hannahhoward Aug 25, 2023
86f2b5c
fix(lint): handle unhandled error
hannahhoward Aug 25, 2023
1b5b830
Merge branch 'main' into feat/singularity-status
hannahhoward Aug 25, 2023
a112d8e
chore(deps): update to tagged singularity
hannahhoward Aug 26, 2023
7345404
fix(motion): add commands for lotus vars
hannahhoward Aug 26, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 21 additions & 3 deletions blob/singularity_store.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import (
"github.com/data-preservation-programs/singularity/client"
"github.com/data-preservation-programs/singularity/handler/dataset"
"github.com/data-preservation-programs/singularity/handler/datasource"
"github.com/data-preservation-programs/singularity/service/epochutil"
)

const motionDatasetName = "MOTION_DATASET"
Expand Down Expand Up @@ -115,7 +116,7 @@ func (s *SingularityStore) Get(ctx context.Context, id ID) (io.ReadSeekCloser, e
if err != nil {
return nil, err
}
item, err := s.singularityClient.GetItem(ctx, itemID)
item, err := s.singularityClient.GetFile(ctx, itemID)
var asNotFoundError client.NotFoundError
if errors.As(err, &asNotFoundError) {
return nil, ErrBlobNotFound
Expand Down Expand Up @@ -146,7 +147,7 @@ func (s *SingularityStore) Describe(ctx context.Context, id ID) (*Descriptor, er
if err != nil {
return nil, err
}
item, err := s.singularityClient.GetItem(ctx, itemID)
item, err := s.singularityClient.GetFile(ctx, itemID)
var asNotFoundError client.NotFoundError
if errors.As(err, &asNotFoundError) {
return nil, ErrBlobNotFound
Expand All @@ -159,5 +160,22 @@ func (s *SingularityStore) Describe(ctx context.Context, id ID) (*Descriptor, er
if err != nil {
return nil, err
}
return s.local.Describe(ctx, decoded)
descriptor, err := s.local.Describe(ctx, decoded)
if err != nil {
return nil, err
}
deals, err := s.singularityClient.GetFileDeals(ctx, itemID)
replicas := make([]Replica, 0, len(deals))
for _, deal := range deals {
replicas = append(replicas, Replica{
// TODO: figure out how to get LastVerified
Provider: deal.Provider,
Status: string(deal.State),
Expiration: epochutil.EpochToTime(deal.EndEpoch),
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This assumes mainnet right? We probably want to parameterise this for devnet testing.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Actually no! But yea we'll probably need to call that function. https://github.com/data-preservation-programs/singularity/blob/main/service/epochutil/epoch.go#L37C1-L38C1

})
}
descriptor.Status = &Status{
Replicas: replicas,
}
return descriptor, nil
}
79 changes: 35 additions & 44 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,10 @@ replace github.com/filecoin-project/filecoin-ffi => ./extern/filecoin-ffi

require (
github.com/filecoin-project/go-address v1.1.0
github.com/filecoin-project/go-state-types v0.11.1
github.com/filecoin-project/go-state-types v0.12.0
github.com/filecoin-project/lotus v1.23.2-0.20230628201333-30a9f631653f
github.com/google/uuid v1.3.0
github.com/ipfs/boxo v0.10.2
github.com/ipfs/boxo v0.11.0
github.com/ipfs/go-block-format v0.1.2
github.com/ipfs/go-cid v0.4.1
github.com/ipfs/go-log/v2 v2.5.1
Expand Down Expand Up @@ -81,7 +81,7 @@ require (
github.com/filecoin-project/go-data-transfer/v2 v2.0.0-rc7 // indirect
github.com/filecoin-project/go-ds-versioning v0.1.2 // indirect
github.com/filecoin-project/go-fil-commcid v0.1.0 // indirect
github.com/filecoin-project/go-fil-commp-hashhash v0.2.0 // indirect
github.com/filecoin-project/go-fil-commp-hashhash v0.2.1-0.20230811065821-2e9c683db589 // indirect
github.com/filecoin-project/go-fil-markets v1.28.3 // indirect
github.com/filecoin-project/go-hamt-ipld v0.1.5 // indirect
github.com/filecoin-project/go-hamt-ipld/v2 v2.0.0 // indirect
Expand All @@ -93,7 +93,7 @@ require (
github.com/filecoin-project/go-statemachine v1.0.3 // indirect
github.com/filecoin-project/go-statestore v0.2.0 // indirect
github.com/filecoin-project/kubo-api-client v0.0.1 // indirect
github.com/filecoin-project/lassie v0.13.0 // indirect
github.com/filecoin-project/lassie v0.15.1-0.20230824015921-0b0d13c3a980 // indirect
github.com/filecoin-project/specs-actors v0.9.15 // indirect
github.com/filecoin-project/specs-actors/v2 v2.3.6 // indirect
github.com/filecoin-project/specs-actors/v3 v3.1.2 // indirect
Expand All @@ -120,17 +120,14 @@ require (
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 // indirect
github.com/golang-sql/sqlexp v0.1.0 // indirect
github.com/golang/glog v1.1.0 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/mock v1.6.0 // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/google/go-querystring v1.1.0 // indirect
github.com/google/gopacket v1.1.19 // indirect
github.com/google/pprof v0.0.0-20230602150820-91b7bce49751 // indirect
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
github.com/google/pprof v0.0.0-20230821062121-407c9e7a662f // indirect
github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect
github.com/googleapis/gax-go/v2 v2.7.0 // indirect
github.com/gorilla/websocket v1.5.0 // indirect
Expand All @@ -141,7 +138,7 @@ require (
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/hashicorp/go-uuid v1.0.3 // indirect
github.com/hashicorp/golang-lru v0.6.0 // indirect
github.com/hashicorp/golang-lru/v2 v2.0.2 // indirect
github.com/hashicorp/golang-lru/v2 v2.0.5 // indirect
github.com/hirochachacha/go-smb2 v1.1.0 // indirect
github.com/huin/goupnp v1.2.0 // indirect
github.com/icza/backscanner v0.0.0-20210726202459-ac2ffc679f94 // indirect
Expand All @@ -154,7 +151,7 @@ require (
github.com/ipfs/go-ds-leveldb v0.5.0 // indirect
github.com/ipfs/go-ds-measure v0.2.0 // indirect
github.com/ipfs/go-fs-lock v0.0.7 // indirect
github.com/ipfs/go-graphsync v0.14.6 // indirect
github.com/ipfs/go-graphsync v0.14.7 // indirect
github.com/ipfs/go-ipfs-blockstore v1.3.0 // indirect
github.com/ipfs/go-ipfs-chunker v0.0.5 // indirect
github.com/ipfs/go-ipfs-cmds v0.9.0 // indirect
Expand Down Expand Up @@ -194,7 +191,6 @@ require (
github.com/jcmturner/goidentity/v6 v6.0.1 // indirect
github.com/jcmturner/gokrb5/v8 v8.4.4 // indirect
github.com/jcmturner/rpc/v2 v2.0.3 // indirect
github.com/jellydator/ttlcache/v3 v3.0.1 // indirect
github.com/jessevdk/go-flags v1.4.0 // indirect
github.com/jinzhu/inflection v1.0.0 // indirect
github.com/jinzhu/now v1.1.5 // indirect
Expand All @@ -204,7 +200,7 @@ require (
github.com/jtolio/eventkit v0.0.0-20221004135224-074cf276595b // indirect
github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004 // indirect
github.com/kelseyhightower/envconfig v1.4.0 // indirect
github.com/klauspost/compress v1.16.5 // indirect
github.com/klauspost/compress v1.16.7 // indirect
github.com/klauspost/cpuid/v2 v2.2.5 // indirect
github.com/koofr/go-httpclient v0.0.0-20230225102643-5d51a2e9dea6 // indirect
github.com/koofr/go-koofrclient v0.0.0-20221207135200-cbd7fc9ad6a6 // indirect
Expand All @@ -218,26 +214,24 @@ require (
github.com/libp2p/go-buffer-pool v0.1.0 // indirect
github.com/libp2p/go-cidranger v1.1.0 // indirect
github.com/libp2p/go-flow-metrics v0.1.0 // indirect
github.com/libp2p/go-libp2p v0.27.8 // indirect
github.com/libp2p/go-libp2p v0.30.0 // indirect
github.com/libp2p/go-libp2p-asn-util v0.3.0 // indirect
github.com/libp2p/go-libp2p-gostream v0.6.0 // indirect
github.com/libp2p/go-libp2p-pubsub v0.9.3 // indirect
github.com/libp2p/go-libp2p-record v0.2.0 // indirect
github.com/libp2p/go-libp2p-routing-helpers v0.7.0 // indirect
github.com/libp2p/go-mplex v0.7.0 // indirect
github.com/libp2p/go-msgio v0.3.0 // indirect
github.com/libp2p/go-nat v0.1.0 // indirect
github.com/libp2p/go-nat v0.2.0 // indirect
github.com/libp2p/go-netroute v0.2.1 // indirect
github.com/libp2p/go-reuseport v0.3.0 // indirect
github.com/libp2p/go-yamux/v4 v4.0.0 // indirect
github.com/libp2p/go-reuseport v0.4.0 // indirect
github.com/libp2p/go-yamux/v4 v4.0.1 // indirect
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
github.com/magefile/mage v1.9.0 // indirect
github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-sqlite3 v1.14.17 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
github.com/microsoft/go-mssqldb v0.21.0 // indirect
github.com/miekg/dns v1.1.54 // indirect
github.com/miekg/dns v1.1.55 // indirect
github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect
github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect
github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 // indirect
Expand All @@ -246,7 +240,7 @@ require (
github.com/mr-tron/base58 v1.2.0 // indirect
github.com/multiformats/go-base32 v0.1.0 // indirect
github.com/multiformats/go-base36 v0.2.0 // indirect
github.com/multiformats/go-multiaddr v0.9.0 // indirect
github.com/multiformats/go-multiaddr v0.11.0 // indirect
github.com/multiformats/go-multiaddr-dns v0.3.1 // indirect
github.com/multiformats/go-multiaddr-fmt v0.1.0 // indirect
github.com/multiformats/go-multibase v0.2.0 // indirect
Expand All @@ -256,8 +250,8 @@ require (
github.com/ncw/go-acd v0.0.0-20201019170801-fe55f33415b1 // indirect
github.com/ncw/swift/v2 v2.0.1 // indirect
github.com/nkovacs/streamquote v1.0.0 // indirect
github.com/onsi/ginkgo/v2 v2.9.7 // indirect
github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417 // indirect
github.com/onsi/ginkgo/v2 v2.11.0 // indirect
github.com/opencontainers/runtime-spec v1.1.0 // indirect
github.com/opentracing/opentracing-go v1.2.0 // indirect
github.com/oracle/oci-go-sdk/v65 v65.32.0 // indirect
github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect
Expand All @@ -270,23 +264,22 @@ require (
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/polydawn/refmt v0.89.0 // indirect
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
github.com/prometheus/client_golang v1.14.0 // indirect
github.com/prometheus/client_golang v1.16.0 // indirect
github.com/prometheus/client_model v0.4.0 // indirect
github.com/prometheus/common v0.42.0 // indirect
github.com/prometheus/procfs v0.9.0 // indirect
github.com/prometheus/common v0.44.0 // indirect
github.com/prometheus/procfs v0.11.1 // indirect
github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8 // indirect
github.com/quic-go/qpack v0.4.0 // indirect
github.com/quic-go/qtls-go1-19 v0.3.3 // indirect
github.com/quic-go/qtls-go1-20 v0.2.3 // indirect
github.com/quic-go/quic-go v0.33.0 // indirect
github.com/quic-go/qtls-go1-20 v0.3.2 // indirect
github.com/quic-go/quic-go v0.38.0 // indirect
github.com/quic-go/webtransport-go v0.5.3 // indirect
github.com/raulk/clock v1.1.0 // indirect
github.com/raulk/go-watchdog v1.3.0 // indirect
github.com/rclone/rclone v1.62.2 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
github.com/rfjakob/eme v1.1.2 // indirect
github.com/rjNemo/underscore v0.5.0 // indirect
github.com/rogpeppe/go-internal v1.9.0 // indirect
github.com/rogpeppe/go-internal v1.10.0 // indirect
github.com/rs/cors v1.7.0 // indirect
github.com/shirou/gopsutil/v3 v3.23.3 // indirect
github.com/shoenig/go-m1cpu v0.1.4 // indirect
Expand All @@ -300,7 +293,7 @@ require (
github.com/tklauser/go-sysconf v0.3.11 // indirect
github.com/tklauser/numcpus v0.6.0 // indirect
github.com/vivint/infectious v0.0.0-20200605153912-25a574ae18a3 // indirect
github.com/whyrusleeping/cbor-gen v0.0.0-20230126041949-52956bd4c9aa // indirect
github.com/whyrusleeping/cbor-gen v0.0.0-20230418232409-daab9ece03a0 // indirect
github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f // indirect
github.com/x448/float16 v0.8.4 // indirect
github.com/xanzy/ssh-agent v0.3.3 // indirect
Expand All @@ -314,43 +307,41 @@ require (
go.opentelemetry.io/otel v1.16.0 // indirect
go.opentelemetry.io/otel/trace v1.16.0 // indirect
go.uber.org/dig v1.17.0 // indirect
go.uber.org/fx v1.19.3 // indirect
golang.org/x/crypto v0.11.0 // indirect
golang.org/x/exp v0.0.0-20230321023759-10a507213a29 // indirect
go.uber.org/fx v1.20.0 // indirect
golang.org/x/crypto v0.12.0 // indirect
golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 // indirect
golang.org/x/mod v0.12.0 // indirect
golang.org/x/net v0.12.0 // indirect
golang.org/x/oauth2 v0.6.0 // indirect
golang.org/x/net v0.14.0 // indirect
golang.org/x/oauth2 v0.8.0 // indirect
golang.org/x/sync v0.3.0 // indirect
golang.org/x/term v0.10.0 // indirect
golang.org/x/text v0.11.0 // indirect
golang.org/x/term v0.11.0 // indirect
golang.org/x/text v0.12.0 // indirect
golang.org/x/time v0.3.0 // indirect
golang.org/x/tools v0.11.0 // indirect
golang.org/x/tools v0.12.1-0.20230815132531-74c255bcf846 // indirect
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
google.golang.org/api v0.112.0 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4 // indirect
google.golang.org/grpc v1.55.0 // indirect
google.golang.org/protobuf v1.30.0 // indirect
google.golang.org/protobuf v1.31.0 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gorm.io/driver/mysql v1.5.0 // indirect
gorm.io/driver/postgres v1.5.0 // indirect
gorm.io/driver/sqlite v1.5.2 // indirect
gorm.io/driver/sqlserver v1.4.3 // indirect
gorm.io/gorm v1.25.2-0.20230530020048-26663ab9bf55 // indirect
lukechampine.com/blake3 v1.2.1 // indirect
modernc.org/libc v1.22.3 // indirect
modernc.org/mathutil v1.5.0 // indirect
modernc.org/memory v1.5.0 // indirect
modernc.org/sqlite v1.21.1 // indirect
nhooyr.io/websocket v1.8.7 // indirect
storj.io/common v0.0.0-20221123115229-fed3e6651b63 // indirect
storj.io/drpc v0.0.32 // indirect
storj.io/uplink v1.10.0 // indirect
)

require (
github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
github.com/data-preservation-programs/singularity v0.2.48-0.20230728231729-36c9e43d76d7
github.com/data-preservation-programs/singularity v0.2.48-0.20230823005535-a3e1aad1386d
github.com/mattn/go-isatty v0.0.19 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible // indirect
Expand All @@ -368,8 +359,8 @@ require (
go.opentelemetry.io/otel/metric v1.16.0 // indirect
go.uber.org/atomic v1.11.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
go.uber.org/zap v1.24.0 // indirect
go.uber.org/zap v1.25.0 // indirect
go4.org v0.0.0-20230225012048-214862532bf5 // indirect
golang.org/x/sys v0.10.0 // indirect
golang.org/x/sys v0.11.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
Loading
Loading