Use UUID to represent blob identifiers

In order to maximise for future flexibility to deal with large blobs,
use random UUIDs as blob IDs. This allows the ID generation to be
independent from the blob upload completion and or chunking.

Update documentation for the exported APIs.

README.md

@@ -9,7 +9,13 @@ To run the server locally, execute:
 go run ./cmd/motion
 ```
 
-The above command starts the Motion HTTP API exposed on default listen address: http://localhost:40080
+Alternatively, run the latest motion as a container by executing:
+
+```shell
+docker run --rm ghcr.io/filecoin-project/motion:main
+```
+
+The above starts the Motion HTTP API exposed on default listen address: http://localhost:40080
 
 For more information, see the [Motion OpenAPI specification](openapi.yaml).
 

api/model.go

@@ -1,10 +1,14 @@
 package api
 
 type (
+	// PostBlobResponse represents the response to a successful POST request to upload a blob.
 	PostBlobResponse struct {
+		// ID is the unique identifier for the uploaded blob.
 		ID string `json:"id"`
 	}
+	// ErrorResponse represents the response that signal an error has occurred.
 	ErrorResponse struct {
+		// Error is the description of the error.
 		Error string `json:"error"`
 	}
 )

api/server/handler.go

@@ -5,7 +5,6 @@ import (
 	"net/http"
 	"strconv"
 	"strings"
-	"time"
 
 	"github.com/filecoin-project/motion/api"
 	"github.com/filecoin-project/motion/blob"
@@ -95,7 +94,7 @@ func (m *HttpServer) handleBlobGetByID(w http.ResponseWriter, r *http.Request, i
 		return
 	}
 	logger := logger.With("id", id)
-	blobReader, err := m.store.Get(r.Context(), id)
+	blobReader, blobDesc, err := m.store.Get(r.Context(), id)
 	switch err {
 	case nil:
 	case blob.ErrBlobNotFound:
@@ -108,10 +107,9 @@ func (m *HttpServer) handleBlobGetByID(w http.ResponseWriter, r *http.Request, i
 	defer blobReader.Close()
 	w.Header().Set(httpHeaderContentTypeOctetStream())
 	w.Header().Set(httpHeaderContentTypeOptionsNoSniff())
+	w.Header().Set(httpHeaderContentLength(blobDesc.Size))
 	w.Header().Set("Content-Disposition", fmt.Sprintf(`attachement; filename="%s.bin"`, id.String()))
-	// TODO: Discuss supporting blob creation time, which can then be passed as last modified time
-	//       below instead of zero-time. This has added benefit of handling specific HTTP range request.
-	http.ServeContent(w, r, "", time.Time{}, blobReader)
+	http.ServeContent(w, r, "", blobDesc.ModificationTime, blobReader)
 	logger.Debug("Blob fetched successfully")
 }
 

api/server/options.go

@@ -1,6 +1,7 @@
 package server
 
 type (
+	// Option is a configurable parameter in HttpServer.
 	Option  func(*options) error
 	options struct {
 		httpListenAddr string
@@ -21,13 +22,17 @@ func newOptions(o ...Option) (*options, error) {
 	return opts, nil
 }
 
+// WithHttpListenAddr sets the HTTP server listen address.
+// Defaults to 0.0.0.0:40080 if unspecified.
 func WithHttpListenAddr(addr string) Option {
 	return func(o *options) error {
 		o.httpListenAddr = addr
 		return nil
 	}
 }
 
+// WithMaxBlobLength sets the maximum blob length accepted by the HTTP blob upload API.
+// Defaults to 32 GiB.
 func WithMaxBlobLength(l uint64) Option {
 	return func(o *options) error {
 		o.maxBlobLength = l

api/server/server.go

@@ -12,14 +12,15 @@ import (
 
 var logger = log.Logger("motion/api/server")
 
-type (
-	HttpServer struct {
-		*options
-		httpServer *http.Server
-		store      blob.Store
-	}
-)
+// HttpServer is Motion API the HTTP server.
+type HttpServer struct {
+	*options
+	httpServer *http.Server
+	store      blob.Store
+}
 
+// NewHttpServer instantiates a new HTTP server that stores and retrieves blobs via the given store.
+// See Option.
 func NewHttpServer(store blob.Store, o ...Option) (*HttpServer, error) {
 	opts, err := newOptions(o...)
 	if err != nil {
@@ -35,6 +36,8 @@ func NewHttpServer(store blob.Store, o ...Option) (*HttpServer, error) {
 	return server, nil
 }
 
+// Start Starts the HTTP server.
+// See Shutdown.
 func (m *HttpServer) Start(_ context.Context) error {
 	listener, err := net.Listen("tcp", m.httpListenAddr)
 	if err != nil {
@@ -51,6 +54,7 @@ func (m *HttpServer) Start(_ context.Context) error {
 	return nil
 }
 
+// ServeMux returns a new HTTP handler for the endpoints supported by the server.
 func (m *HttpServer) ServeMux() *http.ServeMux {
 	mux := http.NewServeMux()
 	mux.HandleFunc("/v0/blob", m.handleBlobRoot)
@@ -59,6 +63,7 @@ func (m *HttpServer) ServeMux() *http.ServeMux {
 	return mux
 }
 
+// Shutdown shuts down the HTTP Server.
 func (m *HttpServer) Shutdown(ctx context.Context) error {
 	return m.httpServer.Shutdown(ctx)
 }

api/server/util.go

@@ -3,6 +3,7 @@ package server
 import (
 	"encoding/json"
 	"net/http"
+	"strconv"
 	"strings"
 
 	"github.com/filecoin-project/motion/api"
@@ -19,6 +20,10 @@ func httpHeaderContentTypeOptionsNoSniff() (string, string) {
 	return "X-Content-Type-Options", "nosniff"
 }
 
+func httpHeaderContentLength(length uint64) (string, string) {
+	return "Content-Length", strconv.FormatUint(length, 10)
+}
+
 func httpHeaderAllow(methods ...string) (string, string) {
 	return "Allow", strings.Join(methods, ",")
 }

blob/blob.go

@@ -4,8 +4,9 @@ import (
 	"context"
 	"errors"
 	"io"
+	"time"
 
-	"github.com/ipfs/go-cid"
+	"github.com/google/uuid"
 )
 
 var (
@@ -14,26 +15,44 @@ var (
 )
 
 type (
-	ID         cid.Cid // TODO: Discuss if everyone is on board with using CIDs as blob ID.
+	// ID uniquely identifies a blob.
+	ID uuid.UUID
+	// Descriptor describes a created blob.
 	Descriptor struct {
-		ID   ID // TODO: Discuss whether to use CIDs straight up.
+		// ID is the blob identifier.
+		ID ID
+		// Size is the size of blob in bytes.
 		Size uint64
+		// ModificationTime is the latest time at which the blob was modified.
+		ModificationTime time.Time
 	}
 	Store interface {
 		Put(context.Context, io.ReadCloser) (*Descriptor, error)
-		Get(context.Context, ID) (io.ReadSeekCloser, error)
+		Get(context.Context, ID) (io.ReadSeekCloser, *Descriptor, error)
 	}
 )
 
+// NewID instantiates a new randomly generated ID.
+func NewID() (*ID, error) {
+	id, err := uuid.NewRandom()
+	if err != nil {
+		return nil, err
+	}
+	i := ID(id)
+	return &i, nil
+}
+
+// String returns the string representation of ID.
 func (i *ID) String() string {
-	return cid.Cid(*i).String()
+	return uuid.UUID(*i).String()
 }
 
+// Decode instantiates the ID from the decoded string value.
 func (i *ID) Decode(v string) error {
-	decode, err := cid.Decode(v)
+	id, err := uuid.Parse(v)
 	if err != nil {
 		return err
 	}
-	*i = ID(decode)
+	*i = ID(id)
 	return nil
 }

blob/local_store.go

@@ -6,60 +6,75 @@ import (
 	"io"
 	"os"
 	"path"
-
-	"github.com/ipfs/go-cid"
-	"github.com/multiformats/go-multihash"
 )
 
 var _ Store = (*LocalStore)(nil)
 
+// LocalStore is a Store that stores blobs as flat files in a configured directory.
+// Blobs are stored as flat files, named by their ID with .bin extension.
+// This store is used primarily for testing purposes.
 type LocalStore struct {
 	dir string
 }
 
+// NewLocalStore instantiates a new LocalStore and uses the given dir as the place to store blobs.
+// Blobs are stored as flat files, named by their ID with .bin extension.
 func NewLocalStore(dir string) *LocalStore {
 	return &LocalStore{
 		dir: dir,
 	}
 }
 
+// Put reads the given reader fully and stores its content in the store directory as flat files.
+// The reader content is first stored in a temporary directory and upon successful storage is moved to the store directory.
+// The Descriptor.ModificationTime is set to the modification date of the file that corresponds to the content.
+// The Descriptor.ID is randomly generated; see NewID.
 func (l *LocalStore) Put(_ context.Context, reader io.ReadCloser) (*Descriptor, error) {
-	hasher, err := multihash.GetHasher(multihash.SHA2_256)
+	id, err := NewID()
 	if err != nil {
 		return nil, err
 	}
-	teeReader := io.TeeReader(reader, hasher)
 	dest, err := os.CreateTemp("", "motion_local_store_*.bin")
 	if err != nil {
 		return nil, err
 	}
 	defer dest.Close()
-	written, err := io.Copy(dest, teeReader)
+	written, err := io.Copy(dest, reader)
 	if err != nil {
+		os.Remove(dest.Name())
 		return nil, err
 	}
-	sum := hasher.Sum(nil)
-	mh, err := multihash.Encode(sum, multihash.SHA2_256)
-	if err != nil {
+	if err = os.Rename(dest.Name(), path.Join(l.dir, id.String()+".bin")); err != nil {
 		return nil, err
 	}
-	id := cid.NewCidV1(cid.Raw, mh)
-	if err = os.Rename(dest.Name(), path.Join(l.dir, id.String()+".bin")); err != nil {
+	stat, err := dest.Stat()
+	if err != nil {
 		return nil, err
 	}
 	return &Descriptor{
-		ID:   ID(id),
-		Size: uint64(written),
+		ID:               *id,
+		Size:             uint64(written),
+		ModificationTime: stat.ModTime(),
 	}, nil
 }
 
-func (l *LocalStore) Get(_ context.Context, id ID) (io.ReadSeekCloser, error) {
+// Get Retrieves the content of blob log with its Descriptor.
+// If no file is found for the given id, ErrBlobNotFound is returned.
+func (l *LocalStore) Get(_ context.Context, id ID) (io.ReadSeekCloser, *Descriptor, error) {
 	switch blob, err := os.Open(path.Join(l.dir, id.String()+".bin")); {
 	case err == nil:
-		return blob, nil
+		stat, err := blob.Stat()
+		if err != nil {
+			return nil, nil, err
+		}
+		return blob, &Descriptor{
+			ID:               id,
+			Size:             uint64(stat.Size()),
+			ModificationTime: stat.ModTime(),
+		}, nil
 	case errors.Is(err, os.ErrNotExist):
-		return nil, ErrBlobNotFound
+		return nil, nil, ErrBlobNotFound
 	default:
-		return nil, err
+		return nil, nil, err
 	}
 }

cmd/motion/main.go

@@ -13,7 +13,9 @@ var logger = log.Logger("motion/cmd")
 
 func main() {
 	// TODO: add flags, options and all that jazz
-	_ = log.SetLogLevel("*", "INFO")
+	if _, set := os.LookupEnv("GOLOG_LOG_LEVEL"); !set {
+		_ = log.SetLogLevel("*", "INFO")
+	}
 	m, err := motion.New()
 	if err != nil {
 		logger.Fatalw("Failed to instantiate Motion", "err", err)

go.mod

@@ -3,25 +3,14 @@ module github.com/filecoin-project/motion
 go 1.20
 
 require (
-	github.com/ipfs/go-cid v0.4.1
+	github.com/google/uuid v1.3.0
 	github.com/ipfs/go-log/v2 v2.5.1
-	github.com/multiformats/go-multihash v0.2.3
 )
 
 require (
-	github.com/klauspost/cpuid/v2 v2.0.9 // indirect
 	github.com/mattn/go-isatty v0.0.14 // indirect
-	github.com/minio/sha256-simd v1.0.0 // indirect
-	github.com/mr-tron/base58 v1.2.0 // indirect
-	github.com/multiformats/go-base32 v0.0.3 // indirect
-	github.com/multiformats/go-base36 v0.1.0 // indirect
-	github.com/multiformats/go-multibase v0.0.3 // indirect
-	github.com/multiformats/go-varint v0.0.6 // indirect
-	github.com/spaolacci/murmur3 v1.1.0 // indirect
 	go.uber.org/atomic v1.7.0 // indirect
 	go.uber.org/multierr v1.6.0 // indirect
 	go.uber.org/zap v1.19.1 // indirect
-	golang.org/x/crypto v0.1.0 // indirect
 	golang.org/x/sys v0.1.0 // indirect
-	lukechampine.com/blake3 v1.1.6 // indirect
 )