modules/lfstransfer: add a backend and runner

Also add handler in runServ()

The protocol lib supports locking but the backend does not,
as neither does Gitea. Support can be added later and the
capability advertised.

cmd/serv.go

@@ -22,6 +22,7 @@ import (
 	"code.gitea.io/gitea/models/perm"
 	"code.gitea.io/gitea/modules/git"
 	"code.gitea.io/gitea/modules/json"
+	"code.gitea.io/gitea/modules/lfstransfer"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/pprof"
 	"code.gitea.io/gitea/modules/private"
@@ -40,6 +41,7 @@ const (
 	verbUploadArchive   = "git-upload-archive"
 	verbReceivePack     = "git-receive-pack"
 	verbLfsAuthenticate = "git-lfs-authenticate"
+	verbLfsTransfer     = "git-lfs-transfer"
 )
 
 // CmdServ represents the available serv sub-command.
@@ -83,9 +85,11 @@ var (
 		verbUploadArchive:   true,
 		verbReceivePack:     true,
 		verbLfsAuthenticate: true,
+		verbLfsTransfer:     true,
 	}
 	allowedCommandsLfs = map[string]bool{
 		verbLfsAuthenticate: true,
+		verbLfsTransfer:     true,
 	}
 	alphaDashDotPattern = regexp.MustCompile(`[^\w-\.]`)
 )
@@ -138,7 +142,7 @@ func getAccessMode(verb string, lfsVerb string) perm.AccessMode {
 		return perm.AccessModeRead
 	case verbReceivePack:
 		return perm.AccessModeWrite
-	case verbLfsAuthenticate:
+	case verbLfsAuthenticate, verbLfsTransfer:
 		switch lfsVerb {
 		case "upload":
 			return perm.AccessModeWrite
@@ -276,6 +280,11 @@ func runServ(c *cli.Context) error {
 		return fail(ctx, extra.UserMsg, "ServCommand failed: %s", extra.Error)
 	}
 
+	// LFS SSH protocol
+	if verb == verbLfsTransfer {
+		return lfstransfer.Main(ctx, repoPath, lfsVerb)
+	}
+
 	// LFS token authentication
 	if verb == verbLfsAuthenticate {
 		url := fmt.Sprintf("%s%s/%s.git/info/lfs", setting.AppURL, url.PathEscape(results.OwnerName), url.PathEscape(results.RepoName))

modules/lfstransfer/backend/backend.go

@@ -0,0 +1,177 @@
+// Copyright 2024 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package backend
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"io"
+
+	git_model "code.gitea.io/gitea/models/git"
+	repo_model "code.gitea.io/gitea/models/repo"
+	"code.gitea.io/gitea/modules/lfs"
+	"code.gitea.io/gitea/modules/lfstransfer/transfer"
+)
+
+// Version is the git-lfs-transfer protocol version number.
+const Version = "1"
+
+// Capabilities is a list of Git LFS capabilities supported by this package.
+var Capabilities = []string{
+	"version=" + Version,
+	// "locking", // no support yet in gitea backend
+}
+
+// GiteaBackend is an adapter between git-lfs-transfer library and Gitea's internal LFS API
+type GiteaBackend struct {
+	ctx   context.Context
+	repo  *repo_model.Repository
+	store *lfs.ContentStore
+}
+
+var _ transfer.Backend = &GiteaBackend{}
+
+// Batch implements transfer.Backend
+func (g *GiteaBackend) Batch(_ string, pointers []transfer.BatchItem, _ transfer.Args) ([]transfer.BatchItem, error) {
+	for i := range pointers {
+		pointers[i].Present = false
+		pointer := lfs.Pointer{Oid: pointers[i].Oid, Size: pointers[i].Size}
+		exists, err := g.store.Verify(pointer)
+		if err != nil || !exists {
+			continue
+		}
+		accessible, err := g.repoHasAccess(pointers[i].Oid)
+		if err != nil || !accessible {
+			continue
+		}
+		pointers[i].Present = true
+	}
+	return pointers, nil
+}
+
+// Download implements transfer.Backend. The returned reader must be closed by the
+// caller.
+func (g *GiteaBackend) Download(oid string, _ transfer.Args) (io.ReadCloser, int64, error) {
+	pointer := lfs.Pointer{Oid: oid}
+	pointer, err := g.store.GetMeta(pointer)
+	if errors.Is(err, lfs.ErrObjectNotInStore) {
+		return nil, 0, transfer.ErrNotFound
+	}
+	if err != nil {
+		return nil, 0, err
+	}
+	obj, err := g.store.Get(pointer)
+	if err != nil {
+		return nil, 0, err
+	}
+	accessible, err := g.repoHasAccess(oid)
+	if err != nil {
+		return nil, 0, err
+	}
+	if !accessible {
+		return nil, 0, transfer.ErrNotFound
+	}
+	return obj, pointer.Size, nil
+}
+
+// StartUpload implements transfer.Backend.
+func (g *GiteaBackend) Upload(oid string, size int64, r io.Reader, _ transfer.Args) error {
+	if r == nil {
+		return fmt.Errorf("%w: received null data", transfer.ErrMissingData)
+	}
+	pointer := lfs.Pointer{Oid: oid, Size: size}
+	exists, err := g.store.Verify(pointer)
+	if err != nil {
+		return err
+	}
+	if exists {
+		accessible, err := g.repoHasAccess(oid)
+		if err != nil {
+			return err
+		}
+		if accessible {
+			// we already have this object in the store and metadata
+			return nil
+		}
+		// we have this object in the store but not accessible
+		// so verify hash and size, and add it to metadata
+		hash := sha256.New()
+		written, err := io.Copy(hash, r)
+		if err != nil {
+			return fmt.Errorf("error hashing data: %w", err)
+		}
+		recvOid := hex.EncodeToString(hash.Sum(nil))
+		if written != size {
+			return fmt.Errorf("%w: size mismatch: expected %v", transfer.ErrCorruptData, size)
+		}
+		if recvOid != oid {
+			return fmt.Errorf("%w: OID mismatch: expected %v", transfer.ErrCorruptData, oid)
+		}
+	} else {
+		err = g.store.Put(pointer, r)
+		if errors.Is(err, lfs.ErrSizeMismatch) {
+			return fmt.Errorf("%w: size mismatch: expected %v", transfer.ErrCorruptData, size)
+		}
+		if errors.Is(err, lfs.ErrHashMismatch) {
+			return fmt.Errorf("%w: OID mismatch: expected %v", transfer.ErrCorruptData, oid)
+		}
+		if err != nil {
+			return err
+		}
+	}
+	_, err = git_model.NewLFSMetaObject(g.ctx, g.repo.ID, pointer)
+	if err != nil {
+		return fmt.Errorf("could not create LFS Meta Object: %w", err)
+	}
+	return nil
+}
+
+// Verify implements transfer.Backend.
+func (g *GiteaBackend) Verify(oid string, size int64, args transfer.Args) (transfer.Status, error) {
+	pointer := lfs.Pointer{Oid: oid, Size: size}
+	exists, err := g.store.Verify(pointer)
+	if err != nil {
+		return transfer.NewStatus(transfer.StatusNotFound, "not found"), err
+	}
+	if !exists {
+		return transfer.NewStatus(transfer.StatusNotFound, "not found"), transfer.ErrNotFound
+	}
+	accessible, err := g.repoHasAccess(oid)
+	if err != nil {
+		return transfer.NewStatus(transfer.StatusNotFound, "not found"), err
+	}
+	if !accessible {
+		return transfer.NewStatus(transfer.StatusNotFound, "not found"), transfer.ErrNotFound
+	}
+	return transfer.SuccessStatus(), nil
+}
+
+// LockBackend implements transfer.Backend.
+func (g *GiteaBackend) LockBackend(_ transfer.Args) transfer.LockBackend {
+	// Gitea doesn't support the locking API
+	// this should never be called as we don't advertise the capability
+	return (transfer.LockBackend)(nil)
+}
+
+// repoHasAccess checks if the repo already has the object with OID stored
+func (g *GiteaBackend) repoHasAccess(oid string) (bool, error) {
+	// check if OID is in global LFS store
+	exists, err := g.store.Exists(lfs.Pointer{Oid: oid})
+	if err != nil || !exists {
+		return false, err
+	}
+	// check if OID is in repo LFS store
+	metaObj, err := git_model.GetLFSMetaObjectByOid(g.ctx, g.repo.ID, oid)
+	if err != nil || metaObj == nil {
+		return false, err
+	}
+	return true, nil
+}
+
+func New(ctx context.Context, r *repo_model.Repository, s *lfs.ContentStore) transfer.Backend {
+	return &GiteaBackend{ctx: ctx, repo: r, store: s}
+}

modules/lfstransfer/logger.go

@@ -0,0 +1,21 @@
+// Copyright 2024 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package lfstransfer
+
+import (
+	"code.gitea.io/gitea/modules/lfstransfer/transfer"
+)
+
+// noop logger for passing into transfer
+type GiteaLogger struct{}
+
+// Log implements transfer.Logger
+func (g *GiteaLogger) Log(msg string, itms ...interface{}) {
+}
+
+var _ transfer.Logger = (*GiteaLogger)(nil)
+
+func newLogger() transfer.Logger {
+	return &GiteaLogger{}
+}

modules/lfstransfer/main.go

@@ -0,0 +1,73 @@
+// Copyright 2024 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package lfstransfer
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"strings"
+
+	db_model "code.gitea.io/gitea/models/db"
+	repo_model "code.gitea.io/gitea/models/repo"
+	"code.gitea.io/gitea/modules/lfs"
+	"code.gitea.io/gitea/modules/lfstransfer/backend"
+	"code.gitea.io/gitea/modules/lfstransfer/transfer"
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/storage"
+)
+
+func initServices(ctx context.Context) error {
+	setting.MustInstalled()
+	setting.LoadDBSetting()
+	setting.InitSQLLoggersForCli(log.INFO)
+	if err := db_model.InitEngine(ctx); err != nil {
+		return fmt.Errorf("unable to initialize the database using configuration [%q]: %w", setting.CustomConf, err)
+	}
+	if err := storage.Init(); err != nil {
+		return fmt.Errorf("unable to initialise storage: %v", err)
+	}
+	return nil
+}
+
+func getRepo(ctx context.Context, path string) (*repo_model.Repository, error) {
+	// runServ ensures repoPath is [owner]/[name].git
+	pathSeg := strings.Split(path, "/")
+	pathSeg[1] = strings.TrimSuffix(pathSeg[1], ".git")
+	return repo_model.GetRepositoryByOwnerAndName(ctx, pathSeg[0], pathSeg[1])
+}
+
+func Main(ctx context.Context, repoPath string, verb string) error {
+	if err := initServices(ctx); err != nil {
+		return err
+	}
+
+	logger := newLogger()
+	pktline := transfer.NewPktline(os.Stdin, os.Stdout, logger)
+	repo, err := getRepo(ctx, repoPath)
+	if err != nil {
+		return fmt.Errorf("unable to get repository: %s Error: %v", repoPath, err)
+	}
+	giteaBackend := backend.New(ctx, repo, lfs.NewContentStore())
+
+	for _, cap := range backend.Capabilities {
+		if err := pktline.WritePacketText(cap); err != nil {
+			log.Error("error sending capability [%v] due to error: %v", cap, err)
+		}
+	}
+	if err := pktline.WriteFlush(); err != nil {
+		log.Error("error flushing capabilities: %v", err)
+	}
+	p := transfer.NewProcessor(pktline, giteaBackend, logger)
+	defer log.Info("done processing commands")
+	switch verb {
+	case "upload":
+		return p.ProcessCommands(transfer.UploadOperation)
+	case "download":
+		return p.ProcessCommands(transfer.DownloadOperation)
+	default:
+		return fmt.Errorf("unknown operation %q", verb)
+	}
+}