diff --git a/cmd/serv.go b/cmd/serv.go index 36064928444e3..bf6b765c9445a 100644 --- a/cmd/serv.go +++ b/cmd/serv.go @@ -22,6 +22,7 @@ import ( "code.gitea.io/gitea/models/perm" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/json" + "code.gitea.io/gitea/modules/lfstransfer" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/pprof" "code.gitea.io/gitea/modules/private" @@ -40,6 +41,7 @@ const ( verbUploadArchive = "git-upload-archive" verbReceivePack = "git-receive-pack" verbLfsAuthenticate = "git-lfs-authenticate" + verbLfsTransfer = "git-lfs-transfer" ) // CmdServ represents the available serv sub-command. @@ -83,9 +85,11 @@ var ( verbUploadArchive: true, verbReceivePack: true, verbLfsAuthenticate: true, + verbLfsTransfer: true, } allowedCommandsLfs = map[string]bool{ verbLfsAuthenticate: true, + verbLfsTransfer: true, } alphaDashDotPattern = regexp.MustCompile(`[^\w-\.]`) ) @@ -138,7 +142,7 @@ func getAccessMode(verb string, lfsVerb string) perm.AccessMode { return perm.AccessModeRead case verbReceivePack: return perm.AccessModeWrite - case verbLfsAuthenticate: + case verbLfsAuthenticate, verbLfsTransfer: switch lfsVerb { case "upload": return perm.AccessModeWrite @@ -276,6 +280,11 @@ func runServ(c *cli.Context) error { return fail(ctx, extra.UserMsg, "ServCommand failed: %s", extra.Error) } + // LFS SSH protocol + if verb == verbLfsTransfer { + return lfstransfer.Main(ctx, repoPath, lfsVerb) + } + // LFS token authentication if verb == verbLfsAuthenticate { url := fmt.Sprintf("%s%s/%s.git/info/lfs", setting.AppURL, url.PathEscape(results.OwnerName), url.PathEscape(results.RepoName)) diff --git a/modules/lfstransfer/backend/backend.go b/modules/lfstransfer/backend/backend.go new file mode 100644 index 0000000000000..fd90b3aae1540 --- /dev/null +++ b/modules/lfstransfer/backend/backend.go @@ -0,0 +1,177 @@ +// Copyright 2024 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package backend + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + + git_model "code.gitea.io/gitea/models/git" + repo_model "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/modules/lfs" + "code.gitea.io/gitea/modules/lfstransfer/transfer" +) + +// Version is the git-lfs-transfer protocol version number. +const Version = "1" + +// Capabilities is a list of Git LFS capabilities supported by this package. +var Capabilities = []string{ + "version=" + Version, + // "locking", // no support yet in gitea backend +} + +// GiteaBackend is an adapter between git-lfs-transfer library and Gitea's internal LFS API +type GiteaBackend struct { + ctx context.Context + repo *repo_model.Repository + store *lfs.ContentStore +} + +var _ transfer.Backend = &GiteaBackend{} + +// Batch implements transfer.Backend +func (g *GiteaBackend) Batch(_ string, pointers []transfer.BatchItem, _ transfer.Args) ([]transfer.BatchItem, error) { + for i := range pointers { + pointers[i].Present = false + pointer := lfs.Pointer{Oid: pointers[i].Oid, Size: pointers[i].Size} + exists, err := g.store.Verify(pointer) + if err != nil || !exists { + continue + } + accessible, err := g.repoHasAccess(pointers[i].Oid) + if err != nil || !accessible { + continue + } + pointers[i].Present = true + } + return pointers, nil +} + +// Download implements transfer.Backend. The returned reader must be closed by the +// caller. +func (g *GiteaBackend) Download(oid string, _ transfer.Args) (io.ReadCloser, int64, error) { + pointer := lfs.Pointer{Oid: oid} + pointer, err := g.store.GetMeta(pointer) + if errors.Is(err, lfs.ErrObjectNotInStore) { + return nil, 0, transfer.ErrNotFound + } + if err != nil { + return nil, 0, err + } + obj, err := g.store.Get(pointer) + if err != nil { + return nil, 0, err + } + accessible, err := g.repoHasAccess(oid) + if err != nil { + return nil, 0, err + } + if !accessible { + return nil, 0, transfer.ErrNotFound + } + return obj, pointer.Size, nil +} + +// StartUpload implements transfer.Backend. +func (g *GiteaBackend) Upload(oid string, size int64, r io.Reader, _ transfer.Args) error { + if r == nil { + return fmt.Errorf("%w: received null data", transfer.ErrMissingData) + } + pointer := lfs.Pointer{Oid: oid, Size: size} + exists, err := g.store.Verify(pointer) + if err != nil { + return err + } + if exists { + accessible, err := g.repoHasAccess(oid) + if err != nil { + return err + } + if accessible { + // we already have this object in the store and metadata + return nil + } + // we have this object in the store but not accessible + // so verify hash and size, and add it to metadata + hash := sha256.New() + written, err := io.Copy(hash, r) + if err != nil { + return fmt.Errorf("error hashing data: %w", err) + } + recvOid := hex.EncodeToString(hash.Sum(nil)) + if written != size { + return fmt.Errorf("%w: size mismatch: expected %v", transfer.ErrCorruptData, size) + } + if recvOid != oid { + return fmt.Errorf("%w: OID mismatch: expected %v", transfer.ErrCorruptData, oid) + } + } else { + err = g.store.Put(pointer, r) + if errors.Is(err, lfs.ErrSizeMismatch) { + return fmt.Errorf("%w: size mismatch: expected %v", transfer.ErrCorruptData, size) + } + if errors.Is(err, lfs.ErrHashMismatch) { + return fmt.Errorf("%w: OID mismatch: expected %v", transfer.ErrCorruptData, oid) + } + if err != nil { + return err + } + } + _, err = git_model.NewLFSMetaObject(g.ctx, g.repo.ID, pointer) + if err != nil { + return fmt.Errorf("could not create LFS Meta Object: %w", err) + } + return nil +} + +// Verify implements transfer.Backend. +func (g *GiteaBackend) Verify(oid string, size int64, args transfer.Args) (transfer.Status, error) { + pointer := lfs.Pointer{Oid: oid, Size: size} + exists, err := g.store.Verify(pointer) + if err != nil { + return transfer.NewStatus(transfer.StatusNotFound, "not found"), err + } + if !exists { + return transfer.NewStatus(transfer.StatusNotFound, "not found"), transfer.ErrNotFound + } + accessible, err := g.repoHasAccess(oid) + if err != nil { + return transfer.NewStatus(transfer.StatusNotFound, "not found"), err + } + if !accessible { + return transfer.NewStatus(transfer.StatusNotFound, "not found"), transfer.ErrNotFound + } + return transfer.SuccessStatus(), nil +} + +// LockBackend implements transfer.Backend. +func (g *GiteaBackend) LockBackend(_ transfer.Args) transfer.LockBackend { + // Gitea doesn't support the locking API + // this should never be called as we don't advertise the capability + return (transfer.LockBackend)(nil) +} + +// repoHasAccess checks if the repo already has the object with OID stored +func (g *GiteaBackend) repoHasAccess(oid string) (bool, error) { + // check if OID is in global LFS store + exists, err := g.store.Exists(lfs.Pointer{Oid: oid}) + if err != nil || !exists { + return false, err + } + // check if OID is in repo LFS store + metaObj, err := git_model.GetLFSMetaObjectByOid(g.ctx, g.repo.ID, oid) + if err != nil || metaObj == nil { + return false, err + } + return true, nil +} + +func New(ctx context.Context, r *repo_model.Repository, s *lfs.ContentStore) transfer.Backend { + return &GiteaBackend{ctx: ctx, repo: r, store: s} +} diff --git a/modules/lfstransfer/logger.go b/modules/lfstransfer/logger.go new file mode 100644 index 0000000000000..fd15019c8bd1c --- /dev/null +++ b/modules/lfstransfer/logger.go @@ -0,0 +1,21 @@ +// Copyright 2024 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package lfstransfer + +import ( + "code.gitea.io/gitea/modules/lfstransfer/transfer" +) + +// noop logger for passing into transfer +type GiteaLogger struct{} + +// Log implements transfer.Logger +func (g *GiteaLogger) Log(msg string, itms ...interface{}) { +} + +var _ transfer.Logger = (*GiteaLogger)(nil) + +func newLogger() transfer.Logger { + return &GiteaLogger{} +} diff --git a/modules/lfstransfer/main.go b/modules/lfstransfer/main.go new file mode 100644 index 0000000000000..6129085fe5094 --- /dev/null +++ b/modules/lfstransfer/main.go @@ -0,0 +1,73 @@ +// Copyright 2024 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package lfstransfer + +import ( + "context" + "fmt" + "os" + "strings" + + db_model "code.gitea.io/gitea/models/db" + repo_model "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/modules/lfs" + "code.gitea.io/gitea/modules/lfstransfer/backend" + "code.gitea.io/gitea/modules/lfstransfer/transfer" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/storage" +) + +func initServices(ctx context.Context) error { + setting.MustInstalled() + setting.LoadDBSetting() + setting.InitSQLLoggersForCli(log.INFO) + if err := db_model.InitEngine(ctx); err != nil { + return fmt.Errorf("unable to initialize the database using configuration [%q]: %w", setting.CustomConf, err) + } + if err := storage.Init(); err != nil { + return fmt.Errorf("unable to initialise storage: %v", err) + } + return nil +} + +func getRepo(ctx context.Context, path string) (*repo_model.Repository, error) { + // runServ ensures repoPath is [owner]/[name].git + pathSeg := strings.Split(path, "/") + pathSeg[1] = strings.TrimSuffix(pathSeg[1], ".git") + return repo_model.GetRepositoryByOwnerAndName(ctx, pathSeg[0], pathSeg[1]) +} + +func Main(ctx context.Context, repoPath string, verb string) error { + if err := initServices(ctx); err != nil { + return err + } + + logger := newLogger() + pktline := transfer.NewPktline(os.Stdin, os.Stdout, logger) + repo, err := getRepo(ctx, repoPath) + if err != nil { + return fmt.Errorf("unable to get repository: %s Error: %v", repoPath, err) + } + giteaBackend := backend.New(ctx, repo, lfs.NewContentStore()) + + for _, cap := range backend.Capabilities { + if err := pktline.WritePacketText(cap); err != nil { + log.Error("error sending capability [%v] due to error: %v", cap, err) + } + } + if err := pktline.WriteFlush(); err != nil { + log.Error("error flushing capabilities: %v", err) + } + p := transfer.NewProcessor(pktline, giteaBackend, logger) + defer log.Info("done processing commands") + switch verb { + case "upload": + return p.ProcessCommands(transfer.UploadOperation) + case "download": + return p.ProcessCommands(transfer.DownloadOperation) + default: + return fmt.Errorf("unknown operation %q", verb) + } +}