From fcb84f194b4a3d2013d2417f1294efbcb654e5e5 Mon Sep 17 00:00:00 2001 From: dragon Date: Thu, 19 Sep 2024 09:13:48 +0800 Subject: [PATCH 01/21] fix anything ... --- models/packages/descriptor.go | 3 + models/packages/package.go | 6 + modules/packages/arch/metadata.go | 316 ++++++++++++++ modules/packages/arch/metadata_test.go | 445 ++++++++++++++++++++ modules/setting/packages.go | 3 + options/locale/locale_en-US.ini | 16 + public/assets/img/svg/gitea-arch.svg | 1 + routers/api/packages/api.go | 12 + routers/api/packages/arch/arch.go | 248 +++++++++++ routers/web/user/package.go | 19 + services/packages/arch/repository.go | 353 ++++++++++++++++ services/packages/cleanup/cleanup.go | 5 + services/packages/packages.go | 2 + templates/package/content/arch.tmpl | 143 +++++++ templates/package/metadata/arch.tmpl | 4 + templates/package/view.tmpl | 2 + tests/integration/api_packages_arch_test.go | 327 ++++++++++++++ web_src/svg/gitea-arch.svg | 1 + 18 files changed, 1906 insertions(+) create mode 100644 modules/packages/arch/metadata.go create mode 100644 modules/packages/arch/metadata_test.go create mode 100644 public/assets/img/svg/gitea-arch.svg create mode 100644 routers/api/packages/arch/arch.go create mode 100644 services/packages/arch/repository.go create mode 100644 templates/package/content/arch.tmpl create mode 100644 templates/package/metadata/arch.tmpl create mode 100644 tests/integration/api_packages_arch_test.go create mode 100644 web_src/svg/gitea-arch.svg diff --git a/models/packages/descriptor.go b/models/packages/descriptor.go index b8ef698d3822..803b73c96899 100644 --- a/models/packages/descriptor.go +++ b/models/packages/descriptor.go @@ -13,6 +13,7 @@ import ( user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/json" "code.gitea.io/gitea/modules/packages/alpine" + "code.gitea.io/gitea/modules/packages/arch" "code.gitea.io/gitea/modules/packages/cargo" "code.gitea.io/gitea/modules/packages/chef" "code.gitea.io/gitea/modules/packages/composer" @@ -150,6 +151,8 @@ func GetPackageDescriptor(ctx context.Context, pv *PackageVersion) (*PackageDesc switch p.Type { case TypeAlpine: metadata = &alpine.VersionMetadata{} + case TypeArch: + metadata = &arch.VersionMetadata{} case TypeCargo: metadata = &cargo.Metadata{} case TypeChef: diff --git a/models/packages/package.go b/models/packages/package.go index 65a25741509e..417d62d19931 100644 --- a/models/packages/package.go +++ b/models/packages/package.go @@ -31,6 +31,7 @@ type Type string // List of supported packages const ( TypeAlpine Type = "alpine" + TypeArch Type = "arch" TypeCargo Type = "cargo" TypeChef Type = "chef" TypeComposer Type = "composer" @@ -55,6 +56,7 @@ const ( var TypeList = []Type{ TypeAlpine, + TypeArch, TypeCargo, TypeChef, TypeComposer, @@ -82,6 +84,8 @@ func (pt Type) Name() string { switch pt { case TypeAlpine: return "Alpine" + case TypeArch: + return "Arch" case TypeCargo: return "Cargo" case TypeChef: @@ -131,6 +135,8 @@ func (pt Type) SVGName() string { switch pt { case TypeAlpine: return "gitea-alpine" + case TypeArch: + return "gitea-arch" case TypeCargo: return "gitea-cargo" case TypeChef: diff --git a/modules/packages/arch/metadata.go b/modules/packages/arch/metadata.go new file mode 100644 index 000000000000..fc66d288cc5b --- /dev/null +++ b/modules/packages/arch/metadata.go @@ -0,0 +1,316 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package arch + +import ( + "bufio" + "bytes" + "encoding/hex" + "errors" + "fmt" + "io" + "regexp" + "strconv" + "strings" + + "code.gitea.io/gitea/modules/packages" + "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/modules/validation" + + "github.com/mholt/archiver/v3" +) + +// Arch Linux Packages +// https://man.archlinux.org/man/PKGBUILD.5 + +const ( + PropertyDescription = "arch.description" + PropertyArch = "arch.architecture" + PropertyDistribution = "arch.distribution" + + SettingKeyPrivate = "arch.key.private" + SettingKeyPublic = "arch.key.public" + + RepositoryPackage = "_arch" + RepositoryVersion = "_repository" +) + +var ( + reName = regexp.MustCompile(`^[a-zA-Z0-9@._+-]+$`) + reVer = regexp.MustCompile(`^[a-zA-Z0-9:_.+]+-+[0-9]+$`) + reOptDep = regexp.MustCompile(`^[a-zA-Z0-9@._+-]+$|^[a-zA-Z0-9@._+-]+(:.*)`) + rePkgVer = regexp.MustCompile(`^[a-zA-Z0-9@._+-]+$|^[a-zA-Z0-9@._+-]+(>.*)|^[a-zA-Z0-9@._+-]+(<.*)|^[a-zA-Z0-9@._+-]+(=.*)`) +) + +type Package struct { + Name string `json:"name"` + Version string `json:"version"` // Includes version, release and epoch + VersionMetadata VersionMetadata + FileMetadata FileMetadata +} + +// Arch package metadata related to specific version. +// Version metadata the same across different architectures and distributions. +type VersionMetadata struct { + Base string `json:"base"` + Description string `json:"description"` + ProjectURL string `json:"project_url"` + Groups []string `json:"groups,omitempty"` + Provides []string `json:"provides,omitempty"` + License []string `json:"license,omitempty"` + Depends []string `json:"depends,omitempty"` + OptDepends []string `json:"opt_depends,omitempty"` + MakeDepends []string `json:"make_depends,omitempty"` + CheckDepends []string `json:"check_depends,omitempty"` + Conflicts []string `json:"conflicts,omitempty"` + Replaces []string `json:"replaces,omitempty"` + Backup []string `json:"backup,omitempty"` + Xdata []string `json:"xdata,omitempty"` +} + +// FileMetadata Metadata related to specific package file. +// This metadata might vary for different architecture and distribution. +type FileMetadata struct { + CompressedSize int64 `json:"compressed_size"` + InstalledSize int64 `json:"installed_size"` + MD5 string `json:"md5"` + SHA256 string `json:"sha256"` + BuildDate int64 `json:"build_date"` + Packager string `json:"packager"` + Arch string `json:"arch"` + PgpSigned string `json:"pgp"` +} + +// ParsePackage Function that receives arch package archive data and returns it's metadata. +func ParsePackage(r *packages.HashedBuffer) (*Package, error) { + md5, _, sha256, _ := r.Sums() + _, err := r.Seek(0, io.SeekStart) + if err != nil { + return nil, err + } + zstd := archiver.NewTarZstd() + err = zstd.Open(r, 0) + if err != nil { + return nil, err + } + defer zstd.Close() + + var pkg *Package + var mtree bool + + for { + f, err := zstd.Read() + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + defer f.Close() + + switch f.Name() { + case ".PKGINFO": + pkg, err = ParsePackageInfo(f) + if err != nil { + return nil, err + } + case ".MTREE": + mtree = true + } + } + + if pkg == nil { + return nil, util.NewInvalidArgumentErrorf(".PKGINFO file not found") + } + + if !mtree { + return nil, util.NewInvalidArgumentErrorf(".MTREE file not found") + } + + pkg.FileMetadata.CompressedSize = r.Size() + pkg.FileMetadata.MD5 = hex.EncodeToString(md5) + pkg.FileMetadata.SHA256 = hex.EncodeToString(sha256) + + return pkg, nil +} + +// ParsePackageInfo Function that accepts reader for .PKGINFO file from package archive, +// validates all field according to PKGBUILD spec and returns package. +func ParsePackageInfo(r io.Reader) (*Package, error) { + p := &Package{} + + scanner := bufio.NewScanner(r) + for scanner.Scan() { + line := scanner.Text() + + if strings.HasPrefix(line, "#") { + continue + } + + key, value, find := strings.Cut(line, "=") + if !find { + continue + } + key = strings.TrimSpace(key) + value = strings.TrimSpace(value) + switch key { + case "pkgname": + p.Name = value + case "pkgbase": + p.VersionMetadata.Base = value + case "pkgver": + p.Version = value + case "pkgdesc": + p.VersionMetadata.Description = value + case "url": + p.VersionMetadata.ProjectURL = value + case "packager": + p.FileMetadata.Packager = value + case "arch": + p.FileMetadata.Arch = value + case "provides": + p.VersionMetadata.Provides = append(p.VersionMetadata.Provides, value) + case "license": + p.VersionMetadata.License = append(p.VersionMetadata.License, value) + case "depend": + p.VersionMetadata.Depends = append(p.VersionMetadata.Depends, value) + case "optdepend": + p.VersionMetadata.OptDepends = append(p.VersionMetadata.OptDepends, value) + case "makedepend": + p.VersionMetadata.MakeDepends = append(p.VersionMetadata.MakeDepends, value) + case "checkdepend": + p.VersionMetadata.CheckDepends = append(p.VersionMetadata.CheckDepends, value) + case "backup": + p.VersionMetadata.Backup = append(p.VersionMetadata.Backup, value) + case "group": + p.VersionMetadata.Groups = append(p.VersionMetadata.Groups, value) + case "conflict": + p.VersionMetadata.Conflicts = append(p.VersionMetadata.Conflicts, value) + case "replaces": + p.VersionMetadata.Replaces = append(p.VersionMetadata.Replaces, value) + case "xdata": + p.VersionMetadata.Xdata = append(p.VersionMetadata.Xdata, value) + case "builddate": + bd, err := strconv.ParseInt(value, 10, 64) + if err != nil { + return nil, err + } + p.FileMetadata.BuildDate = bd + case "size": + is, err := strconv.ParseInt(value, 10, 64) + if err != nil { + return nil, err + } + p.FileMetadata.InstalledSize = is + default: + return nil, util.NewInvalidArgumentErrorf("property is not supported %s", key) + } + } + + return p, errors.Join(scanner.Err(), ValidatePackageSpec(p)) +} + +// ValidatePackageSpec Arch package validation according to PKGBUILD specification. +func ValidatePackageSpec(p *Package) error { + if !reName.MatchString(p.Name) { + return util.NewInvalidArgumentErrorf("invalid package name") + } + if !reName.MatchString(p.VersionMetadata.Base) { + return util.NewInvalidArgumentErrorf("invalid package base") + } + if !reVer.MatchString(p.Version) { + return util.NewInvalidArgumentErrorf("invalid package version") + } + if p.FileMetadata.Arch == "" { + return util.NewInvalidArgumentErrorf("architecture should be specified") + } + if p.VersionMetadata.ProjectURL != "" { + if !validation.IsValidURL(p.VersionMetadata.ProjectURL) { + return util.NewInvalidArgumentErrorf("invalid project URL") + } + } + for _, cd := range p.VersionMetadata.CheckDepends { + if !rePkgVer.MatchString(cd) { + return util.NewInvalidArgumentErrorf("invalid check dependency: " + cd) + } + } + for _, d := range p.VersionMetadata.Depends { + if !rePkgVer.MatchString(d) { + return util.NewInvalidArgumentErrorf("invalid dependency: " + d) + } + } + for _, md := range p.VersionMetadata.MakeDepends { + if !rePkgVer.MatchString(md) { + return util.NewInvalidArgumentErrorf("invalid make dependency: " + md) + } + } + for _, p := range p.VersionMetadata.Provides { + if !rePkgVer.MatchString(p) { + return util.NewInvalidArgumentErrorf("invalid provides: " + p) + } + } + for _, p := range p.VersionMetadata.Conflicts { + if !rePkgVer.MatchString(p) { + return util.NewInvalidArgumentErrorf("invalid conflicts: " + p) + } + } + for _, p := range p.VersionMetadata.Replaces { + if !rePkgVer.MatchString(p) { + return util.NewInvalidArgumentErrorf("invalid replaces: " + p) + } + } + for _, p := range p.VersionMetadata.Replaces { + if !rePkgVer.MatchString(p) { + return util.NewInvalidArgumentErrorf("invalid xdata: " + p) + } + } + for _, od := range p.VersionMetadata.OptDepends { + if !reOptDep.MatchString(od) { + return util.NewInvalidArgumentErrorf("invalid optional dependency: " + od) + } + } + for _, bf := range p.VersionMetadata.Backup { + if strings.HasPrefix(bf, "/") { + return util.NewInvalidArgumentErrorf("backup file contains leading forward slash") + } + } + return nil +} + +// Desc Create pacman package description file. +func (p *Package) Desc() string { + entries := []string{ + "FILENAME", fmt.Sprintf("%s-%s-%s.pkg.tar.zst", p.Name, p.Version, p.FileMetadata.Arch), + "NAME", p.Name, + "BASE", p.VersionMetadata.Base, + "VERSION", p.Version, + "DESC", p.VersionMetadata.Description, + "GROUPS", strings.Join(p.VersionMetadata.Groups, "\n"), + "CSIZE", fmt.Sprintf("%d", p.FileMetadata.CompressedSize), + "ISIZE", fmt.Sprintf("%d", p.FileMetadata.InstalledSize), + "MD5SUM", p.FileMetadata.MD5, + "SHA256SUM", p.FileMetadata.SHA256, + "PGPSIG", p.FileMetadata.PgpSigned, + "URL", p.VersionMetadata.ProjectURL, + "LICENSE", strings.Join(p.VersionMetadata.License, "\n"), + "ARCH", p.FileMetadata.Arch, + "BUILDDATE", fmt.Sprintf("%d", p.FileMetadata.BuildDate), + "PACKAGER", p.FileMetadata.Packager, + "REPLACES", strings.Join(p.VersionMetadata.Replaces, "\n"), + "CONFLICTS", strings.Join(p.VersionMetadata.Conflicts, "\n"), + "PROVIDES", strings.Join(p.VersionMetadata.Provides, "\n"), + "DEPENDS", strings.Join(p.VersionMetadata.Depends, "\n"), + "OPTDEPENDS", strings.Join(p.VersionMetadata.OptDepends, "\n"), + "MAKEDEPENDS", strings.Join(p.VersionMetadata.MakeDepends, "\n"), + "CHECKDEPENDS", strings.Join(p.VersionMetadata.CheckDepends, "\n"), + } + + var buf bytes.Buffer + for i := 0; i < len(entries); i += 2 { + if entries[i+1] != "" { + _, _ = fmt.Fprintf(&buf, "%%%s%%\n%s\n\n", entries[i], entries[i+1]) + } + } + return buf.String() +} diff --git a/modules/packages/arch/metadata_test.go b/modules/packages/arch/metadata_test.go new file mode 100644 index 000000000000..b084762fb69a --- /dev/null +++ b/modules/packages/arch/metadata_test.go @@ -0,0 +1,445 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package arch + +import ( + "bytes" + "errors" + "os" + "strings" + "testing" + "testing/fstest" + "time" + + "code.gitea.io/gitea/modules/packages" + + "github.com/mholt/archiver/v3" + "github.com/stretchr/testify/require" +) + +func TestParsePackage(t *testing.T) { + // Minimal PKGINFO contents and test FS + const PKGINFO = `pkgname = a +pkgbase = b +pkgver = 1-2 +arch = x86_64 +` + fs := fstest.MapFS{ + "pkginfo": &fstest.MapFile{ + Data: []byte(PKGINFO), + Mode: os.ModePerm, + ModTime: time.Now(), + }, + "mtree": &fstest.MapFile{ + Data: []byte("data"), + Mode: os.ModePerm, + ModTime: time.Now(), + }, + } + + // Test .PKGINFO file + pinf, err := fs.Stat("pkginfo") + require.NoError(t, err) + + pfile, err := fs.Open("pkginfo") + require.NoError(t, err) + + parcname, err := archiver.NameInArchive(pinf, ".PKGINFO", ".PKGINFO") + require.NoError(t, err) + + // Test .MTREE file + minf, err := fs.Stat("mtree") + require.NoError(t, err) + + mfile, err := fs.Open("mtree") + require.NoError(t, err) + + marcname, err := archiver.NameInArchive(minf, ".MTREE", ".MTREE") + require.NoError(t, err) + + t.Run("normal archive", func(t *testing.T) { + var buf bytes.Buffer + + archive := archiver.NewTarZstd() + archive.Create(&buf) + + err = archive.Write(archiver.File{ + FileInfo: archiver.FileInfo{ + FileInfo: pinf, + CustomName: parcname, + }, + ReadCloser: pfile, + }) + require.NoError(t, errors.Join(pfile.Close(), err)) + + err = archive.Write(archiver.File{ + FileInfo: archiver.FileInfo{ + FileInfo: minf, + CustomName: marcname, + }, + ReadCloser: mfile, + }) + require.NoError(t, errors.Join(mfile.Close(), archive.Close(), err)) + + reader, err := packages.CreateHashedBufferFromReader(&buf) + if err != nil { + t.Fatal(err) + } + defer reader.Close() + _, err = ParsePackage(reader) + + require.NoError(t, err) + }) + + t.Run("missing .PKGINFO", func(t *testing.T) { + var buf bytes.Buffer + + archive := archiver.NewTarZstd() + archive.Create(&buf) + require.NoError(t, archive.Close()) + + reader, err := packages.CreateHashedBufferFromReader(&buf) + require.NoError(t, err) + + defer reader.Close() + _, err = ParsePackage(reader) + + require.Error(t, err) + require.Contains(t, err.Error(), ".PKGINFO file not found") + }) + + t.Run("missing .MTREE", func(t *testing.T) { + var buf bytes.Buffer + + pfile, err := fs.Open("pkginfo") + require.NoError(t, err) + + archive := archiver.NewTarZstd() + archive.Create(&buf) + + err = archive.Write(archiver.File{ + FileInfo: archiver.FileInfo{ + FileInfo: pinf, + CustomName: parcname, + }, + ReadCloser: pfile, + }) + require.NoError(t, errors.Join(pfile.Close(), archive.Close(), err)) + reader, err := packages.CreateHashedBufferFromReader(&buf) + require.NoError(t, err) + + defer reader.Close() + _, err = ParsePackage(reader) + + require.Error(t, err) + require.Contains(t, err.Error(), ".MTREE file not found") + }) +} + +func TestParsePackageInfo(t *testing.T) { + const PKGINFO = `# Generated by makepkg 6.0.2 +# using fakeroot version 1.31 +pkgname = a +pkgbase = b +pkgver = 1-2 +pkgdesc = comment +url = https://example.com/ +group = group +builddate = 3 +packager = Name Surname +size = 5 +arch = x86_64 +license = BSD +provides = pvd +depend = smth +optdepend = hex +checkdepend = ola +makedepend = cmake +backup = usr/bin/paket1 +` + p, err := ParsePackageInfo(strings.NewReader(PKGINFO)) + require.NoError(t, err) + require.Equal(t, Package{ + Name: "a", + Version: "1-2", + VersionMetadata: VersionMetadata{ + Base: "b", + Description: "comment", + ProjectURL: "https://example.com/", + Groups: []string{"group"}, + Provides: []string{"pvd"}, + License: []string{"BSD"}, + Depends: []string{"smth"}, + OptDepends: []string{"hex"}, + MakeDepends: []string{"cmake"}, + CheckDepends: []string{"ola"}, + Backup: []string{"usr/bin/paket1"}, + }, + FileMetadata: FileMetadata{ + InstalledSize: 5, + BuildDate: 3, + Packager: "Name Surname ", + Arch: "x86_64", + }, + }, *p) +} + +func TestValidatePackageSpec(t *testing.T) { + newpkg := func() Package { + return Package{ + Name: "abc", + Version: "1-1", + VersionMetadata: VersionMetadata{ + Base: "ghx", + Description: "whoami", + ProjectURL: "https://example.com/", + Groups: []string{"gnome"}, + Provides: []string{"abc", "def"}, + License: []string{"GPL"}, + Depends: []string{"go", "gpg=1", "curl>=3", "git<=7"}, + OptDepends: []string{"git: something", "make"}, + MakeDepends: []string{"chrom"}, + CheckDepends: []string{"bariy"}, + Backup: []string{"etc/pacman.d/filo"}, + }, + FileMetadata: FileMetadata{ + CompressedSize: 1, + InstalledSize: 2, + SHA256: "def", + BuildDate: 3, + Packager: "smon", + Arch: "x86_64", + }, + } + } + + t.Run("valid package", func(t *testing.T) { + p := newpkg() + + err := ValidatePackageSpec(&p) + + require.NoError(t, err) + }) + + t.Run("invalid package name", func(t *testing.T) { + p := newpkg() + p.Name = "!$%@^!*&()" + + err := ValidatePackageSpec(&p) + + require.Error(t, err) + require.Contains(t, err.Error(), "invalid package name") + }) + + t.Run("invalid package base", func(t *testing.T) { + p := newpkg() + p.VersionMetadata.Base = "!$%@^!*&()" + + err := ValidatePackageSpec(&p) + + require.Error(t, err) + require.Contains(t, err.Error(), "invalid package base") + }) + + t.Run("invalid package version", func(t *testing.T) { + p := newpkg() + p.VersionMetadata.Base = "una-luna?" + + err := ValidatePackageSpec(&p) + + require.Error(t, err) + require.Contains(t, err.Error(), "invalid package base") + }) + + t.Run("invalid package version", func(t *testing.T) { + p := newpkg() + p.Version = "una-luna" + + err := ValidatePackageSpec(&p) + + require.Error(t, err) + require.Contains(t, err.Error(), "invalid package version") + }) + + t.Run("missing architecture", func(t *testing.T) { + p := newpkg() + p.FileMetadata.Arch = "" + + err := ValidatePackageSpec(&p) + + require.Error(t, err) + require.Contains(t, err.Error(), "architecture should be specified") + }) + + t.Run("invalid URL", func(t *testing.T) { + p := newpkg() + p.VersionMetadata.ProjectURL = "http%%$#" + + err := ValidatePackageSpec(&p) + + require.Error(t, err) + require.Contains(t, err.Error(), "invalid project URL") + }) + + t.Run("invalid check dependency", func(t *testing.T) { + p := newpkg() + p.VersionMetadata.CheckDepends = []string{"Err^_^"} + + err := ValidatePackageSpec(&p) + + require.Error(t, err) + require.Contains(t, err.Error(), "invalid check dependency") + }) + + t.Run("invalid dependency", func(t *testing.T) { + p := newpkg() + p.VersionMetadata.Depends = []string{"^^abc"} + + err := ValidatePackageSpec(&p) + + require.Error(t, err) + require.Contains(t, err.Error(), "invalid dependency") + }) + + t.Run("invalid make dependency", func(t *testing.T) { + p := newpkg() + p.VersionMetadata.MakeDepends = []string{"^m^"} + + err := ValidatePackageSpec(&p) + + require.Error(t, err) + require.Contains(t, err.Error(), "invalid make dependency") + }) + + t.Run("invalid provides", func(t *testing.T) { + p := newpkg() + p.VersionMetadata.Provides = []string{"^m^"} + + err := ValidatePackageSpec(&p) + + require.Error(t, err) + require.Contains(t, err.Error(), "invalid provides") + }) + + t.Run("invalid optional dependency", func(t *testing.T) { + p := newpkg() + p.VersionMetadata.OptDepends = []string{"^m^:MM"} + + err := ValidatePackageSpec(&p) + + require.Error(t, err) + require.Contains(t, err.Error(), "invalid optional dependency") + }) + + t.Run("invalid optional dependency", func(t *testing.T) { + p := newpkg() + p.VersionMetadata.Backup = []string{"/ola/cola"} + + err := ValidatePackageSpec(&p) + + require.Error(t, err) + require.Contains(t, err.Error(), "backup file contains leading forward slash") + }) +} + +func TestDescString(t *testing.T) { + const pkgdesc = `%FILENAME% +zstd-1.5.5-1-x86_64.pkg.tar.zst + +%NAME% +zstd + +%BASE% +zstd + +%VERSION% +1.5.5-1 + +%DESC% +Zstandard - Fast real-time compression algorithm + +%GROUPS% +dummy1 +dummy2 + +%CSIZE% +401 + +%ISIZE% +1500453 + +%MD5SUM% +5016660ef3d9aa148a7b72a08d3df1b2 + +%SHA256SUM% +9fa4ede47e35f5971e4f26ecadcbfb66ab79f1d638317ac80334a3362dedbabd + +%URL% +https://facebook.github.io/zstd/ + +%LICENSE% +BSD +GPL2 + +%ARCH% +x86_64 + +%BUILDDATE% +1681646714 + +%PACKAGER% +Jelle van der Waa + +%PROVIDES% +libzstd.so=1-64 + +%DEPENDS% +glibc +gcc-libs +zlib +xz +lz4 + +%OPTDEPENDS% +dummy3 +dummy4 + +%MAKEDEPENDS% +cmake +gtest +ninja + +%CHECKDEPENDS% +dummy5 +dummy6 + +` + + md := &Package{ + Name: "zstd", + Version: "1.5.5-1", + VersionMetadata: VersionMetadata{ + Base: "zstd", + Description: "Zstandard - Fast real-time compression algorithm", + ProjectURL: "https://facebook.github.io/zstd/", + Groups: []string{"dummy1", "dummy2"}, + Provides: []string{"libzstd.so=1-64"}, + License: []string{"BSD", "GPL2"}, + Depends: []string{"glibc", "gcc-libs", "zlib", "xz", "lz4"}, + OptDepends: []string{"dummy3", "dummy4"}, + MakeDepends: []string{"cmake", "gtest", "ninja"}, + CheckDepends: []string{"dummy5", "dummy6"}, + }, + FileMetadata: FileMetadata{ + CompressedSize: 401, + InstalledSize: 1500453, + MD5: "5016660ef3d9aa148a7b72a08d3df1b2", + SHA256: "9fa4ede47e35f5971e4f26ecadcbfb66ab79f1d638317ac80334a3362dedbabd", + BuildDate: 1681646714, + Packager: "Jelle van der Waa ", + Arch: "x86_64", + }, + } + require.Equal(t, pkgdesc, md.Desc()) +} diff --git a/modules/setting/packages.go b/modules/setting/packages.go index bc093e7ea6c1..b5f08edf0ce1 100644 --- a/modules/setting/packages.go +++ b/modules/setting/packages.go @@ -22,6 +22,7 @@ var ( LimitTotalOwnerCount int64 LimitTotalOwnerSize int64 LimitSizeAlpine int64 + LimitSizeArch int64 LimitSizeCargo int64 LimitSizeChef int64 LimitSizeComposer int64 @@ -79,6 +80,8 @@ func loadPackagesFrom(rootCfg ConfigProvider) (err error) { Packages.LimitTotalOwnerSize = mustBytes(sec, "LIMIT_TOTAL_OWNER_SIZE") Packages.LimitSizeAlpine = mustBytes(sec, "LIMIT_SIZE_ALPINE") + Packages.LimitSizeArch = mustBytes(sec, "LIMIT_SIZE_ARCH") + Packages.LimitSizeArch = mustBytes(sec, "LIMIT_SIZE_ARCH") Packages.LimitSizeCargo = mustBytes(sec, "LIMIT_SIZE_CARGO") Packages.LimitSizeChef = mustBytes(sec, "LIMIT_SIZE_CHEF") Packages.LimitSizeComposer = mustBytes(sec, "LIMIT_SIZE_COMPOSER") diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 957e73b171d3..d8fc1809d3a8 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -3509,6 +3509,22 @@ alpine.repository = Repository Info alpine.repository.branches = Branches alpine.repository.repositories = Repositories alpine.repository.architectures = Architectures +arch.pacman.helper.gpg = Add trust certificate for pacman: +arch.pacman.repo.multi = %s has the same version in different distributions. +arch.pacman.repo.multi.item = Configuration for %s +arch.pacman.conf = Add server with related distribution and architecture to /etc/pacman.conf : +arch.pacman.sync = Sync package with pacman: +arch.version.properties = Version Properties +arch.version.description = Description +arch.version.provides = Provides +arch.version.groups = Group +arch.version.depends = Depends +arch.version.optdepends = Optional depends +arch.version.makedepends = Make depends +arch.version.checkdepends = Check depends +arch.version.conflicts = Conflicts +arch.version.replaces = Replaces +arch.version.backup = Backup cargo.registry = Setup this registry in the Cargo configuration file (for example ~/.cargo/config.toml): cargo.install = To install the package using Cargo, run the following command: chef.registry = Setup this registry in your ~/.chef/config.rb file: diff --git a/public/assets/img/svg/gitea-arch.svg b/public/assets/img/svg/gitea-arch.svg new file mode 100644 index 000000000000..943a92c57946 --- /dev/null +++ b/public/assets/img/svg/gitea-arch.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/routers/api/packages/api.go b/routers/api/packages/api.go index 0f42e8f59ebb..12077753b333 100644 --- a/routers/api/packages/api.go +++ b/routers/api/packages/api.go @@ -14,6 +14,7 @@ import ( "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/web" "code.gitea.io/gitea/routers/api/packages/alpine" + "code.gitea.io/gitea/routers/api/packages/arch" "code.gitea.io/gitea/routers/api/packages/cargo" "code.gitea.io/gitea/routers/api/packages/chef" "code.gitea.io/gitea/routers/api/packages/composer" @@ -121,6 +122,17 @@ func CommonRoutes() *web.Router { }) }) }, reqPackageAccess(perm.AccessModeRead)) + r.Group("/arch", func() { + r.Group("/repository.key", func() { + r.Head("", arch.GetRepositoryKey) + r.Get("", arch.GetRepositoryKey) + }) + r.Group("/{distro}", func() { + r.Put("", reqPackageAccess(perm.AccessModeWrite), arch.PushPackage) + r.Get("/{arch}/{file}", arch.GetPackageOrDB) + r.Delete("/{package}/{version}", reqPackageAccess(perm.AccessModeWrite), arch.RemovePackage) + }) + }, reqPackageAccess(perm.AccessModeRead)) r.Group("/cargo", func() { r.Group("/api/v1/crates", func() { r.Get("", cargo.SearchPackages) diff --git a/routers/api/packages/arch/arch.go b/routers/api/packages/arch/arch.go new file mode 100644 index 000000000000..22cad2a918be --- /dev/null +++ b/routers/api/packages/arch/arch.go @@ -0,0 +1,248 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package arch + +import ( + "encoding/base64" + "errors" + "fmt" + "io" + "net/http" + "strings" + + packages_model "code.gitea.io/gitea/models/packages" + packages_module "code.gitea.io/gitea/modules/packages" + arch_module "code.gitea.io/gitea/modules/packages/arch" + "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/routers/api/packages/helper" + "code.gitea.io/gitea/services/context" + packages_service "code.gitea.io/gitea/services/packages" + arch_service "code.gitea.io/gitea/services/packages/arch" +) + +func apiError(ctx *context.Context, status int, obj any) { + helper.LogAndProcessError(ctx, status, obj, func(message string) { + ctx.PlainText(status, message) + }) +} + +func GetRepositoryKey(ctx *context.Context) { + _, pub, err := arch_service.GetOrCreateKeyPair(ctx, ctx.Package.Owner.ID) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + ctx.ServeContent(strings.NewReader(pub), &context.ServeHeaderOptions{ + ContentType: "application/pgp-keys", + Filename: "repository.key", + }) +} + +func PushPackage(ctx *context.Context) { + distro := ctx.PathParam("distro") + + upload, needToClose, err := ctx.UploadStream() + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + if needToClose { + defer upload.Close() + } + + buf, err := packages_module.CreateHashedBufferFromReader(upload) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + defer buf.Close() + + p, err := arch_module.ParsePackage(buf) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + _, err = buf.Seek(0, io.SeekStart) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + sign, err := arch_service.NewFileSign(ctx, ctx.Package.Owner.ID, buf) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + defer sign.Close() + _, err = buf.Seek(0, io.SeekStart) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + // update gpg sign + pgp, err := io.ReadAll(sign) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + p.FileMetadata.PgpSigned = base64.StdEncoding.EncodeToString(pgp) + _, err = sign.Seek(0, io.SeekStart) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + properties := map[string]string{ + arch_module.PropertyDescription: p.Desc(), + arch_module.PropertyArch: p.FileMetadata.Arch, + arch_module.PropertyDistribution: distro, + } + + version, _, err := packages_service.CreatePackageOrAddFileToExisting( + ctx, + &packages_service.PackageCreationInfo{ + PackageInfo: packages_service.PackageInfo{ + Owner: ctx.Package.Owner, + PackageType: packages_model.TypeArch, + Name: p.Name, + Version: p.Version, + }, + Creator: ctx.Doer, + Metadata: p.VersionMetadata, + }, + &packages_service.PackageFileCreationInfo{ + PackageFileInfo: packages_service.PackageFileInfo{ + Filename: fmt.Sprintf("%s-%s-%s.pkg.tar.zst", p.Name, p.Version, p.FileMetadata.Arch), + CompositeKey: distro, + }, + OverwriteExisting: false, + IsLead: true, + Creator: ctx.ContextUser, + Data: buf, + Properties: properties, + }, + ) + if err != nil { + switch { + case errors.Is(err, packages_model.ErrDuplicatePackageVersion), errors.Is(err, packages_model.ErrDuplicatePackageFile): + apiError(ctx, http.StatusConflict, err) + case errors.Is(err, packages_service.ErrQuotaTotalCount), errors.Is(err, packages_service.ErrQuotaTypeSize), errors.Is(err, packages_service.ErrQuotaTotalSize): + apiError(ctx, http.StatusForbidden, err) + default: + apiError(ctx, http.StatusInternalServerError, err) + } + return + } + // add sign file + _, err = packages_service.AddFileToPackageVersionInternal(ctx, version, &packages_service.PackageFileCreationInfo{ + PackageFileInfo: packages_service.PackageFileInfo{ + CompositeKey: distro, + Filename: fmt.Sprintf("%s-%s-%s.pkg.tar.zst.sig", p.Name, p.Version, p.FileMetadata.Arch), + }, + OverwriteExisting: true, + IsLead: false, + Creator: ctx.Doer, + Data: sign, + }) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + } + if err = arch_service.BuildPacmanDB(ctx, ctx.Package.Owner.ID, distro, p.FileMetadata.Arch); err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + ctx.Status(http.StatusCreated) +} + +func GetPackageOrDB(ctx *context.Context) { + var ( + file = ctx.PathParam("file") + distro = ctx.PathParam("distro") + arch = ctx.PathParam("arch") + ) + + if strings.HasSuffix(file, ".pkg.tar.zst") || strings.HasSuffix(file, ".pkg.tar.zst.sig") { + pkg, err := arch_service.GetPackageFile(ctx, distro, file, ctx.Package.Owner.ID) + if err != nil { + if errors.Is(err, util.ErrNotExist) { + apiError(ctx, http.StatusNotFound, err) + } else { + apiError(ctx, http.StatusInternalServerError, err) + } + return + } + + ctx.ServeContent(pkg, &context.ServeHeaderOptions{ + Filename: file, + }) + return + } + + if strings.HasSuffix(file, ".db.tar.gz") || + strings.HasSuffix(file, ".db") || + strings.HasSuffix(file, ".db.tar.gz.sig") || + strings.HasSuffix(file, ".db.sig") { + pkg, err := arch_service.GetPackageDBFile(ctx, distro, arch, ctx.Package.Owner.ID, + strings.HasSuffix(file, ".sig")) + if err != nil { + if errors.Is(err, util.ErrNotExist) { + apiError(ctx, http.StatusNotFound, err) + } else { + apiError(ctx, http.StatusInternalServerError, err) + } + return + } + ctx.ServeContent(pkg, &context.ServeHeaderOptions{ + Filename: file, + }) + return + } + + ctx.Status(http.StatusNotFound) +} + +func RemovePackage(ctx *context.Context) { + var ( + distro = ctx.PathParam("distro") + pkg = ctx.PathParam("package") + ver = ctx.PathParam("version") + ) + pv, err := packages_model.GetVersionByNameAndVersion( + ctx, ctx.Package.Owner.ID, packages_model.TypeArch, pkg, ver, + ) + if err != nil { + if errors.Is(err, util.ErrNotExist) { + apiError(ctx, http.StatusNotFound, err) + } else { + apiError(ctx, http.StatusInternalServerError, err) + } + return + } + files, err := packages_model.GetFilesByVersionID(ctx, pv.ID) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + deleted := false + for _, file := range files { + if file.CompositeKey == distro { + deleted = true + err := packages_service.RemovePackageFileAndVersionIfUnreferenced(ctx, ctx.ContextUser, file) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + } + } + if deleted { + err = arch_service.BuildCustomRepositoryFiles(ctx, ctx.Package.Owner.ID, distro) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + } + ctx.Status(http.StatusNoContent) + } else { + ctx.Error(http.StatusNotFound) + } +} diff --git a/routers/web/user/package.go b/routers/web/user/package.go index 4e39eabc0ffd..6289e85849e1 100644 --- a/routers/web/user/package.go +++ b/routers/web/user/package.go @@ -4,6 +4,7 @@ package user import ( + "fmt" "net/http" "net/url" @@ -20,6 +21,7 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/optional" alpine_module "code.gitea.io/gitea/modules/packages/alpine" + arch_model "code.gitea.io/gitea/modules/packages/arch" debian_module "code.gitea.io/gitea/modules/packages/debian" rpm_module "code.gitea.io/gitea/modules/packages/rpm" "code.gitea.io/gitea/modules/setting" @@ -206,6 +208,23 @@ func ViewPackageVersion(ctx *context.Context) { ctx.Data["Branches"] = util.Sorted(branches.Values()) ctx.Data["Repositories"] = util.Sorted(repositories.Values()) ctx.Data["Architectures"] = util.Sorted(architectures.Values()) + case packages_model.TypeArch: + registryAppURL, err := url.Parse(httplib.GuessCurrentAppURL(ctx)) + if err != nil { + registryAppURL, _ = url.Parse(setting.AppURL) + } + ctx.Data["RegistryHost"] = registryAppURL.Host + ctx.Data["SignMail"] = fmt.Sprintf("%s@noreply.%s", ctx.Package.Owner.Name, registryAppURL.Host) + groups := make(container.Set[string]) + for _, f := range pd.Files { + for _, pp := range f.Properties { + switch pp.Name { + case arch_model.PropertyDistribution: + groups.Add(pp.Value) + } + } + } + ctx.Data["Groups"] = util.Sorted(groups.Values()) case packages_model.TypeDebian: distributions := make(container.Set[string]) components := make(container.Set[string]) diff --git a/services/packages/arch/repository.go b/services/packages/arch/repository.go new file mode 100644 index 000000000000..41c2c7954147 --- /dev/null +++ b/services/packages/arch/repository.go @@ -0,0 +1,353 @@ +// Copyright 2024 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package arch + +import ( + "archive/tar" + "compress/gzip" + "context" + "errors" + "fmt" + "io" + "net/url" + "os" + "sort" + "strings" + + packages_model "code.gitea.io/gitea/models/packages" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/httplib" + packages_module "code.gitea.io/gitea/modules/packages" + arch_module "code.gitea.io/gitea/modules/packages/arch" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/util" + packages_service "code.gitea.io/gitea/services/packages" + + "github.com/ProtonMail/go-crypto/openpgp" + "github.com/ProtonMail/go-crypto/openpgp/armor" + "github.com/ProtonMail/go-crypto/openpgp/packet" +) + +func GetOrCreateRepositoryVersion(ctx context.Context, ownerID int64) (*packages_model.PackageVersion, error) { + return packages_service.GetOrCreateInternalPackageVersion(ctx, ownerID, packages_model.TypeArch, arch_module.RepositoryPackage, arch_module.RepositoryVersion) +} + +func BuildAllRepositoryFiles(ctx context.Context, ownerID int64) error { + pv, err := GetOrCreateRepositoryVersion(ctx, ownerID) + if err != nil { + return err + } + // remove old db files + pfs, err := packages_model.GetFilesByVersionID(ctx, pv.ID) + if err != nil { + return err + } + for _, pf := range pfs { + if strings.HasSuffix(pf.Name, ".db") { + arch := strings.TrimSuffix(strings.TrimPrefix(pf.Name, fmt.Sprintf("%s-", pf.CompositeKey)), ".db") + if err := BuildPacmanDB(ctx, ownerID, pf.CompositeKey, arch); err != nil { + return err + } + } + } + return nil +} + +func BuildCustomRepositoryFiles(ctx context.Context, ownerID int64, disco string) error { + pv, err := GetOrCreateRepositoryVersion(ctx, ownerID) + if err != nil { + return err + } + // remove old db files + pfs, err := packages_model.GetFilesByVersionID(ctx, pv.ID) + if err != nil { + return err + } + for _, pf := range pfs { + if strings.HasSuffix(pf.Name, ".db") && pf.CompositeKey == disco { + arch := strings.TrimSuffix(strings.TrimPrefix(pf.Name, fmt.Sprintf("%s-", pf.CompositeKey)), ".db") + if err := BuildPacmanDB(ctx, ownerID, pf.CompositeKey, arch); err != nil { + return err + } + } + } + return nil +} + +func NewFileSign(ctx context.Context, ownerID int64, input io.Reader) (*packages_module.HashedBuffer, error) { + // If no signature is specified, it will be generated by Gitea. + priv, _, err := GetOrCreateKeyPair(ctx, ownerID) + if err != nil { + return nil, err + } + block, err := armor.Decode(strings.NewReader(priv)) + if err != nil { + return nil, err + } + e, err := openpgp.ReadEntity(packet.NewReader(block.Body)) + if err != nil { + return nil, err + } + pkgSig, err := packages_module.NewHashedBuffer() + if err != nil { + return nil, err + } + defer pkgSig.Close() + if err := openpgp.DetachSign(pkgSig, e, input, nil); err != nil { + return nil, err + } + return pkgSig, nil +} + +// BuildPacmanDB Create db signature cache +func BuildPacmanDB(ctx context.Context, ownerID int64, distro, arch string) error { + pv, err := GetOrCreateRepositoryVersion(ctx, ownerID) + if err != nil { + return err + } + // remove old db files + pfs, err := packages_model.GetFilesByVersionID(ctx, pv.ID) + if err != nil { + return err + } + for _, pf := range pfs { + if pf.CompositeKey == distro && strings.HasPrefix(pf.Name, fmt.Sprintf("%s-%s", distro, arch)) { + // remove distro and arch + if err := packages_service.DeletePackageFile(ctx, pf); err != nil { + return err + } + } + } + + db, err := flushDB(ctx, ownerID, distro, arch) + if errors.Is(err, io.EOF) { + return nil + } else if err != nil { + return err + } + defer db.Close() + // Create db signature cache + _, err = db.Seek(0, io.SeekStart) + if err != nil { + return err + } + sig, err := NewFileSign(ctx, ownerID, db) + if err != nil { + return err + } + defer sig.Close() + _, err = db.Seek(0, io.SeekStart) + if err != nil { + return err + } + for name, data := range map[string]*packages_module.HashedBuffer{ + fmt.Sprintf("%s-%s.db", distro, arch): db, + fmt.Sprintf("%s-%s.db.sig", distro, arch): sig, + } { + _, err = packages_service.AddFileToPackageVersionInternal(ctx, pv, &packages_service.PackageFileCreationInfo{ + PackageFileInfo: packages_service.PackageFileInfo{ + Filename: name, + CompositeKey: distro, + }, + Creator: user_model.NewGhostUser(), + Data: data, + IsLead: false, + OverwriteExisting: true, + }) + if err != nil { + return err + } + } + return nil +} + +func flushDB(ctx context.Context, ownerID int64, distro, arch string) (*packages_module.HashedBuffer, error) { + pkgs, err := packages_model.GetPackagesByType(ctx, ownerID, packages_model.TypeArch) + if err != nil { + return nil, err + } + if len(pkgs) == 0 { + return nil, io.EOF + } + db, err := packages_module.NewHashedBuffer() + if err != nil { + return nil, err + } + gw := gzip.NewWriter(db) + tw := tar.NewWriter(gw) + count := 0 + for _, pkg := range pkgs { + versions, err := packages_model.GetVersionsByPackageName( + ctx, ownerID, packages_model.TypeArch, pkg.Name, + ) + if err != nil { + return nil, errors.Join(tw.Close(), gw.Close(), db.Close(), err) + } + sort.Slice(versions, func(i, j int) bool { + return versions[i].CreatedUnix > versions[j].CreatedUnix + }) + for _, ver := range versions { + file := fmt.Sprintf("%s-%s-%s.pkg.tar.zst", pkg.Name, ver.Version, arch) + pf, err := packages_model.GetFileForVersionByName(ctx, ver.ID, file, distro) + if err != nil { + // add any arch package + file = fmt.Sprintf("%s-%s-any.pkg.tar.zst", pkg.Name, ver.Version) + pf, err = packages_model.GetFileForVersionByName(ctx, ver.ID, file, distro) + if err != nil { + continue + } + } + pps, err := packages_model.GetPropertiesByName( + ctx, packages_model.PropertyTypeFile, pf.ID, arch_module.PropertyDescription, + ) + if err != nil { + return nil, errors.Join(tw.Close(), gw.Close(), db.Close(), err) + } + if len(pps) >= 1 { + meta := []byte(pps[0].Value) + header := &tar.Header{ + Name: pkg.Name + "-" + ver.Version + "/desc", + Size: int64(len(meta)), + Mode: int64(os.ModePerm), + } + if err = tw.WriteHeader(header); err != nil { + return nil, errors.Join(tw.Close(), gw.Close(), db.Close(), err) + } + if _, err := tw.Write(meta); err != nil { + return nil, errors.Join(tw.Close(), gw.Close(), db.Close(), err) + } + count++ + break + } + } + } + defer gw.Close() + defer tw.Close() + if count == 0 { + return nil, errors.Join(db.Close(), io.EOF) + } + return db, nil +} + +// GetPackageFile Get data related to provided filename and distribution, for package files +// update download counter. +func GetPackageFile(ctx context.Context, distro, file string, ownerID int64) (io.ReadSeekCloser, error) { + pf, err := getPackageFile(ctx, distro, file, ownerID) + if err != nil { + return nil, err + } + + filestream, _, _, err := packages_service.GetPackageFileStream(ctx, pf) + return filestream, err +} + +// Ejects parameters required to get package file property from file name. +func getPackageFile(ctx context.Context, distro, file string, ownerID int64) (*packages_model.PackageFile, error) { + var ( + splt = strings.Split(file, "-") + pkgname = strings.Join(splt[0:len(splt)-3], "-") + vername = splt[len(splt)-3] + "-" + splt[len(splt)-2] + ) + + version, err := packages_model.GetVersionByNameAndVersion(ctx, ownerID, packages_model.TypeArch, pkgname, vername) + if err != nil { + return nil, err + } + + pkgfile, err := packages_model.GetFileForVersionByName(ctx, version.ID, file, distro) + if err != nil { + return nil, err + } + return pkgfile, nil +} + +func GetPackageDBFile(ctx context.Context, distro, arch string, ownerID int64, signFile bool) (io.ReadSeekCloser, error) { + pv, err := GetOrCreateRepositoryVersion(ctx, ownerID) + if err != nil { + return nil, err + } + fileName := fmt.Sprintf("%s-%s.db", distro, arch) + if signFile { + fileName = fmt.Sprintf("%s-%s.db.sig", distro, arch) + } + file, err := packages_model.GetFileForVersionByName(ctx, pv.ID, fileName, distro) + if err != nil { + return nil, err + } + filestream, _, _, err := packages_service.GetPackageFileStream(ctx, file) + return filestream, err +} + +// GetOrCreateKeyPair gets or creates the PGP keys used to sign repository metadata files +func GetOrCreateKeyPair(ctx context.Context, ownerID int64) (string, string, error) { + priv, err := user_model.GetSetting(ctx, ownerID, arch_module.SettingKeyPrivate) + if err != nil && !errors.Is(err, util.ErrNotExist) { + return "", "", err + } + + pub, err := user_model.GetSetting(ctx, ownerID, arch_module.SettingKeyPublic) + if err != nil && !errors.Is(err, util.ErrNotExist) { + return "", "", err + } + + if priv == "" || pub == "" { + user, err := user_model.GetUserByID(ctx, ownerID) + if err != nil && !errors.Is(err, util.ErrNotExist) { + return "", "", err + } + registryAppURL, err := url.Parse(httplib.GuessCurrentAppURL(ctx)) + if err != nil { + registryAppURL, _ = url.Parse(setting.AppURL) + } + priv, pub, err = generateKeypair(user.Name, registryAppURL.Host) + if err != nil { + return "", "", err + } + + if err := user_model.SetUserSetting(ctx, ownerID, arch_module.SettingKeyPrivate, priv); err != nil { + return "", "", err + } + + if err := user_model.SetUserSetting(ctx, ownerID, arch_module.SettingKeyPublic, pub); err != nil { + return "", "", err + } + } + + return priv, pub, nil +} + +func generateKeypair(owner, host string) (string, string, error) { + e, err := openpgp.NewEntity( + owner, + "Arch Package signature only", + fmt.Sprintf("%s@noreply.%s", owner, host), &packet.Config{ + RSABits: 4096, + }) + if err != nil { + return "", "", err + } + + var priv strings.Builder + var pub strings.Builder + + w, err := armor.Encode(&priv, openpgp.PrivateKeyType, nil) + if err != nil { + return "", "", err + } + if err := e.SerializePrivate(w, nil); err != nil { + return "", "", err + } + w.Close() + + w, err = armor.Encode(&pub, openpgp.PublicKeyType, nil) + if err != nil { + return "", "", err + } + if err := e.Serialize(w); err != nil { + return "", "", err + } + w.Close() + + return priv.String(), pub.String(), nil +} diff --git a/services/packages/cleanup/cleanup.go b/services/packages/cleanup/cleanup.go index 5d5120c6a0a8..7fbdc7cc9104 100644 --- a/services/packages/cleanup/cleanup.go +++ b/services/packages/cleanup/cleanup.go @@ -16,6 +16,7 @@ import ( packages_module "code.gitea.io/gitea/modules/packages" packages_service "code.gitea.io/gitea/services/packages" alpine_service "code.gitea.io/gitea/services/packages/alpine" + arch_service "code.gitea.io/gitea/services/packages/arch" cargo_service "code.gitea.io/gitea/services/packages/cargo" container_service "code.gitea.io/gitea/services/packages/container" debian_service "code.gitea.io/gitea/services/packages/debian" @@ -132,6 +133,10 @@ func ExecuteCleanupRules(outerCtx context.Context) error { if err := rpm_service.BuildAllRepositoryFiles(ctx, pcr.OwnerID); err != nil { return fmt.Errorf("CleanupRule [%d]: rpm.BuildAllRepositoryFiles failed: %w", pcr.ID, err) } + } else if pcr.Type == packages_model.TypeArch { + if err := arch_service.BuildAllRepositoryFiles(ctx, pcr.OwnerID); err != nil { + return fmt.Errorf("CleanupRule [%d]: arch.BuildAllRepositoryFiles failed: %w", pcr.ID, err) + } } } return nil diff --git a/services/packages/packages.go b/services/packages/packages.go index 64b1ddd86963..797029f0cb47 100644 --- a/services/packages/packages.go +++ b/services/packages/packages.go @@ -355,6 +355,8 @@ func CheckSizeQuotaExceeded(ctx context.Context, doer, owner *user_model.User, p switch packageType { case packages_model.TypeAlpine: typeSpecificSize = setting.Packages.LimitSizeAlpine + case packages_model.TypeArch: + typeSpecificSize = setting.Packages.LimitSizeArch case packages_model.TypeCargo: typeSpecificSize = setting.Packages.LimitSizeCargo case packages_model.TypeChef: diff --git a/templates/package/content/arch.tmpl b/templates/package/content/arch.tmpl new file mode 100644 index 000000000000..bcc24b585bb5 --- /dev/null +++ b/templates/package/content/arch.tmpl @@ -0,0 +1,143 @@ +{{if eq .PackageDescriptor.Package.Type "arch"}} +

{{ctx.Locale.Tr "packages.installation"}}

+
+
+
+ +
+ 
wget -O sign.gpg 
+pacman-key --add sign.gpg
+pacman-key --lsign-key '{{$.SignMail}}'
+
+
+
+ +
+ 

+{{- if gt (len $.Groups) 1 -}}
+# {{ctx.Locale.Tr "packages.arch.pacman.repo.multi"  $.PackageDescriptor.Package.LowerName}}
+
+{{end -}}
+{{- $GroupSize := (len .Groups) -}}
+{{-  range $i,$v :=  .Groups -}}
+{{- if gt $i 0}}
+{{end -}}{{- if gt $GroupSize 1 -}}
+# {{ctx.Locale.Tr "packages.arch.pacman.repo.multi.item" .}}
+{{end -}}
+[{{$.PackageDescriptor.Owner.LowerName}}.{{$.RegistryHost}}]
+SigLevel = Required
+Server = 
+{{end -}}
+
+
+
+
+ +
+ 
pacman -Sy {{.PackageDescriptor.Package.LowerName}}
+
+
+
+ +
+
+
+ +

{{ctx.Locale.Tr "packages.arch.version.properties"}}

+
+ + + + + + + + {{if .PackageDescriptor.Metadata.Groups}} + + + + + {{end}} + + {{if .PackageDescriptor.Metadata.Provides}} + + + + + {{end}} + + {{if .PackageDescriptor.Metadata.Depends}} + + + + + {{end}} + + {{if .PackageDescriptor.Metadata.OptDepends}} + + + + + {{end}} + + {{if .PackageDescriptor.Metadata.MakeDepends}} + + + + + {{end}} + + {{if .PackageDescriptor.Metadata.CheckDepends}} + + + + + {{end}} + + {{if .PackageDescriptor.Metadata.Conflicts}} + + + + + {{end}} + + {{if .PackageDescriptor.Metadata.Replaces}} + + + + + {{end}} + + {{if .PackageDescriptor.Metadata.Backup}} + + + + + {{end}} + +
+
{{ctx.Locale.Tr "packages.arch.version.description"}}
+ 		{{.PackageDescriptor.Metadata.Description}}
+
{{ctx.Locale.Tr "packages.arch.version.groups"}}
+ 		{{StringUtils.Join $.PackageDescriptor.Metadata.Groups ", "}}
+
{{ctx.Locale.Tr "packages.arch.version.provides"}}
+ 		{{StringUtils.Join $.PackageDescriptor.Metadata.Provides ", "}}
+
{{ctx.Locale.Tr "packages.arch.version.depends"}}
+ 		{{StringUtils.Join $.PackageDescriptor.Metadata.Depends ", "}}
+
{{ctx.Locale.Tr "packages.arch.version.optdepends"}}
+ 		{{StringUtils.Join $.PackageDescriptor.Metadata.OptDepends ", "}}
+
{{ctx.Locale.Tr "packages.arch.version.makedepends"}}
+ 		{{StringUtils.Join $.PackageDescriptor.Metadata.MakeDepends ", "}}
+
{{ctx.Locale.Tr "packages.arch.version.checkdepends"}}
+ 		{{StringUtils.Join $.PackageDescriptor.Metadata.CheckDepends ", "}}
+
{{ctx.Locale.Tr "packages.arch.version.conflicts"}}
+ 		{{StringUtils.Join $.PackageDescriptor.Metadata.Conflicts ", "}}
+
{{ctx.Locale.Tr "packages.arch.version.replaces"}}
+ 		{{StringUtils.Join $.PackageDescriptor.Metadata.Replaces ", "}}
+
{{ctx.Locale.Tr "packages.arch.version.backup"}}
+ 		{{StringUtils.Join $.PackageDescriptor.Metadata.Backup ", "}}
+
+ +{{end}} diff --git a/templates/package/metadata/arch.tmpl b/templates/package/metadata/arch.tmpl new file mode 100644 index 000000000000..822973eb7d98 --- /dev/null +++ b/templates/package/metadata/arch.tmpl @@ -0,0 +1,4 @@ +{{if eq .PackageDescriptor.Package.Type "arch"}} + {{range .PackageDescriptor.Metadata.License}}
{{svg "octicon-law" 16 "gt-mr-3"}} {{.}}
{{end}} + {{if .PackageDescriptor.Metadata.ProjectURL}}
{{svg "octicon-link-external" 16 "mr-3"}} {{ctx.Locale.Tr "packages.details.project_site"}}
{{end}} +{{end}} diff --git a/templates/package/view.tmpl b/templates/package/view.tmpl index 6beb249a7ffc..2c80dd494d47 100644 --- a/templates/package/view.tmpl +++ b/templates/package/view.tmpl @@ -19,6 +19,7 @@
{{template "package/content/alpine" .}} + {{template "package/content/arch" .}} {{template "package/content/cargo" .}} {{template "package/content/chef" .}} {{template "package/content/composer" .}} @@ -50,6 +51,7 @@
{{svg "octicon-calendar" 16 "tw-mr-2"}} {{TimeSinceUnix .PackageDescriptor.Version.CreatedUnix ctx.Locale}}
{{svg "octicon-download" 16 "tw-mr-2"}} {{.PackageDescriptor.Version.DownloadCount}}
{{template "package/metadata/alpine" .}} + {{template "package/metadata/arch" .}} {{template "package/metadata/cargo" .}} {{template "package/metadata/chef" .}} {{template "package/metadata/composer" .}} diff --git a/tests/integration/api_packages_arch_test.go b/tests/integration/api_packages_arch_test.go new file mode 100644 index 000000000000..6062a88ea08d --- /dev/null +++ b/tests/integration/api_packages_arch_test.go @@ -0,0 +1,327 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package integration + +import ( + "archive/tar" + "bufio" + "bytes" + "compress/gzip" + "encoding/base64" + "errors" + "fmt" + "io" + "net/http" + "strings" + "testing" + "testing/fstest" + + "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/models/packages" + "code.gitea.io/gitea/models/unittest" + user_model "code.gitea.io/gitea/models/user" + arch_model "code.gitea.io/gitea/modules/packages/arch" + "code.gitea.io/gitea/tests" + + "github.com/ProtonMail/go-crypto/openpgp/armor" + "github.com/ProtonMail/go-crypto/openpgp/packet" + "github.com/stretchr/testify/require" +) + +func TestPackageArch(t *testing.T) { + defer tests.PrepareTestEnv(t)() + user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) + unPack := func(s string) []byte { + data, _ := base64.StdEncoding.DecodeString(strings.ReplaceAll(strings.ReplaceAll(strings.TrimSpace(s), "\n", ""), "\r", "")) + return data + } + rootURL := fmt.Sprintf("/api/packages/%s/arch", user.Name) + + pkgs := map[string][]byte{ + "any": unPack(` +KLUv/QBYXRMABmOHSbCWag6dY6d8VNtVR3rpBnWdBbkDAxM38Dj3XG3FK01TCKlWtMV9QpskYdsm +e6fh5gWqM8edeurYNESoIUz/RmtyQy68HVrBj1p+AIoAYABFSJh4jcDyWNQgHIKIuNgIll64S4oY +FFIUk6vJQBMIIl2iYtIysqKWVYMCYvXDpAKTMzVGwZTUWhbciFCglIMH1QMbEtjHpohSi8XRYwPr +AwACSy/fzxO1FobizlP7sFgHcpx90Pus94Edjcc9GOustbD3PBprLUxH50IGC1sfw31c7LOfT4Qe +nh0KP1uKywwdPrRYmuyIkWBHRlcLfeBIDpKKqw44N0K2nNAfFW5grHRfSShyVgaEIZwIVVmFGL7O +88XDE5whJm4NkwA91dRoPBCcrgqozKSyah1QygsWkCshAaYrvbHCFdUTJCOgBpeUTMuJJ6+SRtcj +wIRua8mGJyg7qWoqJQq9z/4+DU1rHrEO8f6QZ3HUu3IM7GY37u+jeWjUu45637yN+qj338cdi0Uc +y0a9a+e5//1cYnPUu37dxr15khzNQ9/PE80aC/1okjz9mGo3bqP5Ue+scflGshdzx2g28061k2PW +uKwzjmV/XzTzzmKdcfz3eRbJoRPddcaP/n4PSZqQeYa1PDtPQzOHJK0amfjvz0IUV/v38xHJK/rz +JtFpalPD30drDWi7Bl8NB3J/P3csijQyldWZ8gy3TNslLsozMw74DhoAXoAfnE8xydUUHPZ3hML4 +2zVDGiEXSGYRx4BKQDcDJA5S9Ca25FRgPtSWSowZJpJTYAR9WCPHUDgACm6+hBecGDPNClpwHZ2A +EQ== +`), + "x86_64": unPack(` +KLUv/QBYnRMAFmOJS7BUbg7Un8q21hxCopsOMn6UGTzJRbHI753uOeMdxZ+V7ajoETVxl9CSBCR5 +2a3K1vr1gwyp9gCTH422bRNxHEg7Z0z9HV4rH/DGFn8AjABjAFQ2oaUVMRRGViVoqmxAVKuoKQVM +NJRwTDl9NcHCClliWjTpWin6sRUZsXSipWlAipQnleThRgFF5QTAzpth0UPFkhQeJRnYOaqSScEC +djCPDwE8pQTfVXW9F7bmznX3YTNZDeP7IHgxDazNQhp+UDa798KeRgvvvbCamgsYdL461TfvcmlY +djFowWYH5yaH5ztZcemh4omAkm7iQIWvGypNIXJQNgc7DVuHjx06I4MZGTIkeEBIOIL0OxcvnGps +0TwxycqKYESrwwQYEDKI2F0hNXH1/PCQ2BS4Ykki48EAaflAbRHxYrRQbdAZ4oXVAMGCkYOXkBRb +NkwjNCoIF07ByTlyfJhmoHQtCbFYDN+941783KqzusznmPePXJPluS1+cL/74Rd/1UHluW15blFv +ol6e+8XPPZNDPN/Kc9vOdX/xNZrT8twWnH34U9Xkqw76rqqrPjPQl6nJde9i74e/8Mtz6zOjT3R7 +Uve8BrabpT4zanE83158MtVbkxbH84vPNWkGqeu2OF704vfRzAGl6mhRtXPdmOrRzFla+BO+DL34 +uHHN9r74usjkduX5VEhNz9TnxV9trSabvYAwuIZffN0zSeZM3c3GUHX8dG6jeUgHGgBbgB9cUDHJ +1RR09teBwvjbNUMaIRdIZhHHgEpANwMkDpL0JsbkVFA+0JZKjBkmklNgBH1YI8dQOAAKbr6EF5wY +M80KWnAdnYAR +`), + "aarch64": unPack(` +KLUv/QBYdRQAVuSMS7BUbg7Un8q21hxCopsOMn6UGTzJRbHI753uOeMdxZ+V7ajoEbUkUXbXhXW/ +7FanWzv7B/EcMxhodFqyZkUcB9LOGVN/h9MqG7zFFmoAaQB8AEFrvpXntn3V/cXXaE7Lc9uP5uFP +VXPl+ue7qnJ9Zp8vU3PVvYu9HvbAL8+tz4y+0O1J3TPXqbZ5l3+lapk5ee+L577qXvdf+Atn+P69 +4Qz8QhpYw4/xd78Q3/v6Wg28974u1Ojc2ODseAGpHs2crYG4kef84uNGnu198fWQuVq+8ymQmp5p +z4vPbRjOaBC+FxziF1/3TJI5U3ezMlQdPZ3baA7SMhnMunvHvfg5rrO6zOeY94+rJstzW/zgetfD +Lz7XP+W5bXluUW+hXp77xc89kwFRTF1PrKxAFpgXT7ZWhjzYjpRIStGyNCAGBYM6AnGrkKKCAmAH +k3HBI8VyBBYdGdApmoqJYQE62EeIADCkBF1VOW0WYnz/+y6ufTMaDQ2GDDme7Wapz4xa3JpvLz6Z +6q1Ji1vzi79q0vxR+ba4dejF76OZ80nV0aJqX3VjKCsuP1g0EWDSURyw0JVDZWlEzsnmYLdh8wDS +I2dkIEMjxsSOiAlJjH4HIwbTjayZJidXVxKQYH2gICOCBhK7KqMlLZ4gMCU1BapYlsTAXnywepyy +jMBmtEhxyCnCZdUAwYKxAxeRFVk4TCL0aYgWjt3kHTg9SjVStppI2YCSWshUEFGdmJmyCVGpnqIU +KNlA0hEjIOACGSLqYpXAD5SSNVT2MJRJwREAF4FRHPBlCJMSNwFguGAWDJBg+KIArkIJGNtCydUL +TuN1oBh/+zKkEblAsgjGqVgUwKLP+UOMOGCpAhICtg6ncFJH`), + "other": unPack(` +KLUv/QBYbRMABuOHS9BSNQdQ56F+xNFoV3CijY54JYt3VqV1iUU3xmj00y2pyBOCuokbhDYpvNsj +ZJeCxqH+nQFpMf4Wa92okaZoF4eH6HsXXCBo+qy3Fn4AigBgAEaYrLCQEuAom6YbHyuKZAFYksqi +sSOFiRs0WDmlACk0CnpnaAeKiCS3BlwVkViJEbDS43lFNbLkZEmGhc305Nn4AMLGiUkBDiMTG5Vz +q4ZISjCofEfR1NpXijvP2X95Hu1e+zLalc0+mjeT3Z/FPGvt62WymbX2dXMDIYKDLjjP8n03RrPf +A1vOApwGOh2MgE2LpgZrgXLDF2CUJ15idG2J8GCSgcc2ZVRgA8+RHD0k2VJjg6mRUgGGhBWEyEcz +5EePLhUeWlYhoFCKONxUiBiIUiQeDIqiQwkjLiyqnF5eGs6a2gGRapbU9JRyuXAlPemYajlJojJd +GBBJjo5GxFRkITOAvLhSCr2TDz4uzdU8Yh3i/SHP4qh3vTG2s9198NP8M+pdR73BvIP6qPeDjzsW +gTi+jXrXWOe5P/jZxOeod/287v6JljzNP99RNM0a+/x4ljz3LNV2t5v9qHfW2Pyg24u54zSfObWX +Y9bYrCTHtwdfPPPOYiU5fvB5FssfNN2V5EIPfg9LnM+JhtVEO8+FZw5LXA068YNPhimu9sHPQiWv +qc6fE9BTnxIe/LTKatab+WYu7T74uWNRxJW5W5Ux0bDLuG1ioCwjg4DvGgBcgB8cUDHJ1RQ89neE +wvjbNUMiIZdo5hbHgEpANwMkDnL0Jr7kVFg+0pZKjBkmklNgBH1YI8dQOAAKbr6EF5wYM80KWnAd +nYAR`), + } + + t.Run("RepositoryKey", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + req := NewRequest(t, "GET", rootURL+"/repository.key") + resp := MakeRequest(t, req, http.StatusOK) + + require.Equal(t, "application/pgp-keys", resp.Header().Get("Content-Type")) + require.Contains(t, resp.Body.String(), "-----BEGIN PGP PUBLIC KEY BLOCK-----") + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + req := NewRequestWithBody(t, "PUT", rootURL+"/default", bytes.NewReader(pkgs["any"])) + MakeRequest(t, req, http.StatusUnauthorized) + + req = NewRequestWithBody(t, "PUT", rootURL+"/default", bytes.NewReader(pkgs["any"])). + AddBasicAuth(user.Name) + MakeRequest(t, req, http.StatusCreated) + + pvs, err := packages.GetVersionsByPackageType(db.DefaultContext, user.ID, packages.TypeArch) + require.NoError(t, err) + require.Len(t, pvs, 1) + + pd, err := packages.GetPackageDescriptor(db.DefaultContext, pvs[0]) + require.NoError(t, err) + require.Nil(t, pd.SemVer) + require.IsType(t, &arch_model.VersionMetadata{}, pd.Metadata) + require.Equal(t, "test", pd.Package.Name) + require.Equal(t, "1.0.0-1", pd.Version.Version) + + pfs, err := packages.GetFilesByVersionID(db.DefaultContext, pvs[0].ID) + require.NoError(t, err) + require.Len(t, pfs, 2) // zst and zst.sig + require.True(t, pfs[0].IsLead) + + pb, err := packages.GetBlobByID(db.DefaultContext, pfs[0].BlobID) + require.NoError(t, err) + require.Equal(t, int64(len(pkgs["any"])), pb.Size) + + req = NewRequestWithBody(t, "PUT", rootURL+"/default", bytes.NewReader(pkgs["any"])). + AddBasicAuth(user.Name) + MakeRequest(t, req, http.StatusConflict) + req = NewRequestWithBody(t, "PUT", rootURL+"/default", bytes.NewReader(pkgs["x86_64"])). + AddBasicAuth(user.Name) + MakeRequest(t, req, http.StatusCreated) + req = NewRequestWithBody(t, "PUT", rootURL+"/other", bytes.NewReader(pkgs["any"])). + AddBasicAuth(user.Name) + MakeRequest(t, req, http.StatusCreated) + req = NewRequestWithBody(t, "PUT", rootURL+"/other", bytes.NewReader(pkgs["aarch64"])). + AddBasicAuth(user.Name) + MakeRequest(t, req, http.StatusCreated) + + req = NewRequestWithBody(t, "PUT", rootURL+"/base", bytes.NewReader(pkgs["other"])). + AddBasicAuth(user.Name) + MakeRequest(t, req, http.StatusCreated) + req = NewRequestWithBody(t, "PUT", rootURL+"/base", bytes.NewReader(pkgs["x86_64"])). + AddBasicAuth(user.Name) + MakeRequest(t, req, http.StatusCreated) + req = NewRequestWithBody(t, "PUT", rootURL+"/base", bytes.NewReader(pkgs["aarch64"])). + AddBasicAuth(user.Name) + MakeRequest(t, req, http.StatusCreated) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + req := NewRequest(t, "GET", rootURL+"/default/x86_64/test-1.0.0-1-x86_64.pkg.tar.zst") + resp := MakeRequest(t, req, http.StatusOK) + require.Equal(t, pkgs["x86_64"], resp.Body.Bytes()) + + req = NewRequest(t, "GET", rootURL+"/default/x86_64/test-1.0.0-1-any.pkg.tar.zst") + resp = MakeRequest(t, req, http.StatusOK) + require.Equal(t, pkgs["any"], resp.Body.Bytes()) + + req = NewRequest(t, "GET", rootURL+"/default/x86_64/test-1.0.0-1-aarch64.pkg.tar.zst") + MakeRequest(t, req, http.StatusNotFound) + + req = NewRequest(t, "GET", rootURL+"/other/x86_64/test-1.0.0-1-x86_64.pkg.tar.zst") + MakeRequest(t, req, http.StatusNotFound) + + req = NewRequest(t, "GET", rootURL+"/other/x86_64/test-1.0.0-1-any.pkg.tar.zst") + resp = MakeRequest(t, req, http.StatusOK) + require.Equal(t, pkgs["any"], resp.Body.Bytes()) + }) + + t.Run("SignVerify", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + req := NewRequest(t, "GET", rootURL+"/repository.key") + respPub := MakeRequest(t, req, http.StatusOK) + + req = NewRequest(t, "GET", rootURL+"/other/x86_64/test-1.0.0-1-any.pkg.tar.zst") + respPkg := MakeRequest(t, req, http.StatusOK) + + req = NewRequest(t, "GET", rootURL+"/other/x86_64/test-1.0.0-1-any.pkg.tar.zst.sig") + respSig := MakeRequest(t, req, http.StatusOK) + + if err := gpgVerify(respPub.Body.Bytes(), respSig.Body.Bytes(), respPkg.Body.Bytes()); err != nil { + t.Fatal(err) + } + }) + + t.Run("Repository", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + req := NewRequest(t, "GET", rootURL+"/repository.key") + respPub := MakeRequest(t, req, http.StatusOK) + + req = NewRequest(t, "GET", rootURL+"/base/x86_64/base.db") + respPkg := MakeRequest(t, req, http.StatusOK) + + req = NewRequest(t, "GET", rootURL+"/base/x86_64/base.db.sig") + respSig := MakeRequest(t, req, http.StatusOK) + + if err := gpgVerify(respPub.Body.Bytes(), respSig.Body.Bytes(), respPkg.Body.Bytes()); err != nil { + t.Fatal(err) + } + files, err := listGzipFiles(respPkg.Body.Bytes()) + require.NoError(t, err) + require.Len(t, files, 2) + for s, d := range files { + name := getProperty(string(d.Data), "NAME") + ver := getProperty(string(d.Data), "VERSION") + require.Equal(t, name+"-"+ver+"/desc", s) + fn := getProperty(string(d.Data), "FILENAME") + pgp := getProperty(string(d.Data), "PGPSIG") + req = NewRequest(t, "GET", rootURL+"/base/x86_64/"+fn+".sig") + respSig := MakeRequest(t, req, http.StatusOK) + decodeString, err := base64.StdEncoding.DecodeString(pgp) + require.NoError(t, err) + require.Equal(t, respSig.Body.Bytes(), decodeString) + } + }) + t.Run("Delete", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + req := NewRequestWithBody(t, "DELETE", rootURL+"/base/notfound/1.0.0-1", nil). + AddBasicAuth(user.Name) + MakeRequest(t, req, http.StatusNotFound) + + req = NewRequestWithBody(t, "DELETE", rootURL+"/base/test/1.0.0-1", nil). + AddBasicAuth(user.Name) + MakeRequest(t, req, http.StatusNoContent) + + req = NewRequest(t, "GET", rootURL+"/base/x86_64/base.db") + respPkg := MakeRequest(t, req, http.StatusOK) + files, err := listGzipFiles(respPkg.Body.Bytes()) + require.NoError(t, err) + require.Len(t, files, 1) + + req = NewRequestWithBody(t, "DELETE", rootURL+"/base/test2/1.0.0-1", nil). + AddBasicAuth(user.Name) + MakeRequest(t, req, http.StatusNoContent) + req = NewRequest(t, "GET", rootURL+"/base/x86_64/base.db") + MakeRequest(t, req, http.StatusNotFound) + + req = NewRequest(t, "GET", rootURL+"/default/x86_64/base.db") + respPkg = MakeRequest(t, req, http.StatusOK) + files, err = listGzipFiles(respPkg.Body.Bytes()) + require.NoError(t, err) + require.Len(t, files, 1) + }) +} + +func getProperty(data, key string) string { + r := bufio.NewReader(strings.NewReader(data)) + for { + line, _, err := r.ReadLine() + if err != nil { + return "" + } + if strings.Contains(string(line), "%"+key+"%") { + readLine, _, _ := r.ReadLine() + return string(readLine) + } + } +} + +func listGzipFiles(data []byte) (fstest.MapFS, error) { + reader, err := gzip.NewReader(bytes.NewBuffer(data)) + defer reader.Close() + if err != nil { + return nil, err + } + tarRead := tar.NewReader(reader) + files := make(fstest.MapFS) + for { + cur, err := tarRead.Next() + if err == io.EOF { + break + } else if err != nil { + return nil, err + } + if cur.Typeflag != tar.TypeReg { + continue + } + data, err := io.ReadAll(tarRead) + if err != nil { + return nil, err + } + files[cur.Name] = &fstest.MapFile{Data: data} + } + return files, nil +} + +func gpgVerify(pub, sig, data []byte) error { + sigPack, err := packet.Read(bytes.NewBuffer(sig)) + if err != nil { + return err + } + signature, ok := sigPack.(*packet.Signature) + if !ok { + return errors.New("invalid sign key") + } + pubBlock, err := armor.Decode(bytes.NewReader(pub)) + if err != nil { + return err + } + pack, err := packet.Read(pubBlock.Body) + if err != nil { + return err + } + publicKey, ok := pack.(*packet.PublicKey) + if !ok { + return errors.New("invalid public key") + } + hash := signature.Hash.New() + _, err = hash.Write(data) + if err != nil { + return err + } + return publicKey.VerifySignature(hash, signature) +} diff --git a/web_src/svg/gitea-arch.svg b/web_src/svg/gitea-arch.svg new file mode 100644 index 000000000000..ba8254d8049e --- /dev/null +++ b/web_src/svg/gitea-arch.svg @@ -0,0 +1 @@ + \ No newline at end of file From c98a705f142443849089bfd27e869bfb781b77c9 Mon Sep 17 00:00:00 2001 From: dragon Date: Thu, 19 Sep 2024 09:21:53 +0800 Subject: [PATCH 02/21] fix anything v2 ... --- modules/packages/arch/metadata.go | 42 ++- modules/packages/arch/metadata_test.go | 12 +- routers/api/packages/api.go | 57 +++- routers/api/packages/arch/arch.go | 49 ++-- services/forms/package_form.go | 2 +- services/packages/arch/repository.go | 61 ++-- tests/integration/api_packages_arch_test.go | 299 ++++++++++---------- 7 files changed, 307 insertions(+), 215 deletions(-) diff --git a/modules/packages/arch/metadata.go b/modules/packages/arch/metadata.go index fc66d288cc5b..9b443899bb4c 100644 --- a/modules/packages/arch/metadata.go +++ b/modules/packages/arch/metadata.go @@ -41,11 +41,15 @@ var ( reVer = regexp.MustCompile(`^[a-zA-Z0-9:_.+]+-+[0-9]+$`) reOptDep = regexp.MustCompile(`^[a-zA-Z0-9@._+-]+$|^[a-zA-Z0-9@._+-]+(:.*)`) rePkgVer = regexp.MustCompile(`^[a-zA-Z0-9@._+-]+$|^[a-zA-Z0-9@._+-]+(>.*)|^[a-zA-Z0-9@._+-]+(<.*)|^[a-zA-Z0-9@._+-]+(=.*)`) + + magicZSTD = []byte{0x28, 0xB5, 0x2F, 0xFD} + magicXZ = []byte{0xFD, 0x37, 0x7A, 0x58, 0x5A} ) type Package struct { Name string `json:"name"` Version string `json:"version"` // Includes version, release and epoch + CompressType string `json:"compress_type"` VersionMetadata VersionMetadata FileMetadata FileMetadata } @@ -89,18 +93,38 @@ func ParsePackage(r *packages.HashedBuffer) (*Package, error) { if err != nil { return nil, err } - zstd := archiver.NewTarZstd() - err = zstd.Open(r, 0) + header := make([]byte, 5) + _, err = r.Read(header) + if err != nil { + return nil, err + } + _, err = r.Seek(0, io.SeekStart) + if err != nil { + return nil, err + } + + var tarball archiver.Reader + var tarballType string + if bytes.Equal(header[:len(magicZSTD)], magicZSTD) { + tarballType = "zst" + tarball = archiver.NewTarZstd() + } else if bytes.Equal(header[:len(magicXZ)], magicXZ) { + tarballType = "xz" + tarball = archiver.NewTarXz() + } else { + return nil, errors.New("not supported compression") + } + err = tarball.Open(r, 0) if err != nil { return nil, err } - defer zstd.Close() + defer tarball.Close() var pkg *Package var mtree bool for { - f, err := zstd.Read() + f, err := tarball.Read() if err == io.EOF { break } @@ -111,7 +135,7 @@ func ParsePackage(r *packages.HashedBuffer) (*Package, error) { switch f.Name() { case ".PKGINFO": - pkg, err = ParsePackageInfo(f) + pkg, err = ParsePackageInfo(tarballType, f) if err != nil { return nil, err } @@ -137,8 +161,10 @@ func ParsePackage(r *packages.HashedBuffer) (*Package, error) { // ParsePackageInfo Function that accepts reader for .PKGINFO file from package archive, // validates all field according to PKGBUILD spec and returns package. -func ParsePackageInfo(r io.Reader) (*Package, error) { - p := &Package{} +func ParsePackageInfo(compressType string, r io.Reader) (*Package, error) { + p := &Package{ + CompressType: compressType, + } scanner := bufio.NewScanner(r) for scanner.Scan() { @@ -281,7 +307,7 @@ func ValidatePackageSpec(p *Package) error { // Desc Create pacman package description file. func (p *Package) Desc() string { entries := []string{ - "FILENAME", fmt.Sprintf("%s-%s-%s.pkg.tar.zst", p.Name, p.Version, p.FileMetadata.Arch), + "FILENAME", fmt.Sprintf("%s-%s-%s.pkg.tar.%s", p.Name, p.Version, p.FileMetadata.Arch, p.CompressType), "NAME", p.Name, "BASE", p.VersionMetadata.Base, "VERSION", p.Version, diff --git a/modules/packages/arch/metadata_test.go b/modules/packages/arch/metadata_test.go index b084762fb69a..bd4c2a9ad87d 100644 --- a/modules/packages/arch/metadata_test.go +++ b/modules/packages/arch/metadata_test.go @@ -158,11 +158,12 @@ checkdepend = ola makedepend = cmake backup = usr/bin/paket1 ` - p, err := ParsePackageInfo(strings.NewReader(PKGINFO)) + p, err := ParsePackageInfo("zst", strings.NewReader(PKGINFO)) require.NoError(t, err) require.Equal(t, Package{ - Name: "a", - Version: "1-2", + CompressType: "zst", + Name: "a", + Version: "1-2", VersionMetadata: VersionMetadata{ Base: "b", Description: "comment", @@ -417,8 +418,9 @@ dummy6 ` md := &Package{ - Name: "zstd", - Version: "1.5.5-1", + CompressType: "zst", + Name: "zstd", + Version: "1.5.5-1", VersionMetadata: VersionMetadata{ Base: "zstd", Description: "Zstandard - Fast real-time compression algorithm", diff --git a/routers/api/packages/api.go b/routers/api/packages/api.go index 12077753b333..b40477036e69 100644 --- a/routers/api/packages/api.go +++ b/routers/api/packages/api.go @@ -127,10 +127,59 @@ func CommonRoutes() *web.Router { r.Head("", arch.GetRepositoryKey) r.Get("", arch.GetRepositoryKey) }) - r.Group("/{distro}", func() { - r.Put("", reqPackageAccess(perm.AccessModeWrite), arch.PushPackage) - r.Get("/{arch}/{file}", arch.GetPackageOrDB) - r.Delete("/{package}/{version}", reqPackageAccess(perm.AccessModeWrite), arch.RemovePackage) + + r.Methods("HEAD,GET,PUT,DELETE", "*", func(ctx *context.Context) { + pathGroups := strings.Split(strings.Trim(ctx.PathParam("*"), "/"), "/") + groupLen := len(pathGroups) + isGetHead := ctx.Req.Method == "HEAD" || ctx.Req.Method == "GET" + isPut := ctx.Req.Method == "PUT" + isDelete := ctx.Req.Method == "DELETE" + if isGetHead { + if groupLen < 2 { + ctx.Status(http.StatusNotFound) + return + } + if groupLen == 2 { + ctx.SetPathParam("group", "") + ctx.SetPathParam("arch", pathGroups[0]) + ctx.SetPathParam("file", pathGroups[1]) + } else { + ctx.SetPathParam("group", strings.Join(pathGroups[:groupLen-2], "/")) + ctx.SetPathParam("arch", pathGroups[groupLen-2]) + ctx.SetPathParam("file", pathGroups[groupLen-1]) + } + arch.GetPackageOrDB(ctx) + return + } else if isPut { + ctx.SetPathParam("group", strings.Join(pathGroups, "/")) + reqPackageAccess(perm.AccessModeWrite)(ctx) + if ctx.Written() { + return + } + arch.PushPackage(ctx) + return + } else if isDelete { + if groupLen < 2 { + ctx.Status(http.StatusBadRequest) + return + } + if groupLen == 2 { + ctx.SetPathParam("group", "") + ctx.SetPathParam("package", pathGroups[0]) + ctx.SetPathParam("version", pathGroups[1]) + } else { + ctx.SetPathParam("group", strings.Join(pathGroups[:groupLen-2], "/")) + ctx.SetPathParam("package", pathGroups[groupLen-2]) + ctx.SetPathParam("version", pathGroups[groupLen-1]) + } + reqPackageAccess(perm.AccessModeWrite)(ctx) + if ctx.Written() { + return + } + arch.RemovePackage(ctx) + return + } + ctx.Status(http.StatusNotFound) }) }, reqPackageAccess(perm.AccessModeRead)) r.Group("/cargo", func() { diff --git a/routers/api/packages/arch/arch.go b/routers/api/packages/arch/arch.go index 22cad2a918be..835820b40856 100644 --- a/routers/api/packages/arch/arch.go +++ b/routers/api/packages/arch/arch.go @@ -9,6 +9,7 @@ import ( "fmt" "io" "net/http" + "regexp" "strings" packages_model "code.gitea.io/gitea/models/packages" @@ -21,6 +22,11 @@ import ( arch_service "code.gitea.io/gitea/services/packages/arch" ) +var ( + archPkgOrSig = regexp.MustCompile(`^.*\.pkg\.tar\.\w+(\.sig)*$`) + archDBOrSig = regexp.MustCompile(`^.*.db(\.tar\.gz)*(\.sig)*$`) +) + func apiError(ctx *context.Context, status int, obj any) { helper.LogAndProcessError(ctx, status, obj, func(message string) { ctx.PlainText(status, message) @@ -41,7 +47,7 @@ func GetRepositoryKey(ctx *context.Context) { } func PushPackage(ctx *context.Context) { - distro := ctx.PathParam("distro") + group := ctx.PathParam("group") upload, needToClose, err := ctx.UploadStream() if err != nil { @@ -61,7 +67,7 @@ func PushPackage(ctx *context.Context) { p, err := arch_module.ParsePackage(buf) if err != nil { - apiError(ctx, http.StatusInternalServerError, err) + apiError(ctx, http.StatusBadRequest, err) return } @@ -97,7 +103,7 @@ func PushPackage(ctx *context.Context) { properties := map[string]string{ arch_module.PropertyDescription: p.Desc(), arch_module.PropertyArch: p.FileMetadata.Arch, - arch_module.PropertyDistribution: distro, + arch_module.PropertyDistribution: group, } version, _, err := packages_service.CreatePackageOrAddFileToExisting( @@ -114,8 +120,8 @@ func PushPackage(ctx *context.Context) { }, &packages_service.PackageFileCreationInfo{ PackageFileInfo: packages_service.PackageFileInfo{ - Filename: fmt.Sprintf("%s-%s-%s.pkg.tar.zst", p.Name, p.Version, p.FileMetadata.Arch), - CompositeKey: distro, + Filename: fmt.Sprintf("%s-%s-%s.pkg.tar.%s", p.Name, p.Version, p.FileMetadata.Arch, p.CompressType), + CompositeKey: group, }, OverwriteExisting: false, IsLead: true, @@ -138,8 +144,8 @@ func PushPackage(ctx *context.Context) { // add sign file _, err = packages_service.AddFileToPackageVersionInternal(ctx, version, &packages_service.PackageFileCreationInfo{ PackageFileInfo: packages_service.PackageFileInfo{ - CompositeKey: distro, - Filename: fmt.Sprintf("%s-%s-%s.pkg.tar.zst.sig", p.Name, p.Version, p.FileMetadata.Arch), + CompositeKey: group, + Filename: fmt.Sprintf("%s-%s-%s.pkg.tar.%s.sig", p.Name, p.Version, p.FileMetadata.Arch, p.CompressType), }, OverwriteExisting: true, IsLead: false, @@ -149,7 +155,7 @@ func PushPackage(ctx *context.Context) { if err != nil { apiError(ctx, http.StatusInternalServerError, err) } - if err = arch_service.BuildPacmanDB(ctx, ctx.Package.Owner.ID, distro, p.FileMetadata.Arch); err != nil { + if err = arch_service.BuildPacmanDB(ctx, ctx.Package.Owner.ID, group, p.FileMetadata.Arch); err != nil { apiError(ctx, http.StatusInternalServerError, err) return } @@ -158,13 +164,13 @@ func PushPackage(ctx *context.Context) { func GetPackageOrDB(ctx *context.Context) { var ( - file = ctx.PathParam("file") - distro = ctx.PathParam("distro") - arch = ctx.PathParam("arch") + file = ctx.PathParam("file") + group = ctx.PathParam("group") + arch = ctx.PathParam("arch") ) - if strings.HasSuffix(file, ".pkg.tar.zst") || strings.HasSuffix(file, ".pkg.tar.zst.sig") { - pkg, err := arch_service.GetPackageFile(ctx, distro, file, ctx.Package.Owner.ID) + if archPkgOrSig.MatchString(file) { + pkg, err := arch_service.GetPackageFile(ctx, group, file, ctx.Package.Owner.ID) if err != nil { if errors.Is(err, util.ErrNotExist) { apiError(ctx, http.StatusNotFound, err) @@ -180,11 +186,8 @@ func GetPackageOrDB(ctx *context.Context) { return } - if strings.HasSuffix(file, ".db.tar.gz") || - strings.HasSuffix(file, ".db") || - strings.HasSuffix(file, ".db.tar.gz.sig") || - strings.HasSuffix(file, ".db.sig") { - pkg, err := arch_service.GetPackageDBFile(ctx, distro, arch, ctx.Package.Owner.ID, + if archDBOrSig.MatchString(file) { + pkg, err := arch_service.GetPackageDBFile(ctx, group, arch, ctx.Package.Owner.ID, strings.HasSuffix(file, ".sig")) if err != nil { if errors.Is(err, util.ErrNotExist) { @@ -205,9 +208,9 @@ func GetPackageOrDB(ctx *context.Context) { func RemovePackage(ctx *context.Context) { var ( - distro = ctx.PathParam("distro") - pkg = ctx.PathParam("package") - ver = ctx.PathParam("version") + group = ctx.PathParam("group") + pkg = ctx.PathParam("package") + ver = ctx.PathParam("version") ) pv, err := packages_model.GetVersionByNameAndVersion( ctx, ctx.Package.Owner.ID, packages_model.TypeArch, pkg, ver, @@ -227,7 +230,7 @@ func RemovePackage(ctx *context.Context) { } deleted := false for _, file := range files { - if file.CompositeKey == distro { + if file.CompositeKey == group { deleted = true err := packages_service.RemovePackageFileAndVersionIfUnreferenced(ctx, ctx.ContextUser, file) if err != nil { @@ -237,7 +240,7 @@ func RemovePackage(ctx *context.Context) { } } if deleted { - err = arch_service.BuildCustomRepositoryFiles(ctx, ctx.Package.Owner.ID, distro) + err = arch_service.BuildCustomRepositoryFiles(ctx, ctx.Package.Owner.ID, group) if err != nil { apiError(ctx, http.StatusInternalServerError, err) } diff --git a/services/forms/package_form.go b/services/forms/package_form.go index cc940d42d34d..9b6f9071647b 100644 --- a/services/forms/package_form.go +++ b/services/forms/package_form.go @@ -15,7 +15,7 @@ import ( type PackageCleanupRuleForm struct { ID int64 Enabled bool - Type string `binding:"Required;In(alpine,cargo,chef,composer,conan,conda,container,cran,debian,generic,go,helm,maven,npm,nuget,pub,pypi,rpm,rubygems,swift,vagrant)"` + Type string `binding:"Required;In(alpine,arch,cargo,chef,composer,conan,conda,container,cran,debian,generic,go,helm,maven,npm,nuget,pub,pypi,rpm,rubygems,swift,vagrant)"` KeepCount int `binding:"In(0,1,5,10,25,50,100)"` KeepPattern string `binding:"RegexPattern"` RemoveDays int `binding:"In(0,7,14,30,60,90,180)"` diff --git a/services/packages/arch/repository.go b/services/packages/arch/repository.go index 41c2c7954147..a442b0c23f1b 100644 --- a/services/packages/arch/repository.go +++ b/services/packages/arch/repository.go @@ -12,6 +12,7 @@ import ( "io" "net/url" "os" + "path/filepath" "sort" "strings" @@ -45,7 +46,7 @@ func BuildAllRepositoryFiles(ctx context.Context, ownerID int64) error { } for _, pf := range pfs { if strings.HasSuffix(pf.Name, ".db") { - arch := strings.TrimSuffix(strings.TrimPrefix(pf.Name, fmt.Sprintf("%s-", pf.CompositeKey)), ".db") + arch := strings.TrimSuffix(pf.Name, ".db") if err := BuildPacmanDB(ctx, ownerID, pf.CompositeKey, arch); err != nil { return err } @@ -101,7 +102,7 @@ func NewFileSign(ctx context.Context, ownerID int64, input io.Reader) (*packages } // BuildPacmanDB Create db signature cache -func BuildPacmanDB(ctx context.Context, ownerID int64, distro, arch string) error { +func BuildPacmanDB(ctx context.Context, ownerID int64, group, arch string) error { pv, err := GetOrCreateRepositoryVersion(ctx, ownerID) if err != nil { return err @@ -112,15 +113,15 @@ func BuildPacmanDB(ctx context.Context, ownerID int64, distro, arch string) erro return err } for _, pf := range pfs { - if pf.CompositeKey == distro && strings.HasPrefix(pf.Name, fmt.Sprintf("%s-%s", distro, arch)) { - // remove distro and arch + if pf.CompositeKey == group && pf.Name == fmt.Sprintf("%s.db", arch) { + // remove group and arch if err := packages_service.DeletePackageFile(ctx, pf); err != nil { return err } } } - db, err := flushDB(ctx, ownerID, distro, arch) + db, err := createDB(ctx, ownerID, group, arch) if errors.Is(err, io.EOF) { return nil } else if err != nil { @@ -142,13 +143,13 @@ func BuildPacmanDB(ctx context.Context, ownerID int64, distro, arch string) erro return err } for name, data := range map[string]*packages_module.HashedBuffer{ - fmt.Sprintf("%s-%s.db", distro, arch): db, - fmt.Sprintf("%s-%s.db.sig", distro, arch): sig, + fmt.Sprintf("%s.db", arch): db, + fmt.Sprintf("%s.db.sig", arch): sig, } { _, err = packages_service.AddFileToPackageVersionInternal(ctx, pv, &packages_service.PackageFileCreationInfo{ PackageFileInfo: packages_service.PackageFileInfo{ Filename: name, - CompositeKey: distro, + CompositeKey: group, }, Creator: user_model.NewGhostUser(), Data: data, @@ -162,7 +163,7 @@ func BuildPacmanDB(ctx context.Context, ownerID int64, distro, arch string) erro return nil } -func flushDB(ctx context.Context, ownerID int64, distro, arch string) (*packages_module.HashedBuffer, error) { +func createDB(ctx context.Context, ownerID int64, group, arch string) (*packages_module.HashedBuffer, error) { pkgs, err := packages_model.GetPackagesByType(ctx, ownerID, packages_model.TypeArch) if err != nil { return nil, err @@ -187,17 +188,29 @@ func flushDB(ctx context.Context, ownerID int64, distro, arch string) (*packages sort.Slice(versions, func(i, j int) bool { return versions[i].CreatedUnix > versions[j].CreatedUnix }) + for _, ver := range versions { - file := fmt.Sprintf("%s-%s-%s.pkg.tar.zst", pkg.Name, ver.Version, arch) - pf, err := packages_model.GetFileForVersionByName(ctx, ver.ID, file, distro) + files, err := packages_model.GetFilesByVersionID(ctx, ver.ID) if err != nil { - // add any arch package - file = fmt.Sprintf("%s-%s-any.pkg.tar.zst", pkg.Name, ver.Version) - pf, err = packages_model.GetFileForVersionByName(ctx, ver.ID, file, distro) - if err != nil { - continue + return nil, errors.Join(tw.Close(), gw.Close(), db.Close(), err) + } + var pf *packages_model.PackageFile + for _, file := range files { + ext := filepath.Ext(file.Name) + if file.CompositeKey == group && ext != "" && ext != ".db" && ext != ".sig" { + if pf == nil && strings.HasSuffix(file.Name, fmt.Sprintf("any.pkg.tar%s", ext)) { + pf = file + } + if strings.HasSuffix(file.Name, fmt.Sprintf("%s.pkg.tar%s", arch, ext)) { + pf = file + break + } } } + if pf == nil { + // file not exists + continue + } pps, err := packages_model.GetPropertiesByName( ctx, packages_model.PropertyTypeFile, pf.ID, arch_module.PropertyDescription, ) @@ -232,8 +245,8 @@ func flushDB(ctx context.Context, ownerID int64, distro, arch string) (*packages // GetPackageFile Get data related to provided filename and distribution, for package files // update download counter. -func GetPackageFile(ctx context.Context, distro, file string, ownerID int64) (io.ReadSeekCloser, error) { - pf, err := getPackageFile(ctx, distro, file, ownerID) +func GetPackageFile(ctx context.Context, group, file string, ownerID int64) (io.ReadSeekCloser, error) { + pf, err := getPackageFile(ctx, group, file, ownerID) if err != nil { return nil, err } @@ -243,7 +256,7 @@ func GetPackageFile(ctx context.Context, distro, file string, ownerID int64) (io } // Ejects parameters required to get package file property from file name. -func getPackageFile(ctx context.Context, distro, file string, ownerID int64) (*packages_model.PackageFile, error) { +func getPackageFile(ctx context.Context, group, file string, ownerID int64) (*packages_model.PackageFile, error) { var ( splt = strings.Split(file, "-") pkgname = strings.Join(splt[0:len(splt)-3], "-") @@ -255,23 +268,23 @@ func getPackageFile(ctx context.Context, distro, file string, ownerID int64) (*p return nil, err } - pkgfile, err := packages_model.GetFileForVersionByName(ctx, version.ID, file, distro) + pkgfile, err := packages_model.GetFileForVersionByName(ctx, version.ID, file, group) if err != nil { return nil, err } return pkgfile, nil } -func GetPackageDBFile(ctx context.Context, distro, arch string, ownerID int64, signFile bool) (io.ReadSeekCloser, error) { +func GetPackageDBFile(ctx context.Context, group, arch string, ownerID int64, signFile bool) (io.ReadSeekCloser, error) { pv, err := GetOrCreateRepositoryVersion(ctx, ownerID) if err != nil { return nil, err } - fileName := fmt.Sprintf("%s-%s.db", distro, arch) + fileName := fmt.Sprintf("%s.db", arch) if signFile { - fileName = fmt.Sprintf("%s-%s.db.sig", distro, arch) + fileName = fmt.Sprintf("%s.db.sig", arch) } - file, err := packages_model.GetFileForVersionByName(ctx, pv.ID, fileName, distro) + file, err := packages_model.GetFileForVersionByName(ctx, pv.ID, fileName, group) if err != nil { return nil, err } diff --git a/tests/integration/api_packages_arch_test.go b/tests/integration/api_packages_arch_test.go index 6062a88ea08d..74a6bb8bee76 100644 --- a/tests/integration/api_packages_arch_test.go +++ b/tests/integration/api_packages_arch_test.go @@ -81,18 +81,18 @@ jMBmtEhxyCnCZdUAwYKxAxeRFVk4TCL0aYgWjt3kHTg9SjVStppI2YCSWshUEFGdmJmyCVGpnqIU KNlA0hEjIOACGSLqYpXAD5SSNVT2MJRJwREAF4FRHPBlCJMSNwFguGAWDJBg+KIArkIJGNtCydUL TuN1oBh/+zKkEblAsgjGqVgUwKLP+UOMOGCpAhICtg6ncFJH`), "other": unPack(` -KLUv/QBYbRMABuOHS9BSNQdQ56F+xNFoV3CijY54JYt3VqV1iUU3xmj00y2pyBOCuokbhDYpvNsj -ZJeCxqH+nQFpMf4Wa92okaZoF4eH6HsXXCBo+qy3Fn4AigBgAEaYrLCQEuAom6YbHyuKZAFYksqi -sSOFiRs0WDmlACk0CnpnaAeKiCS3BlwVkViJEbDS43lFNbLkZEmGhc305Nn4AMLGiUkBDiMTG5Vz -q4ZISjCofEfR1NpXijvP2X95Hu1e+zLalc0+mjeT3Z/FPGvt62WymbX2dXMDIYKDLjjP8n03RrPf -A1vOApwGOh2MgE2LpgZrgXLDF2CUJ15idG2J8GCSgcc2ZVRgA8+RHD0k2VJjg6mRUgGGhBWEyEcz -5EePLhUeWlYhoFCKONxUiBiIUiQeDIqiQwkjLiyqnF5eGs6a2gGRapbU9JRyuXAlPemYajlJojJd -GBBJjo5GxFRkITOAvLhSCr2TDz4uzdU8Yh3i/SHP4qh3vTG2s9198NP8M+pdR73BvIP6qPeDjzsW -gTi+jXrXWOe5P/jZxOeod/287v6JljzNP99RNM0a+/x4ljz3LNV2t5v9qHfW2Pyg24u54zSfObWX -Y9bYrCTHtwdfPPPOYiU5fvB5FssfNN2V5EIPfg9LnM+JhtVEO8+FZw5LXA068YNPhimu9sHPQiWv -qc6fE9BTnxIe/LTKatab+WYu7T74uWNRxJW5W5Ux0bDLuG1ioCwjg4DvGgBcgB8cUDHJ1RQ89neE -wvjbNUMiIZdo5hbHgEpANwMkDnL0Jr7kVFg+0pZKjBkmklNgBH1YI8dQOAAKbr6EF5wYM80KWnAd -nYAR`), +/Td6WFoAAATm1rRGBMCyBIAYIQEWAAAAAAAAABaHRszgC/8CKl0AFxNGhTWwfXmuDQEJlHgNLrkq +VxpJY6d9iRTt6gB4uCj0481rnYfXaUADHzOFuF3490RPrM6juPXrknqtVyuWJ5efW19BgwctN6xk +UiXiZaXVAWVWJWy2XHJiyYCMWBfIjUfo1ccOgwolwgFHJ64ZJjbayA3k6lYPcImuAqYL5NEVHpwl +Z8CWIjiXXSMQGsB3gxMdq9nySZbHQLK/KCKQ+oseF6kXyIgSEyuG4HhjVBBYIwTvWzI06kjNUXEy +2sw0n50uocLSAwJ/3mdX3n3XF5nmmuQMPtFbdQgQtC2VhyVd3TdIF+pT6zAEzXFJJ3uLkNbKSS88 +ZdBny6X/ftT5lQpNi/Wg0xLEQA4m4fu4fRAR0kOKzHM2svNLbTxa/wOPidqPzR6b/jfKmHkXxBNa +jFafty0a5K2S3F6JpwXZ2fqti/zG9NtMc+bbuXycC327EofXRXNtuOupELDD+ltTOIBF7CcTswyi +MZDP1PBie6GqDV2GuPz+0XXmul/ds+XysG19HIkKbJ+cQKp5o7Y0tI7EHM8GhwMl7MjgpQGj5nuv +0u2hqt4NXPNYqaMm9bFnnIUxEN82HgNWBcXf2baWKOdGzPzCuWg2fAM4zxHnBWcimxLXiJgaI8mU +J/QqTPWE0nJf1PW/J9yFQVR1Xo0TJyiX8/ObwmbqUPpxRGjKlYRBvn0jbTdUAENBSn+QVcASRGFE +SB9OM2B8Bg4jR/oojs8Beoq7zbIblgAAAACfRtXvhmznOgABzgSAGAAAKklb4rHEZ/sCAAAAAARZ +Wg==`), // this is tar.xz file } t.Run("RepositoryKey", func(t *testing.T) { @@ -105,155 +105,154 @@ nYAR`), require.Contains(t, resp.Body.String(), "-----BEGIN PGP PUBLIC KEY BLOCK-----") }) - t.Run("Upload", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - - req := NewRequestWithBody(t, "PUT", rootURL+"/default", bytes.NewReader(pkgs["any"])) - MakeRequest(t, req, http.StatusUnauthorized) - - req = NewRequestWithBody(t, "PUT", rootURL+"/default", bytes.NewReader(pkgs["any"])). - AddBasicAuth(user.Name) - MakeRequest(t, req, http.StatusCreated) - - pvs, err := packages.GetVersionsByPackageType(db.DefaultContext, user.ID, packages.TypeArch) - require.NoError(t, err) - require.Len(t, pvs, 1) - - pd, err := packages.GetPackageDescriptor(db.DefaultContext, pvs[0]) - require.NoError(t, err) - require.Nil(t, pd.SemVer) - require.IsType(t, &arch_model.VersionMetadata{}, pd.Metadata) - require.Equal(t, "test", pd.Package.Name) - require.Equal(t, "1.0.0-1", pd.Version.Version) - - pfs, err := packages.GetFilesByVersionID(db.DefaultContext, pvs[0].ID) - require.NoError(t, err) - require.Len(t, pfs, 2) // zst and zst.sig - require.True(t, pfs[0].IsLead) - - pb, err := packages.GetBlobByID(db.DefaultContext, pfs[0].BlobID) - require.NoError(t, err) - require.Equal(t, int64(len(pkgs["any"])), pb.Size) - - req = NewRequestWithBody(t, "PUT", rootURL+"/default", bytes.NewReader(pkgs["any"])). - AddBasicAuth(user.Name) - MakeRequest(t, req, http.StatusConflict) - req = NewRequestWithBody(t, "PUT", rootURL+"/default", bytes.NewReader(pkgs["x86_64"])). - AddBasicAuth(user.Name) - MakeRequest(t, req, http.StatusCreated) - req = NewRequestWithBody(t, "PUT", rootURL+"/other", bytes.NewReader(pkgs["any"])). - AddBasicAuth(user.Name) - MakeRequest(t, req, http.StatusCreated) - req = NewRequestWithBody(t, "PUT", rootURL+"/other", bytes.NewReader(pkgs["aarch64"])). - AddBasicAuth(user.Name) - MakeRequest(t, req, http.StatusCreated) - - req = NewRequestWithBody(t, "PUT", rootURL+"/base", bytes.NewReader(pkgs["other"])). - AddBasicAuth(user.Name) - MakeRequest(t, req, http.StatusCreated) - req = NewRequestWithBody(t, "PUT", rootURL+"/base", bytes.NewReader(pkgs["x86_64"])). - AddBasicAuth(user.Name) - MakeRequest(t, req, http.StatusCreated) - req = NewRequestWithBody(t, "PUT", rootURL+"/base", bytes.NewReader(pkgs["aarch64"])). - AddBasicAuth(user.Name) - MakeRequest(t, req, http.StatusCreated) - }) - - t.Run("Download", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - req := NewRequest(t, "GET", rootURL+"/default/x86_64/test-1.0.0-1-x86_64.pkg.tar.zst") - resp := MakeRequest(t, req, http.StatusOK) - require.Equal(t, pkgs["x86_64"], resp.Body.Bytes()) - - req = NewRequest(t, "GET", rootURL+"/default/x86_64/test-1.0.0-1-any.pkg.tar.zst") - resp = MakeRequest(t, req, http.StatusOK) - require.Equal(t, pkgs["any"], resp.Body.Bytes()) - - req = NewRequest(t, "GET", rootURL+"/default/x86_64/test-1.0.0-1-aarch64.pkg.tar.zst") - MakeRequest(t, req, http.StatusNotFound) + for _, group := range []string{"", "arch", "arch/os", "x86_64"} { + groupURL := rootURL + if group != "" { + groupURL = groupURL + "/" + group + } + t.Run(fmt.Sprintf("Upload[%s]", group), func(t *testing.T) { + defer tests.PrintCurrentTest(t)() - req = NewRequest(t, "GET", rootURL+"/other/x86_64/test-1.0.0-1-x86_64.pkg.tar.zst") - MakeRequest(t, req, http.StatusNotFound) + req := NewRequestWithBody(t, "PUT", groupURL, bytes.NewReader(pkgs["any"])) + MakeRequest(t, req, http.StatusUnauthorized) - req = NewRequest(t, "GET", rootURL+"/other/x86_64/test-1.0.0-1-any.pkg.tar.zst") - resp = MakeRequest(t, req, http.StatusOK) - require.Equal(t, pkgs["any"], resp.Body.Bytes()) - }) + req = NewRequestWithBody(t, "PUT", groupURL, bytes.NewReader(pkgs["any"])). + AddBasicAuth(user.Name) + MakeRequest(t, req, http.StatusCreated) - t.Run("SignVerify", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - req := NewRequest(t, "GET", rootURL+"/repository.key") - respPub := MakeRequest(t, req, http.StatusOK) + req = NewRequestWithBody(t, "PUT", groupURL, bytes.NewBuffer([]byte("any string"))). + AddBasicAuth(user.Name) + MakeRequest(t, req, http.StatusBadRequest) - req = NewRequest(t, "GET", rootURL+"/other/x86_64/test-1.0.0-1-any.pkg.tar.zst") - respPkg := MakeRequest(t, req, http.StatusOK) + pvs, err := packages.GetVersionsByPackageType(db.DefaultContext, user.ID, packages.TypeArch) + require.NoError(t, err) + require.Len(t, pvs, 1) - req = NewRequest(t, "GET", rootURL+"/other/x86_64/test-1.0.0-1-any.pkg.tar.zst.sig") - respSig := MakeRequest(t, req, http.StatusOK) + pd, err := packages.GetPackageDescriptor(db.DefaultContext, pvs[0]) + require.NoError(t, err) + require.Nil(t, pd.SemVer) + require.IsType(t, &arch_model.VersionMetadata{}, pd.Metadata) + require.Equal(t, "test", pd.Package.Name) + require.Equal(t, "1.0.0-1", pd.Version.Version) - if err := gpgVerify(respPub.Body.Bytes(), respSig.Body.Bytes(), respPkg.Body.Bytes()); err != nil { - t.Fatal(err) - } - }) + pfs, err := packages.GetFilesByVersionID(db.DefaultContext, pvs[0].ID) + require.NoError(t, err) + size := 0 + for _, pf := range pfs { + if pf.CompositeKey == group { + size++ + } + } + require.Equal(t, 2, size) // zst and zst.sig + + pb, err := packages.GetBlobByID(db.DefaultContext, pfs[0].BlobID) + require.NoError(t, err) + require.Equal(t, int64(len(pkgs["any"])), pb.Size) + + req = NewRequestWithBody(t, "PUT", groupURL, bytes.NewReader(pkgs["any"])). + AddBasicAuth(user.Name) // exists + MakeRequest(t, req, http.StatusConflict) + req = NewRequestWithBody(t, "PUT", groupURL, bytes.NewReader(pkgs["x86_64"])). + AddBasicAuth(user.Name) + MakeRequest(t, req, http.StatusCreated) + req = NewRequestWithBody(t, "PUT", groupURL, bytes.NewReader(pkgs["aarch64"])). + AddBasicAuth(user.Name) + MakeRequest(t, req, http.StatusCreated) + req = NewRequestWithBody(t, "PUT", groupURL, bytes.NewReader(pkgs["aarch64"])). + AddBasicAuth(user.Name) // exists again + MakeRequest(t, req, http.StatusConflict) + }) + + t.Run(fmt.Sprintf("Download[%s]", group), func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + req := NewRequest(t, "GET", groupURL+"/x86_64/test-1.0.0-1-x86_64.pkg.tar.zst") + resp := MakeRequest(t, req, http.StatusOK) + require.Equal(t, pkgs["x86_64"], resp.Body.Bytes()) + + req = NewRequest(t, "GET", groupURL+"/x86_64/test-1.0.0-1-any.pkg.tar.zst") + resp = MakeRequest(t, req, http.StatusOK) + require.Equal(t, pkgs["any"], resp.Body.Bytes()) + + // get other group + req = NewRequest(t, "GET", rootURL+"/unknown/x86_64/test-1.0.0-1-aarch64.pkg.tar.zst") + MakeRequest(t, req, http.StatusNotFound) + }) + + t.Run(fmt.Sprintf("SignVerify[%s]", group), func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + req := NewRequest(t, "GET", rootURL+"/repository.key") + respPub := MakeRequest(t, req, http.StatusOK) + + req = NewRequest(t, "GET", groupURL+"/x86_64/test-1.0.0-1-any.pkg.tar.zst") + respPkg := MakeRequest(t, req, http.StatusOK) + + req = NewRequest(t, "GET", groupURL+"/x86_64/test-1.0.0-1-any.pkg.tar.zst.sig") + respSig := MakeRequest(t, req, http.StatusOK) - t.Run("Repository", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - req := NewRequest(t, "GET", rootURL+"/repository.key") - respPub := MakeRequest(t, req, http.StatusOK) + if err := gpgVerify(respPub.Body.Bytes(), respSig.Body.Bytes(), respPkg.Body.Bytes()); err != nil { + t.Fatal(err) + } + }) - req = NewRequest(t, "GET", rootURL+"/base/x86_64/base.db") - respPkg := MakeRequest(t, req, http.StatusOK) + t.Run(fmt.Sprintf("RepositoryDB[%s]", group), func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + req := NewRequest(t, "GET", rootURL+"/repository.key") + respPub := MakeRequest(t, req, http.StatusOK) - req = NewRequest(t, "GET", rootURL+"/base/x86_64/base.db.sig") - respSig := MakeRequest(t, req, http.StatusOK) + req = NewRequest(t, "GET", groupURL+"/x86_64/base.db") + respPkg := MakeRequest(t, req, http.StatusOK) - if err := gpgVerify(respPub.Body.Bytes(), respSig.Body.Bytes(), respPkg.Body.Bytes()); err != nil { - t.Fatal(err) - } - files, err := listGzipFiles(respPkg.Body.Bytes()) - require.NoError(t, err) - require.Len(t, files, 2) - for s, d := range files { - name := getProperty(string(d.Data), "NAME") - ver := getProperty(string(d.Data), "VERSION") - require.Equal(t, name+"-"+ver+"/desc", s) - fn := getProperty(string(d.Data), "FILENAME") - pgp := getProperty(string(d.Data), "PGPSIG") - req = NewRequest(t, "GET", rootURL+"/base/x86_64/"+fn+".sig") + req = NewRequest(t, "GET", groupURL+"/x86_64/base.db.sig") respSig := MakeRequest(t, req, http.StatusOK) - decodeString, err := base64.StdEncoding.DecodeString(pgp) + + if err := gpgVerify(respPub.Body.Bytes(), respSig.Body.Bytes(), respPkg.Body.Bytes()); err != nil { + t.Fatal(err) + } + files, err := listTarGzFiles(respPkg.Body.Bytes()) require.NoError(t, err) - require.Equal(t, respSig.Body.Bytes(), decodeString) - } - }) - t.Run("Delete", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - req := NewRequestWithBody(t, "DELETE", rootURL+"/base/notfound/1.0.0-1", nil). - AddBasicAuth(user.Name) - MakeRequest(t, req, http.StatusNotFound) - - req = NewRequestWithBody(t, "DELETE", rootURL+"/base/test/1.0.0-1", nil). - AddBasicAuth(user.Name) - MakeRequest(t, req, http.StatusNoContent) - - req = NewRequest(t, "GET", rootURL+"/base/x86_64/base.db") - respPkg := MakeRequest(t, req, http.StatusOK) - files, err := listGzipFiles(respPkg.Body.Bytes()) - require.NoError(t, err) - require.Len(t, files, 1) - - req = NewRequestWithBody(t, "DELETE", rootURL+"/base/test2/1.0.0-1", nil). - AddBasicAuth(user.Name) - MakeRequest(t, req, http.StatusNoContent) - req = NewRequest(t, "GET", rootURL+"/base/x86_64/base.db") - MakeRequest(t, req, http.StatusNotFound) - - req = NewRequest(t, "GET", rootURL+"/default/x86_64/base.db") - respPkg = MakeRequest(t, req, http.StatusOK) - files, err = listGzipFiles(respPkg.Body.Bytes()) - require.NoError(t, err) - require.Len(t, files, 1) - }) + require.Len(t, files, 1) + for s, d := range files { + name := getProperty(string(d.Data), "NAME") + ver := getProperty(string(d.Data), "VERSION") + require.Equal(t, name+"-"+ver+"/desc", s) + fn := getProperty(string(d.Data), "FILENAME") + pgp := getProperty(string(d.Data), "PGPSIG") + req = NewRequest(t, "GET", groupURL+"/x86_64/"+fn+".sig") + respSig := MakeRequest(t, req, http.StatusOK) + decodeString, err := base64.StdEncoding.DecodeString(pgp) + require.NoError(t, err) + require.Equal(t, respSig.Body.Bytes(), decodeString) + } + }) + + t.Run(fmt.Sprintf("Delete[%s]", group), func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + // test data + req := NewRequestWithBody(t, "PUT", groupURL, bytes.NewReader(pkgs["other"])). + AddBasicAuth(user.Name) + MakeRequest(t, req, http.StatusCreated) + + req = NewRequestWithBody(t, "DELETE", rootURL+"/base/notfound/1.0.0-1", nil). + AddBasicAuth(user.Name) + MakeRequest(t, req, http.StatusNotFound) + + req = NewRequestWithBody(t, "DELETE", groupURL+"/test/1.0.0-1", nil). + AddBasicAuth(user.Name) + MakeRequest(t, req, http.StatusNoContent) + + req = NewRequest(t, "GET", groupURL+"/x86_64/base.db") + respPkg := MakeRequest(t, req, http.StatusOK) + files, err := listTarGzFiles(respPkg.Body.Bytes()) + require.NoError(t, err) + require.Len(t, files, 1) // other pkg in L225 + + req = NewRequestWithBody(t, "DELETE", groupURL+"/test2/1.0.0-1", nil). + AddBasicAuth(user.Name) + MakeRequest(t, req, http.StatusNoContent) + req = NewRequest(t, "GET", groupURL+"/x86_64/base.db") + MakeRequest(t, req, http.StatusNotFound) + }) + } } func getProperty(data, key string) string { @@ -270,7 +269,7 @@ func getProperty(data, key string) string { } } -func listGzipFiles(data []byte) (fstest.MapFS, error) { +func listTarGzFiles(data []byte) (fstest.MapFS, error) { reader, err := gzip.NewReader(bytes.NewBuffer(data)) defer reader.Close() if err != nil { From 97aefc5fd614d934cc700ec2e355def3d2eac0a3 Mon Sep 17 00:00:00 2001 From: dragon Date: Thu, 19 Sep 2024 09:29:07 +0800 Subject: [PATCH 03/21] fix anything v3 ... --- modules/packages/arch/metadata.go | 8 +++- modules/packages/arch/metadata_test.go | 2 +- tests/integration/api_packages_arch_test.go | 53 +++++++++++++++++++-- 3 files changed, 57 insertions(+), 6 deletions(-) diff --git a/modules/packages/arch/metadata.go b/modules/packages/arch/metadata.go index 9b443899bb4c..219cdcd702dd 100644 --- a/modules/packages/arch/metadata.go +++ b/modules/packages/arch/metadata.go @@ -39,11 +39,12 @@ const ( var ( reName = regexp.MustCompile(`^[a-zA-Z0-9@._+-]+$`) reVer = regexp.MustCompile(`^[a-zA-Z0-9:_.+]+-+[0-9]+$`) - reOptDep = regexp.MustCompile(`^[a-zA-Z0-9@._+-]+$|^[a-zA-Z0-9@._+-]+(:.*)`) - rePkgVer = regexp.MustCompile(`^[a-zA-Z0-9@._+-]+$|^[a-zA-Z0-9@._+-]+(>.*)|^[a-zA-Z0-9@._+-]+(<.*)|^[a-zA-Z0-9@._+-]+(=.*)`) + reOptDep = regexp.MustCompile(`^[a-zA-Z0-9@._+-]+([<>]?=?[a-zA-Z0-9@._+-]+)?(:.*)?$`) + rePkgVer = regexp.MustCompile(`^[a-zA-Z0-9@._+-]+([<>]?=?[a-zA-Z0-9@._+-]+)?$`) magicZSTD = []byte{0x28, 0xB5, 0x2F, 0xFD} magicXZ = []byte{0xFD, 0x37, 0x7A, 0x58, 0x5A} + magicGZ = []byte{0x1F, 0x8B} ) type Package struct { @@ -111,6 +112,9 @@ func ParsePackage(r *packages.HashedBuffer) (*Package, error) { } else if bytes.Equal(header[:len(magicXZ)], magicXZ) { tarballType = "xz" tarball = archiver.NewTarXz() + } else if bytes.Equal(header[:len(magicGZ)], magicGZ) { + tarballType = "gz" + tarball = archiver.NewTarGz() } else { return nil, errors.New("not supported compression") } diff --git a/modules/packages/arch/metadata_test.go b/modules/packages/arch/metadata_test.go index bd4c2a9ad87d..ddb35ca83738 100644 --- a/modules/packages/arch/metadata_test.go +++ b/modules/packages/arch/metadata_test.go @@ -199,7 +199,7 @@ func TestValidatePackageSpec(t *testing.T) { Provides: []string{"abc", "def"}, License: []string{"GPL"}, Depends: []string{"go", "gpg=1", "curl>=3", "git<=7"}, - OptDepends: []string{"git: something", "make"}, + OptDepends: []string{"git", "libgcc=1.0", "gzip>1.0", "gz>=1.0", "lz<1.0", "gzip<=1.0", "zstd>1.0:foo bar"}, MakeDepends: []string{"chrom"}, CheckDepends: []string{"bariy"}, Backup: []string{"etc/pacman.d/filo"}, diff --git a/tests/integration/api_packages_arch_test.go b/tests/integration/api_packages_arch_test.go index 74a6bb8bee76..af275de442f3 100644 --- a/tests/integration/api_packages_arch_test.go +++ b/tests/integration/api_packages_arch_test.go @@ -80,7 +80,7 @@ I2dkIEMjxsSOiAlJjH4HIwbTjayZJidXVxKQYH2gICOCBhK7KqMlLZ4gMCU1BapYlsTAXnywepyy jMBmtEhxyCnCZdUAwYKxAxeRFVk4TCL0aYgWjt3kHTg9SjVStppI2YCSWshUEFGdmJmyCVGpnqIU KNlA0hEjIOACGSLqYpXAD5SSNVT2MJRJwREAF4FRHPBlCJMSNwFguGAWDJBg+KIArkIJGNtCydUL TuN1oBh/+zKkEblAsgjGqVgUwKLP+UOMOGCpAhICtg6ncFJH`), - "other": unPack(` + "otherXZ": unPack(` /Td6WFoAAATm1rRGBMCyBIAYIQEWAAAAAAAAABaHRszgC/8CKl0AFxNGhTWwfXmuDQEJlHgNLrkq VxpJY6d9iRTt6gB4uCj0481rnYfXaUADHzOFuF3490RPrM6juPXrknqtVyuWJ5efW19BgwctN6xk UiXiZaXVAWVWJWy2XHJiyYCMWBfIjUfo1ccOgwolwgFHJ64ZJjbayA3k6lYPcImuAqYL5NEVHpwl @@ -92,7 +92,33 @@ MZDP1PBie6GqDV2GuPz+0XXmul/ds+XysG19HIkKbJ+cQKp5o7Y0tI7EHM8GhwMl7MjgpQGj5nuv 0u2hqt4NXPNYqaMm9bFnnIUxEN82HgNWBcXf2baWKOdGzPzCuWg2fAM4zxHnBWcimxLXiJgaI8mU J/QqTPWE0nJf1PW/J9yFQVR1Xo0TJyiX8/ObwmbqUPpxRGjKlYRBvn0jbTdUAENBSn+QVcASRGFE SB9OM2B8Bg4jR/oojs8Beoq7zbIblgAAAACfRtXvhmznOgABzgSAGAAAKklb4rHEZ/sCAAAAAARZ -Wg==`), // this is tar.xz file +Wg==`), + "otherZST": unPack(` +KLUv/QRYbRMABuOHS9BSNQdQ56F+xNFoV3CijY54JYt3VqV1iUU3xmj00y2pyBOCuokbhDYpvNsj +ZJeCxqH+nQFpMf4Wa92okaZoF4eH6HsXXCBo+qy3Fn4AigBgAEaYrLCQEuAom6YbHyuKZAFYksqi +sSOFiRs0WDmlACk0CnpnaAeKiCS3BlwVkViJEbDS43lFNbLkZEmGhc305Nn4AMLGiUkBDiMTG5Vz +q4ZISjCofEfR1NpXijvP2X95Hu1e+zLalc0+mjeT3Z/FPGvt62WymbX2dXMDIYKDLjjP8n03RrPf +A1vOApwGOh2MgE2LpgZrgXLDF2CUJ15idG2J8GCSgcc2ZVRgA8+RHD0k2VJjg6mRUgGGhBWEyEcz +5EePLhUeWlYhoFCKONxUiBiIUiQeDIqiQwkjLiyqnF5eGs6a2gGRapbU9JRyuXAlPemYajlJojJd +GBBJjo5GxFRkITOAvLhSCr2TDz4uzdU8Yh3i/SHP4qh3vTG2s9198NP8M+pdR73BvIP6qPeDjzsW +gTi+jXrXWOe5P/jZxOeod/287v6JljzNP99RNM0a+/x4ljz3LNV2t5v9qHfW2Pyg24u54zSfObWX +Y9bYrCTHtwdfPPPOYiU5fvB5FssfNN2V5EIPfg9LnM+JhtVEO8+FZw5LXA068YNPhimu9sHPQiWv +qc6fE9BTnxIe/LTKatab+WYu7T74uWNRxJW5W5Ux0bDLuG1ioCwjg4DvGgBcgB8cUDHJ1RQ89neE +wvjbNUMiIZdo5hbHgEpANwMkDnL0Jr7kVFg+0pZKjBkmklNgBH1YI8dQOAAKbr6EF5wYM80KWnAd +nYARrByncQ==`), + "otherGZ": unPack(` +H4sIAAAAAAAAA9PzDQlydWWgKTAwMDAzMVEA0UCAThsYGBuZKRiamBmbm5qZGJqbKBgYGpobGzMo +GNDWWRBQWlySWAR0SlF+fgk+dYTk0T03RIB8NweEwVx71tDviIFA60O75Rtc5s+9YbxteUHzhUWi +HBkWDcbGcUqCukrLGi4Lv8jIqNsbXhueXW8uzTe79Lr9/TVbnl69c3wR652f21+7rnU5kmjTc/38 +8t+zLx/+ePFr6lajpZ2dzCkyB3NPTxdVOfFk2/RXmq+Ktq2dbnY6RcPCMW8Kg9aGszs1f6+YsTlf +x5j5eIpXnXzStAbJvQvPP3su//3lu2/2pj++XO9hbJS+puPmqJKREff4X+RUqdYTbpGTBGYuefH9 +mNbGzKNdiUmS+xgt7J+5iTMObIgOLaAX4O3u6efmT0s7COV/UwNztPxvZGhqOpr/6QGUFdxT81KL +EktSUxSSKhVyE7NTC7LTFcz0DPUMuJQVSosz89IV0oCiIP8rlKUWFWfm5ykY6hmbcgHV5SXmpirY +KpSkFpcYgfhJicUIfkVKYkkikAcUL6ksSLUF0iA1QDOAgkDj9Qx0DUECKanFyVBNCgWJydmJ6alc +pUU5QKGMkpKCYit9/dSKxNyCnFS95Pxcfa6k0sycFKDRIIsMzQ0tTS2NDSxMuKA6QWaH5mXn5Zfn +KQRAhbiKM6tAqg24EouSM4CMxLxKrpzM5NQ8sGuTgUkgP5crOT8vDShYAhSpKs7gKijKL8sEOg2k +HMhNSS1IzUsBcpJAPFAwwUXSM0u4BjoaR8EoGAWjgGQAAILFeyQADAAA +`), } t.Run("RepositoryKey", func(t *testing.T) { @@ -228,7 +254,7 @@ Wg==`), // this is tar.xz file t.Run(fmt.Sprintf("Delete[%s]", group), func(t *testing.T) { defer tests.PrintCurrentTest(t)() // test data - req := NewRequestWithBody(t, "PUT", groupURL, bytes.NewReader(pkgs["other"])). + req := NewRequestWithBody(t, "PUT", groupURL, bytes.NewReader(pkgs["otherXZ"])). AddBasicAuth(user.Name) MakeRequest(t, req, http.StatusCreated) @@ -252,6 +278,27 @@ Wg==`), // this is tar.xz file req = NewRequest(t, "GET", groupURL+"/x86_64/base.db") MakeRequest(t, req, http.StatusNotFound) }) + + for tp, key := range map[string]string{ + "GZ": "otherGZ", + "XZ": "otherXZ", + "ZST": "otherZST", + } { + t.Run(fmt.Sprintf("Upload%s[%s]", tp, group), func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + req := NewRequestWithBody(t, "PUT", groupURL, bytes.NewReader(pkgs[key])). + AddBasicAuth(user.Name) + MakeRequest(t, req, http.StatusCreated) + + req = NewRequest(t, "GET", groupURL+"/x86_64/test2-1.0.0-1-any.pkg.tar."+strings.ToLower(tp)) + resp := MakeRequest(t, req, http.StatusOK) + require.Equal(t, pkgs[key], resp.Body.Bytes()) + + req = NewRequestWithBody(t, "DELETE", groupURL+"/test2/1.0.0-1", nil). + AddBasicAuth(user.Name) + MakeRequest(t, req, http.StatusNoContent) + }) + } } } From 2667260674487026b73b1bf33e7382e8f9d2e112 Mon Sep 17 00:00:00 2001 From: dragon Date: Thu, 19 Sep 2024 10:54:11 +0800 Subject: [PATCH 04/21] fix anything v4 ... --- modules/packages/arch/metadata.go | 16 ++++++------- routers/api/packages/api.go | 12 ++++++---- routers/api/packages/arch/arch.go | 16 +++++++++---- tests/integration/api_packages_arch_test.go | 26 ++++++++++++++++----- 4 files changed, 47 insertions(+), 23 deletions(-) diff --git a/modules/packages/arch/metadata.go b/modules/packages/arch/metadata.go index 219cdcd702dd..72e4be591340 100644 --- a/modules/packages/arch/metadata.go +++ b/modules/packages/arch/metadata.go @@ -262,42 +262,42 @@ func ValidatePackageSpec(p *Package) error { } for _, cd := range p.VersionMetadata.CheckDepends { if !rePkgVer.MatchString(cd) { - return util.NewInvalidArgumentErrorf("invalid check dependency: " + cd) + return util.NewInvalidArgumentErrorf("invalid check dependency: %s", cd) } } for _, d := range p.VersionMetadata.Depends { if !rePkgVer.MatchString(d) { - return util.NewInvalidArgumentErrorf("invalid dependency: " + d) + return util.NewInvalidArgumentErrorf("invalid dependency: %s", d) } } for _, md := range p.VersionMetadata.MakeDepends { if !rePkgVer.MatchString(md) { - return util.NewInvalidArgumentErrorf("invalid make dependency: " + md) + return util.NewInvalidArgumentErrorf("invalid make dependency: %s ", md) } } for _, p := range p.VersionMetadata.Provides { if !rePkgVer.MatchString(p) { - return util.NewInvalidArgumentErrorf("invalid provides: " + p) + return util.NewInvalidArgumentErrorf("invalid provides: %s", p) } } for _, p := range p.VersionMetadata.Conflicts { if !rePkgVer.MatchString(p) { - return util.NewInvalidArgumentErrorf("invalid conflicts: " + p) + return util.NewInvalidArgumentErrorf("invalid conflicts: %s", p) } } for _, p := range p.VersionMetadata.Replaces { if !rePkgVer.MatchString(p) { - return util.NewInvalidArgumentErrorf("invalid replaces: " + p) + return util.NewInvalidArgumentErrorf("invalid replaces: %s", p) } } for _, p := range p.VersionMetadata.Replaces { if !rePkgVer.MatchString(p) { - return util.NewInvalidArgumentErrorf("invalid xdata: " + p) + return util.NewInvalidArgumentErrorf("invalid xdata: %s", p) } } for _, od := range p.VersionMetadata.OptDepends { if !reOptDep.MatchString(od) { - return util.NewInvalidArgumentErrorf("invalid optional dependency: " + od) + return util.NewInvalidArgumentErrorf("invalid optional dependency: %s", od) } } for _, bf := range p.VersionMetadata.Backup { diff --git a/routers/api/packages/api.go b/routers/api/packages/api.go index b40477036e69..b3867807cf49 100644 --- a/routers/api/packages/api.go +++ b/routers/api/packages/api.go @@ -159,18 +159,20 @@ func CommonRoutes() *web.Router { arch.PushPackage(ctx) return } else if isDelete { - if groupLen < 2 { + if groupLen < 3 { ctx.Status(http.StatusBadRequest) return } - if groupLen == 2 { + if groupLen == 3 { ctx.SetPathParam("group", "") ctx.SetPathParam("package", pathGroups[0]) ctx.SetPathParam("version", pathGroups[1]) + ctx.SetPathParam("arch", pathGroups[2]) } else { - ctx.SetPathParam("group", strings.Join(pathGroups[:groupLen-2], "/")) - ctx.SetPathParam("package", pathGroups[groupLen-2]) - ctx.SetPathParam("version", pathGroups[groupLen-1]) + ctx.SetPathParam("group", strings.Join(pathGroups[:groupLen-3], "/")) + ctx.SetPathParam("package", pathGroups[groupLen-3]) + ctx.SetPathParam("version", pathGroups[groupLen-2]) + ctx.SetPathParam("arch", pathGroups[groupLen-1]) } reqPackageAccess(perm.AccessModeWrite)(ctx) if ctx.Written() { diff --git a/routers/api/packages/arch/arch.go b/routers/api/packages/arch/arch.go index 835820b40856..9521be377a29 100644 --- a/routers/api/packages/arch/arch.go +++ b/routers/api/packages/arch/arch.go @@ -9,6 +9,7 @@ import ( "fmt" "io" "net/http" + "path/filepath" "regexp" "strings" @@ -208,9 +209,10 @@ func GetPackageOrDB(ctx *context.Context) { func RemovePackage(ctx *context.Context) { var ( - group = ctx.PathParam("group") - pkg = ctx.PathParam("package") - ver = ctx.PathParam("version") + group = ctx.PathParam("group") + pkg = ctx.PathParam("package") + ver = ctx.PathParam("version") + pkgArch = ctx.PathParam("arch") ) pv, err := packages_model.GetVersionByNameAndVersion( ctx, ctx.Package.Owner.ID, packages_model.TypeArch, pkg, ver, @@ -230,7 +232,13 @@ func RemovePackage(ctx *context.Context) { } deleted := false for _, file := range files { - if file.CompositeKey == group { + extName := fmt.Sprintf("-%s.pkg.tar%s", pkgArch, filepath.Ext(file.LowerName)) + if strings.HasSuffix(file.LowerName, ".sig") { + extName = fmt.Sprintf("-%s.pkg.tar%s.sig", pkgArch, + filepath.Ext(strings.TrimSuffix(file.LowerName, filepath.Ext(file.LowerName)))) + } + if file.CompositeKey == group && + strings.HasSuffix(file.LowerName, extName) { deleted = true err := packages_service.RemovePackageFileAndVersionIfUnreferenced(ctx, ctx.ContextUser, file) if err != nil { diff --git a/tests/integration/api_packages_arch_test.go b/tests/integration/api_packages_arch_test.go index af275de442f3..291c6d3feaf2 100644 --- a/tests/integration/api_packages_arch_test.go +++ b/tests/integration/api_packages_arch_test.go @@ -258,11 +258,15 @@ HMhNSS1IzUsBcpJAPFAwwUXSM0u4BjoaR8EoGAWjgGQAAILFeyQADAAA AddBasicAuth(user.Name) MakeRequest(t, req, http.StatusCreated) - req = NewRequestWithBody(t, "DELETE", rootURL+"/base/notfound/1.0.0-1", nil). + req = NewRequestWithBody(t, "DELETE", rootURL+"/base/notfound/1.0.0-1/any", nil). AddBasicAuth(user.Name) MakeRequest(t, req, http.StatusNotFound) - req = NewRequestWithBody(t, "DELETE", groupURL+"/test/1.0.0-1", nil). + req = NewRequestWithBody(t, "DELETE", groupURL+"/test/1.0.0-1/x86_64", nil). + AddBasicAuth(user.Name) + MakeRequest(t, req, http.StatusNoContent) + + req = NewRequestWithBody(t, "DELETE", groupURL+"/test/1.0.0-1/any", nil). AddBasicAuth(user.Name) MakeRequest(t, req, http.StatusNoContent) @@ -270,12 +274,22 @@ HMhNSS1IzUsBcpJAPFAwwUXSM0u4BjoaR8EoGAWjgGQAAILFeyQADAAA respPkg := MakeRequest(t, req, http.StatusOK) files, err := listTarGzFiles(respPkg.Body.Bytes()) require.NoError(t, err) - require.Len(t, files, 1) // other pkg in L225 + require.Len(t, files, 1) - req = NewRequestWithBody(t, "DELETE", groupURL+"/test2/1.0.0-1", nil). + req = NewRequestWithBody(t, "DELETE", groupURL+"/test2/1.0.0-1/any", nil). AddBasicAuth(user.Name) MakeRequest(t, req, http.StatusNoContent) - req = NewRequest(t, "GET", groupURL+"/x86_64/base.db") + + req = NewRequest(t, "GET", groupURL+"/x86_64/base.db"). + AddBasicAuth(user.Name) + MakeRequest(t, req, http.StatusNotFound) + + req = NewRequestWithBody(t, "DELETE", groupURL+"/test/1.0.0-1/aarch64", nil). + AddBasicAuth(user.Name) + MakeRequest(t, req, http.StatusNoContent) + + req = NewRequest(t, "GET", groupURL+"/aarch64/base.db"). + AddBasicAuth(user.Name) MakeRequest(t, req, http.StatusNotFound) }) @@ -294,7 +308,7 @@ HMhNSS1IzUsBcpJAPFAwwUXSM0u4BjoaR8EoGAWjgGQAAILFeyQADAAA resp := MakeRequest(t, req, http.StatusOK) require.Equal(t, pkgs[key], resp.Body.Bytes()) - req = NewRequestWithBody(t, "DELETE", groupURL+"/test2/1.0.0-1", nil). + req = NewRequestWithBody(t, "DELETE", groupURL+"/test2/1.0.0-1/any", nil). AddBasicAuth(user.Name) MakeRequest(t, req, http.StatusNoContent) }) From 317ea7f5cd4e8b0ce9965c155211e3d4ce9f2a86 Mon Sep 17 00:00:00 2001 From: dragon Date: Fri, 20 Sep 2024 09:25:46 +0800 Subject: [PATCH 05/21] fix defer error --- modules/packages/arch/metadata.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/packages/arch/metadata.go b/modules/packages/arch/metadata.go index 72e4be591340..2029c637cd09 100644 --- a/modules/packages/arch/metadata.go +++ b/modules/packages/arch/metadata.go @@ -135,7 +135,6 @@ func ParsePackage(r *packages.HashedBuffer) (*Package, error) { if err != nil { return nil, err } - defer f.Close() switch f.Name() { case ".PKGINFO": @@ -146,6 +145,7 @@ func ParsePackage(r *packages.HashedBuffer) (*Package, error) { case ".MTREE": mtree = true } + _ = f.Close() } if pkg == nil { From fbe09b2e773440badce8069c9c61ff2199754a7b Mon Sep 17 00:00:00 2001 From: dragon Date: Fri, 20 Sep 2024 14:05:12 +0800 Subject: [PATCH 06/21] fix cache download --- routers/api/packages/arch/arch.go | 14 +++++--------- services/packages/arch/repository.go | 16 +++++++--------- 2 files changed, 12 insertions(+), 18 deletions(-) diff --git a/routers/api/packages/arch/arch.go b/routers/api/packages/arch/arch.go index 9521be377a29..f0f1370b551e 100644 --- a/routers/api/packages/arch/arch.go +++ b/routers/api/packages/arch/arch.go @@ -171,7 +171,7 @@ func GetPackageOrDB(ctx *context.Context) { ) if archPkgOrSig.MatchString(file) { - pkg, err := arch_service.GetPackageFile(ctx, group, file, ctx.Package.Owner.ID) + pkg, u, pf, err := arch_service.GetPackageFile(ctx, group, file, ctx.Package.Owner.ID) if err != nil { if errors.Is(err, util.ErrNotExist) { apiError(ctx, http.StatusNotFound, err) @@ -180,15 +180,12 @@ func GetPackageOrDB(ctx *context.Context) { } return } - - ctx.ServeContent(pkg, &context.ServeHeaderOptions{ - Filename: file, - }) + helper.ServePackageFile(ctx, pkg, u, pf) return } if archDBOrSig.MatchString(file) { - pkg, err := arch_service.GetPackageDBFile(ctx, group, arch, ctx.Package.Owner.ID, + pkg, u, pf, err := arch_service.GetPackageDBFile(ctx, group, arch, ctx.Package.Owner.ID, strings.HasSuffix(file, ".sig")) if err != nil { if errors.Is(err, util.ErrNotExist) { @@ -198,9 +195,8 @@ func GetPackageOrDB(ctx *context.Context) { } return } - ctx.ServeContent(pkg, &context.ServeHeaderOptions{ - Filename: file, - }) + + helper.ServePackageFile(ctx, pkg, u, pf) return } diff --git a/services/packages/arch/repository.go b/services/packages/arch/repository.go index a442b0c23f1b..e83616063054 100644 --- a/services/packages/arch/repository.go +++ b/services/packages/arch/repository.go @@ -245,14 +245,13 @@ func createDB(ctx context.Context, ownerID int64, group, arch string) (*packages // GetPackageFile Get data related to provided filename and distribution, for package files // update download counter. -func GetPackageFile(ctx context.Context, group, file string, ownerID int64) (io.ReadSeekCloser, error) { +func GetPackageFile(ctx context.Context, group, file string, ownerID int64) (io.ReadSeekCloser, *url.URL, *packages_model.PackageFile, error) { pf, err := getPackageFile(ctx, group, file, ownerID) if err != nil { - return nil, err + return nil, nil, nil, err } - filestream, _, _, err := packages_service.GetPackageFileStream(ctx, pf) - return filestream, err + return packages_service.GetPackageFileStream(ctx, pf) } // Ejects parameters required to get package file property from file name. @@ -275,10 +274,10 @@ func getPackageFile(ctx context.Context, group, file string, ownerID int64) (*pa return pkgfile, nil } -func GetPackageDBFile(ctx context.Context, group, arch string, ownerID int64, signFile bool) (io.ReadSeekCloser, error) { +func GetPackageDBFile(ctx context.Context, group, arch string, ownerID int64, signFile bool) (io.ReadSeekCloser, *url.URL, *packages_model.PackageFile, error) { pv, err := GetOrCreateRepositoryVersion(ctx, ownerID) if err != nil { - return nil, err + return nil, nil, nil, err } fileName := fmt.Sprintf("%s.db", arch) if signFile { @@ -286,10 +285,9 @@ func GetPackageDBFile(ctx context.Context, group, arch string, ownerID int64, si } file, err := packages_model.GetFileForVersionByName(ctx, pv.ID, fileName, group) if err != nil { - return nil, err + return nil, nil, nil, err } - filestream, _, _, err := packages_service.GetPackageFileStream(ctx, file) - return filestream, err + return packages_service.GetPackageFileStream(ctx, file) } // GetOrCreateKeyPair gets or creates the PGP keys used to sign repository metadata files From 0328200a1f73d87e74c0449cd7057f2d90f43699 Mon Sep 17 00:00:00 2001 From: dragon Date: Fri, 20 Sep 2024 17:21:02 +0800 Subject: [PATCH 07/21] fix some bugs --- modules/setting/packages.go | 1 - routers/api/packages/arch/arch.go | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/setting/packages.go b/modules/setting/packages.go index b5f08edf0ce1..3f618cfd6411 100644 --- a/modules/setting/packages.go +++ b/modules/setting/packages.go @@ -81,7 +81,6 @@ func loadPackagesFrom(rootCfg ConfigProvider) (err error) { Packages.LimitTotalOwnerSize = mustBytes(sec, "LIMIT_TOTAL_OWNER_SIZE") Packages.LimitSizeAlpine = mustBytes(sec, "LIMIT_SIZE_ALPINE") Packages.LimitSizeArch = mustBytes(sec, "LIMIT_SIZE_ARCH") - Packages.LimitSizeArch = mustBytes(sec, "LIMIT_SIZE_ARCH") Packages.LimitSizeCargo = mustBytes(sec, "LIMIT_SIZE_CARGO") Packages.LimitSizeChef = mustBytes(sec, "LIMIT_SIZE_CHEF") Packages.LimitSizeComposer = mustBytes(sec, "LIMIT_SIZE_COMPOSER") diff --git a/routers/api/packages/arch/arch.go b/routers/api/packages/arch/arch.go index f0f1370b551e..251c7713162d 100644 --- a/routers/api/packages/arch/arch.go +++ b/routers/api/packages/arch/arch.go @@ -155,6 +155,7 @@ func PushPackage(ctx *context.Context) { }) if err != nil { apiError(ctx, http.StatusInternalServerError, err) + return } if err = arch_service.BuildPacmanDB(ctx, ctx.Package.Owner.ID, group, p.FileMetadata.Arch); err != nil { apiError(ctx, http.StatusInternalServerError, err) @@ -247,6 +248,7 @@ func RemovePackage(ctx *context.Context) { err = arch_service.BuildCustomRepositoryFiles(ctx, ctx.Package.Owner.ID, group) if err != nil { apiError(ctx, http.StatusInternalServerError, err) + return } ctx.Status(http.StatusNoContent) } else { From 7b53f05c1a2b6afb8fa8fcc6e70c77e0c6d0dab5 Mon Sep 17 00:00:00 2001 From: dragon Date: Fri, 20 Sep 2024 17:25:59 +0800 Subject: [PATCH 08/21] fix ui bugs --- templates/package/metadata/arch.tmpl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/package/metadata/arch.tmpl b/templates/package/metadata/arch.tmpl index 822973eb7d98..89001b979c43 100644 --- a/templates/package/metadata/arch.tmpl +++ b/templates/package/metadata/arch.tmpl @@ -1,4 +1,4 @@ {{if eq .PackageDescriptor.Package.Type "arch"}} - {{range .PackageDescriptor.Metadata.License}}
{{svg "octicon-law" 16 "gt-mr-3"}} {{.}}
{{end}} - {{if .PackageDescriptor.Metadata.ProjectURL}}
{{svg "octicon-link-external" 16 "mr-3"}} {{ctx.Locale.Tr "packages.details.project_site"}}
{{end}} + {{if .PackageDescriptor.Metadata.ProjectURL}}
{{svg "octicon-link-external" 16 "tw-mr-2"}} {{ctx.Locale.Tr "packages.details.project_site"}}
{{end}} + {{range .PackageDescriptor.Metadata.License}}
{{svg "octicon-law" 16 "tw-mr-2"}} {{.}}
{{end}} {{end}} From c71fd97f3311d5911c42437bf739a14d4587a38f Mon Sep 17 00:00:00 2001 From: dragon Date: Mon, 23 Sep 2024 08:36:10 +0800 Subject: [PATCH 09/21] fix url typo --- templates/package/content/arch.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/package/content/arch.tmpl b/templates/package/content/arch.tmpl index bcc24b585bb5..134c6222115c 100644 --- a/templates/package/content/arch.tmpl +++ b/templates/package/content/arch.tmpl @@ -40,7 +40,7 @@ Server = + "https://docs.gitea.com/usage/packages/arch/"}}
From 5c2625a989f7de098f5a86ec621cd0268244970f Mon Sep 17 00:00:00 2001 From: dragon Date: Tue, 24 Sep 2024 09:50:48 +0800 Subject: [PATCH 10/21] fix typo and errors --- modules/packages/arch/metadata.go | 1 + services/packages/arch/repository.go | 17 +++++++++-------- tests/integration/api_packages_arch_test.go | 3 +++ 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/modules/packages/arch/metadata.go b/modules/packages/arch/metadata.go index 2029c637cd09..3296fb1e849c 100644 --- a/modules/packages/arch/metadata.go +++ b/modules/packages/arch/metadata.go @@ -140,6 +140,7 @@ func ParsePackage(r *packages.HashedBuffer) (*Package, error) { case ".PKGINFO": pkg, err = ParsePackageInfo(tarballType, f) if err != nil { + _ = f.Close() return nil, err } case ".MTREE": diff --git a/services/packages/arch/repository.go b/services/packages/arch/repository.go index e83616063054..b7d70634200f 100644 --- a/services/packages/arch/repository.go +++ b/services/packages/arch/repository.go @@ -175,15 +175,18 @@ func createDB(ctx context.Context, ownerID int64, group, arch string) (*packages if err != nil { return nil, err } + defer db.Close() gw := gzip.NewWriter(db) + defer gw.Close() tw := tar.NewWriter(gw) + defer tw.Close() count := 0 for _, pkg := range pkgs { versions, err := packages_model.GetVersionsByPackageName( ctx, ownerID, packages_model.TypeArch, pkg.Name, ) if err != nil { - return nil, errors.Join(tw.Close(), gw.Close(), db.Close(), err) + return nil, err } sort.Slice(versions, func(i, j int) bool { return versions[i].CreatedUnix > versions[j].CreatedUnix @@ -192,7 +195,7 @@ func createDB(ctx context.Context, ownerID int64, group, arch string) (*packages for _, ver := range versions { files, err := packages_model.GetFilesByVersionID(ctx, ver.ID) if err != nil { - return nil, errors.Join(tw.Close(), gw.Close(), db.Close(), err) + return nil, err } var pf *packages_model.PackageFile for _, file := range files { @@ -215,7 +218,7 @@ func createDB(ctx context.Context, ownerID int64, group, arch string) (*packages ctx, packages_model.PropertyTypeFile, pf.ID, arch_module.PropertyDescription, ) if err != nil { - return nil, errors.Join(tw.Close(), gw.Close(), db.Close(), err) + return nil, err } if len(pps) >= 1 { meta := []byte(pps[0].Value) @@ -225,20 +228,18 @@ func createDB(ctx context.Context, ownerID int64, group, arch string) (*packages Mode: int64(os.ModePerm), } if err = tw.WriteHeader(header); err != nil { - return nil, errors.Join(tw.Close(), gw.Close(), db.Close(), err) + return nil, err } if _, err := tw.Write(meta); err != nil { - return nil, errors.Join(tw.Close(), gw.Close(), db.Close(), err) + return nil, err } count++ break } } } - defer gw.Close() - defer tw.Close() if count == 0 { - return nil, errors.Join(db.Close(), io.EOF) + return nil, io.EOF } return db, nil } diff --git a/tests/integration/api_packages_arch_test.go b/tests/integration/api_packages_arch_test.go index 291c6d3feaf2..4750ceb0e73b 100644 --- a/tests/integration/api_packages_arch_test.go +++ b/tests/integration/api_packages_arch_test.go @@ -332,6 +332,9 @@ func getProperty(data, key string) string { func listTarGzFiles(data []byte) (fstest.MapFS, error) { reader, err := gzip.NewReader(bytes.NewBuffer(data)) + if err != nil { + return nil, err + } defer reader.Close() if err != nil { return nil, err From bc72567a415d4aedd112b1f8c0404fd796b040d0 Mon Sep 17 00:00:00 2001 From: dragon Date: Tue, 24 Sep 2024 10:07:33 +0800 Subject: [PATCH 11/21] add locker --- routers/api/packages/arch/arch.go | 18 +++++++++++++++++- services/packages/arch/repository.go | 6 ++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/routers/api/packages/arch/arch.go b/routers/api/packages/arch/arch.go index 251c7713162d..463c8870298e 100644 --- a/routers/api/packages/arch/arch.go +++ b/routers/api/packages/arch/arch.go @@ -14,6 +14,7 @@ import ( "strings" packages_model "code.gitea.io/gitea/models/packages" + "code.gitea.io/gitea/modules/globallock" packages_module "code.gitea.io/gitea/modules/packages" arch_module "code.gitea.io/gitea/modules/packages/arch" "code.gitea.io/gitea/modules/util" @@ -47,9 +48,18 @@ func GetRepositoryKey(ctx *context.Context) { }) } +func refreshLocker(ctx *context.Context, group string) (globallock.ReleaseFunc, error) { + return globallock.Lock(ctx, fmt.Sprintf("pkg_arch_pkg_%s", group)) +} + func PushPackage(ctx *context.Context) { group := ctx.PathParam("group") - + releaser, err := refreshLocker(ctx, group) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + defer releaser() upload, needToClose, err := ctx.UploadStream() if err != nil { apiError(ctx, http.StatusInternalServerError, err) @@ -211,6 +221,12 @@ func RemovePackage(ctx *context.Context) { ver = ctx.PathParam("version") pkgArch = ctx.PathParam("arch") ) + releaser, err := refreshLocker(ctx, group) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + defer releaser() pv, err := packages_model.GetVersionByNameAndVersion( ctx, ctx.Package.Owner.ID, packages_model.TypeArch, pkg, ver, ) diff --git a/services/packages/arch/repository.go b/services/packages/arch/repository.go index b7d70634200f..702f1a99503c 100644 --- a/services/packages/arch/repository.go +++ b/services/packages/arch/repository.go @@ -18,6 +18,7 @@ import ( packages_model "code.gitea.io/gitea/models/packages" user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/globallock" "code.gitea.io/gitea/modules/httplib" packages_module "code.gitea.io/gitea/modules/packages" arch_module "code.gitea.io/gitea/modules/packages/arch" @@ -103,6 +104,11 @@ func NewFileSign(ctx context.Context, ownerID int64, input io.Reader) (*packages // BuildPacmanDB Create db signature cache func BuildPacmanDB(ctx context.Context, ownerID int64, group, arch string) error { + lock, err := globallock.Lock(ctx, fmt.Sprintf("pkg_arch_db_%s", group)) + if err != nil { + return err + } + defer lock() pv, err := GetOrCreateRepositoryVersion(ctx, ownerID) if err != nil { return err From f4092d6f01564ac1f21cb57dc53ab6bb9ff5256b Mon Sep 17 00:00:00 2001 From: dragon Date: Tue, 24 Sep 2024 10:14:33 +0800 Subject: [PATCH 12/21] fix tests --- tests/integration/api_packages_arch_test.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/integration/api_packages_arch_test.go b/tests/integration/api_packages_arch_test.go index 4750ceb0e73b..1c75e11a25a6 100644 --- a/tests/integration/api_packages_arch_test.go +++ b/tests/integration/api_packages_arch_test.go @@ -336,9 +336,6 @@ func listTarGzFiles(data []byte) (fstest.MapFS, error) { return nil, err } defer reader.Close() - if err != nil { - return nil, err - } tarRead := tar.NewReader(reader) files := make(fstest.MapFS) for { From 3caa520ed94e8b1a52946fed27709a73db22a588 Mon Sep 17 00:00:00 2001 From: dragon Date: Tue, 24 Sep 2024 10:41:00 +0800 Subject: [PATCH 13/21] add comments. --- services/packages/arch/repository.go | 1 + 1 file changed, 1 insertion(+) diff --git a/services/packages/arch/repository.go b/services/packages/arch/repository.go index 702f1a99503c..835dfd971c93 100644 --- a/services/packages/arch/repository.go +++ b/services/packages/arch/repository.go @@ -104,6 +104,7 @@ func NewFileSign(ctx context.Context, ownerID int64, input io.Reader) (*packages // BuildPacmanDB Create db signature cache func BuildPacmanDB(ctx context.Context, ownerID int64, group, arch string) error { + // Prevent `update/delete` and `cron clean task` from running concurrently. lock, err := globallock.Lock(ctx, fmt.Sprintf("pkg_arch_db_%s", group)) if err != nil { return err From c35f354eb8e89557b426e9d0d0643abdf362f445 Mon Sep 17 00:00:00 2001 From: dragon Date: Tue, 24 Sep 2024 10:56:19 +0800 Subject: [PATCH 14/21] add Concurrent tests. --- tests/integration/api_packages_arch_test.go | 22 +++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/integration/api_packages_arch_test.go b/tests/integration/api_packages_arch_test.go index 1c75e11a25a6..2cf018641627 100644 --- a/tests/integration/api_packages_arch_test.go +++ b/tests/integration/api_packages_arch_test.go @@ -14,6 +14,7 @@ import ( "io" "net/http" "strings" + "sync" "testing" "testing/fstest" @@ -314,6 +315,27 @@ HMhNSS1IzUsBcpJAPFAwwUXSM0u4BjoaR8EoGAWjgGQAAILFeyQADAAA }) } } + t.Run("Concurrent Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + var wg sync.WaitGroup + + targets := []string{"any", "aarch64", "x86_64"} + for _, tag := range targets { + wg.Add(1) + go func(i string) { + defer wg.Done() + req := NewRequestWithBody(t, "PUT", rootURL, bytes.NewReader(pkgs[i])). + AddBasicAuth(user.Name) + MakeRequest(t, req, http.StatusCreated) + }(tag) + } + wg.Wait() + for _, target := range targets { + req := NewRequestWithBody(t, "DELETE", rootURL+"/test/1.0.0-1/"+target, nil). + AddBasicAuth(user.Name) + MakeRequest(t, req, http.StatusNoContent) + } + }) } func getProperty(data, key string) string { From 1b0a4e2b7b16844a1fa13537344059d796e2935b Mon Sep 17 00:00:00 2001 From: dragon Date: Tue, 24 Sep 2024 11:01:40 +0800 Subject: [PATCH 15/21] fix errors. --- modules/packages/arch/metadata.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/packages/arch/metadata.go b/modules/packages/arch/metadata.go index 3296fb1e849c..1f24b068feaf 100644 --- a/modules/packages/arch/metadata.go +++ b/modules/packages/arch/metadata.go @@ -291,7 +291,7 @@ func ValidatePackageSpec(p *Package) error { return util.NewInvalidArgumentErrorf("invalid replaces: %s", p) } } - for _, p := range p.VersionMetadata.Replaces { + for _, p := range p.VersionMetadata.Xdata { if !rePkgVer.MatchString(p) { return util.NewInvalidArgumentErrorf("invalid xdata: %s", p) } From 1b256e492f96576ce70781df319cdc6b8faaa647 Mon Sep 17 00:00:00 2001 From: dragon Date: Tue, 24 Sep 2024 11:21:06 +0800 Subject: [PATCH 16/21] fix errors. --- modules/packages/arch/metadata.go | 56 ++++++++++++++-------------- routers/api/packages/arch/arch.go | 2 +- services/packages/arch/repository.go | 2 +- 3 files changed, 30 insertions(+), 30 deletions(-) diff --git a/modules/packages/arch/metadata.go b/modules/packages/arch/metadata.go index 1f24b068feaf..791172642f46 100644 --- a/modules/packages/arch/metadata.go +++ b/modules/packages/arch/metadata.go @@ -71,7 +71,7 @@ type VersionMetadata struct { Conflicts []string `json:"conflicts,omitempty"` Replaces []string `json:"replaces,omitempty"` Backup []string `json:"backup,omitempty"` - Xdata []string `json:"xdata,omitempty"` + XData []string `json:"xdata,omitempty"` } // FileMetadata Metadata related to specific package file. @@ -221,7 +221,7 @@ func ParsePackageInfo(compressType string, r io.Reader) (*Package, error) { case "replaces": p.VersionMetadata.Replaces = append(p.VersionMetadata.Replaces, value) case "xdata": - p.VersionMetadata.Xdata = append(p.VersionMetadata.Xdata, value) + p.VersionMetadata.XData = append(p.VersionMetadata.XData, value) case "builddate": bd, err := strconv.ParseInt(value, 10, 64) if err != nil { @@ -261,48 +261,48 @@ func ValidatePackageSpec(p *Package) error { return util.NewInvalidArgumentErrorf("invalid project URL") } } - for _, cd := range p.VersionMetadata.CheckDepends { - if !rePkgVer.MatchString(cd) { - return util.NewInvalidArgumentErrorf("invalid check dependency: %s", cd) + for _, checkDepend := range p.VersionMetadata.CheckDepends { + if !rePkgVer.MatchString(checkDepend) { + return util.NewInvalidArgumentErrorf("invalid check dependency: %s", checkDepend) } } - for _, d := range p.VersionMetadata.Depends { - if !rePkgVer.MatchString(d) { - return util.NewInvalidArgumentErrorf("invalid dependency: %s", d) + for _, depend := range p.VersionMetadata.Depends { + if !rePkgVer.MatchString(depend) { + return util.NewInvalidArgumentErrorf("invalid dependency: %s", depend) } } - for _, md := range p.VersionMetadata.MakeDepends { - if !rePkgVer.MatchString(md) { - return util.NewInvalidArgumentErrorf("invalid make dependency: %s ", md) + for _, makeDepend := range p.VersionMetadata.MakeDepends { + if !rePkgVer.MatchString(makeDepend) { + return util.NewInvalidArgumentErrorf("invalid make dependency: %s ", makeDepend) } } - for _, p := range p.VersionMetadata.Provides { - if !rePkgVer.MatchString(p) { - return util.NewInvalidArgumentErrorf("invalid provides: %s", p) + for _, provide := range p.VersionMetadata.Provides { + if !rePkgVer.MatchString(provide) { + return util.NewInvalidArgumentErrorf("invalid provides: %s", provide) } } - for _, p := range p.VersionMetadata.Conflicts { - if !rePkgVer.MatchString(p) { - return util.NewInvalidArgumentErrorf("invalid conflicts: %s", p) + for _, conflict := range p.VersionMetadata.Conflicts { + if !rePkgVer.MatchString(conflict) { + return util.NewInvalidArgumentErrorf("invalid conflicts: %s", conflict) } } - for _, p := range p.VersionMetadata.Replaces { - if !rePkgVer.MatchString(p) { - return util.NewInvalidArgumentErrorf("invalid replaces: %s", p) + for _, replace := range p.VersionMetadata.Replaces { + if !rePkgVer.MatchString(replace) { + return util.NewInvalidArgumentErrorf("invalid replaces: %s", replace) } } - for _, p := range p.VersionMetadata.Xdata { - if !rePkgVer.MatchString(p) { - return util.NewInvalidArgumentErrorf("invalid xdata: %s", p) + for _, data := range p.VersionMetadata.XData { + if !rePkgVer.MatchString(data) { + return util.NewInvalidArgumentErrorf("invalid xdata: %s", data) } } - for _, od := range p.VersionMetadata.OptDepends { - if !reOptDep.MatchString(od) { - return util.NewInvalidArgumentErrorf("invalid optional dependency: %s", od) + for _, optDepend := range p.VersionMetadata.OptDepends { + if !reOptDep.MatchString(optDepend) { + return util.NewInvalidArgumentErrorf("invalid optional dependency: %s", optDepend) } } - for _, bf := range p.VersionMetadata.Backup { - if strings.HasPrefix(bf, "/") { + for _, backup := range p.VersionMetadata.Backup { + if strings.HasPrefix(backup, "/") { return util.NewInvalidArgumentErrorf("backup file contains leading forward slash") } } diff --git a/routers/api/packages/arch/arch.go b/routers/api/packages/arch/arch.go index 463c8870298e..37a8c7381918 100644 --- a/routers/api/packages/arch/arch.go +++ b/routers/api/packages/arch/arch.go @@ -49,7 +49,7 @@ func GetRepositoryKey(ctx *context.Context) { } func refreshLocker(ctx *context.Context, group string) (globallock.ReleaseFunc, error) { - return globallock.Lock(ctx, fmt.Sprintf("pkg_arch_pkg_%s", group)) + return globallock.Lock(ctx, fmt.Sprintf("pkg_%d_arch_pkg_%s", ctx.Package.Owner.ID, group)) } func PushPackage(ctx *context.Context) { diff --git a/services/packages/arch/repository.go b/services/packages/arch/repository.go index 835dfd971c93..d9fca4e926eb 100644 --- a/services/packages/arch/repository.go +++ b/services/packages/arch/repository.go @@ -105,7 +105,7 @@ func NewFileSign(ctx context.Context, ownerID int64, input io.Reader) (*packages // BuildPacmanDB Create db signature cache func BuildPacmanDB(ctx context.Context, ownerID int64, group, arch string) error { // Prevent `update/delete` and `cron clean task` from running concurrently. - lock, err := globallock.Lock(ctx, fmt.Sprintf("pkg_arch_db_%s", group)) + lock, err := globallock.Lock(ctx, fmt.Sprintf("pkg_%d_arch_db_%s", ownerID, group)) if err != nil { return err } From d80ca1f683125c52fec9d942add477e3b2df5e22 Mon Sep 17 00:00:00 2001 From: dragon Date: Tue, 24 Sep 2024 11:27:55 +0800 Subject: [PATCH 17/21] remove xdata check --- modules/packages/arch/metadata.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/modules/packages/arch/metadata.go b/modules/packages/arch/metadata.go index 791172642f46..e5ec82bae0c1 100644 --- a/modules/packages/arch/metadata.go +++ b/modules/packages/arch/metadata.go @@ -291,11 +291,6 @@ func ValidatePackageSpec(p *Package) error { return util.NewInvalidArgumentErrorf("invalid replaces: %s", replace) } } - for _, data := range p.VersionMetadata.XData { - if !rePkgVer.MatchString(data) { - return util.NewInvalidArgumentErrorf("invalid xdata: %s", data) - } - } for _, optDepend := range p.VersionMetadata.OptDepends { if !reOptDep.MatchString(optDepend) { return util.NewInvalidArgumentErrorf("invalid optional dependency: %s", optDepend) From 3c71f34de265be10daebbfcfd7e543ceecf336b5 Mon Sep 17 00:00:00 2001 From: dragon Date: Tue, 24 Sep 2024 11:38:11 +0800 Subject: [PATCH 18/21] add more version checks --- modules/packages/arch/metadata.go | 4 ++-- modules/packages/arch/metadata_test.go | 6 ++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/modules/packages/arch/metadata.go b/modules/packages/arch/metadata.go index e5ec82bae0c1..dd982a1648bc 100644 --- a/modules/packages/arch/metadata.go +++ b/modules/packages/arch/metadata.go @@ -39,8 +39,8 @@ const ( var ( reName = regexp.MustCompile(`^[a-zA-Z0-9@._+-]+$`) reVer = regexp.MustCompile(`^[a-zA-Z0-9:_.+]+-+[0-9]+$`) - reOptDep = regexp.MustCompile(`^[a-zA-Z0-9@._+-]+([<>]?=?[a-zA-Z0-9@._+-]+)?(:.*)?$`) - rePkgVer = regexp.MustCompile(`^[a-zA-Z0-9@._+-]+([<>]?=?[a-zA-Z0-9@._+-]+)?$`) + reOptDep = regexp.MustCompile(`^[a-zA-Z0-9@._+-]+([<>]?=?([0-9]+:)?[a-zA-Z0-9@._+-]+)?(:.*)?$`) + rePkgVer = regexp.MustCompile(`^[a-zA-Z0-9@._+-]+([<>]?=?([0-9]+:)?[a-zA-Z0-9@._+-]+)?$`) magicZSTD = []byte{0x28, 0xB5, 0x2F, 0xFD} magicXZ = []byte{0xFD, 0x37, 0x7A, 0x58, 0x5A} diff --git a/modules/packages/arch/metadata_test.go b/modules/packages/arch/metadata_test.go index ddb35ca83738..301a90032c87 100644 --- a/modules/packages/arch/metadata_test.go +++ b/modules/packages/arch/metadata_test.go @@ -445,3 +445,9 @@ dummy6 } require.Equal(t, pkgdesc, md.Desc()) } + +func TestOptVersionCheck(t *testing.T) { + for _, s := range []string{"foo", "foo>=1.0.0", "foo<=1.0.0", "foo>=1:1.0.0: aaa"} { + require.True(t, reOptDep.MatchString(s), s) + } +} From 568f7a60b1234bd39eca2932e59f77c2ebae1f56 Mon Sep 17 00:00:00 2001 From: ExplodingDragon Date: Wed, 25 Sep 2024 14:00:50 +0800 Subject: [PATCH 19/21] clean style --- modules/packages/arch/metadata.go | 56 +++++++++++++++++++++++-------- 1 file changed, 42 insertions(+), 14 deletions(-) diff --git a/modules/packages/arch/metadata.go b/modules/packages/arch/metadata.go index dd982a1648bc..ce89093c8235 100644 --- a/modules/packages/arch/metadata.go +++ b/modules/packages/arch/metadata.go @@ -42,11 +42,40 @@ var ( reOptDep = regexp.MustCompile(`^[a-zA-Z0-9@._+-]+([<>]?=?([0-9]+:)?[a-zA-Z0-9@._+-]+)?(:.*)?$`) rePkgVer = regexp.MustCompile(`^[a-zA-Z0-9@._+-]+([<>]?=?([0-9]+:)?[a-zA-Z0-9@._+-]+)?$`) - magicZSTD = []byte{0x28, 0xB5, 0x2F, 0xFD} - magicXZ = []byte{0xFD, 0x37, 0x7A, 0x58, 0x5A} - magicGZ = []byte{0x1F, 0x8B} + maxMagicLength = 0 + magics = map[string]struct { + magic []byte + archiver func() archiver.Reader + }{ + "zst": { + magic: []byte{0x28, 0xB5, 0x2F, 0xFD}, + archiver: func() archiver.Reader { + return archiver.NewTarZstd() + }, + }, + "xz": { + magic: []byte{0xFD, 0x37, 0x7A, 0x58, 0x5A}, + archiver: func() archiver.Reader { + return archiver.NewTarXz() + }, + }, + "gz": { + magic: []byte{0x1F, 0x8B}, + archiver: func() archiver.Reader { + return archiver.NewTarGz() + }, + }, + } ) +func init() { + for _, i := range magics { + if nLen := len(i.magic); nLen > maxMagicLength { + maxMagicLength = nLen + } + } +} + type Package struct { Name string `json:"name"` Version string `json:"version"` // Includes version, release and epoch @@ -94,7 +123,8 @@ func ParsePackage(r *packages.HashedBuffer) (*Package, error) { if err != nil { return nil, err } - header := make([]byte, 5) + + header := make([]byte, maxMagicLength) _, err = r.Read(header) if err != nil { return nil, err @@ -106,16 +136,14 @@ func ParsePackage(r *packages.HashedBuffer) (*Package, error) { var tarball archiver.Reader var tarballType string - if bytes.Equal(header[:len(magicZSTD)], magicZSTD) { - tarballType = "zst" - tarball = archiver.NewTarZstd() - } else if bytes.Equal(header[:len(magicXZ)], magicXZ) { - tarballType = "xz" - tarball = archiver.NewTarXz() - } else if bytes.Equal(header[:len(magicGZ)], magicGZ) { - tarballType = "gz" - tarball = archiver.NewTarGz() - } else { + for tarType, info := range magics { + if bytes.Equal(header[:len(info.magic)], info.magic) { + tarballType = tarType + tarball = info.archiver() + break + } + } + if tarballType == "" || tarball == nil { return nil, errors.New("not supported compression") } err = tarball.Open(r, 0) From 0b95ef22deeb96497ef8001fe6b61965ab16169e Mon Sep 17 00:00:00 2001 From: ExplodingDragon Date: Thu, 26 Sep 2024 13:46:23 +0800 Subject: [PATCH 20/21] fix bugs --- services/packages/arch/repository.go | 29 +++++++++++----------------- 1 file changed, 11 insertions(+), 18 deletions(-) diff --git a/services/packages/arch/repository.go b/services/packages/arch/repository.go index d9fca4e926eb..a326ac325a75 100644 --- a/services/packages/arch/repository.go +++ b/services/packages/arch/repository.go @@ -254,32 +254,25 @@ func createDB(ctx context.Context, ownerID int64, group, arch string) (*packages // GetPackageFile Get data related to provided filename and distribution, for package files // update download counter. func GetPackageFile(ctx context.Context, group, file string, ownerID int64) (io.ReadSeekCloser, *url.URL, *packages_model.PackageFile, error) { - pf, err := getPackageFile(ctx, group, file, ownerID) - if err != nil { - return nil, nil, nil, err + fileSplit := strings.Split(file, "-") + if len(fileSplit) <= 3 { + return nil, nil, nil, errors.New("invalid file format, need ---.pkg.") } - - return packages_service.GetPackageFileStream(ctx, pf) -} - -// Ejects parameters required to get package file property from file name. -func getPackageFile(ctx context.Context, group, file string, ownerID int64) (*packages_model.PackageFile, error) { var ( - splt = strings.Split(file, "-") - pkgname = strings.Join(splt[0:len(splt)-3], "-") - vername = splt[len(splt)-3] + "-" + splt[len(splt)-2] + pkgName = strings.Join(fileSplit[0:len(fileSplit)-3], "-") + pkgVer = fileSplit[len(fileSplit)-3] + "-" + fileSplit[len(fileSplit)-2] ) - - version, err := packages_model.GetVersionByNameAndVersion(ctx, ownerID, packages_model.TypeArch, pkgname, vername) + version, err := packages_model.GetVersionByNameAndVersion(ctx, ownerID, packages_model.TypeArch, pkgName, pkgVer) if err != nil { - return nil, err + return nil, nil, nil, err } - pkgfile, err := packages_model.GetFileForVersionByName(ctx, version.ID, file, group) + pkgFile, err := packages_model.GetFileForVersionByName(ctx, version.ID, file, group) if err != nil { - return nil, err + return nil, nil, nil, err } - return pkgfile, nil + + return packages_service.GetPackageFileStream(ctx, pkgFile) } func GetPackageDBFile(ctx context.Context, group, arch string, ownerID int64, signFile bool) (io.ReadSeekCloser, *url.URL, *packages_model.PackageFile, error) { From 07c2b6d41d278028905d42d1f503045f64b93831 Mon Sep 17 00:00:00 2001 From: ExplodingDragon Date: Thu, 26 Sep 2024 13:50:04 +0800 Subject: [PATCH 21/21] fix typo --- templates/package/content/arch.tmpl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/package/content/arch.tmpl b/templates/package/content/arch.tmpl index 134c6222115c..8e964d5effe7 100644 --- a/templates/package/content/arch.tmpl +++ b/templates/package/content/arch.tmpl @@ -16,11 +16,11 @@ pacman-key --lsign-key '{{$.SignMail}}' 

 {{- if gt (len $.Groups) 1 -}}
-# {{ctx.Locale.Tr "packages.arch.pacman.repo.multi"  $.PackageDescriptor.Package.LowerName}}
+# {{ctx.Locale.Tr "packages.arch.pacman.repo.multi" $.PackageDescriptor.Package.LowerName}}
 
 {{end -}}
 {{- $GroupSize := (len .Groups) -}}
-{{-  range $i,$v :=  .Groups -}}
+{{- range $i,$v := .Groups -}}
 {{- if gt $i 0}}
 {{end -}}{{- if gt $GroupSize 1 -}}
 # {{ctx.Locale.Tr "packages.arch.pacman.repo.multi.item" .}}