From 7d6bbbc6fcf7b2b1ca2f47a99f21d252619377ce Mon Sep 17 00:00:00 2001 From: Adam Martin Date: Mon, 2 Oct 2023 21:15:02 -0400 Subject: [PATCH 01/11] pull back in ocil --- cmd/hauler/cli/download/download.go | 2 +- cmd/hauler/cli/store/copy.go | 2 +- cmd/hauler/cli/store/extract.go | 2 +- cmd/hauler/cli/store/flags.go | 4 +- cmd/hauler/cli/store/load.go | 4 +- cmd/hauler/cli/store/serve.go | 2 +- internal/mapper/mappers.go | 2 +- pkg/artifacts/config.go | 92 +++++++++++++ pkg/artifacts/file/file.go | 116 ++++++++++++++++ pkg/artifacts/file/file_test.go | 166 +++++++++++++++++++++++ pkg/artifacts/file/getter/directory.go | 165 ++++++++++++++++++++++ pkg/artifacts/file/getter/file.go | 53 ++++++++ pkg/artifacts/file/getter/getter.go | 148 ++++++++++++++++++++ pkg/artifacts/file/getter/getter_test.go | 139 +++++++++++++++++++ pkg/artifacts/file/getter/https.go | 67 +++++++++ pkg/artifacts/file/options.go | 26 ++++ pkg/artifacts/image/image.go | 53 ++++++++ pkg/artifacts/image/image_test.go | 1 + pkg/artifacts/memory/memory.go | 78 +++++++++++ pkg/artifacts/memory/memory_test.go | 61 +++++++++ pkg/artifacts/memory/options.go | 17 +++ pkg/artifacts/ocis.go | 21 +++ pkg/collection/chart/chart.go | 4 +- pkg/collection/imagetxt/imagetxt.go | 6 +- pkg/collection/imagetxt/imagetxt_test.go | 4 +- pkg/collection/k3s/k3s.go | 8 +- pkg/content/chart/chart.go | 9 +- pkg/content/chart/chart_test.go | 2 +- pkg/layer/cache.go | 106 +++++++++++++++ pkg/layer/filesystem.go | 118 ++++++++++++++++ pkg/layer/layer.go | 127 +++++++++++++++++ pkg/store/store_test.go | 105 ++++++++++++++ 32 files changed, 1685 insertions(+), 25 deletions(-) create mode 100644 pkg/artifacts/config.go create mode 100644 pkg/artifacts/file/file.go create mode 100644 pkg/artifacts/file/file_test.go create mode 100644 pkg/artifacts/file/getter/directory.go create mode 100644 pkg/artifacts/file/getter/file.go create mode 100644 pkg/artifacts/file/getter/getter.go create mode 100644 pkg/artifacts/file/getter/getter_test.go create mode 100644 pkg/artifacts/file/getter/https.go create mode 100644 pkg/artifacts/file/options.go create mode 100644 pkg/artifacts/image/image.go create mode 100644 pkg/artifacts/image/image_test.go create mode 100644 pkg/artifacts/memory/memory.go create mode 100644 pkg/artifacts/memory/memory_test.go create mode 100644 pkg/artifacts/memory/options.go create mode 100644 pkg/artifacts/ocis.go create mode 100644 pkg/layer/cache.go create mode 100644 pkg/layer/filesystem.go create mode 100644 pkg/layer/layer.go create mode 100644 pkg/store/store_test.go diff --git a/cmd/hauler/cli/download/download.go b/cmd/hauler/cli/download/download.go index b3ec0425..d63f1f66 100644 --- a/cmd/hauler/cli/download/download.go +++ b/cmd/hauler/cli/download/download.go @@ -11,7 +11,7 @@ import ( "oras.land/oras-go/pkg/content" "oras.land/oras-go/pkg/oras" - "github.com/rancherfederal/ocil/pkg/consts" + "github.com/rancherfederal/hauler/pkg/consts" "github.com/rancherfederal/hauler/internal/mapper" "github.com/rancherfederal/hauler/pkg/log" diff --git a/cmd/hauler/cli/store/copy.go b/cmd/hauler/cli/store/copy.go index ac270d0f..0eafb123 100644 --- a/cmd/hauler/cli/store/copy.go +++ b/cmd/hauler/cli/store/copy.go @@ -9,7 +9,7 @@ import ( "github.com/spf13/cobra" "oras.land/oras-go/pkg/content" - "github.com/rancherfederal/ocil/pkg/store" + "github.com/rancherfederal/hauler/pkg/store" "github.com/rancherfederal/hauler/pkg/log" "github.com/rancherfederal/hauler/pkg/reference" diff --git a/cmd/hauler/cli/store/extract.go b/cmd/hauler/cli/store/extract.go index de29b92e..9b09755a 100644 --- a/cmd/hauler/cli/store/extract.go +++ b/cmd/hauler/cli/store/extract.go @@ -8,7 +8,7 @@ import ( ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/spf13/cobra" - "github.com/rancherfederal/ocil/pkg/store" + "github.com/rancherfederal/hauler/pkg/store" "github.com/rancherfederal/hauler/internal/mapper" "github.com/rancherfederal/hauler/pkg/log" diff --git a/cmd/hauler/cli/store/flags.go b/cmd/hauler/cli/store/flags.go index dbb1b7ff..3fcb27d1 100644 --- a/cmd/hauler/cli/store/flags.go +++ b/cmd/hauler/cli/store/flags.go @@ -6,8 +6,8 @@ import ( "os" "path/filepath" - "github.com/rancherfederal/ocil/pkg/layer" - "github.com/rancherfederal/ocil/pkg/store" + "github.com/rancherfederal/hauler/pkg/layer" + "github.com/rancherfederal/hauler/pkg/store" "github.com/spf13/cobra" "github.com/rancherfederal/hauler/pkg/log" diff --git a/cmd/hauler/cli/store/load.go b/cmd/hauler/cli/store/load.go index a9a4774f..5de6a94b 100644 --- a/cmd/hauler/cli/store/load.go +++ b/cmd/hauler/cli/store/load.go @@ -5,8 +5,8 @@ import ( "os" "github.com/mholt/archiver/v3" - "github.com/rancherfederal/ocil/pkg/content" - "github.com/rancherfederal/ocil/pkg/store" + "github.com/rancherfederal/hauler/pkg/content" + "github.com/rancherfederal/hauler/pkg/store" "github.com/spf13/cobra" "github.com/rancherfederal/hauler/pkg/log" diff --git a/cmd/hauler/cli/store/serve.go b/cmd/hauler/cli/store/serve.go index 6f394419..a6ebad7e 100644 --- a/cmd/hauler/cli/store/serve.go +++ b/cmd/hauler/cli/store/serve.go @@ -14,7 +14,7 @@ import ( "github.com/distribution/distribution/v3/version" "github.com/spf13/cobra" - "github.com/rancherfederal/ocil/pkg/store" + "github.com/rancherfederal/hauler/pkg/store" "github.com/rancherfederal/hauler/internal/server" ) diff --git a/internal/mapper/mappers.go b/internal/mapper/mappers.go index 1cc60d48..05cf89bb 100644 --- a/internal/mapper/mappers.go +++ b/internal/mapper/mappers.go @@ -6,7 +6,7 @@ import ( ocispec "github.com/opencontainers/image-spec/specs-go/v1" "oras.land/oras-go/pkg/target" - "github.com/rancherfederal/ocil/pkg/consts" + "github.com/rancherfederal/hauler/pkg/consts" ) type Fn func(desc ocispec.Descriptor) (string, error) diff --git a/pkg/artifacts/config.go b/pkg/artifacts/config.go new file mode 100644 index 00000000..b25bb441 --- /dev/null +++ b/pkg/artifacts/config.go @@ -0,0 +1,92 @@ +package artifacts + +import ( + "bytes" + "encoding/json" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/partial" + "github.com/google/go-containerregistry/pkg/v1/types" + + "github.com/rancherfederal/hauler/pkg/consts" +) + +var _ partial.Describable = (*marshallableConfig)(nil) + +type Config interface { + // Raw returns the config bytes + Raw() ([]byte, error) + + Digest() (v1.Hash, error) + + MediaType() (types.MediaType, error) + + Size() (int64, error) +} + +type Marshallable interface{} + +type ConfigOption func(*marshallableConfig) + +// ToConfig takes anything that is marshallabe and converts it into a Config +func ToConfig(i Marshallable, opts ...ConfigOption) Config { + mc := &marshallableConfig{Marshallable: i} + for _, o := range opts { + o(mc) + } + return mc +} + +func WithConfigMediaType(mediaType string) ConfigOption { + return func(config *marshallableConfig) { + config.mediaType = mediaType + } +} + +// marshallableConfig implements Config using helper methods +type marshallableConfig struct { + Marshallable + + mediaType string +} + +func (c *marshallableConfig) MediaType() (types.MediaType, error) { + mt := c.mediaType + if mt == "" { + mt = consts.UnknownManifest + } + return types.MediaType(mt), nil +} + +func (c *marshallableConfig) Raw() ([]byte, error) { + return json.Marshal(c.Marshallable) +} + +func (c *marshallableConfig) Digest() (v1.Hash, error) { + return Digest(c) +} + +func (c *marshallableConfig) Size() (int64, error) { + return Size(c) +} + +type WithRawConfig interface { + Raw() ([]byte, error) +} + +func Digest(c WithRawConfig) (v1.Hash, error) { + b, err := c.Raw() + if err != nil { + return v1.Hash{}, err + } + digest, _, err := v1.SHA256(bytes.NewReader(b)) + return digest, err +} + +func Size(c WithRawConfig) (int64, error) { + b, err := c.Raw() + if err != nil { + return -1, err + } + return int64(len(b)), nil +} diff --git a/pkg/artifacts/file/file.go b/pkg/artifacts/file/file.go new file mode 100644 index 00000000..3be515ef --- /dev/null +++ b/pkg/artifacts/file/file.go @@ -0,0 +1,116 @@ +package file + +import ( + "context" + + gv1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/partial" + gtypes "github.com/google/go-containerregistry/pkg/v1/types" + + "github.com/rancherfederal/hauler/pkg/artifacts" + "github.com/rancherfederal/hauler/pkg/artifacts/file/getter" + "github.com/rancherfederal/hauler/pkg/consts" +) + +// interface guard +var _ artifacts.OCI = (*File)(nil) + +// File implements the OCI interface for File API objects. API spec information is +// stored into the Path field. +type File struct { + Path string + + computed bool + client *getter.Client + config artifacts.Config + blob gv1.Layer + manifest *gv1.Manifest + annotations map[string]string +} + +func NewFile(path string, opts ...Option) *File { + client := getter.NewClient(getter.ClientOptions{}) + + f := &File{ + client: client, + Path: path, + } + + for _, opt := range opts { + opt(f) + } + return f +} + +// Name is the name of the file's reference +func (f *File) Name(path string) string { + return f.client.Name(path) +} + +func (f *File) MediaType() string { + return consts.OCIManifestSchema1 +} + +func (f *File) RawConfig() ([]byte, error) { + if err := f.compute(); err != nil { + return nil, err + } + return f.config.Raw() +} + +func (f *File) Layers() ([]gv1.Layer, error) { + if err := f.compute(); err != nil { + return nil, err + } + var layers []gv1.Layer + layers = append(layers, f.blob) + return layers, nil +} + +func (f *File) Manifest() (*gv1.Manifest, error) { + if err := f.compute(); err != nil { + return nil, err + } + return f.manifest, nil +} + +func (f *File) compute() error { + if f.computed { + return nil + } + + ctx := context.TODO() + blob, err := f.client.LayerFrom(ctx, f.Path) + if err != nil { + return err + } + + layer, err := partial.Descriptor(blob) + if err != nil { + return err + } + + cfg := f.client.Config(f.Path) + if cfg == nil { + cfg = f.client.Config(f.Path) + } + + cfgDesc, err := partial.Descriptor(cfg) + if err != nil { + return err + } + + m := &gv1.Manifest{ + SchemaVersion: 2, + MediaType: gtypes.MediaType(f.MediaType()), + Config: *cfgDesc, + Layers: []gv1.Descriptor{*layer}, + Annotations: f.annotations, + } + + f.manifest = m + f.config = cfg + f.blob = blob + f.computed = true + return nil +} diff --git a/pkg/artifacts/file/file_test.go b/pkg/artifacts/file/file_test.go new file mode 100644 index 00000000..729b5c4f --- /dev/null +++ b/pkg/artifacts/file/file_test.go @@ -0,0 +1,166 @@ +package file_test + +import ( + "bytes" + "context" + "io" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path/filepath" + "testing" + + "github.com/spf13/afero" + + "github.com/rancherfederal/hauler/pkg/artifacts/file" + "github.com/rancherfederal/hauler/pkg/artifacts/file/getter" + "github.com/rancherfederal/hauler/pkg/consts" +) + +var ( + filename = "myfile.yaml" + data = []byte(`data`) + + ts *httptest.Server + tfs afero.Fs + mc *getter.Client +) + +func TestMain(m *testing.M) { + teardown := setup() + defer teardown() + code := m.Run() + os.Exit(code) +} + +func Test_file_Config(t *testing.T) { + tests := []struct { + name string + ref string + want string + wantErr bool + }{ + { + name: "should properly type local file", + ref: filename, + want: consts.FileLocalConfigMediaType, + wantErr: false, + }, + { + name: "should properly type remote file", + ref: ts.URL + "/" + filename, + want: consts.FileHttpConfigMediaType, + wantErr: false, + }, + // TODO: Add directory test + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + f := file.NewFile(tt.ref, file.WithClient(mc)) + + f.MediaType() + + m, err := f.Manifest() + if err != nil { + t.Fatal(err) + } + + got := string(m.Config.MediaType) + if got != tt.want { + t.Errorf("unxpected mediatype; got %s, want %s", got, tt.want) + } + }) + } +} + +func Test_file_Layers(t *testing.T) { + tests := []struct { + name string + ref string + want []byte + wantErr bool + }{ + { + name: "should load a local file and preserve contents", + ref: filename, + want: data, + wantErr: false, + }, + { + name: "should load a remote file and preserve contents", + ref: ts.URL + "/" + filename, + want: data, + wantErr: false, + }, + // TODO: Add directory test + } + for _, tt := range tests { + t.Run(tt.name, func(it *testing.T) { + f := file.NewFile(tt.ref, file.WithClient(mc)) + + layers, err := f.Layers() + if (err != nil) != tt.wantErr { + it.Fatalf("unexpected Layers() error: got %v, want %v", err, tt.wantErr) + } + + rc, err := layers[0].Compressed() + if err != nil { + it.Fatal(err) + } + + got, err := io.ReadAll(rc) + if err != nil { + it.Fatal(err) + } + + if !bytes.Equal(got, tt.want) { + it.Fatalf("unexpected Layers(): got %v, want %v", layers, tt.want) + } + }) + } +} + +func setup() func() { + tfs = afero.NewMemMapFs() + afero.WriteFile(tfs, filename, data, 0644) + + mf := &mockFile{File: getter.NewFile(), fs: tfs} + + mockHttp := getter.NewHttp() + mhttp := afero.NewHttpFs(tfs) + fileserver := http.FileServer(mhttp.Dir(".")) + http.Handle("/", fileserver) + ts = httptest.NewServer(fileserver) + + mc = &getter.Client{ + Options: getter.ClientOptions{}, + Getters: map[string]getter.Getter{ + "file": mf, + "http": mockHttp, + }, + } + + teardown := func() { + defer ts.Close() + } + + return teardown +} + +type mockFile struct { + *getter.File + fs afero.Fs +} + +func (m mockFile) Open(ctx context.Context, u *url.URL) (io.ReadCloser, error) { + return m.fs.Open(filepath.Join(u.Host, u.Path)) +} + +func (m mockFile) Detect(u *url.URL) bool { + fi, err := m.fs.Stat(filepath.Join(u.Host, u.Path)) + if err != nil { + return false + } + return !fi.IsDir() +} diff --git a/pkg/artifacts/file/getter/directory.go b/pkg/artifacts/file/getter/directory.go new file mode 100644 index 00000000..ab640d89 --- /dev/null +++ b/pkg/artifacts/file/getter/directory.go @@ -0,0 +1,165 @@ +package getter + +import ( + "archive/tar" + "compress/gzip" + "context" + "io" + "net/url" + "os" + "path/filepath" + "time" + + "github.com/opencontainers/go-digest" + "github.com/pkg/errors" + + "github.com/rancherfederal/hauler/pkg/artifacts" + "github.com/rancherfederal/hauler/pkg/consts" +) + +type directory struct { + *File +} + +func NewDirectory() *directory { + return &directory{File: NewFile()} +} + +func (d directory) Open(ctx context.Context, u *url.URL) (io.ReadCloser, error) { + tmpfile, err := os.CreateTemp("", "hauler") + if err != nil { + return nil, err + } + + digester := digest.Canonical.Digester() + zw := gzip.NewWriter(io.MultiWriter(tmpfile, digester.Hash())) + defer zw.Close() + + tarDigester := digest.Canonical.Digester() + if err := tarDir(d.path(u), d.Name(u), io.MultiWriter(zw, tarDigester.Hash()), false); err != nil { + return nil, err + } + + if err := zw.Close(); err != nil { + return nil, err + } + if err := tmpfile.Sync(); err != nil { + return nil, err + } + + fi, err := os.Open(tmpfile.Name()) + if err != nil { + return nil, err + } + + // rc := &closer{ + // t: io.TeeReader(tmpfile, fi), + // closes: []func() error{fi.Close, tmpfile.Close, zw.Close}, + // } + return fi, nil +} + +func (d directory) Detect(u *url.URL) bool { + if len(d.path(u)) == 0 { + return false + } + + fi, err := os.Stat(d.path(u)) + if err != nil { + return false + } + return fi.IsDir() +} + +func (d directory) Config(u *url.URL) artifacts.Config { + c := &directoryConfig{ + config{Reference: u.String()}, + } + return artifacts.ToConfig(c, artifacts.WithConfigMediaType(consts.FileDirectoryConfigMediaType)) +} + +type directoryConfig struct { + config `json:",inline,omitempty"` +} + +func tarDir(root string, prefix string, w io.Writer, stripTimes bool) error { + tw := tar.NewWriter(w) + defer tw.Close() + if err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + // Rename path + name, err := filepath.Rel(root, path) + if err != nil { + return err + } + name = filepath.Join(prefix, name) + name = filepath.ToSlash(name) + + // Generate header + var link string + mode := info.Mode() + if mode&os.ModeSymlink != 0 { + if link, err = os.Readlink(path); err != nil { + return err + } + } + header, err := tar.FileInfoHeader(info, link) + if err != nil { + return errors.Wrap(err, path) + } + header.Name = name + header.Uid = 0 + header.Gid = 0 + header.Uname = "" + header.Gname = "" + + if stripTimes { + header.ModTime = time.Time{} + header.AccessTime = time.Time{} + header.ChangeTime = time.Time{} + } + + // Write file + if err := tw.WriteHeader(header); err != nil { + return errors.Wrap(err, "tar") + } + if mode.IsRegular() { + file, err := os.Open(path) + if err != nil { + return err + } + defer file.Close() + if _, err := io.Copy(tw, file); err != nil { + return errors.Wrap(err, path) + } + } + + return nil + }); err != nil { + return err + } + return nil +} + +type closer struct { + t io.Reader + closes []func() error +} + +func (c *closer) Read(p []byte) (n int, err error) { + return c.t.Read(p) +} + +func (c *closer) Close() error { + var err error + for _, c := range c.closes { + lastErr := c() + if err == nil { + err = lastErr + } + } + return err +} diff --git a/pkg/artifacts/file/getter/file.go b/pkg/artifacts/file/getter/file.go new file mode 100644 index 00000000..222a80cf --- /dev/null +++ b/pkg/artifacts/file/getter/file.go @@ -0,0 +1,53 @@ +package getter + +import ( + "context" + "io" + "net/url" + "os" + "path/filepath" + + "github.com/rancherfederal/hauler/pkg/artifacts" + "github.com/rancherfederal/hauler/pkg/consts" +) + +type File struct{} + +func NewFile() *File { + return &File{} +} + +func (f File) Name(u *url.URL) string { + return filepath.Base(f.path(u)) +} + +func (f File) Open(ctx context.Context, u *url.URL) (io.ReadCloser, error) { + return os.Open(f.path(u)) +} + +func (f File) Detect(u *url.URL) bool { + if len(f.path(u)) == 0 { + return false + } + + fi, err := os.Stat(f.path(u)) + if err != nil { + return false + } + return !fi.IsDir() +} + +func (f File) path(u *url.URL) string { + return filepath.Join(u.Host, u.Path) +} + +func (f File) Config(u *url.URL) artifacts.Config { + c := &fileConfig{ + config{Reference: u.String()}, + } + return artifacts.ToConfig(c, artifacts.WithConfigMediaType(consts.FileLocalConfigMediaType)) +} + +type fileConfig struct { + config `json:",inline,omitempty"` +} diff --git a/pkg/artifacts/file/getter/getter.go b/pkg/artifacts/file/getter/getter.go new file mode 100644 index 00000000..ff2e0cf4 --- /dev/null +++ b/pkg/artifacts/file/getter/getter.go @@ -0,0 +1,148 @@ +package getter + +import ( + "context" + "fmt" + "io" + "net/url" + + v1 "github.com/google/go-containerregistry/pkg/v1" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/pkg/errors" + "oras.land/oras-go/pkg/content" + + content2 "github.com/rancherfederal/hauler/pkg/artifacts" + "github.com/rancherfederal/hauler/pkg/consts" + "github.com/rancherfederal/hauler/pkg/layer" +) + +type Client struct { + Getters map[string]Getter + Options ClientOptions +} + +// ClientOptions provides options for the client +type ClientOptions struct { + NameOverride string +} + +var ( + ErrGetterTypeUnknown = errors.New("no getter type found matching reference") +) + +type Getter interface { + Open(context.Context, *url.URL) (io.ReadCloser, error) + + Detect(*url.URL) bool + + Name(*url.URL) string + + Config(*url.URL) content2.Config +} + +func NewClient(opts ClientOptions) *Client { + defaults := map[string]Getter{ + "file": NewFile(), + "directory": NewDirectory(), + "http": NewHttp(), + } + + c := &Client{ + Getters: defaults, + Options: opts, + } + return c +} + +func (c *Client) LayerFrom(ctx context.Context, source string) (v1.Layer, error) { + u, err := url.Parse(source) + if err != nil { + return nil, err + } + + g, err := c.getterFrom(u) + if err != nil { + if errors.Is(err, ErrGetterTypeUnknown) { + return nil, err + } + return nil, fmt.Errorf("create getter: %w", err) + } + + opener := func() (io.ReadCloser, error) { + return g.Open(ctx, u) + } + + annotations := make(map[string]string) + annotations[ocispec.AnnotationTitle] = c.Name(source) + + switch g.(type) { + case *directory: + annotations[content.AnnotationUnpack] = "true" + } + + l, err := layer.FromOpener(opener, + layer.WithMediaType(consts.FileLayerMediaType), + layer.WithAnnotations(annotations)) + if err != nil { + return nil, err + } + return l, nil +} + +func (c *Client) ContentFrom(ctx context.Context, source string) (io.ReadCloser, error) { + u, err := url.Parse(source) + if err != nil { + return nil, fmt.Errorf("parse source %s: %w", source, err) + } + g, err := c.getterFrom(u) + if err != nil { + if errors.Is(err, ErrGetterTypeUnknown) { + return nil, err + } + return nil, fmt.Errorf("create getter: %w", err) + } + return g.Open(ctx, u) +} + +func (c *Client) getterFrom(srcUrl *url.URL) (Getter, error) { + for _, g := range c.Getters { + if g.Detect(srcUrl) { + return g, nil + } + } + return nil, errors.Wrapf(ErrGetterTypeUnknown, "source %s", srcUrl.String()) +} + +func (c *Client) Name(source string) string { + if c.Options.NameOverride != "" { + return c.Options.NameOverride + } + u, err := url.Parse(source) + if err != nil { + return source + } + for _, g := range c.Getters { + if g.Detect(u) { + return g.Name(u) + } + } + return source +} + +func (c *Client) Config(source string) content2.Config { + u, err := url.Parse(source) + if err != nil { + return nil + } + for _, g := range c.Getters { + if g.Detect(u) { + return g.Config(u) + } + } + return nil +} + +type config struct { + Reference string `json:"reference"` + Annotations map[string]string `json:"annotations,omitempty"` +} diff --git a/pkg/artifacts/file/getter/getter_test.go b/pkg/artifacts/file/getter/getter_test.go new file mode 100644 index 00000000..e0da7585 --- /dev/null +++ b/pkg/artifacts/file/getter/getter_test.go @@ -0,0 +1,139 @@ +package getter_test + +import ( + "net/url" + "os" + "path/filepath" + "testing" + + "github.com/rancherfederal/hauler/pkg/artifacts/file/getter" +) + +func TestClient_Detect(t *testing.T) { + teardown := setup(t) + defer teardown() + + c := getter.NewClient(getter.ClientOptions{}) + + type args struct { + source string + } + tests := []struct { + name string + args args + want string + }{ + { + name: "should identify a file", + args: args{ + source: fileWithExt, + }, + want: "file", + }, + { + name: "should identify a directory", + args: args{ + source: rootDir, + }, + want: "directory", + }, + { + name: "should identify a http", + args: args{ + source: "http://my.cool.website", + }, + want: "http", + }, + { + name: "should identify a http", + args: args{ + source: "https://my.cool.website", + }, + want: "http", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := identify(c, tt.args.source); got != tt.want { + t.Errorf("identify() = %v, want %v", got, tt.want) + } + }) + } +} + +func identify(c *getter.Client, source string) string { + u, _ := url.Parse(source) + for t, g := range c.Getters { + if g.Detect(u) { + return t + } + } + return "" +} + +func TestClient_Name(t *testing.T) { + teardown := setup(t) + defer teardown() + + type args struct { + source string + opts getter.ClientOptions + } + tests := []struct { + name string + args args + want string + }{ + { + name: "should correctly name a file with an extension", + args: args{ + source: fileWithExt, + opts: getter.ClientOptions{}, + }, + want: "file.yaml", + }, + { + name: "should correctly name a directory", + args: args{ + source: rootDir, + opts: getter.ClientOptions{}, + }, + want: rootDir, + }, + { + name: "should correctly override a files name", + args: args{ + source: fileWithExt, + opts: getter.ClientOptions{NameOverride: "myfile"}, + }, + want: "myfile", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := getter.NewClient(tt.args.opts) + if got := c.Name(tt.args.source); got != tt.want { + t.Errorf("Name() = %v, want %v", got, tt.want) + } + }) + } +} + +var ( + rootDir = "gettertests" + fileWithExt = filepath.Join(rootDir, "file.yaml") +) + +func setup(t *testing.T) func() { + if err := os.MkdirAll(rootDir, os.ModePerm); err != nil { + t.Fatal(err) + } + + if err := os.WriteFile(fileWithExt, []byte(""), 0644); err != nil { + t.Fatal(err) + } + + return func() { + os.RemoveAll(rootDir) + } +} diff --git a/pkg/artifacts/file/getter/https.go b/pkg/artifacts/file/getter/https.go new file mode 100644 index 00000000..3fb11284 --- /dev/null +++ b/pkg/artifacts/file/getter/https.go @@ -0,0 +1,67 @@ +package getter + +import ( + "context" + "io" + "mime" + "net/http" + "net/url" + "path/filepath" + "strings" + + "github.com/rancherfederal/hauler/pkg/artifacts" + "github.com/rancherfederal/hauler/pkg/consts" +) + +type Http struct{} + +func NewHttp() *Http { + return &Http{} +} + +func (h Http) Name(u *url.URL) string { + resp, err := http.Head(u.String()) + if err != nil { + return "" + } + + contentType := resp.Header.Get("Content-Type") + for _, v := range strings.Split(contentType, ",") { + t, _, err := mime.ParseMediaType(v) + if err != nil { + break + } + // TODO: Identify known mimetypes for hints at a filename + _ = t + } + + // TODO: Not this + return filepath.Base(u.String()) +} + +func (h Http) Open(ctx context.Context, u *url.URL) (io.ReadCloser, error) { + resp, err := http.Get(u.String()) + if err != nil { + return nil, err + } + return resp.Body, nil +} + +func (h Http) Detect(u *url.URL) bool { + switch u.Scheme { + case "http", "https": + return true + } + return false +} + +func (h *Http) Config(u *url.URL) artifacts.Config { + c := &httpConfig{ + config{Reference: u.String()}, + } + return artifacts.ToConfig(c, artifacts.WithConfigMediaType(consts.FileHttpConfigMediaType)) +} + +type httpConfig struct { + config `json:",inline,omitempty"` +} diff --git a/pkg/artifacts/file/options.go b/pkg/artifacts/file/options.go new file mode 100644 index 00000000..3efdd63b --- /dev/null +++ b/pkg/artifacts/file/options.go @@ -0,0 +1,26 @@ +package file + +import ( + "github.com/rancherfederal/hauler/pkg/artifacts" + "github.com/rancherfederal/hauler/pkg/artifacts/file/getter" +) + +type Option func(*File) + +func WithClient(c *getter.Client) Option { + return func(f *File) { + f.client = c + } +} + +func WithConfig(obj interface{}, mediaType string) Option { + return func(f *File) { + f.config = artifacts.ToConfig(obj, artifacts.WithConfigMediaType(mediaType)) + } +} + +func WithAnnotations(m map[string]string) Option { + return func(f *File) { + f.annotations = m + } +} diff --git a/pkg/artifacts/image/image.go b/pkg/artifacts/image/image.go new file mode 100644 index 00000000..a4a0cfd3 --- /dev/null +++ b/pkg/artifacts/image/image.go @@ -0,0 +1,53 @@ +package image + +import ( + "github.com/google/go-containerregistry/pkg/authn" + gname "github.com/google/go-containerregistry/pkg/name" + gv1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/remote" + + "github.com/rancherfederal/hauler/pkg/artifacts" +) + +var _ artifacts.OCI = (*Image)(nil) + +func (i *Image) MediaType() string { + mt, err := i.Image.MediaType() + if err != nil { + return "" + } + return string(mt) +} + +func (i *Image) RawConfig() ([]byte, error) { + return i.RawConfigFile() +} + +// Image implements the OCI interface for Image API objects. API spec information +// is stored into the Name field. +type Image struct { + Name string + gv1.Image +} + +func NewImage(name string, opts ...remote.Option) (*Image, error) { + r, err := gname.ParseReference(name) + if err != nil { + return nil, err + } + + defaultOpts := []remote.Option{ + remote.WithAuthFromKeychain(authn.DefaultKeychain), + } + opts = append(opts, defaultOpts...) + + img, err := remote.Image(r, opts...) + if err != nil { + return nil, err + } + + return &Image{ + Name: name, + Image: img, + }, nil +} diff --git a/pkg/artifacts/image/image_test.go b/pkg/artifacts/image/image_test.go new file mode 100644 index 00000000..aa66c334 --- /dev/null +++ b/pkg/artifacts/image/image_test.go @@ -0,0 +1 @@ +package image_test diff --git a/pkg/artifacts/memory/memory.go b/pkg/artifacts/memory/memory.go new file mode 100644 index 00000000..05bb7529 --- /dev/null +++ b/pkg/artifacts/memory/memory.go @@ -0,0 +1,78 @@ +package memory + +import ( + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/partial" + "github.com/google/go-containerregistry/pkg/v1/static" + "github.com/google/go-containerregistry/pkg/v1/types" + + "github.com/rancherfederal/hauler/pkg/artifacts" + "github.com/rancherfederal/hauler/pkg/consts" +) + +var _ artifacts.OCI = (*Memory)(nil) + +// Memory implements the OCI interface for a generic set of bytes stored in memory. +type Memory struct { + blob v1.Layer + annotations map[string]string + config artifacts.Config +} + +type defaultConfig struct { + MediaType string `json:"mediaType,omitempty"` +} + +func NewMemory(data []byte, mt string, opts ...Option) *Memory { + blob := static.NewLayer(data, types.MediaType(mt)) + + cfg := defaultConfig{MediaType: consts.MemoryConfigMediaType} + m := &Memory{ + blob: blob, + config: artifacts.ToConfig(cfg), + } + + for _, opt := range opts { + opt(m) + } + return m +} + +func (m *Memory) MediaType() string { + return consts.OCIManifestSchema1 +} + +func (m *Memory) Manifest() (*v1.Manifest, error) { + layer, err := partial.Descriptor(m.blob) + if err != nil { + return nil, err + } + + cfgDesc, err := partial.Descriptor(m.config) + if err != nil { + return nil, err + } + + manifest := &v1.Manifest{ + SchemaVersion: 2, + MediaType: types.MediaType(m.MediaType()), + Config: *cfgDesc, + Layers: []v1.Descriptor{*layer}, + Annotations: m.annotations, + } + + return manifest, nil +} + +func (m *Memory) RawConfig() ([]byte, error) { + if m.config == nil { + return []byte(`{}`), nil + } + return m.config.Raw() +} + +func (m *Memory) Layers() ([]v1.Layer, error) { + var layers []v1.Layer + layers = append(layers, m.blob) + return layers, nil +} diff --git a/pkg/artifacts/memory/memory_test.go b/pkg/artifacts/memory/memory_test.go new file mode 100644 index 00000000..529ce9d0 --- /dev/null +++ b/pkg/artifacts/memory/memory_test.go @@ -0,0 +1,61 @@ +package memory_test + +import ( + "math/rand" + "testing" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/opencontainers/go-digest" + + "github.com/rancherfederal/hauler/pkg/artifacts/memory" +) + +func TestMemory_Layers(t *testing.T) { + tests := []struct { + name string + want *v1.Manifest + wantErr bool + }{ + { + name: "should preserve content", + want: nil, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + data, m := setup(t) + + layers, err := m.Layers() + if err != nil { + t.Fatal(err) + } + + if len(layers) != 1 { + t.Fatalf("Expected 1 layer, got %d", len(layers)) + } + + h, err := layers[0].Digest() + if err != nil { + t.Fatal(err) + } + + d := digest.FromBytes(data) + + if d.String() != h.String() { + t.Fatalf("bytes do not match, got %s, expected %s", h.String(), d.String()) + } + }) + } +} + +func setup(t *testing.T) ([]byte, *memory.Memory) { + block := make([]byte, 2048) + _, err := rand.Read(block) + if err != nil { + t.Fatal(err) + } + + mem := memory.NewMemory(block, "random") + return block, mem +} diff --git a/pkg/artifacts/memory/options.go b/pkg/artifacts/memory/options.go new file mode 100644 index 00000000..032aa3cc --- /dev/null +++ b/pkg/artifacts/memory/options.go @@ -0,0 +1,17 @@ +package memory + +import "github.com/rancherfederal/hauler/pkg/artifacts" + +type Option func(*Memory) + +func WithConfig(obj interface{}, mediaType string) Option { + return func(m *Memory) { + m.config = artifacts.ToConfig(obj, artifacts.WithConfigMediaType(mediaType)) + } +} + +func WithAnnotations(annotations map[string]string) Option { + return func(m *Memory) { + m.annotations = annotations + } +} diff --git a/pkg/artifacts/ocis.go b/pkg/artifacts/ocis.go new file mode 100644 index 00000000..c1fe25be --- /dev/null +++ b/pkg/artifacts/ocis.go @@ -0,0 +1,21 @@ +package artifacts + +import "github.com/google/go-containerregistry/pkg/v1" + +// OCI is the bare minimum we need to represent an artifact in an oci layout +// At a high level, it is not constrained by an Image's config, manifests, and layer ordinality +// This specific implementation fully encapsulates v1.Layer's within a more generic form +type OCI interface { + MediaType() string + + Manifest() (*v1.Manifest, error) + + RawConfig() ([]byte, error) + + Layers() ([]v1.Layer, error) +} + +type OCICollection interface { + // Contents returns the list of contents in the collection + Contents() (map[string]OCI, error) +} diff --git a/pkg/collection/chart/chart.go b/pkg/collection/chart/chart.go index 032ce401..05be6c34 100644 --- a/pkg/collection/chart/chart.go +++ b/pkg/collection/chart/chart.go @@ -1,8 +1,8 @@ package chart import ( - "github.com/rancherfederal/ocil/pkg/artifacts" - "github.com/rancherfederal/ocil/pkg/artifacts/image" + "github.com/rancherfederal/hauler/pkg/artifacts" + "github.com/rancherfederal/hauler/pkg/artifacts/image" "helm.sh/helm/v3/pkg/action" "github.com/rancherfederal/hauler/pkg/apis/hauler.cattle.io/v1alpha1" diff --git a/pkg/collection/imagetxt/imagetxt.go b/pkg/collection/imagetxt/imagetxt.go index 0143a7c4..b9fc4055 100644 --- a/pkg/collection/imagetxt/imagetxt.go +++ b/pkg/collection/imagetxt/imagetxt.go @@ -12,9 +12,9 @@ import ( "github.com/rancherfederal/hauler/pkg/log" "github.com/google/go-containerregistry/pkg/name" - artifact "github.com/rancherfederal/ocil/pkg/artifacts" - "github.com/rancherfederal/ocil/pkg/artifacts/file/getter" - "github.com/rancherfederal/ocil/pkg/artifacts/image" + artifact "github.com/rancherfederal/hauler/pkg/artifacts" + "github.com/rancherfederal/hauler/pkg/artifacts/file/getter" + "github.com/rancherfederal/hauler/pkg/artifacts/image" ) type ImageTxt struct { diff --git a/pkg/collection/imagetxt/imagetxt_test.go b/pkg/collection/imagetxt/imagetxt_test.go index 0dfa8efa..8dcee138 100644 --- a/pkg/collection/imagetxt/imagetxt_test.go +++ b/pkg/collection/imagetxt/imagetxt_test.go @@ -8,8 +8,8 @@ import ( "os" "testing" - "github.com/rancherfederal/ocil/pkg/artifacts" - "github.com/rancherfederal/ocil/pkg/artifacts/image" + "github.com/rancherfederal/hauler/pkg/artifacts" + "github.com/rancherfederal/hauler/pkg/artifacts/image" ) var ( diff --git a/pkg/collection/k3s/k3s.go b/pkg/collection/k3s/k3s.go index 28aae224..890db13a 100644 --- a/pkg/collection/k3s/k3s.go +++ b/pkg/collection/k3s/k3s.go @@ -10,12 +10,12 @@ import ( "path" "strings" - "github.com/rancherfederal/ocil/pkg/artifacts" - "github.com/rancherfederal/ocil/pkg/artifacts/image" + "github.com/rancherfederal/hauler/pkg/artifacts" + "github.com/rancherfederal/hauler/pkg/artifacts/image" - "github.com/rancherfederal/ocil/pkg/artifacts/file" + "github.com/rancherfederal/hauler/pkg/artifacts/file" - "github.com/rancherfederal/ocil/pkg/artifacts/file/getter" + "github.com/rancherfederal/hauler/pkg/artifacts/file/getter" "github.com/rancherfederal/hauler/pkg/reference" ) diff --git a/pkg/content/chart/chart.go b/pkg/content/chart/chart.go index 329cf126..ae01d65a 100644 --- a/pkg/content/chart/chart.go +++ b/pkg/content/chart/chart.go @@ -14,15 +14,15 @@ import ( "github.com/google/go-containerregistry/pkg/v1/partial" gtypes "github.com/google/go-containerregistry/pkg/v1/types" ocispec "github.com/opencontainers/image-spec/specs-go/v1" - "github.com/rancherfederal/ocil/pkg/artifacts" + "github.com/rancherfederal/hauler/pkg/artifacts" "helm.sh/helm/v3/pkg/action" "helm.sh/helm/v3/pkg/chart" "helm.sh/helm/v3/pkg/chart/loader" "helm.sh/helm/v3/pkg/cli" - "github.com/rancherfederal/ocil/pkg/layer" + "github.com/rancherfederal/hauler/pkg/layer" - "github.com/rancherfederal/ocil/pkg/consts" + "github.com/rancherfederal/hauler/pkg/consts" ) var _ artifacts.OCI = (*Chart)(nil) @@ -137,7 +137,8 @@ func (h *Chart) RawChartData() ([]byte, error) { } // chartData loads the chart contents into memory and returns a NopCloser for the contents -// Normally we avoid loading into memory, but charts sizes are strictly capped at ~1MB +// +// Normally we avoid loading into memory, but charts sizes are strictly capped at ~1MB func (h *Chart) chartData() (gv1.Layer, error) { info, err := os.Stat(h.path) if err != nil { diff --git a/pkg/content/chart/chart_test.go b/pkg/content/chart/chart_test.go index 7128c8f7..462f5ebc 100644 --- a/pkg/content/chart/chart_test.go +++ b/pkg/content/chart/chart_test.go @@ -10,7 +10,7 @@ import ( ocispec "github.com/opencontainers/image-spec/specs-go/v1" "helm.sh/helm/v3/pkg/action" - "github.com/rancherfederal/ocil/pkg/consts" + "github.com/rancherfederal/hauler/pkg/consts" "github.com/rancherfederal/hauler/pkg/content/chart" ) diff --git a/pkg/layer/cache.go b/pkg/layer/cache.go new file mode 100644 index 00000000..45d27f7b --- /dev/null +++ b/pkg/layer/cache.go @@ -0,0 +1,106 @@ +package layer + +import ( + "errors" + "io" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/types" + + "github.com/rancherfederal/hauler/pkg/artifacts" +) + +/* +This package is _heavily_ influenced by go-containerregistry and it's cache implementation: https://github.com/google/go-containerregistry/tree/main/pkg/v1/cache +*/ + +type Cache interface { + Put(v1.Layer) (v1.Layer, error) + + Get(v1.Hash) (v1.Layer, error) +} + +var ErrLayerNotFound = errors.New("layer not found") + +type oci struct { + artifacts.OCI + + c Cache +} + +func OCICache(o artifacts.OCI, c Cache) artifacts.OCI { + return &oci{ + OCI: o, + c: c, + } +} + +func (o *oci) Layers() ([]v1.Layer, error) { + ls, err := o.OCI.Layers() + if err != nil { + return nil, err + } + + var out []v1.Layer + for _, l := range ls { + out = append(out, &lazyLayer{inner: l, c: o.c}) + } + return out, nil +} + +type lazyLayer struct { + inner v1.Layer + c Cache +} + +func (l *lazyLayer) Compressed() (io.ReadCloser, error) { + digest, err := l.inner.Digest() + if err != nil { + return nil, err + } + + layer, err := l.getOrPut(digest) + if err != nil { + return nil, err + } + + return layer.Compressed() +} + +func (l *lazyLayer) Uncompressed() (io.ReadCloser, error) { + diffID, err := l.inner.DiffID() + if err != nil { + return nil, err + } + + layer, err := l.getOrPut(diffID) + if err != nil { + return nil, err + } + + return layer.Uncompressed() +} + +func (l *lazyLayer) getOrPut(h v1.Hash) (v1.Layer, error) { + var layer v1.Layer + if cl, err := l.c.Get(h); err == nil { + layer = cl + + } else if err == ErrLayerNotFound { + rl, err := l.c.Put(l.inner) + if err != nil { + return nil, err + } + layer = rl + + } else { + return nil, err + } + + return layer, nil +} + +func (l *lazyLayer) Size() (int64, error) { return l.inner.Size() } +func (l *lazyLayer) DiffID() (v1.Hash, error) { return l.inner.Digest() } +func (l *lazyLayer) Digest() (v1.Hash, error) { return l.inner.Digest() } +func (l *lazyLayer) MediaType() (types.MediaType, error) { return l.inner.MediaType() } diff --git a/pkg/layer/filesystem.go b/pkg/layer/filesystem.go new file mode 100644 index 00000000..8330fdbd --- /dev/null +++ b/pkg/layer/filesystem.go @@ -0,0 +1,118 @@ +package layer + +import ( + "io" + "os" + "path/filepath" + + v1 "github.com/google/go-containerregistry/pkg/v1" +) + +type fs struct { + root string +} + +func NewFilesystemCache(root string) Cache { + return &fs{root: root} +} + +func (f *fs) Put(l v1.Layer) (v1.Layer, error) { + digest, err := l.Digest() + if err != nil { + return nil, err + } + diffID, err := l.DiffID() + if err != nil { + return nil, err + } + return &cachedLayer{ + Layer: l, + root: f.root, + digest: digest, + diffID: diffID, + }, nil +} + +func (f *fs) Get(h v1.Hash) (v1.Layer, error) { + opener := f.open(h) + l, err := FromOpener(opener) + if os.IsNotExist(err) { + return nil, ErrLayerNotFound + } + return l, err +} + +func (f *fs) open(h v1.Hash) Opener { + return func() (io.ReadCloser, error) { + return os.Open(layerpath(f.root, h)) + } +} + +type cachedLayer struct { + v1.Layer + + root string + digest, diffID v1.Hash +} + +func (l *cachedLayer) create(h v1.Hash) (io.WriteCloser, error) { + lp := layerpath(l.root, h) + if err := os.MkdirAll(filepath.Dir(lp), os.ModePerm); err != nil { + return nil, err + } + return os.Create(lp) +} + +func (l *cachedLayer) Compressed() (io.ReadCloser, error) { + f, err := l.create(l.digest) + if err != nil { + return nil, nil + } + rc, err := l.Layer.Compressed() + if err != nil { + return nil, err + } + return &readcloser{ + t: io.TeeReader(rc, f), + closes: []func() error{rc.Close, f.Close}, + }, nil +} + +func (l *cachedLayer) Uncompressed() (io.ReadCloser, error) { + f, err := l.create(l.diffID) + if err != nil { + return nil, err + } + rc, err := l.Layer.Uncompressed() + if err != nil { + return nil, err + } + return &readcloser{ + t: io.TeeReader(rc, f), + closes: []func() error{rc.Close, f.Close}, + }, nil +} + +func layerpath(root string, h v1.Hash) string { + return filepath.Join(root, h.Algorithm, h.Hex) +} + +type readcloser struct { + t io.Reader + closes []func() error +} + +func (rc *readcloser) Read(b []byte) (int, error) { + return rc.t.Read(b) +} + +func (rc *readcloser) Close() error { + var err error + for _, c := range rc.closes { + lastErr := c() + if err == nil { + err = lastErr + } + } + return err +} diff --git a/pkg/layer/layer.go b/pkg/layer/layer.go new file mode 100644 index 00000000..fdfbf6a8 --- /dev/null +++ b/pkg/layer/layer.go @@ -0,0 +1,127 @@ +package layer + +import ( + "io" + + v1 "github.com/google/go-containerregistry/pkg/v1" + gtypes "github.com/google/go-containerregistry/pkg/v1/types" + + "github.com/rancherfederal/hauler/pkg/consts" +) + +type Opener func() (io.ReadCloser, error) + +func FromOpener(opener Opener, opts ...Option) (v1.Layer, error) { + var err error + + layer := &layer{ + mediaType: consts.UnknownLayer, + annotations: make(map[string]string, 1), + } + + layer.uncompressedOpener = opener + layer.compressedOpener = func() (io.ReadCloser, error) { + rc, err := opener() + if err != nil { + return nil, err + } + + return rc, nil + } + + for _, opt := range opts { + opt(layer) + } + + if layer.digest, layer.size, err = compute(layer.uncompressedOpener); err != nil { + return nil, err + } + + if layer.diffID, _, err = compute(layer.compressedOpener); err != nil { + return nil, err + } + + return layer, nil +} + +func compute(opener Opener) (v1.Hash, int64, error) { + rc, err := opener() + if err != nil { + return v1.Hash{}, 0, err + } + defer rc.Close() + return v1.SHA256(rc) +} + +type Option func(*layer) + +func WithMediaType(mt string) Option { + return func(l *layer) { + l.mediaType = mt + } +} + +func WithAnnotations(annotations map[string]string) Option { + return func(l *layer) { + if l.annotations == nil { + l.annotations = make(map[string]string) + } + l.annotations = annotations + } +} + +type layer struct { + digest v1.Hash + diffID v1.Hash + size int64 + compressedOpener Opener + uncompressedOpener Opener + mediaType string + annotations map[string]string + urls []string +} + +func (l layer) Descriptor() (*v1.Descriptor, error) { + digest, err := l.Digest() + if err != nil { + return nil, err + } + mt, err := l.MediaType() + if err != nil { + return nil, err + } + return &v1.Descriptor{ + MediaType: mt, + Size: l.size, + Digest: digest, + Annotations: l.annotations, + URLs: l.urls, + + // TODO: Allow platforms + Platform: nil, + }, nil +} + +func (l layer) Digest() (v1.Hash, error) { + return l.digest, nil +} + +func (l layer) DiffID() (v1.Hash, error) { + return l.diffID, nil +} + +func (l layer) Compressed() (io.ReadCloser, error) { + return l.compressedOpener() +} + +func (l layer) Uncompressed() (io.ReadCloser, error) { + return l.uncompressedOpener() +} + +func (l layer) Size() (int64, error) { + return l.size, nil +} + +func (l layer) MediaType() (gtypes.MediaType, error) { + return gtypes.MediaType(l.mediaType), nil +} diff --git a/pkg/store/store_test.go b/pkg/store/store_test.go new file mode 100644 index 00000000..54b4319a --- /dev/null +++ b/pkg/store/store_test.go @@ -0,0 +1,105 @@ +package store_test + +import ( + "context" + "os" + "testing" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/random" + + "github.com/rancherfederal/hauler/pkg/artifacts" + "github.com/rancherfederal/hauler/pkg/store" +) + +var ( + ctx context.Context + root string +) + +func TestLayout_AddOCI(t *testing.T) { + teardown := setup(t) + defer teardown() + + type args struct { + ref string + } + tests := []struct { + name string + args args + wantErr bool + }{ + { + name: "", + args: args{ + ref: "hello/world:v1", + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s, err := store.NewLayout(root) + if (err != nil) != tt.wantErr { + t.Errorf("NewOCI() error = %v, wantErr %v", err, tt.wantErr) + return + } + moci := genArtifact(t, tt.args.ref) + + got, err := s.AddOCI(ctx, moci, tt.args.ref) + if (err != nil) != tt.wantErr { + t.Errorf("AddOCI() error = %v, wantErr %v", err, tt.wantErr) + return + } + _ = got + + _, err = s.AddOCI(ctx, moci, tt.args.ref) + if err != nil { + t.Errorf("AddOCI() error = %v, wantErr %v", err, tt.wantErr) + return + } + }) + } +} + +func setup(t *testing.T) func() error { + tmpdir, err := os.MkdirTemp("", "hauler") + if err != nil { + t.Fatal(err) + } + root = tmpdir + + ctx = context.Background() + + return func() error { + os.RemoveAll(tmpdir) + return nil + } +} + +type mockArtifact struct { + v1.Image +} + +func (m mockArtifact) MediaType() string { + mt, err := m.Image.MediaType() + if err != nil { + return "" + } + return string(mt) +} + +func (m mockArtifact) RawConfig() ([]byte, error) { + return m.RawConfigFile() +} + +func genArtifact(t *testing.T, ref string) artifacts.OCI { + img, err := random.Image(1024, 3) + if err != nil { + t.Fatal(err) + } + + return &mockArtifact{ + img, + } +} From 214ed48829fa7a76d84d80b9190edcaf791d2c81 Mon Sep 17 00:00:00 2001 From: Adam Martin Date: Fri, 6 Oct 2023 14:34:48 -0400 Subject: [PATCH 02/11] updates to OCIL funcs to handle cosign changes --- pkg/consts/consts.go | 50 ++++++++ pkg/content/oci.go | 272 +++++++++++++++++++++++++++++++++++++++++++ pkg/store/store.go | 262 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 584 insertions(+) create mode 100644 pkg/consts/consts.go create mode 100644 pkg/content/oci.go create mode 100644 pkg/store/store.go diff --git a/pkg/consts/consts.go b/pkg/consts/consts.go new file mode 100644 index 00000000..0263408b --- /dev/null +++ b/pkg/consts/consts.go @@ -0,0 +1,50 @@ +package consts + +const ( + OCIManifestSchema1 = "application/vnd.oci.image.manifest.v1+json" + DockerManifestSchema2 = "application/vnd.docker.distribution.manifest.v2+json" + DockerManifestListSchema2 = "application/vnd.docker.distribution.manifest.list.v2+json" + + DockerConfigJSON = "application/vnd.docker.container.image.v1+json" + DockerLayer = "application/vnd.docker.image.rootfs.diff.tar.gzip" + DockerForeignLayer = "application/vnd.docker.image.rootfs.foreign.diff.tar.gzip" + DockerUncompressedLayer = "application/vnd.docker.image.rootfs.diff.tar" + OCILayer = "application/vnd.oci.image.layer.v1.tar+gzip" + + // ChartConfigMediaType is the reserved media type for the Helm chart manifest config + ChartConfigMediaType = "application/vnd.cncf.helm.config.v1+json" + + // ChartLayerMediaType is the reserved media type for Helm chart package content + ChartLayerMediaType = "application/vnd.cncf.helm.chart.content.v1.tar+gzip" + + // ProvLayerMediaType is the reserved media type for Helm chart provenance files + ProvLayerMediaType = "application/vnd.cncf.helm.chart.provenance.v1.prov" + + // FileLayerMediaType is the reserved media type for File content layers + FileLayerMediaType = "application/vnd.content.hauler.file.layer.v1" + + // FileLocalConfigMediaType is the reserved media type for File config + FileLocalConfigMediaType = "application/vnd.content.hauler.file.local.config.v1+json" + FileDirectoryConfigMediaType = "application/vnd.content.hauler.file.directory.config.v1+json" + FileHttpConfigMediaType = "application/vnd.content.hauler.file.http.config.v1+json" + + // MemoryConfigMediaType + MemoryConfigMediaType = "application/vnd.content.hauler.memory.config.v1+json" + + // WasmArtifactLayerMediaType is the reserved media type for WASM artifact layers + WasmArtifactLayerMediaType = "application/vnd.wasm.content.layer.v1+wasm" + + // WasmConfigMediaType is the reserved media type for WASM configs + WasmConfigMediaType = "application/vnd.wasm.config.v1+json" + + UnknownManifest = "application/vnd.hauler.cattle.io.unknown.v1+json" + UnknownLayer = "application/vnd.content.hauler.unknown.layer" + + OCIVendorPrefix = "vnd.oci" + DockerVendorPrefix = "vnd.docker" + HaulerVendorPrefix = "vnd.hauler" + OCIImageIndexFile = "index.json" + + KindAnnotationName = "kind" + KindAnnotation = "dev.cosignproject.cosign/image" +) diff --git a/pkg/content/oci.go b/pkg/content/oci.go new file mode 100644 index 00000000..1c488dd3 --- /dev/null +++ b/pkg/content/oci.go @@ -0,0 +1,272 @@ +package content + +import ( + "context" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "os" + "path/filepath" + "sort" + "strings" + "sync" + + ccontent "github.com/containerd/containerd/content" + "github.com/containerd/containerd/remotes" + "github.com/opencontainers/image-spec/specs-go" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "oras.land/oras-go/pkg/content" + "oras.land/oras-go/pkg/target" + + "github.com/rancherfederal/hauler/pkg/consts" +) + +var _ target.Target = (*OCI)(nil) + +type OCI struct { + root string + index *ocispec.Index + nameMap *sync.Map // map[string]ocispec.Descriptor +} + +func NewOCI(root string) (*OCI, error) { + o := &OCI{ + root: root, + nameMap: &sync.Map{}, + } + return o, nil +} + +// AddIndex adds a descriptor to the index and updates it +// +// The descriptor must use AnnotationRefName to identify itself +func (o *OCI) AddIndex(desc ocispec.Descriptor) error { + if _, ok := desc.Annotations[ocispec.AnnotationRefName]; !ok { + return fmt.Errorf("descriptor must contain a reference from the annotation: %s", ocispec.AnnotationRefName) + } + key := fmt.Sprintf("%s-%s-%s", desc.Digest.String(), desc.Annotations[ocispec.AnnotationRefName], desc.Annotations[consts.KindAnnotationName]) + o.nameMap.Store(key, desc) + return o.SaveIndex() +} + +// LoadIndex will load the index from disk +func (o *OCI) LoadIndex() error { + path := o.path(consts.OCIImageIndexFile) + idx, err := os.Open(path) + if err != nil { + if !os.IsNotExist(err) { + return err + } + o.index = &ocispec.Index{ + Versioned: specs.Versioned{ + SchemaVersion: 2, + }, + } + return nil + } + defer idx.Close() + + if err := json.NewDecoder(idx).Decode(&o.index); err != nil { + return err + } + + for _, desc := range o.index.Manifests { + key := fmt.Sprintf("%s-%s-%s", desc.Digest.String(), desc.Annotations[ocispec.AnnotationRefName], desc.Annotations[consts.KindAnnotationName]) + if strings.TrimSpace(key) != "--" { + o.nameMap.Store(key, desc) + } + } + return nil +} + +// SaveIndex will update the index on disk +func (o *OCI) SaveIndex() error { + var descs []ocispec.Descriptor + o.nameMap.Range(func(name, desc interface{}) bool { + n := desc.(ocispec.Descriptor).Annotations[ocispec.AnnotationRefName] + d := desc.(ocispec.Descriptor) + + if d.Annotations == nil { + d.Annotations = make(map[string]string) + } + d.Annotations[ocispec.AnnotationRefName] = n + descs = append(descs, d) + return true + }) + + // sort index to ensure that images come before any signatures and attestations. + sort.SliceStable(descs, func(i, j int) bool { + kindI := descs[i].Annotations["kind"] + kindJ := descs[j].Annotations["kind"] + + // Objects with the prefix of "dev.cosignproject.cosign/image" should be at the top. + if strings.HasPrefix(kindI, consts.KindAnnotation) && !strings.HasPrefix(kindJ, consts.KindAnnotation) { + return true + } else if !strings.HasPrefix(kindI, consts.KindAnnotation) && strings.HasPrefix(kindJ, consts.KindAnnotation) { + return false + } + return false // Default: maintain the order. + }) + + o.index.Manifests = descs + data, err := json.Marshal(o.index) + if err != nil { + return err + } + return os.WriteFile(o.path(consts.OCIImageIndexFile), data, 0644) +} + +// Resolve attempts to resolve the reference into a name and descriptor. +// +// The argument `ref` should be a scheme-less URI representing the remote. +// Structurally, it has a host and path. The "host" can be used to directly +// reference a specific host or be matched against a specific handler. +// +// The returned name should be used to identify the referenced entity. +// Dependending on the remote namespace, this may be immutable or mutable. +// While the name may differ from ref, it should itself be a valid ref. +// +// If the resolution fails, an error will be returned. +func (o *OCI) Resolve(ctx context.Context, ref string) (name string, desc ocispec.Descriptor, err error) { + if err := o.LoadIndex(); err != nil { + return "", ocispec.Descriptor{}, err + } + d, ok := o.nameMap.Load(ref) + if !ok { + return "", ocispec.Descriptor{}, err + } + desc = d.(ocispec.Descriptor) + return ref, desc, nil +} + +// Fetcher returns a new fetcher for the provided reference. +// All content fetched from the returned fetcher will be +// from the namespace referred to by ref. +func (o *OCI) Fetcher(ctx context.Context, ref string) (remotes.Fetcher, error) { + if err := o.LoadIndex(); err != nil { + return nil, err + } + if _, ok := o.nameMap.Load(ref); !ok { + return nil, nil + } + return o, nil +} + +func (o *OCI) Fetch(ctx context.Context, desc ocispec.Descriptor) (io.ReadCloser, error) { + readerAt, err := o.blobReaderAt(desc) + if err != nil { + return nil, err + } + return readerAt, nil +} + +// Pusher returns a new pusher for the provided reference +// The returned Pusher should satisfy content.Ingester and concurrent attempts +// to push the same blob using the Ingester API should result in ErrUnavailable. +func (o *OCI) Pusher(ctx context.Context, ref string) (remotes.Pusher, error) { + if err := o.LoadIndex(); err != nil { + return nil, err + } + + var baseRef, hash string + parts := strings.SplitN(ref, "@", 2) + baseRef = parts[0] + if len(parts) > 1 { + hash = parts[1] + } + return &ociPusher{ + oci: o, + ref: baseRef, + digest: hash, + }, nil +} + +func (o *OCI) Walk(fn func(reference string, desc ocispec.Descriptor) error) error { + if err := o.LoadIndex(); err != nil { + return err + } + + var errst []string + o.nameMap.Range(func(key, value interface{}) bool { + if err := fn(key.(string), value.(ocispec.Descriptor)); err != nil { + errst = append(errst, err.Error()) + } + return true + }) + if errst != nil { + return fmt.Errorf(strings.Join(errst, "; ")) + } + return nil +} + +func (o *OCI) blobReaderAt(desc ocispec.Descriptor) (*os.File, error) { + blobPath, err := o.ensureBlob(desc.Digest.Algorithm().String(), desc.Digest.Hex()) + if err != nil { + return nil, err + } + return os.Open(blobPath) +} + +func (o *OCI) blobWriterAt(desc ocispec.Descriptor) (*os.File, error) { + blobPath, err := o.ensureBlob(desc.Digest.Algorithm().String(), desc.Digest.Hex()) + if err != nil { + return nil, err + } + return os.OpenFile(blobPath, os.O_WRONLY|os.O_CREATE, 0644) +} + +func (o *OCI) ensureBlob(alg string, hex string) (string, error) { + dir := o.path("blobs", alg) + if err := os.MkdirAll(dir, os.ModePerm); err != nil && !os.IsExist(err) { + return "", err + } + return filepath.Join(dir, hex), nil +} + +func (o *OCI) path(elem ...string) string { + complete := []string{string(o.root)} + return filepath.Join(append(complete, elem...)...) +} + +type ociPusher struct { + oci *OCI + ref string + digest string +} + +// Push returns a content writer for the given resource identified +// by the descriptor. +func (p *ociPusher) Push(ctx context.Context, d ocispec.Descriptor) (ccontent.Writer, error) { + switch d.MediaType { + case ocispec.MediaTypeImageManifest, ocispec.MediaTypeImageIndex, consts.DockerManifestSchema2, consts.DockerManifestListSchema2: + // if the hash of the content matches that which was provided as the hash for the root, mark it + if p.digest != "" && p.digest == d.Digest.String() { + if err := p.oci.LoadIndex(); err != nil { + return nil, err + } + p.oci.nameMap.Store(p.ref, d) + if err := p.oci.SaveIndex(); err != nil { + return nil, err + } + } + } + + blobPath, err := p.oci.ensureBlob(d.Digest.Algorithm().String(), d.Digest.Hex()) + if err != nil { + return nil, err + } + + if _, err := os.Stat(blobPath); err == nil { + // file already exists, discard (but validate digest) + return content.NewIoContentWriter(ioutil.Discard, content.WithOutputHash(d.Digest)), nil + } + + f, err := os.Create(blobPath) + if err != nil { + return nil, err + } + + w := content.NewIoContentWriter(f, content.WithInputHash(d.Digest), content.WithOutputHash(d.Digest)) + return w, nil +} diff --git a/pkg/store/store.go b/pkg/store/store.go new file mode 100644 index 00000000..ef4b2d54 --- /dev/null +++ b/pkg/store/store.go @@ -0,0 +1,262 @@ +package store + +import ( + "context" + "encoding/json" + "io" + "os" + "path/filepath" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/static" + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "golang.org/x/sync/errgroup" + "oras.land/oras-go/pkg/oras" + "oras.land/oras-go/pkg/target" + + "github.com/rancherfederal/hauler/pkg/artifacts" + "github.com/rancherfederal/hauler/pkg/consts" + "github.com/rancherfederal/hauler/pkg/content" + "github.com/rancherfederal/hauler/pkg/layer" +) + +type Layout struct { + *content.OCI + Root string + cache layer.Cache +} + +type Options func(*Layout) + +func WithCache(c layer.Cache) Options { + return func(l *Layout) { + l.cache = c + } +} + +func NewLayout(rootdir string, opts ...Options) (*Layout, error) { + ociStore, err := content.NewOCI(rootdir) + if err != nil { + return nil, err + } + + if err := ociStore.LoadIndex(); err != nil { + return nil, err + } + + l := &Layout{ + Root: rootdir, + OCI: ociStore, + } + + for _, opt := range opts { + opt(l) + } + + return l, nil +} + +// AddOCI adds an artifacts.OCI to the store +// +// The method to achieve this is to save artifact.OCI to a temporary directory in an OCI layout compatible form. Once +// saved, the entirety of the layout is copied to the store (which is just a registry). This allows us to not only use +// strict types to define generic content, but provides a processing pipeline suitable for extensibility. In the +// future we'll allow users to define their own content that must adhere either by artifact.OCI or simply an OCI layout. +func (l *Layout) AddOCI(ctx context.Context, oci artifacts.OCI, ref string) (ocispec.Descriptor, error) { + if l.cache != nil { + cached := layer.OCICache(oci, l.cache) + oci = cached + } + + // Write manifest blob + m, err := oci.Manifest() + if err != nil { + return ocispec.Descriptor{}, err + } + + mdata, err := json.Marshal(m) + if err != nil { + return ocispec.Descriptor{}, err + } + if err := l.writeBlobData(mdata); err != nil { + return ocispec.Descriptor{}, err + } + + // Write config blob + cdata, err := oci.RawConfig() + if err != nil { + return ocispec.Descriptor{}, err + } + + static.NewLayer(cdata, "") + + if err := l.writeBlobData(cdata); err != nil { + return ocispec.Descriptor{}, err + } + + // write blob layers concurrently + layers, err := oci.Layers() + if err != nil { + return ocispec.Descriptor{}, err + } + + var g errgroup.Group + for _, lyr := range layers { + lyr := lyr + g.Go(func() error { + return l.writeLayer(lyr) + }) + } + if err := g.Wait(); err != nil { + return ocispec.Descriptor{}, err + } + + // Build index + idx := ocispec.Descriptor{ + MediaType: string(m.MediaType), + Digest: digest.FromBytes(mdata), + Size: int64(len(mdata)), + Annotations: map[string]string{ + consts.KindAnnotationName: consts.KindAnnotation, + ocispec.AnnotationRefName: ref, + }, + URLs: nil, + Platform: nil, + } + + return idx, l.OCI.AddIndex(idx) +} + +// AddOCICollection . +func (l *Layout) AddOCICollection(ctx context.Context, collection artifacts.OCICollection) ([]ocispec.Descriptor, error) { + cnts, err := collection.Contents() + if err != nil { + return nil, err + } + + var descs []ocispec.Descriptor + for ref, oci := range cnts { + desc, err := l.AddOCI(ctx, oci, ref) + if err != nil { + return nil, err + } + descs = append(descs, desc) + } + return descs, nil +} + +// Flush is a fancy name for delete-all-the-things, in this case it's as trivial as deleting oci-layout content +// +// This can be a highly destructive operation if the store's directory happens to be inline with other non-store contents +// To reduce the blast radius and likelihood of deleting things we don't own, Flush explicitly deletes oci-layout content only +func (l *Layout) Flush(ctx context.Context) error { + blobs := filepath.Join(l.Root, "blobs") + if err := os.RemoveAll(blobs); err != nil { + return err + } + + index := filepath.Join(l.Root, "index.json") + if err := os.RemoveAll(index); err != nil { + return err + } + + layout := filepath.Join(l.Root, "oci-layout") + if err := os.RemoveAll(layout); err != nil { + return err + } + + return nil +} + +// Copy will copy a given reference to a given target.Target +// +// This is essentially a wrapper around oras.Copy, but locked to this content store +func (l *Layout) Copy(ctx context.Context, ref string, to target.Target, toRef string) (ocispec.Descriptor, error) { + return oras.Copy(ctx, l.OCI, ref, to, toRef, + oras.WithAdditionalCachedMediaTypes(consts.DockerManifestSchema2, consts.DockerManifestListSchema2)) +} + +// CopyAll performs bulk copy operations on the stores oci layout to a provided target.Target +func (l *Layout) CopyAll(ctx context.Context, to target.Target, toMapper func(string) (string, error)) ([]ocispec.Descriptor, error) { + var descs []ocispec.Descriptor + err := l.OCI.Walk(func(reference string, desc ocispec.Descriptor) error { + toRef := "" + if toMapper != nil { + tr, err := toMapper(reference) + if err != nil { + return err + } + toRef = tr + } + + desc, err := l.Copy(ctx, reference, to, toRef) + if err != nil { + return err + } + + descs = append(descs, desc) + return nil + }) + if err != nil { + return nil, err + } + return descs, nil +} + +// Identify is a helper function that will identify a human-readable content type given a descriptor +func (l *Layout) Identify(ctx context.Context, desc ocispec.Descriptor) string { + rc, err := l.OCI.Fetch(ctx, desc) + if err != nil { + return "" + } + defer rc.Close() + + m := struct { + Config struct { + MediaType string `json:"mediaType"` + } `json:"config"` + }{} + if err := json.NewDecoder(rc).Decode(&m); err != nil { + return "" + } + + return m.Config.MediaType +} + +func (l *Layout) writeBlobData(data []byte) error { + blob := static.NewLayer(data, "") // NOTE: MediaType isn't actually used in the writing + return l.writeLayer(blob) +} + +func (l *Layout) writeLayer(layer v1.Layer) error { + d, err := layer.Digest() + if err != nil { + return err + } + + r, err := layer.Compressed() + if err != nil { + return err + } + + dir := filepath.Join(l.Root, "blobs", d.Algorithm) + if err := os.MkdirAll(dir, os.ModePerm); err != nil && !os.IsExist(err) { + return err + } + + blobPath := filepath.Join(dir, d.Hex) + // Skip entirely if something exists, assume layer is present already + if _, err := os.Stat(blobPath); err == nil { + return nil + } + + w, err := os.Create(blobPath) + if err != nil { + return err + } + defer w.Close() + + _, err = io.Copy(w, r) + return err +} From 58c55d7aebb14db29d5c53ff7889b6ab2774ba56 Mon Sep 17 00:00:00 2001 From: Adam Martin Date: Fri, 6 Oct 2023 14:36:07 -0400 Subject: [PATCH 03/11] add cosign logic --- pkg/cosign/cosign.go | 57 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 pkg/cosign/cosign.go diff --git a/pkg/cosign/cosign.go b/pkg/cosign/cosign.go new file mode 100644 index 00000000..c7476e12 --- /dev/null +++ b/pkg/cosign/cosign.go @@ -0,0 +1,57 @@ +package cosign + +import ( + "fmt" + "oras.land/oras-go/pkg/content" + "os/exec" +) + +// VerifyFileSignature verifies the digital signature of a file using Sigstore/Cosign. +func VerifySignature(filePath string, keyPath string) error { + // Command to verify the signature using Cosign. + cmd := exec.Command("cosign", "verify", "--insecure-ignore-tlog", "--key", keyPath, filePath) + + // Run the command and capture its output. + output, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("error verifying signature: %v, output: %s", err, output) + } + + return nil +} + +// SaveImage saves image and any signatures/attestations to the store. +func SaveImage(storePath string, ref string) error { + // Command to verify the signature using Cosign. + cmd := exec.Command("cosign", "save", ref, "--dir", storePath) + + // Run the command and capture its output. + output, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("error adding image to store: %v, output: %s", err, output) + } + + return nil +} + +// LoadImage loads store to a remote registry. +func LoadImage(storePath string, registry string, ropts content.RegistryOptions) error { + // Command to verify the signature using Cosign. + cmd := exec.Command("cosign", "load", "--registry", registry, "--dir", storePath) + + // Conditionally add extra registry flags. + if ropts.Insecure { + cmd.Args = append(cmd.Args, "--allow-insecure-registry=true") + } + if ropts.PlainHTTP { + cmd.Args = append(cmd.Args, "--allow-http-registry=true") + } + + // Run the command and capture its output. + output, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("error adding image to store: %v, output: %s", err, output) + } + + return nil +} From ece463bc1c57692bd6a9b2673663c538f3f32747 Mon Sep 17 00:00:00 2001 From: Adam Martin Date: Fri, 6 Oct 2023 14:37:08 -0400 Subject: [PATCH 04/11] adjust Makefile to be a little more generic --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 398cfbfd..bc504c5e 100644 --- a/Makefile +++ b/Makefile @@ -11,13 +11,13 @@ all: fmt vet install test build: mkdir bin;\ - $(GO_BUILD_ENV) go build -o bin ./cmd/...;\ + GOENV=GOARCH=$(uname -m) CGO_ENABLED=0 go build -o bin ./cmd/...;\ build-all: fmt vet goreleaser build --rm-dist --snapshot install: - $(GO_BUILD_ENV) go install + GOENV=GOARCH=$(uname -m) CGO_ENABLED=0 go install ./cmd/...;\ vet: go vet $(GO_FILES) From 3049846a46041b786733a8e907f9af82fd454783 Mon Sep 17 00:00:00 2001 From: Adam Martin Date: Fri, 6 Oct 2023 14:40:06 -0400 Subject: [PATCH 05/11] cli updates to accomodate the cosign additions --- cmd/hauler/cli/serve/registry.go | 1 + cmd/hauler/cli/store/add.go | 30 ++++++++++++++++++------------ cmd/hauler/cli/store/copy.go | 24 ++++-------------------- cmd/hauler/cli/store/info.go | 12 +++++++----- cmd/hauler/cli/store/sync.go | 17 +++++++++++++++-- internal/mapper/mappers.go | 2 +- 6 files changed, 46 insertions(+), 40 deletions(-) diff --git a/cmd/hauler/cli/serve/registry.go b/cmd/hauler/cli/serve/registry.go index 93363143..99218bdf 100644 --- a/cmd/hauler/cli/serve/registry.go +++ b/cmd/hauler/cli/serve/registry.go @@ -74,6 +74,7 @@ func (o *RegistryOpts) defaultConfig() *configuration.Configuration { cfg.HTTP.Addr = fmt.Sprintf(":%d", o.Port) cfg.HTTP.Headers = http.Header{ "X-Content-Type-Options": []string{"nosniff"}, + "Accept": []string{"application/vnd.dsse.envelope.v1+json, application/json"}, } return cfg diff --git a/cmd/hauler/cli/store/add.go b/cmd/hauler/cli/store/add.go index 666839b4..865d9df2 100644 --- a/cmd/hauler/cli/store/add.go +++ b/cmd/hauler/cli/store/add.go @@ -4,17 +4,17 @@ import ( "context" "github.com/google/go-containerregistry/pkg/name" - "github.com/rancherfederal/ocil/pkg/artifacts/file/getter" + "github.com/rancherfederal/hauler/pkg/artifacts/file/getter" "github.com/spf13/cobra" "helm.sh/helm/v3/pkg/action" - "github.com/rancherfederal/ocil/pkg/artifacts/file" - "github.com/rancherfederal/ocil/pkg/artifacts/image" + "github.com/rancherfederal/hauler/pkg/artifacts/file" - "github.com/rancherfederal/ocil/pkg/store" + "github.com/rancherfederal/hauler/pkg/store" "github.com/rancherfederal/hauler/pkg/apis/hauler.cattle.io/v1alpha1" "github.com/rancherfederal/hauler/pkg/content/chart" + "github.com/rancherfederal/hauler/pkg/cosign" "github.com/rancherfederal/hauler/pkg/log" "github.com/rancherfederal/hauler/pkg/reference" ) @@ -62,11 +62,12 @@ func storeFile(ctx context.Context, s *store.Layout, fi v1alpha1.File) error { type AddImageOpts struct { *RootOpts Name string + Key string } func (o *AddImageOpts) AddFlags(cmd *cobra.Command) { f := cmd.Flags() - _ = f + f.StringVarP(&o.Key, "key", "k", "", "(Optional) Path to the key for digital signature verification") } func AddImageCmd(ctx context.Context, o *AddImageOpts, s *store.Layout, reference string) error { @@ -74,28 +75,33 @@ func AddImageCmd(ctx context.Context, o *AddImageOpts, s *store.Layout, referenc Name: reference, } + // Check if the user provided a key. + if o.Key != "" { + // verify signature using the provided key. + err := cosign.VerifySignature(reference, o.Key) + if err != nil { + return err + } + } + return storeImage(ctx, s, cfg) } func storeImage(ctx context.Context, s *store.Layout, i v1alpha1.Image) error { l := log.FromContext(ctx) - img, err := image.NewImage(i.Name) - if err != nil { - return err - } - r, err := name.ParseReference(i.Name) if err != nil { return err } - desc, err := s.AddOCI(ctx, img, r.Name()) + err = cosign.SaveImage(s.Root, r.Name()) + //desc, err := s.AddOCI(ctx, img, r.Name()) if err != nil { return err } - l.Infof("added 'image' to store at [%s], with digest [%s]", r.Name(), desc.Digest.String()) + l.Infof("added 'image' to store at [%s]", r.Name()) return nil } diff --git a/cmd/hauler/cli/store/copy.go b/cmd/hauler/cli/store/copy.go index 0eafb123..064f2dab 100644 --- a/cmd/hauler/cli/store/copy.go +++ b/cmd/hauler/cli/store/copy.go @@ -5,14 +5,13 @@ import ( "fmt" "strings" - ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/spf13/cobra" "oras.land/oras-go/pkg/content" + "github.com/rancherfederal/hauler/pkg/cosign" "github.com/rancherfederal/hauler/pkg/store" "github.com/rancherfederal/hauler/pkg/log" - "github.com/rancherfederal/hauler/pkg/reference" ) type CopyOpts struct { @@ -36,7 +35,6 @@ func (o *CopyOpts) AddFlags(cmd *cobra.Command) { func CopyCmd(ctx context.Context, o *CopyOpts, s *store.Layout, targetRef string) error { l := log.FromContext(ctx) - var descs []ocispec.Descriptor components := strings.SplitN(targetRef, "://", 2) switch components[0] { case "dir": @@ -44,11 +42,10 @@ func CopyCmd(ctx context.Context, o *CopyOpts, s *store.Layout, targetRef string fs := content.NewFile(components[1]) defer fs.Close() - ds, err := s.CopyAll(ctx, fs, nil) + _, err := s.CopyAll(ctx, fs, nil) if err != nil { return err } - descs = ds case "registry": l.Debugf("identified registry target reference") @@ -58,29 +55,16 @@ func CopyCmd(ctx context.Context, o *CopyOpts, s *store.Layout, targetRef string Insecure: o.Insecure, PlainHTTP: o.PlainHTTP, } - r, err := content.NewRegistry(ropts) - if err != nil { - return err - } - - mapperFn := func(ref string) (string, error) { - r, err := reference.Relocate(ref, components[1]) - if err != nil { - return "", err - } - return r.Name(), nil - } - ds, err := s.CopyAll(ctx, r, mapperFn) + err := cosign.LoadImage(s.Root, components[1], ropts) if err != nil { return err } - descs = ds default: return fmt.Errorf("detecting protocol from [%s]", targetRef) } - l.Infof("Copied [%d] artifacts to [%s]", len(descs), components[1]) + l.Infof("Copied artifacts to [%s]", components[1]) return nil } diff --git a/cmd/hauler/cli/store/info.go b/cmd/hauler/cli/store/info.go index 74cdce70..f4de62df 100644 --- a/cmd/hauler/cli/store/info.go +++ b/cmd/hauler/cli/store/info.go @@ -10,9 +10,9 @@ import ( ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/spf13/cobra" - "github.com/rancherfederal/ocil/pkg/consts" + "github.com/rancherfederal/hauler/pkg/consts" - "github.com/rancherfederal/ocil/pkg/store" + "github.com/rancherfederal/hauler/pkg/store" "github.com/rancherfederal/hauler/pkg/reference" ) @@ -78,9 +78,11 @@ func buildTable(items ...item) string { fmt.Fprintf(tw, "---------\t----\t--------\t----\n") for _, i := range items { - fmt.Fprintf(tw, "%s\t%s\t%d\t%s\n", - i.Reference, i.Type, i.Layers, i.Size, - ) + if i.Type != "unknown" { + fmt.Fprintf(tw, "%s\t%s\t%d\t%s\n", + i.Reference, i.Type, i.Layers, i.Size, + ) + } } tw.Flush() return b.String() diff --git a/cmd/hauler/cli/store/sync.go b/cmd/hauler/cli/store/sync.go index 5eb46417..a5930db4 100644 --- a/cmd/hauler/cli/store/sync.go +++ b/cmd/hauler/cli/store/sync.go @@ -11,25 +11,28 @@ import ( "helm.sh/helm/v3/pkg/action" "k8s.io/apimachinery/pkg/util/yaml" - "github.com/rancherfederal/ocil/pkg/store" + "github.com/rancherfederal/hauler/pkg/store" "github.com/rancherfederal/hauler/pkg/apis/hauler.cattle.io/v1alpha1" tchart "github.com/rancherfederal/hauler/pkg/collection/chart" "github.com/rancherfederal/hauler/pkg/collection/imagetxt" "github.com/rancherfederal/hauler/pkg/collection/k3s" "github.com/rancherfederal/hauler/pkg/content" + "github.com/rancherfederal/hauler/pkg/cosign" "github.com/rancherfederal/hauler/pkg/log" ) type SyncOpts struct { *RootOpts ContentFiles []string + Key string } func (o *SyncOpts) AddFlags(cmd *cobra.Command) { f := cmd.Flags() f.StringSliceVarP(&o.ContentFiles, "files", "f", []string{}, "Path to content files") + f.StringVarP(&o.Key, "key", "k", "", "(Optional) Path to the key for digital signature verification") } func SyncCmd(ctx context.Context, o *SyncOpts, s *store.Layout) error { @@ -94,7 +97,17 @@ func SyncCmd(ctx context.Context, o *SyncOpts, s *store.Layout) error { } for _, i := range cfg.Spec.Images { - err := storeImage(ctx, s, i) + + // Check if the user provided a key. + if o.Key != "" { + // verify signature using the provided key. + err := cosign.VerifySignature(i.Name, o.Key) + if err != nil { + return err + } + } + + err = storeImage(ctx, s, i) if err != nil { return err } diff --git a/internal/mapper/mappers.go b/internal/mapper/mappers.go index 05cf89bb..7942f886 100644 --- a/internal/mapper/mappers.go +++ b/internal/mapper/mappers.go @@ -39,7 +39,7 @@ func Images() map[string]Fn { return "manifest.json", nil }) - for _, l := range []string{consts.DockerManifestSchema2, consts.OCIManifestSchema1} { + for _, l := range []string{consts.DockerManifestSchema2, consts.DockerManifestListSchema2, consts.OCIManifestSchema1} { m[l] = manifestMapperFn } From 220eeedb2c2f82dd1f252ed5b1324636427a7beb Mon Sep 17 00:00:00 2001 From: Adam Martin Date: Tue, 10 Oct 2023 10:47:11 -0400 Subject: [PATCH 06/11] add cosign drop-in funcs Signed-off-by: Adam Martin --- pkg/cosign/cosign.go | 245 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 238 insertions(+), 7 deletions(-) diff --git a/pkg/cosign/cosign.go b/pkg/cosign/cosign.go index c7476e12..65db45b6 100644 --- a/pkg/cosign/cosign.go +++ b/pkg/cosign/cosign.go @@ -2,14 +2,39 @@ package cosign import ( "fmt" - "oras.land/oras-go/pkg/content" + "io" + "net/http" + "os" "os/exec" + "os/user" + "path/filepath" + "runtime" + "context" + "strings" + "encoding/json" + + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "oras.land/oras-go/pkg/content" + "github.com/rancherfederal/hauler/pkg/store" + "github.com/rancherfederal/hauler/pkg/log" + "github.com/rancherfederal/hauler/internal/mapper" + "github.com/rancherfederal/hauler/pkg/reference" + "github.com/rancherfederal/hauler/pkg/artifacts/file" + "github.com/rancherfederal/hauler/pkg/artifacts/file/getter" + "github.com/rancherfederal/hauler/pkg/apis/hauler.cattle.io/v1alpha1" ) // VerifyFileSignature verifies the digital signature of a file using Sigstore/Cosign. -func VerifySignature(filePath string, keyPath string) error { +func VerifySignature(ctx context.Context, s *store.Layout, keyPath string) error { + + // Ensure that the cosign binary is installed or download it if needed + cosignBinaryPath, err := ensureCosignBinary(ctx, s) + if err != nil { + return err + } + // Command to verify the signature using Cosign. - cmd := exec.Command("cosign", "verify", "--insecure-ignore-tlog", "--key", keyPath, filePath) + cmd := exec.Command(cosignBinaryPath, "verify", "--insecure-ignore-tlog", "--key", keyPath, s.Root) // Run the command and capture its output. output, err := cmd.CombinedOutput() @@ -21,9 +46,17 @@ func VerifySignature(filePath string, keyPath string) error { } // SaveImage saves image and any signatures/attestations to the store. -func SaveImage(storePath string, ref string) error { +func SaveImage(ctx context.Context, s *store.Layout, ref string) error { + + // Ensure that the cosign binary is installed or download it if needed + cosignBinaryPath, err := ensureCosignBinary(ctx, s) + if err != nil { + return err + } + println(cosignBinaryPath) + // Command to verify the signature using Cosign. - cmd := exec.Command("cosign", "save", ref, "--dir", storePath) + cmd := exec.Command("/Users/amartin/go/bin/cosign", "save", ref, "--dir", s.Root) // Run the command and capture its output. output, err := cmd.CombinedOutput() @@ -35,9 +68,17 @@ func SaveImage(storePath string, ref string) error { } // LoadImage loads store to a remote registry. -func LoadImage(storePath string, registry string, ropts content.RegistryOptions) error { +func LoadImage(ctx context.Context, s *store.Layout, registry string, ropts content.RegistryOptions) error { + + //Ensure that the cosign binary is installed or download it if needed + cosignBinaryPath, err := ensureCosignBinary(ctx, s) + if err != nil { + return err + } + println(cosignBinaryPath) + // Command to verify the signature using Cosign. - cmd := exec.Command("cosign", "load", "--registry", registry, "--dir", storePath) + cmd := exec.Command("/Users/amartin/go/bin/cosign", "load", "--registry", registry, "--dir", s.Root) // Conditionally add extra registry flags. if ropts.Insecure { @@ -55,3 +96,193 @@ func LoadImage(storePath string, registry string, ropts content.RegistryOptions) return nil } + +// ensureCosignBinary checks if the cosign binary exists in the specified directory and installs it if not. +func ensureCosignBinary(ctx context.Context, s *store.Layout) (string, error) { + l := log.FromContext(ctx) + + // Get the current user's information + currentUser, err := user.Current() + if err != nil { + return "", fmt.Errorf("Error: %v\n", err) + } + + // Get the user's home directory + homeDir := currentUser.HomeDir + + // Construct the path to the .hauler directory + haulerDir := filepath.Join(homeDir, ".hauler") + + // Create the .hauler directory if it doesn't exist + if _, err := os.Stat(haulerDir); os.IsNotExist(err) { + // .hauler directory does not exist, create it + if err := os.MkdirAll(haulerDir, 0755); err != nil { + return "", fmt.Errorf("Error creating .hauler directory: %v\n", err) + } + l.Infof("Created .hauler directory at: %s", haulerDir) + } + + // Check if the cosign binary exists in the specified directory. + binaryPath := filepath.Join(haulerDir, "cosign") + _, err = os.Stat(binaryPath) + if err == nil { + // Cosign binary is already installed in the specified directory. + return binaryPath, nil + } + + // Cosign binary is not found. + l.Infof("Cosign binary not found. Checking to see if it exists in the store...") + + // grab binary from store if it exists, otherwise try to download it from GitHub. + // if the binary has to be downloaded, then automatically add it to the store afterwards. + err = copyCosignFromStore(ctx, s, haulerDir) + if err != nil { + l.Warnf("%s", err) + err = downloadCosign(ctx, haulerDir) + if err != nil { + return "", err + } + err = addCosignToStore(ctx, s, binaryPath) + if err != nil { + return "", err + } + } + + return binaryPath, nil +} + +// used to check if the cosign binary is in the store and if so copy it to the .hauler directory +func copyCosignFromStore(ctx context.Context, s *store.Layout, destDir string) error { + l := log.FromContext(ctx) + + ref := "hauler/cosign:latest" + r, err := reference.Parse(ref) + if err != nil { + return err + } + + found := false + if err := s.Walk(func(reference string, desc ocispec.Descriptor) error { + + if !strings.Contains(reference, r.Name()) { + return nil + } + found = true + + rc, err := s.Fetch(ctx, desc) + if err != nil { + return err + } + defer rc.Close() + + var m ocispec.Manifest + if err := json.NewDecoder(rc).Decode(&m); err != nil { + return err + } + + mapperStore, err := mapper.FromManifest(m, destDir) + if err != nil { + return err + } + + pushedDesc, err := s.Copy(ctx, reference, mapperStore, "") + if err != nil { + return err + } + + l.Infof("extracted [%s] from store with digest [%s]", ref, pushedDesc.Digest.String()) + + return nil + }); err != nil { + return err + } + + if !found { + return fmt.Errorf("Reference [%s] not found in store. Hauler will attempt to download it from Github.", ref) + } + + return nil +} + +// adds the cosign binary to the store. +// this is to help with airgapped situations where you cannot access the internet. +func addCosignToStore(ctx context.Context, s *store.Layout, binaryPath string) error { + l := log.FromContext(ctx) + + fi := v1alpha1.File{ + Path: binaryPath, + } + + copts := getter.ClientOptions{ + NameOverride: fi.Name, + } + + f := file.NewFile(fi.Path, file.WithClient(getter.NewClient(copts))) + ref, err := reference.NewTagged(f.Name(fi.Path), reference.DefaultTag) + if err != nil { + return err + } + + desc, err := s.AddOCI(ctx, f, ref.Name()) + if err != nil { + return err + } + + l.Infof("added 'file' to store at [%s], with digest [%s]", ref.Name(), desc.Digest.String()) + return nil +} + + +// used to check if the cosign binary is in the store and if so copy it to the .hauler directory +func downloadCosign(ctx context.Context, haulerDir string) error { + l := log.FromContext(ctx) + + // Define the GitHub release URL and architecture-specific binary name. + releaseURL := "https://github.com/rancher-government-solutions/cosign/releases/latest/download" + + // Determine the architecture and add it to the binary name. + arch := runtime.GOARCH + rOS := runtime.GOOS + binaryName := "cosign" + if rOS == "windows" { + binaryName = fmt.Sprintf("cosign-%s-%s.exe", rOS, arch) + } else { + binaryName = fmt.Sprintf("cosign-%s-%s", rOS, arch) + } + + // Download the binary. + downloadURL := fmt.Sprintf("%s/%s", releaseURL, binaryName) + resp, err := http.Get(downloadURL) + if err != nil { + return fmt.Errorf("error downloading cosign binary: %v", err) + } + defer resp.Body.Close() + + // Create the cosign binary file in the specified directory. + binaryFile, err := os.Create(filepath.Join(haulerDir, binaryName)) + if err != nil { + return fmt.Errorf("error creating cosign binary: %v", err) + } + defer binaryFile.Close() + + // Copy the downloaded binary to the file. + _, err = io.Copy(binaryFile, resp.Body) + if err != nil { + return fmt.Errorf("error saving cosign binary: %v", err) + } + + // Make the binary executable. + if err := os.Chmod(binaryFile.Name(), 0755); err != nil { + return fmt.Errorf("error setting executable permission: %v", err) + } + + // Rename the binary to "cosign" + oldBinaryPath := filepath.Join(haulerDir, binaryName) + newBinaryPath := filepath.Join(haulerDir, "cosign") + if err := os.Rename(oldBinaryPath, newBinaryPath); err != nil { + return fmt.Errorf("error renaming cosign binary: %v", err) + } + + l.Infof("Cosign binary downloaded and installed to %s", haulerDir) + return nil +} \ No newline at end of file From 96d92e3248f9dcbb3aeebca1546c839a18eec9b3 Mon Sep 17 00:00:00 2001 From: Adam Martin Date: Tue, 10 Oct 2023 10:48:48 -0400 Subject: [PATCH 07/11] impl for cosign functions for images & store copy Signed-off-by: Adam Martin --- cmd/hauler/cli/store/add.go | 4 ++-- cmd/hauler/cli/store/copy.go | 2 +- cmd/hauler/cli/store/extract.go | 6 ++++-- cmd/hauler/cli/store/sync.go | 2 +- 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/cmd/hauler/cli/store/add.go b/cmd/hauler/cli/store/add.go index 865d9df2..1c9ec6fb 100644 --- a/cmd/hauler/cli/store/add.go +++ b/cmd/hauler/cli/store/add.go @@ -78,7 +78,7 @@ func AddImageCmd(ctx context.Context, o *AddImageOpts, s *store.Layout, referenc // Check if the user provided a key. if o.Key != "" { // verify signature using the provided key. - err := cosign.VerifySignature(reference, o.Key) + err := cosign.VerifySignature(ctx, s, o.Key) if err != nil { return err } @@ -95,7 +95,7 @@ func storeImage(ctx context.Context, s *store.Layout, i v1alpha1.Image) error { return err } - err = cosign.SaveImage(s.Root, r.Name()) + err = cosign.SaveImage(ctx, s, r.Name()) //desc, err := s.AddOCI(ctx, img, r.Name()) if err != nil { return err diff --git a/cmd/hauler/cli/store/copy.go b/cmd/hauler/cli/store/copy.go index 064f2dab..158f4331 100644 --- a/cmd/hauler/cli/store/copy.go +++ b/cmd/hauler/cli/store/copy.go @@ -56,7 +56,7 @@ func CopyCmd(ctx context.Context, o *CopyOpts, s *store.Layout, targetRef string PlainHTTP: o.PlainHTTP, } - err := cosign.LoadImage(s.Root, components[1], ropts) + err := cosign.LoadImage(ctx, s, components[1], ropts) if err != nil { return err } diff --git a/cmd/hauler/cli/store/extract.go b/cmd/hauler/cli/store/extract.go index 9b09755a..3d9630da 100644 --- a/cmd/hauler/cli/store/extract.go +++ b/cmd/hauler/cli/store/extract.go @@ -2,6 +2,7 @@ package store import ( "context" + "strings" "encoding/json" "fmt" @@ -36,7 +37,8 @@ func ExtractCmd(ctx context.Context, o *ExtractOpts, s *store.Layout, ref string found := false if err := s.Walk(func(reference string, desc ocispec.Descriptor) error { - if reference != r.Name() { + + if !strings.Contains(reference, r.Name()) { return nil } found = true @@ -57,7 +59,7 @@ func ExtractCmd(ctx context.Context, o *ExtractOpts, s *store.Layout, ref string return err } - pushedDesc, err := s.Copy(ctx, r.Name(), mapperStore, "") + pushedDesc, err := s.Copy(ctx, reference, mapperStore, "") if err != nil { return err } diff --git a/cmd/hauler/cli/store/sync.go b/cmd/hauler/cli/store/sync.go index a5930db4..3d153b49 100644 --- a/cmd/hauler/cli/store/sync.go +++ b/cmd/hauler/cli/store/sync.go @@ -101,7 +101,7 @@ func SyncCmd(ctx context.Context, o *SyncOpts, s *store.Layout) error { // Check if the user provided a key. if o.Key != "" { // verify signature using the provided key. - err := cosign.VerifySignature(i.Name, o.Key) + err := cosign.VerifySignature(ctx, s, o.Key) if err != nil { return err } From bb9a088a84c6052fda851c0287df11b7f11c13fb Mon Sep 17 00:00:00 2001 From: Adam Martin Date: Wed, 11 Oct 2023 11:24:11 -0400 Subject: [PATCH 08/11] fixes and logging for cosign verify Signed-off-by: Adam Martin --- cmd/hauler/cli/store/add.go | 4 +++- cmd/hauler/cli/store/sync.go | 3 ++- pkg/cosign/cosign.go | 10 ++++------ 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/cmd/hauler/cli/store/add.go b/cmd/hauler/cli/store/add.go index 1c9ec6fb..4b26621f 100644 --- a/cmd/hauler/cli/store/add.go +++ b/cmd/hauler/cli/store/add.go @@ -71,6 +71,7 @@ func (o *AddImageOpts) AddFlags(cmd *cobra.Command) { } func AddImageCmd(ctx context.Context, o *AddImageOpts, s *store.Layout, reference string) error { + l := log.FromContext(ctx) cfg := v1alpha1.Image{ Name: reference, } @@ -78,10 +79,11 @@ func AddImageCmd(ctx context.Context, o *AddImageOpts, s *store.Layout, referenc // Check if the user provided a key. if o.Key != "" { // verify signature using the provided key. - err := cosign.VerifySignature(ctx, s, o.Key) + err := cosign.VerifySignature(ctx, s, o.Key, cfg.Name) if err != nil { return err } + l.Infof("Signature verified for image [%s]", cfg.Name) } return storeImage(ctx, s, cfg) diff --git a/cmd/hauler/cli/store/sync.go b/cmd/hauler/cli/store/sync.go index 3d153b49..9bee71f1 100644 --- a/cmd/hauler/cli/store/sync.go +++ b/cmd/hauler/cli/store/sync.go @@ -101,10 +101,11 @@ func SyncCmd(ctx context.Context, o *SyncOpts, s *store.Layout) error { // Check if the user provided a key. if o.Key != "" { // verify signature using the provided key. - err := cosign.VerifySignature(ctx, s, o.Key) + err := cosign.VerifySignature(ctx, s, o.Key, i.Name) if err != nil { return err } + l.Infof("Signature verified for image [%s]", cfg.Name) } err = storeImage(ctx, s, i) diff --git a/pkg/cosign/cosign.go b/pkg/cosign/cosign.go index 65db45b6..91b06a2a 100644 --- a/pkg/cosign/cosign.go +++ b/pkg/cosign/cosign.go @@ -25,7 +25,7 @@ import ( ) // VerifyFileSignature verifies the digital signature of a file using Sigstore/Cosign. -func VerifySignature(ctx context.Context, s *store.Layout, keyPath string) error { +func VerifySignature(ctx context.Context, s *store.Layout, keyPath string, ref string) error { // Ensure that the cosign binary is installed or download it if needed cosignBinaryPath, err := ensureCosignBinary(ctx, s) @@ -34,7 +34,7 @@ func VerifySignature(ctx context.Context, s *store.Layout, keyPath string) error } // Command to verify the signature using Cosign. - cmd := exec.Command(cosignBinaryPath, "verify", "--insecure-ignore-tlog", "--key", keyPath, s.Root) + cmd := exec.Command(cosignBinaryPath, "verify", "--insecure-ignore-tlog", "--key", keyPath, ref) // Run the command and capture its output. output, err := cmd.CombinedOutput() @@ -53,10 +53,9 @@ func SaveImage(ctx context.Context, s *store.Layout, ref string) error { if err != nil { return err } - println(cosignBinaryPath) // Command to verify the signature using Cosign. - cmd := exec.Command("/Users/amartin/go/bin/cosign", "save", ref, "--dir", s.Root) + cmd := exec.Command(cosignBinaryPath, "save", ref, "--dir", s.Root) // Run the command and capture its output. output, err := cmd.CombinedOutput() @@ -75,10 +74,9 @@ func LoadImage(ctx context.Context, s *store.Layout, registry string, ropts cont if err != nil { return err } - println(cosignBinaryPath) // Command to verify the signature using Cosign. - cmd := exec.Command("/Users/amartin/go/bin/cosign", "load", "--registry", registry, "--dir", s.Root) + cmd := exec.Command(cosignBinaryPath, "load", "--registry", registry, "--dir", s.Root) // Conditionally add extra registry flags. if ropts.Insecure { From 323b93ae204d58ed1a605c76c4249dd5c606962f Mon Sep 17 00:00:00 2001 From: Adam Martin Date: Wed, 11 Oct 2023 11:53:16 -0400 Subject: [PATCH 09/11] fix cosign verify logging Signed-off-by: Adam Martin --- cmd/hauler/cli/store/add.go | 2 +- cmd/hauler/cli/store/sync.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/hauler/cli/store/add.go b/cmd/hauler/cli/store/add.go index 4b26621f..a9dceb7f 100644 --- a/cmd/hauler/cli/store/add.go +++ b/cmd/hauler/cli/store/add.go @@ -83,7 +83,7 @@ func AddImageCmd(ctx context.Context, o *AddImageOpts, s *store.Layout, referenc if err != nil { return err } - l.Infof("Signature verified for image [%s]", cfg.Name) + l.Infof("signature verified for image [%s]", cfg.Name) } return storeImage(ctx, s, cfg) diff --git a/cmd/hauler/cli/store/sync.go b/cmd/hauler/cli/store/sync.go index 9bee71f1..b8b9044e 100644 --- a/cmd/hauler/cli/store/sync.go +++ b/cmd/hauler/cli/store/sync.go @@ -105,7 +105,7 @@ func SyncCmd(ctx context.Context, o *SyncOpts, s *store.Layout) error { if err != nil { return err } - l.Infof("Signature verified for image [%s]", cfg.Name) + l.Infof("signature verified for image [%s]", i.Name) } err = storeImage(ctx, s, i) From 356c46fe2871b9f60c31e004898c730d47a45e92 Mon Sep 17 00:00:00 2001 From: Adam Martin Date: Thu, 12 Oct 2023 10:34:40 -0400 Subject: [PATCH 10/11] update go.mod Signed-off-by: Adam Martin --- go.mod | 7 +++---- go.sum | 2 -- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 72c4f2e8..f5ac8c58 100644 --- a/go.mod +++ b/go.mod @@ -10,12 +10,14 @@ require ( github.com/gorilla/handlers v1.5.1 github.com/gorilla/mux v1.8.0 github.com/mholt/archiver/v3 v3.5.1 + github.com/opencontainers/go-digest v1.0.0 github.com/opencontainers/image-spec v1.1.0-rc5 github.com/pkg/errors v0.9.1 - github.com/rancherfederal/ocil v0.1.9 github.com/rs/zerolog v1.31.0 github.com/sirupsen/logrus v1.9.3 + github.com/spf13/afero v1.10.0 github.com/spf13/cobra v1.7.0 + golang.org/x/sync v0.4.0 helm.sh/helm/v3 v3.13.0 k8s.io/apimachinery v0.28.2 k8s.io/client-go v0.28.2 @@ -110,7 +112,6 @@ require ( github.com/morikuni/aec v1.0.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/nwaples/rardecode v1.1.0 // indirect - github.com/opencontainers/go-digest v1.0.0 // indirect github.com/peterbourgon/diskv v2.0.1+incompatible // indirect github.com/pierrec/lz4/v4 v4.1.2 // indirect github.com/prometheus/client_golang v1.16.0 // indirect @@ -123,7 +124,6 @@ require ( github.com/rubenv/sql-migrate v1.5.2 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/shopspring/decimal v1.3.1 // indirect - github.com/spf13/afero v1.10.0 // indirect github.com/spf13/cast v1.5.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/ulikunitz/xz v0.5.9 // indirect @@ -140,7 +140,6 @@ require ( golang.org/x/crypto v0.13.0 // indirect golang.org/x/net v0.13.0 // indirect golang.org/x/oauth2 v0.8.0 // indirect - golang.org/x/sync v0.4.0 // indirect golang.org/x/sys v0.12.0 // indirect golang.org/x/term v0.12.0 // indirect golang.org/x/text v0.13.0 // indirect diff --git a/go.sum b/go.sum index d326077a..28b3d189 100644 --- a/go.sum +++ b/go.sum @@ -454,8 +454,6 @@ github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsT github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg= github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= -github.com/rancherfederal/ocil v0.1.9 h1:pmiUQCh2HTIMDD9tDj/UqBAAxq4yloLFgd2WnrZnQgc= -github.com/rancherfederal/ocil v0.1.9/go.mod h1:l4d1cHHfdXDGtio32AYDjG6n1i1JxQK+kAom0cVf0SY= github.com/redis/go-redis/extra/rediscmd/v9 v9.0.5 h1:EaDatTxkdHG+U3Bk4EUr+DZ7fOGwTfezUiUJMaIcaho= github.com/redis/go-redis/extra/rediscmd/v9 v9.0.5/go.mod h1:fyalQWdtzDBECAQFBJuQe5bzQ02jGd5Qcbgb97Flm7U= github.com/redis/go-redis/extra/redisotel/v9 v9.0.5 h1:EfpWLLCyXw8PSM2/XNJLjI3Pb27yVE+gIAfeqp8LUCc= From f2b0c44af3a35e4b4d0c7bd1f15c215ed871c193 Mon Sep 17 00:00:00 2001 From: Adam Martin Date: Thu, 12 Oct 2023 12:05:35 -0400 Subject: [PATCH 11/11] polish up cosign verify for hauler store sync Signed-off-by: Adam Martin --- cmd/hauler/cli/store/sync.go | 18 +++++++++++++----- go.mod | 2 +- pkg/apis/hauler.cattle.io/v1alpha1/image.go | 4 ++++ pkg/cosign/cosign.go | 4 ++-- 4 files changed, 20 insertions(+), 8 deletions(-) diff --git a/cmd/hauler/cli/store/sync.go b/cmd/hauler/cli/store/sync.go index b8b9044e..7f90b345 100644 --- a/cmd/hauler/cli/store/sync.go +++ b/cmd/hauler/cli/store/sync.go @@ -10,6 +10,7 @@ import ( "github.com/spf13/cobra" "helm.sh/helm/v3/pkg/action" "k8s.io/apimachinery/pkg/util/yaml" + "github.com/mitchellh/go-homedir" "github.com/rancherfederal/hauler/pkg/store" @@ -32,7 +33,7 @@ func (o *SyncOpts) AddFlags(cmd *cobra.Command) { f := cmd.Flags() f.StringSliceVarP(&o.ContentFiles, "files", "f", []string{}, "Path to content files") - f.StringVarP(&o.Key, "key", "k", "", "(Optional) Path to the key for digital signature verification") + f.StringVarP(&o.Key, "key", "k", "", "(Optional) Path to the key for image signature verification") } func SyncCmd(ctx context.Context, o *SyncOpts, s *store.Layout) error { @@ -99,15 +100,22 @@ func SyncCmd(ctx context.Context, o *SyncOpts, s *store.Layout) error { for _, i := range cfg.Spec.Images { // Check if the user provided a key. - if o.Key != "" { + if o.Key != "" || i.Key != "" { + key := o.Key + if i.Key != "" { + key, err = homedir.Expand(i.Key) + } + l.Debugf("key for image [%s]", key) + // verify signature using the provided key. - err := cosign.VerifySignature(ctx, s, o.Key, i.Name) + err := cosign.VerifySignature(ctx, s, key, i.Name) if err != nil { - return err + l.Errorf("signature verification failed for image [%s]. ** hauler will skip adding this image to the store **:\n%v", i.Name, err) + continue } l.Infof("signature verified for image [%s]", i.Name) } - + err = storeImage(ctx, s, i) if err != nil { return err diff --git a/go.mod b/go.mod index f5ac8c58..775b9f07 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/gorilla/handlers v1.5.1 github.com/gorilla/mux v1.8.0 github.com/mholt/archiver/v3 v3.5.1 + github.com/mitchellh/go-homedir v1.1.0 github.com/opencontainers/go-digest v1.0.0 github.com/opencontainers/image-spec v1.1.0-rc5 github.com/pkg/errors v0.9.1 @@ -100,7 +101,6 @@ require ( github.com/mattn/go-runewidth v0.0.9 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect - github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/go-wordwrap v1.0.1 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/moby/locker v1.0.1 // indirect diff --git a/pkg/apis/hauler.cattle.io/v1alpha1/image.go b/pkg/apis/hauler.cattle.io/v1alpha1/image.go index cde4d2c0..b7a23602 100644 --- a/pkg/apis/hauler.cattle.io/v1alpha1/image.go +++ b/pkg/apis/hauler.cattle.io/v1alpha1/image.go @@ -20,4 +20,8 @@ type ImageSpec struct { type Image struct { // Name is the full location for the image, can be referenced by tags or digests Name string `json:"name"` + + // Path is the path to the cosign public key used for verifying image signatures + //Key string `json:"key,omitempty"` + Key string `json:"key"` } diff --git a/pkg/cosign/cosign.go b/pkg/cosign/cosign.go index 91b06a2a..a6787122 100644 --- a/pkg/cosign/cosign.go +++ b/pkg/cosign/cosign.go @@ -54,7 +54,7 @@ func SaveImage(ctx context.Context, s *store.Layout, ref string) error { return err } - // Command to verify the signature using Cosign. + // Command to save/download an image using Cosign. cmd := exec.Command(cosignBinaryPath, "save", ref, "--dir", s.Root) // Run the command and capture its output. @@ -75,7 +75,7 @@ func LoadImage(ctx context.Context, s *store.Layout, registry string, ropts cont return err } - // Command to verify the signature using Cosign. + // Command to upload index to a remote registry using Cosign. cmd := exec.Command(cosignBinaryPath, "load", "--registry", registry, "--dir", s.Root) // Conditionally add extra registry flags.