Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add OCI hauler manifests. #136

Merged
merged 12 commits into from
Nov 3, 2023
Merged

Add OCI hauler manifests. #136

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
39e0fed
pull carbide flavored hauler manifests from reg
amartin120 Oct 20, 2023
566c918
remove temp constant
amartin120 Oct 20, 2023
b811810
remove temp hardcoding
amartin120 Oct 20, 2023
fd27851
add comments for new sync flags
amartin120 Oct 20, 2023
d9f2b23
fixes for version and registry serve
amartin120 Oct 21, 2023
e0a1d13
band-aid for store info... needs love
amartin120 Oct 21, 2023
80fc969
add sbom to info logic
amartin120 Oct 24, 2023
ce8f9d6
adjust a few text descriptions
amartin120 Oct 26, 2023
65426ff
adjust tag names with +
amartin120 Nov 3, 2023
5b8caac
Merge branch 'main' of github.com:rancherfederal/hauler into oci-haul…
amartin120 Nov 3, 2023
d4db80f
removed testing file
amartin120 Nov 3, 2023
b9f49b1
resolved conflicts
amartin120 Nov 3, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 12 additions & 5 deletions cmd/hauler/cli/store/info.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,6 @@ func InfoCmd(ctx context.Context, o *InfoOpts, s *store.Layout) error {
if _, ok := desc.Annotations[ocispec.AnnotationRefName]; !ok {
return nil
}

rc, err := s.Fetch(ctx, desc)
if err != nil {
return err
Expand All @@ -49,9 +48,11 @@ func InfoCmd(ctx context.Context, o *InfoOpts, s *store.Layout) error {
if err := json.NewDecoder(rc).Decode(&m); err != nil {
return err
}

i := newItem(s, desc, m)
items = append(items, i)
var emptyItem item
if i != emptyItem {
items = append(items, i)
}

return nil
}); err != nil {
Expand All @@ -78,7 +79,7 @@ func buildTable(items ...item) string {
fmt.Fprintf(tw, "---------\t----\t--------\t----\n")

for _, i := range items {
if i.Type != "unknown" {
if i.Type != "" {
fmt.Fprintf(tw, "%s\t%s\t%d\t%s\n",
i.Reference, i.Type, i.Layers, i.Size,
)
Expand All @@ -104,6 +105,12 @@ type item struct {
}

func newItem(s *store.Layout, desc ocispec.Descriptor, m ocispec.Manifest) item {
if desc.Annotations["kind"] == "dev.cosignproject.cosign/atts" ||
desc.Annotations["kind"] == "dev.cosignproject.cosign/sigs" ||
desc.Annotations["kind"] == "dev.cosignproject.cosign/sboms" {
return item{}
}

var size int64 = 0
for _, l := range m.Layers {
size = +l.Size
Expand All @@ -119,7 +126,7 @@ func newItem(s *store.Layout, desc ocispec.Descriptor, m ocispec.Manifest) item
case consts.FileLocalConfigMediaType, consts.FileHttpConfigMediaType:
ctype = "file"
default:
ctype = "unknown"
ctype = "image"
}

ref, err := reference.Parse(desc.Annotations[ocispec.AnnotationRefName])
Expand Down
261 changes: 152 additions & 109 deletions cmd/hauler/cli/store/sync.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,18 +6,19 @@ import (
"fmt"
"io"
"os"
"strings"

"github.com/spf13/cobra"
"helm.sh/helm/v3/pkg/action"
"k8s.io/apimachinery/pkg/util/yaml"
"github.com/mitchellh/go-homedir"

"github.com/rancherfederal/hauler/pkg/store"

"github.com/rancherfederal/hauler/pkg/apis/hauler.cattle.io/v1alpha1"
tchart "github.com/rancherfederal/hauler/pkg/collection/chart"
"github.com/rancherfederal/hauler/pkg/collection/imagetxt"
"github.com/rancherfederal/hauler/pkg/collection/k3s"
"github.com/rancherfederal/hauler/pkg/consts"
"github.com/rancherfederal/hauler/pkg/content"
"github.com/rancherfederal/hauler/pkg/cosign"
"github.com/rancherfederal/hauler/pkg/log"
Expand All @@ -27,13 +28,15 @@ type SyncOpts struct {
*RootOpts
ContentFiles []string
Key string
Products []string
}

func (o *SyncOpts) AddFlags(cmd *cobra.Command) {
f := cmd.Flags()

f.StringSliceVarP(&o.ContentFiles, "files", "f", []string{}, "Path to content files")
f.StringVarP(&o.Key, "key", "k", "", "(Optional) Path to the key for image signature verification")
f.StringVarP(&o.Key, "key", "k", "", "(Optional) Path to the key for signature verification")
f.StringSliceVar(&o.Products, "products", []string{}, "Used for RGS Carbide customers to supply a product and version and Hauler will retrieve the images. i.e. '--product rancher=v2.7.6'")
}

func SyncCmd(ctx context.Context, o *SyncOpts, s *store.Layout) error {
Expand All @@ -45,157 +48,197 @@ func SyncCmd(ctx context.Context, o *SyncOpts, s *store.Layout) error {
return err
}

// if passed products, check for a remote manifest to retrieve and use.
for _, product := range o.Products {
l.Infof("processing content file for product: '%s'", product)
parts := strings.Split(product, "=")
tag := strings.ReplaceAll(parts[1], "+", "-")
manifestLoc := fmt.Sprintf("%s/hauler/%s-manifest.yaml:%s", consts.CarbideRegistry, parts[0], tag)
l.Infof("retrieving product manifest from: '%s'", manifestLoc)
img := v1alpha1.Image{
Name: manifestLoc,
}
err := storeImage(ctx, s, img)
if err != nil {
return err
}
err = ExtractCmd(ctx, &ExtractOpts{RootOpts: o.RootOpts}, s, fmt.Sprintf("hauler/%s-manifest.yaml:%s", parts[0],tag))
if err != nil {
return err
}
filename := fmt.Sprintf("%s-manifest.yaml", parts[0])

fi, err := os.Open(filename)
if err != nil {
return err
}
err = processContent(ctx, fi, o, s)
if err != nil {
return err
}
}

// if passed a local manifest, process it
for _, filename := range o.ContentFiles {
l.Debugf("processing content file: '%s'", filename)
fi, err := os.Open(filename)
if err != nil {
return err
}
err = processContent(ctx, fi, o, s)
if err != nil {
return err
}
}

return nil
}

reader := yaml.NewYAMLReader(bufio.NewReader(fi))
func processContent(ctx context.Context, fi *os.File, o *SyncOpts, s *store.Layout) error {
l := log.FromContext(ctx)

var docs [][]byte
for {
raw, err := reader.Read()
if err == io.EOF {
break
}
if err != nil {
return err
}
reader := yaml.NewYAMLReader(bufio.NewReader(fi))

docs = append(docs, raw)
var docs [][]byte
for {
raw, err := reader.Read()
if err == io.EOF {
break
}
if err != nil {
return err
}

for _, doc := range docs {
obj, err := content.Load(doc)
if err != nil {
l.Debugf("skipping sync of unknown content")
continue
}
docs = append(docs, raw)
}

l.Infof("syncing [%s] to store", obj.GroupVersionKind().String())
for _, doc := range docs {
obj, err := content.Load(doc)
if err != nil {
l.Debugf("skipping sync of unknown content")
continue
}

// TODO: Should type switch instead...
switch obj.GroupVersionKind().Kind {
case v1alpha1.FilesContentKind:
var cfg v1alpha1.Files
if err := yaml.Unmarshal(doc, &cfg); err != nil {
return err
}
l.Infof("syncing [%s] to store", obj.GroupVersionKind().String())

for _, f := range cfg.Spec.Files {
err := storeFile(ctx, s, f)
if err != nil {
return err
}
}
// TODO: Should type switch instead...
switch obj.GroupVersionKind().Kind {
case v1alpha1.FilesContentKind:
var cfg v1alpha1.Files
if err := yaml.Unmarshal(doc, &cfg); err != nil {
return err
}

case v1alpha1.ImagesContentKind:
var cfg v1alpha1.Images
if err := yaml.Unmarshal(doc, &cfg); err != nil {
for _, f := range cfg.Spec.Files {
err := storeFile(ctx, s, f)
if err != nil {
return err
}
}

case v1alpha1.ImagesContentKind:
var cfg v1alpha1.Images
if err := yaml.Unmarshal(doc, &cfg); err != nil {
return err
}

for _, i := range cfg.Spec.Images {

for _, i := range cfg.Spec.Images {

// Check if the user provided a key.
if o.Key != "" || i.Key != "" {
key := o.Key
if i.Key != "" {
key, err = homedir.Expand(i.Key)
}
l.Debugf("key for image [%s]", key)

// verify signature using the provided key.
err := cosign.VerifySignature(ctx, s, key, i.Name)
if err != nil {
l.Errorf("signature verification failed for image [%s]. ** hauler will skip adding this image to the store **:\n%v", i.Name, err)
continue
}
l.Infof("signature verified for image [%s]", i.Name)
// Check if the user provided a key.
if o.Key != "" || i.Key != "" {
key := o.Key
if i.Key != "" {
key, err = homedir.Expand(i.Key)
}
l.Debugf("key for image [%s]", key)

err = storeImage(ctx, s, i)
// verify signature using the provided key.
err := cosign.VerifySignature(ctx, s, key, i.Name)
if err != nil {
return err
l.Errorf("signature verification failed for image [%s]. ** hauler will skip adding this image to the store **:\n%v", i.Name, err)
continue
}
l.Infof("signature verified for image [%s]", i.Name)
}

case v1alpha1.ChartsContentKind:
var cfg v1alpha1.Charts
if err := yaml.Unmarshal(doc, &cfg); err != nil {

err = storeImage(ctx, s, i)
if err != nil {
return err
}
}

for _, ch := range cfg.Spec.Charts {
// TODO: Provide a way to configure syncs
err := storeChart(ctx, s, ch, &action.ChartPathOptions{})
if err != nil {
return err
}
}

case v1alpha1.K3sCollectionKind:
var cfg v1alpha1.K3s
if err := yaml.Unmarshal(doc, &cfg); err != nil {
return err
}
case v1alpha1.ChartsContentKind:
var cfg v1alpha1.Charts
if err := yaml.Unmarshal(doc, &cfg); err != nil {
return err
}

k, err := k3s.NewK3s(cfg.Spec.Version)
for _, ch := range cfg.Spec.Charts {
// TODO: Provide a way to configure syncs
err := storeChart(ctx, s, ch, &action.ChartPathOptions{})
if err != nil {
return err
}
}

if _, err := s.AddOCICollection(ctx, k); err != nil {
return err
}
case v1alpha1.K3sCollectionKind:
var cfg v1alpha1.K3s
if err := yaml.Unmarshal(doc, &cfg); err != nil {
return err
}

case v1alpha1.ChartsCollectionKind:
var cfg v1alpha1.ThickCharts
if err := yaml.Unmarshal(doc, &cfg); err != nil {
return err
}
k, err := k3s.NewK3s(cfg.Spec.Version)
if err != nil {
return err
}

for _, cfg := range cfg.Spec.Charts {
tc, err := tchart.NewThickChart(cfg, &action.ChartPathOptions{
RepoURL: cfg.RepoURL,
Version: cfg.Version,
})
if err != nil {
return err
}
if _, err := s.AddOCICollection(ctx, k); err != nil {
return err
}

if _, err := s.AddOCICollection(ctx, tc); err != nil {
return err
}
case v1alpha1.ChartsCollectionKind:
var cfg v1alpha1.ThickCharts
if err := yaml.Unmarshal(doc, &cfg); err != nil {
return err
}

for _, cfg := range cfg.Spec.Charts {
tc, err := tchart.NewThickChart(cfg, &action.ChartPathOptions{
RepoURL: cfg.RepoURL,
Version: cfg.Version,
})
if err != nil {
return err
}

case v1alpha1.ImageTxtsContentKind:
var cfg v1alpha1.ImageTxts
if err := yaml.Unmarshal(doc, &cfg); err != nil {
if _, err := s.AddOCICollection(ctx, tc); err != nil {
return err
}
}

for _, cfgIt := range cfg.Spec.ImageTxts {
it, err := imagetxt.New(cfgIt.Ref,
imagetxt.WithIncludeSources(cfgIt.Sources.Include...),
imagetxt.WithExcludeSources(cfgIt.Sources.Exclude...),
)
if err != nil {
return fmt.Errorf("convert ImageTxt %s: %v", cfg.Name, err)
}
case v1alpha1.ImageTxtsContentKind:
var cfg v1alpha1.ImageTxts
if err := yaml.Unmarshal(doc, &cfg); err != nil {
return err
}

if _, err := s.AddOCICollection(ctx, it); err != nil {
return fmt.Errorf("add ImageTxt %s to store: %v", cfg.Name, err)
}
for _, cfgIt := range cfg.Spec.ImageTxts {
it, err := imagetxt.New(cfgIt.Ref,
imagetxt.WithIncludeSources(cfgIt.Sources.Include...),
imagetxt.WithExcludeSources(cfgIt.Sources.Exclude...),
)
if err != nil {
return fmt.Errorf("convert ImageTxt %s: %v", cfg.Name, err)
}

default:
return fmt.Errorf("unrecognized content/collection type: %s", obj.GroupVersionKind().String())
if _, err := s.AddOCICollection(ctx, it); err != nil {
return fmt.Errorf("add ImageTxt %s to store: %v", cfg.Name, err)
}
}

default:
return fmt.Errorf("unrecognized content/collection type: %s", obj.GroupVersionKind().String())
}
}

return nil
}
Loading
Loading