Skip to content

Commit

Permalink
Add OCI hauler manifests. (#136)
Browse files Browse the repository at this point in the history 
* pull carbide flavored hauler manifests from reg
* remove temp constant
* remove temp hardcoding
* add comments for new sync flags
* fixes for version and registry serve
* band-aid for store info... needs love
* add sbom to info logic
* adjust a few text descriptions
* adjust tag names with +
* removed testing file

Signed-off-by: Adam Martin <[email protected]>
  • Loading branch information
@amartin120
amartin120 committed Nov 3, 2023
1 parent f8c16a1 commit 4d950f7
Show file tree
Hide file tree
Showing 7 changed files with 415 additions and 189 deletions.
17 changes: 12 additions & 5 deletions cmd/hauler/cli/store/info.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,6 @@ func InfoCmd(ctx context.Context, o *InfoOpts, s *store.Layout) error {
if _, ok := desc.Annotations[ocispec.AnnotationRefName]; !ok {
return nil
}

rc, err := s.Fetch(ctx, desc)
if err != nil {
return err
Expand All @@ -49,9 +48,11 @@ func InfoCmd(ctx context.Context, o *InfoOpts, s *store.Layout) error {
if err := json.NewDecoder(rc).Decode(&m); err != nil {
return err
}

i := newItem(s, desc, m)
items = append(items, i)
var emptyItem item
if i != emptyItem {
items = append(items, i)
}

return nil
}); err != nil {
Expand All @@ -78,7 +79,7 @@ func buildTable(items ...item) string {
fmt.Fprintf(tw, "---------\t----\t--------\t----\n")

for _, i := range items {
if i.Type != "unknown" {
if i.Type != "" {
fmt.Fprintf(tw, "%s\t%s\t%d\t%s\n",
i.Reference, i.Type, i.Layers, i.Size,
)
Expand All @@ -104,6 +105,12 @@ type item struct {
}

func newItem(s *store.Layout, desc ocispec.Descriptor, m ocispec.Manifest) item {
if desc.Annotations["kind"] == "dev.cosignproject.cosign/atts" ||
desc.Annotations["kind"] == "dev.cosignproject.cosign/sigs" ||
desc.Annotations["kind"] == "dev.cosignproject.cosign/sboms" {
return item{}
}

var size int64 = 0
for _, l := range m.Layers {
size = +l.Size
Expand All @@ -119,7 +126,7 @@ func newItem(s *store.Layout, desc ocispec.Descriptor, m ocispec.Manifest) item
case consts.FileLocalConfigMediaType, consts.FileHttpConfigMediaType:
ctype = "file"
default:
ctype = "unknown"
ctype = "image"
}

ref, err := reference.Parse(desc.Annotations[ocispec.AnnotationRefName])
Expand Down
261 changes: 152 additions & 109 deletions cmd/hauler/cli/store/sync.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,18 +6,19 @@ import (
"fmt"
"io"
"os"
"strings"

"github.com/spf13/cobra"
"helm.sh/helm/v3/pkg/action"
"k8s.io/apimachinery/pkg/util/yaml"
"github.com/mitchellh/go-homedir"

"github.com/rancherfederal/hauler/pkg/store"

"github.com/rancherfederal/hauler/pkg/apis/hauler.cattle.io/v1alpha1"
tchart "github.com/rancherfederal/hauler/pkg/collection/chart"
"github.com/rancherfederal/hauler/pkg/collection/imagetxt"
"github.com/rancherfederal/hauler/pkg/collection/k3s"
"github.com/rancherfederal/hauler/pkg/consts"
"github.com/rancherfederal/hauler/pkg/content"
"github.com/rancherfederal/hauler/pkg/cosign"
"github.com/rancherfederal/hauler/pkg/log"
Expand All @@ -27,13 +28,15 @@ type SyncOpts struct {
*RootOpts
ContentFiles []string
Key string
Products []string
}

func (o *SyncOpts) AddFlags(cmd *cobra.Command) {
f := cmd.Flags()

f.StringSliceVarP(&o.ContentFiles, "files", "f", []string{}, "Path to content files")
f.StringVarP(&o.Key, "key", "k", "", "(Optional) Path to the key for image signature verification")
f.StringVarP(&o.Key, "key", "k", "", "(Optional) Path to the key for signature verification")
f.StringSliceVar(&o.Products, "products", []string{}, "Used for RGS Carbide customers to supply a product and version and Hauler will retrieve the images. i.e. '--product rancher=v2.7.6'")
}

func SyncCmd(ctx context.Context, o *SyncOpts, s *store.Layout) error {
Expand All @@ -45,157 +48,197 @@ func SyncCmd(ctx context.Context, o *SyncOpts, s *store.Layout) error {
return err
}

// if passed products, check for a remote manifest to retrieve and use.
for _, product := range o.Products {
l.Infof("processing content file for product: '%s'", product)
parts := strings.Split(product, "=")
tag := strings.ReplaceAll(parts[1], "+", "-")
manifestLoc := fmt.Sprintf("%s/hauler/%s-manifest.yaml:%s", consts.CarbideRegistry, parts[0], tag)
l.Infof("retrieving product manifest from: '%s'", manifestLoc)
img := v1alpha1.Image{
Name: manifestLoc,
}
err := storeImage(ctx, s, img)
if err != nil {
return err
}
err = ExtractCmd(ctx, &ExtractOpts{RootOpts: o.RootOpts}, s, fmt.Sprintf("hauler/%s-manifest.yaml:%s", parts[0],tag))
if err != nil {
return err
}
filename := fmt.Sprintf("%s-manifest.yaml", parts[0])

fi, err := os.Open(filename)
if err != nil {
return err
}
err = processContent(ctx, fi, o, s)
if err != nil {
return err
}
}

// if passed a local manifest, process it
for _, filename := range o.ContentFiles {
l.Debugf("processing content file: '%s'", filename)
fi, err := os.Open(filename)
if err != nil {
return err
}
err = processContent(ctx, fi, o, s)
if err != nil {
return err
}
}

return nil
}

reader := yaml.NewYAMLReader(bufio.NewReader(fi))
func processContent(ctx context.Context, fi *os.File, o *SyncOpts, s *store.Layout) error {
l := log.FromContext(ctx)

var docs [][]byte
for {
raw, err := reader.Read()
if err == io.EOF {
break
}
if err != nil {
return err
}
reader := yaml.NewYAMLReader(bufio.NewReader(fi))

docs = append(docs, raw)
var docs [][]byte
for {
raw, err := reader.Read()
if err == io.EOF {
break
}
if err != nil {
return err
}

for _, doc := range docs {
obj, err := content.Load(doc)
if err != nil {
l.Debugf("skipping sync of unknown content")
continue
}
docs = append(docs, raw)
}

l.Infof("syncing [%s] to store", obj.GroupVersionKind().String())
for _, doc := range docs {
obj, err := content.Load(doc)
if err != nil {
l.Debugf("skipping sync of unknown content")
continue
}

// TODO: Should type switch instead...
switch obj.GroupVersionKind().Kind {
case v1alpha1.FilesContentKind:
var cfg v1alpha1.Files
if err := yaml.Unmarshal(doc, &cfg); err != nil {
return err
}
l.Infof("syncing [%s] to store", obj.GroupVersionKind().String())

for _, f := range cfg.Spec.Files {
err := storeFile(ctx, s, f)
if err != nil {
return err
}
}
// TODO: Should type switch instead...
switch obj.GroupVersionKind().Kind {
case v1alpha1.FilesContentKind:
var cfg v1alpha1.Files
if err := yaml.Unmarshal(doc, &cfg); err != nil {
return err
}

case v1alpha1.ImagesContentKind:
var cfg v1alpha1.Images
if err := yaml.Unmarshal(doc, &cfg); err != nil {
for _, f := range cfg.Spec.Files {
err := storeFile(ctx, s, f)
if err != nil {
return err
}
}

case v1alpha1.ImagesContentKind:
var cfg v1alpha1.Images
if err := yaml.Unmarshal(doc, &cfg); err != nil {
return err
}

for _, i := range cfg.Spec.Images {

for _, i := range cfg.Spec.Images {

// Check if the user provided a key.
if o.Key != "" || i.Key != "" {
key := o.Key
if i.Key != "" {
key, err = homedir.Expand(i.Key)
}
l.Debugf("key for image [%s]", key)

// verify signature using the provided key.
err := cosign.VerifySignature(ctx, s, key, i.Name)
if err != nil {
l.Errorf("signature verification failed for image [%s]. ** hauler will skip adding this image to the store **:\n%v", i.Name, err)
continue
}
l.Infof("signature verified for image [%s]", i.Name)
// Check if the user provided a key.
if o.Key != "" || i.Key != "" {
key := o.Key
if i.Key != "" {
key, err = homedir.Expand(i.Key)
}
l.Debugf("key for image [%s]", key)

err = storeImage(ctx, s, i)
// verify signature using the provided key.
err := cosign.VerifySignature(ctx, s, key, i.Name)
if err != nil {
return err
l.Errorf("signature verification failed for image [%s]. ** hauler will skip adding this image to the store **:\n%v", i.Name, err)
continue
}
l.Infof("signature verified for image [%s]", i.Name)
}

case v1alpha1.ChartsContentKind:
var cfg v1alpha1.Charts
if err := yaml.Unmarshal(doc, &cfg); err != nil {

err = storeImage(ctx, s, i)
if err != nil {
return err
}
}

for _, ch := range cfg.Spec.Charts {
// TODO: Provide a way to configure syncs
err := storeChart(ctx, s, ch, &action.ChartPathOptions{})
if err != nil {
return err
}
}

case v1alpha1.K3sCollectionKind:
var cfg v1alpha1.K3s
if err := yaml.Unmarshal(doc, &cfg); err != nil {
return err
}
case v1alpha1.ChartsContentKind:
var cfg v1alpha1.Charts
if err := yaml.Unmarshal(doc, &cfg); err != nil {
return err
}

k, err := k3s.NewK3s(cfg.Spec.Version)
for _, ch := range cfg.Spec.Charts {
// TODO: Provide a way to configure syncs
err := storeChart(ctx, s, ch, &action.ChartPathOptions{})
if err != nil {
return err
}
}

if _, err := s.AddOCICollection(ctx, k); err != nil {
return err
}
case v1alpha1.K3sCollectionKind:
var cfg v1alpha1.K3s
if err := yaml.Unmarshal(doc, &cfg); err != nil {
return err
}

case v1alpha1.ChartsCollectionKind:
var cfg v1alpha1.ThickCharts
if err := yaml.Unmarshal(doc, &cfg); err != nil {
return err
}
k, err := k3s.NewK3s(cfg.Spec.Version)
if err != nil {
return err
}

for _, cfg := range cfg.Spec.Charts {
tc, err := tchart.NewThickChart(cfg, &action.ChartPathOptions{
RepoURL: cfg.RepoURL,
Version: cfg.Version,
})
if err != nil {
return err
}
if _, err := s.AddOCICollection(ctx, k); err != nil {
return err
}

if _, err := s.AddOCICollection(ctx, tc); err != nil {
return err
}
case v1alpha1.ChartsCollectionKind:
var cfg v1alpha1.ThickCharts
if err := yaml.Unmarshal(doc, &cfg); err != nil {
return err
}

for _, cfg := range cfg.Spec.Charts {
tc, err := tchart.NewThickChart(cfg, &action.ChartPathOptions{
RepoURL: cfg.RepoURL,
Version: cfg.Version,
})
if err != nil {
return err
}

case v1alpha1.ImageTxtsContentKind:
var cfg v1alpha1.ImageTxts
if err := yaml.Unmarshal(doc, &cfg); err != nil {
if _, err := s.AddOCICollection(ctx, tc); err != nil {
return err
}
}

for _, cfgIt := range cfg.Spec.ImageTxts {
it, err := imagetxt.New(cfgIt.Ref,
imagetxt.WithIncludeSources(cfgIt.Sources.Include...),
imagetxt.WithExcludeSources(cfgIt.Sources.Exclude...),
)
if err != nil {
return fmt.Errorf("convert ImageTxt %s: %v", cfg.Name, err)
}
case v1alpha1.ImageTxtsContentKind:
var cfg v1alpha1.ImageTxts
if err := yaml.Unmarshal(doc, &cfg); err != nil {
return err
}

if _, err := s.AddOCICollection(ctx, it); err != nil {
return fmt.Errorf("add ImageTxt %s to store: %v", cfg.Name, err)
}
for _, cfgIt := range cfg.Spec.ImageTxts {
it, err := imagetxt.New(cfgIt.Ref,
imagetxt.WithIncludeSources(cfgIt.Sources.Include...),
imagetxt.WithExcludeSources(cfgIt.Sources.Exclude...),
)
if err != nil {
return fmt.Errorf("convert ImageTxt %s: %v", cfg.Name, err)
}

default:
return fmt.Errorf("unrecognized content/collection type: %s", obj.GroupVersionKind().String())
if _, err := s.AddOCICollection(ctx, it); err != nil {
return fmt.Errorf("add ImageTxt %s to store: %v", cfg.Name, err)
}
}

default:
return fmt.Errorf("unrecognized content/collection type: %s", obj.GroupVersionKind().String())
}
}

return nil
}
Loading

0 comments on commit 4d950f7

Please sign in to comment.