feat: scaffolded functions certs (#1823)

.gitignore

@@ -4,6 +4,7 @@
 /cmd/func.yaml
 /templates/typescript/cloudevents/build
 /templates/typescript/http/build
+/templates/go/cloudevents/go.sum
 /coverage.out
 /coverage.txt
 /.coverage

Makefile

@@ -27,8 +27,11 @@ KVER    ?= $(shell git describe --tags --match 'knative-*')
 LDFLAGS := "-X main.date=$(DATE) -X main.vers=$(VERS) -X main.kver=$(KVER) -X main.hash=$(HASH)"
 
 # All Code prerequisites, including generated files, etc.
-CODE := $(shell find . -name '*.go') generate/zz_filesystem_generated.go go.mod schema/func_yaml-schema.json
-TEMPLATES := $(shell find templates -name '*' -type f)
+CODE := $(shell find . -name '*.go') \
+				generate/zz_filesystem_generated.go \
+				schema/func_yaml-schema.json \
+				templates/certs/ca-certificates.crt \
+				go.mod
 
 .PHONY: test docs
 
@@ -153,6 +156,13 @@ update-runtimes:  ## Update Scaffolding Runtimes
 	cd templates/go/scaffolding/instanced-cloudevents && go get -u github.com/lkingland/func-runtime-go/cloudevents
 	cd templates/go/scaffolding/static-cloudevents && go get -u github.com/lkingland/func-runtime-go/cloudevents
 
+.PHONY: cert
+certs: templates/certs/ca-certificates.crt ## Update root certificates
+
+.PHONY: templates/certs/ca-certificates.crt
+templates/certs/ca-certificates.crt:
+	# Updating root certificates
+	curl --output templates/certs/ca-certificates.crt https://curl.se/ca/cacert.pem
 
 ###################
 ##@ Extended Testing (cluster required)
@@ -219,6 +229,7 @@ $(BIN_WINDOWS): generate/zz_filesystem_generated.go
 ######################
 ##@ Schemas
 ######################
+
 schema-generate: schema/func_yaml-schema.json ## Generate func.yaml schema
 schema/func_yaml-schema.json: pkg/functions/function.go pkg/functions/function_*.go
 	go run schema/generator/main.go

generate/templates/main.go

@@ -80,7 +80,7 @@ func main() {
 			if err != nil {
 				return err
 			}
-			_, err = w.Write([]byte(symlinkTarget))
+			_, err = w.Write([]byte(filepath.ToSlash(symlinkTarget)))
 			return err
 		case info.Mode()&fs.ModeType == 0: // regular file
 			f, err := os.Open(path)

pkg/filesystem/filesystem.go

@@ -173,7 +173,11 @@ func (o osFilesystem) Stat(name string) (fs.FileInfo, error) {
 
 func (o osFilesystem) Readlink(link string) (string, error) {
 	link = filepath.FromSlash(link)
-	return os.Readlink(filepath.Join(o.root, link))
+	t, err := os.Readlink(filepath.Join(o.root, link))
+	if err != nil {
+		return "", err
+	}
+	return filepath.ToSlash(t), nil
 }
 
 // subFS exposes subdirectory of underlying FS, this is similar to `chroot`.

pkg/filesystem/filesystem_test.go

@@ -158,7 +158,7 @@ func loadLocalFiles(root string) ([]FileInfo, error) {
 			}
 		case fs.ModeSymlink:
 			t, _ := os.Readlink(path)
-			bs = []byte(t)
+			bs = []byte(filepath.ToSlash(t))
 		}
 		path, err = filepath.Rel(root, path)
 		if err != nil {

pkg/functions/repository.go

@@ -275,6 +275,10 @@ func repositoryRuntimes(fs filesystem.Filesystem, repoName string, repoConfig re
 		if !fi.IsDir() || strings.HasPrefix(fi.Name(), ".") {
 			continue
 		}
+		// ignore the reserved word "certs"
+		if fi.Name() == "certs" {
+			continue
+		}
 		// Runtime, defaulted to values inherited from the repository
 		runtime := Runtime{
 			Name: fi.Name(),

pkg/oci/builder_test.go

@@ -17,9 +17,9 @@ import (
 
 var TestPlatforms = []fn.Platform{{OS: runtime.GOOS, Architecture: runtime.GOARCH}}
 
-// TestBuilder ensures that, when given a Go Function, an OCI-compliant
+// TestBuilder_Build ensures that, when given a Go Function, an OCI-compliant
 // directory structure is created on .Build in the expected path.
-func TestBuilder(t *testing.T) {
+func TestBuilder_Build(t *testing.T) {
 	root, done := Mktemp(t)
 	defer done()
 
@@ -32,7 +32,7 @@ func TestBuilder(t *testing.T) {
 
 	builder := NewBuilder("", true)
 
-	if err := builder.Build(context.Background(), f, nil); err != nil {
+	if err := builder.Build(context.Background(), f, TestPlatforms); err != nil {
 		t.Fatal(err)
 	}
 

pkg/oci/containerize.go

@@ -67,21 +67,17 @@ func containerize(cfg *buildConfig) (err error) {
 		return
 	}
 
-	// TODO: if the base image is not provided, create a certificates layer
-	// which includes root certificates such that the resultant container
-	// can validate SSL (make HTTPS requests)
-	/*
-		certsDesc, certsLayer, err := newCerts(cfg) // shared
-		if err != nil {
-			return
-		}
-	*/
+	// Create the root certificates layer and its decriptor
+	certsDesc, certsLayer, err := newCertsLayer(cfg) // shared
+	if err != nil {
+		return
+	}
 
-	// Create an image for each platform consisting of the shared data layer
-	// and an os/platform specific layer.
+	// Create an image for each platform consisting of the shared data layer,
+	// the shared root certs layer, and an os/platform specific layer.
 	imageDescs := []v1.Descriptor{}
 	for _, p := range cfg.platforms {
-		imageDesc, err := newImage(cfg, dataDesc, dataLayer, p, cfg.verbose)
+		imageDesc, err := newImage(cfg, dataDesc, dataLayer, certsDesc, certsLayer, p, cfg.verbose)
 		if err != nil {
 			return err
 		}
@@ -188,6 +184,90 @@ func newDataTarball(source, target string, ignored []string, verbose bool) error
 	})
 }
 
+// newCertLayer creates the shared data layer in the container file hierarchy and
+// returns both its descriptor and layer metadata.
+func newCertsLayer(cfg *buildConfig) (desc v1.Descriptor, layer v1.Layer, err error) {
+
+	// Create the data tarball
+	// TODO: try WithCompressedCaching?
+	source := filepath.Join(cfg.buildDir(), "ca-certificates.crt")
+	target := path(cfg.buildDir(), "certslayer.tar.gz")
+
+	if err = newCertsTarball(source, target, defaultIgnored, cfg.verbose); err != nil {
+		return
+	}
+
+	// Layer
+	if layer, err = tarball.LayerFromFile(target); err != nil {
+		return
+	}
+
+	// Descriptor
+	if desc, err = newDescriptor(layer); err != nil {
+		return
+	}
+
+	// Blob
+	blob := path(cfg.blobsDir(), desc.Digest.Hex)
+	if cfg.verbose {
+		fmt.Printf("mv %v %v\n", rel(cfg.buildDir(), target), rel(cfg.buildDir(), blob))
+	}
+	err = os.Rename(target, blob)
+	return
+}
+
+func newCertsTarball(source, target string, ignored []string, verbose bool) error {
+	targetFile, err := os.Create(target)
+	if err != nil {
+		return err
+	}
+	defer targetFile.Close()
+
+	gw := gzip.NewWriter(targetFile)
+	defer gw.Close()
+
+	tw := tar.NewWriter(gw)
+	defer tw.Close()
+
+	paths := []string{
+		"/etc/ssl/certs/ca-certificates.crt",
+		"/etc/pki/tls/certs/ca-certificates.crt",
+	}
+
+	fi, err := os.Stat(source)
+	if err != nil {
+		return err
+	}
+
+	// For each ssl certs path we want to create
+	for _, path := range paths {
+		// Create a header for it
+		header, err := tar.FileInfoHeader(fi, "")
+		if err != nil {
+			return err
+		}
+		header.Name = path
+
+		if err := tw.WriteHeader(header); err != nil {
+			return err
+		}
+		if verbose {
+			fmt.Printf("→ %v \n", header.Name)
+		}
+		file, err := os.Open(source)
+		if err != nil {
+			return err
+		}
+		defer file.Close()
+		_, err = io.Copy(tw, file)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
 func newDescriptor(layer v1.Layer) (desc v1.Descriptor, err error) {
 	size, err := layer.Size()
 	if err != nil {
@@ -206,7 +286,7 @@ func newDescriptor(layer v1.Layer) (desc v1.Descriptor, err error) {
 
 // newImage creates an image for the given platform.
 // The image consists of the shared data layer which is provided
-func newImage(cfg *buildConfig, dataDesc v1.Descriptor, dataLayer v1.Layer, p v1.Platform, verbose bool) (imageDesc v1.Descriptor, err error) {
+func newImage(cfg *buildConfig, dataDesc v1.Descriptor, dataLayer v1.Layer, certsDesc v1.Descriptor, certsLayer v1.Layer, p v1.Platform, verbose bool) (imageDesc v1.Descriptor, err error) {
 	buildFn, err := getLanguageLayerBuilder(cfg)
 	if err != nil {
 		return
@@ -219,7 +299,7 @@ func newImage(cfg *buildConfig, dataDesc v1.Descriptor, dataLayer v1.Layer, p v1
 	}
 
 	// Write Config Layer as Blob -> Layer
-	configDesc, _, err := newConfig(cfg, p, dataLayer, execLayer)
+	configDesc, _, err := newConfig(cfg, p, dataLayer, certsLayer, execLayer)
 	if err != nil {
 		return
 	}
@@ -229,7 +309,7 @@ func newImage(cfg *buildConfig, dataDesc v1.Descriptor, dataLayer v1.Layer, p v1
 		SchemaVersion: 2,
 		MediaType:     types.OCIManifestSchema1,
 		Config:        configDesc,
-		Layers:        []v1.Descriptor{dataDesc, execDesc},
+		Layers:        []v1.Descriptor{dataDesc, certsDesc, execDesc},
 	}
 
 	// Write image manifest out as json to a tempfile

pkg/scaffolding/scaffold.go

@@ -48,6 +48,14 @@ func Write(out, src, runtime, invoke string, fs filesystem.Filesystem) (err erro
 		return ScaffoldingError{"filesystem copy failed", err}
 	}
 
+	// Copy the certs from the filesystem to the build directory
+	if _, err := fs.Stat("certs"); err != nil {
+		return ScaffoldingError{"certs directory not found in filesystem", err}
+	}
+	if err := filesystem.CopyFromFS("certs", out, fs); err != nil {
+		return ScaffoldingError{"certs copy failed", err}
+	}
+
 	// Replace the 'f' link of the scaffolding (which is now incorrect) to
 	// link to the function's root.
 	rel, err := filepath.Rel(out, src)

pkg/scaffolding/scaffold_test.go

@@ -51,7 +51,7 @@ func TestWrite_RuntimeErrors(t *testing.T) {
 }
 
 // TestWrite ensures that the Write method writes Scaffolding to the given
-// destination.  This is a failry shallow test.  See the Scaffolding and
+// destination.  This is a fairly shallow test.  See the Scaffolding and
 // Detector tests for more depth.
 func TestWrite(t *testing.T) {
 	// The filesystem containing scaffolding is expected to conform to the

pkg/scaffolding/testdata/testwrite/certs/README.md

@@ -0,0 +1,11 @@
+# Test Certs
+
+This directory mocks the existence of a 'certs' directory peer to the various
+languages in the templates directory.  This directory is required for the
+error-free execution of Scaffolding Write tests, since it is used to
+create the certificates container layer.
+
+The actual certs used are located in templates/certs, and are refreshed
+by running `make certs` or by running a `make clean`
+before a `make`.
+