From b01588d82f73b70a7a7a93d602094585f852a069 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Sch=C3=BCller?= Date: Wed, 14 Aug 2024 13:01:02 +0200 Subject: [PATCH 1/6] splunk_logger: move environment hook to splunk_logger pt2 for reusability also in image-builder --- cmd/osbuild-composer/main.go | 2 +- go.mod | 2 +- go.sum | 4 +-- internal/common/environment_hook_test.go | 30 ------------------- .../pkg/splunk_logger}/environment_hook.go | 2 +- vendor/modules.txt | 4 +-- 6 files changed, 7 insertions(+), 37 deletions(-) delete mode 100644 internal/common/environment_hook_test.go rename {internal/common => vendor/github.com/osbuild/osbuild-composer/pkg/splunk_logger}/environment_hook.go (96%) diff --git a/cmd/osbuild-composer/main.go b/cmd/osbuild-composer/main.go index f9e16172f5..eeb724c9eb 100644 --- a/cmd/osbuild-composer/main.go +++ b/cmd/osbuild-composer/main.go @@ -85,7 +85,7 @@ func main() { } if config.DeploymentChannel != "" { - logrus.AddHook(&common.EnvironmentHook{Channel: config.DeploymentChannel}) + logrus.AddHook(&slogger.EnvironmentHook{Channel: config.DeploymentChannel}) } if config.SplunkHost != "" { diff --git a/go.mod b/go.mod index f7f817052a..ff30fc556b 100644 --- a/go.mod +++ b/go.mod @@ -47,7 +47,7 @@ require ( github.com/openshift-online/ocm-sdk-go v0.1.432 github.com/oracle/oci-go-sdk/v54 v54.0.0 github.com/osbuild/images v0.79.0 - github.com/osbuild/osbuild-composer/pkg/splunk_logger v0.0.0-20231117174845-e969a9dc3cd1 + github.com/osbuild/osbuild-composer/pkg/splunk_logger v0.0.0-20240814102216-0239db53236d github.com/osbuild/pulp-client v0.1.0 github.com/prometheus/client_golang v1.19.1 github.com/segmentio/ksuid v1.0.4 diff --git a/go.sum b/go.sum index da7bd60f0b..52160a2dcf 100644 --- a/go.sum +++ b/go.sum @@ -510,8 +510,8 @@ github.com/oracle/oci-go-sdk/v54 v54.0.0 h1:CDLjeSejv2aDpElAJrhKpi6zvT/zhZCZuXch github.com/oracle/oci-go-sdk/v54 v54.0.0/go.mod h1:+t+yvcFGVp+3ZnztnyxqXfQDsMlq8U25faBLa+mqCMc= github.com/osbuild/images v0.79.0 h1:6kBRo0WzmyQTZ4ojX1oU9nIxBazZOO4I3FLsZaz8268= github.com/osbuild/images v0.79.0/go.mod h1:Rzud9PFt0L9qFFlisL9I6OsZ2h0M6viMBdPhPtpwfwg= -github.com/osbuild/osbuild-composer/pkg/splunk_logger v0.0.0-20231117174845-e969a9dc3cd1 h1:UFEJIcPa46W8gtWgOYzriRKYyy1t6SWL0BI7fPTuVvc= -github.com/osbuild/osbuild-composer/pkg/splunk_logger v0.0.0-20231117174845-e969a9dc3cd1/go.mod h1:z+WA+dX6qMwc7fqY5jCzESDIlg4WR2sBQezxsoXv9Ik= +github.com/osbuild/osbuild-composer/pkg/splunk_logger v0.0.0-20240814102216-0239db53236d h1:r9BFPDv0uuA9k1947Jybcxs36c/pTywWS1gjeizvtcQ= +github.com/osbuild/osbuild-composer/pkg/splunk_logger v0.0.0-20240814102216-0239db53236d/go.mod h1:zR1iu/hOuf+OQNJlk70tju9IqzzM4ycq0ectkFBm94U= github.com/osbuild/pulp-client v0.1.0 h1:L0C4ezBJGTamN3BKdv+rKLuq/WxXJbsFwz/Hj7aEmJ8= github.com/osbuild/pulp-client v0.1.0/go.mod h1:rd/MLdfwwO2cQI1s056h8z32zAi3Bo90XhlAAryIvWc= github.com/ostreedev/ostree-go v0.0.0-20210805093236-719684c64e4f h1:/UDgs8FGMqwnHagNDPGOlts35QkhAZ8by3DR7nMih7M= diff --git a/internal/common/environment_hook_test.go b/internal/common/environment_hook_test.go deleted file mode 100644 index 7de77d6071..0000000000 --- a/internal/common/environment_hook_test.go +++ /dev/null @@ -1,30 +0,0 @@ -package common - -import ( - "bytes" - "testing" - - "github.com/sirupsen/logrus" - "github.com/stretchr/testify/require" -) - -func makeLogrus(buf *bytes.Buffer) *logrus.Logger { - return &logrus.Logger{ - Out: buf, - Formatter: &logrus.TextFormatter{ - DisableTimestamp: true, - DisableColors: true, - }, - Hooks: make(logrus.LevelHooks), - Level: logrus.DebugLevel, - } - -} - -func TestInfoWithEnvironment(t *testing.T) { - buf := &bytes.Buffer{} - l := makeLogrus(buf) - l.AddHook(&EnvironmentHook{Channel: "test_framework"}) - l.Info("test message") - require.Equal(t, "level=info msg=\"test message\" channel=test_framework\n", buf.String()) -} diff --git a/internal/common/environment_hook.go b/vendor/github.com/osbuild/osbuild-composer/pkg/splunk_logger/environment_hook.go similarity index 96% rename from internal/common/environment_hook.go rename to vendor/github.com/osbuild/osbuild-composer/pkg/splunk_logger/environment_hook.go index 2d0052ebac..152b97197c 100644 --- a/internal/common/environment_hook.go +++ b/vendor/github.com/osbuild/osbuild-composer/pkg/splunk_logger/environment_hook.go @@ -1,4 +1,4 @@ -package common +package logger import ( "github.com/sirupsen/logrus" diff --git a/vendor/modules.txt b/vendor/modules.txt index 50d38e4ff0..800c015dd2 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -989,8 +989,8 @@ github.com/osbuild/images/pkg/rhsm github.com/osbuild/images/pkg/rhsm/facts github.com/osbuild/images/pkg/rpmmd github.com/osbuild/images/pkg/runner -# github.com/osbuild/osbuild-composer/pkg/splunk_logger v0.0.0-20231117174845-e969a9dc3cd1 -## explicit; go 1.19 +# github.com/osbuild/osbuild-composer/pkg/splunk_logger v0.0.0-20240814102216-0239db53236d +## explicit; go 1.21 github.com/osbuild/osbuild-composer/pkg/splunk_logger # github.com/osbuild/pulp-client v0.1.0 ## explicit; go 1.19 From c51197c1610ea8f3ec6f6cf2bd9a8118b421fe66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Sch=C3=BCller?= Date: Wed, 14 Aug 2024 16:50:39 +0200 Subject: [PATCH 2/6] workflows: include splunk_logger sub module in tests --- .github/workflows/tests.yml | 2 +- .gitignore | 3 +++ Makefile | 7 ++++++- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index e49714f7bc..a16fea8f58 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -33,7 +33,7 @@ jobs: run: git config --global --add safe.directory "$(pwd)" - name: Run unit tests - run: go test -race -covermode=atomic -coverprofile=coverage.txt -coverpkg=$(go list ./... | grep -v rpmmd/test$ | tr "\n" ",") ./... + run: make unit-tests - uses: codecov/codecov-action@v4 if: env.CODECOV_TOKEN diff --git a/.gitignore b/.gitignore index b83ec19551..2056a883b6 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,6 @@ __pycache__ container_composer_golangci_built.info processed-templates +coverage.txt + +coverage_splunk_logger.txt diff --git a/Makefile b/Makefile index 92be7c0013..04ddd2274e 100644 --- a/Makefile +++ b/Makefile @@ -234,8 +234,13 @@ worker-key-pair: ca rm /etc/osbuild-composer/worker-csr.pem .PHONY: unit-tests +.ONESHELL: unit-tests: - go test -race ./... + go test -race -covermode=atomic -coverprofile=coverage.txt -coverpkg=$$(go list ./... | grep -v rpmmd/test$ | tr "\n" ",") ./... + # go modules with go.mod in subdirs are not tested automatically + cd pkg/splunk_logger + go test -race -covermode=atomic -coverprofile=../../coverage_splunk_logger.txt -coverpkg=$$(go list ./... | grep -v rpmmd/test$ | tr "\n" ",") ./... + # # Building packages From d6c78e77a1e29d2b6e0969e5daf65749acedb042 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Sch=C3=BCller?= Date: Wed, 14 Aug 2024 16:51:06 +0200 Subject: [PATCH 3/6] Makefile: implement a target to create coverage-report --- .gitignore | 2 ++ Makefile | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/.gitignore b/.gitignore index 2056a883b6..72bc9c3c44 100644 --- a/.gitignore +++ b/.gitignore @@ -18,5 +18,7 @@ container_composer_golangci_built.info processed-templates coverage.txt +coverage.html coverage_splunk_logger.txt +coverage_splunk_logger.html diff --git a/Makefile b/Makefile index 04ddd2274e..f5e5977d69 100644 --- a/Makefile +++ b/Makefile @@ -90,6 +90,7 @@ help: @echo " (do this before pushing!)" @echo " lint: Runs linters as close as github workflow as possible" @echo " process-templates: Execute the OpenShift CLI to check the templates" + @echo " coverage-report: Run unit tests and generate HTML coverage reports" $(BUILDDIR)/: mkdir -p "$@" @@ -241,6 +242,10 @@ unit-tests: cd pkg/splunk_logger go test -race -covermode=atomic -coverprofile=../../coverage_splunk_logger.txt -coverpkg=$$(go list ./... | grep -v rpmmd/test$ | tr "\n" ",") ./... +.PHONY: coverage-report +coverage-report: unit-tests + go tool cover -o coverage.html -html coverage.txt + go tool cover -o coverage_splunk_logger.html -html coverage_splunk_logger.txt # # Building packages From c06ccafb9933c60c9caf1792c71cae9ead2ad7f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Sch=C3=BCller?= Date: Wed, 21 Aug 2024 12:22:26 +0200 Subject: [PATCH 4/6] Makefile: remove old remnant package filter --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index f5e5977d69..5cce210e64 100644 --- a/Makefile +++ b/Makefile @@ -237,10 +237,10 @@ worker-key-pair: ca .PHONY: unit-tests .ONESHELL: unit-tests: - go test -race -covermode=atomic -coverprofile=coverage.txt -coverpkg=$$(go list ./... | grep -v rpmmd/test$ | tr "\n" ",") ./... + go test -race -covermode=atomic -coverprofile=coverage.txt -coverpkg=$$(go list ./... | tr "\n" ",") ./... # go modules with go.mod in subdirs are not tested automatically cd pkg/splunk_logger - go test -race -covermode=atomic -coverprofile=../../coverage_splunk_logger.txt -coverpkg=$$(go list ./... | grep -v rpmmd/test$ | tr "\n" ",") ./... + go test -race -covermode=atomic -coverprofile=../../coverage_splunk_logger.txt -coverpkg=$$(go list ./... | tr "\n" ",") ./... .PHONY: coverage-report coverage-report: unit-tests From 06da729557e7654c3e5fd832358dbaebde63780d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Sch=C3=BCller?= Date: Thu, 22 Aug 2024 10:08:38 +0200 Subject: [PATCH 5/6] tmt/plans/edge-test.fmf: increase disksize for qcow2 test --- tmt/plans/edge-test.fmf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tmt/plans/edge-test.fmf b/tmt/plans/edge-test.fmf index 0d5e2ef9b3..8c8235fb27 100644 --- a/tmt/plans/edge-test.fmf +++ b/tmt/plans/edge-test.fmf @@ -11,6 +11,8 @@ provision: cpu: processors: ">= 2" memory: ">= 6 GB" + disk: + - size: ">= 15 GB" /edge-x86-commit: summary: Test edge commit From b1985e16b8f8a64ed956812b3302fdd6bbb2f4e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Sch=C3=BCller?= Date: Wed, 21 Aug 2024 10:24:21 +0200 Subject: [PATCH 6/6] osbuild-worker: add CHANNEL to worker logs aka "the deployment channel" like "staging" or "production" --- cmd/osbuild-worker/config.go | 3 +++ cmd/osbuild-worker/config_test.go | 13 +++++++++++++ cmd/osbuild-worker/main.go | 5 +++++ .../client_credentials.sh | 1 + 4 files changed, 22 insertions(+) diff --git a/cmd/osbuild-worker/config.go b/cmd/osbuild-worker/config.go index f28cf0effc..ac3dae5926 100644 --- a/cmd/osbuild-worker/config.go +++ b/cmd/osbuild-worker/config.go @@ -103,6 +103,8 @@ type workerConfig struct { // default value: &{ Type: host } OSBuildExecutor *executorConfig `toml:"osbuild_executor"` RepositoryMTLSConfig *repositoryMTLSConfig `toml:"repository_mtls"` + // something like "production" or "staging" to be added to logging + DeploymentChannel string `toml:"deployment_channel"` } func parseConfig(file string) (*workerConfig, error) { @@ -112,6 +114,7 @@ func parseConfig(file string) (*workerConfig, error) { OSBuildExecutor: &executorConfig{ Type: "host", }, + DeploymentChannel: "local", } _, err := toml.DecodeFile(file, &config) diff --git a/cmd/osbuild-worker/config_test.go b/cmd/osbuild-worker/config_test.go index 46aa7f8999..540570d2fd 100644 --- a/cmd/osbuild-worker/config_test.go +++ b/cmd/osbuild-worker/config_test.go @@ -136,6 +136,7 @@ cloudwatch_group = "osbuild-worker" Credentials: "/etc/osbuild-worker/pulp-creds", ServerURL: "https://example.com/pulp", }, + DeploymentChannel: "local", }, }, { @@ -146,6 +147,18 @@ cloudwatch_group = "osbuild-worker" OSBuildExecutor: &executorConfig{ Type: "host", }, + DeploymentChannel: "local", + }, + }, + { + name: "set_channel", + config: `deployment_channel = "staging"`, + want: &workerConfig{ + BasePath: "/api/worker/v1", + OSBuildExecutor: &executorConfig{ + Type: "host", + }, + DeploymentChannel: "staging", }, }, } diff --git a/cmd/osbuild-worker/main.go b/cmd/osbuild-worker/main.go index d7fe8b0735..d81c754600 100644 --- a/cmd/osbuild-worker/main.go +++ b/cmd/osbuild-worker/main.go @@ -7,6 +7,7 @@ import ( "errors" "flag" "fmt" + slogger "github.com/osbuild/osbuild-composer/pkg/splunk_logger" "net/url" "os" "path" @@ -193,6 +194,10 @@ func main() { logrus.Fatalf("Could not print config: %v", err) } + if config.DeploymentChannel != "" { + logrus.AddHook(&slogger.EnvironmentHook{Channel: config.DeploymentChannel}) + } + cacheDirectory, ok := os.LookupEnv("CACHE_DIRECTORY") if !ok { logrus.Fatal("CACHE_DIRECTORY is not set. Is the service file missing CacheDirectory=?") diff --git a/templates/packer/ansible/roles/common/files/worker-initialization-scripts/client_credentials.sh b/templates/packer/ansible/roles/common/files/worker-initialization-scripts/client_credentials.sh index a8497c1214..7c983c169d 100755 --- a/templates/packer/ansible/roles/common/files/worker-initialization-scripts/client_credentials.sh +++ b/templates/packer/ansible/roles/common/files/worker-initialization-scripts/client_credentials.sh @@ -20,6 +20,7 @@ jq -r ".client_secret" /tmp/client-credentials.json > /etc/osbuild-worker/client rm -f /tmp/client-credentials.json sudo tee -a /etc/osbuild-worker/osbuild-worker.toml > /dev/null << EOF +deployment_channel = "${CHANNEL:-local}" [authentication] oauth_url = "${TOKEN_URL:-https://sso.redhat.com/auth/realms/redhat-external/protocol/openid-connect/token}" client_id = "${CLIENT_ID}"