From 8bce72057cbef247390cc540d0557ea9856e0488 Mon Sep 17 00:00:00 2001 From: lostbean Date: Wed, 31 Jan 2024 15:48:20 -0300 Subject: [PATCH 01/15] add nix build backend service --- .../docker_kurtosis_backend.go | 10 +- .../docker/docker_manager/docker_manager.go | 18 +- .../kubernetes_kurtosis_backend.go | 9 +- .../metrics_reporting_kurtosis_backend.go | 10 +- .../lib/backend_interface/kurtosis_backend.go | 8 +- .../mock_kurtosis_backend.go | 57 +++++- .../objects/nix_build_spec/nix_build_spec.go | 59 ++++++ .../objects/service/service_config.go | 4 + .../service_config/nix_build_spec.go | 189 ++++++++++++++++++ flake.nix | 1 + 10 files changed, 351 insertions(+), 14 deletions(-) create mode 100644 container-engine-lib/lib/backend_interface/objects/nix_build_spec/nix_build_spec.go create mode 100644 core/server/api_container/server/startosis_engine/kurtosis_types/service_config/nix_build_spec.go diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend.go index 7632be76aa..d663de40a9 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend.go @@ -2,11 +2,13 @@ package docker_kurtosis_backend import ( "context" - "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/image_build_spec" - "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/image_registry_spec" "io" "sync" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/image_build_spec" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/image_registry_spec" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/nix_build_spec" + "github.com/sirupsen/logrus" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/engine_functions" @@ -544,6 +546,10 @@ func (backend *DockerKurtosisBackend) BuildImage(ctx context.Context, imageName return backend.dockerManager.BuildImage(ctx, imageName, imageBuildSpec) } +func (backend *DockerKurtosisBackend) NixBuild(ctx context.Context, nixBuildSpec *nix_build_spec.NixBuildSpec) (string, error) { + return backend.dockerManager.NixBuild(ctx, nixBuildSpec) +} + // ==================================================================================================== // // Private helper functions shared by multiple subfunctions files diff --git a/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go b/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go index ad5e386806..64ccaee7d3 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go @@ -11,12 +11,6 @@ import ( "context" "encoding/json" "fmt" - "github.com/docker/docker/api/types/registry" - "github.com/docker/go-units" - "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/image_build_spec" - "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/image_registry_spec" - "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/uuid_generator" - "github.com/kurtosis-tech/kurtosis/utils" "io" "math" "net" @@ -25,6 +19,14 @@ import ( "sync" "time" + "github.com/docker/docker/api/types/registry" + "github.com/docker/go-units" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/image_build_spec" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/image_registry_spec" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/nix_build_spec" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/uuid_generator" + "github.com/kurtosis-tech/kurtosis/utils" + "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/filters" @@ -1313,6 +1315,10 @@ func (manager *DockerManager) FetchImage(ctx context.Context, image string, regi return pulledFromRemote, imageArchitecture, nil } +func (manager *DockerManager) NixBuild(ctx context.Context, nixBuildSpec *nix_build_spec.NixBuildSpec) (string, error) { + return "", nil +} + func (manager *DockerManager) BuildImage(ctx context.Context, imageName string, imageBuildSpec *image_build_spec.ImageBuildSpec) (string, error) { buildContextDirPath := imageBuildSpec.GetBuildContextDir() buildContextTarReader, err := getBuildContextReader(buildContextDirPath) diff --git a/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/kubernetes_kurtosis_backend.go b/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/kubernetes_kurtosis_backend.go index d87cd0b358..b096a5339b 100644 --- a/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/kubernetes_kurtosis_backend.go +++ b/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/kubernetes_kurtosis_backend.go @@ -2,9 +2,11 @@ package kubernetes_kurtosis_backend import ( "context" + "io" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/image_build_spec" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/image_registry_spec" - "io" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/nix_build_spec" apiv1 "k8s.io/api/core/v1" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/engine_functions" @@ -487,6 +489,11 @@ func (backend *KubernetesKurtosisBackend) BuildImage(ctx context.Context, imageN return "", stacktrace.NewError("Building images isn't yet implemented in Kubernetes.") } +func (backend *KubernetesKurtosisBackend) NixBuild(ctx context.Context, nixBuildSpec *nix_build_spec.NixBuildSpec) (string, error) { + // TODO IMPLEMENT + return "", stacktrace.NewError("Loading images isn't yet implemented in Kubernetes.") +} + // ==================================================================================================== // // Private Helper Functions diff --git a/container-engine-lib/lib/backend_impls/metrics_reporting/metrics_reporting_kurtosis_backend.go b/container-engine-lib/lib/backend_impls/metrics_reporting/metrics_reporting_kurtosis_backend.go index 5a695e7537..c96e23449e 100644 --- a/container-engine-lib/lib/backend_impls/metrics_reporting/metrics_reporting_kurtosis_backend.go +++ b/container-engine-lib/lib/backend_impls/metrics_reporting/metrics_reporting_kurtosis_backend.go @@ -2,11 +2,13 @@ package metrics_reporting import ( "context" - "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/image_build_spec" - "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/image_registry_spec" "io" "time" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/image_build_spec" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/image_registry_spec" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/nix_build_spec" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/api_container" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/compute_resources" @@ -470,3 +472,7 @@ func (backend *MetricsReportingKurtosisBackend) GetAvailableCPUAndMemory(ctx con func (backend *MetricsReportingKurtosisBackend) BuildImage(ctx context.Context, imageName string, imageBuildSpec *image_build_spec.ImageBuildSpec) (string, error) { return backend.underlying.BuildImage(ctx, imageName, imageBuildSpec) } + +func (backend *MetricsReportingKurtosisBackend) NixBuild(ctx context.Context, nixBuildSpec *nix_build_spec.NixBuildSpec) (string, error) { + return backend.underlying.NixBuild(ctx, nixBuildSpec) +} diff --git a/container-engine-lib/lib/backend_interface/kurtosis_backend.go b/container-engine-lib/lib/backend_interface/kurtosis_backend.go index be9a65024f..71b3c16394 100644 --- a/container-engine-lib/lib/backend_interface/kurtosis_backend.go +++ b/container-engine-lib/lib/backend_interface/kurtosis_backend.go @@ -2,11 +2,13 @@ package backend_interface import ( "context" - "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/image_build_spec" - "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/image_registry_spec" "io" "time" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/image_build_spec" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/image_registry_spec" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/nix_build_spec" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/api_container" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/compute_resources" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/enclave" @@ -356,4 +358,6 @@ type KurtosisBackend interface { // BuildImage builds a container image based on the [imageBuildSpec] with [imageName] // Returns image architecture and if error occurred BuildImage(ctx context.Context, imageName string, imageBuildSpec *image_build_spec.ImageBuildSpec) (string, error) + + NixBuild(ctx context.Context, nixBuildSpec *nix_build_spec.NixBuildSpec) (string, error) } diff --git a/container-engine-lib/lib/backend_interface/mock_kurtosis_backend.go b/container-engine-lib/lib/backend_interface/mock_kurtosis_backend.go index 62e2d0eb5f..aea4d57d20 100644 --- a/container-engine-lib/lib/backend_interface/mock_kurtosis_backend.go +++ b/container-engine-lib/lib/backend_interface/mock_kurtosis_backend.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.23.1. DO NOT EDIT. +// Code generated by mockery v2.36.0. DO NOT EDIT. package backend_interface @@ -28,6 +28,8 @@ import ( mock "github.com/stretchr/testify/mock" + nix_build_spec "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/nix_build_spec" + reverse_proxy "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/reverse_proxy" service "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service" @@ -1631,6 +1633,59 @@ func (_c *MockKurtosisBackend_GetUserServices_Call) RunAndReturn(run func(contex return _c } +// NixBuild provides a mock function with given fields: ctx, nixBuildSpec +func (_m *MockKurtosisBackend) NixBuild(ctx context.Context, nixBuildSpec *nix_build_spec.NixBuildSpec) (string, error) { + ret := _m.Called(ctx, nixBuildSpec) + + var r0 string + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *nix_build_spec.NixBuildSpec) (string, error)); ok { + return rf(ctx, nixBuildSpec) + } + if rf, ok := ret.Get(0).(func(context.Context, *nix_build_spec.NixBuildSpec) string); ok { + r0 = rf(ctx, nixBuildSpec) + } else { + r0 = ret.Get(0).(string) + } + + if rf, ok := ret.Get(1).(func(context.Context, *nix_build_spec.NixBuildSpec) error); ok { + r1 = rf(ctx, nixBuildSpec) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockKurtosisBackend_NixBuild_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'NixBuild' +type MockKurtosisBackend_NixBuild_Call struct { + *mock.Call +} + +// NixBuild is a helper method to define mock.On call +// - ctx context.Context +// - nixBuildSpec *nix_build_spec.NixBuildSpec +func (_e *MockKurtosisBackend_Expecter) NixBuild(ctx interface{}, nixBuildSpec interface{}) *MockKurtosisBackend_NixBuild_Call { + return &MockKurtosisBackend_NixBuild_Call{Call: _e.mock.On("NixBuild", ctx, nixBuildSpec)} +} + +func (_c *MockKurtosisBackend_NixBuild_Call) Run(run func(ctx context.Context, nixBuildSpec *nix_build_spec.NixBuildSpec)) *MockKurtosisBackend_NixBuild_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(*nix_build_spec.NixBuildSpec)) + }) + return _c +} + +func (_c *MockKurtosisBackend_NixBuild_Call) Return(_a0 string, _a1 error) *MockKurtosisBackend_NixBuild_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *MockKurtosisBackend_NixBuild_Call) RunAndReturn(run func(context.Context, *nix_build_spec.NixBuildSpec) (string, error)) *MockKurtosisBackend_NixBuild_Call { + _c.Call.Return(run) + return _c +} + // PruneUnusedImages provides a mock function with given fields: ctx func (_m *MockKurtosisBackend) PruneUnusedImages(ctx context.Context) ([]string, error) { ret := _m.Called(ctx) diff --git a/container-engine-lib/lib/backend_interface/objects/nix_build_spec/nix_build_spec.go b/container-engine-lib/lib/backend_interface/objects/nix_build_spec/nix_build_spec.go new file mode 100644 index 0000000000..3706b5395c --- /dev/null +++ b/container-engine-lib/lib/backend_interface/objects/nix_build_spec/nix_build_spec.go @@ -0,0 +1,59 @@ +package nix_build_spec + +import ( + "encoding/json" + + "github.com/kurtosis-tech/stacktrace" +) + +type NixBuildSpec struct { + // we do this way in order to have exported fields which can be marshalled + // and an unexported type for encapsulation + privateNixBuildSpec *privateNixBuildSpec +} + +// NixBuildSpec contains the information need for building a container from nix. +type privateNixBuildSpec struct { + ContainerNixFilePath string + ContextDirPath string + Flake string +} + +func NewNixBuildSpec(contextDirPath string, containerNixFilePath string, flake string) *NixBuildSpec { + internalNixBuildSpec := &privateNixBuildSpec{ + ContainerNixFilePath: containerNixFilePath, + ContextDirPath: contextDirPath, + Flake: flake, + } + return &NixBuildSpec{internalNixBuildSpec} +} + +func (nixBuildSpec *NixBuildSpec) GetContainerNixFilePath() string { + return nixBuildSpec.privateNixBuildSpec.ContainerNixFilePath +} + +func (nixBuildSpec *NixBuildSpec) GetBuildContextDir() string { + return nixBuildSpec.privateNixBuildSpec.ContextDirPath +} + +func (nixBuildSpec *NixBuildSpec) GetFlake() string { + return nixBuildSpec.privateNixBuildSpec.Flake +} + +func (nixBuildSpec *NixBuildSpec) MarshalJSON() ([]byte, error) { + return json.Marshal(nixBuildSpec.privateNixBuildSpec) +} + +func (nixBuildSpec *NixBuildSpec) UnmarshalJSON(data []byte) error { + + // Suppressing exhaustruct requirement because we want an object with zero values + // nolint: exhaustruct + unmarshalledPrivateStructPtr := &privateNixBuildSpec{} + + if err := json.Unmarshal(data, unmarshalledPrivateStructPtr); err != nil { + return stacktrace.Propagate(err, "An error occurred unmarshalling the private struct") + } + + nixBuildSpec.privateNixBuildSpec = unmarshalledPrivateStructPtr + return nil +} diff --git a/container-engine-lib/lib/backend_interface/objects/service/service_config.go b/container-engine-lib/lib/backend_interface/objects/service/service_config.go index 44b5daddba..6afb146558 100644 --- a/container-engine-lib/lib/backend_interface/objects/service/service_config.go +++ b/container-engine-lib/lib/backend_interface/objects/service/service_config.go @@ -2,8 +2,10 @@ package service import ( "encoding/json" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/image_build_spec" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/image_registry_spec" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/nix_build_spec" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/port_spec" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service_directory" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service_user" @@ -30,6 +32,8 @@ type privateServiceConfig struct { // Mutually exclusive from ImageBuildSpec, ContainerImageName ImagerRegistrySpec *image_registry_spec.ImageRegistrySpec + NixBuildSpec *nix_build_spec.NixBuildSpec + PrivatePorts map[string]*port_spec.PortSpec PublicPorts map[string]*port_spec.PortSpec //TODO this is a huge hack to temporarily enable static ports for NEAR until we have a more productized solution diff --git a/core/server/api_container/server/startosis_engine/kurtosis_types/service_config/nix_build_spec.go b/core/server/api_container/server/startosis_engine/kurtosis_types/service_config/nix_build_spec.go new file mode 100644 index 0000000000..4e6bfd0db3 --- /dev/null +++ b/core/server/api_container/server/startosis_engine/kurtosis_types/service_config/nix_build_spec.go @@ -0,0 +1,189 @@ +package service_config + +import ( + "path" + "path/filepath" + + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/nix_build_spec" + "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework" + "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/builtin_argument" + "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/kurtosis_type_constructor" + "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_constants" + "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_errors" + "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_packages" + "go.starlark.net/starlark" +) + +const ( + NixBuildSpecTypeName = "NixBuildSpec" + + NixNameAttr = "nix_name" + NixContextAttr = "build_context_dir" + FlakeAttr = "target_stage" + + // Currently only supports container nixs named Dockerfile + defaultContainerNixFileName = "Dockerfile" +) + +func NewNixBuildSpecType() *kurtosis_type_constructor.KurtosisTypeConstructor { + return &kurtosis_type_constructor.KurtosisTypeConstructor{ + KurtosisBaseBuiltin: &kurtosis_starlark_framework.KurtosisBaseBuiltin{ + Name: NixBuildSpecTypeName, + Arguments: []*builtin_argument.BuiltinArgument{ + { + Name: NixNameAttr, + IsOptional: false, + ZeroValueProvider: builtin_argument.ZeroValueProvider[starlark.String], + Validator: func(value starlark.Value) *startosis_errors.InterpretationError { + return builtin_argument.NonEmptyString(value, NixNameAttr) + }, + }, + { + Name: NixContextAttr, + IsOptional: false, + ZeroValueProvider: builtin_argument.ZeroValueProvider[starlark.String], + Validator: func(value starlark.Value) *startosis_errors.InterpretationError { + return builtin_argument.NonEmptyString(value, NixContextAttr) + }, + }, + { + Name: FlakeAttr, + IsOptional: true, + ZeroValueProvider: builtin_argument.ZeroValueProvider[starlark.String], + Validator: func(value starlark.Value) *startosis_errors.InterpretationError { + return builtin_argument.NonEmptyString(value, FlakeAttr) + }, + }, + }, + }, + Instantiate: instantiateNixBuildSpec, + } +} + +func instantiateNixBuildSpec(arguments *builtin_argument.ArgumentValuesSet) (builtin_argument.KurtosisValueType, *startosis_errors.InterpretationError) { + kurtosisValueType, err := kurtosis_type_constructor.CreateKurtosisStarlarkTypeDefault(NixBuildSpecTypeName, arguments) + if err != nil { + return nil, err + } + return &NixBuildSpec{ + KurtosisValueTypeDefault: kurtosisValueType, + }, nil +} + +// NixBuildSpec is a starlark.Value that holds all the information for the startosis_engine to initiate an nix build +type NixBuildSpec struct { + *kurtosis_type_constructor.KurtosisValueTypeDefault +} + +func (nixBuildSpec *NixBuildSpec) Copy() (builtin_argument.KurtosisValueType, error) { + copiedValueType, err := nixBuildSpec.KurtosisValueTypeDefault.Copy() + if err != nil { + return nil, err + } + return &NixBuildSpec{ + KurtosisValueTypeDefault: copiedValueType, + }, nil +} + +// Name to give nix built from NixBuildSpec +func (nixBuildSpec *NixBuildSpec) GetNixName() (string, *startosis_errors.InterpretationError) { + nixName, found, interpretationErr := kurtosis_type_constructor.ExtractAttrValue[starlark.String](nixBuildSpec.KurtosisValueTypeDefault, NixNameAttr) + if interpretationErr != nil { + return "", interpretationErr + } + if !found { + return "", startosis_errors.NewInterpretationError("Required attribute '%s' could not be found on type '%s'", + NixNameAttr, NixBuildSpecTypeName) + } + nixNameStr := nixName.GoString() + return nixNameStr, nil +} + +// Relative locator of build context directory +func (nixBuildSpec *NixBuildSpec) GetBuildContextLocator() (string, *startosis_errors.InterpretationError) { + buildContextLocator, found, interpretationErr := kurtosis_type_constructor.ExtractAttrValue[starlark.String](nixBuildSpec.KurtosisValueTypeDefault, NixContextAttr) + if interpretationErr != nil { + return "", interpretationErr + } + if !found { + return "", startosis_errors.NewInterpretationError("Required attribute '%s' could not be found on type '%s'", + NixContextAttr, NixBuildSpecTypeName) + } + buildContextLocatorStr := buildContextLocator.GoString() + return buildContextLocatorStr, nil +} + +// GetTargetStage is used for specifying which stage of a multi-stage container nix build to execute +// Default value is the empty string for single stage nix builds (common case) +// Info on target stage and multi-stag builds for Docker nixs: https://docs.docker.com/build/building/multi-stage/ +func (nixBuildSpec *NixBuildSpec) GetTargetStage() (string, *startosis_errors.InterpretationError) { + targetStage, found, interpretationErr := kurtosis_type_constructor.ExtractAttrValue[starlark.String](nixBuildSpec.KurtosisValueTypeDefault, FlakeAttr) + if interpretationErr != nil { + return "", interpretationErr + } + if !found { + return "", nil + } + targetStageStr := targetStage.GoString() + return targetStageStr, nil +} + +func (nixBuildSpec *NixBuildSpec) ToKurtosisType( + locatorOfModuleInWhichThisBuiltInIsBeingCalled string, + packageId string, + packageContentProvider startosis_packages.PackageContentProvider, + packageReplaceOptions map[string]string) (*nix_build_spec.NixBuildSpec, *startosis_errors.InterpretationError) { + // get locator of context directory (relative or absolute) + buildContextLocator, interpretationErr := nixBuildSpec.GetBuildContextLocator() + if interpretationErr != nil { + return nil, interpretationErr + } + + buildContextDirPathOnDisk, containerNixFilePathOnDisk, interpretationErr := getOnDiskNixBuildSpecPaths( + buildContextLocator, + packageId, + locatorOfModuleInWhichThisBuiltInIsBeingCalled, + packageContentProvider, + packageReplaceOptions) + if interpretationErr != nil { + return nil, interpretationErr + } + + targetStageStr, interpretationErr := nixBuildSpec.GetTargetStage() + if interpretationErr != nil { + return nil, interpretationErr + } + + return nix_build_spec.NewNixBuildSpec(buildContextDirPathOnDisk, containerNixFilePathOnDisk, targetStageStr), nil +} + +// Returns the filepath of the build context directory and container nix on APIC based on package info +func getOnDiskNixBuildSpecPaths( + buildContextLocator string, + packageId string, + locatorOfModuleInWhichThisBuiltInIsBeingCalled string, + packageContentProvider startosis_packages.PackageContentProvider, + packageReplaceOptions map[string]string) (string, string, *startosis_errors.InterpretationError) { + if packageId == startosis_constants.PackageIdPlaceholderForStandaloneScript { + return "", "", startosis_errors.NewInterpretationError("Cannot use NixBuildSpec in a standalone script; create a package and rerun to use NixBuildSpec.") + } + + // get absolute locator of context directory + contextDirAbsoluteLocator, interpretationErr := packageContentProvider.GetAbsoluteLocator(packageId, locatorOfModuleInWhichThisBuiltInIsBeingCalled, buildContextLocator, packageReplaceOptions) + if interpretationErr != nil { + return "", "", interpretationErr + } + + // get on disk directory path of Dockerfile + containerNixAbsoluteLocator := path.Join(contextDirAbsoluteLocator, defaultContainerNixFileName) + + containerNixPathOnDisk, interpretationErr := packageContentProvider.GetOnDiskAbsolutePackageFilePath(containerNixAbsoluteLocator) + if interpretationErr != nil { + return "", "", interpretationErr + } + + // Assume, that container nix sits at the same level as context directory to get context dir path on disk + contextDirPathOnDisk := filepath.Dir(containerNixPathOnDisk) + + return contextDirPathOnDisk, containerNixPathOnDisk, nil +} diff --git a/flake.nix b/flake.nix index 44d25796ac..ef4d08fceb 100644 --- a/flake.nix +++ b/flake.nix @@ -34,6 +34,7 @@ golangci-lint delve enumer + go-mockery nodejs_20 node2nix yarn From 2e50c89f571afac691c0e7d98a95ceb7b7adebf0 Mon Sep 17 00:00:00 2001 From: lostbean Date: Wed, 31 Jan 2024 17:34:25 -0300 Subject: [PATCH 02/15] add nix build service to starlark --- .../objects/service/service_config.go | 6 +++ .../objects/service/service_config_test.go | 11 ++++- .../service_registration/repository_test.go | 12 +++--- .../default_service_network_test.go | 1 + .../add_service/add_service_shared.go | 4 +- .../add_service/add_service_shared_test.go | 9 ++++- .../tasks/tasks_shared.go | 8 ++-- .../test_engine/add_service_framework_test.go | 4 +- .../add_services_framework_test.go | 5 ++- .../service_config_image_build_spec_test.go | 4 +- ...service_config_image_registry_spec_test.go | 4 +- .../service_config_minimal_framework_test.go | 4 +- .../service_config_toleration_test.go | 4 +- .../service_config/service_config.go | 33 ++++++++++----- .../startosis_validator/images_validator.go | 40 ++++++++++++++++++- .../validator_environment.go | 9 ++++- 16 files changed, 126 insertions(+), 32 deletions(-) diff --git a/container-engine-lib/lib/backend_interface/objects/service/service_config.go b/container-engine-lib/lib/backend_interface/objects/service/service_config.go index 6afb146558..000aec8676 100644 --- a/container-engine-lib/lib/backend_interface/objects/service/service_config.go +++ b/container-engine-lib/lib/backend_interface/objects/service/service_config.go @@ -73,6 +73,7 @@ func CreateServiceConfig( containerImageName string, imageBuildSpec *image_build_spec.ImageBuildSpec, imageRegistrySpec *image_registry_spec.ImageRegistrySpec, + nixBuildSpec *nix_build_spec.NixBuildSpec, privatePorts map[string]*port_spec.PortSpec, publicPorts map[string]*port_spec.PortSpec, entrypointArgs []string, @@ -99,6 +100,7 @@ func CreateServiceConfig( ContainerImageName: containerImageName, ImageBuildSpec: imageBuildSpec, ImagerRegistrySpec: imageRegistrySpec, + NixBuildSpec: nixBuildSpec, PrivatePorts: privatePorts, PublicPorts: publicPorts, EntrypointArgs: entrypointArgs, @@ -132,6 +134,10 @@ func (serviceConfig *ServiceConfig) GetImageRegistrySpec() *image_registry_spec. return serviceConfig.privateServiceConfig.ImagerRegistrySpec } +func (serviceConfig *ServiceConfig) GetNixBuildSpec() *nix_build_spec.NixBuildSpec { + return serviceConfig.privateServiceConfig.NixBuildSpec +} + func (serviceConfig *ServiceConfig) GetPrivatePorts() map[string]*port_spec.PortSpec { return serviceConfig.privateServiceConfig.PrivatePorts } diff --git a/container-engine-lib/lib/backend_interface/objects/service/service_config_test.go b/container-engine-lib/lib/backend_interface/objects/service/service_config_test.go index e40d56b436..db2632fa12 100644 --- a/container-engine-lib/lib/backend_interface/objects/service/service_config_test.go +++ b/container-engine-lib/lib/backend_interface/objects/service/service_config_test.go @@ -2,15 +2,17 @@ package service import ( "encoding/json" + "testing" + "time" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/image_build_spec" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/image_registry_spec" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/nix_build_spec" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/port_spec" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service_directory" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service_user" "github.com/stretchr/testify/require" v1 "k8s.io/api/core/v1" - "testing" - "time" ) func TestServiceConfigMarshallers(t *testing.T) { @@ -66,6 +68,7 @@ func getServiceConfigForTest(t *testing.T, imageName string) *ServiceConfig { imageName, testImageBuildSpec(), testImageRegistrySpec(), + testNixBuildSpec(), testPrivatePorts(t), testPublicPorts(t), []string{"bin", "bash", "ls"}, @@ -187,6 +190,10 @@ func testImageRegistrySpec() *image_registry_spec.ImageRegistrySpec { return image_registry_spec.NewImageRegistrySpec("test-image", "test-userename", "test-password", "test-registry.io") } +func testNixBuildSpec() *nix_build_spec.NixBuildSpec { + return nix_build_spec.NewNixBuildSpec("test-image", "path", "") +} + func testServiceUser() *service_user.ServiceUser { su := service_user.NewServiceUser(100) su.SetGID(100) diff --git a/container-engine-lib/lib/database_accessors/enclave_db/service_registration/repository_test.go b/container-engine-lib/lib/database_accessors/enclave_db/service_registration/repository_test.go index 92bf06365a..82311e59ce 100644 --- a/container-engine-lib/lib/database_accessors/enclave_db/service_registration/repository_test.go +++ b/container-engine-lib/lib/database_accessors/enclave_db/service_registration/repository_test.go @@ -2,6 +2,12 @@ package service_registration import ( "fmt" + "math/rand" + "net" + "os" + "testing" + "time" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/enclave" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/port_spec" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service" @@ -9,11 +15,6 @@ import ( "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/database_accessors/enclave_db" "github.com/stretchr/testify/require" bolt "go.etcd.io/bbolt" - "math/rand" - "net" - "os" - "testing" - "time" ) const ( @@ -306,6 +307,7 @@ func getServiceConfigForTest(t *testing.T, imageName string) *service.ServiceCon imageName, nil, nil, + nil, testPrivatePorts(t), testPublicPorts(t), []string{"bin", "bash", "ls"}, diff --git a/core/server/api_container/server/service_network/default_service_network_test.go b/core/server/api_container/server/service_network/default_service_network_test.go index 9456aa35d1..516f3aa651 100644 --- a/core/server/api_container/server/service_network/default_service_network_test.go +++ b/core/server/api_container/server/service_network/default_service_network_test.go @@ -1215,6 +1215,7 @@ func testServiceConfig(t *testing.T, imageName string) *service.ServiceConfig { nil, nil, nil, + nil, 0, 0, "", diff --git a/core/server/api_container/server/startosis_engine/kurtosis_instruction/add_service/add_service_shared.go b/core/server/api_container/server/startosis_engine/kurtosis_instruction/add_service/add_service_shared.go index e0a264c518..bfedb357f0 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_instruction/add_service/add_service_shared.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_instruction/add_service/add_service_shared.go @@ -3,6 +3,8 @@ package add_service import ( "context" "fmt" + "time" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service_directory" "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/service_network" @@ -17,7 +19,6 @@ import ( "github.com/kurtosis-tech/stacktrace" "github.com/sirupsen/logrus" "go.starlark.net/starlark" - "time" ) const ( @@ -199,6 +200,7 @@ func replaceMagicStrings( serviceConfig.GetContainerImageName(), serviceConfig.GetImageBuildSpec(), serviceConfig.GetImageRegistrySpec(), + serviceConfig.GetNixBuildSpec(), serviceConfig.GetPrivatePorts(), serviceConfig.GetPublicPorts(), entrypoints, diff --git a/core/server/api_container/server/startosis_engine/kurtosis_instruction/add_service/add_service_shared_test.go b/core/server/api_container/server/startosis_engine/kurtosis_instruction/add_service/add_service_shared_test.go index 24627af559..042c0bbe17 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_instruction/add_service/add_service_shared_test.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_instruction/add_service/add_service_shared_test.go @@ -2,6 +2,9 @@ package add_service import ( "fmt" + "os" + "testing" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/database_accessors/enclave_db" "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/kurtosis_instruction/shared_helpers" @@ -10,8 +13,6 @@ import ( "github.com/stretchr/testify/require" bolt "go.etcd.io/bbolt" "go.starlark.net/starlark" - "os" - "testing" ) const ( @@ -42,6 +43,7 @@ func TestAddServiceShared_EntryPointArgsRuntimeValueAreReplaced(t *testing.T) { nil, nil, nil, + nil, []string{"-- " + runtimeValue}, nil, nil, @@ -89,6 +91,7 @@ func TestAddServiceShared_CmdArgsRuntimeValueAreReplaced(t *testing.T) { nil, nil, nil, + nil, []string{"bash", "-c", "sleep " + runtimeValue}, nil, nil, @@ -136,6 +139,7 @@ func TestAddServiceShared_EnvVarsWithRuntimeValueAreReplaced(t *testing.T) { nil, nil, nil, + nil, map[string]string{ "PORT": runtimeValue, }, @@ -190,6 +194,7 @@ func TestAddServiceShared_ServiceNameWithRuntimeValuesAreReplaced(t *testing.T) nil, nil, nil, + nil, 0, 0, "", diff --git a/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/tasks_shared.go b/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/tasks_shared.go index 53119e30c1..2632a5eb6c 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/tasks_shared.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/tasks_shared.go @@ -3,6 +3,10 @@ package tasks import ( "context" "fmt" + "reflect" + "strings" + "time" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/exec_result" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service_directory" @@ -19,9 +23,6 @@ import ( "github.com/sirupsen/logrus" "go.starlark.net/starlark" "go.starlark.net/starlarkstruct" - "reflect" - "strings" - "time" ) // shared constants @@ -260,6 +261,7 @@ func getServiceConfig( nil, nil, nil, + nil, // This make sure that the container does not stop as soon as it starts // This only is needed for kubernetes at the moment // TODO: Instead of creating a service and running exec commands diff --git a/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/add_service_framework_test.go b/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/add_service_framework_test.go index f5d5692864..adb5eb05ac 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/add_service_framework_test.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/add_service_framework_test.go @@ -2,9 +2,10 @@ package test_engine import ( "fmt" - "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_packages/mock_package_content_provider" "testing" + "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_packages/mock_package_content_provider" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/container" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/port_spec" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service" @@ -36,6 +37,7 @@ func (suite *KurtosisPlanInstructionTestSuite) TestAddService() { testContainerImageName, nil, nil, + nil, map[string]*port_spec.PortSpec{}, map[string]*port_spec.PortSpec{}, nil, diff --git a/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/add_services_framework_test.go b/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/add_services_framework_test.go index 8ac6c02305..7912c700a0 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/add_services_framework_test.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/add_services_framework_test.go @@ -2,13 +2,14 @@ package test_engine import ( "fmt" - "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_packages/mock_package_content_provider" "io" "net/http" "net/url" "strings" "testing" + "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_packages/mock_package_content_provider" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/container" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/port_spec" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service" @@ -52,6 +53,7 @@ func (suite *KurtosisPlanInstructionTestSuite) TestAddServices() { testContainerImageName, nil, nil, + nil, map[string]*port_spec.PortSpec{}, map[string]*port_spec.PortSpec{}, nil, @@ -78,6 +80,7 @@ func (suite *KurtosisPlanInstructionTestSuite) TestAddServices() { testContainerImageName, nil, nil, + nil, map[string]*port_spec.PortSpec{}, map[string]*port_spec.PortSpec{}, nil, diff --git a/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/service_config_image_build_spec_test.go b/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/service_config_image_build_spec_test.go index bf69c89472..64740547f7 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/service_config_image_build_spec_test.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/service_config_image_build_spec_test.go @@ -2,6 +2,8 @@ package test_engine import ( "fmt" + "testing" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/image_build_spec" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/port_spec" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service" @@ -10,7 +12,6 @@ import ( "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/kurtosis_types/service_config" "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_packages" "github.com/stretchr/testify/require" - "testing" ) type serviceConfigImageBuildSpecTestCase struct { @@ -71,6 +72,7 @@ func (t *serviceConfigImageBuildSpecTestCase) Assert(typeValue builtin_argument. testContainerImageName, expectedImageBuildSpec, nil, + nil, map[string]*port_spec.PortSpec{}, map[string]*port_spec.PortSpec{}, nil, diff --git a/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/service_config_image_registry_spec_test.go b/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/service_config_image_registry_spec_test.go index 227d5ef7de..af3fa83ba2 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/service_config_image_registry_spec_test.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/service_config_image_registry_spec_test.go @@ -2,6 +2,8 @@ package test_engine import ( "fmt" + "testing" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/image_registry_spec" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/port_spec" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service" @@ -10,7 +12,6 @@ import ( "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/kurtosis_types/service_config" "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_packages" "github.com/stretchr/testify/require" - "testing" ) type serviceConfigImageRegistrySpecTest struct { @@ -61,6 +62,7 @@ func (t *serviceConfigImageRegistrySpecTest) Assert(typeValue builtin_argument.K testContainerImageName, nil, expectedImageRegistrySpec, + nil, map[string]*port_spec.PortSpec{}, map[string]*port_spec.PortSpec{}, nil, diff --git a/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/service_config_minimal_framework_test.go b/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/service_config_minimal_framework_test.go index 6e123bb42f..1199948dc0 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/service_config_minimal_framework_test.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/service_config_minimal_framework_test.go @@ -2,6 +2,8 @@ package test_engine import ( "fmt" + "testing" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/port_spec" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service" "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/service_network" @@ -9,7 +11,6 @@ import ( "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/kurtosis_types/service_config" "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_packages" "github.com/stretchr/testify/require" - "testing" ) type serviceConfigMinimalTestCase struct { @@ -48,6 +49,7 @@ func (t *serviceConfigMinimalTestCase) Assert(typeValue builtin_argument.Kurtosi testContainerImageName, nil, nil, + nil, map[string]*port_spec.PortSpec{}, map[string]*port_spec.PortSpec{}, nil, diff --git a/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/service_config_toleration_test.go b/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/service_config_toleration_test.go index b932a212ff..09f32a09a5 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/service_config_toleration_test.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/service_config_toleration_test.go @@ -2,6 +2,8 @@ package test_engine import ( "fmt" + "testing" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/port_spec" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service" "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/service_network" @@ -10,7 +12,6 @@ import ( "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_packages" "github.com/stretchr/testify/require" v1 "k8s.io/api/core/v1" - "testing" ) type serviceConfigTolerationTest struct { @@ -63,6 +64,7 @@ func (t *serviceConfigTolerationTest) Assert(typeValue builtin_argument.Kurtosis testContainerImageName, nil, nil, + nil, map[string]*port_spec.PortSpec{}, map[string]*port_spec.PortSpec{}, nil, diff --git a/core/server/api_container/server/startosis_engine/kurtosis_types/service_config/service_config.go b/core/server/api_container/server/startosis_engine/kurtosis_types/service_config/service_config.go index 0ec4b6d8f4..ccd44840ac 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_types/service_config/service_config.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_types/service_config/service_config.go @@ -2,8 +2,12 @@ package service_config import ( "fmt" + "math" + "path" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/image_build_spec" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/image_registry_spec" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/nix_build_spec" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/port_spec" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service_directory" @@ -22,8 +26,6 @@ import ( "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_packages" "go.starlark.net/starlark" v1 "k8s.io/api/core/v1" - "math" - "path" ) const ( @@ -250,6 +252,7 @@ func (config *ServiceConfig) ToKurtosisType( var ok bool var imageName string + var maybeNixBuildSpec *nix_build_spec.NixBuildSpec var maybeImageBuildSpec *image_build_spec.ImageBuildSpec var maybeImageRegistrySpec *image_registry_spec.ImageRegistrySpec rawImageAttrValue, found, interpretationErr := kurtosis_type_constructor.ExtractAttrValue[starlark.Value](config.KurtosisValueTypeDefault, ImageAttr) @@ -259,7 +262,7 @@ func (config *ServiceConfig) ToKurtosisType( if !found { return nil, startosis_errors.NewInterpretationError("Required attribute '%s' could not be found on type '%s'", ImageAttr, ServiceConfigTypeName) } - imageName, maybeImageBuildSpec, maybeImageRegistrySpec, interpretationErr = convertImage( + imageName, maybeImageBuildSpec, maybeImageRegistrySpec, maybeNixBuildSpec, interpretationErr = convertImage( rawImageAttrValue, locatorOfModuleInWhichThisBuiltInIsBeingCalled, packageId, @@ -501,6 +504,7 @@ func (config *ServiceConfig) ToKurtosisType( imageName, maybeImageBuildSpec, maybeImageRegistrySpec, + maybeNixBuildSpec, privatePorts, publicPorts, entryPointArgs, @@ -672,31 +676,38 @@ func convertImage( locatorOfModuleInWhichThisBuiltInIsBeingCalled string, packageId string, packageContentProvider startosis_packages.PackageContentProvider, - packageReplaceOptions map[string]string) (string, *image_build_spec.ImageBuildSpec, *image_registry_spec.ImageRegistrySpec, *startosis_errors.InterpretationError) { + packageReplaceOptions map[string]string) (string, *image_build_spec.ImageBuildSpec, *image_registry_spec.ImageRegistrySpec, *nix_build_spec.NixBuildSpec, *startosis_errors.InterpretationError) { imageBuildSpecStarlarkType, isImageBuildSpecStarlarkType := image.(*ImageBuildSpec) imageRegistrySpecStarlarkType, isImageRegistrySpecStarlarkType := image.(*ImageRegistrySpec) + NixBuildSpecStarlarkType, isNixBuildSpecStarlarkType := image.(*NixBuildSpec) if isImageBuildSpecStarlarkType { imageBuildSpec, interpretationErr := imageBuildSpecStarlarkType.ToKurtosisType(locatorOfModuleInWhichThisBuiltInIsBeingCalled, packageId, packageContentProvider, packageReplaceOptions) if interpretationErr != nil { - return "", nil, nil, interpretationErr + return "", nil, nil, nil, interpretationErr } imageName, interpretationErr := imageBuildSpecStarlarkType.GetImageName() if interpretationErr != nil { - return "", nil, nil, interpretationErr + return "", nil, nil, nil, interpretationErr } - return imageName, imageBuildSpec, nil, nil + return imageName, imageBuildSpec, nil, nil, nil } else if isImageRegistrySpecStarlarkType { imageRegistrySpec, interpretationErr := imageRegistrySpecStarlarkType.ToKurtosisType() if interpretationErr != nil { - return "", nil, nil, interpretationErr + return "", nil, nil, nil, interpretationErr + } + return imageRegistrySpec.GetImageName(), nil, imageRegistrySpec, nil, nil + } else if isNixBuildSpecStarlarkType { + nixBuildSpec, interpretationErr := NixBuildSpecStarlarkType.ToKurtosisType(locatorOfModuleInWhichThisBuiltInIsBeingCalled, packageId, packageContentProvider, packageReplaceOptions) + if interpretationErr != nil { + return "", nil, nil, nil, interpretationErr } - return imageRegistrySpec.GetImageName(), nil, imageRegistrySpec, nil + return nixBuildSpec.GetFlake(), nil, nil, nixBuildSpec, nil } else { imageName, interpretationErr := kurtosis_types.SafeCastToString(image, ImageAttr) if interpretationErr != nil { - return "", nil, nil, interpretationErr + return "", nil, nil, nil, interpretationErr } - return imageName, nil, nil, nil + return imageName, nil, nil, nil, nil } } diff --git a/core/server/api_container/server/startosis_engine/startosis_validator/images_validator.go b/core/server/api_container/server/startosis_engine/startosis_validator/images_validator.go index 8cc701b9aa..b3c6fe134f 100644 --- a/core/server/api_container/server/startosis_engine/startosis_validator/images_validator.go +++ b/core/server/api_container/server/startosis_engine/startosis_validator/images_validator.go @@ -2,9 +2,11 @@ package startosis_validator import ( "context" + "sync" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/image_build_spec" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/image_registry_spec" - "sync" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/nix_build_spec" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/image_download_mode" @@ -58,6 +60,10 @@ func (validator *ImagesValidator) Validate( wg.Add(1) go validator.buildImageUsingBackend(ctx, wg, imageCurrentlyValidating, validator.kurtosisBackend, imageName, imageBuildSpec, imageValidationErrors, imageValidationStarted, imageValidationFinished) } + for imageName, imageBuildSpec := range environment.nixToBuild { + wg.Add(1) + go validator.nixBuildUsingBackend(ctx, wg, imageCurrentlyValidating, validator.kurtosisBackend, imageName, imageBuildSpec, imageValidationErrors, imageValidationStarted, imageValidationFinished) + } wg.Wait() logrus.Debug("All image validation submitted, currently in progress.") } @@ -116,3 +122,35 @@ func (validator *ImagesValidator) buildImageUsingBackend( } logrus.Debugf("Container image '%s' successfully built", imageName) } + +func (validator *ImagesValidator) nixBuildUsingBackend( + ctx context.Context, + wg *sync.WaitGroup, + imageCurrentlyBuilding chan bool, + backend *backend_interface.KurtosisBackend, + imageName string, + nixBuildSpec *nix_build_spec.NixBuildSpec, + buildErrors chan<- error, + nixBuildStarted chan<- string, + nixBuildFinished chan<- *ValidatedImage) { + logrus.Debugf("Requesting the build of image: '%s'", imageName) + var imageArch string + imageBuiltLocally := true + imagePulledFromRemote := false + defer wg.Done() + imageCurrentlyBuilding <- true + nixBuildStarted <- imageName + defer func() { + <-imageCurrentlyBuilding + nixBuildFinished <- NewValidatedImage(imageName, imagePulledFromRemote, imageBuiltLocally, imageArch) + }() + + logrus.Debugf("Starting the build of image: '%s'", imageName) + imageArch, err := (*backend).NixBuild(ctx, nixBuildSpec) + if err != nil { + logrus.Warnf("Container image '%s' build failed. Error was: '%s'", imageName, err.Error()) + buildErrors <- startosis_errors.WrapWithValidationError(err, "Failed to build the required image '%v'.", imageName) + return + } + logrus.Debugf("Container image '%s' successfully built", imageName) +} diff --git a/core/server/api_container/server/startosis_engine/startosis_validator/validator_environment.go b/core/server/api_container/server/startosis_engine/startosis_validator/validator_environment.go index b125a0a358..a0968d8457 100644 --- a/core/server/api_container/server/startosis_engine/startosis_validator/validator_environment.go +++ b/core/server/api_container/server/startosis_engine/startosis_validator/validator_environment.go @@ -5,6 +5,7 @@ import ( "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/image_build_spec" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/image_download_mode" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/image_registry_spec" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/nix_build_spec" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service_directory" "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_errors" @@ -15,6 +16,7 @@ import ( type ValidatorEnvironment struct { imagesToPull map[string]*image_registry_spec.ImageRegistrySpec // "set" of images that need to be downloaded imagesToBuild map[string]*image_build_spec.ImageBuildSpec + nixToBuild map[string]*nix_build_spec.NixBuildSpec serviceNames map[service.ServiceName]ComponentExistence artifactNames map[string]ComponentExistence persistentKeys map[service_directory.DirectoryPersistentKey]ComponentExistence @@ -39,6 +41,7 @@ func NewValidatorEnvironment(serviceNames map[service.ServiceName]bool, artifact return &ValidatorEnvironment{ imagesToPull: map[string]*image_registry_spec.ImageRegistrySpec{}, imagesToBuild: map[string]*image_build_spec.ImageBuildSpec{}, + nixToBuild: map[string]*nix_build_spec.NixBuildSpec{}, serviceNames: serviceNamesWithComponentExistence, artifactNames: artifactNamesWithComponentExistence, serviceNameToPrivatePortIDs: serviceNameToPrivatePortIds, @@ -65,8 +68,12 @@ func (environmemt *ValidatorEnvironment) AppendImageToPullWithAuth(containerImag environmemt.imagesToPull[containerImage] = registrySpec } +func (environment *ValidatorEnvironment) AppendRequiredNixBuild(containerImage string, nixBuildSpec *nix_build_spec.NixBuildSpec) { + environment.nixToBuild[containerImage] = nixBuildSpec +} + func (environment *ValidatorEnvironment) GetNumberOfContainerImagesToProcess() uint32 { - return uint32(len(environment.imagesToPull) + len(environment.imagesToBuild)) + return uint32(len(environment.imagesToPull) + len(environment.imagesToBuild) + len(environment.nixToBuild)) } func (environment *ValidatorEnvironment) AddServiceName(serviceName service.ServiceName) { From 22dd144a1973119b53fa2f6ea56dd09982f5c448 Mon Sep 17 00:00:00 2001 From: lostbean Date: Thu, 1 Feb 2024 11:57:28 -0300 Subject: [PATCH 03/15] add nix to APIC image --- core/server/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/server/Dockerfile b/core/server/Dockerfile index df50957179..7536aa33b6 100644 --- a/core/server/Dockerfile +++ b/core/server/Dockerfile @@ -1,7 +1,8 @@ FROM alpine:3.17 # We need protobut-dev to run protobuf compiler against startosis .proto files -RUN apk update && apk add --no-cache bash protobuf-dev +RUN apk update && apk add --no-cache sudo shadow bash curl xz +RUN sh <(curl -L https://nixos.org/nix/install) --daemon --yes ARG TARGETARCH From 4f05f8864850a6e359778c2e77b4b4fd089d5307 Mon Sep 17 00:00:00 2001 From: lostbean Date: Thu, 1 Feb 2024 13:21:34 -0300 Subject: [PATCH 04/15] add nix to APIC image --- core/server/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/server/Dockerfile b/core/server/Dockerfile index 7536aa33b6..cd57de82c6 100644 --- a/core/server/Dockerfile +++ b/core/server/Dockerfile @@ -1,7 +1,7 @@ FROM alpine:3.17 # We need protobut-dev to run protobuf compiler against startosis .proto files -RUN apk update && apk add --no-cache sudo shadow bash curl xz +RUN apk update && apk add --no-cache bash protobuf-dev sudo shadow curl xz RUN sh <(curl -L https://nixos.org/nix/install) --daemon --yes ARG TARGETARCH From 869a1de94f03e3fa30abb627f8c691d4f8e67160 Mon Sep 17 00:00:00 2001 From: lostbean Date: Thu, 1 Feb 2024 21:02:21 -0300 Subject: [PATCH 05/15] update attrs --- .../objects/nix_build_spec/nix_build_spec.go | 12 +-- .../startosis_engine/kurtosis_builtins.go | 1 + .../nix_build_spec_framework_test.go | 60 +++++++++++++++ .../test_engine/static_constants.go | 7 ++ .../service_config/nix_build_spec.go | 74 +++++++++---------- .../service_config/service_config.go | 2 +- .../startosis_validator/images_validator.go | 5 +- 7 files changed, 115 insertions(+), 46 deletions(-) create mode 100644 core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/nix_build_spec_framework_test.go diff --git a/container-engine-lib/lib/backend_interface/objects/nix_build_spec/nix_build_spec.go b/container-engine-lib/lib/backend_interface/objects/nix_build_spec/nix_build_spec.go index 3706b5395c..f82ad9232d 100644 --- a/container-engine-lib/lib/backend_interface/objects/nix_build_spec/nix_build_spec.go +++ b/container-engine-lib/lib/backend_interface/objects/nix_build_spec/nix_build_spec.go @@ -16,19 +16,19 @@ type NixBuildSpec struct { type privateNixBuildSpec struct { ContainerNixFilePath string ContextDirPath string - Flake string + flakeOutput string } -func NewNixBuildSpec(contextDirPath string, containerNixFilePath string, flake string) *NixBuildSpec { +func NewNixBuildSpec(contextDirPath string, containerNixFilePath string, flakeOutput string) *NixBuildSpec { internalNixBuildSpec := &privateNixBuildSpec{ ContainerNixFilePath: containerNixFilePath, ContextDirPath: contextDirPath, - Flake: flake, + flakeOutput: flakeOutput, } return &NixBuildSpec{internalNixBuildSpec} } -func (nixBuildSpec *NixBuildSpec) GetContainerNixFilePath() string { +func (nixBuildSpec *NixBuildSpec) GetNixFlakeFilePath() string { return nixBuildSpec.privateNixBuildSpec.ContainerNixFilePath } @@ -36,8 +36,8 @@ func (nixBuildSpec *NixBuildSpec) GetBuildContextDir() string { return nixBuildSpec.privateNixBuildSpec.ContextDirPath } -func (nixBuildSpec *NixBuildSpec) GetFlake() string { - return nixBuildSpec.privateNixBuildSpec.Flake +func (nixBuildSpec *NixBuildSpec) GetFlakeOutput() string { + return nixBuildSpec.privateNixBuildSpec.flakeOutput } func (nixBuildSpec *NixBuildSpec) MarshalJSON() ([]byte, error) { diff --git a/core/server/api_container/server/startosis_engine/kurtosis_builtins.go b/core/server/api_container/server/startosis_engine/kurtosis_builtins.go index e8a4f6a8ac..5aeae0154e 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_builtins.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_builtins.go @@ -112,6 +112,7 @@ func KurtosisTypeConstructors() []*starlark.Builtin { starlark.NewBuiltin(service_config.ServiceConfigTypeName, service_config.NewServiceConfigType().CreateBuiltin()), starlark.NewBuiltin(service_config.ReadyConditionTypeName, service_config.NewReadyConditionType().CreateBuiltin()), starlark.NewBuiltin(service_config.ImageBuildSpecTypeName, service_config.NewImageBuildSpecType().CreateBuiltin()), + starlark.NewBuiltin(service_config.NixBuildSpecTypeName, service_config.NewNixBuildSpecType().CreateBuiltin()), starlark.NewBuiltin(service_config.ImageRegistrySpecTypeName, service_config.NewImageRegistrySpec().CreateBuiltin()), starlark.NewBuiltin(service_config.UserTypeName, service_config.NewUserType().CreateBuiltin()), starlark.NewBuiltin(service_config.TolerationTypeName, service_config.NewTolerationType().CreateBuiltin()), diff --git a/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/nix_build_spec_framework_test.go b/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/nix_build_spec_framework_test.go new file mode 100644 index 0000000000..15315d2874 --- /dev/null +++ b/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/nix_build_spec_framework_test.go @@ -0,0 +1,60 @@ +package test_engine + +import ( + "fmt" + "testing" + + "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/builtin_argument" + "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/kurtosis_types/service_config" + "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_packages" + "github.com/stretchr/testify/require" +) + +type nixBuildSpecTest struct { + *testing.T + + packageContentProvider *startosis_packages.MockPackageContentProvider +} + +func (suite *KurtosisTypeConstructorTestSuite) TestNixBuildSpecTest() { + suite.packageContentProvider.EXPECT(). + GetAbsoluteLocator(testModulePackageId, testModuleMainFileLocator, testBuildContextDir, testNoPackageReplaceOptions). + Times(1). + Return(testOnDiskNixContextDirPath, nil) + + suite.packageContentProvider.EXPECT(). + GetOnDiskAbsolutePackageFilePath(testNixFlakeLocator). + Times(1). + Return(testOnDiskNixFlakePath, nil) + + suite.run(&nixBuildSpecTest{ + T: suite.T(), + packageContentProvider: suite.packageContentProvider, + }) +} + +func (t *nixBuildSpecTest) GetStarlarkCode() string { + return fmt.Sprintf("%s(%s=%q, %s=%q, %s=%q)", + service_config.NixBuildSpecTypeName, + service_config.FlakeLocationDir, + testNixFlakeLocationDir, + service_config.NixContextAttr, + testNixContextDir, + service_config.FlakeOutputAttr, + testNixFlakeOutput) +} + +func (t *nixBuildSpecTest) Assert(typeValue builtin_argument.KurtosisValueType) { + nixBuildSpecStarlark, ok := typeValue.(*service_config.NixBuildSpec) + require.True(t, ok) + + nixBuildSpec, err := nixBuildSpecStarlark.ToKurtosisType( + testModuleMainFileLocator, + testModulePackageId, + t.packageContentProvider, + testNoPackageReplaceOptions) + require.Nil(t, err) + require.Equal(t, testOnDiskNixFlakePath, nixBuildSpec.GetNixFlakeFilePath()) + require.Equal(t, testOnDiskNixContextDirPath, nixBuildSpec.GetBuildContextDir()) + require.Equal(t, testNixFlakeOutput, nixBuildSpec.GetFlakeOutput()) +} diff --git a/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/static_constants.go b/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/static_constants.go index 607e480d9f..505d83e52d 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/static_constants.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/static_constants.go @@ -34,6 +34,13 @@ var ( testOnDiskContextDirPath = "kurtosis-data/test-package" testOnDiskContainerImagePath = "kurtosis-data/test-package/Dockerfile" + testNixContextDir = "./" + testNixFlakeOutput = ".#foo" + testNixFlakeLocationDir = "./" + testOnDiskNixContextDirPath = "kurtosis-data/test-package" + testOnDiskNixFlakePath = "kurtosis-data/test-package/flake.nix" + testNixFlakeLocator = "kurtosis-data/test-package/flake.nix" + testRegistryAddr = "http://registry.test.io" testRegistryUsername = "kurtosis" testRegistryPassword = "password" diff --git a/core/server/api_container/server/startosis_engine/kurtosis_types/service_config/nix_build_spec.go b/core/server/api_container/server/startosis_engine/kurtosis_types/service_config/nix_build_spec.go index 4e6bfd0db3..7dfac42d19 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_types/service_config/nix_build_spec.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_types/service_config/nix_build_spec.go @@ -17,12 +17,12 @@ import ( const ( NixBuildSpecTypeName = "NixBuildSpec" - NixNameAttr = "nix_name" - NixContextAttr = "build_context_dir" - FlakeAttr = "target_stage" + FlakeLocationDir = "flake_location_dir" + FlakeOutputAttr = "flake_output" + NixContextAttr = "build_context_dir" // Currently only supports container nixs named Dockerfile - defaultContainerNixFileName = "Dockerfile" + defaultNixFlakeFilePath = "flake.nix" ) func NewNixBuildSpecType() *kurtosis_type_constructor.KurtosisTypeConstructor { @@ -31,11 +31,11 @@ func NewNixBuildSpecType() *kurtosis_type_constructor.KurtosisTypeConstructor { Name: NixBuildSpecTypeName, Arguments: []*builtin_argument.BuiltinArgument{ { - Name: NixNameAttr, + Name: FlakeLocationDir, IsOptional: false, ZeroValueProvider: builtin_argument.ZeroValueProvider[starlark.String], Validator: func(value starlark.Value) *startosis_errors.InterpretationError { - return builtin_argument.NonEmptyString(value, NixNameAttr) + return builtin_argument.NonEmptyString(value, FlakeLocationDir) }, }, { @@ -47,11 +47,11 @@ func NewNixBuildSpecType() *kurtosis_type_constructor.KurtosisTypeConstructor { }, }, { - Name: FlakeAttr, + Name: FlakeOutputAttr, IsOptional: true, ZeroValueProvider: builtin_argument.ZeroValueProvider[starlark.String], Validator: func(value starlark.Value) *startosis_errors.InterpretationError { - return builtin_argument.NonEmptyString(value, FlakeAttr) + return builtin_argument.NonEmptyString(value, FlakeOutputAttr) }, }, }, @@ -85,47 +85,40 @@ func (nixBuildSpec *NixBuildSpec) Copy() (builtin_argument.KurtosisValueType, er }, nil } -// Name to give nix built from NixBuildSpec -func (nixBuildSpec *NixBuildSpec) GetNixName() (string, *startosis_errors.InterpretationError) { - nixName, found, interpretationErr := kurtosis_type_constructor.ExtractAttrValue[starlark.String](nixBuildSpec.KurtosisValueTypeDefault, NixNameAttr) +// Relative locator of build context directory +func (nixBuildSpec *NixBuildSpec) GetBuildContextLocator() (string, *startosis_errors.InterpretationError) { + buildContextLocator, found, interpretationErr := kurtosis_type_constructor.ExtractAttrValue[starlark.String](nixBuildSpec.KurtosisValueTypeDefault, NixContextAttr) if interpretationErr != nil { return "", interpretationErr } if !found { return "", startosis_errors.NewInterpretationError("Required attribute '%s' could not be found on type '%s'", - NixNameAttr, NixBuildSpecTypeName) + NixContextAttr, NixBuildSpecTypeName) } - nixNameStr := nixName.GoString() - return nixNameStr, nil + buildContextLocatorStr := buildContextLocator.GoString() + return buildContextLocatorStr, nil } -// Relative locator of build context directory -func (nixBuildSpec *NixBuildSpec) GetBuildContextLocator() (string, *startosis_errors.InterpretationError) { - buildContextLocator, found, interpretationErr := kurtosis_type_constructor.ExtractAttrValue[starlark.String](nixBuildSpec.KurtosisValueTypeDefault, NixContextAttr) +func (nixBuildSpec *NixBuildSpec) GetFlakeOutput() (string, *startosis_errors.InterpretationError) { + flakeOutput, found, interpretationErr := kurtosis_type_constructor.ExtractAttrValue[starlark.String](nixBuildSpec.KurtosisValueTypeDefault, FlakeOutputAttr) if interpretationErr != nil { return "", interpretationErr } if !found { - return "", startosis_errors.NewInterpretationError("Required attribute '%s' could not be found on type '%s'", - NixContextAttr, NixBuildSpecTypeName) + return "", nil } - buildContextLocatorStr := buildContextLocator.GoString() - return buildContextLocatorStr, nil + return flakeOutput.GoString(), nil } -// GetTargetStage is used for specifying which stage of a multi-stage container nix build to execute -// Default value is the empty string for single stage nix builds (common case) -// Info on target stage and multi-stag builds for Docker nixs: https://docs.docker.com/build/building/multi-stage/ -func (nixBuildSpec *NixBuildSpec) GetTargetStage() (string, *startosis_errors.InterpretationError) { - targetStage, found, interpretationErr := kurtosis_type_constructor.ExtractAttrValue[starlark.String](nixBuildSpec.KurtosisValueTypeDefault, FlakeAttr) +func (nixBuildSpec *NixBuildSpec) GetFlakeLocationDir() (string, *startosis_errors.InterpretationError) { + flakeLocator, found, interpretationErr := kurtosis_type_constructor.ExtractAttrValue[starlark.String](nixBuildSpec.KurtosisValueTypeDefault, FlakeLocationDir) if interpretationErr != nil { return "", interpretationErr } if !found { return "", nil } - targetStageStr := targetStage.GoString() - return targetStageStr, nil + return flakeLocator.GoString(), nil } func (nixBuildSpec *NixBuildSpec) ToKurtosisType( @@ -139,8 +132,14 @@ func (nixBuildSpec *NixBuildSpec) ToKurtosisType( return nil, interpretationErr } - buildContextDirPathOnDisk, containerNixFilePathOnDisk, interpretationErr := getOnDiskNixBuildSpecPaths( + flakeLocationDir, interpretationErr := nixBuildSpec.GetFlakeLocationDir() + if interpretationErr != nil { + return nil, interpretationErr + } + + buildContextDirPathOnDisk, flakeNixFilePathOnDisk, interpretationErr := getOnDiskNixBuildSpecPaths( buildContextLocator, + flakeLocationDir, packageId, locatorOfModuleInWhichThisBuiltInIsBeingCalled, packageContentProvider, @@ -149,17 +148,18 @@ func (nixBuildSpec *NixBuildSpec) ToKurtosisType( return nil, interpretationErr } - targetStageStr, interpretationErr := nixBuildSpec.GetTargetStage() + flakeOutputStr, interpretationErr := nixBuildSpec.GetFlakeOutput() if interpretationErr != nil { return nil, interpretationErr } - return nix_build_spec.NewNixBuildSpec(buildContextDirPathOnDisk, containerNixFilePathOnDisk, targetStageStr), nil + return nix_build_spec.NewNixBuildSpec(buildContextDirPathOnDisk, flakeNixFilePathOnDisk, flakeOutputStr), nil } -// Returns the filepath of the build context directory and container nix on APIC based on package info +// Returns the filepath of the build context directory and flake nix on APIC based on package info func getOnDiskNixBuildSpecPaths( buildContextLocator string, + flakeLocationDir string, packageId string, locatorOfModuleInWhichThisBuiltInIsBeingCalled string, packageContentProvider startosis_packages.PackageContentProvider, @@ -175,15 +175,15 @@ func getOnDiskNixBuildSpecPaths( } // get on disk directory path of Dockerfile - containerNixAbsoluteLocator := path.Join(contextDirAbsoluteLocator, defaultContainerNixFileName) + flakeNixAbsoluteLocator := path.Join(contextDirAbsoluteLocator, flakeLocationDir, defaultNixFlakeFilePath) - containerNixPathOnDisk, interpretationErr := packageContentProvider.GetOnDiskAbsolutePackageFilePath(containerNixAbsoluteLocator) + flakeNixPathOnDisk, interpretationErr := packageContentProvider.GetOnDiskAbsolutePackageFilePath(flakeNixAbsoluteLocator) if interpretationErr != nil { return "", "", interpretationErr } - // Assume, that container nix sits at the same level as context directory to get context dir path on disk - contextDirPathOnDisk := filepath.Dir(containerNixPathOnDisk) + // Assume, that flake nix sits at the same level as context directory to get context dir path on disk + contextDirPathOnDisk := filepath.Dir(flakeNixPathOnDisk) - return contextDirPathOnDisk, containerNixPathOnDisk, nil + return contextDirPathOnDisk, flakeNixPathOnDisk, nil } diff --git a/core/server/api_container/server/startosis_engine/kurtosis_types/service_config/service_config.go b/core/server/api_container/server/startosis_engine/kurtosis_types/service_config/service_config.go index ccd44840ac..b090298956 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_types/service_config/service_config.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_types/service_config/service_config.go @@ -701,7 +701,7 @@ func convertImage( if interpretationErr != nil { return "", nil, nil, nil, interpretationErr } - return nixBuildSpec.GetFlake(), nil, nil, nixBuildSpec, nil + return nixBuildSpec.GetFlakeOutput(), nil, nil, nixBuildSpec, nil } else { imageName, interpretationErr := kurtosis_types.SafeCastToString(image, ImageAttr) if interpretationErr != nil { diff --git a/core/server/api_container/server/startosis_engine/startosis_validator/images_validator.go b/core/server/api_container/server/startosis_engine/startosis_validator/images_validator.go index b3c6fe134f..480b90738d 100644 --- a/core/server/api_container/server/startosis_engine/startosis_validator/images_validator.go +++ b/core/server/api_container/server/startosis_engine/startosis_validator/images_validator.go @@ -60,9 +60,10 @@ func (validator *ImagesValidator) Validate( wg.Add(1) go validator.buildImageUsingBackend(ctx, wg, imageCurrentlyValidating, validator.kurtosisBackend, imageName, imageBuildSpec, imageValidationErrors, imageValidationStarted, imageValidationFinished) } - for imageName, imageBuildSpec := range environment.nixToBuild { + for imageName, nixBuildSpec := range environment.nixToBuild { wg.Add(1) - go validator.nixBuildUsingBackend(ctx, wg, imageCurrentlyValidating, validator.kurtosisBackend, imageName, imageBuildSpec, imageValidationErrors, imageValidationStarted, imageValidationFinished) + logrus.Warnf("%v - %v", imageName, nixBuildSpec) + go validator.nixBuildUsingBackend(ctx, wg, imageCurrentlyValidating, validator.kurtosisBackend, imageName, nixBuildSpec, imageValidationErrors, imageValidationStarted, imageValidationFinished) } wg.Wait() logrus.Debug("All image validation submitted, currently in progress.") From 3dcf944213dae0537bf70d98fc1502bb0da57ccd Mon Sep 17 00:00:00 2001 From: lostbean Date: Mon, 5 Feb 2024 18:42:01 -0300 Subject: [PATCH 06/15] add nix to the building queue --- .../kurtosis_instruction/add_service/add_service_shared.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/server/api_container/server/startosis_engine/kurtosis_instruction/add_service/add_service_shared.go b/core/server/api_container/server/startosis_engine/kurtosis_instruction/add_service/add_service_shared.go index bfedb357f0..93c28bfe95 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_instruction/add_service/add_service_shared.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_instruction/add_service/add_service_shared.go @@ -108,6 +108,8 @@ func validateSingleService(validatorEnvironment *startosis_validator.ValidatorEn validatorEnvironment.AppendRequiredImageBuild(serviceConfig.GetContainerImageName(), serviceConfig.GetImageBuildSpec()) } else if serviceConfig.GetImageRegistrySpec() != nil { validatorEnvironment.AppendImageToPullWithAuth(serviceConfig.GetContainerImageName(), serviceConfig.GetImageRegistrySpec()) + } else if serviceConfig.GetNixBuildSpec() != nil { + validatorEnvironment.AppendRequiredNixBuild(serviceConfig.GetContainerImageName(), serviceConfig.GetNixBuildSpec()) } else { validatorEnvironment.AppendRequiredImagePull(serviceConfig.GetContainerImageName()) } From 668efbf682fd82ba0d6d85bc1fde1fcfcabe9b73 Mon Sep 17 00:00:00 2001 From: lostbean Date: Tue, 6 Feb 2024 10:06:23 -0300 Subject: [PATCH 07/15] add nix to debug image --- core/server/Dockerfile.debug | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/server/Dockerfile.debug b/core/server/Dockerfile.debug index a8b268dc55..1aceb832e5 100644 --- a/core/server/Dockerfile.debug +++ b/core/server/Dockerfile.debug @@ -1,7 +1,8 @@ FROM alpine:3.19 # We need protobut-dev to run protobuf compiler against startosis .proto files -RUN apk update && apk add --no-cache bash protobuf-dev +RUN apk update && apk add --no-cache bash protobuf-dev sudo shadow curl xz +RUN sh <(curl -L https://nixos.org/nix/install) --daemon --yes # Make sure that you changed the port inside the APIC's code before changing it here EXPOSE 50103 From 093083cb8dc491e7c2713a507beda610902513b2 Mon Sep 17 00:00:00 2001 From: lostbean Date: Tue, 6 Feb 2024 16:01:42 -0300 Subject: [PATCH 08/15] get nix to build image --- .../docker/docker_manager/docker_manager.go | 115 +++++++++++++++++- .../objects/nix_build_spec/nix_build_spec.go | 20 +-- .../nix_build_spec_framework_test.go | 9 +- .../test_engine/static_constants.go | 9 +- .../service_config/nix_build_spec.go | 12 +- .../startosis_validator/images_validator.go | 6 +- 6 files changed, 147 insertions(+), 24 deletions(-) diff --git a/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go b/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go index 64ccaee7d3..5a3a4f2c22 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go @@ -6,14 +6,19 @@ package docker_manager import ( + "archive/tar" "bufio" "bytes" + "compress/gzip" "context" "encoding/json" "fmt" "io" + "log" "math" "net" + "os" + "os/exec" "regexp" "strings" "sync" @@ -1315,8 +1320,116 @@ func (manager *DockerManager) FetchImage(ctx context.Context, image string, regi return pulledFromRemote, imageArchitecture, nil } +// ImageManifest represents the structure of the manifest.json file +type ImageManifest struct { + Config string `json:"Config"` + RepoTags []string `json:"RepoTags"` + Layers []string `json:"Layers"` +} +type Manifest struct { + Images []ImageManifest +} + +// Add other fields if needed + +func readTarGz(filename string) ImageManifest { + cmd := exec.Command("tar", "-axf", filename, "manifest.json", "-O") + stdout, err := cmd.Output() + if err != nil { + log.Fatal(err) + } + var data ImageManifest + err = json.Unmarshal([]byte(stdout), &data) + if err != nil { + fmt.Println("error:", err) + } + return data +} + +// GetRepoTags extracts the RepoTags from a Docker image file +func GetRepoTags(imageFilePath string) ([]string, error) { + imageFile, err := os.Open(imageFilePath) + if err != nil { + return nil, stacktrace.Propagate(err, "Fail to open image file %s", imageFilePath) + } + defer imageFile.Close() + + gzipReader, err := gzip.NewReader(imageFile) + if err != nil { + return nil, stacktrace.Propagate(err, "Fail to ungzip image file %s", stacktrace.Propagate(err, "Fail to read the image files")) + } + defer gzipReader.Close() + + tarReader := tar.NewReader(gzipReader) + + var found bool = false + for { + header, err := tarReader.Next() + if err == io.EOF { + break + } + if err != nil { + return nil, stacktrace.Propagate(err, "Fail to read the image files") + } + + if header.Name == "manifest.json" { + found = true + break + } + } + + if !found { + return nil, stacktrace.NewError("manifest.json not found in the image") + } + + var imageManifest []ImageManifest + jsonDecoder := json.NewDecoder(tarReader) + if err := jsonDecoder.Decode(&imageManifest); err != nil { + return nil, stacktrace.Propagate(err, "Could not parse the manifest.json") + } + + return imageManifest[0].RepoTags, nil +} + func (manager *DockerManager) NixBuild(ctx context.Context, nixBuildSpec *nix_build_spec.NixBuildSpec) (string, error) { - return "", nil + getBuildContextDir := nixBuildSpec.GetBuildContextDir() + logrus.Warnf("Nix Build Dir: %s", getBuildContextDir) + flakeOutput := nixBuildSpec.GetFlakeOutput() + logrus.Warnf("Nix Flake Output: %s", flakeOutput) + flakeOutputFilePath := nixBuildSpec.GetNixFlakeDir() + logrus.Warnf("Nix Build Flake: %s", flakeOutputFilePath) + + entries, err := os.ReadDir(getBuildContextDir) + if err != nil { + logrus.Fatal(err) + } + + for _, e := range entries { + fmt.Println(e.Name()) + } + + cmd := exec.Command("/nix/var/nix/profiles/default/bin/nix", "build", fmt.Sprintf("%s/.#%s", flakeOutputFilePath, flakeOutput), "--print-out-paths", "--extra-experimental-features", "flakes nix-command", "--out-link", "result-123") + + var errBuffer strings.Builder + cmd.Stderr = &errBuffer + imageFileRaw, err := cmd.Output() + if err != nil { + errMsg := errBuffer.String() + logrus.WithError(err).Error(errMsg) + return "", stacktrace.Propagate(err, "Failed to build nix image with Nix.") + } + imageFile := strings.TrimSpace(string(imageFileRaw)) + + imageName, err := GetRepoTags(imageFile) + if err != nil { + return "", err + } + + image, err := os.Open(imageFile) + imgRes, err := manager.dockerClient.ImageLoad(ctx, image, false) + logrus.Debugf("%v", imgRes) + + return imageName[0], nil } func (manager *DockerManager) BuildImage(ctx context.Context, imageName string, imageBuildSpec *image_build_spec.ImageBuildSpec) (string, error) { diff --git a/container-engine-lib/lib/backend_interface/objects/nix_build_spec/nix_build_spec.go b/container-engine-lib/lib/backend_interface/objects/nix_build_spec/nix_build_spec.go index f82ad9232d..d7c193079e 100644 --- a/container-engine-lib/lib/backend_interface/objects/nix_build_spec/nix_build_spec.go +++ b/container-engine-lib/lib/backend_interface/objects/nix_build_spec/nix_build_spec.go @@ -14,22 +14,22 @@ type NixBuildSpec struct { // NixBuildSpec contains the information need for building a container from nix. type privateNixBuildSpec struct { - ContainerNixFilePath string - ContextDirPath string - flakeOutput string + NixFlakeDir string + ContextDirPath string + FlakeOutput string } -func NewNixBuildSpec(contextDirPath string, containerNixFilePath string, flakeOutput string) *NixBuildSpec { +func NewNixBuildSpec(contextDirPath string, nixFlakeDir string, flakeOutput string) *NixBuildSpec { internalNixBuildSpec := &privateNixBuildSpec{ - ContainerNixFilePath: containerNixFilePath, - ContextDirPath: contextDirPath, - flakeOutput: flakeOutput, + NixFlakeDir: nixFlakeDir, + ContextDirPath: contextDirPath, + FlakeOutput: flakeOutput, } return &NixBuildSpec{internalNixBuildSpec} } -func (nixBuildSpec *NixBuildSpec) GetNixFlakeFilePath() string { - return nixBuildSpec.privateNixBuildSpec.ContainerNixFilePath +func (nixBuildSpec *NixBuildSpec) GetNixFlakeDir() string { + return nixBuildSpec.privateNixBuildSpec.NixFlakeDir } func (nixBuildSpec *NixBuildSpec) GetBuildContextDir() string { @@ -37,7 +37,7 @@ func (nixBuildSpec *NixBuildSpec) GetBuildContextDir() string { } func (nixBuildSpec *NixBuildSpec) GetFlakeOutput() string { - return nixBuildSpec.privateNixBuildSpec.flakeOutput + return nixBuildSpec.privateNixBuildSpec.FlakeOutput } func (nixBuildSpec *NixBuildSpec) MarshalJSON() ([]byte, error) { diff --git a/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/nix_build_spec_framework_test.go b/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/nix_build_spec_framework_test.go index 15315d2874..9da723fa2e 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/nix_build_spec_framework_test.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/nix_build_spec_framework_test.go @@ -20,13 +20,18 @@ func (suite *KurtosisTypeConstructorTestSuite) TestNixBuildSpecTest() { suite.packageContentProvider.EXPECT(). GetAbsoluteLocator(testModulePackageId, testModuleMainFileLocator, testBuildContextDir, testNoPackageReplaceOptions). Times(1). - Return(testOnDiskNixContextDirPath, nil) + Return(testBuildContextLocator, nil) suite.packageContentProvider.EXPECT(). GetOnDiskAbsolutePackageFilePath(testNixFlakeLocator). Times(1). Return(testOnDiskNixFlakePath, nil) + suite.packageContentProvider.EXPECT(). + GetOnDiskAbsolutePath(testBuildContextLocator). + Times(1). + Return(testOnDiskContextDirPath, nil) + suite.run(&nixBuildSpecTest{ T: suite.T(), packageContentProvider: suite.packageContentProvider, @@ -54,7 +59,7 @@ func (t *nixBuildSpecTest) Assert(typeValue builtin_argument.KurtosisValueType) t.packageContentProvider, testNoPackageReplaceOptions) require.Nil(t, err) - require.Equal(t, testOnDiskNixFlakePath, nixBuildSpec.GetNixFlakeFilePath()) + require.Equal(t, testOnDiskNixFlakeDir, nixBuildSpec.GetNixFlakeDir()) require.Equal(t, testOnDiskNixContextDirPath, nixBuildSpec.GetBuildContextDir()) require.Equal(t, testNixFlakeOutput, nixBuildSpec.GetFlakeOutput()) } diff --git a/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/static_constants.go b/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/static_constants.go index 505d83e52d..c57d4e3f8c 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/static_constants.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/static_constants.go @@ -35,11 +35,12 @@ var ( testOnDiskContainerImagePath = "kurtosis-data/test-package/Dockerfile" testNixContextDir = "./" - testNixFlakeOutput = ".#foo" - testNixFlakeLocationDir = "./" + testNixFlakeOutput = "foo" + testNixFlakeLocationDir = "./server/app" testOnDiskNixContextDirPath = "kurtosis-data/test-package" - testOnDiskNixFlakePath = "kurtosis-data/test-package/flake.nix" - testNixFlakeLocator = "kurtosis-data/test-package/flake.nix" + testOnDiskNixFlakePath = "kurtosis-data/test-package/server/app/flake.nix" + testOnDiskNixFlakeDir = "kurtosis-data/test-package/server/app" + testNixFlakeLocator = "github.com/kurtosistech/test-package/server/app/flake.nix" testRegistryAddr = "http://registry.test.io" testRegistryUsername = "kurtosis" diff --git a/core/server/api_container/server/startosis_engine/kurtosis_types/service_config/nix_build_spec.go b/core/server/api_container/server/startosis_engine/kurtosis_types/service_config/nix_build_spec.go index 7dfac42d19..d23a312118 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_types/service_config/nix_build_spec.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_types/service_config/nix_build_spec.go @@ -22,7 +22,7 @@ const ( NixContextAttr = "build_context_dir" // Currently only supports container nixs named Dockerfile - defaultNixFlakeFilePath = "flake.nix" + defaultNixFlakeFile = "flake.nix" ) func NewNixBuildSpecType() *kurtosis_type_constructor.KurtosisTypeConstructor { @@ -175,15 +175,19 @@ func getOnDiskNixBuildSpecPaths( } // get on disk directory path of Dockerfile - flakeNixAbsoluteLocator := path.Join(contextDirAbsoluteLocator, flakeLocationDir, defaultNixFlakeFilePath) + flakeNixAbsoluteLocator := path.Join(contextDirAbsoluteLocator, flakeLocationDir, defaultNixFlakeFile) flakeNixPathOnDisk, interpretationErr := packageContentProvider.GetOnDiskAbsolutePackageFilePath(flakeNixAbsoluteLocator) if interpretationErr != nil { return "", "", interpretationErr } + contextDirOnDisk, interpretationErr := packageContentProvider.GetOnDiskAbsolutePath(contextDirAbsoluteLocator) + if interpretationErr != nil { + return "", "", interpretationErr + } // Assume, that flake nix sits at the same level as context directory to get context dir path on disk - contextDirPathOnDisk := filepath.Dir(flakeNixPathOnDisk) + FlakeDirOnDisk := filepath.Dir(flakeNixPathOnDisk) - return contextDirPathOnDisk, flakeNixPathOnDisk, nil + return contextDirOnDisk, FlakeDirOnDisk, nil } diff --git a/core/server/api_container/server/startosis_engine/startosis_validator/images_validator.go b/core/server/api_container/server/startosis_engine/startosis_validator/images_validator.go index 480b90738d..dbe74dce91 100644 --- a/core/server/api_container/server/startosis_engine/startosis_validator/images_validator.go +++ b/core/server/api_container/server/startosis_engine/startosis_validator/images_validator.go @@ -63,7 +63,7 @@ func (validator *ImagesValidator) Validate( for imageName, nixBuildSpec := range environment.nixToBuild { wg.Add(1) logrus.Warnf("%v - %v", imageName, nixBuildSpec) - go validator.nixBuildUsingBackend(ctx, wg, imageCurrentlyValidating, validator.kurtosisBackend, imageName, nixBuildSpec, imageValidationErrors, imageValidationStarted, imageValidationFinished) + go validator.nixBuildUsingBackend(ctx, wg, imageCurrentlyValidating, validator.kurtosisBackend, nixBuildSpec, imageValidationErrors, imageValidationStarted, imageValidationFinished) } wg.Wait() logrus.Debug("All image validation submitted, currently in progress.") @@ -129,11 +129,11 @@ func (validator *ImagesValidator) nixBuildUsingBackend( wg *sync.WaitGroup, imageCurrentlyBuilding chan bool, backend *backend_interface.KurtosisBackend, - imageName string, nixBuildSpec *nix_build_spec.NixBuildSpec, buildErrors chan<- error, nixBuildStarted chan<- string, nixBuildFinished chan<- *ValidatedImage) { + var imageName string logrus.Debugf("Requesting the build of image: '%s'", imageName) var imageArch string imageBuiltLocally := true @@ -147,7 +147,7 @@ func (validator *ImagesValidator) nixBuildUsingBackend( }() logrus.Debugf("Starting the build of image: '%s'", imageName) - imageArch, err := (*backend).NixBuild(ctx, nixBuildSpec) + imageName, err := (*backend).NixBuild(ctx, nixBuildSpec) if err != nil { logrus.Warnf("Container image '%s' build failed. Error was: '%s'", imageName, err.Error()) buildErrors <- startosis_errors.WrapWithValidationError(err, "Failed to build the required image '%v'.", imageName) From 99dc3ae608bc460928c9f6523a0064010dc899f6 Mon Sep 17 00:00:00 2001 From: lostbean Date: Tue, 6 Feb 2024 17:39:15 -0300 Subject: [PATCH 09/15] specify image name --- .../mock_kurtosis_backend.go | 13 ++---- .../objects/nix_build_spec/nix_build_spec.go | 13 +++++- .../objects/service/service_config_test.go | 2 +- .../nix_build_spec_framework_test.go | 4 +- .../test_engine/static_constants.go | 1 + .../service_config/nix_build_spec.go | 45 +++++++++++++++++-- .../service_config/service_config.go | 6 +-- .../startosis_validator/images_validator.go | 13 +++--- 8 files changed, 73 insertions(+), 24 deletions(-) diff --git a/container-engine-lib/lib/backend_interface/mock_kurtosis_backend.go b/container-engine-lib/lib/backend_interface/mock_kurtosis_backend.go index aea4d57d20..63260d872b 100644 --- a/container-engine-lib/lib/backend_interface/mock_kurtosis_backend.go +++ b/container-engine-lib/lib/backend_interface/mock_kurtosis_backend.go @@ -154,10 +154,6 @@ func (_c *MockKurtosisBackend_CopyFilesFromUserService_Call) RunAndReturn(run fu func (_m *MockKurtosisBackend) CreateAPIContainer(ctx context.Context, image string, enclaveUuid enclave.EnclaveUUID, grpcPortNum uint16, enclaveDataVolumeDirpath string, ownIpAddressEnvVar string, customEnvVars map[string]string, shouldStartInDebugMode bool) (*api_container.APIContainer, error) { ret := _m.Called(ctx, image, enclaveUuid, grpcPortNum, enclaveDataVolumeDirpath, ownIpAddressEnvVar, customEnvVars, shouldStartInDebugMode) - if len(ret) == 0 { - panic("no return value specified for CreateAPIContainer") - } - var r0 *api_container.APIContainer var r1 error if rf, ok := ret.Get(0).(func(context.Context, string, enclave.EnclaveUUID, uint16, string, string, map[string]string, bool) (*api_container.APIContainer, error)); ok { @@ -2433,13 +2429,12 @@ func (_c *MockKurtosisBackend_UpdateEnclave_Call) RunAndReturn(run func(context. return _c } -type mockConstructorTestingTNewMockKurtosisBackend interface { +// NewMockKurtosisBackend creates a new instance of MockKurtosisBackend. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewMockKurtosisBackend(t interface { mock.TestingT Cleanup(func()) -} - -// NewMockKurtosisBackend creates a new instance of MockKurtosisBackend. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewMockKurtosisBackend(t mockConstructorTestingTNewMockKurtosisBackend) *MockKurtosisBackend { +}) *MockKurtosisBackend { mock := &MockKurtosisBackend{} mock.Mock.Test(t) diff --git a/container-engine-lib/lib/backend_interface/objects/nix_build_spec/nix_build_spec.go b/container-engine-lib/lib/backend_interface/objects/nix_build_spec/nix_build_spec.go index d7c193079e..5de9eafa0b 100644 --- a/container-engine-lib/lib/backend_interface/objects/nix_build_spec/nix_build_spec.go +++ b/container-engine-lib/lib/backend_interface/objects/nix_build_spec/nix_build_spec.go @@ -2,6 +2,7 @@ package nix_build_spec import ( "encoding/json" + "fmt" "github.com/kurtosis-tech/stacktrace" ) @@ -17,17 +18,23 @@ type privateNixBuildSpec struct { NixFlakeDir string ContextDirPath string FlakeOutput string + ImageName string } -func NewNixBuildSpec(contextDirPath string, nixFlakeDir string, flakeOutput string) *NixBuildSpec { +func NewNixBuildSpec(imageName string, contextDirPath string, nixFlakeDir string, flakeOutput string) *NixBuildSpec { internalNixBuildSpec := &privateNixBuildSpec{ NixFlakeDir: nixFlakeDir, ContextDirPath: contextDirPath, FlakeOutput: flakeOutput, + ImageName: imageName, } return &NixBuildSpec{internalNixBuildSpec} } +func (nixBuildSpec *NixBuildSpec) GetImageName() string { + return nixBuildSpec.privateNixBuildSpec.ImageName +} + func (nixBuildSpec *NixBuildSpec) GetNixFlakeDir() string { return nixBuildSpec.privateNixBuildSpec.NixFlakeDir } @@ -40,6 +47,10 @@ func (nixBuildSpec *NixBuildSpec) GetFlakeOutput() string { return nixBuildSpec.privateNixBuildSpec.FlakeOutput } +func (nixBuildSpec *NixBuildSpec) GetFullFlakeReference() string { + return fmt.Sprintf("%s/.#%s", nixBuildSpec.privateNixBuildSpec.NixFlakeDir, nixBuildSpec.privateNixBuildSpec.FlakeOutput) +} + func (nixBuildSpec *NixBuildSpec) MarshalJSON() ([]byte, error) { return json.Marshal(nixBuildSpec.privateNixBuildSpec) } diff --git a/container-engine-lib/lib/backend_interface/objects/service/service_config_test.go b/container-engine-lib/lib/backend_interface/objects/service/service_config_test.go index db2632fa12..3f3df193f8 100644 --- a/container-engine-lib/lib/backend_interface/objects/service/service_config_test.go +++ b/container-engine-lib/lib/backend_interface/objects/service/service_config_test.go @@ -191,7 +191,7 @@ func testImageRegistrySpec() *image_registry_spec.ImageRegistrySpec { } func testNixBuildSpec() *nix_build_spec.NixBuildSpec { - return nix_build_spec.NewNixBuildSpec("test-image", "path", "") + return nix_build_spec.NewNixBuildSpec("test-image", "path", "", "") } func testServiceUser() *service_user.ServiceUser { diff --git a/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/nix_build_spec_framework_test.go b/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/nix_build_spec_framework_test.go index 9da723fa2e..f4c5909dd2 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/nix_build_spec_framework_test.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/nix_build_spec_framework_test.go @@ -39,12 +39,14 @@ func (suite *KurtosisTypeConstructorTestSuite) TestNixBuildSpecTest() { } func (t *nixBuildSpecTest) GetStarlarkCode() string { - return fmt.Sprintf("%s(%s=%q, %s=%q, %s=%q)", + return fmt.Sprintf("%s(%s=%q, %s=%q, %s=%q, %s=%q)", service_config.NixBuildSpecTypeName, service_config.FlakeLocationDir, testNixFlakeLocationDir, service_config.NixContextAttr, testNixContextDir, + service_config.NixImageName, + testNixImageName, service_config.FlakeOutputAttr, testNixFlakeOutput) } diff --git a/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/static_constants.go b/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/static_constants.go index c57d4e3f8c..d6cebcc8ca 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/static_constants.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/static_constants.go @@ -35,6 +35,7 @@ var ( testOnDiskContainerImagePath = "kurtosis-data/test-package/Dockerfile" testNixContextDir = "./" + testNixImageName = "test-image" testNixFlakeOutput = "foo" testNixFlakeLocationDir = "./server/app" testOnDiskNixContextDirPath = "kurtosis-data/test-package" diff --git a/core/server/api_container/server/startosis_engine/kurtosis_types/service_config/nix_build_spec.go b/core/server/api_container/server/startosis_engine/kurtosis_types/service_config/nix_build_spec.go index d23a312118..b9bbfcc9a1 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_types/service_config/nix_build_spec.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_types/service_config/nix_build_spec.go @@ -1,6 +1,7 @@ package service_config import ( + "fmt" "path" "path/filepath" @@ -20,6 +21,7 @@ const ( FlakeLocationDir = "flake_location_dir" FlakeOutputAttr = "flake_output" NixContextAttr = "build_context_dir" + NixImageName = "image_name" // Currently only supports container nixs named Dockerfile defaultNixFlakeFile = "flake.nix" @@ -46,6 +48,14 @@ func NewNixBuildSpecType() *kurtosis_type_constructor.KurtosisTypeConstructor { return builtin_argument.NonEmptyString(value, NixContextAttr) }, }, + { + Name: NixImageName, + IsOptional: false, + ZeroValueProvider: builtin_argument.ZeroValueProvider[starlark.String], + Validator: func(value starlark.Value) *startosis_errors.InterpretationError { + return builtin_argument.NonEmptyString(value, NixImageName) + }, + }, { Name: FlakeOutputAttr, IsOptional: true, @@ -110,15 +120,39 @@ func (nixBuildSpec *NixBuildSpec) GetFlakeOutput() (string, *startosis_errors.In return flakeOutput.GoString(), nil } +func (nixBuildSpec *NixBuildSpec) GetImageName() (string, *startosis_errors.InterpretationError) { + imageName, found, interpretationErr := kurtosis_type_constructor.ExtractAttrValue[starlark.String](nixBuildSpec.KurtosisValueTypeDefault, NixImageName) + if interpretationErr != nil { + return "", interpretationErr + } + if !found { + return "", nil + } + return imageName.GoString(), nil +} + func (nixBuildSpec *NixBuildSpec) GetFlakeLocationDir() (string, *startosis_errors.InterpretationError) { - flakeLocator, found, interpretationErr := kurtosis_type_constructor.ExtractAttrValue[starlark.String](nixBuildSpec.KurtosisValueTypeDefault, FlakeLocationDir) + flakeDir, found, interpretationErr := kurtosis_type_constructor.ExtractAttrValue[starlark.String](nixBuildSpec.KurtosisValueTypeDefault, FlakeLocationDir) if interpretationErr != nil { return "", interpretationErr } if !found { return "", nil } - return flakeLocator.GoString(), nil + return flakeDir.GoString(), nil +} + +func (nixBuildSpec *NixBuildSpec) GetFullFlakeReference() (string, *startosis_errors.InterpretationError) { + flakeDir, err := nixBuildSpec.GetFlakeLocationDir() + if err != nil { + return "", err + } + flakeAttr, err := nixBuildSpec.GetFlakeOutput() + if err != nil { + return "", err + } + fullLocator := fmt.Sprintf("%s/.#%s", flakeDir, flakeAttr) + return fullLocator, nil } func (nixBuildSpec *NixBuildSpec) ToKurtosisType( @@ -148,12 +182,17 @@ func (nixBuildSpec *NixBuildSpec) ToKurtosisType( return nil, interpretationErr } + imageName, interpretationErr := nixBuildSpec.GetImageName() + if interpretationErr != nil { + return nil, interpretationErr + } + flakeOutputStr, interpretationErr := nixBuildSpec.GetFlakeOutput() if interpretationErr != nil { return nil, interpretationErr } - return nix_build_spec.NewNixBuildSpec(buildContextDirPathOnDisk, flakeNixFilePathOnDisk, flakeOutputStr), nil + return nix_build_spec.NewNixBuildSpec(imageName, buildContextDirPathOnDisk, flakeNixFilePathOnDisk, flakeOutputStr), nil } // Returns the filepath of the build context directory and flake nix on APIC based on package info diff --git a/core/server/api_container/server/startosis_engine/kurtosis_types/service_config/service_config.go b/core/server/api_container/server/startosis_engine/kurtosis_types/service_config/service_config.go index b090298956..467bc51e00 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_types/service_config/service_config.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_types/service_config/service_config.go @@ -679,7 +679,7 @@ func convertImage( packageReplaceOptions map[string]string) (string, *image_build_spec.ImageBuildSpec, *image_registry_spec.ImageRegistrySpec, *nix_build_spec.NixBuildSpec, *startosis_errors.InterpretationError) { imageBuildSpecStarlarkType, isImageBuildSpecStarlarkType := image.(*ImageBuildSpec) imageRegistrySpecStarlarkType, isImageRegistrySpecStarlarkType := image.(*ImageRegistrySpec) - NixBuildSpecStarlarkType, isNixBuildSpecStarlarkType := image.(*NixBuildSpec) + nixBuildSpecStarlarkType, isNixBuildSpecStarlarkType := image.(*NixBuildSpec) if isImageBuildSpecStarlarkType { imageBuildSpec, interpretationErr := imageBuildSpecStarlarkType.ToKurtosisType(locatorOfModuleInWhichThisBuiltInIsBeingCalled, packageId, packageContentProvider, packageReplaceOptions) if interpretationErr != nil { @@ -697,11 +697,11 @@ func convertImage( } return imageRegistrySpec.GetImageName(), nil, imageRegistrySpec, nil, nil } else if isNixBuildSpecStarlarkType { - nixBuildSpec, interpretationErr := NixBuildSpecStarlarkType.ToKurtosisType(locatorOfModuleInWhichThisBuiltInIsBeingCalled, packageId, packageContentProvider, packageReplaceOptions) + nixBuildSpec, interpretationErr := nixBuildSpecStarlarkType.ToKurtosisType(locatorOfModuleInWhichThisBuiltInIsBeingCalled, packageId, packageContentProvider, packageReplaceOptions) if interpretationErr != nil { return "", nil, nil, nil, interpretationErr } - return nixBuildSpec.GetFlakeOutput(), nil, nil, nixBuildSpec, nil + return nixBuildSpec.GetImageName(), nil, nil, nixBuildSpec, nil } else { imageName, interpretationErr := kurtosis_types.SafeCastToString(image, ImageAttr) if interpretationErr != nil { diff --git a/core/server/api_container/server/startosis_engine/startosis_validator/images_validator.go b/core/server/api_container/server/startosis_engine/startosis_validator/images_validator.go index dbe74dce91..4934fd989f 100644 --- a/core/server/api_container/server/startosis_engine/startosis_validator/images_validator.go +++ b/core/server/api_container/server/startosis_engine/startosis_validator/images_validator.go @@ -133,25 +133,26 @@ func (validator *ImagesValidator) nixBuildUsingBackend( buildErrors chan<- error, nixBuildStarted chan<- string, nixBuildFinished chan<- *ValidatedImage) { + imageRef := nixBuildSpec.GetFullFlakeReference() + logrus.Debugf("Requesting the build of image: '%s'", imageRef) var imageName string - logrus.Debugf("Requesting the build of image: '%s'", imageName) var imageArch string imageBuiltLocally := true imagePulledFromRemote := false defer wg.Done() imageCurrentlyBuilding <- true - nixBuildStarted <- imageName + nixBuildStarted <- imageRef defer func() { <-imageCurrentlyBuilding nixBuildFinished <- NewValidatedImage(imageName, imagePulledFromRemote, imageBuiltLocally, imageArch) }() - logrus.Debugf("Starting the build of image: '%s'", imageName) + logrus.Debugf("Starting the build of image: '%s'", imageRef) imageName, err := (*backend).NixBuild(ctx, nixBuildSpec) if err != nil { - logrus.Warnf("Container image '%s' build failed. Error was: '%s'", imageName, err.Error()) - buildErrors <- startosis_errors.WrapWithValidationError(err, "Failed to build the required image '%v'.", imageName) + logrus.Warnf("Container image '%s' build failed. Error was: '%s'", imageRef, err.Error()) + buildErrors <- startosis_errors.WrapWithValidationError(err, "Failed to build the required image '%v'.", imageRef) return } - logrus.Debugf("Container image '%s' successfully built", imageName) + logrus.Debugf("Container image '%s' successfully built from Nix definition %s", imageName, imageRef) } From 804b0a6c450c8dc19fd15ddb081ed8f843942b45 Mon Sep 17 00:00:00 2001 From: lostbean Date: Wed, 7 Feb 2024 18:25:52 -0300 Subject: [PATCH 10/15] clean-up and lint --- cli/cli/commands/import/import.go | 10 +-- .../docker/docker_manager/docker_manager.go | 68 ++++++++----------- .../engine_functions/create_engine.go | 3 + ...urtosis_backend_api_container_functions.go | 10 ++- .../start_user_services.go | 6 +- .../docker_compose_transpiler.go | 12 ++-- enclave-manager/server/server.go | 2 + .../server/engine/streaming/websocket_pump.go | 1 + 8 files changed, 61 insertions(+), 51 deletions(-) diff --git a/cli/cli/commands/import/import.go b/cli/cli/commands/import/import.go index 13583d192f..6058d95cc5 100644 --- a/cli/cli/commands/import/import.go +++ b/cli/cli/commands/import/import.go @@ -3,6 +3,11 @@ package _import import ( "context" "fmt" + "os" + "path/filepath" + "strconv" + "strings" + "github.com/compose-spec/compose-go/loader" "github.com/compose-spec/compose-go/types" "github.com/joho/godotenv" @@ -25,10 +30,6 @@ import ( "github.com/kurtosis-tech/kurtosis/name_generator" "github.com/kurtosis-tech/stacktrace" "github.com/sirupsen/logrus" - "os" - "path/filepath" - "strconv" - "strings" ) const ( @@ -175,6 +176,7 @@ func run( func convertComposeFileToStarlark(path string, dotEnvMap map[string]string) (string, map[string]string, error) { project, err := loader.Load(types.ConfigDetails{ //nolint:exhaustruct + // nolint: exhaustruct ConfigFiles: []types.ConfigFile{{Filename: path}}, Environment: dotEnvMap, }) diff --git a/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go b/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go index 5a3a4f2c22..8cc7327b25 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go @@ -11,10 +11,10 @@ import ( "bytes" "compress/gzip" "context" + "crypto/md5" "encoding/json" "fmt" "io" - "log" "math" "net" "os" @@ -159,6 +159,8 @@ const ( // Per https://github.com/hashicorp/waypoint/pull/1937/files buildkitSessionSharedKey = "" + + nixCmdPath = "/nix/var/nix/profiles/default/bin/nix" ) type RestartPolicy string @@ -1326,25 +1328,6 @@ type ImageManifest struct { RepoTags []string `json:"RepoTags"` Layers []string `json:"Layers"` } -type Manifest struct { - Images []ImageManifest -} - -// Add other fields if needed - -func readTarGz(filename string) ImageManifest { - cmd := exec.Command("tar", "-axf", filename, "manifest.json", "-O") - stdout, err := cmd.Output() - if err != nil { - log.Fatal(err) - } - var data ImageManifest - err = json.Unmarshal([]byte(stdout), &data) - if err != nil { - fmt.Println("error:", err) - } - return data -} // GetRepoTags extracts the RepoTags from a Docker image file func GetRepoTags(imageFilePath string) ([]string, error) { @@ -1371,7 +1354,6 @@ func GetRepoTags(imageFilePath string) ([]string, error) { if err != nil { return nil, stacktrace.Propagate(err, "Fail to read the image files") } - if header.Name == "manifest.json" { found = true break @@ -1388,27 +1370,29 @@ func GetRepoTags(imageFilePath string) ([]string, error) { return nil, stacktrace.Propagate(err, "Could not parse the manifest.json") } + if len(imageManifest) > 1 { + return nil, stacktrace.NewError("Image has more than 1 label/tag, don't know which one to pick: %v", imageManifest) + } else if len(imageManifest) < 1 { + return nil, stacktrace.NewError("Image has no label/tag") + } + return imageManifest[0].RepoTags, nil } func (manager *DockerManager) NixBuild(ctx context.Context, nixBuildSpec *nix_build_spec.NixBuildSpec) (string, error) { - getBuildContextDir := nixBuildSpec.GetBuildContextDir() - logrus.Warnf("Nix Build Dir: %s", getBuildContextDir) - flakeOutput := nixBuildSpec.GetFlakeOutput() - logrus.Warnf("Nix Flake Output: %s", flakeOutput) - flakeOutputFilePath := nixBuildSpec.GetNixFlakeDir() - logrus.Warnf("Nix Build Flake: %s", flakeOutputFilePath) + flakeReference := nixBuildSpec.GetFullFlakeReference() - entries, err := os.ReadDir(getBuildContextDir) - if err != nil { - logrus.Fatal(err) - } - - for _, e := range entries { - fmt.Println(e.Name()) - } + // Flake generates a link to the nix store containing the image result, to avoid collision with a possible existing one from the + // build context (from the user env) and which would result when trying to overwrite it, we create a unique one + hasher := md5.New() + hasher.Write([]byte(flakeReference)) + resultLink := fmt.Sprintf("nix-result-%x", hasher.Sum(nil)) - cmd := exec.Command("/nix/var/nix/profiles/default/bin/nix", "build", fmt.Sprintf("%s/.#%s", flakeOutputFilePath, flakeOutput), "--print-out-paths", "--extra-experimental-features", "flakes nix-command", "--out-link", "result-123") + cmd := exec.Command( + nixCmdPath, "build", flakeReference, + "--print-out-paths", + "--extra-experimental-features", "flakes nix-command", + "--out-link", resultLink) var errBuffer strings.Builder cmd.Stderr = &errBuffer @@ -1419,6 +1403,7 @@ func (manager *DockerManager) NixBuild(ctx context.Context, nixBuildSpec *nix_bu return "", stacktrace.Propagate(err, "Failed to build nix image with Nix.") } imageFile := strings.TrimSpace(string(imageFileRaw)) + logrus.Debugf("Nix flake image on attribute %s, result on image file %s", flakeReference, imageFile) imageName, err := GetRepoTags(imageFile) if err != nil { @@ -1426,8 +1411,15 @@ func (manager *DockerManager) NixBuild(ctx context.Context, nixBuildSpec *nix_bu } image, err := os.Open(imageFile) - imgRes, err := manager.dockerClient.ImageLoad(ctx, image, false) - logrus.Debugf("%v", imgRes) + if err != nil { + return "", stacktrace.Propagate(err, "Failed to open generated Nix image on %s", imageFile) + } + + _, err = manager.dockerClient.ImageLoad(ctx, image, false) + if err != nil { + return "", stacktrace.Propagate(err, "Failed to load Nix image %s in docker", imageFile) + } + logrus.Debugf("Nix generated image file %s is loaded into docker", imageFile) return imageName[0], nil } diff --git a/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/engine_functions/create_engine.go b/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/engine_functions/create_engine.go index ca132b4d1c..3c8f9e9207 100644 --- a/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/engine_functions/create_engine.go +++ b/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/engine_functions/create_engine.go @@ -321,6 +321,7 @@ func createEngineClusterRole( } clusterRoleName := clusterRolesAttributes.GetName().GetString() clusterRoleLabels := shared_helpers.GetStringMapFromLabelMap(clusterRolesAttributes.GetLabels()) + // nolint: exhaustruct clusterRolePolicyRules := []rbacv1.PolicyRule{ { Verbs: []string{ @@ -388,6 +389,7 @@ func createEngineClusterRoleBindings( } clusterRoleBindingsName := clusterRoleBindingsAttributes.GetName().GetString() clusterRoleBindingsLabels := shared_helpers.GetStringMapFromLabelMap(clusterRoleBindingsAttributes.GetLabels()) + // nolint: exhaustruct clusterRoleBindingsSubjects := []rbacv1.Subject{ { Kind: rbacv1.ServiceAccountKind, @@ -452,6 +454,7 @@ func createEnginePod( } engineContainerEnvVars = append(engineContainerEnvVars, envVar) } + // nolint: exhaustruct engineContainers := []apiv1.Container{ { Name: kurtosisEngineContainerName, diff --git a/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/kubernetes_kurtosis_backend_api_container_functions.go b/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/kubernetes_kurtosis_backend_api_container_functions.go index 31b7f861cb..0cfe1986f4 100644 --- a/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/kubernetes_kurtosis_backend_api_container_functions.go +++ b/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/kubernetes_kurtosis_backend_api_container_functions.go @@ -3,6 +3,9 @@ package kubernetes_kurtosis_backend import ( "context" "fmt" + "net" + "time" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/consts" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/shared_helpers" kubernetes_manager_consts "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_manager/consts" @@ -17,8 +20,6 @@ import ( apiv1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" applyconfigurationsv1 "k8s.io/client-go/applyconfigurations/core/v1" - "net" - "time" ) const ( @@ -233,6 +234,7 @@ func (backend *KubernetesKurtosisBackend) CreateAPIContainer( clusterRoleName := clusterRolesAttributes.GetName().GetString() clusterRoleLabels := shared_helpers.GetStringMapFromLabelMap(clusterRolesAttributes.GetLabels()) + // nolint: exhaustruct clusterRolePolicyRules := []rbacv1.PolicyRule{ { Verbs: []string{ @@ -292,6 +294,7 @@ func (backend *KubernetesKurtosisBackend) CreateAPIContainer( clusterRoleBindingName := clusterRoleBindingsAttributes.GetName().GetString() clusterRoleBindingsLabels := shared_helpers.GetStringMapFromLabelMap(clusterRoleBindingsAttributes.GetLabels()) + // nolint: exhaustruct clusterRoleBindingsSubjects := []rbacv1.Subject{ { Kind: rbacv1.ServiceAccountKind, @@ -335,6 +338,7 @@ func (backend *KubernetesKurtosisBackend) CreateAPIContainer( roleName := rolesAttributes.GetName().GetString() roleLabels := shared_helpers.GetStringMapFromLabelMap(rolesAttributes.GetLabels()) + // nolint: exhaustruct rolePolicyRules := []rbacv1.PolicyRule{ { Verbs: []string{ @@ -394,6 +398,7 @@ func (backend *KubernetesKurtosisBackend) CreateAPIContainer( roleBindingName := roleBindingsAttributes.GetName().GetString() roleBindingsLabels := shared_helpers.GetStringMapFromLabelMap(roleBindingsAttributes.GetLabels()) + // nolint: exhaustruct roleBindingsSubjects := []rbacv1.Subject{ { Kind: rbacv1.ServiceAccountKind, @@ -1085,6 +1090,7 @@ func getApiContainerContainersAndVolumes( } containerEnvVars = append(containerEnvVars, ownNamespaceEnvVar) + // nolint: exhaustruct containers := []apiv1.Container{ { Name: kurtosisApiContainerContainerName, diff --git a/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/user_services_functions/start_user_services.go b/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/user_services_functions/start_user_services.go index 42a473b848..5c4a2c97a2 100644 --- a/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/user_services_functions/start_user_services.go +++ b/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/user_services_functions/start_user_services.go @@ -3,9 +3,10 @@ package user_services_functions import ( "context" "fmt" - "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service_user" "strings" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service_user" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/consts" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/shared_helpers" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_manager" @@ -681,7 +682,7 @@ func getUserServicePodContainerSpecs( Limits: resourceLimitsList, Requests: resourceRequestsList, } - + // nolint: exhaustruct containers := []apiv1.Container{ { Name: userServiceContainerName, @@ -871,6 +872,7 @@ func createRegisterUserServiceOperation( // Kubernetes doesn't allow us to create services without any ports, so we need to set this to a notional value // until the user calls StartService + // nolint: exhaustruct notionalServicePorts := []apiv1.ServicePort{ { Name: unboundPortName, diff --git a/core/server/api_container/server/docker_compose_transpiler/docker_compose_transpiler.go b/core/server/api_container/server/docker_compose_transpiler/docker_compose_transpiler.go index 366fbd3eb8..b0cca993e4 100644 --- a/core/server/api_container/server/docker_compose_transpiler/docker_compose_transpiler.go +++ b/core/server/api_container/server/docker_compose_transpiler/docker_compose_transpiler.go @@ -3,6 +3,12 @@ package docker_compose_transpiler import ( "errors" "fmt" + "os" + "path" + "sort" + "strconv" + "strings" + "github.com/compose-spec/compose-go/loader" "github.com/compose-spec/compose-go/types" "github.com/joho/godotenv" @@ -15,11 +21,6 @@ import ( "github.com/kurtosis-tech/stacktrace" "github.com/sirupsen/logrus" "go.starlark.net/starlark" - "os" - "path" - "sort" - "strconv" - "strings" ) const ( @@ -122,6 +123,7 @@ func convertComposeToStarlarkScript(composeBytes []byte, envVars map[string]stri func convertComposeBytesToComposeStruct(composeBytes []byte, envVars map[string]string) (*types.Project, error) { composeParseConfig := types.ConfigDetails{ //nolint:exhaustruct // Note that we might be able to use the WorkingDir property instead, to parse the entire directory + // nolint: exhaustruct ConfigFiles: []types.ConfigFile{{ Content: composeBytes, }}, diff --git a/enclave-manager/server/server.go b/enclave-manager/server/server.go index 3b4dabb8ff..5952e7a188 100644 --- a/enclave-manager/server/server.go +++ b/enclave-manager/server/server.go @@ -204,6 +204,7 @@ func (c *WebServer) ListFilesArtifactNamesAndUuids(ctx context.Context, req *con return nil, stacktrace.Propagate(err, "Failed to create the APIC client") } + // nolint: exhaustruct serviceRequest := &connect.Request[emptypb.Empty]{} result, err := (*apiContainerServiceClient).ListFilesArtifactNamesAndUuids(ctx, serviceRequest) if err != nil { @@ -291,6 +292,7 @@ func (c *WebServer) DestroyEnclave(ctx context.Context, req *connect.Request[kur if err != nil { return nil, err } + // nolint: exhaustruct return &connect.Response[emptypb.Empty]{}, nil } diff --git a/engine/server/engine/streaming/websocket_pump.go b/engine/server/engine/streaming/websocket_pump.go index ede4a7efa1..f9d89d9121 100644 --- a/engine/server/engine/streaming/websocket_pump.go +++ b/engine/server/engine/streaming/websocket_pump.go @@ -52,6 +52,7 @@ func NewWebsocketPump[T interface{}](ctx echo.Context, cors cors.Cors) (*Websock ctxWithCancel, cancelFunc := context.WithCancel(context.Background()) + // nolint: exhaustruct pump := &WebsocketPump[T]{ websocket: conn, inputChan: make(chan *T), From b3be6e0675d6f99b7611245a35c78d65a204cf29 Mon Sep 17 00:00:00 2001 From: lostbean Date: Wed, 7 Feb 2024 19:41:24 -0300 Subject: [PATCH 11/15] add Nix support to docs --- .../starlark-reference/nix-support.md | 53 +++++++++++++++++++ .../starlark-reference/service-config.md | 22 ++++++++ 2 files changed, 75 insertions(+) create mode 100644 docs/docs/api-reference/starlark-reference/nix-support.md diff --git a/docs/docs/api-reference/starlark-reference/nix-support.md b/docs/docs/api-reference/starlark-reference/nix-support.md new file mode 100644 index 0000000000..08bc8f4c16 --- /dev/null +++ b/docs/docs/api-reference/starlark-reference/nix-support.md @@ -0,0 +1,53 @@ +--- +title: NixSupport +sidebar_label: NixSupport +--- + +You can provide Kurtosis just with source code and a Nix definition on how to build an image. Kurtosis will take care of building and deploying the image directly into the enclave without the need to upload or register the image beforehand. + +For that, we use Nix flakes, which is a way to package build definitions and dependencies in a reproducible manner. Using Nix flakes, you can define your system configurations and dependencies in a single file (`flake.nix`), making it easier to manage and share. + +Here's a basic explanation of how you can generate Docker images from services using Nix flakes: + +1. **Install Nix**: Installing Nix isn't strictly necessary with Kurtosis, but it's recommended if you are creating or developing the package. You can install it by following the instructions on the Nix website: [https://nixos.org/download.html](https://nixos.org/download.html) + +2. **Create a Nix Flake**: Create a directory for your project and initialize a Nix flake. You can do this by running: + ```bash + mkdir myproject + cd myproject + nix flake init -t simple + ``` + +3. **Define Your Services**: Inside the `flake.nix` file, you can define your services and their dependencies. For example: + ```nix + { + description = "My project"; + + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; + }; + + outputs = { self, nixpkgs, myservice }: { + defaultPackage.aarch64-darwin = nixpkgs.lib.dockerTools.buildImage { + name = "myservice"; + tag = "latest"; + contents = [ myservice ]; + config.Cmd = [ "myservice-binary" ]; + }; + }; + } + ``` + +4. **Add a service definition to Starlark**: Now just add a service to your Starlark configuration: + ```python + plan.add_service( + name = "nix-example", + config = ServiceConfig( + image = NixBuildSpec(image_name = "myservice", flake_location_dir = ".", build_context_dir = "./"), + ), + ) + ``` + +5. **Build and Deploy with Kurtosis**: From your package folder, simply run `kurtosis run .` to get your cluster up and running. + +This is just a basic example. Depending on your specific use case and requirements, you may need to adjust the configuration and dependencies in your `flake.nix` file accordingly. Additionally, you can add more services, configure networking, volumes, environment variables, etc., based on your needs. \ No newline at end of file diff --git a/docs/docs/api-reference/starlark-reference/service-config.md b/docs/docs/api-reference/starlark-reference/service-config.md index 13efc1046d..38bf857593 100644 --- a/docs/docs/api-reference/starlark-reference/service-config.md +++ b/docs/docs/api-reference/starlark-reference/service-config.md @@ -52,6 +52,26 @@ config = ServiceConfig( # The URL of the registry registry = "http://my.registry.io/" ) + + OR + + image = NixBuildSpec( + # The name of the image that needs to be pulled qualified with the registry + # MANDATORY + image_name = "hello-world-server", + + # Locator to build context within the Kurtosis package + # As of now, Kurtosis expects a Dockerfile at the root of the build context + # MANDATORY + build_context_dir = "./" + + # The relative path (from the `build_context_dir`) to the folder containing the flake.nix file + # MANDATORY + flake_location_dir = "./hello-go", + + # The selector for the Flake output with the image derivation. Fallbacks to the default package. + flake_output = "containerImage", + ) # The ports that the container should listen on, identified by a user-friendly ID that can be used to select the port again in the future. # If no ports are provided, no ports will be exposed on the host machine, unless there is an EXPOSE in the Dockerfile @@ -194,6 +214,8 @@ The `ports` dictionary argument accepts a key value pair, where `key` is a user The `files` dictionary argument accepts a key value pair, where `key` is the path where the contents of the artifact will be mounted to and `value` is a [Directory][directory] object or files artifact name. Using a `Directory` object with `artifact_name` is strictly equivalent to directly using the files artifact name as the value of the dictionary. This is just to simplify usage. +See [NixSupport](NixSupport) for more information on how to use the Nix and Kurtosis together. + You can view more information on [configuring the `ReadyCondition` type here][ready-condition]. :::tip From 5c51bc7728be9f0ac836a5cb649072ad05c2ac19 Mon Sep 17 00:00:00 2001 From: lostbean Date: Thu, 8 Feb 2024 07:47:19 -0300 Subject: [PATCH 12/15] fix doc link --- docs/docs/api-reference/starlark-reference/service-config.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/docs/api-reference/starlark-reference/service-config.md b/docs/docs/api-reference/starlark-reference/service-config.md index 38bf857593..221b6018bf 100644 --- a/docs/docs/api-reference/starlark-reference/service-config.md +++ b/docs/docs/api-reference/starlark-reference/service-config.md @@ -214,7 +214,7 @@ The `ports` dictionary argument accepts a key value pair, where `key` is a user The `files` dictionary argument accepts a key value pair, where `key` is the path where the contents of the artifact will be mounted to and `value` is a [Directory][directory] object or files artifact name. Using a `Directory` object with `artifact_name` is strictly equivalent to directly using the files artifact name as the value of the dictionary. This is just to simplify usage. -See [NixSupport](NixSupport) for more information on how to use the Nix and Kurtosis together. +See [Nix Support](nix-support) for more information on how to use the Nix and Kurtosis together. You can view more information on [configuring the `ReadyCondition` type here][ready-condition]. From c9d1026778ffd31820983a3081f2e3b00daec55a Mon Sep 17 00:00:00 2001 From: lostbean Date: Thu, 8 Feb 2024 08:13:32 -0300 Subject: [PATCH 13/15] another doc fix --- docs/docs/api-reference/starlark-reference/service-config.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/docs/api-reference/starlark-reference/service-config.md b/docs/docs/api-reference/starlark-reference/service-config.md index 221b6018bf..f9f7c8ec82 100644 --- a/docs/docs/api-reference/starlark-reference/service-config.md +++ b/docs/docs/api-reference/starlark-reference/service-config.md @@ -214,7 +214,7 @@ The `ports` dictionary argument accepts a key value pair, where `key` is a user The `files` dictionary argument accepts a key value pair, where `key` is the path where the contents of the artifact will be mounted to and `value` is a [Directory][directory] object or files artifact name. Using a `Directory` object with `artifact_name` is strictly equivalent to directly using the files artifact name as the value of the dictionary. This is just to simplify usage. -See [Nix Support](nix-support) for more information on how to use the Nix and Kurtosis together. +See [Nix Support][nix-support] for more information on how to use the Nix and Kurtosis together. You can view more information on [configuring the `ReadyCondition` type here][ready-condition]. From 30602feabe2ed547cc82e68ae707944957f401c6 Mon Sep 17 00:00:00 2001 From: lostbean Date: Thu, 8 Feb 2024 08:19:45 -0300 Subject: [PATCH 14/15] add doc ref --- docs/docs/api-reference/starlark-reference/service-config.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/docs/api-reference/starlark-reference/service-config.md b/docs/docs/api-reference/starlark-reference/service-config.md index f9f7c8ec82..7027a5f20d 100644 --- a/docs/docs/api-reference/starlark-reference/service-config.md +++ b/docs/docs/api-reference/starlark-reference/service-config.md @@ -267,3 +267,4 @@ The `tolerations` field expects a list of [`Toleration`][toleration] objects bei [package]: ../../advanced-concepts/packages.md [user]: ./user.md [toleration]: ./toleration.md +[nix-support]: ./nix-support.md From 1d23a800e37f7f027c4a7b15d66283ef0561d7eb Mon Sep 17 00:00:00 2001 From: lostbean Date: Fri, 9 Feb 2024 12:54:40 -0300 Subject: [PATCH 15/15] Apply improvements from code review --- .../docker/docker_manager/docker_manager.go | 73 +++---------------- .../kubernetes_kurtosis_backend.go | 2 +- .../lib/image_utils/image_utils.go | 72 ++++++++++++++++++ .../service_config/nix_build_spec.go | 6 +- .../starlark-reference/nix-support.md | 3 +- .../starlark-reference/service-config.md | 2 +- 6 files changed, 89 insertions(+), 69 deletions(-) create mode 100644 container-engine-lib/lib/image_utils/image_utils.go diff --git a/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go b/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go index 8cc7327b25..cdcf06fac3 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go @@ -6,10 +6,8 @@ package docker_manager import ( - "archive/tar" "bufio" "bytes" - "compress/gzip" "context" "crypto/md5" "encoding/json" @@ -29,6 +27,7 @@ import ( "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/image_build_spec" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/image_registry_spec" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/nix_build_spec" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/image_utils" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/uuid_generator" "github.com/kurtosis-tech/kurtosis/utils" @@ -1322,63 +1321,6 @@ func (manager *DockerManager) FetchImage(ctx context.Context, image string, regi return pulledFromRemote, imageArchitecture, nil } -// ImageManifest represents the structure of the manifest.json file -type ImageManifest struct { - Config string `json:"Config"` - RepoTags []string `json:"RepoTags"` - Layers []string `json:"Layers"` -} - -// GetRepoTags extracts the RepoTags from a Docker image file -func GetRepoTags(imageFilePath string) ([]string, error) { - imageFile, err := os.Open(imageFilePath) - if err != nil { - return nil, stacktrace.Propagate(err, "Fail to open image file %s", imageFilePath) - } - defer imageFile.Close() - - gzipReader, err := gzip.NewReader(imageFile) - if err != nil { - return nil, stacktrace.Propagate(err, "Fail to ungzip image file %s", stacktrace.Propagate(err, "Fail to read the image files")) - } - defer gzipReader.Close() - - tarReader := tar.NewReader(gzipReader) - - var found bool = false - for { - header, err := tarReader.Next() - if err == io.EOF { - break - } - if err != nil { - return nil, stacktrace.Propagate(err, "Fail to read the image files") - } - if header.Name == "manifest.json" { - found = true - break - } - } - - if !found { - return nil, stacktrace.NewError("manifest.json not found in the image") - } - - var imageManifest []ImageManifest - jsonDecoder := json.NewDecoder(tarReader) - if err := jsonDecoder.Decode(&imageManifest); err != nil { - return nil, stacktrace.Propagate(err, "Could not parse the manifest.json") - } - - if len(imageManifest) > 1 { - return nil, stacktrace.NewError("Image has more than 1 label/tag, don't know which one to pick: %v", imageManifest) - } else if len(imageManifest) < 1 { - return nil, stacktrace.NewError("Image has no label/tag") - } - - return imageManifest[0].RepoTags, nil -} - func (manager *DockerManager) NixBuild(ctx context.Context, nixBuildSpec *nix_build_spec.NixBuildSpec) (string, error) { flakeReference := nixBuildSpec.GetFullFlakeReference() @@ -1405,10 +1347,17 @@ func (manager *DockerManager) NixBuild(ctx context.Context, nixBuildSpec *nix_bu imageFile := strings.TrimSpace(string(imageFileRaw)) logrus.Debugf("Nix flake image on attribute %s, result on image file %s", flakeReference, imageFile) - imageName, err := GetRepoTags(imageFile) + imageTags, err := image_utils.GetRepoTags(imageFile) if err != nil { - return "", err + return "", stacktrace.Propagate(err, "Failed to get image tags from Nix image %s", imageFile) + } + + if len(imageTags) == 0 { + return "", stacktrace.NewError("Generated image %s did not have any tags", imageFile) + } else if len(imageTags) > 1 { + logrus.Warnf("Generated image %s had multiple tags: %v. We'll select the first.", imageFile, imageTags) } + imageTag := imageTags[0] image, err := os.Open(imageFile) if err != nil { @@ -1421,7 +1370,7 @@ func (manager *DockerManager) NixBuild(ctx context.Context, nixBuildSpec *nix_bu } logrus.Debugf("Nix generated image file %s is loaded into docker", imageFile) - return imageName[0], nil + return imageTag, nil } func (manager *DockerManager) BuildImage(ctx context.Context, imageName string, imageBuildSpec *image_build_spec.ImageBuildSpec) (string, error) { diff --git a/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/kubernetes_kurtosis_backend.go b/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/kubernetes_kurtosis_backend.go index b096a5339b..8f835e4e7e 100644 --- a/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/kubernetes_kurtosis_backend.go +++ b/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/kubernetes_kurtosis_backend.go @@ -491,7 +491,7 @@ func (backend *KubernetesKurtosisBackend) BuildImage(ctx context.Context, imageN func (backend *KubernetesKurtosisBackend) NixBuild(ctx context.Context, nixBuildSpec *nix_build_spec.NixBuildSpec) (string, error) { // TODO IMPLEMENT - return "", stacktrace.NewError("Loading images isn't yet implemented in Kubernetes.") + return "", stacktrace.NewError("Nix image building isn't yet implemented in Kubernetes.") } // ==================================================================================================== diff --git a/container-engine-lib/lib/image_utils/image_utils.go b/container-engine-lib/lib/image_utils/image_utils.go new file mode 100644 index 0000000000..5b9224df71 --- /dev/null +++ b/container-engine-lib/lib/image_utils/image_utils.go @@ -0,0 +1,72 @@ +package image_utils + +import ( + "archive/tar" + "compress/gzip" + "encoding/json" + "io" + "os" + + "github.com/kurtosis-tech/stacktrace" +) + +const ( + manifestFileName = "manifest.json" +) + +// ImageManifest represents the structure of the manifest.json file +type ImageManifest struct { + Config string `json:"Config"` + RepoTags []string `json:"RepoTags"` + Layers []string `json:"Layers"` +} + +// GetRepoTags extracts the RepoTags from a Docker image file +func GetRepoTags(imageFilePath string) ([]string, error) { + imageFile, err := os.Open(imageFilePath) + if err != nil { + return nil, stacktrace.Propagate(err, "Fail to open image file %s", imageFilePath) + } + defer imageFile.Close() + + gzipReader, err := gzip.NewReader(imageFile) + if err != nil { + return nil, stacktrace.Propagate(err, "Fail to ungzip image file %s", stacktrace.Propagate(err, "Fail to read the image files")) + } + defer gzipReader.Close() + + tarReader := tar.NewReader(gzipReader) + + var found bool = false + for { + header, err := tarReader.Next() + if err == io.EOF { + break + } + if err != nil { + return nil, stacktrace.Propagate(err, "Fail to read the image files") + } + if header.Name == manifestFileName { + found = true + break + } + } + + if !found { + return nil, stacktrace.NewError("manifest.json not found in the image") + } + + var imageManifest []ImageManifest + jsonDecoder := json.NewDecoder(tarReader) + if err := jsonDecoder.Decode(&imageManifest); err != nil { + return nil, stacktrace.Propagate(err, "Could not parse the manifest.json") + } + + if len(imageManifest) > 1 { + return nil, stacktrace.NewError("Image has more than 1 label/tag, don't know which one to pick: %v", imageManifest) + } else if len(imageManifest) < 1 { + return nil, stacktrace.NewError("Image has no label/tag") + } + + return imageManifest[0].RepoTags, nil +} diff --git a/core/server/api_container/server/startosis_engine/kurtosis_types/service_config/nix_build_spec.go b/core/server/api_container/server/startosis_engine/kurtosis_types/service_config/nix_build_spec.go index b9bbfcc9a1..5e82f085e5 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_types/service_config/nix_build_spec.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_types/service_config/nix_build_spec.go @@ -23,7 +23,7 @@ const ( NixContextAttr = "build_context_dir" NixImageName = "image_name" - // Currently only supports container nixs named Dockerfile + // Currently only supports container nix flakes defaultNixFlakeFile = "flake.nix" ) @@ -226,7 +226,7 @@ func getOnDiskNixBuildSpecPaths( return "", "", interpretationErr } // Assume, that flake nix sits at the same level as context directory to get context dir path on disk - FlakeDirOnDisk := filepath.Dir(flakeNixPathOnDisk) + flakeDirOnDisk := filepath.Dir(flakeNixPathOnDisk) - return contextDirOnDisk, FlakeDirOnDisk, nil + return contextDirOnDisk, flakeDirOnDisk, nil } diff --git a/docs/docs/api-reference/starlark-reference/nix-support.md b/docs/docs/api-reference/starlark-reference/nix-support.md index 08bc8f4c16..19443d7f23 100644 --- a/docs/docs/api-reference/starlark-reference/nix-support.md +++ b/docs/docs/api-reference/starlark-reference/nix-support.md @@ -11,9 +11,8 @@ Here's a basic explanation of how you can generate Docker images from services u 1. **Install Nix**: Installing Nix isn't strictly necessary with Kurtosis, but it's recommended if you are creating or developing the package. You can install it by following the instructions on the Nix website: [https://nixos.org/download.html](https://nixos.org/download.html) -2. **Create a Nix Flake**: Create a directory for your project and initialize a Nix flake. You can do this by running: +2. **Create a Nix Flake**: Go to your project root directory and initialize a Nix flake. You can do this by running: ```bash - mkdir myproject cd myproject nix flake init -t simple ``` diff --git a/docs/docs/api-reference/starlark-reference/service-config.md b/docs/docs/api-reference/starlark-reference/service-config.md index 7027a5f20d..b2dab77535 100644 --- a/docs/docs/api-reference/starlark-reference/service-config.md +++ b/docs/docs/api-reference/starlark-reference/service-config.md @@ -61,7 +61,7 @@ config = ServiceConfig( image_name = "hello-world-server", # Locator to build context within the Kurtosis package - # As of now, Kurtosis expects a Dockerfile at the root of the build context + # This allows to select a sub-package where the context is going be used to build the image # MANDATORY build_context_dir = "./"