From 59ba36d6bfbff676cd56deeff5d6e0ce7fe75b62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marijan=20Petri=C4=8Devi=C4=87?= Date: Sat, 25 May 2024 17:57:09 +0200 Subject: [PATCH 01/31] fix: add enclave-manager/local to workspace --- enclave-manager/local/go.mod | 15 +++++++++++++++ enclave-manager/local/go.sum | 15 +++++++++++++++ go.work | 1 + 3 files changed, 31 insertions(+) create mode 100644 enclave-manager/local/go.mod create mode 100644 enclave-manager/local/go.sum diff --git a/enclave-manager/local/go.mod b/enclave-manager/local/go.mod new file mode 100644 index 0000000000..0d00c4cf4a --- /dev/null +++ b/enclave-manager/local/go.mod @@ -0,0 +1,15 @@ +module github.com/kurtosis-tech/kurtosis/enclave-manager/local + +go 1.20 +replace ( + github.com/kurtosis-tech/kurtosis/enclave-manager => ../server + github.com/kurtosis-tech/kurtosis/contexts-config-store => ../../contexts-config-store + github.com/kurtosis-tech/kurtosis/kurtosis_version => ../../kurtosis_version +) + +require ( + github.com/kurtosis-tech/kurtosis/enclave-manager v0.0.0-20230828153722-32770ca96513 + github.com/sirupsen/logrus v1.9.3 +) + +require golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 // indirect diff --git a/enclave-manager/local/go.sum b/enclave-manager/local/go.sum new file mode 100644 index 0000000000..21f9bfb372 --- /dev/null +++ b/enclave-manager/local/go.sum @@ -0,0 +1,15 @@ +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 h1:0A+M6Uqn+Eje4kHMK80dtF3JCXC4ykBgQG4Fe06QRhQ= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/go.work b/go.work index 318f912450..9a20f823bb 100644 --- a/go.work +++ b/go.work @@ -10,6 +10,7 @@ use ( ./core/launcher ./core/server ./enclave-manager/api/golang + ./enclave-manager/local ./enclave-manager/server ./engine/launcher ./engine/server From bc0e31cdb51f3375468ecf84360b20288481cb36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marijan=20Petri=C4=8Devi=C4=87?= Date: Sat, 25 May 2024 18:01:43 +0200 Subject: [PATCH 02/31] feat: package kurtosis with nix --- flake.lock | 6 +++--- flake.nix | 13 ++++++++++--- package.nix | 54 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 67 insertions(+), 6 deletions(-) create mode 100644 package.nix diff --git a/flake.lock b/flake.lock index 60e9d3d49c..9c05eae1a9 100644 --- a/flake.lock +++ b/flake.lock @@ -58,11 +58,11 @@ }, "unstable": { "locked": { - "lastModified": 1706371002, - "narHash": "sha256-dwuorKimqSYgyu8Cw6ncKhyQjUDOyuXoxDTVmAXq88s=", + "lastModified": 1716509168, + "narHash": "sha256-4zSIhSRRIoEBwjbPm3YiGtbd8HDWzFxJjw5DYSDy1n8=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "c002c6aa977ad22c60398daaa9be52f2203d0006", + "rev": "bfb7a882678e518398ce9a31a881538679f6f092", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index ef4d08fceb..cb00960565 100644 --- a/flake.nix +++ b/flake.nix @@ -7,7 +7,7 @@ flake-utils.url = "github:numtide/flake-utils"; }; - outputs = { nixpkgs, unstable, flake-utils, ... }: + outputs = { self, nixpkgs, unstable, flake-utils, ... }: let utils = flake-utils; in utils.lib.eachDefaultSystem (system: let @@ -17,9 +17,15 @@ inherit pkgs system; nodejs = pkgs.nodejs_20; }; - in { + in + { formatter = pkgs.nixpkgs-fmt; + packages = rec { + default = kurtosis; + kurtosis = unstable_pkgs.callPackage ./package.nix { }; + }; + devShell = pkgs.mkShell { nativeBuildInputs = with pkgs; let @@ -27,7 +33,8 @@ import ./nix-pkgs/openapi-codegen.nix { inherit pkgs; }; grpc-tools-node = import ./nix-pkgs/grpc-tools-node.nix { inherit pkgs; }; - in [ + in + [ goreleaser go_1_20 gopls diff --git a/package.nix b/package.nix new file mode 100644 index 0000000000..92aead7926 --- /dev/null +++ b/package.nix @@ -0,0 +1,54 @@ +{ lib +, buildGoModule +, writeTextFile +}: + +buildGoModule rec { + pname = "kurtosis"; + version = "0.89.11"; + + src = ./.; + + proxyVendor = true; + vendorHash = "sha256-GaEIitoRiuYxtS7cDKobFyIlraDNQjcvbRvzG3nUKFU="; + + postPatch = + let + kurtosisVersion = writeTextFile { + name = "kurtosis_verion.go"; + text = '' + package kurtosis_version + const ( + KurtosisVersion = "${version}" + ) + ''; + }; + in + '' + ln -s ${kurtosisVersion} kurtosis_version/kurtosis_version.go + ''; + + # disable checks temporarily since they connect to the internet + # namely user_support_constants_test.go + doCheck = false; + + # keep this for future reference + preCheck = '' + # some tests in commands use XDG home related environment variables + export HOME=/tmp + ''; + + postInstall = '' + mv $out/bin/cli $out/bin/kurtosis + mv $out/bin/files_artifacts_expander $out/bin/files-artifacts-expander + mv $out/bin/api_container $out/bin/api-container + + ''; + + meta = with lib; { + description = "A platform for launching an ephemeral Ethereum backend"; + mainProgram = "kurtosis"; + homepage = "https://github.com/kurtosis-tech/kurtosis"; + license = licenses.asl20; + }; +} From 41bd2dc103c0aa00504af884e8dc4d13f586be30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marijan=20Petri=C4=8Devi=C4=87?= Date: Sat, 25 May 2024 18:02:00 +0200 Subject: [PATCH 03/31] feat: ignore nix related files --- .gitignore | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 21b3b83910..7c9666618d 100644 --- a/.gitignore +++ b/.gitignore @@ -131,4 +131,8 @@ lerna-debug.log # contains several libraries that get shipped yarn.lock package.json -package-lock.json \ No newline at end of file +package-lock.json + +# Nix +# ignores the default result symlink created when building with nix +result From 744e330693ca37ed4ecd29c1ae86d7a8eff019fc Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Fri, 12 Jul 2024 15:17:47 -0400 Subject: [PATCH 04/31] use podman socket --- .../backend_creator/backend_creator.go | 5 +++-- .../docker/docker_kurtosis_backend/consts/consts.go | 7 +++++-- .../implementations/traefik/consts.go | 2 +- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/backend_creator/backend_creator.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/backend_creator/backend_creator.go index 86d787d599..234cedff40 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/backend_creator/backend_creator.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/backend_creator/backend_creator.go @@ -3,11 +3,12 @@ package backend_creator import ( "context" "fmt" - "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/database_accessors/enclave_db" "net" "os" "path" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/database_accessors/enclave_db" + "github.com/docker/docker/client" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_collector_functions" @@ -27,7 +28,7 @@ import ( const ( unixSocketPrefix = "unix://" - systemDaemonSocket = "/var/run/docker.sock" + systemDaemonSocket = "/var/run/podman/podman.sock" userOwnDaemonSocket = "/.docker/run/docker.sock" noTempDirPrefix = "" diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/consts/consts.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/consts/consts.go index a13632c54e..2da615c2bd 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/consts/consts.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/consts/consts.go @@ -22,14 +22,17 @@ const ( EngineTransportProtocol = port_spec.TransportProtocol_TCP // This needs to be bind-mounted into the engine & API containers so they can manipulate Docker - DockerSocketFilepath = "/var/run/docker.sock" + DockerSocketFilepath = "/var/run/podman/podman.sock" // The host engine config directory to mount and its local mapping HostEngineConfigDirToMount = "/root/engine_config" EngineConfigLocalDir = "/run/engine" //The Docker network name where all the containers in the engine and logs service context will be added - NameOfNetworkToStartEngineAndLogServiceContainersIn = "bridge" + // NameOfNetworkToStartEngineAndLogServiceContainersIn = "bridge" + // HttpApplicationProtocol = "http" + // The Docker network name where all the containers in the engine and logs service context will be added + NameOfNetworkToStartEngineAndLogServiceContainersIn = "podman" HttpApplicationProtocol = "http" GitHubAuthStorageDirPath = "/kurtosis-data/github-auth/" diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/implementations/traefik/consts.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/implementations/traefik/consts.go index 7df8087c81..bf2b63fc56 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/implementations/traefik/consts.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/implementations/traefik/consts.go @@ -28,7 +28,7 @@ entryPoints: providers: docker: - endpoint: "unix:///var/run/docker.sock" + endpoint: "unix:///var/run/podman/podman.sock" exposedByDefault: false network: "{{ .NetworkId }}" ` From 70fa36ae0f167e3d27badcfe5fac440dbd14f248 Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Fri, 12 Jul 2024 15:19:29 -0400 Subject: [PATCH 05/31] turn off log collector --- .../start_user_services.go | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/user_services_functions/start_user_services.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/user_services_functions/start_user_services.go index 371937bbf6..ce2759b592 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/user_services_functions/start_user_services.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/user_services_functions/start_user_services.go @@ -669,15 +669,15 @@ func createStartServiceOperation( } } - if logsCollectorAddress == "" { - return nil, stacktrace.NewError("Expected to have a logs collector server address value to send the user service logs, but it is empty") - } + // if logsCollectorAddress == "" { + // return nil, stacktrace.NewError("Expected to have a logs collector server address value to send the user service logs, but it is empty") + // } - // The container will be configured to send the logs to the Fluentbit logs collector server - fluentdLoggingDriverCnfg := docker_manager.NewFluentdLoggingDriver( - logsCollectorAddress, - logsCollectorLabels, - ) + // // The container will be configured to send the logs to the Fluentbit logs collector server + // fluentdLoggingDriverCnfg := docker_manager.NewFluentdLoggingDriver( + // logsCollectorAddress, + // logsCollectorLabels, + // ) createAndStartArgsBuilder := docker_manager.NewCreateAndStartContainerArgsBuilder( containerImageName, @@ -703,8 +703,8 @@ func createStartServiceOperation( tiniEnabled, ).WithVolumeMounts( volumeMounts, - ).WithLoggingDriver( - fluentdLoggingDriverCnfg, + // ).WithLoggingDriver( + // fluentdLoggingDriverCnfg, ).WithRestartPolicy( restartPolicy, ).WithUser( From df42701d46c537543825085da082de4ce45c51e0 Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Fri, 12 Jul 2024 15:21:15 -0400 Subject: [PATCH 06/31] fix volume inconsistency match --- .../docker/docker_manager/docker_manager.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go b/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go index bd4e771c04..7949a88621 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go @@ -481,6 +481,21 @@ func (manager *DockerManager) GetVolumesByLabels(ctx context.Context, labels map result := []*volume.Volume{} if resp.Volumes != nil { + // Podman API inconsistency - filter out the union matches that podman returns while docker only returns the intersect matches when filtering by label + for _, vol := range resp.Volumes { + allLabelsMatch := true + + for label, val := range labels { + if volValue, exists := vol.Labels[label]; !exists || volValue != val { + allLabelsMatch = false + break + } + } + + if allLabelsMatch { + result = append(result, vol) + } + } result = resp.Volumes } From 68c13874d27d88c51d05b91e2145b0ecd0c382f1 Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Fri, 12 Jul 2024 15:22:36 -0400 Subject: [PATCH 07/31] retrieve network id --- .../docker/docker_manager/docker_manager.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go b/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go index 7949a88621..73fee107ba 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go @@ -819,8 +819,17 @@ func (manager *DockerManager) GetContainerIps(ctx context.Context, containerId s return nil, stacktrace.Propagate(err, "An error occurred inspecting container with ID '%v'", containerId) } allNetworkInfo := resp.NetworkSettings.Networks - for _, networkInfo := range allNetworkInfo { - containerIps[networkInfo.NetworkID] = networkInfo.IPAddress + // for _, networkInfo := range allNetworkInfo { + // containerIps[networkInfo.NetworkID] = networkInfo.IPAddress + // } + for networkKey, networkInfo := range allNetworkInfo { + // podman does not return the networkID properly and as such we need to make sure we get it. + network, err := manager.dockerClient.NetworkInspect(ctx, networkInfo.NetworkID, types.NetworkInspectOptions{}) + if err != nil { + return nil, stacktrace.Propagate(err, "An error occurred inspecting network: '%v'", networkKey) + } + + containerIps[network.ID] = networkInfo.IPAddress } return containerIps, nil } From e062bf17e5c20ae0598a58c641ef23d3243e0b7b Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Fri, 12 Jul 2024 15:23:09 -0400 Subject: [PATCH 08/31] change host ip --- .../lib/backend_impls/docker/docker_manager/docker_manager.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go b/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go index 73fee107ba..1779f8eedc 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go @@ -1762,7 +1762,7 @@ func (manager *DockerManager) getContainerHostConfig( portMap[containerPort] = []nat.PortBinding{ // Leaving this struct empty will cause Docker to automatically choose an interface IP & port on the host machine { - HostIP: "", + HostIP: "0.0.0.0", HostPort: "", }, } From 15aa10a93f5cfe70e305e43cc9bcf233163e4599 Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Fri, 12 Jul 2024 15:24:47 -0400 Subject: [PATCH 09/31] add xdg fallback --- .../host_machine_directories.go | 22 +++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/cli/cli/helpers/host_machine_directories/host_machine_directories.go b/cli/cli/helpers/host_machine_directories/host_machine_directories.go index d78a53dbb9..3d210728be 100644 --- a/cli/cli/helpers/host_machine_directories/host_machine_directories.go +++ b/cli/cli/helpers/host_machine_directories/host_machine_directories.go @@ -1,9 +1,13 @@ package host_machine_directories import ( + "io/ioutil" + "path" + "path/filepath" + "github.com/adrg/xdg" "github.com/kurtosis-tech/stacktrace" - "path" + "github.com/sirupsen/logrus" ) const ( @@ -107,9 +111,19 @@ func GetLastPesteredUserAboutOldVersionsFilepath() (string, error) { func GetKurtosisCliLogsFileDirPath(fileName string) (string, error) { xdgRelDirPath := getRelativeFilePathForKurtosisCliLogs() - kurtosisCliLogFilePath, err := xdg.DataFile(path.Join(xdgRelDirPath, fileName)) - if err != nil { - return "", stacktrace.Propagate(err, "An error occurred getting the kurtosis cli logs file path using '%v'", kurtosisCliLogFilePath) + // kurtosisCliLogFilePath, err := xdg.DataFile(path.Join(xdgRelDirPath, fileName)) + // if err != nil { + // return "", stacktrace.Propagate(err, "An error occurred getting the kurtosis cli logs file path using '%v'", kurtosisCliLogFilePath) + // } + kurtosisCliLogFilePath, errXdg := xdg.DataFile(path.Join(xdgRelDirPath, fileName)) + if errXdg != nil { + // Fallback to temp folder if XDG fails to find a suitable location. For instance XDG will fail when testing inside Nix sandbox. + logrus.WithError(errXdg).Warnf("Couldn't create kurtosis cli logs file path in the user space '%v'. Trying on temp folder.", kurtosisCliLogFilePath) + kurtosisCliLogDir, errTemp := ioutil.TempDir("", applicationDirname) + if errTemp != nil { + return "", stacktrace.Propagate(errTemp, "An error occurred creating kurtosis cli logs file path using '%v'", kurtosisCliLogDir) + } + kurtosisCliLogFilePath = filepath.Join(kurtosisCliLogDir, fileName) } return kurtosisCliLogFilePath, nil } From 555fcb3b326d460fe8b7c563f7684a736af4b75c Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Fri, 12 Jul 2024 16:00:23 -0400 Subject: [PATCH 10/31] use local goreleaser --- cli/cli/scripts/build.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/cli/cli/scripts/build.sh b/cli/cli/scripts/build.sh index 0f8401c209..026a0de506 100755 --- a/cli/cli/scripts/build.sh +++ b/cli/cli/scripts/build.sh @@ -97,10 +97,11 @@ fi exit 1 fi # Executing goreleaser v1.26.2 without needing to install it - if ! curl -sfL https://goreleaser.com/static/run | VERSION=v1.26.2 DISTRIBUTION=oss bash -s -- ${goreleaser_verb_and_flags}; then - echo "Error: Couldn't build the CLI binary for the current OS/arch" >&2 - exit 1 - fi +# if ! curl -sfL https://goreleaser.com/static/run | VERSION=v1.26.2 DISTRIBUTION=oss bash -s -- ${goreleaser_verb_and_flags}; then + if ! GORELEASER_CURRENT_TAG=$(cat $root_dirpath/version.txt) goreleaser ${goreleaser_verb_and_flags}; then + echo "Error: Couldn't build the CLI binary for the current OS/arch" >&2 + exit 1 + fi ) # Final verification From 3e20f4c2012a0f6100bd3e01db15cbca6e769606 Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Fri, 12 Jul 2024 16:17:05 -0400 Subject: [PATCH 11/31] Revert "use podman socket" This reverts commit 744e330693ca37ed4ecd29c1ae86d7a8eff019fc. --- .../backend_creator/backend_creator.go | 5 ++--- .../docker/docker_kurtosis_backend/consts/consts.go | 7 ++----- .../implementations/traefik/consts.go | 2 +- 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/backend_creator/backend_creator.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/backend_creator/backend_creator.go index 234cedff40..86d787d599 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/backend_creator/backend_creator.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/backend_creator/backend_creator.go @@ -3,12 +3,11 @@ package backend_creator import ( "context" "fmt" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/database_accessors/enclave_db" "net" "os" "path" - "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/database_accessors/enclave_db" - "github.com/docker/docker/client" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_collector_functions" @@ -28,7 +27,7 @@ import ( const ( unixSocketPrefix = "unix://" - systemDaemonSocket = "/var/run/podman/podman.sock" + systemDaemonSocket = "/var/run/docker.sock" userOwnDaemonSocket = "/.docker/run/docker.sock" noTempDirPrefix = "" diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/consts/consts.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/consts/consts.go index 2da615c2bd..a13632c54e 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/consts/consts.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/consts/consts.go @@ -22,17 +22,14 @@ const ( EngineTransportProtocol = port_spec.TransportProtocol_TCP // This needs to be bind-mounted into the engine & API containers so they can manipulate Docker - DockerSocketFilepath = "/var/run/podman/podman.sock" + DockerSocketFilepath = "/var/run/docker.sock" // The host engine config directory to mount and its local mapping HostEngineConfigDirToMount = "/root/engine_config" EngineConfigLocalDir = "/run/engine" //The Docker network name where all the containers in the engine and logs service context will be added - // NameOfNetworkToStartEngineAndLogServiceContainersIn = "bridge" - // HttpApplicationProtocol = "http" - // The Docker network name where all the containers in the engine and logs service context will be added - NameOfNetworkToStartEngineAndLogServiceContainersIn = "podman" + NameOfNetworkToStartEngineAndLogServiceContainersIn = "bridge" HttpApplicationProtocol = "http" GitHubAuthStorageDirPath = "/kurtosis-data/github-auth/" diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/implementations/traefik/consts.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/implementations/traefik/consts.go index bf2b63fc56..7df8087c81 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/implementations/traefik/consts.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/implementations/traefik/consts.go @@ -28,7 +28,7 @@ entryPoints: providers: docker: - endpoint: "unix:///var/run/podman/podman.sock" + endpoint: "unix:///var/run/docker.sock" exposedByDefault: false network: "{{ .NetworkId }}" ` From f54369b7346ad650f587fb3a1285b044df217ebc Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Fri, 12 Jul 2024 17:21:27 -0400 Subject: [PATCH 12/31] fix get engine logs and components --- .../shared_helpers/shared_helpers.go | 39 ++++++++++--------- .../docker/docker_manager/docker_manager.go | 26 +++++++++++++ 2 files changed, 46 insertions(+), 19 deletions(-) diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/shared_helpers/shared_helpers.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/shared_helpers/shared_helpers.go index 18c3e533e2..ba9c236e66 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/shared_helpers/shared_helpers.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/shared_helpers/shared_helpers.go @@ -5,7 +5,6 @@ import ( "context" "encoding/json" "fmt" - "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/docker_label_key" "io" "net" "os" @@ -14,6 +13,8 @@ import ( "strings" "time" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/docker_label_key" + "github.com/docker/docker/pkg/stdcopy" "github.com/docker/go-connections/nat" "github.com/gammazero/workerpool" @@ -417,24 +418,24 @@ func GetEngineAndLogsComponentsNetwork( ctx context.Context, dockerManager *docker_manager.DockerManager, ) (*types.Network, error) { - matchingNetworks, err := dockerManager.GetNetworksByName(ctx, consts.NameOfNetworkToStartEngineAndLogServiceContainersIn) - if err != nil { - return nil, stacktrace.Propagate( - err, - "An error occurred getting networks matching the network we want to start the engine in, '%v'", - consts.NameOfNetworkToStartEngineAndLogServiceContainersIn, - ) - } - numMatchingNetworks := len(matchingNetworks) - if numMatchingNetworks == 0 && numMatchingNetworks > 1 { - return nil, stacktrace.NewError( - "Expected exactly one network matching the name of the network that we want to start the engine in, '%v', but got %v", - consts.NameOfNetworkToStartEngineAndLogServiceContainersIn, - numMatchingNetworks, - ) - } - targetNetwork := matchingNetworks[0] - return targetNetwork, nil + // matchingNetworks, err := dockerManager.GetNetworksByName(ctx, consts.NameOfNetworkToStartEngineAndLogServiceContainersIn) + // if err != nil { + // return nil, stacktrace.Propagate( + // err, + // "An error occurred getting networks matching the network we want to start the engine in, '%v'", + // consts.NameOfNetworkToStartEngineAndLogServiceContainersIn, + // ) + // } + // numMatchingNetworks := len(matchingNetworks) + // if numMatchingNetworks == 0 && numMatchingNetworks > 1 { + // return nil, stacktrace.NewError( + // "Expected exactly one network matching the name of the network that we want to start the engine in, '%v', but got %v", + // consts.NameOfNetworkToStartEngineAndLogServiceContainersIn, + // numMatchingNetworks, + // ) + // } + // targetNetwork := matchingNetworks[0] + return dockerManager.GetDefaultNetwork(ctx) } func DumpContainers(ctx context.Context, dockerManager *docker_manager.DockerManager, containers []*types.Container, outputDirpath string) error { diff --git a/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go b/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go index 1779f8eedc..1e5909af96 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go @@ -413,6 +413,32 @@ func (manager *DockerManager) RemoveNetwork(context context.Context, networkId s return nil } +func (manager *DockerManager) GetDefaultNetwork(ctx context.Context) (*docker_manager_types.Network, error) { + + matchingNetworks, err := manager.GetNetworksByName(ctx, consts.NameOfNetworkToStartEngineAndLogServiceContainersIn) + if err != nil { + return nil, stacktrace.Propagate( + err, + "An error occurred getting networks matching the network we want to start the engine in, '%v'", + consts.NameOfNetworkToStartEngineAndLogServiceContainersIn, + ) + } + numMatchingNetworks := len(matchingNetworks) + if numMatchingNetworks > 1 { + return nil, stacktrace.NewError( + "Expected exactly one network matching the name of the network that we want to start the engine in, '%v', but got %v", + consts.NameOfNetworkToStartEngineAndLogServiceContainersIn, + numMatchingNetworks, + ) + } + + if numMatchingNetworks == 0 { + return nil, stacktrace.NewError(fmt.Sprintf("No matching network found with the configured name: %v", consts.NameOfNetworkToStartEngineAndLogServiceContainersIn)) + } + + return matchingNetworks[0], nil +} + /* CreateVolume Creates a Docker volume identified by the given name. From e48fc234ea263a89aa4958d7ffbd6bd591008a7e Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Fri, 12 Jul 2024 17:32:46 -0400 Subject: [PATCH 13/31] bridge -> podman --- .../docker/docker_kurtosis_backend/consts/consts.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/consts/consts.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/consts/consts.go index a13632c54e..dc5816377b 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/consts/consts.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/consts/consts.go @@ -29,7 +29,7 @@ const ( EngineConfigLocalDir = "/run/engine" //The Docker network name where all the containers in the engine and logs service context will be added - NameOfNetworkToStartEngineAndLogServiceContainersIn = "bridge" + NameOfNetworkToStartEngineAndLogServiceContainersIn = "podman" HttpApplicationProtocol = "http" GitHubAuthStorageDirPath = "/kurtosis-data/github-auth/" From 9789f1cee6f4cd135d6b4acfee3d348a90c8ccda Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Thu, 25 Jul 2024 11:46:58 -0400 Subject: [PATCH 14/31] disable security opts --- .../docker/docker_manager/docker_manager.go | 38 +++++++++++-------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go b/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go index 1e5909af96..cd43767885 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go @@ -624,7 +624,8 @@ func (manager *DockerManager) CreateAndStartContainer( } containerHostConfigPtr, err := manager.getContainerHostConfig( args.addedCapabilities, - args.securityOpts, + //args.securityOpts, + args.networkMode, args.bindMounts, args.volumeMounts, @@ -1823,6 +1824,7 @@ func (manager *DockerManager) getContainerHostConfig( securityOptStr := string(securityOpt) securityOptsSlice = append(securityOptsSlice, securityOptStr) } + securityOptsSlice = append(securityOptsSlice, "") extraHosts := []string{} if needsToAccessDockerHostMachine { @@ -1925,21 +1927,25 @@ func (manager *DockerManager) getContainerHostConfig( Privileged: false, PublishAllPorts: false, ReadonlyRootfs: false, - SecurityOpt: securityOptsSlice, - StorageOpt: nil, - Tmpfs: nil, - UTSMode: "", - UsernsMode: "", - ShmSize: 0, - Sysctls: nil, - Runtime: "", - ConsoleSize: [2]uint{}, - Isolation: "", - Resources: resources, - Mounts: nil, - MaskedPaths: nil, - ReadonlyPaths: nil, - Init: &useInit, + //SecurityOpt: securityOptsSlice, + SecurityOpt: []string{ + "label=disable", // Disables SELinux + "apparmor:unconfined", // Disables AppArmor + }, + StorageOpt: nil, + Tmpfs: nil, + UTSMode: "", + UsernsMode: "", + ShmSize: 0, + Sysctls: nil, + Runtime: "", + ConsoleSize: [2]uint{}, + Isolation: "", + Resources: resources, + Mounts: nil, + MaskedPaths: nil, + ReadonlyPaths: nil, + Init: &useInit, } return containerHostConfigPtr, nil } From 4edfe23b263e121279b5cd10b771e85ff67be6c4 Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Thu, 25 Jul 2024 11:47:08 -0400 Subject: [PATCH 15/31] use podman for image building --- scripts/docker-image-builder.sh | 72 ++++++++++++++++----------------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/scripts/docker-image-builder.sh b/scripts/docker-image-builder.sh index 7c470f322f..ca47784529 100755 --- a/scripts/docker-image-builder.sh +++ b/scripts/docker-image-builder.sh @@ -70,33 +70,33 @@ done ## Start by making sure the builder and the context do not already exist. If that's the case remove them kurtosis_docker_builder="kurtosis-docker-builder" docker_buildx_context='kurtosis-docker-builder-context' -if docker buildx inspect "${kurtosis_docker_builder}" &>/dev/null; then - echo "Removing docker buildx builder ${kurtosis_docker_builder} as it seems to already exist" - if ! docker buildx rm ${kurtosis_docker_builder} &>/dev/null; then - echo "Failed removing docker buildx builder ${kurtosis_docker_builder}. Try removing it manually with 'docker buildx rm ${kurtosis_docker_builder}' before re-running this script" - exit 1 - fi -fi -if docker context inspect "${docker_buildx_context}" &>/dev/null; then - echo "Removing docker context ${docker_buildx_context} as it seems to already exist" - if ! docker context rm ${docker_buildx_context} &>/dev/null; then - echo "Failed removing docker context ${docker_buildx_context}. Try removing it manually with 'docker context rm ${docker_buildx_context}' before re-running this script" - exit 1 - fi -fi - -## Create Docker context and buildx builder -if ! docker context create "${docker_buildx_context}" &>/dev/null; then - echo "Error: Docker context creation for buildx failed" >&2 - exit 1 -fi -if ! docker buildx create --use --name "${kurtosis_docker_builder}" "${docker_buildx_context}" &>/dev/null; then - echo "Error: Docker context switch for buildx failed" >&2d - exit 1 -fi +#if docker buildx inspect "${kurtosis_docker_builder}" &>/dev/null; then +# echo "Removing docker buildx builder ${kurtosis_docker_builder} as it seems to already exist" +# if ! docker buildx rm ${kurtosis_docker_builder} &>/dev/null; then +# echo "Failed removing docker buildx builder ${kurtosis_docker_builder}. Try removing it manually with 'docker buildx rm ${kurtosis_docker_builder}' before re-running this script" +# exit 1 +# fi +#fi +#if docker context inspect "${docker_buildx_context}" &>/dev/null; then +# echo "Removing docker context ${docker_buildx_context} as it seems to already exist" +# if ! docker context rm ${docker_buildx_context} &>/dev/null; then +# echo "Failed removing docker context ${docker_buildx_context}. Try removing it manually with 'docker context rm ${docker_buildx_context}' before re-running this script" +# exit 1 +# fi +#fi +# +### Create Docker context and buildx builder +#if ! docker context create "${docker_buildx_context}" &>/dev/null; then +# echo "Error: Docker context creation for buildx failed" >&2 +# exit 1 +#fi +#if ! docker buildx create --use --name "${kurtosis_docker_builder}" "${docker_buildx_context}" &>/dev/null; then +# echo "Error: Docker context switch for buildx failed" >&2d +# exit 1 +#fi ## Actually build the Docker image -docker_buildx_cmd="docker buildx build ${push_flag} --platform ${buildx_platform_arg} ${image_tags_concatenated} -f ${dockerfile_filepath} ${dockerfile_dirpath}" +docker_buildx_cmd="sudo podman buildx build ${push_flag} --platform ${buildx_platform_arg} ${image_tags_concatenated} -f ${dockerfile_filepath} ${dockerfile_dirpath}" echo "Running the following docker buildx command:" echo "${docker_buildx_cmd}" if ! eval "${docker_buildx_cmd}"; then @@ -104,14 +104,14 @@ if ! eval "${docker_buildx_cmd}"; then exit 1 fi -# Cleanup context and buildx runner -echo "Cleaning up remaining resources" -if ! docker buildx rm "${kurtosis_docker_builder}" &>/dev/null; then - echo "Warn: Failed removing the buildx builder '${kurtosis_docker_builder}'. Try manually removing it with 'docker buildx rm ${kurtosis_docker_builder}'" >&2 - exit 1 -fi -if ! docker context rm "${docker_buildx_context}" &>/dev/null; then - echo "Warn: Failed removing the buildx context '${docker_buildx_context}'. Try manually removing it with 'docker context rm ${docker_buildx_context}'" >&2 - exit 1 -fi -echo "Successfully built docker image" +## Cleanup context and buildx runner +#echo "Cleaning up remaining resources" +#if ! docker buildx rm "${kurtosis_docker_builder}" &>/dev/null; then +# echo "Warn: Failed removing the buildx builder '${kurtosis_docker_builder}'. Try manually removing it with 'docker buildx rm ${kurtosis_docker_builder}'" >&2 +# exit 1 +#fi +#if ! docker context rm "${docker_buildx_context}" &>/dev/null; then +# echo "Warn: Failed removing the buildx context '${docker_buildx_context}'. Try manually removing it with 'docker context rm ${docker_buildx_context}'" >&2 +# exit 1 +#fi +#echo "Successfully built docker image" From aaca583056888cb9196f9c8e93a683a3b11d721a Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Thu, 25 Jul 2024 12:08:34 -0400 Subject: [PATCH 16/31] fix --- .../lib/backend_impls/docker/docker_manager/docker_manager.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go b/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go index cd43767885..0293d194f8 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go @@ -624,8 +624,7 @@ func (manager *DockerManager) CreateAndStartContainer( } containerHostConfigPtr, err := manager.getContainerHostConfig( args.addedCapabilities, - //args.securityOpts, - + args.securityOpts, args.networkMode, args.bindMounts, args.volumeMounts, From 900a5315fb5bbbcc8706ad9a2ff6d243ac6a46ed Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Wed, 31 Jul 2024 16:29:21 -0400 Subject: [PATCH 17/31] wait and list networks before connecting --- .../reverse_proxy_functions/network_reverse_proxy.go | 5 +++++ .../docker/docker_manager/docker_manager.go | 9 +++++++++ 2 files changed, 14 insertions(+) diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/network_reverse_proxy.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/network_reverse_proxy.go index 109e4c1c20..539e91eb9c 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/network_reverse_proxy.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/network_reverse_proxy.go @@ -2,7 +2,9 @@ package reverse_proxy_functions import ( "context" + "github.com/sirupsen/logrus" "net" + "time" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_manager" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/docker_label_key" @@ -34,6 +36,9 @@ func ConnectReverseProxyToNetwork(ctx context.Context, dockerManager *docker_man return nil } + logrus.Infof("Waiting a few second before connecting reverse proxy to network...") + time.Sleep(10 * time.Second) + logrus.Infof("Okay lets try and connect now.") if err = dockerManager.ConnectContainerToNetwork(ctx, networkId, maybeReverseProxyContainerId, autoAssignIpAddressToReverseProxy, emptyAliasForReverseProxy); err != nil { return stacktrace.Propagate(err, "An error occurred while connecting the reverse proxy with container id '%v' to the enclave network '%v'", maybeReverseProxyContainerId, networkId) } diff --git a/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go b/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go index 0293d194f8..4e38b5afe7 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go @@ -1222,6 +1222,15 @@ func (manager *DockerManager) ConnectContainerToNetwork(ctx context.Context, net config := getEndpointSettingsForIpAddress(staticIpAddressStr, alias) + logrus.Infof("Listing all networks right before attempting to connect to network...") + networkResources, err := manager.dockerClient.NetworkList(ctx, types.NetworkListOptions{}) + for _, networkResource := range networkResources { + logrus.Infof("Information about network '%v' that exists: %v", networkResource.Name, networkResource) + if networkResource.ID == networkId { + logrus.Infof("The network we are trying to connect to exists right before connection.") + } + } + err = manager.dockerClient.NetworkConnect( ctx, networkId, From 572f4b859e5ea74fe300140c8f0ca67d7324d81d Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Thu, 1 Aug 2024 12:17:02 -0400 Subject: [PATCH 18/31] improve err logs for storing pkg contents --- .../git_package_content_provider.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/server/api_container/server/startosis_engine/startosis_packages/git_package_content_provider/git_package_content_provider.go b/core/server/api_container/server/startosis_engine/startosis_packages/git_package_content_provider/git_package_content_provider.go index af1c8b1f5d..d3a1303ac0 100644 --- a/core/server/api_container/server/startosis_engine/startosis_packages/git_package_content_provider/git_package_content_provider.go +++ b/core/server/api_container/server/startosis_engine/startosis_packages/git_package_content_provider/git_package_content_provider.go @@ -230,7 +230,7 @@ func (provider *GitPackageContentProvider) StorePackageContents(packageId string tempFile, err := os.CreateTemp(defaultTmpDir, temporaryArchiveFilePattern) if err != nil { - return "", startosis_errors.NewInterpretationError("An error occurred while creating temporary file to write compressed '%v' to", packageId) + return "", startosis_errors.WrapWithInterpretationError(err, "An error occurred while creating temporary file to write compressed '%v' to temporary directory '%v' with temporary archive file pattern '%v'.", packageId, defaultTmpDir, temporaryArchiveFilePattern) } defer os.Remove(tempFile.Name()) From aba722acf18fd6e1640a8067bef7d4f2c253f802 Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Fri, 23 Aug 2024 17:06:13 -0400 Subject: [PATCH 19/31] start dependency list --- .../builtins/import_module/import_module.go | 2 +- .../startosis_engine/plan_yaml/plan_yaml.go | 22 +++++++++++++++---- .../startosis_engine/startosis_interpreter.go | 1 + 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/core/server/api_container/server/startosis_engine/builtins/import_module/import_module.go b/core/server/api_container/server/startosis_engine/builtins/import_module/import_module.go index 7e2351748f..143c9ce354 100644 --- a/core/server/api_container/server/startosis_engine/builtins/import_module/import_module.go +++ b/core/server/api_container/server/startosis_engine/builtins/import_module/import_module.go @@ -19,7 +19,7 @@ const ( /* NewImportModule returns a sequential (not parallel) implementation of an equivalent or `load` in Starlark - This function returns a starlarkstruct.Module object that can then me used to get variables and call functions from the loaded module. + This function returns a starlarkstruct.Module object that can then be used to get variables and call functions from the loaded module. How does the returned function work? 1. The function first checks whether a module is currently loading. If so then there's cycle and it errors immediately, diff --git a/core/server/api_container/server/startosis_engine/plan_yaml/plan_yaml.go b/core/server/api_container/server/startosis_engine/plan_yaml/plan_yaml.go index 3677b2004c..e7f6167719 100644 --- a/core/server/api_container/server/startosis_engine/plan_yaml/plan_yaml.go +++ b/core/server/api_container/server/startosis_engine/plan_yaml/plan_yaml.go @@ -9,10 +9,11 @@ const ( ) type privatePlanYaml struct { - PackageId string `yaml:"packageId,omitempty"` - Services []*Service `yaml:"services,omitempty"` - FilesArtifacts []*FilesArtifact `yaml:"filesArtifacts,omitempty"` - Tasks []*Task `yaml:"tasks,omitempty"` + PackageId string `yaml:"packageId,omitempty"` + Services []*Service `yaml:"services,omitempty"` + FilesArtifacts []*FilesArtifact `yaml:"filesArtifacts,omitempty"` + Tasks []*Task `yaml:"tasks,omitempty"` + PackageDependencyGraph PackageDependencyGraph `yaml:"packagDependencyGraph,omitempty"` } // Service represents a service in the system. @@ -110,3 +111,16 @@ type Task struct { // TaskType represents the type of task (either python or shell) type TaskType string + +type Package struct { + PackageId string + ContainerImages []string +} + +type PackageDependencyGraph struct { + RootPackageId string + + PackageIndex map[string]Package + + PackageGraph map[string][]Package +} diff --git a/core/server/api_container/server/startosis_engine/startosis_interpreter.go b/core/server/api_container/server/startosis_engine/startosis_interpreter.go index 8660df5ce4..a737262db0 100644 --- a/core/server/api_container/server/startosis_engine/startosis_interpreter.go +++ b/core/server/api_container/server/startosis_engine/startosis_interpreter.go @@ -358,6 +358,7 @@ func (interpreter *StartosisInterpreter) buildBindings( packageReplaceOptions map[string]string, ) (*starlark.StringDict, *startosis_errors.InterpretationError) { recursiveInterpretForModuleLoading := func(moduleId string, serializedStartosis string) (starlark.StringDict, *startosis_errors.InterpretationError) { + logrus.Infof("PACKAGE DEPENDENCY: %v", moduleId) result, err := interpreter.interpretInternal(packageId, moduleId, serializedStartosis, instructionPlan, moduleGlobalCache, packageReplaceOptions) if err != nil { return nil, err From 507ef1329ba03973b07c6b1d2e96bfa7c3d132e9 Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Thu, 29 Aug 2024 22:55:57 -0400 Subject: [PATCH 20/31] add images and package dependencies to plan yaml --- .../instructions_plan/instructions_plan.go | 9 +++++++ .../startosis_engine/plan_yaml/plan_yaml.go | 11 +++++---- .../plan_yaml/plan_yaml_generator.go | 24 +++++++++++++++---- .../startosis_engine/startosis_interpreter.go | 1 + 4 files changed, 36 insertions(+), 9 deletions(-) diff --git a/core/server/api_container/server/startosis_engine/instructions_plan/instructions_plan.go b/core/server/api_container/server/startosis_engine/instructions_plan/instructions_plan.go index 38e42655d4..1ec0e1e309 100644 --- a/core/server/api_container/server/startosis_engine/instructions_plan/instructions_plan.go +++ b/core/server/api_container/server/startosis_engine/instructions_plan/instructions_plan.go @@ -21,6 +21,9 @@ type InstructionsPlan struct { scheduledInstructionsIndex map[ScheduledInstructionUuid]*ScheduledInstruction instructionsSequence []ScheduledInstructionUuid + + // list of package names that this instructions plan relies on + packageDependencies []string } func NewInstructionsPlan() *InstructionsPlan { @@ -28,6 +31,7 @@ func NewInstructionsPlan() *InstructionsPlan { indexOfFirstInstruction: 0, scheduledInstructionsIndex: map[ScheduledInstructionUuid]*ScheduledInstruction{}, instructionsSequence: []ScheduledInstructionUuid{}, + packageDependencies: []string{}, } } @@ -88,9 +92,14 @@ func (plan *InstructionsPlan) GenerateYaml(planYaml *plan_yaml.PlanYaml) (string return "", startosis_errors.WrapWithInterpretationError(err, "An error occurred updating the plan with instruction: %v.", instructionUuid) } } + planYaml.AddPackageDependencies(plan.packageDependencies) return planYaml.GenerateYaml() } +func (plan *InstructionsPlan) AddPackageDependency(packageDependency string) { + plan.packageDependencies = append(plan.packageDependencies, packageDependency) +} + func (plan *InstructionsPlan) Size() int { return len(plan.instructionsSequence) } diff --git a/core/server/api_container/server/startosis_engine/plan_yaml/plan_yaml.go b/core/server/api_container/server/startosis_engine/plan_yaml/plan_yaml.go index e7f6167719..df7b2d17dc 100644 --- a/core/server/api_container/server/startosis_engine/plan_yaml/plan_yaml.go +++ b/core/server/api_container/server/startosis_engine/plan_yaml/plan_yaml.go @@ -9,11 +9,12 @@ const ( ) type privatePlanYaml struct { - PackageId string `yaml:"packageId,omitempty"` - Services []*Service `yaml:"services,omitempty"` - FilesArtifacts []*FilesArtifact `yaml:"filesArtifacts,omitempty"` - Tasks []*Task `yaml:"tasks,omitempty"` - PackageDependencyGraph PackageDependencyGraph `yaml:"packagDependencyGraph,omitempty"` + PackageId string `yaml:"packageId,omitempty"` + Services []*Service `yaml:"services,omitempty"` + FilesArtifacts []*FilesArtifact `yaml:"filesArtifacts,omitempty"` + Tasks []*Task `yaml:"tasks,omitempty"` + Images []string `yaml:"images,omitempty"` + PackageDependencies []string `yaml:"packageDependencies,omitempty"` } // Service represents a service in the system. diff --git a/core/server/api_container/server/startosis_engine/plan_yaml/plan_yaml_generator.go b/core/server/api_container/server/startosis_engine/plan_yaml/plan_yaml_generator.go index 36fa1ea211..cdc2ac5ce8 100644 --- a/core/server/api_container/server/startosis_engine/plan_yaml/plan_yaml_generator.go +++ b/core/server/api_container/server/startosis_engine/plan_yaml/plan_yaml_generator.go @@ -29,15 +29,19 @@ type PlanYaml struct { futureReferenceIndex map[string]string filesArtifactIndex map[string]*FilesArtifact latestUuid int + images []string + packageDependencies []string } func CreateEmptyPlan(packageId string) *PlanYaml { return &PlanYaml{ privatePlanYaml: &privatePlanYaml{ - PackageId: packageId, - Services: []*Service{}, - Tasks: []*Task{}, - FilesArtifacts: []*FilesArtifact{}, + PackageId: packageId, + Services: []*Service{}, + Tasks: []*Task{}, + FilesArtifacts: []*FilesArtifact{}, + Images: []string{}, + PackageDependencies: []string{}, }, futureReferenceIndex: map[string]string{}, filesArtifactIndex: map[string]*FilesArtifact{}, @@ -84,6 +88,8 @@ func (planYaml *PlanYaml) AddService( imageYaml := &ImageSpec{} //nolint:exhaustruct imageYaml.ImageName = serviceConfig.GetContainerImageName() + planYaml.addImage(imageYaml.ImageName) + imageYaml.BuildContextLocator = imageBuildContextLocator imageYaml.TargetStage = imageTargetStage imageYaml.Registry = imageRegistryAddress @@ -168,6 +174,7 @@ func (planYaml *PlanYaml) AddRunSh( taskYaml.RunCmd = []string{planYaml.swapFutureReference(runCommand)} taskYaml.Image = serviceConfig.GetContainerImageName() + planYaml.addImage(taskYaml.Image) var envVars []*EnvironmentVariable for key, val := range serviceConfig.GetEnvVars() { @@ -243,6 +250,7 @@ func (planYaml *PlanYaml) AddRunPython( taskYaml.RunCmd = []string{planYaml.swapFutureReference(runCommand)} taskYaml.Image = serviceConfig.GetContainerImageName() + planYaml.addImage(taskYaml.Image) var envVars []*EnvironmentVariable for key, val := range serviceConfig.GetEnvVars() { @@ -375,6 +383,10 @@ func (planYaml *PlanYaml) RemoveService(serviceName string) { } } +func (planYaml *PlanYaml) AddPackageDependencies(packageDependency []string) { + planYaml.privatePlanYaml.PackageDependencies = append(planYaml.privatePlanYaml.PackageDependencies, packageDependency...) +} + // getFileMountsFromFilesArtifacts turns filesArtifactExpansions into FileMount's // file mounts have two cases: // 1. the referenced files artifact already exists in the planYaml, in which case add the referenced files artifact to the proper filepath as a file mount @@ -433,6 +445,10 @@ func (planYaml *PlanYaml) addTaskYaml(task *Task) { planYaml.privatePlanYaml.Tasks = append(planYaml.privatePlanYaml.Tasks, task) } +func (planYaml *PlanYaml) addImage(img string) { + planYaml.privatePlanYaml.Images = append(planYaml.privatePlanYaml.Images, img) +} + // yaml future reference format: {{ kurtosis.. Date: Thu, 29 Aug 2024 22:57:32 -0400 Subject: [PATCH 21/31] hook up to cli --- cli/cli/commands/run/run.go | 64 ++++++++++++++++++++++++++++++++++++- 1 file changed, 63 insertions(+), 1 deletion(-) mode change 100644 => 100755 cli/cli/commands/run/run.go diff --git a/cli/cli/commands/run/run.go b/cli/cli/commands/run/run.go old mode 100644 new mode 100755 index e271c849ae..d61e4fd51d --- a/cli/cli/commands/run/run.go +++ b/cli/cli/commands/run/run.go @@ -55,6 +55,9 @@ const ( dryRunFlagKey = "dry-run" defaultDryRun = "false" + dependenciesFlagKey = "dependencies" + defaultDependencies = "false" + fullUuidsFlagKey = "full-uuids" fullUuidFlagKeyDefault = "false" @@ -128,6 +131,12 @@ var StarlarkRunCmd = &engine_consuming_kurtosis_command.EngineConsumingKurtosisC Type: flags.FlagType_Bool, Default: defaultDryRun, }, + { + Key: dependenciesFlagKey, + Usage: "If true, a yaml will be returned with a list of images and packages that this run depends on.", + Type: flags.FlagType_Bool, + Default: defaultDependencies, + }, { Key: enclaveIdentifierFlagKey, Usage: "The enclave identifier of the enclave in which the script or package will be ran. " + @@ -262,6 +271,11 @@ func run( return stacktrace.Propagate(err, "Expected a boolean flag with key '%v' but none was found; this is an error in Kurtosis!", dryRunFlagKey) } + dependencies, err := flags.GetBool(dependenciesFlagKey) + if err != nil { + return stacktrace.Propagate(err, "Expectew a boolean flag with key '%v' but none was found; this is an error in Kurtosis!", dependencies) + } + parallelism, err := flags.GetUint32(parallelismFlagKey) if err != nil { return stacktrace.Propagate(err, "Expected a integer flag with key '%v' but none was found; this is an error in Kurtosis!", parallelismFlagKey) @@ -366,6 +380,17 @@ func run( defer output_printers.PrintEnclaveName(enclaveCtx.GetEnclaveName()) } + isRemotePackage := strings.HasPrefix(starlarkScriptOrPackagePath, githubDomainPrefix) + + if dependencies { + packageDependencyYamlStr, err := getPackageDependencyYamlStr(ctx, enclaveCtx, starlarkScriptOrPackagePath, isRemotePackage, packageArgs) + if err != nil { + return stacktrace.Propagate(err, "An error occurred getting package dependencies") + } + fmt.Printf("%v\n", packageDependencyYamlStr) + return nil + } + var responseLineChan <-chan *kurtosis_core_rpc_api_bindings.StarlarkRunResponseLine var cancelFunc context.CancelFunc var errRunningKurtosis error @@ -375,7 +400,6 @@ func run( connect = kurtosis_core_rpc_api_bindings.Connect_NO_CONNECT } - isRemotePackage := strings.HasPrefix(starlarkScriptOrPackagePath, githubDomainPrefix) if isRemotePackage { responseLineChan, cancelFunc, errRunningKurtosis = executeRemotePackage(ctx, enclaveCtx, starlarkScriptOrPackagePath, starlarkRunConfig) } else { @@ -588,6 +612,44 @@ func getOrCreateEnclaveContext( return enclaveContext, isNewEnclaveFlagWhenCreated, nil } +type PackageDependency struct { + Packages []string `yaml:"packages,omitempty"` + Images []string `yaml:"images,omitempty"` +} + +func getPackageDependencyYamlStr( + ctx context.Context, + enclaveCtx *enclaves.EnclaveContext, + starlarkScriptOrPackageId string, + isRemote bool, + packageArgs string, +) (string, error) { + var packageYaml *kurtosis_core_rpc_api_bindings.PlanYaml + var err error + if isRemote { + packageYaml, err = enclaveCtx.GetStarlarkPackagePlanYaml(ctx, starlarkScriptOrPackageId, packageArgs) + } else { + packageYaml, err = enclaveCtx.GetStarlarkScriptPlanYaml(ctx, starlarkScriptOrPackageId, packageArgs) + } + if err != nil { + return "", stacktrace.Propagate(err, "An error occurred retrieving plan yaml for provided package.") + } + //var dependencies map[string][]string + //if err = yaml.Unmarshal([]byte(packageYaml.PlanYaml), dependencies); err != nil { + // return "", stacktrace.Propagate(err, "An error occurred unmarshaling plan yaml string: %v.", packageYaml.PlanYaml) + //} + //pd := PackageDependency{ + // Packages: dependencies["packageDependencies"], + // Images: dependencies["images"], + //} + //packageDependencyYamlBytes, err := yaml.Marshal(&pd) + //if err != nil { + // return "", stacktrace.Propagate(err, "") + //} + //return string(packageDependencyYamlBytes), nil + return packageYaml.PlanYaml, nil +} + // validatePackageArgs just validates the args is a valid JSON or YAML string func validatePackageArgs(_ context.Context, _ *flags.ParsedFlags, args *args.ParsedArgs) error { serializedArgs, err := args.GetNonGreedyArg(inputArgsArgKey) From 8df95e7ae0f767ebab404b1e287f78f7d416e462 Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Thu, 29 Aug 2024 22:58:00 -0400 Subject: [PATCH 22/31] add get starlark plan yaml to api --- .../core/lib/enclaves/enclave_context.go | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) mode change 100644 => 100755 api/golang/core/lib/enclaves/enclave_context.go diff --git a/api/golang/core/lib/enclaves/enclave_context.go b/api/golang/core/lib/enclaves/enclave_context.go old mode 100644 new mode 100755 index 9849652114..bb54691c61 --- a/api/golang/core/lib/enclaves/enclave_context.go +++ b/api/golang/core/lib/enclaves/enclave_context.go @@ -531,6 +531,31 @@ func (enclaveCtx *EnclaveContext) GetStarlarkRun(ctx context.Context) (*kurtosis return response, nil } +func (enclaveCtx *EnclaveContext) GetStarlarkPackagePlanYaml(ctx context.Context, packageId string, serializedParams string) (*kurtosis_core_rpc_api_bindings.PlanYaml, error) { + response, err := enclaveCtx.client.GetStarlarkPackagePlanYaml(ctx, &kurtosis_core_rpc_api_bindings.StarlarkPackagePlanYamlArgs{ + PackageId: packageId, + SerializedParams: &serializedParams, + RelativePathToMainFile: nil, + MainFunctionName: nil, + }) + if err != nil { + return nil, stacktrace.Propagate(err, "An error occurred while getting the starlark package plan yaml run.") + } + return response, nil +} + +func (enclaveCtx *EnclaveContext) GetStarlarkScriptPlanYaml(ctx context.Context, serializedScript string, serializedParams string) (*kurtosis_core_rpc_api_bindings.PlanYaml, error) { + response, err := enclaveCtx.client.GetStarlarkScriptPlanYaml(ctx, &kurtosis_core_rpc_api_bindings.StarlarkScriptPlanYamlArgs{ + SerializedScript: serializedScript, + SerializedParams: &serializedParams, + MainFunctionName: nil, + }) + if err != nil { + return nil, stacktrace.Propagate(err, "An error occurred while getting the last starlark script plan yaml run.") + } + return response, nil +} + // ==================================================================================================== // // Private helper methods From d0bba73c93ab207005192c1fffb04ae959777ef8 Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Fri, 30 Aug 2024 10:54:06 -0400 Subject: [PATCH 23/31] update tests --- .../startosis_interpreter_plan_yaml_test.go | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/core/server/api_container/server/startosis_engine/startosis_interpreter_plan_yaml_test.go b/core/server/api_container/server/startosis_engine/startosis_interpreter_plan_yaml_test.go index 73838db876..9618d10827 100644 --- a/core/server/api_container/server/startosis_engine/startosis_interpreter_plan_yaml_test.go +++ b/core/server/api_container/server/startosis_engine/startosis_interpreter_plan_yaml_test.go @@ -147,6 +147,8 @@ services: filesArtifacts: - uuid: "2" name: hi-file +images: +- kurtosistech/example-datastore-server ` require.Equal(suite.T(), expectedYaml, planYaml) } @@ -212,6 +214,8 @@ tasks: envVar: - key: HELLO value: Hello! +images: +- badouralix/curl-jq ` require.Equal(suite.T(), expectedYaml, planYaml) } @@ -288,6 +292,8 @@ tasks: - requests pythonArgs: - something +images: +- python:3.11-alpine ` require.Equal(suite.T(), expectedYaml, planYaml) } @@ -362,6 +368,8 @@ tasks: serviceName: db acceptableCodes: - 0 +images: +- postgres:latest ` require.Equal(suite.T(), expectedYaml, planYaml) } @@ -428,6 +436,8 @@ tasks: filesArtifacts: - uuid: "1" name: bye-file +images: +- badouralix/curl-jq ` require.Equal(suite.T(), expectedYaml, planYaml) } @@ -490,6 +500,8 @@ services: filesArtifacts: - uuid: "2" name: hi-file +images: +- kurtosistech/example-datastore-server ` require.Equal(suite.T(), expectedYaml, planYaml) } @@ -550,6 +562,8 @@ services: filesArtifacts: - uuid: "2" name: hi-file +images: +- kurtosistech/example-datastore-server ` require.Equal(suite.T(), expectedYaml, planYaml) } @@ -610,6 +624,8 @@ tasks: filesArtifacts: - uuid: "1" name: dockerfile +images: +- badouralix/curl-jq ` require.Equal(suite.T(), expectedYaml, planYaml) } @@ -673,6 +689,8 @@ filesArtifacts: name: bye-file files: - bye.txt +images: +- postgres:latest ` require.Equal(suite.T(), expectedYaml, planYaml) } @@ -717,6 +735,8 @@ func (suite *StartosisIntepreterPlanYamlTestSuite) TestRemoveService() { filesArtifacts: - uuid: "2" name: hi-file +images: +- postgres:latest ` require.Equal(suite.T(), expectedYaml, planYaml) } @@ -813,6 +833,10 @@ tasks: command: - echo {{ kurtosis.4.code }} {{ kurtosis.4.output }} image: badouralix/curl-jq +images: +- postgres:latest +- badouralix/curl-jq +- badouralix/curl-jq ` require.Equal(suite.T(), expectedYaml, planYaml) } From 3831674792484c1088b284dfcf477c501c0fe87f Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Fri, 30 Aug 2024 11:06:43 -0400 Subject: [PATCH 24/31] make imgs unique and sort --- .../instructions_plan/instructions_plan.go | 6 +++--- .../plan_yaml/plan_yaml_generator.go | 21 ++++++++++++++----- .../startosis_interpreter_plan_yaml_test.go | 3 +-- 3 files changed, 20 insertions(+), 10 deletions(-) diff --git a/core/server/api_container/server/startosis_engine/instructions_plan/instructions_plan.go b/core/server/api_container/server/startosis_engine/instructions_plan/instructions_plan.go index 1ec0e1e309..4b2a871d33 100644 --- a/core/server/api_container/server/startosis_engine/instructions_plan/instructions_plan.go +++ b/core/server/api_container/server/startosis_engine/instructions_plan/instructions_plan.go @@ -23,7 +23,7 @@ type InstructionsPlan struct { instructionsSequence []ScheduledInstructionUuid // list of package names that this instructions plan relies on - packageDependencies []string + packageDependencies map[string]bool } func NewInstructionsPlan() *InstructionsPlan { @@ -31,7 +31,7 @@ func NewInstructionsPlan() *InstructionsPlan { indexOfFirstInstruction: 0, scheduledInstructionsIndex: map[ScheduledInstructionUuid]*ScheduledInstruction{}, instructionsSequence: []ScheduledInstructionUuid{}, - packageDependencies: []string{}, + packageDependencies: map[string]bool{}, } } @@ -97,7 +97,7 @@ func (plan *InstructionsPlan) GenerateYaml(planYaml *plan_yaml.PlanYaml) (string } func (plan *InstructionsPlan) AddPackageDependency(packageDependency string) { - plan.packageDependencies = append(plan.packageDependencies, packageDependency) + plan.packageDependencies[packageDependency] = true } func (plan *InstructionsPlan) Size() int { diff --git a/core/server/api_container/server/startosis_engine/plan_yaml/plan_yaml_generator.go b/core/server/api_container/server/startosis_engine/plan_yaml/plan_yaml_generator.go index cdc2ac5ce8..f2aaa3c708 100644 --- a/core/server/api_container/server/startosis_engine/plan_yaml/plan_yaml_generator.go +++ b/core/server/api_container/server/startosis_engine/plan_yaml/plan_yaml_generator.go @@ -29,8 +29,7 @@ type PlanYaml struct { futureReferenceIndex map[string]string filesArtifactIndex map[string]*FilesArtifact latestUuid int - images []string - packageDependencies []string + imageSet map[string]bool } func CreateEmptyPlan(packageId string) *PlanYaml { @@ -43,6 +42,7 @@ func CreateEmptyPlan(packageId string) *PlanYaml { Images: []string{}, PackageDependencies: []string{}, }, + imageSet: map[string]bool{}, futureReferenceIndex: map[string]string{}, filesArtifactIndex: map[string]*FilesArtifact{}, latestUuid: 0, @@ -50,6 +50,7 @@ func CreateEmptyPlan(packageId string) *PlanYaml { } func (planYaml *PlanYaml) GenerateYaml() (string, error) { + planYaml.AddImages() yamlBytes, err := yaml.Marshal(planYaml.privatePlanYaml) if err != nil { return "", stacktrace.Propagate(err, "An error occurred generating plan yaml.") @@ -383,8 +384,18 @@ func (planYaml *PlanYaml) RemoveService(serviceName string) { } } -func (planYaml *PlanYaml) AddPackageDependencies(packageDependency []string) { - planYaml.privatePlanYaml.PackageDependencies = append(planYaml.privatePlanYaml.PackageDependencies, packageDependency...) +func (planYaml *PlanYaml) AddPackageDependencies(packageDependency map[string]bool) { + for dependency := range packageDependency { + planYaml.privatePlanYaml.PackageDependencies = append(planYaml.privatePlanYaml.PackageDependencies, dependency) + } + slices.Sort(planYaml.privatePlanYaml.PackageDependencies) +} + +func (planYaml *PlanYaml) AddImages() { + for img := range planYaml.imageSet { + planYaml.privatePlanYaml.Images = append(planYaml.privatePlanYaml.Images, img) + } + slices.Sort(planYaml.privatePlanYaml.Images) } // getFileMountsFromFilesArtifacts turns filesArtifactExpansions into FileMount's @@ -446,7 +457,7 @@ func (planYaml *PlanYaml) addTaskYaml(task *Task) { } func (planYaml *PlanYaml) addImage(img string) { - planYaml.privatePlanYaml.Images = append(planYaml.privatePlanYaml.Images, img) + planYaml.imageSet[img] = true } // yaml future reference format: {{ kurtosis.. Date: Fri, 30 Aug 2024 11:29:18 -0400 Subject: [PATCH 25/31] refactor plan yaml --- cli/cli/commands/run/run.go | 28 +++++------ .../instructions_plan/instructions_plan.go | 2 +- .../add_service/add_service.go | 2 +- .../add_service/add_services.go | 2 +- .../kurtosis_instruction/exec/exec.go | 2 +- .../get_files_artifact/get_files_artifact.go | 2 +- .../get_service/get_service.go | 2 +- .../get_services/get_services.go | 2 +- .../kurtosis_instruction.go | 2 +- .../kurtosis_print/kurtosis_print.go | 2 +- .../mock_kurtosis_instruction.go | 12 ++--- .../remove_service/remove_service.go | 2 +- .../render_templates/render_templates.go | 2 +- .../kurtosis_instruction/request/request.go | 2 +- .../set_service/set_service.go | 2 +- .../start_service/start_service.go | 2 +- .../stop_service/stop_service.go | 2 +- .../store_service_files.go | 2 +- .../kurtosis_instruction/tasks/run_python.go | 2 +- .../kurtosis_instruction/tasks/run_sh.go | 2 +- .../upload_files/upload_files.go | 2 +- .../kurtosis_instruction/verify/verify.go | 2 +- .../kurtosis_instruction/wait/wait.go | 2 +- .../kurtosis_plan_instruction_capabilities.go | 2 +- .../kurtosis_plan_instruction_internal.go | 2 +- .../startosis_engine/plan_yaml/plan_yaml.go | 2 +- .../plan_yaml/plan_yaml_generator.go | 50 +++++++++---------- 27 files changed, 69 insertions(+), 69 deletions(-) diff --git a/cli/cli/commands/run/run.go b/cli/cli/commands/run/run.go index d61e4fd51d..ce6358ce9e 100755 --- a/cli/cli/commands/run/run.go +++ b/cli/cli/commands/run/run.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/user_support_constants" + "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/plan_yaml" "io" "net/http" "net/url" @@ -634,20 +635,19 @@ func getPackageDependencyYamlStr( if err != nil { return "", stacktrace.Propagate(err, "An error occurred retrieving plan yaml for provided package.") } - //var dependencies map[string][]string - //if err = yaml.Unmarshal([]byte(packageYaml.PlanYaml), dependencies); err != nil { - // return "", stacktrace.Propagate(err, "An error occurred unmarshaling plan yaml string: %v.", packageYaml.PlanYaml) - //} - //pd := PackageDependency{ - // Packages: dependencies["packageDependencies"], - // Images: dependencies["images"], - //} - //packageDependencyYamlBytes, err := yaml.Marshal(&pd) - //if err != nil { - // return "", stacktrace.Propagate(err, "") - //} - //return string(packageDependencyYamlBytes), nil - return packageYaml.PlanYaml, nil + var planYaml plan_yaml.PlanYaml + if err = yaml.Unmarshal([]byte(packageYaml.PlanYaml), &planYaml); err != nil { + return "", stacktrace.Propagate(err, "An error occurred unmarshaling plan yaml string: %v.", packageYaml.PlanYaml) + } + planYaml.Tasks = nil + planYaml.FilesArtifacts = nil + planYaml.Services = nil + packageDependencyYamlBytes, err := yaml.Marshal(&planYaml) + if err != nil { + return "", stacktrace.Propagate(err, "") + } + return string(packageDependencyYamlBytes), nil + //return packageYaml.PlanYaml, nil } // validatePackageArgs just validates the args is a valid JSON or YAML string diff --git a/core/server/api_container/server/startosis_engine/instructions_plan/instructions_plan.go b/core/server/api_container/server/startosis_engine/instructions_plan/instructions_plan.go index 4b2a871d33..4b5f8b2dc6 100644 --- a/core/server/api_container/server/startosis_engine/instructions_plan/instructions_plan.go +++ b/core/server/api_container/server/startosis_engine/instructions_plan/instructions_plan.go @@ -81,7 +81,7 @@ func (plan *InstructionsPlan) GeneratePlan() ([]*ScheduledInstruction, *startosi } // GenerateYaml takes in an existing planYaml (usually empty) and returns a yaml string containing the effects of the plan -func (plan *InstructionsPlan) GenerateYaml(planYaml *plan_yaml.PlanYaml) (string, error) { +func (plan *InstructionsPlan) GenerateYaml(planYaml *plan_yaml.PlanYamlGenerator) (string, error) { for _, instructionUuid := range plan.instructionsSequence { instruction, found := plan.scheduledInstructionsIndex[instructionUuid] if !found { diff --git a/core/server/api_container/server/startosis_engine/kurtosis_instruction/add_service/add_service.go b/core/server/api_container/server/startosis_engine/kurtosis_instruction/add_service/add_service.go index 903b30c358..b92b70528c 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_instruction/add_service/add_service.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_instruction/add_service/add_service.go @@ -286,7 +286,7 @@ func (builtin *AddServiceCapabilities) FillPersistableAttributes(builder *enclav ) } -func (builtin *AddServiceCapabilities) UpdatePlan(planYaml *plan_yaml.PlanYaml) error { +func (builtin *AddServiceCapabilities) UpdatePlan(planYaml *plan_yaml.PlanYamlGenerator) error { var buildContextLocator string var targetStage string var registryAddress string diff --git a/core/server/api_container/server/startosis_engine/kurtosis_instruction/add_service/add_services.go b/core/server/api_container/server/startosis_engine/kurtosis_instruction/add_service/add_services.go index c221488f85..6e1ec87f8a 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_instruction/add_service/add_services.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_instruction/add_service/add_services.go @@ -370,7 +370,7 @@ func (builtin *AddServicesCapabilities) allServicesReadinessCheck( return failedServiceChecksRegularMap } -func (builtin *AddServicesCapabilities) UpdatePlan(plan *plan_yaml.PlanYaml) error { +func (builtin *AddServicesCapabilities) UpdatePlan(plan *plan_yaml.PlanYamlGenerator) error { // TOOD: Implement logrus.Warn("ADD SERVICES NOT IMPLEMENTED YET FOR UPDATING PLAN YAML.") return nil diff --git a/core/server/api_container/server/startosis_engine/kurtosis_instruction/exec/exec.go b/core/server/api_container/server/startosis_engine/kurtosis_instruction/exec/exec.go index 14f9ef0821..bcea5246b1 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_instruction/exec/exec.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_instruction/exec/exec.go @@ -214,7 +214,7 @@ func (builtin *ExecCapabilities) FillPersistableAttributes(builder *enclave_plan builder.SetType(ExecBuiltinName) } -func (builtin *ExecCapabilities) UpdatePlan(planYaml *plan_yaml.PlanYaml) error { +func (builtin *ExecCapabilities) UpdatePlan(planYaml *plan_yaml.PlanYamlGenerator) error { err := planYaml.AddExec(string(builtin.serviceName), builtin.description, builtin.returnValue, builtin.cmdList, builtin.acceptableCodes) if err != nil { return stacktrace.Propagate(err, "An error occurred updating plan with exec.") diff --git a/core/server/api_container/server/startosis_engine/kurtosis_instruction/get_files_artifact/get_files_artifact.go b/core/server/api_container/server/startosis_engine/kurtosis_instruction/get_files_artifact/get_files_artifact.go index da3b565027..60122ee167 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_instruction/get_files_artifact/get_files_artifact.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_instruction/get_files_artifact/get_files_artifact.go @@ -93,7 +93,7 @@ func (builtin *GetFilesArtifactCapabilities) FillPersistableAttributes(builder * builder.SetType(GetFilesArtifactBuiltinName).AddFilesArtifact(builtin.artifactName, nil) } -func (builtin *GetFilesArtifactCapabilities) UpdatePlan(planYaml *plan_yaml.PlanYaml) error { +func (builtin *GetFilesArtifactCapabilities) UpdatePlan(planYaml *plan_yaml.PlanYamlGenerator) error { // get files artifact does not affect the planYaml return nil } diff --git a/core/server/api_container/server/startosis_engine/kurtosis_instruction/get_service/get_service.go b/core/server/api_container/server/startosis_engine/kurtosis_instruction/get_service/get_service.go index 17b5aa7a2a..98ce8d9fd1 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_instruction/get_service/get_service.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_instruction/get_service/get_service.go @@ -103,7 +103,7 @@ func (builtin *GetServiceCapabilities) FillPersistableAttributes(builder *enclav builder.SetType(GetServiceBuiltinName).AddServiceName(builtin.serviceName) } -func (builtin *GetServiceCapabilities) UpdatePlan(planYaml *plan_yaml.PlanYaml) error { +func (builtin *GetServiceCapabilities) UpdatePlan(planYaml *plan_yaml.PlanYamlGenerator) error { // get service does not affect the plan return nil } diff --git a/core/server/api_container/server/startosis_engine/kurtosis_instruction/get_services/get_services.go b/core/server/api_container/server/startosis_engine/kurtosis_instruction/get_services/get_services.go index 05d2667a24..991cbdb907 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_instruction/get_services/get_services.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_instruction/get_services/get_services.go @@ -92,7 +92,7 @@ func (builtin *GetServicesCapabilities) FillPersistableAttributes(builder *encla builder.SetType(GetServicesBuiltinName) } -func (builtin *GetServicesCapabilities) UpdatePlan(planYaml *plan_yaml.PlanYaml) error { +func (builtin *GetServicesCapabilities) UpdatePlan(planYaml *plan_yaml.PlanYamlGenerator) error { // get services does not affect the plan return nil } diff --git a/core/server/api_container/server/startosis_engine/kurtosis_instruction/kurtosis_instruction.go b/core/server/api_container/server/startosis_engine/kurtosis_instruction/kurtosis_instruction.go index 97c2fd1749..cf959b6c98 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_instruction/kurtosis_instruction.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_instruction/kurtosis_instruction.go @@ -36,5 +36,5 @@ type KurtosisInstruction interface { GetPersistableAttributes() *enclave_plan_persistence.EnclavePlanInstructionBuilder // UpdatePlan updates the plan with the effects of running this instruction. - UpdatePlan(plan *plan_yaml.PlanYaml) error + UpdatePlan(plan *plan_yaml.PlanYamlGenerator) error } diff --git a/core/server/api_container/server/startosis_engine/kurtosis_instruction/kurtosis_print/kurtosis_print.go b/core/server/api_container/server/startosis_engine/kurtosis_instruction/kurtosis_print/kurtosis_print.go index c111b0a098..6608f7e68c 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_instruction/kurtosis_print/kurtosis_print.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_instruction/kurtosis_print/kurtosis_print.go @@ -106,7 +106,7 @@ func (builtin *PrintCapabilities) FillPersistableAttributes(builder *enclave_pla builder.SetType(PrintBuiltinName) } -func (builitin *PrintCapabilities) UpdatePlan(plan *plan_yaml.PlanYaml) error { +func (builitin *PrintCapabilities) UpdatePlan(plan *plan_yaml.PlanYamlGenerator) error { // print does not affect the plan return nil } diff --git a/core/server/api_container/server/startosis_engine/kurtosis_instruction/mock_instruction/mock_kurtosis_instruction.go b/core/server/api_container/server/startosis_engine/kurtosis_instruction/mock_instruction/mock_kurtosis_instruction.go index 6d22f66a9c..a81b33fe94 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_instruction/mock_instruction/mock_kurtosis_instruction.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_instruction/mock_instruction/mock_kurtosis_instruction.go @@ -347,11 +347,11 @@ func (_c *MockKurtosisInstruction_TryResolveWith_Call) RunAndReturn(run func(*en } // UpdatePlan provides a mock function with given fields: plan -func (_m *MockKurtosisInstruction) UpdatePlan(plan *plan_yaml.PlanYaml) error { +func (_m *MockKurtosisInstruction) UpdatePlan(plan *plan_yaml.PlanYamlGenerator) error { ret := _m.Called(plan) var r0 error - if rf, ok := ret.Get(0).(func(*plan_yaml.PlanYaml) error); ok { + if rf, ok := ret.Get(0).(func(*plan_yaml.PlanYamlGenerator) error); ok { r0 = rf(plan) } else { r0 = ret.Error(0) @@ -366,14 +366,14 @@ type MockKurtosisInstruction_UpdatePlan_Call struct { } // UpdatePlan is a helper method to define mock.On call -// - plan *plan_yaml.PlanYaml +// - plan *plan_yaml.PlanYamlGenerator func (_e *MockKurtosisInstruction_Expecter) UpdatePlan(plan interface{}) *MockKurtosisInstruction_UpdatePlan_Call { return &MockKurtosisInstruction_UpdatePlan_Call{Call: _e.mock.On("UpdatePlan", plan)} } -func (_c *MockKurtosisInstruction_UpdatePlan_Call) Run(run func(plan *plan_yaml.PlanYaml)) *MockKurtosisInstruction_UpdatePlan_Call { +func (_c *MockKurtosisInstruction_UpdatePlan_Call) Run(run func(plan *plan_yaml.PlanYamlGenerator)) *MockKurtosisInstruction_UpdatePlan_Call { _c.Call.Run(func(args mock.Arguments) { - run(args[0].(*plan_yaml.PlanYaml)) + run(args[0].(*plan_yaml.PlanYamlGenerator)) }) return _c } @@ -383,7 +383,7 @@ func (_c *MockKurtosisInstruction_UpdatePlan_Call) Return(_a0 error) *MockKurtos return _c } -func (_c *MockKurtosisInstruction_UpdatePlan_Call) RunAndReturn(run func(*plan_yaml.PlanYaml) error) *MockKurtosisInstruction_UpdatePlan_Call { +func (_c *MockKurtosisInstruction_UpdatePlan_Call) RunAndReturn(run func(*plan_yaml.PlanYamlGenerator) error) *MockKurtosisInstruction_UpdatePlan_Call { _c.Call.Return(run) return _c } diff --git a/core/server/api_container/server/startosis_engine/kurtosis_instruction/remove_service/remove_service.go b/core/server/api_container/server/startosis_engine/kurtosis_instruction/remove_service/remove_service.go index d905da7e81..d4358a62b8 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_instruction/remove_service/remove_service.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_instruction/remove_service/remove_service.go @@ -114,7 +114,7 @@ func (builtin *RemoveServiceCapabilities) FillPersistableAttributes(builder *enc ) } -func (builtin *RemoveServiceCapabilities) UpdatePlan(plan *plan_yaml.PlanYaml) error { +func (builtin *RemoveServiceCapabilities) UpdatePlan(plan *plan_yaml.PlanYamlGenerator) error { plan.RemoveService(string(builtin.serviceName)) return nil } diff --git a/core/server/api_container/server/startosis_engine/kurtosis_instruction/render_templates/render_templates.go b/core/server/api_container/server/startosis_engine/kurtosis_instruction/render_templates/render_templates.go index 40e92fbf56..e2a05e08a6 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_instruction/render_templates/render_templates.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_instruction/render_templates/render_templates.go @@ -174,7 +174,7 @@ func (builtin *RenderTemplatesCapabilities) FillPersistableAttributes(builder *e ) } -func (builtin *RenderTemplatesCapabilities) UpdatePlan(plan *plan_yaml.PlanYaml) error { +func (builtin *RenderTemplatesCapabilities) UpdatePlan(plan *plan_yaml.PlanYamlGenerator) error { filepaths := []string{} for filepath := range builtin.templatesAndDataByDestRelFilepath { filepaths = append(filepaths, filepath) diff --git a/core/server/api_container/server/startosis_engine/kurtosis_instruction/request/request.go b/core/server/api_container/server/startosis_engine/kurtosis_instruction/request/request.go index 2a893d2672..538ec28afa 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_instruction/request/request.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_instruction/request/request.go @@ -209,7 +209,7 @@ func (builtin *RequestCapabilities) FillPersistableAttributes(builder *enclave_p builder.SetType(RequestBuiltinName) } -func (builtin *RequestCapabilities) UpdatePlan(plan *plan_yaml.PlanYaml) error { +func (builtin *RequestCapabilities) UpdatePlan(plan *plan_yaml.PlanYamlGenerator) error { // TODO: Implement logrus.Warn("REQUEST NOT IMPLEMENTED YET FOR UPDATING PLAN") return nil diff --git a/core/server/api_container/server/startosis_engine/kurtosis_instruction/set_service/set_service.go b/core/server/api_container/server/startosis_engine/kurtosis_instruction/set_service/set_service.go index 03f3d75094..d741ec199d 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_instruction/set_service/set_service.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_instruction/set_service/set_service.go @@ -187,7 +187,7 @@ func (builtin *SetServiceCapabilities) FillPersistableAttributes(builder *enclav builder.SetType(SetServiceBuiltinName).AddServiceName(builtin.serviceName) } -func (builtin *SetServiceCapabilities) UpdatePlan(planYaml *plan_yaml.PlanYaml) error { +func (builtin *SetServiceCapabilities) UpdatePlan(planYaml *plan_yaml.PlanYamlGenerator) error { // update service does not affect the plan return nil } diff --git a/core/server/api_container/server/startosis_engine/kurtosis_instruction/start_service/start_service.go b/core/server/api_container/server/startosis_engine/kurtosis_instruction/start_service/start_service.go index 2856274444..05822d05bc 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_instruction/start_service/start_service.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_instruction/start_service/start_service.go @@ -107,7 +107,7 @@ func (builtin *StartServiceCapabilities) FillPersistableAttributes(builder *encl ) } -func (builtin *StartServiceCapabilities) UpdatePlan(plan *plan_yaml.PlanYaml) error { +func (builtin *StartServiceCapabilities) UpdatePlan(plan *plan_yaml.PlanYamlGenerator) error { // start services doesn't affect the plan return nil } diff --git a/core/server/api_container/server/startosis_engine/kurtosis_instruction/stop_service/stop_service.go b/core/server/api_container/server/startosis_engine/kurtosis_instruction/stop_service/stop_service.go index 82fc1a86c7..6e7cb18898 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_instruction/stop_service/stop_service.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_instruction/stop_service/stop_service.go @@ -107,7 +107,7 @@ func (builtin *StopServiceCapabilities) FillPersistableAttributes(builder *encla ) } -func (builtin *StopServiceCapabilities) UpdatePlan(plan *plan_yaml.PlanYaml) error { +func (builtin *StopServiceCapabilities) UpdatePlan(plan *plan_yaml.PlanYamlGenerator) error { // stop service does not affect the plan return nil } diff --git a/core/server/api_container/server/startosis_engine/kurtosis_instruction/store_service_files/store_service_files.go b/core/server/api_container/server/startosis_engine/kurtosis_instruction/store_service_files/store_service_files.go index ab17ad1b5b..a5cbdc4de5 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_instruction/store_service_files/store_service_files.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_instruction/store_service_files/store_service_files.go @@ -180,7 +180,7 @@ func (builtin *StoreServiceFilesCapabilities) FillPersistableAttributes(builder ) } -func (builtin *StoreServiceFilesCapabilities) UpdatePlan(plan *plan_yaml.PlanYaml) error { +func (builtin *StoreServiceFilesCapabilities) UpdatePlan(plan *plan_yaml.PlanYamlGenerator) error { err := plan.AddStoreServiceFiles(builtin.artifactName, builtin.src) if err != nil { return stacktrace.Propagate(err, "An error occurred updating plan with store service files") diff --git a/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/run_python.go b/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/run_python.go index fd1e820452..86e1f66f85 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/run_python.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/run_python.go @@ -373,7 +373,7 @@ func (builtin *RunPythonCapabilities) FillPersistableAttributes(builder *enclave builder.SetType(RunPythonBuiltinName) } -func (builtin *RunPythonCapabilities) UpdatePlan(plan *plan_yaml.PlanYaml) error { +func (builtin *RunPythonCapabilities) UpdatePlan(plan *plan_yaml.PlanYamlGenerator) error { err := plan.AddRunPython(builtin.run, builtin.description, builtin.returnValue, builtin.serviceConfig, builtin.storeSpecList, builtin.pythonArguments, builtin.packages) if err != nil { return stacktrace.Propagate(err, "An error occurred updating plan with run python") diff --git a/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/run_sh.go b/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/run_sh.go index afe8de2ee2..a1e0b1cf4b 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/run_sh.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/run_sh.go @@ -329,7 +329,7 @@ func (builtin *RunShCapabilities) FillPersistableAttributes(builder *enclave_pla builder.SetType(RunShBuiltinName) } -func (builtin *RunShCapabilities) UpdatePlan(plan *plan_yaml.PlanYaml) error { +func (builtin *RunShCapabilities) UpdatePlan(plan *plan_yaml.PlanYamlGenerator) error { err := plan.AddRunSh(builtin.run, builtin.description, builtin.returnValue, builtin.serviceConfig, builtin.storeSpecList) if err != nil { return stacktrace.Propagate(err, "An error occurred adding run sh task to the plan") diff --git a/core/server/api_container/server/startosis_engine/kurtosis_instruction/upload_files/upload_files.go b/core/server/api_container/server/startosis_engine/kurtosis_instruction/upload_files/upload_files.go index b1b02397fc..80e6e2572b 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_instruction/upload_files/upload_files.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_instruction/upload_files/upload_files.go @@ -219,7 +219,7 @@ func (builtin *UploadFilesCapabilities) FillPersistableAttributes(builder *encla ) } -func (builtin *UploadFilesCapabilities) UpdatePlan(plan *plan_yaml.PlanYaml) error { +func (builtin *UploadFilesCapabilities) UpdatePlan(plan *plan_yaml.PlanYamlGenerator) error { err := plan.AddUploadFiles(builtin.artifactName, builtin.src) if err != nil { return stacktrace.Propagate(err, "An error occurred updating plan with upload files.") diff --git a/core/server/api_container/server/startosis_engine/kurtosis_instruction/verify/verify.go b/core/server/api_container/server/startosis_engine/kurtosis_instruction/verify/verify.go index 842f5d0ddf..b5944d85ea 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_instruction/verify/verify.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_instruction/verify/verify.go @@ -161,7 +161,7 @@ func (builtin *VerifyCapabilities) FillPersistableAttributes(builder *enclave_pl builder.SetType(VerifyBuiltinName) } -func (builtin *VerifyCapabilities) UpdatePlan(plan *plan_yaml.PlanYaml) error { +func (builtin *VerifyCapabilities) UpdatePlan(plan *plan_yaml.PlanYamlGenerator) error { // verify does not affect the plan return nil } diff --git a/core/server/api_container/server/startosis_engine/kurtosis_instruction/wait/wait.go b/core/server/api_container/server/startosis_engine/kurtosis_instruction/wait/wait.go index ee89af066e..6526cf6860 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_instruction/wait/wait.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_instruction/wait/wait.go @@ -294,7 +294,7 @@ func (builtin *WaitCapabilities) FillPersistableAttributes(builder *enclave_plan builder.SetType(WaitBuiltinName) } -func (builtin *WaitCapabilities) UpdatePlan(plan *plan_yaml.PlanYaml) error { +func (builtin *WaitCapabilities) UpdatePlan(plan *plan_yaml.PlanYamlGenerator) error { // wait does not affect the plan return nil } diff --git a/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/kurtosis_plan_instruction/kurtosis_plan_instruction_capabilities.go b/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/kurtosis_plan_instruction/kurtosis_plan_instruction_capabilities.go index 5037f15fad..47a2b32853 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/kurtosis_plan_instruction/kurtosis_plan_instruction_capabilities.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/kurtosis_plan_instruction/kurtosis_plan_instruction_capabilities.go @@ -28,5 +28,5 @@ type KurtosisPlanInstructionCapabilities interface { Description() string // UpdatePlan applies the effect of this instruction capabilities onto the yaml representation of the instruction plan. - UpdatePlan(plan *plan_yaml.PlanYaml) error + UpdatePlan(plan *plan_yaml.PlanYamlGenerator) error } diff --git a/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/kurtosis_plan_instruction/kurtosis_plan_instruction_internal.go b/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/kurtosis_plan_instruction/kurtosis_plan_instruction_internal.go index 968181cb55..41aa54aeda 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/kurtosis_plan_instruction/kurtosis_plan_instruction_internal.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/kurtosis_plan_instruction/kurtosis_plan_instruction_internal.go @@ -98,7 +98,7 @@ func (builtin *kurtosisPlanInstructionInternal) GetPersistableAttributes() *encl return enclavePlaneInstructionBuilder.SetStarlarkCode(builtin.String()) } -func (builtin *kurtosisPlanInstructionInternal) UpdatePlan(plan *plan_yaml.PlanYaml) error { +func (builtin *kurtosisPlanInstructionInternal) UpdatePlan(plan *plan_yaml.PlanYamlGenerator) error { return builtin.capabilities.UpdatePlan(plan) } diff --git a/core/server/api_container/server/startosis_engine/plan_yaml/plan_yaml.go b/core/server/api_container/server/startosis_engine/plan_yaml/plan_yaml.go index df7b2d17dc..fe04379788 100644 --- a/core/server/api_container/server/startosis_engine/plan_yaml/plan_yaml.go +++ b/core/server/api_container/server/startosis_engine/plan_yaml/plan_yaml.go @@ -8,7 +8,7 @@ const ( exec TaskType = "exec" ) -type privatePlanYaml struct { +type PlanYaml struct { PackageId string `yaml:"packageId,omitempty"` Services []*Service `yaml:"services,omitempty"` FilesArtifacts []*FilesArtifact `yaml:"filesArtifacts,omitempty"` diff --git a/core/server/api_container/server/startosis_engine/plan_yaml/plan_yaml_generator.go b/core/server/api_container/server/startosis_engine/plan_yaml/plan_yaml_generator.go index f2aaa3c708..e7098c83db 100644 --- a/core/server/api_container/server/startosis_engine/plan_yaml/plan_yaml_generator.go +++ b/core/server/api_container/server/startosis_engine/plan_yaml/plan_yaml_generator.go @@ -22,9 +22,9 @@ const ( outputFutureRefType = "output" ) -// PlanYaml is a yaml representation of the effect of an Instructions Plan or sequence of instructions on the state of the Enclave. -type PlanYaml struct { - privatePlanYaml *privatePlanYaml +// PlanYamlGenerator is a yaml representation of the effect of an Instructions Plan or sequence of instructions on the state of the Enclave. +type PlanYamlGenerator struct { + privatePlanYaml *PlanYaml futureReferenceIndex map[string]string filesArtifactIndex map[string]*FilesArtifact @@ -32,9 +32,9 @@ type PlanYaml struct { imageSet map[string]bool } -func CreateEmptyPlan(packageId string) *PlanYaml { - return &PlanYaml{ - privatePlanYaml: &privatePlanYaml{ +func CreateEmptyPlan(packageId string) *PlanYamlGenerator { + return &PlanYamlGenerator{ + privatePlanYaml: &PlanYaml{ PackageId: packageId, Services: []*Service{}, Tasks: []*Task{}, @@ -49,7 +49,7 @@ func CreateEmptyPlan(packageId string) *PlanYaml { } } -func (planYaml *PlanYaml) GenerateYaml() (string, error) { +func (planYaml *PlanYamlGenerator) GenerateYaml() (string, error) { planYaml.AddImages() yamlBytes, err := yaml.Marshal(planYaml.privatePlanYaml) if err != nil { @@ -58,7 +58,7 @@ func (planYaml *PlanYaml) GenerateYaml() (string, error) { return string(yamlBytes), nil } -func (planYaml *PlanYaml) AddService( +func (planYaml *PlanYamlGenerator) AddService( serviceName service.ServiceName, serviceInfo *kurtosis_types.Service, serviceConfig *service.ServiceConfig, @@ -138,7 +138,7 @@ func (planYaml *PlanYaml) AddService( return nil } -func (planYaml *PlanYaml) AddRunSh( +func (planYaml *PlanYamlGenerator) AddRunSh( runCommand string, description string, returnValue *starlarkstruct.Struct, @@ -213,7 +213,7 @@ func (planYaml *PlanYaml) AddRunSh( return nil } -func (planYaml *PlanYaml) AddRunPython( +func (planYaml *PlanYamlGenerator) AddRunPython( runCommand string, description string, returnValue *starlarkstruct.Struct, @@ -293,7 +293,7 @@ func (planYaml *PlanYaml) AddRunPython( return nil } -func (planYaml *PlanYaml) AddExec( +func (planYaml *PlanYamlGenerator) AddExec( serviceName string, description string, returnValue *starlark.Dict, @@ -345,7 +345,7 @@ func (planYaml *PlanYaml) AddExec( return nil } -func (planYaml *PlanYaml) AddRenderTemplates(filesArtifactName string, filepaths []string) error { +func (planYaml *PlanYamlGenerator) AddRenderTemplates(filesArtifactName string, filepaths []string) error { uuid := planYaml.generateUuid() filesArtifactYaml := &FilesArtifact{} //nolint exhaustruct filesArtifactYaml.Uuid = uuid @@ -355,7 +355,7 @@ func (planYaml *PlanYaml) AddRenderTemplates(filesArtifactName string, filepaths return nil } -func (planYaml *PlanYaml) AddUploadFiles(filesArtifactName, locator string) error { +func (planYaml *PlanYamlGenerator) AddUploadFiles(filesArtifactName, locator string) error { uuid := planYaml.generateUuid() filesArtifactYaml := &FilesArtifact{} //nolint exhauststruct filesArtifactYaml.Uuid = uuid @@ -365,7 +365,7 @@ func (planYaml *PlanYaml) AddUploadFiles(filesArtifactName, locator string) erro return nil } -func (planYaml *PlanYaml) AddStoreServiceFiles(filesArtifactName, locator string) error { +func (planYaml *PlanYamlGenerator) AddStoreServiceFiles(filesArtifactName, locator string) error { uuid := planYaml.generateUuid() filesArtifactYaml := &FilesArtifact{} //nolint exhaustruct filesArtifactYaml.Uuid = uuid @@ -375,7 +375,7 @@ func (planYaml *PlanYaml) AddStoreServiceFiles(filesArtifactName, locator string return nil } -func (planYaml *PlanYaml) RemoveService(serviceName string) { +func (planYaml *PlanYamlGenerator) RemoveService(serviceName string) { for idx, service := range planYaml.privatePlanYaml.Services { if service.Name == serviceName { planYaml.privatePlanYaml.Services = slices.Delete(planYaml.privatePlanYaml.Services, idx, idx+1) @@ -384,14 +384,14 @@ func (planYaml *PlanYaml) RemoveService(serviceName string) { } } -func (planYaml *PlanYaml) AddPackageDependencies(packageDependency map[string]bool) { +func (planYaml *PlanYamlGenerator) AddPackageDependencies(packageDependency map[string]bool) { for dependency := range packageDependency { planYaml.privatePlanYaml.PackageDependencies = append(planYaml.privatePlanYaml.PackageDependencies, dependency) } slices.Sort(planYaml.privatePlanYaml.PackageDependencies) } -func (planYaml *PlanYaml) AddImages() { +func (planYaml *PlanYamlGenerator) AddImages() { for img := range planYaml.imageSet { planYaml.privatePlanYaml.Images = append(planYaml.privatePlanYaml.Images, img) } @@ -406,7 +406,7 @@ func (planYaml *PlanYaml) AddImages() { // - create new files artifact // - add the files artifact to the plan // - add it to as a file mount accordingly -func (planYaml *PlanYaml) getFileMountsFromFilesArtifacts(filesArtifactExpansion *service_directory.FilesArtifactsExpansion) []*FileMount { +func (planYaml *PlanYamlGenerator) getFileMountsFromFilesArtifacts(filesArtifactExpansion *service_directory.FilesArtifactsExpansion) []*FileMount { var fileMounts []*FileMount if filesArtifactExpansion == nil { return fileMounts @@ -443,30 +443,30 @@ func (planYaml *PlanYaml) getFileMountsFromFilesArtifacts(filesArtifactExpansion return fileMounts } -func (planYaml *PlanYaml) addServiceYaml(service *Service) { +func (planYaml *PlanYamlGenerator) addServiceYaml(service *Service) { planYaml.privatePlanYaml.Services = append(planYaml.privatePlanYaml.Services, service) } -func (planYaml *PlanYaml) addFilesArtifactYaml(filesArtifact *FilesArtifact) { +func (planYaml *PlanYamlGenerator) addFilesArtifactYaml(filesArtifact *FilesArtifact) { planYaml.filesArtifactIndex[filesArtifact.Name] = filesArtifact planYaml.privatePlanYaml.FilesArtifacts = append(planYaml.privatePlanYaml.FilesArtifacts, filesArtifact) } -func (planYaml *PlanYaml) addTaskYaml(task *Task) { +func (planYaml *PlanYamlGenerator) addTaskYaml(task *Task) { planYaml.privatePlanYaml.Tasks = append(planYaml.privatePlanYaml.Tasks, task) } -func (planYaml *PlanYaml) addImage(img string) { +func (planYaml *PlanYamlGenerator) addImage(img string) { planYaml.imageSet[img] = true } // yaml future reference format: {{ kurtosis.. Date: Fri, 30 Aug 2024 11:37:34 -0400 Subject: [PATCH 26/31] only add module prefix and dont add parent package --- .../server/startosis_engine/startosis_interpreter.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/core/server/api_container/server/startosis_engine/startosis_interpreter.go b/core/server/api_container/server/startosis_engine/startosis_interpreter.go index 9f1bd3fba9..b26fef893a 100644 --- a/core/server/api_container/server/startosis_engine/startosis_interpreter.go +++ b/core/server/api_container/server/startosis_engine/startosis_interpreter.go @@ -350,6 +350,10 @@ func (interpreter *StartosisInterpreter) interpretInternal( return globalVariables, nil } +func getModulePrefix(moduleId string) string { + return strings.Join(strings.SplitN(moduleId, "/", 4)[:3], "/") +} + func (interpreter *StartosisInterpreter) buildBindings( packageId string, thread *starlark.Thread, @@ -357,9 +361,12 @@ func (interpreter *StartosisInterpreter) buildBindings( moduleGlobalCache map[string]*startosis_packages.ModuleCacheEntry, packageReplaceOptions map[string]string, ) (*starlark.StringDict, *startosis_errors.InterpretationError) { + packagePrefix := getModulePrefix(packageId) recursiveInterpretForModuleLoading := func(moduleId string, serializedStartosis string) (starlark.StringDict, *startosis_errors.InterpretationError) { - logrus.Infof("PACKAGE DEPENDENCY: %v", moduleId) - instructionPlan.AddPackageDependency(moduleId) + modulePrefix := getModulePrefix(moduleId) + if modulePrefix != packagePrefix { + instructionPlan.AddPackageDependency(modulePrefix) + } result, err := interpreter.interpretInternal(packageId, moduleId, serializedStartosis, instructionPlan, moduleGlobalCache, packageReplaceOptions) if err != nil { return nil, err From c1cd1fc37b6e7e75671ec7fe8fee91333e3d9b09 Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Fri, 30 Aug 2024 12:20:36 -0400 Subject: [PATCH 27/31] maybe parse yaml --- .../core/lib/enclaves/enclave_context.go | 8 ++++++ cli/cli/commands/run/run.go | 27 +++++++++---------- 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/api/golang/core/lib/enclaves/enclave_context.go b/api/golang/core/lib/enclaves/enclave_context.go index bb54691c61..589dbd0055 100755 --- a/api/golang/core/lib/enclaves/enclave_context.go +++ b/api/golang/core/lib/enclaves/enclave_context.go @@ -532,6 +532,10 @@ func (enclaveCtx *EnclaveContext) GetStarlarkRun(ctx context.Context) (*kurtosis } func (enclaveCtx *EnclaveContext) GetStarlarkPackagePlanYaml(ctx context.Context, packageId string, serializedParams string) (*kurtosis_core_rpc_api_bindings.PlanYaml, error) { + serializedParams, err := maybeParseYaml(serializedParams) + if err != nil { + return nil, stacktrace.Propagate(err, "An error occurred when parsing YAML args for package '%v'", serializedParams) + } response, err := enclaveCtx.client.GetStarlarkPackagePlanYaml(ctx, &kurtosis_core_rpc_api_bindings.StarlarkPackagePlanYamlArgs{ PackageId: packageId, SerializedParams: &serializedParams, @@ -545,6 +549,10 @@ func (enclaveCtx *EnclaveContext) GetStarlarkPackagePlanYaml(ctx context.Context } func (enclaveCtx *EnclaveContext) GetStarlarkScriptPlanYaml(ctx context.Context, serializedScript string, serializedParams string) (*kurtosis_core_rpc_api_bindings.PlanYaml, error) { + serializedParams, err := maybeParseYaml(serializedParams) + if err != nil { + return nil, stacktrace.Propagate(err, "An error occurred when parsing YAML args for package '%v'", serializedParams) + } response, err := enclaveCtx.client.GetStarlarkScriptPlanYaml(ctx, &kurtosis_core_rpc_api_bindings.StarlarkScriptPlanYamlArgs{ SerializedScript: serializedScript, SerializedParams: &serializedParams, diff --git a/cli/cli/commands/run/run.go b/cli/cli/commands/run/run.go index ce6358ce9e..8f0529ec4c 100755 --- a/cli/cli/commands/run/run.go +++ b/cli/cli/commands/run/run.go @@ -5,7 +5,6 @@ import ( "encoding/json" "fmt" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/user_support_constants" - "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/plan_yaml" "io" "net/http" "net/url" @@ -635,19 +634,19 @@ func getPackageDependencyYamlStr( if err != nil { return "", stacktrace.Propagate(err, "An error occurred retrieving plan yaml for provided package.") } - var planYaml plan_yaml.PlanYaml - if err = yaml.Unmarshal([]byte(packageYaml.PlanYaml), &planYaml); err != nil { - return "", stacktrace.Propagate(err, "An error occurred unmarshaling plan yaml string: %v.", packageYaml.PlanYaml) - } - planYaml.Tasks = nil - planYaml.FilesArtifacts = nil - planYaml.Services = nil - packageDependencyYamlBytes, err := yaml.Marshal(&planYaml) - if err != nil { - return "", stacktrace.Propagate(err, "") - } - return string(packageDependencyYamlBytes), nil - //return packageYaml.PlanYaml, nil + //var planYaml plan_yaml.PlanYaml + //if err = yaml.Unmarshal([]byte(packageYaml.PlanYaml), &planYaml); err != nil { + // return "", stacktrace.Propagate(err, "An error occurred unmarshaling plan yaml string: %v.", packageYaml.PlanYaml) + //} + //planYaml.Tasks = nil + //planYaml.FilesArtifacts = nil + //planYaml.Services = nil + //packageDependencyYamlBytes, err := yaml.Marshal(&planYaml) + //if err != nil { + // return "", stacktrace.Propagate(err, "") + //} + //return string(packageDependencyYamlBytes), nil + return packageYaml.PlanYaml, nil } // validatePackageArgs just validates the args is a valid JSON or YAML string From 554b754abc75b7998b3fb510ca0288a573864403 Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Wed, 11 Sep 2024 11:05:45 -0400 Subject: [PATCH 28/31] pull dependencies locally --- .../api_container_service.pb.go | 73 ++++--- .../core/lib/enclaves/enclave_context.go | 6 +- api/protobuf/core/api_container_service.proto | 6 + api/rust/src/api_container_api.rs | 6 + .../api_container_service_pb.d.ts | 22 +++ .../api_container_service_pb.js | 100 +++++++++- .../connect/api_container_service_pb.d.ts | 14 ++ .../connect/api_container_service_pb.js | 2 + cli/cli/commands/run/run.go | 178 ++++++++++++++---- .../server/api_container_service.go | 6 +- .../instructions_plan/instructions_plan.go | 4 +- .../plan_yaml/plan_yaml_generator.go | 17 +- .../startosis_engine/startosis_interpreter.go | 14 +- .../startosis_interpreter_plan_yaml_test.go | 22 +-- .../startosis_interpreter_test.go | 11 ++ 15 files changed, 398 insertions(+), 83 deletions(-) diff --git a/api/golang/core/kurtosis_core_rpc_api_bindings/api_container_service.pb.go b/api/golang/core/kurtosis_core_rpc_api_bindings/api_container_service.pb.go index ebb3f041d2..af0a975ee5 100644 --- a/api/golang/core/kurtosis_core_rpc_api_bindings/api_container_service.pb.go +++ b/api/golang/core/kurtosis_core_rpc_api_bindings/api_container_service.pb.go @@ -3381,6 +3381,8 @@ type StarlarkScriptPlanYamlArgs struct { SerializedParams *string `protobuf:"bytes,2,opt,name=serialized_params,json=serializedParams,proto3,oneof" json:"serialized_params,omitempty"` // The name of the main function, the default value is "run" MainFunctionName *string `protobuf:"bytes,5,opt,name=main_function_name,json=mainFunctionName,proto3,oneof" json:"main_function_name,omitempty"` + // Whether to return only the dependencies of the starlark script + DependenciesOnly *bool `protobuf:"varint,6,opt,name=dependencies_only,json=dependenciesOnly,proto3,oneof" json:"dependencies_only,omitempty"` } func (x *StarlarkScriptPlanYamlArgs) Reset() { @@ -3436,6 +3438,13 @@ func (x *StarlarkScriptPlanYamlArgs) GetMainFunctionName() string { return "" } +func (x *StarlarkScriptPlanYamlArgs) GetDependenciesOnly() bool { + if x != nil && x.DependenciesOnly != nil { + return *x.DependenciesOnly + } + return false +} + type StarlarkPackagePlanYamlArgs struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -3449,6 +3458,8 @@ type StarlarkPackagePlanYamlArgs struct { RelativePathToMainFile *string `protobuf:"bytes,3,opt,name=relative_path_to_main_file,json=relativePathToMainFile,proto3,oneof" json:"relative_path_to_main_file,omitempty"` // The name of the main function, the default value is "run" MainFunctionName *string `protobuf:"bytes,4,opt,name=main_function_name,json=mainFunctionName,proto3,oneof" json:"main_function_name,omitempty"` + // Whether to return only the dependencies of the Starlark package + DependenciesOnly *bool `protobuf:"varint,5,opt,name=dependencies_only,json=dependenciesOnly,proto3,oneof" json:"dependencies_only,omitempty"` } func (x *StarlarkPackagePlanYamlArgs) Reset() { @@ -3511,6 +3522,13 @@ func (x *StarlarkPackagePlanYamlArgs) GetMainFunctionName() string { return "" } +func (x *StarlarkPackagePlanYamlArgs) GetDependenciesOnly() bool { + if x != nil && x.DependenciesOnly != nil { + return *x.DependenciesOnly + } + return false +} + var File_api_container_service_proto protoreflect.FileDescriptor var file_api_container_service_proto_rawDesc = []byte{ @@ -4085,7 +4103,7 @@ var file_api_container_service_proto_rawDesc = []byte{ 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x64, 0x5f, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x22, 0x27, 0x0a, 0x08, 0x50, 0x6c, 0x61, 0x6e, 0x59, 0x61, 0x6d, 0x6c, 0x12, 0x1b, 0x0a, 0x09, 0x70, 0x6c, 0x61, 0x6e, 0x5f, 0x79, 0x61, 0x6d, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x70, - 0x6c, 0x61, 0x6e, 0x59, 0x61, 0x6d, 0x6c, 0x22, 0xdb, 0x01, 0x0a, 0x1a, 0x53, 0x74, 0x61, 0x72, + 0x6c, 0x61, 0x6e, 0x59, 0x61, 0x6d, 0x6c, 0x22, 0xa3, 0x02, 0x0a, 0x1a, 0x53, 0x74, 0x61, 0x72, 0x6c, 0x61, 0x72, 0x6b, 0x53, 0x63, 0x72, 0x69, 0x70, 0x74, 0x50, 0x6c, 0x61, 0x6e, 0x59, 0x61, 0x6d, 0x6c, 0x41, 0x72, 0x67, 0x73, 0x12, 0x2b, 0x0a, 0x11, 0x73, 0x65, 0x72, 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x64, 0x5f, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, @@ -4096,29 +4114,38 @@ var file_api_container_service_proto_rawDesc = []byte{ 0x6d, 0x73, 0x88, 0x01, 0x01, 0x12, 0x31, 0x0a, 0x12, 0x6d, 0x61, 0x69, 0x6e, 0x5f, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x48, 0x01, 0x52, 0x10, 0x6d, 0x61, 0x69, 0x6e, 0x46, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, - 0x6e, 0x4e, 0x61, 0x6d, 0x65, 0x88, 0x01, 0x01, 0x42, 0x14, 0x0a, 0x12, 0x5f, 0x73, 0x65, 0x72, - 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x64, 0x5f, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x42, 0x15, - 0x0a, 0x13, 0x5f, 0x6d, 0x61, 0x69, 0x6e, 0x5f, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, - 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x22, 0xae, 0x02, 0x0a, 0x1b, 0x53, 0x74, 0x61, 0x72, 0x6c, 0x61, - 0x72, 0x6b, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x50, 0x6c, 0x61, 0x6e, 0x59, 0x61, 0x6d, - 0x6c, 0x41, 0x72, 0x67, 0x73, 0x12, 0x1d, 0x0a, 0x0a, 0x70, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, - 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x70, 0x61, 0x63, 0x6b, 0x61, - 0x67, 0x65, 0x49, 0x64, 0x12, 0x30, 0x0a, 0x11, 0x73, 0x65, 0x72, 0x69, 0x61, 0x6c, 0x69, 0x7a, - 0x65, 0x64, 0x5f, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x48, - 0x00, 0x52, 0x10, 0x73, 0x65, 0x72, 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x64, 0x50, 0x61, 0x72, - 0x61, 0x6d, 0x73, 0x88, 0x01, 0x01, 0x12, 0x3f, 0x0a, 0x1a, 0x72, 0x65, 0x6c, 0x61, 0x74, 0x69, - 0x76, 0x65, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x5f, 0x74, 0x6f, 0x5f, 0x6d, 0x61, 0x69, 0x6e, 0x5f, - 0x66, 0x69, 0x6c, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x48, 0x01, 0x52, 0x16, 0x72, 0x65, - 0x6c, 0x61, 0x74, 0x69, 0x76, 0x65, 0x50, 0x61, 0x74, 0x68, 0x54, 0x6f, 0x4d, 0x61, 0x69, 0x6e, - 0x46, 0x69, 0x6c, 0x65, 0x88, 0x01, 0x01, 0x12, 0x31, 0x0a, 0x12, 0x6d, 0x61, 0x69, 0x6e, 0x5f, - 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x04, 0x20, - 0x01, 0x28, 0x09, 0x48, 0x02, 0x52, 0x10, 0x6d, 0x61, 0x69, 0x6e, 0x46, 0x75, 0x6e, 0x63, 0x74, - 0x69, 0x6f, 0x6e, 0x4e, 0x61, 0x6d, 0x65, 0x88, 0x01, 0x01, 0x42, 0x14, 0x0a, 0x12, 0x5f, 0x73, + 0x6e, 0x4e, 0x61, 0x6d, 0x65, 0x88, 0x01, 0x01, 0x12, 0x30, 0x0a, 0x11, 0x64, 0x65, 0x70, 0x65, + 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x69, 0x65, 0x73, 0x5f, 0x6f, 0x6e, 0x6c, 0x79, 0x18, 0x06, 0x20, + 0x01, 0x28, 0x08, 0x48, 0x02, 0x52, 0x10, 0x64, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, + 0x69, 0x65, 0x73, 0x4f, 0x6e, 0x6c, 0x79, 0x88, 0x01, 0x01, 0x42, 0x14, 0x0a, 0x12, 0x5f, 0x73, 0x65, 0x72, 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x64, 0x5f, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, - 0x42, 0x1d, 0x0a, 0x1b, 0x5f, 0x72, 0x65, 0x6c, 0x61, 0x74, 0x69, 0x76, 0x65, 0x5f, 0x70, 0x61, - 0x74, 0x68, 0x5f, 0x74, 0x6f, 0x5f, 0x6d, 0x61, 0x69, 0x6e, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x42, - 0x15, 0x0a, 0x13, 0x5f, 0x6d, 0x61, 0x69, 0x6e, 0x5f, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, - 0x6e, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x2a, 0x36, 0x0a, 0x0d, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, + 0x42, 0x15, 0x0a, 0x13, 0x5f, 0x6d, 0x61, 0x69, 0x6e, 0x5f, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, + 0x6f, 0x6e, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x42, 0x14, 0x0a, 0x12, 0x5f, 0x64, 0x65, 0x70, 0x65, + 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x69, 0x65, 0x73, 0x5f, 0x6f, 0x6e, 0x6c, 0x79, 0x22, 0xf6, 0x02, + 0x0a, 0x1b, 0x53, 0x74, 0x61, 0x72, 0x6c, 0x61, 0x72, 0x6b, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, + 0x65, 0x50, 0x6c, 0x61, 0x6e, 0x59, 0x61, 0x6d, 0x6c, 0x41, 0x72, 0x67, 0x73, 0x12, 0x1d, 0x0a, + 0x0a, 0x70, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x09, 0x70, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x49, 0x64, 0x12, 0x30, 0x0a, 0x11, + 0x73, 0x65, 0x72, 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x64, 0x5f, 0x70, 0x61, 0x72, 0x61, 0x6d, + 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x10, 0x73, 0x65, 0x72, 0x69, 0x61, + 0x6c, 0x69, 0x7a, 0x65, 0x64, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x88, 0x01, 0x01, 0x12, 0x3f, + 0x0a, 0x1a, 0x72, 0x65, 0x6c, 0x61, 0x74, 0x69, 0x76, 0x65, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x5f, + 0x74, 0x6f, 0x5f, 0x6d, 0x61, 0x69, 0x6e, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x09, 0x48, 0x01, 0x52, 0x16, 0x72, 0x65, 0x6c, 0x61, 0x74, 0x69, 0x76, 0x65, 0x50, 0x61, + 0x74, 0x68, 0x54, 0x6f, 0x4d, 0x61, 0x69, 0x6e, 0x46, 0x69, 0x6c, 0x65, 0x88, 0x01, 0x01, 0x12, + 0x31, 0x0a, 0x12, 0x6d, 0x61, 0x69, 0x6e, 0x5f, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, + 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x48, 0x02, 0x52, 0x10, 0x6d, + 0x61, 0x69, 0x6e, 0x46, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4e, 0x61, 0x6d, 0x65, 0x88, + 0x01, 0x01, 0x12, 0x30, 0x0a, 0x11, 0x64, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x69, + 0x65, 0x73, 0x5f, 0x6f, 0x6e, 0x6c, 0x79, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x48, 0x03, 0x52, + 0x10, 0x64, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x69, 0x65, 0x73, 0x4f, 0x6e, 0x6c, + 0x79, 0x88, 0x01, 0x01, 0x42, 0x14, 0x0a, 0x12, 0x5f, 0x73, 0x65, 0x72, 0x69, 0x61, 0x6c, 0x69, + 0x7a, 0x65, 0x64, 0x5f, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x42, 0x1d, 0x0a, 0x1b, 0x5f, 0x72, + 0x65, 0x6c, 0x61, 0x74, 0x69, 0x76, 0x65, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x5f, 0x74, 0x6f, 0x5f, + 0x6d, 0x61, 0x69, 0x6e, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x42, 0x15, 0x0a, 0x13, 0x5f, 0x6d, 0x61, + 0x69, 0x6e, 0x5f, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6e, 0x61, 0x6d, 0x65, + 0x42, 0x14, 0x0a, 0x12, 0x5f, 0x64, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x69, 0x65, + 0x73, 0x5f, 0x6f, 0x6e, 0x6c, 0x79, 0x2a, 0x36, 0x0a, 0x0d, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x0b, 0x0a, 0x07, 0x53, 0x54, 0x4f, 0x50, 0x50, 0x45, 0x44, 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07, 0x52, 0x55, 0x4e, 0x4e, 0x49, 0x4e, 0x47, 0x10, 0x01, 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x02, 0x2a, 0x2c, diff --git a/api/golang/core/lib/enclaves/enclave_context.go b/api/golang/core/lib/enclaves/enclave_context.go index 589dbd0055..eb5707a8be 100755 --- a/api/golang/core/lib/enclaves/enclave_context.go +++ b/api/golang/core/lib/enclaves/enclave_context.go @@ -531,7 +531,7 @@ func (enclaveCtx *EnclaveContext) GetStarlarkRun(ctx context.Context) (*kurtosis return response, nil } -func (enclaveCtx *EnclaveContext) GetStarlarkPackagePlanYaml(ctx context.Context, packageId string, serializedParams string) (*kurtosis_core_rpc_api_bindings.PlanYaml, error) { +func (enclaveCtx *EnclaveContext) GetStarlarkPackagePlanYaml(ctx context.Context, packageId string, serializedParams string, dependenciesOnly bool) (*kurtosis_core_rpc_api_bindings.PlanYaml, error) { serializedParams, err := maybeParseYaml(serializedParams) if err != nil { return nil, stacktrace.Propagate(err, "An error occurred when parsing YAML args for package '%v'", serializedParams) @@ -541,6 +541,7 @@ func (enclaveCtx *EnclaveContext) GetStarlarkPackagePlanYaml(ctx context.Context SerializedParams: &serializedParams, RelativePathToMainFile: nil, MainFunctionName: nil, + DependenciesOnly: &dependenciesOnly, }) if err != nil { return nil, stacktrace.Propagate(err, "An error occurred while getting the starlark package plan yaml run.") @@ -548,7 +549,7 @@ func (enclaveCtx *EnclaveContext) GetStarlarkPackagePlanYaml(ctx context.Context return response, nil } -func (enclaveCtx *EnclaveContext) GetStarlarkScriptPlanYaml(ctx context.Context, serializedScript string, serializedParams string) (*kurtosis_core_rpc_api_bindings.PlanYaml, error) { +func (enclaveCtx *EnclaveContext) GetStarlarkScriptPlanYaml(ctx context.Context, serializedScript string, serializedParams string, dependenciesOnly bool) (*kurtosis_core_rpc_api_bindings.PlanYaml, error) { serializedParams, err := maybeParseYaml(serializedParams) if err != nil { return nil, stacktrace.Propagate(err, "An error occurred when parsing YAML args for package '%v'", serializedParams) @@ -557,6 +558,7 @@ func (enclaveCtx *EnclaveContext) GetStarlarkScriptPlanYaml(ctx context.Context, SerializedScript: serializedScript, SerializedParams: &serializedParams, MainFunctionName: nil, + DependenciesOnly: &dependenciesOnly, }) if err != nil { return nil, stacktrace.Propagate(err, "An error occurred while getting the last starlark script plan yaml run.") diff --git a/api/protobuf/core/api_container_service.proto b/api/protobuf/core/api_container_service.proto index cc4d0aacaa..3bded04cad 100644 --- a/api/protobuf/core/api_container_service.proto +++ b/api/protobuf/core/api_container_service.proto @@ -598,6 +598,9 @@ message StarlarkScriptPlanYamlArgs { // The name of the main function, the default value is "run" optional string main_function_name = 5; + + // Whether to return only the dependencies of the starlark script + optional bool dependencies_only = 6; } message StarlarkPackagePlanYamlArgs { @@ -612,4 +615,7 @@ message StarlarkPackagePlanYamlArgs { // The name of the main function, the default value is "run" optional string main_function_name = 4; + + // Whether to return only the dependencies of the Starlark package + optional bool dependencies_only = 5; } diff --git a/api/rust/src/api_container_api.rs b/api/rust/src/api_container_api.rs index a3de2c65bc..c81e2200ad 100644 --- a/api/rust/src/api_container_api.rs +++ b/api/rust/src/api_container_api.rs @@ -709,6 +709,9 @@ pub struct StarlarkScriptPlanYamlArgs { /// The name of the main function, the default value is "run" #[prost(string, optional, tag = "5")] pub main_function_name: ::core::option::Option<::prost::alloc::string::String>, + /// Whether to return only the dependencies of the starlark script + #[prost(bool, optional, tag = "6")] + pub dependencies_only: ::core::option::Option, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -727,6 +730,9 @@ pub struct StarlarkPackagePlanYamlArgs { /// The name of the main function, the default value is "run" #[prost(string, optional, tag = "4")] pub main_function_name: ::core::option::Option<::prost::alloc::string::String>, + /// Whether to return only the dependencies of the Starlark package + #[prost(bool, optional, tag = "5")] + pub dependencies_only: ::core::option::Option, } #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] #[repr(i32)] diff --git a/api/typescript/src/core/kurtosis_core_rpc_api_bindings/api_container_service_pb.d.ts b/api/typescript/src/core/kurtosis_core_rpc_api_bindings/api_container_service_pb.d.ts index 1d23b22991..2fb6b04a0f 100644 --- a/api/typescript/src/core/kurtosis_core_rpc_api_bindings/api_container_service_pb.d.ts +++ b/api/typescript/src/core/kurtosis_core_rpc_api_bindings/api_container_service_pb.d.ts @@ -1511,6 +1511,11 @@ export class StarlarkScriptPlanYamlArgs extends jspb.Message { hasMainFunctionName(): boolean; clearMainFunctionName(): StarlarkScriptPlanYamlArgs; + getDependenciesOnly(): boolean; + setDependenciesOnly(value: boolean): StarlarkScriptPlanYamlArgs; + hasDependenciesOnly(): boolean; + clearDependenciesOnly(): StarlarkScriptPlanYamlArgs; + serializeBinary(): Uint8Array; toObject(includeInstance?: boolean): StarlarkScriptPlanYamlArgs.AsObject; static toObject(includeInstance: boolean, msg: StarlarkScriptPlanYamlArgs): StarlarkScriptPlanYamlArgs.AsObject; @@ -1524,6 +1529,7 @@ export namespace StarlarkScriptPlanYamlArgs { serializedScript: string, serializedParams?: string, mainFunctionName?: string, + dependenciesOnly?: boolean, } export enum SerializedParamsCase { @@ -1535,6 +1541,11 @@ export namespace StarlarkScriptPlanYamlArgs { _MAIN_FUNCTION_NAME_NOT_SET = 0, MAIN_FUNCTION_NAME = 5, } + + export enum DependenciesOnlyCase { + _DEPENDENCIES_ONLY_NOT_SET = 0, + DEPENDENCIES_ONLY = 6, + } } export class StarlarkPackagePlanYamlArgs extends jspb.Message { @@ -1556,6 +1567,11 @@ export class StarlarkPackagePlanYamlArgs extends jspb.Message { hasMainFunctionName(): boolean; clearMainFunctionName(): StarlarkPackagePlanYamlArgs; + getDependenciesOnly(): boolean; + setDependenciesOnly(value: boolean): StarlarkPackagePlanYamlArgs; + hasDependenciesOnly(): boolean; + clearDependenciesOnly(): StarlarkPackagePlanYamlArgs; + serializeBinary(): Uint8Array; toObject(includeInstance?: boolean): StarlarkPackagePlanYamlArgs.AsObject; static toObject(includeInstance: boolean, msg: StarlarkPackagePlanYamlArgs): StarlarkPackagePlanYamlArgs.AsObject; @@ -1570,6 +1586,7 @@ export namespace StarlarkPackagePlanYamlArgs { serializedParams?: string, relativePathToMainFile?: string, mainFunctionName?: string, + dependenciesOnly?: boolean, } export enum SerializedParamsCase { @@ -1586,6 +1603,11 @@ export namespace StarlarkPackagePlanYamlArgs { _MAIN_FUNCTION_NAME_NOT_SET = 0, MAIN_FUNCTION_NAME = 4, } + + export enum DependenciesOnlyCase { + _DEPENDENCIES_ONLY_NOT_SET = 0, + DEPENDENCIES_ONLY = 5, + } } export enum ServiceStatus { diff --git a/api/typescript/src/core/kurtosis_core_rpc_api_bindings/api_container_service_pb.js b/api/typescript/src/core/kurtosis_core_rpc_api_bindings/api_container_service_pb.js index d0c4855650..714c868f8a 100644 --- a/api/typescript/src/core/kurtosis_core_rpc_api_bindings/api_container_service_pb.js +++ b/api/typescript/src/core/kurtosis_core_rpc_api_bindings/api_container_service_pb.js @@ -10882,7 +10882,8 @@ proto.api_container_api.StarlarkScriptPlanYamlArgs.toObject = function(includeIn var f, obj = { serializedScript: jspb.Message.getFieldWithDefault(msg, 1, ""), serializedParams: jspb.Message.getFieldWithDefault(msg, 2, ""), - mainFunctionName: jspb.Message.getFieldWithDefault(msg, 5, "") + mainFunctionName: jspb.Message.getFieldWithDefault(msg, 5, ""), + dependenciesOnly: jspb.Message.getBooleanFieldWithDefault(msg, 6, false) }; if (includeInstance) { @@ -10931,6 +10932,10 @@ proto.api_container_api.StarlarkScriptPlanYamlArgs.deserializeBinaryFromReader = var value = /** @type {string} */ (reader.readString()); msg.setMainFunctionName(value); break; + case 6: + var value = /** @type {boolean} */ (reader.readBool()); + msg.setDependenciesOnly(value); + break; default: reader.skipField(); break; @@ -10981,6 +10986,13 @@ proto.api_container_api.StarlarkScriptPlanYamlArgs.serializeBinaryToWriter = fun f ); } + f = /** @type {boolean} */ (jspb.Message.getField(message, 6)); + if (f != null) { + writer.writeBool( + 6, + f + ); + } }; @@ -11074,6 +11086,42 @@ proto.api_container_api.StarlarkScriptPlanYamlArgs.prototype.hasMainFunctionName }; +/** + * optional bool dependencies_only = 6; + * @return {boolean} + */ +proto.api_container_api.StarlarkScriptPlanYamlArgs.prototype.getDependenciesOnly = function() { + return /** @type {boolean} */ (jspb.Message.getBooleanFieldWithDefault(this, 6, false)); +}; + + +/** + * @param {boolean} value + * @return {!proto.api_container_api.StarlarkScriptPlanYamlArgs} returns this + */ +proto.api_container_api.StarlarkScriptPlanYamlArgs.prototype.setDependenciesOnly = function(value) { + return jspb.Message.setField(this, 6, value); +}; + + +/** + * Clears the field making it undefined. + * @return {!proto.api_container_api.StarlarkScriptPlanYamlArgs} returns this + */ +proto.api_container_api.StarlarkScriptPlanYamlArgs.prototype.clearDependenciesOnly = function() { + return jspb.Message.setField(this, 6, undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.api_container_api.StarlarkScriptPlanYamlArgs.prototype.hasDependenciesOnly = function() { + return jspb.Message.getField(this, 6) != null; +}; + + @@ -11109,7 +11157,8 @@ proto.api_container_api.StarlarkPackagePlanYamlArgs.toObject = function(includeI packageId: jspb.Message.getFieldWithDefault(msg, 1, ""), serializedParams: jspb.Message.getFieldWithDefault(msg, 2, ""), relativePathToMainFile: jspb.Message.getFieldWithDefault(msg, 3, ""), - mainFunctionName: jspb.Message.getFieldWithDefault(msg, 4, "") + mainFunctionName: jspb.Message.getFieldWithDefault(msg, 4, ""), + dependenciesOnly: jspb.Message.getBooleanFieldWithDefault(msg, 5, false) }; if (includeInstance) { @@ -11162,6 +11211,10 @@ proto.api_container_api.StarlarkPackagePlanYamlArgs.deserializeBinaryFromReader var value = /** @type {string} */ (reader.readString()); msg.setMainFunctionName(value); break; + case 5: + var value = /** @type {boolean} */ (reader.readBool()); + msg.setDependenciesOnly(value); + break; default: reader.skipField(); break; @@ -11219,6 +11272,13 @@ proto.api_container_api.StarlarkPackagePlanYamlArgs.serializeBinaryToWriter = fu f ); } + f = /** @type {boolean} */ (jspb.Message.getField(message, 5)); + if (f != null) { + writer.writeBool( + 5, + f + ); + } }; @@ -11348,6 +11408,42 @@ proto.api_container_api.StarlarkPackagePlanYamlArgs.prototype.hasMainFunctionNam }; +/** + * optional bool dependencies_only = 5; + * @return {boolean} + */ +proto.api_container_api.StarlarkPackagePlanYamlArgs.prototype.getDependenciesOnly = function() { + return /** @type {boolean} */ (jspb.Message.getBooleanFieldWithDefault(this, 5, false)); +}; + + +/** + * @param {boolean} value + * @return {!proto.api_container_api.StarlarkPackagePlanYamlArgs} returns this + */ +proto.api_container_api.StarlarkPackagePlanYamlArgs.prototype.setDependenciesOnly = function(value) { + return jspb.Message.setField(this, 5, value); +}; + + +/** + * Clears the field making it undefined. + * @return {!proto.api_container_api.StarlarkPackagePlanYamlArgs} returns this + */ +proto.api_container_api.StarlarkPackagePlanYamlArgs.prototype.clearDependenciesOnly = function() { + return jspb.Message.setField(this, 5, undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.api_container_api.StarlarkPackagePlanYamlArgs.prototype.hasDependenciesOnly = function() { + return jspb.Message.getField(this, 5) != null; +}; + + /** * @enum {number} */ diff --git a/api/typescript/src/core/kurtosis_core_rpc_api_bindings/connect/api_container_service_pb.d.ts b/api/typescript/src/core/kurtosis_core_rpc_api_bindings/connect/api_container_service_pb.d.ts index de9778ca61..b297f0233d 100644 --- a/api/typescript/src/core/kurtosis_core_rpc_api_bindings/connect/api_container_service_pb.d.ts +++ b/api/typescript/src/core/kurtosis_core_rpc_api_bindings/connect/api_container_service_pb.d.ts @@ -1882,6 +1882,13 @@ export declare class StarlarkScriptPlanYamlArgs extends Message); static readonly runtime: typeof proto3; @@ -1928,6 +1935,13 @@ export declare class StarlarkPackagePlanYamlArgs extends Message); static readonly runtime: typeof proto3; diff --git a/api/typescript/src/core/kurtosis_core_rpc_api_bindings/connect/api_container_service_pb.js b/api/typescript/src/core/kurtosis_core_rpc_api_bindings/connect/api_container_service_pb.js index 164d68d7d5..e0812b1b9d 100644 --- a/api/typescript/src/core/kurtosis_core_rpc_api_bindings/connect/api_container_service_pb.js +++ b/api/typescript/src/core/kurtosis_core_rpc_api_bindings/connect/api_container_service_pb.js @@ -659,6 +659,7 @@ export const StarlarkScriptPlanYamlArgs = proto3.makeMessageType( { no: 1, name: "serialized_script", kind: "scalar", T: 9 /* ScalarType.STRING */ }, { no: 2, name: "serialized_params", kind: "scalar", T: 9 /* ScalarType.STRING */, opt: true }, { no: 5, name: "main_function_name", kind: "scalar", T: 9 /* ScalarType.STRING */, opt: true }, + { no: 6, name: "dependencies_only", kind: "scalar", T: 8 /* ScalarType.BOOL */, opt: true }, ], ); @@ -672,6 +673,7 @@ export const StarlarkPackagePlanYamlArgs = proto3.makeMessageType( { no: 2, name: "serialized_params", kind: "scalar", T: 9 /* ScalarType.STRING */, opt: true }, { no: 3, name: "relative_path_to_main_file", kind: "scalar", T: 9 /* ScalarType.STRING */, opt: true }, { no: 4, name: "main_function_name", kind: "scalar", T: 9 /* ScalarType.STRING */, opt: true }, + { no: 5, name: "dependencies_only", kind: "scalar", T: 8 /* ScalarType.BOOL */, opt: true }, ], ); diff --git a/cli/cli/commands/run/run.go b/cli/cli/commands/run/run.go index 8f0529ec4c..8abfea3c01 100755 --- a/cli/cli/commands/run/run.go +++ b/cli/cli/commands/run/run.go @@ -4,8 +4,16 @@ import ( "context" "encoding/json" "fmt" + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing/transport" + "github.com/kurtosis-tech/kurtosis/api/golang/core/lib/starlark_run_config" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/backend_creator" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/configs" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/image_download_mode" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/user_support_constants" + "gopkg.in/yaml.v2" "io" + "k8s.io/utils/strings/slices" "net/http" "net/url" "os" @@ -15,10 +23,6 @@ import ( "regexp" "strings" - "github.com/kurtosis-tech/kurtosis/api/golang/core/lib/starlark_run_config" - "gopkg.in/yaml.v2" - "k8s.io/utils/strings/slices" - "github.com/kurtosis-tech/kurtosis/api/golang/core/kurtosis_core_rpc_api_bindings" "github.com/kurtosis-tech/kurtosis/api/golang/core/lib/enclaves" "github.com/kurtosis-tech/kurtosis/api/golang/core/lib/services" @@ -55,8 +59,10 @@ const ( dryRunFlagKey = "dry-run" defaultDryRun = "false" - dependenciesFlagKey = "dependencies" - defaultDependencies = "false" + dependenciesFlagKey = "dependencies" + defaultDependencies = "false" + pullDependenciesFlagKey = "pull" + defaultPullDependencies = "false" fullUuidsFlagKey = "full-uuids" fullUuidFlagKeyDefault = "false" @@ -137,6 +143,12 @@ var StarlarkRunCmd = &engine_consuming_kurtosis_command.EngineConsumingKurtosisC Type: flags.FlagType_Bool, Default: defaultDependencies, }, + { + Key: pullDependenciesFlagKey, + Usage: "If true, and the dependencies flag is passed, attempts to pull all images and packages that the run depends on locally. kurtosis.yml is updated with the replace directives pointing to locally pulled packages.", + Type: flags.FlagType_Bool, + Default: defaultPullDependencies, + }, { Key: enclaveIdentifierFlagKey, Usage: "The enclave identifier of the enclave in which the script or package will be ran. " + @@ -273,7 +285,12 @@ func run( dependencies, err := flags.GetBool(dependenciesFlagKey) if err != nil { - return stacktrace.Propagate(err, "Expectew a boolean flag with key '%v' but none was found; this is an error in Kurtosis!", dependencies) + return stacktrace.Propagate(err, "Expected a boolean flag with key '%v' but none was found; this is an error in Kurtosis!", dependenciesFlagKey) + } + + pullDependencies, err := flags.GetBool(pullDependenciesFlagKey) + if err != nil { + return stacktrace.Propagate(err, "Expected a boolean flag with key '%v' but none was found; this is an error in Kurtosis!", pullDependenciesFlagKey) } parallelism, err := flags.GetUint32(parallelismFlagKey) @@ -383,11 +400,38 @@ func run( isRemotePackage := strings.HasPrefix(starlarkScriptOrPackagePath, githubDomainPrefix) if dependencies { - packageDependencyYamlStr, err := getPackageDependencyYamlStr(ctx, enclaveCtx, starlarkScriptOrPackagePath, isRemotePackage, packageArgs) + dependencyYaml, err := getPackageDependencyYaml(ctx, enclaveCtx, starlarkScriptOrPackagePath, isRemotePackage, packageArgs) if err != nil { return stacktrace.Propagate(err, "An error occurred getting package dependencies") } - fmt.Printf("%v\n", packageDependencyYamlStr) + fmt.Printf("%v\n", dependencyYaml.PlanYaml) + + if pullDependencies { + type PackageDependencies struct { + Images []string `yaml:"images"` + Packages []string `yaml:"packageDependencies"` + } + var pkgDeps PackageDependencies + err := yaml.Unmarshal([]byte(dependencyYaml.PlanYaml), &pkgDeps) + if err != nil { + return stacktrace.Propagate(err, "An error occurred unmarhsaling dependency yaml string") + } + + err = pullImagesLocally(ctx, pkgDeps.Images) + if err != nil { + return err + } + + packageNamesToLocalFilepaths, err := pullPackagesLocally(pkgDeps.Packages) + if err != nil { + return err + } + + err = updateKurtosisYamlWithReplaceDirectives(packageNamesToLocalFilepaths) + if err != nil { + return err + } + } return nil } @@ -612,41 +656,24 @@ func getOrCreateEnclaveContext( return enclaveContext, isNewEnclaveFlagWhenCreated, nil } -type PackageDependency struct { - Packages []string `yaml:"packages,omitempty"` - Images []string `yaml:"images,omitempty"` -} - -func getPackageDependencyYamlStr( +func getPackageDependencyYaml( ctx context.Context, enclaveCtx *enclaves.EnclaveContext, starlarkScriptOrPackageId string, isRemote bool, packageArgs string, -) (string, error) { +) (*kurtosis_core_rpc_api_bindings.PlanYaml, error) { var packageYaml *kurtosis_core_rpc_api_bindings.PlanYaml var err error if isRemote { - packageYaml, err = enclaveCtx.GetStarlarkPackagePlanYaml(ctx, starlarkScriptOrPackageId, packageArgs) + packageYaml, err = enclaveCtx.GetStarlarkPackagePlanYaml(ctx, starlarkScriptOrPackageId, packageArgs, true) } else { - packageYaml, err = enclaveCtx.GetStarlarkScriptPlanYaml(ctx, starlarkScriptOrPackageId, packageArgs) - } - if err != nil { - return "", stacktrace.Propagate(err, "An error occurred retrieving plan yaml for provided package.") - } - //var planYaml plan_yaml.PlanYaml - //if err = yaml.Unmarshal([]byte(packageYaml.PlanYaml), &planYaml); err != nil { - // return "", stacktrace.Propagate(err, "An error occurred unmarshaling plan yaml string: %v.", packageYaml.PlanYaml) - //} - //planYaml.Tasks = nil - //planYaml.FilesArtifacts = nil - //planYaml.Services = nil - //packageDependencyYamlBytes, err := yaml.Marshal(&planYaml) - //if err != nil { - // return "", stacktrace.Propagate(err, "") - //} - //return string(packageDependencyYamlBytes), nil - return packageYaml.PlanYaml, nil + packageYaml, err = enclaveCtx.GetStarlarkScriptPlanYaml(ctx, starlarkScriptOrPackageId, packageArgs, true) + } + if err != nil { + return nil, stacktrace.Propagate(err, "An error occurred retrieving plan yaml for provided package.") + } + return packageYaml, nil } // validatePackageArgs just validates the args is a valid JSON or YAML string @@ -795,3 +822,84 @@ func isHttpUrl(maybeHttpUrl string) bool { httpProtocolRegex := regexp.MustCompile(httpProtocolRegexStr) return httpProtocolRegex.MatchString(maybeHttpUrl) } + +func pullImagesLocally(ctx context.Context, images []string) error { + kurtosisBackend, err := backend_creator.GetDockerKurtosisBackend(backend_creator.NoAPIContainerModeArgs, configs.NoRemoteBackendConfig) + if err != nil { + return stacktrace.Propagate(err, "An error occurred retrieving Docker Kurtosis Backend") + } + for _, img := range images { + _, _, err := kurtosisBackend.FetchImage(ctx, img, nil, image_download_mode.ImageDownloadMode_Always) + if err != nil { + return stacktrace.Propagate(err, "An error occurred pulling '%v' locally.", img) + } + } + return nil +} + +func pullPackagesLocally(packageDependencies []string) (map[string]string, error) { + var localPackagesToRelativeFilepaths map[string]string + wd, err := os.Getwd() + if err != nil { + return nil, stacktrace.Propagate(err, "An error occurred getting current working directory.") + } + parentCwd := filepath.Dir(wd) + relParentCwd, err := filepath.Rel(wd, parentCwd) + if err != nil { + return nil, stacktrace.Propagate(err, "An error occurred getting rel path between '%v' and '%v'.", wd, relParentCwd) + } + logrus.Infof("current wid: %v", wd) + logrus.Infof("rel parent wd: %v", relParentCwd) + for _, dep := range packageDependencies { + logrus.Infof("Pulling package: %v", dep) + // assume dep doesn't have http:// prefix or .git suffix + repoUrl := fmt.Sprintf("%s%s%s", "http://", dep, ".git") + logrus.Infof("repo url: %v", repoUrl) + _, err := git.PlainClone(relParentCwd, false, &git.CloneOptions{ + URL: repoUrl, + Auth: nil, + RemoteName: "", + ReferenceName: "", + SingleBranch: false, + Mirror: false, + NoCheckout: false, + Depth: 0, + RecurseSubmodules: 0, + ShallowSubmodules: false, + Progress: nil, + Tags: 0, + InsecureSkipTLS: false, + CABundle: nil, + ProxyOptions: transport.ProxyOptions{}, + Shared: false, + }) + if err != nil { + return nil, stacktrace.Propagate(err, "An error occurred cloning package locally.") + } + parts := strings.Split(dep, "/") + packageName := parts[len(parts)-1] + localPackagesToRelativeFilepaths[dep] = fmt.Sprintf("%s/%s", relParentCwd, packageName) + } + return localPackagesToRelativeFilepaths, nil +} + +func updateKurtosisYamlWithReplaceDirectives(packageNamesToLocalFilepaths map[string]string) error { + file, err := os.OpenFile("kurtosis.yml", os.O_APPEND, 0644) + if err != nil { + return stacktrace.Propagate(err, "An error occurred opening kurtosis.yml file.") + } + defer file.Close() + + _, err = file.Write([]byte("replace:")) + if err != nil { + return stacktrace.Propagate(err, "An error occurred writing 'replace:' to kurtosis.yml") + } + + for packageName, localFilepath := range packageNamesToLocalFilepaths { + _, err := file.Write([]byte(fmt.Sprintf("\t %s: %s", packageName, localFilepath))) + if err != nil { + return stacktrace.Propagate(err, "An error occurred writing replace directive for '%v: %v'", packageName, localFilepath) + } + } + return nil +} diff --git a/core/server/api_container/server/api_container_service.go b/core/server/api_container/server/api_container_service.go index eabdf4a60d..ce2ab9db51 100644 --- a/core/server/api_container/server/api_container_service.go +++ b/core/server/api_container/server/api_container_service.go @@ -663,6 +663,7 @@ func (apicService *ApiContainerService) GetStarlarkPackagePlanYaml(ctx context.C serializedParams := args.GetSerializedParams() requestedRelativePathToMainFile := args.GetRelativePathToMainFile() mainFuncName := args.GetMainFunctionName() + dependenciesOnly := args.GetDependenciesOnly() var scriptWithRunFunction string var interpretationError *startosis_errors.InterpretationError @@ -691,7 +692,7 @@ func (apicService *ApiContainerService) GetStarlarkPackagePlanYaml(ctx context.C interpretationError = startosis_errors.NewInterpretationError(apiInterpretationError.GetErrorMessage()) return nil, stacktrace.Propagate(interpretationError, "An interpretation error occurred interpreting package for retrieving plan yaml for package: %v", packageIdFromArgs) } - planYamlStr, err := instructionsPlan.GenerateYaml(plan_yaml.CreateEmptyPlan(packageIdFromArgs)) + planYamlStr, err := instructionsPlan.GenerateYaml(plan_yaml.CreateEmptyPlan(packageIdFromArgs), dependenciesOnly) if err != nil { return nil, stacktrace.Propagate(err, "An error occurred generating plan yaml for package: %v", packageIdFromArgs) } @@ -707,6 +708,7 @@ func (apicService *ApiContainerService) GetStarlarkScriptPlanYaml(ctx context.Co serializedStarlarkScript := args.GetSerializedScript() serializedParams := args.GetSerializedParams() mainFuncName := args.GetMainFunctionName() + dependenciesOnly := args.GetDependenciesOnly() noPackageReplaceOptions := map[string]string{} _, instructionsPlan, apiInterpretationError := apicService.startosisInterpreter.Interpret( @@ -724,7 +726,7 @@ func (apicService *ApiContainerService) GetStarlarkScriptPlanYaml(ctx context.Co if apiInterpretationError != nil { return nil, startosis_errors.NewInterpretationError(apiInterpretationError.GetErrorMessage()) } - planYamlStr, err := instructionsPlan.GenerateYaml(plan_yaml.CreateEmptyPlan(startosis_constants.PackageIdPlaceholderForStandaloneScript)) + planYamlStr, err := instructionsPlan.GenerateYaml(plan_yaml.CreateEmptyPlan(startosis_constants.PackageIdPlaceholderForStandaloneScript), dependenciesOnly) if err != nil { return nil, err } diff --git a/core/server/api_container/server/startosis_engine/instructions_plan/instructions_plan.go b/core/server/api_container/server/startosis_engine/instructions_plan/instructions_plan.go index 4b5f8b2dc6..81dd265ad7 100644 --- a/core/server/api_container/server/startosis_engine/instructions_plan/instructions_plan.go +++ b/core/server/api_container/server/startosis_engine/instructions_plan/instructions_plan.go @@ -81,7 +81,7 @@ func (plan *InstructionsPlan) GeneratePlan() ([]*ScheduledInstruction, *startosi } // GenerateYaml takes in an existing planYaml (usually empty) and returns a yaml string containing the effects of the plan -func (plan *InstructionsPlan) GenerateYaml(planYaml *plan_yaml.PlanYamlGenerator) (string, error) { +func (plan *InstructionsPlan) GenerateYaml(planYaml *plan_yaml.PlanYamlGenerator, dependenciesOnly bool) (string, error) { for _, instructionUuid := range plan.instructionsSequence { instruction, found := plan.scheduledInstructionsIndex[instructionUuid] if !found { @@ -93,7 +93,7 @@ func (plan *InstructionsPlan) GenerateYaml(planYaml *plan_yaml.PlanYamlGenerator } } planYaml.AddPackageDependencies(plan.packageDependencies) - return planYaml.GenerateYaml() + return planYaml.GenerateYaml(dependenciesOnly) } func (plan *InstructionsPlan) AddPackageDependency(packageDependency string) { diff --git a/core/server/api_container/server/startosis_engine/plan_yaml/plan_yaml_generator.go b/core/server/api_container/server/startosis_engine/plan_yaml/plan_yaml_generator.go index e7098c83db..b7891f684b 100644 --- a/core/server/api_container/server/startosis_engine/plan_yaml/plan_yaml_generator.go +++ b/core/server/api_container/server/startosis_engine/plan_yaml/plan_yaml_generator.go @@ -49,9 +49,22 @@ func CreateEmptyPlan(packageId string) *PlanYamlGenerator { } } -func (planYaml *PlanYamlGenerator) GenerateYaml() (string, error) { +func (planYaml *PlanYamlGenerator) GenerateYaml(dependenciesOnly bool) (string, error) { planYaml.AddImages() - yamlBytes, err := yaml.Marshal(planYaml.privatePlanYaml) + + var planYamlCopy PlanYaml + planYamlCopy = *planYaml.privatePlanYaml + if dependenciesOnly { + planYamlCopy = PlanYaml{ + PackageId: "", + Services: nil, + FilesArtifacts: nil, + Tasks: nil, + Images: planYaml.privatePlanYaml.Images, + PackageDependencies: planYaml.privatePlanYaml.PackageDependencies, + } + } + yamlBytes, err := yaml.Marshal(planYamlCopy) if err != nil { return "", stacktrace.Propagate(err, "An error occurred generating plan yaml.") } diff --git a/core/server/api_container/server/startosis_engine/startosis_interpreter.go b/core/server/api_container/server/startosis_engine/startosis_interpreter.go index b26fef893a..48132f77e0 100644 --- a/core/server/api_container/server/startosis_engine/startosis_interpreter.go +++ b/core/server/api_container/server/startosis_engine/startosis_interpreter.go @@ -350,10 +350,6 @@ func (interpreter *StartosisInterpreter) interpretInternal( return globalVariables, nil } -func getModulePrefix(moduleId string) string { - return strings.Join(strings.SplitN(moduleId, "/", 4)[:3], "/") -} - func (interpreter *StartosisInterpreter) buildBindings( packageId string, thread *starlark.Thread, @@ -395,6 +391,16 @@ func (interpreter *StartosisInterpreter) buildBindings( return &predeclared, nil } +const ( + moduleIdSeperator = "/" + numModuleIdSeparators = 4 + prefixNum = 3 +) + +func getModulePrefix(moduleId string) string { + return strings.Join(strings.SplitN(moduleId, moduleIdSeperator, numModuleIdSeparators)[:prefixNum], moduleIdSeperator) +} + func findFirstEqualInstructionPastIndex(currentEnclaveInstructionsList []*enclave_plan_persistence.EnclavePlanInstruction, naiveInstructionsList []*instructions_plan.ScheduledInstruction, minIndex int) int { if len(naiveInstructionsList) == 0 { return -1 // no result as the naiveInstructionsList is empty diff --git a/core/server/api_container/server/startosis_engine/startosis_interpreter_plan_yaml_test.go b/core/server/api_container/server/startosis_engine/startosis_interpreter_plan_yaml_test.go index bccdfe5d59..05d59ef308 100644 --- a/core/server/api_container/server/startosis_engine/startosis_interpreter_plan_yaml_test.go +++ b/core/server/api_container/server/startosis_engine/startosis_interpreter_plan_yaml_test.go @@ -115,7 +115,7 @@ func (suite *StartosisIntepreterPlanYamlTestSuite) TestAddServiceWithFilesArtifa require.Nil(suite.T(), interpretationError) require.Equal(suite.T(), 1, instructionsPlan.Size()) - planYaml, err := instructionsPlan.GenerateYaml(plan_yaml.CreateEmptyPlan(startosis_constants.PackageIdPlaceholderForStandaloneScript)) + planYaml, err := instructionsPlan.GenerateYaml(plan_yaml.CreateEmptyPlan(startosis_constants.PackageIdPlaceholderForStandaloneScript), false) require.NoError(suite.T(), err) expectedYaml := @@ -185,7 +185,7 @@ func (suite *StartosisIntepreterPlanYamlTestSuite) TestRunShWithFilesArtifacts() require.Nil(suite.T(), interpretationError) require.Equal(suite.T(), 1, instructionsPlan.Size()) - planYaml, err := instructionsPlan.GenerateYaml(plan_yaml.CreateEmptyPlan(startosis_constants.PackageIdPlaceholderForStandaloneScript)) + planYaml, err := instructionsPlan.GenerateYaml(plan_yaml.CreateEmptyPlan(startosis_constants.PackageIdPlaceholderForStandaloneScript), false) require.NoError(suite.T(), err) expectedYaml := `packageId: DEFAULT_PACKAGE_ID_FOR_SCRIPT @@ -260,7 +260,7 @@ func (suite *StartosisIntepreterPlanYamlTestSuite) TestRunPython() { require.Nil(suite.T(), interpretationError) require.Equal(suite.T(), 1, instructionsPlan.Size()) - planYaml, err := instructionsPlan.GenerateYaml(plan_yaml.CreateEmptyPlan(startosis_constants.PackageIdPlaceholderForStandaloneScript)) + planYaml, err := instructionsPlan.GenerateYaml(plan_yaml.CreateEmptyPlan(startosis_constants.PackageIdPlaceholderForStandaloneScript), false) require.NoError(suite.T(), err) expectedYaml := `packageId: DEFAULT_PACKAGE_ID_FOR_SCRIPT @@ -336,7 +336,7 @@ func (suite *StartosisIntepreterPlanYamlTestSuite) TestExec() { require.Nil(suite.T(), interpretationError) require.Equal(suite.T(), 2, instructionsPlan.Size()) - planYaml, err := instructionsPlan.GenerateYaml(plan_yaml.CreateEmptyPlan(startosis_constants.PackageIdPlaceholderForStandaloneScript)) + planYaml, err := instructionsPlan.GenerateYaml(plan_yaml.CreateEmptyPlan(startosis_constants.PackageIdPlaceholderForStandaloneScript), false) require.NoError(suite.T(), err) expectedYaml := `packageId: DEFAULT_PACKAGE_ID_FOR_SCRIPT @@ -414,7 +414,7 @@ func (suite *StartosisIntepreterPlanYamlTestSuite) TestRenderTemplate() { require.Nil(suite.T(), interpretationError) require.Equal(suite.T(), 2, instructionsPlan.Size()) - planYaml, err := instructionsPlan.GenerateYaml(plan_yaml.CreateEmptyPlan(startosis_constants.PackageIdPlaceholderForStandaloneScript)) + planYaml, err := instructionsPlan.GenerateYaml(plan_yaml.CreateEmptyPlan(startosis_constants.PackageIdPlaceholderForStandaloneScript), false) require.NoError(suite.T(), err) expectedYaml := `packageId: DEFAULT_PACKAGE_ID_FOR_SCRIPT @@ -481,7 +481,7 @@ func (suite *StartosisIntepreterPlanYamlTestSuite) TestAddServiceWithImageBuildS require.Nil(suite.T(), interpretationError) require.Equal(suite.T(), 1, instructionsPlan.Size()) - planYaml, err := instructionsPlan.GenerateYaml(plan_yaml.CreateEmptyPlan(packageId)) + planYaml, err := instructionsPlan.GenerateYaml(plan_yaml.CreateEmptyPlan(packageId), false) require.NoError(suite.T(), err) expectedYaml := `packageId: ` + packageId + ` @@ -544,7 +544,7 @@ func (suite *StartosisIntepreterPlanYamlTestSuite) TestAddServiceWithImageSpec() require.Nil(suite.T(), interpretationError) require.Equal(suite.T(), 1, instructionsPlan.Size()) - planYaml, err := instructionsPlan.GenerateYaml(plan_yaml.CreateEmptyPlan(packageId)) + planYaml, err := instructionsPlan.GenerateYaml(plan_yaml.CreateEmptyPlan(packageId), false) require.NoError(suite.T(), err) expectedYaml := `packageId: ` + packageId + ` @@ -603,7 +603,7 @@ func (suite *StartosisIntepreterPlanYamlTestSuite) TestUploadFiles() { require.Nil(suite.T(), interpretationError) require.Equal(suite.T(), 2, instructionsPlan.Size()) - planYaml, err := instructionsPlan.GenerateYaml(plan_yaml.CreateEmptyPlan(packageId)) + planYaml, err := instructionsPlan.GenerateYaml(plan_yaml.CreateEmptyPlan(packageId), false) require.NoError(suite.T(), err) expectedYaml := `packageId: ` + packageId + ` @@ -665,7 +665,7 @@ func (suite *StartosisIntepreterPlanYamlTestSuite) TestStoreServiceFiles() { require.Nil(suite.T(), interpretationError) require.Equal(suite.T(), 2, instructionsPlan.Size()) - planYaml, err := instructionsPlan.GenerateYaml(plan_yaml.CreateEmptyPlan(startosis_constants.PackageIdPlaceholderForStandaloneScript)) + planYaml, err := instructionsPlan.GenerateYaml(plan_yaml.CreateEmptyPlan(startosis_constants.PackageIdPlaceholderForStandaloneScript), false) require.NoError(suite.T(), err) expectedYaml := `packageId: DEFAULT_PACKAGE_ID_FOR_SCRIPT @@ -728,7 +728,7 @@ func (suite *StartosisIntepreterPlanYamlTestSuite) TestRemoveService() { require.Nil(suite.T(), interpretationError) require.Equal(suite.T(), 2, instructionsPlan.Size()) - planYaml, err := instructionsPlan.GenerateYaml(plan_yaml.CreateEmptyPlan(startosis_constants.PackageIdPlaceholderForStandaloneScript)) + planYaml, err := instructionsPlan.GenerateYaml(plan_yaml.CreateEmptyPlan(startosis_constants.PackageIdPlaceholderForStandaloneScript), false) require.NoError(suite.T(), err) expectedYaml := `packageId: DEFAULT_PACKAGE_ID_FOR_SCRIPT @@ -789,7 +789,7 @@ func (suite *StartosisIntepreterPlanYamlTestSuite) TestFutureReferencesAreSwappe require.Nil(suite.T(), interpretationError) require.Equal(suite.T(), 4, instructionsPlan.Size()) - planYaml, err := instructionsPlan.GenerateYaml(plan_yaml.CreateEmptyPlan(startosis_constants.PackageIdPlaceholderForStandaloneScript)) + planYaml, err := instructionsPlan.GenerateYaml(plan_yaml.CreateEmptyPlan(startosis_constants.PackageIdPlaceholderForStandaloneScript), false) require.NoError(suite.T(), err) expectedYaml := `packageId: DEFAULT_PACKAGE_ID_FOR_SCRIPT diff --git a/core/server/api_container/server/startosis_engine/startosis_interpreter_test.go b/core/server/api_container/server/startosis_engine/startosis_interpreter_test.go index d2bea6d789..2b95563878 100644 --- a/core/server/api_container/server/startosis_engine/startosis_interpreter_test.go +++ b/core/server/api_container/server/startosis_engine/startosis_interpreter_test.go @@ -1081,6 +1081,17 @@ def run(plan): require.Nil(suite.T(), interpretationError) } +func (suite *StartosisInterpreterTestSuite) TestGetModuleIdPrefix() { + githubModuleId := "github.com/some-person/some-pkg/main.star" + expectedGithubModuleId := "github.com/some-person/some-pkg" + + require.Equal(suite.T(), expectedGithubModuleId, getModulePrefix(githubModuleId)) + + moduleId := "./some-person/some-pkg/main.star" + expectedModuleId := "./some-person/some-pkg" + require.Equal(suite.T(), expectedModuleId, getModulePrefix(moduleId)) +} + // ##################################################################################################################### // // TEST HELPERS From 18c2591796f2d2f3a58342ee6cf0862c2eb890a3 Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Thu, 12 Sep 2024 10:10:07 -0400 Subject: [PATCH 29/31] fix cloning pkgs --- cli/cli/commands/run/run.go | 37 +++++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/cli/cli/commands/run/run.go b/cli/cli/commands/run/run.go index 8abfea3c01..23502f47c5 100755 --- a/cli/cli/commands/run/run.go +++ b/cli/cli/commands/run/run.go @@ -3,6 +3,7 @@ package run import ( "context" "encoding/json" + "errors" "fmt" "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/plumbing/transport" @@ -838,7 +839,8 @@ func pullImagesLocally(ctx context.Context, images []string) error { } func pullPackagesLocally(packageDependencies []string) (map[string]string, error) { - var localPackagesToRelativeFilepaths map[string]string + localPackagesToRelativeFilepaths := map[string]string{} + wd, err := os.Getwd() if err != nil { return nil, stacktrace.Propagate(err, "An error occurred getting current working directory.") @@ -848,14 +850,17 @@ func pullPackagesLocally(packageDependencies []string) (map[string]string, error if err != nil { return nil, stacktrace.Propagate(err, "An error occurred getting rel path between '%v' and '%v'.", wd, relParentCwd) } - logrus.Infof("current wid: %v", wd) - logrus.Infof("rel parent wd: %v", relParentCwd) - for _, dep := range packageDependencies { - logrus.Infof("Pulling package: %v", dep) - // assume dep doesn't have http:// prefix or .git suffix - repoUrl := fmt.Sprintf("%s%s%s", "http://", dep, ".git") + + for _, dependency := range packageDependencies { + packageIdParts := strings.Split(dependency, "/") + packageName := packageIdParts[len(packageIdParts)-1] + logrus.Infof("Pulling package: %v", dependency) + + // assume dependency doesn't have http:// prefix or .git suffix + repoUrl := fmt.Sprintf("%s%s%s", "http://", dependency, ".git") + localPackagePath := fmt.Sprintf("%s/%s", parentCwd, packageName) logrus.Infof("repo url: %v", repoUrl) - _, err := git.PlainClone(relParentCwd, false, &git.CloneOptions{ + _, err := git.PlainClone(localPackagePath, false, &git.CloneOptions{ URL: repoUrl, Auth: nil, RemoteName: "", @@ -873,33 +878,33 @@ func pullPackagesLocally(packageDependencies []string) (map[string]string, error ProxyOptions: transport.ProxyOptions{}, Shared: false, }) - if err != nil { - return nil, stacktrace.Propagate(err, "An error occurred cloning package locally.") + if err != nil && !errors.Is(err, git.ErrRepositoryAlreadyExists) { + return nil, stacktrace.Propagate(err, "An error occurred cloning package '%s' to '%s'.", dependency, localPackagePath) } - parts := strings.Split(dep, "/") - packageName := parts[len(parts)-1] - localPackagesToRelativeFilepaths[dep] = fmt.Sprintf("%s/%s", relParentCwd, packageName) + localPackagesToRelativeFilepaths[dependency] = fmt.Sprintf("%s/%s", relParentCwd, packageName) } + return localPackagesToRelativeFilepaths, nil } func updateKurtosisYamlWithReplaceDirectives(packageNamesToLocalFilepaths map[string]string) error { - file, err := os.OpenFile("kurtosis.yml", os.O_APPEND, 0644) + file, err := os.OpenFile("kurtosis.yml", os.O_WRONLY|os.O_APPEND, 0644) if err != nil { return stacktrace.Propagate(err, "An error occurred opening kurtosis.yml file.") } defer file.Close() - _, err = file.Write([]byte("replace:")) + _, err = file.Write([]byte("replace:\n")) if err != nil { return stacktrace.Propagate(err, "An error occurred writing 'replace:' to kurtosis.yml") } for packageName, localFilepath := range packageNamesToLocalFilepaths { - _, err := file.Write([]byte(fmt.Sprintf("\t %s: %s", packageName, localFilepath))) + _, err := file.Write([]byte(fmt.Sprintf(" %s: %s\n", packageName, localFilepath))) if err != nil { return stacktrace.Propagate(err, "An error occurred writing replace directive for '%v: %v'", packageName, localFilepath) } } + return nil } From 7e56e5b37ff6cbfc96db8d247fd5b7c509e5eeab Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Thu, 12 Sep 2024 10:39:23 -0400 Subject: [PATCH 30/31] replace magic vlues --- cli/cli/commands/run/run.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cli/cli/commands/run/run.go b/cli/cli/commands/run/run.go index 23502f47c5..d9d6fb2b13 100755 --- a/cli/cli/commands/run/run.go +++ b/cli/cli/commands/run/run.go @@ -94,7 +94,8 @@ const ( kurtosisBackendCtxKey = "kurtosis-backend" engineClientCtxKey = "engine-client" - kurtosisYMLFilePath = "kurtosis.yml" + kurtosisYMLFilePath = "kurtosis.yml" + kurtosisYMLFilePerms = 0644 portMappingSeparatorForLogs = ", " @@ -888,7 +889,7 @@ func pullPackagesLocally(packageDependencies []string) (map[string]string, error } func updateKurtosisYamlWithReplaceDirectives(packageNamesToLocalFilepaths map[string]string) error { - file, err := os.OpenFile("kurtosis.yml", os.O_WRONLY|os.O_APPEND, 0644) + file, err := os.OpenFile(kurtosisYMLFilePath, os.O_WRONLY|os.O_APPEND, kurtosisYMLFilePerms) if err != nil { return stacktrace.Propagate(err, "An error occurred opening kurtosis.yml file.") } From 24bbebbd5da5e4071a11dac2387785bc1dff8f8f Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Thu, 12 Sep 2024 10:44:17 -0400 Subject: [PATCH 31/31] exhaust struct --- cli/cli/commands/run/run.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/cli/cli/commands/run/run.go b/cli/cli/commands/run/run.go index d9d6fb2b13..15273af4bb 100755 --- a/cli/cli/commands/run/run.go +++ b/cli/cli/commands/run/run.go @@ -876,8 +876,12 @@ func pullPackagesLocally(packageDependencies []string) (map[string]string, error Tags: 0, InsecureSkipTLS: false, CABundle: nil, - ProxyOptions: transport.ProxyOptions{}, - Shared: false, + ProxyOptions: transport.ProxyOptions{ + URL: "", + Username: "", + Password: "", + }, + Shared: false, }) if err != nil && !errors.Is(err, git.ErrRepositoryAlreadyExists) { return nil, stacktrace.Propagate(err, "An error occurred cloning package '%s' to '%s'.", dependency, localPackagePath)