Skip to content

Commit

Permalink
Add multiple iterations of workload creation test
Browse files Browse the repository at this point in the history
  • Loading branch information
mcornea committed Oct 12, 2023
1 parent 6ae3d1c commit e48542e
Show file tree
Hide file tree
Showing 50 changed files with 6,577 additions and 3 deletions.
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ require (
github.com/onsi/gomega v1.27.8
github.com/openshift-kni/eco-goinfra v0.0.0-20231006185241-4e996b952657
github.com/openshift-kni/k8sreporter v1.0.4
gonum.org/v1/gonum v0.14.0
gopkg.in/yaml.v2 v2.4.0
k8s.io/api v0.27.1
k8s.io/apimachinery v0.27.1
Expand Down
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -1623,6 +1623,8 @@ golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8T
gomodules.xyz/jsonpatch/v2 v2.1.0/go.mod h1:IhYNNY4jnS53ZnfE4PAmpKtDpTCj1JFXc+3mwe7XcUU=
gomodules.xyz/jsonpatch/v2 v2.2.0 h1:4pT439QV83L+G9FkcCriY6EkpcK6r6bK+A5FBUMI7qY=
gomodules.xyz/jsonpatch/v2 v2.2.0/go.mod h1:WXp+iVDkoLQqPudfQ9GBlwB2eZ5DKOnjQZCYdOS8GPY=
gonum.org/v1/gonum v0.14.0 h1:2NiG67LD1tEH0D7kM+ps2V+fXmsAnpUeec7n8tcr4S0=
gonum.org/v1/gonum v0.14.0/go.mod h1:AoWeoz0becf9QMWtE8iWXNXc27fK4fNeHNf/oMejGfU=
google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk=
google.golang.org/api v0.3.2/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk=
google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
Expand Down
7 changes: 4 additions & 3 deletions tests/ran-du/internal/randuconfig/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,10 @@ type RanDuConfig struct {
CreateMethod string `yaml:"create_method" envconfig:"ECO_RANDU_TESTWORKLOAD_CREATE_METHOD"`
CreateShellCmd string `yaml:"create_shell_cmd" envconfig:"ECO_RANDU_TESTWORKLOAD_CREATE_SHELLCMD"`
} `yaml:"randu_test_workload"`
SoftRebootIterations string `yaml:"soft_reboot_iterations" envconfig:"ECO_RANDU_SOFT_REBOOT_ITERATIONS"`
HardRebootIterations string `yaml:"hard_reboot_iterations" envconfig:"ECO_RANDU_HARD_REBOOT_ITERATIONS"`
IpmiToolImage string `yaml:"ipmitool_image" envconfig:"ECO_RANDU_IPMITOOL_IMAGE"`
SoftRebootIterations string `yaml:"soft_reboot_iterations" envconfig:"ECO_RANDU_SOFT_REBOOT_ITERATIONS"`
HardRebootIterations string `yaml:"hard_reboot_iterations" envconfig:"ECO_RANDU_HARD_REBOOT_ITERATIONS"`
IpmiToolImage string `yaml:"ipmitool_image" envconfig:"ECO_RANDU_IPMITOOL_IMAGE"`
LaunchWorkloadIterations string `yaml:"launch_workload_iterations" envconfig:"ECO_RANDU_LAUNCH_WORKLOAD_ITERATIONS"`
}

// NewRanDuConfig returns instance of RanDuConfig config type.
Expand Down
1 change: 1 addition & 0 deletions tests/ran-du/internal/randuconfig/default.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -6,3 +6,4 @@ randu_test_workload:
create_shell_cmd: '/opt/vdu-workload-emulator/add_test-deployments.sh'
soft_reboot_iterations: '5'
hard_reboot_iterations: '5'
launch_workload_iterations: '5'
4 changes: 4 additions & 0 deletions tests/ran-du/internal/randuparams/randuvars.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,4 +22,8 @@ var (

// TestNamespaceName is used for defining the namespace name where test resources are created.
TestNamespaceName = "ran-du-system-tests"

// TestMultipleLaunchWorkloadLoadAvg is used for defining the node load average threshold to be
// used in the LaunchWorkloadMultipleIterations test.
TestMultipleLaunchWorkloadLoadAvg = 100
)
98 changes: 98 additions & 0 deletions tests/ran-du/tests/launch-workload-multiple-iter-loadavg.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
package ran_du_system_test

import (
"fmt"
"strconv"
"strings"
"time"

. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/openshift-kni/eco-goinfra/pkg/namespace"
"github.com/openshift-kni/eco-goinfra/pkg/nodes"
"github.com/openshift-kni/eco-goinfra/pkg/polarion"
"github.com/openshift-kni/eco-gosystem/tests/internal/await"
"github.com/openshift-kni/eco-gosystem/tests/internal/cmd"
"github.com/openshift-kni/eco-gosystem/tests/internal/shell"
. "github.com/openshift-kni/eco-gosystem/tests/ran-du/internal/randuinittools"
"github.com/openshift-kni/eco-gosystem/tests/ran-du/internal/randuparams"
"github.com/openshift-kni/eco-gosystem/tests/ran-du/internal/randutestworkload"
"gonum.org/v1/gonum/floats"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

var _ = Describe(
"LaunchWorkloadMultipleIterations",
Ordered,
ContinueOnFailure,
Label("LaunchWorkloadMultipleIterations"), func() {
It("Launch workload multiple times", polarion.ID("45698"), Label("LaunchWorkloadMultipleIterations"), func() {

By("Clean up workload namespace")
if namespace.NewBuilder(APIClient, RanDuTestConfig.TestWorkload.Namespace).Exists() {
err := randutestworkload.CleanNameSpace(randuparams.DefaultTimeout, RanDuTestConfig.TestWorkload.Namespace)
Expect(err).ToNot(HaveOccurred(), "Failed to clean workload test namespace objects")
}

launchIterations, err := strconv.Atoi(RanDuTestConfig.LaunchWorkloadIterations)
if err != nil {
fmt.Println(err)
}

By("Launch workload")
for iter := 0; iter < launchIterations; iter++ {
fmt.Printf("Launch workload iteration no. %d\n", iter)
if RanDuTestConfig.TestWorkload.CreateMethod == randuparams.TestWorkloadShellLaunchMethod {
By("Launching workload using shell method")
_, err := shell.ExecuteCmd(RanDuTestConfig.TestWorkload.CreateShellCmd)
Expect(err).ToNot(HaveOccurred(), "Failed to launch workload")
}

By("Waiting for deployment replicas to become ready")
_, err := await.WaitUntilAllDeploymentsReady(APIClient, RanDuTestConfig.TestWorkload.Namespace,
randuparams.DefaultTimeout)
Expect(err).ToNot(HaveOccurred(), "error while waiting for deployment to become ready")

By("Waiting for statefulset replicas to become ready")
_, err = await.WaitUntilAllStatefulSetsReady(APIClient, RanDuTestConfig.TestWorkload.Namespace,
randuparams.DefaultTimeout)
Expect(err).ToNot(HaveOccurred(), "error while waiting for statefulsets to become ready")

By("Waiting for all pods to become ready")
_, err = await.WaitUntilAllPodsReady(APIClient, RanDuTestConfig.TestWorkload.Namespace, 10*time.Second)
Expect(err).ToNot(HaveOccurred(), "pod not ready: %s", err)
}
By("Retrieve nodes list")
nodeList, err := nodes.List(
APIClient,
metav1.ListOptions{},
)
Expect(err).ToNot(HaveOccurred(), "Error listing nodes.")

By("Observe node load average while workload is running")
cmdToExec := []string{"awk", "{print $1}", "/proc/loadavg"}
var observedLoadAverage []float64

for n := 0; n < 30; n++ {
buf, err := cmd.ExecCmd(cmdToExec, nodeList[0].Definition.Name)
Expect(err).ToNot(HaveOccurred(), "could not execute command: %s", err)

floatBuf, err := strconv.ParseFloat(strings.ReplaceAll(strings.ReplaceAll(buf, "\r", ""), "\n", ""), 32)
if err != nil {
fmt.Println(err)
}

observedLoadAverage = append(observedLoadAverage, floatBuf)
time.Sleep(10 * time.Second)
}

Expect(floats.Max(observedLoadAverage)).To(BeNumerically("<", randuparams.TestMultipleLaunchWorkloadLoadAvg),
"error: node load average detected above 100")

})
AfterAll(func() {
By("Cleaning up test workload resources")
err := randutestworkload.CleanNameSpace(randuparams.DefaultTimeout, RanDuTestConfig.TestWorkload.Namespace)
Expect(err).ToNot(HaveOccurred(), "Failed to clean workload test namespace objects")
})
})
128 changes: 128 additions & 0 deletions vendor/gonum.org/v1/gonum/AUTHORS

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading

0 comments on commit e48542e

Please sign in to comment.