From 50e88489cfd4b348f64183030bcb03e7794f12eb Mon Sep 17 00:00:00 2001 From: Dhruv-J Date: Wed, 12 Jul 2023 08:30:17 -0700 Subject: [PATCH] finished getting and managing coverage files Signed-off-by: Dhruv-J --- .github/workflows/kind.yml | 4 + build/charts/theia/templates/_helpers.tpl | 11 +- .../templates/theia-manager/deployment.yaml | 8 +- build/charts/theia/values.yaml | 3 +- build/yamls/flow-visibility.yml | 19 +- ci/kind/test-e2e-kind.sh | 12 +- test/e2e/framework.go | 175 ++++++++++++++---- 7 files changed, 184 insertions(+), 48 deletions(-) diff --git a/.github/workflows/kind.yml b/.github/workflows/kind.yml index 1147e39fa..cb0ee729c 100644 --- a/.github/workflows/kind.yml +++ b/.github/workflows/kind.yml @@ -158,6 +158,10 @@ jobs: - name: Tar log files if: ${{ failure() }} run: tar -czf log.tar.gz log + # - name: collect coverage files from node + # run: | + # docker cp $(docker inspect --format="{{.Id}}" kind-worker2):/var/log/cm-coverage/ ~/go/src/github.com/theia/.coverage + # docker cp $(docker inspect --format="{{.Id}}" kind-worker):/var/log/cm-coverage/. ~/go/src/github.com/theia/.coverage/cm-coverage - name: Codecov uses: codecov/codecov-action@v3 with: diff --git a/build/charts/theia/templates/_helpers.tpl b/build/charts/theia/templates/_helpers.tpl index e00b950a0..3609eea8f 100644 --- a/build/charts/theia/templates/_helpers.tpl +++ b/build/charts/theia/templates/_helpers.tpl @@ -4,6 +4,9 @@ - name: clickhouse-monitor image: {{ include "clickHouseMonitorImage" . | quote }} imagePullPolicy: {{ $clickhouse.monitor.image.pullPolicy }} + volumeMounts: + - name: clickhouse-monitor-coverage + mountPath: /clickhouse-monitor-coverage env: - name: CLICKHOUSE_USERNAME valueFrom: @@ -32,7 +35,7 @@ - name: SKIP_ROUNDS_NUM value: {{ $clickhouse.monitor.skipRoundsNum | quote }} - name: GOCOVERDIR - value: {{ $clickhouse.monitor.gocoverdir }} + value: "/clickhouse-monitor-coverage" {{- end }} {{- define "clickhouse.server.container" }} @@ -49,8 +52,6 @@ - name: clickhouse-storage-volume mountPath: /var/lib/clickhouse {{- end }} - - name: clickhouse-monitor-coverage-volume - mountPath: /clickhouse-monitor-coverage env: - name: THEIA_VERSION value: {{ $Chart.Version }} @@ -93,9 +94,9 @@ sizeLimit: {{ $clickhouse.storage.size }} {{- end }} - hostPath: - path: /var/log/coverage + path: /var/log/cm-coverage type: DirectoryOrCreate - name: clickhouse-monitor-coverage-volume + name: clickhouse-monitor-coverage {{- end }} diff --git a/build/charts/theia/templates/theia-manager/deployment.yaml b/build/charts/theia/templates/theia-manager/deployment.yaml index fdc7806a2..455239e7b 100644 --- a/build/charts/theia/templates/theia-manager/deployment.yaml +++ b/build/charts/theia/templates/theia-manager/deployment.yaml @@ -53,7 +53,7 @@ spec: - name: CLICKHOUSE_URL value: "tcp://clickhouse-clickhouse.{{ .Release.Namespace }}.svc:{{ .Values.clickhouse.service.tcpPort }}" - name: GOCOVERDIR - value: {{ .Values.theiaManager.gocoverdir }} + value: "/theia-manager-coverage" ports: - name: "theia-api-http" containerPort: {{ .Values.theiaManager.apiServer.apiPort }} @@ -65,6 +65,8 @@ spec: name: theia-manager-tls - mountPath: /var/log/antrea/theia-manager name: host-var-log-antrea-theia-manager + - mountPath: /theia-manager-coverage + name: theia-manager-coverage nodeSelector: kubernetes.io/os: linux kubernetes.io/arch: amd64 @@ -83,4 +85,8 @@ spec: hostPath: path: /var/log/antrea/theia-manager type: DirectoryOrCreate + - name: theia-manager-coverage + hostPath: + path: /var/log/tm-coverage + type: DirectoryOrCreate {{- end }} diff --git a/build/charts/theia/values.yaml b/build/charts/theia/values.yaml index 0662528cd..a10ec1b37 100644 --- a/build/charts/theia/values.yaml +++ b/build/charts/theia/values.yaml @@ -272,5 +272,4 @@ theiaManager: tlsMinVersion: "" # -- Log verbosity switch for Theia Manager. logVerbosity: 0 - # -- coverage directory to be used - gocoverdir: theia-manager-coverage + # -- coverage directory to be used \ No newline at end of file diff --git a/build/yamls/flow-visibility.yml b/build/yamls/flow-visibility.yml index 51148769f..02864a9e3 100644 --- a/build/yamls/flow-visibility.yml +++ b/build/yamls/flow-visibility.yml @@ -6944,7 +6944,7 @@ spec: - name: CLICKHOUSE_URL value: tcp://clickhouse-clickhouse.flow-visibility.svc:9000 - name: GOCOVERDIR - value: theia-manager-coverage + value: /theia-manager-coverage image: projects.registry.vmware.com/antrea/theia-manager:latest imagePullPolicy: IfNotPresent name: theia-manager @@ -6959,6 +6959,8 @@ spec: name: theia-manager-tls - mountPath: /var/log/antrea/theia-manager name: host-var-log-antrea-theia-manager + - mountPath: /theia-manager-coverage + name: theia-manager-coverage nodeSelector: kubernetes.io/os: linux serviceAccountName: theia-manager @@ -6975,6 +6977,10 @@ spec: path: /var/log/antrea/theia-manager type: DirectoryOrCreate name: host-var-log-antrea-theia-manager + - hostPath: + path: /var/log/tm-coverage + type: DirectoryOrCreate + name: theia-manager-coverage --- apiVersion: apps/v1 kind: StatefulSet @@ -7222,8 +7228,6 @@ spec: name: clickhouse-configmap-volume - mountPath: /var/lib/clickhouse name: clickhouse-storage-volume - - mountPath: /clickhouse-monitor-coverage - name: clickhouse-monitor-coverage-volume - env: - name: CLICKHOUSE_USERNAME valueFrom: @@ -7252,10 +7256,13 @@ spec: - name: SKIP_ROUNDS_NUM value: "3" - name: GOCOVERDIR - value: clickhouse-monitor-coverage + value: /clickhouse-monitor-coverage image: projects.registry.vmware.com/antrea/theia-clickhouse-monitor:latest imagePullPolicy: IfNotPresent name: clickhouse-monitor + volumeMounts: + - mountPath: /clickhouse-monitor-coverage + name: clickhouse-monitor-coverage volumes: - configMap: items: @@ -7290,9 +7297,9 @@ spec: sizeLimit: 8Gi name: clickhouse-storage-volume - hostPath: - path: /var/log/coverage + path: /var/log/cm-coverage type: DirectoryOrCreate - name: clickhouse-monitor-coverage-volume + name: clickhouse-monitor-coverage serviceTemplates: - name: service-template spec: diff --git a/ci/kind/test-e2e-kind.sh b/ci/kind/test-e2e-kind.sh index d94fd5d52..388296e46 100755 --- a/ci/kind/test-e2e-kind.sh +++ b/ci/kind/test-e2e-kind.sh @@ -150,6 +150,7 @@ function setup_cluster { } function run_test { + echo "STARTEDSTARTED_RUNNING" TMP_DIR=$(mktemp -d $(dirname $0)/tmp.XXXXXXXX) curl -o $TMP_DIR/antrea.yml https://raw.githubusercontent.com/antrea-io/antrea/main/build/yamls/antrea.yml # cp ~/go/src/github.com/antrea/build/yamls/antrea.yml $TMP_DIR/flow-aggregator @@ -161,7 +162,7 @@ function run_test { sed -i -e "s/idleFlowExportTimeout: \"15s\"/idleFlowExportTimeout: \"1s\"/g" $TMP_DIR/antrea.yml curl -o $TMP_DIR/flow-aggregator.yml https://raw.githubusercontent.com/antrea-io/antrea/main/build/yamls/flow-aggregator.yml - #cp ~/go/src/github.com/antrea/build/yamls/flow-aggregator.yml $TMP_DIR/flow-aggregator.yml + # cp ~/go/src/github.com/antrea/build/yamls/flow-aggregator.yml $TMP_DIR/flow-aggregator.yml sed -i -e "s|image: projects.registry.vmware.com/antrea/flow-aggregator:latest|image: antrea/flow-aggregator:latest|g" $TMP_DIR/flow-aggregator.yml perl -i -p0e 's/ # Enable is the switch to enable exporting flow records to ClickHouse.\n enable: false/ # Enable is the switch to enable exporting flow records to ClickHouse.\n enable: true/' $TMP_DIR/flow-aggregator.yml sed -i -e "s/ activeFlowRecordTimeout: 60s/ activeFlowRecordTimeout: 3500ms/g" $TMP_DIR/flow-aggregator.yml @@ -169,6 +170,8 @@ function run_test { sed -i -e "s/ podLabels: false/ podLabels: true/g" $TMP_DIR/flow-aggregator.yml sed -i -e "s/ commitInterval: \"8s\"/ commitInterval: \"1s\"/g" $TMP_DIR/flow-aggregator.yml + echo "1MID_RUNNING" + docker exec -i kind-control-plane dd of=/root/antrea.yml < $TMP_DIR/antrea.yml docker exec -i kind-control-plane dd of=/root/flow-aggregator.yml < $TMP_DIR/flow-aggregator.yml docker exec -i kind-control-plane dd of=/root/clickhouse-operator-install-bundle.yaml < $CH_OPERATOR_YML @@ -176,20 +179,27 @@ function run_test { $FLOW_VISIBILITY_WITH_SPARK_CMD | docker exec -i kind-control-plane dd of=/root/flow-visibility-with-spark.yml $FLOW_VISIBILITY_CH_ONLY_CMD | docker exec -i kind-control-plane dd of=/root/flow-visibility-ch-only.yml + echo "2MID_RUNNING" + docker exec -i kind-control-plane dd of=/root/theia < $THEIACTL_BIN rm -rf $TMP_DIR sleep 1 + echo "3MID_RUNNING" + if $coverage; then mkdir -p .coverage/clickhouse-monitor-coverage/ mkdir -p .coverage/theia-manager-coverage/ mkdir -p .coverage/merged + echo "STARTED_RUNNING" go test -v -timeout=30m antrea.io/theia/test/e2e -provider=kind --logs-export-dir=$ANTREA_LOG_DIR -cover -coverprofile=.coverage/.cov.out -covermode=atomic --skip=$skiplist else go test -v -timeout=30m antrea.io/theia/test/e2e -provider=kind --logs-export-dir=$ANTREA_LOG_DIR --skip=$skiplist fi + echo "FINISHED_RUNNING" + go tool covdata merge -i=.coverage/clickhouse-monitor-coverage,.coverage/theia-manager.coverage -o .coverage/merged go tool covdata textfmt -i=.coverage/merged -o .coverage/kind-e2e-coverage.txt } diff --git a/test/e2e/framework.go b/test/e2e/framework.go index 7ad570cec..218125288 100644 --- a/test/e2e/framework.go +++ b/test/e2e/framework.go @@ -1535,48 +1535,99 @@ func (data *TestData) Cleanup(namespaces []string) { } } -func (data *TestData) copyPodFiles(podName string, containerName string, nsName string, fileName string, covDir string) error { - fmt.Println("_______________ copyPodFiles") - // getPodWriter creates the file with name podName-fileName-suffix. It returns nil if the - // file cannot be created. File must be closed by the caller. - getPodWriter := func(podName, fileName string) *os.File { - destFile, err := filepath.Abs(filepath.Join("../.././", covDir, fileName)) +// func (data *TestData) copyPodFiles(podName string, containerName string, nsName string, fileName string, covDir string) error { +// fmt.Println("_______________ copyPodFiles") +// // getPodWriter creates the file with name podName-fileName-suffix. It returns nil if the +// // file cannot be created. File must be closed by the caller. +// getPodWriter := func(podName, fileName string) *os.File { +// destFile, err := filepath.Abs(filepath.Join("../.././", covDir, fileName)) +// if err != nil { +// fmt.Println(fmt.Sprintf("FILE PATH ERROR IS NOT NIL: %v", err)) +// return nil +// } +// fmt.Println("QWERTY destination file is: " + destFile) +// f, err := os.Create(destFile) +// if err != nil { +// fmt.Println(fmt.Sprintf("FILE CREATION ERROR IS NOT NIL: %v", err)) +// _ = fmt.Errorf("error when creating destination file '%s': %v", destFile, err) +// return nil +// } +// if f == nil { +// fmt.Println("NILLED F MID") +// } +// return f +// } + +// // dump the file from Theia Pods to disk. +// w := getPodWriter(podName, fileName) +// if w == nil { +// return nil +// } +// defer w.Close() +// cmd := []string{"cat", fileName} +// stdout, stderr, err := data.RunCommandFromPod(nsName, podName, containerName, cmd) +// if err != nil { +// return fmt.Errorf("cannot retrieve content of file '%s' from Pod '%s', stderr: <%v>, err: <%v>", fileName, podName, stderr, err) +// } +// if stdout == "" { +// return nil +// } +// w.WriteString(stdout) +// return nil +// } + +func (data *TestData) copyNodeFiles(nodeName, fileName, covDir, covPrefix string) error { + fmt.Println("-------------- copyNodeFiles: node " + nodeName + " has file: " + fileName) + // getNodeWriter creates the file with name nodeName-suffix. It returns nil if the file + // cannot be created. File must be closed by the caller. + getNodeWriter := func(nodeName, fileName, suffix string) *os.File { + covdirabs, err := filepath.Abs("../../" + covDir) if err != nil { - fmt.Println(fmt.Sprintf("FILE PATH ERROR IS NOT NIL: %v", err)) - return nil + fmt.Println(fmt.Sprintf("_______________ copyNodeFiles: error creating absolute file path: %v", err)) } - fmt.Println("QWERTY destination file is: " + destFile) - f, err := os.Create(destFile) + covFile := filepath.Join(covdirabs, fmt.Sprintf("%s-%s-%s", nodeName, fileName, suffix)) + fmt.Println("_______________ copyNodeFiles covFile: " + covFile) + f, err := os.Create(covFile) if err != nil { - fmt.Println(fmt.Sprintf("FILE CREATION ERROR IS NOT NIL: %v", err)) - _ = fmt.Errorf("error when creating destination file '%s': %v", destFile, err) + _ = fmt.Errorf("error when creating coverage file '%s': %v", covFile, err) + fmt.Println(fmt.Sprintf("_______________ copyNodeFiles: error creating coverage file: %v", err)) return nil } if f == nil { - fmt.Println("NILLED F MID") + fmt.Println("_______________ copyNodeFiles: file is nil") + } else { + fmt.Println("_______________ copyNodeFiles: file is NOT nil") } return f } - // dump the file from Theia Pods to disk. - w := getPodWriter(podName, fileName) + // dump the file from Antrea Pods to disk. + // a filepath-friendly timestamp format. + const timeFormat = "Jan02-15-04-05" + timeStamp := time.Now().Format(timeFormat) + w := getNodeWriter(nodeName, fileName, timeStamp) if w == nil { + fmt.Println("_______________ copyNodeFiles: nodewriter is null") return nil } defer w.Close() - cmd := []string{"cat", fileName} - stdout, stderr, err := data.RunCommandFromPod(nsName, podName, containerName, cmd) - if err != nil { - return fmt.Errorf("cannot retrieve content of file '%s' from Pod '%s', stderr: <%v>, err: <%v>", fileName, podName, stderr, err) + fileName = "/var/log/" + covPrefix + "-coverage/" + fileName + cmd := fmt.Sprintf("cat %s", fileName) + fmt.Println("_______________ copyNodeFiles: cat command set") + rc, stdout, stderr, err := data.RunCommandOnNode(nodeName, cmd) + if err != nil || rc != 0 { + return fmt.Errorf("cannot retrieve content of file '%s' from Node '%s', stderr: <%v>, err: <%v>", fileName, controlPlaneNodeName(), stderr, err) } if stdout == "" { + fmt.Println("_______________ copyNodeFiles: stdout was nil") return nil } w.WriteString(stdout) + fmt.Println("_______________ copyNodeFiles: finished writing string") return nil } -func (data *TestData) killProcessesAndCollectCovFiles(namespace, podName, containerName, processName, covDir string) error { +func (data *TestData) killProcessesAndCollectCovFiles(namespace, podName, containerName, processName, covDir, covPrefix string) error { cmds := []string{"pgrep", "-f", processName} stdout, stderr, err := data.RunCommandFromPod(namespace, podName, containerName, cmds) if err != nil { @@ -1590,30 +1641,87 @@ func (data *TestData) killProcessesAndCollectCovFiles(namespace, podName, contai return fmt.Errorf("error when sending SIGINT signal to '%s', stderr: <%v>, err: <%v>", processName, stderr, err) } - log.Infof("Copying coverage files from Pod '%s'", podName) - cmds = []string{"bash", "-c", "find / -name 'covmeta.*' -exec basename {} ';'"} - stdout, stderr, err = data.RunCommandFromPod(namespace, podName, containerName, cmds) - if err != nil { - return fmt.Errorf("error when running this find command (for coverage file) '%s' on Pod '%s', stderr: <%v>, err: <%v>", cmds, podName, stderr, err) + log.Infof("Copying coverage files from worker nodes kind-worker and kind-worker2") + cmd := "/bin/sh -c find / -name 'covmeta.*' -exec basename {} ';'" + rc, stdout, stderr, err := data.RunCommandOnNode("kind-worker", cmd) + if err != nil || rc != 0 { + return fmt.Errorf("error when running this find command (for coverage file) '%s' on Node kind-worker, stderr: <%v>, err: <%v>", cmd, stderr, err) } stdout = strings.TrimSpace(stdout) files := strings.Split(stdout, "\n") - cmds = []string{"bash", "-c", "find / -name 'covcounters.*' -exec basename {} ';'"} - stdout, stderr, err = data.RunCommandFromPod(namespace, podName, containerName, cmds) - if err != nil { - return fmt.Errorf("error when running this find command (for coverage file) '%s' on Pod '%s', stderr: <%v>, err: <%v>", cmds, podName, stderr, err) + cmd = "/bin/sh -c find / -name 'covcounters.*' -exec basename {} ';'" + rc, stdout, stderr, err = data.RunCommandOnNode("kind-worker", cmd) + if err != nil || rc != 0 { + return fmt.Errorf("error when running this find command (for coverage file) '%s' on Node kind-worker, stderr: <%v>, err: <%v>", cmd, stderr, err) + } + stdout = strings.TrimSpace(stdout) + files = append(files, strings.Split(stdout, "\n")...) + fmt.Println("-------- length of kind-worker files: " + fmt.Sprint(len(files))) + for i, file := range files { + fmt.Println("copyNodeFiles iteration " + fmt.Sprint(i)) + if len(file) == 0 { + fmt.Println("file " + file + " was empty") + continue + } + err = data.copyNodeFiles("kind-worker", file, covDir, covPrefix) + if err != nil { + return fmt.Errorf("error when copying coverage files from Pod '%s' to coverage directory '%s': %v", podName, covDir, err) + } + } + cmd = "/bin/sh -c find / -name 'covmeta.*' -exec basename {} ';'" + rc, stdout, stderr, err = data.RunCommandOnNode("kind-worker2", cmd) + if err != nil || rc != 0 { + return fmt.Errorf("error when running this find command (for coverage file) '%s' on Node kind-worker2, stderr: <%v>, err: <%v>", cmd, stderr, err) + } + stdout = strings.TrimSpace(stdout) + files = strings.Split(stdout, "\n") + cmd = "/bin/sh -c find / -name 'covcounters.*' -exec basename {} ';'" + rc, stdout, stderr, err = data.RunCommandOnNode("kind-worker2", cmd) + if err != nil || rc != 0 { + return fmt.Errorf("error when running this find command (for coverage file) '%s' on Node kind-worker2, stderr: <%v>, err: <%v>", cmd, stderr, err) } stdout = strings.TrimSpace(stdout) files = append(files, strings.Split(stdout, "\n")...) - for _, file := range files { + fmt.Println("-------- length of kind-worker2 files: " + fmt.Sprint(len(files))) + for i, file := range files { + fmt.Println("copyNodeFiles iteration " + fmt.Sprint(i)) if len(file) == 0 { + fmt.Println("file " + file + " was empty") continue } - err := data.copyPodFiles(podName, containerName, namespace, file, covDir) + err := data.copyNodeFiles("kind-worker2", file, covDir, covPrefix) if err != nil { + fmt.Println(fmt.Sprintf("killProcessesAndCollectCovFiles: error copying node files: %v", err)) return fmt.Errorf("error when copying coverage files from Pod '%s' to coverage directory '%s': %v", podName, covDir, err) } } + // cmd = "/bin/sh -c find / -name 'covmeta.*' -exec basename {} ';'" + // rc, stdout, stderr, err = data.RunCommandOnNode("kind-worker2", cmd) + // if err != nil || rc != 0 { + // return fmt.Errorf("error when running this find command (for coverage file) '%s' on Node kind-worker2, stderr: <%v>, err: <%v>", cmd, stderr, err) + // } + // stdout = strings.TrimSpace(stdout) + // files = strings.Split(stdout, "\n") + // cmd = "/bin/sh -c find / -name 'covcounters.*' -exec basename {} ';'" + // rc, stdout, stderr, err = data.RunCommandOnNode("kind-worker", cmd) + // if err != nil || rc != 0 { + // return fmt.Errorf("error when running this find command (for coverage file) '%s' on Node kind-worker, stderr: <%v>, err: <%v>", cmd, stderr, err) + // } + // stdout = strings.TrimSpace(stdout) + // files = append(files, strings.Split(stdout, "\n")...) + // fmt.Println("-------- length of kind-worker2 files: "+fmt.Sprint(len(files))) + // for _, file := range files { + // fmt.Println("copyNodeFiles iteration ") + // if len(file) == 0 { + // fmt.Println("file "+file+" was empty"); + // continue + // } + // // err := data.copyPodFiles(podName, containerName, namespace, file, covDir) + // err := data.copyNodeFiles("kind-worker2", file, covDir) + // if err != nil { + // return fmt.Errorf("error when copying coverage files from Pod '%s' to coverage directory '%s': %v", podName, covDir, err) + // } + // } return nil } @@ -1632,13 +1740,14 @@ func (data *TestData) copyCovFilesFromPods(covDir string, nodeName string) error for _, pod := range pods.Items { podName := pod.Name if strings.Contains(podName, "chi-clickhouse-clickhouse") { - err = data.killProcessesAndCollectCovFiles("flow-visibility", pod.Name, clickHouseMonitorContName, clickHouseMonitorContName, covDir+"/"+clickHouseMonitorCovFolder) + err = data.killProcessesAndCollectCovFiles("flow-visibility", pod.Name, clickHouseMonitorContName, clickHouseMonitorContName, covDir+"/"+clickHouseMonitorCovFolder, "cm") } else if strings.Contains(podName, "theia-manager") { - err = data.killProcessesAndCollectCovFiles("flow-visibility", pod.Name, theiaManagerContName, theiaManagerContName, covDir+"/"+theiaManagerCovFolder) + err = data.killProcessesAndCollectCovFiles("flow-visibility", pod.Name, theiaManagerContName, theiaManagerContName, covDir+"/"+theiaManagerCovFolder, "tm") } else { fmt.Println("no coverage files to look for") } if err != nil { + fmt.Println(fmt.Sprintf("copyCovFilesFromPods: error copying node files: %v", err)) return fmt.Errorf("error when copying coverage files from pods: copy pod files out, error:%v", err) } }