Allow setting custom-entries-file in formats

This commit adds the the flag customEntriesFormat that sets the custom
entries file format (json/yaml/csv).

We also include 3 example files to help the user to create his own custom
file, and they'll be used in the integration test.

Signed-off-by: Lior Noy <[email protected]>

Makefile

@@ -30,7 +30,7 @@ endif
 generate: oc build
 	rm -rf $(DEST_DIR)/communication-matrix
 	mkdir -p $(DEST_DIR)/communication-matrix
-	./$(EXECUTABLE) -format=$(FORMAT) -env=$(CLUSTER_ENV) -destDir=$(DEST_DIR)/communication-matrix -deployment=$(DEPLOYMENT)
+	./$(EXECUTABLE) -format=$(FORMAT) -env=$(CLUSTER_ENV) -destDir=$(DEST_DIR)/communication-matrix -deployment=$(DEPLOYMENT) -customEntriesPath=$(CUSTOM_ENTRIES_PATH) -customEntriesFormat=$(CUSTOM_ENTRIES_FORMAT)
 
 clean:
 	@rm -f $(EXECUTABLE)

README.md

@@ -25,12 +25,18 @@ the `ss` command on each node, and converts the output into a corresponding ComD
 ### Communication Matrix Creation Guide
 
 Use the `generate` Makefile target to create the matrix.
+Add additional entires to the matrix via a custom file, using
+the variables `CUSTOM_ENTRIES_PATH` and `CUSTOM_ENTRIES_FORMAT`.
+Examples are available in the `example-custom-entries` files.
+
 The following environment variables are used to configure:
 ```
 FORMAT (csv/json/yaml)
 CLUSTER_ENV (baremetal/aws)
 DEST_DIR (path to the directory containing the artifacts)
 DEPLOYMENT (mno/sno)
+CUSTOM_ENTRIES_PATH (path to the file containing custom entries to add to the matrix)
+CUSTOM_ENTRIES_FORMAT (the format of the custom entries file (json,yaml,csv))
 ```
 
 The generated artifcats are:

cmd/main.go

@@ -22,19 +22,21 @@ import (
 
 func main() {
 	var (
-		destDir           string
-		format            string
-		envStr            string
-		deploymentStr     string
-		customEntriesPath string
-		printFn           func(m types.ComMatrix) ([]byte, error)
+		destDir             string
+		format              string
+		envStr              string
+		deploymentStr       string
+		customEntriesPath   string
+		customEntriesFormat string
+		printFn             func(m types.ComMatrix) ([]byte, error)
 	)
 
 	flag.StringVar(&destDir, "destDir", "communication-matrix", "Output files dir")
 	flag.StringVar(&format, "format", "csv", "Desired format (json,yaml,csv)")
 	flag.StringVar(&envStr, "env", "baremetal", "Cluster environment (baremetal/aws)")
 	flag.StringVar(&deploymentStr, "deployment", "mno", "Deployment type (mno/sno)")
-	flag.StringVar(&customEntriesPath, "customEntriesPath", "", "Add custom entries from a JSON file to the matrix")
+	flag.StringVar(&customEntriesPath, "customEntriesPath", "", "Add custom entries from a file to the matrix")
+	flag.StringVar(&customEntriesFormat, "customEntriesFormat", "", "Set the format of the custom entries file (json,yaml,csv)")
 
 	flag.Parse()
 
@@ -74,7 +76,11 @@ func main() {
 		panic(fmt.Sprintf("invalid deployment type: %s", deploymentStr))
 	}
 
-	mat, err := commatrix.New(kubeconfig, customEntriesPath, env, deployment)
+	if customEntriesPath != "" && customEntriesFormat == "" {
+		panic("error, variable customEntriesFormat is not set")
+	}
+
+	mat, err := commatrix.New(kubeconfig, customEntriesPath, customEntriesFormat, env, deployment)
 	if err != nil {
 		panic(fmt.Sprintf("failed to create the communication matrix: %s", err))
 	}

commatrix/commatrix.go

@@ -7,6 +7,9 @@ import (
 	"os"
 	"path/filepath"
 
+	"github.com/gocarina/gocsv"
+	"sigs.k8s.io/yaml"
+
 	"github.com/openshift-kni/commatrix/client"
 	"github.com/openshift-kni/commatrix/endpointslices"
 	"github.com/openshift-kni/commatrix/types"
@@ -34,7 +37,7 @@ const (
 // Custom entries from a JSON file can be added to the matrix by setting `customEntriesPath`.
 // Returns a pointer to ComMatrix and error. Entries include traffic direction, protocol,
 // port number, namespace, service name, pod, container, node role, and flow optionality for OpenShift.
-func New(kubeconfigPath string, customEntriesPath string, e Env, d Deployment) (*types.ComMatrix, error) {
+func New(kubeconfigPath string, customEntriesPath string, customEntriesFormat string, e Env, d Deployment) (*types.ComMatrix, error) {
 	res := make([]types.ComDetails, 0)
 
 	cs, err := client.New(kubeconfigPath)
@@ -55,15 +58,19 @@ func New(kubeconfigPath string, customEntriesPath string, e Env, d Deployment) (
 
 	staticEntries, err := getStaticEntries(e, d)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("failed adding static entries: %s", err)
 	}
 
 	res = append(res, staticEntries...)
 
 	if customEntriesPath != "" {
-		customComDetails, err := addFromFile(customEntriesPath)
+		inputFormat, err := parseFormat(customEntriesFormat)
 		if err != nil {
-			return nil, fmt.Errorf("failed fetching custom entries from file %s err: %w", customEntriesPath, err)
+			return nil, fmt.Errorf("failed adding custom entries: %s", err)
+		}
+		customComDetails, err := addFromFile(customEntriesPath, inputFormat)
+		if err != nil {
+			return nil, fmt.Errorf("failed adding custom entries: %s", err)
 		}
 
 		res = append(res, customComDetails...)
@@ -74,7 +81,7 @@ func New(kubeconfigPath string, customEntriesPath string, e Env, d Deployment) (
 	return &types.ComMatrix{Matrix: cleanedComDetails}, nil
 }
 
-func addFromFile(fp string) ([]types.ComDetails, error) {
+func addFromFile(fp string, format types.Format) ([]types.ComDetails, error) {
 	var res []types.ComDetails
 	f, err := os.Open(filepath.Clean(fp))
 	if err != nil {
@@ -85,10 +92,24 @@ func addFromFile(fp string) ([]types.ComDetails, error) {
 	if err != nil {
 		return nil, fmt.Errorf("failed to read file %s: %v", fp, err)
 	}
-
-	err = json.Unmarshal(raw, &res)
-	if err != nil {
-		return nil, fmt.Errorf("failed to unmarshal custom entries file: %v", err)
+	switch format {
+	case types.JSON:
+		err = json.Unmarshal(raw, &res)
+		if err != nil {
+			return nil, fmt.Errorf("failed to unmarshal custom entries file: %v", err)
+		}
+	case types.YAML:
+		err = yaml.Unmarshal(raw, &res)
+		if err != nil {
+			return nil, fmt.Errorf("failed to unmarshal custom entries file: %v", err)
+		}
+	case types.CSV:
+		err = gocsv.UnmarshalBytes(raw, &res)
+		if err != nil {
+			return nil, fmt.Errorf("failed to unmarshal custom entries file: %v", err)
+		}
+	default:
+		return nil, fmt.Errorf("invalid value for format must be (json,yaml,csv)")
 	}
 
 	return res, nil
@@ -124,3 +145,16 @@ func getStaticEntries(e Env, d Deployment) ([]types.ComDetails, error) {
 
 	return comDetails, nil
 }
+
+func parseFormat(format string) (types.Format, error) {
+	switch format {
+	case types.FormatJSON:
+		return types.JSON, nil
+	case types.FormatYAML:
+		return types.YAML, nil
+	case types.FormatCSV:
+		return types.CSV, nil
+	}
+
+	return types.FormatErr, fmt.Errorf("failed to parse format: %s. options are: (json/yaml/csv)", format)
+}

commatrix/example-custom-entries.csv

@@ -0,0 +1,3 @@
+Direction,Protocol,Port,Namespace,Service,Pod,Container,NodeRole,Optional
+ingress,TCP,9050,example-namespace,example-service,example-pod,example-container,master,false
+ingress,UDP,9051,example-namespace2,example-service2,example-pod2,example-container2,woker,false

commatrix/example-custom-entries.json

@@ -0,0 +1,24 @@
+[
+    {
+        "direction": "ingress",
+        "protocol": "TCP",
+        "port": "9050",
+        "namespace": "example-namespace",
+        "service": "example-service",
+        "pod": "example-pod",
+        "container": "example-container",
+        "nodeRole": "master",
+        "optional": false
+    },
+    {
+        "direction": "ingress",
+        "protocol": "UDP",
+        "port": "9051",
+        "namespace": "example-namespace2",
+        "service": "example-service2",
+        "pod": "example-pod2",
+        "container": "example-container2",
+        "nodeRole": "worker",
+        "optional": false
+    }
+]

commatrix/example-custom-entries.yaml

@@ -0,0 +1,18 @@
+- container: example-container
+  direction: ingress
+  namespace: example-namespace
+  nodeRole: master
+  optional: false
+  pod: example-pod
+  port: "9050"
+  protocol: TCP
+  service: example-service
+- container: example-container2
+  direction: ingress
+  namespace: example-namespace2
+  nodeRole: worker
+  optional: false
+  pod: example-pod2
+  port: "9051"
+  protocol: UDP
+  service: example-service2

go.mod

@@ -3,6 +3,7 @@ module github.com/openshift-kni/commatrix
 go 1.21
 
 require (
+	github.com/gocarina/gocsv v0.0.0-20231116093920-b87c2d0e983a
 	github.com/openshift/client-go v0.0.0-20240415214935-be70f772f157
 	github.com/sirupsen/logrus v1.9.3
 	k8s.io/api v0.29.3

go.sum

@@ -21,6 +21,8 @@ github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/
 github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14=
 github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
 github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls=
+github.com/gocarina/gocsv v0.0.0-20231116093920-b87c2d0e983a h1:RYfmiM0zluBJOiPDJseKLEN4BapJ42uSi9SZBQ2YyiA=
+github.com/gocarina/gocsv v0.0.0-20231116093920-b87c2d0e983a/go.mod h1:5YoVOkjYAQumqlV356Hj3xeYh4BdZuLE0/nRkf2NKkI=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=

types/types.go

@@ -13,20 +13,35 @@ import (
 	"sigs.k8s.io/yaml"
 )
 
+type Format int
+
+const (
+	FormatErr        = -1
+	JSON      Format = iota
+	YAML
+	CSV
+)
+
+const (
+	FormatJSON = "json"
+	FormatYAML = "yaml"
+	FormatCSV  = "csv"
+)
+
 type ComMatrix struct {
 	Matrix []ComDetails
 }
 
 type ComDetails struct {
-	Direction string `json:"direction"`
-	Protocol  string `json:"protocol"`
-	Port      int    `json:"port"`
-	Namespace string `json:"namespace"`
-	Service   string `json:"service"`
-	Pod       string `json:"pod"`
-	Container string `json:"container"`
-	NodeRole  string `json:"nodeRole"`
-	Optional  bool   `json:"optional"`
+	Direction string `json:"direction" yaml:"direction"`
+	Protocol  string `json:"protocol" yaml:"protocol"`
+	Port      string `json:"port" yaml:"port"`
+	Namespace string `json:"namespace" yaml:"namespace"`
+	Service   string `json:"service" yaml:"service"`
+	Pod       string `json:"pod" yaml:"pod"`
+	Container string `json:"container" yaml:"container"`
+	NodeRole  string `json:"nodeRole" yaml:"nodeRole"`
+	Optional  bool   `json:"optional" yaml:"optional"`
 }
 
 func ToCSV(m ComMatrix) ([]byte, error) {
@@ -52,7 +67,7 @@ func ToCSV(m ComMatrix) ([]byte, error) {
 }
 
 func ToJSON(m ComMatrix) ([]byte, error) {
-	out, err := json.Marshal(m.Matrix)
+	out, err := json.MarshalIndent(m.Matrix, "", "    ")
 	if err != nil {
 		return nil, err
 	}