diff --git a/api/config/v1/flags.go b/api/config/v1/flags.go index 2720379e2..60cc0d67c 100644 --- a/api/config/v1/flags.go +++ b/api/config/v1/flags.go @@ -107,6 +107,7 @@ type GFDCommandLineFlags struct { NoTimestamp *bool `json:"noTimestamp" yaml:"noTimestamp"` SleepInterval *Duration `json:"sleepInterval" yaml:"sleepInterval"` OutputFile *string `json:"outputFile" yaml:"outputFile"` + ImexNodesConfig *string `json:"imexNodesConfig" yaml:"imexNodesConfig"` MachineTypeFile *string `json:"machineTypeFile" yaml:"machineTypeFile"` } @@ -162,6 +163,8 @@ func (f *Flags) UpdateFromCLIFlags(c *cli.Context, flags []cli.Flag) { updateFromCLIFlag(&f.GFD.Oneshot, c, n) case "output-file": updateFromCLIFlag(&f.GFD.OutputFile, c, n) + case "imex-nodes-config": + updateFromCLIFlag(&f.GFD.ImexNodesConfig, c, n) case "sleep-interval": updateFromCLIFlag(&f.GFD.SleepInterval, c, n) case "no-timestamp": diff --git a/cmd/gpu-feature-discovery/main.go b/cmd/gpu-feature-discovery/main.go index c824ffcc2..80895a61c 100644 --- a/cmd/gpu-feature-discovery/main.go +++ b/cmd/gpu-feature-discovery/main.go @@ -86,6 +86,12 @@ func main() { Value: "/etc/kubernetes/node-feature-discovery/features.d/gfd", EnvVars: []string{"GFD_OUTPUT_FILE"}, }, + &cli.StringFlag{ + Name: "imex-nodes-config", + Usage: "the path to nvidia-imex nodes config file", + Value: "/etc/nvidia-imex/nodes_config.cfg", + EnvVars: []string{"GFD_IMEX_NODES_CONFIG"}, + }, &cli.StringFlag{ Name: "machine-type-file", Value: "/sys/class/dmi/id/product_name", diff --git a/deployments/container/Dockerfile b/deployments/container/Dockerfile index ae2c6d997..9e38d4467 100644 --- a/deployments/container/Dockerfile +++ b/deployments/container/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -ARG GOLANG_VERSION=1.22.6 +ARG GOLANG_VERSION=1.23.1 FROM nvcr.io/nvidia/cuda:12.6.0-base-ubi9 AS build RUN yum install -y \ @@ -30,7 +30,7 @@ RUN set -eux; \ aarch64) ARCH='arm64' ;; \ *) echo "unsupported architecture" ; exit 1 ;; \ esac; \ - wget -nv -O - https://storage.googleapis.com/golang/go${GOLANG_VERSION}.linux-${ARCH}.tar.gz \ + wget -nv -O - https://go.dev/dl/go1.23.1.linux-arm64.tar.gz \ | tar -C /usr/local -xz ENV GOPATH /go diff --git a/deployments/container/native-only.mk b/deployments/container/native-only.mk index e998246bd..becccedd6 100644 --- a/deployments/container/native-only.mk +++ b/deployments/container/native-only.mk @@ -13,7 +13,7 @@ # limitations under the License. PUSH_ON_BUILD ?= false -DOCKER_BUILD_PLATFORM_OPTIONS = --platform=linux/amd64 +DOCKER_BUILD_PLATFORM_OPTIONS = --platform=linux/aarch64 ifeq ($(PUSH_ON_BUILD),true) $(BUILD_TARGETS): build-%: image-% diff --git a/deployments/devel/Dockerfile b/deployments/devel/Dockerfile index e08a7b205..5284e73cc 100644 --- a/deployments/devel/Dockerfile +++ b/deployments/devel/Dockerfile @@ -14,7 +14,7 @@ # This Dockerfile is also used to define the golang version used in this project # This allows dependabot to manage this version in addition to other images. -FROM golang:1.22.6 +FROM golang:1.23 WORKDIR /work COPY * . diff --git a/deployments/devel/go.mod b/deployments/devel/go.mod index c65570abd..9523451e7 100644 --- a/deployments/devel/go.mod +++ b/deployments/devel/go.mod @@ -1,6 +1,7 @@ module github.com/NVIDIA/k8s-device-plugin/deployments/devel -go 1.22 +go 1.23 + toolchain go1.23.0 require github.com/matryer/moq v0.5.0 diff --git a/deployments/helm/nvidia-device-plugin/templates/daemonset-gfd.yml b/deployments/helm/nvidia-device-plugin/templates/daemonset-gfd.yml index 940dcc902..e724d6ece 100644 --- a/deployments/helm/nvidia-device-plugin/templates/daemonset-gfd.yml +++ b/deployments/helm/nvidia-device-plugin/templates/daemonset-gfd.yml @@ -163,6 +163,10 @@ spec: - name: GFD_USE_NODE_FEATURE_API value: {{ .Values.nfd.enableNodeFeatureApi | quote }} {{- end }} + {{- if typeIs "string" .Values.imexNodesConfigFile }} + - name: GFD_IMEX_NODES_CONFIG + value: {{ .Values.imexNodesConfigFile | quote }} + {{- end }} {{- if $options.hasConfigMap }} - name: CONFIG_FILE value: /config/config.yaml @@ -182,6 +186,10 @@ spec: mountPath: "/etc/kubernetes/node-feature-discovery/features.d" - name: host-sys mountPath: "/sys" + {{- if typeIs "string" .Values.imexNodesConfigFile }} + - name: imex-nodes-config + mountPath: {{ .Values.imexNodesConfigFile | quote }} + {{- end }} {{- if $options.hasConfigMap }} - name: available-configs mountPath: /available-configs @@ -199,6 +207,11 @@ spec: - name: host-sys hostPath: path: "/sys" + {{- if typeIs "string" .Values.imexNodesConfigFile }} + - name: imex-nodes-config + hostPath: + path: {{ .Values.imexNodesConfigFile | quote }} + {{- end }} {{- if $options.hasConfigMap }} - name: available-configs configMap: diff --git a/deployments/helm/nvidia-device-plugin/values.yaml b/deployments/helm/nvidia-device-plugin/values.yaml index b0c624295..cea95c416 100644 --- a/deployments/helm/nvidia-device-plugin/values.yaml +++ b/deployments/helm/nvidia-device-plugin/values.yaml @@ -35,6 +35,8 @@ deviceIDStrategy: null nvidiaDriverRoot: null gdsEnabled: null mofedEnabled: null +# Default value is "/etc/nvidia-imex/nodes_config.cfg" +imexNodesConfigFile: null deviceDiscoveryStrategy: null nameOverride: "" diff --git a/internal/lm/nvml.go b/internal/lm/nvml.go index 0b5ed6e9a..b5b31d47d 100644 --- a/internal/lm/nvml.go +++ b/internal/lm/nvml.go @@ -17,8 +17,13 @@ package lm import ( + "bufio" "errors" "fmt" + "math/rand" + "net" + "os" + "sort" "strconv" "strings" @@ -28,6 +33,7 @@ import ( spec "github.com/NVIDIA/k8s-device-plugin/api/config/v1" "github.com/NVIDIA/k8s-device-plugin/internal/resource" + "github.com/google/uuid" ) var errMPSSharingNotSupported = errors.New("MPS sharing is not supported") @@ -80,6 +86,11 @@ func NewDeviceLabeler(manager resource.Manager, config *spec.Config) (Labeler, e return nil, fmt.Errorf("error creating resource labeler: %v", err) } + imexLabeler, err := newImexDomainLabeler(*config.Flags.GFD.ImexNodesConfig, devices) + if err != nil { + return nil, fmt.Errorf("error creating imex domain labeler: %v", err) + } + l := Merge( machineTypeLabeler, versionLabeler, @@ -87,6 +98,7 @@ func NewDeviceLabeler(manager resource.Manager, config *spec.Config) (Labeler, e sharingLabeler, resourceLabeler, gpuModeLabeler, + imexLabeler, ) return l, nil @@ -218,6 +230,79 @@ func newGPUModeLabeler(devices []resource.Device) (Labeler, error) { return labels, nil } +func newImexDomainLabeler(configFile string, device []resource.Device) (Labeler, error) { + if configFile == "" { + return nil, nil + } + + // read file and parse it + imexConfig, err := os.Open(configFile) + if err != nil { + klog.Warningf("failed to open imex config file: %v", err) + return nil, nil + } + defer imexConfig.Close() + + // Read the file line by line + var ips []string + scanner := bufio.NewScanner(imexConfig) + for scanner.Scan() { + ip := strings.TrimSpace(scanner.Text()) + if net.ParseIP(ip) == nil { + return nil, fmt.Errorf("invalid IP address in imex config file: %s", ip) + } + ips = append(ips, ip) + } + + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("failed to read imex config file: %v", err) + } + + // Sort the IP addresses + sort.Strings(ips) + + // Join the sorted IPs into a single string + sortedIPs := strings.Join(ips, "\n") + + hashedconfig := generateUUIDs(sortedIPs)[0] + + var commonClusterUUID string + var commonCliqueID string + for _, d := range device { + clusterUUID, err := d.GetClusterUUID() + if err != nil { + return nil, fmt.Errorf("error getting cluster UUID: %v", err) + } + if commonClusterUUID == "" { + commonClusterUUID = clusterUUID + } + if commonClusterUUID != clusterUUID { + klog.Warningf("Cluster UUIDs are different: %s != %s", commonClusterUUID, clusterUUID) + return nil, nil + } + + cliqueID, err := d.GetCliqueIP() + if err != nil { + return nil, fmt.Errorf("error getting clique ID: %v", err) + } + if commonCliqueID == "" { + commonCliqueID = cliqueID + } + if commonCliqueID != cliqueID { + klog.Warningf("Clique IDs are different: %s != %s", commonCliqueID, cliqueID) + return nil, nil + } + } + + labels := Labels{ + "nvidia.com/gpu.clusteruuid": commonClusterUUID, + "nvidia.com/gpu.cliqueid": commonCliqueID, + "nvidia.com/gpu.imex-domain": hashedconfig + "-" + commonCliqueID, + } + + return labels, nil +} + func getModeForClasses(classes []uint32) string { if len(classes) == 0 { return "unknown" @@ -254,3 +339,23 @@ func getDeviceClasses(devices []resource.Device) ([]uint32, error) { } return classes, nil } + +func generateUUIDs(seed string) []string { + rand := rand.New(rand.NewSource(hash(seed))) + + uuids := make([]string, 1) + charset := make([]byte, 16) + rand.Read(charset) + uuid, _ := uuid.FromBytes(charset) + uuids[0] = uuid.String() + + return uuids +} + +func hash(s string) int64 { + h := int64(0) + for _, c := range s { + h = 31*h + int64(c) + } + return h +} diff --git a/internal/resource/cuda-device.go b/internal/resource/cuda-device.go index a4f4bc4a4..e3db22770 100644 --- a/internal/resource/cuda-device.go +++ b/internal/resource/cuda-device.go @@ -100,3 +100,11 @@ func (d *cudaDevice) IsMigEnabled() (bool, error) { func (d *cudaDevice) GetPCIClass() (uint32, error) { return 0, nil } + +func (d *cudaDevice) GetClusterUUID() (string, error) { + return "", nil +} + +func (d *cudaDevice) GetCliqueIP() (string, error) { + return "", nil +} diff --git a/internal/resource/device_mock.go b/internal/resource/device_mock.go index 1024ec96b..9f7d68262 100644 --- a/internal/resource/device_mock.go +++ b/internal/resource/device_mock.go @@ -20,6 +20,12 @@ var _ Device = &DeviceMock{} // GetAttributesFunc: func() (map[string]interface{}, error) { // panic("mock out the GetAttributes method") // }, +// GetCliqueIPFunc: func() (string, error) { +// panic("mock out the GetCliqueIP method") +// }, +// GetClusterUUIDFunc: func() (string, error) { +// panic("mock out the GetClusterUUID method") +// }, // GetCudaComputeCapabilityFunc: func() (int, int, error) { // panic("mock out the GetCudaComputeCapability method") // }, @@ -54,6 +60,12 @@ type DeviceMock struct { // GetAttributesFunc mocks the GetAttributes method. GetAttributesFunc func() (map[string]interface{}, error) + // GetCliqueIPFunc mocks the GetCliqueIP method. + GetCliqueIPFunc func() (string, error) + + // GetClusterUUIDFunc mocks the GetClusterUUID method. + GetClusterUUIDFunc func() (string, error) + // GetCudaComputeCapabilityFunc mocks the GetCudaComputeCapability method. GetCudaComputeCapabilityFunc func() (int, int, error) @@ -83,6 +95,12 @@ type DeviceMock struct { // GetAttributes holds details about calls to the GetAttributes method. GetAttributes []struct { } + // GetCliqueIP holds details about calls to the GetCliqueIP method. + GetCliqueIP []struct { + } + // GetClusterUUID holds details about calls to the GetClusterUUID method. + GetClusterUUID []struct { + } // GetCudaComputeCapability holds details about calls to the GetCudaComputeCapability method. GetCudaComputeCapability []struct { } @@ -109,6 +127,8 @@ type DeviceMock struct { } } lockGetAttributes sync.RWMutex + lockGetCliqueIP sync.RWMutex + lockGetClusterUUID sync.RWMutex lockGetCudaComputeCapability sync.RWMutex lockGetDeviceHandleFromMigDeviceHandle sync.RWMutex lockGetMigDevices sync.RWMutex @@ -146,6 +166,60 @@ func (mock *DeviceMock) GetAttributesCalls() []struct { return calls } +// GetCliqueIP calls GetCliqueIPFunc. +func (mock *DeviceMock) GetCliqueIP() (string, error) { + if mock.GetCliqueIPFunc == nil { + panic("DeviceMock.GetCliqueIPFunc: method is nil but Device.GetCliqueIP was just called") + } + callInfo := struct { + }{} + mock.lockGetCliqueIP.Lock() + mock.calls.GetCliqueIP = append(mock.calls.GetCliqueIP, callInfo) + mock.lockGetCliqueIP.Unlock() + return mock.GetCliqueIPFunc() +} + +// GetCliqueIPCalls gets all the calls that were made to GetCliqueIP. +// Check the length with: +// +// len(mockedDevice.GetCliqueIPCalls()) +func (mock *DeviceMock) GetCliqueIPCalls() []struct { +} { + var calls []struct { + } + mock.lockGetCliqueIP.RLock() + calls = mock.calls.GetCliqueIP + mock.lockGetCliqueIP.RUnlock() + return calls +} + +// GetClusterUUID calls GetClusterUUIDFunc. +func (mock *DeviceMock) GetClusterUUID() (string, error) { + if mock.GetClusterUUIDFunc == nil { + panic("DeviceMock.GetClusterUUIDFunc: method is nil but Device.GetClusterUUID was just called") + } + callInfo := struct { + }{} + mock.lockGetClusterUUID.Lock() + mock.calls.GetClusterUUID = append(mock.calls.GetClusterUUID, callInfo) + mock.lockGetClusterUUID.Unlock() + return mock.GetClusterUUIDFunc() +} + +// GetClusterUUIDCalls gets all the calls that were made to GetClusterUUID. +// Check the length with: +// +// len(mockedDevice.GetClusterUUIDCalls()) +func (mock *DeviceMock) GetClusterUUIDCalls() []struct { +} { + var calls []struct { + } + mock.lockGetClusterUUID.RLock() + calls = mock.calls.GetClusterUUID + mock.lockGetClusterUUID.RUnlock() + return calls +} + // GetCudaComputeCapability calls GetCudaComputeCapabilityFunc. func (mock *DeviceMock) GetCudaComputeCapability() (int, int, error) { if mock.GetCudaComputeCapabilityFunc == nil { diff --git a/internal/resource/nvml-device.go b/internal/resource/nvml-device.go index 1184657d2..1e8018ed2 100644 --- a/internal/resource/nvml-device.go +++ b/internal/resource/nvml-device.go @@ -18,6 +18,7 @@ package resource import ( "fmt" + "strconv" "github.com/NVIDIA/go-nvlib/pkg/nvlib/device" "github.com/NVIDIA/go-nvlib/pkg/nvpci" @@ -99,3 +100,31 @@ func (d nvmlDevice) GetPCIClass() (uint32, error) { } return nvDevice.Class, nil } + +func (d nvmlDevice) GetClusterUUID() (string, error) { + gfInfo, ret := d.GetGpuFabricInfo() + if ret != nvml.SUCCESS { + return "", ret + } + + // Convert the array to a byte slice + byteSlice := gfInfo.ClusterUuid[:] + clusterUUID := fmt.Sprintf("%08x-%04x-%04x-%04x-%012x", + byteSlice[0:4], + byteSlice[4:6], + byteSlice[6:8], + byteSlice[8:10], + byteSlice[10:16], + ) + + return clusterUUID, nil +} + +func (d nvmlDevice) GetCliqueIP() (string, error) { + gfInfo, ret := d.GetGpuFabricInfo() + if ret != nvml.SUCCESS { + return "", ret + } + + return strconv.FormatUint(uint64(gfInfo.CliqueId), 10), nil +} diff --git a/internal/resource/nvml-mig-device.go b/internal/resource/nvml-mig-device.go index 8ef933ff5..d82195a55 100644 --- a/internal/resource/nvml-mig-device.go +++ b/internal/resource/nvml-mig-device.go @@ -138,3 +138,11 @@ func (d nvmlMigDevice) GetPCIClass() (uint32, error) { // GPU devices that support MIG do not support switching mode between graphics and compute, so they are always in compute mode. return nvpci.PCI3dControllerClass, nil } + +func (d nvmlMigDevice) GetClusterUUID() (string, error) { + return "", nil +} + +func (d nvmlMigDevice) GetCliqueIP() (string, error) { + return "", nil +} diff --git a/internal/resource/sysfs-device.go b/internal/resource/sysfs-device.go index 105229fe4..f4f10c384 100644 --- a/internal/resource/sysfs-device.go +++ b/internal/resource/sysfs-device.go @@ -68,3 +68,11 @@ func (d vfioDevice) IsMigCapable() (bool, error) { func (d vfioDevice) GetPCIClass() (uint32, error) { return d.nvidiaPCIDevice.Class, nil } + +func (d vfioDevice) GetClusterUUID() (string, error) { + return "", nil +} + +func (d vfioDevice) GetCliqueIP() (string, error) { + return "", nil +} diff --git a/internal/resource/types.go b/internal/resource/types.go index ec89ec579..1ea18c9dd 100644 --- a/internal/resource/types.go +++ b/internal/resource/types.go @@ -40,4 +40,6 @@ type Device interface { GetDeviceHandleFromMigDeviceHandle() (Device, error) GetCudaComputeCapability() (int, int, error) GetPCIClass() (uint32, error) + GetClusterUUID() (string, error) + GetCliqueIP() (string, error) }