diff --git a/go.mod b/go.mod index 348eaf1b..0d03a435 100644 --- a/go.mod +++ b/go.mod @@ -24,6 +24,7 @@ require ( k8s.io/apimachinery v0.25.2 k8s.io/client-go v12.0.0+incompatible k8s.io/code-generator v0.25.2 + k8s.io/utils v0.0.0-20220922133306-665eaaec4324 ) require ( @@ -68,7 +69,6 @@ require ( k8s.io/gengo v0.0.0-20211129171323-c02415ce4185 // indirect k8s.io/klog/v2 v2.80.1 // indirect k8s.io/kube-openapi v0.0.0-20220928191237-829ce0c27909 // indirect - k8s.io/utils v0.0.0-20220922133306-665eaaec4324 // indirect sigs.k8s.io/controller-runtime v0.13.0 // indirect sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect diff --git a/pkg/controllers/tfpcontroller/tfpcontroller.go b/pkg/controllers/tfpcontroller/tfpcontroller.go index e178e725..71045693 100644 --- a/pkg/controllers/tfpcontroller/tfpcontroller.go +++ b/pkg/controllers/tfpcontroller/tfpcontroller.go @@ -174,12 +174,10 @@ func (t *TerraformProvisionerController) processNextVM() bool { // returns an error and a boolean of requeue func (t *TerraformProvisionerController) handleProvision(vm *hfv1.VirtualMachine) (error, bool) { // VM shall not be provisioned by internal terraform controller - if ! vm.Spec.Provision { - if provisionMethod, ok := vm.ObjectMeta.Labels["hobbyfarm.io/provisioner"]; ok { - if provisionMethod == "external" { - glog.V(8).Infof("vm %s ignored due to external provisioning label", vm.Name) - t.vmWorkqueue.Done(vm.Name) - } + if !vm.Spec.Provision { + if prov, ok := vm.ObjectMeta.Labels["hobbyfarm.io/provisioner"]; ok && prov != "" { + glog.V(8).Infof("vm %s ignored by internal provisioner due to 3rd party provisioning label", vm.Name) + t.vmWorkqueue.Done(vm.Name) } glog.V(8).Infof("vm %s was not a provisioned vm", vm.Name) return nil, false @@ -235,18 +233,18 @@ func (t *TerraformProvisionerController) handleProvision(vm *hfv1.VirtualMachine glog.Errorf("error generating keypair %v", err) return err, true } - config := util.GetVMConfig(env,vmt) + config := util.GetVMConfig(env, vmt) config["name"] = vm.Name config["public_key"] = pubKey image, exists := config["image"] - if !exists || image == "" { + if !exists || image == "" { return fmt.Errorf("image does not exist or is empty in vm config for vmt %s", vmt.Name), true } moduleName, exists := config["module"] - if !exists || moduleName == "" { + if !exists || moduleName == "" { return fmt.Errorf("module name does not exist or is empty in vm config for vmt %s", vmt.Name), true } @@ -297,7 +295,7 @@ func (t *TerraformProvisionerController) handleProvision(vm *hfv1.VirtualMachine Data: map[string][]byte{ "private_key": []byte(privKey), "public_key": []byte(pubKey), - "password" : []byte(password), + "password": []byte(password), }, } @@ -348,10 +346,10 @@ func (t *TerraformProvisionerController) handleProvision(vm *hfv1.VirtualMachine toUpdate.Status.Status = hfv1.VmStatusProvisioned toUpdate.Status.TFState = tfs.Name - + toUpdate, updateErr := t.hfClientSet.HobbyfarmV1().VirtualMachines(util.GetReleaseNamespace()).UpdateStatus(t.ctx, toUpdate, metav1.UpdateOptions{}) - if(updateErr != nil){ + if updateErr != nil { glog.Errorf("error while updating VirtualMachine status %s", toUpdate.Name) return updateErr } @@ -362,7 +360,7 @@ func (t *TerraformProvisionerController) handleProvision(vm *hfv1.VirtualMachine toUpdate, updateErr = t.hfClientSet.HobbyfarmV1().VirtualMachines(util.GetReleaseNamespace()).Update(t.ctx, toUpdate, metav1.UpdateOptions{}) - if(updateErr != nil){ + if updateErr != nil { glog.Errorf("error while updating VirtualMachine %s", toUpdate.Name) return updateErr } @@ -465,7 +463,7 @@ func (t *TerraformProvisionerController) handleProvision(vm *hfv1.VirtualMachine toUpdate, updateErr := t.hfClientSet.HobbyfarmV1().VirtualMachines(util.GetReleaseNamespace()).Update(t.ctx, toUpdate, metav1.UpdateOptions{}) - if(updateErr != nil){ + if updateErr != nil { glog.Errorf("error while updating machine: %s", toUpdate.Name) return updateErr } diff --git a/pkg/controllers/vmclaimcontroller/vmclaimcontroller.go b/pkg/controllers/vmclaimcontroller/vmclaimcontroller.go index 6379ce41..1db7fb07 100644 --- a/pkg/controllers/vmclaimcontroller/vmclaimcontroller.go +++ b/pkg/controllers/vmclaimcontroller/vmclaimcontroller.go @@ -7,11 +7,11 @@ import ( "time" "github.com/golang/glog" + "github.com/hobbyfarm/gargantua/pkg/accesscode" hfv1 "github.com/hobbyfarm/gargantua/pkg/apis/hobbyfarm.io/v1" hfClientset "github.com/hobbyfarm/gargantua/pkg/client/clientset/versioned" hfInformers "github.com/hobbyfarm/gargantua/pkg/client/informers/externalversions" hfListers "github.com/hobbyfarm/gargantua/pkg/client/listers/hobbyfarm.io/v1" - "github.com/hobbyfarm/gargantua/pkg/accesscode" "github.com/hobbyfarm/gargantua/pkg/util" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -32,7 +32,7 @@ type VMClaimController struct { vmLister hfListers.VirtualMachineLister vmClaimLister hfListers.VirtualMachineClaimLister - vmtLister hfListers.VirtualMachineTemplateLister + vmtLister hfListers.VirtualMachineTemplateLister vmClaimWorkqueue workqueue.Interface @@ -262,14 +262,14 @@ func (v *VMClaimController) processVMClaim(vmc *hfv1.VirtualMachineClaim) (err e if err != nil { // VirtualMachines could not be submitted. Delete Session glog.Errorf("error processing vmc %s, taint session: %v", vmc.Name, err) - return v.taintSession(vmc.Labels[util.SessionLabel]); + return v.taintSession(vmc.Labels[util.SessionLabel]) } } else if vmc.Status.BindMode == "static" { err = v.findVirtualMachines(vmc) if err != nil { // VirtualMachines could not be bound. Delete Session glog.Errorf("error processing vmc %s, taint session: %v", vmc.Name, err) - return v.taintSession(vmc.Labels[util.SessionLabel]); + return v.taintSession(vmc.Labels[util.SessionLabel]) } } else { glog.Errorf("vmc bind mode needs to be either dynamic or static.. ignoring this object %s", vmc.Name) @@ -300,7 +300,7 @@ func (v *VMClaimController) processVMClaim(vmc *hfv1.VirtualMachineClaim) (err e return nil } -func (v *VMClaimController) taintSession(session string) (err error){ +func (v *VMClaimController) taintSession(session string) (err error) { retryErr := retry.RetryOnConflict(retry.DefaultRetry, func() error { result, getErr := v.hfClientSet.HobbyfarmV1().Sessions(util.GetReleaseNamespace()).Get(v.ctx, session, metav1.GetOptions{}) if getErr != nil { @@ -317,7 +317,7 @@ func (v *VMClaimController) taintSession(session string) (err error){ return updateErr }) - if(retryErr != nil){ + if retryErr != nil { return retryErr } @@ -325,11 +325,11 @@ func (v *VMClaimController) taintSession(session string) (err error){ err = v.hfClientSet.HobbyfarmV1().Progresses(util.GetReleaseNamespace()).DeleteCollection(v.ctx, metav1.DeleteOptions{}, metav1.ListOptions{ LabelSelector: fmt.Sprintf("%s=%s,finished=false", util.SessionLabel, session)}) - return err; + return err } type VMEnvironment struct { - Environment hfv1.Environment + Environment hfv1.Environment DynamicBindConfiguration hfv1.DynamicBindConfiguration } @@ -351,15 +351,15 @@ func (v *VMClaimController) submitVirtualMachines(vmc *hfv1.VirtualMachineClaim) for _, vmDetails := range vmc.Spec.VirtualMachines { if count, found := requiredTemplateCount[vmDetails.Template]; found { requiredTemplateCount[vmDetails.Template] = count + 1 - }else{ + } else { requiredTemplateCount[vmDetails.Template] = 1 } } - environmentMap := make(map[string]VMEnvironment) // Maps node to the environment it should use + environmentMap := make(map[string]VMEnvironment) // Maps node to the environment it should use bestDBC, err := v.findBestDBCForVMs(dbcList, requiredTemplateCount, vmc.Labels[util.ScheduledEventLabel]) // Try to find if one environment can provision all VMs - if(err != nil) { + if err != nil { // We can not provision all VirtualMachines in one environment. Figure out which environments we want to use reservedCapacity := make(map[string]map[string]int) // EnvironmentID -> TemplateID -> Count @@ -373,17 +373,17 @@ func (v *VMClaimController) submitVirtualMachines(vmc *hfv1.VirtualMachineClaim) } for vmName, vmDetails := range vmc.Spec.VirtualMachines { env, dbc, err := v.findSuitableEnvironmentForVMTemplate(environments, dbcList, vmDetails.Template, reservedCapacity, vmc.Labels[util.ScheduledEventLabel]) - if err != nil{ + if err != nil { glog.Errorf("no suitable environment for %s (%s): %v", vmName, vmDetails.Template, err) return err } environmentMap[vmName] = VMEnvironment{env, dbc} reservedCapacity[env.Name][vmDetails.Template] += 1 } - }else{ + } else { // One DBC for them all enviroment := hfv1.Environment{} - for _, e := range environments{ + for _, e := range environments { if e.Name == bestDBC.Spec.Environment { enviroment = e break @@ -411,19 +411,19 @@ func (v *VMClaimController) submitVirtualMachines(vmc *hfv1.VirtualMachineClaim) }, }, Labels: map[string]string{ - "dynamic": "true", - "vmc": vmc.Name, - util.EnvironmentLabel: environment.Name, - "bound": "true", - "ready": "false", - util.VirtualMachineTemplate: vmDetails.Template, - util.ScheduledEventLabel: seName, + "dynamic": "true", + "vmc": vmc.Name, + util.EnvironmentLabel: environment.Name, + "bound": "true", + "ready": "false", + util.VirtualMachineTemplate: vmDetails.Template, + util.ScheduledEventLabel: seName, }, }, Spec: hfv1.VirtualMachineSpec{ VirtualMachineTemplateId: vmDetails.Template, SecretName: "", - Protocol: "ssh", //default protocol is ssh + Protocol: "ssh", //default protocol is ssh VirtualMachineClaimId: vmc.Name, UserId: vmc.Spec.UserId, Provision: true, @@ -442,20 +442,20 @@ func (v *VMClaimController) submitVirtualMachines(vmc *hfv1.VirtualMachineClaim) return err } - config := util.GetVMConfig(&environment,vmt) - - protocol, exists := config["protocol"] - if exists { - vm.Spec.Protocol = protocol + config := util.GetVMConfig(&environment, vmt) + + protocol, exists := config["protocol"] + if exists { + vm.Spec.Protocol = protocol } - - sshUser, exists := config["ssh_username"] + + sshUser, exists := config["ssh_username"] if exists { vm.Spec.SshUsername = sshUser } // extra label to indicate external provisioning so tfpcontroller ignores this request // - if provisionMethod, ok := environment.Annotations["hobbyfarm.io/provisioner"]; ok { + if provisionMethod, ok := environment.Annotations["hobbyfarm.io/provisioner"]; ok && provisionMethod != "" { vm.Labels["hobbyfarm.io/provisioner"] = provisionMethod vm.Spec.Provision = false } @@ -472,7 +472,7 @@ func (v *VMClaimController) submitVirtualMachines(vmc *hfv1.VirtualMachineClaim) return err } - createdVM.Status = hfv1.VirtualMachineStatus{ + vmStatus := hfv1.VirtualMachineStatus{ Status: hfv1.VmStatusRFP, Allocated: true, Tainted: false, @@ -482,9 +482,25 @@ func (v *VMClaimController) submitVirtualMachines(vmc *hfv1.VirtualMachineClaim) PrivateIP: "", } - _, err = v.hfClientSet.HobbyfarmV1().VirtualMachines(util.GetReleaseNamespace()).UpdateStatus(v.ctx, createdVM, metav1.UpdateOptions{}) - if err != nil { - return err + // retry loop here + retryErr := retry.RetryOnConflict(retry.DefaultBackoff, func() error { + createdVM, err := v.hfClientSet.HobbyfarmV1().VirtualMachines(util.GetReleaseNamespace()).Get(v.ctx, createdVM.Name, metav1.GetOptions{}) + if err != nil { + return err + } + + createdVM.Status = vmStatus + + _, err = v.hfClientSet.HobbyfarmV1().VirtualMachines(util.GetReleaseNamespace()).UpdateStatus(v.ctx, createdVM, metav1.UpdateOptions{}) + if err != nil { + return err + } + + return nil + }) + + if retryErr != nil { + return retryErr } } @@ -514,7 +530,7 @@ func (v *VMClaimController) findEnvironmentsForVM(accessCode string, vmc *hfv1.V return environments, seName, dbc, err } - for _ , dbc := range dbcList.Items { + for _, dbc := range dbcList.Items { env, err := v.hfClientSet.HobbyfarmV1().Environments(util.GetReleaseNamespace()).Get(v.ctx, dbc.Spec.Environment, metav1.GetOptions{}) if err != nil { @@ -541,12 +557,12 @@ func (v *VMClaimController) findBestDBCForVMs(dbcList []hfv1.DynamicBindConfigur return hfv1.DynamicBindConfiguration{}, fmt.Errorf("error fetching environment") } for requiredTemplate, requiredCount := range requiredTemplateCount { - dbcCapacity, foundDBC := dbc.Spec.BurstCountCapacity[requiredTemplate]; - envCapacity, foundEnv := env.Spec.CountCapacity[requiredTemplate]; + dbcCapacity, foundDBC := dbc.Spec.BurstCountCapacity[requiredTemplate] + envCapacity, foundEnv := env.Spec.CountCapacity[requiredTemplate] if foundDBC && foundEnv { // Does the DBC satisfy this amount? count, err := util.CountMachinesPerTemplateAndEnvironmentAndScheduledEvent(v.vmLister, requiredTemplate, dbc.Spec.Environment, scheduledEvent) - if(err != nil){ + if err != nil { satisfiedDBC = false break } @@ -557,7 +573,7 @@ func (v *VMClaimController) findBestDBCForVMs(dbcList []hfv1.DynamicBindConfigur // Does the environment satisfy this amount? count, err = util.CountMachinesPerTemplateAndEnvironment(v.vmLister, requiredTemplate, dbc.Spec.Environment) - if(err != nil){ + if err != nil { satisfiedDBC = false break } @@ -566,7 +582,7 @@ func (v *VMClaimController) findBestDBCForVMs(dbcList []hfv1.DynamicBindConfigur break } - }else{ + } else { satisfiedDBC = false break } @@ -584,19 +600,19 @@ func (v *VMClaimController) findBestDBCForVMs(dbcList []hfv1.DynamicBindConfigur func (v *VMClaimController) findSuitableEnvironmentForVMTemplate(environments []hfv1.Environment, dbcList []hfv1.DynamicBindConfiguration, template string, reservedCapacity map[string]map[string]int, scheduledEvent string) (hfv1.Environment, hfv1.DynamicBindConfiguration, error) { for _, environment := range environments { countEnv, err := util.CountMachinesPerTemplateAndEnvironment(v.vmLister, template, environment.Name) - if(err != nil){ + if err != nil { continue } // We have also reserved capacity for other VMs countEnv += reservedCapacity[environment.Name][template] - if(countEnv >= environment.Spec.CountCapacity[template]){ + if countEnv >= environment.Spec.CountCapacity[template] { // Environment is at limit continue } countDBC, err := util.CountMachinesPerTemplateAndEnvironmentAndScheduledEvent(v.vmLister, template, environment.Name, scheduledEvent) - if(err != nil){ + if err != nil { continue } // We have also reserved capacity for other VMs @@ -604,11 +620,11 @@ func (v *VMClaimController) findSuitableEnvironmentForVMTemplate(environments [] // found environment that satisfies capacity for this template for _, dbc := range dbcList { - if(dbc.Spec.Environment == environment.Name){ + if dbc.Spec.Environment == environment.Name { if capacity, found := dbc.Spec.BurstCountCapacity[template]; found { - if(countDBC < capacity){ + if countDBC < capacity { // Capacity also satisfied for environment + scheduledEvent via DBC - return environment, dbc, nil + return environment, dbc, nil } } break @@ -639,7 +655,7 @@ func (v *VMClaimController) checkVMStatus(vmc *hfv1.VirtualMachineClaim) (ready func (v *VMClaimController) findScheduledEvent(accessCode string) (schedEvent string, environments map[string]map[string]int, err error) { ac, err := v.accessCodeClient.GetAccessCodeWithOTACs(accessCode) - if (err != nil){ + if err != nil { return schedEvent, environments, err } @@ -691,18 +707,16 @@ func (v *VMClaimController) findVirtualMachines(vmc *hfv1.VirtualMachineClaim) ( return err } - return nil } -func (v *VMClaimController) assignVM(vmClaimId string, user string, vmId string) (error) { +func (v *VMClaimController) assignVM(vmClaimId string, user string, vmId string) error { retryErr := retry.RetryOnConflict(retry.DefaultRetry, func() error { result, getErr := v.hfClientSet.HobbyfarmV1().VirtualMachines(util.GetReleaseNamespace()).Get(v.ctx, vmId, metav1.GetOptions{}) if getErr != nil { return fmt.Errorf("error retrieving latest version of Virtual Machine %s: %v", vmId, getErr) } - result.Labels["bound"] = "true" result.Spec.VirtualMachineClaimId = vmClaimId result.Spec.UserId = user @@ -735,7 +749,7 @@ func (v *VMClaimController) assignVM(vmClaimId string, user string, vmId string return nil } -func (v *VMClaimController) unassignVM(vmId string) (string, error) { +func (v *VMClaimController) unassignVM(vmId string) (string, error) { retryErr := retry.RetryOnConflict(retry.DefaultRetry, func() error { result, getErr := v.hfClientSet.HobbyfarmV1().VirtualMachines(util.GetReleaseNamespace()).Get(v.ctx, vmId, metav1.GetOptions{}) if getErr != nil { @@ -774,8 +788,8 @@ func (v *VMClaimController) unassignVM(vmId string) (string, error) { func (v *VMClaimController) assignNextFreeVM(vmClaimId string, user string, environments map[string]map[string]int, template string, restrictedBind bool, restrictedBindValue string) (string, error) { vmLabels := labels.Set{ - "bound": "false", - util.VirtualMachineTemplate: template, + "bound": "false", + util.VirtualMachineTemplate: template, } if restrictedBind { @@ -791,7 +805,7 @@ func (v *VMClaimController) assignNextFreeVM(vmClaimId string, user string, envi return "", fmt.Errorf("error while listing all vms %v", err) } - if(len(vms) == 0){ + if len(vms) == 0 { return "", fmt.Errorf("No static VMs matching template: %s. All static VMs are in use.", template) } @@ -805,17 +819,17 @@ func (v *VMClaimController) assignNextFreeVM(vmClaimId string, user string, envi // ... but this environment does not support this virtualmachinetemplate continue } - }else{ + } else { // This virtualmachine is in a non supported environment - continue; + continue } if !vm.Status.Allocated && !vm.Status.Tainted { // we can assign this vm assigned = true vmId = vm.Name - + // Prefer running machines - if( vm.Status.Status == hfv1.VmStatusRunning){ + if vm.Status.Status == hfv1.VmStatusRunning { break } } diff --git a/pkg/controllers/vmsetcontroller/vmsetcontroller.go b/pkg/controllers/vmsetcontroller/vmsetcontroller.go index 644f97e2..83b8545c 100644 --- a/pkg/controllers/vmsetcontroller/vmsetcontroller.go +++ b/pkg/controllers/vmsetcontroller/vmsetcontroller.go @@ -253,10 +253,8 @@ func (v *VirtualMachineSetController) reconcileVirtualMachineSet(vmset *hfv1.Vir env, err := v.envLister.Environments(util.GetReleaseNamespace()).Get(vmset.Spec.Environment) var provision bool provision = true - if provisionMethod, ok := env.Annotations["hobbyfarm.io/provisioner"]; ok { - if provisionMethod == "external" { - provision = false - } + if provisionMethod, ok := env.Annotations["hobbyfarm.io/provisioner"]; ok && provisionMethod != "" { + provision = false } if err != nil { if apierrors.IsNotFound(err) { @@ -323,6 +321,11 @@ func (v *VirtualMachineSetController) reconcileVirtualMachineSet(vmset *hfv1.Vir } else { vm.ObjectMeta.Labels["restrictedbind"] = "false" } + + if provisionMethod, ok := env.Annotations["hobbyfarm.io/provisioner"]; ok && provisionMethod != "" { + vm.ObjectMeta.Labels["hobbyfarm.io/provisioner"] = provisionMethod + } + // adding a custom finalizer for reconcile of vmsets vm.SetFinalizers([]string{vmSetFinalizer}) vm, err := v.hfClientSet.HobbyfarmV1().VirtualMachines(util.GetReleaseNamespace()).Create(v.ctx, vm, metav1.CreateOptions{}) @@ -354,18 +357,20 @@ func (v *VirtualMachineSetController) reconcileVirtualMachineSet(vmset *hfv1.Vir } } } -//-----------------------handle case of scaling down VMSets + //-----------------------handle case of scaling down VMSets if len(currentVMs) > vmset.Spec.Count { needed_delete := len(currentVMs) - vmset.Spec.Count for _, cur_vm := range currentVMs { if !cur_vm.Status.Allocated { - v.hfClientSet.HobbyfarmV1().VirtualMachines(util.GetReleaseNamespace()).Delete(v.ctx, cur_vm.Name, metav1.DeleteOptions{} ) + v.hfClientSet.HobbyfarmV1().VirtualMachines(util.GetReleaseNamespace()).Delete(v.ctx, cur_vm.Name, metav1.DeleteOptions{}) needed_delete-- - if needed_delete == 0 {break} + if needed_delete == 0 { + break + } } } - } -//----------------------------------------------------- + } + //----------------------------------------------------- vms, err := v.vmLister.List(labels.Set{ "vmset": string(vmset.Name), }.AsSelector())