Skip to content

Commit

Permalink
Move TF Controller to TF Service (#198)
Browse files Browse the repository at this point in the history 
* Move TF Controller to TF Service

* requeue VM after deleting the TFState
  • Loading branch information
@jggoebel
jggoebel committed Jul 11, 2024
1 parent 7ba786e commit 602245e
Show file tree
Hide file tree
Showing 3 changed files with 75 additions and 62 deletions.
50 changes: 26 additions & 24 deletions v3/services/vmsvc/internal/controller.go → ...vices/terraformsvc/internal/controller.go
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package vmservice
package terraformsvc

import (
"context"
Expand All @@ -8,6 +8,8 @@ import (
"time"

hfv1 "github.com/hobbyfarm/gargantua/v3/pkg/apis/hobbyfarm.io/v1"
"github.com/hobbyfarm/gargantua/v3/pkg/client/clientset/versioned"
v1 "github.com/hobbyfarm/gargantua/v3/pkg/client/clientset/versioned/typed/hobbyfarm.io/v1"
hfInformers "github.com/hobbyfarm/gargantua/v3/pkg/client/informers/externalversions"
controllers "github.com/hobbyfarm/gargantua/v3/pkg/microservices/controller"
"github.com/hobbyfarm/gargantua/v3/pkg/util"
Expand Down Expand Up @@ -37,22 +39,24 @@ const (
type VMController struct {
controllers.DelayingWorkqueueController
controllers.Reconciler
internalVmServer *GrpcVMServer
VMClient vmpb.VMSvcClient
configMapClient corev1.ConfigMapInterface
environmentClient environmentpb.EnvironmentSvcClient
secretClient corev1.SecretInterface
terraformClient terraformpb.TerraformSvcClient
HFVMClient v1.VirtualMachineInterface
terraformClient *GrpcTerraformServer
vmClaimClient vmclaimpb.VMClaimSvcClient
vmSetClient vmsetpb.VMSetSvcClient
vmTemplateClient vmtemplatepb.VMTemplateSvcClient
}

func NewVMController(
hfClient *versioned.Clientset,
kubeClient *kubernetes.Clientset,
internalVmServer *GrpcVMServer,
VMClient vmpb.VMSvcClient,
hfInformerFactory hfInformers.SharedInformerFactory,
environmentClient environmentpb.EnvironmentSvcClient,
terraformClient terraformpb.TerraformSvcClient,
terraformClient *GrpcTerraformServer,
vmClaimClient vmclaimpb.VMClaimSvcClient,
vmSetClient vmsetpb.VMSetSvcClient,
vmTemplateClient vmtemplatepb.VMTemplateSvcClient,
Expand All @@ -71,10 +75,11 @@ func NewVMController(

vmController := &VMController{
DelayingWorkqueueController: delayingWorkqueueController,
internalVmServer: internalVmServer,
VMClient: VMClient,
configMapClient: kubeClient.CoreV1().ConfigMaps(util.GetReleaseNamespace()),
environmentClient: environmentClient,
secretClient: kubeClient.CoreV1().Secrets(util.GetReleaseNamespace()),
HFVMClient: hfClient.HobbyfarmV1().VirtualMachines(util.GetReleaseNamespace()),
terraformClient: terraformClient,
vmClaimClient: vmClaimClient,
vmSetClient: vmSetClient,
Expand All @@ -89,7 +94,7 @@ func NewVMController(
func (v *VMController) Reconcile(objName string) error {
glog.V(8).Infof("reconciling vm %s inside vm controller", objName)
// fetch vm
vm, err := v.internalVmServer.GetVM(v.Context, &generalpb.GetRequest{Id: objName})
vm, err := v.VMClient.GetVM(v.Context, &generalpb.GetRequest{Id: objName})
if err != nil {
if hferrors.IsGrpcNotFound(err) {
glog.Infof("vm %s not found on queue.. ignoring", objName)
Expand All @@ -100,6 +105,12 @@ func (v *VMController) Reconcile(objName string) error {
}
}

// VM shall not be provisioned by internal terraform controller
if prov, ok := vm.GetLabels()["hobbyfarm.io/provisioner"]; ok && prov != "" {
glog.V(8).Infof("vm %s ignored by terraform controller due to 3rd party provisioning label", vm.GetId())
v.GetWorkqueue().Done(vm.GetId())
}

// trigger reconcile on vmClaims only when associated VM is running
// this should avoid triggering unwanted reconciles of VMClaims until the VM's are running
if vm.GetVmClaimId() != "" && vm.GetStatus().GetStatus() == string(hfv1.VmStatusRunning) {
Expand Down Expand Up @@ -129,7 +140,7 @@ func (v *VMController) handleRequeue(err error, requeue bool, vmId string) {

// returns an error and a boolean of requeue
func (v *VMController) deleteVM(vm *vmpb.VM) (error, bool) {
_, deleteVMErr := v.internalVmServer.DeleteVM(v.Context, &generalpb.ResourceId{Id: vm.GetId()})
_, deleteVMErr := v.VMClient.DeleteVM(v.Context, &generalpb.ResourceId{Id: vm.GetId()})
if deleteVMErr != nil {
return fmt.Errorf("there was an error while deleting the virtual machine %s", vm.GetId()), true
}
Expand All @@ -142,7 +153,7 @@ func (v *VMController) handleDeletion(vm *vmpb.VM) (error, bool) {
if vm.GetVmSetId() != "" && util.ContainsFinalizer(vm.GetFinalizers(), vmSetFinalizer) {
glog.V(4).Infof("requeuing vmset %s to account for tainted vm %s", vm.GetVmSetId(), vm.GetId())
updatedVmFinalizers := util.RemoveFinalizer(vm.GetFinalizers(), vmSetFinalizer)
_, err := v.internalVmServer.UpdateVM(v.Context, &vmpb.UpdateVMRequest{Id: vm.GetId(), Finalizers: &generalpb.StringArray{
_, err := v.VMClient.UpdateVM(v.Context, &vmpb.UpdateVMRequest{Id: vm.GetId(), Finalizers: &generalpb.StringArray{
Values: updatedVmFinalizers,
}})
if err != nil {
Expand All @@ -168,7 +179,7 @@ func (v *VMController) handleDeletion(vm *vmpb.VM) (error, bool) {
} else {
// The terraform state was deleted successfully.
// We still need to requeue, remove the finalizers and confirm that the vm was deleted successfully
return nil, false
return nil, true
}
}

Expand All @@ -181,9 +192,9 @@ func (v *VMController) updateAndVerifyVMDeletion(vm *vmpb.VM) (error, bool) {

// start verification of deletion in a separate goroutine
go func() {
resultCh <- util.VerifyDeletion(ctx, v.internalVmServer.vmClient, vm.GetId())
resultCh <- util.VerifyDeletion(ctx, v.HFVMClient, vm.GetId())
}()
_, err := v.internalVmServer.UpdateVM(v.Context, &vmpb.UpdateVMRequest{
_, err := v.VMClient.UpdateVM(v.Context, &vmpb.UpdateVMRequest{
Id: vm.GetId(),
Finalizers: &generalpb.StringArray{Values: []string{}},
})
Expand All @@ -206,15 +217,6 @@ func (v *VMController) updateAndVerifyVMDeletion(vm *vmpb.VM) (error, bool) {

// returns an error and a boolean of requeue
func (v *VMController) handleProvision(vm *vmpb.VM) (error, bool) {
// VM shall not be provisioned by internal terraform controller
if !vm.GetProvision() {
if prov, ok := vm.GetLabels()["hobbyfarm.io/provisioner"]; ok && prov != "" {
glog.V(8).Infof("vm %s ignored by internal provisioner due to 3rd party provisioning label", vm.GetId())
v.GetWorkqueue().Done(vm.GetId())
}
glog.V(8).Infof("vm %s was not a provisioned vm", vm.GetId())
return nil, false
}
//Status is ReadyForProvisioning AND No Secret provided (Do not provision VM twice, happens due to vm.status being updated after vm.status)
if vm.Status.Status == string(hfv1.VmStatusRFP) {
vmt, err := v.vmTemplateClient.GetVMTemplate(v.Context, &generalpb.GetRequest{Id: vm.GetVmTemplateId(), LoadFromCache: true})
Expand Down Expand Up @@ -333,7 +335,7 @@ func (v *VMController) handleProvision(vm *vmpb.VM) (error, bool) {
glog.Errorf("error creating tfs %v", err)
}

_, err = v.internalVmServer.UpdateVMStatus(v.Context, &vmpb.UpdateVMStatusRequest{
_, err = v.VMClient.UpdateVMStatus(v.Context, &vmpb.UpdateVMStatusRequest{
Id: vm.GetId(),
Status: string(hfv1.VmStatusProvisioned),
Tfstate: tfsId.GetId(),
Expand All @@ -348,7 +350,7 @@ func (v *VMController) handleProvision(vm *vmpb.VM) (error, bool) {
} else {
updatedFinalizers = []string{"vm.controllers.hobbyfarm.io"}
}
_, err = v.internalVmServer.UpdateVM(v.Context, &vmpb.UpdateVMRequest{
_, err = v.VMClient.UpdateVM(v.Context, &vmpb.UpdateVMRequest{
Id: vm.GetId(),
SecretName: keypair.Name,
Finalizers: &generalpb.StringArray{Values: updatedFinalizers},
Expand Down Expand Up @@ -438,7 +440,7 @@ func (v *VMController) handleProvision(vm *vmpb.VM) (error, bool) {
publicIP = translatePrivToPub(env.GetIpTranslationMap(), tfOutput["private_ip"]["value"])
}

_, err = v.internalVmServer.UpdateVMStatus(v.Context, &vmpb.UpdateVMStatusRequest{
_, err = v.VMClient.UpdateVMStatus(v.Context, &vmpb.UpdateVMStatusRequest{
Id: vm.GetId(),
Status: string(hfv1.VmStatusRunning),
PublicIp: wrapperspb.String(publicIP),
Expand Down
50 changes: 48 additions & 2 deletions v3/services/terraformsvc/main.go
View file
Original file line number Diff line number Diff line change
@@ -1,17 +1,24 @@
package main

import (
"context"
"sync"
"time"

"github.com/golang/glog"
"github.com/hobbyfarm/gargantua/v3/pkg/crd"
"github.com/hobbyfarm/gargantua/v3/pkg/microservices"
"github.com/hobbyfarm/gargantua/v3/pkg/signals"
"github.com/hobbyfarm/gargantua/v3/pkg/util"

terraformservice "github.com/hobbyfarm/gargantua/services/terraformsvc/v3/internal"
hfInformers "github.com/hobbyfarm/gargantua/v3/pkg/client/informers/externalversions"
environmentpb "github.com/hobbyfarm/gargantua/v3/protos/environment"
terraformpb "github.com/hobbyfarm/gargantua/v3/protos/terraform"
vmpb "github.com/hobbyfarm/gargantua/v3/protos/vm"
vmclaimpb "github.com/hobbyfarm/gargantua/v3/protos/vmclaim"
vmsetpb "github.com/hobbyfarm/gargantua/v3/protos/vmset"
vmtemplatepb "github.com/hobbyfarm/gargantua/v3/protos/vmtemplate"
)

var (
Expand All @@ -24,19 +31,53 @@ func init() {

func main() {
stopCh := signals.SetupSignalHandler()
ctx := context.Background()

cfg, hfClient, _ := microservices.BuildClusterConfig(serviceConfig)
cfg, hfClient, kubeClient := microservices.BuildClusterConfig(serviceConfig)

namespace := util.GetReleaseNamespace()
hfInformerFactory := hfInformers.NewSharedInformerFactoryWithOptions(hfClient, time.Second*30, hfInformers.WithNamespace(namespace))

crd.InstallCrds(terraformservice.TerraformCRDInstaller{}, cfg, "terraform")

gs := microservices.CreateGRPCServer(serviceConfig.ServerCert.Clone())
services := []microservices.MicroService{
microservices.Environment,
microservices.VMClaim,
microservices.VM,
microservices.VMSet,
microservices.VMTemplate,
}
connections := microservices.EstablishConnections(services, serviceConfig.ClientCert)
for _, conn := range connections {
defer conn.Close()
}
environmentClient := environmentpb.NewEnvironmentSvcClient(connections[microservices.Environment])
vmClaimClient := vmclaimpb.NewVMClaimSvcClient(connections[microservices.VMClaim])
vmClient := vmpb.NewVMSvcClient(connections[microservices.VM])
vmSetClient := vmsetpb.NewVMSetSvcClient(connections[microservices.VMSet])
vmTemplateClient := vmtemplatepb.NewVMTemplateSvcClient(connections[microservices.VMTemplate])

gs := microservices.CreateGRPCServer(serviceConfig.ServerCert.Clone())
ts := terraformservice.NewGrpcTerraformServer(hfClient, hfInformerFactory)
terraformpb.RegisterTerraformSvcServer(gs, ts)

vmController, err := terraformservice.NewVMController(
hfClient,
kubeClient,
vmClient,
hfInformerFactory,
environmentClient,
ts,
vmClaimClient,
vmSetClient,
vmTemplateClient,
ctx,
)

if err != nil {
glog.Fatalf("failed creating vm controller: %s", err.Error())
}

var wg sync.WaitGroup
wg.Add(1)

Expand All @@ -45,6 +86,11 @@ func main() {
microservices.StartGRPCServer(gs, serviceConfig.EnableReflection)
}()

go func() {
defer wg.Done()
vmController.RunSharded(stopCh, microservices.Terraform)
}()

hfInformerFactory.Start(stopCh)

wg.Wait()
Expand Down
37 changes: 1 addition & 36 deletions v3/services/vmsvc/main.go
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,9 @@
package main

import (
"context"
"sync"
"time"

"github.com/golang/glog"
"github.com/hobbyfarm/gargantua/v3/pkg/crd"
"github.com/hobbyfarm/gargantua/v3/pkg/microservices"
"github.com/hobbyfarm/gargantua/v3/pkg/signals"
Expand All @@ -15,11 +13,7 @@ import (
hfInformers "github.com/hobbyfarm/gargantua/v3/pkg/client/informers/externalversions"
authnpb "github.com/hobbyfarm/gargantua/v3/protos/authn"
authrpb "github.com/hobbyfarm/gargantua/v3/protos/authr"
environmentpb "github.com/hobbyfarm/gargantua/v3/protos/environment"
terraformpb "github.com/hobbyfarm/gargantua/v3/protos/terraform"
vmpb "github.com/hobbyfarm/gargantua/v3/protos/vm"
vmclaimpb "github.com/hobbyfarm/gargantua/v3/protos/vmclaim"
vmsetpb "github.com/hobbyfarm/gargantua/v3/protos/vmset"
vmtemplatepb "github.com/hobbyfarm/gargantua/v3/protos/vmtemplate"
)

Expand All @@ -33,9 +27,8 @@ func init() {

func main() {
stopCh := signals.SetupSignalHandler()
ctx := context.Background()

cfg, hfClient, kubeClient := microservices.BuildClusterConfig(serviceConfig)
cfg, hfClient, _ := microservices.BuildClusterConfig(serviceConfig)

namespace := util.GetReleaseNamespace()
hfInformerFactory := hfInformers.NewSharedInformerFactoryWithOptions(hfClient, time.Second*30, hfInformers.WithNamespace(namespace))
Expand All @@ -45,42 +38,19 @@ func main() {
services := []microservices.MicroService{
microservices.AuthN,
microservices.AuthR,
microservices.Environment,
microservices.Terraform,
microservices.VMClaim,
microservices.VMSet,
microservices.VMTemplate,
}
connections := microservices.EstablishConnections(services, serviceConfig.ClientCert)
for _, conn := range connections {
defer conn.Close()
}
authnClient := authnpb.NewAuthNClient(connections[microservices.AuthN])
authrClient := authrpb.NewAuthRClient(connections[microservices.AuthR])
environmentClient := environmentpb.NewEnvironmentSvcClient(connections[microservices.Environment])
terraformClient := terraformpb.NewTerraformSvcClient(connections[microservices.Terraform])
vmClaimClient := vmclaimpb.NewVMClaimSvcClient(connections[microservices.VMClaim])
vmSetClient := vmsetpb.NewVMSetSvcClient(connections[microservices.VMSet])
vmTemplateClient := vmtemplatepb.NewVMTemplateSvcClient(connections[microservices.VMTemplate])

gs := microservices.CreateGRPCServer(serviceConfig.ServerCert.Clone())

vs := vmservice.NewGrpcVMServer(hfClient, hfInformerFactory)
vmpb.RegisterVMSvcServer(gs, vs)
vmController, err := vmservice.NewVMController(
kubeClient,
vs,
hfInformerFactory,
environmentClient,
terraformClient,
vmClaimClient,
vmSetClient,
vmTemplateClient,
ctx,
)
if err != nil {
glog.Fatalf("failed creating vm controller: %s", err.Error())
}

var wg sync.WaitGroup
// only add 1 to our wait group since our service should stop (and restart) as soon as one of the go routines terminates
Expand All @@ -103,11 +73,6 @@ func main() {
microservices.StartAPIServer(vmServer)
}()

go func() {
defer wg.Done()
vmController.RunSharded(stopCh, microservices.VM)
}()

hfInformerFactory.Start(stopCh)

wg.Wait()
Expand Down

0 comments on commit 602245e

Please sign in to comment.