Skip to content

Commit

Permalink
Add pipeline cordon and drain (#515)
Browse files Browse the repository at this point in the history
* feat: 🎸 add pipeline cmd to trigger cordon and drain job from local

* docs(cobra): update auto-generated documentation

---------

Co-authored-by: jaskaransarkaria <[email protected]>
  • Loading branch information
jaskaransarkaria and jaskaransarkaria committed Dec 13, 2023
1 parent 7bd5786 commit ccb12d8
Show file tree
Hide file tree
Showing 5 changed files with 112 additions and 0 deletions.
1 change: 1 addition & 0 deletions doc/cloud-platform_pipeline.md
Original file line number Diff line number Diff line change
Expand Up @@ -17,5 +17,6 @@ Cloud Platform pipeline actions
### SEE ALSO

* [cloud-platform](cloud-platform.md) - Multi-purpose CLI from the Cloud Platform team
* [cloud-platform pipeline cordon-and-drain](cloud-platform_pipeline_cordon-and-drain.md) - cordon and drain a node group on a cluster
* [cloud-platform pipeline delete-cluster](cloud-platform_pipeline_delete-cluster.md) - delete a cloud-platform cluster via the pipeline

44 changes: 44 additions & 0 deletions doc/cloud-platform_pipeline_cordon-and-drain.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
## cloud-platform pipeline cordon-and-drain

cordon and drain a node group on a cluster

### Synopsis


Running this command will cordon and drain an existing node group in a eks cluster in the cloud-platform aws account.
It will not terminate the nodes nor will it delete the node group.

The cordon and drain will run remotely in the pipeline, and under the hood it calls "cloud-platform cluster recycle-node --name <node-name> --drain-only --ignore-label"

You must have the following environment variables set, or passed via arguments:
- a cluster name

** You _must_ have the fly cli installed **
--> https://concourse-ci.org/fly.html

** You must also have wget installed **
--> brew install wget


```
cloud-platform pipeline cordon-and-drain [flags]
```

### Options

```
--cluster-name string cluster to run pipeline cmds against
-h, --help help for cordon-and-drain
--node-group string node group name to cordon and drain
```

### Options inherited from parent commands

```
--skip-version-check don't check for updates
```

### SEE ALSO

* [cloud-platform pipeline](cloud-platform_pipeline.md) - Cloud Platform pipeline actions

3 changes: 3 additions & 0 deletions doc/cloud-platform_pipeline_delete-cluster.md
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,9 @@ You must have the following environment variables set, or passed via arguments:
** You _must_ have the fly cli installed **
--> https://concourse-ci.org/fly.html

** You must also have wget installed **
--> brew install wget


```
cloud-platform pipeline delete-cluster [flags]
Expand Down
55 changes: 55 additions & 0 deletions pkg/commands/pipeline.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,11 +11,62 @@ import (
type pipelineOptions struct {
util.Options
Name string
NodeGroupName string
MaxNameLength int8
}

var cliOpt pipelineOptions

func (opt *pipelineOptions) addPipelineCordonAndDrainClusterFlags(cmd *cobra.Command) {
cmd.Flags().StringVar(&cliOpt.Name, "cluster-name", "", "cluster to run pipeline cmds against")
cmd.Flags().StringVar(&cliOpt.NodeGroupName, "node-group", "", "node group name to cordon and drain")
}

func addPipelineCordonAndDrainClusterCmd(toplevel *cobra.Command) {
cmd := &cobra.Command{
Use: "cordon-and-drain",
Short: `cordon and drain a node group on a cluster`,
Long: heredoc.Doc(`
Running this command will cordon and drain an existing node group in a eks cluster in the cloud-platform aws account.
It will not terminate the nodes nor will it delete the node group.
The cordon and drain will run remotely in the pipeline, and under the hood it calls "cloud-platform cluster recycle-node --name <node-name> --drain-only --ignore-label"
You must have the following environment variables set, or passed via arguments:
- a cluster name
** You _must_ have the fly cli installed **
--> https://concourse-ci.org/fly.html
** You must also have wget installed **
--> brew install wget
`),
PreRun: upgradeIfNotLatest,
Run: func(cmd *cobra.Command, args []string) {
contextLogger := log.WithFields(log.Fields{"subcommand": "cordon-and-drain"})
cliOpt.MaxNameLength = 12

if cliOpt.Name == "" {
contextLogger.Fatal("--cluster-name is required")
}

if err := cliOpt.IsNameValid(); err != nil {
contextLogger.Fatal(err)
}

if cliOpt.NodeGroupName == "" {
contextLogger.Fatal("--node-group-to-drain is required")
}

pipeline.CordonAndDrainPipelineShellCmds(cliOpt.Name, cliOpt.NodeGroupName)
},
}

cliOpt.addPipelineCordonAndDrainClusterFlags(cmd)
toplevel.AddCommand(cmd)
}

func (opt *pipelineOptions) addPipelineDeleteClusterFlags(cmd *cobra.Command) {
cmd.Flags().StringVar(&cliOpt.Name, "cluster-name", "", "cluster to delete")
}
Expand All @@ -36,6 +87,9 @@ func addPipelineDeleteClusterCmd(toplevel *cobra.Command) {
** You _must_ have the fly cli installed **
--> https://concourse-ci.org/fly.html
** You must also have wget installed **
--> brew install wget
`),
PreRun: upgradeIfNotLatest,
Run: func(cmd *cobra.Command, args []string) {
Expand All @@ -62,6 +116,7 @@ func addPipelineCmd(topLevel *cobra.Command) {
topLevel.AddCommand(pipelineCmd)

addPipelineDeleteClusterCmd(pipelineCmd)
addPipelineCordonAndDrainClusterCmd(pipelineCmd)
}

var pipelineCmd = &cobra.Command{
Expand Down
9 changes: 9 additions & 0 deletions pkg/pipeline/pipeline.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,3 +25,12 @@ func DeletePipelineShellCmds(clusterName string) {
runCmd("fly", []string{"--target", "manager", "trigger-job", "-j", "delete-cluster/delete"})
fmt.Println("Deleting... https://concourse.cloud-platform.service.justice.gov.uk/teams/main/pipelines/delete-cluster/jobs/delete/builds/latest")
}

func CordonAndDrainPipelineShellCmds(clusterName, nodeGroup string) {
strCmd := fmt.Sprintf("wget -qO- https://raw.githubusercontent.com/ministryofjustice/cloud-platform-terraform-concourse/main/pipelines/manager/main/cordon-and-drain.yaml | fly -t manager set-pipeline --pipeline cordon-and-drain-nodes --config - -v node_group_to_drain=%s -v cluster_name=%s", nodeGroup, clusterName)

runCmd("fly", []string{"--target", "manager", "login", "--team-name", "main", "--concourse-url", "https://concourse.cloud-platform.service.justice.gov.uk/"})
runCmd("bash", []string{"-c", strCmd})
runCmd("fly", []string{"--target", "manager", "trigger-job", "-j", "cordon-and-drain-nodes/cordon-and-drain-nodes"})
fmt.Println("Cordoning and Draining... https://concourse.cloud-platform.service.justice.gov.uk/teams/main/pipelines/cordon-and-drain-nodes/jobs/cordon-and-drain-nodes/builds/latest")
}

0 comments on commit ccb12d8

Please sign in to comment.