From 84d1d6e02ad1679e3293b9035f420d7760a712f5 Mon Sep 17 00:00:00 2001 From: Eytan Avisror Date: Tue, 28 Jul 2020 10:56:42 -0700 Subject: [PATCH] Reap tainted nodes (#35) * Reap nodes with taint * Update README.md * refactor (coverage) --- cmd/governor/app/nodereaper.go | 2 +- pkg/reaper/README.md | 5 + pkg/reaper/nodereaper/helpers.go | 35 +++ pkg/reaper/nodereaper/nodereaper.go | 74 +++++- pkg/reaper/nodereaper/nodereaper_test.go | 311 ++++++++++++++++++++--- pkg/reaper/nodereaper/types.go | 2 + 6 files changed, 385 insertions(+), 44 deletions(-) diff --git a/cmd/governor/app/nodereaper.go b/cmd/governor/app/nodereaper.go index d6be5e7..56b63ab 100644 --- a/cmd/governor/app/nodereaper.go +++ b/cmd/governor/app/nodereaper.go @@ -62,5 +62,5 @@ func init() { nodeReapCmd.Flags().Int32Var(&nodeReaperArgs.ReapUnjoinedThresholdMinutes, "reap-unjoined-threshold-minutes", 15, "Reap N minute old nodes") nodeReapCmd.Flags().StringVar(&nodeReaperArgs.ReapUnjoinedKey, "reap-unjoined-tag-key", "", "BE CAREFUL! EC2 tag key that identfies a joining node") nodeReapCmd.Flags().StringVar(&nodeReaperArgs.ReapUnjoinedValue, "reap-unjoined-tag-value", "", "BE CAREFUL! EC2 tag value that identfies a joining node") - + nodeReapCmd.Flags().StringArrayVar(&nodeReaperArgs.ReapTainted, "reap-tainted", []string{}, "marks nodes with a given taint reapable, must be in format of comma separated taints key=value:effect, key:effect or key") } diff --git a/pkg/reaper/README.md b/pkg/reaper/README.md index 9b378b1..e0cbf5d 100644 --- a/pkg/reaper/README.md +++ b/pkg/reaper/README.md @@ -218,6 +218,11 @@ Ghost nodes are nodes which point to an instance-id which is invalid or already Unjoined nodes are nodes which fail to join the cluster and remain unjoined while taking capacity from the scaling groups. By default this feature is not enabled, but can be enabled by setting `--reap-unjoined=true`, you must also set `--reap-unjoined-threshold-minutes` which is the number of minutes passed since EC2 launch time to consider a node unjoined (we recommend setting a relatively high number here, e.g. 15), also `--reap-unjoined-tag-key` and `--reap-unjoined-tag-value` are required in order to identify the instances which failed to join, and should match an EC2 tag on the cluster nodes. when this is enabled, node-reaper will actively look at all EC2 instances with the mentioned key/value tag, and make sure they are joined in the cluster as nodes by looking at their `ProviderID`, if a matching node is not found and the EC2 instance has been up for more than the configured thershold, the instance will be terminated. +### Reaping Tainted Nodes + +You can chose to mark nodes with certain taints reapable by using the `--reap-tainted` flag and providing a comma separated list of taint strings. +for example, `--reap-tainted NodeWithImpairedVolumes=true:NoSchedule,MyTaint:NoSchedule`, would mean nodes having either one of these taints will be drained & terminated. You can use the following formats for describing a taint - key=value:effect, key:effect, key. + ### Example ```text diff --git a/pkg/reaper/nodereaper/helpers.go b/pkg/reaper/nodereaper/helpers.go index 0d7724c..0d16a47 100644 --- a/pkg/reaper/nodereaper/helpers.go +++ b/pkg/reaper/nodereaper/helpers.go @@ -29,11 +29,46 @@ import ( "github.com/aws/aws-sdk-go/service/autoscaling/autoscalingiface" "github.com/aws/aws-sdk-go/service/ec2" "github.com/aws/aws-sdk-go/service/ec2/ec2iface" + "github.com/pkg/errors" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" ) +func parseTaint(t string) (v1.Taint, bool, error) { + var key, value string + var effect v1.TaintEffect + var taint v1.Taint + + parts := strings.Split(t, ":") + + switch len(parts) { + case 1: + key = parts[0] + case 2: + effect = v1.TaintEffect(parts[1]) + KV := strings.Split(parts[0], "=") + + if len(KV) > 2 { + return taint, false, errors.Errorf("invalid taint %v provided", t) + } + + key = KV[0] + + if len(KV) == 2 { + value = KV[1] + } + default: + return taint, false, errors.Errorf("invalid taint %v provided", t) + } + + taint.Key = key + taint.Value = value + taint.Effect = effect + taint.TimeAdded = &metav1.Time{Time: time.Time{}} + return taint, true, nil +} + func runCommand(call string, arg []string) (string, error) { log.Infof("invoking >> %s %s", call, arg) out, err := exec.Command(call, arg...).CombinedOutput() diff --git a/pkg/reaper/nodereaper/nodereaper.go b/pkg/reaper/nodereaper/nodereaper.go index 7527430..704e54d 100644 --- a/pkg/reaper/nodereaper/nodereaper.go +++ b/pkg/reaper/nodereaper/nodereaper.go @@ -16,9 +16,9 @@ limitations under the License. package nodereaper import ( - "errors" "fmt" "os" + "reflect" "time" "github.com/aws/aws-sdk-go/aws" @@ -26,6 +26,7 @@ import ( "github.com/aws/aws-sdk-go/service/autoscaling" "github.com/aws/aws-sdk-go/service/ec2" "github.com/keikoproj/governor/pkg/reaper/common" + "github.com/pkg/errors" "github.com/sirupsen/logrus" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -39,10 +40,10 @@ const ( terminatedStateName = "termination-issued" drainingStateName = "draining" reaperDisableLabelKey = "governor.keikoproj.io/node-reaper-disabled" - reapUnreadyDisabledLabelKey = "governor.keikoproj.io/reap-unready-disabled" - reapUnknownDisabledLabelKey = "governor.keikoproj.io/reap-unknown-disabled" - reapFlappyDisabledLabelKey = "governor.keikoproj.io/reap-flappy-disabled" - reapOldDisabledLabelKey = "governor.keikoproj.io/reap-old-disabled" + reapUnreadyDisabledLabelKey = "governor.keikoproj.io/reap-unready-disabled" + reapUnknownDisabledLabelKey = "governor.keikoproj.io/reap-unknown-disabled" + reapFlappyDisabledLabelKey = "governor.keikoproj.io/reap-flappy-disabled" + reapOldDisabledLabelKey = "governor.keikoproj.io/reap-old-disabled" ) // Validate command line arguments @@ -64,6 +65,7 @@ func (ctx *ReaperContext) validateArguments(args *Args) error { ctx.NodeInstanceIDs = make(map[string]string) ctx.AgeDrainReapableInstances = make([]AgeDrainReapableInstance, 0) ctx.AgeKillOrder = make([]string, 0) + ctx.ReapTainted = make([]v1.Taint, 0) ctx.EC2Region = args.EC2Region ctx.ReapOld = args.ReapOld ctx.MaxKill = args.MaxKill @@ -75,6 +77,18 @@ func (ctx *ReaperContext) validateArguments(args *Args) error { log.Infof("ASG Validation = %t", ctx.AsgValidation) log.Infof("Post Reap Throttle = %v seconds", ctx.ReapThrottle) + for _, t := range args.ReapTainted { + var taint v1.Taint + var ok bool + var err error + + if taint, ok, err = parseTaint(t); !ok { + return errors.Wrap(err, "failed to parse taint") + } + + ctx.ReapTainted = append(ctx.ReapTainted, taint) + } + if ctx.MaxKill < 1 { err := fmt.Errorf("--max-kill-nodes must be set to a number greater than or equal to 1") log.Errorln(err) @@ -245,7 +259,14 @@ func Run(args *Args) error { log.Infoln("starting drain condition check for ghost nodes") err = ctx.deriveGhostDrainReapableNodes(awsAuth) if err != nil { - log.Errorf("failed to derive age drain-reapable nodes, %v", err) + log.Errorf("failed to derive ghost nodes, %v", err) + return err + } + + log.Infoln("starting drain condition check for tainted nodes") + err = ctx.deriveTaintDrainReapableNodes() + if err != nil { + log.Errorf("failed to derive taint drain-reapable nodes, %v", err) return err } @@ -281,6 +302,24 @@ func Run(args *Args) error { return nil } +func (ctx *ReaperContext) deriveTaintDrainReapableNodes() error { + if len(ctx.ReapTainted) == 0 { + return nil + } + + log.Infoln("scanning for taint drain-reapable nodes") + for _, node := range ctx.AllNodes { + nodeInstanceID := getNodeInstanceID(&node) + for _, t := range ctx.ReapTainted { + if nodeIsTainted(t, node) { + ctx.addDrainable(node.Name, nodeInstanceID) + ctx.addReapable(node.Name, nodeInstanceID) + } + } + } + return nil +} + // Handle age-reapable nodes func (ctx *ReaperContext) deriveAgeDrainReapableNodes() error { log.Infoln("scanning for age drain-reapable nodes") @@ -298,7 +337,7 @@ func (ctx *ReaperContext) deriveAgeDrainReapableNodes() error { // Drain-Reap old nodes if ctx.ReapOld { - if !nodeHasAnnotation(node, ageUnreapableAnnotationKey, "true") && !hasSkipLabel(node, reapOldDisabledLabelKey){ + if !nodeHasAnnotation(node, ageUnreapableAnnotationKey, "true") && !hasSkipLabel(node, reapOldDisabledLabelKey) { if nodeIsAgeReapable(nodeAgeMinutes, ageThreshold) { log.Infof("node %v is drain-reapable !! State = OldAge, Diff = %v/%v", nodeName, nodeAgeMinutes, ageThreshold) ctx.addAgeDrainReapable(nodeName, nodeInstanceID, nodeAgeMinutes) @@ -326,7 +365,7 @@ func (ctx *ReaperContext) deriveFlappyDrainReapableNodes() error { // Drain-Reap flappy nodes if ctx.ReapFlappy { - if nodeIsFlappy(events, nodeName, countThreshold, "NodeReady") && !hasSkipLabel(node, reapFlappyDisabledLabelKey) { + if nodeIsFlappy(events, nodeName, countThreshold, "NodeReady") && !hasSkipLabel(node, reapFlappyDisabledLabelKey) { log.Infof("node %v is drain-reapable !! State = ReadinessFlapping", nodeName) ctx.addDrainable(nodeName, nodeInstanceID) ctx.addReapable(nodeName, nodeInstanceID) @@ -625,7 +664,7 @@ func (ctx *ReaperContext) scan(w ReaperAwsAuth) error { log.Infof("found %v nodes, %v pods, and %v events", len(ctx.AllNodes), len(ctx.AllPods), len(ctx.AllEvents)) for _, node := range nodeList.Items { ctx.NodeInstanceIDs[getNodeInstanceID(&node)] = node.Name - if (nodeStateIsNotReady(&node) || nodeStateIsUnknown(&node)) { + if nodeStateIsNotReady(&node) || nodeStateIsUnknown(&node) { log.Infof("node %v is not ready", node.ObjectMeta.Name) ctx.UnreadyNodes = append(ctx.UnreadyNodes, node) } @@ -720,6 +759,23 @@ func autoScalingGroupIsStable(w ReaperAwsAuth, instance string) (bool, error) { return true, nil } +func nodeIsTainted(taint v1.Taint, node v1.Node) bool { + for _, t := range node.Spec.Taints { + // ignore timeAdded + t.TimeAdded = &metav1.Time{Time: time.Time{}} + + // handle key only match + if taint.Effect == v1.TaintEffect("") && taint.Value == "" && taint.Key == t.Key { + return true + } + + if reflect.DeepEqual(taint, t) { + return true + } + } + return false +} + func nodeIsFlappy(events []v1.Event, name string, threshold int32, reason string) bool { totalFlapEvents := make(map[string]int32) for _, event := range events { diff --git a/pkg/reaper/nodereaper/nodereaper_test.go b/pkg/reaper/nodereaper/nodereaper_test.go index abe3ab5..8bae273 100644 --- a/pkg/reaper/nodereaper/nodereaper_test.go +++ b/pkg/reaper/nodereaper/nodereaper_test.go @@ -173,6 +173,7 @@ func newFakeReaperContext() *ReaperContext { ctx.GhostInstances = make(map[string]string) ctx.ClusterInstancesData = make(map[string]float64) ctx.NodeInstanceIDs = make(map[string]string) + ctx.ReapTainted = make([]v1.Taint, 0) ctx.AgeDrainReapableInstances = make([]AgeDrainReapableInstance, 0) ctx.AgeKillOrder = make([]string, 0) // Default Flags @@ -232,7 +233,6 @@ func createNodeLabels(node FakeNode, ctx *ReaperContext) map[string]string { return nodeLabels } - func createFakeNodes(nodes []FakeNode, ctx *ReaperContext) { for _, n := range nodes { nodeLabels := createNodeLabels(n, ctx) @@ -335,6 +335,7 @@ func createFakeNodes(nodes []FakeNode, ctx *ReaperContext) { Labels: nodeLabels, }, Spec: v1.NodeSpec{ ProviderID: n.providerID, + Taints: n.nodeTaints, }, Status: v1.NodeStatus{ Conditions: nodeConditions, }} @@ -412,6 +413,7 @@ func runFakeReaper(ctx *ReaperContext, awsAuth ReaperAwsAuth) { ctx.deriveFlappyDrainReapableNodes() ctx.deriveAgeDrainReapableNodes() ctx.deriveGhostDrainReapableNodes(awsAuth) + ctx.deriveTaintDrainReapableNodes() ctx.deriveReapableNodes() ctx.reapUnhealthyNodes(awsAuth) ctx.reapOldNodes(awsAuth) @@ -474,7 +476,6 @@ func (u *ReaperUnitTest) Run(t *testing.T, timeTest bool) { } } - type ReaperUnitTest struct { TestDescription string Nodes []FakeNode @@ -509,6 +510,7 @@ type FakeNode struct { unreadyPods int lostPods int ageMinutes int + nodeTaints []v1.Taint nodeLabels map[string]string } type FakePod struct { @@ -627,6 +629,151 @@ func TestReapOldPositive(t *testing.T) { testCase.Run(t, false) } +func TestReapTaintedPositive(t *testing.T) { + reaper := newFakeReaperContext() + reaper.ReapTainted = []v1.Taint{ + { + Key: "key", + Value: "value", + Effect: "effect", + TimeAdded: &metav1.Time{Time: time.Time{}}, + }, + { + Key: "key", + Effect: "effect", + TimeAdded: &metav1.Time{Time: time.Time{}}, + }, + { + Key: "key", + TimeAdded: &metav1.Time{Time: time.Time{}}, + }, + } + + testCase := ReaperUnitTest{ + TestDescription: "Reap Tainted - should reap tainted nodes", + InstanceGroup: FakeASG{ + Name: "my-ig.cluster.k8s.local", + Healthy: 4, + Unhealthy: 0, + Desired: 4, + }, + Nodes: []FakeNode{ + { + nodeName: "node-1", + state: "Ready", + ageMinutes: 60, + nodeTaints: []v1.Taint{ + { + Key: "key", + Value: "value", + Effect: "effect", + TimeAdded: &metav1.Time{Time: time.Time{}}, + }, + }, + }, + { + nodeName: "node-2", + state: "Ready", + ageMinutes: 120, + nodeTaints: []v1.Taint{ + { + Key: "key", + Effect: "effect", + TimeAdded: &metav1.Time{Time: time.Time{}}, + }, + }, + }, + { + nodeName: "node-3", + state: "Ready", + ageMinutes: 240, + nodeTaints: []v1.Taint{ + { + Key: "key", + Effect: "some-effect", + TimeAdded: &metav1.Time{Time: time.Time{}}, + }, + }, + }, + { + nodeName: "node-4", + state: "Ready", + ageMinutes: 10, + }, + }, + FakeReaper: reaper, + ExpectedReapable: 3, + ExpectedTerminated: 3, + ExpectedDrainable: 3, + ExpectedDrained: 3, + } + testCase.Run(t, false) +} + +func TestReapTaintedNegative(t *testing.T) { + reaper := newFakeReaperContext() + reaper.ReapTainted = []v1.Taint{} + + testCase := ReaperUnitTest{ + TestDescription: "Reap Tainted - should not reap tainted nodes if taints not provided", + InstanceGroup: FakeASG{ + Name: "my-ig.cluster.k8s.local", + Healthy: 4, + Unhealthy: 0, + Desired: 4, + }, + Nodes: []FakeNode{ + { + nodeName: "node-1", + state: "Ready", + ageMinutes: 60, + nodeTaints: []v1.Taint{ + { + Key: "key", + Value: "value", + Effect: "effect", + TimeAdded: &metav1.Time{Time: time.Time{}}, + }, + }, + }, + { + nodeName: "node-2", + state: "Ready", + ageMinutes: 120, + nodeTaints: []v1.Taint{ + { + Key: "key", + Effect: "effect", + TimeAdded: &metav1.Time{Time: time.Time{}}, + }, + }, + }, + { + nodeName: "node-3", + state: "Ready", + ageMinutes: 240, + nodeTaints: []v1.Taint{ + { + Key: "key", + Effect: "some-effect", + TimeAdded: &metav1.Time{Time: time.Time{}}, + }, + }, + }, + { + nodeName: "node-4", + state: "Ready", + ageMinutes: 60, + }, + }, + FakeReaper: reaper, + ExpectedReapable: 0, + ExpectedTerminated: 0, + ExpectedDrained: 0, + } + testCase.Run(t, false) +} + func TestReapOldNegative(t *testing.T) { reaper := newFakeReaperContext() @@ -1428,21 +1575,21 @@ func TestSkipLabelReaper(t *testing.T) { nodeLabels: map[string]string{reaperDisableLabelKey: "true"}, }, { - nodeName: "node-flappy", - state: "Ready", + nodeName: "node-flappy", + state: "Ready", nodeLabels: map[string]string{reaperDisableLabelKey: "true"}, }, { - nodeName: "node-unknown", - state: "Unknown", + nodeName: "node-unknown", + state: "Unknown", lastTransitionMinutes: 6, - nodeLabels: map[string]string{reaperDisableLabelKey: "true"}, + nodeLabels: map[string]string{reaperDisableLabelKey: "true"}, }, { - nodeName: "node-unready", - state: "NotReady", + nodeName: "node-unready", + state: "NotReady", lastTransitionMinutes: 6, - nodeLabels: map[string]string{reaperDisableLabelKey: "true"}, + nodeLabels: map[string]string{reaperDisableLabelKey: "true"}, }, }, Events: []FakeEvent{ @@ -1453,13 +1600,13 @@ func TestSkipLabelReaper(t *testing.T) { kind: "Node", }, }, - FakeReaper: reaper, - ExpectedUnready: 2, - ExpectedReapable: 0, - ExpectedDrainable: 0, + FakeReaper: reaper, + ExpectedUnready: 2, + ExpectedReapable: 0, + ExpectedDrainable: 0, ExpectedOldReapable: 0, - ExpectedTerminated: 0, - ExpectedDrained: 0, + ExpectedTerminated: 0, + ExpectedDrained: 0, } testCase.Run(t, false) } @@ -1479,16 +1626,16 @@ func TestSkipLabelUnknownNodes(t *testing.T) { }, Nodes: []FakeNode{ { - nodeName: "node-unknown-1", - state: "Unknown", + nodeName: "node-unknown-1", + state: "Unknown", lastTransitionMinutes: 6, - nodeLabels: map[string]string{reapUnknownDisabledLabelKey: "true"}, + nodeLabels: map[string]string{reapUnknownDisabledLabelKey: "true"}, }, { - nodeName: "node-unknown-2", - state: "Unknown", + nodeName: "node-unknown-2", + state: "Unknown", lastTransitionMinutes: 6, - nodeLabels: map[string]string{reapUnknownDisabledLabelKey: "false"}, + nodeLabels: map[string]string{reapUnknownDisabledLabelKey: "false"}, }, }, FakeReaper: reaper, @@ -1516,16 +1663,16 @@ func TestSkipLabelUnreadyNodes(t *testing.T) { }, Nodes: []FakeNode{ { - nodeName: "node-unready-1", - state: "NotReady", + nodeName: "node-unready-1", + state: "NotReady", lastTransitionMinutes: 6, - nodeLabels: map[string]string{reapUnreadyDisabledLabelKey: "true"}, + nodeLabels: map[string]string{reapUnreadyDisabledLabelKey: "true"}, }, { - nodeName: "node-unready-2", - state: "NotReady", + nodeName: "node-unready-2", + state: "NotReady", lastTransitionMinutes: 6, - nodeLabels: map[string]string{reapUnreadyDisabledLabelKey: "false"}, + nodeLabels: map[string]string{reapUnreadyDisabledLabelKey: "false"}, }, }, FakeReaper: reaper, @@ -1563,11 +1710,11 @@ func TestSkipLabelOldNodes(t *testing.T) { nodeLabels: map[string]string{reapOldDisabledLabelKey: "false"}, }, }, - FakeReaper: reaper, - ExpectedUnready: 0, + FakeReaper: reaper, + ExpectedUnready: 0, ExpectedOldReapable: 1, - ExpectedTerminated: 1, - ExpectedDrained: 1, + ExpectedTerminated: 1, + ExpectedDrained: 1, } testCase.Run(t, false) } @@ -1591,8 +1738,8 @@ func TestSkipLabelFlappyNodes(t *testing.T) { nodeLabels: map[string]string{reapFlappyDisabledLabelKey: "true"}, }, { - nodeName: "ip-10-10-10-11.us-west-2.compute.local", - state: "Ready", + nodeName: "ip-10-10-10-11.us-west-2.compute.local", + state: "Ready", nodeLabels: map[string]string{reapFlappyDisabledLabelKey: "false"}, }, }, @@ -1626,3 +1773,99 @@ func TestSkipLabelFlappyNodes(t *testing.T) { testCase.Run(t, false) } + +func TestParseTaint(t *testing.T) { + + fullTaint := v1.Taint{ + Key: "fullTaint", + Value: "value", + Effect: "NoSchedule", + TimeAdded: &metav1.Time{Time: time.Time{}}, + } + + keyEffectTaint := v1.Taint{ + Key: "keyEffectTaint", + Effect: "NoSchedule", + TimeAdded: &metav1.Time{Time: time.Time{}}, + } + + keyTaint := v1.Taint{ + Key: "keyTaint", + TimeAdded: &metav1.Time{Time: time.Time{}}, + } + + emptyTaint := v1.Taint{ + TimeAdded: &metav1.Time{Time: time.Time{}}, + } + + tests := []struct { + taintStr string + expectedTaint v1.Taint + }{ + {taintStr: "fullTaint=value:NoSchedule", expectedTaint: fullTaint}, + {taintStr: "keyEffectTaint:NoSchedule", expectedTaint: keyEffectTaint}, + {taintStr: "keyTaint", expectedTaint: keyTaint}, + {taintStr: "invalid:taint:invalid=taint", expectedTaint: v1.Taint{}}, + {taintStr: "invalid=taint=invalid:taint", expectedTaint: v1.Taint{}}, + {taintStr: "", expectedTaint: emptyTaint}, + } + + for i, tc := range tests { + got, _, _ := parseTaint(tc.taintStr) + if !reflect.DeepEqual(got, tc.expectedTaint) { + t.Fatalf("test #%v: expected match: %+v, got: %+v", i, tc.expectedTaint, got) + } + } +} + +func TestNodeIsTainted(t *testing.T) { + fullTaint := v1.Taint{ + Key: "fullTaint", + Value: "value", + Effect: "NoSchedule", + TimeAdded: &metav1.Time{Time: time.Time{}}, + } + + keyEffectTaint := v1.Taint{ + Key: "keyEffectTaint", + Effect: "NoSchedule", + TimeAdded: &metav1.Time{Time: time.Time{}}, + } + + keyTaint := v1.Taint{ + Key: "keyTaint", + TimeAdded: &metav1.Time{Time: time.Time{}}, + } + + otherTaint := v1.Taint{ + Key: "otherTaint", + Value: "value", + Effect: "NoSchedule", + TimeAdded: &metav1.Time{Time: time.Time{}}, + } + + taintedNode := v1.Node{ + Spec: v1.NodeSpec{ + Taints: []v1.Taint{fullTaint, keyEffectTaint, keyTaint}, + }, + } + + tests := []struct { + taint v1.Taint + node v1.Node + shouldMatch bool + }{ + {taint: fullTaint, node: taintedNode, shouldMatch: true}, + {taint: keyEffectTaint, node: taintedNode, shouldMatch: true}, + {taint: keyTaint, node: taintedNode, shouldMatch: true}, + {taint: otherTaint, node: taintedNode, shouldMatch: false}, + {taint: otherTaint, node: v1.Node{}, shouldMatch: false}, + } + + for i, tc := range tests { + got := nodeIsTainted(tc.taint, tc.node) + if !reflect.DeepEqual(got, tc.shouldMatch) { + t.Fatalf("test #%v: expected match: %t, got: %t", i, tc.shouldMatch, got) + } + } +} diff --git a/pkg/reaper/nodereaper/types.go b/pkg/reaper/nodereaper/types.go index b84af18..61b4467 100644 --- a/pkg/reaper/nodereaper/types.go +++ b/pkg/reaper/nodereaper/types.go @@ -56,6 +56,7 @@ type Args struct { ReapThrottle int64 AgeReapThrottle int64 ReapAfter float64 + ReapTainted []string } // ReaperContext holds the context of the node-reaper and target cluster @@ -85,6 +86,7 @@ type ReaperContext struct { FlapCount int32 MaxKill int TimeToReap float64 + ReapTainted []v1.Taint // runtime UnreadyNodes []v1.Node AllNodes []v1.Node