From d9ecf2a9ca6541abdea9676b9efb5759b8024229 Mon Sep 17 00:00:00 2001 From: whitewindmills Date: Tue, 27 Aug 2024 14:32:00 +0800 Subject: [PATCH] Standardize the health probe and metrics arguments of descheduler Signed-off-by: whitewindmills --- artifacts/deploy/karmada-descheduler.yaml | 3 +- .../templates/karmada-descheduler.yaml | 3 +- cmd/descheduler/app/descheduler.go | 67 +++++++++++++++---- cmd/descheduler/app/options/options.go | 31 +++++++++ cmd/descheduler/app/options/validation.go | 10 --- .../app/options/validation_test.go | 14 ---- operator/pkg/controlplane/manifests.go | 3 +- .../addons/descheduler/manifests.go | 4 +- 8 files changed, 93 insertions(+), 42 deletions(-) diff --git a/artifacts/deploy/karmada-descheduler.yaml b/artifacts/deploy/karmada-descheduler.yaml index 53d0d189bd7c..8a52826a0368 100644 --- a/artifacts/deploy/karmada-descheduler.yaml +++ b/artifacts/deploy/karmada-descheduler.yaml @@ -26,7 +26,8 @@ spec: command: - /bin/karmada-descheduler - --kubeconfig=/etc/kubeconfig - - --bind-address=0.0.0.0 + - --metrics-bind-address=0.0.0.0:10358 + - --health-probe-bind-address=0.0.0.0:10358 - --scheduler-estimator-ca-file=/etc/karmada/pki/ca.crt - --scheduler-estimator-cert-file=/etc/karmada/pki/karmada.crt - --scheduler-estimator-key-file=/etc/karmada/pki/karmada.key diff --git a/charts/karmada/templates/karmada-descheduler.yaml b/charts/karmada/templates/karmada-descheduler.yaml index 3b2ae61c8fb5..ee96eda825aa 100644 --- a/charts/karmada/templates/karmada-descheduler.yaml +++ b/charts/karmada/templates/karmada-descheduler.yaml @@ -50,7 +50,8 @@ spec: command: - /bin/karmada-descheduler - --kubeconfig=/etc/kubeconfig - - --bind-address=0.0.0.0 + - --metrics-bind-address=0.0.0.0:10358 + - --health-probe-bind-address=0.0.0.0:10358 - --leader-elect-resource-namespace={{ $systemNamespace }} - --scheduler-estimator-ca-file=/etc/karmada/pki/server-ca.crt - --scheduler-estimator-cert-file=/etc/karmada/pki/karmada.crt diff --git a/cmd/descheduler/app/descheduler.go b/cmd/descheduler/app/descheduler.go index 17183732e16f..b93c4c4be239 100644 --- a/cmd/descheduler/app/descheduler.go +++ b/cmd/descheduler/app/descheduler.go @@ -19,10 +19,8 @@ package app import ( "context" "fmt" - "net" "net/http" "os" - "strconv" "time" "github.com/prometheus/client_golang/prometheus/promhttp" @@ -87,6 +85,11 @@ func NewDeschedulerCommand(stopChan <-chan struct{}) *cobra.Command { if they are failed to be scheduled for a period of time. It relies on karmada-scheduler-estimator to get replica status.`, RunE: func(_ *cobra.Command, _ []string) error { + // complete options + if err := opts.Complete(); err != nil { + return err + } + // validate options if errs := opts.Validate(); len(errs) != 0 { return errs.ToAggregate() @@ -127,7 +130,7 @@ karmada-scheduler-estimator to get replica status.`, func run(opts *options.Options, stopChan <-chan struct{}) error { klog.Infof("karmada-descheduler version: %s", version.Get()) klog.Infof("Please make sure the karmada-scheduler-estimator of all member clusters has been deployed") - go serveHealthzAndMetrics(net.JoinHostPort(opts.BindAddress, strconv.Itoa(opts.SecurePort))) + serveHealthzAndMetrics(opts.HealthProbeBindAddress, opts.MetricsBindAddress) profileflag.ListenAndServe(opts.ProfileOpts) @@ -191,26 +194,64 @@ func run(opts *options.Options, stopChan <-chan struct{}) error { return nil } -func serveHealthzAndMetrics(address string) { +func serveHealthzAndMetrics(healthProbeBindAddress, metricsBindAddress string) { + if healthProbeBindAddress == metricsBindAddress { + if healthProbeBindAddress != "0" { + go serveCombined(healthProbeBindAddress) + } + } else { + if healthProbeBindAddress != "0" { + go serveHealthz(healthProbeBindAddress) + } + if metricsBindAddress != "0" { + go serveMetrics(metricsBindAddress) + } + } +} + +func serveCombined(address string) { mux := http.NewServeMux() - mux.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) { - w.WriteHeader(http.StatusOK) - _, _ = w.Write([]byte("ok")) - }) + mux.HandleFunc("/healthz", healthzHandler) + mux.Handle("/metrics", metricsHandler()) + + serveHTTP(address, mux, "healthz and metrics") +} + +func serveHealthz(address string) { + mux := http.NewServeMux() + mux.HandleFunc("/healthz", healthzHandler) + serveHTTP(address, mux, "healthz") +} + +func serveMetrics(address string) { + mux := http.NewServeMux() + mux.Handle("/metrics", metricsHandler()) + serveHTTP(address, mux, "metrics") +} - mux.Handle("/metrics", promhttp.HandlerFor(ctrlmetrics.Registry, promhttp.HandlerOpts{ +func healthzHandler(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("ok")) +} + +func metricsHandler() http.Handler { + return promhttp.HandlerFor(ctrlmetrics.Registry, promhttp.HandlerOpts{ ErrorHandling: promhttp.HTTPErrorOnError, - })) + }) +} - httpServer := http.Server{ +func serveHTTP(address string, handler http.Handler, name string) { + httpServer := &http.Server{ Addr: address, - Handler: mux, + Handler: handler, ReadHeaderTimeout: ReadHeaderTimeout, WriteTimeout: WriteTimeout, ReadTimeout: ReadTimeout, } + + klog.Infof("Starting %s server on %s", name, address) if err := httpServer.ListenAndServe(); err != nil { - klog.Errorf("Failed to serve healthz and metrics: %v", err) + klog.Errorf("Failed to serve %s on %s: %v", name, address, err) os.Exit(1) } } diff --git a/cmd/descheduler/app/options/options.go b/cmd/descheduler/app/options/options.go index 67e8a54d4e34..79e8a11a6633 100644 --- a/cmd/descheduler/app/options/options.go +++ b/cmd/descheduler/app/options/options.go @@ -17,6 +17,8 @@ limitations under the License. package options import ( + "net" + "strconv" "time" "github.com/spf13/pflag" @@ -48,8 +50,10 @@ type Options struct { KubeConfig string Master string // BindAddress is the IP address on which to listen for the --secure-port port. + // Deprecated: To specify the TCP address for serving health probes, use HealthProbeBindAddress instead. To specify the TCP address for serving prometheus metrics, use MetricsBindAddress instead. This will be removed in release 1.12+. BindAddress string // SecurePort is the port that the server serves at. + // Deprecated: To specify the TCP address for serving health probes, use HealthProbeBindAddress instead. To specify the TCP address for serving prometheus metrics, use MetricsBindAddress instead. This will be removed in release 1.12+. SecurePort int KubeAPIQPS float32 @@ -75,6 +79,16 @@ type Options struct { // UnschedulableThreshold specifies the period of pod unschedulable condition. UnschedulableThreshold metav1.Duration ProfileOpts profileflag.Options + // MetricsBindAddress is the TCP address that the server should bind to + // for serving prometheus metrics. + // It can be set to "0" to disable the metrics serving. + // Defaults to ":10358". + MetricsBindAddress string + // HealthProbeBindAddress is the TCP address that the server should bind to + // for serving health probes + // It can be set to "0" to disable serving the health probe. + // Defaults to ":10358". + HealthProbeBindAddress string } // NewOptions builds a default descheduler options. @@ -103,6 +117,10 @@ func (o *Options) AddFlags(fs *pflag.FlagSet) { fs.StringVar(&o.Master, "master", o.Master, "The address of the Kubernetes API server. Overrides any value in KubeConfig. Only required if out-of-cluster.") fs.StringVar(&o.BindAddress, "bind-address", defaultBindAddress, "The IP address on which to listen for the --secure-port port.") fs.IntVar(&o.SecurePort, "secure-port", defaultPort, "The secure port on which to serve HTTPS.") + // nolint: errcheck + fs.MarkDeprecated("bind-address", "This flag is deprecated and will be removed in release 1.12+. Use --health-probe-bind-address and --metrics-bind-address instead. Note: In release 1.12+, these two addresses cannot be the same.") + // nolint: errcheck + fs.MarkDeprecated("secure-port", "This flag is deprecated and will be removed in release 1.12+. Use --health-probe-bind-address and --metrics-bind-address instead. Note: In release 1.12+, these two addresses cannot be the same.") fs.Float32Var(&o.KubeAPIQPS, "kube-api-qps", 40.0, "QPS to use while talking with karmada-apiserver.") fs.IntVar(&o.KubeAPIBurst, "kube-api-burst", 60, "Burst to use while talking with karmada-apiserver.") fs.DurationVar(&o.SchedulerEstimatorTimeout.Duration, "scheduler-estimator-timeout", 3*time.Second, "Specifies the timeout period of calling the scheduler estimator service.") @@ -114,5 +132,18 @@ func (o *Options) AddFlags(fs *pflag.FlagSet) { fs.StringVar(&o.SchedulerEstimatorServicePrefix, "scheduler-estimator-service-prefix", "karmada-scheduler-estimator", "The prefix of scheduler estimator service name") fs.DurationVar(&o.DeschedulingInterval.Duration, "descheduling-interval", defaultDeschedulingInterval, "Time interval between two consecutive descheduler executions. Setting this value instructs the descheduler to run in a continuous loop at the interval specified.") fs.DurationVar(&o.UnschedulableThreshold.Duration, "unschedulable-threshold", defaultUnschedulableThreshold, "The period of pod unschedulable condition. This value is considered as a classification standard of unschedulable replicas.") + fs.StringVar(&o.MetricsBindAddress, "metrics-bind-address", "", "The TCP address that the server should bind to for serving prometheus metrics(e.g. 127.0.0.1:10358, :10358). It can be set to \"0\" to disable the metrics serving. Defaults to 0.0.0.0:10358.") + fs.StringVar(&o.HealthProbeBindAddress, "health-probe-bind-address", "", "The TCP address that the server should bind to for serving health probes(e.g. 127.0.0.1:10358, :10358). It can be set to \"0\" to disable serving the health probe. Defaults to 0.0.0.0:10358.") o.ProfileOpts.AddFlags(fs) } + +// Complete ensures that options are valid and marshals them if necessary. +func (o *Options) Complete() error { + if len(o.HealthProbeBindAddress) == 0 { + o.HealthProbeBindAddress = net.JoinHostPort(o.BindAddress, strconv.Itoa(o.SecurePort)) + } + if len(o.MetricsBindAddress) == 0 { + o.MetricsBindAddress = net.JoinHostPort(o.BindAddress, strconv.Itoa(o.SecurePort)) + } + return nil +} diff --git a/cmd/descheduler/app/options/validation.go b/cmd/descheduler/app/options/validation.go index 81bf557e0d08..92e0c8dd5d3d 100644 --- a/cmd/descheduler/app/options/validation.go +++ b/cmd/descheduler/app/options/validation.go @@ -17,8 +17,6 @@ limitations under the License. package options import ( - "net" - "k8s.io/apimachinery/pkg/util/validation/field" ) @@ -27,14 +25,6 @@ func (o *Options) Validate() field.ErrorList { errs := field.ErrorList{} newPath := field.NewPath("Options") - if net.ParseIP(o.BindAddress) == nil { - errs = append(errs, field.Invalid(newPath.Child("BindAddress"), o.BindAddress, "not a valid textual representation of an IP address")) - } - - if o.SecurePort < 0 || o.SecurePort > 65535 { - errs = append(errs, field.Invalid(newPath.Child("SecurePort"), o.SecurePort, "must be a valid port between 0 and 65535 inclusive")) - } - if o.SchedulerEstimatorPort < 0 || o.SchedulerEstimatorPort > 65535 { errs = append(errs, field.Invalid(newPath.Child("SchedulerEstimatorPort"), o.SchedulerEstimatorPort, "must be a valid port between 0 and 65535 inclusive")) } diff --git a/cmd/descheduler/app/options/validation_test.go b/cmd/descheduler/app/options/validation_test.go index 28c5ec126c8c..076bf44b10d6 100644 --- a/cmd/descheduler/app/options/validation_test.go +++ b/cmd/descheduler/app/options/validation_test.go @@ -34,8 +34,6 @@ func New(modifyOptions ModifyOptions) Options { LeaderElection: componentbaseconfig.LeaderElectionConfiguration{ LeaderElect: false, }, - BindAddress: "127.0.0.1", - SecurePort: 9000, KubeAPIQPS: 40, KubeAPIBurst: 30, SchedulerEstimatorTimeout: metav1.Duration{Duration: 1 * time.Second}, @@ -78,18 +76,6 @@ func TestValidateKarmadaDescheduler(t *testing.T) { opt Options expectedErrs field.ErrorList }{ - "invalid BindAddress": { - opt: New(func(option *Options) { - option.BindAddress = "127.0.0.1:8080" - }), - expectedErrs: field.ErrorList{field.Invalid(newPath.Child("BindAddress"), "127.0.0.1:8080", "not a valid textual representation of an IP address")}, - }, - "invalid SecurePort": { - opt: New(func(option *Options) { - option.SecurePort = 90000 - }), - expectedErrs: field.ErrorList{field.Invalid(newPath.Child("SecurePort"), 90000, "must be a valid port between 0 and 65535 inclusive")}, - }, "invalid SchedulerEstimatorPort": { opt: New(func(option *Options) { option.SchedulerEstimatorPort = 90000 diff --git a/operator/pkg/controlplane/manifests.go b/operator/pkg/controlplane/manifests.go index fc67c4d2fa96..bf420156d8fe 100644 --- a/operator/pkg/controlplane/manifests.go +++ b/operator/pkg/controlplane/manifests.go @@ -248,7 +248,8 @@ spec: command: - /bin/karmada-descheduler - --kubeconfig=/etc/karmada/kubeconfig - - --bind-address=0.0.0.0 + - --metrics-bind-address=0.0.0.0:10358 + - --health-probe-bind-address=0.0.0.0:10358 - --leader-elect-resource-namespace={{ .SystemNamespace }} - --scheduler-estimator-ca-file=/etc/karmada/pki/ca.crt - --scheduler-estimator-cert-file=/etc/karmada/pki/karmada.crt diff --git a/pkg/karmadactl/addons/descheduler/manifests.go b/pkg/karmadactl/addons/descheduler/manifests.go index 5b54f25ce098..ad28d37d4aa6 100644 --- a/pkg/karmadactl/addons/descheduler/manifests.go +++ b/pkg/karmadactl/addons/descheduler/manifests.go @@ -45,8 +45,8 @@ spec: command: - /bin/karmada-descheduler - --kubeconfig=/etc/kubeconfig - - --bind-address=0.0.0.0 - - --secure-port=10358 + - --metrics-bind-address=0.0.0.0:10358 + - --health-probe-bind-address=0.0.0.0:10358 - --leader-elect-resource-namespace={{ .Namespace }} - --scheduler-estimator-ca-file=/etc/karmada/pki/ca.crt - --scheduler-estimator-cert-file=/etc/karmada/pki/karmada.crt