diff --git a/adapter_test.go b/adapter_test.go new file mode 100644 index 000000000..71ab02705 --- /dev/null +++ b/adapter_test.go @@ -0,0 +1,29 @@ +package main + +import ( + "flag" + "github.com/AliyunContainerService/alibaba-cloud-metrics-adapter/pkg/provider/prometheusProvider" + "testing" +) + +func TestFlags(t *testing.T) { + + opts := prometheusProvider.NewAlibabaMetricsAdapterOptions() + prometheusProvider.GlobalConfig = opts + opts.AddFlags() + fakeCommandLines := []string{ + "--secure-port=443", + "--prometheus-url=https://cn-shanghai.arms.aliyuncs.com:9443/api/v1/prometheus/xxxx/1251182063904492/xxxx/cn-shanghai", + "--config=/etc/adapter/config.yaml", + "--prometheus-header=Authorization=xxxxxxxxxxxxxxxxxxxxxxxxxxx", + "--metrics-relist-interval=1m", + "--v=9", + } + opts.Flags().AddGoFlagSet(flag.CommandLine) + //opts.Flags().AddGoFlagSet(flag.NewFlagSet(fakeCommandLine, 1)) + if err := opts.Flags().Parse(fakeCommandLines); err != nil { + t.Fatalf("unable to parse flags: %v", err) + } + + t.Logf("finish, opts: %v", opts) +} diff --git a/pkg/provider/prometheusProvider/prometheus.go b/pkg/provider/prometheusProvider/prometheus.go index 59a8e1941..dee50c938 100644 --- a/pkg/provider/prometheusProvider/prometheus.go +++ b/pkg/provider/prometheusProvider/prometheus.go @@ -10,6 +10,7 @@ import ( "k8s.io/client-go/tools/clientcmd" "net/http" "net/url" + "os" basecmd "sigs.k8s.io/custom-metrics-apiserver/pkg/cmd" prom "sigs.k8s.io/prometheus-adapter/pkg/client" cfg "sigs.k8s.io/prometheus-adapter/pkg/config" @@ -34,10 +35,16 @@ type AlibabaMetricsAdapterOptions struct { PrometheusAuthConf string // PrometheusCAFile points to the file containing the ca-root for connecting with Prometheus PrometheusCAFile string + // PrometheusClientTLSCertFile points to the file containing the client TLS cert for connecting with Prometheus + PrometheusClientTLSCertFile string + // PrometheusClientTLSKeyFile points to the file containing the client TLS key for connecting with Prometheus + PrometheusClientTLSKeyFile string // PrometheusTokenFile points to the file that contains the bearer token when connecting with Prometheus PrometheusTokenFile string // PrometheusHeaders is a k=v list of headers to set on requests to PrometheusURL PrometheusHeaders []string + // PrometheusVerb is a verb to set on requests to PrometheusURL + PrometheusVerb string // AdapterConfigFile points to the file containing the metrics discovery configuration. AdapterConfigFile string // MetricsRelistInterval is the interval at which to relist the set of available metrics @@ -59,10 +66,16 @@ func (cmd *AlibabaMetricsAdapterOptions) AddFlags() { "kubeconfig file used to configure auth when connecting to Prometheus.") cmd.Flags().StringVar(&cmd.PrometheusCAFile, "prometheus-ca-file", cmd.PrometheusCAFile, "Optional CA file to use when connecting with Prometheus") + cmd.Flags().StringVar(&cmd.PrometheusClientTLSCertFile, "prometheus-client-tls-cert-file", cmd.PrometheusClientTLSCertFile, + "Optional client TLS cert file to use when connecting with Prometheus, auto-renewal is not supported") + cmd.Flags().StringVar(&cmd.PrometheusClientTLSKeyFile, "prometheus-client-tls-key-file", cmd.PrometheusClientTLSKeyFile, + "Optional client TLS key file to use when connecting with Prometheus, auto-renewal is not supported") cmd.Flags().StringVar(&cmd.PrometheusTokenFile, "prometheus-token-file", cmd.PrometheusTokenFile, "Optional file containing the bearer token to use when connecting with Prometheus") cmd.Flags().StringArrayVar(&cmd.PrometheusHeaders, "prometheus-header", cmd.PrometheusHeaders, "Optional header to set on requests to prometheus-url. Can be repeated") + cmd.Flags().StringVar(&cmd.PrometheusVerb, "prometheus-verb", cmd.PrometheusVerb, + "HTTP verb to set on requests to Prometheus. Possible values: \"GET\", \"POST\"") cmd.Flags().StringVar(&cmd.AdapterConfigFile, "config", cmd.AdapterConfigFile, "Configuration file containing details of how to transform between Prometheus metrics "+ "and custom metrics API resources") @@ -94,9 +107,10 @@ func (cmd *AlibabaMetricsAdapterOptions) MakePromClient() (prom.Client, error) { return nil, fmt.Errorf("invalid Prometheus URL %q: %v", baseURL, err) } + // prom client http auth var httpClient *http.Client if cmd.PrometheusCAFile != "" { - prometheusCAClient, err := makePrometheusCAClient(cmd.PrometheusCAFile, cmd.PrometheusInsecure) + prometheusCAClient, err := makePrometheusCAClient(cmd.PrometheusCAFile, cmd.PrometheusClientTLSCertFile, cmd.PrometheusClientTLSKeyFile) if err != nil { return nil, err } @@ -111,7 +125,7 @@ func (cmd *AlibabaMetricsAdapterOptions) MakePromClient() (prom.Client, error) { klog.Info("successfully using in-cluster auth") } else { // return the default client if we're using no auth - //httpClient = http.DefaultClient + // httpClient = http.DefaultClient httpClient = &http.Client{ Transport: &http.Transport{ TLSClientConfig: &tls.Config{ @@ -122,6 +136,7 @@ func (cmd *AlibabaMetricsAdapterOptions) MakePromClient() (prom.Client, error) { klog.Infof("successfully using default http client auth. InsecureSkipVerify: %v", cmd.PrometheusInsecure) } + // prom client token auth if cmd.PrometheusTokenFile != "" { data, err := ioutil.ReadFile(cmd.PrometheusTokenFile) if err != nil { @@ -130,13 +145,14 @@ func (cmd *AlibabaMetricsAdapterOptions) MakePromClient() (prom.Client, error) { httpClient.Transport = transport.NewBearerAuthRoundTripper(string(data), httpClient.Transport) } + // prom client http header genericPromClient := prom.NewGenericAPIClient(httpClient, baseURL, parseHeaderArgs(cmd.PrometheusHeaders)) instrumentedGenericPromClient := utils.InstrumentGenericAPIClient(genericPromClient, baseURL.String()) return prom.NewClientForAPI(instrumentedGenericPromClient), nil } -func makePrometheusCAClient(caFilename string, insecure bool) (*http.Client, error) { - data, err := ioutil.ReadFile(caFilename) +func makePrometheusCAClient(caFilePath string, tlsCertFilePath string, tlsKeyFilePath string) (*http.Client, error) { + data, err := os.ReadFile(caFilePath) if err != nil { return nil, fmt.Errorf("failed to read prometheus-ca-file: %v", err) } @@ -146,11 +162,27 @@ func makePrometheusCAClient(caFilename string, insecure bool) (*http.Client, err return nil, fmt.Errorf("no certs found in prometheus-ca-file") } + if (tlsCertFilePath != "") && (tlsKeyFilePath != "") { + tlsClientCerts, err := tls.LoadX509KeyPair(tlsCertFilePath, tlsKeyFilePath) + if err != nil { + return nil, fmt.Errorf("failed to read TLS key pair: %v", err) + } + return &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{ + RootCAs: pool, + Certificates: []tls.Certificate{tlsClientCerts}, + MinVersion: tls.VersionTLS12, + }, + }, + }, nil + } + return &http.Client{ Transport: &http.Transport{ TLSClientConfig: &tls.Config{ - RootCAs: pool, - InsecureSkipVerify: insecure, + RootCAs: pool, + MinVersion: tls.VersionTLS12, }, }, }, nil diff --git a/pkg/provider/prometheusProvider/prometheus_test.go b/pkg/provider/prometheusProvider/prometheus_test.go new file mode 100644 index 000000000..092fe7f4b --- /dev/null +++ b/pkg/provider/prometheusProvider/prometheus_test.go @@ -0,0 +1,37 @@ +package prometheusProvider + +import ( + "context" + pmodel "github.com/prometheus/common/model" + prom "sigs.k8s.io/prometheus-adapter/pkg/client" + "testing" +) + +func TestMakePromClient(t *testing.T) { + + promUrl := "http://ack-prometheus-operator-prometheus.monitoring.svc:9090" + armsPromAuthToken := "testAuthToken" + + // input param + cmdOpts := NewAlibabaMetricsAdapterOptions() + cmdOpts.PrometheusURL = promUrl + cmdOpts.PrometheusHeaders = []string{ + "Authorization=" + armsPromAuthToken, + } + + // + promClient, createClientErr := cmdOpts.MakePromClient() + if createClientErr != nil { + t.Fatalf("failed create prom client.") + } + + queryPromQL := prom.Selector("up") + + queryResult, queryErr := promClient.Query(context.TODO(), pmodel.Now(), queryPromQL) + if queryErr != nil { + t.Fatalf("failed to query prom") + } + + t.Logf("prom query resutl := %v", queryResult) + +} diff --git a/vendor/k8s.io/metrics/pkg/client/external_metrics/client.go b/vendor/k8s.io/metrics/pkg/client/external_metrics/client.go new file mode 100644 index 000000000..d18d9e124 --- /dev/null +++ b/vendor/k8s.io/metrics/pkg/client/external_metrics/client.go @@ -0,0 +1,100 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package external_metrics + +import ( + "context" + "fmt" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "k8s.io/client-go/util/flowcontrol" + "k8s.io/metrics/pkg/apis/external_metrics/v1beta1" +) + +type externalMetricsClient struct { + client rest.Interface +} + +func New(client rest.Interface) ExternalMetricsClient { + return &externalMetricsClient{ + client: client, + } +} + +func NewForConfig(c *rest.Config) (ExternalMetricsClient, error) { + configShallowCopy := *c + if configShallowCopy.RateLimiter == nil && configShallowCopy.QPS > 0 { + if configShallowCopy.Burst <= 0 { + return nil, fmt.Errorf("burst is required to be greater than 0 when RateLimiter is not set and QPS is set to greater than 0") + } + configShallowCopy.RateLimiter = flowcontrol.NewTokenBucketRateLimiter(configShallowCopy.QPS, configShallowCopy.Burst) + } + configShallowCopy.APIPath = "/apis" + if configShallowCopy.UserAgent == "" { + configShallowCopy.UserAgent = rest.DefaultKubernetesUserAgent() + } + configShallowCopy.GroupVersion = &v1beta1.SchemeGroupVersion + configShallowCopy.NegotiatedSerializer = scheme.Codecs.WithoutConversion() + + client, err := rest.RESTClientFor(&configShallowCopy) + if err != nil { + return nil, err + } + + return New(client), nil +} + +func NewForConfigOrDie(c *rest.Config) ExternalMetricsClient { + client, err := NewForConfig(c) + if err != nil { + panic(err) + } + return client +} + +func (c *externalMetricsClient) NamespacedMetrics(namespace string) MetricsInterface { + return &namespacedMetrics{ + client: c, + namespace: namespace, + } +} + +type namespacedMetrics struct { + client *externalMetricsClient + namespace string +} + +func (m *namespacedMetrics) List(metricName string, metricSelector labels.Selector) (*v1beta1.ExternalMetricValueList, error) { + res := &v1beta1.ExternalMetricValueList{} + err := m.client.client.Get(). + Namespace(m.namespace). + Resource(metricName). + VersionedParams(&metav1.ListOptions{ + LabelSelector: metricSelector.String(), + }, metav1.ParameterCodec). + Do(context.TODO()). + Into(res) + + if err != nil { + return nil, err + } + + return res, nil +} diff --git a/vendor/k8s.io/metrics/pkg/client/external_metrics/interfaces.go b/vendor/k8s.io/metrics/pkg/client/external_metrics/interfaces.go new file mode 100644 index 000000000..d98c161aa --- /dev/null +++ b/vendor/k8s.io/metrics/pkg/client/external_metrics/interfaces.go @@ -0,0 +1,40 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package external_metrics + +import ( + "k8s.io/apimachinery/pkg/labels" + "k8s.io/metrics/pkg/apis/external_metrics/v1beta1" +) + +// ExternalMetricsClient is a client for fetching external metrics. +type ExternalMetricsClient interface { + NamespacedMetricsGetter +} + +// NamespacedMetricsGetter provides access to an interface for fetching +// metrics in a particular namespace. +type NamespacedMetricsGetter interface { + NamespacedMetrics(namespace string) MetricsInterface +} + +// MetricsInterface provides access to external metrics. +type MetricsInterface interface { + // List fetches the metric for the given namespace that maches the given + // metricSelector. + List(metricName string, metricSelector labels.Selector) (*v1beta1.ExternalMetricValueList, error) +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 2e739653b..e901d2db3 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -919,13 +919,6 @@ k8s.io/metrics/pkg/apis/custom_metrics/v1beta2 k8s.io/metrics/pkg/apis/external_metrics k8s.io/metrics/pkg/apis/external_metrics/install k8s.io/metrics/pkg/apis/external_metrics/v1beta1 -k8s.io/metrics/pkg/apis/metrics -k8s.io/metrics/pkg/apis/metrics/v1alpha1 -k8s.io/metrics/pkg/apis/metrics/v1beta1 -k8s.io/metrics/pkg/client/clientset/versioned -k8s.io/metrics/pkg/client/clientset/versioned/scheme -k8s.io/metrics/pkg/client/clientset/versioned/typed/metrics/v1alpha1 -k8s.io/metrics/pkg/client/clientset/versioned/typed/metrics/v1beta1 k8s.io/metrics/pkg/client/external_metrics # k8s.io/utils v0.0.0-20210802155522-efc7438f0176 k8s.io/utils/buffer