diff --git a/Makefile b/Makefile index ec4df20..3bb9eed 100644 --- a/Makefile +++ b/Makefile @@ -36,11 +36,11 @@ vet: .PHONY: build-controller build-controller: - docker build --pull --build-arg ARCH=$(ARCH) --build-arg LDFLAGS="$(LDFLAGS)" -f images/$(CONTROLLER_NAME)/Dockerfile . -t $(CONTROLLER_IMG)-$(ARCH):$(TAG) + docker build --build-arg ARCH=$(ARCH) --build-arg LDFLAGS="$(LDFLAGS)" -f images/$(CONTROLLER_NAME)/Dockerfile . -t $(CONTROLLER_IMG)-$(ARCH):$(TAG) .PHONY: build-approver build-approver: - docker build --pull --build-arg ARCH=$(ARCH) --build-arg LDFLAGS="$(LDFLAGS)" -f images/$(APPROVER_NAME)/Dockerfile . -t $(APPROVER_IMG)-$(ARCH):$(TAG) + docker build --build-arg ARCH=$(ARCH) --build-arg LDFLAGS="$(LDFLAGS)" -f images/$(APPROVER_NAME)/Dockerfile . -t $(APPROVER_IMG)-$(ARCH):$(TAG) .PHONY: docker-build docker-build: build-controller build-approver diff --git a/cmd/pve-csr-approver-manager/main.go b/cmd/pve-csr-approver-manager/main.go index 7fcc6eb..80391c4 100644 --- a/cmd/pve-csr-approver-manager/main.go +++ b/cmd/pve-csr-approver-manager/main.go @@ -1,104 +1,220 @@ package main import ( + "encoding/json" + "errors" "flag" + "io/ioutil" + "math/rand" "os" - "os/signal" - "syscall" "time" - certificatesv1beta1 "k8s.io/api/certificates/v1beta1" - "k8s.io/client-go/informers" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" - "k8s.io/client-go/tools/cache" + "k8s.io/apimachinery/pkg/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + typedcertificatesv1beta1 "k8s.io/client-go/kubernetes/typed/certificates/v1beta1" "k8s.io/klog/klogr" "k8s.io/klog/v2" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/healthz" "git.giftfish.de/ston1th/cloud-controller-manager-pve/pkg/approver" pve "git.giftfish.de/ston1th/pve-go" ) +var ( + scheme = runtime.NewScheme() + setupLog = ctrl.Log.WithName("setup") +) + +func init() { + _ = clientgoscheme.AddToScheme(scheme) +} + func main() { + rand.Seed(time.Now().UnixNano()) klog.InitFlags(nil) + + var ( + leaderElect bool + healthAddr string + cloudConfig string + csrConcurrency int + syncPeriod time.Duration + ) + + flag.BoolVar(&leaderElect, "leader-elect", true, + "Enable leader election for controller manager. "+ + "Enabling this will ensure there is only one active controller manager.") + flag.StringVar(&cloudConfig, + "cloud-config", + "", + "The path to the cloud provider configuration file. Empty string for no configuration file.", + ) + flag.StringVar(&healthAddr, + "health-addr", + ":9449", + "The address the health endpoint binds to.", + ) + flag.IntVar(&csrConcurrency, + "csr-concurrency", + 1, + "Number of CSRs to process simultaneously", + ) + flag.DurationVar(&syncPeriod, + "sync-period", + 10*time.Minute, + "The minimum interval at which watched resources are reconciled (e.g. 15m)", + ) flag.Parse() - log := klogr.New() - opts, err := pve.ClientOptionsFromEnv() + ctrl.SetLogger(klogr.New()) + + rcfg := ctrl.GetConfigOrDie() + + mgr, err := ctrl.NewManager(rcfg, ctrl.Options{ + Scheme: scheme, + LeaderElection: leaderElect, + LeaderElectionID: "pve-csr-approver-manager", + HealthProbeBindAddress: healthAddr, + SyncPeriod: &syncPeriod, + }) if err != nil { - log.Error(err, "could not create pve client") + setupLog.Error(err, "unable to start manager") os.Exit(1) } - pveClient := pve.NewClient(opts...) - - cfg, err := rest.InClusterConfig() - if err != nil { - log.Error(err, "could not configure kubernetes client") - os.Exit(1) - } - - cl, err := kubernetes.NewForConfig(cfg) - if err != nil { - log.Error(err, "could not create kubernetes client") - os.Exit(1) - } - - factory := informers.NewSharedInformerFactory(cl, time.Second*30) - certInformerV1beta1 := factory.Certificates().V1beta1().CertificateSigningRequests().Informer() - fV1beta1 := func(obj interface{}) { - if req, ok := obj.(*certificatesv1beta1.CertificateSigningRequest); ok { - if err := approver.ApproveV1beta1(log, pveClient, cl.CertificatesV1beta1().CertificateSigningRequests(), req); err != nil { - log.Error(err, "csr approval failed", "name", req.ObjectMeta.Name) - return + var opts []pve.ClientOption + config, err := os.Open(cloudConfig) + if err == nil { + buf, _ := ioutil.ReadAll(config) + if len(buf) != 0 { + var cfg pve.Config + err = json.Unmarshal(buf, &cfg) + if err != nil { + setupLog.Error(err, "failed to read config file") + os.Exit(1) + } + opts, err = pve.ClientOptionsFromConfig(cfg) + if err != nil { + setupLog.Error(err, "failed to create client options from file") + os.Exit(1) } } + } else { + opts, err = pve.ClientOptionsFromEnv() + if err != nil { + setupLog.Error(err, "failed to create client options from env") + os.Exit(1) + } } - certInformerV1beta1.AddEventHandler(cache.ResourceEventHandlerFuncs{ - AddFunc: func(obj interface{}) { - fV1beta1(obj) - }, - UpdateFunc: func(_, obj interface{}) { - fV1beta1(obj) - }, - }) - stop := make(chan struct{}) - factory.Start(stop) + if len(opts) == 0 { + setupLog.Error(errors.New("missing pve config"), "") + os.Exit(1) + } + pveClient := pve.NewClient(opts...) - //watchListV1 := cache.NewListWatchFromClient( - // cl.CertificatesV1Client.RESTClient(), - // "certificatesigningrequests", - // v1.NamespaceAll, - // fields.Everything(), - //) - //fV1 := func(obj interface{}) { - // if req, ok := obj.(*certificates.CertificateSigningRequest); ok { - // if err := approver.ApproveV1(pveClient, cl.CertificatesV1Client.CertificateSigningRequests(), req); err != nil { - // log.Error(err, "csr approval failed", "name", req.ObjectMeta.Name) - // return - // } - // log.Info("csr approval successful", "name", req.ObjectMeta.Name) - // } - //} - //_, controllerV1 := cache.NewInformer( - // watchListV1, - // &certificates.CertificateSigningRequest{}, - // time.Second*30, - // cache.ResourceEventHandlerFuncs{ - // AddFunc: func(obj interface{}) { - // fV1(obj) - // }, - // UpdateFunc: func(_, obj interface{}) { - // fV1(obj) - // }, - // }, - //) - //stopV1 := make(chan struct{}) - //go controllerV1.Run(stopV1) - log.Info("csr approver started") + if err = (&approver.CSRReconciler{ + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controllers").WithName("CSR"), + Scheme: mgr.GetScheme(), + Recorder: mgr.GetEventRecorderFor("pvecluster-controller"), + PVEClient: pveClient, + CSRClient: typedcertificatesv1beta1.NewForConfigOrDie(rcfg), + }).SetupWithManager(mgr, controller.Options{MaxConcurrentReconciles: csrConcurrency}); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "CSR") + os.Exit(1) + } + if err := mgr.AddReadyzCheck("ping", healthz.Ping); err != nil { + setupLog.Error(err, "unable to create ready check") + os.Exit(1) + } - sigs := make(chan os.Signal) - signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM) - <-sigs - close(stop) - log.Info("csr approver stopped") + if err := mgr.AddHealthzCheck("ping", healthz.Ping); err != nil { + setupLog.Error(err, "unable to create health check") + os.Exit(1) + } + + setupLog.Info("starting manager") + if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { + setupLog.Error(err, "problem running manager") + os.Exit(1) + } + /* + cfg, err := rest.InClusterConfig() + if err != nil { + log.Error(err, "could not configure kubernetes client") + os.Exit(1) + } + + cl, err := kubernetes.NewForConfig(cfg) + if err != nil { + log.Error(err, "could not create kubernetes client") + os.Exit(1) + } + + id, err := os.Hostname() + if err != nil { + log.Error(err, "could not get hostname") + os.Exit(1) + } + lock := &resourcelock.LeaseLock{ + LeaseMeta: metav1.ObjectMeta{ + Name: csrLock, + Namespace: namespace, + }, + Client: cl.CoordinationV1(), + LockConfig: resourcelock.ResourceLockConfig{ + Identity: id, + }, + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + leaderelection.RunOrDie(ctx, leaderelection.LeaderElectionConfig{ + Lock: lock, + ReleaseOnCancel: true, + LeaseDuration: time.Duration(leaseDuration) * time.Second, + RenewDeadline: time.Duration(renewDeadline) * time.Second, + RetryPeriod: time.Duration(retryPeriod) * time.Second, + Callbacks: leaderelection.LeaderCallbacks{ + OnStartedLeading: func(ctx context.Context) { + log.Info("Leader election successful") + factory := informers.NewSharedInformerFactory(cl, time.Second*30) + certInformerV1beta1 := factory.Certificates().V1beta1().CertificateSigningRequests().Informer() + fV1beta1 := func(obj interface{}) { + if req, ok := obj.(*certificatesv1beta1.CertificateSigningRequest); ok { + if err := approver.ApproveV1beta1(log, pveClient, cl.CertificatesV1beta1().CertificateSigningRequests(), req); err != nil { + log.Error(err, "csr approval failed", "name", req.ObjectMeta.Name) + return + } + } + } + certInformerV1beta1.AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { + fV1beta1(obj) + }, + UpdateFunc: func(_, obj interface{}) { + fV1beta1(obj) + }, + }) + stop := make(chan struct{}) + factory.Start(stop) + <-ctx.Done() + close(stop) + }, + OnStoppedLeading: func() { + cancel() + log.Info("Leader election lost") + }, + }}) + + log.Info("csr approver started") + + sigs := make(chan os.Signal) + signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM) + <-sigs + cancel() + log.Info("csr approver stopped") + */ } diff --git a/deploy/pve-cloud-controller-manager.yaml b/deploy/pve-cloud-controller-manager.yaml index 9344c5b..e694df8 100644 --- a/deploy/pve-cloud-controller-manager.yaml +++ b/deploy/pve-cloud-controller-manager.yaml @@ -28,6 +28,14 @@ rules: - patch - update - watch +- apiGroups: [""] + resources: ["endpoints"] + verbs: + - create + - get + - list + - watch + - update - apiGroups: ["coordination.k8s.io"] resources: ["leases"] verbs: @@ -62,67 +70,61 @@ subjects: name: cloud-controller-manager namespace: kube-system --- -apiVersion: apps/v1 -kind: Deployment +apiVersion: v1 +kind: Pod metadata: name: pve-cloud-controller-manager namespace: kube-system labels: app: pve-cloud-controller-manager spec: - replicas: 1 - selector: - matchLabels: - app: pve-cloud-controller-manager - template: - metadata: - labels: - app: pve-cloud-controller-manager - spec: - priorityClassName: system-node-critical - containers: - - args: - - --cloud-provider=pve - - --leader-elect=false - env: - - name: PVE_API - valueFrom: - secretKeyRef: - key: api - name: pvecloud - - name: PVE_USER - valueFrom: - secretKeyRef: - key: user - name: pvecloud - - name: PVE_PASSWORD - valueFrom: - secretKeyRef: - key: password - name: pvecloud - image: reg.k8s.giftfish.de/library/pve-cloud-controller-manager-amd64:dev - imagePullPolicy: Always - livenessProbe: - httpGet: - path: /healthz - port: 10258 - scheme: HTTPS - initialDelaySeconds: 3 - periodSeconds: 3 - name: manager - resources: - requests: - cpu: 100m - memory: 50Mi - dnsPolicy: Default - serviceAccountName: cloud-controller-manager - tolerations: - - effect: NoSchedule - key: node.cloudprovider.kubernetes.io/uninitialized - value: "true" - - key: CriticalAddonsOnly - operator: Exists - - effect: NoSchedule - key: node-role.kubernetes.io/master - - effect: NoSchedule - key: node.kubernetes.io/not-ready + priorityClassName: system-node-critical + #hostNetwork: true + containers: + - args: + - --cloud-config=/etc/kubernetes/pve.json + - --tls-min-version=VersionTLS12 + - --tls-cipher-suites="TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256" + - --cloud-provider=pve + - --leader-elect=true + image: reg.k8s.giftfish.de/library/pve-cloud-controller-manager-amd64:dev + imagePullPolicy: Always + livenessProbe: + httpGet: + path: /healthz + port: 10258 + scheme: HTTPS + initialDelaySeconds: 3 + periodSeconds: 3 + name: manager + resources: + requests: + cpu: 100m + memory: 50Mi + volumeMounts: + - name: etc-kubernetes + mountPath: /etc/kubernetes + readOnly: true + - name: etc-ssl + mountPath: /etc/ssl + readOnly: true + volumes: + - name: etc-kubernetes + hostPath: + path: /etc/kubernetes + - name: etc-ssl + hostPath: + path: /etc/ssl + dnsPolicy: Default + serviceAccountName: cloud-controller-manager + tolerations: + - effect: NoSchedule + key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + - key: CriticalAddonsOnly + operator: Exists + - effect: NoSchedule + key: node-role.kubernetes.io/master + - effect: NoSchedule + key: node.kubernetes.io/not-ready + diff --git a/deploy/pve-csr-approver-manager.yaml b/deploy/pve-csr-approver-manager.yaml index 6409e71..bac047b 100644 --- a/deploy/pve-csr-approver-manager.yaml +++ b/deploy/pve-csr-approver-manager.yaml @@ -26,6 +26,12 @@ rules: verbs: - create - update +- apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: + - get + - create + - update --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 @@ -40,57 +46,56 @@ subjects: namespace: kube-system name: pve-csr-approver --- -apiVersion: apps/v1 -kind: Deployment +apiVersion: v1 +kind: Pod metadata: name: pve-csr-approver-manager namespace: kube-system labels: app: pve-csr-approver-manager spec: - replicas: 1 - selector: - matchLabels: - app: pve-csr-approver-manager - template: - metadata: - labels: - app: pve-csr-approver-manager - spec: - priorityClassName: system-node-critical - containers: - - name: manager - env: - - name: PVE_API - valueFrom: - secretKeyRef: - key: api - name: pvecloud - - name: PVE_USER - valueFrom: - secretKeyRef: - key: user - name: pvecloud - - name: PVE_PASSWORD - valueFrom: - secretKeyRef: - key: password - name: pvecloud - image: reg.k8s.giftfish.de/library/pve-csr-approver-manager-amd64:dev - imagePullPolicy: Always - resources: - requests: - cpu: 100m - memory: 50Mi - dnsPolicy: Default - serviceAccountName: pve-csr-approver - tolerations: - - effect: NoSchedule - key: node.cloudprovider.kubernetes.io/uninitialized - value: "true" - - key: CriticalAddonsOnly - operator: Exists - - effect: NoSchedule - key: node-role.kubernetes.io/master - - effect: NoSchedule - key: node.kubernetes.io/not-ready + priorityClassName: system-node-critical + #hostNetwork: true + containers: + - args: + - --cloud-config=/etc/kubernetes/pve.json + - --leader-elect=true + name: manager + image: reg.k8s.giftfish.de/library/pve-csr-approver-manager-amd64:dev + imagePullPolicy: Always + livenessProbe: + httpGet: + path: /healthz + port: 9449 + initialDelaySeconds: 3 + periodSeconds: 3 + resources: + requests: + cpu: 100m + memory: 50Mi + volumeMounts: + - name: etc-kubernetes + mountPath: /etc/kubernetes + readOnly: true + - name: etc-ssl + mountPath: /etc/ssl + readOnly: true + volumes: + - name: etc-kubernetes + hostPath: + path: /etc/kubernetes + - name: etc-ssl + hostPath: + path: /etc/ssl + dnsPolicy: Default + serviceAccountName: pve-csr-approver + tolerations: + - effect: NoSchedule + key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + - key: CriticalAddonsOnly + operator: Exists + - effect: NoSchedule + key: node-role.kubernetes.io/master + - effect: NoSchedule + key: node.kubernetes.io/not-ready diff --git a/go.mod b/go.mod index 74f1882..c0edf58 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module git.giftfish.de/ston1th/cloud-controller-manager-pve go 1.13 require ( - git.giftfish.de/ston1th/pve-go v0.0.0-20201031113657-271c91eecbeb + git.giftfish.de/ston1th/pve-go v0.0.0-20201031205459-928086aea9c1 github.com/go-logr/logr v0.2.1 github.com/hashicorp/golang-lru v0.5.4 // indirect github.com/imdario/mergo v0.3.9 // indirect @@ -16,6 +16,7 @@ require ( k8s.io/klog v1.0.0 k8s.io/klog/v2 v2.4.0 k8s.io/kubernetes v1.19.3 + sigs.k8s.io/controller-runtime v0.6.3 ) replace ( diff --git a/go.sum b/go.sum index ee5c96d..b608c40 100644 --- a/go.sum +++ b/go.sum @@ -14,6 +14,8 @@ cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiy dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= git.giftfish.de/ston1th/pve-go v0.0.0-20201031113657-271c91eecbeb h1:4tRFUX6/KiJEjVfIBIdekpXRHCMG0G1ym4YHnDTTo9U= git.giftfish.de/ston1th/pve-go v0.0.0-20201031113657-271c91eecbeb/go.mod h1:WOrvThayhaB89KZwQqGzqcTD9nVeTsnaRB1vBaeb2tM= +git.giftfish.de/ston1th/pve-go v0.0.0-20201031205459-928086aea9c1 h1:DB8S2p46uptQn2dkfb1RC28tzOIED7bYbt+UHQWwEbQ= +git.giftfish.de/ston1th/pve-go v0.0.0-20201031205459-928086aea9c1/go.mod h1:WOrvThayhaB89KZwQqGzqcTD9nVeTsnaRB1vBaeb2tM= github.com/Azure/azure-sdk-for-go v43.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 h1:w+iIsaOQNcT7OZ575w+acHgRric5iCyQh+xv+KJ4HB8= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= @@ -157,6 +159,7 @@ github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/euank/go-kmsg-parser v2.0.0+incompatible/go.mod h1:MhmAMZ8V4CYH4ybgdRwPr2TU5ThnS43puaKEMpja1uw= +github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v4.9.0+incompatible h1:kLcOMZeuLAJvL2BPWLMIj5oaZQobrkAqrL+WFZwQses= github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4= @@ -186,6 +189,7 @@ github.com/go-logr/logr v0.2.0 h1:QvGt2nLcHH0WK9orKa+ppBPAxREcH364nPUedEpK0TY= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v0.2.1 h1:fV3MLmabKIZ383XifUjFSwcoGee0v9qgPp8wy5svibE= github.com/go-logr/logr v0.2.1/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= +github.com/go-logr/zapr v0.1.0/go.mod h1:tabnROwaDl0UNxkVeFRbY8bwB37GwRv0P8lg6aAiEnk= github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70tL6pCuVxPJOHXQ+wIac1FUrvNkHolPie/cLEU6hI= github.com/go-openapi/analysis v0.17.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= github.com/go-openapi/analysis v0.18.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= @@ -293,6 +297,7 @@ github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/gnostic v0.3.1/go.mod h1:on+2t9HRStVgn95RSsFWFz+6Q0Snyqv1awfrALZdbtU= github.com/googleapis/gnostic v0.4.1 h1:DLJCy1n/vrD4HPjOvYcT8aYQXpPIzoRZONaYwyycI+I= github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8= @@ -482,6 +487,7 @@ github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2 h1:6LJUbpNm42llc4HRCuvApCSWB/WfhuNo9K98Q9sNGfs= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.11/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.1.3 h1:F0+tqvhOksq22sc6iCHF5WGlWjdwj92p0udFh1VFBS8= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= @@ -746,6 +752,8 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gomodules.xyz/jsonpatch/v2 v2.0.1 h1:xyiBuvkD2g5n7cYzx6u2sxQvsAy4QJsZFCzGVdzOXZ0= +gomodules.xyz/jsonpatch/v2 v2.0.1/go.mod h1:IhYNNY4jnS53ZnfE4PAmpKtDpTCj1JFXc+3mwe7XcUU= gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= gonum.org/v1/gonum v0.0.0-20190331200053-3d26580ed485/go.mod h1:2ltnJ7xHfj0zHS40VVPYEAAMTa3ZGguvHGBSJeRWqE0= gonum.org/v1/gonum v0.6.2/go.mod h1:9mxDZsDKxgMAuccQkewq682L+0eCu4dCN2yonUJTCLU= @@ -841,6 +849,7 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= k8s.io/api v0.19.3 h1:GN6ntFnv44Vptj/b+OnMW7FmzkpDoIDLZRvKX3XH9aU= k8s.io/api v0.19.3/go.mod h1:VF+5FT1B74Pw3KxMdKyinLo+zynBaMBiAfGMuldcNDs= +k8s.io/apiextensions-apiserver v0.19.3 h1:WZxBypSHW4SdXHbdPTS/Jy7L2la6Niggs8BuU5o+avo= k8s.io/apiextensions-apiserver v0.19.3/go.mod h1:igVEkrE9TzInc1tYE7qSqxaLg/rEAp6B5+k9Q7+IC8Q= k8s.io/apimachinery v0.19.3 h1:bpIQXlKjB4cB/oNpnNnV+BybGPR7iP5oYpsOTEJ4hgc= k8s.io/apimachinery v0.19.3/go.mod h1:DnPGDnARWFvYa3pMHgSxtbZb7gpzzAZ1pTfaUNDVlmA= @@ -884,6 +893,7 @@ k8s.io/metrics v0.19.3/go.mod h1:Eap/Lk1FiAIjkaArFuv41v+ph6dbDpVGwAg7jMI+4vg= k8s.io/sample-apiserver v0.19.3/go.mod h1:lCFCWx71mTeqnj/DioYoSGwC0Ce/KSPFtvt41OKpcMQ= k8s.io/system-validators v1.1.2/go.mod h1:bPldcLgkIUK22ALflnsXk8pvkTEndYdNuaHH6gRrl0Q= k8s.io/utils v0.0.0-20200414100711-2df71ebbae66/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/utils v0.0.0-20200603063816-c1c6865ac451/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20200729134348-d5654de09c73 h1:uJmqzgNWG7XyClnU/mLPBWwfKKF1K8Hf8whTseBgJcg= k8s.io/utils v0.0.0-20200729134348-d5654de09c73/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= modernc.org/cc v1.0.0/go.mod h1:1Sk4//wdnYJiUIxnW8ddKpaOJCF37yAdqYnkxUpaYxw= @@ -895,6 +905,8 @@ rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8 rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.9 h1:rusRLrDhjBp6aYtl9sGEvQJr6faoHoDLd0YcUBTZguI= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.9/go.mod h1:dzAXnQbTRyDlZPJX2SUPEqvnB+j7AJjtlox7PEwigU0= +sigs.k8s.io/controller-runtime v0.6.3 h1:SBbr+inLPEKhvlJtrvDcwIpm+uhDvp63Bl72xYJtoOE= +sigs.k8s.io/controller-runtime v0.6.3/go.mod h1:WlZNXcM0++oyaQt4B7C2lEE5JYRs8vJUzRP4N4JpdAY= sigs.k8s.io/kustomize v2.0.3+incompatible/go.mod h1:MkjgH3RdOWrievjo6c9T245dYlB5QeXV4WCbnt/PEpU= sigs.k8s.io/structured-merge-diff/v4 v4.0.1 h1:YXTMot5Qz/X1iBRJhAt+vI+HVttY0WkSqqhKxQ0xVbA= sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= diff --git a/pkg/approver/approver.go b/pkg/approver/approver.go index 4278f8f..5028e92 100644 --- a/pkg/approver/approver.go +++ b/pkg/approver/approver.go @@ -8,17 +8,98 @@ import ( pve "git.giftfish.de/ston1th/pve-go" - //"k8s.io/client-go/kubernetes/typed/certificates/v1" "github.com/go-logr/logr" certificatesv1beta1 "k8s.io/api/certificates/v1beta1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/client-go/kubernetes/typed/certificates/v1beta1" + typedcertificatesv1beta1 "k8s.io/client-go/kubernetes/typed/certificates/v1beta1" + "k8s.io/client-go/tools/record" "k8s.io/kubernetes/pkg/apis/certificates" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" ) var ErrNoKubeletServingCSR = errors.New("CSR is no kubelet serving certificate") +type CSRReconciler struct { + client.Client + Recorder record.EventRecorder + Log logr.Logger + Scheme *runtime.Scheme + PVEClient *pve.Client + CSRClient *typedcertificatesv1beta1.CertificatesV1beta1Client +} + +func (r *CSRReconciler) SetupWithManager(mgr ctrl.Manager, options controller.Options) error { + return ctrl.NewControllerManagedBy(mgr). + WithOptions(options). + For(&certificatesv1beta1.CertificateSigningRequest{}). + Complete(r) +} + +func (r *CSRReconciler) Reconcile(req ctrl.Request) (res ctrl.Result, err error) { + ctx := context.Background() + log := r.Log + csr := &certificatesv1beta1.CertificateSigningRequest{} + if err = r.Get(ctx, req.NamespacedName, csr); err != nil { + if apierrors.IsNotFound(err) { + err = nil + return + } + return + } + if !csr.DeletionTimestamp.IsZero() { + return + } + err = r.approve(ctx, log, csr) + return +} + +var conditionv1beta1 = certificatesv1beta1.CertificateSigningRequestCondition{ + Type: certificatesv1beta1.CertificateApproved, + Reason: "AutoApproved", + Message: "Auto approving kubelet client certificate in PVE Cluster", +} + +func (r *CSRReconciler) approve(ctx context.Context, log logr.Logger, request *certificatesv1beta1.CertificateSigningRequest) error { + if len(request.Status.Conditions) > 0 { + return nil + } + client := r.CSRClient.CertificateSigningRequests() + err := checkCSR(ctx, r.PVEClient, request) + if err == ErrNoKubeletServingCSR { + return nil + } + if err != nil { + return err + } + + for { + if len(request.Status.Conditions) > 0 { + return nil + } + request.Status.Conditions = append(request.Status.Conditions, conditionv1beta1) + // Submit the updated CSR. + if _, err := client.UpdateApproval(ctx, request, metav1.UpdateOptions{}); err != nil { + if strings.Contains(err.Error(), "the object has been modified") { + // The CSR might have been updated by a third-party, retry until we + // succeed. + request, err = client.Get(ctx, request.ObjectMeta.Name, metav1.GetOptions{}) + if err != nil { + return err + } + continue + } + return err + } + log.Info("csr approval successful", "name", request.ObjectMeta.Name) + return nil + } +} + func checkCSR(ctx context.Context, pveClient *pve.Client, request *certificatesv1beta1.CertificateSigningRequest) error { req, err := certificates.ParseCSR(request.Spec.Request) if err != nil { @@ -85,76 +166,3 @@ func checkCSR(ctx context.Context, pveClient *pve.Client, request *certificatesv } return nil } - -var conditionv1beta1 = certificatesv1beta1.CertificateSigningRequestCondition{ - Type: certificatesv1beta1.CertificateApproved, - Reason: "AutoApproved", - Message: "Auto approving kubelet client certificate in PVE Cluster", -} - -func ApproveV1beta1(log logr.Logger, pveClient *pve.Client, client v1beta1.CertificateSigningRequestInterface, request *certificatesv1beta1.CertificateSigningRequest) error { - if len(request.Status.Conditions) > 0 { - return nil - } - ctx := context.Background() - err := checkCSR(ctx, pveClient, request) - if err == ErrNoKubeletServingCSR { - return nil - } - if err != nil { - return err - } - - for { - if len(request.Status.Conditions) > 0 { - return nil - } - request.Status.Conditions = append(request.Status.Conditions, conditionv1beta1) - // Submit the updated CSR. - if _, err := client.UpdateApproval(ctx, request, metav1.UpdateOptions{}); err != nil { - if strings.Contains(err.Error(), "the object has been modified") { - // The CSR might have been updated by a third-party, retry until we - // succeed. - request, err = client.Get(ctx, request.ObjectMeta.Name, metav1.GetOptions{}) - if err != nil { - return err - } - continue - } - return err - } - log.Info("csr approval successful", "name", request.ObjectMeta.Name) - return nil - } -} - -//func ApproveV1(pveClient *pve.Client, client v1.CertificateSigningRequestInterface, request *certificates.CertificateSigningRequest) error { -// if len(request.Status.Conditions) > 0 { -// return nil -// } -// err := checkCSR(pveClient, request) -// if err != nil { -// return err -// } -// -// for { -// if len(request.Status.Conditions) > 0 { -// return nil -// } -// request.Status.Conditions = append(request.Status.Conditions, condition) -// // Submit the updated CSR. -// if _, err := client.UpdateApproval(request); err != nil { -// if strings.Contains(err.Error(), "the object has been modified") { -// // The CSR might have been updated by a third-party, retry until we -// // succeed. -// request, err = client.Get(request.ObjectMeta.Name) -// if err != nil { -// return err -// } -// continue -// } -// return err -// } -// return nil -// } -//} diff --git a/pkg/pvecloud/cloud.go b/pkg/pvecloud/cloud.go index 1557eb7..4954811 100644 --- a/pkg/pvecloud/cloud.go +++ b/pkg/pvecloud/cloud.go @@ -13,7 +13,10 @@ limitations under the License. package pvecloud import ( + "encoding/json" + "errors" "io" + "io/ioutil" pve "git.giftfish.de/ston1th/pve-go" cloudprovider "k8s.io/cloud-provider" @@ -21,34 +24,49 @@ import ( const ( providerName = pve.PVEScheme - scheme = providerName + "://" + scheme = pve.PVESchemeURL ) type cloud struct { client *pve.Client instances cloudprovider.Instances instancesV2 cloudprovider.InstancesV2 - //zones cloudprovider.Zones - //routes cloudprovider.Routes - //network string } -func newCloud(_ io.Reader) (cloudprovider.Interface, error) { - opts, err := pve.ClientOptionsFromEnv() - if err != nil { - return nil, err +func newCloud(config io.Reader) (c cloudprovider.Interface, err error) { + var opts []pve.ClientOption + buf, _ := ioutil.ReadAll(config) + if len(buf) != 0 { + var cfg pve.Config + err = json.Unmarshal(buf, &cfg) + if err != nil { + return + } + opts, err = pve.ClientOptionsFromConfig(cfg) + if err != nil { + return + } + } else { + opts, err = pve.ClientOptionsFromEnv() + if err != nil { + return + } + } + if len(opts) == 0 { + err = errors.New("missing pve config") + return } client := pve.NewClient(opts...) - - return &cloud{ + c = &cloud{ client: client, instances: newInstances(client), instancesV2: newInstancesV2(client), //zones: newZones(client, nodeName), //routes: nil, //network: network, - }, nil + } + return } func (c *cloud) Initialize(clientBuilder cloudprovider.ControllerClientBuilder, stop <-chan struct{}) { @@ -63,7 +81,6 @@ func (c *cloud) InstancesV2() (cloudprovider.InstancesV2, bool) { } func (c *cloud) Zones() (cloudprovider.Zones, bool) { - //return c.zones, true return nil, false } @@ -77,17 +94,6 @@ func (c *cloud) Clusters() (cloudprovider.Clusters, bool) { func (c *cloud) Routes() (cloudprovider.Routes, bool) { return nil, false - /* - if len(c.network) > 0 { - r, err := newRoutes(c.client, c.network) - if err != nil { - return nil, false - } - return r, true - } - return nil, false // If no network is configured, disable the routes part - */ - } func (c *cloud) ProviderName() string {