csr rewrite

2020-11-01 00:01:57 +01:00 · 2020-11-01 00:01:57 +01:00 · 954b5c0d61
commit 954b5c0d61
parent 211b89236e
8 changed files with 430 additions and 280 deletions
--- a/4
+++ b/4
@ -36,11 +36,11 @@ vet:

 .PHONY: build-controller
 build-controller:
-	docker build --pull --build-arg ARCH=$(ARCH) --build-arg LDFLAGS="$(LDFLAGS)" -f images/$(CONTROLLER_NAME)/Dockerfile . -t $(CONTROLLER_IMG)-$(ARCH):$(TAG)
+	docker build --build-arg ARCH=$(ARCH) --build-arg LDFLAGS="$(LDFLAGS)" -f images/$(CONTROLLER_NAME)/Dockerfile . -t $(CONTROLLER_IMG)-$(ARCH):$(TAG)

 .PHONY: build-approver
 build-approver:
-	docker build --pull --build-arg ARCH=$(ARCH) --build-arg LDFLAGS="$(LDFLAGS)" -f images/$(APPROVER_NAME)/Dockerfile . -t $(APPROVER_IMG)-$(ARCH):$(TAG)
+	docker build --build-arg ARCH=$(ARCH) --build-arg LDFLAGS="$(LDFLAGS)" -f images/$(APPROVER_NAME)/Dockerfile . -t $(APPROVER_IMG)-$(ARCH):$(TAG)

 .PHONY: docker-build
 docker-build: build-controller build-approver
--- a/cmd/pve-csr-approver-manager/main.go
+++ b/cmd/pve-csr-approver-manager/main.go
@ -1,104 +1,220 @@
 package main

 import (
+	"encoding/json"
+	"errors"
 	"flag"
+	"io/ioutil"
+	"math/rand"
 	"os"
-	"os/signal"
-	"syscall"
 	"time"

-	certificatesv1beta1 "k8s.io/api/certificates/v1beta1"
-	"k8s.io/client-go/informers"
-	"k8s.io/client-go/kubernetes"
-	"k8s.io/client-go/rest"
-	"k8s.io/client-go/tools/cache"
+	"k8s.io/apimachinery/pkg/runtime"
+	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+	typedcertificatesv1beta1 "k8s.io/client-go/kubernetes/typed/certificates/v1beta1"
 	"k8s.io/klog/klogr"
 	"k8s.io/klog/v2"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/controller"
+	"sigs.k8s.io/controller-runtime/pkg/healthz"

 	"git.giftfish.de/ston1th/cloud-controller-manager-pve/pkg/approver"
 	pve "git.giftfish.de/ston1th/pve-go"
 )

+var (
+	scheme   = runtime.NewScheme()
+	setupLog = ctrl.Log.WithName("setup")
+)
+
+func init() {
+	_ = clientgoscheme.AddToScheme(scheme)
+}
+
 func main() {
+	rand.Seed(time.Now().UnixNano())
 	klog.InitFlags(nil)
+
+	var (
+		leaderElect    bool
+		healthAddr     string
+		cloudConfig    string
+		csrConcurrency int
+		syncPeriod     time.Duration
+	)
+
+	flag.BoolVar(&leaderElect, "leader-elect", true,
+		"Enable leader election for controller manager. "+
+			"Enabling this will ensure there is only one active controller manager.")
+	flag.StringVar(&cloudConfig,
+		"cloud-config",
+		"",
+		"The path to the cloud provider configuration file. Empty string for no configuration file.",
+	)
+	flag.StringVar(&healthAddr,
+		"health-addr",
+		":9449",
+		"The address the health endpoint binds to.",
+	)
+	flag.IntVar(&csrConcurrency,
+		"csr-concurrency",
+		1,
+		"Number of CSRs to process simultaneously",
+	)
+	flag.DurationVar(&syncPeriod,
+		"sync-period",
+		10*time.Minute,
+		"The minimum interval at which watched resources are reconciled (e.g. 15m)",
+	)
 	flag.Parse()
-	log := klogr.New()
-	opts, err := pve.ClientOptionsFromEnv()
+	ctrl.SetLogger(klogr.New())
+
+	rcfg := ctrl.GetConfigOrDie()
+
+	mgr, err := ctrl.NewManager(rcfg, ctrl.Options{
+		Scheme:                 scheme,
+		LeaderElection:         leaderElect,
+		LeaderElectionID:       "pve-csr-approver-manager",
+		HealthProbeBindAddress: healthAddr,
+		SyncPeriod:             &syncPeriod,
+	})
 	if err != nil {
-		log.Error(err, "could not create pve client")
+		setupLog.Error(err, "unable to start manager")
 		os.Exit(1)
 	}

-	pveClient := pve.NewClient(opts...)
-
-	cfg, err := rest.InClusterConfig()
-	if err != nil {
-		log.Error(err, "could not configure kubernetes client")
-		os.Exit(1)
-	}
-
-	cl, err := kubernetes.NewForConfig(cfg)
-	if err != nil {
-		log.Error(err, "could not create kubernetes client")
-		os.Exit(1)
-	}
-
-	factory := informers.NewSharedInformerFactory(cl, time.Second*30)
-	certInformerV1beta1 := factory.Certificates().V1beta1().CertificateSigningRequests().Informer()
-	fV1beta1 := func(obj interface{}) {
-		if req, ok := obj.(*certificatesv1beta1.CertificateSigningRequest); ok {
-			if err := approver.ApproveV1beta1(log, pveClient, cl.CertificatesV1beta1().CertificateSigningRequests(), req); err != nil {
-				log.Error(err, "csr approval failed", "name", req.ObjectMeta.Name)
-				return
+	var opts []pve.ClientOption
+	config, err := os.Open(cloudConfig)
+	if err == nil {
+		buf, _ := ioutil.ReadAll(config)
+		if len(buf) != 0 {
+			var cfg pve.Config
+			err = json.Unmarshal(buf, &cfg)
+			if err != nil {
+				setupLog.Error(err, "failed to read config file")
+				os.Exit(1)
+			}
+			opts, err = pve.ClientOptionsFromConfig(cfg)
+			if err != nil {
+				setupLog.Error(err, "failed to create client options from file")
+				os.Exit(1)
 			}
 		}
+	} else {
+		opts, err = pve.ClientOptionsFromEnv()
+		if err != nil {
+			setupLog.Error(err, "failed to create client options from env")
+			os.Exit(1)
+		}
 	}
-	certInformerV1beta1.AddEventHandler(cache.ResourceEventHandlerFuncs{
-		AddFunc: func(obj interface{}) {
-			fV1beta1(obj)
-		},
-		UpdateFunc: func(_, obj interface{}) {
-			fV1beta1(obj)
-		},
-	})
-	stop := make(chan struct{})
-	factory.Start(stop)
+	if len(opts) == 0 {
+		setupLog.Error(errors.New("missing pve config"), "")
+		os.Exit(1)
+	}
+	pveClient := pve.NewClient(opts...)

-	//watchListV1 := cache.NewListWatchFromClient(
-	//	cl.CertificatesV1Client.RESTClient(),
-	//	"certificatesigningrequests",
-	//	v1.NamespaceAll,
-	//	fields.Everything(),
-	//)
-	//fV1 := func(obj interface{}) {
-	//	if req, ok := obj.(*certificates.CertificateSigningRequest); ok {
-	//		if err := approver.ApproveV1(pveClient, cl.CertificatesV1Client.CertificateSigningRequests(), req); err != nil {
-	//			log.Error(err, "csr approval failed", "name", req.ObjectMeta.Name)
-	//			return
-	//		}
-	//		log.Info("csr approval successful", "name", req.ObjectMeta.Name)
-	//	}
-	//}
-	//_, controllerV1 := cache.NewInformer(
-	//	watchListV1,
-	//	&certificates.CertificateSigningRequest{},
-	//	time.Second*30,
-	//	cache.ResourceEventHandlerFuncs{
-	//		AddFunc: func(obj interface{}) {
-	//			fV1(obj)
-	//		},
-	//		UpdateFunc: func(_, obj interface{}) {
-	//			fV1(obj)
-	//		},
-	//	},
-	//)
-	//stopV1 := make(chan struct{})
-	//go controllerV1.Run(stopV1)
-	log.Info("csr approver started")
+	if err = (&approver.CSRReconciler{
+		Client:    mgr.GetClient(),
+		Log:       ctrl.Log.WithName("controllers").WithName("CSR"),
+		Scheme:    mgr.GetScheme(),
+		Recorder:  mgr.GetEventRecorderFor("pvecluster-controller"),
+		PVEClient: pveClient,
+		CSRClient: typedcertificatesv1beta1.NewForConfigOrDie(rcfg),
+	}).SetupWithManager(mgr, controller.Options{MaxConcurrentReconciles: csrConcurrency}); err != nil {
+		setupLog.Error(err, "unable to create controller", "controller", "CSR")
+		os.Exit(1)
+	}
+	if err := mgr.AddReadyzCheck("ping", healthz.Ping); err != nil {
+		setupLog.Error(err, "unable to create ready check")
+		os.Exit(1)
+	}

-	sigs := make(chan os.Signal)
-	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
-	<-sigs
-	close(stop)
-	log.Info("csr approver stopped")
+	if err := mgr.AddHealthzCheck("ping", healthz.Ping); err != nil {
+		setupLog.Error(err, "unable to create health check")
+		os.Exit(1)
+	}
+
+	setupLog.Info("starting manager")
+	if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
+		setupLog.Error(err, "problem running manager")
+		os.Exit(1)
+	}
+	/*
+		cfg, err := rest.InClusterConfig()
+		if err != nil {
+			log.Error(err, "could not configure kubernetes client")
+			os.Exit(1)
+		}
+
+		cl, err := kubernetes.NewForConfig(cfg)
+		if err != nil {
+			log.Error(err, "could not create kubernetes client")
+			os.Exit(1)
+		}
+
+		id, err := os.Hostname()
+		if err != nil {
+			log.Error(err, "could not get hostname")
+			os.Exit(1)
+		}
+		lock := &resourcelock.LeaseLock{
+			LeaseMeta: metav1.ObjectMeta{
+				Name:      csrLock,
+				Namespace: namespace,
+			},
+			Client: cl.CoordinationV1(),
+			LockConfig: resourcelock.ResourceLockConfig{
+				Identity: id,
+			},
+		}
+
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+
+		leaderelection.RunOrDie(ctx, leaderelection.LeaderElectionConfig{
+			Lock:            lock,
+			ReleaseOnCancel: true,
+			LeaseDuration:   time.Duration(leaseDuration) * time.Second,
+			RenewDeadline:   time.Duration(renewDeadline) * time.Second,
+			RetryPeriod:     time.Duration(retryPeriod) * time.Second,
+			Callbacks: leaderelection.LeaderCallbacks{
+				OnStartedLeading: func(ctx context.Context) {
+					log.Info("Leader election successful")
+					factory := informers.NewSharedInformerFactory(cl, time.Second*30)
+					certInformerV1beta1 := factory.Certificates().V1beta1().CertificateSigningRequests().Informer()
+					fV1beta1 := func(obj interface{}) {
+						if req, ok := obj.(*certificatesv1beta1.CertificateSigningRequest); ok {
+							if err := approver.ApproveV1beta1(log, pveClient, cl.CertificatesV1beta1().CertificateSigningRequests(), req); err != nil {
+								log.Error(err, "csr approval failed", "name", req.ObjectMeta.Name)
+								return
+							}
+						}
+					}
+					certInformerV1beta1.AddEventHandler(cache.ResourceEventHandlerFuncs{
+						AddFunc: func(obj interface{}) {
+							fV1beta1(obj)
+						},
+						UpdateFunc: func(_, obj interface{}) {
+							fV1beta1(obj)
+						},
+					})
+					stop := make(chan struct{})
+					factory.Start(stop)
+					<-ctx.Done()
+					close(stop)
+				},
+				OnStoppedLeading: func() {
+					cancel()
+					log.Info("Leader election lost")
+				},
+			}})
+
+		log.Info("csr approver started")
+
+		sigs := make(chan os.Signal)
+		signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
+		<-sigs
+		cancel()
+		log.Info("csr approver stopped")
+	*/
 }
--- a/deploy/pve-cloud-controller-manager.yaml
+++ b/deploy/pve-cloud-controller-manager.yaml
@ -28,6 +28,14 @@ rules:
  - patch
  - update
  - watch
+- apiGroups: [""]
+  resources: ["endpoints"]
+  verbs:
+  - create
+  - get
+  - list
+  - watch
+  - update
 - apiGroups: ["coordination.k8s.io"]
  resources: ["leases"]
  verbs:
@ -62,67 +70,61 @@ subjects:
  name: cloud-controller-manager
  namespace: kube-system
 ---
-apiVersion: apps/v1
-kind: Deployment
+apiVersion: v1
+kind: Pod
 metadata:
  name: pve-cloud-controller-manager
  namespace: kube-system
  labels:
    app: pve-cloud-controller-manager
 spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: pve-cloud-controller-manager
-  template:
-    metadata:
-      labels:
-        app: pve-cloud-controller-manager
-    spec:
-      priorityClassName: system-node-critical
-      containers:
-      - args:
-        - --cloud-provider=pve
-        - --leader-elect=false
-        env:
-        - name: PVE_API
-          valueFrom:
-            secretKeyRef:
-              key: api
-              name: pvecloud
-        - name: PVE_USER
-          valueFrom:
-            secretKeyRef:
-              key: user
-              name: pvecloud
-        - name: PVE_PASSWORD
-          valueFrom:
-            secretKeyRef:
-              key: password
-              name: pvecloud
-        image: reg.k8s.giftfish.de/library/pve-cloud-controller-manager-amd64:dev
-        imagePullPolicy: Always
-        livenessProbe:
-          httpGet:
-            path: /healthz
-            port: 10258
-            scheme: HTTPS
-          initialDelaySeconds: 3
-          periodSeconds: 3
-        name: manager
-        resources:
-          requests:
-            cpu: 100m
-            memory: 50Mi
-      dnsPolicy: Default
-      serviceAccountName: cloud-controller-manager
-      tolerations:
-      - effect: NoSchedule
-        key: node.cloudprovider.kubernetes.io/uninitialized
-        value: "true"
-      - key: CriticalAddonsOnly
-        operator: Exists
-      - effect: NoSchedule
-        key: node-role.kubernetes.io/master
-      - effect: NoSchedule
-        key: node.kubernetes.io/not-ready
+  priorityClassName: system-node-critical
+  #hostNetwork: true
+  containers:
+  - args:
+    - --cloud-config=/etc/kubernetes/pve.json
+    - --tls-min-version=VersionTLS12
+    - --tls-cipher-suites="TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256"
+    - --cloud-provider=pve
+    - --leader-elect=true
+    image: reg.k8s.giftfish.de/library/pve-cloud-controller-manager-amd64:dev
+    imagePullPolicy: Always
+    livenessProbe:
+      httpGet:
+        path: /healthz
+        port: 10258
+        scheme: HTTPS
+      initialDelaySeconds: 3
+      periodSeconds: 3
+    name: manager
+    resources:
+      requests:
+        cpu: 100m
+        memory: 50Mi
+    volumeMounts:
+    - name: etc-kubernetes
+      mountPath: /etc/kubernetes
+      readOnly: true
+    - name: etc-ssl
+      mountPath: /etc/ssl
+      readOnly: true
+  volumes:
+  - name: etc-kubernetes
+    hostPath:
+      path: /etc/kubernetes
+  - name: etc-ssl
+    hostPath:
+      path: /etc/ssl
+  dnsPolicy: Default
+  serviceAccountName: cloud-controller-manager
+  tolerations:
+  - effect: NoSchedule
+    key: node.cloudprovider.kubernetes.io/uninitialized
+    value: "true"
+  - key: CriticalAddonsOnly
+    operator: Exists
+  - effect: NoSchedule
+    key: node-role.kubernetes.io/master
+  - effect: NoSchedule
+    key: node.kubernetes.io/not-ready
+
--- a/deploy/pve-csr-approver-manager.yaml
+++ b/deploy/pve-csr-approver-manager.yaml
@ -26,6 +26,12 @@ rules:
  verbs:
  - create
  - update
+- apiGroups: ["coordination.k8s.io"]
+  resources: ["leases"]
+  verbs:
+  - get
+  - create
+  - update
 ---
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
@ -40,57 +46,56 @@ subjects:
  namespace: kube-system
  name: pve-csr-approver
 ---
-apiVersion: apps/v1
-kind: Deployment
+apiVersion: v1
+kind: Pod
 metadata:
  name: pve-csr-approver-manager
  namespace: kube-system
  labels:
    app: pve-csr-approver-manager
 spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: pve-csr-approver-manager
-  template:
-    metadata:
-      labels:
-        app: pve-csr-approver-manager
-    spec:
-      priorityClassName: system-node-critical
-      containers:
-      - name: manager
-        env:
-        - name: PVE_API
-          valueFrom:
-            secretKeyRef:
-              key: api
-              name: pvecloud
-        - name: PVE_USER
-          valueFrom:
-            secretKeyRef:
-              key: user
-              name: pvecloud
-        - name: PVE_PASSWORD
-          valueFrom:
-            secretKeyRef:
-              key: password
-              name: pvecloud
-        image: reg.k8s.giftfish.de/library/pve-csr-approver-manager-amd64:dev
-        imagePullPolicy: Always
-        resources:
-          requests:
-            cpu: 100m
-            memory: 50Mi
-      dnsPolicy: Default
-      serviceAccountName: pve-csr-approver
-      tolerations:
-      - effect: NoSchedule
-        key: node.cloudprovider.kubernetes.io/uninitialized
-        value: "true"
-      - key: CriticalAddonsOnly
-        operator: Exists
-      - effect: NoSchedule
-        key: node-role.kubernetes.io/master
-      - effect: NoSchedule
-        key: node.kubernetes.io/not-ready
+  priorityClassName: system-node-critical
+  #hostNetwork: true
+  containers:
+  - args:
+    - --cloud-config=/etc/kubernetes/pve.json
+    - --leader-elect=true
+    name: manager
+    image: reg.k8s.giftfish.de/library/pve-csr-approver-manager-amd64:dev
+    imagePullPolicy: Always
+    livenessProbe:
+      httpGet:
+        path: /healthz
+        port: 9449
+      initialDelaySeconds: 3
+      periodSeconds: 3
+    resources:
+      requests:
+        cpu: 100m
+        memory: 50Mi
+    volumeMounts:
+    - name: etc-kubernetes
+      mountPath: /etc/kubernetes
+      readOnly: true
+    - name: etc-ssl
+      mountPath: /etc/ssl
+      readOnly: true
+  volumes:
+  - name: etc-kubernetes
+    hostPath:
+      path: /etc/kubernetes
+  - name: etc-ssl
+    hostPath:
+      path: /etc/ssl
+  dnsPolicy: Default
+  serviceAccountName: pve-csr-approver
+  tolerations:
+  - effect: NoSchedule
+    key: node.cloudprovider.kubernetes.io/uninitialized
+    value: "true"
+  - key: CriticalAddonsOnly
+    operator: Exists
+  - effect: NoSchedule
+    key: node-role.kubernetes.io/master
+  - effect: NoSchedule
+    key: node.kubernetes.io/not-ready
--- a/go.mod
+++ b/go.mod
@ -3,7 +3,7 @@ module git.giftfish.de/ston1th/cloud-controller-manager-pve
 go 1.13

 require (
-	git.giftfish.de/ston1th/pve-go v0.0.0-20201031113657-271c91eecbeb
+	git.giftfish.de/ston1th/pve-go v0.0.0-20201031205459-928086aea9c1
 	github.com/go-logr/logr v0.2.1
 	github.com/hashicorp/golang-lru v0.5.4 // indirect
 	github.com/imdario/mergo v0.3.9 // indirect
@ -16,6 +16,7 @@ require (
 	k8s.io/klog v1.0.0
 	k8s.io/klog/v2 v2.4.0
 	k8s.io/kubernetes v1.19.3
+	sigs.k8s.io/controller-runtime v0.6.3
 )

 replace (
--- a/go.sum
+++ b/go.sum
@ -14,6 +14,8 @@ cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiy
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 git.giftfish.de/ston1th/pve-go v0.0.0-20201031113657-271c91eecbeb h1:4tRFUX6/KiJEjVfIBIdekpXRHCMG0G1ym4YHnDTTo9U=
 git.giftfish.de/ston1th/pve-go v0.0.0-20201031113657-271c91eecbeb/go.mod h1:WOrvThayhaB89KZwQqGzqcTD9nVeTsnaRB1vBaeb2tM=
+git.giftfish.de/ston1th/pve-go v0.0.0-20201031205459-928086aea9c1 h1:DB8S2p46uptQn2dkfb1RC28tzOIED7bYbt+UHQWwEbQ=
+git.giftfish.de/ston1th/pve-go v0.0.0-20201031205459-928086aea9c1/go.mod h1:WOrvThayhaB89KZwQqGzqcTD9nVeTsnaRB1vBaeb2tM=
 github.com/Azure/azure-sdk-for-go v43.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
 github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 h1:w+iIsaOQNcT7OZ575w+acHgRric5iCyQh+xv+KJ4HB8=
 github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
@ -157,6 +159,7 @@ github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/euank/go-kmsg-parser v2.0.0+incompatible/go.mod h1:MhmAMZ8V4CYH4ybgdRwPr2TU5ThnS43puaKEMpja1uw=
+github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/evanphx/json-patch v4.9.0+incompatible h1:kLcOMZeuLAJvL2BPWLMIj5oaZQobrkAqrL+WFZwQses=
 github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4=
@ -186,6 +189,7 @@ github.com/go-logr/logr v0.2.0 h1:QvGt2nLcHH0WK9orKa+ppBPAxREcH364nPUedEpK0TY=
 github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
 github.com/go-logr/logr v0.2.1 h1:fV3MLmabKIZ383XifUjFSwcoGee0v9qgPp8wy5svibE=
 github.com/go-logr/logr v0.2.1/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
+github.com/go-logr/zapr v0.1.0/go.mod h1:tabnROwaDl0UNxkVeFRbY8bwB37GwRv0P8lg6aAiEnk=
 github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70tL6pCuVxPJOHXQ+wIac1FUrvNkHolPie/cLEU6hI=
 github.com/go-openapi/analysis v0.17.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik=
 github.com/go-openapi/analysis v0.18.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik=
@ -293,6 +297,7 @@ github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
+github.com/googleapis/gnostic v0.3.1/go.mod h1:on+2t9HRStVgn95RSsFWFz+6Q0Snyqv1awfrALZdbtU=
 github.com/googleapis/gnostic v0.4.1 h1:DLJCy1n/vrD4HPjOvYcT8aYQXpPIzoRZONaYwyycI+I=
 github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg=
 github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8=
@ -482,6 +487,7 @@ github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R
 github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/procfs v0.0.2 h1:6LJUbpNm42llc4HRCuvApCSWB/WfhuNo9K98Q9sNGfs=
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
+github.com/prometheus/procfs v0.0.11/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
 github.com/prometheus/procfs v0.1.3 h1:F0+tqvhOksq22sc6iCHF5WGlWjdwj92p0udFh1VFBS8=
 github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
@ -746,6 +752,8 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+gomodules.xyz/jsonpatch/v2 v2.0.1 h1:xyiBuvkD2g5n7cYzx6u2sxQvsAy4QJsZFCzGVdzOXZ0=
+gomodules.xyz/jsonpatch/v2 v2.0.1/go.mod h1:IhYNNY4jnS53ZnfE4PAmpKtDpTCj1JFXc+3mwe7XcUU=
 gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo=
 gonum.org/v1/gonum v0.0.0-20190331200053-3d26580ed485/go.mod h1:2ltnJ7xHfj0zHS40VVPYEAAMTa3ZGguvHGBSJeRWqE0=
 gonum.org/v1/gonum v0.6.2/go.mod h1:9mxDZsDKxgMAuccQkewq682L+0eCu4dCN2yonUJTCLU=
@ -841,6 +849,7 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh
 honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
 k8s.io/api v0.19.3 h1:GN6ntFnv44Vptj/b+OnMW7FmzkpDoIDLZRvKX3XH9aU=
 k8s.io/api v0.19.3/go.mod h1:VF+5FT1B74Pw3KxMdKyinLo+zynBaMBiAfGMuldcNDs=
+k8s.io/apiextensions-apiserver v0.19.3 h1:WZxBypSHW4SdXHbdPTS/Jy7L2la6Niggs8BuU5o+avo=
 k8s.io/apiextensions-apiserver v0.19.3/go.mod h1:igVEkrE9TzInc1tYE7qSqxaLg/rEAp6B5+k9Q7+IC8Q=
 k8s.io/apimachinery v0.19.3 h1:bpIQXlKjB4cB/oNpnNnV+BybGPR7iP5oYpsOTEJ4hgc=
 k8s.io/apimachinery v0.19.3/go.mod h1:DnPGDnARWFvYa3pMHgSxtbZb7gpzzAZ1pTfaUNDVlmA=
@ -884,6 +893,7 @@ k8s.io/metrics v0.19.3/go.mod h1:Eap/Lk1FiAIjkaArFuv41v+ph6dbDpVGwAg7jMI+4vg=
 k8s.io/sample-apiserver v0.19.3/go.mod h1:lCFCWx71mTeqnj/DioYoSGwC0Ce/KSPFtvt41OKpcMQ=
 k8s.io/system-validators v1.1.2/go.mod h1:bPldcLgkIUK22ALflnsXk8pvkTEndYdNuaHH6gRrl0Q=
 k8s.io/utils v0.0.0-20200414100711-2df71ebbae66/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
+k8s.io/utils v0.0.0-20200603063816-c1c6865ac451/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
 k8s.io/utils v0.0.0-20200729134348-d5654de09c73 h1:uJmqzgNWG7XyClnU/mLPBWwfKKF1K8Hf8whTseBgJcg=
 k8s.io/utils v0.0.0-20200729134348-d5654de09c73/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
 modernc.org/cc v1.0.0/go.mod h1:1Sk4//wdnYJiUIxnW8ddKpaOJCF37yAdqYnkxUpaYxw=
@ -895,6 +905,8 @@ rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8
 rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=
 sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.9 h1:rusRLrDhjBp6aYtl9sGEvQJr6faoHoDLd0YcUBTZguI=
 sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.9/go.mod h1:dzAXnQbTRyDlZPJX2SUPEqvnB+j7AJjtlox7PEwigU0=
+sigs.k8s.io/controller-runtime v0.6.3 h1:SBbr+inLPEKhvlJtrvDcwIpm+uhDvp63Bl72xYJtoOE=
+sigs.k8s.io/controller-runtime v0.6.3/go.mod h1:WlZNXcM0++oyaQt4B7C2lEE5JYRs8vJUzRP4N4JpdAY=
 sigs.k8s.io/kustomize v2.0.3+incompatible/go.mod h1:MkjgH3RdOWrievjo6c9T245dYlB5QeXV4WCbnt/PEpU=
 sigs.k8s.io/structured-merge-diff/v4 v4.0.1 h1:YXTMot5Qz/X1iBRJhAt+vI+HVttY0WkSqqhKxQ0xVbA=
 sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
--- a/pkg/approver/approver.go
+++ b/pkg/approver/approver.go
@ -8,17 +8,98 @@ import (

 	pve "git.giftfish.de/ston1th/pve-go"

-	//"k8s.io/client-go/kubernetes/typed/certificates/v1"
 	"github.com/go-logr/logr"
 	certificatesv1beta1 "k8s.io/api/certificates/v1beta1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/util/sets"
-	"k8s.io/client-go/kubernetes/typed/certificates/v1beta1"
+	typedcertificatesv1beta1 "k8s.io/client-go/kubernetes/typed/certificates/v1beta1"
+	"k8s.io/client-go/tools/record"
 	"k8s.io/kubernetes/pkg/apis/certificates"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller"
 )

 var ErrNoKubeletServingCSR = errors.New("CSR is no kubelet serving certificate")

+type CSRReconciler struct {
+	client.Client
+	Recorder  record.EventRecorder
+	Log       logr.Logger
+	Scheme    *runtime.Scheme
+	PVEClient *pve.Client
+	CSRClient *typedcertificatesv1beta1.CertificatesV1beta1Client
+}
+
+func (r *CSRReconciler) SetupWithManager(mgr ctrl.Manager, options controller.Options) error {
+	return ctrl.NewControllerManagedBy(mgr).
+		WithOptions(options).
+		For(&certificatesv1beta1.CertificateSigningRequest{}).
+		Complete(r)
+}
+
+func (r *CSRReconciler) Reconcile(req ctrl.Request) (res ctrl.Result, err error) {
+	ctx := context.Background()
+	log := r.Log
+	csr := &certificatesv1beta1.CertificateSigningRequest{}
+	if err = r.Get(ctx, req.NamespacedName, csr); err != nil {
+		if apierrors.IsNotFound(err) {
+			err = nil
+			return
+		}
+		return
+	}
+	if !csr.DeletionTimestamp.IsZero() {
+		return
+	}
+	err = r.approve(ctx, log, csr)
+	return
+}
+
+var conditionv1beta1 = certificatesv1beta1.CertificateSigningRequestCondition{
+	Type:    certificatesv1beta1.CertificateApproved,
+	Reason:  "AutoApproved",
+	Message: "Auto approving kubelet client certificate in PVE Cluster",
+}
+
+func (r *CSRReconciler) approve(ctx context.Context, log logr.Logger, request *certificatesv1beta1.CertificateSigningRequest) error {
+	if len(request.Status.Conditions) > 0 {
+		return nil
+	}
+	client := r.CSRClient.CertificateSigningRequests()
+	err := checkCSR(ctx, r.PVEClient, request)
+	if err == ErrNoKubeletServingCSR {
+		return nil
+	}
+	if err != nil {
+		return err
+	}
+
+	for {
+		if len(request.Status.Conditions) > 0 {
+			return nil
+		}
+		request.Status.Conditions = append(request.Status.Conditions, conditionv1beta1)
+		// Submit the updated CSR.
+		if _, err := client.UpdateApproval(ctx, request, metav1.UpdateOptions{}); err != nil {
+			if strings.Contains(err.Error(), "the object has been modified") {
+				// The CSR might have been updated by a third-party, retry until we
+				// succeed.
+				request, err = client.Get(ctx, request.ObjectMeta.Name, metav1.GetOptions{})
+				if err != nil {
+					return err
+				}
+				continue
+			}
+			return err
+		}
+		log.Info("csr approval successful", "name", request.ObjectMeta.Name)
+		return nil
+	}
+}
+
 func checkCSR(ctx context.Context, pveClient *pve.Client, request *certificatesv1beta1.CertificateSigningRequest) error {
 	req, err := certificates.ParseCSR(request.Spec.Request)
 	if err != nil {
@ -85,76 +166,3 @@ func checkCSR(ctx context.Context, pveClient *pve.Client, request *certificatesv
 	}
 	return nil
 }
-
-var conditionv1beta1 = certificatesv1beta1.CertificateSigningRequestCondition{
-	Type:    certificatesv1beta1.CertificateApproved,
-	Reason:  "AutoApproved",
-	Message: "Auto approving kubelet client certificate in PVE Cluster",
-}
-
-func ApproveV1beta1(log logr.Logger, pveClient *pve.Client, client v1beta1.CertificateSigningRequestInterface, request *certificatesv1beta1.CertificateSigningRequest) error {
-	if len(request.Status.Conditions) > 0 {
-		return nil
-	}
-	ctx := context.Background()
-	err := checkCSR(ctx, pveClient, request)
-	if err == ErrNoKubeletServingCSR {
-		return nil
-	}
-	if err != nil {
-		return err
-	}
-
-	for {
-		if len(request.Status.Conditions) > 0 {
-			return nil
-		}
-		request.Status.Conditions = append(request.Status.Conditions, conditionv1beta1)
-		// Submit the updated CSR.
-		if _, err := client.UpdateApproval(ctx, request, metav1.UpdateOptions{}); err != nil {
-			if strings.Contains(err.Error(), "the object has been modified") {
-				// The CSR might have been updated by a third-party, retry until we
-				// succeed.
-				request, err = client.Get(ctx, request.ObjectMeta.Name, metav1.GetOptions{})
-				if err != nil {
-					return err
-				}
-				continue
-			}
-			return err
-		}
-		log.Info("csr approval successful", "name", request.ObjectMeta.Name)
-		return nil
-	}
-}
-
-//func ApproveV1(pveClient *pve.Client, client v1.CertificateSigningRequestInterface, request *certificates.CertificateSigningRequest) error {
-//	if len(request.Status.Conditions) > 0 {
-//		return nil
-//	}
-//	err := checkCSR(pveClient, request)
-//	if err != nil {
-//		return err
-//	}
-//
-//	for {
-//		if len(request.Status.Conditions) > 0 {
-//			return nil
-//		}
-//		request.Status.Conditions = append(request.Status.Conditions, condition)
-//		// Submit the updated CSR.
-//		if _, err := client.UpdateApproval(request); err != nil {
-//			if strings.Contains(err.Error(), "the object has been modified") {
-//				// The CSR might have been updated by a third-party, retry until we
-//				// succeed.
-//				request, err = client.Get(request.ObjectMeta.Name)
-//				if err != nil {
-//					return err
-//				}
-//				continue
-//			}
-//			return err
-//		}
-//		return nil
-//	}
-//}
--- a/pkg/pvecloud/cloud.go
+++ b/pkg/pvecloud/cloud.go
@ -13,7 +13,10 @@ limitations under the License.
 package pvecloud

 import (
+	"encoding/json"
+	"errors"
 	"io"
+	"io/ioutil"

 	pve "git.giftfish.de/ston1th/pve-go"
 	cloudprovider "k8s.io/cloud-provider"
@ -21,34 +24,49 @@ import (

 const (
 	providerName = pve.PVEScheme
-	scheme       = providerName + "://"
+	scheme       = pve.PVESchemeURL
 )

 type cloud struct {
 	client      *pve.Client
 	instances   cloudprovider.Instances
 	instancesV2 cloudprovider.InstancesV2
-	//zones     cloudprovider.Zones
-	//routes    cloudprovider.Routes
-	//network string
 }

-func newCloud(_ io.Reader) (cloudprovider.Interface, error) {
-	opts, err := pve.ClientOptionsFromEnv()
-	if err != nil {
-		return nil, err
+func newCloud(config io.Reader) (c cloudprovider.Interface, err error) {
+	var opts []pve.ClientOption
+	buf, _ := ioutil.ReadAll(config)
+	if len(buf) != 0 {
+		var cfg pve.Config
+		err = json.Unmarshal(buf, &cfg)
+		if err != nil {
+			return
+		}
+		opts, err = pve.ClientOptionsFromConfig(cfg)
+		if err != nil {
+			return
+		}
+	} else {
+		opts, err = pve.ClientOptionsFromEnv()
+		if err != nil {
+			return
+		}
+	}
+	if len(opts) == 0 {
+		err = errors.New("missing pve config")
+		return
 	}

 	client := pve.NewClient(opts...)
-
-	return &cloud{
+	c = &cloud{
 		client:      client,
 		instances:   newInstances(client),
 		instancesV2: newInstancesV2(client),
 		//zones:     newZones(client, nodeName),
 		//routes:    nil,
 		//network:   network,
-	}, nil
+	}
+	return
 }

 func (c *cloud) Initialize(clientBuilder cloudprovider.ControllerClientBuilder, stop <-chan struct{}) {
@ -63,7 +81,6 @@ func (c *cloud) InstancesV2() (cloudprovider.InstancesV2, bool) {
 }

 func (c *cloud) Zones() (cloudprovider.Zones, bool) {
-	//return c.zones, true
 	return nil, false
 }

@ -77,17 +94,6 @@ func (c *cloud) Clusters() (cloudprovider.Clusters, bool) {

 func (c *cloud) Routes() (cloudprovider.Routes, bool) {
 	return nil, false
-	/*
-		if len(c.network) > 0 {
-			r, err := newRoutes(c.client, c.network)
-			if err != nil {
-				return nil, false
-			}
-			return r, true
-		}
-		return nil, false // If no network is configured, disable the routes part
-	*/
-
 }

 func (c *cloud) ProviderName() string {