added csr approver

2020-10-31 16:53:35 +01:00 · 2020-10-31 16:53:35 +01:00 · bc750389dc
commit bc750389dc
parent b0e048ba9d
14 changed files with 852 additions and 310 deletions
--- a/48
+++ b/48
@ -1,7 +1,9 @@
 # Image URL to use all building/pushing image targets
 REGISTRY ?= reg.k8s.giftfish.de/library
-IMAGE_NAME ?= pve-cloud-controller-manager
-CONTROLLER_IMG ?= $(REGISTRY)/$(IMAGE_NAME)
+CONTROLLER_NAME ?= pve-cloud-controller-manager
+APPROVER_NAME ?= pve-csr-approver-manager
+CONTROLLER_IMG ?= $(REGISTRY)/$(CONTROLLER_NAME)
+APPROVER_IMG ?= $(REGISTRY)/$(APPROVER_NAME)
 TAG ?= dev
 ARCH ?= amd64

@ -32,32 +34,24 @@ fmt:
 vet:
 	go vet ./...

-.PHONY: docker-build
-docker-build: ## Build the docker image for controller-manager
-	docker build --pull --build-arg ARCH=$(ARCH) --build-arg LDFLAGS="$(LDFLAGS)" . -t $(CONTROLLER_IMG)-$(ARCH):$(TAG)
-	MANIFEST_IMG=$(CONTROLLER_IMG)-$(ARCH) MANIFEST_TAG=$(TAG) $(MAKE) set-manifest-image
-	$(MAKE) set-manifest-pull-policy
+.PHONY: build-controller
+build-controller:
+	docker build --pull --build-arg ARCH=$(ARCH) --build-arg LDFLAGS="$(LDFLAGS)" -f images/$(CONTROLLER_NAME)/Dockerfile . -t $(CONTROLLER_IMG)-$(ARCH):$(TAG)

-.PHONY: docker-push
-docker-push: ## Push the docker image
+.PHONY: build-approver
+build-approver:
+	docker build --pull --build-arg ARCH=$(ARCH) --build-arg LDFLAGS="$(LDFLAGS)" -f images/$(APPROVER_NAME)/Dockerfile . -t $(APPROVER_IMG)-$(ARCH):$(TAG)
+
+.PHONY: docker-build
+docker-build: build-controller build-approver
+
+.PHONY: push-controller
+push-controller:
 	docker push $(CONTROLLER_IMG)-$(ARCH):$(TAG)

-.PHONY: docker-push-manifest
-docker-push-manifest: ## Push the fat manifest docker image.
-	## Minimum docker version 18.06.0 is required for creating and pushing manifest images.
-	docker manifest create --amend $(CONTROLLER_IMG):$(TAG) $(shell echo $(ALL_ARCH) | sed -e "s~[^ ]*~$(CONTROLLER_IMG)\-&:$(TAG)~g")
-	@for arch in $(ALL_ARCH); do docker manifest annotate --arch $${arch} ${CONTROLLER_IMG}:${TAG} ${CONTROLLER_IMG}-$${arch}:${TAG}; done
-	docker manifest push --purge ${CONTROLLER_IMG}:${TAG}
-	MANIFEST_IMG=$(CONTROLLER_IMG) MANIFEST_TAG=$(TAG) $(MAKE) set-manifest-image
-	$(MAKE) set-manifest-pull-policy
+.PHONY: push-approver
+push-approver:
+	docker push $(APPROVER_IMG)-$(ARCH):$(TAG)

-.PHONY: set-manifest-image
-set-manifest-image:
-	$(info Updating kustomize image patch file for manager resource)
-	sed -i'' -e 's@image: .*@image: '"${MANIFEST_IMG}:$(MANIFEST_TAG)"'@' ./config/manager/manager_image_patch.yaml
-
-
-.PHONY: set-manifest-pull-policy
-set-manifest-pull-policy:
-	$(info Updating kustomize pull policy file for manager resource)
-	sed -i'' -e 's@imagePullPolicy: .*@imagePullPolicy: '"$(PULL_POLICY)"'@' ./config/manager/manager_pull_policy.yaml
+.PHONY: docker-push
+docker-push: push-controller push-approver
--- a/cmd/pve-cloud-controller-manager/main.go
+++ b/cmd/pve-cloud-controller-manager/main.go
@ -21,7 +21,7 @@ import (
 	"os"
 	"time"

-	_ "git.giftfish.de/ston1th/cloud-controller-manager-pve/pvecloud"
+	_ "git.giftfish.de/ston1th/cloud-controller-manager-pve/pkg/pvecloud"
 	"k8s.io/component-base/logs"
 	"k8s.io/kubernetes/cmd/cloud-controller-manager/app"
 	//_ "k8s.io/kubernetes/pkg/client/metrics/prometheus" // for client metric registration
--- a/cmd/pve-csr-approver-manager/main.go
+++ b/cmd/pve-csr-approver-manager/main.go
@ -0,0 +1,104 @@
+package main
+
+import (
+	"flag"
+	"os"
+	"os/signal"
+	"syscall"
+	"time"
+
+	certificatesv1beta1 "k8s.io/api/certificates/v1beta1"
+	"k8s.io/client-go/informers"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/cache"
+	"k8s.io/klog/klogr"
+	"k8s.io/klog/v2"
+
+	"git.giftfish.de/ston1th/cloud-controller-manager-pve/pkg/approver"
+	pve "git.giftfish.de/ston1th/pve-go"
+)
+
+func main() {
+	klog.InitFlags(nil)
+	flag.Parse()
+	log := klogr.New()
+	opts, err := pve.ClientOptionsFromEnv()
+	if err != nil {
+		log.Error(err, "could not create pve client")
+		os.Exit(1)
+	}
+
+	pveClient := pve.NewClient(opts...)
+
+	cfg, err := rest.InClusterConfig()
+	if err != nil {
+		log.Error(err, "could not configure kubernetes client")
+		os.Exit(1)
+	}
+
+	cl, err := kubernetes.NewForConfig(cfg)
+	if err != nil {
+		log.Error(err, "could not create kubernetes client")
+		os.Exit(1)
+	}
+
+	factory := informers.NewSharedInformerFactory(cl, time.Second*30)
+	certInformerV1beta1 := factory.Certificates().V1beta1().CertificateSigningRequests().Informer()
+	fV1beta1 := func(obj interface{}) {
+		if req, ok := obj.(*certificatesv1beta1.CertificateSigningRequest); ok {
+			if err := approver.ApproveV1beta1(log, pveClient, cl.CertificatesV1beta1().CertificateSigningRequests(), req); err != nil {
+				log.Error(err, "csr approval failed", "name", req.ObjectMeta.Name)
+				return
+			}
+		}
+	}
+	certInformerV1beta1.AddEventHandler(cache.ResourceEventHandlerFuncs{
+		AddFunc: func(obj interface{}) {
+			fV1beta1(obj)
+		},
+		UpdateFunc: func(_, obj interface{}) {
+			fV1beta1(obj)
+		},
+	})
+	stop := make(chan struct{})
+	factory.Start(stop)
+
+	//watchListV1 := cache.NewListWatchFromClient(
+	//	cl.CertificatesV1Client.RESTClient(),
+	//	"certificatesigningrequests",
+	//	v1.NamespaceAll,
+	//	fields.Everything(),
+	//)
+	//fV1 := func(obj interface{}) {
+	//	if req, ok := obj.(*certificates.CertificateSigningRequest); ok {
+	//		if err := approver.ApproveV1(pveClient, cl.CertificatesV1Client.CertificateSigningRequests(), req); err != nil {
+	//			log.Error(err, "csr approval failed", "name", req.ObjectMeta.Name)
+	//			return
+	//		}
+	//		log.Info("csr approval successful", "name", req.ObjectMeta.Name)
+	//	}
+	//}
+	//_, controllerV1 := cache.NewInformer(
+	//	watchListV1,
+	//	&certificates.CertificateSigningRequest{},
+	//	time.Second*30,
+	//	cache.ResourceEventHandlerFuncs{
+	//		AddFunc: func(obj interface{}) {
+	//			fV1(obj)
+	//		},
+	//		UpdateFunc: func(_, obj interface{}) {
+	//			fV1(obj)
+	//		},
+	//	},
+	//)
+	//stopV1 := make(chan struct{})
+	//go controllerV1.Run(stopV1)
+	log.Info("csr approver started")
+
+	sigs := make(chan os.Signal)
+	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
+	<-sigs
+	close(stop)
+	log.Info("csr approver stopped")
+}
--- a/deploy/pve-cloud-controller-manager.yaml
+++ b/deploy/pve-cloud-controller-manager.yaml
@ -60,7 +60,7 @@ spec:
          valueFrom:
            fieldRef:
              fieldPath: spec.nodeName
-        image: reg.k8s.giftfish.de/library/pve-cloud-controller-manager-amd64:latest
+        image: reg.k8s.giftfish.de/library/pve-cloud-controller-manager-amd64:dev
        imagePullPolicy: Always
        livenessProbe:
          httpGet:
--- a/deploy/pve-csr-approver-manager.yaml
+++ b/deploy/pve-csr-approver-manager.yaml
@ -0,0 +1,105 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: pve-csr-approver
+  namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRole
+metadata:
+  name: pve-csr-approver
+rules:
+- apiGroups:
+  - certificates.k8s.io
+  resources:
+  - signers
+  # legacy-unknown: support before kubernetes-1.18.0
+  resourceNames:
+  - "kubernetes.io/legacy-unknown"
+  - "kubernetes.io/kubelet-serving"
+  verbs:
+  - approve
+- apiGroups:
+  - certificates.k8s.io
+  resources:
+  - certificatesigningrequests
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - certificates.k8s.io
+  resources:
+  - certificatesigningrequests/approval
+  verbs:
+  - create
+  - update
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: pve-csr-approver
+subjects:
+- kind: ServiceAccount
+  namespace: kube-system
+  name: pve-csr-approver
+roleRef:
+  kind: ClusterRole
+  name: pve-csr-approver
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: pve-csr-approver-manager
+  name: pve-csr-approver-manager
+  namespace: kube-system
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: pve-csr-approver-manager
+  template:
+    metadata:
+      annotations:
+        scheduler.alpha.kubernetes.io/critical-pod: ""
+      labels:
+        app: pve-csr-approver-manager
+    spec:
+      containers:
+      - name: manager
+        env:
+        - name: PVE_API
+          valueFrom:
+            secretKeyRef:
+              key: api
+              name: pvecloud
+        - name: PVE_USER
+          valueFrom:
+            secretKeyRef:
+              key: user
+              name: pvecloud
+        - name: PVE_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              key: password
+              name: pvecloud
+        image: reg.k8s.giftfish.de/library/pve-csr-approver-manager-amd64:dev
+        imagePullPolicy: Always
+        resources:
+          requests:
+            cpu: 100m
+            memory: 50Mi
+      dnsPolicy: Default
+      serviceAccountName: pve-csr-approver
+      tolerations:
+      - effect: NoSchedule
+        key: node.cloudprovider.kubernetes.io/uninitialized
+        value: "true"
+      - key: CriticalAddonsOnly
+        operator: Exists
+      - effect: NoSchedule
+        key: node-role.kubernetes.io/master
+      - effect: NoSchedule
+        key: node.kubernetes.io/not-ready
--- a/go.mod
+++ b/go.mod
@ -3,39 +3,43 @@ module git.giftfish.de/ston1th/cloud-controller-manager-pve
 go 1.13

 require (
-	git.giftfish.de/ston1th/pve-go v0.0.0-20201025202919-78911eca5697
-	k8s.io/api v0.18.6
-	k8s.io/apimachinery v0.18.6
-	k8s.io/client-go v0.18.6
-	k8s.io/cloud-provider v0.18.3
-	k8s.io/component-base v0.18.3
+	git.giftfish.de/ston1th/pve-go v0.0.0-20201031113657-271c91eecbeb
+	github.com/go-logr/logr v0.2.1
+	github.com/hashicorp/golang-lru v0.5.4 // indirect
+	github.com/imdario/mergo v0.3.9 // indirect
+	github.com/onsi/gomega v1.10.1 // indirect
+	k8s.io/api v0.19.3
+	k8s.io/apimachinery v0.19.3
+	k8s.io/client-go v0.19.3
+	k8s.io/cloud-provider v0.19.3
+	k8s.io/component-base v0.19.3
 	k8s.io/klog v1.0.0
-	k8s.io/kubernetes v1.18.3
-	sigs.k8s.io/controller-runtime v0.6.3
+	k8s.io/klog/v2 v2.4.0
+	k8s.io/kubernetes v1.19.3
 )

 replace (
-	k8s.io/api => k8s.io/api v0.18.3
-	k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.18.3
-	k8s.io/apimachinery => k8s.io/apimachinery v0.18.3
-	k8s.io/apiserver => k8s.io/apiserver v0.18.3
-	k8s.io/cli-runtime => k8s.io/cli-runtime v0.18.3
-	k8s.io/client-go => k8s.io/client-go v0.18.3
-	k8s.io/cloud-provider => k8s.io/cloud-provider v0.18.3
-	k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.18.3
-	k8s.io/code-generator => k8s.io/code-generator v0.18.3
-	k8s.io/component-base => k8s.io/component-base v0.18.3
-	k8s.io/cri-api => k8s.io/cri-api v0.18.3
-	k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.18.3
-	k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.18.3
-	k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.18.3
-	k8s.io/kube-proxy => k8s.io/kube-proxy v0.18.3
-	k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.18.3
-	k8s.io/kubectl => k8s.io/kubectl v0.18.3
-	k8s.io/kubelet => k8s.io/kubelet v0.18.3
-	k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.18.3
-	k8s.io/metrics => k8s.io/metrics v0.18.3
-	k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.18.3
-	k8s.io/sample-cli-plugin => k8s.io/sample-cli-plugin v0.18.3
-	k8s.io/sample-controller => k8s.io/sample-controller v0.18.3
+	k8s.io/api => k8s.io/api v0.19.3
+	k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.19.3
+	k8s.io/apimachinery => k8s.io/apimachinery v0.19.3
+	k8s.io/apiserver => k8s.io/apiserver v0.19.3
+	k8s.io/cli-runtime => k8s.io/cli-runtime v0.19.3
+	k8s.io/client-go => k8s.io/client-go v0.19.3
+	k8s.io/cloud-provider => k8s.io/cloud-provider v0.19.3
+	k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.19.3
+	k8s.io/code-generator => k8s.io/code-generator v0.19.3
+	k8s.io/component-base => k8s.io/component-base v0.19.3
+	k8s.io/cri-api => k8s.io/cri-api v0.19.3
+	k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.19.3
+	k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.19.3
+	k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.19.3
+	k8s.io/kube-proxy => k8s.io/kube-proxy v0.19.3
+	k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.19.3
+	k8s.io/kubectl => k8s.io/kubectl v0.19.3
+	k8s.io/kubelet => k8s.io/kubelet v0.19.3
+	k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.19.3
+	k8s.io/metrics => k8s.io/metrics v0.19.3
+	k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.19.3
+	k8s.io/sample-cli-plugin => k8s.io/sample-cli-plugin v0.19.3
+	k8s.io/sample-controller => k8s.io/sample-controller v0.19.3
 )
--- a/go.sum
+++ b/go.sum
--- a/images/pve-cloud-controller-manager/Dockerfile
+++ b/images/pve-cloud-controller-manager/Dockerfile
@ -0,0 +1,17 @@
+FROM golang:alpine as builder
+RUN adduser -D -g 65532 -u 65532 appuser
+RUN mkdir /build 
+WORKDIR /build 
+COPY go.mod go.mod
+COPY go.sum go.sum
+COPY cmd/pve-cloud-controller-manager/main.go main.go
+COPY pkg/pvecloud/ pkg/pvecloud/
+RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -trimpath -gcflags 'all=-e' -ldflags "-s -w" -o manager main.go
+RUN apk add --no-cache ca-certificates
+FROM scratch
+COPY --from=builder /etc/passwd /etc/passwd
+COPY --from=builder /etc/ssl/cert.pem /etc/ssl/cert.pem
+COPY --from=builder /build/manager /manager
+# appuser
+USER 65532
+ENTRYPOINT ["/manager"]
--- a/images/pve-csr-approver-manager/Dockerfile
+++ b/images/pve-csr-approver-manager/Dockerfile
@ -0,0 +1,17 @@
+FROM golang:alpine as builder
+RUN adduser -D -g 65532 -u 65532 appuser
+RUN mkdir /build 
+WORKDIR /build 
+COPY go.mod go.mod
+COPY go.sum go.sum
+COPY cmd/pve-csr-approver-manager/main.go main.go
+COPY pkg/approver/ pkg/approver
+RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -trimpath -gcflags 'all=-e' -ldflags "-s -w" -o manager main.go
+RUN apk add --no-cache ca-certificates
+FROM scratch
+COPY --from=builder /etc/passwd /etc/passwd
+COPY --from=builder /etc/ssl/cert.pem /etc/ssl/cert.pem
+COPY --from=builder /build/manager /manager
+# appuser
+USER 65532
+ENTRYPOINT ["/manager"]
--- a/pkg/approver/approver.go
+++ b/pkg/approver/approver.go
@ -0,0 +1,160 @@
+package approver
+
+import (
+	"context"
+	"errors"
+	"net"
+	"strings"
+
+	pve "git.giftfish.de/ston1th/pve-go"
+
+	//"k8s.io/client-go/kubernetes/typed/certificates/v1"
+	"github.com/go-logr/logr"
+	certificatesv1beta1 "k8s.io/api/certificates/v1beta1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/client-go/kubernetes/typed/certificates/v1beta1"
+	"k8s.io/kubernetes/pkg/apis/certificates"
+)
+
+var ErrNoKubeletServingCSR = errors.New("CSR is no kubelet serving certificate")
+
+func checkCSR(ctx context.Context, pveClient *pve.Client, request *certificatesv1beta1.CertificateSigningRequest) error {
+	req, err := certificates.ParseCSR(request.Spec.Request)
+	if err != nil {
+		return err
+	}
+	usages := make([]string, len(request.Spec.Usages))
+	for i, v := range request.Spec.Usages {
+		usages[i] = string(v)
+	}
+	if !certificates.IsKubeletServingCSR(req, sets.NewString(usages...)) {
+		return ErrNoKubeletServingCSR
+	}
+	hostname := strings.TrimPrefix(req.Subject.CommonName, "system:node:")
+	ref, err := pveClient.Pool.FindServerByName(ctx, "", hostname)
+	if err != nil {
+		return err
+	}
+	if ref.Name != hostname {
+		return errors.New("CSR hostname and PVE server name do not match")
+	}
+	ok := false
+	for _, dns := range req.DNSNames {
+		if dns == hostname {
+			ok = true
+		}
+	}
+	if len(req.DNSNames) == 0 {
+		ok = true
+	}
+	if !ok {
+		return errors.New("req.DNSNames does not contain hostname")
+	}
+	srv, err := pveClient.Server.GetByRef(ctx, ref)
+	if err != nil {
+		return err
+	}
+	ok = false
+	for _, ip := range req.IPAddresses {
+		for _, ipc := range srv.IPConfig {
+			ipv4, _, err := net.ParseCIDR(ipc.IPv4CIDR)
+			if err == nil {
+				if ip.String() == ipv4.String() {
+					ok = true
+					break
+				}
+			}
+			ipv6, _, err := net.ParseCIDR(ipc.IPv6CIDR)
+			if err == nil {
+				if ip.String() == ipv6.String() {
+					ok = true
+					break
+				}
+			}
+		}
+		if ok {
+			break
+		}
+	}
+	if len(req.IPAddresses) == 0 {
+		ok = true
+	}
+	if !ok {
+		return errors.New("req.IPAddresses does not contain a valid IP")
+	}
+	return nil
+}
+
+var conditionv1beta1 = certificatesv1beta1.CertificateSigningRequestCondition{
+	Type:    certificatesv1beta1.CertificateApproved,
+	Reason:  "AutoApproved",
+	Message: "Auto approving kubelet client certificate in PVE Cluster",
+}
+
+func ApproveV1beta1(log logr.Logger, pveClient *pve.Client, client v1beta1.CertificateSigningRequestInterface, request *certificatesv1beta1.CertificateSigningRequest) error {
+	if len(request.Status.Conditions) > 0 {
+		return nil
+	}
+	ctx := context.Background()
+	err := checkCSR(ctx, pveClient, request)
+	if err == ErrNoKubeletServingCSR {
+		return nil
+	}
+	if err != nil {
+		return err
+	}
+
+	for {
+		if len(request.Status.Conditions) > 0 {
+			return nil
+		}
+		request.Status.Conditions = append(request.Status.Conditions, conditionv1beta1)
+		// Submit the updated CSR.
+		if _, err := client.UpdateApproval(ctx, request, metav1.UpdateOptions{}); err != nil {
+			if strings.Contains(err.Error(), "the object has been modified") {
+				// The CSR might have been updated by a third-party, retry until we
+				// succeed.
+				request, err = client.Get(ctx, request.ObjectMeta.Name, metav1.GetOptions{})
+				if err != nil {
+					return err
+				}
+				continue
+			}
+			return err
+		}
+		log.Info("csr approval successful", "name", request.ObjectMeta.Name)
+		return nil
+	}
+}
+
+//func ApproveV1(pveClient *pve.Client, client v1.CertificateSigningRequestInterface, request *certificates.CertificateSigningRequest) error {
+//	if len(request.Status.Conditions) > 0 {
+//		return nil
+//	}
+//	err := checkCSR(pveClient, request)
+//	if err != nil {
+//		return err
+//	}
+//
+//	for {
+//		if len(request.Status.Conditions) > 0 {
+//			return nil
+//		}
+//		request.Status.Conditions = append(request.Status.Conditions, condition)
+//		// Submit the updated CSR.
+//		if _, err := client.UpdateApproval(request); err != nil {
+//			if strings.Contains(err.Error(), "the object has been modified") {
+//				// The CSR might have been updated by a third-party, retry until we
+//				// succeed.
+//				request, err = client.Get(request.ObjectMeta.Name)
+//				if err != nil {
+//					return err
+//				}
+//				continue
+//			}
+//			return err
+//		}
+//		return nil
+//	}
+//}
--- a/pkg/pvecloud/cloud.go
+++ b/pkg/pvecloud/cloud.go
@ -28,8 +28,9 @@ const (
 )

 type cloud struct {
-	client    *pve.Client
-	instances cloudprovider.Instances
+	client      *pve.Client
+	instances   cloudprovider.Instances
+	instancesV2 cloudprovider.InstancesV2
 	//zones     cloudprovider.Zones
 	//routes    cloudprovider.Routes
 	//network string
@ -74,6 +75,10 @@ func (c *cloud) Instances() (cloudprovider.Instances, bool) {
 	return c.instances, true
 }

+func (c *cloud) InstancesV2() (cloudprovider.InstancesV2, bool) {
+	return c.instancesV2, true
+}
+
 func (c *cloud) Zones() (cloudprovider.Zones, bool) {
 	//return c.zones, true
 	return nil, false
--- a/pkg/pvecloud/instances.go
+++ b/pkg/pvecloud/instances.go
--- a/pkg/pvecloud/instancesv2.go
+++ b/pkg/pvecloud/instancesv2.go
@ -0,0 +1,91 @@
+/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package pvecloud
+
+import (
+	"context"
+	"net"
+
+	cloudprovider "k8s.io/cloud-provider"
+
+	pve "git.giftfish.de/ston1th/pve-go"
+	v1 "k8s.io/api/core/v1"
+)
+
+type instancesV2 struct {
+	client *pve.Client
+}
+
+func newInstancesV2(client *pve.Client) *instancesV2 {
+	return &instancesV2{client}
+}
+
+func (i *instancesV2) InstanceExists(ctx context.Context, node *v1.Node) (exists bool, err error) {
+	server, err := i.getServer(ctx, node)
+	if err != nil && err != cloudprovider.InstanceNotFound {
+		return
+	}
+	err = nil
+	exists = server != nil
+	return
+}
+
+func (i *instancesV2) InstanceShutdown(ctx context.Context, node *v1.Node) (isOff bool, err error) {
+	server, err := i.getServer(ctx, node)
+	if err != nil && err != cloudprovider.InstanceNotFound {
+		return
+	}
+	err = nil
+	isOff = server != nil && server.Status == pve.ServerStatusStopped
+	return
+}
+
+func (i *instancesV2) getServer(ctx context.Context, node *v1.Node) (server *pve.Server, err error) {
+	if node.Spec.ProviderID != "" {
+		server, err = getServerByProviderID(ctx, i.client, node.Spec.ProviderID)
+		return
+	}
+	server, err = getServerByName(ctx, i.client, node.Name)
+	if err != nil {
+		return
+	}
+	return getServerByProviderID(ctx, i.client, server.K8sID())
+}
+
+func (i *instancesV2) nodeAddresses(ctx context.Context, srv *pve.Server) []v1.NodeAddress {
+	var internalIP string
+	ipc := srv.IPConfig.Get(0)
+	if ipc != nil {
+		ip, _, err := net.ParseCIDR(ipc.IPv4CIDR)
+		if err == nil {
+			internalIP = ip.String()
+		}
+	}
+	addrs := []v1.NodeAddress{
+		{Type: v1.NodeHostName, Address: srv.Name},
+		{Type: v1.NodeInternalIP, Address: internalIP},
+	}
+	return addrs
+}
+
+func (i *instancesV2) InstanceMetadata(ctx context.Context, node *v1.Node) (*cloudprovider.InstanceMetadata, error) {
+	server, err := i.getServer(ctx, node)
+	if err != nil {
+		return nil, err
+	}
+	return &cloudprovider.InstanceMetadata{
+		ProviderID:    server.K8sID(),
+		InstanceType:  InstanceType,
+		NodeAddresses: i.nodeAddresses(ctx, server),
+	}, nil
+}
--- a/pkg/pvecloud/util.go
+++ b/pkg/pvecloud/util.go