added minimal image

.gitignore

@@ -0,0 +1 @@
+cmd

MINIMAL.md

@@ -0,0 +1,57 @@
+# Minimal Cloud Images
+
+## Cloud-Init Image
+
+How to prepare a cloud-init image.
+
+### Ubuntu 20.04
+
+Download the latest version of the base image to one of your proxmox nodes:
+
+```
+curl -sSL https://cloud-images.ubuntu.com/minimal/releases/focal/release/ubuntu-20.04-minimal-cloudimg-amd64.img >/tmp/ubuntu-20.04-minimal-cloudimg-amd64.img
+```
+
+Create a cloud-init config to bootstrap the template:
+```
+cat <<'EOF'> /var/lib/vz/snippets/9003_bootstrap
+#cloud-config
+package_update: true
+package_upgrade: true
+ntp:
+  enabled: true
+  servers: ["pool.ntp.org"]
+  ntp_client: systemd-timesyncd
+ssh_deletekeys: true
+ssh_genkeytypes: ["ed25519", "rsa"]
+bootcmd:
+- |
+  sed -i 's/ENABLED=1/ENABLED=0/' /etc/default/motd-news
+  systemctl disable motd-news.service motd-news.timer --now
+runcmd:
+- |
+  curl -sSL https://git.giftfish.de/ston1th/cleanup/raw/branch/master/ubuntu_2004_min.sh >/tmp/cleanup.sh
+  [ "$(sha256sum /tmp/cleanup.sh|cut -d" " -f1)" = "9fbd939eb8d1ef6157a61bb984a5045339ce361596b8a99a8847280cc785b726" ] && sh /tmp/cleanup.sh || echo "error: cleanup script hash does not match" >&2
+power_state:
+  mode: poweroff
+EOF
+```
+
+Create the template VM:
+```
+qm stop 9003; qm destroy 9003
+qm create 9003 --name ubuntu-2004-min --memory 2048 --net0 virtio,bridge=vmbr0
+qm importdisk 9003 /tmp/ubuntu-20.04-minimal-cloudimg-amd64.img local
+qm set 9003 --scsihw virtio-scsi-pci --scsi0 local:9003/vm-9003-disk-0.raw
+qm set 9003 --ide0 local:cloudinit
+qm set 9003 --boot c --bootdisk scsi0
+qm set 9003 --serial0 socket --vga serial0
+qm set 9003 --cicustom "user=local:snippets/9003_bootstrap"
+qm set 9003 --ipconfig0 ip=dhcp,ip6=dhcp
+
+qm start 9003
+sleep 10;while :; do qm status 9003|grep -q stopped && break; done
+qm set 9003 --cicustom ""
+qm set 9003 --ipconfig0 ""
+qm template 9003
+```

README.md

@@ -1,5 +1,233 @@
 # pve-go
 
-PVE-Go is a go library for the Porxmox VE API.
+PVE-Go is a go library for the Proxmox VE API.
 
 The design is based on the Hetzner Clound API implementation: https://github.com/hetznercloud/hcloud-go
+
+## Cloud-Init Image
+
+How to prepare a cloud-init image.
+
+### Ubuntu 20.04
+
+Download the latest version of the base image to one of your proxmox nodes:
+
+```
+curl -sSL https://cloud-images.ubuntu.com/focal/current/focal-server-cloudimg-amd64.img >/tmp/focal-server-cloudimg-amd64.img
+```
+
+Create a cloud-init config to bootstrap the template:
+```
+cat <<'EOF'> /var/lib/vz/snippets/9002_bootstrap
+#cloud-config
+package_update: true
+package_upgrade: true
+ntp:
+  enabled: true
+  servers: ["pool.ntp.org"]
+ssh_deletekeys: true
+ssh_genkeytypes: ["ed25519", "rsa"]
+runcmd:
+- "apt-get -y clean"
+- "apt-get -y autoremove --purge"
+- "bash -c 'rm /home/ubuntu/.ssh/authorized_keys; exit 0'"
+- "bash -c 'find /var/log -type f | while read f; do echo -ne >$f; done; exit 0'"
+- "bash -c 'rm -rf /var/lib/cloud/* /var/tmp/* /tmp/* /tmp/.*-unix; exit 0'"
+power_state:
+  mode: poweroff
+EOF
+```
+
+Create the template VM:
+```
+qm stop 9002; qm destroy 9002
+qm create 9002 --name ubuntu-2004 --memory 2048 --net0 virtio,bridge=vmbr0
+qm importdisk 9002 /tmp/focal-server-cloudimg-amd64.img local
+qm set 9002 --scsihw virtio-scsi-pci --scsi0 local:9002/vm-9002-disk-0.raw
+qm set 9002 --ide0 local:cloudinit
+qm set 9002 --boot c --bootdisk scsi0
+qm set 9002 --serial0 socket --vga serial0
+qm set 9002 --cicustom "user=local:snippets/9002_bootstrap"
+qm set 9002 --ipconfig0 ip=dhcp,ip6=dhcp
+
+qm start 9002
+sleep 10;while :; do qm status 9002|grep -q stopped && break; done
+qm set 9002 --cicustom ""
+qm set 9002 --ipconfig0 ""
+qm template 9002
+```
+
+#### More Image Cleanup
+
+```
+mkdir /etc/systemd/system/motd-news.service.d
+cat <<EOF>/etc/systemd/system/motd-news.service.d/override.conf
+[Service]
+ExecStart=/bin/true
+EOF
+
+mkdir /etc/systemd/timesyncd.conf.d
+cat <<EOF>/etc/systemd/timesyncd.conf.d/ntp.conf
+[Time]
+NTP=pool.ntp.org
+EOF
+
+apt -y purge alsa-topology-conf alsa-ucm-conf apport \
+  apport-symptoms at bash-completion bolt byobu eatmydata command-not-found \
+  eject fonts-ubuntu-console htop install-info landscape-common lxd-agent-loader \
+  motd-news-config nano ntfs-3g pastebinit plymouth plymouth-theme-ubuntu-text \
+  popularity-contest policykit-1 python3-apport os-prober snapd \
+  sound-theme-freedesktop ubuntu-advantage-tools ufw
+
+# optional
+# apt -y purge accountsservice multipath-tools packagekit udisks2 unattended-upgrades
+
+apt -y autoremove
+```
+
+Even more cleanup:
+
+```
+apt -y purge lshw lsof ltrace man-db manpages mdadm mtr-tiny screen sosreport strace tcpdump tmux usbutils whiptail
+apt -y autoremove
+```
+
+### Ubuntu 18.04
+
+Download the latest version of the base image to one of your proxmox nodes:
+
+```
+curl -sSL https://cloud-images.ubuntu.com/bionic/current/bionic-server-cloudimg-amd64.img >/tmp/bionic-server-cloudimg-amd64.img
+```
+
+Create a cloud-init config to bootstrap the template:
+```
+cat <<'EOF'> /var/lib/vz/snippets/9000_bootstrap
+#cloud-config
+package_update: true
+package_upgrade: true
+ntp:
+  enabled: true
+  servers: ["pool.ntp.org"]
+ssh_deletekeys: true
+ssh_genkeytypes: ["ed25519", "rsa"]
+runcmd:
+- "apt-get -y clean"
+- "apt-get -y autoremove --purge"
+- "bash -c 'rm /home/ubuntu/.ssh/authorized_keys; exit 0'"
+- "bash -c 'find /var/log -type f | while read f; do echo -ne >$f; done; exit 0'"
+- "bash -c 'rm -rf /var/lib/cloud/*; exit 0'"
+power_state:
+  mode: poweroff
+EOF
+```
+
+Create the template VM:
+```
+qm stop 9000; qm destroy 9000
+qm create 9000 --name ubuntu-1804 --memory 2048 --net0 virtio,bridge=vmbr0
+qm importdisk 9000 /tmp/bionic-server-cloudimg-amd64.img local
+qm set 9000 --scsihw virtio-scsi-pci --scsi0 local:9000/vm-9000-disk-0.raw
+qm set 9000 --ide0 local:cloudinit
+qm set 9000 --boot c --bootdisk scsi0
+qm set 9000 --serial0 socket --vga serial0
+qm set 9000 --cicustom "user=local:snippets/9000_bootstrap"
+qm set 9000 --ipconfig0 ip=dhcp,ip6=dhcp
+
+qm start 9000
+sleep 10;while :; do qm status 9000|grep -q stopped && break; done
+qm set 9000 --cicustom ""
+qm set 9000 --ipconfig0 ""
+qm template 9000
+```
+
+#### More Image Cleanup
+
+```
+apt -y purge apport apport-symptoms bash-completion byobu htop landscape-common lxcfs lxd lxd-client motd-news-config nano ntfs-3g os-prober ufw
+apt -y autoremove
+```
+
+Even more cleanup:
+
+```
+apt -y purge lshw lsof ltrace man-db manpages mdadm mtr-tiny screen sosreport strace tcpdump tmux usbutils whiptail
+apt -y autoremove
+```
+
+## Minimal Cloud Image
+
+### Ubuntu 20.04
+
+```
+curl -sSL https://cloud-images.ubuntu.com/focal/current/focal-server-cloudimg-amd64.img >/tmp/focal-server-cloudimg-amd64.img
+
+cat <<'EOF'> /var/lib/vz/snippets/9002_bootstrap
+#cloud-config
+package_update: true
+package_upgrade: true
+ssh_deletekeys: true
+ssh_genkeytypes: ["ed25519", "rsa"]
+runcmd:
+- "curl -sSL https://git.giftfish.de/ston1th/cleanup/raw/branch/master/ubuntu_2004_full.sh >/tmp/cleanup.sh"
+- |
+  bash -c '[ "$(sha256sum /tmp/cleanup.sh|cut -d" " -f1)" = "fe2827d03ebba6e058ed7b9e56116eef78a45606f2e5f2956b2b2f495f042724" ] && sh /tmp/cleanup.sh || echo "error: cleanup script hash does not match" >&2'
+power_state:
+  mode: poweroff
+EOF
+
+qm stop 9002; qm destroy 9002
+qm create 9002 --name ubuntu-2004 --memory 2048 --net0 virtio,bridge=vmbr0
+qm importdisk 9002 /tmp/focal-server-cloudimg-amd64.img local
+qm set 9002 --scsihw virtio-scsi-pci --scsi0 local:9002/vm-9002-disk-0.raw
+qm set 9002 --ide0 local:cloudinit
+qm set 9002 --boot c --bootdisk scsi0
+qm set 9002 --serial0 socket --vga serial0
+qm set 9002 --cicustom "user=local:snippets/9002_bootstrap"
+qm set 9002 --ipconfig0 ip=dhcp,ip6=dhcp
+
+qm start 9002
+sleep 10;while :; do qm status 9002|grep -q stopped && break; done
+qm set 9002 --cicustom ""
+qm set 9002 --ipconfig0 ""
+qm template 9002
+```
+
+### Ubuntu 18.04
+
+```
+curl -sSL https://cloud-images.ubuntu.com/minimal/releases/bionic/release/ubuntu-18.04-minimal-cloudimg-amd64.img >/tmp/ubuntu-18.04-minimal-cloudimg-amd64.img
+
+cat <<'EOF'> /var/lib/vz/snippets/9001_bootstrap
+#cloud-config
+package_update: true
+package_upgrade: true
+ntp:
+  enabled: true
+  servers: ["pool.ntp.org"]
+ssh_deletekeys: true
+ssh_genkeytypes: ["ed25519", "rsa"]
+runcmd:
+- "curl -sSL https://git.giftfish.de/ston1th/cleanup/raw/branch/master/ubuntu_1804.sh >/tmp/cleanup.sh"
+- |
+  bash -c '[ "$(sha256sum /tmp/cleanup.sh|cut -d" " -f1)" = "ecb2ae04cdd6b91fdc56a5db2015f401e474196b8eaaaf8e3aa5dc4b5e4c087d" ] && sh /tmp/cleanup.sh || echo "error: cleanup script hash does not match" >&2'
+power_state:
+  mode: poweroff
+EOF
+
+qm stop 9001; qm destroy 9001
+qm create 9001 --name ubuntu-1804-min --memory 2048 --net0 virtio,bridge=vmbr0
+qm importdisk 9001 /tmp/ubuntu-18.04-minimal-cloudimg-amd64.img local
+qm set 9001 --scsihw virtio-scsi-pci --scsi0 local:9001/vm-9001-disk-0.raw
+qm set 9001 --ide0 local:cloudinit
+qm set 9001 --boot c --bootdisk scsi0
+qm set 9001 --serial0 socket --vga serial0
+qm set 9001 --cicustom "user=local:snippets/9001_bootstrap"
+qm set 9001 --ipconfig0 ip=dhcp,ip6=dhcp
+
+qm start 9001
+sleep 10;while :; do qm status 9001|grep -q stopped && break; done
+qm set 9001 --cicustom ""
+qm set 9001 --ipconfig0 ""
+qm template 9001
+```

TODO

@@ -0,0 +1 @@
+remove 18.04 fat cloud images

client.go

@@ -9,10 +9,11 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net"
 	"net/http"
 	"net/http/httputil"
+	"os"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -35,21 +36,22 @@ func ExponentialBackoff(b float64, d time.Duration) BackoffFunc {
 */
 
 type Client struct {
-	sync.Mutex
+	mu        sync.Mutex
-	endpoint string
+	endpoints []string
-	username string
+	username  string
-	password string
+	password  string
-	session  session
+	session   session
 	//pollInterval time.Duration
 	//backoffFunc  BackoffFunc
 	httpClient  *http.Client
 	debugWriter io.Writer
 	insecure    bool
 
-	Server ServerClient
+	Server  ServerClient
-	Task   TaskClient
+	Task    TaskClient
-	Pool   PoolClient
+	Pool    PoolClient
-	Node   NodeClient
+	Node    NodeClient
+	Snippet SnippetClient
 }
 
 type session struct {
@@ -63,7 +65,15 @@ type ClientOption func(*Client)
 
 func WithEndpoint(endpoint string) ClientOption {
 	return func(client *Client) {
-		client.endpoint = strings.TrimRight(endpoint, "/")
+		client.endpoints = append(client.endpoints, strings.TrimRight(endpoint, "/"))
+	}
+}
+
+func WithEndpoints(endpoints []string) ClientOption {
+	return func(client *Client) {
+		for _, e := range endpoints {
+			client.endpoints = append(client.endpoints, strings.TrimRight(e, "/"))
+		}
 	}
 }
 
@@ -106,6 +116,72 @@ func WithCredentials(username, password string) ClientOption {
 	}
 }
 
+var (
+	ErrMissingEndpointsEnv = errors.New("missing environment variable PVE_ENDPOINTS")
+	ErrMissingUserEnv      = errors.New("missing environment variable PVE_USER")
+	ErrMissingPasswordEnv  = errors.New("missing environment variable PVE_PASSWORD")
+
+	ErrMissingEndpoints = errors.New("missing Endpoints in config")
+	ErrMissingUser      = errors.New("missing User in config")
+	ErrMissingPassword  = errors.New("missing Password in config")
+)
+
+type Config struct {
+	Endpoints []string `json:"endpoints"`
+	User      string   `json:"user"`
+	Password  string   `json:"password"`
+	Insecure  bool     `json:"insecure"`
+}
+
+func ClientOptionsFromConfig(cfg Config) (options []ClientOption, err error) {
+	if len(cfg.Endpoints) == 0 {
+		err = ErrMissingEndpoints
+		return
+	}
+	if cfg.User == "" {
+		err = ErrMissingUser
+		return
+	}
+	if cfg.Password == "" {
+		err = ErrMissingPassword
+		return
+	}
+	options = []ClientOption{
+		WithEndpoints(cfg.Endpoints),
+		WithCredentials(cfg.User, cfg.Password),
+		WithInsecureClient(cfg.Insecure),
+	}
+	return
+}
+
+func ClientOptionsFromEnv() (options []ClientOption, err error) {
+	endpoints := os.Getenv("PVE_ENDPOINTS")
+	if endpoints == "" {
+		err = ErrMissingEndpointsEnv
+		return
+	}
+	user := os.Getenv("PVE_USER")
+	if user == "" {
+		err = ErrMissingUserEnv
+		return
+	}
+
+	pass := os.Getenv("PVE_PASSWORD")
+	if pass == "" {
+		err = ErrMissingPasswordEnv
+		return
+	}
+
+	insecure, _ := strconv.ParseBool(os.Getenv("PVE_INSECURE"))
+
+	options = []ClientOption{
+		WithEndpoints(strings.Split(endpoints, ",")),
+		WithCredentials(user, pass),
+		WithInsecureClient(insecure),
+	}
+	return
+}
+
 func NewClient(options ...ClientOption) *Client {
 	client := &Client{}
 	/*
@@ -121,8 +197,14 @@ func NewClient(options ...ClientOption) *Client {
 	if client.httpClient == nil {
 		client.httpClient = &http.Client{Transport: &http.Transport{
 			DialContext: (&net.Dialer{
-				Timeout: 30 * time.Second,
+				Timeout:   30 * time.Second,
+				KeepAlive: 30 * time.Second,
 			}).DialContext,
+			ForceAttemptHTTP2:     true,
+			MaxIdleConns:          100,
+			IdleConnTimeout:       90 * time.Second,
+			TLSHandshakeTimeout:   10 * time.Second,
+			ExpectContinueTimeout: 1 * time.Second,
 			TLSClientConfig: &tls.Config{
 				InsecureSkipVerify: client.insecure,
 			},
@@ -133,20 +215,27 @@ func NewClient(options ...ClientOption) *Client {
 	client.Task = TaskClient{client: client}
 	client.Pool = PoolClient{client: client}
 	client.Node = NodeClient{client: client}
+	client.Snippet = SnippetClient{client: client}
 
 	return client
 }
+func (c *Client) getEndpoint() string {
+	if len(c.endpoints) == 0 {
+		return ""
+	}
+	return c.endpoints[0]
+}
 
 func (c *Client) Auth() (err error) {
 	if !time.Now().After(c.session.Time) {
 		return
 	}
-	c.Lock()
+	c.mu.Lock()
-	defer c.Unlock()
+	defer c.mu.Unlock()
 	if !time.Now().After(c.session.Time) {
 		return
 	}
-	u := c.endpoint + "/access/ticket"
+	u := c.getEndpoint() + "/access/ticket"
 	body := httpbody{"username": c.username, "password": c.password}
 	r, err := http.NewRequest("POST", u, body.Reader())
 	if err != nil {
@@ -172,7 +261,7 @@ func (c *Client) NewRequest(ctx context.Context, method, path string, body io.Re
 	if err != nil {
 		return nil, err
 	}
-	url := c.endpoint + path
+	url := c.getEndpoint() + path
 	req, err := http.NewRequest(method, url, body)
 	if err != nil {
 		return nil, err
@@ -188,25 +277,26 @@ func (c *Client) NewRequest(ctx context.Context, method, path string, body io.Re
 var HTTPErr = errors.New("http error")
 
 func (c *Client) Do(r *http.Request, v interface{}) (resp *http.Response, err error) {
+	if c.debugWriter != nil {
+		dumpReq, err := httputil.DumpRequestOut(r, true)
+		if err != nil {
+			return &http.Response{}, err
+		}
+		fmt.Fprintf(c.debugWriter, "--- Request:\n%s\n\n", dumpReq)
+	}
 	resp, err = c.httpClient.Do(r)
 	if err != nil {
 		return
 	}
-	body, err := ioutil.ReadAll(resp.Body)
+	body, err := io.ReadAll(resp.Body)
 	if err != nil {
 		resp.Body.Close()
 		return
 	}
 	resp.Body.Close()
-	resp.Body = ioutil.NopCloser(bytes.NewReader(body))
+	resp.Body = io.NopCloser(bytes.NewReader(body))
 
 	if c.debugWriter != nil {
-		dumpReq, err := httputil.DumpRequest(r, true)
-		if err != nil {
-			return resp, err
-		}
-		fmt.Fprintf(c.debugWriter, "--- Request:\n%s\n\n", dumpReq)
-
 		dumpResp, err := httputil.DumpResponse(resp, true)
 		if err != nil {
 			return resp, err

docs/dump.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+#===================================================================================
+#
+# FILE: dump.sh
+# USAGE: dump.sh [-i interface] [tcpdump-parameters]
+# DESCRIPTION: tcpdump on any interface and add the prefix [Interace:xy] in front of the dump data.
+# OPTIONS: same as tcpdump
+# REQUIREMENTS: tcpdump, sed, ifconfig, kill, awk, grep, posix regex matching
+# BUGS:  ---
+# FIXED: - In 1.0 The parameter -w would not work without -i parameter as multiple tcpdumps are started.
+#        - In 1.1 VLAN's would not be shown if a single interface was dumped.
+# NOTES: ---
+#        - 1.2 git initial
+# AUTHOR: Sebastian Haas
+# COMPANY: pharma mall
+# VERSION: 1.2
+# CREATED: 16.09.2014
+# REVISION: 22.09.2014
+#
+#===================================================================================
+
+# When this exits, exit all background processes:
+trap 'kill $(jobs -p) &> /dev/null && sleep 0.2 &&  echo ' EXIT
+# Create one tcpdump output per interface and add an identifier to the beginning of each line:
+if [[ $@ =~ -i[[:space:]]?[^[:space:]]+ ]]; then
+    tcpdump -l $@ | sed 's/^/[Interface:'"${BASH_REMATCH[0]:2}"'] /' &
+else
+    for interface in $(ifconfig | grep '^[a-z0-9]' | awk '{print $1}')
+    do
+       tcpdump -l -i $interface -nn $@ | sed 's/^/[Interface:'"$interface"']    /' &
+    done
+fi
+# wait .. until CTRL+C
+wait

docs/proxmox_do.txt

@@ -0,0 +1,206 @@
+apt update && apt install -y curl tcpdump
+echo "deb http://download.proxmox.com/debian/pve buster pve-no-subscription" > /etc/apt/sources.list.d/pve-install-repo.list
+curl -sSL http://download.proxmox.com/debian/proxmox-ve-release-6.x.gpg | apt-key add -
+
+rm /etc/network/interfaces.d/50-cloud-init
+echo "alias ll='ls -lah'" >>/root/.bashrc
+cat <<EOF>/root/.vimrc
+syntax on
+set mouse=
+set ttymouse=
+EOF
+# source /etc/network/interfaces.d/*
+sed -i -e '24,29d' /etc/network/interfaces
+sed -i '1isource /etc/network/interfaces.d/*' /etc/network/interfaces
+iface=eth1
+ip=$(ip a show $iface | grep "inet " | cut -d" " -f6 | cut -d"/" -f1)
+hw=$(ip a show $iface | grep "link/" | cut -d" " -f6)
+sed -i "s/127.0.1.1/$ip/" /etc/hosts
+cat <<EOF>>/etc/network/interfaces
+auto $iface
+iface $iface inet static
+    hwaddress $hw
+
+auto vmbr0
+iface vmbr0 inet static
+    address $ip/24
+    bridge-ports $iface
+    bridge-stp off
+    bridge-fd 0
+EOF
+
+apt update && apt purge cloud-init -y && apt install ifupdown2 -y && systemctl restart networking
+
+# install ifupdown2 ?
+
+DEBIAN_FRONTEND=noninteractive apt dist-upgrade -y -o DPkg::options::="--force-confold"; apt autoremove -y; DEBIAN_FRONTEND=noninteractive apt install --assume-yes -o DPkg::options::="--force-confdef" -y proxmox-ve vim isc-dhcp-server libpve-network-perl;
+#systemctl disable hc-net-ifup@
+reboot
+apt remove -y os-prober linux-image-amd64 'linux-image-4.19*'; apt autoremove -y;
+
+cat <<EOF>>/etc/network/interfaces
+
+auto vmbr1
+iface vmbr1 inet static
+    address 10.0.0.254/24
+    bridge-ports none
+    bridge-stp off
+    bridge-fd 0
+    post-up echo 1 > /proc/sys/net/ipv4/ip_forward
+    post-up iptables -t nat -A POSTROUTING -s '10.0.0.0/24' -o eth0 -j MASQUERADE
+    post-down iptables -t nat -F
+EOF
+systemctl restart networking
+
+pvecm create pmx -link0 172.16.0.2
+
+cat <<EOF>/etc/default/isc-dhcp-server
+INTERFACESv4="vmbr1"
+INTERFACESv6=""
+EOF
+cat <<EOF>/etc/dhcp/dhcpd.conf
+default-lease-time 600;
+max-lease-time 7200;
+
+subnet 10.0.0.0 netmask 255.255.255.0 {
+  range 10.0.0.1 10.0.0.253;
+  option domain-name-servers 1.1.1.1, 8.8.8.8;
+  option routers 10.0.0.254;
+}
+EOF
+systemctl restart isc-dhcp-server
+
+# controllers
+# id: evpn
+# asn: 65000
+# peers: 172.16.0.2,172.16.0.3,172.16.0.4
+
+# zones
+# id: evpn
+# tag: 10000
+# exit-nodes: pmx01, pmx02
+# controller: evpn
+# mtu 1400
+
+# vnets
+# name: vnet1
+# zone: evpn
+# tag: 11000
+# gateway: 192.168.1.1
+
+# name: vnet2
+# zone: evpn
+# tag: 12000
+# gateway: 192.168.2.1
+
+cd /var/lib/vz/template/iso
+curl -sSLO https://dl-cdn.alpinelinux.org/alpine/v3.14/releases/x86_64/alpine-virt-3.14.1-x86_64.iso
+
+ip link set up dev eth0
+ip link set mtu 1450 dev eth0
+ip a add 192.168.1.2/24 dev eth0
+ip route add default via 192.168.1.1
+
+ip a add 192.168.2.2/24 dev eth0
+ip route add default via 192.168.2.1
+
+rm /etc/apt/sources.list.d/pve-enterprise.list
+sed -i 's/buster\/updates/bullseye-security/g;s/buster/bullseye/g' /etc/apt/sources.list
+sed -i -e 's/buster/bullseye/g' /etc/apt/sources.list.d/pve-install-repo.list
+
+apt update && apt dist-upgrade
+
+
+cd /tmp
+curl -sSLO https://mutulin1.odiso.net/frr_7.5.1-2+pve_amd64.deb
+curl -sSLO https://mutulin1.odiso.net/frr-pythontools_7.5.1-2+pve_all.deb
+dpkg -i frr_7.5.1-2+pve_amd64.deb
+dpkg -i frr-pythontools_7.5.1-2+pve_all.deb
+
+ip route add table vrf_evpn default dev eth0 via 164.90.160.
+
+# /etc/frr/frr.conf
+ bgp listen range 172.16.0.0/24
+
+pvecm add 172.16.0.2
+
+# vxlan zone
+# id: vxlan
+# peer: 172.16.0.2,172.16.0.3,172.16.0.4
+# mut: 1400
+
+# vnets
+# name: vnet1
+# zone: vxlan
+# tag: 100000
+
+fallocate -l 10G /var/osd.img
+losetup -l -P /dev/loop1 /var/osd.img
+wipefs -a /dev/loop1
+lsblk
+sed -i "s/'mpath'/'mpath', 'loop'/g" /usr/lib/python3/dist-packages/ceph_volume/util/disk.py
+
+/usr/share/perl5/PVE/Diskmanage.pm
+get_disks
+$dev !~ m/^loop\d+$/;
+
+get_sysdir_info
+my $data = {};
+if ($sysdir =~ /loop\d+/) {
+$data->{size} = 1024;
+$data->{rotational} = 1;
+$data->{vendor} = 'unknown';
+$data->{model} = 'unknown';
+return $data;
+}
+systemctl restart pvedaemon
+
+
+cat <<'EOF'> /var/lib/vz/snippets/9000_bootstrap
+#cloud-config
+package_update: true
+package_upgrade: true
+ntp:
+  enabled: true
+  servers: ["pool.ntp.org"]
+ssh_deletekeys: true
+ssh_genkeytypes: ["ed25519", "rsa"]
+runcmd:
+- "apt-get -y clean"
+- "apt-get -y autoremove --purge"
+- "bash -c 'rm /home/ubuntu/.ssh/authorized_keys; exit 0'"
+- "bash -c 'find /var/log -type f | while read f; do echo -ne >$f; done; exit 0'"
+- "bash -c 'rm -rf /var/lib/cloud/*; exit 0'"
+power_state:
+  mode: poweroff
+EOF
+
+
+curl -sSL https://cloud-images.ubuntu.com/bionic/current/bionic-server-cloudimg-amd64.img >/tmp/bionic-server-cloudimg-amd64.img
+
+qm destroy 9000
+qm create 9000 --memory 512 --net0 virtio,bridge=vmbr1
+qm importdisk 9000 /tmp/bionic-server-cloudimg-amd64.img local
+qm set 9000 --scsihw virtio-scsi-pci --scsi0 local:9000/vm-9000-disk-0.raw
+qm set 9000 --ide0 local:cloudinit
+qm set 9000 --boot c --bootdisk scsi0
+qm set 9000 --serial0 socket --vga serial0
+qm set 9000 --cicustom "user=local:snippets/9000_bootstrap"
+qm set 9000 --ipconfig0 ip=dhcp,ip6=dhcp
+qm set 9000 --kvm 0
+qm set 9000 --balloon 0
+qm set 9000 --cpu kvm64
+
+qm start 9000
+sleep 10;while :; do qm status 9000|grep -q stopped && break; done
+qm set 9000 --cicustom ""
+qm set 9000 --ipconfig0 ""
+qm template 9000
+
+
+# iptables
+iptables -t nat -A PREROUTING -i eth0 -p tcp --dport 222 -j DNAT --to 10.0.0.11:22
+iptables -t nat -A POSTROUTING -s '10.0.0.0/24' -o eth0 -j MASQUERADE
+
+# ceph
+https://www.netways.de/blog/2018/11/14/ceph-mimic-using-loop-devices-as-osd/

docs/proxmox_hcloud.txt

@@ -0,0 +1,177 @@
+echo "deb http://download.proxmox.com/debian/pve buster pve-no-subscription" > /etc/apt/sources.list.d/pve-install-repo.list
+curl -sSL http://download.proxmox.com/debian/proxmox-ve-release-6.x.gpg | apt-key add -
+
+cat <<EOF>>/etc/network/interfaces
+auto eth0
+iface eth0 inet dhcp
+    dns-nameservers 1.1.1.1 8.8.8.8
+
+EOF
+rm /etc/network/interfaces.d/50-cloud-init
+echo "alias ll='ls -lah'" >>/root/.bashrc
+cat <<EOF>/root/.vimrc
+syntax on
+set mouse=
+set ttymouse=
+EOF
+iface=enp7s0
+ip=$(ip a show $iface | grep "inet " | cut -d" " -f6 | cut -d"/" -f1)
+sed -i "s/127.0.1.1/$ip/" /etc/hosts
+cat <<EOF>>/etc/network/interfaces
+auto $iface
+iface $iface inet static
+    mtu 1450
+
+auto vmbr0
+iface vmbr0 inet static
+    address $ip/32
+    bridge-ports $iface
+    bridge-stp off
+    bridge-fd 0
+    mtu 1450
+    post-up ip route add 172.16.0.1/32 dev vmbr0
+    post-up ip route add 172.16.0.0/24 via 172.16.0.1
+EOF
+
+apt update && apt purge cloud-init -y && apt install ifupdown2 -y && systemctl restart networking
+
+# install ifupdown2 ?
+
+DEBIAN_FRONTEND=noninteractive apt dist-upgrade -y -o DPkg::options::="--force-confold"; apt autoremove -y; DEBIAN_FRONTEND=noninteractive apt install --assume-yes -o DPkg::options::="--force-confdef" -y proxmox-ve vim isc-dhcp-server libpve-network-perl;
+systemctl disable hc-net-ifup@
+reboot
+apt remove -y os-prober linux-image-amd64 'linux-image-4.19*'; apt autoremove -y;
+
+cat <<EOF>>/etc/network/interfaces
+
+auto vmbr1
+iface vmbr1 inet static
+    address 10.0.0.254/24
+    bridge-ports none
+    bridge-stp off
+    bridge-fd 0
+    post-up echo 1 > /proc/sys/net/ipv4/ip_forward
+    post-up iptables -t nat -A POSTROUTING -s '10.0.0.0/24' -o eth0 -j MASQUERADE
+    post-down iptables -t nat -F
+EOF
+systemctl restart networking
+
+cat <<EOF>/etc/default/isc-dhcp-server
+INTERFACESv4="vmbr1"
+INTERFACESv6=""
+EOF
+cat <<EOF>/etc/dhcp/dhcpd.conf
+default-lease-time 600;
+max-lease-time 7200;
+
+subnet 10.0.0.0 netmask 255.255.255.0 {
+  range 10.0.0.1 10.0.0.253;
+  option domain-name-servers 1.1.1.1, 8.8.8.8;
+  option routers 10.0.0.254;
+}
+EOF
+systemctl restart isc-dhcp-server
+
+# controllers
+# id: router
+# asn: 65534
+# peers: 172.16.0.2,172.16.0.3,172.16.0.4
+# gateway-nodes: pmx01, pmx02
+
+# zones
+# id: evpn
+# tag: 10000
+# controller: router
+# mtu 1400
+
+# vnets
+# name: vnet2
+# zone: evpn
+# tag: 11000
+# anycast: 10.0.0.1/24
+
+# /etc/frr/frr.conf
+ bgp listen range 172.16.0.0/24
+
+pvecm add 172.16.0.2
+
+# vxlan zone
+# id: vxlan
+# peer: 172.16.0.2,172.16.0.3,172.16.0.4
+# mut: 1400
+
+# vnets
+# name: vnet1
+# zone: vxlan
+# tag: 100000
+
+fallocate -l 10G /var/osd.img
+losetup -l -P /dev/loop1 /var/osd.img
+wipefs -a /dev/loop1
+lsblk
+sed -i "s/'mpath'/'mpath', 'loop'/g" /usr/lib/python3/dist-packages/ceph_volume/util/disk.py
+
+/usr/share/perl5/PVE/Diskmanage.pm
+get_disks
+$dev !~ m/^loop\d+$/;
+
+get_sysdir_info
+my $data = {};
+if ($sysdir =~ /loop\d+/) {
+$data->{size} = 1024;
+$data->{rotational} = 1;
+$data->{vendor} = 'unknown';
+$data->{model} = 'unknown';
+return $data;
+}
+systemctl restart pvedaemon
+
+
+cat <<'EOF'> /var/lib/vz/snippets/9000_bootstrap
+#cloud-config
+package_update: true
+package_upgrade: true
+ntp:
+  enabled: true
+  servers: ["pool.ntp.org"]
+ssh_deletekeys: true
+ssh_genkeytypes: ["ed25519", "rsa"]
+runcmd:
+- "apt-get -y clean"
+- "apt-get -y autoremove --purge"
+- "bash -c 'rm /home/ubuntu/.ssh/authorized_keys; exit 0'"
+- "bash -c 'find /var/log -type f | while read f; do echo -ne >$f; done; exit 0'"
+- "bash -c 'rm -rf /var/lib/cloud/*; exit 0'"
+power_state:
+  mode: poweroff
+EOF
+
+
+curl -sSL https://cloud-images.ubuntu.com/bionic/current/bionic-server-cloudimg-amd64.img >/tmp/bionic-server-cloudimg-amd64.img
+
+qm destroy 9000
+qm create 9000 --memory 512 --net0 virtio,bridge=vmbr1
+qm importdisk 9000 /tmp/bionic-server-cloudimg-amd64.img local
+qm set 9000 --scsihw virtio-scsi-pci --scsi0 local:9000/vm-9000-disk-0.raw
+qm set 9000 --ide0 local:cloudinit
+qm set 9000 --boot c --bootdisk scsi0
+qm set 9000 --serial0 socket --vga serial0
+qm set 9000 --cicustom "user=local:snippets/9000_bootstrap"
+qm set 9000 --ipconfig0 ip=dhcp,ip6=dhcp
+qm set 9000 --kvm 0
+qm set 9000 --balloon 0
+qm set 9000 --cpu kvm64
+
+qm start 9000
+sleep 10;while :; do qm status 9000|grep -q stopped && break; done
+qm set 9000 --cicustom ""
+qm set 9000 --ipconfig0 ""
+qm template 9000
+
+
+# iptables
+iptables -t nat -A PREROUTING -i eth0 -p tcp --dport 222 -j DNAT --to 10.0.0.11:22
+iptables -t nat -A POSTROUTING -s '10.0.0.0/24' -o eth0 -j MASQUERADE
+
+# ceph
+https://www.netways.de/blog/2018/11/14/ceph-mimic-using-loop-devices-as-osd/

helper.go

@@ -3,6 +3,7 @@
 package pve
 
 import (
+	"bytes"
 	"encoding/json"
 	"errors"
 	"io"
@@ -43,3 +44,25 @@ func (b httpbody) Reader() io.Reader {
 	}
 	return strings.NewReader(data.Encode())
 }
+
+type unreadBuffer struct {
+	b    *bytes.Buffer
+	save []byte
+}
+
+func newUnreadBuffer() (ub *unreadBuffer, buf *bytes.Buffer) {
+	buf = new(bytes.Buffer)
+	ub = &unreadBuffer{b: buf}
+	return
+}
+
+func (u *unreadBuffer) Save() {
+	u.save = u.b.Bytes()
+}
+
+func (u *unreadBuffer) Reset(b **bytes.Buffer) {
+	if u.save != nil && b != nil {
+		u.b = bytes.NewBuffer(u.save)
+		*b = u.b
+	}
+}

ipconfig.go

@@ -8,9 +8,25 @@ import (
 
 type IPConfigs []*IPConfig
 
-func (c IPConfigs) Get(index int) *IPConfig {
+func (c *IPConfigs) Set(index int, ipc *IPConfig) {
-	if len(c)-1 >= index {
+	l := len(*c)
-		return c[index]
+	if index >= l {
+		n := make(IPConfigs, index+1)
+		copy(n, *c)
+		n[index] = ipc
+		*c = n
+		return
+	}
+	if l == 0 {
+		*c = append(*c, ipc)
+		return
+	}
+	(*c)[index] = ipc
+}
+
+func (c *IPConfigs) Get(index int) *IPConfig {
+	if len(*c)-1 >= index {
+		return (*c)[index]
 	}
 	return nil
 }
@@ -20,7 +36,6 @@ type IPConfig struct {
 	IPv4Gateway string
 	IPv6CIDR    string
 	IPv6Gateway string
-	MTU         string
 }
 
 func (c *IPConfig) String() string {
@@ -37,9 +52,6 @@ func (c *IPConfig) String() string {
 			str = append(str, "gw6="+c.IPv6Gateway)
 		}
 	}
-	if c.MTU != "" {
-		str = append(str, "mtu="+c.MTU)
-	}
 	return strings.Join(str, ",")
 }
 
@@ -63,8 +75,6 @@ func parseIPConfig(s string) (c *IPConfig) {
 			c.IPv6CIDR = v[1]
 		case "gw6":
 			c.IPv6Gateway = v[1]
-		case "mtu":
-			c.MTU = v[1]
 		}
 	}
 	return

network.go

@@ -9,9 +9,25 @@ import (
 
 type NetworkDevices []*NetworkDevice
 
-func (c NetworkDevices) Get(index int) *NetworkDevice {
+func (c *NetworkDevices) Set(index int, d *NetworkDevice) {
-	if len(c)-1 >= index {
+	l := len(*c)
-		return c[index]
+	if index >= l {
+		n := make(NetworkDevices, index+1)
+		copy(n, *c)
+		n[index] = d
+		*c = n
+		return
+	}
+	if l == 0 {
+		*c = append(*c, d)
+		return
+	}
+	(*c)[index] = d
+}
+
+func (c *NetworkDevices) Get(index int) *NetworkDevice {
+	if len(*c)-1 >= index {
+		return (*c)[index]
 	}
 	return nil
 }
@@ -20,10 +36,15 @@ type NetworkDevice struct {
 	Type       string
 	MACAddress string
 	Bridge     string
+	MTU        string
 }
 
 func (n *NetworkDevice) String() string {
-	return fmt.Sprintf("%s=%s,bridge=%s", n.Type, n.MACAddress, n.Bridge)
+	var mtu string
+	if n.MTU != "" {
+		mtu = "," + n.MTU
+	}
+	return fmt.Sprintf("%s=%s,bridge=%s%s", n.Type, n.MACAddress, n.Bridge, mtu)
 }
 func parseNetworkDevice(s string) (n *NetworkDevice) {
 	cfg := strings.Split(s, ",")
@@ -41,7 +62,8 @@ func parseNetworkDevice(s string) (n *NetworkDevice) {
 	for _, o := range cfg {
 		if strings.HasPrefix(o, "bridge=") {
 			n.Bridge = o[7:]
-			break
+		} else if strings.HasPrefix(o, "mtu=") {
+			n.MTU = o[4:]
 		}
 	}
 	return

node.go

@@ -6,9 +6,15 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"sort"
 )
 
-var ErrServerNotFound = errors.New("server not found")
+var (
+	ErrNodesOffline        = errors.New("one or more nodes are offline")
+	ErrNodesNotSearched    = errors.New("one or more nodes could not be searched")
+	ErrUnschedulable       = errors.New("no schedulable node found")
+	ErrTooManyServersFound = errors.New("too many servers found with the same name")
+)
 
 type NodeStatus string
 
@@ -19,18 +25,163 @@ const (
 )
 
 type Node struct {
-	Name   string     `json:"node"`
+	Name       string        `json:"node"`
-	Status NodeStatus `json:"status"`
+	Status     NodeStatus    `json:"status"`
+	CPU        float64       `json:"cpu"`
+	MaxCPU     float64       `json:"maxcpu"`
+	Mem        uint64        `json:"mem"`
+	MaxMem     uint64        `json:"maxmem"`
+	Disk       uint64        `json:"disk"`
+	MaxDisk    uint64        `json:"maxdisk"`
+	serverList ServerRefList `json:"-"`
+}
+
+func (n *Node) memFreePercent() float64 {
+	return float64(n.Mem) / float64(n.MaxMem) * 100
+}
+
+func (n *Node) memUsedPercent() float64 {
+	return float64(n.MaxMem-n.Mem) / float64(n.MaxMem) * 100
+}
+
+func (n *Node) weight() int {
+	return int(n.memUsedPercent()) + (len(n.serverList) * 10)
 }
 
 type NodeList []*Node
 
-type ServerList []*Server
+func (nl NodeList) sortByWeight() {
+	sort.Sort(weightSorter(nl))
+}
+
+func (nl *NodeList) remove(i int) {
+	(*nl)[i] = (*nl)[len(*nl)-1]
+	*nl = (*nl)[:len(*nl)-1]
+}
+
+func (nl *NodeList) filter(filters ...NodeFilter) {
+	for _, f := range filters {
+		f(nl)
+	}
+}
+
+func (nl *NodeList) getServerList(ctx context.Context, c *NodeClient) (err error) {
+	for _, n := range *nl {
+		n.serverList, err = c.ListServers(ctx, n)
+		if err != nil {
+			return
+		}
+	}
+	return
+}
+
+type weightSorter NodeList
+
+func (s weightSorter) Len() int { return len(s) }
+func (s weightSorter) Less(i, j int) bool {
+	return s[i].weight() < s[j].weight()
+}
+func (s weightSorter) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
+
+type NodeFilter func(*NodeList)
+
+func FilterOnlineNode() NodeFilter {
+	return func(nl *NodeList) {
+		for i, n := range *nl {
+			if n.Status != NodeStatusOnline {
+				nl.remove(i)
+			}
+		}
+	}
+}
+
+func FilterNodeName(name string) NodeFilter {
+	return func(nl *NodeList) {
+		for i, n := range *nl {
+			if n.Name == name {
+				nl.remove(i)
+				return
+			}
+		}
+	}
+}
+
+func FilterMaxCores(cores uint64) NodeFilter {
+	return func(nl *NodeList) {
+		for i, n := range *nl {
+			if cores > uint64(n.MaxCPU) {
+				nl.remove(i)
+			}
+		}
+	}
+}
+
+const (
+	mib = 1024 * 1024
+	gib = mib * 1024
+)
+
+func FilterFreeMem(mem uint64) NodeFilter {
+	return func(nl *NodeList) {
+		for i, n := range *nl {
+			if (mem * gib) >= n.MaxMem-n.Mem {
+				nl.remove(i)
+			}
+		}
+	}
+}
+
+// Useful? only local disk?
+func FilterFreeDisk(disk uint64) NodeFilter {
+	return func(nl *NodeList) {
+		for i, n := range *nl {
+			if (disk * gib) >= n.MaxDisk-n.Disk {
+				nl.remove(i)
+			}
+		}
+	}
+}
+
+func FilterVMAntiAffinity(url string) NodeFilter {
+	return func(nl *NodeList) {
+		_, _, id, err := ParseURL(url)
+		if err != nil {
+			return
+		}
+		for i, n := range *nl {
+			for _, s := range n.serverList {
+				if s.ID == id {
+					nl.remove(i)
+					return
+				}
+			}
+		}
+	}
+}
 
 type NodeClient struct {
 	client *Client
 }
 
+func (c *NodeClient) Schedule(ctx context.Context, filters ...NodeFilter) (n *Node, err error) {
+	nl, err := c.List(ctx)
+	if err != nil {
+		return
+	}
+	err = nl.getServerList(ctx, c)
+	if err != nil {
+		return
+	}
+	nl.filter(filters...)
+	nl.sortByWeight()
+	if len(nl) == 0 {
+		err = ErrUnschedulable
+		return
+	}
+	n = nl[0]
+	return
+}
+
 func (c *NodeClient) List(ctx context.Context) (nl NodeList, err error) {
 	req, err := c.client.NewRequest(ctx, "GET", "/nodes", nil)
 	if err != nil {
@@ -40,7 +191,7 @@ func (c *NodeClient) List(ctx context.Context) (nl NodeList, err error) {
 	return
 }
 
-func (c *NodeClient) ListServers(ctx context.Context, n *Node) (sl ServerList, err error) {
+func (c *NodeClient) ListServers(ctx context.Context, n *Node) (sl ServerRefList, err error) {
 	req, err := c.client.NewRequest(ctx, "GET", fmt.Sprintf("/nodes/%s/qemu", n.Name), nil)
 	if err != nil {
 		return
@@ -54,43 +205,66 @@ func (c *NodeClient) ListServers(ctx context.Context, n *Node) (sl ServerList, e
 	return
 }
 
-func (c *NodeClient) FindServer(ctx context.Context, name, id string) (s *Server, err error) {
+func (c *NodeClient) FindServer(ctx context.Context, name string, id int) (ref *ServerRef, err error) {
+	valid := ValidateID(id) == nil
 	nl, err := c.List(ctx)
 	if err != nil {
 		return
 	}
+	var (
+		nodeOffline     bool
+		nodeNotSearched bool
+	)
 	for _, node := range nl {
 		if node.Status != NodeStatusOnline {
+			nodeOffline = true
 			continue
 		}
 		sl, err := c.ListServers(ctx, node)
 		if err != nil {
+			nodeNotSearched = true
 			continue
 		}
 		for _, s := range sl {
-			if id != "" && s.ID == id {
+			if valid && s.ID == id {
 				return s, nil
 			}
 			if name != "" && s.Name == name {
-				return s, nil
+				if ref != nil {
+					return nil, ErrTooManyServersFound
+				}
+				ref = s
 			}
 		}
 	}
-	return nil, ErrServerNotFound
+	if nodeOffline {
+		return nil, ErrNodesOffline
+	}
+	if nodeNotSearched {
+		return nil, ErrNodesNotSearched
+	}
+	if ref == nil {
+		err = ErrServerNotFound
+	}
+	return
 }
 
-func (c *NodeClient) FindServerByName(ctx context.Context, name string) (s *Server, err error) {
+func (c *NodeClient) FindServerByName(ctx context.Context, name string) (s *ServerRef, err error) {
-	return c.FindServer(ctx, name, "")
+	return c.FindServer(ctx, name, InvalidID)
 }
 
-func (c *NodeClient) FindServerByID(ctx context.Context, id string) (s *Server, err error) {
+func (c *NodeClient) FindServerByID(ctx context.Context, id int) (s *ServerRef, err error) {
-	return c.FindServer(ctx, "", id)
+	err = ValidateID(id)
-}
-
-func (c *NodeClient) FindServerByURL(ctx context.Context, url string) (s *Server, err error) {
-	_, id, err := ParseURL(url)
 	if err != nil {
 		return
 	}
 	return c.FindServer(ctx, "", id)
 }
+
+func (c *NodeClient) FindServerByURL(ctx context.Context, url string) (s *ServerRef, err error) {
+	_, _, id, err := ParseURL(url)
+	if err != nil {
+		return
+	}
+	return c.FindServerByID(ctx, id)
+}

patches/snippets_6.3-3.patch

@@ -0,0 +1,29 @@
+--- PVE/Storage.pm.old	2021-04-18 12:35:48.201033594 +0200
++++ PVE/Storage.pm	2021-04-09 23:28:07.554075001 +0200
+@@ -420,6 +420,15 @@
+     return $plugin->get_subdir($scfg, 'iso');
+ }
+ 
++sub get_snippet_dir {
++    my ($cfg, $storeid) = @_;
++
++    my $scfg = storage_config($cfg, $storeid);
++    my $plugin = PVE::Storage::Plugin->lookup($scfg->{type});
++
++    return $plugin->get_subdir($scfg, 'snippets');
++}
++
+ sub get_vztmpl_dir {
+     my ($cfg, $storeid) = @_;
+
+--- PVE/API2/Storage/Status.pm.old	2021-04-18 12:36:10.512862744 +0200
++++ PVE/API2/Storage/Status.pm	2021-04-09 23:28:46.429777635 +0200
+@@ -424,6 +424,8 @@
+ 		raise_param_exc({ filename => "missing '.tar.gz' or '.tar.xz' extension" });
+ 	    }
+ 	    $path = PVE::Storage::get_vztmpl_dir($cfg, $param->{storage});
++	} elsif ($content eq 'snippets') {
++	    $path = PVE::Storage::get_snippet_dir($cfg, $param->{storage});
+ 	} else {
+ 	    raise_param_exc({ content => "upload content type '$content' not allowed" });
+ 	}

patches/snippets_7.0-11.patch

@@ -0,0 +1,60 @@
+--- Storage.pm.old	2021-11-28 12:58:53.026308987 +0100
++++ Storage.pm	2021-11-28 12:59:14.394144995 +0100
+@@ -427,6 +427,15 @@
+     return $plugin->get_subdir($scfg, 'vztmpl');
+ }
+ 
++sub get_snippet_dir {
++    my ($cfg, $storeid) = @_;
++
++    my $scfg = storage_config($cfg, $storeid);
++    my $plugin = PVE::Storage::Plugin->lookup($scfg->{type});
++
++    return $plugin->get_subdir($scfg, 'snippets');
++}
++
+ sub get_backup_dir {
+     my ($cfg, $storeid) = @_;
+
+
+--- API2/Storage/Config.pm.old	2021-11-28 14:09:07.861745249 +0100
++++ API2/Storage/Config.pm	2021-11-28 14:00:09.133897607 +0100
+@@ -19,7 +19,7 @@
+ 
+ use base qw(PVE::RESTHandler);
+ 
+-my @ctypes = qw(images vztmpl iso backup);
++my @ctypes = qw(images vztmpl iso backup snippets);
+ 
+ my $storage_type_enum = PVE::Storage::Plugin->lookup_types();
+ 
+
+--- API2/Storage/Status.pm.old	2021-11-28 13:51:17.609996827 +0100
++++ API2/Storage/Status.pm	2021-11-28 13:51:02.190115817 +0100
+@@ -381,7 +381,7 @@
+ 	    content => {
+ 		description => "Content type.",
+ 		type => 'string', format => 'pve-storage-content',
+-		enum => ['iso', 'vztmpl'],
++		enum => ['iso', 'vztmpl', 'snippets'],
+ 	    },
+ 	    filename => {
+ 		description => "The name of the file to create. Caution: This will be normalized!",
+@@ -433,6 +433,8 @@
+ 		raise_param_exc({ filename => "wrong file extension" });
+ 	    }
+ 	    $path = PVE::Storage::get_vztmpl_dir($cfg, $param->{storage});
++	} elsif ($content eq 'snippets') {
++	    $path = PVE::Storage::get_snippet_dir($cfg, $param->{storage});
+ 	} else {
+ 	    raise_param_exc({ content => "upload content type '$content' not allowed" });
+ 	}
+@@ -534,7 +536,7 @@
+ 	    content => {
+ 		description => "Content type.", # TODO: could be optional & detected in most cases
+ 		type => 'string', format => 'pve-storage-content',
+-		enum => ['iso', 'vztmpl'],
++		enum => ['iso', 'vztmpl', 'snippets'],
+ 	    },
+ 	    filename => {
+ 		description => "The name of the file to create. Caution: This will be normalized!",

pool.go

@@ -12,6 +12,7 @@ import (
 var (
 	ErrPoolExists    = errors.New("pool already exists")
 	ErrPoolNotExists = errors.New("pool does not exist")
+	ErrEmptyPoolName = errors.New("pool name is empty")
 )
 
 func MakePoolName(namespace, name string) string {
@@ -72,3 +73,113 @@ func (c *PoolClient) Delete(ctx context.Context, name string) (err error) {
 	_, err = c.client.Do(req, nil)
 	return
 }
+
+type pools struct {
+	PoolID string `json:"poolid"`
+}
+
+func (c *PoolClient) ListPools(ctx context.Context) (list []string, err error) {
+	var ps []pools
+	req, err := c.client.NewRequest(ctx, "GET", "/pools", nil)
+	if err != nil {
+		return
+	}
+	_, err = c.client.Do(req, &ps)
+	if err != nil {
+		return
+	}
+	for _, p := range ps {
+		list = append(list, p.PoolID)
+	}
+	return
+}
+
+type poolMember struct {
+	ID   int    `json:"vmid"`
+	Name string `json:"name"`
+	Node string `json:"node"`
+}
+
+type poolMembers []poolMember
+
+func (pms poolMembers) ServerList() (sl ServerRefList) {
+	for _, pm := range pms {
+		sl = append(sl, &ServerRef{
+			ID:   pm.ID,
+			Name: pm.Name,
+			Node: pm.Node,
+		})
+	}
+	return
+}
+
+type pool struct {
+	Members poolMembers `json:"members"`
+}
+
+func (c *PoolClient) ListMembers(ctx context.Context, name string) (sl ServerRefList, err error) {
+	if name == "" {
+		err = ErrEmptyPoolName
+		return
+	}
+	var p pool
+	req, err := c.client.NewRequest(ctx, "GET", fmt.Sprintf("/pools/%s", name), nil)
+	if err != nil {
+		return
+	}
+	_, err = c.client.Do(req, &p)
+	sl = p.Members.ServerList()
+	return
+}
+
+func (c *PoolClient) FindServer(ctx context.Context, poolname, servername string, id int) (ref *ServerRef, err error) {
+	valid := ValidateID(id) == nil
+	var pools []string
+	if poolname != "" {
+		pools = append(pools, poolname)
+	} else {
+		pools, err = c.ListPools(ctx)
+		if err != nil {
+			return
+		}
+	}
+	for _, pool := range pools {
+		sl, err := c.ListMembers(ctx, pool)
+		if err != nil {
+			return nil, err
+		}
+		for _, s := range sl {
+			if valid && s.ID == id {
+				s.Pool = pool
+				return s, nil
+			}
+			if servername != "" && s.Name == servername {
+				if ref != nil {
+					return nil, ErrTooManyServersFound
+				}
+				s.Pool = pool
+				ref = s
+			}
+		}
+	}
+	if ref == nil {
+		err = ErrServerNotFound
+	}
+	return
+}
+
+func (c *PoolClient) FindServerByName(ctx context.Context, poolname, servername string) (s *ServerRef, err error) {
+	return c.FindServer(ctx, poolname, servername, InvalidID)
+}
+
+func (c *PoolClient) FindServerByID(ctx context.Context, poolname string, id int) (s *ServerRef, err error) {
+	return c.FindServer(ctx, poolname, "", id)
+}
+
+func (c *PoolClient) FindServerByURL(ctx context.Context, url string) (s *ServerRef, err error) {
+	pool, _, id, err := ParseURL(url)
+	if err != nil {
+		return
+	}
+	return c.FindServerByID(ctx, pool, id)
+}

scheme.go

@@ -6,14 +6,33 @@ import (
 	"errors"
 	"fmt"
 	"net/url"
+	"strconv"
 	"strings"
 )
 
-const PVEScheme = "pve"
+const (
+	PVEScheme    = "pve"
+	PVESchemeURL = PVEScheme + "://"
 
-var ErrInvalidPVEURL = errors.New("invalid pve url scheme")
+	MinID     = 100
+	InvalidID = 0
+)
 
-func ParseURL(s string) (node, id string, err error) {
+var (
+	ErrInvalidPVEURL = errors.New("invalid pve url scheme")
+	ErrNoID          = errors.New("missing id in url")
+	ErrInvalidID     = errors.New("invalid id")
+	ErrParsingID     = errors.New("error parsing id in url")
+)
+
+func ValidateID(id int) error {
+	if id < MinID {
+		return ErrInvalidID
+	}
+	return nil
+}
+
+func ParseURL(s string) (pool, node string, id int, err error) {
 	u, err := url.Parse(s)
 	if err != nil {
 		return
@@ -22,9 +41,43 @@ func ParseURL(s string) (node, id string, err error) {
 		err = ErrInvalidPVEURL
 		return
 	}
-	return u.Host, strings.TrimPrefix(u.Path, "/"), nil
+	a := strings.Split(u.Path, "/")
+	if len(a) != 3 {
+		err = ErrInvalidPVEURL
+		return
+	}
+	if a[2] == "" {
+		err = ErrNoID
+		return
+	}
+	id, err = strconv.Atoi(a[2])
+	if err != nil {
+		err = ErrParsingID
+	}
+	return u.Host, a[1], id, ValidateID(id)
 }
 
-func NewURL(node, id string) string {
+func ServerRefFromURL(s string) (ref *ServerRef, err error) {
-	return fmt.Sprintf("%s://%s/%s", PVEScheme, node, id)
+	pool, node, id, err := ParseURL(s)
+	if err != nil {
+		return
+	}
+	ref = &ServerRef{
+		ID:   id,
+		Node: node,
+		Pool: pool,
+	}
+	return
+}
+
+func NewURL(pool, node string, id int) string {
+	return fmt.Sprintf("%s%s/%s/%d", PVESchemeURL, pool, node, id)
+}
+
+func K8sURL(url string) (string, error) {
+	pool, _, id, err := ParseURL(url)
+	if err != nil {
+		return "", nil
+	}
+	return NewURL(pool, "", id), nil
 }

server.go

@@ -10,51 +10,125 @@ import (
 	"strings"
 )
 
+var ErrServerNotFound = errors.New("server not found")
+
 type Server struct {
-	ID           string `json:"vmid"`
+	ID              int    `json:"vmid"`
-	Name         string `json:"name"`
+	Name            string `json:"name"`
-	Node         string
+	Node            string
-	Status       ServerStatus `json:"status"`
+	Pool            string
-	Resources    Resources
+	Status          ServerStatus `json:"status"`
-	NetDevices   NetworkDevices
+	Resources       Resources
-	IPConfig     IPConfigs
+	NetDevices      NetworkDevices
-	Nameserver   Nameserver
+	IPConfig        IPConfigs
-	SearchDomain string
+	Nameserver      Nameserver
-	UserData     *UserData
+	SearchDomain    string
+	UserData        *UserData
+	UserDataStorage string
+	cicustom        bool
 }
 
-func (s *Server) body() (httpbody, error) {
+func (s *Server) UserDataSnippetName() string {
-	body := httpbody{
+	return fmt.Sprintf("%d_userdata", s.ID)
-		"memory":       strconv.FormatInt(s.Resources.Memory, 10),
+}
-		"cores":        strconv.FormatInt(s.Resources.Cores, 10),
+
+func (s *Server) UserDataSnippet(storage string) string {
+	return fmt.Sprintf("user=%s", SnippetVolume(storage, s.UserDataSnippetName()))
+}
+
+func (s *Server) body() (b httpbody) {
+	b = httpbody{
+		"memory":       s.Resources.Memory.String(),
+		"cores":        s.Resources.Cores.String(),
 		"name":         s.Name,
 		"nameserver":   s.Nameserver.String(),
 		"searchdomain": s.SearchDomain,
 	}
-	userdata, err := s.UserData.String()
+	if s.UserData != nil || s.cicustom {
-	body["ciuserdata"] = userdata
+		b["cicustom"] = s.UserDataSnippet(s.UserDataStorage)
+	}
 	for i, n := range s.NetDevices {
-		body["net"+strconv.Itoa(i)] = n.String()
+		if n == nil {
+			b["net"+strconv.Itoa(i)] = ""
+			continue
+		}
+		b["net"+strconv.Itoa(i)] = n.String()
 	}
 	for i, ip := range s.IPConfig {
-		body["ipconfig"+strconv.Itoa(i)] = ip.String()
+		if ip == nil {
+			b["ipconfig"+strconv.Itoa(i)] = ""
+			continue
+		}
+		b["ipconfig"+strconv.Itoa(i)] = ip.String()
 	}
-	return body, err
+	return
 }
 
 func (s *Server) InstanceID() string {
-	return s.String()
+	return NewURL("", s.Node, s.ID)
 }
 
-func (s *Server) String() string {
+func (s *Server) K8sID() string {
-	return NewURL(s.Node, s.ID)
+	return NewURL(s.Pool, "", s.ID)
+}
+
+func (s *Server) Ref() *ServerRef {
+	return &ServerRef{
+		ID:   s.ID,
+		Node: s.Node,
+		Name: s.Name,
+		Pool: s.Pool,
+	}
+}
+
+type ServerRefList []*ServerRef
+
+type ServerRef struct {
+	ID       int    `json:"vmid"`
+	Name     string `json:"name"`
+	Template int    `json:"template,omitempty"`
+	Node     string `json:"node,omitempty"`
+	Pool     string `json:"pool,omitempty"`
+}
+
+func (ref *ServerRef) InstanceID() string {
+	return NewURL("", ref.Node, ref.ID)
+}
+
+func (ref *ServerRef) K8sID() string {
+	return NewURL(ref.Pool, "", ref.ID)
+}
+
+func (ref *ServerRef) IsTemplate() bool {
+	return ref.Template == 1
 }
 
 type Resources struct {
-	Memory       int64
+	Cores  Cores  `json:"cores"`
-	Cores        int64
+	Memory Memory `json:"memory"`
-	BootDisk     string
+	Disk   Disk   `json:"disk"`
-	BootDiskSize string
+}
+
+type Cores uint64
+
+func (c Cores) String() string {
+	return strconv.FormatUint(uint64(c), 10)
+}
+
+type Memory uint64
+
+func (m Memory) String() string {
+	return strconv.FormatUint(uint64(m)*1024, 10)
+}
+
+type Disk struct {
+	Storage string `json:"storage"`
+	Name    string `json:"name,omitempty"`
+	Size    uint64 `json:"size"`
+}
+
+func (d Disk) String() string {
+	return strconv.FormatUint(d.Size, 10) + "G"
 }
 
 type Nameserver []string
@@ -69,8 +143,11 @@ func (n Nameserver) String() string {
 
 type serverConfig map[string]interface{}
 
-func (sc serverConfig) Server(node, id string) (s *Server, err error) {
+func (sc serverConfig) Server(ref *ServerRef) (s *Server, err error) {
-	s = &Server{ID: id, Node: node}
+	if len(sc) == 0 {
+		return
+	}
+	s = &Server{ID: ref.ID, Node: ref.Node}
 
 	for k := range sc {
 		switch k {
@@ -78,25 +155,33 @@ func (sc serverConfig) Server(node, id string) (s *Server, err error) {
 			if v, ok := sc[k].(string); ok {
 				s.Name = v
 			}
-		case "ciuserdata":
+		case "cicustom":
-			if v, ok := sc[k].(string); ok {
+			if _, ok := sc[k].(string); ok {
-				s.UserData, err = parseUserData(v)
+				s.cicustom = true
 			}
 		case "memory":
 			if v, ok := sc[k].(float64); ok {
-				s.Resources.Memory = int64(v)
+				s.Resources.Memory = Memory(uint64(v) / 1024)
 			}
 		case "cores":
 			if v, ok := sc[k].(float64); ok {
-				s.Resources.Cores = int64(v)
+				s.Resources.Cores = Cores(uint64(v))
 			}
 		case "bootdisk":
 			if v, ok := sc[k].(string); ok {
-				s.Resources.BootDisk = v
+				s.Resources.Disk.Name = v
 				if val, ok := sc[v].(string); ok {
-					for _, o := range strings.Split(val, ",") {
+					vals := strings.Split(val, ",")
+					if len(vals) > 0 {
+						storage := strings.Split(vals[0], ":")
+						if len(storage) > 0 {
+							s.Resources.Disk.Storage = storage[0]
+						}
+					}
+					for _, o := range vals {
 						if strings.HasPrefix(o, "size=") {
-							s.Resources.BootDiskSize = o[5:]
+							size, _ := strconv.ParseUint(o[5:], 10, 64)
+							s.Resources.Disk.Size = size
 							break
 						}
 					}
@@ -140,18 +225,23 @@ type ServerClient struct {
 	client *Client
 }
 
-func (c *ServerClient) NextID(ctx context.Context) (id string, err error) {
+func (c *ServerClient) NextID(ctx context.Context) (id int, err error) {
+	id = InvalidID
 	req, err := c.client.NewRequest(ctx, "GET", "/cluster/nextid", nil)
 	if err != nil {
 		return
 	}
-	_, err = c.client.Do(req, &id)
+	var sid string
-	return
+	_, err = c.client.Do(req, &sid)
+	if err != nil {
+		return
+	}
+	return strconv.Atoi(sid)
 }
 
-func (c *ServerClient) getConfig(ctx context.Context, node, id string) (cfg serverConfig, err error) {
+func (c *ServerClient) getConfig(ctx context.Context, ref *ServerRef) (cfg serverConfig, err error) {
 	cfg = make(serverConfig)
-	req, err := c.client.NewRequest(ctx, "GET", fmt.Sprintf("/nodes/%s/qemu/%s/config", node, id), nil)
+	req, err := c.client.NewRequest(ctx, "GET", fmt.Sprintf("/nodes/%s/qemu/%d/config?current=1", ref.Node, ref.ID), nil)
 	if err != nil {
 		return
 	}
@@ -159,21 +249,17 @@ func (c *ServerClient) getConfig(ctx context.Context, node, id string) (cfg serv
 	return
 }
 
-func (c *ServerClient) getServer(ctx context.Context, node, id string) (s *Server, cfg serverConfig, err error) {
+func (c *ServerClient) getServer(ctx context.Context, ref *ServerRef) (s *Server, cfg serverConfig, err error) {
-	cfg, err = c.getConfig(ctx, node, id)
+	cfg, err = c.getConfig(ctx, ref)
 	if err != nil {
 		return
 	}
-	s, err = cfg.Server(node, id)
+	s, err = cfg.Server(ref)
 	return
 }
 
-func (c *ServerClient) GetByURL(ctx context.Context, url string) (s *Server, err error) {
+func (c *ServerClient) GetByRef(ctx context.Context, ref *ServerRef) (s *Server, err error) {
-	node, id, err := ParseURL(url)
+	s, cfg, err := c.getServer(ctx, ref)
-	if err != nil {
-		return
-	}
-	s, cfg, err := c.getServer(ctx, node, id)
 	if err != nil {
 		return
 	}
@@ -187,25 +273,69 @@ func (c *ServerClient) GetByURL(ctx context.Context, url string) (s *Server, err
 	return
 }
 
+func (c *ServerClient) GetByURL(ctx context.Context, url string) (s *Server, err error) {
+	ref, err := ServerRefFromURL(url)
+	if err != nil {
+		return
+	}
+	if ref.Node == "" && ref.Pool != "" {
+		ref, err = c.client.Pool.FindServerByID(ctx, ref.Pool, ref.ID)
+		if err != nil {
+			return
+		}
+	} else if ref.Node == "" && ref.Pool == "" {
+		ref, err = c.client.Node.FindServerByID(ctx, ref.ID)
+		if err != nil {
+			return
+		}
+	}
+	s, err = c.GetByRef(ctx, ref)
+	return
+}
+
 type ServerTemplateOpts struct {
-	Name          string
+	Name         string
-	TemplateURL   string
+	TemplateID   int
+	TemplateName string
+	// TemplateNode is optional
+	TemplateNode  string
 	Pool          string
 	TargetStorage string
 	TargetNode    string
 }
 
-func (o ServerTemplateOpts) Validate() error {
+func (o *ServerTemplateOpts) Validate(ctx context.Context, c *Client) error {
 	if o.Name == "" {
 		return errors.New("missing name")
 	}
-	if o.TemplateURL == "" {
+	if ValidateID(o.TemplateID) != nil {
-		return errors.New("missing template url")
+		if o.TemplateName == "" {
+			return errors.New("missing or invalid template id")
+		}
+		ref, err := c.Node.FindServerByName(ctx, o.TemplateName)
+		if err != nil {
+			return fmt.Errorf("template id not found: %w", err)
+		}
+		if !ref.IsTemplate() {
+			return errors.New("server is not a vm template")
+		}
+		o.TemplateID = ref.ID
+		o.TemplateNode = ref.Node
+	}
+	if o.TemplateNode == "" {
+		ref, err := c.Node.FindServerByID(ctx, o.TemplateID)
+		if err != nil {
+			return fmt.Errorf("template node not found err: %w", err)
+		}
+		if ref.Node == "" {
+			return errors.New("missing template node")
+		}
+		o.TemplateNode = ref.Node
 	}
 	return nil
 }
 
-func (o ServerTemplateOpts) body() httpbody {
+func (o *ServerTemplateOpts) body() httpbody {
 	body := httpbody{"name": o.Name}
 	if o.TargetStorage != "" {
 		body["storage"] = o.TargetStorage
@@ -217,12 +347,8 @@ func (o ServerTemplateOpts) body() httpbody {
 	return body
 }
 
-func (c *ServerClient) CreateFromTemplate(ctx context.Context, opts ServerTemplateOpts) (t *Task, url string, err error) {
+func (c *ServerClient) CreateFromTemplate(ctx context.Context, opts *ServerTemplateOpts) (t *Task, url string, err error) {
-	err = opts.Validate()
+	err = opts.Validate(ctx, c.client)
-	if err != nil {
-		return
-	}
-	tempnode, id, err := ParseURL(opts.TemplateURL)
 	if err != nil {
 		return
 	}
@@ -231,14 +357,14 @@ func (c *ServerClient) CreateFromTemplate(ctx context.Context, opts ServerTempla
 		return
 	}
 	body := opts.body()
-	body["newid"] = nextid
+	body["newid"] = strconv.Itoa(nextid)
-	if opts.TargetNode != "" && tempnode != opts.TargetNode {
+	node := opts.TemplateNode
+	if opts.TargetNode != "" && opts.TemplateNode != opts.TargetNode {
+		node = opts.TargetNode
 		body["target"] = opts.TargetNode
-		url = NewURL(opts.TargetNode, nextid)
-	} else {
-		url = NewURL(tempnode, nextid)
 	}
-	req, err := c.client.NewRequest(ctx, "POST", fmt.Sprintf("/nodes/%s/qemu/%s/clone", tempnode, id), body.Reader())
+	url = NewURL(opts.Pool, node, nextid)
+	req, err := c.client.NewRequest(ctx, "POST", fmt.Sprintf("/nodes/%s/qemu/%d/clone", opts.TemplateNode, opts.TemplateID), body.Reader())
 	if err != nil {
 		return
 	}
@@ -248,36 +374,36 @@ func (c *ServerClient) CreateFromTemplate(ctx context.Context, opts ServerTempla
 	return
 }
 
-func (c *ServerClient) Start(ctx context.Context, s *Server) (t *Task, err error) {
+func (c *ServerClient) Start(ctx context.Context, ref *ServerRef) (t *Task, err error) {
-	return c.setStatus(ctx, s, "start")
+	return c.setStatus(ctx, ref, "start")
 }
 
-func (c *ServerClient) Reboot(ctx context.Context, s *Server) (t *Task, err error) {
+func (c *ServerClient) Reboot(ctx context.Context, ref *ServerRef) (t *Task, err error) {
-	return c.setStatus(ctx, s, "reboot")
+	return c.setStatus(ctx, ref, "reboot")
 }
 
-func (c *ServerClient) Shutdown(ctx context.Context, s *Server) (t *Task, err error) {
+func (c *ServerClient) Shutdown(ctx context.Context, ref *ServerRef) (t *Task, err error) {
-	return c.setStatus(ctx, s, "shutdown")
+	return c.setStatus(ctx, ref, "shutdown")
 }
 
-func (c *ServerClient) Reset(ctx context.Context, s *Server) (t *Task, err error) {
+func (c *ServerClient) Reset(ctx context.Context, ref *ServerRef) (t *Task, err error) {
-	return c.setStatus(ctx, s, "reset")
+	return c.setStatus(ctx, ref, "reset")
 }
 
-func (c *ServerClient) Suspend(ctx context.Context, s *Server) (t *Task, err error) {
+func (c *ServerClient) Suspend(ctx context.Context, ref *ServerRef) (t *Task, err error) {
-	return c.setStatus(ctx, s, "suspent")
+	return c.setStatus(ctx, ref, "suspent")
 }
 
-func (c *ServerClient) Resume(ctx context.Context, s *Server) (t *Task, err error) {
+func (c *ServerClient) Resume(ctx context.Context, ref *ServerRef) (t *Task, err error) {
-	return c.setStatus(ctx, s, "resume")
+	return c.setStatus(ctx, ref, "resume")
 }
 
-func (c *ServerClient) Stop(ctx context.Context, s *Server) (t *Task, err error) {
+func (c *ServerClient) Stop(ctx context.Context, ref *ServerRef) (t *Task, err error) {
-	return c.setStatus(ctx, s, "stop")
+	return c.setStatus(ctx, ref, "stop")
 }
 
-func (c *ServerClient) setStatus(ctx context.Context, s *Server, status string) (t *Task, err error) {
+func (c *ServerClient) setStatus(ctx context.Context, ref *ServerRef, status string) (t *Task, err error) {
-	req, err := c.client.NewRequest(ctx, "POST", fmt.Sprintf("/nodes/%s/qemu/%s/status/%s", s.Node, s.ID, status), nil)
+	req, err := c.client.NewRequest(ctx, "POST", fmt.Sprintf("/nodes/%s/qemu/%d/status/%s", ref.Node, ref.ID, status), nil)
 	if err != nil {
 		return
 	}
@@ -294,7 +420,7 @@ type statusObj struct {
 func (c *ServerClient) GetStatus(ctx context.Context, s *Server, lock *string) (err error) {
 	var lockval string
 	if lock == nil {
-		cfg, err := c.getConfig(ctx, s.Node, s.ID)
+		cfg, err := c.getConfig(ctx, s.Ref())
 		if err != nil {
 			return err
 		}
@@ -305,7 +431,7 @@ func (c *ServerClient) GetStatus(ctx context.Context, s *Server, lock *string) (
 		lockval = *lock
 	}
 	var status statusObj
-	req, err := c.client.NewRequest(ctx, "GET", fmt.Sprintf("/nodes/%s/qemu/%s/status/current", s.Node, s.ID), nil)
+	req, err := c.client.NewRequest(ctx, "GET", fmt.Sprintf("/nodes/%s/qemu/%d/status/current", s.Node, s.ID), nil)
 	if err != nil {
 		s.Status = ServerStatusUnknown
 		return
@@ -319,15 +445,9 @@ func (c *ServerClient) GetStatus(ctx context.Context, s *Server, lock *string) (
 	return
 }
 
-func diffBody(cur, n *Server) (b httpbody, err error) {
+func diffBody(cur, n *Server) (b httpbody) {
-	b, err = n.body()
+	b = n.body()
-	if err != nil {
+	curbody := cur.body()
-		return
-	}
-	curbody, err := cur.body()
-	if err != nil {
-		return
-	}
 	for k, v := range b {
 		if curbody[k] == v {
 			delete(b, k)
@@ -337,7 +457,7 @@ func diffBody(cur, n *Server) (b httpbody, err error) {
 }
 
 func (c *ServerClient) updateConfig(ctx context.Context, s *Server, body httpbody) (err error) {
-	req, err := c.client.NewRequest(ctx, "PUT", fmt.Sprintf("/nodes/%s/qemu/%s/config", s.Node, s.ID), body.Reader())
+	req, err := c.client.NewRequest(ctx, "PUT", fmt.Sprintf("/nodes/%s/qemu/%d/config", s.Node, s.ID), body.Reader())
 	if err != nil {
 		return err
 	}
@@ -347,10 +467,10 @@ func (c *ServerClient) updateConfig(ctx context.Context, s *Server, body httpbod
 
 func (c *ServerClient) resizeDisk(ctx context.Context, s *Server) (err error) {
 	body := httpbody{
-		"disk": s.Resources.BootDisk,
+		"disk": s.Resources.Disk.Name,
-		"size": s.Resources.BootDiskSize,
+		"size": s.Resources.Disk.String(),
 	}
-	req, err := c.client.NewRequest(ctx, "PUT", fmt.Sprintf("/nodes/%s/qemu/%s/resize", s.Node, s.ID), body.Reader())
+	req, err := c.client.NewRequest(ctx, "PUT", fmt.Sprintf("/nodes/%s/qemu/%d/resize", s.Node, s.ID), body.Reader())
 	if err != nil {
 		return err
 	}
@@ -358,38 +478,47 @@ func (c *ServerClient) resizeDisk(ctx context.Context, s *Server) (err error) {
 	return
 }
 
-func (c *ServerClient) Update(ctx context.Context, s *Server) (err error) {
+func (c *ServerClient) Update(ctx context.Context, s *Server) (t *Task, err error) {
-	cur, _, err := c.getServer(ctx, s.Node, s.ID)
+	cur, _, err := c.getServer(ctx, s.Ref())
 	if err != nil {
 		return
 	}
-	body, err := diffBody(cur, s)
+	body := diffBody(cur, s)
-	if err != nil {
+	var e []string
-		return
+	if s.UserData != nil {
+		b, userdata := s.UserData.Bytes()
+		if userdata == nil {
+			t, userdata = c.client.Snippet.Create(ctx, s.Node, s.UserDataStorage, s.UserDataSnippetName(), b)
+		}
+		if userdata != nil {
+			e = append(e, fmt.Sprintf("userdata: %s", userdata))
+		}
 	}
-	var (
-		cfg    error
-		resize error
-	)
 	if len(body) > 0 {
-		cfg = c.updateConfig(ctx, s, body)
+		cfg := c.updateConfig(ctx, s, body)
+		if cfg != nil {
+			e = append(e, fmt.Sprintf("config: %s", cfg))
+		}
 	}
-	if cur.Resources.BootDiskSize != s.Resources.BootDiskSize {
+	if cur.Resources.Disk.Size != s.Resources.Disk.Size {
-		resize = c.resizeDisk(ctx, s)
+		resize := c.resizeDisk(ctx, s)
+		if resize != nil {
+			e = append(e, fmt.Sprintf("resize: %s", resize))
+		}
 	}
-	if cfg != nil && resize != nil {
+	if len(e) > 0 {
-		return fmt.Errorf("config: %s resize: %s", cfg, resize)
+		err = fmt.Errorf("%s", strings.Join(e, ""))
-	} else if cfg != nil {
+	}
-		return fmt.Errorf("config: %s", cfg)
+	if t == nil {
-	} else if resize != nil {
+		dummy := DummyTask
-		return fmt.Errorf("resize: %s", resize)
+		t = &dummy
 	}
 	return
 }
 
 func (c *ServerClient) Delete(ctx context.Context, s *Server, f OnTaskChange) (t *Task, err error) {
 	if s.Status == ServerStatusRunning || s.Status == ServerStatusUnknown {
-		task, err := c.Stop(ctx, s)
+		task, err := c.Stop(ctx, s.Ref())
 		if err != nil {
 			return nil, err
 		}
@@ -398,7 +527,7 @@ func (c *ServerClient) Delete(ctx context.Context, s *Server, f OnTaskChange) (t
 			return nil, err
 		}
 	}
-	req, err := c.client.NewRequest(ctx, "DELETE", fmt.Sprintf("/nodes/%s/qemu/%s", s.Node, s.ID), nil)
+	req, err := c.client.NewRequest(ctx, "DELETE", fmt.Sprintf("/nodes/%s/qemu/%d", s.Node, s.ID), nil)
 	if err != nil {
 		return
 	}

serverstatus.go

@@ -21,6 +21,9 @@ const (
 	ServerStatusSespending  ServerStatus = "suspending"
 	ServerStatusSespended   ServerStatus = "suspended"
 	ServerStatusResuming    ServerStatus = "resuming"
+
+	ServerStatusError         ServerStatus = "error"
+	ServerStatusScheduleError ServerStatus = "schedule-error"
 )
 
 func ServerStatusFromLock(status, lock string) ServerStatus {
@@ -61,7 +64,6 @@ func ServerStatusFromTask(t *Task) ServerStatus {
 		if t.Exitstatus == TaskExitStatusOK {
 			return ServerStatusSespended
 		}
-		return ServerStatusUnknown
 	case "qmresume":
 		if t.Status == TaskStatusRunning {
 			return ServerStatusResuming
@@ -69,7 +71,6 @@ func ServerStatusFromTask(t *Task) ServerStatus {
 		if t.Exitstatus == TaskExitStatusOK {
 			return ServerStatusRunning
 		}
-		return ServerStatusUnknown
 	case "qmclone", "qmcreate":
 		if t.Status == TaskStatusRunning {
 			return ServerStatusInitializing
@@ -77,7 +78,6 @@ func ServerStatusFromTask(t *Task) ServerStatus {
 		if t.Exitstatus == TaskExitStatusOK {
 			return ServerStatusStopped
 		}
-		return ServerStatusUnknown
 	case "qmreboot", "qmreset":
 		if t.Status == TaskStatusRunning {
 			return ServerStatusRebooting
@@ -85,7 +85,6 @@ func ServerStatusFromTask(t *Task) ServerStatus {
 		if t.Exitstatus == TaskExitStatusOK {
 			return ServerStatusRunning
 		}
-		return ServerStatusUnknown
 	case "qmstart":
 		if t.Status == TaskStatusRunning {
 			return ServerStatusStarting
@@ -101,12 +100,10 @@ func ServerStatusFromTask(t *Task) ServerStatus {
 		if t.Exitstatus == TaskExitStatusOK {
 			return ServerStatusStopped
 		}
-		return ServerStatusUnknown
 	case "qmdestroy":
 		if t.Status == TaskStatusRunning {
 			return ServerStatusDeleting
 		}
-		return ServerStatusUnknown
 	}
 	return ServerStatusUnknown
 }

snippet.go

@@ -0,0 +1,85 @@
+// Copyright (C) 2020 Marius Schellenberger
+
+package pve
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"mime/multipart"
+	"net/http"
+	"time"
+)
+
+var (
+	ErrNoSnippetNode    = errors.New("missing snippet target node")
+	ErrNoSnippetStorage = errors.New("missing snippet target storage")
+	ErrNoSnippetName    = errors.New("missing snippet name")
+)
+
+type SnippetClient struct {
+	client *Client
+}
+
+func (c *SnippetClient) Create(ctx context.Context, node, storage, name string, data []byte) (t *Task, err error) {
+	if node == "" {
+		err = ErrNoSnippetNode
+		return
+	}
+	if storage == "" {
+		err = ErrNoSnippetStorage
+		return
+	}
+	if name == "" {
+		err = ErrNoSnippetName
+		return
+	}
+	ub, b := newUnreadBuffer()
+	m := multipart.NewWriter(b)
+	err = m.WriteField("content", "snippets")
+	if err != nil {
+		return
+	}
+	fw, err := m.CreateFormFile("filename", name)
+	if err != nil {
+		return
+	}
+	if _, err = fw.Write(data); err != nil {
+		return
+	}
+	m.Close()
+	ub.Save()
+	ct := m.FormDataContentType()
+	var taskid string
+	var req *http.Request
+	var resp *http.Response
+	for i := 0; i < 3; i++ {
+		req, err = c.client.NewRequest(ctx, "POST", fmt.Sprintf("/nodes/%s/storage/%s/upload", node, storage), b)
+		req.Header.Set("Content-Type", ct)
+		resp, err = c.client.Do(req, &taskid)
+		if err != nil || resp.StatusCode != http.StatusOK {
+			ub.Reset(&b)
+			time.Sleep(time.Millisecond * 500)
+			continue
+		}
+		break
+	}
+	t = c.client.Task.MustGet(ctx, taskid)
+	return
+}
+
+func SnippetVolume(storage, name string) string {
+	return fmt.Sprintf("%s:snippets/%s", storage, name)
+}
+
+func (c *SnippetClient) Delete(ctx context.Context, node, storage, name string) (t *Task, err error) {
+	volume := SnippetVolume(storage, name)
+	req, err := c.client.NewRequest(ctx, "DELETE", fmt.Sprintf("/nodes/%s/storage/%s/content/%s", node, storage, volume), nil)
+	if err != nil {
+		return
+	}
+	var taskid string
+	_, err = c.client.Do(req, &taskid)
+	t = c.client.Task.MustGet(ctx, taskid)
+	return
+}

task.go

@@ -16,8 +16,12 @@ const (
 	TaskExitStatusOK        = "OK"
 	TaskStatusRunning       = "running"
 	TaskStatusStopped       = "stopped"
+
+	DummyTaskID = "dummy"
 )
 
+var DummyTask = Task{ID: DummyTaskID}
+
 type Task struct {
 	ID         string `json:"upid"`
 	Type       string `json:"type"`
@@ -63,6 +67,9 @@ func (c *TaskClient) Get(ctx context.Context, taskid string) (t *Task, err error
 type OnTaskChange func(t *Task)
 
 func (c *TaskClient) Wait(ctx context.Context, t *Task, f OnTaskChange) error {
+	if t.ID == DummyTaskID {
+		return nil
+	}
 	if f != nil && t.Status != "" {
 		f(t)
 	}
@@ -90,7 +97,7 @@ func (c *TaskClient) Wait(ctx context.Context, t *Task, f OnTaskChange) error {
 			}
 		}
 		time.Sleep(TaskStatusCheckInterval * time.Second)
-		waited = waited + TaskStatusCheckInterval
+		waited += TaskStatusCheckInterval
 	}
 	return errors.New("task wait timeout for: " + t.ID)
 }

userdata.go

@@ -3,31 +3,49 @@
 package pve
 
 import (
-	"encoding/base64"
 	"gopkg.in/yaml.v2"
-	"net/url"
 )
 
-func parseUserData(s string) (u *UserData, err error) {
+var cloudConfig = []byte("#cloud-config\n")
-	s, err = url.QueryUnescape(s)
+
-	if err != nil {
+const (
-		return
+	minBytes = 87
-	}
+	pad      = "########################################################################\n"
-	b, err := base64.StdEncoding.DecodeString(s)
+)
-	if err != nil {
+
-		return
+//TODO
-	}
+//func parseUserData(s string) (u *UserData, err error) {
-	return UnmarshalUserData(b)
+//	s, err = url.QueryUnescape(s)
-}
+//	if err != nil {
+//		return
+//	}
+//	b, err := base64.StdEncoding.DecodeString(s)
+//	if err != nil {
+//		return
+//	}
+//	return UnmarshalUserData(b)
+//}
 
 func UnmarshalUserData(b []byte) (u *UserData, err error) {
 	u = new(UserData)
 	err = yaml.Unmarshal(b, u)
 	return
 }
-func MarshalUserData(u *UserData) ([]byte, error) {
+
-	return yaml.Marshal(u)
+//TODO
-}
+//func MarshalUserData(u *UserData) ([]byte, error) {
+//	if u == nil {
+//		return nil, nil
+//	}
+//	b, err := yaml.Marshal(u)
+//	if err == nil {
+//		if len(b)+len(cloudConfig) < minBytes {
+//			b = append([]byte(pad[len(b):]), b...)
+//		}
+//		b = append(cloudConfig, b...)
+//	}
+//	return b, err
+//}
 
 type UserData struct {
 	Hostname          string     `yaml:"hostname"`
@@ -44,12 +62,33 @@ type UserData struct {
 	WriteFiles        []File     `yaml:"write_files,omitempty"`
 }
 
+// TODO
+// func (u *UserData) String() (s string, err error) {
+// 	b, err := MarshalUserData(u)
+// 	if err != nil {
+// 		return
+// 	}
+// 	s = url.QueryEscape(base64.StdEncoding.EncodeToString(b))
+// 	return
+// }
+
 func (u *UserData) String() (s string, err error) {
-	b, err := MarshalUserData(u)
+	b, err := u.Bytes()
 	if err != nil {
 		return
 	}
-	s = base64.StdEncoding.EncodeToString(b)
+	s = string(b)
+	return
+}
+
+func (u *UserData) Bytes() (b []byte, err error) {
+	b, err = yaml.Marshal(u)
+	if err == nil {
+		if len(b)+len(cloudConfig) < minBytes {
+			b = append([]byte(pad[len(b):]), b...)
+		}
+		b = append(cloudConfig, b...)
+	}
 	return
 }