initial commit

This commit is contained in:
ston1th 2019-08-03 22:31:25 +02:00
commit 0e6be50e37
23 changed files with 1069 additions and 0 deletions

5
.gitignore vendored Normal file
View file

@ -0,0 +1,5 @@
cluster
drone
grafana
netpol
minikube

131
README.md Normal file
View file

@ -0,0 +1,131 @@
# kube
This repo contains some kubernetes test stuff
## testapp
The testapp consists of three parts:
* srv - the frontend server
* db - simulating a database backend that gets called
* api - another service that calls an external service (httpbin.org)
Topology:
```
+--- db
/
srv ---+
\
+--- api --- httpbin.org
```
### Prepare
You need to have access from you local machine to the container registry deployed in you cluster on port 5000:
You can use ssh port-forwarding: `ssh k8s-master -L 5000:127.0.0.1:5000`
You need a local docker installation to build and upload the containers.
For the normal deployment the nginx ingress controller is used.
#### Ingress traffic
To get traffic to the ingress controller, install an nginx tcp proxy on the master node:
Install nginx: `apt install nginx -y`
Configure nginx to forward traffic to the node ports.
First get the ports of your ingress controller.
Look for the `80:<nodeport>` and `443:<nodeport>` mappings.
For nginx: `kubectl -n ingress-nginx get svc ingress-nginx`
For Istio: `kubectl -n istio-system get svc istio-ingressgateway`
Now add all your nodes as stream backends with the ports:
```
cat <<EOF> /etc/nginx/nginx.conf
load_module /usr/lib/nginx/modules/ngx_stream_module.so;
events {
worker_connections 1024;
}
stream {
upstream stream_backend_80 {
server 192.168.0.3:31380;
server 192.168.0.4:31380;
server 192.168.0.5:31380;
}
server {
listen 80;
proxy_pass stream_backend_80;
}
upstream stream_backend_443 {
server 192.168.0.3:31390;
server 192.168.0.4:31390;
server 192.168.0.5:31390;
}
server {
listen 443;
proxy_pass stream_backend_443;
}
}
EOF
```
Restart nginx: `systemctl restart nginx`
Now you can access your ingress via the masters IP on port 80 and 443.
### Build
New build the containers and upload them.
#### SRV
```
cd testapp/srv
VERSION=v1 make
```
#### DB
```
cd testapp/db
VERSION=v1 make
```
#### API
```
cd testapp/api
VERSION=v1 make
```
### Deploy
Deploy your application stack either using the simple versions or the `-istio` versions.
To use `-istio` versions you need to install istio in your cluster first.
**Note:** Change the `k8s-testapp.example.com` domain in the `srv` manifests to your domain.
#### Simple
```
cd testapp
kubectl apply -f api/api-v1.yaml
kubectl apply -f db/db.yaml
kubectl apply -f srv/srv-v1.yaml
```
#### Istio
```
cd testapp
kubectl apply -f api/api-v1-istio.yaml
kubectl apply -f db/db.yaml
kubectl apply -f srv/srv-v1-istio.yaml
```

2
loop.sh Executable file
View file

@ -0,0 +1,2 @@
#!/bin/sh
while true; do curl $1 2>/dev/null;sleep ${2:-1}; done

120
registry/registry-pvc.yaml Normal file
View file

@ -0,0 +1,120 @@
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: kube-registry-pvc
namespace: kube-system
labels:
k8s-app: kube-registry
spec:
storageClassName: rook-ceph-block
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 1Gi
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: kube-registry-v0
namespace: kube-system
labels:
k8s-app: kube-registry
version: v0
kubernetes.io/cluster-service: "true"
spec:
replicas: 1
selector:
matchLabels:
k8s-app: kube-registry
version: v0
template:
metadata:
labels:
k8s-app: kube-registry
version: v0
kubernetes.io/cluster-service: "true"
spec:
containers:
- name: registry
image: registry:2
resources:
limits:
cpu: 100m
memory: 100Mi
env:
- name: REGISTRY_HTTP_ADDR
value: :5000
- name: REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY
value: /var/lib/registry
volumeMounts:
- name: kube-registry
mountPath: /var/lib/registry
ports:
- containerPort: 5000
name: registry
protocol: TCP
volumes:
- name: kube-registry
persistentVolumeClaim:
claimName: kube-registry-pvc
---
apiVersion: v1
kind: Service
metadata:
name: kube-registry
namespace: kube-system
labels:
k8s-app: kube-registry
spec:
selector:
k8s-app: kube-registry
ports:
- name: registry
port: 5000
protocol: TCP
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: kube-registry-proxy
namespace: kube-system
labels:
k8s-app: kube-registry-proxy
tier: node
kubernetes.io/cluster-service: "true"
spec:
selector:
matchLabels:
name: kube-registry-proxy
tier: node
kubernetes.io/cluster-service: "true"
template:
metadata:
labels:
name: kube-registry-proxy
tier: node
kubernetes.io/cluster-service: "true"
annotations:
scheduler.alpha.kubernetes.io/critical-pod: ''
spec:
tolerations:
- key: node-role.kubernetes.io/master
effect: NoSchedule
containers:
- name: kube-registry-proxy
# image: gcr.io/google_containers/kube-registry-proxy:0.4
image: tapjoy/kube-registry-proxy:latest
resources:
limits:
cpu: 100m
memory: 50Mi
env:
- name: REGISTRY_HOST
value: kube-registry.kube-system.svc.cluster.local
- name: REGISTRY_PORT
value: "5000"
ports:
- name: registry
containerPort: 5000
hostPort: 5000

104
registry/registry.yaml Normal file
View file

@ -0,0 +1,104 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: kube-registry-v0
namespace: kube-system
labels:
k8s-app: kube-registry
version: v0
kubernetes.io/cluster-service: "true"
spec:
replicas: 1
selector:
matchLabels:
k8s-app: kube-registry
version: v0
template:
metadata:
labels:
k8s-app: kube-registry
version: v0
kubernetes.io/cluster-service: "true"
spec:
containers:
- name: registry
image: registry:2
resources:
limits:
cpu: 100m
memory: 100Mi
env:
- name: REGISTRY_HTTP_ADDR
value: :5000
- name: REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY
value: /var/lib/registry
volumeMounts:
- name: kube-registry
mountPath: /var/lib/registry
ports:
- containerPort: 5000
name: registry
protocol: TCP
volumes:
- name: kube-registry
emptyDir: {}
---
apiVersion: v1
kind: Service
metadata:
name: kube-registry
namespace: kube-system
labels:
k8s-app: kube-registry
spec:
selector:
k8s-app: kube-registry
ports:
- name: registry
port: 5000
protocol: TCP
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: kube-registry-proxy
namespace: kube-system
labels:
k8s-app: kube-registry-proxy
tier: node
kubernetes.io/cluster-service: "true"
spec:
selector:
matchLabels:
name: kube-registry-proxy
tier: node
kubernetes.io/cluster-service: "true"
template:
metadata:
labels:
name: kube-registry-proxy
tier: node
kubernetes.io/cluster-service: "true"
annotations:
scheduler.alpha.kubernetes.io/critical-pod: ''
spec:
tolerations:
- key: node-role.kubernetes.io/master
effect: NoSchedule
containers:
- name: kube-registry-proxy
# image: gcr.io/google_containers/kube-registry-proxy:0.4
image: tapjoy/kube-registry-proxy:latest
resources:
limits:
cpu: 100m
memory: 50Mi
env:
- name: REGISTRY_HOST
value: kube-registry.kube-system.svc.cluster.local
- name: REGISTRY_PORT
value: "5000"
ports:
- name: registry
containerPort: 5000
hostPort: 5000

15
testapp/api/Dockerfile Normal file
View file

@ -0,0 +1,15 @@
FROM golang:alpine
ARG VERSION
RUN mkdir /build
ADD . /build/
WORKDIR /build
RUN CGO_ENABLED=0 GOOS=linux go build -a -gcflags '-e' -ldflags "-X main.version=$VERSION -s -w" -o main .
RUN echo -e "root:x:0:0:root:/:\nappuser:x:1000:1000:appuser:/:" > /etc/passwd
RUN echo -e "root:x:0:root\nappuser:x:1000:" > /etc/group
FROM scratch
COPY --from=0 /build/main /
COPY --from=0 /etc/passwd /etc/
COPY --from=0 /etc/group /etc/
USER appuser
EXPOSE 8080
ENTRYPOINT ["/main"]

13
testapp/api/Makefile Normal file
View file

@ -0,0 +1,13 @@
CC=docker
APP=api
TAG=127.0.0.1:5000/testapp/$(APP):$(VERSION)
all: build push
build:
$(CC) build --build-arg VERSION=$(VERSION) -t $(TAG) .
push:
$(CC) push $(TAG)
.PHONY: build push

View file

@ -0,0 +1,109 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: testapp-api
labels:
app: testapp-api
version: v1
spec:
replicas: 1
selector:
matchLabels:
app: testapp-api
version: v1
template:
metadata:
labels:
app: testapp-api
version: v1
spec:
containers:
- name: testapp-api
image: 127.0.0.1:5000/testapp/api:v1
imagePullPolicy: Always
ports:
- containerPort: 8080
---
apiVersion: v1
kind: Service
metadata:
name: testapp-api
labels:
app: testapp-api
spec:
ports:
- port: 8080
name: http
selector:
app: testapp-api
---
apiVersion: networking.istio.io/v1alpha3
kind: ServiceEntry
metadata:
name: httpbin
spec:
hosts:
- httpbin.org
ports:
- number: 80
name: http-port
protocol: HTTP
resolution: DNS
location: MESH_EXTERNAL
---
apiVersion: networking.istio.io/v1alpha3
kind: Gateway
metadata:
name: httpbin
spec:
selector:
istio: egressgateway
servers:
- port:
number: 80
name: http
protocol: HTTP
hosts:
- httpbin.org
---
apiVersion: networking.istio.io/v1alpha3
kind: DestinationRule
metadata:
name: httpbin
spec:
host: istio-egressgateway.istio-system.svc.cluster.local
subsets:
- name: httpbin
---
apiVersion: networking.istio.io/v1alpha3
kind: VirtualService
metadata:
name: httpbin-egress
spec:
hosts:
- httpbin.org
gateways:
- httpbin
- mesh
http:
- match:
- gateways:
- mesh
port: 80
route:
- destination:
host: istio-egressgateway.istio-system.svc.cluster.local
subset: httpbin
port:
number: 80
weight: 100
- match:
- gateways:
- httpbin
port: 80
route:
- destination:
host: httpbin.org
port:
number: 80
weight: 100

38
testapp/api/api-v1.yaml Normal file
View file

@ -0,0 +1,38 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: testapp-api
labels:
app: testapp-api
version: v1
spec:
replicas: 1
selector:
matchLabels:
app: testapp-api
version: v1
template:
metadata:
labels:
app: testapp-api
version: v1
spec:
containers:
- name: testapp-api
image: 127.0.0.1:5000/testapp/api:v1
imagePullPolicy: Always
ports:
- containerPort: 8080
---
apiVersion: v1
kind: Service
metadata:
name: testapp-api
labels:
app: testapp-api
spec:
ports:
- port: 8080
name: http
selector:
app: testapp-api

63
testapp/api/main.go Normal file
View file

@ -0,0 +1,63 @@
package main
import (
"fmt"
"io/ioutil"
"log"
"net"
"net/http"
"os"
"time"
)
var version string
func get(c *http.Client, url string) (body string, err error) {
resp, err := c.Get(url)
if err != nil {
return
}
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return
}
body = string(b)
return
}
func main() {
http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
log.Printf("%s %s %s", r.RemoteAddr, r.Method, r.URL)
h, err := os.Hostname()
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
c := &http.Client{Transport: &http.Transport{
DialContext: (&net.Dialer{
Timeout: 5 * time.Second,
}).DialContext,
}}
health, err := get(c, "http://testapp-db:8080/healthz")
if err != nil {
health = err.Error()
}
ext, err := get(c, "http://httpbin.org/robots.txt")
if err != nil {
ext = err.Error()
}
fmt.Fprintf(w, "%s (%s) DB Health: '%s' external GET: '%s'", h, version, health, ext)
})
http.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
log.Printf("%s %s %s", r.RemoteAddr, r.Method, r.URL)
fmt.Fprintf(w, "api: ok")
})
s := &http.Server{
ReadTimeout: 5 * time.Second,
WriteTimeout: 10 * time.Second,
IdleTimeout: 120 * time.Second,
Addr: ":8080",
}
log.Fatal(s.ListenAndServe())
}

15
testapp/db/Dockerfile Normal file
View file

@ -0,0 +1,15 @@
FROM golang:alpine
ARG VERSION
RUN mkdir /build
ADD . /build/
WORKDIR /build
RUN CGO_ENABLED=0 GOOS=linux go build -a -gcflags '-e' -ldflags "-X main.version=$VERSION -s -w" -o main .
RUN echo -e "root:x:0:0:root:/:\nappuser:x:1000:1000:appuser:/:" > /etc/passwd
RUN echo -e "root:x:0:root\nappuser:x:1000:" > /etc/group
FROM scratch
COPY --from=0 /build/main /
COPY --from=0 /etc/passwd /etc/
COPY --from=0 /etc/group /etc/
USER appuser
EXPOSE 8080
ENTRYPOINT ["/main"]

13
testapp/db/Makefile Normal file
View file

@ -0,0 +1,13 @@
CC=docker
APP=db
TAG=127.0.0.1:5000/testapp/$(APP):$(VERSION)
all: build push
build:
$(CC) build --build-arg VERSION=$(VERSION) -t $(TAG) .
push:
$(CC) push $(TAG)
.PHONY: build push

38
testapp/db/db-v2.yaml Normal file
View file

@ -0,0 +1,38 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: testapp-db-v2
labels:
app: testapp-db
version: v2
spec:
replicas: 1
selector:
matchLabels:
app: testapp-db
version: v2
template:
metadata:
labels:
app: testapp-db
version: v2
spec:
containers:
- name: testapp-db
image: 127.0.0.1:5000/testapp/db:v2
imagePullPolicy: Always
ports:
- containerPort: 8080
---
apiVersion: v1
kind: Service
metadata:
name: testapp-db
labels:
app: testapp-db
spec:
ports:
- port: 8080
name: http
selector:
app: testapp-db

38
testapp/db/db.yaml Normal file
View file

@ -0,0 +1,38 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: testapp-db
labels:
app: testapp-db
version: v1
spec:
replicas: 1
selector:
matchLabels:
app: testapp-db
version: v1
template:
metadata:
labels:
app: testapp-db
version: v1
spec:
containers:
- name: testapp-db
image: 127.0.0.1:5000/testapp/db:v1
imagePullPolicy: Always
ports:
- containerPort: 8080
---
apiVersion: v1
kind: Service
metadata:
name: testapp-db
labels:
app: testapp-db
spec:
ports:
- port: 8080
name: http
selector:
app: testapp-db

View file

@ -0,0 +1,24 @@
apiVersion: "config.istio.io/v1alpha2"
kind: denier
metadata:
name: deny-testapp-db
spec:
status:
code: 7
message: Not allowed
---
apiVersion: "config.istio.io/v1alpha2"
kind: checknothing
metadata:
name: deny-testapp-db
spec:
---
apiVersion: "config.istio.io/v1alpha2"
kind: rule
metadata:
name: deny-testapp-db
spec:
match: destination.labels["app"] == "testapp-db"
actions:
- handler: deny-testapp-db.denier
instances: [ deny-testapp-db.checknothing ]

59
testapp/db/main.go Normal file
View file

@ -0,0 +1,59 @@
package main
import (
"fmt"
"io/ioutil"
"log"
"net"
"net/http"
"os"
"time"
)
var version string
func get(c *http.Client, url string) (body string, err error) {
resp, err := c.Get(url)
if err != nil {
return
}
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return
}
body = string(b)
return
}
func main() {
http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
log.Printf("%s %s %s", r.RemoteAddr, r.Method, r.URL)
h, err := os.Hostname()
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
c := &http.Client{Transport: &http.Transport{
DialContext: (&net.Dialer{
Timeout: 5 * time.Second,
}).DialContext,
}}
body, err := get(c, "http://testapp-api:8080/healthz")
if err != nil {
body = err.Error()
}
fmt.Fprintf(w, "%s (%s) API Health: '%s'", h, version, body)
})
http.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
log.Printf("%s %s %s", r.RemoteAddr, r.Method, r.URL)
fmt.Fprintf(w, "db: ok")
})
s := &http.Server{
ReadTimeout: 5 * time.Second,
WriteTimeout: 10 * time.Second,
IdleTimeout: 120 * time.Second,
Addr: ":8080",
}
log.Fatal(s.ListenAndServe())
}

34
testapp/db/route.yaml Normal file
View file

@ -0,0 +1,34 @@
apiVersion: networking.istio.io/v1alpha3
kind: VirtualService
metadata:
name: testapp-db
spec:
hosts:
- testapp-db.default.svc.cluster.local
http:
- route:
- destination:
host: testapp-db.default.svc.cluster.local
subset: v1
weight: 50
- destination:
host: testapp-db.default.svc.cluster.local
subset: v2
weight: 50
---
apiVersion: networking.istio.io/v1alpha3
kind: DestinationRule
metadata:
name: testapp-db
spec:
host: testapp-db.default.svc.cluster.local
trafficPolicy:
loadBalancer:
simple: RANDOM
subsets:
- name: v1
labels:
version: v1
- name: v2
labels:
version: v2

15
testapp/srv/Dockerfile Normal file
View file

@ -0,0 +1,15 @@
FROM golang:alpine
ARG VERSION
RUN mkdir /build
ADD . /build/
WORKDIR /build
RUN CGO_ENABLED=0 GOOS=linux go build -a -gcflags '-e' -ldflags "-X main.version=$VERSION -s -w" -o main .
RUN echo -e "root:x:0:0:root:/:\nappuser:x:1000:1000:appuser:/:" > /etc/passwd
RUN echo -e "root:x:0:root\nappuser:x:1000:" > /etc/group
FROM scratch
COPY --from=0 /build/main /
COPY --from=0 /etc/passwd /etc/
COPY --from=0 /etc/group /etc/
USER appuser
EXPOSE 8080
ENTRYPOINT ["/main"]

13
testapp/srv/Makefile Normal file
View file

@ -0,0 +1,13 @@
CC=docker
APP=srv
TAG=127.0.0.1:5000/testapp/$(APP):$(VERSION)
all: build push
build:
$(CC) build --build-arg VERSION=$(VERSION) -t $(TAG) .
push:
$(CC) push $(TAG)
.PHONY: build push

73
testapp/srv/main.go Normal file
View file

@ -0,0 +1,73 @@
package main
import (
"fmt"
"io/ioutil"
"log"
"net"
"net/http"
"os"
"os/user"
"strconv"
"time"
)
var version string
func get(c *http.Client, url string) (body string, err error) {
resp, err := c.Get(url)
if err != nil {
return
}
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return
}
body = string(b)
return
}
func main() {
http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
log.Printf("%s %s %s", r.RemoteAddr, r.Method, r.URL)
h, err := os.Hostname()
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
u, err := user.LookupId(strconv.Itoa(os.Getuid()))
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
//url := "http://" + os.Getenv("DB_SERVICE_HOST") + ":" + os.Getenv("DB_SERVICE_PORT")
//url := "http://testapp-db.default.svc.cluster.local:8080"
c := &http.Client{Transport: &http.Transport{
DialContext: (&net.Dialer{
Timeout: 5 * time.Second,
}).DialContext,
}}
dbUrl := "http://testapp-db:8080"
dbBody, err := get(c, dbUrl)
if err != nil {
dbBody = err.Error()
}
apiUrl := "http://testapp-api:8080"
apiBody, err := get(c, apiUrl)
if err != nil {
apiBody = err.Error()
}
fmt.Fprintf(w, "Version: %s\nHostname: %s\nUser: %s (%s) (%s)\nDB URL: %s\nDB Response: %s\nAPI URL: %s\nAPI Response: %s\n",
version, h, u.Username, u.Uid, u.Gid,
dbUrl, dbBody,
apiUrl, apiBody)
})
s := &http.Server{
ReadTimeout: 5 * time.Second,
WriteTimeout: 10 * time.Second,
IdleTimeout: 120 * time.Second,
Addr: ":8080",
}
log.Fatal(s.ListenAndServe())
}

View file

@ -0,0 +1,68 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: testapp-srv
labels:
app: testapp-srv
version: v1
spec:
replicas: 3
selector:
matchLabels:
app: testapp-srv
version: v1
template:
metadata:
labels:
app: testapp-srv
version: v1
spec:
containers:
- name: testapp-srv
image: 127.0.0.1:5000/testapp/srv:v1
imagePullPolicy: Always
ports:
- containerPort: 8080
---
apiVersion: v1
kind: Service
metadata:
name: testapp-srv
spec:
type: ClusterIP
ports:
- name: http
port: 8080
selector:
app: testapp-srv
---
apiVersion: networking.istio.io/v1alpha3
kind: Gateway
metadata:
name: testapp-srv
spec:
selector:
istio: ingressgateway
servers:
- port:
number: 80
name: http
protocol: HTTP
hosts:
- "k8s-testapp.example.com"
---
apiVersion: networking.istio.io/v1alpha3
kind: VirtualService
metadata:
name: testapp-srv
spec:
gateways:
- testapp-srv
hosts:
- "k8s-testapp.example.com"
http:
- route:
- destination:
port:
number: 8080
host: testapp-srv.default.svc.cluster.local

53
testapp/srv/srv-v1.yaml Normal file
View file

@ -0,0 +1,53 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: testapp-srv
labels:
app: testapp-srv
version: v1
spec:
replicas: 3
selector:
matchLabels:
app: testapp-srv
version: v1
template:
metadata:
labels:
app: testapp-srv
version: v1
spec:
containers:
- name: testapp-srv
image: 127.0.0.1:5000/testapp/srv:v1
imagePullPolicy: Always
ports:
- containerPort: 8080
---
apiVersion: v1
kind: Service
metadata:
name: testapp-srv
spec:
type: ClusterIP
ports:
- name: http
port: 8080
selector:
app: testapp-srv
---
---
apiVersion: networking.k8s.io/v1beta1
kind: Ingress
metadata:
name: testapp-srv
annotations:
kubernetes.io/ingress.class: "nginx"
spec:
rules:
- host: k8s-testapp.example.com
http:
paths:
- backend:
serviceName: testapp-srv
servicePort: 8080

26
wrk.go Normal file
View file

@ -0,0 +1,26 @@
package main
import (
"net/http"
"os"
"os/signal"
"syscall"
"time"
)
func get() {
for {
http.Get("http://127.0.0.1:8001/srv")
time.Sleep(time.Millisecond * 10)
}
}
func main() {
t := 4
for i := 0; i < t; i++ {
go get()
}
sigs := make(chan os.Signal)
signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
<-sigs
}