diff --git a/cmd/vipman/main.go b/cmd/vipman/main.go index dc8ed03..1cb66e7 100644 --- a/cmd/vipman/main.go +++ b/cmd/vipman/main.go @@ -1,19 +1,17 @@ package main import ( - "context" "flag" "os" - "os/exec" "os/signal" - "path/filepath" "syscall" - "time" "git.giftfish.de/ston1th/vipman/pkg/cluster" "git.giftfish.de/ston1th/vipman/pkg/config" + "git.giftfish.de/ston1th/vipman/pkg/controller" + "git.giftfish.de/ston1th/vipman/pkg/etcd" "git.giftfish.de/ston1th/vipman/pkg/raft" - "git.giftfish.de/ston1th/vipman/pkg/vip" + "github.com/go-logr/logr" "k8s.io/klog/v2" "k8s.io/klog/v2/klogr" ) @@ -21,105 +19,40 @@ import ( var ( configFile string version string + log logr.Logger ) -func AddIPs(n *vip.Network, cidrs []string) error { - for _, cidr := range cidrs { - err := n.AddIP(cidr) - if err != nil { - klog.Infof("error adding ip %s to interface %s: %s", cidr, n.Interface(), err) - } - } - return nil -} - -func DeleteIPs(n *vip.Network, cidrs []string) error { - for _, cidr := range cidrs { - err := n.DeleteIP(cidr) - if err != nil { - klog.Infof("error deleting ip %s from interface %s: %s", cidr, n.Interface(), err) - } - } - return nil -} - func main() { klog.InitFlags(nil) flag.StringVar(&configFile, "config", "vipman.yaml", "vipman config file") flag.Parse() + log = klogr.New().WithName("main") + log.Info("starting vipman", "version", version) cfg, err := config.ParseFile(configFile) if err != nil { - klog.Fatalf("Initialization failed: %s", err) + klog.Fatalf("init failed: %s", err) } - n, err := vip.NewNetworkWithLabel(cfg.Interface, cfg.IPLabel) - if err != nil { - klog.Fatalf("Initialization failed: %s", err) - } - cc := cfg.Cluster.Raft - c, err := raft.NewCluster(cluster.Callbacks{ - Leader: func(ctx context.Context, _ cluster.KV) { - t := time.NewTicker(time.Second * 10) - if cfg.LeaderHook != "" { - if !filepath.IsAbs(cfg.LeaderHook) { - klog.Infof("error running leaderHook %s: path is not absolute", cfg.LeaderHook) - } else { - go func() { - err := exec.CommandContext(ctx, cfg.LeaderHook).Run() - if err != nil { - klog.Infof("error running leaderHook %s: %s", cfg.LeaderHook, err) - } - }() - } - } - for { - klog.Infof("[%s] leading", cc.ID) - AddIPs(n, cfg.VirtualIPs) - select { - case <-ctx.Done(): - t.Stop() - DeleteIPs(n, cfg.VirtualIPs) - klog.Infof("[%s] leading canceled", cc.ID) - return - case <-t.C: - } - } - }, - Follower: func(ctx context.Context, _ cluster.KV) { - t := time.NewTicker(time.Second * 10) - if cfg.FollowerHook != "" { - if !filepath.IsAbs(cfg.FollowerHook) { - klog.Infof("error running followerHook %s: path is not absolute", cfg.FollowerHook) - } else { - go func() { - err := exec.CommandContext(ctx, cfg.FollowerHook).Run() - if err != nil { - klog.Infof("error running FollowerHook %s: %s", cfg.FollowerHook, err) - } - }() - } - } - for { - klog.Infof("[%s] following", cc.ID) - //DeleteIPs(n, cfg.VirtualIPs) - select { - case <-ctx.Done(): - t.Stop() - klog.Infof("[%s] following canceled", cc.ID) - return - case <-t.C: - } - } - }, - }) - if err != nil { - klog.Fatalf("Initialization failed: %s", err) + var c cluster.Cluster + if cfg.Cluster.Raft != nil { + log.Info("creating cluster", "mode", "raft") + c = raft.NewCluster(klogr.New().WithName("raft")) + } else if cfg.Cluster.Etcd != nil { + log.Info("creating cluster", "mode", "etcd") + c = etcd.NewCluster(klogr.New().WithName("etcd")) + } + callbacks, err := controller.NewVIPController(cfg) + if err != nil { + klog.Fatalf("init failed: %s", err) + } + err = c.SetCallbacks(callbacks) + if err != nil { + klog.Fatalf("init failed: %s", err) } - log := klogr.New() go func() { - err = c.Start(log, cfg) + err = c.Start(cfg) if err != nil { - klog.Fatalf("Initialization failed: %s", err) + klog.Fatalf("init failed: %s", err) } }() go func() { @@ -133,8 +66,9 @@ func main() { sigs := make(chan os.Signal) signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM) <-sigs + log.Info("stopping vipman") c.Stop() - DeleteIPs(n, cfg.VirtualIPs) + //DeleteIPs(n, cfg.VirtualIPs) } /* diff --git a/etcd.sh b/etcd.sh new file mode 100755 index 0000000..3a7046f --- /dev/null +++ b/etcd.sh @@ -0,0 +1,12 @@ +REGISTRY=gcr.io/etcd-development/etcd +NODE1=127.0.0.1 +docker run \ + --rm -ti \ + -p 2379:2379 \ + -p 2380:2380 \ + --name etcd ${REGISTRY}:latest \ + /usr/local/bin/etcd \ + --data-dir=/etcd-data --name node1 \ + --initial-advertise-peer-urls http://${NODE1}:2380 --listen-peer-urls http://0.0.0.0:2380 \ + --advertise-client-urls http://${NODE1}:2379 --listen-client-urls http://0.0.0.0:2379 \ + --initial-cluster node1=http://${NODE1}:2380 diff --git a/go.mod b/go.mod index cda5aac..cf5d57d 100644 --- a/go.mod +++ b/go.mod @@ -4,12 +4,24 @@ go 1.15 require ( git.giftfish.de/ston1th/raftbbolt v0.0.0-20201114165203-b85c94bfc9b0 - github.com/fatih/color v1.10.0 // indirect + github.com/coreos/etcd v3.3.25+incompatible + github.com/coreos/go-semver v0.3.0 // indirect + github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf // indirect + github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect + github.com/envoyproxy/go-control-plane v0.9.4 // indirect github.com/go-logr/logr v0.3.0 + github.com/gogo/protobuf v1.3.1 // indirect + github.com/golang/protobuf v1.4.3 // indirect + github.com/google/go-cmp v0.5.0 // indirect + github.com/google/uuid v1.1.2 // indirect github.com/hashicorp/go-hclog v0.15.0 github.com/hashicorp/raft v1.2.0 github.com/vishvananda/netlink v1.1.0 - golang.org/x/sys v0.0.0-20201113233024-12cec1faf1ba // indirect + go.etcd.io/etcd v3.3.25+incompatible + go.uber.org/zap v1.16.0 + google.golang.org/grpc v1.26.0 // indirect gopkg.in/yaml.v2 v2.3.0 k8s.io/klog/v2 v2.4.0 ) + +replace github.com/coreos/bbolt => go.etcd.io/bbolt v1.3.5 diff --git a/go.sum b/go.sum index 6a3a140..7d9fc0c 100644 --- a/go.sum +++ b/go.sum @@ -1,66 +1,100 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= git.giftfish.de/ston1th/raftbbolt v0.0.0-20201114165203-b85c94bfc9b0 h1:DU0PVdA2EHz2LFGEdkwgJopL9HQh5nFC9GJ7oqzZ300= git.giftfish.de/ston1th/raftbbolt v0.0.0-20201114165203-b85c94bfc9b0/go.mod h1:9dF/aHYkXts6/6oVyJ+vzXCn7NURDaoREaanX0aspx0= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 h1:EFSB7Zo9Eg91v7MJPVsifUysc/wPdN+NOnVe6bWbdBM= github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878/go.mod h1:3AMJUQhVx52RsWOnlkpikZr01T/yAVN2gn0861vByNg= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4= github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/coreos/etcd v0.5.0-alpha.5 h1:0Qi6Jzjk2CDuuGlIeecpu+em2nrjhOgz2wsIwCmQHmc= +github.com/coreos/etcd v3.3.25+incompatible h1:0GQEw6h3YnuOVdtwygkIfJ+Omx0tZ8/QkVyXI4LkbeY= +github.com/coreos/etcd v3.3.25+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= +github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf h1:iW4rZ826su+pqaw19uhpSCzhj44qo35pNgKFGqzDKkU= +github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg= -github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v0.3.0 h1:q4c+kbcR0d5rSurhBR8dIgieOaYpXtsdTYfx22Cu6rs= github.com/go-logr/logr v0.3.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= +github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= +github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1 h1:ZFgWrT+bLgsYPirOnRfKLYJLvssAegOj/hgyMFdJZe0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= -github.com/hashicorp/go-hclog v0.9.1 h1:9PZfAcVEvez4yhLH2TBU64/h/z4xlFI80cWXRrxuKuM= github.com/hashicorp/go-hclog v0.9.1/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= github.com/hashicorp/go-hclog v0.15.0 h1:qMuK0wxsoW4D0ddCCYwPSTm4KQv1X1ke3WmPWZ0Mvsk= github.com/hashicorp/go-hclog v0.15.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= github.com/hashicorp/go-immutable-radix v1.0.0 h1:AKDB1HM5PWEA7i4nhcpwOrO2byshxBjXVn/J/3+z5/0= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-msgpack v0.5.5 h1:i9R9JSrqIz0QVLz3sz+i3YJdT7TTSLcfLLzJi9aZTuI= github.com/hashicorp/go-msgpack v0.5.5/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-msgpack v1.1.5 h1:9byZdVjKTe5mce63pRVNP1L7UAmdHOTEMGehn6KvJWs= github.com/hashicorp/go-msgpack v1.1.5/go.mod h1:gWVc3sv/wbDmR3rQsj1CAktEZzoz1YNK9NfGLXJ69/4= github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= -github.com/hashicorp/go-uuid v1.0.0 h1:RS8zrF7PhGwyNPOtxSClXXj9HA8feRnJzgnI1RJCSnM= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/raft v1.2.0 h1:mHzHIrF0S91d3A7RPBvuqkgB4d/7oFJZyvf1Q4m7GA0= github.com/hashicorp/raft v1.2.0/go.mod h1:vPAJM8Asw6u8LxC3eJCUZmRP/E4QmUGE1R7g7k8sG/8= github.com/hashicorp/raft-boltdb v0.0.0-20171010151810-6e5ba93211ea/go.mod h1:pNv7Wc3ycL6F5oOWn+tPGo2gWD4a5X+yp/ntwdKLjRk= +github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= -github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8= -github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.10 h1:qxFzApOv4WsAL965uUPIsXzAKCZxN2p9UqdhFS4ZW10= github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= -github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= -github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/vishvananda/netlink v1.1.0 h1:1iyaYNBLmP6L0220aDnYQpo1QEV4t4hJ+xEEhhJH8j0= github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= @@ -68,27 +102,95 @@ github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df h1:OviZH7qLw/7Zo github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= +go.etcd.io/etcd v0.5.0-alpha.5 h1:VOolFSo3XgsmnYDLozjvZ6JL6AAwIDu1Yx1y+4EYLDo= +go.etcd.io/etcd v3.3.25+incompatible h1:V1RzkZJj9LqsJRy+TUBgpWSbZXITLB819lstuTFoZOY= +go.etcd.io/etcd v3.3.25+incompatible/go.mod h1:yaeTdrJi5lOmYerz05bd8+V7KubZs8YSFZfzsF9A6aI= +go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk= +go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/multierr v1.5.0 h1:KCa4XfM8CWFCpxXRGok+Q0SS/0XBhMDbHHGABQLvD2A= +go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= +go.uber.org/zap v1.16.0 h1:uFRZXykJGK9lLY4HtgSw44DnIcAM+kRBP7x5m+NpAOM= +go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190523142557-0e01d883c5c5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5 h1:LfCXLvNmTYH9kEmVgqbnsWfruoXZIrh4YBgqVHtDvw0= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201113233024-12cec1faf1ba h1:xmhUJGQGbxlod18iJGqVEp9cHIPLl7QiX2aA3to708s= -golang.org/x/sys v0.0.0-20201113233024-12cec1faf1ba/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190424220101-1e8e1cfdf96b/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.26.0 h1:2dTRdpdFEEhJYQD8EMLB61nnrzSCTbG38PhqdhvOltg= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0 h1:rRYRFMVgRv6E0D70Skyfsr28tDXIuuPZyWGMPdMcnXg= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.33.2 h1:EQyQC3sa8M+p6Ulc8yy9SWSS2GVwyRc83gAbG8lrl4o= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= k8s.io/klog/v2 v2.4.0 h1:7+X0fUguPyrKEC4WjH8iGDg3laWgMo5tMnRTIGTTxGQ= k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index b4eaef1..599da12 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -8,7 +8,8 @@ import ( ) type Cluster interface { - Start(logr.Logger, *config.Config) error + Start(*config.Config) error + SetCallbacks(Callbacks) error Stepdown() Stop() } @@ -19,7 +20,13 @@ type KV interface { Delete(k string) error } -type Callbacks struct { - Leader func(context.Context, KV) - Follower func(context.Context, KV) +type CallbackContext interface { + logr.Logger + KV + ID() string +} + +type Callbacks struct { + Leader func(context.Context, CallbackContext) + Follower func(context.Context, CallbackContext) } diff --git a/pkg/config/ca.go b/pkg/config/ca.go new file mode 100644 index 0000000..2304117 --- /dev/null +++ b/pkg/config/ca.go @@ -0,0 +1,19 @@ +package config + +import ( + "crypto/x509" + "io/ioutil" +) + +func LoadCertPool(file string) (p *x509.CertPool, err error) { + if file == "" { + return + } + buf, err := ioutil.ReadFile(file) + if err != nil { + return + } + p = x509.NewCertPool() + p.AppendCertsFromPEM(buf) + return +} diff --git a/pkg/config/config.go b/pkg/config/config.go index da723ca..6827588 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -4,13 +4,19 @@ import ( "errors" "fmt" "os" + "path/filepath" + "strings" "gopkg.in/yaml.v2" ) const ( - RaftDir = "raft" - RaftPort = 7001 + RaftDir = "raft" + RaftPort = 7001 + RaftLogLevel = "off" + + EtcdLogLevel = "error" + EtcdPrefix = "/vipman" ) func ParseFile(file string) (cfg *Config, err error) { @@ -35,14 +41,24 @@ func Validate(c *Config) error { if len(c.VirtualIPs) == 0 { return errors.New("missing virtualIPs config") } + if c.LeaderHook != "" { + if !filepath.IsAbs(c.LeaderHook) { + return errors.New("leaderHook path must be absolute") + } + } + if c.FollowerHook != "" { + if !filepath.IsAbs(c.FollowerHook) { + return errors.New("followerHook path must be absolute") + } + } raft := c.Cluster.Raft if raft != nil { + if raft.LogLevel == "" { + raft.LogLevel = RaftLogLevel + } if raft.Dir == "" { raft.Dir = RaftDir } - if raft.Level == "" { - raft.Level = "off" - } if raft.ID == "" { return errors.New("missing raft.id config") } @@ -63,7 +79,29 @@ func Validate(c *Config) error { } etcd := c.Cluster.Etcd if etcd != nil { - + if etcd.LogLevel == "" { + etcd.LogLevel = EtcdLogLevel + } + if len(etcd.Endpoints) == 0 { + return errors.New("missing etcd.endpoints config") + } + if etcd.Prefix == "" { + etcd.Prefix = EtcdPrefix + } else { + if !strings.HasPrefix(etcd.Prefix, "/") { + etcd.Prefix = "/" + etcd.Prefix + } + if strings.HasSuffix(etcd.Prefix, "/") { + etcd.Prefix = strings.TrimSuffix(etcd.Prefix, "/") + } + } + if etcd.ClusterName == "" { + return errors.New("missing etcd.clusterName config") + } else { + if strings.HasSuffix(etcd.ClusterName, "/") { + etcd.ClusterName = strings.TrimSuffix(etcd.ClusterName, "/") + } + } } if raft == nil && etcd == nil { return errors.New("missing cluster config: raft or etcd") @@ -75,17 +113,12 @@ func Validate(c *Config) error { } type Config struct { - VirtualIPs []string `yaml:"virtualIPs"` - - Interface string `yaml:"interface,omitempty"` - - IPLabel string `yaml:"label,omitempty"` - - LeaderHook string `yaml:"leaderHook,omitempty"` - - FollowerHook string `yaml:"followerHook,omitempty"` - - Cluster Cluster `yaml:"cluster"` + VirtualIPs []string `yaml:"virtualIPs"` + Interface string `yaml:"interface,omitempty"` + Label string `yaml:"label,omitempty"` + LeaderHook string `yaml:"leaderHook,omitempty"` + FollowerHook string `yaml:"followerHook,omitempty"` + Cluster Cluster `yaml:"cluster"` } type Cluster struct { @@ -94,17 +127,17 @@ type Cluster struct { } type Raft struct { - Dir string `yaml:"dir,omitempty"` - ID string `yaml:"id"` - Address string `yaml:"address"` - Level string `yaml:"level"` - Peers []RaftPeer `yaml:"peers"` - TLS *TLS `yaml:"tls,omitempty"` + Dir string `yaml:"dir,omitempty"` + ID string `yaml:"id"` + Address string `yaml:"address"` + Peers []RaftPeer `yaml:"peers"` + TLS *TLS `yaml:"tls,omitempty"` + LogLevel string `yaml:"logLevel,omitempty"` } type TLS struct { - Cert string `yaml:"cert"` - Key string `yaml:"key"` + Cert string `yaml:"cert,omitempty"` + Key string `yaml:"key,omitempty"` CA string `yaml:"ca,omitempty"` Insecure bool `yaml:"insecure,omitempty"` } @@ -119,4 +152,8 @@ type Etcd struct { Endpoints []string `yaml:"endpoints"` Prefix string `yaml:"prefix"` ClusterName string `yaml:"clusterName"` + Username string `yaml:"username,omitempty"` + Password string `yaml:"password,omitempty"` + TLS *TLS `yaml:"tls,omitempty"` + LogLevel string `yaml:"logLevel,omitempty"` } diff --git a/pkg/controller/vip.go b/pkg/controller/vip.go new file mode 100644 index 0000000..353882c --- /dev/null +++ b/pkg/controller/vip.go @@ -0,0 +1,88 @@ +package controller + +import ( + "context" + "os/exec" + "time" + + "git.giftfish.de/ston1th/vipman/pkg/cluster" + "git.giftfish.de/ston1th/vipman/pkg/config" + "git.giftfish.de/ston1th/vipman/pkg/vip" + "github.com/go-logr/logr" +) + +func NewVIPController(cfg *config.Config) (callbacks cluster.Callbacks, err error) { + n, err := vip.NewNetworkWithLabel(cfg.Interface, cfg.Label) + if err != nil { + return + } + callbacks = cluster.Callbacks{ + Leader: func(ctx context.Context, cc cluster.CallbackContext) { + t := time.NewTicker(time.Second * 10) + if cfg.LeaderHook != "" { + go func() { + err := exec.CommandContext(ctx, cfg.LeaderHook).Run() + if err != nil { + cc.Error(err, "error running hook", "leaderHook", cfg.LeaderHook) + } + }() + } + for { + cc.Info("leading", "id", cc.ID()) + addIPs(cc, n, cfg.VirtualIPs) + select { + case <-ctx.Done(): + t.Stop() + deleteIPs(cc, n, cfg.VirtualIPs) + cc.Info("leading canceled", "id", cc.ID()) + return + case <-t.C: + } + } + }, + Follower: func(ctx context.Context, cc cluster.CallbackContext) { + t := time.NewTicker(time.Second * 10) + if cfg.FollowerHook != "" { + go func() { + err := exec.CommandContext(ctx, cfg.FollowerHook).Run() + if err != nil { + cc.Error(err, "error running hook", "followerHook", cfg.FollowerHook) + } + }() + } + for { + cc.Info("following", "id", cc.ID()) + //TODO + //deleteIPs(cc, n, cfg.VirtualIPs) + select { + case <-ctx.Done(): + t.Stop() + cc.Info("following canceled", "id", cc.ID()) + return + case <-t.C: + } + } + }, + } + return +} + +func addIPs(log logr.Logger, n *vip.Network, cidrs []string) error { + for _, cidr := range cidrs { + err := n.AddIP(cidr) + if err != nil { + log.Error(err, "adding ip", "interface", n.Interface(), "ip", cidr) + } + } + return nil +} + +func deleteIPs(log logr.Logger, n *vip.Network, cidrs []string) error { + for _, cidr := range cidrs { + err := n.DeleteIP(cidr) + if err != nil { + log.Error(err, "deleting ip", "interface", n.Interface(), "ip", cidr) + } + } + return nil +} diff --git a/pkg/etcd/cluster.go b/pkg/etcd/cluster.go new file mode 100644 index 0000000..10d8897 --- /dev/null +++ b/pkg/etcd/cluster.go @@ -0,0 +1,55 @@ +package etcd + +import ( + "errors" + + "git.giftfish.de/ston1th/vipman/pkg/cluster" + //"go.etcd.io/etcd/clientv3" + "github.com/coreos/etcd/clientv3" + "github.com/go-logr/logr" +) + +type Cluster struct { + logr.Logger + cluster.KV + cli *clientv3.Client + + stepdown chan struct{} + stop chan struct{} + done chan struct{} + callbacks cluster.Callbacks + + id string + keyID string + session bool + cancelSession func() +} + +func NewCluster(log logr.Logger) cluster.Cluster { + return &Cluster{ + Logger: log, + KV: &kv{}, + stepdown: make(chan struct{}, 1), + stop: make(chan struct{}, 1), + done: make(chan struct{}, 1), + } +} + +func (c *Cluster) SetCallbacks(callbacks cluster.Callbacks) error { + c.callbacks = callbacks + return c.checkConfig() +} + +func (c *Cluster) ID() string { + return c.id +} + +func (c *Cluster) checkConfig() error { + if c.callbacks.Leader == nil { + return errors.New("Leader callback is nil") + } + if c.callbacks.Follower == nil { + return errors.New("Follower callback is nil") + } + return nil +} diff --git a/pkg/etcd/etcd.go b/pkg/etcd/etcd.go new file mode 100644 index 0000000..9c70479 --- /dev/null +++ b/pkg/etcd/etcd.go @@ -0,0 +1,217 @@ +package etcd + +import ( + "context" + "crypto/tls" + "fmt" + "time" + + "git.giftfish.de/ston1th/vipman/pkg/config" + "git.giftfish.de/ston1th/vipman/pkg/util" + "github.com/coreos/etcd/clientv3" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + //"go.etcd.io/etcd/clientv3" + "go.etcd.io/etcd/clientv3/concurrency" +) + +func (c *Cluster) Start(etcdcfg *config.Config) error { + err := c.checkConfig() + if err != nil { + return err + } + cfg := etcdcfg.Cluster.Etcd + var level zapcore.Level + err = level.Set(cfg.LogLevel) + if err != nil { + return err + } + al := zap.NewAtomicLevel() + al.SetLevel(level) + var tlsConfig *tls.Config + if cfg.TLS != nil { + ca, err := config.LoadCertPool(cfg.TLS.CA) + if err != nil { + return err + } + tlsConfig = &tls.Config{ + MinVersion: tls.VersionTLS12, + InsecureSkipVerify: cfg.TLS.Insecure, + RootCAs: ca, + } + } + + c.cli, err = clientv3.New(clientv3.Config{ + Endpoints: cfg.Endpoints, + TLS: tlsConfig, + DialTimeout: time.Second * 5, + Username: cfg.Username, + Password: cfg.Password, + LogConfig: &zap.Config{ + Level: al, + Encoding: "console", + DisableCaller: true, + DisableStacktrace: true, + EncoderConfig: zap.NewDevelopmentEncoderConfig(), + OutputPaths: []string{"stderr"}, + ErrorOutputPaths: []string{"stderr"}, + }, + }) + if err != nil { + return err + } + + ctxSsession, cancelSession := context.WithCancel(context.Background()) + c.cancelSession = cancelSession + s, err := concurrency.NewSession(c.cli, + concurrency.WithContext(ctxSsession), + concurrency.WithTTL(10), + ) + if err != nil { + return err + } + c.session = true + + prefix := cfg.Prefix + "/" + cfg.ClusterName + c.id = fmt.Sprintf("%x", s.Lease()) + c.keyID = fmt.Sprintf("%s/%s", prefix, c.id) + + e := concurrency.NewElection(s, prefix) + ctxCampaign, cancelCampaign := context.WithCancel(context.Background()) + go func() { + errc := make(chan error, 1) + for { + go func() { + errc <- e.Campaign(ctxCampaign, "leader") + }() + select { + case err := <-errc: + if err != nil { + time.Sleep(time.Second) + } + case <-ctxCampaign.Done(): + return + } + } + }() + + t := time.NewTicker(time.Second) + ctx, cancel := context.WithCancel(context.Background()) + ctxObserve, cancelObserve := context.WithCancel(context.Background()) + leading := false + following := false + leaderChan := c.observe(ctxObserve, e.Observe(ctxObserve)) + for { + if !leading && c.leaderID(e.Leader(ctxObserve)) == c.keyID { + leading = true + go c.leader(ctx) + } else if !following && !leading { + following = true + go c.follower(ctx) + } + + select { + case <-t.C: + case leader := <-leaderChan: + if leader { + if following { + cancel() + following = false + ctx, cancel = context.WithCancel(context.Background()) + } + if !leading { + leading = true + go c.leader(ctx) + } + } else { + if leading { + cancel() + leading = false + ctx, cancel = context.WithCancel(context.Background()) + } + if !following { + following = true + go c.follower(ctx) + } + } + case <-c.stepdown: + cancel() + ctx, cancel = context.WithCancel(context.Background()) + ctxResign, _ := context.WithTimeout(context.Background(), time.Second*2) + e.Resign(ctxResign) + case <-c.stop: + t.Stop() + cancelObserve() + cancelCampaign() + cancel() + if leading { + ctxResign, _ := context.WithTimeout(context.Background(), time.Second*2) + e.Resign(ctxResign) + } + s.Close() + close(c.done) + return nil + } + } + return nil +} + +func (c *Cluster) leaderID(r *clientv3.GetResponse, err error) string { + if err != nil { + return "" + } + if len(r.Kvs) == 0 { + return "" + } + return string(r.Kvs[0].Key) +} + +func (c *Cluster) observe(ctx context.Context, resp <-chan clientv3.GetResponse) <-chan bool { + leader := make(chan bool) + go func() { + for { + select { + case r := <-resp: + if c.leaderID(&r, nil) == c.keyID { + leader <- true + } else { + leader <- false + } + case <-ctx.Done(): + return + } + } + }() + return leader +} + +func (c *Cluster) Stepdown() { + select { + case c.stepdown <- struct{}{}: + case <-time.After(time.Second * 2): + } +} + +func (c *Cluster) Stop() { + if !c.session { + c.cancelSession() + return + } + close(c.stop) + <-c.done + c.cli.Close() +} + +func (c *Cluster) leader(ctx context.Context) { + defer func() { + util.HandleCrash() + }() + c.callbacks.Leader(ctx, c) +} + +func (c *Cluster) follower(ctx context.Context) { + defer func() { + util.HandleCrash() + }() + c.callbacks.Follower(ctx, c) +} diff --git a/pkg/etcd/kv.go b/pkg/etcd/kv.go new file mode 100644 index 0000000..f229335 --- /dev/null +++ b/pkg/etcd/kv.go @@ -0,0 +1,18 @@ +package etcd + +//TODO + +type kv struct { +} + +func (kv *kv) Get(k string) []byte { + return nil +} + +func (kv *kv) Set(k string, v []byte) error { + return nil +} + +func (kv *kv) Delete(k string) error { + return nil +} diff --git a/pkg/raft/cluster.go b/pkg/raft/cluster.go index 1910240..e581102 100644 --- a/pkg/raft/cluster.go +++ b/pkg/raft/cluster.go @@ -4,33 +4,53 @@ import ( "errors" "git.giftfish.de/ston1th/vipman/pkg/cluster" + "github.com/go-logr/logr" "github.com/hashicorp/raft" ) type Cluster struct { - kv *KV - r *raft.Raft + logr.Logger + cluster.KV + r *raft.Raft + kvm *kv stepdown chan struct{} stop chan struct{} done chan struct{} - Callbacks cluster.Callbacks + callbacks cluster.Callbacks + + id string } -func NewCluster(callbacks cluster.Callbacks) (cluster.Cluster, error) { - if callbacks.Leader == nil { - return nil, errors.New("Leader is nil") - } - if callbacks.Follower == nil { - return nil, errors.New("Follower is nil") +func NewCluster(log logr.Logger) cluster.Cluster { + kvm := &kv{ + m: make(map[string][]byte), } return &Cluster{ - kv: &KV{ - m: make(map[string][]byte), - }, - stepdown: make(chan struct{}, 1), - stop: make(chan struct{}, 1), - done: make(chan struct{}, 1), - Callbacks: callbacks, - }, nil + Logger: log, + KV: kvm, + kvm: kvm, + stepdown: make(chan struct{}, 1), + stop: make(chan struct{}, 1), + done: make(chan struct{}, 1), + } +} + +func (c *Cluster) ID() string { + return c.id +} + +func (c *Cluster) SetCallbacks(callbacks cluster.Callbacks) error { + c.callbacks = callbacks + return c.checkConfig() +} + +func (c *Cluster) checkConfig() error { + if c.callbacks.Leader == nil { + return errors.New("Leader callback is nil") + } + if c.callbacks.Follower == nil { + return errors.New("Follower callback is nil") + } + return nil } diff --git a/pkg/raft/fsm.go b/pkg/raft/fsm.go index 5a3de37..33817ee 100644 --- a/pkg/raft/fsm.go +++ b/pkg/raft/fsm.go @@ -27,7 +27,7 @@ type raftCmd struct { V []byte } -type fsm KV +type fsm kv func (f *fsm) Apply(l *raft.Log) interface{} { var c raftCmd @@ -49,7 +49,7 @@ func (f *fsm) Apply(l *raft.Log) interface{} { func (f *fsm) Snapshot() (raft.FSMSnapshot, error) { f.mu.Lock() defer f.mu.Unlock() - m := make(kv) + m := make(kvm) for k, v := range f.m { m[k] = v } @@ -57,7 +57,7 @@ func (f *fsm) Snapshot() (raft.FSMSnapshot, error) { } func (f *fsm) Restore(rc io.ReadCloser) error { - o := make(kv) + o := make(kvm) if err := msgpack.NewDecoder(rc).Decode(&o); err != nil { return err } @@ -78,7 +78,7 @@ func (f *fsm) applyDelete(k string) { } type fsmSnapshot struct { - m kv + m kvm } func (f *fsmSnapshot) Persist(sink raft.SnapshotSink) error { diff --git a/pkg/raft/kv.go b/pkg/raft/kv.go index 69ee90f..baaed80 100644 --- a/pkg/raft/kv.go +++ b/pkg/raft/kv.go @@ -10,21 +10,21 @@ import ( var ErrNotLeader = errors.New("not leader") -type kv map[string][]byte +type kvm map[string][]byte -type KV struct { +type kv struct { mu sync.Mutex - m kv + m kvm r *raft.Raft } -func (kv *KV) Get(k string) []byte { +func (kv *kv) Get(k string) []byte { kv.mu.Lock() defer kv.mu.Unlock() return kv.m[k] } -func (kv *KV) Set(k string, v []byte) error { +func (kv *kv) Set(k string, v []byte) error { if kv.r.State() != raft.Leader { return ErrNotLeader } @@ -40,7 +40,7 @@ func (kv *KV) Set(k string, v []byte) error { } // Delete deletes the given key. -func (kv *KV) Delete(k string) error { +func (kv *kv) Delete(k string) error { if kv.r.State() != raft.Leader { return ErrNotLeader } diff --git a/pkg/raft/raft.go b/pkg/raft/raft.go index cf82225..59fa2b7 100644 --- a/pkg/raft/raft.go +++ b/pkg/raft/raft.go @@ -4,32 +4,27 @@ import ( "context" "fmt" "net" - "net/http" "os" "path/filepath" - "runtime" "time" "git.giftfish.de/ston1th/raftbbolt" "git.giftfish.de/ston1th/vipman/pkg/config" "git.giftfish.de/ston1th/vipman/pkg/raft/tls" - "github.com/go-logr/logr" + "git.giftfish.de/ston1th/vipman/pkg/util" hclog "github.com/hashicorp/go-hclog" "github.com/hashicorp/raft" - "k8s.io/klog/v2" ) -type raftLog string - -func (r *raftLog) Write(p []byte) (n int, err error) { - return os.Stdout.Write(append([]byte(*r), p...)) -} - -func (c *Cluster) Start(log logr.Logger, raftcfg *config.Config) error { +func (c *Cluster) Start(raftcfg *config.Config) error { + err := c.checkConfig() + if err != nil { + return err + } cfg := raftcfg.Cluster.Raft hcl := hclog.New(&hclog.LoggerOptions{ Name: "raft", - Level: hclog.LevelFromString(cfg.Level), + Level: hclog.LevelFromString(cfg.LogLevel), Output: hclog.DefaultOutput, TimeFormat: "I0102 15:04:05.000000", }) @@ -38,19 +33,20 @@ func (c *Cluster) Start(log logr.Logger, raftcfg *config.Config) error { rc.ProtocolVersion = 3 rc.LocalID = raft.ServerID(cfg.ID) rc.Logger = hcl + c.id = cfg.ID pool := 3 timeout := 10 * time.Second var transport *raft.NetworkTransport if cfg.TLS != nil { - log.Info("transport: tls") + c.V(1).Info("setup", "transport", "tls") s, err := tls.NewStream(cfg) if err != nil { return err } transport = raft.NewNetworkTransportWithLogger(s, pool, timeout, hcl) } else { - log.Info("transport: tcp") + c.V(1).Info("setup", "transport", "tcp") address, err := net.ResolveTCPAddr("tcp", cfg.Address) if err != nil { return err @@ -104,16 +100,13 @@ func (c *Cluster) Start(log logr.Logger, raftcfg *config.Config) error { } } - // Create RAFT instance - c.r, err = raft.NewRaft(rc, (*fsm)(c.kv), logStore, stableStore, snapshots, transport) + c.r, err = raft.NewRaft(rc, (*fsm)(c.kvm), logStore, stableStore, snapshots, transport) if err != nil { return err } - c.kv.r = c.r + c.kvm.r = c.r t := time.NewTicker(time.Second) - time.Sleep(time.Second * 3) - ctx, cancel := context.WithCancel(context.Background()) defer cancel() leading := false @@ -128,6 +121,7 @@ func (c *Cluster) Start(log logr.Logger, raftcfg *config.Config) error { } select { + case <-t.C: case leader := <-c.r.LeaderCh(): if leader { if following { @@ -163,7 +157,6 @@ func (c *Cluster) Start(log logr.Logger, raftcfg *config.Config) error { } close(c.done) return nil - case <-t.C: } } return nil @@ -171,49 +164,16 @@ func (c *Cluster) Start(log logr.Logger, raftcfg *config.Config) error { func (c *Cluster) leader(ctx context.Context) { defer func() { - handleCrash() + util.HandleCrash() }() - c.Callbacks.Leader(ctx, c.kv) + c.callbacks.Leader(ctx, c) } func (c *Cluster) follower(ctx context.Context) { defer func() { - handleCrash() + util.HandleCrash() }() - c.Callbacks.Follower(ctx, c.kv) -} - -var reallyCrash = false - -func handleCrash() { - if r := recover(); r != nil { - logPanic(r) - if reallyCrash { - panic(r) - } - } -} - -// logPanic logs the caller tree when a panic occurs (except in the special case of http.ErrAbortHandler). -func logPanic(r interface{}) { - if r == http.ErrAbortHandler { - // honor the http.ErrAbortHandler sentinel panic value: - // ErrAbortHandler is a sentinel panic value to abort a handler. - // While any panic from ServeHTTP aborts the response to the client, - // panicking with ErrAbortHandler also suppresses logging of a stack trace to the server's error log. - return - } - - // Same as stdlib http server code. Manually allocate stack trace buffer size - // to prevent excessively large logs - const size = 64 << 10 - stacktrace := make([]byte, size) - stacktrace = stacktrace[:runtime.Stack(stacktrace, false)] - if _, ok := r.(string); ok { - klog.Errorf("Observed a panic: %s\n%s", r, stacktrace) - } else { - klog.Errorf("Observed a panic: %#v (%v)\n%s", r, r, stacktrace) - } + c.callbacks.Follower(ctx, c) } func (c *Cluster) Stepdown() { diff --git a/pkg/raft/tls/stream.go b/pkg/raft/tls/stream.go index 77c74e6..bde01f1 100644 --- a/pkg/raft/tls/stream.go +++ b/pkg/raft/tls/stream.go @@ -3,7 +3,6 @@ package tls import ( "crypto/tls" "crypto/x509" - "io/ioutil" "net" "time" @@ -20,19 +19,6 @@ var ( } ) -func loadCertPool(file string) (p *x509.CertPool, err error) { - if file == "" { - return - } - buf, err := ioutil.ReadFile(file) - if err != nil { - return - } - p = x509.NewCertPool() - p.AppendCertsFromPEM(buf) - return -} - type stream struct { net.Listener insecure bool @@ -44,7 +30,7 @@ func NewStream(cfg *config.Raft) (s *stream, err error) { if err != nil { return } - ca, err := loadCertPool(cfg.TLS.CA) + ca, err := config.LoadCertPool(cfg.TLS.CA) if err != nil { return } diff --git a/pkg/util/crash.go b/pkg/util/crash.go new file mode 100644 index 0000000..91d5a97 --- /dev/null +++ b/pkg/util/crash.go @@ -0,0 +1,41 @@ +package util + +import ( + "net/http" + "runtime" + + "k8s.io/klog/v2" +) + +var reallyCrash = true + +func HandleCrash() { + if r := recover(); r != nil { + logPanic(r) + if reallyCrash { + panic(r) + } + } +} + +// logPanic logs the caller tree when a panic occurs (except in the special case of http.ErrAbortHandler). +func logPanic(r interface{}) { + if r == http.ErrAbortHandler { + // honor the http.ErrAbortHandler sentinel panic value: + // ErrAbortHandler is a sentinel panic value to abort a handler. + // While any panic from ServeHTTP aborts the response to the client, + // panicking with ErrAbortHandler also suppresses logging of a stack trace to the server's error log. + return + } + + // Same as stdlib http server code. Manually allocate stack trace buffer size + // to prevent excessively large logs + const size = 64 << 10 + stacktrace := make([]byte, size) + stacktrace = stacktrace[:runtime.Stack(stacktrace, false)] + if _, ok := r.(string); ok { + klog.Errorf("Observed a panic: %s\n%s", r, stacktrace) + } else { + klog.Errorf("Observed a panic: %#v (%v)\n%s", r, r, stacktrace) + } +} diff --git a/run.sh b/run.sh deleted file mode 100755 index 6343783..0000000 --- a/run.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/sh - -cd data -rm -rf node* -../vipman -config config1.yaml & -../vipman -config config2.yaml & -../vipman -config config3.yaml diff --git a/run_etcd.sh b/run_etcd.sh new file mode 100755 index 0000000..ab75ec7 --- /dev/null +++ b/run_etcd.sh @@ -0,0 +1,6 @@ +#!/bin/sh + +cd data +../vipman -config etcd.yaml & +../vipman -config etcd.yaml & +../vipman -config etcd.yaml diff --git a/run_raft.sh b/run_raft.sh new file mode 100755 index 0000000..d218498 --- /dev/null +++ b/run_raft.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +cd data +rm -rf node* +../vipman -config raft1.yaml & +../vipman -config raft2.yaml & +../vipman -config raft3.yaml diff --git a/vipman.yaml b/vipman.yaml index 34ea273..096f184 100644 --- a/vipman.yaml +++ b/vipman.yaml @@ -9,7 +9,6 @@ cluster: dir: "" id: "" address: "" - level: "" peers: - id: "" address: "" @@ -19,7 +18,8 @@ cluster: cert: "" key: "" ca: "" - insecure: true + insecure: false + logLevel: "" etcd: endpoints: - "" @@ -27,3 +27,9 @@ cluster: - "" prefix: "" clusterName: "" + username: "" + password: "" + tls: + ca: "" + insecure: false + logLevel: ""