diff --git a/TODO b/TODO index 19a472d..1a8d110 100644 --- a/TODO +++ b/TODO @@ -1,7 +1 @@ -VIP for haproxy-lb -* take from pool? -* extra config? - -cleanup pkg/vip - -test api in followers +add logger to vip package? diff --git a/cmd/haproxy-lb/main.go b/cmd/haproxy-lb/main.go index d927c17..0392bda 100644 --- a/cmd/haproxy-lb/main.go +++ b/cmd/haproxy-lb/main.go @@ -59,7 +59,7 @@ func main() { log.Info("creating cluster", "mode", "etcd") c = etcd.NewCluster(logfmtr.New().WithName("etcd")) } - log.Info("starting api server", "address", cfg.Server.Listen, "tls", cfg.Server.TLS != nil) + log.Info("starting api server", "address", cfg.APIServer.Listen, "tls", cfg.APIServer.TLS != nil) srv, err := api.NewServer(cfg, logfmtr.New().WithName("api")) if err != nil { fatal(err, "init failed") diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000..a0b6a1b --- /dev/null +++ b/config.yaml @@ -0,0 +1,58 @@ +#leaderHook: "/bin/leader_hook.sh" +#followerHook: "/bin/follower_hook.sh" +#haproxyConfig: "/etc/haproxy/haproxy.cfg" +#haproxyService: haproxy +#failover: +# ip: "192.168.1.1/32" +# interface: eth0 +vip: + prefix: "192.168.101.0/24" + store: + local: {} +# netbox: +# endpoint: "https://your.net.box" +# token: asdf +# insecure: false +# interface: eth1 +# label: LB +cluster: +# raft: +# dir: raft +# id: node1 +# listen: 192.168.1.2:7001 +# peers: +# - id: node2 +# address: 192.168.1.3:7002 +# - id: node3 +# address: 192.168.1.4:7003 +# tls: +# cert: cert.pem +# key: key.pem +# ca: cert.pem +# insecure: false +# logLevel: off + etcd: + endpoints: + - "http://127.0.0.1:2379" + clusterName: "lb" +# prefix: "/haproxy-lb" +# username: user +# password: password +# tls: +# cert: cert.pem +# key: key.pem +# ca: cert.pem +# insecure: false +# logLevel: error +apiServer: + listen: "127.0.0.1:8001" +# tls: +# cert: cert.pem +# key: key.pem +# ca: cert.pem +# insecure: false + basicAuth: + - user: admin + # 123456 + hash: $2b$10$zNfGTAQ94vifj2wtGsv1W.yZRe4/rDBzQixpB6FbrTed4BnHuNqBS + prefix: / diff --git a/go.mod b/go.mod index cbaaf22..0266da2 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module git.giftfish.de/ston1th/haproxy-lb go 1.16 require ( - git.giftfish.de/ston1th/netalloc v0.0.0-20210926174801-c39acaee43a5 + git.giftfish.de/ston1th/netalloc v0.0.0-20211010134723-1daa3f8e0e64 git.giftfish.de/ston1th/raftbbolt v0.0.0-20201114165203-b85c94bfc9b0 github.com/armon/go-metrics v0.3.6 // indirect github.com/fatih/color v1.10.0 // indirect @@ -17,6 +17,7 @@ require ( github.com/iand/logfmtr v0.2.1 github.com/mdlayher/arp v0.0.0-20191213142603-f72070a231fc github.com/mdlayher/ethernet v0.0.0-20190606142754-0394541c37b7 + github.com/netbox-community/go-netbox v0.0.0-20210916100335-c6261e6b3c39 // indirect github.com/vishvananda/netlink v1.1.0 github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f // indirect go.etcd.io/etcd/client/v3 v3.5.0 diff --git a/go.sum b/go.sum index b37495d..33a1c25 100644 --- a/go.sum +++ b/go.sum @@ -2,6 +2,8 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= git.giftfish.de/ston1th/netalloc v0.0.0-20210926174801-c39acaee43a5 h1:DQzBQS2lYd4FrHHvVnqIEEfKUEXfbj/NwAAq+vRC4dg= git.giftfish.de/ston1th/netalloc v0.0.0-20210926174801-c39acaee43a5/go.mod h1:ECRm6vIqNEDXBgJmKyXaBAWucdCzCVH5FG9Hfk6QX0E= +git.giftfish.de/ston1th/netalloc v0.0.0-20211010134723-1daa3f8e0e64 h1:YOJZsnU7JwiVureMZ04LeZYW1i/qId75No8m0F0w29Q= +git.giftfish.de/ston1th/netalloc v0.0.0-20211010134723-1daa3f8e0e64/go.mod h1:/+hsFsLucNK49c3N4YPFnseyUIJOiGvNKCjObeBQvqY= git.giftfish.de/ston1th/raftbbolt v0.0.0-20201114165203-b85c94bfc9b0 h1:DU0PVdA2EHz2LFGEdkwgJopL9HQh5nFC9GJ7oqzZ300= git.giftfish.de/ston1th/raftbbolt v0.0.0-20201114165203-b85c94bfc9b0/go.mod h1:9dF/aHYkXts6/6oVyJ+vzXCn7NURDaoREaanX0aspx0= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= diff --git a/pkg/api/server.go b/pkg/api/server.go index 01fd1bc..6ab9448 100644 --- a/pkg/api/server.go +++ b/pkg/api/server.go @@ -13,6 +13,7 @@ import ( "git.giftfish.de/ston1th/haproxy-lb/pkg/db" "git.giftfish.de/ston1th/netalloc" + "git.giftfish.de/ston1th/netalloc/netbox" "github.com/go-logr/logr" "github.com/gorilla/mux" @@ -25,7 +26,6 @@ type Server struct { log logr.Logger Data *types.ContextData - cidr string init chan struct{} stop chan struct{} @@ -33,6 +33,9 @@ type Server struct { tlsConfig *tls.Config listen net.Listener + + prefix string + store config.Store } type notFoundHandler struct { @@ -48,17 +51,19 @@ func NewServer(c *config.Config, log logr.Logger) (*Server, error) { s := &Server{ mux: mux.NewRouter(), log: log, - cidr: c.VIP.Prefix, Data: &types.ContextData{}, init: make(chan struct{}), stop: make(chan struct{}), stopKeyReset: make(chan struct{}), - } - s.Data.Auth = types.NewAuth(c.Server.BasicAuth) - if c.Server.TLS != nil { - cert, err := tls.LoadX509KeyPair(c.Server.TLS.Cert, c.Server.TLS.Key) + prefix: c.VIP.Prefix, + store: c.VIP.Store, + } + s.Data.Auth = types.NewAuth(c.APIServer.BasicAuth) + + if c.APIServer.TLS != nil { + cert, err := tls.LoadX509KeyPair(c.APIServer.TLS.Cert, c.APIServer.TLS.Key) if err != nil { return nil, err } @@ -83,13 +88,13 @@ func NewServer(c *config.Config, log logr.Logger) (*Server, error) { PreferServerCipherSuites: true, Certificates: []tls.Certificate{cert}, } - l, err := tls.Listen("tcp", c.Server.Listen, s.tlsConfig) + l, err := tls.Listen("tcp", c.APIServer.Listen, s.tlsConfig) if err != nil { return nil, err } s.listen = l } else { - l, err := net.Listen("tcp", c.Server.Listen) + l, err := net.Listen("tcp", c.APIServer.Listen) if err != nil { return nil, err } @@ -107,15 +112,32 @@ func NewServer(c *config.Config, log logr.Logger) (*Server, error) { func (s *Server) UpdateDB(db *db.DB, leader bool) error { s.Data.DB = db s.Data.Leader = leader - if s.Data.Net == nil { - na, err := netalloc.NewGenericAlloc(db, s.cidr) - if err != nil { - return err + if s.store.Local != nil { + if s.Data.Net == nil { + na, err := netalloc.NewGenericAlloc(db, s.prefix) + if err != nil { + return err + } + s.Data.Net = na + close(s.init) + } else { + s.Data.Net.SetStore(db) } - s.Data.Net = na - close(s.init) } else { - s.Data.Net.SetStore(db) + if s.Data.Net == nil { + nb := s.store.Netbox + na, err := netbox.NewAllocator( + netbox.WithEndpoint(nb.Endpoint), + netbox.WithToken(nb.Token), + netbox.WithPrefix(s.prefix), + netbox.WithInsecureClient(nb.Insecure), + ) + if err != nil { + return err + } + s.Data.Net = na + close(s.init) + } } return nil } diff --git a/pkg/config/config.go b/pkg/config/config.go index c0b5567..745b88c 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -3,6 +3,7 @@ package config import ( "errors" "fmt" + "net" "os" "path/filepath" "strings" @@ -13,7 +14,7 @@ import ( const ( RaftDir = "raft" - RaftPort = 7001 + RaftListen = ":7001" RaftLogLevel = "off" EtcdLogLevel = "error" @@ -48,9 +49,19 @@ func Validate(c *Config) error { if c.HAProxyService == "" { c.HAProxyService = HAProxyService } - if c.VIP.Prefix == "" { - return errors.New("missing server.prefix config") + + failover := c.Failover + if failover != nil { + if failover.IP == "" { + return errors.New("missing failover.ip config") + + } + _, _, err := net.ParseCIDR(failover.IP) + if err != nil { + return fmt.Errorf("failed to parse failover.ip: %w", err) + } } + if c.LeaderHook != "" { if !filepath.IsAbs(c.LeaderHook) { return errors.New("leaderHook path must be absolute") @@ -61,6 +72,28 @@ func Validate(c *Config) error { return errors.New("followerHook path must be absolute") } } + + vip := c.VIP + if vip.Prefix == "" { + return errors.New("missing vip.prefix config") + } + local := vip.Store.Local + netbox := vip.Store.Netbox + if local == nil && netbox == nil { + return errors.New("missing vip.store config: local or netbox") + } + if local != nil && netbox != nil { + return errors.New("only one vip.store config allowed: local or netbox") + } + if netbox != nil { + if netbox.Endpoint == "" { + return errors.New("missing vip.store.netbox.endpoint config") + } + if netbox.Token == "" { + return errors.New("missing vip.store.netbox.token config") + } + } + raft := c.Cluster.Raft if raft != nil { if raft.LogLevel == "" { @@ -72,8 +105,8 @@ func Validate(c *Config) error { if raft.ID == "" { return errors.New("missing raft.id config") } - if raft.Address == "" { - return errors.New("missing raft.address config") + if raft.Listen == "" { + return errors.New("missing raft.listen config") } if len(raft.Peers) < 2 { return errors.New("minimum number of remote peers: 2") @@ -119,10 +152,10 @@ func Validate(c *Config) error { if raft != nil && etcd != nil { return errors.New("only one cluster config allowed: raft or etcd") } - if len(c.Server.BasicAuth) == 0 { + if len(c.APIServer.BasicAuth) == 0 { return errors.New("missing basic auth configuration for api users") } - for _, u := range c.Server.BasicAuth { + for _, u := range c.APIServer.BasicAuth { if u.User == "" { return errors.New("basic auth username can not be empty") } @@ -135,31 +168,37 @@ func Validate(c *Config) error { } type Config struct { - LeaderHook string `yaml:"leaderHook,omitempty"` - FollowerHook string `yaml:"followerHook,omitempty"` - HAProxyConfig string `yaml:"haproxyConfig,omitempty"` - HAProxyService string `yaml:"haproxyService,omitempty"` - VIP VIP `yaml:"vip"` - Cluster Cluster `yaml:"cluster"` - Server Server `yaml:"server"` + LeaderHook string `yaml:"leaderHook,omitempty"` + FollowerHook string `yaml:"followerHook,omitempty"` + HAProxyConfig string `yaml:"haproxyConfig,omitempty"` + HAProxyService string `yaml:"haproxyService,omitempty"` + Failover *Failover `yaml:"failover"` + VIP VIP `yaml:"vip"` + Cluster Cluster `yaml:"cluster"` + APIServer APIServer `yaml:"apiServer"` } -type Server struct { - Listen string `yaml:"listen"` - TLS *TLS `yaml:"tls"` - BasicAuth []BasicAuth `yaml:"basicAuth"` +type Failover struct { + IP string `yaml:"ip"` + Interface string `yaml:"interface"` } type VIP struct { Prefix string `yaml:"prefix"` + Store Store `yaml:"store"` Interface string `yaml:"interface,omitempty"` Label string `yaml:"label,omitempty"` } -type BasicAuth struct { - User string `yaml:"user"` - Hash string `yaml:"hash"` - Prefix string `yaml:"prefix"` +type Store struct { + Local *struct{} `yaml:"local"` + Netbox *Netbox `yaml:"netbox"` +} + +type Netbox struct { + Endpoint string `yaml:"endpoint"` + Token string `yaml:"token"` + Insecure bool `yaml:"insecure,omitempty"` } type Cluster struct { @@ -170,7 +209,7 @@ type Cluster struct { type Raft struct { Dir string `yaml:"dir,omitempty"` ID string `yaml:"id"` - Address string `yaml:"address"` + Listen string `yaml:"listen"` Peers []RaftPeer `yaml:"peers"` TLS *TLS `yaml:"tls,omitempty"` LogLevel string `yaml:"logLevel,omitempty"` @@ -191,10 +230,22 @@ type RaftPeer struct { type Etcd struct { Endpoints []string `yaml:"endpoints"` - Prefix string `yaml:"prefix"` ClusterName string `yaml:"clusterName"` + Prefix string `yaml:"prefix,omitempty"` Username string `yaml:"username,omitempty"` Password string `yaml:"password,omitempty"` TLS *TLS `yaml:"tls,omitempty"` LogLevel string `yaml:"logLevel,omitempty"` } + +type APIServer struct { + Listen string `yaml:"listen"` + TLS *TLS `yaml:"tls"` + BasicAuth []BasicAuth `yaml:"basicAuth"` +} + +type BasicAuth struct { + User string `yaml:"user"` + Hash string `yaml:"hash"` + Prefix string `yaml:"prefix"` +} diff --git a/pkg/controller/loadbalancer.go b/pkg/controller/loadbalancer.go index 4db46f6..0d7d745 100644 --- a/pkg/controller/loadbalancer.go +++ b/pkg/controller/loadbalancer.go @@ -15,6 +15,14 @@ import ( ) func NewLBController(cfg *config.Config, srv *api.Server, log logr.Logger) (callbacks cluster.Callbacks, err error) { + var fo *vip.Network + if cfg.Failover != nil { + fo, err = vip.NewNetwork(cfg.Failover.Interface) + if err != nil { + return + } + } + n, err := vip.NewNetworkWithLabel(cfg.VIP.Interface, cfg.VIP.Label) if err != nil { return @@ -57,6 +65,9 @@ func NewLBController(cfg *config.Config, srv *api.Server, log logr.Logger) (call return } log.Info("leading", "id", cc.ID()) + if cfg.Failover != nil { + addIPs(log, fo, []string{cfg.Failover.IP}) + } ips, err := db.GetDeletedIPs() if err != nil && err != cluster.ErrPrefixNotFound { log.Error(err, "error reading deleted ips list") @@ -150,6 +161,9 @@ func NewLBController(cfg *config.Config, srv *api.Server, log logr.Logger) (call } }, Cleanup: func(ctx context.Context, cc cluster.CallbackContext) { + if cfg.Failover != nil { + deleteIPs(log, fo, fo.GetIPs()) + } deleteIPs(log, n, n.GetIPs()) }, } diff --git a/pkg/raft/raft.go b/pkg/raft/raft.go index f20cf9b..e92d0b3 100644 --- a/pkg/raft/raft.go +++ b/pkg/raft/raft.go @@ -48,11 +48,11 @@ func (c *Cluster) Start(raftcfg *config.Config) error { transport = raft.NewNetworkTransportWithLogger(s, pool, timeout, hcl) } else { log.V(1).Info("setup", "transport", "tcp") - address, err := net.ResolveTCPAddr("tcp", cfg.Address) + address, err := net.ResolveTCPAddr("tcp", cfg.Listen) if err != nil { return err } - transport, err = raft.NewTCPTransportWithLogger(cfg.Address, address, pool, timeout, hcl) + transport, err = raft.NewTCPTransportWithLogger(cfg.Listen, address, pool, timeout, hcl) if err != nil { return err } @@ -85,10 +85,10 @@ func (c *Cluster) Start(raftcfg *config.Config) error { // Add Local Peer configuration.Servers = append(configuration.Servers, raft.Server{ ID: raft.ServerID(cfg.ID), - Address: raft.ServerAddress(cfg.Address), + Address: raft.ServerAddress(cfg.Listen), }) for _, p := range cfg.Peers { - if cfg.Address != p.Address { + if cfg.Listen != p.Address { configuration.Servers = append(configuration.Servers, raft.Server{ ID: raft.ServerID(p.ID), Address: raft.ServerAddress(p.Address)}) @@ -113,7 +113,7 @@ func (c *Cluster) Start(raftcfg *config.Config) error { leading := false following := false for { - if !leading && cfg.Address == string(c.r.Leader()) { + if !leading && cfg.Listen == string(c.r.Leader()) { leading = true go c.leader(ctx) } else if !following && !leading { diff --git a/pkg/raft/tls/stream.go b/pkg/raft/tls/stream.go index 834df88..0385a18 100644 --- a/pkg/raft/tls/stream.go +++ b/pkg/raft/tls/stream.go @@ -34,7 +34,7 @@ func NewStream(cfg *config.Raft) (s *stream, err error) { if err != nil { return } - l, err := tls.Listen("tcp", cfg.Address, &tls.Config{ + l, err := tls.Listen("tcp", cfg.Listen, &tls.Config{ MinVersion: tls.VersionTLS13, CipherSuites: ciphers, CurvePreferences: curves,