ston1th/haproxy-lb
0
0
Fork 0
Code Issues Releases Activity

more work done

Browse source
This commit is contained in:
ston1th 2021-03-06 16:51:36 +01:00
commit 6fda229a8e
23 changed files with 384 additions and 177 deletions
Download patch file Download diff file Expand all files Collapse all files

49
pkg/alloc/alloc.go
View file

@ -4,18 +4,21 @@ import (
"context"
"fmt"
"net"
"strings"
"sync"
"git.giftfish.de/ston1th/haproxy-lb/pkg/cluster"
"git.giftfish.de/ston1th/haproxy-lb/pkg/db"
"github.com/mikioh/ipaddr"
)
type Alloc struct {
kv cluster.KV
sync.Mutex
db *db.DB
pool []*net.IPNet
gateway *net.IPNet
}
func NewAlloc(kv cluster.KV, cidrs []string, gateway string) (*Alloc, error) {
func NewAlloc(db *db.DB, cidrs []string, gateway string) (*Alloc, error) {
pool, err := parseRange(cidrs)
if err != nil {
return nil, err
@ -24,26 +27,27 @@ func NewAlloc(kv cluster.KV, cidrs []string, gateway string) (*Alloc, error) {
if err != nil {
return nil, err
}
return &Alloc{kv, pool, gw}, nil
return &Alloc{db: db, pool: pool, gateway: gw}, nil
}
func (a *Alloc) UpdateDB(db *db.DB) {
a.Lock()
a.db = db
a.Unlock()
}
func (a *Alloc) AllocIP(ctx context.Context, name string) (addr string, err error) {
a.Lock()
defer a.Unlock()
for _, cidr := range a.pool {
c := ipaddr.NewCursor([]ipaddr.Prefix{*ipaddr.NewPrefix(a.gateway)})
c := ipaddr.NewCursor([]ipaddr.Prefix{*ipaddr.NewPrefix(cidr)})
for pos := c.First(); pos != nil; pos = c.Next() {
ip := pos.IP.String()
v, _ := a.kv.Get(ip)
vs := string(v)
if v == "" {
err = a.kv.Set(ip, []byte(name))
if !a.db.IPExists(ip) {
err = a.db.SetIP(ip, name)
if err != nil {
return
}
err = a.kv.Set(name, []byte(ip))
if err != nil {
a.kv.Delete(ip)
return
}
var gw *net.IPNet
*gw = *a.gateway
gw.IP = pos.IP
@ -52,28 +56,21 @@ func (a *Alloc) AllocIP(ctx context.Context, name string) (addr string, err erro
}
}
}
err = fmt.Errorf("no available IPs in pool")
err = fmt.Errorf("no more IPs available in pool")
return
}
func (a *Alloc) FreeIP(ctx context.Context, name string) (err error) {
a.Lock()
defer a.Unlock()
if name == "" {
return
}
v, err := a.kv.Get(name)
ip, err := a.db.GetName(name)
if err != nil {
return
}
vs := string(v)
err = a.kv.Delete(vs)
if err != nil {
return
}
err = a.kv.Delete(name)
if err != nil {
a.kv.Set(vs, []byte(name))
}
return
return a.db.DeleteIP(ip, name)
}
func parseRange(cidrs []string) (nets []*net.IPNet, err error) {

52
pkg/api/server.go
View file

@ -3,15 +3,15 @@ package api
import (
"context"
stdtls "crypto/tls"
"io/ioutil"
"log"
"net"
"net/http"
"path/filepath"
"time"
"git.giftfish.de/ston1th/haproxy-lb/pkg/alloc"
"git.giftfish.de/ston1th/haproxy-lb/pkg/api/types"
serverv1 "git.giftfish.de/ston1th/haproxy-lb/pkg/api/v1/server"
"git.giftfish.de/ston1th/haproxy-lb/pkg/config"
"git.giftfish.de/ston1th/haproxy-lb/pkg/db"
"github.com/go-logr/logr"
"github.com/gorilla/mux"
@ -23,14 +23,19 @@ type Server struct {
mux *mux.Router
log logr.Logger
Alloc *alloc.Alloc
DB *db.DB
cidrs []string
gw string
Auth []config.BasicAuth
init chan struct{}
stop chan struct{}
stopKeyReset chan struct{}
cert string
key string
laddr string
debug bool
}
type notFoundHandler struct {
@ -42,19 +47,21 @@ func (nf *notFoundHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
}
// NewHTTPServer returns a new HTTPServer
func NewServer(log logr.Logger) *Server {
func NewServer(c *config.Config, log logr.Logger) *Server {
s := &Server{
mux: mux.NewRouter(),
log: log.WithName("api"),
mux: mux.NewRouter(),
log: log.WithName("api"),
cidrs: c.VIP.VirtualIPs,
gw: c.VIP.Gateway,
Auth: c.Server.BasicAuth,
init: make(chan struct{}),
stop: make(chan struct{}),
stopKeyReset: make(chan struct{}),
cert: filepath.Join(core.C.DataDir, core.C.HTTP.Cert),
key: filepath.Join(core.C.DataDir, core.C.HTTP.Key),
laddr: net.JoinHostPort(core.C.HTTP.Address, core.C.HTTP.Port),
debug: core.C.Debug,
cert: c.Server.TLS.Cert,
key: c.Server.TLS.Key,
laddr: c.Server.Listen,
}
s.mux.NotFoundHandler = &notFoundHandler{s}
for _, v := range serverv1.Routes {
@ -64,11 +71,28 @@ func NewServer(log logr.Logger) *Server {
return s
}
func (s *Server) UpdateDB(db *db.DB) error {
s.db = db
if s.Alloc == nil {
alloc, err := alloc.NewAlloc(db, s.cidrs, s.gw)
if err != nil {
return err
}
s.alloc = alloc
close(s.init)
} else {
s.Alloc.UpdateDB(db)
}
return nil
}
func (s *Server) start() error {
l, err := tls.Listen("tcp", h.laddr, cfg)
<-s.init
l, err := net.Listen("tcp", h.laddr)
if err != nil {
return err
}
// TODO tls config
h.srv = &http.Server{
Handler: h.mux,
TLSConfig: cfg,

7
pkg/api/types/context.go
View file

@ -2,7 +2,6 @@ package types
import (
"encoding/json"
"fmt"
"net/http"
"github.com/go-logr/logr"
@ -16,7 +15,7 @@ func NewContext(w http.ResponseWriter, r *http.Request, log logr.Logger) *Contex
Status: http.StatusOK,
Request: r,
Response: w,
log: log,
logger: log,
}
}
@ -27,7 +26,7 @@ type Context struct {
Request *http.Request
Response http.ResponseWriter
log logr.Logger
logger logr.Logger
}
// Method returns the request method
@ -58,7 +57,7 @@ func (c *Context) Var(name string) (ret string) {
}
func (c *Context) log() {
c.log.Info("access",
c.logger.Info("access",
"addr", c.Request.RemoteAddr,
"method", c.Request.Method,
"url", c.Request.URL,

45
pkg/api/types/error.go Normal file
View file

@ -0,0 +1,45 @@
package types
import (
"net/http"
"strconv"
)
// Error interface is a custom api error
type Error interface {
Error() string
Status() int
}
// Err implements the Error interface
type Err struct {
err string
status int
}
// Error returns the error string
func (a Err) Error() string {
return strconv.Itoa(a.status) + ": " + a.err
}
// Status returns the HTTP status code
func (a Err) Status() int {
return a.status
}
func newError(err string, status int) Error {
return Err{
err: err,
status: status,
}
}
var (
badreq = newError("bad request", http.StatusBadRequest)
invalid = newError("invalid data", http.StatusBadRequest)
forbid = newError("forbidden", http.StatusForbidden)
nf = newError("resource not found", http.StatusNotFound)
mna = newError("method not allowed", http.StatusMethodNotAllowed)
exists = newError("resource already exists", http.StatusConflict)
ise = newError("internal server error", http.StatusInternalServerError)
)

2
pkg/api/types/types.go
View file

@ -2,6 +2,6 @@ package types
type Route struct {
Path string
Handler http.HandlerFunc
Handler CtxHandler
Methods []string
}

2
pkg/api/v1/schema/schema.go
View file

@ -5,6 +5,8 @@ healthCheckNodePort
*/
const Version = "v1"
type LoadBalancer struct {
Name string `json:"name"`
IP string `json:"ip,omitempty"`

12
pkg/api/v1/server/auth.go
View file

@ -1,10 +1,12 @@
package server
import (
"git.gitfish.de/ston1th/haproxy-lb/pkg/config"
"git.giftfish.de/ston1th/haproxy-lb/pkg/config"
"golang.org/x/crypto/bcrypt"
weakrand "math/rand"
"net/http"
//"net/http"
"encoding/hex"
"strings"
"sync"
"time"
)
@ -90,7 +92,7 @@ func (a *auth) Auth(user, pass, path string) bool {
a.bcryptMtx.Unlock()
authOk = err == nil
u.cache.set(cacheKey, authOk)
a.cache.set(cacheKey, authOk)
}
return authOk && valid && strings.HasPrefix(path, creds.Prefix)
@ -103,10 +105,10 @@ type creds struct {
func newAuth(conf config.Config) (a *auth) {
a = &auth{
users: make(map[string]creds),
users: make(map[string]*creds),
cache: newCache(),
}
for _, u := range conf.BasicAuth {
for _, u := range conf.Server.BasicAuth {
a.users[u.User] = &creds{u.Hash, u.Prefix}
}
return

10
pkg/api/v1/server/handler.go
View file

@ -2,7 +2,7 @@ package server
import (
//"golang.org/x/crypto/bcrypt"
"net/http"
//"net/http"
"git.giftfish.de/ston1th/haproxy-lb/pkg/api/types"
)
@ -21,3 +21,11 @@ func healthzHandler(ctx *types.Context) {
// TODO maybe report etcd/raft stats
ctx.OK()
}
func lbListHandler(ctx *types.Context) {
ctx.OK()
}
func lbHandler(ctx *types.Context) {
ctx.OK()
}

3
pkg/cluster/cluster.go
View file

@ -3,7 +3,7 @@ package cluster
import (
"context"
"git.giftfish.de/ston1th/vipman/pkg/config"
"git.giftfish.de/ston1th/haproxy-lb/pkg/config"
"github.com/go-logr/logr"
)
@ -25,6 +25,7 @@ type CallbackContext interface {
logr.Logger
KV
ID() string
Fatal(error, string)
}
type Callbacks struct {

44
pkg/config/config.go
View file

@ -17,7 +17,9 @@ const (
RaftLogLevel = "off"
EtcdLogLevel = "error"
EtcdPrefix = "/vipman"
EtcdPrefix = "/haproxy-lb"
HAProxyConfigFile = "/etc/haproxy/haproxy.cf"
)
func ParseFile(file string) (cfg *Config, err error) {
@ -39,8 +41,14 @@ func ParseFile(file string) (cfg *Config, err error) {
}
func Validate(c *Config) error {
if len(c.VirtualIPs) == 0 {
return errors.New("missing virtualIPs config")
if c.HAProxyConfig == "" {
c.HAProxyConfig = HAProxyConfigFile
}
if len(c.VIP.VirtualIPs) == 0 {
return errors.New("missing server.virtualIPs config")
}
if c.VIP.Gateway == "" {
return errors.New("missing server.gateway config")
}
if c.LeaderHook != "" {
if !filepath.IsAbs(c.LeaderHook) {
@ -110,10 +118,10 @@ func Validate(c *Config) error {
if raft != nil && etcd != nil {
return errors.New("only one cluster config allowed: raft or etcd")
}
if len(c.BasicAuth) == 0 {
if len(c.Server.BasicAuth) == 0 {
return errors.New("missing basic auth configuration for api users")
}
for _, u := range c.BasicAuth {
for _, u := range c.Server.BasicAuth {
if u.User == "" {
return errors.New("basic auth username can not be empty")
}
@ -126,13 +134,25 @@ func Validate(c *Config) error {
}
type Config struct {
VirtualIPs []string `yaml:"virtualIPs"`
Interface string `yaml:"interface,omitempty"`
Label string `yaml:"label,omitempty"`
LeaderHook string `yaml:"leaderHook,omitempty"`
FollowerHook string `yaml:"followerHook,omitempty"`
Cluster Cluster `yaml:"cluster"`
BasicAuth []BasicAuth `yaml:"basicAuth"`
LeaderHook string `yaml:"leaderHook,omitempty"`
FollowerHook string `yaml:"followerHook,omitempty"`
HAProxyConfig string `yaml:"haproxyConfig,omitempty"`
VIP VIP `yaml:"vip"`
Cluster Cluster `yaml:"cluster"`
Server Server `yaml:"server"`
}
type Server struct {
Listen string `yaml:"listen"`
TLS TLS `yaml:"tls"`
BasicAuth []BasicAuth `yaml:"basicAuth"`
}
type VIP struct {
VirtualIPs []string `yaml:"virtualIPs"`
Gateway string `yaml:"gateway"`
Interface string `yaml:"interface,omitempty"`
Label string `yaml:"label,omitempty"`
}
type BasicAuth struct {

91
pkg/controller/vip.go
View file

@ -5,62 +5,33 @@ import (
"os/exec"
"time"
"git.giftfish.de/ston1th/haproxy-kb/pkg/cluster"
"git.giftfish.de/ston1th/haproxy-kb/pkg/config"
"git.giftfish.de/ston1th/haproxy-kb/pkg/haproxy"
"git.giftfish.de/ston1th/haproxy-kb/pkg/vip"
"git.giftfish.de/ston1th/haproxy-lb/pkg/api"
"git.giftfish.de/ston1th/haproxy-lb/pkg/cluster"
"git.giftfish.de/ston1th/haproxy-lb/pkg/config"
"git.giftfish.de/ston1th/haproxy-lb/pkg/db"
"git.giftfish.de/ston1th/haproxy-lb/pkg/haproxy"
"git.giftfish.de/ston1th/haproxy-lb/pkg/vip"
"github.com/go-logr/logr"
)
type UpdateAction int
const (
Add UpdateAction = iota
Delete
)
type ConfigUpdate struct {
Action UpdateAction
Config haproxy.LoadBalancer
}
func readConfigUpdates(c chan ConfigUpdate) (cu []ConfigUpdate) {
for len(c) > 0 {
select {
case config := <-c:
cu = append(cu, config)
}
}
return
}
func applyConfigUpdates(cc cluster.CallbackContext, cu []ConfigUpdate) error {
for _, c := range cu {
switch c.Action {
case Add:
addIPs(cc, n, []string{c.Config.IP})
cc.Set(c.Config.IP, nil)
case Delete:
cc.Delete(c.Config.IP)
deleteIPs(cc, n, []string{c.Config.IP})
}
}
}
func NewLBController(cfg *config.Config) (callbacks cluster.Callbacks, err error) {
n, err := vip.NewNetworkWithLabel(cfg.Interface, cfg.Label)
func NewLBController(cfg *config.Config, srv *api.Server, log logr.Logger) (callbacks cluster.Callbacks, err error) {
n, err := vip.NewNetworkWithLabel(cfg.VIP.Interface, cfg.VIP.Label)
if err != nil {
return
}
//TODO
ha, err := haproxy.NewHAProxyManager("")
ha, err := haproxy.NewHAProxyManager(cfg.HAProxyConfig)
if err != nil {
return
}
configUpdate := make(chan ConfigUpdate, 100)
callbacks = cluster.Callbacks{
Leader: func(ctx context.Context, cc cluster.CallbackContext) {
db := db.New(cc)
err := srv.UpdateDB(db)
if err != nil {
cc.Fatal(err, "error initialising api server as leader")
return
}
t := time.NewTicker(time.Second * 10)
if cfg.LeaderHook != "" {
go func() {
@ -72,13 +43,20 @@ func NewLBController(cfg *config.Config) (callbacks cluster.Callbacks, err error
}
for {
cc.Info("leading", "id", cc.ID())
ips, err := db.GetIPs()
if err != nil {
cc.Error("error updating haproxy config", err)
<-t.C
continue
}
addIPs(cc, n, ips)
cfg, err := cc.Get("config")
if err != nil {
cc.Error("error getting config key", err)
<-t.C
continue
}
addIPs(cc, n, cfg.VirtualIPs)
var lbs haproxy.Config
err = ha.UpdateConfig(ctx, lbs)
if err != nil {
@ -89,7 +67,7 @@ func NewLBController(cfg *config.Config) (callbacks cluster.Callbacks, err error
select {
case <-ctx.Done():
t.Stop()
deleteIPs(cc, n, cfg.VirtualIPs)
deleteIPs(cc, n, ips)
cc.Info("leading canceled", "id", cc.ID())
return
case <-t.C:
@ -97,6 +75,12 @@ func NewLBController(cfg *config.Config) (callbacks cluster.Callbacks, err error
}
},
Follower: func(ctx context.Context, cc cluster.CallbackContext) {
db := db.New(cc)
err := srv.UpdateDB(db)
if err != nil {
cc.Fatal(err, "error initialising api server as leader")
return
}
t := time.NewTicker(time.Second * 10)
if cfg.FollowerHook != "" {
go func() {
@ -109,7 +93,13 @@ func NewLBController(cfg *config.Config) (callbacks cluster.Callbacks, err error
for {
cc.Info("following", "id", cc.ID())
//TODO
//deleteIPs(cc, n, cfg.VirtualIPs)
//ips, err := db.GetIPs()
//if err != nil {
// cc.Error("error getting config key", err)
// <-t.C
// continue
//}
//deleteIPs(cc, n, ips)
select {
case <-ctx.Done():
t.Stop()
@ -120,7 +110,12 @@ func NewLBController(cfg *config.Config) (callbacks cluster.Callbacks, err error
}
},
Cleanup: func(ctx context.Context, cc cluster.CallbackContext) {
deleteIPs(cc, n, cfg.VirtualIPs)
db := db.New(cc)
ips, err := db.GetIPs()
if err != nil {
return
}
deleteIPs(cc, n, ips)
},
}
return

56
pkg/db/db.go Normal file
View file

@ -0,0 +1,56 @@
package db
import (
"git.giftfish.de/ston1th/haproxy-lb/pkg/cluster"
)
type DB struct {
kv cluster.KV
}
func New(kv cluster.KV) *DB {
return &DB{kv}
}
const (
ipPrefix = "/ip/"
namePrefix = "/name/"
)
func (db *DB) GetIPs() (ips []string, err error) {
m, err := db.kv.GetPrefix(namePrefix)
if err != nil {
return
}
for _, v := range m {
ips = append(ips, string(v))
}
return
}
func (db *DB) GetIP(ip string) (string, error) {
v, err := db.kv.Get(ipPrefix + ip)
return string(v), err
}
func (db *DB) IPExists(ip string) bool {
_, err := db.GetIP(ipPrefix + ip)
return err == nil
}
func (db *DB) SetIP(ip, name string) error {
err := db.kv.Set(ipPrefix+ip, []byte(name))
if err != nil {
return err
}
return db.kv.Set(namePrefix+name, []byte(ip))
}
func (db *DB) DeleteIP(ip, name string) error {
err := db.kv.Delete(ipPrefix + ip)
if err != nil {
return err
}
return db.kv.Delete(namePrefix + name)
}
func (db *DB) GetName(name string) (string, error) {
v, err := db.kv.Get(namePrefix + name)
return string(v), err
}

9
pkg/etcd/cluster.go
View file

@ -5,13 +5,16 @@ import (
"errors"
"strings"
"git.giftfish.de/ston1th/vipman/pkg/cluster"
"git.giftfish.de/ston1th/haproxy-lb/pkg/cluster"
//"go.etcd.io/etcd/clientv3"
"github.com/coreos/etcd/clientv3"
"github.com/go-logr/logr"
)
var ErrKeyNotFound = errors.New("key not found")
var (
ErrKeyNotFound = errors.New("etcd: key not found")
ErrPrefixNotFound = errors.New("etcd: prefix not found")
)
type Cluster struct {
logr.Logger
@ -59,7 +62,7 @@ func (c *Cluster) GetPrefix(k string) (m map[string][]byte, err error) {
return
}
if len(r.Kvs) == 0 {
err = ErrKeyNotFound
err = ErrPrefixNotFound
return
}
m = make(map[string][]byte)

9
pkg/etcd/etcd.go
View file

@ -6,8 +6,8 @@ import (
"fmt"
"time"
"git.giftfish.de/ston1th/vipman/pkg/config"
"git.giftfish.de/ston1th/vipman/pkg/util"
"git.giftfish.de/ston1th/haproxy-lb/pkg/config"
"git.giftfish.de/ston1th/haproxy-lb/pkg/util"
"github.com/coreos/etcd/clientv3"
"go.uber.org/zap"
"go.uber.org/zap/zapcore"
@ -198,6 +198,11 @@ func (c *Cluster) Stepdown() {
}
}
func (c *Cluster) Fatal(err error, msg string) {
c.Error(err, "a fatal error occurred", "message", msg)
c.Stop()
}
func (c *Cluster) Stop() {
if !c.session {
c.cancelSession()

7
pkg/haproxy/haproxy.go
View file

@ -7,12 +7,13 @@ import (
"io"
"io/ioutil"
"os"
"os/exec"
"sync"
"text/template"
)
type ServiceManager interface {
Reload()
Reload(context.Context) error
}
type HAProxyManager struct {
@ -25,7 +26,7 @@ type HAProxyManager struct {
}
func NewHAProxyManager(configFile string) (*HAProxyManager, error) {
t, err := template.New("haproxy").Parse(haproxyTemplate)
t, err := template.New("haproxy").Parse(haproxyConfigTemplate)
if err != nil {
return nil, err
}
@ -74,5 +75,5 @@ func (ha *HAProxyManager) UpdateConfig(ctx context.Context, lbs Config) error {
}
file.Close()
ha.hash = hash
return ha.serviceManager.Reload()
return ha.serviceManager.Reload(ctx)
}

2
pkg/raft/cluster.go
View file

@ -3,7 +3,7 @@ package raft
import (
"errors"
"git.giftfish.de/ston1th/vipman/pkg/cluster"
"git.giftfish.de/ston1th/haproxy-lb/pkg/cluster"
"github.com/go-logr/logr"
"github.com/hashicorp/raft"
)

16
pkg/raft/kv.go
View file

@ -2,13 +2,18 @@ package raft
import (
"errors"
"strings"
"sync"
"git.giftfish.de/ston1th/raftbbolt/msgpack"
"github.com/hashicorp/raft"
)
var ErrNotLeader = errors.New("not leader")
var (
ErrNotLeader = errors.New("raft: not leader")
ErrKeyNotFound = errors.New("raft: key not found")
ErrPrefixNotFound = errors.New("raft: prefix not found")
)
type kvm map[string][]byte
@ -21,7 +26,11 @@ type kv struct {
func (kv *kv) Get(k string) ([]byte, error) {
kv.mu.Lock()
defer kv.mu.Unlock()
return kv.m[k], nil
v, ok := kv.m[k]
if !ok {
return nil, ErrKeyNotFound
}
return v, nil
}
func (kv *kv) GetPrefix(pk string) (m map[string][]byte, err error) {
@ -33,6 +42,9 @@ func (kv *kv) GetPrefix(pk string) (m map[string][]byte, err error) {
m[k] = v
}
}
if len(m) == 0 {
err = ErrPrefixNotFound
}
return
}

11
pkg/raft/raft.go
View file

@ -8,10 +8,10 @@ import (
"path/filepath"
"time"
"git.giftfish.de/ston1th/haproxy-lb/pkg/config"
"git.giftfish.de/ston1th/haproxy-lb/pkg/raft/tls"
"git.giftfish.de/ston1th/haproxy-lb/pkg/util"
"git.giftfish.de/ston1th/raftbbolt"
"git.giftfish.de/ston1th/vipman/pkg/config"
"git.giftfish.de/ston1th/vipman/pkg/raft/tls"
"git.giftfish.de/ston1th/vipman/pkg/util"
hclog "github.com/hashicorp/go-hclog"
"github.com/hashicorp/raft"
)
@ -187,6 +187,11 @@ func (c *Cluster) Stepdown() {
}
}
func (c *Cluster) Fatal(err error, msg string) {
c.Error(err, "a fatal error occurred", "message", msg)
c.Stop()
}
func (c *Cluster) Stop() {
close(c.stop)
<-c.done

2
pkg/raft/tls/stream.go
View file

@ -6,7 +6,7 @@ import (
"net"
"time"
"git.giftfish.de/ston1th/vipman/pkg/config"
"git.giftfish.de/ston1th/haproxy-lb/pkg/config"
"github.com/hashicorp/raft"
)