initial commit

pkg/cluster/cluster.go

@@ -0,0 +1,45 @@
+package cluster
+
+import (
+	"context"
+	"errors"
+)
+
+// Cluster - The Cluster object manages the state of the cluster for a particular node
+type Cluster struct {
+	stateMachine FSM
+	stop         chan struct{}
+	done         chan struct{}
+	Callbacks    LeaderCallbacks
+}
+
+type LeaderCallbacks struct {
+	// OnStartedLeading is called when a LeaderElector client starts leading
+	OnStartedLeading func(context.Context)
+	// OnStoppedLeading is called when a LeaderElector client stops leading
+	OnStoppedLeading func()
+	// OnNewLeader is called when the client observes a leader that is
+	// not the previously observed leader. This includes the first observed
+	// leader when the client starts.
+	OnNewLeader func(string)
+}
+
+// InitCluster - Will attempt to initialise all of the required settings for the cluster
+func InitCluster(callbacks LeaderCallbacks) (*Cluster, error) {
+	if callbacks.OnStartedLeading == nil {
+		return nil, errors.New("OnStartedLeading is nil")
+	}
+	if callbacks.OnStoppedLeading == nil {
+		return nil, errors.New("OnStoppedLeading is nil")
+	}
+	if callbacks.OnNewLeader == nil {
+		return nil, errors.New("OnNewLeader is nil")
+	}
+	c := &Cluster{
+		stop:      make(chan struct{}, 1),
+		done:      make(chan struct{}, 1),
+		Callbacks: callbacks,
+	}
+
+	return c, nil
+}

pkg/cluster/raft.go

@@ -0,0 +1,200 @@
+package cluster
+
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	"net"
+	"net/http"
+	"os"
+	"runtime"
+	"time"
+
+	"git.giftfish.de/ston1th/vipman/pkg/config"
+	"github.com/go-logr/logr"
+	"github.com/hashicorp/raft"
+	raftboltdb "github.com/hashicorp/raft-boltdb"
+	"k8s.io/klog"
+)
+
+type raftLog string
+
+func (r *raftLog) Write(p []byte) (n int, err error) {
+	return os.Stdout.Write(append([]byte(*r), p...))
+}
+
+var _ = ioutil.Discard
+
+// StartRaftCluster - Begins a running instance of the Raft cluster
+func (c *Cluster) StartRaftCluster(log logr.Logger, cfg *config.Config) error {
+
+	// Create local configuration address
+	localAddress := fmt.Sprintf("%s:%d", cfg.LocalPeer.Address, cfg.LocalPeer.Port)
+
+	// Begin the Raft configuration
+	rc := raft.DefaultConfig()
+	rc.LocalID = raft.ServerID(cfg.LocalPeer.ID)
+	rl := raftLog(cfg.LocalPeer.ID + "  ")
+	rc.LogOutput = &rl
+	//rc.LogOutput = ioutil.Discard
+
+	// Initialize communication
+	address, err := net.ResolveTCPAddr("tcp", localAddress)
+	if err != nil {
+		return err
+	}
+
+	// Create transport
+	transport, err := raft.NewTCPTransport(localAddress, address, 3, 10*time.Second, os.Stdout)
+	if err != nil {
+		return err
+	}
+
+	// Create Raft structures
+	snapshots := raft.NewInmemSnapshotStore()
+	//logStore := raft.NewInmemStore()
+	bootstrap := true
+	if _, err := os.Stat(cfg.DBFile); err == nil {
+		bootstrap = false
+	}
+	stableStore, err := raftboltdb.NewBoltStore(cfg.DBFile)
+	if err != nil {
+		return err
+	}
+	logStore, err := raftboltdb.NewBoltStore(cfg.DBFile + ".log")
+	if err != nil {
+		return err
+	}
+
+	// Cluster configuration
+	configuration := raft.Configuration{}
+
+	// Add Local Peer
+	configuration.Servers = append(configuration.Servers, raft.Server{
+		ID:      raft.ServerID(cfg.LocalPeer.ID),
+		Address: raft.ServerAddress(localAddress),
+	})
+
+	// If we want to start a node as leader then we will not add any remote peers, this will leave this as a cluster of one
+	// The remotePeers will add themselves to the cluster as they're added
+	if !cfg.Leader {
+		for _, p := range cfg.RemotePeers {
+			peerAddress := fmt.Sprintf("%s:%d", p.Address, p.Port)
+			if localAddress != peerAddress {
+				configuration.Servers = append(configuration.Servers, raft.Server{
+					ID:      raft.ServerID(p.ID),
+					Address: raft.ServerAddress(peerAddress)})
+			}
+		}
+		log.Info("This node will attempt to start as Follower")
+	} else {
+		log.Info("This node will attempt to start as Leader")
+	}
+
+	if bootstrap {
+		if err := raft.BootstrapCluster(rc, logStore, stableStore, snapshots, transport, configuration); err != nil {
+			return err
+		}
+	}
+	//else {
+	//	if err := raft.RecoverCluster(rc, c.stateMachine, logStore, stableStore, snapshots, transport, configuration); err != nil {
+	//		return err
+	//	}
+	//}
+
+	// Create RAFT instance
+	raftServer, err := raft.NewRaft(rc, c.stateMachine, logStore, stableStore, snapshots, transport)
+	if err != nil {
+		return err
+	}
+
+	ticker := time.NewTicker(time.Second)
+	time.Sleep(time.Second * 5)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	lead := raftServer.Leader()
+	leading := false
+	for {
+		l := raftServer.Leader()
+		if lead != l && string(l) != "" {
+			go c.Callbacks.OnNewLeader(string(l))
+			lead = l
+		}
+		if !leading && localAddress == string(raftServer.Leader()) {
+			leading = true
+			go c.Run(ctx)
+		}
+
+		select {
+		case leader := <-raftServer.LeaderCh():
+			if leader {
+				if !leading {
+					leading = true
+					go c.Run(ctx)
+				}
+			} else {
+				if leading {
+					cancel()
+					leading = false
+					ctx, cancel = context.WithCancel(context.Background())
+				}
+			}
+		case <-ticker.C:
+		case <-c.stop:
+			if leading {
+				cancel()
+				raftServer.LeadershipTransfer()
+			}
+			close(c.done)
+			return nil
+		}
+	}
+	return nil
+}
+
+func (c *Cluster) Run(ctx context.Context) {
+	defer func() {
+		handleCrash()
+		c.Callbacks.OnStoppedLeading()
+	}()
+	c.Callbacks.OnStartedLeading(ctx)
+}
+
+var reallyCrash = false
+
+func handleCrash() {
+	if r := recover(); r != nil {
+		logPanic(r)
+		if reallyCrash {
+			panic(r)
+		}
+	}
+}
+
+// logPanic logs the caller tree when a panic occurs (except in the special case of http.ErrAbortHandler).
+func logPanic(r interface{}) {
+	if r == http.ErrAbortHandler {
+		// honor the http.ErrAbortHandler sentinel panic value:
+		//   ErrAbortHandler is a sentinel panic value to abort a handler.
+		//   While any panic from ServeHTTP aborts the response to the client,
+		//   panicking with ErrAbortHandler also suppresses logging of a stack trace to the server's error log.
+		return
+	}
+
+	// Same as stdlib http server code. Manually allocate stack trace buffer size
+	// to prevent excessively large logs
+	const size = 64 << 10
+	stacktrace := make([]byte, size)
+	stacktrace = stacktrace[:runtime.Stack(stacktrace, false)]
+	if _, ok := r.(string); ok {
+		klog.Errorf("Observed a panic: %s\n%s", r, stacktrace)
+	} else {
+		klog.Errorf("Observed a panic: %#v (%v)\n%s", r, r, stacktrace)
+	}
+}
+
+func (c *Cluster) Stop() {
+	close(c.stop)
+	<-c.done
+}

pkg/cluster/state.go

@@ -0,0 +1,32 @@
+package cluster
+
+import (
+	"io"
+
+	"github.com/hashicorp/raft"
+)
+
+type FSM struct {
+}
+
+func (fsm FSM) Apply(log *raft.Log) interface{} {
+	return nil
+}
+
+func (fsm FSM) Restore(snap io.ReadCloser) error {
+	return nil
+}
+
+func (fsm FSM) Snapshot() (raft.FSMSnapshot, error) {
+	return Snapshot{}, nil
+}
+
+type Snapshot struct {
+}
+
+func (snapshot Snapshot) Persist(sink raft.SnapshotSink) error {
+	return nil
+}
+
+func (snapshot Snapshot) Release() {
+}

pkg/config/config.go

@@ -0,0 +1,48 @@
+package config
+
+import (
+	"errors"
+	"gopkg.in/yaml.v2"
+	"os"
+)
+
+func ParseFile(file string) (cfg *Config, err error) {
+	if file == "" {
+		return nil, errors.New("missing config file")
+	}
+	f, err := os.Open(file)
+	if err != nil {
+		return
+	}
+	cfg = new(Config)
+	err = yaml.NewDecoder(f).Decode(cfg)
+	return
+}
+
+type Config struct {
+	DBFile string `yaml:"dbFile"`
+
+	Leader bool `yaml:"leader"`
+
+	AddressRange []string `yaml:"addressRange"`
+
+	// LocalPeer is the configuration of this host
+	LocalPeer RaftPeer `yaml:"localPeer"`
+
+	// Peers are all of the peers within the RAFT cluster
+	RemotePeers []RaftPeer `yaml:"remotePeers"`
+
+	// Interface is the network interface to bind to (default: First Adapter)
+	Interface string `yaml:"interface,omitempty"`
+}
+
+// RaftPeer details the configuration of all cluster peers
+type RaftPeer struct {
+	ID string `yaml:"id"`
+
+	// IP Address of a peer instance
+	Address string `yaml:"address"`
+
+	// Listening port of this peer instance
+	Port int `yaml:"port"`
+}

pkg/vip/arp.go

@@ -0,0 +1,10 @@
+// +build !linux
+
+package vip
+
+import "fmt"
+
+// ARPSendGratuitous is only supported on Linux, so return an error
+func ARPSendGratuitous(address, ifaceName string) error {
+	return fmt.Errorf("Unsupported on this OS")
+}

pkg/vip/arp_linux.go

@@ -0,0 +1,176 @@
+// +build linux
+
+// These syscalls are only supported on Linux, so this uses a build directive during compilation. Other OS's will use the arp_unsupported.go and recieve an error
+
+package vip
+
+import (
+	"bytes"
+	"encoding/binary"
+	"fmt"
+	"net"
+	"syscall"
+	"unsafe"
+
+	log "github.com/sirupsen/logrus"
+)
+
+const (
+	opARPRequest = 1
+	opARPReply   = 2
+	hwLen        = 6
+)
+
+var (
+	ethernetBroadcast = net.HardwareAddr{0xff, 0xff, 0xff, 0xff, 0xff, 0xff}
+	// arpRequest is used to flip between garp request or garp reply
+	arpRequest = true
+)
+
+func htons(p uint16) uint16 {
+	var b [2]byte
+	binary.BigEndian.PutUint16(b[:], p)
+	return *(*uint16)(unsafe.Pointer(&b))
+}
+
+// arpHeader specifies the header for an ARP message.
+type arpHeader struct {
+	hardwareType          uint16
+	protocolType          uint16
+	hardwareAddressLength uint8
+	protocolAddressLength uint8
+	opcode                uint16
+}
+
+// arpMessage represents an ARP message.
+type arpMessage struct {
+	arpHeader
+	senderHardwareAddress []byte
+	senderProtocolAddress []byte
+	targetHardwareAddress []byte
+	targetProtocolAddress []byte
+}
+
+// bytes returns the wire representation of the ARP message.
+func (m *arpMessage) bytes() ([]byte, error) {
+	buf := new(bytes.Buffer)
+
+	if err := binary.Write(buf, binary.BigEndian, m.arpHeader); err != nil {
+		return nil, fmt.Errorf("binary write failed: %v", err)
+	}
+	buf.Write(m.senderHardwareAddress)
+	buf.Write(m.senderProtocolAddress)
+	buf.Write(m.targetHardwareAddress)
+	buf.Write(m.targetProtocolAddress)
+
+	return buf.Bytes(), nil
+}
+
+// gratuitousARP return a gARP request or gARP reply alternatively
+// because different devices may support either one of them
+func gratuitousARP(ip net.IP, mac net.HardwareAddr) (*arpMessage, error) {
+	if ip.To4() == nil {
+		return nil, fmt.Errorf("%q is not an IPv4 address", ip)
+	}
+	if len(mac) != hwLen {
+		return nil, fmt.Errorf("%q is not an Ethernet MAC address", mac)
+	}
+
+	m := &arpMessage{
+		arpHeader: arpHeader{
+			1,           // Ethernet
+			0x0800,      // IPv4
+			hwLen,       // 48-bit MAC Address
+			net.IPv4len, // 32-bit IPv4 Address
+			opARPReply,  // ARP Reply
+		},
+	}
+
+	// https://tools.ietf.org/html/rfc5944#section-4.6
+	// In either case, the ARP Sender Hardware Address is
+	// set to the link-layer address to which this cache entry should be
+	// updated.
+	m.senderHardwareAddress = mac
+
+	// When using an ARP Reply packet, the Target Hardware
+	// Address is also set to the link-layer address to which this cache
+	// entry should be updated (this field is not used in an ARP Request
+	// packet).
+	m.targetHardwareAddress = mac
+
+	// In either case, the ARP Sender Protocol Address and
+	// ARP Target Protocol Address are both set to the IP address of the
+	// cache entry to be updated,
+	m.senderProtocolAddress = ip.To4()
+	m.targetProtocolAddress = ip.To4()
+
+	// send arpRequest and arpReply alternatively
+	arpRequest = !arpRequest
+	if arpRequest {
+		m.arpHeader.opcode = opARPRequest
+
+		// this field is not used in an ARP Request packet
+		m.targetHardwareAddress = ethernetBroadcast
+	}
+
+	return m, nil
+}
+
+// sendARP sends the given ARP message via the specified interface.
+func sendARP(iface *net.Interface, m *arpMessage) error {
+	fd, err := syscall.Socket(syscall.AF_PACKET, syscall.SOCK_DGRAM, int(htons(syscall.ETH_P_ARP)))
+	if err != nil {
+		return fmt.Errorf("failed to get raw socket: %v", err)
+	}
+	defer syscall.Close(fd)
+
+	if err := syscall.BindToDevice(fd, iface.Name); err != nil {
+		return fmt.Errorf("failed to bind to device: %v", err)
+	}
+
+	ll := syscall.SockaddrLinklayer{
+		Protocol: htons(syscall.ETH_P_ARP),
+		Ifindex:  iface.Index,
+		Pkttype:  0, // syscall.PACKET_HOST
+		Hatype:   m.hardwareType,
+		Halen:    m.hardwareAddressLength,
+	}
+	target := ethernetBroadcast
+	for i := 0; i < len(target); i++ {
+		ll.Addr[i] = target[i]
+	}
+
+	b, err := m.bytes()
+	if err != nil {
+		return fmt.Errorf("failed to convert ARP message: %v", err)
+	}
+
+	if err := syscall.Bind(fd, &ll); err != nil {
+		return fmt.Errorf("failed to bind: %v", err)
+	}
+	if err := syscall.Sendto(fd, b, 0, &ll); err != nil {
+		return fmt.Errorf("failed to send: %v", err)
+	}
+
+	return nil
+}
+
+// ARPSendGratuitous sends a gratuitous ARP message via the specified interface.
+func ARPSendGratuitous(address, ifaceName string) error {
+	iface, err := net.InterfaceByName(ifaceName)
+	if err != nil {
+		return fmt.Errorf("failed to get interface %q: %v", ifaceName, err)
+	}
+
+	ip := net.ParseIP(address)
+	if ip == nil {
+		return fmt.Errorf("failed to parse address %s", ip)
+	}
+
+	log.Infof("Broadcasting ARP update for %s (%s) via %s", address, iface.HardwareAddr, iface.Name)
+	m, err := gratuitousARP(ip, iface.HardwareAddr)
+	if err != nil {
+		return err
+	}
+	return sendARP(iface, m)
+}

pkg/vip/vip.go

@@ -0,0 +1,142 @@
+package vip
+
+import (
+	"sync"
+
+	"github.com/pkg/errors"
+	"github.com/vishvananda/netlink"
+)
+
+const (
+	defaultValidLft = 60
+)
+
+// Network is an interface that enable managing operations for a given IP
+type Network interface {
+	AddIP() error
+	DeleteIP() error
+	IsSet() (bool, error)
+	IP() string
+	SetIP(ip string) error
+	Interface() string
+}
+
+// network - This allows network configuration
+type network struct {
+	mu sync.Mutex
+
+	address *netlink.Addr
+	link    netlink.Link
+	isDNS   bool
+}
+
+// NewConfig will attempt to provide an interface to the kernel network configuration
+func NewConfig(address string, iface string) (Network, error) {
+	result := &network{}
+
+	link, err := netlink.LinkByName(iface)
+	if err != nil {
+		return result, errors.Wrapf(err, "could not get link for interface '%s'", iface)
+	}
+	result.link = link
+
+	if IsIP(address) {
+		result.address, err = netlink.ParseAddr(address + "/32")
+		if err != nil {
+			return result, errors.Wrapf(err, "could not parse address '%s'", address)
+		}
+		return result, nil
+	}
+
+	// try to resolve the address
+	ip, err := lookupHost(address)
+	result.isDNS = err == nil
+	if err != nil {
+		return nil, err
+	}
+
+	// we're able to resolve store this as the initial IP
+	if result.address, err = netlink.ParseAddr(ip + "/32"); err != nil {
+		return result, err
+	}
+	// set ValidLft so that the VIP expires if the DNS entry is updated, otherwise it'll be refreshed by the DNS prober
+	result.address.ValidLft = defaultValidLft
+
+	return result, err
+}
+
+//AddIP - Add an IP address to the interface
+func (configurator *network) AddIP() error {
+	if err := netlink.AddrReplace(configurator.link, configurator.address); err != nil {
+		return errors.Wrap(err, "could not add ip")
+	}
+	return nil
+}
+
+//DeleteIP - Remove an IP address from the interface
+func (configurator *network) DeleteIP() error {
+	result, err := configurator.IsSet()
+	if err != nil {
+		return errors.Wrap(err, "ip check in DeleteIP failed")
+	}
+
+	// Nothing to delete
+	if !result {
+		return nil
+	}
+
+	if err = netlink.AddrDel(configurator.link, configurator.address); err != nil {
+		return errors.Wrap(err, "could not delete ip")
+	}
+
+	return nil
+}
+
+// IsSet - Check to see if VIP is set
+func (configurator *network) IsSet() (result bool, err error) {
+	var addresses []netlink.Addr
+
+	addresses, err = netlink.AddrList(configurator.link, 0)
+	if err != nil {
+		err = errors.Wrap(err, "could not list addresses")
+
+		return
+	}
+
+	for _, address := range addresses {
+		if address.Equal(*configurator.address) {
+			return true, nil
+		}
+	}
+
+	return false, nil
+}
+
+// SetIP updates the IP that is used
+func (configurator *network) SetIP(ip string) error {
+	configurator.mu.Lock()
+	defer configurator.mu.Unlock()
+
+	addr, err := netlink.ParseAddr(ip + "/32")
+	if err != nil {
+		return err
+	}
+	if configurator.address != nil && configurator.isDNS {
+		addr.ValidLft = defaultValidLft
+	}
+	configurator.address = addr
+	return nil
+}
+
+// IP - return the IP Address
+func (configurator *network) IP() string {
+	configurator.mu.Lock()
+	defer configurator.mu.Unlock()
+
+	return configurator.address.IP.String()
+}
+
+// Interface - return the Interface name
+func (configurator *network) Interface() string {
+	return configurator.link.Attrs().Name
+}