added raft over tls

pkg/raft/raft.go

@@ -0,0 +1,235 @@
+package raft
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"net/http"
+	"os"
+	"path/filepath"
+	"runtime"
+	"time"
+
+	"git.giftfish.de/ston1th/raftbbolt"
+	"git.giftfish.de/ston1th/vipman/pkg/config"
+	"git.giftfish.de/ston1th/vipman/pkg/raft/tls"
+	"github.com/go-logr/logr"
+	hclog "github.com/hashicorp/go-hclog"
+	"github.com/hashicorp/raft"
+	"k8s.io/klog/v2"
+)
+
+type raftLog string
+
+func (r *raftLog) Write(p []byte) (n int, err error) {
+	return os.Stdout.Write(append([]byte(*r), p...))
+}
+
+func (c *Cluster) Start(log logr.Logger, raftcfg *config.Config) error {
+	cfg := raftcfg.Cluster.Raft
+	hcl := hclog.New(&hclog.LoggerOptions{
+		Name:       "raft",
+		Level:      hclog.LevelFromString(cfg.Level),
+		Output:     hclog.DefaultOutput,
+		TimeFormat: "I0102 15:04:05.000000",
+	})
+
+	rc := raft.DefaultConfig()
+	rc.ProtocolVersion = 3
+	rc.LocalID = raft.ServerID(cfg.ID)
+	rc.Logger = hcl
+
+	pool := 3
+	timeout := 10 * time.Second
+	var transport *raft.NetworkTransport
+	if cfg.TLS != nil {
+		log.Info("transport: tls")
+		s, err := tls.NewStream(cfg)
+		if err != nil {
+			return err
+		}
+		transport = raft.NewNetworkTransportWithLogger(s, pool, timeout, hcl)
+	} else {
+		log.Info("transport: tcp")
+		address, err := net.ResolveTCPAddr("tcp", cfg.Address)
+		if err != nil {
+			return err
+		}
+		transport, err = raft.NewTCPTransportWithLogger(cfg.Address, address, pool, timeout, hcl)
+		if err != nil {
+			return err
+		}
+	}
+
+	// Create Raft structures
+	//snapshots := raft.NewInmemSnapshotStore()
+	snapshots, err := raft.NewFileSnapshotStoreWithLogger(cfg.Dir, retainSnapshotCount, hcl)
+	if err != nil {
+		return fmt.Errorf("file snapshot store: %s", err)
+	}
+
+	store := filepath.Join(cfg.Dir, "store.db")
+
+	//logStore := raft.NewInmemStore()
+	bootstrap := true
+	if _, err := os.Stat(store); err == nil {
+		bootstrap = false
+	}
+
+	stableStore, err := raftbbolt.NewStore(store)
+	if err != nil {
+		return err
+	}
+	logStore := stableStore
+
+	// Cluster configuration
+	configuration := raft.Configuration{}
+
+	// Add Local Peer
+	configuration.Servers = append(configuration.Servers, raft.Server{
+		ID:      raft.ServerID(cfg.ID),
+		Address: raft.ServerAddress(cfg.Address),
+	})
+	for _, p := range cfg.Peers {
+		if cfg.Address != p.Address {
+			configuration.Servers = append(configuration.Servers, raft.Server{
+				ID:      raft.ServerID(p.ID),
+				Address: raft.ServerAddress(p.Address)})
+		}
+	}
+
+	if bootstrap {
+		if err := raft.BootstrapCluster(rc, logStore, stableStore, snapshots, transport, configuration); err != nil {
+			return err
+		}
+	}
+
+	// Create RAFT instance
+	c.r, err = raft.NewRaft(rc, (*fsm)(c.kv), logStore, stableStore, snapshots, transport)
+	if err != nil {
+		return err
+	}
+	c.kv.r = c.r
+
+	t := time.NewTicker(time.Second)
+	time.Sleep(time.Second * 3)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	leading := false
+	following := false
+	for {
+		if !leading && cfg.Address == string(c.r.Leader()) {
+			leading = true
+			go c.leader(ctx)
+		} else if !following && !leading {
+			following = true
+			go c.follower(ctx)
+		}
+
+		select {
+		case leader := <-c.r.LeaderCh():
+			if leader {
+				if following {
+					cancel()
+					following = false
+					ctx, cancel = context.WithCancel(context.Background())
+				}
+				if !leading {
+					leading = true
+					go c.leader(ctx)
+				}
+			} else if c.r.State() == raft.Follower {
+				if leading {
+					cancel()
+					leading = false
+					ctx, cancel = context.WithCancel(context.Background())
+				}
+				if !following {
+					following = true
+					go c.follower(ctx)
+				}
+			}
+		case <-c.stepdown:
+			if leading {
+				cancel()
+				c.r.LeadershipTransfer().Error()
+			}
+		case <-c.stop:
+			t.Stop()
+			cancel()
+			if leading {
+				c.r.LeadershipTransfer().Error()
+			}
+			close(c.done)
+			return nil
+		case <-t.C:
+		}
+	}
+	return nil
+}
+
+func (c *Cluster) leader(ctx context.Context) {
+	defer func() {
+		handleCrash()
+	}()
+	c.Callbacks.Leader(ctx, c.kv)
+}
+
+func (c *Cluster) follower(ctx context.Context) {
+	defer func() {
+		handleCrash()
+	}()
+	c.Callbacks.Follower(ctx, c.kv)
+}
+
+var reallyCrash = false
+
+func handleCrash() {
+	if r := recover(); r != nil {
+		logPanic(r)
+		if reallyCrash {
+			panic(r)
+		}
+	}
+}
+
+// logPanic logs the caller tree when a panic occurs (except in the special case of http.ErrAbortHandler).
+func logPanic(r interface{}) {
+	if r == http.ErrAbortHandler {
+		// honor the http.ErrAbortHandler sentinel panic value:
+		//   ErrAbortHandler is a sentinel panic value to abort a handler.
+		//   While any panic from ServeHTTP aborts the response to the client,
+		//   panicking with ErrAbortHandler also suppresses logging of a stack trace to the server's error log.
+		return
+	}
+
+	// Same as stdlib http server code. Manually allocate stack trace buffer size
+	// to prevent excessively large logs
+	const size = 64 << 10
+	stacktrace := make([]byte, size)
+	stacktrace = stacktrace[:runtime.Stack(stacktrace, false)]
+	if _, ok := r.(string); ok {
+		klog.Errorf("Observed a panic: %s\n%s", r, stacktrace)
+	} else {
+		klog.Errorf("Observed a panic: %#v (%v)\n%s", r, r, stacktrace)
+	}
+}
+
+func (c *Cluster) Stepdown() {
+	c.stepdown <- struct{}{}
+}
+
+func (c *Cluster) Stop() {
+	close(c.stop)
+	<-c.done
+	s := make(chan struct{})
+	go func() {
+		c.r.Shutdown().Error()
+		close(s)
+	}()
+	select {
+	case <-s:
+	case <-time.After(time.Second * 5):
+	}
+}