ston1th/vipman
0
0
Fork 0
Code Issues 2 Releases Activity

added raft over tls

Browse source
This commit is contained in:
ston1th 2020-11-15 00:30:35 +01:00
commit 585baea183
16 changed files with 563 additions and 300 deletions
Download patch file Download diff file Expand all files Collapse all files

44
pkg/cluster/cluster.go
View file

@ -2,42 +2,24 @@ package cluster
import (
"context"
"errors"
"github.com/hashicorp/raft"
"git.giftfish.de/ston1th/vipman/pkg/config"
"github.com/go-logr/logr"
)
// Cluster - The Cluster object manages the state of the cluster for a particular node
type Cluster struct {
kv *KV
r *raft.Raft
type Cluster interface {
Start(logr.Logger, *config.Config) error
Stepdown()
Stop()
}
stop chan struct{}
done chan struct{}
Callbacks Callbacks
type KV interface {
Get(k string) []byte
Set(k string, v []byte) error
Delete(k string) error
}
type Callbacks struct {
Leader func(context.Context, *KV)
Follower func(context.Context, *KV)
}
// InitCluster - Will attempt to initialise all of the required settings for the cluster
func InitCluster(callbacks Callbacks) (*Cluster, error) {
if callbacks.Leader == nil {
return nil, errors.New("Leader is nil")
}
if callbacks.Follower == nil {
return nil, errors.New("Follower is nil")
}
c := &Cluster{
kv: &KV{
m: make(map[string][]byte),
},
stop: make(chan struct{}, 1),
done: make(chan struct{}, 1),
Callbacks: callbacks,
}
return c, nil
Leader func(context.Context, KV)
Follower func(context.Context, KV)
}

99
pkg/cluster/fsm.go
View file

@ -1,99 +0,0 @@
package cluster
import (
gogob "encoding/gob"
"fmt"
"io"
"time"
"github.com/hashicorp/raft"
)
const (
retainSnapshotCount = 2
raftTimeout = 10 * time.Second
)
type Op uint8
const (
SET Op = iota
DELETE
)
type raftCmd struct {
Op Op
K string
V []byte
}
type fsm KV
func (f *fsm) Apply(l *raft.Log) interface{} {
var c raftCmd
if err := gob.Unmarshal(l.Data, &c); err != nil {
panic(fmt.Sprintf("failed to unmarshal command: %s", err.Error()))
}
switch c.Op {
case SET:
f.applySet(c.K, c.V)
case DELETE:
f.applyDelete(c.K)
default:
panic(fmt.Sprintf("unrecognized command op: %s", c.Op))
}
return nil
}
func (f *fsm) Snapshot() (raft.FSMSnapshot, error) {
f.mu.Lock()
defer f.mu.Unlock()
m := make(kv)
for k, v := range f.m {
m[k] = v
}
return &fsmSnapshot{m: m}, nil
}
func (f *fsm) Restore(rc io.ReadCloser) error {
o := make(kv)
if err := gogob.NewDecoder(rc).Decode(&o); err != nil {
return err
}
f.m = o
return nil
}
func (f *fsm) applySet(k string, v []byte) {
f.mu.Lock()
f.m[k] = v
f.mu.Unlock()
}
func (f *fsm) applyDelete(k string) {
f.mu.Lock()
delete(f.m, k)
f.mu.Unlock()
}
type fsmSnapshot struct {
m kv
}
func (f *fsmSnapshot) Persist(sink raft.SnapshotSink) error {
err := func() error {
err := gogob.NewEncoder(sink).Encode(f.m)
if err != nil {
return err
}
return sink.Close()
}()
if err != nil {
sink.Cancel()
}
return err
}
func (f *fsmSnapshot) Release() {}

55
pkg/cluster/kv.go
View file

@ -1,55 +0,0 @@
package cluster
import (
"errors"
"sync"
"github.com/hashicorp/raft"
)
var ErrNotLeader = errors.New("not leader")
type kv map[string][]byte
type KV struct {
mu sync.Mutex
m kv
r *raft.Raft
}
func (kv *KV) Get(k string) []byte {
kv.mu.Lock()
defer kv.mu.Unlock()
return kv.m[k]
}
func (kv *KV) Set(k string, v []byte) error {
if kv.r.State() != raft.Leader {
return ErrNotLeader
}
c := &raftCmd{SET, k, v}
b, err := gob.Marshal(c)
if err != nil {
return err
}
f := kv.r.Apply(b, raftTimeout)
return f.Error()
}
// Delete deletes the given key.
func (kv *KV) Delete(k string) error {
if kv.r.State() != raft.Leader {
return ErrNotLeader
}
c := &raftCmd{Op: DELETE, K: k}
b, err := gob.Marshal(c)
if err != nil {
return err
}
f := kv.r.Apply(b, raftTimeout)
return f.Error()
}

21
pkg/cluster/marshaler.go
View file

@ -1,21 +0,0 @@
package cluster
import (
"bytes"
gogob "encoding/gob"
)
type gobMarshaler struct{}
func (gobMarshaler) Marshal(v interface{}) (b []byte, err error) {
buf := new(bytes.Buffer)
err = gogob.NewEncoder(buf).Encode(v)
b = buf.Bytes()
return
}
func (gobMarshaler) Unmarshal(data []byte, v interface{}) error {
return gogob.NewDecoder(bytes.NewBuffer(data)).Decode(v)
}
var gob = gobMarshaler{}

248
pkg/cluster/raft.go
View file

@ -1,248 +0,0 @@
package cluster
import (
"context"
"fmt"
"io/ioutil"
"net"
"net/http"
"os"
"path/filepath"
"runtime"
"time"
"git.giftfish.de/ston1th/vipman/pkg/config"
"github.com/go-logr/logr"
"github.com/hashicorp/raft"
raftboltdb "github.com/hashicorp/raft-boltdb"
"k8s.io/klog"
)
type raftLog string
func (r *raftLog) Write(p []byte) (n int, err error) {
return os.Stdout.Write(append([]byte(*r), p...))
}
var _ = ioutil.Discard
// StartRaftCluster - Begins a running instance of the Raft cluster
func (c *Cluster) StartRaftCluster(log logr.Logger, cfg *config.Config) error {
// Create local configuration address
localAddress := fmt.Sprintf("%s:%d", cfg.LocalPeer.Address, cfg.LocalPeer.Port)
// Begin the Raft configuration
rc := raft.DefaultConfig()
rc.ProtocolVersion = 3
rc.LocalID = raft.ServerID(cfg.LocalPeer.ID)
//rl := raftLog(cfg.LocalPeer.ID + " ")
//rc.LogOutput = &rl
rc.LogOutput = ioutil.Discard
// Initialize communication
address, err := net.ResolveTCPAddr("tcp", localAddress)
if err != nil {
return err
}
// Create transport
transport, err := raft.NewTCPTransport(localAddress, address, 3, 10*time.Second, os.Stdout)
if err != nil {
return err
}
// Create Raft structures
//snapshots := raft.NewInmemSnapshotStore()
snapshots, err := raft.NewFileSnapshotStore(cfg.RaftDir, retainSnapshotCount, os.Stderr)
if err != nil {
return fmt.Errorf("file snapshot store: %s", err)
}
store := filepath.Join(cfg.RaftDir, "store.db")
//logStore := raft.NewInmemStore()
bootstrap := true
if _, err := os.Stat(store); err == nil {
bootstrap = false
}
stableStore, err := raftboltdb.NewBoltStore(store)
if err != nil {
return err
}
logStore := stableStore
// Cluster configuration
configuration := raft.Configuration{}
// Add Local Peer
configuration.Servers = append(configuration.Servers, raft.Server{
Suffrage: raft.Voter,
ID: raft.ServerID(cfg.LocalPeer.ID),
Address: raft.ServerAddress(localAddress),
})
// If we want to start a node as leader then we will not add any remote peers, this will leave this as a cluster of one
// The remotePeers will add themselves to the cluster as they're added
//if !cfg.Leader {
// for _, p := range cfg.RemotePeers {
// peerAddress := fmt.Sprintf("%s:%d", p.Address, p.Port)
// if localAddress != peerAddress {
// configuration.Servers = append(configuration.Servers, raft.Server{
// ID: raft.ServerID(p.ID),
// Address: raft.ServerAddress(peerAddress)})
// }
// }
// log.Info("This node will attempt to start as Follower")
//} else {
// log.Info("This node will attempt to start as Leader")
//}
for _, p := range cfg.RemotePeers {
peerAddress := fmt.Sprintf("%s:%d", p.Address, p.Port)
if localAddress != peerAddress {
configuration.Servers = append(configuration.Servers, raft.Server{
Suffrage: raft.Voter,
ID: raft.ServerID(p.ID),
Address: raft.ServerAddress(peerAddress)})
}
}
if bootstrap {
if err := raft.BootstrapCluster(rc, logStore, stableStore, snapshots, transport, configuration); err != nil {
return err
}
}
//else {
// if err := raft.RecoverCluster(rc, c.stateMachine, logStore, stableStore, snapshots, transport, configuration); err != nil {
// return err
// }
//}
// Create RAFT instance
c.r, err = raft.NewRaft(rc, (*fsm)(c.kv), logStore, stableStore, snapshots, transport)
if err != nil {
return err
}
c.kv.r = c.r
ticker := time.NewTicker(time.Second)
time.Sleep(time.Second * 3)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
leading := false
following := false
for {
if !leading && localAddress == string(c.r.Leader()) {
leading = true
go c.leader(ctx)
} else if !following && !leading {
following = true
go c.follower(ctx)
}
select {
case leader := <-c.r.LeaderCh():
if leader {
if following {
cancel()
following = false
ctx, cancel = context.WithCancel(context.Background())
}
if !leading {
leading = true
//cf := c.r.GetConfiguration()
//cf.Error()
//log.Info(fmt.Sprintf("[%s] %#v", cfg.LocalPeer.ID, cf.Configuration()))
go c.leader(ctx)
}
} else if c.r.State() == raft.Follower {
if leading {
cancel()
leading = false
ctx, cancel = context.WithCancel(context.Background())
}
if !following {
following = true
//cf := c.r.GetConfiguration()
//cf.Error()
//log.Info(fmt.Sprintf("[%s] %#v", cfg.LocalPeer.ID, cf.Configuration()))
go c.follower(ctx)
}
}
case <-ticker.C:
case <-c.stop:
cancel()
if leading {
c.r.LeadershipTransfer().Error()
}
close(c.done)
return nil
}
}
return nil
}
func (c *Cluster) leader(ctx context.Context) {
defer func() {
handleCrash()
//c.Callbacks.OnStoppedLeading()
}()
c.Callbacks.Leader(ctx, c.kv)
}
func (c *Cluster) follower(ctx context.Context) {
defer func() {
handleCrash()
//c.Callbacks.OnStoppedLeading()
}()
c.Callbacks.Follower(ctx, c.kv)
}
var reallyCrash = false
func handleCrash() {
if r := recover(); r != nil {
logPanic(r)
if reallyCrash {
panic(r)
}
}
}
// logPanic logs the caller tree when a panic occurs (except in the special case of http.ErrAbortHandler).
func logPanic(r interface{}) {
if r == http.ErrAbortHandler {
// honor the http.ErrAbortHandler sentinel panic value:
// ErrAbortHandler is a sentinel panic value to abort a handler.
// While any panic from ServeHTTP aborts the response to the client,
// panicking with ErrAbortHandler also suppresses logging of a stack trace to the server's error log.
return
}
// Same as stdlib http server code. Manually allocate stack trace buffer size
// to prevent excessively large logs
const size = 64 << 10
stacktrace := make([]byte, size)
stacktrace = stacktrace[:runtime.Stack(stacktrace, false)]
if _, ok := r.(string); ok {
klog.Errorf("Observed a panic: %s\n%s", r, stacktrace)
} else {
klog.Errorf("Observed a panic: %#v (%v)\n%s", r, r, stacktrace)
}
}
func (c *Cluster) Stop() {
close(c.stop)
<-c.done
s := make(chan struct{})
go func() {
c.r.Shutdown().Error()
close(s)
}()
select {
case <-s:
case <-time.After(time.Second * 5):
}
}