added raft over tls

This commit is contained in:
ston1th 2020-11-15 00:30:35 +01:00
commit 585baea183
16 changed files with 563 additions and 300 deletions

View file

@ -2,42 +2,24 @@ package cluster
import (
"context"
"errors"
"github.com/hashicorp/raft"
"git.giftfish.de/ston1th/vipman/pkg/config"
"github.com/go-logr/logr"
)
// Cluster - The Cluster object manages the state of the cluster for a particular node
type Cluster struct {
kv *KV
r *raft.Raft
type Cluster interface {
Start(logr.Logger, *config.Config) error
Stepdown()
Stop()
}
stop chan struct{}
done chan struct{}
Callbacks Callbacks
type KV interface {
Get(k string) []byte
Set(k string, v []byte) error
Delete(k string) error
}
type Callbacks struct {
Leader func(context.Context, *KV)
Follower func(context.Context, *KV)
}
// InitCluster - Will attempt to initialise all of the required settings for the cluster
func InitCluster(callbacks Callbacks) (*Cluster, error) {
if callbacks.Leader == nil {
return nil, errors.New("Leader is nil")
}
if callbacks.Follower == nil {
return nil, errors.New("Follower is nil")
}
c := &Cluster{
kv: &KV{
m: make(map[string][]byte),
},
stop: make(chan struct{}, 1),
done: make(chan struct{}, 1),
Callbacks: callbacks,
}
return c, nil
Leader func(context.Context, KV)
Follower func(context.Context, KV)
}

View file

@ -1,21 +0,0 @@
package cluster
import (
"bytes"
gogob "encoding/gob"
)
type gobMarshaler struct{}
func (gobMarshaler) Marshal(v interface{}) (b []byte, err error) {
buf := new(bytes.Buffer)
err = gogob.NewEncoder(buf).Encode(v)
b = buf.Bytes()
return
}
func (gobMarshaler) Unmarshal(data []byte, v interface{}) error {
return gogob.NewDecoder(bytes.NewBuffer(data)).Decode(v)
}
var gob = gobMarshaler{}

View file

@ -2,8 +2,15 @@ package config
import (
"errors"
"gopkg.in/yaml.v2"
"fmt"
"os"
"gopkg.in/yaml.v2"
)
const (
RaftDir = "raft"
RaftPort = 7001
)
func ParseFile(file string) (cfg *Config, err error) {
@ -16,33 +23,99 @@ func ParseFile(file string) (cfg *Config, err error) {
}
cfg = new(Config)
err = yaml.NewDecoder(f).Decode(cfg)
if err != nil {
return
}
err = Validate(cfg)
return
}
func Validate(c *Config) error {
if len(c.VirtualIPs) == 0 {
return errors.New("missing virtualIPs config")
}
raft := c.Cluster.Raft
if raft != nil {
if raft.Dir == "" {
raft.Dir = RaftDir
}
if raft.Level == "" {
raft.Level = "off"
}
if raft.ID == "" {
return errors.New("missing raft.id config")
}
if raft.Address == "" {
return errors.New("missing raft.address config")
}
if len(raft.Peers) < 2 {
return errors.New("minimum number of remote peers: 2")
}
for i, v := range raft.Peers {
if v.ID == "" {
return fmt.Errorf("missing raft.peers[%d].id config", i)
}
if v.Address == "" {
return fmt.Errorf("missing raft.peers[%d].address config", i)
}
}
}
etcd := c.Cluster.Etcd
if etcd != nil {
}
if raft == nil && etcd == nil {
return errors.New("missing cluster config: raft or etcd")
}
if raft != nil && etcd != nil {
return errors.New("only one cluster config allowed: raft or etcd")
}
return nil
}
type Config struct {
RaftDir string `yaml:"raftDir"`
VirtualIPs []string `yaml:"virtualIPs"`
Leader bool `yaml:"leader"`
AddressRange []string `yaml:"addressRange"`
// LocalPeer is the configuration of this host
LocalPeer RaftPeer `yaml:"localPeer"`
// Peers are all of the peers within the RAFT cluster
RemotePeers []RaftPeer `yaml:"remotePeers"`
// Interface is the network interface to bind to (default: First Adapter)
Interface string `yaml:"interface,omitempty"`
IPLabel string `yaml:"label,omitempty"`
LeaderHook string `yaml:"leaderHook,omitempty"`
FollowerHook string `yaml:"followerHook,omitempty"`
Cluster Cluster `yaml:"cluster"`
}
type Cluster struct {
Raft *Raft `yaml:"raft,omitempty"`
Etcd *Etcd `yaml:"etcd,omitempty"`
}
type Raft struct {
Dir string `yaml:"dir,omitempty"`
ID string `yaml:"id"`
Address string `yaml:"address"`
Level string `yaml:"level"`
Peers []RaftPeer `yaml:"peers"`
TLS *TLS `yaml:"tls,omitempty"`
}
type TLS struct {
Cert string `yaml:"cert"`
Key string `yaml:"key"`
CA string `yaml:"ca,omitempty"`
Insecure bool `yaml:"insecure,omitempty"`
}
// RaftPeer details the configuration of all cluster peers
type RaftPeer struct {
ID string `yaml:"id"`
// IP Address of a peer instance
ID string `yaml:"id"`
Address string `yaml:"address"`
// Listening port of this peer instance
Port int `yaml:"port"`
}
type Etcd struct {
Endpoints []string `yaml:"endpoints"`
Prefix string `yaml:"prefix"`
ClusterName string `yaml:"clusterName"`
}

36
pkg/raft/cluster.go Normal file
View file

@ -0,0 +1,36 @@
package raft
import (
"errors"
"git.giftfish.de/ston1th/vipman/pkg/cluster"
"github.com/hashicorp/raft"
)
type Cluster struct {
kv *KV
r *raft.Raft
stepdown chan struct{}
stop chan struct{}
done chan struct{}
Callbacks cluster.Callbacks
}
func NewCluster(callbacks cluster.Callbacks) (cluster.Cluster, error) {
if callbacks.Leader == nil {
return nil, errors.New("Leader is nil")
}
if callbacks.Follower == nil {
return nil, errors.New("Follower is nil")
}
return &Cluster{
kv: &KV{
m: make(map[string][]byte),
},
stepdown: make(chan struct{}, 1),
stop: make(chan struct{}, 1),
done: make(chan struct{}, 1),
Callbacks: callbacks,
}, nil
}

View file

@ -1,11 +1,11 @@
package cluster
package raft
import (
gogob "encoding/gob"
"fmt"
"io"
"time"
"git.giftfish.de/ston1th/raftbbolt/msgpack"
"github.com/hashicorp/raft"
)
@ -31,7 +31,7 @@ type fsm KV
func (f *fsm) Apply(l *raft.Log) interface{} {
var c raftCmd
if err := gob.Unmarshal(l.Data, &c); err != nil {
if err := msgpack.Unmarshal(l.Data, &c); err != nil {
panic(fmt.Sprintf("failed to unmarshal command: %s", err.Error()))
}
@ -58,7 +58,7 @@ func (f *fsm) Snapshot() (raft.FSMSnapshot, error) {
func (f *fsm) Restore(rc io.ReadCloser) error {
o := make(kv)
if err := gogob.NewDecoder(rc).Decode(&o); err != nil {
if err := msgpack.NewDecoder(rc).Decode(&o); err != nil {
return err
}
f.m = o
@ -83,7 +83,7 @@ type fsmSnapshot struct {
func (f *fsmSnapshot) Persist(sink raft.SnapshotSink) error {
err := func() error {
err := gogob.NewEncoder(sink).Encode(f.m)
err := msgpack.NewEncoder(sink).Encode(f.m)
if err != nil {
return err
}

View file

@ -1,9 +1,10 @@
package cluster
package raft
import (
"errors"
"sync"
"git.giftfish.de/ston1th/raftbbolt/msgpack"
"github.com/hashicorp/raft"
)
@ -29,7 +30,7 @@ func (kv *KV) Set(k string, v []byte) error {
}
c := &raftCmd{SET, k, v}
b, err := gob.Marshal(c)
b, err := msgpack.Marshal(c)
if err != nil {
return err
}
@ -45,7 +46,7 @@ func (kv *KV) Delete(k string) error {
}
c := &raftCmd{Op: DELETE, K: k}
b, err := gob.Marshal(c)
b, err := msgpack.Marshal(c)
if err != nil {
return err
}

View file

@ -1,9 +1,8 @@
package cluster
package raft
import (
"context"
"fmt"
"io/ioutil"
"net"
"net/http"
"os"
@ -11,11 +10,13 @@ import (
"runtime"
"time"
"git.giftfish.de/ston1th/raftbbolt"
"git.giftfish.de/ston1th/vipman/pkg/config"
"git.giftfish.de/ston1th/vipman/pkg/raft/tls"
"github.com/go-logr/logr"
hclog "github.com/hashicorp/go-hclog"
"github.com/hashicorp/raft"
raftboltdb "github.com/hashicorp/raft-boltdb"
"k8s.io/klog"
"k8s.io/klog/v2"
)
type raftLog string
@ -24,42 +25,50 @@ func (r *raftLog) Write(p []byte) (n int, err error) {
return os.Stdout.Write(append([]byte(*r), p...))
}
var _ = ioutil.Discard
func (c *Cluster) Start(log logr.Logger, raftcfg *config.Config) error {
cfg := raftcfg.Cluster.Raft
hcl := hclog.New(&hclog.LoggerOptions{
Name: "raft",
Level: hclog.LevelFromString(cfg.Level),
Output: hclog.DefaultOutput,
TimeFormat: "I0102 15:04:05.000000",
})
// StartRaftCluster - Begins a running instance of the Raft cluster
func (c *Cluster) StartRaftCluster(log logr.Logger, cfg *config.Config) error {
// Create local configuration address
localAddress := fmt.Sprintf("%s:%d", cfg.LocalPeer.Address, cfg.LocalPeer.Port)
// Begin the Raft configuration
rc := raft.DefaultConfig()
rc.ProtocolVersion = 3
rc.LocalID = raft.ServerID(cfg.LocalPeer.ID)
//rl := raftLog(cfg.LocalPeer.ID + " ")
//rc.LogOutput = &rl
rc.LogOutput = ioutil.Discard
rc.LocalID = raft.ServerID(cfg.ID)
rc.Logger = hcl
// Initialize communication
address, err := net.ResolveTCPAddr("tcp", localAddress)
if err != nil {
return err
}
// Create transport
transport, err := raft.NewTCPTransport(localAddress, address, 3, 10*time.Second, os.Stdout)
if err != nil {
return err
pool := 3
timeout := 10 * time.Second
var transport *raft.NetworkTransport
if cfg.TLS != nil {
log.Info("transport: tls")
s, err := tls.NewStream(cfg)
if err != nil {
return err
}
transport = raft.NewNetworkTransportWithLogger(s, pool, timeout, hcl)
} else {
log.Info("transport: tcp")
address, err := net.ResolveTCPAddr("tcp", cfg.Address)
if err != nil {
return err
}
transport, err = raft.NewTCPTransportWithLogger(cfg.Address, address, pool, timeout, hcl)
if err != nil {
return err
}
}
// Create Raft structures
//snapshots := raft.NewInmemSnapshotStore()
snapshots, err := raft.NewFileSnapshotStore(cfg.RaftDir, retainSnapshotCount, os.Stderr)
snapshots, err := raft.NewFileSnapshotStoreWithLogger(cfg.Dir, retainSnapshotCount, hcl)
if err != nil {
return fmt.Errorf("file snapshot store: %s", err)
}
store := filepath.Join(cfg.RaftDir, "store.db")
store := filepath.Join(cfg.Dir, "store.db")
//logStore := raft.NewInmemStore()
bootstrap := true
@ -67,7 +76,7 @@ func (c *Cluster) StartRaftCluster(log logr.Logger, cfg *config.Config) error {
bootstrap = false
}
stableStore, err := raftboltdb.NewBoltStore(store)
stableStore, err := raftbbolt.NewStore(store)
if err != nil {
return err
}
@ -78,33 +87,14 @@ func (c *Cluster) StartRaftCluster(log logr.Logger, cfg *config.Config) error {
// Add Local Peer
configuration.Servers = append(configuration.Servers, raft.Server{
Suffrage: raft.Voter,
ID: raft.ServerID(cfg.LocalPeer.ID),
Address: raft.ServerAddress(localAddress),
ID: raft.ServerID(cfg.ID),
Address: raft.ServerAddress(cfg.Address),
})
// If we want to start a node as leader then we will not add any remote peers, this will leave this as a cluster of one
// The remotePeers will add themselves to the cluster as they're added
//if !cfg.Leader {
// for _, p := range cfg.RemotePeers {
// peerAddress := fmt.Sprintf("%s:%d", p.Address, p.Port)
// if localAddress != peerAddress {
// configuration.Servers = append(configuration.Servers, raft.Server{
// ID: raft.ServerID(p.ID),
// Address: raft.ServerAddress(peerAddress)})
// }
// }
// log.Info("This node will attempt to start as Follower")
//} else {
// log.Info("This node will attempt to start as Leader")
//}
for _, p := range cfg.RemotePeers {
peerAddress := fmt.Sprintf("%s:%d", p.Address, p.Port)
if localAddress != peerAddress {
for _, p := range cfg.Peers {
if cfg.Address != p.Address {
configuration.Servers = append(configuration.Servers, raft.Server{
Suffrage: raft.Voter,
ID: raft.ServerID(p.ID),
Address: raft.ServerAddress(peerAddress)})
ID: raft.ServerID(p.ID),
Address: raft.ServerAddress(p.Address)})
}
}
@ -113,11 +103,6 @@ func (c *Cluster) StartRaftCluster(log logr.Logger, cfg *config.Config) error {
return err
}
}
//else {
// if err := raft.RecoverCluster(rc, c.stateMachine, logStore, stableStore, snapshots, transport, configuration); err != nil {
// return err
// }
//}
// Create RAFT instance
c.r, err = raft.NewRaft(rc, (*fsm)(c.kv), logStore, stableStore, snapshots, transport)
@ -126,7 +111,7 @@ func (c *Cluster) StartRaftCluster(log logr.Logger, cfg *config.Config) error {
}
c.kv.r = c.r
ticker := time.NewTicker(time.Second)
t := time.NewTicker(time.Second)
time.Sleep(time.Second * 3)
ctx, cancel := context.WithCancel(context.Background())
@ -134,7 +119,7 @@ func (c *Cluster) StartRaftCluster(log logr.Logger, cfg *config.Config) error {
leading := false
following := false
for {
if !leading && localAddress == string(c.r.Leader()) {
if !leading && cfg.Address == string(c.r.Leader()) {
leading = true
go c.leader(ctx)
} else if !following && !leading {
@ -152,9 +137,6 @@ func (c *Cluster) StartRaftCluster(log logr.Logger, cfg *config.Config) error {
}
if !leading {
leading = true
//cf := c.r.GetConfiguration()
//cf.Error()
//log.Info(fmt.Sprintf("[%s] %#v", cfg.LocalPeer.ID, cf.Configuration()))
go c.leader(ctx)
}
} else if c.r.State() == raft.Follower {
@ -165,20 +147,23 @@ func (c *Cluster) StartRaftCluster(log logr.Logger, cfg *config.Config) error {
}
if !following {
following = true
//cf := c.r.GetConfiguration()
//cf.Error()
//log.Info(fmt.Sprintf("[%s] %#v", cfg.LocalPeer.ID, cf.Configuration()))
go c.follower(ctx)
}
}
case <-ticker.C:
case <-c.stepdown:
if leading {
cancel()
c.r.LeadershipTransfer().Error()
}
case <-c.stop:
t.Stop()
cancel()
if leading {
c.r.LeadershipTransfer().Error()
}
close(c.done)
return nil
case <-t.C:
}
}
return nil
@ -187,7 +172,6 @@ func (c *Cluster) StartRaftCluster(log logr.Logger, cfg *config.Config) error {
func (c *Cluster) leader(ctx context.Context) {
defer func() {
handleCrash()
//c.Callbacks.OnStoppedLeading()
}()
c.Callbacks.Leader(ctx, c.kv)
}
@ -195,7 +179,6 @@ func (c *Cluster) leader(ctx context.Context) {
func (c *Cluster) follower(ctx context.Context) {
defer func() {
handleCrash()
//c.Callbacks.OnStoppedLeading()
}()
c.Callbacks.Follower(ctx, c.kv)
}
@ -233,6 +216,10 @@ func logPanic(r interface{}) {
}
}
func (c *Cluster) Stepdown() {
c.stepdown <- struct{}{}
}
func (c *Cluster) Stop() {
close(c.stop)
<-c.done

89
pkg/raft/tls/stream.go Normal file
View file

@ -0,0 +1,89 @@
package tls
import (
"crypto/tls"
"crypto/x509"
"io/ioutil"
"net"
"os"
"time"
"git.giftfish.de/ston1th/vipman/pkg/config"
"github.com/hashicorp/raft"
)
var (
ciphers = []uint16{
tls.TLS_CHACHA20_POLY1305_SHA256,
}
curves = []tls.CurveID{
tls.X25519,
}
)
func loadCertPool(file string) (p *x509.CertPool, err error) {
if file == "" {
return
}
f, err := os.Open(file)
if err != nil {
return
}
defer f.Close()
buf, err := ioutil.ReadAll(f)
if err != nil {
return
}
certs, err := x509.ParseCertificates(buf)
if err != nil {
return
}
p = x509.NewCertPool()
for _, c := range certs {
p.AddCert(c)
}
return
}
type stream struct {
net.Listener
insecure bool
ca *x509.CertPool
}
func NewStream(cfg *config.Raft) (s *stream, err error) {
cert, err := tls.LoadX509KeyPair(cfg.TLS.Cert, cfg.TLS.Key)
if err != nil {
return
}
ca, err := loadCertPool(cfg.TLS.CA)
if err != nil {
return
}
l, err := tls.Listen("tcp", cfg.Address, &tls.Config{
MinVersion: tls.VersionTLS13,
CipherSuites: ciphers,
CurvePreferences: curves,
PreferServerCipherSuites: true,
Certificates: []tls.Certificate{cert},
})
if err != nil {
return
}
s = &stream{l, cfg.TLS.Insecure, ca}
return
}
func (s *stream) Dial(address raft.ServerAddress, timeout time.Duration) (net.Conn, error) {
return tls.DialWithDialer(
&net.Dialer{Timeout: timeout},
"tcp", string(address),
&tls.Config{
MinVersion: tls.VersionTLS13,
CipherSuites: ciphers,
CurvePreferences: curves,
InsecureSkipVerify: s.insecure,
RootCAs: s.ca,
},
)
}

View file

@ -5,6 +5,6 @@ package vip
import "fmt"
// ARPSendGratuitous is only supported on Linux, so return an error
func ARPSendGratuitous(address, ifaceName string) error {
func ARPSendGratuitous(cidr, iface string) error {
return fmt.Errorf("Unsupported on this OS")
}

View file

@ -11,8 +11,6 @@ import (
"net"
"syscall"
"unsafe"
log "github.com/sirupsen/logrus"
)
const (
@ -156,21 +154,20 @@ func sendARP(iface *net.Interface, m *arpMessage) error {
}
// ARPSendGratuitous sends a gratuitous ARP message via the specified interface.
func ARPSendGratuitous(address, ifaceName string) error {
iface, err := net.InterfaceByName(ifaceName)
func ARPSendGratuitous(cidr, iface string) error {
i, err := net.InterfaceByName(iface)
if err != nil {
return fmt.Errorf("failed to get interface %q: %v", ifaceName, err)
return fmt.Errorf("failed to get interface %q: %v", iface, err)
}
ip, _, err := net.ParseCIDR(cidr)
if err != nil {
return fmt.Errorf("failed to parse cidr: %s", cidr)
}
ip := net.ParseIP(address)
if ip == nil {
return fmt.Errorf("failed to parse address %s", ip)
}
log.Infof("Broadcasting ARP update for %s (%s) via %s", address, iface.HardwareAddr, iface.Name)
m, err := gratuitousARP(ip, iface.HardwareAddr)
//log.Infof("Broadcasting ARP update for %s (%s) via %s", address, iface.HardwareAddr, iface.Name)
m, err := gratuitousARP(ip, i.HardwareAddr)
if err != nil {
return err
}
return sendARP(iface, m)
return sendARP(i, m)
}

View file

@ -1,142 +1,97 @@
package vip
import (
"sync"
"github.com/pkg/errors"
"errors"
"github.com/vishvananda/netlink"
"net"
)
const (
defaultValidLft = 60
)
var ErrNoDefaultInterface = errors.New("no default interface found")
// Network is an interface that enable managing operations for a given IP
type Network interface {
AddIP() error
DeleteIP() error
IsSet() (bool, error)
IP() string
SetIP(ip string) error
Interface() string
}
// network - This allows network configuration
type network struct {
mu sync.Mutex
address *netlink.Addr
link netlink.Link
isDNS bool
}
// NewConfig will attempt to provide an interface to the kernel network configuration
func NewConfig(address string, iface string) (Network, error) {
result := &network{}
link, err := netlink.LinkByName(iface)
func DefaultInterface() (iface string, err error) {
routes, err := netlink.RouteGet(net.IPv4bcast)
if err != nil {
return result, errors.Wrapf(err, "could not get link for interface '%s'", iface)
}
result.link = link
if IsIP(address) {
result.address, err = netlink.ParseAddr(address + "/32")
if err != nil {
return result, errors.Wrapf(err, "could not parse address '%s'", address)
}
return result, nil
}
// try to resolve the address
ip, err := lookupHost(address)
result.isDNS = err == nil
if err != nil {
return nil, err
}
// we're able to resolve store this as the initial IP
if result.address, err = netlink.ParseAddr(ip + "/32"); err != nil {
return result, err
}
// set ValidLft so that the VIP expires if the DNS entry is updated, otherwise it'll be refreshed by the DNS prober
result.address.ValidLft = defaultValidLft
return result, err
}
//AddIP - Add an IP address to the interface
func (configurator *network) AddIP() error {
if err := netlink.AddrReplace(configurator.link, configurator.address); err != nil {
return errors.Wrap(err, "could not add ip")
}
return nil
}
//DeleteIP - Remove an IP address from the interface
func (configurator *network) DeleteIP() error {
result, err := configurator.IsSet()
if err != nil {
return errors.Wrap(err, "ip check in DeleteIP failed")
}
// Nothing to delete
if !result {
return nil
}
if err = netlink.AddrDel(configurator.link, configurator.address); err != nil {
return errors.Wrap(err, "could not delete ip")
}
return nil
}
// IsSet - Check to see if VIP is set
func (configurator *network) IsSet() (result bool, err error) {
var addresses []netlink.Addr
addresses, err = netlink.AddrList(configurator.link, 0)
if err != nil {
err = errors.Wrap(err, "could not list addresses")
return
}
for _, address := range addresses {
if address.Equal(*configurator.address) {
return true, nil
if len(routes) >= 1 {
link, e := netlink.LinkByIndex(routes[0].LinkIndex)
if e != nil {
err = e
return
}
return link.Attrs().Name, nil
}
return false, nil
err = ErrNoDefaultInterface
return
}
// SetIP updates the IP that is used
func (configurator *network) SetIP(ip string) error {
configurator.mu.Lock()
defer configurator.mu.Unlock()
type Network struct {
iface string
label string
link netlink.Link
}
addr, err := netlink.ParseAddr(ip + "/32")
func NewNetwork(iface string) (n *Network, err error) {
return NewNetworkWithLabel(iface, "")
}
func NewNetworkWithLabel(iface, label string) (n *Network, err error) {
if iface == "" {
iface, err = DefaultInterface()
if err != nil {
return
}
}
link, err := netlink.LinkByName(iface)
if err != nil {
return
}
n = &Network{iface, label, link}
return
}
func (n *Network) Interface() string {
return n.iface
}
func (n *Network) HasIP(a *netlink.Addr) bool {
addrs, err := netlink.AddrList(n.link, 0)
if err != nil {
return false
}
for _, addr := range addrs {
if addr.Equal(*a) {
return true
}
}
return false
}
func (n *Network) AddIP(cidr string) error {
addr, err := netlink.ParseAddr(cidr)
if err != nil {
return err
}
if configurator.address != nil && configurator.isDNS {
addr.ValidLft = defaultValidLft
if n.HasIP(addr) {
return nil
}
configurator.address = addr
return nil
if n.label != "" {
addr.Label = n.iface + ":" + n.label
}
err = netlink.AddrReplace(n.link, addr)
if err != nil {
return err
}
return ARPSendGratuitous(cidr, n.iface)
}
// IP - return the IP Address
func (configurator *network) IP() string {
configurator.mu.Lock()
defer configurator.mu.Unlock()
return configurator.address.IP.String()
}
// Interface - return the Interface name
func (configurator *network) Interface() string {
return configurator.link.Attrs().Name
func (n *Network) DeleteIP(cidr string) error {
addr, err := netlink.ParseAddr(cidr)
if err != nil {
return err
}
if !n.HasIP(addr) {
return nil
}
return netlink.AddrDel(n.link, addr)
}