fix(cluster): keepalived Split-Brain + Boot-Race behoben — v1.2.101
Ursache der „WireGuard reißt immer wieder ab"-Abrisse war NICHT die UniFi, sondern keepalived-Flapping im HA-Cluster: die VIP 89.163.205.100 (an der die UniFi-Site-to-Site hängt) wanderte bei ~17 VRRP-Wahlen/Tag zwischen utm-1/utm-2 → Tunnel-Abriss bei jeder Wahl. Drei Bugs: 1) Firewall ließ VRRP (IP-Proto 112) zwischen den Cluster-Peers NICHT zu (policy drop). Adverts überlebten nur via conntrack-Reverse-Matching → bei conntrack-Ablauf gedroppt → Peer promotet sich → Split-Brain. Fix: ruleset.nft.tpl erlaubt `ip/ip6 ... vrrp saddr @peer_ipv4/6`; firewall.go nimmt zusätzlich hb_src_ip/hb_peer_ip aus cluster_settings ins Peer-Set (deckt den Heartbeat-Pfad 169.254.0.x ab). 2) Kein nopreempt → erholter Node riss die VIP sofort zurück (Flap-Back); aggressiver gw-Check (fall 2 → 10s-Blip = Failover). Fix: keepalived.conf.tpl mit `nopreempt` in VI_1+VI_HB, chk_gateway fall 2→5; keepalived.go setzt State immer BACKUP (nopreempt wirkt nur in BACKUP), Priorität 200/100 aus pg_role bleibt → deckt sich mit „manuelles Promote". 3) keepalived-Boot-Race: Unit startete vor vlan500 (nur After=network-online) → „interface vlan500 doesn't exist" → permanenter CONFIG-Crash ohne Recovery (keepalived nach Reboot tot). Fix: postinst legt Drop-in mit After=/Wants=edgeguard-interfaces.service + Restart=on-failure an. Neuer Test internal/keepalived/keepalived_test.go (nopreempt/BACKUP/fall). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -16,7 +16,7 @@ vrrp_script chk_gateway {
|
||||
script "/usr/lib/edgeguard/keepalived-gw-check.sh {{ .GWCheckIP }}"
|
||||
interval 5
|
||||
weight -110
|
||||
fall 2
|
||||
fall 5
|
||||
rise 2
|
||||
}
|
||||
{{ end }}
|
||||
@@ -34,6 +34,7 @@ vrrp_instance VI_1 {
|
||||
virtual_router_id {{ .RouterID }}
|
||||
priority {{ .Priority }}
|
||||
advert_int 1
|
||||
nopreempt
|
||||
{{ if .SrcIP }} unicast_src_ip {{ .SrcIP }}
|
||||
unicast_peer {
|
||||
{{ .PeerIP }}
|
||||
@@ -60,6 +61,7 @@ vrrp_instance VI_HB {
|
||||
virtual_router_id {{ .HBRouterID }}
|
||||
priority {{ .Priority }}
|
||||
advert_int 1
|
||||
nopreempt
|
||||
{{ if .HBSrcIP }} unicast_src_ip {{ .HBSrcIP }}
|
||||
unicast_peer {
|
||||
{{ .HBPeerIP }}
|
||||
|
||||
@@ -167,17 +167,19 @@ func (g *generator) buildView(cs *models.ClusterSettings, vips []VIPEntry, local
|
||||
v.HBRouterID = 52
|
||||
}
|
||||
|
||||
// pg_role=standby ist das härtere Signal — ein Standby-Node ist niemals
|
||||
// MASTER, auch wenn role='primary' noch aus dem Join-Prozess stammt.
|
||||
// Reihenfolge: standby → BACKUP; sonst primary-Check.
|
||||
// State IMMER BACKUP: das Template setzt `nopreempt`, und nopreempt wirkt
|
||||
// in keepalived NUR, wenn die Instanz im BACKUP-Zustand startet (bei state
|
||||
// MASTER wird nopreempt ignoriert). Die Priorität entscheidet weiterhin die
|
||||
// Initial-Election (primary=200 gewinnt), aber ein erholter Node reißt die
|
||||
// VIP NICHT mehr zurück → kein Flap-Back / Split-Brain. Deckt sich mit der
|
||||
// "kein Auto-Promote"-Philosophie: Promotion bleibt manuell.
|
||||
// pg_role=standby ist das härtere Signal (Standby ist nie bevorzugter Node).
|
||||
v.State = "BACKUP"
|
||||
if local.PGRole == "standby" {
|
||||
v.State = "BACKUP"
|
||||
v.Priority = 100
|
||||
} else if local.PGRole == "primary" || local.Role == "primary" {
|
||||
v.State = "MASTER"
|
||||
v.Priority = 200
|
||||
} else {
|
||||
v.State = "BACKUP"
|
||||
v.Priority = 100
|
||||
}
|
||||
|
||||
|
||||
79
internal/keepalived/keepalived_test.go
Normal file
79
internal/keepalived/keepalived_test.go
Normal file
@@ -0,0 +1,79 @@
|
||||
package keepalived
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.netcell-it.de/projekte/edgeguard-native/internal/models"
|
||||
)
|
||||
|
||||
func render(t *testing.T, v View) string {
|
||||
t.Helper()
|
||||
var buf bytes.Buffer
|
||||
if err := tpl.Execute(&buf, v); err != nil {
|
||||
t.Fatalf("template execute: %v", err)
|
||||
}
|
||||
return buf.String()
|
||||
}
|
||||
|
||||
// Split-Brain-Schutz: jede vrrp_instance MUSS `nopreempt` tragen, sonst reißt
|
||||
// ein erholter Node die VIP zurück → Flapping. nopreempt wirkt nur bei state
|
||||
// BACKUP — also muss auch der bevorzugte Node BACKUP starten.
|
||||
func testView() View {
|
||||
return View{
|
||||
State: "BACKUP", Interface: "eth0", RouterID: 51, Priority: 200,
|
||||
SrcIP: "89.163.205.6", PeerIP: "89.163.205.8", AuthPass: "edgeguard",
|
||||
VIPs: []VIPEntry{{Address: "89.163.205.100", Prefix: 24, Device: "eth0"}},
|
||||
HBInterface: "ens19", HBSrcIP: "169.254.0.1", HBPeerIP: "169.254.0.2", HBRouterID: 52,
|
||||
GWCheckIP: "89.163.205.1",
|
||||
}
|
||||
}
|
||||
|
||||
func TestTemplateNopreemptOnBothInstances(t *testing.T) {
|
||||
out := render(t, testView())
|
||||
if n := strings.Count(out, "nopreempt"); n != 2 {
|
||||
t.Fatalf("erwarte nopreempt in VI_1 UND VI_HB (2×), gefunden: %d\n%s", n, out)
|
||||
}
|
||||
if strings.Contains(out, "state MASTER") {
|
||||
t.Fatalf("kein Node darf state MASTER starten (nopreempt würde ignoriert):\n%s", out)
|
||||
}
|
||||
if c := strings.Count(out, "state BACKUP"); c != 2 {
|
||||
t.Fatalf("erwarte state BACKUP in beiden Instanzen, gefunden: %d", c)
|
||||
}
|
||||
}
|
||||
|
||||
// gw-Check darf nicht zu zucken (fall 5, nicht fall 2) — ein kurzer Upstream-
|
||||
// Blip soll keinen Failover erzwingen.
|
||||
func TestTemplateGatewayCheckNotTwitchy(t *testing.T) {
|
||||
out := render(t, testView())
|
||||
if !strings.Contains(out, "fall 5") {
|
||||
t.Fatalf("chk_gateway sollte fall 5 nutzen:\n%s", out)
|
||||
}
|
||||
}
|
||||
|
||||
// buildView: State immer BACKUP, Priorität aus pg_role.
|
||||
func TestBuildViewStateAlwaysBackup(t *testing.T) {
|
||||
g := &generator{localID: "n1"}
|
||||
cs := &models.ClusterSettings{VRRPRouterID: 51}
|
||||
pub := "89.163.205.6"
|
||||
cases := []struct {
|
||||
pgRole, role string
|
||||
wantPrio int
|
||||
}{
|
||||
{"primary", "primary", 200},
|
||||
{"standby", "primary", 100},
|
||||
{"", "primary", 200},
|
||||
{"", "", 100},
|
||||
}
|
||||
for _, c := range cases {
|
||||
local := &models.HANode{ID: "n1", PGRole: c.pgRole, Role: c.role, PublicIP: &pub}
|
||||
v := g.buildView(cs, nil, local, nil)
|
||||
if v.State != "BACKUP" {
|
||||
t.Errorf("pg_role=%q role=%q: State=%q, erwarte immer BACKUP (nopreempt)", c.pgRole, c.role, v.State)
|
||||
}
|
||||
if v.Priority != c.wantPrio {
|
||||
t.Errorf("pg_role=%q role=%q: Priority=%d, erwarte %d", c.pgRole, c.role, v.Priority, c.wantPrio)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user