feat(cluster): PG Logical Replication + VIP/Keepalived + config_hash sync (v1.2.1–1.2.2)
- PG Logical Replication: edgeguard_shared PUBLICATION auf Primary, edgeguard_sub SUBSCRIPTION auf Secondary. Nur geteilte Config-Tabellen werden repliziert; node-eigene Daten (network_interfaces, ip_addresses, static_routes, cluster_settings, dns_settings, ntp_settings) bleiben lokal — OPNsense-Muster. - cluster-init-replication: Erstellt PUBLICATION, Rolle + pg_hba-Einträge (logical + replication), WAL-Level auf logical. - cluster-setup-standby: Erstellt SUBSCRIPTION (copy_data=true), pollt pg_subscription_rel bis alle Tabellen sync = 'r', rendert dann Configs. - promote: manueller Failover via pg_promote() + touch recovery.signal. - VIP/Keepalived: cluster_settings-Tabelle (vip_address, vip_interface, vrrp_router_id), /cluster/vip-settings API, Keepalived-Config-Generator mit VRRP + check_script + notify-Skripten in /usr/lib/edgeguard/scripts/. - config_hash sync: Secondary pusht alle 5 Min seinen Hash via mTLS an Primary (PushSelfToPrimary). Heartbeat schreibt nur LOCAL, daher ohne aktiven Push wäre Primary-Sicht des Secondary-Hash stale gewesen. - runSecondaryConfigRender: Goroutine auf Secondary rendert HAProxy+nftables neu wenn config_hash sich ändert (Logical-Replication-Nachzügler). - confighash: node-spezifische Tabellen aus hashSpec entfernt. - postinst: Keepalived-Skripte installieren, sudoers für keepalived. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Debian
154
cmd/edgeguard-ctl/promote.go
Normal file
154
cmd/edgeguard-ctl/promote.go
Normal file
@@ -0,0 +1,154 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.netcell-it.de/projekte/edgeguard-native/internal/cluster"
|
||||
"git.netcell-it.de/projekte/edgeguard-native/internal/database"
|
||||
"git.netcell-it.de/projekte/edgeguard-native/internal/keepalived"
|
||||
)
|
||||
|
||||
// cmdPromote promotes this node's PostgreSQL instance from Hot-Standby
|
||||
// to Primary. Manual failover — keine automatische Promotion, um Split-Brain
|
||||
// in 2-Node-Clustern ohne externen Quorum zu verhindern.
|
||||
//
|
||||
// Ablauf:
|
||||
// 1. Prüfen ob standby.signal vorhanden (wir sind wirklich Standby)
|
||||
// 2. pg_ctlcluster promote → PG wird Primary
|
||||
// 3. Warten bis pg_is_in_recovery() = false
|
||||
// 4. ha_nodes.pg_role auf 'primary' setzen
|
||||
// 5. KeyDB cluster:pg-primary-url auf lokal setzen
|
||||
// 6. keepalived.conf neu rendern (Primary bekommt Priorität 200)
|
||||
// 7. keepalived reload
|
||||
func cmdPromote(args []string) int {
|
||||
// 1. Standby-Signal prüfen
|
||||
signalPath := filepath.Join(pgDataDir, "standby.signal")
|
||||
if _, err := os.Stat(signalPath); os.IsNotExist(err) {
|
||||
fmt.Fprintf(os.Stderr,
|
||||
"promote: %s nicht gefunden — diese Node ist kein PG-Standby oder wurde bereits promoted.\n",
|
||||
signalPath)
|
||||
return 1
|
||||
}
|
||||
|
||||
fmt.Println("→ Promoting PostgreSQL zu Primary...")
|
||||
if out, err := exec.Command("pg_ctlcluster", pgVersion, pgCluster, "promote").
|
||||
CombinedOutput(); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "promote: pg_ctlcluster promote: %v\n%s\n", err, out)
|
||||
return 1
|
||||
}
|
||||
fmt.Println("✓ pg_ctlcluster promote gesendet")
|
||||
|
||||
// 2. Warten bis PG wirklich Primary ist (pg_is_in_recovery = false)
|
||||
fmt.Print("→ Warte auf PG Primary-Mode")
|
||||
deadline := time.Now().Add(60 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
out, err := psqlRun([]string{"-tA", "-c", "SELECT pg_is_in_recovery();"})
|
||||
if err == nil && strings.TrimSpace(string(out)) == "f" {
|
||||
break
|
||||
}
|
||||
fmt.Print(".")
|
||||
time.Sleep(2 * time.Second)
|
||||
}
|
||||
fmt.Println()
|
||||
// Nochmal prüfen
|
||||
out, err := psqlRun([]string{"-tA", "-c", "SELECT pg_is_in_recovery();"})
|
||||
if err != nil || strings.TrimSpace(string(out)) != "f" {
|
||||
fmt.Fprintln(os.Stderr, "promote: PG ist nach 60s noch in recovery — prüfe PG-Logs")
|
||||
return 1
|
||||
}
|
||||
fmt.Println("✓ PostgreSQL ist jetzt Primary")
|
||||
|
||||
// 3. ha_nodes.pg_role + role aktualisieren
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer cancel()
|
||||
|
||||
pool, err := database.Open(ctx, database.ConnStringFromEnv())
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, "promote: db connect:", err)
|
||||
fmt.Println(" → ha_nodes manuell updaten: UPDATE ha_nodes SET pg_role='primary', role='primary' WHERE id='<local-id>';")
|
||||
} else {
|
||||
defer pool.Close()
|
||||
localID, err := loadLocalID()
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, "promote: local node ID:", err)
|
||||
} else {
|
||||
_, err = pool.Exec(ctx, `UPDATE ha_nodes SET pg_role='primary', role='primary', status='online', updated_at=NOW() WHERE id=$1`, localID)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, "promote: update ha_nodes:", err)
|
||||
} else {
|
||||
fmt.Println("✓ ha_nodes.pg_role = 'primary' gesetzt")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 4. KeyDB cluster:pg-primary-url updaten
|
||||
if err := updateKeyDBPrimaryURL(); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "promote: KeyDB update: %v\n", err)
|
||||
fmt.Println(" → Manuell: redis-cli SET cluster:pg-primary-url 'postgres://edgeguard@/edgeguard'")
|
||||
} else {
|
||||
fmt.Println("✓ KeyDB cluster:pg-primary-url aktualisiert")
|
||||
}
|
||||
|
||||
// 5. Keepalived.conf neu rendern (Primary = Priorität 200)
|
||||
if pool != nil {
|
||||
localID, _ := loadLocalID()
|
||||
kg := keepalived.New(pool, localID)
|
||||
renderCtx, renderCancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer renderCancel()
|
||||
if err := kg.Render(renderCtx); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "promote: keepalived render: %v\n", err)
|
||||
fmt.Println(" → Manuell: edgeguard-ctl render-config --only=keepalived")
|
||||
} else {
|
||||
fmt.Println("✓ keepalived.conf neu gerendert (Priority 200)")
|
||||
}
|
||||
}
|
||||
|
||||
fmt.Println()
|
||||
fmt.Println("✓ Promotion abgeschlossen. Diese Node ist jetzt der primäre EdgeGuard-Knoten.")
|
||||
fmt.Println()
|
||||
fmt.Println("Empfohlene Nachschritte:")
|
||||
fmt.Println(" 1) sudo systemctl restart edgeguard-api (falls noch nicht laufend)")
|
||||
fmt.Println(" 2) Alte Primary-Node nach Recovery als neuen Standby einrichten:")
|
||||
fmt.Println(" edgeguard-ctl cluster-setup-standby <diese-node-ip>")
|
||||
return 0
|
||||
}
|
||||
|
||||
// loadLocalID liest die Node-ID aus /var/lib/edgeguard/node.conf.
|
||||
func loadLocalID() (string, error) {
|
||||
c, err := cluster.LoadLocalConfig("")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if c.NodeID == "" {
|
||||
return "", fmt.Errorf("NODE_ID in node.conf ist leer")
|
||||
}
|
||||
return c.NodeID, nil
|
||||
}
|
||||
|
||||
// updateKeyDBPrimaryURL schreibt den lokalen PG-DSN als cluster:pg-primary-url
|
||||
// in KeyDB, damit alle Nodes im Cluster Writes an diese Node schicken.
|
||||
func updateKeyDBPrimaryURL() error {
|
||||
// edgeguard-api nutzt Unix-Socket-Auth, der DSN ist immer lokal.
|
||||
const localDSN = "postgres://edgeguard@/edgeguard?host=/var/run/postgresql"
|
||||
out, err := exec.Command("redis-cli",
|
||||
"-s", "/var/run/keydb/keydb.sock",
|
||||
"SET", "cluster:pg-primary-url", localDSN,
|
||||
).CombinedOutput()
|
||||
if err != nil {
|
||||
// Fallback: Standard-Port
|
||||
out2, err2 := exec.Command("redis-cli",
|
||||
"-p", "6379",
|
||||
"SET", "cluster:pg-primary-url", localDSN,
|
||||
).CombinedOutput()
|
||||
if err2 != nil {
|
||||
return fmt.Errorf("%v: %s / %v: %s", err, out, err2, out2)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
Reference in New Issue
Block a user