feat: HA-Cluster v1.2.x — Split-Brain, TOTP, Enterprise-FW, Drift-Fix, VIP-Recovery
- keepalived: pg_role='standby' hat Vorrang vor role für BACKUP-Bestimmung - keepalived-master.sh: gecrasht Dienste beim MASTER-Übergang starten (nicht nur reload) - confighash: ip_addresses per Interface-Name hashen statt per FK (Cross-Node-Drift-Fix) - TOTP/2FA: RFC 6238 — Setup-Flow, QR-Code, Admin-Disable; two-step Login - Firewall-UI: Enterprise-Design — auto-Beschreibung, icon-only Actions, zero-hit Indikator - fe80-Filter: Link-local IPv6 aus NTP/DNS Listen-Dropdowns entfernen - VIP-Dashboard, Dual-Path VRRP, GW-Tracking (Migrations 0033/0034) - Forward Proxy + DNS erweiterte Einstellungen (Migrations 0031/0032) - unbound-control: edgeguard in unbound-Gruppe via postinst Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Debian
319
internal/handlers/cluster_viptest.go
Normal file
319
internal/handlers/cluster_viptest.go
Normal file
@@ -0,0 +1,319 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os/exec"
|
||||
"strings"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"git.netcell-it.de/projekte/edgeguard-native/internal/handlers/response"
|
||||
"git.netcell-it.de/projekte/edgeguard-native/internal/models"
|
||||
)
|
||||
|
||||
// vipInfo enthält die für einen VIP-Schwenk notwendigen Daten.
|
||||
type vipInfo struct {
|
||||
ID int64 `json:"id"`
|
||||
Address string `json:"address"`
|
||||
Prefix int `json:"prefix"`
|
||||
Device string `json:"device"`
|
||||
}
|
||||
|
||||
// VIPStatusEntry kombiniert einen VIP mit den Nodes die ihn gerade halten.
|
||||
type VIPStatusEntry struct {
|
||||
VIP vipInfo `json:"vip"`
|
||||
ActiveOn []string `json:"active_on"` // FQDNs der Nodes mit diesem VIP
|
||||
}
|
||||
|
||||
// AgentActiveIPs gibt alle aktiven IPv4-Adressen dieses Nodes zurück.
|
||||
// Wird vom Primary genutzt um zu prüfen welcher Node welchen VIP hält.
|
||||
func (h *ClusterHandler) AgentActiveIPs(c *gin.Context) {
|
||||
ips, err := localActiveIPs()
|
||||
if err != nil {
|
||||
response.Internal(c, err)
|
||||
return
|
||||
}
|
||||
response.OK(c, gin.H{"ips": ips})
|
||||
}
|
||||
|
||||
// vipCmdRequest ist der Body für den AgentVIPCmd-Endpoint.
|
||||
type vipCmdRequest struct {
|
||||
Action string `json:"action"` // "add" | "del"
|
||||
Address string `json:"address"` // z.B. "10.0.5.1"
|
||||
Prefix int `json:"prefix"` // z.B. 24
|
||||
Device string `json:"device"` // z.B. "vlan100"
|
||||
}
|
||||
|
||||
// AgentVIPCmd führt `ip addr add/del` auf diesem Node aus.
|
||||
// Wird vom Primary via mTLS für VIP-Schwenk-Tests aufgerufen.
|
||||
func (h *ClusterHandler) AgentVIPCmd(c *gin.Context) {
|
||||
var req vipCmdRequest
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
response.BadRequest(c, err)
|
||||
return
|
||||
}
|
||||
if req.Action != "add" && req.Action != "del" {
|
||||
response.BadRequest(c, simpleError("action must be 'add' or 'del'"))
|
||||
return
|
||||
}
|
||||
if req.Address == "" || req.Device == "" || req.Prefix <= 0 || req.Prefix > 128 {
|
||||
response.BadRequest(c, simpleError("address, device, prefix required"))
|
||||
return
|
||||
}
|
||||
if err := runVIPCmd(req.Action, req.Address, req.Prefix, req.Device); err != nil {
|
||||
slog.Warn("cluster: agent vip-cmd failed",
|
||||
"action", req.Action, "addr", req.Address, "dev", req.Device, "error", err)
|
||||
response.Internal(c, err)
|
||||
return
|
||||
}
|
||||
slog.Info("cluster: agent vip-cmd ok",
|
||||
"action", req.Action, "addr", req.Address, "prefix", req.Prefix,
|
||||
"dev", req.Device, "caller", c.ClientIP())
|
||||
response.OK(c, gin.H{"ok": true})
|
||||
}
|
||||
|
||||
// VIPStatus liest alle VIPs (is_vip=true) aus der DB und fragt alle Nodes
|
||||
// welche davon sie gerade aktiv haben. Nur sinnvoll im Cluster-Modus.
|
||||
func (h *ClusterHandler) VIPStatus(c *gin.Context) {
|
||||
vips, err := loadVIPs(c.Request.Context(), h.Store.Pool)
|
||||
if err != nil {
|
||||
response.Internal(c, err)
|
||||
return
|
||||
}
|
||||
nodeIPs := h.collectActiveIPs(c.Request.Context())
|
||||
result := make([]VIPStatusEntry, 0, len(vips))
|
||||
for _, v := range vips {
|
||||
entry := VIPStatusEntry{VIP: v}
|
||||
for fqdn, ips := range nodeIPs {
|
||||
for _, ip := range ips {
|
||||
if ip == v.Address {
|
||||
entry.ActiveOn = append(entry.ActiveOn, fqdn)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
result = append(result, entry)
|
||||
}
|
||||
response.OK(c, gin.H{"vips": result})
|
||||
}
|
||||
|
||||
// vipTestRequest steuert einen VIP-Schwenk.
|
||||
type vipTestRequest struct {
|
||||
IPAddressID int64 `json:"ip_address_id"`
|
||||
Action string `json:"action"` // "to_secondary" | "restore"
|
||||
}
|
||||
|
||||
// vipTestStep beschreibt einen Schritt des Schwenk-Prozesses.
|
||||
type vipTestStep struct {
|
||||
Step string `json:"step"`
|
||||
OK bool `json:"ok"`
|
||||
Message string `json:"message,omitempty"`
|
||||
}
|
||||
|
||||
// VIPTest schwenkt einen VIP vom Primary auf den Secondary ("to_secondary")
|
||||
// oder zurück ("restore"). Nur vom Primary aufzurufen.
|
||||
func (h *ClusterHandler) VIPTest(c *gin.Context) {
|
||||
var req vipTestRequest
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
response.BadRequest(c, err)
|
||||
return
|
||||
}
|
||||
if req.Action != "to_secondary" && req.Action != "restore" {
|
||||
response.BadRequest(c, simpleError("action must be 'to_secondary' or 'restore'"))
|
||||
return
|
||||
}
|
||||
|
||||
vips, err := loadVIPs(c.Request.Context(), h.Store.Pool)
|
||||
if err != nil {
|
||||
response.Internal(c, err)
|
||||
return
|
||||
}
|
||||
var target *vipInfo
|
||||
for i := range vips {
|
||||
if vips[i].ID == req.IPAddressID {
|
||||
target = &vips[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
if target == nil {
|
||||
response.NotFound(c, simpleError("VIP not found or not marked as VIP"))
|
||||
return
|
||||
}
|
||||
|
||||
all, err := h.Store.List(c.Request.Context())
|
||||
if err != nil {
|
||||
response.Internal(c, err)
|
||||
return
|
||||
}
|
||||
var peer *models.HANode
|
||||
for i := range all {
|
||||
if all[i].ID != h.LocalID {
|
||||
peer = &all[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
if peer == nil {
|
||||
response.BadRequest(c, simpleError("kein Secondary-Peer gefunden"))
|
||||
return
|
||||
}
|
||||
|
||||
var steps []vipTestStep
|
||||
addrPfx := fmt.Sprintf("%s/%d", target.Address, target.Prefix)
|
||||
|
||||
if req.Action == "to_secondary" {
|
||||
// 1. VIP auf Secondary via mTLS hinzufügen
|
||||
steps = append(steps, h.peerVIPCmd(c.Request.Context(), *peer, target, "add",
|
||||
fmt.Sprintf("add %s dev %s auf %s", addrPfx, target.Device, peer.FQDN)))
|
||||
// 2. VIP vom Primary entfernen (nur wenn Secondary-Add erfolgreich)
|
||||
if steps[0].OK {
|
||||
steps = append(steps, localVIPStep(target, "del",
|
||||
fmt.Sprintf("del %s dev %s lokal", addrPfx, target.Device)))
|
||||
}
|
||||
} else {
|
||||
// 1. VIP auf Primary zurückholen
|
||||
steps = append(steps, localVIPStep(target, "add",
|
||||
fmt.Sprintf("add %s dev %s lokal", addrPfx, target.Device)))
|
||||
// 2. VIP auf Secondary entfernen
|
||||
steps = append(steps, h.peerVIPCmd(c.Request.Context(), *peer, target, "del",
|
||||
fmt.Sprintf("del %s dev %s auf %s", addrPfx, target.Device, peer.FQDN)))
|
||||
}
|
||||
|
||||
slog.Info("cluster: vip-test", "action", req.Action, "vip", target.Address,
|
||||
"dev", target.Device, "peer", peer.FQDN, "actor", actorOf(c))
|
||||
response.OK(c, gin.H{"steps": steps})
|
||||
}
|
||||
|
||||
// ── Hilfsfunktionen ───────────────────────────────────────────────────────
|
||||
|
||||
func loadVIPs(ctx context.Context, pool *pgxpool.Pool) ([]vipInfo, error) {
|
||||
rows, err := pool.Query(ctx, `
|
||||
SELECT ia.id, ia.address, ia.prefix, ni.name
|
||||
FROM ip_addresses ia
|
||||
JOIN network_interfaces ni ON ni.id = ia.interface_id
|
||||
WHERE ia.is_vip = true AND ia.active = true
|
||||
ORDER BY ni.name, ia.address`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []vipInfo
|
||||
for rows.Next() {
|
||||
var v vipInfo
|
||||
if err := rows.Scan(&v.ID, &v.Address, &v.Prefix, &v.Device); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, v)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// collectActiveIPs fragt alle Cluster-Nodes (lokal + Peers via mTLS) nach
|
||||
// ihren aktiven IPv4-Adressen und gibt eine Map[fqdn][]ip zurück.
|
||||
func (h *ClusterHandler) collectActiveIPs(ctx context.Context) map[string][]string {
|
||||
result := make(map[string][]string)
|
||||
if h.Store == nil {
|
||||
return result
|
||||
}
|
||||
all, err := h.Store.List(ctx)
|
||||
if err != nil {
|
||||
return result
|
||||
}
|
||||
// Lokaler Node
|
||||
if ips, err := localActiveIPs(); err == nil {
|
||||
for _, n := range all {
|
||||
if n.ID == h.LocalID {
|
||||
result[n.FQDN] = ips
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
// Peers via mTLS-Aggregator
|
||||
if h.Aggregator != nil {
|
||||
var peers []models.HANode
|
||||
for _, n := range all {
|
||||
if n.ID != h.LocalID {
|
||||
peers = append(peers, n)
|
||||
}
|
||||
}
|
||||
if len(peers) > 0 {
|
||||
peerResults := h.Aggregator.FanOut(ctx, peers, "/agent/cluster/active-ips", h.LocalID)
|
||||
for _, pr := range peerResults {
|
||||
if !pr.OK || len(pr.Data) == 0 {
|
||||
continue
|
||||
}
|
||||
var payload struct {
|
||||
IPs []string `json:"ips"`
|
||||
}
|
||||
if err := json.Unmarshal(pr.Data, &payload); err == nil {
|
||||
result[pr.FQDN] = payload.IPs
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// localActiveIPs liest alle aktiven IPv4-Adressen des lokalen Nodes via `ip`.
|
||||
func localActiveIPs() ([]string, error) {
|
||||
out, err := exec.Command("ip", "-4", "-o", "addr", "show").Output()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var ips []string
|
||||
for _, line := range strings.Split(string(out), "\n") {
|
||||
parts := strings.Fields(line)
|
||||
for i, p := range parts {
|
||||
if p == "inet" && i+1 < len(parts) {
|
||||
addr := strings.SplitN(parts[i+1], "/", 2)[0]
|
||||
ips = append(ips, addr)
|
||||
}
|
||||
}
|
||||
}
|
||||
return ips, nil
|
||||
}
|
||||
|
||||
// peerVIPCmd ruft AgentVIPCmd auf dem Peer via mTLS auf.
|
||||
func (h *ClusterHandler) peerVIPCmd(ctx context.Context, peer models.HANode, vip *vipInfo, action, stepLabel string) vipTestStep {
|
||||
step := vipTestStep{Step: stepLabel}
|
||||
if h.Aggregator == nil {
|
||||
step.Message = "aggregator nicht verfügbar"
|
||||
return step
|
||||
}
|
||||
body, _ := json.Marshal(vipCmdRequest{
|
||||
Action: action,
|
||||
Address: vip.Address,
|
||||
Prefix: vip.Prefix,
|
||||
Device: vip.Device,
|
||||
})
|
||||
res := h.Aggregator.PostPeerWithBody(ctx, peer, "/agent/cluster/vip-cmd", body)
|
||||
step.OK = res.OK
|
||||
if !res.OK {
|
||||
step.Message = res.Err
|
||||
}
|
||||
return step
|
||||
}
|
||||
|
||||
// localVIPStep führt ip addr add/del auf dem lokalen Node aus.
|
||||
func localVIPStep(vip *vipInfo, action, stepLabel string) vipTestStep {
|
||||
step := vipTestStep{Step: stepLabel}
|
||||
if err := runVIPCmd(action, vip.Address, vip.Prefix, vip.Device); err != nil {
|
||||
step.Message = err.Error()
|
||||
return step
|
||||
}
|
||||
step.OK = true
|
||||
return step
|
||||
}
|
||||
|
||||
// runVIPCmd führt `sudo /usr/lib/edgeguard/vip-cmd.sh {action} {addr/prefix} {dev}` aus.
|
||||
func runVIPCmd(action, address string, prefix int, device string) error {
|
||||
addrPfx := fmt.Sprintf("%s/%d", address, prefix)
|
||||
out, err := exec.Command("sudo", "-n", "/usr/lib/edgeguard/vip-cmd.sh", action, addrPfx, device).CombinedOutput()
|
||||
if err != nil {
|
||||
return fmt.Errorf("vip-cmd.sh %s %s %s: %s", action, addrPfx, device, strings.TrimSpace(string(out)))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
Reference in New Issue
Block a user