feat(cluster): PG Logical Replication + VIP/Keepalived + config_hash sync (v1.2.1–1.2.2)

- PG Logical Replication: edgeguard_shared PUBLICATION auf Primary,
  edgeguard_sub SUBSCRIPTION auf Secondary. Nur geteilte Config-Tabellen
  werden repliziert; node-eigene Daten (network_interfaces, ip_addresses,
  static_routes, cluster_settings, dns_settings, ntp_settings) bleiben
  lokal — OPNsense-Muster.
- cluster-init-replication: Erstellt PUBLICATION, Rolle + pg_hba-Einträge
  (logical + replication), WAL-Level auf logical.
- cluster-setup-standby: Erstellt SUBSCRIPTION (copy_data=true), pollt
  pg_subscription_rel bis alle Tabellen sync = 'r', rendert dann Configs.
- promote: manueller Failover via pg_promote() + touch recovery.signal.
- VIP/Keepalived: cluster_settings-Tabelle (vip_address, vip_interface,
  vrrp_router_id), /cluster/vip-settings API, Keepalived-Config-Generator
  mit VRRP + check_script + notify-Skripten in /usr/lib/edgeguard/scripts/.
- config_hash sync: Secondary pusht alle 5 Min seinen Hash via mTLS an
  Primary (PushSelfToPrimary). Heartbeat schreibt nur LOCAL, daher ohne
  aktiven Push wäre Primary-Sicht des Secondary-Hash stale gewesen.
- runSecondaryConfigRender: Goroutine auf Secondary rendert HAProxy+nftables
  neu wenn config_hash sich ändert (Logical-Replication-Nachzügler).
- confighash: node-spezifische Tabellen aus hashSpec entfernt.
- postinst: Keepalived-Skripte installieren, sudoers für keepalived.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Debian
2026-05-29 23:40:37 +02:00
parent c1a4ccff8f
commit 25c7cd0cb5
19 changed files with 1128 additions and 40 deletions

View File

@@ -35,10 +35,12 @@ import (
// hashTable beschreibt eine Tabelle die in den config-hash einfließt.
type hashTable struct {
Name string
Singleton bool // dns_settings, ntp_settings → eine row, id=1
ExtraExclude []string // Spalten die zusätzlich aus to_jsonb gefiltert werden
SkipUpdatedAt bool // setze true wenn updated_at semantisch relevant ist
Name string
Singleton bool // dns_settings, ntp_settings → eine row, id=1
ExtraExclude []string // Spalten die zusätzlich aus to_jsonb gefiltert werden
SkipUpdatedAt bool // setze true wenn updated_at semantisch relevant ist
MigrationDefault bool // Tabelle hat migrations-erzeugte Default-Rows (firewall_zones, ntp_pools…)
// → zählt nicht als "user hat config" bei der Empty-DB-Erkennung
}
// hashSpec ist die Reihenfolge-stabile Liste. NEUE Tabellen hier
@@ -49,15 +51,13 @@ var hashSpec = []hashTable{
{Name: "backends"},
{Name: "backend_servers"},
{Name: "routing_rules"},
{Name: "network_interfaces"},
{Name: "ip_addresses"},
{Name: "tls_certs", ExtraExclude: []string{"last_renewed_at", "last_error"}},
{Name: "firewall_zones"},
{Name: "firewall_zones", MigrationDefault: true},
{Name: "firewall_address_objects"},
{Name: "firewall_address_groups"},
{Name: "firewall_services"},
{Name: "firewall_service_groups"},
{Name: "firewall_services", MigrationDefault: true},
{Name: "firewall_service_groups", MigrationDefault: true},
{Name: "firewall_rules"},
{Name: "firewall_nat_rules"},
@@ -67,12 +67,12 @@ var hashSpec = []hashTable{
{Name: "dns_zones"},
{Name: "dns_records"},
{Name: "dns_settings", Singleton: true},
{Name: "ntp_pools"},
{Name: "ntp_settings", Singleton: true},
{Name: "ntp_pools", MigrationDefault: true},
{Name: "static_routes"},
// network_interfaces, ip_addresses, static_routes, dns_settings, ntp_settings
// sind node-spezifisch (jeder Node hat eigene IPs/Routes/Listen-Adressen)
// und fließen NICHT in den Drift-Hash ein.
}
// hashSQL rendert die SHA-Input-SQL für eine Tabelle.
@@ -100,22 +100,35 @@ func hashSQL(t hashTable) string {
// ComputeConfigHash gibt den 16-hex-char-Hash über alle Spec-Tabellen
// zurück. Fehlende Tabellen (transienter schema-flux) werden als
// leerer Per-Table-Hash behandelt — kein Abbruch.
//
// Gibt "" zurück wenn alle user-konfigurierbaren Tabellen leer sind
// (Singleton- und MigrationDefault-Tabellen zählen nicht als User-Config).
// Das verhindert False-Positive-Drift-Banner auf frisch gejointen Secondaries.
func ComputeConfigHash(ctx context.Context, pool *pgxpool.Pool) (string, error) {
if pool == nil {
return "", fmt.Errorf("nil pool")
}
h := sha256.New()
hasUserConfig := false
for _, t := range hashSpec {
var s string
if err := pool.QueryRow(ctx, hashSQL(t)).Scan(&s); err != nil {
// Migration fehlt o.ä. → leeren string nehmen, weiter.
s = ""
}
if s != "" && !t.Singleton && !t.MigrationDefault {
hasUserConfig = true
}
h.Write([]byte(t.Name))
h.Write([]byte{0})
h.Write([]byte(s))
h.Write([]byte{0})
}
if !hasUserConfig {
// Frisch gejoincter Secondary oder komplett leere DB →
// leerer String signalisiert "kein Drift prüfen" im Status-Handler.
return "", nil
}
return hex.EncodeToString(h.Sum(nil))[:16], nil
}

View File

@@ -0,0 +1,36 @@
-- +goose Up
-- +goose StatementBegin
-- pg_role: Rolle dieser Node in der PG-Replikation.
-- "standalone" = kein Streaming-Replication-Setup
-- "primary" = WAL-Sender, repliziert an Standby(s)
-- "standby" = Hot-Standby, liest WAL vom Primary
ALTER TABLE ha_nodes ADD COLUMN IF NOT EXISTS pg_role TEXT NOT NULL DEFAULT 'standalone';
-- cluster_settings: VIP + VRRP-Konfiguration (Singleton, id=1).
-- vip_address = die virtuelle IP-Adresse (z.B. "89.163.205.10")
-- vip_interface = Netzwerk-Interface (z.B. "eth0")
-- vip_auth_pass = VRRP-Authentication-Passwort (max. 8 Zeichen, Keepalived-Limit)
-- vrrp_router_id = VRRP Virtual Router ID (1255, muss im Subnetz eindeutig sein)
CREATE TABLE IF NOT EXISTS cluster_settings (
id INTEGER PRIMARY KEY DEFAULT 1,
vip_address TEXT,
vip_interface TEXT,
vip_auth_pass TEXT,
vrrp_router_id INTEGER NOT NULL DEFAULT 51,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
CONSTRAINT cluster_settings_singleton CHECK (id = 1)
);
INSERT INTO cluster_settings (id) VALUES (1) ON CONFLICT DO NOTHING;
-- +goose StatementEnd
-- +goose Down
-- +goose StatementBegin
DROP TABLE IF EXISTS cluster_settings;
ALTER TABLE ha_nodes DROP COLUMN IF EXISTS pg_role;
-- +goose StatementEnd

View File

@@ -0,0 +1,40 @@
global_defs {
router_id {{ .RouterID }}
script_user root
enable_script_security
vrrp_garp_interval 0
vrrp_gna_interval 0
}
vrrp_script chk_edgeguard {
script "/usr/lib/edgeguard/keepalived-check.sh"
interval 2
weight -50
fall 3
rise 2
}
vrrp_instance VI_1 {
state {{ .State }}
interface {{ .Interface }}
virtual_router_id {{ .RouterID }}
priority {{ .Priority }}
advert_int 1
{{ if .SrcIP }} unicast_src_ip {{ .SrcIP }}
unicast_peer {
{{ .PeerIP }}
}
{{ end }} authentication {
auth_type PASS
auth_pass {{ .AuthPass }}
}
virtual_ipaddress {
{{ .VIP }}
}
track_script {
chk_edgeguard
}
notify_master "/usr/lib/edgeguard/keepalived-master.sh"
notify_backup "/usr/lib/edgeguard/keepalived-backup.sh"
notify_fault "/usr/lib/edgeguard/keepalived-backup.sh"
}

View File

@@ -0,0 +1,154 @@
// Package keepalived rendert /etc/keepalived/keepalived.conf aus
// cluster_settings (VIP/VRRP-Config) und ha_nodes (local vs. peer).
//
// Split-Brain-Strategie: kein Auto-Promote. notify_master loggt nur
// und sendet einen internen Alert. Promotion ist immer manuell via
// "edgeguard-ctl promote" — das ist die einzig sichere Option ohne
// externes Quorum in einem 2-Node-Cluster.
package keepalived
import (
"bytes"
"context"
_ "embed"
"fmt"
"os"
"os/exec"
"text/template"
"github.com/jackc/pgx/v5/pgxpool"
"git.netcell-it.de/projekte/edgeguard-native/internal/configgen"
"git.netcell-it.de/projekte/edgeguard-native/internal/models"
)
const ConfPath = "/etc/keepalived/keepalived.conf"
//go:embed keepalived.conf.tpl
var cfgTpl string
var tpl = template.Must(template.New("keepalived").Parse(cfgTpl))
// View ist der Template-Kontext.
type View struct {
State string // MASTER | BACKUP
Interface string
RouterID int
Priority int // MASTER=200, BACKUP=100
SrcIP string // eigene Public-IP (für unicast_src_ip)
PeerIP string // Peer-Public-IP (für unicast_peer)
AuthPass string
VIP string
}
type generator struct {
pool *pgxpool.Pool
localID string
}
func New(pool *pgxpool.Pool, localID string) configgen.Generator {
return &generator{pool: pool, localID: localID}
}
func (g *generator) Name() string { return "keepalived" }
func (g *generator) Render(ctx context.Context) error {
cs, local, peer, err := g.loadData(ctx)
if err != nil {
return fmt.Errorf("keepalived: load: %w", err)
}
if cs.VIPAddress == nil || *cs.VIPAddress == "" {
// Kein VIP konfiguriert → keepalived.conf nicht schreiben.
return nil
}
v := g.buildView(cs, local, peer)
var buf bytes.Buffer
if err := tpl.Execute(&buf, v); err != nil {
return fmt.Errorf("keepalived: template: %w", err)
}
if err := configgen.AtomicWrite(ConfPath, buf.Bytes(), 0o640); err != nil {
return fmt.Errorf("keepalived: write: %w", err)
}
if err := reloadKeepalived(); err != nil {
return fmt.Errorf("keepalived: reload: %w", err)
}
return nil
}
func (g *generator) loadData(ctx context.Context) (*models.ClusterSettings, *models.HANode, *models.HANode, error) {
var cs models.ClusterSettings
row := g.pool.QueryRow(ctx, `SELECT id, vip_address, vip_interface, vip_auth_pass, vrrp_router_id FROM cluster_settings WHERE id = 1`)
if err := row.Scan(&cs.ID, &cs.VIPAddress, &cs.VIPInterface, &cs.VIPAuthPass, &cs.VRRPRouterID); err != nil {
return nil, nil, nil, fmt.Errorf("cluster_settings: %w", err)
}
rows, err := g.pool.Query(ctx, `SELECT id, fqdn, role, pg_role, public_ip, status FROM ha_nodes ORDER BY joined_at`)
if err != nil {
return nil, nil, nil, fmt.Errorf("ha_nodes: %w", err)
}
defer rows.Close()
var local, peer *models.HANode
for rows.Next() {
n := &models.HANode{}
if err := rows.Scan(&n.ID, &n.FQDN, &n.Role, &n.PGRole, &n.PublicIP, &n.Status); err != nil {
continue
}
if n.ID == g.localID {
local = n
} else {
peer = n
}
}
if local == nil {
return nil, nil, nil, fmt.Errorf("local node %s not in ha_nodes", g.localID)
}
return &cs, local, peer, nil
}
func (g *generator) buildView(cs *models.ClusterSettings, local, peer *models.HANode) View {
v := View{
RouterID: cs.VRRPRouterID,
VIP: deref(cs.VIPAddress),
Interface: deref(cs.VIPInterface),
AuthPass: deref(cs.VIPAuthPass),
}
if v.Interface == "" {
v.Interface = "eth0"
}
if v.AuthPass == "" {
v.AuthPass = "edgeguard"
}
// Primary-Node bekommt höhere Priorität und startet als MASTER.
if local.PGRole == "primary" || local.Role == "primary" {
v.State = "MASTER"
v.Priority = 200
} else {
v.State = "BACKUP"
v.Priority = 100
}
if local.PublicIP != nil {
v.SrcIP = *local.PublicIP
}
if peer != nil && peer.PublicIP != nil {
v.PeerIP = *peer.PublicIP
}
return v
}
func reloadKeepalived() error {
if _, err := os.Stat("/run/keepalived.pid"); os.IsNotExist(err) {
// keepalived läuft noch nicht — erster Render beim Start.
return nil
}
return exec.Command("systemctl", "reload-or-restart", "keepalived").Run()
}
func deref(s *string) string {
if s == nil {
return ""
}
return *s
}

View File

@@ -0,0 +1,17 @@
package models
import "time"
// ClusterSettings ist die Singleton-Tabelle (id=1) für VIP/VRRP-
// und Replikations-Konfiguration. Angelegt in Migration 0029.
type ClusterSettings struct {
ID int `gorm:"column:id;primaryKey" json:"id"`
VIPAddress *string `gorm:"column:vip_address" json:"vip_address,omitempty"`
VIPInterface *string `gorm:"column:vip_interface" json:"vip_interface,omitempty"`
VIPAuthPass *string `gorm:"column:vip_auth_pass" json:"vip_auth_pass,omitempty"`
VRRPRouterID int `gorm:"column:vrrp_router_id" json:"vrrp_router_id"`
CreatedAt time.Time `gorm:"column:created_at" json:"created_at"`
UpdatedAt time.Time `gorm:"column:updated_at" json:"updated_at"`
}
func (ClusterSettings) TableName() string { return "cluster_settings" }

View File

@@ -4,7 +4,7 @@ import "time"
// HANode mirrort eine Row der ha_nodes-Tabelle. Erweitert in Migration
// 0020 um version/config_hash/mgmt_ip/status für Cluster-Phase-3-
// Drift-Detection + Health-State.
// Drift-Detection + Health-State. Migration 0029 fügt PGRole hinzu.
type HANode struct {
ID string `gorm:"column:id;primaryKey" json:"id"`
Name string `gorm:"column:name" json:"name"`
@@ -14,6 +14,7 @@ type HANode struct {
InternalIP *string `gorm:"column:internal_ip;type:inet" json:"internal_ip,omitempty"`
MgmtIP *string `gorm:"column:mgmt_ip;type:inet" json:"mgmt_ip,omitempty"`
Role string `gorm:"column:role" json:"role"`
PGRole string `gorm:"column:pg_role" json:"pg_role"`
Version *string `gorm:"column:version" json:"version,omitempty"`
ConfigHash *string `gorm:"column:config_hash" json:"config_hash,omitempty"`
Status string `gorm:"column:status" json:"status"`

View File

@@ -128,7 +128,7 @@ func Join(req Request) error {
// synchronous on the primary side.
var autoRegErr error
for i := 0; i < 3; i++ {
if err := autoRegister(primary, tlsDir, req.CommonName, req.Version, req.NodeID); err == nil {
if err := autoRegister(primary, tlsDir, req.CommonName, req.Version, req.NodeID, ""); err == nil {
autoRegErr = nil
break
} else {
@@ -217,7 +217,18 @@ func issueCert(primary, token, csr string, insecure bool) (caCert, peerCert stri
return env.Data.CACert, env.Data.PeerCert, nil
}
func autoRegister(primary, tlsDir, commonName, version, nodeID string) error {
// PushSelfToPrimary sends this node's current identity + configHash to the
// primary via mTLS. Exported for use by the API server's periodic push
// goroutine so the primary's ha_nodes always reflects the secondary's actual
// config_hash (not the stale join-time value).
func PushSelfToPrimary(primaryURL, tlsDir, nodeID, fqdn, version, configHash string) error {
if tlsDir == "" {
tlsDir = clustertls.DefaultDir
}
return autoRegister(primaryURL, tlsDir, fqdn, version, nodeID, configHash)
}
func autoRegister(primary, tlsDir, commonName, version, nodeID, configHash string) error {
u, err := url.Parse(primary)
if err != nil {
return err
@@ -231,11 +242,12 @@ func autoRegister(primary, tlsDir, commonName, version, nodeID string) error {
}
hostname, _ := os.Hostname()
body, _ := json.Marshal(map[string]string{
"id": nodeID,
"name": hostname,
"fqdn": commonName,
"api_url": "https://" + commonName + ":3443",
"version": version,
"id": nodeID,
"name": hostname,
"fqdn": commonName,
"api_url": "https://" + commonName + ":3443",
"version": version,
"config_hash": configHash,
})
pair, err := tls.LoadX509KeyPair(tlsDir+"/peer.crt", tlsDir+"/peer.key")