feat(cluster): PG Logical Replication + VIP/Keepalived + config_hash sync (v1.2.1–1.2.2)
- PG Logical Replication: edgeguard_shared PUBLICATION auf Primary, edgeguard_sub SUBSCRIPTION auf Secondary. Nur geteilte Config-Tabellen werden repliziert; node-eigene Daten (network_interfaces, ip_addresses, static_routes, cluster_settings, dns_settings, ntp_settings) bleiben lokal — OPNsense-Muster. - cluster-init-replication: Erstellt PUBLICATION, Rolle + pg_hba-Einträge (logical + replication), WAL-Level auf logical. - cluster-setup-standby: Erstellt SUBSCRIPTION (copy_data=true), pollt pg_subscription_rel bis alle Tabellen sync = 'r', rendert dann Configs. - promote: manueller Failover via pg_promote() + touch recovery.signal. - VIP/Keepalived: cluster_settings-Tabelle (vip_address, vip_interface, vrrp_router_id), /cluster/vip-settings API, Keepalived-Config-Generator mit VRRP + check_script + notify-Skripten in /usr/lib/edgeguard/scripts/. - config_hash sync: Secondary pusht alle 5 Min seinen Hash via mTLS an Primary (PushSelfToPrimary). Heartbeat schreibt nur LOCAL, daher ohne aktiven Push wäre Primary-Sicht des Secondary-Hash stale gewesen. - runSecondaryConfigRender: Goroutine auf Secondary rendert HAProxy+nftables neu wenn config_hash sich ändert (Logical-Replication-Nachzügler). - confighash: node-spezifische Tabellen aus hashSpec entfernt. - postinst: Keepalived-Skripte installieren, sudoers für keepalived. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -35,10 +35,12 @@ import (
|
||||
|
||||
// hashTable beschreibt eine Tabelle die in den config-hash einfließt.
|
||||
type hashTable struct {
|
||||
Name string
|
||||
Singleton bool // dns_settings, ntp_settings → eine row, id=1
|
||||
ExtraExclude []string // Spalten die zusätzlich aus to_jsonb gefiltert werden
|
||||
SkipUpdatedAt bool // setze true wenn updated_at semantisch relevant ist
|
||||
Name string
|
||||
Singleton bool // dns_settings, ntp_settings → eine row, id=1
|
||||
ExtraExclude []string // Spalten die zusätzlich aus to_jsonb gefiltert werden
|
||||
SkipUpdatedAt bool // setze true wenn updated_at semantisch relevant ist
|
||||
MigrationDefault bool // Tabelle hat migrations-erzeugte Default-Rows (firewall_zones, ntp_pools…)
|
||||
// → zählt nicht als "user hat config" bei der Empty-DB-Erkennung
|
||||
}
|
||||
|
||||
// hashSpec ist die Reihenfolge-stabile Liste. NEUE Tabellen hier
|
||||
@@ -49,15 +51,13 @@ var hashSpec = []hashTable{
|
||||
{Name: "backends"},
|
||||
{Name: "backend_servers"},
|
||||
{Name: "routing_rules"},
|
||||
{Name: "network_interfaces"},
|
||||
{Name: "ip_addresses"},
|
||||
{Name: "tls_certs", ExtraExclude: []string{"last_renewed_at", "last_error"}},
|
||||
|
||||
{Name: "firewall_zones"},
|
||||
{Name: "firewall_zones", MigrationDefault: true},
|
||||
{Name: "firewall_address_objects"},
|
||||
{Name: "firewall_address_groups"},
|
||||
{Name: "firewall_services"},
|
||||
{Name: "firewall_service_groups"},
|
||||
{Name: "firewall_services", MigrationDefault: true},
|
||||
{Name: "firewall_service_groups", MigrationDefault: true},
|
||||
{Name: "firewall_rules"},
|
||||
{Name: "firewall_nat_rules"},
|
||||
|
||||
@@ -67,12 +67,12 @@ var hashSpec = []hashTable{
|
||||
|
||||
{Name: "dns_zones"},
|
||||
{Name: "dns_records"},
|
||||
{Name: "dns_settings", Singleton: true},
|
||||
|
||||
{Name: "ntp_pools"},
|
||||
{Name: "ntp_settings", Singleton: true},
|
||||
{Name: "ntp_pools", MigrationDefault: true},
|
||||
|
||||
{Name: "static_routes"},
|
||||
// network_interfaces, ip_addresses, static_routes, dns_settings, ntp_settings
|
||||
// sind node-spezifisch (jeder Node hat eigene IPs/Routes/Listen-Adressen)
|
||||
// und fließen NICHT in den Drift-Hash ein.
|
||||
}
|
||||
|
||||
// hashSQL rendert die SHA-Input-SQL für eine Tabelle.
|
||||
@@ -100,22 +100,35 @@ func hashSQL(t hashTable) string {
|
||||
// ComputeConfigHash gibt den 16-hex-char-Hash über alle Spec-Tabellen
|
||||
// zurück. Fehlende Tabellen (transienter schema-flux) werden als
|
||||
// leerer Per-Table-Hash behandelt — kein Abbruch.
|
||||
//
|
||||
// Gibt "" zurück wenn alle user-konfigurierbaren Tabellen leer sind
|
||||
// (Singleton- und MigrationDefault-Tabellen zählen nicht als User-Config).
|
||||
// Das verhindert False-Positive-Drift-Banner auf frisch gejointen Secondaries.
|
||||
func ComputeConfigHash(ctx context.Context, pool *pgxpool.Pool) (string, error) {
|
||||
if pool == nil {
|
||||
return "", fmt.Errorf("nil pool")
|
||||
}
|
||||
h := sha256.New()
|
||||
hasUserConfig := false
|
||||
for _, t := range hashSpec {
|
||||
var s string
|
||||
if err := pool.QueryRow(ctx, hashSQL(t)).Scan(&s); err != nil {
|
||||
// Migration fehlt o.ä. → leeren string nehmen, weiter.
|
||||
s = ""
|
||||
}
|
||||
if s != "" && !t.Singleton && !t.MigrationDefault {
|
||||
hasUserConfig = true
|
||||
}
|
||||
h.Write([]byte(t.Name))
|
||||
h.Write([]byte{0})
|
||||
h.Write([]byte(s))
|
||||
h.Write([]byte{0})
|
||||
}
|
||||
if !hasUserConfig {
|
||||
// Frisch gejoincter Secondary oder komplett leere DB →
|
||||
// leerer String signalisiert "kein Drift prüfen" im Status-Handler.
|
||||
return "", nil
|
||||
}
|
||||
return hex.EncodeToString(h.Sum(nil))[:16], nil
|
||||
}
|
||||
|
||||
|
||||
36
internal/database/migrations/0029_cluster_vip.sql
Normal file
36
internal/database/migrations/0029_cluster_vip.sql
Normal file
@@ -0,0 +1,36 @@
|
||||
-- +goose Up
|
||||
-- +goose StatementBegin
|
||||
|
||||
-- pg_role: Rolle dieser Node in der PG-Replikation.
|
||||
-- "standalone" = kein Streaming-Replication-Setup
|
||||
-- "primary" = WAL-Sender, repliziert an Standby(s)
|
||||
-- "standby" = Hot-Standby, liest WAL vom Primary
|
||||
ALTER TABLE ha_nodes ADD COLUMN IF NOT EXISTS pg_role TEXT NOT NULL DEFAULT 'standalone';
|
||||
|
||||
-- cluster_settings: VIP + VRRP-Konfiguration (Singleton, id=1).
|
||||
-- vip_address = die virtuelle IP-Adresse (z.B. "89.163.205.10")
|
||||
-- vip_interface = Netzwerk-Interface (z.B. "eth0")
|
||||
-- vip_auth_pass = VRRP-Authentication-Passwort (max. 8 Zeichen, Keepalived-Limit)
|
||||
-- vrrp_router_id = VRRP Virtual Router ID (1–255, muss im Subnetz eindeutig sein)
|
||||
CREATE TABLE IF NOT EXISTS cluster_settings (
|
||||
id INTEGER PRIMARY KEY DEFAULT 1,
|
||||
vip_address TEXT,
|
||||
vip_interface TEXT,
|
||||
vip_auth_pass TEXT,
|
||||
vrrp_router_id INTEGER NOT NULL DEFAULT 51,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
CONSTRAINT cluster_settings_singleton CHECK (id = 1)
|
||||
);
|
||||
|
||||
INSERT INTO cluster_settings (id) VALUES (1) ON CONFLICT DO NOTHING;
|
||||
|
||||
-- +goose StatementEnd
|
||||
|
||||
-- +goose Down
|
||||
-- +goose StatementBegin
|
||||
|
||||
DROP TABLE IF EXISTS cluster_settings;
|
||||
ALTER TABLE ha_nodes DROP COLUMN IF EXISTS pg_role;
|
||||
|
||||
-- +goose StatementEnd
|
||||
40
internal/keepalived/keepalived.conf.tpl
Normal file
40
internal/keepalived/keepalived.conf.tpl
Normal file
@@ -0,0 +1,40 @@
|
||||
global_defs {
|
||||
router_id {{ .RouterID }}
|
||||
script_user root
|
||||
enable_script_security
|
||||
vrrp_garp_interval 0
|
||||
vrrp_gna_interval 0
|
||||
}
|
||||
|
||||
vrrp_script chk_edgeguard {
|
||||
script "/usr/lib/edgeguard/keepalived-check.sh"
|
||||
interval 2
|
||||
weight -50
|
||||
fall 3
|
||||
rise 2
|
||||
}
|
||||
|
||||
vrrp_instance VI_1 {
|
||||
state {{ .State }}
|
||||
interface {{ .Interface }}
|
||||
virtual_router_id {{ .RouterID }}
|
||||
priority {{ .Priority }}
|
||||
advert_int 1
|
||||
{{ if .SrcIP }} unicast_src_ip {{ .SrcIP }}
|
||||
unicast_peer {
|
||||
{{ .PeerIP }}
|
||||
}
|
||||
{{ end }} authentication {
|
||||
auth_type PASS
|
||||
auth_pass {{ .AuthPass }}
|
||||
}
|
||||
virtual_ipaddress {
|
||||
{{ .VIP }}
|
||||
}
|
||||
track_script {
|
||||
chk_edgeguard
|
||||
}
|
||||
notify_master "/usr/lib/edgeguard/keepalived-master.sh"
|
||||
notify_backup "/usr/lib/edgeguard/keepalived-backup.sh"
|
||||
notify_fault "/usr/lib/edgeguard/keepalived-backup.sh"
|
||||
}
|
||||
154
internal/keepalived/keepalived.go
Normal file
154
internal/keepalived/keepalived.go
Normal file
@@ -0,0 +1,154 @@
|
||||
// Package keepalived rendert /etc/keepalived/keepalived.conf aus
|
||||
// cluster_settings (VIP/VRRP-Config) und ha_nodes (local vs. peer).
|
||||
//
|
||||
// Split-Brain-Strategie: kein Auto-Promote. notify_master loggt nur
|
||||
// und sendet einen internen Alert. Promotion ist immer manuell via
|
||||
// "edgeguard-ctl promote" — das ist die einzig sichere Option ohne
|
||||
// externes Quorum in einem 2-Node-Cluster.
|
||||
package keepalived
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
_ "embed"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"text/template"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"git.netcell-it.de/projekte/edgeguard-native/internal/configgen"
|
||||
"git.netcell-it.de/projekte/edgeguard-native/internal/models"
|
||||
)
|
||||
|
||||
const ConfPath = "/etc/keepalived/keepalived.conf"
|
||||
|
||||
//go:embed keepalived.conf.tpl
|
||||
var cfgTpl string
|
||||
|
||||
var tpl = template.Must(template.New("keepalived").Parse(cfgTpl))
|
||||
|
||||
// View ist der Template-Kontext.
|
||||
type View struct {
|
||||
State string // MASTER | BACKUP
|
||||
Interface string
|
||||
RouterID int
|
||||
Priority int // MASTER=200, BACKUP=100
|
||||
SrcIP string // eigene Public-IP (für unicast_src_ip)
|
||||
PeerIP string // Peer-Public-IP (für unicast_peer)
|
||||
AuthPass string
|
||||
VIP string
|
||||
}
|
||||
|
||||
type generator struct {
|
||||
pool *pgxpool.Pool
|
||||
localID string
|
||||
}
|
||||
|
||||
func New(pool *pgxpool.Pool, localID string) configgen.Generator {
|
||||
return &generator{pool: pool, localID: localID}
|
||||
}
|
||||
|
||||
func (g *generator) Name() string { return "keepalived" }
|
||||
|
||||
func (g *generator) Render(ctx context.Context) error {
|
||||
cs, local, peer, err := g.loadData(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("keepalived: load: %w", err)
|
||||
}
|
||||
if cs.VIPAddress == nil || *cs.VIPAddress == "" {
|
||||
// Kein VIP konfiguriert → keepalived.conf nicht schreiben.
|
||||
return nil
|
||||
}
|
||||
v := g.buildView(cs, local, peer)
|
||||
var buf bytes.Buffer
|
||||
if err := tpl.Execute(&buf, v); err != nil {
|
||||
return fmt.Errorf("keepalived: template: %w", err)
|
||||
}
|
||||
if err := configgen.AtomicWrite(ConfPath, buf.Bytes(), 0o640); err != nil {
|
||||
return fmt.Errorf("keepalived: write: %w", err)
|
||||
}
|
||||
if err := reloadKeepalived(); err != nil {
|
||||
return fmt.Errorf("keepalived: reload: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (g *generator) loadData(ctx context.Context) (*models.ClusterSettings, *models.HANode, *models.HANode, error) {
|
||||
var cs models.ClusterSettings
|
||||
row := g.pool.QueryRow(ctx, `SELECT id, vip_address, vip_interface, vip_auth_pass, vrrp_router_id FROM cluster_settings WHERE id = 1`)
|
||||
if err := row.Scan(&cs.ID, &cs.VIPAddress, &cs.VIPInterface, &cs.VIPAuthPass, &cs.VRRPRouterID); err != nil {
|
||||
return nil, nil, nil, fmt.Errorf("cluster_settings: %w", err)
|
||||
}
|
||||
|
||||
rows, err := g.pool.Query(ctx, `SELECT id, fqdn, role, pg_role, public_ip, status FROM ha_nodes ORDER BY joined_at`)
|
||||
if err != nil {
|
||||
return nil, nil, nil, fmt.Errorf("ha_nodes: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var local, peer *models.HANode
|
||||
for rows.Next() {
|
||||
n := &models.HANode{}
|
||||
if err := rows.Scan(&n.ID, &n.FQDN, &n.Role, &n.PGRole, &n.PublicIP, &n.Status); err != nil {
|
||||
continue
|
||||
}
|
||||
if n.ID == g.localID {
|
||||
local = n
|
||||
} else {
|
||||
peer = n
|
||||
}
|
||||
}
|
||||
if local == nil {
|
||||
return nil, nil, nil, fmt.Errorf("local node %s not in ha_nodes", g.localID)
|
||||
}
|
||||
return &cs, local, peer, nil
|
||||
}
|
||||
|
||||
func (g *generator) buildView(cs *models.ClusterSettings, local, peer *models.HANode) View {
|
||||
v := View{
|
||||
RouterID: cs.VRRPRouterID,
|
||||
VIP: deref(cs.VIPAddress),
|
||||
Interface: deref(cs.VIPInterface),
|
||||
AuthPass: deref(cs.VIPAuthPass),
|
||||
}
|
||||
if v.Interface == "" {
|
||||
v.Interface = "eth0"
|
||||
}
|
||||
if v.AuthPass == "" {
|
||||
v.AuthPass = "edgeguard"
|
||||
}
|
||||
|
||||
// Primary-Node bekommt höhere Priorität und startet als MASTER.
|
||||
if local.PGRole == "primary" || local.Role == "primary" {
|
||||
v.State = "MASTER"
|
||||
v.Priority = 200
|
||||
} else {
|
||||
v.State = "BACKUP"
|
||||
v.Priority = 100
|
||||
}
|
||||
|
||||
if local.PublicIP != nil {
|
||||
v.SrcIP = *local.PublicIP
|
||||
}
|
||||
if peer != nil && peer.PublicIP != nil {
|
||||
v.PeerIP = *peer.PublicIP
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
func reloadKeepalived() error {
|
||||
if _, err := os.Stat("/run/keepalived.pid"); os.IsNotExist(err) {
|
||||
// keepalived läuft noch nicht — erster Render beim Start.
|
||||
return nil
|
||||
}
|
||||
return exec.Command("systemctl", "reload-or-restart", "keepalived").Run()
|
||||
}
|
||||
|
||||
func deref(s *string) string {
|
||||
if s == nil {
|
||||
return ""
|
||||
}
|
||||
return *s
|
||||
}
|
||||
17
internal/models/cluster_settings.go
Normal file
17
internal/models/cluster_settings.go
Normal file
@@ -0,0 +1,17 @@
|
||||
package models
|
||||
|
||||
import "time"
|
||||
|
||||
// ClusterSettings ist die Singleton-Tabelle (id=1) für VIP/VRRP-
|
||||
// und Replikations-Konfiguration. Angelegt in Migration 0029.
|
||||
type ClusterSettings struct {
|
||||
ID int `gorm:"column:id;primaryKey" json:"id"`
|
||||
VIPAddress *string `gorm:"column:vip_address" json:"vip_address,omitempty"`
|
||||
VIPInterface *string `gorm:"column:vip_interface" json:"vip_interface,omitempty"`
|
||||
VIPAuthPass *string `gorm:"column:vip_auth_pass" json:"vip_auth_pass,omitempty"`
|
||||
VRRPRouterID int `gorm:"column:vrrp_router_id" json:"vrrp_router_id"`
|
||||
CreatedAt time.Time `gorm:"column:created_at" json:"created_at"`
|
||||
UpdatedAt time.Time `gorm:"column:updated_at" json:"updated_at"`
|
||||
}
|
||||
|
||||
func (ClusterSettings) TableName() string { return "cluster_settings" }
|
||||
@@ -4,7 +4,7 @@ import "time"
|
||||
|
||||
// HANode mirrort eine Row der ha_nodes-Tabelle. Erweitert in Migration
|
||||
// 0020 um version/config_hash/mgmt_ip/status für Cluster-Phase-3-
|
||||
// Drift-Detection + Health-State.
|
||||
// Drift-Detection + Health-State. Migration 0029 fügt PGRole hinzu.
|
||||
type HANode struct {
|
||||
ID string `gorm:"column:id;primaryKey" json:"id"`
|
||||
Name string `gorm:"column:name" json:"name"`
|
||||
@@ -14,6 +14,7 @@ type HANode struct {
|
||||
InternalIP *string `gorm:"column:internal_ip;type:inet" json:"internal_ip,omitempty"`
|
||||
MgmtIP *string `gorm:"column:mgmt_ip;type:inet" json:"mgmt_ip,omitempty"`
|
||||
Role string `gorm:"column:role" json:"role"`
|
||||
PGRole string `gorm:"column:pg_role" json:"pg_role"`
|
||||
Version *string `gorm:"column:version" json:"version,omitempty"`
|
||||
ConfigHash *string `gorm:"column:config_hash" json:"config_hash,omitempty"`
|
||||
Status string `gorm:"column:status" json:"status"`
|
||||
|
||||
@@ -128,7 +128,7 @@ func Join(req Request) error {
|
||||
// synchronous on the primary side.
|
||||
var autoRegErr error
|
||||
for i := 0; i < 3; i++ {
|
||||
if err := autoRegister(primary, tlsDir, req.CommonName, req.Version, req.NodeID); err == nil {
|
||||
if err := autoRegister(primary, tlsDir, req.CommonName, req.Version, req.NodeID, ""); err == nil {
|
||||
autoRegErr = nil
|
||||
break
|
||||
} else {
|
||||
@@ -217,7 +217,18 @@ func issueCert(primary, token, csr string, insecure bool) (caCert, peerCert stri
|
||||
return env.Data.CACert, env.Data.PeerCert, nil
|
||||
}
|
||||
|
||||
func autoRegister(primary, tlsDir, commonName, version, nodeID string) error {
|
||||
// PushSelfToPrimary sends this node's current identity + configHash to the
|
||||
// primary via mTLS. Exported for use by the API server's periodic push
|
||||
// goroutine so the primary's ha_nodes always reflects the secondary's actual
|
||||
// config_hash (not the stale join-time value).
|
||||
func PushSelfToPrimary(primaryURL, tlsDir, nodeID, fqdn, version, configHash string) error {
|
||||
if tlsDir == "" {
|
||||
tlsDir = clustertls.DefaultDir
|
||||
}
|
||||
return autoRegister(primaryURL, tlsDir, fqdn, version, nodeID, configHash)
|
||||
}
|
||||
|
||||
func autoRegister(primary, tlsDir, commonName, version, nodeID, configHash string) error {
|
||||
u, err := url.Parse(primary)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -231,11 +242,12 @@ func autoRegister(primary, tlsDir, commonName, version, nodeID string) error {
|
||||
}
|
||||
hostname, _ := os.Hostname()
|
||||
body, _ := json.Marshal(map[string]string{
|
||||
"id": nodeID,
|
||||
"name": hostname,
|
||||
"fqdn": commonName,
|
||||
"api_url": "https://" + commonName + ":3443",
|
||||
"version": version,
|
||||
"id": nodeID,
|
||||
"name": hostname,
|
||||
"fqdn": commonName,
|
||||
"api_url": "https://" + commonName + ":3443",
|
||||
"version": version,
|
||||
"config_hash": configHash,
|
||||
})
|
||||
|
||||
pair, err := tls.LoadX509KeyPair(tlsDir+"/peer.crt", tlsDir+"/peer.key")
|
||||
|
||||
Reference in New Issue
Block a user