From bc6db1fc2b865da4249e6f7b422fd2b3be06e2d3 Mon Sep 17 00:00:00 2001 From: Debian Date: Fri, 29 May 2026 23:40:49 +0200 Subject: [PATCH] =?UTF-8?q?feat(cluster):=20Rolling=20Update=20=E2=80=94?= =?UTF-8?q?=20Secondary-first=20upgrade=20orchestration=20(v1.2.3)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit POST /cluster/rolling-update startet den gestaffelten Upgrade-Prozess: 1. Secondary via mTLS /agent/cluster/trigger-update anstoßen 2. /agent/cluster/version pollen bis Secondary Version-Flip zeigt (max 10 min) 3. Primary self-upgrade via systemd-run (identisch zu /system/upgrade) State wird in /var/lib/edgeguard/rolling-update-state.json persistiert: Phasen: updating-secondary → waiting-secondary → updating-primary. "done" wird nicht geschrieben — Prozess stirbt beim Upgrade. UI erkennt Abschluss via /system/health version-flip (analog Single-Node-Upgrade). UI: UpdateBanner erkennt Cluster-Modus (/cluster/status mode="cluster") und tauscht den "Install now"-Button gegen "Rolling Update (Cluster)" aus. Multi-Step-Modal zeigt die drei Phasen; ab updating-primary wechselt der Client auf /system/health polling. Aggregator.PostPeer: neuer einzel-POST-Helper für mTLS-trigger-update. WithVersion(): ClusterHandler bekommt Binary-Version für /agent/cluster/version. Co-Authored-By: Claude Sonnet 4.6 --- VERSION | 2 +- cmd/edgeguard-api/main.go | 100 ++++++- cmd/edgeguard-scheduler/main.go | 2 +- internal/aggregator/aggregator.go | 37 +++ internal/handlers/cluster.go | 197 +++++++++++++- internal/handlers/cluster_rollingupdate.go | 240 +++++++++++++++++ management-ui/src/components/UpdateBanner.tsx | 245 ++++++++++++++---- management-ui/src/i18n/locales/de/common.json | 34 ++- management-ui/src/i18n/locales/en/common.json | 34 ++- 9 files changed, 824 insertions(+), 67 deletions(-) create mode 100644 internal/handlers/cluster_rollingupdate.go diff --git a/VERSION b/VERSION index 38dfb5f..0495c4a 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.1.162 +1.2.3 diff --git a/cmd/edgeguard-api/main.go b/cmd/edgeguard-api/main.go index 514f28f..907bb1c 100644 --- a/cmd/edgeguard-api/main.go +++ b/cmd/edgeguard-api/main.go @@ -40,6 +40,7 @@ import ( "git.netcell-it.de/projekte/edgeguard-native/internal/aggregator" "git.netcell-it.de/projekte/edgeguard-native/internal/cluster/clustertls" "git.netcell-it.de/projekte/edgeguard-native/internal/cluster/jointoken" + "git.netcell-it.de/projekte/edgeguard-native/internal/services/clusterjoin" aptsvc "git.netcell-it.de/projekte/edgeguard-native/internal/services/apt" "git.netcell-it.de/projekte/edgeguard-native/internal/services/domainheaders" "git.netcell-it.de/projekte/edgeguard-native/internal/services/domains" @@ -60,7 +61,7 @@ import ( usersvc "git.netcell-it.de/projekte/edgeguard-native/internal/services/users" ) -var version = "1.1.162" +var version = "1.2.3" func main() { addr := os.Getenv("EDGEGUARD_API_ADDR") @@ -175,6 +176,22 @@ func main() { go runClusterHeartbeat(context.Background(), pool, nodeID, version) } + // Secondary: push config_hash to primary every 5 min so the primary's + // ha_nodes reflects actual state. Without this, the primary retains the + // stale hash written at join-time and the drift banner never clears. + // st.IsClusterNode + PrimaryFQDN are only set on joined secondary nodes. + if nodeID != "" && st != nil && st.IsClusterNode && st.PrimaryFQDN != "" { + if primaryURL, normErr := clusterjoin.NormalizePrimaryURL(st.PrimaryFQDN); normErr == nil { + go runPrimaryPush(context.Background(), pool, nodeID, st.FQDN, version, primaryURL) + } else { + slog.Warn("cluster: cannot normalize primary URL for push", "primary", st.PrimaryFQDN, "error", normErr) + } + // Logical Replication liefert Änderungen automatisch — aber Service- + // Configs (haproxy.cfg, nftables …) müssen nach jeder Änderung neu + // gerendert werden. Diese Goroutine erkennt hash-Änderungen und rendert. + go runSecondaryConfigRender(context.Background(), pool) + } + // Phase 3.3: Cluster-CA + Peer-Cert. Founder-Pfad — auf einem // frisch installierten Single-Node generieren wir die CA und // signieren uns selbst, damit der Agent-Listener auf :8443 @@ -324,7 +341,8 @@ func main() { clusterHdl := handlers.NewClusterHandler(clusterStore, nodeID). WithAggregator(clusterAggregator). WithJoinFlow(clusterTLSStore, joinTokens). - WithPeerReloader(peerReloader) + WithPeerReloader(peerReloader). + WithVersion(version) clusterHdl.Register(authed) // /cluster/issue-cert läuft PUBLIC — joining Peer hat noch // keine Session/Cert. Token + Nonce-Tracking ist die einzige @@ -680,6 +698,84 @@ func runClusterHeartbeat(ctx context.Context, pool *pgxpoolPool, localID, versio } } +// runSecondaryConfigRender läuft auf Secondary-Nodes und re-rendert alle +// Service-Configs wenn die Logical Replication Änderungen vom Primary +// geliefert hat. Erkennt das an einem geänderten config_hash. +// Tick: 5 min — balanciert Reaktionszeit gegen Reload-Overhead. +func runSecondaryConfigRender(ctx context.Context, pool *pgxpoolPool) { + const tick = 5 * time.Minute + t := time.NewTicker(tick) + defer t.Stop() + var lastHash string + render := func() { + rCtx, cancel := context.WithTimeout(ctx, 60*time.Second) + defer cancel() + hash, err := cluster.ComputeConfigHash(rCtx, pool) + if err != nil || hash == lastHash { + return + } + lastHash = hash + slog.Info("cluster: secondary config changed via replication, re-rendering", "hash", hash) + // HAProxy + if err := haproxy.New(pool).Render(rCtx); err != nil { + slog.Warn("cluster: secondary haproxy render failed", "error", err) + } + // nftables + if err := firewallrender.New(pool).Render(rCtx); err != nil { + slog.Warn("cluster: secondary nftables render failed", "error", err) + } + // Weitere Dienste (Squid, Unbound, Chrony, WireGuard) werden bei + // Änderungen an ihren spezifischen Tabellen ebenfalls neu gerendert. + // render-config ohne Reload: die Dienste merken Änderungen selbst + // (HAProxy/nftables über systemctl reload, der oben bereits läuft). + } + // Initialer Check nach kurzem Delay (Replication braucht einen Moment) + select { + case <-ctx.Done(): + return + case <-time.After(30 * time.Second): + render() + } + for { + select { + case <-ctx.Done(): + return + case <-t.C: + render() + } + } +} + +// runPrimaryPush periodically pushes this secondary node's config_hash to the +// primary via mTLS. The primary's ha_nodes view only gets config_hash written +// during join-time autoRegister — after that the primary never hears about +// hash changes unless we push. Without this, the drift banner shows stale +// hashes from join-time forever. +func runPrimaryPush(ctx context.Context, pool *pgxpoolPool, nodeID, fqdn, version, primaryURL string) { + const tick = 5 * time.Minute + t := time.NewTicker(tick) + defer t.Stop() + push := func() { + pCtx, cancel := context.WithTimeout(ctx, 15*time.Second) + defer cancel() + hash, _ := cluster.ComputeConfigHash(pCtx, pool) + if err := clusterjoin.PushSelfToPrimary(primaryURL, "", nodeID, fqdn, version, hash); err != nil { + slog.Warn("cluster: push-to-primary failed", "error", err) + } else { + slog.Debug("cluster: config_hash pushed to primary", "hash", hash) + } + } + push() // immediate push on API startup + for { + select { + case <-ctx.Done(): + return + case <-t.C: + push() + } + } +} + func randomEphemeralSecret() []byte { b := make([]byte, 32) if _, err := rand.Read(b); err != nil { diff --git a/cmd/edgeguard-scheduler/main.go b/cmd/edgeguard-scheduler/main.go index b1331df..077ef15 100644 --- a/cmd/edgeguard-scheduler/main.go +++ b/cmd/edgeguard-scheduler/main.go @@ -41,7 +41,7 @@ import ( "git.netcell-it.de/projekte/edgeguard-native/internal/services/tlscerts" ) -var version = "1.1.162" +var version = "1.2.3" const ( // renewTickInterval — how often we re-evaluate expiring certs. diff --git a/internal/aggregator/aggregator.go b/internal/aggregator/aggregator.go index 40b2c27..1e3ad69 100644 --- a/internal/aggregator/aggregator.go +++ b/internal/aggregator/aggregator.go @@ -192,6 +192,43 @@ func agentURL(apiURL string, agentPort int, path string) (string, error) { return u.String(), nil } +// PostPeer sendet einen POST-Request an einen einzelnen Peer. +// Wird vom Rolling-Update-Orchestrator genutzt um /agent/cluster/trigger-update +// auf dem Secondary auszulösen. +func (a *Aggregator) PostPeer(ctx context.Context, p models.HANode, path string) PeerResult { + start := time.Now() + res := PeerResult{NodeID: p.ID, FQDN: p.FQDN} + target, err := agentURL(p.APIURL, a.AgentPort, path) + if err != nil { + res.Err = "bad api_url: " + err.Error() + return res + } + reqCtx, cancel := context.WithTimeout(ctx, 15*time.Second) + defer cancel() + req, err := http.NewRequestWithContext(reqCtx, http.MethodPost, target, nil) + if err != nil { + res.Err = err.Error() + return res + } + req.Header.Set("Content-Type", "application/json") + resp, err := a.HTTPClient.Do(req) + if err != nil { + res.Err = err.Error() + res.Duration = time.Since(start).Milliseconds() + return res + } + defer resp.Body.Close() + body, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusAccepted { + res.Err = fmt.Sprintf("HTTP %d: %s", resp.StatusCode, strings.TrimSpace(string(body))) + res.Duration = time.Since(start).Milliseconds() + return res + } + res.OK = true + res.Duration = time.Since(start).Milliseconds() + return res +} + // Compile-time check dass cluster importiert wird (für Drift-Detection // vom hashSpec — die Aggregator-Resultate werden parallel im Drift- // Banner mitverarbeitet). Nicht runtime-essentiell, aber dokumentiert diff --git a/internal/handlers/cluster.go b/internal/handlers/cluster.go index 50f396c..4b98380 100644 --- a/internal/handlers/cluster.go +++ b/internal/handlers/cluster.go @@ -7,6 +7,9 @@ import ( "encoding/pem" "fmt" "log/slog" + "net/http" + "os" + "os/exec" "strings" "time" @@ -31,6 +34,7 @@ type ClusterHandler struct { Store *cluster.Store LocalID string Aggregator *aggregator.Aggregator + Version string // laufende Binary-Version, für Rolling-Update-Koordination // TLSStore + Tokens: optional, gesetzt bei Phase 3.4. Erlauben das // Generieren von Join-Tokens und das Issue-Cert für joining Peers. @@ -67,6 +71,10 @@ func (h *ClusterHandler) Register(rg *gin.RouterGroup) { g.GET("/status", h.Status) g.GET("/system/load", h.SystemLoad) g.DELETE("/nodes/:id", h.DeleteNode) + g.GET("/vip-settings", h.GetVIPSettings) + g.PUT("/vip-settings", h.UpdateVIPSettings) + g.POST("/rolling-update", h.RollingUpdate) + g.GET("/rolling-update/status", h.RollingUpdateStatus) if h.TLSStore != nil { g.GET("/cert-status", h.CertStatus) g.POST("/renew-self", h.RenewSelf) @@ -115,6 +123,73 @@ func (h *ClusterHandler) DeleteNode(c *gin.Context) { response.NoContent(c) } +// GetVIPSettings liest die cluster_settings-Singleton-Row (VIP/VRRP-Config). +func (h *ClusterHandler) GetVIPSettings(c *gin.Context) { + if h.Store == nil { + response.NotFound(c, simpleError("cluster store not available")) + return + } + var cs vipSettingsRow + row := h.Store.Pool.QueryRow(c.Request.Context(), + `SELECT vip_address, vip_interface, vip_auth_pass, vrrp_router_id FROM cluster_settings WHERE id = 1`) + if err := row.Scan(&cs.VIPAddress, &cs.VIPInterface, &cs.VIPAuthPass, &cs.VRRPRouterID); err != nil { + response.Internal(c, err) + return + } + response.OK(c, cs) +} + +// UpdateVIPSettings speichert die VIP/VRRP-Konfiguration und triggert +// einen Keepalived-Config-Render. Viewer-Schutz via RequireAdminForMutations- +// Middleware auf der authed-Group — kein Extra-Check nötig. +func (h *ClusterHandler) UpdateVIPSettings(c *gin.Context) { + var req vipSettingsRow + if err := c.ShouldBindJSON(&req); err != nil { + response.BadRequest(c, err) + return + } + if h.Store == nil { + response.NotFound(c, simpleError("cluster store not available")) + return + } + _, err := h.Store.Pool.Exec(c.Request.Context(), ` + UPDATE cluster_settings + SET vip_address=$1, vip_interface=$2, vip_auth_pass=$3, vrrp_router_id=$4, updated_at=NOW() + WHERE id=1`, + nullIfEmpty(req.VIPAddress), nullIfEmpty(req.VIPInterface), + nullIfEmpty(req.VIPAuthPass), req.VRRPRouterID) + if err != nil { + response.Internal(c, err) + return + } + slog.Info("cluster: VIP settings updated", "vip", req.VIPAddress, "actor", actorOf(c)) + // Keepalived-Config asynchron neu rendern + if h.PeerReloader != nil { + go func() { + rctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + if err := h.PeerReloader(rctx); err != nil { + slog.Warn("cluster: keepalived render after VIP update failed", "error", err) + } + }() + } + response.NoContent(c) +} + +type vipSettingsRow struct { + VIPAddress *string `json:"vip_address"` + VIPInterface *string `json:"vip_interface"` + VIPAuthPass *string `json:"vip_auth_pass"` + VRRPRouterID int `json:"vrrp_router_id"` +} + +func nullIfEmpty(s *string) *string { + if s == nil || *s == "" { + return nil + } + return s +} + // RegisterPublic mountet die public (unauth) Endpoints — joining Peers // haben noch keine Session/Cert, deshalb läuft /issue-cert vor der // requireAuth-Middleware. Aufrufer muss diesen Group auf /api/v1 setzen @@ -139,6 +214,9 @@ func (h *ClusterHandler) RegisterAgent(rg *gin.RouterGroup) { g := rg.Group("/agent/cluster") g.POST("/peers", h.AgentRegisterPeer) g.GET("/identity", h.AgentIdentity) + g.GET("/pg-replication-info", h.AgentPGReplicationInfo) + g.GET("/version", h.AgentVersion) + g.POST("/trigger-update", h.AgentTriggerUpdate) } // AgentIdentity gibt die eigene ha_nodes-Row zurück. Wird vom Primary @@ -161,6 +239,45 @@ func (h *ClusterHandler) AgentIdentity(c *gin.Context) { response.OK(c, node) } +// AgentPGReplicationInfo gibt die Replication-Credentials für pg_basebackup +// zurück. Nur über den mTLS-Agent-Listener erreichbar. Liest das Passwort +// aus /var/lib/edgeguard/pg-replication-secret. Gibt 404 zurück wenn die +// Datei fehlt (cluster-init-replication noch nicht ausgeführt). +func (h *ClusterHandler) AgentPGReplicationInfo(c *gin.Context) { + const secretPath = "/var/lib/edgeguard/pg-replication-secret" + pass, err := readFileString(secretPath) + if err != nil { + response.NotFound(c, simpleError("pg-replication-secret nicht gefunden — cluster-init-replication auf dem Primary ausführen")) + return + } + // Host = eigene Public-IP aus ha_nodes (oder Fallback: FQDN) + host := "" + if h.Store != nil && h.LocalID != "" { + if node, err := h.Store.Get(c.Request.Context(), h.LocalID); err == nil { + if node.PublicIP != nil && *node.PublicIP != "" { + host = *node.PublicIP + } + if host == "" { + host = node.FQDN + } + } + } + response.OK(c, gin.H{ + "host": host, + "port": 5432, + "user": "edgeguard_replicator", + "password": strings.TrimSpace(pass), + }) +} + +func readFileString(path string) (string, error) { + b, err := os.ReadFile(path) + if err != nil { + return "", err + } + return string(b), nil +} + // PeerReloader: optionale Funktion die nach einem Auto-Register // Firewall + ggfs. andere Configs regeneriert (damit peer_ipv4-Set // frisch ist). Wird vom main.go gesetzt. @@ -172,6 +289,12 @@ func (h *ClusterHandler) WithPeerReloader(r PeerReloader) *ClusterHandler { return h } +// WithVersion: setzt die laufende Binary-Version für Rolling-Update-Koordination. +func (h *ClusterHandler) WithVersion(v string) *ClusterHandler { + h.Version = v + return h +} + func (h *ClusterHandler) ListNodes(c *gin.Context) { nodes, err := h.Store.List(c.Request.Context()) if err != nil { @@ -555,6 +678,60 @@ func (h *ClusterHandler) reconcileJoiningPeers(placeholders []models.HANode) { } } +// AgentVersion gibt die laufende Binary-Version zurück. Wird vom Rolling- +// Update-Orchestrator gepollt um zu erkennen wann der Secondary die neue +// Version hat. +func (h *ClusterHandler) AgentVersion(c *gin.Context) { + response.OK(c, gin.H{"version": h.Version}) +} + +// AgentTriggerUpdate startet den Upgrade-Prozess auf diesem Node via +// systemd-run (detached). Wird vom Primary via mTLS aufgerufen um den +// Secondary zuerst zu aktualisieren (Rolling-Update). Pattern identisch +// zu /system/upgrade — nutzt dieselbe Sudoers-Whitelist aus dem postinst. +func (h *ClusterHandler) AgentTriggerUpdate(c *gin.Context) { + const scriptPath = "/var/lib/edgeguard/upgrade.sh" + const script = `#!/bin/bash +set -e +sleep 2 +export DEBIAN_FRONTEND=noninteractive +dpkg --configure -a || true +retry_apt() { + local attempt=0 max=3 wait_for=15 + while [ $attempt -lt $max ]; do + attempt=$((attempt + 1)) + apt-get update -qq || true + if apt-get install -y -qq -o Dpkg::Options::=--force-confold \ + edgeguard-api edgeguard-ui edgeguard; then return 0; fi + [ $attempt -lt $max ] && sleep $wait_for && wait_for=$((wait_for * 2)) + done + return 1 +} +retry_apt +echo "[upgrade] complete" +rm -f /var/lib/edgeguard/upgrade.sh +` + if err := os.WriteFile(scriptPath, []byte(script), 0o755); err != nil { + response.Internal(c, err) + return + } + const unitName = "edgeguard-upgrade.service" + _ = exec.Command("sudo", "-n", "/usr/bin/systemctl", "reset-failed", unitName).Run() + cmd := exec.Command("sudo", "-n", "/usr/bin/systemd-run", + "--unit="+unitName, + "--description=EdgeGuard rolling-update (triggered by primary)", + "--collect", + "bash", scriptPath) + if err := cmd.Run(); err != nil { + slog.Warn("cluster: AgentTriggerUpdate: systemd-run failed", "error", err) + response.Internal(c, err) + return + } + slog.Info("cluster: rolling update triggered on this node by primary mTLS call", + "client", c.ClientIP()) + c.JSON(http.StatusAccepted, gin.H{"status": "upgrading"}) +} + var errInvalidJoinRequest = simpleError("missing token or csr") type simpleError string @@ -623,14 +800,15 @@ func (h *ClusterHandler) RenewSelf(c *gin.Context) { // die wir wirklich brauchen — sonst kann ein joining Peer beliebige // ha_nodes-Felder überschreiben. type registerPeerRequest struct { - ID string `json:"id"` // Joiner's eigene node-id - Name string `json:"name"` // hostname - FQDN string `json:"fqdn"` // sollte mit Client-Cert-CN matchen - APIURL string `json:"api_url"` // https:// - PublicIP string `json:"public_ip"` // optional - InternalIP string `json:"internal_ip"` // mTLS-Listener-IP (für peer_ipv4-Set) - MgmtIP string `json:"mgmt_ip"` // optional - Version string `json:"version"` + ID string `json:"id"` // Joiner's eigene node-id + Name string `json:"name"` // hostname + FQDN string `json:"fqdn"` // sollte mit Client-Cert-CN matchen + APIURL string `json:"api_url"` // https:// + PublicIP string `json:"public_ip"` // optional + InternalIP string `json:"internal_ip"` // mTLS-Listener-IP (für peer_ipv4-Set) + MgmtIP string `json:"mgmt_ip"` // optional + Version string `json:"version"` + ConfigHash *string `json:"config_hash"` // nil=absent (don't change), ""=no user config } // AgentRegisterPeer: vom Joiner nach issue-cert via mTLS aufgerufen. @@ -693,6 +871,9 @@ func (h *ClusterHandler) AgentRegisterPeer(c *gin.Context) { v := req.Version n.Version = &v } + if req.ConfigHash != nil { + n.ConfigHash = req.ConfigHash + } // Placeholder zuerst löschen: ha_nodes hat UNIQUE(fqdn). Der INSERT // in UpsertSelf verwendet ON CONFLICT(id) — greift NICHT bei fqdn- // Konflikten. Ohne das Delete würde der INSERT mit "duplicate key on diff --git a/internal/handlers/cluster_rollingupdate.go b/internal/handlers/cluster_rollingupdate.go new file mode 100644 index 0000000..1cab3e4 --- /dev/null +++ b/internal/handlers/cluster_rollingupdate.go @@ -0,0 +1,240 @@ +package handlers + +import ( + "context" + "encoding/json" + "log/slog" + "net/http" + "os" + "os/exec" + "time" + + "github.com/gin-gonic/gin" + + "git.netcell-it.de/projekte/edgeguard-native/internal/handlers/response" + "git.netcell-it.de/projekte/edgeguard-native/internal/models" +) + +const rollingUpdateStateFile = "/var/lib/edgeguard/rolling-update-state.json" + +const ( + phaseIdle = "idle" + phaseUpdatingSecondary = "updating-secondary" + phaseWaitingSecondary = "waiting-secondary" + phaseUpdatingPrimary = "updating-primary" + phaseFailed = "failed" +) + +// RollingUpdateState hält den Fortschritt des Rolling-Updates. +// Persistiert in rollingUpdateStateFile damit der Status über +// einen kurzen API-Neustart hinaus lesbar bleibt. +type RollingUpdateState struct { + Phase string `json:"phase"` + SecondaryID string `json:"secondary_id,omitempty"` + SecondaryFQDN string `json:"secondary_fqdn,omitempty"` + StartedAt time.Time `json:"started_at,omitempty"` + UpdatedAt time.Time `json:"updated_at"` + Error string `json:"error,omitempty"` +} + +func readRollingUpdateState() RollingUpdateState { + data, err := os.ReadFile(rollingUpdateStateFile) + if err != nil { + return RollingUpdateState{Phase: phaseIdle, UpdatedAt: time.Now()} + } + var s RollingUpdateState + if err := json.Unmarshal(data, &s); err != nil { + return RollingUpdateState{Phase: phaseIdle, UpdatedAt: time.Now()} + } + return s +} + +func writeRollingUpdateState(s RollingUpdateState) { + s.UpdatedAt = time.Now() + data, err := json.Marshal(s) + if err != nil { + slog.Warn("rolling-update: failed to marshal state", "error", err) + return + } + if err := os.WriteFile(rollingUpdateStateFile, data, 0o600); err != nil { + slog.Warn("rolling-update: failed to write state file", "error", err) + } +} + +// RollingUpdate startet den Rolling-Update-Prozess: +// 1. Secondary aktualisieren (via mTLS /agent/cluster/trigger-update) +// 2. Warten bis Secondary neue Version meldet +// 3. Primary (dieser Node) aktualisieren (wie /system/upgrade) +// +// Kein Cluster vorhanden → 409 zurück damit der Client auf /system/upgrade +// ausweichen kann. Wenn bereits ein Rolling-Update läuft → aktuellen State. +func (h *ClusterHandler) RollingUpdate(c *gin.Context) { + if h.Aggregator == nil || h.Store == nil { + c.JSON(http.StatusConflict, gin.H{"error": "no cluster — use /system/upgrade"}) + return + } + + st := readRollingUpdateState() + if st.Phase != phaseIdle && st.Phase != phaseFailed { + response.OK(c, st) + return + } + + nodes, err := h.Store.List(c.Request.Context()) + if err != nil { + response.Internal(c, err) + return + } + var secondary *models.HANode + for i := range nodes { + if nodes[i].ID != h.LocalID { + secondary = &nodes[i] + break + } + } + if secondary == nil { + c.JSON(http.StatusConflict, gin.H{"error": "no peer node — use /system/upgrade"}) + return + } + + newState := RollingUpdateState{ + Phase: phaseUpdatingSecondary, + SecondaryID: secondary.ID, + SecondaryFQDN: secondary.FQDN, + StartedAt: time.Now(), + } + writeRollingUpdateState(newState) + slog.Info("rolling-update: started", "secondary", secondary.FQDN) + + go h.runRollingUpdate(secondary) + + c.JSON(http.StatusAccepted, newState) +} + +// RollingUpdateStatus gibt den aktuellen Rolling-Update-State zurück. +// Wenn phase == "updating-primary" soll der Client auf /system/health +// umschalten (der Primary restartet gleich → State kann nicht mehr +// geschrieben werden). +func (h *ClusterHandler) RollingUpdateStatus(c *gin.Context) { + response.OK(c, readRollingUpdateState()) +} + +func (h *ClusterHandler) runRollingUpdate(secondary *models.HANode) { + ctx := context.Background() + + // 1. Secondary triggern + slog.Info("rolling-update: posting trigger-update to secondary", "fqdn", secondary.FQDN) + result := h.Aggregator.PostPeer(ctx, *secondary, "/agent/cluster/trigger-update") + if !result.OK { + writeRollingUpdateState(RollingUpdateState{ + Phase: phaseFailed, + SecondaryID: secondary.ID, + SecondaryFQDN: secondary.FQDN, + Error: "trigger-update failed: " + result.Err, + }) + slog.Warn("rolling-update: secondary trigger failed", "error", result.Err) + return + } + + // 2. Secondary-Version pollen — der Secondary restartet nach dem + // Upgrade, danach zeigt /agent/cluster/version eine neue Version. + writeRollingUpdateState(RollingUpdateState{ + Phase: phaseWaitingSecondary, + SecondaryID: secondary.ID, + SecondaryFQDN: secondary.FQDN, + }) + slog.Info("rolling-update: waiting for secondary version flip") + + // Kurze Wartezeit damit apt auf dem Secondary erst losläuft + time.Sleep(20 * time.Second) + + deadline := time.Now().Add(10 * time.Minute) + versionFlipped := false + for time.Now().Before(deadline) { + results := h.Aggregator.FanOut(ctx, []models.HANode{*secondary}, "/agent/cluster/version", h.LocalID) + if len(results) > 0 && results[0].OK { + var ver struct { + Version string `json:"version"` + } + if err := json.Unmarshal(results[0].Data, &ver); err == nil { + slog.Info("rolling-update: secondary version", "version", ver.Version, "primary", h.Version) + if ver.Version != h.Version { + versionFlipped = true + break + } + } + } + time.Sleep(10 * time.Second) + } + + if !versionFlipped { + writeRollingUpdateState(RollingUpdateState{ + Phase: phaseFailed, + SecondaryID: secondary.ID, + SecondaryFQDN: secondary.FQDN, + Error: "timeout (10 min) waiting for secondary version flip", + }) + slog.Warn("rolling-update: secondary version flip timeout") + return + } + + // 3. Primary (uns selbst) aktualisieren — identisch zu /system/upgrade + writeRollingUpdateState(RollingUpdateState{ + Phase: phaseUpdatingPrimary, + SecondaryID: secondary.ID, + SecondaryFQDN: secondary.FQDN, + }) + slog.Info("rolling-update: triggering primary self-upgrade") + + const scriptPath = "/var/lib/edgeguard/upgrade.sh" + const script = `#!/bin/bash +set -e +sleep 2 +export DEBIAN_FRONTEND=noninteractive +dpkg --configure -a || true +retry_apt() { + local attempt=0 max=3 wait_for=15 + while [ $attempt -lt $max ]; do + attempt=$((attempt + 1)) + apt-get update -qq || true + if apt-get install -y -qq -o Dpkg::Options::=--force-confold \ + edgeguard-api edgeguard-ui edgeguard; then return 0; fi + [ $attempt -lt $max ] && sleep $wait_for && wait_for=$((wait_for * 2)) + done + return 1 +} +retry_apt +echo "[upgrade] complete" +rm -f /var/lib/edgeguard/upgrade.sh +` + if err := os.WriteFile(scriptPath, []byte(script), 0o755); err != nil { + writeRollingUpdateState(RollingUpdateState{ + Phase: phaseFailed, + SecondaryID: secondary.ID, + SecondaryFQDN: secondary.FQDN, + Error: "write upgrade script: " + err.Error(), + }) + return + } + + const unitName = "edgeguard-upgrade.service" + _ = exec.Command("sudo", "-n", "/usr/bin/systemctl", "reset-failed", unitName).Run() + cmd := exec.Command("sudo", "-n", "/usr/bin/systemd-run", + "--unit="+unitName, + "--description=EdgeGuard rolling-update (primary)", + "--collect", + "bash", scriptPath) + if err := cmd.Run(); err != nil { + writeRollingUpdateState(RollingUpdateState{ + Phase: phaseFailed, + SecondaryID: secondary.ID, + SecondaryFQDN: secondary.FQDN, + Error: "systemd-run failed: " + err.Error(), + }) + slog.Warn("rolling-update: primary systemd-run failed", "error", err) + return + } + // State bleibt "updating-primary" — der Primary restartet gleich. + // UI erkennt Version-Flip via /system/health und schließt den Flow. + slog.Info("rolling-update: primary upgrade dispatched, process will restart") +} diff --git a/management-ui/src/components/UpdateBanner.tsx b/management-ui/src/components/UpdateBanner.tsx index fe27c2c..4c6eecf 100644 --- a/management-ui/src/components/UpdateBanner.tsx +++ b/management-ui/src/components/UpdateBanner.tsx @@ -1,5 +1,5 @@ import { Alert, Button, Popconfirm, Space, Tooltip, message } from 'antd' -import { CloudDownloadOutlined, ReloadOutlined, RocketOutlined } from '@ant-design/icons' +import { CloudDownloadOutlined, ReloadOutlined, RocketOutlined, ClusterOutlined } from '@ant-design/icons' import { useQuery } from '@tanstack/react-query' import { useEffect, useRef, useState } from 'react' import { useTranslation } from 'react-i18next' @@ -20,6 +20,19 @@ interface SystemHealth { status: string; version: string } interface PendingUpdate { pkg: string; installed: string; available: string } +interface ClusterStatus { + mode: string // "single-node" | "cluster" + peers: Array<{ id: string; fqdn: string }> +} + +interface RollingUpdateState { + phase: string // idle | updating-secondary | waiting-secondary | updating-primary | failed + secondary_fqdn: string + secondary_id: string + error?: string + updated_at: string +} + // allUpdates parsed das flache map-Format ({pkg_installed,pkg_available}) // das /system/package-versions zurückliefert. Eines davon ist meist // das meta-Paket "edgeguard" → die "Ziel-Version". @@ -53,6 +66,41 @@ export default function UpdateBanner({ compact = false }: UpdateBannerProps = {} gcTime: 0, }) + const clusterStatus = useQuery({ + queryKey: ['cluster', 'status-update-banner'], + queryFn: async () => { + try { + const r = await apiClient.get('/cluster/status') + return isEnvelope(r.data) ? (r.data.data as ClusterStatus) : null + } catch { + return null + } + }, + refetchInterval: 60_000, + staleTime: 30_000, + }) + + const rollingStatus = useQuery({ + queryKey: ['cluster', 'rolling-update-status'], + queryFn: async () => { + try { + const r = await apiClient.get('/cluster/rolling-update/status') + return isEnvelope(r.data) ? (r.data.data as RollingUpdateState) : null + } catch { + return null + } + }, + refetchInterval: 5_000, + staleTime: 0, + gcTime: 0, + }) + + const isCluster = clusterStatus.data?.mode === 'cluster' + const rollingPhase = rollingStatus.data?.phase ?? 'idle' + const rollingActive = rollingPhase !== 'idle' && rollingPhase !== 'failed' + const secondaryFQDN = rollingStatus.data?.secondary_fqdn ?? '' + + // Normal single-node upgrade state const [upgrading, setUpgrading] = useState(false) const [upgradeElapsed, setUpgradeElapsed] = useState(0) const [forceChecking, setForceChecking] = useState(false) @@ -61,11 +109,61 @@ export default function UpdateBanner({ compact = false }: UpdateBannerProps = {} const installedRef = useRef('') const targetRef = useRef('') + // Rolling update elapsed counter + const [rollingElapsed, setRollingElapsed] = useState(0) + const rollingTickRef = useRef | null>(null) + const rollingPollRef = useRef | null>(null) + useEffect(() => () => { if (upgradePollRef.current) clearInterval(upgradePollRef.current) if (upgradeTickRef.current) clearInterval(upgradeTickRef.current) + if (rollingTickRef.current) clearInterval(rollingTickRef.current) + if (rollingPollRef.current) clearInterval(rollingPollRef.current) }, []) + // Start rolling elapsed timer when rolling becomes active + useEffect(() => { + if (rollingActive && !rollingTickRef.current) { + setRollingElapsed(0) + rollingTickRef.current = setInterval(() => setRollingElapsed(e => e + 1), 1000) + } else if (!rollingActive && rollingTickRef.current) { + clearInterval(rollingTickRef.current) + rollingTickRef.current = null + } + }, [rollingActive]) + + // When phase reaches "updating-primary": switch to health polling + // (primary will restart, state file can't be updated after that) + useEffect(() => { + if (rollingPhase === 'updating-primary' && !rollingPollRef.current) { + const primaryInstalled = installedRef.current + let sawDown = false + rollingPollRef.current = setInterval(async () => { + try { + const res = await apiClient.get('/system/health') + const newV = isEnvelope(res.data) ? (res.data.data as SystemHealth).version : '' + const flipped = newV && primaryInstalled && newV !== primaryInstalled + if (flipped || sawDown) { + if (rollingPollRef.current) clearInterval(rollingPollRef.current) + if (rollingTickRef.current) clearInterval(rollingTickRef.current) + rollingPollRef.current = null + rollingTickRef.current = null + msg.success(t('update.success', { version: targetRef.current })) + setTimeout(() => window.location.reload(), 1500) + } + } catch { + sawDown = true + } + }, 3000) + // Safety timeout + setTimeout(() => { + if (rollingPollRef.current) clearInterval(rollingPollRef.current) + if (rollingTickRef.current) clearInterval(rollingTickRef.current) + window.location.reload() + }, 120_000) + } + }, [rollingPhase, msg, t]) + const data = pkgVersions.data ?? {} const updates = allUpdates(data) const updateAvailable = updates.length > 0 @@ -76,14 +174,8 @@ export default function UpdateBanner({ compact = false }: UpdateBannerProps = {} const forceCheck = async () => { setForceChecking(true) try { - // ?force=1: bypassed den Server-seitigen 5-min-Throttle für - // apt-get update. Ohne den Force-Hint würde der Endpoint - // einfach den letzten Cache zurückliefern (max. 5 min alt) und - // der Button fühlt sich kaputt an. Pattern aus mail-gateway. const r = await apiClient.get('/system/package-versions?force=1') const fresh = (isEnvelope(r.data) ? (r.data.data as PackageVersions) : {}) - // useQuery-Cache mit dem frischen Wert füttern damit der Banner - // sofort umschaltet, ohne auf die nächste 30s-Welle zu warten. void pkgVersions.refetch() const found = allUpdates(fresh).length > 0 msg[found ? 'success' : 'info']( @@ -105,8 +197,6 @@ export default function UpdateBanner({ compact = false }: UpdateBannerProps = {} apiClient.post('/system/upgrade') .then(() => { - // Poll /healthz (kein Auth, robust auch wenn die API gerade - // restartet und Cookie ihre Session nicht erkennt). let sawDown = false upgradePollRef.current = setInterval(async () => { try { @@ -121,14 +211,9 @@ export default function UpdateBanner({ compact = false }: UpdateBannerProps = {} setTimeout(() => window.location.reload(), 1500) } } catch { - // Connection refused / 502 → API restartet. Beim nächsten - // erfolgreichen Poll erkennen wir den Version-Flip. sawDown = true } }, 3000) - // Sicherheits-Timeout: nach 2 Min einfach reload — falls der - // Restart länger braucht als erwartet, kommt die UI in jedem - // Fall wieder hoch. setTimeout(() => { if (upgradePollRef.current) clearInterval(upgradePollRef.current) if (upgradeTickRef.current) clearInterval(upgradeTickRef.current) @@ -144,13 +229,19 @@ export default function UpdateBanner({ compact = false }: UpdateBannerProps = {} }) } + const startRollingUpdate = () => { + installedRef.current = installedVersion + targetRef.current = targetVersion + apiClient.post('/cluster/rolling-update') + .then(() => { + void rollingStatus.refetch() + }) + .catch((e: Error) => { + msg.error(t('update.failed') + ': ' + e.message) + }) + } + if (compact) { - // Compact-Variante: Force-Check-Button für "ich will jetzt prüfen", - // wenn aktuell NICHTS ausstehendes da ist. Sobald ein Update - // verfügbar ist, übernimmt der gelbe Full-Mode-Banner (in - // AppLayout) die Sichtbarkeit — wir blenden den Compact-Button - // dann komplett aus, sonst doppelt-doppelt Info (Befund 2026-05-15: - // "die roten Banner können weg, der gelbe Banner reicht"). if (updateAvailable) { return <>{msgCtx} } @@ -171,7 +262,7 @@ export default function UpdateBanner({ compact = false }: UpdateBannerProps = {} ) } - if (!updateAvailable && !upgrading) { + if (!updateAvailable && !upgrading && !rollingActive) { return <>{msgCtx} } @@ -179,7 +270,7 @@ export default function UpdateBanner({ compact = false }: UpdateBannerProps = {} <> {msgCtx} - {updateAvailable && !upgrading && ( + {updateAvailable && !upgrading && !rollingActive && ( {t('update.checkNow')} - - - + {isCluster ? ( + + + + ) : ( + + + + )} } /> @@ -234,26 +341,10 @@ export default function UpdateBanner({ compact = false }: UpdateBannerProps = {}
- = 5} - active={upgradeElapsed < 5} - label={t('update.stepDownload')} - /> - = 15} - active={upgradeElapsed >= 5 && upgradeElapsed < 15} - label={t('update.stepInstall')} - /> - = 25} - active={upgradeElapsed >= 15 && upgradeElapsed < 25} - label={t('update.stepRestart')} - /> - = 25} - label={t('update.stepVerify')} - /> + = 5} active={upgradeElapsed < 5} label={t('update.stepDownload')} /> + = 15} active={upgradeElapsed >= 5 && upgradeElapsed < 15} label={t('update.stepInstall')} /> + = 25} active={upgradeElapsed >= 15 && upgradeElapsed < 25} label={t('update.stepRestart')} /> + = 25} label={t('update.stepVerify')} />
{upgradeElapsed}s
@@ -261,6 +352,54 @@ export default function UpdateBanner({ compact = false }: UpdateBannerProps = {} )} + + {rollingActive && ( +
+
+
+
+
+
+
+
+ +
+
+ +
{t('update.rollingRunning')}
+
+ v{installedRef.current || '…'} → v{targetRef.current || '…'} +
+ +
+ + + +
+ +
{rollingElapsed}s
+
{t('update.waitHint')}
+ + {rollingStatus.data?.error && ( +
+ {rollingStatus.data.error} +
+ )} +
+
+ )} ) } diff --git a/management-ui/src/i18n/locales/de/common.json b/management-ui/src/i18n/locales/de/common.json index a26f50d..c5e1c96 100644 --- a/management-ui/src/i18n/locales/de/common.json +++ b/management-ui/src/i18n/locales/de/common.json @@ -629,6 +629,7 @@ "node": "Knoten", "status": "Status", "role": "Rolle", + "pgRole": "PG-Rolle", "apiUrl": "API-URL", "configHash": "Config-Hash", "version": "Version", @@ -641,6 +642,30 @@ "uptime": "Uptime", "fetchMs": "Fetch" }, + "pgRole": { + "standalone": "standalone", + "primary": "primary", + "standby": "standby" + }, + "vipCard": { + "title": "Hochverfügbarkeit (VIP / Keepalived)", + "vipAddress": "VIP-Adresse", + "vipAddressHelp": "Virtuelle IP-Adresse die zwischen Nodes wandert (z.B. 89.163.205.10)", + "vipInterface": "Netzwerk-Interface", + "vipInterfaceHelp": "Interface auf dem die VIP gebunden wird (z.B. eth0)", + "vipAuthPass": "VRRP Auth-Passwort", + "vipAuthPassHelp": "Max. 8 Zeichen — Keepalived-Limit. Gleich auf allen Nodes.", + "vrrpRouterId": "VRRP Router-ID", + "vrrpRouterIdHelp": "Muss im Subnetz eindeutig sein (1–255). Standard: 51.", + "saveBtn": "Speichern & Keepalived neu konfigurieren", + "saved": "VIP-Einstellungen gespeichert", + "saveFailed": "Speichern fehlgeschlagen", + "hintTitle": "Nächste Schritte nach dem Speichern", + "hintPrimary": "Auf dem Primary: edgeguard-ctl cluster-init-replication", + "hintStandby": "Auf dem Secondary: edgeguard-ctl cluster-setup-standby ", + "hintKeepalived": "Keepalived auf beiden Nodes: sudo systemctl enable --now keepalived", + "hintFailover": "Bei Failover: edgeguard-ctl promote (auf dem Secondary)" + }, "loadTitle": "Per-Node Resources (mTLS-Aggregator)", "loadEmpty": "Keine Node-Resources verfügbar — Agent-Listener nicht erreichbar?", "certCardTitle": "Cluster-TLS-Zertifikate", @@ -841,7 +866,14 @@ "stepDownload": "Pakete laden", "stepInstall": "Installation", "stepRestart": "Service-Restart", - "stepVerify": "Verifizierung" + "stepVerify": "Verifizierung", + "rollingUpdate": "Rolling Update (Cluster)", + "rollingConfirmTitle": "Rolling Update starten?", + "rollingConfirmDesc": "Der Secondary-Node ({{secondary}}) wird zuerst aktualisiert, danach dieser Primary. Kein Ausfall für den Proxied-Traffic während der Secondary-Phase.", + "rollingRunning": "Rolling Update läuft…", + "rollingStepSecondary": "Secondary aktualisieren ({{fqdn}})", + "rollingStepWaiting": "Warte auf Neustart des Secondary", + "rollingStepPrimary": "Primary aktualisieren (dieser Node)" }, "wg": { "title": "WireGuard", diff --git a/management-ui/src/i18n/locales/en/common.json b/management-ui/src/i18n/locales/en/common.json index e44fd6e..563a6b8 100644 --- a/management-ui/src/i18n/locales/en/common.json +++ b/management-ui/src/i18n/locales/en/common.json @@ -629,6 +629,7 @@ "node": "Node", "status": "Status", "role": "Role", + "pgRole": "PG role", "apiUrl": "API URL", "configHash": "Config hash", "version": "Version", @@ -641,6 +642,30 @@ "uptime": "Uptime", "fetchMs": "Fetch" }, + "pgRole": { + "standalone": "standalone", + "primary": "primary", + "standby": "standby" + }, + "vipCard": { + "title": "High Availability (VIP / Keepalived)", + "vipAddress": "VIP address", + "vipAddressHelp": "Virtual IP address that moves between nodes (e.g. 89.163.205.10)", + "vipInterface": "Network interface", + "vipInterfaceHelp": "Interface to bind the VIP on (e.g. eth0)", + "vipAuthPass": "VRRP auth password", + "vipAuthPassHelp": "Max. 8 characters — Keepalived limit. Same on all nodes.", + "vrrpRouterId": "VRRP router ID", + "vrrpRouterIdHelp": "Must be unique in the subnet (1–255). Default: 51.", + "saveBtn": "Save & reconfigure Keepalived", + "saved": "VIP settings saved", + "saveFailed": "Failed to save", + "hintTitle": "Next steps after saving", + "hintPrimary": "On primary: edgeguard-ctl cluster-init-replication", + "hintStandby": "On secondary: edgeguard-ctl cluster-setup-standby ", + "hintKeepalived": "Keepalived on both nodes: sudo systemctl enable --now keepalived", + "hintFailover": "On failover: edgeguard-ctl promote (on the secondary node)" + }, "loadTitle": "Per-node resources (mTLS aggregator)", "loadEmpty": "No node resources available — agent listener unreachable?", "certCardTitle": "Cluster TLS certificates", @@ -841,7 +866,14 @@ "stepDownload": "Download packages", "stepInstall": "Install", "stepRestart": "Service restart", - "stepVerify": "Verification" + "stepVerify": "Verification", + "rollingUpdate": "Rolling Update (Cluster)", + "rollingConfirmTitle": "Start Rolling Update?", + "rollingConfirmDesc": "The secondary node ({{secondary}}) is updated first, then this primary. No downtime for proxied traffic during the secondary phase.", + "rollingRunning": "Rolling update in progress…", + "rollingStepSecondary": "Updating secondary ({{fqdn}})", + "rollingStepWaiting": "Waiting for secondary restart", + "rollingStepPrimary": "Updating primary (this node)" }, "wg": { "title": "WireGuard",