feat(cluster): Fix "joining" status + cross-node auth federation

1. preRegisterJoiner now runs SYNCHRONOUSLY before IssueCert responds,
   so nftables @peer_ipv4 is updated before the joiner calls autoRegister.
   Previously it was a goroutine → race → autoRegister failed → "joining"
   forever.

2. Stable node ID for pre-registered placeholder (prenode-{fqdn}) instead
   of time-based ID — re-joins are now idempotent.

3. AgentRegisterPeer sets status="online" immediately (peer proved it is
   online by connecting via mTLS) and deletes the prenode-{fqdn} placeholder.

4. autoRegister retries 3× with 2s delay in case of transient nftables lag.

5. Auth federation: cluster nodes forward failed logins to the primary via
   mTLS /agent/auth/check so users can log in on any node with primary
   credentials (no PG replication needed).
   - SystemHandler.AgentAuthCheck: new endpoint on :8443
   - AuthHandler.checkWithPrimary: mTLS call to primary when local auth fails
   - AuthHandler.WithClusterTLS: inject cluster TLS store
   - startAgentListener now uses the wired systemHdl with Users repo

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

internal/handlers/auth.go

@@ -1,13 +1,19 @@
 package handlers
 
 import (
+	"bytes"
+	"context"
+	"encoding/json"
 	"errors"
+	"io"
+	"log/slog"
 	"net/http"
 	"strings"
 	"time"
 
 	"github.com/gin-gonic/gin"
 
+	"git.netcell-it.de/projekte/edgeguard-native/internal/cluster/clustertls"
 	"git.netcell-it.de/projekte/edgeguard-native/internal/handlers/response"
 	"git.netcell-it.de/projekte/edgeguard-native/internal/services/audit"
 	"git.netcell-it.de/projekte/edgeguard-native/internal/services/session"
@@ -21,11 +27,12 @@ import (
 // the account is auto-migrated into the DB (Upsert) so it shows up in
 // user management from that point on.
 type AuthHandler struct {
-	Setup  *setup.Store
-	Signer *session.Signer
-	Audit  *audit.Repo
-	NodeID string
-	Users  *usersvc.Repo // optional — nil on first boot before DB is ready
+	Setup      *setup.Store
+	Signer     *session.Signer
+	Audit      *audit.Repo
+	NodeID     string
+	Users      *usersvc.Repo     // optional — nil on first boot before DB is ready
+	ClusterTLS *clustertls.Store // optional — enables auth federation on cluster nodes
 }
 
 func NewAuthHandler(s *setup.Store, sig *session.Signer) *AuthHandler {
@@ -46,6 +53,13 @@ func (h *AuthHandler) WithUsers(u *usersvc.Repo) *AuthHandler {
 	return h
 }
 
+// WithClusterTLS enables auth federation: when local auth fails on a
+// cluster node, Login tries the primary via mTLS /agent/auth/check.
+func (h *AuthHandler) WithClusterTLS(store *clustertls.Store) *AuthHandler {
+	h.ClusterTLS = store
+	return h
+}
+
 // Register mounts /auth/login + /logout (public) and /auth/me
 // (gated by requireAuth, passed in as a per-route middleware).
 func (h *AuthHandler) Register(rg *gin.RouterGroup, requireAuth gin.HandlerFunc) {
@@ -115,24 +129,38 @@ func (h *AuthHandler) Login(c *gin.Context) {
 	}
 
 	// 2. Fallback: setup-store admin (backwards compat for pre-DB installs).
-	if actor == "" {
-		if !strings.EqualFold(st.AdminEmail, email) || !st.VerifyAdminPassword(req.Password) {
-			if h.Audit != nil {
-				_ = h.Audit.Log(c.Request.Context(), email, "auth.login.failed",
-					email, gin.H{"reason": "invalid_credentials", "remote": remote}, h.NodeID)
+	if actor == "" && st.AdminEmail != "" {
+		if strings.EqualFold(st.AdminEmail, email) && st.VerifyAdminPassword(req.Password) {
+			actor = st.AdminEmail
+			role = "admin"
+			// Auto-migrate: insert the setup-store admin into the DB so it
+			// shows up in user management from this point on.
+			if h.Users != nil {
+				_, _ = h.Users.Upsert(c.Request.Context(), st.AdminEmail, req.Password, "admin", true)
 			}
-			response.Unauthorized(c, errors.New("invalid_credentials"))
-			return
 		}
-		actor = st.AdminEmail
-		role = "admin"
-		// Auto-migrate: insert the setup-store admin into the DB so it
-		// shows up in user management from this point on.
-		if h.Users != nil {
-			_, _ = h.Users.Upsert(c.Request.Context(), st.AdminEmail, req.Password, "admin", true)
+	}
+
+	// 3. Auth federation: cluster nodes forward failed auth to the primary
+	// via mTLS so users can log in with their primary credentials on any node.
+	if actor == "" && st.IsClusterNode && st.PrimaryFQDN != "" && h.ClusterTLS != nil {
+		if a, r, err := h.checkWithPrimary(c.Request.Context(), st.PrimaryFQDN, email, req.Password); err == nil {
+			actor = a
+			role = r
+		} else {
+			slog.Debug("auth: primary auth check failed", "primary", st.PrimaryFQDN, "error", err)
 		}
 	}
 
+	if actor == "" {
+		if h.Audit != nil {
+			_ = h.Audit.Log(c.Request.Context(), email, "auth.login.failed",
+				email, gin.H{"reason": "invalid_credentials", "remote": remote}, h.NodeID)
+		}
+		response.Unauthorized(c, errors.New("invalid_credentials"))
+		return
+	}
+
 	raw, tok, err := h.Signer.IssueWithRole(actor, role)
 	if err != nil {
 		response.Internal(c, err)
@@ -278,6 +306,48 @@ func (h *AuthHandler) ChangePassword(c *gin.Context) {
 	response.OK(c, gin.H{"ok": true})
 }
 
+// checkWithPrimary verifies credentials against the primary node via mTLS.
+// Returns actor+role on success, error on failure.
+func (h *AuthHandler) checkWithPrimary(ctx context.Context, primaryFQDN, email, password string) (string, string, error) {
+	clientTLS, err := h.ClusterTLS.ClientTLSConfig()
+	if err != nil {
+		return "", "", err
+	}
+	tr := &http.Transport{TLSClientConfig: clientTLS, TLSHandshakeTimeout: 5 * time.Second}
+	client := &http.Client{Transport: tr, Timeout: 8 * time.Second}
+
+	body, _ := json.Marshal(map[string]string{"email": email, "password": password})
+	reqURL := "https://" + primaryFQDN + ":8443/agent/auth/check"
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, reqURL, bytes.NewReader(body))
+	if err != nil {
+		return "", "", err
+	}
+	req.Header.Set("Content-Type", "application/json")
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return "", "", err
+	}
+	defer resp.Body.Close()
+	raw, _ := io.ReadAll(io.LimitReader(resp.Body, 64*1024))
+	if resp.StatusCode != http.StatusOK {
+		return "", "", errors.New("primary: " + strings.TrimSpace(string(raw)))
+	}
+	var env struct {
+		Data struct {
+			Actor string `json:"actor"`
+			Role  string `json:"role"`
+		} `json:"data"`
+	}
+	if err := json.Unmarshal(raw, &env); err != nil {
+		return "", "", err
+	}
+	if env.Data.Actor == "" {
+		return "", "", errors.New("primary returned empty actor")
+	}
+	return env.Data.Actor, env.Data.Role, nil
+}
+
 func setSessionCookie(c *gin.Context, raw string, expUnix int64) {
 	maxAge := int(time.Until(time.Unix(expUnix, 0)).Seconds())
 	if maxAge < 0 {

internal/handlers/cluster.go

@@ -7,6 +7,7 @@ import (
 	"encoding/pem"
 	"fmt"
 	"log/slog"
+	"strings"
 	"time"
 
 	"github.com/gin-gonic/gin"
@@ -392,11 +393,13 @@ func (h *ClusterHandler) IssueCert(c *gin.Context) {
 		return
 	}
 
-	// Pre-register the joining node so its IP lands in @peer_ipv4
-	// immediately — otherwise port 8443 stays blocked and auto-register
-	// via mTLS can never succeed (chicken-and-egg).
+	// Pre-register the joining node SYNCHRONOUSLY before returning the
+	// cert so that nftables @peer_ipv4 already contains the joiner's IP
+	// by the time they call autoRegister on port 8443. A goroutine here
+	// caused a race: cert returned → joiner calls autoRegister → nftables
+	// not updated yet → connection refused → status stays "joining".
 	if h.Store != nil && h.PeerReloader != nil {
-		go h.preRegisterJoiner(clientIP, req.CSR)
+		h.preRegisterJoiner(c.Request.Context(), clientIP, req.CSR)
 	}
 
 	response.OK(c, issueCertResponse{
@@ -409,15 +412,16 @@ func (h *ClusterHandler) IssueCert(c *gin.Context) {
 // (using the CSR CN as FQDN and the HTTP client IP as public_ip), then
 // triggers a firewall reload so @peer_ipv4 contains the new IP before
 // the peer tries to call /agent/cluster/peers on port 8443.
-func (h *ClusterHandler) preRegisterJoiner(clientIP, csrPEM string) {
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+// Uses a stable deterministic ID so re-joins are idempotent.
+func (h *ClusterHandler) preRegisterJoiner(parent context.Context, clientIP, csrPEM string) {
+	ctx, cancel := context.WithTimeout(parent, 10*time.Second)
 	defer cancel()
 
 	fqdn := cnFromCSR(csrPEM)
 	if fqdn == "" {
 		fqdn = "joining-" + clientIP
 	}
-	nodeID := fmt.Sprintf("pre-%x", time.Now().UnixNano())
+	nodeID := fmt.Sprintf("prenode-%s", strings.ReplaceAll(fqdn, ".", "-"))
 
 	n := models.HANode{
 		ID:     nodeID,
@@ -574,7 +578,7 @@ func (h *ClusterHandler) AgentRegisterPeer(c *gin.Context) {
 		FQDN:   req.FQDN,
 		APIURL: req.APIURL,
 		Role:   "peer",
-		Status: "joining",
+		Status: "online", // peer IS online — it just connected via mTLS
 	}
 	if req.PublicIP != "" {
 		v := req.PublicIP
@@ -598,6 +602,13 @@ func (h *ClusterHandler) AgentRegisterPeer(c *gin.Context) {
 		return
 	}
 
+	// Clean up the prenode-{fqdn} placeholder that preRegisterJoiner
+	// created during cert issuance — the real row just took its place.
+	placeholderID := fmt.Sprintf("prenode-%s", strings.ReplaceAll(req.FQDN, ".", "-"))
+	if placeholderID != req.ID {
+		_ = h.Store.Delete(c.Request.Context(), placeholderID)
+	}
+
 	// Firewall-Reload damit peer_ipv4-Set die neue IP aufnimmt. Best-
 	// effort: Fehler loggen, Response weiter durchreichen — der Peer
 	// hat seine Identity erfolgreich registriert, Operator kann manuell

internal/handlers/system.go

@@ -3,6 +3,7 @@ package handlers
 import (
 	"bufio"
 	stdcontext "context"
+	"errors"
 	"log/slog"
 	"net"
 	"net/http"
@@ -22,6 +23,7 @@ import (
 	aptsvc "git.netcell-it.de/projekte/edgeguard-native/internal/services/apt"
 	"git.netcell-it.de/projekte/edgeguard-native/internal/services/audit"
 	"git.netcell-it.de/projekte/edgeguard-native/internal/services/setup"
+	usersvc "git.netcell-it.de/projekte/edgeguard-native/internal/services/users"
 )
 
 // SystemHandler covers /system/health, /system/package-versions,
@@ -44,6 +46,9 @@ type SystemHandler struct {
 	// ExtraReloaders: additional service renderers triggered by
 	// RenderConfigs. Keyed by service name (nftables, wireguard, etc.).
 	ExtraReloaders map[string]func(stdcontext.Context) error
+	// Users: optional — wired after DB pool opens. Used by AgentAuthCheck
+	// so cluster peers can verify credentials against this node's DB.
+	Users *usersvc.Repo
 }
 
 func NewSystemHandler(version string) *SystemHandler {
@@ -87,6 +92,12 @@ func (h *SystemHandler) WithAllReloaders(extras map[string]func(stdcontext.Conte
 	return h
 }
 
+// WithUsers injectet das Users-Repo für AgentAuthCheck.
+func (h *SystemHandler) WithUsers(u *usersvc.Repo) *SystemHandler {
+	h.Users = u
+	return h
+}
+
 func (h *SystemHandler) Register(rg *gin.RouterGroup) {
 	g := rg.Group("/system")
 	g.GET("/health", h.Health)
@@ -124,8 +135,50 @@ func (h *SystemHandler) RegisterAgent(rg *gin.RouterGroup) {
 	g := rg.Group("/agent/system")
 	g.GET("/health", h.Health)
 	g.GET("/resources", h.Resources)
+	// Auth-Federation: Cluster-Nodes verifizieren Credentials gegen diesen
+	// Node via mTLS. Nur über den Agent-Listener (:8443) erreichbar.
+	rg.POST("/agent/auth/check", h.AgentAuthCheck)
 }
 
+// AgentAuthCheck verifies email+password against the local users table
+// and setup.json. Called by cluster nodes over mTLS when local auth fails
+// so users can log in on any cluster node with the primary's credentials.
+func (h *SystemHandler) AgentAuthCheck(c *gin.Context) {
+	var req struct {
+		Email    string `json:"email"`
+		Password string `json:"password"`
+	}
+	if err := c.ShouldBindJSON(&req); err != nil {
+		response.BadRequest(c, err)
+		return
+	}
+	email := strings.TrimSpace(strings.ToLower(req.Email))
+	if email == "" || req.Password == "" {
+		response.Unauthorized(c, errInvalidCreds)
+		return
+	}
+
+	// Check local users table first.
+	if h.Users != nil {
+		u, hash, err := h.Users.FindByEmail(c.Request.Context(), email)
+		if err == nil && u.Active && usersvc.VerifyPassword(hash, req.Password) {
+			response.OK(c, gin.H{"actor": u.Email, "role": u.Role})
+			return
+		}
+	}
+	// Fallback: setup.json admin.
+	if h.Setup != nil {
+		st, _ := h.Setup.Load()
+		if st != nil && strings.EqualFold(st.AdminEmail, email) && st.VerifyAdminPassword(req.Password) {
+			response.OK(c, gin.H{"actor": st.AdminEmail, "role": "admin"})
+			return
+		}
+	}
+	response.Unauthorized(c, errInvalidCreds)
+}
+
+var errInvalidCreds = errors.New("invalid_credentials")
+
 // servicesToCheck is the curated list shown on the dashboard
 // service-health-grid. Order matters (UI renders in this sequence).
 // Each entry is a (label, systemd-unit) pair — label is what the

internal/services/clusterjoin/join.go

@@ -11,6 +11,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"log/slog"
 	"net"
 	"net/http"
 	"net/url"
@@ -122,8 +123,17 @@ func Join(req Request) error {
 		}
 	}
 
-	// Auto-register: best-effort — cert material is already written.
-	_ = autoRegister(primary, tlsDir, req.CommonName, req.Version, req.NodeID)
+	// Auto-register: retry a few times because the primary's nftables may
+	// need a moment to reload even though preRegisterJoiner is now
+	// synchronous on the primary side.
+	for i := 0; i < 3; i++ {
+		if err := autoRegister(primary, tlsDir, req.CommonName, req.Version, req.NodeID); err == nil {
+			break
+		} else if i < 2 {
+			slog.Warn("clusterjoin: autoRegister failed, retrying", "attempt", i+1, "error", err)
+			time.Sleep(2 * time.Second)
+		}
+	}
 	return nil
 }