#15 waf/alerts.go: AlertWriter.Close() flusht gepufferte Alerts + stoppt die Goroutine (stop/done-Channels, sync.Once, atomic closed; Kanal wird NIE geschlossen → Send racet ohne Panic). Wiring in cmd/edgeguard-waf nach ListenAndServe (graceful shutdown). -race-Test alerts_test.go. #19 handlers/cluster_rollingupdate.go: (a) RollingUpdateStatus mutiert State nicht mehr beim GET — terminale Zustände altern in readRollingUpdateState nach 10 min aus (kein verlorenes 'done' bei parallelen Pollern). (b) State-File via sync.Mutex + configgen.AtomicWrite (kein partieller Read / Race zwischen Handler & Goroutine). (c) Version-Flip wird gegen die VORHER erfasste Secondary-Baseline geprüft statt gegen die Primary-Version (verhindert sofort-/nie-Flip). Bewusst belassen: geteilter upgrade.sh-Pfad ist deterministischer Inhalt + an exakte sudoers-Zeile gebunden → Überschreib-Race benign; MST-Timestamp-Parse locale (Server laufen C-Locale). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -83,6 +83,8 @@ func main() {
|
|||||||
slog.Error("waf: SPOE agent stopped", "error", err)
|
slog.Error("waf: SPOE agent stopped", "error", err)
|
||||||
os.Exit(1)
|
os.Exit(1)
|
||||||
}
|
}
|
||||||
|
// Graceful shutdown (ctx cancelled): gepufferte Alerts flushen.
|
||||||
|
alertWriter.Close()
|
||||||
}
|
}
|
||||||
|
|
||||||
// reload fetches all domain+waf_config pairs from DB and rebuilds engines.
|
// reload fetches all domain+waf_config pairs from DB and rebuilds engines.
|
||||||
|
|||||||
@@ -7,14 +7,20 @@ import (
|
|||||||
"net/http"
|
"net/http"
|
||||||
"os"
|
"os"
|
||||||
"os/exec"
|
"os/exec"
|
||||||
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/gin-gonic/gin"
|
"github.com/gin-gonic/gin"
|
||||||
|
|
||||||
|
"git.netcell-it.de/projekte/edgeguard-native/internal/configgen"
|
||||||
"git.netcell-it.de/projekte/edgeguard-native/internal/handlers/response"
|
"git.netcell-it.de/projekte/edgeguard-native/internal/handlers/response"
|
||||||
"git.netcell-it.de/projekte/edgeguard-native/internal/models"
|
"git.netcell-it.de/projekte/edgeguard-native/internal/models"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// ruStateMu serialisiert Lesen/Schreiben der Rolling-Update-State-Datei
|
||||||
|
// (HTTP-Handler + Hintergrund-Goroutine greifen gleichzeitig zu).
|
||||||
|
var ruStateMu sync.Mutex
|
||||||
|
|
||||||
const rollingUpdateStateFile = "/var/lib/edgeguard/rolling-update-state.json"
|
const rollingUpdateStateFile = "/var/lib/edgeguard/rolling-update-state.json"
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -53,6 +59,8 @@ type RollingUpdateState struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func readRollingUpdateState() RollingUpdateState {
|
func readRollingUpdateState() RollingUpdateState {
|
||||||
|
ruStateMu.Lock()
|
||||||
|
defer ruStateMu.Unlock()
|
||||||
data, err := os.ReadFile(rollingUpdateStateFile)
|
data, err := os.ReadFile(rollingUpdateStateFile)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return RollingUpdateState{Phase: phaseIdle, UpdatedAt: time.Now()}
|
return RollingUpdateState{Phase: phaseIdle, UpdatedAt: time.Now()}
|
||||||
@@ -61,6 +69,13 @@ func readRollingUpdateState() RollingUpdateState {
|
|||||||
if err := json.Unmarshal(data, &s); err != nil {
|
if err := json.Unmarshal(data, &s); err != nil {
|
||||||
return RollingUpdateState{Phase: phaseIdle, UpdatedAt: time.Now()}
|
return RollingUpdateState{Phase: phaseIdle, UpdatedAt: time.Now()}
|
||||||
}
|
}
|
||||||
|
// Terminale Zustände altern aus (statt Mutation-on-GET): nach 10 min
|
||||||
|
// gilt done/failed als idle — so verliert kein paralleler Poller das
|
||||||
|
// Ergebnis und ein alter Stand bleibt nicht hängen.
|
||||||
|
if (s.Phase == phaseDone || s.Phase == phaseFailed) && !s.UpdatedAt.IsZero() &&
|
||||||
|
time.Since(s.UpdatedAt) > 10*time.Minute {
|
||||||
|
return RollingUpdateState{Phase: phaseIdle, UpdatedAt: time.Now()}
|
||||||
|
}
|
||||||
return s
|
return s
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -71,7 +86,10 @@ func writeRollingUpdateState(s RollingUpdateState) {
|
|||||||
slog.Warn("rolling-update: failed to marshal state", "error", err)
|
slog.Warn("rolling-update: failed to marshal state", "error", err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if err := os.WriteFile(rollingUpdateStateFile, data, 0o600); err != nil {
|
ruStateMu.Lock()
|
||||||
|
defer ruStateMu.Unlock()
|
||||||
|
// AtomicWrite (temp+rename) → Leser sehen nie einen partiellen Stand.
|
||||||
|
if err := configgen.AtomicWrite(rollingUpdateStateFile, data, 0o600); err != nil {
|
||||||
slog.Warn("rolling-update: failed to write state file", "error", err)
|
slog.Warn("rolling-update: failed to write state file", "error", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -127,19 +145,24 @@ func (h *ClusterHandler) RollingUpdate(c *gin.Context) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// RollingUpdateStatus gibt den aktuellen Rolling-Update-State zurück.
|
// RollingUpdateStatus gibt den aktuellen Rolling-Update-State zurück.
|
||||||
// Bei phase == "done" wird nach Auslieferung sofort auf idle zurückgesetzt
|
// Read-only — terminale Zustände altern in readRollingUpdateState aus
|
||||||
// damit der nächste Pageload keinen Stale-done vorfindet.
|
// (kein Reset-on-GET mehr, das parallelen Pollern das "done" wegnahm).
|
||||||
func (h *ClusterHandler) RollingUpdateStatus(c *gin.Context) {
|
func (h *ClusterHandler) RollingUpdateStatus(c *gin.Context) {
|
||||||
st := readRollingUpdateState()
|
response.OK(c, readRollingUpdateState())
|
||||||
response.OK(c, st)
|
|
||||||
if st.Phase == phaseDone {
|
|
||||||
writeRollingUpdateState(RollingUpdateState{Phase: phaseIdle})
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *ClusterHandler) runRollingUpdate(secondary *models.HANode) {
|
func (h *ClusterHandler) runRollingUpdate(secondary *models.HANode) {
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
|
|
||||||
|
// Vor dem Upgrade die aktuelle Secondary-Version als Baseline merken —
|
||||||
|
// der Flip wird gegen DIESEN Wert geprüft (nicht gegen die Primary-
|
||||||
|
// Version, die fälschlich sofort/nie „flippen" konnte).
|
||||||
|
baseline := secondaryVersion(ctx, h, secondary)
|
||||||
|
target := baseline
|
||||||
|
if target == "" {
|
||||||
|
target = h.Version // Fallback, falls Baseline nicht abrufbar
|
||||||
|
}
|
||||||
|
|
||||||
// 1. Secondary triggern
|
// 1. Secondary triggern
|
||||||
slog.Info("rolling-update: posting trigger-update to secondary", "fqdn", secondary.FQDN)
|
slog.Info("rolling-update: posting trigger-update to secondary", "fqdn", secondary.FQDN)
|
||||||
result := h.Aggregator.PostPeer(ctx, *secondary, "/agent/cluster/trigger-update")
|
result := h.Aggregator.PostPeer(ctx, *secondary, "/agent/cluster/trigger-update")
|
||||||
@@ -175,8 +198,8 @@ func (h *ClusterHandler) runRollingUpdate(secondary *models.HANode) {
|
|||||||
Version string `json:"version"`
|
Version string `json:"version"`
|
||||||
}
|
}
|
||||||
if err := json.Unmarshal(results[0].Data, &ver); err == nil {
|
if err := json.Unmarshal(results[0].Data, &ver); err == nil {
|
||||||
slog.Info("rolling-update: secondary version", "version", ver.Version, "primary", h.Version)
|
slog.Info("rolling-update: secondary version", "version", ver.Version, "baseline", target)
|
||||||
if ver.Version != h.Version {
|
if ver.Version != "" && ver.Version != target {
|
||||||
versionFlipped = true
|
versionFlipped = true
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
@@ -256,3 +279,17 @@ rm -f /var/lib/edgeguard/upgrade.sh
|
|||||||
// UI erkennt Version-Flip via /system/health und schließt den Flow.
|
// UI erkennt Version-Flip via /system/health und schließt den Flow.
|
||||||
slog.Info("rolling-update: primary upgrade dispatched, process will restart")
|
slog.Info("rolling-update: primary upgrade dispatched, process will restart")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// secondaryVersion holt best-effort die laufende Version des Peers via mTLS.
|
||||||
|
func secondaryVersion(ctx context.Context, h *ClusterHandler, secondary *models.HANode) string {
|
||||||
|
results := h.Aggregator.FanOut(ctx, []models.HANode{*secondary}, "/agent/cluster/version", h.LocalID)
|
||||||
|
if len(results) > 0 && results[0].OK {
|
||||||
|
var ver struct {
|
||||||
|
Version string `json:"version"`
|
||||||
|
}
|
||||||
|
if json.Unmarshal(results[0].Data, &ver) == nil {
|
||||||
|
return ver.Version
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|||||||
@@ -3,6 +3,8 @@ package waf
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/jackc/pgx/v5/pgxpool"
|
"github.com/jackc/pgx/v5/pgxpool"
|
||||||
@@ -26,8 +28,12 @@ type Alert struct {
|
|||||||
// AlertWriter accepts Alert values via a buffered channel and writes
|
// AlertWriter accepts Alert values via a buffered channel and writes
|
||||||
// them to PostgreSQL asynchronously so SPOE handling stays low-latency.
|
// them to PostgreSQL asynchronously so SPOE handling stays low-latency.
|
||||||
type AlertWriter struct {
|
type AlertWriter struct {
|
||||||
pool *pgxpool.Pool
|
pool *pgxpool.Pool
|
||||||
ch chan Alert
|
ch chan Alert
|
||||||
|
stop chan struct{}
|
||||||
|
done chan struct{}
|
||||||
|
closeOnce sync.Once
|
||||||
|
closed atomic.Bool
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewAlertWriter creates an AlertWriter and starts its background goroutine.
|
// NewAlertWriter creates an AlertWriter and starts its background goroutine.
|
||||||
@@ -36,14 +42,19 @@ func NewAlertWriter(pool *pgxpool.Pool, bufSize int) *AlertWriter {
|
|||||||
aw := &AlertWriter{
|
aw := &AlertWriter{
|
||||||
pool: pool,
|
pool: pool,
|
||||||
ch: make(chan Alert, bufSize),
|
ch: make(chan Alert, bufSize),
|
||||||
|
stop: make(chan struct{}),
|
||||||
|
done: make(chan struct{}),
|
||||||
}
|
}
|
||||||
go aw.run()
|
go aw.run()
|
||||||
return aw
|
return aw
|
||||||
}
|
}
|
||||||
|
|
||||||
// Send enqueues an alert. Drops silently if the channel is full to
|
// Send enqueues an alert. Drops silently if the channel is full (or the
|
||||||
// avoid slowing down SPOE request handling.
|
// writer is closing) to avoid slowing down / panicking SPOE handling.
|
||||||
func (aw *AlertWriter) Send(a Alert) {
|
func (aw *AlertWriter) Send(a Alert) {
|
||||||
|
if aw.closed.Load() {
|
||||||
|
return
|
||||||
|
}
|
||||||
select {
|
select {
|
||||||
case aw.ch <- a:
|
case aw.ch <- a:
|
||||||
default:
|
default:
|
||||||
@@ -51,9 +62,34 @@ func (aw *AlertWriter) Send(a Alert) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Close stops the writer and flushes buffered alerts (best-effort).
|
||||||
|
// Safe to call multiple times. The channel is never closed → Send never
|
||||||
|
// panics even if it races with Close.
|
||||||
|
func (aw *AlertWriter) Close() {
|
||||||
|
aw.closeOnce.Do(func() {
|
||||||
|
aw.closed.Store(true)
|
||||||
|
close(aw.stop)
|
||||||
|
})
|
||||||
|
<-aw.done
|
||||||
|
}
|
||||||
|
|
||||||
func (aw *AlertWriter) run() {
|
func (aw *AlertWriter) run() {
|
||||||
for a := range aw.ch {
|
defer close(aw.done)
|
||||||
aw.write(a)
|
for {
|
||||||
|
select {
|
||||||
|
case a := <-aw.ch:
|
||||||
|
aw.write(a)
|
||||||
|
case <-aw.stop:
|
||||||
|
// Restliche gepufferte Alerts noch wegschreiben, dann Ende.
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case a := <-aw.ch:
|
||||||
|
aw.write(a)
|
||||||
|
default:
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
61
internal/waf/alerts_test.go
Normal file
61
internal/waf/alerts_test.go
Normal file
@@ -0,0 +1,61 @@
|
|||||||
|
package waf
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"os"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.netcell-it.de/projekte/edgeguard-native/internal/database"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Beweist Fix #15: AlertWriter.Close() flusht, ist idempotent, und Send/Close
|
||||||
|
// racen ohne Panic (Kanal wird nie geschlossen). Guarded per EG_FWTEST_DSN.
|
||||||
|
func TestAlertWriter_CloseFlush(t *testing.T) {
|
||||||
|
dsn := os.Getenv("EG_FWTEST_DSN")
|
||||||
|
if dsn == "" {
|
||||||
|
t.Skip("set EG_FWTEST_DSN to run the alert-writer test")
|
||||||
|
}
|
||||||
|
ctx := context.Background()
|
||||||
|
var mErr error
|
||||||
|
for i := 0; i < 3; i++ {
|
||||||
|
if mErr = database.Migrate(ctx, dsn); mErr == nil {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
time.Sleep(700 * time.Millisecond)
|
||||||
|
}
|
||||||
|
if mErr != nil {
|
||||||
|
t.Fatalf("migrate: %v", mErr)
|
||||||
|
}
|
||||||
|
pool, err := database.Open(ctx, dsn)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("open: %v", err)
|
||||||
|
}
|
||||||
|
defer pool.Close()
|
||||||
|
|
||||||
|
aw := NewAlertWriter(pool, 64)
|
||||||
|
for i := 0; i < 20; i++ {
|
||||||
|
aw.Send(Alert{Hostname: "t.local", ClientIP: "203.0.113.1", Method: "GET", URI: "/", Action: "detected"})
|
||||||
|
}
|
||||||
|
|
||||||
|
// Send parallel zu Close → darf nicht paniken.
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
for i := 0; i < 10; i++ {
|
||||||
|
wg.Add(1)
|
||||||
|
go func() { defer wg.Done(); aw.Send(Alert{Hostname: "t.local", Action: "detected"}) }()
|
||||||
|
}
|
||||||
|
|
||||||
|
done := make(chan struct{})
|
||||||
|
go func() { aw.Close(); close(done) }()
|
||||||
|
select {
|
||||||
|
case <-done:
|
||||||
|
case <-time.After(10 * time.Second):
|
||||||
|
t.Fatal("Close() did not return (flush hung)")
|
||||||
|
}
|
||||||
|
wg.Wait()
|
||||||
|
|
||||||
|
// Idempotent + Send nach Close ist No-op (kein Panic).
|
||||||
|
aw.Close()
|
||||||
|
aw.Send(Alert{Hostname: "after.local", Action: "detected"})
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user