diff --git a/VERSION b/VERSION index d0ab9f9..67452e4 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.1.105 +1.1.106 diff --git a/cmd/edgeguard-api/main.go b/cmd/edgeguard-api/main.go index 1c007fb..245a4e5 100644 --- a/cmd/edgeguard-api/main.go +++ b/cmd/edgeguard-api/main.go @@ -60,7 +60,7 @@ import ( usersvc "git.netcell-it.de/projekte/edgeguard-native/internal/services/users" ) -var version = "1.1.105" +var version = "1.1.106" func main() { addr := os.Getenv("EDGEGUARD_API_ADDR") diff --git a/cmd/edgeguard-ctl/main.go b/cmd/edgeguard-ctl/main.go index 26a61b6..2223dba 100644 --- a/cmd/edgeguard-ctl/main.go +++ b/cmd/edgeguard-ctl/main.go @@ -11,7 +11,7 @@ import ( "git.netcell-it.de/projekte/edgeguard-native/internal/services/setup" ) -var version = "1.1.105" +var version = "1.1.106" const usage = `edgeguard-ctl — EdgeGuard CLI diff --git a/cmd/edgeguard-scheduler/main.go b/cmd/edgeguard-scheduler/main.go index 4d75455..3b73b65 100644 --- a/cmd/edgeguard-scheduler/main.go +++ b/cmd/edgeguard-scheduler/main.go @@ -9,12 +9,16 @@ package main import ( + "bufio" "context" "encoding/json" "fmt" "log/slog" + "net" "os" + "regexp" "strconv" + "strings" "syscall" "time" @@ -27,6 +31,7 @@ import ( "git.netcell-it.de/projekte/edgeguard-native/internal/services/acme" "git.netcell-it.de/projekte/edgeguard-native/internal/services/alerts" "git.netcell-it.de/projekte/edgeguard-native/internal/services/audit" + "git.netcell-it.de/projekte/edgeguard-native/internal/services/backends" "git.netcell-it.de/projekte/edgeguard-native/internal/services/backup" backupremote "git.netcell-it.de/projekte/edgeguard-native/internal/services/backup/remote" "git.netcell-it.de/projekte/edgeguard-native/internal/services/certrenewer" @@ -35,7 +40,7 @@ import ( "git.netcell-it.de/projekte/edgeguard-native/internal/services/tlscerts" ) -var version = "1.1.105" +var version = "1.1.106" const ( // renewTickInterval — how often we re-evaluate expiring certs. @@ -97,6 +102,12 @@ const ( // ist passiert nichts. auditCleanupInterval = 24 * time.Hour auditRetentionDays = 90 + + // backendDownCheckInterval — alle 2 Minuten HAProxy-Stats lesen und + // prüfen ob ein Backend komplett ausgefallen ist (alle Server DOWN). + // Dedupe 12h pro Backend → kein Alert-Spam. Frischer Alert wenn das + // Backend nach 12h immer noch unten ist. + backendDownCheckInterval = 2 * time.Minute ) func main() { @@ -180,6 +191,12 @@ func main() { auditTick := time.NewTicker(auditCleanupInterval) defer auditTick.Stop() + backendDownTick := time.NewTicker(backendDownCheckInterval) + defer backendDownTick.Stop() + // Initial-Check direkt beim Start — wenn ein Backend seit dem letzten + // Scheduler-Restart down ist, brauchen wir nicht 2 Minuten zu warten. + runBackendDownCheck(ctx, pool, alertSvc, alertDedupe) + for { select { case <-renewTick.C: @@ -203,6 +220,8 @@ func main() { runDiskCheck(ctx, alertSvc, alertDedupe) case <-auditTick.C: runAuditCleanup(ctx, auditRepo, setupStore) + case <-backendDownTick.C: + runBackendDownCheck(ctx, pool, alertSvc, alertDedupe) } } } @@ -307,6 +326,120 @@ func runDiskCheck(ctx context.Context, a *alerts.Service, d *dedupe) { // remaining hat UND eine lokale CA existiert, wird automatisch neu // signiert. Restart-Hinweis als Info-Alert — wir starten edgeguard-api // nicht selbst neu, das passiert beim nächsten geplanten Update/Reboot. +var egBackendRE = regexp.MustCompile(`^eg_backend_(\d+)$`) + +// runBackendDownCheck liest HAProxy-Stats via Admin-Socket und feuert +// einen Error-Alert für jedes Backend bei dem alle Server DOWN sind +// (und mind. einer einen echten Health-Check hat). Dedupe 12h pro Backend. +func runBackendDownCheck(ctx context.Context, pool *pgxpool.Pool, a *alerts.Service, d *dedupe) { + if a == nil || d == nil { + return + } + conn, err := net.DialTimeout("unix", "/run/haproxy/admin.sock", 2*time.Second) + if err != nil { + // HAProxy läuft nicht oder Socket nicht erreichbar — kein Alert, + // das ist der Dienst selbst nicht der Scheduler. + return + } + defer conn.Close() + _ = conn.SetDeadline(time.Now().Add(3 * time.Second)) + if _, err := conn.Write([]byte("show stat\n")); err != nil { + return + } + + type srvEntry struct{ status string; hasCheck bool } + byBackend := map[string][]srvEntry{} + colIdx := map[string]int{} + scanner := bufio.NewScanner(conn) + scanner.Buffer(make([]byte, 64*1024), 1024*1024) + for scanner.Scan() { + line := scanner.Text() + if line == "" { + continue + } + fields := strings.Split(line, ",") + if strings.HasPrefix(line, "# ") { + fields[0] = strings.TrimPrefix(fields[0], "# ") + for i, name := range fields { + colIdx[name] = i + } + continue + } + px := fieldAt(fields, colIdx["pxname"]) + sv := fieldAt(fields, colIdx["svname"]) + if !strings.HasPrefix(px, "eg_backend_") || sv == "BACKEND" || sv == "FRONTEND" || sv == "" { + continue + } + status := fieldAt(fields, colIdx["status"]) + byBackend[px] = append(byBackend[px], srvEntry{ + status: status, + hasCheck: status != "no check", + }) + } + if len(byBackend) == 0 { + return + } + + // Friendly Backend-Namen aus DB — best-effort, Fehler = anonyme ID. + bkRepo := backends.New(pool) + bklist, _ := bkRepo.List(ctx) + nameOf := func(id int64) string { + for _, b := range bklist { + if b.ID == id { + return b.Name + } + } + return fmt.Sprintf("#%d", id) + } + + for haName, servers := range byBackend { + hasRealCheck, allDown := false, true + for _, s := range servers { + if s.hasCheck { + hasRealCheck = true + } + if s.status == "UP" { + allDown = false + break + } + } + if !hasRealCheck || !allDown { + continue + } + m := egBackendRE.FindStringSubmatch(haName) + if m == nil { + continue + } + id, _ := strconv.ParseInt(m[1], 10, 64) + name := nameOf(id) + + key := "backend.down." + haName + if !d.shouldFire(key) { + continue + } + msg := fmt.Sprintf( + "Alle Server in Backend \"%s\" sind DOWN — HAProxy liefert 503 für alle Requests zu diesem Backend.\n\n"+ + "HAProxy-Backend-Name: %s\n\n"+ + "Nächste Schritte:\n"+ + " • Dienst auf Backend-Host prüfen (systemctl status / docker ps)\n"+ + " • Health-Check-Pfad erreichbar? (curl http://:)\n"+ + " • Firewall-Regeln zwischen EdgeGuard und Backend-Host prüfen", + name, haName) + if _, err := a.Fire(ctx, "backend.down", alerts.SeverityError, + fmt.Sprintf("Backend DOWN: %s", name), msg); err != nil { + slog.Warn("scheduler: backend-down alert fire failed", + "backend", name, "error", err) + } + } +} + +func fieldAt(fields []string, i int) string { + if i <= 0 || i >= len(fields) { + return "" + } + return fields[i] +} + func runClusterCertExpiryCheck(ctx context.Context, a *alerts.Service, d *dedupe) { if a == nil || d == nil { return diff --git a/management-ui/src/i18n/locales/de/common.json b/management-ui/src/i18n/locales/de/common.json index fd8c975..95dbb6c 100644 --- a/management-ui/src/i18n/locales/de/common.json +++ b/management-ui/src/i18n/locales/de/common.json @@ -1116,7 +1116,7 @@ "title": "Health-Alarme", "intro": "Notification-Channels für kritische Events. Webhook (Slack/Discord/Teams/Generic-HTTP) oder Email (SMTP). Triggers: cert.expiring (<14 d), cert.renew_failed, backup.failed, license.invalid.", "scopeTitle": "Was triggert Alarme?", - "scopeDesc": "cert.expiring — TLS-Zertifikat <14 Tage Restzeit (dedupe 12h). cert.renew_failed — ACME-Renewer hat Fails. backup.failed — Scheduled Backup konnte nicht erstellt werden. license.invalid — License-Server liefert valid=false. Mehr Triggers folgen (Backend-Down, Disk-Usage).", + "scopeDesc": "cert.expiring — TLS-Zertifikat <14 Tage Restzeit (dedupe 12h). cert.renew_failed — ACME-Renewer hat Fails. backup.failed — Scheduled Backup konnte nicht erstellt werden. license.invalid — License-Server liefert valid=false. backend.down — alle Server eines Backend-Pools sind DOWN (2-Min-Check, dedupe 12h). disk.full — Root-Filesystem ≥80% Warnung, ≥90% Critical (stündlich, dedupe 12h).", "tabs": { "channels": "Channels", "events": "History" }, "add": "Channel hinzufügen", "addTitle": "Notification-Channel anlegen", diff --git a/management-ui/src/i18n/locales/en/common.json b/management-ui/src/i18n/locales/en/common.json index 25d223e..b479c95 100644 --- a/management-ui/src/i18n/locales/en/common.json +++ b/management-ui/src/i18n/locales/en/common.json @@ -1116,7 +1116,7 @@ "title": "Health alerts", "intro": "Notification channels for critical events. Webhook (Slack/Discord/Teams/generic-HTTP) or email (SMTP). Triggers: cert.expiring (<14 d), cert.renew_failed, backup.failed, license.invalid.", "scopeTitle": "What triggers alerts?", - "scopeDesc": "cert.expiring — TLS cert <14 days remaining (12 h dedupe). cert.renew_failed — ACME renewer cycle had failures. backup.failed — scheduled backup couldn't run. license.invalid — License server returns valid=false. More triggers coming (backend-down, disk usage).", + "scopeDesc": "cert.expiring — TLS cert <14 days remaining (12 h dedupe). cert.renew_failed — ACME renewer cycle had failures. backup.failed — scheduled backup couldn't run. license.invalid — License server returns valid=false. backend.down — all servers in a backend pool are DOWN (2 min check, 12 h dedupe). disk.full — root filesystem ≥80% warning, ≥90% critical (hourly check, 12 h dedupe).", "tabs": { "channels": "Channels", "events": "History" }, "add": "Add channel", "addTitle": "Add notification channel",