feat(scheduler): NTP-Sync-Alert — warnt wenn chrony keine Zeitquelle hat — v1.1.109

- runNTPSyncCheck() läuft alle 10 Minuten: ruft chronyc tracking auf,
  prüft ob Stratum 0 / ≥16 oder Reference ID 00000000 — feuert
  Warning mit Fix-Hints (1h dedupe)
- Initial-Check absichtlich NICHT beim Boot, da chrony nach dem Start
  einige Sekunden zur ersten Synchronisation braucht
- ntp.unsync-Trigger in beiden i18n-Dateien dokumentiert

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Debian
2026-05-26 17:40:16 +02:00
parent b9dfab6664
commit feae18772c
6 changed files with 95 additions and 6 deletions

View File

@@ -1 +1 @@
1.1.108 1.1.109

View File

@@ -60,7 +60,7 @@ import (
usersvc "git.netcell-it.de/projekte/edgeguard-native/internal/services/users" usersvc "git.netcell-it.de/projekte/edgeguard-native/internal/services/users"
) )
var version = "1.1.108" var version = "1.1.109"
func main() { func main() {
addr := os.Getenv("EDGEGUARD_API_ADDR") addr := os.Getenv("EDGEGUARD_API_ADDR")

View File

@@ -11,7 +11,7 @@ import (
"git.netcell-it.de/projekte/edgeguard-native/internal/services/setup" "git.netcell-it.de/projekte/edgeguard-native/internal/services/setup"
) )
var version = "1.1.108" var version = "1.1.109"
const usage = `edgeguard-ctl — EdgeGuard CLI const usage = `edgeguard-ctl — EdgeGuard CLI

View File

@@ -16,6 +16,7 @@ import (
"log/slog" "log/slog"
"net" "net"
"os" "os"
"os/exec"
"regexp" "regexp"
"strconv" "strconv"
"strings" "strings"
@@ -40,7 +41,7 @@ import (
"git.netcell-it.de/projekte/edgeguard-native/internal/services/tlscerts" "git.netcell-it.de/projekte/edgeguard-native/internal/services/tlscerts"
) )
var version = "1.1.108" var version = "1.1.109"
const ( const (
// renewTickInterval — how often we re-evaluate expiring certs. // renewTickInterval — how often we re-evaluate expiring certs.
@@ -124,6 +125,14 @@ const (
conntrackCheckInterval = 2 * time.Minute conntrackCheckInterval = 2 * time.Minute
conntrackWarnPct = 80.0 conntrackWarnPct = 80.0
conntrackCriticalPct = 90.0 conntrackCriticalPct = 90.0
// ntpSyncCheckInterval — alle 10 Minuten chronyc tracking aufrufen.
// Keine Sync bedeutet: Uhr driftet → TLS-Cert-Prüfung schlägt fehl
// wenn die Abweichung > Toleranz des Gegenstücks (i.d.R. ±1 min),
// JWT-Ablauf inkonsistent, Cluster-Split-Brain möglich. Dedupe 1h
// damit ein kurzer Upstream-Ausfall (Reboot, DHCP-Pause) keinen
// Alert-Regen produziert.
ntpSyncCheckInterval = 10 * time.Minute
) )
func main() { func main() {
@@ -221,6 +230,12 @@ func main() {
defer conntrackTick.Stop() defer conntrackTick.Stop()
runConntrackCheck(ctx, alertSvc, alertDedupe) runConntrackCheck(ctx, alertSvc, alertDedupe)
ntpSyncTick := time.NewTicker(ntpSyncCheckInterval)
defer ntpSyncTick.Stop()
// Kein Initial-Check bei Boot: chrony braucht nach dem Start
// einige Sekunden bis zur ersten Synchronisation — ein
// sofortiger Check würde immer feuern.
for { for {
select { select {
case <-renewTick.C: case <-renewTick.C:
@@ -250,6 +265,8 @@ func main() {
runMemoryCheck(ctx, alertSvc, alertDedupe) runMemoryCheck(ctx, alertSvc, alertDedupe)
case <-conntrackTick.C: case <-conntrackTick.C:
runConntrackCheck(ctx, alertSvc, alertDedupe) runConntrackCheck(ctx, alertSvc, alertDedupe)
case <-ntpSyncTick.C:
runNTPSyncCheck(ctx, alertSvc, alertDedupe)
} }
} }
} }
@@ -474,6 +491,78 @@ func runConntrackCheck(ctx context.Context, a *alerts.Service, d *dedupe) {
} }
} }
// runNTPSyncCheck ruft chronyc tracking auf und feuert einen Alert wenn
// chrony keine synchronisierte Zeitquelle hat (Stratum 0 oder ≥ 16).
// Zeitdrift > ~1 Minute führt zu TLS-Handshake-Fehlern, JWT-Ablauf-
// Inkonsistenzen und möglichen Cluster-Problemen. Dedupe 1h.
func runNTPSyncCheck(ctx context.Context, a *alerts.Service, d *dedupe) {
if a == nil || d == nil {
return
}
out, err := exec.Command("chronyc", "tracking").Output()
if err != nil {
// chrony nicht installiert oder nicht gestartet — kein Alert,
// weil wir nicht wissen ob chrony hier überhaupt erwartet wird.
return
}
synced, stratum, ref := parseChronyTrackingForAlert(string(out))
if synced {
return
}
const key = "ntp.unsync"
if !d.shouldFire(key) {
return
}
refStr := ref
if refStr == "" {
refStr = "(keine Referenz)"
}
title := fmt.Sprintf("NTP nicht synchronisiert (Stratum %d)", stratum)
desc := fmt.Sprintf(
"chrony hat keine synchronisierte Zeitquelle.\n"+
"Referenz: %s Stratum: %d\n\n"+
"Mögliche Ursachen:\n"+
" • Upstream-NTP-Server nicht erreichbar (UDP/123 blockiert?)\n"+
" • Pool-DNS-Einträge lösen nicht auf\n"+
" • chrony läuft, braucht aber noch Zeit nach Boot (warten)\n\n"+
"Prüfen: chronyc sources -v — chronyc tracking",
refStr, stratum)
if _, err := a.Fire(ctx, "ntp.unsync", alerts.SeverityWarning, title, desc); err != nil {
slog.Warn("scheduler: ntp-sync-check alert fire failed", "error", err)
}
}
// parseChronyTrackingForAlert ist eine schlanke Variante des NTP-Handler-
// Parsers: liefert nur synced/stratum/reference ohne die vollen Felder.
func parseChronyTrackingForAlert(out string) (synced bool, stratum int, reference string) {
for _, line := range strings.Split(out, "\n") {
line = strings.TrimSpace(line)
key, val, ok := strings.Cut(line, ":")
if !ok {
continue
}
key = strings.TrimSpace(key)
val = strings.TrimSpace(val)
switch key {
case "Reference ID":
if i := strings.Index(val, "("); i >= 0 {
reference = strings.Trim(val[i:], "()")
}
if val != "00000000 ()" {
synced = true
}
case "Stratum":
fmt.Sscanf(val, "%d", &stratum)
if stratum > 0 && stratum < 16 {
synced = true
} else if stratum == 0 || stratum >= 16 {
synced = false
}
}
}
return
}
var egBackendRE = regexp.MustCompile(`^eg_backend_(\d+)$`) var egBackendRE = regexp.MustCompile(`^eg_backend_(\d+)$`)
// runBackendDownCheck liest HAProxy-Stats via Admin-Socket und feuert // runBackendDownCheck liest HAProxy-Stats via Admin-Socket und feuert

View File

@@ -1119,7 +1119,7 @@
"title": "Health-Alarme", "title": "Health-Alarme",
"intro": "Notification-Channels für kritische Events. Webhook (Slack/Discord/Teams/Generic-HTTP) oder Email (SMTP). Triggers: cert.expiring (<14 d), cert.renew_failed, backup.failed, license.invalid.", "intro": "Notification-Channels für kritische Events. Webhook (Slack/Discord/Teams/Generic-HTTP) oder Email (SMTP). Triggers: cert.expiring (<14 d), cert.renew_failed, backup.failed, license.invalid.",
"scopeTitle": "Was triggert Alarme?", "scopeTitle": "Was triggert Alarme?",
"scopeDesc": "cert.expiring — TLS-Zertifikat <14 Tage Restzeit (dedupe 12h). cert.renew_failed — ACME-Renewer hat Fails. backup.failed — Scheduled Backup konnte nicht erstellt werden. license.invalid — License-Server liefert valid=false. backend.down — alle Server eines Backend-Pools sind DOWN (2-Min-Check, dedupe 12h). disk.full — Root-Filesystem ≥80% Warnung, ≥90% Critical (stündlich, dedupe 12h). mem.high — RAM-Auslastung ≥85% Warnung, ≥95% Critical (5-Min-Check, dedupe 1h). conntrack.high — Conntrack-Tabelle ≥80% Warnung, ≥90% Critical; bei 100% werden alle neuen Verbindungen lautlos verworfen (2-Min-Check, dedupe 1h).", "scopeDesc": "cert.expiring — TLS-Zertifikat <14 Tage Restzeit (dedupe 12h). cert.renew_failed — ACME-Renewer hat Fails. backup.failed — Scheduled Backup konnte nicht erstellt werden. license.invalid — License-Server liefert valid=false. backend.down — alle Server eines Backend-Pools sind DOWN (2-Min-Check, dedupe 12h). disk.full — Root-Filesystem ≥80% Warnung, ≥90% Critical (stündlich, dedupe 12h). mem.high — RAM-Auslastung ≥85% Warnung, ≥95% Critical (5-Min-Check, dedupe 1h). conntrack.high — Conntrack-Tabelle ≥80% Warnung, ≥90% Critical; bei 100% werden alle neuen Verbindungen lautlos verworfen (2-Min-Check, dedupe 1h). ntp.unsync — chrony hat keine synchronisierte Zeitquelle; Drift führt zu TLS- und JWT-Fehlern (10-Min-Check, dedupe 1h).",
"tabs": { "channels": "Channels", "events": "History" }, "tabs": { "channels": "Channels", "events": "History" },
"add": "Channel hinzufügen", "add": "Channel hinzufügen",
"addTitle": "Notification-Channel anlegen", "addTitle": "Notification-Channel anlegen",

View File

@@ -1119,7 +1119,7 @@
"title": "Health alerts", "title": "Health alerts",
"intro": "Notification channels for critical events. Webhook (Slack/Discord/Teams/generic-HTTP) or email (SMTP). Triggers: cert.expiring (<14 d), cert.renew_failed, backup.failed, license.invalid.", "intro": "Notification channels for critical events. Webhook (Slack/Discord/Teams/generic-HTTP) or email (SMTP). Triggers: cert.expiring (<14 d), cert.renew_failed, backup.failed, license.invalid.",
"scopeTitle": "What triggers alerts?", "scopeTitle": "What triggers alerts?",
"scopeDesc": "cert.expiring — TLS cert <14 days remaining (12 h dedupe). cert.renew_failed — ACME renewer cycle had failures. backup.failed — scheduled backup couldn't run. license.invalid — License server returns valid=false. backend.down — all servers in a backend pool are DOWN (2 min check, 12 h dedupe). disk.full — root filesystem ≥80% warning, ≥90% critical (hourly check, 12 h dedupe). mem.high — RAM usage ≥85% warning, ≥95% critical (5 min check, 1 h dedupe). conntrack.high — conntrack table ≥80% warning, ≥90% critical; at 100% all new connections are silently dropped (2 min check, 1 h dedupe).", "scopeDesc": "cert.expiring — TLS cert <14 days remaining (12 h dedupe). cert.renew_failed — ACME renewer cycle had failures. backup.failed — scheduled backup couldn't run. license.invalid — License server returns valid=false. backend.down — all servers in a backend pool are DOWN (2 min check, 12 h dedupe). disk.full — root filesystem ≥80% warning, ≥90% critical (hourly check, 12 h dedupe). mem.high — RAM usage ≥85% warning, ≥95% critical (5 min check, 1 h dedupe). conntrack.high — conntrack table ≥80% warning, ≥90% critical; at 100% all new connections are silently dropped (2 min check, 1 h dedupe). ntp.unsync — chrony has no synchronized time source; clock drift causes TLS and JWT failures (10 min check, 1 h dedupe).",
"tabs": { "channels": "Channels", "events": "History" }, "tabs": { "channels": "Channels", "events": "History" },
"add": "Add channel", "add": "Add channel",
"addTitle": "Add notification channel", "addTitle": "Add notification channel",