feat(scheduler): NTP-Sync-Alert — warnt wenn chrony keine Zeitquelle hat — v1.1.109
- runNTPSyncCheck() läuft alle 10 Minuten: ruft chronyc tracking auf, prüft ob Stratum 0 / ≥16 oder Reference ID 00000000 — feuert Warning mit Fix-Hints (1h dedupe) - Initial-Check absichtlich NICHT beim Boot, da chrony nach dem Start einige Sekunden zur ersten Synchronisation braucht - ntp.unsync-Trigger in beiden i18n-Dateien dokumentiert Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -60,7 +60,7 @@ import (
|
|||||||
usersvc "git.netcell-it.de/projekte/edgeguard-native/internal/services/users"
|
usersvc "git.netcell-it.de/projekte/edgeguard-native/internal/services/users"
|
||||||
)
|
)
|
||||||
|
|
||||||
var version = "1.1.108"
|
var version = "1.1.109"
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
addr := os.Getenv("EDGEGUARD_API_ADDR")
|
addr := os.Getenv("EDGEGUARD_API_ADDR")
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ import (
|
|||||||
"git.netcell-it.de/projekte/edgeguard-native/internal/services/setup"
|
"git.netcell-it.de/projekte/edgeguard-native/internal/services/setup"
|
||||||
)
|
)
|
||||||
|
|
||||||
var version = "1.1.108"
|
var version = "1.1.109"
|
||||||
|
|
||||||
const usage = `edgeguard-ctl — EdgeGuard CLI
|
const usage = `edgeguard-ctl — EdgeGuard CLI
|
||||||
|
|
||||||
|
|||||||
@@ -16,6 +16,7 @@ import (
|
|||||||
"log/slog"
|
"log/slog"
|
||||||
"net"
|
"net"
|
||||||
"os"
|
"os"
|
||||||
|
"os/exec"
|
||||||
"regexp"
|
"regexp"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
@@ -40,7 +41,7 @@ import (
|
|||||||
"git.netcell-it.de/projekte/edgeguard-native/internal/services/tlscerts"
|
"git.netcell-it.de/projekte/edgeguard-native/internal/services/tlscerts"
|
||||||
)
|
)
|
||||||
|
|
||||||
var version = "1.1.108"
|
var version = "1.1.109"
|
||||||
|
|
||||||
const (
|
const (
|
||||||
// renewTickInterval — how often we re-evaluate expiring certs.
|
// renewTickInterval — how often we re-evaluate expiring certs.
|
||||||
@@ -124,6 +125,14 @@ const (
|
|||||||
conntrackCheckInterval = 2 * time.Minute
|
conntrackCheckInterval = 2 * time.Minute
|
||||||
conntrackWarnPct = 80.0
|
conntrackWarnPct = 80.0
|
||||||
conntrackCriticalPct = 90.0
|
conntrackCriticalPct = 90.0
|
||||||
|
|
||||||
|
// ntpSyncCheckInterval — alle 10 Minuten chronyc tracking aufrufen.
|
||||||
|
// Keine Sync bedeutet: Uhr driftet → TLS-Cert-Prüfung schlägt fehl
|
||||||
|
// wenn die Abweichung > Toleranz des Gegenstücks (i.d.R. ±1 min),
|
||||||
|
// JWT-Ablauf inkonsistent, Cluster-Split-Brain möglich. Dedupe 1h
|
||||||
|
// damit ein kurzer Upstream-Ausfall (Reboot, DHCP-Pause) keinen
|
||||||
|
// Alert-Regen produziert.
|
||||||
|
ntpSyncCheckInterval = 10 * time.Minute
|
||||||
)
|
)
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
@@ -221,6 +230,12 @@ func main() {
|
|||||||
defer conntrackTick.Stop()
|
defer conntrackTick.Stop()
|
||||||
runConntrackCheck(ctx, alertSvc, alertDedupe)
|
runConntrackCheck(ctx, alertSvc, alertDedupe)
|
||||||
|
|
||||||
|
ntpSyncTick := time.NewTicker(ntpSyncCheckInterval)
|
||||||
|
defer ntpSyncTick.Stop()
|
||||||
|
// Kein Initial-Check bei Boot: chrony braucht nach dem Start
|
||||||
|
// einige Sekunden bis zur ersten Synchronisation — ein
|
||||||
|
// sofortiger Check würde immer feuern.
|
||||||
|
|
||||||
for {
|
for {
|
||||||
select {
|
select {
|
||||||
case <-renewTick.C:
|
case <-renewTick.C:
|
||||||
@@ -250,6 +265,8 @@ func main() {
|
|||||||
runMemoryCheck(ctx, alertSvc, alertDedupe)
|
runMemoryCheck(ctx, alertSvc, alertDedupe)
|
||||||
case <-conntrackTick.C:
|
case <-conntrackTick.C:
|
||||||
runConntrackCheck(ctx, alertSvc, alertDedupe)
|
runConntrackCheck(ctx, alertSvc, alertDedupe)
|
||||||
|
case <-ntpSyncTick.C:
|
||||||
|
runNTPSyncCheck(ctx, alertSvc, alertDedupe)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -474,6 +491,78 @@ func runConntrackCheck(ctx context.Context, a *alerts.Service, d *dedupe) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// runNTPSyncCheck ruft chronyc tracking auf und feuert einen Alert wenn
|
||||||
|
// chrony keine synchronisierte Zeitquelle hat (Stratum 0 oder ≥ 16).
|
||||||
|
// Zeitdrift > ~1 Minute führt zu TLS-Handshake-Fehlern, JWT-Ablauf-
|
||||||
|
// Inkonsistenzen und möglichen Cluster-Problemen. Dedupe 1h.
|
||||||
|
func runNTPSyncCheck(ctx context.Context, a *alerts.Service, d *dedupe) {
|
||||||
|
if a == nil || d == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
out, err := exec.Command("chronyc", "tracking").Output()
|
||||||
|
if err != nil {
|
||||||
|
// chrony nicht installiert oder nicht gestartet — kein Alert,
|
||||||
|
// weil wir nicht wissen ob chrony hier überhaupt erwartet wird.
|
||||||
|
return
|
||||||
|
}
|
||||||
|
synced, stratum, ref := parseChronyTrackingForAlert(string(out))
|
||||||
|
if synced {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
const key = "ntp.unsync"
|
||||||
|
if !d.shouldFire(key) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
refStr := ref
|
||||||
|
if refStr == "" {
|
||||||
|
refStr = "(keine Referenz)"
|
||||||
|
}
|
||||||
|
title := fmt.Sprintf("NTP nicht synchronisiert (Stratum %d)", stratum)
|
||||||
|
desc := fmt.Sprintf(
|
||||||
|
"chrony hat keine synchronisierte Zeitquelle.\n"+
|
||||||
|
"Referenz: %s Stratum: %d\n\n"+
|
||||||
|
"Mögliche Ursachen:\n"+
|
||||||
|
" • Upstream-NTP-Server nicht erreichbar (UDP/123 blockiert?)\n"+
|
||||||
|
" • Pool-DNS-Einträge lösen nicht auf\n"+
|
||||||
|
" • chrony läuft, braucht aber noch Zeit nach Boot (warten)\n\n"+
|
||||||
|
"Prüfen: chronyc sources -v — chronyc tracking",
|
||||||
|
refStr, stratum)
|
||||||
|
if _, err := a.Fire(ctx, "ntp.unsync", alerts.SeverityWarning, title, desc); err != nil {
|
||||||
|
slog.Warn("scheduler: ntp-sync-check alert fire failed", "error", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseChronyTrackingForAlert ist eine schlanke Variante des NTP-Handler-
|
||||||
|
// Parsers: liefert nur synced/stratum/reference ohne die vollen Felder.
|
||||||
|
func parseChronyTrackingForAlert(out string) (synced bool, stratum int, reference string) {
|
||||||
|
for _, line := range strings.Split(out, "\n") {
|
||||||
|
line = strings.TrimSpace(line)
|
||||||
|
key, val, ok := strings.Cut(line, ":")
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
key = strings.TrimSpace(key)
|
||||||
|
val = strings.TrimSpace(val)
|
||||||
|
switch key {
|
||||||
|
case "Reference ID":
|
||||||
|
if i := strings.Index(val, "("); i >= 0 {
|
||||||
|
reference = strings.Trim(val[i:], "()")
|
||||||
|
}
|
||||||
|
if val != "00000000 ()" {
|
||||||
|
synced = true
|
||||||
|
}
|
||||||
|
case "Stratum":
|
||||||
|
fmt.Sscanf(val, "%d", &stratum)
|
||||||
|
if stratum > 0 && stratum < 16 {
|
||||||
|
synced = true
|
||||||
|
} else if stratum == 0 || stratum >= 16 {
|
||||||
|
synced = false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
var egBackendRE = regexp.MustCompile(`^eg_backend_(\d+)$`)
|
var egBackendRE = regexp.MustCompile(`^eg_backend_(\d+)$`)
|
||||||
|
|
||||||
// runBackendDownCheck liest HAProxy-Stats via Admin-Socket und feuert
|
// runBackendDownCheck liest HAProxy-Stats via Admin-Socket und feuert
|
||||||
|
|||||||
@@ -1119,7 +1119,7 @@
|
|||||||
"title": "Health-Alarme",
|
"title": "Health-Alarme",
|
||||||
"intro": "Notification-Channels für kritische Events. Webhook (Slack/Discord/Teams/Generic-HTTP) oder Email (SMTP). Triggers: cert.expiring (<14 d), cert.renew_failed, backup.failed, license.invalid.",
|
"intro": "Notification-Channels für kritische Events. Webhook (Slack/Discord/Teams/Generic-HTTP) oder Email (SMTP). Triggers: cert.expiring (<14 d), cert.renew_failed, backup.failed, license.invalid.",
|
||||||
"scopeTitle": "Was triggert Alarme?",
|
"scopeTitle": "Was triggert Alarme?",
|
||||||
"scopeDesc": "cert.expiring — TLS-Zertifikat <14 Tage Restzeit (dedupe 12h). cert.renew_failed — ACME-Renewer hat Fails. backup.failed — Scheduled Backup konnte nicht erstellt werden. license.invalid — License-Server liefert valid=false. backend.down — alle Server eines Backend-Pools sind DOWN (2-Min-Check, dedupe 12h). disk.full — Root-Filesystem ≥80% Warnung, ≥90% Critical (stündlich, dedupe 12h). mem.high — RAM-Auslastung ≥85% Warnung, ≥95% Critical (5-Min-Check, dedupe 1h). conntrack.high — Conntrack-Tabelle ≥80% Warnung, ≥90% Critical; bei 100% werden alle neuen Verbindungen lautlos verworfen (2-Min-Check, dedupe 1h).",
|
"scopeDesc": "cert.expiring — TLS-Zertifikat <14 Tage Restzeit (dedupe 12h). cert.renew_failed — ACME-Renewer hat Fails. backup.failed — Scheduled Backup konnte nicht erstellt werden. license.invalid — License-Server liefert valid=false. backend.down — alle Server eines Backend-Pools sind DOWN (2-Min-Check, dedupe 12h). disk.full — Root-Filesystem ≥80% Warnung, ≥90% Critical (stündlich, dedupe 12h). mem.high — RAM-Auslastung ≥85% Warnung, ≥95% Critical (5-Min-Check, dedupe 1h). conntrack.high — Conntrack-Tabelle ≥80% Warnung, ≥90% Critical; bei 100% werden alle neuen Verbindungen lautlos verworfen (2-Min-Check, dedupe 1h). ntp.unsync — chrony hat keine synchronisierte Zeitquelle; Drift führt zu TLS- und JWT-Fehlern (10-Min-Check, dedupe 1h).",
|
||||||
"tabs": { "channels": "Channels", "events": "History" },
|
"tabs": { "channels": "Channels", "events": "History" },
|
||||||
"add": "Channel hinzufügen",
|
"add": "Channel hinzufügen",
|
||||||
"addTitle": "Notification-Channel anlegen",
|
"addTitle": "Notification-Channel anlegen",
|
||||||
|
|||||||
@@ -1119,7 +1119,7 @@
|
|||||||
"title": "Health alerts",
|
"title": "Health alerts",
|
||||||
"intro": "Notification channels for critical events. Webhook (Slack/Discord/Teams/generic-HTTP) or email (SMTP). Triggers: cert.expiring (<14 d), cert.renew_failed, backup.failed, license.invalid.",
|
"intro": "Notification channels for critical events. Webhook (Slack/Discord/Teams/generic-HTTP) or email (SMTP). Triggers: cert.expiring (<14 d), cert.renew_failed, backup.failed, license.invalid.",
|
||||||
"scopeTitle": "What triggers alerts?",
|
"scopeTitle": "What triggers alerts?",
|
||||||
"scopeDesc": "cert.expiring — TLS cert <14 days remaining (12 h dedupe). cert.renew_failed — ACME renewer cycle had failures. backup.failed — scheduled backup couldn't run. license.invalid — License server returns valid=false. backend.down — all servers in a backend pool are DOWN (2 min check, 12 h dedupe). disk.full — root filesystem ≥80% warning, ≥90% critical (hourly check, 12 h dedupe). mem.high — RAM usage ≥85% warning, ≥95% critical (5 min check, 1 h dedupe). conntrack.high — conntrack table ≥80% warning, ≥90% critical; at 100% all new connections are silently dropped (2 min check, 1 h dedupe).",
|
"scopeDesc": "cert.expiring — TLS cert <14 days remaining (12 h dedupe). cert.renew_failed — ACME renewer cycle had failures. backup.failed — scheduled backup couldn't run. license.invalid — License server returns valid=false. backend.down — all servers in a backend pool are DOWN (2 min check, 12 h dedupe). disk.full — root filesystem ≥80% warning, ≥90% critical (hourly check, 12 h dedupe). mem.high — RAM usage ≥85% warning, ≥95% critical (5 min check, 1 h dedupe). conntrack.high — conntrack table ≥80% warning, ≥90% critical; at 100% all new connections are silently dropped (2 min check, 1 h dedupe). ntp.unsync — chrony has no synchronized time source; clock drift causes TLS and JWT failures (10 min check, 1 h dedupe).",
|
||||||
"tabs": { "channels": "Channels", "events": "History" },
|
"tabs": { "channels": "Channels", "events": "History" },
|
||||||
"add": "Add channel",
|
"add": "Add channel",
|
||||||
"addTitle": "Add notification channel",
|
"addTitle": "Add notification channel",
|
||||||
|
|||||||
Reference in New Issue
Block a user