From b9dfab6664129312bf0ae089b7909aae039d4259 Mon Sep 17 00:00:00 2001 From: Debian Date: Tue, 26 May 2026 06:21:16 +0200 Subject: [PATCH] =?UTF-8?q?feat(scheduler):=20Conntrack-Alert=20=E2=80=94?= =?UTF-8?q?=20silent-drop=20Warnung=20bevor=20Tabelle=20voll=20l=C3=A4uft?= =?UTF-8?q?=20=E2=80=94=20v1.1.108?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - runConntrackCheck() liest /proc/sys/net/netfilter/nf_conntrack_count + nf_conntrack_max, feuert Warning bei ≥80% und Critical bei ≥90% (2-Min-Takt, 1h dedupe pro Severity-Key) - Bei 100% Auslastung dropped nftables alle neuen Verbindungen lautlos; der Alert kommt damit rechtzeitig bevor das passiert - conntrack.high-Trigger in beiden i18n-Dateien dokumentiert Co-Authored-By: Claude Sonnet 4.6 --- VERSION | 2 +- cmd/edgeguard-api/main.go | 2 +- cmd/edgeguard-ctl/main.go | 2 +- cmd/edgeguard-scheduler/main.go | 75 ++++++++++++++++++- management-ui/src/i18n/locales/de/common.json | 2 +- management-ui/src/i18n/locales/en/common.json | 2 +- 6 files changed, 79 insertions(+), 6 deletions(-) diff --git a/VERSION b/VERSION index 8c43a3a..c69d5ce 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.1.107 +1.1.108 diff --git a/cmd/edgeguard-api/main.go b/cmd/edgeguard-api/main.go index 4dc65a0..b05a932 100644 --- a/cmd/edgeguard-api/main.go +++ b/cmd/edgeguard-api/main.go @@ -60,7 +60,7 @@ import ( usersvc "git.netcell-it.de/projekte/edgeguard-native/internal/services/users" ) -var version = "1.1.107" +var version = "1.1.108" func main() { addr := os.Getenv("EDGEGUARD_API_ADDR") diff --git a/cmd/edgeguard-ctl/main.go b/cmd/edgeguard-ctl/main.go index d98a00a..0776139 100644 --- a/cmd/edgeguard-ctl/main.go +++ b/cmd/edgeguard-ctl/main.go @@ -11,7 +11,7 @@ import ( "git.netcell-it.de/projekte/edgeguard-native/internal/services/setup" ) -var version = "1.1.107" +var version = "1.1.108" const usage = `edgeguard-ctl — EdgeGuard CLI diff --git a/cmd/edgeguard-scheduler/main.go b/cmd/edgeguard-scheduler/main.go index d47d1ad..58ee47e 100644 --- a/cmd/edgeguard-scheduler/main.go +++ b/cmd/edgeguard-scheduler/main.go @@ -40,7 +40,7 @@ import ( "git.netcell-it.de/projekte/edgeguard-native/internal/services/tlscerts" ) -var version = "1.1.107" +var version = "1.1.108" const ( // renewTickInterval — how often we re-evaluate expiring certs. @@ -115,6 +115,15 @@ const ( memCheckInterval = 5 * time.Minute memWarnPct = 85.0 memCriticalPct = 95.0 + + // conntrackCheckInterval — alle 2 Minuten /proc/sys/net/netfilter/ + // nf_conntrack_count+max lesen. Eine volle conntrack-Tabelle verwirft + // alle neuen Verbindungen ohne jegliche Rückmeldung. 2-Minuten-Takt + // erlaubt früh zu warnen bevor die Tabelle überläuft. + // Schwellen analog Disk: 80% Warning, 90% Critical. Dedupe 1h. + conntrackCheckInterval = 2 * time.Minute + conntrackWarnPct = 80.0 + conntrackCriticalPct = 90.0 ) func main() { @@ -208,6 +217,10 @@ func main() { defer memTick.Stop() runMemoryCheck(ctx, alertSvc, alertDedupe) + conntrackTick := time.NewTicker(conntrackCheckInterval) + defer conntrackTick.Stop() + runConntrackCheck(ctx, alertSvc, alertDedupe) + for { select { case <-renewTick.C: @@ -235,6 +248,8 @@ func main() { runBackendDownCheck(ctx, pool, alertSvc, alertDedupe) case <-memTick.C: runMemoryCheck(ctx, alertSvc, alertDedupe) + case <-conntrackTick.C: + runConntrackCheck(ctx, alertSvc, alertDedupe) } } } @@ -401,6 +416,64 @@ func runMemoryCheck(ctx context.Context, a *alerts.Service, d *dedupe) { } } +// runConntrackCheck liest die conntrack-Tabellen-Belegung aus /proc und +// feuert bei hoher Auslastung. Eine volle conntrack-Tabelle (100%) +// verwirft alle neuen TCP/UDP-Verbindungen ohne ICMP-Rückmeldung — +// der Operator sieht auf der Gegenstelle nur Timeouts. +// +// Schwellen: 80% Warning, 90% Critical (wie Disk, niedriger als RAM weil +// der Impact sofortig ist). Dedupe 1h pro Severity. +func runConntrackCheck(ctx context.Context, a *alerts.Service, d *dedupe) { + if a == nil || d == nil { + return + } + readInt := func(path string) int64 { + b, err := os.ReadFile(path) + if err != nil { + return 0 + } + v, _ := strconv.ParseInt(strings.TrimSpace(string(b)), 10, 64) + return v + } + count := readInt("/proc/sys/net/netfilter/nf_conntrack_count") + max := readInt("/proc/sys/net/netfilter/nf_conntrack_max") + if max <= 0 { + return + } + usedPct := float64(count) * 100 / float64(max) + + var key, title string + var sev alerts.Severity + switch { + case usedPct >= conntrackCriticalPct: + key = "conntrack.high.critical" + sev = alerts.SeverityError + title = fmt.Sprintf("Conntrack-Tabelle kritisch voll: %.0f%%", usedPct) + case usedPct >= conntrackWarnPct: + key = "conntrack.high.warning" + sev = alerts.SeverityWarning + title = fmt.Sprintf("Conntrack-Tabelle fast voll: %.0f%%", usedPct) + default: + return + } + if !d.shouldFire(key) { + return + } + desc := fmt.Sprintf( + "Conntrack-Auslastung: %.1f%% — %d von %d Einträgen belegt.\n\n"+ + "Wenn die Tabelle auf 100%% steigt, werden alle neuen Verbindungen\n"+ + "ohne Fehlermeldung verworfen (Silent Drop).\n\n"+ + "Maßnahmen:\n"+ + " • Zeitweilige Spikes: nf_conntrack_max erhöhen\n"+ + " (sysctl net.netfilter.nf_conntrack_max)\n"+ + " • Leaks: conntrack -L | sort | head zeigt häufige Quellen\n"+ + " • Timeouts reduzieren (z.B. nf_conntrack_tcp_timeout_established)", + usedPct, count, max) + if _, err := a.Fire(ctx, "conntrack.high", sev, title, desc); err != nil { + slog.Warn("scheduler: conntrack-check alert fire failed", "error", err) + } +} + var egBackendRE = regexp.MustCompile(`^eg_backend_(\d+)$`) // runBackendDownCheck liest HAProxy-Stats via Admin-Socket und feuert diff --git a/management-ui/src/i18n/locales/de/common.json b/management-ui/src/i18n/locales/de/common.json index 1fde6a5..1ef7c76 100644 --- a/management-ui/src/i18n/locales/de/common.json +++ b/management-ui/src/i18n/locales/de/common.json @@ -1119,7 +1119,7 @@ "title": "Health-Alarme", "intro": "Notification-Channels für kritische Events. Webhook (Slack/Discord/Teams/Generic-HTTP) oder Email (SMTP). Triggers: cert.expiring (<14 d), cert.renew_failed, backup.failed, license.invalid.", "scopeTitle": "Was triggert Alarme?", - "scopeDesc": "cert.expiring — TLS-Zertifikat <14 Tage Restzeit (dedupe 12h). cert.renew_failed — ACME-Renewer hat Fails. backup.failed — Scheduled Backup konnte nicht erstellt werden. license.invalid — License-Server liefert valid=false. backend.down — alle Server eines Backend-Pools sind DOWN (2-Min-Check, dedupe 12h). disk.full — Root-Filesystem ≥80% Warnung, ≥90% Critical (stündlich, dedupe 12h). mem.high — RAM-Auslastung ≥85% Warnung, ≥95% Critical (5-Min-Check, dedupe 1h).", + "scopeDesc": "cert.expiring — TLS-Zertifikat <14 Tage Restzeit (dedupe 12h). cert.renew_failed — ACME-Renewer hat Fails. backup.failed — Scheduled Backup konnte nicht erstellt werden. license.invalid — License-Server liefert valid=false. backend.down — alle Server eines Backend-Pools sind DOWN (2-Min-Check, dedupe 12h). disk.full — Root-Filesystem ≥80% Warnung, ≥90% Critical (stündlich, dedupe 12h). mem.high — RAM-Auslastung ≥85% Warnung, ≥95% Critical (5-Min-Check, dedupe 1h). conntrack.high — Conntrack-Tabelle ≥80% Warnung, ≥90% Critical; bei 100% werden alle neuen Verbindungen lautlos verworfen (2-Min-Check, dedupe 1h).", "tabs": { "channels": "Channels", "events": "History" }, "add": "Channel hinzufügen", "addTitle": "Notification-Channel anlegen", diff --git a/management-ui/src/i18n/locales/en/common.json b/management-ui/src/i18n/locales/en/common.json index d36c692..8192eec 100644 --- a/management-ui/src/i18n/locales/en/common.json +++ b/management-ui/src/i18n/locales/en/common.json @@ -1119,7 +1119,7 @@ "title": "Health alerts", "intro": "Notification channels for critical events. Webhook (Slack/Discord/Teams/generic-HTTP) or email (SMTP). Triggers: cert.expiring (<14 d), cert.renew_failed, backup.failed, license.invalid.", "scopeTitle": "What triggers alerts?", - "scopeDesc": "cert.expiring — TLS cert <14 days remaining (12 h dedupe). cert.renew_failed — ACME renewer cycle had failures. backup.failed — scheduled backup couldn't run. license.invalid — License server returns valid=false. backend.down — all servers in a backend pool are DOWN (2 min check, 12 h dedupe). disk.full — root filesystem ≥80% warning, ≥90% critical (hourly check, 12 h dedupe). mem.high — RAM usage ≥85% warning, ≥95% critical (5 min check, 1 h dedupe).", + "scopeDesc": "cert.expiring — TLS cert <14 days remaining (12 h dedupe). cert.renew_failed — ACME renewer cycle had failures. backup.failed — scheduled backup couldn't run. license.invalid — License server returns valid=false. backend.down — all servers in a backend pool are DOWN (2 min check, 12 h dedupe). disk.full — root filesystem ≥80% warning, ≥90% critical (hourly check, 12 h dedupe). mem.high — RAM usage ≥85% warning, ≥95% critical (5 min check, 1 h dedupe). conntrack.high — conntrack table ≥80% warning, ≥90% critical; at 100% all new connections are silently dropped (2 min check, 1 h dedupe).", "tabs": { "channels": "Channels", "events": "History" }, "add": "Add channel", "addTitle": "Add notification channel",