feat(scheduler): Conntrack-Alert — silent-drop Warnung bevor Tabelle voll läuft — v1.1.108

- runConntrackCheck() liest /proc/sys/net/netfilter/nf_conntrack_count +
  nf_conntrack_max, feuert Warning bei ≥80% und Critical bei ≥90%
  (2-Min-Takt, 1h dedupe pro Severity-Key)
- Bei 100% Auslastung dropped nftables alle neuen Verbindungen lautlos;
  der Alert kommt damit rechtzeitig bevor das passiert
- conntrack.high-Trigger in beiden i18n-Dateien dokumentiert

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Debian
2026-05-26 06:21:16 +02:00
parent ac6c580318
commit b9dfab6664
6 changed files with 79 additions and 6 deletions

View File

@@ -1 +1 @@
1.1.107
1.1.108

View File

@@ -60,7 +60,7 @@ import (
usersvc "git.netcell-it.de/projekte/edgeguard-native/internal/services/users"
)
var version = "1.1.107"
var version = "1.1.108"
func main() {
addr := os.Getenv("EDGEGUARD_API_ADDR")

View File

@@ -11,7 +11,7 @@ import (
"git.netcell-it.de/projekte/edgeguard-native/internal/services/setup"
)
var version = "1.1.107"
var version = "1.1.108"
const usage = `edgeguard-ctl — EdgeGuard CLI

View File

@@ -40,7 +40,7 @@ import (
"git.netcell-it.de/projekte/edgeguard-native/internal/services/tlscerts"
)
var version = "1.1.107"
var version = "1.1.108"
const (
// renewTickInterval — how often we re-evaluate expiring certs.
@@ -115,6 +115,15 @@ const (
memCheckInterval = 5 * time.Minute
memWarnPct = 85.0
memCriticalPct = 95.0
// conntrackCheckInterval — alle 2 Minuten /proc/sys/net/netfilter/
// nf_conntrack_count+max lesen. Eine volle conntrack-Tabelle verwirft
// alle neuen Verbindungen ohne jegliche Rückmeldung. 2-Minuten-Takt
// erlaubt früh zu warnen bevor die Tabelle überläuft.
// Schwellen analog Disk: 80% Warning, 90% Critical. Dedupe 1h.
conntrackCheckInterval = 2 * time.Minute
conntrackWarnPct = 80.0
conntrackCriticalPct = 90.0
)
func main() {
@@ -208,6 +217,10 @@ func main() {
defer memTick.Stop()
runMemoryCheck(ctx, alertSvc, alertDedupe)
conntrackTick := time.NewTicker(conntrackCheckInterval)
defer conntrackTick.Stop()
runConntrackCheck(ctx, alertSvc, alertDedupe)
for {
select {
case <-renewTick.C:
@@ -235,6 +248,8 @@ func main() {
runBackendDownCheck(ctx, pool, alertSvc, alertDedupe)
case <-memTick.C:
runMemoryCheck(ctx, alertSvc, alertDedupe)
case <-conntrackTick.C:
runConntrackCheck(ctx, alertSvc, alertDedupe)
}
}
}
@@ -401,6 +416,64 @@ func runMemoryCheck(ctx context.Context, a *alerts.Service, d *dedupe) {
}
}
// runConntrackCheck liest die conntrack-Tabellen-Belegung aus /proc und
// feuert bei hoher Auslastung. Eine volle conntrack-Tabelle (100%)
// verwirft alle neuen TCP/UDP-Verbindungen ohne ICMP-Rückmeldung —
// der Operator sieht auf der Gegenstelle nur Timeouts.
//
// Schwellen: 80% Warning, 90% Critical (wie Disk, niedriger als RAM weil
// der Impact sofortig ist). Dedupe 1h pro Severity.
func runConntrackCheck(ctx context.Context, a *alerts.Service, d *dedupe) {
if a == nil || d == nil {
return
}
readInt := func(path string) int64 {
b, err := os.ReadFile(path)
if err != nil {
return 0
}
v, _ := strconv.ParseInt(strings.TrimSpace(string(b)), 10, 64)
return v
}
count := readInt("/proc/sys/net/netfilter/nf_conntrack_count")
max := readInt("/proc/sys/net/netfilter/nf_conntrack_max")
if max <= 0 {
return
}
usedPct := float64(count) * 100 / float64(max)
var key, title string
var sev alerts.Severity
switch {
case usedPct >= conntrackCriticalPct:
key = "conntrack.high.critical"
sev = alerts.SeverityError
title = fmt.Sprintf("Conntrack-Tabelle kritisch voll: %.0f%%", usedPct)
case usedPct >= conntrackWarnPct:
key = "conntrack.high.warning"
sev = alerts.SeverityWarning
title = fmt.Sprintf("Conntrack-Tabelle fast voll: %.0f%%", usedPct)
default:
return
}
if !d.shouldFire(key) {
return
}
desc := fmt.Sprintf(
"Conntrack-Auslastung: %.1f%% — %d von %d Einträgen belegt.\n\n"+
"Wenn die Tabelle auf 100%% steigt, werden alle neuen Verbindungen\n"+
"ohne Fehlermeldung verworfen (Silent Drop).\n\n"+
"Maßnahmen:\n"+
" • Zeitweilige Spikes: nf_conntrack_max erhöhen\n"+
" (sysctl net.netfilter.nf_conntrack_max)\n"+
" • Leaks: conntrack -L | sort | head zeigt häufige Quellen\n"+
" • Timeouts reduzieren (z.B. nf_conntrack_tcp_timeout_established)",
usedPct, count, max)
if _, err := a.Fire(ctx, "conntrack.high", sev, title, desc); err != nil {
slog.Warn("scheduler: conntrack-check alert fire failed", "error", err)
}
}
var egBackendRE = regexp.MustCompile(`^eg_backend_(\d+)$`)
// runBackendDownCheck liest HAProxy-Stats via Admin-Socket und feuert

View File

@@ -1119,7 +1119,7 @@
"title": "Health-Alarme",
"intro": "Notification-Channels für kritische Events. Webhook (Slack/Discord/Teams/Generic-HTTP) oder Email (SMTP). Triggers: cert.expiring (<14 d), cert.renew_failed, backup.failed, license.invalid.",
"scopeTitle": "Was triggert Alarme?",
"scopeDesc": "cert.expiring — TLS-Zertifikat <14 Tage Restzeit (dedupe 12h). cert.renew_failed — ACME-Renewer hat Fails. backup.failed — Scheduled Backup konnte nicht erstellt werden. license.invalid — License-Server liefert valid=false. backend.down — alle Server eines Backend-Pools sind DOWN (2-Min-Check, dedupe 12h). disk.full — Root-Filesystem ≥80% Warnung, ≥90% Critical (stündlich, dedupe 12h). mem.high — RAM-Auslastung ≥85% Warnung, ≥95% Critical (5-Min-Check, dedupe 1h).",
"scopeDesc": "cert.expiring — TLS-Zertifikat <14 Tage Restzeit (dedupe 12h). cert.renew_failed — ACME-Renewer hat Fails. backup.failed — Scheduled Backup konnte nicht erstellt werden. license.invalid — License-Server liefert valid=false. backend.down — alle Server eines Backend-Pools sind DOWN (2-Min-Check, dedupe 12h). disk.full — Root-Filesystem ≥80% Warnung, ≥90% Critical (stündlich, dedupe 12h). mem.high — RAM-Auslastung ≥85% Warnung, ≥95% Critical (5-Min-Check, dedupe 1h). conntrack.high — Conntrack-Tabelle ≥80% Warnung, ≥90% Critical; bei 100% werden alle neuen Verbindungen lautlos verworfen (2-Min-Check, dedupe 1h).",
"tabs": { "channels": "Channels", "events": "History" },
"add": "Channel hinzufügen",
"addTitle": "Notification-Channel anlegen",

View File

@@ -1119,7 +1119,7 @@
"title": "Health alerts",
"intro": "Notification channels for critical events. Webhook (Slack/Discord/Teams/generic-HTTP) or email (SMTP). Triggers: cert.expiring (<14 d), cert.renew_failed, backup.failed, license.invalid.",
"scopeTitle": "What triggers alerts?",
"scopeDesc": "cert.expiring — TLS cert <14 days remaining (12 h dedupe). cert.renew_failed — ACME renewer cycle had failures. backup.failed — scheduled backup couldn't run. license.invalid — License server returns valid=false. backend.down — all servers in a backend pool are DOWN (2 min check, 12 h dedupe). disk.full — root filesystem ≥80% warning, ≥90% critical (hourly check, 12 h dedupe). mem.high — RAM usage ≥85% warning, ≥95% critical (5 min check, 1 h dedupe).",
"scopeDesc": "cert.expiring — TLS cert <14 days remaining (12 h dedupe). cert.renew_failed — ACME renewer cycle had failures. backup.failed — scheduled backup couldn't run. license.invalid — License server returns valid=false. backend.down — all servers in a backend pool are DOWN (2 min check, 12 h dedupe). disk.full — root filesystem ≥80% warning, ≥90% critical (hourly check, 12 h dedupe). mem.high — RAM usage ≥85% warning, ≥95% critical (5 min check, 1 h dedupe). conntrack.high — conntrack table ≥80% warning, ≥90% critical; at 100% all new connections are silently dropped (2 min check, 1 h dedupe).",
"tabs": { "channels": "Channels", "events": "History" },
"add": "Add channel",
"addTitle": "Add notification channel",