feat(scheduler): Conntrack-Alert — silent-drop Warnung bevor Tabelle voll läuft — v1.1.108
- runConntrackCheck() liest /proc/sys/net/netfilter/nf_conntrack_count + nf_conntrack_max, feuert Warning bei ≥80% und Critical bei ≥90% (2-Min-Takt, 1h dedupe pro Severity-Key) - Bei 100% Auslastung dropped nftables alle neuen Verbindungen lautlos; der Alert kommt damit rechtzeitig bevor das passiert - conntrack.high-Trigger in beiden i18n-Dateien dokumentiert Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -60,7 +60,7 @@ import (
|
||||
usersvc "git.netcell-it.de/projekte/edgeguard-native/internal/services/users"
|
||||
)
|
||||
|
||||
var version = "1.1.107"
|
||||
var version = "1.1.108"
|
||||
|
||||
func main() {
|
||||
addr := os.Getenv("EDGEGUARD_API_ADDR")
|
||||
|
||||
@@ -11,7 +11,7 @@ import (
|
||||
"git.netcell-it.de/projekte/edgeguard-native/internal/services/setup"
|
||||
)
|
||||
|
||||
var version = "1.1.107"
|
||||
var version = "1.1.108"
|
||||
|
||||
const usage = `edgeguard-ctl — EdgeGuard CLI
|
||||
|
||||
|
||||
@@ -40,7 +40,7 @@ import (
|
||||
"git.netcell-it.de/projekte/edgeguard-native/internal/services/tlscerts"
|
||||
)
|
||||
|
||||
var version = "1.1.107"
|
||||
var version = "1.1.108"
|
||||
|
||||
const (
|
||||
// renewTickInterval — how often we re-evaluate expiring certs.
|
||||
@@ -115,6 +115,15 @@ const (
|
||||
memCheckInterval = 5 * time.Minute
|
||||
memWarnPct = 85.0
|
||||
memCriticalPct = 95.0
|
||||
|
||||
// conntrackCheckInterval — alle 2 Minuten /proc/sys/net/netfilter/
|
||||
// nf_conntrack_count+max lesen. Eine volle conntrack-Tabelle verwirft
|
||||
// alle neuen Verbindungen ohne jegliche Rückmeldung. 2-Minuten-Takt
|
||||
// erlaubt früh zu warnen bevor die Tabelle überläuft.
|
||||
// Schwellen analog Disk: 80% Warning, 90% Critical. Dedupe 1h.
|
||||
conntrackCheckInterval = 2 * time.Minute
|
||||
conntrackWarnPct = 80.0
|
||||
conntrackCriticalPct = 90.0
|
||||
)
|
||||
|
||||
func main() {
|
||||
@@ -208,6 +217,10 @@ func main() {
|
||||
defer memTick.Stop()
|
||||
runMemoryCheck(ctx, alertSvc, alertDedupe)
|
||||
|
||||
conntrackTick := time.NewTicker(conntrackCheckInterval)
|
||||
defer conntrackTick.Stop()
|
||||
runConntrackCheck(ctx, alertSvc, alertDedupe)
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-renewTick.C:
|
||||
@@ -235,6 +248,8 @@ func main() {
|
||||
runBackendDownCheck(ctx, pool, alertSvc, alertDedupe)
|
||||
case <-memTick.C:
|
||||
runMemoryCheck(ctx, alertSvc, alertDedupe)
|
||||
case <-conntrackTick.C:
|
||||
runConntrackCheck(ctx, alertSvc, alertDedupe)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -401,6 +416,64 @@ func runMemoryCheck(ctx context.Context, a *alerts.Service, d *dedupe) {
|
||||
}
|
||||
}
|
||||
|
||||
// runConntrackCheck liest die conntrack-Tabellen-Belegung aus /proc und
|
||||
// feuert bei hoher Auslastung. Eine volle conntrack-Tabelle (100%)
|
||||
// verwirft alle neuen TCP/UDP-Verbindungen ohne ICMP-Rückmeldung —
|
||||
// der Operator sieht auf der Gegenstelle nur Timeouts.
|
||||
//
|
||||
// Schwellen: 80% Warning, 90% Critical (wie Disk, niedriger als RAM weil
|
||||
// der Impact sofortig ist). Dedupe 1h pro Severity.
|
||||
func runConntrackCheck(ctx context.Context, a *alerts.Service, d *dedupe) {
|
||||
if a == nil || d == nil {
|
||||
return
|
||||
}
|
||||
readInt := func(path string) int64 {
|
||||
b, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
v, _ := strconv.ParseInt(strings.TrimSpace(string(b)), 10, 64)
|
||||
return v
|
||||
}
|
||||
count := readInt("/proc/sys/net/netfilter/nf_conntrack_count")
|
||||
max := readInt("/proc/sys/net/netfilter/nf_conntrack_max")
|
||||
if max <= 0 {
|
||||
return
|
||||
}
|
||||
usedPct := float64(count) * 100 / float64(max)
|
||||
|
||||
var key, title string
|
||||
var sev alerts.Severity
|
||||
switch {
|
||||
case usedPct >= conntrackCriticalPct:
|
||||
key = "conntrack.high.critical"
|
||||
sev = alerts.SeverityError
|
||||
title = fmt.Sprintf("Conntrack-Tabelle kritisch voll: %.0f%%", usedPct)
|
||||
case usedPct >= conntrackWarnPct:
|
||||
key = "conntrack.high.warning"
|
||||
sev = alerts.SeverityWarning
|
||||
title = fmt.Sprintf("Conntrack-Tabelle fast voll: %.0f%%", usedPct)
|
||||
default:
|
||||
return
|
||||
}
|
||||
if !d.shouldFire(key) {
|
||||
return
|
||||
}
|
||||
desc := fmt.Sprintf(
|
||||
"Conntrack-Auslastung: %.1f%% — %d von %d Einträgen belegt.\n\n"+
|
||||
"Wenn die Tabelle auf 100%% steigt, werden alle neuen Verbindungen\n"+
|
||||
"ohne Fehlermeldung verworfen (Silent Drop).\n\n"+
|
||||
"Maßnahmen:\n"+
|
||||
" • Zeitweilige Spikes: nf_conntrack_max erhöhen\n"+
|
||||
" (sysctl net.netfilter.nf_conntrack_max)\n"+
|
||||
" • Leaks: conntrack -L | sort | head zeigt häufige Quellen\n"+
|
||||
" • Timeouts reduzieren (z.B. nf_conntrack_tcp_timeout_established)",
|
||||
usedPct, count, max)
|
||||
if _, err := a.Fire(ctx, "conntrack.high", sev, title, desc); err != nil {
|
||||
slog.Warn("scheduler: conntrack-check alert fire failed", "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
var egBackendRE = regexp.MustCompile(`^eg_backend_(\d+)$`)
|
||||
|
||||
// runBackendDownCheck liest HAProxy-Stats via Admin-Socket und feuert
|
||||
|
||||
@@ -1119,7 +1119,7 @@
|
||||
"title": "Health-Alarme",
|
||||
"intro": "Notification-Channels für kritische Events. Webhook (Slack/Discord/Teams/Generic-HTTP) oder Email (SMTP). Triggers: cert.expiring (<14 d), cert.renew_failed, backup.failed, license.invalid.",
|
||||
"scopeTitle": "Was triggert Alarme?",
|
||||
"scopeDesc": "cert.expiring — TLS-Zertifikat <14 Tage Restzeit (dedupe 12h). cert.renew_failed — ACME-Renewer hat Fails. backup.failed — Scheduled Backup konnte nicht erstellt werden. license.invalid — License-Server liefert valid=false. backend.down — alle Server eines Backend-Pools sind DOWN (2-Min-Check, dedupe 12h). disk.full — Root-Filesystem ≥80% Warnung, ≥90% Critical (stündlich, dedupe 12h). mem.high — RAM-Auslastung ≥85% Warnung, ≥95% Critical (5-Min-Check, dedupe 1h).",
|
||||
"scopeDesc": "cert.expiring — TLS-Zertifikat <14 Tage Restzeit (dedupe 12h). cert.renew_failed — ACME-Renewer hat Fails. backup.failed — Scheduled Backup konnte nicht erstellt werden. license.invalid — License-Server liefert valid=false. backend.down — alle Server eines Backend-Pools sind DOWN (2-Min-Check, dedupe 12h). disk.full — Root-Filesystem ≥80% Warnung, ≥90% Critical (stündlich, dedupe 12h). mem.high — RAM-Auslastung ≥85% Warnung, ≥95% Critical (5-Min-Check, dedupe 1h). conntrack.high — Conntrack-Tabelle ≥80% Warnung, ≥90% Critical; bei 100% werden alle neuen Verbindungen lautlos verworfen (2-Min-Check, dedupe 1h).",
|
||||
"tabs": { "channels": "Channels", "events": "History" },
|
||||
"add": "Channel hinzufügen",
|
||||
"addTitle": "Notification-Channel anlegen",
|
||||
|
||||
@@ -1119,7 +1119,7 @@
|
||||
"title": "Health alerts",
|
||||
"intro": "Notification channels for critical events. Webhook (Slack/Discord/Teams/generic-HTTP) or email (SMTP). Triggers: cert.expiring (<14 d), cert.renew_failed, backup.failed, license.invalid.",
|
||||
"scopeTitle": "What triggers alerts?",
|
||||
"scopeDesc": "cert.expiring — TLS cert <14 days remaining (12 h dedupe). cert.renew_failed — ACME renewer cycle had failures. backup.failed — scheduled backup couldn't run. license.invalid — License server returns valid=false. backend.down — all servers in a backend pool are DOWN (2 min check, 12 h dedupe). disk.full — root filesystem ≥80% warning, ≥90% critical (hourly check, 12 h dedupe). mem.high — RAM usage ≥85% warning, ≥95% critical (5 min check, 1 h dedupe).",
|
||||
"scopeDesc": "cert.expiring — TLS cert <14 days remaining (12 h dedupe). cert.renew_failed — ACME renewer cycle had failures. backup.failed — scheduled backup couldn't run. license.invalid — License server returns valid=false. backend.down — all servers in a backend pool are DOWN (2 min check, 12 h dedupe). disk.full — root filesystem ≥80% warning, ≥90% critical (hourly check, 12 h dedupe). mem.high — RAM usage ≥85% warning, ≥95% critical (5 min check, 1 h dedupe). conntrack.high — conntrack table ≥80% warning, ≥90% critical; at 100% all new connections are silently dropped (2 min check, 1 h dedupe).",
|
||||
"tabs": { "channels": "Channels", "events": "History" },
|
||||
"add": "Add channel",
|
||||
"addTitle": "Add notification channel",
|
||||
|
||||
Reference in New Issue
Block a user