feat(scheduler): Conntrack-Alert — silent-drop Warnung bevor Tabelle voll läuft — v1.1.108
- runConntrackCheck() liest /proc/sys/net/netfilter/nf_conntrack_count + nf_conntrack_max, feuert Warning bei ≥80% und Critical bei ≥90% (2-Min-Takt, 1h dedupe pro Severity-Key) - Bei 100% Auslastung dropped nftables alle neuen Verbindungen lautlos; der Alert kommt damit rechtzeitig bevor das passiert - conntrack.high-Trigger in beiden i18n-Dateien dokumentiert Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -60,7 +60,7 @@ import (
|
|||||||
usersvc "git.netcell-it.de/projekte/edgeguard-native/internal/services/users"
|
usersvc "git.netcell-it.de/projekte/edgeguard-native/internal/services/users"
|
||||||
)
|
)
|
||||||
|
|
||||||
var version = "1.1.107"
|
var version = "1.1.108"
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
addr := os.Getenv("EDGEGUARD_API_ADDR")
|
addr := os.Getenv("EDGEGUARD_API_ADDR")
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ import (
|
|||||||
"git.netcell-it.de/projekte/edgeguard-native/internal/services/setup"
|
"git.netcell-it.de/projekte/edgeguard-native/internal/services/setup"
|
||||||
)
|
)
|
||||||
|
|
||||||
var version = "1.1.107"
|
var version = "1.1.108"
|
||||||
|
|
||||||
const usage = `edgeguard-ctl — EdgeGuard CLI
|
const usage = `edgeguard-ctl — EdgeGuard CLI
|
||||||
|
|
||||||
|
|||||||
@@ -40,7 +40,7 @@ import (
|
|||||||
"git.netcell-it.de/projekte/edgeguard-native/internal/services/tlscerts"
|
"git.netcell-it.de/projekte/edgeguard-native/internal/services/tlscerts"
|
||||||
)
|
)
|
||||||
|
|
||||||
var version = "1.1.107"
|
var version = "1.1.108"
|
||||||
|
|
||||||
const (
|
const (
|
||||||
// renewTickInterval — how often we re-evaluate expiring certs.
|
// renewTickInterval — how often we re-evaluate expiring certs.
|
||||||
@@ -115,6 +115,15 @@ const (
|
|||||||
memCheckInterval = 5 * time.Minute
|
memCheckInterval = 5 * time.Minute
|
||||||
memWarnPct = 85.0
|
memWarnPct = 85.0
|
||||||
memCriticalPct = 95.0
|
memCriticalPct = 95.0
|
||||||
|
|
||||||
|
// conntrackCheckInterval — alle 2 Minuten /proc/sys/net/netfilter/
|
||||||
|
// nf_conntrack_count+max lesen. Eine volle conntrack-Tabelle verwirft
|
||||||
|
// alle neuen Verbindungen ohne jegliche Rückmeldung. 2-Minuten-Takt
|
||||||
|
// erlaubt früh zu warnen bevor die Tabelle überläuft.
|
||||||
|
// Schwellen analog Disk: 80% Warning, 90% Critical. Dedupe 1h.
|
||||||
|
conntrackCheckInterval = 2 * time.Minute
|
||||||
|
conntrackWarnPct = 80.0
|
||||||
|
conntrackCriticalPct = 90.0
|
||||||
)
|
)
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
@@ -208,6 +217,10 @@ func main() {
|
|||||||
defer memTick.Stop()
|
defer memTick.Stop()
|
||||||
runMemoryCheck(ctx, alertSvc, alertDedupe)
|
runMemoryCheck(ctx, alertSvc, alertDedupe)
|
||||||
|
|
||||||
|
conntrackTick := time.NewTicker(conntrackCheckInterval)
|
||||||
|
defer conntrackTick.Stop()
|
||||||
|
runConntrackCheck(ctx, alertSvc, alertDedupe)
|
||||||
|
|
||||||
for {
|
for {
|
||||||
select {
|
select {
|
||||||
case <-renewTick.C:
|
case <-renewTick.C:
|
||||||
@@ -235,6 +248,8 @@ func main() {
|
|||||||
runBackendDownCheck(ctx, pool, alertSvc, alertDedupe)
|
runBackendDownCheck(ctx, pool, alertSvc, alertDedupe)
|
||||||
case <-memTick.C:
|
case <-memTick.C:
|
||||||
runMemoryCheck(ctx, alertSvc, alertDedupe)
|
runMemoryCheck(ctx, alertSvc, alertDedupe)
|
||||||
|
case <-conntrackTick.C:
|
||||||
|
runConntrackCheck(ctx, alertSvc, alertDedupe)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -401,6 +416,64 @@ func runMemoryCheck(ctx context.Context, a *alerts.Service, d *dedupe) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// runConntrackCheck liest die conntrack-Tabellen-Belegung aus /proc und
|
||||||
|
// feuert bei hoher Auslastung. Eine volle conntrack-Tabelle (100%)
|
||||||
|
// verwirft alle neuen TCP/UDP-Verbindungen ohne ICMP-Rückmeldung —
|
||||||
|
// der Operator sieht auf der Gegenstelle nur Timeouts.
|
||||||
|
//
|
||||||
|
// Schwellen: 80% Warning, 90% Critical (wie Disk, niedriger als RAM weil
|
||||||
|
// der Impact sofortig ist). Dedupe 1h pro Severity.
|
||||||
|
func runConntrackCheck(ctx context.Context, a *alerts.Service, d *dedupe) {
|
||||||
|
if a == nil || d == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
readInt := func(path string) int64 {
|
||||||
|
b, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
v, _ := strconv.ParseInt(strings.TrimSpace(string(b)), 10, 64)
|
||||||
|
return v
|
||||||
|
}
|
||||||
|
count := readInt("/proc/sys/net/netfilter/nf_conntrack_count")
|
||||||
|
max := readInt("/proc/sys/net/netfilter/nf_conntrack_max")
|
||||||
|
if max <= 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
usedPct := float64(count) * 100 / float64(max)
|
||||||
|
|
||||||
|
var key, title string
|
||||||
|
var sev alerts.Severity
|
||||||
|
switch {
|
||||||
|
case usedPct >= conntrackCriticalPct:
|
||||||
|
key = "conntrack.high.critical"
|
||||||
|
sev = alerts.SeverityError
|
||||||
|
title = fmt.Sprintf("Conntrack-Tabelle kritisch voll: %.0f%%", usedPct)
|
||||||
|
case usedPct >= conntrackWarnPct:
|
||||||
|
key = "conntrack.high.warning"
|
||||||
|
sev = alerts.SeverityWarning
|
||||||
|
title = fmt.Sprintf("Conntrack-Tabelle fast voll: %.0f%%", usedPct)
|
||||||
|
default:
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if !d.shouldFire(key) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
desc := fmt.Sprintf(
|
||||||
|
"Conntrack-Auslastung: %.1f%% — %d von %d Einträgen belegt.\n\n"+
|
||||||
|
"Wenn die Tabelle auf 100%% steigt, werden alle neuen Verbindungen\n"+
|
||||||
|
"ohne Fehlermeldung verworfen (Silent Drop).\n\n"+
|
||||||
|
"Maßnahmen:\n"+
|
||||||
|
" • Zeitweilige Spikes: nf_conntrack_max erhöhen\n"+
|
||||||
|
" (sysctl net.netfilter.nf_conntrack_max)\n"+
|
||||||
|
" • Leaks: conntrack -L | sort | head zeigt häufige Quellen\n"+
|
||||||
|
" • Timeouts reduzieren (z.B. nf_conntrack_tcp_timeout_established)",
|
||||||
|
usedPct, count, max)
|
||||||
|
if _, err := a.Fire(ctx, "conntrack.high", sev, title, desc); err != nil {
|
||||||
|
slog.Warn("scheduler: conntrack-check alert fire failed", "error", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
var egBackendRE = regexp.MustCompile(`^eg_backend_(\d+)$`)
|
var egBackendRE = regexp.MustCompile(`^eg_backend_(\d+)$`)
|
||||||
|
|
||||||
// runBackendDownCheck liest HAProxy-Stats via Admin-Socket und feuert
|
// runBackendDownCheck liest HAProxy-Stats via Admin-Socket und feuert
|
||||||
|
|||||||
@@ -1119,7 +1119,7 @@
|
|||||||
"title": "Health-Alarme",
|
"title": "Health-Alarme",
|
||||||
"intro": "Notification-Channels für kritische Events. Webhook (Slack/Discord/Teams/Generic-HTTP) oder Email (SMTP). Triggers: cert.expiring (<14 d), cert.renew_failed, backup.failed, license.invalid.",
|
"intro": "Notification-Channels für kritische Events. Webhook (Slack/Discord/Teams/Generic-HTTP) oder Email (SMTP). Triggers: cert.expiring (<14 d), cert.renew_failed, backup.failed, license.invalid.",
|
||||||
"scopeTitle": "Was triggert Alarme?",
|
"scopeTitle": "Was triggert Alarme?",
|
||||||
"scopeDesc": "cert.expiring — TLS-Zertifikat <14 Tage Restzeit (dedupe 12h). cert.renew_failed — ACME-Renewer hat Fails. backup.failed — Scheduled Backup konnte nicht erstellt werden. license.invalid — License-Server liefert valid=false. backend.down — alle Server eines Backend-Pools sind DOWN (2-Min-Check, dedupe 12h). disk.full — Root-Filesystem ≥80% Warnung, ≥90% Critical (stündlich, dedupe 12h). mem.high — RAM-Auslastung ≥85% Warnung, ≥95% Critical (5-Min-Check, dedupe 1h).",
|
"scopeDesc": "cert.expiring — TLS-Zertifikat <14 Tage Restzeit (dedupe 12h). cert.renew_failed — ACME-Renewer hat Fails. backup.failed — Scheduled Backup konnte nicht erstellt werden. license.invalid — License-Server liefert valid=false. backend.down — alle Server eines Backend-Pools sind DOWN (2-Min-Check, dedupe 12h). disk.full — Root-Filesystem ≥80% Warnung, ≥90% Critical (stündlich, dedupe 12h). mem.high — RAM-Auslastung ≥85% Warnung, ≥95% Critical (5-Min-Check, dedupe 1h). conntrack.high — Conntrack-Tabelle ≥80% Warnung, ≥90% Critical; bei 100% werden alle neuen Verbindungen lautlos verworfen (2-Min-Check, dedupe 1h).",
|
||||||
"tabs": { "channels": "Channels", "events": "History" },
|
"tabs": { "channels": "Channels", "events": "History" },
|
||||||
"add": "Channel hinzufügen",
|
"add": "Channel hinzufügen",
|
||||||
"addTitle": "Notification-Channel anlegen",
|
"addTitle": "Notification-Channel anlegen",
|
||||||
|
|||||||
@@ -1119,7 +1119,7 @@
|
|||||||
"title": "Health alerts",
|
"title": "Health alerts",
|
||||||
"intro": "Notification channels for critical events. Webhook (Slack/Discord/Teams/generic-HTTP) or email (SMTP). Triggers: cert.expiring (<14 d), cert.renew_failed, backup.failed, license.invalid.",
|
"intro": "Notification channels for critical events. Webhook (Slack/Discord/Teams/generic-HTTP) or email (SMTP). Triggers: cert.expiring (<14 d), cert.renew_failed, backup.failed, license.invalid.",
|
||||||
"scopeTitle": "What triggers alerts?",
|
"scopeTitle": "What triggers alerts?",
|
||||||
"scopeDesc": "cert.expiring — TLS cert <14 days remaining (12 h dedupe). cert.renew_failed — ACME renewer cycle had failures. backup.failed — scheduled backup couldn't run. license.invalid — License server returns valid=false. backend.down — all servers in a backend pool are DOWN (2 min check, 12 h dedupe). disk.full — root filesystem ≥80% warning, ≥90% critical (hourly check, 12 h dedupe). mem.high — RAM usage ≥85% warning, ≥95% critical (5 min check, 1 h dedupe).",
|
"scopeDesc": "cert.expiring — TLS cert <14 days remaining (12 h dedupe). cert.renew_failed — ACME renewer cycle had failures. backup.failed — scheduled backup couldn't run. license.invalid — License server returns valid=false. backend.down — all servers in a backend pool are DOWN (2 min check, 12 h dedupe). disk.full — root filesystem ≥80% warning, ≥90% critical (hourly check, 12 h dedupe). mem.high — RAM usage ≥85% warning, ≥95% critical (5 min check, 1 h dedupe). conntrack.high — conntrack table ≥80% warning, ≥90% critical; at 100% all new connections are silently dropped (2 min check, 1 h dedupe).",
|
||||||
"tabs": { "channels": "Channels", "events": "History" },
|
"tabs": { "channels": "Channels", "events": "History" },
|
||||||
"add": "Add channel",
|
"add": "Add channel",
|
||||||
"addTitle": "Add notification channel",
|
"addTitle": "Add notification channel",
|
||||||
|
|||||||
Reference in New Issue
Block a user