feat: per-Backend timeout server + Alerts-Deeplink + Retention + Cert-Prune — v1.3.4
- feat(backends): per-Backend `server_timeout_seconds` (nullable, Default 60s). Rendert `timeout server <N>s` im HAProxy-Backend-Block — für langsam antwortende Upstreams (KI-/Inferenz-Server mit gepufferter Antwort). Migration 0044 (+CHECK 1..86400), Model/Repo/Template/UI + Render-Test. - fix(ui): Dashboard-Alert-Karte verlinkt auf /alerts?tab=events; Alerts-Seite respektiert ?tab= (Deeplink landete bisher auf leerem Channels-Tab). - feat(scheduler): alert_events-Retention (90d) im täglichen Cleanup-Tick — Schutz vor unbounded growth der node-lokalen Health-Event-History. - fix(cluster): Cert-Sync prunt jetzt lokale .pem die der Primary nicht mehr hat (Waisen gelöschter Domains); schützt _default.pem + eigenen Node-Cert. - fix(security): x/text v0.37→v0.39 (GO-2026-5970, Infinite-Loop; via ACME+goose aktiv aufgerufen — govulncheck-Release-Gate). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
27
internal/database/migrations/0044_backend_server_timeout.sql
Normal file
27
internal/database/migrations/0044_backend_server_timeout.sql
Normal file
@@ -0,0 +1,27 @@
|
||||
-- +goose Up
|
||||
-- +goose StatementBegin
|
||||
|
||||
-- Per-Backend `timeout server` (Sekunden). NULL = defaults-Timeout (60s,
|
||||
-- siehe haproxy.cfg.tpl). Gedacht für Upstreams die LANGE für die Antwort
|
||||
-- brauchen und dabei NICHT streamen — z. B. KI-/Inferenz-Server, die eine
|
||||
-- gepufferte Antwort erst nach Minuten schicken. Ohne Override kappt der
|
||||
-- 60s-defaults-Timeout diese Requests.
|
||||
--
|
||||
-- Bewusst NULL-per-default: Backends ohne Langläufer-Workload behalten den
|
||||
-- kurzen Timeout (Connection-Hygiene / Slowloris-Schutz, vgl. v1.3.2).
|
||||
-- Der Renderer setzt `timeout server <N>s` NUR wenn ein Wert gesetzt ist.
|
||||
--
|
||||
-- CHECK 1..86400: mind. 1s, max. 24h — verhindert 0/negativ (würde HAProxy-
|
||||
-- Config sprengen bzw. „unendlich" bedeuten) und absurd hohe Werte.
|
||||
ALTER TABLE backends
|
||||
ADD COLUMN IF NOT EXISTS server_timeout_seconds INTEGER
|
||||
CONSTRAINT backends_server_timeout_range
|
||||
CHECK (server_timeout_seconds IS NULL
|
||||
OR (server_timeout_seconds BETWEEN 1 AND 86400));
|
||||
|
||||
-- +goose StatementEnd
|
||||
|
||||
-- +goose Down
|
||||
-- +goose StatementBegin
|
||||
ALTER TABLE backends DROP COLUMN IF EXISTS server_timeout_seconds;
|
||||
-- +goose StatementEnd
|
||||
@@ -109,6 +109,40 @@ func SyncTLSCertsFromPrimary(ctx context.Context, pool *pgxpool.Pool, agg *aggre
|
||||
slog.Info("cert-sync: updated", "file", name)
|
||||
}
|
||||
|
||||
// Prune: lokale .pem entfernen, die der Primary NICHT (mehr) hat.
|
||||
// Ohne diesen Schritt bleiben Zertifikate gelöschter Domains auf dem
|
||||
// Secondary als Waisen liegen — der Sync oben ist write-only, „nicht
|
||||
// mitgeschickt" ≠ „gelöscht". Geschützt bleiben:
|
||||
// _default.pem — Self-Signed-Fallback
|
||||
// <lokaler-FQDN>.pem — eigener Node-Cert (steht NICHT im Primary-Payload)
|
||||
// Nur prunen wenn der Payload nicht leer ist — Schutz gegen ein
|
||||
// versehentliches Leerräumen bei unvollständiger Primary-Antwort.
|
||||
if len(payload.Certs) > 0 {
|
||||
protected := map[string]bool{"_default.pem": true}
|
||||
var localFQDN string
|
||||
if err := pool.QueryRow(ctx,
|
||||
`SELECT fqdn FROM ha_nodes WHERE id = $1`, localID).Scan(&localFQDN); err == nil && localFQDN != "" {
|
||||
protected[localFQDN+".pem"] = true
|
||||
}
|
||||
if entries, err := os.ReadDir(tlsCertDir); err == nil {
|
||||
for _, e := range entries {
|
||||
name := e.Name()
|
||||
if e.IsDir() || !strings.HasSuffix(name, ".pem") || protected[name] {
|
||||
continue
|
||||
}
|
||||
if _, ok := payload.Certs[name]; ok {
|
||||
continue // vom Primary gepflegt — behalten
|
||||
}
|
||||
if err := os.Remove(filepath.Join(tlsCertDir, name)); err != nil {
|
||||
slog.Warn("cert-sync: prune failed", "file", name, "error", err)
|
||||
continue
|
||||
}
|
||||
changed = true
|
||||
slog.Info("cert-sync: pruned orphan", "file", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if changed {
|
||||
if err := exec.Command("sudo", "-n", "/usr/bin/systemctl", "reload", "haproxy.service").Run(); err != nil {
|
||||
slog.Warn("cert-sync: haproxy reload failed", "error", err)
|
||||
|
||||
@@ -226,6 +226,11 @@ backend eg_backend_{{$b.ID}}
|
||||
{{- if $b.WebSocket}}
|
||||
timeout tunnel 1h
|
||||
{{- end}}
|
||||
{{- if $b.ServerTimeoutSeconds}}
|
||||
# Override des defaults-`timeout server 60s` für langsame Upstreams
|
||||
# (z. B. KI-Server mit gepufferter Antwort). Wert per Backend gepflegt.
|
||||
timeout server {{$b.ServerTimeoutSeconds}}s
|
||||
{{- end}}
|
||||
{{- if $b.HealthCheckPath}}
|
||||
option httpchk
|
||||
http-check send meth GET uri {{$b.HealthCheckPath}}
|
||||
|
||||
@@ -554,6 +554,44 @@ func TestRender_WebSocketEmitsTunnelTimeout(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRender_ServerTimeoutOverride(t *testing.T) {
|
||||
tmo := 300
|
||||
v := View{
|
||||
Backends: []BackendView{
|
||||
{
|
||||
Backend: models.Backend{ID: 11, Name: "ai", Scheme: "http",
|
||||
LBAlgorithm: "roundrobin", ServerTimeoutSeconds: &tmo, Active: true},
|
||||
Servers: []models.BackendServer{
|
||||
{BackendID: 11, Name: "ai-1", Address: "10.0.5.30", Port: 8000, Weight: 100, Active: true},
|
||||
},
|
||||
},
|
||||
{
|
||||
Backend: models.Backend{ID: 12, Name: "web", Scheme: "http",
|
||||
LBAlgorithm: "roundrobin", Active: true},
|
||||
Servers: []models.BackendServer{
|
||||
{BackendID: 12, Name: "web-1", Address: "10.0.5.31", Port: 8080, Weight: 100, Active: true},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
out := renderView(t, v)
|
||||
idxAI := strings.Index(out, "backend eg_backend_11")
|
||||
idxWeb := strings.Index(out, "backend eg_backend_12")
|
||||
if idxAI < 0 || idxWeb < 0 {
|
||||
t.Fatalf("backend sections missing in output:\n%s", out)
|
||||
}
|
||||
aiBlock := out[idxAI:idxWeb]
|
||||
webBlock := out[idxWeb:]
|
||||
// ai (nil-Override gesetzt) soll `timeout server 300s` bekommen …
|
||||
if !strings.Contains(aiBlock, "timeout server 300s") {
|
||||
t.Errorf("ai-Block sollte `timeout server 300s` enthalten:\n%s", aiBlock)
|
||||
}
|
||||
// … web (kein Override) soll KEINE eigene timeout-server-Zeile bekommen.
|
||||
if strings.Contains(webBlock, "timeout server") {
|
||||
t.Errorf("web-Block soll KEIN eigenes `timeout server` enthalten:\n%s", webBlock)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRender_MultiServerPool(t *testing.T) {
|
||||
v := View{
|
||||
Backends: []BackendView{
|
||||
|
||||
@@ -12,6 +12,10 @@ type Backend struct {
|
||||
LBAlgorithm string `gorm:"column:lb_algorithm" json:"lb_algorithm"`
|
||||
WebSocket bool `gorm:"column:websocket" json:"websocket"`
|
||||
ForceHTTP1 bool `gorm:"column:force_http1" json:"force_http1"`
|
||||
// ServerTimeoutSeconds überschreibt `timeout server` für dieses
|
||||
// Backend (Sekunden). nil = defaults-Timeout (60s). Für langsam
|
||||
// antwortende Upstreams (KI-/Inferenz-Server ohne Streaming).
|
||||
ServerTimeoutSeconds *int `gorm:"column:server_timeout_seconds" json:"server_timeout_seconds,omitempty"`
|
||||
Active bool `gorm:"column:active" json:"active"`
|
||||
CreatedAt time.Time `gorm:"column:created_at" json:"created_at"`
|
||||
UpdatedAt time.Time `gorm:"column:updated_at" json:"updated_at"`
|
||||
|
||||
@@ -194,6 +194,24 @@ FROM alert_events ORDER BY fired_at DESC, id DESC LIMIT $1`, limit)
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// Cleanup löscht alert_events älter als keepDays und liefert die Anzahl
|
||||
// gelöschter Rows. make_interval(days => $1) nimmt $1 sauber als int —
|
||||
// der frühere ($1 || ' days')::interval-Ansatz erzwang text und scheiterte
|
||||
// unter pgx mit einem Encode-Fehler (vgl. waf PurgeAlerts, v1.3.3).
|
||||
func (s *Service) Cleanup(ctx context.Context, keepDays int) (int64, error) {
|
||||
if keepDays <= 0 {
|
||||
return 0, nil
|
||||
}
|
||||
tag, err := s.Pool.Exec(ctx,
|
||||
`DELETE FROM alert_events WHERE fired_at < NOW() - make_interval(days => $1)`,
|
||||
keepDays,
|
||||
)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return tag.RowsAffected(), nil
|
||||
}
|
||||
|
||||
// Fire dispatch'ed einen Event an alle aktiven Channels und persistiert
|
||||
// das Ergebnis. Non-fatal — Send-Failures werden im sent_to-JSON
|
||||
// dokumentiert, der Event selbst landet in jedem Fall in der History.
|
||||
|
||||
@@ -26,8 +26,8 @@ type Repo struct {
|
||||
func New(pool *pgxpool.Pool) *Repo { return &Repo{Pool: pool} }
|
||||
|
||||
const baseSelect = `
|
||||
SELECT id, name, scheme, health_check_path, lb_algorithm, websocket, force_http1, active,
|
||||
created_at, updated_at
|
||||
SELECT id, name, scheme, health_check_path, lb_algorithm, websocket, force_http1,
|
||||
server_timeout_seconds, active, created_at, updated_at
|
||||
FROM backends
|
||||
`
|
||||
|
||||
@@ -65,11 +65,13 @@ func (r *Repo) Create(ctx context.Context, b models.Backend) (*models.Backend, e
|
||||
b.LBAlgorithm = "roundrobin"
|
||||
}
|
||||
row := r.Pool.QueryRow(ctx, `
|
||||
INSERT INTO backends (name, scheme, health_check_path, lb_algorithm, websocket, force_http1, active)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7)
|
||||
RETURNING id, name, scheme, health_check_path, lb_algorithm, websocket, force_http1, active,
|
||||
created_at, updated_at`,
|
||||
b.Name, b.Scheme, b.HealthCheckPath, b.LBAlgorithm, b.WebSocket, b.ForceHTTP1, b.Active)
|
||||
INSERT INTO backends (name, scheme, health_check_path, lb_algorithm, websocket, force_http1,
|
||||
server_timeout_seconds, active)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
|
||||
RETURNING id, name, scheme, health_check_path, lb_algorithm, websocket, force_http1,
|
||||
server_timeout_seconds, active, created_at, updated_at`,
|
||||
b.Name, b.Scheme, b.HealthCheckPath, b.LBAlgorithm, b.WebSocket, b.ForceHTTP1,
|
||||
b.ServerTimeoutSeconds, b.Active)
|
||||
return scanBackend(row)
|
||||
}
|
||||
|
||||
@@ -85,12 +87,14 @@ UPDATE backends SET
|
||||
lb_algorithm = $4,
|
||||
websocket = $5,
|
||||
force_http1 = $6,
|
||||
active = $7,
|
||||
server_timeout_seconds = $7,
|
||||
active = $8,
|
||||
updated_at = NOW()
|
||||
WHERE id = $8
|
||||
RETURNING id, name, scheme, health_check_path, lb_algorithm, websocket, force_http1, active,
|
||||
created_at, updated_at`,
|
||||
b.Name, b.Scheme, b.HealthCheckPath, b.LBAlgorithm, b.WebSocket, b.ForceHTTP1, b.Active, id)
|
||||
WHERE id = $9
|
||||
RETURNING id, name, scheme, health_check_path, lb_algorithm, websocket, force_http1,
|
||||
server_timeout_seconds, active, created_at, updated_at`,
|
||||
b.Name, b.Scheme, b.HealthCheckPath, b.LBAlgorithm, b.WebSocket, b.ForceHTTP1,
|
||||
b.ServerTimeoutSeconds, b.Active, id)
|
||||
out, err := scanBackend(row)
|
||||
if err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
@@ -125,7 +129,8 @@ func scanBackend(row interface{ Scan(...any) error }) (*models.Backend, error) {
|
||||
var b models.Backend
|
||||
if err := row.Scan(
|
||||
&b.ID, &b.Name, &b.Scheme,
|
||||
&b.HealthCheckPath, &b.LBAlgorithm, &b.WebSocket, &b.ForceHTTP1, &b.Active,
|
||||
&b.HealthCheckPath, &b.LBAlgorithm, &b.WebSocket, &b.ForceHTTP1,
|
||||
&b.ServerTimeoutSeconds, &b.Active,
|
||||
&b.CreatedAt, &b.UpdatedAt,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
|
||||
Reference in New Issue
Block a user