feat(errors, watchdog): Fehler-Stream mit Ignore-Regeln, Metrik-Verlauf, Abhängigkeits-Alarme
Fehler-Schnittstelle - Neuer schlanker Eingang POST /api/errors/v1/report für den globalen Exception-Handler einer Anwendung. Titel und Dringlichkeit leitet der Server ab; gespeichert wird in derselben Tabelle wie der Bugtracker. Ein zweiter Speicher wäre nur ein zweiter Ort, an dem man suchen müsste. - error_level (fatal/error/warning) trennt die technische Art des Ereignisses von der geschäftlichen Dringlichkeit. Ein Duplicate-Entry ist technisch ein error, geschäftlich belanglos — beides zu vermischen war der Grund, warum solche Meldungen als Bug im Dashboard landeten. Ignore-Regeln gegen bekanntes Rauschen - bugtracker_ignore_rules mit contains/regex/exception_class, Pflichtfeld für die Begründung und optionaler Alarmschwelle. - Ein Treffer bedeutet nicht "wegwerfen": Der Fehler wird weiterhin erfasst und hochgezählt, bleibt aber aus der Übersicht heraus und löst keine Benachrichtigung aus. Der Zähler ist der eigentliche Zweck — dass ein bekannter Fehler auftritt, ist normal; dass er plötzlich hundertmal so oft auftritt, ist ein Signal. Dafür das rollende Stundenfenster und error.rate_exceeded. - Neue Regeln lassen sich rückwirkend auf bestehende Einträge anwenden. Gruppierung überarbeitet - Der Schlüssel nahm bisher 300 Zeichen Stacktrace auf. Derselbe Fehler zersplitterte dadurch, sobald ein Aufrufer den Stack einmal mitschickte und einmal nicht. Jetzt zählt der Ursprungsort: bevorzugt die Dateiangabe, sonst der erste Rahmen des Stacktrace. - Die Normalisierung ersetzte nur Zahlen ab vier Stellen, wodurch 'AA-1' und 'BB-2' getrennt blieben. Werte in Anführungszeichen, die Ziffern enthalten, gelten jetzt als veränderlich — der Schlüsselname bleibt erhalten, sodass verschiedene Unique-Keys unterscheidbar sind. Mit 9 Testfällen belegt. Metrik-Verlauf - watchdog_metrics speichert numerische Heartbeat-Werte mit Zeitstempel. Zuvor wurde metrics_json bei jedem Heartbeat überschrieben; damit ließ sich "die Platte läuft seit drei Tagen voll" nicht erkennen, nur "sie ist voll". - GET /api/watchdog/v1/metrics liefert den verdichteten Verlauf und die Abweichung vom eigenen Sieben-Tage-Durchschnitt. Dieser relative Ansatz braucht keine projektspezifischen Schwellwerte. - Aufbewahrung 14 Tage, Bereinigung stündlich durch den Evaluator. Health-Checks per Push statt Abruf - Der Heartbeat nimmt ein checks-Objekt entgegen, das die Anwendung selbst ermittelt. Das Deploymentcenter interpretiert die Namen nicht, es liest nur ok und message — was "gesund" bedeutet, entscheidet jede Anwendung selbst. Schlägt eine Prüfung fehl, wird ein als ok gemeldeter Heartbeat auf warning herabgestuft. - Bewusst ausgehend: auf den Zielmaschinen müssen keine Ports geöffnet werden. Abhängigkeitsbewusste Alarmierung - Fällt ein Monitor aus, dessen Parent selbst unten ist, wird der Alarm unterdrückt. Der Zustand bleibt sichtbar. Vorher erzeugte ein ausgefallener Hypervisor mit zwölf VMs dreizehn Meldungen für ein Problem. - Mehrere Ebenen und fehlerhafte Hierarchien (Zyklen, gelöschte Parents) sind abgesichert; mit 10 Testfällen belegt. WebUI - Neue Ansicht "Fehler-Stream" mit Filtern nach Projekt, Fehlerklasse, Umgebung, Zeitraum und Sichtbarkeit sowie Volltextsuche und Pagination. Stummgeschaltete Einträge sind standardmäßig ausgeblendet. - Verwaltung der Ignore-Regeln inklusive Trefferzähler. - Die Detailansicht zeigt Fehlerklasse, Stummschaltungsgrund und die Häufung im laufenden Stundenfenster. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Deploymentcenter Bot
co-authored by
Claude Opus 5
Claude Opus 5
parent
a74c6fd990
commit
60e34b29f6
@@ -30,11 +30,20 @@ final class BugRepo
|
||||
public const TYPES = ['bug', 'feature_request'];
|
||||
public const ENVIRONMENTS = ['production', 'development', 'staging', 'testing'];
|
||||
public const SEVERITIES = ['idea', 'wishlist', 'low', 'medium', 'high', 'critical'];
|
||||
public const STATUSES = ['open', 'planned', 'in_progress', 'resolved', 'closed', 'rejected'];
|
||||
public const STATUSES = ['open', 'planned', 'in_progress', 'resolved', 'closed', 'rejected', 'ignored'];
|
||||
|
||||
/**
|
||||
* Technische Art des Ereignisses, unabhaengig von der Dringlichkeit.
|
||||
* Ein "Duplicate entry" ist technisch ein error, geschaeftlich belanglos.
|
||||
*/
|
||||
public const ERROR_LEVELS = ['fatal', 'error', 'warning'];
|
||||
|
||||
/** Status, in denen ein Item als offen/aktiv gilt. */
|
||||
public const ACTIVE_STATUSES = ['open', 'planned', 'in_progress'];
|
||||
|
||||
/** Laenge des rollenden Fensters fuer die Ratenerkennung, in Minuten. */
|
||||
private const RATE_WINDOW_MINUTES = 60;
|
||||
|
||||
private const DEFAULT_LIMIT = 100;
|
||||
private const MAX_LIMIT = 500;
|
||||
|
||||
@@ -77,6 +86,17 @@ final class BugRepo
|
||||
$createdBy = self::text($data['created_by'] ?? null) ?? 'agent';
|
||||
$clientRef = self::text($data['client_ref'] ?? null);
|
||||
|
||||
$exceptionClass = self::text($data['exception'] ?? $data['exception_class'] ?? null);
|
||||
$errorLevel = self::oneOfOrNull($data['error_level'] ?? null, self::ERROR_LEVELS);
|
||||
|
||||
// Bekannte, harmlose Fehler werden erfasst und gezaehlt, aber nicht
|
||||
// gemeldet. Der Zaehler bleibt erhalten, damit eine auffaellige
|
||||
// Haeufung trotzdem auffaellt.
|
||||
$ignoreRules = new IgnoreRules($this->db);
|
||||
$ignoreRule = $type === 'bug'
|
||||
? $ignoreRules->match($projectSlug, $exceptionClass, $errorMessage, $title)
|
||||
: null;
|
||||
|
||||
// Strukturierter Code-Kontext
|
||||
$repoUrl = self::text($data['repo_url'] ?? null);
|
||||
$gitBranch = self::text($data['git_branch'] ?? null);
|
||||
@@ -107,11 +127,26 @@ final class BugRepo
|
||||
}
|
||||
|
||||
// --- 2. Deduplizierung ueber einen stabilen Schluessel ---
|
||||
$dedupKey = self::buildDedupKey($type, $projectSlug, $environment, $title, $errorMessage, $stackTrace);
|
||||
$dedupKey = self::buildDedupKey(
|
||||
$type,
|
||||
$projectSlug,
|
||||
$environment,
|
||||
$title,
|
||||
$errorMessage,
|
||||
$stackTrace,
|
||||
$exceptionClass,
|
||||
$filePath
|
||||
);
|
||||
$duplicate = $this->findByDedupKey($dedupKey);
|
||||
|
||||
if ($duplicate !== null && in_array($duplicate['status'], self::ACTIVE_STATUSES, true)) {
|
||||
return $this->registerRecurrence($duplicate, $pushId, $buildVersion, $severity);
|
||||
// Auch stummgeschaltete Items werden weiter hochgezaehlt.
|
||||
$countable = array_merge(self::ACTIVE_STATUSES, ['ignored']);
|
||||
|
||||
if ($duplicate !== null && in_array($duplicate['status'], $countable, true)) {
|
||||
if ($ignoreRule !== null) {
|
||||
$ignoreRules->recordMatch((int)$ignoreRule['id']);
|
||||
}
|
||||
return $this->registerRecurrence($duplicate, $pushId, $buildVersion, $severity, $ignoreRule);
|
||||
}
|
||||
|
||||
// Bereits geloest und tritt erneut auf: neues Item mit Verweis auf das alte.
|
||||
@@ -148,19 +183,23 @@ final class BugRepo
|
||||
// --- 3. Neues Item anlegen ---
|
||||
$projectId = $this->projectIdForSlug($projectSlug);
|
||||
|
||||
$initialStatus = $ignoreRule !== null ? 'ignored' : 'open';
|
||||
|
||||
$stmt = $this->db->prepare('
|
||||
INSERT INTO bugtracker_items (
|
||||
project_id, project_slug, type, title, description,
|
||||
error_message, stack_trace, error_hash, dedup_key,
|
||||
build_version, repo_url, git_branch, commit_sha, file_path, line_no, context_json,
|
||||
environment, severity, status, occurrence_count,
|
||||
environment, severity, error_level, ignore_rule_id, status, occurrence_count,
|
||||
rate_window_start, rate_window_count,
|
||||
push_id, target_agent, tags, client_ref, regression_of,
|
||||
first_seen_at, last_seen_at, created_by, created_at
|
||||
) VALUES (
|
||||
:project_id, :project_slug, :type, :title, :description,
|
||||
:error_message, :stack_trace, :error_hash, :dedup_key,
|
||||
:build_version, :repo_url, :git_branch, :commit_sha, :file_path, :line_no, :context_json,
|
||||
:environment, :severity, "open", 1,
|
||||
:environment, :severity, :error_level, :ignore_rule_id, :status, 1,
|
||||
UTC_TIMESTAMP(), 1,
|
||||
:push_id, :target_agent, :tags, :client_ref, :regression_of,
|
||||
UTC_TIMESTAMP(), UTC_TIMESTAMP(), :created_by, UTC_TIMESTAMP()
|
||||
)
|
||||
@@ -186,6 +225,9 @@ final class BugRepo
|
||||
':context_json' => $contextJson,
|
||||
':environment' => $environment,
|
||||
':severity' => $severity,
|
||||
':error_level' => $errorLevel,
|
||||
':ignore_rule_id' => $ignoreRule !== null ? (int)$ignoreRule['id'] : null,
|
||||
':status' => $initialStatus,
|
||||
':push_id' => $pushId,
|
||||
':target_agent' => $targetAgent,
|
||||
':tags' => $tags,
|
||||
@@ -222,6 +264,40 @@ final class BugRepo
|
||||
|
||||
$newId = (int)$this->db->lastInsertId();
|
||||
|
||||
if ($ignoreRule !== null) {
|
||||
$ignoreRules->recordMatch((int)$ignoreRule['id']);
|
||||
|
||||
$this->addComment(
|
||||
$newId,
|
||||
'system',
|
||||
sprintf(
|
||||
"Automatisch stummgeschaltet durch Regel #%d (%s: \"%s\").\n\nBegruendung: %s",
|
||||
$ignoreRule['id'],
|
||||
$ignoreRule['match_type'],
|
||||
$ignoreRule['pattern'],
|
||||
$ignoreRule['reason']
|
||||
),
|
||||
'auto_ignored'
|
||||
);
|
||||
|
||||
// Bewusst keine Benachrichtigung: genau dafuer ist die Regel da.
|
||||
return [
|
||||
'id' => $newId,
|
||||
'is_new' => true,
|
||||
'idempotent_hit' => false,
|
||||
'occurrence_count' => 1,
|
||||
'dedup_key' => $dedupKey,
|
||||
'error_hash' => $dedupKey,
|
||||
'type' => $type,
|
||||
'status' => 'ignored',
|
||||
'ignored' => true,
|
||||
'ignore_rule_id' => (int)$ignoreRule['id'],
|
||||
'environment' => $environment,
|
||||
'push_id' => $pushId,
|
||||
'regression_of' => $regressionOf,
|
||||
];
|
||||
}
|
||||
|
||||
$note = $type === 'bug'
|
||||
? 'Bug im System erfasst.'
|
||||
: ($severity === 'idea' ? 'Neue Idee hinterlegt.' : 'Feature-Request eingereicht.');
|
||||
@@ -232,13 +308,15 @@ final class BugRepo
|
||||
'environment' => $environment,
|
||||
'project_slug' => $projectSlug,
|
||||
'title' => $title,
|
||||
'error_level' => $errorLevel,
|
||||
]);
|
||||
|
||||
if ($severity === 'critical') {
|
||||
if ($severity === 'critical' || $errorLevel === 'fatal') {
|
||||
$this->notify('bug.critical', $newId, [
|
||||
'project_slug' => $projectSlug,
|
||||
'title' => $title,
|
||||
'environment' => $environment,
|
||||
'error_level' => $errorLevel,
|
||||
]);
|
||||
}
|
||||
|
||||
@@ -251,26 +329,54 @@ final class BugRepo
|
||||
'error_hash' => $type === 'bug' ? $dedupKey : null,
|
||||
'type' => $type,
|
||||
'status' => 'open',
|
||||
'ignored' => false,
|
||||
'environment' => $environment,
|
||||
'push_id' => $pushId,
|
||||
'regression_of' => $regressionOf,
|
||||
];
|
||||
}
|
||||
|
||||
/** Zaehlt ein wiederkehrendes Vorkommnis hoch. */
|
||||
private function registerRecurrence(array $existing, ?string $pushId, string $build, string $severity): array
|
||||
{
|
||||
/**
|
||||
* Zaehlt ein wiederkehrendes Vorkommnis hoch und pflegt das Ratenfenster.
|
||||
*
|
||||
* @param array<string,mixed>|null $ignoreRule
|
||||
*/
|
||||
private function registerRecurrence(
|
||||
array $existing,
|
||||
?string $pushId,
|
||||
string $build,
|
||||
string $severity,
|
||||
?array $ignoreRule = null
|
||||
): array {
|
||||
$itemId = (int)$existing['id'];
|
||||
$isIgnored = (string)$existing['status'] === 'ignored';
|
||||
$newCount = (int)$existing['occurrence_count'] + 1;
|
||||
|
||||
// Eskalation: ein bereits offener Bug, der erneut mit hoeherem
|
||||
// Schweregrad gemeldet wird, wird hochgestuft - nie herabgestuft.
|
||||
// Fuer stummgeschaltete Items entfaellt das; dort entscheidet die Rate.
|
||||
$currentRank = array_search((string)$existing['severity'], self::SEVERITIES, true);
|
||||
$incomingRank = array_search($severity, self::SEVERITIES, true);
|
||||
$escalate = is_int($currentRank) && is_int($incomingRank) && $incomingRank > $currentRank;
|
||||
$escalate = !$isIgnored
|
||||
&& is_int($currentRank)
|
||||
&& is_int($incomingRank)
|
||||
&& $incomingRank > $currentRank;
|
||||
|
||||
// Reihenfolge beachten: rate_window_count muss den alten Wert von
|
||||
// rate_window_start sehen und steht deshalb davor.
|
||||
$stmt = $this->db->prepare('
|
||||
UPDATE bugtracker_items
|
||||
SET occurrence_count = :count,
|
||||
SET rate_window_count = IF(
|
||||
rate_window_start IS NULL
|
||||
OR rate_window_start < UTC_TIMESTAMP() - INTERVAL ' . self::RATE_WINDOW_MINUTES . ' MINUTE,
|
||||
1, rate_window_count + 1
|
||||
),
|
||||
rate_window_start = IF(
|
||||
rate_window_start IS NULL
|
||||
OR rate_window_start < UTC_TIMESTAMP() - INTERVAL ' . self::RATE_WINDOW_MINUTES . ' MINUTE,
|
||||
UTC_TIMESTAMP(), rate_window_start
|
||||
),
|
||||
occurrence_count = :count,
|
||||
last_seen_at = UTC_TIMESTAMP(),
|
||||
push_id = COALESCE(:push_id, push_id),
|
||||
build_version = COALESCE(:build, build_version),
|
||||
@@ -283,20 +389,22 @@ final class BugRepo
|
||||
':build' => $build,
|
||||
':escalate' => $escalate ? 1 : 0,
|
||||
':severity' => $severity,
|
||||
':id' => (int)$existing['id'],
|
||||
':id' => $itemId,
|
||||
]);
|
||||
|
||||
if ($escalate) {
|
||||
$this->addComment(
|
||||
(int)$existing['id'],
|
||||
$itemId,
|
||||
'system',
|
||||
sprintf('Schweregrad automatisch hochgestuft: %s -> %s (erneutes Auftreten).', $existing['severity'], $severity),
|
||||
'severity_escalated'
|
||||
);
|
||||
}
|
||||
|
||||
$rateAlerted = $this->checkRateThreshold($itemId, $ignoreRule, $existing);
|
||||
|
||||
return [
|
||||
'id' => (int)$existing['id'],
|
||||
'id' => $itemId,
|
||||
'is_new' => false,
|
||||
'idempotent_hit' => false,
|
||||
'occurrence_count' => $newCount,
|
||||
@@ -304,11 +412,95 @@ final class BugRepo
|
||||
'error_hash' => $existing['error_hash'],
|
||||
'type' => $existing['type'],
|
||||
'status' => $existing['status'],
|
||||
'ignored' => $isIgnored,
|
||||
'rate_alerted' => $rateAlerted,
|
||||
'environment' => $existing['environment'],
|
||||
'push_id' => $pushId ?? $existing['push_id'],
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* Prueft, ob ein stummgeschalteter Fehler auffaellig haeufig auftritt.
|
||||
*
|
||||
* Das ist der eigentliche Grund, warum bekannte Fehler weiter gezaehlt und
|
||||
* nicht verworfen werden: Dass ein Duplicate-Entry auftritt, ist normal.
|
||||
* Dass er ploetzlich hundertmal so oft auftritt, bedeutet, dass sich am
|
||||
* Datenfeed etwas geaendert hat.
|
||||
*
|
||||
* @param array<string,mixed>|null $ignoreRule
|
||||
* @param array<string,mixed> $existing
|
||||
*/
|
||||
private function checkRateThreshold(int $itemId, ?array $ignoreRule, array $existing): bool
|
||||
{
|
||||
$threshold = null;
|
||||
|
||||
if ($ignoreRule !== null && $ignoreRule['alert_on_rate'] !== null) {
|
||||
$threshold = (int)$ignoreRule['alert_on_rate'];
|
||||
} elseif (!empty($existing['ignore_rule_id'])) {
|
||||
// Regel ueber das Item nachschlagen, wenn sie nicht mitgeliefert wurde.
|
||||
$lookup = $this->db->prepare('SELECT alert_on_rate, pattern, reason FROM bugtracker_ignore_rules WHERE id = :id');
|
||||
$lookup->execute([':id' => (int)$existing['ignore_rule_id']]);
|
||||
$row = $lookup->fetch();
|
||||
|
||||
if (is_array($row) && $row['alert_on_rate'] !== null) {
|
||||
$threshold = (int)$row['alert_on_rate'];
|
||||
$ignoreRule = $row;
|
||||
}
|
||||
}
|
||||
|
||||
if ($threshold === null || $threshold <= 0) {
|
||||
return false;
|
||||
}
|
||||
|
||||
$stmt = $this->db->prepare('
|
||||
SELECT rate_window_count, rate_alerted_at, rate_window_start, project_slug, title
|
||||
FROM bugtracker_items WHERE id = :id
|
||||
');
|
||||
$stmt->execute([':id' => $itemId]);
|
||||
$current = $stmt->fetch();
|
||||
|
||||
if (!is_array($current) || (int)$current['rate_window_count'] < $threshold) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Nur einmal je Fenster alarmieren, sonst entsteht genau der
|
||||
// Meldungssturm, den die Regel verhindern soll.
|
||||
if ($current['rate_alerted_at'] !== null
|
||||
&& $current['rate_window_start'] !== null
|
||||
&& $current['rate_alerted_at'] >= $current['rate_window_start']) {
|
||||
return false;
|
||||
}
|
||||
|
||||
$this->db->prepare('UPDATE bugtracker_items SET rate_alerted_at = UTC_TIMESTAMP() WHERE id = :id')
|
||||
->execute([':id' => $itemId]);
|
||||
|
||||
$message = sprintf(
|
||||
'Auffaellige Haeufung: %d Vorkommnisse in %d Minuten (Schwelle %d). '
|
||||
. 'Der Fehler gilt als bekannt und harmlos, tritt aber deutlich haeufiger auf als erwartet.',
|
||||
(int)$current['rate_window_count'],
|
||||
self::RATE_WINDOW_MINUTES,
|
||||
$threshold
|
||||
);
|
||||
|
||||
$this->addComment($itemId, 'system', $message, 'rate_exceeded');
|
||||
|
||||
$this->notify('error.rate_exceeded', $itemId, [
|
||||
'project_slug' => $current['project_slug'],
|
||||
'title' => $current['title'],
|
||||
'count' => (int)$current['rate_window_count'],
|
||||
'threshold' => $threshold,
|
||||
'window_min' => self::RATE_WINDOW_MINUTES,
|
||||
]);
|
||||
|
||||
Logger::warning('Ratenschwelle eines stummgeschalteten Fehlers ueberschritten', [
|
||||
'item_id' => $itemId,
|
||||
'count' => (int)$current['rate_window_count'],
|
||||
'threshold' => $threshold,
|
||||
]);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
// ==================================================================
|
||||
// Abfragen
|
||||
// ==================================================================
|
||||
@@ -399,8 +591,15 @@ final class BugRepo
|
||||
}
|
||||
}
|
||||
|
||||
// status und severity duerfen als Liste kommen: status=open,in_progress
|
||||
foreach (['status' => self::STATUSES, 'severity' => self::SEVERITIES] as $key => $allowed) {
|
||||
// status, severity und error_level duerfen als Liste kommen,
|
||||
// z. B. status=open,in_progress
|
||||
$listFilters = [
|
||||
'status' => self::STATUSES,
|
||||
'severity' => self::SEVERITIES,
|
||||
'error_level' => self::ERROR_LEVELS,
|
||||
];
|
||||
|
||||
foreach ($listFilters as $key => $allowed) {
|
||||
$value = $filters[$key] ?? null;
|
||||
if ($value === null || $value === '' || $value === 'all') {
|
||||
continue;
|
||||
@@ -438,6 +637,33 @@ final class BugRepo
|
||||
$where[] = '(lease_until IS NULL OR lease_until < UTC_TIMESTAMP())';
|
||||
}
|
||||
|
||||
// Stummgeschaltete Fehler bleiben standardmaessig aussen vor. Sie sind
|
||||
// bekannt und harmlos - genau deshalb sollen sie die Uebersicht nicht
|
||||
// fuellen. Wer sie sehen will, fragt sie ausdruecklich an.
|
||||
$includeIgnored = isset($filters['include_ignored'])
|
||||
&& filter_var($filters['include_ignored'], FILTER_VALIDATE_BOOLEAN);
|
||||
$onlyIgnored = isset($filters['only_ignored'])
|
||||
&& filter_var($filters['only_ignored'], FILTER_VALIDATE_BOOLEAN);
|
||||
|
||||
$statusRequested = ($filters['status'] ?? 'all') !== 'all' && ($filters['status'] ?? '') !== '';
|
||||
|
||||
if ($onlyIgnored) {
|
||||
$where[] = 'status = "ignored"';
|
||||
} elseif (!$includeIgnored && !$statusRequested) {
|
||||
$where[] = 'status <> "ignored"';
|
||||
}
|
||||
|
||||
// Nur Eintraege, die ueber die Fehler-Schnittstelle kamen.
|
||||
if (!empty($filters['errors_only'])) {
|
||||
$where[] = 'error_level IS NOT NULL';
|
||||
}
|
||||
|
||||
// Zeitraum, z. B. "letzte 24 Stunden"
|
||||
$sinceHours = isset($filters['since_hours']) ? (int)$filters['since_hours'] : 0;
|
||||
if ($sinceHours > 0) {
|
||||
$where[] = 'last_seen_at > (UTC_TIMESTAMP() - INTERVAL ' . min($sinceHours, 8760) . ' HOUR)';
|
||||
}
|
||||
|
||||
// Delta-Abfrage fuer Polling.
|
||||
$since = $filters['updated_since'] ?? null;
|
||||
if (is_string($since) && trim($since) !== '') {
|
||||
@@ -759,6 +985,19 @@ final class BugRepo
|
||||
$params[':line_no'] = is_numeric($updates['line_no']) ? (int)$updates['line_no'] : null;
|
||||
}
|
||||
|
||||
if (array_key_exists('error_level', $updates)) {
|
||||
$fields[] = 'error_level = :error_level';
|
||||
$params[':error_level'] = self::oneOfOrNull($updates['error_level'], self::ERROR_LEVELS);
|
||||
}
|
||||
|
||||
// Wird ein stummgeschaltetes Item wieder geoeffnet, verliert es die
|
||||
// Regelbindung - sonst wuerde das naechste Vorkommnis es sofort
|
||||
// wieder stummschalten.
|
||||
if (isset($updates['status']) && $updates['status'] !== 'ignored'
|
||||
&& (string)($existing['status'] ?? '') === 'ignored') {
|
||||
$fields[] = 'ignore_rule_id = NULL';
|
||||
}
|
||||
|
||||
if ($fields === []) {
|
||||
return true;
|
||||
}
|
||||
@@ -902,7 +1141,17 @@ final class BugRepo
|
||||
SUM(status = "resolved") AS resolved_total,
|
||||
SUM(type = "bug" AND severity = "critical" AND status IN ("open","in_progress")) AS critical_bugs,
|
||||
SUM(claimed_by IS NOT NULL AND lease_until > UTC_TIMESTAMP()) AS in_progress_by_agents,
|
||||
SUM(status IN ("open","planned","in_progress")) AS open_total
|
||||
SUM(status IN ("open","planned","in_progress")) AS open_total,
|
||||
|
||||
-- Fehler-Stream
|
||||
SUM(error_level = "fatal" AND status IN ("open","in_progress")) AS fatal_open,
|
||||
SUM(error_level IS NOT NULL AND status = "ignored") AS ignored_groups,
|
||||
-- Nur was noch offen ist: bereits abgehakte oder stummgeschaltete
|
||||
-- Gruppen sind keine aktuellen Fehler mehr.
|
||||
SUM(error_level IS NOT NULL
|
||||
AND status IN ("open","planned","in_progress")
|
||||
AND last_seen_at > UTC_TIMESTAMP() - INTERVAL 24 HOUR) AS errors_24h,
|
||||
COALESCE(SUM(CASE WHEN status = "ignored" THEN occurrence_count ELSE 0 END), 0) AS ignored_occurrences
|
||||
FROM bugtracker_items' . $where;
|
||||
|
||||
$stmt = $this->db->prepare($sql);
|
||||
@@ -912,6 +1161,7 @@ final class BugRepo
|
||||
$keys = [
|
||||
'open_bugs_prod', 'open_bugs_dev', 'open_features', 'ideas_count',
|
||||
'resolved_total', 'critical_bugs', 'in_progress_by_agents', 'open_total',
|
||||
'fatal_open', 'ignored_groups', 'errors_24h', 'ignored_occurrences',
|
||||
];
|
||||
|
||||
$stats = [];
|
||||
@@ -995,14 +1245,17 @@ final class BugRepo
|
||||
string $environment,
|
||||
string $title,
|
||||
?string $errorMessage,
|
||||
?string $stackTrace
|
||||
?string $stackTrace,
|
||||
?string $exceptionClass = null,
|
||||
?string $filePath = null
|
||||
): string {
|
||||
if ($type === 'bug') {
|
||||
$signature = implode('|', [
|
||||
$projectSlug,
|
||||
$environment,
|
||||
$exceptionClass !== null ? mb_strtolower(trim($exceptionClass)) : '',
|
||||
self::normalizeForHash($errorMessage ?? $title),
|
||||
self::normalizeForHash(mb_substr($stackTrace ?? '', 0, 300)),
|
||||
self::originOf($stackTrace, $filePath),
|
||||
]);
|
||||
} else {
|
||||
$signature = implode('|', [
|
||||
@@ -1015,6 +1268,39 @@ final class BugRepo
|
||||
return substr(hash('sha256', $signature), 0, 40);
|
||||
}
|
||||
|
||||
/**
|
||||
* Stabile Herkunftsangabe fuer den Deduplizierungsschluessel.
|
||||
*
|
||||
* Zuvor flossen 300 Zeichen Stacktrace ein. Damit zersplitterte derselbe
|
||||
* Fehler in mehrere Gruppen, sobald ein Aufrufer den Stack einmal mitschickte
|
||||
* und einmal nicht - oder wenn er aus unterschiedlicher Aufruftiefe kam.
|
||||
* Jetzt zaehlt nur der Ursprungsort: bevorzugt die ausdrueckliche
|
||||
* Dateiangabe, sonst der erste verwertbare Rahmen des Stacktrace.
|
||||
*/
|
||||
private static function originOf(?string $stackTrace, ?string $filePath): string
|
||||
{
|
||||
if ($filePath !== null && trim($filePath) !== '') {
|
||||
// Zeilennummern verschieben sich bei jeder Aenderung an der Datei;
|
||||
// die Datei selbst bleibt dieselbe Fehlerquelle.
|
||||
return mb_strtolower(trim($filePath));
|
||||
}
|
||||
|
||||
if ($stackTrace === null || trim($stackTrace) === '') {
|
||||
return '';
|
||||
}
|
||||
|
||||
foreach (preg_split('/\r?\n/', trim($stackTrace)) ?: [] as $line) {
|
||||
$line = trim($line);
|
||||
if ($line === '') {
|
||||
continue;
|
||||
}
|
||||
// Erster nicht leerer Rahmen, von veraenderlichen Anteilen befreit.
|
||||
return self::normalizeForHash(mb_substr($line, 0, 200));
|
||||
}
|
||||
|
||||
return '';
|
||||
}
|
||||
|
||||
/**
|
||||
* Entfernt Rauschen, das denselben Fehler sonst als neu erscheinen liesse:
|
||||
* Zeilennummern, Speicheradressen, Zeitstempel, GUIDs, Mehrfach-Leerzeichen.
|
||||
@@ -1022,13 +1308,25 @@ final class BugRepo
|
||||
private static function normalizeForHash(string $value): string
|
||||
{
|
||||
$value = mb_strtolower(trim($value));
|
||||
|
||||
$patterns = [
|
||||
'/0x[0-9a-f]+/' => '0xADDR',
|
||||
'/0x[0-9a-f]+/' => '0xADDR',
|
||||
'/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/' => 'GUID',
|
||||
'/\d{4}-\d{2}-\d{2}[t ]\d{2}:\d{2}:\d{2}/' => 'TIMESTAMP',
|
||||
'/:line \d+/' => ':line N',
|
||||
'/\b\d{4,}\b/' => 'N',
|
||||
'/\s+/' => ' ',
|
||||
'/\d{4}-\d{2}-\d{2}[t ]\d{2}:\d{2}:\d{2}/' => 'TIMESTAMP',
|
||||
|
||||
// Werte in Anfuehrungszeichen, die mindestens eine Ziffer
|
||||
// enthalten, sind praktisch immer konkrete Datensatzwerte und
|
||||
// gehoeren nicht zur Identitaet des Fehlers:
|
||||
// Duplicate entry 'MKT-88213' for key 'uq_market'
|
||||
// Der Schluesselname bleibt erhalten, weil er ohne Ziffern
|
||||
// auskommt - unterschiedliche Unique-Keys bleiben damit
|
||||
// unterscheidbar.
|
||||
"/'[^']*\d[^']*'/" => "'VALUE'",
|
||||
'/"[^"]*\d[^"]*"/' => '"VALUE"',
|
||||
|
||||
'/:line \d+/' => ':line N',
|
||||
'/\b\d{3,}\b/' => 'N',
|
||||
'/\s+/' => ' ',
|
||||
];
|
||||
|
||||
foreach ($patterns as $pattern => $replacement) {
|
||||
@@ -1072,4 +1370,15 @@ final class BugRepo
|
||||
}
|
||||
return $default;
|
||||
}
|
||||
|
||||
/**
|
||||
* Wie oneOf(), liefert aber null statt eines Standardwerts.
|
||||
*
|
||||
* @param mixed $value
|
||||
* @param list<string> $allowed
|
||||
*/
|
||||
private static function oneOfOrNull($value, array $allowed): ?string
|
||||
{
|
||||
return is_string($value) && in_array($value, $allowed, true) ? $value : null;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,278 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace Deploymentcenter\Modules\Bugtracker;
|
||||
|
||||
use Deploymentcenter\Core\Logger;
|
||||
use PDO;
|
||||
|
||||
/**
|
||||
* Regeln fuer bekannte, harmlose Fehler.
|
||||
*
|
||||
* Anwendungen erzeugen betriebsbedingt Fehler, die niemanden interessieren -
|
||||
* etwa doppelte Datensaetze aus einem Datenfeed, die ohnehin nur einmal
|
||||
* gebraucht werden. Ohne Klassifizierung landen sie im Dashboard und
|
||||
* verdraengen dort das Wesentliche.
|
||||
*
|
||||
* Ein Treffer bedeutet nicht "wegwerfen": Das Item wird angelegt und weiter
|
||||
* hochgezaehlt, bleibt aber im Status "ignored" und loest keine
|
||||
* Benachrichtigung aus. Der Zaehler ist der eigentliche Gewinn - denn
|
||||
* interessant ist nicht, dass ein bekannter Fehler auftritt, sondern wenn er
|
||||
* ploetzlich um ein Vielfaches haeufiger auftritt.
|
||||
*/
|
||||
final class IgnoreRules
|
||||
{
|
||||
public const MATCH_TYPES = ['contains', 'regex', 'exception_class'];
|
||||
|
||||
/** @var array<string,list<array<string,mixed>>>|null Regeln je Projekt, pro Request zwischengespeichert */
|
||||
private static ?array $cache = null;
|
||||
|
||||
private PDO $db;
|
||||
|
||||
public function __construct(PDO $db)
|
||||
{
|
||||
$this->db = $db;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sucht die erste zutreffende Regel.
|
||||
*
|
||||
* @return array<string,mixed>|null
|
||||
*/
|
||||
public function match(string $projectSlug, ?string $exceptionClass, ?string $message, ?string $title = null): ?array
|
||||
{
|
||||
$haystack = trim(implode("\n", array_filter([$exceptionClass, $message, $title])));
|
||||
if ($haystack === '') {
|
||||
return null;
|
||||
}
|
||||
|
||||
foreach ($this->rulesFor($projectSlug) as $rule) {
|
||||
if (self::ruleMatches($rule, $exceptionClass, $haystack)) {
|
||||
return $rule;
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/** @param array<string,mixed> $rule */
|
||||
private static function ruleMatches(array $rule, ?string $exceptionClass, string $haystack): bool
|
||||
{
|
||||
$pattern = (string)$rule['pattern'];
|
||||
if ($pattern === '') {
|
||||
return false;
|
||||
}
|
||||
|
||||
switch ((string)$rule['match_type']) {
|
||||
case 'exception_class':
|
||||
return $exceptionClass !== null
|
||||
&& strcasecmp(trim($exceptionClass), $pattern) === 0;
|
||||
|
||||
case 'regex':
|
||||
// Das Muster stammt aus der Verwaltungsoberflaeche und wird als
|
||||
// reiner Ausdruck ohne Begrenzer gespeichert. Der Begrenzer wird
|
||||
// hier gesetzt, damit kein eigener Modifikator untergeschoben
|
||||
// werden kann.
|
||||
$delimited = '/' . str_replace('/', '\/', $pattern) . '/i';
|
||||
$result = @preg_match($delimited, $haystack);
|
||||
|
||||
if ($result === false) {
|
||||
Logger::warning('Ignore-Regel enthaelt einen ungueltigen regulaeren Ausdruck', [
|
||||
'rule_id' => $rule['id'] ?? null,
|
||||
'pattern' => $pattern,
|
||||
]);
|
||||
return false;
|
||||
}
|
||||
|
||||
return $result === 1;
|
||||
|
||||
case 'contains':
|
||||
default:
|
||||
return stripos($haystack, $pattern) !== false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Regeln des Projekts plus die projektuebergreifenden.
|
||||
*
|
||||
* @return list<array<string,mixed>>
|
||||
*/
|
||||
private function rulesFor(string $projectSlug): array
|
||||
{
|
||||
if (self::$cache === null) {
|
||||
self::$cache = [];
|
||||
|
||||
try {
|
||||
$rows = $this->db->query('
|
||||
SELECT * FROM bugtracker_ignore_rules
|
||||
WHERE enabled = 1
|
||||
ORDER BY project_slug IS NULL ASC, id ASC
|
||||
')->fetchAll() ?: [];
|
||||
|
||||
foreach ($rows as $row) {
|
||||
$key = $row['project_slug'] !== null && $row['project_slug'] !== ''
|
||||
? (string)$row['project_slug']
|
||||
: '*';
|
||||
self::$cache[$key][] = $row;
|
||||
}
|
||||
} catch (\Throwable $e) {
|
||||
// Tabelle fehlt (Migration noch nicht gelaufen): dann greift
|
||||
// eben keine Regel. Kein Grund, die Erfassung scheitern zu lassen.
|
||||
Logger::warning('Ignore-Regeln nicht ladbar', ['error' => $e->getMessage()]);
|
||||
self::$cache = [];
|
||||
}
|
||||
}
|
||||
|
||||
return array_merge(
|
||||
self::$cache[$projectSlug] ?? [],
|
||||
self::$cache['*'] ?? []
|
||||
);
|
||||
}
|
||||
|
||||
/** Vermerkt einen Treffer fuer die Statistik in der Verwaltungsansicht. */
|
||||
public function recordMatch(int $ruleId): void
|
||||
{
|
||||
try {
|
||||
$stmt = $this->db->prepare('
|
||||
UPDATE bugtracker_ignore_rules
|
||||
SET match_count = match_count + 1, last_match_at = UTC_TIMESTAMP()
|
||||
WHERE id = :id
|
||||
');
|
||||
$stmt->execute([':id' => $ruleId]);
|
||||
} catch (\Throwable $e) {
|
||||
Logger::warning('Treffer der Ignore-Regel nicht vermerkt', ['rule_id' => $ruleId]);
|
||||
}
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// Verwaltung
|
||||
// ------------------------------------------------------------------
|
||||
|
||||
/** @return list<array<string,mixed>> */
|
||||
public function all(): array
|
||||
{
|
||||
$stmt = $this->db->query('
|
||||
SELECT r.*,
|
||||
(SELECT COUNT(*) FROM bugtracker_items i WHERE i.ignore_rule_id = r.id) AS item_count
|
||||
FROM bugtracker_ignore_rules r
|
||||
ORDER BY r.enabled DESC, r.last_match_at DESC, r.id DESC
|
||||
');
|
||||
return $stmt->fetchAll() ?: [];
|
||||
}
|
||||
|
||||
public function create(
|
||||
?string $projectSlug,
|
||||
string $matchType,
|
||||
string $pattern,
|
||||
string $reason,
|
||||
?int $alertOnRate,
|
||||
string $createdBy
|
||||
): int {
|
||||
$pattern = trim($pattern);
|
||||
$reason = trim($reason);
|
||||
|
||||
if ($pattern === '') {
|
||||
throw new \InvalidArgumentException('Das Suchmuster darf nicht leer sein.');
|
||||
}
|
||||
if ($reason === '') {
|
||||
throw new \InvalidArgumentException('Bitte begruenden, warum dieser Fehler harmlos ist.');
|
||||
}
|
||||
if (!in_array($matchType, self::MATCH_TYPES, true)) {
|
||||
$matchType = 'contains';
|
||||
}
|
||||
|
||||
// Ungueltige Ausdruecke sollen beim Anlegen auffallen, nicht spaeter
|
||||
// still bei jedem eingehenden Fehler.
|
||||
if ($matchType === 'regex' && @preg_match('/' . str_replace('/', '\/', $pattern) . '/i', '') === false) {
|
||||
throw new \InvalidArgumentException('Der regulaere Ausdruck ist ungueltig.');
|
||||
}
|
||||
|
||||
$stmt = $this->db->prepare('
|
||||
INSERT INTO bugtracker_ignore_rules
|
||||
(project_slug, match_type, pattern, reason, alert_on_rate, enabled, created_by, created_at)
|
||||
VALUES (:slug, :match_type, :pattern, :reason, :rate, 1, :created_by, UTC_TIMESTAMP())
|
||||
');
|
||||
$stmt->execute([
|
||||
':slug' => $projectSlug !== null && $projectSlug !== '' && $projectSlug !== 'all' ? $projectSlug : null,
|
||||
':match_type' => $matchType,
|
||||
':pattern' => $pattern,
|
||||
':reason' => $reason,
|
||||
':rate' => $alertOnRate !== null && $alertOnRate > 0 ? $alertOnRate : null,
|
||||
':created_by' => $createdBy,
|
||||
]);
|
||||
|
||||
self::$cache = null;
|
||||
|
||||
return (int)$this->db->lastInsertId();
|
||||
}
|
||||
|
||||
public function setEnabled(int $ruleId, bool $enabled): bool
|
||||
{
|
||||
$stmt = $this->db->prepare('UPDATE bugtracker_ignore_rules SET enabled = :enabled WHERE id = :id');
|
||||
$stmt->execute([':enabled' => $enabled ? 1 : 0, ':id' => $ruleId]);
|
||||
|
||||
self::$cache = null;
|
||||
|
||||
return $stmt->rowCount() > 0;
|
||||
}
|
||||
|
||||
public function delete(int $ruleId): bool
|
||||
{
|
||||
$stmt = $this->db->prepare('DELETE FROM bugtracker_ignore_rules WHERE id = :id');
|
||||
$stmt->execute([':id' => $ruleId]);
|
||||
|
||||
self::$cache = null;
|
||||
|
||||
return $stmt->rowCount() > 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Wendet eine neu angelegte Regel rueckwirkend an: passende offene Items
|
||||
* werden stummgeschaltet. Ohne das muesste man sie einzeln nachpflegen.
|
||||
*
|
||||
* @return int Anzahl der betroffenen Items
|
||||
*/
|
||||
public function applyRetroactively(int $ruleId): int
|
||||
{
|
||||
$stmt = $this->db->prepare('SELECT * FROM bugtracker_ignore_rules WHERE id = :id');
|
||||
$stmt->execute([':id' => $ruleId]);
|
||||
$rule = $stmt->fetch();
|
||||
|
||||
if (!is_array($rule)) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
$where = ['status IN ("open","planned","in_progress")'];
|
||||
$params = [];
|
||||
|
||||
if ($rule['project_slug'] !== null && $rule['project_slug'] !== '') {
|
||||
$where[] = 'project_slug = :slug';
|
||||
$params[':slug'] = $rule['project_slug'];
|
||||
}
|
||||
|
||||
$candidates = $this->db->prepare(
|
||||
'SELECT id, title, error_message FROM bugtracker_items WHERE ' . implode(' AND ', $where) . ' LIMIT 2000'
|
||||
);
|
||||
$candidates->execute($params);
|
||||
|
||||
$affected = 0;
|
||||
$update = $this->db->prepare('
|
||||
UPDATE bugtracker_items
|
||||
SET status = "ignored", ignore_rule_id = :rule_id
|
||||
WHERE id = :id
|
||||
');
|
||||
|
||||
foreach ($candidates->fetchAll() ?: [] as $item) {
|
||||
$haystack = trim(($item['error_message'] ?? '') . "\n" . ($item['title'] ?? ''));
|
||||
if ($haystack === '' || !self::ruleMatches($rule, null, $haystack)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$update->execute([':rule_id' => $ruleId, ':id' => (int)$item['id']]);
|
||||
$affected++;
|
||||
}
|
||||
|
||||
return $affected;
|
||||
}
|
||||
}
|
||||
@@ -9,6 +9,9 @@ use Deploymentcenter\Modules\Bugtracker\BugRepo;
|
||||
use Deploymentcenter\Modules\Notify\WebhookDispatcher;
|
||||
use PDO;
|
||||
|
||||
// MetricStore und MonitorRepo liegen im selben Namensraum und werden
|
||||
// automatisch geladen.
|
||||
|
||||
/**
|
||||
* Watchdog-Evaluator.
|
||||
*
|
||||
@@ -50,8 +53,21 @@ final class Evaluator
|
||||
|
||||
$candidates = $monitorRepo->getEvaluationCandidates();
|
||||
|
||||
// Zustaende aller Monitore, damit die Abhaengigkeitspruefung ohne
|
||||
// weitere Abfragen auskommt.
|
||||
$stateBySource = [];
|
||||
$parentBySource = [];
|
||||
foreach ($candidates as $monitor) {
|
||||
$source = (string)$monitor['source'];
|
||||
$stateBySource[$source] = (string)$monitor['state'];
|
||||
$parentBySource[$source] = !empty($monitor['parent_source'])
|
||||
? (string)$monitor['parent_source']
|
||||
: null;
|
||||
}
|
||||
|
||||
$changes = [];
|
||||
$checked = 0;
|
||||
$suppressed = [];
|
||||
|
||||
foreach ($candidates as $monitor) {
|
||||
$checked++;
|
||||
@@ -66,6 +82,10 @@ final class Evaluator
|
||||
$reason = self::reasonFor($monitor, $target);
|
||||
$monitorRepo->setState((int)$monitor['id'], $target, $reason);
|
||||
|
||||
// Zustandswechsel im Bild mitfuehren, damit ein spaeter geprueftes
|
||||
// Kind den frisch gefallenen Parent bereits sieht.
|
||||
$stateBySource[(string)$monitor['source']] = $target;
|
||||
|
||||
$eventLog->logEvent(
|
||||
(string)$monitor['source'],
|
||||
(string)$monitor['instance'],
|
||||
@@ -76,13 +96,34 @@ final class Evaluator
|
||||
$reason
|
||||
);
|
||||
|
||||
// Faellt ein Hypervisor aus, sind seine VMs zwangslaeufig auch weg.
|
||||
// Ohne diese Pruefung erzeugt ein einzelner Ausfall so viele
|
||||
// Meldungen, wie Kinder daran haengen - bei einem Node mit zwoelf
|
||||
// VMs also dreizehn Alarme fuer ein Problem.
|
||||
$blockingParent = self::findDownAncestor(
|
||||
(string)$monitor['source'],
|
||||
$parentBySource,
|
||||
$stateBySource
|
||||
);
|
||||
|
||||
$changes[] = [
|
||||
'source' => $monitor['source'],
|
||||
'from' => $current,
|
||||
'to' => $target,
|
||||
'reason' => $reason,
|
||||
'source' => $monitor['source'],
|
||||
'from' => $current,
|
||||
'to' => $target,
|
||||
'reason' => $reason,
|
||||
'suppressed' => $blockingParent,
|
||||
];
|
||||
|
||||
$monitorRepo->setAlertSuppression((int)$monitor['id'], $blockingParent);
|
||||
|
||||
if ($blockingParent !== null) {
|
||||
$suppressed[] = [
|
||||
'source' => $monitor['source'],
|
||||
'parent' => $blockingParent,
|
||||
];
|
||||
continue;
|
||||
}
|
||||
|
||||
// Stummgeschaltete Monitore erscheinen im Dashboard, loesen aber
|
||||
// keine Benachrichtigung aus.
|
||||
if (empty($monitor['is_muted'])) {
|
||||
@@ -99,6 +140,12 @@ final class Evaluator
|
||||
Logger::warning('Lease-Bereinigung fehlgeschlagen', ['error' => $e->getMessage()]);
|
||||
}
|
||||
|
||||
// Alten Metrik-Verlauf abraeumen (hoechstens einmal pro Stunde).
|
||||
$purgedMetrics = 0;
|
||||
if (self::shouldPurgeMetrics($db)) {
|
||||
$purgedMetrics = (new MetricStore($db))->purge();
|
||||
}
|
||||
|
||||
$durationMs = (int)round((microtime(true) - $started) * 1000);
|
||||
self::recordRun($db, $durationMs, count($changes));
|
||||
|
||||
@@ -110,11 +157,73 @@ final class Evaluator
|
||||
'checked' => $checked,
|
||||
'changed' => count($changes),
|
||||
'changes' => $changes,
|
||||
'suppressed' => $suppressed,
|
||||
'released_leases' => $releasedLeases,
|
||||
'purged_metrics' => $purgedMetrics,
|
||||
'duration_ms' => $durationMs,
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* Sucht den naechsten Vorfahren, der selbst unten ist.
|
||||
*
|
||||
* @param array<string,?string> $parentBySource
|
||||
* @param array<string,string> $stateBySource
|
||||
* @return string|null Name des ausgefallenen Vorfahren, oder null
|
||||
*/
|
||||
private static function findDownAncestor(
|
||||
string $source,
|
||||
array $parentBySource,
|
||||
array $stateBySource
|
||||
): ?string {
|
||||
$seen = [];
|
||||
$current = $parentBySource[$source] ?? null;
|
||||
|
||||
// Tiefenbegrenzung und Zyklusschutz: ein falsch gesetzter Parent darf
|
||||
// keine Endlosschleife ausloesen.
|
||||
while ($current !== null && !isset($seen[$current]) && count($seen) < 10) {
|
||||
$seen[$current] = true;
|
||||
|
||||
$parentState = $stateBySource[$current] ?? null;
|
||||
if ($parentState === 'down' || $parentState === 'error') {
|
||||
return $current;
|
||||
}
|
||||
|
||||
$current = $parentBySource[$current] ?? null;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Der Verlauf wird hoechstens stuendlich bereinigt, nicht bei jedem Lauf. */
|
||||
private static function shouldPurgeMetrics(PDO $db): bool
|
||||
{
|
||||
try {
|
||||
$stmt = $db->query("
|
||||
SELECT last_run_utc FROM watchdog_cron_jobs WHERE name = 'metrics_cleanup'
|
||||
");
|
||||
$lastRun = $stmt !== false ? $stmt->fetchColumn() : false;
|
||||
|
||||
if ($lastRun === false || $lastRun === null) {
|
||||
$due = true;
|
||||
} else {
|
||||
$due = (time() - (int)strtotime((string)$lastRun . ' UTC')) > 3600;
|
||||
}
|
||||
|
||||
if ($due) {
|
||||
$db->prepare('
|
||||
INSERT INTO watchdog_cron_jobs (name, interval_sec, last_run_utc, enabled)
|
||||
VALUES ("metrics_cleanup", 86400, UTC_TIMESTAMP(), 1)
|
||||
ON DUPLICATE KEY UPDATE last_run_utc = UTC_TIMESTAMP()
|
||||
')->execute();
|
||||
}
|
||||
|
||||
return $due;
|
||||
} catch (\Throwable $e) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Ermittelt den Zustand, den ein Monitor haben sollte.
|
||||
* null bedeutet: keine Aenderung noetig.
|
||||
|
||||
@@ -0,0 +1,270 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace Deploymentcenter\Modules\Watchdog;
|
||||
|
||||
use Deploymentcenter\Core\Logger;
|
||||
use PDO;
|
||||
|
||||
/**
|
||||
* Verlauf der Heartbeat-Metriken.
|
||||
*
|
||||
* Bisher wurde metrics_json bei jedem Heartbeat ueberschrieben - es gab immer
|
||||
* nur den letzten Moment. Damit laesst sich "die Platte laeuft seit drei Tagen
|
||||
* voll" nicht erkennen, sondern nur "die Platte ist voll".
|
||||
*
|
||||
* Bewusst schmal gehalten: Das hier ist die Datengrundlage fuer Sparklines und
|
||||
* Abweichungsalarme im Dashboard, keine Zeitreihendatenbank. Wer echte Analyse
|
||||
* braucht, ist mit Prometheus besser bedient.
|
||||
*/
|
||||
final class MetricStore
|
||||
{
|
||||
/** Aufbewahrung der Rohwerte in Tagen. */
|
||||
private const RETENTION_DAYS = 14;
|
||||
|
||||
/** Maximale Anzahl Metriken je Heartbeat - schuetzt vor Ausreissern. */
|
||||
private const MAX_KEYS_PER_BEAT = 25;
|
||||
|
||||
private PDO $db;
|
||||
|
||||
public function __construct(PDO $db)
|
||||
{
|
||||
$this->db = $db;
|
||||
}
|
||||
|
||||
/**
|
||||
* Nimmt die numerischen Werte eines Heartbeats auf.
|
||||
* Nicht numerische Werte werden uebergangen.
|
||||
*
|
||||
* @param mixed $metrics
|
||||
*/
|
||||
public function record(string $source, string $instance, $metrics): int
|
||||
{
|
||||
if (!is_array($metrics) && !is_object($metrics)) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
$flat = self::flatten((array)$metrics);
|
||||
if ($flat === []) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
try {
|
||||
$stmt = $this->db->prepare('
|
||||
INSERT INTO watchdog_metrics (source, instance, metric_key, metric_value, recorded_utc)
|
||||
VALUES (:source, :instance, :metric_key, :metric_value, UTC_TIMESTAMP())
|
||||
');
|
||||
|
||||
$written = 0;
|
||||
foreach ($flat as $key => $value) {
|
||||
if ($written >= self::MAX_KEYS_PER_BEAT) {
|
||||
break;
|
||||
}
|
||||
|
||||
$stmt->execute([
|
||||
':source' => mb_substr($source, 0, 100),
|
||||
':instance' => mb_substr($instance, 0, 100),
|
||||
':metric_key' => mb_substr($key, 0, 64),
|
||||
':metric_value' => $value,
|
||||
]);
|
||||
$written++;
|
||||
}
|
||||
|
||||
return $written;
|
||||
} catch (\Throwable $e) {
|
||||
// Der Verlauf ist Beiwerk; ein Heartbeat darf daran nicht scheitern.
|
||||
Logger::warning('Metriken nicht gespeichert', [
|
||||
'source' => $source,
|
||||
'error' => $e->getMessage(),
|
||||
]);
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Verlauf einer Metrik, auf Zeitfenster verdichtet.
|
||||
*
|
||||
* @return list<array{bucket:string,avg:float,min:float,max:float,samples:int}>
|
||||
*/
|
||||
public function history(
|
||||
string $source,
|
||||
string $metricKey,
|
||||
int $hours = 24,
|
||||
string $instance = 'default',
|
||||
int $bucketMinutes = 15
|
||||
): array {
|
||||
$hours = max(1, min($hours, 24 * self::RETENTION_DAYS));
|
||||
$bucketMinutes = max(1, min($bucketMinutes, 1440));
|
||||
|
||||
try {
|
||||
// Zeitstempel auf das Raster runden, damit gleichmaessige
|
||||
// Stuetzstellen entstehen.
|
||||
$stmt = $this->db->prepare('
|
||||
SELECT
|
||||
FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(recorded_utc) / (' . $bucketMinutes . ' * 60))
|
||||
* (' . $bucketMinutes . ' * 60)) AS bucket,
|
||||
AVG(metric_value) AS avg_value,
|
||||
MIN(metric_value) AS min_value,
|
||||
MAX(metric_value) AS max_value,
|
||||
COUNT(*) AS samples
|
||||
FROM watchdog_metrics
|
||||
WHERE source = :source
|
||||
AND instance = :instance
|
||||
AND metric_key = :metric_key
|
||||
AND recorded_utc > (UTC_TIMESTAMP() - INTERVAL ' . $hours . ' HOUR)
|
||||
GROUP BY bucket
|
||||
ORDER BY bucket ASC
|
||||
');
|
||||
$stmt->execute([
|
||||
':source' => $source,
|
||||
':instance' => $instance,
|
||||
':metric_key' => $metricKey,
|
||||
]);
|
||||
|
||||
$out = [];
|
||||
foreach ($stmt->fetchAll() ?: [] as $row) {
|
||||
$out[] = [
|
||||
'bucket' => (string)$row['bucket'],
|
||||
'avg' => (float)$row['avg_value'],
|
||||
'min' => (float)$row['min_value'],
|
||||
'max' => (float)$row['max_value'],
|
||||
'samples' => (int)$row['samples'],
|
||||
];
|
||||
}
|
||||
|
||||
return $out;
|
||||
} catch (\Throwable $e) {
|
||||
Logger::warning('Metrik-Verlauf nicht abrufbar', ['error' => $e->getMessage()]);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Welche Metriken liefert dieser Monitor ueberhaupt?
|
||||
*
|
||||
* @return list<string>
|
||||
*/
|
||||
public function keysFor(string $source, string $instance = 'default'): array
|
||||
{
|
||||
try {
|
||||
$stmt = $this->db->prepare('
|
||||
SELECT DISTINCT metric_key
|
||||
FROM watchdog_metrics
|
||||
WHERE source = :source AND instance = :instance
|
||||
AND recorded_utc > (UTC_TIMESTAMP() - INTERVAL 7 DAY)
|
||||
ORDER BY metric_key ASC
|
||||
');
|
||||
$stmt->execute([':source' => $source, ':instance' => $instance]);
|
||||
return $stmt->fetchAll(PDO::FETCH_COLUMN) ?: [];
|
||||
} catch (\Throwable $e) {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Vergleicht den aktuellen Wert mit dem eigenen Verlauf.
|
||||
*
|
||||
* Dieser Ansatz braucht kein Projektwissen: Statt fester Schwellwerte je
|
||||
* Anwendung wird gemeldet, was deutlich vom bisherigen Verhalten desselben
|
||||
* Monitors abweicht.
|
||||
*
|
||||
* @return array{deviates:bool,current:float,baseline:float,factor:float}|null
|
||||
*/
|
||||
public function deviation(string $source, string $metricKey, string $instance = 'default', float $factor = 3.0): ?array
|
||||
{
|
||||
try {
|
||||
$stmt = $this->db->prepare('
|
||||
SELECT
|
||||
(SELECT metric_value FROM watchdog_metrics
|
||||
WHERE source = :s1 AND instance = :i1 AND metric_key = :k1
|
||||
ORDER BY recorded_utc DESC LIMIT 1) AS current_value,
|
||||
(SELECT AVG(metric_value) FROM watchdog_metrics
|
||||
WHERE source = :s2 AND instance = :i2 AND metric_key = :k2
|
||||
AND recorded_utc BETWEEN (UTC_TIMESTAMP() - INTERVAL 7 DAY)
|
||||
AND (UTC_TIMESTAMP() - INTERVAL 1 HOUR)) AS baseline_value
|
||||
');
|
||||
$stmt->execute([
|
||||
':s1' => $source, ':i1' => $instance, ':k1' => $metricKey,
|
||||
':s2' => $source, ':i2' => $instance, ':k2' => $metricKey,
|
||||
]);
|
||||
$row = $stmt->fetch();
|
||||
|
||||
if (!is_array($row) || $row['current_value'] === null || $row['baseline_value'] === null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
$current = (float)$row['current_value'];
|
||||
$baseline = (float)$row['baseline_value'];
|
||||
|
||||
if (abs($baseline) < 0.0001) {
|
||||
return null;
|
||||
}
|
||||
|
||||
$ratio = $current / $baseline;
|
||||
|
||||
return [
|
||||
'deviates' => $ratio >= $factor || $ratio <= (1 / $factor),
|
||||
'current' => $current,
|
||||
'baseline' => $baseline,
|
||||
'factor' => $ratio,
|
||||
];
|
||||
} catch (\Throwable $e) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** Entfernt Werte, die aelter als die Aufbewahrungsfrist sind. */
|
||||
public function purge(): int
|
||||
{
|
||||
try {
|
||||
$stmt = $this->db->prepare(
|
||||
'DELETE FROM watchdog_metrics
|
||||
WHERE recorded_utc < (UTC_TIMESTAMP() - INTERVAL ' . self::RETENTION_DAYS . ' DAY)
|
||||
LIMIT 50000'
|
||||
);
|
||||
$stmt->execute();
|
||||
return $stmt->rowCount();
|
||||
} catch (\Throwable $e) {
|
||||
Logger::warning('Metrik-Bereinigung fehlgeschlagen', ['error' => $e->getMessage()]);
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Verschachtelte Metriken flach klopfen: {"cpu":{"load":1.2}} -> "cpu.load".
|
||||
*
|
||||
* @return array<string,float>
|
||||
*/
|
||||
private static function flatten(array $metrics, string $prefix = '', int $depth = 0): array
|
||||
{
|
||||
if ($depth > 3) {
|
||||
return [];
|
||||
}
|
||||
|
||||
$out = [];
|
||||
foreach ($metrics as $key => $value) {
|
||||
if (!is_string($key) && !is_int($key)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$name = $prefix === '' ? (string)$key : $prefix . '.' . $key;
|
||||
|
||||
if (is_array($value)) {
|
||||
$out += self::flatten($value, $name, $depth + 1);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (is_bool($value)) {
|
||||
$out[$name] = $value ? 1.0 : 0.0;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (is_numeric($value)) {
|
||||
$out[$name] = (float)$value;
|
||||
}
|
||||
}
|
||||
|
||||
return $out;
|
||||
}
|
||||
}
|
||||
@@ -72,6 +72,10 @@ final class MonitorRepo
|
||||
/**
|
||||
* Nimmt einen Heartbeat entgegen und legt den Monitor bei Bedarf an.
|
||||
*/
|
||||
/**
|
||||
* @param mixed $metrics
|
||||
* @param mixed $checks Gesundheitszustand, den die Anwendung selbst ermittelt hat.
|
||||
*/
|
||||
public function upsertHeartbeat(
|
||||
string $source,
|
||||
string $instance,
|
||||
@@ -81,18 +85,35 @@ final class MonitorRepo
|
||||
string $status,
|
||||
?string $message,
|
||||
?string $groupKey = null,
|
||||
?string $os = null
|
||||
?string $os = null,
|
||||
$checks = null
|
||||
): array {
|
||||
$metricsJson = (is_array($metrics) || is_object($metrics))
|
||||
? json_encode($metrics, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE)
|
||||
: null;
|
||||
|
||||
// Die Anwendung meldet ihren Gesundheitszustand selbst mit. Das
|
||||
// Deploymentcenter interpretiert die Namen der Pruefungen nicht - es
|
||||
// liest nur ok und message. Damit muss auf der Zielmaschine kein Port
|
||||
// geoeffnet werden, und jede Anwendung entscheidet selbst, was bei ihr
|
||||
// "gesund" bedeutet.
|
||||
$failing = self::failingChecks($checks);
|
||||
$healthJson = (is_array($checks) || is_object($checks))
|
||||
? json_encode($checks, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE)
|
||||
: null;
|
||||
|
||||
$state = match ($status) {
|
||||
'ok' => 'up',
|
||||
'warning' => 'warning',
|
||||
default => 'down',
|
||||
};
|
||||
|
||||
// Eine fehlgeschlagene Pruefung stuft einen als "ok" gemeldeten
|
||||
// Heartbeat herab: der Prozess laeuft, tut aber nicht, was er soll.
|
||||
if ($failing !== [] && $state === 'up') {
|
||||
$state = 'warning';
|
||||
}
|
||||
|
||||
$previous = $this->getMonitor($source, $instance);
|
||||
$previousState = $previous !== null ? (string)$previous['state'] : null;
|
||||
|
||||
@@ -102,12 +123,12 @@ final class MonitorRepo
|
||||
$stmt = $this->db->prepare('
|
||||
INSERT INTO watchdog_monitors (
|
||||
source, instance, type, state, last_state_change_utc, expected_interval_sec,
|
||||
last_seen_utc, last_status, last_message, metrics_json, group_key, os,
|
||||
created_utc, updated_utc
|
||||
last_seen_utc, last_status, last_message, metrics_json, health_json,
|
||||
failing_checks, group_key, os, created_utc, updated_utc
|
||||
) VALUES (
|
||||
:source, :instance, :type, :state, UTC_TIMESTAMP(), :interval,
|
||||
UTC_TIMESTAMP(), :last_status, :message, :metrics, :group_key, :os,
|
||||
UTC_TIMESTAMP(), UTC_TIMESTAMP()
|
||||
UTC_TIMESTAMP(), :last_status, :message, :metrics, :health,
|
||||
:failing, :group_key, :os, UTC_TIMESTAMP(), UTC_TIMESTAMP()
|
||||
)
|
||||
ON DUPLICATE KEY UPDATE
|
||||
-- Reihenfolge ist relevant: MySQL wertet die Zuweisungen von
|
||||
@@ -121,6 +142,8 @@ final class MonitorRepo
|
||||
last_status = VALUES(last_status),
|
||||
last_message = VALUES(last_message),
|
||||
metrics_json = VALUES(metrics_json),
|
||||
health_json = COALESCE(VALUES(health_json), health_json),
|
||||
failing_checks = VALUES(failing_checks),
|
||||
group_key = COALESCE(VALUES(group_key), group_key),
|
||||
os = COALESCE(VALUES(os), os),
|
||||
updated_utc = VALUES(updated_utc)
|
||||
@@ -135,6 +158,8 @@ final class MonitorRepo
|
||||
':last_status' => in_array($status, ['ok', 'warning', 'error'], true) ? $status : 'error',
|
||||
':message' => $message,
|
||||
':metrics' => $metricsJson,
|
||||
':health' => $healthJson,
|
||||
':failing' => $failing !== [] ? mb_substr(implode(', ', $failing), 0, 255) : null,
|
||||
':group_key' => $groupKey,
|
||||
':os' => $os,
|
||||
]);
|
||||
@@ -146,10 +171,52 @@ final class MonitorRepo
|
||||
|
||||
$monitor['_previous_state'] = $previousState;
|
||||
$monitor['_state_changed'] = $previousState !== null && $previousState !== $state;
|
||||
$monitor['_failing_checks'] = $failing;
|
||||
|
||||
return $monitor;
|
||||
}
|
||||
|
||||
/**
|
||||
* Ermittelt die Namen aller fehlgeschlagenen Pruefungen.
|
||||
*
|
||||
* Erwartetes Format, das die Anwendung mitschickt:
|
||||
* { "db": { "ok": true }, "feed": { "ok": false, "message": "..." } }
|
||||
*
|
||||
* Akzeptiert zur Bequemlichkeit auch { "db": true, "feed": false }.
|
||||
*
|
||||
* @param mixed $checks
|
||||
* @return list<string>
|
||||
*/
|
||||
public static function failingChecks($checks): array
|
||||
{
|
||||
if (!is_array($checks) && !is_object($checks)) {
|
||||
return [];
|
||||
}
|
||||
|
||||
$failing = [];
|
||||
foreach ((array)$checks as $name => $check) {
|
||||
if (!is_string($name)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (is_bool($check)) {
|
||||
if (!$check) {
|
||||
$failing[] = $name;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if (is_array($check) || is_object($check)) {
|
||||
$data = (array)$check;
|
||||
if (array_key_exists('ok', $data) && !filter_var($data['ok'], FILTER_VALIDATE_BOOLEAN)) {
|
||||
$failing[] = $name;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return $failing;
|
||||
}
|
||||
|
||||
/**
|
||||
* Legt einen Monitor manuell an (ohne Heartbeat).
|
||||
*
|
||||
@@ -357,6 +424,23 @@ final class MonitorRepo
|
||||
return $stmt->rowCount() > 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Vermerkt, dass die Alarmierung dieses Monitors unterdrueckt wird, weil
|
||||
* ein uebergeordnetes System ausgefallen ist. Der Zustand bleibt sichtbar,
|
||||
* nur die Benachrichtigung entfaellt.
|
||||
*/
|
||||
public function setAlertSuppression(int $id, ?string $blockingParent): bool
|
||||
{
|
||||
$stmt = $this->db->prepare('
|
||||
UPDATE watchdog_monitors
|
||||
SET alert_suppressed_by = :parent
|
||||
WHERE id = :id
|
||||
');
|
||||
$stmt->execute([':parent' => $blockingParent, ':id' => $id]);
|
||||
|
||||
return $stmt->rowCount() > 0;
|
||||
}
|
||||
|
||||
public function setMaintenance(string $source, string $instance, ?string $untilUtc): bool
|
||||
{
|
||||
$stmt = $this->db->prepare('
|
||||
|
||||
Reference in New Issue
Block a user