using System.Text.Json.Nodes; namespace MaksIT.ClusterConsole.Shared; public sealed record ClusterIssue( string Id, string Message, string ObjectName, string Kind, string Age, DateTimeOffset OccurredAt, string Severity, string State); public sealed record ClusterIssueSet( IReadOnlyList Warnings, IReadOnlyList Errors) { public static ClusterIssueSet Empty { get; } = new([], []); } public static class ClusterIssues { public const string Active = "Active"; public const string Resolved = "Resolved"; public static readonly TimeSpan StaleAfter = TimeSpan.FromMinutes(10); private static readonly HashSet HealthyNodeConditions = new(StringComparer.Ordinal) { "Ready", "HostUpgrades", "SchedulingDisabled", "EtcdIsVoter" }; private static readonly HashSet UnhealthyWhenFalseNodeConditions = new(StringComparer.Ordinal) { "EtcdIsVoter" }; public static ClusterIssueSet Collect( IEnumerable nodes, IEnumerable events, IEnumerable pods, DateTimeOffset? utcNow = null, IEnumerable? services = null, IEnumerable? claims = null) { var now = utcNow ?? DateTimeOffset.UtcNow; var podByUid = pods .Select(p => (Uid: JsonPath.Uid(p), Pod: p)) .Where(p => !string.IsNullOrWhiteSpace(p.Uid)) .ToDictionary(p => p.Uid, p => p.Pod, StringComparer.Ordinal); var warnings = new List(); var errors = new List(); foreach (var node in nodes) warnings.AddRange(NodeWarnings(node, now)); foreach (var service in services ?? []) warnings.AddRange(ServiceWarnings(service, now)); foreach (var claim in claims ?? []) warnings.AddRange(ClaimWarnings(claim, now)); foreach (var issue in EventIssues(events, podByUid, now)) { if (issue.Severity == "Error") errors.Add(issue); else warnings.Add(issue); } return new ClusterIssueSet(Rank(warnings), Rank(errors)); } public static string Caption(string noun, IReadOnlyList issues) { var resolved = 0; foreach (var issue in issues) { if (issue.State == Resolved) resolved++; } var active = issues.Count - resolved; if (resolved == 0) return $"{noun}: {active}"; if (active == 0) return $"{noun}: {resolved} resolved"; return $"{noun}: {active} ({resolved} resolved)"; } public static string Format(ClusterIssueSet issues) { var lines = new List(); foreach (var error in issues.Errors) lines.Add(Line("ERROR", error)); foreach (var warning in issues.Warnings) lines.Add(Line("WARN", warning)); return lines.Count == 0 ? "No overview warnings or errors." : string.Join(Environment.NewLine, lines); } private static string Line(string severity, ClusterIssue issue) => $"{severity} {issue.State} {issue.Kind} {issue.ObjectName} ({issue.Age}): {issue.Message}"; private static List Rank(IEnumerable issues) => issues .OrderBy(i => i.State == Resolved ? 1 : 0) .ThenByDescending(i => i.OccurredAt) .ToList(); private static IEnumerable NodeWarnings(JsonObject node, DateTimeOffset now) { var name = JsonPath.Name(node); JsonPath.TryTimestamp(node["metadata"]?["creationTimestamp"], out var created); var createdAt = created == default ? now : created; var nodeAge = JsonPath.Age(createdAt, now); var conditions = node["status"]?["conditions"] as JsonArray; if (conditions is null) yield break; foreach (var condition in conditions.OfType()) { var type = Text(condition["type"]); if (!IsUnhealthyNodeCondition(type, condition["status"])) continue; var message = Text(condition["message"]); if (string.IsNullOrWhiteSpace(message)) message = type; yield return new ClusterIssue( $"node/{JsonPath.Uid(node)}/{type}", message, name, "Node", nodeAge, createdAt, "Warning", Active); } } private static IEnumerable ServiceWarnings(JsonObject service, DateTimeOffset now) { var status = JsonPath.ServiceStatus(service); if (status is not ("Unreachable" or "Pending")) yield break; JsonPath.TryTimestamp(service["metadata"]?["creationTimestamp"], out var created); var createdAt = created == default ? now : created; var message = status == "Unreachable" ? "LoadBalancer address does not match the requested address" : "LoadBalancer has no address"; yield return new ClusterIssue( $"service/{JsonPath.Uid(service)}/{status}", message, NamespacedName(service), "Service", JsonPath.Age(createdAt, now), createdAt, "Warning", Active); } private static IEnumerable ClaimWarnings(JsonObject claim, DateTimeOffset now) { var phase = Text(claim["status"]?["phase"]); if (!phase.Equals("Pending", StringComparison.OrdinalIgnoreCase)) yield break; JsonPath.TryTimestamp(claim["metadata"]?["creationTimestamp"], out var created); var createdAt = created == default ? now : created; var detail = (claim["status"]?["conditions"] as JsonArray)? .OfType() .Select(condition => Text(condition["message"])) .FirstOrDefault(message => !string.IsNullOrWhiteSpace(message)); yield return new ClusterIssue( $"pvc/{JsonPath.Uid(claim)}/Pending", string.IsNullOrWhiteSpace(detail) ? "PersistentVolumeClaim is Pending" : detail, NamespacedName(claim), "PersistentVolumeClaim", JsonPath.Age(createdAt, now), createdAt, "Warning", Active); } private static string NamespacedName(JsonObject item) { var name = JsonPath.Name(item); var ns = JsonPath.Namespace(item); return string.IsNullOrWhiteSpace(ns) ? name : $"{ns}/{name}"; } private static IEnumerable EventIssues( IEnumerable events, IReadOnlyDictionary pods, DateTimeOffset now) { var latestWarning = new Dictionary(StringComparer.Ordinal); var latestNormal = new Dictionary(StringComparer.Ordinal); foreach (var ev in events) { var type = Text(ev["type"]); var involved = ev["involvedObject"] as JsonObject; var key = InvolvedKey(involved); var at = EventTime(ev); if (type.Equals("Normal", StringComparison.OrdinalIgnoreCase)) { var normalKey = $"{key}\0{Text(ev["reason"])}"; if (!latestNormal.TryGetValue(normalKey, out var existing) || at > existing) latestNormal[normalKey] = at; continue; } if (!type.Equals("Warning", StringComparison.OrdinalIgnoreCase) && !type.Equals("Error", StringComparison.OrdinalIgnoreCase)) continue; if (latestWarning.TryGetValue(key, out var previous) && previous.At >= at) continue; latestWarning[key] = (ev, at); } foreach (var (ev, at) in latestWarning.Values) { var involved = ev["involvedObject"] as JsonObject; var kind = Text(involved?["kind"]); var podStillUnhealthy = kind.Equals("Pod", StringComparison.OrdinalIgnoreCase) && ShouldKeepPodEvent(involved, pods); if (kind.Equals("Pod", StringComparison.OrdinalIgnoreCase) && !podStillUnhealthy) continue; var message = Text(ev["message"]); if (string.IsNullOrWhiteSpace(message)) message = Text(ev["reason"]); var severity = Text(ev["type"]).Equals("Error", StringComparison.OrdinalIgnoreCase) ? "Error" : "Warning"; var reasonKey = $"{InvolvedKey(involved)}\0{Text(ev["reason"])}"; var resolvedByNormal = latestNormal.TryGetValue(reasonKey, out var normalAt) && normalAt >= at; var stale = now - at >= StaleAfter; var state = resolvedByNormal || (stale && !podStillUnhealthy) ? Resolved : Active; yield return new ClusterIssue( JsonPath.Uid(ev), message, Text(involved?["name"]), "Event", JsonPath.Age(at, now), at, severity, state); } } private static string InvolvedKey(JsonObject? involved) { var uid = Text(involved?["uid"]); if (!string.IsNullOrWhiteSpace(uid)) return uid; return $"{Text(involved?["kind"])}/{Text(involved?["namespace"])}/{Text(involved?["name"])}"; } private static bool ShouldKeepPodEvent(JsonObject? involved, IReadOnlyDictionary pods) { var uid = Text(involved?["uid"]); if (string.IsNullOrWhiteSpace(uid) || !pods.TryGetValue(uid, out var pod)) return false; if (PodHasIssues(pod)) return true; var priority = pod["spec"]?["priority"] as JsonValue; return priority is not null && priority.TryGetValue(out var value) && value >= 500_000; } private static bool PodHasIssues(JsonObject pod) { var phase = Text(pod["status"]?["phase"]); if (!phase.Equals("Running", StringComparison.OrdinalIgnoreCase)) return true; var conditions = pod["status"]?["conditions"] as JsonArray; var ready = conditions?.OfType() .FirstOrDefault(c => Text(c["type"]) == "Ready"); if (ready is not null && !IsTrue(ready["status"])) return true; var statuses = pod["status"]?["containerStatuses"] as JsonArray; return statuses?.OfType().Any(status => { var waiting = Text(status["state"]?["waiting"]?["reason"]); return waiting.Equals("CrashLoopBackOff", StringComparison.OrdinalIgnoreCase); }) == true; } private static DateTimeOffset EventTime(JsonObject ev) { foreach (var node in new[] { ev["lastTimestamp"], ev["eventTime"], ev["series"]?["lastObservedTime"], ev["metadata"]?["creationTimestamp"] }) { if (JsonPath.TryTimestamp(node, out var when)) return when; } return DateTimeOffset.UnixEpoch; } private static string Text(JsonNode? node) => node is JsonValue value ? value.TryGetValue(out var text) ? text ?? string.Empty : value.ToString() ?? string.Empty : node?.ToString() ?? string.Empty; private static bool IsUnhealthyNodeCondition(string type, JsonNode? status) { if (UnhealthyWhenFalseNodeConditions.Contains(type)) return IsFalse(status); return IsTrue(status) && !HealthyNodeConditions.Contains(type); } private static bool IsTrue(JsonNode? node) => IsConditionStatus(node, true, "True"); private static bool IsFalse(JsonNode? node) => IsConditionStatus(node, false, "False"); private static bool IsConditionStatus(JsonNode? node, bool flagValue, string textValue) { if (node is not JsonValue value) return false; if (value.TryGetValue(out var flag)) return flag == flagValue; return value.TryGetValue(out var text) && text.Equals(textValue, StringComparison.OrdinalIgnoreCase); } }