File size: 3,694 Bytes
23804b3 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 |
# Prometheus Alert Rules for Cyber-LLM
groups:
- name: cyber-llm-alerts
rules:
# High error rate
- alert: HighErrorRate
expr: rate(http_requests_total{status=~"5.."}[5m]) / rate(http_requests_total[5m]) > 0.05
for: 5m
labels:
severity: critical
service: cyber-llm
annotations:
summary: "High error rate detected"
description: "Error rate is above 5% for {{ $labels.instance }}"
# High response time
- alert: HighResponseTime
expr: histogram_quantile(0.95, rate(http_request_duration_seconds_bucket[5m])) > 2
for: 3m
labels:
severity: warning
service: cyber-llm
annotations:
summary: "High response time detected"
description: "95th percentile response time is {{ $value }}s"
# Low availability
- alert: ServiceDown
expr: up{job="cyber-llm-api"} == 0
for: 1m
labels:
severity: critical
service: cyber-llm
annotations:
summary: "Cyber-LLM API is down"
description: "Service {{ $labels.instance }} is down"
# High memory usage
- alert: HighMemoryUsage
expr: process_resident_memory_bytes / node_memory_MemTotal_bytes > 0.8
for: 5m
labels:
severity: warning
service: cyber-llm
annotations:
summary: "High memory usage detected"
description: "Memory usage is {{ $value | humanizePercentage }}"
# High CPU usage
- alert: HighCPUUsage
expr: rate(process_cpu_seconds_total[5m]) > 0.8
for: 10m
labels:
severity: warning
service: cyber-llm
annotations:
summary: "High CPU usage detected"
description: "CPU usage is {{ $value | humanizePercentage }}"
# GPU utilization
- alert: HighGPUUtilization
expr: nvidia_gpu_utilization > 90
for: 15m
labels:
severity: warning
service: cyber-llm
annotations:
summary: "High GPU utilization"
description: "GPU utilization is {{ $value }}%"
# Low disk space
- alert: LowDiskSpace
expr: (node_filesystem_avail_bytes / node_filesystem_size_bytes) < 0.1
for: 5m
labels:
severity: critical
service: cyber-llm
annotations:
summary: "Low disk space"
description: "Disk space is below 10% on {{ $labels.instance }}"
# Failed model predictions
- alert: HighModelFailureRate
expr: rate(model_prediction_failures_total[5m]) / rate(model_predictions_total[5m]) > 0.1
for: 3m
labels:
severity: warning
service: cyber-llm
annotations:
summary: "High model failure rate"
description: "Model failure rate is {{ $value | humanizePercentage }}"
# Suspicious activity detection
- alert: SuspiciousActivity
expr: rate(security_violations_total[5m]) > 5
for: 1m
labels:
severity: critical
service: cyber-llm-security
annotations:
summary: "Suspicious activity detected"
description: "Security violations detected at {{ $value }} per second"
# Agent orchestration failures
- alert: AgentOrchestrationFailure
expr: rate(agent_orchestration_failures_total[5m]) > 0.1
for: 2m
labels:
severity: warning
service: cyber-llm-agents
annotations:
summary: "Agent orchestration failures"
description: "Agent orchestration failure rate is high"
|