[Chore] Prometheus 알림 룰을 레포로 편입하고 PodNotReady 오탐 수정

k8s/kustomization.yaml

@@ -13,6 +13,7 @@ resources:
   - redis/pvc.yaml
   - minio/deployment.yaml
   - minio/service.yaml
+  - monitoring/prometheusrule.yaml
 images:
   - name: ghcr.io/semosan/semosan_be
     newTag: ae0e07ca129b8116f9f3428bed3462d9ba49b96d

k8s/monitoring/prometheusrule.yaml

@@ -0,0 +1,72 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: semosan-app-alerts
+  namespace: monitoring
+  labels:
+    release: prometheus
+spec:
+  groups:
+    - name: semosan-app
+      rules:
+        - alert: HighErrorRate
+          expr: sum(rate(http_server_requests_seconds_count{namespace="semosan",status=~"5.."}[5m])) / sum(rate(http_server_requests_seconds_count{namespace="semosan"}[5m])) > 0.05
+          for: 3m
+          labels:
+            severity: critical
+          annotations:
+            summary: "API 5xx 에러율 5% 초과"
+            description: "최근 5분간 에러율: {{ $value | humanizePercentage }}"
+            dashboard: "http://10.99.110.31/d/semosan-app/semosan-application-monitor"
+
+        - alert: HighResponseTime
+          expr: histogram_quantile(0.95, sum(rate(http_server_requests_seconds_bucket{namespace="semosan"}[5m])) by (le)) > 3
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: "API 응답시간 p95 > 3초"
+            description: "p95 응답시간: {{ $value }}초"
+            dashboard: "http://10.99.110.31/d/semosan-app/semosan-application-monitor"
+
+        - alert: JvmHeapHigh
+          expr: jvm_memory_used_bytes{namespace="semosan",area="heap"} / jvm_memory_max_bytes{namespace="semosan",area="heap"} > 0.85
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: "JVM Heap 사용률 85% 초과"
+            description: "현재 Heap 사용률: {{ $value | humanizePercentage }}"
+            dashboard: "http://10.99.110.31/d/semosan-app/semosan-application-monitor"
+
+        - alert: PodRestart
+          expr: increase(kube_pod_container_status_restarts_total{namespace="semosan"}[1h]) > 3
+          for: 1m
+          labels:
+            severity: critical
+          annotations:
+            summary: "Pod 1시간 내 3회 이상 재시작"
+            description: "Pod {{ $labels.pod }} 재시작 횟수: {{ $value }}"
+            dashboard: "http://10.99.110.31/d/semosan-app/semosan-application-monitor"
+
+        - alert: PodNotReady
+          expr: |
+            kube_pod_status_ready{namespace="semosan",condition="true"} == 0
+            unless on(namespace,pod) kube_pod_status_phase{phase=~"Succeeded|Failed"} == 1
+          for: 5m
+          labels:
+            severity: critical
+          annotations:
+            summary: "Pod가 5분 이상 Ready 아님"
+            description: "Pod {{ $labels.pod }}이 Ready 상태가 아닙니다"
+            dashboard: "http://10.99.110.31/d/semosan-app/semosan-application-monitor"
+
+        - alert: HighDbConnectionUsage
+          expr: hikaricp_connections_active{namespace="semosan"} / hikaricp_connections_max{namespace="semosan"} > 0.8
+          for: 3m
+          labels:
+            severity: warning
+          annotations:
+            summary: "DB 커넥션 풀 80% 초과"
+            description: "활성 커넥션: {{ $value | humanizePercentage }}"
+            dashboard: "http://10.99.110.31/d/semosan-app/semosan-application-monitor"