1tar xvfz prometheus-*.tar.gz
2cd prometheus-*

1./prometheus --help

 1global:
 2  scrape_interval:     15s    # 全局抓取间隔
 3  evaluation_interval: 15s    # 规则评估间隔
 4
 5rule_files:
 6  # - "first.rules"
 7  # - "second.rules"
 8
 9scrape_configs:
10  - job_name: prometheus
11    static_configs:
12      - targets: ['localhost:9090']

1./prometheus --config.file=prometheus.yml

1# 查询 Prometheus 服务器处理的 /metrics 请求总数
2promhttp_metric_handler_requests_total
3
4# 仅查询 HTTP 状态码 200 的请求
5promhttp_metric_handler_requests_total{code="200"}
6
7# 计算返回的时间序列数量
8count(promhttp_metric_handler_requests_total)

1rate(promhttp_metric_handler_requests_total{code="200"}[1m])

1<metric name>{<label name>="<label value>", ...}

1# 标准表示法
2api_http_requests_total{method="POST", handler="/messages"}
3
4# UTF-8 字符需要引号
5{"特殊指标名", label="value"}
6
7# 使用 __name__ 标签（内部表示）
8{__name__="api_http_requests_total", method="POST", handler="/messages"}

1# HTTP 请求总数
2http_requests_total{method="POST", endpoint="/api/users"}

1# 当前内存使用量（字节）
2memory_usage_bytes{instance="localhost:9090"}
3
4# 当前并发请求数
5http_concurrent_requests{endpoint="/api"}

 1# 原始直方图指标
 2http_request_duration_seconds_bucket{le="0.1"} 1000
 3http_request_duration_seconds_bucket{le="0.5"} 1500
 4http_request_duration_seconds_bucket{le="1.0"} 1800
 5http_request_duration_seconds_bucket{le="+Inf"} 2000
 6http_request_duration_seconds_sum 850
 7http_request_duration_seconds_count 2000
 8
 9# 查询 95 分位数
10histogram_quantile(0.95, rate(http_request_duration_seconds_bucket[5m]))

1# 摘要指标
2http_request_duration_seconds{quantile="0.5"} 0.05
3http_request_duration_seconds{quantile="0.9"} 0.2
4http_request_duration_seconds{quantile="0.99"} 0.5
5http_request_duration_seconds_sum 850
6http_request_duration_seconds_count 2000

1job: api-server
2  instance 1: 1.2.3.4:5670
3  instance 2: 1.2.3.4:5671
4  instance 3: 5.6.7.8:5670
5  instance 4: 5.6.7.8:5671

 1# 检查所有实例的健康状态
 2up
 3
 4# 查询所有不健康的实例
 5up == 0
 6
 7# 查询特定任务的所有实例
 8up{job="api-server"}
 9
10# 计算任务的健康实例数
11sum(up{job="api-server"})
12
13# 查询抓取时间超过 1 秒的实例
14scrape_duration_seconds > 1

1tar xvfz prometheus-*.tar.gz
2cd prometheus-*

 1global:
 2  scrape_interval: 15s  # 默认每 15 秒抓取目标一次
 3  
 4  # 与外部系统通信时附加这些标签
 5  # (federation, remote storage, Alertmanager)
 6  external_labels:
 7    monitor: 'codelab-monitor'
 8
 9# 包含一个要抓取的端点的抓取配置
10# 这里是 Prometheus 自身
11scrape_configs:
12  # job 名称作为标签 `job=<job_name>` 添加到从此配置抓取的任何时间序列
13  - job_name: 'prometheus'
14  
15    # 覆盖全局默认值，每 5 秒抓取此任务的目标
16    scrape_interval: 5s
17  
18    static_configs:
19      - targets: ['localhost:9090']

1# 启动 Prometheus
2# 默认情况下，Prometheus 将数据库存储在 ./data (标志 --storage.tsdb.path)
3./prometheus --config.file=prometheus.yml

1# 查询目标抓取间隔
2prometheus_target_interval_length_seconds
3
4# 仅查询 99 分位数延迟
5prometheus_target_interval_length_seconds{quantile="0.99"}
6
7# 计算返回的时间序列数量
8count(prometheus_target_interval_length_seconds)

1rate(prometheus_tsdb_head_chunks_created_total[1m])

1tar -xzvf node_exporter-*.*.tar.gz
2cd node_exporter-*.*
3
4# 在不同终端启动 3 个示例目标
5./node_exporter --web.listen-address 127.0.0.1:8080
6./node_exporter --web.listen-address 127.0.0.1:8081
7./node_exporter --web.listen-address 127.0.0.1:8082

 1scrape_configs:
 2  - job_name: 'node'
 3  
 4    # 每 5 秒抓取一次
 5    scrape_interval: 5s
 6  
 7    static_configs:
 8      - targets: ['localhost:8080', 'localhost:8081']
 9        labels:
10          group: 'production'
11    
12      - targets: ['localhost:8082']
13        labels:
14          group: 'canary'

1groups:
2- name: cpu-node
3  rules:
4  - record: job_instance_mode:node_cpu_seconds:avg_rate5m
5    expr: avg by (job, instance, mode) (rate(node_cpu_seconds_total[5m]))

 1global:
 2  scrape_interval: 15s
 3  evaluation_interval: 15s  # 每 15 秒评估规则
 4  
 5  external_labels:
 6    monitor: 'codelab-monitor'
 7
 8rule_files:
 9  - 'prometheus.rules.yml'
10
11scrape_configs:
12  - job_name: 'prometheus'
13    scrape_interval: 5s
14    static_configs:
15      - targets: ['localhost:9090']
16  
17  - job_name: 'node'
18    scrape_interval: 5s
19    static_configs:
20      - targets: ['localhost:8080', 'localhost:8081']
21        labels:
22          group: 'production'
23      - targets: ['localhost:8082']
24        labels:
25          group: 'canary'

1# Linux 系统
2kill -s SIGHUP <PID>

1# Linux 系统
2kill -s SIGTERM <PID>
3# 或
4kill -s SIGINT <PID>
5# 或在终端按 Control-C

1tar xvfz prometheus-*.tar.gz
2cd prometheus-*
3./prometheus --config.file=prometheus.yml

1# 通常流程
2git clone https://github.com/prometheus/prometheus.git
3cd prometheus
4make build

1# 使用示例配置启动 Prometheus
2docker run -p 9090:9090 prom/prometheus

1docker run \
2    -p 9090:9090 \
3    -v /path/to/prometheus.yml:/etc/prometheus/prometheus.yml \
4    prom/prometheus

1docker run \
2    -p 9090:9090 \
3    -v /path/to/config:/etc/prometheus \
4    prom/prometheus

1# 创建持久卷
2docker volume create prometheus-data
3
4# 使用持久存储启动 Prometheus
5docker run \
6    -p 9090:9090 \
7    -v /path/to/prometheus.yml:/etc/prometheus/prometheus.yml \
8    -v prometheus-data:/prometheus \
9    prom/prometheus

1FROM prom/prometheus
2ADD prometheus.yml /etc/prometheus/

1docker build -t my-prometheus .
2docker run -p 9090:9090 my-prometheus

 1# 1. 创建持久卷
 2docker volume create prometheus-data
 3docker volume create prometheus-config
 4
 5# 2. 创建配置文件
 6cat > /tmp/prometheus.yml <<EOF
 7global:
 8  scrape_interval: 15s
 9  evaluation_interval: 15s
10
11scrape_configs:
12  - job_name: 'prometheus'
13    static_configs:
14      - targets: ['localhost:9090']
15EOF
16
17# 3. 启动 Prometheus
18docker run -d \
19    --name prometheus \
20    -p 9090:9090 \
21    -v /tmp/prometheus.yml:/etc/prometheus/prometheus.yml \
22    -v prometheus-data:/prometheus \
23    prom/prometheus \
24    --config.file=/etc/prometheus/prometheus.yml \
25    --storage.tsdb.path=/prometheus
26
27# 4. 查看日志
28docker logs -f prometheus
29
30# 5. 访问 Web UI
31# http://localhost:9090

1./prometheus -h

1./prometheus --config.file=prometheus.yml

1# 方法 1: 发送 SIGHUP 信号
2kill -s SIGHUP <PID>
3
4# 方法 2: HTTP POST (需启用 --web.enable-lifecycle)
5curl -X POST http://localhost:9090/-/reload

 1global:
 2  # 抓取目标的默认频率
 3  scrape_interval: 15s  # 默认 1m
 4  
 5  # 抓取请求超时时间 (不能大于 scrape_interval)
 6  scrape_timeout: 10s  # 默认 10s
 7  
 8  # 规则评估频率
 9  evaluation_interval: 15s  # 默认 1m
10  
11  # 规则评估时间戳偏移(确保底层指标已收到)
12  rule_query_offset: 0s  # 默认 0s
13  
14  # 与外部系统通信时添加的标签
15  # (federation, remote storage, Alertmanager)
16  external_labels:
17    cluster: 'production'
18    region: 'us-east-1'
19  
20  # PromQL 查询日志文件
21  query_log_file: '/var/log/prometheus/query.log'
22  
23  # 抓取失败日志文件
24  scrape_failure_log_file: '/var/log/prometheus/scrape_failures.log'

1runtime:
2  # 配置 Go 垃圾回收器 GOGC 参数
3  # 降低此数字会增加 CPU 使用率
4  gogc: 75  # 默认 75

 1scrape_configs:
 2  - job_name: 'prometheus'  # 任务名称(必需,唯一)
 3  
 4    # 抓取间隔(覆盖全局设置)
 5    scrape_interval: 5s
 6  
 7    # 抓取超时
 8    scrape_timeout: 5s
 9  
10    # 指标路径
11    metrics_path: /metrics
12  
13    # 协议方案
14    scheme: http
15  
16    # honor_labels 控制标签冲突处理
17    # true: 保留抓取数据中的标签,忽略服务端标签
18    # false: 重命名冲突标签为 exported_<label>
19    honor_labels: false
20  
21    # honor_timestamps 控制是否使用目标的时间戳
22    # true: 使用目标提供的时间戳
23    # false: 忽略目标时间戳
24    honor_timestamps: true
25  
26    # 静态配置的目标列表
27    static_configs:
28      - targets: ['localhost:9090']
29        labels:
30          env: 'production'

 1scrape_configs:
 2  # 监控 Prometheus 自身
 3  - job_name: 'prometheus'
 4    scrape_interval: 5s
 5    static_configs:
 6      - targets: ['localhost:9090']
 7  
 8  # 监控 Node Exporter
 9  - job_name: 'node'
10    scrape_interval: 10s
11    static_configs:
12      - targets: 
13          - 'node1:9100'
14          - 'node2:9100'
15        labels:
16          group: 'production'
17      - targets: ['node3:9100']
18        labels:
19          group: 'testing'
20  
21  # 使用 Kubernetes 服务发现
22  - job_name: 'kubernetes-pods'
23    kubernetes_sd_configs:
24      - role: pod
25    relabel_configs:
26      - source_labels: [__meta_kubernetes_pod_label_app]
27        target_label: app

 1scrape_configs:
 2  - job_name: 'secure-app'
 3    # Basic 认证
 4    basic_auth:
 5      username: 'admin'
 6      password: 'secret123'
 7  
 8    # 或使用文件
 9    # basic_auth:
10    #   username_file: /path/to/username
11    #   password_file: /path/to/password
12  
13    # 或使用 Bearer Token
14    # authorization:
15    #   type: Bearer
16    #   credentials: 'mytoken123'
17    #   # 或从文件读取
18    #   # credentials_file: /path/to/token
19  
20    # TLS 配置
21    tls_config:
22      # CA 证书
23      ca_file: /path/to/ca.crt
24      # 客户端证书
25      cert_file: /path/to/client.crt
26      key_file: /path/to/client.key
27      # 跳过证书验证(不建议)
28      insecure_skip_verify: false
29      # 最小 TLS 版本
30      min_version: TLS12
31  
32    # 代理设置
33    proxy_url: http://proxy.example.com:8080
34  
35    # 自定义 HTTP 头
36    http_headers:
37      X-Custom-Header:
38        values: ['custom-value']
39  
40    static_configs:
41      - targets: ['app.example.com:443']
42  
43    scheme: https

 1scrape_configs:
 2  - job_name: 'kubernetes-pods'
 3    kubernetes_sd_configs:
 4      - role: pod
 5        namespaces:
 6          names:
 7            - default
 8            - production
 9  
10    relabel_configs:
11      # 仅抓取带有 prometheus.io/scrape=true 注解的 Pod
12      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
13        action: keep
14        regex: true
15    
16      # 使用自定义路径
17      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
18        action: replace
19        target_label: __metrics_path__
20        regex: (.+)
21    
22      # 使用自定义端口
23      - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
24        action: replace
25        regex: ([^:]+)(?::\d+)?;(\d+)
26        replacement: $1:$2
27        target_label: __address__

 1scrape_configs:
 2  - job_name: 'consul-services'
 3    consul_sd_configs:
 4      - server: 'localhost:8500'
 5        datacenter: 'dc1'
 6        services: ['web', 'api', 'database']
 7        tags: ['prometheus']
 8  
 9    relabel_configs:
10      - source_labels: [__meta_consul_service]
11        target_label: service

1scrape_configs:
2  - job_name: 'dns-discovery'
3    dns_sd_configs:
4      - names:
5          - 'tasks.myservice.example.com'
6        type: 'A'
7        port: 9100
8    refresh_interval: 30s

 1scrape_configs:
 2  - job_name: 'ec2-nodes'
 3    ec2_sd_configs:
 4      - region: us-east-1
 5        port: 9100
 6        filters:
 7          - name: tag:Environment
 8            values: ['production']
 9          - name: instance-state-name
10            values: ['running']
11  
12    relabel_configs:
13      - source_labels: [__meta_ec2_tag_Name]
14        target_label: instance_name

1scrape_configs:
2  - job_name: 'file-sd'
3    file_sd_configs:
4      - files:
5          - '/etc/prometheus/targets/*.json'
6          - '/etc/prometheus/targets/*.yaml'
7        refresh_interval: 30s

 1[
 2  {
 3    "targets": ["host1:9100", "host2:9100"],
 4    "labels": {
 5      "env": "production",
 6      "team": "backend"
 7    }
 8  },
 9  {
10    "targets": ["host3:9100"],
11    "labels": {
12      "env": "staging"
13    }
14  }
15]

 1scrape_configs:
 2  - job_name: 'example'
 3    static_configs:
 4      - targets: ['localhost:9090']
 5  
 6    relabel_configs:
 7      # 保留特定标签的目标
 8      - source_labels: [__meta_kubernetes_namespace]
 9        action: keep
10        regex: 'production|staging'
11    
12      # 删除特定标签的目标
13      - source_labels: [__meta_kubernetes_pod_name]
14        action: drop
15        regex: 'test-.*'
16    
17      # 替换标签值
18      - source_labels: [__address__]
19        target_label: instance
20        regex: '(.+):.*'
21        replacement: '$1'
22    
23      # 添加新标签
24      - target_label: cluster
25        replacement: 'us-central'
26    
27      # 保留标签
28      - action: labelkeep
29        regex: '__meta_kubernetes_(namespace|pod_name|pod_ip)'
30    
31      # 删除标签
32      - action: labeldrop
33        regex: '__meta_kubernetes_pod_label_.*'

 1alerting:
 2  # 告警重新标记
 3  alert_relabel_configs:
 4    - source_labels: [dc]
 5      regex: 'dc1'
 6      target_label: severity
 7      replacement: 'critical'
 8  
 9  # Alertmanager 配置
10  alertmanagers:
11    - static_configs:
12        - targets:
13            - 'alertmanager1:9093'
14            - 'alertmanager2:9093'
15    
16      # 超时设置
17      timeout: 10s
18    
19      # 路径前缀
20      path_prefix: /alertmanager

 1# 远程写入(用于长期存储)
 2remote_write:
 3  - url: 'http://remote-storage:9201/write'
 4  
 5    # 写入超时
 6    remote_timeout: 30s
 7  
 8    # 队列配置
 9    queue_config:
10      capacity: 10000
11      max_shards: 50
12      max_samples_per_send: 1000
13  
14    # 写入重新标记(过滤不需要的指标)
15    write_relabel_configs:
16      - source_labels: [__name__]
17        regex: 'go_.*'
18        action: drop
19
20# 远程读取
21remote_read:
22  - url: 'http://remote-storage:9201/read'
23    read_recent: true

 1storage:
 2  tsdb:
 3    # 数据保留时间
 4    retention_time: 15d
 5  
 6    # 最大数据块大小
 7    retention_size: 512GB
 8  
 9  exemplars:
10    # Exemplar 最大存储大小
11    max_exemplars: 100000

 1# 全局配置
 2global:
 3  scrape_interval: 15s
 4  evaluation_interval: 15s
 5  external_labels:
 6    cluster: 'production'
 7    datacenter: 'us-east-1'
 8
 9# 运行时配置
10runtime:
11  gogc: 75
12
13# 规则文件
14rule_files:
15  - '/etc/prometheus/rules/*.yml'
16
17# 抓取配置
18scrape_configs:
19  # Prometheus 自身
20  - job_name: 'prometheus'
21    static_configs:
22      - targets: ['localhost:9090']
23  
24  # Node Exporters (使用文件服务发现)
25  - job_name: 'node'
26    file_sd_configs:
27      - files:
28          - '/etc/prometheus/targets/nodes/*.json'
29  
30    relabel_configs:
31      - source_labels: [__address__]
32        target_label: instance
33  
34  # Kubernetes Pods
35  - job_name: 'kubernetes-pods'
36    kubernetes_sd_configs:
37      - role: pod
38  
39    relabel_configs:
40      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
41        action: keep
42        regex: true
43      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
44        action: replace
45        target_label: __metrics_path__
46        regex: (.+)
47
48# 告警配置
49alerting:
50  alertmanagers:
51    - static_configs:
52        - targets: ['alertmanager:9093']
53
54# 远程写入
55remote_write:
56  - url: 'http://thanos-receive:19291/api/v1/receive'
57    queue_config:
58      capacity: 10000
59      max_samples_per_send: 1000
60
61# 存储配置
62storage:
63  tsdb:
64    retention_time: 30d
65    retention_size: 1TB

1rule_files:
2  - '/etc/prometheus/rules/*.yml'
3  - '/etc/prometheus/alerts/*.yml'

1# 发送 SIGHUP 信号
2kill -s SIGHUP <PID>
3
4# 或使用 HTTP 接口
5curl -X POST http://localhost:9090/-/reload

1promtool check rules /path/to/example.rules.yml

1groups:
2  - name: <组名>              # 必需,文件内唯一
3    interval: <持续时间>       # 可选,规则评估频率
4    limit: <整数>              # 可选,限制告警/序列数量
5    query_offset: <持续时间>   # 可选,查询时间偏移
6    labels:                    # 可选,添加到所有规则的标签
7      <标签名>: <标签值>
8    rules:
9      - <规则定义>

1groups:
2  - name: example
3    interval: 30s
4    rules:
5      # 记录规则
6      - record: code:prometheus_http_requests_total:sum
7        expr: sum by (code) (prometheus_http_requests_total)

 1# 记录规则名称(必需,必须是有效的指标名称)
 2record: <string>
 3
 4# PromQL 表达式(必需)
 5# 每次评估时执行,结果保存为新的时间序列
 6expr: <string>
 7
 8# 添加或覆盖的标签(可选)
 9labels:
10  <labelname>: <labelvalue>

 1groups:
 2  # 1. HTTP 请求聚合
 3  - name: http_requests_aggregation
 4    interval: 30s
 5    rules:
 6      # 按状态码聚合请求总数
 7      - record: code:prometheus_http_requests_total:sum
 8        expr: sum by (code) (prometheus_http_requests_total)
 9    
10      # 按 job 聚合请求总数
11      - record: job:prometheus_http_requests_total:sum
12        expr: sum by (job) (prometheus_http_requests_total)
13    
14      # 计算每个 job 的请求速率 (5分钟平均)
15      - record: job:prometheus_http_requests_total:rate5m
16        expr: sum by (job) (rate(prometheus_http_requests_total[5m]))
17  
18  # 2. 资源使用率聚合
19  - name: resource_usage
20    interval: 1m
21    rules:
22      # CPU 使用率聚合
23      - record: instance:node_cpu_utilization:rate5m
24        expr: 1 - avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m]))
25    
26      # 内存使用率
27      - record: instance:node_memory_utilization:ratio
28        expr: |
29          1 - (
30            node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes
31          )
32        labels:
33          team: infrastructure
34  
35  # 3. 业务指标聚合
36  - name: business_metrics
37    interval: 5m
38    rules:
39      # 计算 API 请求成功率
40      - record: job:api_success_rate:ratio
41        expr: |
42          sum by (job) (rate(api_requests_total{status=~"2.."}[5m]))
43          /
44          sum by (job) (rate(api_requests_total[5m]))
45    
46      # 计算 99 分位延迟
47      - record: job:request_latency_seconds:p99
48        expr: histogram_quantile(0.99, sum by (job, le) (rate(request_duration_seconds_bucket[5m])))

 1groups:
 2  # 高频评估 - 关键指标
 3  - name: critical_metrics
 4    interval: 15s
 5    rules:
 6      - record: instance:up:count
 7        expr: count by (job) (up == 1)
 8  
 9  # 低频评估 - 聚合指标
10  - name: daily_aggregations
11    interval: 5m
12    rules:
13      - record: cluster:cpu_usage:avg24h
14        expr: avg_over_time(cluster:cpu_usage:ratio[24h])

1groups:
2  - name: remote_write_metrics
3    # 偏移 1 分钟,确保远程写入的数据已到达
4    query_offset: 1m
5    rules:
6      - record: job:requests_total:rate5m
7        expr: sum by (job) (rate(requests_total[5m]))

1groups:
2  - name: high_cardinality_metrics
3    # 限制最多产生 10000 个时间序列
4    limit: 10000
5    rules:
6      - record: path:http_requests_total:sum
7        expr: sum by (path) (http_requests_total)

 1# 告警名称(必需,必须是有效的标签值)
 2alert: <string>
 3
 4# PromQL 表达式(必需)
 5# 当表达式结果为真时,告警激活
 6expr: <string>
 7
 8# 持续时间阈值(可选)
 9# 告警在持续触发这段时间后才会 firing
10[ for: <duration> | default = 0s ]
11
12# 保持触发时间(可选)
13# 告警条件消失后继续触发的时间
14[ keep_firing_for: <duration> | default = 0s ]
15
16# 附加标签(可选)
17# 标签值支持模板化
18labels:
19  [ <labelname>: <tmpl_string> ]
20
21# 注解(可选)
22# 用于存储描述、Runbook 链接等信息
23# 注解值支持模板化
24annotations:
25  [ <labelname>: <tmpl_string> ]

 1groups:
 2  - name: instance_alerts
 3    rules:
 4      # 实例宕机告警
 5      - alert: InstanceDown
 6        expr: up == 0
 7        for: 5m
 8        labels:
 9          severity: critical
10          team: infrastructure
11        annotations:
12          summary: "实例 {{ $labels.instance }} 宕机"
13          description: "{{ $labels.instance }} (job: {{ $labels.job }}) 已宕机超过 5 分钟"

 1groups:
 2  - name: application_alerts
 3    labels:
 4      team: backend
 5    rules:
 6      # 1. API 高延迟告警
 7      - alert: APIHighRequestLatency
 8        expr: |
 9          histogram_quantile(0.5, 
10            rate(api_http_request_latencies_second_bucket[5m])
11          ) > 1
12        for: 10m
13        labels:
14          severity: warning
15          service: api
16        annotations:
17          summary: "{{ $labels.instance }} API 延迟过高"
18          description: "实例 {{ $labels.instance }} 的中位数请求延迟超过 1s (当前值: {{ $value }}s)"
19          dashboard: "https://grafana.example.com/d/api-dashboard"
20          runbook: "https://wiki.example.com/runbooks/api-latency"
21    
22      # 2. 错误率过高告警
23      - alert: HighErrorRate
24        expr: |
25          (
26            sum by (job) (rate(http_requests_total{status=~"5.."}[5m]))
27            /
28            sum by (job) (rate(http_requests_total[5m]))
29          ) > 0.05
30        for: 5m
31        keep_firing_for: 10m
32        labels:
33          severity: critical
34        annotations:
35          summary: "{{ $labels.job }} 错误率过高"
36          description: "{{ $labels.job }} 的 5xx 错误率超过 5% (当前: {{ $value | humanizePercentage }})"
37    
38      # 3. 磁盘空间不足告警
39      - alert: DiskSpaceLow
40        expr: |
41          (
42            node_filesystem_avail_bytes{fstype!~"tmpfs|fuse.lxcfs"}
43            /
44            node_filesystem_size_bytes{fstype!~"tmpfs|fuse.lxcfs"}
45          ) < 0.1
46        for: 15m
47        labels:
48          severity: warning
49        annotations:
50          summary: "{{ $labels.instance }} 磁盘空间不足"
51          description: |
52            实例 {{ $labels.instance }} 的挂载点 {{ $labels.mountpoint }} 
53            可用空间低于 10% (当前: {{ $value | humanizePercentage }})
54    
55      # 4. 内存使用率过高告警
56      - alert: HighMemoryUsage
57        expr: |
58          (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) > 0.9
59        for: 5m
60        labels:
61          severity: warning
62        annotations:
63          summary: "{{ $labels.instance }} 内存使用率过高"
64          description: "实例 {{ $labels.instance }} 内存使用率超过 90% (当前: {{ $value | humanizePercentage }})"

 1annotations:
 2  # 格式化百分比
 3  usage: "{{ $value | humanizePercentage }}"
 4  
 5  # 格式化数字(添加单位)
 6  memory: "{{ $value | humanize }}B"
 7  
 8  # 格式化为 1024 进制
 9  disk: "{{ $value | humanize1024 }}B"
10  
11  # 格式化时间戳
12  timestamp: "{{ $value | humanizeTimestamp }}"

 1groups:
 2  - name: template_examples
 3    rules:
 4      - alert: PodCrashLooping
 5        expr: rate(kube_pod_container_status_restarts_total[15m]) > 0
 6        for: 5m
 7        labels:
 8          severity: warning
 9          namespace: "{{ $labels.namespace }}"
10          pod: "{{ $labels.pod }}"
11        annotations:
12          summary: "Pod {{ $labels.namespace }}/{{ $labels.pod }} 频繁重启"
13          description: |
14            命名空间: {{ $labels.namespace }}
15            Pod: {{ $labels.pod }}
16            容器: {{ $labels.container }}
17            重启速率: {{ $value | humanize }} 次/秒
18            集群: {{ $externalLabels.cluster }}
19          grafana: "https://grafana/d/pods?var-namespace={{ $labels.namespace }}&var-pod={{ $labels.pod }}"

1# ❌ 不好 - 可能产生大量瞬时告警
2- alert: HighCPU
3  expr: cpu_usage > 0.8
4  # 没有 for,立即告警
5
6# ✅ 好 - 只在持续高负载时告警
7- alert: HighCPU
8  expr: cpu_usage > 0.8
9  for: 10m  # 持续 10 分钟才告警

1- alert: ServiceDown
2  expr: up{job="myservice"} == 0
3  for: 5m
4  keep_firing_for: 10m  # 服务恢复后继续告警 10 分钟
5  annotations:
6    summary: "服务可能存在不稳定问题"

 1groups:
 2  - name: disk_alerts
 3    rules:
 4      # 警告级别
 5      - alert: DiskSpaceWarning
 6        expr: disk_free_percent < 20
 7        for: 30m
 8        labels:
 9          severity: warning
10    
11      # 严重级别
12      - alert: DiskSpaceCritical
13        expr: disk_free_percent < 10
14        for: 15m
15        labels:
16          severity: critical
17    
18      # 紧急级别
19      - alert: DiskSpaceEmergency
20        expr: disk_free_percent < 5
21        for: 5m
22        labels:
23          severity: emergency

 1- alert: DatabaseConnectionPoolExhausted
 2  expr: db_connection_pool_usage > 0.95
 3  for: 5m
 4  labels:
 5    severity: critical
 6    component: database
 7  annotations:
 8    summary: "{{ $labels.instance }} 数据库连接池即将耗尽"
 9    description: |
10      数据库连接池使用率: {{ $value | humanizePercentage }}
11      实例: {{ $labels.instance }}
12      数据库: {{ $labels.database }}
13    impact: "可能导致应用无法建立新的数据库连接,影响业务"
14    action: |
15      1. 检查是否存在连接泄漏
16      2. 考虑增加连接池大小
17      3. 检查慢查询
18    runbook: "https://wiki.example.com/runbooks/db-connection-pool"
19    dashboard: "https://grafana.example.com/d/db-dashboard?var-instance={{ $labels.instance }}"