使用 Alertmanager 管理 zabbix 告警消息

[
  {
    "labels": {
      "alertname": "<requiredAlertName>",
      "<labelname>": "<labelvalue>",
      ...
    },
    "annotations": {
      "<labelname>": "<labelvalue>",
    },
    "startsAt": "<rfc3339>",
    "endsAt": "<rfc3339>",
    "generatorURL": "<generator_url>"
  },
  ...
]

{
    "labels": {
       "alertname": "{ITEM.NAME}",
       "instance": "{HOST.NAME}({HOST.IP})",
       "severity": "{TRIGGER.SEVERITY}"
     },
     "annotations": {
        "info": "{TRIGGER.NAME} - {ITEM.NAME}:{ITEM.VALUE}"
      },
     "failure_time": "{EVENT.DATE} {EVENT.TIME}",
     "status": "{TRIGGER.STATUS}"
}

{
    "labels": {
       "alertname": "{ITEM.NAME}",
       "instance": "{HOST.NAME}({HOST.IP})",
       "severity": "{TRIGGER.SEVERITY}"
     },
     "annotations": {
        "info": "{TRIGGER.NAME} - {ITEM.NAME}:{ITEM.VALUE}"
      },
     "failure_time": "{EVENT.DATE} {EVENT.TIME}",
     "recovery_time": "{EVENT.RECOVERY.DATE} {EVENT.RECOVERY.TIME}",
     "status": "{TRIGGER.STATUS}"
}

#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import sys
import json

from http.client import HTTPConnection
from urllib.parse import urlsplit
from datetime import datetime, timedelta

url = "http://127.0.0.1:9093"
api_url = f"{url}/api/v2/alerts"
headers = {"Content-Type": "application/json"}


def to_rfc_3339(dt, flag=False):
    """以 rfc-3339 标准格式化时间
    AlertManager 接口只接受 rfc-3339 标准的时间
    """
    _d, _t = dt.split()
    y, m, d = (int(i) for i in _d.split('.'))
    H, M, S = (int(i) for i in _t.split(':'))
    t = datetime(y, m, d, H, M, S)
    if flag:
        new = t + timedelta(days=2)
    else:
        new = t
    return new.isoformat() + "+08:00"


def post(url, data, headers):
    u = urlsplit(url)
    if not isinstance(data, str):
        data = json.dumps(data)
    conn = HTTPConnection(u.netloc)
    conn.request('POST', u.path, data, headers)
    response = conn.getresponse()
    text = response.read().decode()
    print(text)


def main(msg):
    try:
        msg = json.loads(msg)
    except Exception as e:
        with open('/tmp/alert.txt', 'a') as f:
            f.write(f"{datetime.now().isoformat(' ')} {str(e)}\n")

    alert_msg = {}
    if msg['status'] == "OK":
        # alert_msg['startsAt'] = to_rfc_3339(msg['failure_time'])
        alert_msg['endsAt'] = to_rfc_3339(msg['recovery_time'])
    else:
        alert_msg['startsAt'] = to_rfc_3339(msg['failure_time'])
        alert_msg['endsAt'] = to_rfc_3339(msg['failure_time'], True)

    alert_msg['labels'] = msg['labels']
    alert_msg['annotations'] = msg['annotations']

    # with open('/tmp/alert.txt','a') as f:
    #     f.write(json.dumps(alert_msg, indent=4))

    post(api_url, [alert_msg], headers)


if __name__ == "__main__":
    main(sys.argv[1])

cd /usr/local/src
wget https://github.com/prometheus/alertmanager/releases/download/v0.22.2/alertmanager-0.22.2.linux-amd64.tar.gz
tar xf alertmanager-0.22.2.linux-amd64.tar.gz -C ../
ln -s /usr/local/alertmanager-0.22.2.linux-amd64 /usr/local/alertmanager
cat > /usr/lib/systemd/system/alertmanager.service <<EOF
[Unit]
Description=Prometheus Alertmanager
Documentation=https://prometheus.io
After=network.target

[Service]
Type=simple
WorkingDirectory=/usr/local/alertmanager
ExecStart=/usr/local/alertmanager/alertmanager --config.file=alertmanager.yml
Restart=on-failure

[Install]
WantedBy=multi-user.target
EOF

systemctl enable --now alertmanager

global:
  resolve_timeout: 5m
  wechat_api_url: https://qyapi.weixin.qq.com/cgi-bin/
  wechat_api_secret:  <企业微信应用密钥>
  wechat_api_corp_id: <企业微信 ID>

# 告警消息模板
templates:
- /usr/local/alertmanager/templates/*.tmpl

# 告警路由规则
route:
  group_by: ['alertname']
  group_wait: 30s
  group_interval: 5m
  repeat_interval: 1h
  receiver: 'wechat-default'  # 默认告警消息接收器，通过 receivers 定义
  routes:  # 定义路由规则，没有时发送至默认接收器（这里为 wechat-default）
  - receiver: wechat-info  # 定义消息接受者
    match:
      severity: info  # 消息匹配规则, 如果告警级别为 info 消息就会发送到 wechat-ops 

# 接受器配置
# 企业微信相关配置参考: https://prometheus.io/docs/alerting/latest/configuration/#wechat_config
receivers:
- name: wechat-default
  wechat_configs:   # 企业微信配置
  - send_resolved: true  # 是否接收恢复消息，默认为 false
    message_type: text   # 消息类型
    agent_id: 1000002    # 企业微信应用 ID
    to_user: LiWanggui   # 消息接收人
- name: wechat-info
  wechat_configs:
  - send_resolved: true
    message_type: text
    agent_id: 1000002
    to_user: WangXiaoer

# 消息抑制配置项
inhibit_rules:
  - source_match:
      severity: 'critical'
    target_match:
      severity: 'warning'
    equal: ['alertname', 'dev', 'instance']

{{ define "wechat.default.message" }}
{{- if gt (len .Alerts.Firing) 0 -}}
    {{/* 从 .Alerts.Firing 中取数据可以防止发送恢复通知时数据重复 */}}
    {{- range $index, $alert := .Alerts.Firing -}}
        {{- if eq $index 0 -}}
**********告警通知**********
告警类型: {{ $alert.Labels.alertname }}
告警级别: {{ $alert.Labels.severity }}
        {{- end }}
=====================
告警详情: {{ $alert.Annotations.info }}
故障时间: {{ $alert.StartsAt.Local.Format "2006-01-02 15:04:05" }}
{{ if gt (len $alert.Labels.instance) 0 -}}故障实例: {{ $alert.Labels.instance }}{{- end -}}
    {{- end }}
{{- end }}

{{- if gt (len .Alerts.Resolved) 0 -}}
{{- range $index, $alert := .Alerts.Resolved -}}
{{- if eq $index 0 -}}

**********恢复通知**********
告警类型: {{ $alert.Labels.alertname }}
告警级别: {{ $alert.Labels.severity }}
{{- end }}
=====================
告警详情: {{ $alert.Annotations.info }}
故障时间: {{ $alert.StartsAt.Local.Format "2006-01-02 15:04:05" }}
恢复时间: {{ $alert.EndsAt.Local.Format "2006-01-02 15:04:05" }}
{{ if gt (len $alert.Labels.instance) 0 -}}故障实例: {{ $alert.Labels.instance }}{{- end -}}
{{- end }}
{{- end }}
{{- end }}