配置钉钉机器人
获取token
https://oapi.dingtalk.com/robot/send?access_token=a85f438fa00c451c0b32ed0fc799609093436f07e93e518bde59dd6d26f930e9
配置钉钉告警插件
vim dingtalk-webhook.yaml
---
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
app: prometheus-webhook-dingtalk
name: prometheus-webhook-dingtalk
namespace: monitoring
spec:
replicas: 1
selector:
matchLabels:
app: prometheus-webhook-dingtalk
template:
metadata:
labels:
app: prometheus-webhook-dingtalk
spec:
containers:
- name: prometheus-webhook-dingtalk
image: timonwong/prometheus-webhook-dingtalk:v0.3.0
imagePullPolicy: IfNotPresent
args:
- --ding.profile=webhook1=获取的钉钉token
- --template.file=/usr/share/prometheus-webhook-dingtalk/template/webhook-dingtalk.tmpl
volumeMounts:
- mountPath: /usr/share/prometheus-webhook-dingtalk/template/
name: webhook-dingtalk-template
ports:
- containerPort: 8060
protocol: TCP
resources:
requests:
cpu: 100m
memory: 100Mi
limits:
cpu: 200m
memory: 1000Mi
volumes:
- name: webhook-dingtalk-template
configMap:
name: webhook-dingtalk-template
defaultMode: 420
---
apiVersion: v1
kind: Service
metadata:
labels:
app: prometheus-webhook-dingtalk
name: prometheus-webhook-dingtalk
namespace: monitoring
spec:
ports:
- port: 8060
protocol: TCP
targetPort: 8060
selector:
app: prometheus-webhook-dingtalk
sessionAffinity: None
配置报警模板
mkdir -p /usr/share/prometheus-webhook-dingtalk/template/
vim /usr/share/prometheus-webhook-dingtalk/template/webhook-dingtalk.tmpl
{{ define "__subject" }}[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ .GroupLabels.SortedPairs.Values | join " " }} {{ if gt (len .CommonLabels) (len .GroupLabels) }}({{ with .CommonLabels.Remove .GroupLabels.Names }}{{ .Values | join " " }}{{ end }}){{ end }}{{ end }}
{{ define "__alertmanagerURL" }}{{ .ExternalURL }}/#/alerts?receiver={{ .Receiver }}{{ end }}
{{ define "__text_alert_list" }}{{ range . }}
**报警内容**
{{ range .Labels.SortedPairs }}> - {{ .Name }}: {{ .Value | markdown | html }}
{{ end }}
**报警信息**
{{ range .Annotations.SortedPairs }}> - {{ .Name }}: {{ .Value | markdown | html }}
{{ end }}
{{ end }}{{ end }}
{{ define "ding.link.title" }}{{ template "__subject" . }}{{ end }}
{{ define "ding.link.content" }}
### k8s集群测试环境监控报警
---
{{ index .GroupLabels "alertname" }}
{{ template "__text_alert_list" .Alerts.Firing }}
{{ end }}
{{ if gt (len .Alerts.Resolved) 0 -}}
**恢复内容**
{{ template "__text_resolve_list" .Alerts.Resolved }}
{{ end }}
创建configmap 报警模板生成
kubectl create configmap webhook-dingtalk-template --from-file=webhook-dingtalk.tmpl -n monitoring
应用dingtalk-webhook.yaml
kubectl apply -f dingtalk-webhook.yaml
添加告警接受器
vim kube-prometheus-0.9.0/manifests/alertmanager.yaml
global:
resolve_timeout: 5m
route:
group_by: ['job']
group_wait: 0s
group_interval: 5m
repeat_interval: 2h
receiver: webhook
receivers:
- name: 'webhook'
webhook_configs:
- url: 'http://prometheus-webhook-dingtalk:8060/dingtalk/webhook1/send'
send_resolved: true
注:上述配置url: 'http://webhook-dingtalk.monitoring.svc.cluster.local:8060/dingtalk/webhook1/send' 是dingtalk-webhook.yaml文件中svc的地址。
替换原有secret
cd kube-prometheus-0.9.0/manifests/
kubectl delete secret alertmanager-main -n monitoring
kubectl create secret generic alertmanager-main --from-file=alertmanager.yaml -n monitoring
测试
#! /bin/bash
alert_payload='[
{
"labels": {
"alertname": "报警通道测试",
"dev": "sda1",
"instance": "127.0.0.1",
"severity": "critical"
},
"annotations": {
"info": "报警通道测试",
"summary": "报警通道测试"
}
}
]'
alertmanager_0_ip=`kubectl describe po alertmanager-main-0 -n monitoring | grep "IP: " | awk 'END{print $2}'`
curl -XPOST -H "Content-Type: application/json" -d"${alert_payload}" http://${alertmanager_0_ip}:9093/api/v1/alerts
Comments | NOTHING