from kubernetes import client, config, watch from elasticsearch import Elasticsearch import arrow import sys import requests import json dingding_webhook = "https://oapi.dingtalk.com/robot/send?access_token=xxxxx" hosts = [ '172.16.21.39:9200', '172.16.21.40:9200', '172.16.21.41:9200' ] def send_text(content): data = { "msgtype": "text", "text": { "content": content } } requests.post(url=dingding_webhook, json=data) now = arrow.now('Asia/Shanghai') index = "kube-events" index_today = "{}.{}".format(index, now.format("YYYY.MM.DD")) cluster_name = sys.argv[1] kube_config = sys.argv[2] with Elasticsearch(hosts=hosts, timeout=120) as es: if not es.indices.exists(index=index_today): es.indices.create(index=index_today) # Configs can be set in Configuration class directly or using helper utility config.load_kube_config(config_file=kube_config) v1 = client.CoreV1Api() w = watch.Watch() try: for event in w.stream(v1.list_event_for_all_namespaces): doc = { "cluster_name": cluster_name, "time_iso8601": arrow.Arrow.fromdatetime(event['object'].metadata.creation_timestamp).isoformat(), "namespace": event['object'].metadata.namespace, "type": event['object'].type, "reason": event['object'].reason, "message": event['object'].message } # print(json.dumps(doc)) # if doc["type"] == "Warning": # send_text('[ {} ]- {}'.format(cluster_name, doc["message"])) if es.exists(index=index_today, doc_type=index, id=event['object'].metadata.uid): continue es.create(index=index_today, doc_type=index, body=doc, id=event['object'].metadata.uid) except Exception as e: pass w.stop()
使用 supervisord 将程序后台运行。
[program:ali-14] command = /usr/local/bin/python3.6 /ops/scripts/kube_events.py ali-14 /ops/k8s_config/ali-14 autorestart = true redirect_stderr = true stdout_logfile = /ops/logs/ali-14.log stopasgroup = true [program:m7] command = /usr/local/bin/python3.9 /ops/scripts/kube_events.py m7 /ops/k8s_config/m7 autorestart = true redirect_stderr = true stdout_logfile = /ops/logs/m7.log stopasgroup = true [program:cm] command = /usr/local/bin/python3.9 /ops/scripts/kube_events.py cm /ops/k8s_config/cm autorestart = true redirect_stderr = true stdout_logfile = /ops/logs/cm.log stopasgroup = true [program:meteo] command = /usr/local/bin/python3.6 /ops/scripts/kube_events.py meteo /ops/k8s_config/meteo autorestart = true redirect_stderr = true stdout_logfile = /ops/logs/meteo.log stopasgroup = true
写入 ES 之后就可以通过 kibana 查看了。