\u76ee\u6807：\u5982\u679c\u670d\u52a1\u5668GPU\u7a7a\u7f6e，\u53ef\u4ee5\u53ca\u65f6\u63a8\u9001\u6d88\u606f\u5230\u98de\u4e66\u7fa4\u3002<\/p>\n
\u5176\u4ed6\u7c7b\u4f3c\u7684\u76d1\u63a7\u76ee\u6807\u4e5f\u53ef\u4ee5\u4fee\u6539\u4ee3\u7801\u5b9e\u73b0\u3002<\/p>\n<\/p>\n
\u6b65\u9aa4：<\/p>\n
(1) \u9996\u5148\u5728\u7fa4\u804a\u8bbe\u7f6e\u52a0\u5165\u673a\u5668\u4eba，\u590d\u5236webhook_url<\/p>\n
(2) \u5728\u670d\u52a1\u5668\u540e\u53f0\u8fd0\u884c\u5982\u4e0b\u4ee3\u7801\u3002\u6ce8\u610f\u66ff\u6362webhook_url<\/p>\n
"""
\nnohup python monitor_gpu.py my_server > monitor_gpu.log 2>&1 &
\n"""
\nimport sys
\nimport subprocess
\nimport requests
\nimport time
\nfrom datetime import datetime<\/p>\n
import logging
\nlogger = logging.getLogger(__name__)
\nlogging.basicConfig(level=logging.INFO)<\/p>\n
# \u98de\u4e66\u673a\u5668\u4eba webhook \u5730\u5740
\nfeishu_webhook_url = "https:\/\/open.feishu.cn\/open-apis\/bot\/v2\/hook\/xxxxxxx"
\nif len(sys.argv) < 2:
\n logger.error(f"Usage: python {__file__} <machine_id>")
\n sys.exit(1)
\nmachine_id = sys.argv[1]<\/p>\n
def check_gpu_usage(util_threshold=10):
\n # \u6267\u884c nvidia-smi \u547d\u4ee4\u83b7\u53d6\u663e\u5361\u4f7f\u7528\u60c5\u51b5
\n result = subprocess.run(['nvidia-smi', '–query-gpu=utilization.gpu,memory.used,memory.total', '–format=csv,noheader,nounits'], stdout=subprocess.PIPE)
\n output = result.stdout.decode('utf-8').strip()<\/p>\n
# \u89e3\u6790\u8f93\u51fa
\n utils = []
\n mems = []
\n for info in output.split('\\\\n'):
\n if info:
\n gpu_utilization, memory_used, memory_total = map(int, info.split(', '))
\n utils.append(gpu_utilization)
\n mems.append(memory_used)
\n gpu_utilization_avg = sum(utils) \/ len(utils)
\n memory_used_avg = sum(mems) \/ len(mems)<\/p>\n
# \u5224\u65ad\u662f\u5426\u7a7a\u95f2（\u5047\u8bbe\u5229\u7528\u7387\u5c0f\u4e8e10%\u4e3a\u7a7a\u95f2）
\n if gpu_utilization_avg < util_threshold:
\n return True, f"\u26a0\ufe0f {machine_id} GPU\u7a7a\u95f2：\u5e73\u5747\u5229\u7528\u7387 {gpu_utilization_avg}% \u663e\u5b58 {memory_used_avg\/1000} GB"
\n return False, f"\u2705 {machine_id} GPUs are in usage."<\/p>\n
def send_feishu_message(message):
\n headers = {
\n "Content-Type": "application\/json"
\n }
\n data = {
\n "timestamp": int(datetime.now().timestamp()),
\n "msg_type": "text",
\n "content": {
\n "text": message
\n }
\n }
\n t = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
\n response = requests.post(feishu_webhook_url, json=data, headers=headers)
\n response.raise_for_status()
\n result = response.json()
\n if result.get("code") and result.get("code") != 0:
\n logger.info(f"{t} \u53d1\u9001\\\\"{message}\\\\"\u5931\u8d25：{result['msg']}")
\n else:
\n logger.info(f"{t} \u53d1\u9001\\\\"{message}\\\\"\u6210\u529f")<\/p>\n
send_feishu_message(f"🎉 {machine_id} GPU\u7a7a\u95f2\u72b6\u6001\u76d1\u63a7\u5df2\u5f00\u542f! \u521d\u59cb\u5316\u72b6\u6001\u4e3a：GPU\u6b63\u5728\u88ab\u4f7f\u7528\u4e2d")
\nstate_prev = False
\nmax_retry = 3
\nwhile True:
\n # \u5355\u6b21\u6700\u591a\u68c0\u67e5k\u6b21，\u5168\u7a7a\u624d\u89c6\u4e3a\u7a7a
\n is_idle, message = None, None
\n for _ in range(max_retry):
\n is_idle, message = check_gpu_usage()
\n if not is_idle:
\n break
\n time.sleep(60)<\/p>\n
if is_idle ^ state_prev:
\n send_feishu_message(message)
\n state_prev = is_idle
\n time.sleep(10 * 60) # \u6bcf10\u5206\u949f\u68c0\u67e5\u4e00\u6b21<\/p>\n","protected":false},"excerpt":{"rendered":"
\u6587\u7ae0\u6d4f\u89c8\u9605\u8bfb1k\u6b21\u3002(2) \u5728\u670d\u52a1\u5668\u540e\u53f0\u8fd0\u884c\u5982\u4e0b\u4ee3\u7801\u3002\u6ce8\u610f\u66ff\u6362webhook_url\u3002(1) \u9996\u5148\u5728\u7fa4\u804a\u8bbe\u7f6e\u52a0\u5165\u673a\u5668\u4eba\uff0c\u590d\u5236webhook_url\u3002\u76ee\u6807\uff1a\u5982\u679c\u670d\u52a1\u5668GPU\u7a7a\u7f6e\uff0c\u53ef\u4ee5\u53ca\u65f6\u63a8\u9001\u6d88\u606f\u5230\u98de\u4e66\u7fa4\u3002\u5176\u4ed6\u7c7b\u4f3c\u7684\u76d1\u63a7\u76ee\u6807\u4e5f\u53ef\u4ee5\u4fee\u6539\u4ee3\u7801\u5b9e\u73b0\u3002_\u98de\u4e66\u4ecb\u5165gpu api<\/p>\n","protected":false},"author":2,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[1],"tags":[43,1388],"topic":[],"class_list":["post-18393","post","type-post","status-publish","format-standard","hentry","category-server","tag-43","tag-1388"],"yoast_head":"\n