cfdaily
126 lines
3.9 KiB
Bash
Executable File
126 lines
3.9 KiB
Bash
Executable File
#!/bin/bash
|
||
# gateway-watchdog.sh — 检测 OpenClaw Gateway 429 并自动重启
|
||
#
|
||
# 机制：
|
||
# 1. 检查最近 CHECK_WINDOW 秒内的 session jsonl 日志
|
||
# 2. 统计 429 错误次数（errorCode=1305 或 errorMessage 含 "429"）
|
||
# 3. 连续 THRESHOLD 次检测都发现新 429 → 重启 Gateway
|
||
#
|
||
# 部署：cron 每分钟执行一次
|
||
# * * * * * /Users/chufeng/.openclaw/sanguo_projects/sanguo_moziplus_v2/scripts/gateway-watchdog.sh >> /tmp/gateway-watchdog.log 2>&1
|
||
|
||
set -euo pipefail
|
||
|
||
# crontab 环境下 PATH 不完整，补上
|
||
export PATH="/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin:$PATH"
|
||
|
||
# === 配置 ===
|
||
CHECK_WINDOW=120 # 检查最近多少秒的日志
|
||
THRESHOLD=3 # 连续检测到多少次 429 才重启
|
||
STATE_FILE="/tmp/gateway-watchdog-429-count"
|
||
LOG_DIR="/Users/chufeng/.openclaw/agents"
|
||
RESTART_CMD="openclaw gateway restart"
|
||
HEALTH_CMD="openclaw gateway health"
|
||
|
||
# === 函数 ===
|
||
|
||
log() {
|
||
echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*"
|
||
}
|
||
|
||
# 统计最近的 429 错误数
|
||
count_recent_429() {
|
||
local cutoff=$(date -u -v-${CHECK_WINDOW}S '+%Y-%m-%dT%H:%M:%S' 2>/dev/null || date -u -d "${CHECK_WINDOW} seconds ago" '+%Y-%m-%dT%H:%M:%S')
|
||
local count=0
|
||
|
||
# 只看 session jsonl（排除 trajectory）
|
||
for jsonl in "$LOG_DIR"/*/sessions/*.jsonl; do
|
||
[[ "$jsonl" == *"trajectory"* ]] && continue
|
||
[ -f "$jsonl" ] || continue
|
||
# 只看最近修改的文件（性能优化）
|
||
local mtime
|
||
mtime=$(stat -f '%m' "$jsonl" 2>/dev/null | tr -d ' \n' || stat -c '%Y' "$jsonl" 2>/dev/null | tr -d ' \n')
|
||
[ -z "$mtime" ] && continue
|
||
local now
|
||
now=$(date +%s)
|
||
local age=$(( now - mtime ))
|
||
# 文件超过 CHECK_WINDOW 秒没修改就跳过
|
||
[ "$age" -gt "$CHECK_WINDOW" ] && continue
|
||
# 文件小于 100 字节就跳过（空/刚创建）
|
||
local fsize
|
||
fsize=$(wc -c < "$jsonl" | tr -d ' ')
|
||
[ "$fsize" -lt 100 ] && continue
|
||
|
||
# 搜索 429 错误：errorCode=1305 或 errorMessage 含 "429"
|
||
local found
|
||
found=$(grep -c '"errorCode".*"1305"\|"errorMessage".*"429"' "$jsonl" 2>/dev/null || echo 0)
|
||
# 进一步过滤：只统计时间窗口内的
|
||
if [ "$found" -gt 0 ]; then
|
||
local recent
|
||
recent=$(python3 -c "
|
||
import json, sys
|
||
cutoff = '${cutoff}'
|
||
count = 0
|
||
with open('${jsonl}') as f:
|
||
for line in f:
|
||
try:
|
||
d = json.loads(line)
|
||
ts = d.get('timestamp', '')[:19]
|
||
if ts < cutoff:
|
||
continue
|
||
msg = d.get('message', {})
|
||
if msg.get('stopReason') == 'error':
|
||
err = str(msg.get('errorMessage', ''))
|
||
code = str(msg.get('errorCode', ''))
|
||
if '429' in err or '1305' in code:
|
||
count += 1
|
||
except:
|
||
pass
|
||
print(count)
|
||
" 2>/dev/null || echo 0)
|
||
count=$(( count + recent ))
|
||
fi
|
||
done
|
||
echo "$count"
|
||
}
|
||
|
||
# === 主逻辑 ===
|
||
|
||
# 1. 先检查 Gateway 是否在运行
|
||
if ! $HEALTH_CMD &>/dev/null; then
|
||
log "WARN: Gateway health check failed, attempting restart"
|
||
$RESTART_CMD 2>&1 || true
|
||
echo 0 > "$STATE_FILE"
|
||
exit 0
|
||
fi
|
||
|
||
# 2. 统计最近 429 错误
|
||
four29_count=$(count_recent_429)
|
||
|
||
# 3. 读取连续计数
|
||
consecutive=0
|
||
if [ -f "$STATE_FILE" ]; then
|
||
consecutive=$(cat "$STATE_FILE")
|
||
fi
|
||
|
||
# 4. 判断
|
||
if [ "$four29_count" -gt 0 ]; then
|
||
consecutive=$(( consecutive + 1 ))
|
||
log "429 detected: ${four29_count} recent errors, consecutive=${consecutive}/${THRESHOLD}"
|
||
else
|
||
if [ "$consecutive" -gt 0 ]; then
|
||
log "429 cleared (was ${consecutive} consecutive, now 0)"
|
||
fi
|
||
consecutive=0
|
||
fi
|
||
|
||
echo "$consecutive" > "$STATE_FILE"
|
||
|
||
# 5. 达到阈值 → 重启
|
||
if [ "$consecutive" -ge "$THRESHOLD" ]; then
|
||
log "ALERT: ${consecutive} consecutive 429 detections, restarting Gateway"
|
||
$RESTART_CMD 2>&1
|
||
log "Gateway restart completed"
|
||
echo 0 > "$STATE_FILE"
|
||
fi
|