This commit was merged in pull request #56.
This commit is contained in:
2026-06-13 01:34:39 +00:00
3 changed files with 122 additions and 24 deletions
+114 -21
View File
@@ -1,4 +1,4 @@
"""Mail 失败通知 — 以 system 身份通知发件人""" """Mail 失败通知 v2.0 — 以 system 身份通知发件人AI Native"""
from __future__ import annotations from __future__ import annotations
@@ -6,7 +6,7 @@ import json
import logging import logging
from datetime import datetime from datetime import datetime
from pathlib import Path from pathlib import Path
from typing import Optional from typing import Dict, Optional
from src.blackboard.models import Task from src.blackboard.models import Task
from src.blackboard.operations import Blackboard from src.blackboard.operations import Blackboard
@@ -15,21 +15,118 @@ from src.config.agents import AGENT_IDS
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
# 邮件通知正文模板(统一模板,包含所有可能的失败原因和建议) # ── Reason 人话翻译 + detail 提取 ──────────────────────────────
_NOTIFY_TEMPLATE = """你的邮件投递失败了。
📧 原始邮件:「{title} def _extract_stderr(detail: dict, max_len: int = 200) -> str:
👤 收件人:{to_agent} """从 detail 中提取 stderr_preview"""
❌ 失败原因:{reason} preview = (detail or {}).get("stderr_preview", "")
if preview and len(preview) > max_len:
preview = preview[:max_len] + "..."
return preview
常见失败原因及处理建议:
• no_reply_found:收件人未回复。建议重发邮件,或通过黑板任务方式联系
• auth_failed:收件人认证失败。需检查 Agent 配置,联系姜维(jiangwei-infra)排查
• crash_limit:收件人处理时多次崩溃。系统异常,建议稍后重试
• task_timeout:处理超时。建议重发或通过其他方式联系
• 其他原因:建议联系副军师(pangtong-fujunshi)排查
——系统自动通知""" def _fmt_retry_info(reason: str, detail: dict) -> str:
"""格式化重试情况描述"""
_NO_RETRY_REASONS = {
"no_reply_found", "auth_failed", "agent_error",
"agent_failed", "compact_failed",
}
if reason in _NO_RETRY_REASONS:
reason_human = _REASON_MAP.get(reason, _REASON_MAP.get("_default", ("未知原因", lambda d: "")))[0]
return f"无法重试({reason_human}"
count = (detail or {}).get("count", 0)
fallback_count = (detail or {}).get("fallback_count", 0)
if count > 0:
return f"已自动重试 {count}"
if fallback_count > 0:
return f"已自动重试 {fallback_count} 次(fallback"
return "系统已尝试恢复,但仍失败"
# reason_raw → (reason_human_readable, detail_format_fn)
_REASON_MAP: Dict[str, tuple] = {
"no_reply_found": ("收件人未回复(Agent 未能识别或处理此邮件)", lambda d: ""),
"crashed": ("处理时进程崩溃", lambda d: f"stderr: {_extract_stderr(d)}" if _extract_stderr(d) else "无 stderr 输出"),
"max_crash_count": ("连续崩溃达上限", lambda d: f"崩溃 {d.get('count', '?')}"),
"max_retries": ("续杯耗尽(已自动重试)", lambda d: f"重试 {d.get('count', '?')}"),
"max_api_retry_count": ("API 连续失败达上限", lambda d: f"API 重试 {d.get('count', '?')}"),
"max_monitor_timeouts": (
"处理超时达上限",
lambda d: f"超时 {d.get('count', '?')} 次,"
f"共约 {d.get('elapsed_seconds', 0) // 60} 分钟"),
"gateway_timeout": ("Agent 执行超时(已续杯重试)", lambda d: ""),
"session_stuck": ("会话假死(lock PID 死亡)", lambda d: f"假死 {d.get('stuck_count', '?')}"),
"revive_failed": ("会话恢复失败", lambda d: f"假死 {d.get('stuck_count', '?')}"),
"auth_failed": ("Agent 认证失败(配置问题)", lambda d: f"stderr: {_extract_stderr(d)}" if _extract_stderr(d) else ""),
"fallback_exhausted": (
"主模型和备用模型均失败",
lambda d: f"fallback {d.get('fallback_count', '?')} 次,"
f"原因: {d.get('fallback_reason', '未知')}"),
"agent_failed": ("收件人主动标记失败", lambda d: ""),
"compact_failed": ("上下文压缩失败", lambda d: f"stderr: {_extract_stderr(d)}" if _extract_stderr(d) else ""),
"compact_hanging": ("上下文压缩长时间未完成", lambda d: ""),
"compact_interrupted": ("上下文压缩被中断(已自动重试)", lambda d: ""),
"gateway_unreachable": (
"Gateway 不可达(已自动重试)",
lambda d: f"stderr: {_extract_stderr(d)}"
if _extract_stderr(d) else ""),
"lock_conflict": ("会话锁冲突(已自动重试)", lambda d: ""),
"max_retry_count": ("重试耗尽", lambda d: f"重试 {d.get('count', '?')}"),
"max_lock_retry_count": ("锁冲突重试耗尽", lambda d: f"重试 {d.get('count', '?')}"),
"max_connect_retry_count": ("连接重试耗尽", lambda d: f"重试 {d.get('count', '?')}"),
"_default": ("未知原因", lambda d: f"stderr: {_extract_stderr(d)}" if _extract_stderr(d) else ""),
}
# 常见失败原因参考(AI Native:提供知识库让收件 AI 自行判断)
_REASON_REFERENCE = """常见失败原因参考:
• no_reply_found:收件人未回复(Agent 未能识别或处理此邮件)
• crashed / max_crash_count:收件人处理时进程崩溃(已自动重试 3 次)
• max_retries:续杯耗尽(已自动重试 3 次,共约 34 分钟)
• max_api_retry_countAPI 连续失败达上限(rate_limit/500/503
• max_monitor_timeouts:处理超时达上限(共约 31.5 分钟)
• gateway_timeoutAgent 执行超时(已续杯重试)
• session_stuckAgent 会话假死(lock PID 死亡,revive 失败)
• revive_failed:会话假死后恢复失败
• auth_failedAgent 认证失败(配置问题)
• fallback_exhausted:主模型和备用模型均失败
• agent_failed:收件人主动标记失败
• compact_failed:上下文压缩失败
• compact_hanging:上下文压缩长时间未完成(等待超 31.5 分钟)
• compact_interrupted:上下文压缩被中断(已自动重试 3 次)
• gateway_unreachableGateway 不可达(已自动重试 3 次)
• lock_conflict:会话锁冲突(已自动重试 3 次)
• 其他:建议排查系统日志"""
def _build_notify_text(title: str, to_agent: str, reason: str,
detail: Optional[dict] = None) -> str:
"""构建通知正文(v2.0 AI Native"""
reason_human, detail_fn = _REASON_MAP.get(reason, _REASON_MAP["_default"])
detail_info = detail_fn(detail or {})
retry_info = _fmt_retry_info(reason, detail or {})
lines = [
"邮件投递失败通知",
"",
f"📧 原始邮件:「{title}",
f"👤 收件人:{to_agent}",
f"❌ 失败原因:{reason_human}{reason}",
f"📊 重试情况:{retry_info}",
]
if detail_info:
lines.append("📋 上下文信息:")
lines.append(f" {detail_info}")
lines.append("")
lines.append(_REASON_REFERENCE)
lines.append("")
lines.append("——系统自动通知")
return "\n".join(lines)
def _is_mail_project(db_path: Path) -> bool: def _is_mail_project(db_path: Path) -> bool:
@@ -43,7 +140,7 @@ def notify_mail_failed(db_path: Path, original_mail_id: str,
"""Mail 失败后以 system 身份给发件人发通知邮件 """Mail 失败后以 system 身份给发件人发通知邮件
直接通过 Blackboard 创建 Task,不走 HTTP API。 直接通过 Blackboard 创建 Task,不走 HTTP API。
防递归:检查原邮件 must_hives.system_notify,为 true 则跳过。 防递归:检查原邮件 must_haves.system_notify,为 true 则跳过。
发件人不是有效 Agent(如 system)→ 通知庞统代处理,避免广播风暴。 发件人不是有效 Agent(如 system)→ 通知庞统代处理,避免广播风暴。
""" """
try: try:
@@ -83,12 +180,8 @@ def notify_mail_failed(db_path: Path, original_mail_id: str,
original_mail_id, from_agent) original_mail_id, from_agent)
target_agent = "pangtong-fujunshi" target_agent = "pangtong-fujunshi"
# 构造通知正文 # 构造通知正文v2.0 AI Native
text = _NOTIFY_TEMPLATE.format( text = _build_notify_text(title, to_agent, reason, detail)
title=title,
to_agent=to_agent,
reason=reason,
)
# 创建通知邮件 Task # 创建通知邮件 Task
notify_id = f"mail-{int(datetime.now().timestamp() * 1000)}" notify_id = f"mail-{int(datetime.now().timestamp() * 1000)}"
+4 -1
View File
@@ -845,6 +845,8 @@ curl -X POST http://{api_host}:{api_port}/api/projects/{project_id}/tasks/{task_
cls.get("retry_field", "retry_count") cls.get("retry_field", "retry_count")
) )
elif outcome == "api_error": elif outcome == "api_error":
# A9: [DEPRECATED] api_error 已改为 should_retry=True 走续杯路径。
# 此分支理论上不再命中,保留作为安全兜底。
# A9: 429/API 错误 → release counter(on_complete)+ 推回 pending + 冷却 # A9: 429/API 错误 → release counter(on_complete)+ 推回 pending + 冷却
# 有上限:api_retry_count 累计达 max_retries 则标 failed # 有上限:api_retry_count 累计达 max_retries 则标 failed
await self._do_on_complete_async(on_complete, agent_id, outcome) await self._do_on_complete_async(on_complete, agent_id, outcome)
@@ -1842,7 +1844,8 @@ curl -X POST http://{api_host}:{api_port}/api/projects/{project_id}/tasks/{task_
"retry_field": "retry_count", "cooldown_seconds": 60} "retry_field": "retry_count", "cooldown_seconds": 60}
if any(kw in stderr_lower for kw in [ if any(kw in stderr_lower for kw in [
"rate_limit", "500", "503", "api error"]): "rate_limit", "500", "503", "api error"]):
return {"outcome": "api_error", "should_retry": False} return {"outcome": "api_error", "should_retry": True,
"retry_field": "retry_count", "cooldown_seconds": 60}
if any(kw in stderr_lower for kw in [ if any(kw in stderr_lower for kw in [
"compaction-diag", "context-overflow"]): "compaction-diag", "context-overflow"]):
return {"outcome": "compact_failed", "should_retry": False} return {"outcome": "compact_failed", "should_retry": False}
+4 -2
View File
@@ -165,14 +165,16 @@ class TestClassifyErrorApi:
1, {"status": "error"}, "rate_limit exceeded", None 1, {"status": "error"}, "rate_limit exceeded", None
) )
assert result["outcome"] == "api_error" assert result["outcome"] == "api_error"
assert result["should_retry"] is False assert result["should_retry"] is True
assert result["cooldown_seconds"] == 60
def test_stderr_500(self): def test_stderr_500(self):
result = Spawner._classify_outcome( result = Spawner._classify_outcome(
1, {"status": "error"}, "HTTP 500 Internal Server Error", None 1, {"status": "error"}, "HTTP 500 Internal Server Error", None
) )
assert result["outcome"] == "api_error" assert result["outcome"] == "api_error"
assert result["should_retry"] is False assert result["should_retry"] is True
assert result["cooldown_seconds"] == 60
class TestClassifyErrorCompact: class TestClassifyErrorCompact: