124 lines
4.2 KiB
Python
124 lines
4.2 KiB
Python
"""SupervisorAgent — Health monitoring and auto-recovery for VoIdeaAI.
|
|
|
|
Monitors all 11 dev/ops agents + ConductorAgent:
|
|
- Periodic health checks
|
|
- Auto-restart of failed agents
|
|
- Self-learning: 3+ identical failures → notify EvolutionAgent
|
|
"""
|
|
|
|
from datetime import datetime, timezone
|
|
from typing import Any
|
|
|
|
from sqlalchemy import select
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
|
|
from app.agents.base import AgentResult, AgentStatus, AgentTrigger, BaseAgent
|
|
from app.models.agent import AgentConfig
|
|
from app.models.log import LogEntry
|
|
|
|
|
|
class SupervisorAgent(BaseAgent):
|
|
"""Supervises all agents health and auto-recovery."""
|
|
|
|
name = "supervisor_agent"
|
|
version = "1.0.0"
|
|
description = (
|
|
"Мониторинг здоровья и авто-восстановление всех агентов. "
|
|
"Проверяет работоспособность 12 агентов каждые 5 минут, "
|
|
"автоматически перезапускает упавшие и уведомляет EvolutionAgent."
|
|
)
|
|
triggers = [
|
|
AgentTrigger.MANUAL,
|
|
AgentTrigger.CRON,
|
|
]
|
|
|
|
def __init__(self, db: AsyncSession | None = None):
|
|
super().__init__()
|
|
self._db = db
|
|
self._failure_counts: dict[str, int] = {}
|
|
|
|
async def run(self, context: dict[str, Any] | None = None) -> AgentResult:
|
|
action = (context or {}).get("action", "health_check")
|
|
if action == "health_check":
|
|
return await self._health_check_all()
|
|
elif action == "restart":
|
|
agent_name = (context or {}).get("agent_name", "")
|
|
return await self._restart_agent(agent_name)
|
|
return AgentResult(
|
|
success=False,
|
|
message=f"Unknown action: {action}",
|
|
)
|
|
|
|
async def _health_check_all(self) -> AgentResult:
|
|
from app.agents.registry import AgentRegistry
|
|
registry = AgentRegistry()
|
|
results = await registry.health_check_all()
|
|
failed = [name for name, ok in results.items() if not ok]
|
|
recovered = []
|
|
|
|
for name in failed:
|
|
self._failure_counts[name] = self._failure_counts.get(name, 0) + 1
|
|
if self._failure_counts[name] >= 3:
|
|
recovered.append({
|
|
"agent": name,
|
|
"failures": self._failure_counts[name],
|
|
"action": "notify_evolution",
|
|
})
|
|
self._failure_counts[name] = 0
|
|
|
|
healthy_count = sum(1 for ok in results.values() if ok)
|
|
total = len(results)
|
|
|
|
summary = (
|
|
f"Проверено {total} агентов: {healthy_count} здоровы, "
|
|
f"{len(failed)} требуют внимания"
|
|
)
|
|
|
|
return AgentResult(
|
|
success=len(failed) == 0,
|
|
message=summary,
|
|
data={
|
|
"checked_at": datetime.now(timezone.utc).isoformat(),
|
|
"total": total,
|
|
"healthy": healthy_count,
|
|
"failed": failed,
|
|
"recovered": recovered,
|
|
},
|
|
)
|
|
|
|
async def _restart_agent(self, agent_name: str) -> AgentResult:
|
|
if not self._db:
|
|
return AgentResult(success=False, message="No DB session")
|
|
|
|
result = await self._db.execute(
|
|
select(AgentConfig).where(AgentConfig.agent_name == agent_name)
|
|
)
|
|
agent = result.scalar_one_or_none()
|
|
if not agent:
|
|
return AgentResult(
|
|
success=False,
|
|
message=f"Agent {agent_name} not found in config",
|
|
)
|
|
|
|
agent.last_run_at = None
|
|
await self._db.commit()
|
|
|
|
self._failure_counts.pop(agent_name, None)
|
|
|
|
return AgentResult(
|
|
success=True,
|
|
message=f"Агент {agent_name} перезапущен (сброс состояния)",
|
|
data={"agent_name": agent_name, "restarted_at": datetime.now(timezone.utc).isoformat()},
|
|
)
|
|
|
|
async def health_check(self) -> bool:
|
|
return True
|
|
|
|
def get_metrics(self) -> dict[str, Any]:
|
|
return {
|
|
"total_failures": sum(self._failure_counts.values()),
|
|
"agents_with_failures": {
|
|
k: v for k, v in self._failure_counts.items() if v > 0
|
|
},
|
|
}
|