458 lines
19 KiB
Python
458 lines
19 KiB
Python
"""QATesterAgent v2 — real functional testing with API + Playwright E2E.
|
|
|
|
Modes:
|
|
- api: HTTP tests against backend endpoints
|
|
- e2e: Playwright headless browser tests against UI (skip if unavailable)
|
|
- full: both modes
|
|
|
|
Creates temp users, runs tests, cleans up. Reports bugs to FixAgent via LogEntry.
|
|
"""
|
|
|
|
import asyncio
|
|
import json
|
|
import logging
|
|
import os
|
|
import time
|
|
from datetime import datetime, timezone
|
|
from typing import Any
|
|
from uuid import uuid4
|
|
|
|
from sqlalchemy import delete, select
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
|
|
from app.agents.base import AgentResult, AgentStatus, AgentTrigger, BaseAgent
|
|
from app.models.log import LogEntry
|
|
from app.models.user import User
|
|
from app.core.config import get_settings
|
|
from app.core.security import get_password_hash
|
|
|
|
logger = logging.getLogger("voidea.qa_tester")
|
|
|
|
PLAYWRIGHT_AVAILABLE = False
|
|
try:
|
|
from playwright.async_api import async_playwright
|
|
PLAYWRIGHT_AVAILABLE = True
|
|
except ImportError:
|
|
pass
|
|
|
|
TEST_BASE_URL = os.environ.get("TEST_BASE_URL", "http://localhost:8020")
|
|
TEST_FRONTEND_URL = os.environ.get("TEST_FRONTEND_URL", "http://localhost:3000")
|
|
|
|
|
|
class QATesterAgent(BaseAgent):
|
|
"""Functional testing agent with API + E2E browser tests."""
|
|
|
|
name = "qa_tester_agent"
|
|
version = "2.0.0"
|
|
description = "API + E2E testing with temp users and auto-cleanup"
|
|
triggers = [
|
|
AgentTrigger.MANUAL,
|
|
AgentTrigger.CRON,
|
|
AgentTrigger.PRE_COMMIT,
|
|
]
|
|
|
|
def __init__(self, session: AsyncSession | None = None):
|
|
super().__init__()
|
|
self._session = session
|
|
self._temp_users: list[dict[str, Any]] = []
|
|
self._bugs: list[dict[str, Any]] = []
|
|
self._settings = get_settings()
|
|
|
|
# ── Main ──
|
|
|
|
async def run(self, context: dict[str, Any] | None = None) -> AgentResult:
|
|
start = time.time()
|
|
await self.set_running("qa_testing")
|
|
|
|
try:
|
|
action = (context or {}).get("action", "full")
|
|
|
|
if action == "api":
|
|
result = await self._run_api_tests()
|
|
elif action == "e2e":
|
|
result = await self._run_e2e_tests()
|
|
elif action == "vitest":
|
|
result = await self._run_vitest_tests()
|
|
elif action == "cleanup":
|
|
result = await self._cleanup_all()
|
|
elif action == "status":
|
|
result = await self._get_status()
|
|
else:
|
|
result = await self._run_full()
|
|
|
|
result.duration_ms = int((time.time() - start) * 1000)
|
|
await self.set_idle()
|
|
await self._report_bugs()
|
|
return result
|
|
|
|
except Exception as e:
|
|
await self.set_error(str(e))
|
|
return AgentResult(
|
|
success=False,
|
|
message=f"QATesterAgent failed: {str(e)}",
|
|
errors=[str(e)],
|
|
)
|
|
|
|
# ── Full suite ──
|
|
|
|
async def _run_full(self) -> AgentResult:
|
|
api_result = await self._run_api_tests()
|
|
e2e_result = await self._run_e2e_tests()
|
|
vitest_result = await self._run_vitest_tests()
|
|
total_bugs = self._bugs.copy()
|
|
combined_success = api_result.success or e2e_result.success or vitest_result.success
|
|
combined_msg = f"API: {api_result.message} | E2E: {e2e_result.message} | Vitest: {vitest_result.message}"
|
|
combined_data = {
|
|
"api": api_result.data,
|
|
"e2e": e2e_result.data,
|
|
"vitest": vitest_result.data,
|
|
"bugs": total_bugs,
|
|
}
|
|
await self._cleanup_all()
|
|
return AgentResult(
|
|
success=combined_success,
|
|
message=combined_msg,
|
|
data=combined_data,
|
|
errors=api_result.errors + e2e_result.errors + vitest_result.errors,
|
|
)
|
|
|
|
# ── API tests ──
|
|
|
|
async def _run_api_tests(self) -> AgentResult:
|
|
import httpx
|
|
|
|
temp_user = await self._create_temp_user()
|
|
if not temp_user:
|
|
return AgentResult(success=False, message="Failed to create temp user", errors=["Temp user creation failed"])
|
|
|
|
tests = []
|
|
base = TEST_BASE_URL
|
|
|
|
async with httpx.AsyncClient(base_url=base, timeout=15.0) as client:
|
|
# 1. Health check
|
|
try:
|
|
r = await client.get("/health")
|
|
tests.append({"name": "health_check", "passed": r.status_code == 200, "detail": f"GET /health → {r.status_code}"})
|
|
except Exception as e:
|
|
tests.append({"name": "health_check", "passed": False, "detail": str(e)})
|
|
|
|
# 2. Register (same user could conflict, so check)
|
|
email = temp_user["email"]
|
|
password = temp_user["password"]
|
|
try:
|
|
r = await client.post("/api/v1/auth/register", json={"email": email, "password": password, "display_name": "Test User", "accepted_terms": True})
|
|
tests.append({"name": "register", "passed": r.status_code in (201, 409), "detail": f"POST /auth/register → {r.status_code}"})
|
|
except Exception as e:
|
|
tests.append({"name": "register", "passed": False, "detail": str(e)})
|
|
|
|
# 3. Login
|
|
access_token = None
|
|
try:
|
|
r = await client.post("/api/v1/auth/login", json={"email": email, "password": password})
|
|
if r.status_code == 200:
|
|
data = r.json()
|
|
access_token = data.get("access_token")
|
|
tests.append({"name": "login", "passed": r.status_code == 200, "detail": f"POST /auth/login → {r.status_code}"})
|
|
except Exception as e:
|
|
tests.append({"name": "login", "passed": False, "detail": str(e)})
|
|
|
|
if access_token:
|
|
headers = {"Authorization": f"Bearer {access_token}"}
|
|
|
|
# 4. Get me
|
|
try:
|
|
r = await client.get("/api/v1/users/me", headers=headers)
|
|
tests.append({"name": "get_me", "passed": r.status_code == 200, "detail": f"GET /users/me → {r.status_code}"})
|
|
except Exception as e:
|
|
tests.append({"name": "get_me", "passed": False, "detail": str(e)})
|
|
|
|
# 5. Create idea
|
|
idea_id = None
|
|
try:
|
|
r = await client.post("/api/v1/ideas", headers=headers, json={"title": "Test idea from QA", "content": "This is a test idea created by QATesterAgent"})
|
|
if r.status_code in (200, 201):
|
|
idea_data = r.json()
|
|
idea_id = idea_data.get("id")
|
|
tests.append({"name": "create_idea", "passed": r.status_code in (200, 201), "detail": f"POST /ideas → {r.status_code}"})
|
|
except Exception as e:
|
|
tests.append({"name": "create_idea", "passed": False, "detail": str(e)})
|
|
|
|
# 6. List ideas
|
|
try:
|
|
r = await client.get("/api/v1/ideas", headers=headers)
|
|
tests.append({"name": "list_ideas", "passed": r.status_code == 200, "detail": f"GET /ideas → {r.status_code}"})
|
|
except Exception as e:
|
|
tests.append({"name": "list_ideas", "passed": False, "detail": str(e)})
|
|
|
|
# 7. Delete test idea
|
|
if idea_id:
|
|
try:
|
|
r = await client.delete(f"/api/v1/ideas/{idea_id}", headers=headers)
|
|
tests.append({"name": "delete_idea", "passed": r.status_code in (204, 200), "detail": f"DELETE /ideas/{idea_id} → {r.status_code}"})
|
|
except Exception as e:
|
|
tests.append({"name": "delete_idea", "passed": False, "detail": str(e)})
|
|
|
|
# 8. Voice settings
|
|
try:
|
|
r = await client.get("/api/v1/users/me/voice-settings", headers=headers)
|
|
tests.append({"name": "voice_settings", "passed": r.status_code == 200, "detail": f"GET /voice-settings → {r.status_code}"})
|
|
except Exception as e:
|
|
tests.append({"name": "voice_settings", "passed": False, "detail": str(e)})
|
|
|
|
# 9. Public config
|
|
try:
|
|
r = await client.get("/api/v1/config/public")
|
|
tests.append({"name": "public_config", "passed": r.status_code == 200, "detail": f"GET /config/public → {r.status_code}"})
|
|
except Exception as e:
|
|
tests.append({"name": "public_config", "passed": False, "detail": str(e)})
|
|
|
|
passed = sum(1 for t in tests if t["passed"])
|
|
failed = [t for t in tests if not t["passed"]]
|
|
if failed:
|
|
self._bugs.extend({
|
|
"test": t["name"],
|
|
"detail": t["detail"],
|
|
"source": "api",
|
|
"severity": "medium",
|
|
} for t in failed)
|
|
|
|
return AgentResult(
|
|
success=len(failed) == 0,
|
|
message=f"API tests: {passed}/{len(tests)} passed",
|
|
data={"total": len(tests), "passed": passed, "failed": len(failed), "tests": tests, "bugs": self._bugs},
|
|
errors=[f"{t['name']}: {t['detail']}" for t in failed],
|
|
)
|
|
|
|
# ── Vitest (frontend unit tests) ──
|
|
|
|
async def _run_vitest_tests(self) -> AgentResult:
|
|
webui_dir = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), "webui")
|
|
if not os.path.isdir(os.path.join(webui_dir, "node_modules")):
|
|
return AgentResult(
|
|
success=True,
|
|
message="Vitest пропущен: node_modules не найдены",
|
|
data={"skipped": True, "reason": "node_modules not found"},
|
|
)
|
|
|
|
import subprocess
|
|
|
|
try:
|
|
proc = await asyncio.create_subprocess_exec(
|
|
"npx", "vitest", "run", "--reporter=json",
|
|
stdout=asyncio.subprocess.PIPE,
|
|
stderr=asyncio.subprocess.PIPE,
|
|
cwd=webui_dir,
|
|
)
|
|
stdout, stderr = await proc.communicate()
|
|
output = stdout.decode("utf-8", errors="replace")
|
|
|
|
if proc.returncode != 0:
|
|
logger.warning("Vitest exited with code %d: %s", proc.returncode, stderr.decode()[:200])
|
|
|
|
# Vitest JSON output starts after potential Vite banner
|
|
json_start = output.find("{")
|
|
if json_start == -1:
|
|
return AgentResult(
|
|
success=False,
|
|
message="Vitest: JSON output not found",
|
|
errors=[output[:500]],
|
|
)
|
|
|
|
import json as json_mod
|
|
data = json_mod.loads(output[json_start:])
|
|
total = data.get("total", 0)
|
|
passed = sum(1 for f in data.get("files", []) if f.get("result") == "passed")
|
|
failed_files = [f for f in data.get("files", []) if f.get("result") != "passed"]
|
|
|
|
if failed_files:
|
|
for ff in failed_files:
|
|
filepath = ff.get("filepath", ff.get("name", "unknown"))
|
|
self._bugs.append({
|
|
"test": filepath,
|
|
"detail": f"Vitest failed: {json_mod.dumps(ff.get('failureMessage', 'unknown'))}",
|
|
"source": "vitest",
|
|
"severity": "high",
|
|
})
|
|
|
|
return AgentResult(
|
|
success=len(failed_files) == 0,
|
|
message=f"Vitest: {passed}/{total} passed",
|
|
data={"total": total, "passed": passed, "failed": len(failed_files), "files": data.get("files", [])},
|
|
errors=[f"{f.get('filepath', '?')} failed" for f in failed_files],
|
|
)
|
|
|
|
except FileNotFoundError:
|
|
return AgentResult(
|
|
success=True,
|
|
message="Vitest пропущен: npx не найден",
|
|
data={"skipped": True, "reason": "npx not found"},
|
|
)
|
|
|
|
# ── E2E tests (Playwright) ──
|
|
|
|
async def _run_e2e_tests(self) -> AgentResult:
|
|
if not PLAYWRIGHT_AVAILABLE:
|
|
return AgentResult(
|
|
success=True,
|
|
message="Playwright не установлен, E2E тесты пропущены",
|
|
data={"skipped": True, "reason": "playwright not installed"},
|
|
)
|
|
|
|
tests = []
|
|
frontend_url = TEST_FRONTEND_URL
|
|
|
|
try:
|
|
async with async_playwright() as p:
|
|
browser = await p.chromium.launch(headless=True, args=["--no-sandbox"])
|
|
page = await browser.new_page()
|
|
|
|
# 1. Landing page loads
|
|
try:
|
|
await page.goto(frontend_url, wait_until="networkidle", timeout=30000)
|
|
title = await page.title()
|
|
tests.append({"name": "landing_loads", "passed": bool(title), "detail": f"Title: {title[:50]}"})
|
|
except Exception as e:
|
|
tests.append({"name": "landing_loads", "passed": False, "detail": str(e)})
|
|
|
|
# 2. Navigate to /login
|
|
try:
|
|
await page.goto(f"{frontend_url}/login", wait_until="networkidle", timeout=15000)
|
|
has_form = await page.query_selector('input[type="email"], input[name="email"]') is not None
|
|
tests.append({"name": "login_page", "passed": has_form, "detail": "Login form found" if has_form else "No email input"})
|
|
except Exception as e:
|
|
tests.append({"name": "login_page", "passed": False, "detail": str(e)})
|
|
|
|
# 3. Navigate to /register
|
|
try:
|
|
await page.goto(f"{frontend_url}/register", wait_until="networkidle", timeout=15000)
|
|
has_register_form = await page.query_selector('input[type="password"]') is not None
|
|
tests.append({"name": "register_page", "passed": has_register_form, "detail": "Register form found" if has_register_form else "No password input"})
|
|
except Exception as e:
|
|
tests.append({"name": "register_page", "passed": False, "detail": str(e)})
|
|
|
|
# 4. Dashboard (may redirect to login)
|
|
try:
|
|
await page.goto(f"{frontend_url}/dashboard", wait_until="networkidle", timeout=15000)
|
|
tests.append({"name": "dashboard_redirect", "passed": True, "detail": f"URL: {page.url[:60]}"})
|
|
except Exception as e:
|
|
tests.append({"name": "dashboard_redirect", "passed": False, "detail": str(e)})
|
|
|
|
# 5. Check dark mode toggle exists
|
|
try:
|
|
await page.goto(f"{frontend_url}/settings", wait_until="networkidle", timeout=15000)
|
|
tests.append({"name": "settings_page", "passed": True, "detail": f"Settings loaded: {page.url[:60]}"})
|
|
except Exception as e:
|
|
tests.append({"name": "settings_page", "passed": False, "detail": str(e)})
|
|
|
|
await browser.close()
|
|
|
|
except Exception as e:
|
|
return AgentResult(
|
|
success=False,
|
|
message=f"E2E tests failed: {str(e)}",
|
|
errors=[str(e)],
|
|
)
|
|
|
|
passed = sum(1 for t in tests if t["passed"])
|
|
failed = [t for t in tests if not t["passed"]]
|
|
if failed:
|
|
self._bugs.extend({
|
|
"test": t["name"],
|
|
"detail": t["detail"],
|
|
"source": "e2e",
|
|
"severity": "high",
|
|
} for t in failed)
|
|
|
|
return AgentResult(
|
|
success=len(failed) == 0,
|
|
message=f"E2E tests: {passed}/{len(tests)} passed",
|
|
data={"total": len(tests), "passed": passed, "failed": len(failed), "tests": tests, "bugs": self._bugs},
|
|
errors=[f"{t['name']}: {t['detail']}" for t in failed],
|
|
)
|
|
|
|
# ── Temp user management ──
|
|
|
|
async def _create_temp_user(self) -> dict[str, Any] | None:
|
|
if not self._session:
|
|
return {"email": "test@voidea.test", "password": "test123", "simulated": True}
|
|
|
|
temp_id = str(uuid4())[:8]
|
|
email = f"qa_test_{temp_id}@voidea.test"
|
|
password = f"qa_pass_{temp_id}"
|
|
|
|
user = User(
|
|
id=uuid4(),
|
|
email=email,
|
|
password_hash=get_password_hash(password),
|
|
is_active=True,
|
|
role="user",
|
|
display_name=f"QA Test {temp_id}",
|
|
)
|
|
self._session.add(user)
|
|
await self._session.commit()
|
|
|
|
entry = {"id": str(user.id), "email": email, "password": password}
|
|
self._temp_users.append(entry)
|
|
return entry
|
|
|
|
async def _cleanup_all(self) -> AgentResult:
|
|
if not self._session or not self._temp_users:
|
|
return AgentResult(success=True, message="No cleanup needed", data={"cleaned": 0})
|
|
|
|
cleaned = 0
|
|
for entry in self._temp_users:
|
|
try:
|
|
stmt = delete(User).where(User.id == entry.get("id"))
|
|
await self._session.execute(stmt)
|
|
cleaned += 1
|
|
except Exception as e:
|
|
logger.warning("Cleanup failed for %s: %s", entry.get("email"), e)
|
|
|
|
await self._session.commit()
|
|
self._temp_users = []
|
|
return AgentResult(success=True, message=f"Cleaned up {cleaned} temp users", data={"cleaned": cleaned})
|
|
|
|
async def _report_bugs(self):
|
|
"""Write bugs as LogEntries for FixAgent."""
|
|
if not self._bugs or not self._session:
|
|
return
|
|
for bug in self._bugs:
|
|
log = LogEntry(
|
|
level="ERROR" if bug.get("severity") == "high" else "WARNING",
|
|
source="qa_tester_agent",
|
|
message=f"Bug: {bug['test']} — {bug['detail']}",
|
|
details=json.dumps(bug, ensure_ascii=False),
|
|
created_at=datetime.now(timezone.utc),
|
|
)
|
|
self._session.add(log)
|
|
try:
|
|
await self._session.commit()
|
|
except Exception:
|
|
pass
|
|
|
|
async def _get_status(self) -> AgentResult:
|
|
return AgentResult(
|
|
success=True,
|
|
message="QATesterAgent v2 status",
|
|
data={
|
|
"temp_users": len(self._temp_users),
|
|
"bugs_found": len(self._bugs),
|
|
"playwright_available": PLAYWRIGHT_AVAILABLE,
|
|
"api_base_url": TEST_BASE_URL,
|
|
"frontend_url": TEST_FRONTEND_URL,
|
|
"status": self.status.value,
|
|
},
|
|
)
|
|
|
|
async def health_check(self) -> bool:
|
|
return True
|
|
|
|
async def get_metrics(self) -> dict[str, Any]:
|
|
return {
|
|
"agent_id": self.name,
|
|
"version": self.version,
|
|
"status": self.status.value,
|
|
"temp_users_current": len(self._temp_users),
|
|
"bugs_found": len(self._bugs),
|
|
"playwright_available": PLAYWRIGHT_AVAILABLE,
|
|
} |