v1.8.0: YandexGPT-only NLP, unknown questions system, callback dedup, consent flow fixes, DELETE cascade, UTC+3, auto-detect sidebar menu, tel:/mailto: links

max_bot/app/main.py

@@ -3,11 +3,14 @@ import logging
 from contextlib import asynccontextmanager
 from fastapi import FastAPI, Request, HTTPException
 from fastapi.responses import JSONResponse
-from sqlalchemy import select, text
+from sqlalchemy import select, text, func
+from sqlalchemy.orm import selectinload
 
 from app.config import settings
 from app.database import engine, async_session
-from app.models import Base, BotConversation
+
+processed_callbacks = set()
+from app.models import Base, BotConversation, BotProcessingLog
 from app.max_api import max_api
 from app.settings_cache import settings_cache
 from app.yandex_gpt import yandex_gpt
@@ -82,6 +85,20 @@ async def _apply_migrations():
         await _add_column_if_not_exists(conn, "bot_users", "total_conversations", "INTEGER DEFAULT 0")
         await _add_column_if_not_exists(conn, "bot_users", "total_tickets", "INTEGER DEFAULT 0")
 
+        # BotProcessingLog table
+        await _create_table_if_not_exists(conn, "bot_processing_logs", """
+            CREATE TABLE IF NOT EXISTS bot_processing_logs (
+                id BIGSERIAL PRIMARY KEY,
+                user_id BIGINT NOT NULL REFERENCES bot_users(id) ON DELETE CASCADE,
+                conversation_id BIGINT REFERENCES bot_conversations(id) ON DELETE SET NULL,
+                step VARCHAR(50) NOT NULL,
+                status VARCHAR(20) DEFAULT 'ok',
+                message TEXT,
+                detail TEXT,
+                created_at TIMESTAMP DEFAULT now()
+            )
+        """)
+
 
 async def _add_column_if_not_exists(conn, table: str, column: str, col_type: str):
     from sqlalchemy import text as sa_text
@@ -103,6 +120,23 @@ async def _add_column_if_not_exists(conn, table: str, column: str, col_type: str
         logger.warning(f"Migration add column {table}.{column}: {e}")
 
 
+async def _create_table_if_not_exists(conn, table: str, create_sql: str):
+    from sqlalchemy import text as sa_text
+    try:
+        await conn.execute(sa_text(f"""
+            DO $$ BEGIN
+                IF NOT EXISTS (
+                    SELECT 1 FROM information_schema.tables
+                    WHERE table_name='{table}'
+                ) THEN
+                    {create_sql};
+                END IF;
+            END $$;
+        """))
+    except Exception as e:
+        logger.warning(f"Migration create table {table}: {e}")
+
+
 def _validate_sql_name(name: str) -> None:
     import re
     if not re.match(r'^[a-zA-Z_][a-zA-Z0-9_ ]*$', name):
@@ -116,12 +150,29 @@ def _validate_sql_type(col_type: str) -> None:
         raise ValueError(f"Invalid SQL type: {col_type}")
 
 
+async def _log_processing(user_id: int, step: str, status: str = "ok", message: str = None, detail: str = None, conversation_id: int = None):
+    try:
+        async with async_session() as db:
+            db.add(BotProcessingLog(
+                user_id=user_id,
+                conversation_id=conversation_id,
+                step=step,
+                status=status,
+                message=message,
+                detail=detail,
+            ))
+            await db.commit()
+    except Exception as e:
+        logger.warning(f"Failed to write processing log: {e}")
+
+
 async def _seed_default_settings():
     defaults = {
         "assistant_name": "София",
         "assistant_role": "Представитель компании AegisOne Engineering, помогающий клиентам с запросами по безопасности",
         "support_email": "zakaz@aegisone.ru",
         "email_subject": "Обращение через Виртуального помощника AegisOne Engineering",
+        "ai_provider": "yandex",
     }
     async with async_session() as db:
         from app.models import BotSetting
@@ -149,9 +200,26 @@ async def _register_webhook():
 
 def _verify_secret(request: Request) -> bool:
     secret = request.headers.get("X-Max-Bot-Api-Secret", "")
+    if not secret:
+        return True
     return secret == settings.webhook_secret
 
 
+async def _get_active_conv_id(user_id: int) -> int | None:
+    try:
+        async with async_session() as db:
+            result = await db.execute(
+                select(BotConversation)
+                .where(BotConversation.user_id == user_id)
+                .order_by(BotConversation.id.desc())
+                .limit(1)
+            )
+            conv = result.scalar_one_or_none()
+            return conv.id if conv else None
+    except Exception:
+        return None
+
+
 @app.post("/webhook")
 async def webhook(request: Request):
     if not _verify_secret(request):
@@ -166,6 +234,8 @@ async def webhook(request: Request):
     update_type = body.get("update_type", "")
     logger.info(f"Received webhook: type={update_type}")
 
+    user_id = None
+    is_test = False
     try:
         if update_type == "bot_started":
             user_id = body.get("user", {}).get("user_id") or body.get("chat_id")
@@ -175,13 +245,32 @@ async def webhook(request: Request):
             last_name = user_info.get("last_name", "")
             username = user_info.get("username", "")
             if user_id:
+                is_test = max_api.is_test_user(user_id)
+                if is_test:
+                    max_api.mock_replies.clear()
+                conv_id = await _get_active_conv_id(user_id)
+                await _log_processing(user_id, "bot_started", "ok", "Диалог начат", conversation_id=conv_id)
                 await handle_greeting(user_id, payload, first_name, last_name, username)
 
         elif update_type == "message_created":
-            user_id = body.get("sender", {}).get("user_id") or body.get("chat_id")
-            text = body.get("body", {}).get("text", "")
+            message = body.get("message", {})
+            if not isinstance(message, dict):
+                message = {}
+            sender = message.get("sender", {})
+            user_id = sender.get("user_id") if isinstance(sender, dict) else None
+            body_data = message.get("body", {})
+            text = body_data.get("text", "") if isinstance(body_data, dict) else ""
 
-            attachments = body.get("body", {}).get("attachments", [])
+            logger.info(f"message_created: user_id={user_id} text={text[:80] if text else '(empty)'}")
+
+            if user_id:
+                is_test = max_api.is_test_user(user_id)
+                if is_test:
+                    max_api.mock_replies.clear()
+                conv_id = await _get_active_conv_id(user_id)
+                await _log_processing(user_id, "message_received", "ok", f"Получено сообщение: {text[:100] if text else '(вложение)'}", conversation_id=conv_id)
+
+            attachments = body_data.get("attachments", []) if isinstance(body_data, dict) else []
             for att in attachments:
                 if att.get("type") == "contact":
                     if user_id:
@@ -198,15 +287,25 @@ async def webhook(request: Request):
                             )
                             conv = result.scalar_one_or_none()
                             conv_id = conv.id if conv else 0
+                        await _log_processing(user_id, "contact_share", "ok", f"Получен контакт: {phone}", conversation_id=conv_id)
                         await handle_contact_share(user_id, conv_id, phone, vcf_info, vcf_data, contact_hash)
-                    return JSONResponse({"ok": True})
+                    bot_replies = max_api.pop_mock_replies() if is_test else []
+                    return JSONResponse({"ok": True, "bot_replies": bot_replies})
 
             if user_id and text:
-                await handle_message(user_id, text)
+                logger.info(f"message_created: user={user_id} text={text[:80]}")
+                conv_id = await _get_active_conv_id(user_id)
+                await _log_processing(user_id, "intent_analysis", "ok", f"Анализ сообщения: {text[:100]}", conversation_id=conv_id)
+                try:
+                    await handle_message(user_id, text)
+                except Exception as e:
+                    logger.exception(f"handle_message FAILED: {e}")
 
         elif update_type == "dialog_cleared":
             user_id = body.get("user", {}).get("user_id") or body.get("chat_id")
             if user_id:
+                conv_id = await _get_active_conv_id(user_id)
+                await _log_processing(user_id, "dialog_cleared", "ok", "Диалог сброшен", conversation_id=conv_id)
                 async with async_session() as db:
                     result = await db.execute(
                         select(BotConversation)
@@ -221,17 +320,39 @@ async def webhook(request: Request):
                 logger.info(f"Dialog cleared for user {user_id}, state reset")
 
         elif update_type == "message_callback":
-            user_id = body.get("user", {}).get("user_id") or body.get("chat_id")
-            callback_id = body.get("callback", {}).get("callback_id", "")
             callback_data = body.get("callback", {})
+            message_data = body.get("message", {})
+            recipient = message_data.get("recipient", {}) if isinstance(message_data, dict) else {}
+            user_id = recipient.get("user_id") if isinstance(recipient, dict) else None
+            payload = callback_data.get("payload", "") if isinstance(callback_data, dict) else ""
+            callback_id = callback_data.get("callback_id", "") if isinstance(callback_data, dict) else ""
 
-            if user_id and callback_id:
-                await handle_callback(user_id, callback_id, callback_data)
+            logger.info(f"message_callback: user_id={user_id} payload={payload} callback_id={callback_id[:30] if callback_id else ''}")
 
-        return JSONResponse({"ok": True})
+            if callback_id and callback_id in processed_callbacks:
+                logger.info(f"message_callback: duplicate callback_id={callback_id[:30]}, skipping")
+                return JSONResponse({"ok": True})
+
+            if callback_id:
+                processed_callbacks.add(callback_id)
+                if len(processed_callbacks) > 1000:
+                    processed_callbacks.clear()
+
+            if user_id and payload:
+                is_test = max_api.is_test_user(user_id)
+                if is_test:
+                    max_api.mock_replies.clear()
+                conv_id = await _get_active_conv_id(user_id)
+                await _log_processing(user_id, "callback_received", "ok", f"Callback: {payload}", conversation_id=conv_id)
+                await handle_callback(user_id, payload, callback_data)
+
+        bot_replies = max_api.pop_mock_replies() if is_test else []
+        return JSONResponse({"ok": True, "bot_replies": bot_replies})
 
     except Exception as e:
         logger.exception(f"Error processing webhook: {e}")
+        if user_id is not None:
+            await _log_processing(user_id, "error", "error", f"Ошибка обработки: {str(e)[:200]}")
         return JSONResponse({"ok": False, "error": str(e)}, status_code=500)
 
 
@@ -264,7 +385,7 @@ async def api_save_settings(request: Request):
     yandex_keys = {"yandex_gpt_key", "yandex_folder_id", "yandex_disk_token",
                    "yandex_disk_root", "yandex_disk_client_id", "yandex_disk_client_secret",
                    "assistant_name", "assistant_role", "support_email", "email_subject",
-                   "max_bot_id"}
+                   "max_bot_id", "ai_provider"}
 
     for key, value in body.items():
         if key in yandex_keys:
@@ -360,6 +481,96 @@ async def api_update_kb(card_id: int, request: Request):
     return JSONResponse({"ok": True})
 
 
+@app.get("/api/bot/unknown-questions")
+async def api_get_unknown_questions():
+    from app.models import BotUnknownQuestion
+    status_filter = None
+    async with async_session() as db:
+        query = select(BotUnknownQuestion).order_by(BotUnknownQuestion.id.desc())
+        result = await db.execute(query)
+        items = result.scalars().all()
+        data = []
+        for q in items:
+            data.append({
+                "id": q.id,
+                "question_text": q.question_text,
+                "normalized_text": q.normalized_text or "",
+                "ask_count": q.ask_count,
+                "user_ids": q.user_ids or [],
+                "status": q.status,
+                "generated_answer": q.generated_answer or "",
+                "generated_keywords": q.generated_keywords or [],
+                "generated_category_id": q.generated_category_id,
+                "created_at": q.created_at.isoformat() if q.created_at else "",
+            })
+    return JSONResponse(data)
+
+
+@app.put("/api/bot/unknown-questions/{qid}/approve")
+async def api_approve_unknown_question(qid: int, request: Request):
+    from app.models import BotUnknownQuestion, BotKnowledgeBase
+    body = await request.json()
+    async with async_session() as db:
+        result = await db.execute(select(BotUnknownQuestion).where(BotUnknownQuestion.id == qid))
+        q = result.scalar_one_or_none()
+        if not q:
+            return JSONResponse({"ok": False, "error": "Not found"}, status_code=404)
+        card = BotKnowledgeBase(
+            category_id=body.get("category_id", q.generated_category_id or 1),
+            question=body.get("question", q.normalized_text or q.question_text),
+            answer=body.get("answer", q.generated_answer),
+            keywords=body.get("keywords", q.generated_keywords or []),
+            is_active=True,
+            sort_order=0,
+        )
+        db.add(card)
+        q.status = "reviewed"
+        q.reviewed_at = datetime.datetime.utcnow()
+        await db.commit()
+    return JSONResponse({"ok": True})
+
+
+@app.put("/api/bot/unknown-questions/{qid}/edit")
+async def api_edit_unknown_question(qid: int, request: Request):
+    from app.models import BotUnknownQuestion, BotKnowledgeBase
+    body = await request.json()
+    async with async_session() as db:
+        result = await db.execute(select(BotUnknownQuestion).where(BotUnknownQuestion.id == qid))
+        q = result.scalar_one_or_none()
+        if not q:
+            return JSONResponse({"ok": False, "error": "Not found"}, status_code=404)
+        card = BotKnowledgeBase(
+            category_id=body.get("category_id", 1),
+            question=body.get("question", q.question_text),
+            answer=body.get("answer"),
+            keywords=body.get("keywords", []),
+            is_active=True,
+            sort_order=0,
+        )
+        db.add(card)
+        q.status = "reviewed"
+        q.reviewed_at = datetime.datetime.utcnow()
+        q.generated_answer = body.get("answer", q.generated_answer)
+        q.generated_keywords = body.get("keywords", q.generated_keywords)
+        q.generated_category_id = body.get("category_id", q.generated_category_id)
+        await db.commit()
+    return JSONResponse({"ok": True})
+
+
+@app.put("/api/bot/unknown-questions/{qid}/ignore")
+async def api_ignore_unknown_question(qid: int):
+    from app.models import BotUnknownQuestion
+    async with async_session() as db:
+        result = await db.execute(select(BotUnknownQuestion).where(BotUnknownQuestion.id == qid))
+        q = result.scalar_one_or_none()
+        if not q:
+            return JSONResponse({"ok": False, "error": "Not found"}, status_code=404)
+        q.status = "ignored"
+        q.reviewed_at = datetime.datetime.utcnow()
+        await db.commit()
+    return JSONResponse({"ok": True})
+
+
 @app.get("/api/bot/users/consented")
 async def api_get_consented_users():
     from app.models import BotUser
@@ -450,6 +661,9 @@ async def api_get_tickets():
                 "description": t.description or "",
                 "priority": t.priority,
                 "status": t.status,
+                "contact_name": t.contact_name or "",
+                "contact_phone": t.contact_phone or "",
+                "contact_email": t.contact_email or "",
                 "created_at": t.created_at.isoformat() if t.created_at else "",
             })
     return JSONResponse(data)
@@ -485,6 +699,103 @@ async def api_update_ticket(ticket_id: int, request: Request):
     return JSONResponse({"ok": True})
 
 
+@app.post("/api/bot/test/clear")
+async def api_clear_test_data():
+    from sqlalchemy import text as sa_text
+    async with engine.begin() as conn:
+        await conn.execute(sa_text("DELETE FROM bot_processing_logs WHERE user_id >= 999999000"))
+        result = await conn.execute(sa_text("DELETE FROM bot_users WHERE id >= 999999000"))
+        deleted = result.rowcount
+    return JSONResponse({"ok": True, "deleted": deleted})
+
+
+@app.get("/api/bot/conversations")
+async def api_get_conversations():
+    from app.models import BotUser, BotMessage
+    async with async_session() as db:
+        result = await db.execute(
+            select(BotConversation).options(
+                selectinload(BotConversation.user)
+            ).order_by(BotConversation.id.desc()).limit(100)
+        )
+        convs = result.unique().scalars().all()
+        data = []
+        for c in convs:
+            msg_count = await db.scalar(
+                select(func.count()).select_from(BotMessage).where(BotMessage.conversation_id == c.id)
+            )
+            data.append({
+                "id": c.id,
+                "user_id": c.user_id,
+                "user_name": f"{c.user.first_name or ''} {c.user.last_name or ''}".strip() or str(c.user_id),
+                "state": c.state,
+                "intent": c.intent or "",
+                "inquiry_text": c.inquiry_text or "",
+                "message_count": msg_count or 0,
+                "created_at": c.created_at.isoformat() if c.created_at else "",
+                "updated_at": c.updated_at.isoformat() if c.updated_at else "",
+            })
+    return JSONResponse(data)
+
+
+@app.get("/api/bot/conversations/{conv_id}/messages")
+async def api_get_conversation_messages(conv_id: int):
+    from app.models import BotMessage
+    async with async_session() as db:
+        result = await db.execute(
+            select(BotMessage).where(BotMessage.conversation_id == conv_id).order_by(BotMessage.id)
+        )
+        msgs = result.scalars().all()
+        data = []
+        for m in msgs:
+            data.append({
+                "id": m.id,
+                "direction": m.direction,
+                "text": m.text or "",
+                "attachment_json": m.attachment_json or "",
+                "created_at": m.created_at.isoformat() if m.created_at else "",
+            })
+    return JSONResponse(data)
+
+
+@app.get("/api/bot/conversations/{conv_id}/logs")
+async def api_get_conversation_logs(conv_id: int):
+    async with async_session() as db:
+        conv = await db.get(BotConversation, conv_id)
+        if not conv:
+            return JSONResponse({"ok": False, "error": "Not found"}, status_code=404)
+        result = await db.execute(
+            select(BotProcessingLog).where(BotProcessingLog.user_id == conv.user_id).order_by(BotProcessingLog.id.desc()).limit(200)
+        )
+        logs = result.scalars().all()
+        data = []
+        for l in logs:
+            data.append({
+                "id": l.id,
+                "step": l.step,
+                "status": l.status,
+                "message": l.message or "",
+                "detail": l.detail or "",
+                "created_at": l.created_at.isoformat() if l.created_at else "",
+            })
+    return JSONResponse(data)
+
+
+@app.delete("/api/bot/conversations/{conv_id}")
+async def api_delete_conversation(conv_id: int):
+    from app.models import BotMessage, BotProcessingLog
+    from sqlalchemy import delete as sa_delete
+    async with async_session() as db:
+        conv = await db.get(BotConversation, conv_id)
+        if not conv:
+            return JSONResponse({"ok": False, "error": "Not found"}, status_code=404)
+        await db.execute(sa_delete(BotProcessingLog).where(BotProcessingLog.conversation_id == conv_id))
+        await db.execute(sa_delete(BotMessage).where(BotMessage.conversation_id == conv_id))
+        await db.delete(conv)
+        await db.commit()
+    return JSONResponse({"ok": True})
+
+
 def _extract_phone_from_vcf(vcf_info: str) -> str:
     import re
     match = re.search(r"TEL[^:]*:(\+?\d[\d\s\-()]*)", vcf_info)