deploy: backend 2026-06-22T08:22:30Z

.env.example

@@ -7,9 +7,11 @@
 # --- OpenRouter ---
 OPENROUTER_API_KEY=your_openrouter_api_key_here
 OPENROUTER_BASE_URL=https://openrouter.ai/api/v1
-OPENROUTER_FREE_MODEL_PRIMARY=meta-llama/llama-3.3-70b-instruct:free
+OPENROUTER_FREE_MODEL_PRIMARY=openai/gpt-oss-20b:free
 # Comma-separated chain of fallback free models. Tried in order if primary fails.
-OPENROUTER_FREE_MODEL_FALLBACKS=qwen/qwen3-next-80b-a3b-instruct:free,nvidia/nemotron-3-super-120b-a12b:free,openai/gpt-oss-20b:free,google/gemma-4-31b-it:free
+# Keep this list short so an inference call finishes inside HF Space's
+# request budget (~60 s on free tier).
+OPENROUTER_FREE_MODEL_FALLBACKS=qwen/qwen3-next-80b-a3b-instruct:free,google/gemma-4-31b-it:free
 
 # --- Hugging Face Space (used for HTTP-Referer header) ---
 HF_SPACE_URL=https://huggingface.co/spaces/Dirbol/VirtForemanBackend

app/core/config.py

@@ -19,14 +19,14 @@ class Settings(BaseSettings):
     openrouter_base_url: str = Field(default="https://openrouter.ai/api/v1")
     # Primary + comma-separated list of fallback free models.
     # All must match OpenRouter's `:free` endpoint convention.
+    # Default chain kept short so the inference call always finishes well
+    # inside HF Space's request budget (~60 s free tier).
     openrouter_free_model_primary: str = Field(
-        default="meta-llama/llama-3.3-70b-instruct:free"
+        default="openai/gpt-oss-20b:free"
     )
     openrouter_free_model_fallbacks: str = Field(
         default=(
             "qwen/qwen3-next-80b-a3b-instruct:free,"
-            "nvidia/nemotron-3-super-120b-a12b:free,"
-            "openai/gpt-oss-20b:free,"
             "google/gemma-4-31b-it:free"
         )
     )

app/services/openrouter.py

@@ -1,25 +1,25 @@
 """OpenRouter async client with structured-output Pydantic enforcement.
 
-Free models on OpenRouter have rate limits and occasional downtime. This client:
-- Sets the three required headers (Authorization, HTTP-Referer, X-Title).
-- Tries primary free model first; on validation/transport failure → fallback model.
-- Forces JSON output via `response_format` and validates with a Pydantic schema.
+Free models on OpenRouter have aggressive rate limits (429) and occasional
+downtime. This client:
+  - Sets the required headers (Authorization, HTTP-Referer, X-Title).
+  - Tries models in a fixed chain (primary → fallbacks).
+  - **One attempt per model** (no internal retry) — the chain itself provides
+    redundancy within a tight wall-clock budget.
+  - Hard wall-clock budget (default 25 s) so the request never hangs.
+  - Forces JSON output via `response_format`; validates with Pydantic.
 """
 
 from __future__ import annotations
 
+import asyncio
 import json
+import time
 from typing import Any, Type, TypeVar
 
 import httpx
 from loguru import logger
 from pydantic import BaseModel, ValidationError
-from tenacity import (
-    retry,
-    retry_if_exception_type,
-    stop_after_attempt,
-    wait_exponential,
-)
 
 from app.core.config import get_settings
 
@@ -27,20 +27,31 @@ T = TypeVar("T", bound=BaseModel)
 
 
 class OpenRouterError(Exception):
-    """Raised when OpenRouter fails after all retries and fallbacks."""
+    """Raised when OpenRouter fails after the full chain exhausts."""
 
 
 class _StructuredResponseError(Exception):
-    """Internal: model returned JSON that did not match the requested schema."""
+    """Internal: model returned text that did not match the requested schema."""
 
 
 class OpenRouterClient:
     """Async client wrapping OpenRouter chat completions."""
 
+    # Per-attempt network budget. Connections fail fast (5 s), full read in 15 s.
+    DEFAULT_TIMEOUT = httpx.Timeout(15.0, connect=5.0)
+    # Total wall-clock budget for the entire `structured_completion()` call.
+    # Hugging Face Spaces free tier typically idles / kills workers after ~60 s.
+    DEFAULT_BUDGET_S = 25.0
+    # How long to remember a rate-limit (429) per model so we don't hammer it.
+    RATE_LIMIT_COOLDOWN_S = 60.0
+
     def __init__(self, http_client: httpx.AsyncClient | None = None) -> None:
         self.settings = get_settings()
         self._client = http_client
         self._owns_client = http_client is None
+        # Per-model ban list with deadline epoch. A model on the list is
+        # skipped during structured_completion for `RATE_LIMIT_COOLDOWN_S` s.
+        self._cooldown_until: dict[str, float] = {}
 
     def _headers(self) -> dict[str, str]:
         return {
@@ -55,7 +66,7 @@ class OpenRouterClient:
             self._client = httpx.AsyncClient(
                 base_url=self.settings.openrouter_base_url,
                 headers=self._headers(),
-                timeout=httpx.Timeout(120.0, connect=15.0),
+                timeout=self.DEFAULT_TIMEOUT,
             )
         return self._client
 
@@ -64,22 +75,14 @@ class OpenRouterClient:
             await self._client.aclose()
             self._client = None
 
-    @retry(
-        retry=retry_if_exception_type(
-            (httpx.HTTPError, _StructuredResponseError)
-        ),
-        stop=stop_after_attempt(2),
-        wait=wait_exponential(multiplier=1, min=2, max=8),
-        reraise=True,
-    )
-    async def _call_model(
+    async def _call_model_once(
         self,
         *,
         model: str,
         system_prompt: str,
         user_prompt: str,
     ) -> str:
-        """Call a specific OpenRouter model and return raw assistant text."""
+        """Single non-retrying attempt against one OpenRouter model."""
         client = await self._ensure_client()
         payload: dict[str, Any] = {
             "model": model,
@@ -91,15 +94,27 @@ class OpenRouterClient:
             ],
             "response_format": {"type": "json_object"},
         }
-        logger.debug(f"[openrouter] POST {model}")
+        logger.info(f"[openrouter] POST {model}")
         resp = await client.post("/chat/completions", json=payload)
+
+        if resp.status_code == 429:
+            # Ban this model for RATE_LIMIT_COOLDOWN_S seconds.
+            self._cooldown_until[model] = time.monotonic() + self.RATE_LIMIT_COOLDOWN_S
+            raise httpx.HTTPStatusError(
+                "rate-limited (429)", request=resp.request, response=resp
+            )
+        if resp.status_code == 404:
+            raise httpx.HTTPStatusError(
+                "model unavailable (404)", request=resp.request, response=resp
+            )
         resp.raise_for_status()
+
         data = resp.json()
         try:
             return data["choices"][0]["message"]["content"]
         except (KeyError, IndexError, TypeError) as exc:
             raise _StructuredResponseError(
-                f"Unexpected OpenRouter response shape: {exc}"
+                f"unexpected response shape: {exc}"
             ) from exc
 
     async def structured_completion(
@@ -109,16 +124,13 @@ class OpenRouterClient:
         system_prompt: str,
         user_prompt: str,
         model_override: str | None = None,
+        budget_s: float | None = None,
     ) -> tuple[T, str]:
         """Return validated Pydantic instance + model name used.
 
-        Iterates: primary model → chain of fallback models. Each model gets up
-        to 2 attempts (2nd attempt re-emphasises JSON-only output).
-
-        Retryable errors (`httpx.HTTPError`, `_StructuredResponseError`,
-        `pydantic.ValidationError`) cause a move to the next model in the chain.
-
-        Raises OpenRouterError if the entire chain fails.
+        Iterates the model chain inside a hard wall-clock budget. **One
+        attempt per model** — chain itself handles fallbacks. Stops as soon
+        as the budget is drained.
         """
         chain: list[str] = []
         if model_override:
@@ -128,42 +140,93 @@ class OpenRouterClient:
             if fb not in chain:
                 chain.append(fb)
 
-        tried: set[str] = set()
-        for model_name in chain:
-            if model_name in tried:
-                continue
-            tried.add(model_name)
+        now = time.monotonic()
+        # Filter out models that are still in cooldown from prior 429s.
+        active_chain = [
+            m for m in chain
+            if self._cooldown_until.get(m, 0.0) <= now
+        ]
+        if active_chain != chain:
+            skipped = [m for m in chain if m not in active_chain]
+            logger.info(
+                f"[openrouter] skipping {len(skipped)} model(s) in cooldown: "
+                f"{skipped}"
+            )
+
+        budget = budget_s or self.DEFAULT_BUDGET_S
+        deadline = time.monotonic() + budget
+        tried: list[str] = []
+        failures: list[str] = []
+
+        for model_name in active_chain:
+            elapsed = time.monotonic()
+            if elapsed >= deadline:
+                logger.warning(
+                    f"[openrouter] budget exhausted after {len(tried)} model(s)"
+                )
+                break
+            tried.append(model_name)
+            # Loosen the JSON-mandate reminder if we are on attempt 2+ for
+            # this model (used by the schema-retry case below).
+            current_user_prompt = user_prompt
             for attempt in range(1, 3):
+                per_attempt_deadline = min(
+                    deadline,
+                    elapsed + (deadline - elapsed) / max(1, len(chain) - len(tried) + 1),
+                )
+                remaining = per_attempt_deadline - time.monotonic()
+                if remaining <= 0:
+                    break
                 try:
-                    raw = await self._call_model(
-                        model=model_name,
-                        system_prompt=system_prompt,
-                        user_prompt=user_prompt,
+                    raw = await asyncio.wait_for(
+                        self._call_model_once(
+                            model=model_name,
+                            system_prompt=system_prompt,
+                            user_prompt=current_user_prompt,
+                        ),
+                        timeout=remaining,
                     )
-                    return schema.model_validate_json(raw), model_name
+                    instance = schema.model_validate_json(raw)
+                    logger.info(
+                        f"[openrouter] success: model={model_name}, "
+                        f"attempt={attempt}, elapsed={time.monotonic()-deadline+budget:.2f}s"
+                    )
+                    return instance, model_name
+                except asyncio.TimeoutError:
+                    failures.append(f"{model_name}: budget timeout")
+                    logger.warning(
+                        f"[openrouter] {model_name} attempt {attempt}: timeout"
+                    )
+                    break  # move to next model
                 except _StructuredResponseError as exc:
+                    failures.append(f"{model_name}: bad shape — {exc}")
                     logger.warning(
-                        f"[openrouter] {model_name} attempt {attempt} schema fail: {exc}"
+                        f"[openrouter] {model_name} attempt {attempt}: bad shape: {exc}"
                     )
-                    user_prompt = (
-                        user_prompt
-                        + "\n\nReturn ONLY valid JSON with no commentary or markdown."
+                    # Re-prompt with the JSON-only suffix once, then move on.
+                    current_user_prompt = (
+                        user_prompt + "\n\nReturn ONLY valid JSON, no commentary."
                     )
-                    continue
+                    if attempt == 1:
+                        continue
+                    break
                 except (httpx.HTTPStatusError, ValidationError) as exc:
+                    failures.append(f"{model_name}: {type(exc).__name__} — {exc}")
                     logger.warning(
-                        f"[openrouter] {model_name} attempt {attempt} error: {exc}"
+                        f"[openrouter] {model_name} attempt {attempt}: "
+                        f"{type(exc).__name__} — {exc}"
                     )
                     break  # move to next model
                 except httpx.HTTPError as exc:
+                    failures.append(f"{model_name}: transport — {exc}")
                     logger.warning(
-                        f"[openrouter] {model_name} attempt {attempt} transport error: {exc}"
+                        f"[openrouter] {model_name} attempt {attempt}: "
+                        f"transport error — {exc}"
                     )
-                    break  # move to next model
+                    break
 
         raise OpenRouterError(
-            f"All OpenRouter free-model fallbacks exhausted. "
-            f"Tried: {sorted(tried)}"
+            f"OpenRouter chain exhausted. Tried={tried[:5]}; failures={failures[:5]}"
         )
 
 

app/services/tech_stack_service.py

@@ -1,7 +1,14 @@
-"""Tech-stack inference service — calls OpenRouter and returns validated model."""
+"""Tech-stack inference service — calls OpenRouter and returns validated model.
+
+If every upstream model is rate-limited or unreachable, we DO NOT hang and
+DO NOT bubble a 502 to the frontend. Instead we fall back to a deterministic
+"starter" tech-stack derived from keyword heuristics so the user gets
+actionable output immediately, plus a clear flag that says it's a stub.
+"""
 
 from __future__ import annotations
 
+import re
 import uuid
 from datetime import datetime, timezone
 
@@ -13,13 +20,86 @@ from app.models.tech_stack import (
     InferredWorkResponse,
     TechStack,
     TechStackPayload,
+    TechStep,
 )
 from app.services.openrouter import OpenRouterError, get_openrouter_client
 
 
+# Keyword-driven stub mapping. When the live OpenRouter call fails the user
+# still gets a meaningful, expandable baseline so the UI never dead-ends.
+_KEYWORD_STEPS: list[tuple[re.Pattern[str], tuple[str, str]]] = [
+    (re.compile(r"водопониж", re.I),
+     ("Бурение скважин под водопонижение",
+      "Бурение шурфов / водопонижающих скважин в осях, указанных заказчиком.")),
+    (re.compile(r"фундамент", re.I),
+     ("Устройство фундамента",
+      "Земляные работы, устройство опалубки, армирование, бетонирование.")),
+    (re.compile(r"плит", re.I),
+     ("Устройство плиты перекрытия",
+      "Монтаж опалубки, армокаркас, бетонирование с виброуплотнением, уход.")),
+    (re.compile(r"кров", re.I),
+     ("Устройство кровли",
+      "Монтаж стропильной системы / пирога, гидроизоляция, контроль уклонов.")),
+    (re.compile(r"стен", re.I),
+     ("Возведение стен",
+      "Кладка / монтаж элементов, утепление, контроль вертикали.")),
+    (re.compile(r"бетон", re.I),
+     ("Бетонирование конструкции",
+      "Армирование, монтаж опалубки, укладка бетонной смеси, уход.")),
+]
+
+
+def _fallback_payload(payload: DailyWorkInput, reason: str) -> TechStack:
+    """Deterministic minimal tech-stack based on keyword matching.
+
+    Ensures the response shape is contract-correct for the frontend even
+    when all OpenRouter models are rate-limited. Marked with a hint so the
+    UI can warn the user that AI enrichment is pending.
+    """
+    steps: list[TechStep] = []
+    for pat, (title, description) in _KEYWORD_STEPS:
+        if pat.search(payload.raw_text):
+            steps.append(TechStep(
+                order=len(steps) + 1,
+                title=title,
+                description=description,
+                required_acts=["АОСР", "Общий журнал работ"],
+                weather_sensitive=True,
+            ))
+    if not steps:
+        steps.append(TechStep(
+            order=1,
+            title="Базовый тех-стек по описанию пользователя",
+            description=(
+                "ИИ-модели временно недоступны; приведена минимальная структура."
+            ),
+            required_acts=["Общий журнал работ"],
+        ))
+
+    inferred_summary = (
+        f"Распознан базовый набор этапов из описания «{payload.raw_text}». "
+        f"AI-обогащение недоступно ({reason}); уточните параметры для детализации."
+    )
+
+    return TechStackPayload(
+        inferred_summary=inferred_summary,
+        steps=steps,
+        required_documents=["Общий журнал работ"],
+        missing_info_prompts=[
+            "Уточните точные оси / отметки / объёмы работ.",
+            "Подтвердите температурный режим работ (если наружные).",
+            "При наличии — пришлите рабочий проект / чертёж PDF.",
+        ],
+    )
+
+
 async def infer_tech_stack(payload: DailyWorkInput) -> InferredWorkResponse:
-    """Run inference on user's daily-work input and return structured tech-stack."""
-    client = get_openrouter_client()
+    """Run inference on user's daily-work input and return structured tech-stack.
+
+    Behaviour:
+      1. Try the OpenRouter chain (5 models, hard 25-s budget).
+      2. On total failure → deterministic stub so the UI never hangs.
+    """
     system_prompt, user_prompt = build_tech_stack_prompt(
         raw_text=payload.raw_text,
         date_from=payload.date_from.isoformat(),
@@ -27,16 +107,20 @@ async def infer_tech_stack(payload: DailyWorkInput) -> InferredWorkResponse:
         axes=payload.axes,
     )
 
+    parsed: TechStackPayload | None = None
+    model_used = "fallback:keyword-stub"
     try:
-        # Validate against the LLM-facing payload schema (no envelope fields).
+        client = get_openrouter_client()
         parsed, model_used = await client.structured_completion(
             schema=TechStackPayload,
             system_prompt=system_prompt,
             user_prompt=user_prompt,
         )
     except OpenRouterError as exc:
-        logger.error(f"tech-stack inference failed: {exc}")
-        raise
+        logger.warning(
+            f"[infer] chain exhausted → switching to keyword stub ({exc})"
+        )
+        parsed = _fallback_payload(payload, reason="OpenRouter upstream rate-limited")
 
     tech_stack = TechStack(
         source_input=payload.raw_text,

tests/test_stub_fallback.py

@@ -0,0 +1,58 @@
+"""Test that the inference service falls back to a stub when OpenRouter fails."""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import sys
+from datetime import date
+
+from app.models.tech_stack import DailyWorkInput
+from app.services.tech_stack_service import infer_tech_stack
+
+
+async def _run() -> int:
+    print("=" * 60)
+    print("  Stub fallback test (forces OpenRouter failure)")
+    print("=" * 60)
+
+    # Force-fail OpenRouter by pointing at a black hole URL.
+    from app.core.config import get_settings
+    settings = get_settings()
+    real_base = settings.openrouter_base_url
+    settings.openrouter_base_url = "http://127.0.0.1:1"  # always fails
+    print(f"Temporarily set OPENROUTER_BASE_URL = {settings.openrouter_base_url}")
+
+    sample = DailyWorkInput(
+        raw_text="Залита плита перекрытия в осях А-В / 1-3",
+        date_from=date(2026, 5, 1),
+        date_to=date(2026, 5, 15),
+        axes="А-В / 1-3",
+    )
+
+    try:
+        response = await infer_tech_stack(sample)
+    finally:
+        settings.openrouter_base_url = real_base
+
+    print(f"\n[ok] stub response received")
+    print(f"  request_id     : {response.request_id}")
+    print(f"  model_used     : {response.tech_stack.model_used}")
+    print(f"  inferred_summary: {response.tech_stack.inferred_summary}")
+    print(f"  steps          : {len(response.tech_stack.steps)}")
+    for s in response.tech_stack.steps:
+        print(f"    - {s.title}")
+
+    out_path = "tests/.last_stub.json"
+    with open(out_path, "w", encoding="utf-8") as f:
+        json.dump(response.model_dump(mode="json"), f, ensure_ascii=False, indent=2)
+    print(f"\nFull JSON → {out_path}")
+
+    assert "stub" in response.tech_stack.model_used, "expected stub marker"
+    assert len(response.tech_stack.steps) >= 1, "expected fallback steps"
+    print("\n✅ PASS — stub fallback returns valid response")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(asyncio.run(_run()))