Spaces:
Running
Running
Commit ·
53d0a86
0
Parent(s):
Decision 2.0 Studio: open foundation decision models, archived (replays recorded results; see Decision 3.0)
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- .dockerignore +27 -0
- .gitattributes +45 -0
- .gitignore +4 -0
- DEPLOYMENT.md +219 -0
- Dockerfile +16 -0
- LICENSE +21 -0
- MODEL_RELEASES.json +95 -0
- README.md +57 -0
- SYSTEMONE_API.md +121 -0
- SYSTEM_ONE_MAPPING.md +51 -0
- app.py +655 -0
- archive_cache.py +85 -0
- cache/arena.json +0 -0
- cache/studio.json +0 -0
- contract.py +221 -0
- deploy/decision2/RUNBOOK.md +147 -0
- deploy/decision2/check_runtime.py +103 -0
- deploy/decision2/compose.yaml +151 -0
- deploy/decision2/e2e_check.py +169 -0
- deploy/decision2/fla_kernel_choices.py +70 -0
- deploy/decision2/registry.json +128 -0
- deploy/edge/Caddyfile +72 -0
- deploy/edge/verify_adapted.py +193 -0
- deploy/runtime/Dockerfile +31 -0
- deploy/runtime/fetch_pins.py +47 -0
- deploy/worker/Dockerfile +10 -0
- deploy/worker/compose.yaml +59 -0
- deploy/worker/requirements.txt +1 -0
- direct_contract.py +248 -0
- direct_gateway.py +528 -0
- engine.py +84 -0
- examples.json +0 -0
- http_pull_worker.py +183 -0
- model_registry.py +145 -0
- model_runtime_client.py +410 -0
- pull_worker.py +173 -0
- relay.py +318 -0
- requirements.txt +3 -0
- static/app.js +438 -0
- static/assets/decision-studio-card.png +3 -0
- static/assets/fonts/Geist-Variable.woff2 +3 -0
- static/assets/fonts/GeistMono-Variable.woff2 +3 -0
- static/assets/fonts/Inter-OFL.txt +92 -0
- static/assets/fonts/InterVariable.woff2 +3 -0
- static/assets/fonts/OFL-Geist.txt +92 -0
- static/assets/hero-interstellar.png +3 -0
- static/assets/horizon-1080.webp +3 -0
- static/assets/horizon.webp +3 -0
- static/assets/social-banner.png +3 -0
- static/assets/vllm-sr-logo.svg +1 -0
.dockerignore
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
**
|
| 2 |
+
!Dockerfile
|
| 3 |
+
!requirements.txt
|
| 4 |
+
!app.py
|
| 5 |
+
!archive_cache.py
|
| 6 |
+
!cache/
|
| 7 |
+
!cache/**
|
| 8 |
+
!systemone_api.py
|
| 9 |
+
!contract.py
|
| 10 |
+
!engine.py
|
| 11 |
+
!relay.py
|
| 12 |
+
!model_registry.py
|
| 13 |
+
!direct_contract.py
|
| 14 |
+
!direct_gateway.py
|
| 15 |
+
!tetris_arena.py
|
| 16 |
+
!examples.json
|
| 17 |
+
!MODEL_RELEASES.json
|
| 18 |
+
!static/
|
| 19 |
+
!static/**
|
| 20 |
+
!LICENSE
|
| 21 |
+
!http_pull_worker.py
|
| 22 |
+
!model_runtime_client.py
|
| 23 |
+
!pull_worker.py
|
| 24 |
+
!deploy/
|
| 25 |
+
!deploy/worker/
|
| 26 |
+
!deploy/worker/Dockerfile
|
| 27 |
+
!deploy/worker/requirements.txt
|
.gitattributes
ADDED
|
@@ -0,0 +1,45 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
*.7z filter=lfs diff=lfs merge=lfs -text
|
| 2 |
+
*.arrow filter=lfs diff=lfs merge=lfs -text
|
| 3 |
+
*.bin filter=lfs diff=lfs merge=lfs -text
|
| 4 |
+
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
| 5 |
+
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
| 6 |
+
*.ftz filter=lfs diff=lfs merge=lfs -text
|
| 7 |
+
*.gz filter=lfs diff=lfs merge=lfs -text
|
| 8 |
+
*.h5 filter=lfs diff=lfs merge=lfs -text
|
| 9 |
+
*.joblib filter=lfs diff=lfs merge=lfs -text
|
| 10 |
+
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
| 11 |
+
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
| 12 |
+
*.model filter=lfs diff=lfs merge=lfs -text
|
| 13 |
+
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
| 14 |
+
*.npy filter=lfs diff=lfs merge=lfs -text
|
| 15 |
+
*.npz filter=lfs diff=lfs merge=lfs -text
|
| 16 |
+
*.onnx filter=lfs diff=lfs merge=lfs -text
|
| 17 |
+
*.ot filter=lfs diff=lfs merge=lfs -text
|
| 18 |
+
*.parquet filter=lfs diff=lfs merge=lfs -text
|
| 19 |
+
*.pb filter=lfs diff=lfs merge=lfs -text
|
| 20 |
+
*.pickle filter=lfs diff=lfs merge=lfs -text
|
| 21 |
+
*.pkl filter=lfs diff=lfs merge=lfs -text
|
| 22 |
+
*.pt filter=lfs diff=lfs merge=lfs -text
|
| 23 |
+
*.pth filter=lfs diff=lfs merge=lfs -text
|
| 24 |
+
*.rar filter=lfs diff=lfs merge=lfs -text
|
| 25 |
+
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
| 26 |
+
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
| 27 |
+
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
| 28 |
+
*.tar filter=lfs diff=lfs merge=lfs -text
|
| 29 |
+
*.tflite filter=lfs diff=lfs merge=lfs -text
|
| 30 |
+
*.tgz filter=lfs diff=lfs merge=lfs -text
|
| 31 |
+
*.wasm filter=lfs diff=lfs merge=lfs -text
|
| 32 |
+
*.xz filter=lfs diff=lfs merge=lfs -text
|
| 33 |
+
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
+
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
+
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
+
static/assets/decision-studio-card.png filter=lfs diff=lfs merge=lfs -text
|
| 37 |
+
static/assets/fonts/InterVariable.woff2 filter=lfs diff=lfs merge=lfs -text
|
| 38 |
+
|
| 39 |
+
static/assets/horizon.webp filter=lfs diff=lfs merge=lfs -text
|
| 40 |
+
static/assets/horizon-1080.webp filter=lfs diff=lfs merge=lfs -text
|
| 41 |
+
static/assets/fonts/Geist-Variable.woff2 filter=lfs diff=lfs merge=lfs -text
|
| 42 |
+
static/assets/fonts/GeistMono-Variable.woff2 filter=lfs diff=lfs merge=lfs -text
|
| 43 |
+
|
| 44 |
+
static/assets/hero-interstellar.png filter=lfs diff=lfs merge=lfs -text
|
| 45 |
+
static/assets/social-banner.png filter=lfs diff=lfs merge=lfs -text
|
.gitignore
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
__pycache__/
|
| 2 |
+
*.py[cod]
|
| 3 |
+
.ruff_cache/
|
| 4 |
+
.pytest_cache/
|
DEPLOYMENT.md
ADDED
|
@@ -0,0 +1,219 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Decision Gateway deployment
|
| 2 |
+
|
| 3 |
+
## Direct serving mode
|
| 4 |
+
|
| 5 |
+
`DECISION_BACKEND=direct` is the production direct-routing mode. The Space keeps no model weights: it validates the public request, resolves its exact canonical Hugging Face model ID, and sends it to that model's private `vllm-sr decision serve` origin. Both public inference routes keep the same path end to end:
|
| 6 |
+
|
| 7 |
+
- `POST /v1/systemone`
|
| 8 |
+
- `POST /v1/systemone/batches`
|
| 9 |
+
|
| 10 |
+
Public requests must include one exact canonical model ID advertised by `/v1/models`. The Studio-only `/api/evaluate` route requires a configured short wire ID or canonical ID; the Gateway converts it to the canonical ID before forwarding and returns the strict backend response unchanged. Direct mode advertises no default model in `/v1/models`; the Studio editor chooses its model locally and sends it explicitly. There is no default-model inference and no fallback to another model, the legacy native engine, or a queue worker.
|
| 11 |
+
|
| 12 |
+
Set the backend, the pinned `DECISION_MODEL_REGISTRY_V2`, and a server-only JSON map whose keys cover that registry exactly. Every direct-mode registry entry must include the full immutable 40-character Hub `revision` and 64-character `manifest_sha256`; it may also pin the runtime's 64-character `content_sha256`. These values come from each released artifact and may change through deployment configuration without a Gateway source release. The loopback example below works only when the Gateway and every Decision runtime listener share the host network namespace:
|
| 13 |
+
|
| 14 |
+
`/v1/models` reads presentation dates from the validated `MODEL_RELEASES.json` metadata shipped with the Gateway image. Its top-level `release` is the default date; a model row can set its own ISO `release_date`. A date is published only when that row's canonical ID, revision, and manifest digest match the active registry. An artifact with newer deployment pins still serves normally when the presentation file has not caught up; its date is omitted rather than reported incorrectly. If the file is missing or invalid, `/v1/models` still lists every configured model but omits all release dates; direct serving is unaffected. Operators can call `load_model_releases()` explicitly to fail on invalid metadata. Update the JSON metadata and rebuild the Gateway image when the public date should change. This file does not allowlist model files or decide which artifacts the Decision runtime may load.
|
| 15 |
+
|
| 16 |
+
```text
|
| 17 |
+
DECISION_BACKEND=direct
|
| 18 |
+
DECISION_DIRECT_ENDPOINTS={"vllm-sr/Decision-1.0-Kai-0.6B":"http://127.0.0.1:8101","vllm-sr/Decision-1.0-Lux-9B":"http://127.0.0.1:8106"}
|
| 19 |
+
DECISION_DIRECT_TIMEOUT_SECONDS=45
|
| 20 |
+
```
|
| 21 |
+
|
| 22 |
+
The example assumes a two-model registry; the endpoint map must include every model in the active registry, and every model must have a distinct origin. A Decision runtime instance binds to host loopback by default. `127.0.0.1` inside a separate Gateway or Tetris container is that container itself, **not** the model host, so loopback endpoints do not work across ordinary container namespaces. Use one of these explicit deployment boundaries:
|
| 23 |
+
|
| 24 |
+
- Run the Gateway with host networking on the same trusted Linux host, keep every Decision runtime listener on host loopback, and retain the loopback map above.
|
| 25 |
+
- Attach the Gateway and model containers to a dedicated private network, bind each Decision runtime listener intentionally to its assigned private interface, and admit only the Gateway with host firewall/container-network policy. Never publish those ports on a public interface.
|
| 26 |
+
- For a remote Space, establish an authenticated private overlay or tunnel first and use its private IP literals. Direct mode is not ready if the Space can reach the models only through public origins.
|
| 27 |
+
|
| 28 |
+
For a private-network deployment, keep the same JSON map structure and replace
|
| 29 |
+
each loopback origin with the private IP origin assigned to that model in your
|
| 30 |
+
environment. Configure the active registry with the same model IDs. Store the
|
| 31 |
+
map only in the server-side secret setting; no private-network addresses belong
|
| 32 |
+
in this repository.
|
| 33 |
+
|
| 34 |
+
Store `DECISION_DIRECT_ENDPOINTS` as a private Space secret or equivalent server-only setting because it describes private topology. Values must be bare `http` or `https` origins with an explicit port and a literal loopback, RFC 1918, or IPv6 unique-local address. DNS names, public addresses, link-local addresses, URL credentials, paths, queries, and fragments fail startup. `DECISION_DIRECT_TIMEOUT_SECONDS` is bounded to 0.1–300 seconds.
|
| 35 |
+
|
| 36 |
+
The direct transport ignores ambient proxy variables, does not follow redirects, requests identity encoding, reuses private keep-alive connections, bounds single requests at 256 KiB, batch requests at 2 MiB, and inference responses at 8 MiB. Bind each Decision runtime service to loopback or enforce a network policy that admits only the Gateway; direct mode intentionally does not forward browser authorization. Neither backend origins nor private configuration are returned by `/api/status`, `/v1/models`, error responses, or browser configuration.
|
| 37 |
+
|
| 38 |
+
Each successful private inference response carries `X-Decision-Artifact-Model`, `X-Decision-Artifact-Revision`, `X-Decision-Artifact-Manifest-Sha256`, and `X-Decision-Artifact-Content-Sha256` headers from the resident Decision runtime process. The Gateway requires exactly one well-formed value for each, matches the canonical model, pinned Hub revision and manifest digest, and matches the content digest when configured. Missing, duplicate, malformed, or mismatched identity discards that response. It also verifies the exact requested model, strict official JSON field set, answer IDs and types, state order, probability distributions, Choice winner, Score legend/value, type-aware Decision confidence, and batch usage aggregation. Confidence is accepted only from the validated runtime response and is never replaced with `max(probabilities)`. Explicit `/api/status?model=<wire-id>` probes report readiness plus the configured and live artifact digests without disclosing private origins. A Gateway using response attestation requires the matching Decision runtime version; older runtimes fail closed.
|
| 39 |
+
|
| 40 |
+
`GET /api/ready` probes every configured model and returns HTTP 200 only when all are loaded with the pinned artifact; otherwise it returns HTTP 503 with public model IDs and Boolean readiness, never private origins. The Docker health check uses this aggregate route in direct mode, while legacy queue deployments keep their existing `/api/status` health check. The Studio menu reports an unavailable model distinctly from one still loading. Tetris configuration probes live Decision readiness before offering a local competitor, and a race start rechecks each selected local model so a disconnected instance returns 503 before the runners start. These control probes occur at page load and race admission, not on every game turn.
|
| 41 |
+
|
| 42 |
+
For rollout, start and warm every pinned Decision runtime instance first, verify reachability **from the Gateway network namespace**, and verify that each private origin's live artifact report matches its assigned model and deployment pins. Then switch the Space to `DECISION_BACKEND=direct`. Keep the prior queue workers and their exact settings available but inactive until direct-mode probes and representative single/batch calls pass. A response identity mismatch is a deployment failure, never a reason to replay against another backend.
|
| 43 |
+
|
| 44 |
+
The public reverse proxy must route `/tetris/` and `/api/tetris/*` to the same pinned Gateway release. Preserve the `/tetris` path when forwarding: the new Gateway serves the page at `/tetris/`, while an older standalone Tetris service used a stripped prefix. Keep `/v1/systemone` bounded at 256 KiB and admit up to 2 MiB at `/v1/systemone/batches`; a site-wide 256 KiB body limit would reject valid batch traffic before it reaches this application. Align the proxy's response-header timeout with the selected synchronous runtime timeout. Validate the proxy configuration before reload, retain its exact prior version, and test both paths before retiring the old page.
|
| 45 |
+
|
| 46 |
+
### Versioned public edge configuration
|
| 47 |
+
|
| 48 |
+
[`deploy/edge/Caddyfile`](deploy/edge/Caddyfile) is the source-controlled Caddy 2.11.4 route template. It uses one Gateway proxy snippet for `/tetris/`, `/api/tetris/*`, batch requests, and the ordinary fallback. The Tetris path is forwarded unchanged; `/tetris` alone redirects to `/tetris/`. Only the exact `/v1/systemone/batches` path receives a 2 MiB body allowance. All other proxied paths, including `/v1/systemone` and other APIs, retain 256 KiB. The `/internal/*` block remains inaccessible from the public edge. No deployment origin or credential is committed here.
|
| 49 |
+
|
| 50 |
+
Before production rendering, pin a tested Gateway revision and prepare the edge's private deployment settings `DECISION_EDGE_SITE` and `DECISION_EDGE_GATEWAY_ORIGIN`. The latter is one bare HTTP(S) origin reachable from the edge, with no path, query, or credentials. Supply those values to the Caddy process on **both reload and restart**; setting variables only for a one-off validation command is insufficient. Keep the exact current mounted Caddyfile, edge image identity, and previous Tetris service configuration in a restricted rollback copy outside this repository. Never infer the rollback target from the new template.
|
| 51 |
+
|
| 52 |
+
From a staging environment with the same Caddy release and those two variables exported, validate the candidate without reloading the live server:
|
| 53 |
+
|
| 54 |
+
```bash
|
| 55 |
+
caddy adapt --config deploy/edge/Caddyfile --adapter caddyfile | python deploy/edge/verify_adapted.py
|
| 56 |
+
caddy validate --config deploy/edge/Caddyfile --adapter caddyfile
|
| 57 |
+
python -m unittest tests.test_edge_caddy
|
| 58 |
+
```
|
| 59 |
+
|
| 60 |
+
The verifier inspects Caddy's adapted route tree, not only the template text: it requires one mutually exclusive routing group, an unmodified Tetris path, one common upstream, the two exact body limits, and the expected slash redirect. The unit test runs the adapter itself when `caddy` is installed and always runs static and mutation checks. Validation with placeholder values does **not** establish that the selected production Gateway is reachable; probe it from the edge network namespace before cutover. The template's 65-second upstream response-header timeout must remain greater than the deployed synchronous Gateway timeout; adjust and revalidate together if that timeout changes.
|
| 61 |
+
|
| 62 |
+
After the pinned Gateway passes its own six-model readiness and representative inference probes, stage this template as the edge's next mounted configuration, validate it again with the real private settings, and atomically reload Caddy. Check `/tetris/` and `/api/tetris/config` through the public host, then a race creation/stream, `/api/ready`, a canonical single-state call, and a valid batch. A 2 MiB malformed JSON request to `/v1/systemone/batches` should reach the application and return 422, while 2 MiB + 1 byte should return 413; a 256 KiB + 1 byte request to `/v1/systemone` must still return 413. These malformed probes perform no inference. Also confirm the same Gateway revision serves the page and race API. Keep the prior standalone Tetris service available but inactive until these checks pass.
|
| 63 |
+
|
| 64 |
+
If any check fails, restore the exact previous mounted Caddyfile and persistent Caddy environment, reload the previous edge configuration, then restore the previous pinned Gateway and Tetris service together. Recheck the prior `/tetris/`, `/api/status`, and representative inference behavior. If reverting to the old pull queue, restore its matching registry and Worker images as one unit; do not mix the old and new Kai wire IDs. Do not leave the 2 MiB exception enabled on unrelated paths after rollback.
|
| 65 |
+
|
| 66 |
+
One edge-limit smoke check uses a malformed 300 KiB JSON body, so it performs no inference. The batch route should return the application's 422 rather than an edge 413; the single route should remain limited to 256 KiB:
|
| 67 |
+
|
| 68 |
+
```bash
|
| 69 |
+
head -c 307200 /dev/zero | tr '\0' x | curl -sS -o /dev/null -w '%{http_code}\n' -H 'Content-Type: application/json' --data-binary @- 'https://YOUR-DECISION-HOST/v1/systemone/batches'
|
| 70 |
+
head -c 307200 /dev/zero | tr '\0' x | curl -sS -o /dev/null -w '%{http_code}\n' -H 'Content-Type: application/json' --data-binary @- 'https://YOUR-DECISION-HOST/v1/systemone'
|
| 71 |
+
```
|
| 72 |
+
|
| 73 |
+
Confirm `GET /api/ready` is 200 only after every configured model passes live artifact readiness, `GET /api/tetris/config` marks a disconnected local model unavailable, and the served `/tetris/` page and race API use the same release. A Cloud-only race does not depend on local model availability. Roll back the proxy, Gateway image/configuration, and old Tetris service together if the cutover fails; restore the previous queue registry and workers when reverting the Decision API.
|
| 74 |
+
|
| 75 |
+
Rollback is explicit and mode-level:
|
| 76 |
+
|
| 77 |
+
1. Restore the previous pinned Gateway image, `DECISION_BACKEND=pull_queue`, its matching registry and previous Worker image/configuration together; or restore the previous pinned native image and Kai package settings together. The Kai internal wire ID changed from `decision-nano-preview` to `decision-kai`: an old Worker cannot join a new queue, and a new Worker cannot join an old queue. Do not mix these generations or add an alias to bridge them.
|
| 78 |
+
2. Confirm `/api/status` and representative inference through that mode.
|
| 79 |
+
3. Remove the direct endpoint setting only after the old mode is healthy. Do not run two fallback modes inside one request.
|
| 80 |
+
|
| 81 |
+
## Decision 2.0 on the built-in model runtime
|
| 82 |
+
|
| 83 |
+
The hosted Space runs `DECISION_BACKEND=pull_queue`. Decision 2.0 models are
|
| 84 |
+
served by the vLLM Semantic Router built-in model runtime (`vllm-sr-runtime
|
| 85 |
+
serve`, from `src/model-runtime` on the router's `main` branch) on private AMD
|
| 86 |
+
GPU hosts, one runtime container per model, each with one outbound HTTP pull
|
| 87 |
+
worker started with `DECISION_RUNTIME_API=vllm_sr_runtime_v1`. That runtime
|
| 88 |
+
reports its artifact on `/v1/models` and in each response's `meta` rather than
|
| 89 |
+
in `X-Decision-Artifact-*` headers, and answers one state per request: the
|
| 90 |
+
worker checks the repository, revision, manifest and model identity on every
|
| 91 |
+
answer, requires the `exact` profile, strips `meta`, and turns a
|
| 92 |
+
shared-question batch into one runtime request per state. Decision 2.0 reports
|
| 93 |
+
`confidence` as one minus the normalized entropy (`normalized_entropy_v2`);
|
| 94 |
+
the Gateway validates that statistic for Decision 2.0 queues and the Decision
|
| 95 |
+
1.0 margin / ordinal statistic (`margin_ordinal_v1`) for Decision 1.0 queues.
|
| 96 |
+
|
| 97 |
+
The deployment files are in [`deploy/decision2/`](deploy/decision2/): the
|
| 98 |
+
Compose project, the pinned registry for `DECISION_MODEL_REGISTRY_V2`, the
|
| 99 |
+
reference-answer check and the operations runbook
|
| 100 |
+
([`deploy/decision2/RUNBOOK.md`](deploy/decision2/RUNBOOK.md)). The ROCm
|
| 101 |
+
runtime image is built by [`deploy/runtime/Dockerfile`](deploy/runtime/Dockerfile).
|
| 102 |
+
|
| 103 |
+
## Legacy pull-queue and native modes
|
| 104 |
+
|
| 105 |
+
### Outbound HTTP workers for private Decision runtimes
|
| 106 |
+
|
| 107 |
+
The pull-queue Gateway now accepts a distinct `http_runtime_v1` worker result. It
|
| 108 |
+
validates the returned strict SystemOne response against the queued request and
|
| 109 |
+
publishes the canonical response unchanged. Deploy this Gateway version before
|
| 110 |
+
starting HTTP pull workers; an older Gateway rejects their result messages.
|
| 111 |
+
The original native worker result format remains supported. Each HTTP worker
|
| 112 |
+
claims one queue and calls one already running private Decision runtime; it
|
| 113 |
+
does not load weights. A heartbeat is sent only while that runtime reports the
|
| 114 |
+
configured artifact as ready. Inference responses must carry the matching
|
| 115 |
+
artifact headers. A failed call is reported once, without retrying inference.
|
| 116 |
+
|
| 117 |
+
Build the CPU-only [`worker image`](deploy/worker/Dockerfile) from this checkout.
|
| 118 |
+
The [`Compose template`](deploy/worker/compose.yaml) starts six independently
|
| 119 |
+
restartable containers on an existing private runtime network and publishes no
|
| 120 |
+
ports. Its external network name and image tag come from a protected
|
| 121 |
+
`/etc/decision-studio/compose.env` file with keys
|
| 122 |
+
`DECISION_RUNTIME_NETWORK` and `DECISION_HTTP_WORKER_IMAGE`.
|
| 123 |
+
|
| 124 |
+
Keep `/etc/decision-studio/worker-common.env` restricted to administrators;
|
| 125 |
+
it contains `DECISION_STUDIO_URL` (the fixed HTTPS Space origin), the shared
|
| 126 |
+
`DECISION_WORKER_TOKEN`, and the exact six-model
|
| 127 |
+
`DECISION_MODEL_REGISTRY_V2` used by the Space. Each protected
|
| 128 |
+
`/etc/decision-studio/workers/<wire-id>.env` contains its private, literal-IP
|
| 129 |
+
`DECISION_RUNTIME_URL` with an explicit port. The Compose service fixes each
|
| 130 |
+
exact `DECISION_WORKER_MODEL` wire ID. Optional
|
| 131 |
+
`DECISION_RUNTIME_TIMEOUT_SECONDS` defaults to 40
|
| 132 |
+
and is limited to 0.1–90. The same files can include this setting for all
|
| 133 |
+
workers or override it per model. Keep credentials, pins, and private origins
|
| 134 |
+
out of shell arguments, source, image layers, and logs.
|
| 135 |
+
|
| 136 |
+
After placing the protected files, build and validate the container set. The
|
| 137 |
+
image tag in the first command must equal `DECISION_HTTP_WORKER_IMAGE` in the
|
| 138 |
+
protected Compose file:
|
| 139 |
+
|
| 140 |
+
```bash
|
| 141 |
+
docker build -f deploy/worker/Dockerfile -t decision-http-pull:release .
|
| 142 |
+
docker compose --env-file /etc/decision-studio/compose.env -f deploy/worker/compose.yaml config --quiet
|
| 143 |
+
docker compose --env-file /etc/decision-studio/compose.env -f deploy/worker/compose.yaml up -d
|
| 144 |
+
```
|
| 145 |
+
|
| 146 |
+
The Kai ID is `decision-kai`; the previous preview ID cannot join this queue.
|
| 147 |
+
Check `/api/status?model=<wire-id>` for all six `context_batch: true` workers,
|
| 148 |
+
then run representative canonical single and batch requests through the Space.
|
| 149 |
+
The Space worker endpoint must be reachable from these processes over outbound
|
| 150 |
+
HTTPS, and the private runtime origins must be reachable from the same host.
|
| 151 |
+
The Space still needs one process because queues are in memory.
|
| 152 |
+
|
| 153 |
+
For a hosted Space currently using the previous Kai wire ID, coordinate the
|
| 154 |
+
source and secret changes in one maintenance window:
|
| 155 |
+
|
| 156 |
+
1. Prepare the new Gateway source and the six protected per-model worker files.
|
| 157 |
+
Confirm each runtime reports the exact registry artifact and retain the
|
| 158 |
+
previous Gateway source and registry settings for a deliberate rollback.
|
| 159 |
+
2. Push the new Gateway main source and update the hosted
|
| 160 |
+
`DECISION_MODEL_REGISTRY_V2` to use `decision-kai` with the unchanged Kai
|
| 161 |
+
artifact pins. Rotate the hosted `DECISION_WORKER_TOKEN` and place that same
|
| 162 |
+
new token in the protected worker common environment file. Expect the
|
| 163 |
+
previous workers to lose authorization and a short offline interval while
|
| 164 |
+
the Space rebuilds; neither old nor new Kai IDs are accepted as aliases.
|
| 165 |
+
3. Once the new Space is healthy, start the six adapter containers above. Verify
|
| 166 |
+
each worker heartbeat, a single request, and a shared-question batch for
|
| 167 |
+
its canonical model. Retire any remaining old workers after confirming they
|
| 168 |
+
can no longer claim jobs.
|
| 169 |
+
|
| 170 |
+
The prior hosted token is not recoverable after rotation. Rolling back to old
|
| 171 |
+
workers requires assigning them a newly controlled token and setting that
|
| 172 |
+
same token on the restored Space; restoring only the old source and registry
|
| 173 |
+
would leave the workers unauthorized.
|
| 174 |
+
|
| 175 |
+
In hosted `pull_queue` mode, Tetris sends each turn to the selected model's
|
| 176 |
+
queue inside the gateway process, through the same validation, queue, and
|
| 177 |
+
response path as `/v1/systemone`, so every local competitor plays under its
|
| 178 |
+
canonical model ID. It needs no model URL and ignores `TETRIS_LOCAL_*` and
|
| 179 |
+
`LOCAL_API_URL`. `/api/tetris/config` uses each queue's live heartbeat to mark
|
| 180 |
+
disconnected local models unavailable, and race admission rechecks the selected
|
| 181 |
+
queues. Check that status and run a short local race after worker readiness.
|
| 182 |
+
|
| 183 |
+
The Docker Space is a CPU-only gateway: no weights or tokenization. One gateway process maintains one independent bounded queue per configured model, selected by an explicit model ID and pinned manifest. Keep one replica because queue state is in memory.
|
| 184 |
+
|
| 185 |
+
Internal queues use short wire IDs such as `decision-kai` for Kai. `/v1/models` exposes canonical Hugging Face repository IDs, actual sizes, complete-input limits, and per-model examples; `wire_id` is supplied for the Studio editor. `/api/status?model=…` reports the selected queue. Public `/v1/systemone` and `/v1/systemone/batches` calls must select a canonical model explicitly.
|
| 186 |
+
|
| 187 |
+
Configure `DECISION_BACKEND=pull_queue`, the private `DECISION_WORKER_TOKEN` secret, and `DECISION_MODEL_REGISTRY_V2`. Each registry item includes `id`, `label`, `version`, `manifest_sha256`, `complete_input_tokens` and optional presentation/example metadata. The old `DECISION_MODEL_REGISTRY` remains a fallback for legacy two-model deployments. The newer variable allows existing code to ignore it safely during a coordinated upgrade.
|
| 188 |
+
|
| 189 |
+
The registry accepts a newly built artifact for any canonical Decision model ID in `model_registry.PROFILES` without a code-level manifest allowlist. Pin its actual manifest SHA-256 and, when supplied, the full immutable Hub revision in the deployment configuration, then verify the worker/runtime loaded that artifact. A changed model file needs a new deployment receipt and rollback value, not a Gateway source release. The configured manifest remains an exact worker-to-queue identity check; it is not permission to skip artifact integrity or compatibility checks in the serving runtime.
|
| 190 |
+
|
| 191 |
+
Each resident AMD worker binds exactly one model and manifest and makes outbound HTTPS heartbeat/claim/result calls. Private worker addresses are never exposed. Kai/Lex retain their native FP32 runtime and complete1K admission. Sol/Nox use their isolated published BF16-backbone/FP32-head runtime and complete16K admission. The selected model checks all questions before inference and rejects overflow without truncation. Physical batches contain at most eight questions.
|
| 192 |
+
|
| 193 |
+
A leased request is evaluated once. Identical result delivery may be retried; inference is not repeated. Worker IDs, model IDs, manifests and leases are checked independently. Switching detaches/cancels the previous job and clears old results; an in-flight old result cannot render under the new model. Decoders sharing a device serialize their loading and forwards through an owner-only lock while keeping independent queues.
|
| 194 |
+
|
| 195 |
+
For upgrades, verify idle queues and preserve exact previous sources and registry values for rollback. Keep compatible residents running across an ordinary Gateway rebuild; a temporary offline transition is possible. The Kai wire-ID migration is not an ordinary compatible rebuild: deploy its new registry and new Workers with the new Gateway as one release boundary. A previous `decision-nano-preview` Worker must be stopped before admitting the `decision-kai` queue; the old Worker receives 422 at the new Worker endpoint. Reverse the entire image, registry, and Worker set together if rollback is needed. New residents warm the selected model before advertising readiness. Never put worker credentials in command arguments, source, images, public variables, logs or browser responses.
|
| 196 |
+
|
| 197 |
+
## Shared-question context batches
|
| 198 |
+
|
| 199 |
+
The public `/v1/systemone` endpoint accepts one `state`; `/v1/systemone/batches` accepts ordered `states` with shared questions. Batch support is negotiated by authenticated worker heartbeat (`context_batch_v1`); `/api/status` exposes `context_batch`. A legacy worker continues serving single-context requests and is never given a batch it cannot handle.
|
| 200 |
+
|
| 201 |
+
Deploy a compatible gateway first. Upgrade one idle worker at a time, retaining its exact released manifest and runtime. Stop the old process explicitly and let its heartbeat expire before the replacement claims that model. Keep decoder workers under their existing shared GPU lock. Enable the new batch interface after each registered model reports the capability. A model mismatch, expired lease or missing result is an error, never a trigger for fallback inference or automatic replay.
|
| 202 |
+
|
| 203 |
+
Public single-state request bodies remain bounded at 256 KiB; public batch bodies are bounded at 2 MiB, with at most 1,024 state/question decisions. Runtime inference responses remain bounded at 8 MiB; the internal worker result route allows another 64 KiB for the worker/lease envelope. Other internal messages retain their prior bound. Credentials remain exclusively in the existing private worker secret mount and Space secret. No model weights are shipped in this Space.
|
| 204 |
+
|
| 205 |
+
## Tetris Race
|
| 206 |
+
|
| 207 |
+
`/tetris/` is a passive SSE client for server-authoritative races. Starting a race creates two independent asynchronous runners immediately; connecting, reconnecting, or slowing the event stream does not pause either model. A disconnect starts a bounded reconnection lease, after which an abandoned race is cancelled. Only the seeded tetromino sequence is shared. The server gives both models the same shortlist of up to five placements with next-piece lookahead. An invalid choice string or a clearly hazardous choice is transparently replaced with the top-ranked safe placement; `side.step` events and traces record `requested_choice`, `applied_choice`, and `override_reason`, while side results and summaries count `safety_overrides`. A missing or non-string choice, or an upstream error, still ends that side. The server retains separate board snapshots, request states, model actions, and traces at `/api/tetris/races/{id}/trace`. The browser paints each side from its own bounded step queue; a short 40-step burst keeps every step, while a longer burst may skip intermediate visual frames without changing the server race, final board, or result. The server records side completion order in the SSE result and snapshot so recovery preserves the real finishing order.
|
| 208 |
+
|
| 209 |
+
All endpoints and credentials are server-only Space variables or secrets. In `DECISION_BACKEND=direct` mode, Tetris selects local models from the same private `DECISION_DIRECT_ENDPOINTS` map and sends each turn through the Gateway's attested direct client. It ignores legacy `TETRIS_LOCAL_*_API_URL` and local bearer variables, and so does `pull_queue` mode, which uses the queues as described above. In native mode, configure the shared Decision gateway with `TETRIS_LOCAL_API_URL` pointing to its `/v1/systemone` route, or configure per-model `TETRIS_LOCAL_LUX_API_URL`, `TETRIS_LOCAL_NOX_API_URL`, `TETRIS_LOCAL_SOL_API_URL`, `TETRIS_LOCAL_EOS_API_URL`, `TETRIS_LOCAL_KAI_API_URL`, and `TETRIS_LOCAL_LEX_API_URL`; optional local bearer secrets use the corresponding `_API_KEY` name or the shared `TETRIS_LOCAL_API_KEY`. A Decision response must return the exact Hugging Face canonical model ID selected by the player, such as `vllm-sr/Decision-2.0-Lux-9B`; a different or missing identity ends that side instead of being displayed under the wrong model name. This strict response identity is a deployment prerequisite. Legacy wire IDs such as `decision-lux` are not compatibility aliases here and deliberately fail closed during integration with an outdated Gateway.
|
| 210 |
+
|
| 211 |
+
Configure Jev Cloud with `TETRIS_JEV_API_URL`, the `TETRIS_JEV_API_KEY` secret, and optional `TETRIS_JEV_MODEL`. Configure the alternate SystemOne provider with `TETRIS_SYSTEMONE_API_URL`, the `TETRIS_SYSTEMONE_API_KEY` secret, and optional `TETRIS_SYSTEMONE_MODEL`. Existing `JEV_API_*` and `JEV_MIRROR_API_*` variables, and in native mode `LOCAL_API_URL`, remain accepted while the deployment migrates. The `jev-latest` alias accepts only itself or a versioned `jev-x.y.z` response; an explicitly pinned Jev model must match exactly. `/api/tetris/config` exposes only readiness and display metadata; it never returns endpoint URLs or credentials.
|
| 212 |
+
|
| 213 |
+
Step races accept 1 through `TETRIS_MAX_STEPS` pieces. The default server limit is 500, and the configured limit can be raised to 2,000; `/api/tetris/config` supplies the active limit to the browser. The browser initially selects 120 steps. `first_failure` has no user-visible step target and stops after the first game-over or model failure. Its operational guard defaults to the configured step limit and can be lowered with `TETRIS_FIRST_FAILURE_GUARD`. Each race makes at most two provider calls per allowed step. Retained events and traces are bounded by that race limit; completed races expire or are evicted from the in-memory cache.
|
| 214 |
+
|
| 215 |
+
Tetris imposes no per-visitor start frequency, call window, or paid Cloud call budget. `TETRIS_MAX_CONCURRENT_RACES` controls the number of simultaneous races in one Gateway process (default 4, maximum 32); starting another race at capacity returns HTTP 503 until a slot is free. Provider runtimes and Cloud APIs can still enforce their own capacity or rate limits. Configure paid Cloud credentials and upstream limits with the possible call volume in mind.
|
| 216 |
+
|
| 217 |
+
`TETRIS_RACE_TIMEOUT_SECONDS` sets a 1–3,600 second whole-race deadline (default 900); raise it deliberately for slow long runs. `TETRIS_STREAM_GRACE_SECONDS` sets the 1–300 second reconnection lease after the last event stream disconnects (default 30); reconnecting resumes retained SSE events, and the browser can recover the latest board and terminal result from the race snapshot. An abandoned race is cancelled. `TETRIS_COMPLETED_TTL_SECONDS` controls completed-race retention from 30–3,600 seconds (default 300). Shutdown cancels and awaits every active race before closing the provider transport. A failed, timed-out, or otherwise incomplete race does not award a score winner. The speed percentage is available only when both sides complete the same step target; it compares the sums of complete server-observed adapter calls. For a private Decision runtime, that includes its inference POST, response attestation, and response validation. Cloud competitors include the provider POST, response read, and response validation. Local `inference_ms`, when returned, is retained per turn in the trace for diagnosis and never enters that comparison. Whole-run durations remain in the race summary as diagnostics; browser rendering delay does not enter the speed percentage.
|
| 218 |
+
|
| 219 |
+
`TETRIS_REQUEST_TIMEOUT_SECONDS` bounds each server-to-provider call to 0.1–300 seconds (default 45). The adapter sends optional bearer credentials only from server configuration, rejects redirects, ignores ambient proxy settings, bounds response bytes, and never returns provider URLs or credentials to the browser. Validate each configured endpoint with its selected canonical model and expected response identity before enabling that competitor; publishing a Space commit rebuilds the live Space.
|
Dockerfile
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# CPU gateway only: no Torch, model weights, native package, or AMD ingress.
|
| 2 |
+
FROM python:3.12-slim
|
| 3 |
+
ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 PIP_NO_CACHE_DIR=1 \
|
| 4 |
+
DECISION_BACKEND=cache DECISION_STUDIO_GENERATION=2.0
|
| 5 |
+
RUN useradd --create-home --uid 1000 studio
|
| 6 |
+
WORKDIR /app
|
| 7 |
+
COPY requirements.txt /app/requirements.txt
|
| 8 |
+
RUN pip install -r /app/requirements.txt
|
| 9 |
+
COPY --chown=studio:studio app.py archive_cache.py systemone_api.py contract.py engine.py relay.py model_registry.py direct_contract.py direct_gateway.py tetris_arena.py examples.json MODEL_RELEASES.json LICENSE /app/
|
| 10 |
+
COPY --chown=studio:studio static /app/static
|
| 11 |
+
COPY --chown=studio:studio cache /app/cache
|
| 12 |
+
USER studio
|
| 13 |
+
EXPOSE 7860
|
| 14 |
+
HEALTHCHECK --interval=30s --timeout=5s --start-period=20s CMD python -c "import os, urllib.request; path = '/api/ready' if os.getenv('DECISION_BACKEND') == 'direct' else '/api/status'; urllib.request.urlopen('http://127.0.0.1:7860' + path, timeout=3).close()"
|
| 15 |
+
# One process preserves the legacy in-memory queue/lease rollback mode.
|
| 16 |
+
CMD ["python", "-m", "uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860", "--workers", "1", "--no-access-log"]
|
LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
MIT License
|
| 2 |
+
|
| 3 |
+
Copyright (c) 2026 Decision Nano contributors
|
| 4 |
+
|
| 5 |
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
| 6 |
+
of this software and associated documentation files (the "Software"), to deal
|
| 7 |
+
in the Software without restriction, including without limitation the rights
|
| 8 |
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
| 9 |
+
copies of the Software, and to permit persons to whom the Software is
|
| 10 |
+
furnished to do so, subject to the following conditions:
|
| 11 |
+
|
| 12 |
+
The above copyright notice and this permission notice shall be included in all
|
| 13 |
+
copies or substantial portions of the Software.
|
| 14 |
+
|
| 15 |
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
| 16 |
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
| 17 |
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
| 18 |
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
| 19 |
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
| 20 |
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
| 21 |
+
SOFTWARE.
|
MODEL_RELEASES.json
ADDED
|
@@ -0,0 +1,95 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"release": "2026-09-22",
|
| 3 |
+
"models": [
|
| 4 |
+
{
|
| 5 |
+
"label": "Vega",
|
| 6 |
+
"repo_id": "vllm-sr/Decision-2.0-Vega-27B",
|
| 7 |
+
"revision": "7aec49ae11a18741706da549ab626b9052795fe7",
|
| 8 |
+
"manifest_sha256": "24dcf251f5b50a2259201cc5e11889b6d2545afd4ccecabb0e2344549542bdd3",
|
| 9 |
+
"complete_input_tokens": 32768,
|
| 10 |
+
"release_date": "2026-10-03"
|
| 11 |
+
},
|
| 12 |
+
{
|
| 13 |
+
"label": "Lux",
|
| 14 |
+
"repo_id": "vllm-sr/Decision-2.0-Lux-9B",
|
| 15 |
+
"revision": "78bf3c03d9147aeb30b641edfe0e30ed04887ca5",
|
| 16 |
+
"manifest_sha256": "4e0f4ca0cf3c833933f9bed3021012d061910016c9f4ca43b34f5d66e8d0a260",
|
| 17 |
+
"complete_input_tokens": 16384,
|
| 18 |
+
"release_date": "2026-10-03"
|
| 19 |
+
},
|
| 20 |
+
{
|
| 21 |
+
"label": "Nox",
|
| 22 |
+
"repo_id": "vllm-sr/Decision-2.0-Nox-4B",
|
| 23 |
+
"revision": "25e8f67d1b486c647222df3aac640d2d5d736bbe",
|
| 24 |
+
"manifest_sha256": "49771ea33a451274687ad2a246ce5fb78f8f42ec904fc91a467477179975bb16",
|
| 25 |
+
"complete_input_tokens": 16384,
|
| 26 |
+
"release_date": "2026-10-03"
|
| 27 |
+
},
|
| 28 |
+
{
|
| 29 |
+
"label": "Sol",
|
| 30 |
+
"repo_id": "vllm-sr/Decision-2.0-Sol-2B",
|
| 31 |
+
"revision": "64235bef55dad29387dd16da7c90e038bf2f0972",
|
| 32 |
+
"manifest_sha256": "a91c440ca371668d488a788bdfe12cc574a97123df60730afedf419737ced970",
|
| 33 |
+
"complete_input_tokens": 16384,
|
| 34 |
+
"release_date": "2026-10-03"
|
| 35 |
+
},
|
| 36 |
+
{
|
| 37 |
+
"label": "Eos",
|
| 38 |
+
"repo_id": "vllm-sr/Decision-2.0-Eos-0.8B",
|
| 39 |
+
"revision": "3594047d69f476f1d01cf84c593e213fc3a4dfe0",
|
| 40 |
+
"manifest_sha256": "e8b1081be4a76deca5247792a4031c8e19e2b52b95d91775c13d9e257c407101",
|
| 41 |
+
"complete_input_tokens": 16384,
|
| 42 |
+
"release_date": "2026-10-03"
|
| 43 |
+
},
|
| 44 |
+
{
|
| 45 |
+
"label": "Kai",
|
| 46 |
+
"repo_id": "vllm-sr/Decision-2.0-Kai-0.6B",
|
| 47 |
+
"revision": "cd49ea3813fd8ba0928a9a23ef6c9a0f2f0cd764",
|
| 48 |
+
"manifest_sha256": "ca7bf2cff1494a5bfc51f4271f52557da0f3ce3b6f0f6e7a2b4118cb4965e92e",
|
| 49 |
+
"complete_input_tokens": 8192,
|
| 50 |
+
"release_date": "2026-10-03"
|
| 51 |
+
},
|
| 52 |
+
{
|
| 53 |
+
"label": "Kai",
|
| 54 |
+
"repo_id": "vllm-sr/Decision-1.0-Kai-0.6B",
|
| 55 |
+
"revision": "7185f514f54b8f93c55998b1e8f9c5cc67f0d029",
|
| 56 |
+
"manifest_sha256": "c1bf07ab1c4c3fa1f819256d3de858d1ed87869bdfa663553280d7e78b88bee4",
|
| 57 |
+
"complete_input_tokens": 1024
|
| 58 |
+
},
|
| 59 |
+
{
|
| 60 |
+
"label": "Lex",
|
| 61 |
+
"repo_id": "vllm-sr/Decision-1.0-Lex-0.6B",
|
| 62 |
+
"revision": "ee8e74d912fca8328a353c11d174b44da3f91781",
|
| 63 |
+
"manifest_sha256": "f288d873999832a3f37c6a7c4268c2ab309691e621794dbf7acab891acbbb7e6",
|
| 64 |
+
"complete_input_tokens": 1024
|
| 65 |
+
},
|
| 66 |
+
{
|
| 67 |
+
"label": "Eos",
|
| 68 |
+
"repo_id": "vllm-sr/Decision-1.0-Eos-0.8B",
|
| 69 |
+
"revision": "3c2d632609ceb66f3a13bbc5f77f3ab8cdeebcdd",
|
| 70 |
+
"manifest_sha256": "4f1886e3df327cb0726086df1f41f1ab584ecc7ab80c0e065e68c7cb8dc96ec6",
|
| 71 |
+
"complete_input_tokens": 16384
|
| 72 |
+
},
|
| 73 |
+
{
|
| 74 |
+
"label": "Sol",
|
| 75 |
+
"repo_id": "vllm-sr/Decision-1.0-Sol-2B",
|
| 76 |
+
"revision": "38a8fb05dad80e62ec32dba27ee3e31be5ece989",
|
| 77 |
+
"manifest_sha256": "1498cc7aa42f5884ab6ca828b5c23f23d4965b7e950a2ef86d530eed99ad78f1",
|
| 78 |
+
"complete_input_tokens": 16384
|
| 79 |
+
},
|
| 80 |
+
{
|
| 81 |
+
"label": "Nox",
|
| 82 |
+
"repo_id": "vllm-sr/Decision-1.0-Nox-4B",
|
| 83 |
+
"revision": "39f0f0e74ef0ec55dc729437c719547e750d02ff",
|
| 84 |
+
"manifest_sha256": "7d9b06bc25a75b1f131aabd280df0ef2777bf69db98574a300067ace40c61c9c",
|
| 85 |
+
"complete_input_tokens": 16384
|
| 86 |
+
},
|
| 87 |
+
{
|
| 88 |
+
"label": "Lux",
|
| 89 |
+
"repo_id": "vllm-sr/Decision-1.0-Lux-9B",
|
| 90 |
+
"revision": "ec7001aa04b2fe2a682e02aed9572e047bd68993",
|
| 91 |
+
"manifest_sha256": "d968f7e3f1b8e8302909da4631304cd939f721d3109f49f95591b4af1852e67f",
|
| 92 |
+
"complete_input_tokens": 16384
|
| 93 |
+
}
|
| 94 |
+
]
|
| 95 |
+
}
|
README.md
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
title: "Decision 2.0: Open Foundation Decision Models"
|
| 3 |
+
emoji: ⚡
|
| 4 |
+
colorFrom: purple
|
| 5 |
+
colorTo: red
|
| 6 |
+
sdk: docker
|
| 7 |
+
app_port: 7860
|
| 8 |
+
header: mini
|
| 9 |
+
pinned: false
|
| 10 |
+
license: mit
|
| 11 |
+
short_description: One pass. Many decisions. Six Decision 2.0 models.
|
| 12 |
+
thumbnail: https://huggingface.co/spaces/vllm-sr/decision-studio/resolve/main/static/assets/social-banner.png
|
| 13 |
+
models:
|
| 14 |
+
- vllm-sr/Decision-2.0-Vega-27B
|
| 15 |
+
- vllm-sr/Decision-2.0-Lux-9B
|
| 16 |
+
- vllm-sr/Decision-2.0-Nox-4B
|
| 17 |
+
- vllm-sr/Decision-2.0-Sol-2B
|
| 18 |
+
- vllm-sr/Decision-2.0-Eos-0.8B
|
| 19 |
+
- vllm-sr/Decision-2.0-Kai-0.6B
|
| 20 |
+
---
|
| 21 |
+
|
| 22 |
+
# Decision 2.0: Open Foundation Decision Models ⚡
|
| 23 |
+
|
| 24 |
+
> **This Space is an archive.** Every example, and every Arena race of seed 42 up to 120 steps, replays a result recorded on the Decision 2.0 models; nothing here calls a model. For live decisions, try the newest [Decision 3.0 Space](https://huggingface.co/spaces/vllm-sr/Decision-3.0), or call the d3 models at [decision.vllm-sr.ai](https://decision.vllm-sr.ai) with a free API key.
|
| 25 |
+
|
| 26 |
+
**One pass. Many decisions.**
|
| 27 |
+
|
| 28 |
+
Ask many Choice, Noul and Score questions about one input and read every probability. Decision 2.0 models answer all questions about the same input together in one forward pass; Explore the original generation in the [Decision 1.0 model collection](https://huggingface.co/collections/vllm-sr/decision-10-6ab12177bd0002394d8409f9). Route a prompt before calling an LLM, compare a batch of contexts with shared questions, or ask several related questions about one case, with nine LLM-routing scenarios plus model-specific English scenarios.
|
| 29 |
+
|
| 30 |
+
[Decision 2.0 collection](https://huggingface.co/collections/vllm-sr/decision-20-6ab7cf7bdfb506bf8269cb00) · [Decision 1.0 collection](https://huggingface.co/collections/vllm-sr/decision-10-6ab12177bd0002394d8409f9)
|
| 31 |
+
|
| 32 |
+
The deployed service also includes a server-authoritative Tetris Race playground. It compares any two configured Decision 2.0 models. Both runners receive the same seeded piece sequence but keep independent boards, requests, actions, timing, and traces.
|
| 33 |
+
|
| 34 |
+
The race length follows the server's configured limit (120 steps by default, up to 2,000 when enabled). “Until first failure” is bounded by that server guard; reaching it without a game-over is shown as inconclusive, not as a win. A provider failure or incomplete run has no score winner. For equal completed step races, the speed percentage compares complete server-observed adapter calls on both sides, including direct Decision response attestation; a provider's inference-only time is diagnostic.
|
| 35 |
+
|
| 36 |
+
Each Tetris turn runs on the selected model's queue, the same path as the public `/v1/systemone` API, and its response must carry the selected Hugging Face repository ID exactly (for example, `vllm-sr/Decision-2.0-Lux-9B`) in the `model` field. Legacy wire aliases such as `decision-lux` are intentionally rejected; a misrouted response fails closed instead of attributing one model's play to another.
|
| 37 |
+
|
| 38 |
+
| Model | Version | Focus | Model size | Complete input per question |
|
| 39 |
+
|---|---|---|---:|---:|
|
| 40 |
+
| [Vega](https://huggingface.co/vllm-sr/Decision-2.0-Vega-27B) | 2.0 | Strongest Decision model | 27B | 32K |
|
| 41 |
+
| [Lux](https://huggingface.co/vllm-sr/Decision-2.0-Lux-9B) | 2.0 | Large hybrid decoder | 9B | 16K |
|
| 42 |
+
| [Nox](https://huggingface.co/vllm-sr/Decision-2.0-Nox-4B) | 2.0 | Capable hybrid decoder | 4B | 16K |
|
| 43 |
+
| [Sol](https://huggingface.co/vllm-sr/Decision-2.0-Sol-2B) | 2.0 | Compact hybrid decoder | 2B | 16K |
|
| 44 |
+
| [Eos](https://huggingface.co/vllm-sr/Decision-2.0-Eos-0.8B) | 2.0 | Fast hybrid decoder | 0.8B | 16K |
|
| 45 |
+
| [Kai](https://huggingface.co/vllm-sr/Decision-2.0-Kai-0.6B) | 2.0 | Fastest compact decoder | 0.6B | 8K |
|
| 46 |
+
|
| 47 |
+
Choose a model from the model menu in the Studio bar, the family lineup at the top, or the models table; this Space offers the six Decision 2.0 models. The first tab, **LLM routing**, combines Choice, Score, and Noul questions to assess the target LLM capability, complexity, speed priority, a possible instruction-override attempt, whether a factual claim needs source verification, and the necessary tool. Eight examples cover quick arithmetic, architecture, code, screenshots, translation, proofs and fact-checking; a ninth harmless instruction-override example asks only the safety question, before ordinary model routing. Choose **One context** to inspect several aspects of one case, or **Batch contexts** to apply shared questions to a compact list of inputs. Single-context results show every candidate probability without opening a detail row; in the batch matrix, open a row for the full distribution. Each model and mode keeps its own draft.
|
| 48 |
+
|
| 49 |
+
The default example is a quick LLM-routing prompt available across the six Decision 2.0 models. Some routing examples use model-specific wording so the same task is clear to both compact encoders and larger decoders; switching models shows the actual prompt and questions sent to that model. The screenshot example classifies the *text request* as requiring vision downstream; Studio does not inspect the image itself. Other examples cover reviews, deliveries, account security, stock requests, travel refunds, purchasing policy, expenses and document sensitivity. Desktop input and output panels keep equal heights with independent scrolling; actions remain visible. Context/question pairs are combined into real GPU batches. The API accepts up to 1,024 state/question decisions per batch within its byte and token budgets; live microbatch capacity belongs to the selected runtime. An unavailable batch worker is reported explicitly.
|
| 50 |
+
|
| 51 |
+
Complete input includes context, instructions, candidates and special tokens. Nothing is truncated. Examples are curated demonstrations, not benchmark results; probabilities do not guarantee correctness. In particular, `jailbreak_attempt` is exploratory: quoted attack text can be misclassified, and the score is not a security filter or a substitute for instruction hierarchy enforcement. One shared context in the request does not imply that the model encodes it only once. Context IDs only associate inputs with outputs; they are not model evidence.
|
| 52 |
+
|
| 53 |
+
The editor runs on this CPU Space. Production direct mode maps each canonical model ID to its own private `vllm-sr decision serve` instance; the authenticated pull-queue deployment remains an explicit rollback mode. Switching clears old results, and an unavailable or misidentified model never silently falls back to another. Requests and results are not application-logged.
|
| 54 |
+
|
| 55 |
+
[Historical presentation metadata](MODEL_RELEASES.json) · [Request contract](SYSTEM_ONE_MAPPING.md) · [Deployment](DEPLOYMENT.md). Active revision and manifest pins come from the deployment registry and are checked against each live Decision runtime service; the presentation file is not a serving allowlist. The interface follows the [Vela 2.0 Studio](https://huggingface.co/spaces/vllm-sr/vela2-studio) visual system: a dusk horizon, self-hosted Geist fonts, warm accents and dark glass panels.
|
| 56 |
+
|
| 57 |
+
Studio code is MIT licensed. Models have their own Apache-2.0 project licenses and retained third-party notices. This repository contains no model weights or worker credentials.
|
SYSTEMONE_API.md
ADDED
|
@@ -0,0 +1,121 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Decision SystemOne API
|
| 2 |
+
|
| 3 |
+
Call the Decision 2.0 models (Vega, Lux, Nox, Sol, Eos and Kai) and the Decision 1.0 models (Lux, Nox, Sol, Eos, Kai and Lex) from your computer using the same `state / questions / model` request format as the [SystemOne API](https://docs.typesafe.ai/api).
|
| 4 |
+
|
| 5 |
+
- Base URL: `https://YOUR-DECISION-HOST` (replace with your deployment origin)
|
| 6 |
+
- Single-state inference: `POST /v1/systemone`
|
| 7 |
+
- Shared-question batches: `POST /v1/systemone/batches` (Decision extension)
|
| 8 |
+
- Model discovery: `GET /v1/models`
|
| 9 |
+
|
| 10 |
+
This is a public demo endpoint backed by the released models on AMD GPUs. It requires no API key. Inference runs on the connected server; your computer sends the request. It is a shared, bounded service rather than a dedicated production deployment. Responses are real model outputs; there is no chat-completion endpoint.
|
| 11 |
+
|
| 12 |
+
## Send a request
|
| 13 |
+
|
| 14 |
+
```bash
|
| 15 |
+
curl --fail-with-body https://YOUR-DECISION-HOST/v1/systemone \
|
| 16 |
+
-H 'Content-Type: application/json' \
|
| 17 |
+
-d '{"model":"vllm-sr/Decision-2.0-Lux-9B","state":"My subscription was charged twice. Please refund the duplicate charge.","questions":{"billing":{"type":"noul","instructions":"Does the customer report a billing problem?"},"team":{"type":"choice","instructions":"Which team should handle this message?","criteria":{"Billing":"Charges, invoices, and refunds","Accounts":"Login, passwords, and account access"}},"urgency":{"type":"score","instructions":"Rate how urgently this needs attention.","criteria":["Routine","Needs prompt attention","Critical"]}}}'
|
| 18 |
+
```
|
| 19 |
+
|
| 20 |
+
Ask multiple named questions in one call. The same state is used for every question and the worker processes the admitted decisions in GPU batches. This does not imply shared-state encoder caching.
|
| 21 |
+
|
| 22 |
+
`/v1/systemone/batches` keeps the same `model`, question types, and per-state
|
| 23 |
+
answer shapes as `/v1/systemone`, but replaces the single `state` with an
|
| 24 |
+
ordered `states: [{id, state}, ...]` array. It returns ordered `results`
|
| 25 |
+
instead of one top-level `answers` map. This is our multi-state extension;
|
| 26 |
+
the official SystemOne SDKs call only the single-state endpoint.
|
| 27 |
+
|
| 28 |
+
To apply one question set to multiple independent states, send a batch:
|
| 29 |
+
|
| 30 |
+
```bash
|
| 31 |
+
curl --fail-with-body https://YOUR-DECISION-HOST/v1/systemone/batches \
|
| 32 |
+
-H 'Content-Type: application/json' \
|
| 33 |
+
-d '{
|
| 34 |
+
"model": "vllm-sr/Decision-2.0-Lux-9B",
|
| 35 |
+
"states": [
|
| 36 |
+
{"id": "ticket-a", "state": "I was charged twice. Please refund the duplicate."},
|
| 37 |
+
{"id": "ticket-b", "state": "How do I change my password?"}
|
| 38 |
+
],
|
| 39 |
+
"questions": {
|
| 40 |
+
"billing": {"type": "noul", "instructions": "Is this a billing issue?"},
|
| 41 |
+
"team": {"type": "choice", "instructions": "Which team should handle this?", "criteria": {"Billing": "Charges and refunds", "Accounts": "Account access"}}
|
| 42 |
+
}
|
| 43 |
+
}'
|
| 44 |
+
```
|
| 45 |
+
|
| 46 |
+
Read `results[0].id` and `results[0].answers` for `ticket-a`, then
|
| 47 |
+
`results[1]` for `ticket-b`. The `results` array preserves input order and
|
| 48 |
+
each row has its own `usage`; top-level `usage` is the total. This is one
|
| 49 |
+
request with four decisions, not the same as four HTTP requests.
|
| 50 |
+
|
| 51 |
+
## Official Python SDK
|
| 52 |
+
|
| 53 |
+
Use Python 3.10 or later:
|
| 54 |
+
|
| 55 |
+
```bash
|
| 56 |
+
python -m pip install typesafe-sdk==0.7.1
|
| 57 |
+
```
|
| 58 |
+
|
| 59 |
+
```python
|
| 60 |
+
from typesafe_sdk import Choice, Noul, Score, TypeSafeClient, RetryPolicy
|
| 61 |
+
|
| 62 |
+
with TypeSafeClient(
|
| 63 |
+
base_url="https://YOUR-DECISION-HOST",
|
| 64 |
+
api_key="decision-public", # SDK-required placeholder, not a credential
|
| 65 |
+
model="vllm-sr/Decision-2.0-Lux-9B",
|
| 66 |
+
timeout=60,
|
| 67 |
+
retry=RetryPolicy(max_retries=0),
|
| 68 |
+
) as client:
|
| 69 |
+
print([model.name for model in client.models.list().models])
|
| 70 |
+
result = client.system_one(
|
| 71 |
+
state="My subscription was charged twice. Please refund the duplicate charge.",
|
| 72 |
+
questions={
|
| 73 |
+
"billing": Noul(instructions="Does the customer report a billing problem?"),
|
| 74 |
+
"team": Choice(
|
| 75 |
+
instructions="Which team should handle this message?",
|
| 76 |
+
criteria={"Billing": "Charges, invoices, and refunds",
|
| 77 |
+
"Accounts": "Login, passwords, and account access"},
|
| 78 |
+
),
|
| 79 |
+
"urgency": Score(
|
| 80 |
+
instructions="Rate how urgently this needs attention.",
|
| 81 |
+
criteria=["Routine", "Needs prompt attention", "Critical"],
|
| 82 |
+
),
|
| 83 |
+
},
|
| 84 |
+
)
|
| 85 |
+
print(result.nouls["billing"].noul)
|
| 86 |
+
print(result.choices["team"].choice)
|
| 87 |
+
print(result.scores["urgency"].score)
|
| 88 |
+
```
|
| 89 |
+
|
| 90 |
+
Select any exact Hugging Face repository ID returned as `id` by `/v1/models`. The model is required on public inference requests; short names, wire aliases, and case variants are rejected. The response repeats the selected canonical ID. The Studio editor uses a same-origin endpoint; in direct serving mode it receives the same strict response envelope, while legacy modes retain their internal queue/diagnostic contracts.
|
| 91 |
+
|
| 92 |
+
Direct mode does not advertise a default model. The Studio editor selects a model locally and includes it explicitly in every `/api/evaluate` request; a missing model receives HTTP 422. Legacy rollback modes may retain their previous discovery default.
|
| 93 |
+
|
| 94 |
+
| Model | Generation | Parameters | Complete input per question |
|
| 95 |
+
|---|---|---:|---:|
|
| 96 |
+
| Vega | 2.0 | 27B | 32,768 tokens |
|
| 97 |
+
| Lux | 2.0 | 9B | 16,384 tokens |
|
| 98 |
+
| Nox | 2.0 | 4B | 16,384 tokens |
|
| 99 |
+
| Sol | 2.0 | 2B | 16,384 tokens |
|
| 100 |
+
| Eos | 2.0 | 0.8B | 16,384 tokens |
|
| 101 |
+
| Kai | 2.0 | 0.6B | 8,192 tokens |
|
| 102 |
+
| Lux | 1.0 | 9B | 16,384 tokens |
|
| 103 |
+
| Nox | 1.0 | 4B | 16,384 tokens |
|
| 104 |
+
| Sol | 1.0 | 2B | 16,384 tokens |
|
| 105 |
+
| Eos | 1.0 | 0.8B | 16,384 tokens |
|
| 106 |
+
| Kai | 1.0 | 0.6B | 1,024 tokens |
|
| 107 |
+
| Lex | 1.0 | 0.6B | 1,024 tokens |
|
| 108 |
+
|
| 109 |
+
`GET /v1/models` returns models in this order, including pinned revision/manifest metadata. When present, `release_date` comes from `MODEL_RELEASES.json` and applies to the currently configured Hub revision and manifest. The date is omitted for a newly deployed artifact until its presentation metadata is updated. If that presentation file is missing or invalid, the endpoint still lists all configured models without release dates; serving does not depend on the file. Explicit metadata validation still reports the error.
|
| 110 |
+
|
| 111 |
+
## Responses and limits
|
| 112 |
+
|
| 113 |
+
- Noul returns P(true); Choice returns its winning label and probabilities; Score returns the expected ordinal level and its distribution.
|
| 114 |
+
- `confidence` is a versioned statistic of the answer distribution, named per model by `confidence_definition` in `/v1/models`. Decision 1.0 (`margin_ordinal_v1`): Choice reports the top-two probability margin and Score the ordinal concentration around its expected level, normalized against uniform variance. Decision 2.0 (`normalized_entropy_v2`): Choice and Score report one minus the distribution's entropy divided by the log of its candidate count. These are Decision-owned statistics, not calibrated probabilities of correctness or a claim of equivalence to TypeSafe's undisclosed calculation. The direct Gateway validates and forwards the runtime value unchanged; it never substitutes `max(probabilities)`. Public responses contain only `model`, `answers`, and `usage`.
|
| 115 |
+
- The hosted endpoint accepts one or more named questions without a fixed question-count cap. In direct mode, each model instance owns its physical microbatch setting. Choice accepts 2–255 options, Score accepts 2–10 ordered levels, and JSON requests are limited to 256 KiB. Expanded state/question input is limited to 16 MiB so large Cartesian workloads stay bounded. All question types require nonempty instructions. State plus question plus all candidates must fit the model limit; overflowing requests are rejected without truncation.
|
| 116 |
+
- In direct serving mode, the synchronous wait is configurable and each instance owns its own concurrency, queue, and physical-batch limits; `/v1/models` does not invent fixed values for them. A busy or timed-out service may return 529/503/504 (legacy queue mode may return 429). Completed requests do not consume admission capacity. Do not treat transport errors as predictions. SDK example retries are disabled to avoid duplicate inference.
|
| 117 |
+
- `POST /v1/systemone/batches` applies one shared `questions` map to ordered `states: [{id, state}, ...]`. It requires the same explicit canonical `model`, unique state IDs, at most 1,024 states, questions, and total decisions, a 2 MiB JSON body, and the same 16 MiB expanded-input and per-question token budgets. Its atomic response contains `model`, ordered `results: [{id, answers, usage}, ...]`, and aggregate `usage`. This Decision extension is separate from the SDK's single-state method.
|
| 118 |
+
|
| 119 |
+
See [SYSTEM_ONE_MAPPING.md](SYSTEM_ONE_MAPPING.md) for the complete adapter contract. The standard single-state call, all three answer types, and model discovery are validated with the official SDK.
|
| 120 |
+
|
| 121 |
+
In legacy pull-queue mode, completed asynchronous results are kept for up to 120 seconds, with the oldest completed records evicted when the 64-result cache fills. Direct serving mode does not create Gateway jobs or retain completion receipts. `GET /v1/models` publishes the service limits. The original Space origin remains available.
|
SYSTEM_ONE_MAPPING.md
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# System One input mapping
|
| 2 |
+
|
| 3 |
+
Verified against TypeSafe's primary documentation on 2026-09-21: [HTTP API](https://docs.typesafe.ai/api), [Choice](https://docs.typesafe.ai/primitives/choice), [Score](https://docs.typesafe.ai/primitives/score), [Noul](https://docs.typesafe.ai/primitives/noul), and [confidence](https://docs.typesafe.ai/confidence). The reference style is the existing `llm-semantic-router/vela-studio` FastAPI/static application; its CPU inference implementation is not reused for model serving.
|
| 4 |
+
|
| 5 |
+
Studio exposes a **SystemOne-format Decision service** with official Python SDK support for model discovery and single-state Noul, Choice, and Score requests. See [the callable API guide](SYSTEMONE_API.md). It is not a Jev service; no TypeSafe key or paid TypeSafe request is involved. Hosted input bounds and the batch-context extension remain distinct from upstream behavior.
|
| 6 |
+
|
| 7 |
+
| Input | Exact local mapping |
|
| 8 |
+
|---|---|
|
| 9 |
+
| `state` text/object/array | Native `state_text`; structured content serializes as UTF-8 JSON with sorted object keys and compact separators. No truncation. |
|
| 10 |
+
| `states: [{id, state}, ...]` | `POST /v1/systemone/batches` extension for shared-question context batches. IDs are unique output keys and never added to model text. |
|
| 11 |
+
| Question map key | Native question ID and returned answer key. Never added to model text. |
|
| 12 |
+
| `instructions` text/object/array | Required native question text, with the same deterministic JSON rule for structured content. |
|
| 13 |
+
| Choice `criteria` map | Each option name is semantic: null description → name; otherwise name + `: ` + full description. Object insertion order is retained. External native candidate IDs equal the names, but the names reach the model only through this explicit semantic text. |
|
| 14 |
+
| Score `criteria` array | Ordered native levels with IDs and values 0…K−1. Returned score is the probability-weighted level index. |
|
| 15 |
+
| Noul `criteria.false/true` | Native false/true descriptions. An omitted or `null` description uses the original native default no/yes description. |
|
| 16 |
+
|
| 17 |
+
The public single-state endpoint requires exactly `model`, `state`, and `questions`, with a 256 KiB JSON body. The separate batch endpoint requires exactly `model`, `states`, and `questions`, with a 2 MiB JSON body and at most 1,024 states, questions, and total decisions. Both accept 2–255 Choice options, 2–10 Score levels, and up to 16 MiB of expanded context/question input. The direct runtime uses a per-model configurable physical microbatch size rather than a fixed eight rows. The complete-input limit is 1,024 tokens for Kai/Lex and 16,384 for Eos/Sol/Nox/Lux **per question**, including state and all candidate descriptions. Unsupported fields and models are rejected.
|
| 18 |
+
|
| 19 |
+
Single-context responses preserve named `answers`, Choice `choice/probabilities`, Score `score/legend/probabilities`, and Noul `noul`. `usage.output_tokens=0` because these models do not generate text. The public inference envelopes contain only `model`, answers or results, and usage. In direct serving mode, Studio's `/api/evaluate` route returns that same strict envelope; legacy native and pull-queue modes retain their diagnostic response fields for rollback.
|
| 20 |
+
|
| 21 |
+
The public inference responses supply versioned, type-aware Decision confidence, named per model by `confidence_definition` in `/v1/models`. For Decision 1.0 (`margin_ordinal_v1`), Choice uses the top-two probability margin and Score uses ordinal concentration around the expected level relative to uniform variance. For Decision 2.0 (`normalized_entropy_v2`), both report one minus the normalized entropy of the distribution, and a structured Score level appears in the legend as canonical JSON text. It is not calibrated correctness or a claim of equivalence to TypeSafe’s undisclosed confidence formula. The direct Gateway validates and preserves the runtime value rather than recalculating it as `max(probabilities)`. Noul returns its original probability of yes.
|
| 22 |
+
|
| 23 |
+
This mapping is independently testable with small metadata fixtures. It does not change the model's packing, native graph, weights, scoring arithmetic, or export format.
|
| 24 |
+
|
| 25 |
+
Sol and Nox use their published `question_row` and `predict_rows` path, the same numerical path used by `DecisionModel.decide`. Shipped temperatures, candidate order, true-probability orientation and ordinal Score arithmetic are preserved. Legacy Studio modes retain exact per-question input-token diagnostics; direct serving mode returns only strict aggregate usage. Neither path reinterprets a Decision statistic as TypeSafe confidence.
|
| 26 |
+
|
| 27 |
+
## A shared question across contexts
|
| 28 |
+
|
| 29 |
+
```json
|
| 30 |
+
{
|
| 31 |
+
"model": "vllm-sr/Decision-2.0-Kai-0.6B",
|
| 32 |
+
"states": [
|
| 33 |
+
{"id": "T-101", "state": "My subscription was charged twice. Please refund the duplicate payment."},
|
| 34 |
+
{"id": "T-102", "state": "I forgot my password and need help signing in."}
|
| 35 |
+
],
|
| 36 |
+
"questions": {
|
| 37 |
+
"destination": {
|
| 38 |
+
"type": "choice",
|
| 39 |
+
"instructions": "Which team should handle this customer message?",
|
| 40 |
+
"criteria": {
|
| 41 |
+
"Billing": "Charges, invoices, and refunds",
|
| 42 |
+
"Accounts": "Login, passwords, and account access"
|
| 43 |
+
}
|
| 44 |
+
}
|
| 45 |
+
}
|
| 46 |
+
}
|
| 47 |
+
```
|
| 48 |
+
|
| 49 |
+
Submit this payload to `POST /v1/systemone/batches`. Batch responses replace top-level `answers` with ordered `results: [{id, answers, usage}, ...]`. Context order and shared-question order are retained. Top-level `usage` sums all pairs. Legacy native/pull-queue Studio responses also report `contexts`, `questions`, and `decisions` in `timing`; direct responses stay strict. A row can expand to show the original probabilities. Score is an expected ordinal index; Noul is P(Yes), not an independent confidence estimate.
|
| 50 |
+
|
| 51 |
+
The worker admits every complete context/question input before its first forward. Any overflow rejects the whole request: no truncation or partial result. Admitted pairs are flattened into one predictor call with physical batches of up to eight. This is actual GPU batching, not browser-side request fanout or shared-state caching. The native numerical engine, weights, candidate order and published temperatures remain unchanged.
|
app.py
ADDED
|
@@ -0,0 +1,655 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Same-origin Studio and strict Decision Gateway."""
|
| 2 |
+
import asyncio
|
| 3 |
+
import json
|
| 4 |
+
import logging
|
| 5 |
+
import os
|
| 6 |
+
import time
|
| 7 |
+
from contextlib import asynccontextmanager
|
| 8 |
+
from pathlib import Path
|
| 9 |
+
|
| 10 |
+
from fastapi import FastAPI, HTTPException, Request
|
| 11 |
+
from fastapi.responses import JSONResponse, StreamingResponse
|
| 12 |
+
from fastapi.staticfiles import StaticFiles
|
| 13 |
+
from starlette.concurrency import run_in_threadpool
|
| 14 |
+
|
| 15 |
+
from archive_cache import NOTICE as ARCHIVE_NOTICE
|
| 16 |
+
from archive_cache import ArchiveCache
|
| 17 |
+
from contract import MODEL, to_records
|
| 18 |
+
from direct_gateway import DirectGateway, DirectGatewayError
|
| 19 |
+
from engine import Busy, Engine, Unavailable
|
| 20 |
+
from relay import Relay, RelayError
|
| 21 |
+
from systemone_api import (
|
| 22 |
+
arrange_models,
|
| 23 |
+
public_models,
|
| 24 |
+
resolve_canonical_model,
|
| 25 |
+
resolve_model,
|
| 26 |
+
public_response,
|
| 27 |
+
sdk_response,
|
| 28 |
+
)
|
| 29 |
+
from tetris_arena import (
|
| 30 |
+
ArenaBusyError,
|
| 31 |
+
ArenaUpstreamError,
|
| 32 |
+
ArenaValidationError,
|
| 33 |
+
HTTPDecisionAdapter,
|
| 34 |
+
RaceManager,
|
| 35 |
+
encode_sse,
|
| 36 |
+
)
|
| 37 |
+
|
| 38 |
+
ROOT = Path(__file__).resolve().parent
|
| 39 |
+
logger = logging.getLogger("decision.studio")
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
def unique_object(pairs):
|
| 43 |
+
result = {}
|
| 44 |
+
for key, value in pairs:
|
| 45 |
+
if key in result:
|
| 46 |
+
raise ValueError("Duplicate JSON key: " + key)
|
| 47 |
+
result[key] = value
|
| 48 |
+
return result
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
async def read_json(request, limit=256 * 1024, *, with_size=False):
|
| 52 |
+
if request.headers.get("content-type", "").split(";", 1)[0].strip().lower() != "application/json":
|
| 53 |
+
raise HTTPException(415, "Use Content-Type: application/json")
|
| 54 |
+
body = bytearray()
|
| 55 |
+
async for chunk in request.stream():
|
| 56 |
+
body.extend(chunk)
|
| 57 |
+
if len(body) > limit:
|
| 58 |
+
raise HTTPException(413, "Request is too large. No field is truncated.")
|
| 59 |
+
try:
|
| 60 |
+
document = json.loads(body, object_pairs_hook=unique_object,
|
| 61 |
+
parse_constant=lambda x: (_ for _ in ()).throw(ValueError("Nonfinite JSON value")))
|
| 62 |
+
return (document, len(body)) if with_size else document
|
| 63 |
+
except (ValueError, TypeError, UnicodeError, RecursionError) as exc:
|
| 64 |
+
raise HTTPException(422, str(exc)) from None
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
async def read_input(request):
|
| 68 |
+
payload = await read_json(request)
|
| 69 |
+
try:
|
| 70 |
+
records = to_records(payload)
|
| 71 |
+
except (ValueError, TypeError, UnicodeError, RecursionError) as exc:
|
| 72 |
+
raise HTTPException(422, str(exc)) from None
|
| 73 |
+
return payload, records
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
def create_app(
|
| 77 |
+
mode=None,
|
| 78 |
+
*,
|
| 79 |
+
native_engine=None,
|
| 80 |
+
relay=None,
|
| 81 |
+
registry=None,
|
| 82 |
+
relays=None,
|
| 83 |
+
direct_gateway=None,
|
| 84 |
+
tetris_adapter=None,
|
| 85 |
+
tetris_manager=None,
|
| 86 |
+
archive=None,
|
| 87 |
+
):
|
| 88 |
+
mode = mode or os.getenv("DECISION_BACKEND", "native")
|
| 89 |
+
if mode not in {"native", "pull_queue", "direct", "cache"}:
|
| 90 |
+
raise ValueError("DECISION_BACKEND must be native, pull_queue, direct, or cache")
|
| 91 |
+
# cache: an archive that answers only from recorded results and calls no model.
|
| 92 |
+
archive = (archive or ArchiveCache.load(ROOT)) if mode == "cache" else None
|
| 93 |
+
from model_registry import model_registry
|
| 94 |
+
if relay is not None and registry is None:
|
| 95 |
+
registry = [{'id': MODEL, 'label': 'Kai', 'version': '1.0', 'manifest_sha256': relay.manifest}]
|
| 96 |
+
registry = arrange_models(model_registry(registry))
|
| 97 |
+
studio_generation = os.getenv("DECISION_STUDIO_GENERATION", "").strip()
|
| 98 |
+
if studio_generation not in {"", "1.0", "2.0"}:
|
| 99 |
+
raise ValueError("DECISION_STUDIO_GENERATION must be 1.0 or 2.0 when set")
|
| 100 |
+
studio_registry = {
|
| 101 |
+
wire: item for wire, item in registry.items()
|
| 102 |
+
if not studio_generation or item['version'] == studio_generation
|
| 103 |
+
}
|
| 104 |
+
if not studio_registry:
|
| 105 |
+
raise ValueError("The Studio generation must include a configured model")
|
| 106 |
+
default_model = next(iter(studio_registry))
|
| 107 |
+
canonical_models = tuple(item['repo_id'] for item in registry.values())
|
| 108 |
+
expected_artifacts = {item['repo_id']: item for item in registry.values()}
|
| 109 |
+
owns_direct_gateway = mode == "direct" and direct_gateway is None
|
| 110 |
+
if mode == "direct":
|
| 111 |
+
if any('revision' not in item for item in registry.values()):
|
| 112 |
+
raise ValueError("Direct serving mode requires a pinned Hub revision for every model")
|
| 113 |
+
direct_gateway = direct_gateway or DirectGateway.from_environment(expected_artifacts)
|
| 114 |
+
if getattr(direct_gateway, "models", None) != frozenset(canonical_models):
|
| 115 |
+
raise ValueError(
|
| 116 |
+
"The direct Gateway must match every configured canonical model"
|
| 117 |
+
)
|
| 118 |
+
gateway_pins = getattr(direct_gateway, "expected_artifacts", None)
|
| 119 |
+
if gateway_pins is not None and any(
|
| 120 |
+
gateway_pins.get(model) != {
|
| 121 |
+
"revision": item["revision"],
|
| 122 |
+
"manifest_sha256": item["manifest_sha256"],
|
| 123 |
+
**({"content_sha256": item["content_sha256"]}
|
| 124 |
+
if "content_sha256" in item else {}),
|
| 125 |
+
}
|
| 126 |
+
for model, item in expected_artifacts.items()
|
| 127 |
+
):
|
| 128 |
+
raise ValueError("The direct Gateway artifact pins must match the registry")
|
| 129 |
+
elif direct_gateway is not None:
|
| 130 |
+
raise ValueError("A direct Gateway may only be supplied in direct mode")
|
| 131 |
+
owns_tetris_manager = tetris_manager is None
|
| 132 |
+
owns_tetris_adapter = owns_tetris_manager and tetris_adapter is None
|
| 133 |
+
|
| 134 |
+
@asynccontextmanager
|
| 135 |
+
async def lifespan(_api):
|
| 136 |
+
yield
|
| 137 |
+
try:
|
| 138 |
+
if owns_tetris_manager:
|
| 139 |
+
try:
|
| 140 |
+
await tetris_manager.aclose()
|
| 141 |
+
finally:
|
| 142 |
+
if owns_tetris_adapter:
|
| 143 |
+
await tetris_adapter.close()
|
| 144 |
+
finally:
|
| 145 |
+
if owns_direct_gateway:
|
| 146 |
+
await direct_gateway.aclose()
|
| 147 |
+
|
| 148 |
+
api = FastAPI(
|
| 149 |
+
title="Decision Studio",
|
| 150 |
+
version="0.6.0",
|
| 151 |
+
docs_url="/api/docs",
|
| 152 |
+
redoc_url=None,
|
| 153 |
+
lifespan=lifespan,
|
| 154 |
+
)
|
| 155 |
+
api.state.engine = native_engine or Engine()
|
| 156 |
+
if mode == "native" and (set(registry) != {MODEL} or api.state.engine.model != MODEL
|
| 157 |
+
or api.state.engine.manifest != registry[MODEL]['manifest_sha256']):
|
| 158 |
+
raise ValueError("Native mode requires only Decision 1.0 Kai with its matching manifest")
|
| 159 |
+
if mode == "pull_queue":
|
| 160 |
+
relays = relays or ({MODEL: relay} if relay else {
|
| 161 |
+
key: Relay(os.getenv("DECISION_WORKER_TOKEN", ""), item['manifest_sha256'], model=key, complete_input_tokens=item['complete_input_tokens'])
|
| 162 |
+
for key, item in registry.items()})
|
| 163 |
+
if set(relays) != set(registry) or any(r.model != key or r.manifest != registry[key]['manifest_sha256'] or r.complete_input_tokens != registry[key]['complete_input_tokens'] for key, r in relays.items()):
|
| 164 |
+
raise ValueError("Every queue must match its registry ID and native manifest")
|
| 165 |
+
else:
|
| 166 |
+
relays = {}
|
| 167 |
+
if relays and len({r.token for r in relays.values()}) != 1:
|
| 168 |
+
raise ValueError("This gateway uses one shared private worker token")
|
| 169 |
+
api.state.relays = relays
|
| 170 |
+
api.state.relay = relays.get(MODEL) # backward-compatible default for local tooling
|
| 171 |
+
api.state.registry = registry
|
| 172 |
+
api.state.direct = direct_gateway
|
| 173 |
+
api.state.archive = archive
|
| 174 |
+
|
| 175 |
+
def select(model):
|
| 176 |
+
if not isinstance(model, str) or model not in registry:
|
| 177 |
+
raise HTTPException(422, "This model is not available in this Studio")
|
| 178 |
+
return relays.get(model)
|
| 179 |
+
|
| 180 |
+
async def direct_readiness(models):
|
| 181 |
+
"""Probe distinct configured Decision models without exposing their origins."""
|
| 182 |
+
selected = tuple(dict.fromkeys(models))
|
| 183 |
+
observations = await asyncio.gather(
|
| 184 |
+
*(direct_gateway.probe(model) for model in selected),
|
| 185 |
+
return_exceptions=True,
|
| 186 |
+
)
|
| 187 |
+
return {
|
| 188 |
+
model: isinstance(observed, dict) and observed.get('loaded') is True
|
| 189 |
+
for model, observed in zip(selected, observations, strict=True)
|
| 190 |
+
}
|
| 191 |
+
|
| 192 |
+
async def tetris_model_readiness(competitor_ids=None):
|
| 193 |
+
if mode not in {"direct", "pull_queue"}:
|
| 194 |
+
return {}
|
| 195 |
+
competitors = (
|
| 196 |
+
api.state.tetris.catalog()
|
| 197 |
+
if hasattr(api.state.tetris, 'catalog') else ()
|
| 198 |
+
)
|
| 199 |
+
selected = {
|
| 200 |
+
competitor.id: competitor.request_model
|
| 201 |
+
for competitor in competitors
|
| 202 |
+
if competitor.family == 'decision'
|
| 203 |
+
and competitor.ready
|
| 204 |
+
and (competitor_ids is None or competitor.id in competitor_ids)
|
| 205 |
+
}
|
| 206 |
+
if mode == "direct":
|
| 207 |
+
by_model = await direct_readiness(selected.values())
|
| 208 |
+
else:
|
| 209 |
+
wire_by_canonical = {
|
| 210 |
+
item['repo_id']: wire for wire, item in registry.items()
|
| 211 |
+
}
|
| 212 |
+
wires = tuple(dict.fromkeys(
|
| 213 |
+
wire_by_canonical[model]
|
| 214 |
+
for model in selected.values()
|
| 215 |
+
if model in wire_by_canonical
|
| 216 |
+
))
|
| 217 |
+
observations = await asyncio.gather(
|
| 218 |
+
*(relays[wire].status() for wire in wires),
|
| 219 |
+
return_exceptions=True,
|
| 220 |
+
)
|
| 221 |
+
by_model = {
|
| 222 |
+
registry[wire]['repo_id']: isinstance(observed, dict)
|
| 223 |
+
and observed.get('loaded') is True
|
| 224 |
+
for wire, observed in zip(wires, observations, strict=True)
|
| 225 |
+
}
|
| 226 |
+
return {
|
| 227 |
+
competitor_id: by_model.get(model, False)
|
| 228 |
+
for competitor_id, model in selected.items()
|
| 229 |
+
}
|
| 230 |
+
|
| 231 |
+
async def input_for(request, *, public_contract=None):
|
| 232 |
+
payload, request_bytes = await read_json(
|
| 233 |
+
request, limit=2 * 1024 * 1024, with_size=True,
|
| 234 |
+
)
|
| 235 |
+
return prepare_input(payload, request_bytes, public_contract=public_contract)
|
| 236 |
+
|
| 237 |
+
def prepare_input(payload, request_bytes, *, public_contract=None):
|
| 238 |
+
if request_bytes > 256 * 1024 and (
|
| 239 |
+
not isinstance(payload, dict) or 'states' not in payload
|
| 240 |
+
):
|
| 241 |
+
raise HTTPException(413, "Request is too large. No field is truncated.")
|
| 242 |
+
if public_contract is not None:
|
| 243 |
+
expected = {'model', 'questions', 'states' if public_contract == 'batch' else 'state'}
|
| 244 |
+
if not isinstance(payload, dict) or set(payload) != expected:
|
| 245 |
+
raise HTTPException(422, f"Provide exactly {', '.join(sorted(expected))}.")
|
| 246 |
+
requested_model = payload.get('model') if isinstance(payload, dict) else None
|
| 247 |
+
model = (
|
| 248 |
+
resolve_canonical_model(requested_model, registry)
|
| 249 |
+
if public_contract is not None
|
| 250 |
+
else resolve_model(
|
| 251 |
+
requested_model if mode == "direct" else requested_model or default_model,
|
| 252 |
+
registry,
|
| 253 |
+
)
|
| 254 |
+
)
|
| 255 |
+
selected = select(model)
|
| 256 |
+
public_model = registry[model]['repo_id']
|
| 257 |
+
canonical_payload = (
|
| 258 |
+
dict(payload, model=public_model) if isinstance(payload, dict) else payload
|
| 259 |
+
)
|
| 260 |
+
if isinstance(payload, dict):
|
| 261 |
+
payload = dict(payload, model=model)
|
| 262 |
+
try:
|
| 263 |
+
records = to_records(payload, model=model)
|
| 264 |
+
except (ValueError, TypeError, UnicodeError, RecursionError) as exc:
|
| 265 |
+
if (
|
| 266 |
+
mode == "direct"
|
| 267 |
+
and "Expanded context/question input exceeds" in str(exc)
|
| 268 |
+
):
|
| 269 |
+
raise HTTPException(413, str(exc)) from None
|
| 270 |
+
raise HTTPException(422, str(exc)) from None
|
| 271 |
+
return payload, records, selected, public_model, canonical_payload, model
|
| 272 |
+
|
| 273 |
+
async def queued_result(selected, payload, public_model, public_contract, is_disconnected=None):
|
| 274 |
+
job = await selected.submit(payload, keep_until_read=True)
|
| 275 |
+
deadline = time.monotonic() + 45
|
| 276 |
+
try:
|
| 277 |
+
while time.monotonic() < deadline:
|
| 278 |
+
remaining = deadline - time.monotonic()
|
| 279 |
+
current = await selected.wait(job["id"], max(0.0, min(0.25, remaining)))
|
| 280 |
+
if current["status"] == "succeeded":
|
| 281 |
+
if current['result'].get('model') == public_model:
|
| 282 |
+
return current['result']
|
| 283 |
+
if public_contract is not None:
|
| 284 |
+
return public_response(current['result'], public_model=public_model,
|
| 285 |
+
batch=public_contract == 'batch')
|
| 286 |
+
return sdk_response(current['result'])
|
| 287 |
+
if current["status"] not in Relay.ACTIVE:
|
| 288 |
+
raise HTTPException(503, current.get("detail", "The request did not finish"))
|
| 289 |
+
if is_disconnected is not None and await is_disconnected():
|
| 290 |
+
raise asyncio.CancelledError()
|
| 291 |
+
raise HTTPException(504, "Request exceeded the synchronous wait. Use the Studio asynchronous job API.")
|
| 292 |
+
finally:
|
| 293 |
+
await selected.release(job["id"])
|
| 294 |
+
|
| 295 |
+
async def arena_turn(payload):
|
| 296 |
+
"""Run one Tetris turn as a strict /v1/systemone request on the selected model's queue."""
|
| 297 |
+
try:
|
| 298 |
+
payload, _, selected, public_model, _, _ = prepare_input(
|
| 299 |
+
payload, len(json.dumps(payload).encode()), public_contract='single',
|
| 300 |
+
)
|
| 301 |
+
return await queued_result(selected, payload, public_model, 'single')
|
| 302 |
+
except (HTTPException, RelayError) as exc:
|
| 303 |
+
raise ArenaUpstreamError("The selected model could not complete this turn.") from exc
|
| 304 |
+
|
| 305 |
+
if tetris_manager is None:
|
| 306 |
+
tetris_adapter = tetris_adapter or HTTPDecisionAdapter.from_environment(
|
| 307 |
+
# The archive reads no upstream URL or key from the environment.
|
| 308 |
+
{} if archive else None,
|
| 309 |
+
allowed_local_models=canonical_models,
|
| 310 |
+
include_cloud=studio_generation != "2.0" and not archive,
|
| 311 |
+
local_endpoints=(
|
| 312 |
+
direct_gateway.local_systemone_endpoints() if mode == "direct" else None
|
| 313 |
+
),
|
| 314 |
+
direct_gateway=direct_gateway if mode == "direct" else None,
|
| 315 |
+
decision_queue=(
|
| 316 |
+
arena_turn if mode == "pull_queue"
|
| 317 |
+
else archive.arena_turn if archive else None
|
| 318 |
+
),
|
| 319 |
+
)
|
| 320 |
+
tetris_manager = RaceManager.from_environment(
|
| 321 |
+
tetris_adapter, archive.race_environment(os.environ) if archive else None,
|
| 322 |
+
)
|
| 323 |
+
api.state.tetris = tetris_manager
|
| 324 |
+
|
| 325 |
+
@api.exception_handler(RelayError)
|
| 326 |
+
async def relay_error(request, exc):
|
| 327 |
+
return JSONResponse({"detail": exc.message}, status_code=exc.code,
|
| 328 |
+
headers={"Retry-After": "2"} if exc.code in {429, 503} else None)
|
| 329 |
+
|
| 330 |
+
@api.exception_handler(DirectGatewayError)
|
| 331 |
+
async def direct_error(request, exc):
|
| 332 |
+
headers = {"Retry-After": exc.retry_after} if exc.retry_after else None
|
| 333 |
+
return JSONResponse({"detail": exc.message}, status_code=exc.code, headers=headers)
|
| 334 |
+
|
| 335 |
+
@api.get("/api/status")
|
| 336 |
+
async def status(model: str = default_model):
|
| 337 |
+
model = resolve_model(model, registry)
|
| 338 |
+
selected = select(model)
|
| 339 |
+
if archive:
|
| 340 |
+
return {
|
| 341 |
+
"configured": True,
|
| 342 |
+
"loaded": True,
|
| 343 |
+
"phase": "ready",
|
| 344 |
+
"model": model,
|
| 345 |
+
"manifest_sha256": registry[model]['manifest_sha256'],
|
| 346 |
+
"revision": registry[model].get('revision'),
|
| 347 |
+
"complete_input_tokens": registry[model]['complete_input_tokens'],
|
| 348 |
+
"context_batch": True,
|
| 349 |
+
"backend": "cache",
|
| 350 |
+
"live_inference": False,
|
| 351 |
+
"running": 0,
|
| 352 |
+
"queued": 0,
|
| 353 |
+
"readiness": "recorded results only; this archive calls no model",
|
| 354 |
+
}
|
| 355 |
+
if mode == "direct":
|
| 356 |
+
observed = await direct_gateway.probe(registry[model]['repo_id'])
|
| 357 |
+
loaded = observed['loaded']
|
| 358 |
+
artifact = observed['artifact'] or {}
|
| 359 |
+
return {
|
| 360 |
+
"configured": True,
|
| 361 |
+
"loaded": loaded,
|
| 362 |
+
"phase": "ready" if loaded else "unavailable",
|
| 363 |
+
"model": model,
|
| 364 |
+
"manifest_sha256": registry[model]['manifest_sha256'],
|
| 365 |
+
"revision": registry[model]['revision'],
|
| 366 |
+
"loaded_revision": artifact.get('revision'),
|
| 367 |
+
"loaded_manifest_sha256": artifact.get('manifest_sha256'),
|
| 368 |
+
"loaded_content_sha256": artifact.get('content_sha256'),
|
| 369 |
+
"complete_input_tokens": registry[model]['complete_input_tokens'],
|
| 370 |
+
"context_batch": loaded,
|
| 371 |
+
"backend": "direct",
|
| 372 |
+
"live_inference": loaded,
|
| 373 |
+
"running": False,
|
| 374 |
+
"queued": 0,
|
| 375 |
+
}
|
| 376 |
+
return await selected.status() if selected else api.state.engine.status()
|
| 377 |
+
|
| 378 |
+
@api.get("/api/ready")
|
| 379 |
+
async def ready():
|
| 380 |
+
"""Return HTTP 503 when any configured model is unavailable."""
|
| 381 |
+
if archive:
|
| 382 |
+
loaded = {model: True for model in canonical_models}
|
| 383 |
+
elif mode == 'direct':
|
| 384 |
+
loaded = await direct_readiness(canonical_models)
|
| 385 |
+
elif relays:
|
| 386 |
+
observations = await asyncio.gather(
|
| 387 |
+
*(relay.status() for relay in relays.values()),
|
| 388 |
+
return_exceptions=True,
|
| 389 |
+
)
|
| 390 |
+
loaded = {
|
| 391 |
+
registry[key]['repo_id']: isinstance(observed, dict)
|
| 392 |
+
and observed.get('loaded') is True
|
| 393 |
+
for key, observed in zip(relays, observations, strict=True)
|
| 394 |
+
}
|
| 395 |
+
else:
|
| 396 |
+
observed = api.state.engine.status()
|
| 397 |
+
loaded = {
|
| 398 |
+
registry[default_model]['repo_id']: observed.get('loaded') is True
|
| 399 |
+
}
|
| 400 |
+
healthy = bool(loaded) and all(loaded.values())
|
| 401 |
+
return JSONResponse(
|
| 402 |
+
{
|
| 403 |
+
'backend': mode,
|
| 404 |
+
'status': 'ready' if healthy else 'unavailable',
|
| 405 |
+
'models': loaded,
|
| 406 |
+
},
|
| 407 |
+
status_code=200 if healthy else 503,
|
| 408 |
+
)
|
| 409 |
+
|
| 410 |
+
@api.get("/api/examples")
|
| 411 |
+
def examples():
|
| 412 |
+
return json.loads((ROOT / "examples.json").read_text())
|
| 413 |
+
|
| 414 |
+
@api.get("/api/tetris/config")
|
| 415 |
+
async def tetris_config():
|
| 416 |
+
# Credentials and upstream URLs intentionally never enter this response.
|
| 417 |
+
config = api.state.tetris.public_config()
|
| 418 |
+
if studio_generation:
|
| 419 |
+
visible_models = {item['repo_id'] for item in studio_registry.values()}
|
| 420 |
+
visible_competitors = {
|
| 421 |
+
competitor.id for competitor in api.state.tetris.catalog()
|
| 422 |
+
if competitor.family == 'decision'
|
| 423 |
+
and competitor.request_model in visible_models
|
| 424 |
+
}
|
| 425 |
+
config['competitors'] = [
|
| 426 |
+
competitor for competitor in config.get('competitors', ())
|
| 427 |
+
if (competitor['family'] != 'decision'
|
| 428 |
+
or competitor['id'] in visible_competitors)
|
| 429 |
+
and (studio_generation != "2.0" or competitor['family'] != 'cloud')
|
| 430 |
+
]
|
| 431 |
+
readiness = await tetris_model_readiness({
|
| 432 |
+
competitor['id'] for competitor in config.get('competitors', ())
|
| 433 |
+
})
|
| 434 |
+
for competitor in config.get('competitors', ()):
|
| 435 |
+
if competitor['id'] in readiness:
|
| 436 |
+
competitor['ready'] = competitor['ready'] and readiness[competitor['id']]
|
| 437 |
+
return config
|
| 438 |
+
|
| 439 |
+
@api.post("/api/tetris/races", status_code=201)
|
| 440 |
+
async def create_tetris_race(request: Request):
|
| 441 |
+
payload = await read_json(request)
|
| 442 |
+
if archive and isinstance(payload, dict) and not archive.replays_race(payload):
|
| 443 |
+
raise HTTPException(422, archive.race_notice())
|
| 444 |
+
if mode in {'direct', 'pull_queue'} and isinstance(payload, dict):
|
| 445 |
+
selected_ids = {
|
| 446 |
+
value for side in ('left', 'right')
|
| 447 |
+
if isinstance(value := payload.get(side), str)
|
| 448 |
+
}
|
| 449 |
+
readiness = await tetris_model_readiness(selected_ids)
|
| 450 |
+
if any(not ready for ready in readiness.values()):
|
| 451 |
+
raise HTTPException(503, "The selected Decision model is unavailable.")
|
| 452 |
+
try:
|
| 453 |
+
race = await api.state.tetris.create(payload)
|
| 454 |
+
except ArenaValidationError as exc:
|
| 455 |
+
headers = (
|
| 456 |
+
{"Retry-After": str(exc.retry_after)}
|
| 457 |
+
if isinstance(exc, ArenaBusyError)
|
| 458 |
+
else None
|
| 459 |
+
)
|
| 460 |
+
raise HTTPException(exc.status_code, str(exc), headers=headers) from None
|
| 461 |
+
return {
|
| 462 |
+
"id": race.id,
|
| 463 |
+
"status": race.status,
|
| 464 |
+
"events_url": f"/api/tetris/races/{race.id}/events",
|
| 465 |
+
"result_url": f"/api/tetris/races/{race.id}",
|
| 466 |
+
}
|
| 467 |
+
|
| 468 |
+
def get_tetris_race(race_id):
|
| 469 |
+
try:
|
| 470 |
+
return api.state.tetris.get(race_id)
|
| 471 |
+
except KeyError:
|
| 472 |
+
raise HTTPException(404, "Race not found") from None
|
| 473 |
+
|
| 474 |
+
@api.get("/api/tetris/races/{race_id}")
|
| 475 |
+
async def tetris_race(race_id: str):
|
| 476 |
+
return get_tetris_race(race_id).snapshot()
|
| 477 |
+
|
| 478 |
+
@api.get("/api/tetris/races/{race_id}/trace")
|
| 479 |
+
async def tetris_race_trace(race_id: str):
|
| 480 |
+
return get_tetris_race(race_id).snapshot(include_traces=True)
|
| 481 |
+
|
| 482 |
+
@api.get("/api/tetris/races/{race_id}/events")
|
| 483 |
+
async def tetris_race_events(race_id: str, request: Request, after: int = 0):
|
| 484 |
+
race = get_tetris_race(race_id)
|
| 485 |
+
header_cursor = request.headers.get("last-event-id")
|
| 486 |
+
if header_cursor:
|
| 487 |
+
try:
|
| 488 |
+
after = max(after, int(header_cursor))
|
| 489 |
+
except ValueError:
|
| 490 |
+
raise HTTPException(400, "Last-Event-ID must be an integer") from None
|
| 491 |
+
if after < 0:
|
| 492 |
+
raise HTTPException(400, "after must not be negative")
|
| 493 |
+
|
| 494 |
+
async def stream():
|
| 495 |
+
async for event in race.iter_events(after):
|
| 496 |
+
yield encode_sse(event)
|
| 497 |
+
|
| 498 |
+
return StreamingResponse(
|
| 499 |
+
stream(),
|
| 500 |
+
media_type="text/event-stream",
|
| 501 |
+
headers={
|
| 502 |
+
"Cache-Control": "no-store",
|
| 503 |
+
"Connection": "keep-alive",
|
| 504 |
+
"X-Accel-Buffering": "no",
|
| 505 |
+
},
|
| 506 |
+
)
|
| 507 |
+
|
| 508 |
+
@api.delete("/api/tetris/races/{race_id}", status_code=204)
|
| 509 |
+
async def cancel_tetris_race(race_id: str):
|
| 510 |
+
try:
|
| 511 |
+
await api.state.tetris.cancel(race_id)
|
| 512 |
+
except KeyError:
|
| 513 |
+
raise HTTPException(404, "Race not found") from None
|
| 514 |
+
|
| 515 |
+
@api.get("/v1/models")
|
| 516 |
+
def models():
|
| 517 |
+
synchronous_wait = (
|
| 518 |
+
getattr(direct_gateway, "timeout_seconds", 45)
|
| 519 |
+
if mode == "direct"
|
| 520 |
+
else 45
|
| 521 |
+
)
|
| 522 |
+
limits = {
|
| 523 |
+
"request_bytes": 256 * 1024,
|
| 524 |
+
"expanded_input_bytes": 16 * 1024 * 1024,
|
| 525 |
+
"questions": None,
|
| 526 |
+
"contexts": None,
|
| 527 |
+
# direct capacity belongs to each live model instance, not the
|
| 528 |
+
# retired Gateway's fixed eight-request/eight-row profile.
|
| 529 |
+
"active_requests_per_model": None if mode == "direct" else 8,
|
| 530 |
+
"gpu_microbatch": None if mode == "direct" else 8,
|
| 531 |
+
"synchronous_wait_seconds": synchronous_wait,
|
| 532 |
+
}
|
| 533 |
+
result = {
|
| 534 |
+
"models": public_models(studio_registry),
|
| 535 |
+
"limits": limits,
|
| 536 |
+
}
|
| 537 |
+
if mode != "direct":
|
| 538 |
+
result["default"] = registry[default_model]['repo_id']
|
| 539 |
+
return result
|
| 540 |
+
|
| 541 |
+
@api.post("/v1/systemone")
|
| 542 |
+
async def evaluate_systemone(request: Request):
|
| 543 |
+
return await evaluate(request, public_contract='single')
|
| 544 |
+
|
| 545 |
+
@api.post("/v1/systemone/batches")
|
| 546 |
+
async def evaluate_decision_batches(request: Request):
|
| 547 |
+
return await evaluate(request, public_contract='batch')
|
| 548 |
+
|
| 549 |
+
@api.post("/api/evaluate")
|
| 550 |
+
async def evaluate_studio(request: Request):
|
| 551 |
+
return await evaluate(request, public_contract=None)
|
| 552 |
+
|
| 553 |
+
async def evaluate(request: Request, *, public_contract: str | None):
|
| 554 |
+
payload, records, selected, public_model, canonical_payload, _ = await input_for(
|
| 555 |
+
request, public_contract=public_contract
|
| 556 |
+
)
|
| 557 |
+
if archive:
|
| 558 |
+
recorded = (
|
| 559 |
+
archive.turn(canonical_payload) if public_contract == 'single'
|
| 560 |
+
else archive.answer(canonical_payload) if public_contract is None
|
| 561 |
+
else None
|
| 562 |
+
)
|
| 563 |
+
if recorded is None:
|
| 564 |
+
raise HTTPException(410, ARCHIVE_NOTICE)
|
| 565 |
+
return recorded
|
| 566 |
+
if mode == "direct":
|
| 567 |
+
return await direct_gateway.evaluate(
|
| 568 |
+
canonical_payload,
|
| 569 |
+
batch='states' in canonical_payload,
|
| 570 |
+
)
|
| 571 |
+
if selected:
|
| 572 |
+
return await queued_result(selected, payload, public_model, public_contract,
|
| 573 |
+
request.is_disconnected)
|
| 574 |
+
try:
|
| 575 |
+
result = await run_in_threadpool(api.state.engine.evaluate, payload, records)
|
| 576 |
+
if public_contract is not None:
|
| 577 |
+
return public_response(result, public_model=public_model,
|
| 578 |
+
batch=public_contract == 'batch')
|
| 579 |
+
return sdk_response(result)
|
| 580 |
+
except Unavailable as exc:
|
| 581 |
+
raise HTTPException(503, str(exc)) from None
|
| 582 |
+
except Busy as exc:
|
| 583 |
+
raise HTTPException(429, str(exc), headers={"Retry-After": "2"}) from None
|
| 584 |
+
except ValueError as exc:
|
| 585 |
+
if "tokens" in str(exc) or "room for state" in str(exc):
|
| 586 |
+
raise HTTPException(422, str(exc)) from None
|
| 587 |
+
logger.warning("Native validation failure: %s", type(exc).__name__)
|
| 588 |
+
raise HTTPException(503, "The configured native package could not be validated. Check the server configuration.") from None
|
| 589 |
+
except Exception as exc:
|
| 590 |
+
logger.warning("Inference failure: %s", type(exc).__name__)
|
| 591 |
+
raise HTTPException(503, "The model could not finish this request. No prediction was substituted.") from None
|
| 592 |
+
|
| 593 |
+
if relays:
|
| 594 |
+
@api.post("/api/jobs", status_code=202)
|
| 595 |
+
async def submit(request: Request):
|
| 596 |
+
payload, _, selected, _, _, _ = await input_for(request)
|
| 597 |
+
return await selected.submit(payload)
|
| 598 |
+
|
| 599 |
+
@api.get("/api/jobs/{job_id}")
|
| 600 |
+
async def read_job(job_id: str, model: str = default_model):
|
| 601 |
+
return await select(resolve_model(model, registry)).read(job_id)
|
| 602 |
+
|
| 603 |
+
@api.delete("/api/jobs/{job_id}")
|
| 604 |
+
async def cancel_job(job_id: str, model: str = default_model):
|
| 605 |
+
return await select(resolve_model(model, registry)).cancel(job_id)
|
| 606 |
+
|
| 607 |
+
async def worker_body(request, required, optional=(), *, limit=512 * 1024):
|
| 608 |
+
# All queues use the same private trust boundary, but identities are mandatory.
|
| 609 |
+
# Authentication precedes reading user data. The token is never returned or logged.
|
| 610 |
+
next(iter(relays.values())).authenticate(request.headers.get("authorization", ""))
|
| 611 |
+
body = await read_json(request, limit=limit)
|
| 612 |
+
required = set(required) | {"model", "manifest_sha256"}
|
| 613 |
+
if not isinstance(body, dict) or set(body) - required - set(optional) or required - set(body):
|
| 614 |
+
raise HTTPException(422, "Invalid worker request fields")
|
| 615 |
+
selected = select(body['model'])
|
| 616 |
+
selected.authenticate(request.headers.get("authorization", ""))
|
| 617 |
+
if body['manifest_sha256'] != selected.manifest:
|
| 618 |
+
raise HTTPException(409, "Worker model and native manifest do not match this queue")
|
| 619 |
+
return body, selected
|
| 620 |
+
|
| 621 |
+
@api.post("/internal/worker/heartbeat")
|
| 622 |
+
async def heartbeat(request: Request):
|
| 623 |
+
body, selected = await worker_body(request, {"worker_id", "phase"}, {"capabilities"})
|
| 624 |
+
if any(other is not selected and other.worker and other.worker['id'] == body['worker_id']
|
| 625 |
+
for other in relays.values()):
|
| 626 |
+
raise HTTPException(409, "A worker identity cannot serve multiple models")
|
| 627 |
+
return await selected.heartbeat(body["worker_id"], body["manifest_sha256"], body["phase"], body.get("capabilities"))
|
| 628 |
+
|
| 629 |
+
@api.post("/internal/worker/claim")
|
| 630 |
+
async def claim(request: Request):
|
| 631 |
+
body, selected = await worker_body(request, {"worker_id"}, {"wait_seconds"})
|
| 632 |
+
return await selected.claim(body["worker_id"], body.get("wait_seconds", 20))
|
| 633 |
+
|
| 634 |
+
@api.post("/internal/worker/result")
|
| 635 |
+
async def complete(request: Request):
|
| 636 |
+
# Leave room for the worker/lease envelope around an 8 MiB
|
| 637 |
+
# attested runtime response.
|
| 638 |
+
body, selected = await worker_body(request, {"worker_id", "id", "lease_token"}, {"result", "error_code"}, limit=8 * 1024 * 1024 + 64 * 1024)
|
| 639 |
+
return await selected.complete(body["worker_id"], body["id"], body["lease_token"],
|
| 640 |
+
body.get("result"), body.get("error_code"))
|
| 641 |
+
|
| 642 |
+
@api.middleware("http")
|
| 643 |
+
async def response_headers(request, call_next):
|
| 644 |
+
response = await call_next(request)
|
| 645 |
+
response.headers["X-Content-Type-Options"] = "nosniff"
|
| 646 |
+
response.headers["Referrer-Policy"] = "no-referrer"
|
| 647 |
+
if request.url.path.startswith(("/api/", "/v1/", "/internal/", "/tetris/")):
|
| 648 |
+
response.headers["Cache-Control"] = "no-store"
|
| 649 |
+
return response
|
| 650 |
+
|
| 651 |
+
api.mount("/", StaticFiles(directory=ROOT / "static", html=True), name="studio")
|
| 652 |
+
return api
|
| 653 |
+
|
| 654 |
+
|
| 655 |
+
app = create_app()
|
archive_cache.py
ADDED
|
@@ -0,0 +1,85 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Archive mode: every answer is a result recorded on the live Decision 2.0 workers.
|
| 2 |
+
|
| 3 |
+
In this mode the Space calls no model, queue or cloud API. The Studio examples answer
|
| 4 |
+
from cache/studio.json and the Arena replays the turns of cache/arena.json (each turn
|
| 5 |
+
waits as long as it took when it was recorded); any other request is refused with a
|
| 6 |
+
pointer to Decision 3.0.
|
| 7 |
+
"""
|
| 8 |
+
import asyncio
|
| 9 |
+
import hashlib
|
| 10 |
+
import json
|
| 11 |
+
from pathlib import Path
|
| 12 |
+
|
| 13 |
+
from tetris_arena import ArenaUpstreamError
|
| 14 |
+
|
| 15 |
+
NOTICE = (
|
| 16 |
+
"Decision 2.0 Studio is an archive: it replays the recorded answers of its curated "
|
| 17 |
+
"examples and calls no model. Run your own requests live on Decision 3.0 at "
|
| 18 |
+
"https://huggingface.co/spaces/vllm-sr/Decision-3.0, or with a free API key at "
|
| 19 |
+
"https://decision.vllm-sr.ai."
|
| 20 |
+
)
|
| 21 |
+
RACE_NOTICE = (
|
| 22 |
+
"This archived Arena replays the recorded races of seed {seed}, up to {steps} steps. "
|
| 23 |
+
"Live races and your own requests: Decision 3.0 at "
|
| 24 |
+
"https://huggingface.co/spaces/vllm-sr/Decision-3.0 or https://decision.vllm-sr.ai."
|
| 25 |
+
)
|
| 26 |
+
# A recorded turn is replayed with its recorded duration, bounded so a slow recording
|
| 27 |
+
# cannot stall a race.
|
| 28 |
+
MAX_TURN_SECONDS = 10.0
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
def request_key(payload):
|
| 32 |
+
"""The cache key of a request: its JSON with sorted keys, so field order is irrelevant."""
|
| 33 |
+
text = json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
|
| 34 |
+
return hashlib.sha256(text.encode()).hexdigest()
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
class ArchiveCache:
|
| 38 |
+
def __init__(self, answers, turns, *, seed, steps, sleep=asyncio.sleep):
|
| 39 |
+
self.answers = answers
|
| 40 |
+
self.turns = turns
|
| 41 |
+
self.seed = seed
|
| 42 |
+
self.steps = steps
|
| 43 |
+
self._sleep = sleep
|
| 44 |
+
|
| 45 |
+
@classmethod
|
| 46 |
+
def load(cls, root):
|
| 47 |
+
root = Path(root)
|
| 48 |
+
studio = json.loads((root / "cache" / "studio.json").read_text())
|
| 49 |
+
arena = json.loads((root / "cache" / "arena.json").read_text())
|
| 50 |
+
return cls(studio["answers"], arena["turns"], seed=arena["seed"], steps=arena["steps"])
|
| 51 |
+
|
| 52 |
+
def answer(self, payload):
|
| 53 |
+
"""The recorded Studio answer of this exact request, or None."""
|
| 54 |
+
entry = self.answers.get(request_key(payload))
|
| 55 |
+
return None if entry is None else entry["answer"]
|
| 56 |
+
|
| 57 |
+
def turn(self, payload):
|
| 58 |
+
"""The recorded answer of this exact Arena turn, or None."""
|
| 59 |
+
entry = self.turns.get(request_key(payload))
|
| 60 |
+
return None if entry is None else entry["document"]
|
| 61 |
+
|
| 62 |
+
def race_notice(self):
|
| 63 |
+
return RACE_NOTICE.format(seed=self.seed, steps=self.steps)
|
| 64 |
+
|
| 65 |
+
def replays_race(self, request):
|
| 66 |
+
"""True when a race request asks for the recorded seed (the default) and steps."""
|
| 67 |
+
seed = request.get("seed", self.seed)
|
| 68 |
+
if isinstance(seed, bool) or seed != self.seed:
|
| 69 |
+
return False
|
| 70 |
+
steps = request.get("max_steps")
|
| 71 |
+
return steps is None or (isinstance(steps, int) and not isinstance(steps, bool)
|
| 72 |
+
and 1 <= steps <= self.steps)
|
| 73 |
+
|
| 74 |
+
def race_environment(self, environ):
|
| 75 |
+
"""The Arena's limits in archive mode: the recorded number of steps."""
|
| 76 |
+
return dict(environ, TETRIS_MAX_STEPS=str(self.steps),
|
| 77 |
+
TETRIS_FIRST_FAILURE_GUARD=str(self.steps))
|
| 78 |
+
|
| 79 |
+
async def arena_turn(self, payload):
|
| 80 |
+
"""One Arena turn, replayed from the recording."""
|
| 81 |
+
turn = self.turns.get(request_key(payload))
|
| 82 |
+
if turn is None:
|
| 83 |
+
raise ArenaUpstreamError(self.race_notice())
|
| 84 |
+
await self._sleep(min(MAX_TURN_SECONDS, max(0.0, turn["ms"] / 1000)))
|
| 85 |
+
return json.loads(json.dumps(turn["document"]))
|
cache/arena.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
cache/studio.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
contract.py
ADDED
|
@@ -0,0 +1,221 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""System One shaped inputs; this adapter makes no TypeSafe model claims."""
|
| 2 |
+
import json
|
| 3 |
+
import math
|
| 4 |
+
|
| 5 |
+
MODEL = "decision-kai"
|
| 6 |
+
MAX_EXPANDED_INPUT_BYTES = 16 * 1024 * 1024
|
| 7 |
+
MAX_REQUEST_BYTES = 256 * 1024
|
| 8 |
+
MAX_BATCH_REQUEST_BYTES = 2 * 1024 * 1024
|
| 9 |
+
MAX_BATCH_DECISIONS = 1024
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
def content(value, label):
|
| 13 |
+
if isinstance(value, str):
|
| 14 |
+
if not value.strip():
|
| 15 |
+
raise ValueError(f"{label} must not be empty")
|
| 16 |
+
return value
|
| 17 |
+
if isinstance(value, (dict, list)):
|
| 18 |
+
return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":"), allow_nan=False)
|
| 19 |
+
raise ValueError(f"{label} must be text, an object, or an array")
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
def identifier(value, label, *, max_length=None):
|
| 23 |
+
if not isinstance(value, str) or not value.strip():
|
| 24 |
+
raise ValueError(f"{label} must be a nonempty string")
|
| 25 |
+
if max_length is not None and len(value) > max_length:
|
| 26 |
+
raise ValueError(f"{label} must be at most {max_length} characters")
|
| 27 |
+
return value
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def _single_records(body, *, model=MODEL):
|
| 31 |
+
if not isinstance(body, dict) or set(body) - {"state", "questions", "model"}:
|
| 32 |
+
raise ValueError("Use state, questions, and optional model only")
|
| 33 |
+
if body.get("model", MODEL) != model:
|
| 34 |
+
raise ValueError("Request model does not match the selected model")
|
| 35 |
+
state = content(body.get("state"), "state")
|
| 36 |
+
questions = body.get("questions")
|
| 37 |
+
if not isinstance(questions, dict) or not questions:
|
| 38 |
+
raise ValueError("Provide at least one named question")
|
| 39 |
+
records = []
|
| 40 |
+
for index, (qid, item) in enumerate(questions.items()):
|
| 41 |
+
identifier(qid, "Question ID")
|
| 42 |
+
if not isinstance(item, dict) or set(item) - {"type", "instructions", "criteria"}:
|
| 43 |
+
raise ValueError(f"{qid}: use type, instructions, criteria only")
|
| 44 |
+
kind = item.get("type")
|
| 45 |
+
if kind not in ("choice", "noul", "score"):
|
| 46 |
+
raise ValueError(f"{qid}: type must be choice, noul, or score")
|
| 47 |
+
q = {
|
| 48 |
+
"id": qid,
|
| 49 |
+
"type": kind.capitalize(),
|
| 50 |
+
"text": content(item.get("instructions"), qid + ".instructions"),
|
| 51 |
+
}
|
| 52 |
+
criteria = item.get("criteria")
|
| 53 |
+
if kind == "choice":
|
| 54 |
+
if not isinstance(criteria, dict) or not 2 <= len(criteria) <= 255:
|
| 55 |
+
raise ValueError(f"{qid}: Choice requires 2–255 options")
|
| 56 |
+
q["options"] = []
|
| 57 |
+
for name, description in criteria.items():
|
| 58 |
+
identifier(name, "Option name")
|
| 59 |
+
# Official Choice option names ARE semantic; opaque native IDs are not.
|
| 60 |
+
text = name if description is None else name + ": " + content(description, qid + ".criteria")
|
| 61 |
+
q["options"].append({"id": name, "text": text})
|
| 62 |
+
elif kind == "score":
|
| 63 |
+
if not isinstance(criteria, list) or not 2 <= len(criteria) <= 10:
|
| 64 |
+
raise ValueError(f"{qid}: Score requires 2–10 ordered levels")
|
| 65 |
+
q["levels"] = [{"id": str(i), "value": i, "text": content(v, qid + ".criteria")} for i, v in enumerate(criteria)]
|
| 66 |
+
elif criteria is not None:
|
| 67 |
+
if not isinstance(criteria, dict) or set(criteria) - {"false", "true"}:
|
| 68 |
+
raise ValueError(f"{qid}: Noul criteria accept false and true only")
|
| 69 |
+
for key in ("false", "true"):
|
| 70 |
+
if criteria.get(key) is not None:
|
| 71 |
+
q[key + "_criterion"] = content(criteria[key], qid + ".criteria." + key)
|
| 72 |
+
records.append({"id": f"studio:{index}", "state_text": state, "question": q})
|
| 73 |
+
return records
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
def _single_response(body, records, predictions, *, model=MODEL, complete_input_tokens=1024, precision="fp32"):
|
| 77 |
+
if len(predictions) != len(records):
|
| 78 |
+
raise RuntimeError("Incomplete model response")
|
| 79 |
+
answers = {}
|
| 80 |
+
total_tokens = 0
|
| 81 |
+
for row, pred in zip(records, predictions):
|
| 82 |
+
q = row["question"]
|
| 83 |
+
kind = q["type"].lower()
|
| 84 |
+
expected_ids = (["no", "yes"] if kind == "noul" else [o["id"] for o in q["options" if kind == "choice" else "levels"]])
|
| 85 |
+
if (pred["id"] != row["id"] or pred["question_id"] != q["id"] or pred["candidate_ids"] != expected_ids
|
| 86 |
+
or pred["type"].lower() != kind or pred["state_tokens_original"] != pred["state_tokens_kept"]
|
| 87 |
+
or not 1 <= pred["input_tokens"] <= complete_input_tokens):
|
| 88 |
+
raise RuntimeError("Model response identity or complete-input mismatch")
|
| 89 |
+
p = pred["probabilities"]
|
| 90 |
+
if (len(p) != len(expected_ids) or not all(math.isfinite(x) and 0 <= x <= 1 for x in p)
|
| 91 |
+
or abs(sum(p) - 1) > 2e-5):
|
| 92 |
+
raise RuntimeError("Invalid probability distribution")
|
| 93 |
+
total_tokens += pred["input_tokens"]
|
| 94 |
+
answer = {"type": kind, "input_tokens": pred["input_tokens"]}
|
| 95 |
+
if kind == "noul":
|
| 96 |
+
answer["noul"] = p[1]
|
| 97 |
+
else:
|
| 98 |
+
answer["probabilities"] = dict(zip(expected_ids, p))
|
| 99 |
+
answer["top_probability"] = max(p)
|
| 100 |
+
if kind == "choice":
|
| 101 |
+
answer["choice"] = expected_ids[max(range(len(p)), key=p.__getitem__)]
|
| 102 |
+
else:
|
| 103 |
+
answer["score"] = sum(i * prob for i, prob in enumerate(p))
|
| 104 |
+
answer["legend"] = {str(i): v for i, v in enumerate(body["questions"][q["id"]]["criteria"])}
|
| 105 |
+
answers[q["id"]] = answer
|
| 106 |
+
return {"model": model, "answers": answers, "usage": {"input_tokens": total_tokens, "output_tokens": 0},
|
| 107 |
+
"profile": {"complete_input_tokens": complete_input_tokens, "precision": precision, "status": "preview", "confidence_definition": None}}
|
| 108 |
+
|
| 109 |
+
|
| 110 |
+
def contexts(body, *, model=MODEL):
|
| 111 |
+
"""Validate the whole request before any model work; IDs are bookkeeping only."""
|
| 112 |
+
if not isinstance(body, dict) or set(body) - {"state", "states", "questions", "model"}:
|
| 113 |
+
raise ValueError("Use state OR states, questions, and optional model only")
|
| 114 |
+
if ("state" in body) == ("states" in body):
|
| 115 |
+
raise ValueError("Provide exactly one of state or states")
|
| 116 |
+
if body.get("model", MODEL) != model:
|
| 117 |
+
raise ValueError("Request model does not match the selected model")
|
| 118 |
+
limit = MAX_BATCH_REQUEST_BYTES if "states" in body else MAX_REQUEST_BYTES
|
| 119 |
+
if len(json.dumps(body, ensure_ascii=False, separators=(",", ":"), allow_nan=False).encode("utf-8")) > limit:
|
| 120 |
+
raise ValueError("Request exceeds the endpoint byte limit; no field is truncated")
|
| 121 |
+
questions = body.get("questions")
|
| 122 |
+
if not isinstance(questions, dict) or not questions:
|
| 123 |
+
raise ValueError("Provide at least one named question")
|
| 124 |
+
if "state" in body:
|
| 125 |
+
result = [(None, body["state"])]
|
| 126 |
+
_expanded_budget(result, questions)
|
| 127 |
+
return result
|
| 128 |
+
states = body["states"]
|
| 129 |
+
if not isinstance(states, list) or not states:
|
| 130 |
+
raise ValueError("Provide at least one context")
|
| 131 |
+
if len(states) > MAX_BATCH_DECISIONS or len(questions) > MAX_BATCH_DECISIONS or len(states) * len(questions) > MAX_BATCH_DECISIONS:
|
| 132 |
+
raise ValueError("A batch may contain at most 1024 states, questions, and decisions")
|
| 133 |
+
result, seen = [], set()
|
| 134 |
+
for item in states:
|
| 135 |
+
if not isinstance(item, dict) or set(item) != {"id", "state"}:
|
| 136 |
+
raise ValueError("Each context must contain exactly id and state")
|
| 137 |
+
cid = identifier(item["id"], "Context ID", max_length=128)
|
| 138 |
+
if cid in seen:
|
| 139 |
+
raise ValueError("Context IDs must be unique")
|
| 140 |
+
seen.add(cid)
|
| 141 |
+
result.append((cid, item["state"]))
|
| 142 |
+
_expanded_budget(result, questions)
|
| 143 |
+
return result
|
| 144 |
+
|
| 145 |
+
|
| 146 |
+
|
| 147 |
+
def _expanded_budget(contexts, questions):
|
| 148 |
+
# Bound the Cartesian expansion before constructing records. Count limits
|
| 149 |
+
# are not model limits; repeated long inputs still require bounded work.
|
| 150 |
+
state_bytes = sum(len(content(state, "state").encode("utf-8")) for _, state in contexts)
|
| 151 |
+
question_bytes = len(json.dumps(questions, ensure_ascii=False, separators=(",", ":"), allow_nan=False).encode("utf-8"))
|
| 152 |
+
expanded_bytes = state_bytes * len(questions) + question_bytes * len(contexts)
|
| 153 |
+
if expanded_bytes > MAX_EXPANDED_INPUT_BYTES:
|
| 154 |
+
raise ValueError("Expanded context/question input exceeds 16 MiB; split this workload into requests. No field is truncated.")
|
| 155 |
+
|
| 156 |
+
|
| 157 |
+
def to_records(body, *, model=MODEL):
|
| 158 |
+
cc = contexts(body, model=model)
|
| 159 |
+
if "states" not in body:
|
| 160 |
+
return _single_records(body, model=model)
|
| 161 |
+
per_context = [_single_records({"state": state, "questions": body["questions"], "model": model}, model=model)
|
| 162 |
+
for _, state in cc]
|
| 163 |
+
records = []
|
| 164 |
+
# Question-major order puts different contexts in the same physical B8.
|
| 165 |
+
for qi in range(len(body["questions"])):
|
| 166 |
+
for ci in range(len(cc)):
|
| 167 |
+
row = per_context[ci][qi]
|
| 168 |
+
opaque = f"studio:batch:{ci}:{qi}"
|
| 169 |
+
row["id"] = opaque
|
| 170 |
+
row["question"]["id"] = opaque
|
| 171 |
+
records.append(row)
|
| 172 |
+
return records
|
| 173 |
+
|
| 174 |
+
|
| 175 |
+
def response_answers(body, result):
|
| 176 |
+
"""Return answers in physical flattened order; reject missing/reordered groups."""
|
| 177 |
+
qids = list(body["questions"])
|
| 178 |
+
if "states" not in body:
|
| 179 |
+
if not isinstance(result.get("answers"), dict) or list(result["answers"]) != qids:
|
| 180 |
+
raise ValueError("Question order mismatch")
|
| 181 |
+
return list(result["answers"].values())
|
| 182 |
+
groups = result.get("results")
|
| 183 |
+
if not isinstance(groups, list) or len(groups) != len(body["states"]):
|
| 184 |
+
raise ValueError("Context result count mismatch")
|
| 185 |
+
for context, group in zip(body["states"], groups):
|
| 186 |
+
if (not isinstance(group, dict) or set(group) != {"id", "answers", "usage"}
|
| 187 |
+
or group["id"] != context["id"] or not isinstance(group["answers"], dict)
|
| 188 |
+
or list(group["answers"]) != qids):
|
| 189 |
+
raise ValueError("Context/question result identity or order mismatch")
|
| 190 |
+
return [group["answers"][qid] for qid in qids for group in groups]
|
| 191 |
+
|
| 192 |
+
|
| 193 |
+
def to_response(body, records, predictions, *, model=MODEL, complete_input_tokens=1024, precision="fp32"):
|
| 194 |
+
if records != to_records(body, model=model):
|
| 195 |
+
raise RuntimeError("Request records changed")
|
| 196 |
+
if "states" not in body:
|
| 197 |
+
return _single_response(body, records, predictions, model=model,
|
| 198 |
+
complete_input_tokens=complete_input_tokens, precision=precision)
|
| 199 |
+
if len(predictions) != len(records):
|
| 200 |
+
raise RuntimeError("Incomplete model response")
|
| 201 |
+
cc = body["states"]; qids = list(body["questions"]); groups = []; total = 0
|
| 202 |
+
for ci, context in enumerate(cc):
|
| 203 |
+
rr, pp = [], []
|
| 204 |
+
for qi, qid in enumerate(qids):
|
| 205 |
+
i = qi * len(cc) + ci; row = records[i]; pred = predictions[i]
|
| 206 |
+
if pred["id"] != row["id"] or pred["question_id"] != row["question"]["id"]:
|
| 207 |
+
raise RuntimeError("Flattened context/question identity mismatch")
|
| 208 |
+
rr.append(dict(row, question=dict(row["question"], id=qid)))
|
| 209 |
+
pp.append(dict(pred, question_id=qid))
|
| 210 |
+
one = _single_response(body, rr, pp, model=model, complete_input_tokens=complete_input_tokens, precision=precision)
|
| 211 |
+
groups.append({"id": context["id"], "answers": one["answers"], "usage": one["usage"]})
|
| 212 |
+
total += one["usage"]["input_tokens"]
|
| 213 |
+
return {"model": model, "results": groups, "usage": {"input_tokens": total, "output_tokens": 0},
|
| 214 |
+
"profile": one["profile"]}
|
| 215 |
+
|
| 216 |
+
|
| 217 |
+
def timing_shape(body, records):
|
| 218 |
+
out = {"questions": len(body["questions"]), "batch_size": min(8, len(records))}
|
| 219 |
+
if "states" in body:
|
| 220 |
+
out.update(contexts=len(body["states"]), decisions=len(records))
|
| 221 |
+
return out
|
deploy/decision2/RUNBOOK.md
ADDED
|
@@ -0,0 +1,147 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Decision 2.0 serving runbook
|
| 2 |
+
|
| 3 |
+
The Space (`DECISION_BACKEND=pull_queue`) keeps no weights. Each Decision 2.0
|
| 4 |
+
model runs in its own `vllm-sr-runtime serve` container on a private AMD GPU
|
| 5 |
+
host, on the internal Docker network `decision2-runtime` (no published ports,
|
| 6 |
+
no internet). One HTTP pull worker per model claims that model's queue from
|
| 7 |
+
the Space over outbound HTTPS and calls its runtime (`DECISION_RUNTIME_API=
|
| 8 |
+
vllm_sr_runtime_v1`). Host addresses, the worker token and the host-private
|
| 9 |
+
files below never go into this repository.
|
| 10 |
+
|
| 11 |
+
| Wire ID | Model | Runtime origin (internal network) | Compose profile |
|
| 12 |
+
|---|---|---|---|
|
| 13 |
+
| `decision2-kai` | `vllm-sr/Decision-2.0-Kai-0.6B` | `http://172.31.231.11:8100` | `small` |
|
| 14 |
+
| `decision2-eos` | `vllm-sr/Decision-2.0-Eos-0.8B` | `http://172.31.231.12:8100` | `small` |
|
| 15 |
+
| `decision2-sol` | `vllm-sr/Decision-2.0-Sol-2B` | `http://172.31.231.13:8100` | `small` |
|
| 16 |
+
| `decision2-nox` | `vllm-sr/Decision-2.0-Nox-4B` | `http://172.31.231.14:8100` | `small` |
|
| 17 |
+
| `decision2-lux` | `vllm-sr/Decision-2.0-Lux-9B` | `http://172.31.231.15:8100` | `small` |
|
| 18 |
+
| `decision2-vega` | `vllm-sr/Decision-2.0-Vega-27B` | `http://172.31.231.16:8100` | `vega` |
|
| 19 |
+
|
| 20 |
+
All six fit on one 192 GB MI300X (about 126 GB resident after warm-up). The
|
| 21 |
+
deployed layout gives Vega a host of its own (`--profile vega`, about 79 GB)
|
| 22 |
+
and the other five another host (`--profile small`, about 58 GB). Each host
|
| 23 |
+
starts only its own profile; on the small-model host, Vega's containers stay
|
| 24 |
+
stopped as a rollback target.
|
| 25 |
+
|
| 26 |
+
## Host files
|
| 27 |
+
|
| 28 |
+
| Path (root, mode 600 / 700) | Contents |
|
| 29 |
+
|---|---|
|
| 30 |
+
| `/etc/decision-studio/decision2.env` | Compose interpolation: `DECISION_RUNTIME_IMAGE`, `DECISION_HTTP_WORKER_IMAGE`, `VIDEO_GID`, `RENDER_GID`, `HF_CACHE_DIR`, `AUTOTUNE_DIR`, `FLA_CHOICES_DIR`, `<MODEL>_REVISION` |
|
| 31 |
+
| `/etc/decision-studio/worker-common.env` | `DECISION_STUDIO_URL` (the Space origin), `DECISION_WORKER_TOKEN` (= the Space secret), `DECISION_MODEL_REGISTRY_V2` (= the Space variable, compact JSON of `registry.json`) |
|
| 32 |
+
| `/etc/decision-studio/workers/<wire-id>.env` | `DECISION_RUNTIME_URL` from the table above |
|
| 33 |
+
| `/root/.cache/huggingface/token` | Hugging Face token (read only by the fetch job) |
|
| 34 |
+
| `/srv/decision-studio/hf-cache` | Verified Hugging Face cache, mounted read-only into the runtimes |
|
| 35 |
+
| `/srv/decision-studio/autotune/<model>` | Triton cache per model (`--autotune-cache`) |
|
| 36 |
+
| `/srv/decision-studio/fla-config/<model>` | Released FLA kernel choices (`fla_kernel_choices.py`) |
|
| 37 |
+
| `/opt/decision-studio/compose/decision2.compose.yaml` | Copy of `compose.yaml` from the deployed Space commit |
|
| 38 |
+
| `/opt/decision-studio/bin`, `/opt/decision-studio/reference` | `check_runtime.py`, `fla_kernel_choices.py`, `fetch_pins.py`; `golden_answers.json` and `kernel_choices.json` from the router commit that records the pinned revisions |
|
| 39 |
+
|
| 40 |
+
```bash
|
| 41 |
+
C="docker compose --env-file /etc/decision-studio/decision2.env -f /opt/decision-studio/compose/decision2.compose.yaml"
|
| 42 |
+
P="--profile small" # the profile this host serves (`--profile vega` on the Vega host)
|
| 43 |
+
```
|
| 44 |
+
|
| 45 |
+
## Start, stop, status
|
| 46 |
+
|
| 47 |
+
```bash
|
| 48 |
+
$C $P up -d # start or reconcile everything (workers wait for healthy runtimes)
|
| 49 |
+
$C $P up -d --no-deps runtime-lux # one runtime; start runtimes one at a time after a host reboot
|
| 50 |
+
$C $P stop worker-lux # take one model off the Space (its queue goes offline in ~35 s)
|
| 51 |
+
$C $P stop # stop everything
|
| 52 |
+
$C $P ps; $C $P logs --tail 50 worker-lux
|
| 53 |
+
```
|
| 54 |
+
|
| 55 |
+
Restart policy is `unless-stopped`; a runtime exits on a device error and is
|
| 56 |
+
restarted. Check readiness from the Space: `GET /api/status?model=<wire-id>`
|
| 57 |
+
(`loaded` and `context_batch` true) and `GET /api/ready`. Check a runtime
|
| 58 |
+
against its pins and recorded reference answers from the host:
|
| 59 |
+
|
| 60 |
+
```bash
|
| 61 |
+
docker run --rm --network decision2-runtime -v /opt/decision-studio/bin:/check:ro \
|
| 62 |
+
-v /opt/decision-studio/reference:/reference:ro --entrypoint python3 "$DECISION_RUNTIME_IMAGE" \
|
| 63 |
+
/check/check_runtime.py --origin http://172.31.231.15:8100 --model vllm-sr/Decision-2.0-Lux-9B \
|
| 64 |
+
--revision <40-hex> --manifest-sha256 <64-hex> --model-sha256 <64-hex> \
|
| 65 |
+
--reference /reference/golden_answers.json
|
| 66 |
+
```
|
| 67 |
+
|
| 68 |
+
It must report every identity field true, `deterministic`, and
|
| 69 |
+
`reference_identical` (byte-identical to the reference recorded on gfx942).
|
| 70 |
+
|
| 71 |
+
End to end, `e2e_check.py space` sends the Studio's examples for each model
|
| 72 |
+
through the public single and batch routes and records answers and latency;
|
| 73 |
+
`e2e_check.py direct`, run in the runtime image on `decision2-runtime`, replays
|
| 74 |
+
them against the runtimes and must report `answers identical`.
|
| 75 |
+
|
| 76 |
+
## Update a model's pins
|
| 77 |
+
|
| 78 |
+
1. Read the new revision's `MODEL_MANIFEST.json` digest and model identity:
|
| 79 |
+
`fetch_pins.py --cache-dir /hf-cache <repo>@<40-hex>` in the runtime image
|
| 80 |
+
(network `bridge`, token file mounted read-only) downloads and verifies it.
|
| 81 |
+
2. Update `<MODEL>_REVISION` in `decision2.env`, the entry in `registry.json`
|
| 82 |
+
(`revision`, `manifest_sha256`, `content_sha256` = model identity) and its
|
| 83 |
+
`MODEL_RELEASES.json` row; push the Space.
|
| 84 |
+
3. Set the Space variable `DECISION_MODEL_REGISTRY_V2` to the compact JSON of
|
| 85 |
+
`registry.json` and write the same value into `worker-common.env`.
|
| 86 |
+
4. `$C $P up -d --force-recreate runtime-<model> worker-<model>`, then run the
|
| 87 |
+
check above with a reference that records the new revision (or record one).
|
| 88 |
+
|
| 89 |
+
## Update the runtime image
|
| 90 |
+
|
| 91 |
+
Mirror `src/model-runtime` of a `main` commit to the host, build with
|
| 92 |
+
`deploy/runtime/Dockerfile` on the packages' release ROCm image, set
|
| 93 |
+
`DECISION_RUNTIME_IMAGE`, and recreate runtimes one at a time with the check
|
| 94 |
+
after each. If that commit pins FLA kernel choices itself, the
|
| 95 |
+
`FLA_CONFIG_DIR` mounts become redundant but harmless.
|
| 96 |
+
|
| 97 |
+
## Add a model (for example a Reasoning release)
|
| 98 |
+
|
| 99 |
+
1. Add its wire ID to `model_registry.PROFILES` and `MODEL_ORDER` (repository,
|
| 100 |
+
complete-input limit, confidence definition) with a test, and its entry to
|
| 101 |
+
`registry.json` and `MODEL_RELEASES.json`; push the Space.
|
| 102 |
+
2. Add a `runtime-<name>` / `worker-<name>` pair to `compose.yaml` with the next
|
| 103 |
+
free internal address, a `<NAME>_REVISION`, and a `workers/<wire-id>.env`.
|
| 104 |
+
Private repositories need the host token only for the fetch job.
|
| 105 |
+
3. Fetch, start the runtime, check it, set the Space variable and the worker
|
| 106 |
+
registry, start the worker.
|
| 107 |
+
|
| 108 |
+
## Move Vega to its own host
|
| 109 |
+
|
| 110 |
+
On the new host: same host files (copy the protected ones over SSH), the
|
| 111 |
+
runtime image, `fetch_pins.py` for Vega, `fla_kernel_choices.py` for Vega,
|
| 112 |
+
then `$C --profile vega up -d runtime-vega` and the check. Stop `worker-vega`
|
| 113 |
+
on the old host, wait 40 s for its heartbeat to expire (a queue has one
|
| 114 |
+
owner), start `worker-vega` on the new host, confirm
|
| 115 |
+
`/api/status?model=decision2-vega`, then stop `runtime-vega` on the old host.
|
| 116 |
+
Without a direct route between hosts, stream the images host to host
|
| 117 |
+
(`docker save | zstd` over SSH with a forwarded agent and a pinned known-hosts
|
| 118 |
+
file) and fetch the weights from the Hub with `fetch_pins.py`; check that the
|
| 119 |
+
loaded image IDs match the old host's. To roll back, reverse the worker
|
| 120 |
+
handover and `$C --profile vega start runtime-vega worker-vega` on the old
|
| 121 |
+
host.
|
| 122 |
+
|
| 123 |
+
## Decision 1.0 queues (removed)
|
| 124 |
+
|
| 125 |
+
The six Decision 1.0 queues are no longer in the deployed registry: on
|
| 126 |
+
2026-10-11 their runtimes and HTTP pull workers (`decision-http-pull` Compose
|
| 127 |
+
project, `decision_serve_v1` API) were stopped and their host went to another
|
| 128 |
+
model. `PROFILES` keeps the 1.0 entries (`runtime_model` is the
|
| 129 |
+
`llm-semantic-router/...` ID those runtimes attest), so a rollback needs no code
|
| 130 |
+
change: restore `registry.json` from Space commit `6b75cce8`, set
|
| 131 |
+
`DECISION_MODEL_REGISTRY_V2` to its compact JSON, start the stopped 1.0 runtimes
|
| 132 |
+
and then their workers on that host, and confirm `/api/ready` lists twelve
|
| 133 |
+
models. The 1.0 workers keep their own 1.0-only registry in `worker-common.env`;
|
| 134 |
+
only `DECISION_STUDIO_URL` and `DECISION_WORKER_TOKEN` must track the Space.
|
| 135 |
+
|
| 136 |
+
## Rotate the worker token
|
| 137 |
+
|
| 138 |
+
Generate 48 random bytes (URL-safe), set the Space secret
|
| 139 |
+
`DECISION_WORKER_TOKEN` (the Space restarts), replace it in every host's
|
| 140 |
+
`worker-common.env`, and `$C $P up -d --force-recreate` the workers. Decision
|
| 141 |
+
1.0 workers share the same token.
|
| 142 |
+
|
| 143 |
+
## Roll back
|
| 144 |
+
|
| 145 |
+
Restore the previous Space commit and `DECISION_MODEL_REGISTRY_V2` together.
|
| 146 |
+
A worker whose model is missing from the restored registry is rejected at its
|
| 147 |
+
first heartbeat and keeps restarting; stop it (`$C $P stop worker-<model>`).
|
deploy/decision2/check_runtime.py
ADDED
|
@@ -0,0 +1,103 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Check one running `vllm-sr-runtime` instance against its pins and reference answers.
|
| 2 |
+
|
| 3 |
+
Runs inside the runtime image on the private runtime network. It checks the
|
| 4 |
+
served artifact (repository, revision, manifest and model identity, exact
|
| 5 |
+
profile, readiness), asks the runtime's own golden request twice on the exact
|
| 6 |
+
profile and compares the answers with the recorded reference answers for the
|
| 7 |
+
device class: `vllm_sr_runtime/registry/golden_answers.json` of a
|
| 8 |
+
vllm-project/semantic-router commit that records the same revision.
|
| 9 |
+
|
| 10 |
+
python3 check_runtime.py --origin http://172.31.231.11:8100 \
|
| 11 |
+
--model vllm-sr/Decision-2.0-Kai-0.6B --revision <40-hex> \
|
| 12 |
+
--manifest-sha256 <64-hex> --model-sha256 <64-hex> --reference golden_answers.json
|
| 13 |
+
"""
|
| 14 |
+
|
| 15 |
+
from __future__ import annotations
|
| 16 |
+
|
| 17 |
+
import argparse
|
| 18 |
+
import json
|
| 19 |
+
import sys
|
| 20 |
+
import time
|
| 21 |
+
import urllib.request
|
| 22 |
+
|
| 23 |
+
from vllm_sr_runtime.families.decision2.family import GOLDEN_QUESTIONS, GOLDEN_STATE
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
def call(origin: str, path: str, body: dict | None = None) -> tuple[int, dict, float]:
|
| 27 |
+
data = None if body is None else json.dumps(body).encode()
|
| 28 |
+
request = urllib.request.Request(
|
| 29 |
+
origin + path, data=data, method="GET" if body is None else "POST",
|
| 30 |
+
headers={"Content-Type": "application/json"},
|
| 31 |
+
)
|
| 32 |
+
started = time.perf_counter()
|
| 33 |
+
try:
|
| 34 |
+
with urllib.request.urlopen(request, timeout=60) as response:
|
| 35 |
+
payload = json.load(response)
|
| 36 |
+
status = response.status
|
| 37 |
+
except urllib.error.HTTPError as error:
|
| 38 |
+
payload, status = json.load(error), error.code
|
| 39 |
+
return status, payload, (time.perf_counter() - started) * 1000
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
def max_delta(left: dict, right: dict) -> float:
|
| 43 |
+
worst = 0.0
|
| 44 |
+
for key, answer in right.items():
|
| 45 |
+
mine = left.get(key, {})
|
| 46 |
+
if answer.get("type") == "noul":
|
| 47 |
+
worst = max(worst, abs(mine.get("noul", 2.0) - answer["noul"]))
|
| 48 |
+
continue
|
| 49 |
+
for candidate, value in answer.get("probabilities", {}).items():
|
| 50 |
+
worst = max(worst, abs(mine.get("probabilities", {}).get(candidate, 2.0) - value))
|
| 51 |
+
return worst
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
def main() -> int:
|
| 55 |
+
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
| 56 |
+
parser.add_argument("--origin", required=True)
|
| 57 |
+
parser.add_argument("--model", required=True)
|
| 58 |
+
parser.add_argument("--revision", required=True)
|
| 59 |
+
parser.add_argument("--manifest-sha256", required=True)
|
| 60 |
+
parser.add_argument("--model-sha256", required=True)
|
| 61 |
+
parser.add_argument("--reference", required=True)
|
| 62 |
+
parser.add_argument("--device-class", default="rocm")
|
| 63 |
+
args = parser.parse_args()
|
| 64 |
+
|
| 65 |
+
_, health, _ = call(args.origin, "/health")
|
| 66 |
+
_, listing, _ = call(args.origin, "/v1/models")
|
| 67 |
+
card = listing["data"][0]
|
| 68 |
+
identity = {
|
| 69 |
+
"ready": health.get("status") == "ready" and card.get("ready") is True,
|
| 70 |
+
"repo": card.get("repo") == args.model and card.get("id") == args.model,
|
| 71 |
+
"revision": card.get("revision") == args.revision,
|
| 72 |
+
"manifest_sha256": card.get("manifest_sha256") == args.manifest_sha256,
|
| 73 |
+
"model_sha256": card.get("model_sha256") == args.model_sha256,
|
| 74 |
+
"profile": card.get("profile") == "exact",
|
| 75 |
+
}
|
| 76 |
+
body = {"model": args.model, "state": GOLDEN_STATE, "questions": GOLDEN_QUESTIONS,
|
| 77 |
+
"options": {"profile": "exact", "return_meta": True}}
|
| 78 |
+
status_a, first, ms_a = call(args.origin, "/v1/systemone", body)
|
| 79 |
+
status_b, second, ms_b = call(args.origin, "/v1/systemone", body)
|
| 80 |
+
recorded = json.load(open(args.reference, encoding="utf-8")).get(args.model, {})
|
| 81 |
+
reference = (recorded.get("answers") or {}).get(args.device_class)
|
| 82 |
+
answers = first.get("answers", {})
|
| 83 |
+
report = {
|
| 84 |
+
"model": args.model,
|
| 85 |
+
"identity": identity,
|
| 86 |
+
"device": card.get("device"),
|
| 87 |
+
"accelerator_validated": card.get("accelerator_validated"),
|
| 88 |
+
"golden_status": (card.get("golden") or {}).get("status"),
|
| 89 |
+
"deterministic": status_a == status_b == 200 and first.get("answers") == second.get("answers"),
|
| 90 |
+
"meta_revision": first.get("meta", {}).get("revision") == args.revision,
|
| 91 |
+
"reference_revision": recorded.get("revision"),
|
| 92 |
+
"reference_identical": reference is not None and answers == reference,
|
| 93 |
+
"reference_max_abs_delta": None if reference is None else max_delta(answers, reference),
|
| 94 |
+
"latency_ms": [round(ms_a, 2), round(ms_b, 2)],
|
| 95 |
+
}
|
| 96 |
+
print(json.dumps(report, sort_keys=True))
|
| 97 |
+
ok = (all(identity.values()) and report["deterministic"] and report["meta_revision"]
|
| 98 |
+
and reference is not None and report["reference_max_abs_delta"] <= 0.02)
|
| 99 |
+
return 0 if ok else 1
|
| 100 |
+
|
| 101 |
+
|
| 102 |
+
if __name__ == "__main__":
|
| 103 |
+
sys.exit(main())
|
deploy/decision2/compose.yaml
ADDED
|
@@ -0,0 +1,151 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Decision 2.0 serving for the Decision Studio Space, on one AMD GPU host.
|
| 2 |
+
#
|
| 3 |
+
# Each model runs in its own `vllm-sr-runtime serve` container on an internal
|
| 4 |
+
# Docker network with a fixed private address and no published ports; the
|
| 5 |
+
# runtimes start offline from a verified Hugging Face cache. Each runtime has
|
| 6 |
+
# one outbound HTTP pull worker that claims that model's queue from the Space
|
| 7 |
+
# over HTTPS. Profiles select the models a host serves:
|
| 8 |
+
# small Kai, Eos, Sol, Nox and Lux
|
| 9 |
+
# vega Vega-27B (its own host when one has room)
|
| 10 |
+
#
|
| 11 |
+
# docker compose --env-file /etc/decision-studio/decision2.env \
|
| 12 |
+
# -f deploy/decision2/compose.yaml --profile small --profile vega up -d
|
| 13 |
+
#
|
| 14 |
+
# /etc/decision-studio/decision2.env (mode 600) supplies the images, the GPU
|
| 15 |
+
# device group IDs, the host cache and kernel-choice paths and the pinned
|
| 16 |
+
# revisions; see the runbook. Worker credentials and the Space registry stay in
|
| 17 |
+
# /etc/decision-studio/worker-common.env, and each worker's private runtime
|
| 18 |
+
# origin in /etc/decision-studio/workers/<wire-id>.env.
|
| 19 |
+
name: decision-studio-2
|
| 20 |
+
|
| 21 |
+
x-runtime: &runtime
|
| 22 |
+
image: ${DECISION_RUNTIME_IMAGE:?Set DECISION_RUNTIME_IMAGE}
|
| 23 |
+
restart: unless-stopped
|
| 24 |
+
devices: [/dev/kfd, /dev/dri]
|
| 25 |
+
group_add: ["${VIDEO_GID:?Set VIDEO_GID}", "${RENDER_GID:?Set RENDER_GID}"]
|
| 26 |
+
cap_drop: [ALL]
|
| 27 |
+
security_opt: ["no-new-privileges:true"]
|
| 28 |
+
ipc: private
|
| 29 |
+
shm_size: 2gb
|
| 30 |
+
stop_grace_period: 30s
|
| 31 |
+
environment:
|
| 32 |
+
HF_HUB_OFFLINE: "1"
|
| 33 |
+
HIP_VISIBLE_DEVICES: "0"
|
| 34 |
+
healthcheck:
|
| 35 |
+
test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8100/health', timeout=3).close()"]
|
| 36 |
+
interval: 15s
|
| 37 |
+
timeout: 5s
|
| 38 |
+
retries: 3
|
| 39 |
+
start_period: 900s
|
| 40 |
+
logging:
|
| 41 |
+
driver: json-file
|
| 42 |
+
options: {max-size: 20m, max-file: "3"}
|
| 43 |
+
|
| 44 |
+
# The Qwen3.5-based sizes run FLA's gated-delta kernels with the released
|
| 45 |
+
# kernel choices (FLA_CACHE_MODE=full) instead of timing them per process.
|
| 46 |
+
x-fla-environment: &fla-environment
|
| 47 |
+
HF_HUB_OFFLINE: "1"
|
| 48 |
+
HIP_VISIBLE_DEVICES: "0"
|
| 49 |
+
FLA_CONFIG_DIR: /fla-config
|
| 50 |
+
FLA_CACHE_MODE: full
|
| 51 |
+
|
| 52 |
+
x-worker: &worker
|
| 53 |
+
image: ${DECISION_HTTP_WORKER_IMAGE:?Set DECISION_HTTP_WORKER_IMAGE}
|
| 54 |
+
restart: unless-stopped
|
| 55 |
+
read_only: true
|
| 56 |
+
cap_drop: [ALL]
|
| 57 |
+
security_opt: ["no-new-privileges:true"]
|
| 58 |
+
stop_grace_period: 45s
|
| 59 |
+
networks: [runtime, egress]
|
| 60 |
+
logging:
|
| 61 |
+
driver: json-file
|
| 62 |
+
options: {max-size: 10m, max-file: "3"}
|
| 63 |
+
|
| 64 |
+
services:
|
| 65 |
+
runtime-kai:
|
| 66 |
+
<<: *runtime
|
| 67 |
+
profiles: [small]
|
| 68 |
+
command: [serve, vllm-sr/Decision-2.0-Kai-0.6B, --revision, "${KAI_REVISION:?}", --served-model-name, vllm-sr/Decision-2.0-Kai-0.6B, --device, "rocm:0", --host, 0.0.0.0, --port, "8100", --offline, --cache-dir, /hf-cache, --autotune-cache, /autotune]
|
| 69 |
+
volumes: ["${HF_CACHE_DIR:?}:/hf-cache:ro", "${AUTOTUNE_DIR:?}/kai:/autotune"]
|
| 70 |
+
networks: {runtime: {ipv4_address: 172.31.231.11}}
|
| 71 |
+
runtime-eos:
|
| 72 |
+
<<: *runtime
|
| 73 |
+
profiles: [small]
|
| 74 |
+
command: [serve, vllm-sr/Decision-2.0-Eos-0.8B, --revision, "${EOS_REVISION:?}", --served-model-name, vllm-sr/Decision-2.0-Eos-0.8B, --device, "rocm:0", --host, 0.0.0.0, --port, "8100", --offline, --cache-dir, /hf-cache, --autotune-cache, /autotune]
|
| 75 |
+
environment: *fla-environment
|
| 76 |
+
volumes: ["${HF_CACHE_DIR:?}:/hf-cache:ro", "${AUTOTUNE_DIR:?}/eos:/autotune", "${FLA_CHOICES_DIR:?}/eos:/fla-config:ro"]
|
| 77 |
+
networks: {runtime: {ipv4_address: 172.31.231.12}}
|
| 78 |
+
runtime-sol:
|
| 79 |
+
<<: *runtime
|
| 80 |
+
profiles: [small]
|
| 81 |
+
command: [serve, vllm-sr/Decision-2.0-Sol-2B, --revision, "${SOL_REVISION:?}", --served-model-name, vllm-sr/Decision-2.0-Sol-2B, --device, "rocm:0", --host, 0.0.0.0, --port, "8100", --offline, --cache-dir, /hf-cache, --autotune-cache, /autotune]
|
| 82 |
+
environment: *fla-environment
|
| 83 |
+
volumes: ["${HF_CACHE_DIR:?}:/hf-cache:ro", "${AUTOTUNE_DIR:?}/sol:/autotune", "${FLA_CHOICES_DIR:?}/sol:/fla-config:ro"]
|
| 84 |
+
networks: {runtime: {ipv4_address: 172.31.231.13}}
|
| 85 |
+
runtime-nox:
|
| 86 |
+
<<: *runtime
|
| 87 |
+
profiles: [small]
|
| 88 |
+
command: [serve, vllm-sr/Decision-2.0-Nox-4B, --revision, "${NOX_REVISION:?}", --served-model-name, vllm-sr/Decision-2.0-Nox-4B, --device, "rocm:0", --host, 0.0.0.0, --port, "8100", --offline, --cache-dir, /hf-cache, --autotune-cache, /autotune]
|
| 89 |
+
environment: *fla-environment
|
| 90 |
+
volumes: ["${HF_CACHE_DIR:?}:/hf-cache:ro", "${AUTOTUNE_DIR:?}/nox:/autotune", "${FLA_CHOICES_DIR:?}/nox:/fla-config:ro"]
|
| 91 |
+
networks: {runtime: {ipv4_address: 172.31.231.14}}
|
| 92 |
+
runtime-lux:
|
| 93 |
+
<<: *runtime
|
| 94 |
+
profiles: [small]
|
| 95 |
+
command: [serve, vllm-sr/Decision-2.0-Lux-9B, --revision, "${LUX_REVISION:?}", --served-model-name, vllm-sr/Decision-2.0-Lux-9B, --device, "rocm:0", --host, 0.0.0.0, --port, "8100", --offline, --cache-dir, /hf-cache, --autotune-cache, /autotune]
|
| 96 |
+
environment: *fla-environment
|
| 97 |
+
volumes: ["${HF_CACHE_DIR:?}:/hf-cache:ro", "${AUTOTUNE_DIR:?}/lux:/autotune", "${FLA_CHOICES_DIR:?}/lux:/fla-config:ro"]
|
| 98 |
+
networks: {runtime: {ipv4_address: 172.31.231.15}}
|
| 99 |
+
runtime-vega:
|
| 100 |
+
<<: *runtime
|
| 101 |
+
profiles: [vega]
|
| 102 |
+
command: [serve, vllm-sr/Decision-2.0-Vega-27B, --revision, "${VEGA_REVISION:?}", --served-model-name, vllm-sr/Decision-2.0-Vega-27B, --device, "rocm:0", --host, 0.0.0.0, --port, "8100", --offline, --cache-dir, /hf-cache, --autotune-cache, /autotune]
|
| 103 |
+
environment: *fla-environment
|
| 104 |
+
volumes: ["${HF_CACHE_DIR:?}:/hf-cache:ro", "${AUTOTUNE_DIR:?}/vega:/autotune", "${FLA_CHOICES_DIR:?}/vega:/fla-config:ro"]
|
| 105 |
+
networks: {runtime: {ipv4_address: 172.31.231.16}}
|
| 106 |
+
|
| 107 |
+
worker-kai:
|
| 108 |
+
<<: *worker
|
| 109 |
+
profiles: [small]
|
| 110 |
+
depends_on: {runtime-kai: {condition: service_healthy}}
|
| 111 |
+
environment: {DECISION_WORKER_MODEL: decision2-kai, DECISION_RUNTIME_API: vllm_sr_runtime_v1}
|
| 112 |
+
env_file: [/etc/decision-studio/worker-common.env, /etc/decision-studio/workers/decision2-kai.env]
|
| 113 |
+
worker-eos:
|
| 114 |
+
<<: *worker
|
| 115 |
+
profiles: [small]
|
| 116 |
+
depends_on: {runtime-eos: {condition: service_healthy}}
|
| 117 |
+
environment: {DECISION_WORKER_MODEL: decision2-eos, DECISION_RUNTIME_API: vllm_sr_runtime_v1}
|
| 118 |
+
env_file: [/etc/decision-studio/worker-common.env, /etc/decision-studio/workers/decision2-eos.env]
|
| 119 |
+
worker-sol:
|
| 120 |
+
<<: *worker
|
| 121 |
+
profiles: [small]
|
| 122 |
+
depends_on: {runtime-sol: {condition: service_healthy}}
|
| 123 |
+
environment: {DECISION_WORKER_MODEL: decision2-sol, DECISION_RUNTIME_API: vllm_sr_runtime_v1}
|
| 124 |
+
env_file: [/etc/decision-studio/worker-common.env, /etc/decision-studio/workers/decision2-sol.env]
|
| 125 |
+
worker-nox:
|
| 126 |
+
<<: *worker
|
| 127 |
+
profiles: [small]
|
| 128 |
+
depends_on: {runtime-nox: {condition: service_healthy}}
|
| 129 |
+
environment: {DECISION_WORKER_MODEL: decision2-nox, DECISION_RUNTIME_API: vllm_sr_runtime_v1}
|
| 130 |
+
env_file: [/etc/decision-studio/worker-common.env, /etc/decision-studio/workers/decision2-nox.env]
|
| 131 |
+
worker-lux:
|
| 132 |
+
<<: *worker
|
| 133 |
+
profiles: [small]
|
| 134 |
+
depends_on: {runtime-lux: {condition: service_healthy}}
|
| 135 |
+
environment: {DECISION_WORKER_MODEL: decision2-lux, DECISION_RUNTIME_API: vllm_sr_runtime_v1}
|
| 136 |
+
env_file: [/etc/decision-studio/worker-common.env, /etc/decision-studio/workers/decision2-lux.env]
|
| 137 |
+
worker-vega:
|
| 138 |
+
<<: *worker
|
| 139 |
+
profiles: [vega]
|
| 140 |
+
depends_on: {runtime-vega: {condition: service_healthy}}
|
| 141 |
+
environment: {DECISION_WORKER_MODEL: decision2-vega, DECISION_RUNTIME_API: vllm_sr_runtime_v1}
|
| 142 |
+
env_file: [/etc/decision-studio/worker-common.env, /etc/decision-studio/workers/decision2-vega.env]
|
| 143 |
+
|
| 144 |
+
networks:
|
| 145 |
+
runtime:
|
| 146 |
+
name: decision2-runtime
|
| 147 |
+
internal: true
|
| 148 |
+
ipam:
|
| 149 |
+
config: [{subnet: 172.31.231.0/24}]
|
| 150 |
+
egress:
|
| 151 |
+
name: decision2-egress
|
deploy/decision2/e2e_check.py
ADDED
|
@@ -0,0 +1,169 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""End-to-end check of Decision models through the public Space against their runtimes.
|
| 2 |
+
|
| 3 |
+
`space` sends the Studio's own examples for each selected model through the
|
| 4 |
+
public SystemOne routes (single and batch), records every response with its
|
| 5 |
+
client-observed latency, and checks `/api/ready` and `/v1/models`. `direct`
|
| 6 |
+
replays the recorded requests against each private `vllm-sr-runtime` origin
|
| 7 |
+
(one runtime request per state, exact profile) on the host's runtime network
|
| 8 |
+
and reports whether every public answer equals the runtime's answer. With
|
| 9 |
+
`--runtime-api decision_serve_v1` it replays against a Decision 1.0 runtime
|
| 10 |
+
instead: the recorded request as is (native batches), under the model ID the
|
| 11 |
+
runtime attests (`--attested repo=attested-id`).
|
| 12 |
+
|
| 13 |
+
python3 e2e_check.py space --url https://vllm-sr-decision-studio.hf.space --out run.json \
|
| 14 |
+
--model vllm-sr/Decision-2.0-Kai-0.6B ...
|
| 15 |
+
python3 e2e_check.py direct --run run.json \
|
| 16 |
+
--origin vllm-sr/Decision-2.0-Kai-0.6B=http://172.31.231.11:8100 ...
|
| 17 |
+
"""
|
| 18 |
+
|
| 19 |
+
from __future__ import annotations
|
| 20 |
+
|
| 21 |
+
import argparse
|
| 22 |
+
import json
|
| 23 |
+
import statistics
|
| 24 |
+
import sys
|
| 25 |
+
import time
|
| 26 |
+
import urllib.error
|
| 27 |
+
import urllib.request
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def call(url: str, body: dict | None = None, timeout: float = 120) -> tuple[int, object, float]:
|
| 31 |
+
data = None if body is None else json.dumps(body, ensure_ascii=False).encode("utf-8")
|
| 32 |
+
request = urllib.request.Request(url, data=data, method="GET" if body is None else "POST",
|
| 33 |
+
headers={"Content-Type": "application/json"})
|
| 34 |
+
started = time.perf_counter()
|
| 35 |
+
try:
|
| 36 |
+
with urllib.request.urlopen(request, timeout=timeout) as response:
|
| 37 |
+
payload, status = json.load(response), response.status
|
| 38 |
+
except urllib.error.HTTPError as error:
|
| 39 |
+
payload, status = json.load(error), error.code
|
| 40 |
+
return status, payload, (time.perf_counter() - started) * 1000
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
def percentile(values: list[float], fraction: float) -> float:
|
| 44 |
+
ordered = sorted(values)
|
| 45 |
+
return ordered[min(len(ordered) - 1, round(fraction * (len(ordered) - 1)))]
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
def requests_for(model: dict, examples: list[dict]) -> list[dict]:
|
| 49 |
+
by_id = {example["id"]: example for example in examples}
|
| 50 |
+
chosen = [example for example in examples if example.get("surface") == "routing"]
|
| 51 |
+
chosen += [by_id[i] for i in model.get("example_ids", []) if i in by_id]
|
| 52 |
+
result = []
|
| 53 |
+
for example in chosen:
|
| 54 |
+
body = {"model": model["id"], "questions": example["questions"]}
|
| 55 |
+
if "states" in example:
|
| 56 |
+
body["states"] = example["states"]
|
| 57 |
+
else:
|
| 58 |
+
body["state"] = example["state"]
|
| 59 |
+
result.append({"example": example["id"], "body": body})
|
| 60 |
+
return result
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
def space(args: argparse.Namespace) -> int:
|
| 64 |
+
url = args.url.rstrip("/")
|
| 65 |
+
_, ready, _ = call(url + "/api/ready")
|
| 66 |
+
_, listing, _ = call(url + "/v1/models")
|
| 67 |
+
_, examples, _ = call(url + "/api/examples")
|
| 68 |
+
models = {row["id"]: row for row in listing["models"]}
|
| 69 |
+
run = {"url": url, "ready": ready, "models": {}}
|
| 70 |
+
failures = 0
|
| 71 |
+
for repo in args.model:
|
| 72 |
+
rows, timings = [], {"single": [], "batch": []}
|
| 73 |
+
for _ in range(args.repeat):
|
| 74 |
+
for item in requests_for(models[repo], examples):
|
| 75 |
+
route = "/v1/systemone/batches" if "states" in item["body"] else "/v1/systemone"
|
| 76 |
+
status, response, ms = call(url + route, item["body"])
|
| 77 |
+
kind = "batch" if "states" in item["body"] else "single"
|
| 78 |
+
failures += status != 200
|
| 79 |
+
if status == 200:
|
| 80 |
+
timings[kind].append(ms)
|
| 81 |
+
rows.append(dict(item, route=route, status=status, response=response, ms=round(ms, 1)))
|
| 82 |
+
run["models"][repo] = {
|
| 83 |
+
"release": {k: models[repo].get(k) for k in ("revision", "manifest_sha256", "release_date")},
|
| 84 |
+
"requests": rows,
|
| 85 |
+
"latency_ms": {
|
| 86 |
+
kind: {"n": len(v), "p50": round(statistics.median(v), 1),
|
| 87 |
+
"p95": round(percentile(v, 0.95), 1)} for kind, v in timings.items() if v
|
| 88 |
+
},
|
| 89 |
+
}
|
| 90 |
+
print(repo, json.dumps(run["models"][repo]["latency_ms"]),
|
| 91 |
+
"ok" if all(r["status"] == 200 for r in rows) else "FAILED", flush=True)
|
| 92 |
+
with open(args.out, "w", encoding="utf-8") as handle:
|
| 93 |
+
json.dump(run, handle, ensure_ascii=False)
|
| 94 |
+
print("ready:", json.dumps(ready))
|
| 95 |
+
return 1 if failures else 0
|
| 96 |
+
|
| 97 |
+
|
| 98 |
+
def direct(args: argparse.Namespace) -> int:
|
| 99 |
+
run = json.load(open(args.run, encoding="utf-8"))
|
| 100 |
+
origins = dict(item.split("=", 1) for item in args.origin)
|
| 101 |
+
attested = dict(item.split("=", 1) for item in args.attested)
|
| 102 |
+
mismatches = 0
|
| 103 |
+
for repo, record in run["models"].items():
|
| 104 |
+
if repo not in origins:
|
| 105 |
+
continue
|
| 106 |
+
timings, compared = [], 0
|
| 107 |
+
for row in record["requests"]:
|
| 108 |
+
body, public = row["body"], row["response"]
|
| 109 |
+
states = body.get("states") or [{"id": None, "state": body["state"]}]
|
| 110 |
+
answers = []
|
| 111 |
+
if args.runtime_api == "decision_serve_v1":
|
| 112 |
+
route = "/v1/systemone/batches" if "states" in body else "/v1/systemone"
|
| 113 |
+
status, runtime, ms = call(origins[repo] + route,
|
| 114 |
+
dict(body, model=attested.get(repo, repo)))
|
| 115 |
+
timings.append(ms)
|
| 116 |
+
if "states" in body:
|
| 117 |
+
same = [r.get("answers") for r in runtime.get("results", [])] == \
|
| 118 |
+
[r["answers"] for r in public.get("results", [])]
|
| 119 |
+
else:
|
| 120 |
+
same = runtime.get("answers") == public.get("answers")
|
| 121 |
+
same = same and status == 200 and runtime.get("usage") == public.get("usage")
|
| 122 |
+
compared += 1
|
| 123 |
+
if not same or row["status"] != 200:
|
| 124 |
+
mismatches += 1
|
| 125 |
+
print("MISMATCH", repo, row["example"], flush=True)
|
| 126 |
+
continue
|
| 127 |
+
for state in states:
|
| 128 |
+
status, runtime, ms = call(origins[repo] + "/v1/systemone", {
|
| 129 |
+
"model": repo, "state": state["state"], "questions": body["questions"],
|
| 130 |
+
"options": {"profile": "exact", "return_meta": False}})
|
| 131 |
+
timings.append(ms)
|
| 132 |
+
answers.append((status, runtime))
|
| 133 |
+
if "states" in body:
|
| 134 |
+
expected = [r["answers"] for r in public.get("results", [])]
|
| 135 |
+
same = [a[1].get("answers") for a in answers] == expected
|
| 136 |
+
else:
|
| 137 |
+
same = answers[0][1].get("answers") == public.get("answers") and \
|
| 138 |
+
answers[0][1].get("usage") == public.get("usage")
|
| 139 |
+
compared += 1
|
| 140 |
+
if not same or row["status"] != 200:
|
| 141 |
+
mismatches += 1
|
| 142 |
+
print("MISMATCH", repo, row["example"], flush=True)
|
| 143 |
+
print(repo, f"{compared} requests compared,",
|
| 144 |
+
f"direct p50 {statistics.median(timings):.1f} ms p95 {percentile(timings, 0.95):.1f} ms",
|
| 145 |
+
f"({len(timings)} runtime calls)", flush=True)
|
| 146 |
+
print("answers identical" if not mismatches else f"{mismatches} mismatches")
|
| 147 |
+
return 1 if mismatches else 0
|
| 148 |
+
|
| 149 |
+
|
| 150 |
+
def main() -> int:
|
| 151 |
+
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
| 152 |
+
commands = parser.add_subparsers(dest="command", required=True)
|
| 153 |
+
record = commands.add_parser("space")
|
| 154 |
+
record.add_argument("--url", required=True)
|
| 155 |
+
record.add_argument("--out", required=True)
|
| 156 |
+
record.add_argument("--model", action="append", required=True)
|
| 157 |
+
record.add_argument("--repeat", type=int, default=1)
|
| 158 |
+
replay = commands.add_parser("direct")
|
| 159 |
+
replay.add_argument("--run", required=True)
|
| 160 |
+
replay.add_argument("--origin", action="append", required=True)
|
| 161 |
+
replay.add_argument("--runtime-api", choices=("vllm_sr_runtime_v1", "decision_serve_v1"),
|
| 162 |
+
default="vllm_sr_runtime_v1")
|
| 163 |
+
replay.add_argument("--attested", action="append", default=[])
|
| 164 |
+
args = parser.parse_args()
|
| 165 |
+
return space(args) if args.command == "space" else direct(args)
|
| 166 |
+
|
| 167 |
+
|
| 168 |
+
if __name__ == "__main__":
|
| 169 |
+
sys.exit(main())
|
deploy/decision2/fla_kernel_choices.py
ADDED
|
@@ -0,0 +1,70 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Write FLA kernel-choice config directories from recorded release choices.
|
| 2 |
+
|
| 3 |
+
FLA's gated-delta kernels pick block sizes by timing them in each process, so
|
| 4 |
+
two runtime processes can answer the Qwen3.5-based Decision 2.0 sizes
|
| 5 |
+
differently by rounding. vllm-project/semantic-router records each model's
|
| 6 |
+
released kernel choices in `src/model-runtime/vllm_sr_runtime/registry/
|
| 7 |
+
kernel_choices.json`. This script turns one device class of that file into
|
| 8 |
+
FLA's own config files (one directory per model, one `<kernel>.json` per
|
| 9 |
+
kernel), in the format FLA reads with `FLA_CONFIG_DIR=<dir>` and
|
| 10 |
+
`FLA_CACHE_MODE=full`, so every process runs the released configurations and
|
| 11 |
+
times none.
|
| 12 |
+
|
| 13 |
+
python3 fla_kernel_choices.py kernel_choices.json /srv/decision-studio/fla-config \
|
| 14 |
+
--model vllm-sr/Decision-2.0-Eos-0.8B=eos --model vllm-sr/Decision-2.0-Lux-9B=lux
|
| 15 |
+
"""
|
| 16 |
+
|
| 17 |
+
from __future__ import annotations
|
| 18 |
+
|
| 19 |
+
import argparse
|
| 20 |
+
import hashlib
|
| 21 |
+
import json
|
| 22 |
+
import sys
|
| 23 |
+
from pathlib import Path
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
def fla_key_hash(key: list) -> str:
|
| 27 |
+
"""FLA's `AutotuneKey.key_hash` of a tuning key."""
|
| 28 |
+
serialized = json.dumps(key, separators=(",", ":"), sort_keys=True)
|
| 29 |
+
return hashlib.md5(serialized.encode(), usedforsecurity=False).hexdigest()
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
def write_model(choices: dict, directory: Path) -> int:
|
| 33 |
+
directory.mkdir(parents=True, exist_ok=True)
|
| 34 |
+
for stale in directory.glob("*.json"):
|
| 35 |
+
stale.unlink()
|
| 36 |
+
for name, entries in choices["kernels"].items():
|
| 37 |
+
document = {
|
| 38 |
+
"kernel_name": name,
|
| 39 |
+
"autotune_entries": {
|
| 40 |
+
fla_key_hash(entry["key"]): {"autotune_key": entry["key"], "config": entry["config"]}
|
| 41 |
+
for entry in entries
|
| 42 |
+
},
|
| 43 |
+
"default_config": entries[0]["config"],
|
| 44 |
+
}
|
| 45 |
+
(directory / f"{name}.json").write_text(json.dumps(document, sort_keys=True), encoding="utf-8")
|
| 46 |
+
return len(choices["kernels"])
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
def main(argv: list[str] | None = None) -> int:
|
| 50 |
+
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
| 51 |
+
parser.add_argument("kernel_choices")
|
| 52 |
+
parser.add_argument("output")
|
| 53 |
+
parser.add_argument("--device-class", default="rocm:gfx942")
|
| 54 |
+
parser.add_argument("--fla", default="0.5.2", help="FLA version installed in the runtime image")
|
| 55 |
+
parser.add_argument("--model", action="append", required=True, metavar="REPO=NAME")
|
| 56 |
+
args = parser.parse_args(argv)
|
| 57 |
+
recorded = json.loads(Path(args.kernel_choices).read_text(encoding="utf-8"))
|
| 58 |
+
for item in args.model:
|
| 59 |
+
repo, _, name = item.partition("=")
|
| 60 |
+
entry = recorded.get(repo)
|
| 61 |
+
choices = (entry or {}).get("choices", {}).get(args.device_class)
|
| 62 |
+
if not choices or choices.get("fla") != args.fla:
|
| 63 |
+
parser.error(f"{repo}: no {args.device_class} choices recorded with FLA {args.fla}")
|
| 64 |
+
count = write_model(choices, Path(args.output) / name)
|
| 65 |
+
print(f"{repo}@{entry.get('revision')} -> {name}: {count} kernels", flush=True)
|
| 66 |
+
return 0
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
if __name__ == "__main__":
|
| 70 |
+
sys.exit(main())
|
deploy/decision2/registry.json
ADDED
|
@@ -0,0 +1,128 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[
|
| 2 |
+
{
|
| 3 |
+
"id": "decision2-vega",
|
| 4 |
+
"label": "Vega",
|
| 5 |
+
"version": "2.0",
|
| 6 |
+
"manifest_sha256": "24dcf251f5b50a2259201cc5e11889b6d2545afd4ccecabb0e2344549542bdd3",
|
| 7 |
+
"revision": "7aec49ae11a18741706da549ab626b9052795fe7",
|
| 8 |
+
"content_sha256": "584697310502cd20e91728e69f724c028b47b95daf3790256dea3e434c48be82",
|
| 9 |
+
"repo_id": "vllm-sr/Decision-2.0-Vega-27B",
|
| 10 |
+
"complete_input_tokens": 32768,
|
| 11 |
+
"parameter_label": "27B",
|
| 12 |
+
"description": "Strongest Decision model",
|
| 13 |
+
"example_ids": [
|
| 14 |
+
"lux-inbox-contexts-v1",
|
| 15 |
+
"lux-policy-batch",
|
| 16 |
+
"lux-purchase-eight-v2",
|
| 17 |
+
"lux-data-eight-v2",
|
| 18 |
+
"lux-expense-contexts-v2",
|
| 19 |
+
"lux-sensitivity-contexts-v2"
|
| 20 |
+
],
|
| 21 |
+
"default_example_id": "lux-inbox-contexts-v1"
|
| 22 |
+
},
|
| 23 |
+
{
|
| 24 |
+
"id": "decision2-lux",
|
| 25 |
+
"label": "Lux",
|
| 26 |
+
"version": "2.0",
|
| 27 |
+
"manifest_sha256": "4e0f4ca0cf3c833933f9bed3021012d061910016c9f4ca43b34f5d66e8d0a260",
|
| 28 |
+
"revision": "78bf3c03d9147aeb30b641edfe0e30ed04887ca5",
|
| 29 |
+
"content_sha256": "0ece5faa210173f443f353474b419fd913a253b06645da3e8370c2db0e1c3339",
|
| 30 |
+
"repo_id": "vllm-sr/Decision-2.0-Lux-9B",
|
| 31 |
+
"complete_input_tokens": 16384,
|
| 32 |
+
"parameter_label": "9B",
|
| 33 |
+
"description": "Large hybrid decoder",
|
| 34 |
+
"example_ids": [
|
| 35 |
+
"lux-inbox-contexts-v1",
|
| 36 |
+
"lux-policy-batch",
|
| 37 |
+
"lux-purchase-eight-v2",
|
| 38 |
+
"lux-data-eight-v2",
|
| 39 |
+
"lux-expense-contexts-v2",
|
| 40 |
+
"lux-sensitivity-contexts-v2"
|
| 41 |
+
],
|
| 42 |
+
"default_example_id": "lux-inbox-contexts-v1"
|
| 43 |
+
},
|
| 44 |
+
{
|
| 45 |
+
"id": "decision2-nox",
|
| 46 |
+
"label": "Nox",
|
| 47 |
+
"version": "2.0",
|
| 48 |
+
"manifest_sha256": "49771ea33a451274687ad2a246ce5fb78f8f42ec904fc91a467477179975bb16",
|
| 49 |
+
"revision": "25e8f67d1b486c647222df3aac640d2d5d736bbe",
|
| 50 |
+
"content_sha256": "31d4ee9213686eeed6283d08ed7bce52ebef94ec47b7f97f7e1209f916fe51ea",
|
| 51 |
+
"repo_id": "vllm-sr/Decision-2.0-Nox-4B",
|
| 52 |
+
"complete_input_tokens": 16384,
|
| 53 |
+
"parameter_label": "4B",
|
| 54 |
+
"description": "Capable hybrid decoder",
|
| 55 |
+
"example_ids": [
|
| 56 |
+
"nox-inbox-contexts-v1",
|
| 57 |
+
"nox-policy-batch",
|
| 58 |
+
"nox-purchase-eight-v2",
|
| 59 |
+
"nox-data-eight-v2",
|
| 60 |
+
"nox-expense-contexts-v2",
|
| 61 |
+
"nox-sensitivity-contexts-v2"
|
| 62 |
+
],
|
| 63 |
+
"default_example_id": "nox-inbox-contexts-v1"
|
| 64 |
+
},
|
| 65 |
+
{
|
| 66 |
+
"id": "decision2-sol",
|
| 67 |
+
"label": "Sol",
|
| 68 |
+
"version": "2.0",
|
| 69 |
+
"manifest_sha256": "a91c440ca371668d488a788bdfe12cc574a97123df60730afedf419737ced970",
|
| 70 |
+
"revision": "64235bef55dad29387dd16da7c90e038bf2f0972",
|
| 71 |
+
"content_sha256": "e20df76c14edb0d28ef17593f49a1870fdea9438ff4805594402e39766f3546c",
|
| 72 |
+
"repo_id": "vllm-sr/Decision-2.0-Sol-2B",
|
| 73 |
+
"complete_input_tokens": 16384,
|
| 74 |
+
"parameter_label": "2B",
|
| 75 |
+
"description": "Compact hybrid decoder",
|
| 76 |
+
"example_ids": [
|
| 77 |
+
"sol-inbox-contexts-v1",
|
| 78 |
+
"sol-billing-batch",
|
| 79 |
+
"sol-order-team-contexts-v2",
|
| 80 |
+
"sol-stock-six-v3",
|
| 81 |
+
"sol-travel-six-v3",
|
| 82 |
+
"sol-refund-six-v3"
|
| 83 |
+
],
|
| 84 |
+
"default_example_id": "sol-inbox-contexts-v1"
|
| 85 |
+
},
|
| 86 |
+
{
|
| 87 |
+
"id": "decision2-eos",
|
| 88 |
+
"label": "Eos",
|
| 89 |
+
"version": "2.0",
|
| 90 |
+
"manifest_sha256": "e8b1081be4a76deca5247792a4031c8e19e2b52b95d91775c13d9e257c407101",
|
| 91 |
+
"revision": "3594047d69f476f1d01cf84c593e213fc3a4dfe0",
|
| 92 |
+
"content_sha256": "d97127991870ae202017a99eb496bdc56a62afd038594a67febfab0aea9d76fe",
|
| 93 |
+
"repo_id": "vllm-sr/Decision-2.0-Eos-0.8B",
|
| 94 |
+
"complete_input_tokens": 16384,
|
| 95 |
+
"parameter_label": "0.8B",
|
| 96 |
+
"description": "Fast hybrid decoder",
|
| 97 |
+
"example_ids": [
|
| 98 |
+
"eos-inbox-contexts-v2",
|
| 99 |
+
"eos-review-eight-en-v1",
|
| 100 |
+
"eos-hotel-eight-v2",
|
| 101 |
+
"eos-parcel-six-v3",
|
| 102 |
+
"eos-sentiment-six-v3",
|
| 103 |
+
"eos-review-topic-six-v3"
|
| 104 |
+
],
|
| 105 |
+
"default_example_id": "eos-inbox-contexts-v2"
|
| 106 |
+
},
|
| 107 |
+
{
|
| 108 |
+
"id": "decision2-kai",
|
| 109 |
+
"label": "Kai",
|
| 110 |
+
"version": "2.0",
|
| 111 |
+
"manifest_sha256": "ca7bf2cff1494a5bfc51f4271f52557da0f3ce3b6f0f6e7a2b4118cb4965e92e",
|
| 112 |
+
"revision": "cd49ea3813fd8ba0928a9a23ef6c9a0f2f0cd764",
|
| 113 |
+
"content_sha256": "bb806f31a14d4532a1b5e00984442128f47f46cbbc68e57e7623327c81cfb983",
|
| 114 |
+
"repo_id": "vllm-sr/Decision-2.0-Kai-0.6B",
|
| 115 |
+
"complete_input_tokens": 8192,
|
| 116 |
+
"parameter_label": "0.6B",
|
| 117 |
+
"description": "Fastest compact decoder",
|
| 118 |
+
"example_ids": [
|
| 119 |
+
"kai-inbox-contexts-v2",
|
| 120 |
+
"kai-review-eight-en-v1",
|
| 121 |
+
"kai-hotel-eight-v2",
|
| 122 |
+
"kai-parcel-six-v3",
|
| 123 |
+
"kai-sentiment-six-v3",
|
| 124 |
+
"kai-review-topic-six-v3"
|
| 125 |
+
],
|
| 126 |
+
"default_example_id": "kai-inbox-contexts-v2"
|
| 127 |
+
}
|
| 128 |
+
]
|
deploy/edge/Caddyfile
ADDED
|
@@ -0,0 +1,72 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Decision Studio public edge, validated with Caddy 2.11.4.
|
| 2 |
+
# Required at Caddy adaptation/reload time:
|
| 3 |
+
# DECISION_EDGE_SITE public site name
|
| 4 |
+
# DECISION_EDGE_GATEWAY_ORIGIN pinned Gateway origin (scheme + host[:port])
|
| 5 |
+
# Keep deployment-specific origins and credentials outside this repository.
|
| 6 |
+
{
|
| 7 |
+
admin localhost:2019
|
| 8 |
+
}
|
| 9 |
+
|
| 10 |
+
(gateway_proxy) {
|
| 11 |
+
reverse_proxy {$DECISION_EDGE_GATEWAY_ORIGIN} {
|
| 12 |
+
transport http {
|
| 13 |
+
dial_timeout 10s
|
| 14 |
+
response_header_timeout 65s
|
| 15 |
+
}
|
| 16 |
+
}
|
| 17 |
+
}
|
| 18 |
+
|
| 19 |
+
(gateway_standard) {
|
| 20 |
+
request_body {
|
| 21 |
+
max_size 256KiB
|
| 22 |
+
}
|
| 23 |
+
import gateway_proxy
|
| 24 |
+
}
|
| 25 |
+
|
| 26 |
+
{$DECISION_EDGE_SITE} {
|
| 27 |
+
encode zstd gzip
|
| 28 |
+
header {
|
| 29 |
+
-Server
|
| 30 |
+
X-Content-Type-Options nosniff
|
| 31 |
+
Referrer-Policy no-referrer
|
| 32 |
+
}
|
| 33 |
+
|
| 34 |
+
@internal path /internal /internal/*
|
| 35 |
+
handle @internal {
|
| 36 |
+
respond 404
|
| 37 |
+
}
|
| 38 |
+
|
| 39 |
+
@health path /_decision/health
|
| 40 |
+
handle @health {
|
| 41 |
+
respond "decision-studio-edge-ready" 200
|
| 42 |
+
}
|
| 43 |
+
|
| 44 |
+
@tetris_root path /tetris
|
| 45 |
+
handle @tetris_root {
|
| 46 |
+
redir * /tetris/ 308
|
| 47 |
+
}
|
| 48 |
+
|
| 49 |
+
@batch path /v1/systemone/batches
|
| 50 |
+
handle @batch {
|
| 51 |
+
request_body {
|
| 52 |
+
max_size 2MiB
|
| 53 |
+
}
|
| 54 |
+
import gateway_proxy
|
| 55 |
+
}
|
| 56 |
+
|
| 57 |
+
# Both paths are forwarded without a URI rewrite to the same Gateway.
|
| 58 |
+
@tetris_page path /tetris/*
|
| 59 |
+
handle @tetris_page {
|
| 60 |
+
import gateway_standard
|
| 61 |
+
}
|
| 62 |
+
|
| 63 |
+
@tetris_api path /api/tetris/*
|
| 64 |
+
handle @tetris_api {
|
| 65 |
+
import gateway_standard
|
| 66 |
+
}
|
| 67 |
+
|
| 68 |
+
# /v1/systemone, other APIs, and Studio retain the 256 KiB edge cap.
|
| 69 |
+
handle {
|
| 70 |
+
import gateway_standard
|
| 71 |
+
}
|
| 72 |
+
}
|
deploy/edge/verify_adapted.py
ADDED
|
@@ -0,0 +1,193 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Fail closed if an adapted Decision edge config breaks its route contract.
|
| 2 |
+
|
| 3 |
+
Usage: caddy adapt --config deploy/edge/Caddyfile --adapter caddyfile |
|
| 4 |
+
python deploy/edge/verify_adapted.py
|
| 5 |
+
|
| 6 |
+
DECISION_EDGE_SITE and DECISION_EDGE_GATEWAY_ORIGIN must be set for both commands.
|
| 7 |
+
No deployment-specific origin is printed on validation failure.
|
| 8 |
+
"""
|
| 9 |
+
|
| 10 |
+
import json
|
| 11 |
+
import os
|
| 12 |
+
import sys
|
| 13 |
+
from urllib.parse import urlsplit
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
STANDARD_BYTES = 256 * 1024
|
| 17 |
+
BATCH_BYTES = 2 * 1024 * 1024
|
| 18 |
+
EXPECTED_PATHS = {
|
| 19 |
+
("/internal", "/internal/*"),
|
| 20 |
+
("/_decision/health",),
|
| 21 |
+
("/tetris",),
|
| 22 |
+
("/v1/systemone/batches",),
|
| 23 |
+
("/tetris/*",),
|
| 24 |
+
("/api/tetris/*",),
|
| 25 |
+
(), # The fallback covers every other API and Studio route.
|
| 26 |
+
}
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def require(condition, message):
|
| 30 |
+
if not condition:
|
| 31 |
+
raise ValueError(message)
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
def expected_dial(origin):
|
| 35 |
+
try:
|
| 36 |
+
parsed = urlsplit(origin)
|
| 37 |
+
port = parsed.port
|
| 38 |
+
except (TypeError, ValueError):
|
| 39 |
+
raise ValueError("Invalid Gateway origin") from None
|
| 40 |
+
require(
|
| 41 |
+
parsed.scheme in {"http", "https"}
|
| 42 |
+
and parsed.hostname
|
| 43 |
+
and not parsed.username
|
| 44 |
+
and not parsed.password
|
| 45 |
+
and not parsed.path
|
| 46 |
+
and not parsed.query
|
| 47 |
+
and not parsed.fragment
|
| 48 |
+
and (port is None or port > 0),
|
| 49 |
+
"Gateway origin must be a bare HTTP(S) origin",
|
| 50 |
+
)
|
| 51 |
+
hostname = parsed.hostname
|
| 52 |
+
host = f"[{hostname}]" if ":" in hostname else hostname
|
| 53 |
+
port = port or (443 if parsed.scheme == "https" else 80)
|
| 54 |
+
return f"{host}:{port}", parsed.scheme == "https"
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
def leaf_handlers(route):
|
| 58 |
+
wrappers = route.get("handle")
|
| 59 |
+
require(
|
| 60 |
+
isinstance(wrappers, list)
|
| 61 |
+
and len(wrappers) == 1
|
| 62 |
+
and wrappers[0].get("handler") == "subroute",
|
| 63 |
+
"A protected path no longer has one exclusive subroute",
|
| 64 |
+
)
|
| 65 |
+
nested = wrappers[0].get("routes")
|
| 66 |
+
require(
|
| 67 |
+
isinstance(nested, list)
|
| 68 |
+
and len(nested) == 1
|
| 69 |
+
and not nested[0].get("match"),
|
| 70 |
+
"A protected path contains a nested matcher",
|
| 71 |
+
)
|
| 72 |
+
return nested[0].get("handle")
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
def path_key(route):
|
| 76 |
+
matches = route.get("match")
|
| 77 |
+
if matches is None:
|
| 78 |
+
return ()
|
| 79 |
+
require(
|
| 80 |
+
isinstance(matches, list)
|
| 81 |
+
and len(matches) == 1
|
| 82 |
+
and set(matches[0]) == {"path"}
|
| 83 |
+
and isinstance(matches[0]["path"], list),
|
| 84 |
+
"A protected route uses an unexpected matcher",
|
| 85 |
+
)
|
| 86 |
+
return tuple(matches[0]["path"])
|
| 87 |
+
|
| 88 |
+
|
| 89 |
+
def verify(document, *, site, origin):
|
| 90 |
+
"""Check Caddy's adapted JSON, not just the visual Caddyfile order."""
|
| 91 |
+
require(isinstance(site, str) and site, "Set DECISION_EDGE_SITE")
|
| 92 |
+
dial, use_tls = expected_dial(origin)
|
| 93 |
+
require(isinstance(document, dict), "Expected a Caddy JSON object")
|
| 94 |
+
servers = document.get("apps", {}).get("http", {}).get("servers", {})
|
| 95 |
+
require(isinstance(servers, dict) and len(servers) == 1, "Expected one edge HTTP server")
|
| 96 |
+
outer = next(iter(servers.values())).get("routes")
|
| 97 |
+
require(isinstance(outer, list) and len(outer) == 1, "Expected one public site route")
|
| 98 |
+
root = outer[0]
|
| 99 |
+
require(root.get("match") == [{"host": [site]}], "Unexpected edge site host")
|
| 100 |
+
handlers = root.get("handle")
|
| 101 |
+
require(
|
| 102 |
+
isinstance(handlers, list)
|
| 103 |
+
and len(handlers) == 1
|
| 104 |
+
and handlers[0].get("handler") == "subroute",
|
| 105 |
+
"Expected one public site subroute",
|
| 106 |
+
)
|
| 107 |
+
routes = handlers[0].get("routes")
|
| 108 |
+
require(isinstance(routes, list), "Missing edge routes")
|
| 109 |
+
groups = [route for route in routes if route.get("group")]
|
| 110 |
+
ungrouped = [route for route in routes if not route.get("group")]
|
| 111 |
+
require(
|
| 112 |
+
len(groups) == len(EXPECTED_PATHS)
|
| 113 |
+
and len({route["group"] for route in groups}) == 1,
|
| 114 |
+
"Protected routes must be one mutually exclusive handle group",
|
| 115 |
+
)
|
| 116 |
+
require(
|
| 117 |
+
all(
|
| 118 |
+
all(handler.get("handler") in {"headers", "encode"} for handler in route.get("handle", []))
|
| 119 |
+
for route in ungrouped
|
| 120 |
+
),
|
| 121 |
+
"Unexpected site-wide handler or body limit",
|
| 122 |
+
)
|
| 123 |
+
by_path = {}
|
| 124 |
+
for route in groups:
|
| 125 |
+
key = path_key(route)
|
| 126 |
+
require(key not in by_path, "Duplicate edge path matcher")
|
| 127 |
+
by_path[key] = leaf_handlers(route)
|
| 128 |
+
require(set(by_path) == EXPECTED_PATHS, "Missing or unexpected edge path matcher")
|
| 129 |
+
require(path_key(groups[-1]) == (), "Catch-all route must be last")
|
| 130 |
+
|
| 131 |
+
def static(path, status, *, location=None):
|
| 132 |
+
leaf = by_path[path]
|
| 133 |
+
require(
|
| 134 |
+
isinstance(leaf, list)
|
| 135 |
+
and len(leaf) == 1
|
| 136 |
+
and leaf[0].get("handler") == "static_response"
|
| 137 |
+
and leaf[0].get("status_code") == status,
|
| 138 |
+
"Edge static response changed",
|
| 139 |
+
)
|
| 140 |
+
if location is not None:
|
| 141 |
+
require(
|
| 142 |
+
leaf[0].get("headers", {}).get("Location") == [location],
|
| 143 |
+
"Tetris slash redirect changed",
|
| 144 |
+
)
|
| 145 |
+
|
| 146 |
+
static(("/internal", "/internal/*"), 404)
|
| 147 |
+
static(("/_decision/health",), 200)
|
| 148 |
+
static(("/tetris",), 308, location="/tetris/")
|
| 149 |
+
|
| 150 |
+
def proxy(path, max_bytes):
|
| 151 |
+
leaf = by_path[path]
|
| 152 |
+
require(
|
| 153 |
+
isinstance(leaf, list)
|
| 154 |
+
and [handler.get("handler") for handler in leaf] == ["request_body", "reverse_proxy"]
|
| 155 |
+
and leaf[0].get("max_size") == max_bytes,
|
| 156 |
+
"Edge route body limit or handler order changed",
|
| 157 |
+
)
|
| 158 |
+
upstream = leaf[1]
|
| 159 |
+
transport = upstream.get("transport", {})
|
| 160 |
+
require(
|
| 161 |
+
upstream.get("upstreams") == [{"dial": dial}]
|
| 162 |
+
and ("tls" in transport) == use_tls
|
| 163 |
+
and transport.get("response_header_timeout") == 65_000_000_000,
|
| 164 |
+
"Edge route no longer targets the pinned Gateway transport",
|
| 165 |
+
)
|
| 166 |
+
return upstream
|
| 167 |
+
|
| 168 |
+
batch = proxy(("/v1/systemone/batches",), BATCH_BYTES)
|
| 169 |
+
page = proxy(("/tetris/*",), STANDARD_BYTES)
|
| 170 |
+
api = proxy(("/api/tetris/*",), STANDARD_BYTES)
|
| 171 |
+
fallback = proxy((), STANDARD_BYTES)
|
| 172 |
+
require(
|
| 173 |
+
page == api == fallback == batch,
|
| 174 |
+
"Tetris and API routes do not share one Gateway target",
|
| 175 |
+
)
|
| 176 |
+
|
| 177 |
+
|
| 178 |
+
def main():
|
| 179 |
+
try:
|
| 180 |
+
verify(
|
| 181 |
+
json.load(sys.stdin),
|
| 182 |
+
site=os.environ.get("DECISION_EDGE_SITE", ""),
|
| 183 |
+
origin=os.environ.get("DECISION_EDGE_GATEWAY_ORIGIN", ""),
|
| 184 |
+
)
|
| 185 |
+
except (ValueError, TypeError, KeyError, json.JSONDecodeError) as exc:
|
| 186 |
+
print(f"Edge route contract failed: {exc}", file=sys.stderr)
|
| 187 |
+
return 1
|
| 188 |
+
print("Edge route contract OK")
|
| 189 |
+
return 0
|
| 190 |
+
|
| 191 |
+
|
| 192 |
+
if __name__ == "__main__":
|
| 193 |
+
raise SystemExit(main())
|
deploy/runtime/Dockerfile
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# ROCm image of the vLLM Semantic Router built-in model runtime (`vllm-sr-runtime`)
|
| 2 |
+
# for the Decision 2.0 Studio hosts.
|
| 3 |
+
#
|
| 4 |
+
# Build context: `src/model-runtime` of a vllm-project/semantic-router commit on
|
| 5 |
+
# `main`. BASE_IMAGE is the Decision 2.0 packages' release ROCm image (PyTorch
|
| 6 |
+
# 2.12.0 on ROCm 7.2, Triton 3.7.1, FLA 0.5.2, causal-conv1d 1.7.0). The runtime's
|
| 7 |
+
# `exact` profile answers bit for bit like the released packages on that image;
|
| 8 |
+
# another PyTorch, Triton or FLA build can move answers by rounding.
|
| 9 |
+
#
|
| 10 |
+
# docker build -f deploy/runtime/Dockerfile \
|
| 11 |
+
# --build-arg BASE_IMAGE=<release-image> --build-arg VLLM_SR_COMMIT=<40-hex> \
|
| 12 |
+
# -t decision-studio/vllm-sr-runtime-rocm:<12-hex> <checkout>/src/model-runtime
|
| 13 |
+
ARG BASE_IMAGE
|
| 14 |
+
FROM ${BASE_IMAGE}
|
| 15 |
+
ARG VLLM_SR_COMMIT
|
| 16 |
+
ENV PYTHONDONTWRITEBYTECODE=1 \
|
| 17 |
+
PYTHONUNBUFFERED=1 \
|
| 18 |
+
PIP_NO_CACHE_DIR=1 \
|
| 19 |
+
PIP_DISABLE_PIP_VERSION_CHECK=1 \
|
| 20 |
+
HF_HUB_DISABLE_TELEMETRY=1
|
| 21 |
+
WORKDIR /opt/vllm-sr-runtime
|
| 22 |
+
COPY pyproject.toml README.md ./
|
| 23 |
+
COPY vllm_sr_runtime ./vllm_sr_runtime
|
| 24 |
+
RUN test -n "$VLLM_SR_COMMIT" \
|
| 25 |
+
&& python3 -m pip install --no-deps --no-build-isolation . \
|
| 26 |
+
&& python3 -c "import causal_conv1d, fla, huggingface_hub, numpy, prometheus_client, safetensors, starlette, tokenizers, torch, triton, uvicorn, yaml, vllm_sr_runtime; print('torch', torch.__version__, 'triton', triton.__version__, 'fla', fla.__version__, 'causal_conv1d', causal_conv1d.__version__)"
|
| 27 |
+
LABEL org.opencontainers.image.source="https://github.com/vllm-project/semantic-router" \
|
| 28 |
+
org.opencontainers.image.revision="${VLLM_SR_COMMIT}"
|
| 29 |
+
WORKDIR /
|
| 30 |
+
ENTRYPOINT ["vllm-sr-runtime"]
|
| 31 |
+
CMD ["--help"]
|
deploy/runtime/fetch_pins.py
ADDED
|
@@ -0,0 +1,47 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Download and verify pinned Decision 2.0 packages into a Hugging Face cache.
|
| 2 |
+
|
| 3 |
+
Runs inside the runtime image with network access and the host's Hugging Face
|
| 4 |
+
token file mounted read-only; the serving containers then start offline from
|
| 5 |
+
the same cache. Verification is the runtime's own: every file listed in the
|
| 6 |
+
package manifest (and an adapter package's pinned base) must match its digest,
|
| 7 |
+
and the model identity must match the manifest. No weights are loaded.
|
| 8 |
+
|
| 9 |
+
python3 fetch_pins.py --cache-dir /hf-cache vllm-sr/Decision-2.0-Kai-0.6B@<40-hex> ...
|
| 10 |
+
"""
|
| 11 |
+
|
| 12 |
+
from __future__ import annotations
|
| 13 |
+
|
| 14 |
+
import argparse
|
| 15 |
+
import re
|
| 16 |
+
import sys
|
| 17 |
+
|
| 18 |
+
from vllm_sr_runtime.families.decision2.family import Decision2Family
|
| 19 |
+
from vllm_sr_runtime.plugins.base import RegistryOptions
|
| 20 |
+
from vllm_sr_runtime.registry.resolve import resolve
|
| 21 |
+
|
| 22 |
+
PIN = re.compile(r"(?P<repo>[A-Za-z0-9][\w.-]*/[\w.-]+)@(?P<revision>[0-9a-f]{40})\Z")
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
def main(argv: list[str] | None = None) -> int:
|
| 26 |
+
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
| 27 |
+
parser.add_argument("pins", nargs="+", help="repository@40-hex revision")
|
| 28 |
+
parser.add_argument("--cache-dir", required=True)
|
| 29 |
+
args = parser.parse_args(argv)
|
| 30 |
+
family = Decision2Family(RegistryOptions(cache_dir=args.cache_dir))
|
| 31 |
+
for pin in args.pins:
|
| 32 |
+
match = PIN.fullmatch(pin)
|
| 33 |
+
if match is None:
|
| 34 |
+
parser.error(f"not a repository@40-hex pin: {pin}")
|
| 35 |
+
ref = resolve(match["repo"], revision=match["revision"], cache_dir=args.cache_dir)
|
| 36 |
+
package = family.verify(ref)
|
| 37 |
+
print(
|
| 38 |
+
f"{match['repo']}@{match['revision']} "
|
| 39 |
+
f"manifest_sha256={package.manifest_sha256} model_sha256={package.model_sha256} "
|
| 40 |
+
f"max_input_tokens={package.max_input_tokens}",
|
| 41 |
+
flush=True,
|
| 42 |
+
)
|
| 43 |
+
return 0
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
if __name__ == "__main__":
|
| 47 |
+
sys.exit(main())
|
deploy/worker/Dockerfile
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# CPU-only outbound adapter. The private runtime owns the GPU and model files.
|
| 2 |
+
FROM python:3.12-slim
|
| 3 |
+
ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 PIP_NO_CACHE_DIR=1
|
| 4 |
+
RUN useradd --system --uid 10001 --create-home decision
|
| 5 |
+
WORKDIR /opt/decision-worker
|
| 6 |
+
COPY deploy/worker/requirements.txt /opt/decision-worker/requirements.txt
|
| 7 |
+
RUN pip install -r /opt/decision-worker/requirements.txt
|
| 8 |
+
COPY contract.py direct_contract.py direct_gateway.py engine.py model_registry.py model_runtime_client.py pull_worker.py http_pull_worker.py /opt/decision-worker/
|
| 9 |
+
USER decision
|
| 10 |
+
CMD ["python", "-m", "http_pull_worker"]
|
deploy/worker/compose.yaml
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
name: decision-http-pull
|
| 2 |
+
|
| 3 |
+
x-worker: &worker
|
| 4 |
+
image: ${DECISION_HTTP_WORKER_IMAGE:?Set DECISION_HTTP_WORKER_IMAGE}
|
| 5 |
+
restart: unless-stopped
|
| 6 |
+
read_only: true
|
| 7 |
+
cap_drop: [ALL]
|
| 8 |
+
security_opt: ["no-new-privileges:true"]
|
| 9 |
+
stop_grace_period: 45s
|
| 10 |
+
networks: [runtime]
|
| 11 |
+
|
| 12 |
+
services:
|
| 13 |
+
decision-lux:
|
| 14 |
+
<<: *worker
|
| 15 |
+
environment:
|
| 16 |
+
DECISION_WORKER_MODEL: decision-lux
|
| 17 |
+
env_file:
|
| 18 |
+
- /etc/decision-studio/worker-common.env
|
| 19 |
+
- /etc/decision-studio/workers/decision-lux.env
|
| 20 |
+
decision-nox:
|
| 21 |
+
<<: *worker
|
| 22 |
+
environment:
|
| 23 |
+
DECISION_WORKER_MODEL: decision-nox
|
| 24 |
+
env_file:
|
| 25 |
+
- /etc/decision-studio/worker-common.env
|
| 26 |
+
- /etc/decision-studio/workers/decision-nox.env
|
| 27 |
+
decision-sol:
|
| 28 |
+
<<: *worker
|
| 29 |
+
environment:
|
| 30 |
+
DECISION_WORKER_MODEL: decision-sol
|
| 31 |
+
env_file:
|
| 32 |
+
- /etc/decision-studio/worker-common.env
|
| 33 |
+
- /etc/decision-studio/workers/decision-sol.env
|
| 34 |
+
decision-eos:
|
| 35 |
+
<<: *worker
|
| 36 |
+
environment:
|
| 37 |
+
DECISION_WORKER_MODEL: decision-eos
|
| 38 |
+
env_file:
|
| 39 |
+
- /etc/decision-studio/worker-common.env
|
| 40 |
+
- /etc/decision-studio/workers/decision-eos.env
|
| 41 |
+
decision-kai:
|
| 42 |
+
<<: *worker
|
| 43 |
+
environment:
|
| 44 |
+
DECISION_WORKER_MODEL: decision-kai
|
| 45 |
+
env_file:
|
| 46 |
+
- /etc/decision-studio/worker-common.env
|
| 47 |
+
- /etc/decision-studio/workers/decision-kai.env
|
| 48 |
+
decision-lex:
|
| 49 |
+
<<: *worker
|
| 50 |
+
environment:
|
| 51 |
+
DECISION_WORKER_MODEL: decision-lex
|
| 52 |
+
env_file:
|
| 53 |
+
- /etc/decision-studio/worker-common.env
|
| 54 |
+
- /etc/decision-studio/workers/decision-lex.env
|
| 55 |
+
|
| 56 |
+
networks:
|
| 57 |
+
runtime:
|
| 58 |
+
external: true
|
| 59 |
+
name: ${DECISION_RUNTIME_NETWORK:?Set DECISION_RUNTIME_NETWORK}
|
deploy/worker/requirements.txt
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
httpx==0.28.1
|
direct_contract.py
ADDED
|
@@ -0,0 +1,248 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Request-relative validation for strict Decision runtime responses."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import json
|
| 6 |
+
import math
|
| 7 |
+
from collections.abc import Collection, Mapping
|
| 8 |
+
|
| 9 |
+
PROBABILITY_TOLERANCE = 2e-5
|
| 10 |
+
MARGIN_ORDINAL_V1 = "margin_ordinal_v1"
|
| 11 |
+
NORMALIZED_ENTROPY_V2 = "normalized_entropy_v2"
|
| 12 |
+
CONFIDENCE_DEFINITIONS = frozenset({MARGIN_ORDINAL_V1, NORMALIZED_ENTROPY_V2})
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
def _probability(value: object, path: str) -> float:
|
| 16 |
+
if isinstance(value, bool) or not isinstance(value, (int, float)):
|
| 17 |
+
raise ValueError(f"{path} must be a finite probability")
|
| 18 |
+
try:
|
| 19 |
+
valid = math.isfinite(value) and 0.0 <= value <= 1.0
|
| 20 |
+
except OverflowError:
|
| 21 |
+
valid = False
|
| 22 |
+
if not valid:
|
| 23 |
+
raise ValueError(f"{path} must be a finite probability")
|
| 24 |
+
return float(value)
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
def _number(value: object, path: str) -> float:
|
| 28 |
+
if isinstance(value, bool) or not isinstance(value, (int, float)):
|
| 29 |
+
raise ValueError(f"{path} must be finite")
|
| 30 |
+
try:
|
| 31 |
+
valid = math.isfinite(value)
|
| 32 |
+
except OverflowError:
|
| 33 |
+
valid = False
|
| 34 |
+
if not valid:
|
| 35 |
+
raise ValueError(f"{path} must be finite")
|
| 36 |
+
return float(value)
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
def _usage(value: object, path: str) -> tuple[int, int]:
|
| 40 |
+
if not isinstance(value, dict) or set(value) != {"input_tokens", "output_tokens"}:
|
| 41 |
+
raise ValueError(f"{path} has an invalid shape")
|
| 42 |
+
input_tokens = value["input_tokens"]
|
| 43 |
+
output_tokens = value["output_tokens"]
|
| 44 |
+
if type(input_tokens) is not int or input_tokens < 0:
|
| 45 |
+
raise ValueError(f"{path}.input_tokens must be a nonnegative integer")
|
| 46 |
+
if type(output_tokens) is not int or output_tokens < 0:
|
| 47 |
+
raise ValueError(f"{path}.output_tokens must be a nonnegative integer")
|
| 48 |
+
return input_tokens, output_tokens
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
def _distribution(value: object, expected: list[str], path: str) -> dict[str, float]:
|
| 52 |
+
if not isinstance(value, dict) or set(value) != set(expected):
|
| 53 |
+
raise ValueError(f"{path} keys must match the request criteria")
|
| 54 |
+
probabilities = {key: _probability(value[key], f"{path}.{key}") for key in expected}
|
| 55 |
+
if not math.isclose(
|
| 56 |
+
math.fsum(probabilities.values()),
|
| 57 |
+
1.0,
|
| 58 |
+
rel_tol=0.0,
|
| 59 |
+
abs_tol=PROBABILITY_TOLERANCE,
|
| 60 |
+
):
|
| 61 |
+
raise ValueError(f"{path} must sum to one")
|
| 62 |
+
return probabilities
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
def _choice_confidence(values: Collection[float]) -> float:
|
| 66 |
+
ordered = sorted(values, reverse=True)
|
| 67 |
+
if len(ordered) == 1:
|
| 68 |
+
return 1.0
|
| 69 |
+
first, second = ordered[:2]
|
| 70 |
+
return min(1.0, max(0.0, first - second))
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
def _score_confidence(values: Collection[float]) -> float:
|
| 74 |
+
probabilities = tuple(values)
|
| 75 |
+
if len(probabilities) == 1:
|
| 76 |
+
return 1.0
|
| 77 |
+
mean = math.fsum(index * value for index, value in enumerate(probabilities))
|
| 78 |
+
variance = math.fsum(
|
| 79 |
+
value * (index - mean) ** 2 for index, value in enumerate(probabilities)
|
| 80 |
+
)
|
| 81 |
+
uniform_variance = (len(probabilities) ** 2 - 1) / 12
|
| 82 |
+
return min(1.0, max(0.0, 1.0 - variance / uniform_variance))
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
def _entropy_confidence(values: Collection[float]) -> float:
|
| 86 |
+
probabilities = tuple(values)
|
| 87 |
+
if len(probabilities) == 1:
|
| 88 |
+
return 1.0
|
| 89 |
+
entropy = -math.fsum(value * math.log(value) for value in probabilities if value > 0)
|
| 90 |
+
return min(1.0, max(0.0, 1.0 - entropy / math.log(len(probabilities))))
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
def _legend_text(criterion: object, confidence: str) -> object:
|
| 94 |
+
# Decision 2.0 runtimes render a structured Score level as canonical JSON text.
|
| 95 |
+
if confidence == NORMALIZED_ENTROPY_V2 and not isinstance(criterion, str):
|
| 96 |
+
return json.dumps(
|
| 97 |
+
criterion, ensure_ascii=False, sort_keys=True, separators=(",", ":"), allow_nan=False
|
| 98 |
+
)
|
| 99 |
+
return criterion
|
| 100 |
+
|
| 101 |
+
|
| 102 |
+
def _answers_for_questions(
|
| 103 |
+
questions: object, answers: object, path: str, confidence: str = MARGIN_ORDINAL_V1
|
| 104 |
+
) -> None:
|
| 105 |
+
entropy = confidence == NORMALIZED_ENTROPY_V2
|
| 106 |
+
if not isinstance(questions, dict) or not isinstance(answers, dict):
|
| 107 |
+
raise ValueError(f"{path} has an invalid shape")
|
| 108 |
+
if set(answers) != set(questions):
|
| 109 |
+
raise ValueError(f"{path} keys must match the request questions")
|
| 110 |
+
for question_id, question in questions.items():
|
| 111 |
+
answer = answers[question_id]
|
| 112 |
+
answer_path = f"{path}.{question_id}"
|
| 113 |
+
if not isinstance(question, dict) or not isinstance(answer, dict):
|
| 114 |
+
raise ValueError(f"{answer_path} has an invalid shape")
|
| 115 |
+
kind = question.get("type")
|
| 116 |
+
if kind == "noul":
|
| 117 |
+
if set(answer) != {"type", "noul"} or answer.get("type") != "noul":
|
| 118 |
+
raise ValueError(f"{answer_path} must be a strict Noul answer")
|
| 119 |
+
_probability(answer["noul"], f"{answer_path}.noul")
|
| 120 |
+
continue
|
| 121 |
+
if kind == "choice":
|
| 122 |
+
if set(answer) != {"type", "choice", "confidence", "probabilities"}:
|
| 123 |
+
raise ValueError(f"{answer_path} must be a strict Choice answer")
|
| 124 |
+
if answer.get("type") != "choice" or not isinstance(answer.get("choice"), str):
|
| 125 |
+
raise ValueError(f"{answer_path} has an invalid Choice identity")
|
| 126 |
+
criteria = question.get("criteria")
|
| 127 |
+
if not isinstance(criteria, dict) or not criteria:
|
| 128 |
+
raise ValueError(f"{answer_path} request criteria are invalid")
|
| 129 |
+
probabilities = _distribution(
|
| 130 |
+
answer["probabilities"], list(criteria), f"{answer_path}.probabilities"
|
| 131 |
+
)
|
| 132 |
+
reported = _probability(answer["confidence"], f"{answer_path}.confidence")
|
| 133 |
+
expected_confidence = (
|
| 134 |
+
_entropy_confidence if entropy else _choice_confidence
|
| 135 |
+
)(probabilities.values())
|
| 136 |
+
if not math.isclose(
|
| 137 |
+
reported,
|
| 138 |
+
expected_confidence,
|
| 139 |
+
rel_tol=0.0,
|
| 140 |
+
abs_tol=PROBABILITY_TOLERANCE,
|
| 141 |
+
):
|
| 142 |
+
raise ValueError(f"{answer_path}.confidence is invalid")
|
| 143 |
+
choice = answer["choice"]
|
| 144 |
+
if choice not in probabilities or not math.isclose(
|
| 145 |
+
probabilities[choice],
|
| 146 |
+
max(probabilities.values()),
|
| 147 |
+
rel_tol=0.0,
|
| 148 |
+
abs_tol=PROBABILITY_TOLERANCE,
|
| 149 |
+
):
|
| 150 |
+
raise ValueError(f"{answer_path}.choice is invalid")
|
| 151 |
+
continue
|
| 152 |
+
if kind != "score" or set(answer) != {
|
| 153 |
+
"type",
|
| 154 |
+
"score",
|
| 155 |
+
"confidence",
|
| 156 |
+
"legend",
|
| 157 |
+
"probabilities",
|
| 158 |
+
}:
|
| 159 |
+
raise ValueError(f"{answer_path} must be a strict Score answer")
|
| 160 |
+
if answer.get("type") != "score":
|
| 161 |
+
raise ValueError(f"{answer_path} has an invalid Score identity")
|
| 162 |
+
criteria = question.get("criteria")
|
| 163 |
+
if not isinstance(criteria, list) or not criteria:
|
| 164 |
+
raise ValueError(f"{answer_path} request criteria are invalid")
|
| 165 |
+
keys = [str(index) for index in range(len(criteria))]
|
| 166 |
+
probabilities = _distribution(
|
| 167 |
+
answer["probabilities"], keys, f"{answer_path}.probabilities"
|
| 168 |
+
)
|
| 169 |
+
reported = _probability(answer["confidence"], f"{answer_path}.confidence")
|
| 170 |
+
expected_confidence = (
|
| 171 |
+
_entropy_confidence if entropy else _score_confidence
|
| 172 |
+
)(probabilities.values())
|
| 173 |
+
if not math.isclose(
|
| 174 |
+
reported,
|
| 175 |
+
expected_confidence,
|
| 176 |
+
rel_tol=0.0,
|
| 177 |
+
abs_tol=PROBABILITY_TOLERANCE,
|
| 178 |
+
):
|
| 179 |
+
raise ValueError(f"{answer_path}.confidence is invalid")
|
| 180 |
+
legend = {
|
| 181 |
+
str(index): _legend_text(criterion, confidence)
|
| 182 |
+
for index, criterion in enumerate(criteria)
|
| 183 |
+
}
|
| 184 |
+
if answer["legend"] != legend:
|
| 185 |
+
raise ValueError(f"{answer_path}.legend is invalid")
|
| 186 |
+
score = _number(answer["score"], f"{answer_path}.score")
|
| 187 |
+
expected_score = math.fsum(
|
| 188 |
+
index * probabilities[str(index)] for index in range(len(criteria))
|
| 189 |
+
)
|
| 190 |
+
if not math.isclose(
|
| 191 |
+
score,
|
| 192 |
+
expected_score,
|
| 193 |
+
rel_tol=0.0,
|
| 194 |
+
abs_tol=PROBABILITY_TOLERANCE,
|
| 195 |
+
):
|
| 196 |
+
raise ValueError(f"{answer_path}.score is invalid")
|
| 197 |
+
|
| 198 |
+
|
| 199 |
+
def validate_response_for_request(
|
| 200 |
+
request: Mapping[str, object],
|
| 201 |
+
response: object,
|
| 202 |
+
*,
|
| 203 |
+
batch: bool,
|
| 204 |
+
confidence: str = MARGIN_ORDINAL_V1,
|
| 205 |
+
) -> None:
|
| 206 |
+
"""Validate a strict direct response without changing any returned statistic."""
|
| 207 |
+
|
| 208 |
+
if confidence not in CONFIDENCE_DEFINITIONS:
|
| 209 |
+
raise ValueError("Unknown Decision confidence definition")
|
| 210 |
+
expected_top = {"model", "results", "usage"} if batch else {
|
| 211 |
+
"model",
|
| 212 |
+
"answers",
|
| 213 |
+
"usage",
|
| 214 |
+
}
|
| 215 |
+
if not isinstance(response, dict) or set(response) != expected_top:
|
| 216 |
+
raise ValueError("The Decision response envelope is not strict")
|
| 217 |
+
if response["model"] != request.get("model"):
|
| 218 |
+
raise ValueError("The Decision response model does not match the request")
|
| 219 |
+
questions = request.get("questions")
|
| 220 |
+
if not batch:
|
| 221 |
+
_answers_for_questions(questions, response["answers"], "answers", confidence)
|
| 222 |
+
_usage(response["usage"], "usage")
|
| 223 |
+
return
|
| 224 |
+
states = request.get("states")
|
| 225 |
+
results = response["results"]
|
| 226 |
+
if not isinstance(states, list) or not isinstance(results, list):
|
| 227 |
+
raise ValueError("Batch results have an invalid shape")
|
| 228 |
+
if len(results) != len(states):
|
| 229 |
+
raise ValueError("Batch results are incomplete")
|
| 230 |
+
input_tokens = 0
|
| 231 |
+
output_tokens = 0
|
| 232 |
+
for index, (state, result) in enumerate(zip(states, results, strict=True)):
|
| 233 |
+
if (
|
| 234 |
+
not isinstance(state, dict)
|
| 235 |
+
or not isinstance(result, dict)
|
| 236 |
+
or set(result) != {"id", "answers", "usage"}
|
| 237 |
+
or result["id"] != state.get("id")
|
| 238 |
+
):
|
| 239 |
+
raise ValueError("Batch result identity or order does not match the request")
|
| 240 |
+
_answers_for_questions(
|
| 241 |
+
questions, result["answers"], f"results.{index}.answers", confidence
|
| 242 |
+
)
|
| 243 |
+
row_input, row_output = _usage(result["usage"], f"results.{index}.usage")
|
| 244 |
+
input_tokens += row_input
|
| 245 |
+
output_tokens += row_output
|
| 246 |
+
aggregate_input, aggregate_output = _usage(response["usage"], "usage")
|
| 247 |
+
if aggregate_input != input_tokens or aggregate_output != output_tokens:
|
| 248 |
+
raise ValueError("Batch aggregate usage does not match the result rows")
|
direct_gateway.py
ADDED
|
@@ -0,0 +1,528 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Private transport from the public Gateway to per-model Decision runtimes.
|
| 2 |
+
|
| 3 |
+
The browser never receives backend locations. Every inference response is
|
| 4 |
+
validated against the originating strict request before it crosses the public
|
| 5 |
+
Gateway boundary.
|
| 6 |
+
"""
|
| 7 |
+
|
| 8 |
+
from __future__ import annotations
|
| 9 |
+
|
| 10 |
+
import asyncio
|
| 11 |
+
import ipaddress
|
| 12 |
+
import json
|
| 13 |
+
import math
|
| 14 |
+
import os
|
| 15 |
+
import time
|
| 16 |
+
from collections.abc import Callable, Collection, Mapping
|
| 17 |
+
from types import MappingProxyType
|
| 18 |
+
from urllib.parse import urlsplit
|
| 19 |
+
|
| 20 |
+
import httpx
|
| 21 |
+
|
| 22 |
+
from direct_contract import (
|
| 23 |
+
CONFIDENCE_DEFINITIONS,
|
| 24 |
+
MARGIN_ORDINAL_V1,
|
| 25 |
+
validate_response_for_request,
|
| 26 |
+
)
|
| 27 |
+
|
| 28 |
+
SINGLE_PATH = "/v1/systemone"
|
| 29 |
+
BATCH_PATH = "/v1/systemone/batches"
|
| 30 |
+
STATUS_PATH = "/api/status"
|
| 31 |
+
ARTIFACT_RESPONSE_HEADERS = {
|
| 32 |
+
"model": "X-Decision-Artifact-Model",
|
| 33 |
+
"revision": "X-Decision-Artifact-Revision",
|
| 34 |
+
"manifest_sha256": "X-Decision-Artifact-Manifest-Sha256",
|
| 35 |
+
"content_sha256": "X-Decision-Artifact-Content-Sha256",
|
| 36 |
+
}
|
| 37 |
+
|
| 38 |
+
MAX_ENDPOINT_CONFIG_BYTES = 64 * 1024
|
| 39 |
+
MAX_SINGLE_REQUEST_BYTES = 256 * 1024
|
| 40 |
+
MAX_BATCH_REQUEST_BYTES = 2 * 1024 * 1024
|
| 41 |
+
MAX_INFERENCE_RESPONSE_BYTES = 8 * 1024 * 1024
|
| 42 |
+
MAX_CONTROL_RESPONSE_BYTES = 64 * 1024
|
| 43 |
+
MAX_CONTROL_TIMEOUT_SECONDS = 2.0
|
| 44 |
+
MIN_TIMEOUT_SECONDS = 0.1
|
| 45 |
+
MAX_TIMEOUT_SECONDS = 300.0
|
| 46 |
+
PRIVATE_NETWORKS = (
|
| 47 |
+
ipaddress.ip_network("10.0.0.0/8"),
|
| 48 |
+
ipaddress.ip_network("172.16.0.0/12"),
|
| 49 |
+
ipaddress.ip_network("192.168.0.0/16"),
|
| 50 |
+
ipaddress.ip_network("fc00::/7"),
|
| 51 |
+
)
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
def _hex_digest(value: object, length: int) -> bool:
|
| 55 |
+
return (
|
| 56 |
+
isinstance(value, str)
|
| 57 |
+
and len(value) == length
|
| 58 |
+
and all(character in "0123456789abcdef" for character in value)
|
| 59 |
+
)
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
class DirectGatewayError(RuntimeError):
|
| 63 |
+
"""A sanitized direct-backend failure safe to return to a caller."""
|
| 64 |
+
|
| 65 |
+
def __init__(self, code: int, message: str, *, retry_after: str | None = None):
|
| 66 |
+
super().__init__(message)
|
| 67 |
+
self.code = code
|
| 68 |
+
self.message = message
|
| 69 |
+
self.retry_after = retry_after
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
def _unique_object(pairs):
|
| 73 |
+
result = {}
|
| 74 |
+
for key, value in pairs:
|
| 75 |
+
if key in result:
|
| 76 |
+
raise ValueError("Duplicate JSON key")
|
| 77 |
+
result[key] = value
|
| 78 |
+
return result
|
| 79 |
+
|
| 80 |
+
|
| 81 |
+
def _loads_strict(data: bytes | str):
|
| 82 |
+
return json.loads(
|
| 83 |
+
data,
|
| 84 |
+
object_pairs_hook=_unique_object,
|
| 85 |
+
parse_constant=lambda _value: (_ for _ in ()).throw(
|
| 86 |
+
ValueError("Nonfinite JSON value")
|
| 87 |
+
),
|
| 88 |
+
)
|
| 89 |
+
|
| 90 |
+
|
| 91 |
+
def _private_origin(value: object) -> str:
|
| 92 |
+
"""Accept an explicit private/loopback IP origin, never a URL path or DNS name."""
|
| 93 |
+
|
| 94 |
+
if not isinstance(value, str) or not value or len(value) > 2048:
|
| 95 |
+
raise ValueError("Every direct endpoint must be a bounded URL string")
|
| 96 |
+
try:
|
| 97 |
+
parsed = urlsplit(value)
|
| 98 |
+
port = parsed.port
|
| 99 |
+
except ValueError:
|
| 100 |
+
raise ValueError("Every direct endpoint must be a valid private origin") from None
|
| 101 |
+
if (
|
| 102 |
+
parsed.scheme not in {"http", "https"}
|
| 103 |
+
or parsed.username is not None
|
| 104 |
+
or parsed.password is not None
|
| 105 |
+
or parsed.query
|
| 106 |
+
or parsed.fragment
|
| 107 |
+
or parsed.path not in {"", "/"}
|
| 108 |
+
or not parsed.hostname
|
| 109 |
+
or port is None
|
| 110 |
+
or not 1 <= port <= 65535
|
| 111 |
+
):
|
| 112 |
+
raise ValueError(
|
| 113 |
+
"Every direct endpoint must be an http(s) origin with an explicit port"
|
| 114 |
+
)
|
| 115 |
+
if "%" in parsed.hostname:
|
| 116 |
+
raise ValueError("Scoped IPv6 direct endpoints are not supported")
|
| 117 |
+
try:
|
| 118 |
+
address = ipaddress.ip_address(parsed.hostname)
|
| 119 |
+
except ValueError:
|
| 120 |
+
raise ValueError(
|
| 121 |
+
"Every direct endpoint host must be a private or loopback IP literal"
|
| 122 |
+
) from None
|
| 123 |
+
if isinstance(address, ipaddress.IPv6Address) and address.ipv4_mapped:
|
| 124 |
+
address = address.ipv4_mapped
|
| 125 |
+
if not (
|
| 126 |
+
address.is_loopback
|
| 127 |
+
or any(
|
| 128 |
+
address.version == network.version and address in network
|
| 129 |
+
for network in PRIVATE_NETWORKS
|
| 130 |
+
)
|
| 131 |
+
):
|
| 132 |
+
raise ValueError("Every direct endpoint must stay on a private or loopback IP")
|
| 133 |
+
host = f"[{address.compressed}]" if address.version == 6 else address.compressed
|
| 134 |
+
return f"{parsed.scheme}://{host}:{port}"
|
| 135 |
+
|
| 136 |
+
|
| 137 |
+
def endpoints_from_environment(expected_models: Collection[str]) -> dict[str, str]:
|
| 138 |
+
raw = os.getenv("DECISION_DIRECT_ENDPOINTS", "")
|
| 139 |
+
if not raw:
|
| 140 |
+
raise ValueError("DECISION_DIRECT_ENDPOINTS is required for the direct backend")
|
| 141 |
+
if len(raw.encode("utf-8")) > MAX_ENDPOINT_CONFIG_BYTES:
|
| 142 |
+
raise ValueError("DECISION_DIRECT_ENDPOINTS is too large")
|
| 143 |
+
try:
|
| 144 |
+
value = _loads_strict(raw)
|
| 145 |
+
except (TypeError, ValueError, UnicodeError, RecursionError) as exc:
|
| 146 |
+
raise ValueError("DECISION_DIRECT_ENDPOINTS must be a strict JSON object") from exc
|
| 147 |
+
if not isinstance(value, dict) or set(value) != set(expected_models):
|
| 148 |
+
raise ValueError(
|
| 149 |
+
"DECISION_DIRECT_ENDPOINTS must cover exactly the configured canonical models"
|
| 150 |
+
)
|
| 151 |
+
return {model: _private_origin(endpoint) for model, endpoint in value.items()}
|
| 152 |
+
|
| 153 |
+
|
| 154 |
+
def timeout_from_environment() -> float:
|
| 155 |
+
raw = os.getenv("DECISION_DIRECT_TIMEOUT_SECONDS", "45")
|
| 156 |
+
try:
|
| 157 |
+
timeout = float(raw)
|
| 158 |
+
except ValueError as exc:
|
| 159 |
+
raise ValueError("DECISION_DIRECT_TIMEOUT_SECONDS must be numeric") from exc
|
| 160 |
+
if not math.isfinite(timeout) or not MIN_TIMEOUT_SECONDS <= timeout <= MAX_TIMEOUT_SECONDS:
|
| 161 |
+
raise ValueError(
|
| 162 |
+
"DECISION_DIRECT_TIMEOUT_SECONDS must be between 0.1 and 300"
|
| 163 |
+
)
|
| 164 |
+
return timeout
|
| 165 |
+
|
| 166 |
+
|
| 167 |
+
class DirectGateway:
|
| 168 |
+
"""Keep-alive direct client with fail-closed model and response validation."""
|
| 169 |
+
|
| 170 |
+
def __init__(
|
| 171 |
+
self,
|
| 172 |
+
endpoints: Mapping[str, str],
|
| 173 |
+
*,
|
| 174 |
+
expected_artifacts: Mapping[str, Mapping[str, str]],
|
| 175 |
+
timeout_seconds: float = 45.0,
|
| 176 |
+
client: httpx.AsyncClient | None = None,
|
| 177 |
+
max_response_bytes: int = MAX_INFERENCE_RESPONSE_BYTES,
|
| 178 |
+
clock: Callable[[], float] = time.perf_counter,
|
| 179 |
+
):
|
| 180 |
+
expected = tuple(expected_artifacts)
|
| 181 |
+
if (
|
| 182 |
+
not expected
|
| 183 |
+
or any(not isinstance(model, str) or not model for model in expected)
|
| 184 |
+
or len(set(expected)) != len(expected)
|
| 185 |
+
):
|
| 186 |
+
raise ValueError("Configure a nonempty set of canonical Decision models")
|
| 187 |
+
if set(endpoints) != set(expected):
|
| 188 |
+
raise ValueError("direct endpoints must cover exactly the configured models")
|
| 189 |
+
pinned = {}
|
| 190 |
+
confidence = {}
|
| 191 |
+
for model in expected:
|
| 192 |
+
item = expected_artifacts[model]
|
| 193 |
+
if not isinstance(item, Mapping):
|
| 194 |
+
raise ValueError("Pin each Decision model to a released artifact")
|
| 195 |
+
revision = item.get("revision")
|
| 196 |
+
manifest = item.get("manifest_sha256")
|
| 197 |
+
content = item.get("content_sha256")
|
| 198 |
+
if (
|
| 199 |
+
not _hex_digest(revision, 40)
|
| 200 |
+
or not _hex_digest(manifest, 64)
|
| 201 |
+
or (content is not None and not _hex_digest(content, 64))
|
| 202 |
+
):
|
| 203 |
+
raise ValueError("Pin each Decision model revision and manifest digest")
|
| 204 |
+
confidence[model] = item.get("confidence_definition", MARGIN_ORDINAL_V1)
|
| 205 |
+
if confidence[model] not in CONFIDENCE_DEFINITIONS:
|
| 206 |
+
raise ValueError("Unknown Decision confidence definition")
|
| 207 |
+
pinned[model] = MappingProxyType(
|
| 208 |
+
{
|
| 209 |
+
"revision": revision,
|
| 210 |
+
"manifest_sha256": manifest,
|
| 211 |
+
**({"content_sha256": content} if content is not None else {}),
|
| 212 |
+
}
|
| 213 |
+
)
|
| 214 |
+
timeout_valid = False
|
| 215 |
+
if not isinstance(timeout_seconds, bool) and isinstance(
|
| 216 |
+
timeout_seconds, (int, float)
|
| 217 |
+
):
|
| 218 |
+
try:
|
| 219 |
+
timeout_valid = math.isfinite(timeout_seconds) and (
|
| 220 |
+
MIN_TIMEOUT_SECONDS <= timeout_seconds <= MAX_TIMEOUT_SECONDS
|
| 221 |
+
)
|
| 222 |
+
except OverflowError:
|
| 223 |
+
timeout_valid = False
|
| 224 |
+
if not timeout_valid:
|
| 225 |
+
raise ValueError("direct timeout must be between 0.1 and 300 seconds")
|
| 226 |
+
if (
|
| 227 |
+
type(max_response_bytes) is not int
|
| 228 |
+
or not 1 <= max_response_bytes <= MAX_INFERENCE_RESPONSE_BYTES
|
| 229 |
+
):
|
| 230 |
+
raise ValueError("Invalid direct response limit")
|
| 231 |
+
normalized = {model: _private_origin(endpoints[model]) for model in expected}
|
| 232 |
+
if len(set(normalized.values())) != len(normalized):
|
| 233 |
+
raise ValueError("Each configured Decision model requires its own direct origin")
|
| 234 |
+
self._endpoints = MappingProxyType(normalized)
|
| 235 |
+
self.expected_artifacts = MappingProxyType(pinned)
|
| 236 |
+
self.confidence_definitions = MappingProxyType(confidence)
|
| 237 |
+
self.models = frozenset(expected)
|
| 238 |
+
self.timeout_seconds = float(timeout_seconds)
|
| 239 |
+
self._max_response_bytes = max_response_bytes
|
| 240 |
+
self._clock = clock
|
| 241 |
+
self._owns_client = client is None
|
| 242 |
+
self._client = client or httpx.AsyncClient(
|
| 243 |
+
follow_redirects=False,
|
| 244 |
+
timeout=self.timeout_seconds,
|
| 245 |
+
trust_env=False,
|
| 246 |
+
)
|
| 247 |
+
|
| 248 |
+
@classmethod
|
| 249 |
+
def from_environment(
|
| 250 |
+
cls, expected_artifacts: Mapping[str, Mapping[str, str]]
|
| 251 |
+
) -> DirectGateway:
|
| 252 |
+
return cls(
|
| 253 |
+
endpoints_from_environment(expected_artifacts),
|
| 254 |
+
expected_artifacts=expected_artifacts,
|
| 255 |
+
timeout_seconds=timeout_from_environment(),
|
| 256 |
+
)
|
| 257 |
+
|
| 258 |
+
async def aclose(self) -> None:
|
| 259 |
+
if self._owns_client:
|
| 260 |
+
await self._client.aclose()
|
| 261 |
+
|
| 262 |
+
def local_systemone_endpoints(self) -> Mapping[str, str]:
|
| 263 |
+
"""Server-only Tetris routes to the same private, pinned model instances."""
|
| 264 |
+
|
| 265 |
+
return MappingProxyType(
|
| 266 |
+
{model: origin + SINGLE_PATH for model, origin in self._endpoints.items()}
|
| 267 |
+
)
|
| 268 |
+
|
| 269 |
+
async def evaluate(self, payload: Mapping[str, object], *, batch: bool):
|
| 270 |
+
response, _request_ms = await self.evaluate_with_timing(payload, batch=batch)
|
| 271 |
+
return response
|
| 272 |
+
|
| 273 |
+
async def evaluate_with_timing(
|
| 274 |
+
self, payload: Mapping[str, object], *, batch: bool
|
| 275 |
+
) -> tuple[dict, float]:
|
| 276 |
+
"""Bind the inference response to its resident artifact in one POST."""
|
| 277 |
+
|
| 278 |
+
model = payload.get("model") if isinstance(payload, Mapping) else None
|
| 279 |
+
if not isinstance(model, str) or model not in self.models:
|
| 280 |
+
raise DirectGatewayError(422, "The selected canonical model is not configured")
|
| 281 |
+
if batch != ("states" in payload) or ("state" in payload and "states" in payload):
|
| 282 |
+
raise DirectGatewayError(422, "The Decision request does not match its route")
|
| 283 |
+
try:
|
| 284 |
+
encoded = json.dumps(
|
| 285 |
+
payload,
|
| 286 |
+
ensure_ascii=False,
|
| 287 |
+
separators=(",", ":"),
|
| 288 |
+
allow_nan=False,
|
| 289 |
+
).encode("utf-8")
|
| 290 |
+
except (TypeError, ValueError, UnicodeError, RecursionError) as exc:
|
| 291 |
+
raise DirectGatewayError(422, "The Decision request is not valid JSON") from exc
|
| 292 |
+
request_limit = MAX_BATCH_REQUEST_BYTES if batch else MAX_SINGLE_REQUEST_BYTES
|
| 293 |
+
if len(encoded) > request_limit:
|
| 294 |
+
raise DirectGatewayError(413, "Request body exceeds the endpoint limit")
|
| 295 |
+
started = self._clock()
|
| 296 |
+
response = await self._request_json(
|
| 297 |
+
"POST",
|
| 298 |
+
model,
|
| 299 |
+
BATCH_PATH if batch else SINGLE_PATH,
|
| 300 |
+
content=encoded,
|
| 301 |
+
limit=self._max_response_bytes,
|
| 302 |
+
attest_model=model,
|
| 303 |
+
)
|
| 304 |
+
request_ms = max(0.0, (self._clock() - started) * 1_000)
|
| 305 |
+
try:
|
| 306 |
+
validate_response_for_request(
|
| 307 |
+
payload, response, batch=batch,
|
| 308 |
+
confidence=self.confidence_definitions[model],
|
| 309 |
+
)
|
| 310 |
+
except (TypeError, ValueError, KeyError, OverflowError, RecursionError) as exc:
|
| 311 |
+
raise DirectGatewayError(
|
| 312 |
+
502, "The selected Decision runtime returned an invalid response"
|
| 313 |
+
) from exc
|
| 314 |
+
return response, request_ms
|
| 315 |
+
|
| 316 |
+
async def ready(self, model: str) -> bool:
|
| 317 |
+
"""Check the live artifact against the deployment pins."""
|
| 318 |
+
|
| 319 |
+
return (await self.probe(model))["loaded"]
|
| 320 |
+
|
| 321 |
+
async def probe(self, model: str) -> dict[str, object]:
|
| 322 |
+
"""Return readiness and parsed live identity, with no private topology."""
|
| 323 |
+
|
| 324 |
+
if model not in self.models:
|
| 325 |
+
return {"loaded": False, "artifact": None}
|
| 326 |
+
try:
|
| 327 |
+
control_timeout = min(self.timeout_seconds, MAX_CONTROL_TIMEOUT_SECONDS)
|
| 328 |
+
status = await self._request_json(
|
| 329 |
+
"GET",
|
| 330 |
+
model,
|
| 331 |
+
STATUS_PATH,
|
| 332 |
+
limit=MAX_CONTROL_RESPONSE_BYTES,
|
| 333 |
+
timeout_seconds=control_timeout,
|
| 334 |
+
)
|
| 335 |
+
artifact = self._parse_artifact(status)
|
| 336 |
+
loaded = (
|
| 337 |
+
isinstance(status, dict)
|
| 338 |
+
and status.get("status") == "ready"
|
| 339 |
+
and self._artifact_matches(model, artifact)
|
| 340 |
+
)
|
| 341 |
+
return {"loaded": bool(loaded), "artifact": artifact}
|
| 342 |
+
except DirectGatewayError:
|
| 343 |
+
return {"loaded": False, "artifact": None}
|
| 344 |
+
|
| 345 |
+
@staticmethod
|
| 346 |
+
def _parse_artifact(status: object) -> dict[str, str] | None:
|
| 347 |
+
if not isinstance(status, dict):
|
| 348 |
+
return None
|
| 349 |
+
artifact = status.get("artifact")
|
| 350 |
+
if not isinstance(artifact, dict) or set(artifact) != {
|
| 351 |
+
"model", "revision", "manifest_sha256", "content_sha256"
|
| 352 |
+
}:
|
| 353 |
+
return None
|
| 354 |
+
if (
|
| 355 |
+
not isinstance(artifact["model"], str)
|
| 356 |
+
or not 1 <= len(artifact["model"]) <= 128
|
| 357 |
+
or any(
|
| 358 |
+
not 33 <= ord(character) <= 126 or character == ","
|
| 359 |
+
for character in artifact["model"]
|
| 360 |
+
)
|
| 361 |
+
or not _hex_digest(artifact["revision"], 40)
|
| 362 |
+
or not _hex_digest(artifact["manifest_sha256"], 64)
|
| 363 |
+
or not _hex_digest(artifact["content_sha256"], 64)
|
| 364 |
+
):
|
| 365 |
+
return None
|
| 366 |
+
return dict(artifact)
|
| 367 |
+
|
| 368 |
+
@classmethod
|
| 369 |
+
def _parse_response_artifact(cls, headers: httpx.Headers) -> dict[str, str] | None:
|
| 370 |
+
fields = {}
|
| 371 |
+
for field, header in ARTIFACT_RESPONSE_HEADERS.items():
|
| 372 |
+
values = headers.get_list(header)
|
| 373 |
+
if len(values) != 1:
|
| 374 |
+
return None
|
| 375 |
+
fields[field] = values[0]
|
| 376 |
+
return cls._parse_artifact({"artifact": fields})
|
| 377 |
+
|
| 378 |
+
def _artifact_matches(self, model: str, artifact: dict[str, str] | None) -> bool:
|
| 379 |
+
if artifact is None:
|
| 380 |
+
return False
|
| 381 |
+
pinned = self.expected_artifacts[model]
|
| 382 |
+
return (
|
| 383 |
+
artifact["model"] == model
|
| 384 |
+
and artifact["revision"] == pinned["revision"]
|
| 385 |
+
and artifact["manifest_sha256"] == pinned["manifest_sha256"]
|
| 386 |
+
and (
|
| 387 |
+
"content_sha256" not in pinned
|
| 388 |
+
or artifact["content_sha256"] == pinned["content_sha256"]
|
| 389 |
+
)
|
| 390 |
+
)
|
| 391 |
+
|
| 392 |
+
async def _request_json(
|
| 393 |
+
self,
|
| 394 |
+
method: str,
|
| 395 |
+
model: str,
|
| 396 |
+
path: str,
|
| 397 |
+
*,
|
| 398 |
+
content: bytes | None = None,
|
| 399 |
+
limit: int,
|
| 400 |
+
timeout_seconds: float | None = None,
|
| 401 |
+
attest_model: str | None = None,
|
| 402 |
+
):
|
| 403 |
+
endpoint = self._endpoints.get(model)
|
| 404 |
+
if endpoint is None:
|
| 405 |
+
raise DirectGatewayError(422, "The selected canonical model is not configured")
|
| 406 |
+
request_timeout = timeout_seconds or self.timeout_seconds
|
| 407 |
+
try:
|
| 408 |
+
headers = {
|
| 409 |
+
"Accept": "application/json",
|
| 410 |
+
"Accept-Encoding": "identity",
|
| 411 |
+
}
|
| 412 |
+
if content is not None:
|
| 413 |
+
headers["Content-Type"] = "application/json"
|
| 414 |
+
async with asyncio.timeout(request_timeout):
|
| 415 |
+
async with self._client.stream(
|
| 416 |
+
method,
|
| 417 |
+
endpoint + path,
|
| 418 |
+
content=content,
|
| 419 |
+
headers=headers,
|
| 420 |
+
follow_redirects=False,
|
| 421 |
+
timeout=request_timeout,
|
| 422 |
+
) as response:
|
| 423 |
+
if 300 <= response.status_code < 400:
|
| 424 |
+
raise DirectGatewayError(
|
| 425 |
+
502,
|
| 426 |
+
"The selected Decision runtime returned an unsupported redirect",
|
| 427 |
+
)
|
| 428 |
+
if response.status_code >= 400:
|
| 429 |
+
self._raise_status(response)
|
| 430 |
+
if attest_model is not None and not self._artifact_matches(
|
| 431 |
+
attest_model, self._parse_response_artifact(response.headers)
|
| 432 |
+
):
|
| 433 |
+
raise DirectGatewayError(
|
| 434 |
+
503, "The selected Decision runtime artifact is unavailable"
|
| 435 |
+
)
|
| 436 |
+
return await self._read_json_response(response, limit)
|
| 437 |
+
except asyncio.CancelledError:
|
| 438 |
+
raise
|
| 439 |
+
except DirectGatewayError:
|
| 440 |
+
raise
|
| 441 |
+
except (TimeoutError, httpx.TimeoutException) as exc:
|
| 442 |
+
raise DirectGatewayError(
|
| 443 |
+
504, "The selected Decision runtime did not respond in time"
|
| 444 |
+
) from exc
|
| 445 |
+
except httpx.HTTPError as exc:
|
| 446 |
+
raise DirectGatewayError(
|
| 447 |
+
503, "The selected Decision runtime is unavailable"
|
| 448 |
+
) from exc
|
| 449 |
+
|
| 450 |
+
@staticmethod
|
| 451 |
+
def _raise_status(response: httpx.Response) -> None:
|
| 452 |
+
status = response.status_code
|
| 453 |
+
if status == 413:
|
| 454 |
+
raise DirectGatewayError(
|
| 455 |
+
413, "Decision input exceeds the selected model limit"
|
| 456 |
+
)
|
| 457 |
+
if status in {429, 529}:
|
| 458 |
+
raise DirectGatewayError(
|
| 459 |
+
status,
|
| 460 |
+
"The selected Decision runtime is temporarily overloaded",
|
| 461 |
+
retry_after=_safe_retry_after(response.headers.get("retry-after")),
|
| 462 |
+
)
|
| 463 |
+
if status == 503:
|
| 464 |
+
raise DirectGatewayError(
|
| 465 |
+
503,
|
| 466 |
+
"The selected Decision runtime is unavailable",
|
| 467 |
+
retry_after=_safe_retry_after(response.headers.get("retry-after")),
|
| 468 |
+
)
|
| 469 |
+
if status == 504:
|
| 470 |
+
raise DirectGatewayError(
|
| 471 |
+
504, "The selected Decision runtime did not respond in time"
|
| 472 |
+
)
|
| 473 |
+
raise DirectGatewayError(
|
| 474 |
+
502, "The selected Decision runtime rejected a validated request"
|
| 475 |
+
)
|
| 476 |
+
|
| 477 |
+
@staticmethod
|
| 478 |
+
async def _read_json_response(response: httpx.Response, limit: int):
|
| 479 |
+
content_type = (
|
| 480 |
+
response.headers.get("content-type", "").split(";", 1)[0].strip().lower()
|
| 481 |
+
)
|
| 482 |
+
if content_type != "application/json":
|
| 483 |
+
raise DirectGatewayError(
|
| 484 |
+
502, "The selected Decision runtime returned an invalid content type"
|
| 485 |
+
)
|
| 486 |
+
encoding = response.headers.get("content-encoding", "identity").strip().lower()
|
| 487 |
+
if encoding not in {"", "identity"}:
|
| 488 |
+
raise DirectGatewayError(
|
| 489 |
+
502, "The selected Decision runtime returned an unsupported encoding"
|
| 490 |
+
)
|
| 491 |
+
declared = response.headers.get("content-length")
|
| 492 |
+
if declared is not None:
|
| 493 |
+
try:
|
| 494 |
+
declared_size = int(declared)
|
| 495 |
+
except ValueError as exc:
|
| 496 |
+
raise DirectGatewayError(
|
| 497 |
+
502, "The selected Decision runtime returned an invalid response"
|
| 498 |
+
) from exc
|
| 499 |
+
if declared_size < 0 or declared_size > limit:
|
| 500 |
+
raise DirectGatewayError(
|
| 501 |
+
502, "The selected Decision runtime returned an oversized response"
|
| 502 |
+
)
|
| 503 |
+
body = bytearray()
|
| 504 |
+
# httpx may already hold a completed response body (for example when
|
| 505 |
+
# using a local transport). aiter_bytes supports both that case and a
|
| 506 |
+
# live stream. Non-identity content encodings were rejected above.
|
| 507 |
+
async for chunk in response.aiter_bytes(chunk_size=64 * 1024):
|
| 508 |
+
if len(chunk) > limit - len(body):
|
| 509 |
+
raise DirectGatewayError(
|
| 510 |
+
502, "The selected Decision runtime returned an oversized response"
|
| 511 |
+
)
|
| 512 |
+
body.extend(chunk)
|
| 513 |
+
try:
|
| 514 |
+
return _loads_strict(bytes(body))
|
| 515 |
+
except (TypeError, ValueError, UnicodeError, RecursionError) as exc:
|
| 516 |
+
raise DirectGatewayError(
|
| 517 |
+
502, "The selected Decision runtime returned invalid JSON"
|
| 518 |
+
) from exc
|
| 519 |
+
|
| 520 |
+
|
| 521 |
+
def _safe_retry_after(value: str | None) -> str | None:
|
| 522 |
+
if value is None:
|
| 523 |
+
return None
|
| 524 |
+
try:
|
| 525 |
+
seconds = int(value)
|
| 526 |
+
except ValueError:
|
| 527 |
+
return None
|
| 528 |
+
return str(seconds) if 1 <= seconds <= 300 else None
|
engine.py
ADDED
|
@@ -0,0 +1,84 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""One persistent native model, explicit offline state, no synthetic predictions."""
|
| 2 |
+
import os
|
| 3 |
+
import sys
|
| 4 |
+
import threading
|
| 5 |
+
import time
|
| 6 |
+
from pathlib import Path
|
| 7 |
+
from contract import MODEL, to_records, to_response, timing_shape
|
| 8 |
+
|
| 9 |
+
DEFAULT_MANIFEST = "da603662bc57e89ccfb51c972ed9c1f2825f267597353cf1337df9117a3dfabe"
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
class Unavailable(RuntimeError):
|
| 13 |
+
pass
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
class Busy(RuntimeError):
|
| 17 |
+
pass
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
class Engine:
|
| 21 |
+
def __init__(self):
|
| 22 |
+
self.model = os.getenv("DECISION_MODEL_ID", MODEL)
|
| 23 |
+
self.native = None
|
| 24 |
+
self.lock = threading.Lock()
|
| 25 |
+
self.phase = "not_loaded"
|
| 26 |
+
self.native_path = os.getenv("DECISION_NATIVE", "")
|
| 27 |
+
self.manifest = os.getenv("DECISION_MANIFEST_SHA256", DEFAULT_MANIFEST)
|
| 28 |
+
|
| 29 |
+
def status(self):
|
| 30 |
+
return {"configured": bool(self.native_path), "loaded": self.native is not None,
|
| 31 |
+
"phase": self.phase if self.native_path else "offline", "model": self.model,
|
| 32 |
+
"manifest_sha256": self.manifest, "complete_input_tokens": 1024,
|
| 33 |
+
"context_batch": True, "backend": "native_rocm", "live_inference": self.native is not None}
|
| 34 |
+
|
| 35 |
+
def _load(self):
|
| 36 |
+
if self.native is not None:
|
| 37 |
+
return 0.0
|
| 38 |
+
if not self.native_path:
|
| 39 |
+
raise Unavailable("No model is connected. Set DECISION_NATIVE to a verified native export on the AMD server.")
|
| 40 |
+
runtime = Path(os.getenv("DECISION_RUNTIME_PATH", str(Path(__file__).resolve().parents[1] / "release")))
|
| 41 |
+
if str(runtime) not in sys.path:
|
| 42 |
+
sys.path.insert(0, str(runtime))
|
| 43 |
+
started = time.perf_counter()
|
| 44 |
+
self.phase = "loading"
|
| 45 |
+
import torch
|
| 46 |
+
from decision_runtime import load_native
|
| 47 |
+
if torch.version.hip is None or not torch.cuda.is_available() or torch.cuda.device_count() != 1:
|
| 48 |
+
raise Unavailable("Expose exactly one AMD ROCm GPU for the native preview. CPU fallback is not enabled.")
|
| 49 |
+
torch.cuda.set_device(0)
|
| 50 |
+
torch.set_num_threads(2)
|
| 51 |
+
torch.backends.cuda.matmul.allow_tf32 = False
|
| 52 |
+
torch.backends.cudnn.allow_tf32 = False
|
| 53 |
+
torch.backends.mha.set_fastpath_enabled(False)
|
| 54 |
+
self.native = load_native(self.native_path, expected_manifest_sha256=self.manifest, device="cuda:0")
|
| 55 |
+
self.phase = "ready"
|
| 56 |
+
return (time.perf_counter() - started) * 1000
|
| 57 |
+
|
| 58 |
+
def evaluate(self, body, records):
|
| 59 |
+
if body.get("model", MODEL) != self.model:
|
| 60 |
+
raise ValueError("Request model does not match this worker")
|
| 61 |
+
if records != to_records(body, model=self.model):
|
| 62 |
+
raise ValueError("Request records changed")
|
| 63 |
+
if not self.lock.acquire(blocking=False):
|
| 64 |
+
raise Busy("The model is finishing another request. Try again in a moment.")
|
| 65 |
+
started = time.perf_counter()
|
| 66 |
+
try:
|
| 67 |
+
load_ms = self._load()
|
| 68 |
+
import torch
|
| 69 |
+
from decision_inference._grouped import predict_grouped_1k as predict_1k
|
| 70 |
+
self.phase = "running"
|
| 71 |
+
torch.cuda.synchronize()
|
| 72 |
+
begin = time.perf_counter()
|
| 73 |
+
predictions = predict_1k(self.native, records, batch_size=8)
|
| 74 |
+
torch.cuda.synchronize()
|
| 75 |
+
inference_ms = (time.perf_counter() - begin) * 1000
|
| 76 |
+
result = to_response(body, records, predictions, model=self.model)
|
| 77 |
+
result["timing"] = {"model_load_ms": round(load_ms, 2), "inference_ms": round(inference_ms, 2),
|
| 78 |
+
"server_ms": round((time.perf_counter() - started) * 1000, 2),
|
| 79 |
+
"cold_start": load_ms > 0, **timing_shape(body, records)}
|
| 80 |
+
result["source"] = "live_native"
|
| 81 |
+
return result
|
| 82 |
+
finally:
|
| 83 |
+
self.phase = "ready" if self.native is not None else "not_loaded"
|
| 84 |
+
self.lock.release()
|
examples.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
http_pull_worker.py
ADDED
|
@@ -0,0 +1,183 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Outbound queue worker for a pinned private Decision HTTP runtime."""
|
| 2 |
+
|
| 3 |
+
import asyncio
|
| 4 |
+
import concurrent.futures
|
| 5 |
+
import json
|
| 6 |
+
import logging
|
| 7 |
+
import math
|
| 8 |
+
import os
|
| 9 |
+
import signal
|
| 10 |
+
import threading
|
| 11 |
+
|
| 12 |
+
from contract import to_records
|
| 13 |
+
from direct_gateway import DirectGateway, DirectGatewayError
|
| 14 |
+
from model_registry import PROFILES, model_registry
|
| 15 |
+
from model_runtime_client import ModelRuntimeGateway
|
| 16 |
+
from pull_worker import Gateway, Worker
|
| 17 |
+
|
| 18 |
+
LOG = logging.getLogger("decision.http_worker")
|
| 19 |
+
# `decision_serve_v1`: `vllm-sr decision serve` (artifact headers, native batches).
|
| 20 |
+
# `vllm_sr_runtime_v1`: the built-in model runtime (`vllm-sr-runtime serve`).
|
| 21 |
+
RUNTIME_APIS = {
|
| 22 |
+
"decision_serve_v1": DirectGateway,
|
| 23 |
+
"vllm_sr_runtime_v1": ModelRuntimeGateway,
|
| 24 |
+
}
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
class HTTPRuntime:
|
| 28 |
+
"""Run one attested private client on its own event loop for probe and inference."""
|
| 29 |
+
|
| 30 |
+
def __init__(self, model, origin, artifact, *, timeout_seconds=40, client=None,
|
| 31 |
+
api="decision_serve_v1"):
|
| 32 |
+
if model not in PROFILES or artifact.get("repo_id") != PROFILES[model]["repo_id"]:
|
| 33 |
+
raise ValueError("Select one exact Decision model and canonical artifact")
|
| 34 |
+
if api not in RUNTIME_APIS:
|
| 35 |
+
raise ValueError("Select a supported Decision runtime API")
|
| 36 |
+
if (
|
| 37 |
+
isinstance(timeout_seconds, bool)
|
| 38 |
+
or not isinstance(timeout_seconds, (int, float))
|
| 39 |
+
or not math.isfinite(timeout_seconds)
|
| 40 |
+
or not 0.1 <= timeout_seconds <= 90
|
| 41 |
+
):
|
| 42 |
+
raise ValueError("Runtime timeout must be between 0.1 and 90 seconds")
|
| 43 |
+
self.model = model
|
| 44 |
+
self.canonical_model = PROFILES[model].get("runtime_model", artifact["repo_id"])
|
| 45 |
+
self.manifest = artifact["manifest_sha256"]
|
| 46 |
+
self.phase = "not_loaded"
|
| 47 |
+
self._gateway = RUNTIME_APIS[api](
|
| 48 |
+
{self.canonical_model: origin},
|
| 49 |
+
expected_artifacts={self.canonical_model: {
|
| 50 |
+
"revision": artifact["revision"],
|
| 51 |
+
"manifest_sha256": self.manifest,
|
| 52 |
+
**({"content_sha256": artifact["content_sha256"]}
|
| 53 |
+
if "content_sha256" in artifact else {}),
|
| 54 |
+
"confidence_definition": PROFILES[model]["confidence"],
|
| 55 |
+
}},
|
| 56 |
+
timeout_seconds=timeout_seconds,
|
| 57 |
+
client=client,
|
| 58 |
+
)
|
| 59 |
+
self._loop = None
|
| 60 |
+
self._thread = None
|
| 61 |
+
|
| 62 |
+
def _submit(self, coroutine, timeout):
|
| 63 |
+
if self._loop is None:
|
| 64 |
+
coroutine.close()
|
| 65 |
+
raise RuntimeError("Runtime client is not started")
|
| 66 |
+
future = asyncio.run_coroutine_threadsafe(coroutine, self._loop)
|
| 67 |
+
try:
|
| 68 |
+
return future.result(timeout=timeout)
|
| 69 |
+
except concurrent.futures.TimeoutError:
|
| 70 |
+
future.cancel()
|
| 71 |
+
raise TimeoutError("Runtime call exceeded its deadline") from None
|
| 72 |
+
|
| 73 |
+
def _load(self):
|
| 74 |
+
if self._loop is not None:
|
| 75 |
+
return
|
| 76 |
+
self._loop = asyncio.new_event_loop()
|
| 77 |
+
|
| 78 |
+
def serve():
|
| 79 |
+
asyncio.set_event_loop(self._loop)
|
| 80 |
+
self._loop.run_forever()
|
| 81 |
+
|
| 82 |
+
self._thread = threading.Thread(target=serve, name="decision-runtime-http", daemon=True)
|
| 83 |
+
self._thread.start()
|
| 84 |
+
if not self.ready():
|
| 85 |
+
raise RuntimeError("Selected Decision runtime is not ready with its pinned artifact")
|
| 86 |
+
self.phase = "ready"
|
| 87 |
+
|
| 88 |
+
def ready(self):
|
| 89 |
+
observed = self._submit(self._gateway.probe(self.canonical_model), 5)
|
| 90 |
+
return observed["loaded"] is True
|
| 91 |
+
|
| 92 |
+
def evaluate(self, body, records):
|
| 93 |
+
if body.get("model") != self.model or records != to_records(body, model=self.model):
|
| 94 |
+
raise ValueError("Request does not match this worker")
|
| 95 |
+
request = dict(body, model=self.canonical_model)
|
| 96 |
+
self.phase = "running"
|
| 97 |
+
try:
|
| 98 |
+
try:
|
| 99 |
+
result = self._submit(
|
| 100 |
+
self._gateway.evaluate(request, batch="states" in body),
|
| 101 |
+
self._gateway.timeout_seconds + 5,
|
| 102 |
+
)
|
| 103 |
+
except DirectGatewayError as exc:
|
| 104 |
+
if exc.code == 413:
|
| 105 |
+
raise ValueError("Selected model rejected the complete input") from None
|
| 106 |
+
raise
|
| 107 |
+
return {"kind": "http_runtime_v1", "response": result}
|
| 108 |
+
finally:
|
| 109 |
+
self.phase = "ready"
|
| 110 |
+
|
| 111 |
+
def close(self):
|
| 112 |
+
if self._loop is None:
|
| 113 |
+
return
|
| 114 |
+
try:
|
| 115 |
+
self._submit(self._gateway.aclose(), 5)
|
| 116 |
+
finally:
|
| 117 |
+
self._loop.call_soon_threadsafe(self._loop.stop)
|
| 118 |
+
self._thread.join(timeout=5)
|
| 119 |
+
if self._thread.is_alive():
|
| 120 |
+
raise RuntimeError("Runtime client did not stop")
|
| 121 |
+
self._loop.close()
|
| 122 |
+
self._loop = None
|
| 123 |
+
self._thread = None
|
| 124 |
+
|
| 125 |
+
|
| 126 |
+
class HTTPWorker(Worker):
|
| 127 |
+
heartbeat_join_timeout = 38
|
| 128 |
+
|
| 129 |
+
def heartbeat(self):
|
| 130 |
+
if self.runtime.phase != "running" and not self.runtime.ready():
|
| 131 |
+
raise RuntimeError("Selected Decision runtime is unavailable")
|
| 132 |
+
return super().heartbeat()
|
| 133 |
+
|
| 134 |
+
def run(self):
|
| 135 |
+
try:
|
| 136 |
+
super().run()
|
| 137 |
+
finally:
|
| 138 |
+
self.runtime.close()
|
| 139 |
+
|
| 140 |
+
|
| 141 |
+
def runtime_from_environment():
|
| 142 |
+
raw_registry = os.getenv("DECISION_MODEL_REGISTRY_V2", "")
|
| 143 |
+
if not raw_registry:
|
| 144 |
+
raise ValueError("DECISION_MODEL_REGISTRY_V2 is required")
|
| 145 |
+
try:
|
| 146 |
+
registry = model_registry(json.loads(raw_registry))
|
| 147 |
+
except (TypeError, ValueError, KeyError, RecursionError) as exc:
|
| 148 |
+
raise ValueError("Invalid Decision model registry") from exc
|
| 149 |
+
model = os.getenv("DECISION_WORKER_MODEL", "")
|
| 150 |
+
if model not in registry:
|
| 151 |
+
raise ValueError("DECISION_WORKER_MODEL must be an exact configured wire ID")
|
| 152 |
+
item = registry[model]
|
| 153 |
+
if "revision" not in item:
|
| 154 |
+
raise ValueError("The selected model requires a pinned Hub revision")
|
| 155 |
+
try:
|
| 156 |
+
timeout = float(os.getenv("DECISION_RUNTIME_TIMEOUT_SECONDS", "40"))
|
| 157 |
+
except ValueError as exc:
|
| 158 |
+
raise ValueError("DECISION_RUNTIME_TIMEOUT_SECONDS must be numeric") from exc
|
| 159 |
+
return HTTPRuntime(model, os.getenv("DECISION_RUNTIME_URL", ""), item,
|
| 160 |
+
timeout_seconds=timeout,
|
| 161 |
+
api=os.getenv("DECISION_RUNTIME_API", "decision_serve_v1"))
|
| 162 |
+
|
| 163 |
+
|
| 164 |
+
def main():
|
| 165 |
+
logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s")
|
| 166 |
+
# Every heartbeat probes the runtime; keep per-request transport lines out of the log.
|
| 167 |
+
logging.getLogger("httpx").setLevel(logging.WARNING)
|
| 168 |
+
runtime = runtime_from_environment()
|
| 169 |
+
gateway = Gateway(os.getenv("DECISION_STUDIO_URL", ""),
|
| 170 |
+
os.getenv("DECISION_WORKER_TOKEN", ""),
|
| 171 |
+
allow_loopback=os.getenv("DECISION_ALLOW_HTTP_LOOPBACK") == "1")
|
| 172 |
+
worker = HTTPWorker(gateway, runtime)
|
| 173 |
+
for sig in (signal.SIGINT, signal.SIGTERM):
|
| 174 |
+
signal.signal(sig, lambda *_: worker.stop.set())
|
| 175 |
+
worker.run()
|
| 176 |
+
|
| 177 |
+
|
| 178 |
+
if __name__ == "__main__":
|
| 179 |
+
try:
|
| 180 |
+
main()
|
| 181 |
+
except Exception as exc: # noqa: BLE001 - never print secret-bearing exception text
|
| 182 |
+
LOG.error("HTTP worker stopped: %s", type(exc).__name__)
|
| 183 |
+
raise SystemExit(1) from None
|
model_registry.py
ADDED
|
@@ -0,0 +1,145 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Explicit released identities and model-specific UI/input contracts."""
|
| 2 |
+
import json
|
| 3 |
+
import os
|
| 4 |
+
from contract import MODEL
|
| 5 |
+
from engine import DEFAULT_MANIFEST
|
| 6 |
+
|
| 7 |
+
REQUIRED = {'id', 'label', 'version', 'manifest_sha256'}
|
| 8 |
+
OPTIONAL = {'complete_input_tokens', 'description', 'parameter_label', 'example_ids', 'default_example_id', 'repo_id', 'revision', 'content_sha256'}
|
| 9 |
+
# Versioned, Decision-owned confidence statistics. Decision 1.0 runtimes report
|
| 10 |
+
# the top-two margin (Choice) and ordinal concentration (Score); the Decision 2.0
|
| 11 |
+
# packages report one minus the normalized entropy for both.
|
| 12 |
+
MARGIN_ORDINAL_V1 = 'margin_ordinal_v1'
|
| 13 |
+
NORMALIZED_ENTROPY_V2 = 'normalized_entropy_v2'
|
| 14 |
+
# The one ordered catalog used by model discovery and Tetris presentation:
|
| 15 |
+
# newest generation first, then descending published size, with general Kai
|
| 16 |
+
# before specialist Lex at 0.6B.
|
| 17 |
+
MODEL_ORDER = (
|
| 18 |
+
'decision2-vega', 'decision2-lux', 'decision2-nox', 'decision2-sol', 'decision2-eos', 'decision2-kai',
|
| 19 |
+
'decision-lux', 'decision-nox', 'decision-sol', 'decision-eos', MODEL, 'decision-lex',
|
| 20 |
+
)
|
| 21 |
+
# A deployed Decision 1.0 runtime attests the artifact under the Hub ID it was
|
| 22 |
+
# loaded from, before the repositories moved to `vllm-sr`. `runtime_model` is that
|
| 23 |
+
# attested ID; every public surface still reports `repo_id`.
|
| 24 |
+
PROFILES = {
|
| 25 |
+
'decision2-vega': {
|
| 26 |
+
'complete_input_tokens': 32768,
|
| 27 |
+
'repo_id': 'vllm-sr/Decision-2.0-Vega-27B',
|
| 28 |
+
'confidence': NORMALIZED_ENTROPY_V2,
|
| 29 |
+
},
|
| 30 |
+
'decision2-lux': {
|
| 31 |
+
'complete_input_tokens': 16384,
|
| 32 |
+
'repo_id': 'vllm-sr/Decision-2.0-Lux-9B',
|
| 33 |
+
'confidence': NORMALIZED_ENTROPY_V2,
|
| 34 |
+
},
|
| 35 |
+
'decision2-nox': {
|
| 36 |
+
'complete_input_tokens': 16384,
|
| 37 |
+
'repo_id': 'vllm-sr/Decision-2.0-Nox-4B',
|
| 38 |
+
'confidence': NORMALIZED_ENTROPY_V2,
|
| 39 |
+
},
|
| 40 |
+
'decision2-sol': {
|
| 41 |
+
'complete_input_tokens': 16384,
|
| 42 |
+
'repo_id': 'vllm-sr/Decision-2.0-Sol-2B',
|
| 43 |
+
'confidence': NORMALIZED_ENTROPY_V2,
|
| 44 |
+
},
|
| 45 |
+
'decision2-eos': {
|
| 46 |
+
'complete_input_tokens': 16384,
|
| 47 |
+
'repo_id': 'vllm-sr/Decision-2.0-Eos-0.8B',
|
| 48 |
+
'confidence': NORMALIZED_ENTROPY_V2,
|
| 49 |
+
},
|
| 50 |
+
'decision2-kai': {
|
| 51 |
+
'complete_input_tokens': 8192,
|
| 52 |
+
'repo_id': 'vllm-sr/Decision-2.0-Kai-0.6B',
|
| 53 |
+
'confidence': NORMALIZED_ENTROPY_V2,
|
| 54 |
+
},
|
| 55 |
+
MODEL: {
|
| 56 |
+
'complete_input_tokens': 1024,
|
| 57 |
+
'repo_id': 'vllm-sr/Decision-1.0-Kai-0.6B',
|
| 58 |
+
'runtime_model': 'llm-semantic-router/Decision-1.0-Kai-0.6B',
|
| 59 |
+
'confidence': MARGIN_ORDINAL_V1,
|
| 60 |
+
},
|
| 61 |
+
'decision-lex': {
|
| 62 |
+
'complete_input_tokens': 1024,
|
| 63 |
+
'repo_id': 'vllm-sr/Decision-1.0-Lex-0.6B',
|
| 64 |
+
'runtime_model': 'llm-semantic-router/Decision-1.0-Lex-0.6B',
|
| 65 |
+
'confidence': MARGIN_ORDINAL_V1,
|
| 66 |
+
},
|
| 67 |
+
'decision-eos': {
|
| 68 |
+
'complete_input_tokens': 16384,
|
| 69 |
+
'repo_id': 'vllm-sr/Decision-1.0-Eos-0.8B',
|
| 70 |
+
'runtime_model': 'llm-semantic-router/Decision-1.0-Eos-0.8B',
|
| 71 |
+
'confidence': MARGIN_ORDINAL_V1,
|
| 72 |
+
},
|
| 73 |
+
'decision-sol': {
|
| 74 |
+
'complete_input_tokens': 16384,
|
| 75 |
+
'repo_id': 'vllm-sr/Decision-1.0-Sol-2B',
|
| 76 |
+
'runtime_model': 'llm-semantic-router/Decision-1.0-Sol-2B',
|
| 77 |
+
'confidence': MARGIN_ORDINAL_V1,
|
| 78 |
+
},
|
| 79 |
+
'decision-nox': {
|
| 80 |
+
'complete_input_tokens': 16384,
|
| 81 |
+
'repo_id': 'vllm-sr/Decision-1.0-Nox-4B',
|
| 82 |
+
'runtime_model': 'llm-semantic-router/Decision-1.0-Nox-4B',
|
| 83 |
+
'confidence': MARGIN_ORDINAL_V1,
|
| 84 |
+
},
|
| 85 |
+
'decision-lux': {
|
| 86 |
+
'complete_input_tokens': 16384,
|
| 87 |
+
'repo_id': 'vllm-sr/Decision-1.0-Lux-9B',
|
| 88 |
+
'runtime_model': 'llm-semantic-router/Decision-1.0-Lux-9B',
|
| 89 |
+
'confidence': MARGIN_ORDINAL_V1,
|
| 90 |
+
},
|
| 91 |
+
}
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
def model_registry(value=None):
|
| 95 |
+
if value is None:
|
| 96 |
+
raw = os.getenv('DECISION_MODEL_REGISTRY_V2') or os.getenv('DECISION_MODEL_REGISTRY')
|
| 97 |
+
value = json.loads(raw) if raw else [{'id': MODEL, 'label': 'Kai', 'version': '1.0',
|
| 98 |
+
'manifest_sha256': os.getenv('DECISION_MANIFEST_SHA256', DEFAULT_MANIFEST)}]
|
| 99 |
+
if not isinstance(value, list) or not 1 <= len(value) <= len(PROFILES):
|
| 100 |
+
raise ValueError(f'Configure one to {len(PROFILES)} explicitly released models')
|
| 101 |
+
result = {}
|
| 102 |
+
for item in value:
|
| 103 |
+
if not isinstance(item, dict) or REQUIRED - set(item) or set(item) - REQUIRED - OPTIONAL:
|
| 104 |
+
raise ValueError('Invalid model metadata fields')
|
| 105 |
+
if any(not isinstance(item[k], str) or not 1 <= len(item[k]) <= 128 for k in REQUIRED):
|
| 106 |
+
raise ValueError('Invalid model metadata')
|
| 107 |
+
if any(c not in 'abcdefghijklmnopqrstuvwxyz0123456789-_' for c in item['id']):
|
| 108 |
+
raise ValueError('Invalid model wire identifier')
|
| 109 |
+
h = item['manifest_sha256']
|
| 110 |
+
if len(h) != 64 or any(c not in '0123456789abcdef' for c in h):
|
| 111 |
+
raise ValueError('Pin the released manifest SHA-256')
|
| 112 |
+
if item['id'] in result or h in {m['manifest_sha256'] for m in result.values()}:
|
| 113 |
+
raise ValueError('Model IDs and manifests must be distinct')
|
| 114 |
+
profile = PROFILES.get(item['id'])
|
| 115 |
+
if profile is None:
|
| 116 |
+
raise ValueError('Use an explicitly released Decision model')
|
| 117 |
+
cap = item.get('complete_input_tokens', profile['complete_input_tokens'])
|
| 118 |
+
if type(cap) is not int or not 1 <= cap <= 262144:
|
| 119 |
+
raise ValueError('Invalid complete-input token limit')
|
| 120 |
+
if 'repo_id' in item and item['repo_id'] != profile['repo_id']:
|
| 121 |
+
raise ValueError('Model Hub repository mismatch')
|
| 122 |
+
if 'revision' in item and (not isinstance(item['revision'], str) or len(item['revision']) != 40 or any(c not in '0123456789abcdef' for c in item['revision'])):
|
| 123 |
+
raise ValueError('Pin a full Hub revision')
|
| 124 |
+
if 'content_sha256' in item and (not isinstance(item['content_sha256'], str) or len(item['content_sha256']) != 64 or any(c not in '0123456789abcdef' for c in item['content_sha256'])):
|
| 125 |
+
raise ValueError('Pin a valid artifact content SHA-256')
|
| 126 |
+
for key in ('description', 'parameter_label'):
|
| 127 |
+
if key in item and (not isinstance(item[key], str) or not 1 <= len(item[key]) <= 500):
|
| 128 |
+
raise ValueError('Invalid model display metadata')
|
| 129 |
+
if ('example_ids' in item) != ('default_example_id' in item):
|
| 130 |
+
raise ValueError('Example allowlist and default must be supplied together')
|
| 131 |
+
if 'example_ids' in item:
|
| 132 |
+
ids = item['example_ids']
|
| 133 |
+
if (not isinstance(ids, list) or not ids or len(ids) > 32
|
| 134 |
+
or any(not isinstance(x, str) or not 1 <= len(x) <= 128 for x in ids)
|
| 135 |
+
or len(set(ids)) != len(ids) or item['default_example_id'] not in ids):
|
| 136 |
+
raise ValueError('Invalid example allowlist/default')
|
| 137 |
+
# Internal Studio and queue traffic uses the short ID, but public
|
| 138 |
+
# SystemOne traffic requires the canonical Hub repository ID.
|
| 139 |
+
result[item['id']] = dict(
|
| 140 |
+
item,
|
| 141 |
+
repo_id=profile['repo_id'],
|
| 142 |
+
complete_input_tokens=cap,
|
| 143 |
+
confidence_definition=profile['confidence'],
|
| 144 |
+
)
|
| 145 |
+
return result
|
model_runtime_client.py
ADDED
|
@@ -0,0 +1,410 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Private client for the vLLM Semantic Router built-in model runtime.
|
| 2 |
+
|
| 3 |
+
`vllm-sr-runtime serve` (``vllm-sr serve <model>``) answers one state per
|
| 4 |
+
request on ``/v1/systemone`` and reports its artifact on ``/v1/models`` and in
|
| 5 |
+
each response's ``meta``, instead of the ``X-Decision-Artifact-*`` headers of
|
| 6 |
+
``vllm-sr decision serve``. This client keeps the same guarantees as
|
| 7 |
+
``DirectGateway``: every answer is bound to the pinned repository, revision,
|
| 8 |
+
manifest and model identity on the ``exact`` profile; the runtime's answers are
|
| 9 |
+
projected onto the strict SystemOne envelope without changing a value; and a
|
| 10 |
+
shared-question batch becomes one runtime request per state, reassembled in
|
| 11 |
+
request order.
|
| 12 |
+
"""
|
| 13 |
+
|
| 14 |
+
from __future__ import annotations
|
| 15 |
+
|
| 16 |
+
import asyncio
|
| 17 |
+
import json
|
| 18 |
+
import math
|
| 19 |
+
from collections.abc import Mapping
|
| 20 |
+
from types import MappingProxyType
|
| 21 |
+
|
| 22 |
+
import httpx
|
| 23 |
+
|
| 24 |
+
from direct_contract import (
|
| 25 |
+
CONFIDENCE_DEFINITIONS,
|
| 26 |
+
MARGIN_ORDINAL_V1,
|
| 27 |
+
validate_response_for_request,
|
| 28 |
+
)
|
| 29 |
+
from direct_gateway import (
|
| 30 |
+
MAX_BATCH_REQUEST_BYTES,
|
| 31 |
+
MAX_INFERENCE_RESPONSE_BYTES,
|
| 32 |
+
MAX_SINGLE_REQUEST_BYTES,
|
| 33 |
+
MAX_TIMEOUT_SECONDS,
|
| 34 |
+
MIN_TIMEOUT_SECONDS,
|
| 35 |
+
DirectGatewayError,
|
| 36 |
+
_hex_digest,
|
| 37 |
+
_loads_strict,
|
| 38 |
+
_private_origin,
|
| 39 |
+
_safe_retry_after,
|
| 40 |
+
)
|
| 41 |
+
|
| 42 |
+
SINGLE_PATH = "/v1/systemone"
|
| 43 |
+
HEALTH_PATH = "/health"
|
| 44 |
+
MODELS_PATH = "/v1/models"
|
| 45 |
+
MAX_CONTROL_RESPONSE_BYTES = 256 * 1024
|
| 46 |
+
MAX_CONTROL_TIMEOUT_SECONDS = 2.0
|
| 47 |
+
EXACT = "exact"
|
| 48 |
+
# Golden statuses of a runtime that verified its package and answers
|
| 49 |
+
# deterministically; a "failed" golden check never reaches readiness.
|
| 50 |
+
SERVING_GOLDEN = frozenset({"matched", "unverified"})
|
| 51 |
+
INPUT_ERRORS = frozenset({"invalid_question", "max_length_exceeded"})
|
| 52 |
+
RESPONSE_FIELDS = frozenset({"model", "answers", "usage", "meta"})
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
class ModelRuntimeGateway:
|
| 56 |
+
"""Attested client for one or more built-in runtime instances."""
|
| 57 |
+
|
| 58 |
+
def __init__(
|
| 59 |
+
self,
|
| 60 |
+
endpoints: Mapping[str, str],
|
| 61 |
+
*,
|
| 62 |
+
expected_artifacts: Mapping[str, Mapping[str, str]],
|
| 63 |
+
timeout_seconds: float = 45.0,
|
| 64 |
+
client: httpx.AsyncClient | None = None,
|
| 65 |
+
max_concurrency: int = 4,
|
| 66 |
+
):
|
| 67 |
+
expected = tuple(expected_artifacts)
|
| 68 |
+
if not expected or len(set(expected)) != len(expected):
|
| 69 |
+
raise ValueError("Configure a nonempty set of canonical Decision models")
|
| 70 |
+
if set(endpoints) != set(expected):
|
| 71 |
+
raise ValueError("runtime endpoints must cover exactly the configured models")
|
| 72 |
+
pinned = {}
|
| 73 |
+
confidence = {}
|
| 74 |
+
for model in expected:
|
| 75 |
+
item = expected_artifacts[model]
|
| 76 |
+
if not isinstance(item, Mapping):
|
| 77 |
+
raise ValueError("Pin each Decision model to a released artifact")
|
| 78 |
+
revision = item.get("revision")
|
| 79 |
+
manifest = item.get("manifest_sha256")
|
| 80 |
+
content = item.get("content_sha256")
|
| 81 |
+
if (
|
| 82 |
+
not _hex_digest(revision, 40)
|
| 83 |
+
or not _hex_digest(manifest, 64)
|
| 84 |
+
or (content is not None and not _hex_digest(content, 64))
|
| 85 |
+
):
|
| 86 |
+
raise ValueError("Pin each Decision model revision and manifest digest")
|
| 87 |
+
pinned[model] = MappingProxyType(
|
| 88 |
+
{
|
| 89 |
+
"revision": revision,
|
| 90 |
+
"manifest_sha256": manifest,
|
| 91 |
+
**({"content_sha256": content} if content is not None else {}),
|
| 92 |
+
}
|
| 93 |
+
)
|
| 94 |
+
confidence[model] = item.get("confidence_definition", MARGIN_ORDINAL_V1)
|
| 95 |
+
if confidence[model] not in CONFIDENCE_DEFINITIONS:
|
| 96 |
+
raise ValueError("Unknown Decision confidence definition")
|
| 97 |
+
if (
|
| 98 |
+
isinstance(timeout_seconds, bool)
|
| 99 |
+
or not isinstance(timeout_seconds, (int, float))
|
| 100 |
+
or not math.isfinite(timeout_seconds)
|
| 101 |
+
or not MIN_TIMEOUT_SECONDS <= timeout_seconds <= MAX_TIMEOUT_SECONDS
|
| 102 |
+
):
|
| 103 |
+
raise ValueError("runtime timeout must be between 0.1 and 300 seconds")
|
| 104 |
+
if type(max_concurrency) is not int or not 1 <= max_concurrency <= 64:
|
| 105 |
+
raise ValueError("Invalid runtime request concurrency")
|
| 106 |
+
normalized = {model: _private_origin(endpoints[model]) for model in expected}
|
| 107 |
+
if len(set(normalized.values())) != len(normalized):
|
| 108 |
+
raise ValueError("Each configured Decision model requires its own runtime origin")
|
| 109 |
+
self._endpoints = MappingProxyType(normalized)
|
| 110 |
+
self.expected_artifacts = MappingProxyType(pinned)
|
| 111 |
+
self.confidence_definitions = MappingProxyType(confidence)
|
| 112 |
+
self.models = frozenset(expected)
|
| 113 |
+
self.timeout_seconds = float(timeout_seconds)
|
| 114 |
+
self._max_concurrency = max_concurrency
|
| 115 |
+
self._owns_client = client is None
|
| 116 |
+
self._client = client or httpx.AsyncClient(
|
| 117 |
+
follow_redirects=False,
|
| 118 |
+
timeout=self.timeout_seconds,
|
| 119 |
+
trust_env=False,
|
| 120 |
+
)
|
| 121 |
+
|
| 122 |
+
async def aclose(self) -> None:
|
| 123 |
+
if self._owns_client:
|
| 124 |
+
await self._client.aclose()
|
| 125 |
+
|
| 126 |
+
# -- readiness -----------------------------------------------------------
|
| 127 |
+
|
| 128 |
+
async def ready(self, model: str) -> bool:
|
| 129 |
+
return (await self.probe(model))["loaded"]
|
| 130 |
+
|
| 131 |
+
async def probe(self, model: str) -> dict[str, object]:
|
| 132 |
+
"""Readiness plus the live artifact, with no private topology."""
|
| 133 |
+
|
| 134 |
+
if model not in self.models:
|
| 135 |
+
return {"loaded": False, "artifact": None}
|
| 136 |
+
timeout = min(self.timeout_seconds, MAX_CONTROL_TIMEOUT_SECONDS)
|
| 137 |
+
try:
|
| 138 |
+
health = await self._request_json("GET", model, HEALTH_PATH, timeout=timeout)
|
| 139 |
+
listing = await self._request_json("GET", model, MODELS_PATH, timeout=timeout)
|
| 140 |
+
except DirectGatewayError:
|
| 141 |
+
return {"loaded": False, "artifact": None}
|
| 142 |
+
artifact = self._card_artifact(model, listing)
|
| 143 |
+
loaded = (
|
| 144 |
+
isinstance(health, dict)
|
| 145 |
+
and health.get("status") == "ready"
|
| 146 |
+
and health.get("model") == model
|
| 147 |
+
and artifact is not None
|
| 148 |
+
and self._artifact_matches(model, artifact)
|
| 149 |
+
)
|
| 150 |
+
return {"loaded": bool(loaded), "artifact": artifact}
|
| 151 |
+
|
| 152 |
+
@staticmethod
|
| 153 |
+
def _card_artifact(model: str, listing: object) -> dict[str, str] | None:
|
| 154 |
+
if not isinstance(listing, dict) or not isinstance(listing.get("data"), list):
|
| 155 |
+
return None
|
| 156 |
+
if len(listing["data"]) != 1 or not isinstance(listing["data"][0], dict):
|
| 157 |
+
return None
|
| 158 |
+
card = listing["data"][0]
|
| 159 |
+
golden = card.get("golden")
|
| 160 |
+
if (
|
| 161 |
+
card.get("id") != model
|
| 162 |
+
or card.get("repo") != model
|
| 163 |
+
or card.get("ready") is not True
|
| 164 |
+
or card.get("profile") != EXACT
|
| 165 |
+
or not isinstance(golden, dict)
|
| 166 |
+
or golden.get("status") not in SERVING_GOLDEN
|
| 167 |
+
or not _hex_digest(card.get("revision"), 40)
|
| 168 |
+
or not _hex_digest(card.get("manifest_sha256"), 64)
|
| 169 |
+
or not _hex_digest(card.get("model_sha256"), 64)
|
| 170 |
+
):
|
| 171 |
+
return None
|
| 172 |
+
return {
|
| 173 |
+
"model": card["repo"],
|
| 174 |
+
"revision": card["revision"],
|
| 175 |
+
"manifest_sha256": card["manifest_sha256"],
|
| 176 |
+
"content_sha256": card["model_sha256"],
|
| 177 |
+
}
|
| 178 |
+
|
| 179 |
+
def _artifact_matches(self, model: str, artifact: Mapping[str, str]) -> bool:
|
| 180 |
+
pinned = self.expected_artifacts[model]
|
| 181 |
+
return (
|
| 182 |
+
artifact["model"] == model
|
| 183 |
+
and artifact["revision"] == pinned["revision"]
|
| 184 |
+
and artifact["manifest_sha256"] == pinned["manifest_sha256"]
|
| 185 |
+
and (
|
| 186 |
+
"content_sha256" not in pinned
|
| 187 |
+
or artifact["content_sha256"] == pinned["content_sha256"]
|
| 188 |
+
)
|
| 189 |
+
)
|
| 190 |
+
|
| 191 |
+
# -- inference -----------------------------------------------------------
|
| 192 |
+
|
| 193 |
+
async def evaluate(self, payload: Mapping[str, object], *, batch: bool) -> dict:
|
| 194 |
+
model = payload.get("model") if isinstance(payload, Mapping) else None
|
| 195 |
+
if not isinstance(model, str) or model not in self.models:
|
| 196 |
+
raise DirectGatewayError(422, "The selected canonical model is not configured")
|
| 197 |
+
if batch != ("states" in payload) or ("state" in payload and "states" in payload):
|
| 198 |
+
raise DirectGatewayError(422, "The Decision request does not match its route")
|
| 199 |
+
try:
|
| 200 |
+
encoded = json.dumps(
|
| 201 |
+
payload, ensure_ascii=False, separators=(",", ":"), allow_nan=False
|
| 202 |
+
).encode("utf-8")
|
| 203 |
+
except (TypeError, ValueError, UnicodeError, RecursionError) as exc:
|
| 204 |
+
raise DirectGatewayError(422, "The Decision request is not valid JSON") from exc
|
| 205 |
+
if len(encoded) > (MAX_BATCH_REQUEST_BYTES if batch else MAX_SINGLE_REQUEST_BYTES):
|
| 206 |
+
raise DirectGatewayError(413, "Request body exceeds the endpoint limit")
|
| 207 |
+
try:
|
| 208 |
+
async with asyncio.timeout(self.timeout_seconds):
|
| 209 |
+
if batch:
|
| 210 |
+
response = await self._batch(model, payload)
|
| 211 |
+
else:
|
| 212 |
+
response = await self._single(model, payload["state"], payload["questions"])
|
| 213 |
+
except TimeoutError as exc:
|
| 214 |
+
raise DirectGatewayError(
|
| 215 |
+
504, "The selected Decision runtime did not respond in time"
|
| 216 |
+
) from exc
|
| 217 |
+
try:
|
| 218 |
+
validate_response_for_request(
|
| 219 |
+
payload, response, batch=batch,
|
| 220 |
+
confidence=self.confidence_definitions[model],
|
| 221 |
+
)
|
| 222 |
+
except (TypeError, ValueError, KeyError, OverflowError, RecursionError) as exc:
|
| 223 |
+
raise DirectGatewayError(
|
| 224 |
+
502, "The selected Decision runtime returned an invalid response"
|
| 225 |
+
) from exc
|
| 226 |
+
return response
|
| 227 |
+
|
| 228 |
+
async def _batch(self, model: str, payload: Mapping[str, object]) -> dict:
|
| 229 |
+
states = payload["states"]
|
| 230 |
+
if not isinstance(states, list) or not states:
|
| 231 |
+
raise DirectGatewayError(422, "The Decision request does not match its route")
|
| 232 |
+
gate = asyncio.Semaphore(self._max_concurrency)
|
| 233 |
+
|
| 234 |
+
async def one(item):
|
| 235 |
+
if not isinstance(item, dict) or set(item) != {"id", "state"}:
|
| 236 |
+
raise DirectGatewayError(422, "Each context must contain exactly id and state")
|
| 237 |
+
async with gate:
|
| 238 |
+
return await self._single(model, item["state"], payload["questions"])
|
| 239 |
+
|
| 240 |
+
tasks = [asyncio.create_task(one(item)) for item in states]
|
| 241 |
+
try:
|
| 242 |
+
rows = await asyncio.gather(*tasks)
|
| 243 |
+
except BaseException:
|
| 244 |
+
for task in tasks:
|
| 245 |
+
task.cancel()
|
| 246 |
+
await asyncio.gather(*tasks, return_exceptions=True)
|
| 247 |
+
raise
|
| 248 |
+
results = [
|
| 249 |
+
{"id": item["id"], "answers": row["answers"], "usage": row["usage"]}
|
| 250 |
+
for item, row in zip(states, rows, strict=True)
|
| 251 |
+
]
|
| 252 |
+
return {
|
| 253 |
+
"model": model,
|
| 254 |
+
"results": results,
|
| 255 |
+
"usage": {
|
| 256 |
+
"input_tokens": sum(row["usage"]["input_tokens"] for row in rows),
|
| 257 |
+
"output_tokens": sum(row["usage"]["output_tokens"] for row in rows),
|
| 258 |
+
},
|
| 259 |
+
}
|
| 260 |
+
|
| 261 |
+
async def _single(self, model: str, state: object, questions: object) -> dict:
|
| 262 |
+
body = {
|
| 263 |
+
"model": model,
|
| 264 |
+
"state": state,
|
| 265 |
+
"questions": questions,
|
| 266 |
+
"options": {"profile": EXACT, "return_meta": True},
|
| 267 |
+
}
|
| 268 |
+
response = await self._request_json(
|
| 269 |
+
"POST",
|
| 270 |
+
model,
|
| 271 |
+
SINGLE_PATH,
|
| 272 |
+
content=json.dumps(
|
| 273 |
+
body, ensure_ascii=False, separators=(",", ":"), allow_nan=False
|
| 274 |
+
).encode("utf-8"),
|
| 275 |
+
limit=MAX_INFERENCE_RESPONSE_BYTES,
|
| 276 |
+
)
|
| 277 |
+
if (
|
| 278 |
+
not isinstance(response, dict)
|
| 279 |
+
or set(response) != RESPONSE_FIELDS
|
| 280 |
+
or response["model"] != model
|
| 281 |
+
or not isinstance(response["answers"], dict)
|
| 282 |
+
or not isinstance(response["usage"], dict)
|
| 283 |
+
):
|
| 284 |
+
raise DirectGatewayError(
|
| 285 |
+
502, "The selected Decision runtime returned an invalid response"
|
| 286 |
+
)
|
| 287 |
+
if not self._meta_matches(model, response["meta"]):
|
| 288 |
+
raise DirectGatewayError(
|
| 289 |
+
503, "The selected Decision runtime artifact is unavailable"
|
| 290 |
+
)
|
| 291 |
+
errors = {
|
| 292 |
+
answer.get("error")
|
| 293 |
+
for answer in response["answers"].values()
|
| 294 |
+
if isinstance(answer, dict) and "error" in answer
|
| 295 |
+
}
|
| 296 |
+
if errors & INPUT_ERRORS:
|
| 297 |
+
raise DirectGatewayError(413, "Decision input exceeds the selected model limit")
|
| 298 |
+
if errors:
|
| 299 |
+
raise DirectGatewayError(503, "The selected Decision runtime is unavailable")
|
| 300 |
+
return {
|
| 301 |
+
"model": response["model"],
|
| 302 |
+
"answers": response["answers"],
|
| 303 |
+
"usage": response["usage"],
|
| 304 |
+
}
|
| 305 |
+
|
| 306 |
+
def _meta_matches(self, model: str, meta: object) -> bool:
|
| 307 |
+
pinned = self.expected_artifacts[model]
|
| 308 |
+
return (
|
| 309 |
+
isinstance(meta, dict)
|
| 310 |
+
and meta.get("revision") == pinned["revision"]
|
| 311 |
+
and meta.get("profile") == EXACT
|
| 312 |
+
and meta.get("numerics") == EXACT
|
| 313 |
+
and _hex_digest(meta.get("model_sha256"), 64)
|
| 314 |
+
and (
|
| 315 |
+
"content_sha256" not in pinned
|
| 316 |
+
or meta["model_sha256"] == pinned["content_sha256"]
|
| 317 |
+
)
|
| 318 |
+
)
|
| 319 |
+
|
| 320 |
+
async def _request_json(
|
| 321 |
+
self,
|
| 322 |
+
method: str,
|
| 323 |
+
model: str,
|
| 324 |
+
path: str,
|
| 325 |
+
*,
|
| 326 |
+
content: bytes | None = None,
|
| 327 |
+
limit: int = MAX_CONTROL_RESPONSE_BYTES,
|
| 328 |
+
timeout: float | None = None,
|
| 329 |
+
):
|
| 330 |
+
endpoint = self._endpoints[model]
|
| 331 |
+
request_timeout = timeout or self.timeout_seconds
|
| 332 |
+
headers = {"Accept": "application/json", "Accept-Encoding": "identity"}
|
| 333 |
+
if content is not None:
|
| 334 |
+
headers["Content-Type"] = "application/json"
|
| 335 |
+
try:
|
| 336 |
+
async with asyncio.timeout(request_timeout):
|
| 337 |
+
async with self._client.stream(
|
| 338 |
+
method,
|
| 339 |
+
endpoint + path,
|
| 340 |
+
content=content,
|
| 341 |
+
headers=headers,
|
| 342 |
+
follow_redirects=False,
|
| 343 |
+
timeout=request_timeout,
|
| 344 |
+
) as response:
|
| 345 |
+
if 300 <= response.status_code < 400:
|
| 346 |
+
raise DirectGatewayError(
|
| 347 |
+
502, "The selected Decision runtime returned an unsupported redirect"
|
| 348 |
+
)
|
| 349 |
+
if response.status_code >= 400 and not (
|
| 350 |
+
path == HEALTH_PATH and response.status_code == 503
|
| 351 |
+
):
|
| 352 |
+
self._raise_status(response)
|
| 353 |
+
return await self._read_json(response, limit)
|
| 354 |
+
except asyncio.CancelledError:
|
| 355 |
+
raise
|
| 356 |
+
except DirectGatewayError:
|
| 357 |
+
raise
|
| 358 |
+
except (TimeoutError, httpx.TimeoutException) as exc:
|
| 359 |
+
raise DirectGatewayError(
|
| 360 |
+
504, "The selected Decision runtime did not respond in time"
|
| 361 |
+
) from exc
|
| 362 |
+
except httpx.HTTPError as exc:
|
| 363 |
+
raise DirectGatewayError(503, "The selected Decision runtime is unavailable") from exc
|
| 364 |
+
|
| 365 |
+
@staticmethod
|
| 366 |
+
def _raise_status(response: httpx.Response) -> None:
|
| 367 |
+
status = response.status_code
|
| 368 |
+
if status in {400, 413}:
|
| 369 |
+
raise DirectGatewayError(413, "Decision input exceeds the selected model limit")
|
| 370 |
+
if status in {429, 529}:
|
| 371 |
+
raise DirectGatewayError(
|
| 372 |
+
status,
|
| 373 |
+
"The selected Decision runtime is temporarily overloaded",
|
| 374 |
+
retry_after=_safe_retry_after(response.headers.get("retry-after")),
|
| 375 |
+
)
|
| 376 |
+
if status == 503:
|
| 377 |
+
raise DirectGatewayError(
|
| 378 |
+
503,
|
| 379 |
+
"The selected Decision runtime is unavailable",
|
| 380 |
+
retry_after=_safe_retry_after(response.headers.get("retry-after")),
|
| 381 |
+
)
|
| 382 |
+
if status == 504:
|
| 383 |
+
raise DirectGatewayError(504, "The selected Decision runtime did not respond in time")
|
| 384 |
+
raise DirectGatewayError(502, "The selected Decision runtime rejected a validated request")
|
| 385 |
+
|
| 386 |
+
@staticmethod
|
| 387 |
+
async def _read_json(response: httpx.Response, limit: int):
|
| 388 |
+
content_type = response.headers.get("content-type", "").split(";", 1)[0].strip().lower()
|
| 389 |
+
if content_type != "application/json":
|
| 390 |
+
raise DirectGatewayError(
|
| 391 |
+
502, "The selected Decision runtime returned an invalid content type"
|
| 392 |
+
)
|
| 393 |
+
encoding = response.headers.get("content-encoding", "identity").strip().lower()
|
| 394 |
+
if encoding not in {"", "identity"}:
|
| 395 |
+
raise DirectGatewayError(
|
| 396 |
+
502, "The selected Decision runtime returned an unsupported encoding"
|
| 397 |
+
)
|
| 398 |
+
body = bytearray()
|
| 399 |
+
async for chunk in response.aiter_bytes(chunk_size=64 * 1024):
|
| 400 |
+
if len(chunk) > limit - len(body):
|
| 401 |
+
raise DirectGatewayError(
|
| 402 |
+
502, "The selected Decision runtime returned an oversized response"
|
| 403 |
+
)
|
| 404 |
+
body.extend(chunk)
|
| 405 |
+
try:
|
| 406 |
+
return _loads_strict(bytes(body))
|
| 407 |
+
except (TypeError, ValueError, UnicodeError, RecursionError) as exc:
|
| 408 |
+
raise DirectGatewayError(
|
| 409 |
+
502, "The selected Decision runtime returned invalid JSON"
|
| 410 |
+
) from exc
|
pull_worker.py
ADDED
|
@@ -0,0 +1,173 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Resident queue worker; only outbound HTTPS. Never retries inference for a job."""
|
| 2 |
+
import json
|
| 3 |
+
import logging
|
| 4 |
+
import os
|
| 5 |
+
import secrets
|
| 6 |
+
import signal
|
| 7 |
+
import threading
|
| 8 |
+
import time
|
| 9 |
+
import urllib.error
|
| 10 |
+
import urllib.parse
|
| 11 |
+
import urllib.request
|
| 12 |
+
|
| 13 |
+
from contract import to_records
|
| 14 |
+
from engine import Engine
|
| 15 |
+
|
| 16 |
+
LOG = logging.getLogger("decision.worker")
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
class NoRedirect(urllib.request.HTTPRedirectHandler):
|
| 20 |
+
def redirect_request(self, req, fp, code, msg, headers, newurl):
|
| 21 |
+
# Do not forward the worker bearer or user input to a redirected origin.
|
| 22 |
+
return None
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
class GatewayError(Exception):
|
| 26 |
+
def __init__(self, code):
|
| 27 |
+
self.code = code
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
class Gateway:
|
| 31 |
+
def __init__(self, url, token, *, allow_loopback=False):
|
| 32 |
+
parsed = urllib.parse.urlsplit(url)
|
| 33 |
+
loopback = allow_loopback and parsed.hostname in {"localhost", "127.0.0.1", "::1"}
|
| 34 |
+
if (parsed.scheme != "https" and not (loopback and parsed.scheme == "http")
|
| 35 |
+
or not parsed.hostname or parsed.username or parsed.password or parsed.query or parsed.fragment
|
| 36 |
+
or parsed.path not in {"", "/"}):
|
| 37 |
+
raise ValueError("Use a fixed HTTPS gateway origin; no credentials, path, query, or fragment")
|
| 38 |
+
if not isinstance(token, str) or len(token) < 32:
|
| 39 |
+
raise ValueError("A worker token of at least 32 characters is required")
|
| 40 |
+
self.url, self.token = url.rstrip("/"), token
|
| 41 |
+
handlers = [NoRedirect()]
|
| 42 |
+
if loopback:
|
| 43 |
+
# A developer's system proxy must not receive loopback test credentials.
|
| 44 |
+
handlers.append(urllib.request.ProxyHandler({}))
|
| 45 |
+
self.opener = urllib.request.build_opener(*handlers)
|
| 46 |
+
|
| 47 |
+
def post(self, action, payload):
|
| 48 |
+
body = json.dumps(payload, ensure_ascii=False, separators=(",", ":"), allow_nan=False).encode()
|
| 49 |
+
request = urllib.request.Request(self.url + "/internal/worker/" + action, data=body, method="POST",
|
| 50 |
+
headers={"Content-Type": "application/json", "Authorization": "Bearer " + self.token})
|
| 51 |
+
try:
|
| 52 |
+
with self.opener.open(request, timeout=30) as response:
|
| 53 |
+
limit = 2 * 1024 * 1024 + 64 * 1024 if action == "claim" else 64 * 1024
|
| 54 |
+
data = response.read(limit + 1)
|
| 55 |
+
if len(data) > limit:
|
| 56 |
+
raise ValueError("Gateway response is too large")
|
| 57 |
+
return json.loads(data)
|
| 58 |
+
except urllib.error.HTTPError as exc:
|
| 59 |
+
# Do not log HTTP body, URL, or request headers.
|
| 60 |
+
raise GatewayError(exc.code) from None
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
class Worker:
|
| 64 |
+
heartbeat_join_timeout = 2
|
| 65 |
+
|
| 66 |
+
def __init__(self, gateway, runtime=None):
|
| 67 |
+
self.gateway, self.runtime = gateway, runtime or Engine()
|
| 68 |
+
self.worker_id, self.stop = secrets.token_hex(16), threading.Event()
|
| 69 |
+
|
| 70 |
+
def post(self, action, payload):
|
| 71 |
+
payload = dict(payload, model=self.runtime.model, manifest_sha256=self.runtime.manifest)
|
| 72 |
+
return self.gateway.post(action, payload)
|
| 73 |
+
|
| 74 |
+
def heartbeat(self):
|
| 75 |
+
phase = "running" if self.runtime.phase == "running" else "ready"
|
| 76 |
+
return self.post("heartbeat", {"worker_id": self.worker_id,
|
| 77 |
+
"manifest_sha256": self.runtime.manifest, "phase": phase,
|
| 78 |
+
"capabilities": ["context_batch_v1"]})
|
| 79 |
+
|
| 80 |
+
def heartbeat_loop(self):
|
| 81 |
+
while not self.stop.wait(10):
|
| 82 |
+
try:
|
| 83 |
+
self.heartbeat()
|
| 84 |
+
except GatewayError as exc:
|
| 85 |
+
LOG.warning("Heartbeat HTTP status: %d", exc.code)
|
| 86 |
+
if exc.code in {401, 403, 409}:
|
| 87 |
+
self.stop.set()
|
| 88 |
+
except Exception as exc:
|
| 89 |
+
LOG.warning("Heartbeat unavailable: %s", type(exc).__name__)
|
| 90 |
+
|
| 91 |
+
def execute(self, job):
|
| 92 |
+
"""One evaluate, followed only by bounded identical result retransmissions."""
|
| 93 |
+
started = time.monotonic()
|
| 94 |
+
if not isinstance(job, dict) or set(job) != {"id", "lease_token", "body", "seconds_remaining"}:
|
| 95 |
+
raise ValueError("Invalid gateway job")
|
| 96 |
+
seconds = job["seconds_remaining"]
|
| 97 |
+
if type(seconds) not in (int, float) or not 0 < seconds <= 180:
|
| 98 |
+
raise ValueError("Invalid lease duration")
|
| 99 |
+
reply = {"worker_id": self.worker_id, "id": job["id"], "lease_token": job["lease_token"]}
|
| 100 |
+
try:
|
| 101 |
+
records = to_records(job["body"], model=self.runtime.model)
|
| 102 |
+
reply["result"] = self.runtime.evaluate(job["body"], records)
|
| 103 |
+
except ValueError:
|
| 104 |
+
reply["error_code"] = "input_rejected"
|
| 105 |
+
except Exception as exc:
|
| 106 |
+
LOG.warning("Inference failed: %s", type(exc).__name__)
|
| 107 |
+
reply["error_code"] = "inference_failed"
|
| 108 |
+
# The same reply can be acknowledged twice; the model is never called twice.
|
| 109 |
+
for attempt in range(4):
|
| 110 |
+
if time.monotonic() - started >= seconds or self.stop.is_set():
|
| 111 |
+
LOG.warning("Result discarded after lease deadline or worker stop")
|
| 112 |
+
return
|
| 113 |
+
try:
|
| 114 |
+
self.post("result", reply)
|
| 115 |
+
LOG.info("Job finished: %s", "prediction" if "result" in reply else reply["error_code"])
|
| 116 |
+
return
|
| 117 |
+
except GatewayError as exc:
|
| 118 |
+
if exc.code in {409, 410}:
|
| 119 |
+
LOG.warning("Result no longer accepted: HTTP %d", exc.code)
|
| 120 |
+
return
|
| 121 |
+
if exc.code in {401, 403, 422}:
|
| 122 |
+
self.stop.set()
|
| 123 |
+
LOG.error("Worker result rejected: HTTP %d", exc.code)
|
| 124 |
+
return
|
| 125 |
+
LOG.warning("Result delivery HTTP status: %d", exc.code)
|
| 126 |
+
except Exception as exc:
|
| 127 |
+
LOG.warning("Result delivery unavailable: %s", type(exc).__name__)
|
| 128 |
+
self.stop.wait(2 ** attempt)
|
| 129 |
+
LOG.warning("Result delivery unconfirmed; no inference retry")
|
| 130 |
+
|
| 131 |
+
def run(self):
|
| 132 |
+
# A runtime must validate its artifact before advertising readiness.
|
| 133 |
+
self.runtime._load()
|
| 134 |
+
self.heartbeat()
|
| 135 |
+
heartbeat = threading.Thread(target=self.heartbeat_loop, daemon=True)
|
| 136 |
+
heartbeat.start()
|
| 137 |
+
LOG.info("Resident worker ready")
|
| 138 |
+
try:
|
| 139 |
+
while not self.stop.is_set():
|
| 140 |
+
try:
|
| 141 |
+
response = self.post("claim", {"worker_id": self.worker_id, "wait_seconds": 20})
|
| 142 |
+
if response.get("job") is not None:
|
| 143 |
+
self.execute(response["job"])
|
| 144 |
+
except GatewayError as exc:
|
| 145 |
+
if exc.code in {401, 403, 422}:
|
| 146 |
+
raise
|
| 147 |
+
# An uncertain claim may remain leased. Let it expire; do not rerun.
|
| 148 |
+
LOG.warning("Claim unavailable: HTTP %d", exc.code)
|
| 149 |
+
self.stop.wait(3)
|
| 150 |
+
except (OSError, ValueError) as exc:
|
| 151 |
+
LOG.warning("Claim unavailable: %s", type(exc).__name__)
|
| 152 |
+
self.stop.wait(3)
|
| 153 |
+
finally:
|
| 154 |
+
self.stop.set()
|
| 155 |
+
heartbeat.join(timeout=self.heartbeat_join_timeout)
|
| 156 |
+
|
| 157 |
+
|
| 158 |
+
def main():
|
| 159 |
+
logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s")
|
| 160 |
+
gateway = Gateway(os.getenv("DECISION_STUDIO_URL", ""), os.getenv("DECISION_WORKER_TOKEN", ""),
|
| 161 |
+
allow_loopback=os.getenv("DECISION_ALLOW_HTTP_LOOPBACK") == "1")
|
| 162 |
+
worker = Worker(gateway)
|
| 163 |
+
for sig in (signal.SIGINT, signal.SIGTERM):
|
| 164 |
+
signal.signal(sig, lambda *_: worker.stop.set())
|
| 165 |
+
worker.run()
|
| 166 |
+
|
| 167 |
+
|
| 168 |
+
if __name__ == "__main__":
|
| 169 |
+
try:
|
| 170 |
+
main()
|
| 171 |
+
except Exception as exc:
|
| 172 |
+
LOG.error("Worker stopped: %s", type(exc).__name__)
|
| 173 |
+
raise SystemExit(1) from None
|
relay.py
ADDED
|
@@ -0,0 +1,318 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Bounded, ephemeral queue. No model weights, disk persistence, or request logging."""
|
| 2 |
+
import asyncio
|
| 3 |
+
import hashlib
|
| 4 |
+
import hmac
|
| 5 |
+
import json
|
| 6 |
+
import math
|
| 7 |
+
import secrets
|
| 8 |
+
import time
|
| 9 |
+
|
| 10 |
+
from contract import MODEL, response_answers, timing_shape, to_records, to_response
|
| 11 |
+
from direct_contract import validate_response_for_request
|
| 12 |
+
from model_registry import PROFILES
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
class RelayError(Exception):
|
| 16 |
+
def __init__(self, code, message):
|
| 17 |
+
self.code, self.message = code, message
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
def validated_result(body, result, *, model=MODEL, complete_input_tokens=1024):
|
| 21 |
+
"""Recheck the mapped response against its original request without inference."""
|
| 22 |
+
answer_field = "results" if "states" in body else "answers"
|
| 23 |
+
if not isinstance(result, dict) or set(result) != {"model", answer_field, "usage", "profile", "timing", "source"}:
|
| 24 |
+
raise ValueError("Invalid result envelope")
|
| 25 |
+
if result["source"] != "live_native" or result["model"] != model:
|
| 26 |
+
raise ValueError("A live native result is required")
|
| 27 |
+
rows = to_records(body, model=model)
|
| 28 |
+
answers = response_answers(body, result)
|
| 29 |
+
predictions = []
|
| 30 |
+
for row, a in zip(rows, answers):
|
| 31 |
+
q = row["question"]
|
| 32 |
+
kind = q["type"].lower()
|
| 33 |
+
ids = ["no", "yes"] if kind == "noul" else [c["id"] for c in q["options" if kind == "choice" else "levels"]]
|
| 34 |
+
if type(a["input_tokens"]) is not int:
|
| 35 |
+
raise ValueError("Invalid complete input length")
|
| 36 |
+
if kind == "noul":
|
| 37 |
+
p = [1 - a["noul"], a["noul"]]
|
| 38 |
+
else:
|
| 39 |
+
if list(a["probabilities"]) != ids:
|
| 40 |
+
raise ValueError("Candidate order mismatch")
|
| 41 |
+
p = list(a["probabilities"].values())
|
| 42 |
+
predictions.append({"id": row["id"], "question_id": q["id"], "type": kind,
|
| 43 |
+
"candidate_ids": ids, "probabilities": p, "input_tokens": a["input_tokens"],
|
| 44 |
+
"state_tokens_original": 0, "state_tokens_kept": 0})
|
| 45 |
+
canonical = to_response(body, rows, predictions, model=model, complete_input_tokens=complete_input_tokens,
|
| 46 |
+
precision="fp32" if complete_input_tokens == 1024 else "bf16_backbone_fp32_head")
|
| 47 |
+
if any(result[key] != value for key, value in canonical.items()):
|
| 48 |
+
raise ValueError("Response math or profile mismatch")
|
| 49 |
+
timing = result["timing"]
|
| 50 |
+
shape = timing_shape(body, rows)
|
| 51 |
+
if set(timing) != {"model_load_ms", "inference_ms", "server_ms", "cold_start"} | set(shape):
|
| 52 |
+
raise ValueError("Invalid timing fields")
|
| 53 |
+
if (type(timing["cold_start"]) is not bool
|
| 54 |
+
or any(type(timing[k]) is not int or timing[k] != v for k, v in shape.items())):
|
| 55 |
+
raise ValueError("Invalid batch metadata")
|
| 56 |
+
for name in ("model_load_ms", "inference_ms", "server_ms"):
|
| 57 |
+
if type(timing[name]) not in (int, float) or not math.isfinite(timing[name]) or timing[name] < 0:
|
| 58 |
+
raise ValueError("Invalid measured timing")
|
| 59 |
+
return result
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
class Relay:
|
| 63 |
+
"""Single-process/single-worker queue; a failed lease is never requeued."""
|
| 64 |
+
ACTIVE = {"queued", "running"}
|
| 65 |
+
|
| 66 |
+
def __init__(self, token, manifest, *, model=MODEL, clock=time.monotonic, capacity=8,
|
| 67 |
+
retention=64, ttl=180, lease=120, result_ttl=120, heartbeat_ttl=35, complete_input_tokens=1024):
|
| 68 |
+
if not isinstance(token, str) or len(token) < 32:
|
| 69 |
+
raise ValueError("DECISION_WORKER_TOKEN must contain at least 32 characters")
|
| 70 |
+
if len(manifest) != 64 or any(c not in "0123456789abcdef" for c in manifest):
|
| 71 |
+
raise ValueError("Expected native manifest must be an exact SHA-256")
|
| 72 |
+
if type(complete_input_tokens) is not int or not 1 <= complete_input_tokens <= 262144:
|
| 73 |
+
raise ValueError("Unsupported complete-input profile")
|
| 74 |
+
self.complete_input_tokens = complete_input_tokens
|
| 75 |
+
self.model = model
|
| 76 |
+
if model not in PROFILES:
|
| 77 |
+
raise ValueError("Use a configured Decision model wire ID")
|
| 78 |
+
self.canonical_model = PROFILES[model]["repo_id"]
|
| 79 |
+
self.runtime_model = PROFILES[model].get("runtime_model", self.canonical_model)
|
| 80 |
+
self.confidence = PROFILES[model]["confidence"]
|
| 81 |
+
self.token, self.manifest, self.clock = token, manifest, clock
|
| 82 |
+
self.capacity, self.retention = capacity, retention
|
| 83 |
+
self.ttl, self.lease, self.result_ttl, self.heartbeat_ttl = ttl, lease, result_ttl, heartbeat_ttl
|
| 84 |
+
self.jobs, self.worker = {}, None
|
| 85 |
+
self.changed = asyncio.Condition()
|
| 86 |
+
|
| 87 |
+
def authenticate(self, authorization):
|
| 88 |
+
value = authorization[7:] if authorization.startswith("Bearer ") else ""
|
| 89 |
+
if not hmac.compare_digest(value.encode(), self.token.encode()):
|
| 90 |
+
raise RelayError(401, "Invalid worker authorization")
|
| 91 |
+
|
| 92 |
+
def _finish(self, job, status, *, error=None, result=None):
|
| 93 |
+
job.update(status=status, body=None, error=error, result=result, retain_until=self.clock() + self.result_ttl)
|
| 94 |
+
self._trim_completed()
|
| 95 |
+
self.changed.notify_all()
|
| 96 |
+
|
| 97 |
+
def _trim_completed(self):
|
| 98 |
+
completed = [(key, job) for key, job in self.jobs.items()
|
| 99 |
+
if job["status"] not in self.ACTIVE and not job.get("keep_until_read")]
|
| 100 |
+
for key, _ in sorted(completed, key=lambda item: item[1]["retain_until"])[:max(0, len(completed) - self.retention)]:
|
| 101 |
+
del self.jobs[key]
|
| 102 |
+
|
| 103 |
+
def _clean(self):
|
| 104 |
+
now = self.clock()
|
| 105 |
+
for key, job in list(self.jobs.items()):
|
| 106 |
+
if job["status"] in self.ACTIVE and (now >= job["deadline"] or now >= job.get("lease_until", float("inf"))):
|
| 107 |
+
self._finish(job, "expired", error="The request expired. No prediction was substituted.")
|
| 108 |
+
elif job["status"] not in self.ACTIVE and now >= job["retain_until"]:
|
| 109 |
+
del self.jobs[key]
|
| 110 |
+
|
| 111 |
+
def _ready(self):
|
| 112 |
+
return self.worker is not None and self.clock() - self.worker["seen"] < self.heartbeat_ttl
|
| 113 |
+
|
| 114 |
+
def _public(self, job):
|
| 115 |
+
out = {"id": job["id"], "status": job["status"], "model": self.model, "manifest_sha256": self.manifest}
|
| 116 |
+
if job.get("result") is not None:
|
| 117 |
+
result = job["result"]
|
| 118 |
+
out["result"] = result["response"] if result.get("kind") == "http_runtime_v1" else result
|
| 119 |
+
if job.get("error"):
|
| 120 |
+
out["detail"] = job["error"]
|
| 121 |
+
return out
|
| 122 |
+
|
| 123 |
+
async def status(self):
|
| 124 |
+
async with self.changed:
|
| 125 |
+
self._clean()
|
| 126 |
+
ready = self._ready()
|
| 127 |
+
return {"backend": "pull_queue", "model": self.model, "manifest_sha256": self.manifest,
|
| 128 |
+
"complete_input_tokens": self.complete_input_tokens, "configured": ready, "loaded": ready,
|
| 129 |
+
"context_batch": ready and "context_batch_v1" in self.worker.get("capabilities", []),
|
| 130 |
+
"live_inference": ready, "phase": "ready" if ready else "offline",
|
| 131 |
+
"worker_phase": self.worker["phase"] if ready else "offline",
|
| 132 |
+
"queued": sum(j["status"] == "queued" for j in self.jobs.values()),
|
| 133 |
+
"running": sum(j["status"] == "running" for j in self.jobs.values()),
|
| 134 |
+
"readiness": "authenticated worker heartbeat; not a quality claim"}
|
| 135 |
+
|
| 136 |
+
async def heartbeat(self, worker_id, manifest, phase, capabilities=None):
|
| 137 |
+
capabilities = [] if capabilities is None else capabilities
|
| 138 |
+
if (not isinstance(capabilities, list) or any(not isinstance(c, str) or c != "context_batch_v1" for c in capabilities)
|
| 139 |
+
or len(capabilities) != len(set(capabilities))):
|
| 140 |
+
raise RelayError(422, "Invalid worker capabilities")
|
| 141 |
+
if not isinstance(worker_id, str) or len(worker_id) != 32 or any(c not in "0123456789abcdef" for c in worker_id):
|
| 142 |
+
raise RelayError(422, "Invalid worker ID")
|
| 143 |
+
if manifest != self.manifest or phase not in ("ready", "running"):
|
| 144 |
+
raise RelayError(409, "Worker native identity or phase does not match this gateway")
|
| 145 |
+
async with self.changed:
|
| 146 |
+
self._clean()
|
| 147 |
+
if self.worker and self.worker["id"] != worker_id:
|
| 148 |
+
if self._ready() or any(j["status"] == "running" for j in self.jobs.values()):
|
| 149 |
+
raise RelayError(409, "An existing worker still owns this queue")
|
| 150 |
+
self.worker = {"id": worker_id, "seen": self.clock(), "phase": phase, "capabilities": list(capabilities)}
|
| 151 |
+
self.changed.notify_all()
|
| 152 |
+
return {"accepted": True, "manifest_sha256": self.manifest}
|
| 153 |
+
|
| 154 |
+
async def submit(self, body, *, keep_until_read=False):
|
| 155 |
+
to_records(body, model=self.model)
|
| 156 |
+
async with self.changed:
|
| 157 |
+
self._clean()
|
| 158 |
+
if not self._ready():
|
| 159 |
+
raise RelayError(503, "The AMD worker is offline. Nothing was queued.")
|
| 160 |
+
if "states" in body and "context_batch_v1" not in self.worker.get("capabilities", []):
|
| 161 |
+
raise RelayError(503, "A context-batch worker is not connected for this model yet.")
|
| 162 |
+
if sum(j["status"] in self.ACTIVE for j in self.jobs.values()) >= self.capacity:
|
| 163 |
+
raise RelayError(429, "The queue is full. Try again later.")
|
| 164 |
+
key = secrets.token_urlsafe(24)
|
| 165 |
+
job = {"id": key, "status": "queued", "body": body, "deadline": self.clock() + self.ttl,
|
| 166 |
+
"keep_until_read": keep_until_read}
|
| 167 |
+
self.jobs[key] = job
|
| 168 |
+
self.changed.notify_all()
|
| 169 |
+
return self._public(job)
|
| 170 |
+
|
| 171 |
+
async def read(self, key):
|
| 172 |
+
async with self.changed:
|
| 173 |
+
self._clean()
|
| 174 |
+
if key not in self.jobs:
|
| 175 |
+
raise RelayError(404, "Unknown or forgotten request")
|
| 176 |
+
return self._public(self.jobs[key])
|
| 177 |
+
|
| 178 |
+
async def wait(self, key, timeout):
|
| 179 |
+
"""Wait for a job transition without polling or missing an early wakeup.
|
| 180 |
+
|
| 181 |
+
The caller may use a short timeout to perform transport-level checks,
|
| 182 |
+
such as detecting a disconnected synchronous client. Completion,
|
| 183 |
+
cancellation, and expiry still wake the waiter immediately.
|
| 184 |
+
"""
|
| 185 |
+
if (type(timeout) not in (int, float) or not math.isfinite(timeout)
|
| 186 |
+
or not 0 <= timeout <= 45):
|
| 187 |
+
raise ValueError("Wait timeout must be between 0 and 45 seconds")
|
| 188 |
+
stop = time.monotonic() + timeout
|
| 189 |
+
async with self.changed:
|
| 190 |
+
while True:
|
| 191 |
+
self._clean()
|
| 192 |
+
job = self.jobs.get(key)
|
| 193 |
+
if job is None:
|
| 194 |
+
raise RelayError(404, "Unknown or forgotten request")
|
| 195 |
+
current = self._public(job)
|
| 196 |
+
if current["status"] not in self.ACTIVE:
|
| 197 |
+
return current
|
| 198 |
+
caller_remaining = stop - time.monotonic()
|
| 199 |
+
if caller_remaining <= 0:
|
| 200 |
+
return current
|
| 201 |
+
job_remaining = min(
|
| 202 |
+
job["deadline"], job.get("lease_until", float("inf"))
|
| 203 |
+
) - self.clock()
|
| 204 |
+
if job_remaining <= 0:
|
| 205 |
+
continue
|
| 206 |
+
try:
|
| 207 |
+
await asyncio.wait_for(
|
| 208 |
+
self.changed.wait(), min(caller_remaining, job_remaining)
|
| 209 |
+
)
|
| 210 |
+
except asyncio.TimeoutError:
|
| 211 |
+
continue
|
| 212 |
+
|
| 213 |
+
async def cancel(self, key):
|
| 214 |
+
async with self.changed:
|
| 215 |
+
self._clean()
|
| 216 |
+
if key not in self.jobs:
|
| 217 |
+
raise RelayError(404, "Unknown or forgotten request")
|
| 218 |
+
job = self.jobs[key]
|
| 219 |
+
if job["status"] in self.ACTIVE:
|
| 220 |
+
self._finish(job, "cancelled", error="Cancelled; an already running forward may still finish and will be discarded.")
|
| 221 |
+
return self._public(job)
|
| 222 |
+
|
| 223 |
+
async def release(self, key):
|
| 224 |
+
"""Release a synchronous result after it has been copied into the response.
|
| 225 |
+
|
| 226 |
+
Keep only a small completion receipt for idempotent worker acknowledgments.
|
| 227 |
+
Async Studio results retain their normal bounded polling lifetime.
|
| 228 |
+
"""
|
| 229 |
+
async with self.changed:
|
| 230 |
+
job = self.jobs.get(key)
|
| 231 |
+
if job is None:
|
| 232 |
+
return
|
| 233 |
+
if job["status"] in self.ACTIVE:
|
| 234 |
+
self._finish(job, "cancelled", error="Cancelled; an in-flight result will be discarded.")
|
| 235 |
+
job["keep_until_read"] = False
|
| 236 |
+
job["body"] = job["result"] = None
|
| 237 |
+
job["retain_until"] = self.clock() + self.result_ttl
|
| 238 |
+
self.jobs.pop(key)
|
| 239 |
+
self.jobs[key] = job
|
| 240 |
+
self._trim_completed()
|
| 241 |
+
|
| 242 |
+
async def claim(self, worker_id, wait_seconds=20):
|
| 243 |
+
if type(wait_seconds) not in (int, float) or not math.isfinite(wait_seconds) or not 0 <= wait_seconds <= 20:
|
| 244 |
+
raise RelayError(422, "Long-poll wait must be between 0 and 20 seconds")
|
| 245 |
+
stop = time.monotonic() + wait_seconds
|
| 246 |
+
async with self.changed:
|
| 247 |
+
while True:
|
| 248 |
+
self._clean()
|
| 249 |
+
if not self._ready() or self.worker["id"] != worker_id:
|
| 250 |
+
raise RelayError(409, "Send a matching, loaded worker heartbeat before claiming")
|
| 251 |
+
# A cancelled in-flight job is not claimed again; the worker serializes forwards.
|
| 252 |
+
if any(j["status"] == "running" for j in self.jobs.values()):
|
| 253 |
+
raise RelayError(409, "A job is already in flight")
|
| 254 |
+
for job in self.jobs.values():
|
| 255 |
+
if job["status"] == "queued":
|
| 256 |
+
if ("states" in job["body"]
|
| 257 |
+
and "context_batch_v1" not in self.worker.get("capabilities", [])):
|
| 258 |
+
self._finish(job, "failed", error="A context-batch worker is no longer connected; no prediction was substituted.")
|
| 259 |
+
continue
|
| 260 |
+
job.update(status="running", worker_id=worker_id, lease_token=secrets.token_urlsafe(24),
|
| 261 |
+
lease_until=min(job["deadline"], self.clock() + self.lease))
|
| 262 |
+
self.changed.notify_all()
|
| 263 |
+
return {"job": {"id": job["id"], "lease_token": job["lease_token"], "body": job["body"],
|
| 264 |
+
"seconds_remaining": job["lease_until"] - self.clock()}}
|
| 265 |
+
remaining = stop - time.monotonic()
|
| 266 |
+
if remaining <= 0:
|
| 267 |
+
return {"job": None}
|
| 268 |
+
try:
|
| 269 |
+
await asyncio.wait_for(self.changed.wait(), remaining)
|
| 270 |
+
except asyncio.TimeoutError:
|
| 271 |
+
return {"job": None}
|
| 272 |
+
|
| 273 |
+
async def complete(self, worker_id, key, lease_token, result=None, error_code=None):
|
| 274 |
+
if not all(isinstance(value, str) for value in (worker_id, key, lease_token)):
|
| 275 |
+
raise RelayError(422, "Worker and lease identities must be strings")
|
| 276 |
+
if (result is None) == (error_code is None):
|
| 277 |
+
raise RelayError(422, "Send either a result or one error code")
|
| 278 |
+
errors = {"input_rejected": "The complete input exceeds this model's limits or was rejected; nothing was truncated.",
|
| 279 |
+
"inference_failed": "The worker could not finish. No prediction was substituted."}
|
| 280 |
+
if error_code is not None and (not isinstance(error_code, str) or error_code not in errors):
|
| 281 |
+
raise RelayError(422, "Unknown worker error code")
|
| 282 |
+
digest = hashlib.sha256(json.dumps({"result": result, "error_code": error_code}, ensure_ascii=False,
|
| 283 |
+
sort_keys=True, separators=(",", ":"), allow_nan=False).encode()).hexdigest()
|
| 284 |
+
async with self.changed:
|
| 285 |
+
self._clean()
|
| 286 |
+
job = self.jobs.get(key)
|
| 287 |
+
if not job:
|
| 288 |
+
raise RelayError(410, "Request is no longer retained")
|
| 289 |
+
if (worker_id != job.get("worker_id")
|
| 290 |
+
or not hmac.compare_digest(lease_token.encode(), job.get("lease_token", "").encode())):
|
| 291 |
+
raise RelayError(409, "Job lease does not match")
|
| 292 |
+
if job["status"] in {"succeeded", "failed"}:
|
| 293 |
+
if digest != job["completion_sha256"]:
|
| 294 |
+
raise RelayError(409, "A different completion already exists")
|
| 295 |
+
return {"accepted": True, "duplicate": True}
|
| 296 |
+
if job["status"] != "running":
|
| 297 |
+
raise RelayError(410, "The request expired or was cancelled; discard the result")
|
| 298 |
+
if result is not None:
|
| 299 |
+
try:
|
| 300 |
+
if isinstance(result, dict) and result.get("kind") == "http_runtime_v1":
|
| 301 |
+
if set(result) != {"kind", "response"}:
|
| 302 |
+
raise ValueError("Invalid HTTP runtime result envelope")
|
| 303 |
+
runtime_request = dict(job["body"], model=self.runtime_model)
|
| 304 |
+
validate_response_for_request(
|
| 305 |
+
runtime_request, result["response"],
|
| 306 |
+
batch="states" in runtime_request,
|
| 307 |
+
confidence=self.confidence,
|
| 308 |
+
)
|
| 309 |
+
result = dict(result, response=dict(
|
| 310 |
+
result["response"], model=self.canonical_model))
|
| 311 |
+
else:
|
| 312 |
+
validated_result(job["body"], result, model=self.model, complete_input_tokens=self.complete_input_tokens)
|
| 313 |
+
except (ValueError, TypeError, KeyError, RuntimeError, OverflowError):
|
| 314 |
+
raise RelayError(422, "Result identity, math, or profile is invalid") from None
|
| 315 |
+
job["completion_sha256"] = digest
|
| 316 |
+
self._finish(job, "succeeded" if result is not None else "failed", result=result,
|
| 317 |
+
error=errors.get(error_code))
|
| 318 |
+
return {"accepted": True, "duplicate": False}
|
requirements.txt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
fastapi==0.115.12
|
| 2 |
+
httpx==0.28.1
|
| 3 |
+
uvicorn==0.34.2
|
static/app.js
ADDED
|
@@ -0,0 +1,438 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import { validateRequest, validateServingResult } from './contract.js?v=archive-1';
|
| 2 |
+
import {validateRegistry, studioExamplesForModel, exampleMode, createModelMenu, modelReadiness, modelPresentation, groupModels, orderedModels, preferredModel, statusName} from './model-menu.js?v=studio-3';
|
| 3 |
+
import {queuedPrediction, requestJSON} from './queue-client.js?v=studio-3';
|
| 4 |
+
const $ = (id) => document.getElementById(id);
|
| 5 |
+
const esc = (value) => String(value).replace(/[&<>"']/g, (x) => ({'&':'&','<':'<','>':'>','"':'"',"'":'''}[x]));
|
| 6 |
+
const clone = (value) => structuredClone(value);
|
| 7 |
+
const display = (value) => value == null ? '' : typeof value === 'string' ? value : JSON.stringify(value, null, 2);
|
| 8 |
+
const pct = (p) => (p * 100).toFixed(1) + '%';
|
| 9 |
+
const TYPE_LABEL = {choice:'Choice', noul:'Noul', score:'Score'};
|
| 10 |
+
const CHEVRON = '<svg class="chevron" viewBox="0 0 16 16" width="14" height="14" aria-hidden="true"><path d="m4 6 4 4 4-4"/></svg>';
|
| 11 |
+
let allExamples = [], examples = [], activeExample, draft, revision = 0, status = null;
|
| 12 |
+
let controller = null, autoTimer, toastTimer, fleetTimer, lastResult = null, userChoseModel = false;
|
| 13 |
+
let models = [], selectedModel = null, modelEpoch = 0;
|
| 14 |
+
let contextMode='routing', statusFlight = null, statusPoll = null, statusTransportError = false;
|
| 15 |
+
let initRetryDelay = 1500, initWarningShown = false;
|
| 16 |
+
const modeWorkspaces=new Map();
|
| 17 |
+
const modelStatuses = new Map(), modelWorkspaces = new Map(), menuStatusVersions = new Map();
|
| 18 |
+
const modelInfo = () => models.find(m => m.id === selectedModel);
|
| 19 |
+
const modelURL = (url, model = selectedModel) => url + '?model=' + encodeURIComponent(model);
|
| 20 |
+
const nameOf = (model = modelInfo()) => model ? statusName(model, models) : 'the model';
|
| 21 |
+
// Decision 2.0 answers every question about one input in a single forward pass.
|
| 22 |
+
const onePass = (model = modelInfo()) => Number.parseFloat(modelPresentation(model || {}).generation) >= 2;
|
| 23 |
+
const isMac = /Mac|iPhone|iPad/.test(navigator.platform);
|
| 24 |
+
$('run-shortcut').textContent = isMac ? '⌘ ↵' : 'Ctrl ↵';
|
| 25 |
+
const reducedMotion = window.matchMedia('(prefers-reduced-motion: reduce)');
|
| 26 |
+
$('motion-toggle').addEventListener('click', () => {
|
| 27 |
+
const paused = document.body.classList.toggle('motion-paused');
|
| 28 |
+
$('motion-toggle').setAttribute('aria-pressed', String(paused));
|
| 29 |
+
$('motion-toggle').setAttribute('aria-label', paused ? 'Resume animations' : 'Pause animations');
|
| 30 |
+
$('motion-toggle').title = paused ? 'Resume animations' : 'Pause animations';
|
| 31 |
+
$('motion-toggle').firstElementChild.textContent = paused ? '▷' : 'Ⅱ';
|
| 32 |
+
});
|
| 33 |
+
const chooseModel = id => {userChoseModel = true; switchModel(id).catch(e => toast(e.message));};
|
| 34 |
+
const modelMenu = createModelMenu({getModels:()=>models, getSelected:()=>selectedModel, getStatus:id=>modelStatuses.get(id), onSelect:chooseModel, onOpen:refreshMenuStatuses, reducedMotion});
|
| 35 |
+
function toast(text) { $('toast').textContent = text; $('toast').hidden = false; clearTimeout(toastTimer); toastTimer = setTimeout(() => $('toast').hidden = true, 4800); }
|
| 36 |
+
// Status regions are live: rewrite them only when the message changes.
|
| 37 |
+
function setRunStatus(text, kind = '') { const el = $('run-status'), className = 'run-status ' + kind; if (el.className === className && el.textContent === text) return; el.className = className; el.replaceChildren(); const led = document.createElement('span'); led.className = 'status-led'; el.append(led, document.createTextNode(text)); }
|
| 38 |
+
function setConnection(text, state) { const el = $('connection'), className = 'connection' + (state ? ' ' + state : ''); if (el.className === className && el.textContent === text) return; el.className = className; el.innerHTML = `<i></i><span>${esc(text)}</span>`; }
|
| 39 |
+
const ACTIVITY = {switching:'Connecting',submitting:'Sending',offline:'Unavailable',ready:'Ready',running:'Reading',loading:'Loading',checking:'Checking',reconnecting:'Reconnecting',queued:'Queued',complete:'Ready',busy:'Busy',error:'Error'};
|
| 40 |
+
function setModelActivity(phase) {
|
| 41 |
+
$('model-card').dataset.state = phase; $('studio-card').dataset.state = phase;
|
| 42 |
+
$('model-card').setAttribute('aria-busy',String(['switching','submitting','queued','running','loading','checking','reconnecting'].includes(phase)));
|
| 43 |
+
$('model-state').textContent = ACTIVITY[phase] || 'Unavailable';
|
| 44 |
+
}
|
| 45 |
+
function updatePass() {
|
| 46 |
+
document.querySelector('.pass-node').textContent = !onePass() ? 'GPU batch' : contextMode === 'batch' ? '1 pass each' : '1 pass';
|
| 47 |
+
}
|
| 48 |
+
function updateRunButton() {
|
| 49 |
+
const q=draft?.questions?.length||0,n=contextMode==='batch'?(draft?.contexts?.length||0):1,total=n*q,label=`Run ${total} decision${total===1?'':'s'}`;
|
| 50 |
+
const batchUnavailable=contextMode==='batch'&&status?.context_batch!==true;
|
| 51 |
+
$('run').disabled=statusTransportError||!status?.configured||!status?.loaded||status?.phase==='unavailable'||!!controller||status?.running||['running','loading'].includes(status?.phase)||batchUnavailable;
|
| 52 |
+
$('run').title=batchUnavailable?'Batch contexts is not ready on this model connection.':'Run the complete current request';
|
| 53 |
+
$('run-label').textContent=controller?`Working on ${total}…`:label;
|
| 54 |
+
$('scenario-summary').textContent=contextMode==='batch'?`${n} context${n===1?'':'s'} × ${q} question${q===1?'':'s'} = ${total} decisions`:contextMode==='routing'?`1 prompt · ${q} routing decision${q===1?'':'s'}`:`1 context · ${q} decision${q===1?'':'s'}`;
|
| 55 |
+
$('context-limit').textContent=batchUnavailable?'Connecting batch mode…':'GPU batches · Complete inputs';
|
| 56 |
+
}
|
| 57 |
+
|
| 58 |
+
function checkStatus() {
|
| 59 |
+
const epoch = modelEpoch, model = selectedModel;
|
| 60 |
+
if (statusFlight?.epoch === epoch && statusFlight.model === model) return statusFlight.promise;
|
| 61 |
+
const promise = checkStatusOnce(epoch, model).finally(() => {
|
| 62 |
+
if (statusFlight?.promise === promise) statusFlight = null;
|
| 63 |
+
});
|
| 64 |
+
statusFlight = {epoch, model, promise};
|
| 65 |
+
return promise;
|
| 66 |
+
}
|
| 67 |
+
async function checkStatusOnce(epoch, model) {
|
| 68 |
+
try {
|
| 69 |
+
const res = await fetch(modelURL('/api/status', model), {cache:'no-store'});
|
| 70 |
+
if (!res.ok) throw Error(); const current = await res.json();
|
| 71 |
+
if(epoch !== modelEpoch) return;
|
| 72 |
+
if(current.model !== model || current.manifest_sha256 !== modelInfo()?.manifest_sha256) throw Error('Model identity mismatch');
|
| 73 |
+
statusTransportError = false;
|
| 74 |
+
status = current; modelStatuses.set(model,current); modelMenu.updateStatuses(); updateFleet();
|
| 75 |
+
const readiness=modelReadiness(status), unavailable=readiness.state==='offline', connecting=readiness.state==='loading', name=nameOf();
|
| 76 |
+
const label = unavailable ? `${name} unavailable` : connecting ? `Connecting to ${name}…` : status.running || status.phase === 'running' ? `${name} is working` : `${name} ready${status.queued ? ' · '+status.queued+' queued' : ''}`;
|
| 77 |
+
setConnection(label, status.loaded ? 'connected' : unavailable ? 'offline' : 'pending');
|
| 78 |
+
$('offline-title').textContent = `${name} is unavailable right now.`;
|
| 79 |
+
$('offline-note').hidden = !unavailable; updateRunButton();
|
| 80 |
+
if((!controller&&!lastResult)||['switching','loading','checking','reconnecting'].includes($('model-card').dataset.state)) setRunStatus(unavailable ? `${name} unavailable · editing only` : connecting ? `Connecting to ${name}…` : status.running || status.phase === 'running' ? `${name} is working` : `${name} ready · run your questions`);
|
| 81 |
+
if (!controller) setModelActivity(unavailable ? 'offline' : connecting ? 'loading' : status.running || status.phase === 'running' ? 'busy' : lastResult && !$('results').classList.contains('stale') ? 'complete' : 'ready');
|
| 82 |
+
else if (status.backend !== 'pull_queue' && ['running','loading'].includes(status.phase)) setModelActivity(status.phase);
|
| 83 |
+
} catch (error) {
|
| 84 |
+
if(epoch !== modelEpoch) return;
|
| 85 |
+
statusTransportError = true;
|
| 86 |
+
const identityChanged = error?.message === 'Model identity mismatch';
|
| 87 |
+
modelStatuses.set(model, status ? {...status, loaded:false, phase:'loading'} : null);
|
| 88 |
+
modelMenu.updateStatuses(); updateFleet();
|
| 89 |
+
setConnection(identityChanged ? 'Model configuration changed · reload' : 'Connection interrupted · retrying', 'pending');
|
| 90 |
+
$('offline-note').hidden = true;
|
| 91 |
+
updateRunButton();
|
| 92 |
+
if (!controller) setModelActivity(identityChanged ? 'error' : 'reconnecting');
|
| 93 |
+
}
|
| 94 |
+
}
|
| 95 |
+
async function refreshMenuStatuses() {
|
| 96 |
+
// Read-only readiness discovery. Selecting a model never creates an inference job.
|
| 97 |
+
await Promise.allSettled(models.map(async info=>{
|
| 98 |
+
const version=(menuStatusVersions.get(info.id)||0)+1;menuStatusVersions.set(info.id,version);
|
| 99 |
+
try {
|
| 100 |
+
const current=await requestJSON(modelURL('/api/status',info.id),{cache:'no-store'});
|
| 101 |
+
if(current.model!==info.id||current.manifest_sha256!==info.manifest_sha256)throw Error('Model identity mismatch');
|
| 102 |
+
if(menuStatusVersions.get(info.id)===version)modelStatuses.set(info.id,current);
|
| 103 |
+
} catch {if(menuStatusVersions.get(info.id)===version){const previous=modelStatuses.get(info.id);modelStatuses.set(info.id,previous?{...previous,loaded:false,phase:'loading'}:null);}}
|
| 104 |
+
modelMenu.updateStatuses();
|
| 105 |
+
}));
|
| 106 |
+
updateFleet();
|
| 107 |
+
}
|
| 108 |
+
function scheduleFleetRefresh() {
|
| 109 |
+
clearTimeout(fleetTimer);
|
| 110 |
+
fleetTimer = setTimeout(async () => { if (!document.hidden) await refreshMenuStatuses(); scheduleFleetRefresh(); }, 60000);
|
| 111 |
+
}
|
| 112 |
+
async function loadModels() {
|
| 113 |
+
const result=validateRegistry(await requestJSON('/v1/models',{cache:'no-store'}));
|
| 114 |
+
models=result.models.map(model=>({...model,id:model.wire_id,repo_id:model.id}));
|
| 115 |
+
const serverDefault=result.default?models.find(model=>model.repo_id===result.default)?.id:undefined;
|
| 116 |
+
const kept=models.find(model=>model.id===selectedModel);
|
| 117 |
+
selectedModel=(kept||preferredModel(models,serverDefault)||models[0]).id;
|
| 118 |
+
modelMenu.identity(modelInfo(),{enabled:false});
|
| 119 |
+
updatePass();renderFleet();
|
| 120 |
+
}
|
| 121 |
+
// Before the first interaction, open on a ready model of the newest generation
|
| 122 |
+
// rather than an unavailable default. Requests never fall back to another model.
|
| 123 |
+
async function settleInitialModel() {
|
| 124 |
+
if(userChoseModel||modelReadiness(modelStatuses.get(selectedModel)).state!=='offline')return;
|
| 125 |
+
const ready=orderedModels(models).filter(m=>modelReadiness(modelStatuses.get(m.id)).state==='ready');
|
| 126 |
+
const choice=ready.find(m=>!modelPresentation(m).reasoning)||ready[0];
|
| 127 |
+
if(choice&&choice.id!==selectedModel)await switchModel(choice.id);
|
| 128 |
+
}
|
| 129 |
+
function saveWorkspace() {
|
| 130 |
+
if(draft){const saved={draft:clone(draft),text:$('state-input').value,contextMode,exampleId:activeExample.id};modelWorkspaces.set(selectedModel,saved);modeWorkspaces.set(selectedModel+'/'+contextMode,saved);}
|
| 131 |
+
}
|
| 132 |
+
function showScenario(example) { $('scenario-title').textContent = example.title; $('scenario-category').textContent = example.category; $('scenario-title').title = example.description || ''; }
|
| 133 |
+
function applyWorkspace(model,previous) {
|
| 134 |
+
const catalog=studioExamplesForModel(model,allExamples);
|
| 135 |
+
examples=catalog.examples;
|
| 136 |
+
const saved=modelWorkspaces.get(model.id);
|
| 137 |
+
if(saved&&examples.some(e=>e.id===saved.exampleId)) {
|
| 138 |
+
activeExample=examples.find(e=>e.id===saved.exampleId);draft=clone(saved.draft);
|
| 139 |
+
showScenario(activeExample);
|
| 140 |
+
contextMode=saved.contextMode||exampleMode(activeExample);$('state-input').value=saved.text;renderContextMode();renderContexts();renderExamples();renderQuestions();updateCount();dirty();
|
| 141 |
+
} else {
|
| 142 |
+
const previousExample=examples.find(e=>e.id===previous?.exampleId);
|
| 143 |
+
const sameMode=examples.find(e=>exampleMode(e)===(previous?.contextMode||'routing'));
|
| 144 |
+
loadExample(previousExample||sameMode||catalog.defaultExample);
|
| 145 |
+
}
|
| 146 |
+
}
|
| 147 |
+
async function switchModel(model) {
|
| 148 |
+
if(model===selectedModel||!models.some(m=>m.id===model))return;
|
| 149 |
+
saveWorkspace();const previous=modelWorkspaces.get(selectedModel);
|
| 150 |
+
++modelEpoch;++revision;
|
| 151 |
+
clearTimeout(autoTimer);
|
| 152 |
+
if(controller){controller.abort();controller=null;}
|
| 153 |
+
selectedModel=model;status=null;statusTransportError=false;lastResult=null;
|
| 154 |
+
$('results').classList.remove('stale');
|
| 155 |
+
$('result-count').textContent='—';$('results-title').textContent='Decisions';$('timing').hidden=true;$('timing').replaceChildren();
|
| 156 |
+
updatePass();
|
| 157 |
+
applyWorkspace(modelInfo(),previous);
|
| 158 |
+
modelMenu.identity(modelInfo(),{animate:true});
|
| 159 |
+
setConnection('Connecting to model','pending');
|
| 160 |
+
setRunStatus(`Connecting to ${nameOf()}…`);setModelActivity('switching');updateRunButton();renderFleet();
|
| 161 |
+
await checkStatus(); // Actual selected identity and readiness, without a cosmetic delay.
|
| 162 |
+
}
|
| 163 |
+
function renderExamples() {
|
| 164 |
+
$('examples').innerHTML = examples.map((e,i) => ({e,i})).filter(({e})=>exampleMode(e)===contextMode).map(({e,i}) => `<button type="button" class="example-button${e.id===activeExample?.id?' selected':''}" data-example="${i}" aria-pressed="${e.id===activeExample?.id}" title="${esc(e.description||e.category||'')}">${esc(e.title)}</button>`).join('');
|
| 165 |
+
const list=$('examples'),chip=list.querySelector('.selected');
|
| 166 |
+
if(chip&&(chip.offsetLeft<list.scrollLeft||chip.offsetLeft+chip.offsetWidth>list.scrollLeft+list.clientWidth))list.scrollLeft=chip.offsetLeft-24;
|
| 167 |
+
}
|
| 168 |
+
function loadExample(example) {
|
| 169 |
+
activeExample = example;
|
| 170 |
+
contextMode=exampleMode(example);
|
| 171 |
+
const states=contextMode==='batch'?example.states:[{id:'context_1',state:example.state}];
|
| 172 |
+
draft = {contexts:states.map(s=>({id:s.id,text:display(s.state),expanded:false})), questions: Object.entries(clone(example.questions)).map(([id,q]) => ({id,...q,expanded:false}))};
|
| 173 |
+
showScenario(example);
|
| 174 |
+
$('state-input').value = draft.contexts[0].text;
|
| 175 |
+
renderContextMode();renderContexts();renderExamples(); renderQuestions(); updateCount(); dirty();
|
| 176 |
+
}
|
| 177 |
+
function updateCount() { $('state-count').textContent = `${$('state-input').value.length.toLocaleString('en')} characters`; }
|
| 178 |
+
function renderContextMode(){
|
| 179 |
+
for(const mode of ['routing','single','batch'])$('context-mode-'+mode).setAttribute('aria-pressed',String(contextMode===mode));
|
| 180 |
+
$('single-context-editor').hidden=contextMode==='batch';$('batch-context-editor').hidden=contextMode!=='batch';
|
| 181 |
+
$('context-heading').textContent=contextMode==='batch'?'Contexts':contextMode==='routing'?'User prompt':'Context';$('context-count').hidden=contextMode!=='batch';
|
| 182 |
+
$('questions-heading').textContent=contextMode==='batch'?'Shared questions':contextMode==='routing'?'Routing decisions':'Questions';$('shared-note').hidden=contextMode!=='batch';
|
| 183 |
+
$('mode-description').textContent=contextMode==='batch'?'Many contexts, one set of questions.':contextMode==='routing'?'Route each request to the right model.':'One context, many decisions.';
|
| 184 |
+
$('workbench').dataset.mode=contextMode;
|
| 185 |
+
updatePass();
|
| 186 |
+
}
|
| 187 |
+
function renderContexts(){
|
| 188 |
+
$('context-count').textContent=draft.contexts.length;$('add-context').disabled=false;
|
| 189 |
+
$('context-list').innerHTML=draft.contexts.map((c,i)=>`<article class="context-item${c.expanded?' is-open':''}"><div class="context-row"><button type="button" class="context-open" data-open-context="${i}" aria-expanded="${c.expanded}" aria-controls="context-editor-${i}" aria-label="${c.expanded?'Collapse':'Edit'} context ${esc(c.id)}">${CHEVRON}<span class="context-row-id">${esc(c.id)}</span><span class="context-preview">${esc(c.text.replace(/\s+/g,' '))||'Write a context…'}</span></button><button type="button" class="icon-button" data-delete-context="${i}" aria-label="Remove context ${esc(c.id)}" ${draft.contexts.length<=1?'disabled':''}>×</button></div><div class="context-editor" id="context-editor-${i}" ${c.expanded?'':'hidden'}><div class="context-editor-head"><label class="inline-field">ID <input data-context-id="${i}" value="${esc(c.id)}" aria-label="Context ${i+1} ID" spellcheck="false"></label></div><textarea data-context-text="${i}" class="context-text" aria-label="Context ${i+1} content" spellcheck="false">${esc(c.text)}</textarea></div></article>`).join('');
|
| 190 |
+
updateRunButton();
|
| 191 |
+
}
|
| 192 |
+
function switchContextMode(mode){
|
| 193 |
+
if(!['routing','single','batch'].includes(mode))return;
|
| 194 |
+
if(mode===contextMode)return;saveWorkspace();const saved=modeWorkspaces.get(selectedModel+'/'+mode);
|
| 195 |
+
if(saved){draft=clone(saved.draft);activeExample=examples.find(e=>e.id===saved.exampleId)||activeExample;contextMode=mode;$('state-input').value=saved.text;showScenario(activeExample);renderContextMode();renderContexts();renderExamples();renderQuestions();updateCount();dirty();}
|
| 196 |
+
else {const example=examples.find(e=>exampleMode(e)===mode);if(example)loadExample(example);else {contextMode=mode;if(mode!=='batch'){draft.contexts=draft.contexts.slice(0,1);$('state-input').value=draft.contexts[0].text;}renderContextMode();renderContexts();renderExamples();renderQuestions();dirty();}}
|
| 197 |
+
}
|
| 198 |
+
function choicesHTML(q, index) {
|
| 199 |
+
if (q.type === 'noul') return `<p class="field-hint">Returns the probability of yes, from 0 to 1.</p><details class="noul-details" ${q.criteria?'open':''}><summary>Define what yes and no mean</summary>${['true','false'].map(k=>`<label class="criterion"><span class="criterion-tag">${k==='true'?'Yes':'No'}</span><input data-index="${index}" data-polarity="${k}" value="${esc(q.criteria?.[k] ? display(q.criteria[k]) : '')}" placeholder="${k==='true'?'Optional yes criterion':'Optional no criterion'}" aria-label="${k==='true'?'Yes':'No'} criterion"></label>`).join('')}</details>`;
|
| 200 |
+
const entries = q.type === 'choice' ? Object.entries(q.criteria) : q.criteria.map((text,i) => [String(i),text]);
|
| 201 |
+
return `<div class="criteria-list ${q.type}-criteria">${entries.map(([name,text],i)=>`<div class="criterion"><span class="criterion-tag">${q.type==='score'?i:String.fromCharCode(65+i%26)}</span>${q.type==='choice'?`<input class="option-name" data-index="${index}" data-option="${i}" data-field="name" value="${esc(name)}" aria-label="Option ${i+1} name">`:''}<input data-index="${index}" data-option="${i}" data-field="description" value="${esc(text===null?'':display(text))}" placeholder="${q.type==='choice'?'Description (optional)':'Describe this level'}" aria-label="${q.type==='choice'?'Option':'Level'} ${i+1} description"><button type="button" class="icon-button" data-remove-option="${i}" data-index="${index}" aria-label="Remove ${q.type==='score'?'level':'option'} ${i+1}" ${entries.length<=2?'disabled':''}>×</button></div>`).join('')}</div><div class="criteria-foot"><span>${q.type==='score'?'Ordered from lowest to highest':'Names and descriptions reach the model'}</span><button type="button" class="button-quiet small" data-add-option="${index}" ${entries.length>=(q.type==='score'?10:255)?'disabled':''}>${q.type==='score'?'Add level':'Add option'}</button></div>`;
|
| 202 |
+
}
|
| 203 |
+
function renderQuestions() {
|
| 204 |
+
$('question-count').textContent = draft.questions.length;
|
| 205 |
+
updateRunButton();
|
| 206 |
+
$('questions').innerHTML = draft.questions.map((q,i)=>`<article class="question-card${q.expanded?' is-open':''}">
|
| 207 |
+
<div class="question-head"><button type="button" class="question-toggle" data-toggle-question="${i}" aria-expanded="${!!q.expanded}" aria-controls="question-body-${i}" aria-label="${q.expanded?'Collapse':'Expand'} ${esc(q.id)}">${CHEVRON}<span class="type-tag ${q.type}">${TYPE_LABEL[q.type]}</span><span class="question-summary"><span class="question-summary-id">${esc(q.id)}</span><span class="question-summary-text">${esc(display(q.instructions))}</span></span></button><button type="button" class="icon-button" data-remove="${i}" aria-label="Remove question ${i+1}" ${draft.questions.length<=1?'disabled':''}>×</button></div>
|
| 208 |
+
<div class="question-body" id="question-body-${i}" ${q.expanded?'':'hidden'}>
|
| 209 |
+
<label class="field"><span>Question ID</span><input class="question-id" data-index="${i}" data-question-id value="${esc(q.id)}" spellcheck="false"></label>
|
| 210 |
+
<label class="field"><span>Instructions</span><textarea class="question-text" data-index="${i}" data-instructions rows="2" aria-label="Question ${i+1} instructions" placeholder="Required instructions">${esc(display(q.instructions))}</textarea></label>${choicesHTML(q,i)}
|
| 211 |
+
</div></article>`).join('');
|
| 212 |
+
}
|
| 213 |
+
function dirty() {
|
| 214 |
+
revision++;
|
| 215 |
+
if (controller) { controller.abort(); controller = null; setRunStatus('Inputs changed · previous request detached'); checkStatus(); }
|
| 216 |
+
if (lastResult) { $('results').classList.add('stale'); setRunStatus('Previous run · inputs have changed'); }
|
| 217 |
+
else renderPending();
|
| 218 |
+
setModelActivity(!status?.configured ? 'offline' : !status.loaded || status.phase==='loading' ? 'loading' : status.running || status.phase==='running' ? 'busy' : 'ready');
|
| 219 |
+
clearTimeout(autoTimer);
|
| 220 |
+
if ($('auto-run').checked && status?.configured) autoTimer = setTimeout(autoRun, 1000);
|
| 221 |
+
updateRunButton();
|
| 222 |
+
}
|
| 223 |
+
async function autoRun() { await checkStatus(); if (['running','loading'].includes(status?.phase)) { autoTimer=setTimeout(autoRun,1000); return; } run(); }
|
| 224 |
+
function requestBody() {
|
| 225 |
+
const questions = Object.create(null);
|
| 226 |
+
for (const q of draft.questions) {
|
| 227 |
+
if (!q.id.trim() || Object.hasOwn(questions, q.id)) throw Error('Every question needs a unique, nonempty ID.');
|
| 228 |
+
if (q.invalid) throw Error(q.invalid);
|
| 229 |
+
questions[q.id] = {type:q.type};
|
| 230 |
+
if(q.type==='noul'||(q.instructions!==null&&q.instructions!==undefined&&q.instructions!=='')) questions[q.id].instructions=q.instructions;
|
| 231 |
+
if (q.type!=='noul' || q.criteria && Object.keys(q.criteria).length) questions[q.id].criteria = clone(q.criteria);
|
| 232 |
+
}
|
| 233 |
+
const input=contextMode==='batch'?{states:draft.contexts.map(c=>({id:c.id,state:c.text}))}:{state:$('state-input').value};
|
| 234 |
+
return validateRequest({model:selectedModel,...input,questions}, selectedModel);
|
| 235 |
+
}
|
| 236 |
+
|
| 237 |
+
// Result rendering. `a` is null while a decision is pending, so the layout of
|
| 238 |
+
// every card is visible before the run and settles in place afterwards.
|
| 239 |
+
const shortLevel = (level) => { const text = display(level); return text.includes(':') ? text.split(':', 1)[0] : text; };
|
| 240 |
+
function choiceBody(q, a) {
|
| 241 |
+
const ids = Object.keys(q.criteria || {});
|
| 242 |
+
const order = a ? [...ids].sort((x, y) => a.probabilities[y] - a.probabilities[x]) : ids;
|
| 243 |
+
const head = a ? `<span class="decision-value">${esc(a.choice)}</span><span class="decision-meta">${pct(a.probabilities[a.choice])}</span>` : `<span class="decision-value is-empty">—</span><span class="decision-meta">${ids.length} options</span>`;
|
| 244 |
+
const rows = order.map((id, i) => {
|
| 245 |
+
const p = a ? a.probabilities[id] : 0, top = a && id === a.choice, description = display(q.criteria[id]);
|
| 246 |
+
return `<div class="dist-row${top ? ' is-top' : ''}" style="--p:${p};--r:${i}"><span class="label" title="${esc(description || id)}">${esc(id)}</span><span class="pct">${a ? pct(p) : '—'}</span><span class="track"><span class="fill"></span></span></div>`;
|
| 247 |
+
}).join('');
|
| 248 |
+
return `<div class="decision-answer">${head}</div><div class="dist">${rows}</div>`;
|
| 249 |
+
}
|
| 250 |
+
function noulBody(q, a) {
|
| 251 |
+
const p = a ? a.noul : 0, yes = a && a.noul >= 0.5;
|
| 252 |
+
const head = a ? `<span class="decision-value">${yes ? 'Yes' : 'No'}</span><span class="decision-meta">P(yes) ${pct(p)}</span>` : `<span class="decision-value is-empty">—</span><span class="decision-meta">Yes or no</span>`;
|
| 253 |
+
return `<div class="decision-answer">${head}</div><div class="gauge${a ? (yes ? ' is-yes' : ' is-no') : ''}" style="--p:${p}" role="img" aria-label="${a ? `Probability of yes ${pct(p)}` : 'Pending'}"><span class="gauge-fill"></span><span class="gauge-mid"></span></div><div class="gauge-scale" aria-hidden="true"><span>No</span><span>50%</span><span>Yes</span></div>`;
|
| 254 |
+
}
|
| 255 |
+
function scoreBody(q, a) {
|
| 256 |
+
const levels = q.criteria || [], n = levels.length;
|
| 257 |
+
const probs = levels.map((_, i) => a ? a.probabilities[String(i)] : 0);
|
| 258 |
+
const top = a ? probs.indexOf(Math.max(...probs)) : -1;
|
| 259 |
+
const head = a ? `<span class="decision-value">${a.score.toFixed(2)}<em> / ${n - 1}</em></span><span class="decision-meta">${esc(shortLevel(levels[top]))} · ${pct(probs[top])}</span>` : `<span class="decision-value is-empty">—</span><span class="decision-meta">${n} levels</span>`;
|
| 260 |
+
const cols = levels.map((level, i) => `<div class="level${i === top ? ' is-top' : ''}" style="--p:${probs[i]};--r:${i}" title="${esc(`${i} · ${display(level)}`)}"><span class="level-pct">${a ? pct(probs[i]) : '—'}</span><span class="level-bar"><span></span></span><span class="level-name"><b>${i}</b> ${esc(shortLevel(level))}</span></div>`).join('');
|
| 261 |
+
const marker = a ? `<span class="level-marker" style="--x:${((a.score + 0.5) / n * 100).toFixed(2)}%" title="Expected level ${a.score.toFixed(2)}"></span>` : '';
|
| 262 |
+
return `<div class="decision-answer">${head}</div><div class="levels${n > 5 ? ' is-dense' : ''}" style="--n:${n}">${cols}${marker}</div>`;
|
| 263 |
+
}
|
| 264 |
+
function decisionCard(id, q, a, i = 0) {
|
| 265 |
+
const body = q.type === 'noul' ? noulBody(q, a) : q.type === 'score' ? scoreBody(q, a) : choiceBody(q, a);
|
| 266 |
+
return `<article class="decision ${q.type}-result${a ? '' : ' is-pending'}" style="--i:${i}" aria-label="${esc(id)} ${a ? 'decision' : '(pending)'}"><div class="decision-head"><span class="decision-id" title="${esc(display(q.instructions))}">${esc(id)}</span><span class="decision-kind">${TYPE_LABEL[q.type]}</span></div><p class="decision-question">${esc(display(q.instructions))}</p>${body}</article>`;
|
| 267 |
+
}
|
| 268 |
+
function renderPending() {
|
| 269 |
+
if (!draft) return;
|
| 270 |
+
const n = contextMode === 'batch' ? draft.contexts.length : 1;
|
| 271 |
+
const cards = draft.questions.map((q, i) => decisionCard(q.id || `question_${i + 1}`, q, null, i)).join('');
|
| 272 |
+
const lede = n > 1 ? `Each of the ${n} contexts receives these ${draft.questions.length} questions.` : '';
|
| 273 |
+
$('results').innerHTML = `${lede ? `<p class="results-lede">${lede}</p>` : ''}<div class="decisions is-pending${draft.questions.length === 1 ? ' single-decision' : ''}">${cards}</div>`;
|
| 274 |
+
}
|
| 275 |
+
function compactDecision(a,q){
|
| 276 |
+
if(a.type==='noul'){const yes=a.noul>=.5;return `<span class="cell-value">${yes?'Yes':'No'}</span><span class="cell-meta">P(yes) ${pct(a.noul)}</span>`;}
|
| 277 |
+
if(a.type==='score'){const best=Object.entries(a.probabilities).reduce((x,y)=>y[1]>x[1]?y:x);return `<span class="cell-value">${a.score.toFixed(2)}<em> / ${q.criteria.length-1}</em></span><span class="cell-meta">Level ${esc(best[0])} · ${pct(best[1])}</span>`;}
|
| 278 |
+
return `<span class="cell-value">${esc(a.choice)}</span><span class="cell-meta">${pct(a.probabilities[a.choice])}</span>`;
|
| 279 |
+
}
|
| 280 |
+
function renderResult(result,payload,backend,elapsed){
|
| 281 |
+
validateServingResult(result,payload,modelInfo().repo_id,backend);setModelActivity('complete');lastResult=result;$('results').classList.remove('stale');
|
| 282 |
+
const qids=Object.keys(payload.questions),isBatch=Array.isArray(payload.states),rows=isBatch?result.results:[{id:'context',answers:result.answers,usage:result.usage}],count=rows.length*qids.length;
|
| 283 |
+
$('result-count').textContent=count;$('results-title').textContent='Decisions';
|
| 284 |
+
if(isBatch){
|
| 285 |
+
$('results').innerHTML=`<p class="results-lede">Shared questions, separate decisions. Open a row for every probability.</p><div class="result-table-scroll" tabindex="0" role="region" aria-label="Context decision table"><table class="result-table"><thead><tr><th scope="col">Context</th>${qids.map(id=>`<th scope="col" title="${esc(display(payload.questions[id].instructions))}"><span class="th-id">${esc(id)}</span><span class="th-kind">${TYPE_LABEL[payload.questions[id].type]}</span></th>`).join('')}</tr></thead><tbody>${rows.map((r,i)=>`<tr class="result-row" data-result-row="${i}"><th scope="row"><button type="button" class="row-toggle" aria-expanded="false" aria-controls="result-detail-${i}" aria-label="Show probabilities for ${esc(r.id)}">${CHEVRON}<span>${esc(r.id)}</span></button></th>${qids.map(id=>`<td class="decision-cell ${payload.questions[id].type}">${compactDecision(r.answers[id],payload.questions[id])}</td>`).join('')}</tr><tr id="result-detail-${i}" class="result-detail-row" hidden><td colspan="${qids.length+1}"><div class="decisions detail-grid">${qids.map((id,j)=>decisionCard(id,payload.questions[id],r.answers[id],j)).join('')}</div></td></tr>`).join('')}</tbody></table></div>`;
|
| 286 |
+
}else{
|
| 287 |
+
$('results').innerHTML=`<div class="decisions is-settled${qids.length===1?' single-decision':''}">${qids.map((id,i)=>decisionCard(id,payload.questions[id],result.answers[id],i)).join('')}</div>`;
|
| 288 |
+
}
|
| 289 |
+
setRunStatus(`${nameOf()} · ${count} decision${count===1?'':'s'} ready`);
|
| 290 |
+
$('timing').hidden=true;
|
| 291 |
+
$('timing').replaceChildren();
|
| 292 |
+
}
|
| 293 |
+
async function nativePrediction(payload, current) {
|
| 294 |
+
return requestJSON('/api/evaluate', {method:'POST', headers:{'Content-Type':'application/json'}, body:JSON.stringify(payload), signal:current.signal});
|
| 295 |
+
}
|
| 296 |
+
async function run() {
|
| 297 |
+
if(controller || !status?.configured || !status?.loaded || status.running || ['running','loading'].includes(status?.phase)) { if(!status?.configured) $('setup-dialog').showModal(); return; }
|
| 298 |
+
if(contextMode==='batch'&&status.context_batch!==true){toast('Batch contexts is not ready on this model connection.');return;}
|
| 299 |
+
let payload; try {payload=requestBody();}catch(e){toast(e.message);return;}
|
| 300 |
+
clearTimeout(toastTimer);$('toast').hidden=true;
|
| 301 |
+
const version=revision, epoch=modelEpoch, backend=status.backend, info=modelInfo(), current=new AbortController(); controller=current; updateRunButton(); clearTimeout(autoTimer); const begin=performance.now();
|
| 302 |
+
setRunStatus(status.backend==='cache' ? 'Replaying the recorded result…' : status.backend==='pull_queue' ? 'Sending your request…' : status.loaded ? 'Reading your context…' : 'Loading the native model…','running');
|
| 303 |
+
setModelActivity(status.backend==='pull_queue' ? 'submitting' : status.loaded ? 'running' : 'loading');
|
| 304 |
+
try {
|
| 305 |
+
const result = backend === 'pull_queue' ? await queuedPrediction(payload, current, info, {
|
| 306 |
+
onProgress: phase => {setRunStatus(phase === 'queued' ? 'Waiting for the model…' : 'Reading your context…', 'running');setModelActivity(phase);},
|
| 307 |
+
validateCompletion: result => validateServingResult(result,payload,info.repo_id,backend),
|
| 308 |
+
}) : await nativePrediction(payload, current);
|
| 309 |
+
if(version!==revision || epoch!==modelEpoch || current.signal.aborted) return;
|
| 310 |
+
if(payload.model!==selectedModel) throw Error('The selected model changed.');
|
| 311 |
+
renderResult(result,payload,backend,performance.now()-begin);
|
| 312 |
+
} catch(e) {if(e.name!=='AbortError' && version===revision){setModelActivity('error');setRunStatus('Could not complete this request','error');toast(e.message); if(!lastResult)$('results').innerHTML=`<div class="error-message" role="alert"><b>No result.</b> ${esc(e.message)}</div>`;}}
|
| 313 |
+
finally{if(controller===current)controller=null;updateRunButton();if(epoch===modelEpoch)await checkStatus();}
|
| 314 |
+
}
|
| 315 |
+
|
| 316 |
+
// Family lineup and model table: views of the same registry.
|
| 317 |
+
const sizeValue = (label) => { const m = /^(\d+(?:\.\d+)?)\s*([BM])$/i.exec(label || ''); return m ? Number(m[1]) / (m[2].toUpperCase() === 'M' ? 1000 : 1) : null; };
|
| 318 |
+
const tokenText = (n) => n % 1024 === 0 ? `${n / 1024}K` : n.toLocaleString('en');
|
| 319 |
+
function readinessOf(model) { return modelReadiness(modelStatuses.get(model.id)); }
|
| 320 |
+
function renderHero() {
|
| 321 |
+
const newest = groupModels(models)[0];
|
| 322 |
+
const generation = document.body.dataset.generation || modelPresentation(newest?.models[0] || {}).generation || '2.0';
|
| 323 |
+
const title = `Decision <span class="grad">${esc(generation)}</span>`;
|
| 324 |
+
if ($('hero-title').innerHTML !== title) $('hero-title').innerHTML = title;
|
| 325 |
+
$('hero-eyebrow').textContent = 'INTRODUCING';
|
| 326 |
+
$('hero-sub').textContent = 'Open Foundation Decision Models';
|
| 327 |
+
$('hero-tag').textContent = Number.parseFloat(generation) >= 2 ? 'One pass. Many decisions.' : 'One input. Many decisions.';
|
| 328 |
+
const sizes = newest ? newest.models.filter(m => !modelPresentation(m).reasoning).map(m => modelPresentation(m).size) : [];
|
| 329 |
+
$('hero-sizes').textContent = [...new Set(sizes)].sort((a, b) => (sizeValue(a) ?? 0) - (sizeValue(b) ?? 0)).join(' · ');
|
| 330 |
+
}
|
| 331 |
+
function renderLineup() {
|
| 332 |
+
const newest = groupModels(models)[0];
|
| 333 |
+
const base = newest ? newest.models.filter(m => !modelPresentation(m).reasoning) : [];
|
| 334 |
+
if (base.length < 2) { $('lineup').hidden = true; return; }
|
| 335 |
+
const sizes = base.map(m => sizeValue(modelPresentation(m).size));
|
| 336 |
+
const known = sizes.filter(Number.isFinite), lo = Math.log(Math.min(...known)), hi = Math.log(Math.max(...known));
|
| 337 |
+
const selected = modelInfo(), sp = selected && modelPresentation(selected);
|
| 338 |
+
$('lineup-title').textContent = newest.label;
|
| 339 |
+
const first = modelPresentation(base.at(-1)).size, last = modelPresentation(base[0]).size;
|
| 340 |
+
const distinct = new Set(base.map(m => modelPresentation(m).size)).size;
|
| 341 |
+
$('lineup-meta').textContent = `${base.length} ${distinct === base.length ? 'sizes' : 'models'}${known.length === base.length ? ` · ${[first, last].sort((x, y) => sizeValue(x) - sizeValue(y)).join(' – ')}` : ''}`;
|
| 342 |
+
const ordered = base.map((m, i) => ({m, size: sizes[i]})).sort((x, y) => (x.size ?? 0) - (y.size ?? 0));
|
| 343 |
+
$('lineup-stage').innerHTML = ordered.map(({m, size}) => {
|
| 344 |
+
const p = modelPresentation(m), ratio = Number.isFinite(size) && hi > lo ? (Math.log(size) - lo) / (hi - lo) : 0.5;
|
| 345 |
+
const active = sp && sp.generation === p.generation && sp.codename === p.codename && sp.size === p.size;
|
| 346 |
+
const r = readinessOf(m);
|
| 347 |
+
return `<button type="button" class="size-option" data-model="${esc(m.id)}" aria-pressed="${!!active}" data-readiness="${r.state}" style="--d:${ratio.toFixed(3)}" aria-label="${esc(p.name)}, ${r.label}"><span class="orb-wrap"><span class="orb"></span></span><span class="size-name">${esc(p.codename)}</span><span class="size-label"><span class="status-dot" aria-hidden="true"></span>${esc(p.size)}</span></button>`;
|
| 348 |
+
}).join('');
|
| 349 |
+
const reasoning = newest.models.length - base.length;
|
| 350 |
+
$('lineup-foot').textContent = reasoning ? `Plus ${reasoning} Reasoning variant${reasoning === 1 ? '' : 's'}. One API for every size.` : 'One API for every size.';
|
| 351 |
+
const facts = selected ? [sp.description, `${tokenText(selected.complete_input_tokens)} complete input`, sp.releaseLabel && `Released ${sp.releaseLabel}`].filter(Boolean) : [];
|
| 352 |
+
$('lineup-detail').innerHTML = selected ? `<b>${esc(sp.name)}${sp.generation ? ` <span class="lineup-generation">${esc(sp.generation)}</span>` : ''}</b><span>${esc(facts.join(' · '))}</span>` : '';
|
| 353 |
+
$('lineup').hidden = false;
|
| 354 |
+
}
|
| 355 |
+
function renderModelTable() {
|
| 356 |
+
const body = $('model-table').tBodies[0];
|
| 357 |
+
body.innerHTML = groupModels(models).map(group => `<tr class="group-row"><th colspan="6" scope="colgroup"><span>${esc(group.label)}</span><small>${group.models.length} model${group.models.length === 1 ? '' : 's'}</small></th></tr>` + group.models.map(m => {
|
| 358 |
+
const p = modelPresentation(m), r = readinessOf(m), current = m.id === selectedModel;
|
| 359 |
+
const compact = [p.description, `${tokenText(m.complete_input_tokens)} input`, p.releaseLabel].filter(Boolean);
|
| 360 |
+
return `<tr class="model-row${current ? ' is-selected' : ''}"><th scope="row"><span class="model-cell"><span class="codename">${esc(p.codename)}</span><span class="compact-only size-inline" aria-hidden="true">${esc(p.size)}</span>${p.reasoning ? '<span class="badge badge-reasoning">Reasoning</span>' : ''}</span>${p.description ? `<span class="model-desc wide-only">${esc(p.description)}</span>` : ''}<span class="model-desc compact-only" aria-hidden="true">${esc(compact.join(' · '))}</span></th><td class="num">${esc(p.size || '—')}</td><td class="num">${tokenText(m.complete_input_tokens)} tokens</td><td class="num">${esc(p.releaseLabel || '—')}</td><td><span class="readiness" data-readiness-for="${esc(m.id)}" data-readiness="${r.state}"><span class="status-dot" aria-hidden="true"></span>${r.label}</span></td><td class="row-actions"><button type="button" class="button-quiet small" data-use-model="${esc(m.id)}" aria-pressed="${current}">${current ? 'In Studio' : 'Try'}</button><a class="text-link" href="https://huggingface.co/${esc(m.repo_id)}" target="_blank" rel="noopener" aria-label="${esc(p.name)} on Hugging Face">Hub <span aria-hidden="true">↗</span></a></td></tr>`;
|
| 361 |
+
}).join('')).join('');
|
| 362 |
+
}
|
| 363 |
+
// Re-rendering replaces buttons, so keep keyboard focus on the same model.
|
| 364 |
+
function keepFocus(container, render) {
|
| 365 |
+
const active = document.activeElement, inside = container.contains(active);
|
| 366 |
+
const attr = inside && active.dataset.model ? 'data-model' : inside && active.dataset.useModel ? 'data-use-model' : null;
|
| 367 |
+
const key = attr && active.getAttribute(attr);
|
| 368 |
+
render();
|
| 369 |
+
if (key) container.querySelector(`[${attr}="${CSS.escape(key)}"]`)?.focus({preventScroll: true});
|
| 370 |
+
}
|
| 371 |
+
function renderFleet() { if (!models.length) return; renderHero(); keepFocus($('lineup'), renderLineup); keepFocus($('model-table'), renderModelTable); }
|
| 372 |
+
function updateFleet() {
|
| 373 |
+
for (const button of $('lineup-stage').querySelectorAll('[data-model]')) {
|
| 374 |
+
const model = models.find(m => m.id === button.dataset.model); if (!model) continue;
|
| 375 |
+
const r = readinessOf(model); button.dataset.readiness = r.state; button.setAttribute('aria-label', `${modelPresentation(model).name}, ${r.label}`);
|
| 376 |
+
}
|
| 377 |
+
for (const cell of $('model-table').querySelectorAll('[data-readiness-for]')) {
|
| 378 |
+
const model = models.find(m => m.id === cell.dataset.readinessFor); if (!model) continue;
|
| 379 |
+
const r = readinessOf(model); cell.dataset.readiness = r.state; cell.lastChild.textContent = r.label;
|
| 380 |
+
}
|
| 381 |
+
}
|
| 382 |
+
$('examples').addEventListener('click',e=>{const b=e.target.closest('[data-example]');if(b)loadExample(examples[Number(b.dataset.example)]);});
|
| 383 |
+
$('reset').onclick=()=>loadExample(activeExample);
|
| 384 |
+
$('context-mode-routing').onclick=()=>switchContextMode('routing');$('context-mode-single').onclick=()=>switchContextMode('single');$('context-mode-batch').onclick=()=>switchContextMode('batch');
|
| 385 |
+
$('shared-toggle').onclick=()=>{const closed=$('shared-questions').hidden;$('shared-questions').hidden=!closed;$('shared-toggle').setAttribute('aria-expanded',String(closed));};
|
| 386 |
+
$('add-context').onclick=()=>{let n=1;while(draft.contexts.some(c=>c.id==='context_'+n))n++;draft.contexts.push({id:'context_'+n,text:'',expanded:true});renderContexts();dirty();$('context-list').querySelector(`[data-context-text="${draft.contexts.length-1}"]`).focus();};
|
| 387 |
+
$('context-list').addEventListener('click',e=>{const b=e.target.closest('button');if(!b)return;
|
| 388 |
+
if(b.dataset.openContext!==undefined){const i=Number(b.dataset.openContext);draft.contexts[i].expanded=!draft.contexts[i].expanded;renderContexts();$('context-list').querySelector(`[data-open-context="${i}"]`).focus();}
|
| 389 |
+
else if(b.dataset.deleteContext!==undefined){if(draft.contexts.length<=1)return;const i=Number(b.dataset.deleteContext);draft.contexts.splice(i,1);renderContexts();dirty();$('context-list').querySelector(`[data-open-context="${Math.min(i,draft.contexts.length-1)}"]`).focus();}
|
| 390 |
+
});
|
| 391 |
+
$('context-list').addEventListener('input',e=>{const el=e.target;
|
| 392 |
+
if(el.dataset.contextText!==undefined)draft.contexts[Number(el.dataset.contextText)].text=el.value;
|
| 393 |
+
else if(el.dataset.contextId!==undefined){const c=draft.contexts[Number(el.dataset.contextId)];c.id=el.value;el.closest('.context-item').querySelector('.context-row-id').textContent=el.value;}
|
| 394 |
+
else return;dirty();
|
| 395 |
+
});
|
| 396 |
+
function toggleResultRow(row){const detail=$('result-detail-'+row.dataset.resultRow),toggle=row.querySelector('.row-toggle'),open=detail.hidden;detail.hidden=!open;toggle.setAttribute('aria-expanded',String(open));}
|
| 397 |
+
$('results').addEventListener('click',e=>{const row=e.target.closest('[data-result-row]');if(row)toggleResultRow(row);});
|
| 398 |
+
$('state-input').addEventListener('input',()=>{draft.contexts[0].text=$('state-input').value;updateCount();dirty();});
|
| 399 |
+
$('questions').addEventListener('input',e=>{
|
| 400 |
+
const el=e.target,index=Number(el.dataset.index),q=draft.questions[index];if(!q)return;
|
| 401 |
+
if(el.hasAttribute('data-question-id')){q.id=el.value;el.closest('.question-card').querySelector('.question-summary-id').textContent=el.value||'Untitled question';}
|
| 402 |
+
else if(el.hasAttribute('data-instructions')){q.instructions=el.value;el.closest('.question-card').querySelector('.question-summary-text').textContent=el.value;}
|
| 403 |
+
else if(el.dataset.polarity){q.criteria ||= {};if(el.value)q.criteria[el.dataset.polarity]=el.value;else delete q.criteria[el.dataset.polarity];}
|
| 404 |
+
else if(el.dataset.option!==undefined){const i=Number(el.dataset.option);if(q.type==='score')q.criteria[i]=el.value;else{const entries=Object.entries(q.criteria);if(el.dataset.field==='name'){entries[i][0]=el.value;if(new Set(entries.map(x=>x[0])).size!==entries.length){q.invalid='Choice option names must be unique.';dirty();return;}q.invalid=null;}else entries[i][1]=el.value||null;q.criteria=Object.fromEntries(entries);}}
|
| 405 |
+
dirty();
|
| 406 |
+
});
|
| 407 |
+
$('questions').addEventListener('click',e=>{const b=e.target.closest('button');if(!b)return;
|
| 408 |
+
if(b.dataset.toggleQuestion!==undefined){const i=Number(b.dataset.toggleQuestion);draft.questions[i].expanded=!draft.questions[i].expanded;renderQuestions();$('questions').querySelector(`[data-toggle-question="${i}"]`).focus();return;}
|
| 409 |
+
if(b.dataset.remove!==undefined){draft.questions.splice(Number(b.dataset.remove),1);renderQuestions();dirty();}
|
| 410 |
+
if(b.dataset.addOption!==undefined){const q=draft.questions[Number(b.dataset.addOption)];if(q.type==='score')q.criteria.push('Describe this level');else{let i=Object.keys(q.criteria).length+1;while(Object.hasOwn(q.criteria,'Option '+i))i++;q.criteria['Option '+i]=null;}renderQuestions();dirty();}
|
| 411 |
+
if(b.dataset.removeOption!==undefined){const q=draft.questions[Number(b.dataset.index)],i=Number(b.dataset.removeOption);if(q.type==='score')q.criteria.splice(i,1);else{const entries=Object.entries(q.criteria);entries.splice(i,1);q.criteria=Object.fromEntries(entries);}renderQuestions();dirty();}
|
| 412 |
+
});
|
| 413 |
+
$('add-toggle').onclick=()=>{const open=$('add-options').hidden;$('add-options').hidden=!open;$('add-toggle').setAttribute('aria-expanded',String(open));if(open)$('add-options').querySelector('button').focus();};
|
| 414 |
+
$('add-options').onclick=e=>{const button=e.target.closest('[data-add]');if(!button)return;const type=button.dataset.add;let i=draft.questions.length+1;while(draft.questions.some(q=>q.id==='question_'+i))i++;const q={id:'question_'+i,type,expanded:true,instructions:type==='choice'?'Which option best fits the context?':type==='noul'?'Is the statement supported by the context?':'How would you rate the context using these levels?'};if(type==='choice')q.criteria={'Option A':'Describe the first option','Option B':'Describe the second option'};if(type==='score')q.criteria=['Low','Medium','High'];draft.questions.push(q);$('add-options').hidden=true;$('add-toggle').setAttribute('aria-expanded','false');renderQuestions();const added=$('questions').lastElementChild;added.scrollIntoView({block:'nearest'});added.querySelector('.question-id').focus({preventScroll:true});dirty();};
|
| 415 |
+
document.addEventListener('pointerdown',e=>{if(!$('add-options').hidden&&!e.target.closest('.add-menu')){$('add-options').hidden=true;$('add-toggle').setAttribute('aria-expanded','false');}});
|
| 416 |
+
$('run').onclick=run;$('auto-run').onchange=()=>{if($('auto-run').checked)toast('Auto-run sends edits after you pause. A detached request may still finish on the server.');dirty();};
|
| 417 |
+
document.addEventListener('keydown',e=>{if(e.key==='Escape'&&!$('add-options').hidden){$('add-options').hidden=true;$('add-toggle').setAttribute('aria-expanded','false');$('add-toggle').focus();}if((e.metaKey||e.ctrlKey)&&e.key==='Enter'&&!document.querySelector('dialog[open]')){e.preventDefault();if(!$('run').disabled)run();}});
|
| 418 |
+
$('lineup-stage').addEventListener('click',e=>{const b=e.target.closest('[data-model]');if(b){chooseModel(b.dataset.model);$('studio').scrollIntoView({behavior:reducedMotion.matches?'auto':'smooth',block:'start'});}});
|
| 419 |
+
$('model-table').addEventListener('click',e=>{const b=e.target.closest('[data-use-model]');if(!b)return;chooseModel(b.dataset.useModel);$('studio').scrollIntoView({behavior:reducedMotion.matches?'auto':'smooth',block:'start'});});
|
| 420 |
+
$('setup-help').onclick=()=>$('setup-dialog').showModal();$('close-setup').onclick=()=>$('setup-dialog').close();
|
| 421 |
+
for(const dialog of document.querySelectorAll('dialog'))dialog.addEventListener('click',e=>{if(e.target===dialog){const r=dialog.getBoundingClientRect();if(e.clientX<r.left||e.clientX>r.right||e.clientY<r.top||e.clientY>r.bottom)dialog.close();}});
|
| 422 |
+
async function init(){try{
|
| 423 |
+
await loadModels();allExamples=await requestJSON('/api/examples',{cache:'no-store'});
|
| 424 |
+
for(const model of models)studioExamplesForModel(model,allExamples);
|
| 425 |
+
applyWorkspace(modelInfo());modelMenu.identity(modelInfo());
|
| 426 |
+
await checkStatus();
|
| 427 |
+
const poll=async()=>{await checkStatus();statusPoll=setTimeout(poll,3000);};
|
| 428 |
+
statusPoll=setTimeout(poll,3000);
|
| 429 |
+
await refreshMenuStatuses();await settleInitialModel();scheduleFleetRefresh();
|
| 430 |
+
}catch(e){
|
| 431 |
+
if(!initWarningShown)toast(e.message||'Connecting to Studio…');
|
| 432 |
+
initWarningShown=true;
|
| 433 |
+
setConnection('Connecting to Studio…','pending');
|
| 434 |
+
setModelActivity('checking');
|
| 435 |
+
setTimeout(init,initRetryDelay);
|
| 436 |
+
initRetryDelay=Math.min(15000,Math.round(initRetryDelay*1.6));
|
| 437 |
+
}}
|
| 438 |
+
init();
|
static/assets/decision-studio-card.png
ADDED
|
Git LFS Details
|
static/assets/fonts/Geist-Variable.woff2
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:a369fcf5628ea2aa4e1b9e2ec6a5b3624e365bda588e1f0f2f12b564f728fbb8
|
| 3 |
+
size 69652
|
static/assets/fonts/GeistMono-Variable.woff2
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:fba8f577f38a2bbcbe818efa6348dd58f36303a10b8737c42fefad275be563ab
|
| 3 |
+
size 71368
|
static/assets/fonts/Inter-OFL.txt
ADDED
|
@@ -0,0 +1,92 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Copyright (c) 2016 The Inter Project Authors (https://github.com/rsms/inter)
|
| 2 |
+
|
| 3 |
+
This Font Software is licensed under the SIL Open Font License, Version 1.1.
|
| 4 |
+
This license is copied below, and is also available with a FAQ at:
|
| 5 |
+
http://scripts.sil.org/OFL
|
| 6 |
+
|
| 7 |
+
-----------------------------------------------------------
|
| 8 |
+
SIL OPEN FONT LICENSE Version 1.1 - 26 February 2007
|
| 9 |
+
-----------------------------------------------------------
|
| 10 |
+
|
| 11 |
+
PREAMBLE
|
| 12 |
+
The goals of the Open Font License (OFL) are to stimulate worldwide
|
| 13 |
+
development of collaborative font projects, to support the font creation
|
| 14 |
+
efforts of academic and linguistic communities, and to provide a free and
|
| 15 |
+
open framework in which fonts may be shared and improved in partnership
|
| 16 |
+
with others.
|
| 17 |
+
|
| 18 |
+
The OFL allows the licensed fonts to be used, studied, modified and
|
| 19 |
+
redistributed freely as long as they are not sold by themselves. The
|
| 20 |
+
fonts, including any derivative works, can be bundled, embedded,
|
| 21 |
+
redistributed and/or sold with any software provided that any reserved
|
| 22 |
+
names are not used by derivative works. The fonts and derivatives,
|
| 23 |
+
however, cannot be released under any other type of license. The
|
| 24 |
+
requirement for fonts to remain under this license does not apply
|
| 25 |
+
to any document created using the fonts or their derivatives.
|
| 26 |
+
|
| 27 |
+
DEFINITIONS
|
| 28 |
+
"Font Software" refers to the set of files released by the Copyright
|
| 29 |
+
Holder(s) under this license and clearly marked as such. This may
|
| 30 |
+
include source files, build scripts and documentation.
|
| 31 |
+
|
| 32 |
+
"Reserved Font Name" refers to any names specified as such after the
|
| 33 |
+
copyright statement(s).
|
| 34 |
+
|
| 35 |
+
"Original Version" refers to the collection of Font Software components as
|
| 36 |
+
distributed by the Copyright Holder(s).
|
| 37 |
+
|
| 38 |
+
"Modified Version" refers to any derivative made by adding to, deleting,
|
| 39 |
+
or substituting -- in part or in whole -- any of the components of the
|
| 40 |
+
Original Version, by changing formats or by porting the Font Software to a
|
| 41 |
+
new environment.
|
| 42 |
+
|
| 43 |
+
"Author" refers to any designer, engineer, programmer, technical
|
| 44 |
+
writer or other person who contributed to the Font Software.
|
| 45 |
+
|
| 46 |
+
PERMISSION AND CONDITIONS
|
| 47 |
+
Permission is hereby granted, free of charge, to any person obtaining
|
| 48 |
+
a copy of the Font Software, to use, study, copy, merge, embed, modify,
|
| 49 |
+
redistribute, and sell modified and unmodified copies of the Font
|
| 50 |
+
Software, subject to the following conditions:
|
| 51 |
+
|
| 52 |
+
1) Neither the Font Software nor any of its individual components,
|
| 53 |
+
in Original or Modified Versions, may be sold by itself.
|
| 54 |
+
|
| 55 |
+
2) Original or Modified Versions of the Font Software may be bundled,
|
| 56 |
+
redistributed and/or sold with any software, provided that each copy
|
| 57 |
+
contains the above copyright notice and this license. These can be
|
| 58 |
+
included either as stand-alone text files, human-readable headers or
|
| 59 |
+
in the appropriate machine-readable metadata fields within text or
|
| 60 |
+
binary files as long as those fields can be easily viewed by the user.
|
| 61 |
+
|
| 62 |
+
3) No Modified Version of the Font Software may use the Reserved Font
|
| 63 |
+
Name(s) unless explicit written permission is granted by the corresponding
|
| 64 |
+
Copyright Holder. This restriction only applies to the primary font name as
|
| 65 |
+
presented to the users.
|
| 66 |
+
|
| 67 |
+
4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font
|
| 68 |
+
Software shall not be used to promote, endorse or advertise any
|
| 69 |
+
Modified Version, except to acknowledge the contribution(s) of the
|
| 70 |
+
Copyright Holder(s) and the Author(s) or with their explicit written
|
| 71 |
+
permission.
|
| 72 |
+
|
| 73 |
+
5) The Font Software, modified or unmodified, in part or in whole,
|
| 74 |
+
must be distributed entirely under this license, and must not be
|
| 75 |
+
distributed under any other license. The requirement for fonts to
|
| 76 |
+
remain under this license does not apply to any document created
|
| 77 |
+
using the Font Software.
|
| 78 |
+
|
| 79 |
+
TERMINATION
|
| 80 |
+
This license becomes null and void if any of the above conditions are
|
| 81 |
+
not met.
|
| 82 |
+
|
| 83 |
+
DISCLAIMER
|
| 84 |
+
THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
| 85 |
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF
|
| 86 |
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT
|
| 87 |
+
OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE
|
| 88 |
+
COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
| 89 |
+
INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL
|
| 90 |
+
DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
| 91 |
+
FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM
|
| 92 |
+
OTHER DEALINGS IN THE FONT SOFTWARE.
|
static/assets/fonts/InterVariable.woff2
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:b528ca17e4e4851855c555b54ccb403acba20cce42ef0c72d9ce0c3bb2a9f9e2
|
| 3 |
+
size 108756
|
static/assets/fonts/OFL-Geist.txt
ADDED
|
@@ -0,0 +1,92 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Copyright (c) 2023 Vercel, in collaboration with basement.studio
|
| 2 |
+
|
| 3 |
+
This Font Software is licensed under the SIL Open Font License, Version 1.1.
|
| 4 |
+
This license is copied below, and is also available with a FAQ at:
|
| 5 |
+
http://scripts.sil.org/OFL
|
| 6 |
+
|
| 7 |
+
-----------------------------------------------------------
|
| 8 |
+
SIL OPEN FONT LICENSE Version 1.1 - 26 February 2007
|
| 9 |
+
-----------------------------------------------------------
|
| 10 |
+
|
| 11 |
+
PREAMBLE
|
| 12 |
+
The goals of the Open Font License (OFL) are to stimulate worldwide
|
| 13 |
+
development of collaborative font projects, to support the font creation
|
| 14 |
+
efforts of academic and linguistic communities, and to provide a free and
|
| 15 |
+
open framework in which fonts may be shared and improved in partnership
|
| 16 |
+
with others.
|
| 17 |
+
|
| 18 |
+
The OFL allows the licensed fonts to be used, studied, modified and
|
| 19 |
+
redistributed freely as long as they are not sold by themselves. The
|
| 20 |
+
fonts, including any derivative works, can be bundled, embedded,
|
| 21 |
+
redistributed and/or sold with any software provided that any reserved
|
| 22 |
+
names are not used by derivative works. The fonts and derivatives,
|
| 23 |
+
however, cannot be released under any other type of license. The
|
| 24 |
+
requirement for fonts to remain under this license does not apply
|
| 25 |
+
to any document created using the fonts or their derivatives.
|
| 26 |
+
|
| 27 |
+
DEFINITIONS
|
| 28 |
+
"Font Software" refers to the set of files released by the Copyright
|
| 29 |
+
Holder(s) under this license and clearly marked as such. This may
|
| 30 |
+
include source files, build scripts and documentation.
|
| 31 |
+
|
| 32 |
+
"Reserved Font Name" refers to any names specified as such after the
|
| 33 |
+
copyright statement(s).
|
| 34 |
+
|
| 35 |
+
"Original Version" refers to the collection of Font Software components as
|
| 36 |
+
distributed by the Copyright Holder(s).
|
| 37 |
+
|
| 38 |
+
"Modified Version" refers to any derivative made by adding to, deleting,
|
| 39 |
+
or substituting -- in part or in whole -- any of the components of the
|
| 40 |
+
Original Version, by changing formats or by porting the Font Software to a
|
| 41 |
+
new environment.
|
| 42 |
+
|
| 43 |
+
"Author" refers to any designer, engineer, programmer, technical
|
| 44 |
+
writer or other person who contributed to the Font Software.
|
| 45 |
+
|
| 46 |
+
PERMISSION AND CONDITIONS
|
| 47 |
+
Permission is hereby granted, free of charge, to any person obtaining
|
| 48 |
+
a copy of the Font Software, to use, study, copy, merge, embed, modify,
|
| 49 |
+
redistribute, and sell modified and unmodified copies of the Font
|
| 50 |
+
Software, subject to the following conditions:
|
| 51 |
+
|
| 52 |
+
1) Neither the Font Software nor any of its individual components,
|
| 53 |
+
in Original or Modified Versions, may be sold by itself.
|
| 54 |
+
|
| 55 |
+
2) Original or Modified Versions of the Font Software may be bundled,
|
| 56 |
+
redistributed and/or sold with any software, provided that each copy
|
| 57 |
+
contains the above copyright notice and this license. These can be
|
| 58 |
+
included either as stand-alone text files, human-readable headers or
|
| 59 |
+
in the appropriate machine-readable metadata fields within text or
|
| 60 |
+
binary files as long as those fields can be easily viewed by the user.
|
| 61 |
+
|
| 62 |
+
3) No Modified Version of the Font Software may use the Reserved Font
|
| 63 |
+
Name(s) unless explicit written permission is granted by the corresponding
|
| 64 |
+
Copyright Holder. This restriction only applies to the primary font name as
|
| 65 |
+
presented to the users.
|
| 66 |
+
|
| 67 |
+
4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font
|
| 68 |
+
Software shall not be used to promote, endorse or advertise any
|
| 69 |
+
Modified Version, except to acknowledge the contribution(s) of the
|
| 70 |
+
Copyright Holder(s) and the Author(s) or with their explicit written
|
| 71 |
+
permission.
|
| 72 |
+
|
| 73 |
+
5) The Font Software, modified or unmodified, in part or in whole,
|
| 74 |
+
must be distributed entirely under this license, and must not be
|
| 75 |
+
distributed under any other license. The requirement for fonts to
|
| 76 |
+
remain under this license does not apply to any document created
|
| 77 |
+
using the Font Software.
|
| 78 |
+
|
| 79 |
+
TERMINATION
|
| 80 |
+
This license becomes null and void if any of the above conditions are
|
| 81 |
+
not met.
|
| 82 |
+
|
| 83 |
+
DISCLAIMER
|
| 84 |
+
THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
| 85 |
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF
|
| 86 |
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT
|
| 87 |
+
OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE
|
| 88 |
+
COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
| 89 |
+
INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL
|
| 90 |
+
DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
| 91 |
+
FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM
|
| 92 |
+
OTHER DEALINGS IN THE FONT SOFTWARE.
|
static/assets/hero-interstellar.png
ADDED
|
Git LFS Details
|
static/assets/horizon-1080.webp
ADDED
|
Git LFS Details
|
static/assets/horizon.webp
ADDED
|
Git LFS Details
|
static/assets/social-banner.png
ADDED
|
Git LFS Details
|
static/assets/vllm-sr-logo.svg
ADDED
|
|