Reduce reader brain Modal memory use

Co-authored-by: Codex <codex@openai.com>

DEPLOY_SPACES.md

@@ -107,7 +107,8 @@ llama-server \
   --alias narrator-brain \
   --host 0.0.0.0 \
   --port 8080 \
-  --ctx-size 0 \
+  --ctx-size 4096 \
+  --parallel 1 \
   --reasoning off \
   --n-gpu-layers 999 \
   --api-key your-random-token
@@ -248,7 +249,7 @@ curl https://<YOUR_USERNAME>-tiny-narrator.hf.space/api/submission-readiness
 ### Modal reader-brain cold starts are slow
 
 - Keep `scaledown_window` higher in `modal_workers/reader_brain.py` if you want the container to stay warm longer.
-- Use a faster GPU by setting `READER_BRAIN_MODAL_GPU` before deploying.
+- Use a larger GPU by editing `gpu="T4"` in `modal_workers/reader_brain.py` before deploying.
 - Keep `min_containers=0` for cheapest operation; use a warm container only if you accept continuous cost.
 
 ### External services unreachable

README.md

@@ -49,7 +49,7 @@ Copy-Item .env.example .env
 Start the llama.cpp reader-brain server locally:
 
 ```powershell
-llama-server -hf nvidia/NVIDIA-Nemotron-3-Nano-4B-GGUF:Q4_K_M --alias narrator-brain --port 8080 --host 0.0.0.0 --ctx-size 0 --reasoning off --n-gpu-layers 999
+llama-server -hf nvidia/NVIDIA-Nemotron-3-Nano-4B-GGUF:Q4_K_M --alias narrator-brain --port 8080 --host 0.0.0.0 --ctx-size 4096 --parallel 1 --reasoning off --n-gpu-layers 999
 ```
 
 Start the app:
@@ -77,7 +77,7 @@ LLAMA_CPP_MODEL=narrator-brain
 LLAMA_CPP_TOKEN=your-random-token
 ```
 
-The Modal worker starts Nemotron with `--ctx-size 0`, `--reasoning off`, full GPU offload, and `--api-key` when `LLAMA_CPP_TOKEN` is configured. It uses the prebuilt `ghcr.io/ggml-org/llama.cpp:server-cuda12` image instead of compiling llama.cpp during deploy, clears that image's entrypoint so Modal can start its Python runner, and allows up to 10 minutes for the first GGUF download/load. It scales down when idle, so the first request after a cold start can be slower.
+The Modal worker starts Nemotron on T4 with `--ctx-size 4096`, `--parallel 1`, `--reasoning off`, full GPU offload, and `--api-key` when `LLAMA_CPP_TOKEN` is configured. It uses the prebuilt `ghcr.io/ggml-org/llama.cpp:server-cuda12` image instead of compiling llama.cpp during deploy, clears that image's entrypoint so Modal can start its Python runner, and allows up to 10 minutes for the first GGUF download/load. It scales down when idle, so the first request after a cold start can be slower.
 
 ## Modal Klein Image Generation
 

app.py

@@ -218,7 +218,7 @@ def runtime_setup_core() -> dict[str, Any]:
                 "command": (
                     "llama-server -hf nvidia/NVIDIA-Nemotron-3-Nano-4B-GGUF:Q4_K_M "
                     "--alias narrator-brain --port 8080 --host 0.0.0.0 "
-                    "--ctx-size 0 --reasoning off --n-gpu-layers 999"
+                    "--ctx-size 4096 --parallel 1 --reasoning off --n-gpu-layers 999"
                 ),
                 "modal_command": "modal deploy modal_workers/reader_brain.py",
                 "env": {

modal_workers/reader_brain.py

@@ -71,7 +71,7 @@ reader_brain_image = (
 
 @app.function(
     image=reader_brain_image,
-    gpu=os.getenv("READER_BRAIN_MODAL_GPU", "T4"),
+    gpu="T4",
     volumes={CACHE_DIR: model_cache},
     secrets=_secret_names(),
     timeout=900,
@@ -81,10 +81,6 @@ reader_brain_image = (
 @modal.concurrent(max_inputs=20)
 @modal.web_server(SERVER_PORT, startup_timeout=600)
 def reader_brain_server():
-    model_ref = os.getenv("READER_BRAIN_MODEL_REF", MODEL_REF)
-    model_alias = os.getenv("READER_BRAIN_MODEL_ALIAS", MODEL_ALIAS)
-    context_size = os.getenv("READER_BRAIN_CTX_SIZE", "0")
-    gpu_layers = os.getenv("READER_BRAIN_GPU_LAYERS", "999")
     api_key = os.getenv("LLAMA_CPP_TOKEN", "")
 
     command = [
@@ -94,15 +90,17 @@ def reader_brain_server():
         "--port",
         str(SERVER_PORT),
         "-hf",
-        model_ref,
+        MODEL_REF,
         "--alias",
-        model_alias,
+        MODEL_ALIAS,
         "--ctx-size",
-        context_size,
+        "4096",
+        "--parallel",
+        "1",
         "--reasoning",
         "off",
         "--n-gpu-layers",
-        gpu_layers,
+        "999",
     ]
     if api_key:
         command.extend(["--api-key", api_key])

scripts/verify.py

@@ -483,7 +483,8 @@ def verify_modal_klein_integration() -> None:
     assert_true("cmake --build" not in reader_worker_source, "Reader-brain worker should avoid slow CMake builds on Modal")
     assert_true("NVIDIA-Nemotron-3-Nano-4B-GGUF:Q4_K_M" in reader_worker_source, "Reader-brain worker should serve the Nemotron GGUF")
     assert_true('"--reasoning"' in reader_worker_source and '"off"' in reader_worker_source, "Reader-brain worker should disable reasoning mode")
-    assert_true('"--ctx-size"' in reader_worker_source and '"0"' in reader_worker_source, "Reader-brain worker should use model context size")
+    assert_true('"--ctx-size"' in reader_worker_source and '"4096"' in reader_worker_source, "Reader-brain worker should use T4-safe context size")
+    assert_true('"--parallel"' in reader_worker_source and '"1"' in reader_worker_source, "Reader-brain worker should use one parallel slot on T4")
     assert_true('"--n-gpu-layers"' in reader_worker_source and '"999"' in reader_worker_source, "Reader-brain worker should request full GPU offload")
     assert_true("modal.Volume" in reader_worker_source, "Reader-brain worker should cache model downloads in a Modal volume")
     assert_true("tiny-narrator-reader-brain-token" in reader_worker_source, "Reader-brain worker should use a fixed Modal token secret")