Spaces:
Running
Running
Sasha Malahov
block 08: synthetic twin — spec rebuilt from docs alone, seed-0 weights + predict.py live, diff target for src/
df60ab0 Download index.html from Kodep/jev-review: direct link, hf CLI and curl.
- Browser
- Download file 30.3 kB
-
https://huggingface.co/spaces/Kodep/jev-review/resolve/main/index.html
- Command line
-
hf download hf://spaces/Kodep/jev-review/index.html
-
curl -L -o index.html https://huggingface.co/spaces/Kodep/jev-review/resolve/main/index.html
30.3 kB
| <html lang="en" data-theme="light"> | |
| <head> | |
| <meta charset="utf-8"> | |
| <meta name="viewport" content="width=device-width, initial-scale=1"> | |
| <title>Replacing Jev with a local topic classifier — review & open questions</title> | |
| <meta name="description" content="Our read of the spike report, the numbers that matter, and the questions we need answered before switching off Jev."> | |
| <style> | |
| :root { | |
| --bg:#fafaf9; --card:#ffffff; --fg:#1c1917; --muted:#57534e; --faint:#78716c; | |
| --border:rgba(0,0,0,.10); --surface:#f5f5f4; --accent:#4f46e5; --accent-soft:#eef2ff; | |
| --warn:#b45309; --warn-soft:#fffbeb; --warn-border:#f5d08a; | |
| --good:#15803d; --chip:#e7e5e4; --ok:#dcfce7; --ok-fg:#166534; | |
| --code:#f4f4f5; | |
| --shadow:0 1px 2px rgba(0,0,0,.04), 0 8px 24px -16px rgba(0,0,0,.18); | |
| } | |
| [data-theme="dark"] { | |
| --bg:#0c0a09; --card:#1a1817; --fg:#fafaf9; --muted:#a8a29e; --faint:#78716c; | |
| --border:rgba(255,255,255,.12); --surface:#231f1e; --accent:#a5b4fc; --accent-soft:#1e1b3a; | |
| --warn:#fbbf24; --warn-soft:#2a2210; --warn-border:#6b4e12; | |
| --good:#4ade80; --chip:#2b2725; --ok:#10291a; --ok-fg:#86efac; | |
| --code:#231f1e; | |
| --shadow:0 1px 2px rgba(0,0,0,.4), 0 12px 32px -18px rgba(0,0,0,.8); | |
| } | |
| * { box-sizing:border-box; } | |
| html { -webkit-text-size-adjust:100%; } | |
| body { | |
| margin:0; background:var(--bg); color:var(--fg); | |
| font:16px/1.6 ui-sans-serif,-apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,Helvetica,Arial,system-ui,sans-serif; | |
| text-rendering:optimizeLegibility; -webkit-font-smoothing:antialiased; | |
| } | |
| a { color:var(--accent); text-decoration:none; } | |
| a:hover { text-decoration:underline; } | |
| .wrap { max-width:860px; margin:0 auto; padding:0 20px 96px; } | |
| header.top { padding:40px 0 8px; } | |
| .kicker { font-size:12.5px; letter-spacing:.14em; text-transform:uppercase; color:var(--faint); font-weight:600; } | |
| h1 { font-size:30px; line-height:1.18; margin:10px 0 10px; letter-spacing:-.02em; font-weight:700; } | |
| .sub { color:var(--muted); font-size:17px; max-width:64ch; } | |
| .bar { display:flex; flex-wrap:wrap; gap:10px; align-items:center; margin-top:18px; } | |
| .chip { display:inline-flex; align-items:center; gap:7px; font-size:13px; font-weight:600; padding:5px 11px; border-radius:999px; background:var(--chip); color:var(--muted); border:1px solid var(--border); } | |
| .chip.live { background:var(--ok); color:var(--ok-fg); } | |
| .chip .dot { width:8px; height:8px; border-radius:50%; background:currentColor; opacity:.8; } | |
| .spacer { flex:1 1 auto; } | |
| .btn { font-size:13.5px; font-weight:600; padding:7px 13px; border-radius:9px; border:1px solid var(--border); background:var(--card); color:var(--fg); cursor:pointer; display:inline-flex; align-items:center; gap:7px; } | |
| .btn:hover { border-color:var(--accent); } | |
| .btn.primary { background:var(--accent); color:#fff; border-color:transparent; } | |
| .btn.primary:hover { filter:brightness(1.06); text-decoration:none; } | |
| [data-theme="dark"] .btn.primary { color:#0c0a09; } | |
| section { background:var(--card); border:1px solid var(--border); border-radius:16px; padding:26px 26px 28px; margin-top:20px; box-shadow:var(--shadow); } | |
| section > h2 { font-size:20px; margin:0 0 4px; letter-spacing:-.01em; display:flex; align-items:center; gap:10px; } | |
| section > h2 .n { font-size:12px; font-weight:700; color:var(--accent); background:var(--accent-soft); border-radius:7px; padding:3px 8px; letter-spacing:0; } | |
| .lead { color:var(--muted); margin:6px 0 18px; } | |
| p { margin:0 0 14px; } | |
| p:last-child { margin-bottom:0; } | |
| .muted { color:var(--muted); } | |
| code, .mono { font-family:ui-monospace,SFMono-Regular,Menlo,Consolas,"Liberation Mono",monospace; font-size:.9em; background:var(--code); padding:.1em .4em; border-radius:6px; } | |
| strong { font-weight:650; } | |
| /* pipeline */ | |
| .pipe { display:flex; flex-wrap:wrap; align-items:stretch; gap:8px; margin:14px 0 4px; } | |
| .step { flex:1 1 120px; border:1px solid var(--border); border-radius:11px; padding:11px 12px; background:var(--surface); font-size:13px; } | |
| .step b { display:block; font-size:13.5px; margin-bottom:2px; } | |
| .step .t { color:var(--faint); font-size:11.5px; } | |
| .step.here { border-color:var(--accent); background:var(--accent-soft); box-shadow:inset 0 0 0 1px var(--accent); } | |
| .arrow { align-self:center; color:var(--faint); font-size:14px; } | |
| /* stat grid */ | |
| .stats { display:grid; grid-template-columns:repeat(auto-fit,minmax(150px,1fr)); gap:12px; margin:6px 0 4px; } | |
| .stat { border:1px solid var(--border); border-radius:13px; padding:14px 15px; background:var(--surface); } | |
| .stat .big { font-size:26px; font-weight:750; letter-spacing:-.02em; line-height:1.1; } | |
| .stat .lab { font-size:12.5px; color:var(--muted); margin-top:4px; } | |
| .stat .fine { font-size:11.5px; color:var(--faint); margin-top:2px; } | |
| /* callouts */ | |
| .note { border:1px solid var(--border); border-left:4px solid var(--accent); background:var(--surface); border-radius:10px; padding:14px 16px; margin:14px 0; } | |
| .note.warn { border-left-color:var(--warn); background:var(--warn-soft); border-color:var(--warn-border); } | |
| .note .hd { font-weight:700; font-size:14.5px; margin-bottom:4px; display:flex; align-items:center; gap:8px; } | |
| .note p:last-child { margin-bottom:0; } | |
| /* silver/gold */ | |
| .tiers { display:grid; grid-template-columns:1fr auto 1fr; gap:14px; align-items:center; margin:16px 0; } | |
| .tier { border:1px solid var(--border); border-radius:13px; padding:16px; background:var(--surface); } | |
| .tier .role { font-size:12px; font-weight:700; letter-spacing:.06em; text-transform:uppercase; color:var(--faint); } | |
| .tier h4 { margin:4px 0 6px; font-size:16px; } | |
| .tier .vol { font-size:13.5px; color:var(--muted); } | |
| .tier.gold { border-color:var(--warn-border); background:var(--warn-soft); } | |
| .tier .pill { display:inline-block; font-size:11.5px; font-weight:600; padding:2px 8px; border-radius:999px; background:var(--chip); margin-top:8px; } | |
| .tier .pill.vol { background:var(--accent-soft); color:var(--accent); } | |
| .tier .pill.tru { background:var(--ok); color:var(--ok-fg); } | |
| @media (max-width:620px){ .tiers { grid-template-columns:1fr; } .tiers .mid { transform:rotate(90deg); } } | |
| /* details */ | |
| details { border:1px solid var(--border); border-radius:11px; background:var(--surface); margin:12px 0; overflow:hidden; } | |
| details > summary { cursor:pointer; padding:13px 16px; font-weight:650; font-size:14.5px; list-style:none; display:flex; align-items:center; gap:9px; } | |
| details > summary::-webkit-details-marker { display:none; } | |
| details > summary .tri { transition:transform .15s; color:var(--faint); } | |
| details[open] > summary .tri { transform:rotate(90deg); } | |
| details .body { padding:4px 16px 16px; border-top:1px solid var(--border); } | |
| /* tables */ | |
| .tw { overflow-x:auto; margin:6px -4px 4px; } | |
| table { border-collapse:collapse; width:100%; font-size:13.5px; min-width:520px; } | |
| th,td { text-align:left; padding:9px 11px; border-bottom:1px solid var(--border); vertical-align:top; } | |
| th { font-size:11.5px; text-transform:uppercase; letter-spacing:.05em; color:var(--faint); font-weight:700; } | |
| tbody tr:last-child td { border-bottom:none; } | |
| td.num, th.num { text-align:right; font-variant-numeric:tabular-nums; } | |
| tr.pick { background:var(--accent-soft); } | |
| tr.pick td { font-weight:600; } | |
| .tag { font-size:11px; font-weight:700; padding:1px 7px; border-radius:999px; background:var(--chip); color:var(--muted); } | |
| .tag.pick { background:var(--accent); color:#fff; } | |
| [data-theme="dark"] .tag.pick { color:#0c0a09; } | |
| /* spec list */ | |
| .spec { display:grid; grid-template-columns:140px 1fr; gap:0; font-size:14px; border:1px solid var(--border); border-radius:11px; overflow:hidden; } | |
| .spec div { padding:10px 14px; border-bottom:1px solid var(--border); } | |
| .spec div:nth-child(2n+1) { background:var(--surface); color:var(--muted); font-weight:600; } | |
| .spec div:nth-last-child(-n+2) { border-bottom:none; } | |
| @media (max-width:560px){ .spec { grid-template-columns:1fr; } .spec div:nth-child(2n+1){ border-bottom:none; } } | |
| /* questions */ | |
| ol.q { counter-reset:q; list-style:none; padding:0; margin:0; } | |
| ol.q li { position:relative; padding:14px 16px 14px 52px; border:1px solid var(--border); border-radius:12px; margin-bottom:11px; background:var(--surface); } | |
| ol.q li::before { counter-increment:q; content:counter(q); position:absolute; left:14px; top:14px; width:24px; height:24px; border-radius:7px; background:var(--accent-soft); color:var(--accent); font-weight:700; font-size:13px; display:flex; align-items:center; justify-content:center; } | |
| ol.q li .q-t { font-weight:650; } | |
| ol.q li .why { color:var(--muted); font-size:14px; margin-top:3px; } | |
| /* checklist */ | |
| ul.check { list-style:none; padding:0; margin:4px 0 0; } | |
| ul.check li { padding:9px 0; border-bottom:1px dashed var(--border); display:flex; gap:11px; align-items:flex-start; font-size:14.5px; } | |
| ul.check li:last-child { border-bottom:none; } | |
| ul.check input { margin-top:3px; width:17px; height:17px; accent-color:var(--accent); flex:none; cursor:pointer; } | |
| ul.check label { cursor:pointer; } | |
| ul.check input:checked + label { color:var(--faint); text-decoration:line-through; } | |
| .progress { height:7px; border-radius:99px; background:var(--surface); overflow:hidden; margin:12px 0 4px; border:1px solid var(--border); } | |
| .progress > i { display:block; height:100%; width:0; background:var(--accent); transition:width .25s ease; } | |
| .pnote { font-size:12.5px; color:var(--faint); } | |
| /* repo map */ | |
| .map { width:100%; min-width:0; } | |
| .map td:first-child { width:46%; } | |
| footer { color:var(--faint); font-size:13px; margin-top:30px; text-align:center; } | |
| footer a { color:var(--muted); text-decoration:underline; } | |
| .refs { display:flex; flex-wrap:wrap; gap:8px; justify-content:center; margin-bottom:12px; } | |
| </style> | |
| </head> | |
| <body> | |
| <div class="wrap"> | |
| <header class="top"> | |
| <div class="kicker">Review & open questions</div> | |
| <h1>Replacing Jev with a local topic classifier</h1> | |
| <p class="sub">Our read of the spike report, the numbers that actually matter, and the questions we need answered before we switch the digest over from Jev.</p> | |
| <div class="bar"> | |
| <span class="chip"><span class="dot"></span>Draft for discussion</span> | |
| <span class="chip">3 Oct 2026</span> | |
| <span class="chip">Claude-drafted → verify in code</span> | |
| <span class="spacer"></span> | |
| <a class="btn primary" href="https://huggingface.co/Kodep/jev-topic-head" target="_blank" rel="noopener">Model repo →</a> | |
| <button class="btn" id="theme" aria-label="Toggle theme">◐</button> | |
| </div> | |
| </header> | |
| <section> | |
| <h2><span class="n">01</span> The job, in one place</h2> | |
| <p class="lead">A family reads a daily digest of school email. Each item gets sorted into a topic; a fixed rules table then decides show / hide. <strong>Only the classifier changes.</strong> The rules don't, so the <em>show-or-hide outcome</em> matters more than the topic name itself.</p> | |
| <div class="pipe"> | |
| <div class="step"><b>Email</b><span class="t">school & team messages</span></div> | |
| <div class="arrow">›</div> | |
| <div class="step"><b>Reader</b><span class="t">big model splits into items</span></div> | |
| <div class="arrow">›</div> | |
| <div class="step here"><b>Classifier</b><span class="t">Jev today → our model</span></div> | |
| <div class="arrow">›</div> | |
| <div class="step"><b>Rules table</b><span class="t">fixed show/hide, no AI</span></div> | |
| <div class="arrow">›</div> | |
| <div class="step"><b>Digest</b><span class="t">what the family sees</span></div> | |
| </div> | |
| <p class="muted" style="margin-top:14px">Goal: run the classifier on our own machine — CPU, no GPU pressure (the GPU is shared with the resident text model), no paid fallback. It picks one of 56 topics, plus a weak <em>child</em> and <em>action</em> answer.</p> | |
| </section> | |
| <section> | |
| <h2><span class="n">02</span> The result — and what it measures</h2> | |
| <p class="lead">These are the numbers from the report. Treat them as <strong>reported, not yet reproduced</strong>; we'll recompute them from the pushed code + raw eval.</p> | |
| <div class="stats"> | |
| <div class="stat"><div class="big">87.6%</div><div class="lab">Same topic as Jev</div><div class="fine">top-1, test 92/105</div></div> | |
| <div class="stat"><div class="big">0.572</div><div class="lab">Macro-F1</div><div class="fine">each topic weighted equally</div></div> | |
| <div class="stat"><div class="big">90.5%</div><div class="lab">Same show/hide as Jev</div><div class="fine">test 95/105</div></div> | |
| <div class="stat"><div class="big">0 / 0.2s</div><div class="lab">GPU / item</div><div class="fine">2.2 GiB RAM, CPU only</div></div> | |
| </div> | |
| <div class="note warn"> | |
| <div class="hd">⚠ Same as Jev ≠ correct</div> | |
| <p>Every score above is measured <strong>against Jev's answer, not against truth.</strong> Jev re-answers the same item identically only <strong>97.1%</strong> of the time, and matched a human label just <strong>11 of 20</strong> on a disagreement set. So 87.6% currently means "agrees with the teacher," not "is right." That's not a criticism of the result — it's the limit of what can be claimed right now.</p> | |
| </div> | |
| </section> | |
| <section> | |
| <h2><span class="n">03</span> Why silver + gold (the part we want to make sure of)</h2> | |
| <p class="lead">We're not just "copying Jev." The point of two label tiers is that <strong>Jev is a black box we don't fully trust</strong>, so we keep <em>volume</em> and <em>trust</em> separate:</p> | |
| <div class="tiers"> | |
| <div class="tier silver"> | |
| <div class="role">Silver · volume</div> | |
| <h4>Jev's predictions</h4> | |
| <div class="vol">Cheap, abundant, <em>usually</em> right but noisy.</div> | |
| <span class="pill vol">~279 training items</span> | |
| </div> | |
| <div class="mid">+</div> | |
| <div class="tier gold"> | |
| <div class="role">Gold · trust</div> | |
| <h4>Human labels</h4> | |
| <div class="vol">Expensive, scarce, human-verified. The only real yardstick.</div> | |
| <span class="pill tru">31 items so far</span> | |
| </div> | |
| </div> | |
| <p><strong>Train on silver, verify and clean with gold.</strong> Gold is what turns "same-as-Jev" into an honest accuracy estimate, and the only way to find where Jev systematically errs — so we don't teach the student its mistakes. Without gold, a student could score 99% "same as Jev" and still be wrong.</p> | |
| <div class="note"> | |
| <div class="hd">The crux — two things we need to confirm</div> | |
| <p><strong>1.</strong> Is the silver used <em>raw</em>, or <strong>gold-corrected before training</strong>? If raw, the student still learns Jev's errors and 87.6% is honestly "just agrees with Jev." If gold cleaned the silver, the system beats a pure copy — and that difference is the whole value of the two-tier setup. <strong>Which is it?</strong></p> | |
| <p style="margin-top:8px"><strong>2.</strong> How big is the gold really? <strong>31 across 56 topics is far too small</strong> to estimate accuracy. Is the fresh, unopened <strong>100-item set human gold</strong>, and is it held out from <em>all</em> tuning (not just the test split)?</p> | |
| </div> | |
| </section> | |
| <section> | |
| <h2><span class="n">04</span> The method, as we understand it</h2> | |
| <p class="lead">It is <strong>not</strong> a plain linear probe on frozen vectors — it's LoRA on the encoder plus a prototype head. Correct us anywhere this is off:</p> | |
| <div class="spec"> | |
| <div>Base</div><div><code>Qwen/Qwen3-Embedding-0.6B</code> → item vector (~1024-d)</div> | |
| <div>Change</div><div>LoRA on attention + feed-forward · rank 16, α 32, dropout 0.05 · rest frozen</div> | |
| <div>Head</div><div>56 prototype vectors · score = cosine(item, proto) × learned scale (init 20)</div> | |
| <div>Seed head</div><div>Prototypes start from topic-description embeddings (14 topics have no training item)</div> | |
| <div>Loss</div><div>softmax cross-entropy, label smoothing 0.05 · single-label</div> | |
| <div>Optim</div><div>AdamW: LoRA 3e-4 (wd 0.01), head 2e-3 (no wd) · warmup 10% → decay 5%</div> | |
| <div>Train</div><div>batch 16 · ≤12 epochs · early-stop on val macro-F1 · bf16</div> | |
| <div>Balance</div><div>sample ∝ 1/√count · hard pairs ×2 · no augmentation / synthetic</div> | |
| <div>Seeds</div><div>3 seeds; macro-F1 0.49–0.59 across them; shipped = seed 0</div> | |
| <div>Also</div><div>child + action answers — weak (matches Jev 51–65)</div> | |
| </div> | |
| </section> | |
| <section> | |
| <h2><span class="n">05</span> Everything that was tried</h2> | |
| <p class="lead">The shortlist idea from "embedding + reranker" survived into the cascade. The trained head won on accuracy-per-watt. <em>Click to expand.</em></p> | |
| <details> | |
| <summary><span class="tri">▶</span> Full model comparison (18 rows)</summary> | |
| <div class="body"> | |
| <div class="tw"><table> | |
| <thead><tr><th>Model / method</th><th class="num">Same topic</th><th>Speed / memory</th><th>Outcome</th></tr></thead> | |
| <tbody> | |
| <tr><td>GLiNER2.5-Decide 340M (shipped)</td><td class="num">24.1%</td><td>~0.5s GPU</td><td>drop — over-picks one topic</td></tr> | |
| <tr><td>GLiNER2.5-Decide 1B</td><td class="num">23.8%</td><td>—</td><td>drop</td></tr> | |
| <tr><td>GLiClass-instruct-large</td><td class="num">1.2%</td><td>—</td><td>drop — scores saturate</td></tr> | |
| <tr><td>BGE-small embeddings, zero-shot</td><td class="num">29.4%</td><td>CPU</td><td>drop</td></tr> | |
| <tr><td>Embedding + reranker</td><td class="num">56–59%</td><td>—</td><td>drop — but shortlist reused</td></tr> | |
| <tr><td>TF-IDF (word-count)</td><td class="num">65–67%</td><td>CPU</td><td>drop</td></tr> | |
| <tr><td>Embedding + simple head</td><td class="num">65–67%</td><td>CPU</td><td>drop</td></tr> | |
| <tr><td>BGE-small, trained</td><td class="num">< Qwen3</td><td>CPU</td><td>drop</td></tr> | |
| <tr><td>Qwen3.5-4B over all 56 topics</td><td class="num">57.5–75.3%</td><td>~7 GiB GPU</td><td>drop alone — child only 45.9%</td></tr> | |
| <tr><td>GLiNER-340M fine-tuned, 10-topic</td><td class="num">81.9%</td><td>3.1 GiB / 1.3s CPU</td><td>not chosen — costs GPU</td></tr> | |
| <tr><td>Cascade (shortlist → Qwen3.5-4B)</td><td class="num">87.6%</td><td>0.41s · 9.3 GiB</td><td><span class="tag">helper</span> too much GPU</td></tr> | |
| <tr class="pick"><td>Qwen3-Embedding-0.6B + trained head</td><td class="num">87.6%</td><td>0.20s CPU · 2.2 GiB · 0 GPU</td><td><span class="tag pick">chosen</span></td></tr> | |
| <tr><td>Clef-flash 9B (Cloudflare)</td><td class="num">84.8%</td><td>1.47s · 8.3 GiB</td><td>drop — slow, drops items</td></tr> | |
| <tr><td>Intern-Decision-4B</td><td class="num">79.0%</td><td>0.8s · 3.7–9 GiB</td><td>parked — not better</td></tr> | |
| <tr><td>Jev-Style 0.8B</td><td class="num">88%*</td><td>0.55s GPU / 7.6s CPU</td><td>drop</td></tr> | |
| <tr><td>Kev-0.8B</td><td class="num">86%*</td><td>0.53s GPU / 10s CPU</td><td>drop</td></tr> | |
| <tr><td>Kev-4B</td><td class="num">—</td><td>46s CPU</td><td>drop — too slow</td></tr> | |
| <tr><td>Open-Jev-DeBERTa</td><td class="num">—</td><td>—</td><td>can't run — 512 vs 843–1923 tokens</td></tr> | |
| </tbody> | |
| </table></div> | |
| <p class="pnote">* scored on a different set with a show/hide measure; the chosen head scored 93% on that same measure. Test set = 105 items, so a 2–3 point gap is 2–3 items — inside the noise.</p> | |
| </div> | |
| </details> | |
| <details> | |
| <summary><span class="tri">▶</span> Finalists, side by side (190 items)</summary> | |
| <div class="body"> | |
| <div class="tw"><table> | |
| <thead><tr><th>Metric</th><th class="num">Head</th><th class="num">Cascade</th><th class="num">GLiNER-FT</th><th class="num">Clef</th><th class="num">Intern</th><th class="num">Jev↔Jev</th></tr></thead> | |
| <tbody> | |
| <tr><td>Same topic, val/test</td><td class="num">75.3/87.6</td><td class="num">88.2/87.6</td><td class="num">84.7/81.9</td><td class="num">77.6/84.8</td><td class="num">80.0/79.0</td><td class="num">96.5/97.1</td></tr> | |
| <tr><td>Same show/hide, val/test</td><td class="num">83.5/90.5</td><td class="num">92.9/94.3</td><td class="num">91.8/90.5</td><td class="num">85.9/92.4</td><td class="num">84.7/89.5</td><td class="num">97.7/98.1</td></tr> | |
| <tr><td>Same final result (of 190)</td><td class="num">176</td><td class="num">185</td><td class="num">181</td><td class="num">176</td><td class="num">174</td><td class="num">189</td></tr> | |
| <tr><td>Important items dropped ↓</td><td class="num">7</td><td class="num">3</td><td class="num">5</td><td class="num">11</td><td class="num">8</td><td class="num">0</td></tr> | |
| <tr><td>Time / item ↓</td><td class="num">0.20s</td><td class="num">0.18–0.41s</td><td class="num">0.09s</td><td class="num">1.47s</td><td class="num">0.80s</td><td class="num">—</td></tr> | |
| <tr><td>GPU memory ↓</td><td class="num">0</td><td class="num">9.3</td><td class="num">3.1</td><td class="num">8.3</td><td class="num">9.05</td><td class="num">—</td></tr> | |
| </tbody> | |
| </table></div> | |
| <p class="pnote">GPU has 16 GiB; the resident text model uses 6.7–11.4 GiB of it, so anything ≥9 GiB doesn't fit alongside it. The head fits precisely because it's 0.</p> | |
| </div> | |
| </details> | |
| </section> | |
| <section> | |
| <h2><span class="n">06</span> Questions for Andrei</h2> | |
| <p class="lead">The report is Claude-drafted, so these are mostly "confirm against the real code / data" items.</p> | |
| <ol class="q"> | |
| <li><span class="q-t">Silver raw, or gold-corrected?</span><div class="why">Was any Jev label changed by your human gold before training, or is silver used as-is and gold only for reporting? This decides whether the model is a copy of Jev or better.</div></li> | |
| <li><span class="q-t">How big is gold, really?</span><div class="why">31 human labels across 56 topics can't establish accuracy. Is the unopened 100-item set the real gold, and fully held out from tuning?</div></li> | |
| <li><span class="q-t">What is the "94.3% outcome" for the cascade?</span><div class="why">It sits next to 87.6% test accuracy but isn't defined. Different metric? A confident-subset score? We want the exact definition.</div></li> | |
| <li><span class="q-t">The test split was used for decisions.</span><div class="why">The report calls it dev data now, but the headline 87.6% comes from it. Can we quote the val number, or a true held-out number, instead?</div></li> | |
| <li><span class="q-t">The sports-class skew.</span><div class="why">~half the items are sports and almost always right. What's the non-sports agreement number (you cite ~78%) — and does the headline hold with sports down-weighted?</div></li> | |
| <li><span class="q-t">Child + action heads.</span><div class="why">They match Jev only 51–65. Are they in scope for the switch, or topic-only for now?</div></li> | |
| </ol> | |
| </section> | |
| <section> | |
| <h2><span class="n">07</span> What we need to reproduce it</h2> | |
| <p class="lead">Drop these into the model repo — each folder's README has the exact schema. Priority top to bottom.</p> | |
| <div class="tw"><table class="map"> | |
| <tbody> | |
| <tr><td><code>src/train_head.py</code> + <code>config.yaml</code></td><td>the actual training code + hyperparameters <span class="tag pick">top</span></td></tr> | |
| <tr><td><code>eval/per_item_test.jsonl</code></td><td>per-item raw output → we recompute 87.6 / 0.572 / 90.5 without re-running</td></tr> | |
| <tr><td><code>src/evaluate.py</code> + <code>rules_table.json</code></td><td>metric definitions + the fixed show/hide rules</td></tr> | |
| <tr><td><code>checkpoints/lora</code> + <code>checkpoints/head</code></td><td>adapter + 56 prototypes (seed 0), with shapes</td></tr> | |
| <tr><td><code>data/*.jsonl</code> + taxonomy + 31 labels</td><td>train/val/test, descriptions, human gold → <a href="https://huggingface.co/datasets/Kodep/jev-topic-data" target="_blank" rel="noopener" style="color:inherit;text-decoration:underline">private data repo</a>, not here</td></tr> | |
| <tr><td><code>requirements.txt</code> + how-to-run</td><td>torch/peft versions, base-model path, seed/config flags</td></tr> | |
| </tbody> | |
| </table></div> | |
| <p style="margin-top:18px"><strong>Verification checklist</strong> — what we'll confirm in the code. Tick as we go (saved in your browser only).</p> | |
| <div class="progress"><i id="pbar"></i></div> | |
| <div class="pnote" id="ptext">0 of 9 confirmed</div> | |
| <ul class="check" id="check"> | |
| <li><input type="checkbox" id="c1"><label for="c1">LoRA on attention + FF, rank 16 / α 32 / dropout 0.05; rest frozen</label></li> | |
| <li><input type="checkbox" id="c2"><label for="c2">Head = 56 prototypes, score = cosine × learned scale (init 20)</label></li> | |
| <li><input type="checkbox" id="c3"><label for="c3">Prototypes initialized from topic-description embeddings</label></li> | |
| <li><input type="checkbox" id="c4"><label for="c4">Loss = softmax CE, label smoothing 0.05, single-label</label></li> | |
| <li><input type="checkbox" id="c5"><label for="c5">AdamW 3e-4/2e-3 with warmup 10% → decay 5%</label></li> | |
| <li><input type="checkbox" id="c6"><label for="c6">Batch 16, ≤12 epochs, early-stop on val macro-F1, bf16</label></li> | |
| <li><input type="checkbox" id="c7"><label for="c7">Class-balanced 1/√count, hard pairs ×2, no augmentation</label></li> | |
| <li><input type="checkbox" id="c8"><label for="c8">3 seeds, macro-F1 0.49–0.59 span, shipped = seed 0</label></li> | |
| <li><input type="checkbox" id="c9"><label for="c9">Recomputed 92/105 top-1, macro-F1 0.572, show/hide 95/105</label></li> | |
| </ul> | |
| </section> | |
| <section> | |
| <h2><span class="n">08</span> The synthetic twin — we rebuilt the recipe from your docs alone</h2> | |
| <p class="lead">While waiting for <code>src/</code>, we asked: is the spike reproducible from the report + checklist with zero code? Answer: yes — <a href="https://huggingface.co/Kodep/jev-topic-head-example" target="_blank" rel="noopener" style="color:inherit;text-decoration:underline"><code>Kodep/jev-topic-head-example</code></a> implements the whole thing: LoRA r16/α32/dropout .05 on attention+FFN, 56 prototypes seeded from topic descriptions, scale init 20, CE + smoothing .05, AdamW 3e-4 / 2e-3, warmup 10% → decay 5%, 1/√count sampling with hard pairs ×2, early stop on val macro-F1, 3 seeds.</p> | |
| <div class="tw"><table class="map"> | |
| <tbody> | |
| <tr><td>What it proves</td><td>the spec is complete enough to rebuild: pipeline trains, early-stops (~epoch 4), ships a 57,345-param head (56×1024+1 exactly), and reproduces the report's "errors hide in low confidence" pattern</td></tr> | |
| <tr><td>What it does NOT prove</td><td>quality — it trained on <strong>synthetic</strong> data with the real split's shape (279/85/105, 56 topics, 15 unseen); the 0.956 synthetic macro-F1 measures label cleanliness, not brains</td></tr> | |
| <tr><td>Ready-made</td><td>seed-0 weights (39 MB adapter + 230 KB head via LFS — base model auto-downloads, we ship only our delta) + <code>predict.py</code>; <code>hf download</code> and try in two commands</td></tr> | |
| <tr><td>Intended use</td><td>a diff target: when your <code>train_head.py</code> lands, the deltas between our guesses (last-token pooling? bf16 = weights or autocast?) become concrete questions instead of unknowns</td></tr> | |
| </tbody> | |
| </table></div> | |
| <p style="margin-top:18px">The ask is unchanged from block 07: <code>src/</code> + <code>config.yaml</code> first (they answer the silver/gold question directly), then data access — and we rerun <em>this</em> pipeline on real labels to face 92/105 and 0.572 honestly.</p> | |
| </section> | |
| <footer> | |
| <div class="refs"> | |
| <a href="https://huggingface.co/Kodep/jev-topic-head" target="_blank" rel="noopener">model repo</a> · | |
| <a href="https://huggingface.co/Kodep/jev-topic-head-example" target="_blank" rel="noopener">synthetic twin</a> · | |
| <a href="https://huggingface.co/Kodep/jev-topic-head/blob/main/docs/report_claude_v1.md" target="_blank" rel="noopener">baseline report (in repo)</a> · | |
| <a href="https://huggingface.co/Qwen/Qwen3-Embedding-0.6B" target="_blank" rel="noopener">base model</a> | |
| </div> | |
| Written by Kodep — a careful reading of the spike report, not a verdict. All numbers are reported-by-report and pending reproduction in code. Open to correction. | |
| </footer> | |
| </div> | |
| <script> | |
| (function(){ | |
| // theme | |
| var root=document.documentElement, key='jev-theme'; | |
| function set(t){ root.setAttribute('data-theme',t); try{localStorage.setItem(key,t);}catch(e){} } | |
| var saved; try{saved=localStorage.getItem(key);}catch(e){} | |
| set(saved || (matchMedia('(prefers-color-scheme: dark)').matches?'dark':'light')); | |
| document.getElementById('theme').onclick=function(){ set(root.getAttribute('data-theme')==='dark'?'light':'dark'); }; | |
| // checklist persistence + progress | |
| var box=document.getElementById('check'), bars=document.getElementById('pbar'), txt=document.getElementById('ptext'), CK='jev-ck'; | |
| var states={}; try{states=JSON.parse(localStorage.getItem(CK)||'{}');}catch(e){} | |
| var boxes=[].slice.call(box.querySelectorAll('input[type=checkbox]')); | |
| boxes.forEach(function(b){ if(states[b.id]) b.checked=true; }); | |
| function prog(){ | |
| var done=boxes.filter(function(b){return b.checked;}).length, n=boxes.length; | |
| bars.style.width=(done/n*100)+'%'; txt.textContent=done+' of '+n+' confirmed'; | |
| } | |
| box.addEventListener('change',function(){ | |
| var s={}; boxes.forEach(function(b){ if(b.checked) s[b.id]=1; }); | |
| try{localStorage.setItem(CK,JSON.stringify(s));}catch(e){} | |
| prog(); | |
| }); | |
| prog(); | |
| })(); | |
| </script> | |
| </body> | |
| </html> | |