jev-review / index.html
Sasha Malahov
block 08: synthetic twin — spec rebuilt from docs alone, seed-0 weights + predict.py live, diff target for src/
df60ab0
Raw History Blame Contribute Delete
30.3 kB
<!doctype html>
<html lang="en" data-theme="light">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Replacing Jev with a local topic classifier — review & open questions</title>
<meta name="description" content="Our read of the spike report, the numbers that matter, and the questions we need answered before switching off Jev.">
<style>
:root {
--bg:#fafaf9; --card:#ffffff; --fg:#1c1917; --muted:#57534e; --faint:#78716c;
--border:rgba(0,0,0,.10); --surface:#f5f5f4; --accent:#4f46e5; --accent-soft:#eef2ff;
--warn:#b45309; --warn-soft:#fffbeb; --warn-border:#f5d08a;
--good:#15803d; --chip:#e7e5e4; --ok:#dcfce7; --ok-fg:#166534;
--code:#f4f4f5;
--shadow:0 1px 2px rgba(0,0,0,.04), 0 8px 24px -16px rgba(0,0,0,.18);
}
[data-theme="dark"] {
--bg:#0c0a09; --card:#1a1817; --fg:#fafaf9; --muted:#a8a29e; --faint:#78716c;
--border:rgba(255,255,255,.12); --surface:#231f1e; --accent:#a5b4fc; --accent-soft:#1e1b3a;
--warn:#fbbf24; --warn-soft:#2a2210; --warn-border:#6b4e12;
--good:#4ade80; --chip:#2b2725; --ok:#10291a; --ok-fg:#86efac;
--code:#231f1e;
--shadow:0 1px 2px rgba(0,0,0,.4), 0 12px 32px -18px rgba(0,0,0,.8);
}
* { box-sizing:border-box; }
html { -webkit-text-size-adjust:100%; }
body {
margin:0; background:var(--bg); color:var(--fg);
font:16px/1.6 ui-sans-serif,-apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,Helvetica,Arial,system-ui,sans-serif;
text-rendering:optimizeLegibility; -webkit-font-smoothing:antialiased;
}
a { color:var(--accent); text-decoration:none; }
a:hover { text-decoration:underline; }
.wrap { max-width:860px; margin:0 auto; padding:0 20px 96px; }
header.top { padding:40px 0 8px; }
.kicker { font-size:12.5px; letter-spacing:.14em; text-transform:uppercase; color:var(--faint); font-weight:600; }
h1 { font-size:30px; line-height:1.18; margin:10px 0 10px; letter-spacing:-.02em; font-weight:700; }
.sub { color:var(--muted); font-size:17px; max-width:64ch; }
.bar { display:flex; flex-wrap:wrap; gap:10px; align-items:center; margin-top:18px; }
.chip { display:inline-flex; align-items:center; gap:7px; font-size:13px; font-weight:600; padding:5px 11px; border-radius:999px; background:var(--chip); color:var(--muted); border:1px solid var(--border); }
.chip.live { background:var(--ok); color:var(--ok-fg); }
.chip .dot { width:8px; height:8px; border-radius:50%; background:currentColor; opacity:.8; }
.spacer { flex:1 1 auto; }
.btn { font-size:13.5px; font-weight:600; padding:7px 13px; border-radius:9px; border:1px solid var(--border); background:var(--card); color:var(--fg); cursor:pointer; display:inline-flex; align-items:center; gap:7px; }
.btn:hover { border-color:var(--accent); }
.btn.primary { background:var(--accent); color:#fff; border-color:transparent; }
.btn.primary:hover { filter:brightness(1.06); text-decoration:none; }
[data-theme="dark"] .btn.primary { color:#0c0a09; }
section { background:var(--card); border:1px solid var(--border); border-radius:16px; padding:26px 26px 28px; margin-top:20px; box-shadow:var(--shadow); }
section > h2 { font-size:20px; margin:0 0 4px; letter-spacing:-.01em; display:flex; align-items:center; gap:10px; }
section > h2 .n { font-size:12px; font-weight:700; color:var(--accent); background:var(--accent-soft); border-radius:7px; padding:3px 8px; letter-spacing:0; }
.lead { color:var(--muted); margin:6px 0 18px; }
p { margin:0 0 14px; }
p:last-child { margin-bottom:0; }
.muted { color:var(--muted); }
code, .mono { font-family:ui-monospace,SFMono-Regular,Menlo,Consolas,"Liberation Mono",monospace; font-size:.9em; background:var(--code); padding:.1em .4em; border-radius:6px; }
strong { font-weight:650; }
/* pipeline */
.pipe { display:flex; flex-wrap:wrap; align-items:stretch; gap:8px; margin:14px 0 4px; }
.step { flex:1 1 120px; border:1px solid var(--border); border-radius:11px; padding:11px 12px; background:var(--surface); font-size:13px; }
.step b { display:block; font-size:13.5px; margin-bottom:2px; }
.step .t { color:var(--faint); font-size:11.5px; }
.step.here { border-color:var(--accent); background:var(--accent-soft); box-shadow:inset 0 0 0 1px var(--accent); }
.arrow { align-self:center; color:var(--faint); font-size:14px; }
/* stat grid */
.stats { display:grid; grid-template-columns:repeat(auto-fit,minmax(150px,1fr)); gap:12px; margin:6px 0 4px; }
.stat { border:1px solid var(--border); border-radius:13px; padding:14px 15px; background:var(--surface); }
.stat .big { font-size:26px; font-weight:750; letter-spacing:-.02em; line-height:1.1; }
.stat .lab { font-size:12.5px; color:var(--muted); margin-top:4px; }
.stat .fine { font-size:11.5px; color:var(--faint); margin-top:2px; }
/* callouts */
.note { border:1px solid var(--border); border-left:4px solid var(--accent); background:var(--surface); border-radius:10px; padding:14px 16px; margin:14px 0; }
.note.warn { border-left-color:var(--warn); background:var(--warn-soft); border-color:var(--warn-border); }
.note .hd { font-weight:700; font-size:14.5px; margin-bottom:4px; display:flex; align-items:center; gap:8px; }
.note p:last-child { margin-bottom:0; }
/* silver/gold */
.tiers { display:grid; grid-template-columns:1fr auto 1fr; gap:14px; align-items:center; margin:16px 0; }
.tier { border:1px solid var(--border); border-radius:13px; padding:16px; background:var(--surface); }
.tier .role { font-size:12px; font-weight:700; letter-spacing:.06em; text-transform:uppercase; color:var(--faint); }
.tier h4 { margin:4px 0 6px; font-size:16px; }
.tier .vol { font-size:13.5px; color:var(--muted); }
.tier.gold { border-color:var(--warn-border); background:var(--warn-soft); }
.tier .pill { display:inline-block; font-size:11.5px; font-weight:600; padding:2px 8px; border-radius:999px; background:var(--chip); margin-top:8px; }
.tier .pill.vol { background:var(--accent-soft); color:var(--accent); }
.tier .pill.tru { background:var(--ok); color:var(--ok-fg); }
@media (max-width:620px){ .tiers { grid-template-columns:1fr; } .tiers .mid { transform:rotate(90deg); } }
/* details */
details { border:1px solid var(--border); border-radius:11px; background:var(--surface); margin:12px 0; overflow:hidden; }
details > summary { cursor:pointer; padding:13px 16px; font-weight:650; font-size:14.5px; list-style:none; display:flex; align-items:center; gap:9px; }
details > summary::-webkit-details-marker { display:none; }
details > summary .tri { transition:transform .15s; color:var(--faint); }
details[open] > summary .tri { transform:rotate(90deg); }
details .body { padding:4px 16px 16px; border-top:1px solid var(--border); }
/* tables */
.tw { overflow-x:auto; margin:6px -4px 4px; }
table { border-collapse:collapse; width:100%; font-size:13.5px; min-width:520px; }
th,td { text-align:left; padding:9px 11px; border-bottom:1px solid var(--border); vertical-align:top; }
th { font-size:11.5px; text-transform:uppercase; letter-spacing:.05em; color:var(--faint); font-weight:700; }
tbody tr:last-child td { border-bottom:none; }
td.num, th.num { text-align:right; font-variant-numeric:tabular-nums; }
tr.pick { background:var(--accent-soft); }
tr.pick td { font-weight:600; }
.tag { font-size:11px; font-weight:700; padding:1px 7px; border-radius:999px; background:var(--chip); color:var(--muted); }
.tag.pick { background:var(--accent); color:#fff; }
[data-theme="dark"] .tag.pick { color:#0c0a09; }
/* spec list */
.spec { display:grid; grid-template-columns:140px 1fr; gap:0; font-size:14px; border:1px solid var(--border); border-radius:11px; overflow:hidden; }
.spec div { padding:10px 14px; border-bottom:1px solid var(--border); }
.spec div:nth-child(2n+1) { background:var(--surface); color:var(--muted); font-weight:600; }
.spec div:nth-last-child(-n+2) { border-bottom:none; }
@media (max-width:560px){ .spec { grid-template-columns:1fr; } .spec div:nth-child(2n+1){ border-bottom:none; } }
/* questions */
ol.q { counter-reset:q; list-style:none; padding:0; margin:0; }
ol.q li { position:relative; padding:14px 16px 14px 52px; border:1px solid var(--border); border-radius:12px; margin-bottom:11px; background:var(--surface); }
ol.q li::before { counter-increment:q; content:counter(q); position:absolute; left:14px; top:14px; width:24px; height:24px; border-radius:7px; background:var(--accent-soft); color:var(--accent); font-weight:700; font-size:13px; display:flex; align-items:center; justify-content:center; }
ol.q li .q-t { font-weight:650; }
ol.q li .why { color:var(--muted); font-size:14px; margin-top:3px; }
/* checklist */
ul.check { list-style:none; padding:0; margin:4px 0 0; }
ul.check li { padding:9px 0; border-bottom:1px dashed var(--border); display:flex; gap:11px; align-items:flex-start; font-size:14.5px; }
ul.check li:last-child { border-bottom:none; }
ul.check input { margin-top:3px; width:17px; height:17px; accent-color:var(--accent); flex:none; cursor:pointer; }
ul.check label { cursor:pointer; }
ul.check input:checked + label { color:var(--faint); text-decoration:line-through; }
.progress { height:7px; border-radius:99px; background:var(--surface); overflow:hidden; margin:12px 0 4px; border:1px solid var(--border); }
.progress > i { display:block; height:100%; width:0; background:var(--accent); transition:width .25s ease; }
.pnote { font-size:12.5px; color:var(--faint); }
/* repo map */
.map { width:100%; min-width:0; }
.map td:first-child { width:46%; }
footer { color:var(--faint); font-size:13px; margin-top:30px; text-align:center; }
footer a { color:var(--muted); text-decoration:underline; }
.refs { display:flex; flex-wrap:wrap; gap:8px; justify-content:center; margin-bottom:12px; }
</style>
</head>
<body>
<div class="wrap">
<header class="top">
<div class="kicker">Review &amp; open questions</div>
<h1>Replacing Jev with a local topic classifier</h1>
<p class="sub">Our read of the spike report, the numbers that actually matter, and the questions we need answered before we switch the digest over from Jev.</p>
<div class="bar">
<span class="chip"><span class="dot"></span>Draft for discussion</span>
<span class="chip">3 Oct 2026</span>
<span class="chip">Claude-drafted → verify in code</span>
<span class="spacer"></span>
<a class="btn primary" href="https://huggingface.co/Kodep/jev-topic-head" target="_blank" rel="noopener">Model repo →</a>
<button class="btn" id="theme" aria-label="Toggle theme">◐</button>
</div>
</header>
<section>
<h2><span class="n">01</span> The job, in one place</h2>
<p class="lead">A family reads a daily digest of school email. Each item gets sorted into a topic; a fixed rules table then decides show / hide. <strong>Only the classifier changes.</strong> The rules don't, so the <em>show-or-hide outcome</em> matters more than the topic name itself.</p>
<div class="pipe">
<div class="step"><b>Email</b><span class="t">school &amp; team messages</span></div>
<div class="arrow">›</div>
<div class="step"><b>Reader</b><span class="t">big model splits into items</span></div>
<div class="arrow">›</div>
<div class="step here"><b>Classifier</b><span class="t">Jev today → our model</span></div>
<div class="arrow">›</div>
<div class="step"><b>Rules table</b><span class="t">fixed show/hide, no AI</span></div>
<div class="arrow">›</div>
<div class="step"><b>Digest</b><span class="t">what the family sees</span></div>
</div>
<p class="muted" style="margin-top:14px">Goal: run the classifier on our own machine — CPU, no GPU pressure (the GPU is shared with the resident text model), no paid fallback. It picks one of 56 topics, plus a weak <em>child</em> and <em>action</em> answer.</p>
</section>
<section>
<h2><span class="n">02</span> The result — and what it measures</h2>
<p class="lead">These are the numbers from the report. Treat them as <strong>reported, not yet reproduced</strong>; we'll recompute them from the pushed code + raw eval.</p>
<div class="stats">
<div class="stat"><div class="big">87.6%</div><div class="lab">Same topic as Jev</div><div class="fine">top-1, test 92/105</div></div>
<div class="stat"><div class="big">0.572</div><div class="lab">Macro-F1</div><div class="fine">each topic weighted equally</div></div>
<div class="stat"><div class="big">90.5%</div><div class="lab">Same show/hide as Jev</div><div class="fine">test 95/105</div></div>
<div class="stat"><div class="big">0 / 0.2s</div><div class="lab">GPU / item</div><div class="fine">2.2 GiB RAM, CPU only</div></div>
</div>
<div class="note warn">
<div class="hd">⚠ Same as Jev ≠ correct</div>
<p>Every score above is measured <strong>against Jev's answer, not against truth.</strong> Jev re-answers the same item identically only <strong>97.1%</strong> of the time, and matched a human label just <strong>11 of 20</strong> on a disagreement set. So 87.6% currently means "agrees with the teacher," not "is right." That's not a criticism of the result — it's the limit of what can be claimed right now.</p>
</div>
</section>
<section>
<h2><span class="n">03</span> Why silver + gold (the part we want to make sure of)</h2>
<p class="lead">We're not just "copying Jev." The point of two label tiers is that <strong>Jev is a black box we don't fully trust</strong>, so we keep <em>volume</em> and <em>trust</em> separate:</p>
<div class="tiers">
<div class="tier silver">
<div class="role">Silver · volume</div>
<h4>Jev's predictions</h4>
<div class="vol">Cheap, abundant, <em>usually</em> right but noisy.</div>
<span class="pill vol">~279 training items</span>
</div>
<div class="mid">+</div>
<div class="tier gold">
<div class="role">Gold · trust</div>
<h4>Human labels</h4>
<div class="vol">Expensive, scarce, human-verified. The only real yardstick.</div>
<span class="pill tru">31 items so far</span>
</div>
</div>
<p><strong>Train on silver, verify and clean with gold.</strong> Gold is what turns "same-as-Jev" into an honest accuracy estimate, and the only way to find where Jev systematically errs — so we don't teach the student its mistakes. Without gold, a student could score 99% "same as Jev" and still be wrong.</p>
<div class="note">
<div class="hd">The crux — two things we need to confirm</div>
<p><strong>1.</strong> Is the silver used <em>raw</em>, or <strong>gold-corrected before training</strong>? If raw, the student still learns Jev's errors and 87.6% is honestly "just agrees with Jev." If gold cleaned the silver, the system beats a pure copy — and that difference is the whole value of the two-tier setup. <strong>Which is it?</strong></p>
<p style="margin-top:8px"><strong>2.</strong> How big is the gold really? <strong>31 across 56 topics is far too small</strong> to estimate accuracy. Is the fresh, unopened <strong>100-item set human gold</strong>, and is it held out from <em>all</em> tuning (not just the test split)?</p>
</div>
</section>
<section>
<h2><span class="n">04</span> The method, as we understand it</h2>
<p class="lead">It is <strong>not</strong> a plain linear probe on frozen vectors — it's LoRA on the encoder plus a prototype head. Correct us anywhere this is off:</p>
<div class="spec">
<div>Base</div><div><code>Qwen/Qwen3-Embedding-0.6B</code> → item vector (~1024-d)</div>
<div>Change</div><div>LoRA on attention + feed-forward · rank 16, α 32, dropout 0.05 · rest frozen</div>
<div>Head</div><div>56 prototype vectors · score = cosine(item, proto) × learned scale (init 20)</div>
<div>Seed head</div><div>Prototypes start from topic-description embeddings (14 topics have no training item)</div>
<div>Loss</div><div>softmax cross-entropy, label smoothing 0.05 · single-label</div>
<div>Optim</div><div>AdamW: LoRA 3e-4 (wd 0.01), head 2e-3 (no wd) · warmup 10% → decay 5%</div>
<div>Train</div><div>batch 16 · ≤12 epochs · early-stop on val macro-F1 · bf16</div>
<div>Balance</div><div>sample ∝ 1/√count · hard pairs ×2 · no augmentation / synthetic</div>
<div>Seeds</div><div>3 seeds; macro-F1 0.49–0.59 across them; shipped = seed 0</div>
<div>Also</div><div>child + action answers — weak (matches Jev 51–65)</div>
</div>
</section>
<section>
<h2><span class="n">05</span> Everything that was tried</h2>
<p class="lead">The shortlist idea from "embedding + reranker" survived into the cascade. The trained head won on accuracy-per-watt. <em>Click to expand.</em></p>
<details>
<summary><span class="tri">▶</span> Full model comparison (18 rows)</summary>
<div class="body">
<div class="tw"><table>
<thead><tr><th>Model / method</th><th class="num">Same topic</th><th>Speed / memory</th><th>Outcome</th></tr></thead>
<tbody>
<tr><td>GLiNER2.5-Decide 340M (shipped)</td><td class="num">24.1%</td><td>~0.5s GPU</td><td>drop — over-picks one topic</td></tr>
<tr><td>GLiNER2.5-Decide 1B</td><td class="num">23.8%</td><td>—</td><td>drop</td></tr>
<tr><td>GLiClass-instruct-large</td><td class="num">1.2%</td><td>—</td><td>drop — scores saturate</td></tr>
<tr><td>BGE-small embeddings, zero-shot</td><td class="num">29.4%</td><td>CPU</td><td>drop</td></tr>
<tr><td>Embedding + reranker</td><td class="num">56–59%</td><td>—</td><td>drop — but shortlist reused</td></tr>
<tr><td>TF-IDF (word-count)</td><td class="num">65–67%</td><td>CPU</td><td>drop</td></tr>
<tr><td>Embedding + simple head</td><td class="num">65–67%</td><td>CPU</td><td>drop</td></tr>
<tr><td>BGE-small, trained</td><td class="num">&lt; Qwen3</td><td>CPU</td><td>drop</td></tr>
<tr><td>Qwen3.5-4B over all 56 topics</td><td class="num">57.5–75.3%</td><td>~7 GiB GPU</td><td>drop alone — child only 45.9%</td></tr>
<tr><td>GLiNER-340M fine-tuned, 10-topic</td><td class="num">81.9%</td><td>3.1 GiB / 1.3s CPU</td><td>not chosen — costs GPU</td></tr>
<tr><td>Cascade (shortlist → Qwen3.5-4B)</td><td class="num">87.6%</td><td>0.41s · 9.3 GiB</td><td><span class="tag">helper</span> too much GPU</td></tr>
<tr class="pick"><td>Qwen3-Embedding-0.6B + trained head</td><td class="num">87.6%</td><td>0.20s CPU · 2.2 GiB · 0 GPU</td><td><span class="tag pick">chosen</span></td></tr>
<tr><td>Clef-flash 9B (Cloudflare)</td><td class="num">84.8%</td><td>1.47s · 8.3 GiB</td><td>drop — slow, drops items</td></tr>
<tr><td>Intern-Decision-4B</td><td class="num">79.0%</td><td>0.8s · 3.7–9 GiB</td><td>parked — not better</td></tr>
<tr><td>Jev-Style 0.8B</td><td class="num">88%*</td><td>0.55s GPU / 7.6s CPU</td><td>drop</td></tr>
<tr><td>Kev-0.8B</td><td class="num">86%*</td><td>0.53s GPU / 10s CPU</td><td>drop</td></tr>
<tr><td>Kev-4B</td><td class="num">—</td><td>46s CPU</td><td>drop — too slow</td></tr>
<tr><td>Open-Jev-DeBERTa</td><td class="num">—</td><td>—</td><td>can't run — 512 vs 843–1923 tokens</td></tr>
</tbody>
</table></div>
<p class="pnote">* scored on a different set with a show/hide measure; the chosen head scored 93% on that same measure. Test set = 105 items, so a 2–3 point gap is 2–3 items — inside the noise.</p>
</div>
</details>
<details>
<summary><span class="tri">▶</span> Finalists, side by side (190 items)</summary>
<div class="body">
<div class="tw"><table>
<thead><tr><th>Metric</th><th class="num">Head</th><th class="num">Cascade</th><th class="num">GLiNER-FT</th><th class="num">Clef</th><th class="num">Intern</th><th class="num">Jev↔Jev</th></tr></thead>
<tbody>
<tr><td>Same topic, val/test</td><td class="num">75.3/87.6</td><td class="num">88.2/87.6</td><td class="num">84.7/81.9</td><td class="num">77.6/84.8</td><td class="num">80.0/79.0</td><td class="num">96.5/97.1</td></tr>
<tr><td>Same show/hide, val/test</td><td class="num">83.5/90.5</td><td class="num">92.9/94.3</td><td class="num">91.8/90.5</td><td class="num">85.9/92.4</td><td class="num">84.7/89.5</td><td class="num">97.7/98.1</td></tr>
<tr><td>Same final result (of 190)</td><td class="num">176</td><td class="num">185</td><td class="num">181</td><td class="num">176</td><td class="num">174</td><td class="num">189</td></tr>
<tr><td>Important items dropped ↓</td><td class="num">7</td><td class="num">3</td><td class="num">5</td><td class="num">11</td><td class="num">8</td><td class="num">0</td></tr>
<tr><td>Time / item ↓</td><td class="num">0.20s</td><td class="num">0.18–0.41s</td><td class="num">0.09s</td><td class="num">1.47s</td><td class="num">0.80s</td><td class="num">—</td></tr>
<tr><td>GPU memory ↓</td><td class="num">0</td><td class="num">9.3</td><td class="num">3.1</td><td class="num">8.3</td><td class="num">9.05</td><td class="num">—</td></tr>
</tbody>
</table></div>
<p class="pnote">GPU has 16 GiB; the resident text model uses 6.7–11.4 GiB of it, so anything ≥9 GiB doesn't fit alongside it. The head fits precisely because it's 0.</p>
</div>
</details>
</section>
<section>
<h2><span class="n">06</span> Questions for Andrei</h2>
<p class="lead">The report is Claude-drafted, so these are mostly "confirm against the real code / data" items.</p>
<ol class="q">
<li><span class="q-t">Silver raw, or gold-corrected?</span><div class="why">Was any Jev label changed by your human gold before training, or is silver used as-is and gold only for reporting? This decides whether the model is a copy of Jev or better.</div></li>
<li><span class="q-t">How big is gold, really?</span><div class="why">31 human labels across 56 topics can't establish accuracy. Is the unopened 100-item set the real gold, and fully held out from tuning?</div></li>
<li><span class="q-t">What is the "94.3% outcome" for the cascade?</span><div class="why">It sits next to 87.6% test accuracy but isn't defined. Different metric? A confident-subset score? We want the exact definition.</div></li>
<li><span class="q-t">The test split was used for decisions.</span><div class="why">The report calls it dev data now, but the headline 87.6% comes from it. Can we quote the val number, or a true held-out number, instead?</div></li>
<li><span class="q-t">The sports-class skew.</span><div class="why">~half the items are sports and almost always right. What's the non-sports agreement number (you cite ~78%) — and does the headline hold with sports down-weighted?</div></li>
<li><span class="q-t">Child + action heads.</span><div class="why">They match Jev only 51–65. Are they in scope for the switch, or topic-only for now?</div></li>
</ol>
</section>
<section>
<h2><span class="n">07</span> What we need to reproduce it</h2>
<p class="lead">Drop these into the model repo — each folder's README has the exact schema. Priority top to bottom.</p>
<div class="tw"><table class="map">
<tbody>
<tr><td><code>src/train_head.py</code> + <code>config.yaml</code></td><td>the actual training code + hyperparameters <span class="tag pick">top</span></td></tr>
<tr><td><code>eval/per_item_test.jsonl</code></td><td>per-item raw output → we recompute 87.6 / 0.572 / 90.5 without re-running</td></tr>
<tr><td><code>src/evaluate.py</code> + <code>rules_table.json</code></td><td>metric definitions + the fixed show/hide rules</td></tr>
<tr><td><code>checkpoints/lora</code> + <code>checkpoints/head</code></td><td>adapter + 56 prototypes (seed 0), with shapes</td></tr>
<tr><td><code>data/*.jsonl</code> + taxonomy + 31 labels</td><td>train/val/test, descriptions, human gold → <a href="https://huggingface.co/datasets/Kodep/jev-topic-data" target="_blank" rel="noopener" style="color:inherit;text-decoration:underline">private data repo</a>, not here</td></tr>
<tr><td><code>requirements.txt</code> + how-to-run</td><td>torch/peft versions, base-model path, seed/config flags</td></tr>
</tbody>
</table></div>
<p style="margin-top:18px"><strong>Verification checklist</strong> — what we'll confirm in the code. Tick as we go (saved in your browser only).</p>
<div class="progress"><i id="pbar"></i></div>
<div class="pnote" id="ptext">0 of 9 confirmed</div>
<ul class="check" id="check">
<li><input type="checkbox" id="c1"><label for="c1">LoRA on attention + FF, rank 16 / α 32 / dropout 0.05; rest frozen</label></li>
<li><input type="checkbox" id="c2"><label for="c2">Head = 56 prototypes, score = cosine × learned scale (init 20)</label></li>
<li><input type="checkbox" id="c3"><label for="c3">Prototypes initialized from topic-description embeddings</label></li>
<li><input type="checkbox" id="c4"><label for="c4">Loss = softmax CE, label smoothing 0.05, single-label</label></li>
<li><input type="checkbox" id="c5"><label for="c5">AdamW 3e-4/2e-3 with warmup 10% → decay 5%</label></li>
<li><input type="checkbox" id="c6"><label for="c6">Batch 16, ≤12 epochs, early-stop on val macro-F1, bf16</label></li>
<li><input type="checkbox" id="c7"><label for="c7">Class-balanced 1/√count, hard pairs ×2, no augmentation</label></li>
<li><input type="checkbox" id="c8"><label for="c8">3 seeds, macro-F1 0.49–0.59 span, shipped = seed 0</label></li>
<li><input type="checkbox" id="c9"><label for="c9">Recomputed 92/105 top-1, macro-F1 0.572, show/hide 95/105</label></li>
</ul>
</section>
<section>
<h2><span class="n">08</span> The synthetic twin — we rebuilt the recipe from your docs alone</h2>
<p class="lead">While waiting for <code>src/</code>, we asked: is the spike reproducible from the report + checklist with zero code? Answer: yes — <a href="https://huggingface.co/Kodep/jev-topic-head-example" target="_blank" rel="noopener" style="color:inherit;text-decoration:underline"><code>Kodep/jev-topic-head-example</code></a> implements the whole thing: LoRA r16/α32/dropout .05 on attention+FFN, 56 prototypes seeded from topic descriptions, scale init 20, CE + smoothing .05, AdamW 3e-4 / 2e-3, warmup 10% → decay 5%, 1/√count sampling with hard pairs ×2, early stop on val macro-F1, 3 seeds.</p>
<div class="tw"><table class="map">
<tbody>
<tr><td>What it proves</td><td>the spec is complete enough to rebuild: pipeline trains, early-stops (~epoch 4), ships a 57,345-param head (56×1024+1 exactly), and reproduces the report's "errors hide in low confidence" pattern</td></tr>
<tr><td>What it does NOT prove</td><td>quality — it trained on <strong>synthetic</strong> data with the real split's shape (279/85/105, 56 topics, 15 unseen); the 0.956 synthetic macro-F1 measures label cleanliness, not brains</td></tr>
<tr><td>Ready-made</td><td>seed-0 weights (39 MB adapter + 230 KB head via LFS — base model auto-downloads, we ship only our delta) + <code>predict.py</code>; <code>hf download</code> and try in two commands</td></tr>
<tr><td>Intended use</td><td>a diff target: when your <code>train_head.py</code> lands, the deltas between our guesses (last-token pooling? bf16 = weights or autocast?) become concrete questions instead of unknowns</td></tr>
</tbody>
</table></div>
<p style="margin-top:18px">The ask is unchanged from block 07: <code>src/</code> + <code>config.yaml</code> first (they answer the silver/gold question directly), then data access — and we rerun <em>this</em> pipeline on real labels to face 92/105 and 0.572 honestly.</p>
</section>
<footer>
<div class="refs">
<a href="https://huggingface.co/Kodep/jev-topic-head" target="_blank" rel="noopener">model repo</a> ·
<a href="https://huggingface.co/Kodep/jev-topic-head-example" target="_blank" rel="noopener">synthetic twin</a> ·
<a href="https://huggingface.co/Kodep/jev-topic-head/blob/main/docs/report_claude_v1.md" target="_blank" rel="noopener">baseline report (in repo)</a> ·
<a href="https://huggingface.co/Qwen/Qwen3-Embedding-0.6B" target="_blank" rel="noopener">base model</a>
</div>
Written by Kodep — a careful reading of the spike report, not a verdict. All numbers are reported-by-report and pending reproduction in code. Open to correction.
</footer>
</div>
<script>
(function(){
// theme
var root=document.documentElement, key='jev-theme';
function set(t){ root.setAttribute('data-theme',t); try{localStorage.setItem(key,t);}catch(e){} }
var saved; try{saved=localStorage.getItem(key);}catch(e){}
set(saved || (matchMedia('(prefers-color-scheme: dark)').matches?'dark':'light'));
document.getElementById('theme').onclick=function(){ set(root.getAttribute('data-theme')==='dark'?'light':'dark'); };
// checklist persistence + progress
var box=document.getElementById('check'), bars=document.getElementById('pbar'), txt=document.getElementById('ptext'), CK='jev-ck';
var states={}; try{states=JSON.parse(localStorage.getItem(CK)||'{}');}catch(e){}
var boxes=[].slice.call(box.querySelectorAll('input[type=checkbox]'));
boxes.forEach(function(b){ if(states[b.id]) b.checked=true; });
function prog(){
var done=boxes.filter(function(b){return b.checked;}).length, n=boxes.length;
bars.style.width=(done/n*100)+'%'; txt.textContent=done+' of '+n+' confirmed';
}
box.addEventListener('change',function(){
var s={}; boxes.forEach(function(b){ if(b.checked) s[b.id]=1; });
try{localStorage.setItem(CK,JSON.stringify(s));}catch(e){}
prog();
});
prog();
})();
</script>
</body>
</html>