tvosch commited on
Commit
ef406e9
ยท
1 Parent(s): 358f1d5

parameters added and parakeet tdt

Browse files
README.md CHANGED
@@ -6,10 +6,26 @@ app_file: app.py
6
  pinned: false
7
  license: apache-2.0
8
  tags:
9
- - ASR
 
 
 
10
  - speech-recognition
 
 
 
11
  - Dutch
12
  - leaderboard
 
 
 
 
 
 
 
 
 
 
13
  ---
14
 
15
  # Dutch ASR Leaderboard
 
6
  pinned: false
7
  license: apache-2.0
8
  tags:
9
+ - automatic-speech-recognition
10
+ - hf-asr-leaderboard
11
+ - asr
12
+ - speech
13
  - speech-recognition
14
+ - leaderboard
15
+ - evaluation
16
+ - benchmark
17
  - Dutch
18
  - leaderboard
19
+ models:
20
+ - openai/whisper-large-v3
21
+ - CohereLabs/cohere-transcribe-03-2026
22
+ - Qwen/Qwen3-ASR-1.7B
23
+ - nvidia/parakeet-tdt-0.6b-v3
24
+ language:
25
+ - nl
26
+ metrics:
27
+ - wer
28
+ - rtf
29
  ---
30
 
31
  # Dutch ASR Leaderboard
app.py CHANGED
@@ -48,9 +48,11 @@ def build_dataframe(
48
 
49
  results = r.get("results", {})
50
 
 
51
  row = {
52
  "Model": model_name,
53
  "License": r.get("license", "?"),
 
54
  }
55
 
56
  wer_values = {}
@@ -89,7 +91,7 @@ def build_dataframe(
89
 
90
  df = pd.DataFrame(rows)
91
 
92
- front = [c for c in ["Model", "License", "Average WER"] if c in df.columns]
93
  rest = [c for c in df.columns if c not in front]
94
  df = df[front + rest]
95
 
@@ -110,10 +112,7 @@ def build_dataframe(
110
  def refresh(sort_by: str, show_proprietary: bool, *dataset_flags: bool):
111
  records = load_results()
112
  datasets_visible = dict(zip(DATASET_KEYS, dataset_flags))
113
- df = build_dataframe(records, sort_by, show_proprietary, datasets_visible)
114
- n = len(records)
115
- status = f"{n} model{'s' if n != 1 else ''} in results/"
116
- return df, status
117
 
118
 
119
  ABOUT_TEXT = """
@@ -144,6 +143,13 @@ Local models are benchmarked on **H100 HBM2e GPUs** for consistent performance m
144
  with gr.Blocks(title="Dutch ASR Leaderboard", theme=gr.themes.Default()) as demo:
145
  gr.Markdown("# Dutch ASR Leaderboard")
146
  gr.Markdown("Community leaderboard for Dutch ASR models. Lower WER is better.")
 
 
 
 
 
 
 
147
 
148
  with gr.Tabs():
149
  with gr.Tab("๐Ÿ† Leaderboard"):
@@ -171,10 +177,6 @@ with gr.Blocks(title="Dutch ASR Leaderboard", theme=gr.themes.Default()) as demo
171
  for key, label in DATASET_LABELS.items()
172
  }
173
 
174
- with gr.Column(scale=1):
175
- status_text = gr.Markdown("")
176
- refresh_btn = gr.Button("๐Ÿ”„ Refresh", variant="primary")
177
-
178
  leaderboard_table = gr.Dataframe(
179
  label="Results",
180
  interactive=False,
@@ -188,12 +190,11 @@ with gr.Blocks(title="Dutch ASR Leaderboard", theme=gr.themes.Default()) as demo
188
 
189
  # Initial load and event handlers
190
  _inputs = [sort_by, show_proprietary, *datasets_visible.values()]
191
- demo.load(fn=refresh, inputs=_inputs, outputs=[leaderboard_table, status_text])
192
- sort_by.change(fn=refresh, inputs=_inputs, outputs=[leaderboard_table, status_text])
193
- show_proprietary.change(fn=refresh, inputs=_inputs, outputs=[leaderboard_table, status_text])
194
  for cb in datasets_visible.values():
195
- cb.change(fn=refresh, inputs=_inputs, outputs=[leaderboard_table, status_text])
196
- refresh_btn.click(fn=refresh, inputs=_inputs, outputs=[leaderboard_table, status_text])
197
 
198
  with gr.Tab("โ„น๏ธ About"):
199
  gr.Markdown(ABOUT_TEXT)
 
48
 
49
  results = r.get("results", {})
50
 
51
+ params = r.get("params_billions")
52
  row = {
53
  "Model": model_name,
54
  "License": r.get("license", "?"),
55
+ "Parameters (B)": f"{params}" if params is not None else "โ€”",
56
  }
57
 
58
  wer_values = {}
 
91
 
92
  df = pd.DataFrame(rows)
93
 
94
+ front = [c for c in ["Model", "License", "Parameters (B)", "Average WER"] if c in df.columns]
95
  rest = [c for c in df.columns if c not in front]
96
  df = df[front + rest]
97
 
 
112
  def refresh(sort_by: str, show_proprietary: bool, *dataset_flags: bool):
113
  records = load_results()
114
  datasets_visible = dict(zip(DATASET_KEYS, dataset_flags))
115
+ return build_dataframe(records, sort_by, show_proprietary, datasets_visible)
 
 
 
116
 
117
 
118
  ABOUT_TEXT = """
 
143
  with gr.Blocks(title="Dutch ASR Leaderboard", theme=gr.themes.Default()) as demo:
144
  gr.Markdown("# Dutch ASR Leaderboard")
145
  gr.Markdown("Community leaderboard for Dutch ASR models. Lower WER is better.")
146
+ gr.Markdown(
147
+ "> **Note:** Some models may be benchmaxxed โ€” trained or fine-tuned on data "
148
+ "that overlaps with these test sets. Treat results as indicative, not definitive."
149
+ )
150
+ gr.Markdown(
151
+ "Want to add your model? [Submit on GitHub โ†’](https://github.com/tvosch/Dutch-ASR-leaderboard)"
152
+ )
153
 
154
  with gr.Tabs():
155
  with gr.Tab("๐Ÿ† Leaderboard"):
 
177
  for key, label in DATASET_LABELS.items()
178
  }
179
 
 
 
 
 
180
  leaderboard_table = gr.Dataframe(
181
  label="Results",
182
  interactive=False,
 
190
 
191
  # Initial load and event handlers
192
  _inputs = [sort_by, show_proprietary, *datasets_visible.values()]
193
+ demo.load(fn=refresh, inputs=_inputs, outputs=[leaderboard_table])
194
+ sort_by.change(fn=refresh, inputs=_inputs, outputs=[leaderboard_table])
195
+ show_proprietary.change(fn=refresh, inputs=_inputs, outputs=[leaderboard_table])
196
  for cb in datasets_visible.values():
197
+ cb.change(fn=refresh, inputs=_inputs, outputs=[leaderboard_table])
 
198
 
199
  with gr.Tab("โ„น๏ธ About"):
200
  gr.Markdown(ABOUT_TEXT)
results/CohereLabs__cohere-transcribe-03-2026.json CHANGED
@@ -7,8 +7,9 @@
7
  "backend_used": "vllm",
8
  "normalize": true,
9
  "run_info": {
10
- "vllm_version": "0.19.0",
11
- "package_version": "asr-nl==0.1.0"
 
12
  },
13
  "results": {
14
  "fleurs_nl": {
 
7
  "backend_used": "vllm",
8
  "normalize": true,
9
  "run_info": {
10
+ "api_mode": "transcriptions",
11
+ "vllm_version": "0.19.1",
12
+ "package_version": "0.1.0"
13
  },
14
  "results": {
15
  "fleurs_nl": {
results/Murmel-2026-05-04.json CHANGED
@@ -7,7 +7,8 @@
7
  "backend_used": "api",
8
  "normalize": true,
9
  "run_info": {
10
- "vllm_version": "0.19.1",
 
11
  "package_version": "0.1.0"
12
  },
13
  "results": {
 
7
  "backend_used": "api",
8
  "normalize": true,
9
  "run_info": {
10
+ "api_mode": "murmel",
11
+ "murmel_version": "0.1.0",
12
  "package_version": "0.1.0"
13
  },
14
  "results": {
results/Qwen__Qwen3-ASR-1.7B.json CHANGED
The diff for this file is too large to render. See raw diff
 
results/Reson8.json CHANGED
@@ -7,8 +7,8 @@
7
  "backend_used": "api",
8
  "normalize": true,
9
  "run_info": {
10
- "vllm_version": "0.19.0",
11
- "package_version": "asr-nl==0.1.0"
12
  },
13
  "results": {
14
  "fleurs_nl": {
 
7
  "backend_used": "api",
8
  "normalize": true,
9
  "run_info": {
10
+ "api_mode": "reson8",
11
+ "package_version": "0.1.0"
12
  },
13
  "results": {
14
  "fleurs_nl": {
results/mistralai__Voxtral-Mini-4B-Realtime-2602.json ADDED
The diff for this file is too large to render. See raw diff
 
results/nvidia__parakeet-tdt-0.6b-v3.json ADDED
The diff for this file is too large to render. See raw diff
 
results/openai__whisper-large-v3.json CHANGED
The diff for this file is too large to render. See raw diff