Spaces:
Running
Running
parameters added and parakeet tdt
Browse files- README.md +17 -1
- app.py +15 -14
- results/CohereLabs__cohere-transcribe-03-2026.json +3 -2
- results/Murmel-2026-05-04.json +2 -1
- results/Qwen__Qwen3-ASR-1.7B.json +0 -0
- results/Reson8.json +2 -2
- results/mistralai__Voxtral-Mini-4B-Realtime-2602.json +0 -0
- results/nvidia__parakeet-tdt-0.6b-v3.json +0 -0
- results/openai__whisper-large-v3.json +0 -0
README.md
CHANGED
|
@@ -6,10 +6,26 @@ app_file: app.py
|
|
| 6 |
pinned: false
|
| 7 |
license: apache-2.0
|
| 8 |
tags:
|
| 9 |
-
-
|
|
|
|
|
|
|
|
|
|
| 10 |
- speech-recognition
|
|
|
|
|
|
|
|
|
|
| 11 |
- Dutch
|
| 12 |
- leaderboard
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 13 |
---
|
| 14 |
|
| 15 |
# Dutch ASR Leaderboard
|
|
|
|
| 6 |
pinned: false
|
| 7 |
license: apache-2.0
|
| 8 |
tags:
|
| 9 |
+
- automatic-speech-recognition
|
| 10 |
+
- hf-asr-leaderboard
|
| 11 |
+
- asr
|
| 12 |
+
- speech
|
| 13 |
- speech-recognition
|
| 14 |
+
- leaderboard
|
| 15 |
+
- evaluation
|
| 16 |
+
- benchmark
|
| 17 |
- Dutch
|
| 18 |
- leaderboard
|
| 19 |
+
models:
|
| 20 |
+
- openai/whisper-large-v3
|
| 21 |
+
- CohereLabs/cohere-transcribe-03-2026
|
| 22 |
+
- Qwen/Qwen3-ASR-1.7B
|
| 23 |
+
- nvidia/parakeet-tdt-0.6b-v3
|
| 24 |
+
language:
|
| 25 |
+
- nl
|
| 26 |
+
metrics:
|
| 27 |
+
- wer
|
| 28 |
+
- rtf
|
| 29 |
---
|
| 30 |
|
| 31 |
# Dutch ASR Leaderboard
|
app.py
CHANGED
|
@@ -48,9 +48,11 @@ def build_dataframe(
|
|
| 48 |
|
| 49 |
results = r.get("results", {})
|
| 50 |
|
|
|
|
| 51 |
row = {
|
| 52 |
"Model": model_name,
|
| 53 |
"License": r.get("license", "?"),
|
|
|
|
| 54 |
}
|
| 55 |
|
| 56 |
wer_values = {}
|
|
@@ -89,7 +91,7 @@ def build_dataframe(
|
|
| 89 |
|
| 90 |
df = pd.DataFrame(rows)
|
| 91 |
|
| 92 |
-
front = [c for c in ["Model", "License", "Average WER"] if c in df.columns]
|
| 93 |
rest = [c for c in df.columns if c not in front]
|
| 94 |
df = df[front + rest]
|
| 95 |
|
|
@@ -110,10 +112,7 @@ def build_dataframe(
|
|
| 110 |
def refresh(sort_by: str, show_proprietary: bool, *dataset_flags: bool):
|
| 111 |
records = load_results()
|
| 112 |
datasets_visible = dict(zip(DATASET_KEYS, dataset_flags))
|
| 113 |
-
|
| 114 |
-
n = len(records)
|
| 115 |
-
status = f"{n} model{'s' if n != 1 else ''} in results/"
|
| 116 |
-
return df, status
|
| 117 |
|
| 118 |
|
| 119 |
ABOUT_TEXT = """
|
|
@@ -144,6 +143,13 @@ Local models are benchmarked on **H100 HBM2e GPUs** for consistent performance m
|
|
| 144 |
with gr.Blocks(title="Dutch ASR Leaderboard", theme=gr.themes.Default()) as demo:
|
| 145 |
gr.Markdown("# Dutch ASR Leaderboard")
|
| 146 |
gr.Markdown("Community leaderboard for Dutch ASR models. Lower WER is better.")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 147 |
|
| 148 |
with gr.Tabs():
|
| 149 |
with gr.Tab("๐ Leaderboard"):
|
|
@@ -171,10 +177,6 @@ with gr.Blocks(title="Dutch ASR Leaderboard", theme=gr.themes.Default()) as demo
|
|
| 171 |
for key, label in DATASET_LABELS.items()
|
| 172 |
}
|
| 173 |
|
| 174 |
-
with gr.Column(scale=1):
|
| 175 |
-
status_text = gr.Markdown("")
|
| 176 |
-
refresh_btn = gr.Button("๐ Refresh", variant="primary")
|
| 177 |
-
|
| 178 |
leaderboard_table = gr.Dataframe(
|
| 179 |
label="Results",
|
| 180 |
interactive=False,
|
|
@@ -188,12 +190,11 @@ with gr.Blocks(title="Dutch ASR Leaderboard", theme=gr.themes.Default()) as demo
|
|
| 188 |
|
| 189 |
# Initial load and event handlers
|
| 190 |
_inputs = [sort_by, show_proprietary, *datasets_visible.values()]
|
| 191 |
-
demo.load(fn=refresh, inputs=_inputs, outputs=[leaderboard_table
|
| 192 |
-
sort_by.change(fn=refresh, inputs=_inputs, outputs=[leaderboard_table
|
| 193 |
-
show_proprietary.change(fn=refresh, inputs=_inputs, outputs=[leaderboard_table
|
| 194 |
for cb in datasets_visible.values():
|
| 195 |
-
cb.change(fn=refresh, inputs=_inputs, outputs=[leaderboard_table
|
| 196 |
-
refresh_btn.click(fn=refresh, inputs=_inputs, outputs=[leaderboard_table, status_text])
|
| 197 |
|
| 198 |
with gr.Tab("โน๏ธ About"):
|
| 199 |
gr.Markdown(ABOUT_TEXT)
|
|
|
|
| 48 |
|
| 49 |
results = r.get("results", {})
|
| 50 |
|
| 51 |
+
params = r.get("params_billions")
|
| 52 |
row = {
|
| 53 |
"Model": model_name,
|
| 54 |
"License": r.get("license", "?"),
|
| 55 |
+
"Parameters (B)": f"{params}" if params is not None else "โ",
|
| 56 |
}
|
| 57 |
|
| 58 |
wer_values = {}
|
|
|
|
| 91 |
|
| 92 |
df = pd.DataFrame(rows)
|
| 93 |
|
| 94 |
+
front = [c for c in ["Model", "License", "Parameters (B)", "Average WER"] if c in df.columns]
|
| 95 |
rest = [c for c in df.columns if c not in front]
|
| 96 |
df = df[front + rest]
|
| 97 |
|
|
|
|
| 112 |
def refresh(sort_by: str, show_proprietary: bool, *dataset_flags: bool):
|
| 113 |
records = load_results()
|
| 114 |
datasets_visible = dict(zip(DATASET_KEYS, dataset_flags))
|
| 115 |
+
return build_dataframe(records, sort_by, show_proprietary, datasets_visible)
|
|
|
|
|
|
|
|
|
|
| 116 |
|
| 117 |
|
| 118 |
ABOUT_TEXT = """
|
|
|
|
| 143 |
with gr.Blocks(title="Dutch ASR Leaderboard", theme=gr.themes.Default()) as demo:
|
| 144 |
gr.Markdown("# Dutch ASR Leaderboard")
|
| 145 |
gr.Markdown("Community leaderboard for Dutch ASR models. Lower WER is better.")
|
| 146 |
+
gr.Markdown(
|
| 147 |
+
"> **Note:** Some models may be benchmaxxed โ trained or fine-tuned on data "
|
| 148 |
+
"that overlaps with these test sets. Treat results as indicative, not definitive."
|
| 149 |
+
)
|
| 150 |
+
gr.Markdown(
|
| 151 |
+
"Want to add your model? [Submit on GitHub โ](https://github.com/tvosch/Dutch-ASR-leaderboard)"
|
| 152 |
+
)
|
| 153 |
|
| 154 |
with gr.Tabs():
|
| 155 |
with gr.Tab("๐ Leaderboard"):
|
|
|
|
| 177 |
for key, label in DATASET_LABELS.items()
|
| 178 |
}
|
| 179 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 180 |
leaderboard_table = gr.Dataframe(
|
| 181 |
label="Results",
|
| 182 |
interactive=False,
|
|
|
|
| 190 |
|
| 191 |
# Initial load and event handlers
|
| 192 |
_inputs = [sort_by, show_proprietary, *datasets_visible.values()]
|
| 193 |
+
demo.load(fn=refresh, inputs=_inputs, outputs=[leaderboard_table])
|
| 194 |
+
sort_by.change(fn=refresh, inputs=_inputs, outputs=[leaderboard_table])
|
| 195 |
+
show_proprietary.change(fn=refresh, inputs=_inputs, outputs=[leaderboard_table])
|
| 196 |
for cb in datasets_visible.values():
|
| 197 |
+
cb.change(fn=refresh, inputs=_inputs, outputs=[leaderboard_table])
|
|
|
|
| 198 |
|
| 199 |
with gr.Tab("โน๏ธ About"):
|
| 200 |
gr.Markdown(ABOUT_TEXT)
|
results/CohereLabs__cohere-transcribe-03-2026.json
CHANGED
|
@@ -7,8 +7,9 @@
|
|
| 7 |
"backend_used": "vllm",
|
| 8 |
"normalize": true,
|
| 9 |
"run_info": {
|
| 10 |
-
"
|
| 11 |
-
"
|
|
|
|
| 12 |
},
|
| 13 |
"results": {
|
| 14 |
"fleurs_nl": {
|
|
|
|
| 7 |
"backend_used": "vllm",
|
| 8 |
"normalize": true,
|
| 9 |
"run_info": {
|
| 10 |
+
"api_mode": "transcriptions",
|
| 11 |
+
"vllm_version": "0.19.1",
|
| 12 |
+
"package_version": "0.1.0"
|
| 13 |
},
|
| 14 |
"results": {
|
| 15 |
"fleurs_nl": {
|
results/Murmel-2026-05-04.json
CHANGED
|
@@ -7,7 +7,8 @@
|
|
| 7 |
"backend_used": "api",
|
| 8 |
"normalize": true,
|
| 9 |
"run_info": {
|
| 10 |
-
"
|
|
|
|
| 11 |
"package_version": "0.1.0"
|
| 12 |
},
|
| 13 |
"results": {
|
|
|
|
| 7 |
"backend_used": "api",
|
| 8 |
"normalize": true,
|
| 9 |
"run_info": {
|
| 10 |
+
"api_mode": "murmel",
|
| 11 |
+
"murmel_version": "0.1.0",
|
| 12 |
"package_version": "0.1.0"
|
| 13 |
},
|
| 14 |
"results": {
|
results/Qwen__Qwen3-ASR-1.7B.json
CHANGED
|
The diff for this file is too large to render.
See raw diff
|
|
|
results/Reson8.json
CHANGED
|
@@ -7,8 +7,8 @@
|
|
| 7 |
"backend_used": "api",
|
| 8 |
"normalize": true,
|
| 9 |
"run_info": {
|
| 10 |
-
"
|
| 11 |
-
"package_version": "
|
| 12 |
},
|
| 13 |
"results": {
|
| 14 |
"fleurs_nl": {
|
|
|
|
| 7 |
"backend_used": "api",
|
| 8 |
"normalize": true,
|
| 9 |
"run_info": {
|
| 10 |
+
"api_mode": "reson8",
|
| 11 |
+
"package_version": "0.1.0"
|
| 12 |
},
|
| 13 |
"results": {
|
| 14 |
"fleurs_nl": {
|
results/mistralai__Voxtral-Mini-4B-Realtime-2602.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
results/nvidia__parakeet-tdt-0.6b-v3.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
results/openai__whisper-large-v3.json
CHANGED
|
The diff for this file is too large to render.
See raw diff
|
|
|