Upload all models and assets for ang (latest)

.gitattributes

@@ -39,3 +39,4 @@ visualizations/position_encoding_comparison.png filter=lfs diff=lfs merge=lfs -t
 visualizations/tsne_sentences.png filter=lfs diff=lfs merge=lfs -text
 visualizations/tsne_words.png filter=lfs diff=lfs merge=lfs -text
 visualizations/zipf_law.png filter=lfs diff=lfs merge=lfs -text
+visualizations/embedding_tsne_multilingual.png filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,6 +1,6 @@
 ---
 language: ang
-language_name: ANG
+language_name: Old English
 language_family: germanic_historical
 tags:
   - wikilangs
@@ -10,11 +10,21 @@ tags:
   - n-gram
   - markov
   - wikipedia
+  - feature-extraction
+  - sentence-similarity
+  - tokenization
+  - n-grams
+  - markov-chain
+  - text-mining
+  - fasttext
+  - babelvec
+  - vocabulous
+  - vocabulary
   - monolingual
   - family-germanic_historical
 license: mit
 library_name: wikilangs
-pipeline_tag: feature-extraction
+pipeline_tag: text-generation
 datasets:
   - omarkamali/wikipedia-monthly
 dataset_info:
@@ -23,20 +33,20 @@ dataset_info:
 metrics:
   - name: best_compression_ratio
     type: compression
-    value: 4.021
+    value: 4.012
   - name: best_isotropy
     type: isotropy
-    value: 0.7825
+    value: 0.8005
   - name: vocabulary_size
     type: vocab
     value: 0
 generated: 2026-01-03
 ---
 
-# ANG - Wikilangs Models
+# Old English - Wikilangs Models
 ## Comprehensive Research Report & Full Ablation Study
 
-This repository contains NLP models trained and evaluated by Wikilangs, specifically on **ANG** Wikipedia data.
+This repository contains NLP models trained and evaluated by Wikilangs, specifically on **Old English** Wikipedia data.
 We analyze tokenizers, n-gram models, Markov chains, vocabulary statistics, and word embeddings.
 
 ## 📋 Repository Contents
@@ -60,7 +70,7 @@ We analyze tokenizers, n-gram models, Markov chains, vocabulary statistics, and
 - [3. Markov Chain Evaluation](#3-markov-chain-evaluation)
 - [4. Vocabulary Analysis](#4-vocabulary-analysis)
 - [5. Word Embeddings Evaluation](#5-word-embeddings-evaluation)
-- [6. Morphological Analysis (Experimental)](#6-morphological-analysis)
+- [6. Morphological Analysis (Experimental)](#6--morphological-analysis-experimental)
 - [7. Summary & Recommendations](#7-summary--recommendations)
 - [Metrics Glossary](#appendix-metrics-glossary--interpretation-guide)
 - [Visualizations Index](#visualizations-index)
@@ -80,47 +90,47 @@ We analyze tokenizers, n-gram models, Markov chains, vocabulary statistics, and
 
 | Vocab Size | Compression | Avg Token Len | UNK Rate | Total Tokens |
 |------------|-------------|---------------|----------|--------------|
-| **8k** | 3.112x | 3.12 | 0.0790% | 253,185 |
-| **16k** | 3.447x | 3.45 | 0.0875% | 228,585 |
-| **32k** | 3.771x | 3.78 | 0.0957% | 208,909 |
-| **64k** | 4.021x 🏆 | 4.03 | 0.1021% | 195,954 |
+| **8k** | 3.107x | 3.11 | 0.0859% | 252,634 |
+| **16k** | 3.441x | 3.45 | 0.0951% | 228,129 |
+| **32k** | 3.763x | 3.77 | 0.1040% | 208,636 |
+| **64k** | 4.012x 🏆 | 4.02 | 0.1109% | 195,650 |
 
 ### Tokenization Examples
 
 Below are sample sentences tokenized with each vocabulary size:
 
-**Sample 1:** `Ƿada (tacn: 16px|♆) is þæt eahtoþa planēta þǣre sunnlican endebyrdnesse. tungol`
+**Sample 1:** `Valladolid is burg on Spēnum. Valladolid hæfþ 319,943 lēoda. on Castillan`
 
 | Vocab | Tokens | Count |
 |-------|--------|-------|
-| 8k | `▁ƿa da ▁( tac n : ▁ 1 6 px ... (+13 more)` | 23 |
-| 16k | `▁ƿa da ▁( tacn : ▁ 1 6 px | ... (+12 more)` | 22 |
-| 32k | `▁ƿada ▁( tacn : ▁ 1 6 px | ♆ ... (+10 more)` | 20 |
-| 64k | `▁ƿada ▁( tacn : ▁ 1 6 px | ♆ ... (+10 more)` | 20 |
+| 8k | `▁val lad ol id ▁is ▁burg ▁on ▁spēnum . ▁val ... (+18 more)` | 28 |
+| 16k | `▁val lad ol id ▁is ▁burg ▁on ▁spēnum . ▁val ... (+17 more)` | 27 |
+| 32k | `▁val ladol id ▁is ▁burg ▁on ▁spēnum . ▁val ladol ... (+14 more)` | 24 |
+| 64k | `▁valladolid ▁is ▁burg ▁on ▁spēnum . ▁valladolid ▁hæfþ ▁ 3 ... (+10 more)` | 20 |
 
-**Sample 2:** `Caþerine, Wēala Þēodienen, (ġeboren Caþerine Elisabeþ Middeltūn; 9 Æfterra Gēola...`
+**Sample 2:** `Cicġan () is burg on Cile. Þǣr oneardiaþ 161,953 lēoda (þæs gēares). His mearc i...`
 
 | Vocab | Tokens | Count |
 |-------|--------|-------|
-| 8k | `▁caþ er ine , ▁wēala ▁þēod ien en , ▁( ... (+28 more)` | 38 |
-| 16k | `▁caþerine , ▁wēala ▁þēod ien en , ▁( ġe boren ... (+24 more)` | 34 |
-| 32k | `▁caþerine , ▁wēala ▁þēodienen , ▁( ġeboren ▁caþerine ▁elisabeþ ▁middeltūn ... (+18 more)` | 28 |
-| 64k | `▁caþerine , ▁wēala ▁þēodienen , ▁( ġeboren ▁caþerine ▁elisabeþ ▁middeltūn ... (+18 more)` | 28 |
+| 8k | `▁c ic ġ an ▁() ▁is ▁burg ▁on ▁cile . ... (+33 more)` | 43 |
+| 16k | `▁cic ġan ▁() ▁is ▁burg ▁on ▁cile . ▁þǣr ▁oneardiaþ ... (+31 more)` | 41 |
+| 32k | `▁cic ġan ▁() ▁is ▁burg ▁on ▁cile . ▁þǣr ▁oneardiaþ ... (+30 more)` | 40 |
+| 64k | `▁cicġan ▁() ▁is ▁burg ▁on ▁cile . ▁þǣr ▁oneardiaþ ▁ ... (+28 more)` | 38 |
 
-**Sample 3:** `Seo burg Hƿītburg ( oþþe Belgrade) oþþe Singidceaster is sēo hēafodburg and sēo ...`
+**Sample 3:** `Welwīc () is þorp in þæm East Þriding, se is Eoferƿicscire dǣl, on Englum. Heo h...`
 
 | Vocab | Tokens | Count |
 |-------|--------|-------|
-| 8k | `▁seo ▁burg ▁hƿīt burg ▁( ▁oþþe ▁belg ra de ) ... (+20 more)` | 30 |
-| 16k | `▁seo ▁burg ▁hƿīt burg ▁( ▁oþþe ▁belg rade ) ▁oþþe ... (+19 more)` | 29 |
-| 32k | `▁seo ▁burg ▁hƿīt burg ▁( ▁oþþe ▁belgrade ) ▁oþþe ▁sing ... (+17 more)` | 27 |
-| 64k | `▁seo ▁burg ▁hƿītburg ▁( ▁oþþe ▁belgrade ) ▁oþþe ▁sing id ... (+16 more)` | 26 |
+| 8k | `▁wel wīc ▁() ▁is ▁þorp ▁in ▁þæm ▁east ▁þriding , ... (+21 more)` | 31 |
+| 16k | `▁wel wīc ▁() ▁is ▁þorp ▁in ▁þæm ▁east ▁þriding , ... (+21 more)` | 31 |
+| 32k | `▁wel wīc ▁() ▁is ▁þorp ▁in ▁þæm ▁east ▁þriding , ... (+21 more)` | 31 |
+| 64k | `▁welwīc ▁() ▁is ▁þorp ▁in ▁þæm ▁east ▁þriding , ▁se ... (+20 more)` | 30 |
 
 
 ### Key Findings
 
-- **Best Compression:** 64k achieves 4.021x compression
-- **Lowest UNK Rate:** 8k with 0.0790% unknown tokens
+- **Best Compression:** 64k achieves 4.012x compression
+- **Lowest UNK Rate:** 8k with 0.0859% unknown tokens
 - **Trade-off:** Larger vocabularies improve compression but increase model size
 - **Recommendation:** 32k vocabulary provides optimal balance for production use
 
@@ -137,12 +147,14 @@ Below are sample sentences tokenized with each vocabulary size:
 
 | N-gram | Variant | Perplexity | Entropy | Unique N-grams | Top-100 Coverage | Top-1000 Coverage |
 |--------|---------|------------|---------|----------------|------------------|-------------------|
-| **2-gram** | Word | 3,511 | 11.78 | 7,045 | 21.4% | 53.3% |
-| **2-gram** | Subword | 365 🏆 | 8.51 | 3,016 | 61.0% | 98.1% |
-| **3-gram** | Word | 3,285 | 11.68 | 6,002 | 21.8% | 50.6% |
-| **3-gram** | Subword | 3,330 | 11.70 | 23,727 | 22.3% | 62.8% |
-| **4-gram** | Word | 6,683 | 12.71 | 11,447 | 16.8% | 36.4% |
-| **4-gram** | Subword | 18,648 | 14.19 | 105,485 | 10.6% | 32.7% |
+| **2-gram** | Word | 3,551 | 11.79 | 7,095 | 21.2% | 53.1% |
+| **2-gram** | Subword | 365 🏆 | 8.51 | 3,006 | 61.0% | 98.1% |
+| **3-gram** | Word | 3,411 | 11.74 | 6,128 | 21.1% | 50.1% |
+| **3-gram** | Subword | 3,332 | 11.70 | 23,711 | 22.3% | 62.8% |
+| **4-gram** | Word | 6,747 | 12.72 | 11,452 | 16.3% | 36.7% |
+| **4-gram** | Subword | 18,651 | 14.19 | 105,677 | 10.6% | 32.7% |
+| **5-gram** | Word | 4,718 | 12.20 | 8,067 | 18.6% | 41.3% |
+| **5-gram** | Subword | 56,790 | 15.79 | 217,768 | 6.4% | 20.2% |
 
 ### Top 5 N-grams by Size
 
@@ -150,21 +162,21 @@ Below are sample sentences tokenized with each vocabulary size:
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `in þǣm` | 768 |
-| 2 | `on þǣm` | 759 |
-| 3 | `in þæm` | 693 |
-| 4 | `of the` | 648 |
-| 5 | `se is` | 547 |
+| 1 | `on þǣm` | 784 |
+| 2 | `in þǣm` | 762 |
+| 3 | `in þæm` | 673 |
+| 4 | `of the` | 645 |
+| 5 | `se is` | 536 |
 
 **3-grams (Word):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
 | 1 | `td valign top` | 529 |
-| 2 | `is þorp in` | 313 |
-| 3 | `þæs geānedan cynerīces` | 312 |
-| 4 | `eoferwicscīre þæs geānedan` | 248 |
-| 5 | `on eoferwicscīre þæs` | 248 |
+| 2 | `þæs geānedan cynerīces` | 312 |
+| 3 | `is þorp in` | 311 |
+| 4 | `on eoferwicscīre þæs` | 248 |
+| 5 | `eoferwicscīre þæs geānedan` | 248 |
 
 **4-grams (Word):**
 
@@ -172,46 +184,66 @@ Below are sample sentences tokenized with each vocabulary size:
 |------|--------|-------|
 | 1 | `on eoferwicscīre þæs geānedan` | 248 |
 | 2 | `eoferwicscīre þæs geānedan cynerīces` | 248 |
-| 3 | `is eoferƿicscire dǣl on` | 244 |
-| 4 | `eoferƿicscire dǣl on englum` | 244 |
-| 5 | `se is eoferƿicscire dǣl` | 242 |
+| 3 | `is eoferƿicscire dǣl on` | 232 |
+| 4 | `eoferƿicscire dǣl on englum` | 231 |
+| 5 | `se is eoferƿicscire dǣl` | 229 |
+
+**5-grams (Word):**
+
+| Rank | N-gram | Count |
+|------|--------|-------|
+| 1 | `on eoferwicscīre þæs geānedan cynerīces` | 248 |
+| 2 | `is eoferƿicscire dǣl on englum` | 231 |
+| 3 | `se is eoferƿicscire dǣl on` | 229 |
+| 4 | `þriding se is eoferƿicscire dǣl` | 224 |
+| 5 | `east þriding se is eoferƿicscire` | 170 |
 
 **2-grams (Subword):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `e _` | 68,661 |
-| 2 | `a n` | 60,782 |
-| 3 | `n _` | 55,172 |
-| 4 | `s _` | 47,775 |
-| 5 | `n d` | 40,577 |
+| 1 | `e _` | 68,542 |
+| 2 | `a n` | 60,904 |
+| 3 | `n _` | 55,318 |
+| 4 | `s _` | 47,837 |
+| 5 | `n d` | 40,759 |
 
 **3-grams (Subword):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `a n d` | 24,204 |
-| 2 | `n d _` | 20,527 |
-| 3 | `a n _` | 17,020 |
-| 4 | `_ a n` | 16,519 |
-| 5 | `o n _` | 15,999 |
+| 1 | `a n d` | 24,396 |
+| 2 | `n d _` | 20,668 |
+| 3 | `a n _` | 16,952 |
+| 4 | `_ a n` | 16,629 |
+| 5 | `o n _` | 16,182 |
 
 **4-grams (Subword):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `a n d _` | 16,546 |
-| 2 | `_ a n d` | 14,727 |
-| 3 | `_ o n _` | 10,205 |
-| 4 | `_ i s _` | 10,081 |
-| 5 | `_ i n _` | 9,962 |
+| 1 | `a n d _` | 16,673 |
+| 2 | `_ a n d` | 14,847 |
+| 3 | `_ o n _` | 10,364 |
+| 4 | `_ i s _` | 10,180 |
+| 5 | `_ i n _` | 9,895 |
+
+**5-grams (Subword):**
+
+| Rank | N-gram | Count |
+|------|--------|-------|
+| 1 | `_ a n d _` | 14,216 |
+| 2 | `_ t h e _` | 3,853 |
+| 3 | `_ þ ǣ m _` | 3,654 |
+| 4 | `_ þ æ s _` | 3,541 |
+| 5 | `_ h i s _` | 3,480 |
 
 
 ### Key Findings
 
 - **Best Perplexity:** 2-gram (subword) with 365
 - **Entropy Trend:** Decreases with larger n-grams (more predictable)
-- **Coverage:** Top-1000 patterns cover ~33% of corpus
+- **Coverage:** Top-1000 patterns cover ~20% of corpus
 - **Recommendation:** 4-gram or 5-gram for best predictive performance
 
 ---
@@ -227,14 +259,14 @@ Below are sample sentences tokenized with each vocabulary size:
 
 | Context | Variant | Avg Entropy | Perplexity | Branching Factor | Unique Contexts | Predictability |
 |---------|---------|-------------|------------|------------------|-----------------|----------------|
-| **1** | Word | 0.6222 | 1.539 | 3.58 | 86,720 | 37.8% |
-| **1** | Subword | 0.8536 | 1.807 | 6.47 | 1,240 | 14.6% |
-| **2** | Word | 0.1549 | 1.113 | 1.30 | 307,843 | 84.5% |
-| **2** | Subword | 0.9630 | 1.949 | 5.87 | 8,021 | 3.7% |
-| **3** | Word | 0.0382 | 1.027 | 1.05 | 397,541 | 96.2% |
-| **3** | Subword | 0.8613 | 1.817 | 4.01 | 47,051 | 13.9% |
-| **4** | Word | 0.0126 🏆 | 1.009 | 1.02 | 415,179 | 98.7% |
-| **4** | Subword | 0.6212 | 1.538 | 2.55 | 188,289 | 37.9% |
+| **1** | Word | 0.6200 | 1.537 | 3.57 | 86,918 | 38.0% |
+| **1** | Subword | 0.8434 | 1.794 | 6.43 | 1,235 | 15.7% |
+| **2** | Word | 0.1550 | 1.113 | 1.30 | 307,624 | 84.5% |
+| **2** | Subword | 0.9640 | 1.951 | 5.90 | 7,944 | 3.6% |
+| **3** | Word | 0.0385 | 1.027 | 1.05 | 397,324 | 96.2% |
+| **3** | Subword | 0.8649 | 1.821 | 4.02 | 46,823 | 13.5% |
+| **4** | Word | 0.0127 🏆 | 1.009 | 1.02 | 415,064 | 98.7% |
+| **4** | Subword | 0.6219 | 1.539 | 2.55 | 188,154 | 37.8% |
 
 ### Generated Text Samples (Word-based)
 
@@ -242,27 +274,27 @@ Below are text samples generated from each word-based Markov chain model:
 
 **Context Size 1:**
 
-1. `and ūtƿeardra ēarena dat gearƿum gearƿum acc hāligne scole basic properties geometry type of snāwdūn...`
-2. `on læt westseaxisc norþanhymbrisc miercisc arun cisseceaster craƿley hæstingas ge forlēosaþ hiera ag...`
-3. `is lēoþ þe roðberht roðberhting beweddod æþelhæþ of indian islamic scholar kenichi fukui geapanisc s...`
+1. `and strætham is eoferƿicscire dǣl on wyrse ġearum ac bowser jr americanisc auto maker grady thomas`
+2. `on pictocusċīre`
+3. `is burg þes geþoftede rīce and gescyldnesse kowane mutum yana da vinci chapter 93 dead rǣdinge`
 
 **Context Size 2:**
 
-1. `in þǣm æt paris and roðem liciaþ on hiere rīce ƿæron corsica sardinia and sicilia īege cartaine`
-2. `on þǣm trēoƿenan hrōfe þǣre byrgenne þæt mægdnes ƿelgeāspared līc nēodlīce geāspared mid mēose and b...`
-3. `in þæm geāre bī marianland and þam sæfaroþum þeodsclandes niðerlandes belgican and franclandes in þæ...`
+1. `on þǣm ġerēfscipe þæs fōresittende sam dōnde swā þurh sūþsið fǣmneland wǣre and ǣrende genōg land fo...`
+2. `in þǣm geānlǣhtum rīcum fram þǣm sericus gārsecge hit stent is 60 mīle geddoburg be sūðƿesten on`
+3. `in þæm suþernan dæle þæs geānedan cynerīces grēatre brytene cynerīce ƿæs þēodland in ƿesternre europ...`
 
 **Context Size 3:**
 
-1. `td valign top td valign top imperator caesar lvcivs septimvs severvs pertinax avgvstvs small procons...`
-2. `is þorp in þæm east þriding se is eoferƿicscire dǣl on englum hit hæfþ 3 178 būendas on`
-3. `on eoferwicscīre þæs geānedan cynerīces fram þǣm gēare oþ þæt gēar belamp þæt hūs and his foregenga ...`
+1. `td valign top efencasere mid numeriane murdered td valign top td valign top td oþ 269 td valign`
+2. `is þorp in þæm east þriding se is eoferƿicscire dǣl on englum on eoferwicscīre þæs geānedan cynerīce...`
+3. `eoferwicscīre þæs geānedan cynerīces on wēalum þrēoscyte is hēo and hæfþ twegen ecgas onmang beorgum...`
 
 **Context Size 4:**
 
 1. `on eoferwicscīre þæs geānedan cynerīces`
-2. `is eoferƿicscire dǣl on englum hit hæfþ 105 būend on eoferwicscīre þæs geānedan cynerīces`
-3. `eoferƿicscire dǣl on englum heo hæfþ 975 buend on eoferwicscīre þæs geānedan cynerīces`
+2. `is eoferƿicscire dǣl on englum hit hæfþ 318 buend on eoferwicscīre þæs geānedan cynerīces`
+3. `eoferƿicscire dǣl on englum on eoferwicscīre þæs geānedan cynerīces`
 
 
 ### Generated Text Samples (Subword-based)
@@ -271,34 +303,34 @@ Below are text samples generated from each subword-based Markov chain model:
 
 **Context Size 1:**
 
-1. `_t_ate_inn,_mis_`
-2. `egmbrōðon,_s_on_`
-3. `n_brls_þ_k_aliea`
+1. `_bor.rm_frsh_ded`
+2. `es_a_urīþofophob`
+3. `nftaner,_a_gavst`
 
 **Context Size 2:**
 
-1. `e_mand_heaxum_sæ_`
-2. `and_ploƿealin_dæg`
-3. `n_enganvicipez)_v`
+1. `e_wærr_ofiret_be_`
+2. `and:_gēac_(144_75`
+3. `n_en_polan_on_þæs`
 
 **Context Size 3:**
 
-1. `andūnsta_hild_on_p`
-2. `nd_(ælesta_æcgrung`
-3. `an_þissibbe._æfn_r`
+1. `and_womericanada_r`
+2. `nd_þā_illinge._þā_`
+3. `an_60_1,000_ause_ƿ`
 
 **Context Size 4:**
 
-1. `and_ƿæs_ƿrīteresfel`
-2. `_and_hīe_(se_ƿord_e`
-3. `_on_villelme._7_heo`
+1. `and_(oððe_stre_in_n`
+2. `_and_scot_ƿæs_þēodi`
+3. `_on_mererīca,_a_cli`
 
 
 ### Key Findings
 
 - **Best Predictability:** Context-4 (word) with 98.7% predictability
 - **Branching Factor:** Decreases with context size (more deterministic)
-- **Memory Trade-off:** Larger contexts require more storage (188,289 contexts)
+- **Memory Trade-off:** Larger contexts require more storage (188,154 contexts)
 - **Recommendation:** Context-3 or Context-4 for text generation
 
 ---
@@ -314,64 +346,64 @@ Below are text samples generated from each subword-based Markov chain model:
 
 | Metric | Value |
 |--------|-------|
-| Vocabulary Size | 31,177 |
-| Total Tokens | 402,508 |
-| Mean Frequency | 12.91 |
+| Vocabulary Size | 31,186 |
+| Total Tokens | 403,003 |
+| Mean Frequency | 12.92 |
 | Median Frequency | 3 |
-| Frequency Std Dev | 155.94 |
+| Frequency Std Dev | 156.70 |
 
 ### Most Common Words
 
 | Rank | Word | Frequency |
 |------|------|-----------|
-| 1 | and | 14,190 |
-| 2 | on | 10,528 |
-| 3 | in | 10,215 |
-| 4 | is | 10,204 |
-| 5 | of | 6,064 |
-| 6 | se | 4,321 |
-| 7 | the | 3,988 |
-| 8 | þǣm | 3,644 |
-| 9 | þæs | 3,627 |
-| 10 | his | 3,498 |
+| 1 | and | 14,299 |
+| 2 | on | 10,683 |
+| 3 | is | 10,302 |
+| 4 | in | 10,147 |
+| 5 | of | 6,062 |
+| 6 | se | 4,316 |
+| 7 | the | 3,973 |
+| 8 | þǣm | 3,669 |
+| 9 | þæs | 3,610 |
+| 10 | his | 3,501 |
 
 ### Least Common Words (from vocabulary)
 
 | Rank | Word | Frequency |
 |------|------|-----------|
-| 1 | orcaneġe | 2 |
-| 2 | laguna | 2 |
-| 3 | stātwīca | 2 |
-| 4 | seolhstrand | 2 |
-| 5 | crosern | 2 |
-| 6 | crosernes | 2 |
-| 7 | sīdesċipes | 2 |
-| 8 | heardran | 2 |
-| 9 | caysċīre | 2 |
-| 10 | gjirokastër | 2 |
+| 1 | minga | 2 |
+| 2 | blæcfugolond | 2 |
+| 3 | ƿīleacstede | 2 |
+| 4 | cōcsċīre | 2 |
+| 5 | winnebagsċīre | 2 |
+| 6 | ælfrēdingtūn | 2 |
+| 7 | irfung | 2 |
+| 8 | larēodo | 2 |
+| 9 | grœndā | 2 |
+| 10 | dǣlungs | 2 |
 
 ### Zipf's Law Analysis
 
 | Metric | Value |
 |--------|-------|
-| Zipf Coefficient | 0.9331 |
-| R² (Goodness of Fit) | 0.998051 |
+| Zipf Coefficient | 0.9344 |
+| R² (Goodness of Fit) | 0.998034 |
 | Adherence Quality | **excellent** |
 
 ### Coverage Analysis
 
 | Top N Words | Coverage |
 |-------------|----------|
-| Top 100 | 37.9% |
-| Top 1,000 | 59.5% |
+| Top 100 | 38.0% |
+| Top 1,000 | 59.6% |
 | Top 5,000 | 77.9% |
 | Top 10,000 | 86.2% |
 
 ### Key Findings
 
-- **Zipf Compliance:** R²=0.9981 indicates excellent adherence to Zipf's law
-- **High Frequency Dominance:** Top 100 words cover 37.9% of corpus
-- **Long Tail:** 21,177 words needed for remaining 13.8% coverage
+- **Zipf Compliance:** R²=0.9980 indicates excellent adherence to Zipf's law
+- **High Frequency Dominance:** Top 100 words cover 38.0% of corpus
+- **Long Tail:** 21,186 words needed for remaining 13.8% coverage
 
 ---
 ## 5. Word Embeddings Evaluation
@@ -387,37 +419,40 @@ Below are text samples generated from each subword-based Markov chain model:
 
 ### 5.1 Cross-Lingual Alignment
 
-> *Note: Multilingual alignment visualization not available for this language.*
+![Alignment Quality](visualizations/embedding_alignment_quality.png)
+
+![Multilingual t-SNE](visualizations/embedding_tsne_multilingual.png)
 
 
 ### 5.2 Model Comparison
 
 | Model | Dimension | Isotropy | Semantic Density | Alignment R@1 | Alignment R@10 |
 |-------|-----------|----------|------------------|---------------|----------------|
-| **mono_32d** | 32 | 0.7825 🏆 | 0.3427 | N/A | N/A |
-| **mono_64d** | 64 | 0.4658 | 0.3135 | N/A | N/A |
-| **mono_128d** | 128 | 0.1306 | 0.3083 | N/A | N/A |
+| **mono_32d** | 32 | 0.8005 🏆 | 0.3579 | N/A | N/A |
+| **mono_64d** | 64 | 0.4615 | 0.3163 | N/A | N/A |
+| **mono_128d** | 128 | 0.1318 | 0.3128 | N/A | N/A |
+| **aligned_32d** | 32 | 0.8005 | 0.3430 | 0.0360 | 0.2720 |
+| **aligned_64d** | 64 | 0.4615 | 0.3200 | 0.0780 | 0.3300 |
+| **aligned_128d** | 128 | 0.1318 | 0.2993 | 0.0840 | 0.3840 |
 
 ### Key Findings
 
-- **Best Isotropy:** mono_32d with 0.7825 (more uniform distribution)
-- **Semantic Density:** Average pairwise similarity of 0.3215. Lower values indicate better semantic separation.
-- **Alignment Quality:** No aligned models evaluated in this run.
+- **Best Isotropy:** mono_32d with 0.8005 (more uniform distribution)
+- **Semantic Density:** Average pairwise similarity of 0.3249. Lower values indicate better semantic separation.
+- **Alignment Quality:** Aligned models achieve up to 8.4% R@1 in cross-lingual retrieval.
 - **Recommendation:** 128d aligned for best cross-lingual performance
 
 ---
 ## 6.  Morphological Analysis (Experimental)
 
-> ⚠️ **Warning:** This language shows low morphological productivity. The statistical signals used for this analysis may be noisy or less reliable than for morphologically rich languages.
-
 This section presents an automated morphological analysis derived from the statistical divergence between word-level and subword-level models. By analyzing where subword predictability spikes and where word-level coverage fails, we can infer linguistic structures without supervised data.
 
 ### 6.1 Productivity & Complexity
 
 | Metric | Value | Interpretation | Recommendation |
 |--------|-------|----------------|----------------|
-| Productivity Index | **0.000** | Low morphological productivity | ⚠️ Likely unreliable |
-| Idiomaticity Gap | **-1.000** | Low formulaic content | - |
+| Productivity Index | **5.000** | High morphological productivity | Reliable analysis |
+| Idiomaticity Gap | **1.044** | High formulaic/idiomatic content | - |
 
 ### 6.2 Affix Inventory (Productive Units)
 
@@ -426,17 +461,17 @@ These are the most productive prefixes and suffixes identified by sampling the v
 #### Productive Prefixes
 | Prefix | Examples |
 |--------|----------|
-| `-ge` | gesetedum, germanisca, getimbrod |
+| `-ge` | geƿinnes, geendebyrded, genered |
 
 #### Productive Suffixes
 | Suffix | Examples |
 |--------|----------|
-| `-e` | participle, ċeampscipe, smǣte |
-| `-es` | pirates, cromwelles, stranges |
-| `-an` | onginnan, lǣdnan, praetorian |
-| `-um` | gesetedum, mǣnum, strengum |
-| `-de` | onƿendode, landede, aspreade |
-| `-ng` | manufacturing, bringing, georging |
+| `-e` | ohthere, tide, crulande |
+| `-an` | iuliscan, weardan, ċiriċeburnan |
+| `-es` | geƿinnes, laurentes, yankees |
+| `-um` | stōwum, sǣfōrum, betweonum |
+| `-de` | tide, crulande, īeglande |
+| `-ng` | hūselhālgung, āmang, rising |
 
 ### 6.3 Bound Stems (Lexical Roots)
 
@@ -444,18 +479,18 @@ Bound stems are high-frequency subword units that are semantically cohesive but
 
 | Stem | Cohesion | Substitutability | Examples |
 |------|----------|------------------|----------|
-| `mani` | 2.06x | 43 contexts | amani, maniȝ, manig |
-| `enne` | 1.98x | 49 contexts | fenne, vienne, etenne |
-| `unge` | 1.86x | 46 contexts | tunge, jungen, ēðunge |
-| `ster` | 1.69x | 59 contexts | buster, easter, ēaster |
-| `tion` | 2.27x | 19 contexts | nation, motion, action |
-| `inga` | 1.74x | 34 contexts | ðinga, minga, þinga |
-| `ning` | 1.67x | 36 contexts | mining, ininga, cyning |
-| `aste` | 1.77x | 27 contexts | easte, ēaste, taste |
-| `ynin` | 2.27x | 11 contexts | cynin, cyninᵹ, cyning |
-| `nden` | 1.74x | 24 contexts | finden, bunden, funden |
-| `afod` | 1.89x | 18 contexts | hēafod, heafod, ƿafode |
-| `nisc` | 1.56x | 30 contexts | denisc, rūnisc, dēnisc |
+| `mani` | 2.03x | 43 contexts | amani, maniġ, maniȝ |
+| `enne` | 1.97x | 48 contexts | fenne, agenne, etenne |
+| `ster` | 1.84x | 59 contexts | buster, easter, noster |
+| `wear` | 1.97x | 43 contexts | weard, wearþ, wearð |
+| `unge` | 1.85x | 46 contexts | tunge, tunges, hunger |
+| `tion` | 2.26x | 19 contexts | action, motion, nation |
+| `inga` | 1.78x | 34 contexts | þinga, minga, ðinga |
+| `ning` | 1.70x | 35 contexts | cyning, ininga, cining |
+| `aste` | 1.76x | 27 contexts | ēaste, easte, taste |
+| `ynin` | 2.28x | 11 contexts | cynin, cyning, cyninȝ |
+| `afod` | 1.85x | 18 contexts | heafod, ƿafode, hēafod |
+| `nisc` | 1.56x | 27 contexts | denisc, cinisc, dēnisc |
 
 ### 6.4 Affix Compatibility (Co-occurrence)
 
@@ -463,12 +498,12 @@ This table shows which prefixes and suffixes most frequently co-occur on the sam
 
 | Prefix | Suffix | Frequency | Examples |
 |--------|--------|-----------|----------|
-| `-ge` | `-e` | 89 words | gesealde, geƿorhte |
-| `-ge` | `-de` | 28 words | gesealde, gebede |
-| `-ge` | `-an` | 21 words | georgian, geƿunelican |
-| `-ge` | `-es` | 19 words | gereces, gewitnes |
-| `-ge` | `-um` | 14 words | gerādum, gelicum |
-| `-ge` | `-ng` | 8 words | gegaderung, gewrixlung |
+| `-ge` | `-e` | 75 words | gearore, gegaderode |
+| `-ge` | `-de` | 27 words | gegaderode, gelytlode |
+| `-ge` | `-um` | 20 words | getremincum, gearum |
+| `-ge` | `-an` | 19 words | gearan, gesecan |
+| `-ge` | `-es` | 18 words | gearƿes, geferscipes |
+| `-ge` | `-ng` | 7 words | geswutelung, geþrang |
 
 ### 6.5 Recursive Morpheme Segmentation
 
@@ -477,25 +512,27 @@ Using **Recursive Hierarchical Substitutability**, we decompose complex words in
 | Word | Suggested Split | Confidence | Stem |
 |------|-----------------|------------|------|
 | gereordes | **`ge-reord-es`** | 6.0 | `reord` |
+| geƿealdes | **`ge-ƿeald-es`** | 6.0 | `ƿeald` |
+| gehealdan | **`ge-heald-an`** | 6.0 | `heald` |
+| gefeohtes | **`ge-feoht-es`** | 6.0 | `feoht` |
+| foresittendlican | **`foresittendlic-an`** | 4.5 | `foresittendlic` |
+| hundgēares | **`hundgēar-es`** | 4.5 | `hundgēar` |
+| ƿīntrēoƿum | **`ƿīntrēoƿ-um`** | 4.5 | `ƿīntrēoƿ` |
 | cræftigum | **`cræftig-um`** | 4.5 | `cræftig` |
-| dƿeligendes | **`dƿeligend-es`** | 4.5 | `dƿeligend` |
-| bisceopes | **`bisceop-es`** | 4.5 | `bisceop` |
-| fylgendan | **`fylgend-an`** | 4.5 | `fylgend` |
-| swisslandes | **`swissland-es`** | 4.5 | `swissland` |
-| norðiscan | **`norðisc-an`** | 4.5 | `norðisc` |
-| þēodlican | **`þēodlic-an`** | 4.5 | `þēodlic` |
-| gregoriscan | **`gregorisc-an`** | 4.5 | `gregorisc` |
-| blōtmōnaðes | **`blōtmōnað-es`** | 4.5 | `blōtmōnað` |
-| dufenales | **`dufenal-es`** | 4.5 | `dufenal` |
-| healdende | **`healden-de`** | 4.5 | `healden` |
-| þrōndhāmes | **`þrōndhām-es`** | 4.5 | `þrōndhām` |
-| niðerlendiscan | **`niðerlendisc-an`** | 4.5 | `niðerlendisc` |
-| antarctiscum | **`antarctisc-um`** | 4.5 | `antarctisc` |
+| bryttiscan | **`bryttisc-an`** | 4.5 | `bryttisc` |
+| speliġendhūses | **`speliġendhūs-es`** | 4.5 | `speliġendhūs` |
+| russiscum | **`russisc-um`** | 4.5 | `russisc` |
+| regollicum | **`regollic-um`** | 4.5 | `regollic` |
+| atlantiscan | **`atlantisc-an`** | 4.5 | `atlantisc` |
+| ƿealdende | **`ƿealden-de`** | 4.5 | `ƿealden` |
+| gegaderunge | **`ge-gaderunge`** | 4.5 | `gaderunge` |
 
 ### 6.6 Linguistic Interpretation
 
 > **Automated Insight:**
-The language ANG appears to be more isolating or has a highly fixed vocabulary. Word-level models perform nearly as well as subword models, indicating fewer productive morphological processes.
+The language Old English shows high morphological productivity. The subword models are significantly more efficient than word models, suggesting a rich system of affixation or compounding.
+
+> **Note on Idiomaticity:** The high Idiomaticity Gap suggests a large number of frequent multi-word expressions or formulaic sequences that are statistically distinct from their component parts.
 
 ---
 ## 7. Summary & Recommendations
@@ -506,7 +543,7 @@ The language ANG appears to be more isolating or has a highly fixed vocabulary.
 
 | Component | Recommended | Rationale |
 |-----------|-------------|-----------|
-| Tokenizer | **64k BPE** | Best compression (4.02x) |
+| Tokenizer | **64k BPE** | Best compression (4.01x) |
 | N-gram | **2-gram** | Lowest perplexity (365) |
 | Markov | **Context-4** | Highest predictability (98.7%) |
 | Embeddings | **100d** | Balanced semantic capture and isotropy |
@@ -722,4 +759,4 @@ MIT License - Free for academic and commercial use.
 ---
 *Generated by Wikilangs Models Pipeline*
 
-*Report Date: 2026-01-03 05:11:41*
+*Report Date: 2026-01-03 14:10:12*

models/embeddings/aligned/ang_128d.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:916924f4cae8ca68c9b328270e6c8f5d750e1f5f8a934a8f14b2a9ed44bfa5e9
+size 1034408961

models/embeddings/aligned/ang_128d.meta.json

@@ -0,0 +1 @@
+{"lang": "ang", "dim": 128, "max_seq_len": 512, "is_aligned": true}

models/embeddings/aligned/ang_128d.projection.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:25502450dc6c859927a71989b5700de37d1474c28657905e5d57935efab0ce21
+size 65664

models/embeddings/aligned/ang_128d_metadata.json

@@ -0,0 +1,8 @@
+{
+  "language": "ang",
+  "dimension": 128,
+  "version": "aligned",
+  "hub_language": "en",
+  "seed_vocab_size": 3813,
+  "vocab_size": 10001
+}

models/embeddings/aligned/ang_32d.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bf60165a1c710ff1513a37905cbab9e4938cc02aa22cdf5c36d13a954072c8da
+size 258728193

models/embeddings/aligned/ang_32d.meta.json

@@ -0,0 +1 @@
+{"lang": "ang", "dim": 32, "max_seq_len": 512, "is_aligned": true}

models/embeddings/aligned/ang_32d.projection.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ddf7e9dc22e45732b340414482d0aa30bca9a7225905b66b922786d632cfb556
+size 4224

models/embeddings/aligned/ang_32d_metadata.json

@@ -0,0 +1,8 @@
+{
+  "language": "ang",
+  "dimension": 32,
+  "version": "aligned",
+  "hub_language": "en",
+  "seed_vocab_size": 3813,
+  "vocab_size": 10001
+}

models/embeddings/aligned/ang_64d.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ca061364183fd6d9b77c2b10561dad103ed397dde669eca49fcc861ac7e34566
+size 517288449

models/embeddings/aligned/ang_64d.meta.json

@@ -0,0 +1 @@
+{"lang": "ang", "dim": 64, "max_seq_len": 512, "is_aligned": true}

models/embeddings/aligned/ang_64d.projection.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a318bf6e939ba9d157a6589439149031e127db782751369e9a4fb9bca1368f69
+size 16512

models/embeddings/aligned/ang_64d_metadata.json

@@ -0,0 +1,8 @@
+{
+  "language": "ang",
+  "dimension": 64,
+  "version": "aligned",
+  "hub_language": "en",
+  "seed_vocab_size": 3813,
+  "vocab_size": 10001
+}

models/embeddings/monolingual/ang_128d.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:dc7bae19e3f22929a7ae4549e0307cea4fff7f0a27889fdd52644f4d57718ccd
-size 1034428777
+oid sha256:916924f4cae8ca68c9b328270e6c8f5d750e1f5f8a934a8f14b2a9ed44bfa5e9
+size 1034408961

models/embeddings/monolingual/ang_128d_metadata.json

@@ -11,5 +11,5 @@
     "encoding_method": "rope",
     "dim": 128
   },
-  "vocab_size": 10020
+  "vocab_size": 10001
 }

models/embeddings/monolingual/ang_32d.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:76152ef959e26af0b83086f074dcef1d392da8b664962055a3b2fc4489805dfb
-size 258733417
+oid sha256:bf60165a1c710ff1513a37905cbab9e4938cc02aa22cdf5c36d13a954072c8da
+size 258728193

models/embeddings/monolingual/ang_32d_metadata.json

@@ -11,5 +11,5 @@
     "encoding_method": "rope",
     "dim": 32
   },
-  "vocab_size": 10020
+  "vocab_size": 10001
 }

models/embeddings/monolingual/ang_64d.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:46f6519234c95a833d0492a96515afc2ecb16722d834b8a1bb54212baff798b7
-size 517298537
+oid sha256:ca061364183fd6d9b77c2b10561dad103ed397dde669eca49fcc861ac7e34566
+size 517288449

models/embeddings/monolingual/ang_64d_metadata.json

@@ -11,5 +11,5 @@
     "encoding_method": "rope",
     "dim": 64
   },
-  "vocab_size": 10020
+  "vocab_size": 10001
 }

models/subword_markov/ang_markov_ctx1_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:90a9a8841ba5050d61a53c8c4b882418b11758d12d4e4bfb0e1c35962a5ff66e
-size 67847
+oid sha256:3c51cb623755a1753881542eeb56f63d0e3cca6b26b36201d37c4673561c4964
+size 66968

models/subword_markov/ang_markov_ctx1_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 1,
   "variant": "subword",
   "language": "ang",
-  "unique_contexts": 1240,
-  "total_transitions": 2931555
+  "unique_contexts": 1235,
+  "total_transitions": 2932601
 }

models/subword_markov/ang_markov_ctx2_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9862fd96d60b2ddcba7f9f843ca6ff64c8e2609911fdd8fe6814efae3de58e89
-size 359618
+oid sha256:75c24d757e1f8ff20404bf43a0494cdc7025f3099ecf2c5fd0ded6196982f719
+size 360335

models/subword_markov/ang_markov_ctx2_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 2,
   "variant": "subword",
   "language": "ang",
-  "unique_contexts": 8021,
-  "total_transitions": 2926656
+  "unique_contexts": 7944,
+  "total_transitions": 2927617
 }

models/subword_markov/ang_markov_ctx3_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bdba84f6d74bc0545de74369839c3baecf845b91493ccd850300aa17dc493368
-size 1382361
+oid sha256:92be48c01deb0d03b413930a4303e8ea68fa19ad34847e3364801ebb65d661f3
+size 1378085

models/subword_markov/ang_markov_ctx3_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 3,
   "variant": "subword",
   "language": "ang",
-  "unique_contexts": 47051,
-  "total_transitions": 2921757
+  "unique_contexts": 46823,
+  "total_transitions": 2922633
 }

models/subword_markov/ang_markov_ctx4_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2975bbac2b2a98d8f25d17a40f1919b0cb2c89af3cb5e283029e0814841eb485
-size 3773700
+oid sha256:63d5bae5a9064ea8d0b55a7323a00e10fd3cc817a992d548d92596656acd9a23
+size 3780418

models/subword_markov/ang_markov_ctx4_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 4,
   "variant": "subword",
   "language": "ang",
-  "unique_contexts": 188289,
-  "total_transitions": 2916858
+  "unique_contexts": 188154,
+  "total_transitions": 2917649
 }

models/subword_ngram/ang_2gram_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ae5772afcd4edeeb505f3f15218338f4ab947f1eb61616ecf75e9477428c3bdf
-size 40718
+oid sha256:3de4ae1fde9e711415a305f0cff7d028d9d6dbb8e9784696204bb787c90dd202
+size 40740

models/subword_ngram/ang_2gram_subword_metadata.json

@@ -2,6 +2,6 @@
   "n": 2,
   "variant": "subword",
   "language": "ang",
-  "unique_ngrams": 3016,
-  "total_ngrams": 2931555
+  "unique_ngrams": 3006,
+  "total_ngrams": 2932601
 }

models/subword_ngram/ang_3gram_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d2c6ba1309e9103dc84eb58507d098ed662ea9cfea0a9c9f9b74ba147f439950
-size 290728
+oid sha256:3c21515a66cb038262024984e5d43dfebb139b67620b893fda4f9839a52c0dc7
+size 290609

models/subword_ngram/ang_3gram_subword_metadata.json

@@ -2,6 +2,6 @@
   "n": 3,
   "variant": "subword",
   "language": "ang",
-  "unique_ngrams": 23727,
-  "total_ngrams": 2926656
+  "unique_ngrams": 23711,
+  "total_ngrams": 2927617
 }

models/subword_ngram/ang_4gram_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:214ae5f8a7aa366afa79d7b55156420e4cde925f114aacb43a0e8b1677ac34eb
-size 1206179
+oid sha256:6a666f8631a2c316853af5db1a673829c50f10adee5b2020606062eb57ed52f6
+size 1209850

models/subword_ngram/ang_4gram_subword_metadata.json

@@ -2,6 +2,6 @@
   "n": 4,
   "variant": "subword",
   "language": "ang",
-  "unique_ngrams": 105485,
-  "total_ngrams": 2921757
+  "unique_ngrams": 105677,
+  "total_ngrams": 2922633
 }

models/subword_ngram/ang_5gram_subword.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7c3056ca0bf5781dd09261541479f028b29a9cc8d5a5a6f2bd27261e1805e569
+size 2555553

models/subword_ngram/ang_5gram_subword_metadata.json

@@ -0,0 +1,7 @@
+{
+  "n": 5,
+  "variant": "subword",
+  "language": "ang",
+  "unique_ngrams": 217768,
+  "total_ngrams": 2917649
+}

models/tokenizer/ang_tokenizer_16k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:10b809027a944dfea82e7565213a5a0dd09ee883ea5c6abc3dc9dae21515155d
-size 507706
+oid sha256:040cdd2ba84e292b44535ad7e47be3d9d4dea1f384c04801879d77cd0895da82
+size 507562

models/tokenizer/ang_tokenizer_32k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:882f479e6372ebf5cf806ece607984b48ce4e93dc139971c738b81acd86b41e5
-size 785063
+oid sha256:11310c4f4880a83c7e0121967baf71b21fdb39d161532062e4d76550c057a832
+size 785162

models/tokenizer/ang_tokenizer_64k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1e5786df616a5b116d734bb3ad420c192cfac85b05de0422ef4883ffab1a5188
-size 1394695
+oid sha256:d5e8fac427c909ddf6ed2f1d118ab76e763479cf812ec71e84f5a78d4c94f50d
+size 1395192

models/tokenizer/ang_tokenizer_8k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9350642caa9ab4fcb09383368a4dd339cdebaa4100ee5aa4e4b6128518a978ae
-size 372384
+oid sha256:de84f48cb9fa027b88c36715acd20bb110a824676f2c89684ff9ac4bc44afca9
+size 372530

models/vocabulary/ang_vocabulary.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ee7206585fc58294269ce4be08117acf83241efa4ebab2fe8ed326955b226129
-size 531959
+oid sha256:113d35307580b6ba8524484649846ca961d60e99640c0a1ec1916e31afebf5d6
+size 532266

models/vocabulary/ang_vocabulary_metadata.json

@@ -1,17 +1,17 @@
 {
   "language": "ang",
-  "vocabulary_size": 31177,
+  "vocabulary_size": 31186,
   "variant": "full",
   "statistics": {
-    "type_token_ratio": 0.19034733881342258,
+    "type_token_ratio": 0.1905231532884854,
     "coverage": {
-      "top_100": 0.33234415541940127,
-      "top_1000": 0.5225039792425267,
-      "top_5000": 0.6832392124370408,
-      "top_10000": 0.7561302138979134
+      "top_100": 0.33298282280713215,
+      "top_1000": 0.5232425489299632,
+      "top_5000": 0.6837001719896371,
+      "top_10000": 0.7563298717697516
     },
-    "hapax_count": 56122,
-    "hapax_ratio": 0.6428710523602791,
-    "total_documents": 4899
+    "hapax_count": 56327,
+    "hapax_ratio": 0.6436415161176053,
+    "total_documents": 4984
   }
 }

models/word_markov/ang_markov_ctx1_word.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:94e97bb71a4cb316f1df8db8559995a89599d765c1ea3b77e8d9298dde91cee5
-size 2838106
+oid sha256:f9404cf8e75c7dafaa37bb38566f58ecbfbcde523666ebe80d07fb25ebd8ff43
+size 2834503

models/word_markov/ang_markov_ctx1_word_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 1,
   "variant": "word",
   "language": "ang",
-  "unique_contexts": 86720,
-  "total_transitions": 453731
+  "unique_contexts": 86918,
+  "total_transitions": 454346
 }

models/word_markov/ang_markov_ctx2_word.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6ed47074ae9df17987a96dedf82859c432c3c11ac930279d48acda20815ee000
-size 5756944
+oid sha256:dd85fc8f75ac79620c6acdaafd11303ab1e84fccbf3610c168b1fc640627a885
+size 5756661

models/word_markov/ang_markov_ctx2_word_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 2,
   "variant": "word",
   "language": "ang",
-  "unique_contexts": 307843,
-  "total_transitions": 448832
+  "unique_contexts": 307624,
+  "total_transitions": 449362
 }