Adding 5gram models

.gitattributes

@@ -25,3 +25,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zstandard filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+*.arpa filter=lfs diff=lfs merge=lfs -text
+*.txt filter=lfs diff=lfs merge=lfs -text
+

all_5gram.arpa

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d5fd8b07a8edcaed3fe0dbc7f30659230f797668b1a798e12ef73526bbf3f702
+size 16086549819

all_5gram.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6e48d3691208ab8219abd2a0349e88f0e9f88ddb56ab5d56c6f9570a013018d6
+size 8035805872

bokmaal_5gram.arpa

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:880a096c854de3c459d182e0acb60f3d0eaf8e7495652f7c280aa6c78c5e4557
+size 8671958280

bokmaal_5gram.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7b41c24c63f2f0585bea83666369593f3b3e6d047f327a90f36ebca2c35ef0ff
+size 4243671427

clean_texts.py

@@ -0,0 +1,37 @@
+import re
+import argparse
+
+#chars_to_ignore_regex = '[,?.!\-\;\:"“%‘”�—’…–]' 
+chars_to_ignore_regex = '[\'=&()\*\/,?.!\-\;\:"“%‘”�—’…–<>#@0123456789°′»«\[\]]'
+radical_regex = r'[^abcdefghijklmnopqrstuvwxyzåæø ]'
+
+
+def extract_text(text, is_radical=False):
+    text = re.sub(chars_to_ignore_regex, " ", text.lower()) + " "
+    if is_radical:
+        text = re.sub(radical_regex, ' ', text)
+    text = re.sub(r'\s+', ' ', text)
+    return text
+
+
+def main(args):
+    with open(args.input_file, 'r') as file:
+        data = file.read()
+
+    data = extract_text(data, bool(args.radical))
+
+    with open(args.output_file, 'w') as outputfile:
+        outputfile.write(data)
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--input_file', required=True, help='Path to input file.')
+    parser.add_argument('--output_file', required=True, help='Path to output file.')
+    parser.add_argument('--radical', dest='radical', action='store_true', help='Delete any non vocab character.')
+    args = parser.parse_args()
+    return args
+
+if __name__ == "__main__":
+    args = parse_args()
+    main(args)

nynorsk_5gram.arpa

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:18a84d547abe6bb8a5441916b296dbdf230e3efc76a953b14660720817bda62d
+size 7718177308

nynorsk_5gram.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ab572020f4871ec611ad95a92071040bafe77c4633de2d26166006749e964129
+size 3981756555

prepare.py

@@ -0,0 +1,85 @@
+#!/usr/bin/env python
+# coding=utf-8
+import re
+from tqdm import tqdm
+
+from datasets import load_dataset, interleave_datasets, concatenate_datasets
+
+TEXT_COLUMN_NAME = "text"
+AUDIO_COLUMN_NAME = "audio"
+CHARS_TO_IGNORE_REGEX = '[\,\?\.\!\-\;\:\"\“\%\‘\”\�\'\–\_\\\+\#\/0-9]'
+
+# Pre-processing dataset
+def replace_hatted_characters(batch):
+    text = batch["text"]
+    text = re.sub(CHARS_TO_IGNORE_REGEX, '', text).lower() + ' '
+    text = re.sub('[áàâ]', 'a', text)
+    text = re.sub('[ä]', 'æ', text)
+    text = re.sub('[éèëê]', 'e', text)
+    text = re.sub('[íìïî]', 'i', text)
+    text = re.sub('[óòöô]', 'o', text)
+    text = re.sub('[ö]', 'ø', text)
+    text = re.sub('[ç]', 'c', text)
+    text = re.sub('[úùüû]', 'u', text)
+    text = re.sub('\xa0', ' ', text)
+    text = re.sub('<ee>', 'eee', text)
+    text = re.sub('<qq>', 'qqq', text)
+    text = re.sub('<mm>', 'mmm', text)
+    text = re.sub('<inaudible>', 'xxx', text)
+    text = re.sub('[<>]', '', text)
+    text = re.sub(r'\s+', ' ', text)
+    return {"text": text}
+
+
+def main():
+    npsc = load_dataset(
+        "NbAiLab/NPSC",
+        "16K_mp3",
+        split="train+validation",
+        use_auth_token=True,
+    )
+    ncc = load_dataset(
+        "NbAiLab/NCC",
+        split="train+validation",
+        use_auth_token=True
+    )
+    dataset = concatenate_datasets([npsc, ncc])
+    dataset = dataset.map(
+        replace_hatted_characters,
+        desc="replacing hesitations and homophones",
+    )
+
+    # Create file with all text together
+    text_count = len(dataset)
+    with open("text.txt", "w") as text_file:
+        for idx, text in tqdm(enumerate(dataset["text"]), total=text_count, desc="Writing text"):
+            if idx == text_count:
+                text_file.write(text)
+            else:
+                text_file.write(text + " ")
+
+    # Create KenLM model
+    !~/bin/lmplz -o 5 --text text.txt --arpa 5gram.arpa.orig -T $(pwd)
+
+    # Adjusting for Huggingface decoding
+    with open("5gram.arpa.orig", "r") as read_file, open("5gram.arpa", "w") as write_file:
+        has_added_eos = False
+        for line in read_file:
+          if not has_added_eos and "ngram 1=" in line:
+            count=line.strip().split("=")[-1]
+            write_file.write(line.replace(f"{count}", f"{int(count)+1}"))
+          elif not has_added_eos and "<s>" in line:
+            write_file.write(line)
+            write_file.write(line.replace("<s>", "</s>"))
+            has_added_eos = True
+          else:
+            write_file.write(line)
+
+    # Compress as binary
+    !~/bin/build_binary 5gram.arpa 5gram.bin -T $(pwd)
+    !rm 5gram.arpa*
+
+
+if __name__ == "__main__":
+    main()
+

texts/all_text.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:77bd024bc200949cf73266459d8e1ead26a8961b6962d415a7cadaf96af24e76
+size 910152429

texts/all_text_clean.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:916471727e5cc2602a3339d94af1cf4e8bc0ea5ca7def6e68dbe53da0a19688a
+size 908965879

texts/bokmaal_clean.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8f26eccf2b2a3076237afc427c93449915053b9b8bd928705b4fbbc539740730
+size 478844863

texts/bokmaal_text.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ae57498e4b97f35f685b8bd30cf2e74f921d3a1fb734d4eaeb437c29aaef0cab
+size 485346069

texts/bokmaal_text_clean.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:243e8924ffb667a7d854ba94e26393933a94f4fa4188ee87487639955eb1e938
+size 484980197

texts/npsc_clean.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:432bc3fc0f5247fde26810f96e6ae4d26f91d87ad6ead2ab37ac934cbdb82a64
+size 6501206

texts/nynorsk_clean.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:53d9360849730d4d46a88d657dfbc0f0e36134e7862c70170f225334234c11fe
+size 424806360

texts/nynorsk_text.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:df6eb3db143c778bc1ac4ac308fb14d750b19860e25a5968d52ca4a7483b695e
+size 431307566

texts/nynorsk_text_clean.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8621ec240641097cfaba360f814d6345da4a391b1b10735a02ae27996c6055f5
+size 430486889