#!/usr/bin/env python3 """Strict, table-driven, REVERSIBLE Devanagari <-> SLP1 and Devanagari <-> IAST. Design rules (E1 of docs/TOKENIZATION.md): 1. NFC is the frozen normalisation. Every public function normalises its input to NFC first and returns NFC output. NFC *decomposes* the precomposed nukta letters U+0958-U+095F, U+0929, U+0931, U+0934 (they are composition-excluded), so after NFC a nukta is always a separate U+093C. Both forms are in the tables anyway. 2. Fail closed, never drop. A code point with no mapping is passed through unchanged AND reported: every function returns ``(text, unmapped)`` where ``unmapped`` is a ``collections.Counter`` keyed by the offending character. Nothing is ever silently lost. 3. Round trip is the contract. deva -> slp1 -> deva and deva -> iast -> deva must return the NFC input exactly, for every input, *unless* the input contains Latin letters (see 5) or malformed Devanagari, both of which are reported. 4. Reserved ASCII. The romanised sides use a few ASCII characters as functional markers: ^ nukta (follows the consonant, precedes the vowel: k^a = U+0915 U+093C) ' avagraha % om (U+0950) | cluster separator, only emitted where IAST would otherwise be ambiguous (k|ha = क्ह, not ख; ka|i = कइ, not कै). It is only ever emitted BETWEEN two letters, so on input a '|' that touches whitespace is read as a literal character -- that is the ASCII dandaa of GRETIL-style romanised sources. ~ candrabindu (SLP1 only) / udatta U+0951 (SLP1 only) + short vowel prefix: +e +o +a (SLP1 only; IAST uses ĕ ŏ ă) * candra vowel prefix: *e *o *a (SLP1 only; IAST uses ê ô â) \\ anudatta U+0952 (SLP1 only) IAST instead writes candrabindu m̐, udatta as combining acute U+0301 and anudatta as combining grave U+0300. A *literal* occurrence of any reserved character in the source is escaped by doubling it, so the mapping stays one-to-one. 5. Latin letters are the one thing that cannot round trip. On the Devanagari side every Latin letter is a legal SLP1/IAST token, so "cafe" inside a Devanagari record comes back as चफे. Such characters are reported (class ``latin``) rather than hidden; E1 counts how many records of the corpus are affected. 6. Things deliberately NOT transliterated (identity in both directions, never reported): danda U+0964, double danda U+0965, Devanagari digits U+0966-U+096F, the abbreviation sign U+0970 and U+0971, ZWJ U+200D and ZWNJ U+200C, whitespace, and ASCII punctuation/digits. SLP1 has no agreed spelling for any of them and inventing one would be lossy -- and giving ZWJ/ZWNJ ASCII markers collided with the literal '_' that the DCS and VedaWeb use as a text marker. Parsing additionally accepts, but never emits, the spellings real-world romanised Sanskrit actually uses: ṁ for the anusvara, and the ISO-15919 vowels ē ō r̥ l̥ r̥̄ l̥̄. 7. Vedic Extensions U+1CD0-U+1CFF are reported as unmapped and passed through, except the two that are clearly SLP1 letters: jihvamuliya U+1CF5 (Z / ẖ) and upadhmaniya U+1CF6 (V / ḫ). Devanagari Extended U+A8E0-U+A8FF is likewise reported and preserved. 8. ऌ and ळ are different letters and get different spellings, which published IAST often conflates: vocalic ऌ/ॡ are ḷ/ḹ (SLP1 x/X) and retroflex ळ is ḻ, U+1E3B (SLP1 L). ISO-15919 uses ḷ for ळ; on input that reading loses to IAST's, so feed ISO through :func:`iso15919_to_iast` first. Philological note on the Vedic accents: the Rigveda's Devanagari convention marks *anudatta* below (U+0952) and *svarita* above (U+0951) and leaves udatta unmarked, while the romanised convention puts an acute on the *udatta* vowel. This module maps the signs, not the accent theory: U+0951 <-> acute, U+0952 <-> grave, as specified for E1. Converting a romanised Vedic text therefore produces a Devanagari string that is faithful as a character mapping but does not match the printed Rigveda's accent placement. The three-line summary of what the encodings look like: धर्मक्षेत्रे कुरुक्षेत्रे Devanagari Darmakzetre kurukzetre SLP1 dharmakṣetre kurukṣetre IAST """ from __future__ import annotations import unicodedata from collections import Counter __all__ = [ 'NFC', 'deva_to_slp1', 'slp1_to_deva', 'deva_to_iast', 'iast_to_deva', 'iso15919_to_iast', 'classify_char', 'AKSHARA_RE', 'split_aksharas', 'RESERVED', ] VIRAMA = '्' NUKTA = '़' ZWNJ = '‌' ZWJ = '‍' RESERVED = "^'%|~/\\+*" # -------------------------------------------------------------------------------------- # Tables. (devanagari, slp1, iast) # -------------------------------------------------------------------------------------- CONSONANTS = [ ('क', 'k', 'k'), ('ख', 'K', 'kh'), ('ग', 'g', 'g'), ('घ', 'G', 'gh'), ('ङ', 'N', 'ṅ'), ('च', 'c', 'c'), ('छ', 'C', 'ch'), ('ज', 'j', 'j'), ('झ', 'J', 'jh'), ('ञ', 'Y', 'ñ'), ('ट', 'w', 'ṭ'), ('ठ', 'W', 'ṭh'), ('ड', 'q', 'ḍ'), ('ढ', 'Q', 'ḍh'), ('ण', 'R', 'ṇ'), ('त', 't', 't'), ('थ', 'T', 'th'), ('द', 'd', 'd'), ('ध', 'D', 'dh'), ('न', 'n', 'n'), ('प', 'p', 'p'), ('फ', 'P', 'ph'), ('ब', 'b', 'b'), ('भ', 'B', 'bh'), ('म', 'm', 'm'), ('य', 'y', 'y'), ('र', 'r', 'r'), ('ल', 'l', 'l'), ('व', 'v', 'v'), ('श', 'S', 'ś'), ('ष', 'z', 'ṣ'), ('स', 's', 's'), ('ह', 'h', 'h'), ('ळ', 'L', 'ḻ'), # ळ -> l with line below ] # (independent, dependent sign or None, slp1, iast) VOWELS = [ ('अ', None, 'a', 'a'), ('आ', 'ा', 'A', 'ā'), ('इ', 'ि', 'i', 'i'), ('ई', 'ी', 'I', 'ī'), ('उ', 'ु', 'u', 'u'), ('ऊ', 'ू', 'U', 'ū'), ('ऋ', 'ृ', 'f', 'ṛ'), ('ॠ', 'ॄ', 'F', 'ṝ'), ('ऌ', 'ॢ', 'x', 'ḷ'), ('ॡ', 'ॣ', 'X', 'ḹ'), ('ए', 'े', 'e', 'e'), ('ऐ', 'ै', 'E', 'ai'), ('ओ', 'ो', 'o', 'o'), ('औ', 'ौ', 'O', 'au'), ('ऎ', 'ॆ', '+e', 'ĕ'), # short e ('ऒ', 'ॊ', '+o', 'ŏ'), # short o ('ऄ', None, '+a', 'ă'), # short a ('ऍ', 'ॅ', '*e', 'ê'), # candra e ('ऑ', 'ॉ', '*o', 'ô'), # candra o ('ॲ', None, '*a', 'â'), # candra a ] # marks that stand on their own (never fuse with the inherent vowel) MARKS = [ ('ं', 'M', 'ṃ'), # anusvara ('ः', 'H', 'ḥ'), # visarga ('ँ', '~', 'm̐'), # candrabindu ('ऽ', "'", "'"), # avagraha ('ॐ', '%', '%'), # om ('॑', '/', '́'), # udatta <-> combining acute ('॒', '\\', '̀'), # anudatta <-> combining grave ('ᳵ', 'Z', 'ẖ'), # jihvamuliya ('ᳶ', 'V', 'ḫ'), # upadhmaniya ] # identity on both sides, never reported. ZWJ/ZWNJ are here rather than in MARKS on # purpose: giving them ASCII markers would collide with the literal '_' that DCS and # VedaWeb use as a text marker, and they survive a round trip untouched as themselves. PASSTHROUGH = set('।॥॰ॱ') | {ZWJ, ZWNJ} | {chr(c) for c in range(0x0966, 0x0970)} # precomposed nukta letters -> base + U+093C (NFC already does this; belt and braces) PRECOMPOSED_NUKTA = { 'ऩ': 'न' + NUKTA, 'ऱ': 'र' + NUKTA, 'ऴ': 'ळ' + NUKTA, 'क़': 'क' + NUKTA, 'ख़': 'ख' + NUKTA, 'ग़': 'ग' + NUKTA, 'ज़': 'ज' + NUKTA, 'ड़': 'ड' + NUKTA, 'ढ़': 'ढ' + NUKTA, 'फ़': 'फ' + NUKTA, 'य़': 'य' + NUKTA, } _SLP1, _IAST = 0, 1 CONS_FWD = {d: (s, i) for d, s, i in CONSONANTS} VSIGN_FWD = {sign: (s, i) for _, sign, s, i in VOWELS if sign} VIND_FWD = {ind: (s, i) for ind, _, s, i in VOWELS} MARK_FWD = {d: (s, i) for d, s, i in MARKS} # reverse: value -> devanagari CONS_REV = [{s: d for d, s, i in CONSONANTS}, {i: d for d, s, i in CONSONANTS}] VOWEL_REV = [{s: (ind, sign) for ind, sign, s, i in VOWELS}, {i: (ind, sign) for ind, sign, s, i in VOWELS}] MARK_REV = [{s: d for d, s, i in MARKS}, {i: d for d, s, i in MARKS}] def _straddles(prev: str, nxt: str, enc: int) -> bool: """Would the junction of two emitted pieces re-parse as a different token? k + ha reads back as ख, ka + i as कै, ka + ī as कै + a stray macron. Checked in NFD, which is what the parser matches on. SLP1 has only single-character tokens, so this is always False there.""" m = MAXTOK[enc] if m < 2 or not prev or not nxt: return False a = unicodedata.normalize('NFD', prev) b = unicodedata.normalize('NFD', nxt) trie = TRIE_NFD[enc] for i in range(1, min(len(a), m - 1) + 1): for j in range(1, min(len(b), m - i) + 1): if a[-i:] + b[:j] in trie: return True return False # functional meaning of each reserved character, per encoding _FUNC_COMMON = {'^': ('nukta', None), '|': ('sep', None)} FUNC = [ dict(_FUNC_COMMON, **{c: ('mark', d) for d, c, _ in MARKS if c in RESERVED}, **{'+': ('vmod', None), '*': ('vmod', None)}), dict(_FUNC_COMMON, **{c: ('mark', d) for d, _, c in MARKS if c in RESERVED}), ] def _build_trie(enc): """token -> (kind, payload); longest match wins. Reserved chars are handled separately.""" t = {} for d, s, i in CONSONANTS: t[(s, i)[enc]] = ('cons', d) for ind, sign, s, i in VOWELS: v = (s, i)[enc] if v[0] not in RESERVED: t[v] = ('vowel', (ind, sign)) for d, s, i in MARKS: v = (s, i)[enc] if v[0] not in RESERVED: t[v] = ('mark', d) for c in PASSTHROUGH: t[c] = ('pass', c) return t # Input-only spellings. Real-world "IAST" is rarely pure: GRETIL and the DCS write the # anusvara as ṁ, VedaWeb writes ISO-15919 vowels. These are accepted when parsing and never # emitted, so the canonical round trip is unaffected while iast -> deva stops mangling them. # ISO's ḷ (= ळ) is NOT aliased: in IAST ḷ is vocalic ऌ and that reading wins. IAST_INPUT_ALIASES = { 'ṁ': ('mark', 'ं'), 'ē': ('vowel', ('ए', 'े')), 'ō': ('vowel', ('ओ', 'ो')), 'r̥': ('vowel', ('ऋ', 'ृ')), 'l̥': ('vowel', ('ऌ', 'ॢ')), 'r̥̄': ('vowel', ('ॠ', 'ॄ')), 'l̥̄': ('vowel', ('ॡ', 'ॣ')), 'ṝ': ('vowel', ('ॠ', 'ॄ')), } TRIE = [_build_trie(_SLP1), _build_trie(_IAST)] # parsing runs on NFD so that composed letters (ī = i + macron) line up with combining # accents; build the NFD tries once, not per call TRIE_NFD = [{unicodedata.normalize('NFD', k): v for k, v in t.items()} for t in TRIE] MAXTOK = [max(len(k) for k in t) for t in TRIE_NFD] # what the parser matches on: canonical tokens plus the input-only aliases TRIE_PARSE = [dict(TRIE_NFD[0]), dict(TRIE_NFD[1], **{unicodedata.normalize('NFD', k): v for k, v in IAST_INPUT_ALIASES.items()})] MAXTOK_PARSE = [max(len(k) for k in t) for t in TRIE_PARSE] # characters that a passthrough would collide with when read back RISKY = [set(k[0] for k in TRIE[0]) - PASSTHROUGH, set(k[0] for k in TRIE[1]) - PASSTHROUGH] def NFC(text: str) -> str: return unicodedata.normalize('NFC', text) def classify_char(ch: str) -> str: """Bucket an unmapped character for reporting.""" cp = ord(ch) if 0x0900 <= cp <= 0x097F: return 'devanagari_unmapped' if 0x1CD0 <= cp <= 0x1CFF: return 'vedic_extensions' if 0xA8E0 <= cp <= 0xA8FF: return 'devanagari_extended' if cp < 0x0250 and ch.isalpha(): return 'latin' if unicodedata.combining(ch): return 'combining' if ch.isalpha(): return 'other_script' return 'other' def _safe_pass(ch: str, enc: int) -> bool: """True if passing this character through is reversible and unremarkable.""" if ch in PASSTHROUGH: return True if ch in RESERVED: return True # escaped by doubling, so reversible if ch in RISKY[enc]: return False cat = unicodedata.category(ch) return cat[0] in 'ZPSN' or ch in '\n\t\r' or cat == 'Cc' # -------------------------------------------------------------------------------------- # Devanagari -> romanised # -------------------------------------------------------------------------------------- def _to_roman(text: str, enc: int): text = NFC(text) unmapped: Counter = Counter() out: list[str] = [] inh = None # index in out of the inherent 'a' still open for replacement bare = False # last piece was a consonant closed by virama def put(piece: str): """Append a piece, inserting the '|' separator wherever the junction would re-parse. Two cases: an IAST digraph straddling the junction (k + ha -> k|ha), and a reserved marker landing next to the same marker, where the doubling escape would otherwise read the pair as a literal (avagraha + avagraha -> '|', not '').""" if out and out[-1] and piece: a, b = out[-1][-1], piece[0] if (a == b and b in RESERVED and b != '|') or _straddles(out[-1], piece, enc): out.append('|') out.append(piece) def raw(ch: str): nonlocal inh, bare inh, bare = None, False put(ch + ch if ch in RESERVED else ch) i, n = 0, len(text) while i < n: ch = text[i] i += 1 if ch in PRECOMPOSED_NUKTA: # only reachable if caller skipped NFC text = text[:i - 1] + PRECOMPOSED_NUKTA[ch] + text[i:] n = len(text) i -= 1 continue if ch in CONS_FWD: put(CONS_FWD[ch][enc]) out.append('a') inh = len(out) - 1 bare = False elif ch == NUKTA: # a second nukta on the same consonant would emit '^^', which reads back as an # escaped literal '^'; refuse rather than lose it silently if inh is not None and inh == len(out) - 1 and out[inh - 1] != '^': out.insert(inh, '^') inh += 1 else: unmapped[ch] += 1 raw(ch) elif ch in VSIGN_FWD: if inh is not None and inh == len(out) - 1: v = VSIGN_FWD[ch][enc] out[inh] = ('|' + v) if (len(out) > 1 and _straddles(out[-2], v, enc)) else v inh = None else: unmapped[ch] += 1 raw(ch) elif ch == VIRAMA: if inh is not None and inh == len(out) - 1: out.pop() inh = None bare = True else: unmapped[ch] += 1 raw(ch) elif ch in VIND_FWD: # an independent vowel right after a bare consonant would be read back as that # consonant's dependent sign (र् + ऋ -> "rf" -> रृ), so separate them if bare: out.append('|') inh, bare = None, False put(VIND_FWD[ch][enc]) elif ch in MARK_FWD: inh, bare = None, False put(MARK_FWD[ch][enc]) elif ch in PASSTHROUGH: inh, bare = None, False out.append(ch) else: if not _safe_pass(ch, enc): unmapped[ch] += 1 raw(ch) return NFC(''.join(out)), unmapped def deva_to_slp1(text: str): """Devanagari -> SLP1. Returns (slp1, Counter of unmapped characters).""" return _to_roman(text, _SLP1) def deva_to_iast(text: str): """Devanagari -> IAST. Returns (iast, Counter of unmapped characters).""" return _to_roman(text, _IAST) # -------------------------------------------------------------------------------------- # romanised -> Devanagari # -------------------------------------------------------------------------------------- def _to_deva(text: str, enc: int): text = unicodedata.normalize('NFD', NFC(text)) unmapped: Counter = Counter() trie, maxtok, func = TRIE_PARSE[enc], MAXTOK_PARSE[enc], FUNC[enc] out: list[str] = [] pend = None # consonant awaiting its vowel nukta = False def close(): nonlocal pend, nukta if pend is not None: out.append(pend) if nukta: out.append(NUKTA) out.append(VIRAMA) pend, nukta = None, False def vowel(ind, sign): nonlocal pend, nukta if pend is not None: out.append(pend) if nukta: out.append(NUKTA) if sign is not None: out.append(sign) elif ind != 'अ': unmapped[ind] += 1 # no dependent form (ऄ, ॲ) after a consonant out.append(VIRAMA) out.append(ind) pend, nukta = None, False else: out.append(ind) i, n = 0, len(text) while i < n: ch = text[i] if ch in RESERVED: if i + 1 < n and text[i + 1] == ch: # escaped literal close() out.append(ch) i += 2 continue kind, payload = func.get(ch, (None, None)) if kind == 'nukta' and pend is not None and not nukta: nukta = True i += 1 continue if kind == 'sep': # Our emitter only ever puts the separator BETWEEN two letters, so a '|' that # touches whitespace is not ours: it is the ASCII dandaa that GRETIL-style # romanised sources use. Keep it (and report it) instead of swallowing it. if (text[i - 1] if i else ' ').isspace() or \ (text[i + 1] if i + 1 < n else ' ').isspace(): unmapped[ch] += 1 close() out.append(ch) else: close() i += 1 continue if kind == 'vmod': tok = text[i:i + 2] hit = VOWEL_REV[enc].get(tok) if hit: vowel(*hit) i += 2 continue if kind == 'mark': close() out.append(payload) i += 1 continue unmapped[ch] += 1 close() out.append(ch) i += 1 continue for L in range(min(maxtok, n - i), 0, -1): hit = trie.get(text[i:i + L]) if hit: kind, payload = hit if kind == 'cons': close() pend = payload elif kind == 'vowel': vowel(*payload) else: close() out.append(payload) i += L break else: if not _safe_pass(ch, enc): unmapped[ch] += 1 close() out.append(ch) i += 1 close() return NFC(''.join(out)), unmapped def slp1_to_deva(text: str): """SLP1 -> Devanagari. Returns (devanagari, Counter of unmapped characters).""" return _to_deva(text, _SLP1) def iast_to_deva(text: str): """IAST -> Devanagari. Returns (devanagari, Counter of unmapped characters).""" return _to_deva(text, _IAST) # -------------------------------------------------------------------------------------- # ISO-15919 (as VedaWeb writes it) -> our canonical IAST # -------------------------------------------------------------------------------------- # Applied on NFD so that the pitch accents (combining acute/grave) survive on the vowel. _ISO_PAIRS = [ ('r̥̄', 'ṝ'), ('l̥̄', 'ḹ'), # r̥̄ l̥̄ -> ṝ ḹ ('r̥', 'ṛ'), ('l̥', 'ḷ'), # r̥ l̥ -> ṛ ḷ ('ḷ', 'ḻ'), # ISO ḷ is ळ, not vocalic l ('ṁ', 'ṃ'), # ṁ -> ṃ ('ē', 'e'), ('ō', 'o'), # ē ō -> e o ('ṉ', 'n'), ('ṟ', 'r'), # ṉ ṟ (nukta-ish ISO) -> n r ] _ISO_ESCAPE = set(RESERVED) - {"'"} # keep ' meaning avagraha def iso15919_to_iast(text: str): """VedaWeb-style ISO-15919 (with pitch accents) -> our strict IAST. Reserved characters other than the avagraha apostrophe are escaped by doubling, so the result is a legal input to :func:`iast_to_deva`. """ src = unicodedata.normalize('NFD', NFC(text)) # longest source first so r̥̄ beats r̥ pairs = sorted(_ISO_PAIRS, key=lambda p: -len(unicodedata.normalize('NFD', p[0]))) pairs = [(unicodedata.normalize('NFD', a), b) for a, b in pairs] out, i, n = [], 0, len(src) unmapped: Counter = Counter() while i < n: for a, b in pairs: if src.startswith(a, i): out.append(b) i += len(a) break else: ch = src[i] out.append(ch + ch if ch in _ISO_ESCAPE else ch) i += 1 return NFC(''.join(out)), unmapped # -------------------------------------------------------------------------------------- # Akshara segmentation (explicit Devanagari regex; NOT \X, which breaks after virama) # -------------------------------------------------------------------------------------- _C = '[क-हक़-य़ॹ-ॿऩऱऴ]' _V = '[ऄ-औॠॡॲ-ॷ]' _S = '[ऺऻा-ौॎॏॕ-ॗॢॣ]' _A = '[ऀ-ः॑-॔᳐-᳨ᳵᳶ꣠-꣱]' AKSHARA_RE = ( f'(?:{_C}़?(?:्[‌‍]?{_C}़?)*(?:{_S}|्)?|{_V}़?{_S}?){_A}*' ) def split_aksharas(text: str): """Devanagari orthographic syllables; anything else comes back one code point per item.""" import regex pat = regex.compile(f'{AKSHARA_RE}|.', regex.DOTALL) return [m.group(0) for m in pat.finditer(text)] if __name__ == '__main__': import sys for line in (sys.argv[1:] or ['धर्मक्षेत्रे कुरुक्षेत्रे समवेता युयुत्सवः ॥ १ ॥']): s, us = deva_to_slp1(line) a, ua = deva_to_iast(line) print('deva :', line) print('slp1 :', s, ' unmapped', dict(us)) print('iast :', a, ' unmapped', dict(ua)) print('back :', slp1_to_deva(s)[0] == NFC(line), iast_to_deva(a)[0] == NFC(line))