sansar-60m / translit.py

v0.1.0: F0 (ex-Gita 0.6434)

bcd28bc verified 18 days ago

22.9 kB

#!/usr/bin/env python3
"""Strict, table-driven, REVERSIBLE Devanagari <-> SLP1 and Devanagari <-> IAST.
Design rules (E1 of docs/TOKENIZATION.md):
1. NFC is the frozen normalisation. Every public function normalises its input to NFC
   first and returns NFC output. NFC *decomposes* the precomposed nukta letters
   U+0958-U+095F, U+0929, U+0931, U+0934 (they are composition-excluded), so after NFC a
   nukta is always a separate U+093C. Both forms are in the tables anyway.
2. Fail closed, never drop. A code point with no mapping is passed through unchanged AND
   reported: every function returns ``(text, unmapped)`` where ``unmapped`` is a
   ``collections.Counter`` keyed by the offending character. Nothing is ever silently lost.
3. Round trip is the contract. deva -> slp1 -> deva and deva -> iast -> deva must return
   the NFC input exactly, for every input, *unless* the input contains Latin letters
   (see 5) or malformed Devanagari, both of which are reported.
4. Reserved ASCII. The romanised sides use a few ASCII characters as functional markers:
       ^  nukta (follows the consonant, precedes the vowel: k^a = U+0915 U+093C)
       '  avagraha                     %  om (U+0950)
       |  cluster separator, only emitted where IAST would otherwise be ambiguous
          (k|ha = क्ह, not ख; ka|i = कइ, not कै). It is only ever emitted BETWEEN two
          letters, so on input a '|' that touches whitespace is read as a literal
          character -- that is the ASCII dandaa of GRETIL-style romanised sources.
       ~  candrabindu   (SLP1 only)    /  udatta U+0951  (SLP1 only)
       +  short vowel prefix: +e +o +a (SLP1 only; IAST uses ĕ ŏ ă)
       *  candra vowel prefix: *e *o *a (SLP1 only; IAST uses ê ô â)
       \\  anudatta U+0952 (SLP1 only)
   IAST instead writes candrabindu m̐, udatta as combining acute U+0301 and anudatta as
   combining grave U+0300. A *literal* occurrence of any reserved character in the source
   is escaped by doubling it, so the mapping stays one-to-one.
5. Latin letters are the one thing that cannot round trip. On the Devanagari side every
   Latin letter is a legal SLP1/IAST token, so "cafe" inside a Devanagari record comes back
   as चफे. Such characters are reported (class ``latin``) rather than hidden; E1 counts how
   many records of the corpus are affected.
6. Things deliberately NOT transliterated (identity in both directions, never reported):
   danda U+0964, double danda U+0965, Devanagari digits U+0966-U+096F, the abbreviation
   sign U+0970 and U+0971, ZWJ U+200D and ZWNJ U+200C, whitespace, and ASCII
   punctuation/digits. SLP1 has no agreed spelling for any of them and inventing one would
   be lossy -- and giving ZWJ/ZWNJ ASCII markers collided with the literal '_' that the DCS
   and VedaWeb use as a text marker.
   Parsing additionally accepts, but never emits, the spellings real-world romanised
   Sanskrit actually uses: ṁ for the anusvara, and the ISO-15919 vowels ē ō r̥ l̥ r̥̄ l̥̄.
7. Vedic Extensions U+1CD0-U+1CFF are reported as unmapped and passed through, except
   the two that are clearly SLP1 letters: jihvamuliya U+1CF5 (Z / ẖ) and upadhmaniya
   U+1CF6 (V / ḫ). Devanagari Extended U+A8E0-U+A8FF is likewise reported and preserved.
8. ऌ and ळ are different letters and get different spellings, which published IAST often
   conflates: vocalic ऌ/ॡ are ḷ/ḹ (SLP1 x/X) and retroflex ळ is ḻ, U+1E3B (SLP1 L). ISO-15919
   uses ḷ for ळ; on input that reading loses to IAST's, so feed ISO through
   :func:`iso15919_to_iast` first.
Philological note on the Vedic accents: the Rigveda's Devanagari convention marks
*anudatta* below (U+0952) and *svarita* above (U+0951) and leaves udatta unmarked, while
the romanised convention puts an acute on the *udatta* vowel. This module maps the signs,
not the accent theory: U+0951 <-> acute, U+0952 <-> grave, as specified for E1. Converting
a romanised Vedic text therefore produces a Devanagari string that is faithful as a
character mapping but does not match the printed Rigveda's accent placement.
The three-line summary of what the encodings look like:
    धर्मक्षेत्रे कुरुक्षेत्रे    Devanagari
    Darmakzetre kurukzetre     SLP1
    dharmakṣetre kurukṣetre    IAST
"""
from __future__ import annotations

import unicodedata
from collections import Counter

__all__ = [
    'NFC', 'deva_to_slp1', 'slp1_to_deva', 'deva_to_iast', 'iast_to_deva',
    'iso15919_to_iast', 'classify_char', 'AKSHARA_RE', 'split_aksharas', 'RESERVED',
]

VIRAMA = '्'
NUKTA = '़'
ZWNJ = '‌'
ZWJ = '‍'

RESERVED = "^'%|~/\\+*"

# --------------------------------------------------------------------------------------
# Tables.  (devanagari, slp1, iast)
# --------------------------------------------------------------------------------------

CONSONANTS = [
    ('क', 'k', 'k'), ('ख', 'K', 'kh'), ('ग', 'g', 'g'), ('घ', 'G', 'gh'),
    ('ङ', 'N', 'ṅ'),
    ('च', 'c', 'c'), ('छ', 'C', 'ch'), ('ज', 'j', 'j'), ('झ', 'J', 'jh'),
    ('ञ', 'Y', 'ñ'),
    ('ट', 'w', 'ṭ'), ('ठ', 'W', 'ṭh'), ('ड', 'q', 'ḍ'), ('ढ', 'Q', 'ḍh'),
    ('ण', 'R', 'ṇ'),
    ('त', 't', 't'), ('थ', 'T', 'th'), ('द', 'd', 'd'), ('ध', 'D', 'dh'),
    ('न', 'n', 'n'),
    ('प', 'p', 'p'), ('फ', 'P', 'ph'), ('ब', 'b', 'b'), ('भ', 'B', 'bh'),
    ('म', 'm', 'm'),
    ('य', 'y', 'y'), ('र', 'r', 'r'), ('ल', 'l', 'l'), ('व', 'v', 'v'),
    ('श', 'S', 'ś'), ('ष', 'z', 'ṣ'), ('स', 's', 's'), ('ह', 'h', 'h'),
    ('ळ', 'L', 'ḻ'),                      # ळ  -> l with line below
]

# (independent, dependent sign or None, slp1, iast)
VOWELS = [
    ('अ', None,       'a',  'a'),
    ('आ', 'ा',   'A',  'ā'),
    ('इ', 'ि',   'i',  'i'),
    ('ई', 'ी',   'I',  'ī'),
    ('उ', 'ु',   'u',  'u'),
    ('ऊ', 'ू',   'U',  'ū'),
    ('ऋ', 'ृ',   'f',  'ṛ'),
    ('ॠ', 'ॄ',   'F',  'ṝ'),
    ('ऌ', 'ॢ',   'x',  'ḷ'),
    ('ॡ', 'ॣ',   'X',  'ḹ'),
    ('ए', 'े',   'e',  'e'),
    ('ऐ', 'ै',   'E',  'ai'),
    ('ओ', 'ो',   'o',  'o'),
    ('औ', 'ौ',   'O',  'au'),
    ('ऎ', 'ॆ',   '+e', 'ĕ'),              # short e
    ('ऒ', 'ॊ',   '+o', 'ŏ'),              # short o
    ('ऄ', None,       '+a', 'ă'),              # short a
    ('ऍ', 'ॅ',   '*e', 'ê'),              # candra e
    ('ऑ', 'ॉ',   '*o', 'ô'),              # candra o
    ('ॲ', None,       '*a', 'â'),              # candra a
]

# marks that stand on their own (never fuse with the inherent vowel)
MARKS = [
    ('ं', 'M',  'ṃ'),          # anusvara
    ('ः', 'H',  'ḥ'),          # visarga
    ('ँ', '~',  'm̐'),    # candrabindu
    ('ऽ', "'",  "'"),          # avagraha
    ('ॐ', '%',  '%'),          # om
    ('॑', '/',  '́'),     # udatta  <-> combining acute
    ('॒', '\\', '̀'),     # anudatta <-> combining grave
    ('ᳵ', 'Z',  'ẖ'),     # jihvamuliya
    ('ᳶ', 'V',  'ḫ'),     # upadhmaniya
]

# identity on both sides, never reported.  ZWJ/ZWNJ are here rather than in MARKS on
# purpose: giving them ASCII markers would collide with the literal '_' that DCS and
# VedaWeb use as a text marker, and they survive a round trip untouched as themselves.
PASSTHROUGH = set('।॥॰ॱ') | {ZWJ, ZWNJ} | {chr(c) for c in range(0x0966, 0x0970)}

# precomposed nukta letters -> base + U+093C (NFC already does this; belt and braces)
PRECOMPOSED_NUKTA = {
    'ऩ': 'न' + NUKTA, 'ऱ': 'र' + NUKTA, 'ऴ': 'ळ' + NUKTA,
    'क़': 'क' + NUKTA, 'ख़': 'ख' + NUKTA, 'ग़': 'ग' + NUKTA,
    'ज़': 'ज' + NUKTA, 'ड़': 'ड' + NUKTA, 'ढ़': 'ढ' + NUKTA,
    'फ़': 'फ' + NUKTA, 'य़': 'य' + NUKTA,
}

_SLP1, _IAST = 0, 1

CONS_FWD = {d: (s, i) for d, s, i in CONSONANTS}
VSIGN_FWD = {sign: (s, i) for _, sign, s, i in VOWELS if sign}
VIND_FWD = {ind: (s, i) for ind, _, s, i in VOWELS}
MARK_FWD = {d: (s, i) for d, s, i in MARKS}

# reverse: value -> devanagari
CONS_REV = [{s: d for d, s, i in CONSONANTS}, {i: d for d, s, i in CONSONANTS}]
VOWEL_REV = [{s: (ind, sign) for ind, sign, s, i in VOWELS},
             {i: (ind, sign) for ind, sign, s, i in VOWELS}]
MARK_REV = [{s: d for d, s, i in MARKS}, {i: d for d, s, i in MARKS}]

def _straddles(prev: str, nxt: str, enc: int) -> bool:
    """Would the junction of two emitted pieces re-parse as a different token?
    k + ha reads back as ख, ka + i as कै, ka + ī as कै + a stray macron.  Checked in NFD,
    which is what the parser matches on.  SLP1 has only single-character tokens, so this is
    always False there."""
    m = MAXTOK[enc]
    if m < 2 or not prev or not nxt:
        return False
    a = unicodedata.normalize('NFD', prev)
    b = unicodedata.normalize('NFD', nxt)
    trie = TRIE_NFD[enc]
    for i in range(1, min(len(a), m - 1) + 1):
        for j in range(1, min(len(b), m - i) + 1):
            if a[-i:] + b[:j] in trie:
                return True
    return False

# functional meaning of each reserved character, per encoding
_FUNC_COMMON = {'^': ('nukta', None), '|': ('sep', None)}
FUNC = [
    dict(_FUNC_COMMON, **{c: ('mark', d) for d, c, _ in MARKS if c in RESERVED},
         **{'+': ('vmod', None), '*': ('vmod', None)}),
    dict(_FUNC_COMMON, **{c: ('mark', d) for d, _, c in MARKS if c in RESERVED}),
]


def _build_trie(enc):
    """token -> (kind, payload); longest match wins.  Reserved chars are handled separately."""
    t = {}
    for d, s, i in CONSONANTS:
        t[(s, i)[enc]] = ('cons', d)
    for ind, sign, s, i in VOWELS:
        v = (s, i)[enc]
        if v[0] not in RESERVED:
            t[v] = ('vowel', (ind, sign))
    for d, s, i in MARKS:
        v = (s, i)[enc]
        if v[0] not in RESERVED:
            t[v] = ('mark', d)
    for c in PASSTHROUGH:
        t[c] = ('pass', c)
    return t


# Input-only spellings.  Real-world "IAST" is rarely pure: GRETIL and the DCS write the
# anusvara as ṁ, VedaWeb writes ISO-15919 vowels.  These are accepted when parsing and never
# emitted, so the canonical round trip is unaffected while iast -> deva stops mangling them.
# ISO's ḷ (= ळ) is NOT aliased: in IAST ḷ is vocalic ऌ and that reading wins.
IAST_INPUT_ALIASES = {
    'ṁ': ('mark', 'ं'),
    'ē': ('vowel', ('ए', 'े')), 'ō': ('vowel', ('ओ', 'ो')),
    'r̥': ('vowel', ('ऋ', 'ृ')), 'l̥': ('vowel', ('ऌ', 'ॢ')),
    'r̥̄': ('vowel', ('ॠ', 'ॄ')), 'l̥̄': ('vowel', ('ॡ', 'ॣ')),
    'ṝ': ('vowel', ('ॠ', 'ॄ')),
}

TRIE = [_build_trie(_SLP1), _build_trie(_IAST)]
# parsing runs on NFD so that composed letters (ī = i + macron) line up with combining
# accents; build the NFD tries once, not per call
TRIE_NFD = [{unicodedata.normalize('NFD', k): v for k, v in t.items()} for t in TRIE]
MAXTOK = [max(len(k) for k in t) for t in TRIE_NFD]
# what the parser matches on: canonical tokens plus the input-only aliases
TRIE_PARSE = [dict(TRIE_NFD[0]),
              dict(TRIE_NFD[1], **{unicodedata.normalize('NFD', k): v
                                   for k, v in IAST_INPUT_ALIASES.items()})]
MAXTOK_PARSE = [max(len(k) for k in t) for t in TRIE_PARSE]
# characters that a passthrough would collide with when read back
RISKY = [set(k[0] for k in TRIE[0]) - PASSTHROUGH, set(k[0] for k in TRIE[1]) - PASSTHROUGH]


def NFC(text: str) -> str:
    return unicodedata.normalize('NFC', text)


def classify_char(ch: str) -> str:
    """Bucket an unmapped character for reporting."""
    cp = ord(ch)
    if 0x0900 <= cp <= 0x097F:
        return 'devanagari_unmapped'
    if 0x1CD0 <= cp <= 0x1CFF:
        return 'vedic_extensions'
    if 0xA8E0 <= cp <= 0xA8FF:
        return 'devanagari_extended'
    if cp < 0x0250 and ch.isalpha():
        return 'latin'
    if unicodedata.combining(ch):
        return 'combining'
    if ch.isalpha():
        return 'other_script'
    return 'other'


def _safe_pass(ch: str, enc: int) -> bool:
    """True if passing this character through is reversible and unremarkable."""
    if ch in PASSTHROUGH:
        return True
    if ch in RESERVED:
        return True            # escaped by doubling, so reversible
    if ch in RISKY[enc]:
        return False
    cat = unicodedata.category(ch)
    return cat[0] in 'ZPSN' or ch in '\n\t\r' or cat == 'Cc'


# --------------------------------------------------------------------------------------
# Devanagari -> romanised
# --------------------------------------------------------------------------------------

def _to_roman(text: str, enc: int):
    text = NFC(text)
    unmapped: Counter = Counter()
    out: list[str] = []
    inh = None                     # index in out of the inherent 'a' still open for replacement
    bare = False                   # last piece was a consonant closed by virama

    def put(piece: str):
        """Append a piece, inserting the '|' separator wherever the junction would re-parse.
        Two cases: an IAST digraph straddling the junction (k + ha -> k|ha), and a reserved
        marker landing next to the same marker, where the doubling escape would otherwise
        read the pair as a literal (avagraha + avagraha -> '|', not '')."""
        if out and out[-1] and piece:
            a, b = out[-1][-1], piece[0]
            if (a == b and b in RESERVED and b != '|') or _straddles(out[-1], piece, enc):
                out.append('|')
        out.append(piece)

    def raw(ch: str):
        nonlocal inh, bare
        inh, bare = None, False
        put(ch + ch if ch in RESERVED else ch)

    i, n = 0, len(text)
    while i < n:
        ch = text[i]
        i += 1
        if ch in PRECOMPOSED_NUKTA:            # only reachable if caller skipped NFC
            text = text[:i - 1] + PRECOMPOSED_NUKTA[ch] + text[i:]
            n = len(text)
            i -= 1
            continue
        if ch in CONS_FWD:
            put(CONS_FWD[ch][enc])
            out.append('a')
            inh = len(out) - 1
            bare = False
        elif ch == NUKTA:
            # a second nukta on the same consonant would emit '^^', which reads back as an
            # escaped literal '^'; refuse rather than lose it silently
            if inh is not None and inh == len(out) - 1 and out[inh - 1] != '^':
                out.insert(inh, '^')
                inh += 1
            else:
                unmapped[ch] += 1
                raw(ch)
        elif ch in VSIGN_FWD:
            if inh is not None and inh == len(out) - 1:
                v = VSIGN_FWD[ch][enc]
                out[inh] = ('|' + v) if (len(out) > 1 and _straddles(out[-2], v, enc)) else v
                inh = None
            else:
                unmapped[ch] += 1
                raw(ch)
        elif ch == VIRAMA:
            if inh is not None and inh == len(out) - 1:
                out.pop()
                inh = None
                bare = True
            else:
                unmapped[ch] += 1
                raw(ch)
        elif ch in VIND_FWD:
            # an independent vowel right after a bare consonant would be read back as that
            # consonant's dependent sign (र् + ऋ -> "rf" -> रृ), so separate them
            if bare:
                out.append('|')
            inh, bare = None, False
            put(VIND_FWD[ch][enc])
        elif ch in MARK_FWD:
            inh, bare = None, False
            put(MARK_FWD[ch][enc])
        elif ch in PASSTHROUGH:
            inh, bare = None, False
            out.append(ch)
        else:
            if not _safe_pass(ch, enc):
                unmapped[ch] += 1
            raw(ch)
    return NFC(''.join(out)), unmapped


def deva_to_slp1(text: str):
    """Devanagari -> SLP1.  Returns (slp1, Counter of unmapped characters)."""
    return _to_roman(text, _SLP1)


def deva_to_iast(text: str):
    """Devanagari -> IAST.  Returns (iast, Counter of unmapped characters)."""
    return _to_roman(text, _IAST)


# --------------------------------------------------------------------------------------
# romanised -> Devanagari
# --------------------------------------------------------------------------------------

def _to_deva(text: str, enc: int):
    text = unicodedata.normalize('NFD', NFC(text))
    unmapped: Counter = Counter()
    trie, maxtok, func = TRIE_PARSE[enc], MAXTOK_PARSE[enc], FUNC[enc]
    out: list[str] = []
    pend = None            # consonant awaiting its vowel
    nukta = False

    def close():
        nonlocal pend, nukta
        if pend is not None:
            out.append(pend)
            if nukta:
                out.append(NUKTA)
            out.append(VIRAMA)
            pend, nukta = None, False

    def vowel(ind, sign):
        nonlocal pend, nukta
        if pend is not None:
            out.append(pend)
            if nukta:
                out.append(NUKTA)
            if sign is not None:
                out.append(sign)
            elif ind != 'अ':
                unmapped[ind] += 1          # no dependent form (ऄ, ॲ) after a consonant
                out.append(VIRAMA)
                out.append(ind)
            pend, nukta = None, False
        else:
            out.append(ind)

    i, n = 0, len(text)
    while i < n:
        ch = text[i]
        if ch in RESERVED:
            if i + 1 < n and text[i + 1] == ch:      # escaped literal
                close()
                out.append(ch)
                i += 2
                continue
            kind, payload = func.get(ch, (None, None))
            if kind == 'nukta' and pend is not None and not nukta:
                nukta = True
                i += 1
                continue
            if kind == 'sep':
                # Our emitter only ever puts the separator BETWEEN two letters, so a '|' that
                # touches whitespace is not ours: it is the ASCII dandaa that GRETIL-style
                # romanised sources use. Keep it (and report it) instead of swallowing it.
                if (text[i - 1] if i else ' ').isspace() or \
                        (text[i + 1] if i + 1 < n else ' ').isspace():
                    unmapped[ch] += 1
                    close()
                    out.append(ch)
                else:
                    close()
                i += 1
                continue
            if kind == 'vmod':
                tok = text[i:i + 2]
                hit = VOWEL_REV[enc].get(tok)
                if hit:
                    vowel(*hit)
                    i += 2
                    continue
            if kind == 'mark':
                close()
                out.append(payload)
                i += 1
                continue
            unmapped[ch] += 1
            close()
            out.append(ch)
            i += 1
            continue
        for L in range(min(maxtok, n - i), 0, -1):
            hit = trie.get(text[i:i + L])
            if hit:
                kind, payload = hit
                if kind == 'cons':
                    close()
                    pend = payload
                elif kind == 'vowel':
                    vowel(*payload)
                else:
                    close()
                    out.append(payload)
                i += L
                break
        else:
            if not _safe_pass(ch, enc):
                unmapped[ch] += 1
            close()
            out.append(ch)
            i += 1
    close()
    return NFC(''.join(out)), unmapped


def slp1_to_deva(text: str):
    """SLP1 -> Devanagari.  Returns (devanagari, Counter of unmapped characters)."""
    return _to_deva(text, _SLP1)


def iast_to_deva(text: str):
    """IAST -> Devanagari.  Returns (devanagari, Counter of unmapped characters)."""
    return _to_deva(text, _IAST)


# --------------------------------------------------------------------------------------
# ISO-15919 (as VedaWeb writes it) -> our canonical IAST
# --------------------------------------------------------------------------------------
# Applied on NFD so that the pitch accents (combining acute/grave) survive on the vowel.
_ISO_PAIRS = [
    ('r̥̄', 'ṝ'), ('l̥̄', 'ḹ'),      # r̥̄ l̥̄ -> ṝ ḹ
    ('r̥', 'ṛ'), ('l̥', 'ḷ'),                  # r̥ l̥  -> ṛ ḷ
    ('ḷ', 'ḻ'),                                          # ISO ḷ is ळ, not vocalic l
    ('ṁ', 'ṃ'),                                          # ṁ -> ṃ
    ('ē', 'e'), ('ō', 'o'),                            # ē ō -> e o
    ('ṉ', 'n'), ('ṟ', 'r'),                              # ṉ ṟ (nukta-ish ISO) -> n r
]
_ISO_ESCAPE = set(RESERVED) - {"'"}          # keep ' meaning avagraha


def iso15919_to_iast(text: str):
    """VedaWeb-style ISO-15919 (with pitch accents) -> our strict IAST.
    Reserved characters other than the avagraha apostrophe are escaped by doubling, so the
    result is a legal input to :func:`iast_to_deva`.
    """
    src = unicodedata.normalize('NFD', NFC(text))
    # longest source first so r̥̄ beats r̥
    pairs = sorted(_ISO_PAIRS, key=lambda p: -len(unicodedata.normalize('NFD', p[0])))
    pairs = [(unicodedata.normalize('NFD', a), b) for a, b in pairs]
    out, i, n = [], 0, len(src)
    unmapped: Counter = Counter()
    while i < n:
        for a, b in pairs:
            if src.startswith(a, i):
                out.append(b)
                i += len(a)
                break
        else:
            ch = src[i]
            out.append(ch + ch if ch in _ISO_ESCAPE else ch)
            i += 1
    return NFC(''.join(out)), unmapped


# --------------------------------------------------------------------------------------
# Akshara segmentation (explicit Devanagari regex; NOT \X, which breaks after virama)
# --------------------------------------------------------------------------------------
_C = '[क-हक़-य़ॹ-ॿऩऱऴ]'
_V = '[ऄ-औॠॡॲ-ॷ]'
_S = '[ऺऻा-ौॎॏॕ-ॗॢॣ]'
_A = '[ऀ-ः॑-॔᳐-᳨ᳵᳶ꣠-꣱]'
AKSHARA_RE = (
    f'(?:{_C}़?(?:्[‌‍]?{_C}़?)*(?:{_S}|्)?|{_V}़?{_S}?){_A}*'
)


def split_aksharas(text: str):
    """Devanagari orthographic syllables; anything else comes back one code point per item."""
    import regex
    pat = regex.compile(f'{AKSHARA_RE}|.', regex.DOTALL)
    return [m.group(0) for m in pat.finditer(text)]


if __name__ == '__main__':
    import sys
    for line in (sys.argv[1:] or ['धर्मक्षेत्रे कुरुक्षेत्रे समवेता युयुत्सवः ॥ १ ॥']):
        s, us = deva_to_slp1(line)
        a, ua = deva_to_iast(line)
        print('deva :', line)
        print('slp1 :', s, '  unmapped', dict(us))
        print('iast :', a, '  unmapped', dict(ua))
        print('back :', slp1_to_deva(s)[0] == NFC(line), iast_to_deva(a)[0] == NFC(line))

	#!/usr/bin/env python3
	"""Strict, table-driven, REVERSIBLE Devanagari <-> SLP1 and Devanagari <-> IAST.

	Design rules (E1 of docs/TOKENIZATION.md):

	1. NFC is the frozen normalisation. Every public function normalises its input to NFC
	first and returns NFC output. NFC decomposes the precomposed nukta letters
	U+0958-U+095F, U+0929, U+0931, U+0934 (they are composition-excluded), so after NFC a
	nukta is always a separate U+093C. Both forms are in the tables anyway.
	2. Fail closed, never drop. A code point with no mapping is passed through unchanged AND
	reported: every function returns ``(text, unmapped)`` where ``unmapped`` is a
	``collections.Counter`` keyed by the offending character. Nothing is ever silently lost.
	3. Round trip is the contract. deva -> slp1 -> deva and deva -> iast -> deva must return
	the NFC input exactly, for every input, unless the input contains Latin letters
	(see 5) or malformed Devanagari, both of which are reported.
	4. Reserved ASCII. The romanised sides use a few ASCII characters as functional markers:

	^ nukta (follows the consonant, precedes the vowel: k^a = U+0915 U+093C)
	' avagraha % om (U+0950)
	\| cluster separator, only emitted where IAST would otherwise be ambiguous
	(k\|ha = क्ह, not ख; ka\|i = कइ, not कै). It is only ever emitted BETWEEN two
	letters, so on input a '\|' that touches whitespace is read as a literal
	character -- that is the ASCII dandaa of GRETIL-style romanised sources.
	~ candrabindu (SLP1 only) / udatta U+0951 (SLP1 only)
	+ short vowel prefix: +e +o +a (SLP1 only; IAST uses ĕ ŏ ă)
	* candra vowel prefix: e o *a (SLP1 only; IAST uses ê ô â)
	\\ anudatta U+0952 (SLP1 only)

	IAST instead writes candrabindu m̐, udatta as combining acute U+0301 and anudatta as
	combining grave U+0300. A literal occurrence of any reserved character in the source
	is escaped by doubling it, so the mapping stays one-to-one.
	5. Latin letters are the one thing that cannot round trip. On the Devanagari side every
	Latin letter is a legal SLP1/IAST token, so "cafe" inside a Devanagari record comes back
	as चफे. Such characters are reported (class ``latin``) rather than hidden; E1 counts how
	many records of the corpus are affected.
	6. Things deliberately NOT transliterated (identity in both directions, never reported):
	danda U+0964, double danda U+0965, Devanagari digits U+0966-U+096F, the abbreviation
	sign U+0970 and U+0971, ZWJ U+200D and ZWNJ U+200C, whitespace, and ASCII
	punctuation/digits. SLP1 has no agreed spelling for any of them and inventing one would
	be lossy -- and giving ZWJ/ZWNJ ASCII markers collided with the literal '_' that the DCS
	and VedaWeb use as a text marker.
	Parsing additionally accepts, but never emits, the spellings real-world romanised
	Sanskrit actually uses: ṁ for the anusvara, and the ISO-15919 vowels ē ō r̥ l̥ r̥̄ l̥̄.
	7. Vedic Extensions U+1CD0-U+1CFF are reported as unmapped and passed through, except
	the two that are clearly SLP1 letters: jihvamuliya U+1CF5 (Z / ẖ) and upadhmaniya
	U+1CF6 (V / ḫ). Devanagari Extended U+A8E0-U+A8FF is likewise reported and preserved.
	8. ऌ and ळ are different letters and get different spellings, which published IAST often
	conflates: vocalic ऌ/ॡ are ḷ/ḹ (SLP1 x/X) and retroflex ळ is ḻ, U+1E3B (SLP1 L). ISO-15919
	uses ḷ for ळ; on input that reading loses to IAST's, so feed ISO through
	:func:`iso15919_to_iast` first.

	Philological note on the Vedic accents: the Rigveda's Devanagari convention marks
	anudatta below (U+0952) and svarita above (U+0951) and leaves udatta unmarked, while
	the romanised convention puts an acute on the udatta vowel. This module maps the signs,
	not the accent theory: U+0951 <-> acute, U+0952 <-> grave, as specified for E1. Converting
	a romanised Vedic text therefore produces a Devanagari string that is faithful as a
	character mapping but does not match the printed Rigveda's accent placement.

	The three-line summary of what the encodings look like:

	धर्मक्षेत्रे कुरुक्षेत्रे Devanagari
	Darmakzetre kurukzetre SLP1
	dharmakṣetre kurukṣetre IAST
	"""
	from __future__ import annotations

	import unicodedata
	from collections import Counter

	__all__ = [
	'NFC', 'deva_to_slp1', 'slp1_to_deva', 'deva_to_iast', 'iast_to_deva',
	'iso15919_to_iast', 'classify_char', 'AKSHARA_RE', 'split_aksharas', 'RESERVED',
	]

	VIRAMA = '्'
	NUKTA = '़'
	ZWNJ = '‌'
	ZWJ = '‍'

	RESERVED = "^'%\|~/\\+*"

	# --------------------------------------------------------------------------------------
	# Tables. (devanagari, slp1, iast)
	# --------------------------------------------------------------------------------------

	CONSONANTS = [
	('क', 'k', 'k'), ('ख', 'K', 'kh'), ('ग', 'g', 'g'), ('घ', 'G', 'gh'),
	('ङ', 'N', 'ṅ'),
	('च', 'c', 'c'), ('छ', 'C', 'ch'), ('ज', 'j', 'j'), ('झ', 'J', 'jh'),
	('ञ', 'Y', 'ñ'),
	('ट', 'w', 'ṭ'), ('ठ', 'W', 'ṭh'), ('ड', 'q', 'ḍ'), ('ढ', 'Q', 'ḍh'),
	('ण', 'R', 'ṇ'),
	('त', 't', 't'), ('थ', 'T', 'th'), ('द', 'd', 'd'), ('ध', 'D', 'dh'),
	('न', 'n', 'n'),
	('प', 'p', 'p'), ('फ', 'P', 'ph'), ('ब', 'b', 'b'), ('भ', 'B', 'bh'),
	('म', 'm', 'm'),
	('य', 'y', 'y'), ('र', 'r', 'r'), ('ल', 'l', 'l'), ('व', 'v', 'v'),
	('श', 'S', 'ś'), ('ष', 'z', 'ṣ'), ('स', 's', 's'), ('ह', 'h', 'h'),
	('ळ', 'L', 'ḻ'), # ळ -> l with line below
	]

	# (independent, dependent sign or None, slp1, iast)
	VOWELS = [
	('अ', None, 'a', 'a'),
	('आ', 'ा', 'A', 'ā'),
	('इ', 'ि', 'i', 'i'),
	('ई', 'ी', 'I', 'ī'),
	('उ', 'ु', 'u', 'u'),
	('ऊ', 'ू', 'U', 'ū'),
	('ऋ', 'ृ', 'f', 'ṛ'),
	('ॠ', 'ॄ', 'F', 'ṝ'),
	('ऌ', 'ॢ', 'x', 'ḷ'),
	('ॡ', 'ॣ', 'X', 'ḹ'),
	('ए', 'े', 'e', 'e'),
	('ऐ', 'ै', 'E', 'ai'),
	('ओ', 'ो', 'o', 'o'),
	('औ', 'ौ', 'O', 'au'),
	('ऎ', 'ॆ', '+e', 'ĕ'), # short e
	('ऒ', 'ॊ', '+o', 'ŏ'), # short o
	('ऄ', None, '+a', 'ă'), # short a
	('ऍ', 'ॅ', '*e', 'ê'), # candra e
	('ऑ', 'ॉ', '*o', 'ô'), # candra o
	('ॲ', None, '*a', 'â'), # candra a
	]

	# marks that stand on their own (never fuse with the inherent vowel)
	MARKS = [
	('ं', 'M', 'ṃ'), # anusvara
	('ः', 'H', 'ḥ'), # visarga
	('ँ', '~', 'm̐'), # candrabindu
	('ऽ', "'", "'"), # avagraha
	('ॐ', '%', '%'), # om
	('॑', '/', '́'), # udatta <-> combining acute
	('॒', '\\', '̀'), # anudatta <-> combining grave
	('ᳵ', 'Z', 'ẖ'), # jihvamuliya
	('ᳶ', 'V', 'ḫ'), # upadhmaniya
	]

	# identity on both sides, never reported. ZWJ/ZWNJ are here rather than in MARKS on
	# purpose: giving them ASCII markers would collide with the literal '_' that DCS and
	# VedaWeb use as a text marker, and they survive a round trip untouched as themselves.
	PASSTHROUGH = set('।॥॰ॱ') \| {ZWJ, ZWNJ} \| {chr(c) for c in range(0x0966, 0x0970)}

	# precomposed nukta letters -> base + U+093C (NFC already does this; belt and braces)
	PRECOMPOSED_NUKTA = {
	'ऩ': 'न' + NUKTA, 'ऱ': 'र' + NUKTA, 'ऴ': 'ळ' + NUKTA,
	'क़': 'क' + NUKTA, 'ख़': 'ख' + NUKTA, 'ग़': 'ग' + NUKTA,
	'ज़': 'ज' + NUKTA, 'ड़': 'ड' + NUKTA, 'ढ़': 'ढ' + NUKTA,
	'फ़': 'फ' + NUKTA, 'य़': 'य' + NUKTA,
	}

	_SLP1, _IAST = 0, 1

	CONS_FWD = {d: (s, i) for d, s, i in CONSONANTS}
	VSIGN_FWD = {sign: (s, i) for _, sign, s, i in VOWELS if sign}
	VIND_FWD = {ind: (s, i) for ind, _, s, i in VOWELS}
	MARK_FWD = {d: (s, i) for d, s, i in MARKS}

	# reverse: value -> devanagari
	CONS_REV = [{s: d for d, s, i in CONSONANTS}, {i: d for d, s, i in CONSONANTS}]
	VOWEL_REV = [{s: (ind, sign) for ind, sign, s, i in VOWELS},
	{i: (ind, sign) for ind, sign, s, i in VOWELS}]
	MARK_REV = [{s: d for d, s, i in MARKS}, {i: d for d, s, i in MARKS}]

	def _straddles(prev: str, nxt: str, enc: int) -> bool:
	"""Would the junction of two emitted pieces re-parse as a different token?

	k + ha reads back as ख, ka + i as कै, ka + ī as कै + a stray macron. Checked in NFD,
	which is what the parser matches on. SLP1 has only single-character tokens, so this is
	always False there."""
	m = MAXTOK[enc]
	if m < 2 or not prev or not nxt:
	return False
	a = unicodedata.normalize('NFD', prev)
	b = unicodedata.normalize('NFD', nxt)
	trie = TRIE_NFD[enc]
	for i in range(1, min(len(a), m - 1) + 1):
	for j in range(1, min(len(b), m - i) + 1):
	if a[-i:] + b[:j] in trie:
	return True
	return False

	# functional meaning of each reserved character, per encoding
	_FUNC_COMMON = {'^': ('nukta', None), '\|': ('sep', None)}
	FUNC = [
	dict(_FUNC_COMMON, **{c: ('mark', d) for d, c, _ in MARKS if c in RESERVED},
	*{'+': ('vmod', None), '': ('vmod', None)}),
	dict(_FUNC_COMMON, **{c: ('mark', d) for d, _, c in MARKS if c in RESERVED}),
	]


	def _build_trie(enc):
	"""token -> (kind, payload); longest match wins. Reserved chars are handled separately."""
	t = {}
	for d, s, i in CONSONANTS:
	t[(s, i)[enc]] = ('cons', d)
	for ind, sign, s, i in VOWELS:
	v = (s, i)[enc]
	if v[0] not in RESERVED:
	t[v] = ('vowel', (ind, sign))
	for d, s, i in MARKS:
	v = (s, i)[enc]
	if v[0] not in RESERVED:
	t[v] = ('mark', d)
	for c in PASSTHROUGH:
	t[c] = ('pass', c)
	return t


	# Input-only spellings. Real-world "IAST" is rarely pure: GRETIL and the DCS write the
	# anusvara as ṁ, VedaWeb writes ISO-15919 vowels. These are accepted when parsing and never
	# emitted, so the canonical round trip is unaffected while iast -> deva stops mangling them.
	# ISO's ḷ (= ळ) is NOT aliased: in IAST ḷ is vocalic ऌ and that reading wins.
	IAST_INPUT_ALIASES = {
	'ṁ': ('mark', 'ं'),
	'ē': ('vowel', ('ए', 'े')), 'ō': ('vowel', ('ओ', 'ो')),
	'r̥': ('vowel', ('ऋ', 'ृ')), 'l̥': ('vowel', ('ऌ', 'ॢ')),
	'r̥̄': ('vowel', ('ॠ', 'ॄ')), 'l̥̄': ('vowel', ('ॡ', 'ॣ')),
	'ṝ': ('vowel', ('ॠ', 'ॄ')),
	}

	TRIE = [_build_trie(_SLP1), _build_trie(_IAST)]
	# parsing runs on NFD so that composed letters (ī = i + macron) line up with combining
	# accents; build the NFD tries once, not per call
	TRIE_NFD = [{unicodedata.normalize('NFD', k): v for k, v in t.items()} for t in TRIE]
	MAXTOK = [max(len(k) for k in t) for t in TRIE_NFD]
	# what the parser matches on: canonical tokens plus the input-only aliases
	TRIE_PARSE = [dict(TRIE_NFD[0]),
	dict(TRIE_NFD[1], **{unicodedata.normalize('NFD', k): v
	for k, v in IAST_INPUT_ALIASES.items()})]
	MAXTOK_PARSE = [max(len(k) for k in t) for t in TRIE_PARSE]
	# characters that a passthrough would collide with when read back
	RISKY = [set(k[0] for k in TRIE[0]) - PASSTHROUGH, set(k[0] for k in TRIE[1]) - PASSTHROUGH]


	def NFC(text: str) -> str:
	return unicodedata.normalize('NFC', text)


	def classify_char(ch: str) -> str:
	"""Bucket an unmapped character for reporting."""
	cp = ord(ch)
	if 0x0900 <= cp <= 0x097F:
	return 'devanagari_unmapped'
	if 0x1CD0 <= cp <= 0x1CFF:
	return 'vedic_extensions'
	if 0xA8E0 <= cp <= 0xA8FF:
	return 'devanagari_extended'
	if cp < 0x0250 and ch.isalpha():
	return 'latin'
	if unicodedata.combining(ch):
	return 'combining'
	if ch.isalpha():
	return 'other_script'
	return 'other'


	def _safe_pass(ch: str, enc: int) -> bool:
	"""True if passing this character through is reversible and unremarkable."""
	if ch in PASSTHROUGH:
	return True
	if ch in RESERVED:
	return True # escaped by doubling, so reversible
	if ch in RISKY[enc]:
	return False
	cat = unicodedata.category(ch)
	return cat[0] in 'ZPSN' or ch in '\n\t\r' or cat == 'Cc'


	# --------------------------------------------------------------------------------------
	# Devanagari -> romanised
	# --------------------------------------------------------------------------------------

	def _to_roman(text: str, enc: int):
	text = NFC(text)
	unmapped: Counter = Counter()
	out: list[str] = []
	inh = None # index in out of the inherent 'a' still open for replacement
	bare = False # last piece was a consonant closed by virama

	def put(piece: str):
	"""Append a piece, inserting the '\|' separator wherever the junction would re-parse.

	Two cases: an IAST digraph straddling the junction (k + ha -> k\|ha), and a reserved
	marker landing next to the same marker, where the doubling escape would otherwise
	read the pair as a literal (avagraha + avagraha -> '\|', not '')."""
	if out and out[-1] and piece:
	a, b = out[-1][-1], piece[0]
	if (a == b and b in RESERVED and b != '\|') or _straddles(out[-1], piece, enc):
	out.append('\|')
	out.append(piece)

	def raw(ch: str):
	nonlocal inh, bare
	inh, bare = None, False
	put(ch + ch if ch in RESERVED else ch)

	i, n = 0, len(text)
	while i < n:
	ch = text[i]
	i += 1
	if ch in PRECOMPOSED_NUKTA: # only reachable if caller skipped NFC
	text = text[:i - 1] + PRECOMPOSED_NUKTA[ch] + text[i:]
	n = len(text)
	i -= 1
	continue
	if ch in CONS_FWD:
	put(CONS_FWD[ch][enc])
	out.append('a')
	inh = len(out) - 1
	bare = False
	elif ch == NUKTA:
	# a second nukta on the same consonant would emit '^^', which reads back as an
	# escaped literal '^'; refuse rather than lose it silently
	if inh is not None and inh == len(out) - 1 and out[inh - 1] != '^':
	out.insert(inh, '^')
	inh += 1
	else:
	unmapped[ch] += 1
	raw(ch)
	elif ch in VSIGN_FWD:
	if inh is not None and inh == len(out) - 1:
	v = VSIGN_FWD[ch][enc]
	out[inh] = ('\|' + v) if (len(out) > 1 and _straddles(out[-2], v, enc)) else v
	inh = None
	else:
	unmapped[ch] += 1
	raw(ch)
	elif ch == VIRAMA:
	if inh is not None and inh == len(out) - 1:
	out.pop()
	inh = None
	bare = True
	else:
	unmapped[ch] += 1
	raw(ch)
	elif ch in VIND_FWD:
	# an independent vowel right after a bare consonant would be read back as that
	# consonant's dependent sign (र् + ऋ -> "rf" -> रृ), so separate them
	if bare:
	out.append('\|')
	inh, bare = None, False
	put(VIND_FWD[ch][enc])
	elif ch in MARK_FWD:
	inh, bare = None, False
	put(MARK_FWD[ch][enc])
	elif ch in PASSTHROUGH:
	inh, bare = None, False
	out.append(ch)
	else:
	if not _safe_pass(ch, enc):
	unmapped[ch] += 1
	raw(ch)
	return NFC(''.join(out)), unmapped


	def deva_to_slp1(text: str):
	"""Devanagari -> SLP1. Returns (slp1, Counter of unmapped characters)."""
	return _to_roman(text, _SLP1)


	def deva_to_iast(text: str):
	"""Devanagari -> IAST. Returns (iast, Counter of unmapped characters)."""
	return _to_roman(text, _IAST)


	# --------------------------------------------------------------------------------------
	# romanised -> Devanagari
	# --------------------------------------------------------------------------------------

	def _to_deva(text: str, enc: int):
	text = unicodedata.normalize('NFD', NFC(text))
	unmapped: Counter = Counter()
	trie, maxtok, func = TRIE_PARSE[enc], MAXTOK_PARSE[enc], FUNC[enc]
	out: list[str] = []
	pend = None # consonant awaiting its vowel
	nukta = False

	def close():
	nonlocal pend, nukta
	if pend is not None:
	out.append(pend)
	if nukta:
	out.append(NUKTA)
	out.append(VIRAMA)
	pend, nukta = None, False

	def vowel(ind, sign):
	nonlocal pend, nukta
	if pend is not None:
	out.append(pend)
	if nukta:
	out.append(NUKTA)
	if sign is not None:
	out.append(sign)
	elif ind != 'अ':
	unmapped[ind] += 1 # no dependent form (ऄ, ॲ) after a consonant
	out.append(VIRAMA)
	out.append(ind)
	pend, nukta = None, False
	else:
	out.append(ind)

	i, n = 0, len(text)
	while i < n:
	ch = text[i]
	if ch in RESERVED:
	if i + 1 < n and text[i + 1] == ch: # escaped literal
	close()
	out.append(ch)
	i += 2
	continue
	kind, payload = func.get(ch, (None, None))
	if kind == 'nukta' and pend is not None and not nukta:
	nukta = True
	i += 1
	continue
	if kind == 'sep':
	# Our emitter only ever puts the separator BETWEEN two letters, so a '\|' that
	# touches whitespace is not ours: it is the ASCII dandaa that GRETIL-style
	# romanised sources use. Keep it (and report it) instead of swallowing it.
	if (text[i - 1] if i else ' ').isspace() or \
	(text[i + 1] if i + 1 < n else ' ').isspace():
	unmapped[ch] += 1
	close()
	out.append(ch)
	else:
	close()
	i += 1
	continue
	if kind == 'vmod':
	tok = text[i:i + 2]
	hit = VOWEL_REV[enc].get(tok)
	if hit:
	vowel(*hit)
	i += 2
	continue
	if kind == 'mark':
	close()
	out.append(payload)
	i += 1
	continue
	unmapped[ch] += 1
	close()
	out.append(ch)
	i += 1
	continue
	for L in range(min(maxtok, n - i), 0, -1):
	hit = trie.get(text[i:i + L])
	if hit:
	kind, payload = hit
	if kind == 'cons':
	close()
	pend = payload
	elif kind == 'vowel':
	vowel(*payload)
	else:
	close()
	out.append(payload)
	i += L
	break
	else:
	if not _safe_pass(ch, enc):
	unmapped[ch] += 1
	close()
	out.append(ch)
	i += 1
	close()
	return NFC(''.join(out)), unmapped


	def slp1_to_deva(text: str):
	"""SLP1 -> Devanagari. Returns (devanagari, Counter of unmapped characters)."""
	return _to_deva(text, _SLP1)


	def iast_to_deva(text: str):
	"""IAST -> Devanagari. Returns (devanagari, Counter of unmapped characters)."""
	return _to_deva(text, _IAST)


	# --------------------------------------------------------------------------------------
	# ISO-15919 (as VedaWeb writes it) -> our canonical IAST
	# --------------------------------------------------------------------------------------
	# Applied on NFD so that the pitch accents (combining acute/grave) survive on the vowel.
	_ISO_PAIRS = [
	('r̥̄', 'ṝ'), ('l̥̄', 'ḹ'), # r̥̄ l̥̄ -> ṝ ḹ
	('r̥', 'ṛ'), ('l̥', 'ḷ'), # r̥ l̥ -> ṛ ḷ
	('ḷ', 'ḻ'), # ISO ḷ is ळ, not vocalic l
	('ṁ', 'ṃ'), # ṁ -> ṃ
	('ē', 'e'), ('ō', 'o'), # ē ō -> e o
	('ṉ', 'n'), ('ṟ', 'r'), # ṉ ṟ (nukta-ish ISO) -> n r
	]
	_ISO_ESCAPE = set(RESERVED) - {"'"} # keep ' meaning avagraha


	def iso15919_to_iast(text: str):
	"""VedaWeb-style ISO-15919 (with pitch accents) -> our strict IAST.

	Reserved characters other than the avagraha apostrophe are escaped by doubling, so the
	result is a legal input to :func:`iast_to_deva`.
	"""
	src = unicodedata.normalize('NFD', NFC(text))
	# longest source first so r̥̄ beats r̥
	pairs = sorted(_ISO_PAIRS, key=lambda p: -len(unicodedata.normalize('NFD', p[0])))
	pairs = [(unicodedata.normalize('NFD', a), b) for a, b in pairs]
	out, i, n = [], 0, len(src)
	unmapped: Counter = Counter()
	while i < n:
	for a, b in pairs:
	if src.startswith(a, i):
	out.append(b)
	i += len(a)
	break
	else:
	ch = src[i]
	out.append(ch + ch if ch in _ISO_ESCAPE else ch)
	i += 1
	return NFC(''.join(out)), unmapped


	# --------------------------------------------------------------------------------------
	# Akshara segmentation (explicit Devanagari regex; NOT \X, which breaks after virama)
	# --------------------------------------------------------------------------------------
	_C = '[क-हक़-य़ॹ-ॿऩऱऴ]'
	_V = '[ऄ-औॠॡॲ-ॷ]'
	_S = '[ऺऻा-ौॎॏॕ-ॗॢॣ]'
	_A = '[ऀ-ः॑-॔᳐-᳨ᳵᳶ꣠-꣱]'
	AKSHARA_RE = (
	f'(?:{_C}़?(?:्[‌‍]?{_C}़?)(?:{_S}\|्)?\|{_V}़?{_S}?){_A}'
	)


	def split_aksharas(text: str):
	"""Devanagari orthographic syllables; anything else comes back one code point per item."""
	import regex
	pat = regex.compile(f'{AKSHARA_RE}\|.', regex.DOTALL)
	return [m.group(0) for m in pat.finditer(text)]


	if __name__ == '__main__':
	import sys
	for line in (sys.argv[1:] or ['धर्मक्षेत्रे कुरुक्षेत्रे समवेता युयुत्सवः ॥ १ ॥']):
	s, us = deva_to_slp1(line)
	a, ua = deva_to_iast(line)
	print('deva :', line)
	print('slp1 :', s, ' unmapped', dict(us))
	print('iast :', a, ' unmapped', dict(ua))
	print('back :', slp1_to_deva(s)[0] == NFC(line), iast_to_deva(a)[0] == NFC(line))