Upload folder using huggingface_hub

app.py

@@ -98,7 +98,7 @@ def tokenize_with_panini(text: str) -> list:
         
         for i, word in enumerate(words):
             prefix = "▁" if i == 0 else ""
-            split_result = PANINI_SPLITTER.split(word)
+            split_result = PANINI_SPLITTER.split_v4(word)  # V1.5: Uses sandhi expansion
             
             if split_result.is_compound and len(split_result.components) > 1:
                 for j, comp in enumerate(split_result.components):

src/__init__.py

@@ -1,10 +1,11 @@
 """
-Panini Tokenizer V3
-Morphology-aware Sanskrit tokenizer using Vidyut.
+Panini Tokenizer
+Morphology-aware Sanskrit tokenizer with Sandhi Expansion.
 """
 
 from .analyzer import VidyutAnalyzer, MorphParse
 from .splitter import SamasaSplitter, CompoundSplit
+from .sandhi_engine import SandhiEngine
 from .tokenizer import PaniniTokenizerV3, create_tokenizer
 
 __all__ = [
@@ -12,8 +13,9 @@ __all__ = [
     "MorphParse", 
     "SamasaSplitter",
     "CompoundSplit",
+    "SandhiEngine",
     "PaniniTokenizerV3",
     "create_tokenizer",
 ]
 
-__version__ = "3.0.0"
+__version__ = "1.5.0"

src/sandhi_engine.py

@@ -0,0 +1,107 @@
+"""
+Sandhi Engine for Panini Tokenizer V4
+Generates pre-sandhi hypotheses for Sanskrit compound splitting.
+Handles vowel coalescence (ac-sandhi) and visarga/consonant assimilation.
+
+Uses table-driven design for maintainability.
+"""
+
+from typing import List, Tuple, Generator
+
+
+class SandhiEngine:
+    """
+    Generates pre-sandhi hypotheses for Sanskrit compound splitting.
+    Handles vowel coalescence (ac-sandhi) and visarga/consonant assimilation.
+    """
+    
+    def __init__(self):
+        # ac-sandhi (vowel merger) tables
+        # Key = surface char, Value = list of (left_end, right_start) pairs
+        self.VOWEL_SPLITS = {
+            # Guṇa
+            'e':  [('a', 'i'), ('A', 'i'), ('a', 'I'), ('A', 'I')],
+            'o':  [('a', 'u'), ('A', 'u'), ('a', 'U'), ('A', 'U')],
+            'ar': [('a', 'f'), ('A', 'f'), ('a', 'F'), ('A', 'F')],  # maharzi -> mahA + fzi
+            
+            # Vṛddhi
+            'E':  [('a', 'e'), ('A', 'e'), ('a', 'E'), ('A', 'E')],  # ai
+            'O':  [('a', 'o'), ('A', 'o'), ('a', 'O'), ('A', 'O')],  # au
+            
+            # Dīrgha (savarṇa dīrgha) - critical for long vowel restoration
+            'A':  [('a', 'a'), ('a', 'A'), ('A', 'a'), ('A', 'A')],
+            'I':  [('i', 'i'), ('i', 'I'), ('I', 'i'), ('I', 'I')],
+            'U':  [('u', 'u'), ('u', 'U'), ('U', 'u'), ('U', 'U')],
+        }
+
+        # Consonant categories
+        self.VOICED = set(['g', 'G', 'j', 'J', 'd', 'D', 'b', 'B', 'n', 'N', 'm', 'y', 'r', 'l', 'v', 'h'])
+        self.HARD = set(['k', 'K', 'c', 'C', 't', 'T', 'w', 'W', 'p', 'P', 'S', 's'])
+    
+    def generate_splits(self, word: str, i: int) -> Generator[Tuple[str, str], None, None]:
+        """
+        Yields (left, right) tuples for a split AT index i.
+        i is the index of the character being considered as the 'pivot'.
+        """
+        if i < 1 or i >= len(word):
+            return
+        
+        char = word[i]
+        
+        # 1. Default: hard cut (no sandhi)
+        # Split BEFORE char: word[:i] | word[i:]
+        yield (word[:i], word[i:])
+        
+        # 2. Vowel coalescence (the char IS the result of merger)
+        # e.g. gaṇ[e]śa -> left ends with 'a', right starts with 'i'
+        if char in self.VOWEL_SPLITS:
+            for left_end, right_start in self.VOWEL_SPLITS[char]:
+                # Replace char at i with the split pair
+                yield (word[:i] + left_end, right_start + word[i+1:])
+                
+        # 3. Yān sandhi (y -> i/I, v -> u/U)
+        # e.g. praty[e]kam -> prati + ekam
+        # CAUTION: Yān happens BEFORE a vowel, check word[i+1]
+        if i + 1 < len(word):
+            next_char = word[i+1]
+            if char == 'y':  # y -> i/I
+                for v in ['i', 'I']:
+                    yield (word[:i] + v, word[i+1:])
+            elif char == 'v':  # v -> u/U
+                for v in ['u', 'U']:
+                    yield (word[:i] + v, word[i+1:])
+                    
+        # 4. Visarga sandhi restoration
+        # 'o' before voiced consonant -> 'aH'
+        if char == 'o' and i + 1 < len(word):
+            if word[i+1] in self.VOICED:
+                yield (word[:i] + "aH", word[i+1:])
+                
+        # 'r' before voiced -> 'H' (punarjanma -> punaH + janma)
+        if char == 'r' and i + 1 < len(word):
+            if word[i+1] in self.VOICED:
+                yield (word[:i] + "H", word[i+1:])
+
+        # 's'/'S' before hard consonant -> 'H'
+        if char in ['s', 'S'] and i + 1 < len(word):
+            if word[i+1] in self.HARD:
+                yield (word[:i] + "H", word[i+1:])
+
+
+# --- TEST ---
+if __name__ == "__main__":
+    engine = SandhiEngine()
+    
+    print("Testing SandhiEngine...")
+    
+    test_cases = [
+        ("gaReSa", 3),   # e: should yield gaRa + iSa
+        ("devendra", 3), # e: should yield deva + indra  
+        ("rAmo", 3),     # o: should yield rAmaH before voiced
+        ("punarjanma", 4), # r: should yield punaH + janma
+    ]
+    
+    for word, pos in test_cases:
+        print(f"\n  {word} at pos {pos}:")
+        for left, right in engine.generate_splits(word, pos):
+            print(f"    {left} | {right}")

src/splitter.py

@@ -6,11 +6,9 @@ Detects and splits Sanskrit compound words at their boundaries.
 from typing import List, Tuple, Optional
 from dataclasses import dataclass
 
-# Import analyzer for Kosha access (use absolute import for standalone execution)
-try:
-    from .analyzer import VidyutAnalyzer, MorphParse
-except ImportError:
-    from analyzer import VidyutAnalyzer, MorphParse
+# Import analyzer for Kosha access
+from .analyzer import VidyutAnalyzer, MorphParse
+from .sandhi_engine import SandhiEngine
 
 
 @dataclass
@@ -57,6 +55,7 @@ class SamasaSplitter:
     def __init__(self, analyzer: Optional[VidyutAnalyzer] = None):
         """Initialize with optional shared analyzer."""
         self.analyzer = analyzer or VidyutAnalyzer(preload_cache=False)
+        self.sandhi_engine = SandhiEngine()  # V4: Generative sandhi expansion
     
     # Sandhi reversal rules: (surface_ending, possible_original_endings)
     # These are common consonant/vowel Sandhi transformations to reverse
@@ -154,6 +153,27 @@ class SamasaSplitter:
             if candidate.endswith('U') and self.analyzer._in_kosha(candidate[:-1] + 'u'):
                 return True
         
+        # Try VISARGA STRIPPING (vAlmIkiH → vAlmIki)
+        if surface.endswith('H') and len(surface) > 2:
+            base = surface[:-1]
+            if self.analyzer._in_kosha(base):
+                return True
+        
+        # Try VIBHAKTI STRIPPING (nominal case endings)
+        VIBHAKTI_ENDINGS = [
+            'am', 'aH', 'ena', 'Aya', 'At', 'asya', 'e', 'AH',  # Masculine a-stem
+            'An', 'EH', 'eBya', 'AnAm', 'ezu',                   # Masculine a-stem plural
+            'au', 'OH', 'AvyAm',                                  # Dual
+        ]
+        for ending in sorted(VIBHAKTI_ENDINGS, key=len, reverse=True):
+            if surface.endswith(ending) and len(surface) > len(ending) + 2:
+                stem = surface[:-len(ending)]
+                if self.analyzer._in_kosha(stem):
+                    return True
+                # Try with 'a' restoration (munipuMgavam → munipuMgava)
+                if self.analyzer._in_kosha(stem + 'a'):
+                    return True
+        
         # Try PRATYAYA STRIPPING (grammatical suffix removal)
         # This is Panini's kRt/taddhita system - generalizes to ALL Sanskrit
         PRATYAYAS = [
@@ -167,6 +187,12 @@ class SamasaSplitter:
             ('in', 2),    # ṇini: possessor
             ('ika', 3),   # ṭhak: related to
             ('Iya', 3),   # cha: related to
+            # Feminine/agent kṛdanta suffixes (Fix 2)
+            ('iRi', 3),   # iṇī: feminine agent (ākarṣiṇī)
+            ('iRI', 3),   # iṇī: alt spelling
+            ('inI', 3),   # inī: feminine possessor (yoginī)
+            ('ikA', 3),   # ikā: feminine derivative (nāyikā)
+            ('trI', 3),   # trī: feminine agent (kartrī)
         ]
         
         for suffix, min_root in PRATYAYAS:
@@ -175,6 +201,13 @@ class SamasaSplitter:
                 # Try the root in Kosha
                 if self.analyzer._in_kosha(root):
                     return True
+                # Try with guṇa 'a' restoration
+                if self.analyzer._in_kosha(root + 'a'):
+                    return True
+                # Try R→f transliteration (MW uses f for ṛ: kartRI → kartf)
+                root_f = root.replace('R', 'f')
+                if root_f != root and self.analyzer._in_kosha(root_f):
+                    return True
                 # Try Sandhi reversal on root
                 for r in self._try_sandhi_reversal(root):
                     if self.analyzer._in_kosha(r):
@@ -698,6 +731,123 @@ class SamasaSplitter:
             compound_type=None  # We don't classify samāsa types
         )
     
+    def _split_dp(self, word: str, memo: dict = None) -> List[List[str]]:
+        """
+        V4 Algorithm: Memoized Dynamic Programming with Sandhi Expansion.
+        
+        Returns all valid splits, cached by suffix.
+        Handles coalescent sandhi (e=a+i, o=a+u, etc.) that V3 misses.
+        """
+        if memo is None:
+            memo = {}
+        
+        if word in memo:
+            return memo[word]
+        
+        # Base: too short to split
+        if len(word) <= 2:
+            if self._is_valid_stem(word):
+                return [[word]]
+            return []
+        
+        valid_splits = []
+
+        # 1. OPTION A: The whole word is a stem (Lexicalized)
+        if self._is_valid_stem(word):
+            valid_splits.append([word])
+            # DO NOT RETURN EARLY. Keep looking for splits!
+
+        # 2. OPTION B: Split it (Generative Sandhi)
+        # Try each split position with sandhi expansion
+        for i in range(2, len(word) - 1):
+            for left, right in self.sandhi_engine.generate_splits(word, i):
+                if len(left) < 2 or len(right) < 2:
+                    continue
+                    
+                if self._is_valid_stem(left):
+                    # Recurse on right (memoized!)
+                    right_splits = self._split_dp(right, memo)
+                    for rs in right_splits:
+                        valid_splits.append([left] + rs)
+        
+        memo[word] = valid_splits
+        return valid_splits
+    
+    def split_v4(self, word: str) -> CompoundSplit:
+        """
+        V4 Split: Uses generative sandhi expansion for coalescent sandhi.
+        
+        Handles:
+        - Vowel coalescence: gaṇeśa → gaṇa + īśa (e = a+i)
+        - Visarga sandhi: punarjanma → punaH + janma
+        - Vṛddhi: tavaiva → tava + eva
+        """
+        if len(word) < 4:
+            return CompoundSplit(
+                surface=word, components=[word],
+                split_points=[], is_compound=False, compound_type=None
+            )
+        
+        # Use V4 DP algorithm
+        all_splits = self._split_dp(word)
+        
+        if not all_splits:
+            return CompoundSplit(
+                surface=word, components=[word],
+                split_points=[], is_compound=False, compound_type=None
+            )
+        
+        # SCORING STRATEGY:
+        # Balance: prefer splits, but penalize over-fragmentation.
+        # 1. Penalize short components (< 3 chars) heavily
+        # 2. Prefer 2-component splits over 3+ components
+        # 3. Single long tokens get moderate penalty
+        # 4. Bonus for components directly in kosha (prefer mahA+rAja over maha+arAja)
+        def score_split(components):
+            base_score = sum(len(c)**2 for c in components)
+            
+            # Penalize short components (garbage like 'ma', 'at')
+            short_penalty = sum(10 for c in components if len(c) < 3)
+            base_score -= short_penalty * 5
+            
+            # Bonus for 2-component splits (optimal granularity)
+            if len(components) == 2:
+                base_score += 20
+            
+            # Penalty for single long tokens (prefer analysis)
+            if len(components) == 1 and len(components[0]) > 6:
+                base_score -= 15
+            
+            # Bonus for components directly in kosha (prefer clean stems)
+            kosha_bonus = sum(25 for c in components if self.analyzer._in_kosha(c))
+            base_score += kosha_bonus
+            
+            # Prefer balanced splits (similar length components)
+            if len(components) == 2:
+                len_diff = abs(len(components[0]) - len(components[1]))
+                if len_diff <= 1:
+                    base_score += 10  # Bonus for balanced split
+            
+            # Penalize splits with expanded length (sandhi artifacts add characters)
+            total_len = sum(len(c) for c in components)
+            if total_len > len(word):
+                base_score -= (total_len - len(word)) * 10  # Penalty per extra char
+            
+            return base_score
+
+        best_split = max(all_splits, key=score_split)
+        
+        if len(best_split) <= 1:
+            return CompoundSplit(
+                surface=word, components=[word],
+                split_points=[], is_compound=False, compound_type=None
+            )
+        
+        return CompoundSplit(
+            surface=word, components=best_split,
+            split_points=[], is_compound=True, compound_type=None
+        )
+    
     def split_multiple(self, words: List[str]) -> List[CompoundSplit]:
         """Split multiple words."""
         return [self.split(w) for w in words]