Merge pull request #910 from PyThaiNLP/add-th_tdtb

Add postag of Thai Discourse Treebank

Author: wannaphong

CONTRIBUTING.md

@@ -157,3 +157,4 @@ Thanks to all [contributors](https://github.com/PyThaiNLP/pythainlp/graphs/contr
 - **[Thai Character Cluster]** -- T. Teeramunkong, V. Sornlertlamvanich, T. Tanhermhong and W. Chinnan, “Character cluster based Thai information retrieval,” in IRAL '00 Proceedings of the fifth international workshop on on Information retrieval with Asian languages, 2000.
 - **[Enhanced Thai Character Cluster]** -- Jeeragone Inrut, Patiroop Yuanghirun, Sarayut Paludkong, Supot Nitsuwat, and Para Limmaneepraserth. “Thai word segmentation using combination of forward and backward longest matching techniques.” In International Symposium on Communications and Information Technology (ISCIT), pp. 37-40. 2001.
 - เพ็ญศิริ ลี้ตระกูล. การเลือกประโยคสำคัญในการสรุปความภาษาไทย โดยใช้แบบจำลองแบบลำดับชั้น (Selection of Important Sentences in Thai Text Summarization Using a Hierarchical Model). Retrieved from http://digi.library.tu.ac.th/thesis/st/0192/
+- **[Thai Discourse Treebank]** -- Ponrawee Prasertsom, Apiwat Jaroonpol, Attapol T. Rutherford; The Thai Discourse Treebank: Annotating and Classifying Thai Discourse Connectives. Transactions of the Association for Computational Linguistics 2024; 12 613–629. doi: https://doi.org/10.1162/tacl_a_00650

pythainlp/corpus/corpus_license.md

@@ -51,6 +51,8 @@ https://creativecommons.org/licenses/by/4.0/
 | pos_ud_perceptron-v0.2.json     | Part-of-speech tagging model, trained from Parallel Universal Dependencies treebank, using perceptron |
 | pos_ud_unigram-v0.2.json       | Part-of-speech tagging model, trained from Parallel Universal Dependencies treebank, using unigram    |
 | sentenceseg_crfcut.model  | Sentence segmentation model, trained from TED subtitles, using CRF                                    |
+| tdtb-pt_tagger.json | Part-of-speech tagging model, trained from The Thai Discourse Treebank, using perceptron                              |
+| tdtb-unigram_tagger.json   | Part-of-speech tagging model, trained from The Thai Discourse Treebank, using unigram                                 |
 | pos_tud_perceptron.json | Part-of-speech tagging model, trained from Thai Universal Dependency Treebank data, using perceptron                              |
 | pos_tud_unigram.json   | Part-of-speech tagging model, trained from Thai Universal Dependency Treebank data, using unigram                                 |
 

pythainlp/corpus/tdtb-pt_tagger.json

@@ -16,6 +16,9 @@
 _PUD_FILENAME = "pos_ud_perceptron-v0.2.json"
 _PUD_PATH = os.path.join(corpus_path(), _PUD_FILENAME)
 
+_TDTB_FILENAME = "tdtb-pt_tagger.json"
+_TDTB_PATH = os.path.join(corpus_path(), _TDTB_FILENAME)
+
 _BLACKBOARD_NAME = "blackboard_pt_tagger"
 
 _TUD_FILENAME = "pos_tud_perceptron.json"
@@ -24,6 +27,7 @@
 _ORCHID_TAGGER = None
 _PUD_TAGGER = None
 _BLACKBOARD_TAGGER = None
+_TDTB_TAGGER = None
 _TUD_TAGGER = None
 
 
@@ -48,6 +52,13 @@ def _blackboard_tagger():
     return _LST20_TAGGER
 
 
+def _tdtb():
+    global _TDTB_TAGGER
+    if not _TDTB_TAGGER:
+        _TDTB_TAGGER = PerceptronTagger(path=_TDTB_PATH)
+    return _TDTB_TAGGER
+
+
 def _tud_tagger():
     global _TUD_TAGGER
     if not _TUD_TAGGER:
@@ -78,6 +89,8 @@ def tag(words: List[str], corpus: str = "pud") -> List[Tuple[str, str]]:
         words = blackboard.pre_process(words)
         word_tags = _blackboard_tagger().tag(words)
         word_tags = blackboard.post_process(word_tags, to_ud)
+    elif corpus in ("tdtb"):
+        word_tags = _tdtb().tag(words)
     elif corpus in ("tud"):
         tagger = _tud_tagger()
         word_tags = tagger.tag(words)

pythainlp/corpus/tdtb-unigram_tagger.json

@@ -28,7 +28,11 @@ def pos_tag(
         * *pud* - `Parallel Universal Dependencies (PUD)\
             <https://github.com/UniversalDependencies/UD_Thai-PUD>`_ \
             treebanks, natively use Universal POS tags
+        * *tdtb* - `Thai Discourse Treebank \
+            <https://github.com/nlp-chula/thai-discourse-treebank/tree/main>`_ \
+            , natively use Universal POS tags
         * *tnc* - Thai National Corpus (support tltk engine only)
+        * *tdtb* - `Thai Discourse Treebank <https://github.com/nlp-chula/thai-discourse-treebank>`_
         * *tud* - `Thai Universal Dependency Treebank (TUD)\
             <https://github.com/nlp-chula/TUD>`_ \
     :return: a list of tuples (word, POS tag)
@@ -98,6 +102,7 @@ def pos_tag(
         "orchid",
         "orchid_ud",
         "pud",
+        "tdtb",
         "tud",
     ]
 

pythainlp/tag/perceptron.py

@@ -17,6 +17,9 @@
 _PUD_FILENAME = "pos_ud_unigram-v0.2.json"
 _PUD_PATH = os.path.join(corpus_path(), _PUD_FILENAME)
 
+_TDTB_FILENAME = "tdtb-unigram_tagger.json"
+_TDTB_PATH = os.path.join(corpus_path(), _TDTB_FILENAME)
+
 _BLACKBOARD_NAME = "blackboard_unigram_tagger"
 
 _TUD_FILENAME = "pos_tud_unigram.json"
@@ -25,6 +28,7 @@
 _ORCHID_TAGGER = None
 _PUD_TAGGER = None
 _BLACKBOARD_TAGGER = None
+_TDTB_TAGGER = None
 _TUD_TAGGER = None
 
 
@@ -53,6 +57,14 @@ def _blackboard_tagger():
     return _BLACKBOARD_TAGGER
 
 
+def _thai_tdtb():
+    global _TDTB_TAGGER
+    if not _TDTB_TAGGER:
+        with open(_TDTB_PATH, encoding="utf-8-sig") as fh:
+            _TDTB_TAGGER = json.load(fh)
+    return _TDTB_TAGGER
+
+
 def _tud_tagger():
     global _TUD_TAGGER
     if not _TUD_TAGGER:
@@ -94,6 +106,8 @@ def tag(words: List[str], corpus: str = "pud") -> List[Tuple[str, str]]:
         words = blackboard.pre_process(words)
         word_tags = _find_tag(words, _blackboard_tagger())
         word_tags = blackboard.post_process(word_tags, to_ud)
+    elif corpus in ("tdtb"):
+        word_tags = _find_tag(words, _thai_tdtb())
     elif corpus in ("tud"):
         word_tags = _find_tag(words, _tud_tagger())
     else:  # by default, use "pud" for corpus

pythainlp/tag/pos_tag.py

@@ -70,6 +70,12 @@ def test_pos_tag(self):
         self.assertIsNotNone(
             pos_tag([""], engine="unigram", corpus="blackboard_ud")
         )
+        self.assertIsNotNone(
+            pos_tag(tokens, engine="unigram", corpus="tdtb")
+        )
+        self.assertIsNotNone(
+            pos_tag([""], engine="unigram", corpus="tdtb")
+        )
         self.assertIsNotNone(pos_tag(tokens, engine="unigram", corpus="tud"))
         self.assertIsNotNone(pos_tag([""], engine="unigram", corpus="tud"))
         self.assertEqual(
@@ -109,6 +115,12 @@ def test_pos_tag(self):
         self.assertIsNotNone(
             pos_tag(tokens, engine="perceptron", corpus="blackboard_ud")
         )
+        self.assertIsNotNone(
+            pos_tag(tokens, engine="perceptron", corpus="tdtb")
+        )
+        self.assertIsNotNone(
+            pos_tag(tokens, engine="perceptron", corpus="tdtb")
+        )
         self.assertIsNotNone(
             pos_tag(tokens, engine="perceptron", corpus="tud")
         )