連文節変換

Makefile

@@ -22,13 +22,21 @@ comb.xml: comb.xml.in
 	    -e "s:@DATADIR@:$(DATADIR):g" $< > $@
 
 comb/config.py: comb/config.py.in
-	sed -e "s:@SYSCONFDIR@:$(SYSCONFDIR):g" $< > $@
+	sed -e "s:@SYSCONFDIR@:$(SYSCONFDIR):g" \
+	    -e "s:@MODELDIR@:$(DESTDIR)/$(DATADIR)/ibus-comb/model:g" \
+		$< > $@
 
-install: all check comb/config.py
-	install -m 0755 -d $(DESTDIR)$(DATADIR)/ibus-comb/comb $(DESTDIR)$(SYSCONFDIR)/xdg/comb $(DESTDIR)$(DATADIR)/ibus/component
+model/jawiki.1gram:
+	make -C model jawiki.1gram
+
+install: all comb/config.py model/jawiki.1gram check
+	install -m 0755 -d $(DESTDIR)$(DATADIR)/ibus-comb/comb $(DESTDIR)$(SYSCONFDIR)/xdg/comb $(DESTDIR)$(DATADIR)/ibus/component $(DESTDIR)$(DATADIR)/ibus-comb/model
+	install -m 0644 model/jawiki.1gram $(DESTDIR)$(DATADIR)/ibus-comb/model/
+	install -m 0644 model/jawiki.2gram $(DESTDIR)$(DATADIR)/ibus-comb/model/
 	install -m 0644 comb.svg $(DESTDIR)$(DATADIR)/ibus-comb
 	install -m 0644 comb/__init__.py $(DESTDIR)$(DATADIR)/ibus-comb/comb/
 	install -m 0644 comb/graph.py $(DESTDIR)$(DATADIR)/ibus-comb/comb/
+	install -m 0644 comb/config.py $(DESTDIR)$(DATADIR)/ibus-comb/comb/
 	install -m 0644 comb/skkdict.py $(DESTDIR)$(DATADIR)/ibus-comb/comb/
 	install -m 0644 comb/combromkan.py $(DESTDIR)$(DATADIR)/ibus-comb/comb/
 	install -m 0644 ibus.py $(DESTDIR)$(DATADIR)/ibus-comb
@@ -49,6 +57,8 @@ uninstall:
 	rm -f $(DESTDIR)$(DATADIR)/ibus-comb/comb/user_dict.py
 	rm -f $(DESTDIR)$(DATADIR)/ibus-comb/comb/system_dict.py
 	rm -f $(DESTDIR)$(DATADIR)/ibus-comb/ibus.py
+	rm -f $(DESTDIR)$(DATADIR)/ibus-comb/model/jawiki.1gram
+	rm -f $(DESTDIR)$(DATADIR)/ibus-comb/model/jawiki.2gram
 	rmdir $(DESTDIR)$(DATADIR)/ibus-comb
 	rmdir $(DESTDIR)$(SYSCONFDIR)/xdg/comb
 	rm -f $(DESTDIR)$(DATADIR)/ibus/component/comb.xml

comb/config.py.in

@@ -1 +1,2 @@
 SYS_CONF_DIR = '@SYSCONFDIR@'
+MODEL_DIR = '@MODELDIR@'

comb/engine.py

@@ -1,3 +1,4 @@
+from logging import Logger
 from typing import List, Any
 
 import os
@@ -8,27 +9,55 @@ from comb import combromkan
 
 from comb.system_dict import SystemDict
 from comb.user_dict import UserDict
+from comb.graph import graph_construct, viterbi, lookup
+from comb.config import MODEL_DIR
+import logging
+import marisa_trie
 
 
 class Comb:
+    logger: Logger
     dictionaries: List[Any]
 
-    def __init__(self, logger, user_dict: UserDict, system_dict: SystemDict):
+    def __init__(self, logger: Logger, user_dict: UserDict, system_dict: SystemDict):
         self.logger = logger
         self.dictionaries = []
         self.user_dict = user_dict
         self.system_dict = system_dict
 
+        self.unigram_score = marisa_trie.RecordTrie('@f')
+        self.unigram_score.load(f"{MODEL_DIR}/jawiki.1gram")
+
+        self.bigram_score = marisa_trie.RecordTrie('@f')
+        self.bigram_score.load(f"{MODEL_DIR}/jawiki.2gram")
+
     def convert(self, src):
         hiragana: str = combromkan.to_hiragana(src)
         katakana: str = jaconv.hira2kata(hiragana)
 
+        self.logger.info(f"convert: src={src} hiragana={hiragana} katakana={katakana}")
+
         candidates = [[hiragana, hiragana]]
 
         for e in self.user_dict.get_candidates(src, hiragana):
             if e not in candidates:
                 candidates.append(e)
 
+        try:
+            ht = dict(lookup(hiragana, self.system_dict))
+            graph = graph_construct(hiragana, ht, self.unigram_score, self.bigram_score)
+            got = viterbi(graph, self.unigram_score)
+
+            phrase = ''.join([x.word for x in got if not x.is_eos()])
+
+            self.logger.info(f"Got phrase: {phrase}")
+
+            if [phrase, phrase] not in candidates:
+                candidates.append([phrase, phrase])
+        except:
+            self.logger.error(f"Cannot convert: {hiragana} {katakana}",
+                              exc_info=True)
+
         if [katakana, katakana] not in candidates:
             candidates.append([katakana, katakana])
 

comb/graph.py

@@ -56,7 +56,7 @@ class Node:
 class Graph:
     d: Dict[int, List[Node]]
 
-    def __init__(self, size: int, unigram_score, bigram_score):
+    def __init__(self, size: int, unigram_score, bigram_score, logger=logging.getLogger(__name__)):
         self.d = {
             0: [Node(start_pos=-9999, word='<S>', yomi='<S>', unigram_score=unigram_score,
                      bigram_score=bigram_score)],
@@ -84,12 +84,8 @@ class Graph:
 
     def __getitem__(self, item):
         ary = [None for _ in range(len(self.d))]
-        try:
         for k in sorted(self.d.keys()):
             ary[k] = self.d[k]
-        except IndexError:
-            logging.error(f"Cannot get entry {self.d[15]} {k}", exc_info=True)
-            sys.exit(1)
         return ary[item]
 
     def dump(self, path: str):
@@ -202,6 +198,7 @@ def main():
 
     bigram_score = marisa_trie.RecordTrie('@f')
     bigram_score.load('model/jawiki.2gram')
+
     system_dict = SystemDict()
 
     # print(ht)

comb/system_dict.py

@@ -46,6 +46,7 @@ class SystemDict:
             trie = marisa_trie.BytesTrie()
             trie.load(cache_file)
             self.trie = trie
+            self.logger.info("loaded cache dictionary")
             return
 
         self.logger.info("loading dictionaries")

comb/ui.py

@@ -35,8 +35,14 @@ user_dict = UserDict(os.path.join(configdir, 'user-dict.txt'), logging.getLogger
 logging.info("Loaded user dictionary")
 
 system_dict = SystemDict()
+logging.info("Loaded system dictionary")
 
+try:
     comb = Comb(logging.getLogger('Comb'), user_dict, system_dict)
+    logging.info("Finished Comb.")
+except:
+    logging.error("Cannot initialize.", exc_info=True)
+    sys.exit(1)
 
 
 # ----------------------------------------------------------------------

ibus.py

@@ -131,4 +131,8 @@ def main():
 
 
 if __name__ == "__main__":
+    try:
         main()
+    except:
+        logging.error("Cannot initialize", exc_info=True)
+        sys.exit(1)