.gitattributes
.gitignore
.gitmodules
LICENSE
MANIFEST.in
README.md
requirements.txt
setup.py
.github/workflows/python-publish.yaml
.github/workflows/test.yaml
SudachiTra.egg-info/PKG-INFO
SudachiTra.egg-info/SOURCES.txt
SudachiTra.egg-info/dependency_links.txt
SudachiTra.egg-info/requires.txt
SudachiTra.egg-info/top_level.txt
misc/license-header.txt
pretraining/bert/README.md
pretraining/bert/__init__.py
pretraining/bert/convert_original_tf2_checkpoint_to_pytorch.py
pretraining/bert/create_pretraining_data.py
pretraining/bert/prepare_dataset.py
pretraining/bert/requirements.txt
pretraining/bert/run_create_pretraining_data.sh
pretraining/bert/run_prepare_dataset.sh
pretraining/bert/split_dataset.py
pretraining/bert/train_pos_substitution_tokenizer.py
pretraining/bert/train_wordpiece_tokenizer.py
sudachitra/__init__.py
sudachitra/slow_tokenizer.py
sudachitra/sudachipy_word_tokenizer.py
sudachitra/tokenization_bert_sudachipy.py
sudachitra/tokenization_electra_sudachipy.py
sudachitra/pretokenizer/__init__.py
sudachitra/pretokenizer/japanese_bert_wordpiece_tokenizer.py
sudachitra/pretokenizer/pos_substitution_tokenizer.py
sudachitra/pretokenizer/sudachipy_pretokenizer.py
tests/__init__.py
tests/test_tokenization_bert_sudachipy.py