MANIFEST.in
README.md
requirements-dev.txt
requirements.txt
setup.py
codeprep/VERSION
codeprep/__init__.py
codeprep/__main__.py
codeprep/config.py
codeprep/dirutils.py
codeprep/fileutils.py
codeprep/logging.yaml
codeprep/noneng.py
codeprep/prepconfig.py
codeprep/stemming.py
codeprep/subtokens.py
codeprep/util.py
codeprep.egg-info/PKG-INFO
codeprep.egg-info/SOURCES.txt
codeprep.egg-info/dependency_links.txt
codeprep.egg-info/entry_points.txt
codeprep.egg-info/not-zip-safe
codeprep.egg-info/requires.txt
codeprep.egg-info/top_level.txt
codeprep/api/__init__.py
codeprep/api/common.py
codeprep/api/corpus.py
codeprep/api/text.py
codeprep/bpepkg/__init__.py
codeprep/bpepkg/bpe_config.py
codeprep/bpepkg/bpe_encode.py
codeprep/bpepkg/bpe_learn.py
codeprep/bpepkg/cache.py
codeprep/bpepkg/merge.py
codeprep/bpepkg/wild_bpe.py
codeprep/cli/__init__.py
codeprep/cli/impl.py
codeprep/cli/spec.py
codeprep/cli/vocab.py
codeprep/data/allamanis_dataset_metadata/small-train-projects.txt
codeprep/data/bpe/10k/merges.txt
codeprep/data/bpe/case/0/merges.txt
codeprep/data/bpe/case/10k/merges.txt
codeprep/data/bpe/case/1k/merges.txt
codeprep/data/bpe/case/5k/merges.txt
codeprep/data/bpe/nocase/0/merges.txt
codeprep/parse/__init__.py
codeprep/parse/core.py
codeprep/parse/matchers.py
codeprep/parse/subtokens.py
codeprep/pipeline/__init__.py
codeprep/pipeline/bpelearner.py
codeprep/pipeline/bperegistry.py
codeprep/pipeline/dataset.py
codeprep/pipeline/parse_projects.py
codeprep/pipeline/stages.py
codeprep/pipeline/to_repr.py
codeprep/pipeline/vocab.py
codeprep/pipeline/vocabloader.py
codeprep/preprocess/__init__.py
codeprep/preprocess/core.py
codeprep/preprocess/metadata.py
codeprep/preprocess/placeholders.py
codeprep/preprocess/reprconfig.py
codeprep/tokens/__init__.py
codeprep/tokens/containers.py
codeprep/tokens/noneng.py
codeprep/tokens/numeric.py
codeprep/tokens/rootclasses.py
codeprep/tokens/whitespace.py
codeprep/tokens/word.py
tests/__init__.py
tests/test_corpus_b2b.py
tests/test_subword_separation.py
tests/test_to_repr.py
tests/api/__init__.py
tests/api/test_corpus.py
tests/bpepkg/__init__.py
tests/bpepkg/test_merge.py
tests/bpepkg/wild_bpe_performance.py
tests/cli/__init__.py
tests/cli/test_spec.py
tests/infrastructure/__init__.py
tests/infrastructure/test_bpelearner.py
tests/infrastructure/test_bperegistry.py
tests/infrastructure/test_dataset.py
tests/parse/__init__.py
tests/parse/test_core.py
tests/parse/test_subtokens.py