fsspec==2023.3.0
pyarrow<=12.0.0
pandas==2.0.0
datasets==2.11.0
loguru
tabulate
tqdm
jsonargparse[signatures]
matplotlib
emoji==2.2.0
regex
requests
wget
zstandard
lz4
pdfplumber
plotly
python-docx
streamlit
spacy==3.5.0
multiprocess==0.70.12
dill==0.3.4
scipy
boto3<=1.15.18
simhash-py
gluonnlp<=0.10.0,>=0.6.0
mxnet<=1.7.0.post2,>=1.4.0
onnxruntime<=1.8.0,==1.8.0
numpy==1.23.1
selectolax
ftfy
nlpaug
ray
fasttext==0.9.2
sentencepiece<=0.1.96,>=0.1.6
pypi-kenlm==0.1.20220713
transformers
fasttext-wheel
scipy
ftfy
simhash-py
selectolax
nlpaug
nlpcda
nltk
transformers
opencc==1.1.6
imagededup
torch<=1.10.1,>=1.7.0
dlib
spacy-pkuseg==0.0.32
fasttext-wheel
kenlm
omegaconf

[all]
fsspec==2023.3.0
pyarrow<=12.0.0
pandas==2.0.0
datasets==2.11.0
loguru
tabulate
tqdm
jsonargparse[signatures]
matplotlib
emoji==2.2.0
regex
requests
wget
zstandard
lz4
pdfplumber
plotly
python-docx
streamlit
spacy==3.5.0
multiprocess==0.70.12
dill==0.3.4
scipy
boto3<=1.15.18
simhash-py
gluonnlp<=0.10.0,>=0.6.0
mxnet<=1.7.0.post2,>=1.4.0
onnxruntime<=1.8.0,==1.8.0
numpy==1.23.1
selectolax
ftfy
nlpaug
ray
fasttext==0.9.2
sentencepiece<=0.1.96,>=0.1.6
pypi-kenlm==0.1.20220713
transformers
fasttext-wheel
nlpcda
nltk
opencc==1.1.6
imagededup
torch<=1.10.1,>=1.7.0
dlib
spacy-pkuseg==0.0.32
kenlm
omegaconf
pre-commit
sphinx
sphinx-autobuild
sphinx_rtd_theme
recommonmark
fire
jsonlines
pyspark

[dev]
pre-commit
sphinx
sphinx-autobuild
sphinx_rtd_theme
recommonmark

[mini]
fsspec==2023.3.0
pyarrow<=12.0.0
pandas==2.0.0
datasets==2.11.0
loguru
tabulate
tqdm
jsonargparse[signatures]
matplotlib
emoji==2.2.0
regex
requests
wget
zstandard
lz4
pdfplumber
plotly
python-docx
streamlit
spacy==3.5.0
multiprocess==0.70.12
dill==0.3.4
scipy
boto3<=1.15.18
simhash-py
gluonnlp<=0.10.0,>=0.6.0
mxnet<=1.7.0.post2,>=1.4.0
onnxruntime<=1.8.0,==1.8.0
numpy==1.23.1
selectolax
ftfy
nlpaug
ray
fasttext==0.9.2
sentencepiece<=0.1.96,>=0.1.6
pypi-kenlm==0.1.20220713
transformers
fasttext-wheel
nlpcda
nltk
opencc==1.1.6
imagededup
torch<=1.10.1,>=1.7.0
dlib
spacy-pkuseg==0.0.32
kenlm
omegaconf

[tools]
fire
jsonlines
pyspark
wget
