boto3==1.28.78
cchardet==2.1.7
inscriptis==2.3.2
loguru==0.7.0
multiprocess==0.70.14
nltk==3.8.1
numpy==1.25.0
python-magic==0.4.27
trafilatura==1.6.1
warcio==1.7.4
zstandard==0.21.0
pyarrow==12.0.1
tokenizers==0.13.3
tldextract==3.4.4
pandas==2.0.3
backoff==2.2.1
fsspec==2023.9.2
humanize==4.8.0
rich==13.7.0

[dev]
black~=23.1
pre-commit>=3.3.3
pytest>=7.2.0
pytest-timeout
pytest-xdist
ruff<=0.0.259,>=0.0.241
