transformers>=4.38.0
huggingface_hub>=0.23.0
torch<2.5,>=2.0
GitPython>=3.1.41
datasets>=2.14.0
termcolor==2.3.0
pytablewriter
colorama
aenum==3.1.15
nltk==3.9.1
scikit-learn
spacy==3.7.2
sacrebleu
rouge_score==0.1.2
sentencepiece>=0.1.99
protobuf==3.20.*
pycountry
fsspec>=2023.12.2

[accelerate]
accelerate

[adapters]
peft==0.3.0

[dev]
lighteval[accelerate,multilingual,quality,tests]

[extended_tasks]
langdetect
openai
tiktoken

[multilingual]
stanza
spacy[ja,ko,th]
jieba
pyvi

[nanotron]
nanotron
tensorboardX

[optimum]
optimum==1.12.0

[quality]
ruff==v0.2.2
pre-commit

[quantization]
bitsandbytes>=0.41.0
auto-gptq>=0.4.2

[s3]
s3fs

[tensorboardX]
tensorboardX

[tests]
pytest==7.4.0

[tgi]
text-generation==0.6.0

[vllm]
vllm
ray
more_itertools
