Something went wrong. Try again.
What happens if I push all of nixpkgs
Something went wrong. Try again.
3.4 kB · 181 lines
Nix
at master
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182{ lib, buildPythonPackage, fetchFromGitHub,
# build-system setuptools-scm,
# dependencies accelerate, datasets, dill, evaluate, jsonlines, more-itertools, numexpr, peft, pybind11, pytablewriter, rouge-score, sacrebleu, scikit-learn, sqlitedict, torch, tqdm-multiprocess, transformers, word2number, zstandard,
# optional-dependencies # api aiohttp, requests, tenacity, tiktoken, tqdm, # hf_transfer hf-transfer, # ifeval immutabledict, langdetect, nltk, # neuronx optimum, # mamba causal-conv1d, mamba-ssm, # math antlr4-python3-runtime, sympy, # sentencepiece sentencepiece, # vllm vllm, # wandb numpy, pandas, wandb,
# tests pytestCheckHook, writableTmpDirAsHomeHook,}:
buildPythonPackage rec { pname = "lm-eval"; version = "0.4.9.1"; pyproject = true;
src = fetchFromGitHub { owner = "EleutherAI"; repo = "lm-evaluation-harness"; tag = "v${version}"; hash = "sha256-N5NRRabjWxPchwOIkjqYTCKInCmVSY6T5cAmdxNbCkU="; };
build-system = [ setuptools-scm ];
dependencies = [ accelerate datasets dill evaluate jsonlines more-itertools numexpr peft pybind11 pytablewriter rouge-score sacrebleu scikit-learn sqlitedict torch tqdm-multiprocess transformers word2number zstandard ];
optional-dependencies = { api = [ aiohttp requests tenacity tiktoken tqdm ]; hf_transfer = [ hf-transfer ]; ifeval = [ immutabledict langdetect nltk ]; neuronx = [ optimum ] ++ optimum.optional-dependencies.neuronx; mamba = [ causal-conv1d mamba-ssm ]; math = [ antlr4-python3-runtime sympy ]; optimum = [ optimum ] ++ optimum.optional-dependencies.openvino; sentencepiece = [ sentencepiece ]; vllm = [ vllm ]; wandb = [ numpy pandas wandb ]; # Still missing dependencies for the following: # deepsparse, gptq, ibm_watsonx_ai, multilingual, promptsource, sparseml, # zeno, gptqmodel, japanese_leaderboard; all = [...]; };
pythonRelaxDeps = [ "datasets" ];
pythonImportsCheck = [ "lm_eval" ];
nativeCheckInputs = [ pytestCheckHook writableTmpDirAsHomeHook ] ++ optional-dependencies.api;
disabledTests = [ "test_deepsparse" # deepsparse is not available
# download models from the internet "test_get_batched_requests_with_no_ssl" "test_model_tokenized_call_usage" ];
disabledTestPaths = [ # attempts to download models "tests/models/test_huggingface.py" "tests/test_evaluator.py" "tests/test_include_path.py" "tests/test_prompt.py" "tests/test_task_manager.py" "tests/test_tasks.py" "tests/test_unitxt_tasks.py"
# optimum-intel is not available "tests/models/test_openvino.py"
# zeno-client is not packaged "tests/scripts/test_zeno_visualize.py" ];
meta = { changelog = "https://github.com/EleutherAI/lm-evaluation-harness/releases/tag/${src.tag}"; description = "Framework for few-shot evaluation of language models"; homepage = "https://github.com/EleutherAI/lm-evaluation-harness"; license = [ lib.licenses.mit ]; maintainers = [ lib.maintainers.booxter ]; };}