Something went wrong. Try again.
What happens if I push all of nixpkgs
Something went wrong. Try again.
3.0 kB · 157 lines
Nix
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158{ lib, buildPythonPackage, fetchFromGitHub, setuptools-scm, accelerate, aiohttp, antlr4-python3-runtime, causal-conv1d, datasets, dill, evaluate, hf-transfer, immutabledict, jsonlines, langdetect, mamba-ssm, more-itertools, nltk, numexpr, numpy, optimum, pandas, peft, pybind11, pytablewriter, pytestCheckHook, requests, rouge-score, sacrebleu, scikit-learn, sentencepiece, sqlitedict, sympy, tenacity, tiktoken, torch, tqdm, tqdm-multiprocess, transformers, vllm, wandb, word2number, zstandard,}:
buildPythonPackage rec { pname = "lm-eval"; version = "0.4.9.1"; pyproject = true;
src = fetchFromGitHub { owner = "EleutherAI"; repo = "lm-evaluation-harness"; tag = "v${version}"; hash = "sha256-N5NRRabjWxPchwOIkjqYTCKInCmVSY6T5cAmdxNbCkU="; };
build-system = [ setuptools-scm ];
dependencies = [ accelerate datasets dill evaluate jsonlines more-itertools numexpr peft pybind11 pytablewriter rouge-score sacrebleu scikit-learn sqlitedict torch tqdm-multiprocess transformers word2number zstandard ];
optional-dependencies = { api = [ requests aiohttp tenacity tqdm tiktoken ]; hf_transfer = [ hf-transfer ]; ifeval = [ langdetect immutabledict nltk ]; neuronx = [ optimum ] ++ optimum.optional-dependencies.neuronx; mamba = [ mamba-ssm causal-conv1d ]; math = [ sympy antlr4-python3-runtime ]; optimum = [ optimum ] ++ optimum.optional-dependencies.openvino; sentencepiece = [ sentencepiece ]; vllm = [ vllm ]; wandb = [ wandb pandas numpy ]; # Still missing dependencies for the following: # deepsparse, gptq, ibm_watsonx_ai, multilingual, promptsource, sparseml, # zeno, gptqmodel, japanese_leaderboard; all = [...]; };
pythonImportsCheck = [ "lm_eval" ];
nativeCheckInputs = [ pytestCheckHook ] ++ optional-dependencies.api;
preCheck = '' export HOME=$TMP '';
disabledTests = [ "test_deepsparse" # deepsparse is not available "test_model_tokenized_call_usage" # downloads a model ];
disabledTestPaths = [ # attempts to download models "tests/models/test_huggingface.py" "tests/test_evaluator.py" "tests/test_include_path.py" "tests/test_prompt.py" "tests/test_task_manager.py" "tests/test_tasks.py"
# optimum-intel is not available "tests/models/test_openvino.py" ];
meta = { changelog = "https://github.com/EleutherAI/lm-evaluation-harness/releases/tag/${src.tag}"; description = "Framework for few-shot evaluation of language models"; homepage = "https://github.com/EleutherAI/lm-evaluation-harness"; license = [ lib.licenses.mit ]; maintainers = [ lib.maintainers.booxter ]; };}