LICENSE
MANIFEST.in
README.md
pyproject.toml
setup.py
src/lighteval/__init__.py
src/lighteval/__main__.py
src/lighteval/data.py
src/lighteval/main_accelerate.py
src/lighteval/main_baseline.py
src/lighteval/main_custom.py
src/lighteval/main_endpoint.py
src/lighteval/main_nanotron.py
src/lighteval/main_sglang.py
src/lighteval/main_tasks.py
src/lighteval/main_vllm.py
src/lighteval/pipeline.py
src/lighteval.egg-info/PKG-INFO
src/lighteval.egg-info/SOURCES.txt
src/lighteval.egg-info/dependency_links.txt
src/lighteval.egg-info/entry_points.txt
src/lighteval.egg-info/requires.txt
src/lighteval.egg-info/top_level.txt
src/lighteval/config/lighteval_config.py
src/lighteval/logging/evaluation_tracker.py
src/lighteval/logging/info_loggers.py
src/lighteval/metrics/__init__.py
src/lighteval/metrics/dynamic_metrics.py
src/lighteval/metrics/judge_prompts.jsonl
src/lighteval/metrics/llm_as_judge.py
src/lighteval/metrics/metrics.py
src/lighteval/metrics/metrics_corpus.py
src/lighteval/metrics/metrics_sample.py
src/lighteval/metrics/normalizations.py
src/lighteval/metrics/sample_preparator.py
src/lighteval/metrics/stderr.py
src/lighteval/metrics/harness_compatibility/drop.py
src/lighteval/metrics/harness_compatibility/truthful_qa.py
src/lighteval/metrics/imports/__init__.py
src/lighteval/metrics/imports/bert_scorer.py
src/lighteval/metrics/imports/data_stats_metric.py
src/lighteval/metrics/imports/data_stats_utils.py
src/lighteval/metrics/imports/summac.py
src/lighteval/metrics/utils/__init__.py
src/lighteval/metrics/utils/extractive_match_utils.py
src/lighteval/metrics/utils/judge_utils.py
src/lighteval/metrics/utils/linguistic_tokenizers.py
src/lighteval/metrics/utils/math_comparison.py
src/lighteval/metrics/utils/metric_utils.py
src/lighteval/models/abstract_model.py
src/lighteval/models/litellm_model.py
src/lighteval/models/model_input.py
src/lighteval/models/model_loader.py
src/lighteval/models/model_output.py
src/lighteval/models/utils.py
src/lighteval/models/custom/custom_model.py
src/lighteval/models/dummy/dummy_model.py
src/lighteval/models/endpoints/endpoint_model.py
src/lighteval/models/endpoints/inference_providers_model.py
src/lighteval/models/endpoints/openai_model.py
src/lighteval/models/endpoints/tgi_model.py
src/lighteval/models/nanotron/nanotron_model.py
src/lighteval/models/sglang/sglang_model.py
src/lighteval/models/transformers/adapter_model.py
src/lighteval/models/transformers/delta_model.py
src/lighteval/models/transformers/transformers_model.py
src/lighteval/models/transformers/vlm_transformers_model.py
src/lighteval/models/vllm/vllm_model.py
src/lighteval/tasks/__init__.py
src/lighteval/tasks/default_prompts.py
src/lighteval/tasks/default_tasks.py
src/lighteval/tasks/lighteval_task.py
src/lighteval/tasks/prompt_manager.py
src/lighteval/tasks/registry.py
src/lighteval/tasks/requests.py
src/lighteval/tasks/extended/__init__.py
src/lighteval/tasks/extended/hle/main.py
src/lighteval/tasks/extended/ifeval/instructions.py
src/lighteval/tasks/extended/ifeval/instructions_registry.py
src/lighteval/tasks/extended/ifeval/instructions_utils.py
src/lighteval/tasks/extended/ifeval/main.py
src/lighteval/tasks/extended/lcb/codegen_metrics.py
src/lighteval/tasks/extended/lcb/main.py
src/lighteval/tasks/extended/mix_eval/judge_prompts.py
src/lighteval/tasks/extended/mix_eval/main.py
src/lighteval/tasks/extended/mix_eval/prompts.py
src/lighteval/tasks/extended/mt_bench/judge_prompt_templates.py
src/lighteval/tasks/extended/mt_bench/main.py
src/lighteval/tasks/extended/olympiade_bench/main.py
src/lighteval/tasks/extended/tiny_benchmarks/main.py
src/lighteval/tasks/multilingual/__init__.py
src/lighteval/tasks/multilingual/adapters.py
src/lighteval/tasks/multilingual/tasks.py
src/lighteval/tasks/multilingual/utils/__init__.py
src/lighteval/tasks/multilingual/utils/adapters_utils.py
src/lighteval/tasks/multilingual/utils/task_utils.py
src/lighteval/tasks/templates/__init__.py
src/lighteval/tasks/templates/boolq.py
src/lighteval/tasks/templates/continuation.py
src/lighteval/tasks/templates/copa.py
src/lighteval/tasks/templates/hellaswag.py
src/lighteval/tasks/templates/multichoice.py
src/lighteval/tasks/templates/nli.py
src/lighteval/tasks/templates/qa.py
src/lighteval/tasks/templates/translation.py
src/lighteval/tasks/templates/utils/__init__.py
src/lighteval/tasks/templates/utils/adapter_utils.py
src/lighteval/tasks/templates/utils/formatting_utils.py
src/lighteval/tasks/templates/utils/formulation.py
src/lighteval/tasks/templates/utils/translation_literals.py
src/lighteval/utils/__init__.py
src/lighteval/utils/imports.py
src/lighteval/utils/language.py
src/lighteval/utils/parallelism.py
src/lighteval/utils/timeout.py
src/lighteval/utils/utils.py
tests/test_prompt_manager.py
tests/test_unit_base_metrics.py
tests/test_unit_harness_metrics.py
tests/test_unit_harness_prompts.py
tests/test_unit_reorder.py