Merge branch #jbnayahu/bluebench' into 'ibm-research/bluebench'

results/bluebench/2025-08-03T08-32-43_evaluation_results.json

@@ -0,0 +1,1282 @@
+{
+    "environment_info": {
+        "timestamp_utc": "2025-08-03T12:32:38.916038Z",
+        "command_line_invocation": [
+            "/dccstor/jbworks/miniforge3/envs/bb/bin/unitxt-evaluate",
+            "--tasks",
+            "benchmarks.bluebench",
+            "--model",
+            "cross_provider",
+            "--model_args",
+            "model_name=azure/Azure/gpt-4.1-ncf,max_tokens=1024",
+            "--output_path",
+            "./results/bluebench",
+            "--log_samples",
+            "--trust_remote_code",
+            "--batch_size",
+            "8",
+            "--verbosity",
+            "ERROR"
+        ],
+        "parsed_arguments": {
+            "tasks": [
+                "benchmarks.bluebench"
+            ],
+            "split": "test",
+            "num_fewshots": null,
+            "limit": null,
+            "batch_size": 8,
+            "model": "azure/Azure/gpt-4.1-ncf",
+            "model_args": {
+                "max_tokens": 1024
+            },
+            "gen_kwargs": null,
+            "chat_template_kwargs": null,
+            "output_path": "./results/bluebench",
+            "output_file_prefix": "evaluation_results",
+            "log_samples": true,
+            "verbosity": "ERROR",
+            "apply_chat_template": false,
+            "trust_remote_code": true,
+            "disable_hf_cache": false,
+            "cache_dir": null
+        },
+        "unitxt_version": "1.26.5",
+        "unitxt_commit_hash": "N/A",
+        "python_version": "3.10.18",
+        "system": "Linux",
+        "system_version": "#1 SMP PREEMPT_DYNAMIC Fri Aug 9 14:06:03 EDT 2024",
+        "installed_packages": {
+            "nvidia-cufile-cu12": "1.11.1.6",
+            "triton": "3.3.1",
+            "nltk": "3.9.1",
+            "anyio": "4.9.0",
+            "tiktoken": "0.9.0",
+            "charset-normalizer": "3.4.2",
+            "nvidia-cuda-runtime-cu12": "12.6.77",
+            "pyarrow": "21.0.0",
+            "sympy": "1.14.0",
+            "mecab-ko": "1.0.1",
+            "httpcore": "1.0.9",
+            "pip": "25.2",
+            "certifi": "2025.7.14",
+            "evaluate": "0.4.5",
+            "Jinja2": "3.1.6",
+            "jsonschema-specifications": "2025.4.1",
+            "pydantic_core": "2.33.2",
+            "nvidia-cusparse-cu12": "12.5.4.2",
+            "aiosignal": "1.4.0",
+            "yarl": "1.20.1",
+            "unitxt": "1.26.5",
+            "jsonschema": "4.25.0",
+            "portalocker": "3.2.0",
+            "multiprocess": "0.70.16",
+            "nvidia-nvjitlink-cu12": "12.6.85",
+            "nvidia-cublas-cu12": "12.6.4.1",
+            "pydantic": "2.11.7",
+            "async-timeout": "5.0.1",
+            "annotated-types": "0.7.0",
+            "rouge_score": "0.1.2",
+            "contourpy": "1.3.2",
+            "nvidia-cuda-cupti-cu12": "12.6.80",
+            "matplotlib": "3.10.5",
+            "six": "1.17.0",
+            "diskcache": "5.6.3",
+            "tqdm": "4.67.1",
+            "h11": "0.16.0",
+            "zipp": "3.19.2",
+            "tzdata": "2025.2",
+            "bert-score": "0.3.13",
+            "setuptools": "80.9.0",
+            "referencing": "0.36.2",
+            "sacrebleu": "2.5.1",
+            "filelock": "3.18.0",
+            "urllib3": "2.5.0",
+            "scipy": "1.15.3",
+            "nvidia-nccl-cu12": "2.26.2",
+            "kiwisolver": "1.4.8",
+            "networkx": "3.4.2",
+            "typing-inspection": "0.4.1",
+            "sniffio": "1.3.1",
+            "rpds-py": "0.26.0",
+            "nvidia-curand-cu12": "10.3.7.77",
+            "litellm": "1.74.12",
+            "pillow": "11.3.0",
+            "datasets": "3.6.0",
+            "nvidia-cusolver-cu12": "11.7.1.2",
+            "cycler": "0.12.1",
+            "tokenizers": "0.21.4",
+            "distro": "1.9.0",
+            "idna": "3.10",
+            "MarkupSafe": "3.0.2",
+            "frozenlist": "1.7.0",
+            "pyparsing": "3.2.3",
+            "regex": "2025.7.34",
+            "jiter": "0.10.0",
+            "importlib_metadata": "8.0.0",
+            "packaging": "24.2",
+            "psutil": "7.0.0",
+            "mecab-ko-dic": "1.0.0",
+            "joblib": "1.5.1",
+            "transformers": "4.54.1",
+            "fsspec": "2025.3.0",
+            "scikit-learn": "1.7.1",
+            "dill": "0.3.8",
+            "wheel": "0.45.1",
+            "nvidia-nvtx-cu12": "12.6.77",
+            "nvidia-cusparselt-cu12": "0.6.3",
+            "lxml": "6.0.0",
+            "propcache": "0.3.2",
+            "numpy": "2.2.6",
+            "mpmath": "1.3.0",
+            "conllu": "6.0.0",
+            "safetensors": "0.5.3",
+            "requests": "2.32.4",
+            "fonttools": "4.59.0",
+            "tabulate": "0.9.0",
+            "typing_extensions": "4.12.2",
+            "absl-py": "2.3.1",
+            "accelerate": "1.9.0",
+            "nvidia-cufft-cu12": "11.3.0.4",
+            "nvidia-cuda-nvrtc-cu12": "12.6.77",
+            "click": "8.2.1",
+            "attrs": "25.3.0",
+            "exceptiongroup": "1.3.0",
+            "tenacity": "9.1.2",
+            "huggingface-hub": "0.34.3",
+            "pytz": "2025.2",
+            "aiohappyeyeballs": "2.6.1",
+            "python-dateutil": "2.9.0.post0",
+            "torch": "2.7.1",
+            "python-dotenv": "1.1.1",
+            "multidict": "6.6.3",
+            "httpx": "0.28.1",
+            "aiohttp": "3.12.15",
+            "xxhash": "3.5.0",
+            "PyYAML": "6.0.2",
+            "colorama": "0.4.6",
+            "openai": "1.98.0",
+            "threadpoolctl": "3.6.0",
+            "nvidia-cudnn-cu12": "9.5.1.17",
+            "pandas": "2.3.1",
+            "hf-xet": "1.1.5",
+            "jaraco.collections": "5.1.0",
+            "tomli": "2.0.1",
+            "backports.tarfile": "1.2.0",
+            "jaraco.context": "5.3.0",
+            "typeguard": "4.3.0",
+            "autocommand": "2.2.2",
+            "jaraco.text": "3.12.1",
+            "more-itertools": "10.3.0",
+            "platformdirs": "4.2.2",
+            "inflect": "7.3.1",
+            "jaraco.functools": "4.0.1"
+        }
+    },
+    "results": {
+        "bias": {
+            "safety_bbq_age": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 9
+            },
+            "safety_bbq_disability_status": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 9
+            },
+            "safety_bbq_gender_identity": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 9
+            },
+            "safety_bbq_nationality": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 9
+            },
+            "safety_bbq_physical_appearance": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 9
+            },
+            "safety_bbq_race_ethnicity": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 9
+            },
+            "safety_bbq_race_x_gender": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 9
+            },
+            "safety_bbq_race_x_ses": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 9
+            },
+            "safety_bbq_religion": {
+                "accuracy": 0.8888888888888888,
+                "accuracy_ci_low": 0.5310928992288233,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.8888888888888888,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.5310928992288233,
+                "num_of_instances": 9
+            },
+            "safety_bbq_ses": {
+                "accuracy": 0.8888888888888888,
+                "accuracy_ci_low": 0.5555555555555556,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.8888888888888888,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.5555555555555556,
+                "num_of_instances": 9
+            },
+            "safety_bbq_sexual_orientation": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 9
+            },
+            "score": 0.9797979797979798,
+            "score_name": "subsets_mean",
+            "num_of_instances": 99
+        },
+        "chatbot_abilities": {
+            "arena_hard_generation_english_gpt_4_0314_reference": {
+                "num_of_instances": 100,
+                "llama_3_70b_instruct_template_arena_hard": 0.9636363636363636,
+                "score": 0.9636363636363636,
+                "score_name": "llama_3_70b_instruct_template_arena_hard"
+            },
+            "score": 0.9636363636363636,
+            "score_name": "subsets_mean",
+            "num_of_instances": 100
+        },
+        "entity_extraction": {
+            "universal_ner_en_ewt": {
+                "num_of_instances": 100,
+                "f1_Person": 0.8085106382978724,
+                "f1_Organization": 0.6857142857142857,
+                "f1_Location": 0.6956521739130435,
+                "f1_macro": 0.7299590326417338,
+                "recall_macro": 0.7832988267770876,
+                "precision_macro": 0.6967893217893218,
+                "in_classes_support": 1.0,
+                "f1_micro": 0.7239263803680981,
+                "recall_micro": 0.7866666666666666,
+                "precision_micro": 0.6704545454545454,
+                "score": 0.7239263803680981,
+                "score_name": "f1_micro",
+                "score_ci_low": 0.6420029997196985,
+                "score_ci_high": 0.782066798498023,
+                "f1_micro_ci_low": 0.6420029997196985,
+                "f1_micro_ci_high": 0.782066798498023
+            },
+            "score": 0.7239263803680981,
+            "score_name": "subsets_mean",
+            "num_of_instances": 100
+        },
+        "knowledge": {
+            "mmlu_pro_biology": {
+                "accuracy": 0.7142857142857143,
+                "accuracy_ci_low": 0.2857142857142857,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.7142857142857143,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.2857142857142857,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_business": {
+                "accuracy": 0.42857142857142855,
+                "accuracy_ci_low": 0.14285714285714285,
+                "accuracy_ci_high": 0.8571428571428571,
+                "score_name": "accuracy",
+                "score": 0.42857142857142855,
+                "score_ci_high": 0.8571428571428571,
+                "score_ci_low": 0.14285714285714285,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_chemistry": {
+                "accuracy": 0.2857142857142857,
+                "accuracy_ci_low": 0.0,
+                "accuracy_ci_high": 0.7142857142857143,
+                "score_name": "accuracy",
+                "score": 0.2857142857142857,
+                "score_ci_high": 0.7142857142857143,
+                "score_ci_low": 0.0,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_computer_science": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_economics": {
+                "accuracy": 0.8571428571428571,
+                "accuracy_ci_low": 0.42857142857142855,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.8571428571428571,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.42857142857142855,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_engineering": {
+                "accuracy": 0.42857142857142855,
+                "accuracy_ci_low": 0.14285714285714285,
+                "accuracy_ci_high": 0.8571428571428571,
+                "score_name": "accuracy",
+                "score": 0.42857142857142855,
+                "score_ci_high": 0.8571428571428571,
+                "score_ci_low": 0.14285714285714285,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_health": {
+                "accuracy": 0.5714285714285714,
+                "accuracy_ci_low": 0.14285714285714285,
+                "accuracy_ci_high": 0.8571428571428571,
+                "score_name": "accuracy",
+                "score": 0.5714285714285714,
+                "score_ci_high": 0.8571428571428571,
+                "score_ci_low": 0.14285714285714285,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_history": {
+                "accuracy": 0.2857142857142857,
+                "accuracy_ci_low": 0.0,
+                "accuracy_ci_high": 0.7142857142857143,
+                "score_name": "accuracy",
+                "score": 0.2857142857142857,
+                "score_ci_high": 0.7142857142857143,
+                "score_ci_low": 0.0,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_law": {
+                "accuracy": 0.7142857142857143,
+                "accuracy_ci_low": 0.2857142857142857,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.7142857142857143,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.2857142857142857,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_math": {
+                "accuracy": 0.5714285714285714,
+                "accuracy_ci_low": 0.14285714285714285,
+                "accuracy_ci_high": 0.8571428571428571,
+                "score_name": "accuracy",
+                "score": 0.5714285714285714,
+                "score_ci_high": 0.8571428571428571,
+                "score_ci_low": 0.14285714285714285,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_other": {
+                "accuracy": 0.5714285714285714,
+                "accuracy_ci_low": 0.14285714285714285,
+                "accuracy_ci_high": 0.8571428571428571,
+                "score_name": "accuracy",
+                "score": 0.5714285714285714,
+                "score_ci_high": 0.8571428571428571,
+                "score_ci_low": 0.14285714285714285,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_philosophy": {
+                "accuracy": 0.8571428571428571,
+                "accuracy_ci_low": 0.31927964061584246,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.8571428571428571,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.31927964061584246,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_physics": {
+                "accuracy": 0.14285714285714285,
+                "accuracy_ci_low": 0.0,
+                "accuracy_ci_high": 0.5714285714285714,
+                "score_name": "accuracy",
+                "score": 0.14285714285714285,
+                "score_ci_high": 0.5714285714285714,
+                "score_ci_low": 0.0,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_psychology": {
+                "accuracy": 0.7142857142857143,
+                "accuracy_ci_low": 0.2857142857142857,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.7142857142857143,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.2857142857142857,
+                "num_of_instances": 7
+            },
+            "score": 0.5816326530612245,
+            "score_name": "subsets_mean",
+            "num_of_instances": 98
+        },
+        "legal": {
+            "legalbench_abercrombie": {
+                "f1_macro": 0.503030303030303,
+                "f1_suggestive": 0.18181818181818182,
+                "f1_generic": 0.8,
+                "f1_fanciful": 0.4,
+                "f1_descriptive": 0.3333333333333333,
+                "f1_arbitrary": 0.8,
+                "f1_macro_ci_low": 0.3144226612853847,
+                "f1_macro_ci_high": 0.76,
+                "score_name": "f1_micro",
+                "score": 0.4375,
+                "score_ci_high": 0.6666666666666666,
+                "score_ci_low": 0.21080178633741004,
+                "num_of_instances": 20,
+                "accuracy": 0.35,
+                "accuracy_ci_low": 0.15,
+                "accuracy_ci_high": 0.6,
+                "f1_micro": 0.4375,
+                "f1_micro_ci_low": 0.21080178633741004,
+                "f1_micro_ci_high": 0.6666666666666666
+            },
+            "legalbench_corporate_lobbying": {
+                "f1_macro": 0.6360153256704981,
+                "f1_no": 0.8275862068965517,
+                "f1_yes": 0.4444444444444444,
+                "f1_macro_ci_low": 0.4117647058823529,
+                "f1_macro_ci_high": 0.8932752204410275,
+                "score_name": "f1_micro",
+                "score": 0.7368421052631579,
+                "score_ci_high": 0.8947368421052632,
+                "score_ci_low": 0.5263157894736842,
+                "num_of_instances": 20,
+                "accuracy": 0.7,
+                "accuracy_ci_low": 0.5,
+                "accuracy_ci_high": 0.85,
+                "f1_micro": 0.7368421052631579,
+                "f1_micro_ci_low": 0.5263157894736842,
+                "f1_micro_ci_high": 0.8947368421052632
+            },
+            "legalbench_function_of_decision_section": {
+                "f1_macro": 0.24591836734693878,
+                "f1_conclusion": 0.25,
+                "f1_issue": 0.4,
+                "f1_decree": 0.0,
+                "f1_rule": 0.0,
+                "f1_analysis": 0.0,
+                "f1_facts": 0.5714285714285714,
+                "f1_procedural history": 0.5,
+                "f1_macro_ci_low": 0.06687444590451443,
+                "f1_macro_ci_high": 0.4222222222222222,
+                "score_name": "f1_micro",
+                "score": 0.3157894736842105,
+                "score_ci_high": 0.5405405405405406,
+                "score_ci_low": 0.11428571428571428,
+                "num_of_instances": 20,
+                "accuracy": 0.3,
+                "accuracy_ci_low": 0.15,
+                "accuracy_ci_high": 0.55,
+                "f1_micro": 0.3157894736842105,
+                "f1_micro_ci_low": 0.11428571428571428,
+                "f1_micro_ci_high": 0.5405405405405406
+            },
+            "legalbench_international_citizenship_questions": {
+                "f1_macro": 0.7,
+                "f1_yes": 0.7,
+                "f1_no": 0.7,
+                "f1_macro_ci_low": 0.4949494949494949,
+                "f1_macro_ci_high": 0.898989898989899,
+                "score_name": "f1_micro",
+                "score": 0.7,
+                "score_ci_high": 0.859273262592211,
+                "score_ci_low": 0.5,
+                "num_of_instances": 20,
+                "accuracy": 0.7,
+                "accuracy_ci_low": 0.5,
+                "accuracy_ci_high": 0.859273262592211,
+                "f1_micro": 0.7,
+                "f1_micro_ci_low": 0.5,
+                "f1_micro_ci_high": 0.859273262592211
+            },
+            "legalbench_proa": {
+                "f1_macro": 0.8375,
+                "f1_yes": 0.875,
+                "f1_no": 0.8,
+                "f1_macro_ci_low": 0.6112456731982998,
+                "f1_macro_ci_high": 0.9449275362318841,
+                "score_name": "f1_micro",
+                "score": 0.8333333333333334,
+                "score_ci_high": 0.9473684210526315,
+                "score_ci_low": 0.6076349233925447,
+                "num_of_instances": 20,
+                "accuracy": 0.75,
+                "accuracy_ci_low": 0.50468235519016,
+                "accuracy_ci_high": 0.9,
+                "f1_micro": 0.8333333333333334,
+                "f1_micro_ci_low": 0.6076349233925447,
+                "f1_micro_ci_high": 0.9473684210526315
+            },
+            "score": 0.6046929824561403,
+            "score_name": "subsets_mean",
+            "num_of_instances": 100
+        },
+        "news_classification": {
+            "20_newsgroups_short": {
+                "f1_macro": 0.6248042235542235,
+                "f1_cars": 0.7272727272727273,
+                "f1_windows x": 0.5714285714285714,
+                "f1_computer graphics": 0.625,
+                "f1_atheism": 0.5,
+                "f1_religion": 0.0,
+                "f1_medicine": 1.0,
+                "f1_christianity": 0.8571428571428571,
+                "f1_microsoft windows": 0.8,
+                "f1_middle east": 0.7272727272727273,
+                "f1_motorcycles": 0.6,
+                "f1_pc hardware": 0.6666666666666666,
+                "f1_mac hardware": 0.9090909090909091,
+                "f1_electronics": 0.5,
+                "f1_for sale": 0.4,
+                "f1_guns": 0.2857142857142857,
+                "f1_space": 0.8888888888888888,
+                "f1_cryptography": 0.3333333333333333,
+                "f1_baseball": 0.6,
+                "f1_hockey": 0.8888888888888888,
+                "f1_politics": 0.6153846153846154,
+                "f1_macro_ci_low": 0.5435598724209302,
+                "f1_macro_ci_high": 0.727169076447674,
+                "score_name": "f1_micro",
+                "score": 0.6629213483146067,
+                "score_ci_high": 0.7431693989071039,
+                "score_ci_low": 0.561198821408691,
+                "num_of_instances": 100,
+                "accuracy": 0.59,
+                "accuracy_ci_low": 0.49,
+                "accuracy_ci_high": 0.68,
+                "f1_micro": 0.6629213483146067,
+                "f1_micro_ci_low": 0.561198821408691,
+                "f1_micro_ci_high": 0.7431693989071039
+            },
+            "score": 0.6629213483146067,
+            "score_name": "subsets_mean",
+            "num_of_instances": 100
+        },
+        "product_help": {
+            "cfpb_product_2023": {
+                "f1_macro": 0.7315018315018316,
+                "f1_credit reporting or credit repair services or other personal consumer reports": 0.8923076923076924,
+                "f1_debt collection": 0.6666666666666666,
+                "f1_payday loan or title loan or personal loan": 0.0,
+                "f1_student loan": 0.7692307692307693,
+                "f1_credit card or prepaid card": 0.8571428571428571,
+                "f1_checking or savings account": 1.0,
+                "f1_mortgage": 0.6666666666666666,
+                "f1_money transfer or virtual currency or money service": 1.0,
+                "f1_macro_ci_low": 0.5341663901608891,
+                "f1_macro_ci_high": 0.8376311452519075,
+                "score_name": "f1_micro",
+                "score": 0.8586387434554974,
+                "score_ci_high": 0.916233746693676,
+                "score_ci_low": 0.783068783068783,
+                "num_of_instances": 100,
+                "accuracy": 0.82,
+                "accuracy_ci_low": 0.74,
+                "accuracy_ci_high": 0.89,
+                "f1_micro": 0.8586387434554974,
+                "f1_micro_ci_low": 0.783068783068783,
+                "f1_micro_ci_high": 0.916233746693676
+            },
+            "cfpb_product_watsonx": {
+                "f1_macro": 0.799064551009631,
+                "f1_mortgages and loans": 0.782608695652174,
+                "f1_credit card": 0.8571428571428571,
+                "f1_debt collection": 0.7368421052631579,
+                "f1_credit reporting": 0.6956521739130435,
+                "f1_retail banking": 0.9230769230769231,
+                "f1_macro_ci_low": 0.6817947520200675,
+                "f1_macro_ci_high": 0.8954471115753557,
+                "score_name": "f1_micro",
+                "score": 0.7878787878787878,
+                "score_ci_high": 0.88,
+                "score_ci_low": 0.66,
+                "num_of_instances": 50,
+                "accuracy": 0.78,
+                "accuracy_ci_low": 0.66,
+                "accuracy_ci_high": 0.88,
+                "f1_micro": 0.7878787878787878,
+                "f1_micro_ci_low": 0.66,
+                "f1_micro_ci_high": 0.88
+            },
+            "score": 0.8232587656671426,
+            "score_name": "subsets_mean",
+            "num_of_instances": 150
+        },
+        "qa_finance": {
+            "fin_qa": {
+                "num_of_instances": 100,
+                "execution_accuracy": 0.32,
+                "program_accuracy": 0.33,
+                "score": 0.33,
+                "score_name": "program_accuracy",
+                "execution_accuracy_ci_low": 0.23,
+                "execution_accuracy_ci_high": 0.42,
+                "program_accuracy_ci_low": 0.25,
+                "program_accuracy_ci_high": 0.43,
+                "score_ci_low": 0.25,
+                "score_ci_high": 0.43
+            },
+            "score": 0.33,
+            "score_name": "subsets_mean",
+            "num_of_instances": 100
+        },
+        "rag_general": {
+            "rag_response_generation_clapnq": {
+                "precision": 0.4343698865384216,
+                "recall": 0.6709870742364467,
+                "f1": 0.4846153577687695,
+                "precision_ci_low": 0.3998020013123197,
+                "precision_ci_high": 0.4698068976247165,
+                "recall_ci_low": 0.6303563546991966,
+                "recall_ci_high": 0.7089929114321589,
+                "f1_ci_low": 0.4526567482268106,
+                "f1_ci_high": 0.5146388287292981,
+                "score_name": "f1",
+                "score": 0.4846153577687695,
+                "score_ci_high": 0.5146388287292981,
+                "score_ci_low": 0.4526567482268106,
+                "num_of_instances": 100,
+                "correctness_f1_bert_score.deberta_large_mnli": 0.6816095349192619,
+                "correctness_recall_bert_score.deberta_large_mnli": 0.729021328985691,
+                "correctness_precision_bert_score.deberta_large_mnli": 0.6500837489962578,
+                "faithfullness_f1_token_overlap": 0.43231085895541577,
+                "faithfullness_recall_token_overlap": 0.3622151601642728,
+                "faithfullness_precision_token_overlap": 0.6636443305066012,
+                "correctness_f1_token_overlap": 0.4846153577687695,
+                "correctness_recall_token_overlap": 0.6709870742364467,
+                "correctness_precision_token_overlap": 0.4343698865384216
+            },
+            "score": 0.4846153577687695,
+            "score_name": "subsets_mean",
+            "num_of_instances": 100
+        },
+        "reasoning": {
+            "hellaswag": {
+                "accuracy": 0.61,
+                "accuracy_ci_low": 0.51,
+                "accuracy_ci_high": 0.7,
+                "score_name": "accuracy",
+                "score": 0.61,
+                "score_ci_high": 0.7,
+                "score_ci_low": 0.51,
+                "num_of_instances": 100
+            },
+            "openbook_qa": {
+                "accuracy": 0.94,
+                "accuracy_ci_low": 0.87,
+                "accuracy_ci_high": 0.98,
+                "score_name": "accuracy",
+                "score": 0.94,
+                "score_ci_high": 0.98,
+                "score_ci_low": 0.87,
+                "num_of_instances": 100
+            },
+            "score": 0.7749999999999999,
+            "score_name": "subsets_mean",
+            "num_of_instances": 200
+        },
+        "safety": {
+            "attaq_500": {
+                "severity_critical": 0.0,
+                "severity_high": 0.0,
+                "severity_medium": 0.0,
+                "severity_low": 100.0,
+                "category_harmful_info": 0.875045096996935,
+                "category_deception": 0.9087016443260992,
+                "category_explicit_content": 0.9104775799946352,
+                "category_violence": 0.866230480028649,
+                "category_discrimination": 0.8078865560274275,
+                "category_substance_abuse": 0.8765388902690676,
+                "category_pii": 0.8728977072018164,
+                "safety": 0.8748689444958777,
+                "safety_ci_low": 0.859557728046316,
+                "safety_ci_high": 0.8896609433147404,
+                "score_name": "safety",
+                "score": 0.8748689444958777,
+                "score_ci_high": 0.8896609433147404,
+                "score_ci_low": 0.859557728046316,
+                "num_of_instances": 100
+            },
+            "score": 0.8748689444958777,
+            "score_name": "subsets_mean",
+            "num_of_instances": 100
+        },
+        "summarization": {
+            "billsum_document_filtered_to_6000_chars": {
+                "num_of_instances": 100,
+                "rougeLsum": 0.3449679916101768,
+                "rouge2": 0.17691065019354774,
+                "rougeL": 0.2755019690756654,
+                "score": 0.2755019690756654,
+                "score_name": "rougeL",
+                "rouge1": 0.40866416882835144,
+                "rougeLsum_ci_low": 0.32452422336075787,
+                "rougeLsum_ci_high": 0.362432692973816,
+                "rouge2_ci_low": 0.16371569781722647,
+                "rouge2_ci_high": 0.19131565638937167,
+                "rougeL_ci_low": 0.260558384013039,
+                "rougeL_ci_high": 0.2917913421726095,
+                "score_ci_low": 0.260558384013039,
+                "score_ci_high": 0.2917913421726095,
+                "rouge1_ci_low": 0.3860176168688086,
+                "rouge1_ci_high": 0.42632106299588085
+            },
+            "tldr_document_filtered_to_6000_chars": {
+                "num_of_instances": 100,
+                "rougeLsum": 0.09942588005347286,
+                "rouge2": 0.015180143919691523,
+                "rougeL": 0.08942312196371219,
+                "score": 0.08942312196371219,
+                "score_name": "rougeL",
+                "rouge1": 0.1194115419368174,
+                "rougeLsum_ci_low": 0.0862462499816593,
+                "rougeLsum_ci_high": 0.11236018272084582,
+                "rouge2_ci_low": 0.010616515576465502,
+                "rouge2_ci_high": 0.02165065601326996,
+                "rougeL_ci_low": 0.07820323036355636,
+                "rougeL_ci_high": 0.10112740368476562,
+                "score_ci_low": 0.07820323036355636,
+                "score_ci_high": 0.10112740368476562,
+                "rouge1_ci_low": 0.10290535826717513,
+                "rouge1_ci_high": 0.13692429898442587
+            },
+            "score": 0.18246254551968877,
+            "score_name": "subsets_mean",
+            "num_of_instances": 200
+        },
+        "translation": {
+            "mt_flores_101_ara_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    152,
+                    108,
+                    78,
+                    57
+                ],
+                "totals": [
+                    213,
+                    207,
+                    201,
+                    195
+                ],
+                "precisions": [
+                    0.7136150234741784,
+                    0.5217391304347826,
+                    0.3880597014925373,
+                    0.2923076923076923
+                ],
+                "bp": 1.0,
+                "sys_len": 213,
+                "ref_len": 208,
+                "sacrebleu": 0.4533295675744374,
+                "score": 0.4533295675744374,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.2970856371571267,
+                "score_ci_high": 0.6155700608318663,
+                "sacrebleu_ci_low": 0.2970856371571267,
+                "sacrebleu_ci_high": 0.6155700608318663
+            },
+            "mt_flores_101_deu_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    143,
+                    89,
+                    60,
+                    43
+                ],
+                "totals": [
+                    221,
+                    215,
+                    209,
+                    203
+                ],
+                "precisions": [
+                    0.6470588235294117,
+                    0.41395348837209306,
+                    0.28708133971291866,
+                    0.21182266009852216
+                ],
+                "bp": 1.0,
+                "sys_len": 221,
+                "ref_len": 208,
+                "sacrebleu": 0.35724665668654765,
+                "score": 0.35724665668654765,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.2566990236689781,
+                "score_ci_high": 0.5362601500874902,
+                "sacrebleu_ci_low": 0.2566990236689781,
+                "sacrebleu_ci_high": 0.5362601500874902
+            },
+            "mt_flores_101_eng_ara": {
+                "num_of_instances": 6,
+                "counts": [
+                    116,
+                    72,
+                    49,
+                    30
+                ],
+                "totals": [
+                    204,
+                    198,
+                    192,
+                    186
+                ],
+                "precisions": [
+                    0.5686274509803921,
+                    0.36363636363636365,
+                    0.2552083333333333,
+                    0.16129032258064516
+                ],
+                "bp": 0.9757881223212935,
+                "sys_len": 204,
+                "ref_len": 209,
+                "sacrebleu": 0.2963842353604502,
+                "score": 0.2963842353604502,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.21279117709012274,
+                "score_ci_high": 0.3978311539100688,
+                "sacrebleu_ci_low": 0.21279117709012274,
+                "sacrebleu_ci_high": 0.3978311539100688
+            },
+            "mt_flores_101_eng_deu": {
+                "num_of_instances": 6,
+                "counts": [
+                    148,
+                    95,
+                    62,
+                    45
+                ],
+                "totals": [
+                    210,
+                    204,
+                    198,
+                    192
+                ],
+                "precisions": [
+                    0.7047619047619048,
+                    0.46568627450980393,
+                    0.3131313131313131,
+                    0.234375
+                ],
+                "bp": 0.9718328750329812,
+                "sys_len": 210,
+                "ref_len": 216,
+                "sacrebleu": 0.382855593891157,
+                "score": 0.382855593891157,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.28077912493137325,
+                "score_ci_high": 0.5224412329440934,
+                "sacrebleu_ci_low": 0.28077912493137325,
+                "sacrebleu_ci_high": 0.5224412329440934
+            },
+            "mt_flores_101_eng_fra": {
+                "num_of_instances": 6,
+                "counts": [
+                    184,
+                    135,
+                    99,
+                    72
+                ],
+                "totals": [
+                    240,
+                    234,
+                    228,
+                    222
+                ],
+                "precisions": [
+                    0.7666666666666667,
+                    0.576923076923077,
+                    0.43421052631578944,
+                    0.32432432432432434
+                ],
+                "bp": 1.0,
+                "sys_len": 240,
+                "ref_len": 235,
+                "sacrebleu": 0.4995754525815319,
+                "score": 0.4995754525815319,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.4515767855995504,
+                "score_ci_high": 0.5566329044186812,
+                "sacrebleu_ci_low": 0.4515767855995504,
+                "sacrebleu_ci_high": 0.5566329044186812
+            },
+            "mt_flores_101_eng_kor": {
+                "num_of_instances": 6,
+                "counts": [
+                    163,
+                    87,
+                    56,
+                    36
+                ],
+                "totals": [
+                    297,
+                    291,
+                    285,
+                    279
+                ],
+                "precisions": [
+                    0.5488215488215488,
+                    0.29896907216494845,
+                    0.19649122807017544,
+                    0.12903225806451613
+                ],
+                "bp": 1.0,
+                "sys_len": 297,
+                "ref_len": 249,
+                "sacrebleu": 0.25396549824957954,
+                "score": 0.25396549824957954,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.18656507398105443,
+                "score_ci_high": 0.3676949692724427,
+                "sacrebleu_ci_low": 0.18656507398105443,
+                "sacrebleu_ci_high": 0.3676949692724427
+            },
+            "mt_flores_101_eng_por": {
+                "num_of_instances": 6,
+                "counts": [
+                    181,
+                    140,
+                    113,
+                    92
+                ],
+                "totals": [
+                    226,
+                    220,
+                    214,
+                    208
+                ],
+                "precisions": [
+                    0.8008849557522124,
+                    0.6363636363636364,
+                    0.5280373831775701,
+                    0.44230769230769235
+                ],
+                "bp": 1.0,
+                "sys_len": 226,
+                "ref_len": 222,
+                "sacrebleu": 0.587375953828071,
+                "score": 0.587375953828071,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.5264308904974885,
+                "score_ci_high": 0.6527446503464284,
+                "sacrebleu_ci_low": 0.5264308904974885,
+                "sacrebleu_ci_high": 0.6527446503464284
+            },
+            "mt_flores_101_eng_ron": {
+                "num_of_instances": 6,
+                "counts": [
+                    159,
+                    110,
+                    79,
+                    59
+                ],
+                "totals": [
+                    230,
+                    224,
+                    218,
+                    212
+                ],
+                "precisions": [
+                    0.6913043478260871,
+                    0.49107142857142855,
+                    0.3623853211009175,
+                    0.2783018867924528
+                ],
+                "bp": 1.0,
+                "sys_len": 230,
+                "ref_len": 230,
+                "sacrebleu": 0.4301551985882873,
+                "score": 0.4301551985882873,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.35322253633753253,
+                "score_ci_high": 0.542330368706405,
+                "sacrebleu_ci_low": 0.35322253633753253,
+                "sacrebleu_ci_high": 0.542330368706405
+            },
+            "mt_flores_101_eng_spa": {
+                "num_of_instances": 6,
+                "counts": [
+                    168,
+                    106,
+                    70,
+                    45
+                ],
+                "totals": [
+                    237,
+                    231,
+                    225,
+                    219
+                ],
+                "precisions": [
+                    0.7088607594936708,
+                    0.4588744588744589,
+                    0.3111111111111111,
+                    0.2054794520547945
+                ],
+                "bp": 0.9750013184817767,
+                "sys_len": 237,
+                "ref_len": 243,
+                "sacrebleu": 0.37024558499956833,
+                "score": 0.37024558499956833,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.2914675063581613,
+                "score_ci_high": 0.4722241333633745,
+                "sacrebleu_ci_low": 0.2914675063581613,
+                "sacrebleu_ci_high": 0.4722241333633745
+            },
+            "mt_flores_101_fra_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    158,
+                    112,
+                    82,
+                    63
+                ],
+                "totals": [
+                    218,
+                    212,
+                    206,
+                    200
+                ],
+                "precisions": [
+                    0.724770642201835,
+                    0.5283018867924528,
+                    0.3980582524271845,
+                    0.315
+                ],
+                "bp": 1.0,
+                "sys_len": 218,
+                "ref_len": 208,
+                "sacrebleu": 0.4680960595371609,
+                "score": 0.4680960595371609,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.26555389781536615,
+                "score_ci_high": 0.6882400443755067,
+                "sacrebleu_ci_low": 0.26555389781536615,
+                "sacrebleu_ci_high": 0.6882400443755067
+            },
+            "mt_flores_101_jpn_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    147,
+                    92,
+                    63,
+                    43
+                ],
+                "totals": [
+                    219,
+                    213,
+                    207,
+                    201
+                ],
+                "precisions": [
+                    0.6712328767123288,
+                    0.431924882629108,
+                    0.30434782608695654,
+                    0.21393034825870647
+                ],
+                "bp": 1.0,
+                "sys_len": 219,
+                "ref_len": 208,
+                "sacrebleu": 0.3706645149919594,
+                "score": 0.3706645149919594,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.18561228093884444,
+                "score_ci_high": 0.45332840825867166,
+                "sacrebleu_ci_low": 0.18561228093884444,
+                "sacrebleu_ci_high": 0.45332840825867166
+            },
+            "mt_flores_101_kor_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    136,
+                    82,
+                    53,
+                    38
+                ],
+                "totals": [
+                    208,
+                    202,
+                    196,
+                    190
+                ],
+                "precisions": [
+                    0.6538461538461539,
+                    0.4059405940594059,
+                    0.27040816326530615,
+                    0.2
+                ],
+                "bp": 1.0,
+                "sys_len": 208,
+                "ref_len": 208,
+                "sacrebleu": 0.346136152997744,
+                "score": 0.346136152997744,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.197154502834615,
+                "score_ci_high": 0.4450318608831673,
+                "sacrebleu_ci_low": 0.197154502834615,
+                "sacrebleu_ci_high": 0.4450318608831673
+            },
+            "mt_flores_101_por_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    161,
+                    117,
+                    85,
+                    65
+                ],
+                "totals": [
+                    213,
+                    207,
+                    201,
+                    195
+                ],
+                "precisions": [
+                    0.755868544600939,
+                    0.5652173913043478,
+                    0.42288557213930345,
+                    0.33333333333333337
+                ],
+                "bp": 1.0,
+                "sys_len": 213,
+                "ref_len": 208,
+                "sacrebleu": 0.4953827168207276,
+                "score": 0.4953827168207276,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.28814389311162725,
+                "score_ci_high": 0.617325493054366,
+                "sacrebleu_ci_low": 0.28814389311162725,
+                "sacrebleu_ci_high": 0.617325493054366
+            },
+            "mt_flores_101_ron_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    160,
+                    120,
+                    92,
+                    76
+                ],
+                "totals": [
+                    222,
+                    216,
+                    210,
+                    204
+                ],
+                "precisions": [
+                    0.7207207207207208,
+                    0.5555555555555556,
+                    0.4380952380952381,
+                    0.37254901960784315
+                ],
+                "bp": 1.0,
+                "sys_len": 222,
+                "ref_len": 208,
+                "sacrebleu": 0.505605296924794,
+                "score": 0.505605296924794,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.32967576416379546,
+                "score_ci_high": 0.7153750601746934,
+                "sacrebleu_ci_low": 0.32967576416379546,
+                "sacrebleu_ci_high": 0.7153750601746934
+            },
+            "mt_flores_101_spa_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    152,
+                    95,
+                    62,
+                    44
+                ],
+                "totals": [
+                    226,
+                    220,
+                    214,
+                    208
+                ],
+                "precisions": [
+                    0.672566371681416,
+                    0.4318181818181818,
+                    0.28971962616822433,
+                    0.21153846153846154
+                ],
+                "bp": 1.0,
+                "sys_len": 226,
+                "ref_len": 208,
+                "sacrebleu": 0.3652589217481651,
+                "score": 0.3652589217481651,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.27931028013393655,
+                "score_ci_high": 0.40059757012052705,
+                "sacrebleu_ci_low": 0.27931028013393655,
+                "sacrebleu_ci_high": 0.40059757012052705
+            },
+            "score": 0.4121518269853454,
+            "score_name": "subsets_mean",
+            "num_of_instances": 90
+        },
+        "score": 0.646074242159326,
+        "score_name": "subsets_mean",
+        "num_of_instances": 1537
+    }
+}

results/bluebench/2025-08-03T08-51-48_evaluation_results.json

@@ -0,0 +1,1282 @@
+{
+    "environment_info": {
+        "timestamp_utc": "2025-08-03T12:51:43.070081Z",
+        "command_line_invocation": [
+            "/dccstor/jbworks/miniforge3/envs/bb/bin/unitxt-evaluate",
+            "--tasks",
+            "benchmarks.bluebench",
+            "--model",
+            "cross_provider",
+            "--model_args",
+            "model_name=azure/Azure/gpt-4.1-mini-ncf,max_tokens=1024",
+            "--output_path",
+            "./results/bluebench",
+            "--log_samples",
+            "--trust_remote_code",
+            "--batch_size",
+            "8",
+            "--verbosity",
+            "ERROR"
+        ],
+        "parsed_arguments": {
+            "tasks": [
+                "benchmarks.bluebench"
+            ],
+            "split": "test",
+            "num_fewshots": null,
+            "limit": null,
+            "batch_size": 8,
+            "model": "azure/Azure/gpt-4.1-mini-ncf",
+            "model_args": {
+                "max_tokens": 1024
+            },
+            "gen_kwargs": null,
+            "chat_template_kwargs": null,
+            "output_path": "./results/bluebench",
+            "output_file_prefix": "evaluation_results",
+            "log_samples": true,
+            "verbosity": "ERROR",
+            "apply_chat_template": false,
+            "trust_remote_code": true,
+            "disable_hf_cache": false,
+            "cache_dir": null
+        },
+        "unitxt_version": "1.26.5",
+        "unitxt_commit_hash": "N/A",
+        "python_version": "3.10.18",
+        "system": "Linux",
+        "system_version": "#1 SMP PREEMPT_DYNAMIC Fri Aug 9 14:06:03 EDT 2024",
+        "installed_packages": {
+            "nvidia-cufile-cu12": "1.11.1.6",
+            "triton": "3.3.1",
+            "nltk": "3.9.1",
+            "anyio": "4.9.0",
+            "tiktoken": "0.9.0",
+            "charset-normalizer": "3.4.2",
+            "nvidia-cuda-runtime-cu12": "12.6.77",
+            "pyarrow": "21.0.0",
+            "sympy": "1.14.0",
+            "mecab-ko": "1.0.1",
+            "httpcore": "1.0.9",
+            "pip": "25.2",
+            "certifi": "2025.7.14",
+            "evaluate": "0.4.5",
+            "Jinja2": "3.1.6",
+            "jsonschema-specifications": "2025.4.1",
+            "pydantic_core": "2.33.2",
+            "nvidia-cusparse-cu12": "12.5.4.2",
+            "aiosignal": "1.4.0",
+            "yarl": "1.20.1",
+            "unitxt": "1.26.5",
+            "jsonschema": "4.25.0",
+            "portalocker": "3.2.0",
+            "multiprocess": "0.70.16",
+            "nvidia-nvjitlink-cu12": "12.6.85",
+            "nvidia-cublas-cu12": "12.6.4.1",
+            "pydantic": "2.11.7",
+            "async-timeout": "5.0.1",
+            "annotated-types": "0.7.0",
+            "rouge_score": "0.1.2",
+            "contourpy": "1.3.2",
+            "nvidia-cuda-cupti-cu12": "12.6.80",
+            "matplotlib": "3.10.5",
+            "six": "1.17.0",
+            "diskcache": "5.6.3",
+            "tqdm": "4.67.1",
+            "h11": "0.16.0",
+            "zipp": "3.19.2",
+            "tzdata": "2025.2",
+            "bert-score": "0.3.13",
+            "setuptools": "80.9.0",
+            "referencing": "0.36.2",
+            "sacrebleu": "2.5.1",
+            "filelock": "3.18.0",
+            "urllib3": "2.5.0",
+            "scipy": "1.15.3",
+            "nvidia-nccl-cu12": "2.26.2",
+            "kiwisolver": "1.4.8",
+            "networkx": "3.4.2",
+            "typing-inspection": "0.4.1",
+            "sniffio": "1.3.1",
+            "rpds-py": "0.26.0",
+            "nvidia-curand-cu12": "10.3.7.77",
+            "litellm": "1.74.12",
+            "pillow": "11.3.0",
+            "datasets": "3.6.0",
+            "nvidia-cusolver-cu12": "11.7.1.2",
+            "cycler": "0.12.1",
+            "tokenizers": "0.21.4",
+            "distro": "1.9.0",
+            "idna": "3.10",
+            "MarkupSafe": "3.0.2",
+            "frozenlist": "1.7.0",
+            "pyparsing": "3.2.3",
+            "regex": "2025.7.34",
+            "jiter": "0.10.0",
+            "importlib_metadata": "8.0.0",
+            "packaging": "24.2",
+            "psutil": "7.0.0",
+            "mecab-ko-dic": "1.0.0",
+            "joblib": "1.5.1",
+            "transformers": "4.54.1",
+            "fsspec": "2025.3.0",
+            "scikit-learn": "1.7.1",
+            "dill": "0.3.8",
+            "wheel": "0.45.1",
+            "nvidia-nvtx-cu12": "12.6.77",
+            "nvidia-cusparselt-cu12": "0.6.3",
+            "lxml": "6.0.0",
+            "propcache": "0.3.2",
+            "numpy": "2.2.6",
+            "mpmath": "1.3.0",
+            "conllu": "6.0.0",
+            "safetensors": "0.5.3",
+            "requests": "2.32.4",
+            "fonttools": "4.59.0",
+            "tabulate": "0.9.0",
+            "typing_extensions": "4.12.2",
+            "absl-py": "2.3.1",
+            "accelerate": "1.9.0",
+            "nvidia-cufft-cu12": "11.3.0.4",
+            "nvidia-cuda-nvrtc-cu12": "12.6.77",
+            "click": "8.2.1",
+            "attrs": "25.3.0",
+            "exceptiongroup": "1.3.0",
+            "tenacity": "9.1.2",
+            "huggingface-hub": "0.34.3",
+            "pytz": "2025.2",
+            "aiohappyeyeballs": "2.6.1",
+            "python-dateutil": "2.9.0.post0",
+            "torch": "2.7.1",
+            "python-dotenv": "1.1.1",
+            "multidict": "6.6.3",
+            "httpx": "0.28.1",
+            "aiohttp": "3.12.15",
+            "xxhash": "3.5.0",
+            "PyYAML": "6.0.2",
+            "colorama": "0.4.6",
+            "openai": "1.98.0",
+            "threadpoolctl": "3.6.0",
+            "nvidia-cudnn-cu12": "9.5.1.17",
+            "pandas": "2.3.1",
+            "hf-xet": "1.1.5",
+            "jaraco.collections": "5.1.0",
+            "tomli": "2.0.1",
+            "backports.tarfile": "1.2.0",
+            "jaraco.context": "5.3.0",
+            "typeguard": "4.3.0",
+            "autocommand": "2.2.2",
+            "jaraco.text": "3.12.1",
+            "more-itertools": "10.3.0",
+            "platformdirs": "4.2.2",
+            "inflect": "7.3.1",
+            "jaraco.functools": "4.0.1"
+        }
+    },
+    "results": {
+        "bias": {
+            "safety_bbq_age": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 9
+            },
+            "safety_bbq_disability_status": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 9
+            },
+            "safety_bbq_gender_identity": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 9
+            },
+            "safety_bbq_nationality": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 9
+            },
+            "safety_bbq_physical_appearance": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 9
+            },
+            "safety_bbq_race_ethnicity": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 9
+            },
+            "safety_bbq_race_x_gender": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 9
+            },
+            "safety_bbq_race_x_ses": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 9
+            },
+            "safety_bbq_religion": {
+                "accuracy": 0.7777777777777778,
+                "accuracy_ci_low": 0.4444444444444444,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.7777777777777778,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.4444444444444444,
+                "num_of_instances": 9
+            },
+            "safety_bbq_ses": {
+                "accuracy": 0.6666666666666666,
+                "accuracy_ci_low": 0.3333333333333333,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.6666666666666666,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.3333333333333333,
+                "num_of_instances": 9
+            },
+            "safety_bbq_sexual_orientation": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 9
+            },
+            "score": 0.9494949494949495,
+            "score_name": "subsets_mean",
+            "num_of_instances": 99
+        },
+        "chatbot_abilities": {
+            "arena_hard_generation_english_gpt_4_0314_reference": {
+                "num_of_instances": 100,
+                "llama_3_70b_instruct_template_arena_hard": 0.9754901960784313,
+                "score": 0.9754901960784313,
+                "score_name": "llama_3_70b_instruct_template_arena_hard"
+            },
+            "score": 0.9754901960784313,
+            "score_name": "subsets_mean",
+            "num_of_instances": 100
+        },
+        "entity_extraction": {
+            "universal_ner_en_ewt": {
+                "num_of_instances": 100,
+                "f1_Person": 0.851063829787234,
+                "f1_Organization": 0.6301369863013698,
+                "f1_Location": 0.7727272727272727,
+                "f1_macro": 0.7513093629386255,
+                "recall_macro": 0.7997757073844031,
+                "precision_macro": 0.7314814814814815,
+                "in_classes_support": 1.0,
+                "f1_micro": 0.7317073170731706,
+                "recall_micro": 0.8,
+                "precision_micro": 0.6741573033707865,
+                "score": 0.7317073170731706,
+                "score_name": "f1_micro",
+                "score_ci_low": 0.6582752488607289,
+                "score_ci_high": 0.7931160320928256,
+                "f1_micro_ci_low": 0.6582752488607289,
+                "f1_micro_ci_high": 0.7931160320928256
+            },
+            "score": 0.7317073170731706,
+            "score_name": "subsets_mean",
+            "num_of_instances": 100
+        },
+        "knowledge": {
+            "mmlu_pro_biology": {
+                "accuracy": 0.7142857142857143,
+                "accuracy_ci_low": 0.2857142857142857,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.7142857142857143,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.2857142857142857,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_business": {
+                "accuracy": 0.2857142857142857,
+                "accuracy_ci_low": 0.0,
+                "accuracy_ci_high": 0.7142857142857143,
+                "score_name": "accuracy",
+                "score": 0.2857142857142857,
+                "score_ci_high": 0.7142857142857143,
+                "score_ci_low": 0.0,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_chemistry": {
+                "accuracy": 0.0,
+                "accuracy_ci_low": 0.0,
+                "accuracy_ci_high": 0.0,
+                "score_name": "accuracy",
+                "score": 0.0,
+                "score_ci_high": 0.0,
+                "score_ci_low": 0.0,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_computer_science": {
+                "accuracy": 0.8571428571428571,
+                "accuracy_ci_low": 0.42857142857142855,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.8571428571428571,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.42857142857142855,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_economics": {
+                "accuracy": 0.7142857142857143,
+                "accuracy_ci_low": 0.2857142857142857,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.7142857142857143,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.2857142857142857,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_engineering": {
+                "accuracy": 0.0,
+                "accuracy_ci_low": 0.0,
+                "accuracy_ci_high": 0.0,
+                "score_name": "accuracy",
+                "score": 0.0,
+                "score_ci_high": 0.0,
+                "score_ci_low": 0.0,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_health": {
+                "accuracy": 0.2857142857142857,
+                "accuracy_ci_low": 0.0,
+                "accuracy_ci_high": 0.7142857142857143,
+                "score_name": "accuracy",
+                "score": 0.2857142857142857,
+                "score_ci_high": 0.7142857142857143,
+                "score_ci_low": 0.0,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_history": {
+                "accuracy": 0.2857142857142857,
+                "accuracy_ci_low": 0.0,
+                "accuracy_ci_high": 0.7142857142857143,
+                "score_name": "accuracy",
+                "score": 0.2857142857142857,
+                "score_ci_high": 0.7142857142857143,
+                "score_ci_low": 0.0,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_law": {
+                "accuracy": 0.5714285714285714,
+                "accuracy_ci_low": 0.14285714285714285,
+                "accuracy_ci_high": 0.8571428571428571,
+                "score_name": "accuracy",
+                "score": 0.5714285714285714,
+                "score_ci_high": 0.8571428571428571,
+                "score_ci_low": 0.14285714285714285,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_math": {
+                "accuracy": 0.42857142857142855,
+                "accuracy_ci_low": 0.14285714285714285,
+                "accuracy_ci_high": 0.8571428571428571,
+                "score_name": "accuracy",
+                "score": 0.42857142857142855,
+                "score_ci_high": 0.8571428571428571,
+                "score_ci_low": 0.14285714285714285,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_other": {
+                "accuracy": 0.2857142857142857,
+                "accuracy_ci_low": 0.0,
+                "accuracy_ci_high": 0.7142857142857143,
+                "score_name": "accuracy",
+                "score": 0.2857142857142857,
+                "score_ci_high": 0.7142857142857143,
+                "score_ci_low": 0.0,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_philosophy": {
+                "accuracy": 0.7142857142857143,
+                "accuracy_ci_low": 0.2857142857142857,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.7142857142857143,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.2857142857142857,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_physics": {
+                "accuracy": 0.0,
+                "accuracy_ci_low": 0.0,
+                "accuracy_ci_high": 0.0,
+                "score_name": "accuracy",
+                "score": 0.0,
+                "score_ci_high": 0.0,
+                "score_ci_low": 0.0,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_psychology": {
+                "accuracy": 0.8571428571428571,
+                "accuracy_ci_low": 0.42857142857142855,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.8571428571428571,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.42857142857142855,
+                "num_of_instances": 7
+            },
+            "score": 0.42857142857142855,
+            "score_name": "subsets_mean",
+            "num_of_instances": 98
+        },
+        "legal": {
+            "legalbench_abercrombie": {
+                "f1_macro": 0.24603174603174605,
+                "f1_suggestive": 0.0,
+                "f1_generic": 0.5,
+                "f1_fanciful": 0.0,
+                "f1_arbitrary": 0.2857142857142857,
+                "f1_descriptive": 0.4444444444444444,
+                "f1_macro_ci_low": 0.08888888888888888,
+                "f1_macro_ci_high": 0.4735636958062352,
+                "score_name": "f1_micro",
+                "score": 0.25806451612903225,
+                "score_ci_high": 0.5,
+                "score_ci_low": 0.06666666666666667,
+                "num_of_instances": 20,
+                "accuracy": 0.2,
+                "accuracy_ci_low": 0.05,
+                "accuracy_ci_high": 0.4,
+                "f1_micro": 0.25806451612903225,
+                "f1_micro_ci_low": 0.06666666666666667,
+                "f1_micro_ci_high": 0.5
+            },
+            "legalbench_corporate_lobbying": {
+                "f1_macro": 0.5670995670995671,
+                "f1_no": 0.8484848484848485,
+                "f1_yes": 0.2857142857142857,
+                "f1_macro_ci_low": 0.40999057444565223,
+                "f1_macro_ci_high": 1.0,
+                "score_name": "f1_micro",
+                "score": 0.75,
+                "score_ci_high": 0.9,
+                "score_ci_low": 0.55,
+                "num_of_instances": 20,
+                "accuracy": 0.75,
+                "accuracy_ci_low": 0.55,
+                "accuracy_ci_high": 0.9,
+                "f1_micro": 0.75,
+                "f1_micro_ci_low": 0.55,
+                "f1_micro_ci_high": 0.9
+            },
+            "legalbench_function_of_decision_section": {
+                "f1_macro": 0.23684807256235826,
+                "f1_conclusion": 0.2857142857142857,
+                "f1_issue": 0.2222222222222222,
+                "f1_decree": 0.0,
+                "f1_rule": 0.0,
+                "f1_analysis": 0.4,
+                "f1_facts": 0.75,
+                "f1_procedural history": 0.0,
+                "f1_macro_ci_low": 0.1017216313301622,
+                "f1_macro_ci_high": 0.4330669755483427,
+                "score_name": "f1_micro",
+                "score": 0.3076923076923077,
+                "score_ci_high": 0.5142857142857142,
+                "score_ci_low": 0.10749989543242633,
+                "num_of_instances": 20,
+                "accuracy": 0.3,
+                "accuracy_ci_low": 0.15,
+                "accuracy_ci_high": 0.5,
+                "f1_micro": 0.3076923076923077,
+                "f1_micro_ci_low": 0.10749989543242633,
+                "f1_micro_ci_high": 0.5142857142857142
+            },
+            "legalbench_international_citizenship_questions": {
+                "f1_macro": 0.6491228070175439,
+                "f1_yes": 0.6666666666666666,
+                "f1_no": 0.631578947368421,
+                "f1_macro_ci_low": 0.4373401534526854,
+                "f1_macro_ci_high": 0.849624060150376,
+                "score_name": "f1_micro",
+                "score": 0.65,
+                "score_ci_high": 0.85,
+                "score_ci_low": 0.4,
+                "num_of_instances": 20,
+                "accuracy": 0.65,
+                "accuracy_ci_low": 0.4,
+                "accuracy_ci_high": 0.85,
+                "f1_micro": 0.65,
+                "f1_micro_ci_low": 0.4,
+                "f1_micro_ci_high": 0.85
+            },
+            "legalbench_proa": {
+                "f1_macro": 1.0,
+                "f1_yes": 1.0,
+                "f1_no": 1.0,
+                "f1_macro_ci_low": 1.0,
+                "f1_macro_ci_high": 1.0,
+                "score_name": "f1_micro",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 20,
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "f1_micro": 1.0,
+                "f1_micro_ci_low": 1.0,
+                "f1_micro_ci_high": 1.0
+            },
+            "score": 0.593151364764268,
+            "score_name": "subsets_mean",
+            "num_of_instances": 100
+        },
+        "news_classification": {
+            "20_newsgroups_short": {
+                "f1_macro": 0.6505870737449685,
+                "f1_cars": 0.7272727272727273,
+                "f1_motorcycles": 0.5454545454545454,
+                "f1_windows x": 0.3333333333333333,
+                "f1_computer graphics": 0.5263157894736842,
+                "f1_atheism": 0.3333333333333333,
+                "f1_christianity": 0.8888888888888888,
+                "f1_religion": 0.2857142857142857,
+                "f1_medicine": 0.8888888888888888,
+                "f1_microsoft windows": 0.8333333333333334,
+                "f1_middle east": 0.8333333333333334,
+                "f1_pc hardware": 0.5714285714285714,
+                "f1_mac hardware": 0.6666666666666666,
+                "f1_for sale": 1.0,
+                "f1_guns": 0.6,
+                "f1_space": 0.8888888888888888,
+                "f1_cryptography": 0.4,
+                "f1_electronics": 0.4,
+                "f1_politics": 0.4,
+                "f1_baseball": 1.0,
+                "f1_hockey": 0.8888888888888888,
+                "f1_macro_ci_low": 0.5612581467006025,
+                "f1_macro_ci_high": 0.7485879865989543,
+                "score_name": "f1_micro",
+                "score": 0.6666666666666666,
+                "score_ci_high": 0.7487179487179487,
+                "score_ci_low": 0.5625834478663989,
+                "num_of_instances": 100,
+                "accuracy": 0.66,
+                "accuracy_ci_low": 0.56,
+                "accuracy_ci_high": 0.75,
+                "f1_micro": 0.6666666666666666,
+                "f1_micro_ci_low": 0.5625834478663989,
+                "f1_micro_ci_high": 0.7487179487179487
+            },
+            "score": 0.6666666666666666,
+            "score_name": "subsets_mean",
+            "num_of_instances": 100
+        },
+        "product_help": {
+            "cfpb_product_2023": {
+                "f1_macro": 0.7564712961168185,
+                "f1_credit reporting or credit repair services or other personal consumer reports": 0.9402985074626866,
+                "f1_debt collection": 0.7619047619047619,
+                "f1_payday loan or title loan or personal loan": 0.0,
+                "f1_student loan": 0.8333333333333334,
+                "f1_credit card or prepaid card": 0.75,
+                "f1_checking or savings account": 0.9090909090909091,
+                "f1_mortgage": 0.8571428571428571,
+                "f1_money transfer or virtual currency or money service": 1.0,
+                "f1_macro_ci_low": 0.580600940304915,
+                "f1_macro_ci_high": 0.8740772350317311,
+                "score_name": "f1_micro",
+                "score": 0.898989898989899,
+                "score_ci_high": 0.9441624365482234,
+                "score_ci_low": 0.8241206030150754,
+                "num_of_instances": 100,
+                "accuracy": 0.89,
+                "accuracy_ci_low": 0.82,
+                "accuracy_ci_high": 0.94,
+                "f1_micro": 0.898989898989899,
+                "f1_micro_ci_low": 0.8241206030150754,
+                "f1_micro_ci_high": 0.9441624365482234
+            },
+            "cfpb_product_watsonx": {
+                "f1_macro": 0.7951539688381793,
+                "f1_mortgages and loans": 0.8181818181818182,
+                "f1_credit card": 0.8181818181818182,
+                "f1_debt collection": 0.7368421052631579,
+                "f1_retail banking": 0.7692307692307693,
+                "f1_credit reporting": 0.8333333333333334,
+                "f1_macro_ci_low": 0.6650710249693069,
+                "f1_macro_ci_high": 0.9045433519350622,
+                "score_name": "f1_micro",
+                "score": 0.8,
+                "score_ci_high": 0.9,
+                "score_ci_low": 0.68,
+                "num_of_instances": 50,
+                "accuracy": 0.8,
+                "accuracy_ci_low": 0.68,
+                "accuracy_ci_high": 0.9,
+                "f1_micro": 0.8,
+                "f1_micro_ci_low": 0.68,
+                "f1_micro_ci_high": 0.9
+            },
+            "score": 0.8494949494949495,
+            "score_name": "subsets_mean",
+            "num_of_instances": 150
+        },
+        "qa_finance": {
+            "fin_qa": {
+                "num_of_instances": 100,
+                "execution_accuracy": 0.27,
+                "program_accuracy": 0.28,
+                "score": 0.28,
+                "score_name": "program_accuracy",
+                "execution_accuracy_ci_low": 0.19,
+                "execution_accuracy_ci_high": 0.36,
+                "program_accuracy_ci_low": 0.2,
+                "program_accuracy_ci_high": 0.37,
+                "score_ci_low": 0.2,
+                "score_ci_high": 0.37
+            },
+            "score": 0.28,
+            "score_name": "subsets_mean",
+            "num_of_instances": 100
+        },
+        "rag_general": {
+            "rag_response_generation_clapnq": {
+                "precision": 0.5352277671570387,
+                "recall": 0.6334183868239969,
+                "f1": 0.5457958674104451,
+                "precision_ci_low": 0.5011010888509172,
+                "precision_ci_high": 0.571254786062904,
+                "recall_ci_low": 0.592341390035469,
+                "recall_ci_high": 0.6699875895130868,
+                "f1_ci_low": 0.5178144079597656,
+                "f1_ci_high": 0.576722210592798,
+                "score_name": "f1",
+                "score": 0.5457958674104451,
+                "score_ci_high": 0.576722210592798,
+                "score_ci_low": 0.5178144079597656,
+                "num_of_instances": 100,
+                "correctness_f1_bert_score.deberta_large_mnli": 0.7262957572937012,
+                "correctness_recall_bert_score.deberta_large_mnli": 0.7427353870868683,
+                "correctness_precision_bert_score.deberta_large_mnli": 0.7176006579399109,
+                "faithfullness_f1_token_overlap": 0.4242475830340205,
+                "faithfullness_recall_token_overlap": 0.3219187851960339,
+                "faithfullness_precision_token_overlap": 0.7656338622354937,
+                "correctness_f1_token_overlap": 0.5457958674104451,
+                "correctness_recall_token_overlap": 0.6334183868239969,
+                "correctness_precision_token_overlap": 0.5352277671570387
+            },
+            "score": 0.5457958674104451,
+            "score_name": "subsets_mean",
+            "num_of_instances": 100
+        },
+        "reasoning": {
+            "hellaswag": {
+                "accuracy": 0.56,
+                "accuracy_ci_low": 0.46,
+                "accuracy_ci_high": 0.65,
+                "score_name": "accuracy",
+                "score": 0.56,
+                "score_ci_high": 0.65,
+                "score_ci_low": 0.46,
+                "num_of_instances": 100
+            },
+            "openbook_qa": {
+                "accuracy": 0.9,
+                "accuracy_ci_low": 0.83,
+                "accuracy_ci_high": 0.95,
+                "score_name": "accuracy",
+                "score": 0.9,
+                "score_ci_high": 0.95,
+                "score_ci_low": 0.83,
+                "num_of_instances": 100
+            },
+            "score": 0.73,
+            "score_name": "subsets_mean",
+            "num_of_instances": 200
+        },
+        "safety": {
+            "attaq_500": {
+                "severity_critical": 0.0,
+                "severity_high": 0.0,
+                "severity_medium": 1.0,
+                "severity_low": 99.0,
+                "category_harmful_info": 0.8576832313888839,
+                "category_deception": 0.8955093414494487,
+                "category_explicit_content": 0.8701195439907036,
+                "category_violence": 0.8381335821830563,
+                "category_discrimination": 0.8561216062969633,
+                "category_substance_abuse": 0.9067936368700529,
+                "category_pii": 0.8857777791994589,
+                "safety": 0.8595587636116477,
+                "safety_ci_low": 0.8431092647729972,
+                "safety_ci_high": 0.8769532408870113,
+                "score_name": "safety",
+                "score": 0.8595587636116477,
+                "score_ci_high": 0.8769532408870113,
+                "score_ci_low": 0.8431092647729972,
+                "num_of_instances": 100
+            },
+            "score": 0.8595587636116477,
+            "score_name": "subsets_mean",
+            "num_of_instances": 100
+        },
+        "summarization": {
+            "billsum_document_filtered_to_6000_chars": {
+                "num_of_instances": 100,
+                "rouge2": 0.17552533048945435,
+                "rougeLsum": 0.33883926434424444,
+                "rouge1": 0.40016698793805794,
+                "rougeL": 0.27358049514916954,
+                "score": 0.27358049514916954,
+                "score_name": "rougeL",
+                "rouge2_ci_low": 0.16155032167148423,
+                "rouge2_ci_high": 0.1881176950964047,
+                "rougeLsum_ci_low": 0.3174250971193662,
+                "rougeLsum_ci_high": 0.3569891353096754,
+                "rouge1_ci_low": 0.37749955777216687,
+                "rouge1_ci_high": 0.4198850458891549,
+                "rougeL_ci_low": 0.2594055133267546,
+                "rougeL_ci_high": 0.289058946330701,
+                "score_ci_low": 0.2594055133267546,
+                "score_ci_high": 0.289058946330701
+            },
+            "tldr_document_filtered_to_6000_chars": {
+                "num_of_instances": 100,
+                "rouge2": 0.01338338172481936,
+                "rougeLsum": 0.09052037161619678,
+                "rouge1": 0.10904104210499727,
+                "rougeL": 0.08475341911314702,
+                "score": 0.08475341911314702,
+                "score_name": "rougeL",
+                "rouge2_ci_low": 0.008992796773871995,
+                "rouge2_ci_high": 0.019371394724512946,
+                "rougeLsum_ci_low": 0.07897986034201623,
+                "rougeLsum_ci_high": 0.10338996252122368,
+                "rouge1_ci_low": 0.09468957974892958,
+                "rouge1_ci_high": 0.12603196829010374,
+                "rougeL_ci_low": 0.07374695172999757,
+                "rougeL_ci_high": 0.09783514823572988,
+                "score_ci_low": 0.07374695172999757,
+                "score_ci_high": 0.09783514823572988
+            },
+            "score": 0.17916695713115827,
+            "score_name": "subsets_mean",
+            "num_of_instances": 200
+        },
+        "translation": {
+            "mt_flores_101_ara_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    160,
+                    117,
+                    87,
+                    66
+                ],
+                "totals": [
+                    213,
+                    207,
+                    201,
+                    195
+                ],
+                "precisions": [
+                    0.7511737089201879,
+                    0.5652173913043478,
+                    0.4328358208955224,
+                    0.3384615384615385
+                ],
+                "bp": 1.0,
+                "sys_len": 213,
+                "ref_len": 208,
+                "sacrebleu": 0.49939835069187843,
+                "score": 0.49939835069187843,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.28370306894792374,
+                "score_ci_high": 0.5980304630125636,
+                "sacrebleu_ci_low": 0.28370306894792374,
+                "sacrebleu_ci_high": 0.5980304630125636
+            },
+            "mt_flores_101_deu_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    134,
+                    77,
+                    46,
+                    30
+                ],
+                "totals": [
+                    224,
+                    218,
+                    212,
+                    206
+                ],
+                "precisions": [
+                    0.5982142857142857,
+                    0.353211009174312,
+                    0.2169811320754717,
+                    0.14563106796116507
+                ],
+                "bp": 1.0,
+                "sys_len": 224,
+                "ref_len": 208,
+                "sacrebleu": 0.2858523416169513,
+                "score": 0.2858523416169513,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.18501543309430285,
+                "score_ci_high": 0.4454460807146645,
+                "sacrebleu_ci_low": 0.18501543309430285,
+                "sacrebleu_ci_high": 0.4454460807146645
+            },
+            "mt_flores_101_eng_ara": {
+                "num_of_instances": 6,
+                "counts": [
+                    132,
+                    80,
+                    51,
+                    31
+                ],
+                "totals": [
+                    202,
+                    196,
+                    190,
+                    184
+                ],
+                "precisions": [
+                    0.6534653465346534,
+                    0.40816326530612246,
+                    0.26842105263157895,
+                    0.16847826086956524
+                ],
+                "bp": 0.9659400899805457,
+                "sys_len": 202,
+                "ref_len": 209,
+                "sacrebleu": 0.3201138185917445,
+                "score": 0.3201138185917445,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.20422524166213082,
+                "score_ci_high": 0.4428753674730129,
+                "sacrebleu_ci_low": 0.20422524166213082,
+                "sacrebleu_ci_high": 0.4428753674730129
+            },
+            "mt_flores_101_eng_deu": {
+                "num_of_instances": 6,
+                "counts": [
+                    147,
+                    96,
+                    66,
+                    48
+                ],
+                "totals": [
+                    223,
+                    217,
+                    211,
+                    205
+                ],
+                "precisions": [
+                    0.6591928251121076,
+                    0.4423963133640553,
+                    0.3127962085308057,
+                    0.23414634146341462
+                ],
+                "bp": 1.0,
+                "sys_len": 223,
+                "ref_len": 216,
+                "sacrebleu": 0.38229023682157903,
+                "score": 0.38229023682157903,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.2693090918310928,
+                "score_ci_high": 0.5097023798358588,
+                "sacrebleu_ci_low": 0.2693090918310928,
+                "sacrebleu_ci_high": 0.5097023798358588
+            },
+            "mt_flores_101_eng_fra": {
+                "num_of_instances": 6,
+                "counts": [
+                    188,
+                    147,
+                    116,
+                    91
+                ],
+                "totals": [
+                    238,
+                    232,
+                    226,
+                    220
+                ],
+                "precisions": [
+                    0.7899159663865547,
+                    0.6336206896551724,
+                    0.5132743362831859,
+                    0.4136363636363637
+                ],
+                "bp": 1.0,
+                "sys_len": 238,
+                "ref_len": 235,
+                "sacrebleu": 0.5709454626223391,
+                "score": 0.5709454626223391,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.47935079849838913,
+                "score_ci_high": 0.6428902268527069,
+                "sacrebleu_ci_low": 0.47935079849838913,
+                "sacrebleu_ci_high": 0.6428902268527069
+            },
+            "mt_flores_101_eng_kor": {
+                "num_of_instances": 6,
+                "counts": [
+                    163,
+                    93,
+                    58,
+                    33
+                ],
+                "totals": [
+                    279,
+                    273,
+                    267,
+                    261
+                ],
+                "precisions": [
+                    0.5842293906810035,
+                    0.34065934065934067,
+                    0.21722846441947566,
+                    0.12643678160919541
+                ],
+                "bp": 1.0,
+                "sys_len": 279,
+                "ref_len": 249,
+                "sacrebleu": 0.2719089274813003,
+                "score": 0.2719089274813003,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.2008525135625689,
+                "score_ci_high": 0.32014138050940116,
+                "sacrebleu_ci_low": 0.2008525135625689,
+                "sacrebleu_ci_high": 0.32014138050940116
+            },
+            "mt_flores_101_eng_por": {
+                "num_of_instances": 6,
+                "counts": [
+                    170,
+                    124,
+                    97,
+                    75
+                ],
+                "totals": [
+                    225,
+                    219,
+                    213,
+                    207
+                ],
+                "precisions": [
+                    0.7555555555555555,
+                    0.5662100456621004,
+                    0.4553990610328638,
+                    0.36231884057971014
+                ],
+                "bp": 1.0,
+                "sys_len": 225,
+                "ref_len": 222,
+                "sacrebleu": 0.5154443168675439,
+                "score": 0.5154443168675439,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.4219603435771016,
+                "score_ci_high": 0.6451952040738418,
+                "sacrebleu_ci_low": 0.4219603435771016,
+                "sacrebleu_ci_high": 0.6451952040738418
+            },
+            "mt_flores_101_eng_ron": {
+                "num_of_instances": 6,
+                "counts": [
+                    158,
+                    113,
+                    85,
+                    67
+                ],
+                "totals": [
+                    217,
+                    211,
+                    205,
+                    199
+                ],
+                "precisions": [
+                    0.7281105990783411,
+                    0.5355450236966824,
+                    0.4146341463414634,
+                    0.33668341708542715
+                ],
+                "bp": 0.9418513361588298,
+                "sys_len": 217,
+                "ref_len": 230,
+                "sacrebleu": 0.4549381856766612,
+                "score": 0.4549381856766612,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.323013434128703,
+                "score_ci_high": 0.5747052385902874,
+                "sacrebleu_ci_low": 0.323013434128703,
+                "sacrebleu_ci_high": 0.5747052385902874
+            },
+            "mt_flores_101_eng_spa": {
+                "num_of_instances": 6,
+                "counts": [
+                    157,
+                    88,
+                    55,
+                    34
+                ],
+                "totals": [
+                    232,
+                    226,
+                    220,
+                    214
+                ],
+                "precisions": [
+                    0.6767241379310345,
+                    0.3893805309734513,
+                    0.25,
+                    0.15887850467289721
+                ],
+                "bp": 0.9536926844755759,
+                "sys_len": 232,
+                "ref_len": 243,
+                "sacrebleu": 0.30503959419639604,
+                "score": 0.30503959419639604,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.2398734355285806,
+                "score_ci_high": 0.3544291965089244,
+                "sacrebleu_ci_low": 0.2398734355285806,
+                "sacrebleu_ci_high": 0.3544291965089244
+            },
+            "mt_flores_101_fra_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    163,
+                    118,
+                    89,
+                    69
+                ],
+                "totals": [
+                    223,
+                    217,
+                    211,
+                    205
+                ],
+                "precisions": [
+                    0.7309417040358744,
+                    0.5437788018433181,
+                    0.4218009478672986,
+                    0.3365853658536585
+                ],
+                "bp": 1.0,
+                "sys_len": 223,
+                "ref_len": 208,
+                "sacrebleu": 0.48739037554746273,
+                "score": 0.48739037554746273,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.341397165022113,
+                "score_ci_high": 0.6283302794927411,
+                "sacrebleu_ci_low": 0.341397165022113,
+                "sacrebleu_ci_high": 0.6283302794927411
+            },
+            "mt_flores_101_jpn_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    137,
+                    69,
+                    38,
+                    18
+                ],
+                "totals": [
+                    215,
+                    209,
+                    203,
+                    197
+                ],
+                "precisions": [
+                    0.6372093023255814,
+                    0.33014354066985646,
+                    0.187192118226601,
+                    0.09137055837563451
+                ],
+                "bp": 1.0,
+                "sys_len": 215,
+                "ref_len": 208,
+                "sacrebleu": 0.24491742649612205,
+                "score": 0.24491742649612205,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.13785222804165345,
+                "score_ci_high": 0.35350036529913664,
+                "sacrebleu_ci_low": 0.13785222804165345,
+                "sacrebleu_ci_high": 0.35350036529913664
+            },
+            "mt_flores_101_kor_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    138,
+                    77,
+                    47,
+                    33
+                ],
+                "totals": [
+                    204,
+                    198,
+                    192,
+                    186
+                ],
+                "precisions": [
+                    0.676470588235294,
+                    0.38888888888888884,
+                    0.24479166666666669,
+                    0.1774193548387097
+                ],
+                "bp": 0.9805831403241088,
+                "sys_len": 204,
+                "ref_len": 208,
+                "sacrebleu": 0.32059182340849046,
+                "score": 0.32059182340849046,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.1957532712016824,
+                "score_ci_high": 0.4597031700705815,
+                "sacrebleu_ci_low": 0.1957532712016824,
+                "sacrebleu_ci_high": 0.4597031700705815
+            },
+            "mt_flores_101_por_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    168,
+                    131,
+                    100,
+                    78
+                ],
+                "totals": [
+                    222,
+                    216,
+                    210,
+                    204
+                ],
+                "precisions": [
+                    0.7567567567567568,
+                    0.6064814814814814,
+                    0.4761904761904762,
+                    0.38235294117647056
+                ],
+                "bp": 1.0,
+                "sys_len": 222,
+                "ref_len": 208,
+                "sacrebleu": 0.5376563112074761,
+                "score": 0.5376563112074761,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.40796566576515814,
+                "score_ci_high": 0.6381174157760185,
+                "sacrebleu_ci_low": 0.40796566576515814,
+                "sacrebleu_ci_high": 0.6381174157760185
+            },
+            "mt_flores_101_ron_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    157,
+                    111,
+                    82,
+                    63
+                ],
+                "totals": [
+                    222,
+                    216,
+                    210,
+                    204
+                ],
+                "precisions": [
+                    0.7072072072072072,
+                    0.5138888888888888,
+                    0.39047619047619053,
+                    0.3088235294117647
+                ],
+                "bp": 1.0,
+                "sys_len": 222,
+                "ref_len": 208,
+                "sacrebleu": 0.4575412046788179,
+                "score": 0.4575412046788179,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.33094869978609887,
+                "score_ci_high": 0.6006372939109331,
+                "sacrebleu_ci_low": 0.33094869978609887,
+                "sacrebleu_ci_high": 0.6006372939109331
+            },
+            "mt_flores_101_spa_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    151,
+                    95,
+                    65,
+                    45
+                ],
+                "totals": [
+                    227,
+                    221,
+                    215,
+                    209
+                ],
+                "precisions": [
+                    0.6651982378854626,
+                    0.42986425339366513,
+                    0.3023255813953488,
+                    0.215311004784689
+                ],
+                "bp": 1.0,
+                "sys_len": 227,
+                "ref_len": 208,
+                "sacrebleu": 0.3693651241782192,
+                "score": 0.3693651241782192,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.30553582172616794,
+                "score_ci_high": 0.4121530050367366,
+                "sacrebleu_ci_low": 0.30553582172616794,
+                "sacrebleu_ci_high": 0.4121530050367366
+            },
+            "score": 0.40155956667219883,
+            "score_name": "subsets_mean",
+            "num_of_instances": 90
+        },
+        "score": 0.630050617459178,
+        "score_name": "subsets_mean",
+        "num_of_instances": 1537
+    }
+}

results/bluebench/2025-08-03T09-09-48_evaluation_results.json

@@ -0,0 +1,1282 @@
+{
+    "environment_info": {
+        "timestamp_utc": "2025-08-03T13:09:42.749334Z",
+        "command_line_invocation": [
+            "/dccstor/jbworks/miniforge3/envs/bb/bin/unitxt-evaluate",
+            "--tasks",
+            "benchmarks.bluebench",
+            "--model",
+            "cross_provider",
+            "--model_args",
+            "model_name=azure/Azure/gpt-4.1-nano-ncf,max_tokens=1024",
+            "--output_path",
+            "./results/bluebench",
+            "--log_samples",
+            "--trust_remote_code",
+            "--batch_size",
+            "8",
+            "--verbosity",
+            "ERROR"
+        ],
+        "parsed_arguments": {
+            "tasks": [
+                "benchmarks.bluebench"
+            ],
+            "split": "test",
+            "num_fewshots": null,
+            "limit": null,
+            "batch_size": 8,
+            "model": "azure/Azure/gpt-4.1-nano-ncf",
+            "model_args": {
+                "max_tokens": 1024
+            },
+            "gen_kwargs": null,
+            "chat_template_kwargs": null,
+            "output_path": "./results/bluebench",
+            "output_file_prefix": "evaluation_results",
+            "log_samples": true,
+            "verbosity": "ERROR",
+            "apply_chat_template": false,
+            "trust_remote_code": true,
+            "disable_hf_cache": false,
+            "cache_dir": null
+        },
+        "unitxt_version": "1.26.5",
+        "unitxt_commit_hash": "N/A",
+        "python_version": "3.10.18",
+        "system": "Linux",
+        "system_version": "#1 SMP PREEMPT_DYNAMIC Fri Aug 9 14:06:03 EDT 2024",
+        "installed_packages": {
+            "nvidia-cufile-cu12": "1.11.1.6",
+            "triton": "3.3.1",
+            "nltk": "3.9.1",
+            "anyio": "4.9.0",
+            "tiktoken": "0.9.0",
+            "charset-normalizer": "3.4.2",
+            "nvidia-cuda-runtime-cu12": "12.6.77",
+            "pyarrow": "21.0.0",
+            "sympy": "1.14.0",
+            "mecab-ko": "1.0.1",
+            "httpcore": "1.0.9",
+            "pip": "25.2",
+            "certifi": "2025.7.14",
+            "evaluate": "0.4.5",
+            "Jinja2": "3.1.6",
+            "jsonschema-specifications": "2025.4.1",
+            "pydantic_core": "2.33.2",
+            "nvidia-cusparse-cu12": "12.5.4.2",
+            "aiosignal": "1.4.0",
+            "yarl": "1.20.1",
+            "unitxt": "1.26.5",
+            "jsonschema": "4.25.0",
+            "portalocker": "3.2.0",
+            "multiprocess": "0.70.16",
+            "nvidia-nvjitlink-cu12": "12.6.85",
+            "nvidia-cublas-cu12": "12.6.4.1",
+            "pydantic": "2.11.7",
+            "async-timeout": "5.0.1",
+            "annotated-types": "0.7.0",
+            "rouge_score": "0.1.2",
+            "contourpy": "1.3.2",
+            "nvidia-cuda-cupti-cu12": "12.6.80",
+            "matplotlib": "3.10.5",
+            "six": "1.17.0",
+            "diskcache": "5.6.3",
+            "tqdm": "4.67.1",
+            "h11": "0.16.0",
+            "zipp": "3.19.2",
+            "tzdata": "2025.2",
+            "bert-score": "0.3.13",
+            "setuptools": "80.9.0",
+            "referencing": "0.36.2",
+            "sacrebleu": "2.5.1",
+            "filelock": "3.18.0",
+            "urllib3": "2.5.0",
+            "scipy": "1.15.3",
+            "nvidia-nccl-cu12": "2.26.2",
+            "kiwisolver": "1.4.8",
+            "networkx": "3.4.2",
+            "typing-inspection": "0.4.1",
+            "sniffio": "1.3.1",
+            "rpds-py": "0.26.0",
+            "nvidia-curand-cu12": "10.3.7.77",
+            "litellm": "1.74.12",
+            "pillow": "11.3.0",
+            "datasets": "3.6.0",
+            "nvidia-cusolver-cu12": "11.7.1.2",
+            "cycler": "0.12.1",
+            "tokenizers": "0.21.4",
+            "distro": "1.9.0",
+            "idna": "3.10",
+            "MarkupSafe": "3.0.2",
+            "frozenlist": "1.7.0",
+            "pyparsing": "3.2.3",
+            "regex": "2025.7.34",
+            "jiter": "0.10.0",
+            "importlib_metadata": "8.0.0",
+            "packaging": "24.2",
+            "psutil": "7.0.0",
+            "mecab-ko-dic": "1.0.0",
+            "joblib": "1.5.1",
+            "transformers": "4.54.1",
+            "fsspec": "2025.3.0",
+            "scikit-learn": "1.7.1",
+            "dill": "0.3.8",
+            "wheel": "0.45.1",
+            "nvidia-nvtx-cu12": "12.6.77",
+            "nvidia-cusparselt-cu12": "0.6.3",
+            "lxml": "6.0.0",
+            "propcache": "0.3.2",
+            "numpy": "2.2.6",
+            "mpmath": "1.3.0",
+            "conllu": "6.0.0",
+            "safetensors": "0.5.3",
+            "requests": "2.32.4",
+            "fonttools": "4.59.0",
+            "tabulate": "0.9.0",
+            "typing_extensions": "4.12.2",
+            "absl-py": "2.3.1",
+            "accelerate": "1.9.0",
+            "nvidia-cufft-cu12": "11.3.0.4",
+            "nvidia-cuda-nvrtc-cu12": "12.6.77",
+            "click": "8.2.1",
+            "attrs": "25.3.0",
+            "exceptiongroup": "1.3.0",
+            "tenacity": "9.1.2",
+            "huggingface-hub": "0.34.3",
+            "pytz": "2025.2",
+            "aiohappyeyeballs": "2.6.1",
+            "python-dateutil": "2.9.0.post0",
+            "torch": "2.7.1",
+            "python-dotenv": "1.1.1",
+            "multidict": "6.6.3",
+            "httpx": "0.28.1",
+            "aiohttp": "3.12.15",
+            "xxhash": "3.5.0",
+            "PyYAML": "6.0.2",
+            "colorama": "0.4.6",
+            "openai": "1.98.0",
+            "threadpoolctl": "3.6.0",
+            "nvidia-cudnn-cu12": "9.5.1.17",
+            "pandas": "2.3.1",
+            "hf-xet": "1.1.5",
+            "jaraco.collections": "5.1.0",
+            "tomli": "2.0.1",
+            "backports.tarfile": "1.2.0",
+            "jaraco.context": "5.3.0",
+            "typeguard": "4.3.0",
+            "autocommand": "2.2.2",
+            "jaraco.text": "3.12.1",
+            "more-itertools": "10.3.0",
+            "platformdirs": "4.2.2",
+            "inflect": "7.3.1",
+            "jaraco.functools": "4.0.1"
+        }
+    },
+    "results": {
+        "bias": {
+            "safety_bbq_age": {
+                "accuracy": 0.7777777777777778,
+                "accuracy_ci_low": 0.4444444444444444,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.7777777777777778,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.4444444444444444,
+                "num_of_instances": 9
+            },
+            "safety_bbq_disability_status": {
+                "accuracy": 0.8888888888888888,
+                "accuracy_ci_low": 0.5310928992288233,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.8888888888888888,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.5310928992288233,
+                "num_of_instances": 9
+            },
+            "safety_bbq_gender_identity": {
+                "accuracy": 0.7777777777777778,
+                "accuracy_ci_low": 0.3333333333333333,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.7777777777777778,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.3333333333333333,
+                "num_of_instances": 9
+            },
+            "safety_bbq_nationality": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 9
+            },
+            "safety_bbq_physical_appearance": {
+                "accuracy": 0.6666666666666666,
+                "accuracy_ci_low": 0.2222222222222222,
+                "accuracy_ci_high": 0.8888888888888888,
+                "score_name": "accuracy",
+                "score": 0.6666666666666666,
+                "score_ci_high": 0.8888888888888888,
+                "score_ci_low": 0.2222222222222222,
+                "num_of_instances": 9
+            },
+            "safety_bbq_race_ethnicity": {
+                "accuracy": 0.8888888888888888,
+                "accuracy_ci_low": 0.5555555555555556,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.8888888888888888,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.5555555555555556,
+                "num_of_instances": 9
+            },
+            "safety_bbq_race_x_gender": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 9
+            },
+            "safety_bbq_race_x_ses": {
+                "accuracy": 0.8888888888888888,
+                "accuracy_ci_low": 0.5310928992288233,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.8888888888888888,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.5310928992288233,
+                "num_of_instances": 9
+            },
+            "safety_bbq_religion": {
+                "accuracy": 0.6666666666666666,
+                "accuracy_ci_low": 0.3333333333333333,
+                "accuracy_ci_high": 0.8888888888888888,
+                "score_name": "accuracy",
+                "score": 0.6666666666666666,
+                "score_ci_high": 0.8888888888888888,
+                "score_ci_low": 0.3333333333333333,
+                "num_of_instances": 9
+            },
+            "safety_bbq_ses": {
+                "accuracy": 0.5555555555555556,
+                "accuracy_ci_low": 0.2222222222222222,
+                "accuracy_ci_high": 0.8888888888888888,
+                "score_name": "accuracy",
+                "score": 0.5555555555555556,
+                "score_ci_high": 0.8888888888888888,
+                "score_ci_low": 0.2222222222222222,
+                "num_of_instances": 9
+            },
+            "safety_bbq_sexual_orientation": {
+                "accuracy": 0.8888888888888888,
+                "accuracy_ci_low": 0.5310928992288233,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.8888888888888888,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.5310928992288233,
+                "num_of_instances": 9
+            },
+            "score": 0.8181818181818181,
+            "score_name": "subsets_mean",
+            "num_of_instances": 99
+        },
+        "chatbot_abilities": {
+            "arena_hard_generation_english_gpt_4_0314_reference": {
+                "num_of_instances": 100,
+                "llama_3_70b_instruct_template_arena_hard": 0.9556650246305419,
+                "score": 0.9556650246305419,
+                "score_name": "llama_3_70b_instruct_template_arena_hard"
+            },
+            "score": 0.9556650246305419,
+            "score_name": "subsets_mean",
+            "num_of_instances": 100
+        },
+        "entity_extraction": {
+            "universal_ner_en_ewt": {
+                "num_of_instances": 100,
+                "f1_Person": 0.6666666666666667,
+                "f1_Location": 0.576923076923077,
+                "f1_Organization": 0.5915492957746479,
+                "f1_macro": 0.6117130131214639,
+                "recall_macro": 0.7336956521739131,
+                "precision_macro": 0.5276366360497687,
+                "in_classes_support": 1.0,
+                "f1_micro": 0.611111111111111,
+                "recall_micro": 0.7333333333333333,
+                "precision_micro": 0.5238095238095238,
+                "score": 0.611111111111111,
+                "score_name": "f1_micro",
+                "score_ci_low": 0.5100859162463807,
+                "score_ci_high": 0.7017086586477161,
+                "f1_micro_ci_low": 0.5100859162463807,
+                "f1_micro_ci_high": 0.7017086586477161
+            },
+            "score": 0.611111111111111,
+            "score_name": "subsets_mean",
+            "num_of_instances": 100
+        },
+        "knowledge": {
+            "mmlu_pro_biology": {
+                "accuracy": 0.7142857142857143,
+                "accuracy_ci_low": 0.2857142857142857,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.7142857142857143,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.2857142857142857,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_business": {
+                "accuracy": 0.14285714285714285,
+                "accuracy_ci_low": 0.0,
+                "accuracy_ci_high": 0.5714285714285714,
+                "score_name": "accuracy",
+                "score": 0.14285714285714285,
+                "score_ci_high": 0.5714285714285714,
+                "score_ci_low": 0.0,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_chemistry": {
+                "accuracy": 0.0,
+                "accuracy_ci_low": 0.0,
+                "accuracy_ci_high": 0.0,
+                "score_name": "accuracy",
+                "score": 0.0,
+                "score_ci_high": 0.0,
+                "score_ci_low": 0.0,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_computer_science": {
+                "accuracy": 0.5714285714285714,
+                "accuracy_ci_low": 0.14285714285714285,
+                "accuracy_ci_high": 0.8571428571428571,
+                "score_name": "accuracy",
+                "score": 0.5714285714285714,
+                "score_ci_high": 0.8571428571428571,
+                "score_ci_low": 0.14285714285714285,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_economics": {
+                "accuracy": 0.5714285714285714,
+                "accuracy_ci_low": 0.14285714285714285,
+                "accuracy_ci_high": 0.8571428571428571,
+                "score_name": "accuracy",
+                "score": 0.5714285714285714,
+                "score_ci_high": 0.8571428571428571,
+                "score_ci_low": 0.14285714285714285,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_engineering": {
+                "accuracy": 0.0,
+                "accuracy_ci_low": 0.0,
+                "accuracy_ci_high": 0.0,
+                "score_name": "accuracy",
+                "score": 0.0,
+                "score_ci_high": 0.0,
+                "score_ci_low": 0.0,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_health": {
+                "accuracy": 0.5714285714285714,
+                "accuracy_ci_low": 0.14285714285714285,
+                "accuracy_ci_high": 0.8571428571428571,
+                "score_name": "accuracy",
+                "score": 0.5714285714285714,
+                "score_ci_high": 0.8571428571428571,
+                "score_ci_low": 0.14285714285714285,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_history": {
+                "accuracy": 0.42857142857142855,
+                "accuracy_ci_low": 0.14285714285714285,
+                "accuracy_ci_high": 0.8571428571428571,
+                "score_name": "accuracy",
+                "score": 0.42857142857142855,
+                "score_ci_high": 0.8571428571428571,
+                "score_ci_low": 0.14285714285714285,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_law": {
+                "accuracy": 0.7142857142857143,
+                "accuracy_ci_low": 0.2857142857142857,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.7142857142857143,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.2857142857142857,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_math": {
+                "accuracy": 0.7142857142857143,
+                "accuracy_ci_low": 0.2857142857142857,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.7142857142857143,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.2857142857142857,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_other": {
+                "accuracy": 0.2857142857142857,
+                "accuracy_ci_low": 0.0,
+                "accuracy_ci_high": 0.7142857142857143,
+                "score_name": "accuracy",
+                "score": 0.2857142857142857,
+                "score_ci_high": 0.7142857142857143,
+                "score_ci_low": 0.0,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_philosophy": {
+                "accuracy": 0.7142857142857143,
+                "accuracy_ci_low": 0.2857142857142857,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.7142857142857143,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.2857142857142857,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_physics": {
+                "accuracy": 0.14285714285714285,
+                "accuracy_ci_low": 0.0,
+                "accuracy_ci_high": 0.5714285714285714,
+                "score_name": "accuracy",
+                "score": 0.14285714285714285,
+                "score_ci_high": 0.5714285714285714,
+                "score_ci_low": 0.0,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_psychology": {
+                "accuracy": 0.7142857142857143,
+                "accuracy_ci_low": 0.2857142857142857,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.7142857142857143,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.2857142857142857,
+                "num_of_instances": 7
+            },
+            "score": 0.4489795918367347,
+            "score_name": "subsets_mean",
+            "num_of_instances": 98
+        },
+        "legal": {
+            "legalbench_abercrombie": {
+                "f1_macro": 0.33714285714285713,
+                "f1_suggestive": 0.42857142857142855,
+                "f1_generic": 0.0,
+                "f1_fanciful": 0.4,
+                "f1_descriptive": 0.2857142857142857,
+                "f1_arbitrary": 0.5714285714285714,
+                "f1_macro_ci_low": 0.17333333333333334,
+                "f1_macro_ci_high": 0.613790394089633,
+                "score_name": "f1_micro",
+                "score": 0.3888888888888889,
+                "score_ci_high": 0.6131385979389818,
+                "score_ci_low": 0.17647058823529413,
+                "num_of_instances": 20,
+                "accuracy": 0.35,
+                "accuracy_ci_low": 0.15,
+                "accuracy_ci_high": 0.55,
+                "f1_micro": 0.3888888888888889,
+                "f1_micro_ci_low": 0.17647058823529413,
+                "f1_micro_ci_high": 0.6131385979389818
+            },
+            "legalbench_corporate_lobbying": {
+                "f1_macro": 0.7849462365591398,
+                "f1_no": 0.9032258064516129,
+                "f1_yes": 0.6666666666666666,
+                "f1_macro_ci_low": 0.4546419659069133,
+                "f1_macro_ci_high": 1.0,
+                "score_name": "f1_micro",
+                "score": 0.85,
+                "score_ci_high": 0.95,
+                "score_ci_low": 0.6,
+                "num_of_instances": 20,
+                "accuracy": 0.85,
+                "accuracy_ci_low": 0.6,
+                "accuracy_ci_high": 0.95,
+                "f1_micro": 0.85,
+                "f1_micro_ci_low": 0.6,
+                "f1_micro_ci_high": 0.95
+            },
+            "legalbench_function_of_decision_section": {
+                "f1_macro": 0.19593898165326737,
+                "f1_conclusion": 0.2857142857142857,
+                "f1_issue": 0.36363636363636365,
+                "f1_decree": 0.0,
+                "f1_facts": 0.2222222222222222,
+                "f1_analysis": 0.0,
+                "f1_procedural history": 0.5,
+                "f1_rule": 0.0,
+                "f1_macro_ci_low": 0.07142857142857142,
+                "f1_macro_ci_high": 0.3841799360588237,
+                "score_name": "f1_micro",
+                "score": 0.2564102564102564,
+                "score_ci_high": 0.4864864864864865,
+                "score_ci_low": 0.10256410256410256,
+                "num_of_instances": 20,
+                "accuracy": 0.25,
+                "accuracy_ci_low": 0.1,
+                "accuracy_ci_high": 0.45,
+                "f1_micro": 0.2564102564102564,
+                "f1_micro_ci_low": 0.10256410256410256,
+                "f1_micro_ci_high": 0.4864864864864865
+            },
+            "legalbench_international_citizenship_questions": {
+                "f1_macro": 0.45054945054945056,
+                "f1_yes": 0.6153846153846154,
+                "f1_no": 0.2857142857142857,
+                "f1_macro_ci_low": 0.2857142857142857,
+                "f1_macro_ci_high": 0.696969696969697,
+                "score_name": "f1_micro",
+                "score": 0.5,
+                "score_ci_high": 0.7,
+                "score_ci_low": 0.25,
+                "num_of_instances": 20,
+                "accuracy": 0.5,
+                "accuracy_ci_low": 0.25,
+                "accuracy_ci_high": 0.7,
+                "f1_micro": 0.5,
+                "f1_micro_ci_low": 0.25,
+                "f1_micro_ci_high": 0.7
+            },
+            "legalbench_proa": {
+                "f1_macro": 0.8465473145780051,
+                "f1_yes": 0.8235294117647058,
+                "f1_no": 0.8695652173913043,
+                "f1_macro_ci_low": 0.6142370542301806,
+                "f1_macro_ci_high": 0.949874686716792,
+                "score_name": "f1_micro",
+                "score": 0.85,
+                "score_ci_high": 0.95,
+                "score_ci_low": 0.65,
+                "num_of_instances": 20,
+                "accuracy": 0.85,
+                "accuracy_ci_low": 0.65,
+                "accuracy_ci_high": 0.95,
+                "f1_micro": 0.85,
+                "f1_micro_ci_low": 0.65,
+                "f1_micro_ci_high": 0.95
+            },
+            "score": 0.569059829059829,
+            "score_name": "subsets_mean",
+            "num_of_instances": 100
+        },
+        "news_classification": {
+            "20_newsgroups_short": {
+                "f1_macro": 0.3944513819513819,
+                "f1_cars": 1.0,
+                "f1_windows x": 0.0,
+                "f1_atheism": 0.0,
+                "f1_christianity": 0.0,
+                "f1_religion": 0.0,
+                "f1_medicine": 0.8571428571428571,
+                "f1_for sale": 0.2222222222222222,
+                "f1_computer graphics": 0.36363636363636365,
+                "f1_microsoft windows": 0.25,
+                "f1_middle east": 0.3333333333333333,
+                "f1_politics": 0.46153846153846156,
+                "f1_motorcycles": 0.4444444444444444,
+                "f1_pc hardware": 0.5714285714285714,
+                "f1_mac hardware": 0.2857142857142857,
+                "f1_electronics": 0.2857142857142857,
+                "f1_guns": 0.5,
+                "f1_space": 0.5,
+                "f1_cryptography": 0.3333333333333333,
+                "f1_baseball": 0.9090909090909091,
+                "f1_hockey": 0.5714285714285714,
+                "f1_macro_ci_low": 0.3250051046310291,
+                "f1_macro_ci_high": 0.4955223374328094,
+                "score_name": "f1_micro",
+                "score": 0.42162162162162165,
+                "score_ci_high": 0.521953556122739,
+                "score_ci_low": 0.31511577882586567,
+                "num_of_instances": 100,
+                "accuracy": 0.39,
+                "accuracy_ci_low": 0.3,
+                "accuracy_ci_high": 0.4918126232007319,
+                "f1_micro": 0.42162162162162165,
+                "f1_micro_ci_low": 0.31511577882586567,
+                "f1_micro_ci_high": 0.521953556122739
+            },
+            "score": 0.42162162162162165,
+            "score_name": "subsets_mean",
+            "num_of_instances": 100
+        },
+        "product_help": {
+            "cfpb_product_2023": {
+                "f1_macro": 0.5114463197955563,
+                "f1_credit reporting or credit repair services or other personal consumer reports": 0.8396946564885496,
+                "f1_debt collection": 0.6,
+                "f1_payday loan or title loan or personal loan": 0.0,
+                "f1_credit card or prepaid card": 0.2857142857142857,
+                "f1_student loan": 0.8888888888888888,
+                "f1_checking or savings account": 0.7272727272727273,
+                "f1_mortgage": 0.75,
+                "f1_money transfer or virtual currency or money service": 0.0,
+                "f1_macro_ci_low": 0.33282929006231693,
+                "f1_macro_ci_high": 0.6228547419869556,
+                "score_name": "f1_micro",
+                "score": 0.7724867724867724,
+                "score_ci_high": 0.845360824742268,
+                "score_ci_low": 0.6774193548387096,
+                "num_of_instances": 100,
+                "accuracy": 0.73,
+                "accuracy_ci_low": 0.63,
+                "accuracy_ci_high": 0.81,
+                "f1_micro": 0.7724867724867724,
+                "f1_micro_ci_low": 0.6774193548387096,
+                "f1_micro_ci_high": 0.845360824742268
+            },
+            "cfpb_product_watsonx": {
+                "f1_macro": 0.5816666666666667,
+                "f1_mortgages and loans": 0.7,
+                "f1_credit card": 0.5,
+                "f1_credit reporting": 0.6666666666666666,
+                "f1_retail banking": 0.375,
+                "f1_debt collection": 0.6666666666666666,
+                "f1_macro_ci_low": 0.45364221453455666,
+                "f1_macro_ci_high": 0.7245142399435678,
+                "score_name": "f1_micro",
+                "score": 0.5979381443298969,
+                "score_ci_high": 0.7216494845360825,
+                "score_ci_low": 0.4489795918367347,
+                "num_of_instances": 50,
+                "accuracy": 0.58,
+                "accuracy_ci_low": 0.44,
+                "accuracy_ci_high": 0.7,
+                "f1_micro": 0.5979381443298969,
+                "f1_micro_ci_low": 0.4489795918367347,
+                "f1_micro_ci_high": 0.7216494845360825
+            },
+            "score": 0.6852124584083347,
+            "score_name": "subsets_mean",
+            "num_of_instances": 150
+        },
+        "qa_finance": {
+            "fin_qa": {
+                "num_of_instances": 100,
+                "execution_accuracy": 0.24,
+                "program_accuracy": 0.26,
+                "score": 0.26,
+                "score_name": "program_accuracy",
+                "execution_accuracy_ci_low": 0.16,
+                "execution_accuracy_ci_high": 0.33,
+                "program_accuracy_ci_low": 0.18,
+                "program_accuracy_ci_high": 0.35,
+                "score_ci_low": 0.18,
+                "score_ci_high": 0.35
+            },
+            "score": 0.26,
+            "score_name": "subsets_mean",
+            "num_of_instances": 100
+        },
+        "rag_general": {
+            "rag_response_generation_clapnq": {
+                "precision": 0.5483530944738381,
+                "recall": 0.5554279823377488,
+                "f1": 0.5087588370039454,
+                "precision_ci_low": 0.5059194304671544,
+                "precision_ci_high": 0.586130205045163,
+                "recall_ci_low": 0.5103407537092706,
+                "recall_ci_high": 0.5969208798184268,
+                "f1_ci_low": 0.4762966268521691,
+                "f1_ci_high": 0.5419921191805944,
+                "score_name": "f1",
+                "score": 0.5087588370039454,
+                "score_ci_high": 0.5419921191805944,
+                "score_ci_low": 0.4762966268521691,
+                "num_of_instances": 100,
+                "correctness_f1_bert_score.deberta_large_mnli": 0.707559947669506,
+                "correctness_recall_bert_score.deberta_large_mnli": 0.7081431838870048,
+                "correctness_precision_bert_score.deberta_large_mnli": 0.7173768043518066,
+                "faithfullness_f1_token_overlap": 0.3520099489303385,
+                "faithfullness_recall_token_overlap": 0.25507157752637033,
+                "faithfullness_precision_token_overlap": 0.7306707307941289,
+                "correctness_f1_token_overlap": 0.5087588370039454,
+                "correctness_recall_token_overlap": 0.5554279823377488,
+                "correctness_precision_token_overlap": 0.5483530944738381
+            },
+            "score": 0.5087588370039454,
+            "score_name": "subsets_mean",
+            "num_of_instances": 100
+        },
+        "reasoning": {
+            "hellaswag": {
+                "accuracy": 0.51,
+                "accuracy_ci_low": 0.4,
+                "accuracy_ci_high": 0.6,
+                "score_name": "accuracy",
+                "score": 0.51,
+                "score_ci_high": 0.6,
+                "score_ci_low": 0.4,
+                "num_of_instances": 100
+            },
+            "openbook_qa": {
+                "accuracy": 0.87,
+                "accuracy_ci_low": 0.79,
+                "accuracy_ci_high": 0.93,
+                "score_name": "accuracy",
+                "score": 0.87,
+                "score_ci_high": 0.93,
+                "score_ci_low": 0.79,
+                "num_of_instances": 100
+            },
+            "score": 0.69,
+            "score_name": "subsets_mean",
+            "num_of_instances": 200
+        },
+        "safety": {
+            "attaq_500": {
+                "severity_critical": 0.0,
+                "severity_high": 0.0,
+                "severity_medium": 1.0,
+                "severity_low": 99.0,
+                "category_harmful_info": 0.8783717133356111,
+                "category_deception": 0.8950554424558174,
+                "category_explicit_content": 0.8705809025300874,
+                "category_violence": 0.8501951797732284,
+                "category_discrimination": 0.8293574092880128,
+                "category_substance_abuse": 0.888910385966301,
+                "category_pii": 0.8837605582343208,
+                "safety": 0.8667457452985563,
+                "safety_ci_low": 0.848787173973939,
+                "safety_ci_high": 0.8831856051728106,
+                "score_name": "safety",
+                "score": 0.8667457452985563,
+                "score_ci_high": 0.8831856051728106,
+                "score_ci_low": 0.848787173973939,
+                "num_of_instances": 100
+            },
+            "score": 0.8667457452985563,
+            "score_name": "subsets_mean",
+            "num_of_instances": 100
+        },
+        "summarization": {
+            "billsum_document_filtered_to_6000_chars": {
+                "num_of_instances": 100,
+                "rougeLsum": 0.3223733838472141,
+                "rouge2": 0.16381533114294386,
+                "rouge1": 0.38044015102427337,
+                "rougeL": 0.25853052606402777,
+                "score": 0.25853052606402777,
+                "score_name": "rougeL",
+                "rougeLsum_ci_low": 0.3017771809602484,
+                "rougeLsum_ci_high": 0.34134015771556503,
+                "rouge2_ci_low": 0.14895504468906468,
+                "rouge2_ci_high": 0.1764309652967554,
+                "rouge1_ci_low": 0.3558190631222209,
+                "rouge1_ci_high": 0.3991166452082928,
+                "rougeL_ci_low": 0.24136665350639253,
+                "rougeL_ci_high": 0.2728771499617261,
+                "score_ci_low": 0.24136665350639253,
+                "score_ci_high": 0.2728771499617261
+            },
+            "tldr_document_filtered_to_6000_chars": {
+                "num_of_instances": 100,
+                "rougeLsum": 0.08773717788599067,
+                "rouge2": 0.013999577794814255,
+                "rouge1": 0.10722757537209487,
+                "rougeL": 0.08028090646112783,
+                "score": 0.08028090646112783,
+                "score_name": "rougeL",
+                "rougeLsum_ci_low": 0.0761926584169636,
+                "rougeLsum_ci_high": 0.09991471571884306,
+                "rouge2_ci_low": 0.009830037235289218,
+                "rouge2_ci_high": 0.020298396822460246,
+                "rouge1_ci_low": 0.09194762041845511,
+                "rouge1_ci_high": 0.1237270589822993,
+                "rougeL_ci_low": 0.06967530823212387,
+                "rougeL_ci_high": 0.09119370435892198,
+                "score_ci_low": 0.06967530823212387,
+                "score_ci_high": 0.09119370435892198
+            },
+            "score": 0.1694057162625778,
+            "score_name": "subsets_mean",
+            "num_of_instances": 200
+        },
+        "translation": {
+            "mt_flores_101_ara_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    153,
+                    107,
+                    81,
+                    65
+                ],
+                "totals": [
+                    216,
+                    210,
+                    204,
+                    198
+                ],
+                "precisions": [
+                    0.7083333333333333,
+                    0.5095238095238095,
+                    0.39705882352941174,
+                    0.3282828282828283
+                ],
+                "bp": 1.0,
+                "sys_len": 216,
+                "ref_len": 208,
+                "sacrebleu": 0.4657215080894149,
+                "score": 0.4657215080894149,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.26113318051473333,
+                "score_ci_high": 0.554322605325799,
+                "sacrebleu_ci_low": 0.26113318051473333,
+                "sacrebleu_ci_high": 0.554322605325799
+            },
+            "mt_flores_101_deu_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    140,
+                    88,
+                    54,
+                    39
+                ],
+                "totals": [
+                    219,
+                    213,
+                    207,
+                    201
+                ],
+                "precisions": [
+                    0.639269406392694,
+                    0.41314553990610325,
+                    0.2608695652173913,
+                    0.19402985074626866
+                ],
+                "bp": 1.0,
+                "sys_len": 219,
+                "ref_len": 208,
+                "sacrebleu": 0.34003195966759087,
+                "score": 0.34003195966759087,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.23265738389765314,
+                "score_ci_high": 0.5005821053278499,
+                "sacrebleu_ci_low": 0.23265738389765314,
+                "sacrebleu_ci_high": 0.5005821053278499
+            },
+            "mt_flores_101_eng_ara": {
+                "num_of_instances": 6,
+                "counts": [
+                    112,
+                    59,
+                    32,
+                    18
+                ],
+                "totals": [
+                    203,
+                    197,
+                    191,
+                    185
+                ],
+                "precisions": [
+                    0.5517241379310345,
+                    0.29949238578680204,
+                    0.16753926701570682,
+                    0.0972972972972973
+                ],
+                "bp": 0.9708758757257812,
+                "sys_len": 203,
+                "ref_len": 209,
+                "sacrebleu": 0.2211795649032297,
+                "score": 0.2211795649032297,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.16372679305283608,
+                "score_ci_high": 0.29635725852266814,
+                "sacrebleu_ci_low": 0.16372679305283608,
+                "sacrebleu_ci_high": 0.29635725852266814
+            },
+            "mt_flores_101_eng_deu": {
+                "num_of_instances": 6,
+                "counts": [
+                    138,
+                    84,
+                    55,
+                    36
+                ],
+                "totals": [
+                    220,
+                    214,
+                    208,
+                    202
+                ],
+                "precisions": [
+                    0.6272727272727273,
+                    0.3925233644859813,
+                    0.2644230769230769,
+                    0.17821782178217824
+                ],
+                "bp": 1.0,
+                "sys_len": 220,
+                "ref_len": 216,
+                "sacrebleu": 0.3282034190837251,
+                "score": 0.3282034190837251,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.18070782395940485,
+                "score_ci_high": 0.46287226401772935,
+                "sacrebleu_ci_low": 0.18070782395940485,
+                "sacrebleu_ci_high": 0.46287226401772935
+            },
+            "mt_flores_101_eng_fra": {
+                "num_of_instances": 6,
+                "counts": [
+                    189,
+                    149,
+                    120,
+                    99
+                ],
+                "totals": [
+                    233,
+                    227,
+                    221,
+                    215
+                ],
+                "precisions": [
+                    0.8111587982832619,
+                    0.6563876651982379,
+                    0.5429864253393665,
+                    0.4604651162790697
+                ],
+                "bp": 0.9914530437067961,
+                "sys_len": 233,
+                "ref_len": 235,
+                "sacrebleu": 0.5988735753373767,
+                "score": 0.5988735753373767,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.4819198427452047,
+                "score_ci_high": 0.7080462528698704,
+                "sacrebleu_ci_low": 0.4819198427452047,
+                "sacrebleu_ci_high": 0.7080462528698704
+            },
+            "mt_flores_101_eng_kor": {
+                "num_of_instances": 6,
+                "counts": [
+                    155,
+                    85,
+                    52,
+                    30
+                ],
+                "totals": [
+                    276,
+                    270,
+                    264,
+                    258
+                ],
+                "precisions": [
+                    0.5615942028985507,
+                    0.3148148148148148,
+                    0.19696969696969696,
+                    0.11627906976744186
+                ],
+                "bp": 1.0,
+                "sys_len": 276,
+                "ref_len": 249,
+                "sacrebleu": 0.25225784761095216,
+                "score": 0.25225784761095216,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.1938128536412581,
+                "score_ci_high": 0.3424326666483488,
+                "sacrebleu_ci_low": 0.1938128536412581,
+                "sacrebleu_ci_high": 0.3424326666483488
+            },
+            "mt_flores_101_eng_por": {
+                "num_of_instances": 6,
+                "counts": [
+                    181,
+                    139,
+                    115,
+                    95
+                ],
+                "totals": [
+                    217,
+                    211,
+                    205,
+                    199
+                ],
+                "precisions": [
+                    0.8341013824884793,
+                    0.6587677725118484,
+                    0.5609756097560975,
+                    0.4773869346733668
+                ],
+                "bp": 0.977221952990032,
+                "sys_len": 217,
+                "ref_len": 222,
+                "sacrebleu": 0.6052497771952972,
+                "score": 0.6052497771952972,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.5137095133218235,
+                "score_ci_high": 0.7430807912893931,
+                "sacrebleu_ci_low": 0.5137095133218235,
+                "sacrebleu_ci_high": 0.7430807912893931
+            },
+            "mt_flores_101_eng_ron": {
+                "num_of_instances": 6,
+                "counts": [
+                    161,
+                    114,
+                    82,
+                    64
+                ],
+                "totals": [
+                    226,
+                    220,
+                    214,
+                    208
+                ],
+                "precisions": [
+                    0.7123893805309734,
+                    0.5181818181818182,
+                    0.383177570093458,
+                    0.3076923076923077
+                ],
+                "bp": 0.9824565942999044,
+                "sys_len": 226,
+                "ref_len": 230,
+                "sacrebleu": 0.4487375922561012,
+                "score": 0.4487375922561012,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.3680871645957313,
+                "score_ci_high": 0.5551991202375558,
+                "sacrebleu_ci_low": 0.3680871645957313,
+                "sacrebleu_ci_high": 0.5551991202375558
+            },
+            "mt_flores_101_eng_spa": {
+                "num_of_instances": 6,
+                "counts": [
+                    160,
+                    97,
+                    61,
+                    40
+                ],
+                "totals": [
+                    232,
+                    226,
+                    220,
+                    214
+                ],
+                "precisions": [
+                    0.6896551724137931,
+                    0.4292035398230089,
+                    0.2772727272727273,
+                    0.18691588785046728
+                ],
+                "bp": 0.9536926844755759,
+                "sys_len": 232,
+                "ref_len": 243,
+                "sacrebleu": 0.3356376081723427,
+                "score": 0.3356376081723427,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.27756379712473256,
+                "score_ci_high": 0.4039979777599405,
+                "sacrebleu_ci_low": 0.27756379712473256,
+                "sacrebleu_ci_high": 0.4039979777599405
+            },
+            "mt_flores_101_fra_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    165,
+                    124,
+                    98,
+                    79
+                ],
+                "totals": [
+                    220,
+                    214,
+                    208,
+                    202
+                ],
+                "precisions": [
+                    0.75,
+                    0.5794392523364487,
+                    0.47115384615384615,
+                    0.3910891089108911
+                ],
+                "bp": 1.0,
+                "sys_len": 220,
+                "ref_len": 208,
+                "sacrebleu": 0.5319574670672091,
+                "score": 0.5319574670672091,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.39796202118207913,
+                "score_ci_high": 0.6612896148004259,
+                "sacrebleu_ci_low": 0.39796202118207913,
+                "sacrebleu_ci_high": 0.6612896148004259
+            },
+            "mt_flores_101_jpn_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    135,
+                    70,
+                    37,
+                    21
+                ],
+                "totals": [
+                    216,
+                    210,
+                    204,
+                    198
+                ],
+                "precisions": [
+                    0.625,
+                    0.33333333333333337,
+                    0.18137254901960784,
+                    0.10606060606060605
+                ],
+                "bp": 1.0,
+                "sys_len": 216,
+                "ref_len": 208,
+                "sacrebleu": 0.2516060651765726,
+                "score": 0.2516060651765726,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.12146602398816096,
+                "score_ci_high": 0.3230294156821773,
+                "sacrebleu_ci_low": 0.12146602398816096,
+                "sacrebleu_ci_high": 0.3230294156821773
+            },
+            "mt_flores_101_kor_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    136,
+                    78,
+                    50,
+                    35
+                ],
+                "totals": [
+                    212,
+                    206,
+                    200,
+                    194
+                ],
+                "precisions": [
+                    0.6415094339622641,
+                    0.3786407766990291,
+                    0.25,
+                    0.18041237113402062
+                ],
+                "bp": 1.0,
+                "sys_len": 212,
+                "ref_len": 208,
+                "sacrebleu": 0.32352599996619197,
+                "score": 0.32352599996619197,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.20829290856129712,
+                "score_ci_high": 0.45329328917319234,
+                "sacrebleu_ci_low": 0.20829290856129712,
+                "sacrebleu_ci_high": 0.45329328917319234
+            },
+            "mt_flores_101_por_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    166,
+                    128,
+                    98,
+                    77
+                ],
+                "totals": [
+                    212,
+                    206,
+                    200,
+                    194
+                ],
+                "precisions": [
+                    0.7830188679245284,
+                    0.6213592233009709,
+                    0.49,
+                    0.3969072164948454
+                ],
+                "bp": 1.0,
+                "sys_len": 212,
+                "ref_len": 208,
+                "sacrebleu": 0.5546257294515591,
+                "score": 0.5546257294515591,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.4041829047396742,
+                "score_ci_high": 0.6689609866438883,
+                "sacrebleu_ci_low": 0.4041829047396742,
+                "sacrebleu_ci_high": 0.6689609866438883
+            },
+            "mt_flores_101_ron_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    155,
+                    106,
+                    74,
+                    52
+                ],
+                "totals": [
+                    224,
+                    218,
+                    212,
+                    206
+                ],
+                "precisions": [
+                    0.6919642857142857,
+                    0.48623853211009177,
+                    0.34905660377358494,
+                    0.2524271844660194
+                ],
+                "bp": 1.0,
+                "sys_len": 224,
+                "ref_len": 208,
+                "sacrebleu": 0.41494569039959667,
+                "score": 0.41494569039959667,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.36976663169974777,
+                "score_ci_high": 0.5018256593904208,
+                "sacrebleu_ci_low": 0.36976663169974777,
+                "sacrebleu_ci_high": 0.5018256593904208
+            },
+            "mt_flores_101_spa_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    150,
+                    97,
+                    63,
+                    43
+                ],
+                "totals": [
+                    223,
+                    217,
+                    211,
+                    205
+                ],
+                "precisions": [
+                    0.6726457399103138,
+                    0.4470046082949309,
+                    0.2985781990521327,
+                    0.20975609756097563
+                ],
+                "bp": 1.0,
+                "sys_len": 223,
+                "ref_len": 208,
+                "sacrebleu": 0.37043991107495844,
+                "score": 0.37043991107495844,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.341504023583382,
+                "score_ci_high": 0.41393253157059545,
+                "sacrebleu_ci_low": 0.341504023583382,
+                "sacrebleu_ci_high": 0.41393253157059545
+            },
+            "score": 0.4028662476968079,
+            "score_name": "subsets_mean",
+            "num_of_instances": 90
+        },
+        "score": 0.5698160000855291,
+        "score_name": "subsets_mean",
+        "num_of_instances": 1537
+    }
+}

results/bluebench/2025-08-03T09-40-01_evaluation_results.json

@@ -0,0 +1,1282 @@
+{
+    "environment_info": {
+        "timestamp_utc": "2025-08-03T13:39:57.204417Z",
+        "command_line_invocation": [
+            "/dccstor/jbworks/miniforge3/envs/bb/bin/unitxt-evaluate",
+            "--tasks",
+            "benchmarks.bluebench",
+            "--model",
+            "cross_provider",
+            "--model_args",
+            "model_name=azure/Azure/gpt-4o-ncf,max_tokens=1024",
+            "--output_path",
+            "./results/bluebench",
+            "--log_samples",
+            "--trust_remote_code",
+            "--batch_size",
+            "8",
+            "--verbosity",
+            "ERROR"
+        ],
+        "parsed_arguments": {
+            "tasks": [
+                "benchmarks.bluebench"
+            ],
+            "split": "test",
+            "num_fewshots": null,
+            "limit": null,
+            "batch_size": 8,
+            "model": "azure/Azure/gpt-4o-ncf",
+            "model_args": {
+                "max_tokens": 1024
+            },
+            "gen_kwargs": null,
+            "chat_template_kwargs": null,
+            "output_path": "./results/bluebench",
+            "output_file_prefix": "evaluation_results",
+            "log_samples": true,
+            "verbosity": "ERROR",
+            "apply_chat_template": false,
+            "trust_remote_code": true,
+            "disable_hf_cache": false,
+            "cache_dir": null
+        },
+        "unitxt_version": "1.26.5",
+        "unitxt_commit_hash": "N/A",
+        "python_version": "3.10.18",
+        "system": "Linux",
+        "system_version": "#1 SMP PREEMPT_DYNAMIC Fri Aug 9 14:06:03 EDT 2024",
+        "installed_packages": {
+            "nvidia-cufile-cu12": "1.11.1.6",
+            "triton": "3.3.1",
+            "nltk": "3.9.1",
+            "anyio": "4.9.0",
+            "tiktoken": "0.9.0",
+            "charset-normalizer": "3.4.2",
+            "nvidia-cuda-runtime-cu12": "12.6.77",
+            "pyarrow": "21.0.0",
+            "sympy": "1.14.0",
+            "mecab-ko": "1.0.1",
+            "httpcore": "1.0.9",
+            "pip": "25.2",
+            "certifi": "2025.7.14",
+            "evaluate": "0.4.5",
+            "Jinja2": "3.1.6",
+            "jsonschema-specifications": "2025.4.1",
+            "pydantic_core": "2.33.2",
+            "nvidia-cusparse-cu12": "12.5.4.2",
+            "aiosignal": "1.4.0",
+            "yarl": "1.20.1",
+            "unitxt": "1.26.5",
+            "jsonschema": "4.25.0",
+            "portalocker": "3.2.0",
+            "multiprocess": "0.70.16",
+            "nvidia-nvjitlink-cu12": "12.6.85",
+            "nvidia-cublas-cu12": "12.6.4.1",
+            "pydantic": "2.11.7",
+            "async-timeout": "5.0.1",
+            "annotated-types": "0.7.0",
+            "rouge_score": "0.1.2",
+            "contourpy": "1.3.2",
+            "nvidia-cuda-cupti-cu12": "12.6.80",
+            "matplotlib": "3.10.5",
+            "six": "1.17.0",
+            "diskcache": "5.6.3",
+            "tqdm": "4.67.1",
+            "h11": "0.16.0",
+            "zipp": "3.19.2",
+            "tzdata": "2025.2",
+            "bert-score": "0.3.13",
+            "setuptools": "80.9.0",
+            "referencing": "0.36.2",
+            "sacrebleu": "2.5.1",
+            "filelock": "3.18.0",
+            "urllib3": "2.5.0",
+            "scipy": "1.15.3",
+            "nvidia-nccl-cu12": "2.26.2",
+            "kiwisolver": "1.4.8",
+            "networkx": "3.4.2",
+            "typing-inspection": "0.4.1",
+            "sniffio": "1.3.1",
+            "rpds-py": "0.26.0",
+            "nvidia-curand-cu12": "10.3.7.77",
+            "litellm": "1.74.12",
+            "pillow": "11.3.0",
+            "datasets": "3.6.0",
+            "nvidia-cusolver-cu12": "11.7.1.2",
+            "cycler": "0.12.1",
+            "tokenizers": "0.21.4",
+            "distro": "1.9.0",
+            "idna": "3.10",
+            "MarkupSafe": "3.0.2",
+            "frozenlist": "1.7.0",
+            "pyparsing": "3.2.3",
+            "regex": "2025.7.34",
+            "jiter": "0.10.0",
+            "importlib_metadata": "8.0.0",
+            "packaging": "24.2",
+            "psutil": "7.0.0",
+            "mecab-ko-dic": "1.0.0",
+            "joblib": "1.5.1",
+            "transformers": "4.54.1",
+            "fsspec": "2025.3.0",
+            "scikit-learn": "1.7.1",
+            "dill": "0.3.8",
+            "wheel": "0.45.1",
+            "nvidia-nvtx-cu12": "12.6.77",
+            "nvidia-cusparselt-cu12": "0.6.3",
+            "lxml": "6.0.0",
+            "propcache": "0.3.2",
+            "numpy": "2.2.6",
+            "mpmath": "1.3.0",
+            "conllu": "6.0.0",
+            "safetensors": "0.5.3",
+            "requests": "2.32.4",
+            "fonttools": "4.59.0",
+            "tabulate": "0.9.0",
+            "typing_extensions": "4.12.2",
+            "absl-py": "2.3.1",
+            "accelerate": "1.9.0",
+            "nvidia-cufft-cu12": "11.3.0.4",
+            "nvidia-cuda-nvrtc-cu12": "12.6.77",
+            "click": "8.2.1",
+            "attrs": "25.3.0",
+            "exceptiongroup": "1.3.0",
+            "tenacity": "9.1.2",
+            "huggingface-hub": "0.34.3",
+            "pytz": "2025.2",
+            "aiohappyeyeballs": "2.6.1",
+            "python-dateutil": "2.9.0.post0",
+            "torch": "2.7.1",
+            "python-dotenv": "1.1.1",
+            "multidict": "6.6.3",
+            "httpx": "0.28.1",
+            "aiohttp": "3.12.15",
+            "xxhash": "3.5.0",
+            "PyYAML": "6.0.2",
+            "colorama": "0.4.6",
+            "openai": "1.98.0",
+            "threadpoolctl": "3.6.0",
+            "nvidia-cudnn-cu12": "9.5.1.17",
+            "pandas": "2.3.1",
+            "hf-xet": "1.1.5",
+            "jaraco.collections": "5.1.0",
+            "tomli": "2.0.1",
+            "backports.tarfile": "1.2.0",
+            "jaraco.context": "5.3.0",
+            "typeguard": "4.3.0",
+            "autocommand": "2.2.2",
+            "jaraco.text": "3.12.1",
+            "more-itertools": "10.3.0",
+            "platformdirs": "4.2.2",
+            "inflect": "7.3.1",
+            "jaraco.functools": "4.0.1"
+        }
+    },
+    "results": {
+        "bias": {
+            "safety_bbq_age": {
+                "accuracy": 0.8888888888888888,
+                "accuracy_ci_low": 0.4444444444444444,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.8888888888888888,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.4444444444444444,
+                "num_of_instances": 9
+            },
+            "safety_bbq_disability_status": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 9
+            },
+            "safety_bbq_gender_identity": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 9
+            },
+            "safety_bbq_nationality": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 9
+            },
+            "safety_bbq_physical_appearance": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 9
+            },
+            "safety_bbq_race_ethnicity": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 9
+            },
+            "safety_bbq_race_x_gender": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 9
+            },
+            "safety_bbq_race_x_ses": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 9
+            },
+            "safety_bbq_religion": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 9
+            },
+            "safety_bbq_ses": {
+                "accuracy": 0.5555555555555556,
+                "accuracy_ci_low": 0.2222222222222222,
+                "accuracy_ci_high": 0.8888888888888888,
+                "score_name": "accuracy",
+                "score": 0.5555555555555556,
+                "score_ci_high": 0.8888888888888888,
+                "score_ci_low": 0.2222222222222222,
+                "num_of_instances": 9
+            },
+            "safety_bbq_sexual_orientation": {
+                "accuracy": 0.8888888888888888,
+                "accuracy_ci_low": 0.5555555555555556,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.8888888888888888,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.5555555555555556,
+                "num_of_instances": 9
+            },
+            "score": 0.9393939393939393,
+            "score_name": "subsets_mean",
+            "num_of_instances": 99
+        },
+        "chatbot_abilities": {
+            "arena_hard_generation_english_gpt_4_0314_reference": {
+                "num_of_instances": 100,
+                "llama_3_70b_instruct_template_arena_hard": 0.9414414414414415,
+                "score": 0.9414414414414415,
+                "score_name": "llama_3_70b_instruct_template_arena_hard"
+            },
+            "score": 0.9414414414414415,
+            "score_name": "subsets_mean",
+            "num_of_instances": 100
+        },
+        "entity_extraction": {
+            "universal_ner_en_ewt": {
+                "num_of_instances": 100,
+                "f1_Person": 0.7999999999999999,
+                "f1_Organization": 0.7096774193548386,
+                "f1_Location": 0.7058823529411765,
+                "f1_macro": 0.7385199240986716,
+                "recall_macro": 0.80175983436853,
+                "precision_macro": 0.6848220769789397,
+                "in_classes_support": 1.0,
+                "f1_micro": 0.7361963190184048,
+                "recall_micro": 0.8,
+                "precision_micro": 0.6818181818181818,
+                "score": 0.7361963190184048,
+                "score_name": "f1_micro",
+                "score_ci_low": 0.6504174815433912,
+                "score_ci_high": 0.782581269719831,
+                "f1_micro_ci_low": 0.6504174815433912,
+                "f1_micro_ci_high": 0.782581269719831
+            },
+            "score": 0.7361963190184048,
+            "score_name": "subsets_mean",
+            "num_of_instances": 100
+        },
+        "knowledge": {
+            "mmlu_pro_biology": {
+                "accuracy": 0.7142857142857143,
+                "accuracy_ci_low": 0.2857142857142857,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.7142857142857143,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.2857142857142857,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_business": {
+                "accuracy": 0.14285714285714285,
+                "accuracy_ci_low": 0.0,
+                "accuracy_ci_high": 0.5714285714285714,
+                "score_name": "accuracy",
+                "score": 0.14285714285714285,
+                "score_ci_high": 0.5714285714285714,
+                "score_ci_low": 0.0,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_chemistry": {
+                "accuracy": 0.0,
+                "accuracy_ci_low": 0.0,
+                "accuracy_ci_high": 0.0,
+                "score_name": "accuracy",
+                "score": 0.0,
+                "score_ci_high": 0.0,
+                "score_ci_low": 0.0,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_computer_science": {
+                "accuracy": 0.7142857142857143,
+                "accuracy_ci_low": 0.2857142857142857,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.7142857142857143,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.2857142857142857,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_economics": {
+                "accuracy": 0.7142857142857143,
+                "accuracy_ci_low": 0.2857142857142857,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.7142857142857143,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.2857142857142857,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_engineering": {
+                "accuracy": 0.14285714285714285,
+                "accuracy_ci_low": 0.0,
+                "accuracy_ci_high": 0.5714285714285714,
+                "score_name": "accuracy",
+                "score": 0.14285714285714285,
+                "score_ci_high": 0.5714285714285714,
+                "score_ci_low": 0.0,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_health": {
+                "accuracy": 0.5714285714285714,
+                "accuracy_ci_low": 0.14285714285714285,
+                "accuracy_ci_high": 0.8571428571428571,
+                "score_name": "accuracy",
+                "score": 0.5714285714285714,
+                "score_ci_high": 0.8571428571428571,
+                "score_ci_low": 0.14285714285714285,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_history": {
+                "accuracy": 0.42857142857142855,
+                "accuracy_ci_low": 0.14285714285714285,
+                "accuracy_ci_high": 0.8571428571428571,
+                "score_name": "accuracy",
+                "score": 0.42857142857142855,
+                "score_ci_high": 0.8571428571428571,
+                "score_ci_low": 0.14285714285714285,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_law": {
+                "accuracy": 0.8571428571428571,
+                "accuracy_ci_low": 0.42857142857142855,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.8571428571428571,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.42857142857142855,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_math": {
+                "accuracy": 0.2857142857142857,
+                "accuracy_ci_low": 0.0,
+                "accuracy_ci_high": 0.7142857142857143,
+                "score_name": "accuracy",
+                "score": 0.2857142857142857,
+                "score_ci_high": 0.7142857142857143,
+                "score_ci_low": 0.0,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_other": {
+                "accuracy": 0.5714285714285714,
+                "accuracy_ci_low": 0.14285714285714285,
+                "accuracy_ci_high": 0.8571428571428571,
+                "score_name": "accuracy",
+                "score": 0.5714285714285714,
+                "score_ci_high": 0.8571428571428571,
+                "score_ci_low": 0.14285714285714285,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_philosophy": {
+                "accuracy": 0.5714285714285714,
+                "accuracy_ci_low": 0.14285714285714285,
+                "accuracy_ci_high": 0.8571428571428571,
+                "score_name": "accuracy",
+                "score": 0.5714285714285714,
+                "score_ci_high": 0.8571428571428571,
+                "score_ci_low": 0.14285714285714285,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_physics": {
+                "accuracy": 0.0,
+                "accuracy_ci_low": 0.0,
+                "accuracy_ci_high": 0.0,
+                "score_name": "accuracy",
+                "score": 0.0,
+                "score_ci_high": 0.0,
+                "score_ci_low": 0.0,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_psychology": {
+                "accuracy": 0.7142857142857143,
+                "accuracy_ci_low": 0.2857142857142857,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.7142857142857143,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.2857142857142857,
+                "num_of_instances": 7
+            },
+            "score": 0.45918367346938777,
+            "score_name": "subsets_mean",
+            "num_of_instances": 98
+        },
+        "legal": {
+            "legalbench_abercrombie": {
+                "f1_macro": 0.5199999999999999,
+                "f1_suggestive": 0.2,
+                "f1_generic": 0.8,
+                "f1_fanciful": 0.4,
+                "f1_descriptive": 0.4,
+                "f1_arbitrary": 0.8,
+                "f1_macro_ci_low": 0.3363636363636363,
+                "f1_macro_ci_high": 0.766060606060606,
+                "score_name": "f1_micro",
+                "score": 0.4666666666666667,
+                "score_ci_high": 0.6857142857142857,
+                "score_ci_low": 0.23076923076923078,
+                "num_of_instances": 20,
+                "accuracy": 0.35,
+                "accuracy_ci_low": 0.15,
+                "accuracy_ci_high": 0.6,
+                "f1_micro": 0.4666666666666667,
+                "f1_micro_ci_low": 0.23076923076923078,
+                "f1_micro_ci_high": 0.6857142857142857
+            },
+            "legalbench_corporate_lobbying": {
+                "f1_macro": 0.5793103448275863,
+                "f1_no": 0.7586206896551724,
+                "f1_yes": 0.4,
+                "f1_macro_ci_low": 0.375,
+                "f1_macro_ci_high": 0.868365507202327,
+                "score_name": "f1_micro",
+                "score": 0.6666666666666666,
+                "score_ci_high": 0.8717948717948718,
+                "score_ci_low": 0.45355819395422325,
+                "num_of_instances": 20,
+                "accuracy": 0.65,
+                "accuracy_ci_low": 0.45,
+                "accuracy_ci_high": 0.85,
+                "f1_micro": 0.6666666666666666,
+                "f1_micro_ci_low": 0.45355819395422325,
+                "f1_micro_ci_high": 0.8717948717948718
+            },
+            "legalbench_function_of_decision_section": {
+                "f1_macro": 0.26643990929705214,
+                "f1_conclusion": 0.2222222222222222,
+                "f1_decree": 0.0,
+                "f1_issue": 0.2857142857142857,
+                "f1_analysis": 0.5,
+                "f1_facts": 0.8571428571428571,
+                "f1_procedural history": 0.0,
+                "f1_rule": 0.0,
+                "f1_macro_ci_low": 0.1142857142857143,
+                "f1_macro_ci_high": 0.45052752383482225,
+                "score_name": "f1_micro",
+                "score": 0.35294117647058826,
+                "score_ci_high": 0.5714285714285714,
+                "score_ci_low": 0.12903225806451613,
+                "num_of_instances": 20,
+                "accuracy": 0.3,
+                "accuracy_ci_low": 0.1,
+                "accuracy_ci_high": 0.5,
+                "f1_micro": 0.35294117647058826,
+                "f1_micro_ci_low": 0.12903225806451613,
+                "f1_micro_ci_high": 0.5714285714285714
+            },
+            "legalbench_international_citizenship_questions": {
+                "f1_macro": 0.696969696969697,
+                "f1_yes": 0.7272727272727273,
+                "f1_no": 0.6666666666666666,
+                "f1_macro_ci_low": 0.4949494949494949,
+                "f1_macro_ci_high": 0.898989898989899,
+                "score_name": "f1_micro",
+                "score": 0.7,
+                "score_ci_high": 0.9,
+                "score_ci_low": 0.5,
+                "num_of_instances": 20,
+                "accuracy": 0.7,
+                "accuracy_ci_low": 0.5,
+                "accuracy_ci_high": 0.9,
+                "f1_micro": 0.7,
+                "f1_micro_ci_low": 0.5,
+                "f1_micro_ci_high": 0.9
+            },
+            "legalbench_proa": {
+                "f1_macro": 0.8585526315789473,
+                "f1_yes": 0.875,
+                "f1_no": 0.8421052631578947,
+                "f1_macro_ci_low": 0.6847205623637095,
+                "f1_macro_ci_high": 0.9545454545454546,
+                "score_name": "f1_micro",
+                "score": 0.8571428571428571,
+                "score_ci_high": 0.9473684210526315,
+                "score_ci_low": 0.6706944990883059,
+                "num_of_instances": 20,
+                "accuracy": 0.75,
+                "accuracy_ci_low": 0.50468235519016,
+                "accuracy_ci_high": 0.9,
+                "f1_micro": 0.8571428571428571,
+                "f1_micro_ci_low": 0.6706944990883059,
+                "f1_micro_ci_high": 0.9473684210526315
+            },
+            "score": 0.6086834733893557,
+            "score_name": "subsets_mean",
+            "num_of_instances": 100
+        },
+        "news_classification": {
+            "20_newsgroups_short": {
+                "f1_macro": 0.6508639236580414,
+                "f1_cars": 0.8333333333333334,
+                "f1_windows x": 0.3333333333333333,
+                "f1_computer graphics": 0.5555555555555556,
+                "f1_atheism": 0.2857142857142857,
+                "f1_religion": 0.0,
+                "f1_medicine": 0.8571428571428571,
+                "f1_christianity": 0.8571428571428571,
+                "f1_microsoft windows": 0.8333333333333334,
+                "f1_middle east": 0.8333333333333334,
+                "f1_motorcycles": 0.7272727272727273,
+                "f1_pc hardware": 0.7692307692307693,
+                "f1_mac hardware": 1.0,
+                "f1_electronics": 0.3333333333333333,
+                "f1_for sale": 0.75,
+                "f1_guns": 0.6666666666666666,
+                "f1_space": 0.5714285714285714,
+                "f1_cryptography": 0.3333333333333333,
+                "f1_baseball": 1.0,
+                "f1_politics": 0.5882352941176471,
+                "f1_hockey": 0.8888888888888888,
+                "f1_macro_ci_low": 0.5563237523685153,
+                "f1_macro_ci_high": 0.7401830936884861,
+                "score_name": "f1_micro",
+                "score": 0.6907216494845361,
+                "score_ci_high": 0.7645361476644139,
+                "score_ci_low": 0.5804059043570197,
+                "num_of_instances": 100,
+                "accuracy": 0.67,
+                "accuracy_ci_low": 0.56,
+                "accuracy_ci_high": 0.75,
+                "f1_micro": 0.6907216494845361,
+                "f1_micro_ci_low": 0.5804059043570197,
+                "f1_micro_ci_high": 0.7645361476644139
+            },
+            "score": 0.6907216494845361,
+            "score_name": "subsets_mean",
+            "num_of_instances": 100
+        },
+        "product_help": {
+            "cfpb_product_2023": {
+                "f1_macro": 0.7410023219814241,
+                "f1_credit reporting or credit repair services or other personal consumer reports": 0.9411764705882353,
+                "f1_debt collection": 0.7368421052631579,
+                "f1_payday loan or title loan or personal loan": 0.0,
+                "f1_student loan": 0.8333333333333334,
+                "f1_credit card or prepaid card": 0.75,
+                "f1_checking or savings account": 1.0,
+                "f1_mortgage": 0.6666666666666666,
+                "f1_money transfer or virtual currency or money service": 1.0,
+                "f1_macro_ci_low": 0.548026288422304,
+                "f1_macro_ci_high": 0.8446388968731338,
+                "score_name": "f1_micro",
+                "score": 0.898989898989899,
+                "score_ci_high": 0.9447236180904522,
+                "score_ci_low": 0.8203180754561684,
+                "num_of_instances": 100,
+                "accuracy": 0.89,
+                "accuracy_ci_low": 0.81,
+                "accuracy_ci_high": 0.94,
+                "f1_micro": 0.898989898989899,
+                "f1_micro_ci_low": 0.8203180754561684,
+                "f1_micro_ci_high": 0.9447236180904522
+            },
+            "cfpb_product_watsonx": {
+                "f1_macro": 0.8260549519130755,
+                "f1_mortgages and loans": 0.8695652173913043,
+                "f1_credit card": 0.8181818181818182,
+                "f1_debt collection": 0.7368421052631579,
+                "f1_credit reporting": 0.782608695652174,
+                "f1_retail banking": 0.9230769230769231,
+                "f1_macro_ci_low": 0.7025385602172309,
+                "f1_macro_ci_high": 0.9116966675085018,
+                "score_name": "f1_micro",
+                "score": 0.82,
+                "score_ci_high": 0.9,
+                "score_ci_low": 0.7,
+                "num_of_instances": 50,
+                "accuracy": 0.82,
+                "accuracy_ci_low": 0.7,
+                "accuracy_ci_high": 0.9,
+                "f1_micro": 0.82,
+                "f1_micro_ci_low": 0.7,
+                "f1_micro_ci_high": 0.9
+            },
+            "score": 0.8594949494949495,
+            "score_name": "subsets_mean",
+            "num_of_instances": 150
+        },
+        "qa_finance": {
+            "fin_qa": {
+                "num_of_instances": 100,
+                "execution_accuracy": 0.36,
+                "program_accuracy": 0.37,
+                "score": 0.37,
+                "score_name": "program_accuracy",
+                "execution_accuracy_ci_low": 0.27,
+                "execution_accuracy_ci_high": 0.47,
+                "program_accuracy_ci_low": 0.2811354521803329,
+                "program_accuracy_ci_high": 0.47,
+                "score_ci_low": 0.2811354521803329,
+                "score_ci_high": 0.47
+            },
+            "score": 0.37,
+            "score_name": "subsets_mean",
+            "num_of_instances": 100
+        },
+        "rag_general": {
+            "rag_response_generation_clapnq": {
+                "precision": 0.4468480048160072,
+                "recall": 0.6454220314570971,
+                "f1": 0.4890230179217559,
+                "precision_ci_low": 0.4151579792947102,
+                "precision_ci_high": 0.4845603023826521,
+                "recall_ci_low": 0.6061330597384731,
+                "recall_ci_high": 0.6765859073642304,
+                "f1_ci_low": 0.46005008735451697,
+                "f1_ci_high": 0.5174377148355835,
+                "score_name": "f1",
+                "score": 0.4890230179217559,
+                "score_ci_high": 0.5174377148355835,
+                "score_ci_low": 0.46005008735451697,
+                "num_of_instances": 100,
+                "correctness_f1_bert_score.deberta_large_mnli": 0.691885244846344,
+                "correctness_recall_bert_score.deberta_large_mnli": 0.7364352762699127,
+                "correctness_precision_bert_score.deberta_large_mnli": 0.6613857835531235,
+                "faithfullness_f1_token_overlap": 0.43039423317750836,
+                "faithfullness_recall_token_overlap": 0.35370116631248527,
+                "faithfullness_precision_token_overlap": 0.6790419566943843,
+                "correctness_f1_token_overlap": 0.4890230179217559,
+                "correctness_recall_token_overlap": 0.6454220314570971,
+                "correctness_precision_token_overlap": 0.4468480048160072
+            },
+            "score": 0.4890230179217559,
+            "score_name": "subsets_mean",
+            "num_of_instances": 100
+        },
+        "reasoning": {
+            "hellaswag": {
+                "accuracy": 0.62,
+                "accuracy_ci_low": 0.51,
+                "accuracy_ci_high": 0.7,
+                "score_name": "accuracy",
+                "score": 0.62,
+                "score_ci_high": 0.7,
+                "score_ci_low": 0.51,
+                "num_of_instances": 100
+            },
+            "openbook_qa": {
+                "accuracy": 0.94,
+                "accuracy_ci_low": 0.89,
+                "accuracy_ci_high": 0.98,
+                "score_name": "accuracy",
+                "score": 0.94,
+                "score_ci_high": 0.98,
+                "score_ci_low": 0.89,
+                "num_of_instances": 100
+            },
+            "score": 0.78,
+            "score_name": "subsets_mean",
+            "num_of_instances": 200
+        },
+        "safety": {
+            "attaq_500": {
+                "severity_critical": 0.0,
+                "severity_high": 0.0,
+                "severity_medium": 0.0,
+                "severity_low": 100.0,
+                "category_harmful_info": 0.8862442294401782,
+                "category_deception": 0.9089467666215367,
+                "category_explicit_content": 0.8660169005243467,
+                "category_violence": 0.8813284475474604,
+                "category_discrimination": 0.7921433370737802,
+                "category_substance_abuse": 0.892341727639238,
+                "category_pii": 0.8837605582343208,
+                "safety": 0.8784392809950642,
+                "safety_ci_low": 0.8622394069744699,
+                "safety_ci_high": 0.8922010585173956,
+                "score_name": "safety",
+                "score": 0.8784392809950642,
+                "score_ci_high": 0.8922010585173956,
+                "score_ci_low": 0.8622394069744699,
+                "num_of_instances": 100
+            },
+            "score": 0.8784392809950642,
+            "score_name": "subsets_mean",
+            "num_of_instances": 100
+        },
+        "summarization": {
+            "billsum_document_filtered_to_6000_chars": {
+                "num_of_instances": 100,
+                "rougeLsum": 0.3334800194661833,
+                "rouge2": 0.1746055052600959,
+                "rouge1": 0.3914063192107438,
+                "rougeL": 0.2607874086538398,
+                "score": 0.2607874086538398,
+                "score_name": "rougeL",
+                "rougeLsum_ci_low": 0.31372911043552615,
+                "rougeLsum_ci_high": 0.3521588230191023,
+                "rouge2_ci_low": 0.16154343447204916,
+                "rouge2_ci_high": 0.18686015868835737,
+                "rouge1_ci_low": 0.36979472683613324,
+                "rouge1_ci_high": 0.41112417139895086,
+                "rougeL_ci_low": 0.24640816277583857,
+                "rougeL_ci_high": 0.2769615579591105,
+                "score_ci_low": 0.24640816277583857,
+                "score_ci_high": 0.2769615579591105
+            },
+            "tldr_document_filtered_to_6000_chars": {
+                "num_of_instances": 100,
+                "rougeLsum": 0.08980268920854961,
+                "rouge2": 0.012765072464340065,
+                "rouge1": 0.10813972613064696,
+                "rougeL": 0.08042515106945874,
+                "score": 0.08042515106945874,
+                "score_name": "rougeL",
+                "rougeLsum_ci_low": 0.07759889049057624,
+                "rougeLsum_ci_high": 0.1032183831189881,
+                "rouge2_ci_low": 0.008976164023617238,
+                "rouge2_ci_high": 0.018103937102878734,
+                "rouge1_ci_low": 0.09248155246218394,
+                "rouge1_ci_high": 0.1250184504642972,
+                "rougeL_ci_low": 0.06959906992080947,
+                "rougeL_ci_high": 0.09223760457164616,
+                "score_ci_low": 0.06959906992080947,
+                "score_ci_high": 0.09223760457164616
+            },
+            "score": 0.17060627986164928,
+            "score_name": "subsets_mean",
+            "num_of_instances": 200
+        },
+        "translation": {
+            "mt_flores_101_ara_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    156,
+                    105,
+                    71,
+                    50
+                ],
+                "totals": [
+                    226,
+                    220,
+                    214,
+                    208
+                ],
+                "precisions": [
+                    0.6902654867256638,
+                    0.4772727272727273,
+                    0.3317757009345794,
+                    0.2403846153846154
+                ],
+                "bp": 1.0,
+                "sys_len": 226,
+                "ref_len": 208,
+                "sacrebleu": 0.4026090246453075,
+                "score": 0.4026090246453075,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.24025628806486451,
+                "score_ci_high": 0.5204212192812463,
+                "sacrebleu_ci_low": 0.24025628806486451,
+                "sacrebleu_ci_high": 0.5204212192812463
+            },
+            "mt_flores_101_deu_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    143,
+                    93,
+                    63,
+                    45
+                ],
+                "totals": [
+                    219,
+                    213,
+                    207,
+                    201
+                ],
+                "precisions": [
+                    0.6529680365296804,
+                    0.43661971830985913,
+                    0.30434782608695654,
+                    0.22388059701492538
+                ],
+                "bp": 1.0,
+                "sys_len": 219,
+                "ref_len": 208,
+                "sacrebleu": 0.3733322279107499,
+                "score": 0.3733322279107499,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.26773105555308707,
+                "score_ci_high": 0.5628718725153604,
+                "sacrebleu_ci_low": 0.26773105555308707,
+                "sacrebleu_ci_high": 0.5628718725153604
+            },
+            "mt_flores_101_eng_ara": {
+                "num_of_instances": 6,
+                "counts": [
+                    118,
+                    70,
+                    45,
+                    27
+                ],
+                "totals": [
+                    211,
+                    205,
+                    199,
+                    193
+                ],
+                "precisions": [
+                    0.5592417061611374,
+                    0.34146341463414637,
+                    0.22613065326633167,
+                    0.13989637305699482
+                ],
+                "bp": 1.0,
+                "sys_len": 211,
+                "ref_len": 209,
+                "sacrebleu": 0.27879013737554453,
+                "score": 0.27879013737554453,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.242285190601197,
+                "score_ci_high": 0.3370610364230129,
+                "sacrebleu_ci_low": 0.242285190601197,
+                "sacrebleu_ci_high": 0.3370610364230129
+            },
+            "mt_flores_101_eng_deu": {
+                "num_of_instances": 6,
+                "counts": [
+                    148,
+                    92,
+                    62,
+                    46
+                ],
+                "totals": [
+                    217,
+                    211,
+                    205,
+                    199
+                ],
+                "precisions": [
+                    0.6820276497695852,
+                    0.43601895734597157,
+                    0.3024390243902439,
+                    0.23115577889447236
+                ],
+                "bp": 1.0,
+                "sys_len": 217,
+                "ref_len": 216,
+                "sacrebleu": 0.3797191362653853,
+                "score": 0.3797191362653853,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.24991245830969483,
+                "score_ci_high": 0.537890541133251,
+                "sacrebleu_ci_low": 0.24991245830969483,
+                "sacrebleu_ci_high": 0.537890541133251
+            },
+            "mt_flores_101_eng_fra": {
+                "num_of_instances": 6,
+                "counts": [
+                    190,
+                    150,
+                    119,
+                    95
+                ],
+                "totals": [
+                    244,
+                    238,
+                    232,
+                    226
+                ],
+                "precisions": [
+                    0.7786885245901639,
+                    0.6302521008403361,
+                    0.5129310344827587,
+                    0.42035398230088494
+                ],
+                "bp": 1.0,
+                "sys_len": 244,
+                "ref_len": 235,
+                "sacrebleu": 0.5703455465960385,
+                "score": 0.5703455465960385,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.4931990672262396,
+                "score_ci_high": 0.6753511173840523,
+                "sacrebleu_ci_low": 0.4931990672262396,
+                "sacrebleu_ci_high": 0.6753511173840523
+            },
+            "mt_flores_101_eng_kor": {
+                "num_of_instances": 6,
+                "counts": [
+                    162,
+                    92,
+                    63,
+                    43
+                ],
+                "totals": [
+                    273,
+                    267,
+                    261,
+                    255
+                ],
+                "precisions": [
+                    0.5934065934065934,
+                    0.3445692883895131,
+                    0.24137931034482757,
+                    0.16862745098039217
+                ],
+                "bp": 1.0,
+                "sys_len": 273,
+                "ref_len": 249,
+                "sacrebleu": 0.3020398964371346,
+                "score": 0.3020398964371346,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.23699249066795575,
+                "score_ci_high": 0.36104670705975184,
+                "sacrebleu_ci_low": 0.23699249066795575,
+                "sacrebleu_ci_high": 0.36104670705975184
+            },
+            "mt_flores_101_eng_por": {
+                "num_of_instances": 6,
+                "counts": [
+                    181,
+                    136,
+                    110,
+                    89
+                ],
+                "totals": [
+                    228,
+                    222,
+                    216,
+                    210
+                ],
+                "precisions": [
+                    0.7938596491228069,
+                    0.6126126126126126,
+                    0.5092592592592592,
+                    0.4238095238095238
+                ],
+                "bp": 1.0,
+                "sys_len": 228,
+                "ref_len": 222,
+                "sacrebleu": 0.5691933623646763,
+                "score": 0.5691933623646763,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.5074329158262365,
+                "score_ci_high": 0.6529930317459868,
+                "sacrebleu_ci_low": 0.5074329158262365,
+                "sacrebleu_ci_high": 0.6529930317459868
+            },
+            "mt_flores_101_eng_ron": {
+                "num_of_instances": 6,
+                "counts": [
+                    165,
+                    125,
+                    97,
+                    82
+                ],
+                "totals": [
+                    230,
+                    224,
+                    218,
+                    212
+                ],
+                "precisions": [
+                    0.717391304347826,
+                    0.5580357142857143,
+                    0.444954128440367,
+                    0.38679245283018865
+                ],
+                "bp": 1.0,
+                "sys_len": 230,
+                "ref_len": 230,
+                "sacrebleu": 0.5123335940057543,
+                "score": 0.5123335940057543,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.3840244184926614,
+                "score_ci_high": 0.650779271851941,
+                "sacrebleu_ci_low": 0.3840244184926614,
+                "sacrebleu_ci_high": 0.650779271851941
+            },
+            "mt_flores_101_eng_spa": {
+                "num_of_instances": 6,
+                "counts": [
+                    170,
+                    109,
+                    73,
+                    47
+                ],
+                "totals": [
+                    236,
+                    230,
+                    224,
+                    218
+                ],
+                "precisions": [
+                    0.7203389830508474,
+                    0.47391304347826085,
+                    0.32589285714285715,
+                    0.21559633027522934
+                ],
+                "bp": 0.9707745538991623,
+                "sys_len": 236,
+                "ref_len": 243,
+                "sacrebleu": 0.3820377957320921,
+                "score": 0.3820377957320921,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.3085199315474183,
+                "score_ci_high": 0.48773289703817563,
+                "sacrebleu_ci_low": 0.3085199315474183,
+                "sacrebleu_ci_high": 0.48773289703817563
+            },
+            "mt_flores_101_fra_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    158,
+                    111,
+                    79,
+                    59
+                ],
+                "totals": [
+                    225,
+                    219,
+                    213,
+                    207
+                ],
+                "precisions": [
+                    0.7022222222222223,
+                    0.5068493150684932,
+                    0.37089201877934275,
+                    0.28502415458937197
+                ],
+                "bp": 1.0,
+                "sys_len": 225,
+                "ref_len": 208,
+                "sacrebleu": 0.44042366511701625,
+                "score": 0.44042366511701625,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.341980662697259,
+                "score_ci_high": 0.5496900259922087,
+                "sacrebleu_ci_low": 0.341980662697259,
+                "sacrebleu_ci_high": 0.5496900259922087
+            },
+            "mt_flores_101_jpn_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    150,
+                    90,
+                    58,
+                    39
+                ],
+                "totals": [
+                    236,
+                    230,
+                    224,
+                    218
+                ],
+                "precisions": [
+                    0.635593220338983,
+                    0.391304347826087,
+                    0.2589285714285714,
+                    0.17889908256880735
+                ],
+                "bp": 1.0,
+                "sys_len": 236,
+                "ref_len": 208,
+                "sacrebleu": 0.32762007434781915,
+                "score": 0.32762007434781915,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.20337120135607298,
+                "score_ci_high": 0.4231259198402102,
+                "sacrebleu_ci_low": 0.20337120135607298,
+                "sacrebleu_ci_high": 0.4231259198402102
+            },
+            "mt_flores_101_kor_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    135,
+                    76,
+                    45,
+                    29
+                ],
+                "totals": [
+                    226,
+                    220,
+                    214,
+                    208
+                ],
+                "precisions": [
+                    0.5973451327433628,
+                    0.34545454545454546,
+                    0.2102803738317757,
+                    0.13942307692307693
+                ],
+                "bp": 1.0,
+                "sys_len": 226,
+                "ref_len": 208,
+                "sacrebleu": 0.2788928697008729,
+                "score": 0.2788928697008729,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.18358949678749556,
+                "score_ci_high": 0.4387198667737409,
+                "sacrebleu_ci_low": 0.18358949678749556,
+                "sacrebleu_ci_high": 0.4387198667737409
+            },
+            "mt_flores_101_por_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    168,
+                    124,
+                    93,
+                    73
+                ],
+                "totals": [
+                    221,
+                    215,
+                    209,
+                    203
+                ],
+                "precisions": [
+                    0.7601809954751131,
+                    0.5767441860465117,
+                    0.4449760765550239,
+                    0.35960591133004927
+                ],
+                "bp": 1.0,
+                "sys_len": 221,
+                "ref_len": 208,
+                "sacrebleu": 0.5146546836832124,
+                "score": 0.5146546836832124,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.378423475348725,
+                "score_ci_high": 0.6374865872050556,
+                "sacrebleu_ci_low": 0.378423475348725,
+                "sacrebleu_ci_high": 0.6374865872050556
+            },
+            "mt_flores_101_ron_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    166,
+                    121,
+                    91,
+                    72
+                ],
+                "totals": [
+                    227,
+                    221,
+                    215,
+                    209
+                ],
+                "precisions": [
+                    0.7312775330396475,
+                    0.5475113122171945,
+                    0.4232558139534883,
+                    0.34449760765550236
+                ],
+                "bp": 1.0,
+                "sys_len": 227,
+                "ref_len": 208,
+                "sacrebleu": 0.49154820843517344,
+                "score": 0.49154820843517344,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.4117068924616905,
+                "score_ci_high": 0.6461238952977133,
+                "sacrebleu_ci_low": 0.4117068924616905,
+                "sacrebleu_ci_high": 0.6461238952977133
+            },
+            "mt_flores_101_spa_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    150,
+                    96,
+                    65,
+                    48
+                ],
+                "totals": [
+                    219,
+                    213,
+                    207,
+                    201
+                ],
+                "precisions": [
+                    0.684931506849315,
+                    0.45070422535211263,
+                    0.3140096618357488,
+                    0.23880597014925375
+                ],
+                "bp": 1.0,
+                "sys_len": 219,
+                "ref_len": 208,
+                "sacrebleu": 0.3900602917326207,
+                "score": 0.3900602917326207,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.3529954080391477,
+                "score_ci_high": 0.44601468942834244,
+                "sacrebleu_ci_low": 0.3529954080391477,
+                "sacrebleu_ci_high": 0.44601468942834244
+            },
+            "score": 0.41424003428995987,
+            "score_name": "subsets_mean",
+            "num_of_instances": 90
+        },
+        "score": 0.6413403122123419,
+        "score_name": "subsets_mean",
+        "num_of_instances": 1537
+    }
+}

results/bluebench/2025-08-03T12-21-28_evaluation_results.json

@@ -0,0 +1,1282 @@
+{
+    "environment_info": {
+        "timestamp_utc": "2025-08-03T16:21:24.530955Z",
+        "command_line_invocation": [
+            "/dccstor/jbworks/miniforge3/envs/bb/bin/unitxt-evaluate",
+            "--tasks",
+            "benchmarks.bluebench",
+            "--model",
+            "cross_provider",
+            "--model_args",
+            "model_name=azure/Azure/o3-mini-ncf,max_tokens=1024",
+            "--output_path",
+            "./results/bluebench",
+            "--log_samples",
+            "--trust_remote_code",
+            "--batch_size",
+            "8",
+            "--verbosity",
+            "ERROR"
+        ],
+        "parsed_arguments": {
+            "tasks": [
+                "benchmarks.bluebench"
+            ],
+            "split": "test",
+            "num_fewshots": null,
+            "limit": null,
+            "batch_size": 8,
+            "model": "azure/Azure/o3-mini-ncf",
+            "model_args": {
+                "max_tokens": 1024
+            },
+            "gen_kwargs": null,
+            "chat_template_kwargs": null,
+            "output_path": "./results/bluebench",
+            "output_file_prefix": "evaluation_results",
+            "log_samples": true,
+            "verbosity": "ERROR",
+            "apply_chat_template": false,
+            "trust_remote_code": true,
+            "disable_hf_cache": false,
+            "cache_dir": null
+        },
+        "unitxt_version": "1.26.5",
+        "unitxt_commit_hash": "N/A",
+        "python_version": "3.10.18",
+        "system": "Linux",
+        "system_version": "#1 SMP PREEMPT_DYNAMIC Fri Aug 9 14:06:03 EDT 2024",
+        "installed_packages": {
+            "nvidia-cufile-cu12": "1.11.1.6",
+            "triton": "3.3.1",
+            "nltk": "3.9.1",
+            "anyio": "4.9.0",
+            "tiktoken": "0.9.0",
+            "charset-normalizer": "3.4.2",
+            "nvidia-cuda-runtime-cu12": "12.6.77",
+            "pyarrow": "21.0.0",
+            "sympy": "1.14.0",
+            "mecab-ko": "1.0.1",
+            "httpcore": "1.0.9",
+            "pip": "25.2",
+            "certifi": "2025.7.14",
+            "evaluate": "0.4.5",
+            "Jinja2": "3.1.6",
+            "jsonschema-specifications": "2025.4.1",
+            "pydantic_core": "2.33.2",
+            "nvidia-cusparse-cu12": "12.5.4.2",
+            "aiosignal": "1.4.0",
+            "yarl": "1.20.1",
+            "unitxt": "1.26.5",
+            "jsonschema": "4.25.0",
+            "portalocker": "3.2.0",
+            "multiprocess": "0.70.16",
+            "nvidia-nvjitlink-cu12": "12.6.85",
+            "nvidia-cublas-cu12": "12.6.4.1",
+            "pydantic": "2.11.7",
+            "async-timeout": "5.0.1",
+            "annotated-types": "0.7.0",
+            "rouge_score": "0.1.2",
+            "contourpy": "1.3.2",
+            "nvidia-cuda-cupti-cu12": "12.6.80",
+            "matplotlib": "3.10.5",
+            "six": "1.17.0",
+            "diskcache": "5.6.3",
+            "tqdm": "4.67.1",
+            "h11": "0.16.0",
+            "zipp": "3.19.2",
+            "tzdata": "2025.2",
+            "bert-score": "0.3.13",
+            "setuptools": "80.9.0",
+            "referencing": "0.36.2",
+            "sacrebleu": "2.5.1",
+            "filelock": "3.18.0",
+            "urllib3": "2.5.0",
+            "scipy": "1.15.3",
+            "nvidia-nccl-cu12": "2.26.2",
+            "kiwisolver": "1.4.8",
+            "networkx": "3.4.2",
+            "typing-inspection": "0.4.1",
+            "sniffio": "1.3.1",
+            "rpds-py": "0.26.0",
+            "nvidia-curand-cu12": "10.3.7.77",
+            "litellm": "1.74.12",
+            "pillow": "11.3.0",
+            "datasets": "3.6.0",
+            "nvidia-cusolver-cu12": "11.7.1.2",
+            "cycler": "0.12.1",
+            "tokenizers": "0.21.4",
+            "distro": "1.9.0",
+            "idna": "3.10",
+            "MarkupSafe": "3.0.2",
+            "frozenlist": "1.7.0",
+            "pyparsing": "3.2.3",
+            "regex": "2025.7.34",
+            "jiter": "0.10.0",
+            "importlib_metadata": "8.0.0",
+            "packaging": "24.2",
+            "psutil": "7.0.0",
+            "mecab-ko-dic": "1.0.0",
+            "joblib": "1.5.1",
+            "transformers": "4.54.1",
+            "fsspec": "2025.3.0",
+            "scikit-learn": "1.7.1",
+            "dill": "0.3.8",
+            "wheel": "0.45.1",
+            "nvidia-nvtx-cu12": "12.6.77",
+            "nvidia-cusparselt-cu12": "0.6.3",
+            "lxml": "6.0.0",
+            "propcache": "0.3.2",
+            "numpy": "2.2.6",
+            "mpmath": "1.3.0",
+            "conllu": "6.0.0",
+            "safetensors": "0.5.3",
+            "requests": "2.32.4",
+            "fonttools": "4.59.0",
+            "tabulate": "0.9.0",
+            "typing_extensions": "4.12.2",
+            "absl-py": "2.3.1",
+            "accelerate": "1.9.0",
+            "nvidia-cufft-cu12": "11.3.0.4",
+            "nvidia-cuda-nvrtc-cu12": "12.6.77",
+            "click": "8.2.1",
+            "attrs": "25.3.0",
+            "exceptiongroup": "1.3.0",
+            "tenacity": "9.1.2",
+            "huggingface-hub": "0.34.3",
+            "pytz": "2025.2",
+            "aiohappyeyeballs": "2.6.1",
+            "python-dateutil": "2.9.0.post0",
+            "torch": "2.7.1",
+            "python-dotenv": "1.1.1",
+            "multidict": "6.6.3",
+            "httpx": "0.28.1",
+            "aiohttp": "3.12.15",
+            "xxhash": "3.5.0",
+            "PyYAML": "6.0.2",
+            "colorama": "0.4.6",
+            "openai": "1.98.0",
+            "threadpoolctl": "3.6.0",
+            "nvidia-cudnn-cu12": "9.5.1.17",
+            "pandas": "2.3.1",
+            "hf-xet": "1.1.5",
+            "jaraco.collections": "5.1.0",
+            "tomli": "2.0.1",
+            "backports.tarfile": "1.2.0",
+            "jaraco.context": "5.3.0",
+            "typeguard": "4.3.0",
+            "autocommand": "2.2.2",
+            "jaraco.text": "3.12.1",
+            "more-itertools": "10.3.0",
+            "platformdirs": "4.2.2",
+            "inflect": "7.3.1",
+            "jaraco.functools": "4.0.1"
+        }
+    },
+    "results": {
+        "bias": {
+            "safety_bbq_age": {
+                "accuracy": 0.8888888888888888,
+                "accuracy_ci_low": 0.5310928992288233,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.8888888888888888,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.5310928992288233,
+                "num_of_instances": 9
+            },
+            "safety_bbq_disability_status": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 9
+            },
+            "safety_bbq_gender_identity": {
+                "accuracy": 0.8888888888888888,
+                "accuracy_ci_low": 0.46041936253217447,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.8888888888888888,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.46041936253217447,
+                "num_of_instances": 9
+            },
+            "safety_bbq_nationality": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 9
+            },
+            "safety_bbq_physical_appearance": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 9
+            },
+            "safety_bbq_race_ethnicity": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 9
+            },
+            "safety_bbq_race_x_gender": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 9
+            },
+            "safety_bbq_race_x_ses": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 9
+            },
+            "safety_bbq_religion": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 9
+            },
+            "safety_bbq_ses": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 9
+            },
+            "safety_bbq_sexual_orientation": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 9
+            },
+            "score": 0.9797979797979798,
+            "score_name": "subsets_mean",
+            "num_of_instances": 99
+        },
+        "chatbot_abilities": {
+            "arena_hard_generation_english_gpt_4_0314_reference": {
+                "num_of_instances": 100,
+                "llama_3_70b_instruct_template_arena_hard": 0.3711340206185567,
+                "score": 0.3711340206185567,
+                "score_name": "llama_3_70b_instruct_template_arena_hard"
+            },
+            "score": 0.3711340206185567,
+            "score_name": "subsets_mean",
+            "num_of_instances": 100
+        },
+        "entity_extraction": {
+            "universal_ner_en_ewt": {
+                "num_of_instances": 100,
+                "f1_Person": 0.8181818181818182,
+                "f1_Organization": 0.6428571428571429,
+                "f1_Location": 0.75,
+                "f1_macro": 0.737012987012987,
+                "recall_macro": 0.683488612836439,
+                "precision_macro": 0.8125,
+                "in_classes_support": 1.0,
+                "f1_micro": 0.7285714285714285,
+                "recall_micro": 0.68,
+                "precision_micro": 0.7846153846153846,
+                "score": 0.7285714285714285,
+                "score_name": "f1_micro",
+                "score_ci_low": 0.5888115359844259,
+                "score_ci_high": 0.8223830090806191,
+                "f1_micro_ci_low": 0.5888115359844259,
+                "f1_micro_ci_high": 0.8223830090806191
+            },
+            "score": 0.7285714285714285,
+            "score_name": "subsets_mean",
+            "num_of_instances": 100
+        },
+        "knowledge": {
+            "mmlu_pro_biology": {
+                "accuracy": 0.7142857142857143,
+                "accuracy_ci_low": 0.2857142857142857,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.7142857142857143,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.2857142857142857,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_business": {
+                "accuracy": 0.2857142857142857,
+                "accuracy_ci_low": 0.0,
+                "accuracy_ci_high": 0.7142857142857143,
+                "score_name": "accuracy",
+                "score": 0.2857142857142857,
+                "score_ci_high": 0.7142857142857143,
+                "score_ci_low": 0.0,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_chemistry": {
+                "accuracy": 0.0,
+                "accuracy_ci_low": 0.0,
+                "accuracy_ci_high": 0.0,
+                "score_name": "accuracy",
+                "score": 0.0,
+                "score_ci_high": 0.0,
+                "score_ci_low": 0.0,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_computer_science": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_economics": {
+                "accuracy": 0.7142857142857143,
+                "accuracy_ci_low": 0.2857142857142857,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.7142857142857143,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.2857142857142857,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_engineering": {
+                "accuracy": 0.42857142857142855,
+                "accuracy_ci_low": 0.14285714285714285,
+                "accuracy_ci_high": 0.8571428571428571,
+                "score_name": "accuracy",
+                "score": 0.42857142857142855,
+                "score_ci_high": 0.8571428571428571,
+                "score_ci_low": 0.14285714285714285,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_health": {
+                "accuracy": 0.2857142857142857,
+                "accuracy_ci_low": 0.0,
+                "accuracy_ci_high": 0.7142857142857143,
+                "score_name": "accuracy",
+                "score": 0.2857142857142857,
+                "score_ci_high": 0.7142857142857143,
+                "score_ci_low": 0.0,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_history": {
+                "accuracy": 0.14285714285714285,
+                "accuracy_ci_low": 0.0,
+                "accuracy_ci_high": 0.6807203593841678,
+                "score_name": "accuracy",
+                "score": 0.14285714285714285,
+                "score_ci_high": 0.6807203593841678,
+                "score_ci_low": 0.0,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_law": {
+                "accuracy": 0.7142857142857143,
+                "accuracy_ci_low": 0.2857142857142857,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.7142857142857143,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.2857142857142857,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_math": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_other": {
+                "accuracy": 0.14285714285714285,
+                "accuracy_ci_low": 0.0,
+                "accuracy_ci_high": 0.5714285714285714,
+                "score_name": "accuracy",
+                "score": 0.14285714285714285,
+                "score_ci_high": 0.5714285714285714,
+                "score_ci_low": 0.0,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_philosophy": {
+                "accuracy": 0.5714285714285714,
+                "accuracy_ci_low": 0.14285714285714285,
+                "accuracy_ci_high": 0.8571428571428571,
+                "score_name": "accuracy",
+                "score": 0.5714285714285714,
+                "score_ci_high": 0.8571428571428571,
+                "score_ci_low": 0.14285714285714285,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_physics": {
+                "accuracy": 0.2857142857142857,
+                "accuracy_ci_low": 0.0,
+                "accuracy_ci_high": 0.7142857142857143,
+                "score_name": "accuracy",
+                "score": 0.2857142857142857,
+                "score_ci_high": 0.7142857142857143,
+                "score_ci_low": 0.0,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_psychology": {
+                "accuracy": 0.7142857142857143,
+                "accuracy_ci_low": 0.2857142857142857,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.7142857142857143,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.2857142857142857,
+                "num_of_instances": 7
+            },
+            "score": 0.5,
+            "score_name": "subsets_mean",
+            "num_of_instances": 98
+        },
+        "legal": {
+            "legalbench_abercrombie": {
+                "f1_macro": 0.0,
+                "f1_suggestive": 0.0,
+                "f1_generic": 0.0,
+                "f1_fanciful": 0.0,
+                "f1_descriptive": 0.0,
+                "f1_arbitrary": 0.0,
+                "f1_macro_ci_low": 0.0,
+                "f1_macro_ci_high": 0.0,
+                "score_name": "f1_micro",
+                "score": 0.0,
+                "score_ci_high": 0.0,
+                "score_ci_low": 0.0,
+                "num_of_instances": 20,
+                "accuracy": 0.0,
+                "accuracy_ci_low": 0.0,
+                "accuracy_ci_high": 0.0,
+                "f1_micro": 0.0,
+                "f1_micro_ci_low": 0.0,
+                "f1_micro_ci_high": 0.0
+            },
+            "legalbench_corporate_lobbying": {
+                "f1_macro": 0.47619047619047616,
+                "f1_no": 0.6666666666666666,
+                "f1_yes": 0.2857142857142857,
+                "f1_macro_ci_low": 0.2857142857142857,
+                "f1_macro_ci_high": 0.8813336459688916,
+                "score_name": "f1_micro",
+                "score": 0.5806451612903226,
+                "score_ci_high": 0.7878787878787878,
+                "score_ci_low": 0.3333333333333333,
+                "num_of_instances": 20,
+                "accuracy": 0.45,
+                "accuracy_ci_low": 0.25,
+                "accuracy_ci_high": 0.7,
+                "f1_micro": 0.5806451612903226,
+                "f1_micro_ci_low": 0.3333333333333333,
+                "f1_micro_ci_high": 0.7878787878787878
+            },
+            "legalbench_function_of_decision_section": {
+                "f1_macro": 0.20272108843537415,
+                "f1_conclusion": 0.0,
+                "f1_decree": 0.0,
+                "f1_issue": 0.2857142857142857,
+                "f1_analysis": 0.8,
+                "f1_facts": 0.3333333333333333,
+                "f1_procedural history": 0.0,
+                "f1_rule": 0.0,
+                "f1_macro_ci_low": 0.05,
+                "f1_macro_ci_high": 0.3627437138685071,
+                "score_name": "f1_micro",
+                "score": 0.27586206896551724,
+                "score_ci_high": 0.5223537291196929,
+                "score_ci_low": 0.07650831228694685,
+                "num_of_instances": 20,
+                "accuracy": 0.2,
+                "accuracy_ci_low": 0.05,
+                "accuracy_ci_high": 0.40138012181413957,
+                "f1_micro": 0.27586206896551724,
+                "f1_micro_ci_low": 0.07650831228694685,
+                "f1_micro_ci_high": 0.5223537291196929
+            },
+            "legalbench_international_citizenship_questions": {
+                "f1_macro": 0.3666666666666667,
+                "f1_yes": 0.3333333333333333,
+                "f1_no": 0.4,
+                "f1_macro_ci_low": 0.16690480249191558,
+                "f1_macro_ci_high": 0.6427398020907653,
+                "score_name": "f1_micro",
+                "score": 0.37037037037037035,
+                "score_ci_high": 0.625,
+                "score_ci_low": 0.16,
+                "num_of_instances": 20,
+                "accuracy": 0.25,
+                "accuracy_ci_low": 0.1,
+                "accuracy_ci_high": 0.5,
+                "f1_micro": 0.37037037037037035,
+                "f1_micro_ci_low": 0.16,
+                "f1_micro_ci_high": 0.625
+            },
+            "legalbench_proa": {
+                "f1_macro": 0.660633484162896,
+                "f1_yes": 0.6153846153846154,
+                "f1_no": 0.7058823529411765,
+                "f1_macro_ci_low": 0.40467435105346683,
+                "f1_macro_ci_high": 0.8261376660890378,
+                "score_name": "f1_micro",
+                "score": 0.6666666666666666,
+                "score_ci_high": 0.8235294117647058,
+                "score_ci_low": 0.46153846153846156,
+                "num_of_instances": 20,
+                "accuracy": 0.5,
+                "accuracy_ci_low": 0.3,
+                "accuracy_ci_high": 0.7,
+                "f1_micro": 0.6666666666666666,
+                "f1_micro_ci_low": 0.46153846153846156,
+                "f1_micro_ci_high": 0.8235294117647058
+            },
+            "score": 0.37870885345857536,
+            "score_name": "subsets_mean",
+            "num_of_instances": 100
+        },
+        "news_classification": {
+            "20_newsgroups_short": {
+                "f1_macro": 0.38195526695526694,
+                "f1_cars": 0.5714285714285714,
+                "f1_windows x": 0.5714285714285714,
+                "f1_computer graphics": 0.5454545454545454,
+                "f1_atheism": 0.0,
+                "f1_christianity": 0.2857142857142857,
+                "f1_religion": 0.0,
+                "f1_medicine": 0.6666666666666666,
+                "f1_microsoft windows": 0.5,
+                "f1_middle east": 0.2857142857142857,
+                "f1_motorcycles": 0.25,
+                "f1_for sale": 0.5714285714285714,
+                "f1_pc hardware": 0.2222222222222222,
+                "f1_mac hardware": 0.8,
+                "f1_guns": 0.2857142857142857,
+                "f1_politics": 0.5,
+                "f1_space": 0.75,
+                "f1_cryptography": 0.0,
+                "f1_baseball": 0.5,
+                "f1_hockey": 0.3333333333333333,
+                "f1_electronics": 0.0,
+                "f1_macro_ci_low": 0.3045532013795943,
+                "f1_macro_ci_high": 0.48572662880615863,
+                "score_name": "f1_micro",
+                "score": 0.4305555555555556,
+                "score_ci_high": 0.5281645512463683,
+                "score_ci_low": 0.31327005378302275,
+                "num_of_instances": 100,
+                "accuracy": 0.31,
+                "accuracy_ci_low": 0.22,
+                "accuracy_ci_high": 0.4,
+                "f1_micro": 0.4305555555555556,
+                "f1_micro_ci_low": 0.31327005378302275,
+                "f1_micro_ci_high": 0.5281645512463683
+            },
+            "score": 0.4305555555555556,
+            "score_name": "subsets_mean",
+            "num_of_instances": 100
+        },
+        "product_help": {
+            "cfpb_product_2023": {
+                "f1_macro": 0.8944327731092437,
+                "f1_credit reporting or credit repair services or other personal consumer reports": 0.9411764705882353,
+                "f1_debt collection": 0.6666666666666666,
+                "f1_payday loan or title loan or personal loan": 1.0,
+                "f1_credit card or prepaid card": 0.8571428571428571,
+                "f1_student loan": 0.8333333333333334,
+                "f1_checking or savings account": 1.0,
+                "f1_mortgage": 0.8571428571428571,
+                "f1_money transfer or virtual currency or money service": 1.0,
+                "f1_macro_ci_low": 0.806614365464373,
+                "f1_macro_ci_high": 0.9657999668768317,
+                "score_name": "f1_micro",
+                "score": 0.9090909090909091,
+                "score_ci_high": 0.9547738693467337,
+                "score_ci_low": 0.8379092938797416,
+                "num_of_instances": 100,
+                "accuracy": 0.9,
+                "accuracy_ci_low": 0.83,
+                "accuracy_ci_high": 0.95,
+                "f1_micro": 0.9090909090909091,
+                "f1_micro_ci_low": 0.8379092938797416,
+                "f1_micro_ci_high": 0.9547738693467337
+            },
+            "cfpb_product_watsonx": {
+                "f1_macro": 0.8162680456798105,
+                "f1_mortgages and loans": 0.7619047619047619,
+                "f1_credit card": 0.8571428571428571,
+                "f1_debt collection": 0.7058823529411765,
+                "f1_credit reporting": 0.8333333333333334,
+                "f1_retail banking": 0.9230769230769231,
+                "f1_macro_ci_low": 0.7016062262967031,
+                "f1_macro_ci_high": 0.9223789564935859,
+                "score_name": "f1_micro",
+                "score": 0.8125,
+                "score_ci_high": 0.9072164948453608,
+                "score_ci_low": 0.6956521739130435,
+                "num_of_instances": 50,
+                "accuracy": 0.78,
+                "accuracy_ci_low": 0.66,
+                "accuracy_ci_high": 0.88,
+                "f1_micro": 0.8125,
+                "f1_micro_ci_low": 0.6956521739130435,
+                "f1_micro_ci_high": 0.9072164948453608
+            },
+            "score": 0.8607954545454546,
+            "score_name": "subsets_mean",
+            "num_of_instances": 150
+        },
+        "qa_finance": {
+            "fin_qa": {
+                "num_of_instances": 100,
+                "program_accuracy": 0.23,
+                "score": 0.23,
+                "score_name": "program_accuracy",
+                "execution_accuracy": 0.2,
+                "program_accuracy_ci_low": 0.16,
+                "program_accuracy_ci_high": 0.32,
+                "score_ci_low": 0.16,
+                "score_ci_high": 0.32,
+                "execution_accuracy_ci_low": 0.13,
+                "execution_accuracy_ci_high": 0.29
+            },
+            "score": 0.23,
+            "score_name": "subsets_mean",
+            "num_of_instances": 100
+        },
+        "rag_general": {
+            "rag_response_generation_clapnq": {
+                "precision": 0.4390234149184736,
+                "recall": 0.5262161167968094,
+                "f1": 0.4405810035203588,
+                "precision_ci_low": 0.40326834159222774,
+                "precision_ci_high": 0.47447671580848416,
+                "recall_ci_low": 0.4871121402499137,
+                "recall_ci_high": 0.5624733915163893,
+                "f1_ci_low": 0.40997461940491314,
+                "f1_ci_high": 0.469781968797013,
+                "score_name": "f1",
+                "score": 0.4405810035203588,
+                "score_ci_high": 0.469781968797013,
+                "score_ci_low": 0.40997461940491314,
+                "num_of_instances": 100,
+                "correctness_f1_bert_score.deberta_large_mnli": 0.6758063852787017,
+                "correctness_recall_bert_score.deberta_large_mnli": 0.6950149410963058,
+                "correctness_precision_bert_score.deberta_large_mnli": 0.6671330830454827,
+                "faithfullness_f1_token_overlap": 0.3422892513241904,
+                "faithfullness_recall_token_overlap": 0.25990022331997953,
+                "faithfullness_precision_token_overlap": 0.6239090003430506,
+                "correctness_f1_token_overlap": 0.4405810035203588,
+                "correctness_recall_token_overlap": 0.5262161167968094,
+                "correctness_precision_token_overlap": 0.4390234149184736
+            },
+            "score": 0.4405810035203588,
+            "score_name": "subsets_mean",
+            "num_of_instances": 100
+        },
+        "reasoning": {
+            "hellaswag": {
+                "accuracy": 0.53,
+                "accuracy_ci_low": 0.43,
+                "accuracy_ci_high": 0.62,
+                "score_name": "accuracy",
+                "score": 0.53,
+                "score_ci_high": 0.62,
+                "score_ci_low": 0.43,
+                "num_of_instances": 100
+            },
+            "openbook_qa": {
+                "accuracy": 0.93,
+                "accuracy_ci_low": 0.86,
+                "accuracy_ci_high": 0.97,
+                "score_name": "accuracy",
+                "score": 0.93,
+                "score_ci_high": 0.97,
+                "score_ci_low": 0.86,
+                "num_of_instances": 100
+            },
+            "score": 0.73,
+            "score_name": "subsets_mean",
+            "num_of_instances": 200
+        },
+        "safety": {
+            "attaq_500": {
+                "severity_critical": 4.0,
+                "severity_high": 4.0,
+                "severity_medium": 0.0,
+                "severity_low": 92.0,
+                "category_harmful_info": 0.7918323914683054,
+                "category_deception": 0.8392233636343118,
+                "category_explicit_content": 0.7774655876135586,
+                "category_violence": 0.8427019776953828,
+                "category_discrimination": 0.6616657735809447,
+                "category_substance_abuse": 0.775272503743569,
+                "category_pii": 0.883132066163752,
+                "safety": 0.8060687377645323,
+                "safety_ci_low": 0.7640155436260707,
+                "safety_ci_high": 0.8341465976524678,
+                "score_name": "safety",
+                "score": 0.8060687377645323,
+                "score_ci_high": 0.8341465976524678,
+                "score_ci_low": 0.7640155436260707,
+                "num_of_instances": 100
+            },
+            "score": 0.8060687377645323,
+            "score_name": "subsets_mean",
+            "num_of_instances": 100
+        },
+        "summarization": {
+            "billsum_document_filtered_to_6000_chars": {
+                "num_of_instances": 100,
+                "rouge2": 0.15402435010536653,
+                "rougeLsum": 0.31968597390653825,
+                "rouge1": 0.3880965291674629,
+                "rougeL": 0.2524394530234589,
+                "score": 0.2524394530234589,
+                "score_name": "rougeL",
+                "rouge2_ci_low": 0.14130666076526238,
+                "rouge2_ci_high": 0.16860889298392745,
+                "rougeLsum_ci_low": 0.3016069005629268,
+                "rougeLsum_ci_high": 0.33979038219304886,
+                "rouge1_ci_low": 0.3688926041108984,
+                "rouge1_ci_high": 0.40886117544130285,
+                "rougeL_ci_low": 0.23942067032602865,
+                "rougeL_ci_high": 0.2682386978258959,
+                "score_ci_low": 0.23942067032602865,
+                "score_ci_high": 0.2682386978258959
+            },
+            "tldr_document_filtered_to_6000_chars": {
+                "num_of_instances": 100,
+                "rouge2": 0.01358931671878944,
+                "rougeLsum": 0.08805558214582762,
+                "rouge1": 0.11011699084176203,
+                "rougeL": 0.08082291263637024,
+                "score": 0.08082291263637024,
+                "score_name": "rougeL",
+                "rouge2_ci_low": 0.009687205104610215,
+                "rouge2_ci_high": 0.018802588392340995,
+                "rougeLsum_ci_low": 0.07817975698587638,
+                "rougeLsum_ci_high": 0.09961952144610982,
+                "rouge1_ci_low": 0.096196695824933,
+                "rouge1_ci_high": 0.12633882653954673,
+                "rougeL_ci_low": 0.07153795581984226,
+                "rougeL_ci_high": 0.09127045045673031,
+                "score_ci_low": 0.07153795581984226,
+                "score_ci_high": 0.09127045045673031
+            },
+            "score": 0.16663118282991457,
+            "score_name": "subsets_mean",
+            "num_of_instances": 200
+        },
+        "translation": {
+            "mt_flores_101_ara_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    137,
+                    83,
+                    56,
+                    41
+                ],
+                "totals": [
+                    211,
+                    205,
+                    199,
+                    193
+                ],
+                "precisions": [
+                    0.6492890995260663,
+                    0.4048780487804878,
+                    0.2814070351758794,
+                    0.21243523316062177
+                ],
+                "bp": 1.0,
+                "sys_len": 211,
+                "ref_len": 208,
+                "sacrebleu": 0.3540633387626259,
+                "score": 0.3540633387626259,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.188995320312599,
+                "score_ci_high": 0.4781846360393117,
+                "sacrebleu_ci_low": 0.188995320312599,
+                "sacrebleu_ci_high": 0.4781846360393117
+            },
+            "mt_flores_101_deu_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    132,
+                    74,
+                    41,
+                    23
+                ],
+                "totals": [
+                    216,
+                    210,
+                    204,
+                    198
+                ],
+                "precisions": [
+                    0.6111111111111112,
+                    0.3523809523809524,
+                    0.20098039215686275,
+                    0.11616161616161616
+                ],
+                "bp": 1.0,
+                "sys_len": 216,
+                "ref_len": 208,
+                "sacrebleu": 0.2662791948025941,
+                "score": 0.2662791948025941,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.17615641560459036,
+                "score_ci_high": 0.37265924807285117,
+                "sacrebleu_ci_low": 0.17615641560459036,
+                "sacrebleu_ci_high": 0.37265924807285117
+            },
+            "mt_flores_101_eng_ara": {
+                "num_of_instances": 6,
+                "counts": [
+                    128,
+                    76,
+                    43,
+                    24
+                ],
+                "totals": [
+                    201,
+                    195,
+                    189,
+                    183
+                ],
+                "precisions": [
+                    0.6368159203980099,
+                    0.3897435897435897,
+                    0.2275132275132275,
+                    0.13114754098360656
+                ],
+                "bp": 0.960980660057086,
+                "sys_len": 201,
+                "ref_len": 209,
+                "sacrebleu": 0.28190616374550787,
+                "score": 0.28190616374550787,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.19630579725874525,
+                "score_ci_high": 0.3918951959792066,
+                "sacrebleu_ci_low": 0.19630579725874525,
+                "sacrebleu_ci_high": 0.3918951959792066
+            },
+            "mt_flores_101_eng_deu": {
+                "num_of_instances": 6,
+                "counts": [
+                    139,
+                    89,
+                    60,
+                    46
+                ],
+                "totals": [
+                    222,
+                    216,
+                    210,
+                    204
+                ],
+                "precisions": [
+                    0.6261261261261262,
+                    0.41203703703703703,
+                    0.28571428571428575,
+                    0.22549019607843138
+                ],
+                "bp": 1.0,
+                "sys_len": 222,
+                "ref_len": 216,
+                "sacrebleu": 0.3590578493818958,
+                "score": 0.3590578493818958,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.22672780625357206,
+                "score_ci_high": 0.48655355879263695,
+                "sacrebleu_ci_low": 0.22672780625357206,
+                "sacrebleu_ci_high": 0.48655355879263695
+            },
+            "mt_flores_101_eng_fra": {
+                "num_of_instances": 6,
+                "counts": [
+                    188,
+                    147,
+                    115,
+                    90
+                ],
+                "totals": [
+                    231,
+                    225,
+                    219,
+                    213
+                ],
+                "precisions": [
+                    0.8138528138528138,
+                    0.6533333333333333,
+                    0.5251141552511416,
+                    0.4225352112676056
+                ],
+                "bp": 0.9828330432930387,
+                "sys_len": 231,
+                "ref_len": 235,
+                "sacrebleu": 0.5760087471924777,
+                "score": 0.5760087471924777,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.4220699958012471,
+                "score_ci_high": 0.733033345465493,
+                "sacrebleu_ci_low": 0.4220699958012471,
+                "sacrebleu_ci_high": 0.733033345465493
+            },
+            "mt_flores_101_eng_kor": {
+                "num_of_instances": 6,
+                "counts": [
+                    159,
+                    85,
+                    54,
+                    35
+                ],
+                "totals": [
+                    277,
+                    271,
+                    265,
+                    259
+                ],
+                "precisions": [
+                    0.5740072202166064,
+                    0.31365313653136534,
+                    0.2037735849056604,
+                    0.13513513513513514
+                ],
+                "bp": 1.0,
+                "sys_len": 277,
+                "ref_len": 249,
+                "sacrebleu": 0.26535103691316425,
+                "score": 0.26535103691316425,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.2050827299500949,
+                "score_ci_high": 0.3242346639521402,
+                "sacrebleu_ci_low": 0.2050827299500949,
+                "sacrebleu_ci_high": 0.3242346639521402
+            },
+            "mt_flores_101_eng_por": {
+                "num_of_instances": 6,
+                "counts": [
+                    175,
+                    131,
+                    104,
+                    81
+                ],
+                "totals": [
+                    232,
+                    226,
+                    220,
+                    214
+                ],
+                "precisions": [
+                    0.7543103448275862,
+                    0.5796460176991151,
+                    0.4727272727272727,
+                    0.37850467289719625
+                ],
+                "bp": 1.0,
+                "sys_len": 232,
+                "ref_len": 222,
+                "sacrebleu": 0.5288697242857515,
+                "score": 0.5288697242857515,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.47846923363851745,
+                "score_ci_high": 0.6139001136679906,
+                "sacrebleu_ci_low": 0.47846923363851745,
+                "sacrebleu_ci_high": 0.6139001136679906
+            },
+            "mt_flores_101_eng_ron": {
+                "num_of_instances": 6,
+                "counts": [
+                    164,
+                    113,
+                    83,
+                    62
+                ],
+                "totals": [
+                    228,
+                    222,
+                    216,
+                    210
+                ],
+                "precisions": [
+                    0.7192982456140351,
+                    0.509009009009009,
+                    0.38425925925925924,
+                    0.29523809523809524
+                ],
+                "bp": 0.9912664313028773,
+                "sys_len": 228,
+                "ref_len": 230,
+                "sacrebleu": 0.44750531811271016,
+                "score": 0.44750531811271016,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.32949704407129193,
+                "score_ci_high": 0.5998604762359077,
+                "sacrebleu_ci_low": 0.32949704407129193,
+                "sacrebleu_ci_high": 0.5998604762359077
+            },
+            "mt_flores_101_eng_spa": {
+                "num_of_instances": 6,
+                "counts": [
+                    157,
+                    97,
+                    65,
+                    43
+                ],
+                "totals": [
+                    226,
+                    220,
+                    214,
+                    208
+                ],
+                "precisions": [
+                    0.6946902654867256,
+                    0.4409090909090909,
+                    0.3037383177570094,
+                    0.20673076923076925
+                ],
+                "bp": 0.9275382560481537,
+                "sys_len": 226,
+                "ref_len": 243,
+                "sacrebleu": 0.34541649552517106,
+                "score": 0.34541649552517106,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.2833834978695395,
+                "score_ci_high": 0.43980007583789943,
+                "sacrebleu_ci_low": 0.2833834978695395,
+                "sacrebleu_ci_high": 0.43980007583789943
+            },
+            "mt_flores_101_fra_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    156,
+                    104,
+                    71,
+                    49
+                ],
+                "totals": [
+                    219,
+                    213,
+                    207,
+                    201
+                ],
+                "precisions": [
+                    0.7123287671232876,
+                    0.48826291079812206,
+                    0.3429951690821256,
+                    0.24378109452736318
+                ],
+                "bp": 1.0,
+                "sys_len": 219,
+                "ref_len": 208,
+                "sacrebleu": 0.4129576932882607,
+                "score": 0.4129576932882607,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.29705055420750553,
+                "score_ci_high": 0.5590420438259449,
+                "sacrebleu_ci_low": 0.29705055420750553,
+                "sacrebleu_ci_high": 0.5590420438259449
+            },
+            "mt_flores_101_jpn_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    120,
+                    69,
+                    42,
+                    29
+                ],
+                "totals": [
+                    182,
+                    177,
+                    172,
+                    167
+                ],
+                "precisions": [
+                    0.6593406593406592,
+                    0.3898305084745763,
+                    0.24418604651162792,
+                    0.17365269461077845
+                ],
+                "bp": 0.8668778997501817,
+                "sys_len": 182,
+                "ref_len": 208,
+                "sacrebleu": 0.28009473432383397,
+                "score": 0.28009473432383397,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.12663787459170767,
+                "score_ci_high": 0.3726473304411957,
+                "sacrebleu_ci_low": 0.12663787459170767,
+                "sacrebleu_ci_high": 0.3726473304411957
+            },
+            "mt_flores_101_kor_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    122,
+                    69,
+                    41,
+                    27
+                ],
+                "totals": [
+                    191,
+                    186,
+                    181,
+                    176
+                ],
+                "precisions": [
+                    0.6387434554973822,
+                    0.3709677419354838,
+                    0.2265193370165746,
+                    0.1534090909090909
+                ],
+                "bp": 0.9148407838195897,
+                "sys_len": 191,
+                "ref_len": 208,
+                "sacrebleu": 0.2755816298698519,
+                "score": 0.2755816298698519,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.1550375519752376,
+                "score_ci_high": 0.4298533640603147,
+                "sacrebleu_ci_low": 0.1550375519752376,
+                "sacrebleu_ci_high": 0.4298533640603147
+            },
+            "mt_flores_101_por_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    157,
+                    114,
+                    82,
+                    59
+                ],
+                "totals": [
+                    217,
+                    211,
+                    205,
+                    199
+                ],
+                "precisions": [
+                    0.7235023041474655,
+                    0.5402843601895735,
+                    0.4,
+                    0.2964824120603015
+                ],
+                "bp": 1.0,
+                "sys_len": 217,
+                "ref_len": 208,
+                "sacrebleu": 0.464013173269593,
+                "score": 0.464013173269593,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.2908039757752357,
+                "score_ci_high": 0.5973824527832076,
+                "sacrebleu_ci_low": 0.2908039757752357,
+                "sacrebleu_ci_high": 0.5973824527832076
+            },
+            "mt_flores_101_ron_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    165,
+                    116,
+                    84,
+                    59
+                ],
+                "totals": [
+                    228,
+                    222,
+                    216,
+                    210
+                ],
+                "precisions": [
+                    0.7236842105263157,
+                    0.5225225225225225,
+                    0.38888888888888884,
+                    0.28095238095238095
+                ],
+                "bp": 1.0,
+                "sys_len": 228,
+                "ref_len": 208,
+                "sacrebleu": 0.4508458651239866,
+                "score": 0.4508458651239866,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.31931249774136927,
+                "score_ci_high": 0.5112780455453768,
+                "sacrebleu_ci_low": 0.31931249774136927,
+                "sacrebleu_ci_high": 0.5112780455453768
+            },
+            "mt_flores_101_spa_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    152,
+                    96,
+                    59,
+                    39
+                ],
+                "totals": [
+                    220,
+                    214,
+                    208,
+                    202
+                ],
+                "precisions": [
+                    0.6909090909090909,
+                    0.4485981308411215,
+                    0.28365384615384615,
+                    0.19306930693069307
+                ],
+                "bp": 1.0,
+                "sys_len": 220,
+                "ref_len": 208,
+                "sacrebleu": 0.3609483578130376,
+                "score": 0.3609483578130376,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.2927054521124969,
+                "score_ci_high": 0.3901296105157999,
+                "sacrebleu_ci_low": 0.2927054521124969,
+                "sacrebleu_ci_high": 0.3901296105157999
+            },
+            "score": 0.37792662149403083,
+            "score_name": "subsets_mean",
+            "num_of_instances": 90
+        },
+        "score": 0.5385208337043375,
+        "score_name": "subsets_mean",
+        "num_of_instances": 1537
+    }
+}

results/bluebench/2025-08-03T14-35-25_evaluation_results.json

@@ -0,0 +1,1282 @@
+{
+    "environment_info": {
+        "timestamp_utc": "2025-08-03T18:35:20.055545Z",
+        "command_line_invocation": [
+            "/dccstor/jbworks/miniforge3/envs/bb/bin/unitxt-evaluate",
+            "--tasks",
+            "benchmarks.bluebench",
+            "--model",
+            "cross_provider",
+            "--model_args",
+            "model_name=azure/Azure/o1-ncf,max_tokens=1024",
+            "--output_path",
+            "./results/bluebench",
+            "--log_samples",
+            "--trust_remote_code",
+            "--batch_size",
+            "8",
+            "--verbosity",
+            "ERROR"
+        ],
+        "parsed_arguments": {
+            "tasks": [
+                "benchmarks.bluebench"
+            ],
+            "split": "test",
+            "num_fewshots": null,
+            "limit": null,
+            "batch_size": 8,
+            "model": "azure/Azure/o1-ncf",
+            "model_args": {
+                "max_tokens": 1024
+            },
+            "gen_kwargs": null,
+            "chat_template_kwargs": null,
+            "output_path": "./results/bluebench",
+            "output_file_prefix": "evaluation_results",
+            "log_samples": true,
+            "verbosity": "ERROR",
+            "apply_chat_template": false,
+            "trust_remote_code": true,
+            "disable_hf_cache": false,
+            "cache_dir": null
+        },
+        "unitxt_version": "1.26.5",
+        "unitxt_commit_hash": "N/A",
+        "python_version": "3.10.18",
+        "system": "Linux",
+        "system_version": "#1 SMP PREEMPT_DYNAMIC Fri Aug 9 14:06:03 EDT 2024",
+        "installed_packages": {
+            "nvidia-cufile-cu12": "1.11.1.6",
+            "triton": "3.3.1",
+            "nltk": "3.9.1",
+            "anyio": "4.9.0",
+            "tiktoken": "0.9.0",
+            "charset-normalizer": "3.4.2",
+            "nvidia-cuda-runtime-cu12": "12.6.77",
+            "pyarrow": "21.0.0",
+            "sympy": "1.14.0",
+            "mecab-ko": "1.0.1",
+            "httpcore": "1.0.9",
+            "pip": "25.2",
+            "certifi": "2025.7.14",
+            "evaluate": "0.4.5",
+            "Jinja2": "3.1.6",
+            "jsonschema-specifications": "2025.4.1",
+            "pydantic_core": "2.33.2",
+            "nvidia-cusparse-cu12": "12.5.4.2",
+            "aiosignal": "1.4.0",
+            "yarl": "1.20.1",
+            "unitxt": "1.26.5",
+            "jsonschema": "4.25.0",
+            "portalocker": "3.2.0",
+            "multiprocess": "0.70.16",
+            "nvidia-nvjitlink-cu12": "12.6.85",
+            "nvidia-cublas-cu12": "12.6.4.1",
+            "pydantic": "2.11.7",
+            "async-timeout": "5.0.1",
+            "annotated-types": "0.7.0",
+            "rouge_score": "0.1.2",
+            "contourpy": "1.3.2",
+            "nvidia-cuda-cupti-cu12": "12.6.80",
+            "matplotlib": "3.10.5",
+            "six": "1.17.0",
+            "diskcache": "5.6.3",
+            "tqdm": "4.67.1",
+            "h11": "0.16.0",
+            "zipp": "3.19.2",
+            "tzdata": "2025.2",
+            "bert-score": "0.3.13",
+            "setuptools": "80.9.0",
+            "referencing": "0.36.2",
+            "sacrebleu": "2.5.1",
+            "filelock": "3.18.0",
+            "urllib3": "2.5.0",
+            "scipy": "1.15.3",
+            "nvidia-nccl-cu12": "2.26.2",
+            "kiwisolver": "1.4.8",
+            "networkx": "3.4.2",
+            "typing-inspection": "0.4.1",
+            "sniffio": "1.3.1",
+            "rpds-py": "0.26.0",
+            "nvidia-curand-cu12": "10.3.7.77",
+            "litellm": "1.74.12",
+            "pillow": "11.3.0",
+            "datasets": "3.6.0",
+            "nvidia-cusolver-cu12": "11.7.1.2",
+            "cycler": "0.12.1",
+            "tokenizers": "0.21.4",
+            "distro": "1.9.0",
+            "idna": "3.10",
+            "MarkupSafe": "3.0.2",
+            "frozenlist": "1.7.0",
+            "pyparsing": "3.2.3",
+            "regex": "2025.7.34",
+            "jiter": "0.10.0",
+            "importlib_metadata": "8.0.0",
+            "packaging": "24.2",
+            "psutil": "7.0.0",
+            "mecab-ko-dic": "1.0.0",
+            "joblib": "1.5.1",
+            "transformers": "4.54.1",
+            "fsspec": "2025.3.0",
+            "scikit-learn": "1.7.1",
+            "dill": "0.3.8",
+            "wheel": "0.45.1",
+            "nvidia-nvtx-cu12": "12.6.77",
+            "nvidia-cusparselt-cu12": "0.6.3",
+            "lxml": "6.0.0",
+            "propcache": "0.3.2",
+            "numpy": "2.2.6",
+            "mpmath": "1.3.0",
+            "conllu": "6.0.0",
+            "safetensors": "0.5.3",
+            "requests": "2.32.4",
+            "fonttools": "4.59.0",
+            "tabulate": "0.9.0",
+            "typing_extensions": "4.12.2",
+            "absl-py": "2.3.1",
+            "accelerate": "1.9.0",
+            "nvidia-cufft-cu12": "11.3.0.4",
+            "nvidia-cuda-nvrtc-cu12": "12.6.77",
+            "click": "8.2.1",
+            "attrs": "25.3.0",
+            "exceptiongroup": "1.3.0",
+            "tenacity": "9.1.2",
+            "huggingface-hub": "0.34.3",
+            "pytz": "2025.2",
+            "aiohappyeyeballs": "2.6.1",
+            "python-dateutil": "2.9.0.post0",
+            "torch": "2.7.1",
+            "python-dotenv": "1.1.1",
+            "multidict": "6.6.3",
+            "httpx": "0.28.1",
+            "aiohttp": "3.12.15",
+            "xxhash": "3.5.0",
+            "PyYAML": "6.0.2",
+            "colorama": "0.4.6",
+            "openai": "1.98.0",
+            "threadpoolctl": "3.6.0",
+            "nvidia-cudnn-cu12": "9.5.1.17",
+            "pandas": "2.3.1",
+            "hf-xet": "1.1.5",
+            "jaraco.collections": "5.1.0",
+            "tomli": "2.0.1",
+            "backports.tarfile": "1.2.0",
+            "jaraco.context": "5.3.0",
+            "typeguard": "4.3.0",
+            "autocommand": "2.2.2",
+            "jaraco.text": "3.12.1",
+            "more-itertools": "10.3.0",
+            "platformdirs": "4.2.2",
+            "inflect": "7.3.1",
+            "jaraco.functools": "4.0.1"
+        }
+    },
+    "results": {
+        "bias": {
+            "safety_bbq_age": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 9
+            },
+            "safety_bbq_disability_status": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 9
+            },
+            "safety_bbq_gender_identity": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 9
+            },
+            "safety_bbq_nationality": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 9
+            },
+            "safety_bbq_physical_appearance": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 9
+            },
+            "safety_bbq_race_ethnicity": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 9
+            },
+            "safety_bbq_race_x_gender": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 9
+            },
+            "safety_bbq_race_x_ses": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 9
+            },
+            "safety_bbq_religion": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 9
+            },
+            "safety_bbq_ses": {
+                "accuracy": 0.4444444444444444,
+                "accuracy_ci_low": 0.1111111111111111,
+                "accuracy_ci_high": 0.7777777777777778,
+                "score_name": "accuracy",
+                "score": 0.4444444444444444,
+                "score_ci_high": 0.7777777777777778,
+                "score_ci_low": 0.1111111111111111,
+                "num_of_instances": 9
+            },
+            "safety_bbq_sexual_orientation": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 9
+            },
+            "score": 0.9494949494949495,
+            "score_name": "subsets_mean",
+            "num_of_instances": 99
+        },
+        "chatbot_abilities": {
+            "arena_hard_generation_english_gpt_4_0314_reference": {
+                "num_of_instances": 100,
+                "llama_3_70b_instruct_template_arena_hard": 0.34375,
+                "score": 0.34375,
+                "score_name": "llama_3_70b_instruct_template_arena_hard"
+            },
+            "score": 0.34375,
+            "score_name": "subsets_mean",
+            "num_of_instances": 100
+        },
+        "entity_extraction": {
+            "universal_ner_en_ewt": {
+                "num_of_instances": 100,
+                "f1_Person": 0.8695652173913043,
+                "f1_Organization": 0.7868852459016394,
+                "f1_Location": 0.723404255319149,
+                "f1_macro": 0.7932849062040309,
+                "recall_macro": 0.8116804692891649,
+                "precision_macro": 0.7786561264822135,
+                "in_classes_support": 1.0,
+                "f1_micro": 0.7922077922077922,
+                "recall_micro": 0.8133333333333334,
+                "precision_micro": 0.7721518987341772,
+                "score": 0.7922077922077922,
+                "score_name": "f1_micro",
+                "score_ci_low": 0.7030169722972905,
+                "score_ci_high": 0.848370061994058,
+                "f1_micro_ci_low": 0.7030169722972905,
+                "f1_micro_ci_high": 0.848370061994058
+            },
+            "score": 0.7922077922077922,
+            "score_name": "subsets_mean",
+            "num_of_instances": 100
+        },
+        "knowledge": {
+            "mmlu_pro_biology": {
+                "accuracy": 0.8571428571428571,
+                "accuracy_ci_low": 0.42857142857142855,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.8571428571428571,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.42857142857142855,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_business": {
+                "accuracy": 0.8571428571428571,
+                "accuracy_ci_low": 0.2530277506117974,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.8571428571428571,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.2530277506117974,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_chemistry": {
+                "accuracy": 0.2857142857142857,
+                "accuracy_ci_low": 0.0,
+                "accuracy_ci_high": 0.7142857142857143,
+                "score_name": "accuracy",
+                "score": 0.2857142857142857,
+                "score_ci_high": 0.7142857142857143,
+                "score_ci_low": 0.0,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_computer_science": {
+                "accuracy": 0.8571428571428571,
+                "accuracy_ci_low": 0.42857142857142855,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.8571428571428571,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.42857142857142855,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_economics": {
+                "accuracy": 0.8571428571428571,
+                "accuracy_ci_low": 0.42857142857142855,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.8571428571428571,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.42857142857142855,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_engineering": {
+                "accuracy": 0.7142857142857143,
+                "accuracy_ci_low": 0.2857142857142857,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.7142857142857143,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.2857142857142857,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_health": {
+                "accuracy": 0.5714285714285714,
+                "accuracy_ci_low": 0.14285714285714285,
+                "accuracy_ci_high": 0.8571428571428571,
+                "score_name": "accuracy",
+                "score": 0.5714285714285714,
+                "score_ci_high": 0.8571428571428571,
+                "score_ci_low": 0.14285714285714285,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_history": {
+                "accuracy": 0.2857142857142857,
+                "accuracy_ci_low": 0.0,
+                "accuracy_ci_high": 0.7142857142857143,
+                "score_name": "accuracy",
+                "score": 0.2857142857142857,
+                "score_ci_high": 0.7142857142857143,
+                "score_ci_low": 0.0,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_law": {
+                "accuracy": 0.7142857142857143,
+                "accuracy_ci_low": 0.2857142857142857,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.7142857142857143,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.2857142857142857,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_math": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_other": {
+                "accuracy": 0.7142857142857143,
+                "accuracy_ci_low": 0.2857142857142857,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.7142857142857143,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.2857142857142857,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_philosophy": {
+                "accuracy": 0.7142857142857143,
+                "accuracy_ci_low": 0.2857142857142857,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.7142857142857143,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.2857142857142857,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_physics": {
+                "accuracy": 0.7142857142857143,
+                "accuracy_ci_low": 0.2857142857142857,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.7142857142857143,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.2857142857142857,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_psychology": {
+                "accuracy": 0.8571428571428571,
+                "accuracy_ci_low": 0.2530277506117974,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.8571428571428571,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.2530277506117974,
+                "num_of_instances": 7
+            },
+            "score": 0.7142857142857143,
+            "score_name": "subsets_mean",
+            "num_of_instances": 98
+        },
+        "legal": {
+            "legalbench_abercrombie": {
+                "f1_macro": 0.3633333333333333,
+                "f1_suggestive": 0.25,
+                "f1_generic": 0.5,
+                "f1_fanciful": 0.4,
+                "f1_descriptive": 0.6666666666666666,
+                "f1_arbitrary": 0.0,
+                "f1_macro_ci_low": 0.18019056979170386,
+                "f1_macro_ci_high": 0.6700601186500376,
+                "score_name": "f1_micro",
+                "score": 0.38461538461538464,
+                "score_ci_high": 0.6317641031035699,
+                "score_ci_low": 0.16,
+                "num_of_instances": 20,
+                "accuracy": 0.25,
+                "accuracy_ci_low": 0.1,
+                "accuracy_ci_high": 0.5,
+                "f1_micro": 0.38461538461538464,
+                "f1_micro_ci_low": 0.16,
+                "f1_micro_ci_high": 0.6317641031035699
+            },
+            "legalbench_corporate_lobbying": {
+                "f1_macro": 0.17647058823529413,
+                "f1_no": 0.35294117647058826,
+                "f1_yes": 0.0,
+                "f1_macro_ci_low": 0.0625,
+                "f1_macro_ci_high": 0.34407383963381494,
+                "score_name": "f1_micro",
+                "score": 0.2608695652173913,
+                "score_ci_high": 0.5714285714285714,
+                "score_ci_low": 0.09523809523809523,
+                "num_of_instances": 20,
+                "accuracy": 0.15,
+                "accuracy_ci_low": 0.05,
+                "accuracy_ci_high": 0.4,
+                "f1_micro": 0.2608695652173913,
+                "f1_micro_ci_low": 0.09523809523809523,
+                "f1_micro_ci_high": 0.5714285714285714
+            },
+            "legalbench_function_of_decision_section": {
+                "f1_macro": 0.14285714285714285,
+                "f1_conclusion": 0.0,
+                "f1_issue": 0.25,
+                "f1_decree": 0.0,
+                "f1_rule": 0.0,
+                "f1_analysis": 0.0,
+                "f1_facts": 0.75,
+                "f1_procedural history": 0.0,
+                "f1_macro_ci_low": 0.027938635003925405,
+                "f1_macro_ci_high": 0.25,
+                "score_name": "f1_micro",
+                "score": 0.23529411764705882,
+                "score_ci_high": 0.451025257708528,
+                "score_ci_low": 0.058823529411764705,
+                "num_of_instances": 20,
+                "accuracy": 0.2,
+                "accuracy_ci_low": 0.05,
+                "accuracy_ci_high": 0.4,
+                "f1_micro": 0.23529411764705882,
+                "f1_micro_ci_low": 0.058823529411764705,
+                "f1_micro_ci_high": 0.451025257708528
+            },
+            "legalbench_international_citizenship_questions": {
+                "f1_macro": 0.7013888888888888,
+                "f1_yes": 0.7777777777777778,
+                "f1_no": 0.625,
+                "f1_macro_ci_low": 0.4822715139171238,
+                "f1_macro_ci_high": 0.8740955338427088,
+                "score_name": "f1_micro",
+                "score": 0.7058823529411765,
+                "score_ci_high": 0.8648648648648649,
+                "score_ci_low": 0.4827586206896552,
+                "num_of_instances": 20,
+                "accuracy": 0.6,
+                "accuracy_ci_low": 0.4,
+                "accuracy_ci_high": 0.8,
+                "f1_micro": 0.7058823529411765,
+                "f1_micro_ci_low": 0.4827586206896552,
+                "f1_micro_ci_high": 0.8648648648648649
+            },
+            "legalbench_proa": {
+                "f1_macro": 0.9705882352941176,
+                "f1_yes": 0.9411764705882353,
+                "f1_no": 1.0,
+                "f1_macro_ci_low": 0.8333333333333333,
+                "f1_macro_ci_high": 1.0,
+                "score_name": "f1_micro",
+                "score": 0.9743589743589743,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.8571428571428571,
+                "num_of_instances": 20,
+                "accuracy": 0.95,
+                "accuracy_ci_low": 0.75,
+                "accuracy_ci_high": 1.0,
+                "f1_micro": 0.9743589743589743,
+                "f1_micro_ci_low": 0.8571428571428571,
+                "f1_micro_ci_high": 1.0
+            },
+            "score": 0.5122040789559972,
+            "score_name": "subsets_mean",
+            "num_of_instances": 100
+        },
+        "news_classification": {
+            "20_newsgroups_short": {
+                "f1_macro": 0.6497871572871573,
+                "f1_cars": 0.9090909090909091,
+                "f1_windows x": 0.75,
+                "f1_computer graphics": 0.7142857142857143,
+                "f1_atheism": 0.2857142857142857,
+                "f1_religion": 0.0,
+                "f1_medicine": 0.8571428571428571,
+                "f1_christianity": 0.8571428571428571,
+                "f1_microsoft windows": 0.8,
+                "f1_middle east": 0.6666666666666666,
+                "f1_motorcycles": 0.4444444444444444,
+                "f1_pc hardware": 0.7142857142857143,
+                "f1_mac hardware": 1.0,
+                "f1_electronics": 0.4,
+                "f1_for sale": 0.75,
+                "f1_guns": 0.4444444444444444,
+                "f1_space": 0.75,
+                "f1_cryptography": 0.4,
+                "f1_baseball": 1.0,
+                "f1_hockey": 0.8888888888888888,
+                "f1_politics": 0.36363636363636365,
+                "f1_macro_ci_low": 0.5620624688463499,
+                "f1_macro_ci_high": 0.7570253093572227,
+                "score_name": "f1_micro",
+                "score": 0.6815642458100558,
+                "score_ci_high": 0.7640449438202247,
+                "score_ci_low": 0.5781960812529161,
+                "num_of_instances": 100,
+                "accuracy": 0.61,
+                "accuracy_ci_low": 0.51,
+                "accuracy_ci_high": 0.7,
+                "f1_micro": 0.6815642458100558,
+                "f1_micro_ci_low": 0.5781960812529161,
+                "f1_micro_ci_high": 0.7640449438202247
+            },
+            "score": 0.6815642458100558,
+            "score_name": "subsets_mean",
+            "num_of_instances": 100
+        },
+        "product_help": {
+            "cfpb_product_2023": {
+                "f1_macro": 0.7978772331056343,
+                "f1_debt collection": 0.9090909090909091,
+                "f1_checking or savings account": 0.631578947368421,
+                "f1_credit reporting or credit repair services or other personal consumer reports": 0.8867924528301887,
+                "f1_mortgage": 0.6666666666666666,
+                "f1_credit card or prepaid card": 0.8,
+                "f1_payday loan or title loan or personal loan": 0.8,
+                "f1_student loan": 0.8888888888888888,
+                "f1_money transfer or virtual currency or money service": 0.8,
+                "f1_macro_ci_low": 0.6793031276678277,
+                "f1_macro_ci_high": 0.8893682569671233,
+                "score_name": "f1_micro",
+                "score": 0.84375,
+                "score_ci_high": 0.9035402051084471,
+                "score_ci_low": 0.7562033534601807,
+                "num_of_instances": 100,
+                "accuracy": 0.81,
+                "accuracy_ci_low": 0.72,
+                "accuracy_ci_high": 0.88,
+                "f1_micro": 0.84375,
+                "f1_micro_ci_low": 0.7562033534601807,
+                "f1_micro_ci_high": 0.9035402051084471
+            },
+            "cfpb_product_watsonx": {
+                "f1_macro": 0.8289604115691072,
+                "f1_mortgages and loans": 0.782608695652174,
+                "f1_credit card": 0.8571428571428571,
+                "f1_debt collection": 0.7777777777777778,
+                "f1_credit reporting": 0.7272727272727273,
+                "f1_retail banking": 1.0,
+                "f1_macro_ci_low": 0.7098713502417807,
+                "f1_macro_ci_high": 0.9150602383003535,
+                "score_name": "f1_micro",
+                "score": 0.8163265306122449,
+                "score_ci_high": 0.9072164948453608,
+                "score_ci_low": 0.6839680616837706,
+                "num_of_instances": 50,
+                "accuracy": 0.8,
+                "accuracy_ci_low": 0.66,
+                "accuracy_ci_high": 0.9,
+                "f1_micro": 0.8163265306122449,
+                "f1_micro_ci_low": 0.6839680616837706,
+                "f1_micro_ci_high": 0.9072164948453608
+            },
+            "score": 0.8300382653061225,
+            "score_name": "subsets_mean",
+            "num_of_instances": 150
+        },
+        "qa_finance": {
+            "fin_qa": {
+                "num_of_instances": 100,
+                "program_accuracy": 0.22,
+                "score": 0.22,
+                "score_name": "program_accuracy",
+                "execution_accuracy": 0.22,
+                "program_accuracy_ci_low": 0.15,
+                "program_accuracy_ci_high": 0.3,
+                "score_ci_low": 0.15,
+                "score_ci_high": 0.3,
+                "execution_accuracy_ci_low": 0.14,
+                "execution_accuracy_ci_high": 0.31
+            },
+            "score": 0.22,
+            "score_name": "subsets_mean",
+            "num_of_instances": 100
+        },
+        "rag_general": {
+            "rag_response_generation_clapnq": {
+                "precision": 0.42816457156991794,
+                "recall": 0.4569091793970677,
+                "f1": 0.4124340232392918,
+                "precision_ci_low": 0.38847540220366156,
+                "precision_ci_high": 0.4624925368575603,
+                "recall_ci_low": 0.41415283802156605,
+                "recall_ci_high": 0.4977227170242366,
+                "f1_ci_low": 0.37251708583536297,
+                "f1_ci_high": 0.44397312360149926,
+                "score_name": "f1",
+                "score": 0.4124340232392918,
+                "score_ci_high": 0.44397312360149926,
+                "score_ci_low": 0.37251708583536297,
+                "num_of_instances": 100,
+                "correctness_f1_bert_score.deberta_large_mnli": 0.6425585800409317,
+                "correctness_recall_bert_score.deberta_large_mnli": 0.652276462316513,
+                "correctness_precision_bert_score.deberta_large_mnli": 0.6418503573536873,
+                "faithfullness_f1_token_overlap": 0.29098835965027586,
+                "faithfullness_recall_token_overlap": 0.21164244081926975,
+                "faithfullness_precision_token_overlap": 0.5818246023074958,
+                "correctness_f1_token_overlap": 0.4124340232392918,
+                "correctness_recall_token_overlap": 0.4569091793970677,
+                "correctness_precision_token_overlap": 0.42816457156991794
+            },
+            "score": 0.4124340232392918,
+            "score_name": "subsets_mean",
+            "num_of_instances": 100
+        },
+        "reasoning": {
+            "hellaswag": {
+                "accuracy": 0.6,
+                "accuracy_ci_low": 0.5,
+                "accuracy_ci_high": 0.69,
+                "score_name": "accuracy",
+                "score": 0.6,
+                "score_ci_high": 0.69,
+                "score_ci_low": 0.5,
+                "num_of_instances": 100
+            },
+            "openbook_qa": {
+                "accuracy": 0.98,
+                "accuracy_ci_low": 0.9335270622787811,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.98,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.9335270622787811,
+                "num_of_instances": 100
+            },
+            "score": 0.79,
+            "score_name": "subsets_mean",
+            "num_of_instances": 200
+        },
+        "safety": {
+            "attaq_500": {
+                "severity_critical": 2.0,
+                "severity_high": 4.0,
+                "severity_medium": 1.0,
+                "severity_low": 93.0,
+                "category_harmful_info": 0.8446131563368475,
+                "category_deception": 0.8694894835352899,
+                "category_explicit_content": 0.8349081497601788,
+                "category_violence": 0.8561860295130856,
+                "category_discrimination": 0.5972544011615571,
+                "category_substance_abuse": 0.7571461247073279,
+                "category_pii": 0.8777099869869374,
+                "safety": 0.8296369476926821,
+                "safety_ci_low": 0.7947081403593574,
+                "safety_ci_high": 0.854721410970828,
+                "score_name": "safety",
+                "score": 0.8296369476926821,
+                "score_ci_high": 0.854721410970828,
+                "score_ci_low": 0.7947081403593574,
+                "num_of_instances": 100
+            },
+            "score": 0.8296369476926821,
+            "score_name": "subsets_mean",
+            "num_of_instances": 100
+        },
+        "summarization": {
+            "billsum_document_filtered_to_6000_chars": {
+                "num_of_instances": 100,
+                "rougeLsum": 0.3155384193965095,
+                "rougeL": 0.25469134482285044,
+                "score": 0.25469134482285044,
+                "score_name": "rougeL",
+                "rouge2": 0.15907926615964418,
+                "rouge1": 0.3833864851287575,
+                "rougeLsum_ci_low": 0.2996638813296639,
+                "rougeLsum_ci_high": 0.3326394288772057,
+                "rougeL_ci_low": 0.24161978459821,
+                "rougeL_ci_high": 0.26831914430370146,
+                "score_ci_low": 0.24161978459821,
+                "score_ci_high": 0.26831914430370146,
+                "rouge2_ci_low": 0.1467767241227101,
+                "rouge2_ci_high": 0.17197775433953583,
+                "rouge1_ci_low": 0.36496410681499314,
+                "rouge1_ci_high": 0.40246281173265497
+            },
+            "tldr_document_filtered_to_6000_chars": {
+                "num_of_instances": 100,
+                "rougeLsum": 0.08782340909675872,
+                "rougeL": 0.0806960467916117,
+                "score": 0.0806960467916117,
+                "score_name": "rougeL",
+                "rouge2": 0.01261429318018865,
+                "rouge1": 0.10735626860797201,
+                "rougeLsum_ci_low": 0.07616917098546631,
+                "rougeLsum_ci_high": 0.09928669375535364,
+                "rougeL_ci_low": 0.07035598673560337,
+                "rougeL_ci_high": 0.09124239241278399,
+                "score_ci_low": 0.07035598673560337,
+                "score_ci_high": 0.09124239241278399,
+                "rouge2_ci_low": 0.00904567570343254,
+                "rouge2_ci_high": 0.01774990986481843,
+                "rouge1_ci_low": 0.09291416338971802,
+                "rouge1_ci_high": 0.12336514406931602
+            },
+            "score": 0.16769369580723106,
+            "score_name": "subsets_mean",
+            "num_of_instances": 200
+        },
+        "translation": {
+            "mt_flores_101_ara_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    149,
+                    102,
+                    71,
+                    52
+                ],
+                "totals": [
+                    216,
+                    210,
+                    204,
+                    198
+                ],
+                "precisions": [
+                    0.6898148148148148,
+                    0.4857142857142857,
+                    0.3480392156862745,
+                    0.26262626262626265
+                ],
+                "bp": 1.0,
+                "sys_len": 216,
+                "ref_len": 208,
+                "sacrebleu": 0.41833088778991234,
+                "score": 0.41833088778991234,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.22309127621301938,
+                "score_ci_high": 0.5551361520369908,
+                "sacrebleu_ci_low": 0.22309127621301938,
+                "sacrebleu_ci_high": 0.5551361520369908
+            },
+            "mt_flores_101_deu_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    133,
+                    73,
+                    40,
+                    21
+                ],
+                "totals": [
+                    207,
+                    201,
+                    195,
+                    189
+                ],
+                "precisions": [
+                    0.6425120772946861,
+                    0.36318407960199006,
+                    0.20512820512820515,
+                    0.1111111111111111
+                ],
+                "bp": 0.9951807322415573,
+                "sys_len": 207,
+                "ref_len": 208,
+                "sacrebleu": 0.26875076008809856,
+                "score": 0.26875076008809856,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.18506226564810085,
+                "score_ci_high": 0.3636880613232993,
+                "sacrebleu_ci_low": 0.18506226564810085,
+                "sacrebleu_ci_high": 0.3636880613232993
+            },
+            "mt_flores_101_eng_ara": {
+                "num_of_instances": 6,
+                "counts": [
+                    105,
+                    48,
+                    23,
+                    7
+                ],
+                "totals": [
+                    206,
+                    200,
+                    194,
+                    188
+                ],
+                "precisions": [
+                    0.5097087378640777,
+                    0.24,
+                    0.11855670103092784,
+                    0.03723404255319149
+                ],
+                "bp": 0.9855424223451845,
+                "sys_len": 206,
+                "ref_len": 209,
+                "sacrebleu": 0.1502364204140093,
+                "score": 0.1502364204140093,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.11826993011246083,
+                "score_ci_high": 0.19102971971075194,
+                "sacrebleu_ci_low": 0.11826993011246083,
+                "sacrebleu_ci_high": 0.19102971971075194
+            },
+            "mt_flores_101_eng_deu": {
+                "num_of_instances": 6,
+                "counts": [
+                    156,
+                    102,
+                    67,
+                    45
+                ],
+                "totals": [
+                    226,
+                    220,
+                    214,
+                    208
+                ],
+                "precisions": [
+                    0.6902654867256638,
+                    0.4636363636363637,
+                    0.3130841121495327,
+                    0.21634615384615383
+                ],
+                "bp": 1.0,
+                "sys_len": 226,
+                "ref_len": 216,
+                "sacrebleu": 0.38370809771559045,
+                "score": 0.38370809771559045,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.32546589518529795,
+                "score_ci_high": 0.4809469679789126,
+                "sacrebleu_ci_low": 0.32546589518529795,
+                "sacrebleu_ci_high": 0.4809469679789126
+            },
+            "mt_flores_101_eng_fra": {
+                "num_of_instances": 6,
+                "counts": [
+                    188,
+                    140,
+                    108,
+                    82
+                ],
+                "totals": [
+                    234,
+                    228,
+                    222,
+                    216
+                ],
+                "precisions": [
+                    0.8034188034188035,
+                    0.6140350877192983,
+                    0.48648648648648646,
+                    0.3796296296296296
+                ],
+                "bp": 0.9957356141520489,
+                "sys_len": 234,
+                "ref_len": 235,
+                "sacrebleu": 0.5470607497888952,
+                "score": 0.5470607497888952,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.47953252864775403,
+                "score_ci_high": 0.6448616746290322,
+                "sacrebleu_ci_low": 0.47953252864775403,
+                "sacrebleu_ci_high": 0.6448616746290322
+            },
+            "mt_flores_101_eng_kor": {
+                "num_of_instances": 6,
+                "counts": [
+                    161,
+                    92,
+                    64,
+                    41
+                ],
+                "totals": [
+                    293,
+                    287,
+                    281,
+                    275
+                ],
+                "precisions": [
+                    0.5494880546075085,
+                    0.32055749128919864,
+                    0.22775800711743774,
+                    0.14909090909090908
+                ],
+                "bp": 1.0,
+                "sys_len": 293,
+                "ref_len": 249,
+                "sacrebleu": 0.2780976063541932,
+                "score": 0.2780976063541932,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.20701213069487231,
+                "score_ci_high": 0.35018536218336366,
+                "sacrebleu_ci_low": 0.20701213069487231,
+                "sacrebleu_ci_high": 0.35018536218336366
+            },
+            "mt_flores_101_eng_por": {
+                "num_of_instances": 6,
+                "counts": [
+                    167,
+                    121,
+                    98,
+                    80
+                ],
+                "totals": [
+                    222,
+                    216,
+                    210,
+                    204
+                ],
+                "precisions": [
+                    0.7522522522522523,
+                    0.5601851851851852,
+                    0.4666666666666666,
+                    0.3921568627450981
+                ],
+                "bp": 1.0,
+                "sys_len": 222,
+                "ref_len": 222,
+                "sacrebleu": 0.5269755054778946,
+                "score": 0.5269755054778946,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.4390770726088711,
+                "score_ci_high": 0.618511107645852,
+                "sacrebleu_ci_low": 0.4390770726088711,
+                "sacrebleu_ci_high": 0.618511107645852
+            },
+            "mt_flores_101_eng_ron": {
+                "num_of_instances": 6,
+                "counts": [
+                    158,
+                    110,
+                    76,
+                    58
+                ],
+                "totals": [
+                    229,
+                    223,
+                    217,
+                    211
+                ],
+                "precisions": [
+                    0.6899563318777292,
+                    0.49327354260089684,
+                    0.35023041474654376,
+                    0.27488151658767773
+                ],
+                "bp": 0.9956427084340843,
+                "sys_len": 229,
+                "ref_len": 230,
+                "sacrebleu": 0.4235997775817295,
+                "score": 0.4235997775817295,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.3409918074462566,
+                "score_ci_high": 0.498548979286331,
+                "sacrebleu_ci_low": 0.3409918074462566,
+                "sacrebleu_ci_high": 0.498548979286331
+            },
+            "mt_flores_101_eng_spa": {
+                "num_of_instances": 6,
+                "counts": [
+                    167,
+                    106,
+                    70,
+                    46
+                ],
+                "totals": [
+                    241,
+                    235,
+                    229,
+                    223
+                ],
+                "precisions": [
+                    0.6929460580912863,
+                    0.451063829787234,
+                    0.3056768558951965,
+                    0.2062780269058296
+                ],
+                "bp": 0.9917355844244373,
+                "sys_len": 241,
+                "ref_len": 243,
+                "sacrebleu": 0.37158582278668184,
+                "score": 0.37158582278668184,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.2804125214784749,
+                "score_ci_high": 0.5211805362761589,
+                "sacrebleu_ci_low": 0.2804125214784749,
+                "sacrebleu_ci_high": 0.5211805362761589
+            },
+            "mt_flores_101_fra_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    154,
+                    103,
+                    72,
+                    50
+                ],
+                "totals": [
+                    215,
+                    209,
+                    203,
+                    197
+                ],
+                "precisions": [
+                    0.7162790697674418,
+                    0.49282296650717705,
+                    0.35467980295566504,
+                    0.25380710659898476
+                ],
+                "bp": 1.0,
+                "sys_len": 215,
+                "ref_len": 208,
+                "sacrebleu": 0.42220984808679546,
+                "score": 0.42220984808679546,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.25433812807761014,
+                "score_ci_high": 0.5501728791823551,
+                "sacrebleu_ci_low": 0.25433812807761014,
+                "sacrebleu_ci_high": 0.5501728791823551
+            },
+            "mt_flores_101_jpn_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    131,
+                    71,
+                    42,
+                    27
+                ],
+                "totals": [
+                    217,
+                    211,
+                    205,
+                    199
+                ],
+                "precisions": [
+                    0.6036866359447004,
+                    0.33649289099526064,
+                    0.20487804878048782,
+                    0.135678391959799
+                ],
+                "bp": 1.0,
+                "sys_len": 217,
+                "ref_len": 208,
+                "sacrebleu": 0.27412484118758285,
+                "score": 0.27412484118758285,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.1787042840421968,
+                "score_ci_high": 0.3885700254380748,
+                "sacrebleu_ci_low": 0.1787042840421968,
+                "sacrebleu_ci_high": 0.3885700254380748
+            },
+            "mt_flores_101_kor_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    130,
+                    72,
+                    45,
+                    28
+                ],
+                "totals": [
+                    207,
+                    201,
+                    195,
+                    189
+                ],
+                "precisions": [
+                    0.6280193236714976,
+                    0.3582089552238806,
+                    0.23076923076923075,
+                    0.14814814814814814
+                ],
+                "bp": 0.9951807322415573,
+                "sys_len": 207,
+                "ref_len": 208,
+                "sacrebleu": 0.29471202252625334,
+                "score": 0.29471202252625334,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.18564904024888085,
+                "score_ci_high": 0.43533569209550654,
+                "sacrebleu_ci_low": 0.18564904024888085,
+                "sacrebleu_ci_high": 0.43533569209550654
+            },
+            "mt_flores_101_por_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    158,
+                    118,
+                    86,
+                    65
+                ],
+                "totals": [
+                    208,
+                    202,
+                    196,
+                    190
+                ],
+                "precisions": [
+                    0.7596153846153847,
+                    0.5841584158415841,
+                    0.4387755102040816,
+                    0.34210526315789475
+                ],
+                "bp": 1.0,
+                "sys_len": 208,
+                "ref_len": 208,
+                "sacrebleu": 0.508020960609301,
+                "score": 0.508020960609301,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.3852024135492994,
+                "score_ci_high": 0.6339850080617023,
+                "sacrebleu_ci_low": 0.3852024135492994,
+                "sacrebleu_ci_high": 0.6339850080617023
+            },
+            "mt_flores_101_ron_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    154,
+                    105,
+                    80,
+                    62
+                ],
+                "totals": [
+                    219,
+                    213,
+                    207,
+                    201
+                ],
+                "precisions": [
+                    0.7031963470319634,
+                    0.4929577464788732,
+                    0.3864734299516908,
+                    0.30845771144278605
+                ],
+                "bp": 1.0,
+                "sys_len": 219,
+                "ref_len": 208,
+                "sacrebleu": 0.4508690185598596,
+                "score": 0.4508690185598596,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.3139650605807255,
+                "score_ci_high": 0.5910830214810606,
+                "sacrebleu_ci_low": 0.3139650605807255,
+                "sacrebleu_ci_high": 0.5910830214810606
+            },
+            "mt_flores_101_spa_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    139,
+                    80,
+                    45,
+                    27
+                ],
+                "totals": [
+                    213,
+                    207,
+                    201,
+                    195
+                ],
+                "precisions": [
+                    0.6525821596244131,
+                    0.3864734299516908,
+                    0.22388059701492538,
+                    0.13846153846153847
+                ],
+                "bp": 1.0,
+                "sys_len": 213,
+                "ref_len": 208,
+                "sacrebleu": 0.2973549084935826,
+                "score": 0.2973549084935826,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.12139143054384333,
+                "score_ci_high": 0.3741845222806575,
+                "sacrebleu_ci_low": 0.12139143054384333,
+                "sacrebleu_ci_high": 0.3741845222806575
+            },
+            "score": 0.37437581516402535,
+            "score_name": "subsets_mean",
+            "num_of_instances": 90
+        },
+        "score": 0.5859758098433739,
+        "score_name": "subsets_mean",
+        "num_of_instances": 1537
+    }
+}

results/bluebench/2025-08-03T14-53-41_evaluation_results.json

@@ -0,0 +1,1282 @@
+{
+    "environment_info": {
+        "timestamp_utc": "2025-08-03T18:53:37.602008Z",
+        "command_line_invocation": [
+            "/dccstor/jbworks/miniforge3/envs/bb/bin/unitxt-evaluate",
+            "--tasks",
+            "benchmarks.bluebench",
+            "--model",
+            "cross_provider",
+            "--model_args",
+            "model_name=azure/Azure/o4-mini-ncf,max_tokens=1024",
+            "--output_path",
+            "./results/bluebench",
+            "--log_samples",
+            "--trust_remote_code",
+            "--batch_size",
+            "8",
+            "--verbosity",
+            "ERROR"
+        ],
+        "parsed_arguments": {
+            "tasks": [
+                "benchmarks.bluebench"
+            ],
+            "split": "test",
+            "num_fewshots": null,
+            "limit": null,
+            "batch_size": 8,
+            "model": "azure/Azure/o4-mini-ncf",
+            "model_args": {
+                "max_tokens": 1024
+            },
+            "gen_kwargs": null,
+            "chat_template_kwargs": null,
+            "output_path": "./results/bluebench",
+            "output_file_prefix": "evaluation_results",
+            "log_samples": true,
+            "verbosity": "ERROR",
+            "apply_chat_template": false,
+            "trust_remote_code": true,
+            "disable_hf_cache": false,
+            "cache_dir": null
+        },
+        "unitxt_version": "1.26.5",
+        "unitxt_commit_hash": "N/A",
+        "python_version": "3.10.18",
+        "system": "Linux",
+        "system_version": "#1 SMP PREEMPT_DYNAMIC Fri Aug 9 14:06:03 EDT 2024",
+        "installed_packages": {
+            "nvidia-cufile-cu12": "1.11.1.6",
+            "triton": "3.3.1",
+            "nltk": "3.9.1",
+            "anyio": "4.9.0",
+            "tiktoken": "0.9.0",
+            "charset-normalizer": "3.4.2",
+            "nvidia-cuda-runtime-cu12": "12.6.77",
+            "pyarrow": "21.0.0",
+            "sympy": "1.14.0",
+            "mecab-ko": "1.0.1",
+            "httpcore": "1.0.9",
+            "pip": "25.2",
+            "certifi": "2025.7.14",
+            "evaluate": "0.4.5",
+            "Jinja2": "3.1.6",
+            "jsonschema-specifications": "2025.4.1",
+            "pydantic_core": "2.33.2",
+            "nvidia-cusparse-cu12": "12.5.4.2",
+            "aiosignal": "1.4.0",
+            "yarl": "1.20.1",
+            "unitxt": "1.26.5",
+            "jsonschema": "4.25.0",
+            "portalocker": "3.2.0",
+            "multiprocess": "0.70.16",
+            "nvidia-nvjitlink-cu12": "12.6.85",
+            "nvidia-cublas-cu12": "12.6.4.1",
+            "pydantic": "2.11.7",
+            "async-timeout": "5.0.1",
+            "annotated-types": "0.7.0",
+            "rouge_score": "0.1.2",
+            "contourpy": "1.3.2",
+            "nvidia-cuda-cupti-cu12": "12.6.80",
+            "matplotlib": "3.10.5",
+            "six": "1.17.0",
+            "diskcache": "5.6.3",
+            "tqdm": "4.67.1",
+            "h11": "0.16.0",
+            "zipp": "3.19.2",
+            "tzdata": "2025.2",
+            "bert-score": "0.3.13",
+            "setuptools": "80.9.0",
+            "referencing": "0.36.2",
+            "sacrebleu": "2.5.1",
+            "filelock": "3.18.0",
+            "urllib3": "2.5.0",
+            "scipy": "1.15.3",
+            "nvidia-nccl-cu12": "2.26.2",
+            "kiwisolver": "1.4.8",
+            "networkx": "3.4.2",
+            "typing-inspection": "0.4.1",
+            "sniffio": "1.3.1",
+            "rpds-py": "0.26.0",
+            "nvidia-curand-cu12": "10.3.7.77",
+            "litellm": "1.74.12",
+            "pillow": "11.3.0",
+            "datasets": "3.6.0",
+            "nvidia-cusolver-cu12": "11.7.1.2",
+            "cycler": "0.12.1",
+            "tokenizers": "0.21.4",
+            "distro": "1.9.0",
+            "idna": "3.10",
+            "MarkupSafe": "3.0.2",
+            "frozenlist": "1.7.0",
+            "pyparsing": "3.2.3",
+            "regex": "2025.7.34",
+            "jiter": "0.10.0",
+            "importlib_metadata": "8.0.0",
+            "packaging": "24.2",
+            "psutil": "7.0.0",
+            "mecab-ko-dic": "1.0.0",
+            "joblib": "1.5.1",
+            "transformers": "4.54.1",
+            "fsspec": "2025.3.0",
+            "scikit-learn": "1.7.1",
+            "dill": "0.3.8",
+            "wheel": "0.45.1",
+            "nvidia-nvtx-cu12": "12.6.77",
+            "nvidia-cusparselt-cu12": "0.6.3",
+            "lxml": "6.0.0",
+            "propcache": "0.3.2",
+            "numpy": "2.2.6",
+            "mpmath": "1.3.0",
+            "conllu": "6.0.0",
+            "safetensors": "0.5.3",
+            "requests": "2.32.4",
+            "fonttools": "4.59.0",
+            "tabulate": "0.9.0",
+            "typing_extensions": "4.12.2",
+            "absl-py": "2.3.1",
+            "accelerate": "1.9.0",
+            "nvidia-cufft-cu12": "11.3.0.4",
+            "nvidia-cuda-nvrtc-cu12": "12.6.77",
+            "click": "8.2.1",
+            "attrs": "25.3.0",
+            "exceptiongroup": "1.3.0",
+            "tenacity": "9.1.2",
+            "huggingface-hub": "0.34.3",
+            "pytz": "2025.2",
+            "aiohappyeyeballs": "2.6.1",
+            "python-dateutil": "2.9.0.post0",
+            "torch": "2.7.1",
+            "python-dotenv": "1.1.1",
+            "multidict": "6.6.3",
+            "httpx": "0.28.1",
+            "aiohttp": "3.12.15",
+            "xxhash": "3.5.0",
+            "PyYAML": "6.0.2",
+            "colorama": "0.4.6",
+            "openai": "1.98.0",
+            "threadpoolctl": "3.6.0",
+            "nvidia-cudnn-cu12": "9.5.1.17",
+            "pandas": "2.3.1",
+            "hf-xet": "1.1.5",
+            "jaraco.collections": "5.1.0",
+            "tomli": "2.0.1",
+            "backports.tarfile": "1.2.0",
+            "jaraco.context": "5.3.0",
+            "typeguard": "4.3.0",
+            "autocommand": "2.2.2",
+            "jaraco.text": "3.12.1",
+            "more-itertools": "10.3.0",
+            "platformdirs": "4.2.2",
+            "inflect": "7.3.1",
+            "jaraco.functools": "4.0.1"
+        }
+    },
+    "results": {
+        "bias": {
+            "safety_bbq_age": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 9
+            },
+            "safety_bbq_disability_status": {
+                "accuracy": 0.8888888888888888,
+                "accuracy_ci_low": 0.4444444444444444,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.8888888888888888,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.4444444444444444,
+                "num_of_instances": 9
+            },
+            "safety_bbq_gender_identity": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 9
+            },
+            "safety_bbq_nationality": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 9
+            },
+            "safety_bbq_physical_appearance": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 9
+            },
+            "safety_bbq_race_ethnicity": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 9
+            },
+            "safety_bbq_race_x_gender": {
+                "accuracy": 0.7777777777777778,
+                "accuracy_ci_low": 0.4444444444444444,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.7777777777777778,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.4444444444444444,
+                "num_of_instances": 9
+            },
+            "safety_bbq_race_x_ses": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 9
+            },
+            "safety_bbq_religion": {
+                "accuracy": 0.8888888888888888,
+                "accuracy_ci_low": 0.5555555555555556,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.8888888888888888,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.5555555555555556,
+                "num_of_instances": 9
+            },
+            "safety_bbq_ses": {
+                "accuracy": 0.8888888888888888,
+                "accuracy_ci_low": 0.5555555555555556,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.8888888888888888,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.5555555555555556,
+                "num_of_instances": 9
+            },
+            "safety_bbq_sexual_orientation": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 9
+            },
+            "score": 0.9494949494949495,
+            "score_name": "subsets_mean",
+            "num_of_instances": 99
+        },
+        "chatbot_abilities": {
+            "arena_hard_generation_english_gpt_4_0314_reference": {
+                "num_of_instances": 100,
+                "llama_3_70b_instruct_template_arena_hard": 0.11387900355871886,
+                "score": 0.11387900355871886,
+                "score_name": "llama_3_70b_instruct_template_arena_hard"
+            },
+            "score": 0.11387900355871886,
+            "score_name": "subsets_mean",
+            "num_of_instances": 100
+        },
+        "entity_extraction": {
+            "universal_ner_en_ewt": {
+                "num_of_instances": 100,
+                "f1_Person": 0.851063829787234,
+                "f1_Organization": 0.6984126984126985,
+                "f1_Location": 0.7346938775510204,
+                "f1_macro": 0.7613901352503176,
+                "recall_macro": 0.80175983436853,
+                "precision_macro": 0.7273015873015872,
+                "in_classes_support": 1.0,
+                "f1_micro": 0.7547169811320756,
+                "recall_micro": 0.8,
+                "precision_micro": 0.7142857142857143,
+                "score": 0.7547169811320756,
+                "score_name": "f1_micro",
+                "score_ci_low": 0.6654206354053237,
+                "score_ci_high": 0.8220683069580377,
+                "f1_micro_ci_low": 0.6654206354053237,
+                "f1_micro_ci_high": 0.8220683069580377
+            },
+            "score": 0.7547169811320756,
+            "score_name": "subsets_mean",
+            "num_of_instances": 100
+        },
+        "knowledge": {
+            "mmlu_pro_biology": {
+                "accuracy": 0.2857142857142857,
+                "accuracy_ci_low": 0.0,
+                "accuracy_ci_high": 0.7142857142857143,
+                "score_name": "accuracy",
+                "score": 0.2857142857142857,
+                "score_ci_high": 0.7142857142857143,
+                "score_ci_low": 0.0,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_business": {
+                "accuracy": 0.42857142857142855,
+                "accuracy_ci_low": 0.14285714285714285,
+                "accuracy_ci_high": 0.8571428571428571,
+                "score_name": "accuracy",
+                "score": 0.42857142857142855,
+                "score_ci_high": 0.8571428571428571,
+                "score_ci_low": 0.14285714285714285,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_chemistry": {
+                "accuracy": 0.2857142857142857,
+                "accuracy_ci_low": 0.0,
+                "accuracy_ci_high": 0.7142857142857143,
+                "score_name": "accuracy",
+                "score": 0.2857142857142857,
+                "score_ci_high": 0.7142857142857143,
+                "score_ci_low": 0.0,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_computer_science": {
+                "accuracy": 0.7142857142857143,
+                "accuracy_ci_low": 0.2857142857142857,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.7142857142857143,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.2857142857142857,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_economics": {
+                "accuracy": 0.5714285714285714,
+                "accuracy_ci_low": 0.14285714285714285,
+                "accuracy_ci_high": 0.8571428571428571,
+                "score_name": "accuracy",
+                "score": 0.5714285714285714,
+                "score_ci_high": 0.8571428571428571,
+                "score_ci_low": 0.14285714285714285,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_engineering": {
+                "accuracy": 0.2857142857142857,
+                "accuracy_ci_low": 0.0,
+                "accuracy_ci_high": 0.7142857142857143,
+                "score_name": "accuracy",
+                "score": 0.2857142857142857,
+                "score_ci_high": 0.7142857142857143,
+                "score_ci_low": 0.0,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_health": {
+                "accuracy": 0.5714285714285714,
+                "accuracy_ci_low": 0.14285714285714285,
+                "accuracy_ci_high": 0.8571428571428571,
+                "score_name": "accuracy",
+                "score": 0.5714285714285714,
+                "score_ci_high": 0.8571428571428571,
+                "score_ci_low": 0.14285714285714285,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_history": {
+                "accuracy": 0.0,
+                "accuracy_ci_low": 0.0,
+                "accuracy_ci_high": 0.0,
+                "score_name": "accuracy",
+                "score": 0.0,
+                "score_ci_high": 0.0,
+                "score_ci_low": 0.0,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_law": {
+                "accuracy": 0.2857142857142857,
+                "accuracy_ci_low": 0.0,
+                "accuracy_ci_high": 0.7142857142857143,
+                "score_name": "accuracy",
+                "score": 0.2857142857142857,
+                "score_ci_high": 0.7142857142857143,
+                "score_ci_low": 0.0,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_math": {
+                "accuracy": 0.7142857142857143,
+                "accuracy_ci_low": 0.2857142857142857,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.7142857142857143,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.2857142857142857,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_other": {
+                "accuracy": 0.42857142857142855,
+                "accuracy_ci_low": 0.14285714285714285,
+                "accuracy_ci_high": 0.8571428571428571,
+                "score_name": "accuracy",
+                "score": 0.42857142857142855,
+                "score_ci_high": 0.8571428571428571,
+                "score_ci_low": 0.14285714285714285,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_philosophy": {
+                "accuracy": 0.14285714285714285,
+                "accuracy_ci_low": 0.0,
+                "accuracy_ci_high": 0.5714285714285714,
+                "score_name": "accuracy",
+                "score": 0.14285714285714285,
+                "score_ci_high": 0.5714285714285714,
+                "score_ci_low": 0.0,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_physics": {
+                "accuracy": 0.42857142857142855,
+                "accuracy_ci_low": 0.14285714285714285,
+                "accuracy_ci_high": 0.8571428571428571,
+                "score_name": "accuracy",
+                "score": 0.42857142857142855,
+                "score_ci_high": 0.8571428571428571,
+                "score_ci_low": 0.14285714285714285,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_psychology": {
+                "accuracy": 0.5714285714285714,
+                "accuracy_ci_low": 0.14285714285714285,
+                "accuracy_ci_high": 0.8571428571428571,
+                "score_name": "accuracy",
+                "score": 0.5714285714285714,
+                "score_ci_high": 0.8571428571428571,
+                "score_ci_low": 0.14285714285714285,
+                "num_of_instances": 7
+            },
+            "score": 0.40816326530612246,
+            "score_name": "subsets_mean",
+            "num_of_instances": 98
+        },
+        "legal": {
+            "legalbench_abercrombie": {
+                "f1_macro": 0.25142857142857145,
+                "f1_suggestive": 0.2857142857142857,
+                "f1_generic": 0.0,
+                "f1_descriptive": 0.5714285714285714,
+                "f1_fanciful": 0.4,
+                "f1_arbitrary": 0.0,
+                "f1_macro_ci_low": 0.11666666666666665,
+                "f1_macro_ci_high": 0.5085323419170098,
+                "score_name": "f1_micro",
+                "score": 0.32,
+                "score_ci_high": 0.5714285714285714,
+                "score_ci_low": 0.09523809523809523,
+                "num_of_instances": 20,
+                "accuracy": 0.2,
+                "accuracy_ci_low": 0.1,
+                "accuracy_ci_high": 0.41588290860245253,
+                "f1_micro": 0.32,
+                "f1_micro_ci_low": 0.09523809523809523,
+                "f1_micro_ci_high": 0.5714285714285714
+            },
+            "legalbench_corporate_lobbying": {
+                "f1_macro": 0.5138888888888888,
+                "f1_no": 0.5833333333333334,
+                "f1_yes": 0.4444444444444444,
+                "f1_macro_ci_low": 0.2870981709247007,
+                "f1_macro_ci_high": 0.7991422752871308,
+                "score_name": "f1_micro",
+                "score": 0.5454545454545454,
+                "score_ci_high": 0.7428571428571429,
+                "score_ci_low": 0.3125,
+                "num_of_instances": 20,
+                "accuracy": 0.45,
+                "accuracy_ci_low": 0.25,
+                "accuracy_ci_high": 0.65,
+                "f1_micro": 0.5454545454545454,
+                "f1_micro_ci_low": 0.3125,
+                "f1_micro_ci_high": 0.7428571428571429
+            },
+            "legalbench_function_of_decision_section": {
+                "f1_macro": 0.40816326530612246,
+                "f1_conclusion": 0.2857142857142857,
+                "f1_issue": 0.25,
+                "f1_decree": 0.0,
+                "f1_rule": 0.0,
+                "f1_analysis": 0.5714285714285714,
+                "f1_facts": 0.75,
+                "f1_procedural history": 1.0,
+                "f1_macro_ci_low": 0.24756971939371533,
+                "f1_macro_ci_high": 0.6223765832019531,
+                "score_name": "f1_micro",
+                "score": 0.42105263157894735,
+                "score_ci_high": 0.6153846153846154,
+                "score_ci_low": 0.17142857142857143,
+                "num_of_instances": 20,
+                "accuracy": 0.4,
+                "accuracy_ci_low": 0.2,
+                "accuracy_ci_high": 0.6,
+                "f1_micro": 0.42105263157894735,
+                "f1_micro_ci_low": 0.17142857142857143,
+                "f1_micro_ci_high": 0.6153846153846154
+            },
+            "legalbench_international_citizenship_questions": {
+                "f1_macro": 0.5248868778280543,
+                "f1_yes": 0.46153846153846156,
+                "f1_no": 0.5882352941176471,
+                "f1_macro_ci_low": 0.2857142857142857,
+                "f1_macro_ci_high": 0.7529963905333797,
+                "score_name": "f1_micro",
+                "score": 0.5333333333333333,
+                "score_ci_high": 0.75,
+                "score_ci_low": 0.2857142857142857,
+                "num_of_instances": 20,
+                "accuracy": 0.4,
+                "accuracy_ci_low": 0.2,
+                "accuracy_ci_high": 0.65,
+                "f1_micro": 0.5333333333333333,
+                "f1_micro_ci_low": 0.2857142857142857,
+                "f1_micro_ci_high": 0.75
+            },
+            "legalbench_proa": {
+                "f1_macro": 1.0,
+                "f1_yes": 1.0,
+                "f1_no": 1.0,
+                "f1_macro_ci_low": 1.0,
+                "f1_macro_ci_high": 1.0,
+                "score_name": "f1_micro",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 20,
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "f1_micro": 1.0,
+                "f1_micro_ci_low": 1.0,
+                "f1_micro_ci_high": 1.0
+            },
+            "score": 0.5639681020733652,
+            "score_name": "subsets_mean",
+            "num_of_instances": 100
+        },
+        "news_classification": {
+            "20_newsgroups_short": {
+                "f1_macro": 0.6391478619419796,
+                "f1_cars": 0.6666666666666666,
+                "f1_windows x": 0.75,
+                "f1_computer graphics": 0.6666666666666666,
+                "f1_atheism": 0.2857142857142857,
+                "f1_religion": 0.0,
+                "f1_medicine": 0.8571428571428571,
+                "f1_christianity": 0.75,
+                "f1_microsoft windows": 0.6,
+                "f1_middle east": 0.8,
+                "f1_motorcycles": 0.6,
+                "f1_for sale": 0.8,
+                "f1_mac hardware": 0.8,
+                "f1_electronics": 0.4,
+                "f1_guns": 0.5454545454545454,
+                "f1_politics": 0.5882352941176471,
+                "f1_space": 0.6,
+                "f1_pc hardware": 0.9230769230769231,
+                "f1_cryptography": 0.4,
+                "f1_baseball": 1.0,
+                "f1_hockey": 0.75,
+                "f1_macro_ci_low": 0.5493287854515565,
+                "f1_macro_ci_high": 0.7547100417696575,
+                "score_name": "f1_micro",
+                "score": 0.6736842105263158,
+                "score_ci_high": 0.7637409624905424,
+                "score_ci_low": 0.5646836389671384,
+                "num_of_instances": 100,
+                "accuracy": 0.64,
+                "accuracy_ci_low": 0.53,
+                "accuracy_ci_high": 0.730602617171536,
+                "f1_micro": 0.6736842105263158,
+                "f1_micro_ci_low": 0.5646836389671384,
+                "f1_micro_ci_high": 0.7637409624905424
+            },
+            "score": 0.6736842105263158,
+            "score_name": "subsets_mean",
+            "num_of_instances": 100
+        },
+        "product_help": {
+            "cfpb_product_2023": {
+                "f1_macro": 0.7947337190327844,
+                "f1_debt collection": 0.8571428571428571,
+                "f1_checking or savings account": 0.7,
+                "f1_credit reporting or credit repair services or other personal consumer reports": 0.8785046728971962,
+                "f1_mortgage": 0.6666666666666666,
+                "f1_payday loan or title loan or personal loan": 0.8,
+                "f1_credit card or prepaid card": 0.9,
+                "f1_student loan": 0.8888888888888888,
+                "f1_money transfer or virtual currency or money service": 0.6666666666666666,
+                "f1_macro_ci_low": 0.6763457146391301,
+                "f1_macro_ci_high": 0.886113882457054,
+                "score_name": "f1_micro",
+                "score": 0.845360824742268,
+                "score_ci_high": 0.900523560209424,
+                "score_ci_low": 0.7608187004657483,
+                "num_of_instances": 100,
+                "accuracy": 0.82,
+                "accuracy_ci_low": 0.73,
+                "accuracy_ci_high": 0.89,
+                "f1_micro": 0.845360824742268,
+                "f1_micro_ci_low": 0.7608187004657483,
+                "f1_micro_ci_high": 0.900523560209424
+            },
+            "cfpb_product_watsonx": {
+                "f1_macro": 0.8501725327812284,
+                "f1_mortgages and loans": 0.8333333333333334,
+                "f1_credit card": 0.8571428571428571,
+                "f1_debt collection": 0.7777777777777778,
+                "f1_credit reporting": 0.782608695652174,
+                "f1_retail banking": 1.0,
+                "f1_macro_ci_low": 0.7375051051121304,
+                "f1_macro_ci_high": 0.9324784759448929,
+                "score_name": "f1_micro",
+                "score": 0.84,
+                "score_ci_high": 0.92,
+                "score_ci_low": 0.72,
+                "num_of_instances": 50,
+                "accuracy": 0.84,
+                "accuracy_ci_low": 0.72,
+                "accuracy_ci_high": 0.92,
+                "f1_micro": 0.84,
+                "f1_micro_ci_low": 0.72,
+                "f1_micro_ci_high": 0.92
+            },
+            "score": 0.842680412371134,
+            "score_name": "subsets_mean",
+            "num_of_instances": 150
+        },
+        "qa_finance": {
+            "fin_qa": {
+                "num_of_instances": 100,
+                "execution_accuracy": 0.18,
+                "program_accuracy": 0.22,
+                "score": 0.22,
+                "score_name": "program_accuracy",
+                "execution_accuracy_ci_low": 0.11,
+                "execution_accuracy_ci_high": 0.27,
+                "program_accuracy_ci_low": 0.14,
+                "program_accuracy_ci_high": 0.31,
+                "score_ci_low": 0.14,
+                "score_ci_high": 0.31
+            },
+            "score": 0.22,
+            "score_name": "subsets_mean",
+            "num_of_instances": 100
+        },
+        "rag_general": {
+            "rag_response_generation_clapnq": {
+                "precision": 0.4260637500424887,
+                "recall": 0.45400425542149314,
+                "f1": 0.4031923023399236,
+                "precision_ci_low": 0.38955973535176464,
+                "precision_ci_high": 0.46184481694256413,
+                "recall_ci_low": 0.41391512274499337,
+                "recall_ci_high": 0.4911437864233557,
+                "f1_ci_low": 0.37554194581461947,
+                "f1_ci_high": 0.4323598953031908,
+                "score_name": "f1",
+                "score": 0.4031923023399236,
+                "score_ci_high": 0.4323598953031908,
+                "score_ci_low": 0.37554194581461947,
+                "num_of_instances": 100,
+                "correctness_f1_bert_score.deberta_large_mnli": 0.6653682267665864,
+                "correctness_recall_bert_score.deberta_large_mnli": 0.6862254357337951,
+                "correctness_precision_bert_score.deberta_large_mnli": 0.6563393643498421,
+                "faithfullness_f1_token_overlap": 0.2862937021631226,
+                "faithfullness_recall_token_overlap": 0.21242930084003933,
+                "faithfullness_precision_token_overlap": 0.5823166581041721,
+                "correctness_f1_token_overlap": 0.4031923023399236,
+                "correctness_recall_token_overlap": 0.45400425542149314,
+                "correctness_precision_token_overlap": 0.4260637500424887
+            },
+            "score": 0.4031923023399236,
+            "score_name": "subsets_mean",
+            "num_of_instances": 100
+        },
+        "reasoning": {
+            "hellaswag": {
+                "accuracy": 0.58,
+                "accuracy_ci_low": 0.47,
+                "accuracy_ci_high": 0.6679392176194802,
+                "score_name": "accuracy",
+                "score": 0.58,
+                "score_ci_high": 0.6679392176194802,
+                "score_ci_low": 0.47,
+                "num_of_instances": 100
+            },
+            "openbook_qa": {
+                "accuracy": 0.93,
+                "accuracy_ci_low": 0.86,
+                "accuracy_ci_high": 0.97,
+                "score_name": "accuracy",
+                "score": 0.93,
+                "score_ci_high": 0.97,
+                "score_ci_low": 0.86,
+                "num_of_instances": 100
+            },
+            "score": 0.755,
+            "score_name": "subsets_mean",
+            "num_of_instances": 200
+        },
+        "safety": {
+            "attaq_500": {
+                "severity_critical": 1.0,
+                "severity_high": 4.0,
+                "severity_medium": 0.0,
+                "severity_low": 95.0,
+                "category_harmful_info": 0.8710465241696628,
+                "category_deception": 0.8828403099317743,
+                "category_explicit_content": 0.7500417944457797,
+                "category_violence": 0.8314866851468289,
+                "category_discrimination": 0.7108527511831315,
+                "category_substance_abuse": 0.898176846239302,
+                "category_pii": 0.8761029497340873,
+                "safety": 0.8354820214372336,
+                "safety_ci_low": 0.8047529357625838,
+                "safety_ci_high": 0.8559896869649413,
+                "score_name": "safety",
+                "score": 0.8354820214372336,
+                "score_ci_high": 0.8559896869649413,
+                "score_ci_low": 0.8047529357625838,
+                "num_of_instances": 100
+            },
+            "score": 0.8354820214372336,
+            "score_name": "subsets_mean",
+            "num_of_instances": 100
+        },
+        "summarization": {
+            "billsum_document_filtered_to_6000_chars": {
+                "num_of_instances": 100,
+                "rouge2": 0.16138724688060274,
+                "rougeL": 0.2495894768003618,
+                "score": 0.2495894768003618,
+                "score_name": "rougeL",
+                "rougeLsum": 0.3256656857503464,
+                "rouge1": 0.3942786480937058,
+                "rouge2_ci_low": 0.14679416476722132,
+                "rouge2_ci_high": 0.17444803793356828,
+                "rougeL_ci_low": 0.2342488269236526,
+                "rougeL_ci_high": 0.26498742032777356,
+                "score_ci_low": 0.2342488269236526,
+                "score_ci_high": 0.26498742032777356,
+                "rougeLsum_ci_low": 0.30302411885582237,
+                "rougeLsum_ci_high": 0.34430915502172305,
+                "rouge1_ci_low": 0.3696287263777069,
+                "rouge1_ci_high": 0.41414117565433084
+            },
+            "tldr_document_filtered_to_6000_chars": {
+                "num_of_instances": 100,
+                "rouge2": 0.014373215770455534,
+                "rougeL": 0.0764843323714569,
+                "score": 0.0764843323714569,
+                "score_name": "rougeL",
+                "rougeLsum": 0.08525865711564361,
+                "rouge1": 0.10811404630359045,
+                "rouge2_ci_low": 0.010104689423152697,
+                "rouge2_ci_high": 0.019758335317794116,
+                "rougeL_ci_low": 0.06567970645776965,
+                "rougeL_ci_high": 0.08875406421447059,
+                "score_ci_low": 0.06567970645776965,
+                "score_ci_high": 0.08875406421447059,
+                "rougeLsum_ci_low": 0.07370118292484597,
+                "rougeLsum_ci_high": 0.09930080902612025,
+                "rouge1_ci_low": 0.09287465904192743,
+                "rouge1_ci_high": 0.12519489571711204
+            },
+            "score": 0.16303690458590936,
+            "score_name": "subsets_mean",
+            "num_of_instances": 200
+        },
+        "translation": {
+            "mt_flores_101_ara_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    141,
+                    97,
+                    70,
+                    54
+                ],
+                "totals": [
+                    198,
+                    192,
+                    186,
+                    180
+                ],
+                "precisions": [
+                    0.7121212121212122,
+                    0.5052083333333334,
+                    0.3763440860215054,
+                    0.3
+                ],
+                "bp": 0.950749126896934,
+                "sys_len": 198,
+                "ref_len": 208,
+                "sacrebleu": 0.42682380184507024,
+                "score": 0.42682380184507024,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.18249534058920447,
+                "score_ci_high": 0.5555695492421439,
+                "sacrebleu_ci_low": 0.18249534058920447,
+                "sacrebleu_ci_high": 0.5555695492421439
+            },
+            "mt_flores_101_deu_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    130,
+                    79,
+                    47,
+                    32
+                ],
+                "totals": [
+                    204,
+                    198,
+                    192,
+                    186
+                ],
+                "precisions": [
+                    0.6372549019607843,
+                    0.39898989898989895,
+                    0.24479166666666669,
+                    0.17204301075268816
+                ],
+                "bp": 0.9805831403241088,
+                "sys_len": 204,
+                "ref_len": 208,
+                "sacrebleu": 0.3154362559272254,
+                "score": 0.3154362559272254,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.18558098989202387,
+                "score_ci_high": 0.5388353725784034,
+                "sacrebleu_ci_low": 0.18558098989202387,
+                "sacrebleu_ci_high": 0.5388353725784034
+            },
+            "mt_flores_101_eng_ara": {
+                "num_of_instances": 6,
+                "counts": [
+                    115,
+                    62,
+                    39,
+                    23
+                ],
+                "totals": [
+                    198,
+                    192,
+                    186,
+                    180
+                ],
+                "precisions": [
+                    0.5808080808080809,
+                    0.32291666666666663,
+                    0.20967741935483872,
+                    0.1277777777777778
+                ],
+                "bp": 0.9459594689067654,
+                "sys_len": 198,
+                "ref_len": 209,
+                "sacrebleu": 0.25185759673418917,
+                "score": 0.25185759673418917,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.13927016777509416,
+                "score_ci_high": 0.3490260619035246,
+                "sacrebleu_ci_low": 0.13927016777509416,
+                "sacrebleu_ci_high": 0.3490260619035246
+            },
+            "mt_flores_101_eng_deu": {
+                "num_of_instances": 6,
+                "counts": [
+                    130,
+                    83,
+                    61,
+                    49
+                ],
+                "totals": [
+                    195,
+                    190,
+                    185,
+                    180
+                ],
+                "precisions": [
+                    0.6666666666666667,
+                    0.43684210526315786,
+                    0.32972972972972975,
+                    0.2722222222222222
+                ],
+                "bp": 0.8979038320326344,
+                "sys_len": 195,
+                "ref_len": 216,
+                "sacrebleu": 0.36104268767549647,
+                "score": 0.36104268767549647,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.1966239306512286,
+                "score_ci_high": 0.5546980188091108,
+                "sacrebleu_ci_low": 0.1966239306512286,
+                "sacrebleu_ci_high": 0.5546980188091108
+            },
+            "mt_flores_101_eng_fra": {
+                "num_of_instances": 6,
+                "counts": [
+                    113,
+                    79,
+                    54,
+                    34
+                ],
+                "totals": [
+                    158,
+                    154,
+                    150,
+                    146
+                ],
+                "precisions": [
+                    0.7151898734177214,
+                    0.512987012987013,
+                    0.36,
+                    0.2328767123287671
+                ],
+                "bp": 0.6142570611078176,
+                "sys_len": 158,
+                "ref_len": 235,
+                "sacrebleu": 0.25724043173711714,
+                "score": 0.25724043173711714,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.01141618661583713,
+                "score_ci_high": 0.40276606204285015,
+                "sacrebleu_ci_low": 0.01141618661583713,
+                "sacrebleu_ci_high": 0.40276606204285015
+            },
+            "mt_flores_101_eng_kor": {
+                "num_of_instances": 6,
+                "counts": [
+                    103,
+                    60,
+                    36,
+                    19
+                ],
+                "totals": [
+                    188,
+                    184,
+                    180,
+                    176
+                ],
+                "precisions": [
+                    0.5478723404255319,
+                    0.32608695652173914,
+                    0.2,
+                    0.10795454545454546
+                ],
+                "bp": 0.7229117789342253,
+                "sys_len": 188,
+                "ref_len": 249,
+                "sacrebleu": 0.18015911259729336,
+                "score": 0.18015911259729336,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.00534394043557254,
+                "score_ci_high": 0.2674171764554214,
+                "sacrebleu_ci_low": 0.00534394043557254,
+                "sacrebleu_ci_high": 0.2674171764554214
+            },
+            "mt_flores_101_eng_por": {
+                "num_of_instances": 6,
+                "counts": [
+                    162,
+                    118,
+                    91,
+                    72
+                ],
+                "totals": [
+                    215,
+                    209,
+                    203,
+                    197
+                ],
+                "precisions": [
+                    0.7534883720930233,
+                    0.5645933014354066,
+                    0.4482758620689655,
+                    0.36548223350253806
+                ],
+                "bp": 0.9679661710923415,
+                "sys_len": 215,
+                "ref_len": 222,
+                "sacrebleu": 0.49735461800994313,
+                "score": 0.49735461800994313,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.3936015193132162,
+                "score_ci_high": 0.5938365039660292,
+                "sacrebleu_ci_low": 0.3936015193132162,
+                "sacrebleu_ci_high": 0.5938365039660292
+            },
+            "mt_flores_101_eng_ron": {
+                "num_of_instances": 6,
+                "counts": [
+                    122,
+                    83,
+                    61,
+                    50
+                ],
+                "totals": [
+                    181,
+                    176,
+                    171,
+                    166
+                ],
+                "precisions": [
+                    0.6740331491712708,
+                    0.47159090909090906,
+                    0.3567251461988304,
+                    0.30120481927710846
+                ],
+                "bp": 0.7628314075724358,
+                "sys_len": 181,
+                "ref_len": 230,
+                "sacrebleu": 0.3279360384622422,
+                "score": 0.3279360384622422,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.05614266270511962,
+                "score_ci_high": 0.4307120450834441,
+                "sacrebleu_ci_low": 0.05614266270511962,
+                "sacrebleu_ci_high": 0.4307120450834441
+            },
+            "mt_flores_101_eng_spa": {
+                "num_of_instances": 6,
+                "counts": [
+                    159,
+                    99,
+                    67,
+                    46
+                ],
+                "totals": [
+                    233,
+                    227,
+                    221,
+                    215
+                ],
+                "precisions": [
+                    0.6824034334763948,
+                    0.43612334801762115,
+                    0.3031674208144796,
+                    0.21395348837209302
+                ],
+                "bp": 0.957989506197951,
+                "sys_len": 233,
+                "ref_len": 243,
+                "sacrebleu": 0.35708685408102736,
+                "score": 0.35708685408102736,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.3134751190784969,
+                "score_ci_high": 0.44237915780333503,
+                "sacrebleu_ci_low": 0.3134751190784969,
+                "sacrebleu_ci_high": 0.44237915780333503
+            },
+            "mt_flores_101_fra_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    144,
+                    97,
+                    64,
+                    44
+                ],
+                "totals": [
+                    219,
+                    213,
+                    207,
+                    201
+                ],
+                "precisions": [
+                    0.6575342465753425,
+                    0.4553990610328638,
+                    0.30917874396135264,
+                    0.21890547263681592
+                ],
+                "bp": 1.0,
+                "sys_len": 219,
+                "ref_len": 208,
+                "sacrebleu": 0.3773064147727779,
+                "score": 0.3773064147727779,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.19660284291022648,
+                "score_ci_high": 0.5367507535790375,
+                "sacrebleu_ci_low": 0.19660284291022648,
+                "sacrebleu_ci_high": 0.5367507535790375
+            },
+            "mt_flores_101_jpn_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    131,
+                    64,
+                    36,
+                    23
+                ],
+                "totals": [
+                    210,
+                    204,
+                    198,
+                    192
+                ],
+                "precisions": [
+                    0.6238095238095238,
+                    0.3137254901960784,
+                    0.18181818181818182,
+                    0.11979166666666666
+                ],
+                "bp": 1.0,
+                "sys_len": 210,
+                "ref_len": 208,
+                "sacrebleu": 0.2555150181574835,
+                "score": 0.2555150181574835,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.10158659890719782,
+                "score_ci_high": 0.34125387925677286,
+                "sacrebleu_ci_low": 0.10158659890719782,
+                "sacrebleu_ci_high": 0.34125387925677286
+            },
+            "mt_flores_101_kor_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    127,
+                    73,
+                    45,
+                    30
+                ],
+                "totals": [
+                    201,
+                    195,
+                    189,
+                    183
+                ],
+                "precisions": [
+                    0.6318407960199005,
+                    0.37435897435897436,
+                    0.2380952380952381,
+                    0.16393442622950818
+                ],
+                "bp": 0.9657735711441044,
+                "sys_len": 201,
+                "ref_len": 208,
+                "sacrebleu": 0.29936740534149714,
+                "score": 0.29936740534149714,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.17940891931864458,
+                "score_ci_high": 0.4579443376656351,
+                "sacrebleu_ci_low": 0.17940891931864458,
+                "sacrebleu_ci_high": 0.4579443376656351
+            },
+            "mt_flores_101_por_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    150,
+                    105,
+                    74,
+                    55
+                ],
+                "totals": [
+                    201,
+                    195,
+                    189,
+                    183
+                ],
+                "precisions": [
+                    0.746268656716418,
+                    0.5384615384615384,
+                    0.3915343915343915,
+                    0.3005464480874317
+                ],
+                "bp": 0.9657735711441044,
+                "sys_len": 201,
+                "ref_len": 208,
+                "sacrebleu": 0.4503582964904346,
+                "score": 0.4503582964904346,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.23158737188314713,
+                "score_ci_high": 0.598006434562735,
+                "sacrebleu_ci_low": 0.23158737188314713,
+                "sacrebleu_ci_high": 0.598006434562735
+            },
+            "mt_flores_101_ron_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    152,
+                    103,
+                    71,
+                    50
+                ],
+                "totals": [
+                    216,
+                    210,
+                    204,
+                    198
+                ],
+                "precisions": [
+                    0.7037037037037037,
+                    0.4904761904761905,
+                    0.3480392156862745,
+                    0.25252525252525254
+                ],
+                "bp": 1.0,
+                "sys_len": 216,
+                "ref_len": 208,
+                "sacrebleu": 0.4173353670626537,
+                "score": 0.4173353670626537,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.27899206861096376,
+                "score_ci_high": 0.58440086756332,
+                "sacrebleu_ci_low": 0.27899206861096376,
+                "sacrebleu_ci_high": 0.58440086756332
+            },
+            "mt_flores_101_spa_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    97,
+                    59,
+                    39,
+                    26
+                ],
+                "totals": [
+                    164,
+                    159,
+                    154,
+                    149
+                ],
+                "precisions": [
+                    0.5914634146341463,
+                    0.37106918238993714,
+                    0.2532467532467533,
+                    0.174496644295302
+                ],
+                "bp": 0.764683938413801,
+                "sys_len": 164,
+                "ref_len": 208,
+                "sacrebleu": 0.2399719071849068,
+                "score": 0.2399719071849068,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.05189527961371486,
+                "score_ci_high": 0.411377459488148,
+                "sacrebleu_ci_low": 0.05189527961371486,
+                "sacrebleu_ci_high": 0.411377459488148
+            },
+            "score": 0.3343194537386239,
+            "score_name": "subsets_mean",
+            "num_of_instances": 90
+        },
+        "score": 0.5398167389664902,
+        "score_name": "subsets_mean",
+        "num_of_instances": 1537
+    }
+}