File size: 3,475 Bytes
dfdc6c0 4a63dcf dfdc6c0 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 |
#########################################################################
# File Name: eval.sh
# Author: Xianchao Wu, Peng Xu
# mail: xianchaow@nvidia.com, pengx@nvidia.com
# Created Time: Mon Sep 4 07:33:40 2024
#########################################################################
#!/bin/bash
# TODO change this to your reference file dir:
data_home="" # data_home https://huggingface.co/nvidia/Llama3-ChatQA-2-70B/tree/main/data
REFDIR=${data_home}
# TODO change to your tstdir
model_path=""
TSTDIR="${model_path}/outputs/"
model_size=70b # TODO change this
retriever="e5_mistral_retriever_chunkbysents1200"
adir=$TSTDIR
echo $adir
declare -A dataset2num_samples
dataset2num_samples["gov_report"]=200
dataset2num_samples["narrative_qa"]=2000
dataset2num_samples["qasper"]=2000
dataset2num_samples["qmsum"]=200
dataset2num_samples["quality"]=2000
dataset2num_samples["summ_screen_fd"]=200
dataset2num_samples["musique"]=200
dataset2num_samples["hotpotqa"]=200
dataset2num_samples["multifieldqa_en"]=200
dataset2num_samples["squality"]=200
dataset2num_samples["doc2dial_full_dialogue"]=1000
echo "ref path = $REFDIR"
echo "tst out path = $TSTDIR"
declare -A sys2name
sys2name["baseline"]=""
sys2name["ret"]="_ctx5"
for system in "baseline" "ret"
do
suffix=${sys2name[${system}]}
echo "--final display----$system----"
for adataset in "qmsum" "qasper" "quality"
do
echo $adataset
ref_fn="${REFDIR}/${adataset}.${retriever}/test.json"
tst_fn="${adir}/${adataset}.e5_mistral_retriever_chunkbysents1200_output_0to${dataset2num_samples[${adataset}]}${suffix}.txt"
echo "ref for ${adataset}", ${ref_fn}
echo "tstout for ${adataset}", ${tst_fn}
if [[ ! -e ${tst_fn} ]]; then
echo "Error: tst_fn=${tst_fn} not exist!"
fi
if [[ ! -e ${ref_fn} ]]; then
echo "Error: ref_fn=${ref_fn} not exist!"
fi
#continue
if [[ -e ${tst_fn} && -e ${ref_fn} ]]
then
python3 dataset_evaluator_retro.py \
--datapath ${ref_fn} \
--gen_test_file ${tst_fn} \
--dataset $adataset
else
if [[ ! -e ${tst_fn} ]]; then
echo "Error: tst_fn=${tst_fn} not exist!"
fi
if [[ ! -e ${ref_fn} ]]; then
echo "Error: ref_fn=${ref_fn} not exist!"
fi
fi
done
for adataset in "musique" "hotpotqa" "multifieldqa_en"
do
echo $adataset
ref_fn="${REFDIR}/${adataset}.${retriever}/test.json"
# TODO change this if necessary, model's prediction output
tst_fn="${adir}/${adataset}.e5_mistral_retriever_chunkbysents1200_output_0to${dataset2num_samples[${adataset}]}${suffix}.txt"
echo "ref for ${adataset}", ${ref_fn}
echo "tstout for ${adataset}", ${tst_fn}
if [[ ! -e ${tst_fn} ]]; then
echo "Error: tst_fn=${tst_fn} not exist!"
fi
if [[ ! -e ${ref_fn} ]]; then
echo "Error: ref_fn=${ref_fn} not exist!"
fi
#continue
if [[ -e ${tst_fn} && -e ${ref_fn} ]]
then
python3 dataset_evaluator_retro_longbench.py \
--datapath ${ref_fn} \
--gen_test_file ${tst_fn} \
--dataset $adataset
else
if [[ ! -e ${tst_fn} ]]; then
echo "Error: tst_fn=${tst_fn} not exist!"
fi
if [[ ! -e ${ref_fn} ]]; then
echo "Error: ref_fn=${ref_fn} not exist!"
fi
fi
done
done
|