o ;;fn@sddlmZddlmZddlmZeGdddZddZedd Gd d d Z d d Z ddZ d"ddZ ddZ ddZddZddZddZdededefd d!ZdS)#) dataclassN) AutoConfigc@s2eZdZUeed<eed<eed<dZeed<dS) ColumnContentnametypeZdisplayed_by_defaultFhiddenN)__name__ __module__ __qualname__str__annotations__boolrrr7/fsx/loubna/tmp/bigcode-models-leaderboard/src/utils.pyrs rcCsdd|jDS)NcSs4g|]\}}|dddkr|dddkr|qS)N__r).0kvrrr s.zfields..)__dict__items)Z raw_classrrrfieldssrT)frozenc@s,eZdZedddZedddZedddZeddd Zed ddZed ddZ ed ddZ ed dd Z edddZ eddd Z eddd Zeddd Zeddd Zeddd Zeddd Zeddd Zeddd Zeddd Zeddd Zeddd Zeddd Zeddd ZedddZeddd ZdS)AutoEvalColumnTr TModelZmarkdownzWin Ratenumber Average scoreFhumaneval-pythonjava javascriptThroughput (tokens/s)cppphprustswiftrluadracketjuliaz #LanguagesThroughput (tokens/s) bs=50Peak Memory (MB)Z Seq_lengthLinksz Submission PRN)rr r rZmodel_type_symbolZmodelZwin_rateaverageZhumaneval_pythonr!r"Z throughputr$r%r&r'r(r)r*r+r,Z languagesZthroughput_bs50Z peak_memoryZ seq_lengthlinkdummyprrrrrrs2                       rcCsd|d|dS)Nzzr)r1 model_namerrrmodel_hyperlink2sr5cCs|jdddd|d<|S)NcSst|d|dS)Nr/r)r5)rowrrr8sz&make_clickable_names..axisr)apply)dfrrrmake_clickable_names6s r=r8cCsl|dkrdnd}d|d<d|d<d|j|d jd df<d |j|d jd df<d |j|d jddf<d|j|d jddf<d|j|d jddf<d|j|d jddf<d|j|d jddf<d|j|d jddf<d|j|d jddf<d|j|d jddf<d|j|d jddf<d|j|d jd df<d!|j|d jd"df<d#|j|d jd$df<d%|j|d jd&df<d'|j|d jd(df<d'|j|d jd(df<d)|j|d jd*df<d+|j|d jd,df<d-|j|d jd.df<d/|j|d jd0df<d1|j|d jd2df<d3|j|d jd4d5df<t}|jD]`}|tj|j||fg|j|d6fgd7t|j|d8fd9g|j|df|j|dfd:|j|d fd;|d<d=d>t|j|d?fd@dAt|j|dBf|j|d fgdCdDqC|j dEdFdGdH|dI|dJdK|S)LNr8r#r-rsymbolcolorZorangerzStarCoder|SantaCoderZpinkZCodeGenZpurpleZReplitZperu WizardCoderZcornflowerblueZCodeGeexZ cadetbluezStableCode-3B-alphaZlimeZ OctoCoderZwheatZOctoGeeXZsalmonZDeciZ palevioletredZ CodeLlamaZ burlywoodZCodeGuruZcrimsonPhindZdimgrayZFalconZyellowRefactZgrayZPhiZ lightskyblue CodeShellZ lightgreenZDeepSeekZoliveCodeFuseZ steelbluezStable-code-3bZredzOpenCodeInterpreter-DSZblackZ CodeGemmaZmaroonZCodeQwenzNxcode-CQ-7B-orpormarkerszSize (B) )sizer@r>z%{text}

z : %{x}
zAverage Score: %{y}
zPeak Memory (MB): r.z
zHuman Eval (Python): r T)xymodemarkerrZ hovertemplatetextZ showlegendFiiXz?Average Score Vs Throughput (A100-80GB, Float16, Batch Size z)zAverage Code Score)ZautosizewidthheighttitleZ xaxis_titleZ yaxis_title) locr containsgoFigureindexZ add_traceZScatterdictZ update_layout)r<bsZthroughput_columnfigirrrplot_throughput=s     rZcC d|dS)Nz<



r)errorrrr styled_error r^cCr[)Nz?

r\r)warnrrrstyled_warningr_racCr[)Nz>

r\r)messagerrrstyled_messager_rccC||jddSNr8r9)notnaallr<columnsrrrhas_no_nan_valuesrjcCrdre)isnaanyrhrrrhas_nan_valuesrkrnr4revisionreturnc Cs^z tj||ddWdStyYdSty.}ztd|WYd}~dSd}~ww)NF)roZtrust_remote_code)TN)Fzneeds to be launched with `trust_remote_code=True`. For safety reason, we do not allow these models to be automatically submitted to the leaderboard.z.Could not get the model config from the hub.: )Fzwas not found on hub!)rZfrom_pretrained ValueError Exceptionprint)r4roerrris_model_on_hubs ru)r8) dataclassesrZplotly.graph_objectsZ graph_objectsrSZ transformersrrrrr5r=rZr^rarcrjrnr r rurrrrs"    E