e'ddlmZddlmZddlmZeGddZdZedGd d Z d Z d Z ddZ dZ dZdZdZdZdededefdZdS)) dataclassN) AutoConfigc<eZdZUeed<eed<eed<dZeed<dS) ColumnContentnametypedisplayed_by_defaultFhiddenN)__name__ __module__ __qualname__str__annotations__boolr zfields..sA   aAbqbETMMafPTnnnnnr)__dict__items) raw_classs rfieldsr s2  (..00   rT)frozenc~eZdZedddZedddZedddZeddd Zed ddZed ddZ ed ddZ ed dd Z edddZ eddd Z eddd Zeddd Zeddd Zeddd Zeddd Zeddd Zeddd Zeddd Zeddd Zeddd Zeddd Zeddd ZedddZeddd ZdS)AutoEvalColumnTrTModelmarkdownzWin Ratenumber Average scoreFhumaneval-pythonjava javascriptThroughput (tokens/s)cppphprustswiftrluadracketjuliaz #LanguagesThroughput (tokens/s) bs=50Peak Memory (MB) Seq_lengthLinksz Submission PRN)r r r rmodel_type_symbolmodelwin_rateaveragehumaneval_pythonr*r+ throughputr-r.r/r0r1r2r3r4r5 languagesthroughput_bs50 peak_memory seq_lengthlinkdummyprrrrr#r#s% c5$77 M':t 4 4E}Z488HmOXu==G$}%74HH =4 0 0D|Xt<zr)rD model_names rmodel_hyperlinkrI2s5 at a aQ[ a a aarc<|dd|d<|S)Nc:t|d|dS)Nr9r%)rI)rows rz&make_clickable_names..8sOCL#g,??raxisr%)apply)dfs rmake_clickable_namesrS6s-((??aBwK IrrNc|dkrdnd}d|d<d|d<d|j|d jd df<d |j|d jd df<d |j|d jddf<d|j|d jddf<d|j|d jddf<d|j|d jddf<d|j|d jddf<d|j|d jddf<d|j|d jddf<d|j|d jddf<d|j|d jddf<d|j|d jd df<d!|j|d jd"df<d#|j|d jd$df<d%|j|d jd&df<d'|j|d jd(df<d'|j|d jd(df<d)|j|d jd*df<d+|j|d jd,df<tj}|jD]}|tj|j||fg|j|d-fgd.t|j|d/fd0zg|j|df|j|df1|j|d fd2|d3zd4zd5zt|j|d6fzd7zd8zt|j|d9fz|j|d fgd:;| dd?|d@|dAB|S)CNrNr,r6rsymbolcolororanger%zStarCoder|SantaCoderpinkCodeGenpurpleReplitperu WizardCodercornflowerblueCodeGeex cadetblue StableCodelime OctoCoderwheatOctoGeeXsalmonDeci palevioletred CodeLlama burlywoodCodeGurucrimsonPhinddimgrayFalconyellowRefactgrayPhi lightskyblue CodeShell lightgreenDeepSeekoliveCodeFuser(markerszSize (B) )sizerWrUz%{text}

z : %{x}
zAverage Score: %{y}
zPeak Memory (MB): r7z
zHuman Eval (Python): r)T)xymodemarkerr hovertemplatetext showlegendFiiXz?Average Score Vs Throughput (A100-80GB, Float16, Batch Size z)zAverage Code Score)autosizewidthheighttitle xaxis_title yaxis_title) locrcontainsgoFigureindex add_traceScatterdict update_layout)rRbsthroughput_columnfigis rplot_throughputr=s#%770MBxLBwKHPBF2g;? # #$: ; ;W DE;ABF2g;? # #I . . 78:BBF2g;? # #H - -w 67?EBF2g;? # #M 2 2G ;<IBF2g;? # #L 1 17 :;=CBF2g;? # #K 0 0' 9:

r)errors r styled_errorr}s U% U U UUrcd|dS)Nz?

rr)warns rstyled_warningrs WT W W WWrcd|dS)Nz>

rr)messages rstyled_messagers YG Y Y YYrc^||dSNrNrO)notnaallrRcolumnss rhas_no_nan_valuesrs) g;     " " " * **rc^||dSr)isnaanyrs rhas_nan_valuesrs) g;     ! !q ! ) ))rrHrevisionreturnc tj||ddS#t$rYdSt$r}t d|Yd}~dSd}~wwxYw)NF)rtrust_remote_code)TN)Fzneeds to be launched with `trust_remote_code=True`. For safety reason, we do not allow these models to be automatically submitted to the leaderboard.z.Could not get the model config from the hub.: )Fzwas not found on hub!)rfrom_pretrained ValueError Exceptionprint)rHres ris_model_on_hubrs .":TYZZZZz       ... BqBBCCC------.s A AA  A)rN) dataclassesrplotly.graph_objects graph_objectsr transformersrrr r#rIrSrrrrrrrrrrrrrs}!!!!!!!!!!!!######    $;;;;;;;;6aaa====@VVVXXXZZZ+++*** . .s .t . . . . . .r