%1gGddlZddlZddlmZddlmZmZmZddlm Z m Z m Z m Z ddl mZmZddlmZddlZGddZGd d ZdS) N) lru_cache)ListTupleDict)END_OF_RULE_MARKEREND_OF_ALTERNATE_MARKER parse_ebnfREF_RULE_MARKER) PartialUTF8 decode_utf8)intervals_intersectc*eZdZdZedZdS) AcceptStatec"||_||_dSN)stacks partial_utf8)selfrrs W/mnt/d/dev/semgus/TinyLlama_v1.1_GAD/../transformers-GAD/transformers_gad/recognizer.py__init__zAcceptState.__init__s (c:tgtSrrr rr empty_statezAcceptState.empty_states2{}}---rN)__name__ __module__ __qualname__r staticmethodrrrrrrs>)))..\...rrc neZdZ d,deededeedeeefdZdedeefdZdedeeefd Zdefd Z defd Z e d de edeeefdZ dedefdZ d-dedeeedefdZ d.dedefdZe d dede e edeeefdZe d dede edeeefdZ d/deedeeedeeefdZ d/deedeeedefdZe d ded edefd!Zd ededefd"Zd#edefd$Zd0d#edefd%Zd0d#edefd&Zdeeefd'Zdeeefd(Ze d d)Z d/deedeeedeeefd*Z!e d  d/de ede edeeefd+Z"dS)1StringRecognizerNgrammar_encoding start_rule_id rule_offsetsrc||_|||_n+|td|||_|||_n+|td|||_||_dS)Nz4start_rule_id cannot be None if rule_offsets is Nonez.start_rule_id cannot be None if stacks is None)r"r$ ValueError init_rulesr init_stackr#)rr"r#r$rs rrzStringRecognizer.__init__s!1  # ,D  $ !WXXX $ > >D    DKK$ !QRRR+/??=+I+IDK*rreturncd}g}|j|dkr|j|}t||kr,|dg|t|z dzz|||<|dz}|j|tkr)|dz|j|z}|j|tk)|dz}|j|dk|j||}||ksJ|S)Nri)r"lenextendr)rr# _rule_offsetr$rule_idsimple_rhs_offsetretrieved_start_rule_ids rr'zStringRecognizer.init_rules7s  #L1V;;+L9G<  G++##RDGc,6G6G,G!,K$LMMM$0L !!-q 0 '(9:>PPP%)D,ABS,TT"'(9:>PPP-q0L'#L1V;;*#'"7 ]8S"T&-7777rcRg}|j|dz}|j|rg}|dz}|j|tkr||||t ||d|j|zz }|j||S)Nr,)r$r"rappendr. advance_stacktuple)rr#rsub_rhs_offsetstackelement_offsets rr(zStringRecognizer.init_stackUs*=9A=#N3 H!E+a/N$^48OOO ^,,, MM$,,U5\\:: ; ; ; a$"7"GG GN#N3 H rcjt||jtSr)rr(r#r rs rget_initial_accept_statez)StringRecognizer.get_initial_accept_stateds%4??4+=>> NNNrc:tgtSrrr;s rget_termination_accept_statez-StringRecognizer.get_termination_accept_stategs2{}}---ri)maxsizer8ct|}t|dkr|gS|d}|j|tkr|gS|j|dz}|j|dz}g}|j|t kr|dd}|dz}|j|t kr|||dz}|j|t kr|||| t|||j|dzz }|j|t k|S)Nrr+r,) listr-r"r r$rrr4r.r5r6) rr8cur_element_offset ref_rule_idref_subrule_offset new_stacks new_stacknext_element_offsetref_element_offsets rr5zStringRecognizer.advance_stackjspU  u::??7N#2Y  !3 4 G G7N/0BQ0FGK!%!2;!?!!C *,J'(:;?QQQ!#2#J &81&<#)*=>.//$$%8999&8!%;"();<@WWW$$%7888!!$"4"4U95E5E"F"FGGG"d&;>T ! !XH(3Hl(K(K% %   MS SSAQSS   ..{FCC   E5zzQtt"2YN22>CSTT tt urc||}|j}|j}t|turt |}t ||\}}|rtjd|d|| ||}g} |D]I} t| dkr| d} | | |r| | Jt| |S)NrRrSrr+)r<rrrTrBrNr rUrVrWr-rXr4r) rrPrKrYrrrZr[rFnew_new_stacksr8r9s rrMzStringRecognizer._consume_bytess  88::L$#0 >>T ! !XH(3Hl(K(K% %   MS SSAQSS   ..{FCC  - -E5zzQ"2YN22>CSTT -%%e,,,>+;<<z8StringRecognizer._consume_code_point..s'H'H'HU 'H'H'Hrr)rBr._consume_code_point_per_stackr6)rr_rrFr8s r_consume_code_pointz$StringRecognizer._consume_code_points "&'H'H'H'H'H"I"I ??   E   22:uU||LL    rcht|}g}|dkr|St|dkr|S|d}|||}|s|S|j|}||dzz }|dd}|j|r|||t |S)rarr+r,N)rBr-accept_code_point_at_elementr"r4r5r6)rr_r8rFr9foundsizerGs rrgz.StringRecognizer._consume_code_point_per_stacksU  ??  u::?? r11*nMM  $^4$("#2#J   0 -   ^ , , ,!!% "2"2333rFrZc&t|D]\}}td|D}|||}t|dkr9|r7|d|dz}t |}t jd|d|d|S)Nc,g|]}t|Srr6rds rrfz9StringRecognizer._consume_code_points..s4V4V4VeU5\\4V4V4Vrrr,z code point z corresponding to z is accepted) enumerater6rhr-chrrUrV) rrZrrYir_ tuple_stacksaccepted_code_pointcorresponding_chars rrWz%StringRecognizer._consume_code_points s'{33  MAz.34V4Vv4V4V4V.W.WL--j,GGF6{{Q7&1'AE'&:#%(__" i"5iiI[iii rcV||||}t|dkSNr)rWr-)rrZrrYs r_accept_code_pointsz$StringRecognizer._accept_code_pointss+**;HH6{{Qrr9c|j|}|dz }td|dD]4}|j||z|cxkr|j||zdzkrn1dS5dS)Nr,rrATF)r"range)rr_r9rlrrs rrjz-StringRecognizer.accept_code_point_at_elements$^4!q$""  A%nq&89AAAA(!);a)?@AAAAAtt A urc \|j}|j}|dkr|dkrdS|dkrdS||dzz}|d|dzzdz z}|dkr|dkrd}n|dkrd }|j|}|dz }td|dD]6}t |||j||z|j||zdzrdS7dS) Nr,rAFrTii)valuen_remainr"rzr ) rr9r partial_valuerlowhighrlrrs rrXz/StringRecognizer.partial_utf8_accept_at_element:s%* ( q==]Q..5 q==41 -qX\*a/0 !881}}Q$^4!q$""  A"%nq&89%nq&81&<=   tt  urstringczd|D}|||j}t||jS)Nc,g|]}t|Sr)ord)rechars rrfz4StringRecognizer._consume_string..qs444Ts4yy444r)rWrrr)rrrKrZrs r_consume_stringz StringRecognizer._consume_stringos@44V444 **; 8KLL6<#<===rc||}|||}t|jdkSrw)r<rr-r)rrrKnew_accept_states r_accept_prefixzStringRecognizer._accept_prefixusF  88::L// EE#*++a//rc||}|||}td|jD}|S)Nc3<K|]}t|dkVdSrNr-rds r z2StringRecognizer._accept_string..s=* * %CJJ!O* * * * * * r)r<ranyr)rrrKrat_least_one_stack_is_emptys r_accept_stringzStringRecognizer._accept_string{sf  88::L// EE&)* * )9)@* * * ' ' #+*rcft|dkrdS|D]}t|dkrdSdS)NrTFr)rrr8s r _can_stopzStringRecognizer._can_stopsI v;;!  4  E5zzQtt5rcZt|dkptd|DS)Nrc3<K|]}t|dkVdSrrrds rrz.StringRecognizer._must_stop..s,&K&K5s5zzQ&K&K&K&K&K&Kr)r-all)rrs r _must_stopzStringRecognizer._must_stops/6{{aK3&K&KF&K&K&K#K#KKrc(tjd|i}|j|}|dz }td|dD]@}|j||z}|j||zdz}t||dzD]}d||<Atj||S)a Caches and returns a dictionary indicating whether a Unicode character is accepted at a given rule position. This function considers Unicode characters, dynamically inserting accepted ranges into a dictionary to optimize memory usage. Args: - rule_offset: The offset in the grammar encoding where the rule starts. Returns: - A dictionary where each key is a Unicode character (or range) and the value is True if accepted. zelement_offset: r,rrAT)rUrVr"rz)rr9 acceptance num_charsrrstartendjs rchar_acceptance_at_elementz+StringRecognizer.char_acceptance_at_elements  999::: ).9 !q)Q'' % %A).1*<=E'(:Q(>?C5#'** % % $ 1  % j!!!rc g}|D]F}||t|t||G|Sr)r._consume_code_points_per_stackr6)rrZrrYrFr8s r_consume_code_points_newz)StringRecognizer._consume_code_points_newse')   E   33+&&e g     rct|}|f}t|D]4\}}|||}td|D}5d|DS)Nc,g|]}t|Srrords rrfzCStringRecognizer._consume_code_points_per_stack..s===UE%LL===rc,g|]}t|Srrcrds rrfzCStringRecognizer._consume_code_points_per_stack..s000U 000r)rBrprhr6)rrZr8rYrrrr_s rrz/StringRecognizer._consume_code_points_per_stacksz;'' &{33 ? ?MAz--j&AAF==f===>>FF000000r)NNN)T)NT)Fr)#rrrrintrr'r(rr<r>rrr5rOrNr r\rMrhrgrWboolrxrjrXstrrrrrrrrrrrrr!r!s""&"& ++s)++3i + T#Y ++++6S <  T#Y    O+OOOO.k....Yu%5:%$tCy/%%%%N9#9[9999 T#Y" :%) ==="====DYu',U3Z'8 d3i$Yu44&+Cj4 d3i4444DHM  9 .249o  d3i     HM9.249o  Yu  /2      6-!-1<- ----j>c>>>>> 00S0 0000 ++S+ ++++ T#Y    Ld3iLLLLYt2HM  9 .249o  d3i    YuBG 1 1 : 1.3Cj 1 d3i 1 1 1 1 1 1rr!)copyrU functoolsrtypingrrrtransformers_gad.parserrrr r transformers_gad.utf8_utilsr r transformers_gad.utilsr rr!rrrrs $$$$$$$$$$ A@@@@@@@666666........m1m1m1m1m1m1m1m1m1m1r