o >g@sddlZddlmZddlZGdddejjZGdddejjZGdddejjZ Gd d d ejjZ Gd d d ejjZ Gd ddejjZ GdddejjZ GdddejjZGdddejjZdS)N)optimized_attention_for_devicec&eZdZfddZdddZZS) BertAttentioncsVt||_|j||d||d|_|j||d||d|_|j||d||d|_dS)NT)biasdtypedevice)super__init__headsLinearquerykeyvalue)self embed_dimr rr operations __class__B/home/comdoleger1/zen-flux-style-shape/comfy/text_encoders/bert.pyr s zBertAttention.__init__NcCs4||}||}||}|||||j|}|SN)r r rr )rxmaskoptimized_attentionqkvoutrrrforwards   zBertAttention.forwardNN__name__ __module__ __qualname__r r __classcell__rrrrrs  rc$eZdZfddZddZZS) BertOutputcs6t|j||||d|_|j||||d|_dS)Nrrepsrr)rr r dense LayerNorm)r input_dim output_dimlayer_norm_epsrrrrrrr s zBertOutput.__init__cCs||}|||}|Sr)r*r+)rryrrrrs zBertOutput.forwardr rrrrr& r&cr%)BertAttentionBlockcs4tt||||||_t|||||||_dSr)rr rrr&output)rrr r.rrrrrrr %s zBertAttentionBlock.__init__cCs||||}|||Sr)rr2rrrrr/rrrr*s zBertAttentionBlock.forwardr rrrrr1$s r1cr%)BertIntermediatecs"t|j||||d|_dS)Nr')rr r r*)rrintermediate_dimrrrrrrr /s zBertIntermediate.__init__cCs||}tjj|Sr)r*torchnn functionalgelu)rrrrrr3s zBertIntermediate.forwardr rrrrr4.s r4cr%) BertBlockcsHtt|||||||_t||||||_t|||||||_dSr)rr r1 attentionr4 intermediater&r2)rrr5r r.rrrrrrr 9s zBertBlock.__init__cCs$||||}||}|||Sr)r;r<r2r3rrrr?s  zBertBlock.forwardr rrrrr:8r0r:cr) BertEncoderc s:ttjfddt|D|_dS)Nc s g|] }tqSr)r:).0irrrr r5r.rrr Gs z(BertEncoder.__init__..)rr r6r7 ModuleListrangelayer) r num_layersrr5r r.rrrrr@rr Es 0zBertEncoder.__init__NcCsnt|j|dudd}|dur|dkrt|j|}d}t|jD]\}}||||}||kr2|}q ||fS)NT)r small_inputr)rrlenrD enumerateclone)rrrintermediate_outputrr<r?lrrrrIs zBertEncoder.forwardrr rrrrr=Ds r=cr)BertEmbeddingsc s`t| j|||||d|_| j||||d|_| j||||d|_| j||||d|_dS)N) padding_idxrrr'r()rr Embeddingword_embeddingsposition_embeddingstoken_type_embeddingsr+) r vocab_sizemax_position_embeddingstype_vocab_size pad_token_idrr.rrrrrrr Xs zBertEmbeddings.__init__NcCsx|j||d}|tj|jjd|jd|7}|dur(||j||jd7}n |tj|jjd|7}| |}|S)N) out_dtyper) rOcomfyops cast_to_inputrPweightshaperQrr+)r input_tokenstoken_type_idsrrrrrr`s$ zBertEmbeddings.forwardrr rrrrrLW rLcs&eZdZfddZdddZZS) BertModel_c slt|d}|d}t|d|d|d|d||||| |_t|d||d|d |||||_dS) N hidden_sizer.rRrSrTrUnum_hidden_layersintermediate_sizenum_attention_heads)rr rL embeddingsr=encoder)r config_dictrrrrr.rrrr ls *(zBertModel_.__init__NTc Cs|j||d}d}|dur=d||j|jddd|jdf|jdd|jd|jd}||tjt d}| |||\}}||fS)N)rg?rrWz-inf) retorreshaper\expand masked_fillr6boolfloatrf) rr]attention_maskrJfinal_layer_norm_intermediaterrrr?rrrrtsHzBertModel_.forward)NNTNr rrrrr`kr_r`cs4eZdZfddZddZddZddZZS) BertModelcs(tt|||||_|d|_dS)Nrb)rr r`bertrE)rrgrrrrrrr s zBertModel.__init__cCs |jjjSrrrrerO)rrrrget_input_embeddingss zBertModel.get_input_embeddingscCs||jj_dSrrs)rrerrrset_input_embeddingsszBertModel.set_input_embeddingscOs|j|i|Sr)rr)rargskwargsrrrrszBertModel.forward)r!r"r#r rtrurr$rrrrrqs  rq)r6comfy.ldm.modules.attentionr comfy.opsrXr7Modulerr&r1r4r:r=rLr`rqrrrrs