o g@sddlZddlmmZddlmZmZmZddlm Z ddl m Z ddZ ej dd d d Z  dd d ZejfddZdS)N)get_sequence_parallel_rank get_sequence_parallel_world_size get_sp_group)xFuserLongContextAttention)sinusoidal_embedding_1dcCsB|j\}}}||}tj||||j|jd}tj||gdd}|S)N)dtypedevicerdim)shapetorchonesrr cat)original_tensorZ target_lenseq_lens1s2pad_sizepadding_tensorZ padded_tensorr5/root/Wan2.1/wan/distributed/xdit_context_parallel.py pad_freqs s rF)enabledc Cs|d|d|dd}}}|j|d|d|d|dgdd}g}t|D]\}\}} } || | } t||d|ftj||dd} tj |dd| |ddd || | d|dd|  d| dd || | d|dd|  dd| d || | dgdd| dd} t }t }t| ||} |}| |||d|ddddf}t| |d} t | |||dfg} || q/t|S)zX x: [B, L, N, C]. grid_sizes: [B, 3]. freqs: [M, C // 2]. rr Nr)sizesplit enumeratetolistr view_as_complextofloat64reshaperviewexpandrrr view_as_realflattenappendstackfloat)x grid_sizesfreqssncoutputifhwrx_ifreqs_isp_sizeZsp_rankZ s_per_rankZ freqs_i_rankrrr rope_applys<&( &&&   r:csjdkr|dur |dusJjjj}jj|kr!j|_|dur/ddt||D}fdd|D}tdd|D}dd|D}tj dd|Dtj d } | ks^Jt fd d|D}t jtjd -tj|} | d d jf} | jtjkr| jtjksJWdn1swYd} tfd d|D}|durɈ|} tj| |gd d}t| | |j|| d}tj|td dt}jD] }||fi|}q || }t!j"|d d}#||}dd|DS)z x: A list of videos each with shape [C, T, H, W]. t: [B]. context: A list of text embeddings each with shape [L, C]. i2vNcSs"g|] \}}tj||gddqS)rr )r r).0uvrrr Xs"z#usp_dit_forward..csg|] }|dqS)r)patch_embedding unsqueezer<r=selfrrr?[scSs&g|]}tj|jddtjdqS)rNr)r tensorr longrBrrrr?]s&cSsg|] }|dddqS)rr)r( transposerBrrrr?^scSsg|]}|dqS)r)rrBrrrr?_srEc s8g|]}tj||d|d|dgddqS)rrr )r r new_zerosrrB)rrrr?as*rc s4g|]}t||j|d|dgqS)rr)r rrItext_lenrrBrCrrr?ps&r )eseq_lensr-r.context context_lenscSsg|]}|qSr)r+rBrrrr?s)$ model_typer@weightr r.r"zipr r*rFrGmaxrampautocastfloat32time_embeddingrfreq_dimr+time_projection unflattenr rtext_embeddingimg_embconcatdictchunkrrblocksheadr all_gather unpatchify)rDr,trNrclip_feayr r-rMrLe0rO context_clipkwargsblockr)rDrrusp_dit_forwardBsj         rkc sg|jddjjR\tjtjffdd}fdd}||\}} } t|||}t| ||} td|||| || jd}| d} |}|S)Nrcs|jvr|S|SN)rr")r,)r half_dtypesrrhalfszusp_attn_forward..halfcsX|}|}|}|||fSrl)norm_qqr%norm_kkr>)r,rprrr>)bdr0r/rDrrqkv_fns z usp_attn_forward..qkv_fn)querykeyvalue window_size) r num_headshead_dimr float16bfloat16r:rryr(o) rDr,rMr-r.rrnrurprrr>r)rsrtrrmr0r/rDrusp_attn_forwards"(     r)NN)r Ztorch.cuda.ampcudarTxfuser.core.distributedrrrxfuser.core.long_ctx_attentionrZ modules.modelrrrUr:rkr}rrrrrs   . X