o h2hc@sJddlZddlmZddZddZddZddd Zd d Zd d Z dS)NcCsts tjdddSdS)z*r initialize sequence parallel group. nccl)backendN)distis_initializedinit_process_grouprr6/home/ubuntu/wan22/wan2.2-main/wan/distributed/util.pyinit_distributed_groupsr cCtSN)rget_rankrrrrr r cCr r )rget_world_sizerrrrrr rcKsbt}|dkr/dd|j||dD}dd|D}tj||fd|i|tj||d}|S)zC `scatter` along one dimension and `gather` along another. cSsg|]}|qSr) contiguous.0urrr szall_to_all..dimcSsg|]}t|qSrtorch empty_likerrrrrgroup)rchunkr all_to_allrcatr)x scatter_dim gather_dimrkwargs world_sizeinputsoutputsrrrrsrcs>t}|dkr gSfddt|D}tj||S)Nrcsg|]}tqSrr)r_tensorrrr&rzall_gather..)rrranger distributed all_gather)r(r# tensor_listrr'rr+"s r+cCs.t}|dkr |St|}tj||dS)Nrr)rrr+rrr)inputrr#outputrrrgather_forward+s r/r ) rtorch.distributedr*rr r rrr+r/rrrrs