o g8G@sddlZddlZddlmZddlmZddlmZddlm Z ddl m Z ddl m ZddlmZddlZddlmZddlZdd lmZddlZddlZejZej jd d d gd ZddgZdedefddZdefddZ GdddZ!GdddZ"GdddZ#Gdddej$j%j&Z'dd Z(e)d!kr gd"Z*d#d$gZ+d%d&gZ,e'e+e,e*d'd(d)d'd*ddd*d*d+d,Z-ddl.Z.e./e-dZ0ddl1Z1e11Z2ee-d-d d*d e(d.Z3dZ4ee3D]%Z5e5d/e5d0e5d1Z6Z7Z8e11e2Z9e:d2e9ddl.Z.e./qe:d3e4dSdS)4N)TextData) FloatData)LrcData)Base) LanceReader) DataLoader)tqdm) pad_sequencespawnT)force)u 纯音乐u编曲u作词u作曲u调音u 制作人u 录音师ZmusicendtimelyriccCsX|dkrdStD] }||vrdSqtD] }||kr!dSqt|dkr*dSdS)Ng?FrT)filter_keyword_listfilter_full_liststriplowerlen)r rZfilter_keywordZ filter_fullr/mnt/sfs/music/hkchen/workspace/F5-TTS-HW/exps/styleprompt_f5_llama_align_rand_start_pure_music/f5_tts/model/custom_dataset_lrc_emb.py check_lyric&s rlyricscCsg}|}|dD]9}z2|dd|dd}}|}|d\}}t|dt|}t||s7Wq |||fWq Yq |S)N  :<)rsplitintfloatrappend)rZlyrics_with_timeliner rminssecsrrr parse_lyrics4s r%c@eZdZddZddZdS)TiktokenTokenizercCstd|_dS)Ngpt2)tiktokenZ get_encoding tokenizerselfrrr__init__HszTiktokenTokenizer.__init__cCs|j|}|SN)r*encode)r,texttokenrrrr/J zTiktokenTokenizer.encodeN__name__ __module__ __qualname__r-r/rrrrr'G r'c@r&) CNENTokenizercCdSr.rr+rrrr-PzCNENTokenizer.__init__cCst|\}}|Sr.)Z chn_eng_g2pr,r0Zphoner1rrrr/Rr2zCNENTokenizer.encodeNr3rrrrr8Or7r8c@r&)MaskGCTTokenizercCr9r.rr+rrrr-Xr:zMaskGCTTokenizer.__init__cCst|dd\}}|S)Nauto)Ztext_tokenizertokenizer;rrrr/ZszMaskGCTTokenizer.encodeNr3rrrrr<Wr7r<c @seZdZedddddddddddf d eed eed eefd d ZddeedefddZddZ ddZ ddZ ddZ dS)LanceDiffusionDatasetrDTFr(fp16 prompt_lances lrc_lances latent_lancescCs||_||_||_| |_||_||_||_d|_||||_g|_ i|_ | |jt dd| |jt d|jrA| |jt dd|_d|_d|_d |_| rS| rSJ||_| |_| |_| |_| |_t|j |_ |d krstj|_n|d kr|tj|_n|d krtj|_td dS)NrAlrcTlatentpromptrricrCbf16fp32*) rDrErFuse_style_prompt max_frames min_frames sampling_ratedownsample_ratemax_secslance_connectionsfile_lst load_lancerr pad_token_idcomma_token_idperiod_token_idstart_token_idr% align_lyrics lyrics_sliceskip_empty_lyrics lyrics_shiftlistvaluestorchfloat16 feature_dtypebfloat16float32randomseed)r,rDrErFrOrPrQrRr%r[r\r]rNr^tokenizer_type precisionrrrr-`sB    zLanceDiffusionDataset.__init__ lance_paths target_clsc Cst|j}g}t|D]K\}}t||d} | } t| d|d|dD]+} |j| jd} | dur@| ||f|j| j|<q%|rO|| ||fi|j| j<q% q%| | q |j |dS)uI lance_paths是路径, target_cls为对应类别 )rkzload z, idx )descN) rrT enumeraterZget_idsrrUgetZdata_idr!extend) r,rjrkZ target_fieldcreateZconnection_offset connectionsidxlpreaderidsiitemrrrrVs   z LanceDiffusionDataset.load_lancec CsRz||\}}|j||jgd}W|Sty(}zWYd}~dSd}~ww)Nr)rTZget_datas_by_rowidsZ_rowid Exception)r,rwfieldZ reader_idxerrr load_items zLanceDiffusionDataset.load_itemcs|d}|j}|j}tt||}jr"t|dkr"td||d}t |j d}j rN|d}t |j  d}|dd}ntdd}td|jd d }ttd jj|}td|} | jj|dd| df}d } | } fd d |D}fdd |D}g} tt|D]%} || \}}| dkr|dkr| dtjgf|}| ||fq| }t|dkr|d dnd }t|dkr|dd n|}tjjftjd}d}d}|D]M\}}fdd |Djg}tj|tjd}|jd}t|jj}d}t|||}t|j|}|d|||||<||7}||}qjrX|dkrXtd|d||dddt|jjf}|j }|j }|||| fS)NrGrzEmpty or short lyrics: rHrrIrirBrAcs(g|]\}}|dkr||fqS)rr.0 time_startr" start_timerr s(z4LanceDiffusionDataset.get_triple..cs"g|] \}}|jkr||fqSr)rSr}r+rrrs"g)dtypecs g|] }|jkr |njqSr)rYrX)r~r1r+rrrs 2zEmpty or short lyrics(z): )!r{r rGr_zipr]r ValueErrorra from_numpydatacopysqueezerN transposezerosmaxshapeminrrQrRrfrandintranger!tensorrZrOlongrYtorc)r,rwZtime_lrcZ input_timesZ input_lrcsZ lrc_with_timerHrIZmax_start_frameZ start_frameZmax_start_timeZnormalized_start_timeZmodified_lrc_with_timervr r"Z line_tokenZlatent_end_timerGZ tokens_countZ last_end_posrtokens num_tokensZgt_frame_start frame_shiftZ frame_startZ frame_lenr)r,rr get_triplesh          $   z LanceDiffusionDataset.get_triplec Cs~|} z||j|\}}}}|jd|jkr||||d}|WSty>}ztd|d}WYd}~qd}~ww)NTrB)rIrGrHrrr)rrUrrPrxrfr__len__) r,indexrrrIrGrHrrwrzrrr __getitem__s z!LanceDiffusionDataset.__getitem__cCs t|jSr.)rrUr+rrrr*s zLanceDiffusionDataset.__len__N)F) r4r5r6MAX_INTr_strr-rrVr{rrrrrrrr?_s $8X r?cCsdd|D}dd|D}dd|D}dd|D}tdd|D}tdd|D}tdd|D}|}|} |} g} |D]} tjj| d || jd f} | | qJg}|D]}tjj|d | |jd f}||qeg}|D]}tjj|d | |jd f}||qg}|D] }|}||qt| }t|}t|}t |}|||||||d S) NcSg|]}|dqS)rHrr~rwrrrr.z%custom_collate_fn..cSr)rIrrrrrr/rcSr)rGrrrrrr0rcSrrrrrrrr1rcSg|]}|jdqSrBr)r~rHrrrr3cSrrr)r~rIrrrr4rcSrrr)r~rGrrrr5rrrB)rIrGrHprompt_lengths lrc_lengthslatent_lengthsr) ra LongTensoramaxnn functionalpadrr!stackr)batchZ latent_listZ prompt_listZlrc_listZstart_time_listrrrZmax_prompt_lengthZmax_latent_lengthZmax_lrc_lengthZpadded_prompt_listrIZ padded_promptZpadded_lrc_listrGZ padded_lrcZpadded_latent_listrHZ padded_latentZpadded_start_time_listrZpadded_start_timeZ prompt_tensorZ lrc_tensorZ latent_tensorZstart_time_tensorrrrcustom_collate_fn-sD        r__main__)z!/mnt/sfs/music/lance/latent-lancez5/mnt/sfs/music/lance/latent-lance-cnen-music-second-1z5/mnt/sfs/music/lance/latent-lance-cnen-music-second-2z%/mnt/sfs/music/lance/style-lance-fullz2/mnt/sfs/music/lance/style-lance-cnen-music-secondz'/mnt/sfs/music/lance/lrc-lance-emb-fullz*/mnt/sfs/music/lance/lrc-lance-cnen-secondrAdr@rrZmaskgct)dataset batch_sizeshuffle num_workers pin_memory collate_fnrIrGrHz#time cost on preparing dataloader: zcnt: );ra torch_npuZ%f5_tts.model.alsp_lance.data.textdatarZ$f5_tts.model.alsp_lance.data.npydatarZ$f5_tts.model.alsp_lance.data.lrcdatarZ!f5_tts.model.alsp_lance.data.baserZf5_tts.model.alsp_lance.toolsrtorch.multiprocessingmultiprocessingmptorch.utils.datarrfrr)Ztorch.nn.utils.rnnr tracebacksysmaxsizerset_start_methodrrr rrr%r'r8r<utilsrDatasetr?rr4Z latent_pthZ prompt_pthZlrc_pthlddpdb set_traceZhhhr r dataloadercntrrIrGrHend_timeprintrrrrsn          O .C"