U 6fe@sddlZddlZddlmZddlmmZddlZddl Z e j ej e ddlmZddlZddlZddlZddlZddZddZGdd d eZGd d d eZGd d d eZddZd ddZddZddZdefddZ e!dkr|e dddd\Z"Z#ddl$Z$e$$Z%e&dD]VZ'e(e"D]F\Z)Ze\Z*Z+e)ddkr0e$$Z,e-e'e)e+.e,e%de,Z%q0q$dS)!N)Dataset)read_and_config_filecCsNt|\}}t|}|dkr,t||d}t|jdkrJ|dddf}|S)N>r)sfread audio_normlibrosaZresamplelenshape)pathdatafsr/mnt/nas_sg/mit_sg/shengkui.zhao/ComplexNN/ComplexNN_16k_V5_UniDFSMN_W40S20_multiUnetx2_residual_mask_Ch_FSMN_full_DNS_Challenge_reverb_20211123/FRCRN_16k_github/tools/time_dataset.py audioreadsrcCsX|dd}d|}||}|d}|}|||kd}d|}||}|S)Ng?g&-`ʬ?)mean)xZrmsscalarZpow_xZ avg_pow_xZrmsxZscalarxrrrrsrc@s,eZdZddZddZddZddZd S) DataReadercCst|dd|_dS)NT)decode)r file_list)self input_pathrrr__init__(szDataReader.__init__cCsD|dd}t|tj}t|d|jdg}|||jdfS)N/rr)splitrastypenpfloat32reshaper )rr Zutt_idr inputsrrrextract_feature+szDataReader.extract_featurecCs t|jSN)r rrrrr__len__2szDataReader.__len__cCs||j|Sr%)r$r)rindexrrr __getitem__5szDataReader.__getitem__N)__name__ __module__ __qualname__rr$r'r)rrrrr'src@seZdZddZdS) Processerc Cst|d}t|d}|jd}|jd|krvtj|tjd}tj|tjd}||d|jd<||d|jd<n0td||} || | |}|| | |}||fS)Nr#labelsrdtype)rr r zerosr!randomrandint) rr start_timesegement_lengthZ wave_inputsZwave_s1Zlen_wavZ padded_inputsZ padded_s1Zst_idxrrrprocess;s  zProcesser.processN)r*r+r,r6rrrrr-9sr-c@s0eZdZddedfddZddZddZdS) TimeDatasetrNcCs@t||_||_t}||_||_t|j|j|j|dS)a scp_file_name: the list include:[input_wave_path, output_wave_path, duration] spk_emb_scp: a speaker embedding ark's scp segement_length: to clip data in a fix length segment, default: 4s sample_rate: the sample rate of wav, default: 16000 processer: a processer class to handle wave data gender2spk: a list include gender2spk, default: None N) rwav_list processermpManagerlistr(r5_dochunk)r scp_file_namer5 sample_rater9Z gender2spkZmgrrrrr\s   zTimeDataset.__init__cCs t|jSr%)r r(r&rrrr'sszTimeDataset.__len__cCs:|j|\}}|j|d|dd||j\}}||fS)Nr#r.)r#r.)r(r9r6r5)rr( data_infor4r#s1rrrr)vs$zTimeDataset.__getitem__)r*r+r,r-rr'r)rrrrr7Zs r7c Cs|||D]}|d}t||}||krNd} |d|kr>q ||dgq d} | ||krv||| g| |7} qR| |kr ||t||gq dS)Ndurationrrr)intappend) Z target_listZ result_liststartendr5r?itemrBlengthZ sample_indexrrrworker{s&     rI c Csg}t||}|dkrJtjt||dt|||fd}|||n`t|D]V}||dkrlt|} n |d|} tjt||||| ||fd}|||qR|D] }|qdS)Ndr)targetargsr)r r:ProcessrIrErDrangejoin) r8r(r5r? num_threadsZpc_liststridepidxrFrrrr=sB         r=cCsptdd|D}tt|||djdg}t|tj}t|D]$\}}|||d|jdddf<qF|S)Ncss|]}|jdVqdS)rN)r ).0inprrr sz"zero_pad_concat..rr)maxr arrayr r r1r! enumerate)r#Zmax_tr Z inputs_matrTrVrrrzero_pad_concats r[cCs@t|\}}tj|tjd}tj|tjd}t|t|fS)Nr/)zipr rYr!torch from_numpy)r r#rArrr collate_fns r_cCs*t||d}tj|||tddd}||fS)N)r9TF) batch_size num_workersr_shuffle drop_last)r7tud DataLoaderr_)r>r`rar9datasetloaderrrr make_loaders rh__main__z../data/cv_wsj0_-5~20.lst )ra rK)rJ)/numpyr r]torch.utils.datarutilsr rdossysr rDdirname__file__miscrmultiprocessingr: soundfilerr r2rrobjectrr-r7rIr=r[r_rhr*Zlaoder_timeZstimerOepochrZrTr#r.Zetimeprintsizerrrrs@    !! *