AudioGPT/NeuralSeq/tasks/__pycache__/base_task.cpython-37.pyc

B
<00>Xd<>.<00>@s
dd
lZdd
l
Z
dd
lZddlmZ
dd
lZe<04>d<03>

ddlmZmZ
dd
l	Z	dd
l
Z
dd
lZdd
l
mZ
ddlmZ
ddlmZmZmZmZ
ddlmZ
dd
lZdd
lZdd
lZdd
lZej<1D>e<1C>dd	<09><02>

d
Z ej!e
j"ej#e dd<0C>
Gd
d<0E>dejj$j%<25>Z&Gdd<10>dej'<27>Z(d
S)<11>N)
<01>datetime<6D>Agg)<02>hparams<6D>set_hparams)
<01>TensorBoardLogger)<04>LatestModelCheckpoint<6E>BaseTrainer<65>data_loader<65>DDP)
<01>nnZTORCH_SHARE_STRATEGY<47>file_systemz%(asctime)s %(message)sz%m/%d %I:%M:%S %p)<04>stream<61>level<65>format<61>datefmtcsdeZ
dZ<02>f
d
d<02>Zedd<04><00>
Zdd<06>Zdd<08>Zd	d
<EFBFBD>Zdd<0C>Z	d
d<0E>Z
dd<10>Zedd<12><00>
Z<0C>Z
S)<13>BaseDatasetcs*t<00><00>
<01>
t|_|
|_td
|_d|_dS)N<>sort_by_len)<06>super<65>__init__r<00>shuffler<00>sizes)<02>selfr)
<01>	__class__<5F><00>R/mnt/sdc/hongzhiqing/code/audio_chatgpt/text_to_sing/DiffSinger/tasks/base_task.pyrs







zBaseDataset.__init__c


Cs|jS)
N)
r)
rrrr<00>_sizes&szBaseDataset._sizesc
Cst<00>
dS)
N)
<01>NotImplementedError)r<00>indexrrr<00>__getitem__*s
zBaseDataset.__getitem__c
Cst<00>
dS)
N)
r)r<00>samplesrrr<00>collater-s
zBaseDataset.collaterc

Cs
t|j
<01>
S)
N)<02>lenr)
rrrr<00>__len__0s
zBaseDataset.__len__cCs
|<00>|
<01>
S)
N)
<01>size)rrrrr<00>
num_tokens3s
zBaseDataset.num_tokenscCst|j
|
td
<00>}|S)z{Return an example's size as a float or tuple. This value is used when
        filtering a dataset with ``--max-positions``.<2E>
max_frames)<03>minrr)rrr#rrrr#6s
zBaseDataset.sizec
CsN|jr<t
j<02>t|<00>
<01>
}
|jrJ|
t
jt
<01>|j<08>
|
d
d<02>}
nt
<01>	t|<00>
<01>
}
|
S)z[Return an ordered list of indices. Batches will be constructed based
        on this order.<2E>	mergesort)
<01>kind)
r<00>np<6E>random<6F>permutationr!r<00>argsort<72>arrayr<00>arange)r<00>indicesrrr<00>ordered_indices<s


 
zBaseDataset.ordered_indicesc

Cstt
<01>d
td<00><02>
S)NZNUM_WORKERSZ
ds_workers)<04>int<6E>os<6F>getenvr)
rrrr<00>num_workersHszBaseDataset.num_workers)<0E>__name__<5F>
__module__<EFBFBD>__qualname__r<00>propertyrrr r"r$r#r0r4<00>
__classcell__rr)
rrrs
rcsP
eZ
dZ<02>f
d
d<02>Zdd<04>ZdOdd	<09>
Zd
d<0B>Zdd
<0A>ZdPdd<10>
Zdd<12>Z	dd<14>Z
dd<16>Zdd<18>Zdd<1A>Z
dd<1C>Zdd<1E>Zdd <20>Zd!d"<22>Zd#d$<24>Zd%d&<26>Zed'd(<28><00>
Zd)d*<2A>Zd+d,<2C>Zd-d.<2E>Zed/d0<64><00>
Zed1d2<64><00>
Zed3d4<64><00>
Zd5d6<64>Zd7d8<64>Zd9d:<3A>Zd;d<<3C>Z d=d><3E>Z!d?d@<40>Z"dAdB<64>Z#dCdD<64>Z$dEdF<64>Z%dGdH<64>Z&dIdJ<64>Z'dKdL<64>Z(dMdN<64>Z)<29>Z*S)Q<>BaseTaskc
s<>tt
|<00>j|
|<02>

d
|_d
|_i|_d|_d|_d|_d|_	d|_
d|_td|_
td|_td|_|jdkr<>|j
td<|_td|_|jdkr<>|jtd<|_d|_d|_dS)Nr
F<>
max_tokens<EFBFBD>
max_sentences<65>max_eval_tokens<6E><73><EFBFBD><EFBFBD><EFBFBD><EFBFBD>max_eval_sentences)rr:r<00>
current_epoch<63>global_stepZloaded_optimizer_states_dict<63>trainer<65>loggerZon_gpuZuse_dpZuse_ddpZexample_input_arrayrr;r<r=r?<00>model<65>training_losses_meter)r<00>args<67>kwargs)
rrrrNs(






















zBaseTask.__init__c


Cst<00>
dS)
N)
r)
rrrr<00>build_modeljs
zBaseTask.build_modelNrDTcCs(|dkr|}t<00>
|<00>|<02>
|
|||<05>
dS)
N)<03>utils<6C>	load_ckpt<70>__getattr__)r<00>
ckpt_base_dirZcurrent_model_name<6D>
model_name<EFBFBD>force<63>strictrrrrJms

zBaseTask.load_ckptc

Csd
t<00>
<01>i
|_dS)N<>
total_loss)rI<00>AvgrageMeterrE)
rrrr<00>on_epoch_startss
zBaseTask.on_epoch_startc
Cst<00>
d
S)zu

        :param sample:
        :param batch_idx:
        :return: total loss: torch.Tensor, loss_log: dict
        N)
r)r<00>sample<6C>	batch_idx<64>
optimizer_idxrrr<00>_training_stepvszBaseTask._training_stepr>cCs<>|<00>|
||<03>}||_
|dkr$d
di
S|\}}t<02>|<06>
}xF|<06><04>D]:\}}||jkr`t<02><06>|j|<t<07>|<08>
s@|j|<00>	|<08>

q@W|jd<00>	|<05>
<EFBFBD><00>

y0|j<0B><0C>|d<t
|dt<0E>r<>|dd|d<Wn


YnX|}	dd<06>|<06><04>D<00>
}
||	|
d<07>S)N<>lossrP<00>lrr
c
Ssi|]\}
}|d|
<01><00><02>qS)
ztr/r)<03>.0<EFBFBD>
k<>
vrrr<00>
<dictcomp><3E>sz*BaseTask.training_step.<locals>.<dictcomp>)rWZprogress_bar<61>log)rVZopt_idxrI<00>tensors_to_scalars<72>itemsrErQr)<00>isnan<61>update<74>item<65>	scheduler<65>get_lr<6C>
isinstance<EFBFBD>list)rrSrTrUZloss_retrP<00>log_outputsrZr[Zprogress_bar_logZtb_logrrr<00>
training_steps.






















zBaseTask.training_stepcCs4|<03><00>
|<03>
<01>
|jdk	r0|j<02>|jtd
<00>

dS)N<>accumulate_grad_batches)<05>step<65>	zero_gradrcrAr)r<00>epochrT<00>	optimizerrUrrr<00>optimizer_step<65>s




zBaseTask.optimizer_stepc
Cs8d
d<02>|j<00>
<01>D<00>
}
td|j<03>d|j<04>d|
<01>d<06><07>

dS)Nc
Ssi|]\}
}t|j
d<00>|
<01>qS)
<01>)<02>round<6E>avg)rYrZr[rrrr\<00>sz)BaseTask.on_epoch_end.<locals>.<dictcomp>z
==============
 Epoch z ended. Steps: z. z
==============
)rEr_<00>printr@rA)rZloss_outputsrrr<00>on_epoch_end<6E>s

zBaseTask.on_epoch_endc
Cst<00>
d
S)zY

        :param sample:
        :param batch_idx:
        :return: output: dict
        N)
r)rrSrTrrr<00>validation_step<65>szBaseTask.validation_stepc
Cst<00>
d
S)zE

        :param outputs:
        :return: loss_output: dict
        N)
r)r<00>outputsrrr<00>_validation_end<6E>szBaseTask._validation_endcCs6|<00>|
<01>
}t
d
|<02>d<02><03>

dd<04>|<02><02>D<00>
|dd<06>S)Nz 
==============
 valid results: z
==============
c
Ssi|]\}
}|d|
<01><00><02>qS)
zval/r)rYrZr[rrrr\<00>sz+BaseTask.validation_end.<locals>.<dictcomp>rP)r]<00>val_loss)rvrrr_)rru<00>loss_outputrrr<00>validation_end<6E>s



zBaseTask.validation_endc
Cst<00>
dS)
N)
r)rrmrrr<00>build_scheduler<65>s
zBaseTask.build_schedulerc
Cst<00>
dS)
N)
r)rrDrrr<00>build_optimizer<65>s
zBaseTask.build_optimizerc
Cs|<00>|j
<01>
}
|<00>|
<01>
|_|
g
S)
N)r{rDrzrc)rZoptmrrr<00>configure_optimizers<72>s


zBaseTask.configure_optimizersc


CsdS)
Nr)
rrrr<00>
test_start<EFBFBD>s
zBaseTask.test_startcCs|<00>|
|<02>S)
N)
rt)rrSrTrrr<00>	test_step<65>s
zBaseTask.test_stepcCs
|<00>|
<01>
S)
N)
ry)rrurrr<00>test_end<6E>s
zBaseTask.test_endc

CsP
t<00>
t
t<02>d
d<02><02>
tjd<t<02>td<00>

tj<02>td<00>

|<00>}
td}t	t
|dddtd	td
tdrldnd
d<0E>t|ddd<11>tdtdtdtdtds<>tdndtdd<1A>}td<00>
sBt<0C>
<0A><00>d<1C>
}|<02>d|<04><00>}tjd|<05>d<1F>dd <20>
x,td!D] }tjd"|<06>d#|<05>d$<24>dd <20>
q<>Wtd%|<05>d&<26><03>

|
|j_|<03>|
<01>

n
|<03>|
<01>

dS)'Ni<4E>:i0u<00>MASTER_PORT<52>seed<65>work_dirTrwr&<00>
num_ckpt_keep<65>	save_bestZ	save_ckpt<70>
i<><69>
)<07>filepath<74>verbose<73>monitor<6F>moder<65>r<><00>periodZlightning_logsZlastest)Zsave_dir<69>name<6D>version<6F>clip_grad_norm<72>val_check_intervalZlog_interval<61>max_updates<65>validate<74>num_sanity_val_stepsi'ri)<08>checkpoint_callbackrCZgradient_clip_valr<6C>Zrow_log_intervalr<6C>r<>ri<00>inferz%Y%m%d%H%M%Sz/codes/z
mkdir -p "<22>
")
<01>shellZ
save_codeszcp -r "z" "z/"z| Copied codes to <20>
.)r<00>strr*<00>randintr2<00>environr<6E>rr)rrrr<00>now<6F>strftime<6D>
subprocess<EFBFBD>
check_callrrr<><00>task<73>fit<69>test)<07>clsr<73>r<>rB<00>
tZcode_dir<69>
crrr<00>start<72>sF




























 


zBaseTask.startcCs`t|
|d
d<02>}
t
<01><02>dkr>tds>ttjd<05>t_ttjd<05>t_	t
<EFBFBD>td<00>

tj
<EFBFBD>td<00>

|
S)NT)<02>
device_ids<EFBFBD>find_unused_parametersr
<00>debug<75>
wr<77>)
r
<00>dist<73>get_rankr<00>openr2<00>devnull<6C>sys<79>stdout<75>stderrr*r<>r))rrDr<>rrr<00>
configure_ddp
s








zBaseTask.configure_ddpc

OsdS)
Nr)rrFrGrrr<00>training_end
s
zBaseTask.training_endcCsntd
d<02>

d}yt
jd}Wn"tk
r>


t|<03>
t
jd<YnXd}|j<05>|<04>
}|t
jd<tjd|
|d<08>
dS)	NF)
<01>
print_hparamsin2r<>z	127.0.0.2<EFBFBD>MASTER_ADDR<44>nccl)<02>rank<6E>
world_size)	rr2r<><00>	Exceptionr<6E>rBZresolve_root_node_addressr<73><00>init_process_group)rZ	proc_rankr<6B><00>default_portZ	root_noderrr<00>init_ddp_connection
s








zBaseTask.init_ddp_connectionc


CsdS)
Nr)
rrrr<00>train_dataloader%
szBaseTask.train_dataloaderc


CsdS)
Nr)
rrrr<00>test_dataloader)
szBaseTask.test_dataloaderc


CsdS)
Nr)
rrrr<00>val_dataloader-
szBaseTask.val_dataloaderc
CsdS)
Nr)r<00>
checkpointrrr<00>on_load_checkpoint1
s
zBaseTask.on_load_checkpointc
CsdS)
Nr)rr<>rrr<00>on_save_checkpoint4
s
zBaseTask.on_save_checkpointc


CsdS)
Nr)
rrrr<00>on_sanity_check_start7
s
zBaseTask.on_sanity_check_startc


CsdS)
Nr)
rrrr<00>on_train_start:
s
zBaseTask.on_train_startc


CsdS)
Nr)
rrrr<00>on_train_end=
s
zBaseTask.on_train_endc
CsdS)
Nr)r<00>batchrrr<00>on_batch_start@
s
zBaseTask.on_batch_startc


CsdS)
Nr)
rrrr<00>on_batch_endC
s
zBaseTask.on_batch_endc


CsdS)
Nr)
rrrr<00>on_pre_performance_checkF
s
z!BaseTask.on_pre_performance_checkc


CsdS)
Nr)
rrrr<00>on_post_performance_checkI
s
z"BaseTask.on_post_performance_checkc
CsdS)
Nr)rrmrrr<00>on_before_zero_gradL
s
zBaseTask.on_before_zero_gradc


CsdS)
Nr)
rrrr<00>on_after_backwardO
s
zBaseTask.on_after_backwardcCs|
<01><00>
dS)
N)
<01>backward)rrWrmrrrr<>R
s
zBaseTask.backwardc		Cs<>i}d
}x<>|<00><00>D]z\}}|j
ryV|jj<03>|
<01>
}|||
7}|d|
}t|j<03><06><00><07><00><08>d
d<03>}||d<04>	|
|<04><Wqt
k
r<EFBFBD>


YqXqW|d|
}t|j<03><06><00><07><00><08>d
d<03>}||d<06>	|
<01>
<|S)Nr
r<><00>zgrad_{}_norm_{}g<00>?zgrad_{}_norm_total)<0B>named_parameters<72>
requires_grad<61>grad<61>data<74>normrp<00>cpu<70>numpy<70>flattenrr<>)	r<00>	norm_type<70>results<74>
total_normr<EFBFBD><00>
pZ
param_normr<EFBFBD>r<>rrr<00>	grad_normU
s 













zBaseTask.grad_norm)NrDTT)
r>)+r5r6r7rrHrJrRrVrhrnrsrtrvryrzr{r|r}r~r<00>classmethodr<64>r<>r<>r<>r	r<>r<>r<>r<>r<>r<>r<>r<>r<>r<>r<>r<>r<>r<>r<>r<>r9rr)
rrr:MsJ

	
	
)
r:))<29>glob<6F>rer<65>r<00>
matplotlib<EFBFBD>use<73>
utils.hparamsrrr*r<>r<>r)<00>torch.distributed<65>distributedr<64>Zpytorch_lightning.loggersr<00>utils.pl_utilsrrr	r
<00>torchr<00>torch.utils.datarI<00>loggingr2<00>multiprocessing<6E>set_sharing_strategyr3Z
log_format<EFBFBD>basicConfigr<67><00>INFOr<4F><00>Datasetr<00>Moduler:rrrr<00><module>
s.

















/