o
    ॵiu                     @   sz  d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlm	Z	 d dl
mZ d dlmZ d dlmZ d dlmZmZmZ d dlZd dlZd dlZd dlZd dlmZ dd	lmZ G d
d dZG dd deZdejdejfddZd'ddZ dd Z!		d(dejfddZ"		d)dejfddZ#		d*dejfddZ$d+dd Z%		d,d!d"Z&	d+d#d$Z'G d%d& d&Z(dS )-    N)OrderedDict)Mapping)Path)FunctionType)AnyDictUnion)nn   )compare_arguments_nestedc                   @   s   e Zd ZdZ			ddededefddZdd	 Zd
d Ze	j
		ddejdefddZe	j
						ddeeef fddZdS )RegressToolzThis class is used to stop inference/training results from changing by some unaware affections by unittests.

    Firstly, run a baseline test to create a result file, then changes can be observed between
    the latest version and the baseline file.
    Nbaseline
store_func	load_funcc                 C   s(   || _ || _|| _tdt   dS )zPA func to store the baseline file and a func to load the baseline file.
        zCurrent working dir is: N)r   r   r   printr   cwd)selfr   r   r    r   W/home/ubuntu/.local/lib/python3.10/site-packages/modelscope/utils/regress_test_utils.py__init__!   s   zRegressTool.__init__c                 C   s`   | j d ur|  || d S tjtjt ddd}tj|dd t	|tj|| d S )Ndatatest
regressionT)exist_ok)
r   ospathabspathjoinr   r   makedirsshutilcopy)r   localremoter   r   r   r   store,   s   
zRegressTool.storec              	   C   s   | j d ur|  || d S tjtjt ddd}tj||}tj|s1td| dt	d| dt
t|d    tj|rPt| tj||d	d
 d S )Nr   r   r   zbase line file z
 not existzlocal file found:z, md5:rbF)target_is_directory)r   r   r   r   r   r   r   exists
ValueErrorr   hashlibmd5openread	hexdigestremovesymlink)r   r!   r"   r   r   r   r   r   load5   s   
"
zRegressTool.loadTmodule	file_namec                 +   s,   t d}|du s| jdu rdV  dS | j}i }d  d}t|tjs/t|ds,J |j}t| | t	|| dV  t|dddd t	|ddd |ryt
|d}	t||	 W d   n1 sdw   Y  | |  d t | dS t j|}
t jt |
}| ||
 t
|d	}	t|	}W d   n1 sw   Y  G d
d dtj}|rtd  fdd| D }| D ]\}}ddd|d< qƇ fdd| D }| D ]\}}ddd|d< qtdtj||d  tdtj||d  t|||fi |stddS )a  Monitor a pytorch module in a single forward.

        Args:
            module: A torch module
            file_name: The file_name to store or load file
            compare_fn: A custom fn used to compare the results manually.
            compare_model_output: Only compare the input module's output, skip all other tensors

        >>> def compare_fn(v1, v2, key, type):
        >>>     return None

        v1 is the baseline value
        v2 is the value of current version
        key is the key of submodules
        type is in one of 'input', 'output'

            kwargs:
            atol: The absolute gap between two np arrays.
            rtol: The relative gap between two np arrays.
        REGRESSION_BASELINEN./.binmodelTrestorewbr$   c                   @      e Zd Zdd Zdd ZdS )zCRegressTool.monitor_module_single_forward.<locals>.SafeNumpyEncoderc                 S   sJ   t |tjr
| S t |tjrt|S t |tjrt|S tj	
| |S N)
isinstancenpndarraytolistfloatingfloatintegerintjsonJSONEncoderdefaultr   objr   r   r   parse_default   s   zQRegressTool.monitor_module_single_forward.<locals>.SafeNumpyEncoder.parse_defaultc                 S   s4   z|  |W S  ty   td|j d Y d S w )NzType z! cannot be serialized and printed)rE   	Exceptionr   	__class__rF   r   r   r   rE      s   zKRegressTool.monitor_module_single_forward.<locals>.SafeNumpyEncoder.defaultN)__name__
__module____qualname__rH   rE   r   r   r   r   SafeNumpyEncoder}   s    rN   zDIgnore inner modules, only the output of the model will be verified.c                       i | ]\}}| kr||qS r   r   .0keyvaluer1   r   r   
<dictcomp>       z=RegressTool.monitor_module_single_forward.<locals>.<dictcomp>argskwargsinputc                    rO   r   r   rP   rT   r   r   rU      rV   z
baseline: )clsz
latest  : zResult not match!)r   getenvr   r;   r	   Modulehasattrr5   hack_forwardintercept_moduler*   pickledumpr#   r-   r   basenamer   tempfile
gettempdirr/   rC   rD   r   itemsdumpscompare_io_and_printr'   )r   r0   r1   
compare_fncompare_model_outputrY   r   io_jsonabsolute_pathfnamebaserN   rR   rS   r   rT   r   monitor_module_single_forwardE   s`   



z)RegressTool.monitor_module_single_forwardconfigtrainerc	              
   +   s   t d}
|
du s| jdu rdV  dS | j}
i }i }d| d}|dkr)td t|ds6d|v s6J dt|tr?|d n|j}t|tj	sRt|dsOJ |j}t|d	s_d	|v s_J d
t|dsld|v slJ dt|tru|d	 n|j
}t|tr|d n|j}tt }tj }t }t|dr|jn	t|dr|jnd}|rt   fdd  | W d   n1 sw   Y  |dkrt||| t|| t||||d dV  t||ddd |dkrt|dddd t|ddd | }|dd |||jj|j|d|jj| dt|dr!|j nd||||dd}|
rZt |d}t!"|| W d   n	1 sEw   Y  | #|| d t $| dS t j%&|}t j%'t() |}
| *|
| t |
d}t!*|}W d   n	1 sw   Y  |dkrt+|d ||fi |	st,dt-|d |f|||d|	st,d |d	 |d |d |sdn|d d!}|d	 |d |d |sdn|d d!}t.|||fi |	st,d"dS )#ay  Monitor a pytorch module's backward data and cfg data within a step of the optimizer.

        This is usually useful when you try to change some dangerous code
        which has the risk of affecting the training loop.

        Args:
            trainer: A dict or an object contains the model/optimizer/lr_scheduler
            file_name: The file_name to store or load file
            level: The regression level.
            'strict' for matching every single tensor.
                     Please make sure the parameters of head are fixed
                     and the drop-out rate is zero.
            'config' for matching the initial config, like cfg file, optimizer param_groups,
                     lr_scheduler params and the random seed.
            'metric' for compare the best metrics in the evaluation loop.
            compare_fn: A custom fn used to compare the results manually.
            ignore_keys: The keys to ignore of the named_parameters.
            compare_random: If to compare random setttings, default True.
            reset_dropout: Reset all dropout modules to 0.0.
            lazy_stop_callback: A callback passed in, when the moniting is over, this callback will be called.
            kwargs:
            atol: The absolute gap between two np arrays.
            rtol: The relative gap between two np arrays.

        >>> def compare_fn(v1, v2, key, type):
        >>>     return None

        v1 is the baseline value
        v2 is the value of current version
        key is the key of modules/parameters
        type is in one of 'input', 'output', 'backward', 'optimizer', 'lr_scheduler', 'cfg', 'state'
        r2   Nr3   r4   strictz[Important] The level of regression is 'strict', please make sure your model's parameters are fixed and all drop-out rates have been set to zero.r5   zmodel must be in trainer	optimizerzoptimizer must be in trainerlr_schedulerzlr_scheduler must be in trainer_seedseedc                    sB   |   D ]\}}t|tjjrt| |tjd q | qd S )Ng        )named_childrenr;   torchr	   Dropoutsetattr)_modulern   	submodulereinit_dropoutr   r   r     s
   
z8RegressTool.monitor_module_train.<locals>.reinit_dropout)lazy_stop_callbackTr6   state)typedefaults
state_dict)r   r   cfg)torch_statenp_staterandom_seedrw   )forwardbackwardrt   ru   r   r   r8   r$   r   zForward not match!r   )ri   ignore_keyslevelzBackward not match!)rt   ru   r   r   zCfg or optimizers not match!)/r   r\   r   r   r^   r;   dictr5   r	   r]   rt   ru   numpify_tensor_nestedry   get_rng_stater<   random	get_stategetstaterv   rw   no_gradr_   r`   hack_backwardr   poprJ   rK   r   r   to_dictr*   ra   rb   r#   r-   r   rc   r   rd   re   r/   rh   RuntimeErrorcompare_backward_and_printcompare_cfg_and_optimizers)r   rr   r1   r   ri   r   compare_randomreset_dropoutr   rY   r   rk   bw_jsonrl   r0   rt   ru   r   r   r   rw   optimizer_dictsummaryrm   rn   baseline_jsoncfg_opt1cfg_opt2r   r~   r   monitor_module_train   s   
+





z RegressTool.monitor_module_train)NNNNT)rq   NNTTN)rK   rL   rM   __doc__boolr   r   r#   r/   
contextlibcontextmanagerr	   r]   strrp   r   r   r   r   r   r   r   r   r      s@    
	d
r   c                   @   s6   e Zd ZG dd deZej					dddZdS )	MsRegressToolc                   @   s   e Zd ZdS )zMsRegressTool.EarlyStopErrorN)rK   rL   rM   r   r   r   r   EarlyStopErrorV  s    r   rq   NTc           
      +   sV    d u rfdd fdd}	j tj |	__ d V  d S )Nc                     s   G dd d}   |   d S )Nc                   @   s   e Zd ZdZdd Zdd Zdd Zdd	 Zd
d Zdd Z	dd Z
dd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zd d! Zd"d# Zd$d% Zd&d' Zd(d) Zd*d+ Zd,d- Zd.d/ Zd0S )1zQMsRegressTool.monitor_ms_train.<locals>.lazy_stop_callback.<locals>.EarlyStopHookZ   c                 S      d S r:   r   r   rr   r   r   r   
before_runk     z\MsRegressTool.monitor_ms_train.<locals>.lazy_stop_callback.<locals>.EarlyStopHook.before_runc                 S   r   r:   r   r   r   r   r   	after_runn  r   z[MsRegressTool.monitor_ms_train.<locals>.lazy_stop_callback.<locals>.EarlyStopHook.after_runc                 S   r   r:   r   r   r   r   r   before_epochq  r   z^MsRegressTool.monitor_ms_train.<locals>.lazy_stop_callback.<locals>.EarlyStopHook.before_epochc                 S   r   r:   r   r   r   r   r   after_epocht  r   z]MsRegressTool.monitor_ms_train.<locals>.lazy_stop_callback.<locals>.EarlyStopHook.after_epochc                 S   r   r:   r   r   r   r   r   before_iterw  r   z]MsRegressTool.monitor_ms_train.<locals>.lazy_stop_callback.<locals>.EarlyStopHook.before_iterc                 S      |  | d S r:   r   r   r   r   r   before_train_epochz     zdMsRegressTool.monitor_ms_train.<locals>.lazy_stop_callback.<locals>.EarlyStopHook.before_train_epochc                 S   r   r:   r   r   r   r   r   before_val_epoch}  r   zbMsRegressTool.monitor_ms_train.<locals>.lazy_stop_callback.<locals>.EarlyStopHook.before_val_epochc                 S   r   r:   r   r   r   r   r   after_train_epoch  r   zcMsRegressTool.monitor_ms_train.<locals>.lazy_stop_callback.<locals>.EarlyStopHook.after_train_epochc                 S   r   r:   r   r   r   r   r   after_val_epoch  r   zaMsRegressTool.monitor_ms_train.<locals>.lazy_stop_callback.<locals>.EarlyStopHook.after_val_epochc                 S   r   r:   r   r   r   r   r   before_train_iter  r   zcMsRegressTool.monitor_ms_train.<locals>.lazy_stop_callback.<locals>.EarlyStopHook.before_train_iterc                 S   r   r:   r   r   r   r   r   before_val_iter  r   zaMsRegressTool.monitor_ms_train.<locals>.lazy_stop_callback.<locals>.EarlyStopHook.before_val_iterc                 S   r   r:   
after_iterr   r   r   r   after_train_iter  r   zbMsRegressTool.monitor_ms_train.<locals>.lazy_stop_callback.<locals>.EarlyStopHook.after_train_iterc                 S   r   r:   r   r   r   r   r   after_val_iter  r   z`MsRegressTool.monitor_ms_train.<locals>.lazy_stop_callback.<locals>.EarlyStopHook.after_val_iterc                 S      |dkr|j d | dkS dS Nr   r
   F)epochr   rr   nr   r   r   every_n_epochs     z`MsRegressTool.monitor_ms_train.<locals>.lazy_stop_callback.<locals>.EarlyStopHook.every_n_epochsc                 S   r   r   )
inner_iter)r   runnerr   r   r   r   every_n_inner_iters  s   zeMsRegressTool.monitor_ms_train.<locals>.lazy_stop_callback.<locals>.EarlyStopHook.every_n_inner_itersc                 S   r   r   )iterr   r   r   r   every_n_iters  r   z_MsRegressTool.monitor_ms_train.<locals>.lazy_stop_callback.<locals>.EarlyStopHook.every_n_itersc                 S      |j d |jkS Nr
   )r   iters_per_epochr   r   r   r   end_of_epoch     z^MsRegressTool.monitor_ms_train.<locals>.lazy_stop_callback.<locals>.EarlyStopHook.end_of_epochc                 S   r   r   )r   
max_epochsr   r   r   r   is_last_epoch  r   z_MsRegressTool.monitor_ms_train.<locals>.lazy_stop_callback.<locals>.EarlyStopHook.is_last_epochc                 S   r   r   )r   	max_itersr   r   r   r   is_last_iter  r   z^MsRegressTool.monitor_ms_train.<locals>.lazy_stop_callback.<locals>.EarlyStopHook.is_last_iterc                 S   s   g S r:   r   r   r   r   r   get_triggered_stages  r   zfMsRegressTool.monitor_ms_train.<locals>.lazy_stop_callback.<locals>.EarlyStopHook.get_triggered_stagesc                 S   s   i S r:   r   r   r   r   r   r     r   z\MsRegressTool.monitor_ms_train.<locals>.lazy_stop_callback.<locals>.EarlyStopHook.state_dictc                 S   r   r:   r   )r   r   r   r   r   load_state_dict  r   zaMsRegressTool.monitor_ms_train.<locals>.lazy_stop_callback.<locals>.EarlyStopHook.load_state_dictc                 S   s
   t d)NzTest finished.)r   r   r   r   r   r   r     s   
z\MsRegressTool.monitor_ms_train.<locals>.lazy_stop_callback.<locals>.EarlyStopHook.after_iterN)rK   rL   rM   PRIORITYr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   EarlyStopHookh  s2    r   )register_hook)r   )rr   r   r   r   f  s   Iz:MsRegressTool.monitor_ms_train.<locals>.lazy_stop_callbackc              	      s|   j | f d$ z| j|i |W W  d    S  tjy+   Y nw W d    d S 1 s7w   Y  d S )N)ri   r   r   r   )r   train_loop_originr   r   )rr   
args_trainkwargs_train)ri   r   r1   r   rY   r   r   r   r   r   _train_loop  s,   	"z3MsRegressTool.monitor_ms_train.<locals>._train_loop)
train_loopr   r   )
r   rr   r1   r   ri   r   r   r   rY   r   r   )	ri   r   r1   r   rY   r   r   r   rr   r   monitor_ms_trainY  s   M
zMsRegressTool.monitor_ms_train)rq   NNTN)rK   rL   rM   rI   r   r   r   r   r   r   r   r   r   T  s    r   module1module2c                 C   s<   t |  | D ]\}}|j|j dkr dS q	dS )Nr   FT)zip
parametersr   nesum)r   r   p1p2r   r   r   compare_module  s
   r   '  c                    s  zddl m} W n ty   t}Y nw 	 t| t|fr*t fdd|  D S t| tr;t fdd| D S t| t	rLt	 fdd| D S t| t
jr|   } d urpt| k |}t|  k   |}dkrz|jtd	S d
kr|jtd	S |S | S )Nr   ModelOutputBasec                    s   i | ]\}}|t | qS r   r   rQ   kt
clip_value	reductionr   r   rU     s    z)numpify_tensor_nested.<locals>.<dictcomp>c                 3       | ]	}t | V  qd S r:   r   rQ   r   r   r   r   	<genexpr>      
z(numpify_tensor_nested.<locals>.<genexpr>c                 3   r   r:   r   r   r   r   r   r     r   r   )dtypemean)modelscope.outputsr   ImportErrorr   r;   r   r   rf   listtuplery   Tensorcpunumpyr<   wherer   r@   r   )tensorsr   r   r   r   r   r   r   r     s:   

r   c                 C   s   zddl m} W n ty   t}Y nw 	 t| t|fr'tdd |  D S t| tr5tdd | D S t| t	rCt	dd | D S t| t
jrM|  S | S )Nr   r   c                 S   s   i | ]	\}}|t |qS r   detach_tensor_nestedr   r   r   r   rU     s    z(detach_tensor_nested.<locals>.<dictcomp>c                 s       | ]}t |V  qd S r:   r  r   r   r   r   r         z'detach_tensor_nested.<locals>.<genexpr>c                 s   r  r:   r  r   r   r   r   r     r	  )r   r   r   r   r;   r   r   rf   r   r   ry   r  detach)r  r   r   r   r   r    s&   

r  Fr0   c                    s`    fdd}|st | ds| jt| j|| | _| _|r,t | dr.| j| _| `d S d S d S )Nc                    s   | j |i |}rtt|}tt|}tt|}n3tt|ddtt|ddd}tt|ddtt|ddd}tt|ddtt|ddd}||d|d < |S )Nr   r   r   r   r   rW   )rZ   output)forward_originr   r  )r   rX   rY   retr  rk   keep_tensorsrn   r   r   _forward  s@   


zhack_forward.<locals>._forwardr  )r^   r   r   r  )r0   rn   rk   r7   r  r  r   r  r   r_      s   )r_   c                    s`    fdd}|st |ds|jt|j|||_|_|r,t |dr.|j|_|`d S d S d S )Nc                    s     D ]-\}}tt|jddtt|jdddtt|jddtt|jdddd |< q| j|i |}  D ]\}}tt|jddtt|jddd | d< q>d ura  |S )Nr   r  r   r  )r   grad
data_after)named_parametersr   r  r   r  step_origin)r   rX   rY   rn   paramr  rk   r   r0   r   r   _step=  s8   






zhack_backward.<locals>._stepr  )r^   stepr   r   r  )r0   rt   rk   r7   r   r  r   r  r   r   7  s   "r   c                 C   sJ   |   D ]\}} |d ur|d | n|}t| ||| t| ||| qd S )N.)rx   r_   r`   )r0   rk   parent_namer7   rn   	full_namer   r   r   r`   g  s
   r`   c                 K   s  |d u rdd }t |  }t | }|| }|| }td| d|  ||}d}	|D ]}
| |
 }||
 }t|d }t|d }||||
d}|d ur`td|
 d| d	 |	o^|}	n(td
|
 d|d |d fi |os|	}	td
|
 d|d |d fi |o|	}	t|d }t|d }||||
d}|d urtd|
 d| d	 |	o|}	q/td
|
 df||d|o|	}	q/|	S )Nc                  _   r   r:   r   rW   r   r   r   ri   t  r   z(compare_io_and_print.<locals>.compare_fnzunmatched keys: , TrZ   z	input of + compared with user compare_fn with result:
unmatched module z input argsrX   z input kwargsrY   r  z
output of z outputsarg1arg2)setkeysr   intersectionr   r   )r   rk   ri   rY   keys1keys2addedremovedshared_keysmatchrR   v1v2v1inputv2inputresv1outputv2outputr   r   r   rh   q  sn   



rh   c                 K   st  |d u rdd }t |  }t | }|| }|| }	td| d|	  ||}
d}|
D ]}|d ur:||v r:q/|| | || |d}|d urYtd| d| d	 |oW|}q/| | d
 | | d | | d }}}|| d
 || d || d }}}td| df||d|o|}|dkrtd| df||d|o|}td| d||fi |o|}q/|S )Nc                  _   r   r:   r   rW   r   r   r   ri     r   z.compare_backward_and_print.<locals>.compare_fnzunmatched backward keys: r  Tr   zbackward data of r  r   r   r  r  r!  z tensor datar"  rs   z
 grad dataz data after step)r%  r&  r   r'  r   )r   r   r   r   ri   rY   r(  r)  r*  r+  r,  r-  rR   r2  data1grad1data_after1data2grad2data_after2r   r   r   r     sz   








r   c                 K   s  |d u rdd }| d | d | d | d f\}}}}|d |d |d | d f\}}	}
}d}|||d d}|d urFt d| d	 |oD|}n6|d
 |d
 kr\t d|d
  d|d
   td|d |d fi |ok|}td|d |d fi |o{|}|||	d d}|d urt d| d	 |o|}n&|d
 |	d
 krt d|d
  d|	d
   td|d |	d fi |o|}|||
d d}|d urt d| d	 |o|}nt	d||
d|o|}|||d d}|d urt d| d	 |o|}|S td||fi |o|}|S )Nc                  _   r   r:   r   rW   r   r   r   ri     r   z.compare_cfg_and_optimizers.<locals>.compare_fnrt   ru   r   r   Tz4optimizer compared with user compare_fn with result:r   r   zOptimizer type not equal:z and zunmatched optimizer defaultsr   zunmatched optimizer state_dictr   z7lr_scheduler compared with user compare_fn with result:z!unmatched lr_scheduler state_dictz.cfg compared with user compare_fn with result:unmatched cfgr"  z7random state compared with user compare_fn with result:zunmatched random state)r;  )r   r   )r   cfg_jsonri   rY   
optimizer1lr_scheduler1cfg1state1
optimizer2lr_scheduler2cfg2state2r-  r2  r   r   r   r     s   




r   c                   @   r9   )IgnoreKeyFnc                 C   s.   t |tr|g}t |tr|| _d S g | _d S r:   )r;   r   r   r&  )r   r&  r   r   r   r     s   
zIgnoreKeyFn.__init__c                 C   s2   | j D ]}t|}|d ur||r dS qd S r   )r&  recompile	fullmatch)r   r3  r4  rR   r   _keypatternr   r   r   __call__  s   

zIgnoreKeyFn.__call__N)rK   rL   rM   r   rK  r   r   r   r   rE    s    rE  )Nr   )FF)FN)NFr:   )NN))r   r(   r   ra   r   rF  r   rd   collectionsr   collections.abcr   pathlibr   typesr   typingr   r   r   rC   r  r<   ry   torch.optimr	   
test_utilsr   r   r   r]   r   r   r  r_   r   r`   rh   r   r   rE  r   r   r   r   <module>   sT     <t
:2

4
4
B