o
    ٷij                     @   s   d dl Z d dlZd dlmZmZmZmZmZ d dlZd dl	m
Z ddlmZ ddlmZ d dlmZ eeZG dd deZG d	d
 d
ZdS )    N)DictOptionalTupleUnionList   )CalibratorBase)CachedContext)init_loggerc                   @   s   e Zd ZdS )ContextNotExistErrorN)__name__
__module____qualname__ r   r   b/home/ubuntu/.local/lib/python3.10/site-packages/cache_dit/caching/cache_contexts/cache_manager.pyr      s    r   c                   @   s`  e Zd ZddedefddZedefdd	Zedefd
dZ	ee
jjdefddZe
jjdefddZe
jjdefddZe
jj	ddeeeB  defddZe
jjdeeB defddZddedefddZdeeB defddZdeeB fddZdd ZejdeeB fdd Ze
jjdefd!d"Ze
jjde
jfd#d$Ze
jjd%d& Ze
jjd'd( Ze
jjd)d* Z e
jjde!fd+d,Z"e
jjdefd-d.Z#e
jjdefd/d0Z$e
jjde!fd1d2Z%e
jjde&e! fd3d4Z'e
jjde&e! fd5d6Z(e
jjde!fd7d8Z)e
jjde!fd9d:Z*e
jjde!fd;d<Z+e
jjde!fd=d>Z,e
jjd?d@ Z-e
jjdAdB Z.e
jjde/eef fdCdDZ0e
jjde/eef fdEdFZ1e
jjdefdGdHZ2e
jjdefdIdJZ3e
jjdefdKdLZ4e
jjdefdMdNZ5e
jjdefdOdPZ6de7e8e8f fdQdRZ9de7e8e8f fdSdTZ:e
jjdefdUdVZ;e
jjdefdWdXZ<e
jjdefdYdZZ=e
jjdefd[d\Z>e
jjdefd]d^Z?e
jjdefd_d`Z@e
jjdefdadbZAe
jjdefdcddZBe
jjdefdedfZCe
jjde!fdgdhZDe
jjde!fdidjZEe
jjdefdkdlZFe
jjdefdmdnZGe
jjdefdodpZHe
jjddqdrdse
jdte
jduedvedwedefdxdyZIdzd{ ZJd|d} ZKe
jjdd~e
jdwefddZLe
jjddwede
jfddZMe
jjdd~e
jdwefddZNe
jjddwede
jfddZOe
jjdd~e
jdwefddZPe
jjddwede
jfddZQe
jjdd~e
jdB dwefddZRe
jjddwede
jfddZSe
jj			dde
jde
jdwedede7e
jeTe
jdf f f
ddZUe
jjdefddZVe
jj			qdde
jdveduee dwedef
ddZWdS )CachedContextManagerNFnamepersistent_contextc                 C   s"   || _ d | _i | _|| _d| _d S NF)r   _current_context_cached_context_manager_persistent_context_current_step_refreshed)selfr   r   r   r   r   __init__   s
   
zCachedContextManager.__init__returnc                 C      | j S N)r   r   r   r   r   r          z'CachedContextManager.persistent_contextc                 C   r   r   )r   r   r   r   r   current_context$   r   z$CachedContextManager.current_contextc                 C   r   r   )r   r   r   r   r   current_step_refreshed(   s   z+CachedContextManager.current_step_refreshedc                 C   s:   | j }|d u r	dS |jj}|d ur| }||d kS dS )NFr   T)r   cache_confignum_inference_stepsget_current_step)r   _contextr#   current_stepr   r   r   is_pre_refreshed-   s   z%CachedContextManager.is_pre_refreshedc                 O   s*   t |i |}||_||_|| j|j< |S r   )r	   
_init_args_init_kwargsr   r   )r   argskwargsr%   r   r   r   new_context<   s
   z CachedContextManager.new_contextcached_contextc              	   C   s   |d u r| j }|d usJ dt|tr|}n|| jvr td| j| }|jj}|d urM| }||krMt	t
jrKtd|j d| d| d dS dS )	NzCurrent context is not set!Context not exist!zRefreshing cache context 'z' as current step: z >= num_inference_steps: .TF)r   
isinstancer	   r   r   r"   r#   r$   loggerisEnabledForloggingDEBUGdebugr   )r   r-   r%   r#   r&   r   r   r   maybe_refreshE   s,   



z"CachedContextManager.maybe_refreshc                 O   s   t |tr	|| _n.|| jvr1| jstdtt|t|fr-||d< | j|i || _n
t	d| j| | _| 
| jrstt|t|fs_t| jdsOJ t| jdsWJ | jj}| jj}| j| jg|R i || _d| _| jS d| _| jS )NzrContext not exist and persistent_context is False. Please create new context first or set persistent_context=True.r   z6To create new context, please provide args and kwargs.r(   r)   TF)r0   r	   r   r   r   r   anyboolr,   
ValueErrorr6   hasattrr(   r)   reset_contextr   )r   r-   r*   r+   r   r   r   set_contextc   s.   

z CachedContextManager.set_contextc                 C   s*   |d ur|| j vrtd| j | S | jS )Nr.   )r   r   r   )r   r   r   r   r   get_context   s
   

z CachedContextManager.get_contextc                 O   s   t |tr%|j}|j| jv r|  | j|j= ||d< | j|i |}|S |}|| jv r7| j|   | j|= ||d< | j|i |}|S )Nr   )r0   r	   r   r   clear_buffersr,   )r   r-   r*   r+   old_context_namer%   r   r   r   r;      s   

	
z"CachedContextManager.reset_contextc                 C   sX   t |tr|  |j| jv r| j|j= d S d S || jv r*| j|   | j|= d S d S r   )r0   r	   r>   r   r   r   r-   r   r   r   remove_context   s   

z#CachedContextManager.remove_contextc                 C   s"   t | j D ]}| | qd S r   )listr   keysrA   )r   context_namer   r   r   clear_contexts   s   z#CachedContextManager.clear_contextsc                 c   sB    | j }t|tr|| _ n| j| | _ z	d V  W || _ d S || _ w r   )r   r0   r	   r   )r   r-   old_cached_contextr   r   r   enter_context   s   
z"CachedContextManager.enter_contextc                 C       |   }|d usJ d| S N!cached_context must be set before)r=   get_residual_diff_thresholdr@   r   r   r   rK         z0CachedContextManager.get_residual_diff_thresholdc                 C   s"   |   }|d usJ d||S rI   )r=   
get_bufferr   r   r-   r   r   r   rM      s   
zCachedContextManager.get_bufferc                 C   s(   |   }|d usJ d||| d S rI   )r=   
set_buffer)r   r   bufferr-   r   r   r   rO      s   zCachedContextManager.set_bufferc                 C   &   |   }|d usJ d|| d S rI   )r=   remove_bufferrN   r   r   r   rR         z"CachedContextManager.remove_bufferc                 C   $   |   }|d usJ d|  d S rI   )r=   mark_step_beginr@   r   r   r   rU         z$CachedContextManager.mark_step_beginc                 C   rH   rI   )r=   r$   r@   r   r   r   r$      rL   z%CachedContextManager.get_current_stepc                 C   @   |   }|d usJ dt|  }|  }||v r|| S d S rI   )r=   strr$   get_residual_diffs)r   r-   stepresidual_diffsr   r   r   get_current_step_residual_diff      z3CachedContextManager.get_current_step_residual_diffc                 C   rW   rI   )r=   rX   r$   get_cfg_residual_diffs)r   r-   rZ   cfg_residual_diffsr   r   r   "get_current_step_cfg_residual_diff   r]   z7CachedContextManager.get_current_step_cfg_residual_diffc                 C   rH   rI   )r=   get_current_transformer_stepr@   r   r   r   ra      rL   z1CachedContextManager.get_current_transformer_stepc                 C   rH   rI   )r=   get_cached_stepsr@   r   r   r   rb     rL   z%CachedContextManager.get_cached_stepsc                 C   rH   rI   )r=   get_cfg_cached_stepsr@   r   r   r   rc     rL   z)CachedContextManager.get_cfg_cached_stepsc                 C       |   }|d usJ d|jjS rI   )r=   r"   max_cached_stepsr@   r   r   r   get_max_cached_steps  rL   z)CachedContextManager.get_max_cached_stepsc                 C   rd   rI   )r=   r"   max_continuous_cached_stepsr@   r   r   r   get_max_continuous_cached_steps  rL   z4CachedContextManager.get_max_continuous_cached_stepsc                 C      |   }|d usJ d|jS rI   )r=   continuous_cached_stepsr@   r   r   r   get_continuous_cached_steps     z0CachedContextManager.get_continuous_cached_stepsc                 C   ri   rI   )r=   cfg_continuous_cached_stepsr@   r   r   r   get_cfg_continuous_cached_steps   rl   z4CachedContextManager.get_cfg_continuous_cached_stepsc                 C   rT   rI   )r=   add_cached_stepr@   r   r   r   ro   &  rV   z$CachedContextManager.add_cached_stepc                 C   rQ   rI   )r=   add_residual_diff)r   diffr-   r   r   r   rp   ,  rS   z&CachedContextManager.add_residual_diffc                 C   rH   rI   )r=   rY   r@   r   r   r   rY   2  rL   z'CachedContextManager.get_residual_diffsc                 C   rH   rI   )r=   r^   r@   r   r   r   r^   8  rL   z+CachedContextManager.get_cfg_residual_diffsc                 C   rH   rI   )r=   get_accumulated_residual_diffr@   r   r   r   rr   >  rL   z2CachedContextManager.get_accumulated_residual_diffc                 C   rH   rI   )r=   !get_cfg_accumulated_residual_diffr@   r   r   r   rs   D  rL   z6CachedContextManager.get_cfg_accumulated_residual_diffc                 C   rd   rI   )r=   r"   'max_accumulated_residual_diff_thresholdr@   r   r   r   rt   J  rL   z<CachedContextManager.max_accumulated_residual_diff_thresholdc                 C   rH   rI   )r=   enable_calibratorr@   r   r   r   is_calibrator_enabledP  rL   z*CachedContextManager.is_calibrator_enabledc                 C   rH   rI   )r=   enable_encoder_calibratorr@   r   r   r   is_encoder_calibrator_enabledV  rL   z2CachedContextManager.is_encoder_calibrator_enabledc                 C   rH   rI   )r=   get_calibratorsr@   r   r   r   get_calibrator\     z#CachedContextManager.get_calibratorc                 C   rH   rI   )r=   get_cfg_calibratorsr@   r   r   r   get_cfg_calibratora  r{   z'CachedContextManager.get_cfg_calibratorc                 C   s$   |   }|d usJ d| dkS )NrJ   residual)r=   calibrator_cache_typer@   r   r   r   is_calibrator_cache_residualf  rV   z1CachedContextManager.is_calibrator_cache_residualc                 C      |   r|  S dS NT)rv   r   r   r   r   r   is_cache_residuall     z&CachedContextManager.is_cache_residualc                 C   r   r   )rx   r   r   r   r   r   is_encoder_cache_residuals  r   z.CachedContextManager.is_encoder_cache_residualc                 C   rH   rI   )r=   is_in_warmupr@   r   r   r   r   z  rL   z!CachedContextManager.is_in_warmupc                 C   rH   rI   )r=   is_in_full_compute_stepsr@   r   r   r   r     rL   z-CachedContextManager.is_in_full_compute_stepsc                 C   s$   |   }|d usJ d|jjd uS rI   )r=   r"   steps_computation_maskr@   r   r   r   !is_steps_computation_mask_enabled  rV   z6CachedContextManager.is_steps_computation_mask_enabledc                 C   rH   rI   )r=   get_steps_computation_policyr@   r   r   r   r     s   z1CachedContextManager.get_steps_computation_policyc                 C   s0   |   }|d usJ d|jjd uo|jjdkS )NrJ           )r=   extra_cache_configl1_hidden_states_diff_thresholdr@   r   r   r   is_l1_diff_enabled  s
   
z'CachedContextManager.is_l1_diff_enabledc                 C   rd   rI   )r=   r   important_condition_thresholdr@   r   r   r   !get_important_condition_threshold  rL   z6CachedContextManager.get_important_condition_thresholdc                 C   4   |   }|d usJ d|jjdksJ d|jjS )NrJ   r   zFn_compute_blocks must be >= 1)r=   r"   Fn_compute_blocksr@   r   r   r   r        z&CachedContextManager.Fn_compute_blocksc                 C   r   )NrJ   r   zBn_compute_blocks must be >= 0)r=   r"   Bn_compute_blocksr@   r   r   r   r     r   z&CachedContextManager.Bn_compute_blocksc                 C   rd   rI   )r=   r"   enable_separate_cfgr@   r   r   r   r     rL   z(CachedContextManager.enable_separate_cfgc                 C   rH   rI   )r=   is_separate_cfg_stepr@   r   r   r   r     rL   z)CachedContextManager.is_separate_cfg_stepc                 C   rd   rI   )r=   r"   cfg_diff_compute_separater@   r   r   r   r     rL   z.CachedContextManager.cfg_diff_compute_separateFn)parallelizedprefixt1t2	thresholdr   r   c                C   s  |dkr|  d dS |dkr|  d dS |j|jkr8ttjr1t| d|j d|j  |  d	 dS t|  | 	 | 
  |  d ufrP|  }nz|  }|dkr||  }|jd
d}	| jd
d}
|	|
 }||k}| dkr|d
}||}||  }||   }n||   }|  }n||   }|  }|rtj|tjjd tj|tjjd ||  }ttjrt| d|dd|d |  | ||k S )Nr   g       Fg      ?g      Tz, shape error: z != g       )dimr   )opz, diff: .6fz, threshold: )rp   shaper1   r2   r3   r4   r5   allr   r   r   r\   r   absmeansum	unsqueeze	expand_asdist
all_reduceReduceOpAVGitem)r   r   r   r   r   r   rq   condition_threshraw_diff
token_m_df
token_m_t1
token_diff	condition	mean_diffmean_t1r   r   r   
similarity  sV   




	


zCachedContextManager.similarityc              	   C   8   t tjrt d| d|   d|    d S d S )Nzset , transformer step: , executed step: r1   r2   r3   r4   r5   ra   r$   r   r   r   r   r   _debugging_set_buffer     z*CachedContextManager._debugging_set_bufferc              	   C   r   )Nzget r   r   r   r   r   r   r   _debugging_get_buffer  r   z*CachedContextManager._debugging_get_bufferrP   c                 C   s   |d u rd S |   }|dkr|dd d |f }| }|  r2| | d | | d| d S | | d | | d| d S )Nr   ._buffer_cfg_buffer)get_downsample_factor
contiguousr   r   rO   )r   rP   r   downsample_factorr   r   r   set_Fn_buffer#  s   z"CachedContextManager.set_Fn_bufferc                 C   H   |   r| | d | | dS | | d | | dS )Nr   r   r   r   rM   r   r   r   r   get_Fn_buffer5  
   z"CachedContextManager.get_Fn_bufferc                 C   s`   |d u rd S |   r| | d | | d| d S | | d | | d| d S N_encoder_buffer_cfg_encoder_buffer)r   r   rO   )r   rP   r   r   r   r   set_Fn_encoder_buffer=  s   z*CachedContextManager.set_Fn_encoder_bufferc                 C   r   r   r   r   r   r   r   get_Fn_encoder_bufferI  r   z*CachedContextManager.get_Fn_encoder_bufferBnc                 C   s  |d u rd S |   r]|  r|  \}}n|  \}}|d ur(|j||d d S ttjr3t	d |  rJ| 
| d | | d| d S | 
| d | | d| d S |  rt| 
| d | | d| d S | 
| d | | d| d S Nr   acalibrator is enabled but not set in the cache context. Falling back to default buffer retrieval.r   r   )rv   r   r}   rz   updater1   r2   r3   r4   r5   r   rO   )r   rP   r   
calibrator_r   r   r   set_Bn_bufferR  ,   z"CachedContextManager.set_Bn_bufferc                 C   s   |   rN|  r|  \}}n|  \}}|d ur|j|dS ttjr*t	d |  r>| 
| d | | dS | 
| d | | dS |  rb| 
| d | | dS | 
| d | | dS r   )rv   r   r}   rz   approximater1   r2   r3   r4   r5   r   rM   )r   r   r   r   r   r   r   get_Bn_bufferw  s(   z"CachedContextManager.get_Bn_bufferc                 C   s  |d u rd S |   r]|  r|  \}}n|  \}}|d ur(|j||d d S ttjr3t	d |  rJ| 
| d | | d| d S | 
| d | | d| d S |  rt| 
| d | | d| d S | 
| d | | d| d S )Nr   zeCalibratorBase is enabled but not set in the cache context. Falling back to default buffer retrieval.r   r   )rx   r   r}   rz   r   r1   r2   r3   r4   r5   r   rO   )r   rP   r   r   encoder_calibratorr   r   r   set_Bn_encoder_buffer  r   z*CachedContextManager.set_Bn_encoder_bufferc                 C   s   |   rN|  r|  \}}n|  \}}|d ur|j|dS ttjr*t	d |  r>| 
| d | | dS | 
| d | | dS |  rb| 
| d | | dS | 
| d | | dS )Nr   r   r   r   )rx   r   r}   rz   r   r1   r2   r3   r4   r5   r   rM   )r   r   r   r   r   r   r   get_Bn_encoder_buffer  s(   z*CachedContextManager.get_Bn_encoder_buffer
Bn_encoderhidden_statesencoder_hidden_statesencoder_prefixc                 C   s   d|v r
|  |}n| |}|d usJ | d|  r#|| }n|}| }|d urOd|v r7| |}n| |}|d urK|  rI|| }n|}| }||fS )Nr   z_buffer must be set before)r   r   r   r   r   r   r   )r   r   r   r   r   hidden_states_prevencoder_hidden_states_prevr   r   r   apply_cache  s$   	



z CachedContextManager.apply_cachec                 C   rd   rI   )r=   r   r   r@   r   r   r   r     rL   z*CachedContextManager.get_downsample_factorstates_tensorc                 C   s  |   rdS |  r|  rdS |  dkrdS |  }|  s%|  }n|  }|dkrFt||krFt	
tjrDt	| d| d dS |  }|  sS|  }n|  }|dkr||krt	
tjrpt	| d| d |  }	|  s}d|	_dS d|	_dS |  }
|
d ur|
dkr|  s|  }n|  }||
krt	
tjrt	| d	|
d
d|d
d dS |d u s|dkr|  }|dkrdS |  }|dkrd|vr|dd d |f }| }d|v r| |}n| |}|d uo| j|||||d}|S )NFstaticTr   z, max_cached_steps reached: z, can not use cache.z', max_continuous_cached_steps reached: r   z3, max_accumulated_residual_diff_threshold reached: r   z, accumulated_residual_diff: r   r   .)r   r   r   )r   r   r   r   rf   r   rb   rc   lenr1   r2   r3   r4   r5   rh   rk   rn   r=   rj   rm   rt   rr   rs   rK   r   r   r   r   r   )r   r   r   r   r   re   cached_stepsrg   rj   r-   rt   accumulated_residual_diffr   prev_states_tensor	can_cacher   r   r   r     s   	



zCachedContextManager.can_cacher   r   )r   )r   )Nr   r   )FNr   )Xr   r   r   rX   r8   r   propertyr   r	   r    torchcompilerdisabler!   r'   r,   r   r6   r<   r=   r;   rA   rE   
contextlibcontextmanagerrG   floatrK   TensorrM   rO   rR   rU   intr$   r\   r`   ra   r   rb   rc   rf   rh   rk   rn   ro   rp   r   rY   r^   rr   rs   rt   rv   rx   r   r   rz   r}   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r      sh   
'




		

M	$$)r   )r3   r   typingr   r   r   r   r   r   torch.distributeddistributedr   calibratorsr   cache_contextr	   cache_dit.loggerr
   r   r1   	Exceptionr   r   r   r   r   r   <module>   s    