o
    8wi-                     @  s   d dl mZ d dlZd dlZd dlmZ d dlmZ d dlm	Z	 d dl
mZ d dlZd dlmZ d dlmZmZmZ d d	lmZmZmZ d
dlmZ d
dlmZmZ G dd deZdS )    )annotationsN)asdict)Enum)chain)Optional)tqdm)	BaseTunerBaseTunerLayercheck_target_module_exists)1TRANSFORMERS_MODELS_TO_C3A_TARGET_MODULES_MAPPINGModulesToSaveWrapper_get_submodules   )	C3AConfig)C3ALayer	C3ALinearc                      s   e Zd ZU dZdZded< d;d< fd
dZd=ddZedd Z	dd Z
dd Zd>ddZedd Zd? fddZd;d@d d!ZdAdBd$d%ZdCd&d'ZdCd(d)ZdDd,d-Zed.d/ Z	"			0dEdFd5d6Z	0dGdHd7d8ZdId9d:Z  ZS )JC3AModela@  
    Creates C3A model from a pretrained transformers model.

    The method is described in detail in [TODO].

    Args:
        model ([`torch.nn.Module`]): The model to be adapted.
        config ([`C3AConfig`]): The configuration of the C3A model.
        adapter_name (`str`): The name of the adapter, defaults to `"default"`.

    Returns:
        `torch.nn.Module`: The C3A model.

    **Attributes**:
        - **model** ([`~transformers.PreTrainedModel`]) -- The model to be adapted.
        - **peft_config** ([`C3AConfig`]): The configuration of the C3A model.
    c3a_strprefixFlow_cpu_mem_usageboolreturnNonec                   s   t  j||||d d S )N)r   )super__init__)selfmodelconfigadapter_namer   	__class__ R/home/ubuntu/sommelier/.venv/lib/python3.10/site-packages/peft/tuners/c3a/model.pyr   :   s   zC3AModel.__init__r   r   c                 C  s2   t | jdkr|jdkrt| jj ddS dS )z
        A helper method to check the config when a new adapter is being added.

        Raise a ValueError if there is something wrong with the config or if it conflicts with existing adapters.

        r   nonezf supports only 1 adapter with bias. When using multiple adapters, set bias to 'none' for all adapters.N)lenpeft_configbias
ValueErrorr!   __name__)r   r   r"   r"   r#   _check_new_adapter_config=   s
   	z"C3AModel._check_new_adapter_configc                 C  s
   t | |S N)r
   )
c3a_configkeyr"   r"   r#   _check_target_module_existsL   s   
z$C3AModel._check_target_module_existsc                   s    d u rt dtt|j }tt fdd| }	|j|	|j}
|
|j	d}t
|tr:|||
|j	 d S | j|||fi |}|| jkrO|d | |||| d S )NzCurrent Key shouldn't be `None`c                   s   t d|  d S )Nz.*\.$)rematch)r-   current_keyr"   r#   <lambda>^   s    z.C3AModel._create_and_replace.<locals>.<lambda>)
block_sizeinit_weightsF)r(   listr   block_size_patternkeysnextfiltergetr5   r6   
isinstancer   update_layer_create_new_moduleactive_adapterrequires_grad__replace_module)r   r,   r   targettarget_nameparentr3   optional_kwargspattern_keystarget_name_keyr5   kwargs
new_moduler"   r2   r#   _create_and_replaceP   s$   



zC3AModel._create_and_replacec                   s   t ||| t|dr|j}t|ds |j|_t|dr |j|_t|dd d ur>t|dr3|j|j_n|j|_||jj t	d |
 D ]\}}| j|v rdt fdd| D sd||jj qGd S )N
base_layerr'   statemetac                 3  s    | ]}|j  kV  qd S r+   )device).0prN   r"   r#   	<genexpr>   s    z+C3AModel._replace_module.<locals>.<genexpr>)setattrhasattrrL   weightr'   getattrrM   torO   torchnamed_modulesr   any
parameters)r   rE   
child_namerJ   childnamemoduler"   rR   r#   rB   s   s&   





zC3AModel._replace_moduler   torch.nn.Modulec                 C  s   |  D ]\}}| j|vrd|_q| jD ]H}| j| j}|dkr!q|dkr6|  D ]\}}d|v r4d|_q)q|dkrU| D ]}t|trSt	|drS|jd urSd|j_q>qt
d| dd S )	NFr$   allr'   Tc3a_onlyzRequested bias: z, is not implemented.)named_parametersr   requires_gradactive_adaptersr&   r'   modulesr=   r   rU   NotImplementedError)r   r   nrQ   r@   r'   mr"   r"   r#    _mark_only_adapters_as_trainable   s,   

z)C3AModel._mark_only_adapters_as_trainablec                 K  s<   t |tr
| }n|}t |tjjrt||fi |}|S r+   )r=   r	   get_base_layerrY   nnLinearr   )r,   r   rC   rI   target_base_layerrJ   r"   r"   r#   r?      s   

zC3AModel._create_new_moduler_   c                   s.   zt  |W S  ty   t| j| Y S w )z1Forward missing attributes to the wrapped module.)r   __getattr__AttributeErrorrW   r   )r   r_   r    r"   r#   rp      s
   zC3AModel.__getattr__	inferencec                 C  sF   i }| j  D ]\}}dd t| D }|rd|d< q|||< |S )Nc                 S  s&   i | ]\}}|t |tr|jn|qS r"   )r=   r   value)rP   kvr"   r"   r#   
<dictcomp>   s   & z4C3AModel.get_peft_config_as_dict.<locals>.<dictcomp>Tinference_mode)r&   itemsr   )r   rr   config_dictr-   rs   r   r"   r"   r#   get_peft_config_as_dict   s   z C3AModel.get_peft_config_as_dictTenabledc                 C  s,   | j  D ]}t|ttfr|| qd S r+   )r   rg   r=   r	   r   enable_adapters)r   r{   r`   r"   r"   r#   _set_adapter_layers   s
   
zC3AModel._set_adapter_layersc                 C  s   | j dd dS )zyEnable all adapters.

        Call this if you have previously disabled all adapters and want to re-enable them.
        Tr{   N)r}   r   r"   r"   r#   enable_adapter_layers   s   zC3AModel.enable_adapter_layersc                 C  sF   | j D ]}| j| j}|dkrd| d}t| q| jdd dS )zDisable all adapters.

        When disabling all adapters, the model output corresponds to the output of the base model.
        r$   z>Careful, disabling adapter layers with bias configured to be 'zP' does not produce the same output as the the base model would without adaption.Fr~   N)rf   r&   r'   warningswarnr}   )r   r@   valmsgr"   r"   r#   disable_adapter_layers   s   


zC3AModel.disable_adapter_layersr   str | list[str]c                 C  sF   | j  D ]}t|tr|jrtd |  || q|| _	dS )zSet the active adapter(s).

        Args:
            adapter_name (`str` or `list[str]`): Name of the adapter(s) to be activated.
        zJAdapter cannot be set when the model is merged. Unmerging the model first.N)
r   rg   r=   r   mergedr   r   unmergeset_adapterr@   )r   r   r`   r"   r"   r#   r      s   



zC3AModel.set_adapterc                 C  s4   | j d u r|d tvrtdtt|d  | _ | S )N
model_typez0Please specify `target_modules` in `peft_config`)target_modulesr   r(   set)r&   model_configr"   r"   r#   _prepare_adapter_config   s   

z C3AModel._prepare_adapter_configNprogressbar
safe_mergeadapter_namesOptional[list[str]]c              	     s    fdd j  D }d|rdnd d }t|| |dD ]?}zt j |\}}	}
W n	 ty4   Y qw t|	drN|rC|	j||d	  ||
|	 |	 qt	|	t
r]t||
|	j|	j  q j S )
Nc                   s   g | ]\}} j |vr|qS r"   )r   )rP   r-   _r   r"   r#   
<listcomp>   s    z9C3AModel._unload_and_optionally_merge.<locals>.<listcomp>z
Unloading zand merging  r   )disabledescrL   )r   r   )r   rZ   r   r   rq   rU   mergerB   rl   r=   r   rT   modules_to_saver@   )r   r   r   r   r   key_listr   r-   rE   rC   rD   r"   r   r#   _unload_and_optionally_merge   s    

z%C3AModel._unload_and_optionally_mergec                 C  s   | j |||dS )a  
        This method merges the C3A layers into the base model. This is needed if someone wants to use the base model as
        a standalone model.

        Args:
            progressbar (`bool`):
                whether to show a progressbar indicating the unload and merge process
            safe_merge (`bool`):
                whether to activate the safe merging check to check if there is any potential Nan in the adapter
                weights
            adapter_names (`list[str]`, *optional*):
                The list of adapter names that should be merged. If None, all active adapters will be merged. Defaults
                to `None`.
        )r   r   r   r   )r   r   r   r   r"   r"   r#   merge_and_unload  s   zC3AModel.merge_and_unloadc                 C  s   | j ddS )z
        Gets back the base model by removing all the C3A modules without merging. This gives back the original base
        model.
        F)r   r   r   r"   r"   r#   unload!  s   zC3AModel.unload)F)r   r   r   r   )r   r   r   r   )r   ra   r   r   )r_   r   )rr   r   )T)r{   r   r   r   )r   r   )r   r   r   r   )TFFN)r   r   r   r   r   r   )FFN)r   r   r   r   r   r   r   ra   )r   ra   )r)   
__module____qualname____doc__r   __annotations__r   r*   staticmethodr.   rK   rB   rk   r?   rp   rz   r}   r   r   r   r   r   r   r   __classcell__r"   r"   r    r#   r   %   s8   
 

#

	



r   )
__future__r   r0   r   dataclassesr   enumr   	itertoolsr   typingr   rY   r   peft.tuners.tuners_utilsr   r	   r
   
peft.utilsr   r   r   r   r   layerr   r   r   r"   r"   r"   r#   <module>   s   