o
    Ni!                     @  s   d dl mZ d dlZd dlZd dlmZ d dlmZ d dlm	Z	m
Z
 d dlmZ ddlmZ ddlmZmZ G d	d
 d
e	ZdS )    )annotationsN)Conv1D)	BaseTunerBaseTunerLayer)4TRANSFORMERS_MODELS_TO_VBLORA_TARGET_MODULES_MAPPING   )VBLoRAConfig)LinearVBLoRALayerc                   @  sd   e Zd ZU dZdZded< eZeZ	dd
dZ
dddZdd Zedd ZddddZdddZdS ) VBLoRAModela  
    Creates VBLoRA model from a pretrained transformers model.

    The method is described in detail in https://huggingface.co/papers/2405.15179.

    Args:
        model ([`~transformers.PreTrainedModel`]): The model to be adapted.
        config ([`VBLoRAConfig`]): The configuration of the VBLoRA model.
        adapter_name (`str`): The name of the adapter, defaults to `"default"`.
        low_cpu_mem_usage (`bool`, `optional`, defaults to `False`):
            Create empty adapter weights on meta device. Useful to speed up the loading process.

    Returns:
        `torch.nn.Module`: The VBLoRA model.

    Example:

        ```py
        >>> from transformers import AutoModelForCausalLM
        >>> from peft import VBLoRAConfig, get_peft_model

        >>> base_model = AutoModelForCausalLM.from_pretrained("facebook/opt-125m")
        >>> config = VBLoRAConfig(
        ...     task_type="SEQ_CLS",
        ...     r=4,
        ...     target_modules=["fc1", "fc2", "k_proj", "out_proj", "q_proj", "v_proj"],
        ...     num_vectors=60,
        ...     vector_length=256,
        ...     save_only_topk_weights=True,
        ... )
        >>> model = get_peft_model(base_model, config)
        ```

    **Attributes**:
        - **model** ([`~transformers.PreTrainedModel`]) -- The model to be adapted.
        - **peft_config** ([`VBLoRAConfig`]): The configuration of the VBLoRAConfig model.
    vblora_strprefixconfigr   adapter_namereturnNonec                 C  s6   t |j|j}t jj||j |j || j|< d S N)	torchzerosnum_vectorsvector_lengthnninituniform_init_vector_bank_boundvblora_vector_bank)selfr   r   r    r   L/home/ubuntu/.local/lib/python3.10/site-packages/peft/tuners/vblora/model.py_init_vblora_vector_bankH   s   z$VBLoRAModel._init_vblora_vector_bankmodel	nn.Modulec                 C  s   t i | _d S r   )r   ParameterDictr   )r   r!   r   r   r   r   r   _pre_injection_hookM   s   zVBLoRAModel._pre_injection_hookc           
   
   C  s   |d u rt dt|do|jd u}|j|d}| || t|tr9|j|| j|j	|j
|j|j|j|jd d S | jd|| j||d|}	|| jvrQ|	d | |||	| d S )NzCurrent Key shouldn't be `None`bias)fan_in_fan_outr%   )r   r   rtopkr   r   vblora_dropoutinit_logits_std)vblora_configr   r   targetFr   )
ValueErrorhasattrr%   r&   r    
isinstancer	   update_layerr   r'   r(   r   r   r)   r*   _create_new_moduleactive_adapterrequires_grad__replace_module)
r   r+   r   r,   target_nameparentcurrent_keyr%   kwargs
new_moduler   r   r   _create_and_replaceP   s:   	



zVBLoRAModel._create_and_replacec                 K  s   t |tr
| }n|}t |tjjr$|d r#td d |d< | _n"t |t	r>d|d< |d s=td d |d< | _nt
d| dtd
|||| j| j| j| j| j| jd		|}|S )Nr&   zjfan_in_fan_out is set to True but the target module is `torch.nn.Linear`. Setting fan_in_fan_out to False.FTis_target_conv_1d_layerzafan_in_fan_out is set to False but the target module is `Conv1D`. Setting fan_in_fan_out to True.zTarget module z is not supported. Currently, only the following modules are supported: `torch.nn.Linear`, `transformers.pytorch_utils.Conv1D`.)	
base_layerr   r   r'   r   r   r(   r)   r*   r   )r/   r   get_base_layerr   r   r	   warningswarnr&   r   r-   r'   r   r   r(   r)   r*   )r+   r   r   r,   r8   target_base_layerr9   r   r   r   r1   |   sF   




zVBLoRAModel._create_new_moduledefaulttuple[int, int]c                 C  s  d}d}d}|   D ]#\}}d|v r|| 7 }q
d|v r$|| 7 }q
|jr-|| 7 }q
| j| jr| j| j}d}|dk rCd}n|dk rJd}n	|d	k rQd}nd
}|| j| j | j| jd  }	|| j| j | j| j | }
t||	 |
 }||fS || }||fS )z`
        Returns the number of savable VB-LoRA parameters and other savable parameters.
        r   vblora_logitsr   r      g      ?i   g      ?l           )named_parametersnumelrequires_gradpeft_configsave_only_topk_weightsr   r(   int)r   adapterlogits_paramsvector_bank_paramsother_paramsnameparamr   factortopk_weight_paramstopk_indices_paramsvblora_paramsr   r   r   get_nb_savable_parameters   s:   z%VBLoRAModel.get_nb_savable_parametersc                 C  s,   |   \}}td|dd|| d dS )z_
        Prints the number of savable VB-LoRA parameters and total savable parameters.
        z1VB-LoRA params to-be-saved (float32-equivalent): z,dz || total params to-be-saved: N)rV   print)r   rU   rO   r   r   r   print_savable_parameters   s   
z$VBLoRAModel.print_savable_parametersN)r   r   r   r   r   r   )r!   r"   r   r   r   r   r   r   )rA   )r   rB   )r   r   )__name__
__module____qualname____doc__r   __annotations__r
   tuner_layer_clsr   target_module_mappingr    r$   r:   staticmethodr1   rV   rX   r   r   r   r   r      s   
 &

,
($r   )
__future__r   r>   r   torch.nnr   transformers.pytorch_utilsr   peft.tuners.tuners_utilsr   r   
peft.utilsr   r   r   layerr	   r
   r   r   r   r   r   <module>   s   