o
    Ni3-                     @  s   d dl mZ d dlZd dlZd dlmZ d dlZd dlmZ d dl	m
Z
 d dlmZ d dlmZmZ d dlmZmZ d dlmZ d	d
lmZ d	dlmZ ddlmZ ddlmZmZ dddZG dd deZdS )    )annotationsN)Union)_calculate_correct_fan)Conv1D)is_bnb_4bit_availableis_bnb_available)	BaseTunerBaseTunerLayer)2TRANSFORMERS_MODELS_TO_VERA_TARGET_MODULES_MAPPING   )
BufferDict) _maybe_include_all_linear_layers   )
VeraConfig)Linear	VeraLayertensor_or_shape$Union[torch.Tensor, tuple[int, ...]]	generatortorch.Generatorreturntorch.Tensorc                 C  s   t | trt| }n| }t|d}td}|t| }td| }t  |j| ||dW  d   S 1 s=w   Y  dS )a  
    Kaiming Uniform Initialisation adapted to accept a `torch.Generator` object for PRNG.

    Args:
        tensor_or_shape (`Union[torch.Tensor, tuple[int, ...]]`):
            Tensor to initialise, or shape of new tensor to create and then initialise.
        generator: (`torch.Generator`):
            Generator object that manages the state of the PRNG algorithm in use.

    Returns:
        `torch.Tensor`: The initialised tensor.
    fan_inr   g      @r   N)	
isinstancetupletorchemptyr   mathsqrtno_graduniform_)r   r   tensorfangainstdbound r'   J/home/ubuntu/.local/lib/python3.10/site-packages/peft/tuners/vera/model.py_kaiming_init&   s   



$r)   c                      sj   e Zd ZU dZdZded< eZeZ	dddZ
dddZdddZd fddZdd Zedd Z  ZS )	VeraModela=  
    Creates Vector-based Random Matrix Adaptation (Vera) model from a pretrained transformers model.

    Args:
        model ([`~transformers.PreTrainedModel`]): The model to be adapted.
        config ([`VeraConfig`]): The configuration of the Vera model.
        adapter_name (`str`): The name of the adapter, defaults to `"default"`.
        low_cpu_mem_usage (`bool`, `optional`, defaults to `False`):
            Create empty adapter weights on meta device. Useful to speed up the loading process.

    Returns:
        `torch.nn.Module`: The Vera model.

    Example:

        ```py
        >>> from transformers import AutoModelForCausalLM
        >>> from peft import VeraConfig, get_peft_model

        >>> base_model = AutoModelForCausalLM.from_pretrained("facebook/opt-125m")
        >>> config = VeraConfig(r=128)
        >>> model = get_peft_model(base_model, config)
        ```

    **Attributes**:
        - **model** ([`~transformers.PreTrainedModel`]) -- The model to be adapted.
        - **peft_config** ([`VeraConfig`]): The configuration of the Vera model.
    vera_lambda_strprefixr   tuple[int, int]c           	      C  s   |  | j}| ||}t|| j}d}| j D ]K\}}| ||s$qt|tjr1|j	|j
f}nt|trLt|jdr@|jjn|jj}|ddd }nq|du rT|}q||krdtdd t||D }q|du rod}t||S )z
        Finds the largest input and output dimensions across linear layers that have been wrapped with VeRA.

        This will be used for determining the size of the shared vera_A and vera_B matrices.
        Nds_shapec                 s  s    | ]
\}}t ||V  qd S N)max).0abr'   r'   r(   	<genexpr>   s    z&VeraModel._find_dim.<locals>.<genexpr>z[No layers types compatible with VeRA were found. Please check `peft_config.target_modules`.)get_model_configmodel_prepare_adapter_configr   named_modules_check_target_module_existsr   nnr   out_featuresin_featuresr   hasattrweightr/   shaper   zip
ValueError)	selfconfigmodel_configpeft_configlargest_shapekeymodulemodule_shapemsgr'   r'   r(   	_find_dime   s.   
zVeraModel._find_dimrE   r   adapter_nameNonec                 C  s~   |  |\}}ti |jd| _ti |jd| _tjdd|j}t	|j
|f|d}t	||j
f|d}|| j|< || j|< d S )N)
persistentcpu)devicer   )rM   r   save_projectionvera_Avera_Br   	Generatormanual_seedprojection_prng_keyr)   r)rD   rE   rN   linear_out_dimlinear_in_dimr   rT   rU   r'   r'   r(   _init_vera_A_vera_B   s   
zVeraModel._init_vera_A_vera_Br8   	nn.Modulec                 C  s   |  || d S r1   )r\   )rD   r8   rE   rN   r'   r'   r(   _pre_injection_hook   s   zVeraModel._pre_injection_hookc                   s   t  | | j D ]}||u rq|j|jkr%td|jd|j dqtdd | j D }t|dkr?td| dS )	z
        A helper method to check the config when a new adapter is being added.

        Raise a ValueError if there is something wrong with the config or if it conflicts with existing adapters.

        z_Vera PRNG initialisation key must be the same for all adapters. Got config.projection_prng_key=z but previous config had .c                 S  s   h | ]}|j qS r'   )rS   )r3   rE   r'   r'   r(   	<setcomp>   s    z6VeraModel._check_new_adapter_config.<locals>.<setcomp>r   zcVeRA projection weights must be saved for all adapters or none, but got multiple different values: N)super_check_new_adapter_configrG   valuesrX   rC   sortedlen)rD   rE   existing_configsave_project_unique_values	__class__r'   r(   rb      s&   
z#VeraModel._check_new_adapter_configc              	   K  s   |d u rt d|j}t|do|jd u}	||j|j|jt| jddt| jddd}
|	|
d< t	|t
rG|j|| j| j||j|j|jd d S | j|| j| j||fi |
}|| jvr`|d | |||| d S )NzCurrent Key shouldn't be `None`biasis_loaded_in_8bitFis_loaded_in_4bit)rY   vera_dropoutfan_in_fan_outinit_weightsloaded_in_8bitloaded_in_4bit)	d_initial)rC   rY   r?   rj   rm   rn   ro   getattrr8   r   r   update_layerrT   rU   rr   _create_new_moduleactive_adapterrequires_grad__replace_module)rD   vera_configrN   targettarget_nameparentcurrent_keyoptional_kwargsrY   rj   kwargs
new_moduler'   r'   r(   _create_and_replace   s4   





zVeraModel._create_and_replacec                 K  s  t  rdd l}ddlm} t rddlm} |dd}	|dd}
|dd}t|t	r2|
 }n|}|
rZt||jjrZ| }||jj|jj|jd	 |||||fi |S |rt||jjr| }||j|jj|jjd
 |||||fi |S t|tjjr|d rtd d |d< | _n"t|trd|d< |d std d |d< | _ntd| dt||||f|	| jd|}|S )Nr   r   )Linear8bitLt)
Linear4bitrj   Frp   rq   )has_fp16_weights	thresholdindex)compute_dtypecompress_statistics
quant_typern   zjfan_in_fan_out is set to True but the target module is `torch.nn.Linear`. Setting fan_in_fan_out to False.Tis_target_conv_1d_layerzafan_in_fan_out is set to False but the target module is `Conv1D`. Setting fan_in_fan_out to True.zTarget module z is not supported. Currently, only the following modules are supported: `torch.nn.Linear`, `transformers.pytorch_utils.Conv1D`.)rj   rr   )r   bitsandbytesbnbr   r   r   popgetr   r	   get_base_layerr<   copyupdatestater   r   r   r   r@   r   r   r   r   warningswarnrn   r   rC   rr   )ry   rT   rU   rN   rz   r   r   r   r   rj   rp   rq   target_base_layereightbit_kwargsfourbit_kwargsr   r'   r'   r(   ru      sv   




zVeraModel._create_new_module)r   r.   )rE   r   rN   r,   r   rO   )r8   r]   rE   r   rN   r,   r   rO   )rE   r   r   rO   )__name__
__module____qualname____doc__r-   __annotations__r   tuner_layer_clsr
   target_module_mappingrM   r\   r^   rb   r   staticmethodru   __classcell__r'   r'   rh   r(   r*   C   s   
 

%
*r*   )r   r   r   r   r   r   ) 
__future__r   r   r   typingr   r   torch.nnr<   torch.nn.initr   transformers.pytorch_utilsr   peft.import_utilsr   r   peft.tuners.tuners_utilsr   r	   
peft.utilsr
   _buffer_dictr   tuners_utilsr   rE   r   layerr   r   r)   r*   r'   r'   r'   r(   <module>   s"   
