o
    NiR;                     @  s   d dl mZ d dlZd dlZd dlmZ d dlZd dlmZ d dl	m
Z
 d dlmZ d dlmZmZ d dlmZmZ d dlmZ d	d
lmZ d	dlmZ ddlmZ ddlmZmZ dddZG dd deZdS )    )annotationsN)Union)is_bf16_available)Conv1D)is_bnb_4bit_availableis_bnb_available)	BaseTunerBaseTunerLayer)6TRANSFORMERS_MODELS_TO_RANDLORA_TARGET_MODULES_MAPPING   )
BufferDict) _maybe_include_all_linear_layers   )RandLoraConfig)LinearRandLoraLayertensor_or_shape$Union[torch.Tensor, tuple[int, ...]]	generatortorch.Generatorreturntorch.Tensorc                 C  sv   t | trtj| t rtjntjd}n| }t  tjj	j
|td|d}|W  d   S 1 s4w   Y  dS )a  
    Kaiming Uniform Initialisation adapted to accept a `torch.Generator` object for PRNG.

    Args:
        tensor_or_shape (`Union[torch.Tensor, tuple[int, ...]]`):
            Tensor to initialise, or shape of new tensor to create and then initialise.
        generator: (`torch.Generator`):
            Generator object that manages the state of the PRNG algorithm in use.

    Returns:
        `torch.Tensor`: The initialised tensor.
    )dtype   )ar   N)
isinstancetupletorchemptyr   bfloat16float16no_gradnninitkaiming_uniform_mathsqrt)r   r   tensorbasis r)   N/home/ubuntu/.local/lib/python3.10/site-packages/peft/tuners/randlora/model.py_kaiming_init&   s   

$r+   c                      sv   e Zd ZU dZdZded< eZeZ	dddZ
dd ddZd!ddZd"ddZd# fddZdd Zedd Z  ZS )$RandLoraModela6  
    Creates a RandLoRA model from a pretrained transformers model.

    Args:
        model ([`~transformers.PreTrainedModel`]): The model to be adapted.
        config ([`RandLoraConfig`]): The configuration of the RandLora model.
        adapter_name (`str`): The name of the adapter, defaults to `"default"`.
        low_cpu_mem_usage (`bool`, `optional`, defaults to `False`):
            Create empty adapter weights on meta device. Useful to speed up the loading process.

    Returns:
        `torch.nn.Module`: The RandLora model.

    Example:

        ```py
        >>> from transformers import AutoModelForCausalLM
        >>> from peft import RandLoraConfig, get_peft_model

        >>> base_model = AutoModelForCausalLM.from_pretrained("facebook/opt-125m")
        >>> config = RandLoraConfig(r=32)
        >>> model = get_peft_model(base_model, config)
        ```

    **Attributes**:
        - **model** ([`~transformers.PreTrainedModel`]) -- The model to be adapted.
        - **peft_config** ([`RandLoraConfig`]): The configuration of the RandLora model.
    	randlora_strprefixr   tuple[int, int]c           	      C  s   |  | j}| ||}t|| j}d}| j D ]K\}}| ||s$qt|tjr1|j	|j
f}nt|trLt|jdr@|jjn|jj}|ddd }nq|du rT|}q||krdtdd t||D }q|du rod}t||S )z
        Finds the largest input and output dimensions across linear layers that have been wrapped with RandLora.

        This will be used for determining the size of the shared randlora_A and randlora_B matrices.
        Nds_shapec                 s  s    | ]
\}}t ||V  qd S )N)max).0r   br)   r)   r*   	<genexpr>   s    z*RandLoraModel._find_dim.<locals>.<genexpr>z_No layers types compatible with RandLora were found. Please check `peft_config.target_modules`.)get_model_configmodel_prepare_adapter_configr   named_modules_check_target_module_existsr   r"   r   out_featuresin_featuresr   hasattrweightr1   shaper   zip
ValueError)	selfconfigmodel_configpeft_configlargest_shapekeymodulemodule_shapemsgr)   r)   r*   	_find_dime   s.   
zRandLoraModel._find_dim   rD   r   adapter_namesparsityintNonec                 C  sL  |  |\}}t||t||}}ti |jd| _ti |jd| _tjdd	|j
}tj|jd|f|d}	||j }
|
 rDt|
nt|
d }
tj||
|jf|d}t|j}t|	j}d||dd|  k < d||ddd|   k< d||	dd|  k < d||	ddd|   k< ||  ||  }	}|	| j|< || j|< dS )	zn
        Sparse random projections as described in https://cs-people.bu.edu/evimaria/cs565/kdd-rp.pdf
        
persistentcpudevicer   r   r2   r   N)rL   r3   minr   save_projection
randlora_A
randlora_Br   	Generatormanual_seedprojection_prng_keyrandr
is_integerrP   zerosr@   std)rC   rD   rN   rO   linear_out_dimlinear_in_dimmax_dimmin_dimr   rZ   	num_basesr[   randlora_B_sparserandlora_A_sparser)   r)   r*   "_init_randlora_A_randlora_B_sparse   s(   



z0RandLoraModel._init_randlora_A_randlora_B_sparsec           	        s   |   \}}t||t||}ti  jd| _ti  jd| _tjdd	 j
t jd|fd}t|| j }| rFt|nt|d }tj fddt|D dd}||  ||  }}|| j|< || j|< d S )	NrR   rT   rU   r   rW   c                   s    g | ]}t d  jfdqS )r   rW   )r+   r`   )r4   _rD   r   rf   r)   r*   
<listcomp>   s     z=RandLoraModel._init_randlora_A_randlora_B.<locals>.<listcomp>)dim)rL   r3   rX   r   rY   rZ   r[   r   r\   r]   r^   r+   r`   ra   rP   catrangerc   )	rC   rD   rN   rd   re   rg   rZ   rh   r[   r)   rm   r*   _init_randlora_A_randlora_B   s   
z)RandLoraModel._init_randlora_A_randlora_Br8   	nn.Modulec                 C  s^   |j r| |\}}| j||tt||d d S |jr'| j||dd d S | || d S )N)rO   rM   )very_sparserL   rk   r%   r&   rX   sparserr   )rC   r8   rD   rN   rd   re   r)   r)   r*   _pre_injection_hook   s   
z!RandLoraModel._pre_injection_hookc                   s   t  | | j D ]}||u rq|j|jkr%td|jd|j dqtdd | j D }t|dkr?td| dS )	z
        A helper method to check the config when a new adapter is being added.

        Raise a ValueError if there is something wrong with the config or if it conflicts with existing adapters.

        zcRandLora PRNG initialisation key must be the same for all adapters. Got config.projection_prng_key=z but previous config had .c                 S  s   h | ]}|j qS r)   )rY   )r4   rD   r)   r)   r*   	<setcomp>   s    z:RandLoraModel._check_new_adapter_config.<locals>.<setcomp>r   zgRandLora projection weights must be saved for all adapters or none, but got multiple different values: N)super_check_new_adapter_configrF   valuesr^   rB   sortedlen)rC   rD   existing_configsave_project_unique_values	__class__r)   r*   rz      s&   
z'RandLoraModel._check_new_adapter_configc              
   K  s   |d u rt d|j}t|do|jd u}	||j|j|j|jt| j	ddt| j	ddd}
|	|
d< t
|trH||| j| j||j|j|j d S | j|| j| j||fi |
}|| jvra|d | |||| d S )NzCurrent Key shouldn't be `None`biasis_loaded_in_8bitFis_loaded_in_4bit)r`   randlora_alpharandlora_dropoutfan_in_fan_outinit_weightsloaded_in_8bitloaded_in_4bit)rB   r`   r>   r   r   r   r   r   getattrr8   r   r   update_layerrZ   r[   _create_new_moduleactive_adapterrequires_grad__replace_module)rC   randlora_configrN   targettarget_nameparentcurrent_keyoptional_kwargsr`   r   kwargs
new_moduler)   r)   r*   _create_and_replace   s>   
	



z!RandLoraModel._create_and_replacec                 K  s  t  rdd l}ddlm} t rddlm} |dd}	|dd}
|dd}t|t	r2|
 }n|}|
rZt||jjrZ| }||jj|jj|jd	 |||||fi |S |rt||jjr| }||j|jj|jjd
 |||||fi |S t|tjjr|d rtd d |d< | _n"t|trd|d< |d std d |d< | _ntd| dt||||fd|	i|}|S )Nr   r   )Linear8bitLt)
Linear4bitr   Fr   r   )has_fp16_weights	thresholdindex)compute_dtypecompress_statistics
quant_typer   zjfan_in_fan_out is set to True but the target module is `torch.nn.Linear`. Setting fan_in_fan_out to False.Tis_target_conv_1d_layerzafan_in_fan_out is set to False but the target module is `Conv1D`. Setting fan_in_fan_out to True.zTarget module z is not supported. Currently, only the following modules are supported: `torch.nn.Linear`, `transformers.pytorch_utils.Conv1D`.)r   bitsandbytesbnbr   r   r   popgetr   r	   get_base_layerr"   copyupdatestater   r   r   r   r?   r   r   r   r   warningswarnr   r   rB   )r   rZ   r[   rN   r   r   r   r   r   r   r   r   target_base_layereightbit_kwargsfourbit_kwargsr   r)   r)   r*   r      st   



	z RandLoraModel._create_new_module)r   r0   )rM   )rD   r   rN   r.   rO   rP   r   rQ   )rD   r   rN   r.   r   rQ   )r8   rs   rD   r   rN   r.   r   rQ   )rD   r   r   rQ   )__name__
__module____qualname____doc__r/   __annotations__r   tuner_layer_clsr
   target_module_mappingrL   rk   rr   rv   rz   r   staticmethodr   __classcell__r)   r)   r   r*   r,   C   s   
 
%
)
,r,   )r   r   r   r   r   r   ) 
__future__r   r%   r   typingr   r   torch.nnr"   accelerate.utils.importsr   transformers.pytorch_utilsr   peft.import_utilsr   r   peft.tuners.tuners_utilsr   r	   
peft.utilsr
   _buffer_dictr   tuners_utilsr   rD   r   layerr   r   r+   r,   r)   r)   r)   r*   <module>   s"   
