o
    8wi_                     @  s  d dl mZ d dlZd dlZd dlmZ d dlmZ d dlm	Z	m
Z
 d dlZd dlmZ d dlmZ d dlmZ d dlmZ d d	lmZmZ d d
lmZmZmZ d dlmZmZmZ ddlmZ ddl m!Z! ddl"m#Z# ddl$m%Z%m&Z& dddZ'G dd deZ(dS )    )annotationsN)asdict)Enum)OptionalUnion)is_bf16_available)tqdm)Conv1D)is_bnb_4bit_availableis_bnb_available)	BaseTunerBaseTunerLayercheck_target_module_exists)6TRANSFORMERS_MODELS_TO_RANDLORA_TARGET_MODULES_MAPPINGModulesToSaveWrapper_get_submodules   )
BufferDict) _maybe_include_all_linear_layers   )RandLoraConfig)LinearRandLoraLayertensor_or_shape$Union[torch.Tensor, tuple[int, ...]]	generatortorch.Generatorreturntorch.Tensorc                 C  sv   t | trtj| t rtjntjd}n| }t  tjj	j
|td|d}|W  d   S 1 s4w   Y  dS )a  
    Kaiming Uniform Initialisation adapted to accept a `torch.Generator` object for PRNG.

    Args:
        tensor_or_shape (`Union[torch.Tensor, tuple[int, ...]]`):
            Tensor to initialise, or shape of new tensor to create and then initialise.
        generator: (`torch.Generator`):
            Generator object that manages the state of the PRNG algorithm in use.

    Returns:
        `torch.Tensor`: The initialised tensor.
    )dtype   )ar   N)
isinstancetupletorchemptyr   bfloat16float16no_gradnninitkaiming_uniform_mathsqrt)r   r   tensorbasis r0   W/home/ubuntu/sommelier/.venv/lib/python3.10/site-packages/peft/tuners/randlora/model.py_kaiming_init+   s   

$r2   c                      s  e Zd ZU dZdZded< dGdH fd
dZdIddZdJdKddZdLddZ	dMddZ
dNddZedd  Zd!d" Zed#d$ ZdOd%d&Zed'd( ZdP fd*d+ZdGdQd-d.ZdRd0d1Zd2d3 Zd4d5 Zd6d7 Zed8d9 Z	/			:dSdTd?d@ZdUdAdBZ	:dVdTdCdDZdEdF Z  ZS )WRandLoraModela6  
    Creates a RandLoRA model from a pretrained transformers model.

    Args:
        model ([`~transformers.PreTrainedModel`]): The model to be adapted.
        config ([`RandLoraConfig`]): The configuration of the RandLora model.
        adapter_name (`str`): The name of the adapter, defaults to `"default"`.
        low_cpu_mem_usage (`bool`, `optional`, defaults to `False`):
            Create empty adapter weights on meta device. Useful to speed up the loading process.

    Returns:
        `torch.nn.Module`: The RandLora model.

    Example:

        ```py
        >>> from transformers import AutoModelForCausalLM
        >>> from peft import RandLoraConfig, get_peft_model

        >>> base_model = AutoModelForCausalLM.from_pretrained("facebook/opt-125m")
        >>> config = RandLoraConfig(r=32)
        >>> model = get_peft_model(base_model, config)
        ```

    **Attributes**:
        - **model** ([`~transformers.PreTrainedModel`]) -- The model to be adapted.
        - **peft_config** ([`RandLoraConfig`]): The configuration of the RandLora model.
    	randlora_strprefixFlow_cpu_mem_usageboolr   Nonec                   s   t  j||||d d S )N)r7   )super__init__)selfmodelconfigadapter_namer7   	__class__r0   r1   r;   h   s   zRandLoraModel.__init__tuple[int, int]c           	      C  s   |  | j}| ||}t|| j}d}| j D ]K\}}| ||s$qt|tjr1|j	|j
f}nt|trLt|jdr@|jjn|jj}|ddd }nq|du rT|}q||krdtdd t||D }q|du rod}t||S )z
        Finds the largest input and output dimensions across linear layers that have been wrapped with RandLora.

        This will be used for determining the size of the shared randlora_A and randlora_B matrices.
        Nds_shapec                 s  s    | ]
\}}t ||V  qd S N)max).0r!   br0   r0   r1   	<genexpr>   s    z*RandLoraModel._find_dim.<locals>.<genexpr>z_No layers types compatible with RandLora were found. Please check `peft_config.target_modules`.)get_model_configr=   _prepare_adapter_configr   named_modules_check_target_module_existsr"   r)   r   out_featuresin_featuresr	   hasattrweightrC   shaper#   zip
ValueError)	r<   r>   model_configpeft_configlargest_shapekeymodulemodule_shapemsgr0   r0   r1   	_find_dimk   s.   
zRandLoraModel._find_dim   r>   r   r?   sparsityintc                 C  sL  |  |\}}t||t||}}ti |jd| _ti |jd| _tjdd	|j
}tj|jd|f|d}	||j }
|
 rDt|
nt|
d }
tj||
|jf|d}t|j}t|	j}d||dd|  k < d||ddd|   k< d||	dd|  k < d||	ddd|   k< ||  ||  }	}|	| j|< || j|< dS )	zn
        Sparse random projections as described in https://cs-people.bu.edu/evimaria/cs565/kdd-rp.pdf
        
persistentcpudevicer   r   rD   r   N)r\   rF   minr   save_projection
randlora_A
randlora_Br$   	Generatormanual_seedprojection_prng_keyrandr
is_integerr_   zerosrR   std)r<   r>   r?   r^   linear_out_dimlinear_in_dimmax_dimmin_dimr   rh   	num_basesri   randlora_B_sparserandlora_A_sparser0   r0   r1   "_init_randlora_A_randlora_B_sparse   s(   



z0RandLoraModel._init_randlora_A_randlora_B_sparsec           	        s   |   \}}t||t||}ti  jd| _ti  jd| _tjdd	 j
t jd|fd}t|| j }| rFt|nt|d }tj fddt|D dd}||  ||  }}|| j|< || j|< d S )	Nr`   rb   rc   r   re   c                   s    g | ]}t d  jfdqS )r   re   )r2   rn   )rG   _r>   r   rt   r0   r1   
<listcomp>   s     z=RandLoraModel._init_randlora_A_randlora_B.<locals>.<listcomp>)dim)r\   rF   rf   r   rg   rh   ri   r$   rj   rk   rl   r2   rn   ro   r_   catrangerq   )	r<   r>   r?   rr   rs   ru   rh   rv   ri   r0   r{   r1   _init_randlora_A_randlora_B   s   
z)RandLoraModel._init_randlora_A_randlora_Br=   	nn.Modulec                 C  s^   |j r| |\}}| j||tt||d d S |jr'| j||dd d S | || d S )N)r^   r]   )very_sparser\   ry   r,   r-   rf   sparser   )r<   r=   r>   r?   rr   rs   r0   r0   r1   _pre_injection_hook   s   
z!RandLoraModel._pre_injection_hookc                 C  s   t | jdkr|jdkrt| jj d| j D ]}||u r!q|j|jkr4td|jd|j dqtdd | j D }t |dkrNtd	| d
S )z
        A helper method to check the config when a new adapter is being added.

        Raise a ValueError if there is something wrong with the config or if it conflicts with existing adapters.

        r   nonezf supports only 1 adapter with bias. When using multiple adapters, set bias to 'none' for all adapters.zcRandLora PRNG initialisation key must be the same for all adapters. Got config.projection_prng_key=z but previous config had .c                 S  s   h | ]}|j qS r0   )rg   )rG   r>   r0   r0   r1   	<setcomp>   s    z:RandLoraModel._check_new_adapter_config.<locals>.<setcomp>zgRandLora projection weights must be saved for all adapters or none, but got multiple different values: N)	lenrV   biasrT   rA   __name__valuesrl   sorted)r<   r>   existing_configsave_project_unique_valuesr0   r0   r1   _check_new_adapter_config   s,   

z'RandLoraModel._check_new_adapter_configc                 C  s
   t | |S rE   )r   )randlora_configrX   r0   r0   r1   rM     s   
z)RandLoraModel._check_target_module_existsc              
   K  s   |d u rt d|j}t|do|jd u}	||j|j|j|jt| j	ddt| j	ddd}
|	|
d< t
|trH||| j| j||j|j|j d S | j|| j| j||fi |
}|| jvra|d | |||| d S )NzCurrent Key shouldn't be `None`r   is_loaded_in_8bitFis_loaded_in_4bit)rn   randlora_alpharandlora_dropoutfan_in_fan_outinit_weightsloaded_in_8bitloaded_in_4bit)rT   rn   rP   r   r   r   r   r   getattrr=   r"   r   update_layerrh   ri   _create_new_moduleactive_adapterrequires_grad__replace_module)r<   r   r?   targettarget_nameparentcurrent_keyoptional_kwargsrn   r   kwargs
new_moduler0   r0   r1   _create_and_replace  s>   
	



z!RandLoraModel._create_and_replacec                   s   t | || t|dr|j}t|ds |j|_t|dr |j|_t|dd d ur>t|dr3|j|j_n|j|_||jj t	d |
 D ]\}}d|v rct fdd| D sc||jj qGd S )N
base_layerr   statemetar4   c                 3  s    | ]}|j  kV  qd S rE   rc   )rG   pr   r0   r1   rI   K  s    z0RandLoraModel._replace_module.<locals>.<genexpr>)setattrrP   r   rQ   r   r   r   tord   r$   rL   any
parameters)r   
child_namer   childnamerY   r0   r   r1   r   1  s&   




zRandLoraModel._replace_modulec                 C  s   |  D ]\}}| j|vrd|_q| jD ]H}| j| j}|dkr!q|dkr6|  D ]\}}d|v r4d|_q)q|dkrU| D ]}t|trSt	|drS|jd urSd|j_q>qt
d| dd S )	NFr   allr   Trandlora_onlyzRequested bias: z, is not implemented.)named_parametersr6   requires_gradactive_adaptersrV   r   modulesr"   r   rP   NotImplementedError)r<   r=   nr   r   r   mr0   r0   r1    _mark_only_adapters_as_trainableN  s,   

z.RandLoraModel._mark_only_adapters_as_trainablec                 K  s  t  rdd l}ddlm} t rddlm} |dd}	|dd}
|dd}t|t	r2|
 }n|}|
rZt||jjrZ| }||jj|jj|jd	 |||||fi |S |rt||jjr| }||j|jj|jjd
 |||||fi |S t|tjjr|d rtd d |d< | _n"t|trd|d< |d std d |d< | _ntd| dt||||fd|	i|}|S )Nr   r   )Linear8bitLt)
Linear4bitr   Fr   r   )has_fp16_weights	thresholdindex)compute_dtypecompress_statistics
quant_typer   zjfan_in_fan_out is set to True but the target module is `torch.nn.Linear`. Setting fan_in_fan_out to False.Tis_target_conv_1d_layerzafan_in_fan_out is set to False but the target module is `Conv1D`. Setting fan_in_fan_out to True.zTarget module z is not supported. Currently, only the following modules are supported: `torch.nn.Linear`, `transformers.pytorch_utils.Conv1D`.)r   bitsandbytesbnbr   r
   r   popgetr"   r   get_base_layerr)   copyupdater   r   r   r   r   rQ   r   r   r$   r   warningswarnr   r	   rT   )r   rh   ri   r?   r   r   r   r   r   r   r   r   target_base_layereightbit_kwargsfourbit_kwargsr   r0   r0   r1   r   c  st   



	z RandLoraModel._create_new_moduler   c                   s8   zt  |W S  ty   |dkr t| j| Y S w )z1Forward missing attributes to the wrapped module.r=   )r:   __getattr__AttributeErrorr   r=   )r<   r   r@   r0   r1   r     s   zRandLoraModel.__getattr__	inferencec                 C  sF   i }| j  D ]\}}dd t| D }|rd|d< q|||< |S )Nc                 S  s&   i | ]\}}|t |tr|jn|qS r0   )r"   r   value)rG   kvr0   r0   r1   
<dictcomp>  s   & z9RandLoraModel.get_peft_config_as_dict.<locals>.<dictcomp>Tinference_mode)rV   itemsr   )r<   r   config_dictrX   r   r>   r0   r0   r1   get_peft_config_as_dict  s   z%RandLoraModel.get_peft_config_as_dictTc                 C  s,   | j  D ]}t|ttfr|| qd S rE   )r=   r   r"   r   r   enable_adapters)r<   enabledrY   r0   r0   r1   _set_adapter_layers  s
   
z!RandLoraModel._set_adapter_layersc                 C  s   | j dd d S )NTr   )r   r<   r0   r0   r1   enable_adapter_layers  s   z#RandLoraModel.enable_adapter_layersc                 C  sF   | j D ]}| j| j}|dkrd| d}t| q| jdd d S )Nr   z>Careful, disabling adapter layers with bias configured to be 'zL' does not produce the same output as the base model would without adaption.Fr   )r   rV   r   r   r   r   )r<   r   valr[   r0   r0   r1   disable_adapter_layers  s   


z$RandLoraModel.disable_adapter_layersc                 C  sF   | j  D ]}t|tr|jrtd |  || q|| _	d S )NzJAdapter cannot be set when the model is merged. Unmerging the model first.)
r=   r   r"   r   mergedr   r   unmergeset_adapterr   )r<   r?   rY   r0   r0   r1   r     s   



zRandLoraModel.set_adapterc                 C  s4   | j d u r|d tvrtdtt|d  | _ | S )N
model_typez0Please specify `target_modules` in `peft_config`)target_modulesr   rT   set)rV   rU   r0   r0   r1   rK     s   

z%RandLoraModel._prepare_adapter_configNprogressbar
safe_mergeadapter_namesOptional[list[str]]c              	   C  s   dd | j  D }d|rdnd d }t|| |dD ]?}zt| j |\}}	}
W n	 ty2   Y qw t|	drL|rA|	j||d	 | ||
|	 |	 qt	|	t
r[t||
|	j|	j  q| j S )
Nc                 S     g | ]
\}}d |vr|qS randlorar0   rG   rX   rz   r0   r0   r1   r|         z>RandLoraModel._unload_and_optionally_merge.<locals>.<listcomp>z
Unloading zand merging  r=   )disabledescr   )r   r   )r=   rL   r   r   r   rP   merger   r   r"   r   r   modules_to_saver   )r<   r   r   r   r   key_listr   rX   r   r   r   r0   r0   r1   _unload_and_optionally_merge  s    

z*RandLoraModel._unload_and_optionally_mergec                 C  s   |t | j vrtd| d| j|= dd | j D }d}|D ] }t| j|\}}}t|trC|	| |du rC|j
dd }q#|pGg | _
| j||d dS )z
        Deletes an existing adapter.

        Args:
            adapter_name (str): Name of the adapter to be deleted.
        zAdapter z does not existc                 S  r   r   r0   r   r0   r0   r1   r|     r   z0RandLoraModel.delete_adapter.<locals>.<listcomp>N)new_active_adapters)listrV   keysrT   r=   rL   r   r"   r   delete_adapterr   _delete_auxiliary_adapter)r<   r?   r   new_adapterrX   rz   r   r0   r0   r1   r     s   


zRandLoraModel.delete_adapterc                 C  s   | j |||dS )aL  
        This method merges the RandLora layers into the base model. This is needed if someone wants to use the base
        model as a standalone model.

        Args:
            progressbar (`bool`):
                whether to show a progressbar indicating the unload and merge process
            safe_merge (`bool`):
                whether to activate the safe merging check to check if there is any potential Nan in the adapter
                weights
            adapter_names (`list[str]`, *optional*):
                The list of adapter names that should be merged. If None, all active adapters will be merged. Defaults
                to `None`.

        Example:

        ```py
        >>> from transformers import AutoModelForCausalLM
        >>> from peft import PeftModel

        >>> base_model = AutoModelForCausalLM.from_pretrained("tiiuae/falcon-40b")
        >>> peft_model_id = "smangrul/falcon-40B-int4-peft-lora-sfttrainer-sample"
        >>> model = PeftModel.from_pretrained(base_model, peft_model_id)
        >>> merged_model = model.merge_and_unload()
        ```
        )r   r   r   r   )r<   r   r   r   r0   r0   r1   merge_and_unload  s   zRandLoraModel.merge_and_unloadc                 C  s   | j ddS )z
        Gets back the base model by removing all the RandLora modules without merging. This gives back the original
        base model.
        F)r   r  r   r0   r0   r1   unload5  s   zRandLoraModel.unload)F)r7   r8   r   r9   )r   rB   )r]   )r>   r   r?   r5   r^   r_   r   r9   )r>   r   r?   r5   r   r9   )r=   r   r>   r   r?   r5   r   r9   )r>   r   r   r9   )r=   r   r   r9   )r   r5   )r   r8   )T)TFFN)r   r8   r   r8   r   r   )r?   r5   )FFN)r   
__module____qualname____doc__r6   __annotations__r;   r\   ry   r   r   r   staticmethodrM   r   r   r   r   r   r   r   r   r   r   rK   r   r   r  r  __classcell__r0   r0   r@   r1   r3   H   sD   
 
%
)

"
,


E	
		

!r3   )r   r   r   r   r   r   ))
__future__r   r,   r   dataclassesr   enumr   typingr   r   r$   torch.nnr)   accelerate.utils.importsr   r   transformers.pytorch_utilsr	   peft.import_utilsr
   r   peft.tuners.tuners_utilsr   r   r   
peft.utilsr   r   r   _buffer_dictr   tuners_utilsr   r>   r   layerr   r   r2   r3   r0   r0   r0   r1   <module>   s(   
