o
    8wik;                     @  s  d dl mZ d dlZd dlmZmZmZ d dlmZ d dl	m	Z	 d dl
mZmZmZmZmZ d dlmZmZmZ d dlmZmZmZmZmZ ejejejejejfZej j!ej"j!ej#j!ej$j!gZ%eej&ej'ej(ej)ej*f Z+ej,j-ej,j.ej,j/ej,j0ej1fZ2G d	d
 d
eZ3dS )    )annotationsN)AnyOptionalUnion)nn)tqdm)adaloralohalokrloraoft)	BaseTunerBaseTunerLayercheck_target_module_exists)2TRANSFORMERS_MODELS_TO_LORA_TARGET_MODULES_MAPPINGModulesToSaveWrapperPeftType_get_submodulesget_auto_gptq_quant_linearc                      s   e Zd ZdZd> fd
dZd?ddZed@ddZdAddZdBddZ	dCddZ
edd ZdD fddZdEd d!Zd"d# Zd$d% ZdFd'd(Zed)d* Z		+	+	,dGdHd2d3ZdId4d5ZdFd6d7Z	,dJdKd8d9ZdLd:d;ZdMd<d=Z  ZS )N
MixedModela  
    A class that allows to mix different types of adapters in a single model.

    Note: This class should usually not be initialized directly. Instead, use `get_peft_model` with the argument
    `mixed=True`.

    Args:
        model (:obj:`nn.Module`):
            The model to be tuned.
        config (:obj:`PeftConfig`):
            The config of the model to be tuned. The adapter type must be compatible.
        adapter_name (:obj:`str`):
            The name of the first adapter.
    model	nn.ModuleconfigConfigsadapter_namestrreturnNonec                   s   t  ||| d S N)super__init__)selfr   r   r   	__class__ T/home/ubuntu/sommelier/.venv/lib/python3.10/site-packages/peft/tuners/mixed/model.pyr    8      zMixedModel.__init__c                 C  sn   t |tjst| jj dt dt| ddd | jD }dd |D }t	|dkr5t| jj d	d
S )z
        A helper method to check the config when a new adapter is being added.

        Raise a ValueError if there is something wrong with the config or if it conflicts with existing adapters.

        z only supports z configs, but got .c                 s  s    | ]	}t |d dV  qdS )biasN)getattr).0r   r$   r$   r%   	<genexpr>G   s    z7MixedModel._check_new_adapter_config.<locals>.<genexpr>c                 S  s   g | ]}|d vr|qS ))Nnoner$   )r*   r(   r$   r$   r%   
<listcomp>H   s    z8MixedModel._check_new_adapter_config.<locals>.<listcomp>   zf supports only 1 adapter with bias. When using multiple adapters, set bias to 'none' for all adapters.N)

isinstancer   __args__
ValueErrorr#   __name__COMPATIBLE_TUNER_TYPEStypepeft_configlen)r!   r   biasesr$   r$   r%   _check_new_adapter_config;   s   z$MixedModel._check_new_adapter_configkeyc                 C  s
   t | |S r   )r   )r   r9   r$   r$   r%   _check_target_module_existsO   s   
z&MixedModel._check_target_module_existsargsr   kwargsc                 O  s   t |tjrtjj| |g|R i | d S t |tjr,tjj| |g|R i | d S t |tj	rBtj
j| |g|R i | d S t |tjrXtjj| |g|R i | d S t |tjrntjj| |g|R i | d S tdt| dt d)NzUnsupported config type , should be one of r'   )r/   r   AdaLoraConfigAdaLoraModel_create_and_replacer   
LoraConfig	LoraModelr	   
LoHaConfig	LoHaModelr
   
LoKrConfig	LoKrModelr   	OFTConfigOFTModelr1   r4   r3   )r!   r   r;   r<   r$   r$   r%   r@   S   s        zMixedModel._create_and_replacec                   s   t ||| t|dr| }nt|dr|j}t|ds*|j|_t|dr*|j|_t|dd d urHt|dr=|j|j_n|j|_|	|jj
 | D ]!\ }t fddtD rb|	|jj
 d v rm|	|jj
 qLd S )N
base_layerquant_linear_moduler(   statec                 3      | ]}| v V  qd S r   r$   r*   prefixnamer$   r%   r+          z-MixedModel._replace_module.<locals>.<genexpr>ranknum)setattrhasattrget_base_layerrJ   weightr(   r)   rK   rI   todevicenamed_modulesanyPREFIXES)r!   parent
child_name
new_modulechildmoduler$   rO   r%   _replace_modulef   s*   





zMixedModel._replace_modulec                   s   |  D ]\ }t fddtD sd|_q| jD ]K}t| j| dd}|dkr*q|dkr?|  D ]\ }d v r=d|_q2q|dkr^| D ]}t|t	r\t
|dr\|jd ur\d|j_qGqtd	| d
d S )Nc                 3  rL   r   r$   rM   nr$   r%   r+      rQ   z>MixedModel._mark_only_adapters_as_trainable.<locals>.<genexpr>Fr(   r,   allT	lora_onlyzRequested bias: z, is not implemented.)named_parametersrZ   r[   requires_gradactive_adaptersr)   r5   modulesr/   LayersrT   r(   r1   )r!   r   pactive_adapterr(   mr$   rb   r%    _mark_only_adapters_as_trainable   s,   
z+MixedModel._mark_only_adapters_as_trainablec           	      K  sN  | dd }t|}|d us|d urtd| jj d|dd}|dd}|s,|r6td| jj dt| tjrJtj	j
| ||fi |}|S t| tjr^tjj
| ||fi |}|S t| tjrrtjj
| ||fi |}|S t| tjrtjj
| ||fi |}|S t| tjrtjj
| ||fi |}|S tdt|  d	t d
)Ngptq_quantization_configz$GPTQ quantization not supported for  (yet).loaded_in_8bitFloaded_in_4bitz-8bit and 4bit quantization not supported for zUnknown config type r=   r'   )getr   r1   	peft_typevaluepopr/   r   r>   r?   _create_new_moduler   rA   rB   r	   rC   rD   r
   rE   rF   r   rG   rH   r4   r3   )	r   r   targetr<   ro   AutoGPTQQuantLinearrq   rr   r^   r$   r$   r%   rw      s0   	zMixedModel._create_new_modulerP   c                   s8   zt  |W S  ty   |dkr t| j| Y S w )z1Forward missing attributes to the wrapped module.r   )r   __getattr__AttributeErrorr)   r   )r!   rP   r"   r$   r%   rz      s   zMixedModel.__getattr__Tc                 C  s,   | j  D ]}t|ttfr|| qd S r   )r   ri   r/   r   r   enable_adapters)r!   enabledr`   r$   r$   r%   _set_adapter_layers   s
   
zMixedModel._set_adapter_layersc                 C  s   | j dd d S )NTr}   )r~   r!   r$   r$   r%   enable_adapter_layers   s   z MixedModel.enable_adapter_layersc                 C  sL   | j D ]}t| j| dd}|dkrd| d}t| q| jdd d S )Nr(   r,   z>Careful, disabling adapter layers with bias configured to be 'zL' does not produce the same output as the base model would without adaption.Fr   )rh   r)   r5   warningswarnr~   )r!   rl   valmsgr$   r$   r%   disable_adapter_layers   s   


z!MixedModel.disable_adapter_layersUnion[str, list[str]]c                 C  sF   | j  D ]}t|tr|jrtd |  || q|| _	d S )NzJAdapter cannot be set when the model is merged. Unmerging the model first.)
r   ri   r/   rj   mergedr   r   unmergeset_adapterrl   )r!   r   r`   r$   r$   r%   r      s   



zMixedModel.set_adapterc                 C  s4   | j d u r|d tvrtdtt|d  | _ | S )N
model_typez0Please specify `target_modules` in `peft_config`)target_modulesr   r1   set)r5   model_configr$   r$   r%   _prepare_adapter_config   s   

z"MixedModel._prepare_adapter_configFNprogressbarbool
safe_mergeadapter_namesOptional[list[str]]c              	     s  |rt | jdd dkrtd fdd}dd | j D }d|r%d	nd
 d }t|| |dD ]P}zt| j|\}	}
}W n	 tyH   Y q2w t|
dr_|rT||
 | |	||
	 |
 q2t
|
tr|
j|
j }t|dr||rx|j d |	 }t|	|| q2| jS )Nquantization_methodgptqz4Cannot merge layers when the model is gptq quantizedc                   sx   g }| }t |dr|| |j}t |ds	t|d d |dd  D ]\}}|j d |j|_q#| j d d S )NrI   r.   r   r   )rT   appendrI   zipmerge)r`   pathlayerlayer_beforelayer_afterr   r   r$   r%   merge_recursively   s   


"
zBMixedModel._unload_and_optionally_merge.<locals>.merge_recursivelyc                   *   g | ]\ }t  fd dtD s qS )c                 3  rL   r   r$   rM   r9   r$   r%   r+      rQ   zEMixedModel._unload_and_optionally_merge.<locals>.<listcomp>.<genexpr>rZ   r[   r*   _r$   r   r%   r-         * z;MixedModel._unload_and_optionally_merge.<locals>.<listcomp>z
Unloading zand merging  r   )disabledescrI   r   )r)   r   r1   rY   r   r   r{   rT   ra   rU   r/   r   modules_to_saverl   r   rS   )r!   r   r   r   r   r   key_listr   r9   r\   rx   target_namer^   r$   r   r%   _unload_and_optionally_merge   s2   


z'MixedModel._unload_and_optionally_mergec                 O  s   t d| jj d)Nz(Weighted adapters are not supported for rp   )NotImplementedErrorr#   r2   r!   r;   r<   r$   r$   r%   add_weighted_adapter  r&   zMixedModel.add_weighted_adapterc           	      C  s   t |tr	|g}n|}t|t| j  }|r)tdt| dt| j  |D ]5}| j|= dd | j D }d}|D ] }t	| j|\}}}t |t
r_|| |du r_|jdd }q?q+|pdg | _| j||d dS )z
        Deletes an existing adapter.

        Args:
            adapter_name (Union[str, list[str]]): Name of the adapter(s) to delete.
        zAdapter(s) z  not found, available adapters: c                   r   )c                 3  rL   r   r$   rM   r   r$   r%   r+   -  rQ   z7MixedModel.delete_adapter.<locals>.<listcomp>.<genexpr>r   r   r$   r   r%   r-   -  r   z-MixedModel.delete_adapter.<locals>.<listcomp>N)new_active_adapters)r/   r   r   r5   keysr1   sortedr   rY   r   r   delete_adapterrh   rl   _delete_auxiliary_adapter)	r!   r   r   
mismatchedr   new_adapterr9   r   rx   r$   r$   r%   r     s,   



zMixedModel.delete_adapterc                 C  s   | j |||dS )a  
        This method merges the layers into the base model. This is needed if someone wants to use the base model as a
        standalone model.

        Args:
            progressbar (`bool`):
                whether to show a progressbar indicating the unload and merge process
            safe_merge (`bool`):
                whether to activate the safe merging check to check if there is any potential Nan in the adapter
                weights
            adapter_names (`List[str]`, *optional*):
                The list of adapter names that should be merged. If None, all active adapters will be merged. Defaults
                to `None`.
        )r   r   r   r   )r!   r   r   r   r$   r$   r%   merge_and_unload9  s   zMixedModel.merge_and_unloadc                 C  s   | j ddS )z
        Gets back the base model by removing all the lora modules without merging. This gives back the original base
        model.
        F)r   r   r   r$   r$   r%   unloadN  s   zMixedModel.unloadc                 O  s   | j j|i |S r   )r   generater   r$   r$   r%   r   U  s   zMixedModel.generate)r   r   r   r   r   r   r   r   )r   r   r   r   )r   r   r9   r   )r   r   r;   r   r<   r   r   r   )r   r   )r   r   r   r   )rP   r   )T)r   r   r   r   )TFFN)r   r   r   r   r   r   )r;   r   r<   r   r   r   )FFN)r   r   r   r   r   r   r   r   )r   r   )r;   r   r<   r   )r2   
__module____qualname____doc__r    r8   staticmethodr:   r@   ra   rn   rw   rz   r~   r   r   r   r   r   r   r   r   r   r   __classcell__r$   r$   r"   r%   r   (   s:    





	
	

0
"
r   )4
__future__r   r   typingr   r   r   torchr   r   peft.tunersr   r	   r
   r   r   peft.tuners.tuners_utilsr   r   r   
peft.utilsr   r   r   r   r   LORALOHALOKRADALORAOFTr3   rB   rN   rF   rD   rH   r[   rA   rC   rE   r>   rG   r   r   	LoraLayer	LoHaLayer	LoKrLayerAdaLoraLayerOFTLayerrj   r   r$   r$   r$   r%   <module>   s   
 