o
    NiG                     @  s.  d dl mZ d dlZd dlZd dlZd dlZd dlmZ d dlZd dl	Z	d dlm
Z
mZ d dlmZmZ d dlmZ d dlmZ d dlmZ d	d
lmZ d	dlmZmZmZmZmZmZmZ d	dl m!Z! dd Z"dd Z#	d6ddZ$	d7d8ddZ%d9d$d%Z&			d:d;d(d)Z'd*d+d,d-Z(	d<d=d4d5Z)dS )>    )annotationsN)Optional)file_existshf_hub_download)EntryNotFoundErrorLocalEntryNotFoundError)	load_file)http_user_agent)PEFT_TYPE_TO_PREFIX_MAPPING   )INCLUDE_LINEAR_LAYERS_SHORTHAND)EMBEDDING_LAYER_NAMESSAFETENSORS_WEIGHTS_NAMEWEIGHTS_NAMEAuxiliaryTrainingWrappercheck_file_exists_on_hf_hubinfer_devicematch_target_against_key)PeftTypec                 C  s"   t | dot| jtjjtjjfS )z.Check if the layer has an embedding base layer
base_layer)hasattr
isinstancer   torchnnLinear	Embedding)layer r   L/home/ubuntu/.local/lib/python3.10/site-packages/peft/utils/save_and_load.pyhas_valid_embedding_base_layer,   s   "r   c                 C  s:   |   D ]\}}|s||ks|t|ddkr|  S qdS )z7Get the name of the embedding module for a given layer.r   N)named_modulesgetattr)modelr   is_embedding_in_target_modulesnamemoduler   r   r   get_embedding_layer_name1   s
   r&   defaultFautoc              	     s  |rt | d| } | j  
du r|  
jtjtjfv rj}|dkr/
fdd
D }n9|dkr=
fdd
D }n+|dkrfi }
D ]}d	|v rd
| ||< |d	d
 d }|
v rd
| ||< qEnt	 fdd|
 D }jtjkrj}	|	dur fdd|	
 D }	|	_| |	| }jrd  dfddfdd|
 D }njtjkrj}|dkrʇ
fdd
D }n|dkrه
fdd
D }n|dkri }
D ]!}d|v r
| ||< |dd
 d }|
v r
| ||< qnt	jtjkr
fdd
D }n|jrVi }jtjkr?| j  j|d< | j  j|d< | j  jj}
njrK| j  jj}
n|  }
|
|d< n@jtjkrtj 		
fdd
D }t dkrxtd |  D ],\}t |dr|j!
 D ]\}}t dkr|"t#j$n|| d| < qq|njtj%krtj 
fd d
D }j&rd!  
vrt'd"
d!   |d!  < 
d#   |d#  < njtj(kr
fd$d
D }njtj)krvi }j*d%k r	t#j+}nj*d&k rt#j,}nj*d'k rt#j-}nt#j.}j/rb
D ]:}d(|v r_
| 0j0\}}|1|d) |j"|d*i |1|d+ t#j2|d,d-dddddd,f 3 i q&n	
fd.d
D }
d/   |d/  < n jt4tv rtj 
fd0d
D }nt'd1j |  D ]3\}t5|t6r̈7d2r8d2fd3d

 D }|1fd4d|9 |
 D  qd5}t d6rt5j:t;rj:t<krt | d7r| = n| }t>fd8d9| D }nj:rt>fd:d9t?D }jtj@kpt d;ddu}|d<kr.|r.|s.td= d>}n`|d<krt t | d?dd@d}t dAd}d5}|durptAjBCtAjBD|dB}|p\tE|dB}|du rntdC| dD d5}n|}|r|r|r|| jFjGH|jIkrtdE d>}nd5}|rt | dFr| J | K fD ]"}|rtL|rtM| ||r|1fdGd

 D  qn|rtdH tNOtNPdI  dJ  fdKdLfdMd|
 D }|S )Nu  
    Get the state dict of the given adapter of the PEFT model.

    This only includes the PEFT parameters, not the parameters of the base model. Thus the returned `state_dict` is
    generally small compared to the full model size. To retrieve the full `state_dict`, just call `model.state_dict()`.

    Note that the adapter name is removed from the `state_dict`, as this is just an arbitrary name that can be changed
    when loading the adapter. So e.g. if the adapter name is `'default'` and the original key is
    `'model.q_proj.lora_A.default.weight'`, the returned key will be `'model.q_proj.lora_A.weight'`. Use this function
    in conjunction with [`set_peft_model_state_dict`] to take care of the adapter name when loading weights.

    Args:
        model ([`PeftModel`]): The Peft model. When using torch.nn.DistributedDataParallel, DeepSpeed or FSDP,
            the model should be the underlying model/unwrapped model (i.e. model.module).
        state_dict (`dict`, *optional*, defaults to `None`):
            The state dict of the model. If not provided, the state dict of the passed model will be used.
        adapter_name (`str`, *optional*, defaults to `"default"`):
            The name of the adapter whose state dict should be returned.
        unwrap_compiled (`bool`, *optional*, defaults to `False`):
            Whether to unwrap the model if torch.compile was used.
        save_embedding_layers (`Union[bool, str]`, , *optional*, defaults to `auto`):
            If `True`, save the embedding layers in addition to adapter weights. If `auto`, checks the common embedding
            layers `peft.utils.other.EMBEDDING_LAYER_NAMES` in config's `target_modules` when available. Based on it
            sets the boolean flag. This only works for 🤗 transformers models.

    	_orig_modNnonec                      i | ]}d |v r| | qS )lora_r   .0k
state_dictr   r   
<dictcomp>d       z-get_peft_model_state_dict.<locals>.<dictcomp>allc                   &   i | ]}d |v sd|v r| | qS r,   biasr   r-   r0   r   r   r2   f      & 	lora_onlyr,   r   r7   c                   s.   i | ]\}}d |v r |v sd|v r||qS r6   r   r.   r/   vadapter_namer   r   r2   q   s   . c                   s$   i | ]\}}| d   d|qS ). )replacer:   r<   r   r   r2   u   s   $ lora_magnitude_vector..weightc                   s   |   r| d d } | S )Niendswithr/   )new_dora_suffixr   r   renamed_dora_weights   s   
z7get_peft_model_state_dict.<locals>.renamed_dora_weightsc                      i | ]	\}} ||qS r   r   r:   rG   r   r   r2          c                   r+   )boft_r   r-   r0   r   r   r2      r3   c                   r5   )rK   r7   r   r-   r0   r   r   r2      r8   	boft_onlyrK   c                   s*   i | ]}| d d dr| | qS )r>   	adaption_)split
startswithr-   r0   r   r   r2      s   * prefix_task_colsprefix_task_rowsprompt_embeddingsc                      i | ]} |v r|| qS r   r   r-   )shira_prefixr1   r   r   r2      r3   WindowszWindows has issues saving integers into safetensors. Hence, we convert shira_indices to float32 before saving on Windows OS. The shira_indices will always be converted to integers when loading.shira_indices.shira_indices.c                   s   i | ]}|v r| | qS r   r   r-   )r1   vera_prefixr   r   r2      r3   zbase_model.vera_A.zModel was initialised to not save vera_A and vera_B but config now specifies to save projection! Set `config.save_projection` to `False`.zbase_model.vera_B.c                   r+   )internal_xlora_classifierr   r-   r0   r   r   r2      r3      i   l        vblora_logits_topk_indices)dtype_topk_weightsrM   dimc                   r+   )r\   r   r-   r0   r   r   r2      r3   zbase_model.vblora_vector_bank.c                   rT   r   r   r-   )prefixr1   r   r   r2      r3   zUnknown PEFT type passed: _fsdp_wrapped_module.c                   s2   i | ]\}}|   d r|  d |qS r>   )rP   removeprefixr:   r$   r   r   r2      s
    c                   s    i | ]\}}  d | |qS rd   r   r:   rf   r   r   r2      s     Ftarget_modulesget_base_modelc                 3  s6    | ]\ }t  fd dtD rtj V  qdS )c                 3  s$    | ]}t d | d V  qdS )z(.*\.)?$N)rematch)r.   erE   r   r   	<genexpr>  s   " z6get_peft_model_state_dict.<locals>.<genexpr>.<genexpr>N)anyr   r   rg   )r.   _configrE   r   rm   
  s    

z,get_peft_model_state_dict.<locals>.<genexpr>c                 3  s    | ]}| j v V  qd S )N)rg   r-   rp   r   r   rm     s    trainable_token_indicesr(   zXSetting `save_embedding_layers` to `True` as embedding layers found in `target_modules`.Trq   
vocab_sizebase_model_name_or_pathzconfig.jsonz Could not find a config file in z4 - will assume that the vocabulary was not modified.zdSetting `save_embedding_layers` to `True` as the embedding layer has been resized during finetuning.get_input_embeddingsc                   s   i | ]\}} |v r||qS r   r   r:   )embedding_module_namer   r   r2   C  r3   uY   Could not identify embedding layer(s) because the model is not a 🤗 transformers model.r>   ri   c                   s   d| vr| S |  d  r| d  S | d\} }}jtjkr8|  dr8| d |  d S d| } |  d| S )Nr>   ro   r?   )	rD   removesuffix
rpartition	peft_typer   VBLORArP   re   sub)keyro   suffix)r=   rq   patternr   r   remove_adapter_nameK  s   z6get_peft_model_state_dict.<locals>.remove_adapter_namec                   rH   r   r   r:   )r   r   r   r2   `  rJ   )Qr!   peft_configr1   ry   r   LORAADALORAr7   rO   NotImplementedErroritemsrank_pattern!resize_state_dict_by_rank_patternuse_doraBOFTADAPTION_PROMPTis_prompt_learningMULTITASK_PROMPT_TUNINGprompt_encoderrQ   rR   	embeddingweightinference_modeget_prompt_embedding_to_saveSHIRAr
   platformsystemwarningswarnr    r   rW   tor   float32VERAsave_projection
ValueErrorXLORArz   num_vectorsuint8int16int32int64save_only_topk_weightstopkupdatesoftmax
contiguouslistr   r   rP   re   adapter_state_dictrg   strr   rh   rn   r   TRAINABLE_TOKENSospathexistsjoinr   rq   	__class__from_pretrainedrs   ru   get_output_embeddingsr   r&   rj   compileescape)r"   r1   r=   unwrap_compiledsave_embedding_layersr7   	to_returnr/   	bias_namer   rS   r%   r;   indices_dtypelogitsindicesmodule_state_dictembedding_is_targeted_modelusing_trainable_tokensrs   model_idhas_base_configlocal_config_existsr   r   r   )r=   rq   rv   r$   rF   r~   rb   r   rG   rU   r1   rY   r   get_peft_model_state_dict9   sf  







6









r   r"   torch.nn.Modulepeft_model_state_dictdict[str, torch.Tensor]ignore_mismatched_sizesboolreturnRtuple[dict[str, torch.Tensor], list[tuple[str, tuple[int, ...], tuple[int, ...]]]]c                 C  s   |s|g fS g }|   }| D ]3\}}||vrq|| jd dkr/||  d | kr/q|| j|jkrC|||j|| jf q|D ]\}}}||= qF||fS )NrM   r      )r1   r   shapenumelappend)r"   r   r   
mismatchedr1   r|   tensorro   r   r   r   _find_mismatched_keysd  s   *r   r1   r=   r   parameter_prefixc           	      C  s   i }|   D ]B\}}||v rD||\}}}d|v r8d|ddd }tt|d | d| |}n| d| }|||< q|||< q|S )zbUtility function to remap the state_dict keys to fit the PEFT model by inserting the adapter name.r>   r   Nri   )r   rx   r   rO   rj   r{   r   )	r1   r=   r   r   r|   valro   r}   suffix_to_replacer   r   r   $_insert_adapter_name_into_state_dict  s   $

r   low_cpu_mem_usageNonec                   s  | j | }|}|  D ]6\}}t|trA||}	|dr#|d}|	D ]}
| d|
 }| d|	|
  }|| ||< ||= q%q|jsK|jt	j
krO|}nK|jt	jkrY|}nA|jtv ri }t|j }|jt	jkr|jr| j| j\}}t| }|D ][}
d|
v r||
 tj}|
dd}||
dd }tj|d|jddd	 gdd
}t|}tg |jdd |td|jd||}|||< ||
= ||
dd= qt|||d}|jt	j kr|j!}|dur| "|| n|jt	j#kr7t$% dkr	t&'d |  D ](\}}t(|dr4| d| |v r4|)| d| }|tj*|j+|< qnc|jt	j,krd|j-rKd|vrKt.d|j-sZd|v rZt&'d n@|j-sct&'d n6|jt	j/krd|   fddfdd|0 D }n|jt	j1krt2dd |D rt.dnt3t4| ||d\}}|r| j5|ddd }| 6 D ]}t(|d!r|7| qn| j5|dd"}|jr| j8| j9j5d#|d$ idd" |jt	j:kr| j8| j5|dd" |r
d%;d&d' |D }d(| j<j= d)| d}t&'| |S )*a  
    Set the state dict of the PEFT model.

    Given a PEFT `state_dict` (as returned by [`get_peft_model_state_dict`]), insert the weights into the model. The
    model needs to have the PEFT adapters already in place (e.g. via [`inject_adapter_in_model`]).

    Setting the adapter weights also takes care of re-inserting the adapter name. This name may be a different name
    than the one originally used to train the adapter.

    Args:
        model ([`PeftModel`]):
            The Peft model.
        peft_model_state_dict (`dict`):
            The state dict of the Peft model.
        adapter_name (`str`, *optional*, defaults to `"default"`):
            The name of the adapter whose state dict should be set.
        ignore_mismatched_sizes (`bool`, *optional*, defaults to `False`):
            Whether to ignore mismatched in the state dict.
        low_cpu_mem_usage (`bool`, `optional`, defaults to `False`):
            This argument must be `True` if the `model` was loaded with adapter weights on the meta device, e.g. after
            calling `inject_adapter_in_model` with `low_cpu_mem_usage=True`. Otherwise, leave it as `False`.

    rc   r>   r]   r?   r_   r   rM   T)keepdimr`   Nz-inf)r=   r   rV   zWindows has issues saving integers into safetensors. Hence, we had converted shira_indices to float32 before saving on Windows OS. The shira_indices will always be converted to integers when loading.rW   rX   zbase_model.vera_AzXSpecified to load vera_A and vera_B from state dictionary however they were not present!zSpecified to not load vera_A and vera_B from state dictionary however they are present in state dictionary! Consider using them to ensure checkpoint loading is correct on all platforms using `peft_config.save_projection = True`zSpecified to not load vera_A and vera_B from state dictionary. This means we will be relying on PRNG initialisation to restore these projections using `config.projection_prng_key`, which may not be accurate on all system configurations.rA   c                   s   |   r	| d } | S )NrB   rC   rE   )old_dora_suffixr   r   rG     s   
z7set_peft_model_state_dict.<locals>.renamed_dora_weightsc                   rH   r   r   r:   rI   r   r   r2   "  rJ   z-set_peft_model_state_dict.<locals>.<dictcomp>c                 s  s    | ]}d |v V  qdS )z.oft_r.Nr   )r.   r|   r   r   r   rm   $  s    z,set_peft_model_state_dict.<locals>.<genexpr>zTrying to load old OFT checkpoint, which is no longer supported. Please install PEFT <= v0.15.2 to load it or train a new OFT adapter.)r   F)strictassign%_move_adapter_to_device_of_base_layer)r   r   rS   
c              	   S  s*   g | ]\}}}d | d| d| dqS )z- z: found shape z in the checkpoint and z in the model instantiatedr   )r.   r|   shape1shape2r   r   r   
<listcomp>B  s    z-set_peft_model_state_dict.<locals>.<listcomp>zSome weights of zy were not initialized from the model checkpoint and are being ignored because you passed `ignore_mismatched_sizes=True`: )>r   r    r   r   adapter_state_dict_load_maprP   re   r   ry   r   r   r   r
   rz   r   vblora_vector_bankr   r   keysr   r   longr@   catsumlogzerosfill_floatdevicescatterr   r   r   resize_modules_by_rank_patternr   r   r   r   r   r   popintrW   r   r   r   r   r   OFTrn   r   r   load_state_dictmodulesr   r   r   r   r   r   __name__)r"   r   r=   r   r   rq   r1   r$   r%   key_mapr/   
lookup_key	store_keyr   r   ro   state_dict_keysr;   original_keytopk_weightstopk_logitsmatrixr   shira_indices_valuesmismatched_keysload_resultmismatched_warningmsgr   )r   rG   r   set_peft_model_state_dict  s   





 


	



r   T)weights_onlyc                 O  s   t j|d| i|S )zCall torch.load and handle weights_only.

    Defaults to weights_only=True to anticipate upcoming switch on the PyTorch side.

    r   )r   load)r   argskwargsr   r   r   
torch_loadP  s   r  r   r   Optional[str]key_mappingOptional[dict[str, str]]dictc                   s    dddurtj|  d n| }|du rt }d fdd	}d vr*t  d< tjtj|tr?tj|t}d}ntjtj|trTtj|t}d}nt	j
jr|dd} d	d zt| |fd	di }d}W nv ty   |dd}t| |fd	di }d}Y n\w   d
d}	|	du r  dd}	|dd}t| |  dd  dd|	d}
|
}|
rt| tfi  }n'zt| tfi  }W n ty   td|  d|  dt dt d|  dw |r	ttjdr|tdkrt|dd}nt||d}n	t|t|d}|s|}|S i }| D ]F\}}|dr,d}n|dr5d}ntd||}| D ]\}}t|||\}}|dkrX|} nqB| | }|||< q|S )aP  
    A helper method to load the PEFT weights from the HuggingFace Hub or locally

    Args:
        model_id (`str`):
            The local path to the adapter weights or the name of the adapter to load from the HuggingFace Hub.
        device (`str`):
            The device to load the weights onto.
        key_mapping (dict, *optional*, defaults to None)
            Extra mapping of PEFT `state_dict` keys applied before loading the `state_dict`. When this mapping is
            applied, the PEFT-specific `"base_model.model"` prefix is removed beforehand and the adapter name (e.g.
            `"default"`) is not inserted yet. Only pass this argument if you know what you're doing.
        hf_hub_download_kwargs (`dict`):
            Additional arguments to pass to the `hf_hub_download` method when loading from the HuggingFace Hub.
    	subfolderNTc                   s2   | rt nt} dd d urtj d |S |S )Nr	  )r   r   getr   r   r   )use_safetensorsweights_namehf_hub_download_kwargsr   r   get_hub_filenamet  s   z+load_peft_weights.<locals>.get_hub_filename
user_agentF)r  local_files_onlytokenuse_auth_tokenrevision	repo_type)repo_idfilenamer  r  r  zCan't find weights for z in z8 or in the Hugging Face Hub. Please check that the file z or z is present at r>   mpscpu)r   )map_locationzbase_model.model.zbase_model.zAn error occurred while trying to load a PEFT state_dict with key_mapping. This should not happen. Please open an issue on https://github.com/huggingface/peft/issues and report the error.r   )T)r
  r   r   r   r   r	   r   r   r   huggingface_hub	constantsHF_HUB_OFFLINEr   r   r   r   r   r   r   r   backendsr   safe_load_filer  r   rP   re   rj   subn)r   r   r  r  r   r  r  r  hub_filenamer  has_remote_safetensors_fileadapters_weightsremapped_adapters_weightsr|   r   rb   r~   replacementkey_new	n_replacekey_with_prefixr   r  r   load_peft_weightsY  s   







r)  )Nr'   Fr(   )F)r"   r   r   r   r   r   r   r   )r1   r   r=   r   r   r   r   r   )r'   FF)r   r   r   r   r   r   )NN)r   r   r   r  r  r  r   r  )*
__future__r   r   r   rj   r   typingr   r  r   r   r   huggingface_hub.errorsr   r   safetensors.torchr   r  transformers.utilsr	   peft.mappingr
   r  r   otherr   r   r   r   r   r   r   
peft_typesr   r   r&   r   r   r   r   r  r)  r   r   r   r   <module>   sB   $		
  .
 <
