o
    8wÖiš  ã                   @  sŽ   d dl mZ d dlZd dlZd dlmZ d dlZd dlmZ d dl	m
Z
mZ ddlmZ eƒ r@d dlmZ G d	d
„ d
ejjeƒZddd„ZdS )é    )ÚannotationsN)ÚOptional)Úis_hqq_available)ÚBaseTunerLayerÚcheck_adapters_to_mergeé   )ÚOFTLayer)Ú	HQQLinearc                      sh   e Zd Z									d,d-‡ fdd„Zd.d/d d!„Zd0d"d#„Zd$d%„ Zd1d(d)„Zd2‡ fd*d+„Z‡  Z	S )3ÚHqqOFTLinearé   r   ç        TFçiUMu?é   Ú
base_layerútorch.nn.ModuleÚadapter_nameÚstrÚrÚintÚoft_block_sizeÚmodule_dropoutÚfloatÚinit_weightsÚboolÚcoftÚepsÚblock_shareÚuse_cayley_neumannÚnum_cayley_neumann_termsÚreturnÚNonec                   sD   t ƒ  ¡  t | |¡ d| _|| _| j||||||||	|
|d
 d S )NF)r   r   r   r   r   r   r   r   )ÚsuperÚ__init__r   Úfan_in_fan_outÚ_active_adapterÚupdate_layer)Úselfr   r   r   r   r   r   r   r   r   r   r   Úkwargs©Ú	__class__© úP/home/ubuntu/sommelier/.venv/lib/python3.10/site-packages/peft/tuners/oft/hqq.pyr"   !   s    

özHqqOFTLinear.__init__NÚ
safe_mergeÚadapter_namesúOptional[list[str]]c           
      C  s  t | |ƒ}|s	dS |D ]y}|| j ¡ vrq|  ¡ }i t |j¡¥d|ji¥}| ¡ }|  	|¡}t
 |dd¡}t
 || |j¡¡}t
 |dd¡}| |j¡ |j¡}|rbt
 |¡ ¡ sbtd|› dƒ‚td||j|jd}	| dd¡ |	j|fi |¤Ž |	| _| j |¡ qdS )a†  
            Merge the active adapter weights into the base weights

            Args:
                safe_merge (`bool`, *optional*):
                    If True, the merge operation will be performed in a copy of the original weights and check for NaNs
                    before merging the weights. This is useful if you want to check if the merge operation will produce
                    NaNs. Defaults to `False`.
                adapter_names (`list[str]`, *optional*):
                    The list of adapter names that should be merged. If None, all active adapters will be merged.
                    Defaults to `None`.
            NÚoffload_metar   r   z1NaNs detected in the merged weights. The adapter z seems to be broken©Úcompute_dtypeÚdevice)r   Úlora_AÚkeysÚget_base_layerÚcopyÚdeepcopyÚquant_configr/   Ú
dequantizeÚget_delta_weightÚtorchÚ	transposeÚmmÚtoÚdtyper2   ÚisfiniteÚallÚ
ValueErrorr	   r1   ÚpopÚquantizer   Úmerged_adaptersÚappend)
r&   r,   r-   Úactive_adapterÚlayerr8   ÚoutputÚoft_dataÚw_dataÚnew_hqq_layerr*   r*   r+   ÚmergeB   s0   


ÿèzHqqOFTLinear.mergec                 C  s  | j s
t d¡ dS t| jƒdkr| j ¡ }|| j ¡ vrq
|  ¡ }i t	 
|j¡¥d|ji¥}| ¡ }|  |¡}t |dd¡}t | ¡ | |j¡¡}t |dd¡}| |j¡ |j¡}td||j|jd}| dd¡ |j|fi |¤Ž || _t| jƒdksdS dS )z_
            This method unmerges all merged adapter layers from the base weights.
            z Already unmerged. Nothing to do.Nr   r/   r   r0   )ÚmergedÚwarningsÚwarnÚlenrE   rC   Úoft_Rr4   r5   r6   r7   r8   r/   r9   r:   r;   r<   r=   Útr>   r?   r2   r	   r1   rD   r   )r&   rG   rH   r8   rI   rJ   rK   rL   r*   r*   r+   Úunmergen   s(   


ízHqqOFTLinear.unmergec                 C  s   | j |  ¡ S ©N)rR   Ú
get_weight)r&   Úadapterr*   r*   r+   r:   ‹   s   zHqqOFTLinear.get_delta_weightÚxútorch.Tensorc           
      O  sî   | j |g|¢R i |¤Ž | dd ¡}| jr)| jr|  ¡  | j|g|¢R i |¤Ž}n9| jr9| j|g|¢R i |¤Ž}n)| jD ]%}|| j ¡ vrFq<| j| }t	 
¡  }|r]|j}	|  ||jj¡}||ƒ}q<| j|g|¢R i |¤Ž}|ru| |	¡}|S )Nr-   )Ú_check_forward_argsrC   Údisable_adaptersrN   rT   r   Úactive_adaptersrR   r4   r;   Úis_autocast_enabledr?   Ú_cast_input_dtypeÚweightr>   )
r&   rX   Úargsr'   r-   ÚresultrG   rR   Úrequires_conversionÚexpected_dtyper*   r*   r+   ÚforwardŽ   s*   




zHqqOFTLinear.forwardc                   s   t ƒ  ¡ }d| S )Nzoft.)r!   Ú__repr__)r&   Úrepr(   r*   r+   re   ª   s   
zHqqOFTLinear.__repr__)	r   r   r   TFr   FFr   )r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r    )FN)r,   r   r-   r.   r   r    )r   r    )rX   rY   r   rY   )r   r   )
Ú__name__Ú
__module__Ú__qualname__r"   rM   rT   r:   rd   re   Ú__classcell__r*   r*   r(   r+   r
      s     ô!
,
r
   Útargetr   r   r   c                 K  sB   d }t | tƒr|  ¡ }n| }tƒ rt |tƒrt||fi |¤Ž}|S rU   )Ú
isinstancer   r5   r   r	   r
   )rk   r   r'   Ú
new_moduleÚtarget_base_layerr*   r*   r+   Údispatch_hqq¯   s   

ro   )rk   r   r   r   )Ú
__future__r   r6   rO   Útypingr   r;   Úpeft.import_utilsr   Úpeft.tuners.tuners_utilsr   r   rH   r   Úhqq.core.quantizer	   ÚnnÚModuler
   ro   r*   r*   r*   r+   Ú<module>   s    