o
    Ni                     @  s   d dl mZ d dlZd dlZd dlmZ d dlZd dlmZ d dl	m
Z
mZ ddlmZ e r@d dlmZ G d	d
 d
ejjeZdddZdS )    )annotationsN)Optional)is_hqq_available)BaseTunerLayercheck_adapters_to_merge   )OFTLayer)	HQQLinearc                      sh   e Zd Z									d,d- fddZd.d/d d!Zd0d"d#Zd$d% Zd1d(d)Zd2 fd*d+Z  Z	S )3HqqOFTLinear   r           TFiUMu?   
base_layertorch.nn.Moduleadapter_namestrrintoft_block_sizemodule_dropoutfloatinit_weightsboolcoftepsblock_shareuse_cayley_neumannnum_cayley_neumann_termsreturnNonec                   sD   t    t| | d| _|| _| j||||||||	|
|d
 d S )NF)r   r   r   r   r   r   r   r   )super__init__r   fan_in_fan_out_active_adapterupdate_layer)selfr   r   r   r   r   r   r   r   r   r   r   kwargs	__class__ G/home/ubuntu/.local/lib/python3.10/site-packages/peft/tuners/oft/hqq.pyr"   !   s    

zHqqOFTLinear.__init__N
safe_mergeadapter_namesOptional[list[str]]c           
      C  s  t | |}|s	dS |D ]y}|| j vrq|  }i t|jd|ji}| }| 	|}t
|dd}t
|||j}t
|dd}||j|j}|rbt
| sbtd| dtd||j|jd}	|dd |	j|fi | |	| _| j| qdS )a  
            Merge the active adapter weights into the base weights

            Args:
                safe_merge (`bool`, *optional*):
                    If True, the merge operation will be performed in a copy of the original weights and check for NaNs
                    before merging the weights. This is useful if you want to check if the merge operation will produce
                    NaNs. Defaults to `False`.
                adapter_names (`list[str]`, *optional*):
                    The list of adapter names that should be merged. If None, all active adapters will be merged.
                    Defaults to `None`.
            Noffload_metar   r   z1NaNs detected in the merged weights. The adapter z seems to be brokencompute_dtypedevice)r   lora_Akeysget_base_layercopydeepcopyquant_configr/   
dequantizeget_delta_weighttorch	transposemmtodtyper2   isfiniteall
ValueErrorr	   r1   popquantizer   merged_adaptersappend)
r&   r,   r-   active_adapterlayerr8   outputoft_dataw_datanew_hqq_layerr*   r*   r+   mergeB   s0   


zHqqOFTLinear.mergec                 C  s  | j s
td dS t| jdkr| j }|| j vrq
|  }i t	
|jd|ji}| }| |}t|dd}t| ||j}t|dd}||j|j}td||j|jd}|dd |j|fi | || _t| jdksdS dS )z_
            This method unmerges all merged adapter layers from the base weights.
            z Already unmerged. Nothing to do.Nr   r/   r   r0   )mergedwarningswarnlenrE   rC   oft_Rr4   r5   r6   r7   r8   r/   r9   r:   r;   r<   r=   tr>   r?   r2   r	   r1   rD   r   )r&   rG   rH   r8   rI   rJ   rK   rL   r*   r*   r+   unmergen   s(   


zHqqOFTLinear.unmergec                 C  s   | j |  S N)rR   
get_weight)r&   adapterr*   r*   r+   r:      s   zHqqOFTLinear.get_delta_weightxtorch.Tensorc           
      O  s   | j |g|R i | |dd }| jr)| jr|   | j|g|R i |}n9| jr9| j|g|R i |}n)| jD ]%}|| j vrFq<| j| }t	
  }|r]|j}	| ||jj}||}q<| j|g|R i |}|ru||	}|S )Nr-   )_check_forward_argsrC   disable_adaptersrN   rT   r   active_adaptersrR   r4   r;   is_autocast_enabledr?   _cast_input_dtypeweightr>   )
r&   rX   argsr'   r-   resultrG   rR   requires_conversionexpected_dtyper*   r*   r+   forward   s*   




zHqqOFTLinear.forwardc                   s   t   }d| S )Nzoft.)r!   __repr__)r&   repr(   r*   r+   re      s   
zHqqOFTLinear.__repr__)	r   r   r   TFr   FFr   )r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r    )FN)r,   r   r-   r.   r   r    )r   r    )rX   rY   r   rY   )r   r   )
__name__
__module____qualname__r"   rM   rT   r:   rd   re   __classcell__r*   r*   r(   r+   r
      s     !
,
r
   targetr   r   r   c                 K  sB   d }t | tr|  }n| }t rt |trt||fi |}|S rU   )
isinstancer   r5   r   r	   r
   )rk   r   r'   
new_moduletarget_base_layerr*   r*   r+   dispatch_hqq   s   

ro   )rk   r   r   r   )
__future__r   r6   rO   typingr   r;   peft.import_utilsr   peft.tuners.tuners_utilsr   r   rH   r   hqq.core.quantizer	   nnModuler
   ro   r*   r*   r*   r+   <module>   s    