o
    Ni(                     @  s   d dl mZ d dlZd dlZd dlmZmZ d dlZd dlm	Z	 d dl
mZmZ d dlmZ ddlmZmZ e	 rJd d	lmZ G d
d dejjeZdddZdS )    )annotationsN)AnyOptional)is_hqq_available)BaseTunerLayercheck_adapters_to_merge)	transpose   )	LoraLayerLoraVariant)	HQQLinearc                      sx   e Zd Z							d1d2 fddZd3ddZd4d5dd Zd6d!d"Zd#d$ Zd7d+d,Zd8d-d.Z	d9 fd/d0Z
  ZS ):HqqLoraLinearr   r	           TF
base_layertorch.nn.Moduleadapter_namestrrint
lora_alphalora_dropoutfloatinit_lora_weightsbool
use_rslorause_dora	lora_biasreturnNonec
              
     sV   |	rt | jj dt   t| | d| _|| _| j||||||||	d d S )Nz0 does not support lora_bias yet, set it to FalseF)r   r   r   r   r   r   )	
ValueError	__class____name__super__init__r
   fan_in_fan_out_active_adapterupdate_layer)selfr   r   r   r   r   r   r   r   r   kwargsr     H/home/ubuntu/.local/lib/python3.10/site-packages/peft/tuners/lora/hqq.pyr#   "   s    

zHqqLoraLinear.__init__Optional[LoraVariant]c                K  s   |sd S ddl m} | S )Nr	   )DoraLinearVariant)variantsr-   )r'   r   r(   r-   r*   r*   r+   resolve_lora_variantB   s   z"HqqLoraLinear.resolve_lora_variantN
safe_mergeadapter_namesOptional[list[str]]c           
      C  s   t | |}|s	dS |D ]k}|| j vrq|  }i t|jd|ji}| }|| j	vr9| 
|}|| }n
| j	| | ||}|rTt| sTtd| dtd||j|jd}	|dd |	j|fi | |	| _| j| qdS )a  
            Merge the active adapter weights into the base weights

            Args:
                safe_merge (`bool`, *optional*):
                    If True, the merge operation will be performed in a copy of the original weights and check for NaNs
                    before merging the weights. This is useful if you want to check if the merge operation will produce
                    NaNs. Defaults to `False`.
                adapter_names (`list[str]`, *optional*):
                    The list of adapter names that should be merged. If None, all active adapters will be merged.
                    Defaults to `None`.
            Noffload_metaz1NaNs detected in the merged weights. The adapter z seems to be brokencompute_dtypedevice)r   lora_Akeysget_base_layercopydeepcopyquant_configr3   
dequantizelora_variantget_delta_weight
merge_safetorchisfiniteallr   r   r5   r6   popquantizer   merged_adaptersappend)
r'   r0   r1   active_adapterlayerr<   output	lora_dataw_datanew_hqq_layerr*   r*   r+   mergeJ   s.   




zHqqLoraLinear.mergec                 C  s   | j s
td dS t| jdkry| j }|| j vrq
|  }i t	
|jd|ji}| }|| jvrJ| |}||j|j| }n
| j| | ||}td||j|jd}|dd |j|fi | || _t| jdksdS dS )z_
            This method unmerges all merged adapter layers from the base weights.
            z Already unmerged. Nothing to do.Nr   r3   r4   )mergedwarningswarnlenrF   rD   r7   r8   r9   r:   r;   r<   r3   r=   r>   r?   todtyper6   unmerger   r5   rE   r   )r'   rH   rI   r<   rJ   rK   rL   rM   r*   r*   r+   rU   u   s&   



zHqqLoraLinear.unmergec                 C  s(   t | j| j| j| j d| j|  S )NF)r   lora_Bweightr7   scaling)r'   adapterr*   r*   r+   r?      s   zHqqLoraLinear.get_delta_weightxtorch.Tensorargsr   	list[str]r(   c                  s  | j |g|R i |}t|}g }|D ] | fddt|D  qt|D ]X\}}	|	dkr2q)|	| j vr:q)| j|	 }
| j|	 }| j|	 }| j|	 }t	
  }|r`|j}| ||
jj}|||  }||
||| }|rw||}|||   |7  < q)|S )Nc                   s   g | ]
\}}| kr|qS r*   r*   ).0indexitemrY   r*   r+   
<listcomp>   s    z6HqqLoraLinear._mixed_batch_forward.<locals>.<listcomp>__base__)r   setrG   	enumerater7   r8   rV   r   rX   rA   is_autocast_enabledrT   _cast_input_dtyperW   rS   )r'   rZ   r1   r\   r(   resultunique_adapterssub_batch_indices_listirH   r7   rV   dropoutrX   requires_conversionexpected_dtype	sub_batchrJ   r*   ra   r+   _mixed_batch_forward   s0   





z"HqqLoraLinear._mixed_batch_forwardc                 O  sl  | j |g|R i | |dd }| jr*| jr|   | j|g|R i |}|S |d ur>| j|g|R d|i|}|S | jrO| j|g|R i |}|S | j|g|R i |}| jD ]U}|| j	 vrhq^| j| }| j
| }| j| }	| j| }
t  }|r|j}| ||jj}|| jvr||||	||
  }n| j| j| |||d}|r||}q^|S )Nr1   )rH   rZ   rh   )_check_forward_argsrD   disable_adaptersrO   rU   r   rp   active_adaptersr7   r8   rV   r   rX   rA   rf   rT   rg   rW   r>   forwardrS   )r'   rZ   r\   r(   r1   rh   rH   r7   rV   rl   rX   rm   rn   r*   r*   r+   rt      sJ   " 








zHqqLoraLinear.forwardc                   s   t   }d| S )Nzlora.)r"   __repr__)r'   repr)   r*   r+   ru      s   
zHqqLoraLinear.__repr__)r   r	   r   TFFF)r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   )r   r   r   r,   )FN)r0   r   r1   r2   r   r   )r   r   )
rZ   r[   r\   r   r1   r]   r(   r   r   r[   )rZ   r[   r   r[   )r   r   )r!   
__module____qualname__r#   r/   rN   rU   r?   rp   rt   ru   __classcell__r*   r*   r)   r+   r       s     
 
+
	
&+r   targetr   r   r   c                 K  sB   d }t | tr|  }n| }t rt |trt||fi |}|S )N)
isinstancer   r9   r   r   r   )rz   r   r(   
new_moduletarget_base_layerr*   r*   r+   dispatch_hqq   s   

r~   )rz   r   r   r   )
__future__r   r:   rP   typingr   r   rA   peft.import_utilsr   peft.tuners.tuners_utilsr   r   peft.utils.otherr   rI   r
   r   hqq.core.quantizer   nnModuler   r~   r*   r*   r*   r+   <module>   s    Q