o
    Ni                     @   s   d dl mZmZ d dlmZ d dlmZmZ ddlm	Z	 ddl
mZ ddlmZ ddlmZ dd	lmZ dd
lmZ ddlmZmZ G dd deZdS )    )is_bnb_4bit_availableis_bnb_available)	BaseTuner)1TRANSFORMERS_MODELS_TO_OFT_TARGET_MODULES_MAPPINGget_quantization_config   )dispatch_aqlm)dispatch_awq)dispatch_eetq)dispatch_gptq)dispatch_hqq)dispatch_inc)OFTLayerdispatch_defaultc                       sJ   e Zd ZU dZdZeed< eZe	Z
dd Zedd Z fdd	Z  ZS )
OFTModelaE  
    Creates Orthogonal Finetuning model from a pretrained model. The method is described in
    https://huggingface.co/papers/2306.07280

    Args:
        model (`torch.nn.Module`): The model to which the adapter tuner layers will be attached.
        config ([`OFTConfig`]): The configuration of the OFT model.
        adapter_name (`str`): The name of the adapter, defaults to `"default"`.
        low_cpu_mem_usage (`bool`, `optional`, defaults to `False`):
            Create empty adapter weights on meta device. Useful to speed up the loading process.

    Returns:
        `torch.nn.Module`: The OFT model.

    Example:
        ```py
        >>> from diffusers import StableDiffusionPipeline
        >>> from peft import OFTModel, OFTConfig

        >>> config_te = OFTConfig(
        ...     r=8,
        ...     target_modules=["k_proj", "q_proj", "v_proj", "out_proj", "fc1", "fc2"],
        ...     module_dropout=0.0,
        ...     init_weights=True,
        ... )
        >>> config_unet = OFTConfig(
        ...     r=8,
        ...     target_modules=[
        ...         "proj_in",
        ...         "proj_out",
        ...         "to_k",
        ...         "to_q",
        ...         "to_v",
        ...         "to_out.0",
        ...         "ff.net.0.proj",
        ...         "ff.net.2",
        ...     ],
        ...     module_dropout=0.0,
        ...     init_weights=True,
        ... )

        >>> model = StableDiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5")
        >>> model.text_encoder = OFTModel(model.text_encoder, config_te, "default")
        >>> model.unet = OFTModel(model.unet, config_unet, "default")
        ```

    **Attributes**:
        - **model** ([`~torch.nn.Module`]) -- The model to be adapted.
        - **peft_config** ([`OFTConfig`]): The configuration of the OFT model.
    oft_prefixc                 K   s$  |d u rt d|j|j|j|j|j|j|j|j|j	|j
t| jddt| jddd}g d}	|	D ]}
t| j|
d}|d urE|||
 d< q1t|tsxt| jd	rU| jjnd }| j|||fd
|i|}|| jvrn|d | |||| d S |j||j|j|j|j|j|j|j|j|j
d
 d S )NzCurrent Key shouldn't be `None`is_loaded_in_8bitFis_loaded_in_4bit)roft_block_sizemodule_dropoutcoftepsblock_shareuse_cayley_neumannnum_cayley_neumann_termsfan_in_fan_outinit_weightsloaded_in_8bitloaded_in_4bit)gptqaqlmawq)method_quantization_confighf_device_map
device_map)	r   r   r   r   r   r   r   r   r   )
ValueErrorr   r   r   r   r   r   r   r   r   r   getattrmodelr   
isinstancer   hasattrr&   _create_new_moduleactive_adaptersrequires_grad__replace_moduleupdate_layer)self
oft_configadapter_nametargettarget_nameparentcurrent_keyoptional_kwargskwargsquant_methodsquant_methodquantization_configr'   
new_module r?   I/home/ubuntu/.local/lib/python3.10/site-packages/peft/tuners/oft/model.py_create_and_replaceZ   sN   




zOFTModel._create_and_replacec           	   	   K   s   g }t  rddlm} || t rddlm} || |ttt	t
tttg d }|D ]}|||fd| i|}|d urA nq.|d u rNtd| d|S )Nr   )dispatch_bnb_8bit)dispatch_bnb_4bitr3   zTarget module zm is not supported. Currently, only the following modules are supported: `torch.nn.Linear`, `torch.nn.Conv2d`.)r   bnbrB   appendr   rC   extendr
   r   r	   r   r   r   r   r(   )	r3   r4   r5   r:   dispatchersrB   rC   r>   
dispatcherr?   r?   r@   r-      s8   


zOFTModel._create_new_modulec                    s<   t    t| jdddkrtd| jdrtddS )zVerify that the configuration supports merging.

        Currently gptq quantization and replicated layers do not support merging.
        quantization_methodNr!   z8Cannot merge OFT layers when the model is gptq quantizedlayer_replicationz=Cannot merge OFT layers when base model layers are replicated)super_check_merge_allowedr)   r*   r(   peft_configget)r2   	__class__r?   r@   rL      s   
zOFTModel._check_merge_allowed)__name__
__module____qualname____doc__r   str__annotations__r   tuner_layer_clsr   target_module_mappingrA   staticmethodr-   rL   __classcell__r?   r?   rO   r@   r   "   s   
 38
+r   N)peft.import_utilsr   r   peft.tuners.tuners_utilsr   
peft.utilsr   r   r"   r   r#   r	   eetqr
   r!   r   hqqr   incr   layerr   r   r   r?   r?   r?   r@   <module>   s   