o Ni'5@sddlmZddlZddlZddlmZddlZddlmZddl m Z m Z ddl m Z mZddlmZmZmZmZmZdd lmZmZmZmZGd d d e ZdS) ) annotationsN)replace)Conv1D)is_bnb_4bit_availableis_bnb_available) BaseTunerBaseTunerLayer)6TRANSFORMERS_MODELS_TO_IA3_FEEDFORWARD_MODULES_MAPPING1TRANSFORMERS_MODELS_TO_IA3_TARGET_MODULES_MAPPINGModulesToSaveWrapper_freeze_adapter_get_submodules)Conv2dConv3dIA3LayerLinearcsteZdZUdZdZded<eZeddZ ddZ edd d Z ed dZ fddZ dddZdddZZS)IA3ModelaP Creates a Infused Adapter by Inhibiting and Amplifying Inner Activations ((IA)^3) model from a pretrained transformers model. The method is described in detail in https://huggingface.co/papers/2205.05638 Args: model ([`~transformers.PreTrainedModel`]): The model to be adapted. config ([`IA3Config`]): The configuration of the (IA)^3 model. adapter_name (`str`): The name of the adapter, defaults to `"default"`. low_cpu_mem_usage (`bool`, `optional`, defaults to `False`): Create empty adapter weights on meta device. Useful to speed up the loading process. Returns: `torch.nn.Module`: The (IA)^3 model. Example: ```py >>> from transformers import AutoModelForSeq2SeqLM, ia3Config >>> from peft import IA3Model, IA3Config >>> config = IA3Config( ... peft_type="IA3", ... task_type="SEQ_2_SEQ_LM", ... target_modules=["k", "v", "w0"], ... feedforward_modules=["w0"], ... ) >>> model = AutoModelForSeq2SeqLM.from_pretrained("t5-base") >>> ia3_model = IA3Model(config, model) ``` **Attributes**: - **model** ([`~transformers.PreTrainedModel`]) -- The model to be adapted. - **peft_config** ([`ia3Config`]): The configuration of the (IA)^3 model. ia3_strprefixcKstr ddl}ddlm}trddlm}|dd}|dd}|dd} t|tr2| } n|} |r\t| |j jr\| } | | j j| j j| jd |||fd| i| } | S|rt| |j jr| } | | j| jj| jjd |||fd| i| } | St|tj jrt||fd| i|} | St|tj jrt||fd| i|} | St| tj jr|d rtd d|d <|_t||fd| i|} | St| tr|d std d|d <|_t||f| dd|} | Std|d)Nrr) Linear8bitLt) Linear4bitloaded_in_8bitFloaded_in_4bitis_feedforward)has_fp16_weights thresholdindex) compute_dtypecompress_statistics quant_typefan_in_fan_outzjfan_in_fan_out is set to True but the target module is `torch.nn.Linear`. Setting fan_in_fan_out to False.zafan_in_fan_out is set to False but the target module is `Conv1D`. Setting fan_in_fan_out to True.T)ris_target_conv_1d_layerzTarget module zd is not supported. Currently, only `torch.nn.Linear`, `torch.nn.Conv2d`, and `Conv1D` are supported.)r bitsandbytesbnbrrrpop isinstancerget_base_layernncopyupdatestaterrrrweightr r!torchrrrwarningswarnr"r ValueError) ia3_config adapter_nametargetkwargsr%rrrrrtarget_base_layereightbit_kwargs new_modulefourbit_kwargsr:I/home/ubuntu/.local/lib/python3.10/site-packages/peft/tuners/ia3/model.py_create_new_moduleLs|       %  zIA3Model._create_new_modulec Cs|||}|j|j|t|jddt|jddd}t|tr(|||jdS|j|||fi|} ||j vr=| d| ||| |dS)Nis_loaded_in_8bitFis_loaded_in_4bit)r"init_ia3_weightsrrr) _check_target_module_feedforwardr"r?getattrmodelr'r update_layerr<active_adaptersrequires_grad__replace_module) selfr2r3r4 target_nameparent current_keyrr5r8r:r:r;_create_and_replaces      zIA3Model._create_and_replacereturnboolcs>t|jtrtt|j}|Stfdd|jD}|S)z A helper private method that checks if the target module `key` matches with a feedforward module specified in `ia3_config` c3s|]}|VqdSN)endswith).0 target_keykeyr:r; z.)r'feedforward_modulesrrMre fullmatchany)r2rSrr:rRr;r@s z)IA3Model._check_target_module_feedforwardcCsd|jdur|dtvrtdtt|d|_|jdur0|dtvr'tdtt|d|_|S)N model_typez0Please specify `target_modules` in `peft_config`z5Please specify `feedforward_modules` in `peft_config`)target_modulesr r1setrVr ) peft_config model_configr:r:r;_prepare_adapter_configs      z IA3Model._prepare_adapter_configcs>t|jddr tdt|jddrtdtj|i|S)a This method merges the (IA)^3 layers into the base model. This is needed if someone wants to use the base model as a standalone model. Args: safe_merge (`bool`, `optional`, defaults to `False`): If True, the merge operation will be performed in a copy of the original weights and check for NaNs before merging the weights. This is useful if you want to check if the merge operation will produce NaNs. Defaults to `False`. adapter_names (`List[str]`, *optional*): The list of adapter names that should be merged. If None, all active adapters will be merged. Defaults to `None`. r=Fz>Cannot merge ia3 layers when the model is loaded in 8-bit moder>z>Cannot merge ia3 layers when the model is loaded in 4-bit mode)rArBr1super_unload_and_optionally_merge)rGargsr5 __class__r:r;ras z%IA3Model._unload_and_optionally_mergeadapters list[str]tuple[str, str]csD]}|jvrtd|dqddD}tfdd|Dr*tdfdd D}fd d D}t|d ksHt|d krLtd t|vr]d fddD}n tjfddD}t|vr}d fddD}||fStjfddD}||fS)z Helper function to check if the arguments to add_weighted_adapter are valid and compatible with the underlying model. zAdapter z does not existcSsg|] }t|tr|qSr:)r'r )rPmoduler:r:r; sz8IA3Model._check_add_weighted_adapter..c3s*|]tfddDdkVqdS)c3s|]}|jvVqdSrN)modules_to_saverPadapterwrapperr:r;rTrUzAIA3Model._check_add_weighted_adapter...rN)sum)rPrermr;rTs z7IA3Model._check_add_weighted_adapter..zLCannot add weighted adapters targeting the same module with modules_to_save.ch|] }tj|jqSr:)typer]r[rkrGr:r; z7IA3Model._check_add_weighted_adapter..crqr:)rrr]rVrkrsr:r;rtrurzQAll adapter configs should have the same type for target and feedforward modules.|c3$|] }dj|jdVqdS()Nr]r[rkrsr:r;rTs"c3|] }j|jVqdSrNr{rkrsr:r;rTc3rwrxr]rVrkrsr:r;rTs c3r|rNr~rkrsr:r;rTr}) r]r1modulesrYlenrjoinr\union)rGrerlmodules_to_save_wrapperstarget_module_typesfeedforward_module_typesnew_target_modulesnew_feedforward_modulesr:)rerGr;_check_add_weighted_adapters2  z$IA3Model._check_add_weighted_adapterweights list[float]r3Nonecs|tjvr dSj|d\}}tj|d||dj|<j|tj|fddjD}|D]B}t j|\}} }t | t r|| j vrX| j |} nq=| j | _ t||D]\} } | | j vrs| j | } nqd| j | j | 7_ qdq=dS)ac This method adds a new adapter by merging the given adapters with the given weights. Args: adapters (`list`): List of adapter names to be merged. weights (`list`): List of weights for each adapter. adapter_name (`str`): Name of the new adapter. Nrpr)r[rVcsg|] \}}j|vr|qSr:)r)rPrS_rsr:r;ri,sz1IA3Model.add_weighted_adapter..)listr]keysrrinject_adapterrBr named_modulesr r'ria3_ldatazero_zip)rGrerr3rrkey_listrSrr4 target_ia3_lrlr-current_adapter_ia3_lr:rsr;add_weighted_adapter s6          zIA3Model.add_weighted_adapter)rLrM)rerfrLrg)rerfrrr3rrLr)__name__ __module__ __qualname____doc__r__annotations__rtuner_layer_cls staticmethodr<rKr@r_rarr __classcell__r:r:rcr;r$s  $ C    (r) __future__rrWr/ dataclassesrr.transformers.pytorch_utilsrpeft.import_utilsrrpeft.tuners.tuners_utilsrr peft.utilsr r r r r layerrrrrrr:r:r:r;s