o
    8wÖi\6  ã                   @   s¤	  d dl Z d dlmZ dd„ Zdd„ ZdeiZeedƒreed	< i d
g d¢“d	g d¢“dg d¢“dddg“dddg“dg d¢“dg d¢“dg d¢“dddg“dg d¢“dg d¢“ddd g“d!g d¢“d"g d#¢“d$g d#¢“d%d&g“d'd&g“Zi dd(d)g“dd(d)g“dd*d+g“dd,g“d	d-g“dg d.¢“d/d*d+g“dd*d+g“d0d-g“d1d*d+g“d2d3d4g“d5d3d4g“d6d3d4g“d7d3d4g“d8d9d:g“d;d<g“d=d3d4g“i d
d*d+g“d>d*d+g“d?d-g“dd,g“d@dAg“dBd-g“dCd-g“dd-g“dDdEd,g“dFdGg“dd*d+g“dHd*d+g“dId*d+g“dg dJ¢“d!d*d+g“d"d*d+g“d$d*d+g“¥d*d+gd*d+gdKœ¥Zi dg dL¢“dg dM¢“dd,dNg“d	d-dOg“d5g dP¢“d/g dQ¢“dg dR¢“d0d-dSg“d1g dT¢“dg dU¢“dd,dNg“d
g dV¢“d>g dW¢“dg dV¢“dHg dX¢“d2g dP¢“d8g dY¢“d<dZgd-dSgd-dSgd-dSgg dU¢g dW¢g dW¢g dW¢g dW¢g dW¢d[œ
¥Z	i dd\g“dg “ddNg“d	dOg“d5dZg“d/d]g“dd^g“d0dSg“d1dEg“dd]g“ddNg“d
d_g“d>d_g“dd_g“dHd`g“d2dZg“d8dZg“dZgdSgdSgdSgd]gd_gd_gd_gd_gd_gdaœ
¥Z
i dg db¢“dg dc¢“dg dd¢“dd,g“d	d-g“d/g dd¢“dd*d+g“d0d-g“d1d*d+g“d
d*d+g“d>d*d+g“d2d3d4g“d5g de¢“d8g df¢“dd,g“d;d<g“d!d*d+g“d*d+gd*d+gd*d+gd*d+gdgœ¥Zi dd(d)g“dd(d)g“dd*d+g“dd,g“d	d-g“dg d.¢“d/d*d+g“dd*d+g“d0d-g“d1d*d+g“d2d3d4g“d5d3d4g“d6d3d4g“d7d3d4g“d8d9d:g“d;d<g“d=d3d4g“i d
d*d+g“d>d*d+g“d?d-g“dd,g“d@dAg“dBd-g“dCd-g“dd-g“dDdEd,g“dFdGg“dd*d+g“dHd*d+g“dId*d+g“dd*d+g“d!d*d+g“d"d*d+g“d$d*d+g“¥d*d+gd*d+gdKœ¥Zi dd(d)g“dd(d)g“dd*d+g“ddNg“d	d-g“dg d.¢“d/d*d+g“dd*d+g“d0d-g“d1d*d+g“d2d3d4g“d5d3d4g“d6d3d4g“d7d3d4g“d8d9d:g“d;d<g“d=d3d4g“i d
d*d+g“d>d*d+g“d?d-g“ddNg“d@dAg“dBd-g“dCd-g“dd-g“dFdGg“dd*d+g“dHd*d+g“dId*d+g“dg dJ¢“d!d*d+g“d"d*d+g“d$d*d+g“d%d*d+g“¥d'd*d+gi¥Zi dg db¢“dg dc¢“dg dd¢“dd,g“d	d-g“d/g dd¢“dd*d+g“d0d-g“d1d*d+g“d
d*d+g“d>d*d+g“d2d3d4g“d5d3d4g“d8g df¢“dd,g“d;d<g“d!d*d+g“d*d+gd*d+gd*d+gd*d+gdgœ¥Zi dd(d)g“dd(d)g“dd*d+g“ddNg“d	d-g“dg d.¢“d/d*d+g“dd*d+g“d0d-g“d1d*d+g“d2d3d4g“d5d3d4g“d6d3d4g“d7d3d4g“d8d9d:g“d;d<g“d=d3d4g“i d
d*d+g“d>d*d+g“d?d-g“ddNg“d@dAg“dBd-g“dCd-g“dd-g“dFdGg“dd*d+g“dHd*d+g“dId*d+g“dg dJ¢“d!d*d+g“d"d*d+g“d$d*d+g“d%d*d+g“¥d'd*d+gi¥ZeZdhZdiZdjZdkdlgZdmdngZdoZdpZdqZdrdsiZdtZdS )ué    N)ÚBloomPreTrainedModelc                 C   s€   t  | ¡} | j\}}}}}| d |d … }| dd¡ |d || ||¡}| |d d … }| |d || ||¡}tt||ƒƒS )Né   é   )ÚtorchÚcatÚshapeÚ	transposeÚreshapeÚtupleÚzip)Úpast_key_valuesÚtotal_layersÚ
batch_sizeÚnum_attention_headsÚnum_virtual_tokensÚhead_dimÚkeysÚvalues© r   úQ/home/ubuntu/sommelier/.venv/lib/python3.10/site-packages/peft/utils/constants.pyÚ&bloom_model_postprocess_past_key_value   s   
ÿr   c                 C   sb   g }| D ](}|d d …d d …df }|  g d¢¡}|jg |jd d… ¢d‘R Ž }| |¡ qt|ƒS )Nr   )é   r   r   r   éþÿÿÿéÿÿÿÿ)Úpermuter	   r   Úappendr
   )r   ÚresultÚkr   r   r   Ú*starcoder_model_postprocess_past_key_value"   s   r   Úgpt_bigcodeÚ_convert_to_standard_cacheÚbloomÚllama)Úinput_layernormÚpost_attention_layernormÚnorm)r#   r$   Úln_fÚllava)Úmulti_modal_projectorr#   r$   r%   Úembed_tokensÚlm_headÚt5Ú
layer_normÚfinal_layer_normÚmt5Úbart)Úself_attn_layer_normÚencoder_attn_layer_normr-   Úgpt2)Úln_1Úln_2r&   zblip-2)Ú	layernormÚ	LayerNormr-   r0   Úgptjr3   r&   ÚfalconÚmistralÚphir#   Úfinal_layernormÚgemmaÚgemma2)r#   r$   Úpre_feedforward_layernormÚpost_feedforward_layernormr%   Úgemma3_textÚqwen2r$   Úqwen3ÚqÚvÚq_projÚv_projÚc_attnÚquery_key_value)rC   rD   rE   rF   ÚoptÚgpt_neoxÚgpt_neoÚbertÚqueryÚvalueÚrobertazxlm-robertaÚelectraz
deberta-v2Ú
query_projÚ
value_projÚdebertaÚin_projÚlayoutlmÚllama4ÚchatglmÚmptÚWqkvÚRefinedWebModelÚ
RefinedWebÚbtlmÚc_projÚcodegenÚqkv_projÚmixtralÚstablelm)rE   rF   Úfc1Úfc2)rA   rB   )r   rD   Úwo)r   rD   Úwi_1z
mlp.c_projzmlp.dense_4h_to_h)ÚkeyrN   úoutput.dense)rE   Úk_projrc   )rE   rF   Úfc_outÚdense_4h_to_h)rE   rF   r]   )rE   rF   rc   )rh   rF   Ú	down_proj)rE   rF   rk   )rh   rF   Úw2)Úkey_projrR   rg   rg   )
rS   rZ   r[   r8   r:   r<   r=   r@   rA   rB   rd   rc   ri   rk   rl   )
rS   r[   rZ   r8   r:   r<   r=   r@   rA   rB   )rC   r   rD   ÚoÚwird   )rC   r   rD   rn   Úwi_0re   rd   )rE   rh   rF   Úout_projrb   rc   )rM   rf   rN   Údense)rQ   rm   rR   rr   )r=   r@   rA   rB   zadapter_model.binzadapter_model.safetensorszadapter_config.jsonr)   r*   ÚscoreÚ
classifierz
all-linearztokenizer_config.jsonzdummy-target-modulesÚ
model_typeÚcustomé   )r   Útransformersr   r   r   Ú8TRANSFORMERS_MODELS_TO_PREFIX_TUNING_POSTPROCESS_MAPPINGÚhasattrÚ6TRANSFORMERS_MODELS_TO_LNTUNING_TARGET_MODULES_MAPPINGÚ2TRANSFORMERS_MODELS_TO_LORA_TARGET_MODULES_MAPPINGÚ1TRANSFORMERS_MODELS_TO_IA3_TARGET_MODULES_MAPPINGÚ6TRANSFORMERS_MODELS_TO_IA3_FEEDFORWARD_MODULES_MAPPINGÚ5TRANSFORMERS_MODELS_TO_ADALORA_TARGET_MODULES_MAPPINGÚ2TRANSFORMERS_MODELS_TO_VERA_TARGET_MODULES_MAPPINGÚ7TRANSFORMERS_MODELS_TO_FOURIERFT_TARGET_MODULES_MAPPINGÚ4TRANSFORMERS_MODELS_TO_VBLORA_TARGET_MODULES_MAPPINGÚ1TRANSFORMERS_MODELS_TO_C3A_TARGET_MODULES_MAPPINGÚ6TRANSFORMERS_MODELS_TO_RANDLORA_TARGET_MODULES_MAPPINGÚWEIGHTS_NAMEÚSAFETENSORS_WEIGHTS_NAMEÚCONFIG_NAMEÚEMBEDDING_LAYER_NAMESÚSEQ_CLS_HEAD_NAMESÚINCLUDE_LINEAR_LAYERS_SHORTHANDÚTOKENIZER_CONFIG_NAMEÚDUMMY_TARGET_MODULESÚDUMMY_MODEL_CONFIGÚ#MIN_TARGET_MODULES_FOR_OPTIMIZATIONr   r   r   r   Ú<module>   s  ÿ
ÿþýõôóòñðïîíìëä#Ý$Ü'ÿþýüûúùø	÷
öõôóòñðïîíìëêéèçæåäãâá à!ß"Þ#Ü'ÿþýüûúùø	÷
öõôóòñðïåÿþýüûúùø	÷
öõôóòñðïåÿþýüûúùø	÷
öõôóðïîìèÿþýüûúùø	÷
öõôóòñðïîíìëêéèçæåäãâá à!ß"Þ#Ü'ÿþýüûúùø	÷
öõôóòñðïîíìëêéèçæåäãâá à!ß"Þ#Ý&ÿþýüûúùø	÷
öõôóòñðïëÿþýüûúùø	÷
öõôóòñðïîíìëêéèçæåäãâá à!ß"Þ#Ý'ÿ