o
    ÁÙ·iSa  ã                   @   sV   d dl Z d dlZd dlZd dlmZmZmZ d dlmZ e  	e
¡ZG dd„ deƒZdS )é    N)ÚTensorProtoÚhelperÚnumpy_helper)ÚBertOnnxModelc                       sŒ   e Zd Z‡ fdd„Zdd„ Zdd„ Zdd„ Zd	d
„ Zdd„ Zdd„ Z	dd„ Z
dd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Z‡  ZS )ÚBertOnnxModelTFc                    s   t ƒ  |||¡ d S ©N)ÚsuperÚ__init__)ÚselfÚmodelÚ	num_headsÚhidden_size©Ú	__class__© ú_/home/ubuntu/.local/lib/python3.10/site-packages/onnxruntime/transformers/onnx_model_bert_tf.pyr	      s   zBertOnnxModelTF.__init__c                 C   sp   g }|   ¡ D ] }|jdkr&|  |jd ¡s&|  |jd |jd ¡ | |¡ q|  |¡ t 	dt
|ƒ› ¡ d S )NÚIdentityr   zRemoved Identity count: )ÚnodesÚop_typeÚfind_graph_outputÚoutputÚreplace_input_of_all_nodesÚinputÚappendÚremove_nodesÚloggerÚinfoÚlen)r
   Únodes_to_removeÚnoder   r   r   Úremove_identity   s   

€
zBertOnnxModelTF.remove_identityc                 C   s^   |   |g d¢g d¢¡}|d ur|S |   |g d¢g d¢¡}|d ur"|S |   |g d¢g d¢¡}|S )N)ÚMulÚSubÚReshapeÚCast)é   Nr%   r   )r!   r"   r$   ÚSliceÚ	Unsqueeze)r%   r   r%   r   r   )r!   r"   r$   r'   r'   )r%   Nr%   r   r   )Úmatch_parent_path)r
   Úadd_or_sub_before_softmaxÚ
mask_nodesr   r   r   Úmatch_mask_path   s(   ýýýzBertOnnxModelTF.match_mask_pathc                 C   s\   |   |g ¡}i }|D ]!}|jD ]}|  |¡}|r*t |¡}t|jƒdkr*|j||j< qq
|S )zi
        Find initializers that is 2D. Returns a dictionary with name as key and shape as value.
        é   )Úget_parent_subgraph_nodesr   Úget_initializerr   Úto_arrayr   ÚshapeÚname)r
   Úcurrent_nodeÚparent_nodesÚinitializersr   r   ÚinitializerÚtempr   r   r   Ú)get_2d_initializers_from_parent_subgraphs7   s   


€ûz9BertOnnxModelTF.get_2d_initializers_from_parent_subgraphsc                 C   sD  |   ¡ }||vr
d S || }t|ƒdkrd S | j|d dd}t|ƒdkr,td|ƒ d S t|ƒdkr<|d |kr<|d S |d jd }|  |d g d¢g d¢fg d¢g d	¢fgd ¡\}}}|r |r ||d
 jd kr t d¡ |d }	|  |	¡j	}
|  
tjd|gdgd|
¡ t |	jd ¡}|  
tjddgdg|d|
¡ d}|S )Nr%   r   T©Ú	recursivez(Found multiple candidates of segment_ids)ÚConstantOfShaper$   ÚConcatr&   r$   ÚShape)r%   r   r   r   r   r   ©r:   r$   r;   r'   ÚSqueezer&   r$   r<   )r%   r   r   r   r   r   r   r   éÿÿÿÿúSimplify semgent id path...r<   Úinput_shape©ÚinputsÚoutputsr:   Úzeros_for_input_shape©rC   rD   Úvalue)Úinput_name_to_nodesr   Úget_graph_inputsÚprintr   Úmatch_parent_pathsr   ÚdebugÚget_graph_by_noder1   Úadd_noder   Ú	make_nodeÚget_attribute_valueÚ	attribute)r
   Úsegment_embeddingÚ	input_idsrH   r   Úgraph_inputsÚsegment_idsÚ_Úsegment_id_pathÚconstantofshape_nodeÚ
graph_nameÚconstantofshape_valuer   r   r   Úfind_segment_idsG   sX   
þ
õû
ë
þüù	z BertOnnxModelTF.find_segment_idsc                 C   s`   |   ¡ }||vr
d S || }t|ƒdkrd S | j|d dd}t|ƒdkr)|d S td|ƒ d S )Nr%   r   Tr8   z&Found multiple candidates of input_ids)rH   r   rI   rJ   )r
   Úword_embeddingrH   r   rT   r   r   r   Úfind_input_ids‡   s   
zBertOnnxModelTF.find_input_idsc              	      sŽ  |   ¡ D ]À}|jdkrÄ|  |g d¢g d¢¡}|d u rq|\}}}}}}	|  |d¡rÄ|  |d¡rÄ| j|dd}
‡ fdd	„|
D ƒ}t|ƒdkrMtd
|ƒ  d S t|ƒdkrY|d   S |  |d g d¢g d¢¡}‡ fdd	„|
D ƒ}|r½|r½t|ƒdkr½|d |d jd kr½t 	d¡ |d }t
 |jd ¡}|  |¡j}|  t
jd|d gdgd|¡ |  t
jddg|	jd g|d|¡ |	jd   S qd S )NÚSoftmax)ÚAddr!   r"   r$   r&   r'   )r   r%   Nr%   r   r   iðØÿÿr%   Tr8   c                    s   g | ]}|ˆ vr|‘qS r   r   ©Ú.0r   ©Úexcluded_graph_inputsr   r   Ú
<listcomp>«   ó    z3BertOnnxModelTF.find_mask_input.<locals>.<listcomp>z'Found multiple candidates of mask inputr   r?   r=   )r   r   r   r   r   r   r   r   c                    s   g | ]}|ˆ v r|‘qS r   r   r`   rb   r   r   rd   À   re   r@   r<   Úinput_shape_for_maskrB   r:   rF   )r   r   r(   Úhas_constant_inputrI   r   rJ   r   r   rL   r   rP   rQ   rM   r1   rN   rO   )r
   rc   r   Ú	mask_pathrN   Úmul_nodeÚsub_nodeÚ	cast_nodeÚ
slice_nodeÚunsqueeze_noderT   rC   Úpath_to_be_simplifiedÚduplicated_inputsrX   rZ   rY   r   rb   r   Úfind_mask_input—   sz   
ýù

ôÿþ
ýú
üù	€zBertOnnxModelTF.find_mask_inputc              
   C   sv  |   |¡}|d u rt d¡ dS |  ||¡}|d u r!t d¡ dS |  ||g¡}|d u r3t d¡ dS |||g| _|  d¡}| j ||¡ |  	|¡j
jjtjkrX| j |¡\}	}|  	|¡rf| j |¡\}	}n| j |¡\}}
|  	|¡r|| j |¡\}	}n| j |¡\}}|  d¡}tjjd||||||jd |jd	 |g||gd
d}d|_|  |jd |¡ |  ||  |¡j¡ d S )Nz6Failed to find input_ids. Cannot fuse embedding layer.Fz8Failed to find segment_ids. Cannot fuse embedding layer.z7Failed to find input_mask. Cannot fuse embedding layer.Ú
mask_indexÚembed_outputÚEmbedLayerNormalizationr%   r,   Ú
EmbedLayer©rC   rD   r1   zcom.microsoftr   )r]   r   r   r[   rp   Úbert_inputsÚcreate_node_nameÚattention_maskÚset_mask_indiceÚfind_graph_inputÚtypeÚtensor_typeÚ	elem_typer   ÚINT32ÚutilsÚcast_graph_input_to_int32Úcast_input_to_int32Úonnxr   rO   r   Údomainr   r   rN   rM   r1   )r
   Únormalize_noder\   rR   Úposition_embeddingrS   rU   Ú
mask_inputrq   ÚcastedÚsegment_id_cast_nodeÚmask_input_cast_noderr   Ú
embed_noder   r   r   Úcreate_embedding_subgraphà   sR   







ø
óz)BertOnnxModelTF.create_embedding_subgraphc                 C   sº  t  d¡ |  ¡ }|  d¡}|D ]Ê}|  |g d¢g d¢|¡}|du r#q|\}}}|  |jd ¡}|du r5qt |¡}	t	|	j
ƒdkrRt  d|j› d	|	j
› ¡ |j}
nt  d
|j› d	|	j
› ¡  dS |  |d|¡}|durÚ|jdkrÚ|  |¡}t	|ƒdkrˆt  d|› ¡  dS d}d}| ¡ D ]%\}}|d dkr¨|}t  d|› d	|› ¡ q|}t  d|› d	|› ¡ q|du s¾|du rÆt  d¡  dS t  d¡ |  ||||
¡ |  ¡   dS qdS )zM
        Automatically detect word, segment and position embeddings.
        z#start processing embedding layer...ÚLayerNormalization)r_   r#   r&   )r   r%   r   Nr   r,   zFound position embedding. name:z, shape:z(Failed to find position embedding. name:r_   zFFailed to find two embeddings (word and segment) from Add node. Found zFound segment embedding. name:zFound words embedding. name:z.Failed to find both word and segment embeddingzCreate Embedding node)r   r   Úoutput_name_to_nodeÚget_nodes_by_op_typer(   r.   r   r   r/   r   r0   r1   Ú
get_parentr   r7   ÚwarningÚitemsr‹   Úprune_graph)r
   r   Úlayer_norm_nodesÚlayer_norm_nodeÚpos_embed_pathrN   Úreshape_noderl   r5   r6   r…   Úfirst_parentÚ
embeddingsr\   rR   r1   r0   r   r   r   Úprocess_embedding  sh   

ü


ÿ

ü€Éz!BertOnnxModelTF.process_embeddingc           	      C   sP   |||fD ] }|j d }|| }||krqt d|› d|jd › ¡  dS dS )Nr   zCheck attention input failed:z, FT)r   r   rL   r   )	r
   Úmatmul_qÚmatmul_kÚmatmul_vÚparentr   ÚxÚ
root_inputÚ	root_noder   r   r   Úcheck_attention_inputW  s   
z%BertOnnxModelTF.check_attention_inputc           #      C   s´  |   ¡ }g }d}g }|  d¡}|  d¡}| |¡ | |¡ |D ]¥}|  |¡j}|jdkr=|  |dd¡}	|	d ur<|	}nq |  |d¡}
|
d u sL|
jdvra|  |d¡}
|
d u s[|
jdvrat 	d¡ q |  
|g d¢g d	¢¡}|d u r”|  
|g d
¢g d¢¡}|d u r”|  
|g d¢g d¢¡}|d u r”t 	d¡ q |d }|  
|g d¢g d¢¡}|d u r¼|  
|ddgddg¡}|d u r¼t 	d¡ q |d }|d }|  
|g d¢g d¢¡}|d u rè|  
|g d¢g d¢¡}|d u rèt 	d¡ q |d }|  
|g d¢g d¢¡}|d u r|  
|ddgddg¡}|d u rt 	d¡ q |d }|d }|  
|g d¢g d¢¡}|d u r@|  
|g d¢g d¢¡}|d u r@t 	d¡ q |d }|d }|  |d ¡}|d u rZt 	d¡ q |  |d d¡sit 	d¡ q |  
|d dgdg¡p|  
|d dgdg¡}d }|d! }|d u r¾t|ƒd"kr¾|  |d jd ¡d u r¾|d jd }|  tjd|g|g|dgd#|¡ ||d jd< |  ||||
|¡}|rÁ| j |d jd ¡}t 	d$¡ | jj|||||||| j| j|
jd |d% jd d&}|d u rûq |d jdkr[tj|d jd' tjd(gt dd| jt | j| j ƒgg¡ !¡ d)d*} |  "| |¡ tjd+|jd |d jd' g|d jd, g|d jd- d.}!|d jd, |d jd< |  |!|¡ |
jd+kr”t# $|  %|
jd ¡¡d }"tj|
jd/ tjd0gt dd|"gg¡ !¡ d)d*} |  "| |¡ |
jd/ |
jd< |  ||¡ |d7 }| |d%d … ¡ | |¡ | |¡ | |¡ | |¡ | |¡ q t 	d1¡ q |  &|¡ |  '¡  t (d2|› ¡ d S )3Nr   ÚSkipLayerNormalizationrŒ   r_   r%   )r¢   rŒ   r#   z(Failed to match parent of normalize_node)r_   ÚMatMulr#   Ú	Transposer£   )r   r   r   r   r   )r£   r#   r¤   r£   )r%   r   r   r   )r_   ÚEinsumr¥   )r   r   r   zFailed to match qkv nodesr?   )r¤   r#   r_   r£   r¥   zFailed to match v pathéþÿÿÿ)r^   r_   r!   r£   )r   r   r   r   )r^   r_   r¥   zFailed to match qk_pathszFailed to match q path)r!   r_   r¥   )r%   r   r   zFailed to match k pathzCannot find mask_nodes.z;Sub node expected to have an input with constant value 1.0.r>   ÚExpandÚSqueeze_3d_to_2d_maskÚ_outputé   )ÚaxeszCreate an Attention node.r,   )rq   Úq_matmulÚk_matmulÚv_matmulÚq_addÚk_addÚv_addr   r   Úfirst_inputr   Ú	_newshapeé   T)r1   Ú	data_typeÚdimsÚvalsÚrawr#   Ú_reshape_outputÚ_reshaperu   Ú	_modifiedé   zRoot node not matched.zFused Attention count:))r   rŽ   ÚextendrM   r1   r   Úmatch_parentr   r   rL   r(   r+   rg   r   rz   r   rN   r   rO   r¡   rx   Úprocess_maskÚattention_fusionÚcreate_attention_noder   r   r   Úmake_tensorr   ÚINT64ÚnpÚint64ÚintÚtobytesÚadd_initializerr   r/   r.   r   Úupdate_graphr   )#r
   r   r   Úattention_countÚstart_nodesÚskip_layer_norm_nodesr“   r„   rY   Úadd_before_layernormr   Ú	qkv_nodesÚ
matmul_qkvÚv_nodesÚadd_vrœ   Úqk_nodesÚ	matmul_qkÚq_nodesÚadd_qrš   Úk_nodesÚadd_kr›   r*   Úsqueeze_nodeÚsqueeze_node_nameÚsqueeze_output_namer†   Úis_same_rootrq   Úattention_nodeÚtensorÚreshape_r   r   r   r   Úfuse_attentionb  sH  






ýý











ÿ2ûø

õ
üÿÿ	òþù	û





zBertOnnxModelTF.fuse_attentionc                 C   s   |   ¡  |  ¡  |  ¡  d S r   )r    r™   Úskip_reshape©r
   r   r   r   Ú
preprocess-  s   zBertOnnxModelTF.preprocessc                 C   sn   d}|   d¡}|D ]}|  |d¡}|d ur&|jdkr&|jd |jd< |d7 }q	|dkr5t d|› ¡ d S d S )Nr   r#   r%   zSkip consequent Reshape count: )rŽ   r   r   r   r   r   )r
   ÚcountÚreshape_nodesr–   r   r   r   r   rà   2  s   
€ÿzBertOnnxModelTF.skip_reshapec                 C   sp   |   d¡}|D ].}|  |ddgddg¡}|d u rqt d¡ |\}}|  |jd |jd ¡ |  |¡  d S d S )NÚ	Attentionr#   rs   r   z+Remove Reshape before first Attention node.)rŽ   r(   r   r   r   r   r   Úremove_node)r
   Úattention_nodesrÜ   ÚpathÚreshaperV   r   r   r   Ú%remove_reshape_before_first_attention>  s   


øz5BertOnnxModelTF.remove_reshape_before_first_attentionc                 C   s   |   ¡  |  ¡  d S r   )rê   r’   rá   r   r   r   ÚpostprocessJ  s   zBertOnnxModelTF.postprocess)Ú__name__Ú
__module__Ú__qualname__r	   r    r+   r7   r[   r]   rp   r‹   r™   r¡   rß   râ   rà   rê   rë   Ú__classcell__r   r   r   r   r      s"    
@I6A Lr   )ÚloggingÚnumpyrÄ   r‚   r   r   r   Úonnx_model_bertr   Ú	getLoggerrì   r   r   r   r   r   r   Ú<module>   s   
