Ë
    �…jÏ�  ã                   óÞ   — d dl Z d dlZd dlmZmZ d dlmZ d dlm	Z	m
Z
 d dlmZ d dlmZmZmZ d dlmZ d dlmZ  e j*                  e«      Z G d	„ d
e«      Z G d„ de«      Z G d„ de«      Zy)é    N)ÚAttentionMaskÚFusionAttention)ÚFusion)Ú"FusionSimplifiedLayerNormalizationÚ&FusionSkipSimplifiedLayerNormalization)ÚNumpyHelper)Ú	NodeProtoÚTensorProtoÚhelper)Ú	OnnxModel)ÚBertOnnxModelc                   óò   ‡ — e Zd ZdZdedededefˆ fd„Zdedz  d	e	d
e	de	dedededededz  de
de	dz  fd„Zdededededz  dedz  dedz  dedz  dededz  dedz  dedede	dz  fd„Zd„ Zd„ Zd„ Zˆ xZS )ÚFusionT5Attentionz=
    Fuse T5 Attention subgraph into one Attention node.
    ÚmodelÚhidden_sizeÚ	num_headsÚattention_maskc                 ó@   •— t         ‰| �  ||||ddg¬«       d| _        y )NFÚSoftmax)Úuse_multi_head_attentionÚsearch_op_typesé   )ÚsuperÚ__init__Ú	static_kv)Úselfr   r   r   r   Ú	__class__s        €úg/root/aria/tools/markitdown-venv/lib/python3.12/site-packages/onnxruntime/transformers/onnx_model_t5.pyr   zFusionT5Attention.__init__   s4   ø€ ô 	‰ÑØØØØØ%*Ø&˜Kð 	ô 	
ð ˆ�ó    Ú
mask_indexNÚq_matmulÚk_matmulÚv_matmulÚinputÚoutputÚ	attn_biasÚscaleÚreturnc                 ó2  — |dkD  sJ ‚|dkD  r$||z  dk7  rt         j                  d|› d|› �«       y| j                  j                  |j                  d   «      }| j                  j                  |j                  d   «      }| j                  j                  |j                  d   «      }|�|�|€&|€|n|€|n|}t        |j                  d   › d�«       yt        j                  |«      }t        j                  |«      }t        j                  |«      }|j                  |j                  k(  sJ ‚|j                  d   }|j                  d   }|j                  d   }||cxk(  r|k(  sJ ‚ J ‚|dkD  r!||k7  rt         j                  d|› d|› d	�«       t        j                  |j                  dd «      }t        j                  |||fd¬
«      }d|z  }| j                  j                  d«      }t        j                  |dz   t         j"                  ||g|j%                  «       d¬«      }| j                  j'                  || j(                  «       ||dz   dg}|r|j+                  |«       n|j+                  d«       |	r"|j+                  d«       |j+                  |	«       |r#|d   dk(  r|j-                  «        |r	|d   dk(  rŒt        j.                  d||g|¬«      }d|_        |j2                  j5                  t        j6                  d|«      g«       |
�0|j2                  j5                  t        j6                  d|
«      g«       | j8                  �C|j2                  j5                  t        j6                  dt;        | j8                  «      «      g«       |S )a¼  Create an Attention node.
        Args:
            mask_index (str): mask input
            q_matmul (NodeProto): MatMul node in fully connection for Q
            k_matmul (NodeProto): MatMul node in fully connection for K
            v_matmul (NodeProto): MatMul node in fully connection for V
            num_heads (int): number of attention heads. If a model is pruned, it is the number of heads after pruning.
            hidden_size (int): hidden dimension. If a model is pruned, it is the hidden dimension after pruning.
            input (str): input name
            output (str): output name
        Returns:
            Union[NodeProto, None]: the node created or None if failed.
        r   úinput hidden size ú# is not a multiple of num of heads Nr   zl is not an initializer. Please set do_constant_folding=True in torch.onnx.export to unblock attention fusionzInput hidden size (z3) is not same as weight matrix dimension of q,k,v (z:). Please provide a correct input hidden size or pass in 0)Úaxisé   Ú	AttentionÚ_qkv_weightT©ÚnameÚ	data_typeÚdimsÚvalsÚrawÚ éÿÿÿÿ©ÚinputsÚoutputsr1   úcom.microsoftr   r'   Úmask_filter_value)ÚloggerÚdebugr   Úget_initializerr$   Úprintr   Úto_arrayÚshapeÚwarningÚnpÚprodÚstackÚcreate_node_namer   Úmake_tensorr
   ÚFLOATÚtobytesÚadd_initializerÚthis_graph_nameÚappendÚpopÚ	make_nodeÚdomainÚ	attributeÚextendÚmake_attributer<   Úfloat)r   r    r!   r"   r#   r   r   r$   r%   r&   r'   Úq_weightÚk_weightÚv_weightÚmatmulÚqwÚkwÚvwÚ
qw_in_sizeÚ
kw_in_sizeÚ
vw_in_sizeÚqw_out_sizeÚ
qkv_weightÚqkv_weight_dimÚattention_node_nameÚweightÚattention_inputsÚattention_nodes                               r   Úmake_attention_nodez%FusionT5Attention.make_attention_node)   sˆ  € ð4 ˜1Š}Ðˆ}à˜Š? ¨iÑ 7¸AÒ=Ü�L‰LÐ-¨k¨]Ð:]Ð^gÐ]hÐiÔjØà—:‘:×-Ñ-¨h¯n©n¸QÑ.?Ó@ˆØ—:‘:×-Ñ-¨h¯n©n¸QÑ.?Ó@ˆØ—:‘:×-Ñ-¨h¯n©n¸QÑ.?Ó@ˆàÐ˜xÐ/°8Ð3CØ!)Ð!1‘XÀ8ÐCS±xÐYaˆFÜØ—<‘< ‘?Ð#ð $gð gôð ä×!Ñ! (Ó+ˆÜ×!Ñ! (Ó+ˆÜ×!Ñ! (Ó+ˆð �x‰x˜2Ÿ8™8Ò#Ð#Ð#à—X‘X˜a‘[ˆ
Ø—X‘X˜a‘[ˆ
Ø—X‘X˜a‘[ˆ
à˜ZÔ5¨:Ò5Ð5Ñ5Ð5Ð5à˜Š?˜{¨jÒ8Ü�N‰NØ% k ]Ð2eÐfpÐeqð rJð Jôô
 —g‘g˜bŸh™h q r˜lÓ+ˆÜ—X‘X˜r 2 r˜l°Ô3ˆ
Ø˜[™ˆà"Ÿj™j×9Ñ9¸+ÓFÐä×#Ñ#Ø$ }Ñ4Ü!×'Ñ'Ø˜nÐ-Ø×#Ñ#Ó%Øô
ˆð 	�
‰
×"Ñ" 6¨4×+?Ñ+?Ô@ð Ø -Ñ/Øð
Ðñ
 Ø×#Ñ# JÕ/à×#Ñ# BÔ'áØ×#Ñ# BÔ'Ø×#Ñ# IÔ.áÐ#3°BÑ#7¸2Ò#=Ø× Ñ Ô"ñ Ð#3°BÑ#7¸2Ó#=ô  ×)Ñ)ØØ#Ø�HØ$ô	
ˆð !0ˆÔØ× Ñ ×'Ñ'¬×)>Ñ)>¸{ÈIÓ)VÐ(WÔXàÐØ×$Ñ$×+Ñ+¬V×-BÑ-BÀ7ÈEÓ-RÐ,SÔTà×!Ñ!Ð-Ø×$Ñ$×+Ñ+¬V×-BÑ-BÐCVÔX]Ð^b×^tÑ^tÓXuÓ-vÐ,wÔxàÐr   ÚqueryÚkeyÚvalueÚpast_keyÚ
past_valueÚpresent_keyÚpresent_valuec                 ó   — |dkD  r|dkD  r|r|r|sJ ‚||z  dk7  rt         j                  d|› d|› �«       y | j                  j                  d«      }|||dg}|r|j	                  |«       n|j	                  d«       |r|j	                  |«       n|j	                  d«       |r&|sJ ‚|j	                  |«       |j	                  |«       |r#|d   dk(  r|j                  «        |r	|d   dk(  rŒ|g}|	r&|
sJ ‚|j	                  |	«       |j	                  |
«       t        d|›d|›d	|›�«       t        j                  d|||¬
«      }d|_	        |j                  j                  t        j                  d|«      g«       |j                  j                  t        j                  dd«      g«       | j                  �C|j                  j                  t        j                  dt        | j                  «      «      g«       | j                  d«       |S )Nr   r*   r+   ÚMultiHeadAttentionr6   r7   zattention_inputs=z, attention_outputs=z, attention_node_name=r8   r;   r   r'   ç      ð?r<   )r=   r>   r   rG   rM   rN   r@   r   rO   rP   rQ   rR   rS   r<   rT   Úincrease_counter)r   rg   rh   ri   r    r&   rj   rk   r%   rl   rm   r   r   rb   rd   Úattention_outputsre   s                    r   Úcreate_mha_nodez!FusionT5Attention.create_mha_nodeš   s  € ð ˜1Š} ¨q¢±U¹sÁuÐLÐLà˜)Ñ#¨Ò)Ü�L‰LÐ-¨k¨]Ð:]Ð^gÐ]hÐiÔjØà"Ÿj™j×9Ñ9Ð:NÓOÐàØØØð	
Ðñ Ø×#Ñ# JÕ/à×#Ñ# BÔ'áØ×#Ñ# IÕ.à×#Ñ# BÔ'áÙÐ�:Ø×#Ñ# HÔ-Ø×#Ñ# JÔ/áÐ#3°BÑ#7¸2Ò#=Ø× Ñ Ô"ñ Ð#3°BÑ#7¸2Ó#=ð $˜HÐÙÙ Ð �=Ø×$Ñ$ [Ô1Ø×$Ñ$ ]Ô3äÐ"Ð!Ð#Ð#8Ð&7Ð%9Ð9PÐ<OÐ;QÐRÔSÜ×)Ñ)Ø Ø#Ø%Ø$ô	
ˆð !0ˆÔØ× Ñ ×'Ñ'¬×)>Ñ)>¸{ÈIÓ)VÐ(WÔXØ× Ñ ×'Ñ'¬×)>Ñ)>¸wÈÓ)LÐ(MÔNØ×!Ñ!Ð-Ø×$Ñ$×+Ñ+¬V×-BÑ-BÐCVÔX]Ð^b×^tÑ^tÓXuÓ-vÐ,wÔxà×ÑÐ2Ô3ØÐr   c                 óR   — | j                  |||«      ry | j                  |||«       y ©N)Úfuse_t5_encoderÚfuse_t5_decoder)r   ÚnodeÚinput_name_to_nodesÚoutput_name_to_nodes       r   ÚfusezFusionT5Attention.fuseà   s-   € Ø×Ñ Ð&9Ð;NÔOØà×Ñ˜TÐ#6Ð8KÕLr   c                 óÂ  ‡ — |j                   dk(  sJ ‚| j                  j                  |g d¢g d¢|¬«      }|€y|\  }}}| j                  j                  |g d¢g d¢|«      }|€y|d   }	| j                  j                  |g d	¢g d
¢|«      }
|
€y|
\  }}}| j                  j                  |g d¢g d¢|«      }|€y|\  }}}| j                  j                  |g d¢g d¢|«      }|d u }|�|d   }nN| j                  j                  |g d¢g d¢|«      }|€%| j                  j                  |g d¢g d¢|«      }|€y|d   }| j                  j	                  |«      \  }}|€y|dk7  rt        |«      | _        | j                  j                  |d   g d¢g d¢|«      Š | j                  j                  |d   g d¢g d¢|«      }‰ �{t        ˆ fd„| j                  j                  «       j                  D «       «      rE|�C‰ d   j                  d   |d   j                  d   k(  rt        ‰ d   j                  «      dk(  rd}n+| j                  j                  |d   j                  d   «      }d }| j                  j                  |ddgddg«      }|€#|r!| j                  j                  |g d¢g d
¢«      }|€y|d   j                  d   }| j                  j                  |g d	¢g d
¢«      }|€y|\  }}}| j                  j                  |g d	¢g d¢«      }|€y|\  }}}|j                  d   |	j                  d   k7  ry| j                  |«      \  }}| j                  |||||||	j                  d   |j                  d   |d¬ «
      }|€y| j                   j#                  |«       | j$                  | j&                  |j(                  <   | j*                  j#                  |«       d!| _        y!)"Nr   ©ÚMatMulÚ	TransposeÚReshape©©r   r   r‚   r‚   ©Úedgesry   F©ÚConcatÚ	UnsqueezeÚGatherÚShape©r   r   r   r   r7   ©r   r€   r~   ©r   r   r   ©r   ÚAddr~   ©r   r   r   ©rŽ   ÚMulÚSubÚCastr‡   r‡   ©r   r   r   r   r   r   r   )rŽ   ÚSlicer‘   r’   r‡   r‡   )r   r   r   r   r   r   ©rŽ   r•   r‘   r’   r“   r‡   r‡   ©r   r   r   r   r   r   r   é   éðØÿÿ)ÚConstantOfShaper†   r‡   rˆ   r‰   ©r   r   r   r   r   )r   r   r   r   r   c              3   ó\   •K  — | ]#  }|j                   ‰d    j                  d   k(  –— Œ% y­w)r7   r   N)r1   r$   )Ú.0r$   Úmask_nodes_2s     €r   ú	<genexpr>z4FusionT5Attention.fuse_t5_encoder.<locals>.<genexpr>F  s*   øè ø€ ÒbÀ�E—J‘J ,¨rÑ"2×"8Ñ"8¸Ñ";Õ;Ñbùs   ƒ),r   r6   rŽ   ÚRelativePositionBias)rŽ   r•   r    rp   )r   r   r$   r%   r&   r'   T)Úop_typer   Úmatch_child_pathÚmatch_parent_pathÚget_constant_inputrT   r<   ÚanyÚgraphr$   Úlenr   Úprocess_maskr%   Úget_num_heads_and_hidden_sizerf   Únodes_to_addrM   rL   Únode_name_to_graph_namer1   Únodes_to_removeÚprune_graph)!r   Úsoftmax_nodery   rz   Ú	qkv_nodesÚ
matmul_qkvÚ_Úreshape_qkvÚqkv_shape_nodesÚinput_shape_nodeÚv_nodesÚ	reshape_vÚmatmul_vÚqk_nodesÚadd_qkÚ	matmul_qkÚ
mask_nodesÚis_pattern_for_one_graph_inputÚmul_nodeÚmul_valÚmask_nodes_3r    Úres_pos_biasÚ	rpb_nodesÚk_nodesÚmatmul_kÚq_nodesÚ	reshape_qÚmatmul_qÚq_num_headsÚq_hidden_sizeÚnew_noderž   s!                                   @r   rv   z!FusionT5Attention.fuse_t5_encoderæ   s\  ø€ Ø×#Ñ# yÒ0Ð0Ð0Ø—J‘J×/Ñ/ØÚ.Ú*Ø 3ð	 0ó 
ˆ	ð ÐØØ%.Ñ"ˆ
�A�{àŸ*™*×6Ñ6ØÚ6ÚØó	
ˆð Ð"ØØ*¨2Ñ.Ðà—*‘*×.Ñ.ØÚ.ÚØó	
ˆð ˆ?ØØ!(Ñˆˆ9�hð —:‘:×/Ñ/ØÚ(ÚØó	
ˆð ÐØØ'Ñˆˆ6�9à—Z‘Z×1Ñ1ØÚCÚØó	
ˆ
ð *4°tÐ);Ð&ØÐ!Ø! !‘}‰Hð Ÿ™×5Ñ5ØÚHÚ"Ø#ó	ˆJð Ð!Ø!ŸZ™Z×9Ñ9ØÚTÚ)Ø'ó	�
ð Ð%Ø Ø! !‘}ˆHà—Z‘Z×2Ñ2°8Ó<‰
ˆˆ7Øˆ?Øà�fÒÜ%*¨7£^ˆDÔ"ð —z‘z×3Ñ3Ø�r‰NÚIÚØó	
ˆð —z‘z×3Ñ3Ø�r‰NÚIÚØó	
ˆð Ð$ÜÓbÈÏÉ×IYÑIYÓI[×IaÑIaÔbÔbØÐ(Ø˜RÑ ×&Ñ& qÑ)¨\¸"Ñ-=×-CÑ-CÀAÑ-FÒFÜ�L ‘O×)Ñ)Ó*¨aÒ/à‰Jà×,Ñ,×9Ñ9¸*ÀR¹.×:NÑ:NÈqÑ:QÓRˆJàˆØ—J‘J×0Ñ0ØØÐ*Ð+Ø�ˆFó
ˆ	ð
 ÐÑ!?àŸ
™
×4Ñ4ØÚ8ÚóˆIð
 ÐØà  ‘}×+Ñ+¨AÑ.ˆà—*‘*×.Ñ.ØÚ.Úó
ˆð
 ˆ?ØØ ‰ˆˆ1ˆhð —*‘*×.Ñ.ØÚ.Úó
ˆð
 ˆ?Øà!(Ñˆˆ9�hð �>‰>˜!ÑÐ 0× 6Ñ 6°qÑ 9Ò9Øà%)×%GÑ%GÈ	Ó%RÑ"ˆ�]à×+Ñ+ØØØØØ!Ø%Ø"×(Ñ(¨Ñ+Ø×%Ñ% aÑ(Ø"Øð ,ó 
ˆð ÐØà×Ñ× Ñ  Ô*Ø6:×6JÑ6Jˆ×$Ñ$ X§]¡]Ñ3à×Ñ×#Ñ# KÔ0ØˆÔØr   c                 ó€  — |j                   dk(  sJ ‚| j                  j                  |g d¢g d¢|¬«      }|€y |\  }}}| j                  j                  |g d¢g d¢«      }|€y |d   }	d }
d }d }| j                  j                  |g d¢g d	¢«      }|€œ| j                  j                  |g d
¢g d¢«      }|�X|\  }}}|j                  d   }
|j
                  d   }d|vry |j                  d   |	j                  d   k7  rd| _        nsd| _        nk|j                  d   }||v ry d|vry d| _        nJ|\  }}}}|j                  d   }||v ry d|vry |j
                  d   }d|vry |j                  d   }
d| _        | j                  j                  |g d¢g d¢«      }|€y |\  }}}d }d }| j                  dk(  r¨| j                  j                  |g d¢g d¢«      }|�|d   }n)| j                  j                  |g d¢g d¢«      }|€y |d   }| j                  j                  |«      \  }}|dk7  r|| _        | j                  j                  |d   j                  d   «      }nY| j                  j                  |ddgddgfddgddgfg|«      \  }}}|dk  rt        j                  d«       y |j                  d   }d }d }d }| j                  dk(  rÓ| j                  j                  |g d
¢g d¢«      }|�o|\  } }!}|!j                  d   }||!j
                  d      }"|"D ]:  }#| j                  j                  |#j
                  d   «      }$|$€Œ.|$j                  } n |€y d|v�rÐy | j                  j                  |dgdg«      }|€y |d   } | j                  d   }||v ry d |v�r�y | j                  j                  |g d!¢g d"¢fg d#¢g d$¢fg|«      \  }%}}d }&d }"|�Ç|d   |d%   }!}'|!j                  d   }|%dk(  r"||'j                  d      }&|&j                  d   }n|'j                  d   }||v ry d&|vry |%dk(  rR||'j
                  d      }"|"D ]:  }#| j                  j                  |#j
                  d   «      }$|$€Œ.|$j                  } n n|'j
                  d   }|€y d'|vr“y | j                  j                  |g d
¢g d¢«      }|€y |\  }}!}|!j                  d   }||!j
                  d      }"|"D ]:  }#| j                  j                  |#j
                  d   «      }$|$€Œ.|$j                  } n |€y d'|vry | j                  j                  |g d
¢g d¢«      }(|(€y |(\  })}*}+|+j                  d   |	j                  d   k7  ry | j!                  |*«      \  },}-| j                  dk(  r
|�|}|}
d }d }|r|
r
|,dkD  r|-dkD  sy | j#                  |+j
                  d   ||
|||||j
                  d   |||,|-¬(«      }.|.rÖ| j$                  j'                  |.«       | j(                  | j*                  |.j                  <   |s|rq||fD ]j  }/|/r| j                  j                  |/«      st-        d)|/›d*�«        y |/|v sJ ‚|/d+z   ||/   j
                  d<   | j                  j/                  |/|/d+z   «       Œl | j0                  j'                  |«       d,| _        y y )-Nr   r}   r�   rƒ   r…   rŠ   r7   )r†   r   r€   r~   )r   r   r   r   r‹   rŒ   r   rm   r   Úpast_value_crossÚpast_value_selfÚpresent_value_selfr�   r�   r�   r”   r–   r—   r˜   r™   rŽ   r•   r    zGSkip MultiHeadAttention fusion since attention bias pattern not matchedÚpresent_key_crossr   Úpast_key_cross)r   r†   r€   r~   )r   r   r   r   )r   r†   r   r€   r~   )r   r   r   r   r   éþÿÿÿÚpast_key_selfÚpresent_key_self)rg   rh   ri   r    r&   rj   rk   r%   rl   rm   r   r   zgraph_output=z does not exist in graph outputÚ_copyF)r¡   r   r¢   r£   r$   r%   r   r¤   r<   r   r¨   Úmatch_parent_pathsr=   r>   Úfind_graph_outputr1   r©   rs   rª   rM   rL   r«   r@   Úreplace_input_of_all_nodesr¬   r­   )0r   r®   ry   rz   r¯   r°   Ú_transpose_qkvr²   r³   r´   ri   rk   rm   rµ   Útranspose_vr¶   r·   Úconcat_vr±   r¸   r¹   rº   r    rÀ   r»   r½   r¾   Úmatched_path_indexrh   rj   rl   rÂ   Útranspose_kÚ	reshape_kÚpresent_key_transpose_nodesÚpresent_key_transpose_nodeÚpresent_key_candidateÚidxÚpast_key_transpose_nodeÚconcat_krÄ   Útranspose_qrÅ   rÆ   rÇ   rÈ   rÉ   Úgraph_outputs0                                                   r   rw   z!FusionT5Attention.fuse_t5_decoder‘  s"  € Ø×#Ñ# yÒ0Ð0Ð0à—J‘J×/Ñ/ØÚ.Ú*Ø 3ð	 0ó 
ˆ	ð ÐØØ2;Ñ/ˆ
�N KàŸ*™*×6Ñ6ØÚ6Úó
ˆð
 Ð"ØØ*¨2Ñ.ÐàˆØˆ
ØˆØ—*‘*×.Ñ.ØÚ8Úó
ˆð
 ˆ?Ø—j‘j×2Ñ2ØÚ2ÚóˆGð
 Ð"Ø3:Ñ0�˜Y¨Ø!Ÿ™¨Ñ*�Ø +× 2Ñ 2°1Ñ 5�Ø"¨-Ñ7ØØ—>‘> !Ñ$Ð(8×(>Ñ(>¸qÑ(AÒAØ%&�D•Nà%&�D•Nà'×-Ñ-¨aÑ0�
ØÐ!4Ñ4ØØ%¨ZÑ7ØØ!"�•à(/Ñ%ˆH�a˜ AØ!Ÿ™¨Ñ*ˆJØÐ0Ñ0ØØ ¨
Ñ2ØØ$ŸO™O¨AÑ.ˆMØ#¨=Ñ8ØØ—O‘O AÑ&ˆEØˆDŒNà—:‘:×/Ñ/ØÚ(Úó
ˆð
 ÐØØ'Ñˆˆ6�9àˆ
ØˆØ�>‰>˜QÒØŸ™×5Ñ5ØÚGÚ"óˆJð
 Ð%Ø% a™=‘à!ŸZ™Z×9Ñ9ØÚTÚ)ó�
ð
 Ð%ØØ% a™=�àŸ™×6Ñ6°xÓ@‰JˆAˆwØ˜&Ò Ø)0�Ô&à×,Ñ,×9Ñ9¸*ÀR¹.×:NÑ:NÈqÑ:QÓR‰Jà'+§z¡z×'DÑ'DØà˜WÐ%¨¨1 vÐ.ØÐ3Ð4°q¸!°fÐ=ðð $ó(Ñ$Ð  1ð " AÒ%Ü—‘ÐfÔgØà!Ÿ<™<¨™?ˆLàˆØˆØˆØ�>‰>˜QÒØ—j‘j×2Ñ2ØÚ2ÚóˆGð
 Ð"Ø,3Ñ)�˜Y¨Ø—o‘o aÑ(�Ø.AÀ)×BRÑBRÐSTÑBUÑ.VÐ+Ø2Mò Ð.Ø,0¯J©J×,HÑ,HÐIc×IjÑIjÐklÑImÓ,nÐ)Ø,Ñ8Ø&;×&@Ñ&@˜Ùð	ð
 Ð&ØØ&¨kÒ9ØàŸ*™*×6Ñ6ØØ �MØ�Có�ð
 �?ØØ% a™j�à&×,Ñ,¨QÑ/�ØÐ2Ñ2ØØ#¨8Ò3Øà"Ÿj™j×;Ñ;ØâAÂ<ÐPÚNÒP_Ð`ðð $ó‰OˆC�˜!ð '+Ð#Ø*.Ð'ØÐ"Ø&-¨a¡j°'¸"±+˜)�Ø—o‘o aÑ(�à˜!’8Ø.AÀ(Ç.Á.ÐQRÑBSÑ.TÐ+Ø6×<Ñ<¸QÑ?‘Hà'Ÿ~™~¨aÑ0�HØÐ2Ñ2ØØ"¨(Ñ2Øà˜!’8Ø2EÀhÇoÁoÐVWÑFXÑ2YÐ/Ø6Qò "Ð2Ø04·
±
×0LÑ0LÐMg×MnÑMnÐopÑMqÓ0rÐ-Ø0Ñ<Ø*?×*DÑ*D˜KÙ!ñ	"ð #+§/¡/°!Ñ"4�KØÐ&ØØ%¨[Ñ8ØàŸ*™*×6Ñ6ØÚ6Úó�ð
 �?ØØ")‘��9˜aØ—o‘o aÑ(�Ø.AÀ)×BRÑBRÐSTÑBUÑ.VÐ+Ø2Mò Ð.Ø,0¯J©J×,HÑ,HÐIc×IjÑIjÐklÑImÓ,nÐ)Ø,Ñ8Ø&;×&@Ñ&@˜Ùð	ð
 Ð&ØØ%¨[Ñ8Øà—*‘*×.Ñ.ØÚ.Úó
ˆð
 ˆ?Øà+2Ñ(ˆ�Y à�>‰>˜!ÑÐ 0× 6Ñ 6°qÑ 9Ò9Øà%)×%GÑ%GÈ	Ó%RÑ"ˆ�]à�>‰>˜QÒ 8Ð#7ØˆCØˆEØˆHØˆJá™ +°¢/°mÀaÒ6GØà×'Ñ'Ø—/‘/ !Ñ$ØØØ!Ø"ØØ!Ø×%Ñ% aÑ(Ø#Ø'Ø!Ø%ð (ó 
ˆñ Ø×Ñ×$Ñ$ XÔ.Ø:>×:NÑ:NˆD×(Ñ(¨¯©Ñ7ñ ™mØ%0°-Ð$@ò `�LÙ(¨T¯Z©Z×-IÑ-IÈ,Ô-WÜ   Ð/NÐOÔPÙØ'Ð+>Ñ>Ð>Ð>ØBNÐQXÑBXÐ'¨Ñ5×<Ñ<¸QÑ?Ø—J‘J×9Ñ9¸,ÈÐW^ÑH^Õ_ð`ð × Ñ ×'Ñ'¨Ô4Ø$ˆDÕð r   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   Úintr   r   Ústrr	   rT   rf   rs   r{   rv   rw   Ú__classcell__©r   s   @r   r   r      s‚  ø„ ñðàðð ðð ð	ð
 &õð"oà˜$‘Jðoð ðoð ð	oð
 ðoð ðoð ðoð ðoð ðoð ˜‘:ðoð ðoð 
�TÑ	óoðbDàðDð ðDð ð	Dð
 ˜$‘JðDð ˜‘:ðDð ˜‘*ðDð ˜$‘JðDð ðDð ˜4‘ZðDð ˜T‘zðDð ðDð ðDð 
�TÑ	óDòLMòiöVI%r   r   c                   ó*   ‡ — e Zd Zdefˆ fd„Zd„ Zˆ xZS )ÚFusionRelativePositionBiasBlockr   c                 ó*   •— t         ‰| �  |ddg«       y )Nr    r   )r   r   )r   r   r   s     €r   r   z(FusionRelativePositionBiasBlock.__init__ž  s   ø€ Ü‰Ñ˜Ð 6¸¸ÕDr   c                 óî  — | j                   j                  |g d¢g d¢|«      }|€%| j                   j                  |g d¢g d¢|«      }|€y |d   }|d   }|d   }|d   }|| j                  v ry | j                   j                  |g d	¢g d
¢|«      }	|	€y | j                   j                  |	d   j                  d   «      }
|	d   }| j                   j                  |g d¢g d¢|«      }d}|€'| j                   j                  |g d¢g d¢|«      }d}|€y |d   }t        t        j                  t        j                  |
«      d|rdndz  z  «      «      }|dk7  rt        j                  d|› d�«       | j                   j                  dd|rdndz   ¬«      }| j                   j                  |j                  d   «      }|€y t        j                  |«      }t        j                  |«      }t!        j"                  |dz   t$        j&                  t        j(                  |«      d   t        j(                  |«      d   g|j+                  «       d¬«      }| j                   j-                  || j.                  «       |j0                  |j                  d   |j                  d   g}|d z   }||j                  d<   t!        j2                  d||g|¬!«      }d"|_        |j6                  j9                  t!        j:                  d#|«      g«       |j6                  j9                  t!        j:                  d$|«      g«       | j.                  | j<                  |j0                  <   | j>                  jA                  |«       d| _!        y )%N)rŽ   rŽ   r•   r‡   r   rˆ   ÚWhere)r   r   r   r   r   r   r   )rŽ   rŽ   r•   r‡   r   rˆ   rŽ   rñ   )r   r   r   r   r   r   r   r   é   r7   r˜   r-   )	ÚMinrš   r‰   rŽ   r“   r‘   ÚDivÚLogrô   )	r˜   r   r   r   r   r   r   r   r   éýÿÿÿr   )r“   ÚNegró   rš   r‰   r’   r‡   ÚRange)r   r   r   r   r   r   r   r   F)r“   ÚAbsr’   r‡   rø   r›   Té    é   é€   zmax_distance is z], which is different from the default value 128. Please double check the model configuration.r    ÚRelPosBias_ÚencoderÚdecoder©Úname_prefixr   Ú_bias_table_weightr0   Ú_rel_pos_biasr8   r;   Úmax_distanceÚis_bidirectional)"r   r£   r¬   Úget_constant_valuer$   ré   rD   ÚroundÚexpr=   rC   rG   r?   r   rA   Ú	transposer   rH   r
   rI   rB   rJ   rK   rL   r1   rO   rP   rQ   rR   rS   r«   rª   rM   r­   )r   rx   ry   rz   Úcompute_bias_nodesÚgatherÚwhereÚsliceÚ	unsqueezeÚcompute_buckets_nodesÚlog_maxÚdivÚrange_nodesr  Ú
range_noder  Ú	node_nameÚtable_weight_iÚtable_weightÚtable_weight_tÚ
bias_tabler9   Úbias_outputÚrpb_nodes                           r   r{   z$FusionRelativePositionBiasBlock.fuse¡  ss  € Ø!ŸZ™Z×9Ñ9ØÚPÚ!Øó	
Ðð Ð%Ø!%§¡×!=Ñ!=ØÚ[Ú(Ø#ó	"Ðð "Ð)Øà# AÑ&ˆØ" 2Ñ&ˆØ" 1Ñ%ˆØ& qÑ)ˆ	ð ˜×,Ñ,Ñ,Øà $§
¡
× <Ñ <ØÚZÚ'Øó	!
Ðð !Ð(Øð —*‘*×/Ñ/Ð0EÀbÑ0I×0OÑ0OÐPQÑ0RÓSˆà# BÑ'ˆà—j‘j×2Ñ2ØÚ[Ú$Øó	
ˆð !ÐØÐØŸ*™*×6Ñ6ØÒAÂ?ÐTgóˆKð  $ÐØÐ"ØØ  ‘_ˆ
ô œ2Ÿ8™8¤B§F¡F¨7£O°rÑCS¹aÐYZÑ7[Ñ$\Ó]Ó^ˆØ˜3ÒÜ�N‰NØ" < .ð 1?ð ?ôð
 —J‘J×/Ñ/Ø"°ÑN^ÁÐdmÑ0nð 0ó 
ˆ	ð Ÿ™×3Ñ3°F·L±LÀ±OÓDˆØÐ!ØÜ"×+Ñ+¨NÓ;ˆÜŸ™ lÓ3ˆÜ×'Ñ'ØÐ1Ñ1Ü!×'Ñ'Ü—(‘(˜<Ó(¨Ñ+¬R¯X©X°lÓ-CÀAÑ-FÐGØ×'Ñ'Ó)Øô
ˆ
ð 	�
‰
×"Ñ" :¨t×/CÑ/CÔDðP —/‘/ :×#3Ñ#3°AÑ#6¸
×8HÑ8HÈÑ8KÐLˆð   /Ñ1ˆØ$ˆ�‰�A‰ä×#Ñ#Ø"ØØ �MØô	
ˆð *ˆŒØ×Ñ×!Ñ!¤6×#8Ñ#8¸ÈÓ#VÐ"WÔXØ×Ñ×!Ñ!¤6×#8Ñ#8Ð9KÐM]Ó#^Ð"_Ô`Ø6:×6JÑ6Jˆ×$Ñ$ X§]¡]Ñ3Ø×Ñ× Ñ  Ô*ØˆÕr   )rå   ræ   rç   r   r   r{   rë   rì   s   @r   rî   rî   �  s   ø„ ðE˜iõ EöR r   rî   c                   ó\   ‡ — e Zd Zddedefˆ fd„Zd„ Zd„ Zdd„Zd„ Zd„ Z	d	„ Z
d
„ Zd„ Zˆ xZS )ÚT5OnnxModelr   r   c                 ó¤  •— t         ‰| �  |||«       t        | «      | _        t	        | j
                  j                  j                  «      dk(  r!ddlm	} |j                  | j                  _        t        | | j                  | j                  | j                  «      | _        t!        | «      | _        t%        | «      | _        t)        | «      | _        y )Nr   r   )ÚAttentionMaskFormat)r   r   r   r   r§   r   r¦   r$   Úfusion_optionsr  ÚNoMaskÚmask_formatr   r   r   Úattention_fusionr   Úlayer_norm_fusionr   Úskip_layer_norm_fusionrî   Ú
rpb_fusion)r   r   r   r   r  r   s        €r   r   zT5OnnxModel.__init__7  s¡   ø€ Ü‰Ñ˜ 	¨;Ô7Ü+¨DÓ1ˆÔô ˆt�z‰z×Ñ×%Ñ%Ó&¨!Ò+Ý:à.A×.HÑ.HˆD×ÑÔ+ä 1°$¸×8HÑ8HÈ$Ï.É.ÐZ^×ZmÑZmÓ nˆÔÜ!CÀDÓ!IˆÔÜ&LÈTÓ&RˆÔ#Ü9¸$Ó?ˆ�r   c                 ó8   — | j                   j                  «        y ru   )r"  Úapply©r   s    r   Úfuse_attentionzT5OnnxModel.fuse_attentionF  s   € Ø×Ñ×#Ñ#Õ%r   c                 ó8   — | j                   j                  «        y ru   )r#  r'  r(  s    r   Úfuse_layer_normzT5OnnxModel.fuse_layer_normI  s   € Ø×Ñ×$Ñ$Õ&r   c                 ó8   — | j                   j                  «        y ru   )r$  r'  )r   Úshape_infers     r   Úfuse_skip_layer_normz T5OnnxModel.fuse_skip_layer_normL  s   € Ø×#Ñ#×)Ñ)Õ+r   c           
      ó  — | j                  «       D �]m  }|j                  dk(  sŒ| j                  |g d¢g d¢«      }|��@| j                  j                  j
                  D �cg c]  }|j                  ‘Œ }}|d   j
                  d   |v rõ| j                  dd¬«      }t        j                  d|d   j
                  d   g|d	z   g|¬
«      }t        j                  dt        j                  dgdg¬«      }| j                  |«       t        j                  d|d	z   dg|dz   g| j                  dd¬«      d¬«      }| j                  |«       | j                  |«       |dz   |j
                  d<   |dz   |j
                  d<    y  y c c}w )Nr    )
rˆ   r‰   r   r€   r†   r‡   rˆ   r‰   ÚSimplifiedLayerNormalizationrˆ   )
r   r   r   r   r   r   r   r   r   r   r7   r   r‰   ÚAdded_Shape_r   Ú_Outputr8   ÚConstant_Index_1)r1   r2   r3   r4   rˆ   Ú_Output_Gather_1ÚAdded_Gather_r   )r9   r:   r1   r,   r˜   )Únodesr¡   r£   r   r¦   r$   r1   rG   r   rO   rH   r
   ÚINT64rK   Úadd_node)	r   rx   r6  r$   Úgraph_input_namesr  Ú
shape_nodeÚ	indices_1r  s	            r   Úadjust_rel_pos_bis_length_inputz+T5OnnxModel.adjust_rel_pos_bis_length_inputO  s—  € ð —J‘J“Ló 3	ˆDØ�|‰|Ð5Ó5Ø×.Ñ.Øòò 3ó�ð" Ñ$ØAEÇÁ×AQÑAQ×AWÑAWÖ(X¸¨¯«Ð(XÐ%Ð(XØ˜R‘y—‘ qÑ)Ð->Ñ>Ø$(×$9Ñ$9¸'È~Ð$9Ó$^˜	Ü%+×%5Ñ%5Ø#Ø$)¨"¡I§O¡O°AÑ$6Ð#7Ø%.°Ñ%:Ð$;Ø!*ô	&˜
ô %+×$6Ñ$6Ø!3Ü&1×&7Ñ&7Ø"# Ø"# ô	%˜	ð ×,Ñ,¨YÔ7ä!'×!1Ñ!1Ø$Ø$-°	Ñ$9Ð;MÐ#NØ%.Ð1CÑ%CÐ$DØ!%×!6Ñ!6°xÈ_Ð!6Ó!]Ø!"ô"˜ð Ÿ™ jÔ1ØŸ™ fÔ-Ø(1Ð4FÑ(F˜Ÿ
™
 1™Ø(1Ð4FÑ(F˜Ÿ
™
 1™áñg3	ùò( )Ys   Á!Fc                 ó^  — g }| j                  «       D ]˜  }|j                  dk(  sŒ| j                  |g d¢g d¢«      }|€Œ-| j                  |dgdg«      }|€ŒE|d   }|j                  d   |j                  d<   |j	                  |«       |j                  |«       | j                  |«       Œš y )NrŽ   )r‘   r’   r‘   r‡   r“   ÚLessOrEqualÚTiler†   r‡   rˆ   r‰   )r   r   r   r   r   r   r   r   r   r   r   r    r   ©r6  r¡   r£   r%   rR   rM   Úremove_nodes©r   r¬   rx   Úextended_mask_nodesrÁ   r  s         r   Ú!remove_extended_mask_decoder_initz-T5OnnxModel.remove_extended_mask_decoder_initˆ  sÀ   € ØˆØ—J‘J“Lò 	3ˆDØ�|‰|˜uÓ$Ø&*×&<Ñ&<Øòò 6ó'Ð#ð" 'Ð.Øà ×2Ñ2°4Ð:PÐ9QÐTUÐSVÓW�	ØÐ$Øà$ Q™<�Ø%)§[¡[°¡^�—‘ Ñ"à×&Ñ&Ð':Ô;Ø×&Ñ& tÔ,Ø×!Ñ! /Õ2ñ?	3r   c                 ób  — g }| j                  «       D ]š  }|j                  dk(  sŒ| j                  |g d¢g d¢«      }|€Œ-| j                  |ddgddg«      }|€ŒG|d   }|j                  d   |j                  d<   |j	                  |«       |j                  |«       | j                  |«       Œœ y )NrŽ   )r‘   r’   r‘   r‡   r†   r“   r>  r?  r†   r‡   rˆ   r‰   )r   r   r   r   r   r   r   r   r   r   r   r   r•   r    r   r@  rB  s         r   Úremove_extended_mask_decoderz(T5OnnxModel.remove_extended_mask_decoder«  sÆ   € ØˆØ—J‘J“Lò  	3ˆDØ�|‰|˜uÓ$Ø&*×&<Ñ&<Øòò 9ó!'Ð#ð$ 'Ð.Øà ×2Ñ2°4¸'ÐCYÐ9ZÐ]^Ð`aÐ\bÓc�	ØÐ$Øà$ Q™<�Ø%)§[¡[°¡^�—‘ Ñ"à×&Ñ&Ð':Ô;Ø×&Ñ& tÔ,Ø×!Ñ! /Õ2ñA 	3r   c                 óX   — | j                  «        | j                  j                  «        y ru   )Úadjust_reshape_and_expandr%  r'  r(  s    r   Ú
preprocesszT5OnnxModel.preprocessÏ  s   € Ø×&Ñ&Ô(Ø�‰×ÑÕr   c                 ó„   — | j                  «        | j                  «        | j                  «        | j                  «        y ru   )rD  rF  r<  r­   r(  s    r   ÚpostprocesszT5OnnxModel.postprocessÓ  s2   € à×.Ñ.Ô0Ø×)Ñ)Ô+Ø×,Ñ,Ô.à×ÑÕr   r‚   )T)rå   ræ   rç   ré   r   r)  r+  r.  r<  rD  rF  rI  rK  rë   rì   s   @r   r  r  6  sD   ø„ ñ@¨ð @¸sõ @ò&ò'ó,ò6òr!3òF"3òH ör   r  )ÚloggingÚnumpyrD   Úfusion_attentionr   r   Úfusion_baser   Úfusion_simplified_layernormr   r   Úfusion_utilsr   Úonnxr	   r
   r   Ú
onnx_modelr   Úonnx_model_bertr   Ú	getLoggerrå   r=   r   rî   r  © r   r   ú<module>rW     sc   ðó
 ã ß ;Ý ß rÝ $ß /Ñ /Ý  Ý )à	ˆ×	Ñ	˜8Ó	$€ôG
%˜ô G
%ôTV  fô V ôrc�-õ cr   