Ë
    �…jÊU  ã                   ó„   — d dl mZ d dlZd dlmZ d dlmZ d dlm	Z	 d dl
mZ  ee«      Z G d„ de«      Z G d	„ d
e«      Zy)é    )Ú	getLoggerN)ÚFusion)ÚFusionUtils)Úhelper)Ú	OnnxModelc                   óD   ‡ — e Zd ZdZdedefˆ fd„Zd„ Zd„ Zd„ Z	d„ Z
ˆ xZS )	ÚFusionGptAttentionPastBasez3Base class for GPT Attention Fusion with past stateÚmodelÚ	num_headsc                 óx   •— t         ‰| �  |dddgd«       || _        t        |«      | _        i | _        d | _        y )NÚ	AttentionÚLayerNormalizationÚSkipLayerNormalizationz	with past)ÚsuperÚ__init__r   r   ÚutilsÚcasted_attention_maskÚmask_filter_value©Úselfr
   r   Ú	__class__s      €ún/root/aria/tools/markitdown-venv/lib/python3.12/site-packages/onnxruntime/transformers/fusion_gpt_attention.pyr   z#FusionGptAttentionPastBase.__init__   sA   ø€ Ü‰Ñ˜ Ð.BÐD\Ð-]Ð_jÔkØ"ˆŒÜ  Ó'ˆŒ
Ø%'ˆÔ"Ø!%ˆÕó    c                 ó²  — | j                   j                  |d|«      }|�|j                  dk7  rt        j	                  d«       y | j                   j                  |d«      dk7  rt        j	                  d«       y |j                  d   }| j                   j                  |d|«      }|r|j                  dk(  r|}n>| j                   j                  |ddgddg«      }|€t        j	                  d«       y |d   }| j                   j                  |d«      dk7  rt        j	                  d	«       y |j                  d   }	||	k7  rt        j	                  d
«       y |S )Nr   ÚGatherz,match_past_pattern_1: expect Gather for pasté   z9match_past_pattern_1: expect indices=1 for Gather of pastÚ	Transposez7match_past_pattern_1: failed match Transpose and Gatheréÿÿÿÿz;match_past_pattern_1: expect indices=0 for Gather k of pastz,match_past_pattern_1: expect past to be same)r
   Ú
get_parentÚop_typeÚloggerÚdebugÚfind_constant_inputÚinputÚmatch_parent_path)
r   Úconcat_kÚconcat_vÚoutput_name_to_nodeÚgatherÚpastÚparentÚgather_past_kÚpast_k_nodesÚpast_ks
             r   Úmatch_past_pattern_1z/FusionGptAttentionPastBase.match_past_pattern_1   s9  € ð& —‘×&Ñ& x°Ð4GÓHˆØˆ>˜VŸ^™^¨xÒ7Ü�L‰LÐGÔHØà�:‰:×)Ñ)¨&°!Ó4¸Ò9Ü�L‰LÐTÔUØØ�|‰|˜A‰ˆà—‘×&Ñ& x°Ð4GÓHˆÙ�f—n‘n¨Ò0Ø"‰MàŸ:™:×7Ñ7¸À;ÐPXÐBYÐ\]Ð_`Ð[aÓbˆLØÐ#Ü—‘ÐVÔWØØ(¨Ñ,ˆMà�:‰:×)Ñ)¨-¸Ó;¸qÒ@Ü�L‰LÐVÔWØØ×$Ñ$ QÑ'ˆØ�6Š>Ü�L‰LÐGÔHØàˆr   c                 ó(  — | j                   j                  |d|«      }|�|j                  dk7  rt        j	                  d«       y | j                   j                  |d|«      }|�|j                  dk7  rt        j	                  d«       y | j                   j                  «       }|dk  r]t        j                  |ddg«      st        j	                  d«       y t        j                  |d	d
d
g«      st        j	                  d«       y | j                  j                  |d
dg«      st        j	                  d«       y | j                  j                  |d
d
d
g«      st        j	                  d«       y t        j                  |ddd¬«      st        j	                  d«       y |j                  d   }| j                   j                  |ddgddg«      }|€t        j	                  d«       y |d   j                  d   }	||	k7  rt        j                  d«       y |S )Nr   ÚSqueezez:match_past_pattern_2: expect Squeeze as parent of concat_vÚSplitz0match_past_pattern_2: expect Split for past pathé   Úaxesz:match_past_pattern_2: axes != [0] for Squeeze in past pathÚsplitr   z<match_past_pattern_2: split != [1, 1] for Split in past pathÚaxis)Údefault_valuezKmatch_past_pattern_2: attribute axis of Split are not expected in past pathz7match_past_pattern_2: failed to match past_k_nodes pathr   z,match_past_pattern_2: expect past to be same)r
   r   r    r!   r"   Úget_opset_versionr   Úcheck_node_attributer   Úcheck_node_input_valuer$   r%   Úinfo)
r   r&   r'   r(   Úsqueezer5   Úopset_versionr*   r-   r.   s
             r   Úmatch_past_pattern_2z/FusionGptAttentionPastBase.match_past_pattern_2K   sÉ  € ð, —*‘*×'Ñ'¨°!Ð5HÓIˆØˆ?˜gŸo™o°Ò:Ü�L‰LÐUÔVØà—
‘
×%Ñ% g¨qÐ2EÓFˆØˆ=˜EŸM™M¨WÒ4Ü�L‰LÐKÔLØàŸ
™
×4Ñ4Ó6ˆØ˜2ÒÜ×3Ñ3°G¸VÀaÀSÔIÜ—‘ÐYÔZØä×3Ñ3°E¸7ÀQÈÀFÔKÜ—‘Ð[Ô\Øà—:‘:×4Ñ4°W¸aÀ!ÀÔEÜ—‘ÐYÔZØà—:‘:×4Ñ4°U¸AÀÀ1¸vÔFÜ—‘Ð[Ô\Øä×/Ñ/°°v¸qÐPQÕRÜ�L‰LÐfÔgØØ�{‰{˜1‰~ˆà—z‘z×3Ñ3°H¸yÈ'Ð>RÐUVÐXYÐTZÓ[ˆØÐÜ�L‰LÐRÔSØØ˜bÑ!×'Ñ'¨Ñ*ˆà�6Š>Ü�K‰KÐFÔGØàˆr   c                 ó   — | j                   j                  |d|d¬«      }|st        j                  d«       y | j                   j                  |d|d¬«      }|st        j                  d«       y |j                  d   }|S )NÚ	UnsqueezeF)Ú	recursivezexpect unsqueeze for presentÚConcatzexpect concat for presentr   )r
   Úfind_first_child_by_typer!   r;   Úoutput)r   r'   Úinput_name_to_nodesÚunsqueeze_present_vÚconcat_presentÚpresents         r   Úmatch_presentz(FusionGptAttentionPastBase.match_presentŽ   s‰   € Ø"Ÿj™j×AÑAØ�kÐ#6À%ð Bó 
Ðñ #Ü�K‰KÐ6Ô7ØØŸ™×<Ñ<Ø Ð+>È%ð =ó 
ˆñ Ü�K‰KÐ3Ô4Øà ×'Ñ'¨Ñ*ˆØˆr   c                 ó2  — || j                   v r| j                   |   }|S | j                  j                  |«      r/| j                  j	                  |«      \  }}|| j                   |<   |S | j                  j                  |«      \  }}|| j                   |<   |S ©N)r   r
   Úfind_graph_inputr   Úcast_graph_input_to_int32Úcast_input_to_int32)r   Ú
input_nameÚattention_mask_input_nameÚcastedÚ	cast_nodes        r   Úcast_attention_maskz.FusionGptAttentionPastBase.cast_attention_maskŸ   s£   € Ø˜×3Ñ3Ñ3Ø(,×(BÑ(BÀ:Ñ(NÐ%ð )Ð(ð �Z‰Z×(Ñ(¨Ô4Ø04·
±
×0TÑ0TÐU_Ó0`Ñ-ˆFÐ-Ø5NˆD×&Ñ& zÑ2ð )Ð(ð 48·:±:×3QÑ3QÐR\Ó3]Ñ0Ð% yØ5NˆD×&Ñ& zÑ2Ø(Ð(r   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   Úintr   r/   r>   rI   rS   Ú__classcell__©r   s   @r   r	   r	      s0   ø„ Ù=ð&˜ið &°Cõ &ò/òbAòFö"	)r   r	   c                   ó8   ‡ — e Zd ZdZdedefˆ fd„Zd„ Zd„ Zˆ xZ	S )ÚFusionGptAttentionzP
    Fuse GPT-2 Attention with past state subgraph into one Attention node.
    r
   r   c                 ó&   •— t         ‰| �  ||«       y rK   )r   r   r   s      €r   r   zFusionGptAttention.__init__°   s   ø€ Ü‰Ñ˜ 	Õ*r   c
                 ó¢  — | j                   j                  d«      }
t        j                  d|||||g|
dz   |g|
¬«      }d|_        |j
                  j                  t        j                  d| j                  «      t        j                  d|	rdnd	«      g«       | j                  �C|j
                  j                  t        j                  d
t        | j                  «      «      g«       t        j                  d|
dz   |j                  d   g|
dz   g|
dz   ¬«      }t        j                  d|
dz   |j                  d   g|g|
dz   ¬«      }| j                  j                  |||g«       | j                  | j                  |j                  <   | j                  | j                  |j                  <   | j                  | j                  |j                  <   y )NÚGptAttentionr   Ú_output)ÚinputsÚoutputsÚnamezcom.microsoftr   Úunidirectionalr   r   r   ÚMatMulÚ_matmul_outputÚ_matmulÚAddé   Ú_add)r
   Úcreate_node_namer   Ú	make_nodeÚdomainÚ	attributeÚextendÚmake_attributer   r   Úfloatr$   Únodes_to_addÚthis_graph_nameÚnode_name_to_graph_namerc   )r   Ú	fc_weightÚfc_biasÚgemm_qkvr*   rH   r$   rD   ÚmaskÚis_unidirectionalÚattention_node_nameÚattention_nodeÚmatmul_nodeÚadd_nodes                 r   Úcreate_attention_nodez(FusionGptAttention.create_attention_node³   s½  € ð #Ÿj™j×9Ñ9¸.ÓIÐÜ×)Ñ)ØØ˜9 g¨t°TÐ:Ø(¨9Ñ4°gÐ>Ø$ô	
ˆð !0ˆÔØ× Ñ ×'Ñ'ä×%Ñ% k°4·>±>ÓBÜ×%Ñ%Ð&6Ñ=N¹ÐTUÓVðô	
ð ×!Ñ!Ð-Ø×$Ñ$×+Ñ+¬V×-BÑ-BÐCVÔX]Ð^b×^tÑ^tÓXuÓ-vÐ,wÔxä×&Ñ&ØØ'¨)Ñ3°X·^±^ÀAÑ5FÐGØ(Ð+;Ñ;Ð<Ø$ yÑ0ô	
ˆô ×#Ñ#ØØ'Ð*:Ñ:¸H¿N¹NÈ1Ñ<MÐNØ�HØ$ vÑ-ô	
ˆð 	×Ñ× Ñ  .°+¸xÐ!HÔIØ<@×<PÑ<Pˆ×$Ñ$ ^×%8Ñ%8Ñ9Ø9=×9MÑ9Mˆ×$Ñ$ [×%5Ñ%5Ñ6Ø6:×6JÑ6Jˆ×$Ñ$ X§]¡]Ò3r   c                 óú  — d }d }g }|j                   dk(  }d }|s%| j                  j                  |g d¢g d¢||¬«      }n$| j                  j                  |g d¢g d¢||¬«      }|€y d }	|s |\  }
}}}}}}|
j                  d|d   z
     }	n	|\  }}}}}}| j                  j                  |g d	¢g d
¢«      }|€t        j                  d«       y |\  }}}}| j                  j                  |g d¢g d¢|«      }|€"| j                  j                  |g d¢g d¢|«      }|€¤| j                  j                  |g d¢g d¢|«      }|€"| j                  j                  |g d¢g d¢|«      }|€t        j                  d«       y |d   j                  d   }| j                  j                  |d   «      \  }}|d   j                  |   }n$|d   j                  d   }|d   j                  d   }|d   }|	�$|	|j                  vrt        j                  d«       y d}d }d }d }| j                  j                  |g d¢g d¢«      } | �¹| \  }!}"}#}$}%| j                  j                  |"g d¢g d¢«      }&|&€t        j                  d«       y |&d   }'|&d   }|$|'k7  rt        j                  d«       y t        |&«      dkD  �r|&d   j                   dk(  �rò| j                  j                  |&d   «      \  }}(|(dk7  �rË|( | _        �nÁ| j                  j                  |g d ¢g d!¢fg d"¢g d#¢fg|«      \  }} }| €t        j                  d$«       y | d%   })| d&   }$| d   }%|dk(  r | d   }*| j                  j                  |*g d'¢g d(¢fg d)¢g d*¢fg d+¢g d,¢fg|«      \  }}}|€t        j                  d-«       y t        |«      dkD  r?|d   j                   dk(  r-| j                  j                  |d   «      \  }}(|(dk7  r|(| _        | j                  j                  |)g d.¢g d/¢fg d0¢g d1¢fg|«      \  }}&}|&€t        j                  d2«       y |&|dk(  rdnd   }| j                  j                  |&d   d|«      }+|+j                   d3k(  r|+}'|$|'k7  r=t        j                  d«       y |+j                   d4k(  r|+}nt        j                  d2«       | j                  j                  |j                  d   «      },t        |,t        j                  «      rIt        |,j                  «      d5k(  r1|,j                  d d d6k(  r|,j                  d   |,j                  d   k(  st        j                  d7«       y t        j                   |,t        j"                  |,«      «      rd8}nRt        j                   |,t        j$                  t        j"                  |,«      «      «      st        j                  d9«       y | j                  j                  |%g d:¢g d;¢«      }-|-€t        j                  d<«       y |-\  }.}/}0||0k7  rt        j                  d=«       y | j                  j                  |%g d	¢g d
¢«      }1|1€B| j                  j                  |%g d>¢g d?¢«      }1|1€t        j                  d@«       y |1\  }}2}3}4}5n|1\  }2}3}4}5||5k7  rt        j                  dA«       y |r|2|k7  rt        j                  dB«       y dC}6|�#|d   j                  d   }7| j'                  |7«      }6| j)                  |2||«      xs | j+                  |2||«      }|€t        j-                  dD«       y | j                  j/                  |«      st        j                  dE«       | j1                  ||«      }|€t        j-                  dF«       y | j                  j3                  |«      st        j-                  dG«       y | j5                  ||||||j6                  d   |j6                  d   |6|«	       d| _        y )HNr   )rh   ÚReshapeÚGemmr€   r€   r   re   )r   Nr   r   r   r   r   )r(   Úreturn_indice)r€   r�   r€   r€   r   re   )Nr   r   r   r   r   r   r   )rB   r   r€   r2   )r   r   r   r   z&fuse_attention: failed to match v path)r€   r�   r€   r   )r   r   r   r   )r€   r�   r€   r   )rh   re   r   )r   Nr   )rh   re   r   z'fuse_attention: failed to match fc pathri   r   zCUpstream Add and (Skip)LayerNormalization shall have one same inputT)ÚSoftmaxÚSubÚMulÚDivre   )r   r   r   r   r   )
r…   r„   ÚSlicer‡   r@   r„   r1   r‡   ÚShaper†   )
r   r   r   r   r   r   r   r   r   r   z8fuse_attention: failed to match unidirectional mask pathé   z-fuse_attention: skip since div_qk != div_maskr…   iðØÿÿ)rƒ   ÚWherer†   re   )r   r   r   r   )rƒ   rh   rŠ   r†   re   )r   r   Nr   r   z(fuse_attention: failed to match qk nodeséýÿÿÿéþÿÿÿ)r…   r„   ÚCastr@   r@   r€   )Nr   r   r   r   r   )r…   r„   r@   r@   r€   )Nr   r   r   r   )r…   r„   r@   r@   )Nr   r   r   z9fuse_attention: failed to match input attention mask path)r�   r‡   r‡   r@   r„   r1   r‡   rˆ   )r   r   r   r   r   r   r   r   )r‡   r‡   r@   r„   r1   r‡   rˆ   )r   r   r   r   r   r   r   z)fuse_attention: failed to match mask pathr†   rB   é   )r   r   z4fuse_attention: skip since mask shape is not 1x1xWxWFzDfuse_attention: skip since mask is neither lower triangular nor ones)r   r€   r2   )r   r   r   z&fuse_attention: failed to match q pathz.fuse_attention: skip since split_fc != split_q)r   rB   r   r€   r2   )r   r   r   r   r   z&fuse_attention: failed to match k pathz.fuse_attention: skip since split_fc != split_kz8fuse_attention: skip since concat_k != concat_k_to_matchÚ z)fuse_attention: failed to match past pathzpast is not graph input.z,fuse_attention: failed to match present pathz!expect present to be graph output)r    r
   r%   r$   r!   r"   Úget_constant_inputÚlenr   Úmatch_parent_pathsr   Úget_constant_valueÚ
isinstanceÚnpÚndarrayÚshapeÚallcloseÚ	ones_likeÚtrilrS   r/   r>   r;   rL   rI   Úfind_graph_outputr~   rD   Úprune_graph)8r   Únormalize_noderE   r(   r*   rH   r‚   Úis_normalize_node_skiplayernormÚ	qkv_nodesÚanother_inputÚadd_qkvÚreshape_qkvrw   Ú	reshape_1Ú	reshape_2Útranspose_qkvÚ
matmul_qkvÚv_nodesr'   Útranspose_vÚ	reshape_vÚsplit_fcÚfc_nodesru   ÚiÚ_rv   Úlayernorm_before_attentionry   Ú
slice_maskÚinput_mask_nodesÚconcat_k_to_matchÚqk_nodesÚ
softmax_qkÚsub_qkÚmul_qkÚdiv_qkÚ	matmul_qkÚ
mask_nodesÚdiv_maskÚmul_valÚwhere_qkÚadd_qkÚdiv_or_concatÚ	mask_dataÚq_nodesÚtranspose_qÚ	reshape_qÚsplit_qÚk_nodesr&   Útranspose_kÚ	reshape_kÚsplit_krP   rO   s8                                                           r   ÚfusezFusionGptAttention.fuseã   sh  € ØˆØˆØˆà*8×*@Ñ*@ÐD\Ñ*\Ð'Øˆ	á.ØŸ
™
×4Ñ4ØÚWÚ(Ø$7Ø+ð 5ó ‰Ið Ÿ
™
×4Ñ4ØÚPÚ%Ø$7Ø+ð 5ó ˆIð ÐØàˆÙ.ð ñØØØØØØØð $ŸM™M¨!¨m¸AÑ.>Ñ*>Ñ?‰Mð ñØØØØØØð —*‘*×.Ñ.¨zÒ;fÒhtÓuˆØˆ?Ü�L‰LÐAÔBØØ7>Ñ4ˆ�; 	¨8ð —:‘:×/Ñ/ØÚ@ÚØó	
ˆð ÐØ—z‘z×3Ñ3ØÚHÚØ#ó	ˆHð Ðà—z‘z×3Ñ3ØÚ7ÚØ#ó	ˆHð ÐØŸ:™:×7Ñ7ØÚ?Ú Ø'ó	�ð ÐÜ—‘ÐFÔGØà  ™×)Ñ)¨!Ñ,ˆIØ—:‘:×0Ñ0°¸!±Ó=‰DˆAˆqØ˜q‘k×'Ñ'¨Ñ*‰Gà  ™×)Ñ)¨!Ñ,ˆIØ˜q‘k×'Ñ'¨Ñ*ˆGà%-¨b¡\Ð"ð Ð$¨Ð>X×>^Ñ>^Ñ)^Ü�L‰LÐ^Ô_Øà ÐØˆ
ØÐØ ÐØ—:‘:×/Ñ/°
Ò<fÒhwÓxˆØÐØ>FÑ;ˆZ˜ ¨°ØŸ™×5Ñ5Øòò /óˆJð  Ð!Ü—‘ÐWÔXØØ! "‘~ˆHØ# A™ˆJà˜Ò!Ü—‘ÐLÔMØä�:‹ Ó" z°!¡}×'<Ñ'<ÀÓ'EØ!ŸZ™Z×:Ñ:¸:Àa¹=ÓI‘
��7Ø˜fÓ$Ø.5¨X�DÖ*ð "ŸZ™Z×:Ñ:Øâ:ºLÐIÚAÒCUÐVðð $ó‰NˆAˆx˜ð ÐÜ—‘ÐGÔHØà ‘|ˆHØ˜b‘\ˆFØ  ™ˆIà�AŠvØ! !™�Ø)-¯©×)FÑ)FØò XÚ1ðò
 PÚ.ðò
 EÚ+ððð (ó!*Ñ&�Ð# Qð$ $Ð+Ü—L‘LÐ!\Ô]ØÜÐ'Ó(¨1Ò,Ð1AÀ!Ñ1D×1LÑ1LÐPUÒ1UØ!%§¡×!>Ñ!>Ð?OÐPQÑ?RÓ!S‘J�A�wØ &Ò(Ø18˜Ô.à#Ÿz™z×<Ñ<Øò dÚ0ðò \Ú-ðð
ð $ó ÑˆAˆz˜1ð Ð!ä—‘ÐHÔIØà#¨¨aª¡A°QÑ7ˆJà ŸJ™J×1Ñ1°*¸R±.À!ÐEXÓYˆMØ×$Ñ$¨Ò-Ø(�Ø˜XÒ%Ü—L‘LÐ!PÔQØØ×&Ñ&¨(Ò2Ø$1Ñ!ä—‘ÐHÔIð —J‘J×1Ñ1°*×2BÑ2BÀ1Ñ2EÓFˆ	ä�y¤"§*¡*Ô-Ü�I—O‘OÓ$¨Ò)Ø—‘  Ð# vÒ-Ø—‘ Ñ" i§o¡o°aÑ&8Ò8ä�L‰LÐOÔPØä�;‰;�y¤"§,¡,¨yÓ"9Ô:Ø %ÑÜ—‘˜Y¬¯©´·±¸YÓ0GÓ(HÔIÜ�L‰LÐ_Ô`Øà—*‘*×.Ñ.¨yÒ:[Ò]fÓgˆØˆ?Ü�L‰LÐAÔBØØ,3Ñ)ˆ�i Ø�wÒÜ�L‰LÐIÔJØà—*‘*×.Ñ.¨yÒ:eÒgsÓtˆØˆ?à—j‘j×2Ñ2ØÚHÚóˆGð
 ˆÜ—‘ÐEÔFØàAHÑ>��H˜k¨9±gà:AÑ7ˆX�{ I¨wØ�wÒÜ�L‰LÐIÔJØá Ð->Ò!>Ü�L‰LÐSÔTØà$&Ð!ØÐ'Ø)¨"Ñ-×3Ñ3°AÑ6ˆJØ(,×(@Ñ(@ÀÓ(LÐ%ð ×(Ñ(¨°8Ð=PÓQò 
ÐUY×UnÑUnØ�hÐ 3óV
ˆð ˆ<Ü�K‰KÐCÔDØØ�z‰z×*Ñ*¨4Ô0Ü�L‰LÐ3Ô4ð ×$Ñ$ XÐ/BÓCˆØˆ?Ü�K‰KÐFÔGØØ�z‰z×+Ñ+¨GÔ4Ü�K‰KÐ;Ô<Øà×"Ñ"ØØØØØØ&×-Ñ-¨aÑ0Ø×Ñ˜qÑ!Ø%Øô
	
ð  ˆÕr   )
rT   rU   rV   rW   r   rX   r   r~   rÇ   rY   rZ   s   @r   r\   r\   «   s(   ø„ ñð+˜ið +°Cõ +ò.Kö` r   r\   )Úloggingr   Únumpyr•   Úfusion_baser   Úfusion_utilsr   Úonnxr   Ú
onnx_modelr   rT   r!   r	   r\   © r   r   ú<module>rÏ      sA   ðõ
 ã Ý Ý $Ý Ý  á	�8Ó	€ôX) ô X)ôvw Ð3õ w r   