j=bzddlmZddlZddlmZddlmZddlm Z m Z m Z m Z ddl mZeeZGddeZy) ) getLoggerN)Fusion) FusionUtils) NodeProto TensorProtohelper numpy_helper) OnnxModelc $eZdZdZdeffd Zd dedefdZdede defd Z d e d e de fd Z d ede dzfdZ d ede dzfdZdede fdZdedee efde fdZd ede dzfdZd ede dzfdZde de dzfdZde de de de dedef dZdZxZS)!FusionMultiHeadAttentionMMDitzO Fuse MultiHeadAttention for Multimodal Diffusion Transformer (MMDiT). modelc:t||ddgi|_y)NMultiHeadAttentionSoftmax) fused_op_typesearch_op_types)super__init__unsqueeze_update_map)selfr __class__s j/root/aria/tools/markitdown-venv/lib/python3.12/site-packages/onnxruntime/transformers/fusion_mha_mmdit.pyrz&FusionMultiHeadAttentionMMDit.__init__s$ .BU^T_`$&! start_nodereturnc0|jj|gd|ddg|}|y|d}t|jdk7ry|jj |jd}|yt|j dk7ryt |dS)a Detect num_heads from Reshape & Transpose of q/k/v for both Stable Diffusion 3.x and Flux 1.x: MatMul .. [-1] [24] .. | | | / / Add Concat(axis=0) | / Reshape | Transpose(perm=0,1,3,2) | (start_node) ) TransposeReshapeConcatroutput_name_to_node)r match_parent_pathleninputget_constant_valueshapeint)rrr" input_indexnodes concat_shapevalues r get_num_headsz+FusionMultiHeadAttentionMMDit.get_num_headss ,, :[!Qr?r<r^rV)r r&r'r(rrWrrFrKrGrHrIrErJr<rO) rrPr"rXconcatrYrZsln_b transpose_bnew_concat_nodes radjust_query_from_bnsh_to_bsdz;FusionMultiHeadAttentionMMDit.adjust_query_from_bnsh_to_bsdsBzz++  C   <%)"{ v||  !zz++  +[ 9 F  <!{// V\R// V\R//B%**1- A$**1- A ** LLOU\\!_5]]1%/0,,X6     1=A=Q=Q$$_%9%9:!!/"8"8";V]]1=MPV=VWWr unsqueezecn|jj|j}|t|jdk(rPt j d|j|jddzg|jjddg}nd}|jj|Ot j|tjdgdg}|jj||jt j d|jd|g|jddzg|jjd }|j j#||j|j$|j<|jd}||j|j<|S) Nr UnsqueezerrUr%)r>r?r<axesunsqueeze_axes_2)r< data_typedimsvalsr=)rgetr<r'r(rrFrKr rGr@ make_tensorrINT64rDrErHrIrJ)rreupdated_unsqueeze_outputnew_nodeinitializer_nameris rupdate_unsqueeze_axes_1_to_2z:FusionMultiHeadAttentionMMDit.update_unsqueeze_axes_1_to_2s#'#<#<#@#@#P # +9??#q(!++$??&--a07:;44[A $6 ::--.>?G'-'9'9-"-"3"3SS ($ JJ../?AUAUV!++%OOA.0@A&--a07:;44[A     $ $X .:>:N:ND ( ( 7'/q'9 $8PD % %inn 5''raddr"clt|jdk7ry|jj|gdgd|}|yt |j}|j |d}||dgk7ry|j |d}||dgk7ry|jj|gdgd|}|y|j |d}||dgk7ry|j |d}||dgk7ry|j |d|djd<|j |d|djd<y) a Update axes of Unsqueeze from [1] to [2] in the following pattern: Unsqueeze Unsqueeze (axes=[0]) (axes=[0]) | | Unsqueeze Unsqueeze ... (axes=[1]) ... (axes=[1]) | / | / Mul Mul | / Add Args: add (NodeProto): the Add node output_name_to_node (Dict[str, NodeProto]): mapping from output name to node Returns: bool: True if the pattern is matched and updated successfully, False otherwise. r%F)Mulrgrg)r r rr r)rr rT)r'r(r r&rget_squeeze_or_unsqueeze_axesrs)rrtr"nodes_b fusion_utilsaxes_1axes_0nodes_as rupdate_unsqueeze_axesz3FusionMultiHeadAttentionMMDit.update_unsqueeze_axes(sL& syy>Q **..s4UW`buv ?"4::. ;;GAJG >Vs];;GAJG >Vs]**..s4UW`buv ?;;GAJG >Vs];;GAJG >Vs]"?? K "?? K rc|jj|gdgd}|y|\}}}}}t|jdk7ry|jj|ddgddg}|y|\} } t j |d gd syt j | d gd syt j |d dsy|j ||sy|jd|jd<| jd| jd<tjd |jd| jdg|jdd zg|jjd d} |jj| |j|j| j<|jj!|jd| jd|j#|jd|jddzS)a3 Adjust graph to change query format from BNSH to BSD for Flux model. Note that the graph pattern is complex, and we only do a shallow match here. Before: | | Transpose(perm=0,2,1,3) Transpose(perm=0,2,1,3) | | SimplifiedLayerNorm SimplifiedLayerNorm | / Concat(axis=2) | Mul Mul | / Add | Mul After (Transpose nods are removed, and a Reshape is added): | | SimplifiedLayerNorm SimplifiedLayerNorm | / Concat(axis=1) | Mul Mul | / Add | Reshape (shape=[0, 0, -1]) )Addrvrr4r)rrrrrNr%r4rr rrRrSr^rrUr_rV)r r&r'r(rrWr}rrFrKrGrHrIrErJr<replace_input_of_all_nodesrO) rrPr"rXrt_mul_ar`rYrZrarbrcs r"adjust_flux_query_from_bnsh_to_bsdz@FusionMultiHeadAttentionMMDit.adjust_flux_query_from_bnsh_to_bsd]sBzz++  Q   <26/VVUK v||  !zz++  +[ 9 F  <!{// V\R// V\R//B))#/BC%**1- A$**1- A ** LLOU\\!_5]]1%/0,,X6     1=A=Q=Q$$_%9%9: --fmmA.>@V@VWX@YZ!!#**Q-A1GHHrc|jj|gdgd}|y|\}}}}tj|dgdsy|j ||sy|j d|j d<|j ddz|j d<|j|j d|j ddzS) a0 Adjust graph to change query format from BNSH to BSD for Flux model. Note that the graph pattern is complex, and we only do a shallow match here. Before: | Transpose(perm=0,2,1,3) | SimplifiedLayerNorm | Mul Mul | / Add | Mul After (Transpose is removed, and a Reshape is added): | SimplifiedLayerNorm | Mul Mul | / Add | Reshape (shape=[0, 0, -1]) )rrvr4r)rrrrNrRrSrrUrV)r r&rrWr}r(rKrO)rrPr"rXrtrrYrZs r)adjust_flux_single_query_from_bnsh_to_bsdzGFusionMultiHeadAttentionMMDit.adjust_flux_single_query_from_bnsh_to_bsds:zz++  G   <*.'VUK// V\R))#/BC%**1- A 1 / 1 !!#**Q-A1GHHrqc &tjd|g|dzg|jjddgd}|jj ||j |j|j<|j|dz|dzS)NrrUTranspose_BNSH_to_BSNH) name_prefixrS)r<rRrV) rrFr rGrHrIrErJr<rO)rrr" transpose_qs rtranspose_reshape_bnsh_to_bsdz;FusionMultiHeadAttentionMMDit.transpose_reshape_bnsh_to_bsds&&  C [M,,[F^,_     -9=9M9M$$[%5%56!!!g+q6z::rkvrK num_headsc|dkDsJ|||g}|g}tjd|||jjd}d|_|j j tjd|g|S)a~ Create a MultiHeadAttention node. Args: q (str): name of q k (str): name of k v (str): name of v output (str): output name of MHA num_heads (int): number of attention heads. If a model is pruned, it is the number of heads after pruning. Returns: NodeProto: the node created. rrr=z com.microsoftr)rrFr rGdomain attributeextendmake_attribute) rrrrrKr mha_inputs mha_outputsmha_nodes rcreate_multihead_attention_nodez=FusionMultiHeadAttentionMMDit.create_multihead_attention_nodes,1}}AY h ## ,,-AB  *!!6#8#8i#P"QRrc|jdk(sJ|}|jj|jdry|jj |gdgd|}|y|\}}}t j |dgdsy|jj|gdgd} | y| \} } } } }}}}| jd}||jdk7ry|jj| d d gd dg}|y|\}}|jd}t j |dgd sy|jj|d dgd dg}|y|djd| jdk7ry|jd }|jj|dd |}|x|jj|d d|}|yt j |dgdsy|jj|d d |}|yt j |dgds=y|jj|d d |}|yt j |dgdsy|r|j||n|j||d }|dk(r|j|||du}|dkry||j| |}n|j| |}|:|j| |}|&|j| |}||j!||}|j#||||jd|}|j$j'||j(|j*|j,<|j.j1|||gd|_y)Nrr)MatMulrr)rrrrrRrS)rrvSqrtDivrCastSliceShape)rrr rr rrrrvrr )rr rTr%rrr)r,r")r,)rrrrKrT)op_typer find_graph_outputrKmatch_child_pathrrWr&r( match_parentr0r5rdr\rrrrrHrIrErJr<nodes_to_remover prune_graph)rnodeinput_name_to_nodesr"softmaxr- matmul_s_v transpose_out reshape_outq_nodes matmul_qkrPsqrt_q_2div_qsqrt_q_shape_qq_bnshk_nodesmul_kr1r k_scale_nodesrconcat_v transpose_1 transpose_2rqueryrqs rfusez"FusionMultiHeadAttentionMMDit.fuses||y((( :: ' 'q(9 :  ++ 79QSf  = 16. M;// v|T **..  N $  ? CJ@ 5(E61aQ W]]1% % **..y5+:NQRTUPVW ? ${   a // V\R  44UVUOaQRVT     ! !! $q(9 9    Q ::**:xQdw*x   **11+1J]2K"33KV**11+1J]2K"33KV**11KQL_2K"33KV    x)< =##J0CQR#S  >11+?RT\dhThiIA~  66u>QRE@@H[\E =;;ECVWE}FFuNab=!>>vGZ[E77%%a( 8    *6:6J6J$$X]]3 ##Z $LM r)r)__name__ __module__ __qualname____doc__r rrr+r0boolr5strrOr\rdrsdictr}rrrrr __classcell__)rs@rr r sv'i' Z]B/ /im/ru/b######4.HY.H`cfj`j.H`MX9MXVY\`V`MX^"(i"(C"(H33cS\nI]3bf3jRI RI[^ae[eRIh1Iy1Ibehlbl1If ;s ;CRVJ ;) ) )  )  )  ) )V rr )loggingrnumpyrB fusion_baserryronnxrrrr onnx_modelr rloggerr rrrs4 $== 8 K  FK  r