Ë
    �…j€W  ã                   ó  — d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlmZm	Z	 d dl
mZ d dlmZ d dlmZ d dlmZ d dlmZ d dlZd dlZd dlZd dlmZ d dlZ ej0                  e«      Z G d	„ d
e«      Z G d„ de«      Z G d„ d«      Zdej<                  iZddddddi fd„Z d(d„Z!d)d„Z"d„ Z#d„ Z$d„ Z%d„ Z&d*d„Z'ejP                  d fd„Z)d„ Z*d+d„Z+de,e-e.ef      dz  fd„Z/ G d „ d!e«      Z0 G d"„ d#e0«      Z1 G d$„ d%e0«      Z2d,d&„Z3d'„ Z4y)-é    N)ÚABCÚabstractmethod)ÚThreadPoolExecutor)Údatetime)ÚEnum)Úsleep)ÚAny)Úversionc                   ó"   — e Zd ZdZdZdZdZd„ Zy)Ú	PrecisionÚfp32Úfp16Úint8Úint4c                 ó   — | j                   S ©N©Úvalue©Úselfs    új/root/aria/tools/markitdown-venv/lib/python3.12/site-packages/onnxruntime/transformers/benchmark_helper.pyÚ__str__zPrecision.__str__%   ó   € Ø�z‰zÐó    N)Ú__name__Ú
__module__Ú__qualname__ÚFLOAT32ÚFLOAT16ÚINT8ÚINT4r   © r   r   r   r      s   „ Ø€GØ€GØ€DØ€Dór   r   c                   ó   — e Zd ZdZdZdZd„ Zy)ÚOptimizerInfoÚno_optÚby_ortÚ	by_scriptc                 ó   — | j                   S r   r   r   s    r   r   zOptimizerInfo.__str__0   r   r   N)r   r   r   ÚNOOPTÚBYORTÚBYSCRIPTr   r"   r   r   r$   r$   )   s   „ ð €EØ€EØ€Hór   r$   c                   ó   — e Zd Zd„ Zd„ Zd„ Zy)ÚConfigModifierc                 ó   — || _         y r   ©Ú
num_layers)r   r0   s     r   Ú__init__zConfigModifier.__init__5   s	   € Ø$ˆ�r   c                 óš  — | j                   €y t        |d«      r3| j                   |_        t        j	                  d| j                   › �«       t        |d«      r3| j                   |_        t        j	                  d| j                   › �«       t        |d«      r4| j                   |_        t        j	                  d| j                   › �«       y y )NÚnum_hidden_layersz6Modifying pytorch model's number of hidden layers to: Úencoder_layersz7Modifying pytorch model's number of encoder layers to: zdecoder_layers z7Modifying pytorch model's number of decoder layers to: )r0   Úhasattrr3   ÚloggerÚinfor4   Údecoder_layers)r   Úconfigs     r   ÚmodifyzConfigModifier.modify8   s­   € Ø�?‰?Ð"ØÜ�6Ð.Ô/Ø'+§¡ˆFÔ$Ü�K‰KÐPÐQU×Q`ÑQ`ÐPaÐbÔcÜ�6Ð+Ô,Ø$(§O¡OˆFÔ!Ü�K‰KÐQÐRV×RaÑRaÐQbÐcÔdÜ�6Ð,Ô-Ø$(§O¡OˆFÔ!Ü�K‰KÐQÐRV×RaÑRaÐQbÐcÕdð .r   c                 ó   — | j                   S r   r/   r   s    r   Úget_layer_numzConfigModifier.get_layer_numE   s   € Ø�‰Ðr   N)r   r   r   r1   r:   r<   r"   r   r   r-   r-   4   s   „ ò%òeór   r-   Úfloat32TéÿÿÿÿFc	                 óä  — t        j                  «       }	|r t         j                  j                  |	_        nt         j                  j
                  |	_        |rd|	_        |dkD  r)||	_        t        j                  d|	j                  › �«       |rd|	_
        nd|	_
        |t        j                  «       v r|g}
n=|r8|dk(  rddg}
n1|dk(  rd	dg}
n'|d
k(  s|€ddg}
n|dk(  rg d¢}
nt        d|› �«      ‚dg}
|r|
D �cg c]  }||v r|||   fn|‘Œ }
}|r|	j                  dd«       d }	 t        j                  | |	|
¬«      }|S c c}w # t        $ r t        j!                  d| › d|
› �«       Y |S w xY w)NTr   z%Session option: intra_op_num_threads=é   ÚdmlÚDmlExecutionProviderÚCPUExecutionProviderÚmigraphxÚMIGraphXExecutionProviderÚcudaÚCUDAExecutionProviderÚtensorrt)ÚTensorrtExecutionProviderrG   rC   z)The execution provider is not supported: z(mlas.enable_gemm_fastmath_arm64_bfloat16Ú1)Ú	providerszFailed to create session for z with providers=)ÚonnxruntimeÚSessionOptionsÚGraphOptimizationLevelÚORT_ENABLE_ALLÚgraph_optimization_levelÚORT_ENABLE_BASICÚenable_profilingÚintra_op_num_threadsr6   ÚdebugÚlog_severity_levelÚget_available_providersÚRuntimeErrorÚadd_session_config_entryÚInferenceSessionÚ	ExceptionÚ	exception)Úonnx_model_pathÚuse_gpuÚproviderÚenable_all_optimizationÚnum_threadsrR   ÚverboseÚ(enable_mlas_gemm_fastmath_arm64_bfloat16Úprovider_optionsÚsess_optionsrK   ÚnameÚsessions                r   Úcreate_onnxruntime_sessionrg   O   s¸  € ô ×-Ñ-Ó/€LáÜ0;×0RÑ0R×0aÑ0aˆÕ-ä0;×0RÑ0R×0cÑ0cˆÔ-áØ(,ˆÔ%à�Q‚Ø,7ˆÔ)Ü�‰Ð<¸\×=^Ñ=^Ð<_Ð`ÔaáØ*+ˆÕ'à*+ˆÔ'à”;×6Ñ6Ó8Ñ8Ø�J‰	Ù	Ø�uÒØ/Ð1GÐH‰IØ˜Ò#à+Ø&ð‰Ið ˜Ò 8Ð#3Ø0Ð2HÐI‰IØ˜Ò#ò‰Iô Ð!JÈ8È*ÐUÓVÐVà+Ð,ˆ	áØfoÖpÐ^b°tÐ?OÑ7O�dÐ,¨TÑ2Ñ3ÐUYÑYÐpˆ	Ðpá/Ø×-Ñ-Ð.XÐZ]Ô^à€GðgÜ×.Ñ.¨ÀÐXaÔbˆð €Nùò qøô ò gÜ×ÑÐ8¸Ð8IÐIYÐZcÐYdÐeÕfà€Nðgús   Ã;EÄ(E Å$E/Å.E/c                 ó  — | r&t        j                  dt         j                  ¬«       y t        j                  dt         j                  ¬«       t        j                  d«      j                  t         j                  «       y )Nz8[%(filename)s:%(lineno)s - %(funcName)20s()] %(message)s)ÚformatÚlevelz%(message)sÚtransformers)ÚloggingÚbasicConfigÚDEBUGÚINFOÚ	getLoggerÚsetLevelÚWARNING)ra   s    r   Úsetup_loggerrs   “   sP   € ÙÜ×ÑØMÜ—-‘-ö	
ô
 	×Ñ =¼¿¹ÕEÜ×Ñ˜.Ó)×2Ñ2´7·?±?ÕCr   c                 óÔ  — | r4t         j                  j                  | «      st        j                  | «       |r4t         j                  j                  |«      st        j                  |«       |rW|dk(  rdt	        j
                  «       v s<J d«       ‚t        t	        j
                  «       «      j                  ddg«      rJ d«       ‚t        j                  dt        j                  › �«       t        j                  dt        j                  › �«       t        j                  d	t        j                  › �«       t        j                  t        j                  «      t        j                  d
«      k\  sJ ‚t        j                  t        j                  «      t        j                  d«      k\  sJ ‚t        j                  t        j                  «      t        j                  d
«      k\  sJ ‚y )NrA   rB   zBPlease install onnxruntime-directml package to test GPU inference.rG   rE   zSPlease install onnxruntime-gpu package, or install migraphx, to test GPU inference.zPyTorch Version:zTransformers Version:zOnnxRuntime Version:z1.10.0z4.12.0)ÚosÚpathÚexistsÚmakedirsrL   rV   ÚsetÚ
isdisjointr6   r7   ÚtorchÚ__version__rk   r
   Úparse)Ú	cache_dirÚ
output_dirr]   r^   s       r   Úprepare_environmentr€   ž   sl  € ÙœŸ™Ÿ™¨	Ô2Ü
�‰�IÔáœ"Ÿ'™'Ÿ.™.¨Ô4Ü
�‰�JÔáØ�uÒØ)¬[×-PÑ-PÓ-RÑRð ØTóÐRô
 œ;×>Ñ>Ó@ÓA×LÑLØ(Ð*EÐFôð eàdóeð ô ‡K�KÐ"¤5×#4Ñ#4Ð"5Ð6Ô7Ü
‡K�KÐ'¬×(@Ñ(@Ð'AÐBÔCÜ
‡K�KÐ&¤{×'>Ñ'>Ð&?Ð@ÔAô �=‰=œ×*Ñ*Ó+¬w¯}©}¸XÓ/FÒFÐFÐFÜ�=‰=œ×1Ñ1Ó2´g·m±mÀHÓ6MÒMÐMÐMÜ�=‰=œ×0Ñ0Ó1´W·]±]À8Ó5LÒLÐLÑLr   c                 óp  — t        | «      t        t        | «      «      z  dz  }t        j                  | t        j
                  ¬«      dz  }|d|z  z  }t        | «      |d›t        j                  | d«      dz  d›t        j                  | d«      dz  d›t        j                  | d«      dz  d›|d›|d›dœS )Ng     @�@)Údtypez.2féZ   é_   éc   )Ú
test_timesÚlatency_varianceÚlatency_90_percentileÚlatency_95_percentileÚlatency_99_percentileÚaverage_latency_msÚQPS)ÚsumÚfloatÚlenÚnumpyÚvarÚfloat64Ú
percentile)Úlatency_listÚ
batch_sizeÚ
latency_msr‡   Ú
throughputs        r   Úget_latency_resultr˜   º   sÆ   € Ü�\Ó"¤U¬3¨|Ó+<Ó%=Ñ=ÀÑF€JÜ—y‘y ´U·]±]ÔCÀfÑLÐØ˜v¨
Ñ2Ñ3€Jô ˜,Ó'Ø/°Ð4Ü$)×$4Ñ$4°\À2Ó$FÈÑ$OÐPSÐ#TÜ$)×$4Ñ$4°\À2Ó$FÈÑ$OÐPSÐ#TÜ$)×$4Ñ$4°\À2Ó$FÈÑ$OÐPSÐ#TØ!+¨CÐ 0Ø˜SÐ!ñð r   c                 ó  — t        |ddd¬«      5 }g d¢}t        j                  ||¬«      }|j                  «        | D ]  }|j	                  |«       Œ 	 d d d «       t
        j                  d|› �«       y # 1 sw Y   Œ"xY w)NÚaÚ Úascii©ÚmodeÚnewlineÚencoding)Úenginer
   rK   ÚdeviceÚ	precisionÚ	optimizerÚ
io_bindingÚ
model_nameÚinputsÚthreadsr•   Úsequence_lengthÚcustom_layer_numr   r†   rŒ   r‹   r‡   rˆ   r‰   rŠ   ©Ú
fieldnamesz&Detail results are saved to csv file: )ÚopenÚcsvÚ
DictWriterÚwriteheaderÚwriterowr6   r7   )ÚresultsÚcsv_filenameÚcsv_fileÚcolumn_namesÚ
csv_writerÚresults         r   Úoutput_detailsr¸   Ê   s   € Ü	ˆl ¨b¸7Ô	Cð (Àxò
ˆô0 —^‘^ H¸ÔFˆ
Ø×ÑÔ Øò 	(ˆFØ×Ñ Õ'ñ	(÷7(ô< ‡K�KÐ8¸¸ÐGÕH÷=(ð (ús   �AA6Á6A?c                 óþ  — t        |ddd¬«      5 }g d¢}g }|j                  D ]O  }|j                  dgk(  r|j                  d|› �«       Œ(|j                  D ]  }|j                  d|› d|› �«       Œ ŒQ t	        j
                  |||z   ¬«      }|j                  «        |j                  D �]  }	d	D �]  }
|j                  D �]   }d
D ]ø  }|j                  D ]ç  }i }| D ]Ê  }|d   |	k(  sŒ|d   |
k(  sŒ|d   |k(  sŒ|d   |k(  sŒ'|d   |k(  sŒ0|j                  «       D ��ci c]  \  }}||v sŒ||“Œ }}}|s7|j                  |«       |j                  t        j                  |d«      «       n|D ]  }||   ||   k(  rŒJ ‚ |d   }|d   }|r|d   |d|› d|› �<   ŒÀ|d   |d|› �<   ŒÌ |sŒ×|j                  |«       Œé Œú �Œ �Œ �Œ 	 d d d «       t        j!                  d|› �«       y c c}}w # 1 sw Y   Œ(xY w)Nrš   r›   rœ   r�   )r¦   r§   rª   r¡   r
   rK   r¢   r£   r¤   r¥   r¨   ÚbÚ_sr«   )é   é   é   )TFr›   r¦   r§   r¡   r¥   r¨   r•   r©   r‹   z'Summary results are saved to csv file: )r­   Úbatch_sizesÚsequence_lengthsÚappendr®   r¯   r°   ÚmodelsÚenginesr`   ÚitemsÚupdateÚdictÚfromkeysr±   r6   r7   )r²   r³   Úargsr´   Úheader_namesÚ
data_namesr•   r©   r¶   r¦   Úinput_countÚengine_namer¥   r¨   Úrowr·   ÚkÚvÚheadersrº   Úss                        r   Úoutput_summaryrÒ   ì   sh  € Ü	ˆl ¨b¸7Ô	Cð 49Àxò
ˆð ˆ
Ø×*Ñ*ò 	KˆJØ×$Ñ$¨¨Ò,Ø×!Ñ! A j \Ð"2Õ3à'+×'<Ñ'<ò K�OØ×%Ñ%¨¨*¨°R¸Ð7HÐ&IÕJñKð		Kô —^‘^ H¸È
Ñ9RÔSˆ
Ø×ÑÔ ØŸ+™+ó 	9ˆJØ(ó 9�Ø#'§<¡<ó 9�KØ&7ò 9˜
Ø'+×'7Ñ'7ò 9˜GØ"$˜CØ*1ò T à$*¨<Ñ$8¸JÓ$FØ(.¨xÑ(8¸KÓ(GØ(.¨xÑ(8¸KÓ(GØ(.¨|Ñ(<À
Ó(JØ(.¨yÑ(9¸WÓ(Dà@FÇÁÃ×.d¹¸¸1ÐRSÐWcÒRc¨q°!©tÐ.d GÑ.dÙ+.Ø(+¯
©
°7Ô(;Ø(+¯
©
´4·=±=ÀÈRÓ3PÕ(Qà1=ò )H¨AØ36°q±6¸WÀQ¹ZÓ3GÐ,GÐ3Gð)Hà(.¨|Ñ(< AØ(.Ð/@Ñ(A AÙ'(Ø<BÐCWÑ<X¨¨a°¨s°"°Q°C¨LÒ(9à7=Ð>RÑ7S¨¨a°¨s¨Gªð)Tò*  #Ø *× 3Ñ 3°CÕ 8ñ19ò9ò9ò9ñ	9÷149ôl ‡K�KÐ9¸,¸ÐHÕIùó! /e÷M49ð 49úsO   �CG3Ã)G3Ã2G3Ã;G3ÄG3ÄG3Ä G-Ä-G-Ä2AG3Å?-G3Æ-G3Ç-G3Ç3G<c           
      ó.  — t        |ddd¬«      5 }ddddgt        t        t        | j	                  «       «      «      j                  «       «      ¢}t        j                  ||¬	«      }|j                  «        | D ]m  }t        t        j                  «       «      | |   d<   t        j                  | |   d<   t        j                  | |   d<   || |   d<   |j                  | |   «       Œo 	 d d d «       t         j#                  d
|› �«       y # 1 sw Y   Œ"xY w)Nrš   r›   rœ   r�   Úmodel_filenamer   rk   r{   r«   z(Fusion statistics is saved to csv file: )r­   ÚlistÚnextÚiterÚvaluesÚkeysr®   r¯   r°   Ústrr   Únowrk   r|   r{   r±   r6   r7   )Úmodel_fusion_statisticsr³   r´   rµ   r¶   Úkeys         r   Úoutput_fusion_statisticsrÞ   &  s  € Ü	ˆl ¨b¸7Ô	Cð >ÀxàØØØð	
ô
 ”$”tÐ3×:Ñ:Ó<Ó=Ó>×CÑCÓEÓFð
ˆô —^‘^ H¸ÔFˆ
Ø×ÑÔ Ø*ò 	>ˆCÜ7:¼8¿<¹<»>Ó7JÐ# CÑ(¨Ñ4Ü;G×;SÑ;SÐ# CÑ(¨Ñ8Ü49×4EÑ4EÐ# CÑ(¨Ñ1Ø=@Ð# CÑ(Ð)9Ñ:Ø×ÑÐ 7¸Ñ <Õ=ñ	>÷>ô  ‡K�KÐ:¸<¸.ÐIÕJ÷!>ð >ús   �CDÄDc                 óü   ‡ ‡— i }t        j                  ˆˆ fd„d|¬«       t        j                  ˆˆ fd„d|¬«      }|j                  |«       |j                  ddi«       |j                  t        ||«      «       |S )Nc                  ó(   •— ‰j                  d ‰ «      S r   ©Úrun©Ú
ort_inputsÚort_sessions   €€r   ú<lambda>zinference_ort.<locals>.<lambda><  s   ø€ ˜+Ÿ/™/¨$°
Ó;€ r   r¼   ©ÚnumberÚrepeatc                  ó(   •— ‰j                  d ‰ «      S r   rá   rã   s   €€r   ræ   zinference_ort.<locals>.<lambda>=  s   ø€ ¨¯©¸¸zÓ)J€ r   r¥   F)Útimeitré   rÅ   r˜   )rå   rä   Úresult_templateÚrepeat_timesr•   Úwarm_up_repeatr·   r”   s   ``      r   Úinference_ortrï   :  sd   ù€ Ø€FÜ
‡M�MÔ;ÀAÈnÕ]Ü—=‘=Ô!JÐSTÐ]iÔj€LØ
‡M�M�/Ô"Ø
‡M�M�< Ð'Ô(Ø
‡M�MÔ$ \°:Ó>Ô?Ø€Mr   c           
      ób  ‡ ‡— i }‰ j                  «       Š|D ]—  }t        j                  ||   «      j                  |	«      }t        j                  t        ||   j                  «      |
«      }‰j                  ||j                  j                  d||j                  |j                  «       «       Œ™ t        |«      dk(  rt        |||	«       t        |«      D ]^  \  }}‰j!                  |||   j                  j                  dt"        j$                  ||   j                  ||   j                  «       «       Œ` t'        j(                  ˆˆ fd„d|¬«       t'        j(                  ˆˆ fd„d|¬«      }|j+                  |«       |j+                  ddi«       |j+                  t-        ||«      «       |S )Nr   c                  ó&   •— ‰j                  ‰ «      S r   ©Úrun_with_iobinding©r¥   rå   s   €€r   ræ   z/inference_ort_with_io_binding.<locals>.<lambda>q  ó   ø€ �×.Ñ.¨zÓ:€ r   r¼   rç   c                  ó&   •— ‰j                  ‰ «      S r   rò   rô   s   €€r   ræ   z/inference_ort_with_io_binding.<locals>.<lambda>w  rõ   r   r¥   T)r¥   r{   Ú
from_numpyÚtoÚIO_BINDING_DATA_TYPE_MAPÚgetrÚ   r‚   Ú
bind_inputr¢   ÚtypeÚshapeÚdata_ptrr�   ÚallocateOutputBuffersÚ	enumerateÚbind_outputr�   r=   rë   ré   rÅ   r˜   )rå   rä   rì   rí   Úort_output_namesÚort_outputsÚoutput_buffersÚoutput_buffer_max_sizesr•   r¢   Ú	data_typerî   r·   re   Únp_inputÚ
input_typeÚiÚort_output_namer”   r¥   s   `                  @r   Úinference_ort_with_io_bindingr  D  s�  ù€ ð €Fð ×'Ñ'Ó)€Jàò 

ˆÜ×#Ñ# J¨tÑ$4Ó5×8Ñ8¸Ó@ˆÜ-×1Ñ1´#°jÀÑ6F×6LÑ6LÓ2MÈyÓYˆ
Ø×ÑØØ�O‰O× Ñ ØØØ�N‰NØ×ÑÓõ	
ð

ô ˆ>Ó˜aÒÜ˜nÐ.EÀvÔNä'Ð(8Ó9ò 
Ñˆˆ?Ø×ÑØØ˜1Ñ×$Ñ$×)Ñ)ØÜ�M‰MØ˜‰N× Ñ Ø˜1Ñ×&Ñ&Ó(õ	
ð
ô ‡M�MÜ:ØØõô —=‘=Ü:ØØô€Lð
 ‡M�M�/Ô"Ø
‡M�M�< Ð&Ô'Ø
‡M�MÔ$ \°:Ó>Ô?Ø€Mr   c                 ó|   — |D ]7  }| j                  t        j                  |t        j                  |¬«      «       Œ9 y )N)r‚   r¢   )rÁ   r{   Úemptyr=   )r  r  r¢   r	  s       r   rÿ   rÿ   �  s4   € ð %ò RˆØ×ÑœeŸk™k¨!´5·=±=ÈÔPÕQñRr   c                 ó  — t        j                  | «       t        j                   j                  | «       t        j                  | «       t        j
                  j	                  | «       t        j
                  j                  | «       y)z5Set random seed manually to get deterministic resultsN)ÚrandomÚseedr�   r{   Úmanual_seedrF   Úmanual_seed_all)r  s    r   Úset_random_seedr  ‰  sR   € ä
‡K�K�ÔÜ	‡L�L×Ñ�dÔÜ	×Ñ�dÔÜ	‡J�J×Ñ˜4Ô Ü	‡J�J×Ñ˜tÕ$r   Úreturnc            	      óš  — ddl m} m}m}m}m}m}m} 	  |«        g } |«       }t        |t        «      sy t        |«      D ]c  }	 | ||	«      «      }
t        |
t        «      r y |j                  |	 | ||	«      «      |
j                  |
j                  |
j                  dœ«       Œe  |«        |S # | $ r}t!        d|«       Y d }~y d }~ww xY w)Nr   ©Ú	NVMLErrorÚnvmlDeviceGetCountÚnvmlDeviceGetHandleByIndexÚnvmlDeviceGetMemoryInfoÚnvmlDeviceGetNameÚnvmlInitÚnvmlShutdown)Úidre   ÚtotalÚfreeÚusedú-Error fetching GPU information using nvml: %s)Úpy3nvml.py3nvmlr  r  r  r  r  r  r  Ú
isinstanceÚintÚrangerÚ   rÁ   r  r   r!  Úprint)r  r  r  r  r  r  r  r·   Údevice_countr	  r7   Úerrors               r   Úget_gpu_infor*  •  sÇ   € ÷÷ ñ ðÙŒ
ØˆÙ)Ó+ˆÜ˜,¬Ô,Øä�|Ó$ò 	ˆAÙ*Ñ+EÀaÓ+HÓIˆDÜ˜$¤Ô$ÙØ�M‰MàÙ-Ñ.HÈÓ.KÓLØ!ŸZ™ZØ ŸI™IØ ŸI™Iñõð		ñ 	ŒØˆøØò ÜÐ=¸uÔEÜûðús#   ” B/ µ-B/ Á#AB/ Â/C
Â4CÃC
c                   óF   — e Zd Zdd„Zd„ Zedeeee	f      dz  fd„«       Z
y)ÚMemoryMonitorc                 ó   — || _         y r   )Úkeep_measuring)r   r.  s     r   r1   zMemoryMonitor.__init__¼  s
   € Ø,ˆÕr   c                 óÖ   — dd l }d}	 t        ||j                  t        j                  «       «      j                  «       j                  dz  «      }t        d«       | j                  s	 |S Œc)Nr   é   ç{®Gázt?)	ÚpsutilÚmaxÚProcessru   ÚgetpidÚmemory_infoÚrssr   r.  )r   r2  Ú	max_usages      r   Úmeasure_cpu_usagezMemoryMonitor.measure_cpu_usage¿  s[   € Ûàˆ	ØÜ˜I v§~¡~´b·i±i³kÓ'B×'NÑ'NÓ'P×'TÑ'TÐW^Ñ'^Ó_ˆIÜ�%ŒLØ×&Ò&ØØÐð r   r  Nc                 ó   — t        «       ‚r   )ÚNotImplementedErrorr   s    r   Úmeasure_gpu_usagezMemoryMonitor.measure_gpu_usageÊ  s   € ä!Ó#Ð#r   ©T)r   r   r   r1   r9  r   rÕ   rÆ   rÚ   r	   r<  r"   r   r   r,  r,  »  s9   „ ó-ò	ð ð$ 4¨¨S°#¨X©Ñ#7¸$Ñ#>ò $ó ñ$r   r,  c                   óB   ‡ — e Zd Zdˆ fd„	Zdeeeef      dz  fd„Zˆ xZ	S )ÚCudaMemoryMonitorc                 ó$   •— t         ‰| �  |«       y r   )Úsuperr1   )r   r.  Ú	__class__s     €r   r1   zCudaMemoryMonitor.__init__Ð  s   ø€ Ü‰Ñ˜Õ(r   r  Nc                 óÚ  — ddl m}m}m}m}m}m}m} g }g }		  |«         |«       }
t        |
t        «      st        j                  d|
› �«       y t        |
«      D �cg c]  }d‘Œ }}t        |
«      D �cg c]  } | ||«      «      ‘Œ }	}	 t        |
«      D ]Y  } | ||«      «      }t        |t        «      rt        j                  d|› �«        y t        ||   |j                  dz  «      ||<   Œ[ t!        d«       | j"                  snŒ€ |«        t        |
«      D �cg c]  }||	|   ||   dœ‘Œ c}S c c}w c c}w c c}w # |$ r }t        j                  d|«       Y d }~y d }~ww xY w)	Nr   r  z*nvmlDeviceGetCount result is not integer: z%nvmlDeviceGetMemoryInfo returns str: r0  r1  ©Ú	device_idre   Úmax_used_MBr"  )r#  r  r  r  r  r  r  r  r$  r%  r6   r)  r&  rÚ   r3  r!  r   r.  )r   r  r  r  r  r  r  r  Úmax_gpu_usageÚgpu_namer(  r	  r7   r)  s                 r   r<  z#CudaMemoryMonitor.measure_gpu_usageÓ  s€  € ÷	
÷ 	
ñ 	
ð ˆØˆð	ÙŒJÙ-Ó/ˆLÜ˜l¬CÔ0Ü—‘ÐIÈ,ÈÐXÔYØä(-¨lÓ(;Ö< 1šQÐ<ˆMÐ<ÜRWÐXdÓReÖfÈQÑ)Ñ*DÀQÓ*GÕHÐfˆHÐfØÜ˜|Ó,ò R�AÙ2Ñ3MÈaÓ3PÓQ�DÜ! $¬Ô,ÜŸ™Ð'LÈTÈFÐ%SÔTÙ#Ü'*¨=¸Ñ+;¸T¿Y¹YÈÑ=PÓ'Q�M !Ò$ðRô �e”Ø×*Ò*Øð ñ ŒNô ˜|Ó,öð ð	 "#Ø$ Q™KØ#0°Ñ#3óòð ùò =ùÚfùòøð ò 	Ü�L‰LÐHÈ%ÔPÜûð	úsO   ˜6E ÁE Á	D6Á%E Á4D;Â	AE ÃAE Ä E Ä3E Ä6E ÅE*Å
E%Å%E*r=  )
r   r   r   r1   rÕ   rÆ   rÚ   r	   r<  Ú__classcell__©rB  s   @r   r?  r?  Ï  s&   ø„ õ)ð+ 4¨¨S°#¨X©Ñ#7¸$Ñ#>÷ +r   r?  c                   ó,   ‡ — e Zd Zdˆ fd„	Zd„ Zd„ Zˆ xZS )ÚRocmMemoryMonitorc                 ó@  •— t         ‰| �  |«       d}t        j                  j	                  |«      r1|t
        j                  vrt
        j                  j                  |«       	 dd l}|| _        | j                  j                  «        y # t        $ r
 d | _        Y y w xY w)Nz/opt/rocm/libexec/rocm_smir   )
rA  r1   ru   rv   rw   ÚsysrÁ   Úrocm_smiÚinitializeRsmiÚImportError)r   r.  Úrocm_smi_pathrO  rB  s       €r   r1   zRocmMemoryMonitor.__init__  sv   ø€ Ü‰Ñ˜Ô(Ø4ˆÜ�7‰7�>‰>˜-Ô(Ø¤C§H¡HÑ,Ü—‘—‘ Ô.ð	!Ûà$ˆDŒMØ�M‰M×(Ñ(Õ*øÜò 	!Ø ˆDŽMð	!ús   Á$%B
 Â
BÂBc                 óf   — | j                   €y| j                   j                  |d«      d   dz  dz  S )Nr>   ÚVRAMr   i   )rO  Ú
getMemInfo)r   Údevs     r   Úget_used_memoryz!RocmMemoryMonitor.get_used_memory  s5   € Ø�=‰=Ð ØØ�}‰}×'Ñ'¨¨VÓ4°QÑ7¸$Ñ>ÀÑEÐEr   c                 óò  — | j                   €y | j                   �#t        | j                   j                  «       «      nd}t        |«      D �cg c]  }d‘Œ }}t        |«      D �cg c]  }d|› �‘Œ	 }}	 t        |«      D ]#  }t	        ||   | j                  |«      «      ||<   Œ% t        j                  d«       | j                  snŒTt        |«      D �cg c]  }|||   ||   dœ‘Œ c}S c c}w c c}w c c}w )Nr   ÚGPUr1  rD  )	rO  r�   ÚlistDevicesr&  r3  rW  Útimer   r.  )r   r(  r	  rG  rH  s        r   r<  z#RocmMemoryMonitor.measure_gpu_usage  s  € Ø�=‰=Ð Øà;?¿=¹=Ð;T”s˜4Ÿ=™=×4Ñ4Ó6Ô7ÐZ[ˆÜ$)¨,Ó$7Ö8˜qšÐ8ˆÐ8Ü',¨\Ó':Ö; !�c˜!˜’IÐ;ˆÐ;ØÜ˜<Ó(ò R�Ü#& }°QÑ'7¸×9MÑ9MÈaÓ9PÓ#Q�˜aÒ ðRä�J‰J�uÔØ×&Ò&Øð ô ˜<Ó(ö
ð ð	 Ø  ™Ø,¨QÑ/óò
ð 	
ùò 9ùÚ;ùò
s   Á	C*Á$C/ÃC4r=  )r   r   r   r1   rW  r<  rI  rJ  s   @r   rL  rL    s   ø„ õ!òFö

r   rL  c                 ó|  — d }|dk(  rt         }nt        } |d«      }| �r#|�|}n|j                  «       }|€y |€|S t        «       5 } |«       }|j	                  |j                  «      }	 |j	                  |«      }	|	j                  «       }
d|_        |j                  «       }|€
	 d d d «       y t        j                  d|› d|› �«       t        |«      dk\  rct        |«      dk\  rUt        |«      t        |«      k(  r>d}t        |«      D ]#  \  }}|d   }||   d   }||z
  }t        ||«      }Œ% |cd d d «       S d d d «       y |�|}n|j                  «       }|€|S t        «       5 } |«       }|j	                  |j                  «      }	 |j	                  |«      }	|	j                  «       }
d|_        |j                  «       }t        j                  d|d	›d
|d	›d�«       ||z
  cd d d «       S # d|_        |j                  «       }w xY w# 1 sw Y   y xY w# d|_        |j                  «       }w xY w# 1 sw Y   y xY w)NÚrocmFzGPU memory usage: before=z  peak=r¼   r   rF  zCPU memory usage: before=z.1fz
 MB, peak=z MB)rL  r?  r<  r   Úsubmitr·   r.  r6   r7   r�   r   r3  r9  )Úis_gpuÚfuncÚmonitor_typeÚstart_memoryÚmemory_monitor_typeÚmonitorÚmemory_before_testÚexecutorÚ
mem_threadÚ	fn_threadÚ_r8  Úmax_usedr	  Úmemory_beforeÚbeforeÚafterr!  s                     r   Úmeasure_memoryrn  ,  s…  € ØÐØ�vÒÜ/Ñä/Ðá! %Ó(€GâØÐ#Ø!-Ñà!(×!:Ñ!:Ó!<ÐØÐ%Øàˆ<Ø%Ð%äÓ!ð 	  XÙ)Ó+ˆGØ!Ÿ™¨×)BÑ)BÓCˆJð0Ø$ŸO™O¨DÓ1�	Ø×$Ñ$Ó&�à).�Ô&Ø&×-Ñ-Ó/�	àÐ Ø÷	 ð 	 ô �K‰KÐ3Ð4FÐ3GÀwÈyÈkÐZÔ[ÜÐ%Ó&¨!Ò+´°I³À!Ò0CÌÐL^ÓH_ÔcfÐgpÓcqÒHqà�Ü(1Ð2DÓ(Eò 3Ñ$�A�}Ø*¨=Ñ9�FØ% a™L¨Ñ7�EØ  6™>�DÜ" 8¨TÓ2‘Hð	3ð
  ÷-	 ÷ 	 ñ 	 ð. ð ÐØ)Ñà$×6Ñ6Ó8Ðà€|Ø!Ð!ä	Ó	ð . Ù%Ó'ˆØ—_‘_ W×%>Ñ%>Ó?ˆ
ð	,Ø Ÿ™¨Ó-ˆIØ× Ñ Ó"ˆAà%*ˆGÔ"Ø"×)Ñ)Ó+ˆIä�‰Ð/Ð0BÀ3Ð/GÀzÐR[Ð\_ÐQ`Ð`cÐdÔeØÐ-Ñ-÷.ñ .øð7 */�Ô&Ø&×-Ñ-Ó/‘	ú÷	 ð. ûð$ &+ˆGÔ"Ø"×)Ñ)Ó+‰Iú÷.ð .úsO   Á#H
Á*!G.ÂH
Â.BH
Å&#H2Æ
!HÆ+9H2Ç.HÈH
È
HÈH/È/H2È2H;c                  ór   — g d¢} d}| D ]+  }t        j                  |«      }|€Œ|r|dz  }||› d|› �z  }Œ- |S )N)ÚORT_DISABLE_FUSED_ATTENTIONÚ!ORT_ENABLE_FUSED_CAUSAL_ATTENTIONÚ!ORT_DISABLE_FUSED_CROSS_ATTENTIONÚORT_DISABLE_TRT_FLASH_ATTENTIONÚ&ORT_DISABLE_MEMORY_EFFICIENT_ATTENTIONÚORT_TRANSFORMER_OPTIONSÚORT_CUDA_GEMM_OPTIONSr›   ú,ú=)ru   Úgetenv)Ú	env_namesÚenvre   r   s       r   Úget_ort_environment_variablesr|  p  s\   € ò€Ið €CØò !ˆÜ—	‘	˜$“ˆØˆ=ØÙØ�3‰JˆCØ�$��q˜˜Ð Ñ ‰ð!ð €Jr   r=  r   )r   )é{   )rF   N)5r®   rl   ru   r  rN  r[  rë   Úabcr   r   Úconcurrent.futuresr   r   Úenumr   r   Útypingr	   r�   r{   rk   Ú	packagingr
   rL   rp   r   r6   r   r$   r-   r=   rù   rg   rs   r€   r˜   r¸   rÒ   rÞ   rï   Úlonglongr  rÿ   r  rÕ   rÆ   rÚ   r*  r,  r?  rL  rn  r|  r"   r   r   ú<module>r„     s:  ðó Û Û 	Û Û 
Û Û ß #Ý 1Ý Ý Ý Ý ã Û Û Ý ã à	ˆ×	Ñ	˜8Ó	$€ô�ô ô�Dô ÷ñ ð, ˆu�}‰}ðÐ ð Ø ØØØØ-2ØóAóHDóMò8ò IòD7JòtKó(ð* �n‰nØó:òzRó%ð#�d˜4  S ™>Ñ*¨TÑ1ó #ôL$�Cô $ô(/˜ô /ôd(
˜ô (
óVA.óHr   