
    jM                     h   U d Z ddlZddlZddlmZmZmZmZmZ ddl	m
Z
 ddlmZmZ ddlmZ ddlmZ dZ	 dd	lmZmZ dd
lmZmZ ddlmZ ddlmZ  G d dee
      Z i de jB                  de jD                  de jF                  de jH                  de jJ                  de jL                  d e jN                  d!e jP                  d"e jR                  d#e jT                  d$e jV                  d%e jX                  d&e jX                  d'e jX                  d(e jZ                  d)e j\                  d*e j^                  i d+e j`                  d,e j`                  d-e jb                  d.e jd                  d/e jf                  d0e jh                  d1e jj                  d2e jl                  d3e jn                  d4e jp                  d5e jr                  d6e jt                  d7e jv                  d8e jx                  d9e jz                  d:e j|                  d;e j~                  Z@eee f   eAd<<   i e jB                  d=d>ge jD                  d?ge jF                  d@ge jH                  dAge jJ                  dBdCge jL                  dDge jN                  dEge jP                  dFdGge jR                  dHdIge jT                  dJge jV                  dKge jX                  dLge jZ                  dMge j\                  dNge j^                  dOge j`                  dPdQge jb                  dRge jd                  dSge jf                  dTge jh                  dUge jj                  dVge jl                  dWge jn                  dXge jp                  dYge jr                  dZd[ge jt                  d\d]ge jv                  g d^e jx                  d_d`ge jz                  dage j|                  dbge j~                  dcgiZBee ee   f   eAdd<   dZd_dedRdfZCeeef   eAdg<   e jB                  e jD                  e jF                  e jH                  e jJ                  e jL                  e jN                  e jP                  e jR                  e jT                  e jV                  hZDe jX                  e jZ                  e j\                  e j^                  e j`                  hZEe jb                  e jd                  e jf                  e jh                  e jj                  e jl                  e jn                  e jp                  hZFe jr                  e jt                  e jv                  e jx                  e jz                  e j|                  e j~                  hZGdhdhdidjdkZH eIe       ZJdle dmefdnZK	 ddoedpeee       dmee    fdqZLdree   dmefdsZMdree   dmefdtZNdle dree   dmefduZOdredveePe       dmee    fdwZQdxdydzd{d|ZReeef   eAd}<   i dhdxd~dxddxddxddxddxddxddxddxddxddxddxddxddxddxddxddxi ddxddxddxddxddxddxddxddxddxddxddxddxddxddxddyddyddzdzdzd{d{d{dZSeeef   eAd<   dededmefdZTdededmeUfdZV G d de      ZWy# e$ rJ  ej<                         Z G d d      Z G d d      Z G d d      Z G d d      Z G d d      Zd ZY ]w xY w)a  Azure Content Understanding converter for MarkItDown.

Converts files using Azure Content Understanding (CU) for high-quality,
multi-modal extraction with structured field output. Supports documents,
images, audio, and video. Fields are serialized as YAML front matter via
the CU SDK's ``to_llm_input()`` helper.

Install dependencies: ``pip install markitdown[az-content-understanding]``
    N)BinaryIOAnyListOptionalDict)Enum   )DocumentConverterDocumentConverterResult)
StreamInfo)MissingDependencyException)ContentUnderstandingClientto_llm_input)AzureKeyCredentialTokenCredential)UserAgentPolicy)DefaultAzureCredentialc                       e Zd Zy)r   N__name__
__module____qualname__     d/root/aria/tools/markitdown-venv/lib/python3.12/site-packages/markitdown/converters/_cu_converter.pyr   r          r   r   c                       e Zd Zy)r   Nr   r   r   r   r   r   "   r   r   r   c                       e Zd Zy)r   Nr   r   r   r   r   r   %   r   r   r   c                       e Zd Zy)r   Nr   r   r   r   r   r   (   r   r   r   c                       e Zd Zy)r   Nr   r   r   r   r   r   +   r   r   r   c                       y Nr   )argskwargss     r   r   r   .   s    r   c                       e Zd ZdZdZdZdZdZdZdZ	dZ
d	Zd
ZdZdZdZdZdZdZdZdZdZdZdZdZdZdZdZdZdZdZdZdZ dZ!d Z"y!)"ContentUnderstandingFileTypez:Supported file types for Content Understanding conversion.pdfdocxpptxxlsxhtmltxtmdrtfxmlemlmsgjpegpngbmptiffheifmp4m4vmovavimkvwebmflvwmvwavmp3m4aflacoggaacwmaN)#r   r   r   __doc__PDFDOCXPPTXXLSXHTMLTXTMDRTFXMLEMLMSGJPEGPNGBMPTIFFHEIFMP4M4VMOVAVIMKVWEBMFLVWMVWAVMP3M4AFLACOGGAACWMAr   r   r   r&   r&   7   s    D CDDDD
C	B
C
C C
C D
C
CDD C
C
C
C
CD
C
C C
C
CD
C
C
Cr   r&   z.pdfz.docxz.pptxz.xlsxz.htmlz.txtz.mdz.rtfz.xmlz.emlz.msgz.jpgz.jpegz.jpez.pngz.bmpz.tiffz.heifz.heicz.mp4z.m4vz.movz.aviz.mkvz.webmz.flvz.wmvz.wavz.mp3z.m4az.flacz.oggz.aacz.wma_EXTENSION_MAPzapplication/pdfzapplication/x-pdfzGapplication/vnd.openxmlformats-officedocument.wordprocessingml.documentz<application/vnd.openxmlformats-officedocument.presentationmlzAapplication/vnd.openxmlformats-officedocument.spreadsheetml.sheetz	text/htmlzapplication/xhtml+xmlz
text/plainztext/markdownztext/rtfzapplication/rtfztext/xmlzapplication/xmlzmessage/rfc822zapplication/vnd.ms-outlookz
image/jpegz	image/pngz	image/bmpz
image/tiffz
image/heifz
image/heicz	video/mp4video/x-m4vzvideo/quicktimezvideo/x-msvideozvideo/x-matroskaz
video/webmzvideo/x-flvzvideo/x-ms-wmvz	audio/wavaudio/x-wavz
audio/mpegz	audio/mp3)	audio/mp4z	audio/m4aaudio/x-m4az
audio/flacaudio/x-flacz	audio/oggz	audio/aaczaudio/x-ms-wma_MIME_PREFIXESri   )rh   rk   rj   rg   _MIME_ALIASESprebuilt-documentSearchprebuilt-videoSearchprebuilt-audioSearch)documentimagevideoaudio	file_typereturnc                 f    | t         v ry| t        v ry| t        v ry| t        v ryt	        d|        )z*Get the modality category for a file type.rq   rr   rs   rt   zUnknown file type: )_DOCUMENT_TYPES_IMAGE_TYPES_VIDEO_TYPES_AUDIO_TYPES
ValueError)ru   s    r   _get_modalityr}      sA    O#	l	"	l	"	l	"
*9+6
77r   stream_info
file_typesc                     |t        |      nd}| j                  xs dj                         }t        j	                  |      }||||v r|S t        | j                        }|syt        ||      S )z<Detect a supported CU file type from extension or MIME type.N )set	extensionlowerrf   get_clean_mime_typemimetype_detect_file_type_from_mime)r~   r   allowedr   ru   r   s         r   _detect_file_typer     sw    
 ",!7c*oTG&&,"335I""9-I'/Y'5I 4 45H&x99r   r   c                 l    | xs dj                  dd      d   j                         j                         S )Nr   ;   r   )splitstripr   )r   s    r   r   r     s0    N!!#q)!,224::<<r   c                 L    t        |       }t        j                  ||      xs dS )Napplication/octet-stream)r   rm   r   )r   cleaneds     r   _canonical_mime_typer     s%    x(GWg.L2LLr   c                     t         j                  | g       }t        |      }|r!|dk7  r|D ]  }|j                  |      s|c S  |rt        |d         S |S )a  Resolve the content type to send to the CU API.

    Uses the resolved ``file_type`` as the source of truth so analyzer
    routing and payload metadata stay consistent. The caller-provided
    ``mimetype`` is only used when it is consistent with ``file_type``
    (e.g., to preserve subtype distinctions like ``image/heic`` vs
    ``image/heif``). When ``mimetype`` disagrees with the resolved
    ``file_type`` (e.g., ``.pdf`` extension with ``audio/mpeg``
    mimetype), the canonical MIME type for ``file_type`` is used.
    r   r   )rl   r   r   
startswith)ru   r   prefixes	canonicalprefixs        r   _content_type_forr   $  sj     !!)R0H$X.I I!;; 	!F##F+  	!
 #HQK00r   r   c                     t         j                         D ]*  \  }}|||vr|D ]  }| j                  |      s|c c S  , y r"   )rl   itemsr   )r   r   	candidater   r   s        r   r   r   B  sY      .335 !	89G#; 	!F""6*  	!! r   rq   rr   rt   rs   )prebuilt-documentprebuilt-imageprebuilt-audioprebuilt-video_BASE_TO_MODALITYzprebuilt-layoutzprebuilt-readr   zprebuilt-invoicezprebuilt-receiptzprebuilt-receipt.genericzprebuilt-receipt.hotelzprebuilt-idDocumentzprebuilt-idDocument.genericzprebuilt-idDocument.passportzprebuilt-healthInsuranceCard.uszprebuilt-contractzprebuilt-creditCardzprebuilt-creditMemozprebuilt-bankStatement.uszprebuilt-check.uszprebuilt-purchaseOrderzprebuilt-procurementzprebuilt-payStub.uszprebuilt-utilityBillzprebuilt-marriageCertificate.uszprebuilt-documentFieldSchemazprebuilt-documentFieldszprebuilt-tax.uszprebuilt-tax.us.w2zprebuilt-tax.us.w4zprebuilt-tax.us.1040zprebuilt-mortgage.uszprebuilt-mortgage.us.1003z&prebuilt-mortgage.us.closingDisclosurer   zprebuilt-imageSearchr   )rp   zprebuilt-callCenterr   ro   zprebuilt-videoSynopsis_KNOWN_PREBUILT_MODALITYclientanalyzer_idc                     |t         v r	t         |   S 	 | j                  |      }|j                  r t
        j                  |j                  d      S y# t        $ r}t        d| d|       |d}~ww xY w)a6  Resolve analyzer modality from cache or via get_analyzer() fallback.

    For known prebuilt-* names, returns the modality from
    ``_KNOWN_PREBUILT_MODALITY`` without an API call.  For unknown
    prebuilt-* names or custom analyzers, calls ``get_analyzer()``
    to inspect ``base_analyzer_id``.

    Args:
        client: A ``ContentUnderstandingClient`` instance.
        analyzer_id: The analyzer ID to resolve.

    Returns:
        Modality string ("document", "image", "audio", or "video").

    Raises:
        ValueError: If ``get_analyzer()`` fails.
    zFailed to resolve analyzer 'z': Nrq   )r   get_analyzer	Exceptionr|   base_analyzer_idr   r   )r   r   analyzer_infoexcs       r   _resolve_analyzer_modalityr     s    & ..'44X++K8 %% $$]%C%CZPP  X7}CuMNTWWXs   A 	A1A,,A1file_modalityanalyzer_modalityc                     |dk(  r| dv S | |k(  S )zBReturn True when an analyzer modality can process a file modality.rq   >   rr   rq   r   )r   r   s     r   _is_analyzer_compatibler     s"    J& 555---r   c                        e Zd ZdZdddddedeez  dz  dee   deee	      f fdZ
d	ed
ededefdZd	ed
ededefdZ xZS )ContentUnderstandingConverterzConverts files using Azure Content Understanding.

    Provides high-quality document, image, audio, and video conversion
    with structured field extraction via YAML front matter.
    N)
credentialr   r   endpointr   r   r   c                   t         |           t        )t        d      t        d   j	                  t        d         ||nt
        | _        || _        d| _        |7t        j                  j                  d      }|t        |      }n
t               }	 ddlm} d| }t#        ||t%        |	      
      | _        | j                  &t)        | j&                  | j                        | _        yy# t         $ r d}Y bw xY w)a  Initialize the Content Understanding converter.

        Args:
            endpoint: CU resource endpoint URL.
            credential: Explicit credential. If None, falls back to
                AZURE_API_KEY env var, then DefaultAzureCredential.
            analyzer_id: Custom analyzer for compatible file types.
                When set, the converter checks the analyzer's base modality
                (via get_analyzer() at init) and routes only compatible
                file types to it. Incompatible modalities auto-route to
                default prebuilts. If None, auto-selects by extension/MIME.
            file_types: Which file types to handle. If None, uses the
                default set (all supported formats).
        NzContentUnderstandingConverter requires the optional dependency [az-content-understanding] (or [all]) to be installed. E.g., `pip install markitdown[az-content-understanding]`r   r	   AZURE_API_KEY)__version__unknownzmarkitdown-cu/)
user_agent)r   r   user_agent_policy)super__init___dependency_exc_infor   with_traceback_ALL_FILE_TYPES_file_types_analyzer_id_analyzer_modalityosenvironr   r   r   	__about__r   ImportErrorr   r   _clientr   )	selfr   r   r   r   api_keyr   r   	__class__s	           r   r   z&ContentUnderstandingConverter.__init__  s   , 	  +,K (n$Q' *4)?:_'15 jjnn_5G"/8
35
	$/ &k]3
 2!-D
 (&@d//'D# )  	$#K	$s   C1 1C?>C?file_streamr~   r$   rv   c                 2    t        || j                        duS )z6Return True if the file type is in the configured set.N)r   r   )r   r   r~   r$   s       r   acceptsz%ContentUnderstandingConverter.accepts  s     !d.>.>?tKKr   c                    t        || j                        }|t        d      t        |      }| j                  /| j
                  #t        || j
                        r| j                  }nt        j                  |d      }|j                         }t        ||j                        }| j                  j                  |||      }	|	j                         }
t        |
      }t!        |      S )zEConvert the file using CU and return Markdown with YAML front matter.z;Unsupported file type for Content Understanding conversion.rn   )r   binary_inputcontent_type)markdown)r   r   r|   r}   r   r   r   _PREBUILT_ANALYZERSr   readr   r   r   begin_analyze_binaryresultr   r   )r   r   r~   r$   ru   r   r   
file_bytesr   pollerr   texts               r   convertz%ContentUnderstandingConverter.convert  s     &k43C3CD	M  &i0 )''3't7N7NO++K-118K
 !%%'
(K4H4HI 22##% 3 
  F# '55r   )r   r   r   rF   strr   r   r   r   r&   r   r   r   r   boolr   r   r   __classcell__)r   s   @r   r   r     s     CG%)CGB B '84?	B
 c]B T">?@BHLL  L 	L
 
L-6-6  -6 	-6
 
!-6r   r   r"   )XrF   sysr   typingr   r   r   r   r   enumr   _base_converterr
   r   _stream_infor   _exceptionsr   r   azure.ai.contentunderstandingr   r   azure.core.credentialsr   r   azure.core.pipeline.policiesr   azure.identityr   r   exc_infor   r&   rG   rH   rI   rJ   rK   rL   rM   rN   rO   rP   rQ   rR   rS   rT   rU   rV   rW   rX   rY   rZ   r[   r\   r]   r^   r_   r`   ra   rb   rc   rd   re   rf   __annotations__rl   rm   rx   ry   rz   r{   r   listr   r}   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   <module>r      sS
    	 6 6  H % 4  VJ<5:*3 *\(;
(,,(; )..(; )..	(;
 )..(; )..(; (,,(; 
'**(; (,,(; (,,(; (,,(; (,,(; (--(;  )..!(;" (--#(;$ (,,%(;& (,,'(;( )..)(;* )..+(;, )..-(;0 (,,1(;2 (,,3(;4 (,,5(;6 (,,7(;8 (,,9(;: )..;(;< (,,=(;> (,,?(;B (,,C(;D (,,E(;F (,,G(;H )..I(;J (,,K(;L (,,M(;N (,,O(;S667 (V+A $$'8:M&N+A !%%Q(+A !%%F(+A !%%K(+A !%%5L'M+A !$$|n+A !##o%6+A !$$z3D&E+A  !$$z3D&E!+A$ !$$'7&8%+A& !$$'C&D'+A* !%%~++A, !$${m-+A. !$${m/+A0 !%%~1+A2 !%%l'C3+A6 !$${m7+A8 !$$}o $$'8&9 $$'8&9 $$'9&: %%~ $$}o $$'7&8 $${M&B $$|[&A $$&O %%n'E $${m $${m $$'7&8U+A149<= +\  	!tCH~  !$$ %% %% %% %% $$ ## $$ $$ $$ $$ !%% $$ $$ %% %% !$$ $$ $$ $$ $$ %% $$ $$	 !$$ $$ $$ %% $$ $$ $$ *&##	  34
89 
8c 
8 @D:::;<: *+:&=x} = =M8C= MS M
+sm 	<

c678
 *+
$ $	% 4S> .,z., z., Z	.,
 ., 
., 
., 
., j., :., ":., #J., &z., ., :.,  :!.,"  #.,$ %.,& j'.,( J).,* :+.,, J-.,. &z/.,0 #J1.,2 z3.,6 z7.,8 *9.,: *;.,< J=.,@ JA.,B  C.,D -jE.,H gI.,J GK.,N gO.,P $"#%[., $sCx. .bs   B.3 .3 .4 .A6$5 A6}  '3<<>     's   Y" "AZ10Z1