Ë
    ¢…j<�  ã                   óÜ   — d Z ddlZddlZddlZddlZddlZddlmZ ddlm	Z	m
Z
mZmZmZmZmZmZ ddlZddlmZ ddlZddlmZ ddlmZmZmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z% dZ& G d„ d	«      Z'y)
z�Magika (the Python library).

This module provides the `Magika` class, the main entry point for using Magika
to identify file content types.
é    N)ÚPath)ÚBinaryIOÚDictÚListÚOptionalÚSequenceÚSetÚTupleÚUnion)Ú
get_logger)ÚContentTypeInfoÚContentTypeLabelÚMagikaErrorÚMagikaPredictionÚMagikaResultÚModelConfigÚModelFeaturesÚModelOutputÚOverwriteReasonÚPredictionModeÚSeekableÚStatusÚstandard_v3_3c                   óò  — e Zd ZdZdej
                  ddddfdee   dedededed	ed
dfd„Z	d
e
fd„Zd
e
fd„Zd
e
fd„Zd
e
fd„Zdee
ej"                  f   d
efd„Zdeee
ej"                  f      d
ee   fd„Zded
efd„Zded
efd„Zd
ee   fd„Zd
ee   fd„Zed
e
fd„«       Zeded
e ee!f   fd„«       Z"eded
e#fd„«       Z$d
e%jL                  fd„Z'd ed
e!fd!„Z(dee   d
ee   fd"„Z)ded
efd#„Z*d$e+d
efd%„Z,ed$e+d&e-d'e-d(e-d)e-d*e-d+ed
e.fd,„«       Z/ed-ed&e-d)e-d
ee-   fd.„«       Z0ed/ed(e-d)e-d
ee-   fd0„«       Z1d1ee2ee.f      d
ee2ee3f      fd2„Z4d1ee2ee.f      d
e e
ef   fd3„Z5d4ed5e6d
e2ee7f   fd6„Z8e7jr                  fded4ed7ed5e6d8e7d
efd9„Z:ded
e2ee   ee.   f   fd:„Z; ed;«      fd$e+ded
e2ee   ee.   f   fd<„Z< ed;«      fdeded
efd=„Z=ded
efd>„Z>d?ee2ee.f      d
e?j€                  fd@„ZAy)AÚMagikaz—Main Magika class for content type identification.

    This class provides methods to identify the content type of files, bytes,
    and streams.
    NFÚ	model_dirÚprediction_modeÚno_dereferenceÚverboseÚdebugÚ
use_colorsÚreturnc                 ó†  — t        |¬«      | _        |r)| j                  j                  t        j                  «       |r)| j                  j                  t        j
                  «       |�|| _        n2t        t        «      j                  dz  | j                  «       z  | _        | j                  dz  | _        | j                  dz  | _        | j                  j                  «       s!t        dt        | j                  «      › �«      ‚| j                  j!                  «       s!t        dt        | j                  «      › �«      ‚| j                  j!                  «       s!t        dt        | j                  «      › �«      ‚t"        j%                  | j                  «      | _        t)        j*                  t-        t/        t        | j&                  j0                  «      «      «      | _        || _        || _        t        t        «      j                  d	z  d
z  }t"        j9                  |«      | _        | j=                  «       | _        y)ai  Initializes the Magika instance.

        Args:
            model_dir: Path to the directory containing the model and its
                configuration. If None, the default model is used.
            prediction_mode: The prediction mode to use.  Defaults to
                PredictionMode.HIGH_CONFIDENCE.
            no_dereference: If True, do not follow symlinks.  Defaults to False.
            verbose: If True, enable verbose logging. Defaults to False.
            debug: If True, enable debug logging. Defaults to False.
            use_colors: If True, use colors in the logger.  Defaults to False.
        )r!   NÚmodelsz
model.onnxzconfig.min.jsonzmodel dir not found at zmodel not found at zmodel config not found at Úconfigzcontent_types_kb.min.json) r   Ú_logÚsetLevelÚloggingÚINFOÚDEBUGÚ
_model_dirr   Ú__file__ÚparentÚ_get_default_model_nameÚ_model_pathÚ_model_config_pathÚis_dirr   ÚstrÚis_filer   Ú_load_model_configÚ_model_configÚnpÚarrayÚlistÚmapÚtarget_labels_spaceÚ_target_labels_space_npÚ_prediction_modeÚ_no_dereferenceÚ_load_content_types_kbÚ
_cts_infosÚ_init_onnx_sessionÚ_onnx_session)Úselfr   r   r   r   r    r!   Úcontent_types_kb_paths           úN/root/aria/tools/markitdown-venv/lib/python3.12/site-packages/magika/magika.pyÚ__init__zMagika.__init__;   sÓ  € ô* ¨*Ô5ˆŒ	áØ�I‰I×ÑœwŸ|™|Ô,áØ�I‰I×ÑœwŸ}™}Ô-àÐ Ø'ˆD�Oô ”X“×%Ñ%¨Ñ0°4×3OÑ3OÓ3QÑQð ŒOð  Ÿ?™?¨\Ñ9ˆÔØ"&§/¡/Ð4EÑ"EˆÔà�‰×%Ñ%Ô'ÜÐ 7¼¸D¿O¹OÓ8LÐ7MÐNÓOÐOØ×Ñ×'Ñ'Ô)ÜÐ 3´C¸×8HÑ8HÓ4IÐ3JÐKÓLÐLØ×&Ñ&×.Ñ.Ô0ÜØ,¬S°×1HÑ1HÓ-IÐ,JÐKóð ô +1×*CÑ*CØ×#Ñ#ó+
ˆÔô (*§x¡xÜ””S˜$×,Ñ,×@Ñ@ÓAÓBó(
ˆÔ$ð !0ˆÔà-ˆÔô ”‹N×!Ñ! HÑ,Ð/JÑJð 	ô !×7Ñ7Ð8MÓNˆŒà!×4Ñ4Ó6ˆÕó    c                 ó   — t        | «      S ©N)r2   ©rB   s    rD   Ú__repr__zMagika.__repr__   s   € Ü�4‹yÐrF   c                 óL   — d| j                  «       › d| j                  «       › d�S )NzMagika(module_version="z", model_name="z"))Úget_module_versionÚget_model_namerI   s    rD   Ú__str__zMagika.__str__‚   s-   € Ø(¨×)@Ñ)@Ó)BÐ(CÀ?ÐSW×SfÑSfÓShÐRiÐikÐlÐlrF   c                 óR   — t        t        | j                  «      j                  «      S )z-Gets the version of the Magika Python module.)r2   Ú
__import__Ú
__module__Ú__version__rI   s    rD   rL   zMagika.get_module_version…   s   € ä”:˜dŸo™oÓ.×:Ñ:Ó;Ð;rF   c                 ó.   — | j                   j                  S )z"Gets the name of the loaded model.)r+   ÚnamerI   s    rD   rM   zMagika.get_model_name‰   s   € à�‰×#Ñ#Ð#rF   Úpathc                 ó®   — t        |t        «      st        |t        j                  «      rt	        |«      }nt        d|› d�«      ‚| j                  |«      S )z3Identify the content type of a file given its path.zPath 'úD' is invalid: input path should be of type `Union[str, os.PathLike]`)Ú
isinstancer2   ÚosÚPathLiker   Ú	TypeErrorÚ_get_result_from_path©rB   rU   s     rD   Úidentify_pathzMagika.identify_path�   sO   € ä�dœCÔ ¤J¨t´R·[±[Ô$AÜ˜“:‰DäØ˜˜ÐbÐcóð ð ×)Ñ)¨$Ó/Ð/rF   Úpathsc                 ó  — t        |t        «      st        d«      ‚g }|D ]U  }t        |t        «      st        |t        j
                  «      r|j                  t        |«      «       ŒHt        d|› d�«      ‚ | j                  |«      S )z?Identify the content type of a list of files given their paths.z,Input paths should be of type Sequence[Path]zInput 'rW   )	rX   r   r[   r2   rY   rZ   Úappendr   Ú_get_results_from_paths)rB   r_   Úpaths_rU   s       rD   Úidentify_pathszMagika.identify_paths˜   s‚   € ô ˜%¤Ô*ÜÐJÓKÐKàˆØò 	ˆDÜ˜$¤Ô$¬
°4¼¿¹Ô(EØ—‘œd 4›jÕ)äØ˜d˜VÐ#gÐhóð ð		ð ×+Ñ+¨FÓ3Ð3rF   Úcontentc                 ó¬   — t        |t        «      st        dt        |«      › d�«      ‚| j	                  t        t        j                  |«      «      «      S )z'Identify the content type of raw bytes.z-Input content should be of type 'bytes', not ú.)rX   Úbytesr[   ÚtypeÚ_get_result_from_seekabler   ÚioÚBytesIO)rB   re   s     rD   Úidentify_byteszMagika.identify_bytesª   sJ   € ä˜'¤5Ô)ÜØ?ÄÀWÃ¸ÈaÐPóð ð ×-Ñ-¬h´r·z±zÀ'Ó7JÓ.KÓLÐLrF   Ústreamc                 ó  — t        |t        j                  «      r|j                  «       st	        d«      ‚t        |t        j
                  «      rt	        d«      ‚t        |t        j                  «      st	        d«      ‚t        |d«      rt        |d«      rt        |d«      st	        d«      ‚	 |j                  «       }| j                  t        |«      «      }|j                  |«       |S # |j                  «       w xY w)aR  Identify the content type of a BinaryIO stream.

        Identifies the content type from an already-open binary file-like object
        (e.g., the output of `open(file_path, 'rb')`). Note: 1) Magika will
        `seek()` around the stream; 2) the stream _is not closed_ (closing it is
        the responsibility of the caller).
        z0Input stream must be a readable BinaryIO object.z<Input stream must be opened in bytes mode, not in text mode.ÚseekÚreadÚtellz4Input stream must have seek, read, and tell methods.)rX   rk   ÚIOBaseÚreadabler[   Ú
TextIOBaseÚBufferedIOBaseÚhasattrrr   rj   r   rp   )rB   rn   Úcurrent_positionÚresults       rD   Úidentify_streamzMagika.identify_stream³   sÜ   € ô ˜&¤"§)¡)Ô,°F·O±OÔ4EÜÐNÓOÐOô �fœbŸm™mÔ,ÜØNóð ô ˜&¤"×"3Ñ"3Ô4ÜÐNÓOÐOô ˜ Ô'Ü˜6 6Ô*Ü˜6 6Ô*äÐRÓSÐSð	*Ø%Ÿ{™{›}ÐØ×3Ñ3´H¸VÓ4DÓEˆFð �K‰KÐ(Ô)Øˆøð �K‰KÐ(Õ)ús   Â0*C- Ã-D c                 ó^  — | j                   j                  }| j                   j                  }t        j                  t        j
                  t        j                  t        j                  t        j                  h}|D ]%  }|j                  ||«      }|j                  |«       Œ' t        |«      S )a®  This method returns the list of all possible output content types.

        I.e., all possible values for `MagikaResult.prediction.output.label`.
        This considers the list of possible outputs from the model itself, but
        also keeps into account additional configuration such as `override_map`
        and special content types such as `empty` or `symlink`.

        Consult the documentation for more details.
        )r5   r:   Úoverwrite_mapr   Ú	DIRECTORYÚEMPTYÚSYMLINKÚTXTÚUNKNOWNÚgetÚaddÚsorted)rB   r:   r|   Úoutput_content_typesÚctÚ	output_cts         rD   Úget_output_content_typeszMagika.get_output_content_types×   sŸ   € ð #×0Ñ0×DÑDÐØ×*Ñ*×8Ñ8ˆô ×&Ñ&Ü×"Ñ"Ü×$Ñ$Ü× Ñ Ü×$Ñ$ð7
Ðð &ò 	0ˆBð &×)Ñ)¨"¨bÓ1ˆIØ ×$Ñ$ YÕ/ð		0ô Ð*Ó+Ð+rF   c                 ó„   — t         j                  h}|j                  | j                  j                  «       t        |«      S )aD  This method returns the list of all possible output of the model.

        I.e., all possible values for `MagikaResult.prediction.dl.label`. Note
        that, in general, the list of "model outputs" is different than the
        "tool outputs" as in some cases the model is not even used, or the
        model's output is overwritten due to a low-confidence score, or other
        reasons. This API is useful mostly for debugging purposes; the vast
        majority of client should use `get_output_content_types()`.

        Consult the documentation for more details.
        )r   Ú	UNDEFINEDÚupdater5   r:   r„   )rB   Úmodel_content_typess     rD   Úget_model_content_typeszMagika.get_model_content_typesó   s=   € ô ×&Ñ&ð6
Ðð 	×"Ñ" 4×#5Ñ#5×#IÑ#IÔJÜÐ)Ó*Ð*rF   c                  ó   — t         S )z±Returns the default model name.

        This method is static so that it can be used by external clients/tests
        without the need to instantiate a Magika object.
        )Ú_DEFAULT_MODEL_NAME© rF   rD   r.   zMagika._get_default_model_name  s
   € ô #Ð"rF   Úcontent_types_kb_json_pathc           	      ó>  — d}d}d}i }t        j                  | j                  «       «      j                  «       D ]`  \  }}|d   }|r|}n|}|d   €|n|d   }	|d   €|n|d   }
|d   €|n|d   }|d   }t	        t        |«      |	|
|||¬	«      |t        |«      <   Œb |S )
Nz
text/plainzapplication/octet-streamÚunknownÚis_textÚ	mime_typeÚgroupÚdescriptionÚ
extensions)Úlabelr•   r–   r—   r˜   r”   )ÚjsonÚloadsÚ	read_textÚitemsr   r   )r‘   ÚTXT_MIME_TYPEÚUNKNOWN_MIME_TYPEÚUNKNOWN_GROUPÚoutÚct_nameÚct_infor”   Údefault_mime_typer•   r–   r—   r˜   s                rD   r>   zMagika._load_content_types_kb  só   € ð %ˆØ6ÐØ!ˆàˆÜ $§
¡
Ø&×0Ñ0Ó2ó!
ç
‰%‹'ò	ÑˆG�Wð ˜iÑ(ˆGÙØ$1Ñ!à$5Ð!ð ˜;Ñ'Ð/ñ "à˜[Ñ)ð ð
 &-¨WÑ%5Ð%=‘MÀ7È7ÑCSˆEà" =Ñ1Ð9‘¸wÀ}Ñ?Uð ð ! Ñ.ˆJÜ-<Ü& wÓ/Ø#ØØ'Ø%Øô.ˆCÔ  Ó)Ò*ð%	ð4 ˆ
rF   Úmodel_config_pathc                 óÄ  — t        j                  | j                  «       «      }t        |d   |d   |d   |d   |d   |d   |d   |d   |d	   D �cg c]  }t	        |«      ‘Œ c}|d
   j                  «       D ��ci c]  \  }}t	        |«      |“Œ c}}|d   j                  «       D ��ci c]  \  }}t	        |«      t	        |«      “Œ c}}¬«      S c c}w c c}}w c c}}w )NÚbeg_sizeÚmid_sizeÚend_sizeÚuse_inputs_at_offsetsÚmedium_confidence_thresholdÚmin_file_size_for_dlÚpadding_tokenÚ
block_sizer:   Ú
thresholdsr|   )r§   r¨   r©   rª   r«   r¬   r­   r®   r:   r¯   r|   )rš   r›   rœ   r   r   r�   )r¥   r%   Úct_strÚkÚvs        rD   r4   zMagika._load_model_config3  s  € ä—‘Ð-×7Ñ7Ó9Ó:ˆäØ˜JÑ'Ø˜JÑ'Ø˜JÑ'Ø"(Ð)@Ñ"AØ(.Ð/LÑ(MØ!'Ð(>Ñ!?Ø  Ñ1Ø˜lÑ+à7=Ð>SÑ7Tö!Ø-3Ô  Õ(ò!ð 4:¸,Ñ3G×3MÑ3MÓ3O÷Ù+/¨1¨aÔ  Ó# QÑ&óð
 # ?Ñ3×9Ñ9Ó;÷á�A�qô ! Ó#Ô%5°aÓ%8Ñ8óô
ð 	
ùò!ùóùós   ÁCÁ:CÂ)Cc                 ó(  — t        j                   «       }t        j                  «        t        j                  | j                  dg¬«      }dt        j                   «       |z
  z  }| j
                  j                  d| j                  › d|d›d�«       |S )NÚCPUExecutionProvider)Ú	providerséè  zONNX DL model "z" loaded in ú.03fú ms)ÚtimeÚrtÚdisable_telemetry_eventsÚInferenceSessionr/   r&   r    )rB   Ú
start_timeÚonnx_sessionÚelapsed_times       rD   r@   zMagika._init_onnx_sessionL  sƒ   € Ü—Y‘Y“[ˆ
Ü
×#Ñ#Ô%ä×*Ñ*Ø×ÑØ-Ð.ô
ˆð œtŸy™y›{¨ZÑ7Ñ8ˆØ�	‰	�‰Ø˜d×.Ñ.Ð/¨|¸LÈÐ;NÈcÐRô	
ð ÐrF   Úcontent_typec                 ó    — | j                   |   S rH   )r?   )rB   rÀ   s     rD   Ú_get_ct_infozMagika._get_ct_infoZ  s   € Ø�‰˜|Ñ,Ð,rF   c                 ó$  — i }g }| j                   j                  dt        |«      › d�«       t        j                  «       }|D ]>  }| j	                  |«      \  }}|�||t        |«      <   Œ(|€J ‚|j                  ||f«       Œ@ dt        j                  «       |z
  z  }| j                   j                  d|d›d�«       | j                  |«      j                  «       D ]
  \  }	}
|
||	<   Œ g }|D ]  }|j                  |t        |«         «       Œ! |S )ao  Get results for a list of paths.

        Given a list of paths, returns a list of MagikaResult objects, which
        contain relevant information, such as: file path, the output of the DL
        model, the confidence score, the output of the tool, and associated
        metadata. The order of the predictions matches the order of the input
        paths.
        z3Processing input files and extracting features for z samplesr¶   z%First pass and features extracted in r·   r¸   )	r&   r    Úlenr¹   Ú!_get_result_or_features_from_pathr2   ra   Ú_get_results_from_featuresr�   )rB   r_   Úall_outputsÚall_featuresr½   rU   ÚoutputÚfeaturesr¿   Úpath_strry   Úsorted_outputss               rD   rb   zMagika._get_results_from_paths]  s0  € ð 02ˆð :<ˆà�	‰	�‰ØAÄ#ÀeÃ*ÀÈXÐVô	
ô —Y‘Y“[ˆ
Øò 	6ˆDØ#×EÑEÀdÓKÑˆF�HØÐ!Ø)/�œC ›IÒ&àÐ+Ð+Ð+Ø×#Ñ# T¨8Ð$4Õ5ð	6ð œtŸy™y›{¨ZÑ7Ñ8ˆØ�	‰	�‰Ð?ÀÈTÐ?RÐRUÐVÔWð !%× ?Ñ ?ÀÓ M× SÑ SÓ Uò 	+ÑˆH�fØ$*ˆK˜Ò!ð	+ð
 ˆØò 	:ˆDØ×!Ñ! +¬c°$«iÑ"8Õ9ð	:àÐrF   c                 ó,   — | j                  |g«      d   S )Nr   )rb   r]   s     rD   r\   zMagika._get_result_from_path‹  s   € Ø×+Ñ+¨T¨FÓ3°AÑ6Ð6rF   Úseekablec                 óz   — | j                  |«      \  }}|�|S |€J ‚| j                  t        d«      |fg«      d   S )Nú-)Ú%_get_result_or_features_from_seekablerÆ   r   )rB   rÎ   ry   rÊ   s       rD   rj   z Magika._get_result_from_seekableŽ  sP   € Ø×EÑEÀhÓOÑˆ�ØÐØˆMØÐ#Ð#Ð#Ø×.Ñ.´°c³¸HÐ0EÐ/FÓGÈÑLÐLrF   r§   r¨   r©   r­   r®   rª   c           	      ó¢  — ||k  sJ ‚|dk(  sJ ‚||k  sJ ‚|rJ ‚t        || j                  «      }|dkD  r:| j                  d|«      }|j                  «       }t        j                  |||«      }	ng }	|dkD  rG| j                  | j                  |z
  |«      }
|
j                  «       }
t        j                  |
||«      }ng }t        |	g |g g g g ¬«      S )a¦  Extract features from an input seekable.

        This implements features extraction v2 from a seekable, which is an
        abstraction about anything that has a size and that can be "read_at" a
        specific offset, such as a file or a buffer. This is implemented so that
        we do not need to load the entire file in memory or scan the entire
        buffer.

        High-level overview on what we do:
        - We read (at most) `block_size` bytes from the beginning and from the
        end.
        - We normalize these bytes by stripping whitespaces.
        - We consider `beg_size` and `end_size` bytes as `beg` and `end`
        features. If we don't have enough bytes, we use `padding_token` as
        padding.

        See comments below for the specifics and handling of corner cases.

        NOTE: This implementation does not support extraction of `mid` features
        and `use_inputs_at_offsets`.
        r   )ÚbegÚmidÚendÚoffset_0x8000_0x8007Úoffset_0x8800_0x8807Úoffset_0x9000_0x9007Úoffset_0x9800_0x9807)	ÚminÚsizeÚread_atÚlstripr   Ú_get_beg_ints_with_paddingÚrstripÚ_get_end_ints_with_paddingr   )rÎ   r§   r¨   r©   r­   r®   rª   Úbytes_num_to_readÚbeg_contentÚbeg_intsÚend_contentÚend_intss               rD   Ú_extract_features_from_seekablez&Magika._extract_features_from_seekable•  s	  € ð> ˜*Ò$Ð$Ð$Ø˜1Š}Ðˆ}Ø˜*Ò$Ð$Ð$Ù(Ð(Ð(ô   
¨H¯M©MÓ:Ðà�aŠ<ð
 #×*Ñ*¨1Ð.?Ó@ˆKØ%×,Ñ,Ó.ˆKÜ×8Ñ8Ø˜X }ó‰Hð ˆHà�aŠ<ð #×*Ñ*Ø—‘Ð 1Ñ1Ð3DóˆKð &×,Ñ,Ó.ˆKÜ×8Ñ8Ø˜X }ó‰Hð ˆHäØØØØ!#Ø!#Ø!#Ø!#ô
ð 	
rF   râ   c                 óÄ   — |t        | «      k  r| d| } t        t        t        | «      «      }t        |«      |k  r||g|t        |«      z
  z  z   }t        |«      |k(  sJ ‚|S )a  Take an (already-stripped) buffer as input and extract beg ints.

        This returns a list of integers whose length is exactly beg_size. If
        the buffer is bigger than required, take only the initial portion. If
        the buffer is shorter, add padding at the end.
        r   ©rÄ   r8   r9   Úint)râ   r§   r­   rã   s       rD   rÞ   z!Magika._get_beg_ints_with_paddingã  sk   € ð ”c˜+Ó&Ò&à% a¨Ð1ˆKäœœC Ó-Ó.ˆäˆx‹=˜8Ò#à M ?°hÄÀXÃÑ6NÑ#OÑPˆHä�8‹} Ò(Ð(Ð(àˆrF   rä   c                 óî   — |t        | «      k  r| t        | «      |z
  t        | «       } t        t        t        | «      «      }t        |«      |k  r|g|t        |«      z
  z  |z   }t        |«      |k(  sJ ‚|S )a  Take an (already-stripped) buffer as input and extract end ints.

        This returns a list of integers whose length is exactly end_size. If the
        buffer is bigger than required, take only the last portion. If the
        buffer is shorter, add padding at the beginning.
        rè   )rä   r©   r­   rå   s       rD   rà   z!Magika._get_end_ints_with_paddingû  sz   € ð ”c˜+Ó&Ò&à%¤c¨+Ó&6¸Ñ&AÄCÈÓDTÐUˆKäœœC Ó-Ó.ˆäˆx‹=˜8Ò#à&˜¨8´c¸(³mÑ+CÑDÈÑPˆHä�8‹} Ò(Ð(Ð(àˆrF   rÈ   c                 ó<  — | j                  |«      }t        j                  |d¬«      }| j                  |   }t        j                  |d¬«      }t        |||«      D ����	cg c]*  \  \  }}}}	|t        t        |«      t        |	«      ¬«      f‘Œ, c}	}}}S c c}	}}}w )Né   )Úaxis)r™   Úscore)	Ú_get_raw_predictionsr6   Úargmaxr;   ÚmaxÚzipr   r   Úfloat)
rB   rÈ   Ú	raw_predsÚtop_preds_idxsÚpreds_content_types_labelsÚscoresrU   Ú_r™   rî   s
             rD   Ú _get_model_outputs_from_featuresz'Magika._get_model_outputs_from_features  s™   € ð ×-Ñ-¨lÓ;ˆ	ÜŸ™ 9°1Ô5ˆØ%)×%AÑ%AÀ.Ñ%QÐ"Ü—‘˜	¨Ô*ˆô ,/ØÐ8¸&ó,÷
ñ 
á'‘	��q˜5 %ð ”;Ô%5°eÓ%<ÄEÈ%ÃLÔQÒRõ
ð 	
ùõ 
s   Á!/B
c                 ó  — t        |«      dk(  ri S i }| j                  |«      D ]d  \  }}| j                  |j                  |j                  «      \  }}| j                  ||j                  ||j                  |¬«      |t        |«      <   Œf |S )Nr   )rU   Údl_labelÚoutput_labelrî   Úoverwrite_reason)rÄ   rù   Ú)_get_output_label_from_dl_label_and_scorer™   rî   Ú!_get_result_from_labels_and_scorer2   )rB   rÈ   ÚresultsrU   Úmodel_outputrü   rý   s          rD   rÆ   z!Magika._get_results_from_features"  s¨   € ô
 ˆ|Ó Ò!àˆIà+-ˆà"&×"GÑ"GÈÓ"Uò 	ÑˆD�,ð ×>Ñ>Ø ×&Ñ&¨×(:Ñ(:óñ +ˆLÐ*ð "&×!GÑ!GØØ%×+Ñ+Ø)Ø"×(Ñ(Ø!1ð "Hó "ˆG”C˜“IÒð	ð* ˆrF   rû   rî   c                 óÚ  — t         j                  }| j                  j                  j	                  ||«      }||k7  rt         j
                  }| j                  t        j                  k(  r	 ||fS | j                  t        j                  k(  rB|| j                  j                  j	                  || j                  j                  «      k\  r	 ||fS | j                  t        j                  k(  r|| j                  j                  k\  r	 ||fS t         j                  }| j                  |«      j                  rt         j"                  }nt         j$                  }||k(  rt         j                  }||fS rH   )r   ÚNONEr5   r|   r‚   ÚOVERWRITE_MAPr<   r   Ú
BEST_GUESSÚHIGH_CONFIDENCEr¯   r«   ÚMEDIUM_CONFIDENCEÚLOW_CONFIDENCErÂ   r”   r   r€   r�   )rB   rû   rî   rý   rü   s        rD   rþ   z0Magika._get_output_label_from_dl_label_and_scoreD  sb  € ô +×/Ñ/Ðð ×)Ñ)×7Ñ7×;Ñ;¸HÀhÓOˆØ˜8Ò#Ü.×<Ñ<Ðð × Ñ ¤N×$=Ñ$=Ò=ð ðL Ð-Ð-Ð-ðI ×!Ñ!¤^×%CÑ%CÒCØØ×!Ñ!×,Ñ,×0Ñ0Ø˜$×,Ñ,×HÑHóòð ð6 Ð-Ð-Ð-ð3 ×!Ñ!¤^×%EÑ%EÒEØ˜×+Ñ+×GÑGÒGð
 ð& Ð-Ð-Ð-ô  /×=Ñ=ÐØ× Ñ  Ó.×6Ò6Ü/×3Ñ3‘ä/×7Ñ7�Ø˜<Ò'ô
 $3×#7Ñ#7Ð àÐ-Ð-Ð-rF   rü   rý   c           	      ór   — t        |t        | j                  |«      | j                  |«      ||¬«      ¬«      S )N)ÚdlrÉ   rî   rý   )rU   Ú
prediction)r   r   rÂ   )rB   rU   rû   rü   rî   rý   s         rD   rÿ   z(Magika._get_result_from_labels_and_score~  s>   € ô ØÜ'Ø×$Ñ$ XÓ.Ø×(Ñ(¨Ó6ØØ!1ô	ô
ð 	
rF   c                 ó"  — | j                   rE|j                  «       r5| j                  |t        j                  t        j
                  d¬«      }|dfS |j                  «       st        |t        j                  ¬«      dfS |j                  «       rrt        j                  |t        j                  «      st        |t        j                  ¬«      dfS t        |d«      5 }| j!                  t#        |«      |«      cddd«       S |j%                  «       r5| j                  |t        j                  t        j&                  d¬«      }|dfS | j                  |t        j                  t        j(                  d¬«      }|dfS # 1 sw Y   t+        d«      ‚xY w)a  Given a path, we return either a MagikaOutput or a MagikaFeatures.

        There are some files and corner cases for which we do not need to use
        deep learning to get the output; in these cases, we already return a
        MagikaOutput object.

        For some other files, we do need to use deep learning, in which case we
        return a MagikaFeatures object. Note that for now we just collect the
        features instead of already performing inference because we want to use
        batching.
        ç      ð?©rU   rû   rü   rî   N)rU   ÚstatusÚrbÚunreachable)r=   Ú
is_symlinkrÿ   r   rŠ   r   Úexistsr   r   ÚFILE_NOT_FOUND_ERRORr3   rY   ÚaccessÚR_OKÚPERMISSION_ERRORÚopenrÑ   r   r1   r}   r�   Ú	Exception)rB   rU   ry   rn   s       rD   rÅ   z(Magika._get_result_or_features_from_path�  sl  € ð ×Ò D§O¡OÔ$5Ø×;Ñ;ØÜ)×3Ñ3Ü-×5Ñ5Øð	 <ó ˆFð ˜4�<Ðà�{‰{Œ}Ü T´&×2MÑ2MÔNÐPTÐTÐTà�<‰<Œ>Ü—9‘9˜T¤2§7¡7Ô+Ü#¨´f×6MÑ6MÔNÐPTÐTÐTô
 ˜$ Ó%ð ¨Ø×EÑEÜ  Ó(¨$ó÷ñ ð
 �[‰[Œ]Ø×;Ñ;ØÜ)×3Ñ3Ü-×7Ñ7Øð	 <ó ˆFð ˜4�<Ðð ×;Ñ;ØÜ)×3Ñ3Ü-×5Ñ5Øð	 <ó ˆFð ˜4�<Ð÷+ô. ˜Ó&Ð&ús   ÃE;Å;FrÐ   c           	      ó„  — |j                   dk(  r5| j                  |t        j                  t        j                  d¬«      }|dfS |j                   | j
                  j                  k  r3|j                  d|j                   «      }| j                  ||¬«      }|dfS t        j                  || j
                  j                  | j
                  j                  | j
                  j                  | j
                  j                  | j
                  j                  | j
                  j                   «      }|j"                  | j
                  j                  dz
     | j
                  j                  k(  rSt%        |j                   | j
                  j                  «      }|j                  d|«      }| j                  ||¬«      }|dfS d|fS )ag  Get result or features from a seekable object.

        Given a Seekable object (which is a wrapper of BinaryIO), we return
        either a MagikaOutput or a MagikaFeatures.

        There are some corner cases for which we do not need to use deep
        learning to get the output; in these cases, we return directly a
        MagikaOutput object.

        For all other cases, we do need to use deep learning, in which case we
        return a MagikaFeatures object. Note that for now we just collect the
        features instead of already performing inference because we want to use
        batching.
        r   r  r  N)rU   rì   )rÛ   rÿ   r   rŠ   r~   r5   r¬   rÜ   Ú_get_result_from_few_bytesr   ræ   r§   r¨   r©   r­   r®   rª   rÓ   rÚ   r  )rB   rÎ   rU   ry   re   Úfile_featuresÚbytes_to_reads          rD   rÑ   z,Magika._get_result_or_features_from_seekableÊ  s–  € ð" �=‰=˜AÒØ×;Ñ;ØÜ)×3Ñ3Ü-×3Ñ3Øð	 <ó ˆFð ˜4�<Ðà�]‰]˜T×/Ñ/×DÑDÒDØ×&Ñ& q¨(¯-©-Ó8ˆGØ×4Ñ4°WÀ4Ð4ÓHˆFØ˜4�<Ðô #×BÑBØØ×"Ñ"×+Ñ+Ø×"Ñ"×+Ñ+Ø×"Ñ"×+Ñ+Ø×"Ñ"×0Ñ0Ø×"Ñ"×-Ñ-Ø×"Ñ"×8Ñ8óˆMð ×!Ñ! $×"4Ñ"4×"IÑ"IÈAÑ"MÑNØ×%Ñ%×3Ñ3ò4ô !$ H§M¡M°4×3EÑ3E×3PÑ3PÓ Q�Ø"×*Ñ*¨1¨mÓ<�Ø×8Ñ8¸ÀtÐ8ÓL�Ø˜t�|Ð#ð
 ˜]Ð*Ð*rF   c                 ó¸   — t        |«      d| j                  j                  z  k  sJ ‚| j                  |«      }| j	                  |t
        j                  |d¬«      S )Né   r  r  )rÄ   r5   r®   Ú_get_label_from_few_bytesrÿ   r   rŠ   )rB   re   rU   r™   s       rD   r  z!Magika._get_result_from_few_bytes  sa   € ô �7‹|˜q 4×#5Ñ#5×#@Ñ#@Ñ@Ò@Ð@Ð@Ø×.Ñ.¨wÓ7ˆØ×5Ñ5ØÜ%×/Ñ/ØØð	 6ó 
ð 	
rF   c                 óŠ   — 	 t         j                  }|j                  d«      }|S # t        $ r t         j                  }Y |S w xY w)Nzutf-8)r   r€   ÚdecodeÚUnicodeDecodeErrorr�   )rB   re   r™   rø   s       rD   r   z Magika._get_label_from_few_bytes  sH   € ð	-Ü$×(Ñ(ˆEØ—‘˜wÓ'ˆAð ˆøô "ò 	-Ü$×,Ñ,‰EØˆð	-ús   ‚!% ¥AÁArÊ   c           	      ó  — t        j                   «       }g }|D ]ú  \  }}g }| j                  j                  dkD  r2|j                  |j                  d| j                  j                   «       | j                  j
                  dkD  r2|j                  |j                  d| j                  j
                   «       | j                  j                  dkD  r3|j                  |j                  | j                  j                   d «       |j                  |«       Œü t        j                  |t        j                  ¬«      }dt        j                   «       |z
  z  }| j                  j                  d|d›d�«       g }	|j                  d   }
d}|
|z  }|
|z  dk7  r|dz  }t!        |«      D ]Ç  }| j                  j                  d	|dz   › d
|› �«       ||z  }t#        |dz   |z  |
«      }t        j                   «       }| j$                  j'                  dgd|||…dd…f   i«      d   }dt        j                   «       |z
  z  }| j                  j                  d|d›d�«       |	j                  |«       ŒÉ t        j(                  |	«      S )z£Get raw predictions from features.

        Given a list of (path, features), return a (files_num, features_size)
        matrix encoding the predictions.
        r   N)Údtyper¶   zDL input prepared in r·   r¸   rì   z-Getting raw predictions for (internal) batch ú/Útarget_labelrh   zDL raw prediction in )r¹   r5   r§   ÚextendrÓ   r¨   rÔ   r©   rÕ   ra   r6   r7   Úint32r&   r    ÚshapeÚrangerÚ   rA   ÚrunÚconcatenate)rB   rÊ   r½   ÚX_bytesrø   ÚfsÚsample_bytesÚXr¿   Úraw_predictions_listÚsamples_numÚmax_internal_batch_sizeÚbatches_numÚ	batch_idxÚ	start_idxÚend_idxÚbatch_raw_predictionss                    rD   rï   zMagika._get_raw_predictions  sc  € ô —Y‘Y“[ˆ
ØˆØò 	)‰EˆAˆrØˆLØ×!Ñ!×*Ñ*¨QÒ.Ø×#Ñ# B§F¡FÐ+H¨T×-?Ñ-?×-HÑ-HÐ$IÔJØ×!Ñ!×*Ñ*¨QÒ.Ø×#Ñ# B§F¡FÐ+H¨T×-?Ñ-?×-HÑ-HÐ$IÔJØ×!Ñ!×*Ñ*¨QÒ.Ø×#Ñ# B§F¡F¨D×,>Ñ,>×,GÑ,GÐ+GÐ+IÐ$JÔKØ�N‰N˜<Õ(ð	)ô �H‰H�W¤B§H¡HÔ-ˆØœtŸy™y›{¨ZÑ7Ñ8ˆØ�	‰	�‰Ð/°¸TÐ/BÀ#ÐFÔGà!ÐØ—g‘g˜a‘jˆà"&ÐØ!Ð%<Ñ<ˆØÐ0Ñ0°AÒ5Ø˜1ÑˆKä˜{Ó+ò 	?ˆIØ�I‰I�O‰OØ?À	È!Á¸}ÈAÈkÈ]Ð[ôð "Ð$;Ñ;ˆIÜ˜9 q™=Ð,CÑCÀ[ÓQˆGäŸ™›ˆJØ$(×$6Ñ$6×$:Ñ$:ØÐ  7¨A¨i¸Ð.?ÂÐ.BÑ,CÐ"Dó%àñ%Ð!ð  ¤4§9¡9£;°Ñ#;Ñ<ˆLØ�I‰I�O‰OÐ3°LÀÐ3FÀcÐJÔKà ×'Ñ'Ð(=Õ>ð	?ô �~‰~Ð2Ó3Ð3rF   )BÚ__name__rQ   Ú__qualname__Ú__doc__r   r  r   r   ÚboolrE   r2   rJ   rN   rL   rM   r   rY   rZ   r   r^   r   r   rd   rh   rm   r   rz   r   rˆ   r�   Ústaticmethodr.   r   r   r>   r   r4   rº   r¼   r@   rÂ   rb   r\   r   rj   ré   r   ræ   rÞ   rà   r
   r   rù   rÆ   ró   r   rþ   r  rÿ   rÅ   rÑ   r  r   ÚnptÚNDArrayrï   r�   rF   rD   r   r   4   s•  „ ñð %)Ø*8×*HÑ*HØ$ØØØ ñB7à˜D‘>ðB7ð (ðB7ð ð	B7ð
 ðB7ð ðB7ð ðB7ð 
óB7ðH˜#ó ðm˜ó mð< Có <ð$ ó $ð	0 %¨¨R¯[©[Ð(8Ñ"9ð 	0¸ló 	0ð4Ø˜e C¨¯©Ð$4Ñ5Ñ6ð4à	ˆlÑ	ó4ð$M eð M°ó Mð" hð "°<ó "ðH,¨$Ð/?Ñ*@ó ,ð8+¨Ð.>Ñ)?ó +ð$ ð# Sò #ó ð#ð ð"Ø$(ð"à	Ð Ð/Ñ	0ò"ó ð"ðH ð
¨dð 
°{ò 
ó ð
ð0 B×$7Ñ$7ó ð-Ð)9ð -¸oó -ð,¨T°$©Zð ,¸DÀÑ<Nó ,ð\7¨$ð 7°<ó 7ðM°(ð M¸|ó Mð ðK
ØðK
àðK
ð ðK
ð ð	K
ð
 ðK
ð ðK
ð  $ðK
ð 
òK
ó ðK
ðZ ðØðØ&)ðØ:=ðà	ˆc‰òó ðð. ðØðØ&)ðØ:=ðà	ˆc‰òó ðð.
Ø   t¨]Ð':Ñ!;Ñ<ð
à	ˆe�D˜+Ð%Ñ&Ñ	'ó
ð Ø   t¨]Ð':Ñ!;Ñ<ð à	ˆc�<ÐÑ	 ó ðD8.Ø(ð8.Ø16ð8.à	Ð Ð0Ñ	1ó8.ð@ -<×,@Ñ,@ñ
àð
ð #ð
ð 'ð	
ð
 ð
ð *ð
ð 
ó
ð$8'Øð8'à	ˆx˜Ñ% x°Ñ'>Ð>Ñ	?ó8'ñv 04°C«yñ<'Ø ð<'Ø(,ð<'à	ˆx˜Ñ% x°Ñ'>Ð>Ñ	?ó<'ñ~ ,0°«9ñ

Øð

Ø$(ð

à	ó

ð°ð Ð;Kó ð.4Ø˜U 4¨Ð#6Ñ7Ñ8ð.4à	�‰ô.4rF   r   )(r<  rk   rš   r(   rY   r¹   Úpathlibr   Útypingr   r   r   r   r   r	   r
   r   Únumpyr6   Únumpy.typingr?  Úonnxruntimerº   Úmagika.loggerr   Úmagika.typesr   r   r   r   r   r   r   r   r   r   r   r   r�   r   r�   rF   rD   ú<module>rH     s[   ðñó 
Û Û Û 	Û Ý ß N× NÓ Nã Ý Û å $÷÷ ÷ ó ð &Ð ÷V4ò V4rF   