
    ju!                     >   d dl Z d dlZd dlZd dlZd dlmZmZmZmZm	Z	 d dl
mZmZmZ ddlmZmZ ddlmZ 	 d dlZ ej(                         5   ej*                  de       d d	lmZ ddd       d
ZddgZddgZ G d de      Zy# 1 sw Y   xY w# e$ r dZY )w xY w)    N)AnyBinaryIODictListUnion)parse_qsurlparseunquote   )DocumentConverterDocumentConverterResult)
StreamInfoignore)category)YouTubeTranscriptApiTFz	text/htmlzapplication/xhtmlz.htmlz.htmc                       e Zd ZdZdedededefdZdededede	fdZ
	 dd	eeef   d
ee   deedf   deedf   fdZdededeedf   fdZddZy)YouTubeConverterzSHandle YouTube specially, focusing on the video title, description, and transcript.file_streamstream_infokwargsreturnc                 h   |j                   xs d}|j                  xs dj                         }|j                  xs dj                         }t	        |      }|j                  dd      j                  dd      }|j                  d      sy|t        v ryt        D ]  }|j                  |      s y y)	zK
        Make sure we're dealing with HTML content *from* YouTube.
         z\??z\==zhttps://www.youtube.com/watch?FT)	urlmimetypelower	extensionr
   replace
startswithACCEPTED_FILE_EXTENSIONSACCEPTED_MIME_TYPE_PREFIXES)selfr   r   r   r   r   r   prefixs           i/root/aria/tools/markitdown-venv/lib/python3.12/site-packages/markitdown/converters/_youtube_converter.pyacceptszYouTubeConverter.accepts(   s     oo#((.B557 **0b779	clkk%%--eS9~~>?001 	F""6*	
     c                     |j                   dn|j                   }t        j                  |d|      }i }|j                  r/|j                  j                  r|j                  j                  |d<    |dg      D ]s  }t        |t        j                        s|j                  D ]G  }|dv st        |j                  |d            }	t        |j                  dd            }
|	r|
r|
||	<    s u 	  |d	g      D ]  }t        |t        j                        s|j                  s+|j                  }
d
|
v s<t        j                  d|
      }|rft        j                  |j                  d            }| j                  |d      }|r.t        |t               rt        |j                  dd            |d<    n d}| j'                  |g d      }t        |t              sJ |r	|d| dz  }d}| j'                  |dg      }|r	|d| dz  }| j'                  |dg      }|r	|d| dz  }| j'                  |dg      }|r	|d| dz  }t)        |      dkD  r	|d| dz  }| j'                  |ddg      }|r	|d| dz  }t*        rt-                d}t/        |j0                        }t3        |j4                        }d|v r|d   d   rt        |d   d          j7                        }dg}|D ]  }|j9                  |j:                          n 	 |j                  d |      | j=                   fd!d"d#$      }|r)d%j?                  |D cg c]  }|j@                   c}      }|r	|d'| dz  }|r|n#|j                  r|j                  j                  nd}t        |t              sJ tI        ||(      S # t"        $ r}t%        d|        Y d }~d }~ww xY wc c}w # t"        $ r}t)        |      dk(  rt%        d&|        n`|jC                  |      jE                  d         jG                         }d%j?                  |D cg c]  }|j@                   nc c}w c}      }Y d }~d }~ww xY w))Nzutf-8zhtml.parser)from_encodingtitlemeta)itemproppropertynamer   contentscriptytInitialDatazvar ytInitialData = ({.*?});   attributedDescriptionBodyTextdescriptionzError extracting description: z
# YouTube
)r+   zog:titler/   z
## 
interactionCountz- **Views:** keywordsz- **Keywords:** durationz- **Runtime:** r   z
### Video Metadata
zog:descriptionz
### Description
venyoutube_transcript_languagesc                  *    j                         S )N)	languages)fetch)video_idr<   ytt_apis   r&   <lambda>z*YouTubeConverter.convert.<locals>.<lambda>   s    $0L !. ! r(      r   )retriesdelay zError fetching transcript: z
### Transcript
)markdownr+   )%charsetbs4BeautifulSoupr+   string
isinstanceTagattrsstrgetresearchjsonloadsgroup_findKeydict	Exceptionprint_getlenIS_YOUTUBE_TRANSCRIPT_CAPABLEr   r	   r   r   querylistappendlanguage_code_retry_operationjointextfind_transcript	translater?   r   )!r$   r   r   r   encodingsoupmetadatar,   akeyr0   r1   matchdataattrdescewebpage_textr+   statsviewsr8   runtimer5   transcript_text
parsed_urlparamstranscript_listr>   
transcriptpartr@   r<   rA   s!                                 @@@r&   convertzYouTubeConverter.convertF   s    *1197{?R?R  m8T $&::$**++ $

 1 1HW&N 
	DdCGG,ZZ 88dhhq"o.C!$((9b"9:Gw(/	
		z* !&#''2}} --"g-II&EwOE#zz%++a.9#'==7V#W#
8T(B69(,,yRT:U6VH]3& %		($AB%%%%eE7"--L		(%7$89}UG2..E99X
|4'z44E))Hzl3wir22Eu:>4UG2>>Lii=:J*KL1+bAAL(*,G O!+//2Jj../Ff}Qvc{1~."),,x"8!F	"1 J$$Z%=%=>W39::6	40 "&!6!6 !" "7 "J "*-((3=>4TYY>+ "4_4ER HH$**TZZ%6%6"%%%%&!
 	
a  	21#67	x ?  W9~* ;A3?@ ,;;IF&Y'CA'FG"UW #
 +.((*3U$DII3U3U*VWsW   3AN7 8BN7 <O! OO! 7	O OOO! !	Q2*AQ-Q
Q--Q2Nrh   keysdefaultc                 *    |D ]  }||v s||   c S  |S )z<Get first non-empty value from metadata matching given keys. )r$   rh   rz   r{   ks        r&   rZ   zYouTubeConverter._get   s+      	#AH}{"	# r(   rS   rj   c                     t        |t              r |D ]  }| j                  ||      }||c S  yt        |t              r= |j                         D ])  \  }}||k(  r||   c S | j                  ||      x}s'|c S  y)zBRecursively search for a key in nested dictionary/list structures.N)rL   r^   rV   rW   items)r$   rS   rj   elmretr~   r:   results           r&   rV   zYouTubeConverter._findKey   s    dD! mmC-?J  d#"

 "187N!]]1c2262!M	"
 r(   c                     d}||k  r	  |       S t        d| d      # t         $ r@}t        d|dz    d|        ||dz
  k  rt        j                  |       |dz  }Y d}~nd}~ww xY w||k  rib)z"Retries the operation if it fails.r   zAttempt r3   z	 failed: NzOperation failed after z
 attempts.)rX   rY   timesleep)r$   	operationrD   rE   attemptrn   s         r&   ra   z!YouTubeConverter._retry_operation   s     {" 1'*EFF  1Yqc:;Wq[(JJu%1	 s    	A(6A##A()N)rC   r   )__name__
__module____qualname____doc__r   r   r   boolr'   r   ry   r   rO   r   r   rZ   rV   ra   r}   r(   r&   r   r   %   s    ]   	
 
<

  
 	

 
!
J %)	
sCx.
 3i
 sDy!	

 
sDy	
S s uS$Y/? Gr(   r   )rS   r   rQ   rI   typingr   r   r   r   r   urllib.parser   r	   r
   _base_converterr   r   _stream_infor   warningscatch_warningsfilterwarningsSyntaxWarningyoutube_transcript_apir   r\   ModuleNotFoundErrorr#   r"   r   r}   r(   r&   <module>r      s      	 
 3 3 4 4 H %*	 	 	 	" @=A?@
 %)!   
 IG( IG+@ @  *$)!*s)   B B(
B BB BB