
    j>P                     `   d dl Z d dlZd dlZd dlmZmZ ddlmZmZ ddl	m
Z
 ddlmZmZ  ej                  d      Zded	efd
ZdZ	 d dlZd dlZd dlZddgZdgZddeee      ded	efdZded	edz  fdZded	eeee         fdZ G d de      Zy# e$ r  e j.                         ZY Xw xY w)    N)BinaryIOAny   )DocumentConverterDocumentConverterResult)
StreamInfo)MissingDependencyExceptionMISSING_DEPENDENCY_MESSAGEz^\.\d+$textreturnc                 J   | j                  d      }g }d}|t        |      k  r||   }|j                         }t        j	                  |      r|dz   }|t        |      k  r:||   j                         s'|dz  }|t        |      k  r||   j                         s'|t        |      k  r/||   j                         }|j                  | d|        |dz   }n-|j                  |       |dz  }n|j                  |       |dz  }|t        |      k  rdj                  |      S )a  
    Post-process extracted text to merge MasterFormat-style partial numbering
    with the following text line.

    MasterFormat documents use partial numbering like:
        .1  The intent of this Request for Proposal...
        .2  Available information relative to...

    Some PDF extractors split these into separate lines:
        .1
        The intent of this Request for Proposal...

    This function merges them back together.
    
r       )splitlenstripPARTIAL_NUMBERING_PATTERNmatchappendjoin)r   linesresult_linesilinestrippedj	next_lines           e/root/aria/tools/markitdown-venv/lib/python3.12/site-packages/markitdown/converters/_pdf_converter.py_merge_partial_numbering_linesr       s    JJtE L	A
c%j.Qx::< %**84AAc%j.q)9Q c%j.q)9 3u:~!!HNN,	##xj)$=>E ##D)Q%FA- c%j.0 99\""    zapplication/pdfzapplication/x-pdf.pdftableinclude_separatorc                 N  	 | sy| D cg c]  }|D cg c]  }||nd
 c} } }}| D cg c]  }t        d |D              s| } }| syt        |  D cg c]  }t        d |D               c}	dt        t           dt        f	fd}|rY| ^}} ||      g}|j                  ddj                  d 	D              z   dz          |D ]  }|j                   ||              n| D cg c]
  } ||       }}d	j                  |      S c c}w c c}}w c c}w c c}w c c}w )
a  Convert a 2D list (rows/columns) into a nicely aligned Markdown table.

    Args:
        table: 2D list of cell values
        include_separator: If True, include header separator row (standard markdown).
                          If False, output simple pipe-separated rows.
     c              3   <   K   | ]  }|j                           y wNr   .0cells     r   	<genexpr>z%_to_markdown_table.<locals>.<genexpr>]   s     (F$(Fs   c              3   D   K   | ]  }t        t        |              y wr(   )r   strr*   s     r   r-   z%_to_markdown_table.<locals>.<genexpr>c   s     5c#d)n5s    rowr   c                 T    ddj                  d t        |       D              z   dz   S )N|c              3   V   K   | ]!  \  }}t        |      j                  |       # y wr(   )r/   ljust)r+   r,   widths      r   r-   z6_to_markdown_table.<locals>.fmt_row.<locals>.<genexpr>h   s!     V+$s4yu-Vs   '))r   zip)r0   
col_widthss    r   fmt_rowz#_to_markdown_table.<locals>.fmt_rowe   s2    hhVS*AUVVW	
r!   r2   c              3   &   K   | ]	  }d |z    yw-N r+   ws     r   r-   z%_to_markdown_table.<locals>.<genexpr>o   s      =Qq =   r   )anyr6   maxlistr/   r   r   )
r#   r$   r0   r,   colr8   headerrowsmdr7   s
            @r   _to_markdown_tablerG   N   s8     LQQC3?4d&dB.?QEQ "GSS(F#(F%FSGEG BEeM##555MJ
T#Y 
3 
 fo
		# =* ===CD 	$CIIgcl#	$ ',,sgcl,,99R=7 @Q H N  -s,   	DDDDDD+D"Dpagec                 &  29:; | j                  ddd      }|syd}i }|D ]3  }t        |d   |z        |z  }||vrg ||<   ||   j                  |       5 t        |j	                               }t        | d      r| j                  nd}g }|D ]  }t        ||   d	 
      }	|	s|	d   d   }
|	d   d   }||
z
  }dj                  d |	D              }|	D cg c]  }|d   	 }}g }t        |      D ]!  }|r||d   z
  dkD  s|j                  |       # ||dz  kD  xr t        |      dkD  }d}|	r-|	d   d   j                         }t        j                  |      rd}|j                  ||	|||t        |      |d        g }|D ]%  }|d   dk\  s|d   r|j                  |d          ' |sy|j                          g }t        t        |      dz
        D ]'  }||dz      ||   z
  }|dkD  s|j                  |       ) |rLt        |      dk\  r>t        |      }t        t        |      dz        }||   }t!        dt#        d|            }nd}g :|D ]!  }:r|:d   z
  |kD  s:j                  |       # t        :      dkD  rb:d   :d   z
  }|t        :      z  }|dk  ryt        :      |dz  z  }|d kD  ryt        d!|dz  z        } t!        d"|       } t        :      | kD  ryy|D ]y  }|d   rd|d#<   |d$   rd|d#<   t%               }!|d%   D ]>  }|d   }"t'        :      D ])  \  }#}$t)        |"|$z
        d&k  s|!j+                  |#        > @ t        |!      d'k\  |d#<   { g }%d}|t        |      k  rf||   d#   rJ|}&|t        |      k  r$||   d#   r|dz  }|t        |      k  r	||   d#   r|}'|%j                  |&|'f       n|dz  }|t        |      k  rft-        d( |%D              }(t        |      dkD  r|(t        |      z  d)k  ryg })t        :      ;d*t.        d+t0        t2           f:;fd,}*d}+|+t        |      k  r||+   }d},|%D ]  \  }-}.|+|-k(  s|-|.f}, n |,r=|,\  }-}.g }/t        |-|.      D ]  }0 |*||0         }1|/j                  |1         |/rt        ;      D 22cg c]  2t!        2fd-|/D               c}299D cg c]  }t!        |d       c}9|/d   }3d.d/j                  9fd0t'        |3      D              z   d1z   }4|)j                  |4       d.d/j                  9fd2t        ;      D              z   d1z   }5|)j                  |5       |/dd D ]<  }6d.d/j                  9fd3t'        |6      D              z   d1z   }7|)j                  |7       > |.}+n9d}8|%D ]  \  }-}.|-|+cxk  r|.k  sn d}8 n |8s|)j                  |d          |+dz  }+|+t        |      k  rd4j                  |)      S c c}w c c}2w c c}w )5a  
    Extract form-style content from a PDF page by analyzing word positions.
    This handles borderless forms/tables where words are aligned in columns.

    Returns markdown with proper table formatting:
    - Tables have pipe-separated columns with header separator rows
    - Non-table content is rendered as plain text

    Returns None if the page doesn't appear to be a form-style document,
    indicating that pdfminer should be used instead for better text spacing.
    T   keep_blank_charsx_tolerancey_toleranceN   topr5   id  c                     | d   S Nx0r<   r>   s    r   <lambda>z2_extract_form_content_from_words.<locals>.<lambda>   s
    1T7 r!   keyr   rS   x1r   c              3   &   K   | ]	  }|d      yw)r   Nr<   r=   s     r   r-   z3_extract_form_content_from_words.<locals>.<genexpr>   s      >q6 >r?   2   g?<   Fr   )y_keywordsr   x_groupsis_paragraphnum_columnshas_partial_numberingra   r`   r_   r   gffffff?   #      H   
         is_table_rowrb   r^   (   r   c              3   ,   K   | ]  \  }}||z
    y wr(   r<   )r+   startends      r   r-   z3_extract_form_content_from_words.<locals>.<genexpr>)  s     G:5#3;Gs   g?infor   c                     t        	      D cg c]  }d }}| d   D ]T  }|d   }	dz
  }t        	dz
        D ]  }|dz      }||dz
  k  s|} n ||   r||xx   d|d   z   z  cc<   M|d   ||<   V |S c c}w )Nr&   r^   rS   r   rh   r   r   )range)
ro   _cellswordword_xassigned_colcol_idxcol_endglobal_columnsnum_colss
           r   extract_cellsz7_extract_form_content_from_words.<locals>.extract_cells2  s    (-h81B88M 	3D$ZF#a<L A. (15GbL(#*L	
 \"l#sT&\'99#&*6ll#	3  9s   	A8c              3   :   K   | ]  }t        |           y wr(   )r   )r+   r0   rC   s     r   r-   z3_extract_form_content_from_words.<locals>.<genexpr>Z  s     <#CH<s   z| z | c              3   L   K   | ]  \  }}|j                  |           y wr(   r4   r+   r   r,   r7   s      r   r-   z3_extract_form_content_from_words.<locals>.<genexpr>c  s'      !6=a

:a=1!   !$z |c              3   .   K   | ]  }d |   z    ywr:   r<   )r+   r   r7   s     r   r-   z3_extract_form_content_from_words.<locals>.<genexpr>m  s      Nz!}!4 Ns   c              3   L   K   | ]  \  }}|j                  |           y wr(   r~   r   s      r   r-   z3_extract_form_content_from_words.<locals>.<genexpr>v  s'      %:A!TDJJz!}5%r   r   )extract_wordsroundr   sortedkeyshasattrr5   r   r   r   r   r   extendsortrq   intrA   minset	enumerateabsaddsumdictrB   r/   )<rH   r^   rN   	rows_by_yrt   r]   sorted_y_keys
page_widthrow_info	row_wordsfirst_x0last_x1
line_widthcombined_textr>   x_positionsr_   xr`   rb   
first_wordall_table_x_positionsro   gapsr   gapsorted_gapspercentile_70_idxadaptive_tolerancecontent_widthavg_col_widthcolumns_per_inchadaptive_max_columnsaligned_columnsru   rw   col_xtable_regions	start_idxend_idxtotal_table_rowsr   r{   idxtable_regionrm   rn   
table_data	table_idxrs   rC   rD   
header_str	separatorr0   row_strin_tabler7   ry   rz   s<                                                     `      @@@r    _extract_form_content_from_wordsr   x   s    !QRSE K)+I &d5kK/0;>	!!Ie%%	& 9>>+,M&tW53J H &
9U+1BC	Q<%B-%x'
 >I >> )221qw22 "$ 	#Aq8B</"4"	#
 "J$55Q#m:Lr:Q !&"1f-335J(..z:(,%"%$ ,"8})>
	
9&
T *, ;!#D,@!((j)9:; !   D3,-12 #AE*-B1-EE7KK D	QTlK 04 78():; !S-?%@A   #%N" %^B%7!7:L!L!!!$% >Q&r*^A->>%N(;; 2 ~.-"2DE b   #2c)9#:;"2';<~!55   9#(D  '(#(D  %(EM 	D$ZF"+N"; v~&+#''0	  #?3q8^)9. ,.M	A
c(m
A;~&Ic(m#N(CQ c(m#N(CG  )W!56FA c(m
 GGG
8}q-H=C !L>"HD T#Y $ C
H
} ' 	JE3e| %s|	
 %JE3*,J"5#. )	%hy&9:!!%()
 HMhADC<<<
 2<<Ac!Qi<
 $Ajj !AJ6AR!  	  ##J/ jj NeHo NNO 
 ##I. &ab> 1C** %ENs^%  	  !''01 C H+ 
s3$$#H
 ##DL11HCI H
L 99\""S 3n =s   X2X	Xc                 :   | j                  ddd      }|sg S d}i }|D ]3  }t        |d   |z        |z  }||vrg ||<   ||   j                  |       5 t        |j	                               }g }|j                         D ]  }|D ]  }|j                  |d            |sg S |j                          d}	g }
|D ]!  }|
r||
d   z
  |	kD  s|
j                  |       # t        |
      dk  st        |
      d	kD  rg S g }|D ]  }t        ||   d
       }dgt        |
      z  }|D ]^  }d}t        d      }t        |
      D ]   \  }}t        |d   |z
        }||k  s|}|}" ||   r||xx   d|d   z   z  cc<   W|d   ||<   ` t        d |D              }|dk\  s|j                  |        t        |      dk  rg S d}d}|D ]A  }|D ]:  }|j                         s|dz  }t        |j                               dkD  s6|dz  }< C |dkD  r
||z  dkD  rg S |gS )a  
    Extract tables from a PDF page by analyzing word positions.
    This handles borderless tables where words are aligned in columns.

    This function is designed for structured tabular data (like invoices),
    not for multi-column text layouts in scientific documents.
    TrJ   rK   rO   rP   rS   rh   rX   rg   c                     | d   S rR   r<   rT   s    r   rU   z,_extract_tables_from_words.<locals>.<lambda>  s
    ag r!   rV   r&   r   infr   r   c              3   B   K   | ]  }|j                         sd   yw)r   Nr)   r*   s     r   r-   z-_extract_tables_from_words.<locals>.<genexpr>  s     ?d$**,?s   r   r   re   g333333?)r   r   r   r   r   valuesr   r   floatr   r   r   r   )rH   r^   rN   r   rt   r]   r   all_x_positionswords_in_rowx_tolerance_colcolumn_startsr   
table_rowsrow_databest_colmin_distr   r   dist	non_emptylong_cell_counttotal_cell_countr0   r,   s                           r   _extract_tables_from_wordsr     s    !QRSE	 K)+I &d5kK/0;>	!!Ie%%	& 9>>+,M O!((* /  	/D""4:.	// 	 O!#M $M"$5 5 G  #$
 =A]!3b!8	 J (i.4EF 4#m,,  	2DHU|H%m4 !54:-.(?#H H	! !"cDL&88"%)&\"	2  ?h??	>h'/(: :	 O ) 	)Dzz| A% tzz|$r)#q(O	)) !2B BS H	<r!   c                   @    e Zd ZdZdedededefdZdededede	fdZ
y)	PdfConverterz
    Converts PDFs to Markdown.
    Supports extracting tables into aligned Markdown format (via pdfplumber).
    Falls back to pdfminer if pdfplumber is missing or fails.
    file_streamstream_infokwargsr   c                     |j                   xs dj                         }|j                  xs dj                         }|t        v ryt        D ]  }|j                  |      s y y)Nr&   TF)mimetypelower	extensionACCEPTED_FILE_EXTENSIONSACCEPTED_MIME_TYPE_PREFIXES
startswith)selfr   r   r   r   r   prefixs          r   acceptszPdfConverter.accepts  sg      ((.B557 **0b779	001 	F""6*	 r!   c                    t         Rt        t        j                  t	        |       j
                  dd            t         d   j                  t         d         t        |t        j                        sJ t        j                  |j                               }	 g }d}g }t        j                  |      5 }t        |j                        D ]  \  }	}
t!        |
      }|'|dz  }|j#                         rd|j%                  |       nR|j%                  |	       |
j'                         }|r/|j#                         r|j%                  |j#                                |
j)                           	 d d d        |dk(  r1|j+                  d       t,        j.                  j'                  |      }ndj1                  |      j#                         }|s0|j+                  d       t,        j.                  j'                  |      }t5        |      }t7        |      S # 1 sw Y   xY w# t2        $ r3 |j+                  d       t,        j.                  j'                  |      }Y w xY w)	Nr"   pdf)	converterr   featurer   r   r   z

)markdown)_dependency_exc_infor	   r
   formattype__name__with_traceback
isinstanceioIOBaseBytesIOread
pdfplumberopenr   pagesr   r   r   extract_textcloseseekpdfminer
high_levelr   	Exceptionr    r   )r   r   r   r   	pdf_bytesmarkdown_chunksform_page_countplain_page_indicesr   page_idxrH   page_contentr   r   s                 r   convertzPdfConverter.convert  s	     +,*11"4j11$! (*99$Q' +ryy111 JJ{//12	%	C *,OO,.+ !s&/		&: !NHd#CD#IL#/'1,'--/+22<@*11(;#002DJJL+224::<@JJL!!$ !#q!#..;;IF!;;7==? NN1**77	BH 2(;&99K! !0  	CNN1**77	BH	Cs,   H 4B4H)AH HH 9IIN)r   
__module____qualname____doc__r   r   r   boolr   r   r   r<   r!   r   r   r     sb       	
 
$E:E:  E: 	E:
 
!E:r!   r   )T) sysr   retypingr   r   _base_converterr   r   _stream_infor   _exceptionsr	   r
   compiler   r/   r    r   r   pdfminer.high_levelr   ImportErrorexc_infor   r   rB   r   rG   r   r   r   r<   r!   r   <module>r     s   
 	 	   H % P 'BJJz2 +# +# +#^  *  
 #8 'd49o '$ 'RU 'TS#3 S#3: S#l^S ^T$tCy/-B ^B^:$ ^:[  *'3<<>*s   B B-,B-