jzddlmZddlmZmZddlZddlmZddlZddl m Z m Z m Z m Z mZmZmZddlZddlZddlmZddlmZddlmZmZmZdd lmZdd lmZdd l m!Z!dd l"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)dd l*m+Z+m,Z,ddl-m.Z.ddl/m0Z0ddl1m2Z2m3Z3ddl4m5Z5ddl6m7Z7ddl8m9Z9m:Z:ddl;mZ>m?Z?m@Z@e r ddlAmBZBmCZCmDZDmEZEddlmFZFmGZGmHZHmIZIddlJmKZKmLZLdZMGdde<ZNGddejZOGddeNZPd dZQy)!) annotations)abc defaultdictN)StringIO)IO TYPE_CHECKINGAny DefaultDictLiteralcastfinal)lib)Scalar)EmptyDataError ParserError ParserWarning)cache_readonly)find_stack_level) astype_array) is_bool_dtypeis_extension_array_dtype is_integeris_numeric_dtypeis_object_dtypeis_string_dtype pandas_dtype)CategoricalDtypeExtensionDtype) is_dict_like) algorithms) CategoricalExtensionArray) BooleanDtype)Index) dedup_namesis_potential_multi_index) ParserBaseevaluate_callable_usecols get_na_valuesparser_defaultsvalidate_parse_dates_presence)HashableIteratorMappingSequence) ArrayLikeDtypeObj ReadCsvBufferT) MultiIndexSeriesuceZdZUded<d fd Zed!dZd"dZ d# d$dZ d%dZ d# d&dZ d'd Z e d( d)d Z e d*d Zed+d Z d,d ZedZ d-dZd.dZd/dZd0dZd.dZd1dZd2dZd3dZd4dZd3dZ d5dZd3dZ d6dZd7dZd#d8dZ d9dZ!d:dZ"xZ#S); PythonParserset[int]_no_thousands_columnsc t|g_g_d_d_|d_tj rj _n fd_t|d_ |d_ |d_ tjtrtj_ |d_|d_|d _|d _|d _|d _|j+d d_|d_|d_|d_t|t4r|_n$t7|dsJj9|_d_j=\}__ jC|jD\_#_"_$}t5jF_%jM\}_%_#jD|_"j:,t5tOtQjF_jS_*tQj0dk7r tWdy)zN Workhorse function for processing nested list into DataFrame rskiprowsc |jvSN)r;)xselfs `/root/aria/tools/markitdown-venv/lib/python3.12/site-packages/pandas/io/parsers/python_parser.pyz'PythonParser.__init__..ssa4==&8 skipfooter delimiter quotechar escapechar doublequoteskipinitialspacelineterminatorquotingskip_blank_lineshas_index_namesF thousandsdecimalcommentreadlineNz'Only length-1 decimal markers supported),super__init__databufposline_posr;callableskipfunc_validate_skipfooter_argrCrDrE isinstancestrrFrGrHrIrJrKgetrLrMrNrOlisthasattr _make_reader _col_indices_infer_columnsnum_original_columns unnamed_cols_extract_multi_indexer_columns index_namescolumns col_names orig_names_get_index_namerangelen_set_no_thousand_columnsr9 ValueError)r?fkwdsrg_rf __class__s` r@rSzPythonParser.__init__csS >@  Z( DMM " MMDM8DM24 3EFk*k* dnnc * 0DN|, . $%7 8"#34I $%7 8 $xx(95Ak*I I  a DI1j) ))))!,DI/3    !    %    / /      L   N +/t||*<595I5I5K2 T_dl    #*D     $ $U3t||+<%= >D %)%B%B%D" t||  !FG G "rBctj|j}|jd|d}n(tj|j}d|d|d}tj|S)Nz^[\-\+]?[0-9]*(z [0-9]*)?([0-9]?(E|e)\-?[0-9]+)?$z^[\-\+]?([0-9]+z |[0-9])*()reescaperNrMcompile)r?rNregexrMs r@numzPythonParser.numsm))DLL) >> !&wi/OPE $..1I"9+Ywi@,- zz%  rBc j t dk(r"jr tdGfddtj }|} w |_j jjrj}|snxjdz c_jjrEn[j}j|ggd}jjs|sZxjdz c_j}j|ggd}jjrW|sZttt|}|d}xjdz c_xjdz c_t jj!|}|j|_t j"t%||}j&j)t|t j"|d} | S fd } | } | S) NrQz.MyDialect N) __name__ __module__ __qualname__rDrErFrGrHrJrIr?sr@ MyDialectr{sC NN  NN !__ ".. #'#8#8 ,,!%rBrr)dialectT)rstrictc3Kj}tj}|j|j D]#}|j|j %ywr=)rPrtrvsplitstrip)linepatroseps r@_readz(PythonParser._make_reader.._readsXzz|jjoii --2D))DJJL112sA0A3)rDrlrIrncsvDialectr;rYrVrP_check_commentsr r^r\rWSniffersniffreaderrrUextend) r?rordiarlines lines_strsniffedline_rdrrrrs `` @r@r`zPythonParser._make_readersnn ;#c(a-"" R &CKK &C # ==,--1 zz|#!A  --1zz|,,tfX6q9mmDHH-UHHMH::z$PythonParser.read..7s'( *+CR$19 %( s rrQ) _get_lines StopIteration _first_chunkcloser^rir%r& index_col_get_empty_metadtype_maybe_make_multi_index_columnsrhrLsumrl _rows_to_cols_exclude_implicit_index _convert_data_do_date_conversions _make_index) r?rowscontentrgnamesindexcol_dict conv_columns indexnamerowalldatarT conv_dataresult_columnss r@readzPythonParser.reads ood+G"'+4??&; (OONNE(,';'; ( $E7H ??XL,0 0   C( /6qz( % \%#1:LabkG$$W-44W= g&&t, --gyA $ 0 0'< P~ni//[     sE)E/.E/cPt|jt|j|j}d}|jrt |j}t |}|j ||t|Dcic]\}}||ks ||||zc}}|fScc}}wNr)r%rir&r_implicit_indexrl_check_data_length enumerate)r?rroffset len_alldatainames r@rz$PythonParser._exclude_implicit_indexGs  OO $     (F'l  w/6?u5E *1!T[D'!f*% %   s  B" B"cB| |j}|j|S)N)r) chunksizer)r?sizes r@ get_chunkzPythonParser.get_chunk`s# <>>Dyydy##rBc|j|j}|j|j}i}i}t|jt rj|jD]Z}||j|}|j |}t|tr||jvr|j|}|||<|||<\n|j}|j }|j|||||Sr=) _clean_mapping convertersrr[ na_valuesdict na_fvaluesrri_convert_to_ndarrays) r?rT clean_conv clean_dtypesclean_na_valuesclean_na_fvaluescolna_value na_fvalues r@rzPythonParser._convert_datals ((9 **4::6  dnnd +~~ 6?#~~c2H $ 4I!#s+4??0J"ooc2+3OC(,5$S) 6#nnO# ((        rBci}t|j|j}|jD]\}} |dn|j |d} t |t r|j |d} n|} |jrt||||j\} } ntt} } ||vrMtj| t| | z}tj| |tj| ||<| h| (t!j"d|dt$t' t)j*| | } |j7| t| | z| dud\}}nt9| }|xs t;| }| xr| }|j7| t| | z| du|\}}| t=| } | rD|j>| k7s|r3|s|dkDrtA| rt-d||jC|| |}|||<|S#t,$rWtj| t/|j1tj2}t)j4| | |} Y)wxYw)Nz5Both a converter and dtype were specified for column z# - only the converter will be used. stacklevelF) try_num_boolrz$Bool column has NA values in column )"r+ parse_datesrgitemsr]r[r na_filterr)keep_default_nasetr isinnpputmasknanwarningswarnrrr map_inferrnr^viewuint8map_infer_mask _infer_typesrrrrr _cast_types)r?dctrrrdtypesresultparse_date_colscvaluesconv_f cast_type col_na_valuescol_na_fvaluesmaskcvalsna_countis_eais_str_or_ea_dtypers r@rz!PythonParser._convert_to_ndarrayss78H8H$,,WG IAv'/TZ^^At5LF&$'"JJq$/ # ~~0=y*d.B.B1- ~14su~ O#"vs=/AN/RS 640"q !(MM**+,OQ#5 F ]]66:F #'"3"3 &7%!& #4#x1;%*%Hoi.H"%.$D2DE #'"3"3 &7% #x( ,Y 7I%++":e X\(3",/STUSV-W"XX ,,UIqAEF1IOG P G"F%??64 ?CHHRD //EFFs/HAI/.I/ct|tr|jdu}t|js|st j |dd}t|djj}tj||j|||j}|St|tr|j} t|t rL|Dcgc] }t#|}}|j%|||j|j&|j(S|j%||St|t,r|j/|d}|St1|j2t"rt j |d d}|S t5||d }|Scc}w#t*$r} t+d|d | d} ~ wwxYw#t6$r} t7d |d || d} ~ wwxYw) ar Cast values to specified type Parameters ---------- values : ndarray or ExtensionArray cast_type : np.dtype or ExtensionDtype dtype to cast values to column : string column name - used only for error reporting Returns ------- converted : ndarray or ExtensionArray NF)skipnaconvert_na_value)copy) true_values)rr false_values none_valuesrzExtension Array: zO must implement _from_sequence_of_strings in order to be used in parser methodsTzUnable to convert column z to type )r[r categoriesrrrensure_string_arrayr$uniquedropnar!_from_inferred_categories get_indexerrrconstruct_array_typer#r\_from_sequence_of_stringsrrNotImplementedErrorr"astype issubclasstyperrn) r?rrcolumn known_catscats array_typeval values_strerrs r@rzPythonParser._cast_typess" i!1 2"--T9J"6<<00055e,335<<>D ::d&&v. tGWGWFV M > 2"779J i67=!=s#c(!=J!=%??"'$($4$4%)%6%6$(NN @&??i?XX /]]95]9F   ,,,teF  %fidC  A">' )' |4VV $  /xy L sH F* F%25F*(F*G %F** G3GG G+G&&G+c|jy|j}t|tttj frt |dkDSy)NFrQ)headerr[r^tuplerndarrayrl)r?rs r@_have_mi_columnszPythonParser._have_mi_columns.s> ;;  ftUBJJ7 8v;? "rBc r |j}d}d}t}|j|j}|j}t |t t tjfr|rgt ||ddz}n|g}g}t|D]u\}} |j} |j| kr |j} |j| kr g}g}t| D]N\}}|d k(r3|r d|d|}nd|}|j+||j+|>|j+|P|st/t0}t3t#|Dcgc]}||vr| c}|z}|D]}||}|}||}|dkDr|dkDr$|dz||<|d|}||vr|dz }n||}|dkDr$|j4t7|j4rl|j4j9|Q|j4j9|6|j4j;||j4j9|i|||<|dz||<nj|rh| |dk(r`t#|}|j<}| t#|nd}t#|}||k7r||z |kDs|dk(rd}dg|z}|j&dg|_|j+||j;|Dchc]}|| c}t#|dk(skt#|}x|r|j&j)| |j}|dn t#|}t#|t#|dkDrt#||kDr t%dt#|dkDr t?d|j@|jC|||n t#|}|jDIt#|t#|jDk7r(tG|jDDcgc]}|| c}g}n|g}n|jC||d|}nt#|jH}|}|s,t t3|g}|jC||d|}n|j@t#||k\r |jC|g||}t#|}nZtK|j@s,t#|t#|j@k7r t%d|g}|jC||d||||fS#t$r} d|jcxkr| krknnh|r| |dk7r^t tt|r|ddn|} ddj!| dt#| d} t%d | d |jd | |rN| dkDrI|r|j&j)|j+dgt#|dz|||fcYd} ~ cS|js t-d | |jdd} Yd} ~  -/BGm4KPRVWPW+0L,06B;L(, ~DH|,##>O$P\!_$PQw<1$+.|+<(Mf =P  &!%!2J+5*wO0O 0 !'$$Wgaj%@,l::u%)4==.B.+rVBZ/?#3s?F3BKPV#WX !#((3- 3s8*BG(,SE4==/X"#'26' HHNN,vGBK0@'@A&(T  X X %X"? X4 XCXX#'XX" X10X1c|jy |j}|S#t$r2}|js t d||jdd}Yd}~|Sd}~wwxYw)Nr)rrrrr)r?rrs r@r(zPythonParser._header_linesd ;; " !&&(D   !::$%DE3N::a=D  !s! A'AAc  |j8t|jrt|j|}ntd|jDrpt |dkDr t dg}|jD]E}t |tr" |j|j|5|j|Gn<|jDcgc] }||k\s | }}|rtd|d|j}|D cgc]%}t|D cgc] \}} ||vs | c} }'}}}} t||_|S#t $r|j|j|YwxYwcc}wcc} }wcc} }}w)zb Sets self._col_indices usecols_key is used if there are string usecols. c3<K|]}t|tywr=)r[r\)rus r@rz/PythonParser._handle_usecols..s>AZ3'>srQz4If using multiple headers, usecols must be integers.z>>w@#ll & )0Etq!A4DEG!'{ 3D ) *T 88{ST #Fs< E E3E3E> E8(E8,E>%E0/E08E>cpt|jdkDr|jdS|jS)zH Return a line from buffer, filling buffer if required. r)rlrUrrs r@rzPythonParser._buffered_line/s/ txx=1 88A; ??$ $rBcR|s|St|dts|S|ds|S|dd}|tk7r|S|d}t|dkDrR|d|jk(r@d}|d}|ddj |dz}|||}t||dzkDr|||dzdz }n|dd}|g}||ddzS)a- Checks whether the file begins with the BOM character. If it does, remove it. In addition, if there is quoting in the field subsequent to the BOM, remove it as well because it technically takes place at the beginning of the name, not the middle of it. rrQN)r[r\_BOMrlrEr) r? first_row first_elt first_row_bomstartquoteendnew_row new_row_lists r@_check_for_bomzPythonParser._check_for_bom8s  )A,, | aLO   !!  }  !mA&6$..&HE!!$E#))%014C$E#.G=!C!G+=q33$AB'G&-Y im++rBc0| xstd|DS)z Check if a line is empty or not. Parameters ---------- line : str, array-like The line of data to check. Returns ------- boolean : Whether or not the line is empty. c3"K|]}|  ywr=)rr>s r@rz.PythonParser._is_line_empty..zs31u3s )all)r?rs r@_is_line_emptyzPythonParser._is_line_emptymsx333d333rBct|jtr|j|jrT|jt |jk\rn1|xjdz c_|j|jrT |j |j|jgd}|xjdz c_|js/|j|j|jdz s|rn|jr|j|g}|r|d}n錤|j|jrF|xjdz c_t|j|j|jrF |j|jdz}|xjdz c_|O|j |gd}|jr|j|g}|r|d}n|j|s|rn|jdk(r|j|}|xjdz c_|j j#||S#t$r }t|d}~wwxYw)NrQrrow_num)r[rTr^rYrVrlrrKrX_remove_empty_lines IndexErrorrnext_next_iter_linerSrWrUr")r?rretr orig_lines r@rzPythonParser._next_line|s dii &--)88s499~-A --) 1//4881D0EFqIDHHMH00++DIIdhhl,CD,,"66v>#&q6D!"--)A TYY--) 00A0F A (// #&q6D!,,Y74& 88q=&&t,D    ?"1'S01s A:I* %I** I?3I::I?c(|j|jjk(r t||j|jjk(st |jr,t jd|d|dttyy)a  Alert a user about a malformed row, depending on value of `self.on_bad_lines` enum. If `self.on_bad_lines` is ERROR, the alert will be `ParserError`. If `self.on_bad_lines` is WARN, the alert will be printed out. Parameters ---------- msg: str The error message to display. row_num: int The row number where the parsing error occurred. Because this row number is displayed, we 1-index, even though we 0-index internally. zSkipping line z: r|rN) on_bad_linesBadLineHandleMethodERRORrWARNrXrrrr)r?r.r[s r@_alert_malformedzPythonParser._alert_malformeds"    8 8 > > >c" "    8 8 = = =   B  MM  C53+- B rBc t|jtrJt|j}|S#tj $r}|j |jj|jjfvr@t|}d|vsd|vrd}|jdkDr d}|d|zz }|j||Yd}~yd}~wwxYw)a[ Wrapper around iterating through `self.data` (CSV source). When a CSV error is raised, we check for specific error messages that allow us to customize the error message displayed to the user. Parameters ---------- row_num: int The row number of the line being parsed. z NULL bytezline contains NULzNULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' insteadrzError could possibly be due to parsing errors in the skipped footer rows (the skipfooter keyword is only applied after Python's csv library has parsed all rows).. N) r[rTr^r^rErrorrcrdrerfr\rCrg)r?r[rer.reasons r@r_zPythonParser._next_iter_lines !$))T2 22 ?DKyy   ((..((--%!f#%)<)C???Q&%4&=(C%%c735 s25C A8CC c`|j|Sg}|D]}g}|D]~}t|tr|j|vs||jvr|j |A|d|j |j}t |dkDr|j |n|j ||Sr)rOr[r\rr"findrl)r?rr`rrlr>s r@rzPythonParser._check_commentss << L DB "1c*||1,DNN*IIaL0AFF4<<01A1vz !   JJrN  rBc|Dcgc]F}t|dkDs4t|dk(r(t|dtr|djr|H}}|Scc}w)a Iterate through the lines and remove any that are either empty or contain only one whitespace value Parameters ---------- lines : list of list of Scalars The array of lines that we are to filter. Returns ------- filtered_lines : list of list of Scalars The same array of lines with the "empty" ones removed. rQr)rlr[r\r)r?rrr`s r@r\z PythonParser._remove_empty_liness\$  D A INJtAw,DQ         sA AcZ|j|S|j||jdS)Nrrsearchreplace)rM_search_replace_num_columnsr?rs r@_check_thousandszPythonParser._check_thousands-s4 >> !L//0  rBc`g}|D]}g}t|D]\}}t|tr;||vs7||jvs)|jj |j s|j|c|j|j|||j||Sr=) rr[r\r9rxrsrr"rt) r?rrsrtr`rrorr>s r@ruz(PythonParser._search_replace_num_columns5s DB!$ :1"1c*QD66688??17795IIaLIIaii89 : JJrN  rBcn|jtdk(r|S|j||jdS)NrNrrr)rNr*rurvs r@_check_decimalzPythonParser._check_decimalHs; <SY1J1J%JJ%)%D *:%;DN#xx|DH%d^-q!,-"&gJ03G D-W44  "#'D ~~%!%e,?&@!AJ/3.E.E/ +ZDN:w..g!    I s#FF( F%$F%( F76F7c|j}|jr|t|jz }t d|D}||kDr"|jdur|j |j r |j nd}g}t|}t|}g}|D]H\}} t| } | |kDr t|jr|j| } | Attt| } t| |kDr7|j||z |zz } |j| t| df| d|} |j| |j|jj |jj"fvs|j||z |zz } |j| | df|j|jj k(s6n|j| K|D]\} } } d|d| dzd | }| dk(r|d z }nK|j$r?t|j$dkDr'|j&t(j*k7r d }|d |zz }|j-|| dztt/j0|| j2}|j r|j4J|j4}|jrQt|Dcgc]9\}}|t|jks|t|jz |vr|;}}}|St|Dcgc] \}}||vs |}}}|Scc}}wcc}}w)Nc32K|]}t|ywr=)rl)rrows r@rz-PythonParser._rows_to_cols..s23c#h2sFrrXnormalz Expected z fields in line rQz, saw z from bad_lines callablezXError could possibly be due to quotes being ignored when a multi-char delimiter is used.ri) min_width)rcrrlrmaxr%rCrrXrcr r^rrVr"rdrerfrDrJr QUOTE_NONErgrto_object_arrayr3ra)r?rcol_lenmax_lenfooters bad_lines iter_content content_lenr_content actual_lennew_lr[sourcer.rlzipped_contentrDas r@rzPythonParser._rows_to_colss=++    s4>>* *G2'22 W u!<AU)-dooaGI$W-Lg,KG+ - 8 ] ' 1 12 $ 1 1( ; ,$(fu$=E"5zG3*.((kAo6O*P ) 0 0'3u:z1R S(-hw#NN51**00660055/#'((kAo.G"H!((':x)HI,,0H0H0N0NN!NN8,- -009 8+Vy(81 VJ<XZ'55CNNDNN+a/ 6G4&=(C%%c7Q;7% 8*c11'WMOOP <<$$0 00++K##!*. 9"1C//s4>>22kA ""#,N";"!QqK?OA""""s>L<& M3Mc|j}d}|Ut|j|k\r%|jd||j|dc}|_n|t|jz}|t|jtr|j t|jkDrt |/|j|j d}t|j}n5|j|j |j |z}|j |z}|j|}|j|||_ng} |d}d}|j |j nd}||krJt|j}|j||zs|dz }|dz }|j|||krJt|} |j|}|j|n>d} |j|j |zdz} |dz }| |j| ; |xj | z c_g|_n|}|jr|d|j }|j|}|jr|j!|}|j#|}|j%|S#t $r?t|} |j|}|j|t|dk(rYwxYw)NrrQrZ)rUrlr[rTr^rVr_remove_skipped_rowsrr^rYr"r_rCrrKr\rwrz) r?rrnew_rowsnew_pos row_indexrow_ctrrQ len_new_rowsnext_rows r@rzPythonParser._get_liness  488}$%)XXet_dhhtuo"$(DHH %  $))T*88c$))n,''<#yy4H!$))nG#yyDHHtODH"hhoG44X> X&"'$% !"-1XX-Aq$tm&*499oG#'==)1C#D &! %NI$OOG4 %tm(+8} #'#<#A.J%-A,J%%AK-,K-c|jr=t|Dcgc]&\}}|j||jzr%|(c}}S|Scc}}wr=)r;rrYrV)r?rrrs r@rz!PythonParser._remove_skipped_rows@sI =="+H"53T]]1txx<=X s &A A ct}|jr@|jr4|jJ|j |j|j}|jr|j r|jJt |j|jdD]\}}t|j ts&t|j s|j|t|j tsa||j vspt|j |rt|j |s|j||S)NTr) rrgrra_set_noconvert_dtype_columnsrzipr[rraddr)r?no_thousands_columnsrrs r@rmz%PythonParser._set_no_thousand_columnsGs), < delimiterspatternrrrrmshiftededges edge_pairss r@rz FixedWidthReader.detect_colspecss_WW?11#h?@ **s:,c23}}[(3 !KL Lc#tn%xx! 3/ << #>BCsCMM$,,/2CDC .C%%c* .,-QWWY) . .''$" $.Q./2#eCaCj%1+dCD !@Ds E/,%E4cJ|j t|j}nt|j}|jDcgc]#\}}|||j |j %c}}S#t$rd|_t|j}YbwxYwcc}}wr=)rr^rrorrrD)r?rfrom_tos r@__next__zFixedWidthReader.__next__s ;; " $DKK( .s(E1z!S))6QWWY6Es-/)rA)r?rrs r@r\z)FixedWidthFieldParser._remove_empty_liness2 EEE    s#N)rozReadCsvBuffer[str]rr)rorrrr)r}r~rrrSr`r\rVrBr@rrs /    rBrcRt|s td|dkr td|S)a Validate the 'skipfooter' parameter. Checks whether 'skipfooter' is a non-negative integer. Raises a ValueError if that is not the case. Parameters ---------- skipfooter : non-negative integer The number of rows to skip at the end of the file. Returns ------- validated_skipfooter : non-negative integer The original input if the validation succeeds. Raises ------ ValueError : 'skipfooter' was not a non-negative integer. zskipfooter must be an integerrzskipfooter cannot be negative)rrn)rCs r@rZrZs3* j !899A~899 rB)rCrrr)R __future__r collectionsrrriorrttypingrrr r r r r rnumpyr pandas._libsrpandas._typingr pandas.errorsrrrpandas.util._decoratorsrpandas.util._exceptionsrpandas.core.dtypes.astyperpandas.core.dtypes.commonrrrrrrrpandas.core.dtypes.dtypesrrpandas.core.dtypes.inferencer pandas.corer pandas.core.arraysr!r"pandas.core.arrays.booleanr#pandas.core.indexes.apir$pandas.io.commonr%r&pandas.io.parsers.base_parserr'r(r)r*r+collections.abcr,r-r.r/r0r1r2r3pandasr4r5rJr7rrrZrVrBr@rs"  ! 3426"4)~$:~$B(qWs||qWh! L! HrB