j.ddlmZddlmZddlZddlmZddlmZddl m Z m Z m Z ddl mZddlmZdd lmZdd lmZdd lmZerddlZdd lmZdd lmZGddeZy)) annotations) TYPE_CHECKINGN)lib)import_optional_dependency)Pandas4Warning ParserError ParserWarning)find_stack_level) pandas_dtype) is_integer)arrow_table_to_pandas) ParserBase) ReadBuffer) DataFramecveZdZdZd fd Zd dZd dZdZddZddZ ddZ dd Z d d Z dd Z xZS)ArrowParserWrapperz7 Wrapper for the pyarrow engine for read_csv() c `t||||_||_|j yN)super__init__kwdssrc _parse_kwds)selfrr __class__s g/root/aria/tools/markitdown-venv/lib/python3.12/site-packages/pandas/io/parsers/arrow_parser_wrapper.pyrzArrowParserWrapper.__init__&s*   c|jjd}|dn||_|jd}t|tr t dt |jd|_y)z? Validates keywords before passing to pyarrow. encodingNzutf-8 na_valuesz?The pyarrow engine doesn't support passing a dict for na_values)rgetr isinstancedict ValueErrorlistr )rrr s rrzArrowParserWrapper._parse_kwds-sc $yy}}Z8#+#3 IIk* i &Q dii 45rc0ddddddd}|jD]X\}}||jvs|jj|1|jj||j|<Z|j}t |t r|g}nd}||jd <|jjDcic]\}}||d vr||c}}|_|jjd }|t|r||jd <n|tjjk(rd|jd <n^|tjjk(rdd }||jd <n-|tjjk(rd|jd <|jjDcic]\}}||dvr||c}}|_d|jdv|jd<|j;d|jvr-|jdD cgc]} d|  c} |jd<|jdu|j |jn|jd|j d|_ycc}}wcc}}wcc} w)z: Rename some arguments to pass to pyarrow include_columns null_values escape_charignore_empty_lines decimal_point quote_char)usecolsr escapecharskip_blank_linesdecimal quotecharNtimestamp_parsers) delimiterr,r)r* on_bad_linesinvalid_row_handlerctjd|jd|jd|jt t y)Nz Expected z columns, but found z: ) stacklevelskip)warningswarnexpected_columnsactual_columnstextr r ) invalid_rows rhandle_warningz?ArrowParserWrapper._get_pyarrow_options..handle_warninglsLMM#K$@$@#AAU&556b9I9I8JL%#3#5  "rcy)Nr8)_s rz9ArrowParserWrapper._get_pyarrow_options..wsr)r'r( true_values false_valuesr+r2strings_can_be_nullfskiprows)autogenerate_column_names skip_rowsr)returnstr)itemsrr!pop date_formatr"rM parse_optionscallablerBadLineHandleMethodERRORWARNSKIPconvert_optionsheaderr read_options) rmapping pandas_name pyarrow_namerP option_name option_valuer4r?ns r_get_pyarrow_optionsz'ArrowParserWrapper._get_pyarrow_options;s )&' 4&%  *1 E %Kdii'DIIMM+,F,R*.))-- *D ,' E&& k3 '&-KK)4 %&.2YY__-> ) \'OP  % yy}}^4  # % ) \'  % 79DIIm "7sJ?J $ Jctd} |jdi|j}|S#t$r}|jj dd}||j ||jj dt }tj|rtd|Ds td|d}~wwxYw)N pyarrow.csvr'r(c3<K|]}t|tywrr"rM.0xs r z:ArrowParserWrapper._get_convert_options..s6'( 1c"6z9The 'pyarrow' engine requires all na_values to be stringsrA) rConvertOptionsrW TypeErrorr!_validate_usecolssetr is_list_likeall)r pyarrow_csvrWerrincludenullss r_get_convert_optionsz'ArrowParserWrapper._get_convert_optionss0?  8k88P4;O;OPO  **../@$GG"&&w/((,,]CEBE##E*#6,163 O  s+ CBB;;CcXt|j}d}|j|jt ||_t|j|k7rMt |t|jz Dcgc] }t |}}||jz|_d}|Scc}w)NTF)lencolumnsrXnamesrangerM)rtablenum_colsmulti_index_namedrgcolumns_prefixs r_adjust_column_namesz'ArrowParserWrapper._adjust_column_namessu}}%  ;; zz!"8_ 4::(* 383tzz?8R2S!TQ#a&!T!T+djj8 $)!  "Us;B'c|jl|jj}t|jD]\}}t|r|j|||<n||jvrt d|d|j N|j j|||j j|fn6|j||j j|j|f\}}|||j|||<|j |=|j|dd|j4|s2dgt|jjz|j_ |S)NzIndex z invalidT)dropinplace) index_colcopy enumerater rwr$dtyper!astype set_indexrXrvindexrx)rframer| index_to_setiitemkey new_dtypes r_finalize_indexz"ArrowParserWrapper._finalize_indexsP >> %>>..0L$T^^4 ,4d#&+mmD&9LO.$vdV8%<==::) ::>>$/;tzz~~d34#mmD14::>>%--PTBU3VW#C !,%*3Z%6%6y%Ac  JJsO! ,$ OOLtTO B{{"+<%)FS1B1B-C$C ! rc|jt|jtrK|jjDcic]\}}||jvr |t |!c}}|_nt |j|_ |j |j}|S|Scc}}w#t$r}tt||d}~wwxYwr) rr"r#rNrwr rrkr$rM)rrkvrqs r_finalize_dtypez"ArrowParserWrapper._finalize_dtypes :: !$**d+!% 0 0 21EMM)|A& *$**5  4 TZZ0 u  4 S*3 4s$B, B22 C;CCc|j|j|}|j||}|j|}|S)a Processes data read in based on kwargs. Parameters ---------- frame : DataFrame The DataFrame to process. multi_index_named : bool Returns ------- DataFrame The processed DataFrame. )_do_date_conversionsrwrr)rrr|s r_finalize_pandas_outputz*ArrowParserWrapper._finalize_pandas_outputsB"))%--?$$U,=>$$U+ rctj|rtd|Ds tdt |r tdy)Nc3<K|]}t|tywrrdres rrhz7ArrowParserWrapper._validate_usecols..s0UAs1C0UrizwThe pyarrow engine does not allow 'usecols' to be integer column positions. Pass a list of string column names instead.z=The pyarrow engine does not allow 'usecols' to be a callable.)rrnror$rR)rr-s rrlz$ArrowParserWrapper._validate_usecolssO   G $S0UW0U-UP g O rc 0td}td}|j|j} |j|j|j d i|j |jd i|j|}|jd}|tjur|j}|j}t!|jj"D]Q\} } |j"j%| s"|j'| |j)| j+|}S|j-|}|j/|} t1j25t1j4ddt6t9||d|j:|j< } ddd|j>|j< _ |jC | S#|j$r}t||d}~wwxYw#1swY[xYw) z Reads the contents of a CSV file into a DataFrame and processes it according to the kwargs passed in the constructor. Returns ------- DataFrame The DataFrame created from the CSV file. pyarrowrb)rYrQrWN dtype_backendignorezmake_block is deprecatedT)r null_to_int64rrxrA)"rr`rtread_csvr ReadOptionsrY ParseOptionsrQ ArrowInvalidrrr no_defaultschemafloat64rtypesis_nullrmfield with_typecastr~r9catch_warningsfilterwarningsrr rrxrXrwr) rparprWrzer new_schemanew_typer arrow_typer|rs rreadzArrowParserWrapper.read s( 20?  !!#335 (((4[44It7H7HI6k66L9K9KL / )E /2  CNN *J!rzz|H!*5<<+=+=!>  :88##J/!+:++A.88B"J  JJz*E 55e<  $ $ &   # #*  *+"jjjj E  ;;  JJEM++E3DEEK (a.a ' ((  s%AG)2AH )H 8 HH  H)rzReadBuffer[bytes]rLNone)rLr)rzzpa.TablerLbool)rrr|rrLr)rrrLr)rLr)__name__ __module__ __qualname____doc__rrr`rtr~rrrrlr __classcell__)rs@rrr!sU 6Y v,! 8&37 , <Frr) __future__rtypingrr9 pandas._libsrpandas.compat._optionalr pandas.errorsrrr pandas.util._exceptionsr pandas.core.dtypes.commonr pandas.core.dtypes.inferencer pandas.io._utilr pandas.io.parsers.base_parserrrrpandas._typingrpandasrrrArrrsO" > 414) gFgFr