
    [6g'                        d Z ddlZddlmZ ddlmZ ddlZddlZddlZ	ddl
Z
ddlmZ ddlmZ ddlmZ ddlmZmZmZ ddlZddlmZ dd	lmZ dd
lmZmZmZm Z m!Z! 	 ddl"Z"dZ#n# e$$ r dZ#Y nw xY w	 ddl%Z%dZ&n# e$$ r dZ&Y nw xY we
j'        (                    d          e
j'        (                    d          gZ) e
j*         e
j+        de
j'        ,                    e& p edd          dk    d                     e
j+        de
j'        ,                    e# d                    g          d             Z-e
j*        d             Z.e
j*        d             Z/e
j*        d             Z0e
j*        d             Z1e
j*        d             Z2 e
j*        ej        3                    ej4        j5                  ej        3                    ej4        j6                  ej        3                    ej4        j7                  ej        8                    dd           ej        8                    d!d           ej        8                    d"d           ej        8                    d#d           g          d$             Z9	 	 	 	 	 	 	 	 	 d9d&Z:d' Z;d( Z<d) Z=d* Z>d+ Z?d, Z@d- ZAd. ZBd/ ZCd0 ZD G d1 d2          ZE G d3 d4eE          ZF G d5 d6eE          ZG G d7 d8eE          ZHdS ):z test parquet compat     N)Decimal)BytesIO)using_copy_on_write)_get_option)is_platform_windows)pa_version_under11p0pa_version_under13p0pa_version_under15p0)Version)FastParquetImplPyArrowImpl
get_engineread_parquet
to_parquetTFz2ignore:DataFrame._data is deprecated:FutureWarningz=ignore:Passing a BlockManager to DataFrame:DeprecationWarningfastparquetmode.data_managersilentarrayz4fastparquet is not installed or ArrayManager is usedreason)markspyarrowpyarrow is not installed)paramsc                     | j         S Nparamrequests    [/var/www/surfInsights/venv3-11/lib/python3.11/site-packages/pandas/tests/io/test_parquet.pyenginer#   ;   s    & =    c                  <    t           st          j        d           dS )Nr   r   )_HAVE_PYARROWpytestskip r$   r"   par*   Q   s      0.///9r$   c                      t           st          j        d           n)t          dd          dk    rt          j        d           dS )Nzfastparquet is not installedr   Tr   r   z.ArrayManager is not supported with fastparquetr   )_HAVE_FASTPARQUETr'   r(   r   r)   r$   r"   fpr-   X   sN     F23333	(	6	6	6'	A	ADEEE=r$   c                  4    t          j        g ddd          S )N         fooAB)pd	DataFramer)   r$   r"   	df_compatr9   a   s    <iiie44555r$   c            
          t          j        t          d          t          t          dd                    t	          j        ddd          g dt          j        d	d
          d          } | S )Nabcr0            @      @float64dtypeTFT20130101r2   periods)abdef)r7   r8   listrangenparange
date_range)dfs    r"   df_cross_compatrQ   f   so    	eeAqkk""39555$$$z1555
	
 
	

 
B Ir$   c                     t          j        t          d          dt          j        dgg dg dg dt          t          dd                    t          j        d	d
                              d          t          j        ddd          dt          j        dgg dt          j        dd	          t          j	        d          t           j
        t          j	        d          gd          S )Nr;   rF   crF   NrS   )   foo   bars   bazr3   barbazr0   r<   r2      u1r=   r>   r?   r@          @      @rB   rC   rD   20130103)stringstring_with_nanstring_with_nonebytesunicodeintuintfloatfloat_with_nanbooldatetimedatetime_with_nat)r7   r8   rK   rM   nanrL   rN   astyperO   	TimestampNaTr)   r$   r"   df_fullro   x   s    <5kk #RVS1 0 0 0---,,,a$$IaOO**400YsCy999"BFC0'''j!<<<Z((Z(("	
 	
  r$   z2019-01-04T16:41:24+0200z%Y-%m-%dT%H:%M:%S%zz2019-01-04T16:41:24+0215z2019-01-04T16:41:24-0200z2019-01-04T16:41:24-0215c                     | j         S r   r   r    s    r"   timezone_aware_date_listrq      s     =r$   r1   c
                      pddipi  |r
|d<   |d<    fd}
9t          j                    5  |
|	           ddd           dS # 1 swxY w Y   dS  |
|	           dS )a  Verify parquet serializer and deserializer produce the same results.

    Performs a pandas to disk and disk to pandas round trip,
    then compares the 2 resulting DataFrames to verify equality.

    Parameters
    ----------
    df: Dataframe
    engine: str, optional
        'pyarrow' or 'fastparquet'
    path: str, optional
    write_kwargs: dict of str:str, optional
    read_kwargs: dict of str:str, optional
    expected: DataFrame, optional
        Expected deserialization result, otherwise will be equal to `df`
    check_names: list of str, optional
        Closed set of column names to be compared
    check_like: bool, optional
        If True, ignore the order of index & columns.
    repeat: int, optional
        How many times to repeat the test
    compressionNr#   c                     t          |           D ]D} j        fi 
 t          fi 	}dv r
d j        d<   t	          j        |           Ed S )Nr`   r0   r`   )check_names
check_likecheck_dtype)rL   r   r   loctmassert_frame_equal)repeat_actualrx   rw   rv   rP   expectedpathread_kwargswrite_kwargss      r"   comparez!check_round_trip.<locals>.compare   s    v 	 	ABM$//,///!$66+66F H,,5912!'%'    	 	r$   )rz   ensure_clean)rP   r#   r   r   r   r   rv   rw   rx   r|   r   s   ` ```````  r"   check_round_tripr      s   D  8M4#8L#K '!'X &H            |_ 	$GFOOO	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	s   AA!$A!c                 n    ddl m} |                    | d          }|j        j        j        |k    sJ dS )zCheck partitions of a parquet file are as expected.

    Parameters
    ----------
    path: str
        Path of the dataset.
    expected: iterable of str
        Expected partition names.
    r   Nhive)partitioning)pyarrow.datasetdatasetr   schemanames)r   r   dsr   s       r"   check_partition_namesr      sK     !     jjFj33G&,888888r$   c                     d}t          j        t          |          5  t          | dd           d d d            d S # 1 swxY w Y   d S )Nz.engine must be one of 'pyarrow', 'fastparquet'matchr3   rX   )r'   raises
ValueErrorr   )r9   msgs     r"   test_invalid_enginer      s    
:C	z	-	-	- 2 2E51112 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2s   =AAc                     t          j        dd          5  t          |            d d d            d S # 1 swxY w Y   d S )Nio.parquet.enginer   r7   option_contextr   )r9   r*   s     r"   test_options_pyr      s     
	.		:	: $ $###$ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $   377c                     t          j        dd          5  t          |            d d d            d S # 1 swxY w Y   d S )Nr   r   r   )r9   r-   s     r"   test_options_fpr      s     
	.	>	> $ $###$ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $r   c                     t          j        dd          5  t          |            d d d            d S # 1 swxY w Y   d S )Nr   autor   )r9   r-   r*   s      r"   test_options_autor     s     
	.	7	7 $ $###$ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $r   c                 4   t          t          d          t                    sJ t          t          d          t                    sJ t	          j        dd          5  t          t          d          t                    sJ t          t          d          t                    sJ t          t          d          t                    sJ 	 d d d            n# 1 swxY w Y   t	          j        dd          5  t          t          d          t                    sJ t          t          d          t                    sJ t          t          d          t                    sJ 	 d d d            n# 1 swxY w Y   t	          j        dd          5  t          t          d          t                    sJ t          t          d          t                    sJ t          t          d          t                    sJ 	 d d d            d S # 1 swxY w Y   d S )Nr   r   r   r   )
isinstancer   r   r   r7   r   )r-   r*   s     r"   test_options_get_enginer     s   j++[99999j//AAAAA		.		:	: F F*V,,k:::::*Y//=====*]33_EEEEEEF F F F F F F F F F F F F F F
 
	.	>	> F F*V,,o>>>>>*Y//=====*]33_EEEEEEF F F F F F F F F F F F F F F
 
	.	7	7 F F*V,,k:::::*Y//=====*]33_EEEEEEF F F F F F F F F F F F F F F F F Fs9   A-CCC8A-E22E69E6A-HHHc                     ddl m}  |                     d          }|                     d          }t          sdn)t	          t
          j                  t	          |          k     }t          sdn)t	          t          j                  t	          |          k     }t          o| }t          o| }|s&|s%|rId| d}t          j
        t          |          5  t          d	           d d d            n# 1 swxY w Y   nDd
}t          j
        t          |          5  t          d	           d d d            n# 1 swxY w Y   |rKd| d}t          j
        t          |          5  t          d	           d d d            d S # 1 swxY w Y   d S d}t          j
        t          |          5  t          d	           d d d            d S # 1 swxY w Y   d S d S d S )Nr   )VERSIONSr   r   FzPandas requires version .z. or newer of .pyarrow.r   r   z%Missing optional dependency .pyarrow.z. or newer of .fastparquet.z)Missing optional dependency .fastparquet.)pandas.compat._optionalr   getr&   r   r   __version__r,   r   r'   r   ImportErrorr   )r   
pa_min_ver
fp_min_verhave_pa_bad_versionhave_fp_bad_versionhave_usable_pahave_usable_fpr   s           r"   "test_get_engine_auto_error_messager      s    100000 i((Jm,,J 	@W())GJ,?,??  !	D[,--
0C0CC  #>+>'>N&B/B+BN #. # 	#S
SSSE{%888 # #6"""# # # # # # # # # # # # # # # <E{%888 # #6"""# # # # # # # # # # # # # # #  	#W
WWWE{%888 # #6"""# # # # # # # # # # # # # # # # # # @E{%888 # #6"""# # # # # # # # # # # # # # # # # ### # # #sH   C00C47C4D55D9<D9#F  FF*GGGc                 @   | }t          j                    5 }|                    ||d            t          ||          }t          j        ||           t          ||ddg          }t          j        ||ddg                    d d d            d S # 1 swxY w Y   d S N)r#   rs   r#   rF   rH   )r#   columnsrz   r   r   r   r{   rQ   r*   r-   rP   r   results         r"   test_cross_engine_pa_fpr   K  s     
B			 6d
d24888d2...
fb)))d2SzBBB
fb#sn5556 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6   A0BBBc                 @   | }t          j                    5 }|                    ||d            t          ||          }t          j        ||           t          ||ddg          }t          j        ||ddg                    d d d            d S # 1 swxY w Y   d S r   r   r   s         r"   test_cross_engine_fp_par   Y  s    	B			 6d
d24888d2...
fb)))d2SzBBB
fb#sn5556 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6r   c                 (   t          j        dg di          }d}t          j                    5 }t          j        t
          |dd          5  |                    ||            d d d            n# 1 swxY w Y   d d d            d S # 1 swxY w Y   d S )NrF   r/   zqStarting with pandas version 3.0 all arguments of to_parquet except for the argument 'path' will be keyword-only.F)r   check_stacklevelraise_on_extra_warnings)r7   r8   rz   r   assert_produces_warningFutureWarningr   )r#   rP   r   r   s       r"   !test_parquet_pos_args_deprecationr   f  s$   	sIII&	'	'B	1  
		 (d'"$)	
 
 
 	( 	( MM$'''	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	(( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( (s5   BA/#B/A3	3B6A3	7BBBc                   h    e Zd Zd Zd Zej        j        ej        j        d                         Z	dS )Basec                     t          j                    5 }t          j        ||          5  t	          |||d            d d d            n# 1 swxY w Y   d d d            d S # 1 swxY w Y   d S )Nr   rs   )rz   r   r'   r   r   )selfrP   r#   excerr_msgr   s         r"   check_error_on_writezBase.check_error_on_writex  s    _ 	?$s'222 ? ?2tV>>>>? ? ? ? ? ? ? ? ? ? ? ? ? ? ?	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	?s3   A#AA#A	A#A	A##A'*A'c                     t          j                    5 }t          j        |          5  t          |||d            d d d            n# 1 swxY w Y   d d d            d S # 1 swxY w Y   d S )Nr   )rz   r   external_error_raisedr   )r   rP   r#   r   r   s        r"   check_external_error_on_writez"Base.check_external_error_on_write~  s    _ 	?$)#.. ? ?2tV>>>>? ? ? ? ? ? ? ? ? ? ? ? ? ? ?	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	?s3   A!A	A!	A	A!A	A!!A%(A%c                 F   |dk    rt          j        |           t           |dddd          d          5 }|                    |                                           t          |j                  }d d d            n# 1 swxY w Y   t          j        ||           d S )	Nr   iodataparquetzsimple.parquetrb)mode)content)	r'   importorskipopenserve_contentreadr   urlrz   r{   )r   
httpserverdatapathr9   r#   rJ   rP   s          r"   test_parquet_read_from_urlzBase.test_parquet_read_from_url  s     V'''((44DEEDQQQ 	.UV$$QVVXX$666jn--B	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	b),,,,,s   =BBBN)
__name__
__module____qualname__r   r   r'   marknetwork
single_cpur   r)   r$   r"   r   r   w  sc        ? ? ?? ? ? [[- -  - - -r$   r   c                       e Zd Zd Zd Zej                            dg d          d             Zd Z	d Z
d Zd	 Zd
 Zd Zd Zd Zd Zd Zd Zd Zej                            dg d          d             ZdS )	TestBasicc                     t          j        g d          ddt          j        d          t          j        g d          fD ]!}d}|                     ||t          |           "d S )Nr/   r0   r3   rC   z+to_parquet only supports IO with DataFrames)r7   Seriesrm   rM   r   r   r   )r   r#   objr   s       r"   
test_errorzTestBasic.test_error  s{    Iiii  L$$HYYY
 	D 	DC @C%%c6:sCCCC	D 	Dr$   c           	          t          j        t          d          t          t          dd                    d          }ddg|_        t          ||           d S )Nr;   r0   r<   r_   rd   r3   rX   )r7   r8   rK   rL   r   r   )r   r#   rP   s      r"   test_columns_dtypeszTestBasic.test_columns_dtypes  sR    \T%[[eAqkk9J9JKKLL U^
V$$$$$r$   rs   )Ngzipsnappybrotlic                 ^    t          j        dg di          }t          ||d|i           d S )Nr5   r/   rs   r   r7   r8   r   )r   r#   rs   rP   s       r"   test_compressionzTestBasic.test_compression  s:    \3			*++V=+2NOOOOOOr$   c           	          t          j        t          d          t          t          dd                    d          }t          j        dt          d          i          }t	          |||ddgi           d S )Nr;   r0   r<   r   r_   r   r   r   r7   r8   rK   rL   r   )r   r#   rP   r   s       r"   test_read_columnszTestBasic.test_read_columns  sz    \T%[[eAqkk9J9JKKLL<4;; 788	H:7N	
 	
 	
 	
 	
 	
r$   c           
          t          j        t          t          d                    t          d          d          }t          j        dddgi          }t	          ||||ddgid	gdgd
d           d S )Nr<   aabb)rd   partrd   r   r0   partition_colsr   )r   ==rF   )filtersr   )r   r   r   r   r|   r   )r   r#   tmp_pathrP   r   s        r"   test_read_filterszTestBasic.test_read_filters  s    \E!HH~~V 
 
 <A00*VH5%8$9ugNN	
 	
 	
 	
 	
 	
r$   c                    |dk    }t          j        dg di          }t          ||           g dt          j        dd          t	          d          g d	g}|D ]T}||_        t          |t           j                  r|j                            d           |_        t          |||
           Ug d|_        d|j        _	        t          ||           d S )Nr   r5   r/   )r1   r2   r<   rC   r2   rD   r;   )r0   r2   r<   )rv   )r   r0   r1   r3   )
r7   r8   r   rO   rK   indexr   DatetimeIndex
_with_freqname)r   r#   rv   rP   indexesr   s         r"   test_write_indexzTestBasic.test_write_index  s    -\3			*++V$$$ IIM*a000KKII	
  	B 	BEBH%!122 58..t44R[AAAAA 99V$$$$$r$   c                     |}t          j        dg di          }t           j                            g d          }||_        t          ||           d S )Nr5   r/   )rF   r0   )rF   r1   )rG   r0   )r7   r8   
MultiIndexfrom_tuplesr   r   )r   r*   r#   rP   r   s        r"   test_write_multiindexzTestBasic.test_write_multiindex  sX    \3			*++))*H*H*HIIV$$$$$r$   c           	         |}t          j        ddd          }t          j        t          j                            d                              dt          |          z  df          t          d                    }t           j	        
                    d	d
g|gddg          }|                    d           }||fD ]8}||_        t          ||           t          ||dddgi|ddg                    9d S )Nz01-Jan-2018z01-Dec-2018MS)freqr1   r2   ABCr   Level1Level2leveldate)r   r   r5   r6   r   r   )r7   rO   r8   rM   randomdefault_rngstandard_normallenrK   r  from_productcopyr   r   )r   r*   r#   datesrP   index1index2r   s           r"   test_multiindex_with_columnsz&TestBasic.test_multiindex_with_columns  s   m]FFF\I!!!$$44a#e**na5HIIKK
 
 
 ++!5)'61B , 
 
 4((f% 	 	EBHR(((FS#J(?"cSVZ.    		 	r$   c                    t          j        g dg dd          }d dd}|                    d          }t          ||||           t          j        g dg ddg d	
          }t          ||||           g dg dg}t          j        t	          t          d                    d t          d          D             d|
          }|                    d          }t          ||||           d S )Nr/   )qrs)rF   rG   F)rs   r   T)dropr   r   )zyxwvutsrr   rX   rX   rY   rY   r3   r3   quxr)  onetwor+  r,  r+  r,  r+  r,     c                     g | ]}| S r)   r)   ).0is     r"   
<listcomp>z7TestBasic.test_write_ignoring_index.<locals>.<listcomp>  s    +A+A+A1QB+A+A+Ar$   )r+  r,  )r7   r8   reset_indexr   rK   rL   )r   r#   rP   r   r   arrayss         r"   test_write_ignoring_indexz#TestBasic.test_write_ignoring_index  s=    \			@@AA'+e<< >>t>,,V,RRRR \))///22:O:O:O
 
 
 	V,RRRR EDDDDD
 \qNN+A+Aa+A+A+ABB&
 
 
 >>t>,,V,RRRRRRr$   c                 N   t           j                            g d          }t          j        t          j                            d                              d          |          }|dk    r|                     ||t          d           d S |dk    rt          ||           d S d S )Nr  r1   )r<   r2   r  r   Column name must be a stringr   )r7   r  r  r8   rM   r  r  r  r   	TypeErrorr   )r   r#   
mi_columnsrP   s       r"   test_write_column_multiindexz&TestBasic.test_write_column_multiindex  s    ]../M/M/MNN
\I!!!$$44V<<j
 
 
 ]""%%FI'E     y  R((((( ! r$   c                 8   g dg dg}t          j        t          j                            d                              d          |          }ddg|j        _        |dk    r|                     ||t          d	           d S |d
k    rt          ||           d S d S )Nr(  )r0   r1   r0   r1   r0   r1   r0   r1   r1   r-  r-  r  r  r  r   zColumn namer   )r7   r8   rM   r  r  r  r   r   r   r   r   r   r#   r3  rP   s       r"   &test_write_column_multiindex_nonstringz0TestBasic.test_write_column_multiindex_nonstring%  s    
 EDD$$$
 \I!!!$$44V<<f
 
 
 %h/
]""%%b&*mLLLLLy  R((((( ! r$   c                     |}g dg dg}t          j        t          j                            d                              d          |          }ddg|j        _        t          ||           d S )Nr(  r*  r1   r;  r  	ColLevel1	ColLevel2)	r7   r8   rM   r  r  r  r   r   r   r   r*   r#   r3  rP   s        r"   #test_write_column_multiindex_stringz-TestBasic.test_write_column_multiindex_string6  s      EDDDDD
 \I!!!$$44V<<f
 
 
 (5
V$$$$$r$   c                     |}g d}t          j        t          j                            d                              d          |          }d|j        _        t          ||           d S )N)rX   rY   r3   r)  r1   r-  r<   r  	StringCol)	r7   r8   rM   r  r  r  r   r  r   rA  s        r"   test_write_column_index_stringz(TestBasic.test_write_column_index_stringG  sp      .--\I!!!$$44V<<f
 
 
 &
V$$$$$r$   c                    g d}t          j        t          j                            d                              d          |          }d|j        _        |dk    r|                     ||t          d           d S t          ||           d S )Nr0   r1   r2   r<   r1   rD  r  NonStringColr   r6  )r7   r8   rM   r  r  r  r   r  r   r7  r   r<  s       r"   !test_write_column_index_nonstringz+TestBasic.test_write_column_index_nonstringU  s     \I!!!$$44V<<f
 
 
 )
]""%%FI'E     R(((((r$   c                    t          j        d          }|dk    r5t           j                            d          }|                    |           t          j        t          j        g dd          t          j        g dd          t          j        g d          t          j        g d	          t          j        g d
d          t          j        g dd          t          j        g dd          d          }t          j	                    5 }|
                    ||           t          ||          }t          ||d          }d d d            n# 1 swxY w Y   |d         j        t          j        d          k    sJ t          j        t          j        g dd          t          j        g dd          t          j        g dd          t          j        g d	d          t          j        g d
d          t          j        g dd          t          j        g dd          d          }	|dk    r.|                    dd          }|	                    dd          }	t          j        ||	           d S )Nzpyarrow.parquetr   z.Fastparquet nullable dtype support is disabledr   r0   r1   r2   Nint64uint8)rF   rG   rS   N)TFTNrH  )      ?r\   r]   Nfloat32r?   )rF   rG   rS   rH   rI   rJ   gr   numpy_nullabler#   dtype_backendrF   Int64r@   UInt8r_   booleanFloat32Float64rS   r0   )axis)r'   r   r   xfailapplymarkerr   tabler   rz   r   write_tabler   rA   rM   r7   r8   r"  r{   )
r   r#   r!   pqr   r]  r   result1result2r   s
             r"   test_dtype_backendzTestBasic.test_dtype_backende  s    !233]"" ;$$G %  D %%%]???G<<]???G<<]#8#8#899]#<#<#<==]<<<99]#8#8#8)DD]#8#8#8)DD
 

 
 _ 	X$NN5$'''"4777G"4FVWWWG		X 	X 	X 	X 	X 	X 	X 	X 	X 	X 	X 	X 	X 	X 	X s|!RXi%8%88888<XoooW===XoooW===X3338DDDX777yIIIXlll':::X3339EEEX3339EEE 

 

 ]"" ll3Ql//G}}Sq}11H
gx00000s   :EEErA   )	rU  rV  rW  objectzdatetime64[ns, UTC]rf   z	period[D]rY  r_   c                     t          j        dt          j        g |          i          }d }|dk    r*t          j        dt          j        g d          i          }t          ||ddi|           d S )Nvaluer@   rf   rY  rT  rR  r  )r7   r8   r   r   )r   r*   rA   rP   r   s        r"   test_read_empty_arrayzTestBasic.test_read_empty_array  s      \"E222
 
 G|RXb	::: H
 	2B Ch	
 	
 	
 	
 	
 	
r$   N)r   r   r   r   r   r'   r   parametrizer   r   r   r  r	  r  r4  r9  r=  rB  rF  rJ  rb  rf  r)   r$   r"   r   r     si       	D 	D 	D% % % [],N,N,NOOP P POP
 
 

 
 
&% % %0% % %  &S S S@) ) )) ) )"% % %"% % %) ) ) /1 /1 /1b [
	
 
	
 
	
 
 
 
 
 
r$   r   c                      e Zd Zd Zd Zd Zd Zd Zd Zd Z	e
j                             e            d	          e
j                            e d
	          e
j                            deej        g          d                                     Zd Ze
j        j        d             Ze
j        j        d             Ze
j        j        e
j                            ddgg g          d                         Zd Zd Zd Zd Ze
j                            ded gddg          d             Zd Zd Z d Z!d Z"d  Z#d! Z$d" Z%d# Z&d$ Z'd% Z(d& Z)d' Z*d( Z+d) Z,d* Z-e
j                            e.d+	          d,             Z/d- Z0d.S )/TestParquetPyArrowc                     |}t          j        ddd          }|                    d           }||d<   g d|d<   t          ||           d S )NrC   r2   Europe/BrusselsrE   tzdatetime_tzTNTbool_with_none)r7   rO   r  r   )r   r*   ro   rP   dtis        r"   
test_basiczTestParquetPyArrow.test_basic  sb     mJ6GHHHnnT""=111R     r$   c                 |    |}t          j        ddd          |d<   t          |||ddg         dddgi	           d S )
NrC   r2   rk  rl  rn  r_   rd   r   r   )r7   rO   r   )r   r*   ro   rP   s       r"   test_basic_subset_columnsz,TestParquetPyArrow.test_basic_subset_columns  sf     M*aDUVVV=5)*"Xu$56		
 	
 	
 	
 	
 	
r$   c                    |                     |          }t          |t                    sJ t          |          }t	          |          }|                                }d |j        d<   t          j        ||           d S )Nr   ru   )	r   r   rb   r   r   r  ry   rz   r{   )r   r*   ro   	buf_bytes
buf_streamresr   s          r"   *test_to_bytes_without_path_or_buf_providedz=TestParquetPyArrow.test_to_bytes_without_path_or_buf_provided  s{    &&b&11	)U+++++Y''
:&&<<>>-1)*
c8,,,,,r$   c                     t          j        t          j        d                              dd          t          d                                                    }|                     ||t          d           d S )N   r<   r2   aaar  zDuplicate column names found	r7   r8   rM   rN   reshaperK   r  r   r   r   r*   rP   s      r"   test_duplicate_columnsz)TestParquetPyArrow.test_duplicate_columns  sa    \")B--//155tE{{KKKPPRR!!"b*6TUUUUUr$   c                 z    t          j        dt          j        dd          i          }t          ||           d S )NrF   1 dayr2   rD   )r7   r8   timedelta_ranger   r  s      r"   test_timedeltaz!TestParquetPyArrow.test_timedelta  s<    \3 27A F F FGHHR     r$   c                 x    t          j        dg di          }|                     ||t          j                   d S )NrF   rF   r0   r\   )r7   r8   r   r   ArrowExceptionr  s      r"   test_unsupportedz#TestParquetPyArrow.test_unsupported  s=    \3.// 	**2r73IJJJJJr$   c                     t          j        ddt           j                  }t          j        |dg          }t
          r#|                     ||t          j                   d S t          ||           d S )Nr1   
   r@   fp16r   r   )
rM   rN   float16r7   r8   r
   r   r   r  r   )r   r*   r   rP   s       r"   test_unsupported_float16z+TestParquetPyArrow.test_unsupported_float16  sq     yBbj111\tfX666 	%..r2w7MNNNNNR$$$$$r$   zqPyArrow does not cleanup of partial files dumps when unsupported dtypes are passed to_parquet function in windowsr   zfloat16 works on 15	path_typec                    t          j        ddt           j                  }t          j        |dg          }t          j                    5 } ||          }t          j        t          j	                  5  |
                    ||           d d d            n# 1 swxY w Y   t          j                            |          rJ 	 d d d            d S # 1 swxY w Y   d S )Nr1   r  r@   r  r  )r   r#   )rM   rN   r  r7   r8   rz   r   r   r   r  r   osr   isfile)r   r*   r  r   rP   path_strr   s          r"    test_unsupported_float16_cleanupz3TestParquetPyArrow.test_unsupported_float16_cleanup  sL    yBbj111\tfX666_ 	,(9X&&D)'*@AA 4 443334 4 4 4 4 4 4 4 4 4 4 4 4 4 4w~~d++++++		, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	,s6   *C6BCB	C!B	"$CCCc                 .   t          j                    }t          j        t          d                    |d<   t          j        g dt          j        g d                    |d<   t          j        g dg dd	
          |d<   t          ||           d S )NabcdefrF   )rX   r3   r3   rX   NrX   rW   r@   rG   )rF   rG   rS   rF   rS   rG   )rG   rS   rH   T)
categoriesorderedrS   )r7   r8   CategoricalrK   CategoricalDtyper   r  s      r"   test_categoricalz#TestParquetPyArrow.test_categorical  s    \^^.h003 .555%&;&;&;<<
 
 
3 .***PT
 
 
3 	R     r$   c                     t          j        d          } |j        di |}d|i}t          |||j         d||           d S )Ns3fs
filesystem/pyarrow.parquetr   r   r   r)   )r'   r   S3FileSystemr   r  )r   r9   s3_public_bucketr*   s3sor  s3kws           r"   test_s3_roundtrip_explicit_fsz0TestParquetPyArrow.test_s3_roundtrip_explicit_fs   sp    "6**T&&&&B$);;;	
 	
 	
 	
 	
 	
r$   c                 H    d|i}t          ||d|j         d||           d S )Nstorage_optionss3://r  r  r   r  )r   r9   r  r*   r  s        r"   test_s3_roundtripz$TestParquetPyArrow.test_s3_roundtrip-  sM     "4(@).@@@	
 	
 	
 	
 	
 	
r$   partition_colr5   c                 V   t          j        d           |                                }|rX|                    t                              |t          j                            }d}||                             |          ||<   t          |||d|j	         dd|i|d |ddd	           d S )
Nr  categoryr  z/parquet_dirr  )r   rs   r  Tr0   )r   r   r   r   rw   r|   )
r'   r   r  rl   dictfromkeysrM   int32r   r  )r   r9   r  r*   r  r  expected_dfpartition_col_types           r"   test_s3_roundtrip_for_dirz,TestParquetPyArrow.test_s3_roundtrip_for_dir9  s     	F###nn&&  	%,,T]]="(-S-STTK!+)4])C)J)J"* *K& 	 <).<<<*D1"/##' 
 	
 	
 	
 	
 	
 	
r$   c                     t          j        d           t                      }|                    |           t	          |          }t          j        ||           d S )Nr   )r'   r   r   r   r   rz   r{   )r   r9   bufferdf_from_bufs       r"   test_read_file_like_obj_supportz2TestParquetPyArrow.test_read_file_like_obj_support`  sV    I&&&V$$$"6**
i55555r$   c                    t          j        d           |                    dd           |                    dd           t          j        t          d          5  t          d           d d d            n# 1 swxY w Y   t          j        t          d          5  |                    d           d d d            d S # 1 swxY w Y   d S )Nr   HOMETestingUserUSERPROFILEz.*TestingUser.*r   z~/file.parquet)r'   r   setenvr   OSErrorr   r   )r   r9   monkeypatchs      r"   test_expand_userz#TestParquetPyArrow.test_expand_userg  sH   I&&&6=111=-888]7*<=== 	+ 	+)***	+ 	+ 	+ 	+ 	+ 	+ 	+ 	+ 	+ 	+ 	+ 	+ 	+ 	+ 	+]7*<=== 	3 	3  !1222	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	3s$   A88A<?A<CCCc                     ddg}|}|                     ||d            t          ||           t          |          j        |j        k    sJ d S )Nrh   rd   r   rs   r   r   r   shape)r   r   r*   ro   r   rP   s         r"   test_partition_cols_supportedz0TestParquetPyArrow.test_partition_cols_supportedp  s\     %
h~4PPPh777H%%+rx777777r$   c                     d}|g}|}|                     ||d            t          ||           t          |          j        |j        k    sJ d S )Nrh   r  r  )r   r   r*   ro   r   partition_cols_listrP   s          r"   test_partition_cols_stringz-TestParquetPyArrow.test_partition_cols_stringx  sb    -.
h~4PPPh(;<<<H%%+rx777777r$   c                     | S r   r)   )xs    r"   <lambda>zTestParquetPyArrow.<lambda>  s    Q r$   r_   zpathlib.Path)idsc                     d}|g}|} ||          }|                     ||           t          |          j        |j        k    sJ d S )Nr6   )r   )r   r   r  )	r   r   r*   r9   r  r   r  rP   r   s	            r"   test_partition_cols_pathlibz.TestParquetPyArrow.test_partition_cols_pathlib  sa     -.y""
d+>???D!!'28333333r$   c                 R    t          j        g g           }t          ||           d S )N)r   r   r   r  s      r"   test_empty_dataframez'TestParquetPyArrow.test_empty_dataframe  s,    \B///R     r$   c                     dd l }t          j        dddgi          } |j         |j        d |j                              g          }|                    t                    }t          ||d|i|           d S )Nr   r  r0   )typer   r#  )	r   r7   r8   r   fieldbool_rl   rh   r   )r   r*   r   rP   r   out_dfs         r"   test_write_with_schemaz)TestParquetPyArrow.test_write_with_schema  s    \3A-((s!I!I!I JKK4Rx.@6RRRRRRr$   c                 z   t          j        d           t          j        t          j        g dd          t          j        g dd          t          j        g dd          d          }t          ||           t          j        d	t          j        g d
d          i          }t          ||           d S )Nr   r/   rU  r@   UInt32rT   r_   rF   rG   rS   rF   rL  )r'   r   r7   r8   r   r   r  s      r"    test_additional_extension_arraysz3TestParquetPyArrow.test_additional_extension_arrays  s     	I&&&\Yyyy888Yyyy999Y///x@@@ 
 
 	R   \3	/// I I IJKKR     r$   c           
      4   t          j        d           t          j        dt          j        g dd          i          }t          j        d|          5  t          |||                    d| d          	           d d d            d S # 1 swxY w Y   d S )
Nr   rF   rT   string[pyarrow]r@   string_storagezstring[]r   )r'   r   r7   r8   r   r   r   rl   )r   r*   r  rP   s       r"    test_pyarrow_backed_string_arrayz3TestParquetPyArrow.test_pyarrow_backed_string_array  s    I&&&\3	*:*:*:BS T T TUVV/@@ 	V 	VRbii8S.8S8S8S.T.TUUUU	V 	V 	V 	V 	V 	V 	V 	V 	V 	V 	V 	V 	V 	V 	V 	V 	V 	Vs   *BBBc                 J   t          j        d           t          j        t          j                            g d          t          j        ddd          t          j                            t          j        ddd                    d          }t          ||           d S )	Nr   ))r   r0   )r0   r1   )r2   r<   z
2012-01-01r2   D)rE   r  r<   )rS   rH   rI   )
r'   r   r7   r8   IntervalIndexr  period_rangefrom_breaksrO   r   r  s      r"   test_additional_extension_typesz2TestParquetPyArrow.test_additional_extension_types  s     	I&&&\%112J2J2JKK_\13GGG%11M,DDD 	 	
 	
 	R     r$   c                     d}t          j        dt          j        ddd          i          }t          ||d|i           d S )	Nz2.6rF   z
2017-01-011nsr  r  rE   versionr   )r7   r8   rO   r   )r   r*   verrP   s       r"   test_timestamp_nanosecondsz-TestParquetPyArrow.test_timestamp_nanoseconds  sO     \3lPR S S STUURy#.>??????r$   c                     |j         t          j        j        k    r3|                    t
          j                            d                     d|gz  }t          j	        |d|i          }t          ||d           d S )Nzitemporary skip this test until it is properly resolved: https://github.com/pandas-dev/pandas/issues/37286r      index_as_colr   r   F)rx   )tzinfori   timezoneutcr\  r'   r   r[  r7   r8   r   )r   r!   r*   rq   idxrP   s         r"   test_timezone_aware_indexz,TestParquetPyArrow.test_timezone_aware_index  s    #*h.?.CCC!!H "     +,,\>3*?@@@ 	RU333333r$   c                 b   t          j        d           t          j        dt	          t          d                    i          }t          j                    5 }|                    ||           t          ||dg          }d d d            n# 1 swxY w Y   t          |          dk    sJ d S )Nr   rF   r2   r   rF   r   r   r   r0   )r'   r   r7   r8   rK   rL   rz   r   r   r   r  )r   r*   rP   r   r   s        r"   test_filter_row_groupsz)TestParquetPyArrow.test_filter_row_groups  s    I&&&\3U1XX/00_ 	F$MM$rM***!$^4DEEEF	F 	F 	F 	F 	F 	F 	F 	F 	F 	F 	F 	F 	F 	F 	F 6{{as   +BBBc                    t          j        t          j                            d                              d          g d          }t          j                    5 }|                    ||           t          ||          }d d d            n# 1 swxY w Y   |r-t          |j        t           j        j        j                  sJ d S t          |j        t           j        j        j                  sJ d S )Nr1   )r  r2   )r5   r6   Cr  r   )r7   r8   rM   r  r  r  rz   r   r   r   r   _mgrcore	internalsArrayManagerBlockManager)r   r*   using_array_managerrP   r   r   s         r"   test_read_parquet_managerz,TestParquetPyArrow.test_read_parquet_manager  s   \I!!!$$44W==
 
 
 _ 	,$MM$rM***!$++F	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	,  	Kfk27+<+IJJJJJJJfk27+<+IJJJJJJJs   (BBBc                 &   dd l }|}t          j        ddd          }|                    d           }||d<   g d|d<   |j                            |          }|                    t          j        	          }t          r||d
         	                    d          |d
<   |d         	                    d          |d<   |d         	                    t          j         |j
        dd                              |d<   t          ||ddi|           d S )Nr   rC   r2   rk  rl  rn  ro  rp  )types_mapperri   ztimestamp[us][pyarrow]rj   us)unitrm  rT  r   r#   r   r   )r   r7   rO   r  Tablefrom_pandas	to_pandas
ArrowDtyper	   rl   	timestampr   )r   r*   ro   r   rP   rq  pa_tabler   s           r"   &test_read_dtype_backend_pyarrow_configz9TestParquetPyArrow.test_read_dtype_backend_pyarrow_config  sB    mJ6GHHHnnT""=111=,,R00%%2=%AA 	#+J#7#>#>?W#X#XHZ ,45H,I,P,P(- -H() '/}&=&D&D/g/T>OPPPQQ' 'H]# 	()4		
 	
 	
 	
 	
 	
r$   c                 J   t          j        dddgit          j        ddgd          d	          }|                                }d
d l}t          |j                  t          d          k    r|j                            d          |_        t          ||ddi|           d S )NrF   r0   r1   r2   r<   testr  zint64[pyarrow])r   rA   r   z11.0.0rT  r   r  )
r7   r8   Indexr  r   r   r   r   rl   r   )r   r*   rP   r   r   s        r"   ,test_read_dtype_backend_pyarrow_config_indexz?TestParquetPyArrow.test_read_dtype_backend_pyarrow_config_index  s    \1a&M1a&v!>!>!>FV
 
 
 77997&'''(*;*;;;%^223CDDHN()4		
 	
 	
 	
 	
 	
r$   c           	         t          j        t          d          t          t          dd                    d          }ddg|_        t          ||           ddg|_        t          j        t          d	          5  t          ||           d d d            n# 1 swxY w Y   t          j	        d
dddd          t          j	        d
dddd          g|_        t          ||           d S )Nr;   r0   r<   r   r   rU   rV   z|S3r     )
r7   r8   rK   rL   r   r   r'   r   NotImplementedErrorri   r  s      r"   test_columns_dtypes_not_invalidz2TestParquetPyArrow.test_columns_dtypes_not_invalid&  s$   \T%[[eAqkk9J9JKKLL V
R    f%
].e<<< 	% 	%R$$$	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% dAq!Q//dAq!Q//

 	R     s   =BB!Bc                 |    t          j        t          j        g dd                    }t          ||           d S )Nr  custom namer  r'  r7   r8   r  r   r  s      r"   test_empty_columnsz%TestParquetPyArrow.test_empty_columns:  s=    \} M M MNNNR     r$   c                     |dz  }t          j        ddgi          }ddi|_        |                    ||           t	          ||          }|j        |j        k    sJ d S )Nztest_df_metadata.pr0   r   test_attributer   )r7   r8   attrsr   r   )r   r   r*   r   rP   new_dfs         r"   test_df_attrs_persistencez,TestParquetPyArrow.test_df_attrs_persistence?  st    ..\A3x((($a(
d2&&&d2...|rx''''''r$   c                 ~   |dz  }t          j        dddgiddg          }|                    |d           t          j        d	d
          5  t	          |d          }d d d            n# 1 swxY w Y   t          j        dddgidt          j        ddgd                    }t          j        ||           d S )Nztest_string_inference.prF   r  yrG   )r   r   r   r   future.infer_stringTstring[pyarrow_numpy]r@   )r   rA   r   )r7   r8   r   r   r   r  rz   r{   r   r   r*   r   rP   r   r   s          r"   test_string_inferencez(TestParquetPyArrow.test_string_inferenceG  s   33\c3Z0c
CCC
d9---4d;; 	: 	:!$y999F	: 	: 	: 	: 	: 	: 	: 	: 	: 	: 	: 	: 	: 	: 	:<Sz")(C:-DEEE
 
 

 	fh/////s   A,,A03A0znot supported before 11.0c           	      ^   dd l }|dz  }t          j        dt          d          gid          }|                    ||                    d|                    d          fg                     t          |          }t          j        dd	gid
          }t          j	        ||           d S )Nr   z	decimal.prF   z123.00r  r@   r  )r   123zstring[python])
r   r7   r8   r   r   r   
decimal128r   rz   r{   r&  s          r"   test_roundtrip_decimalz)TestParquetPyArrow.test_roundtrip_decimalU  s     	+%\3!2!2 34<MNNN
d299sBMM!4D4D.E-F#G#GHHHd##<ug6FGGG
fh/////r$   c                    dd l }dd lm} |dz  }|                    d|                    g d|                                          i          }|                    ||           t          j        dd          5  t          |          }d d d            n# 1 swxY w Y   t          j
        dg didt          j        dgd          	          }t          j        ||           d S )
Nr   zlarge_string.prF   )NrG   rS   r$  Tr%  r@   )r   rA   r   )r   pyarrow.parquetr   r]  r   large_stringr^  r7   r   r   r8   r  rz   r{   )r   r   r*   r_  r   r]  r   r   s           r"   #test_infer_string_large_string_typez6TestParquetPyArrow.test_infer_string_large_string_typea  sB   $$$$$$**#rxx(8(8(8"//:K:KLLMNN
ud###4d;; 	( 	(!$''F	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	(<'''()HcU*ABBB
 
 

 	fh/////s   :BBBN)1r   r   r   rr  rt  ry  r  r  r  r  r'   r   r[  r   skipifr
   rg  strpathlibPathr  r  r   r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r!  r'  r   r+  r/  r)   r$   r"   ri  ri    sj       	! 	! 	!
 
 

- 
- 
-V V V
! ! !K K K% % % [?    [009NOO[[3*=>>, , ?> PO ,! ! !$ [

 

 

 [	
 	
 	
 [[E	
 
 
  
>6 6 63 3 38 8 88 8 8 [c;;'h-G   	4 	4 	4! ! !
S S S! ! ! V V V! ! ! @ @ @4 4 4,     K K K
 
 
<
 
 
 ! ! !(! ! !
( ( (0 0 0 [,5PQQ	0 	0 RQ	00 0 0 0 0r$   ri  c                   V   e Zd Zd Zd Zd Zej                             e	e
j                   e	d          k    d          d             Zd Zd	 Zd
 Zej        j        d             Zd Zd Zd Zd Zej                             e            d          d             Zej                            eo e	ej                   e	d          k    d          d             Zd Zd Zd Zd Zd Z d Z!d Z"ej                             e            d          d             Z#dS )TestParquetFastParquetc                     |}t          j        ddd          }|                    d           }||d<   t          j        dd          |d<   t	          ||           d S )	NrC   r2   z
US/Easternrl  rn  r  rD   	timedelta)r7   rO   r  r  r   )r   r-   ro   rP   rq  s        r"   rr  z!TestParquetFastParquet.test_basic  sg    mJlCCCnnT""=,Wa@@@;R     r$   c           	         t          j        t          d          t          t          dd                    d          }t          }d}ddg|_        |                     ||||           ddg|_        |                     ||||           t          j        d	dddd          t          j        d	dddd          g|_        |                     ||||           d S )
Nr;   r0   r<   r   r6  r   rU   rV   r  )r7   r8   rK   rL   r7  r   r   ri   )r   r-   rP   errr   s        r"   test_columns_dtypes_invalidz2TestParquetFastParquet.test_columns_dtypes_invalid  s    \T%[[eAqkk9J9JKKLL, V
!!"b#s333 f%
!!"b#s333 dAq!Q//dAq!Q//

 	!!"b#s33333r$   c                     t          j        t          j        d                              dd          t          d                                                    }d}|                     ||t          |           d S )Nr{  r<   r2   r|  r  z9Cannot create parquet dataset with duplicate column namesr}  r   r-   rP   r   s       r"   r  z-TestParquetFastParquet.test_duplicate_columns  se    \")B--//155tE{{KKKPPRRI!!"b*c:::::r$   z2.0.0z$fastparquet uses np.float_ in numpy2r   c                     t          j        dg di          }t          j        ddt          j        dgid          }t	          |||d           d S )	NrF   )TNFrO  g        r  r@   F)r   rx   )r7   r8   rM   rk   r   r   r-   rP   r   s       r"   test_bool_with_nonez*TestParquetFastParquet.test_bool_with_none  s`    
 \3 3 3 3455<sBFC&8 9KKK 	R(FFFFFFr$   c                    t          j        dt          j        ddd          i          }|                     ||t          d            t          j        dg di          }d}|                     ||t          |           d S )NrF   2013Mr2   r  r  z"Can't infer object conversion type)r7   r8   r  r   r   r<  s       r"   r  z'TestParquetFastParquet.test_unsupported  s    \3S! L L LMNN!!"b*d;;; \3.//2!!"b*c:::::r$   c                     t          j        dt          j        t          d                    i          }t	          ||           d S )NrF   r;   )r7   r8   r  rK   r   )r   r-   rP   s      r"   r  z'TestParquetFastParquet.test_categorical  s<    \3tE{{ ; ;<==R     r$   c                 B   dt          t          d                    i}t          j        |          }t	          j                    5 }|                    ||d d           t          ||dg          }d d d            n# 1 swxY w Y   t          |          dk    sJ d S )NrF   r2   r0   )r#   rs   row_group_offsetsr  r  )	rK   rL   r7   r8   rz   r   r   r   r  )r   r-   rH   rP   r   r   s         r"   r  z-TestParquetFastParquet.test_filter_row_groups  s    $uQxx..!\!___ 	F$MM$rtqMQQQ!$^4DEEEF	F 	F 	F 	F 	F 	F 	F 	F 	F 	F 	F 	F 	F 	F 	F 6{{as   -A??BBc           	      J    t          ||d|j         dd|id |d           d S )Nr  z/fastparquet.parquetr  )rs   r  r  r  )r   r9   r  r-   r  s        r"   r  z(TestParquetFastParquet.test_s3_roundtrip  sP     	D).DDD*D1)-$GG	
 	
 	
 	
 	
 	
r$   c                     ddg}|}|                     |d|d            t          j                            |          sJ dd l} |j        t          |          d          j        }t          |          dk    sJ d S )Nrh   rd   r   r#   r   rs   r   Fr1   	r   r  r   existsr   ParquetFiler1  catsr  r   r   r-   ro   r   rP   r   actual_partition_colss           r"   r  z4TestParquetFastParquet.test_partition_cols_supported  s     %
 )	 	 	
 	
 	
 w~~h''''' 7 7Hu M M R())Q......r$   c                     d}|}|                     |d|d            t          j                            |          sJ dd l} |j        t          |          d          j        }t          |          dk    sJ d S )Nrh   r   rH  r   Fr0   rI  rM  s           r"   r  z1TestParquetFastParquet.test_partition_cols_string  s    
 )	 	 	
 	
 	
 w~~h''''' 7 7Hu M M R())Q......r$   c                     ddg}|}|                     |dd |           t          j                            |          sJ dd l} |j        t          |          d          j        }t          |          dk    sJ d S )Nrh   rd   r   )r#   rs   partition_onr   Fr1   rI  rM  s           r"   test_partition_on_supportedz2TestParquetFastParquet.test_partition_on_supported  s     %
 '	 	 	
 	
 	
 w~~h''''' 7 7Hu M M R())Q......r$   c                     ddg}|}d}t          j        t          |          5  |                    |dd ||           d d d            d S # 1 swxY w Y   d S )Nrh   rd   zYCannot use both partition_on and partition_cols. Use partition_cols for partitioning datar   r   )r#   rs   rQ  r   )r'   r   r   r   )r   r   r-   ro   r   rP   r   s          r"   3test_error_on_using_partition_cols_and_partition_onzJTestParquetFastParquet.test_error_on_using_partition_cols_and_partition_on  s     !%  	 ]:S111 	 	MM$ +-    	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	s   AAAzfastparquet writes into Indexc                 x    t          j                    }|                                }t          |||           d S )Nr  )r7   r8   r  r   r>  s       r"   r  z+TestParquetFastParquet.test_empty_dataframe  s7     \^^7799R(333333r$   z2022.12zCfastparquet bug, see https://github.com/dask/fastparquet/issues/929c                     d|gz  }t          j        |d|i          }|                                }d|j        _        t          |||           d S )Nr  r  r  r   r  )r7   r8   r  r   r  r   )r   r-   rq   r  rP   r   s         r"   r  z0TestParquetFastParquet.test_timezone_aware_index  s]    
 +,,\>3*?@@@7799%R(333333r$   c                 .   t          j        dddgi          }t          j                    5 }|                    |           t          j        t          d          5  t          j        t                    5  t          |dd           d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   t          j        t          d          5  t          |dd	
           d d d            n# 1 swxY w Y   d d d            d S # 1 swxY w Y   d S )NrF   r0   r1   z!not supported for the fastparquetr   r   T)r#   use_nullable_dtypesr   rS  )r7   r8   rz   r   r   r'   r   r   r   r   r   )r   r-   rP   r   s       r"   &test_use_nullable_dtypes_not_supportedz=TestParquetFastParquet.test_use_nullable_dtypes_not_supported*  sI   \3A-((_ 	R$MM$z1TUUU W W/>> W W mQUVVVVW W W W W W W W W W W W W W WW W W W W W W W W W W W W W W z1TUUU R RT-yQQQQR R R R R R R R R R R R R R R	R 	R 	R 	R 	R 	R 	R 	R 	R 	R 	R 	R 	R 	R 	R 	R 	R 	Rs}   1D
B-7B
B-BB-BB-!D
-B1	1D
4B1	5D
C2&D
2C6	6D
9C6	:D

DDc                    t          j        d          5 }t          j        |                              d           t          j        t          d          5  t          |d           d d d            n# 1 swxY w Y   t          j        |          	                    d           d d d            d S # 1 swxY w Y   d S )	Ntest.parquets   breakit r   r   r   F)
missing_ok)
rz   r   r2  r3  write_bytesr'   r   	Exceptionr   unlink)r   r   s     r"   $test_close_file_handle_on_read_errorz;TestParquetFastParquet.test_close_file_handle_on_read_error5  s1   _^,, 	8L**:666y333 9 9T-88889 9 9 9 9 9 9 9 9 9 9 9 9 9 9 L%%%777	8 	8 	8 	8 	8 	8 	8 	8 	8 	8 	8 	8 	8 	8 	8 	8 	8 	8s6   AB6A6*B66A:	:B6=A:	>+B66B:=B:c                    t          j        ddgddgd          }t          j        d          5 }t	          |                                d          5 }|                    |           d d d            n# 1 swxY w Y   t          ||          }d d d            n# 1 swxY w Y   t          j        ||           d S )Nr   r0   r4   r  r[  wbr   )	r7   r8   rz   r   r   encoder   r   r{   )r   r#   rP   r   rJ   r   s         r"   test_bytes_file_namez+TestParquetFastParquet.test_bytes_file_name=  s1   \aV1a&99:::_^,, 	7dkkmmT** !aa   ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! "$v666F		7 	7 	7 	7 	7 	7 	7 	7 	7 	7 	7 	7 	7 	7 	7
 	fb)))))s5   #BA6*B6A:	:B=A:	>BB"%B"c                    t          j        d           t          j        ddgddgd          }t	          j                    5 }t          j        t          d          5  |                    |dd	           d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   t	          j                    5 }t          j
        |                              d
           t          j        t          d          5  t          |dd	           d d d            n# 1 swxY w Y   d d d            d S # 1 swxY w Y   d S )Nr   r   r0   r4   r  zfilesystem is not implementedr   r3   r#   r  rU   )r'   r   r7   r8   rz   r   r   r  r   r2  r3  r^  r   r   rP   r   s      r"   test_filesystem_notimplementedz5TestParquetFastParquet.test_filesystem_notimplementedG  s~   M***\aV1a&99:::_ 	L$#+J   L L d=UKKKL L L L L L L L L L L L L L L	L 	L 	L 	L 	L 	L 	L 	L 	L 	L 	L 	L 	L 	L 	L _ 	K$L**6222#+J   K K T-EJJJJK K K K K K K K K K K K K K K	K 	K 	K 	K 	K 	K 	K 	K 	K 	K 	K 	K 	K 	K 	K 	K 	K 	Km   B B9BB			BB		BB #B :AD4=DD4D 	 D4#D 	$D44D8;D8c                    t          j        d           t          j        ddgddgd          }t	          j                    5 }t          j        t          d          5  |                    |dd	           d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   t	          j                    5 }t          j
        |                              d
           t          j        t          d          5  t          |dd	           d d d            n# 1 swxY w Y   d d d            d S # 1 swxY w Y   d S )Nr   r   r0   r4   r  z1filesystem must be a pyarrow or fsspec FileSystemr   r3   rg  rU   )r'   r   r7   r8   rz   r   r   r   r   r2  r3  r^  r   rh  s      r"   test_invalid_filesystemz.TestParquetFastParquet.test_invalid_filesystemW  s~   I&&&\aV1a&99:::_ 	H$"U   H H d9GGGH H H H H H H H H H H H H H H	H 	H 	H 	H 	H 	H 	H 	H 	H 	H 	H 	H 	H 	H 	H _ 	G$L**6222"U   G G T)FFFFG G G G G G G G G G G G G G G	G 	G 	G 	G 	G 	G 	G 	G 	G 	G 	G 	G 	G 	G 	G 	G 	G 	Grj  c           	         t          j        d          }t          j        ddgddgd          }t	          j                    5 }t          j        t          d          5  |                    |d|	                                d	d
i           d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   t	          j                    5 }t          j        |                              d           t          j        t          d          5  t          |d|	                                d	d
i           d d d            n# 1 swxY w Y   d d d            d S # 1 swxY w Y   d S )Nz
pyarrow.fsr   r0   r4   r  z8storage_options not supported with a pyarrow FileSystem.r   r   r3   rX   )r#   r  r  rU   )r'   r   r7   r8   rz   r   r   r  r   LocalFileSystemr2  r3  r^  r   )r   pa_fsrP   r   s       r"   .test_unsupported_pa_filesystem_storage_optionszETestParquetFastParquet.test_unsupported_pa_filesystem_storage_optionsg  su   #L11\aV1a&99:::_ 
	$#P   	 	 $$4466%*EN	    		 	 	 	 	 	 	 	 	 	 	 	 	 	 	
	 
	 
	 
	 
	 
	 
	 
	 
	 
	 
	 
	 
	 
	 
	 _ 	$L**6222#P   	 	 $$4466%*EN	   		 	 	 	 	 	 	 	 	 	 	 	 	 	 		 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	sm   B1 .BB1B	B1!B	"B11B58B5AE(E:EE
	
EE
	EE"%E"c           	      x   d}t          j        dt          t          dd                    i          }t	          j        d          5 }|                    |           t          j        t          |          5  t          |d           d d d            n# 1 swxY w Y   d d d            d S # 1 swxY w Y   d S )	NzPdtype_backend numpy is invalid, only 'numpy_nullable' and 'pyarrow' are allowed.rd   r0   r<   ztmp.parquetr   numpy)rT  )r7   r8   rK   rL   rz   r   r   r'   r   r   r   )r   r#   r   rP   r   s        r"   test_invalid_dtype_backendz1TestParquetFastParquet.test_invalid_dtype_backend  s2   % 	 \5$uQ{{"3"3455_]++ 	:tMM$z555 : :T9999: : : : : : : : : : : : : : :	: 	: 	: 	: 	: 	: 	: 	: 	: 	: 	: 	: 	: 	: 	: 	: 	: 	:s6   1B/9BB/B	B/B	B//B36B3c                     t          j        t          j        g dd                    }t          j        t          j        g dd                    }t          |||           d S )Nr  r  r  r'  r  r  r>  s       r"   r  z)TestParquetFastParquet.test_empty_columns  si     \} M M MNNN<bh]&S&S&STTTR(333333r$   N)$r   r   r   rr  r:  r  r'   r   r[  r   rM   r   r?  r  r  r  r   r  r  r  rR  rT  r0  r   r  r,   r   r  rY  ra  re  ri  rl  rp  rs  r  r)   r$   r"   r5  r5    sl       ! ! !4 4 4*; ; ; [777#3#335   G G	 G	; 	; 	;! ! !      [
 
 
/ / / / / / / / /   & [++--6UVV4 4 WV4 [Sggk&=>>ASASST   4 4	 4	R 	R 	R8 8 8* * *K K K G G G   8	: 	: 	: [++--6UVV4 4 WV4 4 4r$   r5  )	NNNNNTFTr1   )I__doc__ri   decimalr   r   r   r  r2  rr  rM   r'   pandas._configr   pandas._config.configr   pandas.compatr   pandas.compat.pyarrowr   r	   r
   pandasr7   pandas._testing_testingrz   pandas.util.versionr   pandas.io.parquetr   r   r   r   r   r   r&   r   r   r,   r   filterwarnings
pytestmarkfixturer   r0  r#   r*   r-   r9   rQ   ro   nowr  r  minmaxstrptimerq   r   r   r   r   r   r   r   r   r   r   r   r   r   ri  r5  r)   r$   r"   <module>r     s                  				       . . . . . . - - - - - - - - - - - -                    ' ' ' ' ' '             NNNMM   MMM    KSTT
KG 
 +$$%% L;24@@@GKM %  	
 	
 	
 	+$$!!*D %  	
 	
 	
  $ % $       6 6 6   "   . h/344h/344h/344""#=?TUU""#=?TUU""#=?TUU""#=?TUU
 
 
 
 
 	? ? ? ?D9 9 9 2 2 2$ $ $$ $ $$ $ $F F F((# (# (#V6 6 6
6 
6 
6( ( ("- - - - - - - -0f
 f
 f
 f
 f
 f
 f
 f
R	z0 z0 z0 z0 z0 z0 z0 z0RR4 R4 R4 R4 R4T R4 R4 R4 R4 R4s$   A% %A/.A/3A: :BB