o
    7?e1                     @   s
  d Z ddlmZmZ ddlZddlZddlZddlm	Z	 ddl
m  mZ ddlmZ ddlmZ ddlmZ ddlmZ ejjejjejdd	d
gejdddgdd Zejjejjdd Zejdd Zejjejde  G dd dZ!dS )zl
Tests parsers ability to read and parse non-local files
and hence require a network connection to be read.
    )BytesIOStringION)is_ci_environment)	DataFrame)read_featherread_csvmodeexplicitinferenginepythoncc                 C   s   |dkr	t d || }t|dddd| d}| j| d W d    n1 s,w   Y  | jd	 | }	|d
kr>|}t|	d||d}
t|
| d S )Ntarz5TODO: Add tar salaraies.csv to pandas/io/parsers/dataioparserdatazsalaries.csvrbcontentz/salaries.csvr
   	)sepcompressionr   )	pytestskipopenserve_contentreadurlr   tmassert_frame_equal)
httpserverdatapathZsalaries_tabler	   r   Zcompression_onlyZcompression_to_extension	extensionfr   Z	url_table r%   d/home/www/facesmatcher.com/pyenv/lib/python3.10/site-packages/pandas/tests/io/parser/test_network.pytest_compressed_urls   s   
r'   c                 C   sj   t |ddddd}| j| d t| jddd	}W d   n1 s%w   Y  |jd
 dks3J dS )zN
    read_csv should honor the requested encoding for URLs.

    GH 10424
    r   r   r   zunicode_series.csvr   r   zlatin-1N)encodingheader)      u$   Á köldum klaka (Cold Fever) (1994))r   r   r   r   r   loc)r!   r"   r$   dfr%   r%   r&   test_url_encoding_csv7   s
   r.   c                 C   s   t | ddddS )z DataFrame with the tips dataset.r   r   csvtips.csvr   )r"   r%   r%   r&   tips_dfE   s   r1   Zs3_resourcec                   @   s   e Zd Zdd Zdd Zdd Zdd Zd	d
 Zdd Zdd Z	dd Z
dd Zdd Zdd Zdd Zejjddddd Zejjddddd Zejjd d! Zejjejje d"d#d$d% Zd&d' Zd(d) Zd*S )+TestS3c                 C   s\   t d dD ]$\}}td|j d| ||d}t|ts J |jr%J t|| qd S )Ns3fs) N).gzgzip).bz2bz2s3://	/tips.csv)r   storage_options	r   importorskipr   name
isinstancer   emptyr   r    selfs3_public_bucket_with_datar1   s3soextcompr-   r%   r%   r&   test_parse_public_s3_bucketO   s   

z"TestS3.test_parse_public_s3_bucketc                 C   sH   t d td|j d|d}t|tsJ |jrJ t|| d S )Nr3   r:   r;   r<   r=   )rC   Zs3_private_bucket_with_datar1   rE   r-   r%   r%   r&   test_parse_private_s3_bucket]   s   

z#TestS3.test_parse_private_s3_bucketc                 C   J   t d|j dd|d}t|tsJ |jrJ t|jd d | d S )Nzs3n://r;   
   nrowsr<   r   r?   r@   r   rA   r   r    ilocrC   rD   r1   rE   r-   r%   r%   r&   test_parse_public_s3n_bucketg      
z#TestS3.test_parse_public_s3n_bucketc                 C   rK   )Nzs3a://r;   rL   rM   rO   rQ   r%   r%   r&   test_parse_public_s3a_bucketr   rS   z#TestS3.test_parse_public_s3a_bucketc                 C   s^   dD ]*\}}t d|j d| d||d}t|tsJ |jr!J t|jd d | qd S )Nr4   r:   r;   rL   )rN   r   r<   rO   rB   r%   r%   r&   !test_parse_public_s3_bucket_nrows}   s   
z(TestS3.test_parse_public_s3_bucket_nrowsc              	   C   s   d}dD ]R\}}t d|j d| |||d7}|j|ksJ dD ]%}| }	t|	ts.J |	jr3J |j|| ||d   }
t	|
|	 q!W d    n1 sQw   Y  qd S )N   r4   r:   r;   )	chunksizer   r<   r   r+      r+   
r   r?   rW   Z	get_chunkr@   r   rA   rP   r   r    rC   rD   r1   rE   rW   rF   rG   Z	df_readerZi_chunkr-   Ztrue_dfr%   r%   r&   #test_parse_public_s3_bucket_chunked   s,   
z*TestS3.test_parse_public_s3_bucket_chunkedc              	   C   s   d}dD ]S\}}t d|j d| ||d|d7}|j|ks J dD ]%}| }	t|	ts/J |	jr4J |j|| ||d   }
t	|
|	 q"W d    n1 sRw   Y  qd S )	NrV   r4   r:   r;   r   )rW   r   r   r<   rX   r+   rZ   r[   r%   r%   r&   *test_parse_public_s3_bucket_chunked_python   s.   
z1TestS3.test_parse_public_s3_bucket_chunked_pythonc                 C   sT   dD ]%\}}t d|j d| d||d}t|tsJ |jr!J t|| qd S )Nr4   r:   r;   r   r   r   r<   r   r?   r@   r   rA   r   r    rB   r%   r%   r&   "test_parse_public_s3_bucket_python   s   
z)TestS3.test_parse_public_s3_bucket_pythonc                 C   sP   dD ]#}t d|j d| dd|d}t|tsJ |jrJ t|| qd S )N)r5   r6   r8   r:   r;   r   r   r^   r_   )rC   rD   r1   rE   rF   r-   r%   r%   r&   test_infer_s3_compression   s   
z TestS3.test_infer_s3_compressionc                 C   s`   dD ]+\}}t d|j d| dd||d}t|tsJ |jr"J t|jd d | qd S )Nr4   r:   r;   r   rL   )r   rN   r   r<   rO   rB   r%   r%   r&   (test_parse_public_s3_bucket_nrows_python   s   
z/TestS3.test_parse_public_s3_bucket_nrows_pythonc                 C   sD   d}t jt|d td|d W d    d S 1 sw   Y  d S )N#The specified bucket does not existmatchzs3://nyqpug/asdf.csvrI   )r   raisesOSErrorr   )rC   rE   msgr%   r%   r&   test_read_s3_fails   s   "zTestS3.test_read_s3_failsc                 C   sJ   d}t jt|d td|j d W d    d S 1 sw   Y  d S )Nrc   rd   r:   z	/file.csv)r   rf   rg   r   r?   )rC   Zs3_private_bucketrE   rh   r%   r%   r&   test_read_s3_fails_private   s   "z!TestS3.test_read_s3_fails_privatezGH#39155 s3fs upgradeF)reasonstrictc                 C   sV   dd l }t|jjf}tj|dd |jd|d W d    d S 1 s$w   Y  d S )Nr   rc   rd   z/s3://an_s3_bucket_data_doesnt_exit/not_real.csvrI   )botocoreFileNotFoundError
exceptionsClientErrorr   rf   to_csvrC   r1   rE   rm   errorr%   r%   r&   test_write_s3_csv_fails   s   "zTestS3.test_write_s3_csv_failsc                 C   s`   t d dd l}t|jjf}t j|dd |jd|d W d    d S 1 s)w   Y  d S )Npyarrowr   rc   rd   z3s3://an_s3_bucket_data_doesnt_exit/not_real.parquetrI   )r   r>   rm   rn   ro   rp   rf   Z
to_parquetrr   r%   r%   r&   test_write_s3_parquet_fails   s   
"z"TestS3.test_write_s3_parquet_failsc                 C   sz   | d}t| d  }t|dd}W d    n1 s w   Y  t|ts,J |jr1J t|}t	|| d S )Nr0   Bodyutf8)r(   )
ZObjectr   getr   r   r@   r   rA   r   r    )rC   rD   Z	tips_fileZ	s3_objectbufferresultexpectedr%   r%   r&   $test_read_csv_handles_boto_s3_object  s   

z+TestS3.test_read_csv_handles_boto_s3_objectz:GH: 45651: This test can hang in our CI min_versions build)rk   c                 C   s   dd l }ttjddtdd}t }|| t	|
 d}|jd|d |j  |jtjd	d
! td|j dd|d ddd |jD v sQJ W d    d S 1 s\w   Y  d S )Nr   rY   )i    abcd)columnszutf-8zlarge-file.csv)Keyrw   r3   )loggerr:   z/large-file.csvrV   rM   )r   i  T c                 s   s    | ]
}|j d d V  qdS )N)args).0xr%   r%   r&   	<genexpr>C  s    z8TestS3.test_read_csv_chunked_download.<locals>.<genexpr>)r3   r   nprandomZdefault_rngZstandard_normallistr   rq   r   getvalueencodeZ
put_objectZS3FileSystemZclear_instance_cacheZat_levelloggingDEBUGr   r?   records)rC   Zs3_public_bucketZcaplogrE   r3   r-   Zstr_bufbufr%   r%   r&   test_read_csv_chunked_download"  s"   

"z%TestS3.test_read_csv_chunked_downloadc                 C   s&   t d|j d|d}t|| d S )Nr:   z/tips#1.csvrI   )r   r?   r   r    )rC   rD   r1   rE   r{   r%   r%   r&   test_read_s3_with_hash_in_keyE  s   z$TestS3.test_read_s3_with_hash_in_keyc                 C   s8   t d t|}td|j d|d}t|| d S )Nru   r:   z/simple_dataset.featherrI   )r   r>   r   r?   r   r    )rC   rD   Zfeather_filerE   r|   resr%   r%   r&   test_read_feather_s3_file_pathL  s   
z%TestS3.test_read_feather_s3_file_pathN)__name__
__module____qualname__rH   rJ   rR   rT   rU   r\   r]   r`   ra   rb   ri   rj   r   markZxfailrt   rv   
single_cpur}   Zskipifr   r   r   r   r%   r%   r%   r&   r2   K   s6    



r2   )"__doc__r   r   r   r   numpyr   r   Zpandas.compatr   Zpandas.util._test_decoratorsutilZ_test_decoratorstdZpandasr   Zpandas._testingZ_testingr   Zpandas.io.feather_formatr   Zpandas.io.parsersr   r   networkr   Zparametrizer'   r.   Zfixturer1   ZusefixturesZskip_if_not_us_localer2   r%   r%   r%   r&   <module>   s2    

