o
    ڷiO                  
   @   s  d Z ddlZddlZddlmZ ddlZddlmZ zddlZW n e	y+   dZY nw zddl
Z
W n e	y=   dZ
Y nw ddlmZ ddlmZmZmZmZmZmZmZmZmZmZ ddlmZ d	d
lmZmZmZmZmZm Z m!Z!m"Z" e Z#e Z$e  Z%e! Z&e" Z'ej(j)edu ddej(*de#d e%gdd Z+ej(j)e
du dddd Z,dd Z-ej(j.dd Z/ej(j.dd Z0ej(j.ej(j*dej1edej(j2dedfedfgg dd d!d" Z3ej(j.ej(j*dej1e$ej(j2de%e'gg dd d#d$ Z4ej(j.d%d& Z5ej(j.ej(j2d'd( Z6ej(j.ej(j2ej(*d)d*d+gd,d- Z7ej(j.d.d/ Z8ej(j.ej(j)e
du d0dd1d2 Z9ej(j.ej(j)e
du d0dd3d4 Z:ej(j)e
dud5dd6d7 Z;ej(j)edud8dej(*d9eeegd:d; Z<ej(j.ej(j)edu ddej(j*d<e#efej1e$eej(j2dgd=d>gd d?d@ Z=ej(j.ej(j)edu dddAdB Z>ej(j.ej(j)edu ddej(j)e
du dddCdD Z?ej(j.dEdF Z@G dGdH dHZAG dIdJ dJZBdS )KzK
Test the downloader classes and functions separately from the Pooch core.
    N)TemporaryDirectory)	HTTPError   )Pooch)
HTTPDownloaderFTPDownloaderSFTPDownloaderDOIDownloaderchoose_downloaderFigshareRepositoryZenodoRepositoryDataverseRepository
doi_to_urlREQUESTS_HEADERS)Unzip   )pooch_test_urlcheck_large_datacheck_tiny_datadata_over_ftppooch_test_figshare_urlpooch_test_zenodo_url pooch_test_zenodo_with_slash_urlpooch_test_dataverse_urlzrequires tqdm)reasonurltiny-data.txtc                 C   s   t | dd}|jdu sJ dS )DThe progressbar keyword argument must pass through choose_downloaderTprogressbarNr
   r   r   
downloader r#   R/home/ubuntu/vllm_env/lib/python3.10/site-packages/pooch/tests/test_downloaders.pytest_progressbar_kwarg_passed?   s   
r%   zrequires paramikoc                  C   s"   d} t | dd}|jdu sJ dS )r   /sftp://test.rebex.net/pub/example/pocketftp.pngTr   Nr    r!   r#   r#   r$   "test_progressbar_kwarg_passed_sftpM   s   r'   c                   C   sj   t t td W d   n1 sw   Y  t t td W d   dS 1 s.w   Y  dS )z6Should raise ValueError when protocol is not supportedzhttpup://some-invalid-url.comNzdoii:XXX/XXX/file)pytestraises
ValueErrorr
   r#   r#   r#   r$   test_unsupported_protocolU   s   

"r+   c                  C   sP   t t} t dddd W d   n1 sw   Y  dt| jv s&J dS )z/Should fail if data repository is not supportedz%doi:10.21105/joss.01943/file_name.txtN)r   output_filepoochz(Invalid data repository 'joss.theoj.org')r(   r)   r*   r	   strvalueexcr#   r#   r$   test_invalid_doi_repository^   s   r2   c                   C   s:   t t tdd W d   dS 1 sw   Y  dS )z#Should fail if the DOI is not foundNOTAREALDOI)doiN)r(   r)   r   r   r#   r#   r#   r$   test_doi_url_not_foundi   s   "r5   zrepository,doiz10.6084/m9.figshare.14763051.v1)marksz10.5281/zenodo.4924875z10.11588/data/TKCFEF)figsharezenodo	dataverse)idsc                 C   s`   t t}t|}| ||}|jdd W d   n1 s w   Y  dt|jv s.J dS )z3Should fail if the file is not found in the archivezbla.txt	file_nameNzFile 'bla.txt' not found)r(   r)   r*   r   
initializedownload_urlr.   r/   )
repositoryr4   r1   r   repor#   r#   r$    test_figshare_url_file_not_foundp   s   rA   c                 C   sX   t  }t }tj|d}|| d |d t| W d   dS 1 s%w   Y  dS )zTest the DOI downloaderr   N)r   r	   ospathjoinr   )r   local_storer"   outfiler#   r#   r$   test_doi_downloader   s   	
"rG   c                  C   s|   t  1} td }t }tj| d}|||d t |ddd}dd |D \}t| W d   dS 1 s7w   Y  dS )zf
    Test the Zenodo downloader when the path contains a forward slash

    Related to issue #336
    z!santisoler/pooch-test-data-v1.zipztest-data.zipNdownload)actionr-   c                 S   s   g | ]}d |v r|qS )r   r#   ).0fr#   r#   r$   
<listcomp>   s    z>test_zenodo_downloader_with_slash_in_fname.<locals>.<listcomp>)r   ZENODOURL_W_SLASHr	   rB   rC   rD   r   r   )rE   base_urlr"   rF   fnamesfnamer#   r#   r$   *test_zenodo_downloader_with_slash_in_fname   s   
"rQ   c               	   C   s   t } | d| d d } | dd }d| d}t :}t }tj|d}tjt	|d	 || d |d W d   n1 sAw   Y  W d   dS W d   dS 1 sYw   Y  dS )
zf
    Test if passing a Figshare url without a version warns about it, but still
    downloads it.
    N./   zThe Figshare DOI 'z#' doesn't specify which version of r   match)
FIGSHAREURLrindexr   r	   rB   rC   rD   r(   warnsUserWarning)r   r4   warning_msgrE   r"   rF   r#   r#   r$   !test_figshare_unspecified_version   s   "r]   zversion, missing, present)r   7LC08_L2SP_218074_20190114_20200829_02_T1-cropped.tar.gzcropped-before.tar.gz)r   r_   r^   c                 C   sL   d|  }d| d}t ||}dd |jD }||v sJ ||vs$J dS )zG
    Test if setting the version in Figshare DOI works as expected
    z10.6084/m9.figshare.21665630.vhttps://doi.org/rS   c                 S   s   g | ]}|d  qS )namer#   )rJ   itemr#   r#   r$   rL      s    z:test_figshare_data_repository_versions.<locals>.<listcomp>N)r   api_response)versionmissingpresentr4   r   r7   	filenamesr#   r#   r$   &test_figshare_data_repository_versions   s   

rh   c              	   C   s   t | d;}t }t| jd}tj|d}|||d t| W d   n1 s+w   Y  W d   dS W d   dS 1 sCw   Y  dS )zTest ftp downloaderr   )portN)r   r   r   server_portrB   rC   rD   r   )	ftpserverr   rE   r"   rF   r#   r#   r$   test_ftp_downloader   s   
"rl   zrequires paramiko to run SFTPc                  C   sf   t  &} tddd}d}tj| d}|||d tj|s!J W d   dS 1 s,w   Y  dS )zTest sftp downloaderdemopasswordusernamern   r&   pocketftp.pngN)r   r   rB   rC   rD   exists)rE   r"   r   rF   r#   r#   r$   test_sftp_downloader   s   "rs   c               
   C   s   t  P} tddd}d}tj| d}t|d#}tt |||d W d   n1 s/w   Y  W d   n1 s>w   Y  W d   dS W d   dS 1 sVw   Y  dS )zFDownloader should fail when a file object rather than string is passedrm   rn   ro   r&   rq   wbN)	r   r   rB   rC   rD   openr(   r)   	TypeError)rE   r"   r   rF   outfile_objr#   r#   r$   (test_sftp_downloader_fail_if_file_object   s   "rx   zparamiko must be missingc                  C   sF   t t} t  W d   n1 sw   Y  dt| jv s!J dS )z+test must fail if paramiko is not installedNz
'paramiko')r(   r)   r*   r   r.   r/   r0   r#   r#   r$   -test_sftp_downloader_fail_if_paramiko_missing  s   ry   ztqdm must be missingr"   c                 C   sJ   t t}| dd W d   n1 sw   Y  dt|jv s#J dS )zFMake sure an error is raised if trying to use progressbar without tqdmTr   Nz'tqdm')r(   r)   r*   r.   r/   )r"   r1   r#   r#   r$   !test_downloader_progressbar_fails
  s   rz   zurl,downloaderhttpr7   c           
      C   s   |dd}t  J}d}| | } tj||}|| |d | }|jdd  }t|dks2J t	j
dkr:d	}	nd
}	|dd |	ksFJ t| W d   dS 1 sUw   Y  dS )z@Setup a downloader function that prints a progress bar for fetchTr   r   NrU   O   win32100%|####################A   100%|████████████████████   )r   rB   rC   rD   
readouterrerrsplitstriplensysplatformr   )
r   r"   capsysrH   rE   rP   rF   capturedprintedprogressr#   r#   r$   test_downloader_progressbar  s   


"r   c           	   	   C   s   t |df}td|jd}t C}tj|d}|||d |  }|j	dd 
 }t|dks4J tjdkr<d	}nd
}|dd |ksHJ t| W d   n1 sVw   Y  W d   dS W d   dS 1 snw   Y  dS )zESetup an FTP downloader function that prints a progress bar for fetchr   T)r   ri   Nr|   rU   r}   r~   r   r   r   )r   r   rj   r   rB   rC   rD   r   r   r   r   r   r   r   r   )	r   rk   r   rH   rE   rF   r   r   r   r#   r#   r$   test_downloader_progressbar_ftp3  s"   

"r   c                 C   s   t dddd}t J}d}tj|d}|||d |  }|jdd	  }t	|d
ks0J t
jdkr8d}nd}|dd |ksDJ tj|sLJ W d   dS 1 sWw   Y  dS )zFSetup an SFTP downloader function that prints a progress bar for fetchTrm   rn   )r   rp   rn   r&   rq   Nr|   rU   r}   r~   r   r   r   )r   r   rB   rC   rD   r   r   r   r   r   r   r   rr   )r   r"   rE   r   rF   r   r   r   r#   r#   r$    test_downloader_progressbar_sftpL  s   
"r   c                 C   s   G dd d}|dd}t |d}t 6}d}t| }tj|d}|||d |  }|jdd 	 }	d	}
|	|
ks>J t
| W d   dS 1 sMw   Y  dS )
zASetup a downloader function with an arbitrary progress bar class.c                   @   sD   e Zd ZdZdd Zdd Zdd Zdd	 Zd
d Ze	dd Z
dS )zEtest_downloader_arbitrary_progressbar.<locals>.MinimalProgressDisplayz&A minimalist replacement for tqdm.tqdmc                 S   s   d| _ || _d S )Nr   )counttotal)selfr   r#   r#   r$   __init__k  s   
zNtest_downloader_arbitrary_progressbar.<locals>.MinimalProgressDisplay.__init__c                 S   s   t | jd t | j S )zrepresent current completionrS   )r.   r   r   r   r#   r#   r$   __repr__o  s   zNtest_downloader_arbitrary_progressbar.<locals>.MinimalProgressDisplay.__repr__c                 S   s   t d|  tjdd dS )zprint self.__repr__ to stderrr|    )fileendNprintr   stderrr   r#   r#   r$   renders  s   zLtest_downloader_arbitrary_progressbar.<locals>.MinimalProgressDisplay.renderc                 S   s   || _ |   dS )zmodify completion and renderN)r   r   )r   ir#   r#   r$   updatew  s   zLtest_downloader_arbitrary_progressbar.<locals>.MinimalProgressDisplay.updatec                 S   s
   d| _ dS )zset counter to 0r   N)r   r   r#   r#   r$   reset|  s   
zKtest_downloader_arbitrary_progressbar.<locals>.MinimalProgressDisplay.resetc                   S   s   t dtjd dS )zprint a new empty liner   )r   Nr   r#   r#   r#   r$   close  s   zKtest_downloader_arbitrary_progressbar.<locals>.MinimalProgressDisplay.closeN)__name__
__module____qualname____doc__r   r   r   r   r   staticmethodr   r#   r#   r#   r$   MinimalProgressDisplayh  s    r   N)r   r   zlarge-data.txtr|   rU   z336/336)r   r   BASEURLrB   rC   rD   r   r   r   r   r   )r   r   pbarrH   rE   rP   r   rF   r   r   r   r#   r#   r$   %test_downloader_arbitrary_progressbard  s   


"r   c                   @   s  e Zd ZdZdZde Zde ZdZdZdZ	ddeeed	ed
e	 deidgdZ
ddeeed	ee	deidgdZddeeed	ee	deidd	ed
e	 deidgdZejdde
fdefdefgdd Zejdde
fdefgdd Zejde
egdd ZdS )TestZenodoAPISupportz0
    Test support for different Zenodo APIs
    i@ z10.0001/zenodo.r`   zmy-file.zipzMhttps://zenodo.org/api/files/513d7033-93a2-4eeb-821c-2fb0bbab0012/my-file.zip 2942bfabb3d05332b66eb128e0842cffz 2021-20-19T08:00:00.000000+00:00z$513d7033-93a2-4eeb-821c-2fb0bbab0012md5:r   )idkeychecksumlinks)createdmodifiedr   r4   doi_urlfiles)r   filenamer   r   zapi_version, api_responselegacynewinvalidc                 C   s   | d| j | t| j| jd}|d|_|dkr'|j|ks%J dS d}t	j
t|d |j}W d   dS 1 s?w   Y  dS )z@
        Test if the API version is correctly detected.
        /zenodo.r4   archive_urlr   r   z0Couldn't determine the version of the Zenodo APIrV   N)expect_request
article_idrespond_with_jsonr   r4   r   url_forbase_api_urlapi_versionr(   r)   r*   )r   
httpserverr   rc   r"   msgr#   r#   r$   test_api_version  s   "z%TestZenodoAPISupport.test_api_versionc                 C   s   | d| j | t| j| jd}|d|_|j| j	d}|dkr.|| j
ks,J d	S d| j d| j	 d}||ks?J d	S )
zK
        Test if the download url is correct for each API version.
        r   r   r   r;   r   zhttps://zenodo.org/records/z/files/z?download=1N)r   r   r   r   r4   r   r   r   r>   r<   file_url)r   r   r   rc   r"   r>   expected_urlr#   r#   r$   test_download_url  s   	z&TestZenodoAPISupport.test_download_urlrc   c                 C   sj   | d| j | td|d}t| j| jd}|d|_|	| |j
| jd| j iks3J dS )zX
        Test if population of registry is correctly done for each API version.
        r   r   )rN   rC   r   r   N)r   r   r   r   r   r4   r   r   r   populate_registryregistryr<   file_checksum)r   r   tmp_pathrc   puppyr"   r#   r#   r$   test_populate_registry  s   	
 z+TestZenodoAPISupport.test_populate_registryN)r   r   r   r   r   r4   r   r<   r   r   legacy_api_responsenew_api_responseinvalid_api_responser(   markparametrizer   r   r   r#   r#   r#   r$   r     s    

	

r   c                   @   s(   e Zd ZdZdd Zdd Zdd ZdS )	TestDOIDownloaderHeadersz+Test the headers argument in DOIDownloader.c                 C   s0   t  }|jtks
J t dd}|jtksJ dS )z#Test the default value for headers.Nheaders)r	   r   r   r   r"   r#   r#   r$   test_default_headers4  s   
z-TestDOIDownloaderHeaders.test_default_headersc                 C   s(   t ddid}ddi}|j|ksJ dS )zTest overwriting for headers.customfieldr   Nr	   r   )r   r"   expected_headersr#   r#   r$   test_overwrite_headers;  s   z/TestDOIDownloaderHeaders.test_overwrite_headersc                 C   s   t i d}|ji ksJ dS )z&Test passing an emtpy dict to headers.r   Nr   r   r#   r#   r$   test_headers_empty_dictC  s   
z0TestDOIDownloaderHeaders.test_headers_empty_dictN)r   r   r   r   r   r   r   r#   r#   r#   r$   r   1  s
    r   )Cr   rB   r   tempfiler   r(   requestsr   tqdmImportErrorparamikor   r   downloadersr   r   r   r	   r
   r   r   r   r   r   
processorsr   utilsr   r   r   r   r   r   r   r   r   rX   	ZENODOURLrM   DATAVERSEURLr   skipifr   r%   r'   r+   networkr2   r5   paramr7   rA   rG   rQ   r]   rh   rl   rs   rx   ry   rz   r   r   r   r   r   r   r#   r#   r#   r$   <module>   s   0(
	


		






2 