o
    ٷig                     @   sL  d Z ddlZddlZddlmZ ddlmZ ddlZddlm	Z	m
Z
mZmZmZ ddlmZmZmZ ddlmZmZ dd	lmZ dd
lmZmZ ddlmZmZmZmZmZmZmZm Z m!Z!m"Z"m#Z# e$ee%j&d Z'e Z(e Z)e Z*e Z+e Z,e Z-ddiZ.ej/dd Z0ej1j2dd Z3ej1j2dd Z4ej1j2dd Z5dd Z6ej1j2ej1j7de)ej8e*ej1j9de+e-gg dddd Z:ej1j2ej1j7de)ej8e*ej1j9de+e-gg ddd d! Z;G d"d# d#Z<ej1j2d$d% Z=G d&d' d'Z>ej1j2d(d) Z?ej1j2d*d+ Z@ej1j2d,d- ZAej1j2d.d/ ZBd0d1 ZCd2d3 ZDd4d5 ZEej1j2d6d7 ZFd8d9 ZGd:d; ZHd<d= ZId>d? ZJd@dA ZKdBdC ZLdDdE ZMej1j2dFdG ZNdHdI ZOdJdK ZPej1j2dLdM ZQdNdO ZRdPdQ ZSdRdS ZTej1j2ej17dTddUgdVdW ZUej1j2ej1j7dej8e*ej1j9de+e-gg dXddYdZ ZVej1j2d[d\ ZWd]d^ ZXdS )_z+
Test the core class and factory function.
    N)Path)TemporaryDirectory   )createPoochretrievedownload_actionstream_download)
get_loggertemporary_fileos_cache)	file_hashhash_matches)core)HTTPDownloaderFTPDownloader   )pooch_test_urldata_over_ftppooch_test_figshare_urlpooch_test_zenodo_url pooch_test_zenodo_with_slash_urlpooch_test_dataverse_urlpooch_test_registrycheck_tiny_datacheck_large_datacapture_logmirror_directorydatatiny-data.txt@098h0894dba14b12085eacb204284b97e362f4f3e5a5807693cc90ef415c1b2dc                 C   s
   t t| S )z
    Mirror the test data folder on a temporary directory. Needed to avoid
    permission errors when pooch is installed on a non-writable path.
    )r   DATA_DIR)tmp_path r#   I/home/ubuntu/.local/lib/python3.10/site-packages/pooch/tests/test_core.pydata_dir_mirror5   s   
r%   c               	   C   sv  t  } d}t| }t +}t|d| d}| }| d dks#J d|v s)J t| |v s1J W d   n1 s;w   Y  ||t| d ksMJ t| t	|t| ks[J t }t|d| d}| dksnJ W d   n1 sxw   Y  t }t|t| | d}| dksJ W d   n1 sw   Y  W d   dS W d   dS 1 sw   Y  dS )z'Try downloading some data with retriever   N)
known_hashpathr   DownloadingSHA256 hash of downloaded file: )
r   BASEURLr   r   getvaluesplitREGISTRYlenr   r   local_store	data_fileurllog_filefnamelogsr#   r#   r$   test_retrieve>   s0   "r7   c               	   C   s   t  `} d}t| }t ,}t|d| |d}| }| d dks$J d|v s*J t| |v s2J W d   n1 s<w   Y  |tj|d ksMJ t	| t
|t| ks[J W d   dS 1 sfw   Y  dS )zATry downloading some data with retrieve and setting the file namer   N)r&   r'   r5   r   r(   r)   r   )r   r+   r   r   r,   r-   r.   osr'   r   r   r0   r#   r#   r$   test_retrieve_fnameX   s   "r9   c               
   C   s  d} t |  }td|  }znt 7}t|d| d}| }| d dks&J ttd |v s2J d|v s8J t|  |v s@J W d   n1 sJw   Y  |t| ksYJ t	| t
|t|  ksgJ W tjt|rytt| dS dS tjt|rtt| w w )zETry downloading some data with retrieve to the default cache locationr   poochN)r&   r5   r   r(   zSHA256 hash of downloaded file)r+   r   r   r   r,   r-   strresolver.   r   r   r8   r'   existsremove)r2   r3   expected_locationr4   r5   r6   r#   r#   r$   test_retrieve_default_pathk   s(   r@   c                 C   s<   t | dtd}t| d }|d}||ksJ t| dS )zASetup a pooch that already has the local data and test the fetch.some bogus URLr'   base_urlregistryr   N)r   r.   r;   fetchr   )r%   puptruer5   r#   r#   r$   test_pooch_local   s
   
rH   r3   )marks)httpsfigsharezenodo	dataverse)idsc              	   C   s  t  {}t|}d| d i}t|dt|d}t )}|d}| }| d dks-J | d d| dks;J W d	   n1 sEw   Y  t| t }|d}| dks_J W d	   n1 siw   Y  W d	   d	S W d	   d	S 1 sw   Y  d	S )
z:Have pooch download the file from URL that is not base_urlr   r*   r'   rC   rD   urlsr   r(   ''.N)	r   r   r   r.   r   rE   r,   r-   r   )r3   r1   r'   rP   rF   r4   r5   r6   r#   r#   r$   test_pooch_custom_url   s$   

"rT   c              	   C   s.  t  }t|}t|d }t|| td}t )}|d}| }| d dks,J | d d| dks:J W d   n1 sDw   Y  ||ksOJ t	| t
|td ks]J t }|d}| d	ksnJ W d   n1 sxw   Y  W d   dS W d   dS 1 sw   Y  dS )
z:Setup a pooch that has no local data and needs to downloadr   rB   r   r(   rQ   rR   rS   Nr*   )r   r   r;   r   r.   r   rE   r,   r-   r   r   )r3   r1   r'   	true_pathrF   r4   r5   r6   r#   r#   r$   test_pooch_download   s(   

"rV   c                   @       e Zd ZdZdd Zdd ZdS )FakeHashMatchesz8Create a fake version of hash_matches that fails n timesc                 C   s   || _ d| _d S )Nr   )	nfailuresfailed)selfrY   r#   r#   r$   __init__   s   
zFakeHashMatches.__init__c                 O   s>   | j | jk r|  j d7  _ t|d dfi |S t|i |S )z#Fail n times before finally passingr   r   bla)rZ   rY   r   )r[   argskwargsr#   r#   r$   r      s   zFakeHashMatches.hash_matchesN)__name__
__module____qualname____doc__r\   r   r#   r#   r#   r$   rX      s    rX   c              
   C   s  t  z}| tdtdj t|}t|ttd}t	
t }t }|d W d   n1 s2w   Y  W d   n1 sAw   Y  dt|v sNJ |  d}t|dks_J |d	 d
shJ |d	 d| dsuJ W d   dS 1 sw   Y  dS )z2Check that retrying the download is off by defaultr      rB   r   Ndoes not match the known hash
r   r   r(   rR   rS   )r   setattrr   rX   r   r   r   r+   r.   pytestraises
ValueErrorr   rE   r;   r,   stripr-   r/   
startswithendswithmonkeypatchr1   r'   rF   errorr4   r6   r#   r#   r$   (test_pooch_download_retry_off_by_default   s    "rq   c                   @   rW   )	FakeSleepz<Create a fake version of sleep that logs the specified timesc                 C   s
   g | _ d S )N)times)r[   r#   r#   r$   r\      s   
zFakeSleep.__init__c                 C   s   | j | dS )z Store the time and doesn't sleepN)rs   append)r[   secsr#   r#   r$   sleep   s   zFakeSleep.sleepN)r`   ra   rb   rc   r\   rv   r#   r#   r#   r$   rr      s    rr   c              	   C   s  t  }| tdtdj t }| tjd|j t|}t	|d }d}t
|tt|d}t l}|d}|  d}	t|	d| ksJJ |	d d	sSJ |	d d
| ds`J tt|dd|	dd D ]\}
}d|v swJ |
dkr}dnd}d|
 d| d|v sJ qm|jg dksJ W d   n1 sw   Y  ||ksJ t| t|td ksJ W d   dS 1 sw   Y  dS )z?Check that retrying the download works if the hash is differentr      rv   r   r'   rC   rD   retry_if_failedrf   r   r   r(   rR   rS   rQ   NFailed to downloadsr*   zdownload again z
 more time.)r   r   rd                  	   
   r   )r   rg   r   rX   r   rr   timerv   r   r;   r   r+   r.   r   rE   r,   rk   r-   r/   rl   rm   ziprangers   r   r   )ro   r1   	fakesleepr'   rU   retriesrF   r4   r5   r6   ilinepluralr#   r#   r$   test_pooch_download_retry   s4   
""r   c              
   C   s0  t  }| tdtdj t|}t|ttdd}t	
t }t }|d W d   n1 s3w   Y  W d   n1 sBw   Y  |  d}t|dksXJ |d	 d
saJ |d	 d| dsnJ d|d v svJ d|d v s~J dt|v sJ W d   dS 1 sw   Y  dS )zFCheck that retrying the download fails after the set amount of retriesr   rd   r   rx   r   Nrf   r   r   r(   rR   rS   rz   zdownload again 1 more time.re   )r   rg   r   rX   r   r   r   r+   r.   rh   ri   rj   r   rE   r,   rk   r-   r/   rl   rm   r;   rn   r#   r#   r$   *test_pooch_download_retry_fails_eventually  s$   "r   c               	   C   s   t  @} t| }dtd i}t|dt|d}td}|d}| dks(J W d   n1 s2w   Y  t| W d   dS 1 sFw   Y  dS )zHSetup a pooch and check that no logging happens when the level is raisedr   r*   rO   CRITICALN)	r   r   r+   r   r.   r   rE   r,   r   )r1   r'   rP   rF   r4   r5   r#   r#   r$   test_pooch_logging_level3  s   


"r   c               	   C   sf  t  } t| }t|d }t|ddd}|d W d   n1 s%w   Y  t|ttd}t )}|	d}|
 }| d d	ksHJ | d
 d| dksVJ W d   n1 s`w   Y  ||kskJ t| t|td ksyJ t }|	d}|
 dksJ W d   n1 sw   Y  W d   dS W d   dS 1 sw   Y  dS )zFSetup a pooch that already has the local data but the file is outdatedr   wutf-8encodingdifferent dataNrB   r   UpdatingrQ   rR   rS   r*   )r   r   r;   openwriter   r+   r.   r   rE   r,   r-   r   r   )r1   r'   rU   finrF   r4   r5   r6   r#   r#   r$   test_pooch_updateB  s.   

"r   c               	   C   s   t  Y} t| }t|d }t|ddd}|d W d   n1 s%w   Y  t|ttdd}t	t
 |d W d   n1 sGw   Y  W d   dS W d   dS 1 s_w   Y  dS )	z$Test that disallowing updates works.r   r   r   r   r   NFr'   rC   rD   allow_updates)r   r   r;   r   r   r   r+   r.   rh   ri   rj   rE   )r1   r'   rU   r   rF   r#   r#   r$   test_pooch_update_disallowed^  s$   "r   c               	   C   s   d} zrdt j| < t P}t|}t|d }t|ddd}|d W d   n1 s-w   Y  t|tt	| d	}t
t |d W d   n1 sOw   Y  W d   n1 s^w   Y  W t j|  dS W t j|  dS t j|  w )
zDTest that disallowing updates works through an environment variable.MYPROJECT_DISALLOW_UPDATESFalser   r   r   r   r   Nr   )r8   environr   r   r;   r   r   r   r+   r.   rh   ri   rj   rE   pop)variable_namer1   r'   rU   r   rF   r#   r#   r$   (test_pooch_update_disallowed_environmentr  s.   
r   c                  C   s(   d} t | dtd}|j| d ksJ dS )zR
    Test if pooch.create appends a trailing slash to the base url if missing
    zhttps://mybase.urlN)rC   rD   r'   /)r   r!   rC   )rC   rF   r#   r#   r$   ,test_pooch_create_base_url_no_trailing_slash  s   r   c              
   C   s  t  g}tj|}t|ttd}t G}t	t
}|d W d   n1 s*w   Y  dt|jv s8J | }| d dksFJ | d d| d	ksTJ W d   n1 s^w   Y  W d   n1 smw   Y  t| ttd}t H}t	t
}|d W d   n1 sw   Y  dt|jv sJ | }| d d
ksJ | d d|  d	ksJ W d   dS 1 sw   Y  dS )z>Raise an exception if the file hash doesn't match the registryrB   r   Nz(tiny-data.txt)r   r(   rQ   rR   rS   r   )r   r8   r'   abspathr   r+   REGISTRY_CORRUPTEDr   rh   ri   rj   rE   r;   valuer,   r-   )r%   r1   r'   rF   r4   rp   r6   r#   r#   r$   test_pooch_corrupted  s0   "r   c                  C   sH   t ddtd} tt | d W d   dS 1 sw   Y  dS )z=Should raise an exception if the file is not in the registry.zit shouldn't matterzthis shouldn't eitherrB   zthis-file-does-not-exit.csvN)r   r.   rh   ri   rj   rE   rF   r#   r#   r$   test_pooch_file_not_in_registry  s   "r   c                  C   L   t ddd} | tjtd | jtksJ | j	 t
t	 ks$J dS )z,Loading the registry from a file should workr*   r'   rC   registry.txtNr   load_registryr8   r'   joinr!   rD   r.   registry_filessortlistr   r#   r#   r$   test_pooch_load_registry     r   c                  C   r   )z8Loading the registry from a file and strip line commentsr*   r   zregistry_comments.txtNr   r   r#   r#   r$   !test_pooch_load_registry_comments  r   r   c                  C   s   t jtd} tddd}t| d}|| W d   n1 s"w   Y  |jtks.J |j	
 tt
 ks;J tddd}t| ddd}|| W d   n1 sXw   Y  |jtksdJ |j	
 tt
 ksqJ dS )	z'Loading the registry from a file objectr   r*   r   rbNrr   r   )r8   r'   r   r!   r   r   r   rD   r.   r   r   r   )r'   rF   r   r#   r#   r$    test_pooch_load_registry_fileobj  s   r   c                  C   sD   t ddd} | tjtd | jtksJ | jddiks J dS )z8Load the registry from a file with a custom URL insertedr*   r   zregistry-custom-url.txtr   zhttps://some-site/tiny-data.txtN)	r   r   r8   r'   r   r!   rD   r.   rP   r   r#   r#   r$   #test_pooch_load_registry_custom_url  s   r   c                  C   sR   t ddi d} tt | tjtd W d   dS 1 s"w   Y  dS )z?Should raise an exception when a line doesn't have two elementsr*   rB   zregistry-invalid.txtN)	r   rh   ri   IOErrorr   r8   r'   r   r!   r   r#   r#   r$   %test_pooch_load_registry_invalid_line  s   "r   c                  C   s@   t ddd} | tjtd d| jv sJ d| jv sJ dS )zCShould check that spaces in filenames are allowed in registry filesr*   r   zregistry-spaces.txtzfile with spaces.txtzother with spaces.txtN)r   r   r8   r'   r   r!   rD   r   r#   r#   r$   $test_pooch_load_registry_with_spaces  s   r   c                  C   sn   t tttd} | dsJ t ttd td} | drJ ddi}|t t tt|d} | dr5J dS )FShould correctly check availability of existing and non existing filesrB   r   z
wrong-url/znot-a-real-data-file.txtnotarealhashN)r   r!   r+   r.   is_availableupdate)rF   rD   r#   r#   r$   test_check_availability  s   
r   c                 C   s|   t | d/}tt|dddddd}t| jd}|jd|ds#J |jd	|dr,J W d
   d
S 1 s7w   Y  d
S )r   r   r*   @baee0894dba14b12085eacb204284b97e362f4f3e5a5807693cc90ef415c1b2djdjdjdjdflld)r   doesnot_exist.ziprB   )port
downloaderr   N)r   r   r!   replacer   server_portr   )	ftpserverr3   rF   r   r#   r#   r$   test_check_availability_on_ftp   s   
"r   c                  C   s\   dd } t tttd}d}tjt|d |jd| d W d   dS 1 s'w   Y  dS )	z@Should raise an exception if the downloader doesn't support thisc                 S   s   dS )z,A downloader that doesn't support check_onlyNr#   )r3   outputr:   r#   r#   r$   r     s   z>test_check_availability_invalid_downloader.<locals>.downloaderrB   z%does not support availability checks.)matchr   r   N)r   r!   r+   r.   rh   ri   NotImplementedErrorr   )r   rF   msgr#   r#   r$   *test_check_availability_invalid_downloader  s   "r   c           	   	   C   s,  dd }t  }t|}t|ttd}t 3}|jd|d}| }| }t	|dks.J |d 
 d dks:J |d	 d
ksBJ W d   n1 sLw   Y  |  jrXJ t| t }|d}| dksmJ W d   n1 sww   Y  W d   dS W d   dS 1 sw   Y  dS )z%Setup a downloader function for fetchc                 S   s   t  d t | || dS )z2Download through HTTP and warn that we're doing itdownloader executedN)r
   infor   )r3   output_filerF   r#   r#   r$   download#  s   z,test_fetch_with_downloader.<locals>.downloadrB   zlarge-data.txtr   r   r   r(   r   r   Nr*   )r   r   r   r+   r.   r   rE   r,   
splitlinesr/   r-   
readouterrerrr   )	capsysr   r1   r'   rF   r4   r5   r6   linesr#   r#   r$   test_fetch_with_downloader  s*   
"r   c                 C   s\   t | tddid}tt}|d W d   n1 sw   Y  dt|jv s,J dS )z!Test an invalid hashing algorithmr   z	blah:1234rB   Nz'blah')r   r+   rh   ri   rj   rE   r;   r   )r%   rF   excr#   r#   r$   test_invalid_hash_alg>  s   
r   c                 C   s   t | d }t| t|d}| }W d   n1 sw   Y  dD ]+}t|}|| d| d|  i}t| d|d}||	dksLJ t| q%dS )z2Test different hashing algorithms using local datar   r   N)sha512md5:rA   rB   )
r;   r   r   readhashlibnewr   	hexdigestr   rE   )r%   r5   r   r   alghasherrD   rF   r#   r#   r$   #test_alternative_hashing_algorithmsI  s   



r   c               	   C   s  t tddd\} }| dksJ |dksJ t }t t|dd\} }W d   n1 s.w   Y  | dks9J |dks?J t 0}t|d	d
d}|d W d   n1 sZw   Y  t t|t|d\} }W d   n1 suw   Y  | dksJ |dksJ dS )z>Test that the right action is performed based on file existingzthis_file_does_not_exist.txtN)r&   r   r(   znot the correct hashr   r   r   r   r   z	some datarE   Fetching)r   r   r   r   r   r   )actionverbtmpr   r#   r#   r$   test_download_actionX  s$   
r   r5   zsubdir/tiny-data.txtc                 C   s   t d |  }t|  }t }t *}t||  }| rJ t||||dd | s,J tt| W d   dS 1 s=w   Y  dS )z9Check that downloading a file over HTTP works as expectedzstore/N)r:   )	r+   r.   r   r   r   r=   r	   r   r;   )r5   r3   r&   r   r1   destinationr#   r#   r$   test_stream_downloadm  s   "r   )rK   rL   rM   c                 C   s   t  ;}tj|}t|| d}|  t|jdksJ d|jv s$J d|jv s+J |jD ]}|| q.W d   dS 1 sAw   Y  dS )z;Check that the registry is correctly populated from the APIr   r   r   z	store.zipN)	r   r8   r'   r   r   load_registry_from_doir/   rD   rE   r3   r1   r'   rF   filenamer#   r#   r$   test_load_registry_from_doi}  s   
"r   c                  C   s   t } t 4}tj|}t|| d}|  t|jdksJ d|jv s&J |jD ]}|	| q)W d   dS 1 s<w   Y  dS )zs
    Check that the registry is correctly populated from the Zenodo API when
    the filename contains a slash
    r   r   z!santisoler/pooch-test-data-v1.zipN)
ZENODOURL_W_SLASHr   r8   r'   r   r   r   r/   rD   rE   r   r#   r#   r$   -test_load_registry_from_doi_zenodo_with_slash  s   
"r   c                  C   sT   t dtd} tt}|   W d   n1 sw   Y  dt|jv s(J dS )z(Check that non-DOI URLs produce an errorr*   r   Nzonly implemented for DOIs)r   r+   rh   ri   rj   r   r;   r   )rF   r   r#   r#   r$   !test_wrong_load_registry_from_doi  s
   
r   )Yrc   r   r8   pathlibr   tempfiler   rh   r   r   r   r   r   r	   utilsr
   r   r   hashesr   r   r*   downloadersr   r   r   r   r   r   r   r   r   r   r   r   r   r;   __file__parentr!   r.   r+   FIGSHAREURL	ZENODOURLr   DATAVERSEURLr   fixturer%   marknetworkr7   r9   r@   rH   parametrizeparamrK   rT   rV   rX   rq   rr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r#   r#   r#   r$   <module>   s   4



	



 


	
	


