o پi= @sUddlZddlZddlZddlZddlZddlZddlZddlZddlZddl Z ddl m Z m Z mZmZmZmZmZmZmZmZmZmZmZddlZddlZddlmZdZdZdZ dZ!dZ"d Z#d Z$d Z%dZ&d Z'd Z(dZ)dZ*ddgZ+e,Z-e$e%d fde.de.de.dee.fddZ/GdddZ0GdddZ1de2de2de2de2fdd Z3Gd!d"d"e4Z5de6deee2ee2ffd#d$Z7d%d&deee2ee2ee2ffd'd(Z8Gd)d*d*e5Z9Gd+d,d,e9Z:Gd-d.d.e9Z;Gd/d0d0e9ZGd5d6d6eZ?Gd7d8d8Z@e@ee d9ZAGd:d;d;ZBGd<d?e2deEfd@dAZGGdBdCdCe4ZHdDe dEejdFeEdGe.de6f dHdIZIdJeBdKe ge0fdd&fdLdMZJGdNdOdOejKZLdPe de fdQdRZMGdSdTdTZNGdUdVdVejOZPGdWdXdXejQZRdYe de6fdZd[ZSdod\eeEd]eeEde2fd^d_ZTd`e2de2fdadbZUdce2dde2de2fdedfZVdJeBdge2deEfdhdiZWeGdjdkdkeZXee2eXdlfZYd`eYde2fdmdnZZdS)pN) AnyCallableDictIteratorListMapping NamedTupleOptionalProtocolSequenceTupleUnionruntime_checkable)_xmli ii,iXg?gN@zhttps://storage.googleapis.com)iiiiz*[SSL: DECRYPTION_FAILED_OR_BAD_RECORD_MAC]z('Connection aborted.',initialmaximum multiplierreturnccs2dV|} |tV||9}||kr|}qNr)rngrandom)rrrbaserD/home/ubuntu/.local/lib/python3.10/site-packages/blobfile/_common.pyexponential_sleep_generator=sr c@szeZdZdZdddddefdededeeeefdeeeefd ed e d e e d e e d dfddZ d efddZ dS)Requestz/ A struct representing an HTTP request NT)methodurlparamsheadersdatapreload_content success_codes retry_codesrc Cs4||_||_||_||_||_||_||_||_dSN)r#r$r%r&r'r(r)r*) selfr#r$r%r&r'r(r)r*rrr__init__Ps  zRequest.__init__cCd|jd|jd|jdS)Nz)r#r$r%r,rrr__repr__ezRequest.__repr__)__name__ __module__ __qualname____doc__DEFAULT_RETRY_CODESstrr rrboolr intr-r1rrrrr!Ks8  r!c@s2eZdZdZdedededdfddZd d ZdS) FileBodyzq A struct for referencing a section of a file on disk to be used as the `data` property on a Request pathstartendrNcCs||_||_||_dSr+r<r=r>)r,r<r=r>rrrr-os zFileBody.__init__cCr.)Nzi}|D]\}}tjj|dd||<q||jdi|S)N)safer)itemsurllibparsequoteformat)r@rAr' escaped_datakvrrr build_urlxsrLcs*eZdZdZdedeffdd ZZS)Errorz#Base class for blobfile exceptions.messageargscs||_tj|g|RdSr+)rNsuperr-)r,rNrO __class__rrr-szError.__init__)r3r4r5r6r8rr- __classcell__rrrQrrMsrMcCs|dr#zt|}|dd|ddfWSty"YdSw|drDzt|}t|d|dfWStyCYdSwdS) Nss z/_extract_error_from_response..)r'rar&joinrD)rberrerr_desc err_headersrrr_extract_error_from_responses  rmcsveZdZdZ ddedededeedeedeef fd d Zd efd d Ze dede ddd e fddZ Z S)RequestFailurezA A request failed, possibly after some number of retries NrNrequest_stringresponse_statusrWrX error_headerscsB||_||_||_||_||_t||j|j|j|j|jdSr+)rorprWrXrqrPr-)r,rNrorprWrXrqrQrrr-s zRequestFailure.__init__rc Cs4d|jd|jd|jd|jd|jd|j S)Nzmessage=z , request=z , status=z, error=z, error_description=z, error_headers=rNrorprWrXrqr0rrr__str__szRequestFailure.__str__requestrbrccCs(t|\}}}||t||j|||dS)Nrr)rmr8status)clsrNrtrbrjrkrlrrrcreate_from_request_responsesz+RequestFailure.create_from_request_responser+)r3r4r5r6r8r:r r-rs classmethodr!rrwrSrrrQrrns6 rnc@eZdZdZdS) RestartableStreamingWriteFailurezo A streaming write failed in a permanent way that requires restarting from the beginning of the stream Nr3r4r5r6rrrrrzrzc@ry)ConcurrentWriteFailurez9 A write failed due to another concurrent writer Nr{rrrrr}r|r}c@ry)DeadlineExceededz7 A read failed after the deadline was exceeded Nr{rrrrr~r|r~c@ry)VersionMismatchza A write failed due to a version mismatch, using ETag for azure or generation for gcloud Nr{rrrrrr|rc@s>eZdZUeed<eed<eed<eeed<eeed<dS)Statsizemtimectimemd5versionN)r3r4r5r:__annotations__floatr r8rrrrrs  rc@s:eZdZUeed<eed<eed<eed<eeed<dS)DirEntryr<nameis_diris_filestatN)r3r4r5r8rr9r rrrrrrs rc@sZeZdZdededdfddZdejfddZdee e ffd d Z d e ddfd d Z dS) PoolDirectorconnection_pool_max_sizemax_connection_pool_countrNcCs&||_||_d|_d|_t|_dSr+)rr pool_manager creation_pid threadingLocklock)r,rrrrrr-  zPoolDirector.__init__cCsf|j&|jdus|jtkrt|_tj|j|jd|_|jWdS1s,wYdS)N)maxsize num_pools) rrrosgetpidurllib3 PoolManagerrrr0rrr get_http_pools $zPoolDirector.get_http_poolcCsdd|jDS)NcSsi|] \}}|dvr||qS))rrr)rerJrKrrr $sz-PoolDirector.__getstate__..)__dict__rDr0rrr __getstate__#szPoolDirector.__getstate__statecCs&|j|d|dd|j|dS)Nrrrr)r-rupdate)r,rrrr __setstate__&s zPoolDirector.__setstate__) r3r4r5r:r-rrrrr8rrrrrrrr s rrc&@seZdZdeegdfdedededededed eed eed eed ed edeegej fdededeegee fdededdf&ddZ dej fddZ dS)Config log_callbackNrrazure_write_chunk_sizegoogle_write_chunk_sizeretry_log_thresholdretry_common_log_threshold retry_limitconnect_timeout read_timeoutoutput_az_paths&use_azure_storage_account_key_fallbackruse_streaming_readdefault_buffer_size get_deadlinesave_access_token_to_diskmultiprocessing_start_methodrcCs||_||_||_||_||_||_||_||_| |_| |_ | |_ | |_ ||_ ||_ ||_||_||_| durC|tks?|tkrC|d| |_dS)Nzwarning: max_connection_pool_count and connection_pool_max_size are no longer supported, set get_http_pool instead if you want to control http pooling)rrrrrrrrrrrrrrrrr!DEFAULT_MAX_CONNECTION_POOL_COUNT DEFAULT_CONNECTION_POOL_MAX_SIZE_get_http_pool)r,rrrrrrrrrrrrrrrrrrrrrr-8s0 zConfig.__init__cCs|jdur tS|Sr+)rglobal_pool_directorrr0rrrris zConfig.get_http_pool) r3r4r5rr8r:r r9rrrr-rrrrrr7sR       1rc@sleZdZdZdedededdfddZdefd d Zej fd ed eddfd dZ dde edefddZ dS) WindowedFilez< A file object that reads from a window into a file fr=r>rNcCs&||_||_||_d|_|ddS)Nr)_f_start_end_posseek)r,rr=r>rrrr-urzWindowedFile.__init__cCs |j|jSr+)rrr0rrrtell|s zWindowedFile.telloffsetwhencecCsL|j|}|tjkr|j|kr|jksJJ|j||||_dSr+)rioSEEK_SETrrrr)r,rrnew_posrrrrs * zWindowedFile.seekncCs|j|jksJ|dur|j|j}t||j|j}|j|}|jt|7_|dkr8t|dkr8td|j|jks@J|S)Nrz0failed to read expected amount of data from file)rrminrreadlenrM)r,rbufrrrrs  zWindowedFile.readr+) r3r4r5r6rr:r-rrrrr rrrrrrps r_hostname_check_cachehostnamecCsZdtdtfdd}t}|tvrt|d|krt|dS||}|d|ft|<|S)Nrrc Ssz tj|dtjdWtStjyM}z4|jtjkr#tWYd}~Sz tjddtjdWntjy@tYWYd}~SwtWYd}~Sd}~ww)N)familyz www.bing.com) socket getaddrinfoAF_INETgaierrorerrno EAI_NONAMEHOSTNAME_STATUS_UNKNOWNHOSTNAME_DOES_NOT_EXISTHOSTNAME_EXISTS)rerrrinners z_check_hostname..innerrrr)r8r:timer)rrnowretrrr_check_hostnames rc@s eZdZdS)TimeoutN)r3r4r5rrrrrsrfpsocknbytes_to_readdeadlinec Cs|t}|dkrtd||z|}Wn tjy%tdwt|}t|}||dt|<t|}||krq|t}|dkrLtd||z | ||d} Wn tjyhtdw|| 7}||ks>t |S)Nrz1Did not read enough bytes before deadline expiredzTimed out waiting for read) rr settimeoutpeekrtimeout bytearray memoryviewr recv_intobytes) rrrrr initial_datarmv nbytes_readrrrr_read_with_deadlines4      rconf build_reqc CsrttD].\}}|}|j}|jdur't|jdkr'|dtj|j7}d}t|j t rBt |j j d}t ||j j|j jd}n|j }|j}d} d} |jdur|jdkr|} | dur|sdJdd}t} | | krtjd|tjjdtd d d | | } d} zrz|j|j||j|tj|j|j | d |ddd } | dur| j!ddkr| j nB| j"j#}|j$j%}zt&||t'| jd| d}| (tjj| j)| j|d} Wnty}z| *tjd||| d d}~ww| j)|j+vr | WW|dur |*SSd| j)}t,}|-t.r;| j)dvr%|d7}n)| j)dkr0|d7}n| j)dkr:|d7}n| j)dkrN| j!ddkrNt/}d}|j||| d } | j)|j0vr_| Wntj1j2tj1j3tj1j4tj1j5t6j5fy}zrt|tj1j7rtj8|j}|j9dusJ|j9:drt;|j9tdur||j>kr| |t?|t@| kr/|Ad"| d#|d$|d%|d&d' tB|qJd())Nr?rbr=r>HEADz9preload_content must be set to True when using a deadlineFz(refusing to send request due to deadline)rubody)rNrtrb)connectrtotal)r#r$r&rrr(retriesredirectzTransfer-EncodingchunkedzContent-Length)rrrr)rur&rzrequest failed with exception zunexpected status )rrz: if you are writing a blob this error may be due to multiple concurrent writers - make sure you are not writing to the same blob from multiple processes simultaneouslyiz: no valid credentials were found, please login with 'gcloud auth application-default login' or else set the 'GOOGLE_APPLICATION_CREDENTIALS' environment variable to the path of a JSON format service account keyiz: credentials were found but do not grant access to this resource, please make sure the account you are using (either via 'gcloud auth application-default login' or the 'GOOGLE_APPLICATION_CREDENTIALS' environment variable) has accessizx-ms-error-codeConditionNotMetz etag mismatchz.blob.core.windows.netzhost does not exist)rtrberror z when executing http request attempt , sleeping for .1f seconds before retrying unreachable)C enumerater r$r%rrErF urlencode isinstancer'r;openr<rr=r>r(rr#rr~rwrrb HTTPResponserBytesIOrrtr&rrrr\_fprraw_sockrr: release_connrucloser)rnrZ GCP_BASE_URLrr* exceptionsConnectTimeoutErrorReadTimeoutError ProtocolErrorSSLErrorsslNewConnectionErrorurlparserendswithrrINVALID_HOSTNAME_STATUSrget_log_threshold_for_errorr8rsleep)rrattemptbackoffreqr$rrr(r total_timeoutrrjresprrrrN error_class fake_resprrrexecute_requests          B          -  r cs&eZdZdedeffdd ZZS) TupleEncoderorcs&dtdtffdd t|S)NitemrcsTt|tr d|dSt|trfdd|DSt|tr(fdd|DS|S)NT) __tuple__rDcsg|]}|qSrr)rer hint_tuplesrr sz.hint_tuples..csi|] \}}||qSrr)rekeyrgr%rrrsz.hint_tuples..)rtuplelistdictrD)r#r%rrr&s    z(TupleEncoder.encode..hint_tuples)rrPencode)r,r"rQr%rr,s zTupleEncoder.encode)r3r4r5rr,rSrrrQrr!sr!objcCsd|vr t|dS|S)Nr$rD)r))r-rrrhinted_tuple_hooks r.c@speZdZdZdeeegeeeffde ddfddZ dd Z d ee gdffd d Z d ededefddZ dS) TokenManagerz7 Automatically refresh tokens when they expire get_token_fnrrNcCsL||_i|_i|_t|_tjd|d|_ tjd|d|_ dS)Nz ~/.blobfile/z _tokens.lockz _tokens.json) _get_token_fn_tokens _expirationsrr_lockrr< expanduser_access_lock_file_access_token_file)r,r0rrrrr-s  zTokenManager.__init__cCstj|jrLt|j6}tj|td}t|d|dD] \}}||j |<qt|d|dD] \}}||j |<q0WddS1sEwYdSdS)N) object_hook token_keys token_valuesexpiration_keysexpiration_values) rr<existsr7rr^loadr.zipr2r3)r,rtokensr(rgrrr_load_token_files   "zTokenManager._load_token_filerc Cstjtj|jddzltj|jddYtjtj|jdd|jd}t|d*}| t t |j t |j t |jt |jdWdn1sYwYt||jWdWdS1sqwYWdStjy|dYdSw) NT)exist_okr)rz.tmpw)r9r:r;r<z>Another instance of this application currently holds the lock.)rmakedirsr<dirnamer6filelockFileLockr7rwriter!r,r*r2keysvaluesr3replacer)r,rtmp_pathrrrr_save_token_files*       &zTokenManager._save_token_filerr(cCs|jkt}|j|}|dus|t|kr[|jrD||j|}|dus/|t|krC|||\|j|<|j|<| |j n|||\|j|<|j|<|j|dus[J||jvsbJ|j|WdS1sqwYdSr+) r4rr3r\EARLY_EXPIRATION_SECONDSrrAr1r2rMr)r,rr(r expirationrrr get_tokens    $zTokenManager.get_token)r3r4r5r6rrrr rr8r-rArMrPrrrrr/s * r/cseZdZdededdfddZdededdfd d Zd"d ededefd dZ d#fdd Z defddZ defddZ de defddZdedefddZdejfddZd$dede fddZdedefd d!ZZS)%BaseStreamingWriteFiler chunk_sizerNcCsd|_t|_||_||_dSr)_offsetr_buf _chunk_size_conf)r,rrRrrrr-s zBaseStreamingWriteFile.__init__chunkfinalizecCtr+NotImplementedError)r,rWrXrrr _upload_chunkz$BaseStreamingWriteFile._upload_chunkFrcCsj|rt|}nt||j|j}|dksJz|d|}||||jt|7_W~~|S~~wr)rrUr\rS)r,rrXrrWrrr _upload_bufs    z"BaseStreamingWriteFile._upload_bufcsF|jrdS|jt|jdd}|t|jksJt|_tdS)NT)rX)closedr^rrTrrrPr r,rrQrrr s zBaseStreamingWriteFile.closecCs|jt|jSr+)rSrrTr0rrrrszBaseStreamingWriteFile.tellcCdSNTrr0rrrwritabler]zBaseStreamingWriteFile.writablebcCst|jdkr!t||jkr!t|}||}t||d|_n(|j|7_t|j|jkrIzt|j}||}t||d|_W~n~wt|j|jksSJt|Sr)rrTrUrr^r)r,rdrrrrrrHs   zBaseStreamingWriteFile.writecC tdNz not readablerUnsupportedOperationr,rdrrrreadinto zBaseStreamingWriteFile.readintocCre)Nzno underlying raw streamrgr0rrrdetachrkzBaseStreamingWriteFile.detachrrcCrerfrgr`rrrread1rkzBaseStreamingWriteFile.read1cCrerfrgrirrr readinto1 rkz BaseStreamingWriteFile.readinto1)FrN)r)r3r4r5rr:r-rr9r\r^r rrcrrHrrjr RawIOBaserlrmrnrSrrrQrrQs rQc seZdZdedededdffdd Z dded ed eedd fd d Z de fddZ de deefddZ ejfdededefddZdefddZd fdd ZdefddZdefddZZS)!BaseStreamingReadFilerr<rrNcs>t||_||_||_d|_d|_d|_d|_d|_ dSr) rPr-rV_size_pathrSrrequestsfailures bytes_read)r,rr<rrQrrr-%s  zBaseStreamingReadFile.__init__ streamingr=r>rccCrYr+rZ)r,rwr=r>rrr_request_chunk1sz$BaseStreamingReadFile._request_chunkcCsjg} |j|j}|dksJd|tt|}|dus!Jd|}t|dkr*n||qd|S)NTrzread more bytes than expectedfile is in non-blocking moder)rrrSrr CHUNK_SIZErappendri)r,piecesbytes_remaining opt_piecepiecerrrreadall6s    zBaseStreamingReadFile.readallrdc CsD|j|j}|dkst|dkrdSt|tst|}t||kr'|d|}d}|jjrttD]\}}|j durU|j d|jd}|j dkrKdS||_ |j d7_ d}z"|j |}|dusfJd|}|dkrutd|j}nWnqWn(tjjtjjtjjtjfy} ztd| d |j}WYd} ~ nd} ~ ww|j d|_ |jd7_|jjdur||jjkr||t|jt|kr|jd |d t|d |jd |d|dd t|q2n(|j d|j|jt|d}|j dkrdS|j d7_ t|j}|j|d|<|j|7_|j|7_|S)NrT)rwr=irryz5failed to read from connection while reading file at z exception z while reading file at rz when executing readinto(z ) at offset rrrrF)rwr=r>) rrrSrrrrVrrr rrxrurtrjrMrsrr rrrrr rurrr8rrrr'rv) r,rdr}rrrrrjopt_nrrrrrjJst         " *   zBaseStreamingReadFile.readintorrc Cs|tjkr|}n*|tjkr|j|}n|tjkr|j|}ntd|dtjdtjdtjd ||jkrG||_|jdurD|jd|_|jS)NzInvalid whence (z , should be rdz, or )) rrSEEK_CURrSSEEK_ENDrr ValueErrorrr )r,rr new_offsetrrrrs     "   zBaseStreamingReadFile.seekcCs|jSr+)rSr0rrrrszBaseStreamingReadFile.tellcs<|jrdSt|dr|jdur|jd|_tdS)Nr)r_hasattrrr rPr0rQrrr s  zBaseStreamingReadFile.closecCrarbrr0rrrreadabler]zBaseStreamingReadFile.readablecCrarbrr0rrrseekabler]zBaseStreamingReadFile.seekabler+ro)r3r4r5rr8r:r-r9r rxrrrrjrrrrr rrrSrrrQrrq$s&  NrqrcCs2t} |t}|dkr |S||q)NTr)hashlibrrrzrdigest)rmblockrrr block_md5s  rr=r>cCsn|dur|durd|d|dS|durd|dS|dur3|dkr+d|dSdt| Std)Nzbytes=-rrzbytes=0-zbytes=-z Invalid range)r:rMrrrr calc_ranges rr<cCs$|dr|dd}|ds|S)N/r)rr<rrr strip_slashess   rardcCsd|vrd|vs J|dkr|S|dkr|S|dddkr$|d}n$|d}|ddkr2|d=||d}dd|ddD|dd<g}|D]}|dkr]t|d kr\|qL|d krbqL||qL|dd vrs|dd|S) Nz://rBrrrcSsg|]}|dkr|qS)rBr)resrrrr'szpath_join....r.)rr)splitrpopr{ri)rrdsegments base_parts resolved_pathsegrrr path_joins0        rrjcs"tfddtDr|jS|jS)Nc3s|]}|vVqdSr+r)resubstrrjrrrhsz.get_log_threshold_for_error..)anyCOMMON_ERROR_SUBSTRINGSrr)rrjrrrrsrc@seZdZdZdefddZdS) BlobPathLikez>Similar to the __fspath__ protocol, but for remote blob paths.rcCsdSr+rr0rrr __blobpath__r]zBlobPathLike.__blobpath__N)r3r4r5r6r8rrrrrrsrzos.PathLike[str]cCs*t|tr |St|tjr|S|Sr+)rrrrPathLike __fspath__rrrr path_to_strs  rrY)[rrr^rrrrrr urllib.parserEtypingrrrrrrrr r r r r rrFrblobfilerr[rzrrPARALLEL_COPY_MINIMUM_PART_SIZErNrBACKOFF_INITIAL BACKOFF_MAXrrrr r7r SystemRandomrrr r!r;r8rLr]rMrrarmrnrzr}r~rrrrrrrrr:rrrrr  JSONEncoderr!r.r/BufferedIOBaserQrprqrrrrrrRemoteOrLocalPathrrrrrs <  " 6&9"& #EJ $