o iA@sdZddlZddlZddlZddlZddlmZddlmZddl m Z ddl m Z ddl mZmZmZmZmZmZmZmZddlmZdd lmZd d lmZmZmZmZmZm Z m!Z!m"Z"d d l#m$Z$d d l%m&Z&m'Z'ertd dl(m)Z)e *e+Z,e-dZ.dZ/dddZ0eGdddZ1e"   dDdee1dee2de2de2dee2dee2deee2e2fdeee3ee3ffddZ4Gdd d eZ5Gd!d"d"eZ6   dDd#d$d%edee2deee2e2fdee2ddf d&d'Z7d(e3fd)d*Z8d+e3fd,d-Z9d.e3fd/d0Z:d#d$d1e2ddfd2d3Z;d#d$d4ed5ed;eedZ?d#d$d?ee2d5es z'post_lfs_batch_info..r) operation transfersobjects hash_algonameref)rBrGjsonrUzMalformed response from servercSsg|] }d|vrt|qSerror)_validate_batch_actionsrQobjr+r+r,rRcSsg|] }d|vrt|qSr[)_validate_batch_errorr^r+r+r,rRr`)rENDPOINTREPO_TYPES_URL_PREFIXESr LFS_HEADERSrrpostrrZget isinstancelist ValueError) rArBrCrDrErFrG url_prefix batch_urlpayloadresp batch_inforUr+r+r,post_lfs_batch_infoes<(       roc@seZdZUeed<eed<dS) PayloadPartT partNumberetagN)r8r9r:r>r=r@r+r+r+r,rps  rpc@s&eZdZUdZeed<eeed<dS)CompletionPayloadTz?Payload that will be sent to the Hub when uploading multi-part.rOpartsN)r8r9r:r;r@r=r rpr+r+r+r,rss rsrSrlfs_batch_actionc CsHt||d}|durtd|jddS|dd}t||dd}|dur2t||di}|d} t|d |d } | durjzt| } Wntt fy`td | d wt ||| | d nt || d|durt|t|d |} t j | t||d|jj|jjdd} t| t|jddS)aw Handles uploading a given object to the Hub with the LFS protocol. Can be a No-op if the content of the file is already present on the hub large file storage. Args: operation (`CommitOperationAdd`): The add operation triggering this upload. lfs_batch_action (`dict`): Upload instructions from the LFS batch endpoint for this object. See [`~utils.lfs.post_lfs_batch_info`] for more details. headers (`dict`, *optional*): Headers to include in the request, including authentication and user agent headers. Raises: [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) If `lfs_batch_action` is improperly formatted [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError) If the upload resulted in an error actionsNzContent of file z. is already present upstream - skipping uploadrKverifyheader chunk_sizehref)rFzTMalformed response from LFS batch endpoint: `chunk_size` should be an integer. Got 'z'.)rSrxry upload_url)rSr{)rBrGrNrYz: Upload successful)r]rfloggerdebug path_in_repo_validate_lfs_actionrr>ri TypeError_upload_multi_part_upload_single_partrrer upload_inforrPrr) rSrurBrGrFrv upload_action verify_actionrxryr{ verify_url verify_respr+r+r, lfs_uploadsB        r lfs_actioncCs:t|dtr|ddust|dtstd|S).validates response from the LFS batch endpointrzrxNz"lfs_action is improperly formatted)rgrfr@dictri)rr+r+r,rs rlfs_batch_actionscCspt|dtrt|dtstd|did}|did}|dur.t||dur6t||S)rrOrz)lfs_batch_actions is improperly formattedrvrKrwN)rgrfr@r>rir)rrrr+r+r,r]s r]lfs_batch_errorcCsht|dtrt|dtstd|d}t|tr.t|dtr.t|dts2td|S)rrOrz'lfs_batch_error is improperly formattedr\messagecode)rgrfr@r>rir)r error_infor+r+r,ras  rar{cCsJ|jdd}td||dd}t|WddS1swYdS)a Uploads `fileobj` as a single PUT HTTP request (basic LFS transfer protocol) Args: upload_url (`str`): The URL to PUT the file to. fileobj: The file-like object holding the data to upload. Returns: `requests.Response` Raises: [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError) If the upload resulted in an error. T with_tqdmPUTiiiir.retry_on_status_codesN)as_filerr)rSr{r2responser+r+r,r,s "rrxrycCst||j|d}tj}tjr!t|jts!t|jts!t dd}|r*t |||dnt |||d}t j |t||jjtd}t|dS)z@ Uploads file using HF multipart LFS transfer protocol. )rxrryzlhf_transfer is enabled but does not support uploading from bytes or BinaryIO, falling back to regular uploadF)rSsorted_parts_urlsry)rZrGN)_get_sorted_parts_urlsrrHF_HUB_ENABLE_HF_TRANSFERrgpath_or_fileobjr@rwarningswarn_upload_parts_hf_transfer_upload_parts_iterativelyrre_get_completion_payloadrrPrdr)rSrxryr{ruse_hf_transferresponse_headerscompletion_resr+r+r,rBs,    rrcCsNddtdd|DdddD}t|}|t|j|kr%td|S)NcSsg|]\}}|qSr+r+)rQ_r{r+r+r,rRfsz*_get_sorted_parts_urls..cSs2g|]\}}|rt|dkrt|d|fqS)r )isdigitr0r>)rQpart_numr{r+r+r,rRis  cSs|dS)Nrr+)tr+r+r,nsz(_get_sorted_parts_urls..)keyz0Invalid server response to upload large LFS file)sorteditemsr0rrri)rxrrysorted_part_upload_urls num_partsr+r+r,res rrrOcCsfg}t|D]'\}}|d}|dus|dkr#td|d|d||d|dq||dS)NrrrIzInvalid etag (`z`) returned for part r)rqrr)rOrt) enumeraterfriappend)rrOrt part_numberrxrrr+r+r,rws  rrc Csg}|jdd=}t|D]/\}}t||||d}td||dd}t|||jWdn1s7wYq Wd|S1sHwY|S)NTr) seek_from read_limitrrr)rrrrrrrG) rSrryrGr2part_idxpart_upload_url fileobj_slicepart_upload_resr+r+r,rs*   rc Cs(zddlm}Wn tytdwdt|jv}|s#td|j j }|j }t |dkr9d|dd}t tjkrBd nd}td d |d||d d ;}z|d|j||d ddd|rcd|jini} Wntyy} ztd| d} ~ ww|s||| WdS1swYdS)Nr)multipart_uploadzFast uploading using 'hf_transfer' is enabled (HF_HUB_ENABLE_HF_TRANSFER=1) but 'hf_transfer' package is not available in your environment. Try `pip install hf_transfer`.callbackzYou are using an outdated version of `hf_transfer`. Consider upgrading to latest version to enable progress bars using `pip install -U hf_transfer`.(u(…)iTBzhuggingface_hub.lfs_upload)unit unit_scaletotalinitialdescdisablerW) file_path parts_urlsry max_filesparallel_failures max_retriesz~An error occurred while uploading using `hf_transfer`. Consider disabling HF_HUB_ENABLE_HF_TRANSFER for better error handling.r+) hf_transferr ImportErrorriinspect signature parametersrrrrr~r0r|getEffectiveLevelrNOTSETrrupdate Exception RuntimeError) rSrryrsupports_callbackrrrprogressoutputer+r+r,rsd     $r)NNN)Br;rr%rer dataclassesrmathros.pathrpathlibrtypingrrrr r r r r urllib.parserhuggingface_hubrutilsrrrrrrrr utils._lfsr utils.sharr _commit_apir get_loggerr8r|compile OID_REGEXLFS_MULTIPART_UPLOAD_COMMANDrdrr@rrorprsrrr]rarr>rrrrrr+r+r+r,s    (  (    ) N  F #