o
    iT                     @   s   d dl Zd dlZd dlZd dlZd dlZd dlZd dlZddlm	Z	 ddl
mZ ddlmZmZ ddlmZ ddlmZmZ dd	lmZmZmZ dd
lmZ ddlmZ G dd deZG dd de	ZdS )    N   )FileDownloader)HttpFD   )aes_cbc_decrypt_bytesunpad_pkcs7)Request)	HTTPErrorIncompleteRead)DownloadErrorRetryManagertraverse_obj)HTTPHeaderDict)ProgressCalculatorc                   @   s   e Zd Zdd ZeZdS )HttpQuietDownloaderc                 O      d S N )selfargskargsr   r   N/home/ubuntu/.local/lib/python3.10/site-packages/yt_dlp/downloader/fragment.py	to_screen      zHttpQuietDownloader.to_screenN)__name__
__module____qualname__r   to_console_titler   r   r   r   r      s    r   c                   @   s   e Zd ZdZdd Zd*ddZdd Zd	d
 Zdd Zdd Z	dd Z
d+ddZdd Zdd Zdd Zdd Zdd Zdd Zdd  Zd!d" Zd#d$ d%d$ ddd&d'd(d)ZdS ),
FragmentFDa  
    A base file downloader class for fragmented media (e.g. f4m/m3u8 manifests).

    Available options:

    fragment_retries:   Number of times to retry a fragment for HTTP error
                        (DASH and hlsnative only). Default is 0 for API, but 10 for CLI
    skip_unavailable_fragments:
                        Skip unavailable fragments (DASH and hlsnative only)
    keep_fragments:     Keep downloaded fragments on disk after downloading is
                        finished
    concurrent_fragment_downloads:  The number of threads to use for native hls and dash downloads
    _no_ytdl_file:      Don't use .ytdl file

    For each incomplete fragment download yt-dlp keeps on disk a special
    bookkeeping file with download state and metadata (in future such files will
    be used for any incomplete download handled by yt-dlp). This file is
    used to properly handle resuming, check download file consistency and detect
    potential errors. The file has a .ytdl extension and represents a standard
    JSON file of the following format:

    extractor:
        Dictionary of extractor related data. TBD.

    downloader:
        Dictionary of downloader related data. May contain following data:
            current_fragment:
                Dictionary with current (being downloaded) fragment data:
                index:  0-based index of current fragment among all fragments
            fragment_count:
                Total count of fragments

    This feature is experimental and file format may change in future.
    c                 C   s   |  d | ||||S )Nz{yt_dlp.downloader.FragmentFD.report_retry_fragment is deprecated. Use yt_dlp.downloader.FileDownloader.report_retry instead)deprecation_warningreport_retry)r   err
frag_indexcountretriesr   r   r   report_retry_fragment>   s   
z FragmentFD.report_retry_fragmentNc                 C   s2   |rd| dnd}|  d| d|dd d S )N ; z
[download]z Skipping fragment dz ...)r   )r   r"   r!   r   r   r   report_skip_fragmentC   s   zFragmentFD.report_skip_fragmentc                 C   s   | d}|rt|d |S |S )Nhttp_headers)getr   )r   	info_dicturlheadersr   r   r   _prepare_urlG   s   
zFragmentFD._prepare_urlc                 C   s   |  | | || d S r   )_prepare_frag_download_start_frag_download)r   ctxr-   r   r   r    _prepare_and_start_frag_downloadK   s   
z+FragmentFD._prepare_and_start_frag_downloadc                 C   s&   |d duo|d dko| j d S )NliveTtmpfilename-_no_ytdl_file)paramsr,   )r   r3   r   r   r   __do_ytdl_fileO   s   &zFragmentFD.__do_ytdl_filec                 C   s   d|vsJ |  | |d d\}}z=z!t| }|d d d |d< d|d v r4|d d |d< W n tyB   d	|d< Y nw W |  d S W |  d S |  w )
Nytdl_corruptfilenamer
downloadercurrent_fragmentindexfragment_indexextra_stateT)sanitize_openytdl_filenamejsonloadsread	Exceptionclose)r   r3   stream_	ytdl_datar   r   r   _read_ytdl_fileR   s   zFragmentFD._read_ytdl_filec                 C   s   |  | |d d\}}z0dd|d ii}d|v r |d |d< |dd ur-|d |d< |td|i W |  d S |  w )	Nr<   wr?   r@   rA   rB   fragment_countr>   )rC   rD   r,   writerE   dumpsrI   )r   r3   frag_index_streamrK   r>   r   r   r   _write_ytdl_file_   s   zFragmentFD._write_ytdl_filec                 C   s   d|d |d f }||p| d|| dd}d}|d j d	d
r,| | |}| |d< |d< |d ||\}	}
|	sBdS | drN| d|d< ||d< d
S )Nz	%s-Frag%dr6   rA   r+   ctx_id)r.   r+   request_datarT   r   dl
continuedlTfrag_resume_lenFfiletimefragment_filetimefragment_filename_sanitized)r,   r9   filesize_or_none	temp_namedownload)r   r3   frag_urlr-   r/   rU   fragment_filenamefragment_info_dictrX   successrK   r   r   r   _download_fragmento   s"   
zFragmentFD._download_fragmentc                 C   sd   | dsd S z| |d d\}}W n ty#   | dr"Y d S  w ||d< | }|  |S )Nr[   rbr5   )r,   rC   FileNotFoundErrorrG   rI   )r   r3   downfrag_sanitizedfrag_contentr   r   r   _read_fragment   s   

zFragmentFD._read_fragmentc              
   C   s   z+|d  | |d   W | |r| | | jdds'| |d  |d= d S | |r6| | | jddsD| |d  |d= w )Ndest_streamkeep_fragmentsFr[   )rP   flush_FragmentFD__do_ytdl_filerS   r9   r,   
try_remove)r   r3   rh   r   r   r   _append_fragment   s   




zFragmentFD._append_fragmentc              	   C   s  | ddsd|d  }|dd}|r|d| 7 }nd}| d	| j d
|  | |d  t| ji | jdddddd}| |d }d}| 	|}|dkrUd}|
|dd | |rtj| |d }| jdd}	|	r|r| | |ddu }
|d dko|dk}|
s|r|
rdnd}| | d d |d< }d|v r|d= | | n|	s|r| | d |d< }| | |d dksJ | ||\}}|
||||d d S )Nr5   F%dtotal_fragsad_fragsr    (not including %d ad)unknown (live)[] Total fragments: r<   T)
noprogresstestsleep_intervalmax_sleep_intervalsleep_interval_subtitleswbabr6   rA   rW   r;   rA   z.ytdl file is corruptz2Inconsistent state of incomplete fragment downloadz#. Restarting from the beginning ...)rV   rj   r6   complete_frags_downloaded_bytes)
setdefaultr,   r   FD_NAMEreport_destinationr   ydlr9   r]   r\   updaterm   ospathisfilerD   rM   report_warningrS   rC   )r   r3   total_frags_strrr   rV   r6   	open_mode
resume_lenytdl_file_existsrW   
is_corruptis_inconsistentmessagerj   r   r   r   r1      sv   






z!FragmentFD._prepare_frag_downloadc                    sz    d } d   dd| d  d  d dt  d	< t| fd
d} d |  d	 S )Nr   rq   rT   downloadingrA   r<   r6   )statusdownloaded_bytesrA   rO   r<   r6   startedc                    sP  | d dvrd S s  dr d d< d ur"|  dkr"d S   dd<   dd< jd< |  dp;d	}| d
i | d<  d sg d | d d   }|_|  d jd< n|  d | d dkrd  d7  < d  d<   j d<  d< jj d<  d< j	jd< 
 d S )Nr   )r   finishedrO   rT   max_progressprogress_idxelapsedtotal_bytesr   r-   ra   r5   r   rA   r   r   total_bytes_estimater   speedeta)r,   r   poptotalr   thread_reset
downloadedr   smoothr   _hook_progress)sfrag_total_bytesestimated_sizer3   rT   r-   progressr   staterq   r   r   frag_progress_hook   s<   


z;FragmentFD._start_frag_download.<locals>.frag_progress_hookrV   )r,   timer   add_progress_hook)r   r3   r-   r   r   r   r   r   r2      s   
	'zFragmentFD._start_frag_downloadc                 C   s2  |d    | |r| | |d  t |d  }|d dk}|r-| |d }n|d }|sC|r<| |d  | d dS |r}| |d |d  |d	}| j	d
r}|r}t
t t|d t |f W d    n1 sxw   Y  | |||d d||d|d|dd| dS )Nrj   r<   r   r6   r7   r   zThe downloaded file is emptyFrZ   
updatetimer   rT   r   r   )r   r   r<   r   r   rT   r   r   T)rI   rm   rn   rD   r   r\   report_error
try_renamer,   r9   
contextlibsuppressrH   r   utimer   )r   r3   r-   r   to_filer   rY   r   r   r   _finish_frag_download  sB   


	
z FragmentFD._finish_frag_downloadc                 C   s   d|vrd|d< |d s!d|d  }| dd}|r |d| 7 }nd}| d	| j d
|  | |d }||dd d S )Nr5   Frp   rq   rr   r   rs   rt   ru   rv   r<   r~   )r,   r   r   r]   r   )r   r3   r   rr   r6   r   r   r   _prepare_external_frag_downloadA  s   z*FragmentFD._prepare_external_frag_downloadc                    s(   i fdd  fdd}|S )Nc                    s,   |  vrj |   | <  |  S r   )r   urlopenr0   rG   )r.   )
_key_cacher-   r   r   r   _get_keyX  s   z&FragmentFD.decrypter.<locals>._get_keyc                    s   |d u rd S |  d}|r|d dkr|S | dp!td| d }| dp1 tdp0|d	 |d< j d
dr=|S tt||d |S )Ndecrypt_infoMETHODzAES-128IVz>8xqmedia_sequenceKEY)hls_aesuriURIrx   F)r,   structpackr   r9   r   r   )fragmentrh   r   iv)r   r-   r   r   r   decrypt_fragment]  s   

z.FragmentFD.decrypter.<locals>.decrypt_fragmentr   )r   r-   r   r   )r   r   r-   r   r   	decrypterU  s   zFragmentFD.decrypterc              
      sj  dg t |dkrj|d i S jdd}dkr% tt|d} fdd}G dd	 d	tjj	}t
jd
krIdd }ndd } fdd}g }	t|D ]#\}
\}}}|t| }|||
|||||}|	||f qYd}|	D ])\}}zz|o||}W n ty   d d< Y nw W |jdd q|jdd w  d s|st|S )z
        @params (ctx1, fragments1, info_dict1), (ctx2, fragments2, info_dict2), ...
                all args must be either tuple or list
        Tr   r   concurrent_fragment_downloads).r   is_livec                    s0   |d< | |d< j |||fi | dS )Nr   r   )tpeinterrupt_trigger)download_and_append_fragments)idxr3   	fragmentsr-   r   r   kwargsr   r   r   r   thread_func}  s   
zFFragmentFD.download_and_append_fragments_multiple.<locals>.thread_funcc                   @   s   e Zd Zdd ZdS )z?FragmentFD.download_and_append_fragments_multiple.<locals>.FTPEc                 S   r   r   r   )r   exc_typeexc_valexc_tbr   r   r   __exit__  r   zHFragmentFD.download_and_append_fragments_multiple.<locals>.FTPE.__exit__N)r   r   r   r   r   r   r   r   FTPE  s    r   ntc                 S   s4   	 z|  dW S  ty     tjjy   Y q w )NTg?)resultKeyboardInterrupt
concurrentfuturesTimeoutErrorfuturer   r   r   future_result  s   zHFragmentFD.download_and_append_fragments_multiple.<locals>.future_resultc                 S   s   |   S r   )r   r   r   r   r   r     s   c                 3   s$    | D ]} d s d S |V  qd S )Nr   r   )fgf)r   r   r   interrupt_trigger_iter  s   zQFragmentFD.download_and_append_fragments_multiple.<locals>.interrupt_trigger_iterFwait)lenr   r9   r,   _prepare_multiline_statusanyr   r   r   ThreadPoolExecutorr   name	enumeratemathceilsubmitappendr   shutdown)r   r   r   max_workersr   r   r   r   r   spinsr   r3   r   r-   r   jobr   r   r   r   &download_and_append_fragments_multipleo  s>   


	z1FragmentFD.download_and_append_fragments_multiplec                 C      dS )NFr   )r   r   r   r   <lambda>      zFragmentFD.<lambda>c                 C   s   | S r   r   )contentr   r   r   r   r     r   )T)is_fatal	pack_funcfinish_funcr   r   c             	      s  j ddsdd fddfdd}	}
tj d	d
 dd
 }|d
kr fdd}|pCtj|O}z,|||D ]#\}}} 	||d |	|
|
 | so W W d    dS qLW n ty     jdddd |jdd  w W d    n1 sw   Y  n6|D ]3}d s n,z|  |	|
|
 |d  }W n ty   drY  n w |s dS q|d ur d |   d    S )Nskip_unavailable_fragmentsTc                 S   r   )NTr   )rK   r   r   r   r     r   z:FragmentFD.download_and_append_fragments.<locals>.<lambda>c                    s  d sd S | d   d< d  d< t d}| d}|r.d|d |d	 d
 f |d< | dp7d
  fdd}tjd|D ]?}z| d d<  | d |dsgW  d S W qK ttfy } z	||_W Y d }~qKd }~w ty   r Y qKw d S )Nr   r"   rA   
last_errorr+   
byte_rangezbytes=%d-%dstartendr   Ranger@   c                    s6   r||kr d    | || |  d< d S )Nrj   r   )rI   r    )r!   r#   r$   )r3   fatalr"   r   r   r   error_callback  s   z[FragmentFD.download_and_append_fragments.<locals>.download_fragment.<locals>.error_callbackfragment_retriesrO   r.   rU   )	r   r,   r   r9   rc   r	   r
   errorr   )r   r3   r/   r   r   retryr!   )r-   r   r   r   )r3   r   r"   r   download_fragment  s8   
zCFragmentFD.download_and_append_fragments.<locals>.download_fragmentc                    sX   | r || | dS  |d s|d dS |d   d| d dS )Nr   zfragment not foundrj   z	fragment z not found, unable to continueFT)ro   r*   rI   r   )rh   r"   r3   )r   r   r   r   r   append_fragment  s   zAFragmentFD.download_and_append_fragments.<locals>.append_fragmentr   r   r   c                    s&      }| | | | d |dfS )Nr"   r[   )copyr,   )r   ctx_copy)r3   r  r   r   rc     s   
zDFragmentFD.download_and_append_fragments.<locals>._download_fragment)r[   rA   Fz;Interrupted by user. Waiting for all threads to shutdown...)is_errortbr   r   r"   r   rj   )r9   r,   r   r   r   r   r   r   mapr   ri   r   _finish_multiline_statusr   r   rP   rl   r   )r   r3   r   r-   r   r   r   r   r   r  r   r   rc   poolr   r"   frag_filenamer   r   )r3   r  r-   r   r   r   r   r   r     sj   !


z(FragmentFD.download_and_append_fragmentsr   )NN)r   r   r   __doc__r%   r*   r0   r4   rm   rM   rS   rc   ri   ro   r1   r2   r   r   r   r   r   r   r   r   r   r      s,    #

D<$Ar   )concurrent.futuresr   r   rE   r   r   r   r   commonr   httpr   aesr   r   
networkingr   networking.exceptionsr	   r
   utilsr   r   r   utils.networkingr   utils.progressr   r   r   r   r   r   r   <module>   s"    