o
    i	G                     @   s   d dl Z d dlZddlmZ ddlmZmZmZmZm	Z	m
Z
mZmZmZmZmZmZmZmZmZ G dd deZG dd	 d	eZG d
d deZG dd deZG dd deZdS )    N   )InfoExtractor   )ExtractorError
clean_htmldetermine_extextract_attributesfloat_or_noneget_elements_html_by_classint_or_nonemerge_dictsmimetype2extparse_iso8601
remove_endremove_startstr_or_nonetraverse_objurl_or_nonec                   @   sB   e Zd ZedZdZdZdZdd Z	dd Z
d	d
 Zdd ZdS )NYTimesBaseIEz$36dd619a-56dc-595b-9e09-37f4152c7b5da  MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAuNIzKBOFB77aT/jN/FQ+/QVKWq5V1ka1AYmCR9hstz1pGNPH5ajOU9gAqta0T89iPnhjwla+3oec/Z3kGjxbpv6miQXufHFq3u2RC6HyU458cLat5kVPSOQCe3VVB5NRpOlRuwKHqn0txfxnwSSj8mqzstR997d3gKB//RO9zE16y3PoWlDQXkASngNJEWvL19iob/xwAkfEWCjyRILWFY0JYX3AvLMSbq7wsqOCE5srJpo7rRU32zsByhsp1D5W9OYqqwDmflsgCEQy2vqTsJjrJohuNg+urMXNNZ7Y3naMoqttsGDrWVxtPBafKMI8pM2ReNZBbGQsQXRzQNo7+QIDAQABz/https://samizdat-graphql.nytimes.com/graphql/v2a  query VideoQuery($id: String!) {
  video(id: $id) {
    ... on Video {
      bylines {
        renderedRepresentation
      }
      duration
      firstPublished
      promotionalHeadline
      promotionalMedia {
        ... on Image {
          crops {
            name
            renditions {
              name
              width
              height
              url
            }
          }
        }
      }
      renditions {
        type
        width
        height
        url
        bitrate
      }
      summary
    }
  }
}c                 C   sr   t | jd}t ||}t| j| j|dtj| jdd| iddd	 dd	d
| j
ddddddthfp8i S )Nvideoz!Downloading JSON from GraphQL APIidznyt://video/)query	variables),:)
separatorszapplication/jsonvhszv3.52.21zhttps://nytimes.com)zContent-TypezNyt-App-TypezNyt-App-Versionz	Nyt-TokenOriginF)dataheadersfatalr   )uuiduuid5_DNS_NAMESPACEr   _download_json_GRAPHQL_APIjsondumps_GRAPHQL_QUERYencode_TOKENdict)selfmedia_id
video_uuid
media_uuid r0   L/home/ubuntu/.local/lib/python3.10/site-packages/yt_dlp/extractor/nytimes.py	_call_api>   s,   


zNYTimesBaseIE._call_apic                 C   s*   t |dd ddthfdthfdfd dS )Nc                 S   s   t |d S )Nurl)r   )_vr0   r0   r1   <lambda>P       z3NYTimesBaseIE._extract_thumbnails.<locals>.<lambda>r3   widthheight)r3   r8   r9   default)r   r   )r,   thumbsr0   r0   r1   _extract_thumbnailsO   s   
z!NYTimesBaseIE._extract_thumbnailsc                 C   s"  g }g }i }t |ddthfD ]}}|d}|d}|r%|dks%||v r&q|| t|dp5t|}	|	dkrV| j||dd	|pCd
dd\}
}||
 | j||d q|	dkr[q||||dph|dt	|dt	|dt |ddt	hfddt	|ddpd |	d q||fS )N
renditions.r3   typer<   mimetypem3u8mp4m3u8_nativehlsF)m3u8_idr    )targetmpdvideoencodingvideo_codecr8   r9   )	file_sizefileSize)Nvalueget_allbitrate  )r3   	format_idvcodecr8   r9   filesizetbrext)
r   r+   getappendr   r   #_extract_m3u8_formats_and_subtitlesextend_merge_subtitlesr   )r,   video_idcontent_media_jsonurlsformats	subtitlesr   	video_urlrQ   rU   	m3u8_fmts	m3u8_subsr0   r0   r1   _extract_formats_and_subtitlesV   sD   





z,NYTimesBaseIE._extract_formats_and_subtitlesc                 C   s|   |  |}| ||\}}||d|dt|dt|ddddt|dd	d
dd hf||| t|dd	S )NpromotionalHeadlinesummaryfirstPublisheddurationrP   scalez, bylines.renderedRepresentationc                 S   s
   t | dS )NzBy )r   xr0   r0   r1   r6      s   
 z.NYTimesBaseIE._extract_video.<locals>.<lambda>promotionalMediacrops.r>   .)	r   titledescription	timestamprg   creatorr^   r_   
thumbnails)r2   rc   rV   r   r	   joinr   r=   )r,   r-   r   r^   r_   r0   r0   r1   _extract_videox   s    
zNYTimesBaseIE._extract_videoN)__name__
__module____qualname__r!   UUIDr#   r*   r%   r(   r2   r=   rc   rw   r0   r0   r0   r1   r      s    
""r   c                   @   sH   e Zd ZdZdgZddddddd	d
dddd	ddddgZdd ZdS )	NYTimesIEzhttps?://(?:(?:www\.)?nytimes\.com/video/(?:[^/]+/)+?|graphics8\.nytimes\.com/bcvideo/\d+(?:\.\d+)?/iframe/embed\.html\?videoId=)(?P<id>\d+)zl<iframe[^>]+src=(["\'])(?P<url>(?:https?:)?//graphics8\.nytimes\.com/bcvideo/[^/]+/iframe/embed\.html.+?)\1>zshttp://www.nytimes.com/video/opinion/100000002847155/verbatim-what-is-a-photocopier.html?playlistId=100000001150263 a553aa344014e3723d33893d89d4defc100000002847155rB   z Verbatim: What Is a Photocopier?z$md5:93603dada88ddbda9395632fdc5da260it]S20140428zBrett Weinerz're:https?://\w+\.nyt.com/images/.+\.jpgi  )	r   rU   rq   rr   rs   upload_datert   	thumbnailrg   r3   md5	info_dictzJhttp://www.nytimes.com/video/travel/100000003550828/36-hours-in-dubai.htmlTr3   only_matchingc                 C   s   |  |}| |S )N)	_match_idrw   )r,   r3   r[   r0   r0   r1   _real_extract   s   

zNYTimesIE._real_extractN)rx   ry   rz   
_VALID_URL_EMBED_REGEX_TESTSr   r0   r0   r0   r1   r|      s*    r|   c                   @   s   e Zd ZdZdddddddd	d
dddd
ddddddddddddgddd	dddddd d!d"d#d$d%gdd&d'
d(d)id*d+d,d-d.d/d0d1gd2d3d4d5d6d7d8d9d:d;gd<d=d>d?d5d@dAdBgZdCdD ZdEdF ZdGS )HNYTimesArticleIEzphttps?://(?:www\.)?nytimes\.com/\d{4}/\d{2}/\d{2}/(?!books|podcasts)[^/?#]+/(?:\w+/)?(?P<id>[^./?#]+)(?:\.html)?zhttp://www.nytimes.com/2015/04/14/business/owner-of-gravity-payments-a-credit-card-processor-is-setting-a-new-minimum-wage-70000-a-year.html?_r=0 3eb5ddb1d6f86254fe4f233826778737100000003628438rB   u0   One Company’s New Minimum Wage: $70,000 a Yearz$md5:89ba9ab67ca767bb92bf823d1f138433i-U20150414zMatthew WilliamszPatricia Cohen're:https?://\w+\.nyt.com/images/.*\.jpgg     ]@)
r   rU   rq   rr   rs   r   uploaderrt   r   rg   z%HTTP Error 500: Internal Server Error)r3   r   r   skipzMhttps://www.nytimes.com/2023/09/29/health/mosquitoes-genetic-engineering.html 2365b3555c8aa7f4dd34ca735ad02e6a100000009110381mp3z<The Gamble: Can Genetically Modified Mosquitoes End Disease?z$md5:9ff8b47acbaf7f3ca8c732f5c815be2eie20230929zStephanie NolenzNatalija Gormalovai*  )	r   rU   rq   rr   rs   r   creatorsr   rg   r   zThttps://www.nytimes.com/2023/11/29/business/dealbook/kamala-harris-biden-voters.html100000009202270uU   Kamala Harris Defends Biden Policies, but Says ‘More Work’ Needed to Reach Votersz$md5:de4212a7e19bb89e4fb14210ca915f1fige20231129zBy The New York TimeszKatie RogersgMbhX@)
r   rU   rq   rr   rs   r   r   r   r   rg   skip_downloadrA   )r3   r   r   paramszOhttps://www.nytimes.com/2023/12/02/business/air-traffic-controllers-safety.htmlzair-traffic-controllers-safetyzHDrunk and Asleep on the Job: Air Traffic Controllers Pushed to the Brinkz$md5:549e5a5e935bf7d048be53ba3d2c863d20231202zEmily SteelzSydney Emberi` ke)r   rq   rr   r   r   rs      r3   r   playlist_countzFhttps://www.nytimes.com/2025/04/30/well/move/hip-mobility-routine.htmlzhip-mobility-routinez!Tight Hips? These Moves Can Help.zRSitting all day is hard on your hips. Try this simple routine for better mobility.zAlyssa AgeszTheodore Taeih20250430)r   rq   rr   r   rs   r      zShttps://www.nytimes.com/2023/12/02/business/media/netflix-squid-game-challenge.htmlTr   c              	   C   s   t |dthfdddthfd dtddhfdthfffd	thfd
thfddd}| |d|\}}t |dthf}|sG|rG||dd i || 	t |d||dS )NsourceIdrj   .rk   rg   rP   rh   lengthrf   podcastSeries)r   r   rg   rs   seriesFrM   r   fileUrlnone)r3   rR   rn   )ru   r^   r_   )
r   strr	   r   r   rc   rV   r   rW   r=   )r,   blockdetailsr^   r_   r3   r0   r0   r1   _extract_content_from_block   s,   z,NYTimesArticleIE._extract_content_from_blockc                 C   sp  |  |}| j||dd}| jd|d|dd dd d	 d
 }|d d }g }dd }t|dd|tf }rD|d|d || |t|d|f |sTt	dt
| |dt|dddddddthfddpr| ddg|t|dthft|dddddthf| t|dd}	g }
|D ]}|
t| ||	 qt|
dkr| j|
|fi |	S d |i|
d! S )"NT)impersonatezwindow\.__preloadedData\s*=zmedia detailsc                 S   s   |  ddS )N	undefinednull)replacerl   r0   r0   r1   r6     r7   z0NYTimesArticleIE._real_extract.<locals>.<lambda>)transform_sourceinitialDatar   articlesprinkledBodycontentc                 S   s   | dko	|d dv S )Nmedia
__typename)VideoAudior0   )kr5   r0   r0   r1   r6         .	ledeMediar   z/Unable to extract any media blocks from webpagez - The New York Timesre   textFrM   og:descriptiontwitter:descriptionrf   rj   r   displayName)ro   
assetCrops.r>   .)rq   rr   rs   r   ru   r   r   r   )r   _download_webpage_search_jsonr   any
setdefaultrV   rW   rY   r   r   _html_extract_titler   _html_search_metar   r=   r   r   lenplaylist_result)r,   r3   page_idwebpageart_jsonr   blocksblock_filterlede_media_blockcommon_infoentriesr   r0   r0   r1   r     sV   

zNYTimesArticleIE._real_extractN)rx   ry   rz   r   r   r   r   r0   r0   r0   r1   r      s    Or   c                   @   sd   e Zd ZdZdZddddddd	d
ddddddddddd	dddddddddgZdd ZdS ) NYTimesCookingIENYTimesCookingGuidez4https?://cooking\.nytimes\.com/guides/(?P<id>[\w-]+)z:https://cooking.nytimes.com/guides/13-how-to-cook-a-turkeyz13-how-to-cook-a-turkeyzHow to Cook a Turkeyz$md5:726cfd3f9b161bdf5c279879e8050ca0)r   rq   rr   r   r   z@https://cooking.nytimes.com/guides/50-how-to-make-mac-and-cheese 64415805fe0b8640fce6b0b9def5989a100000005835845rB   zHow to Make Mac and Cheesez$md5:b8f2f33ec1fb7523b21367147c9594f1i`Z20180405gQ#@zAlison Romanr   )	r   rU   rq   rr   rs   r   rg   rt   r   r   z9https://cooking.nytimes.com/guides/20-how-to-frost-a-cakez20-how-to-frost-a-cakezHow to Frost a Cakez$md5:a31fe3b98a8ce7b98aae097730c269cd   )r3   r   r   r   c              	   C   s   |  |}| ||}| ddg|}| ddg|}| d|d}ttd|dthd	f}|rA|| | t	| j
||||S i | 
|||| jd
|dd ddS )Nzog:titleztwitter:titler   r   z#data-video-player-id="(\d+)"></div>z
lead videoz
video-item.zdata-video-idz*<span itemprop="author">([^<]+)</span></p>authorr:   )rq   rr   rt   )r   r   r   _search_regexr   r
   r   rW   r   maprw   )r,   r3   r   r   rq   rr   lead_video_id	media_idsr0   r0   r1   r   ]  s0   

zNYTimesCookingIE._real_extractN)rx   ry   rz   IE_NAMEr   r   r   r0   r0   r0   r1   r   8  sB    
"r   c                   @   sp   e Zd ZdZdddddddd	d
dddddddddddddddddddddddddddgZdd  Zd!S )"NYTimesCookingRecipeIEz2https?://cooking\.nytimes\.com/recipes/(?P<id>\d+)z?https://cooking.nytimes.com/recipes/1017817-cranberry-curd-tart 579e83bbe8e61e9de67f80edba8a78a81017817rB   zCranberry Curd Tartz$md5:ad77a3fc321db636256d4343c5742152iKV20151118zDavid Tanisr   )r   rU   rq   rr   rs   r   rt   r   r   zKhttps://cooking.nytimes.com/recipes/1024781-neapolitan-checkerboard-cookies 58df35998241dcf0620e99e646331b421024781zNeapolitan Checkerboard Cookiesz$md5:ba12394c585ababea951cb6d2fcc6631ige20231130zSue Liz:https://cooking.nytimes.com/recipes/1019516-overnight-oats 2fe7965a3adc899913b8e25ada3608231019516i +\z$md5:8856ce10239161bd2596ac335b9f9bfb20190102zOvernight OatszGenevieve Ko)r   rU   rs   rr   r   rq   rt   r   c                 C   s   |  |}| ||}| ||d d d }| j|d |ddd\}}i t|dthfd	thfd
thfdthfddthfd||dd t|ddddt	hfD dS )Nprops	pagePropsrecipevideoSrcrB   rD   )rE   r   rq   topnotepublishedAtcontentAttribution
cardByline)r   rq   rr   rs   rt   c                 S   s   g | ]}d |iqS )r3   r0   ).0	thumb_urlr0   r0   r1   
<listcomp>  r   z8NYTimesCookingRecipeIE._real_extract.<locals>.<listcomp>imagerp   .)r^   r_   ru   )
r   r   _search_nextjs_datarX   r   r   r   r   r   r   )r,   r3   r   r   recipe_datar^   r_   r0   r0   r1   r     s*   


z$NYTimesCookingRecipeIE._real_extractN)rx   ry   rz   r   r   r   r0   r0   r0   r1   r   v  sP    )r   )r&   r!   commonr   utilsr   r   r   r   r	   r
   r   r   r   r   r   r   r   r   r   r   r|   r   r   r   r0   r0   r0   r1   <module>   s    Ds >