o
    iV                     @   sl  d dl mZ d dlmZ d dlZd dlmZ d dlmZ d dlZd dl	Z	d dl
mZ d dlmZ d dlmZ d d	lmZ d d
lmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlm Z! d dl"m Z# d dl$m%Z% d dl&m'Z' ddl(m)Z) ddl*m+Z+ ddl*m,Z, ddl*m-Z- ddl.m/Z/ ddl.m0Z0 dd l.m1Z1 dd!l2m3Z3 d"d#l4m5Z5 d"d$l4m6Z6 d"d%l4m7Z7 d"d&l4m8Z8 d"d'l9m:Z: d"d(l9m;Z; d"d)l<m=Z= d"d*l<m>Z> d"d+l<m?Z? d"d,l<m@Z@ eeAZBdZCd-ZDd.ZEd/ZFG d0d1 d1eZGeGjHeGjIeGjJgZKG d2d3 d3eLZMG d4d5 d5eLZNdS )6    )c_int)IntEnumN)Process)Value)Optional)urljoin)ci)!_build_git_packfiles_with_details)-_extract_clone_defaultremotename_with_details)$_extract_latest_commits_with_details)_extract_upstream_sha)_get_rev_list_with_details)#_is_shallow_repository_with_details)_unshallow_repository)"_unshallow_repository_with_details)extract_commit_sha)extract_git_version)extract_remote_url)extract_workspace_path)
get_logger)config)fibonacci_backoff_with_jitter)Tracer   )	telemetry)EVP_PROXY_AGENT_BASE_PATH)EVP_SUBDOMAIN_HEADER_API_VALUE)EVP_SUBDOMAIN_HEADER_NAME)Response)get_connection)
verify_url)	StopWatch   )AGENTLESS_API_KEY_HEADER_NAME)AGENTLESS_DEFAULT_SITE)GIT_API_BASE_PATH)REQUESTS_MODE)ERROR_TYPES)GIT_TELEMETRY_COMMANDS)record_git_command)record_objects_pack_data)record_objects_pack_request)record_search_commitsz.pack   x   c                   @   s    e Zd ZdZdZdZdZdZdS )METADATA_UPLOAD_STATUSr   r"   r         N)__name__
__module____qualname__PENDING
IN_PROCESSSUCCESSFAILEDUNNECESSARY r:   r:   ]/home/ubuntu/.local/lib/python3.10/site-packages/ddtrace/internal/ci_visibility/git_client.pyr/   ?   s    r/   c                
   @   sp   e Zd ZdeddfddZdedee defdd	Zd
edee fddZdedededeee	f fddZ
dS )!CIVisibilityGitClientSerializerV1api_keyreturnNc                 C   s
   || _ d S N)r=   )selfr=   r:   r:   r;   __init__O   s   
z*CIVisibilityGitClientSerializerV1.__init__repo_urllatest_commitsc                 C   s   t d|idd |D dS )Nrepository_urlc                 S   s   g | ]}|d dqS )commitidtyper:   ).0shar:   r:   r;   
<listcomp>T   s    zKCIVisibilityGitClientSerializerV1.search_commits_encode.<locals>.<listcomp>)metadata)jsondumps)r@   rB   rC   r:   r:   r;   search_commits_encodeR   s   z7CIVisibilityGitClientSerializerV1.search_commits_encodepayloadc                 C   s   g }zt |trt| }nt|}dd |d D W S  ty/   tjddd Y |S  tjy@   tjddd Y |S w )Nc                 S   s    g | ]}|d  dkr|d qS )rH   rE   rG   r:   )rI   itemr:   r:   r;   rK   ^   s     zKCIVisibilityGitClientSerializerV1.search_commits_decode.<locals>.<listcomp>rM   z9Expected information not found in search_commits responseTexc_infoz2Unexpected decode error in search_commits response)	
isinstancebytesrN   loadsdecodeKeyErrorlogwarningJSONDecodeError)r@   rQ   resparsedr:   r:   r;   search_commits_decodeW   s   

z7CIVisibilityGitClientSerializerV1.search_commits_decoderJ   	file_pathc              	   C   s   d}d}g }|ddd|id}| d| dd	d
t|dg tj|}t|d}	|	 }
|		  | d| d|d dd
|
g | d| d d
g d|
d ||fS )Ns   ----------boundary------s   
rE   rF   rD   )rM   rL   s   --s0   Content-Disposition: form-data; name="pushedSha"s   Content-Type: application/json    zutf-8rbs>   Content-Disposition: form-data; name="packfile"; filename="%s"s&   Content-Type: application/octet-streamz multipart/form-data; boundary=%s)extendrN   rO   encodeospathbasenameopenreadcloserX   join)r@   rB   rJ   r`   BOUNDARYCRLFbodymetadata	file_nameffile_contentr:   r:   r;   upload_packfile_encodef   s4   	
	z8CIVisibilityGitClientSerializerV1.upload_packfile_encode)r2   r3   r4   strrA   listrP   r_   tuplerV   rs   r:   r:   r:   r;   r<   N   s
    &r<   c                   @   s  e Zd Zejdfdededee ddfddZ	d;dee dee fd	d
Z
d;dee ddfddZdd ZefddZdefddZe				d<	d=ddZed>deeeef  dee defddZed;dee dee fddZededededee dedee deee  fd d!Zeed"d#d$ d%defdeded&ed'eded(ee d)edefd*d+Ze	d>d,ee d-eee  dee defd.d/Ze	d;dededed0ededee dee defd1d2Zed;d3d4Z ed;d5d6Z!ed;d7d8Z"ed;d9d:Z#dS )?CIVisibilityGitClientNr=   requests_modetracerr>   c                 C   s   t || _d | _t| _|| _tttj	dd| _
| jtjkr1tj}|r'|jp&|}t|tt | _d S | jtjkrFtdtdtt| _d S d S )NT)lockzhttps://api.{}DD_SITE)r<   _serializer_workerRESPONSE	_response_requests_moder   r   r/   r5   _metadata_upload_statusr&   EVP_PROXY_EVENTSagent_configtrace_agent_url
_agent_urlr   r   r%   	_base_urlAGENTLESS_EVENTSformatre   getenvr$   )r@   r=   rx   ry   
tracer_urlr:   r:   r;   rA      s$   



zCIVisibilityGitClient.__init__cwdc              	   C   s&   zt |dW S  ttfy   Y d S w )Nr   )r   FileNotFoundError
ValueErrorr@   r   r:   r:   r;   _get_git_dir   s
   z"CIVisibilityGitClient._get_git_dirc                 C   s   | j |dstd tj| j_d S tj|d| _	| j
d u rHttj| j| j| j| jf| j	| j|tjdd| _
| j
  td| j| j
j d S td| j
 d S )Nr   z4Missing .git directory; skipping git metadata upload)_tagsr   r   	log_level)targetargskwargsz1Upload git metadata to URL %s started with PID %sz.git metadata upload worker already started: %s)r   rZ   debugr/   r8   r   valuer   tagsr   r}   r   rw   _run_protocolr|   r   r   r   levelstartpidr   r:   r:   r;   upload_git_metadata   s   



z)CIVisibilityGitClient.upload_git_metadatac                 C   s   | j jtv S r?   )r   r   !FINISHED_METADATA_UPLOAD_STATUSESr@   r:   r:   r;   metadata_upload_finished   s   z.CIVisibilityGitClient.metadata_upload_finishedc                 C   s   t d| | jd u rt d d S t K}|  sYt d| | jj | |kr1td| | jj	d urIt d| jj	| jj t
d| jj	| jjdd td |  rW d    n1 scw   Y  t d	|  d S )
Nz:Waiting up to %s seconds for git metadata upload to finishz%No git metadata upload worker startedzWaited %s so far, status is %szFTimed out waiting for git metadata upload to complete after %s secondszDgit metadata process exited with exitcode %s but upload status is %sz,git metadata process exited with exitcode %sr"   timeoutz0git metadata upload completed, waited %s seconds)rZ   r   r}   r!   r   elapsedr   r   TimeoutErrorexitcoder   rk   timesleep)r@   r   	stopwatchr:   r:   r;   _wait_for_metadata_upload   s0   


z/CIVisibilityGitClient._wait_for_metadata_uploadc                 C   s   |    | jjS r?   )r   r   r   r   r:   r:   r;   wait_for_metadata_upload_status   s   z5CIVisibilityGitClient.wait_for_metadata_upload_statusr   c	              
   C   s  t | tj|_zL|d u ri }| j||d}	| j|d}
| |||	|
||}|d u rBt d tj	|_W dt
_tjjdd d S tt|
t| }t|dkrht d tj|_W dt
_tjjdd d S | j|drt|dd	krt d
 z| j|d W n ty   t jddd Y nw | j|d}
| |||	|
||}|d u rt d tj	|_W dt
_tjjdd d S tt|
t| }| j|||d}|r/t d| t||dG\}}ttj|j|j |jdkr| j|||	||||drtj |_	 W d    W dt
_tjjdd d S tj	|_t d W d    n	1 s)w   Y  nt d tj |_t!dd W dt
_tjjdd d S W dt
_tjjdd d S dt
_tjjdd w )N)r   r   r   z2No initial backend commits found, returning early.FT)force_flushr   z=All latest commits found in backend, skipping metadata upload)r      r   z@Shallow repository detected on git > 2.27 detected, unshallowingz<Failed to unshallow repository, continuing to send pack datarS   z*No backend commits found, returning early.)excluded_commitsincluded_commitsr   z6Building and uploading packfiles for revision list: %sz'Failed to upload git metadata packfilesz5Revision list empty, no packfiles to build and upload)"rZ   setLevelr/   r6   r   _get_repository_url_get_latest_commits_search_commitsr   r8   telemetry_configDEPENDENCY_COLLECTIONr   telemetry_writerperiodicru   setlenr9   _is_shallow_repositoryr   r   r   r[   _get_filtered_revisionsr	   r)   r(   PACK_OBJECTSduration
returncode_upload_packfilesr7   r*   )cls
serializerrx   base_urlr   r   r   r   r   rB   rC   backend_commitscommits_not_in_backendrev_listpackfiles_prefixpackfiles_detailsr:   r:   r;   r      s   

1
*


z#CIVisibilityGitClient._run_protocolr   c                 C   s.   |d u ri }| tjjd}|st|d}|S )N r   )getr   gitREPOSITORY_URLr   )r   r   r   resultr:   r:   r;   r   '  s   
z)CIVisibilityGitClient._get_repository_urlc                 C   sB   t |d\}}}}ttj|| |dkr|r|dS g S t|)Nr   r   
)r   r)   r(   GET_LOCAL_COMMITSsplitr   )r   r   rC   stderrr   r   r:   r:   r;   r   0  s
   z)CIVisibilityGitClient._get_latest_commitsr   rB   rC   r   r   c                 C   s  | ||}d }t }	zz|p| ||d||}
W n# ty=   tj}td Y W t|		 d |d W d    d S w |
j
dkrq|
j
dk rKtjntj}td|
j
|
j td|
j W t|		 d |d W d    d S z||
j}|W W t|		 d |d W  d    S  tjy   tj}td	|
j Y W t|		 d |d W d    d S w t|		 d |d w 1 sw   Y  d S )
Nz/search_commitszTimeout searching commits  )error    zEError searching commits, response status code: %s , response body: %szResponse body: %sz2Error searching commits, response not parsable: %s)rP   r!   _do_requestr   r'   TIMEOUTrZ   r[   r,   r   statusCODE_4XXCODE_5XXr   rn   r_   rN   r\   BAD_JSON)r   rx   r   rB   rC   r   r   rQ   request_errorr   responser   r:   r:   r;   r   8  sP   




z%CIVisibilityGitClient._search_commits   c                 C   s
   t | tS r?   )rU   r   )r   r:   r:   r;   <lambda>d  s   
 zCIVisibilityGitClient.<lambda>)attemptsuntilendpointrQ   headersr   c                 C   s   d ||}t|ji}	|tjkrtti}	|d ur|	| z5t|}
|
j	}t
||d}tdd|||	 |d|||	 | }td|j t|}W |  |S |  w )Nz{}/repository{}r   zSending request: %s %s %s %sPOSTzResponse status: %s)r   r#   r=   r&   r   r   r   updater    rf   r   rZ   r   requestgetresponser   r   from_http_responserj   )r   rx   r   r   rQ   r   r   r   url_headers
parsed_urlurl_pathconnrespr   r:   r:   r;   r   c  s(   


z!CIVisibilityGitClient._do_requestr   r   c                 C   s:   t |||d\}}}}ttj||dkr| |S d  |S Nr   r   )r   r)   r(   GET_OBJECTS)r   r   r   r   filtered_revisions_r   r   r:   r:   r;   r     s   z-CIVisibilityGitClient._get_filtered_revisionsr   c                 C   s  t |d}|d}	d|	d d }
|	d }d}d}t|
D ]}||r,|ts-q tj|
|}|	|||\}}d|i}t
 }d }znz:|pU| j||d|||d}|jdkrf|d	7 }|t|7 }n|jd
krtd|t||j |jdk r~tjntj}W n& ty   tj}td| Y n ty   tj}td|t| Y nw W | d }t|| n| d }t|| w W d    n1 sw   Y  q t|| td|| |jdkS )Nr   /r   zContent-Typez	/packfile)r      r"   r   zJGit packfile upload request for file %s (sizee: %s) failed with status: %sr   zBGit packfile upload request for file %s failed: connection refusedz<Git packfile upload request for file %s (size: %s) timed outr   z>Git packfiles upload succeeded, file count: %s, total size: %s)r   r   rk   re   listdir
startswithendswithPACK_EXTENSIONrf   rs   r!   r   r   r   rZ   r   r'   r   r   ConnectionRefusedErrorNETWORKr   r   r   r+   r*   )r   rx   r   rB   r   r   r   r   rJ   parts	directoryrandpackfiles_uploaded_countpackfiles_uploaded_bytesfilenamer`   content_typerQ   r   r   
error_typer   r   r:   r:   r;   r     sf   





z'CIVisibilityGitClient._upload_packfilesc                 C   s4   t |d\}}}ttj||dkr| |S d  |S r   )r   r)   r(   CHECK_SHALLOW)r   r   is_shallow_repositoryr   r   r:   r:   r;   r     s
   z,CIVisibilityGitClient._is_shallow_repositoryc                 C   s  t  }d }zt|d\}}}}|dkr2|}td| W | d }ttj|| W d    d S ztj	||d W W | d }ttj|| W d    d S  t
yh }	 ztd|	 W Y d }	~	nd }	~	ww ztj||d W W | d }ttj|| W d    d S  t
y }	 ztd|	 W Y d }	~	nd }	~	ww td t||d\}}
}}|dkrtd	 W | d }ttj|| W d    d S td
|
 |}W | d }ttj|| W d    d S | d }ttj|| w 1 sw   Y  d S )Nr   r   z Failed to get default remote: %sr   z0Could not unshallow repository to local head: %sz#Could not unshallow to upstream: %szUnshallowing to default)r   repoz"Unshallowing to default successfulzUnshallowing failed: %s)r!   r
   rZ   r   r   r)   r(   	UNSHALLOWrw   #_unshallow_repository_to_local_headr   !_unshallow_repository_to_upstreamr   )r   r   r   error_exit_coderemoter   r   	exit_coder   eunshallow_errorr:   r:   r;   r     s`   	

z+CIVisibilityGitClient._unshallow_repositoryc                 C   2   t |d}td| t|||d td d S )Nr   zUnshallowing to local head %sr   r  refspecz%Unshallowing to local head successful)r   rZ   r   r   )r   r	  r   headr:   r:   r;   r       
z9CIVisibilityGitClient._unshallow_repository_to_local_headc                 C   r  )Nr   zUnshallowing to upstream %sr  zUnshallowing to upstream)r   rZ   r   r   )r   r	  r   upstreamr:   r:   r;   r    r  z7CIVisibilityGitClient._unshallow_repository_to_upstreamr?   )NNNr   )r>   N)NN)$r2   r3   r4   r&   r   rt   intr   r   rA   r   r   r   DEFAULT_METADATA_UPLOAD_TIMEOUTr   r/   r   classmethodr   dictr   ru   r   r<   r   r   r   DEFAULT_TIMEOUTr   r   boolr   r   r   r  r  r:   r:   r:   r;   rw      s    

O*
*	!
			:#rw   )Octypesr   enumr   rN   multiprocessingr   r   re   r   typingr   urllib.parser   ddtrace.extr   ddtrace.ext.gitr	   r
   r   r   r   r   r   r   r   r   r   r   ddtrace.internal.loggerr    ddtrace.internal.settings._agentr   r   $ddtrace.internal.settings._telemetryr   ddtrace.internal.utils.retryr   ddtrace.tracer   r   r   evp_proxy.constantsr   r   r   
utils.httpr   r   r    
utils.timer!   	constantsr#   r$   r%   r&   telemetry.constantsr'   r(   telemetry.gitr)   r*   r+   r,   r2   rZ   r~   r   r  r  r/   r8   r7   r9   r   objectr<   rw   r:   r:   r:   r;   <module>   sr    	7