o
    -wi                     @   s>  d Z ddlZddlZddlZddlZddlZddlZddlZddlm	Z	m
Z
 ddlmZ ddlmZ ddlmZmZmZmZmZmZmZ ddlmZ ddlZddlZddlZddlZddlm  m Z! ddl"Z"ddl#m$Z$ dd	lm%Z% ddl&Z&dd
l'm(Z(m)Z) ddl*m+Z+ ddl,m-Z-m.Z. ddl/m0Z0 ddl1m2Z2 ddl3m4Z4m5Z5 ddl6m7Z7m8Z8 e&j9Z9e&j:Z:e&j;j<j=Z=dZ>dZ?dZ@dZAdZBdZCdZDdZEdZFdZGeHeIZJeJKejL eMdrddlNmOZO eJPeOddd neQ ZReRKejL eSd ZTeRUeT eJPeR e	G d!d" d"ZVG d#d$ d$ZWG d%d& d&ZXd'e9d(eeYeZf fd)d*Z[d+eYd,eYd-eZfd.d/Z\dNd0eVd1eZfd2d3Z]d4d5 Z^dOd7d8Z_e	G d9d: d:Z`e	G d;d< d<Zad=eYd(eejb fd>d?ZcdPd'e9fd@dAZdd=eYd(dfdBdCZed'e9dDeYd(eeY fdEdFZfdPd'e9dDeeY fdGdHZgdQdIdJZhdKeejb d(ejbfdLdMZidS )RzTooling for the W&B Importer.    N)	dataclassfield)datetime)Path)AnyDictIterableIteratorListOptionalTuple)patch)gql)Report)ArtifactCollectionRun)File)coalesceremove_keys_with_none_values   )
validation)internal)PathStrPolicy)	Namespacefor_eachzartifact_errors.jsonlzartifact_successes.jsonlzrun_errors.jsonlzrun_successes.jsonl"__ART_SEQUENCE_DUMMY_PLACEHOLDER____RUN_DUMMY_PLACEHOLDER____importer_temp____temp__z./artifacts/srcz./artifacts/dst"WANDB_IMPORTER_ENABLE_RICH_LOGGING)RichHandlerT)rich_tracebackstracebacks_show_localsz)%(asctime)s - %(levelname)s - %(message)sc                   @   s|   e Zd ZU eej ed< eed< eed< eed< eed< defddZ	defd	d
Z
edefddZedefddZdS )ArtifactSequence	artifactsentityprojecttype_namereturnc                 C   s
   t | jS N)iterr%   self r/   W/home/ubuntu/sommelier/.venv/lib/python3.10/site-packages/wandb/apis/importers/wandb.py__iter__N   s   
zArtifactSequence.__iter__c                 C   s   d| j  dS )NzArtifactSequence()
identifierr-   r/   r/   r0   __repr__Q      zArtifactSequence.__repr__c                 C   s   d | j| j| j| jgS N/)joinr&   r'   r(   r)   r-   r/   r/   r0   r4   T   s   zArtifactSequence.identifier
collectionc                 C   s0   |  }t|dd d}t||j|j|j|jS )Nc                 S      t | jdS Nvintversionlstripar/   r/   r0   <lambda>[       z2ArtifactSequence.from_collection.<locals>.<lambda>key)r%   sortedr$   r&   r'   typer)   )clsr:   artsr/   r/   r0   from_collectionX   s   z ArtifactSequence.from_collectionN)__name__
__module____qualname__r   wandbArtifact__annotations__strr	   r1   r5   propertyr4   classmethodr   rL   r/   r/   r/   r0   r$   F   s   
 r$   c                   @   s  e Zd Zdedededededdfdd	Zdefd
dZdefddZdefddZdefddZ	de
eef fddZde
eef fddZdee
eef  fddZdee fddZdee fddZdefddZdee fddZdeee  fd d!Zdeee  fd"d#Zdeee  fd$d%Zdee fd&d'Zdee fd(d)Zdee fd*d+Zdee fd,d-Zdee fd.d/Zdee fd0d1Z dee fd2d3Z!dee fd4d5Z"dee# fd6d7Z$dee# fd8d9Z%dee# fd:d;Z&dee# fd<d=Z'dee fd>d?Z(dee fd@dAZ)deee*e+e,f   fdBdCZ-deee  fdDdEZ.de
eef fdFdGZ/de
eef fdHdIZ0dee
eef  fdJdKZ1dee fdLdMZ2dNedee fdOdPZ3dQedee fdRdSZ4dS )TWandbRunrunsrc_base_urlsrc_api_keydst_base_urldst_api_keyr*   Nc                C   sJ   || _ tj|d|id| _tj|d|id| _d | _d | _d | _d | _d S )Nbase_urlapi_key	overrides)	rW   rP   Apiapidst_api_files
_artifacts_used_artifacts_parquet_history_paths)r.   rW   rX   rY   rZ   r[   r/   r/   r0   __init__f   s   	
zWandbRun.__init__c                 C   s(   t j|  |  |  }d| dS )Nz	WandbRun(r2   )ospathr9   r&   r'   run_idr.   sr/   r/   r0   r5      s   zWandbRun.__repr__c                 C      | j jS r+   )rW   idr-   r/   r/   r0   rj         zWandbRun.run_idc                 C   rm   r+   )rW   r&   r-   r/   r/   r0   r&      ro   zWandbRun.entityc                 C   rm   r+   )rW   r'   r-   r/   r/   r0   r'      ro   zWandbRun.projectc                 C   rm   r+   )rW   configr-   r/   r/   r0   rp      ro   zWandbRun.configc                 C   s   | j j}|S r+   )rW   summaryrk   r/   r/   r0   rq      s   zWandbRun.summaryc                 c   sZ    | j d u rt|  | _ | j r|  }n
td | j }|D ]	}t|}|V  q!d S )NzHNo parquet files detected; using scan history (this may not be reliable))	rf   list_get_parquet_history_paths$_get_rows_from_parquet_history_pathsloggerwarningrW   scan_historyr   )r.   rowsrowr/   r/   r0   metrics   s   


zWandbRun.metricsc                 C   rm   r+   )rW   groupr-   r/   r/   r0   	run_group   ro   zWandbRun.run_groupc                 C   rm   r+   )rW   job_typer-   r/   r/   r0   r}      ro   zWandbRun.job_typec                 C   rm   r+   )rW   display_namer-   r/   r/   r0   r~      ro   zWandbRun.display_namec                 C   sD   d| j j }d| j jj }||g}| j jpd}d|d | S )NzImported from: zAuthor:  
z
---
)rW   urluserusernamenotesr9   )r.   previous_linkprevious_authorheaderprevious_notesr/   r/   r0   r      s
   zWandbRun.notesc                 C   rm   r+   )rW   tagsr-   r/   r/   r0   r      ro   zWandbRun.tagsc                 c   H    | j d u rg }| j D ]}t|}|| q|| _ | j E d H  d S r+   )rd   rW   logged_artifacts
_clone_artappend)r.   rd   artrC   r/   r/   r0   r%         
zWandbRun.artifactsc                 c   r   r+   )re   rW   used_artifactsr   r   )r.   re   r   rC   r/   r/   r0   r      r   zWandbRun.used_artifactsc                 C      d S r+   r/   r-   r/   r/   r0   
os_version       zWandbRun.os_versionc                 C      |   dS )Npython_metadata_filegetr-   r/   r/   r0   python_version   r6   zWandbRun.python_versionc                 C   r   r+   r/   r-   r/   r/   r0   cuda_version   r   zWandbRun.cuda_versionc                 C   r   r+   r/   r-   r/   r/   r0   program   r   zWandbRun.programc                 C   r   )Nhostr   r-   r/   r/   r0   r      r6   zWandbRun.hostc                 C   r   r+   r/   r-   r/   r/   r0   r      r   zWandbRun.usernamec                 C   r   r+   r/   r-   r/   r/   r0   
executable   r   zWandbRun.executablec                 C   r   r+   r/   r-   r/   r/   r0   	gpus_used   r   zWandbRun.gpus_usedc                 C   r   r+   r/   r-   r/   r/   r0   	cpus_used      zWandbRun.cpus_usedc                 C   r   r+   r/   r-   r/   r/   r0   memory_used   r   zWandbRun.memory_usedc                 C   sB   | j jdi d}| j jd}t|| }d u r|S t|S )N_wandbruntime_runtime)rW   rq   r   r   r?   )r.   wandb_runtimebase_runtimetr/   r/   r0   r      s
   zWandbRun.runtimec                 C   s   t | jj d }t|S )Ni  )dtfromisoformatrW   
created_at	timestampr?   )r.   r   r/   r/   r0   
start_time   s   zWandbRun.start_timec                 C   s   |   dd}d| S )NcodePathr   zcode/r   )r.   ri   r/   r/   r0   	code_path   s   
zWandbRun.code_pathc                 C   s   |   di di dS )Nr   valuecli_version)_config_filer   r-   r/   r/   r0   r      s   zWandbRun.cli_versionc                 c   s    | j d u r=tj d|   d}g }| j D ]!}|jdkr qd|jv r&q|j|d| j	d}|jdf}|
| q|| _ | j E d H  d S )Nr8   z/filesr   zwandb_manifest.json.deadlistT)exist_okra   end)rc   r   ROOT_DIRrj   rW   filessizer)   downloadra   r   )r.   	files_dirrc   fresultfile_and_policyr/   r/   r0   r      s   



zWandbRun.filesc              	   c   sP    |  d}|D ]}t|}| E d H  W d    n1 s w   Y  qd S )Nz^.*output\.log$)_find_all_in_files_regexopen	readlines)r.   	log_filesri   r   r/   r/   r0   logs  s   

zWandbRun.logsc                 C   sP   |  d }d u ri S t|}t| W  d    S 1 s!w   Y  d S )Nwandb-metadata.json)_find_in_filesr   jsonloadsreadr.   fnamer   r/   r/   r0   r     
   
$zWandbRun._metadata_filec                 C   sP   |  d }d u ri S t|}t|pi W  d    S 1 s!w   Y  d S )Nzconfig.yaml)r   r   yaml	safe_loadr   r/   r/   r0   r     r   zWandbRun._config_filec                 c   sd    |    }si V  d S dd |D }dt| }v r'|tdtj}|jddE d H  d S )Nc                 S   s*   g | ]}t |d D ]}t|qqS )z	*.parquet)r   globplread_parquet).0ri   pr/   r/   r0   
<listcomp>'  s
    
zAWandbRun._get_rows_from_parquet_history_paths.<locals>.<listcomp>_stepTnamed)rs   
_merge_dfswith_columnsr   colcastInt64	iter_rows)r.   pathsdfsdfr/   r/   r0   rt      s   z-WandbRun._get_rows_from_parquet_history_pathsc                 c   sn    | j d u r/g }| j D ]}|jdkrqt|t d|j d }d u r&q|| q|| _ | j E d H  d S )Nwandb-historyr8   root)rf   rW   r   rI   _download_artSRC_ART_PATHr)   r   )r.   r   r   ri   r/   r/   r0   rs   .  s   

z#WandbRun._get_parquet_history_pathsr)   c                 C   s.   |    }r|D ]\}}||v r|  S qd S r+   )r   )r.   r)   r   ri   _r/   r/   r0   r   >  s   zWandbRun._find_in_filesregexc                 c   s6    |    }r|D ]\}}t||r|V  q	d S d S r+   )r   rematch)r.   r   r   ri   r   r/   r/   r0   r   E  s   z!WandbRun._find_all_in_files_regex)5rM   rN   rO   r   rS   rg   r5   rj   r&   r'   r   r   rp   floatrq   r   rz   r   r|   r}   r~   r   r
   r   rQ   r%   r   r   r   r   r   r   r   r   r   r?   r   r   r   r   r   r   r   r   r   r   r   r   r   rt   rs   r   r   r/   r/   r/   r0   rV   e   sd    



rV   c                   @   s  e Zd ZdZdddededededeeeef  d	dfd
dZdd Z	dddde
dee deej d	dfddZ	dxdedee fddZdddedee d	dfddZded	dfddZ	dydedee dee d	efdd Z	!dzd"ed#ed$ed	ee fd%d&Zd"ed#ed	efd'd(Zd"ed#ed	efd)d*Zd	ee fd+d,Zdddd-d.eee  d/ee d0eeeef  d	dfd1d2Zddd3e dee d	dfd4d5Z!ddd6edee fd7d8Z"ddd9d9ddd9d9d9d9d9d9d9d:d.eee  d0eeeef  d;ed<ed=ee# d>ee# d?ed@edAedBedCedDedEefdFdGZ$ddddHd.eee  d>ee# d0eeeef  fdIdJZ%dd9dddKd.eee  d<ed=ee# d0eeeef  fdLdMZ&d9d9d9dd9ddNdOedPedQed.eee  d<ed0eeeef  fdRdSZ'ddTd"ed0eeeef  d	dfdUdVZ(ddTdOee d0eeeef  d	ee fdWdXZ)	!	9d{dedYedZed[ed\ef
d]d^Z*d9dd_dOee
 d`ed0eeeef  fdadbZ+dcdd Z,deee fdfdgZ-d9d!d9ddhdeee d<ed[ed\ed0eeeef  f
didjZ.ddddddkd.eee  d>ee# dleee  dmee d/ee d	ee
 fdndoZ/dddpdee d/ee fdqdrZ0ddddsd.eee  d>ee# d/ee fdtduZ1ddddsd.eee  d>ee# d/ee fdvdwZ2dS )|WandbImporterzFTransfers runs, reports, and artifact sequences between W&B instances.N)custom_api_kwargsrX   rY   rZ   r[   r   r*   c                C   st   || _ || _|| _|| _|d u rddi}tjd|d|id|| _tjd|d|id|| _||||d| _d S )NtimeoutiX  r\   r]   )rX   rY   rZ   r[   r/   )	rX   rY   rZ   r[   rP   r`   src_apirb   run_api_kwargs)r.   rX   rY   rZ   r[   r   r/   r/   r0   rg   O  s.   	zWandbImporter.__init__c                 C   s   d| j  d| j dS )Nz<WandbImporter src=z, dst=>)rX   rZ   r-   r/   r/   r0   r5   r  s   zWandbImporter.__repr__	namespacerp   rW   r   rp   c             	   C   s   |du rt | | }|du rtjdddddddd}| j| jddd}td| tj	||j
||d |jrztd| g }|j D ]}|jd	krRqJtd
|j t|}|| qJtd| tj	|||j
||d dS dS )zzImport one WandbRun.

        Use `namespace` to specify alternate settings like where the run should be uploaded
        NTmetadatar   mediacodehistoryrq   terminal_outputtruer^   r\   resumeresumedzImporting run, run=r_   settings_overriderp   z"Collecting history artifacts, run=r   z%Collecting history artifact art.name=z!Importing history artifacts, run=
extra_artsr_   r   rp   )r   r&   r'   r   SendManagerConfigr[   rZ   ru   debugsend_runsend_manager_overridesr   rW   r   rI   r)   r   r   )r.   rW   r   rp   r   history_artsr   new_artr/   r/   r0   _import_runu  sT   

zWandbImporter._import_runseqc           	      C   s   t |j|j}t |j|j}| d| d|j }|j}td|d|d|d| z	| j||}W n t	j
tfyN   td|d| Y dS w z|  W dS  t	j
tfyz } ztd|d|d	| W Y d}~dS d}~ww )
zDeletes the equivalent artifact collection in destination.

        Intended to clear the destination when an uploaded artifact does not pass validation.
        r8   zDeleting collection entity=
, project=z, art_type=z, art_name=z"Collection doesn't exist art_type=Nz&Collection can't be deleted, art_type=, e=)r   r&   r'   r(   r)   ru   inforb   artifact_collectionrP   	CommError
ValueErrorrv   delete)	r.   r  r   r&   r'   art_typeart_namedst_collectioner/   r/   r0   _delete_collection_in_dst  s,   	z'WandbImporter._delete_collection_in_dstr   c             
   C   sp  |j std|d dS |du rt|j|j}| j| jddd}tj	dd}| 
|| |j d }t|| j}tt|}t|d	D ]d\}}	|	d }|jtkr[t|fi | j}
n2z| }W n	 tyj   Y nw |du r}td
|jd| |}t|}|g}	t|fi | j}
td|d| dt|  tj|
|	|j||d qDtd| | | dS )zImport one artifact sequence.

        Use `namespace` to specify alternate settings like where the artifact sequence should be uploaded
        zArtifact seq=z has no artifacts, skipping.Nr   Tr   )log_artifactsr   r   zRun for art.name=z/ does not exist (deleted?), using run_or_dummy=zUploading partial artifact seq=, r8   r   zFinished uploading seq=)r%   ru   rv   r   r&   r'   r[   rZ   r   r   r  _get_run_or_dummy_from_artr   rr   _make_groups_of_artifacts	enumeratedescriptionART_SEQUENCE_DUMMY_PLACEHOLDERrV   r   	logged_byr
  r)   r   r  lenr   r   _remove_placeholders)r.   r  r   r   send_manager_configr   run_or_dummygroups_of_artifactsir{   rW   	wandb_runr  r/   r/   r0   _import_artifact_sequence  sX   


z'WandbImporter._import_artifact_sequencec                 C   s   zt | jj}t||j|j}W n" tjy&   t	
d|d Y d S  ty4   t	d Y d S w |D ]6}|jtkr?q7|jdv rEq7z|jdd W q7 tjym } zdt|v rbt	
d n W Y d }~q7d }~ww d S )	Nzseq=z5 does not exist in dst.  Has it already been deleted?z/Problem getting dst versions (try again later).)r   jobT)delete_aliasesz%cannot delete system managed artifactz%Cannot delete system managed artifact)r   	exp_retry_dst_apir%   rr   r(   r)   rP   r	  ru   rv   	TypeError	exceptionr  r  rI   r  rS   )r.   r  retry_arts_funcdst_artsr   r  r/   r/   r0   r  "  s6   



z"WandbImporter._remove_placeholderssrc_artr&   r'   c                 C   s:   t ||j}t ||j}|j}| j| d| d| S r7   )r   r&   r'   r)   rb   	_artifact)r.   r*  r&   r'   r)   r/   r/   r0   _get_dst_art=  s   zWandbImporter._get_dst_artFsrc_rundst_runforce_retryc                 C   sZ   g }|r	| d | || }r| dt|  | || }r+| dt|  |S )N__force_retry__z	metadata:zsummary:)r   _compare_run_metadatarS   _compare_run_summary)r.   r-  r.  r/  problemsnon_matching_metadatanon_matching_summaryr/   r/   r0   _get_run_problemsF  s   
zWandbImporter._get_run_problemsc              
   C   s$  d}| |}|jdkri S | |}ztj|j| jj}W n> tj	j
y.   ddi Y S  tjy] } z#|jjdkrIdd| iW  Y d }~S d| d	| d
iW  Y d }~S d }~ww tjjj|}i }	|jr|j D ]\}
}|
|vr|dd|	|
< qp||
 }||kr||d|	|
< qp|	S )Nr   r   zError checkingTimeouti  z
Bad uploadzFile not found: zhttp problemz: (r2   zKEY NOT FOUNDsrcdst)filer   rP   utildownload_file_into_memoryr   rb   r^   urllib3
exceptionsReadTimeoutErrorrequests	HTTPErrorresponsestatus_code	wandb_sdklib	json_utilr   r   items)r.   r-  r.  r   src_fdst_fcontentsr  dst_metanon_matchingksrc_vdst_vr/   r/   r0   r1  X  s:   



 z#WandbImporter._compare_run_metadatac           
      C   s   i }|j  D ]c\}}t|tr|drq|dv rqt|}|j |}t|}t|tr^t|tr^| D ]%\}}t|trF|drFq7||}	t||	s\||	d|| d| < q7qt||sj||d||< q|S )Nzwandb-client-artifact://)r   r   r8  -)	rq   rH  
isinstancerS   
startswith_recursive_cast_to_dictr   dict_almost_equal)
r.   r-  r.  rM  rN  rO  rP  kksvdvr/   r/   r0   r2  {  s0   



z"WandbImporter._compare_run_summaryc           
      c   s    t t }d u rtdtd d S |g d  }|jddD ]<}|d }|d }|d }|d	 }| d
| d
| }| j||}	t|	dd d}	t|	dd d}	t	|	||||V  q"d S )NzARTIFACT_ERRORS_FNAME= is empty, returning nothing)
src_entitysrc_projectr)   rI   Tr   r[  r\  r)   rI   r8   c                 S   r;   r<   r>   rB   r/   r/   r0   rD     rE   zBWandbImporter._collect_failed_artifact_sequences.<locals>.<lambda>rF   c                 S   s   | j S r+   )rI   rB   r/   r/   r0   rD     s    )
_read_ndjsonARTIFACT_ERRORS_FNAMEru   r   uniquer   r   r%   rH   r$   )
r.   r   unique_failed_sequencesry   r&   r'   r)   _typer  rK   r/   r/   r0   "_collect_failed_artifact_sequences  s(   z0WandbImporter._collect_failed_artifact_sequences)
namespacesra   	remappingrc  ra   rd  c                C   s   t || j}t ||  }|D ]Y}|r||v r|| }td| zt|j|jdtid}W n# t	yT } zdt
|v rJtd W Y d }~qW Y d }~nd }~ww |D ]}td| |jdd qWqd S )	NzCleaning up, ns=displayNamefilterszCould not find projectz&Could not find project, does it exist?zDeleting dummy run=F)delete_artifacts)r   rb   _all_namespacesru   r   rr   runsri   RUN_DUMMY_PLACEHOLDERr
  rS   r'  r  )r.   rc  ra   rd  nsrj  r  rW   r/   r/   r0   _cleanup_dummy_runs  s,   
z!WandbImporter._cleanup_dummy_runsreportc          
      C   s  |du rt |j|j}t|j|j}t|j|j}|j}|j}|j}| j}t	d|d| z|
|| W n( tjjy` }	 z|	jjdkrVtd|d|d|	 W Y d}	~	nd}	~	ww t	d|d|d	|d
| |jjtjjd|||||dt|jdd dS )zImport one wandb.Report.

        Use `namespace` to specify alternate settings like where the report should be uploaded
        NzUpserting entity=z	/project=i  zIssue upserting entity=r  zUpserting report entity=r  z, name=z, title=rj  )rn   r)   
entityNameprojectNamer  re  rI   specvariable_values)r   r&   r'   r   r)   titler  rb   ru   r   create_projectrA  r?  rB  rC  rD  rv   clientexecutewrrn  UPSERT_VIEWr   dumpsrq  )
r.   rn  r   r&   r'   r)   rt  r  ra   r  r/   r/   r0   _import_report  s:   "

zWandbImporter._import_reportsequencec          	      C   s   |d u rt |j|j}| j| jddd}td|d| tjdd}|D ]#}|	  }d u r2q'|D ]}t
|fi | j}tj||j||d q4q'd S )Nr   Tr   z/Using artifact sequence with settings_override=, namespace=)use_artifactsr   )r   r&   r'   r[   rZ   ru   r   r   r   used_byrV   r   r   r   )	r.   r|  r   r   r  r   r  r   rW   r/   r/   r0   _use_artifact_sequence  s,   z$WandbImporter._use_artifact_sequenceT)rc  rd  parallelincrementalmax_workerslimitr   r   r   r   r   rq   r   r  r  r  r  r   r   r   r   r   rq   r   c             	      s   t d t d t  tt t d tj||d}t dt| j||d t d t	 }t dt|  f	d	d
}t
||||d t d d S )NzSTART: Import runszSetting up for importzCollecting runsrc  r  zValidating runs, len(runs)=skip_previously_validatedrd  zCollecting failed runszImporting runs, len(runs)=c              	      s   t |  |  }d ur|v r| }tj d}td| d|d| j| ||d td| d|d| d S )Nr   zImporting run=r}  z	, config=r   zFinished importing run=)r   r&   r'   r   r   ru   r   r  )rW   r   rp   	r   r   r   r   r   rd  r.   rq   r   r/   r0   _import_run_wrapped;  s   
 z6WandbImporter.import_runs.<locals>._import_run_wrapped)r  r  zEND: Importing runs)ru   r  _create_files_if_not_exists_clear_fnameRUN_ERRORS_FNAMErr   _collect_runsr  _validate_runs_collect_failed_runsr   )r.   rc  rd  r  r  r  r  r   r   r   r   r   rq   r   rj  r  r/   r  r0   import_runs  s$   



zWandbImporter.import_runs)rc  r  rd  c                   sR   t d t d j||d}t d  fdd}t|| t d d S )NzSTART: Importing reportszCollecting reportsr  zImporting reportsc                    d   t | j| j} d ur| v r | }td| d| j| |d td| d| d S )NzImporting report=r}  r  zFinished importing report=)r   r&   r'   ru   r   r{  )rn  r   rd  r.   r/   r0   _import_report_wrapped_     z<WandbImporter.import_reports.<locals>._import_report_wrappedzEND: Importing reports)ru   r  _collect_reportsr   )r.   rc  r  rd  reportsr  r/   r  r0   import_reportsQ  s   



	zWandbImporter.import_reports)rc  r  r  rd  c                   s   t d tt t d tj|d}t d j|| d t d t }t dt|  fdd	}t	|||d
 t 
dt|  fdd}t	|||d
 t d j| d t d dS )zImport all artifact sequences from `namespaces`.

        Note: There is a known bug with the AWS backend where artifacts > 2048MB will fail to upload.  This seems to be related to multipart uploads, but we don't have a fix yet.
        z#START: Importing artifact sequenceszCollecting artifact sequences)rc  zValidating artifact sequences)r  rd  z$Collecting failed artifact sequencesz(Importing artifact sequences, len(seqs)=c                    r  )Nz Importing artifact sequence seq=r}  r  z)Finished importing artifact sequence seq=)r   r&   r'   ru   r   r!  r  r   r  r/   r0   !_import_artifact_sequence_wrapped  r  zRWandbImporter.import_artifact_sequences.<locals>._import_artifact_sequence_wrapped)r  z$Using artifact sequences, len(seqs)=c                    r  )NzUsing artifact sequence seq=r}  r  z%Finished using artifact sequence seq=)r   r&   r'   ru   r   r  r  r  r/   r0   _use_artifact_sequence_wrapped  r  zOWandbImporter.import_artifact_sequences.<locals>._use_artifact_sequence_wrappedzCleaning up dummy runsrc  rd  z!END: Importing artifact sequencesN)ru   r  r  r^  rr   _collect_artifact_sequences_validate_artifact_sequencesrb  r  r   r   rm  )r.   rc  r  r  rd  seqsr  r  r/   r  r0   import_artifact_sequencesl  s0   



		
z'WandbImporter.import_artifact_sequences)rj  r%   r  rc  r  rd  rj  r%   r  c                C   sd   t d|d|d| |r| j|||d |r!| j||d |r+| j|||d t d d S )NzSTART: Importing all, runs=z, artifacts=z
, reports=)rc  r  rd  r  zEND: Importing all)ru   r  r  r  r  )r.   rj  r%   r  rc  r  rd  r/   r/   r0   
import_all  s&   
zWandbImporter.import_allrd  c             	   C   s4  t |j|j}|d ur||v r|| }|j}|j}|j}z| j| d| d| }W n tjy?   d|d|g}Y nw | ||}|j|j|||d}	|rY||	d< t	}
nt
}
td0 t|
d}|t|	d  W d    n1 s{w   Y  W d    d S W d    d S 1 sw   Y  d S )	Nr8   z(run does not exist in dst at dst_entity=z/dst_project=r[  r\  
dst_entitydst_projectrj   r3  z	runs.lockrC   r   )r   r&   r'   rn   rb   rW   rP   r	  r6  r  RUN_SUCCESSES_FNAMEfilelockFileLockr   writer   rz  )r.   r-  rd  r   r  r  rj   r.  r3  dr   r   r/   r/   r0   _validate_run  s:    "zWandbImporter._validate_runc          	   
   c   s   t t }d u rtdtd |E d H  d S g }|D ]&}t|j|j}|d ur2||v r2|| }||j|j|j|j|j|d qt	
|}tdt|d |j|dg dd}td	t| | s}||d
   }|g d}|jddD ]}|d V  qd S )NzRUN_SUCCESSES_FNAME=z is empty, yielding all runs)r[  r\  r  r  rj   datazStarting with len(runs)=z in namespacesantir  howonz:After filtering out already successful runs, len(results)=rj   Tr   r  )r]  r  ru   r   r   r&   r'   r   rn   r   	DataFramer  r9   is_emptyfilteris_nullr_  r   )	r.   rj  rd  r   r  rr   df2resultsr/   r/   r0   _filter_previously_checked_runs  sF   


z-WandbImporter._filter_previously_checked_runsr  r  download_files_and_comparecheck_entries_are_downloadablec                 C   s$  g }dg}|D ]}t ||jr||||f  S qz	| |||}	W n ty6   |d ||||f Y S w ztd W n& tyd }
 z|d|jd|j	d|jd|
 W Y d }
~
nd }
~
ww |t
||	7 }|rtt
|	 |rtd| zt|t d	|j d
}W n' tjy }
 z|d|jd|j	d|jd|
  W Y d }
~
nd }
~
ww td|	 zt|	t d	|	j d
}W n' tjy }
 z|d|	jd|	j	d|	jd|
  W Y d }
~
nd }
~
ww td|d| t
|| }r|| ||||fS )Nz^job-(.*?)\.py(:v\d+)?$zdestination artifact not foundzComparing artifact manifestsz6Problem getting problems! problem with src_art.entity=z, src_art.project=z, src_art.name=z e=zDownloading src_art=r8   r   z-Invalid download link for src src_art.entity=r  zDownloading dst_art=z-Invalid download link for dst dst_art.entity=z, dst_art.project=z, dst_art.name=z Comparing artifact dirs src_dir=z
, dst_dir=)r   searchr)   r,  	Exceptionr   ru   r   r&   r'   r   _compare_artifact_manifests_check_entries_are_downloadabler   r   rA  rB  DST_ART_PATH_compare_artifact_dirs)r.   r*  r  r  r  r  r3  ignore_patternspatterndst_artr  src_dirdst_dirproblemr/   r/   r0   _validate_artifact$  s^   
 
  
z WandbImporter._validate_artifactr  r  c                   s@   dd |D }|rt j| d} fdd}t|| d S )Nc                 S   s   g | ]}|j qS r/   rW   )r   r  r/   r/   r0   r   h  s    z0WandbImporter._validate_runs.<locals>.<listcomp>r  c                    s2   t d|  j|  d t d|  d S )NzValidating run=r  zFinished validating run=)ru   r   r  r  r  r/   r0   r  q  s   z3WandbImporter._validate_runs.<locals>._validate_run)rr   r  r   )r.   rj  r  rd  	base_runsr  r/   r  r0   r  a  s   zWandbImporter._validate_runsc                 c   s    t t }d u rtdtd d S |g d  }|jddD ]&}|d }|d }|d }| j| d	| d	| }t|fi | j	V  q"d S )
NzRUN_ERRORS_FNAME=rZ  r  Tr   r[  r\  rj   r8   )
r]  r  ru   r   r_  r   r   rW   rV   r   )r.   r   unique_failed_runsry   r[  r\  rj   rW   r/   r/   r0   r  x  s    z"WandbImporter._collect_failed_runsr  c              
   c   s$   t t }d u rtdtd |D ]}|jE d H  qd S |D ]n}|D ]i}zt|| j}W n tjyA   t	d| Y q%w |j
dkrUt|trUtd| q%|j}|j}|j
}t|\}	}
||d |k|d |k@ |d |	k@ |d	 |
k@ |d
 |k@ }t|dkr|V  q%q!d S )NzARTIFACT_SUCCESSES_FNAME=z* is empty, yielding all artifact sequencesz!Failed to get run, skipping: art=r   zSkipping history artifact art=r[  r\  r)   r@   rI   r   )r]  ARTIFACT_SUCCESSES_FNAMEru   r  r%   r  r   rA  rB  r'  rI   rR  	_DummyRunr   r&   r'   _get_art_name_verr  r  )r.   r  r   r  r   r  r&   r'   ra  r)   verfiltered_dfr/   r/   r0   $_filter_previously_checked_artifacts  sN   





	z2WandbImporter._filter_previously_checked_artifacts)r  r  r  rd  c             	      s   |rt d fdd}| }nt d dd D } fdd}d	d
 |D }	t||	}
|
D ]C\}}}}t|\}}|j|j|||||jd}|rZ||d< t}nt	}t
|d}|t|d  W d    n1 svw   Y  q8d S )NzValidating in incremental modec                  3   s    D ]=} | j s	q| j d }zt| j}W n tjy1   td|jd|jd|j	 Y qw |j
dkr=t|tr=q| V  qd S )Nr   z)Validate Artifact http error: art.entity=z, art.project=z, art.name=r   )r%   r  r   rA  rB  ru   r'  r&   r'   r)   rI   rR  r  )r  r   r  )r.   r  r/   r0   filtered_sequences  s*   

zFWandbImporter._validate_artifact_sequences.<locals>.filtered_sequencesz"Validating in non-incremental modec                 S   s   g | ]
}|j D ]}|qqS r/   )r%   )r   r  r   r/   r/   r0   r     s    z>WandbImporter._validate_artifact_sequences.<locals>.<listcomp>c                    s   | \}}}d urt || }v r| }|j}|j}td|d|d| j||| d}td|d|d| |S )NzValidating art=z	, entity=r  )r  r  zFinished validating art=)r   r&   r'   ru   r   r  )argsr   r&   r'   r   remapped_nsr   )r  r  rd  r.   r/   r0   _validate_artifact_wrapped  s    
zNWandbImporter._validate_artifact_sequences.<locals>._validate_artifact_wrappedc                 s   s    | ]
}||j |jfV  qd S r+   )r&   r'   )r   r   r/   r/   r0   	<genexpr>  s    z=WandbImporter._validate_artifact_sequences.<locals>.<genexpr>)r[  r\  r  r  r)   r@   rI   r3  rC   r   )ru   r  r  r   r  r&   r'   rI   r^  r  r   r  r   rz  )r.   r  r  r  r  rd  r  r%   r  r  art_problemsr   r  r  r3  r)   r  r  r   r   r/   )r  r  rd  r.   r  r0   r    s8   	



z*WandbImporter._validate_artifact_sequences)rc  r  skip_ids
start_datera   r  r  c                #   sv    t  j t  i |d urd|id< |d ur$d|id<  fdd}t| |}|E d H  d S )Nz$ninr)   z$gte	createdAtc                  3   sJ    D ]} t d|   j| jdD ]}t|fi jV  qqd S )NzCollecting runs from ns=rf  )ru   r   rj  ri   rV   r   )rl  rW   ra   rg  rc  r.   r/   r0   _runs  s   z*WandbImporter._collect_runs.<locals>._runs)r   r   ri  	itertoolsislice)r.   rc  r  r  r  ra   r  rj  r/   r  r0   r    s   	zWandbImporter._collect_runs)r&   ra   c                c   sB    t || j}t ||j}||}|D ]
}t|j|jV  qd S r+   )r   r   default_entityprojectsr   r&   r)   )r.   r&   ra   r  r   r/   r/   r0   ri    s   
zWandbImporter._all_namespaces)rc  r  ra   c                #   sT    t  | j t |  tj| j| jd  fdd}t| |E d H  d S )N)rG   r   c                  3   s6    D ]}   | jD ]}tjj|j dV  qqd S )N)ra   )r  ri   rx  r   from_urlr   )rl  r  ra   rc  r/   r0   r  4  s   z/WandbImporter._collect_reports.<locals>.reports)	r   r   ri  rP   loginrY   rX   r  r  )r.   rc  r  ra   r  r/   r  r0   r  (  s   zWandbImporter._collect_reportsc                #   sX    t  | j t |   fdd}t| |}dd |D }| E d H  d S )Nc               
   3   s    D ]Y} t d|  g }zdd  | jD }W n ty*   t d Y nw |D ].}g }|jdkr7q-z| }W n tyK   t d Y nw |D ]}| rZt	
|V  qNq-qd S )Nz&Collecting artifact sequences from ns=c                 S   s   g | ]}|qS r/   r/   )r   r   r/   r/   r0   r   J  rE   zYWandbImporter._collect_artifact_sequences.<locals>.artifact_sequences.<locals>.<listcomp>zFailed to get artifact types.r   z#Failed to get artifact collections.)ru   r   artifact_typesri   r  r'  r)   collectionsis_sequencer$   rL   )rl  typesr   r  cr  r/   r0   artifact_sequencesE  s2   
zEWandbImporter._collect_artifact_sequences.<locals>.artifact_sequencesc                 S   s   i | ]}|j |qS r/   r3   )r   r  r/   r/   r0   
<dictcomp>_      z=WandbImporter._collect_artifact_sequences.<locals>.<dictcomp>)r   r   ri  r  r  values)r.   rc  r  ra   r  r  unique_sequencesr/   r  r0   r  ;  s   z)WandbImporter._collect_artifact_sequencesr+   )NN)F)FT)3rM   rN   rO   __doc__rS   r   r   r   rg   r5   rV   r   r   r   r  r$   r  r!  r  r   rQ   r,  boolr
   rU  r6  r1  r2  r   rb  r`   rm  r   r{  r  r?   r  r  r  r  r  r  r  r  r  r  r  r  ri  r  r  r/   r/   r/   r0   r   L  s"   	
#
B
#
O


#


-
$
	

?



F

%
*
3
A
+
U






r   r   r*   c                 C   s   | j d\}}|t|fS )N:v)r)   splitr?   )r   r)   r  r/   r/   r0   r  c  s   r  r)   ra  r  c                 C   sp   t | t}||_t|_tt}|jddd |t| }t	|d W d    n1 s,w   Y  |
| |S )NT)parentsr   w)rQ   ART_DUMMY_PLACEHOLDER_TYPEra  r  _descriptionr   ART_DUMMY_PLACEHOLDER_PATHmkdirrS   r   add_file)r)   ra  r  r   r   r   r/   r/   r0   _make_dummy_arth  s   

r  r  startc                 #   s\    |d }| D ]$ t  \}|| dkr% fddt|d |D V   gV  |}qd S )Nr   c                    s   g | ]	}t  j|qS r/   )r  rI   )r   r=   r   r)   r/   r0   r     s    z-_make_groups_of_artifacts.<locals>.<listcomp>)r  range)r  r  prev_verr  r/   r  r0   r  y  s    r  c                 C   sV   t | trdd | D S t | tst| dr)i }|  D ]
\}}t|||< q|S | S )Nc                 S   s   g | ]}t |qS r/   )rT  )r   itemr/   r/   r0   r     r  z+_recursive_cast_to_dict.<locals>.<listcomp>rH  )rR  rr   rU  hasattrrH  rT  )objnew_dictrG   r   r/   r/   r0   rT    s   
rT  ư>c                    s   t tr!t tr!  krdS t fddD S t tjr5t tjr5t  k S ttur?dS kS )NFc                 3   s$    | ]}t | |  V  qd S r+   )rV  )r   rN  epsxyr/   r0   r    s   " z _almost_equal.<locals>.<genexpr>)rR  rU  keysallnumbersNumberabsrI   )r  r  r  r/   r  r0   rV    s   rV  c                   @   s   e Zd ZU dZeed< dS )
_DummyUserr   r   N)rM   rN   rO   r   rS   rR   r/   r/   r/   r0   r
    s   
 r
  c                   @   s   e Zd ZU dZeed< dZeed< eZeed< eZ	eed< eZ
eed< dZeed< dZeed< dZeed	< d
Zeed< eedZeed< eedZeed< eedZeed< eedZeed< dd ZdS )r  r   r&   r'   rj   rn   r~   r   r   r{   z
2000-01-01r   )default_factoryr   r   rq   rp   c                 C   s   g S r+   r/   r-   r/   r/   r0   r     r   z_DummyRun.filesN)rM   rN   rO   r&   rS   rR   r'   rk  rj   rn   r~   r   r   r{   r   r   r
  r   rr   r   rU  rq   rp   r   r/   r/   r/   r0   r    s   
 r  r   c              
   C   sr   zt | }W |S  ty   Y d S  ty8 } zdt|v r&W Y d }~d S dt|v r3W Y d }~d S  d }~ww )Nz&empty string is not a valid JSON valuezerror parsing ndjson)r   read_ndjsonFileNotFoundErrorRuntimeErrorrS   )r   r   r  r/   r/   r0   r]    s   r]  c                 C   s   d }z|   }W n  ty( } ztd| d|d| W Y d }~nd }~ww |d ur/|S td}|jj|d| jid}|di di }t	| j
| j|d	t|d	td
}|S )Nz2Can't log artifact because run doesn't exist, art=z, run=r  a  
        query ArtifactCreatedBy(
            $id: ID!
        ) {
            artifact(id: $id) {
                createdBy {
                    ... on Run {
                        name
                        project {
                            name
                            entityName
                        }
                    }
                }
            }
        }
    rn   rr  artifact	createdByr)   )r&   r'   rj   rn   )r  r
  ru   rv   r   rv  rw  rn   r   r  r&   r'   rk  )r   ra   rW   r  queryrC  creatorr/   r/   r0   r    s.   

r  c                 C   s   t j d|  }t j d|  }td|d| zt|| W n	 ty,   Y nw t| d	 W d    d S 1 s>w   Y  d S )Nr8   z/prev_zMoving old_fname=z to new_fname=r  )r   r   ru   r   shutilcopy2r  r   )r   	old_fname	new_fnamer/   r/   r0   r    s   "r  r   c                 C   sd   zt d | j|ddW  d    W S 1 sw   Y  W d S  ty1   td|  Y d S w )N
click.echoT)r   
skip_cachezError downloading artifact art=)r   r   r  ru   r'  )r   r   r/   r/   r0   r     s   
(r   c                 C   s   |d u rt  d| j }t| |d }d u rtd| | jd\}}t|t}| j|_| j	|_
| j|_| j|_td || W d    |S 1 sOw   Y  |S )Nr8   r   zProblem downloading art=r  r  )r   r)   r   r
  r  rQ   r  rI   ra  r   _created_ataliases_aliasesr  r  r   add_dir)r   r   ri   r)   r   r  r/   r/   r0   r     s    


r   c               	   C   sV   t tttg} | D ] }td|d t|d W d    n1 s#w   Y  qd S )NzCreating fname=z if not existsrC   )r^  r  r  r  ru   r   r   )fnamesr   r/   r/   r0   r  &  s   r  r   c                    s   t | dkr
t S t | dkr| d S | d  | dd  D ]-} j|ddgd  fdd jD }|D ]\}} |  | } || q5q S )Nr   r   outerr   r  c                    s*   g | ]}| d  j v r|| d fqS )_right)columns)r   r  	merged_dfr/   r0   r   ?  s
    z_merge_dfs.<locals>.<listcomp>)r  r   r  r9   r   	fill_nullr   drop)r   r   	col_pairsr   rightnew_colr/   r!  r0   r   4  s   
r   )r   )r   r+   )r*   N)jr  r  r   loggingr  rh   r   r  dataclassesr   r   r   r   pathlibr   typingr   r   r   r	   r
   r   r   unittest.mockr   r  polarsr   rA  r>  wandb_workspaces.reports.v1r  v1rx  r   	wandb_gqlr   r   rP   wandb.apis.publicr   r   wandb.apis.public.filesr   
wandb.utilr   r   r   r   	internalsr   internals.protocolsr   r   internals.utilr   r   rQ   r`   apispublicProjectr^  r  r  r  r  rk  r  r  r   r  	getLoggerrM   ru   setLevelINFOgetenvrich.loggingr!   
addHandlerStreamHandlerconsole_handler	Formatter	formattersetFormatterr$   rV   r   rS   r?   r  r  r  rT  rV  r
  r  r  r]  r  r  r   r   r  r   r/   r/   r/   r0   <module>   s    $





 h        
+
