o
    i                     @   s>  d Z ddlZddlZddlZddlZddlZddlZzddlZdZW n e	y-   dZY nw z
ddl
mZ dZW n e	yC   dZY nw ddlmZ edZed	Zi Zd
d Zdd Zdd ZG dd dZG dd dZede  G dd dZG dd dZede  G dd dZdd Zdd Zd d! ZdS )"z
To write tf_record into file. Here we use it for tensorboard's event writting.
The code was borrowed from https://github.com/TeamHG-Memex/tensorboard_logger
    NTF)storage   )crc32cz^[A-Za-z0-9.]z[A-Za-z0-9_.\-/]+c                 C   s   d| v rt d|t| < d S )N:zprefix cannot contain a :)
ValueErrorREGISTERED_FACTORIES)prefixfactory r
   N/home/ubuntu/.local/lib/python3.10/site-packages/tensorboardX/record_writer.pyregister_writer_factory%   s   r   c                 C   sT   z|  dd }t| }|| W S  ty)   tj| s&t|  Y dS Y dS w )z'Initialize the directory for log files.r   r   N)splitr   directory_checkKeyErrorospathexistsmakedirsr   r   r	   r
   r
   r   r   +   s   r   c                 C   s@   z|  dd }t| }|| W S  ty   t| d Y S w )z)Open a writer for outputting event files.r   r   wb)r   r   openr   r   r
   r
   r   	open_file6   s   r   c                   @   @   e Zd ZdZdd Zdd Zdd Zdd	 Zd
d Zdd Z	dS )S3RecordWriterz/Writes tensorboard protocol buffer files to S3.c                 C   s&   t std|| _t | _d| _d S )Nz'boto3 must be installed for S3 support.F)
S3_ENABLEDImportErrorr   ioBytesIObufferclosedselfr   r
   r
   r   __init__C   s
   

zS3RecordWriter.__init__c                 C   s   | j s	|   d S d S N)r   closer!   r
   r
   r   __del__J   s   zS3RecordWriter.__del__c                 C   N   | j }|dr|tdd  }|d}|d }|dt| d  }||fS )Nzs3:///r   r   r   
startswithlenr   r!   r   bpbucketr
   r
   r   bucket_and_pathN      

zS3RecordWriter.bucket_and_pathc                 C      | j | d S r#   r   writer!   valr
   r
   r   r3   W      zS3RecordWriter.writec                 C   sJ   t jdtjdd}|  \}}t| j}|d |	||| d S )Ns3S3_ENDPOINT)endpoint_urlr   )
boto3clientr   environgetr/   copyr   seekupload_fileobj)r!   r7   r.   r   upload_bufferr
   r
   r   flushZ   s
   
zS3RecordWriter.flushc                 C   s   |    d| _d S )NT)rB   r   r%   r
   r
   r   r$   a   s   
zS3RecordWriter.closeN
__name__
__module____qualname____doc__r"   r&   r/   r3   rB   r$   r
   r
   r
   r   r   @   s    	r   c                   @       e Zd ZdZdd Zdd ZdS )S3RecordWriterFactoryz.Factory for event protocol buffer files to S3.c                 C      t |S r#   )r   r    r
   r
   r   r   i      zS3RecordWriterFactory.openc                 C      d S r#   r
   r    r
   r
   r   r   l      z%S3RecordWriterFactory.directory_checkNrD   rE   rF   rG   r   r   r
   r
   r
   r   rI   f       rI   r7   c                   @   r   )GCSRecordWriterzAWrites tensorboard protocol buffer files to Google Cloud Storage.c                 C   sN   t std|| _t | _t }|  \}}t	||}t
||| _d S )NzM`google-cloud-storage` must be installed in order to use the 'gs://' protocol)GCS_ENABLEDr   r   r   r   r   r   Clientr/   BucketBlobblob)r!   r   r;   bucket_namefilepathr.   r
   r
   r   r"   x   s   
zGCSRecordWriter.__init__c                 C      |    d S r#   )r$   r%   r
   r
   r   r&         zGCSRecordWriter.__del__c                 C   r'   )Nzgs://r(   r   r   r)   r,   r
   r
   r   r/      r0   zGCSRecordWriter.bucket_and_pathc                 C   r1   r#   r2   r4   r
   r
   r   r3      r6   zGCSRecordWriter.writec                 C   s6   t  | j}|d | jj}| jj| |d d S )Nr   )dataif_generation_match)r>   r   r?   rU   
generationupload_from_stringgetvalue)r!   rA   blob_generationr
   r
   r   rB      s   

zGCSRecordWriter.flushc                 C   rX   r#   )rB   r%   r
   r
   r   r$      rY   zGCSRecordWriter.closeNrC   r
   r
   r
   r   rP   u   s    		rP   c                   @   rH   )GCSRecordWriterFactoryz@Factory for event protocol buffer files to Google Cloud Storage.c                 C   rJ   r#   )rP   r    r
   r
   r   r      rK   zGCSRecordWriterFactory.openc                 C   rL   r#   r
   r    r
   r
   r   r      rM   z&GCSRecordWriterFactory.directory_checkNrN   r
   r
   r
   r   r`      rO   r`   gsc                   @   s,   e Zd Zdd Zdd Zdd Zdd Zd	S )
RecordWriterc                 C   s(   i | _ t | _|| _d | _t|| _d S r#   )_name_to_tf_nameset	_tf_namesr   _writerr   r    r
   r
   r   r"      s
   zRecordWriter.__init__c                 C   sT   | j j}tdt|}|| |tdt| || |tdt| d S )NQI)rf   r3   structpackr+   masked_crc32c)r!   rZ   wheaderr
   r
   r   r3      s   zRecordWriter.writec                 C      | j   d S r#   )rf   rB   r%   r
   r
   r   rB         zRecordWriter.flushc                 C   rn   r#   )rf   r$   r%   r
   r
   r   r$      ro   zRecordWriter.closeN)rD   rE   rF   r"   r3   rB   r$   r
   r
   r
   r   rb      s
    rb   c                 C   s(   t t| }t |d? t |d> B d S )N      l   jE )u32r   )rZ   xr
   r
   r   rk      s   rk   c                 C   s   | d@ S )Nl    r
   )rs   r
   r
   r   rr      rK   rr   c                 C   s"   t | s	d|  } dt| S )N._)_VALID_OP_NAME_STARTmatchjoin_VALID_OP_NAME_PARTfindall)namer
   r
   r   make_valid_tf_name   s   
r|   )rG   r>   r   r   os.pathreri   r:   r   r   google.cloudr   rQ   r   compilerv   ry   r   r   r   r   r   rI   rP   r`   rb   rk   rr   r|   r
   r
   r
   r   <module>   sH    


&,