o
    xi$                     @  s   d Z ddlmZ ddlZddlZddlZddlZddlmZ er.ddlm	Z	m
Z
 ddlmZ eeZdZdZee Zd	Zd
ZdZdZdZdZdZzedd dd ZW n ey_   eZY nw G dd dZdS )a  leveldb log datastore.

Format is described at:
    https://github.com/google/leveldb/blob/master/doc/log_format.md

block := record* trailer?
record :=
  checksum: uint32     // crc32c of type and data[] ; little-endian
  length: uint16       // little-endian
  type: uint8          // One of FULL, FIRST, MIDDLE, LAST
  data: uint8[length]

header :=
  ident: char[4]
  magic: uint16
  version: uint8
    )annotationsN)TYPE_CHECKING)IOAny)Record   i               z:W&Bi   asciic                 C  s
   t | dS )zStrtobytes.z	iso8859-1)bytes)x r   P/home/ubuntu/.local/lib/python3.10/site-packages/wandb/sdk/internal/datastore.py
strtobytes4   s   
r   c                   @  s   e Zd ZU ded< ded< d.ddZd/d
dZdd Zdd Zd0ddZd1ddZ	dd Z
dd Zdd Zdd Zdd Zd2d d!Zd"d# Zd3d%d&Zd4d*d+Zd.d,d-ZdS )5	DataStoreint_index_flush_offsetreturnNonec                 C  sb   d| _ d | _d| _d| _d| _dgtd  | _tdtd D ]}t	t
t|d@ | j|< qd S )NFr   r       )_opened_for_scan_fpr   r   _size_bytesLEVELDBLOG_LAST_crcrangezlibcrc32r   chr)selfr   r   r   r   __init__@   s   zDataStore.__init__fnamestrc                 C  s.   || _ td| d}t||| _|   d S )Nopen: %sxb)_fnameloggerinfoopenr   _write_header)r#   r%   
open_flagsr   r   r   open_for_writeK   s
   zDataStore.open_for_writec                 C  s"   || _ td| t|d| _d S )Nr'   wb)r)   r*   r+   r,   r   r#   r%   r   r   r   open_for_appendR   s   zDataStore.open_for_appendc                 C  sD   || _ td| t|d| _d| _t|j| _	d| _
|   d S )Nzopen for scan: %szr+br   T)r)   r*   r+   r,   r   r   osstatst_sizer   r   _read_headerr1   r   r   r   open_for_scanY   s   zDataStore.open_for_scanoffsetc                 C  s   | j | || _d S N)r   seekr   r#   r8   r   r   r   r:   b   s   
zDataStore.seekc                 C  s   | j  }|S r9   )r   tellr;   r   r   r   
get_offsetf   s   
zDataStore.get_offsetc                 C  s   | j | jt kS )zBDetermine if we're in the last block to handle in-progress writes.)r   r   LEVELDBLOG_DATA_LENr#   r   r   r   in_last_blockj   s   zDataStore.in_last_blockc                 C  s   | j sJ d| jt}t|dkrd S t|tks'J dt| dt td|}|\}}}|  jt7  _| j|}t	|| j
| d@ }||ksRJ d|  j|7  _||fS )Nzfile not open for scanningr   zrecord header is  bytes instead of the expected <IHBr   z/record checksum is invalid, data may be corrupt)r   r   readLEVELDBLOG_HEADER_LENlenstructunpackr   r    r!   r   )r#   headerfieldschecksumdlengthdtypedatachecksum_computedr   r   r   scan_recordn   s"   

zDataStore.scan_recordc           	      C  s   | j t }t| }|tk r(td| }| j|}||ks!J d|  j |7  _ |  }|d u r2d S |\}}|tkr<|S |tksJJ dt d| 	 | j t }|  }|d u rZd S |\}}|t	kri||7 }	 |S |t
kswJ dt
 d| ||7 }qK)N zinvalid paddingzexpected record to be type z but found )r   LEVELDBLOG_BLOCK_LENrD   r   r   rC   rO   LEVELDBLOG_FULLLEVELDBLOG_FIRSTr   LEVELDBLOG_MIDDLE)	r#   r8   
space_left	pad_checkpadrecordrL   rM   new_datar   r   r   	scan_data   s>   



zDataStore.scan_datac                 C  sZ   t dtttt}t|tksJ dt| dt | j	| |  j
t|7  _
d S )N<4sHBzheader size is z bytes, expected )rF   packr   LEVELDBLOG_HEADER_IDENTLEVELDBLOG_HEADER_MAGICLEVELDBLOG_HEADER_VERSIONrE   rD   r   writer   )r#   rM   r   r   r   r-      s   zDataStore._write_headerc                 C  s   | j t}t|tksJ dt| dt td|\}}}|ttkr+td|t	kr3td|t
kr;td|  jt|7  _d S )Nz
header is rA   r[   zInvalid header)r   rC   rD   rE   rF   rG   r   r]   	Exceptionr^   r_   r   )r#   rH   identmagicversionr   r   r   r6      s   zDataStore._read_headerNc                 C  s   t |t t| jt  ksJ dt |}|pt}t|| j| d@ }| j	t
d||| |r8| j	| |  jtt | 7  _dS )z(Write record that must fit into a block.z%not enough space to write new recordsr   rB   N)rE   rD   rQ   r   rR   r    r!   r   r   r`   rF   r\   )r#   srL   rK   rJ   r   r   r   _write_record   s   
zDataStore._write_recordc           	      C  s&  | j }| j t }t| }d}t|}|tk r-d| }| jt| |  j |7  _ d}t}|t |kr9| | nS|t }| |d | t ||7 }||8 }|sUJ d|t	krq| |||t	  t
 |t	7 }|t	8 }|t	ksY| ||d  t | j  t| j  | j | _|| j | jfS )Nr   rP   zdata_left should be non-zero)r   rQ   rE   rD   r   r`   r   rf   rS   r>   rT   r   flushr3   fsyncfilenor   )	r#   re   start_offsetr8   rU   	data_used	data_leftrW   	data_roomr   r   r   _write_data   s>   
	
zDataStore._write_dataoffc                 C  s   | j   d S r9   )r   rg   )r#   ro   r   r   r   ensure_flushed  s   zDataStore.ensure_flushedobjr   tuple[int, int, int]c                 C  s2   |  }| }t||ksJ d| |}|S )zWrite a protocol buffer.

        Args:
            obj: Protocol buffer to write.

        Returns:
            (start_offset, end_offset, flush_offset) if successful,
            None otherwise

        zinvalid serialization)ByteSizeSerializeToStringrE   rn   )r#   rq   raw_sizere   retr   r   r   r`     s
   
zDataStore.writec                 C  s*   | j d urtd| j | j   d S d S )Nz	close: %s)r   r*   r+   r)   closer?   r   r   r   rw     s   
zDataStore.close)r   r   )r%   r&   r   r   )r8   r   r   r   )r   r   r9   )ro   r   r   r   )rq   r   r   rr   )__name__
__module____qualname____annotations__r$   r/   r2   r7   r:   r=   r@   rO   rZ   r-   r6   rf   rn   rp   r`   rw   r   r   r   r   r   <   s&   
 


	
%

-
r   )__doc__
__future__r   loggingr3   rF   r    typingr   r   r   wandb.proto.wandb_internal_pb2r   	getLoggerrx   r*   rD   rQ   r>   rR   rS   rT   r   r]   r^   r_   r   r   ra   r&   r   r   r   r   r   <module>   s:    

