o
    nin&                     @   s   d Z ddlZddlZddlZddlZddlmZmZmZ ddl	Z	er0ddlm
Z
mZ ddlmZ eeZdZdZee ZdZd	Zd
ZdZdZdZdZzedd dd ZW n eya   eZY nw G dd dZdS )a  leveldb log datastore.

Format is described at:
    https://github.com/google/leveldb/blob/master/doc/log_format.md

block := record* trailer?
record :=
  checksum: uint32     // crc32c of type and data[] ; little-endian
  length: uint16       // little-endian
  type: uint8          // One of FULL, FIRST, MIDDLE, LAST
  data: uint8[length]

header :=
  ident: char[4]
  magic: uint16
  version: uint8
    N)TYPE_CHECKINGOptionalTuple)IOAny)Record   i               z:W&Bi   asciic                 C   s
   t | dS )zStrtobytes.z	iso8859-1)bytes)x r   Z/home/ubuntu/SoloSpeech/.venv/lib/python3.10/site-packages/wandb/sdk/internal/datastore.py
strtobytes6   s   
r   c                   @   s   e Zd ZU eed< eed< d*ddZdeddfdd	Zd
d Zdd Z	deddfddZ
defddZdd Zdd Zdd Zdd Zdd Zd+ddZdd  Zd!eddfd"d#Zd$d%deeeef fd&d'Zd*d(d)ZdS ),	DataStore_index_flush_offsetreturnNc                 C   sp   d| _ d | _d| _d| _d| _dgtd  | _tdtd D ]}t	t
t|d@ | j|< qtjs6J dd S )NFr   r	       z2DataStore can only be used in the internal process)_opened_for_scan_fpr   r   _size_bytesLEVELDBLOG_LAST_crcrangezlibcrc32r   chrwandb_assert_is_internal_process)selfr   r   r   r   __init__F   s   zDataStore.__init__fnamec                 C   s.   || _ td| d}t||| _|   d S )Nopen: %sxb)_fnameloggerinfoopenr   _write_header)r$   r&   
open_flagsr   r   r   open_for_writeU   s
   zDataStore.open_for_writec                 C   s"   || _ td| t|d| _d S )Nr'   wb)r)   r*   r+   r,   r   r$   r&   r   r   r   open_for_append\   s   zDataStore.open_for_appendc                 C   sD   || _ td| t|d| _d| _t|j| _	d| _
|   d S )Nzopen for scan: %szr+br   T)r)   r*   r+   r,   r   r   osstatst_sizer   r   _read_headerr1   r   r   r   open_for_scanc   s   zDataStore.open_for_scanoffsetc                 C   s   | j | || _d S N)r   seekr   r$   r8   r   r   r   r:   l   s   
zDataStore.seekc                 C   s   | j  }|S r9   )r   tellr;   r   r   r   
get_offsetp   s   
zDataStore.get_offsetc                 C   s   | j | jt kS )zBDetermine if we're in the last block to handle in-progress writes.)r   r   LEVELDBLOG_DATA_LENr$   r   r   r   in_last_blockt   s   zDataStore.in_last_blockc                 C   s   | j sJ d| jt}t|dkrd S t|tks%J dt|ttd|}|\}}}|  jt7  _| j|}t	
|| j| d@ }||ksPJ d|  j|7  _||fS )Nzfile not open for scanningr   z4record header is {} bytes instead of the expected {}<IHBr   z/record checksum is invalid, data may be corrupt)r   r   readLEVELDBLOG_HEADER_LENlenformatstructunpackr   r   r    r   )r$   headerfieldschecksumdlengthdtypedatachecksum_computedr   r   r   scan_recordx   s&   

zDataStore.scan_recordc           	      C   s   | j t }t| }|tk r(td| }| j|}||ks!J d|  j |7  _ |  }|d u r2d S |\}}|tkr<|S |tksJJ dt d| 	 | j t }|  }|d u rZd S |\}}|t	kri||7 }	 |S |t
kswJ dt
 d| ||7 }qK)N zinvalid paddingzexpected record to be type z but found )r   LEVELDBLOG_BLOCK_LENrC   r   r   rB   rO   LEVELDBLOG_FULLLEVELDBLOG_FIRSTr   LEVELDBLOG_MIDDLE)	r$   r8   
space_left	pad_checkpadrecordrL   rM   new_datar   r   r   	scan_data   s>   



zDataStore.scan_datac                 C   sZ   t dtttt}t|tksJ dt| dt | j	| |  j
t|7  _
d S )N<4sHBzheader size is z bytes, expected )rF   packr   LEVELDBLOG_HEADER_IDENTLEVELDBLOG_HEADER_MAGICLEVELDBLOG_HEADER_VERSIONrD   rC   r   writer   )r$   rM   r   r   r   r-      s   zDataStore._write_headerc                 C   s   | j t}t|tksJ dt|ttd|\}}}|ttkr)t	d|t
kr1t	d|tkr9t	d|  jt|7  _d S )Nz-header is {} bytes instead of the expected {}r[   zInvalid header)r   rB   rC   rD   rE   rF   rG   r   r]   	Exceptionr^   r_   r   )r$   rH   identmagicversionr   r   r   r6      s   zDataStore._read_headerc                 C   s   t |t t| jt  ksJ dt |}|pt}t|| j| d@ }| j	t
d||| |r8| j	| |  jtt | 7  _dS )z(Write record that must fit into a block.z%not enough space to write new recordsr   rA   N)rD   rC   rQ   r   rR   r   r    r   r   r`   rF   r\   )r$   srL   rK   rJ   r   r   r   _write_record   s   
zDataStore._write_recordc           	      C   s&  | j }| j t }t| }d}t|}|tk r-d| }| jt| |  j |7  _ d}t}|t |kr9| | nS|t }| |d | t ||7 }||8 }|sUJ d|t	krq| |||t	  t
 |t	7 }|t	8 }|t	ksY| ||d  t | j  t| j  | j | _|| j | jfS )Nr   rP   zdata_left should be non-zero)r   rQ   rD   rC   r   r`   r   rf   rS   r>   rT   r   flushr3   fsyncfilenor   )	r$   re   start_offsetr8   rU   	data_used	data_leftrW   	data_roomr   r   r   _write_data   s>   
	
zDataStore._write_dataoffc                 C   s   | j   d S r9   )r   rg   )r$   ro   r   r   r   ensure_flushed  s   zDataStore.ensure_flushedobjr   c                 C   s2   |  }| }t||ksJ d| |}|S )zWrite a protocol buffer.

        Args:
            obj: Protocol buffer to write.

        Returns:
            (start_offset, end_offset, flush_offset) if successful,
            None otherwise

        zinvalid serialization)ByteSizeSerializeToStringrD   rn   )r$   rq   raw_sizere   retr   r   r   r`     s
   
zDataStore.writec                 C   s*   | j d urtd| j | j   d S d S )Nz	close: %s)r   r*   r+   r)   closer?   r   r   r   rv   &  s   
zDataStore.close)r   Nr9   )__name__
__module____qualname__int__annotations__r%   strr/   r2   r7   r:   r=   r@   rO   rZ   r-   r6   rf   rn   rp   r   r`   rv   r   r   r   r   r   B   s&   
 
	%
-r   ) __doc__loggingr3   rF   r   typingr   r   r   r"   r   r   wandb.proto.wandb_internal_pb2r   	getLoggerrw   r*   rC   rQ   r>   rR   rS   rT   r   r]   r^   r_   r   r   ra   r|   r   r   r   r   r   <module>   s:    

