o
    }oiwl                     @  st  d dl mZ d dlmZ d dlmZ d dlmZ d dlZd dlZd dlZd dl	Z	d dl
Z
d dlmZ d dlZd dlZd dlmZ d dlmZ d d	lmZ d d
lmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d dl m!Z! d dl"m#Z# d dl$m%Z% d dl$m&Z& ej'(e)Z*dZ+dZ,dZ-G dd dej.Z/G dd deZ0G dd dZ1dS )    )annotations)	Container)Iterable)SequenceN)Any)JSONSerializable)BaseDistribution) check_distribution_compatibilitydistribution_to_jsonjson_to_distribution)DuplicatedStudyError)UpdateFinishedTrialError)BaseStorage)DEFAULT_STUDY_NAME_PREFIX)BaseJournalBackend)BaseJournalSnapshot)FrozenStudyStudyDirection)FrozenTrial)
TrialStatezRecord does not exist.zATrial#{trial_number} has already finished and can not be updated.d   c                   @  s4   e Zd ZdZdZdZdZdZdZdZ	dZ
d	Zd
ZdS )JournalOperationr                           	   N)__name__
__module____qualname__CREATE_STUDYDELETE_STUDYSET_STUDY_USER_ATTRSET_STUDY_SYSTEM_ATTRCREATE_TRIALSET_TRIAL_PARAMSET_TRIAL_STATE_VALUESSET_TRIAL_INTERMEDIATE_VALUESET_TRIAL_USER_ATTRSET_TRIAL_SYSTEM_ATTR r1   r1   T/home/ubuntu/.local/lib/python3.10/site-packages/optuna/storages/journal/_storage.pyr   (   s    r   c                   @  s  e Zd ZdZdbddZdcd	d
ZddddZdeddZdfddZdgddZ		dhdidd Z
djd"d#Zdkd(d)Zdld+d,Zdmd-d.Zdnd/d0Zdod2d3Zdpd4d5Zdpd6d7Zdqd9d:Zdhdrd=d>ZdsdEdFZdtdHdIZ	dhdudNdOZdvdRdSZdwdTdUZdxdVdWZdydYdZZ	[	dzd{d`daZdS )|JournalStoragea  Storage class for Journal storage backend.

    Note that library users can instantiate this class, but the attributes
    provided by this class are not supposed to be directly accessed by them.

    Journal storage writes a record of every operation to the database as it is executed and
    at the same time, keeps a latest snapshot of the database in-memory. If the database crashes
    for any reason, the storage can re-establish the contents in memory by replaying the
    operations stored from the beginning.

    Journal storage has several benefits over the conventional value logging storages.

    1. The number of IOs can be reduced because of larger granularity of logs.
    2. Journal storage has simpler backend API than value logging storage.
    3. Journal storage keeps a snapshot in-memory so no need to add more cache.

    Example:

        .. code::

            import optuna


            def objective(trial): ...


            storage = optuna.storages.JournalStorage(
                optuna.storages.journal.JournalFileBackend("./optuna_journal_storage.log")
            )

            study = optuna.create_study(storage=storage)
            study.optimize(objective)

    In a Windows environment, an error message "A required privilege is not held by the
    client" may appear. In this case, you can solve the problem with creating storage
    by specifying :class:`~optuna.storages.journal.JournalFileOpenLock` as follows.

    .. code::

        file_path = "./optuna_journal_storage.log"
        lock_obj = optuna.storages.journal.JournalFileOpenLock(file_path)

        storage = optuna.storages.JournalStorage(
            optuna.storages.journal.JournalFileBackend(file_path, lock_obj=lock_obj),
        )
    log_storager   returnNonec                 C  s   t t d | _|| _t | _t| j| _	| j! t
| jtr/| j }|d ur/| | |   W d    d S 1 s>w   Y  d S N-)struuiduuid4_worker_id_prefix_backend	threadingLock_thread_lockJournalStorageReplayResult_replay_result
isinstancer   load_snapshotrestore_replay_result_sync_with_backend)selfr4   snapshotr1   r1   r2   __init__e   s   



"zJournalStorage.__init__dict[Any, Any]c                 C  s    | j  }|d= |d= |d= |S )Nr<   rB   r@   )__dict__copyrG   stater1   r1   r2   __getstate__r   s
   
zJournalStorage.__getstate__rN   c                 C  s8   | j | tt d | _t| j| _t	 | _
d S r7   )rK   updater9   r:   r;   r<   rA   rB   r>   r?   r@   rM   r1   r1   r2   __setstate__y   s   zJournalStorage.__setstate__rH   bytesc              	   C  sv   zt |}W n t jtfy   td Y d S w |d u r d S t|ts,td d S | j|_i |_	d|_
|| _d S )Nz/Failed to restore `JournalStorageReplayResult`.z8The restored object is not `JournalStorageReplayResult`.)pickleloadsUnpicklingErrorKeyError_loggerwarningrC   rA   r<   _worker_id_to_owned_trial_id&_last_created_trial_id_by_this_processrB   )rG   rH   rr1   r1   r2   rE      s   



z$JournalStorage.restore_replay_resultop_codeintextra_fieldsdict[str, Any]c                 C  s$   | j j}| j||d|g d S )N)r]   	worker_id)rB   ra   r=   append_logs)rG   r]   r_   ra   r1   r1   r2   
_write_log   s   zJournalStorage._write_logc                 C  s    | j | jj}| j| d S N)r=   	read_logsrB   log_number_read
apply_logs)rG   logsr1   r1   r2   rF      s   z!JournalStorage._sync_with_backendN
directionsSequence[StudyDirection]
study_name
str | Nonec                 C  s   |p	t tt  }| jP | tj||d |   | j	
 D ]7}|j|kr)q!td| |j}t| jtrN|dkrN|t dkrN| jt| j	 |  W  d    S J d1 s`w   Y  d S )N)rk   ri   z,A new study created in Journal with name: {}r   FShould not reach.)r   r9   r:   r;   r@   rc   r   r'   rF   rB   get_all_studiesrk   rX   infoformat	_study_idrC   r=   r   SNAPSHOT_INTERVALsave_snapshotrT   dumps)rG   ri   rk   frozen_studystudy_idr1   r1   r2   create_new_study   s(   

zJournalStorage.create_new_studyrv   c                 C  sF   | j  | tjd|i |   W d    d S 1 sw   Y  d S Nrv   )r@   rc   r   r(   rF   rG   rv   r1   r1   r2   delete_study   s   
"zJournalStorage.delete_studykeyr9   valuer   c                 C  P   |||id}| j  | tj| |   W d    d S 1 s!w   Y  d S )N)rv   	user_attr)r@   rc   r   r)   rF   rG   rv   r{   r|   logr1   r1   r2   set_study_user_attr   
   
"z"JournalStorage.set_study_user_attrr   c                 C  r}   )N)rv   system_attr)r@   rc   r   r*   rF   r   r1   r1   r2   set_study_system_attr   r   z$JournalStorage.set_study_system_attrc                 C  s^   | j " |   | j D ]}|j|kr |j  W  d    S qtt1 s(w   Y  d S rd   )r@   rF   rB   rn   rk   rq   rW   NOT_FOUND_MSG)rG   rk   studyr1   r1   r2   get_study_id_from_name   s   
z%JournalStorage.get_study_id_from_namec                 C  @   | j  |   | j|jW  d    S 1 sw   Y  d S rd   )r@   rF   rB   	get_studyrk   ry   r1   r1   r2   get_study_name_from_id      $z%JournalStorage.get_study_name_from_idlist[StudyDirection]c                 C  r   rd   )r@   rF   rB   r   ri   ry   r1   r1   r2   get_study_directions   r   z#JournalStorage.get_study_directionsc                 C  r   rd   )r@   rF   rB   r   
user_attrsry   r1   r1   r2   get_study_user_attrs   r   z#JournalStorage.get_study_user_attrsc                 C  r   rd   )r@   rF   rB   r   system_attrsry   r1   r1   r2   get_study_system_attrs   r   z%JournalStorage.get_study_system_attrslist[FrozenStudy]c                 C  sB   | j  |   t| j W  d    S 1 sw   Y  d S rd   )r@   rF   rL   deepcopyrB   rn   rG   r1   r1   r2   rn      s   $zJournalStorage.get_all_studiestemplate_trialFrozenTrial | Nonec                   s  |t j  jddd} rx j|d<  jd ur)t jdkr)d |d<  j|d< n	 j|d< d |d<  jr? jjdd|d< nd |d<  jrO jjdd|d	< d
d  j	
 D |d<  fdd j
 D |d<  j|d<  j|d<  j|d< | jJ | tj| |   | jj}t| jtr|dkr|t dkr| jt| j W d    |S W d    |S W d    |S W d    |S 1 sw   Y  |S )Nmicrosecondstimespec)rv   datetime_startrN   r   r|   valuesr   datetime_completec                 S     i | ]	\}}|t |qS r1   r
   ).0kdistr1   r1   r2   
<dictcomp>   s    z3JournalStorage.create_new_trial.<locals>.<dictcomp>distributionsc                   s"   i | ]\}}| j | |qS r1   )r   to_internal_repr)r   r   paramr   r1   r2   r     s    paramsr   r   intermediate_valuesr   )datetimenow	isoformatrN   r   lenr|   r   r   r   itemsr   r   r   r   r@   rc   r   r+   rF   rB   r[   rC   r=   r   rr   rs   rT   rt   )rG   rv   r   r   trial_idr1   r   r2   create_new_trial   sd   













zJournalStorage.create_new_trialr   
param_nameparam_value_internalfloatdistributionr   c                 C  sT   |||t |d}| j | tj| |   W d    d S 1 s#w   Y  d S )N)r   r   r   r   )r   r@   rc   r   r,   rF   )rG   r   r   r   r   r   r1   r1   r2   set_trial_param  s   
"zJournalStorage.set_trial_paramtrial_numberc                 C  sf   | j & |   t| jj| |krtd||| jj| | W  d    S 1 s,w   Y  d S )Nz?No trial with trial number {} exists in study with study_id {}.)r@   rF   r   rB   _study_id_to_trial_idsrW   rp   )rG   rv   r   r1   r1   r2   'get_trial_id_from_study_id_trial_number+  s   $z6JournalStorage.get_trial_id_from_study_id_trial_numberr   r   Sequence[float] | Noneboolc                 C  s  |||d}|t jkrtj jdd|d< n| r&tj jdd|d< | jS |t jkr_|   | jj	
|}|d usBJ d|j rPttj|jd|jt jkr_	 W d    dS | tj| |   |t jkpt|| jjkW  d    S 1 sw   Y  d S )	N)r   rN   r   r   r   r   r   z@Please report your bug on GitHub if this line fails your script.r   F)r   RUNNINGr   r   r   is_finishedr@   rF   rB   _trialsgetrN   r   UNUPDATABLE_MSGrp   numberWAITINGrc   r   r-   owned_trial_id)rG   r   rN   r   r   existing_trialr1   r1   r2   set_trial_state_values6  s4   

	

$z%JournalStorage.set_trial_state_valuesstepintermediate_valuec                 C  sN   |||d}| j  | tj| |   W d    d S 1 s w   Y  d S )N)r   r   r   )r@   rc   r   r.   rF   )rG   r   r   r   r   r1   r1   r2   set_trial_intermediate_valueb  s   
"z+JournalStorage.set_trial_intermediate_valuec                 C  r}   )N)r   r~   )r@   rc   r   r/   rF   rG   r   r{   r|   r   r1   r1   r2   set_trial_user_attro     
"z"JournalStorage.set_trial_user_attrc                 C  r}   )N)r   r   )r@   rc   r   r0   rF   r   r1   r1   r2   set_trial_system_attry  r   z$JournalStorage.set_trial_system_attrr   c                 C  s>   | j  |   | j|W  d    S 1 sw   Y  d S rd   )r@   rF   rB   	get_trialrG   r   r1   r1   r2   r     s   
$zJournalStorage.get_trialTr   statesContainer[TrialState] | Nonelist[FrozenTrial]c                 C  s`   | j # |   | j||}|rt|W  d    S |W  d    S 1 s)w   Y  d S rd   )r@   rF   rB   get_all_trialsrL   r   )rG   rv   r   r   frozen_trialsr1   r1   r2   r     s   $zJournalStorage.get_all_trials)r4   r   r5   r6   )r5   rJ   )rN   rJ   r5   r6   )rH   rR   r5   r6   )r]   r^   r_   r`   r5   r6   )r5   r6   rd   )ri   rj   rk   rl   r5   r^   )rv   r^   r5   r6   )rv   r^   r{   r9   r|   r   r5   r6   )rv   r^   r{   r9   r|   r   r5   r6   )rk   r9   r5   r^   )rv   r^   r5   r9   )rv   r^   r5   r   )rv   r^   r5   r`   r5   r   )rv   r^   r   r   r5   r^   )
r   r^   r   r9   r   r   r   r   r5   r6   )rv   r^   r   r^   r5   r^   )r   r^   rN   r   r   r   r5   r   )r   r^   r   r^   r   r   r5   r6   )r   r^   r{   r9   r|   r   r5   r6   )r   r^   r{   r9   r|   r   r5   r6   r   r^   r5   r   )TN)rv   r^   r   r   r   r   r5   r   )r$   r%   r&   __doc__rI   rO   rQ   rE   rc   rF   rw   rz   r   r   r   r   r   r   r   rn   r   r   r   r   r   r   r   r   r   r1   r1   r1   r2   r3   5   s>    
/














2

,




r3   c                   @  s   e Zd Zd?ddZd@d	d
ZdAddZdBddZdCddZdDddZe	dEddZ
e	dFdd ZdGd$d%ZdHd&d'ZdId(d)ZdId*d+ZdId,d-ZdId.d/ZdId0d1ZdId2d3ZdId4d5ZdId6d7ZdId8d9ZdId:d;ZdJd<d=Zd>S )KrA   worker_id_prefixr9   r5   r6   c                 C  s4   d| _ || _i | _i | _i | _i | _d| _i | _d S )Nr   )rf   r<   _studiesr   r   _trial_id_to_study_id_next_study_idrZ   )rG   r   r1   r1   r2   rI     s   
z#JournalStorageReplayResult.__init__rh   Iterable[dict[str, Any]]c                 C  s  |D ]~}|  j d7  _ |d }|tjkr| | q|tjkr%| | q|tjkr0| | q|tjkr;| 	| q|tj
krF| | q|tjkrQ| | q|tjkr\| | q|tjkrg| | q|tjkrr| | q|tjkr}| | qJ dd S )Nr   r]   Frm   )rf   r   r'   _apply_create_studyr(   _apply_delete_studyr)   _apply_set_study_user_attrr*   _apply_set_study_system_attrr+   _apply_create_trialr,   _apply_set_trial_paramr-   _apply_set_trial_state_valuesr.   #_apply_set_trial_intermediate_valuer/   _apply_set_trial_user_attrr0   _apply_set_trial_system_attr)rG   rh   r   opr1   r1   r2   rg     s2   









z%JournalStorageReplayResult.apply_logsrv   r^   r   c                 C     || j vr	tt| j | S rd   )r   rW   r   ry   r1   r1   r2   r        

z$JournalStorageReplayResult.get_studyr   c                 C  s   t | j S rd   )listr   r   r   r1   r1   r2   rn        z*JournalStorageReplayResult.get_all_studiesr   r   c                 C  r   rd   )r   rW   r   r   r1   r1   r2   r     r   z$JournalStorageReplayResult.get_trialr   r   r   c                 C  sP   || j vr	ttg }| j| D ]}| j| }|d u s |j|v r%|| q|S rd   )r   rW   r   r   r   rN   append)rG   rv   r   r   r   trialr1   r1   r2   r     s   


z)JournalStorageReplayResult.get_all_trialsc                 C  s   | j tt  S rd   )r<   r9   r>   	get_identr   r1   r1   r2   ra     s   z$JournalStorageReplayResult.worker_id
int | Nonec                 C  s   | j | jS rd   )rZ   r   ra   r   r1   r1   r2   r     s   z)JournalStorageReplayResult.owned_trial_idr   r`   r   c                 C  s   |d | j kS )Nra   )ra   )rG   r   r1   r1   r2   _is_issued_by_this_worker  r   z4JournalStorageReplayResult._is_issued_by_this_workerc                 C  s$   || j v rdS | |rttdS )NTF)r   r   rW   r   )rG   rv   r   r1   r1   r2   _study_exists  s
   

z(JournalStorageReplayResult._study_existsc                 C  s   |d }dd |d D }|dd | j  D v r'| |r%td|d S | j}|  jd7  _t|d i i ||d| j |< g | j|< d S )	Nrk   c                 S  s   g | ]}t |qS r1   r   )r   dr1   r1   r2   
<listcomp>  s    zBJournalStorageReplayResult._apply_create_study.<locals>.<listcomp>ri   c                 S  s   g | ]}|j qS r1   )rk   )r   sr1   r1   r2   r     s    zAnother study with name '{}' already exists. Please specify a different name, or reuse the existing one by setting `load_if_exists` (for Python API) or `--skip-if-exists` flag (for CLI).r   )rk   	directionr   r   rv   ri   )r   r   r   r   rp   r   r   r   )rG   r   rk   ri   rv   r1   r1   r2   r     s(   
z.JournalStorageReplayResult._apply_create_studyc                 C  s6   |d }|  ||r| j|}|j|ksJ d S d S rx   )r   r   poprq   )rG   r   rv   fsr1   r1   r2   r     s   z.JournalStorageReplayResult._apply_delete_studyc                 C  F   |d }|  ||r!t|d dksJ | j| j|d  d S d S )Nrv   r~   r   )r   r   r   r   rP   rG   r   rv   r1   r1   r2   r     
   z5JournalStorageReplayResult._apply_set_study_user_attrc                 C  r   )Nrv   r   r   )r   r   r   r   rP   r   r1   r1   r2   r     r   z7JournalStorageReplayResult._apply_set_study_system_attrc                   st  |d }|  ||sd S t| j}i  d|v r"dd |d  D  i }d|v r5 fdd|d  D }|d d urDtj|d }nd }d|v rStj|d }nd }t|t| j| t|	d	tj
j| |	d
i |	di |	dd dd |	di  D |||	dd d| j|< | j| | || j|< | |r|| _| j| jtj
kr|| j| j< d S d S d S )Nrv   r   c                 S  r   r1   r   r   r   vr1   r1   r2   r   "      zBJournalStorageReplayResult._apply_create_trial.<locals>.<dictcomp>r   c                   s    i | ]\}}| |  |qS r1   )to_external_repr)r   r   pr   r1   r2   r   %  s     r   r   rN   r   r   r|   c                 S  s   i | ]	\}}t ||qS r1   )r^   r   r1   r1   r2   r   8  r   r   r   )r   r   rN   r   r   r   r   r|   r   r   r   r   )r   r   r   r   r   fromisoformatr   r   r   r   r   r|   r   r   r   r[   rN   rZ   ra   )rG   r   rv   r   r   r   r   r1   r   r2   r     sL   






z.JournalStorageReplayResult._apply_create_trialc           
   	   C  s   |d }|  ||sd S |d }|d }t|d }| j| }| j| D ],}| j| }||j v rPz
t|j| | W n t	yM   | 
|rI Y  d S w  nq$t| j| }	i t|	j|||i|	_i t|	j||i|	_|	| j|< d S )Nr   r   r   r   )_trial_exists_and_updatabler   r   r   r   r   keysr	   r   	Exceptionr   rL   r   )
rG   r   r   r   r   r   rv   prev_trial_id
prev_trialr   r1   r1   r2   r   F  s:   





z1JournalStorageReplayResult._apply_set_trial_paramc                 C  s   |d }|  ||sd S t|d }|| j| jkr!|tjkr!d S t| j| }|tjkrBtj|d |_| 	|rB|| j
| j< | rOtj|d |_||_|d d ur]|d |_|| j|< d S )Nr   rN   r   r   r   )r   r   r   rN   r   rL   r   r   r   r   rZ   ra   r   r   r   )rG   r   r   rN   r   r1   r1   r2   r   g  s"   


z8JournalStorageReplayResult._apply_set_trial_state_valuesc                 C  sX   |d }|  ||r*t| j| }i t|j|d |d i|_|| j|< d S d S )Nr   r   r   )r   rL   r   r   rG   r   r   r   r1   r1   r2   r     s   
z>JournalStorageReplayResult._apply_set_trial_intermediate_valuec                 C  d   |d }|  ||r0t|d dksJ t| j| }i t|j|d |_|| j|< d S d S )Nr   r~   r   )r   r   rL   r   r   r  r1   r1   r2   r     s   z5JournalStorageReplayResult._apply_set_trial_user_attrc                 C  r  )Nr   r   r   )r   r   rL   r   r   r  r1   r1   r2   r     s   
z7JournalStorageReplayResult._apply_set_trial_system_attrc                 C  sZ   || j vr| |rttdS | j | j r+| |r)ttj| j | j	ddS dS )NFr   T)
r   r   rW   r   rN   r   r   r   rp   r   )rG   r   r   r1   r1   r2   r     s   


z6JournalStorageReplayResult._trial_exists_and_updatableN)r   r9   r5   r6   )rh   r   r5   r6   )rv   r^   r5   r   r   r   )rv   r^   r   r   r5   r   )r5   r9   )r5   r   )r   r`   r5   r   )rv   r^   r   r`   r5   r   )r   r`   r5   r6   )r   r^   r   r`   r5   r   )r$   r%   r&   rI   rg   r   rn   r   r   propertyra   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r1   r1   r1   r2   rA     s0    













-
!


	rA   )2
__future__r   collections.abcr   r   r   rL   r   enumrT   r>   typingr   r:   optunaoptuna._typingr   optuna.distributionsr   r	   r   r   optuna.exceptionsr   r   optuna.storagesr   optuna.storages._baser   optuna.storages.journal._baser   r   optuna.study._frozenr   optuna.study._study_directionr   optuna.trialr   r   logging
get_loggerr$   rX   r   r   rr   IntEnumr   r3   rA   r1   r1   r1   r2   <module>   sH      c