o
    }oiI6                     @  s   d dl mZ d dlZd dlmZ d dlZd dlmZ d dl	m
Z
 er9d dlmZ d dlZd dlmZ d dlmZ nd dlmZ ed	Zed
ZedZe
eZd=ddZd>dd Zd?d#d$Zd@d%d&ZdAd'd(Zd)d*d+dBd/d0Zdd1d2d)dd3dCd;d<ZdS )D    )annotationsN)TYPE_CHECKING)*single_blas_thread_if_scipy_v1_15_or_newer)
get_logger)batched_lbfgsb)BaseAcquisitionFunc)_LazyImportzscipy.optimizetorchzoptuna._gp.batched_lbfgsbacqfr   initial_params_batched
np.ndarrayinitial_fvalscontinuous_indiceslengthscalestolfloatreturn)tuple[np.ndarray, np.ndarray, np.ndarray]c              	     s  |j dksJ tdkr||tjt|tdfS d fd
d}t . tj||ddf  dd | D fdd D t	|dd\}}}	W d   n1 sVw   Y  | }
| |
ddf< | }||k|	dk@ }t
|dddf |
|t
||||fS )a'  
    This function optimizes the acquisition function using preconditioning.
    Preconditioning equalizes the variances caused by each parameter and
    speeds up the convergence.

    In Optuna, acquisition functions use Matern 5/2 kernel, which is a function of `x / l`
    where `x` is `normalized_params` and `l` is the corresponding lengthscales.
    Then acquisition functions are a function of `x / l`, i.e. `f(x / l)`.
    As `l` has different values for each param, it makes the function ill-conditioned.
    By transforming `x / l` to `zl / l = z`, the function becomes `f(z)` and has
    equal variances w.r.t. `z`.
    So optimization w.r.t. `z` instead of `x` is the preconditioning here and
    speeds up the convergence.
    As the domain of `x` is [0, 1], that of `z` becomes [0, 1/l].
       r   dtypescaled_xr   fixed_paramslist[np.ndarray]r   tuple[np.ndarray, np.ndarray]c                   s   t |}| jdkr|jdksJ |  |d d f< t|d} | }|   |j	
  }t |
  }||d d f  fS )Nr   T)nparrayndimr	   
from_numpyrequires_grad_	eval_acqfsumbackwardgraddetachnumpy
atleast_1d)r   r   next_paramsx_tensor	neg_fvalsgrads
neg_fvals_r
   r   r    J/home/ubuntu/.local/lib/python3.10/site-packages/optuna/_gp/optim_mixed.pynegative_acqf_with_grad8   s   
z9_gradient_ascent_batched.<locals>.negative_acqf_with_gradNc                 S  s   g | ]}|qS r-   r-   ).0paramr-   r-   r.   
<listcomp>M   s    z,_gradient_ascent_batched.<locals>.<listcomp>c                 S  s   g | ]}d d| fqS )r      r-   )r0   sr-   r-   r.   r2   N   s       )func_and_grad
x0_batchedbatched_argsboundspgtol	max_iters)r   r   r   r   r   r   )r   lenr   zerosboolr   r   copymathsqrtwhere)r
   r   r   r   r   r   r/   scaled_cont_xs_optneg_fvals_optn_iterationsxs_opt	fvals_optis_updated_batchr-   r,   r.   _gradient_ascent_batched   s,   
rI   initial_paramsinitial_fval	param_idxintchoicestuple[np.ndarray, float, bool]c           	      C  s   t |dkr||dfS |||| k }tj|d d d f t |dd}||d d |f< | |}t|}|| |krH||d d f || dfS ||dfS )Nr3   Fr   )axisT)r<   r   repeateval_acqf_no_gradargmax)	r
   rJ   rK   rL   rN   choices_except_current
all_paramsfvalsbest_idxr-   r-   r.   _exhaustive_searcha   s   
 


rX   gridsxtolc                   s   t dkr||dfS dfdd}|| }t| | s$J || i| d fd
ddfdd}d}	tj|d |	 | d |	 fd|d}
||
j}| }||kru||kru| < |dfS ||dfS )Nr3   Fxr   r   rM   c                   sR   t tt | dt d }t|  |d   t|  |  k r'|d S |S )Nr3   )rM   r   clipsearchsortedr<   abs)r[   i)rY   r-   r.   find_nearest_index   s   "0z1_discrete_line_search.<locals>.find_nearest_indexr_   c                   s>    | }|d ur|S |  < t  }|| < |S )N)getr   rR   )r_   	cache_valnegval)r
   rY   negative_fval_cachenormalized_paramsrL   r-   r.   negative_acqf_with_cache   s   
z7_discrete_line_search.<locals>.negative_acqf_with_cachec                   s   |  d k s|  d krt jS tt t  | dt d }|d }||}} | |   |  |   }d| }|| ||  S )Nr   r3   g      ?)r   infrM   r\   r]   r<   )r[   rightleftneg_acqf_leftneg_acqf_rightw_leftw_right)rY   rf   r-   r.   interpolated_negative_acqf   s   "z9_discrete_line_search.<locals>.interpolated_negative_acqfg-q=r   rg   brent)bracketmethodr   T)r[   r   r   rM   )r_   rM   r   r   )r[   r   r   r   )r<   r   iscloser?   sominimize_scalarr[   )r
   rJ   rK   rL   rY   rZ   r`   current_choice_iro   EPSresopt_idxfval_optr-   )r
   rY   rf   rd   re   rL   r.   _discrete_line_searchy   s,   





r{   c                 C  sB   d}| j j| }|st||krt| ||||S t| |||||S )N   )search_spaceis_categoricalr<   rX   r{   )r
   rJ   rK   rL   rN   rZ    MAX_INT_EXHAUSTIVE_SEARCH_PARAMSr~   r-   r-   r.   _local_search_discrete   s
   
r   c                 C  sr   |  }|  }tjt|td}t|D ]\}	}
t| |
||	 |||\}}}|||	< |||	< |||	< q|||fS )Nr   )r?   r   r=   r<   r>   	enumerater   )r
   r   r   rL   rN   rZ   best_normalized_params_batched
best_fvalsrH   batchre   best_normalized_params	best_fvalupdatedr-   r-   r.   _local_search_discrete_batched   s   	


r   g-C6?d   )r   max_iterxs0r   r   c             	   C  sv  | j | jj } }| jj}| j }dd |D }| |  }	}
d}tjt	|	|t
d}tt	|	}t|D ]x}t| |	| |
| |||\|	|< |
|< }t|||}t|||D ];\}}}|||k }  }||  }|jdkr||	|
f    S t| |	| |
| |||\|	|< |
|< }t|||}q\|||k }  }||  }|jdkr|	|
f  S q9td |	|
fS )Nc                 S  s&   g | ]}t jt |t jd d qS ))initial   )r   mindiffrh   )r0   rN   r-   r-   r.   r2      s    z.local_search_mixed_batched.<locals>.<listcomp>rg   r   r   z2local_search_mixed: Local search did not converge.)length_scalesr}   r   discrete_indicesget_choices_of_discrete_paramsrR   r?   r   fullr<   rM   arangerangerI   rB   zipsizer   _loggerwarning)r
   r   r   r   	cont_indsr   r   choices_of_discrete_paramsdiscrete_xtolsbest_xsr   
CONTINUOUSlast_changed_dimsremaining_inds_r   r_   rN   rZ   is_convergedr-   r-   r.   local_search_mixed_batched   s@   





r   i   
   )!warmstart_normalized_params_arrayn_preliminary_samplesn_local_searchr   rngr   np.ndarray | Noner   r   r   np.random.RandomState | Nonetuple[np.ndarray, float]c                C  sT  |pt j }|d u rt d| jjf}t||d ks J d| jj||d}| |}t	|t j
s5J t |}t |||  }	d|	|< |	|	  }	tt |	dk}
t|t| d |
}||
krjtd t |g}|dkr|jt||d|	d}t ||}t ||d d f |g}t| ||d	\}}t | }|| || fS )
Nr   r3   zPWe must choose at least 1 best sampled point + given_initial_xs as start points.)r   g        zBStudy already converged, so the number of local search is reduced.F)r   replacep)r   )r   randomRandomStateemptyr}   dimr<   sample_normalized_paramsrR   
isinstancendarrayrS   expr!   rM   count_nonzeror   r   r   r   choiceappendvstackr   item)r
   r   r   r   r   r   
sampled_xsf_valsmax_iprobsn_non_zero_probs_improvementn_additional_warmstartchosen_idxsadditional_idxsx_warmstartsr   r   rW   r-   r-   r.   optimize_acqf_mixed  s:   	


r   )r
   r   r   r   r   r   r   r   r   r   r   r   r   r   )r
   r   rJ   r   rK   r   rL   rM   rN   r   r   rO   )r
   r   rJ   r   rK   r   rL   rM   rY   r   rZ   r   r   rO   )r
   r   rJ   r   rK   r   rL   rM   rN   r   rZ   r   r   rO   )r
   r   r   r   r   r   rL   rM   rN   r   rZ   r   r   r   )
r
   r   r   r   r   r   r   rM   r   r   )r
   r   r   r   r   rM   r   rM   r   r   r   r   r   r   )
__future__r   r@   typingr   r%   r   "optuna._gp.scipy_blas_thread_patchr   optuna.loggingr   scipy.optimizeoptimizert   r	   
optuna._gpr   optuna._gp.acqfr   optunar   __name__r   rI   rX   r{   r   r   r   r   r-   r-   r-   r.   <module>   s:    

D

D
3