o ciIC@sddlmZmZddlZddlmZmZmZddl m Z ddl m Z ddl mZddlmZmZddlmZmZdd lmZdd lmZmZdd lmZdd lmZmZmZdd l m!Z!ddl"m#Z#m$Z$ddl%m&Z&errddl'm(Z(e\Z)Z*Z+e\Z,Z-eGdddeZ.dS))BoxDiscreteN)Optional TYPE_CHECKINGUnion)BaseEnv)ActionDistribution)ModelV2) Categorical Deterministic)TorchCategoricalTorchDeterministic) SampleBatch) OldAPIStackoverride) Exploration) get_variable try_import_tftry_import_torch) from_config)softmax SMALL_NUMBER) TensorType)PolicycseZdZdZdddddededed ed ed e ef fd d Z e e ddddde ede e de dfddZe e dedeeefdeee ffddZe e dddddddedede dfddZdd Ze e ddddd!d"Ze e  d>ddd#ede dfd$d%Zdd&d'd(Zd)d*Zdd+d,d-d.Zdd&d/d0Zd1d2Zdd&d3d4Zd5d6Ze e d>d7d8Z e e d>d9ed:e dd;dfdPs z+ParameterNoise.__init__..T)as_dict:r_noisy)dtype)rr# torch_tensordevicetfF EpsilonGreedyPiecewiseSchedule)rrrg{Gz?)type endpoints outside_value)r7epsilon_scheduleOrnsteinUhlenbeckNoise)r7r)r action_spacer r!)*super__init__rrr" stddev_valr!trainable_variablesitemsmodel_variablesnoisenamesplitappendnpzerosshapefloat32r1r2executing_eagerly_tf_sample_new_noise_optf_sample_new_noise_op_tf_add_stored_noise_optf_add_stored_noise_op_tf_remove_noise_optf_remove_noise_optf1control_dependenciesno_optf_sample_new_noise_and_add_opweights_are_currently_noisy isinstancer<rrNotImplementedErrorrrr repisode_started) selfr<rr r!rrrkwargsvarname_add_op __class__r%r)r>*s            zParameterNoise.__init__)timestepexploretf_sessrarbrcz tf.SessioncCsd|dur|n|jd}|jr||||r!|js!|j|ddS|s.|jr0|j|ddSdSdS)Nrbrc)r rY_delayed_on_episode_startrV_add_stored_noise _remove_noise)rZrarbrcr%r%r)before_compute_actionss   z%ParameterNoise.before_compute_actionsaction_distributioncCs|jj|||dS)N)rirarb)rget_exploration_action)rZrirarbr%r%r)rjs z%ParameterNoise.get_exploration_action) environmentepisodercpolicyrrkrlcCs d|_dS)NT)rYrZrmrkrlrcr%r%r)on_episode_starts zParameterNoise.on_episode_startcCs*|r |j|ddn|j|dd|_dS)NTrcrrdF)_sample_new_noise_and_add_sample_new_noiserY)rZrbrcr%r%r)res  z(ParameterNoise._delayed_on_episode_startcCs|jr |j|ddSdS)Nrd)rVrgrnr%r%r)on_episode_endszParameterNoise.on_episode_end sample_batchc Csd}}|j||jd\}}}t|jttfrt|tj}nt|jt t fr-|tj}nt |jr5|}n|}|j||j d\}}}t|jttfrSt|tj}n t|jt t fr`|tj}|durg|}n|}d} } t|jttfrt t |t ||td} |jj|dd} t d| | |jj } n't|jt t frt t t ||} |jj|dd} t|jdd| } | | kr|jd9_n|jd_|j||d|S) N) input_dictrbr5)sess cur_epsilon cur_scaleou_sigmag?g)\(?)compute_actions_from_input_dictrV issubclass dist_classr r rrACTION_DIST_INPUTSr r rXrGnanmeansumlogrr get_stater<nsqrtmeansquaregetattrr? set_state) rZrmrtrcnoisy_action_distnoise_free_action_dist_fetches action_distdeltadistancecurrent_epsilon current_scaler%r%r)postprocess_trajectorysb      z%ParameterNoise.postprocess_trajectoryrdcCsv|jdkr ||jdS|jdkr|dStt|jD]}tjt |j| |j d |j |j|<qdS)z0Samples new noise and stores it in `self.noise`.r2tf2)rstdN)rrunrMrLrangelenrCtorchnormalrHsizer"tor1)rZrcir%r%r)rr2s    z ParameterNoise._sample_new_noisec Cs@g}|jD]}|t|tjj|j|jtj dqtj |S)N)rIr"r/) rCrFrRassignr2randomrrIr"rJgroup)rZ added_noisesrCr%r%r)rL>s   z&ParameterNoise._tf_sample_new_noise_opFrpcCsZ|jdkr|r|jr||j||jn|r |jr |||d|_dS)Nr2T)rrVrrQrUrgrrrf)rZrcrr%r%r)rqKs     z(ParameterNoise._sample_new_noise_and_addcCsr|jdusJ|jdkr||jn!|jdkr|nt|j|jD]\}}d|_| |d|_q$d|_dS)aAdds the stored `self.noise` to the model's parameters. Note: No new sampling of noise here. Args: tf_sess (Optional[tf.Session]): The tf-session to use to add the stored noise to the (currently noise-free) weights. override: If True, undo any currently applied noise first, then add the currently stored noise. Fr2rTN) rVrrrOrNziprBrC requires_gradadd_rZrcr\rCr%r%r)rfXs     z ParameterNoise._add_stored_noisecCsvt}t|j|jD] \}}|t||q tjt |}t |g t WdS1s4wYdS)zGenerates tf-op that assigns the stored noise to weights. Also used by tf-eager. Returns: tf.op: The tf op to apply the already stored noise to the NN. N listrrBrCrFrR assign_addr2rtuplerSrT)rZ add_noise_opsr\rCretr%r%r)rNts$z&ParameterNoise._tf_add_stored_noise_opcCst|jdusJ|jdkr||jn"|jdkr|nt|j|jD]\}}d|_| | d|_q$d|_dS)z Removes the current action noise from the model parameters. Args: tf_sess (Optional[tf.Session]): The tf-session to use to remove the noise from the (currently noisy) weights. Tr2rFN) rVrrrQrPrrBrCrrrr%r%r)rgs     zParameterNoise._remove_noisecCsxt}t|j|jD]\}}|t|| q tjt |}t |g t WdS1s5wYdS)zGenerates a tf-op for removing noise from the model's weights. Also used by tf-eager. Returns: tf.op: The tf op to remve the currently stored noise from the NN. Nr)rZremove_noise_opsr\rCrr%r%r)rPs$z"ParameterNoise._tf_remove_noise_opcCs d|jiS)N cur_stddev)r?)rZrvr%r%r)rs zParameterNoise.get_statestatervreturncCsT|d|_|jdkr|jj|j|ddSt|jtr!|j|_dS|j|jdS)Nrr2)session)r?rr"loadrWfloatr)rZrrvr%r%r)rs    zParameterNoise.set_state)N)#__name__ __module__ __qualname____doc__strdictr rintrr>rrboolrhrrrrjrrorersrrrrrLrqrfrNrgrPrr __classcell__r%r%r_r)rs r        S    &r)/gymnasium.spacesrrnumpyrGtypingrrrray.rllib.env.base_envrray.rllib.models.action_distrray.rllib.models.modelv2r "ray.rllib.models.tf.tf_action_distr r (ray.rllib.models.torch.torch_action_distr r ray.rllib.policy.sample_batchrray.rllib.utils.annotationsrr'ray.rllib.utils.exploration.explorationrray.rllib.utils.frameworkrrrray.rllib.utils.from_configrray.rllib.utils.numpyrrray.rllib.utils.typingrray.rllib.policy.policyrrRr2tfvrrrr%r%r%r)s*