o $i%@sddlZddlZddlmZmZddlmZmZddlm Z m Z m Z m Z m Z mZddlmZmZddlmZmZddlmZddlmZdd lmZmZdd lmZdd lmZdd l m!Z!dd l"m#Z#m$Z$ddl%m&Z&ddl'm(Z(ddl)m*Z*e rddl+m,Z,ddl-m.Z.ddl/m0Z0e&\Z1Z2Z3e4e5Z6edgdZ7e e e Z8GdddeZ9e#GdddeedZ:e#Gddde:Z;dS)N)ABCMetaabstractmethod) defaultdict namedtuple) TYPE_CHECKINGAnyListOptionalTypeUnion)DEPRECATED_VALUEdeprecation_warning)BaseEnvconvert_to_base_env)SampleCollector)SimpleListCollector) EnvRunnerV2 _PerfStats)RolloutMetrics) InputReader)concat_samples) OldAPIStackoverride) try_import_tf)SampleBatchType)log_once) RLlibCallback)ObservationFunction) RolloutWorker_PolicyEvalData)env_idagent_idobsinfo rnn_state prev_action prev_rewardc@seZdZddZdS)_NewEpisodeDefaultDictcCs(|jdur t|||}||<|SN)default_factoryKeyError)selfr retr-Y/home/ubuntu/veenaModal/venv/lib/python3.10/site-packages/ray/rllib/evaluation/sampler.py __missing__-s z"_NewEpisodeDefaultDict.__missing__N)__name__ __module__ __qualname__r/r-r-r-r.r',s r'c@sdeZdZdZeedefddZedefddZ ede e fddZ ede efd d Z d S) SamplerInputz1Reads input experiences from an existing sampler.returncCs4|g}||t|dkrtdt|S)NrzNo data available from sampler.)get_dataextendget_extra_batcheslen RuntimeErrorr)r+batchesr-r-r.next9s  zSamplerInput.nextcCt)zCalled by `self.next()` to return the next batch of data. Override this in child classes. Returns: The next batch of data. NotImplementedErrorr+r-r-r.r5As zSamplerInput.get_datacCr<)a$Returns list of episode metrics since the last call to this method. The list will contain one RolloutMetrics object per completed episode. Returns: List of RolloutMetrics objects, one per completed episode since the last call to this method. r=r?r-r-r. get_metricsLs zSamplerInput.get_metricscCr<)aReturns list of extra batches since the last call to this method. The list will contain all SampleBatches or MultiAgentBatches that the user has provided thus-far. Users can add these "extra batches" to an episode by calling the episode's `add_extra_batch([SampleBatchType])` method. This can be done from inside an overridden `Policy.compute_actions_from_input_dict(..., episodes)` or from a custom callback's `on_episode_[start|step|end]()` methods. Returns: List of SamplesBatches or MultiAgentBatches provided thus-far by the user since the last call to this method. r=r?r-r-r.r7XszSamplerInput.get_extra_batchesN)r0r1r2__doc__rrrr;rr5rrr@r7r-r-r-r.r35s  r3) metaclassc@seZdZdZddddddddddddeeedddd ed eeefd e d e d ddededede dde e e defddZeedefddZeedeefddZeedeefddZdS) SyncSamplerzHSync SamplerInput that collects experiences when `get_data()` is called. env_stepsFTN)count_steps_bymultiple_episodes_in_batchnormalize_actions clip_actionsobservation_fnsample_collector_classrenderpoliciespolicy_mapping_fn preprocessors obs_filterstf_sesshorizon soft_horizonno_done_at_endworkerrenv clip_rewardsrollout_fragment_lengthrE callbacksrrFrGrHrIrrJrKc Cs(tdrO| dur tdd|durtdd|durtdd|dur(tdd|dur1tdd|tkr;td d d |tkrEtd d d |tkrOtd d d t||_||_t|_t |j j d|_ | sht } | |j|||||d|_| |_t||j|||j |||jd|_|j|_t|_dS)aNInitializes a SyncSampler instance. Args: worker: The RolloutWorker that will use this Sampler for sampling. env: Any Env object. Will be converted into an RLlib BaseEnv. clip_rewards: True for +/-1.0 clipping, actual float value for +/- value clipping. False for no clipping. rollout_fragment_length: The length of a fragment to collect before building a SampleBatch from the data and resetting the SampleBatchBuilder object. count_steps_by: One of "env_steps" (default) or "agent_steps". Use "agent_steps", if you want rollout lengths to be counted by individual agent steps. In a multi-agent env, a single env_step contains one or more agent_steps, depending on how many agents are present at any given time in the ongoing episode. callbacks: The RLlibCallback object to use when episode events happen during rollout. multiple_episodes_in_batch: Whether to pack multiple episodes into each batch. This guarantees batches will be exactly `rollout_fragment_length` in size. normalize_actions: Whether to normalize actions to the action space's bounds. clip_actions: Whether to clip actions according to the given action_space's bounds. observation_fn: Optional multi-agent observation func to use for preprocessing observations. sample_collector_class: An optional SampleCollector sub-class to use to collect, store, and retrieve environment-, model-, and sampler data. render: Whether to try to render the environment after each step. deprecated_sync_sampler_argsNrL)oldrMrNrOrPrQT)rZerrorrRrS)ema_coef)rE)rTbase_envrFrX perf_statsrWrErK)rr r rr]rWqueueQueue extra_batchesrconfigsampler_perf_stats_ema_coefr^r policy_mapsample_collectorrKr_env_runner_objrun _env_runner metrics_queue)r+rTrUrVrWrErXrFrGrHrIrJrKrLrMrNrOrPrQrRrSr-r-r.__init__os\<           zSyncSampler.__init__r4cCs* t|j}t|tr|j|n|Sqr()r;rh isinstancerriput)r+itemr-r-r.r5s   zSyncSampler.get_datacCsBg} z||jj|jdWn tjyY|Swq)NT)r^)appendri get_nowait_replacer^getr_Empty)r+ completedr-r-r.r@s zSyncSampler.get_metricscCs4g} z ||jWn tjyY|Swqr()rnraror_rr)r+extrar-r-r.r7szSyncSampler.get_extra_batches)r0r1r2rAr rr boolfloatintstrr r rrjrr3rr5rrr@r7r-r-r-r.rCks`        o rC)<loggingr_abcrr collectionsrrtypingrrrr r r ray._common.deprecationr r ray.rllib.env.base_envrr0ray.rllib.evaluation.collectors.sample_collectorr5ray.rllib.evaluation.collectors.simple_list_collectorr"ray.rllib.evaluation.env_runner_v2rrray.rllib.evaluation.metricsrray.rllib.offlinerray.rllib.policy.sample_batchrray.rllib.utils.annotationsrrray.rllib.utils.frameworkrray.rllib.utils.typingrray.util.debugrray.rllib.callbacks.callbacksr)ray.rllib.evaluation.observation_functionr#ray.rllib.evaluation.rollout_workerrtf1tf_ getLoggerr0loggerr StateBatchr'r3rCr-r-r-r.sB                 5