o S۷iX@sUddlZddlZddlZddlZddlZddlZddlZddlZddlm Z ddl m Z m Z ddl m Z ddlmZddlmZddlmZmZddlmZmZmZmZmZdd lmZerdd lmZdd lmZdd lmZdd lm Z ddlm!Z!ddlm"Z"ddlm#Z#ddlm$Z$ddlm%Z%ddl&m'Z'ddl(m)Z)m*Z*ddlm+Z+m,Z,m-Z-m.Z.m/Z/m0Z0e'de1e.e2dZ3zddl4m5Z5ddl6m7Z8e8Z7de9d<e5ddZ:Wne;yejfd'd(Z?d&e>fd)d*Z@dNd+d,ZAdOd.d/ZBdNd0d1ZCdNd2d3ZDdNd4d5ZEdNd6d7ZFdNd8d9ZGdPd;d<ZHd=d>d&e>fd?d@ZIGdAdBdBZJGdCdDdDZKGdEdFdFeKZLGdGdHdHeKZMdIZNGdJdKdKZOGdLdMdMZPdS)QN)deque)datetimetimezone)VERSION)Envelope)LRUCache)DEFAULT_SAMPLING_FREQUENCY extract_stack)capture_internal_exception is_geventloggernowset_in_app_in_frames) TYPE_CHECKING)Any)Callable)Deque)Dict)List)Optional)Set)Type)Union) TypedDict)ContinuousProfilerModeSDKInfo)ExtractedSampleFrameIdStackIdThreadIdProcessedFrameProcessedStackProcessedSample timestamp thread_idstack_id) get_original) ThreadPoolzOptional[Type[_ThreadPool]]r(timesleepzOptional[ContinuousScheduler] _scheduleroptionsDict[str, Any]sdk_infor capture_funcCallable[[Envelope], None]returncCstdu}|rtdttrtj}ntj}|ddur$|d}n |di}|dp0|}t }|tjkr@t||||an|tjkrMt||||ant d |tdj tjd|set tdS) Nz0[Profiling] Continuous Profiler is already setup profiler_mode _experimentscontinuous_profiling_modez$Unknown continuous profiler mode: {}z9[Profiling] Setting up continuous profiler in {mode} mode)modeT)r+r debugteardown_continuous_profilerr GeventContinuousSchedulerr5ThreadContinuousSchedulergetr ValueErrorformatatexitregister)r,r.r/already_initializeddefault_profiler_moder2 experiments frequencyrC]/home/ubuntu/vllm_env/lib/python3.10/site-packages/sentry_sdk/profiler/continuous_profiler.pysetup_continuous_profilerJs<       rEcCstdurdStjSNF)r+sampledrCrCrCrDis_profile_session_sampledrHcCs$tdurdSts dStdSN)r+is_auto_start_enabled manual_startrCrCrCrD!try_autostart_continuous_profilers  rMUnion[ContinuousProfile, None]cCstdurdStSrJ)r+ auto_startrCrCrCrD!try_profile_lifecycle_trace_startsrPcCtdurdStdSrJ)r+rLrCrCrCrDstart_profiler rRcCtjdtddtdS)NzWThe `start_profile_session` function is deprecated. Please use `start_profile` instead. stacklevel)warningswarnDeprecationWarningrRrCrCrCrDstart_profile_session  r[cCrQrJ)r+ manual_stoprCrCrCrD stop_profilerrSr^cCrT)NzUThe `stop_profile_session` function is deprecated. Please use `stop_profile` instead.rUrV)rXrYrZr^rCrCrCrDstop_profile_sessionr\r_cCstdadSrJ)r^r+rCrCrCrDr7sr7Union[str, None]cCstdurdStjSrJ)r+ profiler_idrCrCrCrDget_profiler_idrIrb sample_ratezUnion[float, None]cCs|sdStt|kSrF)randomfloat)rcrCrCrD+determine_profile_session_sampling_decisionsrfc@s$eZdZUdZeed<dddZdS)ContinuousProfileTactiver1NcCs d|_dSrF)rhselfrCrCrDstops zContinuousProfile.stopr1N)__name__ __module__ __qualname__rhbool__annotations__rkrCrCrCrDrgs rgc @seZdZUdZded<deddddd d d d f d dZd efddZd(ddZ d)ddZ d)ddZ d)ddZ d)ddZ d)ddZd)ddZed*d!d"Zd+d$d%Zd)d&d'Zd S),ContinuousSchedulerunknownrr5rBr,r-r.rr/r0r1NcCszd||_||_||_||_|jd|_|jd}t||_||_ d|_ d|_ d|_ d|_ tdd|_t|_dS)Ng?profile_lifecycleprofile_session_sample_rateF)maxlen)intervalr,r.r/r: lifecyclerfrG make_samplersamplerbufferpidrunning soft_shutdownr new_profilessetactive_profiles)rjrBr,r.r/rurCrCrD__init__s     zContinuousScheduler.__init__cCs0|jtkr dS|jd}|sdS|dS)NFr3continuous_profiling_auto_start)r}osgetpidr,r:)rjrArCrCrDrKs   z)ContinuousScheduler.is_auto_start_enabledrNcCs@|jsdS|jdkr dStdt}|j|||S)Ntracez"[Profiling] Auto starting profiler)rGryr r6rgrappendensure_running)rjprofilerCrCrDrOs   zContinuousScheduler.auto_startcCs$|jsdS|jdkr dS|dSNmanual)rGryrrirCrCrDrLs   z ContinuousScheduler.manual_startcCs|jdkrdS|dSr)ryteardownrirCrCrDr]!s  zContinuousScheduler.manual_stopcCtrJNotImplementedErrorrirCrCrDr'z"ContinuousScheduler.ensure_runningcCrrJrrirCrCrDr*rzContinuousScheduler.teardowncCrrJrrirCrCrDpause-rzContinuousScheduler.pausecCst|j|jt|j|_dSrJ) ProfileBufferr,r.PROFILE_BUFFER_SECONDSr/r|rirCrCrD reset_buffer0s z ContinuousScheduler.reset_bufferr`cCs|jdurdS|jjSrJ)r|rarirCrCrDra5s zContinuousScheduler.profiler_idCallable[..., bool]cs`ttddjdkrdddddtffdd }|Sdddddtffd d }|S) N)max_sizerargsrkwargsr1csjsjsdStj}t}zfddtD}Wnty0tt YdSwt |D] }j j q5g}jD] }|j sP||qF|D]}j|qSjdurhj||dS) Take a sample of the stack on all the threads in the process. This should be called at a regular interval to collect samples. Tc$g|]\}}t|t|fqSrCstrr .0tidframecachecwdrCrD YKContinuousScheduler.make_sampler.._sample_stack..FN)rrlenr sys_current_framesitemsAttributeErrorr exc_inforangeaddpopleftrhrremover|write)rrrtssample_inactive_profilesrrrrjrCrD _sample_stackBs0          z7ContinuousScheduler.make_sampler.._sample_stackcsdt}zfddtD}Wnty#ttYdSwjdur0j||dS)rcrrCrrrrCrDrrrFN) r rrrrr rr|r)rrrrrrCrDrs     )rgetcwdrryrp)rjrrCrrDrz;s  Yz ContinuousScheduler.make_samplercCszt}|jr,||_t|}||jkrt|j||jr%d|_nt}|js|jdur;|jd|_dSdSrF) r) perf_counterr~r{rrx thread_sleepr|flush)rjlastelapsedrCrCrDruns      zContinuousScheduler.runr1rNrlr1r`)r1r)rmrnror5rqintrrprKrOrLr]rrrrpropertyrarzrrCrCrCrDrrs2          brrc s\eZdZUdZdZded<dZdeddd d d d d df fdd ZdddZ dddZ Z S)r9zr This scheduler is based on running a daemon thread that will call the sampler at a regular interval. threadrr5z)sentry.profiler.ThreadContinuousSchedulerrBr,r-r.rr/r0r1Ncs&t||||d|_t|_dSrJ)superrr threadingLocklockrjrBr,r.r/ __class__rCrDrsz"ThreadContinuousScheduler.__init__c Csd|_t}|jr|j|krdS|jN|jr&|j|kr& WddS||_d|_|tj|j |j dd|_ z|j Wnt yQd|_d|_ Yn wWddSWddS1sewYdS)NFT)nametargetdaemon)rrrr~r}rrrThreadrrrstart RuntimeErrorrjr}rCrCrDrs,  "z(ThreadContinuousScheduler.ensure_runningcC0|jrd|_|jdur|jd|_d|_dSrFr~rjoinr|rirCrCrDr    z"ThreadContinuousScheduler.teardownrl) rmrnro__doc__r5rqrrrrr __classcell__rCrCrrDr9s"   $r9c sXeZdZUdZdZded<dedddd d d d d f fdd ZdddZdddZ Z S)r8as This scheduler is based on the thread scheduler but adapted to work with gevent. When using gevent, it may monkey patch the threading modules (`threading` and `_thread`). This results in the use of greenlets instead of native threads. This is an issue because the sampler CANNOT run in a greenlet because 1. Other greenlets doing sync work will prevent the sampler from running 2. The greenlet runs in the same thread as other greenlets so when taking a sample, other greenlets will have been evicted from the thread. This results in a sample containing only the sampler's code. geventrr5rBr,r-r.rr/r0r1Ncs>tdur td|jt||||d|_t|_ dS)Nz"Profiler mode: {} is not available) r(r;r<r5rrrrrrrrrCrDrs z"GeventContinuousScheduler.__init__c Csd|_t}|jr|j|krdS|jJ|jr&|j|kr& WddS||_d|_|td|_z |j |j Wnt yMd|_d|_Yn wWddSWddS1sawYdS)NFT) rrrr~r}rrr(rspawnrrrrCrCrDrs,   "z(GeventContinuousScheduler.ensure_runningcCrrFrrirCrCrDr?rz"GeventContinuousScheduler.teardownrl) rmrnrorr5rqrrrrrrCrCrrDr8s    r8<c @s\eZdZdddddedddd f d d Zd ed ddd fddZd edefddZdddZ d S)rr,r-r.r buffer_sizer/r0r1NcCsP||_||_||_||_tj|_t|_ t |_ t t j|j |_dSrJ)r,r.rr/uuiduuid4hexra ProfileChunkchunkr start_monotonic_timerrutcr$start_timestamp)rjr,r.rr/rCrCrDrNs zProfileBuffer.__init__monotonic_timerrcCs:||r|t|_t|_|j|j||dSrJ) should_flushrrrr rrr)rjrrrCrCrDrjs zProfileBuffer.writecCs||j|jkSrJ)rr)rjrrCrCrDrrszProfileBuffer.should_flushcCs4|j|j|j|j}t}||||dSrJ)rto_jsonrar,r.radd_profile_chunkr/)rjrenveloperCrCrDrws zProfileBuffer.flushrl) rmrnrorrrerrprrrCrCrCrDrMs rc@sFeZdZdddZdeddddfdd Zd ed d d ddd fddZdS)rr1NcCs.tj|_i|_i|_g|_g|_g|_dSrJ) rrrchunk_idindexed_framesindexed_stacksframesstackssamplesrirCrCrDrs  zProfileChunk.__init__rrrc s|D]]\}\}}}zG|jvrBt|D]\}}|jvr,tjj|<j||qtjj|<jfdd|Dj||j|dWqty_t t YqwdS)Ncsg|]}j|qSrC)r)rframe_idrirCrDrsz&ProfileChunk.write..r#) r enumeraterrrrrrrr rr) rjrrrr& frame_idsrirrCrirDrs,   zProfileChunk.writerar,r-r.rcCs|j|j|jddtDd}t|d|d|d|d|j|dtd d ||d d }d D]}||durCt|| ||<q1|S)NcSs"i|] }t|jdt|jiqS)r)ridentr)rrrCrCrD s  z(ProfileChunk.to_json..)rrrthread_metadatarin_app_excludein_app_include project_rootr)rversionpython2)r client_sdkplatformrrar)release environmentdist) rrrrrrrrrstrip)rjrar,r.rpayloadkeyrCrCrDrs6  zProfileChunk.to_jsonrl)rmrnrorrerrrrCrCrCrDr~s  rrlrr)Qr=rrdrrr)rrX collectionsrrrsentry_sdk.constsrsentry_sdk.envelopersentry_sdk._lru_cachersentry_sdk.profiler.utilsrr sentry_sdk.utilsr r r r rtypingrrrrrrrrrrtyping_extensionsrsentry_sdk._typesrrrrrrr r!rerr" gevent.monkeyr'gevent.threadpoolr( _ThreadPoolrqr ImportErrorr*r+rprErHrMrPrRr[r^r_r7rbrfrgrrr9r8rrrrCrCrCrDs                        9       cDJ1