o i :@sddlZddlZddlmZddlmZeeZdejd<dejd<dej j _ dd Z d e efd d ZddedefddZdddZddZeedrvddlmZddlmZddlmZmZeegdej j jd<e e_ee_dSdS)N) init_loggeris_torch_equal1PYTORCH_NVML_BASED_CUDA_CHECKTORCHINDUCTOR_COMPILE_THREADSc sddlmmddlm}m}m}m}m}ddl m dt t ffdd }t jj|r@jjjdur@|jjjj}nj}|jrrt |jd|rr|jdjj|vrr|j|jrrt |jd|rr|jdjj|vsY|g}g} tt|jD]/} |j| } t | |r| |d|j| <qt | |r||qt | |r| |q| |t|dksJdS)Nr)EnterSubgraphLineExitSubgraphLineMemoryPlanningLineMemoryPlanningStateSubgraphPythonWrapperCodegenVreturncsddl}g}|d}|d}|D]3}t|jr)|jjdt|qt|jr>|jjdt|q|| q|S)Nr_none_shape) itertoolscount isinstanceNoneAsConstantBufferappendgraphnamenextShapeAsConstantBufferget_name) graph_outputsrnames shape_counter none_counternoderirG/home/ubuntu/vllm_env/lib/python3.10/site-packages/vllm/env_override.pyget_output_names-s    z3memory_plan_reuse_patched..get_output_names)torch._inductor.ir _inductorr#torch._inductor.codegen.wrapperr r r r r torch._inductor.virtualizedrliststrrr wrapper_codepartition_signatures output_nodesr&linesr!rpoprangelenplanr) selfr r r r r r& out_namesplanning_statespast_planning_statesiliner$r"r%memory_plan_reuse_patched"sF         r<skip_cudagraphscsddlm}ddlm}mmddlmddlm }g}|j } dt dtffdd tt|t|D]\}} |} |D] } | | jqI| |} |jd d |D} |fd d | j| jBD| }|fd d|D}||D]} | jqfdd | D}||fdd|D}fdd|D}fdd |D}| ||fdd| D} fdd | D}fdd |D}||}|||||| |}||||| }q@|dddS)z Gets signature for each graph partition, including input nodes, output nodes, and whether deallocating an input within graph partition. r) dependencies)GraphPartitionSignatureMutationOutput NoneLayoutr) OrderedSetbuf_namercsVj|d}|dur dSt|jjr)t|jr'j|d}r'|SdSdS)z Checks if buf_name is NoneLayout. Buffers with NoneLayout is not allocated so graph partition should not take it as inputs or outputs. NFT) name_to_bufgetrr!layoutmutation_real_name)rCbuf real_name)r@rAis_none_layoutr6r$r%rJws z=get_graph_partition_signature_patched..is_none_layoutcSsg|]}|jqSr$) read_writes).0r!r$r$r% sz9get_graph_partition_signature_patched..csg|] }|js|jqSr$)r)rLx)rJr$r%rMs c3|] }j||VqdSNrGrErLrr6r$r%  z8get_graph_partition_signature_patched..csg|]}|vr|qSr$r$rR name_to_noder$r%rMs csi|] }|vr||qSr$r$rRrVr$r% z9get_graph_partition_signature_patched..csi|] }|vr||vqSr$r$rRbuffer_names_to_freerWr$r%rXrYcs g|] }|vr|vr|qSr$r$rRrZr$r%rMs c3rOrPrQrRrSr$r%rTrUcsg|] }|s|qSr$r$rR)rJrWr$r%rMs csg|] }|jjvr|qSr$)r constantsrRrr$r%rMsNr')torch._inductorr>r(r?r@rAr+rtorch.utils._ordered_setrBrr&get_name_to_nodesr-boolzipreversedupdateoutputs_by_namekeys intersection ReadWrites merge_listreadswrites last_usage!get_graph_partition_symbol_inputsrunion)r6 partitionsr=r>r?rB signaturesunmet_output_names partitionskip_cudagraph output_namesr!returned_output_namesrKpartition_input_namesextra_input_names input_nodesinput_deallocationextra_output_namesr0constant_names symbol_inputspartition_signaturer$)r@rArr[rJrWr6r%%get_graph_partition_signature_patchedfs                     r}F should_logrcsddlmm}ddlm}m}ddlm}m}m }|j } t | t jjj rY| j} rY| } t | t jjr>| d| jn| } | t jjjvsN| t jjjvrYt | t jjsWJdSt jjjjsf|jdurfdSdtd|dBd dfd d } |rw|n| }t ||rtfd d |jDS|j dusJ|s|d|ddSt |j |jr|d|ddSt |j |jr|d|ddSt|j ddr|d|ddS||j r|d|ddSdS)zBReturn True if we should partition the inductor graph on this noderN)BaseSchedulerNodeFusedSchedulerNode)&_unstable_customized_partition_wrapperis_cudagraph_unsafe_opmaybe_log_cudagraph_partition.Tmsgr!rcSsdSrPr$)rr!r$r$r%noop_log9sz*should_partition_patched..noop_logc3s|]}|VqdSrP)should_partition)rLsnoderSr$r%rT?sz+should_partition_patched..z non gpu ops)r!zDeviceCopy opszConditional opsunbacked_bindingszunbacked binding opszCUDAGraph-unsafe custom opsF)r(r)r#torch._inductor.schedulerrrtorch._inductor.utilsrrrr!rtorchFallbackKernel op_overloadr_ops OpOverload _overloadnameconfigcustom_should_partition_opstriton cudagraphswrapperr-anysnodesis_gpu DeviceCopy Conditionalgetattr)r6r!r~r#rrrrrir_nodeopop_overload_packet_nameop_overload_namerlog_partition_reasonr$rSr%should_partition_patchedsV           rcCshddlmm}ddlm}t|_t|_| dd||j |_ WddS1s-wYdS)z (Re)initializes the scheduler member. When initializing the scheduler, no CUBIN files should be generated (to avoid biasing any benchmarks and pessimizing fusion decisions). rN) Schedulerztriton.store_cubinF) torch._inductor.configr)rrrrrr}get_graph_partition_signaturepatch operations scheduler)r6rrr$r$r%_update_scheduler_patched[s "rcCsNddlm}|ds|dr%ddl}ttjdr#ddlm}||_dSdSdS)z;Workaround for TorchInductor autotune get_raw_stream() bug.rr2.9.0z2.9.1N_cuda_getCurrentRawStream)r) vllm.utils.torch_utilsrbuiltinshasattrr_Ctorch._Crget_raw_stream)rr_get_raw_streamr$r$r%_patch_get_raw_stream_if_neededts    rr)PythonWrapperCodegen) GraphLowering)_Config _ConfigEntry)defaultr)F)rN)osr vllm.loggerrrr__name__loggerenvironr)rcompile_threadsr<r,r`r}rrrr*rtorch._inductor.graphrtorch.utils._config_modulerr_configmemory_plan_reuse_update_schedulerr$r$r$r%s6     D # S