o װid@sXdZddlTddlZddlZddlZddlmZdZdZdZ dZ d Z d Z d Z d Zd ZdZdZdZdZdZdZdZdZdZdZdZdZdZdZdZdZdZ dZ!d Z"d!Z#d"Z$d#Z%d$Z&d%Z'd&Z(d'Z)d(Z*d)Z+d*Z,d+Z-d,Z.d-Z/d.Z0d/Z1d0Z2d1Z3d2Z4d3Z5d4Z6d5Z7d6Z8d7Z9d8Z:d9Z;d:Zd=Z?d>Z@d?ZAd@ZBdAZCdBZDdCZEdDZFdEZGdFZHdGZIdHZJdIZKdJZLdKZMdLZNdMZOdNZPdOZQdPZRdQZSdRZTdSZUdTZVdUZWdVZXdWZYdXZZdYZ[dZZ\d[Z]d\Z^d]Z_d^Z`d_Zad`ZbdaZcdbZddcZeddZfdeZgdfZhdgZidhZjdiZkdjZldkZmdlZndmZodnZpdoZqdpZrdqZsdrZtdsZudtZvduZwdvZxdwZydxZzdyZ{dzZ|d{Z}d|Z~d}Zd~ZdZdZidede de de de de de de de dedededededededeidededededededededededede de"de"de#de$de%ide%de&de&de'de'de(de)de_de`deadebdecdeddeedefdegdehide*de,de+de-de/de0de3de4de5de6de7de8de9de:de;de<de>ide?de@de=deAdeBdeCdeDdeEdeGdeHdeIdeFdeJdeKdeLdeMdeNidePdeQdeRdeOdeSdeTdeUdeVdeWdeYdeZde[deXde\de\de]de]ide^deidejdekdeldemdendeodepdeqderderdesdesdetdetdeuideudevdevdewdewdexdexdeydeydezdezde{de{de|de|dede}e~eeedZied ed ed ed ed ededededededededededededZGdddZedkr*ddZedSdS(a nvidia_smi Sample code that attempts to reproduce the output of nvidia-smi -q -x For many cases the output should match. Each query parameter is documented within nvdia-smi --help-query-gpu From Code: DeviceQuery() DeviceQuery("--help") DeviceQuery("--help-query-gpu") DeviceQuery("pci.bus_id,memory.total,memory.free") DeviceQuery([NVSMI_PCI_BUS_ID, NVSMI_MEMORY_TOTAL, NVSMI_MEMORY_FREE]) XmlDeviceQuery() XmlDeviceQuery("--help") XmlDeviceQuery("--help-query-gpu") )*N)Thread  !23456<=>?iiiFGPQRSTUVWXZ[\]^_`abdefghijklnopqrstuvxyz timestampdriver_versioncountnamegpu_nameserial gpu_serialuuidgpu_uuidz pci.bus_id gpu_bus_idz pci.domainzpci.busz pci.devicez pci.device_idzpci.sub_device_idzpcie.link.gen.currentzpcie.link.gen.maxzpcie.link.width.currentzpcie.link.width.maxindex display_modedisplay_activepersistence_modezaccounting.modezaccounting.buffer_sizezdriver_model.currentzdriver_model.pending vbios_versionz inforom.imgz inforom.imagez inforom.oemz inforom.eccz inforom.pwrz inforom.powerz gom.currentzgpu_operation_mode.currentz gom.pendingzgpu_operation_mode.pendingz fan.speedpstatez!clocks_throttle_reasons.supportedzclocks_throttle_reasons.activez clocks_throttle_reasons.gpu_idlez3clocks_throttle_reasons.applications_clocks_settingz$clocks_throttle_reasons.sw_power_capz#clocks_throttle_reasons.hw_slowdownz+clocks_throttle_reasons.hw_thermal_slowdownz/clocks_throttle_reasons.hw_power_brake_slowdownz+clocks_throttle_reasons.sw_thermal_slowdownz"clocks_throttle_reasons.sync_boostz memory.totalz memory.usedz memory.free compute_modezutilization.gpuzutilization.memoryzencoder.stats.sessionCountzencoder.stats.averageFpszencoder.stats.averageLatencyzecc.mode.currentzecc.mode.pendingz+ecc.errors.corrected.volatile.device_memoryz+ecc.errors.corrected.volatile.register_filez&ecc.errors.corrected.volatile.l1_cachez&ecc.errors.corrected.volatile.l2_cachez,ecc.errors.corrected.volatile.texture_memoryz"ecc.errors.corrected.volatile.dramz"ecc.errors.corrected.volatile.sramz!ecc.errors.corrected.volatile.cbuz#ecc.errors.corrected.volatile.totalz,ecc.errors.corrected.aggregate.device_memoryz,ecc.errors.corrected.aggregate.register_filez'ecc.errors.corrected.aggregate.l1_cachez'ecc.errors.corrected.aggregate.l2_cachez-ecc.errors.corrected.aggregate.texture_memoryz#ecc.errors.corrected.aggregate.dramz"ecc.errors.corrected.aggregate.cbuz#ecc.errors.corrected.aggregate.sramz$ecc.errors.corrected.aggregate.totalz-ecc.errors.uncorrected.volatile.device_memoryz-ecc.errors.uncorrected.volatile.register_filez(ecc.errors.uncorrected.volatile.l1_cachez(ecc.errors.uncorrected.volatile.l2_cachez.ecc.errors.uncorrected.volatile.texture_memoryz$ecc.errors.uncorrected.volatile.dramz#ecc.errors.uncorrected.volatile.cbuz$ecc.errors.uncorrected.volatile.sramz%ecc.errors.uncorrected.volatile.totalz.ecc.errors.uncorrected.aggregate.device_memoryz.ecc.errors.uncorrected.aggregate.register_filez)ecc.errors.uncorrected.aggregate.l1_cachez)ecc.errors.uncorrected.aggregate.l2_cachez/ecc.errors.uncorrected.aggregate.texture_memoryz%ecc.errors.uncorrected.aggregate.dramz$ecc.errors.uncorrected.aggregate.cbuz%ecc.errors.uncorrected.aggregate.sramz&ecc.errors.uncorrected.aggregate.totalz"retired_pages.single_bit_ecc.countzretired_pages.sbezretired_pages.double_bit.countzretired_pages.dbezretired_pages.pendingztemperature.gpuztemperature.memoryzpower.managementz power.drawz power.limitzenforced.power.limitzpower.default_limitzpower.min_limitzpower.max_limitzclocks.current.graphicsz clocks.grzclocks.current.smz clocks.smzclocks.current.memoryz clocks.memzclocks.current.videoz clocks.videozclocks.applications.graphicszclocks.applications.grzclocks.applications.memoryzclocks.applications.memz$clocks.default_applications.graphicszclocks.default_applications.grz"clocks.default_applications.memoryzclocks.default_applications.memzclocks.max.graphicsz clocks.max.grz clocks.max.smzclocks.max.memoryzclocks.max.memzsupported-clocksz compute-apps)zaccounted-appsclockszmig.mode.currentzmig.mode.pendingUnknownQuadroTeslaNVSGridGeForceTitanzNVIDIA Virtual ApplicationszNVIDIA Virtual PCzNVIDIA Virtual Compute ServerzNVIDIA RTX Virtual WorkstationzNVIDIA vGamingz Quadro RTXz NVIDIA RTXNVIDIAz Geforce RTXz Titan RTXc@sLeZdZdZdZGdddZeddZed9ddZd d Z ed d Z d dZ eddZ eddZ eddZeddZeddZeddZeddZeddZedd Zed!d"Zed#d$Zed%d&Zed'd(Zed)d*Zed+d,Zed-d.Zed:d/d0Zed:d1d2Zd3d4Zd5d6Z d7d8Z!dS); nvidia_smiNc@s\eZdZdZdZdZdZdddZddZe ddd Z d d Z d d Z ddZ ddZdS)znvidia_smi.loop_asyncNFrcCs8d|_||_ttjj|||tjjfd|_|jdS)NF)targetargs) _loop_async__abort_loop_async__callback_chainrr loop_async_loop_async__loop_task_loop_async__callback_loop_async__taskstart)selftime_in_millisecondsfiltercallbackr>/home/ubuntu/.local/lib/python3.10/site-packages/pynvml/smi.py__init__sznvidia_smi.loop_async.__init__cCsd|_d|_dSNT)rrrrrr__del__s znvidia_smi.loop_async.__del__cCs\|d}t}|dkr,||}||_|dur|||t||dksdSdS)NiF)r getInstance is_aborted DeviceQuery_loop_async__last_resultstimesleep) async_resultsrrr delay_secondsnvsmiresultsrrr __loop_tasks    z!nvidia_smi.loop_async.__loop_taskcCs$||_|jdur|||dSdSN)_loop_async__last_resultr)rresultrrr __callbacks z nvidia_smi.loop_async.__callbackcCs"d|_|jdur|jdSdSr)rrjoinrrrrcancels znvidia_smi.loop_async.cancelcC|jSr)rrrrrrz nvidia_smi.loop_async.is_abortedcCrr)rrrrrrrznvidia_smi.loop_async.resultrNN)__name__ __module__ __qualname__rrrrrr staticmethodrrrrrrrrrr{s    rcCstjdkrttjS)z Static access method. N)r_nvidia_smi__instancerrrrrs znvidia_smi.getInstancercCst|||Sr)rr)rrrrrrloopsznvidia_smi.loopcCs4tjdkr td|t_tjdkrtt_dSdS)z Virtually private constructor. Nz,This class is a singleton, use getInstance())rr Exception_nvidia_smi__handles_nvidia_smi__initialize_nvmlrrrrrs  znvidia_smi.__init__cCs0tt}i}td|D]}t|||<q |S)z Initialize NVML bindings. r)nvmlInitnvmlDeviceGetCountrangenvmlDeviceGetHandleByIndex) deviceCounthandlesirrr__initialize_nvmls znvidia_smi.__initialize_nvmlcCs6tjdkrtj|krt`dt_dt_tdSdSdSr)rrr nvmlShutdownrrrrrs  znvidia_smi.__del__cCs:|d}g}|D]}|}|tvr|t|q |S)N,)splitstripNVSMI_QUERY_GPUappend) queryString parametersvaluesppsrrr__fromDeviceQueryStrings z"nvidia_smi.__fromDeviceQueryStringc Cs$d}z t|||t}Wnty"}z t|}WYd}~nd}~ww|dt|d7}z t|||t}WntyN}z t|}WYd}~nd}~ww|dt|d7}z t|||t}Wntyz}z t|}WYd}~nd}~ww|dt|d7}z t|||t}Wnty}z t|}WYd}~nd}~ww|dt|d 7}z t|||t }Wnty}z t|}WYd}~nd}~ww|d t|d 7}z t|||t } Wnty}z t|} WYd}~nd}~ww|d t| d 7}z t|||t }Wnty+}z t|}WYd}~nd}~ww|dt|d7}z t|||t }WntyX}z t|}WYd}~nd}~ww|dt|d7}z tt |||} Wnty}z t|} WYd}~nd}~ww|d| d7}|S)Nz z z z z z z z z z z z z z z z z )nvmlDeviceGetMemoryErrorCounter"NVML_MEMORY_LOCATION_DEVICE_MEMORY NVMLErrorr_nvidia_smi__handleError_nvidia_smi__toStringNVML_MEMORY_LOCATION_DRAM"NVML_MEMORY_LOCATION_REGISTER_FILENVML_MEMORY_LOCATION_L1_CACHENVML_MEMORY_LOCATION_L2_CACHE#NVML_MEMORY_LOCATION_TEXTURE_MEMORYNVML_MEMORY_LOCATION_CBUNVML_MEMORY_LOCATION_SRAMnvmlDeviceGetTotalEccErrors) handle counterType errorType strResult deviceMemoryerr registerFilel1Cachel2Cache textureMemoryrrrr__xmlGetEccByTypesznvidia_smi.__xmlGetEccByTypec Csd}i}z t|||t}Wnty$}z t|}WYd}~nd}~ww||d<z t|||t}WntyI}z t|}WYd}~nd}~ww||d<z t|||t}Wntyn}z t|}WYd}~nd}~ww||d<z t|||t}Wnty}z t|}WYd}~nd}~ww||d<z t|||t} Wnty}z t|} WYd}~nd}~ww| |d<z t|||t } Wnty}z t|} WYd}~nd}~ww| |d<z t|||t }Wnty}z t|}WYd}~nd}~ww||d<z t|||t }Wnty)}z t|}WYd}~nd}~ww||d <z t t |||} WntyQ}z t|} WYd}~nd}~ww| |d <|S) Nr device_memorydram register_filel1_cachel2_cachetexture_memorycbusramtotal)rrrrrrrrrrrrrr) rrrr eccByTyperrrrrrrrrr__GetEccByType'sznvidia_smi.__GetEccByTypecCsd}t|vsRt|vsRt|vsRt|vsRt|vsRt|vsRt|vsRt|vsRt|vsRt|vsRt |vsRt |vsRt |vsRt |vsRt |vsRt|vsRt|vsRt|vsRt|vsRt|vrf|d7}|tt||t7}|d7}t|vst|vst|vst|vst|vst|vst|vst|vst|vst|vst |vst |vst |vst |vst |vst|vst|vst|vst|vst|vr|d7}|tt||t7}|d7}|S)Nrz z z z ) NVSMI_ALL*NVSMI_ECC_ERROR_CORRECTED_VOLATILE_DEV_MEM*NVSMI_ECC_ERROR_CORRECTED_VOLATILE_REGFILE*NVSMI_ECC_ERROR_CORRECTED_VOLATILE_L1CACHE*NVSMI_ECC_ERROR_CORRECTED_VOLATILE_L2CACHE*NVSMI_ECC_ERROR_CORRECTED_VOLATILE_TEXTURE'NVSMI_ECC_ERROR_CORRECTED_VOLATILE_DRAM'NVSMI_ECC_ERROR_CORRECTED_VOLATILE_SRAM&NVSMI_ECC_ERROR_CORRECTED_VOLATILE_CBU(NVSMI_ECC_ERROR_CORRECTED_VOLATILE_TOTAL+NVSMI_ECC_ERROR_CORRECTED_AGGREGATE_DEV_MEM+NVSMI_ECC_ERROR_CORRECTED_AGGREGATE_REGFILE+NVSMI_ECC_ERROR_CORRECTED_AGGREGATE_L1CACHE+NVSMI_ECC_ERROR_CORRECTED_AGGREGATE_L2CACHE+NVSMI_ECC_ERROR_CORRECTED_AGGREGATE_TEXTURE(NVSMI_ECC_ERROR_CORRECTED_AGGREGATE_DRAM'NVSMI_ECC_ERROR_CORRECTED_AGGREGATE_CBU(NVSMI_ECC_ERROR_CORRECTED_AGGREGATE_SRAM)NVSMI_ECC_ERROR_CORRECTED_AGGREGATE_TOTALrr_nvidia_smi__xmlGetEccByType NVML_MEMORY_ERROR_TYPE_CORRECTED"NVML_MEMORY_ERROR_TYPE_UNCORRECTEDrrr eccByCounterrrr__xmlGetEccByCounterqs`znvidia_smi.__xmlGetEccByCountercCsdi}t|vsRt|vsRt|vsRt|vsRt|vsRt|vsRt|vsRt|vsRt|vsRt|vsRt |vsRt |vsRt |vsRt |vsRt |vsRt|vsRt|vsRt|vsRt|vsRt|vr[t||t|d<t|vst|vst|vst|vst|vst|vst|vst|vst|vst |vst |vst |vst |vst |vst|vst|vst|vst|vst|vrt||t|d<|S)N single_bit double_bit)rrrrrrrrr r r r r rrrrrrr_nvidia_smi__GetEccByTyperrrrrr__GetEccByCountersVznvidia_smi.__GetEccByCountercCsd}d}t|vsPt|vsPt|vsPt|vsPt|vsPt|vsPt|vsPt|vsPt|vsPt |vsPt |vsPt |vsPt |vsPt |vsPt|vsPt|vsPt|vsPt|vsPt|vrf|d7}|tt|t|7}|d7}d}t|vst|vst|vst|vst|vst|vst|vst|vst|vst|vst |vst!|vst"|vst#|vst$|vst%|vst&|vst'|vst(|vr|d7}|tt|t)|7}|d7}d}t*|dkr||fSd|fS) NrFz z Tz z r)+rrrrrrrrr r ,NVSMI_ECC_ERROR_UNCORRECTED_VOLATILE_DEV_MEM,NVSMI_ECC_ERROR_UNCORRECTED_VOLATILE_REGFILE,NVSMI_ECC_ERROR_UNCORRECTED_VOLATILE_L1CACHE,NVSMI_ECC_ERROR_UNCORRECTED_VOLATILE_L2CACHE,NVSMI_ECC_ERROR_UNCORRECTED_VOLATILE_TEXTURE)NVSMI_ECC_ERROR_UNCORRECTED_VOLATILE_DRAM)NVSMI_ECC_ERROR_UNCORRECTED_VOLATILE_SRAM(NVSMI_ECC_ERROR_UNCORRECTED_VOLATILE_CBU*NVSMI_ECC_ERROR_UNCORRECTED_VOLATILE_TOTALrr_nvidia_smi__xmlGetEccByCounterNVML_VOLATILE_ECCr r r rrrrrr-NVSMI_ECC_ERROR_UNCORRECTED_AGGREGATE_DEV_MEM-NVSMI_ECC_ERROR_UNCORRECTED_AGGREGATE_REGFILE-NVSMI_ECC_ERROR_UNCORRECTED_AGGREGATE_L1CACHE-NVSMI_ECC_ERROR_UNCORRECTED_AGGREGATE_L2CACHE-NVSMI_ECC_ERROR_UNCORRECTED_AGGREGATE_TEXTURE*NVSMI_ECC_ERROR_UNCORRECTED_AGGREGATE_DRAM*NVSMI_ECC_ERROR_UNCORRECTED_AGGREGATE_SRAM)NVSMI_ECC_ERROR_UNCORRECTED_AGGREGATE_CBU+NVSMI_ECC_ERROR_UNCORRECTED_AGGREGATE_TOTALNVML_AGGREGATE_ECClenrrecc includeEccrrr __xmlGetEccsbznvidia_smi.__xmlGetEcccCsi}d}t|vsPt|vsPt|vsPt|vsPt|vsPt|vsPt|vsPt|vsPt|vsPt |vsPt |vsPt |vsPt |vsPt |vsPt|vsPt|vsPt|vsPt|vsPt|vr[t|t||d<d}t|vst|vst|vst|vst|vst|vst|vst|vst|vst|vst|vst |vst!|vst"|vst#|vst$|vst%|vst&|vst'|vrt|t(||d<d}t)|*dkr||fSd|fS)NFvolatileT aggregater)+rrrrrrrrr r rrr r!r"r#r$r%r&r_nvidia_smi__GetEccByCounterr(r r r rrrrrrr)r*r+r,r-r.r/r0r1r2r3rr4rrr__GetEcc sZ znvidia_smi.__GetEccc Csd}d}d}zt||}ttt|}Wnty-}z t|}d}WYd}~nd}~ww|dt|d7}|durU|d7}|D] }|dd|d7}qC|d 7}n|d |d 7}|dkrc|SdS) Nrrz z z! z 0x%016xz z" z z )nvmlDeviceGetRetiredPagessummapr3rrrr)rcauseretiredPagedByCauseerrorrpagesrpagerrr__xmlGetRetiredPagesByCause?s&   z&nvidia_smi.__xmlGetRetiredPagesByCausec Csi}d}d}zt||}Wnty&}z t|}d}WYd}~nd}~wwi}|durFd}|D]} d| |dt|<|d7}|d7}q1|durN||d<||d<t|dkr\|nd|d<|dkrf|SdS)Nrrr<retired_page_address_Error retired_countretired_page_addresses)r=rrrstrr3r) rr@rArBrrCrretiredPageAddressesiirDrrr__GetRetiredPagesByCauseWs,  z#nvidia_smi.__GetRetiredPagesByCausec Csd}d}ddg}ttD]4}t|vs"t|vr|dks"t|vr@|dkr@|d||d7}|t||7}|d ||d7}d }q t|vsIt|vr~|d 7}ztt |krY|d 7}n|d 7}Wnt yw}z |t |7}WYd}~nd}~ww|d7}d }t |dkr||fSd|fS)NrFmultiple_single_bit_retirementdouble_bit_retirementrrz z NoYesz ) r NVML_PAGE_RETIREMENT_CAUSE_COUNTr'NVSMI_RETIREDPAGES_SINGLE_BIT_ECC_COUNT'NVSMI_RETIREDPAGES_DOUBLE_BIT_ECC_COUNTr&_nvidia_smi__xmlGetRetiredPagesByCauseNVSMI_RETIREDPAGES_PENDINGNVML_FEATURE_DISABLED&nvmlDeviceGetRetiredPagesPendingStatusrrr3)rr retiredPagesincludeRetiredPagescausesidxrrrr__xmlGetRetiredPagesrs4   znvidia_smi.__xmlGetRetiredPagesc Csi}d}ddg}ttD]"}t|vs"t|vr|dks"t|vr.|dkr.t|||||<d}q t|vs7t|vrdd}z tt |krCd}nd }Wnt y]}z t |}WYd}~nd}~ww||d <d}t | dkrp||fSd|fS) NFrNrOrrTrrPrQpending_retirement)rrRrrSrTr#_nvidia_smi__GetRetiredPagesByCauserVrWrXrrr3r)rrrYrZr[r\r^rrrr__GetRetiredPagess0   znvidia_smi.__GetRetiredPagescCs(|tkrdS|tkr dS|tkrdSdS)NzAll OnComputezLow Double Precisionr)NVML_GOM_ALL_ONNVML_GOM_COMPUTENVML_GOM_LOW_DP)moderrr __toStrGOMsznvidia_smi.__toStrGOMc Cstdgtdgtdgtdgtdgtdgg}d}z6t|}t|}|d7}|D] \}}|dkrE||@r:||@r7d nd }nd }|d |||f7}q%|d 7}W|Styh}z|dt |7}WYd}~|Sd}~ww)Nclocks_throttle_reason_gpu_idle*clocks_throttle_reason_user_defined_clocks2clocks_throttle_reason_applications_clocks_setting#clocks_throttle_reason_sw_power_cap"clocks_throttle_reason_hw_slowdownclocks_throttle_reason_nonerz Active Not ActiveN/Az <%s>%s z z: %s ) nvmlClocksThrottleReasonGpuIdle)nvmlClocksThrottleReasonUserDefinedClocks1nvmlClocksThrottleReasonApplicationsClocksSetting"nvmlClocksThrottleReasonSwPowerCap"nvmlClocksThrottleReasonHwSlowdownnvmlClocksThrottleReasonNone+nvmlDeviceGetSupportedClocksThrottleReasons)nvmlDeviceGetCurrentClocksThrottleReasonsrrr) rthrottleReasonsrsupportedClocksThrottleReasonsclocksThrottleReasonsmaskrvalrrrr__xmlGetClocksThrottleReasonss4   z(nvidia_smi.__xmlGetClocksThrottleReasonsc Cstdgtdgtdgtdgtdgtdgg}i}z(t|}t|}|D]\}}|dkr<||@r6||@r3dnd}nd }|||<q!WntyW}z t ||d <WYd}~nd}~wwt | d krb|SdS) NrgrhrirjrkrlrmrnrorGr) rprqrrrsrtrurvrwrrrr3r) rrxclockThrottleReasonsryrzr{rr|rrrr__GetClocksThrottleReasonss0  z%nvidia_smi.__GetClocksThrottleReasonscCs|jtkrdS|S)Nro)valueNVML_ERROR_NOT_SUPPORTED__str__)rrrr __handleErrors znvidia_smi.__handleErrorcCst|tr |dSt|S)Nzutf-8) isinstancebytesdecoderJ)r|rrr __toStrings  znvidia_smi.__toStringc| Cs'|durtg}n9t|tr=|dks|dkrtjjS|dkr7tdd }|WdS1s1wYn t|}nt |}d}zc|d7}|d 7}|d 7}t|vsYt |vrg|d t t j d 7}t|vsot|vr{|d t td7}t}t|vst|vr|dt |d7}td|D]}|j|}t|}d}t|vst|vr|d|j7}t|vst|vr|dt t|d7}ztt|} Wnty} z t| } WYd} ~ nd} ~ ww|d| d7}t|vst|vr'z t|dkrdnd} Wnty} z t| } WYd} ~ nd} ~ ww|d| d7}t|vs1t |vr_z t!|dkr;dnd} WntyV} z t| } WYd} ~ nd} ~ ww|d| d7}t|vsit"|vrz t#|dkrsdnd} Wnty} z t| } WYd} ~ nd} ~ ww|d| d7}t|vst$|vrz t%|dkrdnd} Wnty} z t| } WYd} ~ nd} ~ ww|d| d 7}t|vst&|vrz t t'|} Wnty} z t| } WYd} ~ nd} ~ ww|d!| d"7}d}d#}t|vst(|vst)|vr]zt*|\}}|t+kr$dnd}|t+kr-dnd}WntyJ} z t| }|}WYd} ~ nd} ~ ww|d$|d%7}|d&|d'7}d(}|rl|d)7}||7}|d*7}d}d#}t|vszt,|vrz t-|t.krd+nd,}Wnty} z t| }WYd} ~ nd} ~ ww|d-|d.7}d(}t|vst/|vrz t0|t.krd+nd,}Wn"ty} zt| }|d/|d07}d(}WYd} ~ nd} ~ ww|r|d17}||7}|d27}t|vst1|vr'zt2|}Wnty} z t| }WYd} ~ nd} ~ ww|d3t |d47}t|vs1t3|vr[zt4|}WntyO} z t| }WYd} ~ nd} ~ ww|d5t |d67}t|vset5|vrzt6|}Wnty} z t| }WYd} ~ nd} ~ ww|d7t |d87}t|vst7|vrzt8|}Wnty} z t| }WYd} ~ nd} ~ ww|d9t |d:7}t|vst7|vrzt9|}Wnty} z t| }WYd} ~ nd} ~ ww|d;kr|d<7}n |r|d=7}n|d>7}t|vs t:|vr@zt;|}Wnty*} z t| }WYd} ~ nd} ~ wwzd?|}Wn|}Y|d@|dA7}d}d#}t|vsNt<|vrzzt=|}Wntyl} z t| }WYd} ~ nd} ~ ww|dBt |dC7}d(}t|vst>|vrzt?|t@}Wnty} z t| }WYd} ~ nd} ~ ww|dDt |dE7}d(}t|vstA|vrzt?|tB}Wnty} z t| }WYd} ~ nd} ~ ww|dFt |dG7}d(}t|vstC|vrzt?|tD} Wnty} z t| } WYd} ~ nd} ~ ww|dHt | dI7}d(}|r.|dJ7}||7}|dK7}d}!d#}"t|vs|>};|>}<|>}=WYd} ~ nd} ~ wwt|v sts|v r|8d|;d7}8t|v stt|v r|8d|<d7}8t|v stu|v r|8d|=d7}8|9 r|d7}||87}|d7}t|v sty|v rSz1tz|}:t |:j{ddd};t |:j|ddd}|>};|>}<|>}=WYd} ~ nd} ~ ww|d7}|d|;d7}|d|<d7}|d|=d7}|d7}t|v s]t}|v rz(t~|} | tk rjd}?n| tk rrd}?n| tk rzd}?n | tk rd}?nd}?Wnt y} z t| }?WYd} ~ nd} ~ ww|d|?d7}d}@d#}At|v st|v st|v rzt|}Bt |Bjd{}Ct |Bjd{}DWnt y} zt| }>|>}C|>}DWYd} ~ nd} ~ wwt|v st|v r|@d|Cd7}@t|v st|v r|@d|Dd7}@d(}At|v st|v rPzt|\}E}Ft |Ed{}GWnt yE} z t| }>|>}GWYd} ~ nd} ~ ww|@d|Gd7}@d(}At|v sZt|v rzt|\}E}Ft |Ed{}HWnt y} z t| }>|>}HWYd} ~ nd} ~ ww|@d|Hd7}@d(}A|A r|d7}||@7}|d7}t|v st|v st|v rzt|\}}|dk rdnd}I|dk rdnd}JWnt y} zt| }>|>}I|>}JWYd} ~ nd} ~ wwd}Kt|v st|v r|Kd|Id7}Kt|v st|v r |Kd|Jd7}K|d7}||K7}|d7}t||\}L}M|M r,|d7}||L7}|d7}t||\}N}O|O rC|d7}||N7}|d7}d}Pd#}Qt|v sQt|v rz t t|td}RWnt yu} z t| }RWYd} ~ nd} ~ ww|Pd|Rd7}Pz t t|td}RWnt y} z t| }RWYd} ~ nd} ~ ww|Pd|Rd7}Pd(}Qz t t|td}RWnt y} z t| }RWYd} ~ nd} ~ ww|Pd|Rd7}Pd(}Q|Q r|d7}||P|d7}d}Sd#}Tt|v st|v r+zt|}U|Udk rdnd;}VWnt y } z t| }VWYd} ~ nd} ~ ww|Sd|Vd7}Sd(}Tt|v s5t|v rdz t|d}Wd|W}XWnt yY} z t| }XWYd} ~ nd} ~ ww|Sd|Xd7}Sd(}Tt|v snt|v rz t|d}Yd|Y}ZWnt y} z t| }ZWYd} ~ nd} ~ ww|Sd|Zd7}Sd(}Tt|v st|v rz t|d}Yd|Y}ZWnt y} z t| }ZWYd} ~ nd} ~ ww|Sd|Zd7}Sd(}Tt|v st|vrz t|d}[d|[}\Wnty} z t| }\WYd} ~ nd} ~ ww|Sd|\d7}Sd(}Tt|vst|vst|vrwzt|}Yd|Ydd}]d|Ydd}^WntyP} zt| }>|>}]|>}^WYd} ~ nd} ~ wwt|vs[t|vrc|Sd|]d7}St|vsmt|vru|Sd|^d7}Sd(}T|Tr|d7}z dt tg|}6Wnty} z t| }6WYd} ~ nd} ~ ww|d|67}||S7}|d7}d}_d#}`t|vst|vrz t t|td}aWnty} z t| }aWYd} ~ nd} ~ ww|_d|ad7}_d(}`t|vst|vr&z t t|td}bWnty} z t| }bWYd} ~ nd} ~ ww|_d|bd7}_d(}`t|vs0t|vr_z t t|td}cWntyT} z t| }cWYd} ~ nd} ~ ww|_d|cd7}_d(}`|`rn|d7}||_7}|d7}d}dd#}et|vs|t|vrz t t|td}aWnty} z t| }aWYd} ~ nd} ~ ww|dd|ad7}dd(}et|vst|vrz t t|td}cWnty} z t| }cWYd} ~ nd} ~ ww|dd|cd7}dd(}e|er|d7}||d7}|d7}d}fd#}gt|vst|vr0z t t|td}aWnty%} z t| }aWYd} ~ nd} ~ ww|fd|ad7}fd(}gt|vs:t|vriz t t|td}cWnty^} z t| }cWYd} ~ nd} ~ ww|fd|cd7}fd(}g|grx|d7}||f7}|d7}d}hd#}it|vst|vrz t t|td}aWnty} z t| }aWYd} ~ nd} ~ ww|hd|ad7}hd(}it|vst|vrz t t|td}bWnty} z t| }bWYd} ~ nd} ~ ww|hd|bd7}hd(}it|vst|vr'z t t|td}cWnty} z t| }cWYd} ~ nd} ~ ww|hd|cd7}hd(}i|ir6|d7}||h7}|d7}t|vs@t|vr|d7}zt|\}j}k|jtkrSd}lnd}l|ktkr]d}mnd}mWn*tynd;}ld;}mYnty} zt| }lt| }mWYd} ~ nd} ~ ww|d|ld7}|d|md7}|d7}t|vst|vrzRt|}n|d7}|nD]A}o|d7}|d|o7}zt||o}_|_D] }p|d|p7}qWnty} z|dt| 7}WYd} ~ nd} ~ ww|d7}q|d7}Wnty} z|dt| d7}WYd} ~ nd} ~ wwt|vs$t|vrzut|}q|d7}|qD]d}rz t t|rj}sWn%ty`} z| jtkrQWYd} ~ q/t| }sWYd} ~ nd} ~ ww|d7}|d|rj7}|d|sd7}|rjdkr}d;}cn d|rjdd}c|d|cd7}|d7}q/|d7}Wnty} z|dt| d7}WYd} ~ nd} ~ wwt|vst|vrzt|}t|d7}|tD]}uz-t||u}vd|vj}wd|vj}x|vjdkrd;}yn d|vjdd}yd|vj}zd|vj}{Wn/ty+} z"| jtkrWYd} ~ qt| } | }w| }x| }y| }z| }{WYd} ~ nd} ~ ww|d7}|d|u7}|d|wd7}|d|xd7}|d|yd7}|d|zd7}|d|{d7}|d7}q|d7}Wn ty} z|dt| d7}WYd} ~ nd} ~ wwt|dkr|d7}||7}|d7}q|d7}W|Sty} z|d| ˡd7}WYd} ~ |Sd} ~ ww(a Provides a Python interface to GPU management and monitoring functions. This is a wrapper around the NVML library. For information about the NVML library, see the NVML developer page http://developer.nvidia.com/nvidia-management-library-nvml Examples: --------------------------------------------------------------------------- For all elements as in XML format. Similiar to nvisia-smi -q -x $ XmlDeviceQuery() --------------------------------------------------------------------------- For XML of filtered elements by string name. Similiar ot nvidia-smi --query-gpu=pci.bus_id,memory.total,memory.free See help_query_gpu.txt or XmlDeviceQuery("--help-query-gpu") for available filter elements $ XmlDeviceQuery("pci.bus_id,memory.total,memory.free") --------------------------------------------------------------------------- For XML of filtered elements by enumeration value. See help_query_gpu.txt or XmlDeviceQuery("--help_query_gpu") for available filter elements $ XmlDeviceQuery([NVSMI_PCI_BUS_ID, NVSMI_MEMORY_TOTAL, NVSMI_MEMORY_FREE]) N--help-h--help-query-gpuhelp_query_gpu.txtrrz z7 z z z z z z z rz %s z z z z EnabledDisabledz z z z z z z z z! z Fz z z z Tz z WDDMTCCz z z z z z z z z z z z z z roz) N/A z) Yes z( No 0x%xz z z z z z z z z z z z z z z z z z z %02X z$ %02X z$ %04X z* %08X z z z2 %08X z z z z z z xz z z z z z z z PLXBR04z z %08Xz, %s z z z z z zz z KB/sz z z z z z %z z P%sz z z MiBz rz z z z z z z z DefaultExclusive Thread ProhibitedExclusive_Processrz z z z z z z z z z z z z z z z z z z z z z z Cz z z z z z z z Supportedz z @@z%.2f Wz z z z z z z z rz z z z z Pz$ %s z z MHzz z z z z z z z z z z" z# z z z OffOnz z z z z z z z %d MHz zD %d MHz z@ %s z z z z z z z %d z z %d MiBz z z z z z z %d %%%d ms%dz z z z z z z z z z z z z nvidia_smi.py:  )rrrJrXmlDeviceQuery__doc__openread"_nvidia_smi__fromDeviceQueryStringlistNVSMI_TIMESTAMPrdatetimedatetodayNVSMI_DRIVER_VERSIONnvmlSystemGetDriverVersionr NVSMI_COUNTrrnvmlDeviceGetPciInfoNVSMI_PCI_BUS_IDbusId NVSMI_NAMEnvmlDeviceGetNameNVSMI_BRAND_NAMESnvmlDeviceGetBrandrrNVSMI_DISPLAY_MODEnvmlDeviceGetDisplayModeNVSMI_DISPLAY_ACTIVEnvmlDeviceGetDisplayActiveNVSMI_PERSISTENCE_MODEnvmlDeviceGetPersistenceModeNVSMI_ACCT_MODEnvmlDeviceGetAccountingModeNVSMI_ACCT_BUFFER_SIZE!nvmlDeviceGetAccountingBufferSizeNVSMI_MIG_MODE_CURRENTNVSMI_MIG_MODE_PENDINGnvmlDeviceGetMigModeNVML_DEVICE_MIG_ENABLENVSMI_DRIVER_MODEL_CURnvmlDeviceGetCurrentDriverModelNVML_DRIVER_WDDMNVSMI_DRIVER_MODEL_PENDINGnvmlDeviceGetPendingDriverModelNVSMI_SERIALNUMBERnvmlDeviceGetSerial NVSMI_UUIDnvmlDeviceGetUUID NVSMI_INDEXnvmlDeviceGetMinorNumberNVSMI_VBIOS_VERnvmlDeviceGetVbiosVersionnvmlDeviceGetMultiGpuBoardNVSMI_BOARD_IDnvmlDeviceGetBoardIdNVSMI_INFOROM_IMG nvmlDeviceGetInforomImageVersionNVSMI_INFOROM_OEMnvmlDeviceGetInforomVersionNVML_INFOROM_OEMNVSMI_INFOROM_ECCNVML_INFOROM_ECCNVSMI_INFOROM_PWRNVML_INFOROM_POWER NVSMI_GOM_CUR_nvidia_smi__toStrGOM$nvmlDeviceGetCurrentGpuOperationModeNVSMI_GOM_PENDING$nvmlDeviceGetPendingGpuOperationMode NVSMI_PCI_BUSbusNVSMI_PCI_DEVICEdeviceNVSMI_PCI_DOMAINdomainNVSMI_PCI_DEVICE_ID pciDeviceIdNVSMI_PCI_SUBDEVICE_IDpciSubSystemIdNVSMI_PCI_LINK_GEN_MAX"nvmlDeviceGetMaxPcieLinkGenerationNVSMI_PCI_LINK_GEN_CUR#nvmlDeviceGetCurrPcieLinkGenerationNVSMI_PCI_LINK_WIDTH_MAXnvmlDeviceGetMaxPcieLinkWidthNVSMI_PCI_LINK_WIDTH_CURnvmlDeviceGetCurrPcieLinkWidthnvmlDeviceGetBridgeChipInfobridgeChipInfotype fwVersionnvmlDeviceGetPcieReplayCounternvmlDeviceGetPcieThroughputNVML_PCIE_UTIL_TX_BYTESNVML_PCIE_UTIL_RX_BYTESNVSMI_FAN_SPEEDnvmlDeviceGetFanSpeed NVSMI_PSTATEnvmlDeviceGetPowerState&NVSMI_CLOCK_THROTTLE_REASONS_SUPPORTED#NVSMI_CLOCK_THROTTLE_REASONS_ACTIVE!NVSMI_CLOCK_THROTTLE_REASONS_IDLE(NVSMI_CLOCK_THROTTLE_REASONS_APP_SETTING'NVSMI_CLOCK_THROTTLE_REASONS_SW_PWR_CAP(NVSMI_CLOCK_THROTTLE_REASONS_HW_SLOWDOWN0NVSMI_CLOCK_THROTTLE_REASONS_HW_THERMAL_SLOWDOWN2NVSMI_CLOCK_THROTTLE_REASONS_HW_PWR_BRAKE_SLOWDOWN0NVSMI_CLOCK_THROTTLE_REASONS_SW_THERMAL_SLOWDOWN'NVSMI_CLOCK_THROTTLE_REASONS_SYNC_BOOST(_nvidia_smi__xmlGetClocksThrottleReasonsNVSMI_MEMORY_TOTALNVSMI_MEMORY_USEDNVSMI_MEMORY_FREEnvmlDeviceGetMemoryInforusedNVSMI_MEMORY_BAR1nvmlDeviceGetBAR1MemoryInfo bar1Totalbar1UsedNVSMI_COMPUTE_MODEnvmlDeviceGetComputeModeNVML_COMPUTEMODE_DEFAULT!NVML_COMPUTEMODE_EXCLUSIVE_THREADNVML_COMPUTEMODE_PROHIBITED"NVML_COMPUTEMODE_EXCLUSIVE_PROCESSNVSMI_UTILIZATION_GPUNVSMI_UTILIZATION_MEMnvmlDeviceGetUtilizationRatesgpumemoryNVSMI_UTILIZATION_ENCODERnvmlDeviceGetEncoderUtilizationNVSMI_UTILIZATION_DECODERnvmlDeviceGetDecoderUtilizationNVSMI_ECC_MODE_CURNVSMI_ECC_MODE_PENDINGnvmlDeviceGetEccMode_nvidia_smi__xmlGetEcc_nvidia_smi__xmlGetRetiredPagesNVSMI_TEMPERATURE_GPUnvmlDeviceGetTemperatureNVML_TEMPERATURE_GPU!nvmlDeviceGetTemperatureThreshold#NVML_TEMPERATURE_THRESHOLD_SHUTDOWN#NVML_TEMPERATURE_THRESHOLD_SLOWDOWNNVSMI_POWER_MGMT nvmlDeviceGetPowerManagementModeNVSMI_POWER_DRAWnvmlDeviceGetPowerUsageNVSMI_POWER_LIMIT!nvmlDeviceGetPowerManagementLimitNVSMI_POWER_LIMIT_DEFAULT(nvmlDeviceGetPowerManagementDefaultLimitNVSMI_POWER_LIMIT_ENFORCEDnvmlDeviceGetEnforcedPowerLimitNVSMI_POWER_LIMIT_MINNVSMI_POWER_LIMIT_MAX,nvmlDeviceGetPowerManagementLimitConstraintsNVSMI_CLOCKS_GRAPHICS_CURnvmlDeviceGetClockInfoNVML_CLOCK_GRAPHICS NVML_CLOCK_SMNVSMI_CLOCKS_MEMORY_CURNVML_CLOCK_MEMNVSMI_CLOCKS_APPL_GRAPHICSnvmlDeviceGetApplicationsClockNVSMI_CLOCKS_APPL_MEMORY"NVSMI_CLOCKS_APPL_GRAPHICS_DEFAULT%nvmlDeviceGetDefaultApplicationsClock NVSMI_CLOCKS_APPL_MEMORY_DEFAULTNVSMI_CLOCKS_GRAPHICS_MAXnvmlDeviceGetMaxClockInfoNVSMI_CLOCKS_SM_MAXNVSMI_CLOCKS_MEMORY_MAXNVSMI_CLOCKS_POLICY%nvmlDeviceGetAutoBoostedClocksEnabledrWNVMLError_NotSupportedNVSMI_CLOCKS_SUPPORTED"nvmlDeviceGetSupportedMemoryClocks$nvmlDeviceGetSupportedGraphicsClocksNVSMI_COMPUTE_APPS$nvmlDeviceGetComputeRunningProcessesnvmlSystemGetProcessNamepidrNVML_ERROR_NOT_FOUND usedGpuMemoryNVSMI_ACCOUNTED_APPSnvmlDeviceGetAccountingPidsnvmlDeviceGetAccountingStatsgpuUtilizationmemoryUtilizationmaxMemoryUsager isRunningr3r)|rrfinrrrrpciInfogpuInfo brandNamerstatere bufferSizemigModeincludeMigModecurrentpending driverModelincludeDriverModelrr minor_numbervbios multiGpuBoolboardIdhexBIDinforomVersionincludeInforomimgoemr5pwrgpuOperationMode includeGOMpci includePcipciGpuLinkInfoincludeLinkInfopciGen includeGengen pciLinkWidthsincludeLinkWidthswidth pciBridgeChipincludeBridgeChipbridgeHierarchy bridge_type strFwVersionreplaytx_bytesrx_bytesfan perfState perfStateStr fbMemoryUsageincludeMemoryUsagememInfo mem_totalmem_usedmem_freerBmodeStr utilizationincludeUtilizationutilgpu_utilmem_utilutil_intssize encoder_util decoder_utilcurr_strpend_streccMode eccErrorsincludeEccErrorsrYrZ temperatureincludeTemperaturetemppower_readingsincludePowerReadingspowMan powManStrpowDraw powDrawStrpowLimit powLimitStrenforcedPowLimitenforcedPowLimitStrpowLimitStrMinpowLimitStrMaxr includeClocksgraphicssmmemapplicationClocksincludeAppClocksdefaultApplicationClocksincludeDefaultAppClocks maxClocksincludeMaxClocks boostedStateboostedDefaultState autoBoostStrautoBoostDefaultStr memClocksmcprocsrrpidsrYstatsr_r`rar is_runningrrrrs  "                     " "  "  "                * *                                         "    "          &   "znvidia_smi.XmlDeviceQueryc Cs"|durtg}n9t|tr=|dks|dkrtjjS|dkr7tdd }|WdS1s1wYn t|}nt |}i}g}z t|vsOt |vrYt t j |d<t|vsat|vrit t|d<t}t|vstt|vrx||d <td |D]}i}|j|}t|} t|vst|vrt | j|d <t|vst|vrt t||d <ztt|} Wnty} z t| } WYd} ~ nd} ~ ww| |d <t|vst|vrz t|d krdnd} Wnty} z t| } WYd} ~ nd} ~ ww| |d<t|vs t |vr5z t!|d krdnd} Wnty0} z t| } WYd} ~ nd} ~ ww| |d<t|vs?t"|vriz t#|d krIdnd} Wntyd} z t| } WYd} ~ nd} ~ ww| |d<i}d}t|vs|t$|vs|t%|vrzt&|\}}Wnty} z t| }|}WYd} ~ nd} ~ ww|t'krdnd|d<|t'krdnd|d<d}|r||d<t|vst(|vrz t)|d krdnd} Wnty} z t| } WYd} ~ nd} ~ ww| |d<t|vst*|vr"z t t+|}Wnty} z t| }WYd} ~ nd} ~ ww||d<i}d}t|vs0t,|vr\z t-|t.kr:dnd}WntyU} z t| }WYd} ~ nd} ~ ww||d<d}t|vsft/|vrz t0|t.krpdnd}Wnty} z t| }WYd} ~ nd} ~ ww||d<d}|r||d<t|vst1|vrzt2|}Wnty} z t| }WYd} ~ nd} ~ wwt ||d<t|vst3|vrzt4|}Wnty} z t| }WYd} ~ nd} ~ wwt ||d <t|vst5|vr)zt6|}Wnty!} z t| }WYd} ~ nd} ~ wwt ||d!<t|vs3t7|vrYzt8|}WntyQ} z t| }WYd} ~ nd} ~ wwt ||d"<t|vsct7|vrzt9|}Wnty} z t| }WYd} ~ nd} ~ ww|d#krd#|d$<n |rd%|d$<nd&|d$<t|vst:|vrzt;|}Wnty} z t| }WYd} ~ nd} ~ wwzd'|}Wn|}Y||d(<i}d}t|vst<|vrzt=|}Wnty} z t| }WYd} ~ nd} ~ wwt ||d)<d}t|vst>|vr;zt?|t@}Wnty1} z t| }WYd} ~ nd} ~ wwt ||d*<d}t|vsEtA|vrnzt?|tB} Wntyd} z t| } WYd} ~ nd} ~ wwt | |d+<d}t|vsxtC|vrzt?|tD}!Wnty} z t| }!WYd} ~ nd} ~ wwt |!|d,<d}|r||d-<i}"d}#t|vstE|vrz tFtG|}Wnty} z t| }WYd} ~ nd} ~ wwt ||"d.<d}#t|vstH|vrz tFtI|}Wnty } z t| }WYd} ~ nd} ~ wwt ||"d/<d}#|#r|"|d0<i}$d}%t|vs+tJ|vr4d1| jK|$d2<d}%t|vs>tL|vrGd1| jM|$d3<d}%t|vsQtN|vrZd4| jO|$d5<d}%t|vsdtP|vrmd6| jQ|$d7<d}%t|vswt|vrt | j|$d8<d}%t|vstR|vrd6| jS|$d9<d}%i}&d}'i}(d})t|vstT|vrz t tU|}*Wnty} z t| }*WYd} ~ nd} ~ ww|*|(d:<d})t|vstV|vrz t tW|}*Wnty} z t| }*WYd} ~ nd} ~ ww|*|(d;<d})|)r |(|&d<<d}'i}+d},t|vstX|vrAz t tY|d=}-Wnty:} z t| }-WYd} ~ nd} ~ ww|-|+d><d},t|vsKtZ|vruz t t[|d=}-Wntyn} z t| }-WYd} ~ nd} ~ ww|-|+d?<d},|,r~|+|&d@<d}'|'r|&|$dA<d}%i}.d}/t|vrz9t\|}0dB}1|0j]d j^d kr|1dC7}1n|1dD7}1|1|.dE<|0j]d j_d krd#}2nd6|0j]d j_}2t |2|.dF<Wn!ty} zt| |.dE<t| |.dF<WYd} ~ nd} ~ wwd}/|/r|.|$dG<d}%t|vr%z t`|}3t |3|$dH<Wnty"} z t| |$dH<WYd} ~ nd} ~ wwd}%t|vrVzta|tb}4|4|$dI<dJ|$dK<WntyS} z t| |$dI<WYd} ~ nd} ~ wwd}%t|vrzta|tc}5|5|$dL<dJ|$dM<Wnty} z t| |$dL<WYd} ~ nd} ~ wwd}%|%r|$|dN<t|vstd|vrzte|}6Wnty} z t| }6WYd} ~ nd} ~ ww|6|dO<dP|dQ<t|vstf|vrz tg|}7dR|7}8Wnty} z t| }8WYd} ~ nd} ~ ww|8|dS<t|vs'th|vs'ti|vs'tj|vs'tk|vs'tl|vs'tm|vs'tn|vs'to|vs'tp|vs'tq|vr.tr||dT<i}9d}:t|vsFts|vsFtt|vsFtu|vrd}:z"tv|};|;jwdUdU}<|;jxdUdU}=|;jwdUdU|;jxdUdU}>Wnty} zt| }?|?}<|?}=|?}>WYd} ~ nd} ~ wwt|vsts|vr|<|9dV<t|vstt|vr|=|9dW<t|vstu|vr|>|9dX<|:rdY|9dZ<|9|d[<t|vsty|v rz"tz|};|;j{dUdU}<|;j|dUdU}=|;j{dUdU|;j|dUdU}>Wnt y} zt| }?|?}<|?}=|?}>WYd} ~ nd} ~ wwi}@|<|@dV<|=|@dW<|>|@dX<dY|@dZ<|@|d\<t|v s)t}|v rnz(t~|} | tk r6d]}An| tk r>d^}An| tk rFd_}An | tk rNd`}Anda}AWnt yi} z t| }AWYd} ~ nd} ~ ww|A|db<i}Bd}Ct|v st|v st|v rz t|}D|Dj}E|Dj}FWnt y} zt| }?|?}E|?}FWYd} ~ nd} ~ wwt|v st|v r|E|Bdc<t|v st|v r|F|Bdd<d}Ct|v st|v rz t|\}G}H|G}IWnt y} z t| }?|?}IWYd} ~ nd} ~ ww|I|Bde<d}Ct|v st|v r2z t|\}G}H|G}JWnt y+} z t| }?|?}JWYd} ~ nd} ~ ww|J|Bdf<d}C|C r=dP|BdZ<|B|dg<t|v sLt|v sLt|v rzt|\}}|d k rZdnd}K|d k rcdnd}LWnt y} zt| }?|?}K|?}LWYd} ~ nd} ~ wwi}Mt|v st|v r|K|Mdh<t|v st|v r|L|Mdi<|M|dj<t||\}N}O|O r|N|dk<t||\}P}Q|Q r|P|dl<i}Rd}St|v st|v rCzt|t}TWnt y} z t| }TWYd} ~ nd} ~ ww|T|Rdm<d}Szt|t}TWnt y} z t| }TWYd} ~ nd} ~ ww|T|Rdn<d}Szt|t}TWnt y<} z t| }TWYd} ~ nd} ~ ww|T|Rdo<d}S|S rNdp|RdZ<|R|dq<i}Ud}Vt|v s\t|v rzt|}W|Wd k rhdrnd#}XWnt y} z t| }XWYd} ~ nd} ~ ww|X|Uds<d}Vt|v st|v rz t|dt}Y|Y}ZWnt y} z t| }ZWYd} ~ nd} ~ ww|Z|Udu<d}Vt|v st|v rz t|dt}[|[}\Wnt y} z t| }\WYd} ~ nd} ~ ww|\|Udv<d}Vt|v st|v r#z t|dt}[|[}\Wnt y} z t| }\WYd} ~ nd} ~ ww|\|Udw<d}Vt|v s-t|v rVz t|dt}]|]}^Wnt yO} z t| }^WYd} ~ nd} ~ ww|^|Udx<d}Vt|v set|v set|v rzt|}[|[d dt}_|[dydt}`Wnt y} zt| }?|?}_|?}`WYd} ~ nd} ~ wwt|v st|v r|_|Udz<t|v st|v r|`|Ud{<d}V|V rz d|t tg|}7Wnt y} z t| }7WYd} ~ nd} ~ ww|7|Ud}<d~|UdZ<|U|d<i}ad}bt|v st|v rzt|t}cWnt y} z t| }cWYd} ~ nd} ~ ww|c|ad<d}bt|v s#t|v rIzt|t}dWnt yB} z t| }dWYd} ~ nd} ~ ww|d|ad<d}bt|v sSt|v ryzt|t}eWnt yr} z t| }eWYd} ~ nd} ~ ww|e|ad<d}b|b rd|adZ<|a|d<i}fd}gt|v st|v rzt|t}cWnt y} z t| }cWYd} ~ nd} ~ ww|c|fd<d}gt|v st|v rzt|t}eWnt y} z t| }eWYd} ~ nd} ~ ww|e|fd<d}g|g rd|fdZ<|f|d<i}hd}it|vst|vr'zt|t}cWnty } z t| }cWYd} ~ nd} ~ ww|c|hd<d}it|vs1t|vrWzt|t}eWntyP} z t| }eWYd} ~ nd} ~ ww|e|hd<d}i|irbd|hdZ<|h|d<i}jd}kt|vspt|vrzt|t}cWnty} z t| }cWYd} ~ nd} ~ ww|c|jd<d}kt|vst|vrzt|t}dWnty} z t| }dWYd} ~ nd} ~ ww|d|jd<d}kt|vst|vrzt|t}eWnty} z t| }eWYd} ~ nd} ~ ww|e|jd<d}k|krd|jdZ<|j|d<t|vs t|vr`i}lzt|\}m}n|mtkrd}ond}o|ntkr&d}pnd}pWn*ty7d#}od#}pYntyS} zt| }ot| }pWYd} ~ nd} ~ ww|o|ld<|p|ld<|l|d<t|vsjt|vrg}qzLt|}r|rD]C}si}t|s|td<d|tdZ<g}uzt||s}a|aD]}v|u|vqWnty} z t| }uWYd} ~ nd} ~ ww|u|td<|q|tqsWnty} z t| |qd<WYd} ~ nd} ~ wwt|qd kr|qnd|d<t|vst|vrxg}wzft|}x|xD]]}yz t t|yj}zWn%ty$} z| jtkrWYd} ~ qt| }zWYd} ~ nd} ~ wwi}{|yj|{d<|z|{d<|yjdkr9d }en t|yjdUdU}e|e|{d<dY|{dZ<|w|{qWntyj} z t| }wWYd} ~ nd} ~ wwt|wd krt|wnd|d<t|vst|vr6zt|}|g}}||D]}~z-t||~}d|j}d|j}|jdkrd#}n d|jdUdU}d|j}d|j}Wn/ty} z"| jtkrWYd} ~ qt| } | }| }| }| }| }WYd} ~ nd} ~ wwi}d|~|d<||dc<||dd<||d<||d<||d<|}|qt|}d kr|}nd|d<Wnty5} z t| |d<WYd} ~ nd} ~ wwt|d krB||q}t|d krQ||d<W|SW|Styq} ztd| ΡdWYd} ~ |Sd} ~ ww)a Provides a Python interface to GPU management and monitoring functions. This is a wrapper around the NVML library. For information about the NVML library, see the NVML developer page http://developer.nvidia.com/nvidia-management-library-nvml Examples: --------------------------------------------------------------------------- For all elements as a list of dictionaries. Similiar to nvisia-smi -q -x $ DeviceQuery() --------------------------------------------------------------------------- For a list of filtered dictionary elements by string name. Similiar ot nvidia-smi --query-gpu=pci.bus_id,memory.total,memory.free See help_query_gpu.txt or DeviceQuery("--help_query_gpu") for available filter elements $ DeviceQuery("pci.bus_id,memory.total,memory.free") --------------------------------------------------------------------------- For a list of filtered dictionary elements by enumeration value. See help_query_gpu.txt or DeviceQuery("--help-query-gpu") for available filter elements $ DeviceQuery([NVSMI_PCI_BUS_ID, NVSMI_MEMORY_TOTAL, NVSMI_MEMORY_FREE]) Nrrrrrr}r~rrid product_name product_brandrrrrrF current_mm pending_mmTmig_modeaccounting_modeaccounting_mode_buffer_sizerr current_dm pending_dm driver_modelrrrorromultigpu_boardrQrPrboard_id img_version oem_object ecc_object pwr_objectinforom_version current_gom pending_gomgpu_operation_modez%02Xpci_bus pci_devicez%04X pci_domainr pci_device_id pci_bus_idpci_sub_system_id max_link_gencurrent_link_genpcie_genrmax_link_widthcurrent_link_width link_widthspci_gpu_link_inforrrbridge_chip_typebridge_chip_fwpci_bridge_chipreplay_countertx_utilzKB/s tx_util_unitrx_util rx_util_unitr{ fan_speed%fan_speed_unitrperformance_stateclocks_throttlerrrfreeMiBunitfb_memory_usagebar1_memory_usagerrrrrrr memory_utilrrr current_ecc pending_eccecc_mode ecc_errors retired_pagesgpu_tempgpu_temp_max_thresholdgpu_temp_slow_thresholdCrrpower_managementr power_draw power_limitdefault_power_limitenforced_power_limitrmin_power_limitmax_power_limitr power_stateWrgraphics_clocksm_clock mem_clockMHzrapplications_clocksdefault_applications_clocks max_clocksrr auto_boostauto_boost_default clock_policyrksupported_graphics_clockrGsupported_clocksrY process_name used_memory processesrrrrmax_memory_usagerraccounted_processesr"rr)rrrJrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrr r r r r r%_nvidia_smi__GetClocksThrottleReasonsrrrrrrrrrrrrrrrrrr r!r"r#r$r%r&r'r(r)r*_nvidia_smi__GetEcc_nvidia_smi__GetRetiredPagesr-r.r/r0r1r2r3r4r5r6r7r8r9r:r;r<r=r>r?r@rArBrCrDrErFrGrHrIrJrKrLrMrNrOrPrQrWrRrSrTrUrr3rVrWrXrYrrZr[intr\r]r^r_r`rarrbprintr)rrrcnvidia_smi_results dictResultrr gpuResultsrrdrfrrgrerirjrkrlrhrmrnrrrorprqrrrsrtrurvrwr5rxryrzr{r|r}r~rrrrrrrrrrrrrrrrrrrrrrrrBbar1MemoryUsagerrrrrrrrrrrrrrrrYrZrrrrrrrrrrrrrrrrrrrrrrrrrr clockPolicyrrrrsupportedClocksrrsupportMemClocksupportedGraphicsClocksrrrrr processInforaccountProcessrYrr_r`rarraccountProcessInforrrrs  "                                                                                            znvidia_smi.DeviceQueryc Cs&d}zu|D]m\}}t|tjr6t|dkr,|d||f7}|||d|7}q|d||df7}qt|turit|dtjrit dt|D]}|d|||dt|f7}||||d|7}qKq|d||t |f7}qW|St y}z|d t |7}WYd}~|Sd}~ww) Nrrz%s%s:  %s%s: %s None%s%s: [%d of %d] rz [Error] ) itemsr collectionsMappingr3r_nvidia_smi__to_str_dictionaryrrrrJr)rrindent strResultskeyr|rerrr__to_str_dictionaryt s* znvidia_smi.__to_str_dictionaryc Csd}d}|D]:\}}t|tur7tdt|D]}|d|||dt|f7}||||d|7}qq|d||t|f7}q|S)Nrr1rr4rr2)r5rrrr3r8rJ)rrr:r9r;r|rrrr__to_str s znvidia_smi.__to_strcCst|tur|S||Sr)rrJ_nvidia_smi__to_str)rrrrrformat s  znvidia_smi.formatrr)"rrrrrrrrrrrrrrrr'r:r+r"rUr_r,r#rrr!rr classmethodrrr8r?r@rrrrrws.      G I 4 0 5 1          F : r__main__cCsvd}d}tdttjD]}tj|}|dkrd}q |}q t}|r-||}n||}t | |dS)NFrxmlT) rr3sysargvlowerrrrrr%r@)as_xmlquery_gpu_argsrvrrrrrmain s     rJ)r pynvml.nvmlrr6r threadingrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrr r$r& NVSMI_ENCODER_STATS_SESSIONCOUNTNVSMI_ENCODER_STATS_AVG_EPSNVSMI_ENCODER_STATS_AVG_LATENCYr(r)rrrrrr rrr r r r rrrrrrrrr r!r"r&r#r%r$r)r*r+r,r-r1r.r0r/rSrTrVrrrrr r r r r rr-NVSMI_TEMPERATURE_MEMORYr3r5r7r;r9r=r>r@NVSMI_CLOCKS_SM_CURrDNVSMI_CLOCKS_VIDEO_CURrFrHrIrKrLrNrOrVr\rPrSrrrNVML_BRAND_UNKNOWNNVML_BRAND_QUADRONVML_BRAND_TESLANVML_BRAND_NVSNVML_BRAND_GRIDNVML_BRAND_GEFORCENVML_BRAND_TITANNVML_BRAND_NVIDIA_VAPPSNVML_BRAND_NVIDIA_VPCNVML_BRAND_NVIDIA_VCSNVML_BRAND_NVIDIA_VWSNVML_BRAND_NVIDIA_VGAMINGNVML_BRAND_QUADRO_RTXNVML_BRAND_NVIDIA_RTXNVML_BRAND_NVIDIANVML_BRAND_GEFORCE_RTXNVML_BRAND_TITAN_RTXrrrrJrrrrs        !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~          <