
    bif                    8   d Z ddl ddlZddlZddlZddlmZ dZdZdZ	dZ
d	Zd
ZdZdZdZdZdZdZdZdZdZdZdZdZdZdZdZdZdZdZdZdZ dZ!d Z"d!Z#d"Z$d#Z%d$Z&d%Z'd&Z(d'Z)d(Z*d)Z+d*Z,d+Z-d,Z.d-Z/d.Z0d/Z1d0Z2d1Z3d2Z4d3Z5d4Z6d5Z7d6Z8d7Z9d8Z:d9Z;d:Z<d;Z=d<Z>d=Z?d>Z@d?ZAd@ZBdAZCdBZDdCZEdDZFdEZGdFZHdGZIdHZJdIZKdJZLdKZMdLZNdMZOdNZPdOZQdPZRdQZSdRZTdSZUdTZVdUZWdVZXdWZYdXZZdYZ[dZZ\d[Z]d\Z^d]Z_d^Z`d_Zad`ZbdaZcdbZddcZeddZfdeZgdfZhdgZidhZjdiZkdjZldkZmdlZndmZodnZpdoZqdpZrdqZsdrZti dsedte	due
dvedwedxedyedzed{ed|ed}ed~edededededei dededededededededededede de"de"de#de$de%i de%de&de&de'de'de(de)deSdeTdeUdeVdeWdeXdeYdeZde[de\i de*de,de+de-de/de0de3de4de5de6de7de8de9de:de;de<de=i de>de?de@deAdeBdeCdeDdeEdeFdeGdeHdeIdeJdeKdeLdeMdeNi deOdePdePdeQdeQdeRde]de^de_de`deadebdecdeddeedefdefi degdegdehdehdeideidejdejdekdekdeldeldemdemdendendeoeoepepeteqeresdZuevdewdexdeydezde{diZ| G d d      Z}e~dk(  rd Z e        yy)a  
nvidia_smi

Sample code that attempts to reproduce the output of nvidia-smi -q -x
For many cases the output should match.
Each query parameter is documented within nvdia-smi --help-query-gpu

From Code:
DeviceQuery()
DeviceQuery("--help")
DeviceQuery("--help_query_gpu")
DeviceQuery("pci.bus_id,memory.total,memory.free")
DeviceQuery([NVSMI_PCI_BUS_ID, NVSMI_MEMORY_TOTAL, NVSMI_MEMORY_FREE])

XmlDeviceQuery()
XmlDeviceQuery("--help")
XmlDeviceQuery("--help_query_gpu")
    )*N)Thread                        	   
                                                                         !   2   3   4   5   6   <   =   >   ?   i  i  i  F   G   P   Q   R   S   T   U   Z   [   \   ]   ^   _   d   e   f   g   h   i   n   o   p   q   r   s   x   y   z                                                                                                         	timestampdriver_versioncountnamegpu_nameserial
gpu_serialuuidgpu_uuidz
pci.bus_id
gpu_bus_idz
pci.domainzpci.busz
pci.devicezpci.device_idzpci.sub_device_idzpcie.link.gen.currentzpcie.link.gen.maxzpcie.link.width.currentzpcie.link.width.maxindexdisplay_modedisplay_activepersistence_modezaccounting.modezaccounting.buffer_sizezdriver_model.currentzdriver_model.pendingvbios_versionzinforom.imgzinforom.imagezinforom.oemzinforom.ecczinforom.pwrzinforom.powerzgom.currentzgpu_operation_mode.currentzgom.pendingzgpu_operation_mode.pendingz	fan.speedpstatez!clocks_throttle_reasons.supportedzclocks_throttle_reasons.activez clocks_throttle_reasons.gpu_idlez3clocks_throttle_reasons.applications_clocks_settingz$clocks_throttle_reasons.sw_power_capz#clocks_throttle_reasons.hw_slowdownz+clocks_throttle_reasons.hw_thermal_slowdownz/clocks_throttle_reasons.hw_power_brake_slowdownz+clocks_throttle_reasons.sw_thermal_slowdownz"clocks_throttle_reasons.sync_boostzmemory.totalzmemory.usedzmemory.freecompute_modezutilization.gpuzutilization.memoryzencoder.stats.sessionCountzencoder.stats.averageFpszencoder.stats.averageLatencyzecc.mode.currentzecc.mode.pendingz+ecc.errors.corrected.volatile.device_memoryz+ecc.errors.corrected.volatile.register_filez&ecc.errors.corrected.volatile.l1_cachez&ecc.errors.corrected.volatile.l2_cachez,ecc.errors.corrected.volatile.texture_memoryz#ecc.errors.corrected.volatile.totalz,ecc.errors.corrected.aggregate.device_memoryz,ecc.errors.corrected.aggregate.register_filez'ecc.errors.corrected.aggregate.l1_cachez'ecc.errors.corrected.aggregate.l2_cachez-ecc.errors.corrected.aggregate.texture_memoryz$ecc.errors.corrected.aggregate.totalz-ecc.errors.uncorrected.volatile.device_memoryz-ecc.errors.uncorrected.volatile.register_filez(ecc.errors.uncorrected.volatile.l1_cachez(ecc.errors.uncorrected.volatile.l2_cachez.ecc.errors.uncorrected.volatile.texture_memoryz%ecc.errors.uncorrected.volatile.totalz.ecc.errors.uncorrected.aggregate.device_memoryz.ecc.errors.uncorrected.aggregate.register_filez)ecc.errors.uncorrected.aggregate.l1_cachez)ecc.errors.uncorrected.aggregate.l2_cachez/ecc.errors.uncorrected.aggregate.texture_memoryz&ecc.errors.uncorrected.aggregate.totalz"retired_pages.single_bit_ecc.countzretired_pages.sbezretired_pages.double_bit.countzretired_pages.dbezretired_pages.pendingztemperature.gpuztemperature.memoryzpower.managementz
power.drawzpower.limitzenforced.power.limitzpower.default_limitzpower.min_limitzpower.max_limitzclocks.current.graphicsz	clocks.grzclocks.current.smz	clocks.smzclocks.current.memoryz
clocks.memzclocks.current.videozclocks.videozclocks.applications.graphicszclocks.applications.grzclocks.applications.memoryzclocks.applications.memz$clocks.default_applications.graphicszclocks.default_applications.grz"clocks.default_applications.memoryzclocks.default_applications.memzclocks.max.graphicszclocks.max.grclocks.max.sm)r   zclocks.max.memoryzclocks.max.memzsupported-clockszcompute-appszaccounted-appsclocksUnknownQuadroTeslaNVSGridGeForcec                      e Zd ZdZdZ G d d      Zed        Zedd       Zd Z	ed        Z
d Zed	        Zed
        Zed        Zed        Zed        Zed        Zed        Zed        Zed        Zed        Zed        Zed        Zed        Zed        Zed        Zed        Zedd       Zedd       Zd Zd Z d Z!y) 
nvidia_smiNc                   T    e Zd ZdZdZdZdZd
dZd Ze	d
d       Z
d Zd Zd Zd	 Zy)nvidia_smi.loop_asyncNFc                     d| _         || _        t        t        j                  j
                  | ||t        j                  j                  f      | _        | j                  j                          y )NF)targetargs)	_loop_async__abort_loop_async__callback_chainr   r   
loop_async_loop_async__loop_task_loop_async__callback_loop_async__taskstart)selftime_in_millisecondsfiltercallbacks       K/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/pynvml_utils/smi.py__init__znvidia_smi.loop_async.__init__Y  so    $,"*$

(=(=(I(ISWYmou  xB  xM  xM  xX  xX  SY  Z$+
++


    c                      d| _         d | _        y NT)r   r   r   s    r   __del__znvidia_smi.loop_async.__del__`  s    $,"&$
r   c                     |dz  }t         j                         }| j                         dk(  rM|j                  |      }|| _        |	 || |       t        j                  |       | j                         dk(  rLy y )Ni  F)r   getInstance
is_abortedDeviceQuery_loop_async__last_resultstimesleep)async_resultsr   r   r   delay_secondsnvsmiresultss          r   __loop_taskz!nvidia_smi.loop_async.__loop_taskd  sr    .5-((*%((*e3))&1g-4m*&='2jj' ((*e3r   c                 P    || _         | j                  | j                  | |       y y N)_loop_async__last_resultr   )r   results     r   
__callbackz nvidia_smi.loop_async.__callbackq  s+    %$
##/##D&1 0r   c                 `    d| _         | j                  | j                  j                          y y r   )r   r   joinr   s    r   cancelznvidia_smi.loop_async.cancelv  s(    $,kk%kk  &r   c                     | j                   S r   )r   r   s    r   r   z nvidia_smi.loop_async.is_aborted{  s    
r   c                     | j                   S r   )r   r   s    r   r   znvidia_smi.loop_async.result~  s    ##
#r   r   NN)__name__
__module____qualname__r   r   r   r   r   r   staticmethodr   r   r   r   r    r   r   r   r   S  sI    mfg' 
( 
(2
!
$r   r   c                  \    t         j                  dk(  r
t                t         j                  S )z Static access method. N)r   _nvidia_smi__instancer   r   r   r   znvidia_smi.getInstance  s#     
		$	&
,"""r   c                 0    t         j                  | ||      S r   )r   r   )r   r   r   s      r   loopznvidia_smi.loop  s    ""#7JJr   c                     t         j                  dk7  rt        d      | t         _        t         j                  dk(  rt         j	                         t         _        yy)z  Virtually private constructor. Nz,This class is a singleton, use getInstance())r   r   	Exception_nvidia_smi__handles_nvidia_smi__initialize_nvmlr   s    r   r   znvidia_smi.__init__  sH    			$	&GH	H!%					% * < < >	 
&r   c                  p    t                t               } i }t        d|       D ]  }t        |      ||<    |S )z Initialize NVML bindings. r   )nvmlInitnvmlDeviceGetCountrangenvmlDeviceGetHandleByIndex)deviceCounthandlesis      r   __initialize_nvmlznvidia_smi.__initialize_nvml  s?     J$&KG1k" 3/2
3Nr   c                     t         j                  d k7  r;t         j                  | k(  r't         `d t         _        d t         _        t                y y y r   )r   r   r   nvmlShutdownr   s    r   r   znvidia_smi.__del__  sA    %J,A,AT,I! $
#
	 -J%r   c                     | j                  d      }g }|D ]3  }|j                         }|t        v s|j                  t        |          5 |S )N,)splitstripNVSMI_QUERY_GPUappend)queryString
parametersvaluesppss        r   __fromDeviceQueryStringz"nvidia_smi.__fromDeviceQueryString  sQ    $$S)jf /!wwy"O#MM/"-./
 mr   c                 \   d}	 t        | ||t              }|dt        j                  |      z   dz   z  }	 t        | ||t              }|dt        j                  |      z   dz   z  }	 t        | ||t              }|dt        j                  |      z   dz   z  }	 t        | ||t              }|dt        j                  |      z   d	z   z  }	 t        | ||t              }	|d
t        j                  |	      z   dz   z  }	 t        j                  t        | ||            }
|d|
z   dz   z  }|S # t        $ r }t        j	                  |      }Y d }~4d }~ww xY w# t        $ r }t        j	                  |      }Y d }~/d }~ww xY w# t        $ r }t        j	                  |      }Y d }~*d }~ww xY w# t        $ r }t        j	                  |      }Y d }~%d }~ww xY w# t        $ r }t        j	                  |      }	Y d }~ d }~ww xY w# t        $ r }t        j	                  |      }
Y d }~d }~ww xY w)N z          <device_memory>z</device_memory>
z          <register_file>z</register_file>
z          <l1_cache>z</l1_cache>
z          <l2_cache>z</l2_cache>
z          <texture_memory>z</texture_memory>
z          <total>	</total>
)nvmlDeviceGetMemoryErrorCounter"NVML_MEMORY_LOCATION_DEVICE_MEMORY	NVMLErrorr   _nvidia_smi__handleError_nvidia_smi__toString"NVML_MEMORY_LOCATION_REGISTER_FILENVML_MEMORY_LOCATION_L1_CACHENVML_MEMORY_LOCATION_L2_CACHE#NVML_MEMORY_LOCATION_TEXTURE_MEMORYnvmlDeviceGetTotalEccErrors)handlecounterType	errorType	strResultdeviceMemoryerrregisterFilel1Cachel2CachetextureMemoryrr   s              r   __xmlGetEccByTypeznvidia_smi.__xmlGetEccByType  s6   i78K9[], .1F1F|1TTWkkki78K9[],
 .1F1F|1TTWkkki23FI{4QS' )J,A,A',JJ_\\i23FI{4QS' )J,A,A',JJ_\\i89&)[:]_- /*2G2G2VVYnnni0''(CFIWb(cd% &.==iO  7#11#6,7  7#11#6,7  2,,S1'2  2,,S1'2  8$2237-8  0**3/%0s   D& E &E> F* G 9 H &	E/E

E	E;E66E;>	F'F""F'*	G3GG	G?G::G?	H+H&&H+c                 P   d}i }	 t        | ||t              }||d<   	 t        | ||t
              }||d<   	 t        | ||t              }||d<   	 t        | ||t              }	|	|d<   	 t        | ||t              }
|
|d<   	 t        j                  t        | ||            }||d<   |S # t        $ r}t        j	                  |      }Y d }~d }~ww xY w# t        $ r}t        j	                  |      }Y d }~d }~ww xY w# t        $ r}t        j	                  |      }Y d }~d }~ww xY w# t        $ r}t        j	                  |      }	Y d }~d }~ww xY w# t        $ r}t        j	                  |      }
Y d }~d }~ww xY w# t        $ r }t        j	                  |      }Y d }~d }~ww xY w)Nr   device_memoryregister_filel1_cachel2_cachetexture_memorytotal)r   r   r   r   r   r   r   r   r   r   r   )r   r   r   r   	eccByTyper   r   r   r   r   r   rr   s               r   __GetEccByTypeznvidia_smi.__GetEccByType  s   ii78K9[],
 $0i 78K9[],
 $0i 23FI{4QS' &i
23FI{4QS' &i
89&)[:]_- %2i !0''(CFIWb(cd% !iQ  7#11#6,7  7#11#6,7  2,,S1'2  2,,S1'2  8$2237-8  0**3/%0s   B% C C; D& &E > E< %	C.CC	C8C33C8;	D#DD#&	E/E		E	E9E44E9<	F%F  F%c                 v   d}t         |v sXt        |v sPt        |v sHt        |v s@t        |v s8t
        |v s0t        |v s(t        |v s t        |v st        |v st        |v st        |v r;|dz  }|t        j                  t        j                  | |t                    z  }|dz  }t         |v sXt        |v sPt        |v sHt        |v s@t        |v s8t
        |v s0t        |v s(t        |v s t        |v st        |v st        |v st        |v r;|dz  }|t        j                  t        j                  | |t                     z  }|dz  }|S )Nr   z        <single_bit>
z        </single_bit>
z        <double_bit>
z        </double_bit>
)	NVSMI_ALL*NVSMI_ECC_ERROR_CORRECTED_VOLATILE_DEV_MEM*NVSMI_ECC_ERROR_CORRECTED_VOLATILE_REGFILE*NVSMI_ECC_ERROR_CORRECTED_VOLATILE_L1CACHE*NVSMI_ECC_ERROR_CORRECTED_VOLATILE_L2CACHE*NVSMI_ECC_ERROR_CORRECTED_VOLATILE_TEXTURE+NVSMI_ECC_ERROR_CORRECTED_AGGREGATE_DEV_MEM+NVSMI_ECC_ERROR_CORRECTED_AGGREGATE_REGFILE+NVSMI_ECC_ERROR_CORRECTED_AGGREGATE_L1CACHE+NVSMI_ECC_ERROR_CORRECTED_AGGREGATE_L2CACHE+NVSMI_ECC_ERROR_CORRECTED_AGGREGATE_TEXTURE)NVSMI_ECC_ERROR_CORRECTED_AGGREGATE_TOTALr   r   _nvidia_smi__xmlGetEccByType NVML_MEMORY_ERROR_TYPE_CORRECTED"NVML_MEMORY_ERROR_TYPE_UNCORRECTEDr   r   r   eccByCounters       r   __xmlGetEccByCounterznvidia_smi.__xmlGetEccByCounter  s_   l
v

4
>
4
>
4
>
4
>
4
>
5
?
5
?
5
?
5
?
5
?
3v
=
2
2,
*//
0L0LVU`  cC  1D  E  E,
3
3,
v

4
>
4
>
4
>
4
>
4
>
5
?
5
?
5
?
5
?
5
?
3v
=
2
2,
*//
0L0LVU`  cE  1F  G  G,
3
3,r   c                    i }t         |v sXt        |v sPt        |v sHt        |v s@t        |v s8t
        |v s0t        |v s(t        |v s t        |v st        |v st        |v st        |v rt        j                  | |t              |d<   t         |v sXt        |v sPt        |v sHt        |v s@t        |v s8t
        |v s0t        |v s(t        |v s t        |v st        |v st        |v st        |v rt        j                  | |t              |d<   |S )N
single_bit
double_bit)r   r   r  r  r  r  r  r  r  r  r	  r
  r   _nvidia_smi__GetEccByTyper  r  r  s       r   __GetEccByCounterznvidia_smi.__GetEccByCounter>  s   l
v

4
>
4
>
4
>
4
>
4
>
5
?
5
?
5
?
5
?
5
?
3v
='1'@'@Vv'w,|
$
v

4
>
4
>
4
>
4
>
4
>
5
?
5
?
5
?
5
?
5
?
3v
='1'@'@Vx'y,|
$r   c                    d}d}t         |v sXt        |v sPt        |v sHt        |v s@t        |v s8t
        |v s0t        |v s(t        |v s t        |v st        |v st        |v st        |v r=|dz  }|t        j                  t        j                  | t        |            z  }|dz  }d}t         |v s`t         |v sXt"        |v sPt$        |v sHt&        |v s@t(        |v s8t*        |v s0t,        |v s(t.        |v s t0        |v st2        |v st4        |v st6        |v r=|dz  }|t        j                  t        j                  | t8        |            z  }|dz  }d}t;        |      dkD  r||fS d |fS )	Nr   Fz      <volatile>
z      </volatile>
Tz      <aggregate>
z      </aggregate>
r   )r   r   r  r  r  r  ,NVSMI_ECC_ERROR_UNCORRECTED_VOLATILE_DEV_MEM,NVSMI_ECC_ERROR_UNCORRECTED_VOLATILE_REGFILE,NVSMI_ECC_ERROR_UNCORRECTED_VOLATILE_L1CACHE,NVSMI_ECC_ERROR_UNCORRECTED_VOLATILE_L2CACHE,NVSMI_ECC_ERROR_UNCORRECTED_VOLATILE_TEXTURE*NVSMI_ECC_ERROR_UNCORRECTED_VOLATILE_TOTALr   r   _nvidia_smi__xmlGetEccByCounterNVML_VOLATILE_ECCr  r  r  r  r	  r
  -NVSMI_ECC_ERROR_UNCORRECTED_AGGREGATE_DEV_MEM-NVSMI_ECC_ERROR_UNCORRECTED_AGGREGATE_REGFILE-NVSMI_ECC_ERROR_UNCORRECTED_AGGREGATE_L1CACHE-NVSMI_ECC_ERROR_UNCORRECTED_AGGREGATE_L2CACHE-NVSMI_ECC_ERROR_UNCORRECTED_AGGREGATE_TEXTURE+NVSMI_ECC_ERROR_UNCORRECTED_AGGREGATE_TOTALNVML_AGGREGATE_ECClenr   r   ecc
includeEccs       r   __xmlGetEccznvidia_smi.__xmlGetEcc`  su   cj
v

4
>
4
>
4
>
4
>
4
>
6&
@
6&
@
6&
@
6&
@
6&
@
4
>
%
%#
&&z'F'FvO`bh'ij
j#
&
&#*
v

5
?
5
?
5
?
5
?
5
?
3v
=
76
A
76
A
76
A
76
A
76
A
5
?
&
&#
&&z'F'FvOaci'jk
k#
'
'#*HqLSJ66dJ66r   c                 b   i }d}t         |v sXt        |v sPt        |v sHt        |v s@t        |v s8t
        |v s0t        |v s(t        |v s t        |v st        |v st        |v st        |v r t        j                  | t        |      |d<   d}t         |v s`t        |v sXt         |v sPt"        |v sHt$        |v s@t&        |v s8t(        |v s0t*        |v s(t,        |v s t.        |v st0        |v st2        |v st4        |v r t        j                  | t6        |      |d<   d}t9        |j;                               dkD  r||fS d |fS )NFvolatileT	aggregater   )r   r   r  r  r  r  r  r  r  r  r  r  r   _nvidia_smi__GetEccByCounterr  r  r  r  r  r	  r
  r  r   r!  r"  r#  r$  r%  r&  r   r'  s       r   __GetEccznvidia_smi.__GetEcc  s:   cj
v

4
>
4
>
4
>
4
>
4
>
6&
@
6&
@
6&
@
6&
@
6&
@
4
>&88ARTZ[#j/*
v

5
?
5
?
5
?
5
?
5
?
3v
=
76
A
76
A
76
A
76
A
76
A
5
?'99&BTV\]#k
*

%)SZ??tZ??r   c                 \   d}d }d}	 t        | |      }t        t        t        |            }|dt
        j                  |      z   dz   z  }| |dz  }|D ]  }|dd|z  z   dz   z  } |d	z  }n|d
|z   dz   z  }|dkD  r|S dS # t        $ r!}t
        j                  |      }d }Y d }~yd }~ww xY w)Nr   r   z        <retired_count>z</retired_count>
z!        <retired_page_addresses>
z           <retired_page_address>0x%016xz</retired_page_address>
z"        </retired_page_addresses>
z         <retired_page_addresses>z</retired_page_addresses>
)nvmlDeviceGetRetiredPagessummapr&  r   r   r   r   )r   causeretiredPagedByCauseerrorrr   pagesr   pages           r   __xmlGetRetiredPagesByCausez&nvidia_smi.__xmlGetRetiredPagesByCause  s    ee+FE:%c#uo&%
 69N9Nu9UUXlll		
!E
E
 yd!%G)VZJZ%Z]x%xx!y
!F
F

!Ce!KNk!k
k
$)AI 525  **3/%%s   %B 	B+
B&&B+c                 H   i }d }d}	 t        | |      }i }|'d}|D ]   }	d|	z  |dt	        |      z   <   |dz  }|dz  }" |||d<   ||d<   t        |j                               dkD  r|nd |d<   |dkD  r|S d S # t        $ r!}t        j                  |      }d }Y d }~d }~ww xY w)Nr   r   r1  retired_page_address_Errorretired_countretired_page_addresses)r2  r   r   r   strr&  r   )
r   r5  r6  r7  rr   r8  r   retiredPageAddressesiir9  s
             r   __GetRetiredPagesByCausez#nvidia_smi.__GetRetiredPagesByCause  s    ee+FE:%
  		" dFORVFV"#:3r7#BC!ebQhe 
	*/
w
'-2/*NQRfRmRmRoNpstNt6Jz~23$)AI 747#  **3/%%s   A7 7	B! BB!c                    d}d}ddg}t        t              D ]]  }t        |v st        |v r|dk(  st        |v s!|dk(  s'|d||   z   dz   z  }|t
        j                  | |      z  }|d	||   z   dz   z  }d
}_ t        |v st        |v r*|dz  }	 t        t        |       k(  r|dz  }n|dz  }|dz  }d
}t        |      dkD  r||fS d |fS # t        $ r"}|t
        j                  |      z  }Y d }~Cd }~ww xY w)Nr   Fmultiple_single_bit_retirementdouble_bit_retirementr   r         <>
z      </Tz      <pending_retirement>NoYesz</pending_retirement>
)r    NVML_PAGE_RETIREMENT_CAUSE_COUNTr   'NVSMI_RETIREDPAGES_SINGLE_BIT_ECC_COUNT'NVSMI_RETIREDPAGES_DOUBLE_BIT_ECC_COUNTr   &_nvidia_smi__xmlGetRetiredPagesByCauseNVSMI_RETIREDPAGES_PENDINGNVML_FEATURE_DISABLED&nvmlDeviceGetRetiredPagesPendingStatusr   r   r&  )r   r   retiredPagesincludeRetiredPagescausesidxr   s          r   __xmlGetRetiredPagesznvidia_smi.__xmlGetRetiredPages  s@   l!13JLf78 )#6!6&@SAX6&@SAXi&+5==ljDDVSQQlj6#;6>>l$(!) v
!;v!E
6
6,<&*PQW*XX$&,%', 3
3, $
 .2\>QQQ>QQQ  <j66s;;l<s   C 	C3C..C3c                    i }d}ddg}t        t              D ]D  }t        |v st        |v r|dk(  st        |v s!|dk(  s't
        j                  | |      |||   <   d}F t        |v st        |v r!d}	 t        t        |       k(  rd}nd	}||d
<   d}t        |j                               dkD  r||fS d |fS # t        $ r}t
        j                  |      }Y d }~Nd }~ww xY w)NFrE  rF  r   r   Tr   rI  rJ  pending_retirement)r   rK  r   rL  rM  r   #_nvidia_smi__GetRetiredPagesByCauserO  rP  rQ  r   r   r&  r   )r   r   rR  rS  rT  rU  rX  r   s           r   __GetRetiredPagesznvidia_smi.__GetRetiredPages  s   l!13JLf78 )#6!6&@SAX6&@SAX*4*M*MfVY*Zl6#;'$(!) v
!;v!E!
A&*PQW*XX'+$',$ 0B,+
, $
 !4!4!67!;\GZZZGZZZ  A#-#;#;C#@ As   2B4 4	C=CCc                 @    | t         k(  ry| t        k(  ry| t        k(  ryy)NzAll OnComputezLow Double Precisionr   )NVML_GOM_ALL_ONNVML_GOM_COMPUTENVML_GOM_LOW_DP)modes    r   
__toStrGOMznvidia_smi.__toStrGOM  s'    		 ##?"'r   c           	      h   t         dgt        dgt        dgt        dgt        dgt
        dgg}d}	 t        |       }t        |       }|dz  }|D ]+  \  }}|dk7  s||z  r
||z  rd	nd
}nd}|d|d|d|dz  }- |dz  }|S # t        $ r&}|dt        j                  |      z  z  }Y d }~|S d }~ww xY w)Nclocks_throttle_reason_gpu_idle*clocks_throttle_reason_user_defined_clocks2clocks_throttle_reason_applications_clocks_setting#clocks_throttle_reason_sw_power_cap"clocks_throttle_reason_hw_slowdownclocks_throttle_reason_unknownr   z    <clocks_throttle_reasons>
Active
Not ActiveN/ArG  >z</rH  z    </clocks_throttle_reasons>
z:    <clocks_throttle_reasons>%s</clocks_throttle_reasons>
)nvmlClocksThrottleReasonGpuIdle)nvmlClocksThrottleReasonUserDefinedClocks1nvmlClocksThrottleReasonApplicationsClocksSetting"nvmlClocksThrottleReasonSwPowerCap"nvmlClocksThrottleReasonHwSlowdownnvmlClocksThrottleReasonUnknown+nvmlDeviceGetSupportedClocksThrottleReasons)nvmlDeviceGetCurrentClocksThrottleReasonsr   r   r   )	r   throttleReasonsr   supportedClocksThrottleReasonsclocksThrottleReasonsmaskrs   valr   s	            r   __xmlGetClocksThrottleReasonsz(nvidia_smi.__xmlGetClocksThrottleReasons$  s    /:[\8:fg@Bvw1:_`1:^_.:Z[o ix+VW]+^
("KF"S

8
8)- JltTFF;;(,/D(DH,c!cc4HH)J 9
9)   x
TXbXpXpqtXuv
v)xs   (B 'B 	B1B,,B1c                 v   t         dgt        dgt        dgt        dgt        dgt
        dgg}i }	 t        |       }t        |       }|D ]!  \  }}|dk7  s||z  r
||z  rdnd}nd	}|||<   # 	 t        |j                               dkD  r|S d S # t        $ r"}t        j                  |      |d
<   Y d }~Fd }~ww xY w)Nrc  rd  re  rf  rg  rh  ri  rj  rk  r=  r   )rm  rn  ro  rp  rq  rr  rs  rt  r   r   r   r&  r   )	r   ru  clockThrottleReasonsrv  rw  rx  rs   ry  r   s	            r   __GetClocksThrottleReasonsz%nvidia_smi.__GetClocksThrottleReasonsB  s     /:[\8:fg@Bvw1:_`1:^_.:Z[o  I+VW]+^
("KF"S
- 3ltTFF;;(,/D(DH,c!c.1&t,3 &))=)D)D)F%G!%K!UQUU  I*4*B*B3*G
w
'Is   #B B 	B8B33B8c                 J    | j                   t        k(  ry| j                         S )Nrk  )valueNVML_ERROR_NOT_SUPPORTED__str__)r   s    r   __handleErrorznvidia_smi.__handleErrora  s    
))/
/
r   c                 Z    t        | t              r| j                  d      S t        |       S )Nzutf-8)
isinstancebytesdecoder@  )ry  s    r   
__toStringznvidia_smi.__toStringh  s$    
S%
 G$
$Xor   c                 $G   |t         g}nt        |t              re|dk(  s|dk(  rt        j                  j
                  S |dk(  r&t        dd      5 }|j                         cddd       S t        j                  |      }nt        |      }d}	 |dz  }|d	z  }|d
z  }t         |v st        |v r:|dt        j                  t        j                  j                               z   dz   z  }t         |v st        |v r&|dt        j                  t!                     z   dz   z  }t#               }t         |v st$        |v r|dt        j                  |      z   dz   z  }t'        d|      D ]  }| j(                  |   }t+        |      }d}t         |v st,        |v r|d|j.                  z  z  }t         |v st0        |v rE|dt        j                  t3        |            z   dz   z  }	 t4        t7        |         }	|d|	z   dz   z  }t         |v st<        |v r	 t?        |      dk7  rdnd}|d|z   dz   z  }t         |v st@        |v r	 tC        |      dk7  rdnd}|d|z   dz   z  }t         |v stD        |v r	 tG        |      dk7  rdnd}|d|z   dz   z  }t         |v stH        |v r	 tK        |      dk7  rdnd}|d|z   d z   z  }t         |v stL        |v r*	 t        j                  tO        |            }|d!|z   d"z   z  }d}d#}t         |v stP        |v r$	 tS        |      tT        k(  rd$nd%}|d&|z   d'z   z  }d(}t         |v stV        |v r	 tY        |      tT        k(  rd$nd%}|r|d+z  }||z  }|d,z  }t         |v stZ        |v r*	 t]        |      }|d-t        j                  |      z   d.z   z  }t         |v st^        |v r*	 ta        |      }|d/t        j                  |      z   d0z   z  }t         |v stb        |v r*	 te        |      }|d1t        j                  |      z   d2z   z  }t         |v stf        |v r*	 ti        |      }|d3t        j                  |      z   d4z   z  }t         |v stf        |v r$	 tk        |      }|d5k(  r|d6z  }n|r|d7z  }n|d8z  }t         |v stl        |v r	 to        |      }	 d9|z  }|d:|z   d;z   z  }d}d#}t         |v stp        |v r,	 ts        |      }|d<t        j                  |      z   d=z   z  }d(}t         |v stt        |v r1	 tw        |tx              }|d>t        j                  |      z   d?z   z  }d(}t         |v stz        |v r1	 tw        |t|              }|d@t        j                  |      z   dAz   z  }d(}t         |v st~        |v r1	 tw        |t              }|dBt        j                  |      z   dCz   z  }d(}|r|dDz  }||z  }|dEz  }d}d#} t         |v st~        |v r?	 t        j                  t        |            }|dFt        j                  |      z   dGz   z  }d(} t         |v st~        |v r?	 t        j                  t        |            }|dHt        j                  |      z   dIz   z  }d(} | r|dJz  }||z  }|dKz  }d}!d#}"t         |v st        |v r|!dL|j                  z  z  }!d(}"t         |v st        |v r|!dM|j                  z  z  }!d(}"t         |v st        |v r|!dN|j                  z  z  }!d(}"t         |v st        |v r|!dO|j                  z  z  }!d(}"t         |v st,        |v r*|!dPt        j                  |j.                        z   dQz   z  }!d(}"t         |v st        |v r|!dR|j                  z  z  }!d(}"d}#d#}$d}%d#}&t         |v st        |v r,	 t        j                  t        |            }'|%dS|'z   dTz   z  }%d(}&t         |v st        |v r,	 t        j                  t        |            }'|%dU|'z   dVz   z  }%d(}&|&r|#dWz  }#|#|%z  }#|#dXz  }#d(}$d}(d#})t         |v st        |v r/	 t        j                  t        |            dYz   }*|(dZ|*z   d[z   z  }(d(})t         |v st        |v r/	 t        j                  t        |            dYz   }*|(d\|*z   d]z   z  }(d(})|)r|#d^z  }#|#|(z  }#|#d_z  }#d(}$|$r|!d`z  }!|!|#z  }!|!daz  }!d}+d#},t         |v r	 t        |      }-d}.|-j                  d   j                  dk(  r|.dbz  }.n|.dcz  }.|+dd|.z   dez   z  }+|-j                  d   j                  dk(  rd5}/ndf|-j                  d   j                  z  }/|+dg|/z  z  }+d(},|,r|!djz  }!|!|+z  }!|!dkz  }!t         |v r,	 t        |      }0|!dlt        j                  |0      z   dmz   z  }!d(}"t         |v r4	 t        |t              }1|!dnt        j                  |1      z   doz   dpz   z  }!d(}"t         |v r4	 t        |t              }2|!dqt        j                  |2      z   doz   drz   z  }!d(}"|"r|dsz  }||!z  }|dtz  }t         |v st        |v r-	 t        j                  t        |            duz   }3|dv|3z   dwz   z  }t         |v st        |v r	 t        |      }4dx|4z  }5|dy|5z   dzz   z  }t         |v sPt        |v sHt        |v s@t        |v s8t        |v s0t        |v s(t        |v s t        |v st        |v st        |v st        |v r|t        j                  |      z  }d}6d#}7t         |v st        |v st        |v st        |v rd(}7	 t        |      }8t        j                  |8j                  d{z  d{z        d|z   }9t        j                  |8j                  d{z  d{z        d|z   }:t        j                  |8j                  d{z  d{z  |8j                  d{z  d{z  z
        d|z   };t         |v st        |v r|6d}|9z   d~z   z  }6t         |v st        |v r|6d|:z   dz   z  }6t         |v st        |v r|6d|;z   dz   z  }6|7r|dz  }||6z  }|dz  }t         |v st        |v r	 t        |      }8t        j                  |8j                  d{z  d{z        d|z   }9t        j                  |8j                  d{z  d{z        d|z   }:t        j                  |8j                  d{z  d{z  |8j                  d{z  d{z  z
        d|z   };|dz  }|d}|9z   d~z   z  }|d|:z   dz   z  }|d|;z   dz   z  }|dz  }t         |v st        |v rI	 t        |      }|t        k(  rd}=n&|t        k(  rd}=n|t        k(  rd}=n|t        k(  rd}=nd}=|d|=z   dz   z  }d}>d#}?t         |v st        |v st        |v r	 t        |      }@t        j                  |@j                         duz   }At        j                  |@j                        duz   }Bt         |v st        |v r|>dAz   dz   z  }>t         |v st        |v r|>dBz   dz   z  }>d(}?t         |v s	t        |v r5	 t        |      \  }C}Dt        j                  |C      duz   }E|>dEz   dz   z  }>d(}?t         |v s	t        |v r5	 t        |      \  }C}Dt        j                  |C      duz   }F|>dFz   dz   z  }>d(}?|?r|dz  }||>z  }|dz  }t         |v st        |v s	t        |v rk	 t        |      \  }}|dk7  rdnd}G|dk7  rdnd}Hd}It         |v s	t        |v rIdGz   dz   z  }It         |v s	t        |v rIdHz   dz   z  }I|dz  }|Iz  }|dz  }t        j                  ||      \  }J}K|Kr|dz  }|Jz  }|dz  }t        j                  ||      \  }L}M|Mr|dz  }|Lz  }|dz  }d}Nd#}Ot         |v s	t        |v r	 t        j                  t        |t                    dz   }PNdPz   dz   z  }N	 t        j                  t        |t                    dz   }PNdPz   dz   z  }Nd(}O	 t        j                  t        |t                     dz   }PNdPz   dz   z  }Nd(}OOr|dz  }|Nz    |dz  }d}Qd#}Rt         |v s	t"        |v r#	 t%        |      }S|Sdk7  rdnd5}TQdTz   dz   z  }Qd(}Rt         |v s	t&        |v r"	 t)        |      dz  }Ud|Uz  }VQdVz   dz   z  }Qd(}Rt         |v s	t*        |v r"	 t-        |      dz  }Wd|Wz  }XQdXz   dz   z  }Qd(}Rt         |v s	t.        |v r"	 t1        |      dz  }Wd|Wz  }XQdXz   dz   z  }Qd(}Rt         |v s	t2        |v r"	 t5        |      dz  }Yd|Yz  }ZQdZz   dz   z  }Qd(}Rt         |v st6        |v s	t8        |v r]	 t;        |      }Wd|Wd   dz  z  }[d|Wd   dz  z  }\t         |v s	t6        |v rQd[z   dz   z  }Qt         |v s	t8        |v rQd\z   dz   z  }Qd(}RRr9|dz  }	 dt        j                  t        |            z   }4|d|4z  z  }|Qz  }|dz  }d}]d#}^t         |v s	t<        |v r6	 t        j                  t?        |t@                    dz   }_]d_z   dz   z  }]d(}^t         |v s	t<        |v r6	 t        j                  t?        |tB                    dz   }`]d`z   dz   z  }]d(}^t         |v s	tD        |v r6	 t        j                  t?        |tF                    dz   }a]daz   dz   z  }]d(}^^r|dz  }|]z  }|dz  }d}bd#}ct         |v s	tH        |v r6	 t        j                  tK        |t@                    dz   }_bd_z   dz   z  }bd(}ct         |v s	tL        |v r6	 t        j                  tK        |tF                    dz   }abdaz   dz   z  }bd(}ccr|dz  }|bz  }|dz  }d}dd#}et         |v s	tN        |v r6	 t        j                  tQ        |t@                    dz   }_dd_z   dz   z  }dd(}et         |v s	tR        |v r6	 t        j                  tQ        |tF                    dz   }addaz   dz   z  }dd(}eer|dz  }|dz  }|dz  }d}fd#}gt         |v s	tT        |v r6	 t        j                  tW        |t@                    dz   }_fd_z   dz   z  }fd(}gt         |v s	tX        |v r6	 t        j                  tW        |tB                    dz   }`fd`z   dz   z  }fd(}gt         |v s	tZ        |v r6	 t        j                  tW        |tF                    dz   }afdaz   dz   z  }fd(}ggr|dz  }|fz  }|dz  }t         |v s	t\        |v rN|dz  }	 t_        |      \  }h}i|ht`        k(  rd}jnd}jit`        k(  rd}knd}k|djz   dz   z  }|dkz   dz   z  }|dz  }t         |v s	td        |v rN	 tg        |      }l|dz  }|lD ]2  }m|dz  }|d|mz  z  }	 ti        |m      }]|]D ]
  }n|d|nz  z  } 	 |dz  }4 |dz  }t         |v s	tj        |v r	 tm        |      }o|dz  }|oD ]  }p	 t        j                  to        pjp                              }q|dz  }|dpjp                  z  z  }|dqz   dz   z  }|pjv                  dk(  rd5}andpjv                  d{z  d{z  z  }a|daz   dz   z  }|dz  } |dz  }t         |v s	tx        |v r	 t{        |      }r|dz  }|rD ]  }s	 t}        |s      }td|tj~                  z  }ud|tj                  z  }v|tj                  dk(  rd5}wndtj                  d{z  d{z  z  }wdtj                  z  }xd|tj                  z  }y|dz  }|dsz  z  }|duz   dz   z  }|dvz   dz   z  }|dwz   dz   z  }|dxz   dz   z  }|dyz   dz   z  }|dz  } |dz  }t        |      dkD  s|dz  }||z  }|dz  } |dz  }|S # 1 sw Y   xY w# t8        $ r }
t        j;                  |
      }	Y d}
~
d}
~
ww xY w# t8        $ r }
t        j;                  |
      }Y d}
~
d}
~
ww xY w# t8        $ r }
t        j;                  |
      }Y d}
~
d}
~
ww xY w# t8        $ r }
t        j;                  |
      }Y d}
~
d}
~
ww xY w# t8        $ r }
t        j;                  |
      }Y d}
~
d}
~
ww xY w# t8        $ r }
t        j;                  |
      }Y d}
~
d}
~
ww xY w# t8        $ r }
t        j;                  |
      }Y d}
~
|d}
~
ww xY w# t8        $ r-}
t        j;                  |
      }|d)|z   d*z   z  }d(}Y d}
~
d}
~
ww xY w# t8        $ r }
t        j;                  |
      }Y d}
~
d}
~
ww xY w# t8        $ r }
t        j;                  |
      }Y d}
~
rd}
~
ww xY w# t8        $ r }
t        j;                  |
      }Y d}
~
dd}
~
ww xY w# t8        $ r }
t        j;                  |
      }Y d}
~
Vd}
~
ww xY w# t8        $ r }
t        j;                  |
      }Y d}
~
Hd}
~
ww xY w# t8        $ r }
t        j;                  |
      }Y d}
~
@d}
~
ww xY w#  |}Y IxY w# t8        $ r }
t        j;                  |
      }Y d}
~
Ed}
~
ww xY w# t8        $ r }
t        j;                  |
      }Y d}
~
0d}
~
ww xY w# t8        $ r }
t        j;                  |
      }Y d}
~
d}
~
ww xY w# t8        $ r }
t        j;                  |
      }Y d}
~
d}
~
ww xY w# t8        $ r }
t        j;                  |
      }Y d}
~
d}
~
ww xY w# t8        $ r }
t        j;                  |
      }Y d}
~
d}
~
ww xY w# t8        $ r }
t        j;                  |
      }'Y d}
~
}d}
~
ww xY w# t8        $ r }
t        j;                  |
      }'Y d}
~
md}
~
ww xY w# t8        $ r }
t        j;                  |
      }*Y d}
~
Cd}
~
ww xY w# t8        $ r }
t        j;                  |
      }*Y d}
~
0d}
~
ww xY w# t8        $ rG}
|+ddt        j;                  |
      z   dez   z  }+|+dht        j;                  |
      z   diz   z  }+Y d}
~
d}
~
ww xY w# t8        $ r)}
|!dlt        j;                  |
      z   dmz   z  }!Y d}
~
d}
~
ww xY w# t8        $ r)}
|!dnt        j;                  |
      z   dpz   z  }!Y d}
~
d}
~
ww xY w# t8        $ r)}
|!dqt        j;                  |
      z   drz   z  }!Y d}
~
d}
~
ww xY w# t8        $ r }
t        j;                  |
      }3Y d}
~
d}
~
ww xY w# t8        $ r }
t        j;                  |
      }5Y d}
~
d}
~
ww xY w# t8        $ r&}
t        j;                  |
      }<|<}9|<}:|<};Y d}
~
d}
~
ww xY w# t8        $ r&}
t        j;                  |
      }<|<}9|<}:|<};Y d}
~
d}
~
ww xY w# t8        $ r }
t        j;                  |
      }=Y d}
~
bd}
~
ww xY w# t8        $ r$}
t        j;                  |
      }<|<}A|<}BY d}
~
d}
~
ww xY w# t8        $ r"}
t        j;                  |
      }<|<}EY d}
~
d}
~
ww xY w# t8        $ r"}
t        j;                  |
      }<|<}FY d}
~
d}
~
ww xY w# t8        $ r$}
t        j;                  |
      }<|<}G|<}HY d}
~
d}
~
ww xY w# t8        $ r }
t        j;                  |
      }PY d}
~
d}
~
ww xY w# t8        $ r }
t        j;                  |
      }PY d}
~
d}
~
ww xY w# t8        $ r }
t        j;                  |
      }PY d}
~
d}
~
ww xY w# t8        $ r }
t        j;                  |
      }TY d}
~
d}
~
ww xY w# t8        $ r }
t        j;                  |
      }VY d}
~
d}
~
ww xY w# t8        $ r }
t        j;                  |
      }XY d}
~
d}
~
ww xY w# t8        $ r }
t        j;                  |
      }XY d}
~
d}
~
ww xY w# t8        $ r }
t        j;                  |
      }ZY d}
~
d}
~
ww xY w# t8        $ r$}
t        j;                  |
      }<|<}[|<}\Y d}
~
~d}
~
ww xY w# t8        $ r }
t        j;                  |
      }4Y d}
~
Gd}
~
ww xY w# t8        $ r }
t        j;                  |
      }_Y d}
~
#d}
~
ww xY w# t8        $ r }
t        j;                  |
      }`Y d}
~
d}
~
ww xY w# t8        $ r }
t        j;                  |
      }aY d}
~
d}
~
ww xY w# t8        $ r }
t        j;                  |
      }_Y d}
~
d}
~
ww xY w# t8        $ r }
t        j;                  |
      }aY d}
~
d}
~
ww xY w# t8        $ r }
t        j;                  |
      }_Y d}
~
rd}
~
ww xY w# t8        $ r }
t        j;                  |
      }aY d}
~
Wd}
~
ww xY w# t8        $ r }
t        j;                  |
      }_Y d}
~
'd}
~
ww xY w# t8        $ r }
t        j;                  |
      }`Y d}
~
d}
~
ww xY w# t8        $ r }
t        j;                  |
      }aY d}
~
d}
~
ww xY w# tb        $ r d5}jd5}kY t8        $ r5}
t        j;                  |
      }jt        j;                  |
      }kY d}
~
d}
~
ww xY w# t8        $ r&}
|dt        j;                  |
      z  z  }Y d}
~
d}
~
ww xY w# t8        $ r)}
|dt        j;                  |
      z   dz   z  }Y d}
~
d}
~
ww xY w# t8        $ r;}
|
jr                  tt        k(  rY d}
~
t        j;                  |
      }qY d}
~
d}
~
ww xY w# t8        $ r)}
|dt        j;                  |
      z   dz   z  }Y d}
~
d}
~
ww xY w# t8        $ rE}
|
jr                  tt        k(  rY d}
~
t        j;                  |
      }
|
}u|
}v|
}w|
}x|
}yY d}
~
Ed}
~
ww xY w# t8        $ r)}
|dt        j;                  |
      z   dz   z  }Y d}
~
*d}
~
ww xY w# t8        $ r&}
|d|
j                         z   d z   z  }Y d}
~
|S d}
~
ww xY w(  a  
      Provides a Python interface to GPU management and monitoring functions.

      This is a wrapper around the NVML library.
      For information about the NVML library, see the NVML developer page
      http://developer.nvidia.com/nvidia-management-library-nvml

      Examples:
      ---------------------------------------------------------------------------
      For all elements as in XML format.  Similiar to nvisia-smi -q -x

      $ XmlDeviceQuery()

      ---------------------------------------------------------------------------
      For XML of filtered elements by string name.
      Similiar ot nvidia-smi --query-gpu=pci.bus_id,memory.total,memory.free
      See help_query_gpu.txt or XmlDeviceQuery("--help_query_gpu") for available filter elements

      $ XmlDeviceQuery("pci.bus_id,memory.total,memory.free")

      ---------------------------------------------------------------------------
      For XML of filtered elements by enumeration value.
      See help_query_gpu.txt or XmlDeviceQuery("--help_query_gpu") for available filter elements

      $ XmlDeviceQuery([NVSMI_PCI_BUS_ID, NVSMI_MEMORY_TOTAL, NVSMI_MEMORY_FREE])

      N--help-h--help-query-gpuhelp_query_gpu.txtrr   z<?xml version="1.0" ?>
z7<!DOCTYPE nvidia_smi_log SYSTEM "nvsmi_device_v4.dtd">
z<nvidia_smi>
z  <timestamp>z</timestamp>
z  <driver_version>z</driver_version>
z	  <count>z	</count>
r   z  <id>%s</id>
z    <product_name>z</product_name>
z    <product_brand>z</product_brand>
EnabledDisabledz    <display_mode>z</display_mode>
z    <display_active>z</display_active>
z    <persistence_mode>z</persistence_mode>
z    <accounting_mode>z</accounting_mode>
z!    <accounting_mode_buffer_size>z</accounting_mode_buffer_size>
FWDDMTCCz      <current_dm>z</current_dm>
Tz      <pending_dm>z</pending_dm>
z    <driver_model>
z    </driver_model>
z    <serial>z
</serial>
z
    <uuid>z</uuid>
z    <minor_number>z</minor_number>
z    <vbios_version>z</vbios_version>
rk  z)    <multigpu_board>N/A</multigpu_board>
z)    <multigpu_board>Yes</multigpu_board>
z(    <multigpu_board>No</multigpu_board>
0x%xz    <board_id>z</board_id>
z      <img_version>z</img_version>
z      <oem_object>z</oem_object>
z      <ecc_object>z</ecc_object>
z      <pwr_object>z</pwr_object>
z    <inforom_version>
z    </inforom_version>
z      <current_gom>z</current_gom>
z      <pending_gom>z</pending_gom>
z    <gpu_operation_mode>
z    </gpu_operation_mode>
z      <pci_bus>%02X</pci_bus>
z$      <pci_device>%02X</pci_device>
z$      <pci_domain>%04X</pci_domain>
z*      <pci_device_id>%08X</pci_device_id>
z      <pci_bus_id>z</pci_bus_id>
z2      <pci_sub_system_id>%08X</pci_sub_system_id>
z          <max_link_gen>z</max_link_gen>
z          <current_link_gen>z</current_link_gen>
z        <pcie_gen>
z        </pcie_gen>
xz          <max_link_width>z</max_link_width>
z          <current_link_width>z</current_link_width>
z        <link_widths>
z        </link_widths>
z      <pci_gpu_link_info>
z      </pci_gpu_link_info>
PLXBR04z        <bridge_chip_type>z</bridge_chip_type>
%08Xz,        <bridge_chip_fw>%s</bridge_chip_fw>
z        <bridge_chip_fw>z</bridge_chip_fw>
z      <pci_bridge_chip>
z      </pci_bridge_chip>
z      <replay_counter>z</replay_counter>z      <tx_util>z KB/sz
</tx_util>z      <rx_util>z
</rx_util>z
    <pci>
z    </pci>
z %z    <fan_speed>z</fan_speed>
P%sz    <performance_state>z</performance_state>
   z MiBz      <total>r   z      <used>z</used>
z      <free>z</free>
z    <fb_memory_usage>
z    </fb_memory_usage>
z    <bar1_memory_usage>
z    </bar1_memory_usage>
DefaultExclusive Thread
ProhibitedExclusive_Processr   z    <compute_mode>z</compute_mode>
z      <gpu_util>z</gpu_util>
z      <memory_util>z</memory_util>
z      <encoder_util>z</encoder_util>
z      <decoder_util>z</decoder_util>
z    <utilization>
z    </utilization>
z      <current_ecc>z</current_ecc>
z      <pending_ecc>z</pending_ecc>
z    <ecc_mode>
z    </ecc_mode>
z    <ecc_errors>
z    </ecc_errors>
z    <retired_pages>
z    </retired_pages>
z Cz      <gpu_temp>z</gpu_temp>
z      <gpu_temp_max_threshold>z</gpu_temp_max_threshold>
z      <gpu_temp_slow_threshold>z</gpu_temp_slow_threshold>
z    <temperature>
z    </temperature>
	Supportedz      <power_management>z</power_management>
     @@z%.2f Wz      <power_draw>z</power_draw>
z      <power_limit>z</power_limit>
z      <default_power_limit>z</default_power_limit>
z      <enforced_power_limit>z</enforced_power_limit>
r   z      <min_power_limit>z</min_power_limit>
z      <max_power_limit>z</max_power_limit>
z    <power_readings>
Pz$      <power_state>%s</power_state>
z    </power_readings>
z MHzz      <graphics_clock>z</graphics_clock>
z      <sm_clock>z</sm_clock>
z      <mem_clock>z</mem_clock>
z    <clocks>
z    </clocks>
z    <applications_clocks>
z    </applications_clocks>
z"    <default_applications_clocks>
z#    </default_applications_clocks>
z    <max_clocks>
z    </max_clocks>
z    <clock_policy>
OffOnz      <auto_boost>z</auto_boost>
z      <auto_boost_default>z</auto_boost_default>
z    </clock_policy>
z    <supported_clocks>
z      <supported_mem_clock>
z        <value>%d MHz</value>
zD        <supported_graphics_clock>%d MHz</supported_graphics_clock>
z@        <supported_graphics_clock>%s</supported_graphics_clock>
z      </supported_mem_clock>
z    </supported_clocks>
z    <supported_clocks>z</supported_clocks>
z    <processes>
z    <process_info>
z      <pid>%d</pid>
z      <process_name>z</process_name>
%d MiBz      <used_memory>z</used_memory>
z    </process_info>
z    </processes>
z    <processes>z</processes>
z    <accounted_processes>
%d %%%d ms%dz    <accounted_process_info>
z      <max_memory_usage>z</max_memory_usage>
z      <time>z</time>
z      <is_running>z</is_running>
z    </accounted_process_info>
z    </accounted_processes>
z    <accounted_processes>z</accounted_processes>
z  <gpu>z	  </gpu>
z</nvidia_smi>
nvidia_smi.py: 
)r   r  r@  r   XmlDeviceQuery__doc__openread"_nvidia_smi__fromDeviceQueryStringlistNVSMI_TIMESTAMPr   datetimedatetodayNVSMI_DRIVER_VERSIONnvmlSystemGetDriverVersionr   NVSMI_COUNTr   r   nvmlDeviceGetPciInfoNVSMI_PCI_BUS_IDbusId
NVSMI_NAMEnvmlDeviceGetNameNVSMI_BRAND_NAMESnvmlDeviceGetBrandr   r   NVSMI_DISPLAY_MODEnvmlDeviceGetDisplayModeNVSMI_DISPLAY_ACTIVEnvmlDeviceGetDisplayActiveNVSMI_PERSISTENCE_MODEnvmlDeviceGetPersistenceModeNVSMI_ACCT_MODEnvmlDeviceGetAccountingModeNVSMI_ACCT_BUFFER_SIZE!nvmlDeviceGetAccountingBufferSizeNVSMI_DRIVER_MODEL_CURnvmlDeviceGetCurrentDriverModelNVML_DRIVER_WDDMNVSMI_DRIVER_MODEL_PENDINGnvmlDeviceGetPendingDriverModelNVSMI_SERIALNUMBERnvmlDeviceGetSerial
NVSMI_UUIDnvmlDeviceGetUUIDNVSMI_INDEXnvmlDeviceGetMinorNumberNVSMI_VBIOS_VERnvmlDeviceGetVbiosVersionnvmlDeviceGetMultiGpuBoardNVSMI_BOARD_IDnvmlDeviceGetBoardIdNVSMI_INFOROM_IMG nvmlDeviceGetInforomImageVersionNVSMI_INFOROM_OEMnvmlDeviceGetInforomVersionNVML_INFOROM_OEMNVSMI_INFOROM_ECCNVML_INFOROM_ECCNVSMI_INFOROM_PWRNVML_INFOROM_POWER_nvidia_smi__toStrGOM$nvmlDeviceGetCurrentGpuOperationMode$nvmlDeviceGetPendingGpuOperationModeNVSMI_PCI_BUSbusNVSMI_PCI_DEVICEdeviceNVSMI_PCI_DOMAINdomainNVSMI_PCI_DEVICE_IDpciDeviceIdNVSMI_PCI_SUBDEVICE_IDpciSubSystemIdNVSMI_PCI_LINK_GEN_MAX"nvmlDeviceGetMaxPcieLinkGenerationNVSMI_PCI_LINK_GEN_CUR#nvmlDeviceGetCurrPcieLinkGenerationNVSMI_PCI_LINK_WIDTH_MAXnvmlDeviceGetMaxPcieLinkWidthNVSMI_PCI_LINK_WIDTH_CURnvmlDeviceGetCurrPcieLinkWidthnvmlDeviceGetBridgeChipInfobridgeChipInfotype	fwVersionnvmlDeviceGetPcieReplayCounternvmlDeviceGetPcieThroughputNVML_PCIE_UTIL_TX_BYTESNVML_PCIE_UTIL_RX_BYTESNVSMI_FAN_SPEEDnvmlDeviceGetFanSpeedNVSMI_PSTATEnvmlDeviceGetPowerState&NVSMI_CLOCK_THROTTLE_REASONS_SUPPORTED#NVSMI_CLOCK_THROTTLE_REASONS_ACTIVE!NVSMI_CLOCK_THROTTLE_REASONS_IDLE(NVSMI_CLOCK_THROTTLE_REASONS_APP_SETTING'NVSMI_CLOCK_THROTTLE_REASONS_SW_PWR_CAP(NVSMI_CLOCK_THROTTLE_REASONS_HW_SLOWDOWN0NVSMI_CLOCK_THROTTLE_REASONS_HW_THERMAL_SLOWDOWN2NVSMI_CLOCK_THROTTLE_REASONS_HW_PWR_BRAKE_SLOWDOWN0NVSMI_CLOCK_THROTTLE_REASONS_SW_THERMAL_SLOWDOWN'NVSMI_CLOCK_THROTTLE_REASONS_SYNC_BOOST(_nvidia_smi__xmlGetClocksThrottleReasonsNVSMI_MEMORY_TOTALNVSMI_MEMORY_USEDNVSMI_MEMORY_FREEnvmlDeviceGetMemoryInfor   usedNVSMI_MEMORY_BAR1nvmlDeviceGetBAR1MemoryInfo	bar1Totalbar1UsedNVSMI_COMPUTE_MODEnvmlDeviceGetComputeModeNVML_COMPUTEMODE_DEFAULT!NVML_COMPUTEMODE_EXCLUSIVE_THREADNVML_COMPUTEMODE_PROHIBITED"NVML_COMPUTEMODE_EXCLUSIVE_PROCESSNVSMI_UTILIZATION_GPUNVSMI_UTILIZATION_MEMnvmlDeviceGetUtilizationRatesgpumemoryNVSMI_UTILIZATION_ENCODERnvmlDeviceGetEncoderUtilizationNVSMI_UTILIZATION_DECODERnvmlDeviceGetDecoderUtilizationNVSMI_ECC_MODE_CURNVSMI_ECC_MODE_PENDINGnvmlDeviceGetEccMode_nvidia_smi__xmlGetEcc_nvidia_smi__xmlGetRetiredPagesNVSMI_TEMPERATURE_GPUnvmlDeviceGetTemperatureNVML_TEMPERATURE_GPU!nvmlDeviceGetTemperatureThreshold#NVML_TEMPERATURE_THRESHOLD_SHUTDOWN#NVML_TEMPERATURE_THRESHOLD_SLOWDOWNNVSMI_POWER_MGMT nvmlDeviceGetPowerManagementModeNVSMI_POWER_DRAWnvmlDeviceGetPowerUsageNVSMI_POWER_LIMIT!nvmlDeviceGetPowerManagementLimitNVSMI_POWER_LIMIT_DEFAULT(nvmlDeviceGetPowerManagementDefaultLimitNVSMI_POWER_LIMIT_ENFORCEDnvmlDeviceGetEnforcedPowerLimitNVSMI_POWER_LIMIT_MINNVSMI_POWER_LIMIT_MAX,nvmlDeviceGetPowerManagementLimitConstraintsNVSMI_CLOCKS_GRAPHICS_CURnvmlDeviceGetClockInfoNVML_CLOCK_GRAPHICSNVML_CLOCK_SMNVSMI_CLOCKS_MEMORY_CURNVML_CLOCK_MEMNVSMI_CLOCKS_APPL_GRAPHICSnvmlDeviceGetApplicationsClockNVSMI_CLOCKS_APPL_MEMORY"NVSMI_CLOCKS_APPL_GRAPHICS_DEFAULT%nvmlDeviceGetDefaultApplicationsClock NVSMI_CLOCKS_APPL_MEMORY_DEFAULTNVSMI_CLOCKS_GRAPHICS_MAXnvmlDeviceGetMaxClockInfoNVSMI_CLOCKS_SM_MAXNVSMI_CLOCKS_MEMORY_MAXNVSMI_CLOCKS_POLICY%nvmlDeviceGetAutoBoostedClocksEnabledrP  NVMLError_NotSupportedNVSMI_CLOCKS_SUPPORTED"nvmlDeviceGetSupportedMemoryClocks$nvmlDeviceGetSupportedGraphicsClocksNVSMI_COMPUTE_APPS$nvmlDeviceGetComputeRunningProcessesnvmlSystemGetProcessNamepidr  NVML_ERROR_NOT_FOUNDusedGpuMemoryNVSMI_ACCOUNTED_APPSnvmlDeviceGetAccountingPidsnvmlDeviceGetAccountingStatsgpuUtilizationmemoryUtilizationmaxMemoryUsager   	isRunningr&  r  )zr   r   finr   r   r   r   pciInfogpuInfo	brandNamer   stater`  
bufferSizedriverModelincludeDriverModelcurrentpendingru   rw   minor_numbervbiosmultiGpuBoolboardIdhexBIDinforomVersionincludeInforomimgoemr(  pwrgpuOperationMode
includeGOMpci
includePcipciGpuLinkInfoincludeLinkInfopciGen
includeGengenpciLinkWidthsincludeLinkWidthswidthpciBridgeChipincludeBridgeChipbridgeHierarchybridge_typestrFwVersionreplaytx_bytesrx_bytesfan	perfStateperfStateStrfbMemoryUsageincludeMemoryUsagememInfo	mem_totalmem_usedmem_freer7  modeStrutilizationincludeUtilizationutilgpu_utilmem_utilutil_intssizeencoder_utildecoder_utilcurr_strpend_streccMode	eccErrorsincludeEccErrorsrR  rS  temperatureincludeTemperaturetemppower_readingsincludePowerReadingspowMan	powManStrpowDraw
powDrawStrpowLimitpowLimitStrenforcedPowLimitenforcedPowLimitStrpowLimitStrMinpowLimitStrMaxr   includeClocksgraphicssmmemapplicationClocksincludeAppClocksdefaultApplicationClocksincludeDefaultAppClocks	maxClocksincludeMaxClocksboostedStateboostedDefaultStateautoBoostStrautoBoostDefaultStr	memClocksmcprocsr   rs   pidsrT  statsrZ  r[  r\  r   
is_runningsz                                                                                                                             r   r  znvidia_smi.XmlDeviceQueryo  s(   < .;&vs# fn..666,,,c2 $c$ $ "99&Af<&i{@
1
1)
Q
Q)
'
')6!_%>?Z-B-B8==CVCVCX-YY\llli6!%9V%C/*2G2GHbHd2eeh}}}i*,+6![F%:;)>)>{)KKlZZiK( h,a~~a(f,V4ggv%)9V)C.>>'v%v)=1J4I4IJ[\bJc4ddgzzz'@"34Fv4N"Oi
 2Y>AUUU'v%);v)E<-Ef-MQR-RyYce 1E9<OOO'v%)=)G<-G-OST-Ty[ee 3e;>SSS'v%)?6)I;+G+OST+TY[ed 5<?VVV'v%F)B;+Fv+NRS+SYZdd 4t;>TTT'v%)?6)IA#-#8#89Z[a9b#cj @:MPrrr'k#( v%)?6)I>+J6+RVf+fmrg !5!?BS!SS+'+$v%)Cv)M.)H)PTd)dfkpG $33'[('44'v%);v)E=26:f ^j.C.CF.KKm[['v%v)=;.v6d \J,A,A$,GG+UU'v%)>C%=f%El 1J4I4I,4WWZmmm'v%F)B<7?e 2Z5J5J55QQThhh'v%F)BD%?%Gl "U*!WWg#!WWg!VVgv%6)A> 4V <g'%/f -6HH'!n$nv%):f)D:<VDc !$9J<Q<QRU<V$VYk$kk.#'.v%):f)D:7@PQc !$8:;P;PQT;U$UXi$ii.#'.v%):f)D:7@PQc !$8:;P;PQT;U$UXi$ii.#'.v%):f)D:7@RSc !$8:;P;PQT;U$UXi$ii.#'.66'^+'77'!# jv%):f)D> * 5 56Z[a6b cg #&;j>S>ST[>\&\_q&qq"#*v%):f)D> * 5 56Z[a6b cg #&;j>S>ST[>\&\_q&qq"#*99'--'::'c jv%&)@:W[[HH##*v%)9V)C@7>>QQ##*v%)9V)C@7>>QQ##*v%)<)FF'J]J]^^##*v%)9V)C-
0E0Egmm0TTWhhh##*v%)?6)INRYRhRhii##*!n %of jv%)?6)I:&112TU[2\]c 6<?RRR&#*v%)?6)I:&112UV\2]^c :S@CZZZ&#* $::. F*. $;;.$(/ m"'v%)AV)K<(334QRX4YZ]``e  #?%#GJ_#__-&*#v%)AV)K<(334RSY4Z[^aae  #Ce#KNg#gg-&*#" $==. M1. $>>.$(/ 66#'#77# !m"'v%z(CF(Ko$&k(77:??1D%.+%/+#'Ck'QTk'kkm(77:DDI).,)/?3Q3QRS3T3^3^)_,#'VZf'ggm '+#"44#&#55#v%l=fEf5
8M8Mf8UUXkkkc  $*v%^!<VE\!]h.1F1Fx1PPSZZ]iiic  $*v%^!<VE\!]h.1F1Fx1PPSZZ]iiic  $*]*'S.'^+'v%F)B:&112G2OPSWWc .47GGG'v%)?C"9&"Ai%*Y%6l 6EH```'v%8FB5?3v=:fD9VC:fDBfLDNBfL9VCZEEfMM' m#( v%$.#v-#v-'+$	' 7 ?g","7"78Lt8S"TW]"]i!+!6!6w||d7JT7Q!RU[![h!+!6!6w}}t7Kd7RU\UaUadhUhkoUo7o!psy!yh  6)-?6-I#'B\'QQm6)->&-H#~'@;'NNm6)->&-H#~'@;'NNm#66']*'77'v%):f)D	' ;F Cg","7"78I8ID8PSW8W"X[a"ai!+!6!6w7G7G$7NQU7U!VY_!_h!+!6!6w7H7H47ORV7VY`YiYilpYpswYw7w!x  |B  "Bh 88'_y8<GG'^h6DD'^h6DD'99'v%);v)E>5f=d!99$-'#DD$6'#>>$0'#EE$7'$-' 1G;>QQQ'k#( v%'61'61':6Bd!+!6!6txx!@4!Gh!+!6!6t{{!Cd!Jh  6)-Bf-L!%7(%B_%TTk6)-Bf-L!%:X%EHZ%ZZk'+$v%)Bf)L+*I&*Q'x%/%:%:8%Dt%Kl
 !7,!FI\!\\+'+$v%)Bf)L+*I&*Q'x%/%:%:8%Dt%Kl
 !7,!FI\!\\+'+$#22'[('33'v%$.(F2'+?+G(w/6!|*h/6!|*h '6)-?6-I!6!ADV!VVg6)-Cv-M!6!ADV!VVg//'W$'00'+5+A+A&&+Q(i(!11'Y&'22'3=3R3RSY[a3b/l/$44'\)'55'k#( v%)>&)H;'223KFTh3ijmqqd !3d!:_!LL+;'223TU[  ^A  4B  C  FJ  Jd !AD!HKh!hh+'+$;'223TU[  ^A  4B  C  FJ  Jd !BT!ILj!jj+'+$#22'K'33'!n%*"v%)9V)C@?Gf171+%i !$>$JMd$dd.)-&v%)9V)CA!8!@6!Ig#+g#5j !$8:$EHY$YY.)-&v%):f)DB"CF"Kf"Th$,x$7k !$9K$GJ\$\\.)-&v%)Bf)LB"J6"RU["[h$,x$7k !$AK$ORl$ll.)-&v%)Cv)MJ*I&*QTZ*Z&,47G,G) !$BEX$X[v$vv.)-&v%'61'61-!Mf!Uh'/8A;3G'Hn'/8A;3G'Hn
  6)-Bf-L$(AN(RUk(kkn6)-Bf-L$(AN(RUk(kkn)-&%55'@"%
(=(=>UV\>](^"^i DyPP'^+'66'f#mf$(AV(K?!+!6!67MfVi7j!knt!th 4h>AVVV&"&-f$(AV(K9%001GP]1^_bhhb .3oEE&"&-f$(?6(I:&112HQ_2`adjjc /#58HHH&"&---'V#'..'"$!&f$(Bf(L?!+!6!67UV\^q7r!sv|!|h $'?'ILa'aa#%)"f$(@F(J:&112PQWYg2hilrrc $':S'@CS'SS#%)"!::'..';;')+&(-%f$(Jf(T?!+!6!67\]cex7y!z  ~D  "Dh +.F.PSh.hh*,0)f$(HF(R:&112WX^`n2opsyyc +.AC.GJZ.ZZ*,0)(BB'55'CC'i!&f$(AV(K?!+!6!67PQWYl7m!nqw!wh 7(BEZZZ)%)"f$(;v(E9%001J6S`1abekkb 1B6HH)%)"f$(?6(I:&112KFTb2cdgmmc 2S8;KKK)%)"!11'Y&'22'v%)<)F33':_`f:g7l$7%)>>).,)-,,0EE05-04- 1L@CTTT'9<OORkkk'44'v%)?6)It"DV"Li!;;g( 	F!!%DD'!%F%JJ'M'KFTU'Vf'- y!")-twx-x"x'y "%EE'	F !<<g v%);v)EfB6Je!44g$ =!G%/%:%:;STUTYTY;Z%[d "%;;'!%<quu%DD'!%;d%BEX%XX'oo5$)c$,$0F0M$Nc!%:S%@CU%UU'!%<<''=* !55g v%)=)G%z8@d!>>g!% G#/&B63&Oe/69M9M/Mn29E<S<S2S/"'"6"6$">38.3;u?S?SVZ?Z]a?a3b.%,uzz%9d+/%//+Aj "%EE'!%<s%BB'!%7.%H?%ZZ'!%:=N%NQc%cc'!%?.%PRi%ii'!^d%:[%HH'!%9J%FIZ%ZZ'!%FF';G> !??g Wqy()w&)|+)Qh,T (
()
 K$ $L # @",":":3"?i@ # <(66s;e< # <(66s;e< # ;'55c:d; # ;'55c:d; # A#-#;#;C#@jA # > * 8 8 =g> ! .(66s;G#7'#ADU#UUK)-&. # =)77<f= # ;'55c:d; # C%/%=%=c%BlC # <(66s;e< # D%/%=%=c%BlD # > * 8 8 =g>
'&f # :&44S9c: # :&44S9c: # :&44S9c: # :&44S9c:  # > * 8 8 =g> # > * 8 8 =g>\ # :&44S9c: # :&44S9c:$ # <(66s;e< # <(66s;e<H # z#'CjF^F^_bFc'cf}'}}m#'AJD\D\]`Da'ady'yymz # l5
8P8PQT8UUXkkkcl # ^.1I1I#1NNQ]]]c^ # ^.1I1I#1NNQ]]]c^ # :&44S9c: # C%/%=%=c%BlC< # '(66s;e"'i!&h!&h	'0 # '(66s;e"'i!&h!&h	'2 # > * 8 8 =g> # '(66s;e!&h!&h'" # +(66s;e%*l+ # +(66s;e%*l+( # '(66s;e!&h!&h'B # ;'55c:d; # ;'55c:d; # ;'55c:d;$ # @",":":3"?i@ # A#-#;#;C#@jA # B$.$<$<S$AkB # B$.$<$<S$AkB # J,6,D,DS,I)J # -(66s;e',n',n- # @",":":3"?i@ # ?!+!9!9#!>h? # 9%33C8b9 # :&44S9c: # ?!+!9!9#!>h? # :&44S9c:  # ?!+!9!9#!>h? # :&44S9c: # ?!+!9!9#!>h? # 9%33C8b9 # :&44S9c:0 0 2%*l,1)" %/%=%=c%Bl,6,D,DS,I)( "+ M%)loy  pH  pH  IL  pM  *M  MgM
 # t!9J<T<TUX<Y!Y\s!ssgt "+ G"%))/C"C"*)3)A)A#)F$G& # f!2Z5M5Mc5R!RUe!eegf& "+ 	/"%))/C"C"*$.$<$<S$Ac/2n25//2n%(d+.j	/* # z!<z?W?WX[?\!\_y!yygz  @
(3;;=84?
?)@s<  A]EBM  A]%-BM  	A^BM  7A^=	BM  %A_)7BM  A`%BM  AaBM  ?Aa-BM  3Ab	!BM  +Ac6.BM  %Ac>0.BM  Ad**.BM  Ae$.BM  Af(BM  Af.AgBM  8Ag$0BM  4Ah0BM  5Ah<0BM  6Ai(ABM  Aj*0BM  Ak 9D;BM  5Ak,BM  1Al4BM  !Am%BM  !Am0$=BM  "BAn$BM   )Ao/)
BM  41Ap$%
BM  01Aq!#BM  !Ar&BM  Ar:B!BM  4BAs&
A2BM  =BAt;BM  =Au
'BM  4AAu6A	BM  'Av&6BM  'Aw<8BM  5!AxB4BM  (Ax23BM  ?(Ay'BM  5(Az
3BM  @Az6@&BM  AA{"ABM  A8A|BBM  B+A|:B?BM  CA}&C2'BM  D"A~D<ABM  E>!AF'BM  G(A.G/BM  H(B@H6BM  I(BAI=3BM  J1(BA2KBM  K8(BBL 3BM  M(BC
M<BM  N(BC6O3BM  O7(BD"PBM  P>(BEQ&BM  R(BE:R-4BM  S"-BF&T,BM  T<#BH)U BG7U<BH)V	BM  VBJ%V2*BIWA#BJ%X?BM  YBL+Y(A6BK[ABL+\.BM  \?BM  ]A]"]%	A^].A^	^BM  ^	A^^BM  ^	A^:^A^5^/BM  ^5A^:^:BM  ^=	A_&_A_!_BM  _!A_&_&BM  _)	A`_2A``BM  `A``BM  `	A`>`A`9`3BM  `9A`>`>BM  a	Aa*a
Aa%aBM  a%Aa*a*BM  a-	Aba6AbbBM  bAbbBM  b	Acb""Ac
cBM  c
AccBM  c	Ac;cAc6c0BM  c6Ac;c;BM  c>	Ad'dAd"dBM  d"Ad'd'BM  d*	Aed3AeeBM  eAeeBM  e	Ae?eAe:e4BM  e:Ae?e?BM  f	Af+fAf&f BM  f&Af+f+BM  f.	Agf7AggBM  gAggBM  gAg!gBM  g$	Ahg-AhhBM  hAhhBM  h	Ah9hAh4h.BM  h4Ah9h9BM  h<	Ai%iAi iBM  i Ai%i%BM  i(	Aji1AjjBM  jAjjBM  j	Aj=jAj8j2BM  j8Aj=j=BM  k 	Ak)k	Ak$kBM  k$Ak)k)BM  k,	Alk5All
BM  lAllBM  l	Aml!Al<l6BM  l<AmmBM  m	Am-mAm(m"BM  m(Am-m-BM  m0	Anm9AnnBM  nAnnBM  n	Ao,n%<Ao'o!BM  o'Ao,o,BM  o/	Ap!o8AppBM  pAp!p!BM  p$	Aqp-AqqBM  qAqqBM  q	Arq"Arr BM  rArrBM  r	Ar7rAr2r,BM  r2Ar7r7BM  r:	As#sAssBM  sAs#s#BM  s&	Ats/Att
BM  tAttBM  t	Aut!Aut<BM  uAuuBM  u
	Au3uAu.u(BM  u.Au3u3BM  u6	Av#u?AvvBM  vAv#v#BM  v&	Awv/AwwBM  wAwwBM  w	Aw?wAw:w4BM  w:Aw?w?BM  x	Ax/xAx*x$BM  x*Ax/x/BM  x2	Ayx;AyyBM  yAyyBM  y	Azy'Azy<BM  zAzzBM  z
	Az3zAz.z(BM  z.Az3z3BM  z6	A{z?A{{BM  {A{{BM  {"	A|{+A|| BM  |A||BM  |	A|7|A|2|,BM  |2A|7|7BM  |:	A}#}A}}BM  }A}#}#BM  }&	A~}/A~
~BM  ~
A~~BM  ~	A~?~A~:~4BM  ~:A~?~?BM  	A+A& BM  &A++BM  .	B@7B@@BM  @B@@BM  @	BA@#B@>@8BM  @>BAABM  A	BA/ABA*A$BM  A*BA/A/BM  A2	BBA;BBBBM  BBBBBM  B	BCB'BCB<BM  CBCCBM  C
	BC3CBC.C(BM  C.BC3C3BM  C6	BDC?BDDBM  DBDDBM  D"	BED+BEE BM  EBEEBM  E	BE7EBE2E,BM  E2BE7E7BM  E:	BF#FBFFBM  FBF#F#BM  F&BG4F4BM  F7BG4F?*BG/G)BM  G/BG4G4BM  G7	BH&H BH!HBH)H!BH&H&BH)H)	BIH2BIIBM  IBIIBM  I	BJ"I'BJI<BJ%JBJJBJ%JBJ"J"BJ%J%	BKJ.BKKBM  KBKKBM  K	BL(K#BL#K8BL+K>BL#LBL+L#BL(L(BL+L+	BML4BMMBM  MBMMBM  M 	BNM)BN
N
BNc                 x>   |t         g}nt        |t              re|dk(  s|dk(  rt        j                  j
                  S |dk(  r&t        dd      5 }|j                         cddd       S t        j                  |      }nt        |      }i }g }	 t         |v st        |v r4t        j                  t        j                  j                               |d<   t         |v st        |v r t        j                  t!                     |d<   t#               }t         |v st$        |v r||d	<   t'        d
|      D ]  }i }| j(                  |   }t+        |      }	t         |v st,        |v r"t        j                  |	j.                        |d<   t         |v st0        |v r9t        j                  t3        |            |d<   	 t4        t7        |         }
|
|d<   t         |v st<        |v r	 t?        |      d
k7  rdnd}||d<   t         |v st@        |v r	 tC        |      d
k7  rdnd}||d<   t         |v stD        |v r	 tG        |      d
k7  rdnd}||d<   t         |v stH        |v r	 tK        |      d
k7  rdnd}||d<   t         |v stL        |v r$	 t        j                  tO        |            }||d<   i }d}t         |v stP        |v r	 tS        |      tT        k(  rdnd}||d<   d}t         |v stV        |v r	 tY        |      tT        k(  rdnd}||d<   d}|r||d<   t         |v stZ        |v r$	 t]        |      }t        j                  |      |d<   t         |v st^        |v r$	 ta        |      }t        j                  |      |d<   t         |v stb        |v r$	 te        |      }t        j                  |      |d<   t         |v stf        |v r$	 ti        |      }t        j                  |      |d<   t         |v stf        |v r$	 tk        |      }|d k(  rd |d!<   n|rd"|d!<   nd#|d!<   t         |v stl        |v r	 to        |      }	 d$|z  }||d%<   i }d}t         |v stp        |v r&	 ts        |      }t        j                  |      |d&<   d}t         |v stt        |v r+	 tw        |tx              }t        j                  |      |d'<   d}t         |v stz        |v r+	 tw        |t|              }t        j                  |      |d(<   d}t         |v st~        |v r+	 tw        |t              }t        j                  |      |d)<   d}|r||d*<   i } d}!t         |v st~        |v r9	 t        j                  t        |            }t        j                  |      | d+<   d}!t         |v st~        |v r9	 t        j                  t        |            }t        j                  |      | d,<   d}!|!r| |d-<   i }"d}#t         |v st        |v rd.|	j                  z  |"d/<   d}#t         |v st        |v rd.|	j                  z  |"d0<   d}#t         |v st        |v rd1|	j                  z  |"d2<   d}#t         |v st        |v rd3|	j                  z  |"d4<   d}#t         |v st,        |v r$t        j                  |	j.                        |"d5<   d}#t         |v st        |v rd3|	j                  z  |"d6<   d}#i }$d}%i }&d}'t         |v st        |v r&	 t        j                  t        |            }(|(|&d7<   d}'t         |v st        |v r&	 t        j                  t        |            }(|(|&d8<   d}'|'r|&|$d9<   d}%i })d}*t         |v st        |v r)	 t        j                  t        |            d:z   }+|+|)d;<   d}*t         |v st        |v r)	 t        j                  t        |            d:z   }+|+|)d<<   d}*|*r|)|$d=<   d}%|%r|$|"d><   d}#i },d}-t         |v r	 t        |      }.d?}/|.j                  d
   j                  d
k(  r|/d@z  }/n|/dAz  }/|/|,dB<   |.j                  d
   j                  d
k(  rd }0nd3|.j                  d
   j                  z  }0t        j                  |0      |,dC<   d}-|-r|,|"dD<   d}#t         |v r&	 t        |      }1t        j                  |1      |"dE<   d}#t         |v r	 t        |t              }2|2|"dF<   dG|"dH<   d}#t         |v r	 t        |t              }3|3|"dI<   dG|"dJ<   d}#|#r|"|dK<   t         |v st        |v r	 t        |      }4|4|dL<   dM|dN<   t         |v st        |v r	 t        |      }5dO|5z  }6|6|dP<   t         |v sPt        |v sHt        |v s@t        |v s8t        |v s0t        |v s(t        |v s t        |v st        |v st        |v st        |v rt        j                  |      |dQ<   i }7d}8t         |v st        |v st        |v st        |v rd}8	 t        |      }9|9j                  dRz  dRz  }:|9j                  dRz  dRz  };|9j                  dRz  dRz  |9j                  dRz  dRz  z
  }<t         |v st        |v r|:|7dS<   t         |v st        |v r|;|7dT<   t         |v st        |v r|<|7dU<   |8r
dV|7dW<   |7|dX<   t         |v st        |v rp	 t        |      }9|9j                  dRz  dRz  }:|9j                  dRz  dRz  };|9j                  dRz  dRz  |9j                  dRz  dRz  z
  }<i }>|:|>dS<   |;|>dT<   |<|>dU<   dV|>dW<   |>|dY<   t         |v st        |v rC	 t        |      }|t        k(  rdZ}?n&|t        k(  rd[}?n|t        k(  rd\}?n|t        k(  rd]}?nd^}?|?|d_<   i }@d}At         |v st        |v st        |v rR	 t        |      }B|Bj                   }C|Bj                  }Dt         |v st        |v rC@d`<   t         |v st        |v rD@da<   d}At         |v s	t        |v r	 t        |      \  }E}F|E}GG@db<   d}At         |v s	t        |v r	 t        |      \  }E}F|E}HH@dc<   d}AAr
dM@dW<   |@|dd<   t         |v st        |v s	t        |v rU	 t        |      \  }}|d
k7  rdnd}I|d
k7  rdnd}Ji }Kt         |v s	t        |v rIKde<   t         |v s	t        |v rJKdf<   K|dg<   t        j                  ||      \  }L}M|MrL|dh<   t        j                  ||      \  }N}O|OrN|di<   i }Pd}Qt         |v s	t        |v rN	 t        |t              }RRPdj<   d}Q	 t        |t              }RRPdk<   d}Q	 t        |t               }RRPdl<   d}QQr
dmPdW<   |P|dn<   i }Sd}Tt         |v s	t"        |v r	 t%        |      }U|Ud
k7  rdond }VVSdp<   d}Tt         |v s	t&        |v r	 t)        |      dqz  }W|W}XXSdr<   d}Tt         |v s	t*        |v r	 t-        |      dqz  }Y|Y}ZZSds<   d}Tt         |v s	t.        |v r	 t1        |      dqz  }Y|Y}ZZSdt<   d}Tt         |v s	t2        |v r	 t5        |      dqz  }[|[}\\Sdu<   d}Tt         |v st6        |v s	t8        |v rK	 t;        |      }Y|Yd
   dqz  }]|Ydv   dqz  }^t         |v s	t6        |v r]Sdw<   t         |v s	t8        |v r^Sdx<   d}TTr1	 dyt        j                  t        |            z   }5|5Sdz<   d{|SdW<   |S|d|<   i }_d}`t         |v s	t<        |v r	 t?        |t@              }aa_d}<   d}`t         |v s	t<        |v r	 t?        |tB              }bb_d~<   d}`t         |v s	tD        |v r	 t?        |tF              }cc_d<   d}``r
d_dW<   |_|d<   i }dd}et         |v s	tH        |v r	 tK        |t@              }aadd}<   d}et         |v s	tL        |v r	 tK        |tF              }ccdd<   d}eer
dddW<   |d|d<   i }fd}gt         |v s	tN        |v r	 tQ        |t@              }aafd}<   d}gt         |v s	tR        |v r	 tQ        |tF              }ccfd<   d}ggr
dfdW<   |f|d<   i }hd}it         |v s	tT        |v r	 tW        |t@              }aahd}<   d}it         |v s	tX        |v r	 tW        |tB              }bbhd~<   d}it         |v s	tZ        |v r	 tW        |tF              }cchd<   d}iir
dhdW<   |h|d<   t         |v s	t\        |v r?i }j	 t_        |      \  }k}l|kt`        k(  rd}mnd}mlt`        k(  rd}nnd}nmjd<   n|jd<   |j|d<   t         |v s	td        |v rzg }o	 tg        |      }p|pD ]O  }qi }r|q|rd<   d|rdW<   g }s	 ti        |q      }_|_D ]  }tsjk                  |t        	 srd<   ojk                  |r       Q 	 tm        o      d
kD  rond|d<   t         |v s	tn        |v rg }u	 tq        |      }v|vD ]  }w	 t        j                  ts        wjt                              }xi }ywjt                  |yd<   x|yd<   |wjz                  dk(  rd
}cnt}        wjz                  dRz  dRz        }ccyd<   dV|ydW<   ujk                  |y        	 tm        u      d
kD  rund|d<   t         |v s	t~        |v r	 t        |      }zg }{|zD ]  }|	 t        ||      }}d|}j                  z  }~d|}j                  z  }|}j                  dk(  rd }nd}j                  dRz  dRz  z  }d}j                  z  }d|}j                  z  }i }d|z  |d<   ~|d`<   |da<   |d<   |d<   |d<   {jk                  |        tm        {      d
kD  r{nd|d<   tm        |      d
kD  s|jk                  |        tm        |      d
kD  r||d<   |S # 1 sw Y   xY w# t8        $ r }t        j;                  |      }
Y d}~d}~ww xY w# t8        $ r }t        j;                  |      }Y d}~d}~ww xY w# t8        $ r }t        j;                  |      }Y d}~d}~ww xY w# t8        $ r }t        j;                  |      }Y d}~d}~ww xY w# t8        $ r }t        j;                  |      }Y d}~d}~ww xY w# t8        $ r }t        j;                  |      }Y d}~d}~ww xY w# t8        $ r }t        j;                  |      }Y d}~d}~ww xY w# t8        $ r }t        j;                  |      }Y d}~d}~ww xY w# t8        $ r }t        j;                  |      }Y d}~d}~ww xY w# t8        $ r }t        j;                  |      }Y d}~d}~ww xY w# t8        $ r }t        j;                  |      }Y d}~d}~ww xY w# t8        $ r }t        j;                  |      }Y d}~d}~ww xY w# t8        $ r }t        j;                  |      }Y d}~d}~ww xY w# t8        $ r }t        j;                  |      }Y d}~d}~ww xY w#  |}Y xY w# t8        $ r }t        j;                  |      }Y d}~d}~ww xY w# t8        $ r }t        j;                  |      }Y d}~d}~ww xY w# t8        $ r }t        j;                  |      }Y d}~wd}~ww xY w# t8        $ r }t        j;                  |      }Y d}~hd}~ww xY w# t8        $ r }t        j;                  |      }Y d}~@d}~ww xY w# t8        $ r }t        j;                  |      }Y d}~#d}~ww xY w# t8        $ r }t        j;                  |      }(Y d}~d}~ww xY w# t8        $ r }t        j;                  |      }(Y d}~d}~ww xY w# t8        $ r }t        j;                  |      }+Y d}~d}~ww xY w# t8        $ r }t        j;                  |      }+Y d}~d}~ww xY w# t8        $ r;}t        j;                  |      |,dB<   t        j;                  |      |,dC<   Y d}~od}~ww xY w# t8        $ r#}t        j;                  |      |"dE<   Y d}~gd}~ww xY w# t8        $ r#}t        j;                  |      |"dF<   Y d}~qd}~ww xY w# t8        $ r#}t        j;                  |      |"dI<   Y d}~{d}~ww xY w# t8        $ r }t        j;                  |      }4Y d}~d}~ww xY w# t8        $ r }t        j;                  |      }6Y d}~d}~ww xY w# t8        $ r&}t        j;                  |      }=|=}:|=};|=}<Y d}~d}~ww xY w# t8        $ r&}t        j;                  |      }=|=}:|=};|=}<Y d}~Gd}~ww xY w# t8        $ r }t        j;                  |      }?Y d}~
d}~ww xY w# t8        $ r$}t        j;                  |      }=|=}C|=}DY d}~d}~ww xY w# t8        $ r"}t        j;                  |      }=|=}GY d}~d}~ww xY w# t8        $ r"}t        j;                  |      }=|=}HY d}~d}~ww xY w# t8        $ r$}t        j;                  |      }=|=}I|=}JY d}~d}~ww xY w# t8        $ r }t        j;                  |      }RY d}~Fd}~ww xY w# t8        $ r }t        j;                  |      }RY d}~Xd}~ww xY w# t8        $ r }t        j;                  |      }RY d}~jd}~ww xY w# t8        $ r }t        j;                  |      }VY d}~Xd}~ww xY w# t8        $ r }t        j;                  |      }XY d}~Zd}~ww xY w# t8        $ r }t        j;                  |      }ZY d}~\d}~ww xY w# t8        $ r }t        j;                  |      }ZY d}~^d}~ww xY w# t8        $ r }t        j;                  |      }\Y d}~`d}~ww xY w# t8        $ r$}t        j;                  |      }=|=}]|=}^Y d}~Rd}~ww xY w# t8        $ r }t        j;                  |      }5Y d}~,d}~ww xY w# t8        $ r }t        j;                  |      }aY d}~!d}~ww xY w# t8        $ r }t        j;                  |      }bY d}~"d}~ww xY w# t8        $ r }t        j;                  |      }cY d}~#d}~ww xY w# t8        $ r }t        j;                  |      }aY d}~d}~ww xY w# t8        $ r }t        j;                  |      }cY d}~d}~ww xY w# t8        $ r }t        j;                  |      }aY d}~d}~ww xY w# t8        $ r }t        j;                  |      }cY d}~d}~ww xY w# t8        $ r }t        j;                  |      }aY d}~d}~ww xY w# t8        $ r }t        j;                  |      }bY d}~d}~ww xY w# t8        $ r }t        j;                  |      }cY d}~d}~ww xY w# tb        $ r d }md }nY t8        $ r5}t        j;                  |      }mt        j;                  |      }nY d}~d}~ww xY w# t8        $ r }t        j;                  |      }sY d}~d}~ww xY w# t8        $ r#}t        j;                  |      od<   Y d}~d}~ww xY w# t8        $ r;}|jv                  tx        k(  rY d}~t        j;                  |      }xY d}~d}~ww xY w# t8        $ r }t        j;                  |      }uY d}~ud}~ww xY w# t8        $ rE}|jv                  tx        k(  rY d}~it        j;                  |      }|}~|}|}|}|}Y d}~d}~ww xY w# t8        $ r#}t        j;                  |      |d<   Y d}~d}~ww xY w# t8        $ r,}t        d|j                         z   dz          Y d}~|S d}~ww xY w)a  
      Provides a Python interface to GPU management and monitoring functions.

      This is a wrapper around the NVML library.
      For information about the NVML library, see the NVML developer page
      http://developer.nvidia.com/nvidia-management-library-nvml

      Examples:
      ---------------------------------------------------------------------------
      For all elements as a list of dictionaries.  Similiar to nvisia-smi -q -x

      $ DeviceQuery()

      ---------------------------------------------------------------------------
      For a list of filtered dictionary elements by string name.
      Similiar ot nvidia-smi --query-gpu=pci.bus_id,memory.total,memory.free
      See help_query_gpu.txt or DeviceQuery("--help_query_gpu") for available filter elements

      $ DeviceQuery("pci.bus_id,memory.total,memory.free")

      ---------------------------------------------------------------------------
      For a list of filtered dictionary elements by enumeration value.
      See help_query_gpu.txt or DeviceQuery("--help_query_gpu") for available filter elements

      $ DeviceQuery([NVSMI_PCI_BUS_ID, NVSMI_MEMORY_TOTAL, NVSMI_MEMORY_FREE])

      Nr  r  r  r  r  rp   rq   rr   r   idproduct_nameproduct_brandr  r  r{   r|   r}   accounting_modeaccounting_mode_buffer_sizeFr  r  
current_dmT
pending_dmdriver_modelru   rw   rh  r~   rk  multigpu_boardrJ  rI  r  board_idimg_version
oem_object
ecc_object
pwr_objectinforom_versioncurrent_gompending_gomgpu_operation_modez%02Xpci_bus
pci_devicez%04X
pci_domainr  pci_device_id
pci_bus_idpci_sub_system_idmax_link_gencurrent_link_genpcie_genr  max_link_widthcurrent_link_widthlink_widthspci_gpu_link_infor   r  r  bridge_chip_typebridge_chip_fwpci_bridge_chipreplay_countertx_utilzKB/stx_util_unitrx_utilrx_util_unitrt  	fan_speed%fan_speed_unitr  performance_stateclocks_throttler  r   r  freeMiBunitfb_memory_usagebar1_memory_usager  r  r  r  r   r   r  memory_utilr  r  r  current_eccpending_eccecc_mode
ecc_errorsretired_pagesgpu_tempgpu_temp_max_thresholdgpu_temp_slow_thresholdCr  r  power_managementr  
power_drawpower_limitdefault_power_limitenforced_power_limitr   min_power_limitmax_power_limitr  power_stateWr  graphics_clocksm_clock	mem_clockMHzr   applications_clocksdefault_applications_clocks
max_clocksr  r  
auto_boostauto_boost_defaultclock_policyrf  supported_graphics_clockr=  supported_clocksrT  process_nameused_memory	processesr  r  r  r  max_memory_usager   r  accounted_processesr  r  r  )r   r  r@  r   r   r  r  r  r  r  r  r   r  r  r  r  r  r   r  r   r   r  r  r  r  r  r  r  r   r   r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r   r  r  r  r  r  r  r  r  r	  %_nvidia_smi__GetClocksThrottleReasonsr  r  r  r  r   r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r   r!  r"  r#  r$  r%  _nvidia_smi__GetEcc_nvidia_smi__GetRetiredPagesr(  r)  r*  r+  r,  r-  r.  r/  r0  r1  r2  r3  r4  r5  r6  r7  r8  r9  r:  r;  r<  r=  r>  r?  r@  rA  rB  rC  rD  rE  rF  rG  rH  rI  rJ  rK  rL  rP  rM  rN  rO  rP  r   r&  rQ  rR  rS  rT  r  rU  rV  intrW  rX  rY  rZ  r[  r\  r   r]  printr  )r   r   r^  nvidia_smi_results
dictResultr   r   
gpuResultsr   r_  ra  r   rb  r`  rc  rd  re  rf  rg  ru   rw   rh  ri  rj  rk  rl  rm  rn  ro  rp  r(  rq  rr  rs  rt  ru  rv  rw  rx  ry  rz  r{  r|  r}  r~  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r7  bar1MemoryUsager  r  r  r  r  r  r  r  r  r  r  r  r  r  r  rR  rS  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r   r  r  r  r  r  r  r  r  r  r  clockPolicyr  r  r  r  supportedClocksr  r  supportMemClocksupportedGraphicsClocksr  r  r  r   rs   processInfor  accountProcessrT  r  rZ  r[  r\  r   r  accountProcessInfos                                                                                                                                       r   r   znvidia_smi.DeviceQuery  sz!   < .;&vs# fn++333,,,c2 $c$ $ "99&Af<&jq<6!_%>.8.C.CHMMDWDWDY.Z -6!%9V%C3=3H3HIcIe3f !12*,+6![F%:*5 )K( a0aj~~a(f,V4gv%)9V)C#-#8#8#G*T"v%v)=-7-B-BCTU[C\-]*^,@"34Fv4N"Oi /8*_- v%);v)E<-Ef-MQR-RyYce .3*^,v%)=)G<-G-OST-Ty[ee 05*-.v%)?6)I;+G+OST+TY[ed 26*/0v%F)B;+Fv+NRS+SYZdd 15*./v%)?6)IA#-#8#89Z[a9b#cj =G*:;k#( v%)?6)I>+J6+RVf+fmrg -4+l+'+$v%)Cv)M>+J6+RVf+fmrg /6+l+'+$#/:*^,v%);v)E=26:f *4)>)>v)F*X&v%v)=;.v6d (2'<'<T'B*V$v%)>C%=f%El 0:/D/D\/R*^,v%F)B<7?e 1;0E0Ee0L*_-v%F)BF'A&'I #e+7<
#34$7<
#347;
#34v%6)A> 4V <g'%/f ,2*Z(!n$nv%):f)D:<VDc 3=2G2G2L./#'.v%):f)D:7@PQc 2<1F1Fs1K..#'.v%):f)D:7@PQc 2<1F1Fs1K..#'.v%):f)D:7@RSc 2<1F1Fs1K..#'.2@*./!# jv%):f)D> * 5 56Z[a6b cg 5?4I4I'4R"=1#*v%):f)D> * 5 56Z[a6b cg 5?4I4I'4R"=1#*5E*12c jv%&)@#)GKK#7#i.#*v%)9V)C&,w~~&=#l##*v%)9V)C&,w~~&=#l##*v%)<)F)/73F3F)G#o&#*v%)9V)C&0&;&;GMM&J#l##*v%)?6)I-3w7M7M-N#)*#*!n %of jv%)?6)I:&112TU[2\]c ,/&(#*v%)?6)I:&112UV\2]^c 03&+,#*/5.,$(/ m"'v%)AV)K<(334QRX4YZ]``e 5:- 01&*#v%)AV)K<(334RSY4Z[^aae 9>- 45&*#"2?./$(/ -;#)*#* m"'v%V(CF(Ko$&k(77:??1D%.+%/+:Em$67(77:DDI).,)/?3Q3QRS3T3^3^)_,8B8M8Ml8[m$45
 '+#"+8#'(#*v%L=fEf.8.C.CF.Kc*+  $*v%E!<VE\!]h'/c)n,2c.)  $*v%E!<VE\!]h'/c)n,2c.)  $*%(*U#v%F)B:1&9c ,/*[)03*-.v%)?C"9&"Ai%*Y%6l 4@*01 v%8FB5?3v=:fD9VC:fDBfLDNBfL9VC1;1V1VW]1^*./ m#( v%$.#v-#v-'+$	' 7 ?g")--$"6"=i!(!4t!;h!(!5!<w||d?RUY?Y!Yh  6)-?6-I-6mG,6)->&-H,4mF+6)->&-H,4mF+#(--'2?*./v%):f)D	' ;F Cg")"3"3d":T"Ai!(!1!1D!84!?h!(!2!2T!9D!@7CSCSVZCZ]aCa!ah %'/+4/'**2/&)*2/&)*//&)4C*01v%);v)E>5f=d!99$-'#DD$6'#>>$0'#EE$7'$-' 07*^,k#( v%'61'61':6Bd!%h!%h  6)-Bf-L08k*-6)-Bf-L3;k-0'+$v%)Bf)L+*I&*Q'x%-l
 1=+n-'+$v%)Bf)L+*I&*Q'x%-l
 1=+n-'+$#(++f%.9*]+v%$.(F2'+?+G(w/6!|*h/6!|*h '6)-?6-I/7gm,6)-Cv-M/7gm,+2*Z( .8-@-@-P(i(!-6*\*2<2N2NvW]2^/l/$0<*_-k#( v%)>&)H;5f>RSd -1+j)'+$;>vGjkd ;?+67'+$;>vGjkd <@+78'+$#(++f%.9*]+!n%*"v%)9V)C@?Gf171+%i 8A.!34)-&v%)9V)CA 7 ?& Hg#*j 2<..)-&v%):f)DB"CF"Kf"Th$,k 3>./)-&v%)Bf)LB"J6"RU["[h$,k ;F.!67)-&v%)Cv)MJ*I&*QTZ*Z&,<) <O.!78)-&v%'61'61-!Mf!Uh'/{V';n'/{V';n  6)-Bf-L:Hn%676)-Bf-L:Hn%67)-&%@"%
(=(=>UV\>](^"^i 3<./+..(1?*-.f#mf$(AV(K?!7@S!Th .6&)*"&-f$(AV(K91&-Hb (*&$"&-f$(?6(I:26>Jc ),&%"&-#(&.)/*X&"$!&f$(Bf(L?!?H[!\h 9A#$45%)"f$(@F(J::6>Rc 47#K0%)"!.3#F+6G*23)+&(-%f$(Jf(T?!FvOb!ch @H*+;<,0)f$(HF(R:A&.Yc ;>*;7,0)(5:*62>V*:;i!&f$(AV(K?!:6CV!Wh 19),-%)"f$(;v(E94V]Kb +-)J'%)"f$(?6(I:5fnMc ,/)K(%)"!&+)F#-6*\*v%)<)F "+J:_`f:g7l$7%)>>).,)-,,0EE05-04- /;+l+6I+23/:*^,v%)?6)I$&/O"DV"Li( D!,./78/)449/&1461V'KFTU'Vf'- D!"9"@"@"CD
 I`/*DE)00/CD* GJ/FZ]^F^?dh*/0v%);v)E )@B6Je  % :!G%/%:%:;STUTYTY;Z%[d )++/0uu+e,8<+n5oo5$%c$'$(>(E$Fc7:+m405+f-#**K9':2 :=Y!9KIQU*[)v%)=)G(X8@d')n!% F#/&B63&Oe/69M9M/Mn29E<S<S2S/"'"6"6$">38.3;u?S?SVZ?Z]a?a3b.%,uzz%9d+/%//+Aj 02,6:Sj,U3;I,Z8>O,];CQ,-?@7;,V4=G,\:(//1CE=F@ MPP^L_bcLc.imj!67 j/A%##J/Ca0F *o!(2 '
  y$ $B # @",":":3"?i@ # <(66s;e< # <(66s;e< # ;'55c:d; # ;'55c:d; # A#-#;#;C#@jA # > * 8 8 =g> # > * 8 8 =g> # =)77<f= # ;'55c:d; # C%/%=%=c%BlC # <(66s;e< $ F'1'?'?'DF # > * 8 8 =g>
'&f # :&44S9c: # :&44S9c: # :&44S9c: # :&44S9c: # > * 8 8 =g> # > * 8 8 =g>X # :&44S9c: # :&44S9c:  # <(66s;e< # <(66s;e<@ # V:D:R:RSV:Wm$678B8P8PQT8Um$45V # L.8.F.Fs.Kc*+L # E'1'?'?'Dc)nE # E'1'?'?'Dc)nE # :&44S9c: # C%/%=%=c%BlC> # '(66s;e"'i!&h!&h	'2 # '(66s;e"'i!&h!&h	'4 # > * 8 8 =g> # '(66s;e!&h!&h'" # +(66s;e%*l+ # +(66s;e%*l+$ # '(66s;e!&h!&h': # ;'55c:d; # ;'55c:d; # ;'55c:d;  # @",":":3"?i@ # A#-#;#;C#@jA # B$.$<$<S$AkB # B$.$<$<S$AkB # J,6,D,DS,I)J # -(66s;e',n',n- # @",":":3"?i@ # ?!+!9!9#!>h? # 9%33C8b9 # :&44S9c: # ?!+!9!9#!>h? # :&44S9c: # ?!+!9!9#!>h? # :&44S9c: # ?!+!9!9#!>h? # 9%33C8b9 # :&44S9c:. 0 2%*l,1)" J%/%=%=c%Bl,6,D,DS,I)J. "+ V8B8P8PQT8U5V # O1;1I1I#1Nog.O "+ G"%))/C"C"*)3)A)A#)F$G& # @",":":3"?i@, "+ 	/"%))/C"C"*$.$<$<S$Ac/2n25//2n%(d+.j	/. # X:D:R:RSV:Wj!67X  <
"S[[]2T9
;
;<s  AMD"A| 3AMA| AM;-A| AN'A| +AO=A| AO?%A| ;AP+A| 3AQ	A| !AR7A| AR/!(A| 
AS(A| >AT	(A| 2AT3=(A| &AU1(A| AV&AV7+A| AW*A| ;AW-*A| 6AX*A| 1AY5A| 7AY1*A|  AZD%A| A[	"A| :A[5$A| =!A\!A| 6!A]%A| =BA]9	A| #A_  
A| A_/%
A| 0A`
A| $Aa/A| 
Aa9BA| 6AAb%
AA| &AAc:+A| &=Ad	#!A| %Ad5*=A| (Ae%9A| Af#-A| !Ag2B
A| =Ag1A| Ah)A| 1Ai	(A| ,Ai5A| Aj!+A| AkA| .Ak9?A| Al%)!A| Am'0A| !An9$A| An-0A| 	AoA| 4Ap(A| /Ap1A| Aq,(A| Ar	'A|  Ar5(A| ;As!@A| @&At@8A| AAt9A#&A| B
-Au%B7"A| CAw"C:&Av6D Aw"D:*A| E%AyE7*AxF!A%AyH(A| H/A{IA6AzJ9AA{LA| L)A| MAMM	AM8MAM3M-A| M3AM8M8A| M;	AN$NANNA| NAN$N$A| N'	AON0AOOA| OAOOA| O	AO<OAO7O1A| O7AO<O<A| O?	AP(PAP#PA| P#AP(P(A| P+	AQP4AQQ	A| QAQQA| Q	AR Q AQ;Q5A| Q;AR R A| R	AR,RAR'R!A| R'AR,R,A| R/	ASR8ASSA| SASSA| S	ATS$AS?S9A| S?ATTA| T	AT0TAT+T%A| T+AT0T0A| T3	AUT<AUUA| UAUUA| U	AVU(AVU=A| VAVVA| V	AV4VAV/V)A| V/AV4V4A| V7AV>V;A| W	AW*W
AW%WA| W%AW*W*A| W-	AXW6AXXA| XAXXA| X	AYX"AX=X7A| X=AYYA| Y	AY.YAY)Y#A| Y)AY.Y.A| Y1	AZY:AZZA| ZAZZA| Z	A[Z&A[Z;A| [A[[A| [		A[2[A[-['A| [-A[2[2A| [5	A\[>A\\A| \A\\A| \!	A]
\*A]\?A| ]A]
]
A| ]	A]6]A]1]+A| ]1A]6]6A| ]9	A^=^0A^8^2A| ^8A^=^=A| _ 	A_,_	A_'_!A| _'A_,_,A| _/	A`_8A``A| `A``A| `	Aa
`'Aa`?A| aAa
a
A| a	Aa6aAa1a+A| a1Aa6a6A| a9	Ab"bAbbA| bAb"b"A| b%	Acb.Acc	A| cAccA| c	Adc Adc;A| dAddA| d		Ad2dAd-d'A| d-Ad2d2A| d5	Ae"d>AeeA| eAe"e"A| e%	Afe.AffA| fAffA| f	Af>fAf9f3A| f9Af>f>A| g	Ag.g
Ag)g#A| g)Ag.g.A| g1	Ahg:AhhA| hAhhA| h	Aih&Aih;A| iAiiA| i		Ai2iAi-i'A| i-Ai2i2A| i5	Aji>AjjA| jAjjA| j!	Ak
j*Akj?A| kAk
k
A| k	Ak6kAk1k+A| k1Ak6k6A| k9	Al"lAllA| lAl"l"A| l%	Aml.Am	mA| m	AmmA| m	Am>mAm9m3A| m9Am>m>A| n	An*n
An%nA| n%An*n*A| n-	Aon6AooA| oAooA| o	Apo"Ao=o7A| o=AppA| p	Ap.pAp)p#A| p)Ap.p.A| p1	Aqp:AqqA| qAqqA| q	Arq&Arq;A| rArrA| r		Ar2rAr-r'A| r-Ar2r2A| r5	Asr>AssA| sAssA| s!	At
s*Ats?A| tAt
t
A| t	At6tAt1t+A| t1At6t6A| t9	Au"uAuuA| uAu"u"A| u%Av3u3A| u6Av3u>*Av.v(A| v.Av3v3A| v6	Awv?AwwAw"wAwwAw"w"	Axw+Ax	xA| x	AxxA| x	AyxAyx/Ayx5Ayy
AyyAyyAyy	Azy!Ay<y6A| y<AzzA| z	A{zA{z"A{z(A{{A{{A{{A{{	A|{A{<{6A| {<A||A| |	A|9|!A|4|4A|9c           
         d}	 |j                         D ]  \  }}t        |t        j                        rMt	        |j                               dkD  r#|||dz  }|| j                  |d|z         z  }_|||dddz  }mt        |      t        u rkt        |d   t        j                        rNt        dt	        |            D ]5  }|d|||d	z   t	        |      fz  z  }|| j                  ||   d|z         z  }7 |||dt        |      dz  } 	 |S # t        $ r}|d
t        |      z   z  }Y d }~|S d }~ww xY w)Nr   r   z:
  : Noner  %s%s: [%d of %d]
r   z	
[Error] )itemsr  collectionsMappingr&  r   _nvidia_smi__to_str_dictionaryr  r  r   r@  r   )r   r  indent
strResultskeyry  r   es           r   __to_str_dictionaryznvidia_smi.__to_str_dictionary
  sK   j. Dgc#C!4!45&* $<<j D$<$<S$v+$NNj F3v$FFjI%Js1v{?R?R,S 3s8, Qa %9F3!CPSH;U$UUj D$<$<SVDK$PPjQ fS#c(CC
D"   .
s1v-
-*.s   DD 	D>"D99D>c           
      (   d}d}|j                         D ]z  \  }}t        |      t        u rNt        dt	        |            D ]5  }|d|||dz   t	        |      fz  z  }|| j                  ||   d|z         z  }7 e|||dt        |      dz  }| |S )Nr   r)  r   r,  r   r*  r  )r-  r  r  r   r&  r0  r@  )r   r   r2  r1  r3  ry  r   s          r   __to_strznvidia_smi.__to_str
  s    jf]]_ B'#c#Y$Qs3x( M!!5sAaCC7Q QQ* 8 8QV LL*M F3CAAjB r   c                 J    t        |      t        u r|S | j                  |      S r   )r  r@  _nvidia_smi__to_str)r   r   s     r   formatznvidia_smi.format
  s"    	g#	.]]7##r   r   r   )"r   r   r   r   r   r   r   r   r   r   r   r   r  r  r  r  r.  r&  r  rN  rY  r'  r  r  r
  r  r   r   classmethodr  r   r0  r9  r:  r   r   r   r   r   O  s   *),$ ,$\ # # K K?     - -^ / /b " "H  B &7 &7P "@ "@H 6 6. 8 84 R R8 [ [4    : V V<     h hT _  _ B,$r   r   __main__c                  t   d} d }t        dt        t        j                              D ];  }t        j                  |   }|j	                         dk(  rd} ,|j	                         }= t
        j                         }| r|j                  |      }n|j                  |      }t        |j                  |             y )NFr   xmlT)r   r&  sysargvlowerr   r   r  r   r  r:  )as_xmlquery_gpu_argsr   vr   r   s         r   mainrE  
  s    qSXX' 	+AA	U"!"	+ &&(**>:G''7Gell7#$r   )r  pynvmlr  r.  r   	threadingr   r   r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  NVSMI_GOM_CURNVSMI_GOM_PENDINGr  r  r  r  r  r  r  r  r  r  r!   NVSMI_ENCODER_STATS_SESSIONCOUNTNVSMI_ENCODER_STATS_AVG_EPSNVSMI_ENCODER_STATS_AVG_LATENCYr#  r$  r   r  r  r  r  (NVSMI_ECC_ERROR_CORRECTED_VOLATILE_TOTALr  r  r  r  r	  r
  r  r  r  r  r  r  r  r   r!  r"  r#  r$  rL  rM  rO  r   r  r  r  r  r  r  r  r  r	  r(  NVSMI_TEMPERATURE_MEMORYr.  r0  r2  r6  r4  r8  r9  r;  NVSMI_CLOCKS_SM_CURr?  NVSMI_CLOCKS_VIDEO_CURrA  rC  rD  rF  rG  rI  rJ  rQ  rW  rK  rN  r   NVML_BRAND_UNKNOWNNVML_BRAND_QUADRONVML_BRAND_TESLANVML_BRAND_NVSNVML_BRAND_GRIDNVML_BRAND_GEFORCEr  r   r   rE  r   r   r   <module>rW     s=
  :&      	 
 
                             #&  ! "%   -/ *-/ *-/ *-/ *-/ *+- (.0 +.0 +.0 +.0 +.0 +,. )/2 ,/2 ,/2 ,/2 ,/2 ,-0 *03 -03 -03 -03 -03 -.1 +*- '*- '  ), &&) #$' !+. (*- '+. (36 058 236 0*- '                 %( "#&         ) K :	
 z  # : z ! ! ! m ! '  .!" 2#$ .%& 6'( 2)* K+, %-. )/0 -12 o34 356 178 59: O;< #=> %?@ #AB #CD #EF %GH -IJ !KL #MN !!2OP QR \ST ((NUV %%HWX ''HYZ ::b[\ ++R]^ **R_` 22bab 66hcd 22bef ))Pgh %ij #kl #mn %op +qr .st !!Auv :wx ##Byz ){| -}~ 22\@ 22\AB --WCD --WEF 33]GH **RIJ 33^KL 33^MN ..YOP ..YQR 44_ST ++TUV 44`WX 44`YZ //[[\ //[]^ 55a_` ,,Vab 55bcd 55bef 00]gh 00]ij 66ckl --Xmn ))Pop ?qr %%Lst ?uv 6wx +yz 1{| '}~ !@ #AB 5CD 3EF +GH +IJ 7KL )MN +OP #QR 3ST (UV 1WX )YZ ##=[\ 7]^ !!9_` 6ab ++Mcd %%Gef ))Igh &&Fij 3kl -mn 'op (/,-%) }B (9&8%7#5$6'9 U&$ U&$pL z%( 	F+ r   