
    bi>              	          d dl Z d dlZd dlZd dlZd dlmZ e j                  Ze j                  Ze j                  ZdZ	 ej                  d      d        Z ej                  d      d        Zej                  d        Zej                  d	        Zej                  d
        Zej                  d        Zej                  d        Zej                  d        Zej                  d        Zd Zd Zd Zd Zd Zd Zd Zd Zej8                  j;                  de j<                  e j>                  g      d        Z ej8                  j;                  de j<                  e j>                  g      d        Z!ej8                  j;                  de jD                  e jF                  e jH                  e jJ                  e jL                  e jN                  g      d        Z(d Z)d Z*d Z+d Z,d  Z-d! Z.ej8                  j;                  d"e j^                  e j`                  e jb                  e jd                  e jf                  e jh                  g      d#        Z5ej8                  j;                  d$d d%g      ej8                  j;                  d&g d'      d(               Z6ej8                  j;                  d)e jn                  e jp                  e jr                  e jt                  g      d*        Z;y)+    N)LooseVersionz$Legacy NVLink test expected to fail.module)scopec                 T    t        j                          d }| j                  |       y )Nc                  ,    t        j                          y N)pynvmlnvmlShutdown     W/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/pynvml_utils/tests/test_nvml.py
nvml_closeznvml.<locals>.nvml_close   s    r   )r	   nvmlInitaddfinalizer)requestr   s     r   nvmlr      s!    
OO $r   c                     t        j                         }t        |t              r|j	                  d      }t        |j                  d      d         S )Nzutf-8.r   )r	   nvmlSystemGetDriverVersion
isinstancebytesdecodeintsplit)r   
driver_vsns     r   driverr      sE    224J*e$&&w/
z$Q'((r   c                 t    t        j                         }|dkD  sJ t        dt        |      z   dz   d       |S )Nr   [z GPUs] end)r	   nvmlDeviceGetCountprintstr)r   results     r   ngpusr&   #   s9    &&(FA::	#F
h
&C0Mr   c                     t        |       D cg c]  }t        j                  |       }}t        |      | k(  sJ |S c c}w r   )ranger	   nvmlDeviceGetHandleByIndexlen)r&   ihandless      r   r,   r,   ,   sA    =B5\Jv003JGJw<5   N K   >c                 n    t        j                  | d         }t        dt        |      z   dz   d       |S )Nr   r   z MIGs]r   r    )r	   nvmlDeviceGetMaxMigDeviceCountr#   r$   )r,   r%   s     r   nmigsr0   4   s3    2271:>F	#F
h
&C0Mr   c                     t        |       D cg c]  }t        j                  |       }}t        |      | k(  sJ |S c c}w r   )r(   r	   #nvmlDeviceGetMigDeviceHandleByIndexr*   )r0   r+   r,   s      r   mig_handlesr3   <   sA    FKElSv99!<SGSw<5   N Tr-   c                     t        |       D cg c]  }t        j                  ||          }}t        |      | k(  sJ |S c c}w r   )r(   r	   nvmlDeviceGetSerialr*   )r&   r,   r+   serialss       r   r6   r6   C   sE    ?DU|L!v))'!*5LGLw<5   N M   Ac                     t        |       D cg c]  }t        j                  ||          }}t        |      | k(  sJ |S c c}w r   )r(   r	   nvmlDeviceGetUUIDr*   )r&   r,   r+   uuidss       r   r:   r:   J   sE    ;@<HaV%%gaj1HEHu:L Ir7   c                     t        |       D cg c]  }t        j                  ||          }}t        |      | k(  sJ |S c c}w r   )r(   r	   nvmlDeviceGetPciInfor*   )r&   r,   r+   pci_infos       r   r=   r=   Q   sE    AFuNA++GAJ7NHNx=E!!!O Or7   c                 v    d}t        j                         }t        d|z   dz   d       |t        d      kD  sJ y )N        z[NVML Version: ]r   r    0.0)r	   nvmlSystemGetNVMLVersionr#   r   r   vsns     r   test_nvmlSystemGetNVMLVersionrE   ]   s=    
C

)
)
+C	
c
!C
'S1e$$$$r   c                 >    d}t        j                         }|dk7  sJ y Nr?   )r	   nvmlSystemGetCudaDriverVersionrC   s     r   #test_nvmlSystemGetCudaDriverVersionrI   d   s!    
C

/
/
1C#::r   c                     d }t        j                  t        j                               }t	        dt        |      z   dz   d       |d k7  sJ y )Nz
[Process: r@   r   r    )r	   nvmlSystemGetProcessNameosgetpidr#   r$   )r   procnames     r   test_nvmlSystemGetProcessNamerO   k   sC    H..ryy{;H	,X
&
,#6tr   c                 v    d}t        j                         }t        d|z   dz   d       |t        d      kD  sJ y )Nr?   z[Driver Version: r@   r   r    rA   )r	   r   r#   r   rC   s     r   test_nvmlSystemGetDriverVersionrQ   s   s=    
C

+
+
-C	

#c
)s3e$$$$r   c                 p    | r | D ]  }t        j                  |      }|J  t        j                          y r   )r	   nvmlDeviceGetAttributespytestskip)r3   handleatts      r   test_nvmlDeviceGetAttributesrX      s;     ! 	#F008C?"?	# KKMr   c                     t        |       D cg c]  }t        j                  ||          }}t        |      | k(  sJ y c c}w r   )r(   r	   nvmlDeviceGetHandleBySerialr*   )r&   r6   r+   r,   s       r    test_nvmlDeviceGetHandleBySerialr[      s@    GLU|T!v11'!*=TGTw<5    U   A c                     t        |       D cg c]  }t        j                  ||          }}t        |      | k(  sJ y c c}w r   )r(   r	   nvmlDeviceGetHandleByUUIDr*   )r&   r:   r+   r,   s       r   test_nvmlDeviceGetHandleByUUIDr_      s@    CH<Pav//a9PGPw<5    Qr\   c                     t        |       D cg c]$  }t        j                  ||   j                        & }}t	        |      | k(  sJ y c c}w r   )r(   r	   nvmlDeviceGetHandleByPciBusIdbusIdr*   )r&   r=   r+   r,   s       r   "test_nvmlDeviceGetHandleByPciBusIdrc      sN    INuDE,,Xa[->->?G  w<5   s   )A
r   c                 L    d}| D ]  }t        j                  |||      }|J  y Ni   )r	   nvmlDeviceGetMemoryAffinity)r,   r   sizerV   nodeSets        r    test_nvmlDeviceGetMemoryAffinityri      s8     D #44VT5I"""#r   c                 L    d}| D ]  }t        j                  |||      }|J  y re   )r	   #nvmlDeviceGetCpuAffinityWithinScope)r,   r   rg   rV   cpuSets        r   (test_nvmlDeviceGetCpuAffinityWithinScoperm      s8     D ";;FD%P!!!"r   indexc                     | D ]Q  }| D ]J  }||ust        j                  |||      }t         j                  |cxk  rt         j                  k  rGJ  J  S y r   )r	   nvmlDeviceGetP2PStatusNVML_P2P_STATUS_OKNVML_P2P_STATUS_UNKNOWN)r,   rn   h1h2statuss        r   test_nvmlDeviceGetP2PStatusrv      sp       	B|66r2uE--655666		r   c                 b    t        |       D ]!  }t        j                  ||         }|dk\  r!J  y rG   )r(   r	   nvmlDeviceGetPowerUsage)r&   r,   r+   power_mWattss       r   test_nvmlDeviceGetPowerUsagerz      s7    5\ #55gajAs"""#r   c                     t        |       D ]k  }t        j                  ||         }t        d      D ]=  }t        j                  d       t        j                  ||         }||k\  sJ ||kD  s= e J d        y )N
   gQ?z.energy did not increase across 150 ms interval)r(   r	   #nvmlDeviceGetTotalEnergyConsumptiontimesleep)r&   r,   r+   energy_mJoules1jenergy_mJoules2s         r   (test_nvmlDeviceGetTotalEnergyConsumptionr      s    5\ 	K DDWQZPr 	KAJJu$HHQRTO"o5550	K KJJ5	Kr   c                     t        |       D ]N  }t        j                  ||         }|j                  |j                  k  r|j
                  |j                  k  rNJ  y r   )r(   r	   nvmlDeviceGetMemoryInfousedtotalfree)r&   r,   r+   meminfos       r   test_nvmlDeviceGetMemoryInfor     sP    5\ S00<-GLLGMM4QRRSr   c                     t        |       D ]<  }t        j                  ||         }|j                  dk\  sJ |j                  dk\  r<J  y Nr   )r(   r	   nvmlDeviceGetUtilizationRatesgpumemory)r&   r,   r+   urates       r   "test_nvmlDeviceGetUtilizationRatesr     sK    5\ !44WQZ@yyA~~||q   !r   c                     t        |       D ]n  }t        j                  ||   t              }|dk\  sJ t        j                  ||   t              }|dk\  sJ t        j                  ||   t
              }|dk\  rnJ  y r   )r(   r	   nvmlDeviceGetPcieThroughputNVML_PCIE_UTIL_TX_BYTESNVML_PCIE_UTIL_RX_BYTESNVML_PCIE_UTIL_COUNT)r&   r,   r+   tx_bytes_tprx_bytes_tpcount_tps         r    test_nvmlDeviceGetPcieThroughputr   L  s    5\ 
88AJ/
 a88AJ/
 a55gajBVW1}}
r   c                    |dkD  rt        j                  t               t        |       D ]  }t        t        j
                        D ]w  }t	        j                  ||   |      }|dk\  sJ t	        j                  ||   |      }|dk\  sJ t	        j                  ||   |      }t        |t        j                        rwJ   y )N      |@   r   )rT   xfailXFAIL_LEGACY_NVLINK_MSGr(   r	   NVML_NVLINK_MAX_LINKSnvmlDeviceGetNvLinkVersionnvmlDeviceGetNvLinkState nvmlDeviceGetNvLinkRemotePciInfor   c_nvmlPciInfo_t)r&   r,   r   r+   r   versionstater=   s           r   test_nvml_nvlink_propertiesr   a  s    ~,-5\ @v334 	@A77
AFGa<<33GAJBEA::>>wqz1MHh(>(>???	@@r   cap_typec                     t        |       D ]A  }t        t        j                        D ]#  }t        j                  ||   ||      }|dk\  r#J  C y r   )r(   r	   r   nvmlDeviceGetNvLinkCapability)r&   r,   r   r+   r   caps         r   test_nvml_nvlink_capabilityr   q  sT     5\ v334 	A66wqz1hOC!8O8	r   counterr   control)r   r      c           	         |dkD  rt        j                  t               d}t        |       D ]  }t        t        j
                        D ]  }t	        j                  ||   ||      t        j                  k(  sJ t	        j                  ||   ||||       t	        j                  ||   ||      }t	        j                  ||   ||      }	|d   dk\  sJ |d   dk\  sJ |	|k(  sJ t	        j                  ||   ||d      t        j                  k(  sJ t	        j                  ||   ||d      t        j                  k(  rJ   y )Nr   r   rxtxr   )rT   r   r   r(   r	   r   'nvmlDeviceResetNvLinkUtilizationCounterNVML_SUCCESS%nvmlDeviceSetNvLinkUtilizationControl%nvmlDeviceGetNvLinkUtilizationCounter%nvmlDeviceGetNvLinkUtilizationControl(nvmlDeviceFreezeNvLinkUtilizationCounter)
r&   r,   r   r   r   resetr+   r   	countdictctls
             r   test_nvml_nvlink_countersr     sj    ~,-E5\ v334 	A>>wqz1gV&&'' 88
Aw DD
AwI >>wqz1gVCT?a'''T?a''''>!>??AJ7A &&'' ??AJ7A &&''-	r   
error_typec                 6   |dkD  rt        j                  t               t        |       D ]m  }t        t        j
                        D ]O  }t	        j                  ||   |      t        j                  k(  sJ t	        j                  ||   ||      }|dk\  rOJ  o y )Nr   r   )	rT   r   r   r(   r	   r   "nvmlDeviceResetNvLinkErrorCountersr   nvmlDeviceGetNvLinkErrorCounter)r&   r,   r   r   r+   r   error_counts          r   test_nvml_nvlink_error_countersr     s     ~,-5\ 	$v334 	$A99'!*aH&&'' !@@
AzK !###	$	$r   )<r	   rT   r~   rL   distutils.versionr   r   r   r   r   fixturer   r   r&   r,   r0   r3   r6   r:   r=   rE   rI   rO   rQ   rX   r[   r_   rc   markparametrizeNVML_AFFINITY_SCOPE_NODENVML_AFFINITY_SCOPE_SOCKETri   rm   NVML_P2P_CAPS_INDEX_READNVML_P2P_CAPS_INDEX_WRITENVML_P2P_CAPS_INDEX_NVLINKNVML_P2P_CAPS_INDEX_ATOMICSNVML_P2P_CAPS_INDEX_PROPNVML_P2P_CAPS_INDEX_UNKNOWNrv   rz   r   r   r   r   r   NVML_NVLINK_CAP_P2P_SUPPORTEDNVML_NVLINK_CAP_SYSMEM_ACCESSNVML_NVLINK_CAP_P2P_ATOMICSNVML_NVLINK_CAP_SYSMEM_ATOMICSNVML_NVLINK_CAP_SLI_BRIDGENVML_NVLINK_CAP_VALIDr   r   NVML_NVLINK_ERROR_DL_REPLAYNVML_NVLINK_ERROR_DL_RECOVERYNVML_NVLINK_ERROR_DL_CRC_FLITNVML_NVLINK_ERROR_DL_CRC_DATAr   r   r   r   <module>r      s5      	 * 88  88 22 @  h%  % h)  )              %%!!! f--v/P/PQ## f--v/P/PQ"" ''(())**''**
	
	Z#
K$S !r*@  ,,,,**--))$$

 QF+I.! / ,!L **,,,,,,	$$r   