
    bi                         d dl Z d dlmZ d dlZd dlZddlmZ ddl	m
Z
 ddlmZ  G d d	eeej                  ef         Zy)
    N)Mapping   )config)
map_nested   )TensorFormatterc                        e Zd Zd fd	Zd Zd Zd ZdefdZde	j                  defd	Zde	j                  dej                  fd
Zde	j                  defdZ xZS )NumpyFormatterc                 6    t         |   ||       || _        y )N)featurestoken_per_repo_id)super__init__np_array_kwargs)selfr   r   r   	__class__s       [/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/datasets/formatting/np_formatter.pyr   zNumpyFormatter.__init__   s    (>OP.    c                     t        t              rVr)t        fdD              rt        j                        S t        j
                  t              t              }|d d  |S S )Nc              3      K   | ]Z  }t        |t        j                        xr: |j                  d    j                  k(  xr |j                  d    j                  k(   \ yw)r   N)
isinstancenpndarrayshapedtype).0xcolumns     r   	<genexpr>z.NumpyFormatter._consolidate.<locals>.<genexpr>!   sU      lm
1bjj)gagg.HgQWWX^_`XaXgXgMggs   A A#)r   )r   listallr   stackemptylenobject)r   r   outs    ` r   _consolidatezNumpyFormatter._consolidate   s^    fd## qw  xx''
 hhs6{&9A
r   c                    t        |t        t        t        d       f      r|S t        |t        j
                  t        j                  f      r0t	        j                  |j                  t        j
                        r|S t        |t        j                        r|S i }t        |t        j                        rAt	        j                  |j                  t        j                        rdt        j                  i}nZt        |t        j                        r@t	        j                  |j                  t        j                        rdt        j                  i}t        j                  rVdt         j"                  v rDdd l}t        ||j&                  j&                        r t	        j(                  |fi | j*                  S t        j,                  r&dt         j"                  v rddlm} t        ||      r|S t        j2                  r*dt         j"                  v rddlm}m} t        |||f      r|S t	        j(                  |fi i || j*                  S )Nr   PILr   torchvision)VideoReader
torchcodec)AudioDecoderVideoDecoder)r   strbytestyper   	characterr   
issubdtyper   numberintegerint64floatingfloat32r   PIL_AVAILABLEsysmodules	PIL.ImageImageasarrayr   TORCHVISION_AVAILABLEtorchvision.ior+   TORCHCODEC_AVAILABLEtorchcodec.decodersr-   r.   )r   valuedefault_dtyper)   r+   r-   r.   s          r   
_tensorizezNumpyFormatter._tensorize.   s   ec5$t*56Lbjj9:r}}U[[Z\ZfZf?gLryy)LeRZZ(R]]5;;

-S$bhh/Mrzz*r}}U[["++/V$bjj1MES[[$8%1zz%@4+?+?@@''MS[[,H2%-&&<3;;+FF%,!=>zz%M#Lm#Lt7K7K#LMMr   c                    t         j                  rjdt        j                  v rXdd l}t        ||j                        r>| j                  |j                         j                         j                         d         S t        |d      rIt        |t        j                  t        j                  t        j                  f      s|j!                         }t        |t        j                        rA|j"                  t$        k(  r.| j'                  |D cg c]  }| j)                  |       c}      S t        |t*        t,        f      r.| j'                  |D cg c]  }| j)                  |       c}      S | j                  |      S c c}w c c}w )Ntorchr    	__array__)r   TORCH_AVAILABLEr:   r;   rG   r   TensorrE   detachcpunumpyhasattrr   r   r2   r4   rI   r   r%   r'   recursive_tensorizer    tuple)r   data_structrG   	substructs       r   _recursive_tensorizez#NumpyFormatter._recursive_tensorizeO   s#   !!g&<+u||4{'9'9';'?'?'A'G'G'I"'MNN;,ZbjjZ\ZfZfhjhqhqMr5s%//1Kk2::.  F*((_j)kR[$*B*B9*M)kllkD%=1$$[f%gid&>&>y&I%ghh{++ *l%gs   F FrR   c                 2    t        | j                  |d      S )NF)map_list)r   rT   )r   rR   s     r   rP   z"NumpyFormatter.recursive_tensorize`   s    $33[5QQr   pa_tablereturnc                     | j                         j                  |      }| j                  j                  |      }| j	                  |      S N)numpy_arrow_extractorextract_rowpython_features_decoder
decode_rowrP   )r   rW   rows      r   
format_rowzNumpyFormatter.format_rowc   sB    ((*66x@**55c:'',,r   c                     | j                         j                  |      }| j                  j                  ||j                  d         }| j                  |      }| j                  |      }|S )Nr   )r[   extract_columnr]   decode_columncolumn_namesrP   r'   )r   rW   r   s      r   format_columnzNumpyFormatter.format_columnh   sd    ++-<<XF--;;FHDYDYZ[D\]))&1""6*r   c                     | j                         j                  |      }| j                  j                  |      }| j	                  |      }|D ]  }| j                  ||         ||<    |S rZ   )r[   extract_batchr]   decode_batchrP   r'   )r   rW   batchcolumn_names       r   format_batchzNumpyFormatter.format_batcho   sq    **,::8D,,99%@((/  	GK!%!2!253E!FE+	Gr   )NN)__name__
__module____qualname__r   r'   rE   rT   dictrP   paTabler   r`   r   r   re   rk   __classcell__)r   s   @r   r
   r
      sp    /NB,"Rt R-288 - -
bhh 2:: RXX ' r   r
   )r:   collections.abcr   rN   r   pyarrowrp    r   utils.py_utilsr   
formattingr   r   r
   rH   r   r   <module>rx      s:     #    ' '[_Wbjj'%AB [r   