
    bi                     b    d dl mZ d dlZddlmZmZ ddlmZ ddlm	Z	 ddl
mZ  G d	 d
e      Zy)    )OptionalN   )Features
NamedSplit)DownloadMode)Spark   )AbstractDatasetReaderc                        e Zd ZdZ	 	 	 	 	 	 	 	 ddej
                  j                  dee   dee	   de
dede
ded	e
d
ef fdZd Z xZS )SparkDatasetReaderzA dataset reader that reads from a Spark DataFrame.

    When caching, cache materialization is parallelized over Spark; an NFS that is accessible to the driver must be
    provided. Streaming is not currently supported.
    dfsplitfeatures	streaming	cache_dirkeep_in_memoryworking_dirload_from_cache_filefile_formatc
           	      v    t        |   d|||||d|
 || _        |	| _        t	        d||||d|
| _        y )N)r   r   r   r   r   )r   r   r   r    )super__init___load_from_cache_file_file_formatr   builder)selfr   r   r   r   r   r   r   r   r   kwargs	__class__s              L/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/datasets/io/spark.pyr   zSparkDatasetReader.__init__   si     	 	
)	
 	
 &:"' 
#	

 
    c                 <   | j                   r&| j                  j                  | j                        S | j                  rd nt
        j                  }| j                  j                  || j                         | j                  j                  | j                        S )N)r   )download_moder   )
r   r   as_streaming_datasetr   r   r   FORCE_REDOWNLOADdownload_and_preparer   
as_dataset)r   r#   s     r    readzSparkDatasetReader.read1   s{    >><<444::4FF $ : :@]@]))')) 	* 	
 ||&&TZZ&88r!   )NNTNFNTarrow)__name__
__module____qualname____doc__pysparksql	DataFramer   r   r   boolstrr   r(   __classcell__)r   s   @r    r   r      s     '+'+$%)"
KK!!
 
#
 8$	

 
 
 
 
 #
 
>9r!   r   )typingr   r.    r   r   downloadr   packaged_modules.spark.sparkr   abcr
   r   r   r!   r    <module>r9      s$      # # 0 &.9. .9r!   