
    bi                         d dl Z d dlmZmZmZ ddlmZmZmZ ddl	m
Z
 ddlmZ ddlmZ dd	lmZ erd dlZd dlZ G d
 de      Z G d d      Zy)    N)TYPE_CHECKINGOptionalUnion   )DatasetFeaturesconfig)query_table)Sql)tqdm   )AbstractDatasetInputStreamc                   `     e Zd Z	 	 	 ddeedf   deedddf   dee   ded	ef
 fd
Zd Z	 xZ
S )SqlDatasetReadersqlzsqlalchemy.sql.Selectableconsqlalchemy.engine.Connectionsqlalchemy.engine.Enginesqlite3.Connectionfeatures	cache_dirkeep_in_memoryc                 V    t        |   d|||d| t        d||||d|| _        y )N)r   r   r   )r   r   r   r    )super__init__r   builder)selfr   r   r   r   r   kwargs	__class__s          J/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/datasets/io/sql.pyr   zSqlDatasetReader.__init__   sG     	i(iP^ibhi 
	

 
    c                     d }d }d }d }| j                   j                  ||||       | j                   j                  d|| j                        }|S )N)download_configdownload_modeverification_mode	base_pathtrain)splitr&   	in_memory)r   download_and_prepare
as_datasetr   )r   r$   r%   r&   r'   datasets         r!   readzSqlDatasetReader.read$   sj     	))+'/	 	* 	
 ,,))->$J]J] * 
 r"   )NNF)__name__
__module____qualname__r   strr   r   boolr   r.   __classcell__)r    s   @r!   r   r      si    
 (,$
3334
 368RThhi
 8$	

 
 
$r"   r   c                   f    e Zd Z	 	 ddededeedddf   dee   d	ee   f
d
ZdefdZ	d Z
defdZy)SqlDatasetWriterNr-   namer   r   r   r   
batch_sizenum_procc                     ||dk  rt        d| d      || _        || _        || _        |r|nt        j
                  | _        || _        || _        y )Nr   z	num_proc z must be an integer > 0.)	
ValueErrorr-   r7   r   r	   DEFAULT_MAX_BATCH_SIZEr8   r9   to_sql_kwargs)r   r-   r7   r   r8   r9   r=   s          r!   r   zSqlDatasetWriter.__init__9   sZ     HMy
2JKLL	(2*8U8U *r"   returnc                     | j                   j                  dd       }| j                   j                  dd       }| j                   j                  dd      } | j                  dd|i| j                   }|S )Nr   r   indexFr   )r=   pop_write)r   _r@   writtens       r!   writezSqlDatasetWriter.writeL   sm    ""5$/""5$/""&&w6$++@E@T-?-?@r"   c                 X   |\  }}}|dkD  ri |ddin|}t        | j                  j                  t        ||| j                  z         | j                  j
                        }|j                         } |j                  | j                  | j                  fd|i|}|xs t        |      S )Nr   	if_existsappend)tablekeyindicesr@   )r
   r-   dataslicer8   _indices	to_pandasto_sqlr7   r   len)r   argsoffsetr@   r=   batchdfnum_rowss           r!   
_batch_sqlzSqlDatasetWriter._batch_sqlT   s    '+$}DJQJ@=@+x@Ta,,##fft67LL))

 __299TYYOOO"3r7"r"   c                 ^   d}| j                   | j                   dk(  rTt        t        dt        | j                        | j
                        dd      D ]  }|| j                  |||f      z  } |S t        | j                        | j
                  }}t        j                  | j                         5 }t        |j                  | j                  t        d||      D cg c]  }|||f	 c}      ||z  r||z  dz   n||z  dd      D ]  }||z  }	 	 ddd       |S c c}w # 1 sw Y   |S xY w)zWrites the pyarrow table as SQL to a database.

        Caller is responsible for opening and closing the SQL connection.
        r   Nr   bazCreating SQL from Arrow format)unitdesc)totalrZ   r[   )
r9   hf_tqdmrangerQ   r-   r8   rW   multiprocessingPoolimap)r   r@   r=   rD   rS   rV   r8   pools           r!   rB   zSqlDatasetWriter._write`   s>   
 == DMMQ$6!aT\\*DOO<5 K
 4??FE=+IJJK(  $'t||#4doojH %%dmm4 
( 'IIFKAxYcFdeF&%7e ;CZ:O8z1Q6U]akUk9! 	(H x'G	(
(  f	
( s   2+D"D))D"D""D,)NN)r/   r0   r1   r   r2   r   r   intr   rE   rW   rB   r   r"   r!   r6   r6   8   su     %)"&++ + 368RThhi	+
 SM+ 3-+&s 
# r"   r6   )r_   typingr   r   r    r   r   r	   
formattingr
   packaged_modules.sql.sqlr   utilsr   r]   abcr   sqlite3
sqlalchemyr   r6   r   r"   r!   <module>rl      sE     1 1 ( ( $ * # + $1 $ND Dr"   