
    bi)                       U d dl mZ d dlZd dlmZmZmZ d dlmZmZm	Z	m
Z
mZmZmZmZmZmZmZmZmZ d dlmZ d dlmZmZmZ d dlmZ d dlmZmZ d dl m!Z! erd d	l"m#Z#m"Z" d d
l$m%Z% d dl&m'Z'm(Z( eedZ)de*d<   	 	 	 	 d	 	 	 	 	 	 	 	 	 	 	 	 	 ddZ+ddZ,edd       Z-e-j]                  e      dd       Z/e-j]                  e      d d       Z/e-j]                  e      d!d       Z/e-j]                  e      d"d       Z/e-j]                  e      d#d       Z/e-j]                  e      d$d       Z/e-j]                  e      d%d       Z/e-j]                  e      d&d       Z/ G d d      Z0y)'    )annotationsN)GtELtLtE)	AttributeBinOpBitAndBitOrCallCompareConstantEqGtInvertListNameUnaryOp)singledispatch)TYPE_CHECKINGAnyCallable)
to_py_dateto_py_datetime)	pyiceberg)datedatetime)Table)	DataFrameSeriesz)dict[str, Callable[..., datetime | date]]_temporal_conversionsc                   ddl m} | j                  ||      }| |j                  | }|(	 t	        |      }t        |      }	|j                  |	      } ||j                               S # t        $ r}
d| }t        |      |
d}
~
ww xY w)a"  
    Take the projected columns and materialize an arrow table.

    Parameters
    ----------
    tbl
        pyarrow dataset
    with_columns
        Columns that are projected
    predicate
        pyarrow expression that can be evaluated with eval
    n_rows:
        Materialize only n rows from the arrow dataset.
    snapshot_id:
        The snapshot ID to scan from.
    batch_size
        The maximum row count for scanned pyarrow record batches.
    kwargs:
        For backward compatibility

    Returns
    -------
    DataFrame
    r   )
from_arrow)limitsnapshot_idNz*Could not convert predicate to PyIceberg: )	polarsr"   scanselect_to_ast_convert_predicate
ValueErrorfilterto_arrow)tblwith_columns	predicaten_rowsr$   kwargsr"   r&   expr_astpyiceberg_expremsgs               S/home/cdr/jupyterlab/.venv/lib/python3.12/site-packages/polars/io/iceberg/_utils.py_scan_pyarrow_dataset_implr7   (   s    @ "88&k8:Dt{{L)	)y)H/9N
 {{>*dmmo&&  	)>ykJCS/q(	)s   A+ +	B
4BB
c                D    t        j                  | d      j                  S )a<  
    Converts a Python string to an AST.

    This will take the Python Arrow expression (as a string), and it will
    be converted into a Python AST that can be traversed to convert it to a PyIceberg
    expression.

    The reason to convert it to an AST is because the PyArrow expression
    itself doesn't have any methods/properties to traverse the expression.
    We need this to convert it into a PyIceberg expression.

    Parameters
    ----------
    expr
        The string expression

    Returns
    -------
    The AST representing the Arrow expression
    eval)mode)astparsebody)exprs    r6   r(   r(   \   s    * 99T',,,    c                "    d|  }t        |      )zJWalks the AST to convert the PyArrow expression to a PyIceberg expression.zUnexpected symbol: )r*   ar5   s     r6   r)   r)   t   s      s
#C
S/r?   c                    | j                   S N)valuerB   s    r6   _rG   {   s    77Nr?   c                    | j                   S rD   )idrF   s    r6   rG   rG      s    44Kr?   c                    t        | j                  t              r2t        j                  j                  t        | j                              S d|  }t        |      )NzUnexpected UnaryOp: )	
isinstanceopr   r   expressionsNotr)   operand	TypeErrorrA   s     r6   rG   rG      sG    !$$$$(();AII)FGG$QC(nr?   c                "   | j                   D cg c]  }t        |       }}t        | j                        }|dk(  r|S |dk(  r|d   S |t        v rt        |   | j	                         S t        | j                  j
                        d   }|dk(  r#t        j                  j                  ||d         S |dk(  rt        j                  j                  |      S |dk(  rt        j                  j                  |      S d|}t        |      c c}w )Nfieldscalarr   isinis_nullis_nanzUnknown call: )argsr)   funcr    	isoformatrE   r   rM   InIsNullIsNaNr*   )rB   argrW   frefr5   s         r6   rG   rG      s    /0vv6s#6D6166"AG|	
hAw	
#	#$Q'.88:: .q1;((++Ca99)^((//44(]((..s331%
 C
S/' 7s   Dc                    | j                   S rD   )attrrF   s    r6   rG   rG      s    66Mr?   c                Z   t        | j                        }t        | j                        }| j                  }t	        |t
              r t        j                  j                  ||      S t	        |t              r t        j                  j                  ||      S d| d| d| }t        |      )Nz	Unknown:  )r)   leftrightrL   rK   r	   r   rM   Andr
   OrrP   )rB   lhsrhsrL   r5   s        r6   rG   rG      s    
QVV
$C
QWW
%C	
B"f$$((c22"e$$''S11#at1SE*nr?   c                   | j                   d   }t        | j                        d   }t        | j                  d         }t	        |t
              r t        j                  j                  ||      S t	        |t              r t        j                  j                  ||      S t	        |t              r t        j                  j                  ||      S t	        |t              r t        j                  j                  ||      S t	        |t              r t        j                  j!                  ||      S d| }t#        |      )Nr   zUnknown comparison: )opsr)   rd   comparatorsrK   r   r   rM   GreaterThanr   GreaterThanOrEqualr   EqualTor   LessThanr   LessThanOrEqualrP   )rB   rL   rh   ri   r5   s        r6   rG   rG      s    	
qB
QVV
$Q
'C
Q]]1-
.C"b$$00c::"c$$77SAA"b$$,,S#66"b$$--c377"c$$44S#>>$RD)nr?   c                R    | j                   D cg c]  }t        |       c}S c c}w rD   )eltsr)   )rB   r4   s     r6   rG   rG      s     +,662aq!222s   $c                  @    e Zd Z	 	 	 	 	 	 ddZ	 	 	 	 	 	 	 	 ddZddZy))IdentityTransformedPartitionValuesBuilderc                d   dd l }ddlm} ddlm} ddlm}m}m}m	}	 |j                  }
i | _        i | _        i | _        |j                         }|j                         D ]  \  }}g }t!        |j"                        D ]a  \  }}|j$                  |
v st'        |j(                  |      s,|j+                  ||j$                  f       g | j                  |j$                  <   c || j                  |<    | j                  D ]"  }|j-                  |      }|j.                  }|j0                  sd| | j                  |<   t3        j4                   ||j6                  j5                  |                  j9                         \  }}|| j                  |<   |j;                         j=                         D ]o  }	 |j-                  |      j.                  }||k(  r%t'        ||	      rt'        ||      r>t'        |||f      rt'        |||f      r[d| d| | j                  |<   q % y # t>        $ r Y w xY w)Nr   )schema_to_pyarrow)IdentityTransform)
DoubleType	FloatTypeIntegerTypeLongTypeznon-primitive type: zunsupported type change: from: z, to: ) pyiceberg.schemapyiceberg.io.pyarrowrw   pyiceberg.transformsrx   pyiceberg.typesry   rz   r{   r|   	field_idspartition_valuespartition_values_dtypes(partition_spec_id_to_identity_transformsspecsitems	enumeratefields	source_idrK   	transformappend
find_field
field_typeis_primitiveplSchemaschemapopitemschemasvaluesr*   )selftableprojected_schemar   rw   rx   ry   rz   r{   r|   projected_idspartition_specsspec_idspecoutfield_indexrR   field_idprojected_fieldprojected_typerG   output_dtyper   type_this_schemas                           r6   __init__z2IdentityTransformedPartitionValuesBuilder.__init__   sM   
 	 ::	
 	
 #3"<"< =? @B$
  	5  ++-,224 
	IMGTC&/&< @"U??m3
OO%69 JJU__=>=?D))%//:@ FID99'B
	I -- #	H.99(CO,77N!..*>*:; %%h/ !ii!)"2"2"9"9/"JKgi A| 6BD((2--/002 '-'8'8'B'M'M$
 #&66">8<&'7E #>J	3JK&'7*i9PQ ::J9K L-.0 ))(3##	$ " s   1H##	H/.H/c          
     n   	 | j                   |   }|D ]h  \  }}||   }t        | j                  |   x}	t              s+|	j                  d t        |t        |	      z
        D               |	j                  |       j y # t        $ r, | j                  D ci c]  }|d| 
 nc c}w c}| _        Y y w xY w)Nzpartition spec ID not found: c              3      K   | ]  }d   y wrD    ).0rG   s     r6   	<genexpr>zRIdentityTransformedPartitionValuesBuilder.push_partition_values.<locals>.<genexpr><  s     OqdOs   )	r   KeyErrorr   rK   listextendrangelenr   )
r   current_indexpartition_spec_idr   identity_transformskisource_field_idpartition_valuer   s
             r6   push_partition_valuesz?IdentityTransformedPartitionValuesBuilder.push_partition_values$  s    		"&"O"O!# #6 	/A.q1OD$9$9/$JJ&DQ OE-#f+2M,NOOo.	/  	 ..% 23D2EFF% %D! 	s   A? ?B4B$#B43B4c                   ddl m}m}m}m}m}m} i }| j                  j                         D ]  \  }}	t        |	t              r|	||<   	 | j                  |   }
t        |
|||f      r|nt        |
|      r|n|
}t        j                  |	|      }t        |
|      r|dz  }|j                  |
      }|||<    |S # t        $ r}d| ||<   Y d }~d }~ww xY w)Nr   )DateDatetimeDurationInt32Int64Time)dtypei  z!failed to load partition values: )polars.datatypesr   r   r   r   r   r   r   r   rK   strr   r   r   cast	Exception)r   r   r   r   r   r   r   r   r   vr   constructor_dtypesr4   s                 r6   finishz0IdentityTransformedPartitionValuesBuilder.finish?  s    QQ*,00668 	LKHa!S! !HL#'#?#?#IL &lXx4NO  &lD9 #) & 		!+<=A!,5 H|,A$%CM1	L: 
 ! L&Gs$KCMLs   
A.B<<	CCCN)r   r   r   zpyiceberg.schema.SchemareturnNone)r   intr   r   r   zpyiceberg.typedef.Recordr   r   )r   zdict[int, pl.Series | str])__name__
__module____qualname__r   r   r   r   r?   r6   ru   ru      sZ    MM 2M 
	M^/ / 	/
 3/ 
/6"r?   ru   )NNNN)r-   r   r.   zlist[str] | Noner/   z
str | Noner0   
int | Noner$   r   r1   r   r   zDataFrame | Series)r>   r   r   zast.expr)rB   r   r   r   )rB   r   r   r   )rB   r   r   r   )rB   r   r   r   )rB   r   r   r   )rB   r   r   r   )rB   r   r   r   )rB   r   r   r   )rB   r   r   r   )1
__future__r   r;   _astr   r   r   r   r   r	   r
   r   r   r   r   r   r   r   r   r   	functoolsr   typingr   r   r   polars._reexport	_reexportr   polars._utils.convertr   r   polars.dependenciesr   r   r   pyiceberg.tabler   r%   r   r   r    __annotations__r7   r(   r)   registerrG   ru   r   r?   r6   <module>r      s   " 
      % / /  < )'%( $D @  &* "1'	1'"1' 1' 	1'
 1' 1' 1'h-0   X& ' T" # W% & T" #. Y' ( U# $ W% &( T"3 #3M Mr?   