U
    {hM                     @   s  d dl Z e jeeeeeeed d dlZd dlZd dlmZ d dlmZ ddl	m
Z
 ddlmZ ddlmZ d	d
lmZmZmZmZmZ d	dlmZmZmZ d	dlmZ d Zd Zd ZdZdadd Z dddddddddddddddddd d!d"d#d$d%d&d'd(d)d*d+d,d-gZ!e!d.d/d0d1d2d3d4d5d6g	 Z"G d7d8 d8Z#d9d: Z$G d;d< d<Z%G d=d> d>e%Z&G d?d@ d@e%Z'G dAdB dBeZ(ee(dCdDdEZ)dS )F    N)make_lexiconlexiconprint_functionerrorwarningosplatform)	normalize)contextmanager   )Utils)Scanner)UnrecognizedInput   )r   r   hold_errorsrelease_errorsCompileError)any_string_prefixr   IDENT)r   c                   C   s   t s
t a t S N)r   r    r   r   </tmp/pip-unpacked-wheel-fhl22ezh/Cython/Compiler/Scanning.pyget_lexicon    s    r   globalnonlocaldefclassprintdelpassbreakcontinuereturnraiseimportexectryexceptfinallywhileifelifelseforinassertandornotislambdafromyieldwithincludeZctypedefZcdefZcpdefZcimportZDEFZIFZELIFELSEc                   @   s>   e Zd ZdddZdd Zdd Zdd	 Zd
d Zdd ZdS )CompileTimeScopeNc                 C   s   i | _ || _d S r   )entriesouter)selfr<   r   r   r   __init__;   s    zCompileTimeScope.__init__c                 C   s   || j |< d S r   r;   )r=   namevaluer   r   r   declare?   s    zCompileTimeScope.declarec                 C   s   | j | d S r   )r;   updater=   otherr   r   r   rC   B   s    zCompileTimeScope.updatec                 C   s
   | j | S r   r?   r=   r@   r   r   r   lookup_hereE   s    zCompileTimeScope.lookup_herec                 C   s
   || j kS r   r?   rF   r   r   r   __contains__H   s    zCompileTimeScope.__contains__c                 C   s@   z|  |W S  tk
r:   | j}|r4|| Y S  Y nX d S r   )rG   KeyErrorr<   lookup)r=   r@   r<   r   r   r   rJ   K   s    zCompileTimeScope.lookup)N)	__name__
__module____qualname__r>   rB   rC   rG   rH   rJ   r   r   r   r   r:   9   s   
r:   c               	   C   s   t  } d}t|t D ]\}}| || qdd l}d}|D ]0}z| |t|| W q> tk
rl   Y q>X q>ddlm	} | d| | dt
 | dt|dt|d | d	t|d	t|d
 t | }|S )N)ZUNAME_SYSNAMEZUNAME_NODENAMEZUNAME_RELEASEZUNAME_VERSIONZUNAME_MACHINEr   )*FalseTrueabsallanyasciibinbool	bytearraybyteschrcmpcomplexdictdivmod	enumeratefilterfloatformat	frozensethashhexintlenlistmapmaxminoctordpowrangereprreversedroundsetslicesortedstrsumtuplezip)reducerx   unicodelongrd   xrangerm   )r:   rw   r   unamerB   builtinsgetattrAttributeError	functoolsrx   rt   )Zbenvnamesr@   rA   r}   rx   Zdenvr   r   r   initial_compile_time_envV   s$    r   c                   @   st   e Zd ZdZdZdZdZdZdZdd Z	dd	 Z
d
d Zdd Zdd Zdd Zdd Zdd Zdd Zdd ZdS )SourceDescriptorz<
    A SourceDescriptor should be considered immutable.
    NFpyx c                 C   s   dst d S NF)AssertionErrorr=   r   r   r   __str__   s    zSourceDescriptor.__str__c                 C   s.   t j|\}}|dkr$|dd  p&d| _d S )N)z.pyxz.pxdz.pyr   r   )r   pathsplitext
_file_type)r=   filenamer@   extr   r   r   set_file_type_from_name   s    z(SourceDescriptor.set_file_type_from_namec                 C   s
   | j dkS )N)r   Zpxdr   r   r   r   r   is_cython_file   s    zSourceDescriptor.is_cython_filec                 C   s
   | j dkS )Npyr   r   r   r   r   is_python_file   s    zSourceDescriptor.is_python_filec                 C   s"   | j d kr|  dd| _ | j S )N\/)_escaped_descriptionget_descriptionreplacer   r   r   r   get_escaped_description   s    
z(SourceDescriptor.get_escaped_descriptionc                 C   s*   z| j |j kW S  tk
r$   Y dS X d S r   	_cmp_namer   rD   r   r   r   __gt__   s    zSourceDescriptor.__gt__c                 C   s*   z| j |j k W S  tk
r$   Y dS X d S r   r   rD   r   r   r   __lt__   s    zSourceDescriptor.__lt__c                 C   s*   z| j |j kW S  tk
r$   Y dS X d S r   r   rD   r   r   r   __le__   s    zSourceDescriptor.__le__c                 C   s   | S r   r   r   r   r   r   __copy__   s    zSourceDescriptor.__copy__c                 C   s   | S r   r   )r=   memor   r   r   __deepcopy__   s    zSourceDescriptor.__deepcopy__)rK   rL   rM   __doc__r   Zin_utility_coder   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   }   s    r   c                   @   s^   e Zd ZdZdddZdddZdddZd	d
 Zdd Zdd Z	dd Z
dd Zdd ZdS )FileSourceDescriptora7  
    Represents a code source. A code source is a more generic abstraction
    for a "filename" (as sometimes the code doesn't come from a file).
    Instances of code sources are passed to Scanner.__init__ as the
    optional name argument and will be passed back when asking for
    the position()-tuple.
    Nc                 C   s   t |}|| _|p|| _ztj| j| _W n tk
rJ   | j| _Y nX tj	dtj
 }||rx|t|d  n|| _| | || _i | _d S )N.)r   decode_filenamer   path_descriptionr   r   relpath_short_path_description
ValueErrorabspathsep
startswithre   	file_pathr   r   _lines)r=   r   r   Zworkdirr   r   r   r>      s    

 
zFileSourceDescriptor.__init__c              	   C   s^   ||f}| j |}|d k	r |S | j||d}| }W 5 Q R X || j krP|nd | j |< |S )Nencodingerror_handling)r   getget_file_object	readlines)r=   r   r   keylinesfr   r   r   	get_lines   s    zFileSourceDescriptor.get_linesc                 C   s   t | j||S r   )r   Zopen_source_filer   r=   r   r   r   r   r   r      s    z$FileSourceDescriptor.get_file_objectc                 C   s   | j S r   )r   r   r   r   r   r      s    z$FileSourceDescriptor.get_descriptionc                 C   s:   | j }tt tjj }||r6|t|d  S |S r   )	r   r   r   r   getcwdr   r   r   re   )r=   r   cwdr   r   r   get_error_description   s
    
z*FileSourceDescriptor.get_error_descriptionc                 C   s   | j S r   )r   r   r   r   r   get_filenametable_entry   s    z,FileSourceDescriptor.get_filenametable_entryc                 C   s   t |to| j|jkS r   )
isinstancer   r   rD   r   r   r   __eq__   s    zFileSourceDescriptor.__eq__c                 C   s
   t | jS r   )rb   r   r   r   r   r   __hash__   s    zFileSourceDescriptor.__hash__c                 C   s
   d| j  S )Nz<FileSourceDescriptor:%s>)r   r   r   r   r   __repr__   s    zFileSourceDescriptor.__repr__)N)NN)NN)rK   rL   rM   r   r>   r   r   r   r   r   r   r   r   r   r   r   r   r      s   


r   c                   @   sN   e Zd ZdZdd ZdddZdd ZeZd	d
 Zdd Z	dd Z
dd ZdS )StringSourceDescriptorzu
    Instances of this class can be used instead of a filenames if the
    code originates from a string object.
    c                 C   s&   || _ dd |dD | _|| _d S )Nc                 S   s   g | ]}|d  qS )
r   ).0xr   r   r   
<listcomp>  s     z3StringSourceDescriptor.__init__.<locals>.<listcomp>r   )r@   split	codelinesr   )r=   r@   coder   r   r   r>      s    zStringSourceDescriptor.__init__Nc                    s$    s
| j S  fdd| j D S d S )Nc                    s   g | ]}|   qS r   )encodedecode)r   liner   r   r   r     s   z4StringSourceDescriptor.get_lines.<locals>.<listcomp>)r   r   r   r   r   r     s
    z StringSourceDescriptor.get_linesc                 C   s   | j S r   r@   r   r   r   r   r     s    z&StringSourceDescriptor.get_descriptionc                 C   s   dS )Nz<stringsource>r   r   r   r   r   r     s    z.StringSourceDescriptor.get_filenametable_entryc                 C   s   t | S r   )idr   r   r   r   r     s    zStringSourceDescriptor.__hash__c                 C   s   t |to| j|jkS r   )r   r   r@   rD   r   r   r   r     s    zStringSourceDescriptor.__eq__c                 C   s
   d| j  S )Nz<StringSourceDescriptor:%s>r   r   r   r   r   r     s    zStringSourceDescriptor.__repr__)NN)rK   rL   rM   r   r>   r   r   r   r   r   r   r   r   r   r   r   r      s   
r   c                   @   s
  e Zd Zd@ddZdd Zdd Zd	d
 Zdd Zdd Zdd Z	dd Z
dddddZedddZdd Zdd Zeddd Zd!d" Zd#d$ Zd%d& Zd'd( ZdAd)d*Zd+d, ZdBd-d.ZdCd/d0ZdDd1d2Zd3d4 Zd5d6 ZdEejd9d:d;Zd<d= Zd>d? Z dS )FPyrexScannerNTc	           
      C   s  t | t ||| | r(d| _t}	n
d| _t}	dd |	D | _d| _|r|j	| _	|j
| _
|j| _|j| _|j| _|jr|   n@|| _	|j
| _
t | _d| _d| _t|jdd r| j|jj || _|| _t| _dg| _d| _d| _d | _| d	 d
| _|   d S )NTFc                 S   s   i | ]
}||qS r   r   )r   keywordr   r   r   
<dictcomp>5  s      z)PyrexScanner.__init__.<locals>.<dictcomp>r   r   compile_time_env INDENTr   )r   r>   r   r   Zin_python_filepy_reserved_wordspyx_reserved_wordskeywordsasync_enabledcontextZincluded_filesr   Zcompile_time_evalZcompile_time_exprenter_asyncr   r~   optionsrC   parse_commentssource_encodingtrace_scannertraceindentation_stackindentation_charbracket_nesting_levelput_back_on_failurebeginsynext)
r=   filer   Zparent_scannerZscoper   r   r   Zinitial_posr   r   r   r   r>   +  sB    

zPyrexScanner.__init__c                 C   s"   |  std|}| t| d S )NNFKC)isasciir	   producer   r=   textr   r   r   normalize_identW  s    
zPyrexScanner.normalize_identc                 C   s   | j r| d| d S )Ncommentline)r   r   r   r   r   r   r   \  s    zPyrexScanner.commentlinec                 C   s   |  ||dd d S )N_r   )r   r   )r=   r   symbolr   r   r   strip_underscores`  s    zPyrexScanner.strip_underscoresc                 C   s
   | j d S )N)r   r   r   r   r   current_levelc  s    zPyrexScanner.current_levelc                 C   s   |  j d7  _ |S Nr   r   r   r   r   r   open_bracket_actionf  s    z PyrexScanner.open_bracket_actionc                 C   s   |  j d8  _ |S r   r   r   r   r   r   close_bracket_actionj  s    z!PyrexScanner.close_bracket_actionc                 C   s$   | j dkr | d | dd d S )Nr   r   NEWLINEr   )r   r   r   r   r   r   r   newline_actionn  s    

zPyrexScanner.newline_actionZ	SQ_STRINGZ	DQ_STRINGZ
TSQ_STRINGZ
TDQ_STRING)'"z'''z""")r   c                 C   s<   |r|d t kr|dd  }q | | j|  | d d S )Nr   r   ZBEGIN_STRING)r   r   string_statesr   r   r   r   r   begin_string_actionz  s    z PyrexScanner.begin_string_actionc                 C   s   |  d | d d S )Nr   Z
END_STRING)r   r   r   r   r   r   end_string_action  s    
zPyrexScanner.end_string_actionc                 C   s   |  | | d d S )NzUnclosed string literal)r   error_at_scanposr   r   r   r   unclosed_string_action  s    
z#PyrexScanner.unclosed_string_actionc                 C   s   |  d |rV|d }| jdkr(|| _n| j|kr<| d ||ddkrV| d |  }t|}||krrd S ||kr| j| | dd n:||  k r| j	  | dd q||  kr| d d S )Nr   r   r   zMixed use of tabs and spacesr   DEDENTzInconsistent indentation)
r   r   r   r   r   re   r   appendr   pop)r=   r   cr   Z	new_levelr   r   r   indentation_action  s*    





zPyrexScanner.indentation_actionc                 C   s6   t | jdkr&| dd | j  q | dd d S )Nr   r  r   EOF)re   r   r   r  r   r   r   r   
eof_action  s    zPyrexScanner.eof_actionc                 C   s   z|   \}}W n  tk
r0   | d Y d S X |tkr|| jkr|dkrht| jjkrh| jdd  n.|dkr| jj	dkr| jdd  n
| j| }| j
|}| jd k	r| j|||  f || _|| _d S )NzUnrecognized characterr   r%      )readr   r   r   r   r   r   Zfuture_directivesr  Zlanguage_levelintern_ustringr   r  positionr   systringr   )r=   r   r  r   r   r   r     s$    



zPyrexScanner.nextc                 C   sT   | j | jf}|  }|   | j | jf}| | j | j|   |\| _ | _|| _|S r   )r   r  r  r   unreadlast_token_position_tuple)r=   Zsaved	saved_posr   r   r   r   peek  s    zPyrexScanner.peekc                 C   s*   |  | j| j| j || _|| _|| _d S r   )r  r   r  r  )r=   r   r  posr   r   r   put_back  s    zPyrexScanner.put_backc                 C   s:   |d kr|   }| jdkr$t|d t||}|r6|d S )Nr   z!Possible inconsistent indentation)r  r   r   )r=   messager  fatalerrr   r   r   r     s    


 zPyrexScanner.errorc                 C   s   |   }| ||d d S )NT)Zget_current_scan_posr   )r=   r  r  r   r   r   r     s    zPyrexScanner.error_at_scanposc                 C   s$   | j |kr|   n| || d S r   )r   r   expectedr=   whatr  r   r   r   expect  s    

zPyrexScanner.expectc                 C   s.   | j tkr| j|kr|   n| || d S r   )r   r   r  r   r  r  r   r   r   expect_keyword  s    
zPyrexScanner.expect_keywordc                 C   s>   |r|  | n*| jtkr"| j}n| j}|  d||f  d S )NzExpected '%s', found '%s')r   r   r   r  )r=   r  r  foundr   r   r   r    s    
zPyrexScanner.expectedc                 C   s   |  dd d S )Nr   z)Expected an increase in indentation levelr  r   r   r   r   expect_indent  s    zPyrexScanner.expect_indentc                 C   s   |  dd d S )Nr  z(Expected a decrease in indentation levelr  r   r   r   r   expect_dedent  s    zPyrexScanner.expect_dedentExpected a newlineF)ignore_semicolonc                 C   sN   d }|r"| j dkr"|  }|   | j dkr8| d| |d k	rJt|d d S )N;r  r   zuseless trailing semicolon)r   r  r   r  r   )r=   r  r   Zuseless_trailing_semicolonr   r   r   expect_newline  s    
zPyrexScanner.expect_newlinec                 C   s0   |  j d7  _ | j dkr,d| jd< d| jd< d S )Nr   asyncawait)r   r   r   r   r   r   r     s    

zPyrexScanner.enter_asyncc                 C   sX   | j dkst|  j d8  _ | j sT| jd= | jd= | jdkrTt| j| j | _| _d S )Nr   r   r$  r#  )r#  r$  )r   r   r   r   r   r   r
  r  r   r   r   r   
exit_async  s    
zPyrexScanner.exit_async)NNNNTN)NT)N)N)N)r  F)!rK   rL   rM   r>   r   r   r   r   r   r   r   r   rt   r   r   r   r  r  r   r  r  r   r   r  r  r  r  r  cythonZbintr"  r   r%  r   r   r   r   r   "  sJ   	          
,$





r   )scannerc              
   c   s   t  }z| j}g | _| j| j|  f}z2z
|V  W n" t	k
rT } zW 5 d }~X Y nX W 5 |r| jrt| jd d D ]}| j|  qv| j|  n|d k	r|| j || _X W 5 tdd X d S )NT)ignorer   )
r   r   r   r   r  r  ro   r  extendr   )r'  errorsr   Zinitial_stater  er   r   r   tentatively_scan"  s$    
r,  )*r&  rB   objectr   r   unicodedatar	   
contextlibr
   r   r   ZPlex.Scannersr   ZPlex.Errorsr   ZErrorsr   r   r   r   r   ZLexiconr   r   r   ZFuturer   Zdebug_scannerr   Zscanner_debug_flagsZscanner_dump_filer   r   r   r   r:   r   r   r   r   r   r,  r   r   r   r   <module>   s      
                                 ';A)  