o
     wi                     @   s4  U d dl mZ d dlZd dlZd dlZd dlmZ d dlm	Z	 d dl
mZ d dlmZmZmZmZmZmZmZ d dlmZ d dlmZ d d	lmZmZ d
ZedZejZejZej Z ej!Z!ej"Z"ej#Z#ej$Z$ej%Z%ej&Z&ej'Z'ej(Z(ej)Z)ej*Z*ej+Z+ej,Z,eddG dd dZ-i Z.eee-f e/d< de0de1de0fddZ2de0de0fddZ3		dHdede4de4dee0 fddZ5de0defdd Z6dede-fd!d"Z7e6d#Z8d$Z9e6d%e9 d& Z:e6d%e9 d' Z;e6d(Z<e6d)Z=dede-fd*d+Z>G d,d- d-e	d-g d.Z?G d/d0 d0e?Z@G d1d2 d2ZAd3d4 ZBd5d6 ZC	7dId8e0ded9eeDeDf dee@ddf fd:d;ZE	7dId<ee0 ded9eeDeDf dee@ddf fd=d>ZF	7dId<ee0 ded?e-d9eeDeDf dee@ddf f
d@dAZG	7dId<ee0 ded?e-d9eeDeDf dee@ddf f
dBdCZHdDe0d9eeDeDf dEe0dee@ddf fdFdGZIdS )J    )absolute_importN)BOM_UTF8)
namedtuple)	dataclass)Dict	GeneratorIterableOptionalPatternSetTuple)CSTLogicError)PythonTokenTypes)PythonVersionInfosplit_linesu   􏿿zutf-8T)frozenc                   @   sb   e Zd ZU eed< ee ed< ee ed< eeef ed< eed< eeef ed< ee ed< dS )	TokenCollectionpseudo_tokensingle_quotedtriple_quotedendpats
whitespacefstring_pattern_mapalways_break_tokensN)__name__
__module____qualname__r
   __annotations__r   strr    r   r   a/home/ubuntu/sommelier/.venv/lib/python3.10/site-packages/libcst/_parser/parso/python/tokenize.pyr   @   s   
 r   _token_collection_cachechoiceskwargsreturnc                  O   s6   | dd}|r
J d}|s|d7 }|d|  d S )NcaptureF(z?:|))popjoin)r"   r#   r%   startr   r   r    groupN   s   r,   c                  G   s   t |  d S )N?)r,   )r"   r   r   r    maybeX      r.   Fversion_infoinclude_fstringonly_fstringc           	      C   s   dd }ddg}| dkr| d | dk s| dkr| d d	h}| d
kr8|r8ddg}|r3|}t }n
||7 }n|r=t S |D ]}t|D ]	}||| qFq?| dkrc||d ||d |S )Nc                 s   s,    t jdd | D  D ]}d|V  qd S )Nc                 S   s   g | ]}||  fqS r   )upper).0cr   r   r    
<listcomp>c   s    zI_all_string_prefixes.<locals>.different_case_versions.<locals>.<listcomp> )
_itertoolsproductr*   )prefixsr   r   r    different_case_versionsb   s   z5_all_string_prefixes.<locals>.different_case_versionsbr   r   br)r@   r@   ur7   r@      ffr)      ur)appendsetr8   permutationsupdate)	r0   r1   r2   r<   valid_string_prefixesresultrE   r:   tr   r   r    _all_string_prefixes]   s.   


rQ   exprc                 C   s   t | t jS N)recompileUNICODE)rR   r   r   r    _compile      rW   c                 C   s2   zt |  W S  ty   t|  t | < }| Y S w rS   )r!   KeyError_create_token_collection)r0   rO   r   r   r    _get_token_collection   s   
r[   z(?:[^{}]+|\{\{|\}\})+z"[A-Za-z0-9\-]+(?: [A-Za-z0-9\-]+)*z(?:\{\{|\}\}|\\N\{z*\}|\\(?:\r\n?|\n)|\\[^\r\nN]|[^{}\r\n\\])+z\}|\\[^N]|[^{}\\])+z(?:\\(?:\r\n?|\n)|[^{}\r\n])+z[^{}]+c           +   	   C   s:  d}d}| j dkrd}ntjd dkrd}ndt d }| d	krKd
}d}d}d}t||||}d}	tddt|	 }
d|	 }t|
|}td|d }n:d}d}| dkrVd}nd}d}t||||}| j dk rj|d7 }d}	tddt|	 }
d|	 }t|
|}td |d }t|||}t| }t| }tt| d!d" }t| d!d!d#}t| }d$}d%}d&}d'}t|d( |d) }td*d+d,d-d.d/d0}d1}g d2}| dkr|dd3 | d4kr|dd5 n|dd6 t| }t|||}t|d7 td8d9 |d: td;d9 }||g}d<} |r||t|    td=g|R  }!t|d!d>t|!||||d!d> }"i }#|D ]#}$t	||#|$d8 < t	||#|$d; < t	||#|$d( < t	||#|$d) < q-t
 }%t
 }&i }'|D ]}(d?D ]
})|%|(|)  q_d@D ]
})|&|(|)  qlq[|D ]}(| D ]	})|)|'|(|) < qq{t	|"}*t|*|%|&|#t	||'h dAS )BNz[ \f\t]*z	#[^\r\n]*rG   z([A-Za-z_0-9]+)r   z(\w+)u   ([A-Za-z_0-9-z]+)rC   z0[xX](?:_?[0-9a-fA-F])+z0[bB](?:_?[01])+z0[oO](?:_?[0-7])+z(?:0(?:_?0)*|[1-9](?:_?[0-9])*)z[eE][-+]?[0-9](?:_?[0-9])*z)[0-9](?:_?[0-9])*\.(?:[0-9](?:_?[0-9])*)?z\.[0-9](?:_?[0-9])*z[0-9](?:_?[0-9])*z[0-9](?:_?[0-9])*[jJ]z[jJ]z0[xX][0-9a-fA-F]+z
0[bB][01]+r?   z0[oO][0-7]+z0[oO]?[0-7]+z(?:0+|[1-9][0-9]*)r@   z[lL]?z[eE][-+]?[0-9]+z[0-9]+\.[0-9]*z\.[0-9]+z[0-9]+z
[0-9]+[jJ]T)r1   )r1   r2   z(?:\\.|[^'\\])*'z(?:\\.|[^"\\])*"z(?:\\.|'(?!'')|[^'\\])*'''z(?:\\.|"(?!"")|[^"\\])*"""'''"""z\*\*=?z>>=?z<<=?z//=?z->z[+\-*/%&@`|^!=<>]=?~z[][(){}])z\r\n?z\nz[;.,@]z\.\.\.)r@      z:=?:z'[^\r\n'\\]*(?:\\.[^\r\n'\\]*)*'z\\(?:\r\n?|\n)z"[^\r\n"\\]*(?:\\.[^\r\n"\\]*)*")rb   ra   r]   r\   z\\(?:\r\n?|\n)|\Z)r%   )rb   ra   )r]   r\   >
   deftrywithclasswhileexceptimportr$   finally;)majorsysr0   MAX_UNICODEr,   r.   rQ   insertrJ   rW   rK   addr   )+r0   
WhitespaceCommentName	Hexnumber	Binnumber	Octnumber	Decnumber	IntnumberExponent
PointfloatExpfloatFloatnumber
ImagnumberNumberpossible_prefixesStringPrefixStringPrefixWithFfstring_prefixesFStringStartSingleDoubleSingle3Double3TripleOperatorBracketspecial_argsSpecialFunnyContStrpseudo_extra_pool
all_quotesPseudoExtrasPseudoTokenr   _prefixr   r   r   rP   quotepseudo_token_compiledr   r   r    rZ      s   



rZ   c                   @   s   e Zd Zedd ZdS )Tokenc                 C   sN   t | j}t|dkr| jd t| d dfS | jd | jd t| j fS N   r   )r   stringlen	start_pos)selflinesr   r   r    end_posC  s   
zToken.end_posN)r   r   r   propertyr   r   r   r   r    r   B  s    r   )typer   r   r:   c                   @   s   e Zd Zdd ZdS )PythonTokenc                 C   s   d| j | jjd S )Nz6TokenInfo(type=%s, string=%r, start_pos=%r, prefix=%r))r   )_replacer   namer   r   r   r    __repr__M  s   zPythonToken.__repr__N)r   r   r   r   r   r   r   r    r   L  s    r   c                   @   s<   e Zd Zdd Zdd Zdd Zdd Zd	d
 Zdd ZdS )FStringNodec                 C   s(   || _ || _d| _d| _d | _d| _d S )Nr   r7   )r   rawparentheses_countprevious_lineslast_string_start_posformat_spec_count)r   r   r   r   r   r    __init__T  s   
zFStringNode.__init__c                 C   s   |  j d7  _ d S )Nr   )r   r   	characterr   r   r    open_parentheses^  s   zFStringNode.open_parenthesesc                 C   s&   |  j d8  _ | j dkrd| _d S d S r   r   r   r   r   r   r    close_parenthesesa  s   

zFStringNode.close_parenthesesc                 C   s   t | jdkS )Nr@   )r   r   r   r   r   r    allow_multilineg  rX   zFStringNode.allow_multilinec                 C   s   | j | jkS rS   r   r   r   r   r    
is_in_exprj  r/   zFStringNode.is_in_exprc                 C   s   |    o| jS rS   )r   r   r   r   r   r    is_in_format_specm  s   zFStringNode.is_in_format_specN)	r   r   r   r   r   r   r   r   r   r   r   r   r    r   S  s    
r   c                 C   sf   t | D ])\}}||jr-tt|j||d}d}|jrJ | |d = |dt|jf  S qd |dfS )Nr:   r7   r   )	enumerate
startswithr   r   FSTRING_ENDr   r   )fstring_stackr   r   additional_prefixfstring_stack_indexnodetokenr   r   r    _close_fstring_if_necessaryq  s   



r   c                 C   s  |d }|  }| r|rt}nt}n|jrt}n|rt}nt}|||}|d u r0|j	|fS |j	s8||f|_
|d}	|D ]}
| |
j |	}|d urZ|dd t|
j  }	q?|}|t|	7 }|	dsm|	drz| j	|	7  _	d}	|	|fS |j	|	 }	|	|fS )Nr   
r7   )r   r   fstring_format_spec_multi_linefstring_format_spec_single_liner   fstring_raw_stringfstring_string_multi_linefstring_string_single_linematchr   r   r,   r   r   endswith)r   r   linelnumpostosr   regexr   r   fstring_stack_node	end_matchnew_posr   r   r    _find_fstring_string~  s<   



r   r   r   coder   c                 C   s   t | dd}t|||dS )z0Generate tokens from a the source code (string).T)keependsr   )r   tokenize_lines)r   r0   r   r   r   r   r    tokenize  s   r   r   c                 C   s6   t |}|tddkrt| |||dS t| |||dS )Nr@   rH   r   )r[   r   _tokenize_lines_py37_or_above_tokenize_lines_py36_or_below)r   r0   r   token_collectionr   r   r    r     s   r   r   c           #   	   #   sH	   d}dgd}d}d}d}d}	d}
d}d}d}d}|d d g }dd dd fdd	}| D ]'}d7 d}t |}|rl|trXt}|dd }t |}d
|d  | }|d }||d 7 }d}|r|du rvtd||}|r|d}|du rtddurtdtt||d|  ||
V  d}d}n	|| }|| }q8||k r`|r|d }| st	|j
|||\}}|rdurtdtt||jddV  d|_q||krny||d }t|||f|\}}}||7 }|durdurtd|V  q|j||}|sL|j||}|dkr.|| E dH  | }d}	tt|| |f||d V  d}|d7 }q||d }
d}|d\}}|f|d}|dkrs|
soJ |
}n|d }|	r|dvrd}	|dkr|sd}|}|| dkr|d7 }|d8 }|| dks|d krdurV  dttddV  | ||E dH  ||v s|dkr|dkr|dkrdurV  dtt||
V  nl|ddur||jv rGg |dd< d}td|d| rG	  } | |kr? r,r,| kr,d dddur6V  dttddV  n|  nqt|rd}!|dv rp rp|dkrett||
V  ntt||
V  d}!nI|dkrstt||
d}!n7|dkrdurd tu rd dkrd d ttd d d V  dnrV  d|!rtt||
V  nt ||
E dH  n|dv rt!dd |D rg |dd< |	s|dkr|s rdrV  dtt"||
V  n|
| }d}	nM|dkr"|#drJ |
| }n:||j$v rb|j
| }|||}|rU|d}||| }durKV  dtt||
V  n|f}||d }|}n||j%v s||dd |j%v s||dd |j%v r|d dv r|f}|j
&|p|j
&|d p|j
&|d }||d }|}ndurV  dtt||
V  n||j'v r|t(|j'| d|v pd|v  dur݈V  dtt)||
V  nv|dkr||d d v r||
||d  7 }na|d!v r|r|d *| n;|d7 }n6|d"v r,|r$|d +| n&|r+|d8 }n|d#krJ|rJ|d j,|d j- dkrJ|d  j-d7  _-durTV  dtt.||
V  ||k sq8|rztt|||
V  |#dsx|#d$rzd}	durV  d|f}"dd D ]} ttd|"dV  qtt/d|"|V  dS )%)  
    A heavily modified Python standard library tokenizer.

    Additionally to the default information, yields also the prefix of each
    token. This idea comes from lib2to3. The prefix contains all information
    that is irrelevant for the parser like newlines in parentheses or comments.
    r   
0123456789r7   NTr   Fc                 3   s    | d k rF| d krt tddfdV  d S d ur"V  d  r2r2d kr2d ddt tddV    | d k sd S d S )Nr   r7   r   Fr   ERROR_DEDENTDEDENTr)   r+   	async_defasync_def_indentasync_def_newlineindentsr   sposstashedr   r    dedent_if_necessary  s   z:_tokenize_lines_py36_or_below.<locals>.dedent_if_necessary^Logic error!r   r   rG   
\#....r@   	[ \f\t]*$)asyncawaitr   rc   
c                 s       | ]}|   V  qd S rS   r   r4   rE   r   r   r    	<genexpr>      z0_tokenize_lines_py36_or_below.<locals>.<genexpr>#r   r>   R\z\
z\
z\([{)]}r`   r   0r   r   BOM_UTF8_STRINGr   r   endr   STRINGr   r   r   FSTRING_STRINGr   r   r   r   r   
ERRORTOKENr,   spanINDENTrJ   NUMBERr   rT   r)   r   r   isidentifierASYNCAWAITNAME_split_illegal_unicode_nameanyNEWLINEr   r   r   getr   r   FSTRING_STARTr   r   r   r   OP	ENDMARKER)#r   r0   r   r   paren_levelmaxnumcharscontstrcontlinenew_liner:   endprogcontstr_startr   firstr   r   r   r   endmatchr   r   restfstring_end_tokenquote_lengthpseudomatchr   r+   r   initialiindent_startindentshould_yield_identifierr   r   r   r    r     s.  

































    
r   c           #   	   #   s    fdd}d}dg d}d}d}d}	d}
d}d}d}d}d}|d d g }| D ]b}d7 d}t |}|r`|trLt}|dd }t |}d	|d  | }|d }||d 7 }d
}|r|du rjtd||}|r|d}|du r~tdtt||d|  ||V  d}d}	n	|| }|	| }	q,||k r|r|d }| st	|j
|||\}}|rtt||jddV  d|_q||krʐn||d }t|||f|\}}}||7 }|dur|V  q|j||}|s)|j||}|dkr|| D ]}|V  q| }d
}
tt|| |f||d V  d}|d7 }q||d }d}|d\}}|f|d}|dkrP|sLJ |}n?|d }|
r|dvrd
}
|dkr|sd}|} || dkr|d7 }| d8 } || dksq|  d krttddV   |  || D ]}|V  q||v s|dkr|dkr|dkrtt||V  n|ddur6||jv rg |dd< d}td|d| r	   }!|!|krttddV  n |! nqt|r'|dkrtt||V  n}|dkrtt||V  nntt||V  ndt ||D ]}|V  q-nU|dv rgt!dd |D rKg |dd< |
s_|dkr_|s_tt"||V  n|| }d}
n$|dkrz|#drtJ || }n||j$v r|j
| }|||}|r|d}||| }tt||V  n|f}||d }|}	n||j%v s|dd |j%v s|dd |j%v r|d dv r|f}|j
&|p|j
&|d p|j
&|d }||d }|}	ntt||V  n||j'v r|t(|j'| d|v pd|v  tt)||V  nl|dkr8||d dv r8||||d  7 }nW|d v rM|rH|d *| n;|d7 }n6|d!v re|r]|d +| n&|rd|d8 }n|d"kr|r|d j,|d j- dkr|d  j-d7  _-tt.||V  ||k sq,|rtt|||V  |#ds|#d#rd}
|f}" dd D ]}!ttd|"dV  qtt/d|"|V  dS )$r   c                 3   s^    |  d k r-|  d krt tddfdV  d S t tddV     |  d k sd S d S )Nr   r   r7   r   r   r   r   r   r   r   r    r   L  s   z:_tokenize_lines_py37_or_above.<locals>.dedent_if_necessaryr   r   r7   NTr   r   Fr   r   r   rG   r   r   r   r   r@   r   r   r   r   c                 s   r   rS   r   r   r   r   r    r     r   z0_tokenize_lines_py37_or_above.<locals>.<genexpr>r   r   r>   r   r   r   r   r   r`   r   r   )#r   r0   r   r   r   r  r  r  r  r  r  r:   r  r  r   r  r   r   r   r  r   r   r  r  r  r   r   rP   r+   r   r!  r"  r#  r$  r   r   r&  r    r   >  s  

























  Pr   r   r:   c                 #   s     fdd}d d|t | D ]G\}}r9t|r4| V  | dd|d |d | fq |7  q | }t|rE| q rW| V  d|d |d | f| dq rd| V  d S d S )Nc                      s   t rtnt S rS   )r   r  r  r   found
is_illegalr   r:   r   r    create_tokenf  s   z1_split_illegal_unicode_name.<locals>.create_tokenr7   Fr   r   T)r   r   r  )r   r   r:   r*  r"  char	new_foundr   r'  r    r  c  s4   


r  )FF)r   )J
__future__r   	itertoolsr8   rT   rm   codecsr   collectionsr   dataclassesr   typingr   r   r   r	   r
   r   r   libcstr   !libcst._parser.parso.python.tokenr   libcst._parser.parso.utilsr   r   rn   decoder   r  r  r  r  r  r  r   r	  r
  r  r  r   r  r  r   r   r!   r   r   objectr,   r.   boolrQ   rW   r[   r   unicode_character_namer   r   r   r   rZ   r   r   r   r   r   intr   r   r   r   r  r   r   r   r    <module>   s  $



*


 
+





   

  '
