o xiNy@sdZddlmZddlZddlZddlZddlmZmZddl m Z ddl m Z m Z mZmZddlmZmZmZmZmZmZmZmZmZddlmZgd Zgd Zed d ZGd dde Z!ee!Gddde"Z#Gddde#Z$Gddde%Z&Gddde"Z'e'Z(Gddde)Z*Gddde"Z+ddZ,Gddde"Z-e-Z.dd Z/Gd!d"d"Z0Gd#d$d$eZ1Gd%d&d&e!Z2ee2Gd'd(d(e#Z3Gd)d*d*e"Z4Gd+d,d,e3Z5d-d.Z6Gd/d0d0e2Z7ee7Gd1d2d2e3Z8dS)3z pygments.lexer ~~~~~~~~~~~~~~ Base lexer classes. :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. )print_functionN) apply_filtersFilter)get_filter_by_name)ErrorTextOther _TokenType) get_bool_opt get_int_opt get_list_optmake_analysator text_type add_metaclass iteritemsFuture guess_decode) regex_opt) Lexer RegexLexerExtendedRegexLexerDelegatingLexer LexerContextincludeinheritbygroupsusingthisdefaultwords))sutf-8)szutf-32)szutf-32be)szutf-16)szutf-16becCdS)Nxr#r#O/home/ubuntu/.local/lib/python3.10/site-packages/wandb/vendor/pygments/lexer.py$r'c@eZdZdZddZdS) LexerMetaz This metaclass automagically converts ``analyse_text`` methods into static methods which always return float values. cCs(d|vr t|d|d<t||||S)N analyse_text)r type__new__)mcsnamebasesdr#r#r&r--szLexerMeta.__new__N)__name__ __module__ __qualname____doc__r-r#r#r#r&r*'s r*c@sZeZdZdZdZgZgZgZgZdZ ddZ ddZ dd Z d d Z dd dZddZdS)ra Lexer for a specific language. Basic options recognized: ``stripnl`` Strip leading and trailing newlines from the input (default: True). ``stripall`` Strip all leading and trailing whitespace from the input (default: False). ``ensurenl`` Make sure that the input ends with a newline (default: True). This is required for some lexers that consume input linewise. .. versionadded:: 1.3 ``tabsize`` If given and greater than 0, expand tabs in the input (default: 0). ``encoding`` If given, must be an encoding name. This encoding will be used to convert the input string to Unicode, if it is not already a Unicode string (default: ``'guess'``, which uses a simple UTF-8 / Locale / Latin1 detection. Can also be ``'chardet'`` to use the chardet library, if it is installed. ``inencoding`` Overrides the ``encoding`` if given. NrcKs||_t|dd|_t|dd|_t|dd|_t|dd|_|dd |_|d p-|j|_g|_ t |d d D]}| |q8dS) NstripnlTstripallFensurenltabsizerencodingguess inencodingfiltersr#) optionsr r6r7r8r r9getr:r=r add_filter)selfr>filter_r#r#r&__init__bs zLexer.__init__cCs$|jr d|jj|jfSd|jjS)Nzz)r> __class__r2rAr#r#r&__repr__ns  zLexer.__repr__cKs*t|ts t|fi|}|j|dS)z8 Add a new stream filter to this lexer. N) isinstancerrr=append)rArBr>r#r#r&r@us zLexer.add_filtercCr!)a~ Has to return a float between ``0`` and ``1`` that indicates if a lexer wants to highlight this text. Used by ``guess_lexer``. If this method returns ``0`` it won't highlight it in any case, if it returns ``1`` highlighting with this lexer is guaranteed. The `LexerMeta` metaclass automatically wraps this function so that it works like a static method (no ``self`` or ``cls`` parameter) and the return value is automatically converted to `float`. If the return value is an object that is boolean `False` it's the same as if the return values was ``0.0``. Nr#)textr#r#r&r+}r(zLexer.analyse_textFc sttssjdkrt\}nojdkr_zddl}Wn ty'tdwd}tD]\}}|rCt|d |d}nq,|dur\| dd} | dpYd d}|n! jd rrtd dn d rtd d d d  d d j rnjrd jdkrjjrd sd 7fdd} | } |st| j} | S)a= Return an iterable of (tokentype, value) pairs generated from `text`. If `unfiltered` is set to `True`, the filtering mechanism is bypassed even if filters are defined. Also preprocess the text, i.e. expand tabs and strip it if wanted and applies registered filters. r;chardetrNzkTo enable chardet encoding guessing, please install the chardet library from http://chardet.feedparser.org/replaceir:r uz   c3s&D] \}}}||fVqdSN)get_tokens_unprocessed)_tvrArIr#r&streamers z"Lexer.get_tokens..streamer)rGrr:rrJ ImportError _encoding_map startswithlendecodedetectr?rKr7stripr6r9 expandtabsr8endswithrr=) rArI unfilteredrPrJdecodedbomr:encrTstreamr#rSr& get_tokenssT               zLexer.get_tokenscCst)z Return an iterable of (index, tokentype, value) pairs where "index" is the starting position of the token within the input text. In subclasses, implement this method as a generator to maximize effectiveness. )NotImplementedErrorrSr#r#r&rOszLexer.get_tokens_unprocessed)F)r2r3r4r5r/aliases filenamesalias_filenames mimetypespriorityrCrFr@r+rcrOr#r#r#r&r3s   ;rc@s$eZdZdZefddZddZdS)ra  This lexer takes two lexer as arguments. A root lexer and a language lexer. First everything is scanned using the language lexer, afterwards all ``Other`` tokens are lexed using the root lexer. The lexers from the ``template`` lexer package use this base lexer. cKs<|di||_|di||_||_tj|fi|dSNr#) root_lexerlanguage_lexerneedlerrC)rA _root_lexer_language_lexer_needler>r#r#r&rCszDelegatingLexer.__init__cCsd}g}g}|j|D]$\}}}||jur(|r#|t||fg}||7}q ||||fq |r<|t||ft||j|S)N)rlrOrmrHrX do_insertionsrk)rArIbuffered insertions lng_bufferirQrRr#r#r&rOs   z&DelegatingLexer.get_tokens_unprocessedN)r2r3r4r5rrCrOr#r#r#r&rs rc@eZdZdZdS)rzI Indicates that a state should include rules from another state. Nr2r3r4r5r#r#r#r&rsrc@r))_inheritzC Indicates the a state should inherit from its superclass. cCr!)Nrr#rEr#r#r&rFz_inherit.__repr__N)r2r3r4r5rFr#r#r#r&rys ryc@s eZdZdZddZddZdS)combinedz: Indicates a state combined from multiple states. cGs t||SrN)tupler-)clsargsr#r#r&r-s zcombined.__new__cGsdSrNr#)rAr~r#r#r&rCszcombined.__init__N)r2r3r4r5r-rCr#r#r#r&r{ s r{c@sFeZdZdZddZdddZdddZdd d Zd d Zd dZ dS) _PseudoMatchz: A pseudo match object constructed from a string. cCs||_||_dSrN)_text_start)rAstartrIr#r#r&rCs z_PseudoMatch.__init__NcCs|jSrN)rrAargr#r#r&rsz_PseudoMatch.startcCs|jt|jSrN)rrXrrr#r#r&end"sz_PseudoMatch.endcCs|rtd|jS)Nz No such group) IndexErrorrrr#r#r&group%sz_PseudoMatch.groupcCs|jfSrN)rrEr#r#r&groups*sz_PseudoMatch.groupscCsiSrNr#rEr#r#r& groupdict-rzz_PseudoMatch.groupdictrN) r2r3r4r5rCrrrrrr#r#r#r&rs    rcsdfdd }|S)zL Callback that yields multiple actions for each group in the match. Nc3stD]O\}}|durqt|tur)||d}|r(||d||fVq||d}|durT|r>||d|_||t||d||D]}|rS|VqLq|r^||_dSdS)N) enumerater,r rrposrr)lexermatchctxrvactiondataitemr~r#r&callback5s, zbygroups..callbackrNr#)r~rr#rr&r1src@rw)_ThiszX Special singleton used for indicating the caller class. Used by ``using``. Nrxr#r#r#r&rKsrc slidvrd}t|ttfr|d<nd|fd<tur+dfdd }|Sdfdd }|S) a Callback that processes the match with a different lexer. The keyword arguments are forwarded to the lexer, except `state` which is handled separately. `state` specifies the state that the new lexer will start in, and can be an enumerable such as ('root', 'inline', 'string') or a simple string which is assumed to be on top of the root state. Note: For that to work, `_other` must not be an `ExtendedRegexLexer`. statestackrootNc3sxr|j|jdi}n|}|}|j|fiD] \}}}||||fVq#|r:||_dSdSrj)updater>rDrrOrrrrrrlxsrvrQrR) gt_kwargskwargsr#r&ris  zusing..callbackc3sl|jdi}|}|j|fiD] \}}}||||fVq|r4||_dSdSrj)rr>rrOrrrr_otherrrr#r&rxs  rN)poprGlistr|r)rrrrr#rr&rSs     rc@r))rz Indicates a state or state action (e.g. #pop) to apply. For example default('#pop') is equivalent to ('', Token, '#pop') Note that state tuples may be used as well. .. versionadded:: 2.0 cCs ||_dSrN)r)rArr#r#r&rCs zdefault.__init__N)r2r3r4r5rCr#r#r#r&rs rc@s"eZdZdZdddZddZdS) rz Indicates a list of literal words that is transformed into an optimized regex that matches any of the words. .. versionadded:: 2.0 rqcCs||_||_||_dSrN)rprefixsuffix)rArrrr#r#r&rCs zwords.__init__cCst|j|j|jdS)Nrr)rrrrrEr#r#r&r?sz words.getN)rqrq)r2r3r4r5rCr?r#r#r#r&rs  rc@sJeZdZdZddZddZddZdd Zdd d Zd dZ ddZ d S)RegexLexerMetazw Metaclass for RegexLexer, creates the self._tokens attribute from self.tokens on the first instantiation. cCs t|tr |}t||jS)zBPreprocess the regular expression component of a token definition.)rGrr?recompiler)r}regexrflagsrr#r#r&_process_regexs zRegexLexerMeta._process_regexcCs&t|tust|sJd|f|S)z5Preprocess the token component of a token definition.z2token type must be simple type or callable, not %r)r,r callable)r}tokenr#r#r&_process_tokenszRegexLexerMeta._process_tokencCst|tr/|dkr dS||vr|fS|dkr|S|dddkr)t|dd SJd|t|trbd |j}|jd 7_g}|D]}||ksPJd ||||||qD|||<|fSt|tr||D]}||vsy|d vsyJd |qi|SJd|)z=Preprocess the state transition action of a token definition.#pop#pushNz#pop:Fzunknown new state %rz_tmp_%drzcircular state ref %r)rrzunknown new state zunknown new state def %r)rGstrintr{_tmpnameextend_process_stater|)r} new_state unprocessed processed tmp_stateitokensistater#r#r&_process_new_states<        z!RegexLexerMeta._process_new_statec Cs~t|tus Jd||ddksJd|||vr ||Sg}||<|j}||D]}t|trK||ks>Jd|||||t|q-t|trQq-t|trk| |j ||}| t djd|fq-t|tuswJd|z ||d||}Wnty} z td |d||| fd} ~ ww||d } t|d krd}n | |d ||}| || |fq-|S) z%Preprocess a single state definition.zwrong state name %rr#zinvalid state name %rzcircular state reference %rrqNzwrong rule def %rz+uncompilable regex %r in state %r of %r: %sr)r,rflagsrGrrrryrrrrHrrrr|r Exception ValueErrorrrX) r}rrrtokensrtdefrrexerrrr#r#r&rsH        zRegexLexerMeta._process_stateNcCs<i}|j|<|p |j|}t|D] }||||q|S)z-Preprocess a dictionary of token definitions.) _all_tokensrrr)r}r/ tokendefsrrr#r#r&process_tokendefs  zRegexLexerMeta.process_tokendefc Csi}i}|jD]_}|jdi}t|D]Q\}}||}|dur;|||<z|t}Wn ty5Yqw|||<q||d}|durFq||||d<z|t} Wn ty^Yqw|| ||<qq|S)a Merge tokens from superclasses in MRO order, returning a single tokendef dictionary. Any state that is not defined by a subclass will be inherited automatically. States that *are* defined by subclasses will, by default, override that state in the superclass. If a subclass wishes to inherit definitions from a superclass, it can use the special value "inherit", which will cause the superclass' state definition to be included at that point in the state. rNr)__mro____dict__r?rindexrrr) r}r inheritablectoksritemscuritems inherit_ndx new_inh_ndxr#r#r& get_tokendefs s6      zRegexLexerMeta.get_tokendefscOsRd|jvri|_d|_t|dr|jrn |d||_tj |g|Ri|S)z:Instantiate cls after preprocessing its token definitions._tokensrtoken_variantsrq) rrrhasattrrrrrr,__call__)r}r~kwdsr#r#r&r;s zRegexLexerMeta.__call__rN) r2r3r4r5rrrrrrrr#r#r#r&rs# , 1rc@s$eZdZdZejZiZdddZdS)rz Base for simple stateful regular expression-based lexers. Simplifies the lexing process so that you need only provide a list of states and regular expressions. rc csd}|j}t|}||d} |D]|\}}} |||} | r|dur=t|tur2||| fVn ||| D]} | Vq7| }| durt| trj| D]} | dkrW|qL| dkrc| |dqL| | qLnt| t ru|| d=n| dkr| |dnJd| ||d}n3qz'||d krd g}|d }|t d fV|d7}Wq|t ||fV|d7}Wn t yYdSwq) z} Split ``text`` into (tokentype, text) pairs. ``stack`` is the inital stack (default: ``['root']``) rrrNrrFwrong state def: %rrLr)rrr,r rrrGr|rrHrrrr) rArIrrr statestack statetokensrexmatchrrmrrr#r#r&rOhsZ              z!RegexLexer.get_tokens_unprocessedNr) r2r3r4r5r MULTILINErrrOr#r#r#r&rIs rc@s"eZdZdZdddZddZdS)rz9 A helper object that holds lexer position data. NcCs*||_||_|p t||_|pdg|_dS)Nr)rIrrXrr)rArIrrrr#r#r&rCszLexerContext.__init__cCsd|j|j|jfS)NzLexerContext(%r, %r, %r))rIrrrEr#r#r&rFs zLexerContext.__repr__NN)r2r3r4r5rCrFr#r#r#r&rs  rc@seZdZdZdddZdS)rzE A RegexLexer that uses a context object to store its state. Nc cs|j}|st|d}|d}n |}||jd}|j} |D]\}}}|||j|j} | r|dur\t|turG|j|| fV| |_n||| |D]} | VqM|s\||jd}|durt |t r|D] } | dkrs|j qg| dkr|j |jdqg|j | qgn t |t r|j|d=n|dkr|j |jdnJd |||jd}nGqz;|j|jkrWdS||jd krdg|_|d}|jtd fV|jd7_Wq|jt||jfV|jd7_Wn tyYdSwq) z Split ``text`` into (tokentype, text) pairs. If ``context`` is given, use this lexer context instead. rrrrNrrFrrL)rrrrIrrr,r rrGr|rrHrrrr) rArIcontextrrrrrrrrrr#r#r&rOsh          z)ExtendedRegexLexer.get_tokens_unprocessedr)r2r3r4r5rOr#r#r#r&rsrc cst|}zt|\}}Wnty|D]}|VqYdSwd}d}|D]s\}}} |dur1|}d} |r|t| |kr| | ||} ||| fV|t| 7}|D]\} } }|| |fV|t|7}qS||} zt|\}}Wn ty|d}Yn w|r|t| |ks=||| | dfV|t| | 7}q&|r|pd}|D]\}}} ||| fV|t| 7}qzt|\}}Wn tyd}YdSw|sdSdS)ag Helper for lexers which must combine the results of several sublexers. ``insertions`` is a list of ``(index, itokens)`` pairs. Each ``itokens`` iterable should be inserted at position ``index`` into the token stream given by the ``tokens`` argument. The result is a combined token stream. TODO: clean up the code here. NTrF)iternext StopIterationrX)rtrrrrrealposinsleftrvrQrRolditmpvalit_indexit_tokenit_valuepr#r#r&rrsX         rrc@r))ProfilingRegexLexerMetaz>Metaclass for ProfilingRegexLexer, collects regex timing info.csLt|trt|j|j|jdn|t|tjffdd }|S)Nrcs`jdfddg}t}|||}t}|dd7<|d||7<|S)Nrrr"r) _prof_data setdefaulttimer)rIrendposinfot0rest1r}compiledrrr#r& match_func@sz:ProfilingRegexLexerMeta._process_regex..match_func) rGrrrrrrsysmaxsize)r}rrrrr#rr&r8s   z&ProfilingRegexLexerMeta._process_regexN)r2r3r4r5rr#r#r#r&r5s rc@s"eZdZdZgZdZdddZdS)ProfilingRegexLexerzFDrop-in replacement for RegexLexer that does profiling of its regexes.rc#sjjit||D]}|Vqjj}tdd|Dfdddd}tdd|D}t t djj t ||ft d t d d t d |D]}t d |qVt d dS)NcssP|]#\\}}\}}|t|ddddd|d|d||fVqdS)zu'z\\\NAi)reprr[rK).0rrnrQr#r#r& Xs  z=ProfilingRegexLexer.get_tokens_unprocessed..cs |jSrN)_prof_sort_indexr$rEr#r&r'[s z.T)keyreversecss|]}|dVqdS)Nr#)rr%r#r#r&r]sz2Profiling result for %s lexing %d chars in %.3f mszn==============================================================================================================z$%-20s %-64s ncalls tottime percall)rrzn--------------------------------------------------------------------------------------------------------------z%-20s %-65s %5d %8.4f %8.4f) rDrrHrrOrsortedrsumprintr2rX)rArIrtokrawdatar sum_totalr1r#rEr&rORs,    z*ProfilingRegexLexer.get_tokens_unprocessedNr)r2r3r4r5rrrOr#r#r#r&rKs r)9r5 __future__rrrrpygments.filterrrpygments.filtersrpygments.tokenrrrr pygments.utilr r r r rrrrrpygments.regexoptr__all__rV staticmethod_default_analyser,r*objectrrrrryrr|r{rrrrrrrrrrrrrrrr#r#r#r&sN  ,   ' 2 )WE?