o
    i$                     @   s  d dl Z d dlmZ d dlmZmZmZmZmZ d dl	Z	d dl
mZ d dlmZ ddlmZmZ ddlmZ dd	lmZ dd
lmZ ddlmZmZmZmZmZmZ edededddeddddeddddddedddddedddd dedd!d"d#dedd$d%d&ddd'ed(d)d*d+dedd,d-d.ded/d0d1d2df
d3ed4ed5ee d6ee d7e d8e!d9ee d:e d;e!d<efd=d>Z"					(		/	dYd3ed4ed5ee d7e d8e!d9ee d:e d?e!d<ed;e!d@eeef fdAdZ#i fd/ddBdCeeef dDeeef d<ed?e!d@eeef f
dEdFZ$	G	H			dZdIee dJedKedLe dMe!dNe!dOe!fdPdQZ%dRedCeeeee&f f dSedTed@df
dUdVZ'dRedCeeeee&f f d@dfdWdXZ(dS )[    N)Path)AnyDictListOptionalUnion)fix_random_seed)Printer   )displacyutil)Scorer)Doc)Corpus   )ArgOptappbenchmark_cliimport_code	setup_gpuaccuracyevaluate.zModel name or path)helpz3Location of binary evaluation data in .spacy formatT)r   existsz--outputz-ozOutput JSON file for metricsF)r   dir_okayz--codez-czNPath to Python file with additional code (registered functions) to be importedz--gpu-idz-gzGPU ID or -1 for CPUz--gold-preprocz-GzUse gold preprocessingz--displacy-pathz-dpz+Directory to output rendered parses as HTML)r   r   	file_okay   z--displacy-limitz-dlz!Limit of parses to render as HTMLz--per-componentz-PzSReturn scores per component, only applicable when an output JSON file is specified.scz--spans-keyz-skz*Spans key to use when evaluating Doc.spansmodel	data_pathoutput	code_pathuse_gpugold_preprocdisplacy_pathdisplacy_limitper_component	spans_keyc
           
      C   s(   t | t| |||||||d|	d
 dS )am  
    Evaluate a trained pipeline. Expects a loadable spaCy pipeline and evaluation
    data in the binary .spacy format. The --gold-preproc option sets up the
    evaluation examples with gold-standard sentences and tokens for the
    predictions. Gold preprocessing helps the annotations align to the
    tokenization, and may result in sequences of more consistent length. However,
    it may reduce runtime accuracy due to train/test skew. To render a sample of
    dependency parses in a HTML file, set as output directory as the
    displacy_path argument.

    DOCS: https://spacy.io/api/cli#benchmark-accuracy
    F)r"   r$   r%   r&   r'   r(   silentr)   N)r   r   )
r    r!   r"   r#   r$   r%   r&   r'   r(   r)    r+   F/home/ubuntu/.local/lib/python3.10/site-packages/spacy/cli/evaluate.pyevaluate_cli   s   
r-   r*   returnc
              	      s  t || d}
t  t||d t|}t|}t|}| s+|
jd|dd |r9| s9|
jd|dd t||d}t|  t	| } j
||	d}|	re|}|d u r_|
d	 n|
d
 ni dddddddddddddddddddddd d!d"d#d$d%d&d'd(| d)d*d(| d+d,d(| d-d.d/i}i }i }| D ]J\}}||v r|d kr|d0 |d1d2 d3 }t|| ttfr|d/kr|| d4||< n|| d5 d6||< nd7||< || |td8d9| < q|
j|d:d; t||||d<}|rJ fd=d> jD }t	 d?d@ |d | D }dA|v }dB|v }dC|v }t||| ||||dD |
dE| dF| |d ur]t|| |
dG|  |S )HNno_printpretty)r*   zEvaluation data not foundr   )exitsz(Visualization output directory not found)r%   )r(   zdThe per-component option is enabled but there is no output JSON file provided to save the scores to.z7Per-component scores will be saved to output JSON file.TOK	token_accTAGtag_accPOSpos_accMORPH	morph_accLEMMA	lemma_accUASdep_uasLASdep_laszNER Pents_pzNER Rents_rzNER Fents_fTEXTCAT
cats_scorezSENT Psents_pzSENT Rsents_rzSENT Fsents_fzSPAN Pspans__pzSPAN R_rzSPAN F_fSPEEDspeedz (cats_score_descunk)z.0fd   .2f-z[\s/]_Results)titler)   r*   c                    s   g | ]}  |jqS r+   )get_pipe_metafactory).0pipenlpr+   r,   
<listcomp>   s    zevaluate.<locals>.<listcomp>c                 s   s    | ]}|j jV  qd S )N)	referencetext)r[   exr+   r+   r,   	<genexpr>   s    zevaluate.<locals>.<genexpr>parsernerspancat)
model_namelimitdepsentsspansz
Generated z parses as HTMLzSaved results to )r	   r   r   r   ensure_pathr   failr   
load_modellistr   warninfoitemsget
isinstanceintfloatresublowertablehandle_scores_per_type
pipe_namesr\   render_parsesgoodsrsly
write_json)r    r!   r"   r$   r%   r&   r'   r*   r)   r(   msgoutput_pathcorpusdev_datasetscoresdatametricsresultsmetrickeyfactory_namesdocsrender_depsrender_entsrender_spansr+   r]   r,   r   =   s   



	
 	
rX   r   r   c                C   sJ  t || d}d| v r| d rt|| d dd | d |d< d| v r5| d r5t|| d dd | d |d< d| v rL| d rLt|| d d	d | d |d< d
| d| v rw| d
| d rwt|| d
| d dd | d
| d |d
| d< d| v r| d rt|| d dd | d |d< d| v r| d rt|| d  | d |d< | S )Nr/   morph_per_featr9   featdep_las_per_typer?   typeents_per_typeNERrI   	_per_typeSPANScats_f_per_typez	Textcat Flabelcats_auc_per_type)r	   print_prf_per_typeprint_textcats_auc_per_cat)r   r   r)   r*   r   r+   r+   r,   r{      s8   r{       r   r   rg   rh   ri   rj   rk   c           	      C   s&  || d j d< |r3tj| d | ddd}|d jddd	}|| W d    n1 s.w   Y  |rbtj| d | d
dddid}|d jddd	}|| W d    n1 s]w   Y  |rtj| d | ddd}|d jddd	}|| W d    d S 1 sw   Y  d S d S )Nr   rW   entT)stylepagezentities.htmlwutf8)encodingdepcompact)r   r   optionszparses.htmlspanz
spans.html)	user_datar   renderopenwrite)	r   r   rg   rh   ri   rj   rk   htmlfile_r+   r+   r,   r}      s&   	"r}   r   namer   c           
      C   s~   g }|  D ]'\}}|g}dD ]}|| }	|t|	ttfr$|	d dn|	 q|| q| j|dd| d| dd d S )	N)prfrR   rS   )r   PRF)lr   r   r   z (per rQ   headeralignsrW   )rr   appendrt   ru   rv   rz   )
r   r   r   r   r   r   valuerowkvr+   r+   r,   r      s   &
r   c                 C   s$   | j dd | D dddd d S )Nc                 S   s.   g | ]\}}|t |ttfr|d n|fqS )rS   )rt   rv   ru   )r[   r   r   r+   r+   r,   r_      s    z.print_textcats_auc_per_cat.<locals>.<listcomp>)r   zROC AUC)r   r   zTextcat ROC AUC (per label)r   )rz   rr   )r   r   r+   r+   r,   r      s   
r   )Nr   FNr   Tr   F)r   r   TTT))rw   pathlibr   typingr   r   r   r   r   r   	thinc.apir   wasabir	   r   r   r   scorerr   tokensr   trainingr   _utilr   r   r   r   r   r   commandstrru   boolr-   r   r{   r}   rv   r   r   r+   r+   r+   r,   <module>   s    
	
,	


c



(

