o
    6iV!                     @   s  d Z ddlZddlm  mZ ddlZddlZddl	Z	ddl
Z
ddlZddlZddlmZ ddlmZ ejdeee jj e
jdd ddlmZ eee jjd  dd	lmZ dd
lmZ ddlm Z m!Z! ddl"m#Z# e	j$e	j%dd e	&dZ'eejd Z(eejd Z)dddZ*dZ+dZ,dedede-e  fddZ.dedede/ee-f de/fdd Z0d!d" Z1e2d#kre3e1  dS dS )$z
Quick 10-sample determinism test for GEMINI_PROJECT2 and GEMINI_PROJECT3.
Reuses canary_data segments (Telugu + Hindi, 5 each).
3 runs per key, compares exact transcript match.
    N)defaultdict)Path	MOCK_MODEfalse)load_dotenvz.env)polish_all_segments)AIStudioProvider)TranscriptionRequestRequestStatus)CacheManagerz1%(asctime)s [%(levelname)s] %(name)s: %(message)s)levelformatkey_testcanary_datakey_test_results5MDwLvubhR8BFU9eSKO_t4)tehi      	lang_codevideo_idreturnc           	   	   C   s   t |  | | d }| std|  g S t|dd td  }|s)g S t|}dd |D d t }g }|D ]#}|jj	}|jj
rQ|jj	 d|jj }|t||j| |jj	d q<|S )	NsegmentszNo segments at z*.flacr   c                 S   s   g | ]}|j js|qS  )	trim_meta	discarded).0pr   r   :/home/ubuntu/transcripts/preflight/key_consistency_test.py
<listcomp>3       z!load_segments.<locals>.<listcomp>_split)
segment_idaudio_base64language_codeoriginal_file)CANARY_DATAexistsloggererrorsortedglobSEGMENTS_PER_LANGr   r   r'   	was_splitsplit_indexappendr	   base64_audio)	r   r   seg_dirflacspolishedvalidrequestssegseg_idr   r   r    load_segments(   s*   
r:   key_nameapi_keyall_requestsc           !         s  t dd  t d|  d|dd  d t d  t|}z| I dH }t d|  W n tyQ } zt d	| d
 d}W Y d}~nd}~ww t||d}tt}g }t	t
D ]}	t d|	d  dt
 d t }
i }t| D ]f\}}t d| dt| d ||I dH }|D ]I}|jtjkr|jr| d|j }|| |j |jdd|jdd|j|jj|jjd||< qt d| d|j d|j d|j  qq~t |
 }||	d t|t|dd t d|	d  dt| d|dd  |	t
d k rt dI dH  qbd!}d!}td"d# }g }| D ]o\}}|!dd! }|| d$  d7  < t|t
k rQ|d7 }q/d%d& |D  t" fd'd( D rt|d7 }|| d)  d7  < q/|d7 }|| t#d*d( t$t% d!  d D t&t d! t d d+ q/|| }|d!kr|| d, nd!}|r|d- ni }t'd.d( | D }| |||t|dt(||dd |d/}t d0|  d1 t d2| d| d3|dd4 t| D ]\}}t d5| d|d)  d|d$   q|rH|dd6 D ]/}t d7|d8  d9|d:   t$|d; D ]\}} t d<|d  d| dd,   q/q|S )=zCRun determinism test for a single key: 3 runs, compare transcripts.
<============================================================zTesting z (N
   z...)z	  Cache: z  Cache setup failed: z, running uncached)r<   cached_content_namez
  --- Run    /z ---z    [z
] Sending z segments...:transcription detected_language)rE   rG   
latency_ms	cache_hitcached_tokensz] : z -    )runsegments_oktime_sz  Run z OK in z.1fsr   c                   S   s
   dddS )Nr   )totaldeterministicr   r   r   r   r    <lambda>|   s   
 ztest_key.<locals>.<lambda>rQ   c                 S   s   g | ]}| d dqS )rE   rF   )get)r   rr   r   r    r!      r"   ztest_key.<locals>.<listcomp>c                 3   s    | ]	}| d  kV  qdS )r   Nr   )r   ttextsr   r    	<genexpr>   s    ztest_key.<locals>.<genexpr>rR   c                 s   s$    | ]\}\}}||kr|V  qd S Nr   )r   iabr   r   r    rY         " )segmentrX   first_diff_chard   c                 s   s$    | ]\}}|r|d  rdV  qdS )rb   rB   Nr   )r   keyrunsr   r   r    rY      r^   )r;   total_segmentsrR   non_deterministicdeterminism_pctby_languagediffsrd   z
  === z RESULTS ===   deterministic (%)z    r   z	    DIFF r_   z: char r`   rX   z	      run))r*   infor   ensure_cache	Exceptionwarningr   r   listrangeDETERMINISM_RUNStime	monotonicr,   itemslen
send_batchstatusr
   SUCCESStranscription_datar$   r1   rT   rH   token_usagerI   rJ   r+   error_messageroundasynciosleepsplitallnext	enumeratezipminsumdict)!r;   r<   r=   cm
cache_nameeproviderall_runsrun_summariesrun_idxstartrun_resultsr   r7   	responsesresprc   elapsedrR   rf   
lang_statsri   rd   langrQ   pctlast_run
cache_hitsresultstatsdr[   rV   r   rW   r    test_keyC   s    *(	
 (&r   c               	      s  t jddd td} td}| r|std d S td i }t D ]'\}}t	||}|rF|||< td| dt
| d	 q(td| d
 q(tdd | D }td| dt
| d |srtd d S td| |I d H }td||I d H }t| dd | D |td||d}	t d }
t|
d}tj|	|ddtd W d    n1 sw   Y  tdd  td td  td|d  d|d   d!|d"  d# td$|d  d|d   d!|d"  d# td%|
  d S )&NT)parentsexist_okGEMINI_PROJECT2GEMINI_PROJECT3z7GEMINI_PROJECT2 and GEMINI_PROJECT3 must be set in .envz$Loading segments from canary_data...rj   rK   z segments readyz: no segments foundc                 s   s    | ]}t |V  qd S rZ   rw   )r   vr   r   r    rY      s    zmain.<locals>.<genexpr>zTotal: z segments across z languages
zANo segments available. Run canary_test.py first to download data.c                 S   s   i | ]	\}}|t |qS r   r   )r   kr   r   r   r    
<dictcomp>   s    zmain.<locals>.<dictcomp>)	languagessegments_per_langre   determinism_runs)test_configr   r   zkey_consistency_results.jsonwrL   F)indentensure_asciidefaultr>   r?   zKEY CONSISTENCY TEST COMPLETEz  PROJECT2: rR   rC   re   rk   rg   rl   z  PROJECT3: z  Results: )RESULTS_DIRmkdirosgetenvr*   r+   rm   
TEST_LANGSrv   r:   rw   rp   r   valuesr   rq   keysrs   openjsondumpstr)key2key3r=   r   r   segsrQ   result2result3combinedout_pathfr   r   r    main   sP   







**r   __main__)4__doc__builtins@py_builtins_pytest.assertion.rewrite	assertionrewrite
@pytest_arr   r   loggingr   sysrt   collectionsr   pathlibr   pathinsertr   __file__resolveparentenviron
setdefaultdotenvr   src.audio_polishr   src.providers.aistudior   src.providers.baser	   r
   src.cache_managerr   basicConfigINFO	getLoggerr*   r(   r   r   r.   rs   rq   r:   r   r   r   __name__rM   r   r   r   r    <module>   s@    "
"r: