o
    lQi`P                     @  s  d dl mZ d dlZd dlm  mZ d dlZd dl	Z	d dl
Z
d dlZd dlZd dlZd dlZd dlZd dlmZ d dlmZ d dlZd dlmZ ee jjZejd ee eed  d dlmZm Z m!Z!m"Z" d dl#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z- ej.ej/d	d
 e0dZ1dZ2ed d Z3ed d Z4dWddZ5dXddZ6dYd!d"Z7dZd#d$Z8d[d'd(Z9d\d+d,Z:d]d/d0Z;d^d4d5Z<G d6d7 d7Z=G d8d9 d9Z>d_d;d<Z?d`d@dAZ@dadFdGZAdbdMdNZBdcdPdQZCdddRdSZDdedTdUZEeFdVkre	GeE  dS dS )f    )annotationsN)Path)Any)load_dotenvz.env)	EnvConfigGEMINI_MODELLANGUAGE_MAPTHINKING_LEVEL)
InputScriptProfileTranscriptVariantBatchResult$build_transcript_variant_user_promptclassify_input_scriptdetect_script_countsextract_protected_spans'get_cacheable_transcript_variant_promptget_target_script_block"get_transcript_variant_json_schemaromanized_text_is_asciiz1%(asctime)s [%(levelname)s] %(name)s: %(message)s)levelformatvariant_prompt_testz0https://generativelanguage.googleapis.com/v1beta
final_dataz+transcript_variant_prompt_test_results.jsondataz&transcript_variant_prompt_samples.jsonpathr   payloaddict[str, Any]returnNonec                 C  s0   | j jddd | tj|ddddd  d S )NT)parentsexist_ok   F)indentensure_ascii	sort_keys
)parentmkdir
write_textjsondumps)r   r    r*   B/home/ubuntu/transcripts/scripts/test_transcript_variant_prompt.py
write_json,   s    r,   strc                  C  s(   t t d d d } d|  S )Nzutf-8   ztranscript-variant-)hashlibsha1r   encode	hexdigest)prompt_hashr*   r*   r+   get_cache_display_name1   s   


r4   itemslist[dict[str, Any]]sizeintlist[list[dict[str, Any]]]c                   s     fddt dt D S )Nc                   s   g | ]
} ||  qS r*   r*   ).0idxr5   r7   r*   r+   
<listcomp>9       zchunked.<locals>.<listcomp>r   )rangelenr<   r*   r<   r+   chunked8   s    rA   c                 C  s*   t |  }t|tstd|  |S )Nz'Sample file must contain a JSON array: )r(   loads	read_text
isinstancelist
ValueError)r   r   r*   r*   r+   load_samples<   s   
rG   sample_pooltarget_countc                 C  sd   |dkr| S g }t |D ]#}t| |t|   }|d |d< |d  d|d|d< || q|S )Nr   id	source_id__03d)r?   dictr@   append)rH   rI   expandedr;   baser*   r*   r+   expand_samplesC   s   rR   shard_index
num_shardsc                   s$    dkr| S  fddt | D S )N   c                   s    g | ]\}}|  kr|qS r*   r*   )r:   r;   itemrT   rS   r*   r+   r=   R        zapply_shard.<locals>.<listcomp>)	enumerate)r5   rS   rT   r*   rW   r+   apply_shardO   s   rZ   samplesAtuple[list[dict[str, Any]], list[dict[str, Any]], dict[str, int]]c              
   C  s   g }g }dd t D }| D ]E}t|d |d }||j  d7  < |d |d|d |d |d |jd}|t jkrM||d |d |jd	d	d
d q|| q|||fS )Nc                 S  s   i | ]}|j d qS )r   )value)r:   profiler*   r*   r+   
<dictcomp>X       z!prepare_items.<locals>.<dictcomp>textlanguage_coderU   rJ   rK   )rJ   rK   rb   ra   input_script_profile fully_roman_local_skip)rJ   rb   rc   native_script_textromanized_textskip_reason)r
   r   r]   getfully_romanrO   )r[   local_skipsmodel_itemsprofile_countssampler^   preparedr*   r*   r+   prepare_itemsU   s2   


rp   configr   dict[int, str]c                 C  s   dd t | jD S )Nc                 S  s   i | ]\}}||qS r*   r*   )r:   r;   keyr*   r*   r+   r_   v   s    zget_key_map.<locals>.<dictcomp>)rY   gemini_keys)rq   r*   r*   r+   get_key_mapu   s   ru   c                   @  s>   e Zd ZdddZdd	d
ZdddZdddZdddZdS )TranscriptVariantCacheManagerapi_keyr-   c                 C  s
   || _ d S N)rw   )selfrw   r*   r*   r+   __init__z   s   
z&TranscriptVariantCacheManager.__init__ttl_sr8   r   r   c                   s(   |   I d H }|r|S | |I d H S rx   )_find_existing_cache_create_cache)ry   r{   existingr*   r*   r+   ensure_cache}   s
   z*TranscriptVariantCacheManager.ensure_cachedict[str, Any] | Nonec              	     s6  t  d| j }t }tjdd4 I d H x}||I d H }|jdkr=td|j|j	d d  	 W d   I d H  d S dt
 }| dg D ]8}|d|kr|d	|kr| |d
 I d H }|rt|  W  d   I d H  S |  W  d   I d H  S qJW d   I d H  d S 1 I d H sw   Y  d S )N/cachedContents?key=      >@timeout   zCould not list caches: %s %smodels/cachedContentsmodeldisplayNamename)AISTUDIO_BASErw   r4   httpxAsyncClientri   status_codeloggerwarningra   r   r(   
_get_cache)ry   urldisplay_nameclientresp
model_namecachedetailedr*   r*   r+   r|      s0   

z2TranscriptVariantCacheManager._find_existing_cache
cache_namec              	     s   t  d| d| j }tjdd4 I d H '}||I d H }|jdkr2| W  d   I d H  S W d   I d H  d S 1 I d H sCw   Y  d S )N/z?key=r   r   r   )r   rw   r   r   ri   r   r(   )ry   r   r   r   r   r*   r*   r+   r      s   
z(TranscriptVariantCacheManager._get_cachec              	     s   t  d| j }dt t ddt igi| dd}tjdd4 I d H .}|j||d	I d H }|jd
krGt	d|j d|j
d d  | W  d   I d H  S 1 I d H s[w   Y  d S )Nr   r   partsra   s)r   r   systemInstructionttl      N@r   r(   r   zCache creation failed:    )r   rw   r   r4   r   r   r   postr   RuntimeErrorra   r(   )ry   r{   r   bodyr   r   r*   r*   r+   r}      s   
 0z+TranscriptVariantCacheManager._create_cacheN)rw   r-   )r{   r8   r   r   )r   r   )r   r-   r   r   )__name__
__module____qualname__rz   r   r|   r   r}   r*   r*   r*   r+   rv   y   s    



rv   c                   @  s    e Zd ZdddZdddZdS )TranscriptVariantClientrw   r-   r   
str | Nonec                 C  s   || _ || _t | _d S rx   )rw   r   r   schema)ry   rw   r   r*   r*   r+   rz      s   z TranscriptVariantClient.__init__r5   r6   r   r   c              	     s  t  dt d| j }ddt|igdgdd| jdt id	d
}| jr-| j|d< n
ddt igi|d< t	j
t	jdddddd4 I d H .}|j||dI d H }|jdkrhtd|j d|jd d  | W  d   I d H  S 1 I d H s|w   Y  d S )Nz/models/z:generateContent?key=userra   )roler   r   zapplication/jsonthinkingLevel)temperatureresponseMimeTyperesponseJsonSchemathinkingConfig)contentsgenerationConfigcachedContentr   r   g      $@r   )connectreadwritepoolr   r   r   zGenerate failed: r   r   )r   r   rw   r   r   r	   upperr   r   r   r   Timeoutr   r   r   ra   r(   )ry   r5   r   r   r   r   r*   r*   r+   generate_batch   s,   $
 0z&TranscriptVariantClient.generate_batchN)rw   r-   r   r   )r5   r6   r   r   )r   r   r   rz   r   r*   r*   r*   r+   r      s    
r   response_jsonc                 C  sR   |  dg }|std|d  di  dg D ]}d|v r$|d   S qtd)N
candidatesz No candidates in Gemini responser   contentr   ra   zNo text part in Gemini response)ri   rF   )r   r   partr*   r*   r+   _extract_response_text   s   r   expectedactual	list[str]c                 C  sl  g }|d | d kr| | d  d|d  d |d  s)| | d  d |d  s9| | d  d t| d D ]&}||d vrS| | d  d	|  ||d vre| | d  d
|  q?t|d sv| | d  d t| d }t|d }t|d }|dkr||ddkr| | d  d |dkr||ddkr| | d  d |S )NrJ   z: id mismatch ()rf   z: empty native_script_textrg   z: empty romanized_textra   z2: protected span missing in native_script_text -> z.: protected span missing in romanized_text -> z;: romanized_text contains non-ASCII outside protected spansrb   Latinr   z5: native_script_text missing target script charactersz8: romanized_text still contains target script characters)rO   stripr   r   r   r   ri   )r   r   errorsspantarget_scriptnative_countsroman_countsr*   r*   r+   validate_item   s.   r   run_outputslist[dict[str, dict[str, Any]]]item_ids tuple[int, list[dict[str, Any]]]c                   sr   d}g }|D ].  fdd| D t fddD r!|d7 }q|  fdd| d d D d	 q||fS )
Nr   c                   s$   g | ]}t j| i d ddqS )FTr"   r#   )r(   r)   ri   r:   runitem_idr*   r+   r=          z compare_runs.<locals>.<listcomp>c                 3  s    | ]	}| d  kV  qdS )r   Nr*   )r:   r]   )
serializedr*   r+   	<genexpr>   s    zcompare_runs.<locals>.<genexpr>rU   c                   s   g | ]}|  i qS r*   ri   r   r   r*   r+   r=     s       )rJ   runs)allrO   )r   r   deterministicdiffsr*   )r   r   r+   compare_runs   s   
r   	key_indexrw   argsargparse.Namespacecheckpoint_payloadc           (        s  t |\}}}dd |D }t|}	|	|jI d H }
|
d }|
di dd}t||d}td| t|t|t|| g  g }g }g }t	|j
D ]}td	| |d
 |j
 t }dd |D }g }g }g }t||j}t|d
dD ]\}}||I d H }|di }|||dd|dd|ddd t|}zt|}W n" ty } z|d|d
  d| d|  W Y d }~qxd }~ww t|jt|kr|d|d
  d| dt| dt|j  qxt||jD ]\}} |  }!t||!}"|"r||" |!||d < q|d
 |t|t|d|di t| < tt|j| qxt t | d}#||d
 |#t|t|t||d  | || || td| |d
 |#t|t| qMt! |\}$}%t! dd |D \}&}'i d| d|d |d!|d"kd#t|d$t|d%t|d&|d'|d(|$t|t |$t"t|d
 d) d|%d d* d+d,|&t|t |&t"t|d
 d) d|'d d* d+d-t|d.t|d/|d d0 d1|d d0 d2 fd3d|d d* D S )4Nc                 S     g | ]}|d  qS rJ   r*   r:   rn   r*   r*   r+   r=     r`   zrun_for_key.<locals>.<listcomp>r   usageMetadatatotalTokenCountr   )rw   r   zI[key %s] samples=%s model_items=%s local_skips=%s cached_prompt_tokens=%sz[key %s] starting run %s/%srU   c                 S  s   i | ]}|d  |qS r   r*   r:   rV   r*   r*   r+   r_   ,  s    
zrun_for_key.<locals>.<dictcomp>)startpromptTokenCountcachedContentTokenCountcandidatesTokenCount)batchprompt_tokenscached_tokensoutput_tokenszrun z batch z: z: expected z results, got rJ   )current_runcurrent_batchtotal_batchescompleted_outputsper_key_progressr    )r   	elapsed_soutputsschema_errorsvalidation_errorsusagezC[key %s] run %s done in %ss (schema_errors=%s validation_errors=%s)c                 S  r   r   r*   r   r*   r*   r+   r=   t  r`   r   r   cached_prompt_tokenscache_threshold_oki   sample_countmodel_item_countlocal_skip_countrm   run_summariesdeterminism_all_itemsd      )r   totalpctdiff_examplesdeterminism_model_itemsschema_error_countvalidation_error_countr      r   sample_outputsc                   s   g | ]
} d   |i qS )r   )r:   r   r   r*   r+   r=     r>   )#rp   rv   r   r{   ri   r   r   infor@   r?   r   time	monotonicrA   
batch_sizerY   r   rO   r   r   model_validate_json	Exceptionresultszip
model_dumpr   extend
setdefaultr-   r,   r   outputroundr   max)(r   rw   r   r[   r   rk   rl   rm   r   cache_manager
cache_infor   r   r   r  validation_errors_allschema_errors_allrun_idx	run_startmerged_outputs
usage_rowsrun_validation_errorsrun_schema_errorsbatches	batch_idxr   r   r   response_textparsedexcr   r   actual_dictr   r   deterministic_all	diffs_alldeterministic_modeldiffs_modelr*   r  r+   run_for_key
  s  	


	 (





		


r2  key_resultsc                 C  s   t | dk ri S | d d }g }| dd  D ]J}d}tt |t |d }t||d D ]\}}tj|dddtj|dddkrD|d7 }q,|| d d |d ||t|t|d d	 dd
 qd|iS )Nr    r   r  rU   FTr   r   r  )left_key_indexright_key_indexexact_match_on_sample_outputssample_sizer  sample_output_cross_key)r@   minr  r(   r)   rO   r  r  )r3  	referencecomparison_rowsothersamer  leftrightr*   r*   r+   compare_keys  s(    
	r@  c                  C  s   t jdd} | jdttd | jdtdd | jdtdd | jd	td
d | jdtdd | jdtdd | jdtdd | jdtdd | jdttd |  S )Nz/Live Gemini test for transcript variant prompt.)descriptionz--sample-file)typedefaultz--sample-targetr  z--runsr  z--batch-size
   z--key-indices0z--num-shardsrU   z--shard-indexr   z--ttl-si  z--output)	argparseArgumentParseradd_argumentr   DEFAULT_SAMPLE_FILEr8   r-   DEFAULT_OUTPUT
parse_args)parserr*   r*   r+   rK    s   rK  c                    s  t  } t }t|}dd | jdD }|D ]}||vr*td| dt| qt| j}t	|| j
}t|| j| j  sDtdtt dtt| jt|| j
| j| j| j| j|d
}t| j| g }|D ]4}t||| |  |d	I d H }	||	 |d
g ||	d |	d d |	d |	d d t| j| qji |tt d|t| fddtdd  D D d}
t| j|
 td| j |D ]}	td|	d |	d |	d d |	d |	d  qd S )Nc                 S  s    g | ]}|  rt|  qS r*   )r   r8   )r:   r   r*   r*   r+   r=     rX   zmain.<locals>.<listcomp>,z
Key index z not available. Have: zNo samples left after sharding.r   )
started_at_epoch_sr   sample_filesample_pool_sizesample_targetrS   rT   r   r  requested_key_indices)r   rw   r   r[   r   completed_key_resultsr   r	  r  r  r
  )r   r   determinism_model_items_pctr  r
  c                   s$   i | ]  t  fd dD qS )c                 3  s     | ]}|d   krdV  qdS )rb   rU   Nr*   r   coder*   r+   r     s    z"main.<locals>.<dictcomp>.<genexpr>)sum)r:   sharded_samplesrU  r+   r_     r   zmain.<locals>.<dictcomp>c                 S  s   h | ]}|d  qS )rb   r*   r   r*   r*   r+   	<setcomp>  r`   zmain.<locals>.<setcomp>)finished_at_epoch_sr3  cross_key_summarylanguage_distributionzSaved results -> %szU[key %s] cache_tokens=%s model_determinism=%s%% validation_errors=%s schema_errors=%sr   )rK  r   ru   key_indicessplitrF   sortedrG   rO  rR   rQ  rZ   rS   rT   r  r  r   r-   r@   r   r  r,   r  r2  rO   r  r@  r   r  )r   rq   key_maprR  r   rH   expanded_samplesr   r3  resultfinal_payloadr*   rX  r+   main  s   


	


re  __main__)r   r   r   r   r   r   )r   r-   )r5   r6   r7   r8   r   r9   )r   r   r   r6   )rH   r6   rI   r8   r   r6   )r5   r6   rS   r8   rT   r8   r   r6   )r[   r6   r   r\   )rq   r   r   rr   )r   r   r   r-   )r   r   r   r   r   r   )r   r   r   r   r   r   )r   r8   rw   r-   r   r   r[   r6   r   r   r   r   )r3  r6   r   r   )r   r   )r   r   )H
__future__r   builtins@py_builtins_pytest.assertion.rewrite	assertionrewrite
@pytest_arrF  asyncior/   r(   loggingossysr  pathlibr   typingr   r   dotenvr   __file__resolver%   ROOTr   insertr-   
src.configr   r   r   r	   src.transcript_variant_promptr
   r   r   r   r   r   r   r   r   r   basicConfigINFO	getLoggerr   r   rJ  rI  r,   r4   rA   rG   rR   rZ   rp   ru   rv   r   r   r   r   r2  r@  rK  re  r   r   r*   r*   r*   r+   <module>   sX    "0








 2
$



 



L