o
    3NPi4                     @   s  d Z ddlZddlZddlZddlZddlmZ ddlmZm	Z	m
Z
mZmZ ddlmZ ddlmZmZ ddlZddlmZ ddlmZ dd	lmZ dd
lmZmZmZmZ ddlmZm Z  ddl!m"Z"m#Z#m$Z$ ddl%m&Z&m'Z' ddl(m)Z) ddl*m+Z+m,Z, ddl-m.Z.m/Z/ ddl0m1Z1m2Z2 ddl3m4Z4m5Z5m6Z6 ddl7m8Z8m9Z9m:Z:m;Z; ddl<m=Z=m>Z>m?Z? e@dZA		d&deBdedeeB dee deeBe	f f
ddZCdeBdedeeeB ee eeD f fddZE		 	 d'd!e
eB dee d"eFd#eFde
e f
d$d%ZGdS )(ug	  
Pipeline v6.7 - Music Detection for TTS Quality

=== NEW IN v6.7: MUSIC DETECTION ===
- PANNs CNN14 for music/instrument detection in 1.5s chunks
- Segments marked as: clean / needs_demucs / heavy_music
- Conservative thresholds (5%/25%) to protect TTS training data
- Reuses existing 1.5s chunk infrastructure (minimal overhead)

=== FROM v6.6: 18x FASTER EMBEDDINGS ===
- FIXED: 35s unexplained overhead in embedding extraction!
- Root cause: Per-batch tensor creation/transfer was serializing GPU work
- Solution: Pre-allocate ALL audio on GPU in ONE transfer, then slice
- Result: 37s → 2s for embeddings (18x faster!)
- GPU utilization: 3.6% → 80%+ during embedding phase

=== FROM v6.5: MAXIMUM GPU SATURATION ===
- Eliminated per-chunk VAD calls (was 55s overhead for 2800 chunks!)
- Fast energy-based eligibility (0.001s vs 0.02s per chunk)
- Pinned memory for faster CPU→GPU transfer
- Removed per-batch empty_cache() sync stalls

=== FROM v6.4: TURBO OPTIMIZATION ===
- Unified Chunk Embedding Cache: Compute ONCE, use EVERYWHERE
- Eliminates "Double Compute" bottleneck (was: Embeddings + ChunkReassignment both used GPU)
- Single mega-batch GPU pass extracts ALL 1.5s chunk embeddings
- Segment embeddings derived from cache (CPU only - instant)
- Chunk reassignment uses cached embeddings (CPU only - instant)

=== FROM v6.3 ===
- Chunk-based reassignment: Detects within-segment speaker changes
- Hybrid approach combining ChatGPT + Gemini Maya + Gemini Stabilized insights
- Look-ahead pattern with severe threshold circuit-breaker
- VAD-based chunk eligibility filtering
- Margin-based confident reassignment

=== FROM v6.2 ===
- Shared audio buffer: Load audio ONCE, pass to all stages
- Persistent VAD workers: Model loaded once per worker
- In-memory diarization chunks: No disk writes
- GPU memory-aware batching: Dynamic batch sizes
- Failure-safe cleanup: atexit handler

Flow (optimized for max TTS data quality):
1. Download (parallel workers for batch)
2. Quick VAD (parallel, get speech outline)
3. Chunk at silence boundaries (VAD-aware, IN-MEMORY)
4. Diarization + OSD (single pass, no disk I/O)
5. Quality filtering
6. UNIFIED Embeddings (single mega-batch for ALL chunks) - v6.4 TURBO
7. Conservative clustering (uses cached segment embeddings)
8. Chunk reassignment (uses cached chunk embeddings - CPU only!) - v6.4 TURBO
9. Duration filtering
10. Output metadata JSON
    N)Path)DictAnyListOptionalTuple)datetime)ThreadPoolExecutorFuture)Config)COMPUTE)MODELS)download_audiovalidate_videoVideoValidationErrorDownloadError)run_vad_parallelfind_optimal_cut_point)detect_nonspeech_regionssplit_on_overlapsfilter_by_quality)create_chunksrun_diarization)extract_embeddings_batched)merge_speakersmerge_adjacent_segments)generate_samplescleanup_chunks)AudioBufferTEMP_MANAGER)PrecisionSurgicalSplitterChunkReassignmentConfigexecute_chunk_reassignment)UnifiedChunkEmbeddingCacheUnifiedEmbeddingConfig#build_segment_embeddings_from_cache!execute_chunk_reassignment_cached)MusicDetectionCacheMusicDetectionConfigbuild_segment_music_stats_batchzFastPipelineV6.Pipeline	video_urlconfigprefetched_audioprefetched_metadatareturnc           O         s4  t jstd t  td td|   td t }i }i }|durD|durDtd |}|}d|d< ddd	|d< n*td
}	t| \}}W d   n1 s[w   Y  |	j	|d< |	j
|	jd	|d< td}	t| W d   n1 sw   Y   j	}
|	j	|d< tjdt|
j d}	t| dW d   n1 sw   Y  |	j	|d< |	j
|	jd	|d< td}	t| dd}W d   n1 sw   Y  |	j	|d< |	j
|	jd	|d< g }g }tjdt|d}	t|jd\}}t|
}W d   n	1 sw   Y  |	j	|d< |	j
|	jd	|d< |rIjrIt|}t||jd}td| dt| d g }jrtd}	t||jj d\}}W d   n	1 smw   Y  |	j	|d< |	j
|	jd	|d< ddlm} ddl}td jd!j j!d"}t" t j#t j$t j%|t & d#}d}d}|' dg fd$d%}|j(|dd&}j)r|*  td'}	|+ }|, }td(| d)|d* d+d, W d   n	1 sw   Y  |	j	|d-< |	j
|	jd	|d-< td.}	t-||}td/t| d0 W d   n	1 s&w   Y  |	j	|d1< |	j
|	jd	|d1< j)rGj.d2d3 d }td4}	t/|||}W d   n	1 s_w   Y  |	j	|d5< |	j
|	jd	|d5< d}j0rtd6s}	dd7l1m2} |t3}t4|D ]\}} ||v r| 5d8d9vr|| d8  6||  qd:d; |7 D }!t8|||!j9d< j9d=d>dd?\}}"G d@dA dA}#|#|"}tdB|"dC  dD|"dE  dF|"dG  dH|"dI   W d   n	1 sw   Y  |	j	|dJ< |	j
|	jd	|dJ< j)r|durtdK}	zV|+ }$t:||\}}tdL|$ dM tdN|dO  dP|5dQd dR tdS|dT  dP|5dUd dR tdV|5dW|5dXd dP|5dY|5dZd dR W nI t;y }% zt<d[|%  d\t=|%i}W Y d}%~%n-d}%~%w t>y }% zt?d]|%  ddl@}&|&A  d\t=|%i}W Y d}%~%nd}%~%ww W d   n	1 sw   Y  |	j	|d^< |	j
|	jd	|d^< d}'d}(td_3}	t3|})|D ]} d`| da< | 5dbdc| db< |)6|  q|D ]} |)6| dd | de | df dgd`dhdi q|)jBdjdk dl tC|)}*dmdn |*D }+dodn |*D },jDdkrvg }-g }.|+D ]} | df jDkrJ|-6|  q9|.6|  q9t|.}'tEdpdq |.D }(|'dkrttdr|' dsjD dt|(dudv |-}+|+|, }dwdn |D }/dxdn |D }0d}1d}2|D ]V} | dd | de dy}3|/D ]$}4|4dd kr|4de krtFfdzdq|0D }5|5sd}3 nq|3r|1d{7 }1|2| df 7 }2q|6| dd | de | df d|d`d}di q|1dkrtd~|1 d|2dud |jBddk dl ddlGmH}6 |6|}W d   n	1 sw   Y  |	j	|d< tI| tJK| tL|d }7i }8jMrVtd}	tN|||7}8W d   n	1 sLw   Y  |	j	|d< t | }9t3tOddq |D }:ddn |D };ddn |D }0ddn |D }<ddn |D }=ddn |D }>tEddq |;D }?tEddq |0D }@tEddq |<D }AtEddq |=D }BtEddq |>D }C|'}D|(}Eg }F|5dd}G|D ]Y} |F6tP| dd dtP| de dtP| df d| d8 | 5dadtP| dd |G dtP| de |G dd d| v r| d |Fd d< | 5drd|Fd d< db| v r| db |Fd db< qi d|d d|5ddd|5d| dtP|5d|
d<dtP|Gd<dtP|5d|
d<d|5ddd|5ddd|5dd|5ddydt|:d|:dt|d|Fdi dt|;dtP|?d<d|
dkrtP|?|
 d d{nddt|0dtP|@d<d|
dkrtP|@|
 d d{nddt|<dtP|Ad<d|
dkrtP|A|
 d d{nddt|=dtP|Bd<dt|DdtPtQ|Ed<dt|>dtP|Cd<d|r|jRndd|r|jSndd^|r	|ni idttPtEddq D d<dddd; |87 D dd; |7 D tP|9d<|i dt jTdÈjUdĈjVdňjWdƈjdǈjDdȈjdɈjdʈj0dˈj!d̈j d͈j)dΈjXdψjYdЈjZdшj[t\] ^ ddt_ v r|ni dԜ}Ht`|7d dփ}Itajb|H|Id<d׍ W d   n	1 sw   Y  |
d }J|Jdkr|9|J nd}Ktd|
d dud|Hd  d|Hd d  d|9dudt|Kdudޝ j[r|5drtL|}LtL|5ddߡ}M|Lc r|Mc r|Ld jed }N|Lf  td|Ndud t jgdd th  |HS )a  
    Process a single video through the complete pipeline with compute monitoring.
    
    === v7.0 OPTIMIZATION: Prefetched Download Support ===
    If prefetched_audio and prefetched_metadata are provided (from batch prefetch),
    the download stage is skipped entirely, saving ~60s per video.
    
    Pipeline stages (per instructions.md):
    1. Download audio (or use prefetched)
    2. VAD (parallel CPU) - get speech activity outline
    3. OSD (GPU) - mark overlaps as unusable FIRST
    4. Chunking (VAD-aware) - cut at silence boundaries
    5. Diarization (GPU) - speaker identification
    6. Embeddings (GPU) - speaker vectors
    7. Clustering (CPU) - conservative merging
    8. Output metadata JSON
    
    Each stage is monitored for CPU/GPU utilization.
    
    NOTE: Models are loaded automatically if not already loaded.
    This allows calling this function directly without going through main().
    
    Args:
        video_url: YouTube URL to process
        config: Pipeline configuration
        prefetched_audio: Optional path to already-downloaded audio (from batch prefetch)
        prefetched_metadata: Optional metadata dict (from batch prefetch)
    z6Loading models (first call to process_single_video)...zF----------------------------------------------------------------------u   📹 Processing: Nu2   ⚡ Using prefetched download (batch optimization)g        downloadr   )cpugpuDownload
LoadBufferload_bufferVAD)items)audio_buffervadChunkingT)r7   	in_memorychunkingzDiarization+OSD)extract_overlapsdiarization)min_segmentz   Split overlaps: u    → z# segments (salvaged clean portions)QualityFilter)
min_snr_dbmin_quality_scorer7   quality_filter)r	   g      ?   )chunk_durationmin_speech_ratio
batch_sizesevere_thresholdnormal_threshold)r7   vad_segmentsembedding_model	vad_model	vad_utilsr+   devicec               
      s<  zzoj sW W   dS tjtt d} tdi djdjdj	dj
djdjd	jd
jdjdjdjdjdjdjdjdj}t | |t dd< d   W n ty } ztd|  W Y d}~nd}~ww W   dS W   dS   w )zOPrepare music detection cache (CPU-intensive: chunk creation + 32kHz resample).N)rM   rD   rF   music_prob_thresholdmusic_ratio_cleanmusic_ratio_demucsmusic_mean_cleanmusic_mean_demucsnoise_prob_thresholdnoise_ratio_cleannoise_ratio_demucsnoise_mean_cleannoise_mean_demucsstrict_tts_modeearly_exit_enabledearly_exit_sample_ratioearly_exit_threshold)r7   rI   panns_modelr+   rM   r   zMusic prep failed:  )enable_music_detectionsetr   
load_pannsstr
get_devicer(   music_chunk_durationmusic_batch_sizerN   rO   rP   rQ   rR   rS   rT   rU   rV   rW   rX   music_early_exitmusic_early_exit_sample_ratiomusic_early_exit_thresholdr'   _precompute_resampled_audio	Exceptionloggerwarning)r\   music_confige)r7   r+   music_cache_holdermusic_prep_readyrI   r]   N/home/ubuntu/.cursor/worktrees/maya3data__SSH__216.81.248.184_/zxg/pipeline.pyprepare_music_cache#  sl   $

z1process_single_video.<locals>.prepare_music_cache)targetdaemonUnifiedEmbeddingsz   Unified cache: z chunk embeddings in build_time_secz.2fsunified_embeddingsSegmentEmbeddingsz   Derived z) segment embeddings from cache (CPU only)segment_embeddings   timeout
Clustering
clusteringChunkReassignment)defaultdictspeakerOVERLAP
NON_SPEECHc                 S   s$   i | ]\}}|r|t j|d dqS )r   )axis)npmean).0spkembsr]   r]   rp   
<dictcomp>  s    z(process_single_video.<locals>.<dictcomp>   g?g?)segmentscachespeaker_centroidsmin_segment_for_analysismin_split_portionassign_min_similarity
margin_mincreate_new_on_ambiguousc                   @   s   e Zd Zdd ZdS )z5process_single_video.<locals>.ReassignmentStatsCompatc                 S   s    | dd| _| dd| _d S )Nsegments_with_changesr   new_speakers_created)getr   r   )self
stats_dictr]   r]   rp   __init__  s   z>process_single_video.<locals>.ReassignmentStatsCompat.__init__N)__name__
__module____qualname__r   r]   r]   r]   rp   ReassignmentStatsCompat  s    r   z    Chunk reassignment (cached): r   z segments split, r   z new speakers, severe=severe_triggersz, lookahead=lookahead_triggerschunk_reassignmentMusicDetectionz   Music detection: z chunks analyzedz
   Clean: segments_cleanz segments (	pct_cleanz%)   Needs Demucs: segments_needs_demucspct_needs_demucsz   Heavy Contamination: segments_heavy_contaminationsegments_heavy_musicpct_heavy_contaminationpct_heavy_musicu!   ⚠️ Music detection disabled: erroru   ❌ Music detection failed: music_detectionFinalizationunusablestatusunusable_reasonlow_qualitystartenddurationr   overlap)r   r   r   r   r   r   c                 S      | d S Nr   r]   xr]   r]   rp   <lambda>      z&process_single_video.<locals>.<lambda>)keyc                 S      g | ]}| d dvr|qS )r   r   r   r   rv   r]   r]   rp   
<listcomp>  s
    z(process_single_video.<locals>.<listcomp>c                 S      g | ]}| d dkr|qS r   r   r   r   r]   r]   rp   r         c                 s       | ]}|d  V  qdS r   Nr]   r   r]   r]   rp   	<genexpr>      z'process_single_video.<locals>.<genexpr>z)   Duration filter (post-merge): dropped z speaker segments <zs (.1fzs total)c                 S   r   )r   )r   r   Nr   r   r]   r]   rp   r     r   c                 S   r   r   r   r   r]   r]   rp   r     r   Fc                 3   s(    | ]}|d   k o|d kV  qdS )r   r   Nr]   )r   ov)ns_endns_startr]   rp   r     s
    
   r   
non_speechz   Filtered z internal pauses (zs) within speaker segmentsc                 S   r   r   r]   r   r]   r]   rp   r   7  r   ) filter_overlap_sandwich_segmentsfinalization
output_dirSamplessamplesc                 s   s$    | ]}|d  dvr|d  V  qdS )r   r   Nr]   r   r]   r]   rp   r   U  s   " c                 S   r   )r   usabler   r   r]   r]   rp   r   V  r   c                 S      g | ]
}|d  dkr|qS r   r]   r   r]   r]   rp   r   W      c                 S   r   )r   r   r]   r   r]   r]   rp   r   X  r   c                 S   r   )r   r   r   r   r]   r]   rp   r   Y  r   c                 S   r   )r   overlap_proximityr   r   r]   r]   rp   r   Z  r   c                 s   r   r   r]   r   r]   r]   rp   r   \  r   c                 s   r   r   r]   r   r]   r]   rp   r   ]  r   c                 s   r   r   r]   r   r]   r]   rp   r   ^  r   c                 s   r   r   r]   r   r]   r]   rp   r   _  r   c                 s   r   r   r]   r   r]   r]   rp   r   `  r   intro_skipped   r   )r   r   r   r   r   original_startoriginal_endmusic_statsneeds_demucsvideo_idvideo_titletitleUnknownyoutube_urloriginal_durationprocessed_durationsample_ratei>  original_sample_rateoriginal_audio_pathoriginal_audio_preservednum_speakersspeakerstotal_segmentsr   quality_statsusable_segmentsusable_durationusable_percentaged   overlap_segmentsoverlap_durationoverlap_percentagenonspeech_segmentsnonspeech_durationnonspeech_percentagelow_quality_segmentslow_quality_durationtoo_short_segmentstoo_short_durationoverlap_proximity_segmentsoverlap_proximity_durationsplit_segmentsnew_speakers_from_splits	vad_statsc                 s   r   r   r]   r   r]   r]   rp   r     r   )speech_segmentstotal_speechsample_clipsc                 S   s    i | ]\}}|d d |D qS )c                 S   s   g | ]}t |jqS r]   )r   name)r   pr]   r]   rp   r     s    z3process_single_video.<locals>.<dictcomp>.<listcomp>r]   )r   r   pathsr]   r]   rp   r     s     c                 S   s   i | ]
\}}|t |d qS )r   )round)r   kvr]   r]   rp   r     r   modelmerge_thresholdvad_workersembedding_batch_sizemin_segment_durationmin_tts_durationr@   r   chunk_reassignment_enabledchunk_reassignment_thresholdchunk_reassignment_severemusic_detection_enabledrO   rP   auto_intro_skippreserve_original_audiozv7.1-strict-ttscache_stats)timingtiming_totalcompute_statsr+   processed_atpipeline_versionembedding_cache_statszmetadata.jsonw)indent  u   📊 Result: <   zmin | z speakers | z% usable | zs/hr) i   u'   🧹 Cleaned up 16kHz processing file (zMB), keeping original quality
aggressive)ir   _loadedrj   infoload_alltimer   monitor_stager   r   cpu_percent_avggpu_percent_avgr   	from_fileintvad_chunk_sizer   r   lenr   detect_overlapr   r   r  r   r@   rA   concurrent.futuresr	   	threadingr$   chunk_reassignment_min_speechr  r  r#   rJ   
silero_vadsilero_utilsrb   EventThreadr^   r   build	get_statsr%   waitr   enable_chunk_reassignmentcollectionsr   list	enumerater   appendr6   r&   chunk_reassignment_min_portionr)   ImportErrorrk   ra   ri   r   	traceback	print_excsortr   r  sumanysrc.overlap_detectionr   r   r   cleanup_for_audior   generate_sample_clipsr   r_   r  floatr   r   _model_namecluster_merge_thresholdr  r  rO   rP   r  r  r   now	isoformatdiropenjsondumpexistsstatst_sizeunlinkclear_cacherefresh_gpu_memory)Or*   r+   r,   r-   total_starttimingsr  
audio_pathmetadatametricstotal_durationchunksr   r   r   original_countr   r	   r1  embedding_configembedding_cachemusic_cachemusic_detection_statsrq   music_prep_thread
num_chunksr  
embeddingschunk_reassignment_statsr   speaker_embeddingsisegr   reassignment_statsr   num_music_chunksrm   rA  dropped_short_countdropped_short_durationspeaker_segments_for_mergemerged_all_segmentsmerged_speaker_segmentsoverlap_output_segmentskeptdroppedspeaker_segsoverlap_segsinternal_filtered_countinternal_filtered_durationis_internalsphas_overlap_nearbyr   r   r  
total_timeall_speakersusable_segsnonspeech_segslow_quality_segsoverlap_proximity_segsr   r   r   r   r   too_short_countr   output_segmentsintro_offsetresultfprocessing_hoursratetrimmed_fileoriginal_filetrimmed_size_mbr]   )r7   r+   rn   ro   r   r   rI   rp   process_single_videoi   sf  $








	



+ 



	


&




		


	
n






	


 
 
 	

(,-	
M

r  urlc              
   C   sN   zt | |dd\}}||dfW S  ty& } zdd|fW  Y d}~S d}~ww )z
    Background download for batch prefetching.
    
    Returns:
        (audio_path, metadata, error) - error is None on success
    F)validateN)r   ri   )r  r+   rZ  r[  rm   r]   r]   rp   _prefetch_download  s   r  T
video_urlsvalidate_videosskip_on_validation_failc           )         s  |pt  }td tdt|  d td td|j  td|j  td|j  td|j d	 td
 td g }i  |rtd t| D ]]\}}td|d  dt|  d|  t	|\}}}	|||	d |< |r|
| |	dddd }
|	dd}td|
 d| d qXtd|  |std| d| qXtdt| dt|  d |std  fd d!| D S n| }t| g }t }| D ]}||vr|
d" | d#  |d$d% qtdd&d'}d}d}t|dkr td( |t|d |}|d }t|D ]h\}}td)d  td*|d  dt| d+t|  d, td  zd}d}|dur||krtd-|  |jd.d/\}}}|du r|dur|}|}td0t|j  n	td1| d2 d}d}|d t|k r||d  }td3|  |t||}|}t||||d4}|
| tjd5d6 t  W q$ ty } ztd7| d|  |
t||d$d% W Y d}~q$d}~w ty } ztd8| d|  |
t||d9d% W Y d}~q$d}~w tyV } z+td:| d|  |
t||d;d% tjd5d6 ddl}|   W Y d}~q$d}~w t!y } z+td<| d|  ddl"}|#  |
t||d=d% tjd5d6 W Y d}~q$d}~ww |j$d>d? t | }td@ tdA td dBd! |D }dCd! |D }tdDt| dt|   tdEt| dt|   |rt%dFdG |D }t%dHdG |D }t&'dId! |D } t%dJdG |D }!t%dKdG |D }"t%dLdG |D }#t%dMdG |D }$tdN tdO|dP dQdR tdS|dP dQdT| dQdU tdV|!dQdW |dkrVtdX|!|dY  dQdZ |"|# |$ dkr|td[ td\|"  td]|#  td^|$  td_ |D ]B}%d`}&|%da dbr|%da db }'dc|'ddd de|'dfd }&tdg|%dh  di|%dj  dk|%da dl  dm|%dn dodW|& 
 q|rtdp |D ]}%|%dqdr}(tds|( dt|%du  di|%dv   qtt(  td |S )wu  
    Process multiple videos with adaptive compute-aware settings.
    
    === v7.0 OPTIMIZATION: Batch Download Overlap ===
    Downloads video N+1 in background while processing video N.
    This hides ~60s download time for each video after the first.
    
    For 5 videos: saves ~4 × 60s = 240s (26% faster batch processing)
    
    Flow:
    1. Detect system resources (COMPUTE)
    2. Apply adaptive settings to config
    3. Validate all videos (optional)
    4. Load models once (hot inference)
    5. Process each video with PREFETCHED download for next video
    6. Print utilization summary
    
    Args:
        video_urls: List of YouTube URLs to process
        config: Pipeline configuration (auto-created if None)
        validate_videos: Whether to pre-validate videos (default: True)
        skip_on_validation_fail: If True, skip invalid videos; if False, raise error
    
    Returns:
        List of result dicts (includes 'error' key for failures)
    zF======================================================================u   🚀 BATCH PROCESSING: z videosz.   Model Priority: community-1 (fallback: 3.1)z   VAD Workers: z   Embedding Batch: z   Merge Threshold: z   Min Segment: zs (for 0.4s events)u2      ⚡ Batch Prefetch: ENABLED (v7.0 optimization)u   
🔍 PRE-VALIDATING VIDEOS...z   [r   /z] Validating: )validmessager%  r   r   N2   r   r   u          ✅ 'z' (zs)u          ❌ zVideo validation failed: z - z
   Validation complete: z videos validu   ❌ No valid videos to process!c                    s$   g | ]}d  | d  |dqS )Validation failed: r  )r   r  r]   )r   r  validation_resultsr]   rp   r   =  s    z!process_batch.<locals>.<listcomp>r  r  
validation)r   r  
error_typeprefetch)max_workersthread_name_prefixu(   ⚡ Starting prefetch for first video...
u   📹 VIDEO z	 (total: )u"   ⚡ Using prefetched download for i,  r{   u      ✅ Prefetch ready: u      ⚠️ Prefetch failed: z, will download in main threadu*   ⚡ Prefetching next video in background: )r,   r-   Tr"  u   ❌ Validation Error: u   ❌ Download Error: r/   u   ❌ Memory Error: memoryu   ❌ Processing Error: 
processingF)r9  zG
======================================================================u   📊 BATCH SUMMARYc                 S   s   g | ]}d |vr|qS r   r]   r   rr]   r]   rp   r         c                 S   s   g | ]}d |v r|qS r  r]   r  r]   r]   rp   r     r  u   ✅ Successful: u   ❌ Failed: c                 s   r   )r   Nr]   r  r]   r]   rp   r     r   z process_batch.<locals>.<genexpr>c                 s   s    | ]	}|d  d V  qdS )r   r   Nr]   r  r]   r]   rp   r     s    c                 S   s   g | ]}|d  d qS )r   r   r]   r  r]   r]   rp   r     r  c                 s   r   )r  Nr]   r  r]   r]   rp   r     r   c                 s   (    | ]}|d   di  ddV  qdS )r   r   r   r   Nr   r  r]   r]   rp   r        & c                 s   r  )r   r   r   r   Nr   r  r]   r]   rp   r     r  c                 s   s<    | ]}|d   di  d|d   di  ddV  qdS )r   r   r   r   r   Nr   r  r]   r]   rp   r     s    u   
📈 AGGREGATE:z   Total processed: r   r   z minz   Total usable: z min (z% avg)z   Total time: rv   z	   Rate: r  zs per hour of audiou   
🎵 MUSIC DETECTION:z   Clean segments: r   z   Heavy Music: u   
📋 PER-VIDEO RESULTS:r!  r   r   u    | 🎵 clean=r   z	, demucs=r   u      • r   z: r   z speakers, r   z
% usable, r  z.0fu   
❌ FAILED VIDEOS:r  unknownu      • [z] r  r   ))r   rj   r%  r.  r  r  rK  r  r=  r   r>  r   rk   r   r   r   r&  r'  r	   submitr  r  r   r  r  rV  r   rW  ra   r   MemoryErrorgccollectri   rA  rB  shutdownrD  r   r   summary))r  r+   r  r  
valid_urlsri  r  is_validr  r%  r   r   resultsrX  prefetch_executorprefetch_futureprefetch_urlr,   r-   rZ  r[  prefetch_errornext_urlr  rm   r  rA  r|  
successfulfailedr]  total_usable
avg_usabletotal_processtotal_cleantotal_demucstotal_heavyr  
music_infomdr  r]   r  rp   process_batch  s:  
 




$
 


	
*
$$	



 


 

(
r  )NN)NTT)H__doc__r'  rP  loggingrandompathlibr   typingr   r   r   r   r   r   r0  r	   r
   numpyr   
src.configr   src.computer   
src.modelsr   src.downloadr   r   r   r   src.vadr   r   rF  r   r   r   src.diarizationr   r   src.embeddingsr   src.clusteringr   r   	src.utilsr   r   src.audio_bufferr   r   src.chunk_reassignmentr    r!   r"   src.unified_embeddingsr#   r$   r%   r&   src.music_detectionr'   r(   r)   	getLoggerrj   ra   r  ri   r  boolr  r]   r]   r]   rp   <module>   st   8


     ,