Output:
Exit code: 0
Wall time: 0 seconds
Total output lines: 320
Output:
#!/usr/bin/env python3
"""
Model management - Hot inference with adaptive loading.

Key improvements:
- community-1 FIRST (better per pyannote docs), 3.1 as fallback
- Lazy loading with compute-aware batching
- Better GPU memory management

=== v7.0 OPTIMIZATION: cuDNN Benchmark + TF32 ===
Enables CUDA optimizations for faster inference:
- cudnn.benchmark=True: Auto-tune convolution algorithms
- TF32 precision: 3x faster matmul on Ampere+ GPUs
Expected gain: 5-10% on diarization (conv-heavy model)
"""

# === COMPATIBILITY FIX: torchaudio 2.9+ removed list_audio_backends() ===
# speechbrain calls this old API; we monkey-patch before any imports
import os
import torchaudio
if not hasattr(torchaudio, 'list_audio_backends'):
    torchaudio.list_audio_backends = lambda: ['soundfile', 'sox']
# === END COMPATIBILITY FIX ===

import time
import logging
import gc
import torch

# === COMPATIBILITY FIX: speechbrain expects hf_hub_download(use_auth_token=...) ===
# Newer huggingface_hub versions renamed this to `token`. Rather than pinning a
# fragile dependency set, we shim the old kwarg to the new one.
try:
    import inspect
    import huggingface_hub  # type: ignore

    if hasattr(huggingface_hub, "hf_hub_download"):
        _orig_hf_hub_download = huggingface_hub.hf_hub_download
        _hf_sig = inspect.signature(_orig_hf_hub_download)
        if "use_auth_token" not in _hf_sig.parameters:
            def _patched_hf_hub_download(*args, use_auth_token=None, token=None, **kwargs):
                if token is None and use_auth_token is not None:
                    token = use_auth_token
                return _orig_hf_hub_download(*args, token=token, **kwargs)

            huggingface_hub.hf_hub_download = _patched_hf_hub_download  # type: ignore
except Exception:
    # Best-effort shim; if anything goes wrong we fall back to stock behavior.
    pass

# === v7.0 OPTIMIZATION: Enable CUDA optimizations ===
# These flags enable hardware-specific optimizations:
# - cudnn.benchmark: Auto-select fastest conv algorithm for input sizes
# - TF32: Use TensorFloat-32 for faster matrix ops (Ampere+ GPUs)
# Must be set BEFORE model loading for full effect
if torch.cuda.is_available():
    torch.backends.cudnn.benchmark = True  # Auto-tune convolutions
    torch.backends.cuda.matmul.allow_tf32 = True  # Fast matmul
    torch.backends.cudnn.allow_tf32 = True  # Fast convolutions

# === CRITICAL: Patch torch.load BEFORE any other imports ===
_original_torch_load = torch.load
def _patched_torch_load(*args, **kwargs):
    kwargs['weights_only'] = False
    return _original_torch_load(*args, **kwargs)
torch.load = _patched_torch_load

# Patch PyTorch Lightning too
try:
    import pytorch_lightning.core.saving as pl_saving
    _orig_pl_load = pl_saving.pl_load
    def _patched_pl_load(path_or_url, map_location=None, **kwargs):
        return _original_torch_load(path_or_url, map_location=map_location, weights_only=False)
    pl_saving.pl_load = _patched_pl_load
except ImportError:
    pass

try:
    import lightning_fabric.utilities.cloud_io as cloud_io
    _orig_fabric_load = cloud_io._load
    def _patched_fabric_load(path_or_url, map_location=None):
        return _original_torch_load(path_or_url, map_location=map_location, weights_only=False)
    cloud_io._load = _patched_fabric_load
except (ImportError, AttributeError):
    pass

logger = logging.getLogger("FastPipelineV6.Models")

class ModelManager:
    """
    Singleton model manager - loads all models once, keeps them hot.

    Model priority (per user request):
    1. community-1 (better performance per pyannote docs)
    2. speaker-diarization-3.1 (fallback)

    Key optimization: Load once, inference many times.
    """
    _instance = None

    def __new__(cls):
        if cls._instance is None:
            cls._instance = super().__new__(cls)
            cls._instance._initialized = False
        return cls._instance

    def __init__(self):
        if self._initialized:
            return
        self._initialized = True
        self.device = None
        self.silero_vad = None
        self.silero_utils = None
        self.diarization_pipeline = None
        self.segmentation_model = None
        self.embedding_model = None
        self.panns_model = None  # v6.7: PANNs CNN14 for music detection
        self._loaded = False
        self._panns_loaded = False  # Separate flag for optional PANNs
        self._model_name = None  # Track which model loaded

    def get_device(self) -> torch.device:
        """Get CUDA device or fallback to CPU."""
        if self.device is None:
            self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
        return self.device

    def load_all(self, config):
        """
        Load all models once into GPU memory.

        Order: VAD -> Diarization (community-1 first) -> Segmentation -> Embedding
        """
        if self._loaded:
            logger.info(f"✅ Models alr…2779 chars truncated…unity-1
        # Fallback: speaker-diarization-3.1 if community-1 fails
        models_to_try = [
            ("pyannote/speaker-diarization-community-1", "community-1"),  # FIRST CHOICE
            ("pyannote/speaker-diarization-3.1", "speaker-diarization-3.1"),  # FALLBACK
        ]

        for model_name, short_name in models_to_try:
            try:
                logger.info(f"Trying {short_name}...")
                # pyannote.audio 4.x uses 'token' parameter
                self.diarization_pipeline = Pipeline.from_pretrained(
                    model_name,
                    token=config.hf_token
                )
                self._model_name = short_name
                logger.info(f"✅ Loaded {short_name} (preferred model)")
                break
            except Exception as e:
                logger.warning(f"Failed to load {short_name}: {e}")
                continue

        if self.diarization_pipeline is None:
            raise RuntimeError("Failed to load any diarization pipeline")

        self.diarization_pipeline.to(device)
        logger.info("✅ PyAnnote Diarization loaded on GPU")

        # 3. Segmentation model - DISABLED (not used in current pipeline)
        # NOTE: segmentation-3.0 was for frame-level OSD but we use community-1 diarization instead
        # The community-1 model handles both diarization AND overlap detection in one pass
        # Keeping this disabled to avoid noisy "gated model" warnings
        self.segmentation_model = None

        # 4. ECAPA-TDNN for embeddings
        logger.info("Loading ECAPA-TDNN...")
        from speechbrain.inference.speaker import EncoderClassifier
        self.embedding_model = EncoderClassifier.from_hparams(
            source="speechbrain/spkrec-ecapa-voxceleb",
            savedir="/ephemeral/models/ecapa",
            run_opts={"device": str(device)}
        )
        logger.info("✅ ECAPA-TDNN loaded")

        self._loaded = True
        elapsed = time.time() - start

        # Log GPU memory after
        if torch.cuda.is_available():
            allocated = torch.cuda.memory_allocated() / 1e9
            reserved = torch.cuda.memory_reserved() / 1e9
            logger.info(f"GPU Memory: {allocated:.2f}GB allocated, {reserved:.2f}GB reserved")

        logger.info(f"✅ All models loaded in {elapsed:.1f}s (STAYING HOT)")
        logger.info("=" * 70)

    def clear_cache(self, aggressive=False):
        """
        Clear GPU cache between operations.

        Args:
            aggressive: If True, also empty reserved memory and run gc
        """
        if torch.cuda.is_available():
            if aggressive:
                gc.collect()
                torch.cuda.empty_cache()
                torch.cuda.reset_peak_memory_stats()
            else:
                torch.cuda.empty_cache()

    def get_vram_free_gb(self) -> float:
        """Get current free VRAM in GB."""
        if torch.cuda.is_available():
            total = torch.cuda.get_device_properties(0).total_memory
            reserved = torch.cuda.memory_reserved(0)
            return (total - reserved) / (1024**3)
        return 0.0

    def load_panns(self, device: str = 'cuda'):
        """
        Load PANNs CNN14 model for music detection (lazy loading).

        === v7.4: Pre-baked model support ===
        If PANNS_MODEL_PATH env var is set, loads from that path (Docker pre-baked).
        Otherwise downloads from default location.

        Args:
            device: 'cuda' or 'cpu'

        Returns:
            Loaded AudioTagging model
        """
        if self._panns_loaded and self.panns_model is not None:
            logger.info("✅ PANNs CNN14 already loaded (hot inference)")
            return self.panns_model

        logger.info("Loading PANNs CNN14 for music detection...")
        start = time.time()

        try:
            from panns_inference import AudioTagging

            # Determine device
            if device == 'cuda' and not torch.cuda.is_available():
                device = 'cpu'
                logger.warning("CUDA not available, using CPU for PANNs")

            # === v7.4: Check for pre-baked model path (Docker) ===
            checkpoint_path = os.environ.get('PANNS_MODEL_PATH')
            if checkpoint_path and os.path.exists(checkpoint_path):
                logger.info(f"   Using pre-baked model: {checkpoint_path}")
                self.panns_model = AudioTagging(checkpoint_path=checkpoint_path, device=device)
            else:
                # Default: downloads model on first use
                self.panns_model = AudioTagging(device=device)

            self._panns_loaded = True
            elapsed = time.time() - start
            logger.info(f"✅ PANNs CNN14 loaded on {device} in {elapsed:.1f}s")

            return self.panns_model


Summary: auto


**Investigating container logs**

I’m looking at the container logs and wondering why certain lines are missing. Could it be that the logger is suppressed due to root access? Maybe the output is truncated, but that seems less likely. I could run the container again while increasing verbosity to capture more detail. It might also help to check the surrounding code in `src/models.py` around the `load_all` function, specifically from line 120 to 260, to understand dependencies better.


nl -ba src/models.py | sed -n '120,260p'


Output:
Exit code: 0
Wall time: 0 seconds
Output:
   120	        self._panns_loaded = False  # Separate flag for optional PANNs
   121	        self._model_name = None  # Track which model loaded
   122
   123	    def get_device(self) -> torch.device:
   124	        """Get CUDA device or fallback to CPU."""
   125	        if self.device is None:
   126	            self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
   127	        return self.device
   128
   129	    def load_all(self, config):
   130	        """
   131	        Load all models once into GPU memory.
   132
   133	        Order: VAD -> Diarization (community-1 first) -> Segmentation -> Embedding
   134	        """
   135	        if self._loaded:
   136	            logger.info(f"✅ Models already loaded: {self._model_name} (hot inference)")
   137	            return
   138
   139	        # Try to get token from config, fallback to environment variable
   140	        hf_token = getattr(config, "hf_token", None) or os.environ.get('HF_TOKEN', '')
   141	        if not hf_token:
   142	            raise RuntimeError(
   143	                "HF_TOKEN is required to load pyannote models.\n"
   144	                "- Set env var HF_TOKEN (do not hard-code it)\n"
   145	                "- Ensure you've accepted the model terms on Hugging Face:\n"
   146	                "  - pyannote/speaker-diarization-community-1\n"
   147	                "  - pyannote/segmentation-3.0 (if needed)\n"
   148	            )
   149	        # Update config with token for later use
   150	        config.hf_token = hf_token
   151
   152	        logger.info("=" * 70)
   153	        logger.info("🚀 LOADING ALL MODELS INTO GPU")
   154	        logger.info("=" * 70)
   155	        start = time.time()
   156
   157	        device = self.get_device()
   158	        logger.info(f"Device: {device}")
   159
   160	        # Log GPU memory before
   161	        if torch.cuda.is_available():
   162	            free = torch.cuda.get_device_properties(0).total_memory - torch.cuda.memory_reserved(0)
   163	            logger.info(f"GPU Memory before: {free / 1e9:.2f}GB free")
   164
   165	        # 1. Silero VAD (CPU - fast)
   166	        logger.debug("Loading Silero VAD...")
   167	        model, utils = torch.hub.load(
   168	            repo_or_dir='snakers4/silero-vad',
   169	            model='silero_vad',
   170	            force_reload=False,
   171	            onnx=False,
   172	            verbose=False  # Suppress "Using cache found" messages
   173	        )
   174	        self.silero_vad = model
   175	        self.silero_utils = utils
   176	        logger.debug("Silero VAD loaded")
   177
   178	        # 2. PyAnnote Diarization - COMMUNITY-1 FIRST (per user request)
   179	        from pyannote.audio import Pipeline
   180	        from pyannote.audio.pipelines import SpeakerDiarization
   181	        import inspect
   182
   183	        logger.info("Loading PyAnnote Diarization...")
   184
   185	        # Patch SpeakerDiarization to ignore unsupported params (like 'plda')
   186	        _original_sd_init = SpeakerDiarization.__init__
   187	        _valid_params = set(inspect.signature(_original_sd_init).parameters.keys())
   188
   189	        def _patched_sd_init(self, *args, **kwargs):
   190	            filtered_kwargs = {k: v for k, v in kwargs.items() if k in _valid_params}
   191	            removed = set(kwargs.keys()) - set(filtered_kwargs.keys())
   192	            if removed:
   193	                logger.debug(f"Filtered out unsupported params: {removed}")
   194	            return _original_sd_init(self, *args, **filtered_kwargs)
   195
   196	        SpeakerDiarization.__init__ = _patched_sd_init
   197
   198	        # === PRIORITY: community-1 FIRST (better performance per pyannote docs) ===
   199	        # Repo: https://huggingface.co/pyannote/speaker-diarization-community-1
   200	        # Fallback: speaker-diarization-3.1 if community-1 fails
   201	        models_to_try = [
   202	            ("pyannote/speaker-diarization-community-1", "community-1"),  # FIRST CHOICE
   203	            ("pyannote/speaker-diarization-3.1", "speaker-diarization-3.1"),  # FALLBACK
   204	        ]
   205
   206	        for model_name, short_name in models_to_try:
   207	            try:
   208	                logger.info(f"Trying {short_name}...")
   209	                # pyannote.audio 4.x uses 'token' parameter
   210	                self.diarization_pipeline = Pipeline.from_pretrained(
   211	                    model_name,
   212	                    token=config.hf_token
   213	                )
   214	                self._model_name = short_name
   215	                logger.info(f"✅ Loaded {short_name} (preferred model)")
   216	                break
   217	            except Exception as e:
   218	                logger.warning(f"Failed to load {short_name}: {e}")
   219	                continue
   220
   221	        if self.diarization_pipeline is None:
   222	            raise RuntimeError("Failed to load any diarization pipeline")
   223
   224	        self.diarization_pipeline.to(device)
   225	        logger.info("✅ PyAnnote Diarization loaded on GPU")
   226
   227	        # 3. Segmentation model - DISABLED (not used in current pipeline)
   228	        # NOTE: segmentation-3.0 was for frame-level OSD but we use community-1 diarization instead
   229	        # The community-1 model handles both diarization AND overlap detection in one pass
   230	        # Keeping this disabled to avoid noisy "gated model" warnings
   231	        self.segmentation_model = None
   232
   233	        # 4. ECAPA-TDNN for embeddings
   234	        logger.info("Loading ECAPA-TDNN...")
   235	        from speechbrain.inference.speaker import EncoderClassifier
   236	        self.embedding_model = EncoderClassifier.from_hparams(
   237	            source="speechbrain/spkrec-ecapa-voxceleb",
   238	            savedir="/ephemeral/models/ecapa",
   239	            run_opts={"device": str(device)}
   240	        )
   241	        logger.info("✅ ECAPA-TDNN loaded")
   242
   243	        self._loaded = True
   244	        elapsed = time.time() - start
   245
   246	        # Log GPU memory after
   247	        if torch.cuda.is_available():
   248	            allocated = torch.cuda.memory_allocated() / 1e9
   249	            reserved = torch.cuda.memory_reserved() / 1e9
   250	            logger.info(f"GPU Memory: {allocated:.2f}GB allocated, {reserved:.2f}GB reserved")
   251
   252	        logger.info(f"✅ All models loaded in {elapsed:.1f}s (STAYING HOT)")
   253	        logger.info("=" * 70)
   254
   255	    def clear_cache(self, aggressive=False):
   256	        """
   257	        Clear GPU cache between operations.
   258
   259	        Args:
   260	            aggressive: If True, also empty reserved memory and run gc


**Troubleshooting audio dependencies**

I'm setting the environment variable HF_TOKEN and passing it to the Pipeline, which seems like it should work. However, I might be missing dependencies for pyannote audio, like `pytorch_lightning` or `torchmetrics`, even though the requirements specify pyannote.audio>=3.1.0. In the host environment, everything works correctly, but the container setup might be different.

I wonder if the PyTorch version difference could be the issue, given that the container has torch 2.5.1 while the host uses torch 2.9.0. It might be fine.


**Diagnosing logging issues**

It looks like the logs aren't showing up in the E2E logs, which might mean the logger is only configured to display "Worker-0" messages. Perhaps the loggers from the pipeline are being suppressed by a custom logger.

To troubleshoot this, I can run a command directly in the container to load the pipeline and print errors. I’ll use `python -c` with a try/except block around `Pipeline.from_pretrained` to capture any exceptions and tracebacks.

I need to make sure to set the environment correctly, possibly using `--gpus all` for GPU support, but CPU will work too.


**Running diagnostic script**

I’m going to run a Python script that loads specific models from the pyannote audio library. I’ll start by importing the necessary libraries and getting the HF_TOKEN environment variable.

Then, I’ll loop through the model names, attempting to load each one with `Pipeline.from_pretrained`. If it loads successfully, I’ll print the type of the pipeline object. If there's an error, I’ll capture it and print the details along with the full traceback. This should provide valuable insight into what’s going wrong. The plan is to execute this script inside the container.


sudo docker run --rm --gpus all --env-file .env --entrypoint bash maya3-pipeline:local -lc 'set -euo pipefail; /workspace/maya3_data/.venv/bin/python - <<"PY"
import os, traceback
from pyannote.audio import Pipeline

token = os.environ.get("HF_TOKEN")
print("HF_TOKEN set", bool(token), "len", len(token) if token else 0)
models = [
    "pyannote/speaker-diarization-community-1",
    "pyannote/speaker-diarization-3.1",
]
for name in models:
    print("\n===", name, "===")
    try:
        p = Pipeline.from_pretrained(name, token=token)
        print("LOADED", type(p))
        break
    except Exception as e:
        print("ERROR:", repr(e))
        traceback.print_exc()
PY'


Output:
Exit code: 0
Wall time: 5.9 seconds
Output:
Traceback (most recent call last):
  File "<stdin>", line 13, in <module>
TypeError: Pipeline.from_pretrained() got an unexpected keyword argument 'token'
Traceback (most recent call last):
  File "<stdin>", line 13, in <module>
TypeError: Pipeline.from_pretrained() got an unexpected keyword argument 'token'
HF_TOKEN set True len 37

=== pyannote/speaker-diarization-community-1 ===
ERROR: TypeError("Pipeline.from_pretrained() got an unexpected keyword argument 'token'")

=== pyannote/speaker-diarization-3.1 ===
ERROR: TypeError("Pipeline.from_pretrained() got an unexpected keyword argument 'token'")


**Debugging pyannote.audio version**

I’ve realized that the error is due to the fact that `Pipeline.from_pretrained()` does not accept the keyword argument 'token'. It looks like I might be using pyannote.audio version 3.x, which requires `use_auth_token` instead. The comment mentioned that version 4.x uses 'token', but since my installed version is 3.x, it expects the `use_auth_token` argument. It seems the container might have a different version. I'll need to double-check the configuration and requirements to clarify the situation.


**Resolving pyannote.audio version issues**

It seems like the host environment may have a pyannote.audio version 4.x, which accepts 'token', while the container is using 3.x, causing some issues. To support both versions, I can modify `src/models.py` to call `Pipeline.from_pretrained()` with the appropriate parameter. I'll check the signature and adapt accordingly.

If 'token' is there, I'll use that. If not, I'll use `use_auth_token`. I want to implement this smartly to handle both cases, ensuring compatibility.


**Patching for token usage**

I need to check if there are other parts of the code using 'token', especially within `Pipeline.from_pretrained()`; this seems to be the main issue. Before I proceed with patching `src/models.py`, I should search for any occurrences of `token=config.hf_token` to ensure I’m not missing anything. Ensuring consistency everywhere will help avoid further errors, so it’s best to be thorough before making any updates.


rg -n "Pipeline\.from_pretrained\(|token=config\.hf_token|use_auth_token=config\.hf_token" src/models.py


Output:
Exit code: 0
Wall time: 0 seconds
Output:
210:                self.diarization_pipeline = Pipeline.from_pretrained(
212:                    token=config.hf_token


Input:
*** Begin Patch
*** Update File: src/models.py
@@
         for model_name, short_name in models_to_try:
             try:
                 logger.info(f"Trying {short_name}...")
-                # pyannote.audio 4.x uses 'token' parameter
-                self.diarization_pipeline = Pipeline.from_pretrained(
-                    model_name,
-                    token=config.hf_token
-                )
+                # pyannote.audio auth kwarg differs across versions:
+                # - 3.x: use_auth_token
+                # - 4.x+: token
+                auth_kwargs = {}
+                try:
+                    _sig = inspect.signature(Pipeline.from_pretrained)
+                    if "token" in _sig.parameters:
+                        auth_kwargs["token"] = config.hf_token
+                    elif "use_auth_token" in _sig.parameters:
+                        auth_kwargs["use_auth_token"] = config.hf_token
+                except Exception:
+                    auth_kwargs["use_auth_token"] = config.hf_token
+
+                self.diarization_pipeline = Pipeline.from_pretrained(model_name, **auth_kwargs)
                 self._model_name = short_name
                 logger.info(f"✅ Loaded {short_name} (preferred model)")
                 break
             except Exception as e:
                 logger.warning(f"Failed to load {short_name}: {e}")
*** End Patch


Output: {"output":"Success. Updated the following files:\nM src/models.py\n","metadata":{"exit_code":0,"duration_seconds":0.0}}


**Running tests and builds**

I need to rebuild the Docker image and then rerun the end-to-end tests. However, before diving into that, it seems wise to run `smoke_test.sh` separately to just check the builds and imports. That should go well, and it’s probably fine. So, I’ll execute the command `sudo bash scripts/docker/smoke_test.sh` without the E2E tests. The building should be quick because of the cache, especially since I'm only copying the changed files!


Output:
Exit code: 0
Wall time: 4.6 seconds
Output:
==> Building image: maya3-pipeline:local
#0 building with "default" instance using docker driver

#1 [internal] load build definition from Dockerfile
#1 transferring dockerfile: 2.44kB done
#1 DONE 0.0s

#2 [internal] load metadata for docker.io/nvidia/cuda:12.1.0-cudnn8-runtime-ubuntu22.04
#2 DONE 0.0s

#3 [internal] load .dockerignore
#3 transferring context: 255B done
#3 DONE 0.0s

#4 [1/8] FROM docker.io/nvidia/cuda:12.1.0-cudnn8-runtime-ubuntu22.04
#4 DONE 0.0s

#5 [internal] load build context
#5 transferring context: 16.14kB done
#5 DONE 0.0s

#6 [5/8] COPY requirements.txt .
#6 CACHED

#7 [2/8] RUN apt-get update && apt-get install -y --no-install-recommends     python3     python3-venv     python3-dev     python3-pip     python-is-python3     build-essential     ffmpeg     libsndfile1     libsox-fmt-all     sox     git     curl     wget     && rm -rf /var/lib/apt/lists/*
#7 CACHED

#8 [4/8] RUN mkdir -p /root/.cache/torch/hub     && mkdir -p /root/.cache/huggingface     && mkdir -p /ephemeral/models
#8 CACHED

#9 [3/8] WORKDIR /workspace/maya3_data
#9 CACHED

#10 [6/8] RUN python3 -m venv .venv &&     .venv/bin/pip install --upgrade pip &&     .venv/bin/pip install --no-cache-dir torch torchaudio --index-url https://download.pytorch.org/whl/cu121 &&     .venv/bin/pip install --no-cache-dir -r requirements.txt &&     .venv/bin/pip install --no-cache-dir supabase &&     .venv/bin/pip install --no-cache-dir "nvidia-nccl-cu12==2.28.9"
#10 CACHED

#11 [7/8] COPY . .
#11 DONE 0.1s

#12 [8/8] RUN mkdir -p ./output
#12 DONE 0.2s

#13 exporting to image
#13 exporting layers 0.0s done
#13 writing image sha256:a65cca12db5ccc343545c5ebec3bcf9a4e804bae574e99ff7d312169813f8d25 done
#13 naming to docker.io/library/maya3-pipeline:local done
#13 DONE 0.0s
==> Basic entrypoint check (help output)
==> GPU + imports smoke test
torch 2.5.1+cu121
cuda_available True
gpu_count 1
==> Supabase connectivity smoke test
{'PENDING': 172611, 'CLAIMED': 0, 'COMPLETED': 0, 'FAILED': 0, 'SKIPPED': 0}
==> R2 connectivity smoke test
source_bucket test
production_bucket 1-cleaned-data
==> Done


**Running E2E Tests**

I'm planning to start the E2E run with `R2_UPLOAD=0` to avoid uploading large files, but I'll save the tar locally and verify it. Then, maybe I can run another test with `R2_UPLOAD=1`, but I’m not sure if it's absolutely necessary. It’s been tested on the host, so testing R2 upload inside the container seems sensible. I’ll do one full E2E run with `R2_UPLOAD=1` while keeping in mind that the script has a safety check. Let’s go for it!


sudo OUTPUT_DIR=./output_docker_e2e2 E2E=1 R2_UPLOAD=1 GPU_DEVICE=0 bash scripts/docker/smoke_test.sh


Output:
Exit code: 1
Wall time: 49.4 seconds
Output:
==> Building image: maya3-pipeline:local
#0 building with "default" instance using docker driver

#1 [internal] load build definition from Dockerfile
#1 transferring dockerfile: 2.44kB done
#1 DONE 0.0s

#2 [internal] load metadata for docker.io/nvidia/cuda:12.1.0-cudnn8-runtime-ubuntu22.04
#2 DONE 0.0s

#3 [internal] load .dockerignore
#3 transferring context: 255B done
#3 DONE 0.0s

#4 [1/8] FROM docker.io/nvidia/cuda:12.1.0-cudnn8-runtime-ubuntu22.04
#4 DONE 0.0s

#5 [internal] load build context
#5 transferring context: 2.39kB done
#5 DONE 0.0s

#6 [4/8] RUN mkdir -p /root/.cache/torch/hub     && mkdir -p /root/.cache/huggingface     && mkdir -p /ephemeral/models
#6 CACHED

#7 [5/8] COPY requirements.txt .
#7 CACHED

#8 [6/8] RUN python3 -m venv .venv &&     .venv/bin/pip install --upgrade pip &&     .venv/bin/pip install --no-cache-dir torch torchaudio --index-url https://download.pytorch.org/whl/cu121 &&     .venv/bin/pip install --no-cache-dir -r requirements.txt &&     .venv/bin/pip install --no-cache-dir supabase &&     .venv/bin/pip install --no-cache-dir "nvidia-nccl-cu12==2.28.9"
#8 CACHED

#9 [3/8] WORKDIR /workspace/maya3_data
#9 CACHED

#10 [2/8] RUN apt-get update && apt-get install -y --no-install-recommends     python3     python3-venv     python3-dev     python3-pip     python-is-python3     build-essential     ffmpeg     libsndfile1     libsox-fmt-all     sox     git     curl     wget     && rm -rf /var/lib/apt/lists/*
#10 CACHED

#11 [7/8] COPY . .
#11 CACHED

#12 [8/8] RUN mkdir -p ./output
#12 CACHED

#13 exporting to image
#13 exporting layers done
#13 writing image sha256:a65cca12db5ccc343545c5ebec3bcf9a4e804bae574e99ff7d312169813f8d25 done
#13 naming to docker.io/library/maya3-pipeline:local done
#13 DONE 0.0s
==> Basic entrypoint check (help output)
==> GPU + imports smoke test
torch 2.5.1+cu121
cuda_available True
gpu_count 1
==> Supabase connectivity smoke test
{'PENDING': 172611, 'CLAIMED': 0, 'COMPLETED': 0, 'FAILED': 0, 'SKIPPED': 0}
==> R2 connectivity smoke test
source_bucket test
production_bucket 1-cleaned-data
==> E2E test: process 1 Supabase-queued video on GPU 0
00:22:02 | MassiveCompute | INFO     | Detected 1 GPUs: [0]
00:22:02 | MassiveCompute | INFO     | Supabase queue mode: videos will be claimed from database
00:22:02 | MassiveCompute | INFO     | HF_TOKEN captured (length=37)
00:22:02 | FastPipelineV6.Export | INFO     | BackgroundExportManager started with 2 workers
00:22:02 | MassiveCompute | INFO     | Background export enabled (2 workers)
00:22:02 | SupabaseClient | INFO     | SupabaseClient initialized: https://exlkkfpymkpqlxulurel.supabase.co
00:22:02 | MassiveCompute | INFO     | ======================================================================
00:22:02 | MassiveCompute | INFO     | SUPABASE QUEUE - Distributed Processing
00:22:02 | MassiveCompute | INFO     | ======================================================================
00:22:02 | MassiveCompute | INFO     | Pending videos: 172,611
00:22:02 | MassiveCompute | INFO     | GPUs: 1 ([0])
00:22:02 | MassiveCompute | INFO     | Workers/GPU: 1 (total: 1)
00:22:02 | MassiveCompute | INFO     | Output: ./output
00:22:02 | MassiveCompute | INFO     | Lease duration: 900s
00:22:02 | MassiveCompute | INFO     | Max videos/worker: 1
00:22:02 | MassiveCompute | INFO     | ======================================================================
00:22:02 | MassiveCompute | INFO     | Started Supabase worker 0 on GPU 0
00:22:02 | Worker-0     | INFO     | Starting worker gpu-0-1771a3784a80 on GPU 0
00:22:07 | Worker-0     | INFO     | Analytics initialized: 1771a3784a80_gpu0
Downloading: "https://github.com/snakers4/silero-vad/zipball/master" to /root/.cache/torch/hub/master.zip
[GPU0] PICK   dQw4w9WgXcQ
00:22:14 | Worker-0     | INFO     | Processing: dQw4w9WgXcQ - Test Video 1
00:22:15 | Worker-0     | ERROR    | Failed: dQw4w9WgXcQ - R2 source not found: s3://test/podcasts/dQw4w9WgXcQ.webm
Traceback (most recent call last):
  File "/workspace/maya3_data/massive_process.py", line 548, in gpu_worker_supabase
    result = process_single_video(video_url, config, supabase_metadata=full_video)
  File "/workspace/maya3_data/pipeline.py", line 219, in process_single_video
    audio_path, metadata = download_audio(video_url, config, supabase_metadata=supabase_metadata)
  File "/workspace/maya3_data/src/download.py", line 966, in download_audio
    remote_key, head = _resolve_r2_source_object(video_id, config)
  File "/workspace/maya3_data/src/download.py", line 513, in _resolve_r2_source_object
    raise DownloadError(
src.download.DownloadError: R2 source not found: s3://test/podcasts/dQw4w9WgXcQ.webm
00:22:15 | MassiveCompute | INFO     | [1] FAIL: dQw4w9WgXcQ (1.0s on GPU 0)
00:22:16 | Worker-0     | INFO     | Reached max videos limit (1)
00:22:16 | Worker-0     | INFO     | Shutting down. Processed 1 videos.
00:22:46 | MassiveCompute | INFO     | ======================================================================
00:22:46 | MassiveCompute | INFO     | SUPABASE QUEUE COMPLETE
00:22:46 | MassiveCompute | INFO     | ======================================================================
00:22:46 | MassiveCompute | INFO     | Processed: 1 (0 OK, 1 failed)
00:22:46 | MassiveCompute | INFO     | Wall time: 0.7 min
00:22:46 | MassiveCompute | INFO     | Audio hours: 0.0h
00:22:46 | MassiveCompute | INFO     | ----------------------------------------------------------------------
00:22:46 | MassiveCompute | INFO     | Queue Status:
00:22:46 | MassiveCompute | INFO     |    CLAIMED: 0
00:22:46 | MassiveCompute | INFO     |    COMPLETED: 0
00:22:46 | MassiveCompute | INFO     |    FAILED: 1
00:22:46 | MassiveCompute | INFO     |    PENDING: 172,610
00:22:46 | MassiveCompute | INFO     |    SKIPPED: 0
00:22:46 | MassiveCompute | INFO     | ======================================================================
00:22:46 | MassiveCompute | WARNING  |
Failed videos:
00:22:46 | MassiveCompute | WARNING  |    dQw4w9WgXcQ: R2 source not found: s3://test/podcasts/dQw4w9WgXcQ.webm
00:22:46 | MassiveCompute | INFO     | Waiting for background exports to complete...
00:22:46 | FastPipelineV6.Export | INFO     | Shutting down ExportManager (pending: 0)
00:22:46 | MassiveCompute | INFO     | Background exports: 0/0 successful
00:22:46 | MassiveCompute | INFO     |   Total segments: 0
00:22:46 | MassiveCompute | INFO     |   Export time: 0.0s
00:22:46 | MassiveCompute | INFO     |   Upload time: 0.0s


**Evaluating video processing**

I was hoping the models would load successfully, but there’s an issue with the R2 source for a specific video, and it got marked as FAILED in Supabase. This suggests there was a missing source object. While it processed one video, marking it as temporarily FAILED might not be the best approach during tests, although I wonder if the user might be concerned.

We might want to skip it instead of failing. For our test, let's use a known good video to ensure everything runs smoothly.


**Considering E2E script execution**

The E2E script uses the Supabase queue, and it's likely to pick a PENDING video that might have a missing object. That could lead to failure and mess with the queue. So, for our local E2E test, it seems wise to run on a known valid video ID or a designated test dataset. I noticed the previous agent used OBMrIJI_dn0.

I'll check the `massive_process.py` to see if it supports `--urls` for direct processing and confirm its help output. Let’s quickly get that information!


python3 massive_process.py --help | head -n 120


Output:
Exit code: 0
Wall time: 0.1 seconds
Output:
usage: massive_process.py [-h]
                          (--input INPUT | --benchmark | --urls URLS [URLS ...] | --supabase-queue)
                          [--gpus GPUS] [--workers-per-gpu WORKERS_PER_GPU]
                          [--num-videos NUM_VIDEOS] [--output OUTPUT]
                          [--merge-threshold MERGE_THRESHOLD]
                          [--min-segment MIN_SEGMENT] [--no-music-detection]
                          [--r2-upload] [--r2-upload-prefix R2_UPLOAD_PREFIX]
                          [--r2-upload-skip-if-exists] [--segment-audio]
                          [--no-segment-audio] [--background-export]
                          [--export-workers EXPORT_WORKERS]
                          [--max-videos MAX_VIDEOS]
                          [--lease-duration LEASE_DURATION]
                          [--poll-interval POLL_INTERVAL]

MASSIVE COMPUTE - 8x H100 Parallel Processing

options:
  -h, --help            show this help message and exit
  --input INPUT, -i INPUT
                        File with video URLs (one per line)
  --benchmark           Run benchmark with test videos
  --urls URLS [URLS ...]
                        Direct URL list
  --supabase-queue      Use Supabase as distributed video queue (claims
                        PENDING videos from DB)
  --gpus GPUS           Comma-separated GPU IDs (default: all available)
  --workers-per-gpu WORKERS_PER_GPU
                        Number of workers per GPU (default: 1, try 2-4 for
                        H100)
  --num-videos NUM_VIDEOS
                        Number of videos for benchmark mode (default: 8)
  --output OUTPUT, -o OUTPUT
                        Output directory (default: data/massive_output)
  --merge-threshold MERGE_THRESHOLD
                        Speaker merge threshold (default: 0.80)
  --min-segment MIN_SEGMENT
                        Minimum segment duration (default: 0.2s)
  --no-music-detection  Disable music detection
  --r2-upload           Upload per-video .tar into R2 bucket 1-cleaned-data
                        (no deletes)
  --r2-upload-prefix R2_UPLOAD_PREFIX
                        Optional prefix within 1-cleaned-data for uploads
                        (e.g. 'benchmarks/rtx4090/2026-01-03')
  --r2-upload-skip-if-exists
                        Skip upload if remote key already exists
  --segment-audio       Segment original audio into individual speaker clips
                        (default: True)
  --no-segment-audio    Disable audio segmentation
  --background-export   Run export (segmentation+tar+upload) in background
                        threads
  --export-workers EXPORT_WORKERS
                        Number of background export workers (default: 4)
  --max-videos MAX_VIDEOS
                        Max videos per worker in supabase-queue mode
                        (0=unlimited)
  --lease-duration LEASE_DURATION
                        Lease duration in seconds for claimed videos (default:
                        900)
  --poll-interval POLL_INTERVAL
                        Seconds to wait when queue empty (default: 10)

Examples:
  # Process videos from file
  python massive_process.py --input videos.txt

  # Benchmark with 16 test videos
  python massive_process.py --benchmark --num-videos 16

  # Use specific GPUs
  python massive_process.py --input videos.txt --gpus 0,1,2,3

  # Custom output directory
  python massive_process.py --input videos.txt --output /data/results


Summary: auto


rg -n "create_reporter\(" massive_process.py


Output:
Exit code: 0
Wall time: 0 seconds
Output:
482:        analytics = create_reporter(


sed -n '1,200p' scripts/find_valid_videos.py


Output:
Exit code: 0
Wall time: 0 seconds
Output:
#!/usr/bin/env python3
"""Find COMPLETED videos that have R2 source and local TAR files for validation."""

import os
import sys

sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

from dotenv import load_dotenv
load_dotenv()

from supabase import create_client
from src.r2_client import R2Client

sb = create_client(os.environ['URL'], os.environ['SUPABASE_ADMIN'])
r2 = R2Client(bucket_type='source')

# Get video IDs from R2
print("Getting videos from R2 source bucket...")
response = r2.client.list_objects_v2(Bucket=r2.bucket, Prefix='podcasts/', MaxKeys=200)
files = response.get('Contents', [])

r2_ids = set()
for f in files:
    key = f['Key']
    if key.endswith('.webm'):
        vid = key.replace('podcasts/', '').replace('.webm', '')
        r2_ids.add(vid)

print(f"Found {len(r2_ids)} videos in R2")

# Cross-reference with Supabase COMPLETED videos
result = sb.table('videos').select(
    'youtube_id, title, source_duration_min, usable_percentage, num_speakers, audio_duration_sec'
).eq('status', 'COMPLETED').order('usable_percentage', desc=True).limit(100).execute()
videos = result.data or []

# Find videos that are COMPLETED and in R2
good_videos = []
for v in videos:
    vid = v['youtube_id']
    if vid in r2_ids:
        good_videos.append(v)

print(f"\nCOMPLETED videos with R2 source: {len(good_videos)}")

# Check which have local TAR files
print("\n" + "=" * 80)
print("VIDEOS READY FOR VALIDATION (COMPLETED + R2 source + local TAR)")
print("=" * 80)

valid_count = 0
for v in good_videos[:20]:
    vid = v['youtube_id']
    tar_path = f"./output/{vid}.tar"
    if os.path.exists(tar_path):
        size_mb = os.path.getsize(tar_path) / (1024*1024)
        dur = v.get('source_duration_min', 0) or 0
        pct = v.get('usable_percentage', 0) or 0
        spk = v.get('num_speakers', 0) or 0
        title = (v.get('title') or '')[:45]
        print(f"  {vid} | {dur:5.1f}min | {pct:4.0f}% | {spk:2}spk | {size_mb:6.1f}MB | {title}")
        valid_count += 1

if valid_count == 0:
    print("  No videos found with all criteria met")
else:
    print(f"\nTotal: {valid_count} videos ready for manual validation")

# Also list what we have locally
print("\n" + "=" * 80)
print("ALL LOCAL TAR FILES (top 10 by size)")
print("=" * 80)

tar_files = []
output_dir = './output'
if os.path.exists(output_dir):
    for f in os.listdir(output_dir):
        if f.endswith('.tar'):
            path = os.path.join(output_dir, f)
            size = os.path.getsize(path)
            tar_files.append((f, size))

tar_files.sort(key=lambda x: -x[1])
for f, size in tar_files[:10]:
    vid = f.replace('.tar', '')
    print(f"  {f}: {size/(1024*1024):.1f}MB")


**Investigating video status**

I’m trying to figure out how to find a PENDING video in the R2 source bucket. The current script isn't helpful since it only looks for completed videos in R2 and local tar. Maybe I should check out `scripts/check_videos.py` to see if it can help with this search. I wonder what specifically I should focus on in that script to get the information I need. Let’s take a closer look!


Output:
Exit code: 0
Wall time: 0 seconds
Output:
#!/usr/bin/env python3
"""Check Supabase video columns and specific videos."""

import os
import sys
import json

sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

from dotenv import load_dotenv
load_dotenv()

from supabase import create_client

sb = create_client(os.environ['URL'], os.environ['SUPABASE_ADMIN'])

# Check a sample COMPLETED video
print('=' * 70)
print('SAMPLE COMPLETED VIDEO - ALL COLUMNS')
print('=' * 70)
result = sb.table('videos').select('*').eq('status', 'COMPLETED').limit(1).execute()
if result.data:
    v = result.data[0]
    for key, val in sorted(v.items()):
        if val is not None and val != '' and val != {} and val != []:
            val_str = str(val)[:100] if len(str(val)) > 100 else str(val)
            print(f'  {key:30}: {val_str}')
else:
    print('No COMPLETED videos found')

# Check specific videos requested by user
print()
print('=' * 70)
print('REQUESTED VIDEOS STATUS')
print('=' * 70)
requested = ['zfnG34iSpbU', 'IDzfs62WnD8', 'fHBR1j1kJ1I']
for vid in requested:
    result = sb.table('videos').select('*').eq('youtube_id', vid).execute()
    if result.data:
        v = result.data[0]
        print(f'\n{vid}:')
        print(f'  status: {v.get("status")}')
        print(f'  r2_tar_key: {v.get("r2_tar_key", "N/A")}')
        print(f'  usable_percentage: {v.get("usable_percentage", "N/A")}')
        print(f'  num_speakers: {v.get("num_speakers", "N/A")}')
        print(f'  total_segments: {v.get("total_segments", "N/A")}')
        print(f'  audio_duration_sec: {v.get("audio_duration_sec", "N/A")}')
    else:
        print(f'\n{vid}: NOT FOUND in Supabase')

# Check local files for these videos
print()
print('=' * 70)
print('LOCAL FILES FOR REQUESTED VIDEOS')
print('=' * 70)
import os
for vid in requested:
    tar_path = f'./output/{vid}.tar'
    dir_path = f'./output/{vid}'
    if os.path.exists(tar_path):
        size_mb = os.path.getsize(tar_path) / (1024*1024)
        print(f'{vid}: TAR exists ({size_mb:.1f}MB)')
    elif os.path.exists(dir_path):
        print(f'{vid}: Directory exists (no TAR)')
    else:
        print(f'{vid}: NOT FOUND locally')


sed -n '1,220p' scripts/process_specific_videos.py


Output:
Exit code: 0
Wall time: 0 seconds
Output:
#!/usr/bin/env python3
"""
Process specific videos and optionally upload to R2.
"""

import os
import sys
import time

sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

from dotenv import load_dotenv
load_dotenv()

# Video IDs to process
VIDEO_IDS = ['IDzfs62WnD8', 'fHBR1j1kJ1I']

from supabase import create_client
from src.r2_client import R2Client

sb = create_client(os.environ['URL'], os.environ['SUPABASE_ADMIN'])
r2_source = R2Client(bucket_type='source')

print("=" * 70)
print("CHECKING REQUESTED VIDEOS")
print("=" * 70)

# Check if these videos have source audio in R2
for vid in VIDEO_IDS:
    print(f"\n{vid}:")

    # Check Supabase
    result = sb.table('videos').select('status, source_duration_min, language, title').eq('youtube_id', vid).execute()
    if result.data:
        v = result.data[0]
        print(f"  Supabase: status={v.get('status')}, duration={v.get('source_duration_min')}min, lang={v.get('language')}")
        print(f"  Title: {v.get('title', 'N/A')[:60]}")
    else:
        print(f"  Supabase: NOT FOUND")

    # Check R2 source
    r2_key = f"podcasts/{vid}.webm"
    exists = r2_source.exists(r2_key)
    print(f"  R2 source: {'EXISTS' if exists else 'NOT FOUND'} ({r2_key})")

    # Check local output
    local_tar = f"./output/{vid}.tar"
    local_dir = f"./output/{vid}"
    if os.path.exists(local_tar):
        size_mb = os.path.getsize(local_tar) / (1024*1024)
        print(f"  Local: TAR exists ({size_mb:.1f}MB)")
    elif os.path.exists(local_dir):
        print(f"  Local: Directory exists")
    else:
        print(f"  Local: NOT FOUND")

print("\n" + "=" * 70)
print("To process these videos, run:")
print("  .venv/bin/python massive_process.py --supabase-queue --max-videos 2 --r2-upload --output ./output")
print()
print("Or set them to PENDING first, then run the full pipeline")
print("=" * 70)


**Considering testing modifications**

I’m thinking we might need to modify the smoke_test.sh script to select a known good video ID instead of claiming from the queue. The user wants to test the supabase queue mode, so we could still test by selecting a specific known good PENDING video first. Alternatively, we could run in supabase queue mode but filter for videos that exist in R2. Since we have full access, we could easily query R2 and supabase for a PENDING video that has an existing R2 source.


**Planning a Python command**

I’m thinking about creating a quick command in Python to query Supabase for 50 pending video IDs and then check R2 for their existence. I need to be careful not to create any unnecessary files; we can run this as a command. I'll look into the Supabase schema, particularly the 'videos' table, to see its columns like youtube_id and status, and possibly r2_source_key.

I’ll run a command to select the youtube_id for PENDING status videos but want to ensure I order them by created_at for efficiency. After retrieving them, I’ll check each ID in R2 using a client. It's all coming together nicely!


python3 - <<'PY'
import os
from dotenv import load_dotenv
load_dotenv()
from supabase import create_client
from src.r2_client import R2Client

sb = create_client(os.environ['URL'], os.environ['SUPABASE_ADMIN'])
r2 = R2Client(bucket_type='source')

# Pull a batch of pending videos
res = sb.table('videos').select('youtube_id,title').eq('status','PENDING').limit(200).execute()
videos = res.data or []
print('pending fetched', len(videos))

for v in videos:
    vid = v.get('youtube_id')
    if not vid:
        continue
    key = f'podcasts/{vid}.webm'
    try:
        if r2.exists(key):
            print('FOUND', vid, '|', (v.get('title') or '')[:80])
            break
    except Exception as e:
        print('ERR', vid, e)
        continue
else:
    print('No pending video with existing R2 source found in this sample')
PY


Output:
Exit code: 1
Wall time: 0.1 seconds
Output:
Traceback (most recent call last):
  File "<stdin>", line 3, in <module>
  File "/home/ubuntu/.local/lib/python3.10/site-packages/dotenv/main.py", line 346, in load_dotenv
    dotenv_path = find_dotenv()
  File "/home/ubuntu/.local/lib/python3.10/site-packages/dotenv/main.py", line 305, in find_dotenv
    assert frame.f_back is not None
AssertionError


**Investigating dotenv issues**

I’m looking into the `load_dotenv` error from python-dotenv, which seems to happen due to assertion failures in interactive contexts. I realize `find_dotenv` relies on the call stack, and this is a known issue when running with `python -c` or similar, possibly because `__file__` isn’t set. To fix this, I can specify the path directly with `load_dotenv(".env")` or set the environment. I’ll rerun the function using `os.path.join(os.getcwd(), ".env")` for clarity.


python3 - <<'PY'
import os
from dotenv import load_dotenv
load_dotenv(dotenv_path=os.path.join(os.getcwd(), '.env'))
from supabase import create_client
from src.r2_client import R2Client

sb = create_client(os.environ['URL'], os.environ['SUPABASE_ADMIN'])
r2 = R2Client(bucket_type='source')

res = sb.table('videos').select('youtube_id,title').eq('status','PENDING').limit(200).execute()
videos = res.data or []
print('pending fetched', len(videos))

for v in videos:
    vid = v.get('youtube_id')
    if not vid:
        continue
    key = f'podcasts/{vid}.webm'
    try:
        if r2.exists(key):
            print('FOUND', vid, '|', (v.get('title') or '')[:80])
            break
    except Exception as e:
        print('ERR', vid, e)
        continue
else:
    print('No pending video with existing R2 source found in this sample')
PY


Output:
Exit code: 124
Wall time: 10 seconds
Output:
command timed out after 10012 milliseconds


**Adjusting timeout settings**

It seems the command timed out after the default 10 seconds because I didn’t set the timeout. I’ll rerun the command with a longer timeout, so let’s set it to 120,000 milliseconds this time. This should give the process enough time to complete without timing out again. I just need to make sure to integrate this change properly before executing it again!


python3 - <<'PY'
import os
from dotenv import load_dotenv
load_dotenv(dotenv_path=os.path.join(os.getcwd(), '.env'))
from supabase import create_client
from src.r2_client import R2Client

sb = create_client(os.environ['URL'], os.environ['SUPABASE_ADMIN'])
r2 = R2Client(bucket_type='source')

res = sb.table('videos').select('youtube_id,title').eq('status','PENDING').limit(200).execute()
videos = res.data or []
print('pending fetched', len(videos))

found = None
for i, v in enumerate(videos, 1):
    vid = v.get('youtube_id')
    if not vid:
        continue
    key = f'podcasts/{vid}.webm'
    if r2.exists(key):
        found = (vid, v.get('title') or '')
        break
    if i % 25 == 0:
        print('checked', i)

if found:
    print('FOUND', found[0], '|', found[1][:80])
else:
    print('No pending video with existing R2 source found in this sample')
PY


Output:
Exit code: 0
Wall time: 52 seconds
Output:
pending fetched 200
checked 25
checked 50
checked 75
checked 100
checked 125
checked 150
checked 175
checked 200
No pending video with existing R2 source found in this sample


**Evaluating video sources**

I noticed that in the sample of 200 pending videos, none had an R2 source file, which seems odd considering there are 172k pending. Maybe the pending videos relate to a playlist but haven't been downloaded yet? It’s possible that the R2 source bucket "test" contains different files. In previous tests, they found a similar number pending. If R2 is missing files, it could lead to worker failures. But some have been processed before, so there might be existing files with different formats.


**Inspecting file resolution**

I’m considering that the expected key pattern might need some tweaking. My check only searched for podcasts/{id}.webm, but there could be files with other extensions like .flac. The resolve function might be set up to consider multiple formats. To clarify this, I need to inspect the `_resolve_r2_source_object` logic in `src/download.py`. I’ll focus around lines 450 to 560 to see what keys it attempts to resolve. Let's get a closer look at this!


nl -ba src/download.py | sed -n '430,580p'


Output:
Exit code: 0
Wall time: 0 seconds
Output:
   430
   431	    # Highest priority: explicit env/config.
   432	    env_prefixes = os.environ.get('R2_SOURCE_PREFIXES', '').strip()
   433	    if env_prefixes:
   434	        prefixes.extend([p.strip() for p in env_prefixes.split(',') if p.strip()])
   435
   436	    cfg_prefix = getattr(config, 'r2_source_prefix', '') or ''
   437	    if cfg_prefix:
   438	        prefixes.append(cfg_prefix)
   439
   440	    # Common defaults seen in earlier code/comments.
   441	    prefixes.extend(['', 'podcasts', 'podcasts/', 'videos', 'videos/'])
   442
   443	    # Normalize to "dir/" or "" and dedupe while preserving order.
   444	    normalized: List[str] = []
   445	    seen = set()
   446	    for p in prefixes:
   447	        p = p.strip()
   448	        if not p:
   449	            p_norm = ''
   450	        else:
   451	            p_norm = p if p.endswith('/') else (p + '/')
   452	        if p_norm not in seen:
   453	            seen.add(p_norm)
   454	            normalized.append(p_norm)
   455	    return normalized
   456
   457
   458	def _iter_r2_source_extensions(config) -> List[str]:
   459	    """Return candidate extensions to try for source objects (small list, no listing)."""
   460	    exts: List[str] = []
   461
   462	    env_exts = os.environ.get('R2_SOURCE_EXTENSIONS', '').strip()
   463	    if env_exts:
   464	        exts.extend([e.strip() for e in env_exts.split(',') if e.strip()])
   465
   466	    cfg_exts = getattr(config, 'r2_source_extensions', None)
   467	    if cfg_exts:
   468	        if isinstance(cfg_exts, str):
   469	            exts.extend([e.strip() for e in cfg_exts.split(',') if e.strip()])
   470	        elif isinstance(cfg_exts, (list, tuple)):
   471	            exts.extend([str(e).strip() for e in cfg_exts if str(e).strip()])
   472
   473	    # Reasonable defaults for YouTube-origin files.
   474	    exts.extend(['', '.webm', '.mp4', '.mkv', '.m4a', '.mp3', '.opus'])
   475
   476	    # Normalize: ensure extensions start with '.' (except empty).
   477	    normalized: List[str] = []
   478	    seen = set()
   479	    for e in exts:
   480	        e = e.strip()
   481	        if not e:
   482	            e_norm = ''
   483	        else:
   484	            e_norm = e if e.startswith('.') else ('.' + e)
   485	        if e_norm not in seen:
   486	            seen.add(e_norm)
   487	            normalized.append(e_norm)
   488	    return normalized
   489
   490
   491	def _resolve_r2_source_object(video_id: str, config) -> Tuple[str, Dict[str, Any]]:
   492	    """
   493	    Resolve a video_id to an object key in the R2 source bucket.
   494
   495	    Simplified: All source videos are at podcasts/{video_id}.webm in the 'test' bucket.
   496
   497	    Returns:
   498	        (remote_key, head_object_response)
   499	    """
   500	    client = _get_r2_source_client(config)
   501
   502	    # Simplified: all videos are at podcasts/{video_id}.webm
   503	    key = f"podcasts/{video_id}.webm"
   504
   505	    # Direct HEAD request (no prefix from client)
   506	    try:
   507	        head = client.client.head_object(Bucket=client.bucket, Key=key)
   508	        if head:
   509	            return key, head
   510	    except Exception:
   511	        pass
   512
   513	    raise DownloadError(
   514	        f"R2 source not found: s3://{client.bucket}/{key}"
   515	    )
   516
   517
   518	def is_r2_url(url: str) -> bool:
   519	    """Check if URL is an R2/S3 presigned URL."""
   520	    return 'r2.cloudflarestorage.com' in url or ('s3.' in url and 'amazonaws.com' in url)
   521
   522
   523	def extract_r2_path(presigned_url: str) -> str:
   524	    """
   525	    Extract clean R2 bucket/key path from a presigned URL.
   526
   527	    Example:
   528	        Input: https://xxx.r2.cloudflarestorage.com/test/podcasts/VIDEO.webm?X-Amz-...
   529	        Output: test/podcasts/VIDEO.webm
   530	    """
   531	    try:
   532	        from urllib.parse import urlparse
   533	        parsed = urlparse(presigned_url)
   534	        # Path is like /bucket/key - remove leading slash
   535	        path = parsed.path.lstrip('/')
   536	        return path
   537	    except:
   538	        return presigned_url  # Fallback to full URL
   539
   540
   541	def _download_from_r2(
   542	    r2_url: str,
   543	    video_id: str,
   544	    output_dir: Path,
   545	    config,
   546	    max_retries: int = 3,
   547	    supabase_metadata: Dict[str, Any] = None,
   548	) -> Tuple[str, Dict[str, Any]]:
   549	    """
   550	    Download audio directly from R2 presigned URL using ffmpeg.
   551
   552	    === v2: R2 DIRECT DOWNLOAD ===
   553	    For pre-cached videos in R2, we bypass yt-dlp entirely and use ffmpeg
   554	    to download and convert in a single pass.
   555
   556	    Creates:
   557	    - {video_id}_trimmed.wav (16kHz mono for processing)
   558	    - {video_id}_original.flac (original quality mono for export)
   559
   560	    Returns:
   561	        (audio_path, metadata): Path to processed audio and basic metadata
   562	    """
   563	    logger.info(f"⚡ R2 direct download: {video_id}")
   564	    start = time.time()
   565
   566	    # Output file paths
   567	    trimmed_file = output_dir / f"{video_id}_trimmed.wav"
   568	    original_file = output_dir / f"{video_id}_original.flac"
   569	    raw_file = output_dir / f"{video_id}_raw.wav"
   570
   571	    # Check cache
   572	    if trimmed_file.exists() and original_file.exists():
   573	        logger.info(f"✅ Using cached audio: {trimmed_file}")
   574	        try:
   575	            # Load a bit to get metadata (torchaudio.info deprecated in newer versions)
   576	            proc_waveform, proc_sr = torchaudio.load(str(trimmed_file))
   577	            orig_waveform, orig_sr = torchaudio.load(str(original_file))
   578	            return str(trimmed_file), {
   579	                'video_id': video_id,
   580	                'youtube_url': f'https://www.youtube.com/watch?v={video_id}',


python3 - <<'PY'
import os
from supabase import create_client
sb = create_client(os.environ['URL'], os.environ['SUPABASE_ADMIN'])
for vid in ['OBMrIJI_dn0','dQw4w9WgXcQ']:
    res = sb.table('videos').select('youtube_id,status,title').eq('youtube_id', vid).execute()
    print(vid, res.data)
PY


Output:
Exit code: 1
Wall time: 0.7 seconds
Output:
Traceback (most recent call last):
  File "<stdin>", line 3, in <module>
  File "/usr/lib/python3.10/os.py", line 680, in __getitem__
    raise KeyError(key) from None
KeyError: 'URL'


**Addressing environment variables**

I’m realizing that when running Python directly on the host, the environment variables from the .env file aren’t loaded automatically. In the past, we loaded them manually, but now I’ll use the `load_dotenv` function to specify the path. This way, I can ensure that all necessary variables are correctly imported into the environment when I'm running my script. It seems important to have everything set up correctly from the start!


python3 - <<'PY'
import os
from dotenv import load_dotenv
load_dotenv(dotenv_path=os.path.join(os.getcwd(), '.env'))
from supabase import create_client
sb = create_client(os.environ['URL'], os.environ['SUPABASE_ADMIN'])
for vid in ['OBMrIJI_dn0','dQw4w9WgXcQ']:
    res = sb.table('videos').select('youtube_id,status,title').eq('youtube_id', vid).execute()
    print(vid, res.data)
PY


Output:
Exit code: 0
Wall time: 1.1 seconds
Output:
OBMrIJI_dn0 [{'youtube_id': 'OBMrIJI_dn0', 'status': 'PENDING', 'title': 'India vs England... Sports Feed || Rj Fida || Facebook Live'}]
dQw4w9WgXcQ [{'youtube_id': 'dQw4w9WgXcQ', 'status': 'FAILED', 'title': 'Test Video 1'}]


Summary: auto


**Considering video processing options**

I want to list objects from the R2 bucket to get some IDs, but there might be many items. We can set `MaxKeys` to 500 to sample them. We know that OBMrIJI_dn0 exists and is pending, so our goal is to process that specific video by ensuring that the `claim_next_video` function returns that ID. The function likely chooses the first available video based on creation time, but I might be able to adjust its status or priority. However, the direct URL mode may not update Supabase, which adds complexity.


**Exploring video claiming methods**

I'm considering an alternative to using the `--supabase-queue` claiming logic. What if I simply call `SupabaseClient.claim_specific_video(video_id)`? I wonder if such a function exists for claiming by ID. To find out, I should search for `claim_next_video` in the `src/supabase_client.py` file. This could streamline the process and ensure I'm handling the video directly by its unique identifier. Let’s see what I can discover there!


rg -n "def claim_next_video|claim_" src/supabase_client.py | head -n 80


Output:
Exit code: 0
Wall time: 0 seconds
Output:
15:    video = client.claim_next_video("worker-1")
132:    def claim_next_video(
154:            result = self._rpc('claim_next_video', {
168:                logger.warning("claim_next_video RPC not found, using REST fallback")
169:                return self._claim_video_rest(worker_id, lease_duration_sec, language)
172:    def _claim_video_rest(
421:    def reclaim_expired_leases(self) -> int:


sed -n '120,250p' src/supabase_client.py


Output:
Exit code: 0
Wall time: 0 seconds
Output:
        body = json.dumps(params or {}).encode('utf-8')
        req = Request(url, data=body, headers=self.headers, method='POST')

        try:
            with urlopen(req, timeout=30) as resp:
                content = resp.read().decode('utf-8')
                return json.loads(content) if content else None
        except HTTPError as e:
            error_body = e.read().decode('utf-8')
            logger.error(f"RPC error {e.code}: {error_body}")
            raise

    def claim_next_video(
        self,
        worker_id: str,
        lease_duration_sec: int = 900,
        language: str = None
    ) -> Optional[Dict[str, Any]]:
        """
        Atomically claim the next PENDING video.

        Uses RPC function with FOR UPDATE SKIP LOCKED for safe concurrent access.
        Falls back to REST API claim if RPC not available.

        Args:
            worker_id: Unique identifier for this worker
            lease_duration_sec: How long to hold the claim (default 15 min)
            language: Optional language filter

        Returns:
            Video dict with youtube_id, youtube_url, title, etc. or None if queue empty
        """
        try:
            # Try RPC function first (atomic)
            result = self._rpc('claim_next_video', {
                'p_worker_id': worker_id,
                'p_lease_duration_sec': lease_duration_sec
            })

            if result and len(result) > 0:
                video = result[0]
                logger.info(f"Claimed video: {video.get('youtube_id')} by {worker_id}")
                return video
            return None

        except HTTPError as e:
            if e.code == 404:
                # RPC function not found, fall back to REST API
                logger.warning("claim_next_video RPC not found, using REST fallback")
                return self._claim_video_rest(worker_id, lease_duration_sec, language)
            raise

    def _claim_video_rest(
        self,
        worker_id: str,
        lease_duration_sec: int,
        language: str = None
    ) -> Optional[Dict[str, Any]]:
        """
        Fallback claim using REST API (less atomic but works without RPC).

        Note: This has a small race condition window between SELECT and UPDATE.
        For production, use the RPC function.
        """
        # Find oldest pending video
        params = {
            'status': f'eq.{STATUS_PENDING}',
            'order': 'created_at.asc',
            'limit': '1'
        }
        if language:
            params['language'] = f'eq.{language}'

        videos = self._request('GET', 'videos', params=params)

        if not videos:
            return None

        video = videos[0]
        video_id = video['youtube_id']

        # Attempt to claim (may race with other workers)
        now = datetime.now(timezone.utc).isoformat()
        lease_expires = datetime.fromtimestamp(
            time.time() + lease_duration_sec,
            tz=timezone.utc
        ).isoformat()

        update_data = {
            'status': STATUS_CLAIMED,
            'claimed_by': worker_id,
            'claimed_at': now,
            'lease_expires_at': lease_expires,
            'attempt_count': (video.get('attempt_count') or 0) + 1
        }

        # Only update if still PENDING (optimistic lock)
        params = {
            'youtube_id': f'eq.{video_id}',
            'status': f'eq.{STATUS_PENDING}'
        }

        result = self._request(
            'PATCH',
            'videos',
            data=update_data,
            params=params
        )

        if result and len(result) > 0:
            logger.info(f"Claimed video (REST): {video_id} by {worker_id}")
            return result[0]

        # Another worker got it first
        logger.debug(f"Lost race for video: {video_id}")
        return None

    def update_status(
        self,
        youtube_id: str,
        status: str,
        results: Dict[str, Any] = None
    ) -> bool:
        """
        Update video status and processing results.

        Args:
            youtube_id: YouTube video ID
            status: New status (COMPLETED, FAILED, SKIPPED)
            results: Processing results dict (num_speakers, usable_percentage, etc.)


**Considering video processing options**

It seems like the SupabaseClient can only claim the next video using the 'claim_next_video' RPC, not a specific one. I could implement a queuing system, but that would require more changes. A possible workaround is setting the created_at timestamp to make sure the correct video is picked. Alternatively, if we run the pipeline directly on OBMrIJI_dn0 using the direct URL mode, we could test the entire processing flow, including analytics and model loading, which sounds practical.


**Testing Supabase analytics**

I realize that while I can't directly test the Supabase claim and analytics together, we've confirmed that the analytics initialization is functioning correctly, as evidenced by the status update for dQw4w9WgXcQ. To monitor analytics, I might run `scripts/test_analytics.py` inside the container to generate heartbeats and events. We can also test the pipeline in direct URL mode and run the analytics script separately. Maybe creating a "test video" record with a known YouTube ID in Supabase could help ensure that it gets picked in the queue.


**Considering timestamp adjustments**

I’m thinking about whether I can insert a test row with an old created_at timestamp. That doesn't seem easy. However, I could set the created_at date to an artificially early time, which might be allowed. An alternative and potentially simpler solution is to update the created_at timestamp of OBMrIJI_dn0 to an older one, making it the first one claimed by claim_next_video. This could be a safe move if I revert the changes afterward.


python3 - <<'PY'
import os
from dotenv import load_dotenv
load_dotenv(dotenv_path=os.path.join(os.getcwd(), '.env'))
from supabase import create_client
sb = create_client(os.environ['URL'], os.environ['SUPABASE_ADMIN'])
res = sb.table('videos').select('youtube_id,created_at,status').eq('youtube_id','OBMrIJI_dn0').execute()
print(res.data)
PY


Output:
Exit code: 0
Wall time: 1.3 seconds
Output:
[{'youtube_id': 'OBMrIJI_dn0', 'created_at': '2026-01-06T13:18:23.347988+00:00', 'status': 'PENDING'}]


**Evaluating data integrity risks**

I'm considering if setting the creation date to 1970 might allow a claim to be picked. The RPC function might order by creation date, but the fallback seems to be in ascending order. Changing the created_at could be risky and affect data integrity. Alternatively, I could run a claim function until I find a valid one, though this might lead to some failures. I’m also thinking about adjusting the source object handling to look beyond specific prefixes. It looks like there could be objects missing in different prefixes.


**Examining video existence issues**

I'm looking into whether our existence check for the video files in the R2 connection is accurate. Initially, it showed that the source and production buckets were accessible, but it might not have verified the actual existence of the claimed videos. The source bucket might not have all pending videos, suggesting the pipeline could fall back to using yt-dlp for downloads from YouTube. The current code seems focused only on R2. I noticed some inconsistencies in the sample data and prefixes, which might indicate a configuration error.


**Investigating the R2 connection**

The summary mentioned that the R2 connection was successful, and the source and production buckets were accessible, but I’m not sure if it was the correct bucket. If the source bucket is indeed "test" and doesn't contain everything, that could explain some issues. There are 172k pending items in the queue, but my check of 200 pending revealed none in R2. This could be due to our r2.exists function being slow. I need to look into the .env variables to ensure the correct bucket is being referenced.


**Reassessing file resolution logic**

I'm wondering if the source bucket might have objects stored not under the expected podcasts/ directory. The function _resolve_r2_source_object is currently simplified to just look for 'podcasts/{id}.webm'. There are earlier helper functions for prefixes and extensions that could help locate files but aren't being used. My previous tests showed some files exist, yet if others have different extensions, we might be missing them. This could lead to false negatives, suggesting we should revisit the function to include more flexible file resolution rather than a fixed key.


**Improving pipeline robustness**

I’m thinking about how to make the pipeline more robust to avoid marking too many files as failed, which is crucial for deployment. I need to check the comments for _resolve_r2_source_object. It suggests that all source videos should be at 'podcasts/{video_id}.webm' in the test bucket, which might have been simplified. We should confirm with the user about the actual structure of the R2 source. If there are many pending items, it’s possible only some have files, meaning others may still need to be downloaded from YouTube. I’ll need to review the download_audio function for any yt-dlp fallback.


nl -ba src/download.py | sed -n '880,1045p'


Output:
Exit code: 0
Wall time: 0 seconds
Output:
   880	        'format': 'bestvideo[height<=720]+bestaudio/best[height<=720]',
   881	        'outtmpl': str(output_dir / f"{video_id}.%(ext)s"),
   882	        'merge_output_format': 'mp4',
   883	        'quiet': True,
   884	        'no_warnings': True,
   885	        'retries': 3,
   886	        'fragment_retries': 3,
   887	    }
   888
   889	    # Add proxy if configured
   890	    proxy = get_ytdlp_proxy()
   891	    if proxy:
   892	        ydl_opts['proxy'] = proxy
   893
   894	    for attempt in range(max_retries):
   895	        try:
   896	            with yt_dlp.YoutubeDL(ydl_opts) as ydl:
   897	                ydl.extract_info(url, download=True)
   898	            if output_file.exists():
   899	                logger.info(f"✅ Video download: {time.time()-start:.1f}s | Size: {output_file.stat().st_size/1e6:.1f}MB")
   900	                return output_file
   901	        except Exception as e:
   902	            logger.warning(f"Video download attempt {attempt+1} failed: {e}")
   903	            if attempt < max_retries - 1:
   904	                time.sleep(2 ** attempt)
   905
   906	    logger.error(f"❌ Video download failed after {max_retries} attempts")
   907	    return None
   908
   909
   910	def download_audio(
   911	    video_url: str,
   912	    config,
   913	    validate: bool = True,
   914	    max_retries: int = 3,
   915	    supabase_metadata: Dict[str, Any] = None,
   916	) -> Tuple[str, Dict[str, Any]]:
   917	    """
   918	    Download audio from YouTube URL or R2 presigned URL and prepare for processing.
   919
   920	    === v7.0 OPTIMIZATION: Download-Time Resample ===
   921	    Uses ffmpeg directly for dual-output in single pass:
   922	    - 16kHz mono WAV for processing pipeline
   923	    - Original quality mono WAV for high-quality export
   924
   925	    Previous: yt-dlp → temp.wav → torchaudio load → Python resample → save both
   926	    Now:      yt-dlp → single ffmpeg → dual output (NO Python resampling!)
   927
   928	    === v2: R2 URL SUPPORT ===
   929	    When video_url is an R2 presigned URL, we use ffmpeg directly instead of yt-dlp.
   930
   931	    === v3: Supabase Metadata ===
   932	    When supabase_metadata is provided (from distributed queue), use it for:
   933	    - Title, chapters, duration (no yt-dlp metadata fetch needed)
   934	    - Chapter-based intro/outro skip
   935
   936	    Args:
   937	        video_url: YouTube URL or R2 presigned URL
   938	        config: Pipeline configuration
   939	        validate: Whether to validate video before download (default: True)
   940	        max_retries: Number of download retries (default: 3)
   941	        supabase_metadata: Pre-fetched metadata from Supabase (chapters, title, etc.)
   942
   943	    Returns:
   944	        (audio_path, metadata): Path to processed audio and video metadata
   945
   946	    Raises:
   947	        VideoValidationError: If video fails validation
   948	        DownloadError: If download fails after retries
   949	    """
   950	    logger.info(f"📥 Downloading: {video_url[:80]}...")
   951	    start = time.time()
   952
   953	    # Extract video ID
   954	    video_id = extract_video_id(video_url)
   955
   956	    output_dir = Path(config.output_dir) / video_id
   957	    output_dir.mkdir(parents=True, exist_ok=True)
   958
   959	    # === R2 SOURCE MODE (video_id -> R2 object) ===
   960	    # For throughput benchmarking we want to avoid YouTube rate limits entirely.
   961	    # When enabled, we resolve the video_id to an object in the R2 *source* bucket
   962	    # (bucket_type='source' -> bucket name "test") using HEAD requests only.
   963	    input_source = getattr(config, 'input_source', 'youtube')
   964	    if input_source == 'r2' and not is_r2_url(video_url):
   965	        client = _get_r2_source_client(config)
   966	        remote_key, head = _resolve_r2_source_object(video_id, config)
   967
   968	        # Presigned URL lets ffmpeg stream directly (no local video download).
   969	        ttl = int(getattr(config, 'r2_presign_ttl_sec', 3600))
   970	        presigned_url = client.generate_presigned_get_url(remote_key, expires_in_sec=ttl)
   971
   972	        logger.info(
   973	            f"📦 R2 source resolved: s3://{client.bucket}/{remote_key} "
   974	            f"({head.get('ContentLength', 0) / (1024**2):.1f} MB)"
   975	        )
   976
   977	        audio_path, metadata = _download_from_r2(
   978	            r2_url=presigned_url,
   979	            video_id=video_id,
   980	            output_dir=output_dir,
   981	            config=config,
   982	            max_retries=max_retries,
   983	            supabase_metadata=supabase_metadata,
   984	        )
   985
   986	        elapsed = time.time() - start
   987	        metadata['download_wall_time_sec'] = elapsed
   988	        metadata['r2_source_bucket'] = client.bucket
   989	        metadata['r2_source_key'] = remote_key
   990	        metadata['r2_source_size_bytes'] = head.get('ContentLength')
   991
   992	        logger.info(f"✅ Download (R2): {elapsed:.1f}s | Duration: {metadata.get('processed_duration', 0):.0f}s")
   993	        return audio_path, metadata
   994
   995	    # === v2: R2 URL HANDLING ===
   996	    # R2 presigned URLs use ffmpeg directly (not yt-dlp)
   997	    if is_r2_url(video_url):
   998	        logger.info(f"📦 R2 source detected, using ffmpeg direct download")
   999	        audio_path, metadata = _download_from_r2(
  1000	            r2_url=video_url,
  1001	            video_id=video_id,
  1002	            output_dir=output_dir,
  1003	            config=config,
  1004	            max_retries=max_retries,
  1005	            supabase_metadata=supabase_metadata,
  1006	        )
  1007	        elapsed = time.time() - start
  1008	        metadata['download_wall_time_sec'] = elapsed
  1009	        logger.info(f"✅ Download (R2 URL): {elapsed:.1f}s | Duration: {metadata.get('processed_duration', 0):.0f}s")
  1010	        return audio_path, metadata
  1011
  1012	    # === Standard YouTube path ===
  1013	    info = {}
  1014	    if validate:
  1015	        is_valid, message, info = validate_video(video_url)
  1016	        if not is_valid:
  1017	            raise VideoValidationError(f"Video validation failed: {message}")
  1018
  1019	    # Fetch metadata if not already fetched during validation
  1020	    if not info:
  1021	        logger.info("📋 Fetching video metadata...")
  1022	        ydl_info_opts = {
  1023	            'quiet': True,
  1024	            'no_warnings': True,
  1025	            'extract_flat': False,
  1026	        }
  1027
  1028	        try:
  1029	            with yt_dlp.YoutubeDL(ydl_info_opts) as ydl:
  1030	                info = ydl.extract_info(video_url, download=False)
  1031	        except Exception as e:
  1032	            logger.warning(f"Could not fetch metadata: {e}")
  1033	            info = {'title': 'Unknown', 'chapters': []}
  1034
  1035	    # Output file paths
  1036	    # v7.2: Original audio saved as FLAC (lossless, ~50% smaller than WAV)
  1037	    trimmed_file = output_dir / f"{video_id}_trimmed.wav"
  1038	    original_file = output_dir / f"{video_id}_original.flac"
  1039
  1040	    # Check cache - both files must exist for full cache hit
  1041	    preserve_original = getattr(config, 'preserve_original_audio', False)
  1042
  1043	    # Also check for legacy .wav original files (backward compatibility)
  1044	    legacy_original_file = output_dir / f"{video_id}_original.wav"
  1045


**Planning E2E test approach**

For the E2E test, I can select a video that exists in R2 (OBMrIJI_dn0) to ensure it succeeds. I'll need to run a Supabase worker, configuring it to pick OBMrIJI_dn0. One strategy is to set other videos like dQw4w9WgXcQ to SKIPPED or FAILED since they’ve already failed. If the claim_next_video process orders them by created_at ascending, I could try adjusting the created_at date of OBMrIJI_dn0 to be much earlier, positioning it first. However, I need to be cautious about updating this in Supabase.


**Exploring video processing options**

I’m considering setting the `input_source` to 'youtube' in the configuration through environment variables, allowing downloads from YouTube in case the file is missing. This would enable processing of any pending video even if R2 is unavailable, although it might slow down the process. However, the user wants to avoid differences in the environment. For deployment, focusing on R2 seems preferable. I could also adjust the _resolve_r2_source_object function to search for different keys to minimize missing files. I’ll check the Config's default `input_source`.


Output:
Exit code: 0
Wall time: 0 seconds
Total output lines: 220
Output:
#!/usr/bin/env python3
"""
Configuration for Fast Pipeline v6.0 - Adaptive Compute-Aware

Key improvements:
- Auto-detects system resources (nproc, vCPUs, GPU vRAM)
- Adapts worker counts and batch sizes to available compute
- Optimized for micro-level speaker detection (0.4s events)
"""

import os
from dataclasses import dataclass, field
from typing import Optional

@dataclass
class Config:
    """
    Pipeline configuration with adaptive compute settings.

    Flow (per instructions.md):
    1. Download (max parallel workers)
    2. Quick VAD (parallel, get speech outline)
    3. Chunk at silence boundaries (VAD-aware)
    4. OSD (mark overlaps as unusable FIRST)
    5. Frame-level segmentation (17ms resolution for micro-changes)
    6. Embeddings + Conservative clustering
    7. Output metadata JSON only
    """

    # === AUTHENTICATION ===
    # NOTE: Must be provided via environment variable HF_TOKEN
    # Never hard-code tokens in repo (HF gated model access + security).
    hf_token: str = os.environ.get('HF_TOKEN', '')

    # === INPUT SOURCE (YouTube vs R2) ===
    # Default behavior remains YouTube URLs. For benchmark/testing you can pass
    # bare 11-char video IDs and set input_source='r2' to stream originals from
    # the R2 source bucket (bucket name: "test") without YouTube rate limits.
    input_source: str = "youtube"  # 'youtube' | 'r2'

    # === R2 SOURCE (input) ===
    # When input_source='r2', download.py will:
    # - Resolve video_id -> object key via HEAD (NO bucket listing)
    # - Generate a presigned GET URL
    # - ffmpeg stream-extract audio
    #
    # Prefix/extension knobs are optional; env overrides exist too:
    # - R2_SOURCE_PREFIX / R2_SOURCE_PREFIXES
    # - R2_SOURCE_EXTENSIONS
    r2_source_prefix: str = ""          # e.g. "podcasts/" (optional)
    r2_source_extensions: str = ""      # e.g. ".webm,.mp4" (optional)
    r2_presign_ttl_sec: int = 3600      # Presigned URL TTL (seconds)

    # Avoid any YouTube metadata calls during R2 runs (rate-limit safe).
    # When False, R2 downloads fall back to duration-based dynamic intro skip.
    fetch_youtube_metadata_for_r2: bool = True

    # === R2 OUTPUT (upload processed artifacts) ===
    # When enabled, pipeline will package each video's output directory into a .tar
    # and upload it to the R2 production bucket (1-cleaned-data) by default.
    r2_upload_enabled: bool = False
    r2_upload_bucket_type: str = "production"  # maps to 1-cleaned-data in src/r2_client.py
    r2_upload_prefix: str = ""                 # e.g. "benchmarks/rtx4090/2026-01-03T16-00Z"
    r2_upload_skip_if_exists: bool = False     # avoid overwriting if True
    r2_upload_delete_local_tar: bool = True    # free local disk after upload

    # === AUDIO SEGMENTATION ===
    # segment_audio: Cut original FLAC into individual speaker segment files
    # Uses parallel ffmpeg (8 workers) - typically 2-3s for 500 segments
    segment_audio: bool = True

    # === BACKGROUND EXPORT ===
    # NOTE: Background export is experimental and not recommended for multiprocessing.
    # In GPU worker subprocesses, background threads die when the process exits.
    # The sync export (5-6s for 60min video) runs AFTER GPU work completes,
    # and overlaps with the NEXT video's download prefetch, so overhead is minimal.
    background_export: bool = False  # Keep False for stability
    background_export_workers: int = 4  # Number of parallel export workers (unused currently)

    # === AUDIO PROCESSING ===
    sample_rate: int = 16000
    intro_skip_seconds: float = 0.0
    outro_skip_seconds: float = 0.0

    # === INTRO SKIP (v6.8 - Dynamic) ===
    # auto_intro_skip: Automatically skip intro based on video duration
    # When True and no chapter-based intro detected:
    #   - Video ≤ 30 min: skip 180s (3 min)
    #   - Video > 30 min: skip 300s (5 min)
    # Set to False to disable automatic intro skipping
    # Manual intro_skip_seconds > 0 always takes priority
    auto_intro_skip: bool = True

    # === HIGH QUALITY AUDIO PRESERVATION (v6.8+) ===
    # preserve_original_audio: Download at highest quality, process at 16kHz
    # When True:
    #   - Downloads original audio format (bestaudio - typically 48kHz)
    #   - Creates 16kHz copy for processing (VAD, diarization, embeddings)
    #   - After processing, deletes 16kHz and keeps only original
    # Final export cuts from original high-quality audio using timestamps
    preserve_original_audio: bool = True
    original_audio_sample_rate: int = 0  # 0 = keep original (highest), or specify (e.g. 48000)

    # === PARALLELISM (Auto-detected, these are fallbacks) ===
    # Actual values computed by COMPUTE.get_optimal_config()
    vad_workers: int = 32          # Will be auto-tuned
    max_workers: int = 32          # Will be auto-tuned
    download_workers: int = 4      # Parallel downloads for batch
    chunk_workers: int = 4         # GPU diarization (VRAM l…985 chars truncated…Window duration (seconds)
    min_segment_duration: float = 0.2   # Min segment (catch 0.4s events)

    # === OVERLAP DETECTION ===
    detect_overlap: bool = True
    overlap_threshold: float = 0.35     # Probability for active speaker
    overlap_min_duration: float = 0.1  # Min overlap to mark
    overlap_padding_ms: int = 100      # Pad overlap boundaries (±100ms)

    # === OVERLAP DENSITY DETECTION (v6.9) ===
    # Mark short segments sandwiched between overlaps as unusable
    # These are "islands" of supposedly clean audio in high-overlap regions
    overlap_density_filter: bool = True
    overlap_density_max_gap: float = 1.0      # Max gap to adjacent overlap (seconds)
    overlap_density_max_duration: float = 3.0  # Max segment duration to mark as unusable

    # === QUALITY FILTERING (v6.1) ===
    filter_by_quality: bool = True     # Enable SNR/quality filtering
    min_snr_db: float = 15.0           # Minimum SNR for TTS usability
    min_quality_score: float = 0.3     # Minimum quality score (0-1)

    # === SEGMENT DURATION FILTERING (v6.1) ===
    # Drop speaker segments shorter than this from the FINAL output.
    #
    # IMPORTANT: we do adjacent same-speaker merging before applying this filter,
    # so short fragments (e.g. 0.4-0.9s) can be absorbed into longer continuous
    # turns when safe to do so.
    min_tts_duration: float = 1.0      # Minimum segment duration for TTS

    # === DIARIZATION ===
    min_speakers: int = 1
    max_speakers: int = 10

    # === EMBEDDINGS (OOM-protected) ===
    embedding_batch_size: int = 8      # Will be auto-tuned
    max_embedding_length: int = 16000 * 10  # Max 10s per segment

    # === CLUSTERING (Conservative - no poison merges) ===
    # Per instructions: 0.75-0.80 to prevent merging different speakers
    cluster_merge_threshold: float = 0.80
    min_segments_for_merge: int = 2

    # === CHUNK REASSIGNMENT (v7.7 - With Validation Gates) ===
    # Detects within-segment speaker changes and surgically splits/reassigns
    #
    # === v7.7 FIX: VALIDATION GATES prevent false positive splits ===
    # Previous issue: 66x speaker fragmentation (6 speakers → 400+)
    # Root cause: Fixed thresholds + create_new_on_ambiguous triggered on natural variation
    #
    # NEW GATES (see unified_embeddings.py):
    # 1. Persistence gate: Require ≥N consecutive low-sim chunks
    # 2. Relative valley gate: Per-segment adaptive threshold (outlier detection)
    # 3. Same-speaker centroid gate: Reject if both sides match original speaker
    # 4. No-new-speaker-unless-proven: Keep original speaker on ambiguous (key fix!)
    enable_chunk_reassignment: bool = True  # RE-ENABLED with validation gates
    chunk_reassignment_threshold: float = 0.30  # Normal threshold (per-segment adaptive overrides)
    chunk_reassignment_severe: float = 0.15  # Severe circuit-breaker (only extreme changes)
    chunk_reassignment_min_speech: float = 0.6  # Min speech ratio for eligibility
    chunk_reassignment_min_portion: float = 1.0  # LOWERED: 1.0s to catch "yeah/ok" (was 2.0s)

    # === v7.8: TUNABLE GATE PARAMETERS for catching small interruptions ===
    # Lower values = more aggressive splitting (catches more but risks false positives)
    chunk_reassignment_min_analysis: float = 2.0  # Min segment duration to analyze (was hardcoded 3.0)
    chunk_reassignment_persistence: int = 1  # Min consecutive low-sim chunks (was hardcoded 2)
    chunk_reassignment_centroid_threshold: float = 0.55  # Match threshold for centroid gate (was hardcoded 0.60)

    # === MUSIC DETECTION (v7.4 - Hybrid INA+PANNs) ===
    # Two-stage detection: Fast INA screening + PANNs verification
    # Stage 1: inaSpeechSegmenter (fast CNN, ~10x realtime)
    # Stage 2: PANNs CNN14 (only on suspicious segments)
    enable_music_detection: bool = True  # Enable music detection
    use_hybrid_detection: bool = False  # DISABLED: Run PANNs on EVERYTHING for strict music detection
    music_chunk_duration: float = 1.5  # Same as embedding chunks
    music_batch_size: int = 128  # Batch size for PANNs inference (v7.5: optimized for A100)

    # === STRICT DETECTION THRESHOLDS FOR TTS QUALITY (v7.1) ===
    # Zero tolerance: ANY contamination → needs_demucs or unusable

    # Per-chunk thresholds (RESTORED to working values)
    music_prob_threshold: float = 0.20  # Chunk "has music" if prob > 20%
    noise_prob_threshold: float = 0.25  # Chunk "has noise" if prob > 25%

    # Per-segment decision thresholds (STRICT but realistic)
    # Based on empirical testing: baseline neural net output is ~0.04-0.06
    music_ratio_clean: float = 0.0      # ANY chunk with music detection → needs_demucs
    music_ratio_demucs: float = 0.15    # 0-15% → demucs, >15% → unusable
    music_mean_clean: float = 0.10      # ADJUSTED: 0.10 accounts for neural net baseline (was 0.05)
    music_mean_demucs: float = 0.25     # Threshold for heavy_contamination


Summary: auto


rg -n "input_source" massive_process.py src/config.py


Output:
Exit code: 0
Wall time: 0 seconds
Output:
src/config.py:38:    # bare 11-char video IDs and set input_source='r2' to stream originals from
src/config.py:40:    input_source: str = "youtube"  # 'youtube' | 'r2'
src/config.py:43:    # When input_source='r2', download.py will:
massive_process.py:1205:        'input_source': 'r2',


nl -ba massive_process.py | sed -n '1160,1245p'


Output:
Exit code: 0
Wall time: 0 seconds
Output:
  1160	    if supabase_mode:
  1161	        if not SUPABASE_AVAILABLE:
  1162	            logger.error("Supabase client not available! Install dependencies or check src/supabase_client.py")
  1163	            sys.exit(1)
  1164	        logger.info("Supabase queue mode: videos will be claimed from database")
  1165	    elif args.benchmark:
  1166	        video_urls = get_test_videos(args.num_videos)
  1167	        logger.info(f"Benchmark mode: {len(video_urls)} test videos")
  1168	    elif args.input:
  1169	        video_urls = load_video_urls(args.input)
  1170	        logger.info(f"Loaded {len(video_urls)} URLs from {args.input}")
  1171	    else:
  1172	        video_urls = args.urls
  1173
  1174	    if not supabase_mode and not video_urls:
  1175	        logger.error("No videos to process!")
  1176	        sys.exit(1)
  1177
  1178	    # CRITICAL: Capture HF_TOKEN before spawning workers
  1179	    # Spawned processes may not inherit environment variables
  1180	    hf_token = os.environ.get('HF_TOKEN', '')
  1181	    if not hf_token:
  1182	        # Try loading from .env file
  1183	        env_file = Path('.env')
  1184	        if env_file.exists():
  1185	            with open(env_file) as f:
  1186	                for line in f:
  1187	                    if line.startswith('HF_TOKEN='):
  1188	                        hf_token = line.strip().split('=', 1)[1].strip('"\'')
  1189	                        break
  1190
  1191	    if not hf_token:
  1192	        logger.error("HF_TOKEN not found! Set via environment or .env file")
  1193	        sys.exit(1)
  1194
  1195	    logger.info(f"HF_TOKEN captured (length={len(hf_token)})")
  1196
  1197	    # Config overrides
  1198	    config_overrides = {
  1199	        'cluster_merge_threshold': args.merge_threshold,
  1200	        'min_segment_duration': args.min_segment,
  1201	        'enable_music_detection': not args.no_music_detection,
  1202	        'workers_per_gpu': args.workers_per_gpu,
  1203	        'hf_token': hf_token,  # Pass token to spawned workers
  1204	        # R2 source (always R2, metadata from Supabase)
  1205	        'input_source': 'r2',
  1206	        'fetch_youtube_metadata_for_r2': False,
  1207	        'r2_source_prefix': 'podcasts/',
  1208	        'r2_source_extensions': '.webm',
  1209	        # R2 output upload
  1210	        'r2_upload_enabled': bool(args.r2_upload),
  1211	        'r2_upload_bucket_type': 'production',
  1212	        'r2_upload_prefix': args.r2_upload_prefix,
  1213	        'r2_upload_skip_if_exists': bool(args.r2_upload_skip_if_exists),
  1214	        # Export options
  1215	        'segment_audio': args.segment_audio,
  1216	        'background_export': args.background_export,
  1217	        'background_export_workers': args.export_workers,
  1218	    }
  1219
  1220	    # Initialize background export manager if enabled
  1221	    export_manager = None
  1222	    if args.background_export:
  1223	        from src.export import init_export_manager
  1224	        export_manager = init_export_manager(max_workers=args.export_workers)
  1225	        logger.info(f"Background export enabled ({args.export_workers} workers)")
  1226
  1227	    # Run!
  1228	    if supabase_mode:
  1229	        # Supabase distributed queue mode
  1230	        stats = run_supabase_queue(
  1231	            gpu_ids=gpu_ids,
  1232	            output_dir=args.output,
  1233	            config_overrides=config_overrides,
  1234	            max_videos_per_worker=args.max_videos,
  1235	            lease_duration_sec=args.lease_duration,
  1236	            poll_interval_sec=args.poll_interval
  1237	        )
  1238	    else:
  1239	        # Traditional file-based mode
  1240	        stats = run_massive_process(
  1241	            video_urls=video_urls,
  1242	            gpu_ids=gpu_ids,
  1243	            output_dir=args.output,
  1244	            config_overrides=config_overrides
  1245	        )


rg -n "claim_next_video" -n scripts/supabase_migrations.sql && rg -n "create or replace function claim_next_video" -n scripts/supabase_migrations.sql -n && sed -n '1,200p' scripts/supabase_migrations.sql


Output:
Exit code: 1
Wall time: 0 seconds
Output:
20:CREATE OR REPLACE FUNCTION claim_next_video(
114:GRANT EXECUTE ON FUNCTION claim_next_video(TEXT, INT) TO anon, authenticated, service_role;
323:--   AND routine_name IN ('claim_next_video', 'reclaim_expired_leases', 'get_queue_stats', 'get_cluster_stats');


nl -ba scripts/supabase_migrations.sql | sed -n '1,160p'