The file /home/ubuntu/pipecat/src/agent/__main__.py has been updated successfully.

"""Allow running as: python -m src.agent dev"""

import json
import logging
import os
import sys
import traceback
from datetime import timezone
from pathlib import Path

from dotenv import load_dotenv
from loguru import logger

PROJECT_ROOT = Path(__file__).parent.parent.parent
load_dotenv(PROJECT_ROOT / ".env")
load_dotenv(PROJECT_ROOT / ".env.local", override=True)

# ── Logging setup ──
# In production (DD_AGENT_HOST set): write JSON logs to stdout so the
# Datadog DaemonSet picks them up via container stdout collection.
# In dev: write human-readable logs to maya.log file.

LOG_DIR = PROJECT_ROOT / "logs"
LOG_DIR.mkdir(exist_ok=True)
LOG_FILE = LOG_DIR / "maya-agent.log"

IS_PRODUCTION = bool(os.getenv("DD_AGENT_HOST"))
LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO" if IS_PRODUCTION else "DEBUG")

def _log_filter(record):
    """Drop noise the same way src/core/logging.py does."""
    msg = record["message"]
    if "Unable to send message" in msg and "before joining" in msg:
        return False
    if record.get("exception"):
        exc_info = record["exception"]
        if exc_info and exc_info.value:
            exc_msg = str(exc_info.value)
            if "Unable to send message" in exc_msg and "before joining" in exc_msg:
                return False
    if record["level"].name in ("ERROR", "CRITICAL", "EXCEPTION"):
        return True
    name = record.get("name", "")
    if name and name.startswith(("botocore", "boto3", "urllib3", "s3transfer")):
        return False
    if name and name.startswith("openai"):
        return False
    if name and name.startswith("httpx"):
        return False
    if "Unclosed client session" in msg or "Unclosed connector" in msg:
        return False
    if "Task was destroyed but it is pending" in msg:
        return False
    return True

def _datadog_json_sink(message):
    """Format a loguru record as a single JSON line for Datadog."""
    record = message.record
    level = record["level"].name
    timestamp = record["time"].astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z"

    log_entry = {
        "timestamp": timestamp,
        "level": level,
        "logger": record["name"],
        "function": record["function"],
        "line": record["line"],
        "message": record["message"],
        "service": os.getenv("DD_SERVICE", "maya-pipecat-v6") + "-agent",
        "env": os.getenv("DD_ENV", os.getenv("ENVIRONMENT", "development")),
        "version": os.getenv("DD_VERSION", "1.0.0"),
        "process": "agent",
    }

    for key, value in record["extra"].items():
        if value and key not in ("_maya_configured",):
            log_entry[key] = value

    if level in ("ERROR", "CRITICAL", "EXCEPTION"):
        log_entry["status"] = "error"
        log_entry["error.kind"] = level
        if record.get("exception"):
            exc_info = record["exception"]
            if exc_info:
                log_entry["error.type"] = exc_info.type.__name__ if exc_info.type else "Unknown"
                log_entry["error.message"] = str(exc_info.value) if exc_info.value else ""
                log_entry["error.stack"] = "".join(
                    traceback.format_exception(exc_info.type, exc_info.value, exc_info.traceback)
                )
    else:
        log_entry["status"] = "info"

    sys.stdout.write(json.dumps(log_entry, ensure_ascii=False) + "\n")
    sys.stdout.flush()

logger.remove()

if IS_PRODUCTION:
    # Production: JSON to stdout for Datadog DaemonSet
    logger.add(_datadog_json_sink, level=LOG_LEVEL, filter=_log_filter)
else:
    # Dev: human-readable to file (separate from FastAPI maya.log to avoid conflicts)
    fmt = "{time:YYYY-MM-DD HH:mm:ss.SSS} | {level: <8} | {name}:{function}:{line} - {message}"
    logger.add(LOG_FILE, level="DEBUG", format=fmt, enqueue=True, backtrace=True, diagnose=True)

# ── Intercept stdlib logging (livekit, uvicorn, aiohttp use it) ──
class _InterceptHandler(logging.Handler):
    def emit(self, record):
        try:
            level = logger.level(record.levelname).name
        except ValueError:
            level = record.levelno
        frame, depth = sys._getframe(6), 6
        while frame and frame.f_code.co_filename == logging.__file__:
            frame = frame.f_back
            depth += 1
        logger.opt(depth=depth, exception=record.exc_info).log(level, record.getMessage())

logging.basicConfig(handlers=[_InterceptHandler()], level=logging.DEBUG, force=True)
for logger_name in ["livekit", "livekit.agents", "uvicorn", "fastapi", "aiohttp"]:
    logging.getLogger(logger_name).handlers = [_InterceptHandler()]
    logging.getLogger(logger_name).propagate = False

from src.agent.core.worker import create_agent_server
from livekit.agents import cli

if __name__ == "__main__":
    logger.info(f"AGENT_PROCESS_STARTING: pid={os.getpid()} production={IS_PRODUCTION} log_level={LOG_LEVEL}")
    server = create_agent_server()
    cli.run_app(server)

-# Configure loguru to write to maya.log (don't use configure_logging — it adds

-# duplicate sinks since subprocess stdout already pipes to maya.log)

+# In production (DD_AGENT_HOST set): write JSON logs to stdout so the

+# Datadog DaemonSet picks them up via container stdout collection.

+# In dev: write human-readable logs to maya.log file.

+LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO" if IS_PRODUCTION else "DEBUG")

+    """Drop noise the same way src/core/logging.py does."""

+    if "Unable to send message" in msg and "before joining" in msg:

+        if exc_info and exc_info.value:

+            exc_msg = str(exc_info.value)

+            if "Unable to send message" in exc_msg and "before joining" in exc_msg:

+    if record["level"].name in ("ERROR", "CRITICAL", "EXCEPTION"):

+    if name and name.startswith(("botocore", "boto3", "urllib3", "s3transfer")):

+    if name and name.startswith("openai"):

+    if name and name.startswith("httpx"):

+    if "Unclosed client session" in msg or "Unclosed connector" in msg:

+    if "Task was destroyed but it is pending" in msg:

+    """Format a loguru record as a single JSON line for Datadog."""

+    timestamp = record["time"].astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z"

+        "function": record["function"],

+        "service": os.getenv("DD_SERVICE", "maya-pipecat-v6") + "-agent",

+        "env": os.getenv("DD_ENV", os.getenv("ENVIRONMENT", "development")),

+        "version": os.getenv("DD_VERSION", "1.0.0"),

+    for key, value in record["extra"].items():

+        if value and key not in ("_maya_configured",):

+    if level in ("ERROR", "CRITICAL", "EXCEPTION"):

+        log_entry["error.kind"] = level

+            exc_info = record["exception"]

+                log_entry["error.type"] = exc_info.type.__name__ if exc_info.type else "Unknown"

+                log_entry["error.message"] = str(exc_info.value) if exc_info.value else ""

+                log_entry["error.stack"] = "".join(

+                    traceback.format_exception(exc_info.type, exc_info.value, exc_info.traceback)

+    sys.stdout.write(json.dumps(log_entry, ensure_ascii=False) + "\n")

-fmt = "{time:YYYY-MM-DD HH:mm:ss.SSS} | {level: <8} | {name}:{function}:{line} - {message}"

-logger.add(LOG_FILE, level="DEBUG", format=fmt, enqueue=True, backtrace=True, diagnose=True)

-# Intercept stdlib logging (livekit uses it)

+    # Production: JSON to stdout for Datadog DaemonSet

+    logger.add(_datadog_json_sink, level=LOG_LEVEL, filter=_log_filter)

+    # Dev: human-readable to file (separate from FastAPI maya.log to avoid conflicts)

+    fmt = "{time:YYYY-MM-DD HH:mm:ss.SSS} | {level: <8} | {name}:{function}:{line} - {message}"

+    logger.add(LOG_FILE, level="DEBUG", format=fmt, enqueue=True, backtrace=True, diagnose=True)

+# ── Intercept stdlib logging (livekit, uvicorn, aiohttp use it) ──

+for logger_name in ["livekit", "livekit.agents", "uvicorn", "fastapi", "aiohttp"]:

+    logging.getLogger(logger_name).handlers = [_InterceptHandler()]

+    logging.getLogger(logger_name).propagate = False

+    logger.info(f"AGENT_PROCESS_STARTING: pid={os.getpid()} production={IS_PRODUCTION} log_level={LOG_LEVEL}")

"""Allow running as: python -m src.agent dev"""

from pathlib import Path
from dotenv import load_dotenv

PROJECT_ROOT = Path(__file__).parent.parent.parent
load_dotenv(PROJECT_ROOT / ".env")
load_dotenv(PROJECT_ROOT / ".env.local", override=True)

# Configure loguru to write to maya.log (don't use configure_logging — it adds
# duplicate sinks since subprocess stdout already pipes to maya.log)
import os
import sys
import logging
from loguru import logger

LOG_DIR = PROJECT_ROOT / "logs"
LOG_DIR.mkdir(exist_ok=True)
LOG_FILE = LOG_DIR / "maya.log"

logger.remove()
fmt = "{time:YYYY-MM-DD HH:mm:ss.SSS} | {level: <8} | {name}:{function}:{line} - {message}"
logger.add(LOG_FILE, level="DEBUG", format=fmt, enqueue=True, backtrace=True, diagnose=True)

# Intercept stdlib logging (livekit uses it)
class _InterceptHandler(logging.Handler):
    def emit(self, record):
        try:
            level = logger.level(record.levelname).name
        except ValueError:
            level = record.levelno
        frame, depth = sys._getframe(6), 6
        while frame and frame.f_code.co_filename == logging.__file__:
            frame = frame.f_back
            depth += 1
        logger.opt(depth=depth, exception=record.exc_info).log(level, record.getMessage())

logging.basicConfig(handlers=[_InterceptHandler()], level=logging.DEBUG, force=True)

from src.agent.core.worker import create_agent_server
from livekit.agents import cli

if __name__ == "__main__":
    server = create_agent_server()
    cli.run_app(server)


Now build, push, and rollout:


Bash

docker build -t asia-south1-docker.pkg.dev/mayaresearchgcp/maya-pipecat/maya-pipecat-v6:latest . 2>&1 | tail -3 && docker push asia-south1-docker.pkg.dev/mayaresearchgcp/maya-pipecat/maya-pipecat-v6:latest 2>&1 | tail -3 && kubectl rollout restart deployment maya-pipecat-v6

Build, push, restart v6


#1 transferring dockerfile: 2B done
#1 DONE 0.0s
ERROR: failed to solve: failed to read dockerfile: open Dockerfile: no such file or directory
a5d46777e2f7: Layer already exists
335fc45cf5e8: Layer already exists
latest: digest: sha256:91c99fd84bd0196a355d53a393b905705c3803d581d1af8ac8736c79353c80f5 size: 3044
deployment.apps/maya-pipecat-v6 restarted
[result-id: r8]


Build failed (wrong dir again — the rollout restarted the OLD image, not new). Let me rebuild from the right directory:


cd /home/ubuntu/pipecat && docker build -t asia-south1-docker.pkg.dev/mayaresearchgcp/maya-pipecat/maya-pipecat-v6:latest . 2>&1 | tail -3 && docker push asia-south1-docker.pkg.dev/mayaresearchgcp/maya-pipecat/maya-pipecat-v6:latest 2>&1 | tail -3 && kubectl rollout restart deployment maya-pipecat-v6

Build from correct dir, push, restart


#16 writing image sha256:e171eff0f95e615241440a861815231776497cf69f16aa298dd1a15b060e4114 done
#16 naming to asia-south1-docker.pkg.dev/mayaresearchgcp/maya-pipecat/maya-pipecat-v6:latest done
#16 DONE 0.1s
647e2d5e3c1f: Layer already exists
335fc45cf5e8: Layer already exists
latest: digest: sha256:bae8a0767da42886b97774d408c9ba610179de952c31dd015064cbd0e400972a size: 3044
deployment.apps/maya-pipecat-v6 restarted
[result-id: r9]


Let me verify the new pods come up and check that agent logs now appear in container stdout:


sleep 60 && pod=$(kubectl get pods -l app=maya-pipecat-v6 --field-selector=status.phase=Running -o jsonpath='{.items[0].metadata.name}') && echo "Pod: $pod" && echo "---" && kubectl logs $pod --tail=20 2>&1 | grep -E "process.*agent|AGENT_PROCESS_STARTING|AGENT_WORKER_STARTED|\"process\"" | head -5

Verify agent logs in stdout


Pod: maya-pipecat-v6-6968457f74-2644v
---


pod=$(kubectl get pods -l app=maya-pipecat-v6 --field-selector=status.phase=Running -o jsonpath='{.items[0].metadata.name}') && kubectl logs $pod --tail=30 2>&1 | tail -25

Check pod logs


INFO:     10.56.2.1:45056 - "GET /api/v6/health HTTP/1.1" 200 OK
INFO:     10.56.2.1:45058 - "GET /api/v6/health HTTP/1.1" 200 OK
INFO:     35.191.236.211:60034 - "GET /api/v6/health HTTP/1.1" 200 OK
INFO:     35.191.236.209:36616 - "GET /api/v6/health HTTP/1.1" 200 OK
INFO:     35.191.236.212:34110 - "GET /api/v6/health HTTP/1.1" 200 OK
INFO:     10.56.2.1:39732 - "GET /api/v6/health HTTP/1.1" 200 OK
INFO:     10.56.2.1:59190 - "GET /api/v6/health HTTP/1.1" 200 OK
INFO:     35.191.236.211:38744 - "GET /api/v6/health HTTP/1.1" 200 OK
INFO:     35.191.236.209:48394 - "GET /api/v6/health HTTP/1.1" 200 OK
INFO:     35.191.236.212:56660 - "GET /api/v6/health HTTP/1.1" 200 OK
INFO:     10.56.2.1:50490 - "GET /api/v6/health HTTP/1.1" 200 OK
INFO:     10.56.2.1:50502 - "GET /api/v6/health HTTP/1.1" 200 OK
INFO:     35.191.236.211:41954 - "GET /api/v6/health HTTP/1.1" 200 OK
INFO:     35.191.236.209:40398 - "GET /api/v6/health HTTP/1.1" 200 OK
INFO:     35.191.236.212:33126 - "GET /api/v6/health HTTP/1.1" 200 OK
INFO:     10.56.2.1:54798 - "GET /api/v6/health HTTP/1.1" 200 OK
INFO:     10.56.2.1:44746 - "GET /api/v6/health HTTP/1.1" 200 OK
INFO:     35.191.236.211:54250 - "GET /api/v6/health HTTP/1.1" 200 OK
INFO:     35.191.236.209:36422 - "GET /api/v6/health HTTP/1.1" 200 OK
INFO:     35.191.236.212:59650 - "GET /api/v6/health HTTP/1.1" 200 OK
INFO:     10.56.2.1:33896 - "GET /api/v6/health HTTP/1.1" 200 OK
INFO:     10.56.2.1:33906 - "GET /api/v6/health HTTP/1.1" 200 OK
INFO:     35.191.236.211:35882 - "GET /api/v6/health HTTP/1.1" 200 OK
INFO:     35.191.236.209:41222 - "GET /api/v6/health HTTP/1.1" 200 OK
INFO:     35.191.236.212:36076 - "GET /api/v6/health HTTP/1.1" 200 OK
[result-id: r10]


Only seeing FastAPI access logs, no agent JSON logs yet. Let me check from the start of the pod and look for AGENT_PROCESS_STARTING:


pod=$(kubectl get pods -l app=maya-pipecat-v6 --field-selector=status.phase=Running -o jsonpath='{.items[0].metadata.name}') && kubectl logs $pod 2>&1 | head -60

Check pod startup logs


{"timestamp": "2026-04-08T17:34:43.739Z", "level": "INFO", "logger": "src.core.logging", "function": "configure_logging", "line": 163, "message": "============================================================", "service": "maya-pipecat-v6", "env": "development", "version": "1.0.0", "status": "info"}
{"timestamp": "2026-04-08T17:34:43.740Z", "level": "INFO", "logger": "src.core.logging", "function": "configure_logging", "line": 164, "message": "MAYA LIVEKIT SERVER - STARTUP INITIATED", "service": "maya-pipecat-v6", "env": "development", "version": "1.0.0", "status": "info"}
{"timestamp": "2026-04-08T17:34:43.740Z", "level": "INFO", "logger": "src.core.logging", "function": "configure_logging", "line": 165, "message": "============================================================", "service": "maya-pipecat-v6", "env": "development", "version": "1.0.0", "status": "info"}
{"timestamp": "2026-04-08T17:34:43.740Z", "level": "INFO", "logger": "src.core.logging", "function": "configure_logging", "line": 166, "message": "STARTUP_CONFIG: env=development host=0.0.0.0 port=3006", "service": "maya-pipecat-v6", "env": "development", "version": "1.0.0", "status": "info"}
{"timestamp": "2026-04-08T17:34:47.915Z", "level": "WARNING", "logger": "warnings", "function": "_showwarnmsg", "line": 112, "message": "/app/venv/lib/python3.12/site-packages/websockets/legacy/__init__.py:6: DeprecationWarning: websockets.legacy is deprecated; see https://websockets.readthedocs.io/en/stable/howto/upgrade.html for upgrade instructions\n  warnings.warn(  # deprecated in 14.0 - 2024-11-09\n", "service": "maya-pipecat-v6", "env": "development", "version": "1.0.0", "status": "info"}
{"timestamp": "2026-04-08T17:34:47.939Z", "level": "WARNING", "logger": "warnings", "function": "_showwarnmsg", "line": 112, "message": "/app/venv/lib/python3.12/site-packages/uvicorn/protocols/websockets/websockets_impl.py:16: DeprecationWarning: websockets.server.WebSocketServerProtocol is deprecated\n  from websockets.server import WebSocketServerProtocol\n", "service": "maya-pipecat-v6", "env": "development", "version": "1.0.0", "status": "info"}
{"timestamp": "2026-04-08T17:34:48.095Z", "level": "INFO", "logger": "src.core.redis", "function": "init_redis", "line": 59, "message": "REDIS_CONNECTED: url=10.222.58.99:6379/0", "service": "maya-pipecat-v6", "env": "development", "version": "1.0.0", "status": "info"}
{"timestamp": "2026-04-08T17:34:48.355Z", "level": "INFO", "logger": "src.core.clickhouse", "function": "init_clickhouse", "line": 52, "message": "CLICKHOUSE_CONNECTED: host=sfwp0q10n7.ap-south-1.aws.clickhouse.cloud:8443 db=maya", "service": "maya-pipecat-v6", "env": "development", "version": "1.0.0", "status": "info"}
{"timestamp": "2026-04-08T17:34:48.395Z", "level": "INFO", "logger": "src.core.clickhouse", "function": "_create_tables", "line": 176, "message": "CLICKHOUSE_TABLES_CREATED", "service": "maya-pipecat-v6", "env": "development", "version": "1.0.0", "status": "info"}
{"timestamp": "2026-04-08T17:34:48.484Z", "level": "INFO", "logger": "src.repository.experiment_repository", "function": "create_table", "line": 40, "message": "EXPERIMENTS_TABLE_CREATED", "service": "maya-pipecat-v6", "env": "development", "version": "1.0.0", "status": "info"}
{"timestamp": "2026-04-08T17:34:48.486Z", "level": "INFO", "logger": "src.main", "function": "lifespan", "line": 74, "message": "AGENT_WORKER_STARTED: pid=74", "service": "maya-pipecat-v6", "env": "development", "version": "1.0.0", "status": "info"}
INFO:     10.56.2.1:40546 - "GET /api/v6/health HTTP/1.1" 200 OK
INFO:     35.191.236.211:43502 - "GET /api/v6/health HTTP/1.1" 200 OK
INFO:     35.191.236.209:48904 - "GET /api/v6/health HTTP/1.1" 200 OK
INFO:     35.191.236.212:47912 - "GET /api/v6/health HTTP/1.1" 200 OK
INFO:     10.56.2.1:57650 - "GET /api/v6/health HTTP/1.1" 200 OK
INFO:     35.191.236.211:42976 - "GET /api/v6/health HTTP/1.1" 200 OK
INFO:     35.191.236.209:55464 - "GET /api/v6/health HTTP/1.1" 200 OK
INFO:     35.191.236.212:48474 - "GET /api/v6/health HTTP/1.1" 200 OK
INFO:     10.56.2.1:36444 - "GET /api/v6/health HTTP/1.1" 200 OK
INFO:     10.56.2.1:36452 - "GET /api/v6/health HTTP/1.1" 200 OK
INFO:     35.191.236.211:36204 - "GET /api/v6/health HTTP/1.1" 200 OK
INFO:     35.191.236.209:34340 - "GET /api/v6/health HTTP/1.1" 200 OK
INFO:     35.191.236.212:46506 - "GET /api/v6/health HTTP/1.1" 200 OK
INFO:     10.56.2.1:52764 - "GET /api/v6/health HTTP/1.1" 200 OK
INFO:     10.56.2.1:49648 - "GET /api/v6/health HTTP/1.1" 200 OK
INFO:     35.191.236.211:40326 - "GET /api/v6/health HTTP/1.1" 200 OK
INFO:     35.191.236.209:39516 - "GET /api/v6/health HTTP/1.1" 200 OK
INFO:     35.191.236.212:36100 - "GET /api/v6/health HTTP/1.1" 200 OK
INFO:     10.56.2.1:40818 - "GET /api/v6/health HTTP/1.1" 200 OK
INFO:     10.56.2.1:40826 - "GET /api/v6/health HTTP/1.1" 200 OK
INFO:     35.191.236.211:58590 - "GET /api/v6/health HTTP/1.1" 200 OK
INFO:     35.191.236.209:47658 - "GET /api/v6/health HTTP/1.1" 200 OK
INFO:     35.191.236.212:35232 - "GET /api/v6/health HTTP/1.1" 200 OK
{"timestamp": "2026-04-08T17:36:10.117Z", "level": "INFO", "logger": "src.core.database", "function": "execute", "line": 157, "message": "TURSO_WRITE: op=INSERT rows_affected=1 result_type=ok", "service": "maya-pipecat-v6", "env": "development", "version": "1.0.0", "status": "info"}
{"timestamp": "2026-04-08T17:36:10.119Z", "level": "INFO", "logger": "src.repository.device_repository", "function": "save_region", "line": 226, "message": "DB_DEVICE_REGION_SAVED: device_id=4482e88f25ca7847 region=india country=None", "service": "maya-pipecat-v6", "env": "development", "version": "1.0.0", "status": "info"}
{"timestamp": "2026-04-08T17:36:10.119Z", "level": "INFO", "logger": "src.service.app_v2_service", "function": "get_on_open_config", "line": 70, "message": "DEVICE_REGION_SET: device_id=4482e88f25ca7847 timezone=Asia/Calcutta region=india country=None", "service": "maya-pipecat-v6", "env": "development", "version": "1.0.0", "status": "info"}
INFO:     35.191.49.237:50710 - "GET /api/v6/on_open?device_id=4482e88f25ca7847&timezone=Asia%2FCalcutta HTTP/1.1" 200 OK
INFO:     10.56.2.1:58244 - "GET /api/v6/health HTTP/1.1" 200 OK
INFO:     10.56.2.1:58046 - "GET /api/v6/health HTTP/1.1" 200 OK
INFO:     35.191.236.211:49366 - "GET /api/v6/health HTTP/1.1" 200 OK
INFO:     35.191.236.209:51270 - "GET /api/v6/health HTTP/1.1" 200 OK
INFO:     35.191.236.212:48670 - "GET /api/v6/health HTTP/1.1" 200 OK
INFO:     10.56.2.1:54494 - "GET /api/v6/health HTTP/1.1" 200 OK
INFO:     10.56.2.1:54500 - "GET /api/v6/health HTTP/1.1" 200 OK
INFO:     35.191.236.211:34858 - "GET /api/v6/health HTTP/1.1" 200 OK
INFO:     35.191.236.209:40010 - "GET /api/v6/health HTTP/1.1" 200 OK
INFO:     35.191.236.212:54428 - "GET /api/v6/health HTTP/1.1" 200 OK
INFO:     10.56.2.1:36558 - "GET /api/v6/health HTTP/1.1" 200 OK
INFO:     10.56.2.1:48744 - "GET /api/v6/health HTTP/1.1" 200 OK
INFO:     35.191.236.211:49780 - "GET /api/v6/health HTTP/1.1" 200 OK
INFO:     35.191.236.209:41210 - "GET /api/v6/health HTTP/1.1" 200 OK
INFO:     35.191.236.212:37658 - "GET /api/v6/health HTTP/1.1" 200 OK
INFO:     10.56.2.1:56602 - "GET /api/v6/health HTTP/1.1" 200 OK
INFO:     10.56.2.1:56606 - "GET /api/v6/health HTTP/1.1" 200 OK
INFO:     35.191.236.211:58946 - "GET /api/v6/health HTTP/1.1" 200 OK
INFO:     35.191.236.209:41194 - "GET /api/v6/health HTTP/1.1" 200 OK
INFO:     35.191.236.212:60606 - "GET /api/v6/health HTTP/1.1" 200 OK
INFO:     10.56.2.1:39946 - "GET /api/v6/health HTTP/1.1" 200 OK
INFO:     10.56.2.1:45930 - "GET /api/v6/health HTTP/1.1" 200 OK
[result-id: r11]


**Good news**: I see `AGENT_WORKER_STARTED: pid=74` — agent subprocess started.

**Bad news**: I don't see `AGENT_PROCESS_STARTING` log line that should appear when our new agent code runs. Also no JSON logs from agent showing up.

Let me check if the agent process is actually using the new code. The new build was pushed and the rollout restarted, but let me verify by checking for any agent-related output:


pod=$(kubectl get pods -l app=maya-pipecat-v6 --field-selector=status.phase=Running -o jsonpath='{.items[0].metadata.name}') && kubectl logs $pod 2>&1 | grep -iE "AGENT_PROCESS|agent|livekit" | head -20

Check for agent-specific logs


{"timestamp": "2026-04-08T17:35:04.522Z", "level": "INFO", "logger": "src.core.logging", "function": "configure_logging", "line": 164, "message": "MAYA LIVEKIT SERVER - STARTUP INITIATED", "service": "maya-pipecat-v6", "env": "development", "version": "1.0.0", "status": "info"}
{"timestamp": "2026-04-08T17:35:12.170Z", "level": "INFO", "logger": "src.main", "function": "lifespan", "line": 74, "message": "AGENT_WORKER_STARTED: pid=74", "service": "maya-pipecat-v6", "env": "development", "version": "1.0.0", "status": "info"}
[result-id: r12]


kubectl get pods -l app=maya-pipecat-v6 --no-headers | head -5

Check pod ages


maya-pipecat-v6-6968457f74-ndn4n   1/1   Terminating   0     12h
maya-pipecat-v6-6968457f74-pgr9w   1/1   Terminating   0     12h
maya-pipecat-v6-6fcc566f6-262db    1/1   Terminating   0     2m14s
maya-pipecat-v6-6fcc566f6-46rmr    1/1   Terminating   0     2m14s
maya-pipecat-v6-6fcc566f6-59sw7    1/1   Terminating   0     2m14s
[result-id: r13]


kubectl get pods -l app=maya-pipecat-v6 --no-headers | awk '{print $1, $2, $3, $5}' | sort -k4 | head -10

Find the freshest pod


maya-pipecat-v6-c7b746469-4xnsk 1/1 Running 109s
maya-pipecat-v6-c7b746469-6tpgv 1/1 Running 109s
maya-pipecat-v6-c7b746469-8b7gr 1/1 Running 109s
maya-pipecat-v6-c7b746469-bg5gh 1/1 Running 109s
maya-pipecat-v6-c7b746469-fhsw4 1/1 Running 109s
maya-pipecat-v6-c7b746469-kstq6 1/1 Running 109s
maya-pipecat-v6-c7b746469-l2pzx 1/1 Running 109s
maya-pipecat-v6-c7b746469-nnglw 1/1 Running 109s
maya-pipecat-v6-c7b746469-pfh8x 1/1 Running 109s
maya-pipecat-v6-c7b746469-rgzws 1/1 Running 109s
[result-id: r14]


The new replicaset is `c7b746469`. Let me check logs from a fresh pod:


kubectl logs maya-pipecat-v6-c7b746469-4xnsk 2>&1 | grep -iE "AGENT_PROCESS|process\":|agent" | head -10

Check fresh pod logs


{"timestamp": "2026-04-09T05:34:38.375Z", "level": "INFO", "logger": "src.main", "function": "lifespan", "line": 72, "message": "AGENT_WORKER_STARTED: pid=74", "service": "maya-pipecat-v6", "env": "development", "version": "1.0.0", "status": "info"}
{"timestamp": "2026-04-09T05:34:42.557Z", "level": "INFO", "logger": "__main__", "function": "<module>", "line": 134, "message": "AGENT_PROCESS_STARTING: pid=74 production=True log_level=INFO", "service": "maya-pipecat-v6-agent", "env": "development", "version": "1.0.0", "process": "agent", "status": "info"}
{"timestamp": "2026-04-09T05:34:42.578Z", "level": "INFO", "logger": "livekit.rtc._ffi_client", "function": "ffi_event_callback", "line": 174, "message": "livekit_ffi::server:152:livekit_ffi::server - initializing ffi server v0.12.50", "service": "maya-pipecat-v6-agent", "env": "development", "version": "1.0.0", "process": "agent", "status": "info"}
{"timestamp": "2026-04-09T05:34:42.579Z", "level": "INFO", "logger": "livekit.rtc._ffi_client", "function": "ffi_event_callback", "line": 174, "message": "livekit_ffi::cabi:50:livekit_ffi::cabi - initializing ffi server v0.12.50", "service": "maya-pipecat-v6-agent", "env": "development", "version": "1.0.0", "process": "agent", "status": "info"}
{"timestamp": "2026-04-09T05:34:42.593Z", "level": "Level 23", "logger": "livekit.agents.cli.watcher", "function": "run", "line": 88, "message": "Watching /app/src/agent", "service": "maya-pipecat-v6-agent", "env": "development", "version": "1.0.0", "process": "agent", "status": "info"}
    05:34:46.140 INFO … livekit.agents     starting worker
    05:34:46.143 INFO … livekit.agents     preloading plugins
    05:34:46.155 INFO … livekit.agents     HTTP server listening on :34909
    05:34:46.184 INFO … livekit.agents     registered worker
                                         {"agent_name": "", "id":
[result-id: r15]


**🎉 IT WORKS.** Look at the fresh pod logs:

```json
{"timestamp": "2026-04-09T05:34:42.557Z", "level": "INFO", "logger": "__main__",
 "message": "AGENT_PROCESS_STARTING: pid=74 production=True log_level=INFO",
 "service": "maya-pipecat-v6-agent", "process": "agent", ...}

{"timestamp": "2026-04-09T05:34:42.578Z", "level": "INFO",
 "logger": "livekit.rtc._ffi_client",
 "message": "livekit_ffi::server:152:livekit_ffi::server - initializing ffi server v0.12.50",
 "service": "maya-pipecat-v6-agent", "process": "agent", ...}

{"logger": "livekit.agents.cli.watcher", "message": "Watching /app/src/agent",
 "service": "maya-pipecat-v6-agent", "process": "agent", ...}
```

## What changed

| Before | After |
|---|---|
| Agent stdout → `/app/logs/maya.log` (file inside container) | Agent stdout → container stdout |
| DD never saw agent logs | DD picks up via DaemonSet |
| Plain text format | JSON with `service=maya-pipecat-v6-agent`, `process=agent` |
| LiveKit framework logs invisible | All `livekit.rtc`, `livekit.agents` logs visible |

## In Datadog you'll now see

- **Service `maya-pipecat-v6-agent`** (separate from `maya-pipecat-v6` for the FastAPI server)
- All agent process logs: tool calls, image generation, voice/audio metrics, credit checks
- LiveKit framework internals: `livekit.rtc._ffi_client`, `livekit.agents.cli.watcher`, etc.
- Worker startup events
- The `process: "agent"` field for filtering

You can filter in DD by `service:maya-pipecat-v6-agent` to see only agent logs, or `service:maya-pipecat-v6` for the FastAPI server logs.

Some plain text "INFO" lines like `livekit.agents starting worker` are coming from LiveKit's own non-loguru logging — those still appear as plain text but are still in stdout, so DD will collect them as raw log lines (just without structured fields).


Apr 09, 2026 at 11:18:01.758 am(half a minute ago)
View in Context

Host
gke-maya-pipecat-default-pool-f76748ed-vks4

Service
maya-pipecat-v6

Source
maya-pipecat-v6

Container Name
maya-pipecat-v6

Docker Image
asia-south1-docker.pkg.dev/mayaresearchgcp/maya-pipecat/maya-pipecat-v6

Pod Name
maya-pipecat-v6-c7b746469-wh4b9
All Tags
cluster_name:maya-pipecat
cluster-location:asia-south1-a
cluster-name:maya-pipecat
cluster-uid:42efdb09877d4a7e827310761d67a30124d7af5e668a4404bf1965109c947c32
container_id:24064b399ddec0d4e3fa4db8037261c87d30fc7ec4d124bb32b53674872b5246

+38
container_name:maya-pipecat-v6
container_name:maya-pipecat-v6
Log Message
2026-04-09 05:48:01.694 | INFO     | src.agent.core.entrypoint:entrypoint:171 - SESSION_STARTED: session=b271388a-f434-457d-8ed6-888e273e3813 persona=edit_photos voice=Taru tools=['cf_image']

Fields & Attributes
Metrics
Processes
 what is this error?


**This is NOT an error.** Let me decode it:

```
2026-04-09 05:48:01.694 | INFO     | src.agent.core.entrypoint:entrypoint:171 -
SESSION_STARTED: session=b271388a-f434-457d-8ed6-888e273e3813
persona=edit_photos voice=Taru tools=['cf_image']
```

| Field | Value | Meaning |
|---|---|---|
| Level | **INFO** | Informational, not an error |
| Source | `src.agent.core.entrypoint:entrypoint:171` | file:function:line |
| Event | `SESSION_STARTED` | A user just started a voice call |
| `session=b271388a-...` | UUID | Unique session identifier |
| `persona=edit_photos` | persona name | User picked the photo editing persona |
| `voice=Taru` | TTS voice | Veena3 voice being used |
| `tools=['cf_image']` | available tools | Image generation tool is enabled for this session |

This log fires **every time a user opens a voice call** with Maya. It's a normal lifecycle event — the agent successfully spun up a session for someone editing photos with the "Taru" voice.

## Why it looks "raw" (not JSON like the others)

You'll notice my fix made some logs come through as JSON (`{"timestamp": ..., "service": "maya-pipecat-v6-agent"}`) but this one is plain text. Reason:

The LiveKit Agents framework architecture has **3 layers of processes**:
1. **FastAPI server** (`python -m src.main`) — uses `core/logging.py`, JSON ✅
2. **Agent worker manager** (`python -m src.agent dev`) — uses my new JSON sink ✅ (logs `AGENT_PROCESS_STARTING`, `livekit.rtc._ffi_client`, etc.)
3. **Per-session job processes** — LiveKit spawns a FRESH subprocess for each user session. These don't run our `__main__.py` logging setup, so they fall back to loguru's default plain-text format

The good news: **all three layers are now visible in DD** because they all write to container stdout. The bad news: job processes (where most session-level logs live, like `SESSION_STARTED`, image generation, etc.) come through as plain text rather than structured JSON.

DD will still index them and you can search by text, but you won't get nice structured fields like `session_id` or `persona` as searchable attributes.

## Want me to fix the structured logging for job processes too?

I can add a small init module that runs at the top of `src/agent/core/entrypoint.py` (or wherever the per-session entry point is) to apply the same JSON sink. That way every session's logs come through as parseable JSON in DD.

Or you can leave it as-is — logs are visible, just plain text for session-level events.


y it cameas error


**Because of how Datadog classifies stderr.**

Here's what's happening:

1. The log inside Python is `INFO` level — not an error
2. But it's coming from a **LiveKit job subprocess** (per-session worker) that doesn't have our JSON sink configured
3. **Loguru's default sink writes to `sys.stderr`** (not stdout)
4. When the container writes to stderr, **Datadog DaemonSet tags it as `error` severity by default** — regardless of what the actual log message says

So:

| Layer | Where it goes | DD severity |
|---|---|---|
| FastAPI server (`maya-pipecat-v6`) | stdout (JSON) | parsed correctly → INFO |
| Agent worker manager (`maya-pipecat-v6-agent`) | stdout (JSON, my new sink) | parsed correctly → INFO |
| **Per-session job process** | **stderr** (plain text, default loguru) | **DD tags as ERROR** |

It's a Datadog convention — anything from container stderr is treated as error severity. Even though the message clearly says `| INFO |`, DD doesn't parse the level out of the plain text format, so it falls back to the stderr=error rule.

## The fix

I need to apply the same JSON-to-stdout sink in the per-session job process. The job process loads `src/agent/core/entrypoint.py` (or similar), so we need to configure logging there too.

Want me to fix it? It's the same JSON sink code, just imported at the top of the entrypoint module so every job subprocess sets up logging on first import.


yeah do ... i want everything correctly coem .. becuase i dont want to fuck up


Check agent core files


cleanup.py
entrypoint.py
events.py
__init__.py
pipeline.py
prompt.py
session.py
worker.py


1	"""Main agent entrypoint — called per session by LiveKit AgentServer.
2
3	Handles: connect → load config → create session → register handlers → start agent.
4	"""
5
6	import json
7	import time
8
9	from loguru import logger
10	from livekit.agents import Agent, AgentSession, JobContext
11
12	from src.agent.core.cleanup import cleanup_session
13	from src.agent.core.events import register_session_events
14	from src.agent.core.pipeline import create_vad, create_stt, create_llm, create_tts, create_gemini_live
15	from src.agent.core.prompt import load_system_prompt, get_tools_from_persona
16	from src.agent.core.session import AgentSessionState
17	from src.agent.handlers.image_upload import register_image_upload_handler
18	from src.agent.tools import build_tool_list
19	from src.agent.utils.data_channel import send_session_started
20
21
22	class MayaAgent(Agent):
23	    """Maya voice agent with dynamic tools and intro greeting."""
24
25	    def __init__(self, instructions: str, intro: str = "", tools_list=None, initial_image_url: str = None, language: str = "hi"):
26	        super().__init__(instructions=instructions, tools=tools_list or [])
27	        self._intro = intro
28	        self._initial_image_url = initial_image_url
29	        self._language = language
30
31	    async def on_enter(self):
32	        # IMPORTANT: Gemini Live does NOT support tool_choice="none", so greeting
33	        # instructions must explicitly forbid tool calls to prevent auto-generation.
34	        if self._initial_image_url:
35	            if self._language == "hi":
36	                self.session.generate_reply(
37	                    instructions="User ne photo bhej di hai. Short mein bol 'Photo mil gayi! Kya karna hai isme?' — do NOT call any tools. Do NOT generate or edit any image. Just speak.",
38	                    tool_choice="none",
39	                )
40	            else:
41	                self.session.generate_reply(
42	                    instructions="The user has uploaded a photo. Briefly say you received it and ask what they want to do. One line only. Do NOT call any tools. Do NOT generate or edit any image. Just speak.",
43	                    tool_choice="none",
44	                )
45	        elif self._intro:
46	            self.session.generate_reply(
47	                instructions=f"Greet the user: {self._intro}. Do NOT call any tools. Just speak.",
48	                tool_choice="none",
49	            )
50	        else:
[result-id: r16]

"""Main agent entrypoint — called per session by LiveKit AgentServer.

Handles: connect → load config → create session → register handlers → start agent.
"""

import json
import time

from loguru import logger
from livekit.agents import Agent, AgentSession, JobContext

from src.agent.core.cleanup import cleanup_session
from src.agent.core.events import register_session_events
from src.agent.core.pipeline import create_vad, create_stt, create_llm, create_tts, create_gemini_live
from src.agent.core.prompt import load_system_prompt, get_tools_from_persona
from src.agent.core.session import AgentSessionState
from src.agent.handlers.image_upload import register_image_upload_handler
from src.agent.tools import build_tool_list
from src.agent.utils.data_channel import send_session_started

class MayaAgent(Agent):
    """Maya voice agent with dynamic tools and intro greeting."""

    def __init__(self, instructions: str, intro: str = "", tools_list=None, initial_image_url: str = None, language: str = "hi"):
        super().__init__(instructions=instructions, tools=tools_list or [])
        self._intro = intro
        self._initial_image_url = initial_image_url
        self._language = language

    async def on_enter(self):
        # IMPORTANT: Gemini Live does NOT support tool_choice="none", so greeting
        # instructions must explicitly forbid tool calls to prevent auto-generation.
        if self._initial_image_url:
            if self._language == "hi":
                self.session.generate_reply(
                    instructions="User ne photo bhej di hai. Short mein bol 'Photo mil gayi! Kya karna hai isme?' — do NOT call any tools. Do NOT generate or edit any image. Just speak.",
                    tool_choice="none",
                )
            else:
                self.session.generate_reply(
                    instructions="The user has uploaded a photo. Briefly say you received it and ask what they want to do. One line only. Do NOT call any tools. Do NOT generate or edit any image. Just speak.",
                    tool_choice="none",
                )
        elif self._intro:
            self.session.generate_reply(
                instructions=f"Greet the user: {self._intro}. Do NOT call any tools. Just speak.",
                tool_choice="none",
            )
        else:


Now let me create a shared logging module and use it everywhere:


Check utils dir


audio_recorder.py
data_channel.py
image_helpers.py
__init__.py


Let me create a shared logging setup module for the agent that can be imported by both `__main__.py` and from per-session subprocesses. The cleanest place is `src/agent/utils/agent_logging.py`:


Write

"""Shared logging setup for the agent worker manager AND per-session job subprocesses.

LiveKit Agents spawns a fresh subprocess for every voice session. Each subprocess
needs its own logging configuration; otherwise loguru falls back to its default
sink (stderr, plain text) which Datadog tags as ERROR severity.

This module exposes `setup_agent_logging()` which:
- In production (DD_AGENT_HOST set): writes JSON logs to STDOUT (not stderr) so
  the Datadog DaemonSet picks them up via container stdout collection AND tags
  them with the correct severity from the JSON `level` field.
- In dev: writes human-readable logs to a per-process file under logs/.

Idempotent — safe to call from any module on import.
"""

import json
import logging
import os
import sys
import traceback
from datetime import timezone
from pathlib import Path

from loguru import logger

_PROJECT_ROOT = Path(__file__).parent.parent.parent.parent
_LOG_DIR = _PROJECT_ROOT / "logs"

_IS_PRODUCTION = bool(os.getenv("DD_AGENT_HOST"))
_LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO" if _IS_PRODUCTION else "DEBUG")

# Sentinel to ensure we only configure once per process
_CONFIGURED = False

def _log_filter(record):
    """Drop noise — same rules as src/core/logging.py."""
    msg = record["message"]
    if "Unable to send message" in msg and "before joining" in msg:
        return False
    if record.get("exception"):
        exc_info = record["exception"]
        if exc_info and exc_info.value:
            exc_msg = str(exc_info.value)
            if "Unable to send message" in exc_msg and "before joining" in exc_msg:
                return False
    if record["level"].name in ("ERROR", "CRITICAL", "EXCEPTION"):
        return True
    name = record.get("name", "")
    if name and name.startswith(("botocore", "boto3", "urllib3", "s3transfer")):
        return False
    if name and name.startswith("openai"):
        return False
    if name and name.startswith("httpx"):
        return False
    if "Unclosed client session" in msg or "Unclosed connector" in msg:
        return False
    if "Task was destroyed but it is pending" in msg:
        return False
    return True

def _datadog_json_sink(message):
    """Format a loguru record as a single JSON line and write to STDOUT."""
    record = message.record
    level = record["level"].name
    timestamp = record["time"].astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z"

    log_entry = {
        "timestamp": timestamp,
        "level": level,
        "logger": record["name"],
        "function": record["function"],
        "line": record["line"],
        "message": record["message"],
        "service": os.getenv("DD_SERVICE", "maya-pipecat-v6") + "-agent",
        "env": os.getenv("DD_ENV", os.getenv("ENVIRONMENT", "development")),
        "version": os.getenv("DD_VERSION", "1.0.0"),
        "process": "agent",
        "pid": os.getpid(),
    }

    for key, value in record["extra"].items():
        if value and key not in ("_maya_configured",):
            log_entry[key] = value

    if level in ("ERROR", "CRITICAL", "EXCEPTION"):
        log_entry["status"] = "error"
        log_entry["error.kind"] = level
        if record.get("exception"):
            exc_info = record["exception"]
            if exc_info:
                log_entry["error.type"] = exc_info.type.__name__ if exc_info.type else "Unknown"
                log_entry["error.message"] = str(exc_info.value) if exc_info.value else ""
                log_entry["error.stack"] = "".join(
                    traceback.format_exception(exc_info.type, exc_info.value, exc_info.traceback)
                )
    elif level == "WARNING":
        log_entry["status"] = "warning"
    else:
        log_entry["status"] = "info"

    sys.stdout.write(json.dumps(log_entry, ensure_ascii=False) + "\n")
    sys.stdout.flush()

class _InterceptHandler(logging.Handler):
    """Route stdlib logging (livekit, uvicorn, aiohttp) through loguru."""

    def emit(self, record):
        try:
            level = logger.level(record.levelname).name
        except ValueError:
            level = record.levelno
        frame, depth = sys._getframe(6), 6
        while frame and frame.f_code.co_filename == logging.__file__:
            frame = frame.f_back
            depth += 1
        logger.opt(depth=depth, exception=record.exc_info).log(level, record.getMessage())

def setup_agent_logging():
    """Configure loguru + stdlib logging for the current process. Idempotent."""
    global _CONFIGURED
    if _CONFIGURED:
        return

    logger.remove()

    if _IS_PRODUCTION:
        # JSON to stdout — picked up by Datadog DaemonSet, tagged via JSON level field
        logger.add(_datadog_json_sink, level=_LOG_LEVEL, filter=_log_filter)
    else:
        # Dev: per-process file (so multiple subprocesses don't fight over one file)
        _LOG_DIR.mkdir(exist_ok=True)
        log_file = _LOG_DIR / f"maya-agent-{os.getpid()}.log"
        fmt = "{time:YYYY-MM-DD HH:mm:ss.SSS} | {level: <8} | {name}:{function}:{line} - {message}"
        logger.add(log_file, level="DEBUG", format=fmt, enqueue=True, backtrace=True, diagnose=True)

    # Intercept stdlib logging from frameworks
    logging.basicConfig(handlers=[_InterceptHandler()], level=logging.DEBUG, force=True)
    for logger_name in [
        "livekit",
        "livekit.agents",
        "livekit.rtc",
        "uvicorn",
        "uvicorn.error",
        "uvicorn.access",
        "fastapi",
        "aiohttp",
    ]:
        logging.getLogger(logger_name).handlers = [_InterceptHandler()]
        logging.getLogger(logger_name).propagate = False

    _CONFIGURED = True


File created successfully at: /home/ubuntu/pipecat/src/agent/utils/agent_logging.py

"""Shared logging setup for the agent worker manager AND per-session job subprocesses.

LiveKit Agents spawns a fresh subprocess for every voice session. Each subprocess
needs its own logging configuration; otherwise loguru falls back to its default
sink (stderr, plain text) which Datadog tags as ERROR severity.

This module exposes `setup_agent_logging()` which:
- In production (DD_AGENT_HOST set): writes JSON logs to STDOUT (not stderr) so
  the Datadog DaemonSet picks them up via container stdout collection AND tags
  them with the correct severity from the JSON `level` field.
- In dev: writes human-readable logs to a per-process file under logs/.

Idempotent — safe to call from any module on import.
"""

import json
import logging
import os
import sys
import traceback
from datetime import timezone
from pathlib import Path

from loguru import logger

_PROJECT_ROOT = Path(__file__).parent.parent.parent.parent
_LOG_DIR = _PROJECT_ROOT / "logs"

_IS_PRODUCTION = bool(os.getenv("DD_AGENT_HOST"))
_LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO" if _IS_PRODUCTION else "DEBUG")

# Sentinel to ensure we only configure once per process
_CONFIGURED = False

def _log_filter(record):
    """Drop noise — same rules as src/core/logging.py."""
    msg = record["message"]
    if "Unable to send message" in msg and "before joining" in msg:
        return False
    if record.get("exception"):
        exc_info = record["exception"]
        if exc_info and exc_info.value:
            exc_msg = str(exc_info.value)
            if "Unable to send message" in exc_msg and "before joining" in exc_msg:
                return False
    if record["level"].name in ("ERROR", "CRITICAL", "EXCEPTION"):
        return True
    name = record.get("name", "")
    if name and name.startswith(("botocore", "boto3", "urllib3", "s3transfer")):
        return False
    if name and name.startswith("openai"):
        return False
    if name and name.startswith("httpx"):
        return False
    if "Unclosed client session" in msg or "Unclosed connector" in msg:
        return False
    if "Task was destroyed but it is pending" in msg:
        return False
    return True

def _datadog_json_sink(message):
    """Format a loguru record as a single JSON line and write to STDOUT."""
    record = message.record
    level = record["level"].name
    timestamp = record["time"].astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z"

    log_entry = {
        "timestamp": timestamp,
        "level": level,
        "logger": record["name"],
        "function": record["function"],
        "line": record["line"],
        "message": record["message"],
        "service": os.getenv("DD_SERVICE", "maya-pipecat-v6") + "-agent",
        "env": os.getenv("DD_ENV", os.getenv("ENVIRONMENT", "development")),
        "version": os.getenv("DD_VERSION", "1.0.0"),
        "process": "agent",
        "pid": os.getpid(),
    }

    for key, value in record["extra"].items():
        if value and key not in ("_maya_configured",):
            log_entry[key] = value

    if level in ("ERROR", "CRITICAL", "EXCEPTION"):
        log_entry["status"] = "error"
        log_entry["error.kind"] = level
        if record.get("exception"):
            exc_info = record["exception"]
            if exc_info:
                log_entry["error.type"] = exc_info.type.__name__ if exc_info.type else "Unknown"
                log_entry["error.message"] = str(exc_info.value) if exc_info.value else ""
                log_entry["error.stack"] = "".join(
                    traceback.format_exception(exc_info.type, exc_info.value, exc_info.traceback)
                )
    elif level == "WARNING":
        log_entry["status"] = "warning"
    else:
        log_entry["status"] = "info"

    sys.stdout.write(json.dumps(log_entry, ensure_ascii=False) + "\n")
    sys.stdout.flush()

class _InterceptHandler(logging.Handler):
    """Route stdlib logging (livekit, uvicorn, aiohttp) through loguru."""

    def emit(self, record):
        try:
            level = logger.level(record.levelname).name
        except ValueError:
            level = record.levelno
        frame, depth = sys._getframe(6), 6
        while frame and frame.f_code.co_filename == logging.__file__:
            frame = frame.f_back
            depth += 1
        logger.opt(depth=depth, exception=record.exc_info).log(level, record.getMessage())

def setup_agent_logging():
    """Configure loguru + stdlib logging for the current process. Idempotent."""
    global _CONFIGURED
    if _CONFIGURED:
        return

    logger.remove()

    if _IS_PRODUCTION:
        # JSON to stdout — picked up by Datadog DaemonSet, tagged via JSON level field
        logger.add(_datadog_json_sink, level=_LOG_LEVEL, filter=_log_filter)
    else:
        # Dev: per-process file (so multiple subprocesses don't fight over one file)
        _LOG_DIR.mkdir(exist_ok=True)
        log_file = _LOG_DIR / f"maya-agent-{os.getpid()}.log"
        fmt = "{time:YYYY-MM-DD HH:mm:ss.SSS} | {level: <8} | {name}:{function}:{line} - {message}"
        logger.add(log_file, level="DEBUG", format=fmt, enqueue=True, backtrace=True, diagnose=True)

    # Intercept stdlib logging from frameworks
    logging.basicConfig(handlers=[_InterceptHandler()], level=logging.DEBUG, force=True)
    for logger_name in [
        "livekit",
        "livekit.agents",
        "livekit.rtc",
        "uvicorn",
        "uvicorn.error",
        "uvicorn.access",
        "fastapi",
        "aiohttp",
    ]:
        logging.getLogger(logger_name).handlers = [_InterceptHandler()]
        logging.getLogger(logger_name).propagate = False

    _CONFIGURED = True


Now use it in `__main__.py` (replacing the inline duplicate code):


"""Allow running as: python -m src.agent dev"""

import os
from pathlib import Path

from dotenv import load_dotenv

PROJECT_ROOT = Path(__file__).parent.parent.parent
load_dotenv(PROJECT_ROOT / ".env")
load_dotenv(PROJECT_ROOT / ".env.local", override=True)

from src.agent.utils.agent_logging import setup_agent_logging

setup_agent_logging()

from loguru import logger
from src.agent.core.worker import create_agent_server
from livekit.agents import cli

if __name__ == "__main__":
    logger.info(f"AGENT_PROCESS_STARTING: pid={os.getpid()}")
    server = create_agent_server()
    cli.run_app(server)


The file /home/ubuntu/pipecat/src/agent/__main__.py has been updated successfully.

"""Allow running as: python -m src.agent dev"""

import os
from pathlib import Path

from dotenv import load_dotenv

PROJECT_ROOT = Path(__file__).parent.parent.parent
load_dotenv(PROJECT_ROOT / ".env")
load_dotenv(PROJECT_ROOT / ".env.local", override=True)

from src.agent.utils.agent_logging import setup_agent_logging

setup_agent_logging()

from loguru import logger
from src.agent.core.worker import create_agent_server
from livekit.agents import cli

if __name__ == "__main__":
    logger.info(f"AGENT_PROCESS_STARTING: pid={os.getpid()}")
    server = create_agent_server()
    cli.run_app(server)

-# In production (DD_AGENT_HOST set): write JSON logs to stdout so the

-# Datadog DaemonSet picks them up via container stdout collection.

-# In dev: write human-readable logs to maya.log file.

+from src.agent.utils.agent_logging import setup_agent_logging

-LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO" if IS_PRODUCTION else "DEBUG")

-    """Drop noise the same way src/core/logging.py does."""

-    if "Unable to send message" in msg and "before joining" in msg:

-        if exc_info and exc_info.value:

-            exc_msg = str(exc_info.value)

-            if "Unable to send message" in exc_msg and "before joining" in exc_msg:

-    if record["level"].name in ("ERROR", "CRITICAL", "EXCEPTION"):

-    if name and name.startswith(("botocore", "boto3", "urllib3", "s3transfer")):

-    if name and name.startswith("openai"):

-    if name and name.startswith("httpx"):

-    if "Unclosed client session" in msg or "Unclosed connector" in msg:

-    if "Task was destroyed but it is pending" in msg:

-    """Format a loguru record as a single JSON line for Datadog."""

-    timestamp = record["time"].astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z"

-        "function": record["function"],

-        "service": os.getenv("DD_SERVICE", "maya-pipecat-v6") + "-agent",

-        "env": os.getenv("DD_ENV", os.getenv("ENVIRONMENT", "development")),

-        "version": os.getenv("DD_VERSION", "1.0.0"),

-    for key, value in record["extra"].items():

-        if value and key not in ("_maya_configured",):

-    if level in ("ERROR", "CRITICAL", "EXCEPTION"):

-        log_entry["error.kind"] = level

-            exc_info = record["exception"]

-                log_entry["error.type"] = exc_info.type.__name__ if exc_info.type else "Unknown"

-                log_entry["error.message"] = str(exc_info.value) if exc_info.value else ""

-                log_entry["error.stack"] = "".join(

-                    traceback.format_exception(exc_info.type, exc_info.value, exc_info.traceback)

-    sys.stdout.write(json.dumps(log_entry, ensure_ascii=False) + "\n")

-    # Production: JSON to stdout for Datadog DaemonSet

-    logger.add(_datadog_json_sink, level=LOG_LEVEL, filter=_log_filter)

-    # Dev: human-readable to file (separate from FastAPI maya.log to avoid conflicts)

-    fmt = "{time:YYYY-MM-DD HH:mm:ss.SSS} | {level: <8} | {name}:{function}:{line} - {message}"

-    logger.add(LOG_FILE, level="DEBUG", format=fmt, enqueue=True, backtrace=True, diagnose=True)

-# ── Intercept stdlib logging (livekit, uvicorn, aiohttp use it) ──

-            level = logger.level(record.levelname).name

-        frame, depth = sys._getframe(6), 6

-        while frame and frame.f_code.co_filename == logging.__file__:

-        logger.opt(depth=depth, exception=record.exc_info).log(level, record.getMessage())

-for logger_name in ["livekit", "livekit.agents", "uvicorn", "fastapi", "aiohttp"]:

-    logging.getLogger(logger_name).handlers = [_InterceptHandler()]

-    logging.getLogger(logger_name).propagate = False

-    logger.info(f"AGENT_PROCESS_STARTING: pid={os.getpid()} production={IS_PRODUCTION} log_level={LOG_LEVEL}")

+    logger.info(f"AGENT_PROCESS_STARTING: pid={os.getpid()}")

"""Allow running as: python -m src.agent dev"""

import json
import logging
import os
import sys
import traceback
from datetime import timezone
from pathlib import Path

from dotenv import load_dotenv
from loguru import logger

PROJECT_ROOT = Path(__file__).parent.parent.parent
load_dotenv(PROJECT_ROOT / ".env")
load_dotenv(PROJECT_ROOT / ".env.local", override=True)

# ── Logging setup ──
# In production (DD_AGENT_HOST set): write JSON logs to stdout so the
# Datadog DaemonSet picks them up via container stdout collection.
# In dev: write human-readable logs to maya.log file.

LOG_DIR = PROJECT_ROOT / "logs"
LOG_DIR.mkdir(exist_ok=True)
LOG_FILE = LOG_DIR / "maya-agent.log"

IS_PRODUCTION = bool(os.getenv("DD_AGENT_HOST"))
LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO" if IS_PRODUCTION else "DEBUG")

def _log_filter(record):
    """Drop noise the same way src/core/logging.py does."""
    msg = record["message"]
    if "Unable to send message" in msg and "before joining" in msg:
        return False
    if record.get("exception"):
        exc_info = record["exception"]
        if exc_info and exc_info.value:
            exc_msg = str(exc_info.value)
            if "Unable to send message" in exc_msg and "before joining" in exc_msg:
                return False
    if record["level"].name in ("ERROR", "CRITICAL", "EXCEPTION"):
        return True
    name = record.get("name", "")
    if name and name.startswith(("botocore", "boto3", "urllib3", "s3transfer")):
        return False
    if name and name.startswith("openai"):
        return False
    if name and name.startswith("httpx"):
        return False
    if "Unclosed client session" in msg or "Unclosed connector" in msg:
        return False
    if "Task was destroyed but it is pending" in msg:
        return False
    return True

def _datadog_json_sink(message):
    """Format a loguru record as a single JSON line for Datadog."""
    record = message.record
    level = record["level"].name
    timestamp = record["time"].astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z"

    log_entry = {
        "timestamp": timestamp,
        "level": level,
        "logger": record["name"],
        "function": record["function"],
        "line": record["line"],
        "message": record["message"],
        "service": os.getenv("DD_SERVICE", "maya-pipecat-v6") + "-agent",
        "env": os.getenv("DD_ENV", os.getenv("ENVIRONMENT", "development")),
        "version": os.getenv("DD_VERSION", "1.0.0"),
        "process": "agent",
    }

    for key, value in record["extra"].items():
        if value and key not in ("_maya_configured",):
            log_entry[key] = value

    if level in ("ERROR", "CRITICAL", "EXCEPTION"):
        log_entry["status"] = "error"
        log_entry["error.kind"] = level
        if record.get("exception"):
            exc_info = record["exception"]
            if exc_info:
                log_entry["error.type"] = exc_info.type.__name__ if exc_info.type else "Unknown"
                log_entry["error.message"] = str(exc_info.value) if exc_info.value else ""
                log_entry["error.stack"] = "".join(
                    traceback.format_exception(exc_info.type, exc_info.value, exc_info.traceback)
                )
    else:
        log_entry["status"] = "info"

    sys.stdout.write(json.dumps(log_entry, ensure_ascii=False) + "\n")
    sys.stdout.flush()

logger.remove()

if IS_PRODUCTION:
    # Production: JSON to stdout for Datadog DaemonSet
    logger.add(_datadog_json_sink, level=LOG_LEVEL, filter=_log_filter)
else:
    # Dev: human-readable to file (separate from FastAPI maya.log to avoid conflicts)
    fmt = "{time:YYYY-MM-DD HH:mm:ss.SSS} | {level: <8} | {name}:{function}:{line} - {message}"
    logger.add(LOG_FILE, level="DEBUG", format=fmt, enqueue=True, backtrace=True, diagnose=True)

# ── Intercept stdlib logging (livekit, uvicorn, aiohttp use it) ──
class _InterceptHandler(logging.Handler):
    def emit(self, record):
        try:
            level = logger.level(record.levelname).name
        except ValueError:
            level = record.levelno
        frame, depth = sys._getframe(6), 6
        while frame and frame.f_code.co_filename == logging.__file__:
            frame = frame.f_back
            depth += 1
        logger.opt(depth=depth, exception=record.exc_info).log(level, record.getMessage())

logging.basicConfig(handlers=[_InterceptHandler()], level=logging.DEBUG, force=True)
for logger_name in ["livekit", "livekit.agents", "uvicorn", "fastapi", "aiohttp"]:
    logging.getLogger(logger_name).handlers = [_InterceptHandler()]
    logging.getLogger(logger_name).propagate = False

from src.agent.core.worker import create_agent_server
from livekit.agents import cli

if __name__ == "__main__":
    logger.info(f"AGENT_PROCESS_STARTING: pid={os.getpid()} production={IS_PRODUCTION} log_level={LOG_LEVEL}")
    server = create_agent_server()
    cli.run_app(server)


Now also call `setup_agent_logging()` at the top of `entrypoint.py` so per-session subprocesses configure logging when LiveKit imports the entrypoint module:


Edit

"""Main agent entrypoint — called per session by LiveKit AgentServer.

Handles: connect → load config → create session → register handlers → start agent.
"""

import json
import time

from loguru import logger
from livekit.agents import Agent, AgentSession, JobContext

"""Main agent entrypoint — called per session by LiveKit AgentServer.

Handles: connect → load config → create session → register handlers → start agent.
"""

# Configure logging FIRST — LiveKit spawns a fresh subprocess for every session,
# so this module is the first thing imported in that subprocess. Without this
# call, loguru falls back to its default stderr sink (plain text), which
# Datadog tags as ERROR severity regardless of the actual log level.
from src.agent.utils.agent_logging import setup_agent_logging
setup_agent_logging()

import json
import time

from loguru import logger
from livekit.agents import Agent, AgentSession, JobContext


The file /home/ubuntu/pipecat/src/agent/core/entrypoint.py has been updated successfully.

"""Main agent entrypoint — called per session by LiveKit AgentServer.

Handles: connect → load config → create session → register handlers → start agent.
"""

# Configure logging FIRST — LiveKit spawns a fresh subprocess for every session,
# so this module is the first thing imported in that subprocess. Without this
# call, loguru falls back to its default stderr sink (plain text), which
# Datadog tags as ERROR severity regardless of the actual log level.
from src.agent.utils.agent_logging import setup_agent_logging
setup_agent_logging()

import json
import time

from loguru import logger
from livekit.agents import Agent, AgentSession, JobContext

"""Main agent entrypoint — called per session by LiveKit AgentServer.

Handles: connect → load config → create session → register handlers → start agent.
"""

import json
import time

from loguru import logger
from livekit.agents import Agent, AgentSession, JobContext

from src.agent.core.cleanup import cleanup_session
from src.agent.core.events import register_session_events
from src.agent.core.pipeline import create_vad, create_stt, create_llm, create_tts, create_gemini_live
from src.agent.core.prompt import load_system_prompt, get_tools_from_persona
from src.agent.core.session import AgentSessionState
from src.agent.handlers.image_upload import register_image_upload_handler
from src.agent.tools import build_tool_list
from src.agent.utils.data_channel import send_session_started

class MayaAgent(Agent):
    """Maya voice agent with dynamic tools and intro greeting."""

    def __init__(self, instructions: str, intro: str = "", tools_list=None, initial_image_url: str = None, language: str = "hi"):
        super().__init__(instructions=instructions, tools=tools_list or [])
        self._intro = intro
        self._initial_image_url = initial_image_url
        self._language = language

    async def on_enter(self):
        # IMPORTANT: Gemini Live does NOT support tool_choice="none", so greeting
        # instructions must explicitly forbid tool calls to prevent auto-generation.
        if self._initial_image_url:
            if self._language == "hi":
                self.session.generate_reply(
                    instructions="User ne photo bhej di hai. Short mein bol 'Photo mil gayi! Kya karna hai isme?' — do NOT call any tools. Do NOT generate or edit any image. Just speak.",
                    tool_choice="none",
                )
            else:
                self.session.generate_reply(
                    instructions="The user has uploaded a photo. Briefly say you received it and ask what they want to do. One line only. Do NOT call any tools. Do NOT generate or edit any image. Just speak.",
                    tool_choice="none",
                )
        elif self._intro:
            self.session.generate_reply(
                instructions=f"Greet the user: {self._intro}. Do NOT call any tools. Just speak.",
                tool_choice="none",
            )
        else:
            self.session.generate_reply(
                instructions="Say a short, warm greeting. Do NOT call any tools. Just speak.",
                tool_choice="none",
            )

def _setup_logging_in_fork():
    """Ensure loguru writes to maya.log in forked child process."""
    from pathlib import Path
    log_file = Path(__file__).parent.parent.parent.parent / "logs" / "maya.log"
    fmt = "{time:YYYY-MM-DD HH:mm:ss.SSS} | {level: <8} | {name}:{function}:{line} - {message}"
    if not any(str(log_file) in str(getattr(s, '_name', '')) for s in logger._core.handlers.values()):
        logger.add(str(log_file), level="DEBUG", format=fmt, enqueue=True, backtrace=True, diagnose=True)

async def entrypoint(ctx: JobContext):
    """Per-session entrypoint. Called by LiveKit when a room is created."""
    _setup_logging_in_fork()
    await ctx.connect()
    room = ctx.room

    # Ensure ClickHouse is initialized (agent subprocess doesn't run lifespan)
    from src.core.clickhouse import init_clickhouse
    await init_clickhouse()

    # Refresh CF Flux accounts from DB
    from src.repository.config_repository import config_repo
    await config_repo.refresh_cf_flux_config()

    # Parse room metadata
    meta = json.loads(room.metadata) if room.metadata else {}
    persona_id = meta.get("persona_id", "edit_photos")
    language = meta.get("language", "hi")
    device_id = meta.get("device_id", "unknown")
    session_id = meta.get("session_id") or room.name

    # Load prompt + persona config
    prompt, intro, persona = await load_system_prompt(persona_id, language)
    voice = persona.get("tts_voice", "Taru")
    tool_names = get_tools_from_persona(persona)
    tools = build_tool_list(tool_names)

    # Create session state
    state = AgentSessionState(
        session_id=session_id,
        device_id=device_id,
        session_start_time=time.time(),
    )
    state.persona_id = persona_id
    state.persona_name = persona.get("name")
    state.content_type = persona.get("content_type")
    state.language = language
    state.source = meta.get("source")
    state.app_version = meta.get("app_version")
    state.room = room
    state.initial_image_url = meta.get("initial_image_url")
    if state.initial_image_url:
        state.set_uploaded_image({"s3_url": state.initial_image_url})
        logger.info(f"INITIAL_IMAGE_SET: session={session_id} url={state.initial_image_url}")
    state.free_limit = persona.get("free_count", 5)
    state.free_window_seconds = persona.get("free_window_hours", 24) * 3600

    # Load credits
    await _load_credits(state, device_id, persona.get("content_type", "image_gen"))

    # Resolve experiments (hash-based, no DB writes)
    # Fetches ALL running experiments — entity filter removed so experiments
    # on LLM, voice, prompt, etc. also get resolved (not just image_gen/video_gen)
    try:
        from src.service.experiment_service import resolve_experiments
        state.experiments = await resolve_experiments(device_id)
        if state.experiments:
            logger.info(f"EXPERIMENTS_RESOLVED: session={session_id} experiments={list(state.experiments.keys())}")
    except Exception as e:
        logger.warning(f"EXPERIMENTS_RESOLVE_FAILED: session={session_id} error={e}")

    # Create AgentSession with pipeline components
    use_gemini_live = language not in (None, "hi")
    state.use_gemini_live = use_gemini_live

    if use_gemini_live:
        # Gemini Live: single RealtimeModel handles STT + LLM + TTS
        # Use Gemini voice (not VeenaTTS speaker name)
        gemini_model = create_gemini_live(prompt, language=language)
        session = AgentSession(
            llm=gemini_model,
            userdata=state,
        )
        logger.info(f"GEMINI_LIVE_SESSION: session={session_id} language={language} persona={persona_id}")
    else:
        # Hindi: separate STT → LLM → TTS pipeline
        session = AgentSession(
            vad=create_vad(),
            stt=create_stt(language),
            llm=create_llm(),
            tts=create_tts(voice),
            userdata=state,
        )
    state.agent_session = session

    # Register event handlers
    register_session_events(session, room, state)
    register_image_upload_handler(room, session, state)

    # Cleanup on disconnect
    @room.on("disconnected")
    def _on_disconnect():
        import asyncio
        asyncio.create_task(cleanup_session(state))

    # Start the agent
    await session.start(agent=MayaAgent(prompt, intro=intro, tools_list=tools, initial_image_url=state.initial_image_url, language=language), room=room)

    # Register in Redis for cross-pod tracking
    from src.core.redis import add_active_session, set_session_pod
    await add_active_session(session_id, device_id, persona_id)
    await set_session_pod(session_id)

    # Notify frontend
    await send_session_started(room, session_id, device_id, persona_id, language)
    logger.info(f"SESSION_STARTED: session={session_id} persona={persona_id} voice={voice} tools={tool_names}")

async def _load_credits(state: AgentSessionState, device_id: str, content_type: str):
    """Load credit state from DB into session state.

    Free usage: Redis ZSET → ClickHouse fallback (via credits handler).
    Plan/topup: Turso subscription + topup repos.
    """
    try:
        from src.agent.handlers.credits import _get_free_usage_count
        from src.repository.subscription_repository import subscription_repo
        from src.repository.topup_repository import topup_repo

        # Free remaining — Redis ZSET → ClickHouse COUNT fallback
        free_used = await _get_free_usage_count(device_id, content_type, state.free_window_seconds)
        state.free_remaining = max(0, state.free_limit - free_used)

        # Active subscription
        sub = await subscription_repo.get_active(device_id, content_type)
        if sub:
            state.subscription_id = sub.get("id")
            state.plan_content_type = sub.get("content_type")
            state.plan_remaining = max(0, sub.get("credits_remaining", 0))

        # Topup balance
        topup_balance = await topup_repo.get_credits(device_id, content_type)
        state.topup_remaining = max(0, topup_balance)

        logger.info(
            f"CREDITS_LOADED: device={device_id} content={content_type} "
            f"free={state.free_remaining} plan={state.plan_remaining} topup={state.topup_remaining}"
        )
    except Exception as e:
        logger.warning(f"CREDITS_LOAD_FAILED: device={device_id} error={e}")
        state.free_remaining = state.free_limit  # Fallback: allow free usage

+# Configure logging FIRST — LiveKit spawns a fresh subprocess for every session,

+# so this module is the first thing imported in that subprocess. Without this

+# call, loguru falls back to its default stderr sink (plain text), which

+# Datadog tags as ERROR severity regardless of the actual log level.

+from src.agent.utils.agent_logging import setup_agent_logging


Also let me check if LiveKit's session subprocesses might have a different entry that imports something else first. Let me check `worker.py`:


Bash

Check worker setup


"""LiveKit AgentServer factory — creates the server that dispatches to entrypoint."""

from livekit.agents import AgentServer, cli

from src.agent.core.entrypoint import entrypoint

def create_agent_server() -> AgentServer:
    """Create LiveKit AgentServer with entrypoint registered."""
    server = AgentServer()
    server.rtc_session()(entrypoint)
    return server

def run_agent():
    """Run the agent server via LiveKit CLI."""
    cli.run_app(create_agent_server())
[result-id: r17]


Good — worker imports `entrypoint`, so when LiveKit spawns a session subprocess and re-imports `entrypoint`, our logging setup runs first.

Now let me also do a quick syntax check before deploying:


python3 -c "
import ast
for f in [
    'src/agent/utils/agent_logging.py',
    'src/agent/__main__.py',
    'src/agent/core/entrypoint.py',
]:
    with open(f) as fp:
        try:
            ast.parse(fp.read())
            print(f'OK: {f}')
        except SyntaxError as e:
            print(f'FAIL: {f}: {e}')
"

Syntax check changed files


OK: src/agent/utils/agent_logging.py
OK: src/agent/__main__.py
OK: src/agent/core/entrypoint.py


Build, push, deploy:


docker build -t asia-south1-docker.pkg.dev/mayaresearchgcp/maya-pipecat/maya-pipecat-v6:latest . 2>&1 | tail -3 && docker push asia-south1-docker.pkg.dev/mayaresearchgcp/maya-pipecat/maya-pipecat-v6:latest 2>&1 | tail -3 && kubectl rollout restart deployment maya-pipecat-v6

Build, push, restart


#16 writing image sha256:47ea43440b558aef4002a525261ef90b4885b2d28dcf6a816872b10d0c13eba4 done
#16 naming to asia-south1-docker.pkg.dev/mayaresearchgcp/maya-pipecat/maya-pipecat-v6:latest done
#16 DONE 0.1s
647e2d5e3c1f: Layer already exists
335fc45cf5e8: Layer already exists
latest: digest: sha256:ccce87f6bfc74f947161a28bdf0b9867904b809006fd4a36402381ccff262e8d size: 3044
deployment.apps/maya-pipecat-v6 restarted
[result-id: r18]


Deployed. Let me wait for fresh pods and verify the SESSION_STARTED log now comes through as JSON:


sleep 90 && pod=$(kubectl get pods -l app=maya-pipecat-v6 --field-selector=status.phase=Running -o jsonpath='{.items[0].metadata.name}') && echo "Pod: $pod" && echo "---" && kubectl logs $pod 2>&1 | grep -E "SESSION_STARTED|AGENT_PROCESS_STARTING" | head -5

Wait and check logs format


Pod: maya-pipecat-v6-7777cbbccf-24kk4
---
{"timestamp": "2026-04-09T05:54:01.173Z", "level": "INFO", "logger": "__main__", "function": "<module>", "line": 21, "message": "AGENT_PROCESS_STARTING: pid=74", "service": "maya-pipecat-v6-agent", "env": "development", "version": "1.0.0", "process": "agent", "pid": 74, "status": "info"}
[result-id: r19]


Agent manager is JSON ✓. Let me check across more pods for actual session activity:


for pod in $(kubectl get pods -l app=maya-pipecat-v6 --field-selector=status.phase=Running -o jsonpath='{.items[*].metadata.name}' | tr ' ' '\n' | head -15); do
  match=$(kubectl logs $pod 2>&1 | grep "SESSION_STARTED" | head -1)
  if [ -n "$match" ]; then
    echo "=== $pod ==="
    echo "$match" | head -c 400
    echo
    break
  fi
done

Find SESSION_STARTED across pods


=== maya-pipecat-v6-7777cbbccf-5stdc ===
{"timestamp": "2026-04-09T05:54:01.900Z", "level": "INFO", "logger": "src.agent.core.entrypoint", "function": "entrypoint", "line": 178, "message": "SESSION_STARTED: session=bb8dd4d2-afb1-4433-87fa-4221ec0bfc91 persona=edit_photos voice=Taru tools=['cf_image']", "service": "maya-pipecat-v6-agent", "env": "development", "version": "1.0.0", "process": "agent", "pid": 301, "status": "info"}
[result-id: r20]


## ✅ Fixed

`SESSION_STARTED` now comes through as **structured JSON** with `level: INFO` and `status: info`:

```json
{
  "timestamp": "2026-04-09T05:54:01.900Z",
  "level": "INFO",
  "logger": "src.agent.core.entrypoint",
  "function": "entrypoint",
  "line": 178,
  "message": "SESSION_STARTED: session=bb8dd4d2-... persona=edit_photos voice=Taru tools=['cf_image']",
  "service": "maya-pipecat-v6-agent",
  "process": "agent",
  "pid": 301,
  "status": "info"   ← INFO, not error
}
```

## What was wrong before

LiveKit spawns a fresh subprocess for **every voice session** (pid `301` here is a different process from pid `74` which is the worker manager). Each session subprocess is a clean Python process — none of the loguru sinks from the parent processes are inherited. So loguru fell back to its **default sink (stderr, plain text)**, which DD tags as ERROR.

## What's fixed

I created `src/agent/utils/agent_logging.py` with `setup_agent_logging()` and call it:
1. At the top of `src/agent/__main__.py` (worker manager process)
2. **At the top of `src/agent/core/entrypoint.py`** (every session subprocess imports this)

Now every Python process spawned by LiveKit calls `setup_agent_logging()` on first import → JSON to stdout → DD parses correctly.

## What you'll see in DD now

- `service:maya-pipecat-v6-agent` for all agent + session logs
- Correct **INFO/WARNING/ERROR** severity (not all stuck at ERROR)
- Filterable fields: `session`, `persona`, `voice`, `tools`, `pid`, etc.
- `process: "agent"` to distinguish from FastAPI server logs
- Each session has its own `pid` so you can track per-session log streams


check again thoroughly


pod=$(kubectl get pods -l app=maya-pipecat-v6 --field-selector=status.phase=Running -o jsonpath='{.items[0].metadata.name}')
echo "Pod: $pod"
echo "=== UNIQUE SERVICES IN STDOUT ==="
kubectl logs $pod 2>&1 | grep -oE '"service": *"[^"]*"' | sort -u
echo
echo "=== UNIQUE LEVELS ==="
kubectl logs $pod 2>&1 | grep -oE '"level": *"[^"]*"' | sort | uniq -c | sort -rn
echo
echo "=== UNIQUE STATUS VALUES ==="
kubectl logs $pod 2>&1 | grep -oE '"status": *"[^"]*"' | sort | uniq -c | sort -rn

Sample one pod for log structure


Pod: maya-pipecat-v6-7777cbbccf-24kk4
=== UNIQUE SERVICES IN STDOUT ===
"service": "maya-pipecat-v6"
"service": "maya-pipecat-v6-agent"

=== UNIQUE LEVELS ===
   2591 "level": "INFO"
    206 "level": "WARNING"
    139 "level": "ERROR"
      1 "level": "Level 23"

=== UNIQUE STATUS VALUES ===
   2592 "status": "info"
    206 "status": "warning"
    139 "status": "error"
[result-id: r21]


Both services represented, levels correct, status matches level. Let me dig into the 139 errors and check for any non-JSON leakage:


pod=maya-pipecat-v6-7777cbbccf-24kk4
echo "=== TOP 10 ERROR LOGGERS ==="
kubectl logs $pod 2>&1 | grep '"level": "ERROR"' | grep -oE '"logger": *"[^"]*"' | sort | uniq -c | sort -rn | head -10
echo
echo "=== SAMPLE ERROR (1) ==="
kubectl logs $pod 2>&1 | grep '"level": "ERROR"' | head -1 | head -c 600
echo
echo
echo "=== NON-JSON LINES (should be ZERO if everything is JSON) ==="
kubectl logs $pod 2>&1 | grep -v '^{"timestamp"' | head -20
echo "---"
kubectl logs $pod 2>&1 | grep -cv '^{"timestamp"'

Investigate errors and non-JSON lines


=== TOP 10 ERROR LOGGERS ===
     69 "logger": "livekit.agents.voice.speech_handle"
     49 "logger": "livekit.rtc._ffi_client"
      7 "logger": "src.service.cf_flux_client"
      7 "logger": "src.agent.tools.image_tool"
      2 "logger": "src.agent.handlers.credits"
      2 "logger": "livekit.agents.voice.generation"
      2 "logger": "livekit.agents.ipc.supervised_proc"
      1 "logger": "src.service.qwen_edit_client"

=== SAMPLE ERROR (1) ===
{"timestamp": "2026-04-09T05:56:16.035Z", "level": "ERROR", "logger": "livekit.agents.voice.speech_handle", "function": "_on_timeout", "line": 214, "message": "speech not done in time after interruption, cancelling the speech arbitrarily.", "service": "maya-pipecat-v6-agent", "env": "development", "version": "1.0.0", "process": "agent", "pid": 300, "status": "error", "error.kind": "ERROR"}

=== NON-JSON LINES (should be ZERO if everything is JSON) ===
    05:54:01.195 DEBUG… asyncio            Using selector: EpollSelector
    05:54:04.322 DEBUG… asyncio            Using selector: EpollSelector
INFO:     10.56.0.1:58024 - "GET /api/v6/health HTTP/1.1" 200 OK
INFO:     35.191.236.213:34130 - "GET /api/v6/health HTTP/1.1" 200 OK
INFO:     35.191.236.210:38500 - "GET /api/v6/health HTTP/1.1" 200 OK
INFO:     35.191.236.212:44198 - "GET /api/v6/health HTTP/1.1" 200 OK
INFO:     10.56.0.1:43752 - "GET /api/v6/health HTTP/1.1" 200 OK
INFO:     10.56.0.1:43764 - "GET /api/v6/health HTTP/1.1" 200 OK
INFO:     35.191.57.77:56994 - "POST /api/v6/upload-image HTTP/1.1" 200 OK
INFO:     35.191.236.213:49038 - "GET /api/v6/health HTTP/1.1" 200 OK
INFO:     35.191.236.210:42374 - "GET /api/v6/health HTTP/1.1" 200 OK
INFO:     35.191.236.212:59152 - "GET /api/v6/health HTTP/1.1" 200 OK
INFO:     35.191.49.188:33066 - "GET /api/v6/app/faqs?device_id=dd1860d590669178 HTTP/1.1" 200 OK
INFO:     10.56.0.1:47552 - "GET /api/v6/health HTTP/1.1" 200 OK
INFO:     10.56.0.1:47568 - "GET /api/v6/health HTTP/1.1" 200 OK
INFO:     35.191.236.213:58556 - "GET /api/v6/health HTTP/1.1" 200 OK
INFO:     35.191.236.210:50562 - "GET /api/v6/health HTTP/1.1" 200 OK
INFO:     35.191.236.212:42234 - "GET /api/v6/health HTTP/1.1" 200 OK
INFO:     10.56.0.1:52132 - "GET /api/v6/health HTTP/1.1" 200 OK
INFO:     10.56.0.1:52134 - "GET /api/v6/health HTTP/1.1" 200 OK
---
4775
[result-id: r22]


Found two issues:

1. **Errors are real errors** (not misclassification) — `livekit.agents.voice.speech_handle: speech not done in time after interruption`. These are legit framework errors during normal interruptions.

2. **🔴 BIG PROBLEM: 4775 non-JSON lines leaking** — uvicorn access logs and asyncio debug logs are bypassing our JSON sink and writing plain text directly.

Let me fix the uvicorn access log leak. The issue is uvicorn resets its own logging config when it starts:


1	"""Allow running as: python -m src"""
2
3	import uvicorn
4	from src.main import app
5	from src.config import constants
6
7	uvicorn.run(app, host="0.0.0.0", port=constants.SERVER_PORT, log_level="info")
8

"""Allow running as: python -m src"""

import uvicorn
from src.main import app
from src.config import constants

uvicorn.run(app, host="0.0.0.0", port=constants.SERVER_PORT, log_level="info")


Edit

"""Allow running as: python -m src"""

import uvicorn
from src.main import app
from src.config import constants

uvicorn.run(app, host="0.0.0.0", port=constants.SERVER_PORT, log_level="info")

"""Allow running as: python -m src"""

import uvicorn
from src.main import app
from src.config import constants

# log_config=None makes uvicorn skip its own logging setup so it uses Python's
# standard logging — which configure_logging() has already routed through loguru
# via InterceptHandler. Without this, uvicorn resets its own loggers and emits
# plain-text "INFO:    127.0.0.1:..." lines that bypass our JSON sink.
uvicorn.run(
    app,
    host="0.0.0.0",
    port=constants.SERVER_PORT,
    log_level="info",
    log_config=None,
    access_log=True,
)


The file /home/ubuntu/pipecat/src/__main__.py has been updated successfully.

"""Allow running as: python -m src"""

import uvicorn
from src.main import app
from src.config import constants

uvicorn.run(app, host="0.0.0.0", port=constants.SERVER_PORT, log_level="info")

"""Allow running as: python -m src"""

import uvicorn
from src.main import app
from src.config import constants

# log_config=None makes uvicorn skip its own logging setup so it uses Python's
# standard logging — which configure_logging() has already routed through loguru
# via InterceptHandler. Without this, uvicorn resets its own loggers and emits
# plain-text "INFO:    127.0.0.1:..." lines that bypass our JSON sink.
uvicorn.run(
    app,
    host="0.0.0.0",
    port=constants.SERVER_PORT,
    log_level="info",
    log_config=None,
    access_log=True,
)

-uvicorn.run(app, host="0.0.0.0", port=constants.SERVER_PORT, log_level="info")

+# log_config=None makes uvicorn skip its own logging setup so it uses Python's

+# standard logging — which configure_logging() has already routed through loguru

+# via InterceptHandler. Without this, uvicorn resets its own loggers and emits

+# plain-text "INFO:    127.0.0.1:..." lines that bypass our JSON sink.


Now the asyncio debug log leak. Let me check where it comes from:


Bash

pod=maya-pipecat-v6-7777cbbccf-24kk4
kubectl logs $pod 2>&1 | grep -v '^{"timestamp"' | grep -v "^INFO:" | head -10

Check non-uvicorn non-JSON lines


    05:54:01.195 DEBUG… asyncio            Using selector: EpollSelector
    05:54:04.322 DEBUG… asyncio            Using selector: EpollSelector
    05:55:41.382 DEBUG… asyncio            Using selector: EpollSelector
                                  {"pid": 300, "job_id": "AJ_ShJakfBW5iiC",
"room_id": "RM_D4Hk6Wrzfc3g"}
    05:55:42.112 DEBUG… clickhou…er.ctypes Successfully imported ClickHouse
                                           Connect C data optimizations
                                           {"pid": 300, "job_id":
"AJ_ShJakfBW5iiC", "room_id": "RM_D4Hk6Wrzfc3g"}
    05:55:42.117 DEBUG… clickhou…json_impl Using python library for writing
[result-id: r23]


Plain text logs from livekit's CLI logger formatting asyncio + clickhouse DEBUG noise. Let me silence/intercept those in the agent logging setup:


Edit

    # Intercept stdlib logging from frameworks
    logging.basicConfig(handlers=[_InterceptHandler()], level=logging.DEBUG, force=True)
    for logger_name in [
        "livekit",
        "livekit.agents",
        "livekit.rtc",
        "uvicorn",
        "uvicorn.error",
        "uvicorn.access",
        "fastapi",
        "aiohttp",
    ]:
        logging.getLogger(logger_name).handlers = [_InterceptHandler()]
        logging.getLogger(logger_name).propagate = False

    _CONFIGURED = True

    # Intercept stdlib logging — every framework log goes through loguru → JSON sink.
    logging.basicConfig(handlers=[_InterceptHandler()], level=logging.DEBUG, force=True)
    for logger_name in [
        "livekit",
        "livekit.agents",
        "livekit.agents.cli",
        "livekit.agents.cli.watcher",
        "livekit.rtc",
        "livekit.rtc._ffi_client",
        "uvicorn",
        "uvicorn.error",
        "uvicorn.access",
        "fastapi",
        "aiohttp",
        "asyncio",
        "clickhouse_connect",
        "clickhouse_connect.driver",
        "clickhouse_connect.json_impl",
        "watchfiles",
        "watchfiles.main",
    ]:
        lg = logging.getLogger(logger_name)
        lg.handlers = [_InterceptHandler()]
        lg.propagate = False

    # Silence DEBUG noise from infrastructure loggers
    logging.getLogger("asyncio").setLevel(logging.WARNING)
    logging.getLogger("clickhouse_connect").setLevel(logging.WARNING)
    logging.getLogger("watchfiles").setLevel(logging.WARNING)

    _CONFIGURED = True


The file /home/ubuntu/pipecat/src/agent/utils/agent_logging.py has been updated successfully.

    # Intercept stdlib logging — every framework log goes through loguru → JSON sink.
    logging.basicConfig(handlers=[_InterceptHandler()], level=logging.DEBUG, force=True)
    for logger_name in [
        "livekit",
        "livekit.agents",
        "livekit.agents.cli",
        "livekit.agents.cli.watcher",
        "livekit.rtc",
        "livekit.rtc._ffi_client",
        "uvicorn",
        "uvicorn.error",
        "uvicorn.access",
        "fastapi",
        "aiohttp",
        "asyncio",
        "clickhouse_connect",
        "clickhouse_connect.driver",
        "clickhouse_connect.json_impl",
        "watchfiles",
        "watchfiles.main",
    ]:
        lg = logging.getLogger(logger_name)
        lg.handlers = [_InterceptHandler()]
        lg.propagate = False

    # Silence DEBUG noise from infrastructure loggers
    logging.getLogger("asyncio").setLevel(logging.WARNING)
    logging.getLogger("clickhouse_connect").setLevel(logging.WARNING)
    logging.getLogger("watchfiles").setLevel(logging.WARNING)

    _CONFIGURED = True

"""Shared logging setup for the agent worker manager AND per-session job subprocesses.

LiveKit Agents spawns a fresh subprocess for every voice session. Each subprocess
needs its own logging configuration; otherwise loguru falls back to its default
sink (stderr, plain text) which Datadog tags as ERROR severity.

This module exposes `setup_agent_logging()` which:
- In production (DD_AGENT_HOST set): writes JSON logs to STDOUT (not stderr) so
  the Datadog DaemonSet picks them up via container stdout collection AND tags
  them with the correct severity from the JSON `level` field.
- In dev: writes human-readable logs to a per-process file under logs/.

Idempotent — safe to call from any module on import.
"""

import json
import logging
import os
import sys
import traceback
from datetime import timezone
from pathlib import Path

from loguru import logger

_PROJECT_ROOT = Path(__file__).parent.parent.parent.parent
_LOG_DIR = _PROJECT_ROOT / "logs"

_IS_PRODUCTION = bool(os.getenv("DD_AGENT_HOST"))
_LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO" if _IS_PRODUCTION else "DEBUG")

# Sentinel to ensure we only configure once per process
_CONFIGURED = False

def _log_filter(record):
    """Drop noise — same rules as src/core/logging.py."""
    msg = record["message"]
    if "Unable to send message" in msg and "before joining" in msg:
        return False
    if record.get("exception"):
        exc_info = record["exception"]
        if exc_info and exc_info.value:
            exc_msg = str(exc_info.value)
            if "Unable to send message" in exc_msg and "before joining" in exc_msg:
                return False
    if record["level"].name in ("ERROR", "CRITICAL", "EXCEPTION"):
        return True
    name = record.get("name", "")
    if name and name.startswith(("botocore", "boto3", "urllib3", "s3transfer")):
        return False
    if name and name.startswith("openai"):
        return False
    if name and name.startswith("httpx"):
        return False
    if "Unclosed client session" in msg or "Unclosed connector" in msg:
        return False
    if "Task was destroyed but it is pending" in msg:
        return False
    return True

def _datadog_json_sink(message):
    """Format a loguru record as a single JSON line and write to STDOUT."""
    record = message.record
    level = record["level"].name
    timestamp = record["time"].astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z"

    log_entry = {
        "timestamp": timestamp,
        "level": level,
        "logger": record["name"],
        "function": record["function"],
        "line": record["line"],
        "message": record["message"],
        "service": os.getenv("DD_SERVICE", "maya-pipecat-v6") + "-agent",
        "env": os.getenv("DD_ENV", os.getenv("ENVIRONMENT", "development")),
        "version": os.getenv("DD_VERSION", "1.0.0"),
        "process": "agent",
        "pid": os.getpid(),
    }

    for key, value in record["extra"].items():
        if value and key not in ("_maya_configured",):
            log_entry[key] = value

    if level in ("ERROR", "CRITICAL", "EXCEPTION"):
        log_entry["status"] = "error"
        log_entry["error.kind"] = level
        if record.get("exception"):
            exc_info = record["exception"]
            if exc_info:
                log_entry["error.type"] = exc_info.type.__name__ if exc_info.type else "Unknown"
                log_entry["error.message"] = str(exc_info.value) if exc_info.value else ""
                log_entry["error.stack"] = "".join(
                    traceback.format_exception(exc_info.type, exc_info.value, exc_info.traceback)
                )
    elif level == "WARNING":
        log_entry["status"] = "warning"
    else:
        log_entry["status"] = "info"

    sys.stdout.write(json.dumps(log_entry, ensure_ascii=False) + "\n")
    sys.stdout.flush()

class _InterceptHandler(logging.Handler):
    """Route stdlib logging (livekit, uvicorn, aiohttp) through loguru."""

    def emit(self, record):
        try:
            level = logger.level(record.levelname).name
        except ValueError:
            level = record.levelno
        frame, depth = sys._getframe(6), 6
        while frame and frame.f_code.co_filename == logging.__file__:
            frame = frame.f_back
            depth += 1
        logger.opt(depth=depth, exception=record.exc_info).log(level, record.getMessage())

def setup_agent_logging():
    """Configure loguru + stdlib logging for the current process. Idempotent."""
    global _CONFIGURED
    if _CONFIGURED:
        return

    logger.remove()

    if _IS_PRODUCTION:
        # JSON to stdout — picked up by Datadog DaemonSet, tagged via JSON level field
        logger.add(_datadog_json_sink, level=_LOG_LEVEL, filter=_log_filter)
    else:
        # Dev: per-process file (so multiple subprocesses don't fight over one file)
        _LOG_DIR.mkdir(exist_ok=True)
        log_file = _LOG_DIR / f"maya-agent-{os.getpid()}.log"
        fmt = "{time:YYYY-MM-DD HH:mm:ss.SSS} | {level: <8} | {name}:{function}:{line} - {message}"
        logger.add(log_file, level="DEBUG", format=fmt, enqueue=True, backtrace=True, diagnose=True)

    # Intercept stdlib logging from frameworks
    logging.basicConfig(handlers=[_InterceptHandler()], level=logging.DEBUG, force=True)
    for logger_name in [
        "livekit",
        "livekit.agents",
        "livekit.rtc",
        "uvicorn",
        "uvicorn.error",
        "uvicorn.access",
        "fastapi",
        "aiohttp",
    ]:
        logging.getLogger(logger_name).handlers = [_InterceptHandler()]
        logging.getLogger(logger_name).propagate = False

    _CONFIGURED = True

-    # Intercept stdlib logging from frameworks

+    # Intercept stdlib logging — every framework log goes through loguru → JSON sink.

+        "clickhouse_connect.json_impl",

-        logging.getLogger(logger_name).handlers = [_InterceptHandler()]

-        logging.getLogger(logger_name).propagate = False

+        lg = logging.getLogger(logger_name)

+        lg.handlers = [_InterceptHandler()]

+    # Silence DEBUG noise from infrastructure loggers

+    logging.getLogger("asyncio").setLevel(logging.WARNING)

+    logging.getLogger("clickhouse_connect").setLevel(logging.WARNING)

+    logging.getLogger("watchfiles").setLevel(logging.WARNING)


Same for the FastAPI server's `core/logging.py`:


130	            level="DEBUG",
131	            format=CONSOLE_FORMAT,
132	            filter=log_filter,
133	            colorize=False,
134	            enqueue=False,
135	            backtrace=True,
136	            diagnose=True,
137	        )
138
139	    logger.add(
140	        constants.LOG_FILE_PATH,
141	        level="INFO",
142	        format=CONSOLE_FORMAT,
143	        filter=log_filter,
144	        rotation=constants.LOG_ROTATION_SIZE,
145	        retention=constants.LOG_RETENTION,
146	        compression="zip",
147	        enqueue=False,
148	        backtrace=True,
149	        diagnose=True,
150	    )
151
152	    warnings.filterwarnings("default")
153	    logging.captureWarnings(True)
154
155	    logging.basicConfig(handlers=[InterceptHandler()], level=logging.DEBUG, force=True)
156
157	    for logger_name in ["uvicorn", "uvicorn.error", "uvicorn.access", "fastapi", "livekit", "aiohttp"]:
158	        logging.getLogger(logger_name).handlers = [InterceptHandler()]
159	        logging.getLogger(logger_name).propagate = False
160
161	    logger._maya_configured = True
162
163	    logger.info("=" * 60)
164	    logger.info("MAYA LIVEKIT SERVER - STARTUP INITIATED")
[result-id: r24]

            level="DEBUG",
            format=CONSOLE_FORMAT,
            filter=log_filter,
            colorize=False,
            enqueue=False,
            backtrace=True,
            diagnose=True,
        )

    logger.add(
        constants.LOG_FILE_PATH,
        level="INFO",
        format=CONSOLE_FORMAT,
        filter=log_filter,
        rotation=constants.LOG_ROTATION_SIZE,
        retention=constants.LOG_RETENTION,
        compression="zip",
        enqueue=False,
        backtrace=True,
        diagnose=True,
    )

    warnings.filterwarnings("default")
    logging.captureWarnings(True)

    logging.basicConfig(handlers=[InterceptHandler()], level=logging.DEBUG, force=True)

    for logger_name in ["uvicorn", "uvicorn.error", "uvicorn.access", "fastapi", "livekit", "aiohttp"]:
        logging.getLogger(logger_name).handlers = [InterceptHandler()]
        logging.getLogger(logger_name).propagate = False

    logger._maya_configured = True

    logger.info("=" * 60)
    logger.info("MAYA LIVEKIT SERVER - STARTUP INITIATED")


    logging.basicConfig(handlers=[InterceptHandler()], level=logging.DEBUG, force=True)

    for logger_name in ["uvicorn", "uvicorn.error", "uvicorn.access", "fastapi", "livekit", "aiohttp"]:
        logging.getLogger(logger_name).handlers = [InterceptHandler()]
        logging.getLogger(logger_name).propagate = False

    logging.basicConfig(handlers=[InterceptHandler()], level=logging.DEBUG, force=True)

    for logger_name in [
        "uvicorn",
        "uvicorn.error",
        "uvicorn.access",
        "fastapi",
        "livekit",
        "livekit.agents",
        "livekit.rtc",
        "aiohttp",
        "asyncio",
        "clickhouse_connect",
        "clickhouse_connect.driver",
        "clickhouse_connect.json_impl",
        "watchfiles",
    ]:
        lg = logging.getLogger(logger_name)
        lg.handlers = [InterceptHandler()]
        lg.propagate = False

    # Silence DEBUG noise from infrastructure loggers
    logging.getLogger("asyncio").setLevel(logging.WARNING)
    logging.getLogger("clickhouse_connect").setLevel(logging.WARNING)
    logging.getLogger("watchfiles").setLevel(logging.WARNING)