Write

apiVersion: apps/v1
kind: Deployment
metadata:
  name: maya-pipecat-v6
  labels:
    app: maya-pipecat-v6
spec:
  replicas: 10
  strategy:
    type: RollingUpdate
    rollingUpdate:
      maxSurge: 25%
      maxUnavailable: 0
  selector:
    matchLabels:
      app: maya-pipecat-v6
  template:
    metadata:
      labels:
        app: maya-pipecat-v6
      annotations:
        ad.datadoghq.com/maya-pipecat-v6.logs: |
          [{
            "source": "maya-pipecat-v6",
            "service": "maya-pipecat-v6",
            "log_processing_rules": [
              {
                "type": "exclude_at_match",
                "name": "exclude_noise",
                "pattern": "Failed to send LogLine|ResponseCanceled|Metrics failed to get snapshot|epoll_wait: Invalid argument|epoll_wait: Bad file descriptor|TrySendError|Disconnected|took longer than 2s"
              }
            ]
          }]
    spec:
      terminationGracePeriodSeconds: 90
      imagePullSecrets:
      - name: ar-pull-secret
      containers:
      - name: maya-pipecat-v6
        image: asia-south1-docker.pkg.dev/mayaresearchgcp/maya-pipecat/maya-pipecat-v6:latest
        imagePullPolicy: Always
        lifecycle:
          preStop:
            exec:
              command: ["/bin/sh", "-c", "sleep 15"]
        ports:
        - containerPort: 3006
        resources:
          requests:
            memory: "3Gi"
            cpu: "2000m"
          limits:
            memory: "12Gi"
            cpu: "8000m"
        livenessProbe:
          httpGet:
            path: /api/v6/health
            port: 3006
          initialDelaySeconds: 30
          periodSeconds: 15
          timeoutSeconds: 15
        readinessProbe:
          httpGet:
            path: /api/v6/health
            port: 3006
          initialDelaySeconds: 15
          periodSeconds: 15
          timeoutSeconds: 10
          failureThreshold: 3
          successThreshold: 1
        envFrom:
        - secretRef:
            name: maya-env-secrets
        env:
        - name: DD_AGENT_HOST
          valueFrom:
            fieldRef:
              fieldPath: status.hostIP
        - name: DD_SERVICE
          value: "maya-pipecat-v6"
        - name: POD_NAME
          valueFrom:
            fieldRef:
              fieldPath: metadata.name
        - name: POD_IP
          valueFrom:
            fieldRef:
              fieldPath: status.podIP
        volumeMounts:
        - name: gcp-creds
          mountPath: /app/gcp-service-account.json
          subPath: gcp-service-account.json
          readOnly: true
        - name: gcp-creds
          mountPath: /app/gcp-new-service-account.json
          subPath: gcp-new-service-account.json
          readOnly: true
        - name: google-play-creds
          mountPath: /app/google-play-service-account.json
          subPath: google-play-service-account.json
          readOnly: true
      volumes:
      - name: gcp-creds
        secret:
          secretName: gcp-credentials
      - name: google-play-creds
        secret:
          secretName: google-play-credentials
---
apiVersion: cloud.google.com/v1
kind: BackendConfig
metadata:
  name: maya-pipecat-v6-backendconfig
spec:
  sessionAffinity:
    affinityType: "CLIENT_IP"
    affinityCookieTtlSec: 300
---
apiVersion: v1
kind: Service
metadata:
  name: maya-pipecat-v6
  labels:
    app: maya-pipecat-v6
  annotations:
    cloud.google.com/backend-config: '{"default": "maya-pipecat-v6-backendconfig"}'
spec:
  type: NodePort
  sessionAffinity: ClientIP
  sessionAffinityConfig:
    clientIP:
      timeoutSeconds: 300
  ports:
  - port: 80
    targetPort: 3006
    protocol: TCP
  selector:
    app: maya-pipecat-v6


File created successfully at: /home/ubuntu/pipecat/deployments/k8s/gke/deployment-v6.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  name: maya-pipecat-v6
  labels:
    app: maya-pipecat-v6
spec:
  replicas: 10
  strategy:
    type: RollingUpdate
    rollingUpdate:
      maxSurge: 25%
      maxUnavailable: 0
  selector:
    matchLabels:
      app: maya-pipecat-v6
  template:
    metadata:
      labels:
        app: maya-pipecat-v6
      annotations:
        ad.datadoghq.com/maya-pipecat-v6.logs: |
          [{
            "source": "maya-pipecat-v6",
            "service": "maya-pipecat-v6",
            "log_processing_rules": [
              {
                "type": "exclude_at_match",
                "name": "exclude_noise",
                "pattern": "Failed to send LogLine|ResponseCanceled|Metrics failed to get snapshot|epoll_wait: Invalid argument|epoll_wait: Bad file descriptor|TrySendError|Disconnected|took longer than 2s"
              }
            ]
          }]
    spec:
      terminationGracePeriodSeconds: 90
      imagePullSecrets:
      - name: ar-pull-secret
      containers:
      - name: maya-pipecat-v6
        image: asia-south1-docker.pkg.dev/mayaresearchgcp/maya-pipecat/maya-pipecat-v6:latest
        imagePullPolicy: Always
        lifecycle:
          preStop:
            exec:
              command: ["/bin/sh", "-c", "sleep 15"]
        ports:
        - containerPort: 3006
        resources:
          requests:
            memory: "3Gi"
            cpu: "2000m"
          limits:
            memory: "12Gi"
            cpu: "8000m"
        livenessProbe:
          httpGet:
            path: /api/v6/health
            port: 3006
          initialDelaySeconds: 30
          periodSeconds: 15
          timeoutSeconds: 15
        readinessProbe:
          httpGet:
            path: /api/v6/health
            port: 3006
          initialDelaySeconds: 15
          periodSeconds: 15
          timeoutSeconds: 10
          failureThreshold: 3
          successThreshold: 1
        envFrom:
        - secretRef:
            name: maya-env-secrets
        env:
        - name: DD_AGENT_HOST
          valueFrom:
            fieldRef:
              fieldPath: status.hostIP
        - name: DD_SERVICE
          value: "maya-pipecat-v6"
        - name: POD_NAME
          valueFrom:
            fieldRef:
              fieldPath: metadata.name
        - name: POD_IP
          valueFrom:
            fieldRef:
              fieldPath: status.podIP
        volumeMounts:
        - name: gcp-creds
          mountPath: /app/gcp-service-account.json
          subPath: gcp-service-account.json
          readOnly: true
        - name: gcp-creds
          mountPath: /app/gcp-new-service-account.json
          subPath: gcp-new-service-account.json
          readOnly: true
        - name: google-play-creds
          mountPath: /app/google-play-service-account.json
          subPath: google-play-service-account.json
          readOnly: true
      volumes:
      - name: gcp-creds
        secret:
          secretName: gcp-credentials
      - name: google-play-creds
        secret:
          secretName: google-play-credentials
---
apiVersion: cloud.google.com/v1
kind: BackendConfig
metadata:
  name: maya-pipecat-v6-backendconfig
spec:
  sessionAffinity:
    affinityType: "CLIENT_IP"
    affinityCookieTtlSec: 300
---
apiVersion: v1
kind: Service
metadata:
  name: maya-pipecat-v6
  labels:
    app: maya-pipecat-v6
  annotations:
    cloud.google.com/backend-config: '{"default": "maya-pipecat-v6-backendconfig"}'
spec:
  type: NodePort
  sessionAffinity: ClientIP
  sessionAffinityConfig:
    clientIP:
      timeoutSeconds: 300
  ports:
  - port: 80
    targetPort: 3006
    protocol: TCP
  selector:
    app: maya-pipecat-v6


Now let me check the routes to confirm the v6 health endpoint exists, and check ingress:


Bash

grep -rn "health\|ready" /home/ubuntu/pipecat/src/routes/ /home/ubuntu/pipecat/src/main.py 2>/dev/null | grep -E "@router|@app|health|ready" | head -10

Check health endpoints


/home/ubuntu/pipecat/src/routes/__init__.py:4:from src.routes.health import router as health_router
/home/ubuntu/pipecat/src/routes/__init__.py:16:    "health_router",
/home/ubuntu/pipecat/src/routes/health.py:4:- /health  (livenessProbe)  → Always returns 200 if process is alive.
/home/ubuntu/pipecat/src/routes/health.py:5:- /ready   (readinessProbe) → Returns 200 with Redis/ClickHouse health.
/home/ubuntu/pipecat/src/routes/health.py:13:router = APIRouter(tags=["health"])
/home/ubuntu/pipecat/src/routes/health.py:16:@router.get("/health")
/home/ubuntu/pipecat/src/routes/health.py:17:async def health_check():
/home/ubuntu/pipecat/src/routes/health.py:19:    return {"status": "healthy"}
/home/ubuntu/pipecat/src/routes/health.py:22:@router.get("/ready")
/home/ubuntu/pipecat/src/routes/health.py:25:    from src.core.redis import redis_health, redis_available
[result-id: r19]


grep -n "include_router\|prefix" /home/ubuntu/pipecat/src/main.py /home/ubuntu/pipecat/src/routes/__init__.py 2>&1 | head -10

Check route prefix


/home/ubuntu/pipecat/src/main.py:102:app.include_router(bot_router, prefix=API_PREFIX)
/home/ubuntu/pipecat/src/main.py:103:app.include_router(health_router, prefix=API_PREFIX)
/home/ubuntu/pipecat/src/main.py:104:app.include_router(personas_router, prefix=API_PREFIX)
/home/ubuntu/pipecat/src/main.py:105:app.include_router(payment_router, prefix=API_PREFIX)
/home/ubuntu/pipecat/src/main.py:106:app.include_router(usage_router, prefix=API_PREFIX)
/home/ubuntu/pipecat/src/main.py:107:app.include_router(client_router, prefix=API_PREFIX)
/home/ubuntu/pipecat/src/main.py:108:app.include_router(video_router, prefix=API_PREFIX)
/home/ubuntu/pipecat/src/main.py:109:app.include_router(app_router, prefix=API_PREFIX)
/home/ubuntu/pipecat/src/main.py:110:app.include_router(device_router, prefix=API_PREFIX)
/home/ubuntu/pipecat/src/main.py:116:    app.include_router(webrtc_fallback_router, prefix=f"{API_PREFIX}/webrtc")
[result-id: r20]


grep -n "API_PREFIX" /home/ubuntu/pipecat/src/main.py /home/ubuntu/pipecat/src/config/constants.py 2>&1 | head -5

Check API_PREFIX value


/home/ubuntu/pipecat/src/main.py:100:API_PREFIX = "/api/v6"
/home/ubuntu/pipecat/src/main.py:102:app.include_router(bot_router, prefix=API_PREFIX)
/home/ubuntu/pipecat/src/main.py:103:app.include_router(health_router, prefix=API_PREFIX)
/home/ubuntu/pipecat/src/main.py:104:app.include_router(personas_router, prefix=API_PREFIX)
/home/ubuntu/pipecat/src/main.py:105:app.include_router(payment_router, prefix=API_PREFIX)
[result-id: r21]


Confirmed: routes are at `/api/v6/health`. Now let me check ingress and check whether the v6 image already exists in registry:


Check ingress


apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: maya-pipecat-ingress
  annotations:
    kubernetes.io/ingress.global-static-ip-name: maya-pipecat-ip
    networking.gke.io/managed-certificates: maya-pipecat-cert
    kubernetes.io/ingress.class: gce
    networking.gke.io/v1beta1.FrontendConfig: maya-pipecat-frontend
spec:
  defaultBackend:
    service:
      name: maya-pipecat
      port:
        number: 80
  rules:
  - host: mayadev.mayaresearch.ai
    http:
      paths:
      - path: /api/v3/*
        pathType: ImplementationSpecific
        backend:
          service:
            name: maya-pipecat-v3
            port:
              number: 80
      - path: /
        pathType: Prefix
        backend:
          service:
            name: maya-pipecat
            port:
              number: 80
---
apiVersion: networking.gke.io/v1beta1
kind: FrontendConfig
metadata:
  name: maya-pipecat-frontend
spec:
  redirectToHttps:
    enabled: true
    responseCodeName: MOVED_PERMANENTLY_DEFAULT
[result-id: r22]


The local file is outdated (no v5). Let me check what's actually in the cluster:


kubectl get ingress maya-pipecat-ingress -o yaml | grep -A 30 "rules:"

Check live ingress rules


  rules:
  - host: mayadev.mayaresearch.ai
    http:
      paths:
      - backend:
          service:
            name: maya-pipecat-v5
            port:
              number: 80
        path: /api/v5/*
        pathType: ImplementationSpecific
      - backend:
          service:
            name: maya-pipecat-v4
            port:
              number: 80
        path: /api/v4/*
        pathType: ImplementationSpecific
      - backend:
          service:
            name: maya-pipecat-v3
            port:
              number: 80
        path: /api/v3/*
        pathType: ImplementationSpecific
      - backend:
          service:
            name: maya-pipecat
            port:
              number: 80
        path: /
[result-id: r23]


Live ingress has v3, v4, v5. I need to add v6. Let me check what v6 image exists in registry:


gcloud artifacts docker images list asia-south1-docker.pkg.dev/mayaresearchgcp/maya-pipecat --format="value(IMAGE,UPDATE_TIME)" 2>&1 | head -10

Check existing images


Listing items under project mayaresearchgcp, location asia-south1, repository maya-pipecat.

asia-south1-docker.pkg.dev/mayaresearchgcp/maya-pipecat/maya-pipecat	2026-02-27T10:32:38
asia-south1-docker.pkg.dev/mayaresearchgcp/maya-pipecat/maya-pipecat	2026-02-20T05:40:57
asia-south1-docker.pkg.dev/mayaresearchgcp/maya-pipecat/maya-pipecat	2026-03-23T18:04:59
asia-south1-docker.pkg.dev/mayaresearchgcp/maya-pipecat/maya-pipecat	2026-03-22T08:42:21
asia-south1-docker.pkg.dev/mayaresearchgcp/maya-pipecat/maya-pipecat	2026-02-19T16:38:33
asia-south1-docker.pkg.dev/mayaresearchgcp/maya-pipecat/maya-pipecat	2026-03-28T08:56:41
asia-south1-docker.pkg.dev/mayaresearchgcp/maya-pipecat/maya-pipecat	2026-02-19T17:07:35
asia-south1-docker.pkg.dev/mayaresearchgcp/maya-pipecat/maya-pipecat	2026-02-20T04:56:20
[result-id: r24]


No v6 image yet. I need to build it. Let me also do a syntax/import sanity check before building:


python3 -c "
import ast, os, sys
errors = []
for root, dirs, files in os.walk('src'):
    if '__pycache__' in root:
        continue
    for f in files:
        if f.endswith('.py'):
            p = os.path.join(root, f)
            try:
                with open(p) as fp:
                    ast.parse(fp.read())
            except SyntaxError as e:
                errors.append(f'{p}:{e.lineno}: {e.msg}')
if errors:
    for e in errors:
        print(e)
    sys.exit(1)
print('OK - all .py files parse')
"

Syntax check all Python files


OK - all .py files parse


Now build the v6 image:


docker build -t asia-south1-docker.pkg.dev/mayaresearchgcp/maya-pipecat/maya-pipecat-v6:latest . 2>&1 | tail -10

Build v6 image


#15 [10/10] RUN mkdir -p /app/logs /app/credentials /app/secrets
#15 DONE 0.2s

#16 exporting to image
#16 exporting layers
#16 exporting layers 3.2s done
#16 writing image sha256:265907f5040f98ff163dd4e66645c0cb37e47479908af18709cf73b583dd7e61 done
#16 naming to asia-south1-docker.pkg.dev/mayaresearchgcp/maya-pipecat/maya-pipecat-v6:latest done
#16 DONE 3.2s
[result-id: r25]


Build succeeded. Let me verify the image starts correctly by checking imports work in it:


docker run --rm asia-south1-docker.pkg.dev/mayaresearchgcp/maya-pipecat/maya-pipecat-v6:latest sh -c "cd /app && python -c 'from src import main; print(\"OK\")'" 2>&1 | tail -20

Verify imports work in image


2026-04-08 00:12:12.884 | INFO     | src.core.logging:configure_logging:163 | ============================================================
2026-04-08 00:12:12.884 | INFO     | src.core.logging:configure_logging:164 | MAYA LIVEKIT SERVER - STARTUP INITIATED
2026-04-08 00:12:12.884 | INFO     | src.core.logging:configure_logging:165 | ============================================================
2026-04-08 00:12:12.884 | INFO     | src.core.logging:configure_logging:166 | STARTUP_CONFIG: env=development host=0.0.0.0 port=3006
OK
[result-id: r26]


All imports load correctly. Now push the image:


docker push asia-south1-docker.pkg.dev/mayaresearchgcp/maya-pipecat/maya-pipecat-v6:latest 2>&1 | tail -5

Push v6 image


a5d46777e2f7: Pushed
bb427b3d570c: Pushed
335fc45cf5e8: Pushed
d5931f08a3c3: Pushed
latest: digest: sha256:2a89b729fc61bbdd5228d9a5b06072f53bb9f29d501fbee8d78dc026b58f227d size: 3044


Image pushed. Now apply the deployment:


kubectl apply -f /home/ubuntu/pipecat/deployments/k8s/gke/deployment-v6.yaml 2>&1

Apply v6 deployment


deployment.apps/maya-pipecat-v6 created
backendconfig.cloud.google.com/maya-pipecat-v6-backendconfig created
service/maya-pipecat-v6 created


Now update ingress to add `/api/v6/*`:


kubectl patch ingress maya-pipecat-ingress --type=json -p='[{"op": "add", "path": "/spec/rules/0/http/paths/0", "value": {"backend": {"service": {"name": "maya-pipecat-v6", "port": {"number": 80}}}, "path": "/api/v6/*", "pathType": "ImplementationSpecific"}}]' 2>&1

Add v6 path to ingress


ingress.networking.k8s.io/maya-pipecat-ingress patched


sleep 30 && kubectl get pods -l app=maya-pipecat-v6 --no-headers

Check v6 pod status


maya-pipecat-v6-549b44979d-95tbx   0/1   ContainerCreating   0     41s
maya-pipecat-v6-549b44979d-9ftfk   0/1   Running             0     41s
maya-pipecat-v6-549b44979d-gltw6   1/1   Running             0     41s
maya-pipecat-v6-549b44979d-h66s9   0/1   ContainerCreating   0     41s
maya-pipecat-v6-549b44979d-k6x6b   0/1   Running             0     41s
maya-pipecat-v6-549b44979d-pgbp7   0/1   Running             0     41s
maya-pipecat-v6-549b44979d-qldl7   0/1   Running             0     41s
maya-pipecat-v6-549b44979d-rkq7n   0/1   ContainerCreating   0     41s
maya-pipecat-v6-549b44979d-w2tqw   0/1   Running             0     41s
maya-pipecat-v6-549b44979d-w2v8f   0/1   ContainerCreating   0     41s
[result-id: r27]


10 pods are spinning up. Let me wait and verify they become ready:


sleep 60 && kubectl get pods -l app=maya-pipecat-v6 --no-headers

Check pods after 60s


maya-pipecat-v6-549b44979d-95tbx   0/1   Running   0     105s
maya-pipecat-v6-549b44979d-9ftfk   1/1   Running   0     105s
maya-pipecat-v6-549b44979d-gltw6   1/1   Running   0     105s
maya-pipecat-v6-549b44979d-h66s9   0/1   Running   0     105s
maya-pipecat-v6-549b44979d-k6x6b   1/1   Running   0     105s
maya-pipecat-v6-549b44979d-pgbp7   1/1   Running   0     105s
maya-pipecat-v6-549b44979d-qldl7   1/1   Running   0     105s
maya-pipecat-v6-549b44979d-rkq7n   0/1   Running   0     105s
maya-pipecat-v6-549b44979d-w2tqw   1/1   Running   0     105s
maya-pipecat-v6-549b44979d-w2v8f   0/1   Running   0     105s
[result-id: r28]


6/10 ready. Let me check if there are any errors in the not-yet-ready pods:


kubectl logs maya-pipecat-v6-549b44979d-95tbx --tail=30 2>&1 | grep -iE "error|fail|exception|critical" | head -10

Check for errors in pod


(Bash completed with no output)


kubectl logs maya-pipecat-v6-549b44979d-95tbx --tail=15 2>&1

Check pod logs


kubectl logs maya-pipecat-v6-549b44979d-95tbx 2>&1 | tail -20

Check logs no tail filter


kubectl describe pod maya-pipecat-v6-549b44979d-95tbx 2>&1 | tail -30

Describe pod


  ContainersReady             False
  PodScheduled                True
Volumes:
  gcp-creds:
    Type:        Secret (a volume populated by a Secret)
    SecretName:  gcp-credentials
    Optional:    false
  google-play-creds:
    Type:        Secret (a volume populated by a Secret)
    SecretName:  google-play-credentials
    Optional:    false
  kube-api-access-z9gdg:
    Type:                    Projected (a volume that contains injected data from multiple sources)
    TokenExpirationSeconds:  3607
    ConfigMapName:           kube-root-ca.crt
    Optional:                false
    DownwardAPI:             true
QoS Class:                   Burstable
Node-Selectors:              <none>
Tolerations:                 node.kubernetes.io/not-ready:NoExecute op=Exists for 300s
                             node.kubernetes.io/unreachable:NoExecute op=Exists for 300s
Events:
  Type     Reason     Age   From               Message
  ----     ------     ----  ----               -------
  Normal   Scheduled  2m2s  default-scheduler  Successfully assigned default/maya-pipecat-v6-549b44979d-95tbx to gke-maya-pipecat-pool-v2-ca055238-xw71
  Normal   Pulling    2m    kubelet            spec.containers{maya-pipecat-v6}: Pulling image "asia-south1-docker.pkg.dev/mayaresearchgcp/maya-pipecat/maya-pipecat-v6:latest"
  Normal   Pulled     34s   kubelet            spec.containers{maya-pipecat-v6}: Successfully pulled image "asia-south1-docker.pkg.dev/mayaresearchgcp/maya-pipecat/maya-pipecat-v6:latest" in 1m25.815s (1m25.815s including waiting). Image size: 361876474 bytes.
  Normal   Created    30s   kubelet            spec.containers{maya-pipecat-v6}: Created container: maya-pipecat-v6
  Normal   Started    27s   kubelet            spec.containers{maya-pipecat-v6}: Started container maya-pipecat-v6
  Warning  Unhealthy  10s   kubelet            spec.containers{maya-pipecat-v6}: Readiness probe failed: Get "http://10.56.3.189:3006/api/v6/health": dial tcp 10.56.3.189:3006: connect: connection refused
[result-id: r29]


Pod just started 30s ago, still warming up. Let me wait and check again:


sleep 60 && kubectl get pods -l app=maya-pipecat-v6 --no-headers

Check pods after another 60s


The user doesn't want to proceed with this tool use. The tool use was rejected (eg. if it was a file edit, the new_string was NOT written to the file). STOP what you are doing and wait for the user to tell you how to proceed.


[Request interrupted by user for tool use]


Can you check, can you do ABA calls and check whether it is up or not?


kubectl get pods -l app=maya-pipecat-v6 --no-headers

Current pod status


maya-pipecat-v6-549b44979d-95tbx   1/1   Running   0     2m49s
maya-pipecat-v6-549b44979d-9ftfk   1/1   Running   0     2m49s
maya-pipecat-v6-549b44979d-gltw6   1/1   Running   0     2m49s
maya-pipecat-v6-549b44979d-h66s9   1/1   Running   0     2m49s
maya-pipecat-v6-549b44979d-k6x6b   1/1   Running   0     2m49s
maya-pipecat-v6-549b44979d-pgbp7   1/1   Running   0     2m49s
maya-pipecat-v6-549b44979d-qldl7   1/1   Running   0     2m49s
maya-pipecat-v6-549b44979d-rkq7n   1/1   Running   0     2m49s
maya-pipecat-v6-549b44979d-w2tqw   1/1   Running   0     2m49s
maya-pipecat-v6-549b44979d-w2v8f   1/1   Running   0     2m49s
[result-id: r30]


**All 10/10 pods are Ready.** Let me verify with API calls — first directly to a pod, then via the ingress:


kubectl exec maya-pipecat-v6-549b44979d-95tbx -- curl -s -w "\nHTTP %{http_code}\n" http://localhost:3006/api/v6/health 2>&1

Test health endpoint inside pod


{"status":"healthy"}
HTTP 200


kubectl exec maya-pipecat-v6-549b44979d-95tbx -- curl -s -w "\nHTTP %{http_code}\n" http://localhost:3006/api/v6/ready 2>&1

Test readiness endpoint


Both Redis and ClickHouse connected. Let me also test via the service (cluster-internal) and via ingress (external):


kubectl run curl-test-v6 --image=curlimages/curl --rm -i --restart=Never -- curl -s -w "\nHTTP %{http_code}\n" http://maya-pipecat-v6/api/v6/health 2>&1 | tail -5

Test via service


curl -s -w "\nHTTP %{http_code}\n" https://mayadev.mayaresearch.ai/api/v6/health 2>&1

Test via public ingress


Warning: BCID failed open: BCID Constraint disabled by Giraffe
{"status":"healthy"}
HTTP 200
pod "curl-test-v6" deleted from default namespace


<meta http-equiv="content-type" content="text/html;charset=utf-8">
<title>502 Server Error</title>

Error: Server Error

The server encountered a temporary error and could not complete your request.
Please try again in 30 seconds.

HTTP 502
[result-id: r31]


Great progress:

| Test | Result |
|---|---|
| Pod direct (`localhost:3006/api/v6/health`) | ✅ 200 `{"status":"healthy"}` |
| Pod direct (`localhost:3006/api/v6/ready`) | ✅ 200 (Redis + ClickHouse connected) |
| Cluster service (`maya-pipecat-v6/api/v6/health`) | ✅ 200 `{"status":"healthy"}` |
| Public ingress (`https://mayadev.mayaresearch.ai/api/v6/health`) | ⏳ 502 (ingress still propagating) |

**The deployment is working** — pods are healthy, Redis + ClickHouse connected, service routing works internally. The 502 from the public URL is expected — GCE ingress takes 1-3 minutes to register the new backend after we patched it.

Let me wait and retest:


curl -s -w "\nHTTP %{http_code}\n" https://mayadev.mayaresearch.ai/api/v6/health 2>&1

Retest public ingress


<identical to result [r31] from your Bash call earlier — refer to that output>


Still propagating. Let me check the ingress backend health:


kubectl get ingress maya-pipecat-ingress -o json | python3 -c "
import json, sys
data = json.load(sys.stdin)
backends = data.get('metadata', {}).get('annotations', {}).get('ingress.kubernetes.io/backends', '{}')
print(json.dumps(json.loads(backends), indent=2))
" 2>&1 | head -20

Check ingress backend health states


{
  "k8s1-5cf08f8d-default-maya-pipecat-80-3bee62a6": "Unknown",
  "k8s1-5cf08f8d-default-maya-pipecat-v3-80-8cf5e2d4": "Unknown",
  "k8s1-5cf08f8d-default-maya-pipecat-v4-80-806d8094": "HEALTHY",
  "k8s1-5cf08f8d-default-maya-pipecat-v5-80-54939ad4": "HEALTHY"
}
[result-id: r32]


v6 isn't in the backends list yet. Let me verify the ingress patch took effect:


kubectl get ingress maya-pipecat-ingress -o yaml | grep -A 3 "v6"

Check v6 in ingress


            name: maya-pipecat-v6
            port:
              number: 80
        path: /api/v6/*
        pathType: ImplementationSpecific
      - backend:
          service:


Path is in the spec. The GCE ingress controller just hasn't synced the new backend yet (typically takes 60-180s). Let me wait and recheck:


sleep 90 && curl -s -w "\nHTTP %{http_code}\n" https://mayadev.mayaresearch.ai/api/v6/health 2>&1

Retest after wait


⏺ Now let me hit a few endpoints to check if it's up.

⏺ Bash(curl -s -o /dev/null -w "transport-type: HTTP %{http_code} (%{time_total}s)\n"
      "https://mayadev.mayaresearch.ai/api/v6/transport-type")
  ⎿  transport-type: HTTP 502 (0.285074s)

⏺ Bash(curl -s -o /dev/null -w "on_open: HTTP %{http_code} (%{time_total}s)\n"
      "https://mayadev.mayaresearch.ai/api/v6/on_open?device_id=test&timezone=Asia/Kolkata")
  ⎿  on_open: HTTP 502 (0.198947s)

⏺ Bash(curl -s -o /dev/null -w "personas: HTTP %{http_code} (%{time_total}s)\n"
      "https://mayadev.mayaresearch.ai/api/v6/personas?page=1&page_size=1&region=IN")
  ⎿  personas: HTTP 502 (0.165434s)

⏺ Bash(curl -s -o /dev/null -w "check-limits: HTTP %{http_code} (%{time_total}s)\n"
      "https://mayadev.mayaresearch.ai/api/v6/check-limits?device_id=test&persona_id=test…)
  ⎿  check-limits: HTTP 502 (0.179492s)

⏺ All endpoints are returning 502 Bad Gateway. The prod server at
  https://mayadev.mayaresearch.ai is down — the reverse proxy (nginx/load balancer) is
  reachable but the backend service behind it isn't running or is crashed.

  This also explains the SSH connection refused error you pasted — the server at 216.81.248.184
   port 22 is refusing connections, so the machine itself may be down or SSH service stopped.

  You'll need to get into the server to restart the backend service. Since SSH is also refused,
   you may need to use your cloud provider's console (e.g., DigitalOcean/AWS dashboard) to
  restart the instance or access it via web console.

✻ Crunched for 33s

───────────────────────────────────────────────────────────────────────────────────────────────
❯
───────────────────────────────────────────────────────────────────────────────────────────────
  ⏵⏵ bypass permissions on (shift+tab to cycle)


The other Claude is wrong about the server being down — pods ARE healthy. The 502 is just the GCE ingress backend not registered yet. Let me check the current state:


kubectl get ingress maya-pipecat-ingress -o json | python3 -c "
import json, sys
data = json.load(sys.stdin)
backends = data.get('metadata', {}).get('annotations', {}).get('ingress.kubernetes.io/backends', '{}')
print(json.dumps(json.loads(backends), indent=2))
"

Check ingress backends now


curl -s -w "\nHTTP %{http_code}\n" https://mayadev.mayaresearch.ai/api/v6/health 2>&1

Test public ingress now


<identical to result [r32] from your Bash call earlier — refer to that output>


Backend state hasn't changed — v6 still missing. Let me check if the service has endpoints and the proper NEG/health-check setup:


kubectl get svc maya-pipecat-v6 -o yaml | grep -E "annotations:|cloud\.google\.com|nodePort:|type:" 2>&1

Check v6 service config


kubectl get endpoints maya-pipecat-v6 2>&1

Check v6 endpoints


  annotations:
    cloud.google.com/backend-config: '{"default": "maya-pipecat-v6-backendconfig"}'
    cloud.google.com/neg: '{"ingress":true}'
    cloud.google.com/neg-status: '{"network_endpoint_groups":{"80":"k8s1-5cf08f8d-default-maya-pipecat-v6-80-5c89f8fd"},"zones":["asia-south1-a"]}'
      {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/backend-config":"{\"default\": \"maya-pipecat-v6-backendconfig\"}"},"labels":{"app":"maya-pipecat-v6"},"name":"maya-pipecat-v6","namespace":"default"},"spec":{"ports":[{"port":80,"protocol":"TCP","targetPort":3006}],"selector":{"app":"maya-pipecat-v6"},"sessionAffinity":"ClientIP","sessionAffinityConfig":{"clientIP":{"timeoutSeconds":300}},"type":"NodePort"}}
  - nodePort: 32675
  type: NodePort
[result-id: r33]


Warning: v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice
NAME              ENDPOINTS                                                     AGE
maya-pipecat-v6   10.56.0.23:3006,10.56.1.74:3006,10.56.2.27:3006 + 7 more...   5m10s


Service has 10 endpoints, NEG was created. The GCE backend service is being provisioned. Let me check the backend status directly via gcloud:


gcloud compute backend-services list --filter="name~maya-pipecat-v6" --format="table(name,protocol)" 2>&1

List v6 backend service


(Bash completed with no output)


gcloud compute backend-services list --filter="name~maya-pipecat" --format="table(name)" 2>&1 | head

List all maya backend services


NAME
k8s1-5cf08f8d-default-maya-pipecat-80-3bee62a6
k8s1-5cf08f8d-default-maya-pipecat-v3-80-8cf5e2d4
k8s1-5cf08f8d-default-maya-pipecat-v4-80-806d8094
k8s1-5cf08f8d-default-maya-pipecat-v5-80-54939ad4


The GCE backend service for v6 hasn't been created yet — that's why ingress returns 502. Let me check if there's an error stopping it:


kubectl describe ingress maya-pipecat-ingress 2>&1 | tail -20

Check ingress events


                           /           maya-pipecat:80 ()
Annotations:               ingress.gcp.kubernetes.io/pre-shared-cert: mcrt-5168d9cd-17f3-4d22-b067-8a7a0ce80ca1
                           ingress.kubernetes.io/backends:
                             {"k8s1-5cf08f8d-default-maya-pipecat-80-3bee62a6":"Unknown","k8s1-5cf08f8d-default-maya-pipecat-v3-80-8cf5e2d4":"Unknown","k8s1-5cf08f8d-d...
                           ingress.kubernetes.io/forwarding-rule: k8s2-fr-ivaq1ofi-default-maya-pipecat-ingress-78hzvowh
                           ingress.kubernetes.io/https-forwarding-rule: k8s2-fs-ivaq1ofi-default-maya-pipecat-ingress-78hzvowh
                           ingress.kubernetes.io/https-target-proxy: k8s2-ts-ivaq1ofi-default-maya-pipecat-ingress-78hzvowh
                           ingress.kubernetes.io/redirect-url-map: k8s2-rm-ivaq1ofi-default-maya-pipecat-ingress-78hzvowh
                           ingress.kubernetes.io/ssl-cert: mcrt-5168d9cd-17f3-4d22-b067-8a7a0ce80ca1
                           ingress.kubernetes.io/target-proxy: k8s2-tp-ivaq1ofi-default-maya-pipecat-ingress-78hzvowh
                           ingress.kubernetes.io/url-map: k8s2-um-ivaq1ofi-default-maya-pipecat-ingress-78hzvowh
                           kubernetes.io/ingress.class: gce
                           kubernetes.io/ingress.global-static-ip-name: maya-pipecat-ip
                           networking.gke.io/managed-certificates: maya-pipecat-cert
                           networking.gke.io/v1beta1.FrontendConfig: maya-pipecat-frontend
Events:
  Type     Reason     Age                       From                     Message
  ----     ------     ----                      ----                     -------
  Normal   Sync       5m33s (x812 over 5d15h)   loadbalancer-controller  Scheduled for sync
  Warning  Translate  5m33s (x1098 over 4d16h)  loadbalancer-controller  Translation failed: invalid ingress spec: could not find service "default/maya-pipecat-v4"
[result-id: r34]


**Found it.** Ingress translation is failing because of `maya-pipecat-v4` — that service was deleted but still referenced in the ingress. **The whole ingress sync is broken — that's why v6 backend isn't being created.**


Verify v4 service is gone


Error: Exit code 1
Error from server (NotFound): services "maya-pipecat-v4" not found


kubectl get ingress maya-pipecat-ingress -o json | python3 -c "
import json, sys
data = json.load(sys.stdin)
paths = data['spec']['rules'][0]['http']['paths']
for i, p in enumerate(paths):
    print(f\"{i}: path={p.get('path')} -> service={p['backend']['service']['name']}\")
"

List ingress paths


<tool_use_error>Cancelled: parallel tool call Bash(kubectl get svc maya-pipecat-v4 2>&1) errored</tool_use_error>


kubectl get ingress maya-pipecat-ingress -o json | python3 -c "
import json, sys
data = json.load(sys.stdin)
paths = data['spec']['rules'][0]['http']['paths']
for i, p in enumerate(paths):
    print(f\"{i}: path={p.get('path')} -> service={p['backend']['service']['name']}\")
"

List ingress paths


0: path=/api/v6/* -> service=maya-pipecat-v6
1: path=/api/v5/* -> service=maya-pipecat-v5
2: path=/api/v4/* -> service=maya-pipecat-v4
3: path=/api/v3/* -> service=maya-pipecat-v3
4: path=/ -> service=maya-pipecat


v4 service was deleted but still in ingress at index 2. Let me remove it:


kubectl patch ingress maya-pipecat-ingress --type=json -p='[{"op": "remove", "path": "/spec/rules/0/http/paths/2"}]' 2>&1

Remove v4 path from ingress


ingress.networking.k8s.io/maya-pipecat-ingress patched


sleep 30 && kubectl describe ingress maya-pipecat-ingress 2>&1 | tail -10

Check ingress sync after fix


                           ingress.kubernetes.io/url-map: k8s2-um-ivaq1ofi-default-maya-pipecat-ingress-78hzvowh
                           kubernetes.io/ingress.class: gce
                           kubernetes.io/ingress.global-static-ip-name: maya-pipecat-ip
                           networking.gke.io/managed-certificates: maya-pipecat-cert
                           networking.gke.io/v1beta1.FrontendConfig: maya-pipecat-frontend
Events:
  Type     Reason     Age                       From                     Message
  ----     ------     ----                      ----                     -------
  Warning  Translate  6m24s (x1098 over 4d16h)  loadbalancer-controller  Translation failed: invalid ingress spec: could not find service "default/maya-pipecat-v4"
  Normal   Sync       37s (x814 over 5d15h)     loadbalancer-controller  Scheduled for sync
[result-id: r35]


No more translation errors. Now let me wait for GCE to provision the v6 backend and test:


sleep 60 && gcloud compute backend-services list --filter="name~maya-pipecat-v6" --format="table(name)" 2>&1 && echo "---" && curl -s -w "\nHTTP %{http_code}\n" https://mayadev.mayaresearch.ai/api/v6/health 2>&1

Wait and test v6 backend


NAME
k8s1-5cf08f8d-default-maya-pipecat-v6-80-5c89f8fd
---

<meta http-equiv="content-type" content="text/html;charset=utf-8">
<title>502 Server Error</title>

Error: Server Error

The server encountered a temporary error and could not complete your request.
Please try again in 30 seconds.

HTTP 502
[result-id: r36]


v6 backend is now created in GCE. The 502 will resolve once health checks complete (~30-60s more). Let me check backend health:


kubectl get ingress maya-pipecat-ingress -o json | python3 -c "
import json, sys
data = json.load(sys.stdin)
backends = data.get('metadata', {}).get('annotations', {}).get('ingress.kubernetes.io/backends', '{}')
print(json.dumps(json.loads(backends), indent=2))
"

Check backend health states


sleep 60 && kubectl get ingress maya-pipecat-ingress -o json | python3 -c "
import json, sys
data = json.load(sys.stdin)
backends = data.get('metadata', {}).get('annotations', {}).get('ingress.kubernetes.io/backends', '{}')
print(json.dumps(json.loads(backends), indent=2))
"

Check backends after wait


gcloud compute backend-services get-health k8s1-5cf08f8d-default-maya-pipecat-v6-80-5c89f8fd --global 2>&1 | head -20

Check v6 backend health directly


---
backend: https://www.googleapis.com/compute/v1/projects/mayaresearchgcp/zones/asia-south1-a/networkEndpointGroups/k8s1-5cf08f8d-default-maya-pipecat-v6-80-5c89f8fd
status:
  healthStatus:
  - healthState: HEALTHY
    instance: https://www.googleapis.com/compute/v1/projects/mayaresearchgcp/zones/asia-south1-a/instances/gke-maya-pipecat-pool-v2-ca055238-xw71
    ipAddress: 10.56.3.188
    port: 3006
  - healthState: HEALTHY
    instance: https://www.googleapis.com/compute/v1/projects/mayaresearchgcp/zones/asia-south1-a/instances/gke-maya-pipecat-pool-v2-ca055238-xw71
    ipAddress: 10.56.3.189
    port: 3006
  - healthState: HEALTHY
    instance: https://www.googleapis.com/compute/v1/projects/mayaresearchgcp/zones/asia-south1-a/instances/gke-maya-pipecat-pool-v2-ca055238-s01j
    ipAddress: 10.56.7.190
    port: 3006
  - healthState: HEALTHY
    instance: https://www.googleapis.com/compute/v1/projects/mayaresearchgcp/zones/asia-south1-a/instances/gke-maya-pipecat-pool-v2-ca055238-s01j
    ipAddress: 10.56.7.191
    port: 3006
[result-id: r37]


**v6 backends are HEALTHY in GCE.** The backend service is up and the URL map should have been updated by now. Let me test:


curl -s -w "\nHTTP %{http_code}\n" https://mayadev.mayaresearch.ai/api/v6/health 2>&1

Test v6 ingress now


gcloud compute url-maps describe k8s2-um-ivaq1ofi-default-maya-pipecat-ingress-78hzvowh --global 2>&1 | grep -A 2 "v6\|api/v6\|pathRules\|hostRules" | head -30

Check URL map for v6 path


hostRules:
- hosts:
  - mayadev.mayaresearch.ai
--
  pathRules:
  - paths:
    - /api/v6/*
    service: https://www.googleapis.com/compute/v1/projects/mayaresearchgcp/global/backendServices/k8s1-5cf08f8d-default-maya-pipecat-v6-80-5c89f8fd
  - paths:
    - /api/v5/*
[result-id: r38]


**URL map IS configured for `/api/v6/*` → v6 backend.** But still 502. The GCE LB sometimes has caching delays. Let me try with curl-forced HTTP/1.1 and a different test path:


curl -s -i --http1.1 https://mayadev.mayaresearch.ai/api/v6/health 2>&1 | head -25

Test with HTTP/1.1 verbose


HTTP/1.1 200 OK
date: Wed, 08 Apr 2026 00:42:44 GMT
server: uvicorn
Content-Length: 20
content-type: application/json
Via: 1.1 google
Alt-Svc: h3=":443"; ma=2592000,h3-29=":443"; ma=2592000

{"status":"healthy"}


**🎉 v6 is up via the public ingress.** Let me test more endpoints:


echo "=== /health ===" && curl -s -w "\nHTTP %{http_code}\n" --http1.1 https://mayadev.mayaresearch.ai/api/v6/health
echo "=== /ready ===" && curl -s -w "\nHTTP %{http_code}\n" --http1.1 https://mayadev.mayaresearch.ai/api/v6/ready
echo "=== /transport-type ===" && curl -s -w "\nHTTP %{http_code}\n" --http1.1 "https://mayadev.mayaresearch.ai/api/v6/transport-type"
echo "=== /on_open ===" && curl -s -w "\nHTTP %{http_code}\n" --http1.1 "https://mayadev.mayaresearch.ai/api/v6/on_open?device_id=test&timezone=Asia/Kolkata"

Test multiple v6 endpoints


=== /health ===
{"status":"healthy"}
HTTP 200
=== /ready ===
{"status":"ready","ready":true,"timestamp":"2026-04-08T00:42:50.678938","redis":{"status":"healthy","connected":true,"latency_ms":3.05,"used_memory_mb":586.11,"connected_clients":1350},"clickhouse":{"status":"connected"}}
HTTP 200
=== /transport-type ===
{"transport":"livekit"}
HTTP 200
=== /on_open ===
{"update_enabled":true,"force_update":true,"show_language_screen":true,"show_call_screen":true,"languages":[{"code":"te","native_label":"తెలుగు","script_char":"త","english_label":"Telugu"},{"code":"hi","native_label":"हिन्दी","script_char":"ह","english_label":"Hindi"}],"region":"india","language":null,"is_premium":false,"transport_type":"livekit","showcase_edits":[{"before":"https://storage.googleapis.com/maya-session-data/showcase/showcase_1_before.jpg","after":"https://storage.googleapis.com/maya-session-data/showcase/showcase_1_after.jpg"},{"before":"https://storage.googleapis.com/maya-session-data/showcase/showcase_2_before.jpg","after":"https://storage.googleapis.com/maya-session-data/showcase/showcase_2_after.jpg"}]}
HTTP 200
[result-id: r39]


## 🎉 v6 is fully deployed and working

| Endpoint | Status | Notes |
|---|---|---|
| `/api/v6/health` | ✅ 200 | `{"status":"healthy"}` |
| `/api/v6/ready` | ✅ 200 | Redis + ClickHouse connected |
| `/api/v6/transport-type` | ✅ 200 | `{"transport":"livekit"}` |
| `/api/v6/on_open` | ✅ 200 | Returns India region, languages, etc. |

## What was deployed

- **10 pods** running `maya-pipecat-v6` image
- All on **port 3006** with **LiveKit transport** (not pipecat)
- Connected to **ClickHouse** for analytics + **Redis** for cross-pod state
- Public ingress route `/api/v6/*` → `maya-pipecat-v6` service
- Cleaned up stale `maya-pipecat-v4` ingress reference that was blocking GCE LB sync

**The other Claude was wrong** — it concluded the server was down based on early 502s from before the ingress finished syncing. Pods were healthy the whole time; the GCE load balancer just needed ~3 minutes to provision the new backend (and we had to fix the broken v4 reference first).

Test from your app/client now — `https://mayadev.mayaresearch.ai/api/v6/*` is live.


pull and deploy again


git status && echo "---" && git pull origin v6_test 2>&1 | tail -10

Pull v6_test changes


On branch v6_test
Your branch is up to date with 'origin/v6_test'.

Changes not staged for commit:
  (use "git add <file>..." to update what will be committed)
  (use "git restore <file>..." to discard changes in working directory)
	modified:   prompts/prompt_character.txt

Untracked files:
  (use "git add <file>..." to include in what will be committed)
	deployments/k8s/gke/deployment-v6.yaml

no changes added to commit (use "git add" and/or "git commit -a")
---
 create mode 100644 default_images/hf_20260407_120549_e3caf572-bac6-4a7b-a746-dc5d61294fe2.png
 create mode 100644 generate_hindi_samples.py
 create mode 100644 qwen_sample/benchmark_results.txt
 create mode 100644 qwen_sample/run_benchmark.py
 create mode 100755 scripts/clickhouse_backup.sh
 create mode 100644 scripts/migrate_turso_to_clickhouse.py
 create mode 100644 scripts/migration.txt
 create mode 100644 src/repository/experiment_repository.py
 create mode 100644 src/service/experiment_service.py
 create mode 100644 src/service/qwen_edit_client.py
[result-id: r40]


Pulled new changes. Let me check what changed and rebuild:


See recent commits


458cc53 updated image models
e099bcf fix: prevent Gemini Live auto-generating images on greeting
c87ec82 feat: add Gemini Live for non-Hindi languages (WIP — has issues with global langs)
b089b1c feat: v6 dual transport — LiveKit primary + SmallWebRTC fallback
3e0114a chore: remove .env and pipecat-ctl.sh
[result-id: r41]


docker build -t asia-south1-docker.pkg.dev/mayaresearchgcp/maya-pipecat/maya-pipecat-v6:latest . 2>&1 | tail -5

Build v6 image


#16 exporting to image
#16 exporting layers 0.0s done
#16 writing image sha256:c685e56b6f7ad9e1ce911f24954cd27a0670cb6a42622285b00748148f1d3e7b done
#16 naming to asia-south1-docker.pkg.dev/mayaresearchgcp/maya-pipecat/maya-pipecat-v6:latest done
#16 DONE 0.1s
[result-id: r42]


docker push asia-south1-docker.pkg.dev/mayaresearchgcp/maya-pipecat/maya-pipecat-v6:latest 2>&1 | tail -5 && kubectl rollout restart deployment maya-pipecat-v6

Push and restart v6


57b8754d9121: Pushed
c79e9e50ecb9: Pushed
647e2d5e3c1f: Layer already exists
335fc45cf5e8: Layer already exists
latest: digest: sha256:8743db9ab694a596c8f46b58bdd463741c4d8ec9c0566600fe9bb06e49184f39 size: 3044
deployment.apps/maya-pipecat-v6 restarted


Deployed. Let me verify the pods come up healthy:


sleep 60 && kubectl get pods -l app=maya-pipecat-v6 --no-headers | awk '{print $2,$3}' | sort | uniq -c

Check pod status after rollout


      2 0/1 Running
     11 1/1 Running
      2 1/1 Terminating


Rollout in progress — 11 ready, 2 starting, 2 terminating. Let me verify health via ingress:


curl -s -w "\nHTTP %{http_code}\n" --http1.1 https://mayadev.mayaresearch.ai/api/v6/health 2>&1

Verify v6 health via ingress


{"status":"healthy"}
HTTP 200


Deployed and healthy. The new commits include:
- `b089b1c` v6 dual transport (LiveKit primary + SmallWebRTC fallback)
- `c87ec82` Gemini Live for non-Hindi languages
- `e099bcf` Fix Gemini Live auto-generating images on greeting
- `458cc53` Updated image models


pull and deplooy again .. and make pods to 40


Pull latest


 default_images/after.png                        | Bin 0 -> 40146689 bytes
 default_images/before.jpeg                      | Bin 0 -> 157778 bytes
 default_images/compressed/showcase_3_after.jpg  | Bin 0 -> 212794 bytes
 default_images/compressed/showcase_3_before.jpg | Bin 0 -> 172358 bytes
 src/service/app_v2_service.py                   |   3 +--
 5 files changed, 1 insertion(+), 2 deletions(-)
 create mode 100644 default_images/after.png
 create mode 100644 default_images/before.jpeg
 create mode 100644 default_images/compressed/showcase_3_after.jpg
 create mode 100644 default_images/compressed/showcase_3_before.jpg
[result-id: r43]


docker build -t asia-south1-docker.pkg.dev/mayaresearchgcp/maya-pipecat/maya-pipecat-v6:latest . 2>&1 | tail -5

Build v6


#16 exporting to image
#16 exporting layers 0.0s done
#16 writing image sha256:43653c92797ba695d7f1b9c166f7ad6d27eec12bb7a86d850a32f63a3b1c79bb done
#16 naming to asia-south1-docker.pkg.dev/mayaresearchgcp/maya-pipecat/maya-pipecat-v6:latest done
#16 DONE 0.1s
[result-id: r44]


docker push asia-south1-docker.pkg.dev/mayaresearchgcp/maya-pipecat/maya-pipecat-v6:latest 2>&1 | tail -3 && kubectl scale deployment maya-pipecat-v6 --replicas=40 && kubectl rollout restart deployment maya-pipecat-v6

Push, scale to 40, restart


647e2d5e3c1f: Layer already exists
335fc45cf5e8: Layer already exists
latest: digest: sha256:069d54a8aafb505b741777bca4b82b9feef5eb7ef08bbc5e52ea878d9919d6fa size: 3044
deployment.apps/maya-pipecat-v6 scaled
deployment.apps/maya-pipecat-v6 restarted


Pushed, scaled to 40, restarted. Let me verify:


sleep 60 && kubectl get pods -l app=maya-pipecat-v6 --no-headers | awk '{print $2,$3}' | sort | uniq -c

Check pod status


chekc is it accesible


      8 0/1 Running
     42 1/1 Running
      6 1/1 Terminating


curl -s -w "\nHTTP %{http_code}\n" --http1.1 https://mayadev.mayaresearch.ai/api/v6/health 2>&1

Test public ingress


✅ **Accessible.** v6 is responding via the public ingress (`https://mayadev.mayaresearch.ai/api/v6/health` → 200).

Status: 42 pods Ready, 8 starting, 6 old terminating. Rollout is finishing — will settle at 40 pods shortly.


pull and deploy


Pull latest


 * branch            v6_test    -> FETCH_HEAD
Updating 820bcaa..d8220ad
Fast-forward
 src/agent/tools/image_tool.py | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)


docker build -t asia-south1-docker.pkg.dev/mayaresearchgcp/maya-pipecat/maya-pipecat-v6:latest . 2>&1 | tail -3 && docker push asia-south1-docker.pkg.dev/mayaresearchgcp/maya-pipecat/maya-pipecat-v6:latest 2>&1 | tail -3 && kubectl rollout restart deployment maya-pipecat-v6

Build, push, restart


#16 writing image sha256:41c4b1d301677a963f945753f69590b0a3fa083451fe26d8482a3605d6cc8604 done
#16 naming to asia-south1-docker.pkg.dev/mayaresearchgcp/maya-pipecat/maya-pipecat-v6:latest done
#16 DONE 0.1s
19a0adfcfddb: Pushed
335fc45cf5e8: Layer already exists
latest: digest: sha256:45a655a5961560e03349dab9d52d96d90e4e673300e6ecf4d477a843c2bfcbed size: 3044
deployment.apps/maya-pipecat-v6 restarted
[result-id: r45]


Deployed.


check again


kubectl get pods -l app=maya-pipecat-v6 --no-headers | awk '{print $2,$3}' | sort | uniq -c

Pod status


curl -s -w "\nHTTP %{http_code}\n" --http1.1 https://mayadev.mayaresearch.ai/api/v6/health 2>&1

Check public ingress


40 1/1 Running


**All 40 pods Running, ingress responding 200.** v6 is fully deployed and accessible.


Pull latest


 qwen_sample/edits_data.json   |   1 +
 src/agent/tools/image_tool.py |   2 +-
 3 files changed, 169 insertions(+), 1 deletion(-)
 create mode 100644 qwen_sample/compare.html
 create mode 100644 qwen_sample/edits_data.json


docker build -t asia-south1-docker.pkg.dev/mayaresearchgcp/maya-pipecat/maya-pipecat-v6:latest . 2>&1 | tail -3 && docker push asia-south1-docker.pkg.dev/mayaresearchgcp/maya-pipecat/maya-pipecat-v6:latest 2>&1 | tail -3 && kubectl rollout restart deployment maya-pipecat-v6

Build, push, restart


#16 writing image sha256:9c47010563a0b56bec6cbe3b5f7f6e962cc507ce0750c4a5cec48332f2620f0e done
#16 naming to asia-south1-docker.pkg.dev/mayaresearchgcp/maya-pipecat/maya-pipecat-v6:latest done
#16 DONE 0.0s
647e2d5e3c1f: Layer already exists
335fc45cf5e8: Layer already exists
latest: digest: sha256:08e628d024d95312a793be333fea602275dbf42bf809f29586748fb8a21817c0 size: 3044
deployment.apps/maya-pipecat-v6 restarted
[result-id: r46]


Pull latest


Updating 377f406..adbb0cb
Fast-forward
 src/agent/tools/image_tool.py   | 3 ++-
 src/service/qwen_edit_client.py | 6 +-----
 2 files changed, 3 insertions(+), 6 deletions(-)


docker build -t asia-south1-docker.pkg.dev/mayaresearchgcp/maya-pipecat/maya-pipecat-v6:latest . 2>&1 | tail -3 && docker push asia-south1-docker.pkg.dev/mayaresearchgcp/maya-pipecat/maya-pipecat-v6:latest 2>&1 | tail -3 && kubectl rollout restart deployment maya-pipecat-v6

Build, push, restart


Error: Exit code 1
#16 writing image sha256:922bfdcce57ac1d050642479abdafefdf56859619ef1b5f070e1e6a364996582 done
#16 naming to asia-south1-docker.pkg.dev/mayaresearchgcp/maya-pipecat/maya-pipecat-v6:latest done
#16 DONE 0.1s
335fc45cf5e8: Waiting
bb427b3d570c: Waiting
denied: Unauthenticated request. Unauthenticated requests do not have permission "artifactregistry.repositories.uploadArtifacts" on resource "projects/mayaresearchgcp/locations/asia-south1/repositories/maya-pipecat" (or it may not exist)
F0408 17:23:41.830178  500970 cred.go:145] print credential failed with error: Failed to retrieve access token:: failure while executing gcloud, with args [config config-helper --format=json]: exit status 1 (err: ERROR: (gcloud.config.config-helper) There was a problem refreshing your current auth tokens: Reauthentication failed. cannot prompt during non-interactive execution.
Please run:

  $ gcloud auth login

to obtain new credentials.

If you have already logged in with a different account, run:

  $ gcloud config set account ACCOUNT

to select an already authenticated account to use.
)
Unable to connect to the server: getting credentials: exec: executable gke-gcloud-auth-plugin failed with exit code 1


gcloud auth expired. Run this in your terminal:

```
! gcloud auth login
```

Then I'll continue. The image is built locally — just need to push and restart after re-auth.


Pull latest


From https://github.com/MayaResearch/pipecat
 * branch            v6_test    -> FETCH_HEAD
Already up to date.


docker push asia-south1-docker.pkg.dev/mayaresearchgcp/maya-pipecat/maya-pipecat-v6:latest 2>&1 | tail -5 && kubectl rollout restart deployment maya-pipecat-v6

Push and restart


34c49996956e: Pushed
883db5f63296: Pushed
647e2d5e3c1f: Layer already exists
335fc45cf5e8: Layer already exists
latest: digest: sha256:91c99fd84bd0196a355d53a393b905705c3803d581d1af8ac8736c79353c80f5 size: 3044
deployment.apps/maya-pipecat-v6 restarted


Pushed and restarted.


what i sratio between v6 pods and v5 pods


v5=$(kubectl get pods -l app=maya-pipecat-v5 --field-selector=status.phase=Running --no-headers 2>/dev/null | wc -l)
v6=$(kubectl get pods -l app=maya-pipecat-v6 --field-selector=status.phase=Running --no-headers 2>/dev/null | wc -l)
total=$((v5 + v6))
echo "v5: $v5 pods"
echo "v6: $v6 pods"
echo "Total: $total"
if [ "$total" -gt 0 ]; then
  echo "v6:v5 ratio = $((v6 * 100 / total))% : $((v5 * 100 / total))%"
fi

Compare v5 and v6 pod counts


v5: 120 pods
v6: 40 pods
Total: 160
v6:v5 ratio = 25% : 75%


| Version | Pods | Share |
|---|---|---|
| v5 | 120 | 75% |
| v6 | 40 | 25% |
| **Total** | **160** | |

**Ratio: 1 v6 pod for every 3 v5 pods (1:3).** v6 has 25% of the cluster capacity, v5 has 75%.


make them v6 to 70 and v5 to 30


Check HPAs


NAME                  REFERENCE                    TARGETS                          MINPODS   MAXPODS   REPLICAS   AGE
maya-pipecat-v5-hpa   Deployment/maya-pipecat-v5   cpu: 41%/70%, memory: 154%/80%   80        120       120        9d


v5 has HPA min=80/max=120 (currently 120). v6 has no HPA. Let me scale both:


kubectl patch hpa maya-pipecat-v5-hpa --type=merge -p '{"spec":{"minReplicas":30,"maxReplicas":40}}' 2>&1

Update v5 HPA min/max


kubectl scale deployment maya-pipecat-v6 --replicas=70 2>&1

Scale v6 to 70


horizontalpodautoscaler.autoscaling/maya-pipecat-v5-hpa patched


deployment.apps/maya-pipecat-v6 scaled


sleep 30 && v5=$(kubectl get pods -l app=maya-pipecat-v5 --field-selector=status.phase=Running --no-headers 2>/dev/null | wc -l)
v6=$(kubectl get pods -l app=maya-pipecat-v6 --field-selector=status.phase=Running --no-headers 2>/dev/null | wc -l)
total=$((v5 + v6))
echo "v5: $v5 pods"
echo "v6: $v6 pods"
echo "Total: $total"

Recheck pod counts


v5: 54 pods
v6: 70 pods
Total: 124


In progress — v5 is scaling down (54 → 30), v6 already at 70. v5 takes longer because of HPA stabilization window. Should settle at v5=30, v6=70 in a few minutes.


is it done ?


v5=$(kubectl get pods -l app=maya-pipecat-v5 --field-selector=status.phase=Running --no-headers 2>/dev/null | wc -l)
v6=$(kubectl get pods -l app=maya-pipecat-v6 --field-selector=status.phase=Running --no-headers 2>/dev/null | wc -l)
echo "v5: $v5 / 30"
echo "v6: $v6 / 70"
kubectl get hpa maya-pipecat-v5-hpa 2>&1

Check pod counts and HPA


v5: 40 / 30
v6: 70 / 70
NAME                  REFERENCE                    TARGETS                          MINPODS   MAXPODS   REPLICAS   AGE
maya-pipecat-v5-hpa   Deployment/maya-pipecat-v5   cpu: 56%/70%, memory: 162%/80%   30        40        40         9d
[result-id: r1]


**v6: 70/70 ✓** done.
**v5: 40/30** — HPA is keeping it at max=40 because memory is at **162%** (way above the 80% target). HPA refuses to scale down when load is high.

Two options to force v5 to exactly 30:
1. Lower HPA `maxReplicas=30` (HPA will be forced to cap, but pods may run hot)
2. Delete the HPA entirely and just `kubectl scale --replicas=30`

Want me to do option 1 (force max=30)?


how many pods in tota;


kubectl get pods --field-selector=status.phase=Running --no-headers 2>/dev/null | wc -l && echo "---" && kubectl get pods --field-selector=status.phase=Running --no-headers 2>/dev/null | awk '{print $1}' | sed 's/-[0-9a-f]*-[0-9a-z]*$//' | sort | uniq -c

Total pods and breakdown


116
---
      1 datadog-agent-286xk
      1 datadog-agent-5j5x5
      1 datadog-agent-6cxbv
      1 datadog-agent-8jm2x
      1 datadog-agent-gtsxq
      1 datadog-agent-zrqnv
     40 maya-pipecat-v5
     70 maya-pipecat-v6


**116 pods total running:**

- **maya-pipecat-v5**: 40
- **maya-pipecat-v6**: 70
- datadog-agent: 6 (one per node)

App pods: **110** (40 v5 + 70 v6).


are we using cluster total compute and memory


Check node resource usage


NAME                                          CPU(cores)   CPU(%)   MEMORY(bytes)   MEMORY(%)
gke-maya-pipecat-default-pool-f76748ed-24sr   5531m        8%       25610Mi         11%
gke-maya-pipecat-default-pool-f76748ed-l70x   8922m        13%      28208Mi         12%
gke-maya-pipecat-default-pool-f76748ed-trnd   11918m       18%      69103Mi         30%
gke-maya-pipecat-default-pool-f76748ed-vks4   7940m        12%      30160Mi         13%
gke-maya-pipecat-pool-v2-ca055238-s01j        12428m       19%      41192Mi         17%
gke-maya-pipecat-pool-v2-ca055238-xw71        14743m       23%      57484Mi         25%
[result-id: r2]


**No — cluster is heavily underutilized.**

| Node | CPU | CPU % | Memory | Mem % |
|------|-----|-------|--------|-------|
| `..24sr` | 5.5 cores | 8% | 25 GB | 11% |
| `..l70x` | 8.9 cores | 13% | 28 GB | 12% |
| `..trnd` | 11.9 cores | 18% | 69 GB | 30% |
| `..vks4` | 7.9 cores | 12% | 30 GB | 13% |
| `..s01j` (new) | 12.4 cores | 19% | 41 GB | 17% |
| `..xw71` (new) | 14.7 cores | 23% | 57 GB | 25% |

**Totals:**
- **CPU**: ~61 / 384 vCPUs (**16%**)
- **Memory**: ~250 / 1.4 TB (**18%**)

Plenty of headroom — you could run 4-5x more pods without breaking a sweat. The cluster is way over-provisioned for the current load.


i mean this pods with max allocation hits the room .


kubectl describe nodes 2>&1 | grep -E "Allocated resources|cpu  |memory  " | head -30

Check allocated requests/limits per node


The user doesn't want to proceed with this tool use. The tool use was rejected (eg. if it was a file edit, the new_string was NOT written to the file). STOP what you are doing and wait for the user to tell you how to proceed.


[Request interrupted by user for tool use]


i mean current max allocation


kubectl describe nodes 2>&1 | awk '/Allocated resources/{flag=1} flag && /cpu/{print $1, $2, $3, $4; flag=0}' | head -20

Get allocated CPU per node


cpu 28669m (44%) 117544m
cpu 34756m (54%) 141500m
cpu 40756m (63%) 165500m
cpu 38496m (60%) 156500m
cpu 38486m (60%) 156500m
cpu 42486m (66%) 172500m


kubectl describe nodes 2>&1 | awk '/Allocated resources/{flag=1; next} flag && /memory/{print $1, $2, $3, $4; flag=0}' | head -20

Get allocated memory per node


memory 46274841472 (19%) 188115466752
memory 55757328640 (23%) 224415753728
memory 65421005056 (27%) 263070459392
memory 62058221824 (25%) 249918170624
memory 62037250304 (25%) 249918170624
memory 68479701248 (28%) 275687974400


Here's the **max allocation** picture (sum across 6 nodes):

| | Requested (reserved) | Limit (max burst) | Cluster capacity |
|---|---|---|---|
| **CPU** | 224 cores (~58%) | 950 cores (**247%**) | 384 cores |
| **Memory** | 343 GB (~24%) | ~1320 GB (**94%**) | 1410 GB |

**Per pod (110 app pods):**
- Requests: 2 cores + 3 GB
- Limits: 8 cores + 12 GB

## What this means

**Requests** (what k8s reserved for scheduling): 58% CPU, 24% memory — **safe**, plenty of room.

**Limits** (max burst if all pods hit max simultaneously):
- **CPU: 247% — overcommitted by 2.5x.** If every pod tried to use 8 cores at once, the cluster physically can't deliver. Pods would get CPU-throttled.
- **Memory: 94% — basically maxed out.** If every pod tried to use 12 GB simultaneously, you'd hit OOM and pods would start getting evicted.

## In practice

CPU overcommit is OK — voice/AI workloads rarely all spike at once.

But **memory at 94% limit is dangerous**. If a few pods leak or sessions pile up, you can hit eviction quickly. This is exactly what caused the cascade failures yesterday (when core dumps filled disk and triggered evictions).

**Current usage** is only 16% CPU and 18% memory — so you have **6x headroom in actual usage**. The risk is the gap between actual usage and the configured limits.

If you want to scale safely, either:
1. **Lower memory limits** from 12Gi → 6Gi per pod (cuts max burst to 47%, much safer)
2. **Add more nodes** (but underutilized currently — wasteful)
3. **Tighten requests** so HPA scales more accurately

Want me to recommend an action?


1 more thing can u go in to pipecat code .. see we not getting all logs in datadog ... i dont know y .. before we used pipecat now we migrtaed code to livekit ...just want to to sure what is the issue .. agent logs not yet al cmng


Check logging config


"""Centralized logging configuration for Maya server."""

import os
import sys
import json
import logging
import warnings
import traceback
from datetime import timezone
from loguru import logger

from src.config import constants

def log_filter(record):
    msg = record["message"]
    if "Unable to send message" in msg and "before joining" in msg:
        return False

    if record.get("exception"):
        exc_info = record["exception"]
        if exc_info and exc_info.value:
            exc_msg = str(exc_info.value)
            if "Unable to send message" in exc_msg and "before joining" in exc_msg:
                return False

    if record["level"].name in ("ERROR", "CRITICAL", "EXCEPTION"):
        return True

    name = record.get("name", "")
    if name and name.startswith(("botocore", "boto3", "urllib3", "s3transfer")):
        return False
    if name and name.startswith("openai"):
        return False
    if name and name.startswith("httpx"):
        return False

    if "Unclosed client session" in msg or "Unclosed connector" in msg:
        return False
    if "Task was destroyed but it is pending" in msg:
        return False
    return True

CONSOLE_FORMAT = "{time:YYYY-MM-DD HH:mm:ss.SSS} | {level: <8} | {name}:{function}:{line} | {message}"

def datadog_sink(message):
    record = message.record
    level = record["level"].name

    timestamp = record["time"].astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z"

    log_entry = {
        "timestamp": timestamp,
        "level": level,
        "logger": record["name"],
        "function": record["function"],
        "line": record["line"],
        "message": record["message"],
        "service": os.getenv("DD_SERVICE", "maya-livekit"),
        "env": os.getenv("DD_ENV", os.getenv("ENVIRONMENT", "development")),
        "version": os.getenv("DD_VERSION", getattr(constants, 'APP_VERSION', "1.0.0")),
    }

    for key, value in record["extra"].items():
        if value and key not in ("_maya_configured",):
            log_entry[key] = value

    if level in ("ERROR", "CRITICAL", "EXCEPTION"):
        log_entry["status"] = "error"
        log_entry["error.kind"] = level

        if record.get("exception"):
            exc_info = record["exception"]
            if exc_info:
                log_entry["error.type"] = exc_info.type.__name__ if exc_info.type else "Unknown"
                log_entry["error.message"] = str(exc_info.value) if exc_info.value else ""
                log_entry["error.stack"] = "".join(traceback.format_exception(exc_info.type, exc_info.value, exc_info.traceback))
    else:
        log_entry["status"] = "info"

    return json.dumps(log_entry, ensure_ascii=False) + "\n"

class DatadogStdoutSink:
    """Writes JSON-formatted logs to stdout for DD DaemonSet collection."""

    def write(self, message):
        json_line = datadog_sink(message)
        sys.stdout.write(json_line)
        sys.stdout.flush()

class InterceptHandler(logging.Handler):
    def emit(self, record):
        try:
            level = logger.level(record.levelname).name
        except ValueError:
            level = record.levelno

        frame, depth = sys._getframe(6), 6
        while frame and frame.f_code.co_filename == logging.__file__:
            frame = frame.f_back
            depth += 1

        logger.opt(depth=depth, exception=record.exc_info).log(level, record.getMessage())

def configure_logging():
    if hasattr(logger, '_maya_configured'):
        return

    os.makedirs(constants.LOG_DIR, exist_ok=True)
    logger.remove()

    is_production = bool(os.getenv("DD_AGENT_HOST"))
    stdout_level = os.getenv("LOG_LEVEL", "INFO" if is_production else "DEBUG")

    if is_production:
        json_stdout_sink = DatadogStdoutSink()
        logger.add(
            json_stdout_sink.write,
            level=stdout_level,
            filter=log_filter,
        )
    else:
        logger.add(
            sys.stdout,
            level="DEBUG",
            format=CONSOLE_FORMAT,
            filter=log_filter,
            colorize=False,
            enqueue=False,
            backtrace=True,
            diagnose=True,
        )

    logger.add(
        constants.LOG_FILE_PATH,
        level="INFO",
        format=CONSOLE_FORMAT,
        filter=log_filter,
        rotation=constants.LOG_ROTATION_SIZE,
        retention=constants.LOG_RETENTION,
        compression="zip",
        enqueue=False,
        backtrace=True,
        diagnose=True,
    )

    warnings.filterwarnings("default")
    logging.captureWarnings(True)

    logging.basicConfig(handlers=[InterceptHandler()], level=logging.DEBUG, force=True)

    for logger_name in ["uvicorn", "uvicorn.error", "uvicorn.access", "fastapi", "livekit", "aiohttp"]:
        logging.getLogger(logger_name).handlers = [InterceptHandler()]
        logging.getLogger(logger_name).propagate = False

    logger._maya_configured = True

    logger.info("=" * 60)
    logger.info("MAYA LIVEKIT SERVER - STARTUP INITIATED")
    logger.info("=" * 60)
    logger.info(f"STARTUP_CONFIG: env={os.getenv('ENVIRONMENT', 'development')} host={constants.SERVER_HOST} port={constants.SERVER_PORT}")

def log_startup_complete(startup_time_ms: float = None):
    logger.info("=" * 60)
    logger.info("MAYA LIVEKIT SERVER - STARTUP COMPLETE")
    if startup_time_ms:
        logger.info(f"STARTUP_TIME: {startup_time_ms:.2f}ms")
    logger.info("=" * 60)

def log_shutdown():
    logger.info("MAYA LIVEKIT SERVER - SHUTDOWN INITIATED")

def log_error(message: str, error: Exception = None, **context):
    full_message = f"ERROR: {message}"
    if error:
        full_message += f" | error_type={type(error).__name__} error_message={str(error)}"

    for key, value in context.items():
        full_message += f" {key}={value}"

    if error:
        logger.opt(exception=error).error(full_message)
    else:
        logger.error(full_message)
[result-id: r3]


Now let me check how the **agent** logs (the LiveKit Agents framework):


ls /home/ubuntu/pipecat/src/agent/ 2>&1 && echo "---" && grep -rn "logging\|logger\|getLogger\|configure_logging" /home/ubuntu/pipecat/src/agent/ 2>&1 | head -20

Check agent logging


core
handlers
__init__.py
__main__.py
pipecat_bot.py
plugins
tools
utils
---
/home/ubuntu/pipecat/src/agent/handlers/image_upload.py:6:from loguru import logger
/home/ubuntu/pipecat/src/agent/handlers/image_upload.py:28:            logger.info(f"IMAGE_UPLOAD: type={msg_type} url={url} session={state.session_id}")
/home/ubuntu/pipecat/src/agent/handlers/image_upload.py:46:                    logger.error(f"IMAGE_TO_LLM_FAIL: {e}")
/home/ubuntu/pipecat/src/agent/handlers/image_upload.py:54:            logger.warning(f"DATA_RECEIVED_ERROR: {e}")
/home/ubuntu/pipecat/src/agent/handlers/credits.py:16:from loguru import logger
/home/ubuntu/pipecat/src/agent/handlers/credits.py:59:            logger.warning(f"REDIS_USAGE_COUNT_FAIL: device={device_id} error={e}")
/home/ubuntu/pipecat/src/agent/handlers/credits.py:88:        logger.debug(f"USAGE_CH_FALLBACK: device={device_id} content={content_type} count={count}")
/home/ubuntu/pipecat/src/agent/handlers/credits.py:92:        logger.error(f"CH_USAGE_FALLBACK_FAIL: device={device_id} error={e}")
/home/ubuntu/pipecat/src/agent/handlers/credits.py:114:            logger.debug(f"REDIS_SEEDED: device={device_id} content={content_type} entries={len(mapping)}")
/home/ubuntu/pipecat/src/agent/handlers/credits.py:117:        logger.warning(f"REDIS_SEED_FAIL: device={device_id} error={e}")
/home/ubuntu/pipecat/src/agent/handlers/credits.py:134:        logger.warning(f"REDIS_USAGE_RECORD_FAIL: device={device_id} error={e}")
/home/ubuntu/pipecat/src/agent/handlers/credits.py:168:        logger.debug(f"CREDIT_USED: type=plan remaining={state.plan_remaining} session={state.session_id}")
/home/ubuntu/pipecat/src/agent/handlers/credits.py:174:        logger.debug(f"CREDIT_USED: type=topup remaining={state.topup_remaining} session={state.session_id}")
/home/ubuntu/pipecat/src/agent/handlers/credits.py:180:        logger.debug(f"CREDIT_USED: type=free remaining={state.free_remaining} session={state.session_id}")
/home/ubuntu/pipecat/src/agent/handlers/credits.py:183:    logger.warning(f"CREDIT_EXHAUSTED: session={state.session_id}")
/home/ubuntu/pipecat/src/agent/handlers/credits.py:202:        logger.info(f"LIMIT_POPUP_SENT: type={popup_type} reason={reason} session={state.session_id}")
/home/ubuntu/pipecat/src/agent/handlers/credits.py:204:        logger.error(f"LIMIT_POPUP_FAILED: session={state.session_id} error={e}")
/home/ubuntu/pipecat/src/agent/plugins/veena_tts.py:10:from loguru import logger
/home/ubuntu/pipecat/src/agent/plugins/veena_tts.py:106:                    logger.error(f"VEENA_TTS_ERROR: status={response.status} error={error_text[:300]}")
/home/ubuntu/pipecat/src/agent/plugins/veena_tts.py:130:                                logger.warning("VEENA_TTS: WAV header but no 'data' chunk")
[result-id: r4]


1	"""Allow running as: python -m src.agent dev"""
2
3	from pathlib import Path
4	from dotenv import load_dotenv
5
6	PROJECT_ROOT = Path(__file__).parent.parent.parent
7	load_dotenv(PROJECT_ROOT / ".env")
8	load_dotenv(PROJECT_ROOT / ".env.local", override=True)
9
10	# Configure loguru to write to maya.log (don't use configure_logging — it adds
11	# duplicate sinks since subprocess stdout already pipes to maya.log)
12	import os
13	import sys
14	import logging
15	from loguru import logger
16
17	LOG_DIR = PROJECT_ROOT / "logs"
18	LOG_DIR.mkdir(exist_ok=True)
19	LOG_FILE = LOG_DIR / "maya.log"
20
21	logger.remove()
22	fmt = "{time:YYYY-MM-DD HH:mm:ss.SSS} | {level: <8} | {name}:{function}:{line} - {message}"
23	logger.add(LOG_FILE, level="DEBUG", format=fmt, enqueue=True, backtrace=True, diagnose=True)
24
25	# Intercept stdlib logging (livekit uses it)
26	class _InterceptHandler(logging.Handler):
27	    def emit(self, record):
28	        try:
29	            level = logger.level(record.levelname).name
30	        except ValueError:
31	            level = record.levelno
32	        frame, depth = sys._getframe(6), 6
33	        while frame and frame.f_code.co_filename == logging.__file__:
34	            frame = frame.f_back
35	            depth += 1
36	        logger.opt(depth=depth, exception=record.exc_info).log(level, record.getMessage())
37
38	logging.basicConfig(handlers=[_InterceptHandler()], level=logging.DEBUG, force=True)
39
40	from src.agent.core.worker import create_agent_server
41	from livekit.agents import cli
42
43	if __name__ == "__main__":
44	    server = create_agent_server()
45	    cli.run_app(server)
46
[result-id: r5]

"""Allow running as: python -m src.agent dev"""

from pathlib import Path
from dotenv import load_dotenv

PROJECT_ROOT = Path(__file__).parent.parent.parent
load_dotenv(PROJECT_ROOT / ".env")
load_dotenv(PROJECT_ROOT / ".env.local", override=True)

# Configure loguru to write to maya.log (don't use configure_logging — it adds
# duplicate sinks since subprocess stdout already pipes to maya.log)
import os
import sys
import logging
from loguru import logger

LOG_DIR = PROJECT_ROOT / "logs"
LOG_DIR.mkdir(exist_ok=True)
LOG_FILE = LOG_DIR / "maya.log"

logger.remove()
fmt = "{time:YYYY-MM-DD HH:mm:ss.SSS} | {level: <8} | {name}:{function}:{line} - {message}"
logger.add(LOG_FILE, level="DEBUG", format=fmt, enqueue=True, backtrace=True, diagnose=True)

# Intercept stdlib logging (livekit uses it)
class _InterceptHandler(logging.Handler):
    def emit(self, record):
        try:
            level = logger.level(record.levelname).name
        except ValueError:
            level = record.levelno
        frame, depth = sys._getframe(6), 6
        while frame and frame.f_code.co_filename == logging.__file__:
            frame = frame.f_back
            depth += 1
        logger.opt(depth=depth, exception=record.exc_info).log(level, record.getMessage())

logging.basicConfig(handlers=[_InterceptHandler()], level=logging.DEBUG, force=True)

from src.agent.core.worker import create_agent_server
from livekit.agents import cli

if __name__ == "__main__":
    server = create_agent_server()
    cli.run_app(server)


**Found the bug.** This is a critical issue.

The agent runs as a **separate process** from the FastAPI server. When the agent starts via `python -m src.agent`, it has its **own logger config that ONLY writes to file** (`LOG_FILE = logs/maya.log`) — **nothing to stdout**.

```python
logger.remove()
logger.add(LOG_FILE, ...)  # ← only writes to file, NO stdout
```

Datadog DaemonSet collects logs from **container stdout/stderr**. Since the agent only writes to a file inside the container (`/app/logs/maya.log`), Datadog never sees those logs.

## Compare with FastAPI server (which works fine in DD)

`src/main.py` calls `configure_logging()` which adds `DatadogStdoutSink()` — JSON logs to stdout → DD picks them up.

`src/agent/__main__.py` does NOT call `configure_logging()`. The comment even says:

```python
# Configure loguru to write to maya.log (don't use configure_logging — it adds
# duplicate sinks since subprocess stdout already pipes to maya.log)
```

That comment is **wrong** for production. It assumes the agent runs as a subprocess of the main server, where stdout is captured and re-piped. But in K8s, the agent runs as the main container process — its stdout IS the container stdout that DD collects.

## How the agent is started in the pod


40
41	@asynccontextmanager
42	async def lifespan(app: FastAPI):
43	    """Init Redis + ClickHouse, start LiveKit agent worker subprocess."""
44	    agent_proc = None
45	    try:
46	        from src.core.redis import init_redis
47	        from src.core.clickhouse import init_clickhouse
48	        await init_redis()
49	        await init_clickhouse()
50
51	        # Ensure experiments table exists in Turso
52	        try:
53	            from src.repository.experiment_repository import experiment_repo
54	            await experiment_repo.create_table()
55	        except Exception as e:
56	            logger.warning(f"EXPERIMENTS_TABLE_INIT_SKIP: {e}")
57
58	        # Start agent worker as subprocess (logs to logs/maya.log)
59	        log_dir = PROJECT_ROOT / "logs"
60	        log_dir.mkdir(exist_ok=True)
61	        agent_log = open(log_dir / "maya.log", "a")
62	        agent_proc = subprocess.Popen(
63	            [sys.executable, "-m", "src.agent", "dev"],
64	            cwd=str(PROJECT_ROOT),
65	            stdout=agent_log,
66	            stderr=subprocess.STDOUT,
67	            env={
68	                **os.environ,
69	                "LIVEKIT_URL": constants.LIVEKIT_URL,
70	                "LIVEKIT_API_KEY": constants.LIVEKIT_API_KEY,
71	                "LIVEKIT_API_SECRET": constants.LIVEKIT_API_SECRET,
72	            },
73	        )
74	        logger.info(f"AGENT_WORKER_STARTED: pid={agent_proc.pid}")
75	        yield
76	    finally:
77	        if agent_proc:
78	            agent_proc.terminate()
79	            agent_proc.wait(timeout=5)
[result-id: r6]

@asynccontextmanager
async def lifespan(app: FastAPI):
    """Init Redis + ClickHouse, start LiveKit agent worker subprocess."""
    agent_proc = None
    try:
        from src.core.redis import init_redis
        from src.core.clickhouse import init_clickhouse
        await init_redis()
        await init_clickhouse()

        # Ensure experiments table exists in Turso
        try:
            from src.repository.experiment_repository import experiment_repo
            await experiment_repo.create_table()
        except Exception as e:
            logger.warning(f"EXPERIMENTS_TABLE_INIT_SKIP: {e}")

        # Start agent worker as subprocess (logs to logs/maya.log)
        log_dir = PROJECT_ROOT / "logs"
        log_dir.mkdir(exist_ok=True)
        agent_log = open(log_dir / "maya.log", "a")
        agent_proc = subprocess.Popen(
            [sys.executable, "-m", "src.agent", "dev"],
            cwd=str(PROJECT_ROOT),
            stdout=agent_log,
            stderr=subprocess.STDOUT,
            env={
                **os.environ,
                "LIVEKIT_URL": constants.LIVEKIT_URL,
                "LIVEKIT_API_KEY": constants.LIVEKIT_API_KEY,
                "LIVEKIT_API_SECRET": constants.LIVEKIT_API_SECRET,
            },
        )
        logger.info(f"AGENT_WORKER_STARTED: pid={agent_proc.pid}")
        yield
    finally:
        if agent_proc:
            agent_proc.terminate()
            agent_proc.wait(timeout=5)


**Confirmed.** Look at lines 61-66:

```python
agent_log = open(log_dir / "maya.log", "a")
agent_proc = subprocess.Popen(
    [sys.executable, "-m", "src.agent", "dev"],
    stdout=agent_log,            # ← stdout REDIRECTED to file
    stderr=subprocess.STDOUT,    # ← stderr also to file
)
```

## Root cause

1. FastAPI server starts the agent as a **subprocess**
2. Agent's **stdout/stderr is redirected to `/app/logs/maya.log`** (a file inside the container)
3. **Nothing flows to the container's stdout** for the agent process
4. **Datadog DaemonSet only reads container stdout**, not files inside containers
5. → **All agent logs are invisible to Datadog**

## What you're missing in DD

Everything from the agent process:
- Image generation logs (`IMAGE_REQUEST`, `IMAGE_GENERATE_*`)
- Video generation logs
- Voice/audio pipeline metrics (TTFB, STT/TTS timing)
- Tool calls
- Per-session credit usage
- Image upload handler
- Veena TTS streaming logs
- All the LiveKit agent worker events

Only the FastAPI server (route handlers, health checks, ingress requests, lifespan setup) shows up in DD.

## Fix options

### Option A: Make agent log to stdout (cleanest)
Change `src/agent/__main__.py` to add a stdout sink (with the same JSON DD format as `src/core/logging.py`). Remove the file-only sink.

### Option B: Pipe subprocess stdout to parent stdout
In `src/main.py` lifespan, change:
```python
agent_log = open(log_dir / "maya.log", "a")
agent_proc = subprocess.Popen([...], stdout=agent_log, stderr=subprocess.STDOUT, ...)
```
to:
```python
agent_proc = subprocess.Popen([...], stdout=None, stderr=None, ...)
# stdout=None inherits parent's stdout (= container stdout = DD)
```

### Option C: Sidecar log forwarder (complex)
Use a sidecar container to tail `/app/logs/maya.log` and forward to DD. Don't recommend.

**Option B is the simplest** — one line change in `src/main.py`. Removes the redirect, agent's stdout becomes container stdout, DD picks it up automatically. The JSON format won't be there but DD parses plain text fine.

**Option A is the proper fix** — same JSON format the FastAPI server uses, so DD sees structured logs with same fields (`service`, `env`, `error.kind`, etc.).

Want me to write Option A (proper fix)? It's about 30 lines of code in `src/agent/__main__.py`.


yeah do


1	"""Maya LiveKit Server — FastAPI app with all routes + agent worker subprocess.
2
3	Entry point: python -m src.main
4	"""
5
6	import os
7	import subprocess
8	import sys
9	from contextlib import asynccontextmanager
10	from pathlib import Path
11
12	from dotenv import load_dotenv
13	from fastapi import FastAPI
14	from fastapi.middleware.cors import CORSMiddleware
15	from loguru import logger
16
17	PROJECT_ROOT = Path(__file__).parent.parent
18	load_dotenv(PROJECT_ROOT / ".env")
19	load_dotenv(PROJECT_ROOT / ".env.local", override=True)
20
21	from src.config import constants
22	from src.core.logging import configure_logging
23
24	configure_logging()
25	from src.routes import (
26	    bot_router,
27	    health_router,
28	    video_router,
29	    app_router,
30	    device_router,
31	    support_router,
32	    client_router,
33	    personas_router,
34	    usage_router,
35	    payment_router,
36	)
37
38
39	# ── Lifespan ──
40
41	@asynccontextmanager
42	async def lifespan(app: FastAPI):
43	    """Init Redis + ClickHouse, start LiveKit agent worker subprocess."""
44	    agent_proc = None
45	    try:
46	        from src.core.redis import init_redis
47	        from src.core.clickhouse import init_clickhouse
48	        await init_redis()
49	        await init_clickhouse()
50
51	        # Ensure experiments table exists in Turso
52	        try:
53	            from src.repository.experiment_repository import experiment_repo
54	            await experiment_repo.create_table()
55	        except Exception as e:
56	            logger.warning(f"EXPERIMENTS_TABLE_INIT_SKIP: {e}")
57
58	        # Start agent worker as subprocess (logs to logs/maya.log)
59	        log_dir = PROJECT_ROOT / "logs"
60	        log_dir.mkdir(exist_ok=True)
61	        agent_log = open(log_dir / "maya.log", "a")
62	        agent_proc = subprocess.Popen(
63	            [sys.executable, "-m", "src.agent", "dev"],
64	            cwd=str(PROJECT_ROOT),
65	            stdout=agent_log,
66	            stderr=subprocess.STDOUT,
67	            env={
68	                **os.environ,
69	                "LIVEKIT_URL": constants.LIVEKIT_URL,
70	                "LIVEKIT_API_KEY": constants.LIVEKIT_API_KEY,
71	                "LIVEKIT_API_SECRET": constants.LIVEKIT_API_SECRET,
72	            },
73	        )
74	        logger.info(f"AGENT_WORKER_STARTED: pid={agent_proc.pid}")
75	        yield
76	    finally:
77	        if agent_proc:
78	            agent_proc.terminate()
79	            agent_proc.wait(timeout=5)
80	        # Clean up any active SmallWebRTC sessions
81	        try:
82	            from src.agent.pipecat_bot import cleanup_smallwebrtc_sessions
83	            await cleanup_smallwebrtc_sessions()
84	        except Exception:
85	            pass
[result-id: r7]

"""Maya LiveKit Server — FastAPI app with all routes + agent worker subprocess.

Entry point: python -m src.main
"""

import os
import subprocess
import sys
from contextlib import asynccontextmanager
from pathlib import Path

from dotenv import load_dotenv
from fastapi import FastAPI
from fastapi.middleware.cors import CORSMiddleware
from loguru import logger

PROJECT_ROOT = Path(__file__).parent.parent
load_dotenv(PROJECT_ROOT / ".env")
load_dotenv(PROJECT_ROOT / ".env.local", override=True)

from src.config import constants
from src.core.logging import configure_logging

configure_logging()
from src.routes import (
    bot_router,
    health_router,
    video_router,
    app_router,
    device_router,
    support_router,
    client_router,
    personas_router,
    usage_router,
    payment_router,
)

# ── Lifespan ──

@asynccontextmanager
async def lifespan(app: FastAPI):
    """Init Redis + ClickHouse, start LiveKit agent worker subprocess."""
    agent_proc = None
    try:
        from src.core.redis import init_redis
        from src.core.clickhouse import init_clickhouse
        await init_redis()
        await init_clickhouse()

        # Ensure experiments table exists in Turso
        try:
            from src.repository.experiment_repository import experiment_repo
            await experiment_repo.create_table()
        except Exception as e:
            logger.warning(f"EXPERIMENTS_TABLE_INIT_SKIP: {e}")

        # Start agent worker as subprocess (logs to logs/maya.log)
        log_dir = PROJECT_ROOT / "logs"
        log_dir.mkdir(exist_ok=True)
        agent_log = open(log_dir / "maya.log", "a")
        agent_proc = subprocess.Popen(
            [sys.executable, "-m", "src.agent", "dev"],
            cwd=str(PROJECT_ROOT),
            stdout=agent_log,
            stderr=subprocess.STDOUT,
            env={
                **os.environ,
                "LIVEKIT_URL": constants.LIVEKIT_URL,
                "LIVEKIT_API_KEY": constants.LIVEKIT_API_KEY,
                "LIVEKIT_API_SECRET": constants.LIVEKIT_API_SECRET,
            },
        )
        logger.info(f"AGENT_WORKER_STARTED: pid={agent_proc.pid}")
        yield
    finally:
        if agent_proc:
            agent_proc.terminate()
            agent_proc.wait(timeout=5)
        # Clean up any active SmallWebRTC sessions
        try:
            from src.agent.pipecat_bot import cleanup_smallwebrtc_sessions
            await cleanup_smallwebrtc_sessions()
        except Exception:
            pass


Now I have the full picture. Let me make both fixes:

**Fix 1**: Remove the stdout redirect in `src/main.py` so agent inherits container stdout


Edit

        # Start agent worker as subprocess (logs to logs/maya.log)
        log_dir = PROJECT_ROOT / "logs"
        log_dir.mkdir(exist_ok=True)
        agent_log = open(log_dir / "maya.log", "a")
        agent_proc = subprocess.Popen(
            [sys.executable, "-m", "src.agent", "dev"],
            cwd=str(PROJECT_ROOT),
            stdout=agent_log,
            stderr=subprocess.STDOUT,
            env={
                **os.environ,
                "LIVEKIT_URL": constants.LIVEKIT_URL,
                "LIVEKIT_API_KEY": constants.LIVEKIT_API_KEY,
                "LIVEKIT_API_SECRET": constants.LIVEKIT_API_SECRET,
            },
        )

        # Start agent worker as subprocess.
        # stdout/stderr inherit parent (=container stdout) so Datadog agent picks up logs.
        agent_proc = subprocess.Popen(
            [sys.executable, "-m", "src.agent", "dev"],
            cwd=str(PROJECT_ROOT),
            stdout=None,
            stderr=None,
            env={
                **os.environ,
                "LIVEKIT_URL": constants.LIVEKIT_URL,
                "LIVEKIT_API_KEY": constants.LIVEKIT_API_KEY,
                "LIVEKIT_API_SECRET": constants.LIVEKIT_API_SECRET,
            },
        )


The file /home/ubuntu/pipecat/src/main.py has been updated successfully.

        # Start agent worker as subprocess.
        # stdout/stderr inherit parent (=container stdout) so Datadog agent picks up logs.
        agent_proc = subprocess.Popen(
            [sys.executable, "-m", "src.agent", "dev"],
            cwd=str(PROJECT_ROOT),
            stdout=None,
            stderr=None,
            env={
                **os.environ,
                "LIVEKIT_URL": constants.LIVEKIT_URL,
                "LIVEKIT_API_KEY": constants.LIVEKIT_API_KEY,
                "LIVEKIT_API_SECRET": constants.LIVEKIT_API_SECRET,
            },
        )

"""Maya LiveKit Server — FastAPI app with all routes + agent worker subprocess.

Entry point: python -m src.main
"""

import os
import subprocess
import sys
from contextlib import asynccontextmanager
from pathlib import Path

from dotenv import load_dotenv
from fastapi import FastAPI
from fastapi.middleware.cors import CORSMiddleware
from loguru import logger

PROJECT_ROOT = Path(__file__).parent.parent
load_dotenv(PROJECT_ROOT / ".env")
load_dotenv(PROJECT_ROOT / ".env.local", override=True)

from src.config import constants
from src.core.logging import configure_logging

configure_logging()
from src.routes import (
    bot_router,
    health_router,
    video_router,
    app_router,
    device_router,
    support_router,
    client_router,
    personas_router,
    usage_router,
    payment_router,
)

# ── Lifespan ──

@asynccontextmanager
async def lifespan(app: FastAPI):
    """Init Redis + ClickHouse, start LiveKit agent worker subprocess."""
    agent_proc = None
    try:
        from src.core.redis import init_redis
        from src.core.clickhouse import init_clickhouse
        await init_redis()
        await init_clickhouse()

        # Ensure experiments table exists in Turso
        try:
            from src.repository.experiment_repository import experiment_repo
            await experiment_repo.create_table()
        except Exception as e:
            logger.warning(f"EXPERIMENTS_TABLE_INIT_SKIP: {e}")

        # Start agent worker as subprocess (logs to logs/maya.log)
        log_dir = PROJECT_ROOT / "logs"
        log_dir.mkdir(exist_ok=True)
        agent_log = open(log_dir / "maya.log", "a")
        agent_proc = subprocess.Popen(
            [sys.executable, "-m", "src.agent", "dev"],
            cwd=str(PROJECT_ROOT),
            stdout=agent_log,
            stderr=subprocess.STDOUT,
            env={
                **os.environ,
                "LIVEKIT_URL": constants.LIVEKIT_URL,
                "LIVEKIT_API_KEY": constants.LIVEKIT_API_KEY,
                "LIVEKIT_API_SECRET": constants.LIVEKIT_API_SECRET,
            },
        )
        logger.info(f"AGENT_WORKER_STARTED: pid={agent_proc.pid}")
        yield
    finally:
        if agent_proc:
            agent_proc.terminate()
            agent_proc.wait(timeout=5)
        # Clean up any active SmallWebRTC sessions
        try:
            from src.agent.pipecat_bot import cleanup_smallwebrtc_sessions
            await cleanup_smallwebrtc_sessions()
        except Exception:
            pass
        from src.core.redis import close_redis
        from src.core.clickhouse import close_clickhouse
        await close_redis()
        await close_clickhouse()
        logger.info("SERVER_STOPPED")

# ── App ──

app = FastAPI(title="Maya LiveKit Server", version="6.0.0", lifespan=lifespan)

app.add_middleware(
    CORSMiddleware,
    allow_origins=["*"],
    allow_credentials=True,
    allow_methods=["*"],
    allow_headers=["*"],
)

# ── Mount all routers under /api/v6 ──

API_PREFIX = "/api/v6"

app.include_router(bot_router, prefix=API_PREFIX)
app.include_router(health_router, prefix=API_PREFIX)
app.include_router(personas_router, prefix=API_PREFIX)
app.include_router(payment_router, prefix=API_PREFIX)
app.include_router(usage_router, prefix=API_PREFIX)
app.include_router(client_router, prefix=API_PREFIX)
app.include_router(video_router, prefix=API_PREFIX)
app.include_router(app_router, prefix=API_PREFIX)
app.include_router(device_router, prefix=API_PREFIX)

# Pipecat SmallWebRTC fallback (lazy import — used when LiveKit is down)
# Endpoints: /api/v6/webrtc/start, /api/v6/webrtc/sessions/{id}/api/offer, etc.
try:
    from src.agent.pipecat_bot import router as webrtc_fallback_router
    app.include_router(webrtc_fallback_router, prefix=f"{API_PREFIX}/webrtc")
except Exception as e:
    logger.warning(f"WEBRTC_FALLBACK_SKIP: {e}")
app.include_router(support_router, prefix=API_PREFIX)

# ── Main ──

if __name__ == "__main__":
    import uvicorn
    uvicorn.run(app, host="0.0.0.0", port=constants.SERVER_PORT, log_level="info")

-        # Start agent worker as subprocess (logs to logs/maya.log)

-        log_dir = PROJECT_ROOT / "logs"

-        agent_log = open(log_dir / "maya.log", "a")

+        # Start agent worker as subprocess.

+        # stdout/stderr inherit parent (=container stdout) so Datadog agent picks up logs.


**Fix 2**: Update `src/agent/__main__.py` to write JSON logs to stdout (same format as `core/logging.py` for consistent DD parsing):


Write

"""Allow running as: python -m src.agent dev"""

import json
import logging
import os
import sys
import traceback
from datetime import timezone
from pathlib import Path

from dotenv import load_dotenv
from loguru import logger

PROJECT_ROOT = Path(__file__).parent.parent.parent
load_dotenv(PROJECT_ROOT / ".env")
load_dotenv(PROJECT_ROOT / ".env.local", override=True)

# ── Logging setup ──
# In production (DD_AGENT_HOST set): write JSON logs to stdout so the
# Datadog DaemonSet picks them up via container stdout collection.
# In dev: write human-readable logs to maya.log file.

LOG_DIR = PROJECT_ROOT / "logs"
LOG_DIR.mkdir(exist_ok=True)
LOG_FILE = LOG_DIR / "maya-agent.log"

IS_PRODUCTION = bool(os.getenv("DD_AGENT_HOST"))
LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO" if IS_PRODUCTION else "DEBUG")

def _log_filter(record):
    """Drop noise the same way src/core/logging.py does."""
    msg = record["message"]
    if "Unable to send message" in msg and "before joining" in msg:
        return False
    if record.get("exception"):
        exc_info = record["exception"]
        if exc_info and exc_info.value:
            exc_msg = str(exc_info.value)
            if "Unable to send message" in exc_msg and "before joining" in exc_msg:
                return False
    if record["level"].name in ("ERROR", "CRITICAL", "EXCEPTION"):
        return True
    name = record.get("name", "")
    if name and name.startswith(("botocore", "boto3", "urllib3", "s3transfer")):
        return False
    if name and name.startswith("openai"):
        return False
    if name and name.startswith("httpx"):
        return False
    if "Unclosed client session" in msg or "Unclosed connector" in msg:
        return False
    if "Task was destroyed but it is pending" in msg:
        return False
    return True

def _datadog_json_sink(message):
    """Format a loguru record as a single JSON line for Datadog."""
    record = message.record
    level = record["level"].name
    timestamp = record["time"].astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z"

    log_entry = {
        "timestamp": timestamp,
        "level": level,
        "logger": record["name"],
        "function": record["function"],
        "line": record["line"],
        "message": record["message"],
        "service": os.getenv("DD_SERVICE", "maya-pipecat-v6") + "-agent",
        "env": os.getenv("DD_ENV", os.getenv("ENVIRONMENT", "development")),
        "version": os.getenv("DD_VERSION", "1.0.0"),
        "process": "agent",
    }

    for key, value in record["extra"].items():
        if value and key not in ("_maya_configured",):
            log_entry[key] = value

    if level in ("ERROR", "CRITICAL", "EXCEPTION"):
        log_entry["status"] = "error"
        log_entry["error.kind"] = level
        if record.get("exception"):
            exc_info = record["exception"]
            if exc_info:
                log_entry["error.type"] = exc_info.type.__name__ if exc_info.type else "Unknown"
                log_entry["error.message"] = str(exc_info.value) if exc_info.value else ""
                log_entry["error.stack"] = "".join(
                    traceback.format_exception(exc_info.type, exc_info.value, exc_info.traceback)
                )
    else:
        log_entry["status"] = "info"

    sys.stdout.write(json.dumps(log_entry, ensure_ascii=False) + "\n")
    sys.stdout.flush()

logger.remove()

if IS_PRODUCTION:
    # Production: JSON to stdout for Datadog DaemonSet
    logger.add(_datadog_json_sink, level=LOG_LEVEL, filter=_log_filter)
else:
    # Dev: human-readable to file (separate from FastAPI maya.log to avoid conflicts)
    fmt = "{time:YYYY-MM-DD HH:mm:ss.SSS} | {level: <8} | {name}:{function}:{line} - {message}"
    logger.add(LOG_FILE, level="DEBUG", format=fmt, enqueue=True, backtrace=True, diagnose=True)

# ── Intercept stdlib logging (livekit, uvicorn, aiohttp use it) ──
class _InterceptHandler(logging.Handler):
    def emit(self, record):
        try:
            level = logger.level(record.levelname).name
        except ValueError:
            level = record.levelno
        frame, depth = sys._getframe(6), 6
        while frame and frame.f_code.co_filename == logging.__file__:
            frame = frame.f_back
            depth += 1
        logger.opt(depth=depth, exception=record.exc_info).log(level, record.getMessage())

logging.basicConfig(handlers=[_InterceptHandler()], level=logging.DEBUG, force=True)
for logger_name in ["livekit", "livekit.agents", "uvicorn", "fastapi", "aiohttp"]:
    logging.getLogger(logger_name).handlers = [_InterceptHandler()]
    logging.getLogger(logger_name).propagate = False

from src.agent.core.worker import create_agent_server
from livekit.agents import cli

if __name__ == "__main__":
    logger.info(f"AGENT_PROCESS_STARTING: pid={os.getpid()} production={IS_PRODUCTION} log_level={LOG_LEVEL}")
    server = create_agent_server()
    cli.run_app(server)