#!/usr/bin/env python3
"""
Deep Pattern Analysis: Find WHY content is being omitted
Analyze correlation between emotion tags, sentence structure, and missing content
"""

import re
import os
from openai import OpenAI

OPENAI_API_KEY = os.environ.get('OPENAI_API_KEY')
if not OPENAI_API_KEY:
    print("❌ OPENAI_API_KEY not set!")
    exit(1)

client = OpenAI(api_key=OPENAI_API_KEY)

# Original text
TEST_TEXT = """<excited> In the heart of an enchanted kingdom, where magic flowed like rivers and mythical beasts roamed freely, there lived a legendary warrior princess named Aisha. </excited> She was known throughout the land for her incredible courage, her mastery of ancient combat arts, and her ability to communicate with all living creatures.

<whisper> But few knew of the prophecy that surrounded her birth. </whisper> The ancient seers had foretold that she would one day face the greatest darkness the world had ever seen, and only she possessed the power to defeat it.

Aisha spent her childhood training in the sacred mountains with the wise monks. <curious> They taught her not just the physical arts of combat, but also the spiritual disciplines that would strengthen her mind and soul. </curious> Every dawn, she would practice her sword techniques as the sun rose over the peaks. Every evening, she would meditate under the stars, learning to harness the cosmic energies that flowed through all things.

<angry> One fateful day, dark clouds gathered over the kingdom! </angry> The evil demon lord Ravaksh had awakened from his thousand-year slumber. His army of shadow warriors began to sweep across the land, corrupting everything they touched. Villages were destroyed, forests withered, and despair spread like wildfire.

<sigh> The kingdom's armies fought valiantly, but they were no match for Ravaksh's dark magic. </sigh> One by one, the greatest warriors fell. The king's advisors urged him to flee, to abandon the kingdom and save what few lives they could.

<excited> But Aisha refused to give up! </excited> She knew this was the moment the prophecy had spoken of. She gathered the remaining defenders and devised a daring plan. They would launch a surprise attack on Ravaksh's fortress while he was still consolidating his power.

एक बार की बात है, ऐशा ने अपने सबसे भरोसेमंद योद्धाओं को इकट्ठा किया। उन्होंने एक खतरनाक योजना बनाई। वे रावक्ष के किले पर हमला करेंगे। यह असंभव लग रहा था, लेकिन उनके पास कोई विकल्प नहीं था। राज्य का भविष्य उनके हाथों में था।

<whisper> Under the cover of darkness, they approached the fortress. </whisper> The shadow guards patrolled the walls, their red eyes glowing in the night. <laugh> But Aisha had learned from the wind spirits how to move unseen! </laugh> She led her team through secret passages known only to the ancient monks.

The infiltration was successful. They reached the inner sanctum where Ravaksh was performing a dark ritual. <angry> He was trying to summon an even greater evil from the void! </angry> There was no time to waste.

<excited> Aisha drew her legendary sword, which began to glow with divine light! </excited> The blade had been forged by the celestial smiths themselves, imbued with the power to vanquish any darkness. As she charged forward, her companions engaged the shadow warriors, giving her a clear path to Ravaksh.

The battle that followed was epic. <curious> Ravaksh wielded powers that defied comprehension - he could bend reality itself, create illusions that seemed more real than reality, and summon storms of pure destruction. </curious> But Aisha had trained for this her entire life.

She dodged his dark bolts, deflected his curse spells, and pressed forward relentlessly. <whisper> She remembered the words of her master: "True strength comes not from power, but from purpose." </whisper> Her purpose was clear - protect her people, save her kingdom, restore the light.

With a mighty cry, she leaped high into the air, her sword blazing like a star. <excited> The blade struck true, piercing through Ravaksh's dark armor and into his corrupted heart! </excited> The demon lord let out a terrible scream as the divine light consumed him.

<laugh> The shadow army dissolved like smoke in the wind! </laugh> The dark clouds parted, and sunlight flooded the land once more. The corrupted forests began to heal, flowers bloomed, and the rivers ran clear again.

<excited> Aisha returned to her kingdom as a hero beyond measure! </excited> The people celebrated for seven days and seven nights. Songs were written about her bravery, statues were erected in her honor, and her legend would be told for a thousand generations.

<whisper> But Aisha remained humble. </whisper> She knew that true victory was not in defeating enemies, but in protecting those she loved. <sigh> She had seen too much loss, too much suffering. </sigh> From that day forward, she dedicated herself to teaching the next generation, ensuring they would be ready when darkness threatened again.

<excited> And so the kingdom prospered under her watchful protection, and peace reigned for many years to come! </excited>"""

# Get transcript
AUDIO_FILE = "fixed_test.wav"

print("="*80)
print("🔬 DEEP PATTERN ANALYSIS")
print("="*80)
print()

print("📝 Analyzing text structure...")

# Split into sentences
sentences = re.split(r'[.!?]+', TEST_TEXT)
sentences = [s.strip() for s in sentences if s.strip()]

print(f"   Total sentences: {len(sentences)}")

# Analyze emotion tag patterns
def analyze_emotion_patterns(text):
    """Analyze where and how emotion tags are used."""
    patterns = []
    
    # Find all emotion tags with context
    for match in re.finditer(r'(<[^>]+>)([^<]*?)(?=<|$)', text):
        tag = match.group(1)
        following_text = match.group(2).strip()
        
        # Check if opening or closing tag
        if tag.startswith('</'):
            tag_type = 'closing'
            emotion = tag[2:-1]
        else:
            tag_type = 'opening'
            emotion = tag[1:-1]
        
        # Get surrounding context
        start = match.start()
        context_before = text[max(0, start-50):start]
        context_after = following_text[:100] if following_text else ""
        
        patterns.append({
            'tag': tag,
            'emotion': emotion,
            'type': tag_type,
            'position': start,
            'following_text': following_text[:50],
            'text_length_after': len(following_text),
            'context_before': context_before[-30:] if context_before else "",
        })
    
    return patterns

emotion_patterns = analyze_emotion_patterns(TEST_TEXT)

print(f"   Emotion tags found: {len(emotion_patterns)}")
print()

# Analyze sentence + emotion combinations
print("📊 Sentence + Emotion Analysis:")
print()

sentence_analysis = []
for i, sent in enumerate(sentences):
    # Check emotion tags in this sentence
    tags_in_sent = re.findall(r'<([^/>]+)>', sent)
    has_emotions = len(tags_in_sent) > 0
    
    # Clean text
    clean_sent = re.sub(r'<[^>]+>', '', sent).strip()
    
    # Check sentence structure
    ends_with_emotion = re.search(r'<[^>]+>\s*$', sent.strip()) is not None
    starts_with_emotion = re.search(r'^\s*<[^>]+>', sent.strip()) is not None
    emotion_in_middle = '<' in sent and not (starts_with_emotion and ends_with_emotion)
    
    sentence_analysis.append({
        'index': i,
        'sentence': sent,
        'clean': clean_sent,
        'length': len(clean_sent),
        'emotions': tags_in_sent,
        'emotion_count': len(tags_in_sent),
        'starts_with_emotion': starts_with_emotion,
        'ends_with_emotion': ends_with_emotion,
        'emotion_in_middle': emotion_in_middle,
        'has_hindi': bool(re.search(r'[\u0900-\u097F]+', sent)),
    })

# Show sentences with multiple emotions
multi_emotion = [s for s in sentence_analysis if s['emotion_count'] > 1]
if multi_emotion:
    print(f"⚠️  Sentences with MULTIPLE emotions: {len(multi_emotion)}")
    for s in multi_emotion[:5]:
        print(f"   Sent {s['index']}: {s['emotion_count']} emotions")
        print(f"      {s['clean'][:80]}")
    print()

# Transcribe to find what's missing
print("🎧 Transcribing audio...")
with open(AUDIO_FILE, 'rb') as f:
    transcription = client.audio.transcriptions.create(
        model="whisper-1",
        file=f,
        response_format="text"
    )

transcript_lower = transcription.lower()
print("✅ Done")
print()

# Match sentences to transcript
print("🔍 Matching sentences to transcript...")
print()

matched = []
missing = []

for sent_data in sentence_analysis:
    clean = sent_data['clean'].lower()
    # Check if significant portion is in transcript (>50% of words)
    words = clean.split()
    if len(words) < 3:
        continue
    
    # Check if at least 60% of words are in transcript
    found_count = sum(1 for w in words if w in transcript_lower)
    match_ratio = found_count / len(words) if words else 0
    
    sent_data['in_transcript'] = match_ratio > 0.6
    sent_data['match_ratio'] = match_ratio
    
    if match_ratio > 0.6:
        matched.append(sent_data)
    else:
        missing.append(sent_data)

print(f"✅ Matched sentences: {len(matched)}/{len(sentence_analysis)}")
print(f"❌ Missing sentences: {len(missing)}/{len(sentence_analysis)}")
print()

# Analyze patterns in missing sentences
if missing:
    print("="*80)
    print("🚨 MISSING SENTENCE ANALYSIS")
    print("="*80)
    print()
    
    # Pattern 1: Emotion tag patterns
    missing_with_emotions = [s for s in missing if s['emotion_count'] > 0]
    missing_without_emotions = [s for s in missing if s['emotion_count'] == 0]
    
    print(f"Missing with emotions: {len(missing_with_emotions)}")
    print(f"Missing without emotions: {len(missing_without_emotions)}")
    print()
    
    # Pattern 2: Position
    missing_at_end = [s for s in missing if s['index'] > len(sentence_analysis) * 0.8]
    missing_in_middle = [s for s in missing if 0.2 < s['index']/len(sentence_analysis) < 0.8]
    
    print(f"Missing at END (last 20%): {len(missing_at_end)}")
    print(f"Missing in MIDDLE: {len(missing_in_middle)}")
    print()
    
    # Pattern 3: Emotion type
    emotion_counts = {}
    for s in missing_with_emotions:
        for emotion in s['emotions']:
            emotion_counts[emotion] = emotion_counts.get(emotion, 0) + 1
    
    if emotion_counts:
        print("Emotions in missing sentences:")
        for emotion, count in sorted(emotion_counts.items(), key=lambda x: -x[1]):
            print(f"   {emotion}: {count} times")
        print()
    
    # Pattern 4: Text after certain tags
    print("📍 Analyzing what comes AFTER emotion tags...")
    print()
    
    # Check if content after closing tags gets dropped
    for pattern in emotion_patterns:
        if pattern['type'] == 'closing' and pattern['text_length_after'] > 20:
            # Check if this following text is in transcript
            following = pattern['following_text'].lower()
            if following and len(following) > 10:
                in_trans = following[:20] in transcript_lower
                status = "✅" if in_trans else "❌"
                print(f"{status} After </{pattern['emotion']}>: '{following}'")
    
    print()
    
    # Show missing sentences
    print("="*80)
    print("❌ MISSING SENTENCES (first 10):")
    print("="*80)
    print()
    
    for i, s in enumerate(missing[:10]):
        print(f"{i+1}. Sentence {s['index']} ({s['emotion_count']} emotions)")
        print(f"   Emotions: {s['emotions']}")
        print(f"   Text: {s['clean'][:100]}")
        print(f"   Match ratio: {s['match_ratio']:.1%}")
        print()

# Final analysis
print("="*80)
print("💡 PATTERN IDENTIFICATION")
print("="*80)
print()

# Check specific patterns
wrapped_sentences = [s for s in sentence_analysis if s['starts_with_emotion'] and s['ends_with_emotion']]
wrapped_missing = [s for s in missing if s['starts_with_emotion'] and s['ends_with_emotion']]

print(f"Sentences WRAPPED in emotions (e.g., <tag> text </tag>):")
print(f"   Total: {len(wrapped_sentences)}")
print(f"   Missing: {len(wrapped_missing)}")
print(f"   Success rate: {(len(wrapped_sentences)-len(wrapped_missing))/len(wrapped_sentences)*100:.1f}%" if wrapped_sentences else "N/A")
print()

# Check sentences with emotion in middle
middle_emotion = [s for s in sentence_analysis if s['emotion_in_middle']]
middle_missing = [s for s in missing if s['emotion_in_middle']]

print(f"Sentences with emotion IN MIDDLE:")
print(f"   Total: {len(middle_emotion)}")
print(f"   Missing: {len(middle_missing)}")
print(f"   Success rate: {(len(middle_emotion)-len(middle_missing))/len(middle_emotion)*100:.1f}%" if middle_emotion else "N/A")
print()

# Check no-emotion sentences
no_emotion = [s for s in sentence_analysis if s['emotion_count'] == 0]
no_emotion_missing = [s for s in missing if s['emotion_count'] == 0]

print(f"Sentences with NO emotions:")
print(f"   Total: {len(no_emotion)}")
print(f"   Missing: {len(no_emotion_missing)}")
print(f"   Success rate: {(len(no_emotion)-len(no_emotion_missing))/len(no_emotion)*100:.1f}%" if no_emotion else "N/A")
print()

# Save detailed analysis
with open("pattern_analysis_report.txt", 'w') as f:
    f.write("DEEP PATTERN ANALYSIS REPORT\n")
    f.write("="*80 + "\n\n")
    
    f.write(f"Total sentences: {len(sentence_analysis)}\n")
    f.write(f"Matched: {len(matched)}\n")
    f.write(f"Missing: {len(missing)}\n\n")
    
    f.write("MISSING SENTENCES:\n")
    f.write("-"*80 + "\n")
    for s in missing:
        f.write(f"\nSentence {s['index']}: {s['clean']}\n")
        f.write(f"  Emotions: {s['emotions']}\n")
        f.write(f"  Emotion count: {s['emotion_count']}\n")
        f.write(f"  Starts with emotion: {s['starts_with_emotion']}\n")
        f.write(f"  Ends with emotion: {s['ends_with_emotion']}\n")
        f.write(f"  Match ratio: {s['match_ratio']:.1%}\n")

print("📄 Detailed report: pattern_analysis_report.txt")
print()