#!/usr/bin/env python3
"""
Qdrant Retrieval Quality Evaluation Script
==========================================
Bu script, Qdrant vektör veritabanının arama kalitesini ölçer.

Metrikler:
- Hit Rate @ K: Beklenen sonuç ilk K sonuç içinde mi?
- MRR (Mean Reciprocal Rank): Beklenen sonuç kaçıncı sırada?

Kullanım:
    export OPENAI_API_KEY="sk-..."
    python3 evaluate_qdrant.py

Gereksinimler:
    pip install qdrant-client openai pandas tabulate colorama
"""

import os
import sys
import json
import time
from typing import List, Dict, Optional, Tuple

# Renk kodları için
try:
    from colorama import init, Fore, Style
    init(autoreset=True)
    HAS_COLORS = True
except ImportError:
    HAS_COLORS = False
    class Fore:
        RED = GREEN = YELLOW = CYAN = MAGENTA = WHITE = RESET = ""
    class Style:
        BRIGHT = RESET_ALL = ""

try:
    from qdrant_client import QdrantClient
    from qdrant_client.models import Distance, VectorParams
except ImportError:
    print(f"{Fore.RED}❌ qdrant-client yüklü değil: pip install qdrant-client{Style.RESET_ALL}")
    sys.exit(1)

try:
    from openai import OpenAI
except ImportError:
    print(f"{Fore.RED}❌ openai yüklü değil: pip install openai{Style.RESET_ALL}")
    sys.exit(1)

try:
    from tabulate import tabulate
except ImportError:
    print(f"{Fore.RED}❌ tabulate yüklü değil: pip install tabulate{Style.RESET_ALL}")
    sys.exit(1)

try:
    import pandas as pd
except ImportError:
    pd = None
    print(f"{Fore.YELLOW}⚠️ pandas yüklü değil, basit mod kullanılacak{Style.RESET_ALL}")


# ==================== CONFIGURATION ====================

QDRANT_HOST = "10.10.10.25"
QDRANT_PORT = 6333
COLLECTION_NAME = "machine_docs"

EMBEDDING_MODEL = "text-embedding-3-large"
EMBEDDING_DIM = 3072

SEARCH_LIMIT = 5  # Top-K sonuç

# ==================== HELPER FUNCTIONS ====================

def get_openai_client() -> OpenAI:
    """OpenAI client oluştur"""
    api_key = os.getenv("OPENAI_API_KEY")
    if not api_key:
        print(f"{Fore.RED}❌ OPENAI_API_KEY ortam değişkeni tanımlı değil!{Style.RESET_ALL}")
        print(f"{Fore.YELLOW}   export OPENAI_API_KEY='sk-...' komutunu çalıştırın{Style.RESET_ALL}")
        sys.exit(1)
    return OpenAI(api_key=api_key)


def get_embedding(client: OpenAI, text: str) -> List[float]:
    """Metni vektöre çevir"""
    try:
        response = client.embeddings.create(
            input=text,
            model=EMBEDDING_MODEL
        )
        embedding = response.data[0].embedding
        
        if len(embedding) != EMBEDDING_DIM:
            print(f"{Fore.YELLOW}⚠️ Beklenmeyen vektör boyutu: {len(embedding)} (beklenen: {EMBEDDING_DIM}){Style.RESET_ALL}")
        
        return embedding
    except Exception as e:
        print(f"{Fore.RED}❌ Embedding hatası: {e}{Style.RESET_ALL}")
        return []


def search_qdrant(qdrant: QdrantClient, query_vector: List[float], limit: int = SEARCH_LIMIT) -> List[Dict]:
    """Qdrant'ta arama yap"""
    try:
        # Yeni qdrant-client versiyonu için query_points kullan
        from qdrant_client.models import PointStruct
        
        results = qdrant.query_points(
            collection_name=COLLECTION_NAME,
            query=query_vector,
            limit=limit,
            with_payload=True
        )
        
        return [
            {
                "score": hit.score,
                "pdf_filename": hit.payload.get("pdf_filename", hit.payload.get("pdf_name", "")),
                "page_number": hit.payload.get("page_number", hit.payload.get("page", None)),
                "text": (hit.payload.get("text", hit.payload.get("content", ""))[:100] + "...")
            }
            for hit in results.points
        ]
    except Exception as e:
        print(f"{Fore.RED}❌ Qdrant arama hatası: {e}{Style.RESET_ALL}")
        return []


def check_hit(results: List[Dict], expected_file: str, expected_page: Optional[int]) -> Tuple[bool, int]:
    """
    Beklenen dosyanın sonuçlarda olup olmadığını kontrol et.
    
    Returns:
        (hit: bool, rank: int) - hit varsa True ve sırası, yoksa False ve 0
    """
    for i, result in enumerate(results):
        result_file = result.get("pdf_filename", "")
        result_page = result.get("page_number")
        
        # Dosya adı eşleşmesi (case-insensitive, kısmi eşleşme)
        file_match = (
            expected_file.lower() in result_file.lower() or
            result_file.lower() in expected_file.lower()
        )
        
        # Sayfa eşleşmesi (opsiyonel)
        if expected_page is not None:
            page_match = (result_page == expected_page)
            if file_match and page_match:
                return True, i + 1
        else:
            if file_match:
                return True, i + 1
    
    return False, 0


def calculate_mrr(ranks: List[int]) -> float:
    """Mean Reciprocal Rank hesapla"""
    reciprocal_ranks = []
    for rank in ranks:
        if rank > 0:
            reciprocal_ranks.append(1.0 / rank)
        else:
            reciprocal_ranks.append(0.0)
    
    if not reciprocal_ranks:
        return 0.0
    
    return sum(reciprocal_ranks) / len(reciprocal_ranks)


def load_evaluation_data(filepath: str) -> List[Dict]:
    """Test verisini yükle"""
    try:
        with open(filepath, 'r', encoding='utf-8') as f:
            data = json.load(f)
        print(f"{Fore.GREEN}✓ {len(data)} test sorgusu yüklendi{Style.RESET_ALL}")
        return data
    except FileNotFoundError:
        print(f"{Fore.RED}❌ Dosya bulunamadı: {filepath}{Style.RESET_ALL}")
        sys.exit(1)
    except json.JSONDecodeError as e:
        print(f"{Fore.RED}❌ JSON parse hatası: {e}{Style.RESET_ALL}")
        sys.exit(1)


# ==================== MAIN EVALUATION ====================

def run_evaluation(test_data: List[Dict], openai_client: OpenAI, qdrant_client: QdrantClient) -> Dict:
    """
    Ana değerlendirme döngüsü
    """
    results = []
    hits = 0
    ranks = []
    failed_queries = []
    
    print(f"\n{Fore.CYAN}{'='*60}{Style.RESET_ALL}")
    print(f"{Fore.CYAN}   🔍 RETRIEVAL KALİTE DEĞERLENDİRMESİ BAŞLIYOR{Style.RESET_ALL}")
    print(f"{Fore.CYAN}{'='*60}{Style.RESET_ALL}\n")
    
    total = len(test_data)
    
    for i, item in enumerate(test_data, 1):
        query = item.get("query", "")
        expected_file = item.get("expected_file", "")
        expected_page = item.get("expected_page")
        
        print(f"{Fore.WHITE}[{i}/{total}] Sorgu: {query[:50]}...{Style.RESET_ALL}", end=" ")
        
        # Embedding oluştur
        start_time = time.time()
        query_vector = get_embedding(openai_client, query)
        embed_time = time.time() - start_time
        
        if not query_vector:
            print(f"{Fore.RED}HATA (embedding){Style.RESET_ALL}")
            failed_queries.append({
                "query": query,
                "expected": expected_file,
                "error": "Embedding oluşturulamadı"
            })
            continue
        
        # Qdrant'ta ara
        start_time = time.time()
        search_results = search_qdrant(qdrant_client, query_vector)
        search_time = time.time() - start_time
        
        if not search_results:
            print(f"{Fore.RED}HATA (arama){Style.RESET_ALL}")
            failed_queries.append({
                "query": query,
                "expected": expected_file,
                "error": "Arama sonuç döndürmedi"
            })
            continue
        
        # Hit kontrolü
        hit, rank = check_hit(search_results, expected_file, expected_page)
        
        if hit:
            hits += 1
            ranks.append(rank)
            status = f"{Fore.GREEN}✓ HIT @ {rank}{Style.RESET_ALL}"
        else:
            ranks.append(0)
            status = f"{Fore.RED}✗ MISS{Style.RESET_ALL}"
            
            # Başarısız sorguları kaydet
            failed_queries.append({
                "query": query,
                "expected": expected_file,
                "got": [r.get("pdf_filename", "?")[:40] for r in search_results[:3]],
                "scores": [f"{r.get('score', 0):.3f}" for r in search_results[:3]]
            })
        
        print(f"{status} ({embed_time*1000:.0f}ms + {search_time*1000:.0f}ms)")
        
        results.append({
            "query": query[:40] + "...",
            "expected": expected_file[:30] + "...",
            "hit": "✓" if hit else "✗",
            "rank": rank if rank > 0 else "-",
            "top_result": search_results[0].get("pdf_filename", "")[:30] + "..." if search_results else "-",
            "score": f"{search_results[0].get('score', 0):.3f}" if search_results else "-"
        })
        
        # Rate limiting
        time.sleep(0.2)
    
    # Metrikleri hesapla
    hit_rate = (hits / total) * 100 if total > 0 else 0
    mrr = calculate_mrr(ranks)
    
    return {
        "results": results,
        "metrics": {
            "total_queries": total,
            "hits": hits,
            "hit_rate": hit_rate,
            "mrr": mrr
        },
        "failed_queries": failed_queries
    }


def print_results(evaluation: Dict):
    """Sonuçları güzel bir şekilde yazdır"""
    
    results = evaluation["results"]
    metrics = evaluation["metrics"]
    failed = evaluation["failed_queries"]
    
    # Ana sonuç tablosu
    print(f"\n{Fore.CYAN}{'='*80}{Style.RESET_ALL}")
    print(f"{Fore.CYAN}   📊 SONUÇ TABLOSU{Style.RESET_ALL}")
    print(f"{Fore.CYAN}{'='*80}{Style.RESET_ALL}\n")
    
    headers = ["Sorgu", "Beklenen", "Hit", "Rank", "Dönen Sonuç", "Skor"]
    table_data = [
        [r["query"], r["expected"], r["hit"], r["rank"], r["top_result"], r["score"]]
        for r in results
    ]
    
    print(tabulate(table_data, headers=headers, tablefmt="pretty"))
    
    # Metrik özeti
    print(f"\n{Fore.CYAN}{'='*80}{Style.RESET_ALL}")
    print(f"{Fore.CYAN}   📈 METRİK ÖZETİ{Style.RESET_ALL}")
    print(f"{Fore.CYAN}{'='*80}{Style.RESET_ALL}\n")
    
    hit_rate = metrics["hit_rate"]
    mrr = metrics["mrr"]
    
    # Renk kodlaması
    if hit_rate >= 80:
        hr_color = Fore.GREEN
    elif hit_rate >= 60:
        hr_color = Fore.YELLOW
    else:
        hr_color = Fore.RED
    
    if mrr >= 0.8:
        mrr_color = Fore.GREEN
    elif mrr >= 0.5:
        mrr_color = Fore.YELLOW
    else:
        mrr_color = Fore.RED
    
    print(f"   Toplam Sorgu     : {metrics['total_queries']}")
    print(f"   Başarılı Hit     : {metrics['hits']}")
    print(f"   {Style.BRIGHT}Hit Rate @ 5    : {hr_color}{hit_rate:.1f}%{Style.RESET_ALL}")
    print(f"   {Style.BRIGHT}MRR             : {mrr_color}{mrr:.4f}{Style.RESET_ALL}")
    
    # Değerlendirme
    print(f"\n   {Fore.WHITE}Değerlendirme:{Style.RESET_ALL}")
    if hit_rate >= 80 and mrr >= 0.7:
        print(f"   {Fore.GREEN}✓ Mükemmel! Retrieval kalitesi çok iyi.{Style.RESET_ALL}")
    elif hit_rate >= 60 and mrr >= 0.5:
        print(f"   {Fore.YELLOW}⚠ Orta seviye. İyileştirme yapılabilir.{Style.RESET_ALL}")
    else:
        print(f"   {Fore.RED}✗ Düşük kalite. Embedding veya veri kalitesi gözden geçirilmeli.{Style.RESET_ALL}")
    
    # Başarısız sorgular
    if failed:
        print(f"\n{Fore.RED}{'='*80}{Style.RESET_ALL}")
        print(f"{Fore.RED}   ❌ BAŞARISIZ SORGULAR ({len(failed)} adet){Style.RESET_ALL}")
        print(f"{Fore.RED}{'='*80}{Style.RESET_ALL}\n")
        
        for i, f in enumerate(failed, 1):
            print(f"   {Fore.RED}{i}. {f['query'][:60]}{Style.RESET_ALL}")
            print(f"      Beklenen : {f.get('expected', '-')}")
            if 'got' in f:
                print(f"      Dönen    : {', '.join(f['got'])}")
                print(f"      Skorlar  : {', '.join(f['scores'])}")
            if 'error' in f:
                print(f"      Hata     : {f['error']}")
            print()


def main():
    """Ana fonksiyon"""
    
    print(f"\n{Fore.MAGENTA}{'='*60}{Style.RESET_ALL}")
    print(f"{Fore.MAGENTA}   🎯 QDRANT RETRIEVAL KALİTE DEĞERLENDİRMESİ{Style.RESET_ALL}")
    print(f"{Fore.MAGENTA}{'='*60}{Style.RESET_ALL}")
    
    # Konfigurasyon bilgisi
    print(f"\n{Fore.WHITE}📌 Konfigürasyon:{Style.RESET_ALL}")
    print(f"   Qdrant        : {QDRANT_HOST}:{QDRANT_PORT}")
    print(f"   Collection    : {COLLECTION_NAME}")
    print(f"   Embedding     : {EMBEDDING_MODEL} ({EMBEDDING_DIM}D)")
    print(f"   Search Limit  : Top-{SEARCH_LIMIT}")
    
    # Test verisini yükle
    script_dir = os.path.dirname(os.path.abspath(__file__))
    eval_data_path = os.path.join(script_dir, "evaluation_data.json")
    
    print(f"\n{Fore.WHITE}📂 Test verisi yükleniyor...{Style.RESET_ALL}")
    test_data = load_evaluation_data(eval_data_path)
    
    # OpenAI client
    print(f"{Fore.WHITE}🔑 OpenAI bağlantısı kuruluyor...{Style.RESET_ALL}")
    openai_client = get_openai_client()
    
    # Qdrant client
    print(f"{Fore.WHITE}🔗 Qdrant bağlantısı kuruluyor...{Style.RESET_ALL}")
    try:
        qdrant_client = QdrantClient(host=QDRANT_HOST, port=QDRANT_PORT, timeout=30)
        
        # Bağlantı testi
        collections = qdrant_client.get_collections()
        collection_names = [c.name for c in collections.collections]
        
        if COLLECTION_NAME not in collection_names:
            print(f"{Fore.RED}❌ Collection '{COLLECTION_NAME}' bulunamadı!{Style.RESET_ALL}")
            print(f"   Mevcut collection'lar: {collection_names}")
            sys.exit(1)
        
        # Collection bilgisi
        info = qdrant_client.get_collection(COLLECTION_NAME)
        print(f"{Fore.GREEN}✓ Qdrant bağlantısı başarılı ({info.points_count:,} vektör){Style.RESET_ALL}")
        
    except Exception as e:
        print(f"{Fore.RED}❌ Qdrant bağlantı hatası: {e}{Style.RESET_ALL}")
        sys.exit(1)
    
    # Değerlendirmeyi çalıştır
    evaluation = run_evaluation(test_data, openai_client, qdrant_client)
    
    # Sonuçları yazdır
    print_results(evaluation)
    
    # Özet
    print(f"\n{Fore.CYAN}{'='*60}{Style.RESET_ALL}")
    print(f"{Fore.CYAN}   ✅ DEĞERLENDİRME TAMAMLANDI{Style.RESET_ALL}")
    print(f"{Fore.CYAN}{'='*60}{Style.RESET_ALL}\n")


if __name__ == "__main__":
    main()