<?php
/**
 * IELearningEngine — Data-driven IE Optimization (Phase 5)
 *
 * Reads REAL production data from:
 *   - ai_logs    → query text, source (ie/ai/cache), latency, service
 *   - ai_requests → ie_hit, ie_confidence, ai_used, latency_ms
 *   - ai_usage   → cost per call, tokens
 *
 * Produces SUGGESTIONS only — does NOT modify InferenceEngine.
 *
 * Usage:
 *   php engine/IELearningEngine.php           (analyze last 1000)
 *   php engine/IELearningEngine.php --n=5000  (analyze last N)
 *   php engine/IELearningEngine.php --dry-run (no DB write)
 */

require_once __DIR__ . '/../config.php';
require_once __DIR__ . '/InferenceEngine.php';
require_once __DIR__ . '/Logger.php';

class IELearningEngine
{
    // ── Thresholds ─────────────────────────────────────────
    const MIN_FREQUENCY     = 3;       // min times a query repeats before suggesting
    const MIN_SLOW_MS       = 2000;    // latency > 2s = slow (AI path)
    const CANDIDATE_CONF_LO = 0.72;    // new rule confidence range
    const CANDIDATE_CONF_HI = 0.78;
    const MAX_PATTERN_LEN   = 60;      // max chars in generated pattern
    const MAX_CANDIDATES    = 20;      // max rules to suggest

    // ── Normalizer: must mirror InferenceEngine::normalize() ──
    public static function normalize(string $q): string {
        $q = mb_strtolower(trim($q));
        $q = preg_replace('/[\x{0610}-\x{061A}\x{064B}-\x{065F}\x{0670}]/u', '', $q) ?? $q;
        $q = preg_replace('/[\x{0622}\x{0623}\x{0625}\x{0671}]/u', 'ا', $q) ?? $q;
        $q = str_replace(['ى','ة'], ['ي','ه'], $q);
        return trim($q);
    }

    // ══════════════════════════════════════════════════════════
    // TASK 1 — Extract & Group Queries
    // ══════════════════════════════════════════════════════════
    public static function extractQueryData(int $limit = 1000): array
    {
        // Pull queries that went to AI (not IE/cache hits)
        $sql = "
            SELECT
                l.question,
                l.source,
                l.service,
                l.latency_ms,
                r.ie_hit,
                r.ie_confidence,
                r.ai_used,
                r.latency_ms   AS req_latency_ms,
                u.cost_usd     AS call_cost_usd,
                u.tokens_in + u.tokens_out AS tokens_total
            FROM ai_logs l
            LEFT JOIN ai_requests r
                ON r.service = l.service
                AND ABS(TIMESTAMPDIFF(SECOND, l.created_at, r.created_at)) <= 2
            LEFT JOIN ai_usage u
                ON u.trace_id = r.request_uuid
            WHERE l.question IS NOT NULL
              AND l.question != ''
              AND l.source NOT IN ('ie', 'cache', 'ie_direct', 'bank_qa')
              AND l.created_at >= DATE_SUB(NOW(), INTERVAL 30 DAY)
            ORDER BY l.created_at DESC
            LIMIT ?
        ";

        try {
            $stmt = db()->prepare($sql);
            $stmt->execute([$limit]);
            $rows = $stmt->fetchAll(\PDO::FETCH_ASSOC);
        } catch (\Throwable $e) {
            error_log('[IELearning] DB error: ' . $e->getMessage());
            return [];
        }

        // Group and count
        $groups = [];
        foreach ($rows as $row) {
            $q      = trim($row['question'] ?? '');
            if (strlen($q) < 3 || strlen($q) > 200) continue;

            $q_norm = self::normalize($q);
            $key    = $q_norm;

            if (!isset($groups[$key])) {
                $groups[$key] = [
                    'query_raw'    => $q,
                    'query_norm'   => $q_norm,
                    'frequency'    => 0,
                    'service'      => $row['service'] ?? 'general',
                    'source'       => $row['source']  ?? 'ai',
                    'total_cost'   => 0.0,
                    'total_tokens' => 0,
                    'avg_latency'  => 0,
                    'latencies'    => [],
                    'ie_hits'      => 0,
                    'ai_hits'      => 0,
                    'ie_confs'     => [],
                ];
            }

            $groups[$key]['frequency']++;
            $groups[$key]['total_cost']   += (float)($row['call_cost_usd'] ?? 0);
            $groups[$key]['total_tokens'] += (int)($row['tokens_total']   ?? 0);
            $latency = (int)($row['req_latency_ms'] ?? $row['latency_ms'] ?? 0);
            if ($latency > 0) $groups[$key]['latencies'][] = $latency;
            if ($row['ie_hit'])  $groups[$key]['ie_hits']++;
            else                  $groups[$key]['ai_hits']++;
            if ($row['ie_confidence'] > 0) $groups[$key]['ie_confs'][] = (float)$row['ie_confidence'];
        }

        // Compute averages
        foreach ($groups as &$g) {
            $g['avg_latency'] = $g['latencies']
                ? round(array_sum($g['latencies']) / count($g['latencies']))
                : 0;
            $g['avg_ie_conf'] = $g['ie_confs']
                ? round(array_sum($g['ie_confs']) / count($g['ie_confs']), 2)
                : 0.0;
        }

        // Sort by frequency desc
        uasort($groups, fn($a,$b) => $b['frequency'] <=> $a['frequency']);
        return $groups;
    }

    // ══════════════════════════════════════════════════════════
    // TASK 2 — Identify Candidates (freq ≥ MIN, AI-handled)
    // ══════════════════════════════════════════════════════════
    public static function identifyCandidates(array $groups): array
    {
        $candidates = [];
        foreach ($groups as $key => $g) {
            // Only queries that:
            // 1. Appear ≥ MIN_FREQUENCY times
            // 2. Are primarily handled by AI (not IE)
            // 3. Are not too short or too long to pattern
            if ($g['frequency'] < self::MIN_FREQUENCY) continue;
            if ($g['ai_hits'] == 0) continue;  // already IE-handled
            $q = $g['query_norm'];
            if (mb_strlen($q) < 4 || mb_strlen($q) > 120) continue;

            $candidates[] = array_merge($g, ['key' => $key]);
        }
        return array_slice($candidates, 0, 50);  // pre-limit
    }

    // ══════════════════════════════════════════════════════════
    // TASK 3 — Generate Safe Pattern for a Query
    // ══════════════════════════════════════════════════════════
    public static function generatePattern(string $q_norm): ?string
    {
        // Strategy: extract meaningful keywords (≥3 chars Arabic or ≥4 Latin)
        // Avoid function words (في، من، على، مع، أن، لأن...)
        $stop_words = [
            'في','من','على','مع','أن','لأن','لكن','هذا','هذه','ذلك','تلك',
            'إذا','حين','عند','بعد','قبل','كيف','لما','لماذا','هل','ما',
            'ماذا','أين','متى','من','لم','لن','كان','كانت','يكون',
            'the','and','or','for','but','what','how','when','where','why',
        ];

        $words = preg_split('/\s+/u', trim($q_norm));
        $keywords = [];
        foreach ($words as $w) {
            $w = trim($w);
            $len = mb_strlen($w);
            if ($len < 3) continue;
            if (in_array($w, $stop_words)) continue;
            // Arabic word ≥ 3 chars or Latin ≥ 4 chars
            if ((preg_match('/[\x{0600}-\x{06ff}]/u', $w) && $len >= 3) ||
                (preg_match('/[a-z]/i', $w) && $len >= 4)) {
                $keywords[] = $w;
            }
        }

        if (count($keywords) < 1) return null;

        // Take up to 3 keywords, join with .*
        $selected = array_slice($keywords, 0, 3);
        if (count($selected) == 1) {
            // Single keyword: too risky as standalone → require context
            return null;
        }

        // Build pattern
        $pat = implode('.*', $selected);
        if (mb_strlen($pat) > self::MAX_PATTERN_LEN) {
            // Too long: use first 2 keywords only
            $pat = implode('.*', array_slice($selected, 0, 2));
        }
        if (mb_strlen($pat) > self::MAX_PATTERN_LEN) return null;

        // Add ة variants for Arabic patterns
        $pat_with_variants = preg_replace_callback(
            '/[\x{0600}-\x{06ff}]+/u',
            fn($m) => str_contains($m[0], 'ه') ? $m[0] . '|' . str_replace('ه','ة',$m[0]) : $m[0],
            $pat
        );

        return '/' . $pat . '/ui';
    }

    // ══════════════════════════════════════════════════════════
    // TASK 4 — Validate: no conflict with existing IE rules
    // ══════════════════════════════════════════════════════════
    public static function validateNoConflict(string $pattern, array $existing_rules): array
    {
        // Test 20 common safe queries — pattern must NOT match them
        $safe_queries = [
            'أريد مساعدة','كيف حالك','مرحبا','شكراً','ما هي سودانا',
            'كم الرصيد','اشحن خط','ادفع كهرباء','حقوقي في العمل',
        ];

        $core = preg_replace('/^\/(.+)\/[iuIsUI]*$/', '$1', $pattern);

        foreach ($safe_queries as $q_safe) {
            $q_norm = self::normalize($q_safe);
            try {
                if (@preg_match('/' . $core . '/ui', $q_norm)) {
                    return ['ok' => false, 'reason' => "Matches safe query: '$q_safe'"];
                }
            } catch (\Throwable $e) {
                return ['ok' => false, 'reason' => 'Invalid regex: ' . $e->getMessage()];
            }
        }

        // Check against existing IE rules by testing overlap
        foreach ($existing_rules as $rule) {
            $rule_core = preg_replace('/^\/(.+)\/[iuIsUI]*$/', '$1', $rule['pattern']);
            // Rough test: if both patterns share 2+ significant keywords → possible overlap
            $pat_words  = preg_split('/[\W.*|]+/u', $core, -1, PREG_SPLIT_NO_EMPTY);
            $rule_words = preg_split('/[\W.*|]+/u', $rule_core, -1, PREG_SPLIT_NO_EMPTY);
            $shared = array_intersect(
                array_filter($pat_words,  fn($w) => mb_strlen($w) >= 3),
                array_filter($rule_words, fn($w) => mb_strlen($w) >= 3)
            );
            if (count($shared) >= 2 && $rule['sector'] !== ($rule['target_sector'] ?? '')) {
                // Different sector + 2 shared words = potential conflict
                return ['ok' => false, 'reason' => "Overlaps with {$rule['id']} ({$rule['sector']})"];
            }
        }

        return ['ok' => true, 'reason' => ''];
    }

    // ══════════════════════════════════════════════════════════
    // TASK 5 — Infer Domain from ai_logs.service
    // ══════════════════════════════════════════════════════════
    public static function inferDomain(string $service, string $q_norm): string
    {
        // ai_logs.service is already the domain (set by Gateway/AssistantEngine)
        $service_map = [
            'medical' => 'medical', 'pharmacy' => 'medical', 'family' => 'medical',
            'legal'   => 'legal',
            'pay'     => 'pay',   'wallet' => 'pay',
            'finance' => 'finance', 'banking' => 'finance',
            'career'  => 'career',
            'education' => 'education', 'academic' => 'education',
            'mental'  => 'mental',
            'agriculture' => 'agriculture',
            'community' => 'community',
            'market'  => 'market',
            'developer' => 'developer', 'tech' => 'developer',
            'enterprise' => 'enterprise',
            'gov'     => 'pay',
            'general' => 'hub', 'platform' => 'hub',
        ];
        return $service_map[$service] ?? 'hub';
    }

    // ══════════════════════════════════════════════════════════
    // MAIN: Full analysis + report
    // ══════════════════════════════════════════════════════════
    public static function analyze(int $n = 1000, bool $verbose = true): array
    {
        $t_start = microtime(true);

        // Step 1: Extract
        if ($verbose) echo "→ Extracting last {$n} AI-handled queries...\n";
        $groups = self::extractQueryData($n);

        if (empty($groups)) {
            return [
                'status'    => 'NO_DATA',
                'message'   => 'No AI-handled queries found in ai_logs. System may be new or DB empty.',
                'candidates'=> [],
                'report'    => [],
            ];
        }

        // Step 2: Identify candidates
        if ($verbose) echo "→ Identifying candidates (freq ≥ " . self::MIN_FREQUENCY . ")...\n";
        $candidates_raw = self::identifyCandidates($groups);

        // Step 3: Load existing IE rules for conflict check
        if ($verbose) echo "→ Loading existing IE rules for conflict check...\n";
        $ie_content = file_get_contents(__DIR__ . '/InferenceEngine.php');
        $existing = [];
        preg_match_all(
            "/'id'=>'([^']+)'.*?'sector'=>'([^']+)'.*?'p'=>\['(\/[^']+\/)'\]/s",
            $ie_content,
            $matches,
            PREG_SET_ORDER
        );
        foreach ($matches as $m) {
            $existing[] = ['id' => $m[1], 'sector' => $m[2], 'pattern' => $m[3]];
        }
        if ($verbose) echo "  Found " . count($existing) . " existing IE rules.\n";

        // Step 4: Generate + validate candidate rules
        $suggestions = [];
        foreach ($candidates_raw as $c) {
            if (count($suggestions) >= self::MAX_CANDIDATES) break;

            $pattern = self::generatePattern($c['query_norm']);
            if (!$pattern) continue;

            $domain = self::inferDomain($c['service'], $c['query_norm']);
            $sector = array_search($domain, [
                'medical'=>'medical','legal'=>'legal','pay'=>'pay','finance'=>'finance',
                'career'=>'career','education'=>'education','mental'=>'mental',
                'agriculture'=>'agriculture','community'=>'community','market'=>'market',
                'developer'=>'developer','enterprise'=>'enterprise','hub'=>'platform',
            ]) ?: $domain;

            $validation = self::validateNoConflict($pattern, $existing);
            if (!$validation['ok']) continue;

            // Assign confidence based on query length and frequency
            $freq      = $c['frequency'];
            $q_len     = mb_strlen($c['query_norm']);
            $conf      = round(min(self::CANDIDATE_CONF_HI,
                          max(self::CANDIDATE_CONF_LO,
                          0.72 + ($freq >= 10 ? 0.04 : ($freq >= 5 ? 0.02 : 0))
                          - ($q_len > 50 ? 0.02 : 0)
                         )), 2);

            $suggestions[] = [
                'query_example' => $c['query_raw'],
                'query_norm'    => $c['query_norm'],
                'pattern'       => $pattern,
                'target_sector' => $sector,
                'target_domain' => $domain,
                'confidence'    => $conf,
                'frequency'     => $c['frequency'],
                'avg_latency_ms'=> $c['avg_latency'],
                'total_cost_usd'=> round($c['total_cost'], 6),
                'total_tokens'  => $c['total_tokens'],
                'is_slow'       => $c['avg_latency'] > self::MIN_SLOW_MS,
                'estimated_ie_rule' => self::formatRule($pattern, $sector, $conf, $c['query_raw']),
            ];
        }

        // Step 5: Build report
        $top20_ai = array_slice(array_values($groups), 0, 20);
        $total_ai_cost    = array_sum(array_column($top20_ai, 'total_cost'));
        $total_ai_queries = array_sum(array_column($top20_ai, 'frequency'));

        $elapsed = round((microtime(true)-$t_start)*1000);
        $report = [
            'analysis_time_ms' => $elapsed,
            'queries_analyzed' => array_sum(array_column($groups, 'frequency')),
            'unique_queries'   => count($groups),
            'candidates_found' => count($candidates_raw),
            'rules_suggested'  => count($suggestions),
            'existing_ie_rules'=> count($existing),
            'top20_ai_queries' => $top20_ai,
            'suggestions'      => $suggestions,
            'impact_estimate'  => self::estimateImpact($groups, $suggestions),
        ];

        return $report;
    }

    // ── Format a valid PHP rule block ───────────────────────
    private static function formatRule(string $pattern, string $sector, float $conf, string $example): string
    {
        $id = 'learned_' . preg_replace('/[^a-z0-9]/i', '_', substr($example, 0, 20));
        $id = strtolower(preg_replace('/_+/', '_', trim($id, '_')));
        $r  = addslashes("💡 {$example}\\n\\n→ سودانا تساعدك في ذلك. أخبرني بتفاصيل أكثر.");
        return "\$R[]=['id'=>'{$id}','sector'=>'{$sector}','c'=>{$conf},\n"
             . "      'p'=>['{$pattern}'],\n"
             . "      'r'=>\"{$r}\",'billing_class'=>'standard','execution_mode'=>'ai_answer','layer'=>'ie'];";
    }

    // ── Estimate impact ─────────────────────────────────────
    private static function estimateImpact(array $groups, array $suggestions): array
    {
        $total_queries   = array_sum(array_column($groups, 'frequency'));
        $total_ai        = array_sum(array_column($groups, 'ai_hits'));
        $total_cost      = array_sum(array_column($groups, 'total_cost'));
        $total_tokens    = array_sum(array_column($groups, 'total_tokens'));

        $covered_freq    = array_sum(array_column($suggestions, 'frequency'));
        $covered_cost    = array_sum(array_column($suggestions, 'total_cost_usd'));
        $covered_tokens  = array_sum(array_column($suggestions, 'total_tokens'));

        $ie_coverage_before = $total_queries > 0
            ? round(100 * ($total_queries - $total_ai) / $total_queries, 1) : 0;
        $ie_coverage_after  = $total_queries > 0
            ? round(100 * ($total_queries - $total_ai + $covered_freq) / $total_queries, 1) : 0;

        return [
            'total_queries_analyzed'    => $total_queries,
            'current_ie_coverage_pct'   => $ie_coverage_before,
            'projected_ie_coverage_pct' => $ie_coverage_after,
            'ie_coverage_gain_pct'      => round($ie_coverage_after - $ie_coverage_before, 1),
            'queries_moved_to_ie'       => $covered_freq,
            'cost_saved_usd'            => round($covered_cost, 6),
            'tokens_saved'              => $covered_tokens,
            'latency_improvement'       => 'IE avg ~15ms vs AI avg ~2000ms → ~133× faster for covered queries',
        ];
    }

    // ── CLI report printer ──────────────────────────────────
    public static function printReport(array $report): void
    {
        $sep = str_repeat('═', 70);
        echo "\n{$sep}\n";
        echo "SUDANNA — IE LEARNING ENGINE REPORT\n";
        echo "{$sep}\n\n";

        if (($report['status'] ?? '') === 'NO_DATA') {
            echo "⚠️  {$report['message']}\n\n";
            echo "This is expected on a new deployment. Run again after collecting\n";
            echo "at least " . self::MIN_FREQUENCY . " days of production traffic.\n\n";
            return;
        }

        echo "Analysis completed in {$report['analysis_time_ms']}ms\n";
        echo "Queries analyzed:  {$report['queries_analyzed']}\n";
        echo "Unique queries:    {$report['unique_queries']}\n";
        echo "Existing IE rules: {$report['existing_ie_rules']}\n";
        echo "Rules suggested:   {$report['rules_suggested']}\n\n";

        // Top 20 AI-handled
        echo "── TOP 20 REPEATED AI-HANDLED QUERIES ──\n";
        echo sprintf("  %-5s  %-45s  %-12s  %-10s  %-8s\n",
            "FREQ", "QUERY", "DOMAIN", "LATENCY", "COST");
        echo "  " . str_repeat("─", 85) . "\n";
        foreach (array_slice($report['top20_ai_queries'] ?? [], 0, 20) as $g) {
            $q   = mb_substr($g['query_raw'], 0, 43);
            $slow= $g['avg_latency'] > self::MIN_SLOW_MS ? '🐢' : '  ';
            echo sprintf("  %-5d  %-45s  %-12s  %4dms%s  $%.6f\n",
                $g['frequency'], $q, $g['service'],
                $g['avg_latency'], $slow,
                $g['total_cost']
            );
        }

        // Suggestions
        echo "\n── SUGGESTED RULES (max 20) ──\n";
        if (empty($report['suggestions'])) {
            echo "  ✅ No new rules needed — all frequent queries already covered by IE.\n";
        } else {
            foreach ($report['suggestions'] as $i => $s) {
                echo "\n  Rule " . ($i+1) . ":\n";
                echo "    Query:      '{$s['query_example']}'\n";
                echo "    Pattern:    {$s['pattern']}\n";
                echo "    Target:     {$s['target_domain']} (c={$s['confidence']})\n";
                echo "    Frequency:  {$s['frequency']}× | Latency: {$s['avg_latency_ms']}ms";
                echo ($s['is_slow'] ? " 🐢 SLOW" : "") . "\n";
                echo "    Cost saved: \${$s['total_cost_usd']} | Tokens: {$s['total_tokens']}\n";
                echo "    PHP rule:\n";
                foreach (explode("\n", $s['estimated_ie_rule']) as $line) {
                    echo "      {$line}\n";
                }
            }
        }

        // Impact
        $imp = $report['impact_estimate'] ?? [];
        echo "\n── IMPACT ESTIMATE ──\n";
        echo "  Current IE coverage:    {$imp['current_ie_coverage_pct']}%\n";
        echo "  Projected IE coverage:  {$imp['projected_ie_coverage_pct']}%\n";
        echo "  Coverage gain:         +{$imp['ie_coverage_gain_pct']}%\n";
        echo "  Queries → IE:           {$imp['queries_moved_to_ie']}\n";
        echo "  Cost saved:            \${$imp['cost_saved_usd']}\n";
        echo "  Tokens saved:           {$imp['tokens_saved']}\n";
        echo "  Latency:                {$imp['latency_improvement']}\n";

        echo "\n{$sep}\n";
        echo "⚠️  SUGGESTIONS ONLY — not auto-injected. Review before adding to IE.\n";
        echo "   To inject: append to engine/InferenceEngine.php::getRules() with peer review.\n";
        echo "{$sep}\n\n";
    }
}

// ══════════════════════════════════════════════════════════════
// CLI RUNNER
// ══════════════════════════════════════════════════════════════
if (php_sapi_name() === 'cli') {
    $opts = getopt('', ['n:', 'dry-run', 'json', 'help']);
    if (isset($opts['help'])) {
        echo "Usage: php engine/IELearningEngine.php [--n=1000] [--dry-run] [--json]\n";
        exit(0);
    }

    $n       = (int)($opts['n'] ?? 1000);
    $as_json = isset($opts['json']);

    echo "IELearningEngine v1.0 — analyzing last {$n} queries...\n";
    $report = IELearningEngine::analyze($n, !$as_json);

    if ($as_json) {
        echo json_encode($report, JSON_PRETTY_PRINT | JSON_UNESCAPED_UNICODE);
    } else {
        IELearningEngine::printReport($report);
    }
}
