LLMVideoPipeline/pipeline/visual_gen.py

"""
Visual Generation & Stock Media Module.
Handles:
1. NvidiaLLM ComfyUI MiniMax H3 Text-to-Video generation
2. NvidiaLLM ComfyUI z-image-turbo / SDXL AI Image generation
3. Internet Stock Video & Stock Image downloaders (Wikimedia Commons, Pexels, Pixabay)
4. Dynamic Ken Burns Motion Effects (pan/zoom) in FFmpeg for still images
5. Intelligent visual curation and multi-tier fallback pipeline
"""

import os
import re
import time
import json
import zlib
import random
import logging
import urllib.request
import urllib.parse
import subprocess
import requests
from typing import Dict, Any, List, Optional
from .remote import RemotePC

logger = logging.getLogger("YTFactory.Visual")

# ─── ComfyUI Model Constants ────────────────────────────────────────────────
H3_UNET = "minimax_h3_fl2va_pruned_int8_convrot.safetensors"
H3_CLIP = "qwen3vl_32b_minimax_h3_nvfp4_awq.safetensors"
H3_VAE = "minimax_h3_video_vae_fp16.safetensors"
H3_FPS = 24
H3_MIN_LEN, H3_MAX_LEN = 124, 362

ZIMAGE_UNET = "z_image_turbo_bf16.safetensors"
ZIMAGE_CLIP = "qwen_3_4b.safetensors"
ZIMAGE_VAE = "ae.safetensors"

SDXL_CKPT = "sd_xl_base_1.0.safetensors"

def h3_snap_length(seconds: float) -> int:
    """Snap a duration in seconds to the MiniMax H3 17k+5 frame grid."""
    n = max(H3_MIN_LEN, min(H3_MAX_LEN, int(round(seconds * H3_FPS))))
    k = max(7, round((n - 5) / 17))
    return 17 * k + 5

# ─── Stock Media Fetcher ────────────────────────────────────────────────────

STOP_WORDS = {
    "the", "a", "an", "and", "or", "but", "in", "on", "at", "to", "for", "of", "with",
    "by", "about", "against", "between", "into", "through", "during", "before", "after",
    "above", "below", "from", "up", "down", "out", "off", "over", "under", "again",
    "further", "then", "once", "here", "there", "when", "where", "why", "how", "all",
    "any", "both", "each", "few", "more", "most", "other", "some", "such", "no", "nor",
    "not", "only", "own", "same", "so", "than", "too", "very", "can", "will", "just",
    "should", "now", "what", "reveals", "stopped", "being", "mankind", "humanity",
    "established", "used", "throughout", "history", "origins", "world", "worlds",
    "has", "have", "had", "been", "was", "were", "are", "is", "its", "their", "they",
    "them", "subjugate", "subjugating", "subjugation", "revealed", "revealing", "secrets",
    "truth", "behind", "explained", "documentary", "mystery", "mysteries"
}

def clean_search_keywords(text: str) -> str:
    """Extract clean, concrete visual search terms from a descriptive prompt or query."""
    words = re.findall(r'[A-Za-z0-9]+', text.lower())
    filtered = [w for w in words if w not in STOP_WORDS and len(w) > 2]
    # Keep up to 3 most specific visual keywords
    return " ".join(filtered[:3]) if filtered else text.strip()

import hashlib

def get_file_hash(path: str) -> str:
    """Compute SHA-256 hash of a file to detect exact duplicate clips or images."""
    if not path or not os.path.exists(path) or os.path.getsize(path) < 100:
        return ""
    h = hashlib.sha256()
    try:
        with open(path, "rb") as f:
            while chunk := f.read(65536):
                h.update(chunk)
        return h.hexdigest()
    except Exception:
        return ""

class StockMediaFetcher:
    def __init__(self, pexels_key: Optional[str] = None, pixabay_key: Optional[str] = None):
        self.pexels_key = pexels_key
        self.pixabay_key = pixabay_key
        self.user_agent = "YouTubeFactory/2.0 (faceless-video-pipeline)"
        self.used_media_urls = set()
        self.used_file_hashes = set()
        self.used_filenames = set()
        self.used_youtube_ids = set()

    def reset_session(self):
        """Reset used media set at the start of a new video job."""
        self.used_media_urls.clear()
        self.used_file_hashes.clear()
        self.used_filenames.clear()
        self.used_youtube_ids.clear()

    def _query_youtube_broll(self, query: str, duration_sec: float, out_path: str, offset_sec: int = 15) -> Optional[str]:
        """Search YouTube for relevant topic documentary/b-roll footage and download muted HD clip."""
        try:
            clean_q = re.sub(r'[^A-Za-z0-9 ]', '', query).strip()
            if not clean_q:
                return None
            search_target = f"ytsearch8:{clean_q} documentary footage"
            logger.info(f"[STOCK:YOUTUBE] Searching YouTube for b-roll footage: '{clean_q}'...")
            cmd_search = [
                "yt-dlp", search_target,
                "--dump-json", "--flat-playlist", "--no-warnings"
            ]
            r = subprocess.run(cmd_search, capture_output=True, text=True, timeout=20)
            lines = [l for l in r.stdout.strip().split("\n") if l]
            for line in lines:
                try:
                    d = json.loads(line)
                    vid_id = d.get("id")
                    dur = d.get("duration") or 60
                    if not vid_id or vid_id in self.used_youtube_ids or (dur and dur < 12):
                        continue

                    # Determine clean mid-video section avoiding intros/outros
                    start_t = min(max(15, offset_sec), max(5, int(dur) - int(duration_sec) - 5)) if dur else 15
                    end_t = start_t + int(duration_sec) + 3

                    raw_tmp = out_path + f".raw_{vid_id}.mp4"
                    cmd_dl = [
                        "yt-dlp", f"https://www.youtube.com/watch?v={vid_id}",
                        "--extractor-args", "youtube:player_client=android,web",
                        "-f", "22/18/best[height<=720]/best",
                        "--download-sections", f"*{start_t}-{end_t}",
                        "--force-keyframes-at-cuts",
                        "-o", raw_tmp,
                        "--no-playlist",
                        "--socket-timeout", "15"
                    ]
                    r_dl = subprocess.run(cmd_dl, capture_output=True, text=True, timeout=35)
                    if r_dl.returncode == 0 and os.path.exists(raw_tmp) and os.path.getsize(raw_tmp) > 5000:
                        # Normalize to 1280x720 25fps muted video (stripping any YouTube audio)
                        cmd_norm = [
                            "ffmpeg", "-y", "-i", raw_tmp,
                            "-vf", "scale=1280:720:force_original_aspect_ratio=decrease,pad=1280:720:(ow-iw)/2:(oh-ih)/2:black,fps=25,format=yuv420p",
                            "-t", f"{duration_sec:.2f}", "-r", "25",
                            "-c:v", "libx264", "-preset", "fast", "-crf", "22", "-pix_fmt", "yuv420p",
                            "-an", out_path
                        ]
                        r_norm = subprocess.run(cmd_norm, capture_output=True, text=True)
                        if os.path.exists(raw_tmp):
                            try:
                                os.remove(raw_tmp)
                            except OSError:
                                pass

                        if r_norm.returncode == 0 and os.path.exists(out_path) and os.path.getsize(out_path) > 5000:
                            h = get_file_hash(out_path)
                            if h and h in self.used_file_hashes:
                                logger.warning(f"[STOCK:YOUTUBE] Duplicate content detected (hash {h[:8]}). Continuing search...")
                                try:
                                    os.remove(out_path)
                                except OSError:
                                    pass
                                continue
                            self.used_youtube_ids.add(vid_id)
                            self.used_media_urls.add(f"https://www.youtube.com/watch?v={vid_id}")
                            if h:
                                self.used_file_hashes.add(h)
                            logger.info(f"[STOCK:YOUTUBE] Extracted high quality muted YouTube b-roll clip from '{d.get('title')}'")
                            return out_path
                except Exception as e:
                    logger.debug(f"[STOCK:YOUTUBE] Candidate parse error: {e}")
                    continue
        except Exception as e:
            logger.debug(f"[STOCK:YOUTUBE] Search failed for '{query}': {e}")
        return None

    def query_person_image(self, person_name: str) -> Optional[str]:
        """Fetch authentic historical portrait/photograph of a person from Wikipedia / Wikimedia Commons."""
        if not person_name or len(person_name.strip()) < 3:
            return None
        clean_name = person_name.strip()
        logger.info(f"[STOCK:PERSON] Searching authentic archival portrait/photo for person: '{clean_name}'...")
        
        # 1. Query Wikipedia PageImage API (1920px curated lead portrait)
        try:
            encoded = urllib.parse.quote(clean_name)
            url = f"https://en.wikipedia.org/w/api.php?action=query&titles={encoded}&prop=pageimages&pithumbsize=1920&format=json"
            req = urllib.request.Request(url, headers={"User-Agent": self.user_agent})
            with urllib.request.urlopen(req, timeout=8) as resp:
                data = json.loads(resp.read().decode("utf-8"))
                pages = data.get("query", {}).get("pages", {})
                for pid, page in pages.items():
                    if pid != "-1" and "thumbnail" in page:
                        src = page["thumbnail"].get("source")
                        if src and src not in self.used_media_urls:
                            return src
        except Exception as e:
            logger.debug(f"[STOCK:PERSON] Wikipedia query note for '{clean_name}': {e}")

        # 2. Query Wikimedia Commons for high-res archival portrait
        try:
            wiki_img = self._query_wikimedia(f"portrait {clean_name}", is_video=False)
            if wiki_img:
                return wiki_img
            wiki_img2 = self._query_wikimedia(clean_name, is_video=False)
            if wiki_img2:
                return wiki_img2
        except Exception as e:
            logger.debug(f"[STOCK:PERSON] Wikimedia query note for '{clean_name}': {e}")

        return None

    def _query_wikimedia(self, query: str, is_video: bool = False) -> Optional[str]:
        """Search Wikimedia Commons for high-resolution images or videos without API keys."""
        try:
            clean_q = re.sub(r'[^A-Za-z0-9 ]', '', query).strip()
            encoded_q = urllib.parse.quote(clean_q)
            api_url = (
                f"https://commons.wikimedia.org/w/api.php?action=query&generator=search"
                f"&gsrsearch={encoded_q}&gsrnamespace=6&gsrlimit=10&prop=imageinfo"
                f"&iiprop=url|mime|size&format=json"
            )
            req = urllib.request.Request(api_url, headers={"User-Agent": self.user_agent})
            with urllib.request.urlopen(req, timeout=10) as resp:
                data = json.loads(resp.read().decode("utf-8"))
                pages = data.get("query", {}).get("pages", {})
                for _, page in pages.items():
                    infos = page.get("imageinfo", [])
                    if not infos:
                        continue
                    url = infos[0].get("url", "")
                    if not url or url in self.used_media_urls:
                        continue
                    clean_name = os.path.basename(urllib.parse.urlparse(url).path)
                    if clean_name and clean_name in self.used_filenames:
                        continue
                    mime = infos[0].get("mime", "").lower()
                    if is_video and ("video" in mime or url.endswith((".webm", ".mp4", ".ogv"))):
                        return url
                    if not is_video and ("image" in mime or url.endswith((".jpg", ".jpeg", ".png", ".webp"))):
                        return url
        except Exception as e:
            logger.debug(f"Wikimedia search failed for '{query}': {e}")
        return None

    def _query_pexels_video(self, query: str) -> Optional[str]:
        """Search Pexels for free HD stock videos if API key is provided."""
        if not self.pexels_key:
            return None
        try:
            clean_q = urllib.parse.quote(query)
            url = f"https://api.pexels.com/videos/search?query={clean_q}&per_page=10&orientation=landscape"
            headers = {"Authorization": self.pexels_key, "User-Agent": self.user_agent}
            req = urllib.request.Request(url, headers=headers)
            with urllib.request.urlopen(req, timeout=10) as resp:
                data = json.loads(resp.read().decode("utf-8"))
                videos = data.get("videos", [])
                for v in videos:
                    files = v.get("video_files", [])
                    hd = [vf for vf in files if vf.get("width", 0) >= 1280 and vf.get("link")]
                    target = sorted(hd, key=lambda x: x["width"])[0]["link"] if hd else (files[0].get("link") if files else None)
                    if target and target not in self.used_media_urls:
                        return target
        except Exception as e:
            logger.debug(f"Pexels video query failed: {e}")
        return None

    def _query_pexels_photo(self, query: str) -> Optional[str]:
        """Search Pexels for free HD stock photos if API key is provided."""
        if not self.pexels_key:
            return None
        try:
            clean_q = urllib.parse.quote(query)
            url = f"https://api.pexels.com/v1/search?query={clean_q}&per_page=10&orientation=landscape"
            headers = {"Authorization": self.pexels_key, "User-Agent": self.user_agent}
            req = urllib.request.Request(url, headers=headers)
            with urllib.request.urlopen(req, timeout=10) as resp:
                data = json.loads(resp.read().decode("utf-8"))
                photos = data.get("photos", [])
                for p in photos:
                    src = p.get("src", {}).get("large2x") or p.get("src", {}).get("large")
                    if src and src not in self.used_media_urls:
                        return src
        except Exception as e:
            logger.debug(f"Pexels photo query failed: {e}")
        return None

    def _query_pixabay_video(self, query: str) -> Optional[str]:
        """Search Pixabay for free HD stock videos if API key is provided."""
        if not self.pixabay_key:
            return None
        try:
            clean_q = urllib.parse.quote(query)
            url = f"https://pixabay.com/api/videos/?key={self.pixabay_key}&q={clean_q}&per_page=10&safesearch=true"
            req = urllib.request.Request(url, headers={"User-Agent": self.user_agent})
            with urllib.request.urlopen(req, timeout=10) as resp:
                data = json.loads(resp.read().decode("utf-8"))
                for hit in data.get("hits", []):
                    vids = hit.get("videos", {})
                    for quality in ("large", "medium", "small"):
                        v = vids.get(quality)
                        if v and v.get("url") and v.get("width", 0) >= 640:
                            target_url = v["url"]
                            if target_url not in self.used_media_urls:
                                return target_url
        except Exception as e:
            logger.debug(f"Pixabay video query failed: {e}")
        return None

    def _query_pixabay_photo(self, query: str) -> Optional[str]:
        """Search Pixabay for free HD stock photos if API key is provided."""
        if not self.pixabay_key:
            return None
        try:
            clean_q = urllib.parse.quote(query)
            url = (f"https://pixabay.com/api/?key={self.pixabay_key}&q={clean_q}"
                   f"&image_type=photo&orientation=horizontal&per_page=10&safesearch=true")
            req = urllib.request.Request(url, headers={"User-Agent": self.user_agent})
            with urllib.request.urlopen(req, timeout=10) as resp:
                data = json.loads(resp.read().decode("utf-8"))
                hits = data.get("hits", [])
                for h in hits:
                    img_url = h.get("largeImageURL") or h.get("webformatURL")
                    if img_url and img_url not in self.used_media_urls:
                        return img_url
        except Exception as e:
            logger.debug(f"Pixabay photo query failed: {e}")
        return None

    def download_media(self, url: str, out_path: str) -> bool:
        """Download remote media with retry, validation, and content hash duplicate prevention."""
        try:
            req = urllib.request.Request(url, headers={"User-Agent": self.user_agent})
            with urllib.request.urlopen(req, timeout=30) as resp, open(out_path, "wb") as f:
                f.write(resp.read())
            if os.path.exists(out_path) and os.path.getsize(out_path) > 5000:
                h = get_file_hash(out_path)
                if h and h in self.used_file_hashes:
                    logger.warning(f"[STOCK] Duplicate asset content detected (hash {h[:8]}). Discarding duplicate...")
                    try:
                        os.remove(out_path)
                    except OSError:
                        pass
                    return False
                self.used_media_urls.add(url)
                if h:
                    self.used_file_hashes.add(h)
                clean_name = os.path.basename(urllib.parse.urlparse(url).path)
                if clean_name:
                    self.used_filenames.add(clean_name)
                return True
        except Exception as e:
            logger.debug(f"Download failed from {url}: {e}")
        if os.path.exists(out_path):
            try:
                os.remove(out_path)
            except OSError:
                pass
        return False

    def search_and_download(self, queries: Any, out_path: str, prefer_video: bool = False, duration_sec: float = 6.0) -> Optional[str]:
        """Search across stock engines with candidate query cascade and download asset."""
        if isinstance(queries, str):
            candidate_list = [queries]
        else:
            candidate_list = list(queries)

        expanded_queries = []
        for raw_q in candidate_list:
            if not raw_q:
                continue
            cleaned = clean_search_keywords(raw_q)
            if cleaned and cleaned not in expanded_queries:
                expanded_queries.append(cleaned)
            words = cleaned.split()
            if len(words) >= 2:
                two_word = " ".join(words[:2])
                if two_word not in expanded_queries:
                    expanded_queries.append(two_word)
                for w in words:
                    if len(w) > 3 and w not in expanded_queries:
                        expanded_queries.append(w)

        for cleaned_q in expanded_queries:
            logger.info(f"[STOCK] Searching stock media for: '{cleaned_q}' (video={prefer_video})...")
            if prefer_video:
                # 1. First priority: Real YouTube documentary B-roll footage without audio
                yt_clip = self._query_youtube_broll(cleaned_q, duration_sec, out_path)
                if yt_clip and os.path.exists(yt_clip) and os.path.getsize(yt_clip) > 5000:
                    return yt_clip

                # 2. Stock APIs (Pexels, Pixabay, Wikimedia)
                for label, fn in (("Pexels", self._query_pexels_video),
                                  ("Pixabay", self._query_pixabay_video),
                                  ("Wikimedia", lambda q: self._query_wikimedia(q, is_video=True))):
                    v_url = fn(cleaned_q)
                    if v_url and self.download_media(v_url, out_path):
                        logger.info(f"[STOCK] Downloaded {label} video for '{cleaned_q}'")
                        return out_path
            else:
                for label, fn in (("Pexels", self._query_pexels_photo),
                                  ("Pixabay", self._query_pixabay_photo),
                                  ("Wikimedia", lambda q: self._query_wikimedia(q, is_video=False))):
                    p_url = fn(cleaned_q)
                    if p_url and self.download_media(p_url, out_path):
                        logger.info(f"[STOCK] Downloaded {label} photo for '{cleaned_q}'")
                        return out_path

        logger.debug(f"[STOCK] No matching stock media found for queries: {expanded_queries}")
        return None

# ─── Ken Burns Motion Engine ────────────────────────────────────────────────

def apply_ken_burns_effect(img_path: str, out_mp4: str, duration_sec: float, motion_type: int = 0) -> str:
    """
    Animate a still image into a cinematic 16:9 25fps video clip using FFmpeg pan & zoom motion.
    Provides 6 distinct cinematic camera movements:
      0: Smooth Center Zoom-In
      1: Smooth Center Zoom-Out
      2: Diagonal Pan Left-Up with subtle zoom
      3: Diagonal Pan Right-Down with subtle zoom
      4: Slow Pan Right across center
      5: Slow Pan Left across center
    """
    total_frames = max(50, int(round(duration_sec * 25)))
    m_idx = motion_type % 6

    if m_idx == 0:
        # Smooth Zoom-In (center)
        z_expr = "min(zoom+0.0012,1.25)"
        x_expr = "iw/2-(iw/zoom/2)"
        y_expr = "ih/2-(ih/zoom/2)"
    elif m_idx == 1:
        # Smooth Zoom-Out (center)
        z_expr = "if(eq(on,1),1.25,max(1.0,zoom-0.0012))"
        x_expr = "iw/2-(iw/zoom/2)"
        y_expr = "ih/2-(ih/zoom/2)"
    elif m_idx == 2:
        # Diagonal Pan Left-Up with subtle zoom
        z_expr = "1.20"
        x_expr = f"if(eq(on,1),iw-iw/zoom,max(0,x-(iw-iw/zoom)/{total_frames}))"
        y_expr = f"if(eq(on,1),ih-ih/zoom,max(0,y-(ih-ih/zoom)/{total_frames}))"
    elif m_idx == 3:
        # Diagonal Pan Right-Down with subtle zoom
        z_expr = "1.20"
        x_expr = f"if(eq(on,1),0,min(iw-iw/zoom,x+(iw-iw/zoom)/{total_frames}))"
        y_expr = f"if(eq(on,1),0,min(ih-ih/zoom,y+(ih-ih/zoom)/{total_frames}))"
    elif m_idx == 4:
        # Slow Pan Right across center
        z_expr = "1.18"
        x_expr = f"if(eq(on,1),0,min(iw-iw/zoom,x+(iw-iw/zoom)/{total_frames}))"
        y_expr = "ih/2-(ih/zoom/2)"
    else:
        # Slow Pan Left across center
        z_expr = "1.18"
        x_expr = f"if(eq(on,1),iw-iw/zoom,max(0,x-(iw-iw/zoom)/{total_frames}))"
        y_expr = "ih/2-(ih/zoom/2)"

    vf = (
        f"scale=1920:1080:force_original_aspect_ratio=increase,crop=1920:1080,"
        f"zoompan=z='{z_expr}':x='{x_expr}':y='{y_expr}':d={total_frames}:s=1280x720:fps=25,"
        f"format=yuv420p"
    )

    cmd = [
        "ffmpeg", "-y", "-loop", "1", "-i", img_path,
        "-vf", vf,
        "-t", f"{duration_sec:.2f}",
        "-r", "25",
        "-c:v", "libx264", "-preset", "fast", "-crf", "22",
        "-pix_fmt", "yuv420p", "-an", out_mp4
    ]
    r = subprocess.run(cmd, capture_output=True, text=True)
    if r.returncode != 0:
        logger.warning(f"Ken Burns animation fallback: {r.stderr[-200:]}")
        # Simple loop fallback
        fallback_vf = "scale=1280:720:force_original_aspect_ratio=decrease,pad=1280:720:(ow-iw)/2:(oh-ih)/2:black,format=yuv420p"
        subprocess.run([
            "ffmpeg", "-y", "-loop", "1", "-i", img_path,
            "-vf", fallback_vf, "-t", f"{duration_sec:.2f}", "-r", "25",
            "-c:v", "libx264", "-preset", "fast", "-crf", "22", "-pix_fmt", "yuv420p", "-an", out_mp4
        ], check=True, capture_output=True)

    return out_mp4

def normalize_clip_to_duration(src_path: str, out_path: str, duration_sec: float) -> str:
    """
    Normalize any video clip to exactly duration_sec at 1280x720 25fps.
    Loops short clips (-stream_loop) and trims long ones (-t) so every segment
    stays in sync with its narration.
    """
    vf = ("scale=1280:720:force_original_aspect_ratio=decrease,"
          "pad=1280:720:(ow-iw)/2:(oh-ih)/2:black,fps=25,format=yuv420p")
    r = subprocess.run([
        "ffmpeg", "-y", "-stream_loop", "-1", "-i", src_path,
        "-vf", vf, "-t", f"{duration_sec:.2f}", "-r", "25",
        "-c:v", "libx264", "-preset", "fast", "-crf", "22",
        "-pix_fmt", "yuv420p", "-an", out_path
    ], capture_output=True, text=True)
    if r.returncode != 0:
        raise RuntimeError(f"clip normalization failed: {r.stderr[-200:]}")
    return out_path

def apply_text_overlay(video_path: str, out_mp4: str, text: str, duration_sec: float) -> str:
    """
    Render a broadcast-style lower-third text banner onto the video clip using UTF-8 textfile.
    Supports full international characters (accents, umlauts, Spanish punctuation) without corruption.
    """
    clean_text = text.strip()
    if not clean_text:
        if video_path != out_mp4:
            import shutil
            shutil.copy(video_path, out_mp4)
        return out_mp4

    font_paths = [
        "/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf",
        "/usr/share/fonts/TTF/DejaVuSans-Bold.ttf",
        "/usr/share/fonts/truetype/liberation/LiberationSans-Bold.ttf",
        "/usr/share/fonts/truetype/freefont/FreeSansBold.ttf",
    ]
    font_file = None
    for fp in font_paths:
        if os.path.exists(fp):
            font_file = fp
            break

    # Write clean UTF-8 text to a temporary textfile for FFmpeg drawtext to avoid character stripping
    import tempfile
    txt_fd, txt_path = tempfile.mkstemp(suffix=".txt", prefix="banner_text_")
    with open(txt_fd, "w", encoding="utf-8") as f:
        f.write(clean_text)

    font_param = f":fontfile='{font_file}'" if font_file else ""
    end_t = max(1.0, duration_sec - 0.8)
    enable_expr = f"between(t,0.8,{end_t:.2f})"
    vf = (
        f"drawtext=textfile='{txt_path}'{font_param}:fontcolor=white:fontsize=36:"
        f"box=1:boxcolor=black@0.65:boxborderw=14:x=60:y=h-th-60:enable='{enable_expr}'"
    )

    cmd = [
        "ffmpeg", "-y", "-i", video_path,
        "-vf", vf,
        "-c:v", "libx264", "-preset", "fast", "-crf", "22", "-pix_fmt", "yuv420p",
        "-an", out_mp4
    ]
    r = subprocess.run(cmd, capture_output=True, text=True)
    if os.path.exists(txt_path):
        try:
            os.remove(txt_path)
        except OSError:
            pass

    if r.returncode != 0:
        logger.warning(f"Text overlay rendering fallback: {r.stderr[-200:]}")
        if video_path != out_mp4:
            import shutil
            shutil.copy(video_path, out_mp4)
    return out_mp4

# ─── Film Grain & Vintage Aging Filter ──────────────────────────────────────

HISTORICAL_KEYWORDS = {
    "ancient", "archaeology", "archaeological", "history", "historical", "century",
    "b.c.", "bc", "a.d.", "ad", "victorian", "medieval", "ruins", "pharaoh",
    "temple", "pyramids", "past", "hunter gatherer", "hunter-gatherer", "gobeklitepe",
    "göbeklitepe", "stone age", "bronze age", "iron age", "1888", "1900", "1920",
    "1930", "1940", "1950", "1960", "1970", "empire", "dynasty", "columbus",
    "alexandria", "antiquity", "paleolithic", "neolithic", "civilization", "excavation",
    "fossil", "scrolls", "papyrus", "jack the ripper", "monument", "artifacts", "artifact",
    "bermuda", "atlantis", "mythology", "prehistoric", "dynastic", "tomb", "sarcophagus",
    "excavations", "relic", "relics", "megalith", "megalithic", "sanctuary"
}

def is_historical_scene(seg: Dict[str, Any], topic: str = "") -> bool:
    """Check if the segment depicts historical/past events based on flag or keywords."""
    if seg.get("is_historical") is True:
        return True
    combined_text = (
        f"{topic} {seg.get('chapter_title', '')} {seg.get('text', '')} "
        f"{seg.get('visual_prompt', '')} {seg.get('stock_query', '')}"
    ).lower()
    return any(kw in combined_text for kw in HISTORICAL_KEYWORDS)

def apply_film_grain_overlay(video_path: str, out_mp4: str) -> str:
    """
    Apply a dynamic 35mm film grain overlay with subtle vintage contrast & vignette
    to give historical footage an authentically aged cinematic look.
    """
    vintage_vf = (
        "noise=alls=14:allf=t+u,"
        "eq=contrast=1.06:brightness=-0.01:saturation=0.88,"
        "vignette=PI/5"
    )
    cmd = [
        "ffmpeg", "-y", "-i", video_path,
        "-vf", vintage_vf,
        "-c:v", "libx264", "-preset", "fast", "-crf", "22", "-pix_fmt", "yuv420p",
        "-an", out_mp4
    ]
    r = subprocess.run(cmd, capture_output=True, text=True)
    if r.returncode != 0:
        logger.warning(f"Film grain filter fallback: {r.stderr[-200:]}")
        if video_path != out_mp4:
            import shutil
            shutil.copy(video_path, out_mp4)
    return out_mp4

# ─── ComfyUI Orchestration ─────────────────────────────────────────────────

class VisualGen:
    def __init__(self, pcs: Dict[str, RemotePC], cfg: Any):
        self.pcs = pcs
        self.cfg = cfg
        self.seen_clip_hashes = set()
        self.stock_fetcher = StockMediaFetcher(
            pexels_key=getattr(cfg, "pexels_api_key", None),
            pixabay_key=getattr(cfg, "pixabay_api_key", None)
        )

    def _h3_workflow(self, prompt: str, seed: int, width: int, height: int, length: int) -> Dict[str, Any]:
        return {
            "1": {"inputs": {"unet_name": H3_UNET, "weight_dtype": "default"}, "class_type": "UNETLoader"},
            "2": {"inputs": {"model": ["1", 0], "shift_video": 12.0, "shift_audio": 3.0}, "class_type": "MiniMaxH3SigmaShift"},
            "3": {"inputs": {"clip_name": H3_CLIP, "type": "minimax"}, "class_type": "CLIPLoader"},
            "4": {"inputs": {"vae_name": H3_VAE}, "class_type": "VAELoader"},
            "5": {"inputs": {"clip": ["3", 0], "vae": ["4", 0], "prompt": prompt,
                             "width": width, "height": height, "length": length},
                  "class_type": "MiniMaxH3ImageToVideo"},
            "6": {"inputs": {"text": "", "clip": ["3", 0]}, "class_type": "CLIPTextEncode"},
            "7": {"inputs": {"seed": seed, "steps": 20, "cfg": 5.0, "sampler_name": "euler",
                             "scheduler": "normal", "denoise": 1.0,
                             "model": ["2", 0], "positive": ["5", 0], "negative": ["6", 0],
                             "latent_image": ["5", 1]}, "class_type": "KSampler"},
            "8": {"inputs": {"samples": ["7", 0], "vae": ["4", 0]}, "class_type": "VAEDecode"},
            "9": {"inputs": {"images": ["8", 0], "fps": float(H3_FPS)}, "class_type": "CreateVideo"},
            "10": {"inputs": {"video": ["9", 0], "filename_prefix": "yt_clip",
                              "format": "auto", "codec": "auto"}, "class_type": "SaveVideo"},
        }

    def _sdxl_workflow(self, prompt: str, seed: int) -> Dict[str, Any]:
        return {
            "1": {"inputs": {"ckpt_name": SDXL_CKPT}, "class_type": "CheckpointLoaderSimple"},
            "2": {"inputs": {"text": prompt + ", 8k resolution, cinematic lighting, photorealistic, documentary b-roll", "clip": ["1", 1]}, "class_type": "CLIPTextEncode"},
            "2b": {"inputs": {"text": "blurry, low quality, watermark, text, logo, ugly, deformed, oversaturated", "clip": ["1", 1]}, "class_type": "CLIPTextEncode"},
            "3": {"inputs": {"width": 1280, "height": 720, "batch_size": 1}, "class_type": "EmptyLatentImage"},
            "4": {"inputs": {"seed": seed, "steps": 22, "cfg": 7.0, "sampler_name": "euler",
                             "scheduler": "normal", "denoise": 1.0,
                             "model": ["1", 0], "positive": ["2", 0], "negative": ["2b", 0],
                             "latent_image": ["3", 0]}, "class_type": "KSampler"},
            "5": {"inputs": {"samples": ["4", 0], "vae": ["1", 2]}, "class_type": "VAEDecode"},
            "6": {"inputs": {"filename_prefix": "yt_img", "images": ["5", 0]}, "class_type": "SaveImage"},
        }

    def _submit(self, host: str, workflow: Dict[str, Any]) -> str:
        r = requests.post(f"http://{host}:8188/prompt", json={"prompt": workflow}, timeout=30)
        data = r.json()
        if r.status_code != 200 or "prompt_id" not in data:
            raise RuntimeError(f"ComfyUI rejected workflow: {json.dumps(data)[:300]}")
        return data["prompt_id"]

    def _cancel_job(self, host: str, pid: str):
        """Interrupt current execution and remove the job from the ComfyUI queue.
        Prevents client-side timeouts from leaving zombie jobs that block later prompts."""
        try:
            requests.post(f"http://{host}:8188/interrupt", timeout=5)
        except Exception:
            pass
        try:
            requests.post(f"http://{host}:8188/queue", json={"delete": [pid]}, timeout=5)
        except Exception:
            pass

    def _wait_result(self, host: str, pid: str, timeout_s: int = 1800) -> Dict[str, Any]:
        deadline = time.time() + timeout_s
        while time.time() < deadline:
            time.sleep(3)
            try:
                h = requests.get(f"http://{host}:8188/history/{pid}", timeout=10).json()
            except Exception:
                continue
            if pid not in h:
                continue
            status = h[pid].get("status", {})
            if status.get("status_str") == "error":
                msgs = h[pid].get("status", {}).get("messages", [])
                self._cancel_job(host, pid)
                raise RuntimeError(f"ComfyUI execution error: {json.dumps(msgs)[:300]}")
            if status.get("completed") or h[pid].get("outputs"):
                return h[pid].get("outputs", {})
        self._cancel_job(host, pid)
        raise RuntimeError("ComfyUI generation timed out.")

    def _download_outputs(self, host: str, outputs: Dict[str, Any], out_path: str) -> str:
        for node_out in outputs.values():
            for key in ("videos", "gifs", "images"):
                for item in node_out.get(key, []):
                    params = {"filename": item["filename"], "type": item.get("type", "output")}
                    if item.get("subfolder"):
                        params["subfolder"] = item["subfolder"]
                    d = requests.get(f"http://{host}:8188/view", params=params, timeout=120).content
                    if len(d) >= 1024:
                        with open(out_path, "wb") as f:
                            f.write(d)
                        return out_path
        raise RuntimeError("No downloadable asset returned by ComfyUI")

    # Maximum frames for a MiniMax H3 clip (~6.5s at 24fps). Longer clips take
    # 30+ min on the RTX 4060; clips are looped/trimmed to segment length anyway.
    H3_CLIP_CAP = 158

    def _try_ai_video(self, pc, prompt, seed, duration_sec, raw_path, final_clip_path, idx):
        """MiniMax H3 text-to-video clip, normalized to the segment duration."""
        if not pc or not pc.check(8188):
            raise RuntimeError("NvidiaLLM ComfyUI is not reachable on port 8188")
        h3_length = min(h3_snap_length(duration_sec), self.H3_CLIP_CAP)
        logger.info(f"[VISUAL] Segment {idx+1}: MiniMax H3 AI video ({h3_length} frames)...")
        wf = self._h3_workflow(prompt, seed, self.cfg.clip_width, self.cfg.clip_height, h3_length)
        pid = self._submit(pc.host, wf)
        outputs = self._wait_result(pc.host, pid, timeout_s=1200)
        self._download_outputs(pc.host, outputs, raw_path)
        return normalize_clip_to_duration(raw_path, final_clip_path, duration_sec)

    def _try_ai_image(self, pc, prompt, seed, duration_sec, raw_path, final_clip_path, idx):
        """SDXL AI image animated with Ken Burns motion."""
        if not pc or not pc.check(8188):
            raise RuntimeError("NvidiaLLM ComfyUI is not reachable on port 8188")
        logger.info(f"[VISUAL] Segment {idx+1}: SDXL AI image + Ken Burns...")
        pid = self._submit(pc.host, self._sdxl_workflow(prompt, seed))
        outputs = self._wait_result(pc.host, pid, timeout_s=300)
        self._download_outputs(pc.host, outputs, raw_path)
        return apply_ken_burns_effect(raw_path, final_clip_path, duration_sec, motion_type=idx)

    def _try_stock(self, query, duration_sec, raw_path, final_clip_path, idx, prefer_video):
        """Stock video (looped/trimmed) or stock image (Ken Burns)."""
        kind = "video" if prefer_video else "image"
        logger.info(f"[VISUAL] Segment {idx+1}: stock {kind} for '{query}'...")
        got = self.stock_fetcher.search_and_download(query, raw_path, prefer_video=prefer_video, duration_sec=duration_sec)
        if not got or not os.path.exists(got):
            raise RuntimeError(f"no stock {kind} found for '{query}'")
        if got.lower().endswith((".mp4", ".webm", ".mov")):
            return normalize_clip_to_duration(got, final_clip_path, duration_sec)
        return apply_ken_burns_effect(got, final_clip_path, duration_sec, motion_type=idx)

    def _try_person(self, person_name: str, duration_sec: float, raw_path: str, final_clip_path: str, idx: int):
        """Fetch authentic photo/portrait of historical person and animate with Ken Burns motion."""
        if not person_name:
            raise RuntimeError("no person name provided")
        img_url = self.stock_fetcher.query_person_image(person_name)
        if not img_url:
            raise RuntimeError(f"no authentic archival photo found for person '{person_name}'")
        if not self.stock_fetcher.download_media(img_url, raw_path):
            raise RuntimeError(f"failed to download person photo from {img_url}")
        logger.info(f"[VISUAL] Segment {idx+1}: Using authentic historical photo/portrait for '{person_name}'")
        return apply_ken_burns_effect(raw_path, final_clip_path, duration_sec, motion_type=idx)

    def _generate_segment_visual(self, pc: RemotePC, seg: Dict[str, Any], idx: int, out_dir: str, duration_sec: float) -> str:
        """
        Produce one visual for a segment as an MP4 of exactly duration_sec.

        "hybrid" rotates across 4 styles for maximum variety:
          YouTube b-roll / stock video -> AI video -> stock image (Ken Burns) -> AI image (Ken Burns)
        Each source falls back to the next one in the rotation on failure or duplicate collision.
        """
        os.makedirs(out_dir, exist_ok=True)
        CAMERA_ANGLES = [
            "wide establishing cinematic drone shot",
            "dramatic macro close-up detail shot",
            "dynamic low-angle heroic perspective",
            "atmospheric high-angle bird's eye view",
            "side profile tracking cinematic perspective",
            "epic panoramic sweeping shot",
            "dramatic cinematic lighting with volumetric shadows",
            "slow dolly-in center focus shot"
        ]
        angle = CAMERA_ANGLES[idx % len(CAMERA_ANGLES)]
        visual_anchor = (
            seg.get("visual_style_anchor") or
            getattr(self.cfg, "visual_style_anchor", "") or
            "cinematic documentary style, consistent warm color grading, 35mm film texture, photorealistic 8k"
        )
        raw_prompt = seg.get("visual_prompt") or seg.get("text", "")
        prompt = f"{raw_prompt}, {angle}, {visual_anchor}"
        primary_stock = seg.get("stock_query") or ""
        chapter_stock = seg.get("chapter_title") or ""
        topic_anchor = clean_search_keywords(getattr(self.cfg, "topic", "") or "")
        person_name = (seg.get("person_name") or "").strip()
        
        # Build prioritized topical query cascade
        query_candidates = []
        if primary_stock:
            query_candidates.append(primary_stock)
        if chapter_stock and topic_anchor:
            query_candidates.append(f"{topic_anchor} {chapter_stock}")
        elif chapter_stock:
            query_candidates.append(chapter_stock)
        if topic_anchor:
            query_candidates.append(topic_anchor)

        import time
        seed = zlib.crc32(f"{prompt}_{idx}_{time.time()}_{os.urandom(4)}".encode()) % (2**31)

        final_clip_path = os.path.join(out_dir, f"clip_{idx:03d}.mp4")
        raw_video = os.path.join(out_dir, f"raw_vid_{idx:03d}.mp4")
        raw_img = os.path.join(out_dir, f"raw_img_{idx:03d}.jpg")

        chosen_clip = None

        # 1. If talking about a specific person, prioritize authentic archival portrait/photo
        if person_name:
            try:
                candidate = self._try_person(person_name, duration_sec, raw_img, final_clip_path, idx)
                cand_hash = get_file_hash(candidate)
                if cand_hash and cand_hash not in self.seen_clip_hashes:
                    chosen_clip = candidate
            except Exception as e:
                logger.debug(f"[VISUAL] Person portrait search note for '{person_name}': {e}")

        if not chosen_clip:
            strategy = getattr(self.cfg, "visual_strategy", "hybrid")
            attempts = {
                "stock_video": lambda: self._try_stock(query_candidates, duration_sec, raw_video, final_clip_path, idx, prefer_video=True),
                "ai_video":    lambda: self._try_ai_video(pc, prompt, seed, duration_sec, raw_video, final_clip_path, idx),
                "stock_image": lambda: self._try_stock(query_candidates, duration_sec, raw_img, final_clip_path, idx, prefer_video=False),
                "ai_image":    lambda: self._try_ai_image(pc, prompt, seed, duration_sec, raw_img, final_clip_path, idx),
            }
            if strategy == "all_ai":
                order = ["ai_video", "ai_image", "stock_video", "stock_image"]
            elif strategy == "stock_focused":
                order = ["stock_video", "stock_image", "ai_video", "ai_image"]
            else:  # hybrid — rotate the starting source so all 4 styles appear
                base = ["stock_video", "ai_video", "stock_image", "ai_image"]
                order = base[idx % 4:] + base[:idx % 4]

            errors = []
            for source in order:
                try:
                    candidate = attempts[source]()
                    cand_hash = get_file_hash(candidate)
                    if cand_hash and cand_hash in self.seen_clip_hashes:
                        logger.warning(f"[VISUAL] Segment {idx+1}: Collision on source '{source}' (hash {cand_hash[:8]}). Trying next source...")
                        continue
                    chosen_clip = candidate
                    break
                except Exception as e:
                    logger.warning(f"[VISUAL] Segment {idx+1} source '{source}' failed: {e}")
                    errors.append(f"{source}: {e}")

        if not chosen_clip:
            # Fallback to unique animated gradient motion card with Ken Burns motion
            logger.warning(f"[VISUAL] Segment {idx+1}: Standard sources failed. Generating standalone gradient visual...")
            c1 = ["0x0a1128", "0x1b1b2f", "0x162447", "0x1f4068", "0x111d5e", "0x0f3460", "0x1e152a", "0x0c2461"][idx % 8]
            c2 = ["0x1a2a6c", "0x2c3e50", "0x20002c", "0x34495e", "0x1e3c72", "0x2a5298", "0x3c1053", "0x1b1464"][idx % 8]
            subprocess.run([
                "ffmpeg", "-y", "-f", "lavfi",
                "-i", f"gradients=s=1280x720:c0={c1}:c1={c2}:d={duration_sec:.2f}:speed=0.005",
                "-vf", "noise=alls=12:allf=t+u,vignette=PI/4,format=yuv420p",
                "-r", "25", "-c:v", "libx264", "-preset", "ultrafast", "-pix_fmt", "yuv420p", "-an", final_clip_path
            ], check=True, capture_output=True)
            chosen_clip = final_clip_path

        # Record fingerprint for zero duplicate guarantee
        clip_hash = get_file_hash(chosen_clip)
        if clip_hash:
            self.seen_clip_hashes.add(clip_hash)

        # Apply dynamic 35mm film grain filter overlay if scene depicts past/historical events
        if is_historical_scene(seg, topic=getattr(self.cfg, "topic", "")):
            aged_tmp = os.path.join(out_dir, f"aged_{idx:03d}.mp4")
            apply_film_grain_overlay(chosen_clip, aged_tmp)
            if os.path.exists(aged_tmp) and os.path.getsize(aged_tmp) > 1000:
                import shutil
                shutil.move(aged_tmp, chosen_clip)
                logger.info(f"[VISUAL] Segment {idx+1}: Applied authentic 35mm film grain & vintage aging overlay.")

        return chosen_clip

    def generate(self, script: Dict[str, Any], job_dir: str, segment_timings: Optional[List[Dict[str, Any]]] = None) -> Dict[str, List[str]]:
        """
        Generate bilingual sequences of video clips matching each script segment.
        Returns: {"en": List[str], "es": List[str]}
        If on-screen text is present in a segment, generates distinct English and Spanish clips.
        """
        self.stock_fetcher.reset_session()
        self.seen_clip_hashes.clear()
        segs = script.get("script_segments", [])
        logger.info(f"[VISUAL] Producing {len(segs)} video clips (bilingual text-matching enabled)...")
        pc = self.pcs.get("nvidiam")
        if not pc or not pc.check(8188):
            raise RuntimeError("NvidiaLLM ComfyUI is unreachable on port 8188")

        clips_dir = os.path.join(job_dir, "clips")
        os.makedirs(clips_dir, exist_ok=True)

        clips_en = []
        clips_es = []

        for i, seg in enumerate(segs):
            t0 = time.time()
            dur = segment_timings[i]["duration"] if segment_timings and i < len(segment_timings) else seg.get("duration_estimate", 18)
            base_clip_path = self._generate_segment_visual(pc, seg, i, clips_dir, dur)

            text_en = (seg.get("on_screen_text") or "").strip()
            text_es = (seg.get("on_screen_text_es") or text_en).strip()

            if text_en or text_es:
                # Segment has on-screen text: render separate English and Spanish video clips
                clip_en = os.path.join(clips_dir, f"clip_en_{i:03d}.mp4")
                clip_es = os.path.join(clips_dir, f"clip_es_{i:03d}.mp4")

                apply_text_overlay(base_clip_path, clip_en, text_en, dur)
                apply_text_overlay(base_clip_path, clip_es, text_es, dur)

                clips_en.append(clip_en)
                clips_es.append(clip_es)
                logger.info(f"[VISUAL] Clip {i+1}/{len(segs)} bilingual text applied: '{text_en}' (EN) / '{text_es}' (ES) in {time.time()-t0:.1f}s")
            else:
                # Clean visual clip without text is shared across both languages
                clips_en.append(base_clip_path)
                clips_es.append(base_clip_path)
                logger.info(f"[VISUAL] Clip {i+1}/{len(segs)} generated in {time.time()-t0:.1f}s -> {os.path.basename(base_clip_path)}")

        return {"en": clips_en, "es": clips_es}