more fixes + yt b-roll

pipeline/research.py

@@ -0,0 +1,184 @@
+"""
+Competitive Topic Research & Viral Explainer Intelligence Engine.
+Researches the highest-viewed YouTube videos and viral narrative structures
+for given topics, and autonomously discovers high-velocity popular topics.
+"""
+
+import re
+import json
+import urllib.request
+import urllib.parse
+import logging
+from typing import Dict, Any, List, Optional
+from .remote import RemotePC
+
+logger = logging.getLogger("YTFactory.Research")
+
+# High-retention explainer niches with proven multi-million view formulas
+POPULAR_EXPLAINER_NICHES = [
+    "Lost Civilizations & Ancient Archaeology",
+    "Astrophysics & Space Paradoxes",
+    "Origins of Human Civilization & Anthropology",
+    "History of Computing & Revolutionary Technology",
+    "Deep Ocean Mysteries & Earth Sciences",
+    "Unsolved Historical Enigmas & Cold Cases",
+    "Economic Scandals & Corporate History",
+    "Neuroscience & Psychology Secrets"
+]
+
+class ViralTopicResearcher:
+    def __init__(self, llm_pc: Optional[RemotePC] = None, llm_port: int = 8002):
+        self.llm_pc = llm_pc
+        self.llm_port = llm_port
+        self.user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
+
+    def _search_web_insights(self, topic: str) -> str:
+        """Fetch search snippets and YouTube search results for the topic."""
+        query = f"{topic} documentary explained"
+        clean_q = urllib.parse.quote(query)
+        url = f"https://html.duckduckgo.com/html/?q={clean_q}"
+        req = urllib.request.Request(url, headers={"User-Agent": self.user_agent})
+        
+        snippets = []
+        try:
+            with urllib.request.urlopen(req, timeout=8) as resp:
+                html = resp.read().decode("utf-8", errors="ignore")
+                # Extract search result snippets
+                matches = re.findall(r'<a class="result__snippet[^>]*>(.*?)</a>', html, re.DOTALL)
+                for m in matches[:6]:
+                    clean_text = re.sub(r'<[^>]+>', '', m).strip()
+                    if len(clean_text) > 30:
+                        snippets.append(clean_text)
+        except Exception as e:
+            logger.debug(f"[RESEARCH] Web search note: {e}")
+
+        return "\n".join(snippets) if snippets else f"Popular documentary subject regarding: {topic}"
+
+    def research_topic(self, topic: str) -> Dict[str, Any]:
+        """
+        Research multi-million view YouTube explainer videos on this topic to extract:
+        - The most viral psychological hook & curiosity gap
+        - Key narrative revelations and counter-intuitive insights
+        - Step-by-step explainer pacing (Kurzgesagt / Veritasium / ColdFusion style)
+        - Core visual motifs for artistic consistency
+        """
+        logger.info(f"[RESEARCH] Researching top-performing viral explainer structures for: '{topic}'...")
+        web_context = self._search_web_insights(topic)
+
+        prompt = f"""You are a master YouTube retention strategist and documentary showrunner (studying 10M+ view explainer channels like Veritasium, Kurzgesagt, ColdFusion, Wendover, and Johnny Harris).
+
+TOPIC: "{topic}"
+WEB SEARCH CONTEXT:
+{web_context[:1500]}
+
+OBJECTIVE:
+Analyze why the most viral videos on this topic succeed and formulate a comprehensive high-retention blueprint for our original 7-12 minute explainer documentary.
+
+Provide:
+1. "viral_hook_angle": The irresistible curiosity gap / central question that hooks viewers in the first 10 seconds.
+2. "common_misconceptions": 2-3 widely believed myths that we will dramatically debunk.
+3. "key_revelations": 3-4 mind-blowing historical/scientific milestones or paradoxes to structure the video around.
+4. "narrative_arc": 5-act explainer progression:
+   - Act 1 (The Hook & Mystery)
+   - Act 2 (The Prevailing Myth)
+   - Act 3 (The Breakthrough Discovery)
+   - Act 4 (The Mechanism / Step-by-Step Reality)
+   - Act 5 (The Paradigm Shift & Modern Epiphany)
+5. "visual_style_anchor": A cohesive 10-15 word visual art direction description to ensure ALL scene clips look visually unified.
+
+STRICT JSON OUTPUT:
+{{
+  "viral_hook_angle": "...",
+  "common_misconceptions": ["...", "..."],
+  "key_revelations": ["...", "...", "..."],
+  "narrative_arc": ["Act 1: ...", "Act 2: ...", "Act 3: ...", "Act 4: ...", "Act 5: ..."],
+  "visual_style_anchor": "cinematic documentary style, warm archival tones, 35mm film texture, photorealistic 8k"
+}}"""
+
+        if self.llm_pc and self.llm_pc.check(self.llm_port):
+            try:
+                payload = {
+                    "model": "llama",
+                    "messages": [{"role": "user", "content": prompt}],
+                    "temperature": 0.7,
+                    "max_tokens": 2048
+                }
+                r = self.llm_pc.post_json(self.llm_port, "/v1/chat/completions", payload, timeout=60)
+                if r and "choices" in r:
+                    content = r["choices"][0]["message"]["content"]
+                    # Extract JSON
+                    s = content.find('{')
+                    e = content.rfind('}')
+                    if s != -1 and e > s:
+                        dossier = json.loads(content[s:e+1])
+                        logger.info(f"[RESEARCH] Extracted viral hook: '{dossier.get('viral_hook_angle', '')[:60]}...'")
+                        return dossier
+            except Exception as e:
+                logger.warning(f"[RESEARCH] LLM topic analysis note: {e}")
+
+        # Fallback structured research dossier
+        return {
+            "viral_hook_angle": f"Everything we thought we understood about {topic} was challenged by a single discovery.",
+            "common_misconceptions": [f"Standard historical accounts of {topic} overlook the key evidence."],
+            "key_revelations": [f"The untold origin and catalyst behind {topic}."],
+            "narrative_arc": ["The Opening Mystery", "The Historical Consensus", "The Revolutionary Discovery", "The Hidden Mechanism", "The Modern Impact"],
+            "visual_style_anchor": "cinematic documentary style, warm archival color grade, 35mm film texture, photorealistic 8k"
+        }
+
+    def discover_popular_trending_topics(self, niche: str = "", count: int = 3) -> List[Dict[str, Any]]:
+        """
+        Autonomously research and generate high-velocity popular YouTube explainer topics.
+        """
+        logger.info(f"[AUTO-RESEARCH] Discovering {count} high-velocity YouTube explainer topics...")
+        target_niche = niche if niche else POPULAR_EXPLAINER_NICHES[0]
+
+        prompt = f"""You are the Head of Programming for top-tier YouTube explainer channels (like Kurzgesagt, ColdFusion, Lemmino, Wendover).
+Generate {count} distinct, multi-million-view potential documentary explainer video topics for the niche: "{target_niche}".
+
+Requirements for EACH topic:
+- Must have massive viral appeal, strong curiosity gaps, and deep explainer substance.
+- Target duration: 7 to 12 minutes.
+- Titles must be high CTR, intriguing, and professional.
+
+STRICT JSON OUTPUT:
+{{
+  "topics": [
+    {{
+      "topic": "The Full Intriguing Video Title",
+      "curiosity_angle": "Why viewers can't resist clicking",
+      "target_niche": "{target_niche}"
+    }}
+  ]
+}}"""
+
+        if self.llm_pc and self.llm_pc.check(self.llm_port):
+            try:
+                payload = {
+                    "model": "llama",
+                    "messages": [{"role": "user", "content": prompt}],
+                    "temperature": 0.8,
+                    "max_tokens": 2048
+                }
+                r = self.llm_pc.post_json(self.llm_port, "/v1/chat/completions", payload, timeout=60)
+                if r and "choices" in r:
+                    content = r["choices"][0]["message"]["content"]
+                    s = content.find('{')
+                    e = content.rfind('}')
+                    if s != -1 and e > s:
+                        data = json.loads(content[s:e+1])
+                        topics = data.get("topics", [])
+                        if topics:
+                            logger.info(f"[AUTO-RESEARCH] Discovered {len(topics)} viral topics ready for production.")
+                            return topics
+            except Exception as e:
+                logger.warning(f"[AUTO-RESEARCH] LLM discovery note: {e}")
+
+        # Fallback curated high-velocity explainer topics
+        fallback_catalog = [
+            {"topic": "The Mystery of Göbekli Tepe and the Civilization Before History", "curiosity_angle": "How hunter gatherers built stone monoliths 7,000 years before Stonehenge."},
+            {"topic": "How the First Microprocessor Silently Changed the Human Race", "curiosity_angle": "The hidden engineering race in 1971 that birthed modern personal computing."},
+            {"topic": "The Black Hole Information Paradox: How the Universe Destroys Reality", "curiosity_angle": "The terrifying theoretical paradox that baffled Stephen Hawking for decades."},
+            {"topic": "The True Story of the Antikythera Mechanism: The 2,000-Year-Old Analog Computer", "curiosity_angle": "Ancient Greek gears that shouldn't have existed for another millennium."},
+            {"topic": "How Beer Built Human Civilization: The Agriculture Hypothesis", "curiosity_angle": "Why humanity abandoned nomadic life to brew the world's first fermented grains."}
+        ]
+        return fallback_catalog[:count]

pipeline/script_gen.py

@@ -9,6 +9,7 @@ import logging
 import re
 from typing import Dict, Any, List, Optional
 from .remote import RemotePC
+from .research import ViralTopicResearcher
 
 logger = logging.getLogger("YTFactory.Script")
 
@@ -16,31 +17,51 @@ class ScriptGen:
     def __init__(self, pcs: Dict[str, RemotePC], llm_port: int = 8002):
         self.pcs = pcs
         self.llm_port = llm_port
+        self.researcher = ViralTopicResearcher(pcs.get("amdllm"), llm_port)
 
-    def _build_master_prompt(self, topic: str, duration: int) -> str:
+    def _build_master_prompt(self, topic: str, duration: int, research_dossier: Optional[Dict[str, Any]] = None) -> str:
         # Pacing: ~140 words per minute for professional documentary delivery.
         num_segments = max(4, duration // 20)
         target_words = max(70, int(duration * 140 / 60))
         words_per_seg = max(25, target_words // num_segments)
 
-        return f"""You are an award-winning documentary writer and YouTube strategist.
-Generate a complete, high-retention faceless documentary script for the topic: "{topic}".
+        dossier = research_dossier or {}
+        viral_hook = dossier.get("viral_hook_angle", "The unexpected mystery behind this subject.")
+        misconceptions = ", ".join(dossier.get("common_misconceptions", [])) or "Standard historical misconceptions"
+        revelations = ", ".join(dossier.get("key_revelations", [])) or "Groundbreaking discoveries"
+        visual_style = dossier.get("visual_style_anchor", "cinematic documentary style, warm archival tones, 35mm film texture, photorealistic 8k")
+
+        return f"""You are a master YouTube retention strategist and award-winning documentary showrunner (modeling 10M+ view explainer channels like Veritasium, Kurzgesagt, ColdFusion, and Wendover).
+Generate an authoritative, high-retention faceless explainer documentary script for the topic: "{topic}".
+
+COMPETITIVE VIRAL INTELLIGENCE & RESEARCH:
+- Primary Curiosity Gap Hook: "{viral_hook}"
+- Critical Myths to Debunk: {misconceptions}
+- Breakthrough Revelations & Step-by-Step Mechanisms: {revelations}
+- Unified Visual Art Direction: "{visual_style}"
 
 SPECIFICATIONS:
 - Target Duration: Exactly ~{duration} seconds of continuous narration
 - Total Word Count: ~{target_words} spoken words across all segments (~{words_per_seg} words per segment)
 - Number of Segments: Exactly {num_segments} chronological segments
-- Tone: Mysterious, engaging, cinematic, informative, highly gripping
+- Tone: Mysterious, authoritative, engaging, cinematic, insightful
+- Narrative Flow (5-Act Explainer Blueprint):
+  * Act 1 (Opening): Irresistible hook & psychological curiosity gap
+  * Act 2 (The Illusion): The common myth or historical puzzle
+  * Act 3 (The Turning Point): The catalyst or critical breakthrough
+  * Act 4 (The Mechanism): Step-by-step breakdown of how it actually happened
+  * Act 5 (The Paradigm Shift): Profound modern conclusion and lasting impact
 - Spoken Depth: Each segment MUST contain a full, rich spoken paragraph ({words_per_seg - 5} to {words_per_seg + 10} words) so the narration fills the entire video duration without silence or rushing.
 
 For EACH segment provide:
 1. "chapter_title": Short 2-4 word chapter title (e.g., "The Ancient Mystery", "The Critical Discovery")
 2. "text": English spoken narration (rich 2-4 sentence narrative paragraph, natural pacing, compelling storytelling)
-3. "visual_prompt": Highly detailed visual scene description for an AI video/image generator (cinematic lighting, camera angle, 8k documentary style)
+3. "visual_prompt": Highly detailed visual scene description for an AI video/image generator (cinematic lighting, camera angle, {visual_style})
 4. "stock_query": 2-4 concrete, high-precision visual search keywords describing physical objects/scenery that exist in stock video libraries (e.g., "ancient temple monoliths", "medieval cathedral altar", "archaeological excavation desert", "prehistoric stone tools", "ancient scrolls manuscript"). Never use abstract concepts, full sentences, or obscure person names.
 5. "on_screen_text": Optional 2-5 word lower-third label, key statistic, date, or scene title to display on screen (e.g., "Depth: 11,034m", "Year: 1960", "Pacific Ocean Abyss", or "" if none)
 6. "is_historical": boolean (true if this segment depicts past events, ancient civilizations, historical figures, archaeological discoveries, or vintage eras; false if modern/contemporary)
-7. "duration_estimate": Estimated duration in seconds ({max(10, duration // num_segments)}s)
+7. "person_name": Name of any specific historical person, scientist, inventor, philosopher, leader, or notable figure discussed in this scene (e.g., "Alan Turing", "Steve Wozniak", "Charles Babbage", "Jack the Ripper", or "" if none)
+8. "duration_estimate": Estimated duration in seconds ({max(10, duration // num_segments)}s)
 
 METADATA REQUIREMENTS:
 - "titles": Exactly 3 distinct viral, high-CTR YouTube titles for A/B testing (max 85 chars each)
@@ -352,7 +373,9 @@ STRICT JSON OUTPUT:
         if not pc or not pc.check(self.llm_port):
             raise RuntimeError(f"AMDLLM llama.cpp not reachable on port {self.llm_port}")
 
-        prompt = self._build_master_prompt(topic, duration)
+        # ── Step 1.0: Viral Topic Research & High-Retention Explainer Blueprint ──
+        research_dossier = self.researcher.research_topic(topic)
+        prompt = self._build_master_prompt(topic, duration, research_dossier=research_dossier)
         
         for attempt in range(1, 4):
             logger.info(f"[SCRIPT] Requesting script generation from AMDLLM (attempt {attempt}/3)...")
@@ -372,6 +395,8 @@ STRICT JSON OUTPUT:
                     titles.append(f"{topic} - Part {len(titles)+1}")
                 data["titles"] = titles[:3]
                 data["title"] = titles[0]
+                data["research_dossier"] = research_dossier
+                data["visual_style_anchor"] = research_dossier.get("visual_style_anchor", "cinematic documentary style, warm archival tones, 35mm film texture, photorealistic 8k")
 
                 segments = data.get("script_segments", [])
                 if not segments:
@@ -381,6 +406,7 @@ STRICT JSON OUTPUT:
                     s.setdefault("chapter_title", f"Part {i+1}")
                     s.setdefault("duration_estimate", max(15, duration // len(segments)))
                     s.setdefault("stock_query", topic)
+                    s.setdefault("visual_style_anchor", data["visual_style_anchor"])
                     if not s.get("visual_prompt"):
                         s["visual_prompt"] = f"Cinematic documentary footage illustrating: {s.get('text', topic)}"
 

pipeline/tts_gen.py

@@ -221,7 +221,8 @@ class TTSGen:
                 "end": current_time + dur,
                 "duration": dur,
                 "chapter_title": chapter_title,
-                "text": text,
+                "text": raw_text,
+                "spoken_text": spoken_text,
                 "wav_file": seg_path
             })
             current_time += dur

pipeline/visual_gen.py

@@ -66,11 +66,149 @@ def clean_search_keywords(text: str) -> str:
     # Keep up to 3 most specific visual keywords
     return " ".join(filtered[:3]) if filtered else text.strip()
 
+import hashlib
+
+def get_file_hash(path: str) -> str:
+    """Compute SHA-256 hash of a file to detect exact duplicate clips or images."""
+    if not path or not os.path.exists(path) or os.path.getsize(path) < 100:
+        return ""
+    h = hashlib.sha256()
+    try:
+        with open(path, "rb") as f:
+            while chunk := f.read(65536):
+                h.update(chunk)
+        return h.hexdigest()
+    except Exception:
+        return ""
+
 class StockMediaFetcher:
     def __init__(self, pexels_key: Optional[str] = None, pixabay_key: Optional[str] = None):
         self.pexels_key = pexels_key
         self.pixabay_key = pixabay_key
         self.user_agent = "YouTubeFactory/2.0 (faceless-video-pipeline)"
+        self.used_media_urls = set()
+        self.used_file_hashes = set()
+        self.used_filenames = set()
+        self.used_youtube_ids = set()
+
+    def reset_session(self):
+        """Reset used media set at the start of a new video job."""
+        self.used_media_urls.clear()
+        self.used_file_hashes.clear()
+        self.used_filenames.clear()
+        self.used_youtube_ids.clear()
+
+    def _query_youtube_broll(self, query: str, duration_sec: float, out_path: str, offset_sec: int = 15) -> Optional[str]:
+        """Search YouTube for relevant topic documentary/b-roll footage and download muted HD clip."""
+        try:
+            clean_q = re.sub(r'[^A-Za-z0-9 ]', '', query).strip()
+            if not clean_q:
+                return None
+            search_target = f"ytsearch8:{clean_q} documentary footage"
+            logger.info(f"[STOCK:YOUTUBE] Searching YouTube for b-roll footage: '{clean_q}'...")
+            cmd_search = [
+                "yt-dlp", search_target,
+                "--dump-json", "--flat-playlist", "--no-warnings"
+            ]
+            r = subprocess.run(cmd_search, capture_output=True, text=True, timeout=20)
+            lines = [l for l in r.stdout.strip().split("\n") if l]
+            for line in lines:
+                try:
+                    d = json.loads(line)
+                    vid_id = d.get("id")
+                    dur = d.get("duration") or 60
+                    if not vid_id or vid_id in self.used_youtube_ids or (dur and dur < 12):
+                        continue
+
+                    # Determine clean mid-video section avoiding intros/outros
+                    start_t = min(max(15, offset_sec), max(5, int(dur) - int(duration_sec) - 5)) if dur else 15
+                    end_t = start_t + int(duration_sec) + 3
+
+                    raw_tmp = out_path + f".raw_{vid_id}.mp4"
+                    cmd_dl = [
+                        "yt-dlp", f"https://www.youtube.com/watch?v={vid_id}",
+                        "--extractor-args", "youtube:player_client=android,web",
+                        "-f", "22/18/best[height<=720]/best",
+                        "--download-sections", f"*{start_t}-{end_t}",
+                        "--force-keyframes-at-cuts",
+                        "-o", raw_tmp,
+                        "--no-playlist",
+                        "--socket-timeout", "15"
+                    ]
+                    r_dl = subprocess.run(cmd_dl, capture_output=True, text=True, timeout=35)
+                    if r_dl.returncode == 0 and os.path.exists(raw_tmp) and os.path.getsize(raw_tmp) > 5000:
+                        # Normalize to 1280x720 25fps muted video (stripping any YouTube audio)
+                        cmd_norm = [
+                            "ffmpeg", "-y", "-i", raw_tmp,
+                            "-vf", "scale=1280:720:force_original_aspect_ratio=decrease,pad=1280:720:(ow-iw)/2:(oh-ih)/2:black,fps=25,format=yuv420p",
+                            "-t", f"{duration_sec:.2f}", "-r", "25",
+                            "-c:v", "libx264", "-preset", "fast", "-crf", "22", "-pix_fmt", "yuv420p",
+                            "-an", out_path
+                        ]
+                        r_norm = subprocess.run(cmd_norm, capture_output=True, text=True)
+                        if os.path.exists(raw_tmp):
+                            try:
+                                os.remove(raw_tmp)
+                            except OSError:
+                                pass
+
+                        if r_norm.returncode == 0 and os.path.exists(out_path) and os.path.getsize(out_path) > 5000:
+                            h = get_file_hash(out_path)
+                            if h and h in self.used_file_hashes:
+                                logger.warning(f"[STOCK:YOUTUBE] Duplicate content detected (hash {h[:8]}). Continuing search...")
+                                try:
+                                    os.remove(out_path)
+                                except OSError:
+                                    pass
+                                continue
+                            self.used_youtube_ids.add(vid_id)
+                            self.used_media_urls.add(f"https://www.youtube.com/watch?v={vid_id}")
+                            if h:
+                                self.used_file_hashes.add(h)
+                            logger.info(f"[STOCK:YOUTUBE] Extracted high quality muted YouTube b-roll clip from '{d.get('title')}'")
+                            return out_path
+                except Exception as e:
+                    logger.debug(f"[STOCK:YOUTUBE] Candidate parse error: {e}")
+                    continue
+        except Exception as e:
+            logger.debug(f"[STOCK:YOUTUBE] Search failed for '{query}': {e}")
+        return None
+
+    def query_person_image(self, person_name: str) -> Optional[str]:
+        """Fetch authentic historical portrait/photograph of a person from Wikipedia / Wikimedia Commons."""
+        if not person_name or len(person_name.strip()) < 3:
+            return None
+        clean_name = person_name.strip()
+        logger.info(f"[STOCK:PERSON] Searching authentic archival portrait/photo for person: '{clean_name}'...")
+        
+        # 1. Query Wikipedia PageImage API (1920px curated lead portrait)
+        try:
+            encoded = urllib.parse.quote(clean_name)
+            url = f"https://en.wikipedia.org/w/api.php?action=query&titles={encoded}&prop=pageimages&pithumbsize=1920&format=json"
+            req = urllib.request.Request(url, headers={"User-Agent": self.user_agent})
+            with urllib.request.urlopen(req, timeout=8) as resp:
+                data = json.loads(resp.read().decode("utf-8"))
+                pages = data.get("query", {}).get("pages", {})
+                for pid, page in pages.items():
+                    if pid != "-1" and "thumbnail" in page:
+                        src = page["thumbnail"].get("source")
+                        if src and src not in self.used_media_urls:
+                            return src
+        except Exception as e:
+            logger.debug(f"[STOCK:PERSON] Wikipedia query note for '{clean_name}': {e}")
+
+        # 2. Query Wikimedia Commons for high-res archival portrait
+        try:
+            wiki_img = self._query_wikimedia(f"portrait {clean_name}", is_video=False)
+            if wiki_img:
+                return wiki_img
+            wiki_img2 = self._query_wikimedia(clean_name, is_video=False)
+            if wiki_img2:
+                return wiki_img2
+        except Exception as e:
+            logger.debug(f"[STOCK:PERSON] Wikimedia query note for '{clean_name}': {e}")
+
+        return None
 
     def _query_wikimedia(self, query: str, is_video: bool = False) -> Optional[str]:
         """Search Wikimedia Commons for high-resolution images or videos without API keys."""
@@ -79,7 +217,7 @@ class StockMediaFetcher:
             encoded_q = urllib.parse.quote(clean_q)
             api_url = (
                 f"https://commons.wikimedia.org/w/api.php?action=query&generator=search"
-                f"&gsrsearch={encoded_q}&gsrnamespace=6&gsrlimit=5&prop=imageinfo"
+                f"&gsrsearch={encoded_q}&gsrnamespace=6&gsrlimit=10&prop=imageinfo"
                 f"&iiprop=url|mime|size&format=json"
             )
             req = urllib.request.Request(api_url, headers={"User-Agent": self.user_agent})
@@ -91,6 +229,11 @@ class StockMediaFetcher:
                     if not infos:
                         continue
                     url = infos[0].get("url", "")
+                    if not url or url in self.used_media_urls:
+                        continue
+                    clean_name = os.path.basename(urllib.parse.urlparse(url).path)
+                    if clean_name and clean_name in self.used_filenames:
+                        continue
                     mime = infos[0].get("mime", "").lower()
                     if is_video and ("video" in mime or url.endswith((".webm", ".mp4", ".ogv"))):
                         return url
@@ -106,7 +249,7 @@ class StockMediaFetcher:
             return None
         try:
             clean_q = urllib.parse.quote(query)
-            url = f"https://api.pexels.com/videos/search?query={clean_q}&per_page=5&orientation=landscape"
+            url = f"https://api.pexels.com/videos/search?query={clean_q}&per_page=10&orientation=landscape"
             headers = {"Authorization": self.pexels_key, "User-Agent": self.user_agent}
             req = urllib.request.Request(url, headers=headers)
             with urllib.request.urlopen(req, timeout=10) as resp:
@@ -114,12 +257,10 @@ class StockMediaFetcher:
                 videos = data.get("videos", [])
                 for v in videos:
                     files = v.get("video_files", [])
-                    # Prefer the smallest HD file (>=1280 wide) to keep downloads fast
                     hd = [vf for vf in files if vf.get("width", 0) >= 1280 and vf.get("link")]
-                    if hd:
-                        return sorted(hd, key=lambda x: x["width"])[0]["link"]
-                    if files and files[0].get("link"):
-                        return files[0]["link"]
+                    target = sorted(hd, key=lambda x: x["width"])[0]["link"] if hd else (files[0].get("link") if files else None)
+                    if target and target not in self.used_media_urls:
+                        return target
         except Exception as e:
             logger.debug(f"Pexels video query failed: {e}")
         return None
@@ -130,14 +271,16 @@ class StockMediaFetcher:
             return None
         try:
             clean_q = urllib.parse.quote(query)
-            url = f"https://api.pexels.com/v1/search?query={clean_q}&per_page=5&orientation=landscape"
+            url = f"https://api.pexels.com/v1/search?query={clean_q}&per_page=10&orientation=landscape"
             headers = {"Authorization": self.pexels_key, "User-Agent": self.user_agent}
             req = urllib.request.Request(url, headers=headers)
             with urllib.request.urlopen(req, timeout=10) as resp:
                 data = json.loads(resp.read().decode("utf-8"))
                 photos = data.get("photos", [])
-                if photos:
-                    return photos[0].get("src", {}).get("large2x") or photos[0].get("src", {}).get("large")
+                for p in photos:
+                    src = p.get("src", {}).get("large2x") or p.get("src", {}).get("large")
+                    if src and src not in self.used_media_urls:
+                        return src
         except Exception as e:
             logger.debug(f"Pexels photo query failed: {e}")
         return None
@@ -148,7 +291,7 @@ class StockMediaFetcher:
             return None
         try:
             clean_q = urllib.parse.quote(query)
-            url = f"https://pixabay.com/api/videos/?key={self.pixabay_key}&q={clean_q}&per_page=5&safesearch=true"
+            url = f"https://pixabay.com/api/videos/?key={self.pixabay_key}&q={clean_q}&per_page=10&safesearch=true"
             req = urllib.request.Request(url, headers={"User-Agent": self.user_agent})
             with urllib.request.urlopen(req, timeout=10) as resp:
                 data = json.loads(resp.read().decode("utf-8"))
@@ -157,7 +300,9 @@ class StockMediaFetcher:
                     for quality in ("large", "medium", "small"):
                         v = vids.get(quality)
                         if v and v.get("url") and v.get("width", 0) >= 640:
-                            return v["url"]
+                            target_url = v["url"]
+                            if target_url not in self.used_media_urls:
+                                return target_url
         except Exception as e:
             logger.debug(f"Pixabay video query failed: {e}")
         return None
@@ -169,24 +314,40 @@ class StockMediaFetcher:
         try:
             clean_q = urllib.parse.quote(query)
             url = (f"https://pixabay.com/api/?key={self.pixabay_key}&q={clean_q}"
-                   f"&image_type=photo&orientation=horizontal&per_page=5&safesearch=true")
+                   f"&image_type=photo&orientation=horizontal&per_page=10&safesearch=true")
             req = urllib.request.Request(url, headers={"User-Agent": self.user_agent})
             with urllib.request.urlopen(req, timeout=10) as resp:
                 data = json.loads(resp.read().decode("utf-8"))
                 hits = data.get("hits", [])
-                if hits:
-                    return hits[0].get("largeImageURL") or hits[0].get("webformatURL")
+                for h in hits:
+                    img_url = h.get("largeImageURL") or h.get("webformatURL")
+                    if img_url and img_url not in self.used_media_urls:
+                        return img_url
         except Exception as e:
             logger.debug(f"Pixabay photo query failed: {e}")
         return None
 
     def download_media(self, url: str, out_path: str) -> bool:
-        """Download remote media with retry and validation."""
+        """Download remote media with retry, validation, and content hash duplicate prevention."""
         try:
             req = urllib.request.Request(url, headers={"User-Agent": self.user_agent})
             with urllib.request.urlopen(req, timeout=30) as resp, open(out_path, "wb") as f:
                 f.write(resp.read())
             if os.path.exists(out_path) and os.path.getsize(out_path) > 5000:
+                h = get_file_hash(out_path)
+                if h and h in self.used_file_hashes:
+                    logger.warning(f"[STOCK] Duplicate asset content detected (hash {h[:8]}). Discarding duplicate...")
+                    try:
+                        os.remove(out_path)
+                    except OSError:
+                        pass
+                    return False
+                self.used_media_urls.add(url)
+                if h:
+                    self.used_file_hashes.add(h)
+                clean_name = os.path.basename(urllib.parse.urlparse(url).path)
+                if clean_name:
+                    self.used_filenames.add(clean_name)
                 return True
         except Exception as e:
             logger.debug(f"Download failed from {url}: {e}")
@@ -197,22 +358,38 @@ class StockMediaFetcher:
                 pass
         return False
 
-    def search_and_download(self, queries: Any, out_path: str, prefer_video: bool = False) -> Optional[str]:
+    def search_and_download(self, queries: Any, out_path: str, prefer_video: bool = False, duration_sec: float = 6.0) -> Optional[str]:
         """Search across stock engines with candidate query cascade and download asset."""
         if isinstance(queries, str):
             candidate_list = [queries]
         else:
             candidate_list = list(queries)
 
+        expanded_queries = []
         for raw_q in candidate_list:
             if not raw_q:
                 continue
-            cleaned_q = clean_search_keywords(raw_q)
-            if not cleaned_q:
-                continue
+            cleaned = clean_search_keywords(raw_q)
+            if cleaned and cleaned not in expanded_queries:
+                expanded_queries.append(cleaned)
+            words = cleaned.split()
+            if len(words) >= 2:
+                two_word = " ".join(words[:2])
+                if two_word not in expanded_queries:
+                    expanded_queries.append(two_word)
+                for w in words:
+                    if len(w) > 3 and w not in expanded_queries:
+                        expanded_queries.append(w)
 
+        for cleaned_q in expanded_queries:
             logger.info(f"[STOCK] Searching stock media for: '{cleaned_q}' (video={prefer_video})...")
             if prefer_video:
+                # 1. First priority: Real YouTube documentary B-roll footage without audio
+                yt_clip = self._query_youtube_broll(cleaned_q, duration_sec, out_path)
+                if yt_clip and os.path.exists(yt_clip) and os.path.getsize(yt_clip) > 5000:
+                    return yt_clip
+
+                # 2. Stock APIs (Pexels, Pixabay, Wikimedia)
                 for label, fn in (("Pexels", self._query_pexels_video),
                                   ("Pixabay", self._query_pixabay_video),
                                   ("Wikimedia", lambda q: self._query_wikimedia(q, is_video=True))):
@@ -229,7 +406,7 @@ class StockMediaFetcher:
                         logger.info(f"[STOCK] Downloaded {label} photo for '{cleaned_q}'")
                         return out_path
 
-        logger.debug(f"[STOCK] No matching stock media found for queries: {candidate_list}")
+        logger.debug(f"[STOCK] No matching stock media found for queries: {expanded_queries}")
         return None
 
 # ─── Ken Burns Motion Engine ────────────────────────────────────────────────
@@ -326,10 +503,10 @@ def normalize_clip_to_duration(src_path: str, out_path: str, duration_sec: float
 
 def apply_text_overlay(video_path: str, out_mp4: str, text: str, duration_sec: float) -> str:
     """
-    Render a broadcast-style lower-third text banner onto the video clip.
-    Only active if text is non-empty.
+    Render a broadcast-style lower-third text banner onto the video clip using UTF-8 textfile.
+    Supports full international characters (accents, umlauts, Spanish punctuation) without corruption.
     """
-    clean_text = re.sub(r"[^A-Za-z0-9 \-\?\!\,\:\'\.\/]", "", text).strip()
+    clean_text = text.strip()
     if not clean_text:
         if video_path != out_mp4:
             import shutil
@@ -348,14 +525,17 @@ def apply_text_overlay(video_path: str, out_mp4: str, text: str, duration_sec: f
             font_file = fp
             break
 
-    font_param = f":fontfile='{font_file}'" if font_file else ""
-    escaped_text = clean_text.replace("'", "'\\''").replace(":", "\\:")
+    # Write clean UTF-8 text to a temporary textfile for FFmpeg drawtext to avoid character stripping
+    import tempfile
+    txt_fd, txt_path = tempfile.mkstemp(suffix=".txt", prefix="banner_text_")
+    with open(txt_fd, "w", encoding="utf-8") as f:
+        f.write(clean_text)
 
-    # Lower-third banner active between 0.8s and (duration - 0.8s)
+    font_param = f":fontfile='{font_file}'" if font_file else ""
     end_t = max(1.0, duration_sec - 0.8)
     enable_expr = f"between(t,0.8,{end_t:.2f})"
     vf = (
-        f"drawtext=text='{escaped_text}'{font_param}:fontcolor=white:fontsize=36:"
+        f"drawtext=textfile='{txt_path}'{font_param}:fontcolor=white:fontsize=36:"
         f"box=1:boxcolor=black@0.65:boxborderw=14:x=60:y=h-th-60:enable='{enable_expr}'"
     )
 
@@ -366,6 +546,12 @@ def apply_text_overlay(video_path: str, out_mp4: str, text: str, duration_sec: f
         "-an", out_mp4
     ]
     r = subprocess.run(cmd, capture_output=True, text=True)
+    if os.path.exists(txt_path):
+        try:
+            os.remove(txt_path)
+        except OSError:
+            pass
+
     if r.returncode != 0:
         logger.warning(f"Text overlay rendering fallback: {r.stderr[-200:]}")
         if video_path != out_mp4:
@@ -427,6 +613,7 @@ class VisualGen:
     def __init__(self, pcs: Dict[str, RemotePC], cfg: Any):
         self.pcs = pcs
         self.cfg = cfg
+        self.seen_clip_hashes = set()
         self.stock_fetcher = StockMediaFetcher(
             pexels_key=getattr(cfg, "pexels_api_key", None),
             pixabay_key=getattr(cfg, "pixabay_api_key", None)
@@ -525,6 +712,8 @@ class VisualGen:
 
     def _try_ai_video(self, pc, prompt, seed, duration_sec, raw_path, final_clip_path, idx):
         """MiniMax H3 text-to-video clip, normalized to the segment duration."""
+        if not pc or not pc.check(8188):
+            raise RuntimeError("NvidiaLLM ComfyUI is not reachable on port 8188")
         h3_length = min(h3_snap_length(duration_sec), self.H3_CLIP_CAP)
         logger.info(f"[VISUAL] Segment {idx+1}: MiniMax H3 AI video ({h3_length} frames)...")
         wf = self._h3_workflow(prompt, seed, self.cfg.clip_width, self.cfg.clip_height, h3_length)
@@ -535,6 +724,8 @@ class VisualGen:
 
     def _try_ai_image(self, pc, prompt, seed, duration_sec, raw_path, final_clip_path, idx):
         """SDXL AI image animated with Ken Burns motion."""
+        if not pc or not pc.check(8188):
+            raise RuntimeError("NvidiaLLM ComfyUI is not reachable on port 8188")
         logger.info(f"[VISUAL] Segment {idx+1}: SDXL AI image + Ken Burns...")
         pid = self._submit(pc.host, self._sdxl_workflow(prompt, seed))
         outputs = self._wait_result(pc.host, pid, timeout_s=300)
@@ -545,25 +736,56 @@ class VisualGen:
         """Stock video (looped/trimmed) or stock image (Ken Burns)."""
         kind = "video" if prefer_video else "image"
         logger.info(f"[VISUAL] Segment {idx+1}: stock {kind} for '{query}'...")
-        got = self.stock_fetcher.search_and_download(query, raw_path, prefer_video=prefer_video)
+        got = self.stock_fetcher.search_and_download(query, raw_path, prefer_video=prefer_video, duration_sec=duration_sec)
         if not got or not os.path.exists(got):
             raise RuntimeError(f"no stock {kind} found for '{query}'")
         if got.lower().endswith((".mp4", ".webm", ".mov")):
             return normalize_clip_to_duration(got, final_clip_path, duration_sec)
         return apply_ken_burns_effect(got, final_clip_path, duration_sec, motion_type=idx)
 
+    def _try_person(self, person_name: str, duration_sec: float, raw_path: str, final_clip_path: str, idx: int):
+        """Fetch authentic photo/portrait of historical person and animate with Ken Burns motion."""
+        if not person_name:
+            raise RuntimeError("no person name provided")
+        img_url = self.stock_fetcher.query_person_image(person_name)
+        if not img_url:
+            raise RuntimeError(f"no authentic archival photo found for person '{person_name}'")
+        if not self.stock_fetcher.download_media(img_url, raw_path):
+            raise RuntimeError(f"failed to download person photo from {img_url}")
+        logger.info(f"[VISUAL] Segment {idx+1}: Using authentic historical photo/portrait for '{person_name}'")
+        return apply_ken_burns_effect(raw_path, final_clip_path, duration_sec, motion_type=idx)
+
     def _generate_segment_visual(self, pc: RemotePC, seg: Dict[str, Any], idx: int, out_dir: str, duration_sec: float) -> str:
         """
         Produce one visual for a segment as an MP4 of exactly duration_sec.
 
         "hybrid" rotates across 4 styles for maximum variety:
-          AI video (MiniMax H3) -> stock video -> AI image (Ken Burns) -> stock image (Ken Burns)
-        Each source falls back to the next one in the rotation on failure.
+          YouTube b-roll / stock video -> AI video -> stock image (Ken Burns) -> AI image (Ken Burns)
+        Each source falls back to the next one in the rotation on failure or duplicate collision.
         """
-        prompt = (seg.get("visual_prompt") or seg.get("text", "")) + ", cinematic documentary style, 8k, detailed"
+        os.makedirs(out_dir, exist_ok=True)
+        CAMERA_ANGLES = [
+            "wide establishing cinematic drone shot",
+            "dramatic macro close-up detail shot",
+            "dynamic low-angle heroic perspective",
+            "atmospheric high-angle bird's eye view",
+            "side profile tracking cinematic perspective",
+            "epic panoramic sweeping shot",
+            "dramatic cinematic lighting with volumetric shadows",
+            "slow dolly-in center focus shot"
+        ]
+        angle = CAMERA_ANGLES[idx % len(CAMERA_ANGLES)]
+        visual_anchor = (
+            seg.get("visual_style_anchor") or
+            getattr(self.cfg, "visual_style_anchor", "") or
+            "cinematic documentary style, consistent warm color grading, 35mm film texture, photorealistic 8k"
+        )
+        raw_prompt = seg.get("visual_prompt") or seg.get("text", "")
+        prompt = f"{raw_prompt}, {angle}, {visual_anchor}"
         primary_stock = seg.get("stock_query") or ""
         chapter_stock = seg.get("chapter_title") or ""
         topic_anchor = clean_search_keywords(getattr(self.cfg, "topic", "") or "")
+        person_name = (seg.get("person_name") or "").strip()
         
         # Build prioritized topical query cascade
         query_candidates = []
@@ -576,45 +798,73 @@ class VisualGen:
         if topic_anchor:
             query_candidates.append(topic_anchor)
 
-        seed = zlib.crc32(prompt.encode()) % (2**31)
+        import time
+        seed = zlib.crc32(f"{prompt}_{idx}_{time.time()}_{os.urandom(4)}".encode()) % (2**31)
 
         final_clip_path = os.path.join(out_dir, f"clip_{idx:03d}.mp4")
         raw_video = os.path.join(out_dir, f"raw_vid_{idx:03d}.mp4")
         raw_img = os.path.join(out_dir, f"raw_img_{idx:03d}.jpg")
 
+        chosen_clip = None
+
+        # 1. If talking about a specific person, prioritize authentic archival portrait/photo
+        if person_name:
+            try:
+                candidate = self._try_person(person_name, duration_sec, raw_img, final_clip_path, idx)
+                cand_hash = get_file_hash(candidate)
+                if cand_hash and cand_hash not in self.seen_clip_hashes:
+                    chosen_clip = candidate
+            except Exception as e:
+                logger.debug(f"[VISUAL] Person portrait search note for '{person_name}': {e}")
+
+        if not chosen_clip:
             strategy = getattr(self.cfg, "visual_strategy", "hybrid")
             attempts = {
-            "ai_video":    lambda: self._try_ai_video(pc, prompt, seed, duration_sec, raw_video, final_clip_path, idx),
                 "stock_video": lambda: self._try_stock(query_candidates, duration_sec, raw_video, final_clip_path, idx, prefer_video=True),
-            "ai_image":    lambda: self._try_ai_image(pc, prompt, seed, duration_sec, raw_img, final_clip_path, idx),
+                "ai_video":    lambda: self._try_ai_video(pc, prompt, seed, duration_sec, raw_video, final_clip_path, idx),
                 "stock_image": lambda: self._try_stock(query_candidates, duration_sec, raw_img, final_clip_path, idx, prefer_video=False),
+                "ai_image":    lambda: self._try_ai_image(pc, prompt, seed, duration_sec, raw_img, final_clip_path, idx),
             }
             if strategy == "all_ai":
                 order = ["ai_video", "ai_image", "stock_video", "stock_image"]
             elif strategy == "stock_focused":
                 order = ["stock_video", "stock_image", "ai_video", "ai_image"]
             else:  # hybrid — rotate the starting source so all 4 styles appear
-            base = ["ai_video", "stock_video", "ai_image", "stock_image"]
+                base = ["stock_video", "ai_video", "stock_image", "ai_image"]
                 order = base[idx % 4:] + base[:idx % 4]
 
-        chosen_clip = None
+            errors = []
             for source in order:
                 try:
-                chosen_clip = attempts[source]()
+                    candidate = attempts[source]()
+                    cand_hash = get_file_hash(candidate)
+                    if cand_hash and cand_hash in self.seen_clip_hashes:
+                        logger.warning(f"[VISUAL] Segment {idx+1}: Collision on source '{source}' (hash {cand_hash[:8]}). Trying next source...")
+                        continue
+                    chosen_clip = candidate
                     break
                 except Exception as e:
                     logger.warning(f"[VISUAL] Segment {idx+1} source '{source}' failed: {e}")
                     errors.append(f"{source}: {e}")
 
         if not chosen_clip:
-            # Emergency color background — pipeline never dies on visuals
-            logger.warning(f"[VISUAL] Segment {idx+1}: all sources failed, using color background")
+            # Fallback to unique animated gradient motion card with Ken Burns motion
+            logger.warning(f"[VISUAL] Segment {idx+1}: Standard sources failed. Generating standalone gradient visual...")
+            c1 = ["0x0a1128", "0x1b1b2f", "0x162447", "0x1f4068", "0x111d5e", "0x0f3460", "0x1e152a", "0x0c2461"][idx % 8]
+            c2 = ["0x1a2a6c", "0x2c3e50", "0x20002c", "0x34495e", "0x1e3c72", "0x2a5298", "0x3c1053", "0x1b1464"][idx % 8]
             subprocess.run([
-                "ffmpeg", "-y", "-f", "lavfi", "-i", f"color=c=0x0a1128:s=1280x720:d={duration_sec:.2f}",
+                "ffmpeg", "-y", "-f", "lavfi",
+                "-i", f"gradients=s=1280x720:c0={c1}:c1={c2}:d={duration_sec:.2f}:speed=0.005",
+                "-vf", "noise=alls=12:allf=t+u,vignette=PI/4,format=yuv420p",
                 "-r", "25", "-c:v", "libx264", "-preset", "ultrafast", "-pix_fmt", "yuv420p", "-an", final_clip_path
             ], check=True, capture_output=True)
             chosen_clip = final_clip_path
 
+        # Record fingerprint for zero duplicate guarantee
+        clip_hash = get_file_hash(chosen_clip)
+        if clip_hash:
+            self.seen_clip_hashes.add(clip_hash)
+
         # Apply dynamic 35mm film grain filter overlay if scene depicts past/historical events
         if is_historical_scene(seg, topic=getattr(self.cfg, "topic", "")):
             aged_tmp = os.path.join(out_dir, f"aged_{idx:03d}.mp4")
@@ -632,6 +882,8 @@ class VisualGen:
         Returns: {"en": List[str], "es": List[str]}
         If on-screen text is present in a segment, generates distinct English and Spanish clips.
         """
+        self.stock_fetcher.reset_session()
+        self.seen_clip_hashes.clear()
         segs = script.get("script_segments", [])
         logger.info(f"[VISUAL] Producing {len(segs)} video clips (bilingual text-matching enabled)...")
         pc = self.pcs.get("nvidiam")

youtube_factory.py

@@ -377,6 +377,9 @@ def main():
     parser.add_argument("--topic", "-t", help="Single video topic to generate")
     parser.add_argument("--duration", "-d", type=int, help="Target duration in seconds")
     parser.add_argument("--batch", "-b", nargs="+", help="List of video topics to process sequentially")
+    parser.add_argument("--auto", action="store_true", help="Autonomous mode: research top-performing viral YouTube explainer topics and produce our own versions")
+    parser.add_argument("--auto-count", type=int, default=3, help="Number of viral topics to auto-research and produce (default 3)")
+    parser.add_argument("--auto-niche", default="", help="Specific niche for auto-research (e.g. ancient history, astrophysics, tech, mysteries)")
     parser.add_argument("--batch-file", "-f", help="Text file containing one topic per line")
     parser.add_argument("--trending", action="store_true", help="Fetch real-time trending topics and generate videos")
     parser.add_argument("--trending-niche", help="Trending niche (history, space, science, tech, mysteries, business)")
@@ -427,6 +430,16 @@ def main():
         logger.info(f"Retried {retried} failed jobs.")
     elif args.daemon:
         factory.run_daemon()
+    elif args.auto:
+        from pipeline.research import ViralTopicResearcher
+        researcher = ViralTopicResearcher(factory.pcs.get("amdllm"))
+        discovered = researcher.discover_popular_trending_topics(niche=args.auto_niche, count=args.auto_count)
+        logger.info(f"🚀 Auto-Research discovered {len(discovered)} high-velocity viral explainer topics!")
+        for item in discovered:
+            t = item.get("topic") if isinstance(item, dict) else str(item)
+            curiosity = item.get("curiosity_angle", "") if isinstance(item, dict) else ""
+            logger.info(f"🎬 Starting autonomous production for viral topic: '{t}' (Curiosity Angle: {curiosity})")
+            factory.run_video_pipeline(t, duration=args.duration, dry_run=args.dry_run)
     elif args.trending:
         fetcher = TrendingFetcher(factory.pcs.get("amdllm"))
         topics = fetcher.get_trending_topics(niche=args.trending_niche, count=args.trending_count)