924 lines
46 KiB
Python
924 lines
46 KiB
Python
"""
|
|
Visual Generation & Stock Media Module.
|
|
Handles:
|
|
1. NvidiaLLM ComfyUI MiniMax H3 Text-to-Video generation
|
|
2. NvidiaLLM ComfyUI z-image-turbo / SDXL AI Image generation
|
|
3. Internet Stock Video & Stock Image downloaders (Wikimedia Commons, Pexels, Pixabay)
|
|
4. Dynamic Ken Burns Motion Effects (pan/zoom) in FFmpeg for still images
|
|
5. Intelligent visual curation and multi-tier fallback pipeline
|
|
"""
|
|
|
|
import os
|
|
import re
|
|
import time
|
|
import json
|
|
import zlib
|
|
import random
|
|
import logging
|
|
import urllib.request
|
|
import urllib.parse
|
|
import subprocess
|
|
import requests
|
|
from typing import Dict, Any, List, Optional
|
|
from .remote import RemotePC
|
|
|
|
logger = logging.getLogger("YTFactory.Visual")
|
|
|
|
# ─── ComfyUI Model Constants ────────────────────────────────────────────────
|
|
H3_UNET = "minimax_h3_fl2va_pruned_int8_convrot.safetensors"
|
|
H3_CLIP = "qwen3vl_32b_minimax_h3_nvfp4_awq.safetensors"
|
|
H3_VAE = "minimax_h3_video_vae_fp16.safetensors"
|
|
H3_FPS = 24
|
|
H3_MIN_LEN, H3_MAX_LEN = 124, 362
|
|
|
|
ZIMAGE_UNET = "z_image_turbo_bf16.safetensors"
|
|
ZIMAGE_CLIP = "qwen_3_4b.safetensors"
|
|
ZIMAGE_VAE = "ae.safetensors"
|
|
|
|
SDXL_CKPT = "sd_xl_base_1.0.safetensors"
|
|
|
|
def h3_snap_length(seconds: float) -> int:
|
|
"""Snap a duration in seconds to the MiniMax H3 17k+5 frame grid."""
|
|
n = max(H3_MIN_LEN, min(H3_MAX_LEN, int(round(seconds * H3_FPS))))
|
|
k = max(7, round((n - 5) / 17))
|
|
return 17 * k + 5
|
|
|
|
# ─── Stock Media Fetcher ────────────────────────────────────────────────────
|
|
|
|
STOP_WORDS = {
|
|
"the", "a", "an", "and", "or", "but", "in", "on", "at", "to", "for", "of", "with",
|
|
"by", "about", "against", "between", "into", "through", "during", "before", "after",
|
|
"above", "below", "from", "up", "down", "out", "off", "over", "under", "again",
|
|
"further", "then", "once", "here", "there", "when", "where", "why", "how", "all",
|
|
"any", "both", "each", "few", "more", "most", "other", "some", "such", "no", "nor",
|
|
"not", "only", "own", "same", "so", "than", "too", "very", "can", "will", "just",
|
|
"should", "now", "what", "reveals", "stopped", "being", "mankind", "humanity",
|
|
"established", "used", "throughout", "history", "origins", "world", "worlds",
|
|
"has", "have", "had", "been", "was", "were", "are", "is", "its", "their", "they",
|
|
"them", "subjugate", "subjugating", "subjugation", "revealed", "revealing", "secrets",
|
|
"truth", "behind", "explained", "documentary", "mystery", "mysteries"
|
|
}
|
|
|
|
def clean_search_keywords(text: str) -> str:
|
|
"""Extract clean, concrete visual search terms from a descriptive prompt or query."""
|
|
words = re.findall(r'[A-Za-z0-9]+', text.lower())
|
|
filtered = [w for w in words if w not in STOP_WORDS and len(w) > 2]
|
|
# Keep up to 3 most specific visual keywords
|
|
return " ".join(filtered[:3]) if filtered else text.strip()
|
|
|
|
import hashlib
|
|
|
|
def get_file_hash(path: str) -> str:
|
|
"""Compute SHA-256 hash of a file to detect exact duplicate clips or images."""
|
|
if not path or not os.path.exists(path) or os.path.getsize(path) < 100:
|
|
return ""
|
|
h = hashlib.sha256()
|
|
try:
|
|
with open(path, "rb") as f:
|
|
while chunk := f.read(65536):
|
|
h.update(chunk)
|
|
return h.hexdigest()
|
|
except Exception:
|
|
return ""
|
|
|
|
class StockMediaFetcher:
|
|
def __init__(self, pexels_key: Optional[str] = None, pixabay_key: Optional[str] = None):
|
|
self.pexels_key = pexels_key
|
|
self.pixabay_key = pixabay_key
|
|
self.user_agent = "YouTubeFactory/2.0 (faceless-video-pipeline)"
|
|
self.used_media_urls = set()
|
|
self.used_file_hashes = set()
|
|
self.used_filenames = set()
|
|
self.used_youtube_ids = set()
|
|
|
|
def reset_session(self):
|
|
"""Reset used media set at the start of a new video job."""
|
|
self.used_media_urls.clear()
|
|
self.used_file_hashes.clear()
|
|
self.used_filenames.clear()
|
|
self.used_youtube_ids.clear()
|
|
|
|
def _query_youtube_broll(self, query: str, duration_sec: float, out_path: str, offset_sec: int = 15) -> Optional[str]:
|
|
"""Search YouTube for relevant topic documentary/b-roll footage and download muted HD clip."""
|
|
try:
|
|
clean_q = re.sub(r'[^A-Za-z0-9 ]', '', query).strip()
|
|
if not clean_q:
|
|
return None
|
|
search_target = f"ytsearch8:{clean_q} documentary footage"
|
|
logger.info(f"[STOCK:YOUTUBE] Searching YouTube for b-roll footage: '{clean_q}'...")
|
|
cmd_search = [
|
|
"yt-dlp", search_target,
|
|
"--dump-json", "--flat-playlist", "--no-warnings"
|
|
]
|
|
r = subprocess.run(cmd_search, capture_output=True, text=True, timeout=20)
|
|
lines = [l for l in r.stdout.strip().split("\n") if l]
|
|
for line in lines:
|
|
try:
|
|
d = json.loads(line)
|
|
vid_id = d.get("id")
|
|
dur = d.get("duration") or 60
|
|
if not vid_id or vid_id in self.used_youtube_ids or (dur and dur < 12):
|
|
continue
|
|
|
|
# Determine clean mid-video section avoiding intros/outros
|
|
start_t = min(max(15, offset_sec), max(5, int(dur) - int(duration_sec) - 5)) if dur else 15
|
|
end_t = start_t + int(duration_sec) + 3
|
|
|
|
raw_tmp = out_path + f".raw_{vid_id}.mp4"
|
|
cmd_dl = [
|
|
"yt-dlp", f"https://www.youtube.com/watch?v={vid_id}",
|
|
"--extractor-args", "youtube:player_client=android,web",
|
|
"-f", "22/18/best[height<=720]/best",
|
|
"--download-sections", f"*{start_t}-{end_t}",
|
|
"--force-keyframes-at-cuts",
|
|
"-o", raw_tmp,
|
|
"--no-playlist",
|
|
"--socket-timeout", "15"
|
|
]
|
|
r_dl = subprocess.run(cmd_dl, capture_output=True, text=True, timeout=35)
|
|
if r_dl.returncode == 0 and os.path.exists(raw_tmp) and os.path.getsize(raw_tmp) > 5000:
|
|
# Normalize to 1280x720 25fps muted video (stripping any YouTube audio)
|
|
cmd_norm = [
|
|
"ffmpeg", "-y", "-i", raw_tmp,
|
|
"-vf", "scale=1280:720:force_original_aspect_ratio=decrease,pad=1280:720:(ow-iw)/2:(oh-ih)/2:black,fps=25,format=yuv420p",
|
|
"-t", f"{duration_sec:.2f}", "-r", "25",
|
|
"-c:v", "libx264", "-preset", "fast", "-crf", "22", "-pix_fmt", "yuv420p",
|
|
"-an", out_path
|
|
]
|
|
r_norm = subprocess.run(cmd_norm, capture_output=True, text=True)
|
|
if os.path.exists(raw_tmp):
|
|
try:
|
|
os.remove(raw_tmp)
|
|
except OSError:
|
|
pass
|
|
|
|
if r_norm.returncode == 0 and os.path.exists(out_path) and os.path.getsize(out_path) > 5000:
|
|
h = get_file_hash(out_path)
|
|
if h and h in self.used_file_hashes:
|
|
logger.warning(f"[STOCK:YOUTUBE] Duplicate content detected (hash {h[:8]}). Continuing search...")
|
|
try:
|
|
os.remove(out_path)
|
|
except OSError:
|
|
pass
|
|
continue
|
|
self.used_youtube_ids.add(vid_id)
|
|
self.used_media_urls.add(f"https://www.youtube.com/watch?v={vid_id}")
|
|
if h:
|
|
self.used_file_hashes.add(h)
|
|
logger.info(f"[STOCK:YOUTUBE] Extracted high quality muted YouTube b-roll clip from '{d.get('title')}'")
|
|
return out_path
|
|
except Exception as e:
|
|
logger.debug(f"[STOCK:YOUTUBE] Candidate parse error: {e}")
|
|
continue
|
|
except Exception as e:
|
|
logger.debug(f"[STOCK:YOUTUBE] Search failed for '{query}': {e}")
|
|
return None
|
|
|
|
def query_person_image(self, person_name: str) -> Optional[str]:
|
|
"""Fetch authentic historical portrait/photograph of a person from Wikipedia / Wikimedia Commons."""
|
|
if not person_name or len(person_name.strip()) < 3:
|
|
return None
|
|
clean_name = person_name.strip()
|
|
logger.info(f"[STOCK:PERSON] Searching authentic archival portrait/photo for person: '{clean_name}'...")
|
|
|
|
# 1. Query Wikipedia PageImage API (1920px curated lead portrait)
|
|
try:
|
|
encoded = urllib.parse.quote(clean_name)
|
|
url = f"https://en.wikipedia.org/w/api.php?action=query&titles={encoded}&prop=pageimages&pithumbsize=1920&format=json"
|
|
req = urllib.request.Request(url, headers={"User-Agent": self.user_agent})
|
|
with urllib.request.urlopen(req, timeout=8) as resp:
|
|
data = json.loads(resp.read().decode("utf-8"))
|
|
pages = data.get("query", {}).get("pages", {})
|
|
for pid, page in pages.items():
|
|
if pid != "-1" and "thumbnail" in page:
|
|
src = page["thumbnail"].get("source")
|
|
if src and src not in self.used_media_urls:
|
|
return src
|
|
except Exception as e:
|
|
logger.debug(f"[STOCK:PERSON] Wikipedia query note for '{clean_name}': {e}")
|
|
|
|
# 2. Query Wikimedia Commons for high-res archival portrait
|
|
try:
|
|
wiki_img = self._query_wikimedia(f"portrait {clean_name}", is_video=False)
|
|
if wiki_img:
|
|
return wiki_img
|
|
wiki_img2 = self._query_wikimedia(clean_name, is_video=False)
|
|
if wiki_img2:
|
|
return wiki_img2
|
|
except Exception as e:
|
|
logger.debug(f"[STOCK:PERSON] Wikimedia query note for '{clean_name}': {e}")
|
|
|
|
return None
|
|
|
|
def _query_wikimedia(self, query: str, is_video: bool = False) -> Optional[str]:
|
|
"""Search Wikimedia Commons for high-resolution images or videos without API keys."""
|
|
try:
|
|
clean_q = re.sub(r'[^A-Za-z0-9 ]', '', query).strip()
|
|
encoded_q = urllib.parse.quote(clean_q)
|
|
api_url = (
|
|
f"https://commons.wikimedia.org/w/api.php?action=query&generator=search"
|
|
f"&gsrsearch={encoded_q}&gsrnamespace=6&gsrlimit=10&prop=imageinfo"
|
|
f"&iiprop=url|mime|size&format=json"
|
|
)
|
|
req = urllib.request.Request(api_url, headers={"User-Agent": self.user_agent})
|
|
with urllib.request.urlopen(req, timeout=10) as resp:
|
|
data = json.loads(resp.read().decode("utf-8"))
|
|
pages = data.get("query", {}).get("pages", {})
|
|
for _, page in pages.items():
|
|
infos = page.get("imageinfo", [])
|
|
if not infos:
|
|
continue
|
|
url = infos[0].get("url", "")
|
|
if not url or url in self.used_media_urls:
|
|
continue
|
|
clean_name = os.path.basename(urllib.parse.urlparse(url).path)
|
|
if clean_name and clean_name in self.used_filenames:
|
|
continue
|
|
mime = infos[0].get("mime", "").lower()
|
|
if is_video and ("video" in mime or url.endswith((".webm", ".mp4", ".ogv"))):
|
|
return url
|
|
if not is_video and ("image" in mime or url.endswith((".jpg", ".jpeg", ".png", ".webp"))):
|
|
return url
|
|
except Exception as e:
|
|
logger.debug(f"Wikimedia search failed for '{query}': {e}")
|
|
return None
|
|
|
|
def _query_pexels_video(self, query: str) -> Optional[str]:
|
|
"""Search Pexels for free HD stock videos if API key is provided."""
|
|
if not self.pexels_key:
|
|
return None
|
|
try:
|
|
clean_q = urllib.parse.quote(query)
|
|
url = f"https://api.pexels.com/videos/search?query={clean_q}&per_page=10&orientation=landscape"
|
|
headers = {"Authorization": self.pexels_key, "User-Agent": self.user_agent}
|
|
req = urllib.request.Request(url, headers=headers)
|
|
with urllib.request.urlopen(req, timeout=10) as resp:
|
|
data = json.loads(resp.read().decode("utf-8"))
|
|
videos = data.get("videos", [])
|
|
for v in videos:
|
|
files = v.get("video_files", [])
|
|
hd = [vf for vf in files if vf.get("width", 0) >= 1280 and vf.get("link")]
|
|
target = sorted(hd, key=lambda x: x["width"])[0]["link"] if hd else (files[0].get("link") if files else None)
|
|
if target and target not in self.used_media_urls:
|
|
return target
|
|
except Exception as e:
|
|
logger.debug(f"Pexels video query failed: {e}")
|
|
return None
|
|
|
|
def _query_pexels_photo(self, query: str) -> Optional[str]:
|
|
"""Search Pexels for free HD stock photos if API key is provided."""
|
|
if not self.pexels_key:
|
|
return None
|
|
try:
|
|
clean_q = urllib.parse.quote(query)
|
|
url = f"https://api.pexels.com/v1/search?query={clean_q}&per_page=10&orientation=landscape"
|
|
headers = {"Authorization": self.pexels_key, "User-Agent": self.user_agent}
|
|
req = urllib.request.Request(url, headers=headers)
|
|
with urllib.request.urlopen(req, timeout=10) as resp:
|
|
data = json.loads(resp.read().decode("utf-8"))
|
|
photos = data.get("photos", [])
|
|
for p in photos:
|
|
src = p.get("src", {}).get("large2x") or p.get("src", {}).get("large")
|
|
if src and src not in self.used_media_urls:
|
|
return src
|
|
except Exception as e:
|
|
logger.debug(f"Pexels photo query failed: {e}")
|
|
return None
|
|
|
|
def _query_pixabay_video(self, query: str) -> Optional[str]:
|
|
"""Search Pixabay for free HD stock videos if API key is provided."""
|
|
if not self.pixabay_key:
|
|
return None
|
|
try:
|
|
clean_q = urllib.parse.quote(query)
|
|
url = f"https://pixabay.com/api/videos/?key={self.pixabay_key}&q={clean_q}&per_page=10&safesearch=true"
|
|
req = urllib.request.Request(url, headers={"User-Agent": self.user_agent})
|
|
with urllib.request.urlopen(req, timeout=10) as resp:
|
|
data = json.loads(resp.read().decode("utf-8"))
|
|
for hit in data.get("hits", []):
|
|
vids = hit.get("videos", {})
|
|
for quality in ("large", "medium", "small"):
|
|
v = vids.get(quality)
|
|
if v and v.get("url") and v.get("width", 0) >= 640:
|
|
target_url = v["url"]
|
|
if target_url not in self.used_media_urls:
|
|
return target_url
|
|
except Exception as e:
|
|
logger.debug(f"Pixabay video query failed: {e}")
|
|
return None
|
|
|
|
def _query_pixabay_photo(self, query: str) -> Optional[str]:
|
|
"""Search Pixabay for free HD stock photos if API key is provided."""
|
|
if not self.pixabay_key:
|
|
return None
|
|
try:
|
|
clean_q = urllib.parse.quote(query)
|
|
url = (f"https://pixabay.com/api/?key={self.pixabay_key}&q={clean_q}"
|
|
f"&image_type=photo&orientation=horizontal&per_page=10&safesearch=true")
|
|
req = urllib.request.Request(url, headers={"User-Agent": self.user_agent})
|
|
with urllib.request.urlopen(req, timeout=10) as resp:
|
|
data = json.loads(resp.read().decode("utf-8"))
|
|
hits = data.get("hits", [])
|
|
for h in hits:
|
|
img_url = h.get("largeImageURL") or h.get("webformatURL")
|
|
if img_url and img_url not in self.used_media_urls:
|
|
return img_url
|
|
except Exception as e:
|
|
logger.debug(f"Pixabay photo query failed: {e}")
|
|
return None
|
|
|
|
def download_media(self, url: str, out_path: str) -> bool:
|
|
"""Download remote media with retry, validation, and content hash duplicate prevention."""
|
|
try:
|
|
req = urllib.request.Request(url, headers={"User-Agent": self.user_agent})
|
|
with urllib.request.urlopen(req, timeout=30) as resp, open(out_path, "wb") as f:
|
|
f.write(resp.read())
|
|
if os.path.exists(out_path) and os.path.getsize(out_path) > 5000:
|
|
h = get_file_hash(out_path)
|
|
if h and h in self.used_file_hashes:
|
|
logger.warning(f"[STOCK] Duplicate asset content detected (hash {h[:8]}). Discarding duplicate...")
|
|
try:
|
|
os.remove(out_path)
|
|
except OSError:
|
|
pass
|
|
return False
|
|
self.used_media_urls.add(url)
|
|
if h:
|
|
self.used_file_hashes.add(h)
|
|
clean_name = os.path.basename(urllib.parse.urlparse(url).path)
|
|
if clean_name:
|
|
self.used_filenames.add(clean_name)
|
|
return True
|
|
except Exception as e:
|
|
logger.debug(f"Download failed from {url}: {e}")
|
|
if os.path.exists(out_path):
|
|
try:
|
|
os.remove(out_path)
|
|
except OSError:
|
|
pass
|
|
return False
|
|
|
|
def search_and_download(self, queries: Any, out_path: str, prefer_video: bool = False, duration_sec: float = 6.0) -> Optional[str]:
|
|
"""Search across stock engines with candidate query cascade and download asset."""
|
|
if isinstance(queries, str):
|
|
candidate_list = [queries]
|
|
else:
|
|
candidate_list = list(queries)
|
|
|
|
expanded_queries = []
|
|
for raw_q in candidate_list:
|
|
if not raw_q:
|
|
continue
|
|
cleaned = clean_search_keywords(raw_q)
|
|
if cleaned and cleaned not in expanded_queries:
|
|
expanded_queries.append(cleaned)
|
|
words = cleaned.split()
|
|
if len(words) >= 2:
|
|
two_word = " ".join(words[:2])
|
|
if two_word not in expanded_queries:
|
|
expanded_queries.append(two_word)
|
|
for w in words:
|
|
if len(w) > 3 and w not in expanded_queries:
|
|
expanded_queries.append(w)
|
|
|
|
for cleaned_q in expanded_queries:
|
|
logger.info(f"[STOCK] Searching stock media for: '{cleaned_q}' (video={prefer_video})...")
|
|
if prefer_video:
|
|
# 1. First priority: Real YouTube documentary B-roll footage without audio
|
|
yt_clip = self._query_youtube_broll(cleaned_q, duration_sec, out_path)
|
|
if yt_clip and os.path.exists(yt_clip) and os.path.getsize(yt_clip) > 5000:
|
|
return yt_clip
|
|
|
|
# 2. Stock APIs (Pexels, Pixabay, Wikimedia)
|
|
for label, fn in (("Pexels", self._query_pexels_video),
|
|
("Pixabay", self._query_pixabay_video),
|
|
("Wikimedia", lambda q: self._query_wikimedia(q, is_video=True))):
|
|
v_url = fn(cleaned_q)
|
|
if v_url and self.download_media(v_url, out_path):
|
|
logger.info(f"[STOCK] Downloaded {label} video for '{cleaned_q}'")
|
|
return out_path
|
|
else:
|
|
for label, fn in (("Pexels", self._query_pexels_photo),
|
|
("Pixabay", self._query_pixabay_photo),
|
|
("Wikimedia", lambda q: self._query_wikimedia(q, is_video=False))):
|
|
p_url = fn(cleaned_q)
|
|
if p_url and self.download_media(p_url, out_path):
|
|
logger.info(f"[STOCK] Downloaded {label} photo for '{cleaned_q}'")
|
|
return out_path
|
|
|
|
logger.debug(f"[STOCK] No matching stock media found for queries: {expanded_queries}")
|
|
return None
|
|
|
|
# ─── Ken Burns Motion Engine ────────────────────────────────────────────────
|
|
|
|
def apply_ken_burns_effect(img_path: str, out_mp4: str, duration_sec: float, motion_type: int = 0) -> str:
|
|
"""
|
|
Animate a still image into a cinematic 16:9 25fps video clip using FFmpeg pan & zoom motion.
|
|
Provides 6 distinct cinematic camera movements:
|
|
0: Smooth Center Zoom-In
|
|
1: Smooth Center Zoom-Out
|
|
2: Diagonal Pan Left-Up with subtle zoom
|
|
3: Diagonal Pan Right-Down with subtle zoom
|
|
4: Slow Pan Right across center
|
|
5: Slow Pan Left across center
|
|
"""
|
|
total_frames = max(50, int(round(duration_sec * 25)))
|
|
m_idx = motion_type % 6
|
|
|
|
if m_idx == 0:
|
|
# Smooth Zoom-In (center)
|
|
z_expr = "min(zoom+0.0012,1.25)"
|
|
x_expr = "iw/2-(iw/zoom/2)"
|
|
y_expr = "ih/2-(ih/zoom/2)"
|
|
elif m_idx == 1:
|
|
# Smooth Zoom-Out (center)
|
|
z_expr = "if(eq(on,1),1.25,max(1.0,zoom-0.0012))"
|
|
x_expr = "iw/2-(iw/zoom/2)"
|
|
y_expr = "ih/2-(ih/zoom/2)"
|
|
elif m_idx == 2:
|
|
# Diagonal Pan Left-Up with subtle zoom
|
|
z_expr = "1.20"
|
|
x_expr = f"if(eq(on,1),iw-iw/zoom,max(0,x-(iw-iw/zoom)/{total_frames}))"
|
|
y_expr = f"if(eq(on,1),ih-ih/zoom,max(0,y-(ih-ih/zoom)/{total_frames}))"
|
|
elif m_idx == 3:
|
|
# Diagonal Pan Right-Down with subtle zoom
|
|
z_expr = "1.20"
|
|
x_expr = f"if(eq(on,1),0,min(iw-iw/zoom,x+(iw-iw/zoom)/{total_frames}))"
|
|
y_expr = f"if(eq(on,1),0,min(ih-ih/zoom,y+(ih-ih/zoom)/{total_frames}))"
|
|
elif m_idx == 4:
|
|
# Slow Pan Right across center
|
|
z_expr = "1.18"
|
|
x_expr = f"if(eq(on,1),0,min(iw-iw/zoom,x+(iw-iw/zoom)/{total_frames}))"
|
|
y_expr = "ih/2-(ih/zoom/2)"
|
|
else:
|
|
# Slow Pan Left across center
|
|
z_expr = "1.18"
|
|
x_expr = f"if(eq(on,1),iw-iw/zoom,max(0,x-(iw-iw/zoom)/{total_frames}))"
|
|
y_expr = "ih/2-(ih/zoom/2)"
|
|
|
|
vf = (
|
|
f"scale=1920:1080:force_original_aspect_ratio=increase,crop=1920:1080,"
|
|
f"zoompan=z='{z_expr}':x='{x_expr}':y='{y_expr}':d={total_frames}:s=1280x720:fps=25,"
|
|
f"format=yuv420p"
|
|
)
|
|
|
|
cmd = [
|
|
"ffmpeg", "-y", "-loop", "1", "-i", img_path,
|
|
"-vf", vf,
|
|
"-t", f"{duration_sec:.2f}",
|
|
"-r", "25",
|
|
"-c:v", "libx264", "-preset", "fast", "-crf", "22",
|
|
"-pix_fmt", "yuv420p", "-an", out_mp4
|
|
]
|
|
r = subprocess.run(cmd, capture_output=True, text=True)
|
|
if r.returncode != 0:
|
|
logger.warning(f"Ken Burns animation fallback: {r.stderr[-200:]}")
|
|
# Simple loop fallback
|
|
fallback_vf = "scale=1280:720:force_original_aspect_ratio=decrease,pad=1280:720:(ow-iw)/2:(oh-ih)/2:black,format=yuv420p"
|
|
subprocess.run([
|
|
"ffmpeg", "-y", "-loop", "1", "-i", img_path,
|
|
"-vf", fallback_vf, "-t", f"{duration_sec:.2f}", "-r", "25",
|
|
"-c:v", "libx264", "-preset", "fast", "-crf", "22", "-pix_fmt", "yuv420p", "-an", out_mp4
|
|
], check=True, capture_output=True)
|
|
|
|
return out_mp4
|
|
|
|
def normalize_clip_to_duration(src_path: str, out_path: str, duration_sec: float) -> str:
|
|
"""
|
|
Normalize any video clip to exactly duration_sec at 1280x720 25fps.
|
|
Loops short clips (-stream_loop) and trims long ones (-t) so every segment
|
|
stays in sync with its narration.
|
|
"""
|
|
vf = ("scale=1280:720:force_original_aspect_ratio=decrease,"
|
|
"pad=1280:720:(ow-iw)/2:(oh-ih)/2:black,fps=25,format=yuv420p")
|
|
r = subprocess.run([
|
|
"ffmpeg", "-y", "-stream_loop", "-1", "-i", src_path,
|
|
"-vf", vf, "-t", f"{duration_sec:.2f}", "-r", "25",
|
|
"-c:v", "libx264", "-preset", "fast", "-crf", "22",
|
|
"-pix_fmt", "yuv420p", "-an", out_path
|
|
], capture_output=True, text=True)
|
|
if r.returncode != 0:
|
|
raise RuntimeError(f"clip normalization failed: {r.stderr[-200:]}")
|
|
return out_path
|
|
|
|
def apply_text_overlay(video_path: str, out_mp4: str, text: str, duration_sec: float) -> str:
|
|
"""
|
|
Render a broadcast-style lower-third text banner onto the video clip using UTF-8 textfile.
|
|
Supports full international characters (accents, umlauts, Spanish punctuation) without corruption.
|
|
"""
|
|
clean_text = text.strip()
|
|
if not clean_text:
|
|
if video_path != out_mp4:
|
|
import shutil
|
|
shutil.copy(video_path, out_mp4)
|
|
return out_mp4
|
|
|
|
font_paths = [
|
|
"/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf",
|
|
"/usr/share/fonts/TTF/DejaVuSans-Bold.ttf",
|
|
"/usr/share/fonts/truetype/liberation/LiberationSans-Bold.ttf",
|
|
"/usr/share/fonts/truetype/freefont/FreeSansBold.ttf",
|
|
]
|
|
font_file = None
|
|
for fp in font_paths:
|
|
if os.path.exists(fp):
|
|
font_file = fp
|
|
break
|
|
|
|
# Write clean UTF-8 text to a temporary textfile for FFmpeg drawtext to avoid character stripping
|
|
import tempfile
|
|
txt_fd, txt_path = tempfile.mkstemp(suffix=".txt", prefix="banner_text_")
|
|
with open(txt_fd, "w", encoding="utf-8") as f:
|
|
f.write(clean_text)
|
|
|
|
font_param = f":fontfile='{font_file}'" if font_file else ""
|
|
end_t = max(1.0, duration_sec - 0.8)
|
|
enable_expr = f"between(t,0.8,{end_t:.2f})"
|
|
vf = (
|
|
f"drawtext=textfile='{txt_path}'{font_param}:fontcolor=white:fontsize=36:"
|
|
f"box=1:boxcolor=black@0.65:boxborderw=14:x=60:y=h-th-60:enable='{enable_expr}'"
|
|
)
|
|
|
|
cmd = [
|
|
"ffmpeg", "-y", "-i", video_path,
|
|
"-vf", vf,
|
|
"-c:v", "libx264", "-preset", "fast", "-crf", "22", "-pix_fmt", "yuv420p",
|
|
"-an", out_mp4
|
|
]
|
|
r = subprocess.run(cmd, capture_output=True, text=True)
|
|
if os.path.exists(txt_path):
|
|
try:
|
|
os.remove(txt_path)
|
|
except OSError:
|
|
pass
|
|
|
|
if r.returncode != 0:
|
|
logger.warning(f"Text overlay rendering fallback: {r.stderr[-200:]}")
|
|
if video_path != out_mp4:
|
|
import shutil
|
|
shutil.copy(video_path, out_mp4)
|
|
return out_mp4
|
|
|
|
# ─── Film Grain & Vintage Aging Filter ──────────────────────────────────────
|
|
|
|
HISTORICAL_KEYWORDS = {
|
|
"ancient", "archaeology", "archaeological", "history", "historical", "century",
|
|
"b.c.", "bc", "a.d.", "ad", "victorian", "medieval", "ruins", "pharaoh",
|
|
"temple", "pyramids", "past", "hunter gatherer", "hunter-gatherer", "gobeklitepe",
|
|
"göbeklitepe", "stone age", "bronze age", "iron age", "1888", "1900", "1920",
|
|
"1930", "1940", "1950", "1960", "1970", "empire", "dynasty", "columbus",
|
|
"alexandria", "antiquity", "paleolithic", "neolithic", "civilization", "excavation",
|
|
"fossil", "scrolls", "papyrus", "jack the ripper", "monument", "artifacts", "artifact",
|
|
"bermuda", "atlantis", "mythology", "prehistoric", "dynastic", "tomb", "sarcophagus",
|
|
"excavations", "relic", "relics", "megalith", "megalithic", "sanctuary"
|
|
}
|
|
|
|
def is_historical_scene(seg: Dict[str, Any], topic: str = "") -> bool:
|
|
"""Check if the segment depicts historical/past events based on flag or keywords."""
|
|
if seg.get("is_historical") is True:
|
|
return True
|
|
combined_text = (
|
|
f"{topic} {seg.get('chapter_title', '')} {seg.get('text', '')} "
|
|
f"{seg.get('visual_prompt', '')} {seg.get('stock_query', '')}"
|
|
).lower()
|
|
return any(kw in combined_text for kw in HISTORICAL_KEYWORDS)
|
|
|
|
def apply_film_grain_overlay(video_path: str, out_mp4: str) -> str:
|
|
"""
|
|
Apply a dynamic 35mm film grain overlay with subtle vintage contrast & vignette
|
|
to give historical footage an authentically aged cinematic look.
|
|
"""
|
|
vintage_vf = (
|
|
"noise=alls=14:allf=t+u,"
|
|
"eq=contrast=1.06:brightness=-0.01:saturation=0.88,"
|
|
"vignette=PI/5"
|
|
)
|
|
cmd = [
|
|
"ffmpeg", "-y", "-i", video_path,
|
|
"-vf", vintage_vf,
|
|
"-c:v", "libx264", "-preset", "fast", "-crf", "22", "-pix_fmt", "yuv420p",
|
|
"-an", out_mp4
|
|
]
|
|
r = subprocess.run(cmd, capture_output=True, text=True)
|
|
if r.returncode != 0:
|
|
logger.warning(f"Film grain filter fallback: {r.stderr[-200:]}")
|
|
if video_path != out_mp4:
|
|
import shutil
|
|
shutil.copy(video_path, out_mp4)
|
|
return out_mp4
|
|
|
|
# ─── ComfyUI Orchestration ─────────────────────────────────────────────────
|
|
|
|
class VisualGen:
|
|
def __init__(self, pcs: Dict[str, RemotePC], cfg: Any):
|
|
self.pcs = pcs
|
|
self.cfg = cfg
|
|
self.seen_clip_hashes = set()
|
|
self.stock_fetcher = StockMediaFetcher(
|
|
pexels_key=getattr(cfg, "pexels_api_key", None),
|
|
pixabay_key=getattr(cfg, "pixabay_api_key", None)
|
|
)
|
|
|
|
def _h3_workflow(self, prompt: str, seed: int, width: int, height: int, length: int) -> Dict[str, Any]:
|
|
return {
|
|
"1": {"inputs": {"unet_name": H3_UNET, "weight_dtype": "default"}, "class_type": "UNETLoader"},
|
|
"2": {"inputs": {"model": ["1", 0], "shift_video": 12.0, "shift_audio": 3.0}, "class_type": "MiniMaxH3SigmaShift"},
|
|
"3": {"inputs": {"clip_name": H3_CLIP, "type": "minimax"}, "class_type": "CLIPLoader"},
|
|
"4": {"inputs": {"vae_name": H3_VAE}, "class_type": "VAELoader"},
|
|
"5": {"inputs": {"clip": ["3", 0], "vae": ["4", 0], "prompt": prompt,
|
|
"width": width, "height": height, "length": length},
|
|
"class_type": "MiniMaxH3ImageToVideo"},
|
|
"6": {"inputs": {"text": "", "clip": ["3", 0]}, "class_type": "CLIPTextEncode"},
|
|
"7": {"inputs": {"seed": seed, "steps": 20, "cfg": 5.0, "sampler_name": "euler",
|
|
"scheduler": "normal", "denoise": 1.0,
|
|
"model": ["2", 0], "positive": ["5", 0], "negative": ["6", 0],
|
|
"latent_image": ["5", 1]}, "class_type": "KSampler"},
|
|
"8": {"inputs": {"samples": ["7", 0], "vae": ["4", 0]}, "class_type": "VAEDecode"},
|
|
"9": {"inputs": {"images": ["8", 0], "fps": float(H3_FPS)}, "class_type": "CreateVideo"},
|
|
"10": {"inputs": {"video": ["9", 0], "filename_prefix": "yt_clip",
|
|
"format": "auto", "codec": "auto"}, "class_type": "SaveVideo"},
|
|
}
|
|
|
|
def _sdxl_workflow(self, prompt: str, seed: int) -> Dict[str, Any]:
|
|
return {
|
|
"1": {"inputs": {"ckpt_name": SDXL_CKPT}, "class_type": "CheckpointLoaderSimple"},
|
|
"2": {"inputs": {"text": prompt + ", 8k resolution, cinematic lighting, photorealistic, documentary b-roll", "clip": ["1", 1]}, "class_type": "CLIPTextEncode"},
|
|
"2b": {"inputs": {"text": "blurry, low quality, watermark, text, logo, ugly, deformed, oversaturated", "clip": ["1", 1]}, "class_type": "CLIPTextEncode"},
|
|
"3": {"inputs": {"width": 1280, "height": 720, "batch_size": 1}, "class_type": "EmptyLatentImage"},
|
|
"4": {"inputs": {"seed": seed, "steps": 22, "cfg": 7.0, "sampler_name": "euler",
|
|
"scheduler": "normal", "denoise": 1.0,
|
|
"model": ["1", 0], "positive": ["2", 0], "negative": ["2b", 0],
|
|
"latent_image": ["3", 0]}, "class_type": "KSampler"},
|
|
"5": {"inputs": {"samples": ["4", 0], "vae": ["1", 2]}, "class_type": "VAEDecode"},
|
|
"6": {"inputs": {"filename_prefix": "yt_img", "images": ["5", 0]}, "class_type": "SaveImage"},
|
|
}
|
|
|
|
def _submit(self, host: str, workflow: Dict[str, Any]) -> str:
|
|
r = requests.post(f"http://{host}:8188/prompt", json={"prompt": workflow}, timeout=30)
|
|
data = r.json()
|
|
if r.status_code != 200 or "prompt_id" not in data:
|
|
raise RuntimeError(f"ComfyUI rejected workflow: {json.dumps(data)[:300]}")
|
|
return data["prompt_id"]
|
|
|
|
def _cancel_job(self, host: str, pid: str):
|
|
"""Interrupt current execution and remove the job from the ComfyUI queue.
|
|
Prevents client-side timeouts from leaving zombie jobs that block later prompts."""
|
|
try:
|
|
requests.post(f"http://{host}:8188/interrupt", timeout=5)
|
|
except Exception:
|
|
pass
|
|
try:
|
|
requests.post(f"http://{host}:8188/queue", json={"delete": [pid]}, timeout=5)
|
|
except Exception:
|
|
pass
|
|
|
|
def _wait_result(self, host: str, pid: str, timeout_s: int = 1800) -> Dict[str, Any]:
|
|
deadline = time.time() + timeout_s
|
|
while time.time() < deadline:
|
|
time.sleep(3)
|
|
try:
|
|
h = requests.get(f"http://{host}:8188/history/{pid}", timeout=10).json()
|
|
except Exception:
|
|
continue
|
|
if pid not in h:
|
|
continue
|
|
status = h[pid].get("status", {})
|
|
if status.get("status_str") == "error":
|
|
msgs = h[pid].get("status", {}).get("messages", [])
|
|
self._cancel_job(host, pid)
|
|
raise RuntimeError(f"ComfyUI execution error: {json.dumps(msgs)[:300]}")
|
|
if status.get("completed") or h[pid].get("outputs"):
|
|
return h[pid].get("outputs", {})
|
|
self._cancel_job(host, pid)
|
|
raise RuntimeError("ComfyUI generation timed out.")
|
|
|
|
def _download_outputs(self, host: str, outputs: Dict[str, Any], out_path: str) -> str:
|
|
for node_out in outputs.values():
|
|
for key in ("videos", "gifs", "images"):
|
|
for item in node_out.get(key, []):
|
|
params = {"filename": item["filename"], "type": item.get("type", "output")}
|
|
if item.get("subfolder"):
|
|
params["subfolder"] = item["subfolder"]
|
|
d = requests.get(f"http://{host}:8188/view", params=params, timeout=120).content
|
|
if len(d) >= 1024:
|
|
with open(out_path, "wb") as f:
|
|
f.write(d)
|
|
return out_path
|
|
raise RuntimeError("No downloadable asset returned by ComfyUI")
|
|
|
|
# Maximum frames for a MiniMax H3 clip (~6.5s at 24fps). Longer clips take
|
|
# 30+ min on the RTX 4060; clips are looped/trimmed to segment length anyway.
|
|
H3_CLIP_CAP = 158
|
|
|
|
def _try_ai_video(self, pc, prompt, seed, duration_sec, raw_path, final_clip_path, idx):
|
|
"""MiniMax H3 text-to-video clip, normalized to the segment duration."""
|
|
if not pc or not pc.check(8188):
|
|
raise RuntimeError("NvidiaLLM ComfyUI is not reachable on port 8188")
|
|
h3_length = min(h3_snap_length(duration_sec), self.H3_CLIP_CAP)
|
|
logger.info(f"[VISUAL] Segment {idx+1}: MiniMax H3 AI video ({h3_length} frames)...")
|
|
wf = self._h3_workflow(prompt, seed, self.cfg.clip_width, self.cfg.clip_height, h3_length)
|
|
pid = self._submit(pc.host, wf)
|
|
outputs = self._wait_result(pc.host, pid, timeout_s=1200)
|
|
self._download_outputs(pc.host, outputs, raw_path)
|
|
return normalize_clip_to_duration(raw_path, final_clip_path, duration_sec)
|
|
|
|
def _try_ai_image(self, pc, prompt, seed, duration_sec, raw_path, final_clip_path, idx):
|
|
"""SDXL AI image animated with Ken Burns motion."""
|
|
if not pc or not pc.check(8188):
|
|
raise RuntimeError("NvidiaLLM ComfyUI is not reachable on port 8188")
|
|
logger.info(f"[VISUAL] Segment {idx+1}: SDXL AI image + Ken Burns...")
|
|
pid = self._submit(pc.host, self._sdxl_workflow(prompt, seed))
|
|
outputs = self._wait_result(pc.host, pid, timeout_s=300)
|
|
self._download_outputs(pc.host, outputs, raw_path)
|
|
return apply_ken_burns_effect(raw_path, final_clip_path, duration_sec, motion_type=idx)
|
|
|
|
def _try_stock(self, query, duration_sec, raw_path, final_clip_path, idx, prefer_video):
|
|
"""Stock video (looped/trimmed) or stock image (Ken Burns)."""
|
|
kind = "video" if prefer_video else "image"
|
|
logger.info(f"[VISUAL] Segment {idx+1}: stock {kind} for '{query}'...")
|
|
got = self.stock_fetcher.search_and_download(query, raw_path, prefer_video=prefer_video, duration_sec=duration_sec)
|
|
if not got or not os.path.exists(got):
|
|
raise RuntimeError(f"no stock {kind} found for '{query}'")
|
|
if got.lower().endswith((".mp4", ".webm", ".mov")):
|
|
return normalize_clip_to_duration(got, final_clip_path, duration_sec)
|
|
return apply_ken_burns_effect(got, final_clip_path, duration_sec, motion_type=idx)
|
|
|
|
def _try_person(self, person_name: str, duration_sec: float, raw_path: str, final_clip_path: str, idx: int):
|
|
"""Fetch authentic photo/portrait of historical person and animate with Ken Burns motion."""
|
|
if not person_name:
|
|
raise RuntimeError("no person name provided")
|
|
img_url = self.stock_fetcher.query_person_image(person_name)
|
|
if not img_url:
|
|
raise RuntimeError(f"no authentic archival photo found for person '{person_name}'")
|
|
if not self.stock_fetcher.download_media(img_url, raw_path):
|
|
raise RuntimeError(f"failed to download person photo from {img_url}")
|
|
logger.info(f"[VISUAL] Segment {idx+1}: Using authentic historical photo/portrait for '{person_name}'")
|
|
return apply_ken_burns_effect(raw_path, final_clip_path, duration_sec, motion_type=idx)
|
|
|
|
def _generate_segment_visual(self, pc: RemotePC, seg: Dict[str, Any], idx: int, out_dir: str, duration_sec: float) -> str:
|
|
"""
|
|
Produce one visual for a segment as an MP4 of exactly duration_sec.
|
|
|
|
"hybrid" rotates across 4 styles for maximum variety:
|
|
YouTube b-roll / stock video -> AI video -> stock image (Ken Burns) -> AI image (Ken Burns)
|
|
Each source falls back to the next one in the rotation on failure or duplicate collision.
|
|
"""
|
|
os.makedirs(out_dir, exist_ok=True)
|
|
CAMERA_ANGLES = [
|
|
"wide establishing cinematic drone shot",
|
|
"dramatic macro close-up detail shot",
|
|
"dynamic low-angle heroic perspective",
|
|
"atmospheric high-angle bird's eye view",
|
|
"side profile tracking cinematic perspective",
|
|
"epic panoramic sweeping shot",
|
|
"dramatic cinematic lighting with volumetric shadows",
|
|
"slow dolly-in center focus shot"
|
|
]
|
|
angle = CAMERA_ANGLES[idx % len(CAMERA_ANGLES)]
|
|
visual_anchor = (
|
|
seg.get("visual_style_anchor") or
|
|
getattr(self.cfg, "visual_style_anchor", "") or
|
|
"cinematic documentary style, consistent warm color grading, 35mm film texture, photorealistic 8k"
|
|
)
|
|
raw_prompt = seg.get("visual_prompt") or seg.get("text", "")
|
|
prompt = f"{raw_prompt}, {angle}, {visual_anchor}"
|
|
primary_stock = seg.get("stock_query") or ""
|
|
chapter_stock = seg.get("chapter_title") or ""
|
|
topic_anchor = clean_search_keywords(getattr(self.cfg, "topic", "") or "")
|
|
person_name = (seg.get("person_name") or "").strip()
|
|
|
|
# Build prioritized topical query cascade
|
|
query_candidates = []
|
|
if primary_stock:
|
|
query_candidates.append(primary_stock)
|
|
if chapter_stock and topic_anchor:
|
|
query_candidates.append(f"{topic_anchor} {chapter_stock}")
|
|
elif chapter_stock:
|
|
query_candidates.append(chapter_stock)
|
|
if topic_anchor:
|
|
query_candidates.append(topic_anchor)
|
|
|
|
import time
|
|
seed = zlib.crc32(f"{prompt}_{idx}_{time.time()}_{os.urandom(4)}".encode()) % (2**31)
|
|
|
|
final_clip_path = os.path.join(out_dir, f"clip_{idx:03d}.mp4")
|
|
raw_video = os.path.join(out_dir, f"raw_vid_{idx:03d}.mp4")
|
|
raw_img = os.path.join(out_dir, f"raw_img_{idx:03d}.jpg")
|
|
|
|
chosen_clip = None
|
|
|
|
# 1. If talking about a specific person, prioritize authentic archival portrait/photo
|
|
if person_name:
|
|
try:
|
|
candidate = self._try_person(person_name, duration_sec, raw_img, final_clip_path, idx)
|
|
cand_hash = get_file_hash(candidate)
|
|
if cand_hash and cand_hash not in self.seen_clip_hashes:
|
|
chosen_clip = candidate
|
|
except Exception as e:
|
|
logger.debug(f"[VISUAL] Person portrait search note for '{person_name}': {e}")
|
|
|
|
if not chosen_clip:
|
|
strategy = getattr(self.cfg, "visual_strategy", "hybrid")
|
|
attempts = {
|
|
"stock_video": lambda: self._try_stock(query_candidates, duration_sec, raw_video, final_clip_path, idx, prefer_video=True),
|
|
"ai_video": lambda: self._try_ai_video(pc, prompt, seed, duration_sec, raw_video, final_clip_path, idx),
|
|
"stock_image": lambda: self._try_stock(query_candidates, duration_sec, raw_img, final_clip_path, idx, prefer_video=False),
|
|
"ai_image": lambda: self._try_ai_image(pc, prompt, seed, duration_sec, raw_img, final_clip_path, idx),
|
|
}
|
|
if strategy == "all_ai":
|
|
order = ["ai_video", "ai_image", "stock_video", "stock_image"]
|
|
elif strategy == "stock_focused":
|
|
order = ["stock_video", "stock_image", "ai_video", "ai_image"]
|
|
else: # hybrid — rotate the starting source so all 4 styles appear
|
|
base = ["stock_video", "ai_video", "stock_image", "ai_image"]
|
|
order = base[idx % 4:] + base[:idx % 4]
|
|
|
|
errors = []
|
|
for source in order:
|
|
try:
|
|
candidate = attempts[source]()
|
|
cand_hash = get_file_hash(candidate)
|
|
if cand_hash and cand_hash in self.seen_clip_hashes:
|
|
logger.warning(f"[VISUAL] Segment {idx+1}: Collision on source '{source}' (hash {cand_hash[:8]}). Trying next source...")
|
|
continue
|
|
chosen_clip = candidate
|
|
break
|
|
except Exception as e:
|
|
logger.warning(f"[VISUAL] Segment {idx+1} source '{source}' failed: {e}")
|
|
errors.append(f"{source}: {e}")
|
|
|
|
if not chosen_clip:
|
|
# Fallback to unique animated gradient motion card with Ken Burns motion
|
|
logger.warning(f"[VISUAL] Segment {idx+1}: Standard sources failed. Generating standalone gradient visual...")
|
|
c1 = ["0x0a1128", "0x1b1b2f", "0x162447", "0x1f4068", "0x111d5e", "0x0f3460", "0x1e152a", "0x0c2461"][idx % 8]
|
|
c2 = ["0x1a2a6c", "0x2c3e50", "0x20002c", "0x34495e", "0x1e3c72", "0x2a5298", "0x3c1053", "0x1b1464"][idx % 8]
|
|
subprocess.run([
|
|
"ffmpeg", "-y", "-f", "lavfi",
|
|
"-i", f"gradients=s=1280x720:c0={c1}:c1={c2}:d={duration_sec:.2f}:speed=0.005",
|
|
"-vf", "noise=alls=12:allf=t+u,vignette=PI/4,format=yuv420p",
|
|
"-r", "25", "-c:v", "libx264", "-preset", "ultrafast", "-pix_fmt", "yuv420p", "-an", final_clip_path
|
|
], check=True, capture_output=True)
|
|
chosen_clip = final_clip_path
|
|
|
|
# Record fingerprint for zero duplicate guarantee
|
|
clip_hash = get_file_hash(chosen_clip)
|
|
if clip_hash:
|
|
self.seen_clip_hashes.add(clip_hash)
|
|
|
|
# Apply dynamic 35mm film grain filter overlay if scene depicts past/historical events
|
|
if is_historical_scene(seg, topic=getattr(self.cfg, "topic", "")):
|
|
aged_tmp = os.path.join(out_dir, f"aged_{idx:03d}.mp4")
|
|
apply_film_grain_overlay(chosen_clip, aged_tmp)
|
|
if os.path.exists(aged_tmp) and os.path.getsize(aged_tmp) > 1000:
|
|
import shutil
|
|
shutil.move(aged_tmp, chosen_clip)
|
|
logger.info(f"[VISUAL] Segment {idx+1}: Applied authentic 35mm film grain & vintage aging overlay.")
|
|
|
|
return chosen_clip
|
|
|
|
def generate(self, script: Dict[str, Any], job_dir: str, segment_timings: Optional[List[Dict[str, Any]]] = None) -> Dict[str, List[str]]:
|
|
"""
|
|
Generate bilingual sequences of video clips matching each script segment.
|
|
Returns: {"en": List[str], "es": List[str]}
|
|
If on-screen text is present in a segment, generates distinct English and Spanish clips.
|
|
"""
|
|
self.stock_fetcher.reset_session()
|
|
self.seen_clip_hashes.clear()
|
|
segs = script.get("script_segments", [])
|
|
logger.info(f"[VISUAL] Producing {len(segs)} video clips (bilingual text-matching enabled)...")
|
|
pc = self.pcs.get("nvidiam")
|
|
if not pc or not pc.check(8188):
|
|
raise RuntimeError("NvidiaLLM ComfyUI is unreachable on port 8188")
|
|
|
|
clips_dir = os.path.join(job_dir, "clips")
|
|
os.makedirs(clips_dir, exist_ok=True)
|
|
|
|
clips_en = []
|
|
clips_es = []
|
|
|
|
for i, seg in enumerate(segs):
|
|
t0 = time.time()
|
|
dur = segment_timings[i]["duration"] if segment_timings and i < len(segment_timings) else seg.get("duration_estimate", 18)
|
|
base_clip_path = self._generate_segment_visual(pc, seg, i, clips_dir, dur)
|
|
|
|
text_en = (seg.get("on_screen_text") or "").strip()
|
|
text_es = (seg.get("on_screen_text_es") or text_en).strip()
|
|
|
|
if text_en or text_es:
|
|
# Segment has on-screen text: render separate English and Spanish video clips
|
|
clip_en = os.path.join(clips_dir, f"clip_en_{i:03d}.mp4")
|
|
clip_es = os.path.join(clips_dir, f"clip_es_{i:03d}.mp4")
|
|
|
|
apply_text_overlay(base_clip_path, clip_en, text_en, dur)
|
|
apply_text_overlay(base_clip_path, clip_es, text_es, dur)
|
|
|
|
clips_en.append(clip_en)
|
|
clips_es.append(clip_es)
|
|
logger.info(f"[VISUAL] Clip {i+1}/{len(segs)} bilingual text applied: '{text_en}' (EN) / '{text_es}' (ES) in {time.time()-t0:.1f}s")
|
|
else:
|
|
# Clean visual clip without text is shared across both languages
|
|
clips_en.append(base_clip_path)
|
|
clips_es.append(base_clip_path)
|
|
logger.info(f"[VISUAL] Clip {i+1}/{len(segs)} generated in {time.time()-t0:.1f}s -> {os.path.basename(base_clip_path)}")
|
|
|
|
return {"en": clips_en, "es": clips_es}
|