627 lines
31 KiB
Python
627 lines
31 KiB
Python
|
"""
|
||
|
|
Visual Generation & Stock Media Module.
|
||
|
|
Handles:
|
||
|
|
1. NvidiaLLM ComfyUI MiniMax H3 Text-to-Video generation
|
||
|
|
2. NvidiaLLM ComfyUI z-image-turbo / SDXL AI Image generation
|
||
|
|
3. Internet Stock Video & Stock Image downloaders (Wikimedia Commons, Pexels, Pixabay)
|
||
|
|
4. Dynamic Ken Burns Motion Effects (pan/zoom) in FFmpeg for still images
|
||
|
|
5. Intelligent visual curation and multi-tier fallback pipeline
|
||
|
|
"""
|
||
|
|
|
||
|
|
import os
|
||
|
|
import re
|
||
|
|
import time
|
||
|
|
import json
|
||
|
|
import zlib
|
||
|
|
import random
|
||
|
|
import logging
|
||
|
|
import urllib.request
|
||
|
|
import urllib.parse
|
||
|
|
import subprocess
|
||
|
|
import requests
|
||
|
|
from typing import Dict, Any, List, Optional
|
||
|
|
from .remote import RemotePC
|
||
|
|
|
||
|
|
logger = logging.getLogger("YTFactory.Visual")
|
||
|
|
|
||
|
|
# ─── ComfyUI Model Constants ────────────────────────────────────────────────
|
||
|
|
H3_UNET = "minimax_h3_fl2va_pruned_int8_convrot.safetensors"
|
||
|
|
H3_CLIP = "qwen3vl_32b_minimax_h3_nvfp4_awq.safetensors"
|
||
|
|
H3_VAE = "minimax_h3_video_vae_fp16.safetensors"
|
||
|
|
H3_FPS = 24
|
||
|
|
H3_MIN_LEN, H3_MAX_LEN = 124, 362
|
||
|
|
|
||
|
|
ZIMAGE_UNET = "z_image_turbo_bf16.safetensors"
|
||
|
|
ZIMAGE_CLIP = "qwen_3_4b.safetensors"
|
||
|
|
ZIMAGE_VAE = "ae.safetensors"
|
||
|
|
|
||
|
|
SDXL_CKPT = "sd_xl_base_1.0.safetensors"
|
||
|
|
|
||
|
|
def h3_snap_length(seconds: float) -> int:
|
||
|
|
"""Snap a duration in seconds to the MiniMax H3 17k+5 frame grid."""
|
||
|
|
n = max(H3_MIN_LEN, min(H3_MAX_LEN, int(round(seconds * H3_FPS))))
|
||
|
|
k = max(7, round((n - 5) / 17))
|
||
|
|
return 17 * k + 5
|
||
|
|
|
||
|
|
# ─── Stock Media Fetcher ────────────────────────────────────────────────────
|
||
|
|
|
||
|
|
class StockMediaFetcher:
|
||
|
|
def __init__(self, pexels_key: Optional[str] = None, pixabay_key: Optional[str] = None):
|
||
|
|
self.pexels_key = pexels_key
|
||
|
|
self.pixabay_key = pixabay_key
|
||
|
|
self.user_agent = "YouTubeFactory/2.0 (faceless-video-pipeline)"
|
||
|
|
|
||
|
|
def _query_wikimedia(self, query: str, is_video: bool = False) -> Optional[str]:
|
||
|
|
"""Search Wikimedia Commons for high-resolution images or videos without API keys."""
|
||
|
|
try:
|
||
|
|
clean_q = re.sub(r'[^A-Za-z0-9 ]', '', query).strip()
|
||
|
|
encoded_q = urllib.parse.quote(clean_q)
|
||
|
|
api_url = (
|
||
|
|
f"https://commons.wikimedia.org/w/api.php?action=query&generator=search"
|
||
|
|
f"&gsrsearch={encoded_q}&gsrnamespace=6&gsrlimit=5&prop=imageinfo"
|
||
|
|
f"&iiprop=url|mime|size&format=json"
|
||
|
|
)
|
||
|
|
req = urllib.request.Request(api_url, headers={"User-Agent": self.user_agent})
|
||
|
|
with urllib.request.urlopen(req, timeout=10) as resp:
|
||
|
|
data = json.loads(resp.read().decode("utf-8"))
|
||
|
|
pages = data.get("query", {}).get("pages", {})
|
||
|
|
for _, page in pages.items():
|
||
|
|
infos = page.get("imageinfo", [])
|
||
|
|
if not infos:
|
||
|
|
continue
|
||
|
|
url = infos[0].get("url", "")
|
||
|
|
mime = infos[0].get("mime", "").lower()
|
||
|
|
if is_video and ("video" in mime or url.endswith((".webm", ".mp4", ".ogv"))):
|
||
|
|
return url
|
||
|
|
if not is_video and ("image" in mime or url.endswith((".jpg", ".jpeg", ".png", ".webp"))):
|
||
|
|
return url
|
||
|
|
except Exception as e:
|
||
|
|
logger.debug(f"Wikimedia search failed for '{query}': {e}")
|
||
|
|
return None
|
||
|
|
|
||
|
|
def _query_pexels_video(self, query: str) -> Optional[str]:
|
||
|
|
"""Search Pexels for free HD stock videos if API key is provided."""
|
||
|
|
if not self.pexels_key:
|
||
|
|
return None
|
||
|
|
try:
|
||
|
|
clean_q = urllib.parse.quote(query)
|
||
|
|
url = f"https://api.pexels.com/videos/search?query={clean_q}&per_page=5&orientation=landscape"
|
||
|
|
headers = {"Authorization": self.pexels_key, "User-Agent": self.user_agent}
|
||
|
|
req = urllib.request.Request(url, headers=headers)
|
||
|
|
with urllib.request.urlopen(req, timeout=10) as resp:
|
||
|
|
data = json.loads(resp.read().decode("utf-8"))
|
||
|
|
videos = data.get("videos", [])
|
||
|
|
for v in videos:
|
||
|
|
files = v.get("video_files", [])
|
||
|
|
# Prefer the smallest HD file (>=1280 wide) to keep downloads fast
|
||
|
|
hd = [vf for vf in files if vf.get("width", 0) >= 1280 and vf.get("link")]
|
||
|
|
if hd:
|
||
|
|
return sorted(hd, key=lambda x: x["width"])[0]["link"]
|
||
|
|
if files and files[0].get("link"):
|
||
|
|
return files[0]["link"]
|
||
|
|
except Exception as e:
|
||
|
|
logger.debug(f"Pexels video query failed: {e}")
|
||
|
|
return None
|
||
|
|
|
||
|
|
def _query_pexels_photo(self, query: str) -> Optional[str]:
|
||
|
|
"""Search Pexels for free HD stock photos if API key is provided."""
|
||
|
|
if not self.pexels_key:
|
||
|
|
return None
|
||
|
|
try:
|
||
|
|
clean_q = urllib.parse.quote(query)
|
||
|
|
url = f"https://api.pexels.com/v1/search?query={clean_q}&per_page=5&orientation=landscape"
|
||
|
|
headers = {"Authorization": self.pexels_key, "User-Agent": self.user_agent}
|
||
|
|
req = urllib.request.Request(url, headers=headers)
|
||
|
|
with urllib.request.urlopen(req, timeout=10) as resp:
|
||
|
|
data = json.loads(resp.read().decode("utf-8"))
|
||
|
|
photos = data.get("photos", [])
|
||
|
|
if photos:
|
||
|
|
return photos[0].get("src", {}).get("large2x") or photos[0].get("src", {}).get("large")
|
||
|
|
except Exception as e:
|
||
|
|
logger.debug(f"Pexels photo query failed: {e}")
|
||
|
|
return None
|
||
|
|
|
||
|
|
def _query_pixabay_video(self, query: str) -> Optional[str]:
|
||
|
|
"""Search Pixabay for free HD stock videos if API key is provided."""
|
||
|
|
if not self.pixabay_key:
|
||
|
|
return None
|
||
|
|
try:
|
||
|
|
clean_q = urllib.parse.quote(query)
|
||
|
|
url = f"https://pixabay.com/api/videos/?key={self.pixabay_key}&q={clean_q}&per_page=5&safesearch=true"
|
||
|
|
req = urllib.request.Request(url, headers={"User-Agent": self.user_agent})
|
||
|
|
with urllib.request.urlopen(req, timeout=10) as resp:
|
||
|
|
data = json.loads(resp.read().decode("utf-8"))
|
||
|
|
for hit in data.get("hits", []):
|
||
|
|
vids = hit.get("videos", {})
|
||
|
|
for quality in ("large", "medium", "small"):
|
||
|
|
v = vids.get(quality)
|
||
|
|
if v and v.get("url") and v.get("width", 0) >= 640:
|
||
|
|
return v["url"]
|
||
|
|
except Exception as e:
|
||
|
|
logger.debug(f"Pixabay video query failed: {e}")
|
||
|
|
return None
|
||
|
|
|
||
|
|
def _query_pixabay_photo(self, query: str) -> Optional[str]:
|
||
|
|
"""Search Pixabay for free HD stock photos if API key is provided."""
|
||
|
|
if not self.pixabay_key:
|
||
|
|
return None
|
||
|
|
try:
|
||
|
|
clean_q = urllib.parse.quote(query)
|
||
|
|
url = (f"https://pixabay.com/api/?key={self.pixabay_key}&q={clean_q}"
|
||
|
|
f"&image_type=photo&orientation=horizontal&per_page=5&safesearch=true")
|
||
|
|
req = urllib.request.Request(url, headers={"User-Agent": self.user_agent})
|
||
|
|
with urllib.request.urlopen(req, timeout=10) as resp:
|
||
|
|
data = json.loads(resp.read().decode("utf-8"))
|
||
|
|
hits = data.get("hits", [])
|
||
|
|
if hits:
|
||
|
|
return hits[0].get("largeImageURL") or hits[0].get("webformatURL")
|
||
|
|
except Exception as e:
|
||
|
|
logger.debug(f"Pixabay photo query failed: {e}")
|
||
|
|
return None
|
||
|
|
|
||
|
|
def download_media(self, url: str, out_path: str) -> bool:
|
||
|
|
"""Download remote media with retry and validation."""
|
||
|
|
try:
|
||
|
|
req = urllib.request.Request(url, headers={"User-Agent": self.user_agent})
|
||
|
|
with urllib.request.urlopen(req, timeout=30) as resp, open(out_path, "wb") as f:
|
||
|
|
f.write(resp.read())
|
||
|
|
if os.path.exists(out_path) and os.path.getsize(out_path) > 5000:
|
||
|
|
return True
|
||
|
|
except Exception as e:
|
||
|
|
logger.debug(f"Download failed from {url}: {e}")
|
||
|
|
if os.path.exists(out_path):
|
||
|
|
try:
|
||
|
|
os.remove(out_path)
|
||
|
|
except OSError:
|
||
|
|
pass
|
||
|
|
return False
|
||
|
|
|
||
|
|
def search_and_download(self, query: str, out_path: str, prefer_video: bool = False) -> Optional[str]:
|
||
|
|
"""Search across stock engines and download asset."""
|
||
|
|
logger.info(f"[STOCK] Searching stock media for: '{query}' (video={prefer_video})...")
|
||
|
|
if prefer_video:
|
||
|
|
for label, fn in (("Pexels", self._query_pexels_video),
|
||
|
|
("Pixabay", self._query_pixabay_video),
|
||
|
|
("Wikimedia", lambda q: self._query_wikimedia(q, is_video=True))):
|
||
|
|
v_url = fn(query)
|
||
|
|
if v_url and self.download_media(v_url, out_path):
|
||
|
|
logger.info(f"[STOCK] Downloaded {label} video for '{query}'")
|
||
|
|
return out_path
|
||
|
|
return None
|
||
|
|
|
||
|
|
for label, fn in (("Pexels", self._query_pexels_photo),
|
||
|
|
("Pixabay", self._query_pixabay_photo),
|
||
|
|
("Wikimedia", lambda q: self._query_wikimedia(q, is_video=False))):
|
||
|
|
p_url = fn(query)
|
||
|
|
if p_url and self.download_media(p_url, out_path):
|
||
|
|
logger.info(f"[STOCK] Downloaded {label} photo for '{query}'")
|
||
|
|
return out_path
|
||
|
|
|
||
|
|
logger.debug(f"[STOCK] No stock media found for: '{query}'")
|
||
|
|
return None
|
||
|
|
|
||
|
|
# ─── Ken Burns Motion Engine ────────────────────────────────────────────────
|
||
|
|
|
||
|
|
def apply_ken_burns_effect(img_path: str, out_mp4: str, duration_sec: float, motion_type: int = 0) -> str:
|
||
|
|
"""
|
||
|
|
Animate a still image into a cinematic 16:9 25fps video clip using FFmpeg pan & zoom motion.
|
||
|
|
Provides 6 distinct cinematic camera movements:
|
||
|
|
0: Smooth Center Zoom-In
|
||
|
|
1: Smooth Center Zoom-Out
|
||
|
|
2: Diagonal Pan Left-Up with subtle zoom
|
||
|
|
3: Diagonal Pan Right-Down with subtle zoom
|
||
|
|
4: Slow Pan Right across center
|
||
|
|
5: Slow Pan Left across center
|
||
|
|
"""
|
||
|
|
total_frames = max(50, int(round(duration_sec * 25)))
|
||
|
|
m_idx = motion_type % 6
|
||
|
|
|
||
|
|
if m_idx == 0:
|
||
|
|
# Smooth Zoom-In (center)
|
||
|
|
z_expr = "min(zoom+0.0012,1.25)"
|
||
|
|
x_expr = "iw/2-(iw/zoom/2)"
|
||
|
|
y_expr = "ih/2-(ih/zoom/2)"
|
||
|
|
elif m_idx == 1:
|
||
|
|
# Smooth Zoom-Out (center)
|
||
|
|
z_expr = "if(eq(on,1),1.25,max(1.0,zoom-0.0012))"
|
||
|
|
x_expr = "iw/2-(iw/zoom/2)"
|
||
|
|
y_expr = "ih/2-(ih/zoom/2)"
|
||
|
|
elif m_idx == 2:
|
||
|
|
# Diagonal Pan Left-Up with subtle zoom
|
||
|
|
z_expr = "1.20"
|
||
|
|
x_expr = f"if(eq(on,1),iw-iw/zoom,max(0,x-(iw-iw/zoom)/{total_frames}))"
|
||
|
|
y_expr = f"if(eq(on,1),ih-ih/zoom,max(0,y-(ih-ih/zoom)/{total_frames}))"
|
||
|
|
elif m_idx == 3:
|
||
|
|
# Diagonal Pan Right-Down with subtle zoom
|
||
|
|
z_expr = "1.20"
|
||
|
|
x_expr = f"if(eq(on,1),0,min(iw-iw/zoom,x+(iw-iw/zoom)/{total_frames}))"
|
||
|
|
y_expr = f"if(eq(on,1),0,min(ih-ih/zoom,y+(ih-ih/zoom)/{total_frames}))"
|
||
|
|
elif m_idx == 4:
|
||
|
|
# Slow Pan Right across center
|
||
|
|
z_expr = "1.18"
|
||
|
|
x_expr = f"if(eq(on,1),0,min(iw-iw/zoom,x+(iw-iw/zoom)/{total_frames}))"
|
||
|
|
y_expr = "ih/2-(ih/zoom/2)"
|
||
|
|
else:
|
||
|
|
# Slow Pan Left across center
|
||
|
|
z_expr = "1.18"
|
||
|
|
x_expr = f"if(eq(on,1),iw-iw/zoom,max(0,x-(iw-iw/zoom)/{total_frames}))"
|
||
|
|
y_expr = "ih/2-(ih/zoom/2)"
|
||
|
|
|
||
|
|
vf = (
|
||
|
|
f"scale=1920:1080:force_original_aspect_ratio=increase,crop=1920:1080,"
|
||
|
|
f"zoompan=z='{z_expr}':x='{x_expr}':y='{y_expr}':d={total_frames}:s=1280x720:fps=25,"
|
||
|
|
f"format=yuv420p"
|
||
|
|
)
|
||
|
|
|
||
|
|
cmd = [
|
||
|
|
"ffmpeg", "-y", "-loop", "1", "-i", img_path,
|
||
|
|
"-vf", vf,
|
||
|
|
"-t", f"{duration_sec:.2f}",
|
||
|
|
"-r", "25",
|
||
|
|
"-c:v", "libx264", "-preset", "fast", "-crf", "22",
|
||
|
|
"-pix_fmt", "yuv420p", "-an", out_mp4
|
||
|
|
]
|
||
|
|
r = subprocess.run(cmd, capture_output=True, text=True)
|
||
|
|
if r.returncode != 0:
|
||
|
|
logger.warning(f"Ken Burns animation fallback: {r.stderr[-200:]}")
|
||
|
|
# Simple loop fallback
|
||
|
|
fallback_vf = "scale=1280:720:force_original_aspect_ratio=decrease,pad=1280:720:(ow-iw)/2:(oh-ih)/2:black,format=yuv420p"
|
||
|
|
subprocess.run([
|
||
|
|
"ffmpeg", "-y", "-loop", "1", "-i", img_path,
|
||
|
|
"-vf", fallback_vf, "-t", f"{duration_sec:.2f}", "-r", "25",
|
||
|
|
"-c:v", "libx264", "-preset", "fast", "-crf", "22", "-pix_fmt", "yuv420p", "-an", out_mp4
|
||
|
|
], check=True, capture_output=True)
|
||
|
|
|
||
|
|
return out_mp4
|
||
|
|
|
||
|
|
def normalize_clip_to_duration(src_path: str, out_path: str, duration_sec: float) -> str:
|
||
|
|
"""
|
||
|
|
Normalize any video clip to exactly duration_sec at 1280x720 25fps.
|
||
|
|
Loops short clips (-stream_loop) and trims long ones (-t) so every segment
|
||
|
|
stays in sync with its narration.
|
||
|
|
"""
|
||
|
|
vf = ("scale=1280:720:force_original_aspect_ratio=decrease,"
|
||
|
|
"pad=1280:720:(ow-iw)/2:(oh-ih)/2:black,fps=25,format=yuv420p")
|
||
|
|
r = subprocess.run([
|
||
|
|
"ffmpeg", "-y", "-stream_loop", "-1", "-i", src_path,
|
||
|
|
"-vf", vf, "-t", f"{duration_sec:.2f}", "-r", "25",
|
||
|
|
"-c:v", "libx264", "-preset", "fast", "-crf", "22",
|
||
|
|
"-pix_fmt", "yuv420p", "-an", out_path
|
||
|
|
], capture_output=True, text=True)
|
||
|
|
if r.returncode != 0:
|
||
|
|
raise RuntimeError(f"clip normalization failed: {r.stderr[-200:]}")
|
||
|
|
return out_path
|
||
|
|
|
||
|
|
def apply_text_overlay(video_path: str, out_mp4: str, text: str, duration_sec: float) -> str:
|
||
|
|
"""
|
||
|
|
Render a broadcast-style lower-third text banner onto the video clip.
|
||
|
|
Only active if text is non-empty.
|
||
|
|
"""
|
||
|
|
clean_text = re.sub(r"[^A-Za-z0-9 \-\?\!\,\:\'\.\/]", "", text).strip()
|
||
|
|
if not clean_text:
|
||
|
|
if video_path != out_mp4:
|
||
|
|
import shutil
|
||
|
|
shutil.copy(video_path, out_mp4)
|
||
|
|
return out_mp4
|
||
|
|
|
||
|
|
font_paths = [
|
||
|
|
"/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf",
|
||
|
|
"/usr/share/fonts/TTF/DejaVuSans-Bold.ttf",
|
||
|
|
"/usr/share/fonts/truetype/liberation/LiberationSans-Bold.ttf",
|
||
|
|
"/usr/share/fonts/truetype/freefont/FreeSansBold.ttf",
|
||
|
|
]
|
||
|
|
font_file = None
|
||
|
|
for fp in font_paths:
|
||
|
|
if os.path.exists(fp):
|
||
|
|
font_file = fp
|
||
|
|
break
|
||
|
|
|
||
|
|
font_param = f":fontfile='{font_file}'" if font_file else ""
|
||
|
|
escaped_text = clean_text.replace("'", "'\\''").replace(":", "\\:")
|
||
|
|
|
||
|
|
# Lower-third banner active between 0.8s and (duration - 0.8s)
|
||
|
|
end_t = max(1.0, duration_sec - 0.8)
|
||
|
|
enable_expr = f"between(t,0.8,{end_t:.2f})"
|
||
|
|
vf = (
|
||
|
|
f"drawtext=text='{escaped_text}'{font_param}:fontcolor=white:fontsize=36:"
|
||
|
|
f"box=1:boxcolor=black@0.65:boxborderw=14:x=60:y=h-th-60:enable='{enable_expr}'"
|
||
|
|
)
|
||
|
|
|
||
|
|
cmd = [
|
||
|
|
"ffmpeg", "-y", "-i", video_path,
|
||
|
|
"-vf", vf,
|
||
|
|
"-c:v", "libx264", "-preset", "fast", "-crf", "22", "-pix_fmt", "yuv420p",
|
||
|
|
"-an", out_mp4
|
||
|
|
]
|
||
|
|
r = subprocess.run(cmd, capture_output=True, text=True)
|
||
|
|
if r.returncode != 0:
|
||
|
|
logger.warning(f"Text overlay rendering fallback: {r.stderr[-200:]}")
|
||
|
|
if video_path != out_mp4:
|
||
|
|
import shutil
|
||
|
|
shutil.copy(video_path, out_mp4)
|
||
|
|
return out_mp4
|
||
|
|
|
||
|
|
# ─── Film Grain & Vintage Aging Filter ──────────────────────────────────────
|
||
|
|
|
||
|
|
HISTORICAL_KEYWORDS = {
|
||
|
|
"ancient", "archaeology", "archaeological", "history", "historical", "century",
|
||
|
|
"b.c.", "bc", "a.d.", "ad", "victorian", "medieval", "ruins", "pharaoh",
|
||
|
|
"temple", "pyramids", "past", "hunter gatherer", "hunter-gatherer", "gobeklitepe",
|
||
|
|
"göbeklitepe", "stone age", "bronze age", "iron age", "1888", "1900", "1920",
|
||
|
|
"1930", "1940", "1950", "1960", "1970", "empire", "dynasty", "columbus",
|
||
|
|
"alexandria", "antiquity", "paleolithic", "neolithic", "civilization", "excavation",
|
||
|
|
"fossil", "scrolls", "papyrus", "jack the ripper", "monument", "artifacts", "artifact",
|
||
|
|
"bermuda", "atlantis", "mythology", "prehistoric", "dynastic", "tomb", "sarcophagus",
|
||
|
|
"excavations", "relic", "relics", "megalith", "megalithic", "sanctuary"
|
||
|
|
}
|
||
|
|
|
||
|
|
def is_historical_scene(seg: Dict[str, Any], topic: str = "") -> bool:
|
||
|
|
"""Check if the segment depicts historical/past events based on flag or keywords."""
|
||
|
|
if seg.get("is_historical") is True:
|
||
|
|
return True
|
||
|
|
combined_text = (
|
||
|
|
f"{topic} {seg.get('chapter_title', '')} {seg.get('text', '')} "
|
||
|
|
f"{seg.get('visual_prompt', '')} {seg.get('stock_query', '')}"
|
||
|
|
).lower()
|
||
|
|
return any(kw in combined_text for kw in HISTORICAL_KEYWORDS)
|
||
|
|
|
||
|
|
def apply_film_grain_overlay(video_path: str, out_mp4: str) -> str:
|
||
|
|
"""
|
||
|
|
Apply a dynamic 35mm film grain overlay with subtle vintage contrast & vignette
|
||
|
|
to give historical footage an authentically aged cinematic look.
|
||
|
|
"""
|
||
|
|
vintage_vf = (
|
||
|
|
"noise=alls=14:allf=t+u,"
|
||
|
|
"eq=contrast=1.06:brightness=-0.01:saturation=0.88,"
|
||
|
|
"vignette=PI/5"
|
||
|
|
)
|
||
|
|
cmd = [
|
||
|
|
"ffmpeg", "-y", "-i", video_path,
|
||
|
|
"-vf", vintage_vf,
|
||
|
|
"-c:v", "libx264", "-preset", "fast", "-crf", "22", "-pix_fmt", "yuv420p",
|
||
|
|
"-an", out_mp4
|
||
|
|
]
|
||
|
|
r = subprocess.run(cmd, capture_output=True, text=True)
|
||
|
|
if r.returncode != 0:
|
||
|
|
logger.warning(f"Film grain filter fallback: {r.stderr[-200:]}")
|
||
|
|
if video_path != out_mp4:
|
||
|
|
import shutil
|
||
|
|
shutil.copy(video_path, out_mp4)
|
||
|
|
return out_mp4
|
||
|
|
|
||
|
|
# ─── ComfyUI Orchestration ─────────────────────────────────────────────────
|
||
|
|
|
||
|
|
class VisualGen:
|
||
|
|
def __init__(self, pcs: Dict[str, RemotePC], cfg: Any):
|
||
|
|
self.pcs = pcs
|
||
|
|
self.cfg = cfg
|
||
|
|
self.stock_fetcher = StockMediaFetcher(
|
||
|
|
pexels_key=getattr(cfg, "pexels_api_key", None),
|
||
|
|
pixabay_key=getattr(cfg, "pixabay_api_key", None)
|
||
|
|
)
|
||
|
|
|
||
|
|
def _h3_workflow(self, prompt: str, seed: int, width: int, height: int, length: int) -> Dict[str, Any]:
|
||
|
|
return {
|
||
|
|
"1": {"inputs": {"unet_name": H3_UNET, "weight_dtype": "default"}, "class_type": "UNETLoader"},
|
||
|
|
"2": {"inputs": {"model": ["1", 0], "shift_video": 12.0, "shift_audio": 3.0}, "class_type": "MiniMaxH3SigmaShift"},
|
||
|
|
"3": {"inputs": {"clip_name": H3_CLIP, "type": "minimax"}, "class_type": "CLIPLoader"},
|
||
|
|
"4": {"inputs": {"vae_name": H3_VAE}, "class_type": "VAELoader"},
|
||
|
|
"5": {"inputs": {"clip": ["3", 0], "vae": ["4", 0], "prompt": prompt,
|
||
|
|
"width": width, "height": height, "length": length},
|
||
|
|
"class_type": "MiniMaxH3ImageToVideo"},
|
||
|
|
"6": {"inputs": {"text": "", "clip": ["3", 0]}, "class_type": "CLIPTextEncode"},
|
||
|
|
"7": {"inputs": {"seed": seed, "steps": 20, "cfg": 5.0, "sampler_name": "euler",
|
||
|
|
"scheduler": "normal", "denoise": 1.0,
|
||
|
|
"model": ["2", 0], "positive": ["5", 0], "negative": ["6", 0],
|
||
|
|
"latent_image": ["5", 1]}, "class_type": "KSampler"},
|
||
|
|
"8": {"inputs": {"samples": ["7", 0], "vae": ["4", 0]}, "class_type": "VAEDecode"},
|
||
|
|
"9": {"inputs": {"images": ["8", 0], "fps": float(H3_FPS)}, "class_type": "CreateVideo"},
|
||
|
|
"10": {"inputs": {"video": ["9", 0], "filename_prefix": "yt_clip",
|
||
|
|
"format": "auto", "codec": "auto"}, "class_type": "SaveVideo"},
|
||
|
|
}
|
||
|
|
|
||
|
|
def _sdxl_workflow(self, prompt: str, seed: int) -> Dict[str, Any]:
|
||
|
|
return {
|
||
|
|
"1": {"inputs": {"ckpt_name": SDXL_CKPT}, "class_type": "CheckpointLoaderSimple"},
|
||
|
|
"2": {"inputs": {"text": prompt + ", 8k resolution, cinematic lighting, photorealistic, documentary b-roll", "clip": ["1", 1]}, "class_type": "CLIPTextEncode"},
|
||
|
|
"2b": {"inputs": {"text": "blurry, low quality, watermark, text, logo, ugly, deformed, oversaturated", "clip": ["1", 1]}, "class_type": "CLIPTextEncode"},
|
||
|
|
"3": {"inputs": {"width": 1280, "height": 720, "batch_size": 1}, "class_type": "EmptyLatentImage"},
|
||
|
|
"4": {"inputs": {"seed": seed, "steps": 22, "cfg": 7.0, "sampler_name": "euler",
|
||
|
|
"scheduler": "normal", "denoise": 1.0,
|
||
|
|
"model": ["1", 0], "positive": ["2", 0], "negative": ["2b", 0],
|
||
|
|
"latent_image": ["3", 0]}, "class_type": "KSampler"},
|
||
|
|
"5": {"inputs": {"samples": ["4", 0], "vae": ["1", 2]}, "class_type": "VAEDecode"},
|
||
|
|
"6": {"inputs": {"filename_prefix": "yt_img", "images": ["5", 0]}, "class_type": "SaveImage"},
|
||
|
|
}
|
||
|
|
|
||
|
|
def _submit(self, host: str, workflow: Dict[str, Any]) -> str:
|
||
|
|
r = requests.post(f"http://{host}:8188/prompt", json={"prompt": workflow}, timeout=30)
|
||
|
|
data = r.json()
|
||
|
|
if r.status_code != 200 or "prompt_id" not in data:
|
||
|
|
raise RuntimeError(f"ComfyUI rejected workflow: {json.dumps(data)[:300]}")
|
||
|
|
return data["prompt_id"]
|
||
|
|
|
||
|
|
def _cancel_job(self, host: str, pid: str):
|
||
|
|
"""Interrupt current execution and remove the job from the ComfyUI queue.
|
||
|
|
Prevents client-side timeouts from leaving zombie jobs that block later prompts."""
|
||
|
|
try:
|
||
|
|
requests.post(f"http://{host}:8188/interrupt", timeout=5)
|
||
|
|
except Exception:
|
||
|
|
pass
|
||
|
|
try:
|
||
|
|
requests.post(f"http://{host}:8188/queue", json={"delete": [pid]}, timeout=5)
|
||
|
|
except Exception:
|
||
|
|
pass
|
||
|
|
|
||
|
|
def _wait_result(self, host: str, pid: str, timeout_s: int = 1800) -> Dict[str, Any]:
|
||
|
|
deadline = time.time() + timeout_s
|
||
|
|
while time.time() < deadline:
|
||
|
|
time.sleep(3)
|
||
|
|
try:
|
||
|
|
h = requests.get(f"http://{host}:8188/history/{pid}", timeout=10).json()
|
||
|
|
except Exception:
|
||
|
|
continue
|
||
|
|
if pid not in h:
|
||
|
|
continue
|
||
|
|
status = h[pid].get("status", {})
|
||
|
|
if status.get("status_str") == "error":
|
||
|
|
msgs = h[pid].get("status", {}).get("messages", [])
|
||
|
|
self._cancel_job(host, pid)
|
||
|
|
raise RuntimeError(f"ComfyUI execution error: {json.dumps(msgs)[:300]}")
|
||
|
|
if status.get("completed") or h[pid].get("outputs"):
|
||
|
|
return h[pid].get("outputs", {})
|
||
|
|
self._cancel_job(host, pid)
|
||
|
|
raise RuntimeError("ComfyUI generation timed out.")
|
||
|
|
|
||
|
|
def _download_outputs(self, host: str, outputs: Dict[str, Any], out_path: str) -> str:
|
||
|
|
for node_out in outputs.values():
|
||
|
|
for key in ("videos", "gifs", "images"):
|
||
|
|
for item in node_out.get(key, []):
|
||
|
|
params = {"filename": item["filename"], "type": item.get("type", "output")}
|
||
|
|
if item.get("subfolder"):
|
||
|
|
params["subfolder"] = item["subfolder"]
|
||
|
|
d = requests.get(f"http://{host}:8188/view", params=params, timeout=120).content
|
||
|
|
if len(d) >= 1024:
|
||
|
|
with open(out_path, "wb") as f:
|
||
|
|
f.write(d)
|
||
|
|
return out_path
|
||
|
|
raise RuntimeError("No downloadable asset returned by ComfyUI")
|
||
|
|
|
||
|
|
# Maximum frames for a MiniMax H3 clip (~6.5s at 24fps). Longer clips take
|
||
|
|
# 30+ min on the RTX 4060; clips are looped/trimmed to segment length anyway.
|
||
|
|
H3_CLIP_CAP = 158
|
||
|
|
|
||
|
|
def _try_ai_video(self, pc, prompt, seed, duration_sec, raw_path, final_clip_path, idx):
|
||
|
|
"""MiniMax H3 text-to-video clip, normalized to the segment duration."""
|
||
|
|
h3_length = min(h3_snap_length(duration_sec), self.H3_CLIP_CAP)
|
||
|
|
logger.info(f"[VISUAL] Segment {idx+1}: MiniMax H3 AI video ({h3_length} frames)...")
|
||
|
|
wf = self._h3_workflow(prompt, seed, self.cfg.clip_width, self.cfg.clip_height, h3_length)
|
||
|
|
pid = self._submit(pc.host, wf)
|
||
|
|
outputs = self._wait_result(pc.host, pid, timeout_s=1200)
|
||
|
|
self._download_outputs(pc.host, outputs, raw_path)
|
||
|
|
return normalize_clip_to_duration(raw_path, final_clip_path, duration_sec)
|
||
|
|
|
||
|
|
def _try_ai_image(self, pc, prompt, seed, duration_sec, raw_path, final_clip_path, idx):
|
||
|
|
"""SDXL AI image animated with Ken Burns motion."""
|
||
|
|
logger.info(f"[VISUAL] Segment {idx+1}: SDXL AI image + Ken Burns...")
|
||
|
|
pid = self._submit(pc.host, self._sdxl_workflow(prompt, seed))
|
||
|
|
outputs = self._wait_result(pc.host, pid, timeout_s=300)
|
||
|
|
self._download_outputs(pc.host, outputs, raw_path)
|
||
|
|
return apply_ken_burns_effect(raw_path, final_clip_path, duration_sec, motion_type=idx)
|
||
|
|
|
||
|
|
def _try_stock(self, query, duration_sec, raw_path, final_clip_path, idx, prefer_video):
|
||
|
|
"""Stock video (looped/trimmed) or stock image (Ken Burns)."""
|
||
|
|
kind = "video" if prefer_video else "image"
|
||
|
|
logger.info(f"[VISUAL] Segment {idx+1}: stock {kind} for '{query}'...")
|
||
|
|
got = self.stock_fetcher.search_and_download(query, raw_path, prefer_video=prefer_video)
|
||
|
|
if not got or not os.path.exists(got):
|
||
|
|
raise RuntimeError(f"no stock {kind} found for '{query}'")
|
||
|
|
if got.lower().endswith((".mp4", ".webm", ".mov")):
|
||
|
|
return normalize_clip_to_duration(got, final_clip_path, duration_sec)
|
||
|
|
return apply_ken_burns_effect(got, final_clip_path, duration_sec, motion_type=idx)
|
||
|
|
|
||
|
|
def _generate_segment_visual(self, pc: RemotePC, seg: Dict[str, Any], idx: int, out_dir: str, duration_sec: float) -> str:
|
||
|
|
"""
|
||
|
|
Produce one visual for a segment as an MP4 of exactly duration_sec.
|
||
|
|
|
||
|
|
"hybrid" rotates across 4 styles for maximum variety:
|
||
|
|
AI video (MiniMax H3) -> stock video -> AI image (Ken Burns) -> stock image (Ken Burns)
|
||
|
|
Each source falls back to the next one in the rotation on failure.
|
||
|
|
"""
|
||
|
|
prompt = (seg.get("visual_prompt") or seg.get("text", "")) + ", cinematic documentary style, 8k, detailed"
|
||
|
|
stock_query = seg.get("stock_query") or seg.get("chapter_title") or seg.get("text", "")[:40]
|
||
|
|
seed = zlib.crc32(prompt.encode()) % (2**31)
|
||
|
|
|
||
|
|
final_clip_path = os.path.join(out_dir, f"clip_{idx:03d}.mp4")
|
||
|
|
raw_video = os.path.join(out_dir, f"raw_vid_{idx:03d}.mp4")
|
||
|
|
raw_img = os.path.join(out_dir, f"raw_img_{idx:03d}.jpg")
|
||
|
|
|
||
|
|
strategy = getattr(self.cfg, "visual_strategy", "hybrid")
|
||
|
|
attempts = {
|
||
|
|
"ai_video": lambda: self._try_ai_video(pc, prompt, seed, duration_sec, raw_video, final_clip_path, idx),
|
||
|
|
"stock_video": lambda: self._try_stock(stock_query, duration_sec, raw_video, final_clip_path, idx, prefer_video=True),
|
||
|
|
"ai_image": lambda: self._try_ai_image(pc, prompt, seed, duration_sec, raw_img, final_clip_path, idx),
|
||
|
|
"stock_image": lambda: self._try_stock(stock_query, duration_sec, raw_img, final_clip_path, idx, prefer_video=False),
|
||
|
|
}
|
||
|
|
if strategy == "all_ai":
|
||
|
|
order = ["ai_video", "ai_image", "stock_video", "stock_image"]
|
||
|
|
elif strategy == "stock_focused":
|
||
|
|
order = ["stock_video", "stock_image", "ai_video", "ai_image"]
|
||
|
|
else: # hybrid — rotate the starting source so all 4 styles appear
|
||
|
|
base = ["ai_video", "stock_video", "ai_image", "stock_image"]
|
||
|
|
order = base[idx % 4:] + base[:idx % 4]
|
||
|
|
|
||
|
|
chosen_clip = None
|
||
|
|
for source in order:
|
||
|
|
try:
|
||
|
|
chosen_clip = attempts[source]()
|
||
|
|
break
|
||
|
|
except Exception as e:
|
||
|
|
logger.warning(f"[VISUAL] Segment {idx+1} source '{source}' failed: {e}")
|
||
|
|
errors.append(f"{source}: {e}")
|
||
|
|
|
||
|
|
if not chosen_clip:
|
||
|
|
# Emergency color background — pipeline never dies on visuals
|
||
|
|
logger.warning(f"[VISUAL] Segment {idx+1}: all sources failed, using color background")
|
||
|
|
subprocess.run([
|
||
|
|
"ffmpeg", "-y", "-f", "lavfi", "-i", f"color=c=0x0a1128:s=1280x720:d={duration_sec:.2f}",
|
||
|
|
"-r", "25", "-c:v", "libx264", "-preset", "ultrafast", "-pix_fmt", "yuv420p", "-an", final_clip_path
|
||
|
|
], check=True, capture_output=True)
|
||
|
|
chosen_clip = final_clip_path
|
||
|
|
|
||
|
|
# Apply dynamic 35mm film grain filter overlay if scene depicts past/historical events
|
||
|
|
if is_historical_scene(seg, topic=getattr(self.cfg, "topic", "")):
|
||
|
|
aged_tmp = os.path.join(out_dir, f"aged_{idx:03d}.mp4")
|
||
|
|
apply_film_grain_overlay(chosen_clip, aged_tmp)
|
||
|
|
if os.path.exists(aged_tmp) and os.path.getsize(aged_tmp) > 1000:
|
||
|
|
import shutil
|
||
|
|
shutil.move(aged_tmp, chosen_clip)
|
||
|
|
logger.info(f"[VISUAL] Segment {idx+1}: Applied authentic 35mm film grain & vintage aging overlay.")
|
||
|
|
|
||
|
|
return chosen_clip
|
||
|
|
|
||
|
|
def generate(self, script: Dict[str, Any], job_dir: str, segment_timings: Optional[List[Dict[str, Any]]] = None) -> Dict[str, List[str]]:
|
||
|
|
"""
|
||
|
|
Generate bilingual sequences of video clips matching each script segment.
|
||
|
|
Returns: {"en": List[str], "es": List[str]}
|
||
|
|
If on-screen text is present in a segment, generates distinct English and Spanish clips.
|
||
|
|
"""
|
||
|
|
segs = script.get("script_segments", [])
|
||
|
|
logger.info(f"[VISUAL] Producing {len(segs)} video clips (bilingual text-matching enabled)...")
|
||
|
|
pc = self.pcs.get("nvidiam")
|
||
|
|
if not pc or not pc.check(8188):
|
||
|
|
raise RuntimeError("NvidiaLLM ComfyUI is unreachable on port 8188")
|
||
|
|
|
||
|
|
clips_dir = os.path.join(job_dir, "clips")
|
||
|
|
os.makedirs(clips_dir, exist_ok=True)
|
||
|
|
|
||
|
|
clips_en = []
|
||
|
|
clips_es = []
|
||
|
|
|
||
|
|
for i, seg in enumerate(segs):
|
||
|
|
t0 = time.time()
|
||
|
|
dur = segment_timings[i]["duration"] if segment_timings and i < len(segment_timings) else seg.get("duration_estimate", 18)
|
||
|
|
base_clip_path = self._generate_segment_visual(pc, seg, i, clips_dir, dur)
|
||
|
|
|
||
|
|
text_en = (seg.get("on_screen_text") or "").strip()
|
||
|
|
text_es = (seg.get("on_screen_text_es") or text_en).strip()
|
||
|
|
|
||
|
|
if text_en or text_es:
|
||
|
|
# Segment has on-screen text: render separate English and Spanish video clips
|
||
|
|
clip_en = os.path.join(clips_dir, f"clip_en_{i:03d}.mp4")
|
||
|
|
clip_es = os.path.join(clips_dir, f"clip_es_{i:03d}.mp4")
|
||
|
|
|
||
|
|
apply_text_overlay(base_clip_path, clip_en, text_en, dur)
|
||
|
|
apply_text_overlay(base_clip_path, clip_es, text_es, dur)
|
||
|
|
|
||
|
|
clips_en.append(clip_en)
|
||
|
|
clips_es.append(clip_es)
|
||
|
|
logger.info(f"[VISUAL] Clip {i+1}/{len(segs)} bilingual text applied: '{text_en}' (EN) / '{text_es}' (ES) in {time.time()-t0:.1f}s")
|
||
|
|
else:
|
||
|
|
# Clean visual clip without text is shared across both languages
|
||
|
|
clips_en.append(base_clip_path)
|
||
|
|
clips_es.append(base_clip_path)
|
||
|
|
logger.info(f"[VISUAL] Clip {i+1}/{len(segs)} generated in {time.time()-t0:.1f}s -> {os.path.basename(base_clip_path)}")
|
||
|
|
|
||
|
|
return {"en": clips_en, "es": clips_es}
|