msrd0/render_video
1
0
Fork 0
Code Issues Pull requests Releases 7 Activity Actions

question overlays work :)

Browse source
This commit is contained in:
Dominic 2024-05-25 16:20:43 +02:00
parent 8b57b97c80
commit b11baf1358
Signed by: msrd0
GPG key ID: AAF7C8430CA3345D
6 changed files with 218 additions and 31 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
230101/project.toml
View file

@ -7,9 +7,13 @@ date = "230101"
[source]
files = ["C01.mp4", "C02.mp4", "C03.mp4"]
stereo = false
start = "2"
start = "1"
end = "12"
fast = [["5", "7"], ["9", "11"]]
fast = [["6", "8"], ["10", "11"]]
questions = [
["1.5", "3", "Hallo liebes Publikum. Ich habe leider meine Frage vergessen. Bitte entschuldigt die Störung."],
["3.5", "5", "Ah jetzt weiß ich es wieder. Meine Frage war: Was war meine Frage?"]
]
[source.metadata]
source_duration = "12.53000"
@ -22,5 +26,6 @@ source_sample_rate = 48000
preprocessed = false
asked_start_end = true
asked_fast = true
asked_questions = true
rendered = false
transcoded = []

11
src/main.rs
View file

@ -216,6 +216,9 @@ struct ProjectProgress {
#[serde(default)]
asked_questions: bool,
#[serde(default)]
rendered_assets: bool,
#[serde(default)]
rendered: bool,
@ -414,6 +417,14 @@ fn main() {
fs::write(&project_path, toml::to_string(&project).unwrap().as_bytes()).unwrap();
}
// render the assets
if !project.progress.rendered_assets {
renderer.render_assets(&project).unwrap();
project.progress.rendered_assets = true;
fs::write(&project_path, toml::to_string(&project).unwrap().as_bytes()).unwrap();
}
// render the video
let mut videos = Vec::new();
videos.push(if project.progress.rendered {

10
src/question.rs
View file

@ -13,7 +13,7 @@ pub(crate) struct Question {
}
impl Question {
pub(crate) fn new(res: Resolution, lang: &Language<'_>, str: String) -> Self {
pub(crate) fn new(res: Resolution, lang: &Language<'_>, str: &str) -> Self {
static FONT: OnceLock<Owned<Font<'static>>> = OnceLock::new();
let font = FONT.get_or_init(|| {
let fc = Fontconfig::new().unwrap();
@ -134,14 +134,6 @@ impl Question {
svg.set_width(self.res.width());
svg.set_height(self.res.height());
svg.set_view_box("0 0 1920 1080");
svg.push(
Rect::new()
.with_fill("#040")
.with_x(0)
.with_y(0)
.with_width(1920)
.with_height(1080)
);
svg.push(self.g);
svg
}

49
src/render/filter.rs
View file

@ -19,6 +19,7 @@ pub(crate) enum Filter {
overlay_input: Cow<'static, str>,
x: Cow<'static, str>,
y: Cow<'static, str>,
repeatlast: bool,
output: Cow<'static, str>
},
@ -37,6 +38,22 @@ pub(crate) enum Filter {
output: Cow<'static, str>
},
/// Fade only video using the alpha channel.
FadeAlpha {
input: Cow<'static, str>,
direction: &'static str,
start: Time,
duration: Time,
output: Cow<'static, str>
},
/// Offset the PTS of the video by the amount of seconds.
VideoOffset {
input: Cow<'static, str>,
seconds: Time,
output: Cow<'static, str>
},
/// Generate silence. The video is copied.
GenerateSilence {
video: Cow<'static, str>,
@ -77,11 +94,13 @@ impl Filter {
overlay_input,
x,
y,
repeatlast,
output
} => {
let repeatlast: u8 = (*repeatlast).into();
writeln!(
complex,
"{}{}overlay=x={x}:y={y}{};",
"{}{}overlay=x={x}:y={y}:repeatlast={repeatlast}:eval=init{};",
channel('v', video_input),
channel('v', overlay_input),
channel('v', output)
@ -129,6 +148,34 @@ impl Filter {
)?;
},
Self::FadeAlpha {
input,
direction,
start,
duration,
output
} => {
writeln!(
complex,
"{}fade={direction}:st={start}:d={duration}:alpha=1{};",
channel('v', input),
channel('v', output)
)?;
},
Self::VideoOffset {
input,
seconds,
output
} => {
writeln!(
complex,
"{}setpts=PTS+{seconds}/TB{};",
channel('v', input),
channel('v', output)
)?;
},
Self::GenerateSilence { video, output } => {
writeln!(
complex,

162
src/render/mod.rs
View file

@ -7,8 +7,9 @@ use self::{
};
use crate::{
iotro::{intro, outro},
question::Question,
render::ffmpeg::{Ffmpeg, FfmpegInput},
time::{format_date, Time},
time::{format_date, format_time, Time},
Project, ProjectLecture, ProjectSourceMetadata, Resolution
};
use anyhow::{bail, Context};
@ -34,6 +35,10 @@ const TRANSITION_LEN: Time = Time {
seconds: 0,
micros: 200_000
};
const QUESTION_FADE_LEN: Time = Time {
seconds: 0,
micros: 400_000
};
const FF_MULTIPLIER: usize = 8;
// logo sizes at full hd, will be scaled to source resolution
const FF_LOGO_SIZE: usize = 128;
@ -152,14 +157,13 @@ fn svg2mkv(
ffmpeg.run()
}
fn svg2png(svg: &Path, png: &Path, size: usize) -> anyhow::Result<()> {
fn svg2png(svg: &Path, png: &Path, width: usize, height: usize) -> anyhow::Result<()> {
let mut cmd = cmd();
let size = size.to_string();
cmd.arg("inkscape")
.arg("-w")
.arg(&size)
.arg(width.to_string())
.arg("-h")
.arg(&size);
.arg(height.to_string());
cmd.arg(svg).arg("-o").arg(png);
let status = cmd.status()?;
@ -218,6 +222,14 @@ impl<'a> Renderer<'a> {
self.target.join("outro.mkv")
}
fn question_svg(&self, q_idx: usize) -> PathBuf {
self.target.join(format!("question{q_idx}.svg"))
}
fn question_png(&self, q_idx: usize) -> PathBuf {
self.target.join(format!("question{q_idx}.png"))
}
pub(crate) fn preprocess(&self, project: &mut Project) -> anyhow::Result<()> {
assert!(!project.progress.preprocessed);
@ -266,6 +278,12 @@ impl<'a> Renderer<'a> {
source_res,
source_sample_rate
});
Ok(())
}
/// Prepare assets like intro, outro and questions.
pub(crate) fn render_assets(&self, project: &Project) -> anyhow::Result<()> {
let metadata = project.source.metadata.as_ref().unwrap();
println!();
@ -279,7 +297,7 @@ impl<'a> Renderer<'a> {
let intro_svg = self.target.join("intro.svg");
fs::write(
&intro_svg,
intro(source_res, &project.lecture)
intro(metadata.source_res, &project.lecture)
.to_string_pretty()
.into_bytes()
)?;
@ -290,7 +308,7 @@ impl<'a> Renderer<'a> {
let outro_svg = self.target.join("outro.svg");
fs::write(
&outro_svg,
outro(&project.lecture.lang, source_res)
outro(&project.lecture.lang, metadata.source_res)
.to_string_pretty()
.into_bytes()
)?;
@ -304,7 +322,8 @@ impl<'a> Renderer<'a> {
include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/assets/logo.svg"))
)?;
let logo_png = self.target.join("logo.png");
svg2png(&logo_svg, &logo_png, LOGO_SIZE * source_res.width() / 1920)?;
let logo_size = LOGO_SIZE * metadata.source_res.width() / 1920;
svg2png(&logo_svg, &logo_png, logo_size, logo_size)?;
// copy fastforward then render to png
let fastforward_svg = self.target.join("fastforward.svg");
@ -316,12 +335,31 @@ impl<'a> Renderer<'a> {
))
)?;
let fastforward_png = self.target.join("fastforward.png");
let ff_logo_size = FF_LOGO_SIZE * metadata.source_res.width() / 1920;
svg2png(
&fastforward_svg,
&fastforward_png,
FF_LOGO_SIZE * source_res.width() / 1920
ff_logo_size,
ff_logo_size
)?;
// write questions then render to png
for (q_idx, (_, _, q_text)) in project.source.questions.iter().enumerate() {
let q = Question::new(metadata.source_res, &project.lecture.lang, q_text)
.finish()
.to_string_pretty()
.into_bytes();
let q_svg = self.question_svg(q_idx);
let q_png = self.question_png(q_idx);
fs::write(&q_svg, q)?;
svg2png(
&q_svg,
&q_png,
metadata.source_res.width(),
metadata.source_res.height()
)?;
}
Ok(())
}
@ -361,21 +399,24 @@ impl<'a> Renderer<'a> {
let mut part3: Cow<'static, str> = outro.into();
// the recording is fun because of all the fast forwarding
let mut part2 = VecDeque::new();
let mut part2 = VecDeque::<Cow<'static, str>>::new();
let mut part2_ts = VecDeque::new();
let mut part2_start_of_the_end = None;
let mut part2_end_of_the_start = None;
// ok so ff is fun. we will add the ff'ed section as well as the part between
// the previous ff'ed section and our new section, unless we are the first
// the previous ff'ed section and our new section, unless we are the first.
project.source.fast.sort();
for (i, (ff_st, ff_end)) in project.source.fast.iter().rev().enumerate() {
if let Some(prev_end) = part2_end_of_the_start {
let duration = prev_end - *ff_end;
let recffbetween = ffmpeg.add_input(FfmpegInput {
start: Some(*ff_end),
duration: Some(prev_end - *ff_end),
duration: Some(duration),
..FfmpegInput::new(rec_file.clone())
});
part2.push_front(recffbetween.into());
part2_ts.push_front(Some((*ff_end, duration)));
} else {
part2_start_of_the_end = Some(*ff_end);
}
@ -395,6 +436,7 @@ impl<'a> Renderer<'a> {
output: recff.clone().into()
});
part2.push_front(recff.into());
part2_ts.push_front(None);
}
// if the recording was not ff'ed, perform a normal trim
@ -409,23 +451,112 @@ impl<'a> Renderer<'a> {
..FfmpegInput::new(rec_file.clone())
});
part2.push_back(rectrim.into());
part2_ts.push_back(Some((start, part2_last_part_duration)));
}
// otherwise add the first and last parts separately
else {
let duration = part2_end_of_the_start.unwrap() - start;
let rectrimst = ffmpeg.add_input(FfmpegInput {
start: Some(start),
duration: Some(part2_end_of_the_start.unwrap() - start),
duration: Some(duration),
..FfmpegInput::new(rec_file.clone())
});
part2.push_front(rectrimst.into());
part2_ts.push_front(Some((start, duration)));
part2_last_part_duration = end - part2_start_of_the_end.unwrap();
let part2_start_of_the_end = part2_start_of_the_end.unwrap();
part2_last_part_duration = end - part2_start_of_the_end;
let rectrimend = ffmpeg.add_input(FfmpegInput {
start: Some(part2_start_of_the_end.unwrap()),
start: Some(part2_start_of_the_end),
duration: Some(part2_last_part_duration),
..FfmpegInput::new(rec_file.clone())
});
part2.push_back(rectrimend.into());
part2_ts.push_back(Some((part2_start_of_the_end, part2_last_part_duration)));
}
// ok now we have a bunch of parts and a bunch of questions that want to get
// overlayed over those parts.
project.source.questions.sort();
let mut q_idx = 0;
for (i, ts) in part2_ts.iter().enumerate() {
let Some((start, duration)) = ts else {
continue;
};
loop {
if q_idx >= project.source.questions.len() {
break;
}
let (q_start, q_end, _) = &project.source.questions[q_idx];
if q_start < start {
bail!(
"Question starting at {} did not fit into the video",
format_time(*q_start)
);
}
if q_start >= start && *q_end <= *start + *duration {
// add the question as input to ffmpeg
let q_inp = ffmpeg.add_input(FfmpegInput {
loop_input: true,
fps: Some(project.source.metadata.as_ref().unwrap().source_fps),
duration: Some(*q_end - *q_start),
..FfmpegInput::new(self.question_png(q_idx))
});
// fade in the question
let q_fadein = format!("q{q_idx}fin");
ffmpeg.add_filter(Filter::FadeAlpha {
input: q_inp.into(),
direction: "in",
start: Time {
seconds: 0,
micros: 0
},
duration: QUESTION_FADE_LEN,
output: q_fadein.clone().into()
});
// fade out the question
let q_fadeout = format!("q{q_idx}fout");
ffmpeg.add_filter(Filter::FadeAlpha {
input: q_fadein.into(),
direction: "out",
start: *q_end - *q_start - QUESTION_FADE_LEN,
duration: QUESTION_FADE_LEN,
output: q_fadeout.clone().into()
});
// move the question to the correct timestamp
let q_pts = format!("q{q_idx}pts");
ffmpeg.add_filter(Filter::VideoOffset {
input: q_fadeout.into(),
seconds: *q_start - *start,
output: q_pts.clone().into()
});
// overlay the part in question
let q_overlay = format!("q{q_idx}o");
ffmpeg.add_filter(Filter::Overlay {
video_input: part2[i].clone(),
overlay_input: q_pts.into(),
x: "0".into(),
y: "0".into(),
repeatlast: false,
output: q_overlay.clone().into()
});
part2[i] = q_overlay.into();
q_idx += 1;
continue;
}
break;
}
}
if q_idx < project.source.questions.len() {
bail!(
"Question starting at {} did not fit into the video before it was over",
format_time(project.source.questions[q_idx].0)
);
}
// fade out the intro
@ -503,6 +634,7 @@ impl<'a> Renderer<'a> {
overlay_input: logoalpha.into(),
x: format!("main_w-overlay_w-{overlay_off_x}").into(),
y: format!("main_h-overlay_h-{overlay_off_y}").into(),
repeatlast: true,
output: overlay.into()
});

8
tmp.sh
View file

@ -4,11 +4,11 @@ set -euo pipefail
rm tmp.mkv || true
ffmpeg -hide_banner \
-loop 1 -r 25 -t 4 -i question.png \
-loop 1 -r 25 -t 3 -i question.png \
-filter_complex "
gradients=s=2560x1440:d=4:c0=#000055:c1=#005500:x0=480:y0=540:x1=1440:y1=540[input];
[0]fade=t=in:st=0:d=1:alpha=1,fade=t=out:st=3:d=1:alpha=1[overlay];
[input][overlay]overlay=eval=frame:x=0:y=0[v]
gradients=s=2560x1440:d=5:c0=#000055:c1=#005500:x0=480:y0=540:x1=1440:y1=540[input];
[0]fade=t=in:st=1:d=0.5:alpha=1,fade=t=out:st=2.5:d=0.5:alpha=1,setpts=PTS+1/TB[overlay];
[input][overlay]overlay=eval=frame:x=0:y=0:repeatlast=0[v]
" \
-map "[v]" \
-c:v libsvtav1 -preset 1 -crf 18 \