feat: add a/v sync indicator scaffolding + PoC

2024-02-18 18:32:58 +01:00 · 2024-02-18 18:32:58 +01:00 · 33b6bbb216
commit 33b6bbb216
parent ee4673737f
21 changed files with 2494 additions and 0 deletions
--- a/.earthlyignore
+++ b/.earthlyignore
@ -0,0 +1 @@
+*/node_modules
--- a/.gitattributes
+++ b/.gitattributes
@ -0,0 +1 @@
+**/*.wav filter=lfs diff=lfs merge=lfs -text
--- a/31
+++ b/31
@ -1,6 +1,37 @@
 VERSION 0.7
 FROM node:lts

+avsync-video-components:
+    # https://pptr.dev/troubleshooting
+    RUN apt-get update && apt-get -y install libgtk-3-dev libnotify-dev libgconf-2-4 libnss3 libxss1 libasound2 && rm -rf /var/lib/apt/lists/*
+    RUN npm install -g pnpm
+    RUN groupadd -r pptruser && useradd -r -g pptruser -G audio,video pptruser && mkdir /home/pptruser && chown -R pptruser:pptruser /home/pptruser
+    USER pptruser
+    COPY av-sync/package.json av-sync/pnpm-lock.yaml /av-sync
+    WORKDIR /av-sync
+    CACHE /home/pptruser/.local/share/pnpm
+    RUN pnpm install
+    COPY av-sync /av-sync
+    ARG FPS=60
+    ARG CYCLES=16
+    ARG SIZE=1200
+    RUN pnpm serve-render --fps $FPS --cycles 1 --size $SIZE --output frames
+    RUN pnpm render-audio -i beep.wav -o track.wav --repeats $CYCLES
+    SAVE ARTIFACT frames
+    SAVE ARTIFACT track.wav
+
+avsync-video:
+    FROM debian:bookworm
+    RUN apt-get update && apt-get install -y ffmpeg && rm -rf /var/lib/apt/lists/*
+    COPY +avsync-video-components/track.wav /track.wav
+    COPY +avsync-video-components/frames /frames
+    RUN find frames -type f | sort | xargs -I {} sh -c 'echo "file {}" >> /frames.txt'
+    ARG CYCLES=16
+    RUN for i in $(seq 1 $CYCLES); do cat /frames.txt >> /final-frames.txt; done
+    ARG FPS=60
+    RUN ffmpeg -r $FPS -f concat -i /final-frames.txt -i track.wav -c:v libvpx-vp9 -pix_fmt yuva420p -shortest avsync.webm
+    SAVE ARTIFACT avsync.webm
+
 site:
    RUN npm install -g pnpm
    COPY package.json pnpm-lock.yaml /site
--- a/av-sync/.gitignore
+++ b/av-sync/.gitignore
@ -0,0 +1,24 @@
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+pnpm-debug.log*
+lerna-debug.log*
+
+node_modules
+dist
+dist-ssr
+*.local
+
+# Editor directories and files
+.vscode/*
+!.vscode/extensions.json
+.idea
+.DS_Store
+*.suo
+*.ntvs*
+*.njsproj
+*.sln
+*.sw?
--- a/av-sync/beep.wav
+++ b/av-sync/beep.wav
--- a/av-sync/index.html
+++ b/av-sync/index.html
@ -0,0 +1,11 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <title>AV SYNC</title>
+  </head>
+  <body>
+    <div id="app"></div>
+    <script type="module" src="/src/main.ts"></script>
+  </body>
+</html>
--- a/av-sync/package.json
+++ b/av-sync/package.json
@ -0,0 +1,29 @@
+{
+  "name": "av-sync",
+  "private": true,
+  "version": "0.0.0",
+  "type": "module",
+  "scripts": {
+    "dev": "vite",
+    "build": "vite build",
+    "preview": "vite preview",
+    "check": "svelte-check --tsconfig ./tsconfig.json",
+    "render": "node render.js",
+    "serve-render": "concurrently -P -k -s command-1 \"pnpm run dev --port 8626\" \"wait-on http://localhost:8626 && pnpm run render --url http://localhost:8626 {@}\" --",
+    "render-audio": "node render-audio.js"
+  },
+  "dependencies": {
+    "@sveltejs/vite-plugin-svelte": "^3.0.2",
+    "@tsconfig/svelte": "^5.0.2",
+    "commander": "^12.0.0",
+    "concurrently": "^8.2.2",
+    "node-wav": "^0.0.2",
+    "puppeteer": "^22.1.0",
+    "svelte": "^4.2.10",
+    "svelte-check": "^3.6.3",
+    "tslib": "^2.6.2",
+    "typescript": "^5.2.2",
+    "vite": "^5.1.0",
+    "wait-on": "^7.2.0"
+  }
+}
--- a/av-sync/pnpm-lock.yaml
+++ b/av-sync/pnpm-lock.yaml
--- a/av-sync/public/vite.svg
+++ b/av-sync/public/vite.svg
@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" class="iconify iconify--logos" width="31.88" height="32" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 257"><defs><linearGradient id="IconifyId1813088fe1fbc01fb466" x1="-.828%" x2="57.636%" y1="7.652%" y2="78.411%"><stop offset="0%" stop-color="#41D1FF"></stop><stop offset="100%" stop-color="#BD34FE"></stop></linearGradient><linearGradient id="IconifyId1813088fe1fbc01fb467" x1="43.376%" x2="50.316%" y1="2.242%" y2="89.03%"><stop offset="0%" stop-color="#FFEA83"></stop><stop offset="8.333%" stop-color="#FFDD35"></stop><stop offset="100%" stop-color="#FFA800"></stop></linearGradient></defs><path fill="url(#IconifyId1813088fe1fbc01fb466)" d="M255.153 37.938L134.897 252.976c-2.483 4.44-8.862 4.466-11.382.048L.875 37.958c-2.746-4.814 1.371-10.646 6.827-9.67l120.385 21.517a6.537 6.537 0 0 0 2.322-.004l117.867-21.483c5.438-.991 9.574 4.796 6.877 9.62Z"></path><path fill="url(#IconifyId1813088fe1fbc01fb467)" d="M185.432.063L96.44 17.501a3.268 3.268 0 0 0-2.634 3.014l-5.474 92.456a3.268 3.268 0 0 0 3.997 3.378l24.777-5.718c2.318-.535 4.413 1.507 3.936 3.838l-7.361 36.047c-.495 2.426 1.782 4.5 4.151 3.78l15.304-4.649c2.372-.72 4.652 1.36 4.15 3.788l-11.698 56.621c-.732 3.542 3.979 5.473 5.943 2.437l1.313-2.028l72.516-144.72c1.215-2.423-.88-5.186-3.54-4.672l-25.505 4.922c-2.396.462-4.435-1.77-3.759-4.114l16.646-57.705c.677-2.35-1.37-4.583-3.769-4.113Z"></path></svg>
--- a/av-sync/render-audio.js
+++ b/av-sync/render-audio.js
@ -0,0 +1,33 @@
+import { Command } from 'commander';
+import wav from 'node-wav';
+import fs from 'fs';
+
+const program = new Command();
+program
+	.requiredOption('-i, --input <input>', 'Input file')
+	.requiredOption('-o, --output <output>', 'Output file')
+	.requiredOption('--repeats <repeats>', 'Number of repeats')
+	.parse(process.argv);
+const options = program.opts();
+
+let beep = wav.decode(fs.readFileSync(options.input));
+let samples = beep.channelData[0];
+
+const sampleRate = beep.sampleRate;
+const silenceDuration = sampleRate - samples.length;
+const silenceSamples = new Float32Array(silenceDuration).fill(0);
+
+let oneSecondChunk = new Float32Array(sampleRate);
+oneSecondChunk.set(samples, 0);
+oneSecondChunk.set(silenceSamples, samples.length);
+
+let numberOfRepeats = parseInt(options.repeats);
+let finalSamples = new Float32Array(sampleRate * numberOfRepeats);
+
+for (let i = 0; i < numberOfRepeats; i++) {
+	finalSamples.set(oneSecondChunk, i * sampleRate);
+}
+
+let finalBuffer = wav.encode([finalSamples], { sampleRate: sampleRate, float: true, bitDepth: 32 });
+
+fs.writeFileSync(options.output, finalBuffer);
--- a/av-sync/render.js
+++ b/av-sync/render.js
@ -0,0 +1,48 @@
+import { Command } from 'commander';
+import puppeteer from 'puppeteer';
+import fs from 'fs';
+
+const program = new Command();
+program
+	.requiredOption('-o, --output <output>', 'Output directory')
+	.requiredOption('--fps <fps>', 'Frames per second')
+	.requiredOption('--cycles <cycles>', 'Number of cycles')
+	.requiredOption('--size <size>', 'Size of the output in pixels')
+	.requiredOption('--url <url>', 'URL to render')
+	.parse(process.argv);
+const options = program.opts();
+
+// mkdir p output path
+if (!fs.existsSync(options.output)) {
+	fs.mkdirSync(options.output, { recursive: true });
+}
+
+const browser = await puppeteer.launch({
+	args: ['--no-sandbox']
+});
+const page = await browser.newPage();
+
+await page.setViewport({ width: parseInt(options.size, 10), height: parseInt(options.size, 10) });
+await page.goto(options.url);
+
+await page.evaluate(async (fps) => {
+	// @ts-ignore
+	await window.setFps(fps);
+}, options.fps);
+
+const totalFrames = parseInt(options.fps) * parseInt(options.cycles);
+for (let frame = 0; frame < totalFrames; frame++) {
+	let start = Date.now();
+	await page.evaluate(async (n) => {
+		// @ts-ignore
+		await window.setFrame(n);
+	}, frame);
+	const path = `${options.output}/${frame.toString().padStart(Math.log10(totalFrames) + 1, '0')}.png`;
+	await page.screenshot({ path, omitBackground: true });
+	let end = Date.now();
+	console.log(`Captured frame ${frame}/${totalFrames} (took ${end - start}ms)`);
+}
+
+console.log('Done.');
+
+await browser.close();
--- a/av-sync/src/App.svelte
+++ b/av-sync/src/App.svelte
@ -0,0 +1,104 @@
+<script lang="ts">
+	// time in seconds
+	import { onMount, tick } from 'svelte';
+	import SectorIndicator from './components/SectorIndicator.svelte';
+	import FlashIndicator from './components/FlashIndicator.svelte';
+
+	export let frame = 0;
+	export let fps = 60;
+	export let debug = false;
+
+	onMount(() => {
+		window.setFps = async (newFps: number) => {
+			fps = newFps;
+			await tick();
+		};
+
+		window.setFrame = async (frameNumber: number) => {
+			frame = frameNumber;
+			await tick();
+		};
+
+		if (window.location.search.includes('debug')) {
+			debug = true;
+		}
+
+		if (window.location.search.includes('play')) {
+			setInterval(() => {
+				frame++;
+				frame %= fps * 4;
+			}, 1000 / fps);
+		}
+
+		if (window.location.search.includes('frame')) {
+			const frameNumber = parseInt(window.location.search.split('frame=')[1]);
+			if (!isNaN(frameNumber)) {
+				frame = frameNumber;
+			}
+		}
+	});
+</script>
+
+<main class:debug>
+	<div class="cyclic">
+		<div class="circular sector">
+			<SectorIndicator {frame} {fps} />
+		</div>
+
+		<div class="circular flash">
+			<FlashIndicator {frame} {fps} />
+		</div>
+	</div>
+
+	{#if debug}
+		<div class="controls">
+			<input type="range" min="0" max={fps * 4} bind:value={frame} />
+			<div class="label">{frame} ({frame % fps}) / {Math.round((frame / fps) * 100) / 100} s)</div>
+		</div>
+	{/if}
+</main>
+
+<style>
+	main {
+		width: 100vw;
+		height: 100vh;
+
+		color: white;
+		--color-active: red;
+		--color-inactive: white;
+	}
+
+	.circular {
+		width: 25vw;
+		height: 25vw;
+	}
+
+	.cyclic {
+		position: absolute;
+		top: 25%;
+		left: 50%;
+		transform: translate(-50%, -50%);
+		width: 100vw;
+
+		display: flex;
+		justify-content: space-evenly;
+	}
+
+	main.debug {
+		background: black;
+	}
+
+	.controls {
+		position: fixed;
+		bottom: 0;
+		left: 50%;
+		transform: translateX(-50%);
+		width: 80vw;
+
+		text-align: center;
+
+		& input {
+			width: 100%;
+		}
+	}
+</style>
--- a/av-sync/src/app.css
+++ b/av-sync/src/app.css
@ -0,0 +1,3 @@
+html, body {
+  margin: 0;
+}
--- a/av-sync/src/components/FlashIndicator.svelte
+++ b/av-sync/src/components/FlashIndicator.svelte
@ -0,0 +1,33 @@
+<script lang="ts">
+	export let frame: number;
+	export let fps: number;
+
+	let el: SVGSVGElement;
+	$: center = el?.clientWidth / 2;
+	$: radius = center;
+
+	let opacity = 1;
+	$: opacity = ease(1 - ((frame % fps) / fps) * 2);
+
+	function ease(x: number) {
+		x = Math.max(0, Math.min(1, x));
+		return 1 - Math.cos((x * Math.PI) / 2);
+	}
+</script>
+
+<svg class="indicator" bind:this={el} style="--opacity: {opacity}">
+	<circle cx={center} cy={center} r={radius}></circle>
+</svg>
+
+<style>
+	.indicator {
+		width: 100%;
+		height: 100%;
+		transform: rotate(-90deg);
+	}
+
+	circle {
+		fill: var(--color-active);
+		opacity: var(--opacity);
+	}
+</style>
--- a/av-sync/src/components/SectorIndicator.svelte
+++ b/av-sync/src/components/SectorIndicator.svelte
@ -0,0 +1,43 @@
+<script lang="ts">
+	export let frame: number;
+	export let fps: number;
+
+	let el: SVGSVGElement;
+	$: center = el?.clientWidth / 2;
+	$: radius = center;
+	let d = '';
+	let circleOpacity = 1;
+
+	$: {
+		const angle = ((frame / fps) * 360) % 360;
+		const radians = (angle * Math.PI) / 180;
+		const x = center + radius * Math.cos(radians);
+		const y = center + radius * Math.sin(radians);
+		d = `M${center},${center} L${center + radius},${center} A${radius},${radius} 0 ${angle > 180 ? 1 : 0} 1 ${x},${y} Z`;
+
+		const flashFrames = fps / 10;
+		circleOpacity = (flashFrames - (frame % fps)) / flashFrames;
+	}
+</script>
+
+<svg class="indicator" style="--circle-opacity: {circleOpacity}" bind:this={el}>
+	<circle cx={center} cy={center} r={radius}></circle>
+	<path {d}></path>
+</svg>
+
+<style>
+	.indicator {
+		width: 100%;
+		height: 100%;
+		transform: rotate(-90deg);
+	}
+
+	circle {
+		fill: var(--color-active);
+		opacity: var(--circle-opacity);
+	}
+
+	path {
+		fill: var(--color-inactive);
+	}
+</style>
--- a/av-sync/src/main.ts
+++ b/av-sync/src/main.ts
@ -0,0 +1,8 @@
+import './app.css';
+import App from './App.svelte';
+
+const app = new App({
+	target: document.getElementById('app')!
+});
+
+export default app;
--- a/av-sync/src/vite-env.d.ts
+++ b/av-sync/src/vite-env.d.ts
@ -0,0 +1,11 @@
+/// <reference types="svelte" />
+/// <reference types="vite/client" />
+
+declare global {
+	interface Window {
+		setFps: (fps: number) => Promise<void>;
+		setFrame: (frame: number) => Promise<void>;
+	}
+}
+
+export {};
--- a/av-sync/svelte.config.js
+++ b/av-sync/svelte.config.js
@ -0,0 +1,7 @@
+import { vitePreprocess } from '@sveltejs/vite-plugin-svelte'
+
+export default {
+  // Consult https://svelte.dev/docs#compile-time-svelte-preprocess
+  // for more information about preprocessors
+  preprocess: vitePreprocess(),
+}
--- a/av-sync/tsconfig.json
+++ b/av-sync/tsconfig.json
@ -0,0 +1,20 @@
+{
+  "extends": "@tsconfig/svelte/tsconfig.json",
+  "compilerOptions": {
+    "target": "ESNext",
+    "useDefineForClassFields": true,
+    "module": "ESNext",
+    "resolveJsonModule": true,
+    /**
+     * Typecheck JS in `.svelte` and `.js` files by default.
+     * Disable checkJs if you'd like to use dynamic types in JS.
+     * Note that setting allowJs false does not prevent the use
+     * of JS in `.svelte` files.
+     */
+    "allowJs": true,
+    "checkJs": true,
+    "isolatedModules": true
+  },
+  "include": ["src/**/*.ts", "src/**/*.js", "src/**/*.svelte"],
+  "references": [{ "path": "./tsconfig.node.json" }]
+}
--- a/av-sync/tsconfig.node.json
+++ b/av-sync/tsconfig.node.json
@ -0,0 +1,10 @@
+{
+  "compilerOptions": {
+    "composite": true,
+    "skipLibCheck": true,
+    "module": "ESNext",
+    "moduleResolution": "bundler",
+    "strict": true
+  },
+  "include": ["vite.config.ts"]
+}
--- a/av-sync/vite.config.ts
+++ b/av-sync/vite.config.ts
@ -0,0 +1,7 @@
+import { defineConfig } from 'vite'
+import { svelte } from '@sveltejs/vite-plugin-svelte'
+
+// https://vitejs.dev/config/
+export default defineConfig({
+  plugins: [svelte()],
+})
				`@ -0,0 +1 @@`
				`*/.wav filter=lfs diff=lfs merge=lfs -text`
				`@ -0,0 +1 @@`
				<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" class="iconify iconify--logos" width="31.88" height="32" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 257"><defs><linearGradient id="IconifyId1813088fe1fbc01fb466" x1="-.828%" x2="57.636%" y1="7.652%" y2="78.411%"><stop offset="0%" stop-color="#41D1FF"></stop><stop offset="100%" stop-color="#BD34FE"></stop></linearGradient><linearGradient id="IconifyId1813088fe1fbc01fb467" x1="43.376%" x2="50.316%" y1="2.242%" y2="89.03%"><stop offset="0%" stop-color="#FFEA83"></stop><stop offset="8.333%" stop-color="#FFDD35"></stop><stop offset="100%" stop-color="#FFA800"></stop></linearGradient></defs><path fill="url(#IconifyId1813088fe1fbc01fb466)" d="M255.153 37.938L134.897 252.976c-2.483 4.44-8.862 4.466-11.382.048L.875 37.958c-2.746-4.814 1.371-10.646 6.827-9.67l120.385 21.517a6.537 6.537 0 0 0 2.322-.004l117.867-21.483c5.438-.991 9.574 4.796 6.877 9.62Z"></path><path fill="url(#IconifyId1813088fe1fbc01fb467)" d="M185.432.063L96.44 17.501a3.268 3.268 0 0 0-2.634 3.014l-5.474 92.456a3.268 3.268 0 0 0 3.997 3.378l24.777-5.718c2.318-.535 4.413 1.507 3.936 3.838l-7.361 36.047c-.495 2.426 1.782 4.5 4.151 3.78l15.304-4.649c2.372-.72 4.652 1.36 4.15 3.788l-11.698 56.621c-.732 3.542 3.979 5.473 5.943 2.437l1.313-2.028l72.516-144.72c1.215-2.423-.88-5.186-3.54-4.672l-25.505 4.922c-2.396.462-4.435-1.77-3.759-4.114l16.646-57.705c.677-2.35-1.37-4.583-3.769-4.113Z"></path></svg>