forked from vgaNAR6ta/drags-and-nerds
edit: intégration fond animé et réédition lien signal vega
This commit is contained in:
Submodule liquid-shape-distortions_EDIT updated: a30cc65745...562c99b15b
@@ -4,8 +4,25 @@
|
||||
<meta charset="UTF-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>Drags and Nerds #2</title>
|
||||
<!--BACKGROUND_HEAD-->
|
||||
<link rel="stylesheet" href="background_styles.css">
|
||||
<script src="https://cdnjs.cloudflare.com/ajax/libs/dat-gui/0.7.9/dat.gui.min.js"></script>
|
||||
|
||||
<link rel="preconnect" href="https://fonts.googleapis.com">
|
||||
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
|
||||
<link href="https://fonts.googleapis.com/css2?family=Modak&display=swap" rel="stylesheet">
|
||||
|
||||
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.0.0/css/all.min.css" integrity="sha512-9usAa10IRO0HhonpyAIVpjrylPvoDwiPUiKdWk5t3PyolY1cOd4DSE0Ga+ri4AuTroPR5aQvXU9xC6qOPnzFeg==" crossorigin="anonymous" referrerpolicy="no-referrer" />
|
||||
<meta property="og:title" content="Liquid Shape Distortions" />
|
||||
<meta property="og:description" content="psychedelic animation generator; (p)art of your next trip" />
|
||||
<meta property="og:type" content="website" />
|
||||
<meta property="og:url" content="https://collidingscopes.github.io/liquid-shape-distortions" />
|
||||
<meta property="og:image" content="https://collidingscopes.github.io/liquid-shape-distortions/assets/siteOGImage2.png">
|
||||
<meta property="og:image:type" content="image/png" >
|
||||
</head>
|
||||
<body>
|
||||
<!--BACKGROUND-->
|
||||
<canvas id="canvas"></canvas>
|
||||
<h1 class="referenceText">
|
||||
<strong>drags and nerds</strong>
|
||||
<p>
|
||||
@@ -18,5 +35,374 @@
|
||||
</h1>
|
||||
<div id="app"></div>
|
||||
<script type="module" src="/src/main.js"></script>
|
||||
<!--SCRIPT BACKGROUND SHADER-->
|
||||
<script id="vertexShader" type="x-shader/x-vertex">
|
||||
attribute vec2 position;
|
||||
void main() {
|
||||
gl_Position = vec4(position, 0.0, 1.0);
|
||||
}
|
||||
</script>
|
||||
<script id="fragmentShader" type="x-shader/x-fragment">
|
||||
precision highp float;
|
||||
|
||||
uniform vec2 resolution;
|
||||
uniform float time;
|
||||
uniform float seed;
|
||||
|
||||
// GUI-controlled parameters
|
||||
uniform float timeScale;
|
||||
uniform float patternAmp;
|
||||
uniform float patternFreq;
|
||||
uniform float bloomStrength;
|
||||
uniform float saturation;
|
||||
uniform float grainAmount;
|
||||
uniform vec3 colorTint;
|
||||
uniform float minCircleSize;
|
||||
uniform float circleStrength;
|
||||
uniform float distortX;
|
||||
uniform float distortY;
|
||||
|
||||
// Noise functions for 3D simplex noise
|
||||
vec3 mod289(vec3 x) { return x - floor(x * (1.0 / 289.0)) * 289.0; }
|
||||
vec4 mod289(vec4 x) { return x - floor(x * (1.0 / 289.0)) * 289.0; }
|
||||
vec4 permute(vec4 x) { return mod289(((x*34.0)+1.0)*x); }
|
||||
vec4 taylorInvSqrt(vec4 r) { return 1.79284291400159 - 0.85373472095314 * r; }
|
||||
|
||||
// Random function that uses the seed
|
||||
float rand(vec3 co) {
|
||||
return fract(sin(dot(co.xyz + vec3(seed * 0.1), vec3(12.9898, 78.233, 53.539))) * 43758.5453);
|
||||
}
|
||||
|
||||
// Pseudo-random function for noise generation
|
||||
float random(vec2 st) {
|
||||
return fract(sin(dot(st.xy, vec2(12.9898, 78.233))) * 43758.5453123);
|
||||
}
|
||||
|
||||
// 3D Simplex noise implementation
|
||||
float snoise3D(vec3 v) {
|
||||
const vec2 C = vec2(1.0/6.0, 1.0/3.0);
|
||||
const vec4 D = vec4(0.0, 0.5, 1.0, 2.0);
|
||||
|
||||
// First corner
|
||||
vec3 i = floor(v + dot(v, C.yyy));
|
||||
vec3 x0 = v - i + dot(i, C.xxx);
|
||||
|
||||
// Other corners
|
||||
vec3 g = step(x0.yzx, x0.xyz);
|
||||
vec3 l = 1.0 - g;
|
||||
vec3 i1 = min(g.xyz, l.zxy);
|
||||
vec3 i2 = max(g.xyz, l.zxy);
|
||||
|
||||
vec3 x1 = x0 - i1 + C.xxx;
|
||||
vec3 x2 = x0 - i2 + C.yyy;
|
||||
vec3 x3 = x0 - D.yyy;
|
||||
|
||||
// Permutations
|
||||
i = mod289(i);
|
||||
vec4 p = permute(permute(permute(
|
||||
i.z + vec4(0.0, i1.z, i2.z, 1.0))
|
||||
+ i.y + vec4(0.0, i1.y, i2.y, 1.0))
|
||||
+ i.x + vec4(0.0, i1.x, i2.x, 1.0));
|
||||
|
||||
// Gradients: 7x7 points over a square, mapped onto an octahedron.
|
||||
// The ring size 17*17 = 289 is close to a multiple of 49 (49*6 = 294)
|
||||
float n_ = 0.142857142857; // 1.0/7.0
|
||||
vec3 ns = n_ * D.wyz - D.xzx;
|
||||
|
||||
vec4 j = p - 49.0 * floor(p * ns.z * ns.z);
|
||||
|
||||
vec4 x_ = floor(j * ns.z);
|
||||
vec4 y_ = floor(j - 7.0 * x_);
|
||||
|
||||
vec4 x = x_ *ns.x + ns.yyyy;
|
||||
vec4 y = y_ *ns.x + ns.yyyy;
|
||||
vec4 h = 1.0 - abs(x) - abs(y);
|
||||
|
||||
vec4 b0 = vec4(x.xy, y.xy);
|
||||
vec4 b1 = vec4(x.zw, y.zw);
|
||||
|
||||
vec4 s0 = floor(b0)*2.0 + 1.0;
|
||||
vec4 s1 = floor(b1)*2.0 + 1.0;
|
||||
vec4 sh = -step(h, vec4(0.0));
|
||||
|
||||
vec4 a0 = b0.xzyw + s0.xzyw*sh.xxyy;
|
||||
vec4 a1 = b1.xzyw + s1.xzyw*sh.zzww;
|
||||
|
||||
vec3 p0 = vec3(a0.xy, h.x);
|
||||
vec3 p1 = vec3(a0.zw, h.y);
|
||||
vec3 p2 = vec3(a1.xy, h.z);
|
||||
vec3 p3 = vec3(a1.zw, h.w);
|
||||
|
||||
// Normalise gradients
|
||||
vec4 norm = taylorInvSqrt(vec4(dot(p0, p0), dot(p1, p1), dot(p2, p2), dot(p3, p3)));
|
||||
p0 *= norm.x;
|
||||
p1 *= norm.y;
|
||||
p2 *= norm.z;
|
||||
p3 *= norm.w;
|
||||
|
||||
// Mix final noise value
|
||||
vec4 m = max(0.6 - vec4(dot(x0, x0), dot(x1, x1), dot(x2, x2), dot(x3, x3)), 0.0);
|
||||
m = m * m;
|
||||
return 42.0 * dot(m*m, vec4(dot(p0, x0), dot(p1, x1), dot(p2, x2), dot(p3, x3)));
|
||||
}
|
||||
|
||||
// Modified fbm function to use 3D noise
|
||||
float fbm3D(vec3 p) {
|
||||
float sum = 0.0;
|
||||
float amp = patternAmp;
|
||||
float freq = patternFreq;
|
||||
|
||||
// Create seed-based offsets using prime multipliers
|
||||
vec3 seedOffset = vec3(
|
||||
sin(seed * 0.731) * cos(seed * 0.293) * 1.0,
|
||||
cos(seed * 0.897) * sin(seed * 0.413) * 1.0,
|
||||
sin(seed * 0.529) * cos(seed * 0.671) * 1.0
|
||||
);
|
||||
|
||||
// Use octaves with better frequency scaling
|
||||
for(int i = 0; i < 2; i++) {
|
||||
// Create unique rotation for each octave to break grid patterns
|
||||
float angle = seed * 0.1 + float(i) * 0.01;
|
||||
mat2 rotation = mat2(
|
||||
cos(angle), -sin(angle),
|
||||
sin(angle), cos(angle)
|
||||
);
|
||||
|
||||
// Rotate coordinates slightly for each octave
|
||||
vec2 rotatedP = rotation * p.xy;
|
||||
vec3 rotated3D = vec3(rotatedP, p.z);
|
||||
|
||||
// Use prime-number-based offsets to avoid repeating patterns
|
||||
vec3 octaveOffset = seedOffset + vec3(
|
||||
sin(float(i) * 1.731 + seed * 0.47),
|
||||
cos(float(i) * 1.293 + seed * 0.83),
|
||||
sin(float(i) * 1.453 + seed * 0.61)
|
||||
);
|
||||
|
||||
// Apply progressive domain warping for more organic results
|
||||
vec3 warpedP = rotated3D + octaveOffset;
|
||||
if (i > 0) {
|
||||
// Add slight domain warping based on previous octave
|
||||
warpedP += vec3(
|
||||
sin(sum * 2.14 + warpedP.y * 1.5),
|
||||
cos(sum * 1.71 + warpedP.x * 1.5),
|
||||
sin(sum * 1.93 + warpedP.z * 1.5)
|
||||
) * 0.1 * float(i);
|
||||
}
|
||||
|
||||
// Add contribution from this octave
|
||||
sum += amp * snoise3D(warpedP * freq);
|
||||
|
||||
// Use better persistence values (slower amplitude reduction)
|
||||
amp *= 0.8;
|
||||
|
||||
// Use better lacunarity values (more moderate frequency increase)
|
||||
freq *= 0.8;
|
||||
}
|
||||
|
||||
// Normalize and add slight contrast adjustment
|
||||
return sum * 0.5 + 0.5;
|
||||
}
|
||||
|
||||
void main() {
|
||||
vec2 uv = gl_FragCoord.xy / resolution.xy;
|
||||
|
||||
// Adjust aspect ratio
|
||||
uv.x *= resolution.x / resolution.y;
|
||||
|
||||
// Apply timeScale from GUI
|
||||
float slowTime = time * timeScale;
|
||||
|
||||
// Create seed-influenced flow vectors for different pattern on each refresh
|
||||
vec2 flow1 = vec2(
|
||||
sin(slowTime * 0.25 + seed * 0.42) * 0.3 + sin(slowTime * 0.14 + seed * 0.23) * 0.2,
|
||||
cos(slowTime * 0.22 + seed * 0.31) * 0.3 + cos(slowTime * 0.11 + seed * 0.17) * 0.2
|
||||
);
|
||||
|
||||
vec2 flow2 = vec2(
|
||||
sin(slowTime * 0.16 + 1.7 + seed * 0.13) * 0.4 + sin(slowTime * 0.18 + seed * 0.29) * 0.1,
|
||||
cos(slowTime * 0.19 + 2.3 + seed * 0.19) * 0.4 + cos(slowTime * 0.11 + seed * 0.33) * 0.1
|
||||
);
|
||||
|
||||
vec2 flow3 = vec2(
|
||||
sin(slowTime * 0.13 + 3.4 + seed * 0.25) * 0.25 + sin(slowTime * 0.22 + seed * 0.11) * 0.15,
|
||||
cos(slowTime * 0.18 + 1.2 + seed * 0.37) * 0.25 + cos(slowTime * 0.25 + seed * 0.27) * 0.15
|
||||
);
|
||||
|
||||
float noiseScale1 = 10000.0;
|
||||
|
||||
// Main light layer with enhanced 3D liquid motion - using 3D noise
|
||||
// The third component determines how the pattern changes over time
|
||||
float timeComponent = slowTime * 5.5 + sin(seed * 0.63) * 0.2;
|
||||
float lightPattern = fbm3D(vec3(uv * noiseScale1, timeComponent));
|
||||
|
||||
float combinedPattern = lightPattern * 0.25;
|
||||
|
||||
// Start with a seed-influenced base color
|
||||
vec3 baseColor = vec3(
|
||||
0.6 + sin(seed * 0.4) * 0.1,
|
||||
0.9 + cos(seed * 0.3) * 0.05,
|
||||
0.92 + sin(seed * 0.5) * 0.05
|
||||
);
|
||||
|
||||
// Define pastel colors with slight seed-based variations
|
||||
float colorSeed1 = sin(seed * 0.73) * 0.88;
|
||||
float colorSeed2 = cos(seed * 0.51) * 0.28;
|
||||
float colorSeed3 = sin(seed * 0.92) * 0.48;
|
||||
|
||||
vec3 pastelGreen = vec3(0.85 + colorSeed1, 0.95 + colorSeed2, 0.85 + colorSeed3);
|
||||
vec3 pastelBlue = vec3(0.85 + colorSeed3, 0.9 + colorSeed1, 0.98 + colorSeed2);
|
||||
vec3 pastelPink = vec3(0.98 + colorSeed2, 0.88 + colorSeed3, 0.92 + colorSeed1);
|
||||
vec3 pastelYellow = vec3(0.98 + colorSeed3, 0.95 + colorSeed1, 0.85 + colorSeed2);
|
||||
vec3 brightPink = vec3(0.98 + colorSeed1, 0.85 + colorSeed2, 0.92 + colorSeed3);
|
||||
vec3 pastelLavender = vec3(0.92 + colorSeed2, 0.88 + colorSeed3, 0.98 + colorSeed1);
|
||||
vec3 pastelPeach = vec3(0.98 + colorSeed3, 0.92 + colorSeed1, 0.87 + colorSeed2);
|
||||
vec3 pastelTeal = vec3(0.85 + colorSeed1, 0.95 + colorSeed2, 0.95 + colorSeed3);
|
||||
vec3 pastelCoral = vec3(0.98 + colorSeed2, 0.88 + colorSeed1, 0.85 + colorSeed3);
|
||||
vec3 pastelMint = vec3(0.88 + colorSeed3, 0.98 + colorSeed2, 0.91 + colorSeed1);
|
||||
vec3 pastelLilac = vec3(0.91 + colorSeed1, 0.85 + colorSeed3, 0.98 + colorSeed2);
|
||||
vec3 pastelSkyBlue = vec3(0.85 + colorSeed2, 0.91 + colorSeed1, 0.98 + colorSeed3);
|
||||
|
||||
// Using larger color patches with 3D liquid-like noise and seed influence
|
||||
float verticalFlow = sin(uv.y * 3.0 + slowTime * 0.2 + seed * 0.4) * 1.0;
|
||||
|
||||
// Creating more complex flow patterns for liquid movement
|
||||
vec2 liquidFlow1 = flow1 + vec2(verticalFlow, sin(uv.x * 2.5 + slowTime * 0.10 + seed * 0.3) * 0.2);
|
||||
vec2 liquidFlow2 = flow2 + vec2(cos(uv.y * 2.2 + slowTime * 0.05 + seed * 0.5) * 0.15, verticalFlow);
|
||||
vec2 liquidFlow3 = flow3 + vec2(verticalFlow * 0.5, cos(uv.x * 1.8 - slowTime * 0.07 + seed * 0.7) * 0.25);
|
||||
|
||||
// Add seed-dependent scale factors for noise
|
||||
float noiseSeedFactor1 = 0.15 * (1.0 + sin(seed * 0.3) * 0.3);
|
||||
float noiseSeedFactor2 = 0.2 * (1.0 + cos(seed * 0.5) * 0.3);
|
||||
float noiseSeedFactor3 = 0.12 * (1.0 + sin(seed * 0.7) * 0.3);
|
||||
|
||||
// Using 3D noise for color patterns with different time components for variety
|
||||
float colorNoise1 = fbm3D(vec3(uv * noiseSeedFactor1 + liquidFlow1 * 0.03, slowTime * 0.02 + seed * 0.27));
|
||||
|
||||
// Adjust thresholds with seed influence for variety
|
||||
float threshSeed1 = 0.0 + sin(seed * 0.4) * 0.15;
|
||||
float threshSeed2 = 0.15 + cos(seed * 0.6) * 0.15;
|
||||
float threshSeed3 = 0.25 + sin(seed * 0.8) * 0.15;
|
||||
|
||||
float colorMixValue1 = smoothstep(0.0, threshSeed1, colorNoise1);
|
||||
float colorMixValue2 = smoothstep(0.0, threshSeed2, colorNoise1);
|
||||
float colorMixValue3 = smoothstep(0.0, threshSeed3, colorNoise1);
|
||||
|
||||
// Start with base color and mix in expanded pastel palette
|
||||
vec3 colorVariation = baseColor;
|
||||
|
||||
// Layer 1: Primary colors with seed-influenced mix factors
|
||||
float mixFactor1 = 2.2 + sin(seed * 0.3) * 0.5;
|
||||
float mixFactor2 = 2.0 + cos(seed * 0.5) * 0.5;
|
||||
float mixFactor3 = 2.0 + sin(seed * 0.7) * 0.5;
|
||||
float mixFactor4 = 2.0 + cos(seed * 0.9) * 0.5;
|
||||
|
||||
colorVariation = mix(colorVariation, pastelBlue, colorMixValue1 * mixFactor1);
|
||||
colorVariation = mix(colorVariation, pastelPink, (1.0 - colorMixValue1) * colorMixValue2 * mixFactor2);
|
||||
colorVariation = mix(colorVariation, pastelGreen, colorMixValue2 * (1.0 - colorMixValue1) * mixFactor3);
|
||||
colorVariation = mix(colorVariation, pastelYellow, (1.0 - colorMixValue2) * colorMixValue1 * mixFactor4);
|
||||
|
||||
// Layer 2: New pastel colors with seed-influenced mix factors
|
||||
float mixFactor5 = 1.8 + sin(seed * 1.1) * 0.4;
|
||||
float mixFactor6 = 1.8 + cos(seed * 1.3) * 0.4;
|
||||
float mixFactor7 = 1.7 + sin(seed * 1.5) * 0.4;
|
||||
float mixFactor8 = 1.6 + cos(seed * 1.7) * 0.4;
|
||||
float mixFactor9 = 1.5 + sin(seed * 1.9) * 0.4;
|
||||
|
||||
colorVariation = mix(colorVariation, brightPink, (colorMixValue1 * colorMixValue2) * mixFactor5);
|
||||
colorVariation = mix(colorVariation, pastelLavender, (1.0 - colorMixValue3) * colorMixValue1 * mixFactor6);
|
||||
colorVariation = mix(colorVariation, pastelPeach, colorMixValue3 * (1.0 - colorMixValue2) * mixFactor7);
|
||||
colorVariation = mix(colorVariation, pastelTeal, (colorMixValue2 * colorMixValue3) * mixFactor8);
|
||||
colorVariation = mix(colorVariation, pastelCoral, ((1.0 - colorMixValue1) * colorMixValue3) * mixFactor9);
|
||||
|
||||
// Layer 3: Additional colors with seed-influenced noise combinations
|
||||
float seedOffset1 = sin(seed * 2.1) * 0.05;
|
||||
float seedOffset2 = cos(seed * 2.3) * 0.05;
|
||||
|
||||
float mixValue4 = smoothstep(0.0, 1.0, float(uv * (0.18 + seedOffset1)));
|
||||
float mixValue5 = mixValue4 - (sin(seed*0.6)*0.4);
|
||||
|
||||
float mixFactor10 = 1.7 + sin(seed * 2.5) * 0.4;
|
||||
float mixFactor11 = 1.8 + cos(seed * 2.7) * 0.4;
|
||||
float mixFactor12 = 1.5 + sin(seed * 2.9) * 0.4;
|
||||
|
||||
colorVariation = mix(colorVariation, pastelMint, mixValue4 * (1.0 - mixValue5) * mixFactor10);
|
||||
colorVariation = mix(colorVariation, pastelLilac, (1.0 - mixValue4) * mixValue5 * mixFactor11);
|
||||
colorVariation = mix(colorVariation, pastelSkyBlue, mixValue4 * mixValue5 * mixFactor12);
|
||||
|
||||
// Adjust pattern brightness with seed influence
|
||||
float brightnessFactor = 1.0 + sin(seed * 0.5) * 2.1;
|
||||
combinedPattern = pow(combinedPattern * 0.2 + 0.8, brightnessFactor);
|
||||
|
||||
// Create light spots with seed-influenced threshold
|
||||
float lightThreshold = 1.0 + sin(seed * 0.6) * 0.8;
|
||||
float lightSpots = smoothstep(0.0, lightThreshold, combinedPattern);
|
||||
|
||||
// Enhanced circular light patterns with seed-influenced distortion
|
||||
float distortionAmount = 1.0 + cos(seed * 0.7) * 5.05;
|
||||
float distortion = sin(slowTime * 0.1 + seed) * distortionAmount;
|
||||
|
||||
vec2 distortedUV = fract(uv * 1.0 + vec2(
|
||||
sin(uv.y * 2.0 + slowTime * 0.15 + seed * 0.8) * distortY,
|
||||
cos(uv.x * 1.8 + slowTime * 0.1 + seed * 0.9) * distortX
|
||||
));
|
||||
|
||||
// Adjust circular spots with seed influence
|
||||
float circleSize = minCircleSize + sin(seed) * 1.3;
|
||||
float circleThreshold = 0.0 + cos(seed * 1.3) * 2.05;
|
||||
float circularSpots = smoothstep(0.0, circleThreshold, 1.0 - length((distortedUV - 0.5) * (circleSize + distortion)));
|
||||
|
||||
// Mix with liquid movement
|
||||
float mixRatio = 0.0 + sin(seed * 1.5) * circleStrength;
|
||||
lightSpots = mix(lightSpots, circularSpots * lightSpots, mixRatio);
|
||||
|
||||
// Apply liquid-like diffusion with seed influence and 3D noise
|
||||
float diffusionScale = 0.0 + cos(seed * 1.7) * 5.0;
|
||||
float diffusedLightSpots = fbm3D(vec3(
|
||||
uv * diffusionScale + vec2(
|
||||
sin(slowTime * 0.03 + uv.x + seed * 1.9) * 0.2,
|
||||
cos(slowTime * 0.02 + uv.y + seed * 2.1) * 0.2
|
||||
),
|
||||
slowTime * 0.05 + sin(seed * 0.76) * 0.5
|
||||
)) * lightSpots;
|
||||
|
||||
// Mix diffusion with seed influence
|
||||
//float diffusionMix = 0.7 + sin(seed * 2.3) * 0.1;
|
||||
float diffusionMix = 1.0;
|
||||
lightSpots = mix(lightSpots, diffusedLightSpots, diffusionMix);
|
||||
|
||||
// Final pattern mix
|
||||
float patternMix = 1.0;
|
||||
combinedPattern = mix(combinedPattern, lightSpots, patternMix);
|
||||
|
||||
float finalValue = combinedPattern;
|
||||
|
||||
// Use the color variation and apply color tint from GUI
|
||||
vec3 color = finalValue * colorVariation * colorTint;
|
||||
|
||||
// Bloom with GUI-controlled bloomStrength
|
||||
float bloomThreshold = 1.0;
|
||||
float bloom = smoothstep(0.0, bloomThreshold, finalValue) * bloomStrength;
|
||||
color += bloom;
|
||||
|
||||
// Grain with GUI-controlled grainAmount
|
||||
vec2 noiseCoord = uv;
|
||||
float noise = random(noiseCoord + time * 0.0015) * grainAmount;
|
||||
color = color + vec3(noise);
|
||||
|
||||
// Saturation with GUI control
|
||||
float luminance = dot(color, vec3(0.299, 0.587, 0.114));
|
||||
vec3 saturatedColor = mix(vec3(luminance), color, saturation);
|
||||
float satMix = 1.0;
|
||||
color = mix(color, saturatedColor, satMix);
|
||||
|
||||
gl_FragColor = vec4(color, 1.0);
|
||||
}
|
||||
</script>
|
||||
</body>
|
||||
<!--SCRIPT BACKGROUND SETUP-->
|
||||
<script src="mp4-muxer-main/build/mp4-muxer.js"></script>
|
||||
<script src="helperFunctions.js"></script>
|
||||
<script src="canvasVideoExport.js"></script>
|
||||
<script src="main.js"></script>
|
||||
</html>
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2025 Alan Ang
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
@@ -0,0 +1,29 @@
|
||||
html, body{
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
inset: 0;
|
||||
position: absolute;
|
||||
z-index: 0;
|
||||
background-color: #000000;
|
||||
height: 100%;
|
||||
width: 100%;
|
||||
overflow: hidden;
|
||||
display: block;
|
||||
}
|
||||
|
||||
canvas {
|
||||
display: block;
|
||||
max-width: 100%;
|
||||
margin: 0 auto;
|
||||
padding: 0;
|
||||
/* margin-top: 0vh; */
|
||||
text-align: center;
|
||||
/* height: 100vh; */
|
||||
}
|
||||
|
||||
|
||||
.dg{
|
||||
display: none !important;
|
||||
height: 0;
|
||||
overflow: hidden;
|
||||
}
|
||||
@@ -0,0 +1,265 @@
|
||||
let projectName = "komorebi"; //to be updated
|
||||
|
||||
//detect user browser
|
||||
var ua = navigator.userAgent;
|
||||
var isSafari = false;
|
||||
var isFirefox = false;
|
||||
var isIOS = false;
|
||||
var isAndroid = false;
|
||||
if(ua.includes("Safari")){
|
||||
isSafari = true;
|
||||
}
|
||||
if(ua.includes("Firefox")){
|
||||
isFirefox = true;
|
||||
}
|
||||
if(ua.includes("iPhone") || ua.includes("iPad") || ua.includes("iPod")){
|
||||
isIOS = true;
|
||||
}
|
||||
if(ua.includes("Android")){
|
||||
isAndroid = true;
|
||||
}
|
||||
console.log("isSafari: "+isSafari+", isFirefox: "+isFirefox+", isIOS: "+isIOS+", isAndroid: "+isAndroid);
|
||||
|
||||
let useMobileRecord = false;
|
||||
if(isIOS || isAndroid || isFirefox){
|
||||
useMobileRecord = true;
|
||||
}
|
||||
|
||||
var mediaRecorder;
|
||||
var recordedChunks;
|
||||
var finishedBlob;
|
||||
var recordingMessageDiv = document.getElementById("videoRecordingMessageDiv");
|
||||
var recordVideoState = false;
|
||||
var videoRecordInterval;
|
||||
var videoEncoder;
|
||||
var muxer;
|
||||
var mobileRecorder;
|
||||
var videofps = 30;
|
||||
let bitrate = 16_000_000;
|
||||
|
||||
function saveImage() {
|
||||
console.log("Export png image");
|
||||
|
||||
// Create a temporary canvas with the same dimensions
|
||||
const tempCanvas = document.createElement('canvas');
|
||||
tempCanvas.width = canvas.width;
|
||||
tempCanvas.height = canvas.height;
|
||||
const tempContext = tempCanvas.getContext('2d', {
|
||||
willReadFrequently: true,
|
||||
alpha: true // Enable alpha for transparency
|
||||
});
|
||||
|
||||
// Skip filling the background, leaving it transparent
|
||||
|
||||
// Force a render frame to ensure latest content
|
||||
drawScene();
|
||||
gl.flush();
|
||||
gl.finish();
|
||||
|
||||
// Draw the WebGL canvas onto the temporary canvas
|
||||
tempContext.drawImage(canvas, 0, 0);
|
||||
|
||||
// Create download link
|
||||
const link = document.createElement('a');
|
||||
link.href = tempCanvas.toDataURL('image/png');
|
||||
|
||||
const date = new Date();
|
||||
const filename = projectName + `_${date.toLocaleDateString()}_${date.toLocaleTimeString()}.png`;
|
||||
link.download = filename;
|
||||
link.click();
|
||||
|
||||
// Cleanup
|
||||
tempCanvas.remove();
|
||||
}
|
||||
|
||||
function toggleVideoRecord(){
|
||||
|
||||
if(recordVideoState == false){
|
||||
recordVideoState = true;
|
||||
chooseRecordingFunction();
|
||||
} else {
|
||||
recordVideoState = false;
|
||||
chooseEndRecordingFunction();
|
||||
}
|
||||
}
|
||||
|
||||
function chooseRecordingFunction(){
|
||||
//resetAnimation();
|
||||
if(useMobileRecord){
|
||||
startMobileRecording();
|
||||
}else {
|
||||
recordVideoMuxer();
|
||||
}
|
||||
}
|
||||
|
||||
function chooseEndRecordingFunction(){
|
||||
if(useMobileRecord){
|
||||
mobileRecorder.stop();
|
||||
}else {
|
||||
finalizeVideo();
|
||||
}
|
||||
}
|
||||
|
||||
//record html canvas element and export as mp4 video
|
||||
//source: https://devtails.xyz/adam/how-to-save-html-canvas-to-mp4-using-web-codecs-api
|
||||
async function recordVideoMuxer() {
|
||||
console.log("start muxer video recording");
|
||||
|
||||
var videoWidth = Math.floor(canvas.width/2)*2;
|
||||
var videoHeight = Math.floor(canvas.height/4)*4; //force a number which is divisible by 4
|
||||
|
||||
console.log("Video dimensions: "+videoWidth+", "+videoHeight);
|
||||
|
||||
//display user message
|
||||
recordingMessageDiv.classList.remove("hidden");
|
||||
|
||||
recordVideoState = true;
|
||||
const ctx = canvas.getContext("2d", {
|
||||
// This forces the use of a software (instead of hardware accelerated) 2D canvas
|
||||
// This isn't necessary, but produces quicker results
|
||||
willReadFrequently: true,
|
||||
// Desynchronizes the canvas paint cycle from the event loop
|
||||
// Should be less necessary with OffscreenCanvas, but with a real canvas you will want this
|
||||
desynchronized: true,
|
||||
});
|
||||
|
||||
muxer = new Mp4Muxer.Muxer({
|
||||
target: new Mp4Muxer.ArrayBufferTarget(),
|
||||
video: {
|
||||
// If you change this, make sure to change the VideoEncoder codec as well
|
||||
codec: "avc",
|
||||
width: videoWidth,
|
||||
height: videoHeight,
|
||||
},
|
||||
|
||||
firstTimestampBehavior: 'offset',
|
||||
|
||||
// mp4-muxer docs claim you should always use this with ArrayBufferTarget
|
||||
fastStart: "in-memory",
|
||||
});
|
||||
|
||||
videoEncoder = new VideoEncoder({
|
||||
output: (chunk, meta) => muxer.addVideoChunk(chunk, meta),
|
||||
error: (e) => console.error(e),
|
||||
});
|
||||
|
||||
// This codec should work in most browsers
|
||||
// See https://dmnsgn.github.io/media-codecs for list of codecs and see if your browser supports
|
||||
videoEncoder.configure({
|
||||
codec: "avc1.4d0032",
|
||||
width: videoWidth,
|
||||
height: videoHeight,
|
||||
bitrate: bitrate,
|
||||
bitrateMode: "variable",
|
||||
});
|
||||
//NEW codec: "avc1.4d0032",
|
||||
//ORIGINAL codec: "avc1.42003e",
|
||||
|
||||
var frameNumber = 0;
|
||||
|
||||
//take a snapshot of the canvas every x miliseconds and encode to video
|
||||
|
||||
videoRecordInterval = setInterval(
|
||||
function(){
|
||||
if(recordVideoState == true){
|
||||
//gl.flush();
|
||||
//gl.finish();
|
||||
drawScene();
|
||||
renderCanvasToVideoFrameAndEncode({
|
||||
canvas,
|
||||
videoEncoder,
|
||||
frameNumber,
|
||||
videofps
|
||||
})
|
||||
frameNumber++;
|
||||
}else{
|
||||
}
|
||||
} , 1000/videofps);
|
||||
|
||||
}
|
||||
|
||||
//finish and export video
|
||||
async function finalizeVideo(){
|
||||
console.log("finalize muxer video");
|
||||
togglePlayPause();
|
||||
clearInterval(videoRecordInterval);
|
||||
//playAnimationToggle = false;
|
||||
recordVideoState = false;
|
||||
|
||||
// Forces all pending encodes to complete
|
||||
await videoEncoder.flush();
|
||||
muxer.finalize();
|
||||
let buffer = muxer.target.buffer;
|
||||
finishedBlob = new Blob([buffer]);
|
||||
downloadBlob(new Blob([buffer]));
|
||||
|
||||
//hide user message
|
||||
recordingMessageDiv.classList.add("hidden");
|
||||
togglePlayPause();
|
||||
}
|
||||
|
||||
async function renderCanvasToVideoFrameAndEncode({
|
||||
canvas,
|
||||
videoEncoder,
|
||||
frameNumber,
|
||||
videofps,
|
||||
}) {
|
||||
let frame = new VideoFrame(canvas, {
|
||||
// Equally spaces frames out depending on frames per second
|
||||
timestamp: (frameNumber * 1e6) / videofps,
|
||||
});
|
||||
|
||||
// The encode() method of the VideoEncoder interface asynchronously encodes a VideoFrame
|
||||
videoEncoder.encode(frame);
|
||||
|
||||
// The close() method of the VideoFrame interface clears all states and releases the reference to the media resource.
|
||||
frame.close();
|
||||
}
|
||||
|
||||
function downloadBlob() {
|
||||
console.log("download video");
|
||||
let url = window.URL.createObjectURL(finishedBlob);
|
||||
let a = document.createElement("a");
|
||||
a.style.display = "none";
|
||||
a.href = url;
|
||||
const date = new Date();
|
||||
const filename = projectName+`_${date.toLocaleDateString()}_${date.toLocaleTimeString()}.mp4`;
|
||||
a.download = filename;
|
||||
document.body.appendChild(a);
|
||||
a.click();
|
||||
window.URL.revokeObjectURL(url);
|
||||
}
|
||||
|
||||
//record and download videos on mobile devices
|
||||
function startMobileRecording(){
|
||||
var stream = canvas.captureStream(videofps);
|
||||
mobileRecorder = new MediaRecorder(stream, { 'type': 'video/mp4' });
|
||||
mobileRecorder.addEventListener('dataavailable', finalizeMobileVideo);
|
||||
|
||||
console.log("start simple video recording");
|
||||
console.log("Video dimensions: "+canvas.width+", "+canvas.height);
|
||||
|
||||
recordingMessageDiv.classList.remove("hidden");
|
||||
|
||||
recordVideoState = true;
|
||||
mobileRecorder.start(); //start mobile video recording
|
||||
|
||||
}
|
||||
|
||||
function finalizeMobileVideo(e) {
|
||||
setTimeout(function(){
|
||||
console.log("finish simple video recording");
|
||||
togglePlayPause();
|
||||
recordVideoState = false;
|
||||
/*
|
||||
mobileRecorder.stop();*/
|
||||
var videoData = [ e.data ];
|
||||
finishedBlob = new Blob(videoData, { 'type': 'video/mp4' });
|
||||
downloadBlob(finishedBlob);
|
||||
|
||||
//hide user message
|
||||
recordingMessageDiv.classList.add("hidden");
|
||||
togglePlayPause();
|
||||
|
||||
},500);
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
// Toggle play/pause
|
||||
function togglePlayPause() {
|
||||
|
||||
if (isPlaying) {
|
||||
cancelAnimationFrame(animationID);
|
||||
isPlaying = false;
|
||||
} else {
|
||||
isPlaying = true;
|
||||
animationID = requestAnimationFrame(render);
|
||||
}
|
||||
}
|
||||
|
||||
// Function to refresh the pattern with a new random seed
|
||||
const selectedSeeds = [53, 118, 506];
|
||||
var seedCount = 1;
|
||||
function refreshPattern() {
|
||||
timeOffset = performance.now();
|
||||
//randomSeed = Math.floor(Math.random() * 1000,0);
|
||||
randomSeed = selectedSeeds[seedCount];
|
||||
gl.uniform1f(seedLocation, randomSeed);
|
||||
if(!isPlaying){
|
||||
isPlaying = true;
|
||||
animationID = requestAnimationFrame(render);
|
||||
}
|
||||
console.log('seed:', randomSeed);
|
||||
}
|
||||
|
||||
function startFromZeroTime(){
|
||||
console.log("Restarting animation from time = 0");
|
||||
|
||||
// Cancel current animation if running
|
||||
if (animationID) {
|
||||
cancelAnimationFrame(animationID);
|
||||
}
|
||||
|
||||
// Set the time offset to the current time
|
||||
// This will be subtracted in the render function
|
||||
timeOffset = performance.now();
|
||||
|
||||
// Reset frame counter for FPS calculation
|
||||
frameCount = 0;
|
||||
lastTime = performance.now();
|
||||
|
||||
// Make sure all other uniforms are updated
|
||||
updateUniforms();
|
||||
|
||||
// Ensure animation is playing
|
||||
isPlaying = true;
|
||||
|
||||
// Start the animation loop from the beginning
|
||||
animationID = requestAnimationFrame(render);
|
||||
}
|
||||
|
||||
// Add this function to handle canvas resizing
|
||||
function updateCanvasSize() {
|
||||
// Update canvas dimensions to window size
|
||||
canvas.width = window.innerWidth;
|
||||
canvas.height = window.innerHeight;
|
||||
|
||||
// Update the WebGL viewport to match
|
||||
gl.viewport(0, 0, canvas.width, canvas.height);
|
||||
|
||||
// Re-render if not already playing
|
||||
if (!isPlaying) {
|
||||
drawScene();
|
||||
}
|
||||
|
||||
// If recording is active, we need to handle that
|
||||
if (recordVideoState) {
|
||||
stopRecording();
|
||||
startRecording();
|
||||
}
|
||||
}
|
||||
|
||||
//intro overlay info screen
|
||||
|
||||
let musicPlaying = false;
|
||||
|
||||
let isZenMode = false;
|
||||
@@ -0,0 +1,200 @@
|
||||
/*
|
||||
To do:
|
||||
Press z for zen mode (hides all control and other display on top of the canvas)
|
||||
Ability to add this shader effect on top of an image?
|
||||
Presets / seed choice??
|
||||
Allow user to upload a song, and then it becomes audio reactive?
|
||||
Generate perfect loops in x seconds
|
||||
*/
|
||||
|
||||
// Initialize WebGL context
|
||||
const canvas = document.getElementById('canvas');
|
||||
let startingWidth = window.innerWidth;
|
||||
let startingHeight = window.innerHeight;
|
||||
canvas.width = startingWidth;
|
||||
canvas.height = startingHeight;
|
||||
console.log("canvas width/height: "+canvas.width+" / "+canvas.height);
|
||||
|
||||
const gl = canvas.getContext('webgl') || canvas.getContext('experimental-webgl');
|
||||
let isPlaying = false;
|
||||
let animationID = null;
|
||||
let randomSeed;
|
||||
let time;
|
||||
let timeOffset = 0;
|
||||
|
||||
// FPS tracking variables
|
||||
let frameCount = 0;
|
||||
let lastTime = 0;
|
||||
let fps = 0;
|
||||
|
||||
if (!gl) {
|
||||
alert('WebGL not supported');
|
||||
}
|
||||
|
||||
// Compile shaders
|
||||
function compileShader(source, type) {
|
||||
const shader = gl.createShader(type);
|
||||
gl.shaderSource(shader, source);
|
||||
gl.compileShader(shader);
|
||||
|
||||
if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) {
|
||||
console.error('Shader compilation error:', gl.getShaderInfoLog(shader));
|
||||
gl.deleteShader(shader);
|
||||
return null;
|
||||
}
|
||||
|
||||
return shader;
|
||||
}
|
||||
|
||||
// Create program
|
||||
const vertexShader = compileShader(document.getElementById('vertexShader').textContent, gl.VERTEX_SHADER);
|
||||
const fragmentShader = compileShader(document.getElementById('fragmentShader').textContent, gl.FRAGMENT_SHADER);
|
||||
|
||||
const program = gl.createProgram();
|
||||
gl.attachShader(program, vertexShader);
|
||||
gl.attachShader(program, fragmentShader);
|
||||
gl.linkProgram(program);
|
||||
|
||||
if (!gl.getProgramParameter(program, gl.LINK_STATUS)) {
|
||||
console.error('Program linking error:', gl.getProgramInfoLog(program));
|
||||
}
|
||||
|
||||
gl.useProgram(program);
|
||||
|
||||
// Create rectangle covering the entire canvas
|
||||
const positionBuffer = gl.createBuffer();
|
||||
gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer);
|
||||
gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([
|
||||
-1.0, -1.0,
|
||||
1.0, -1.0,
|
||||
-1.0, 1.0,
|
||||
1.0, 1.0
|
||||
]), gl.STATIC_DRAW);
|
||||
|
||||
// Set up attributes and uniforms
|
||||
const positionLocation = gl.getAttribLocation(program, 'position');
|
||||
gl.enableVertexAttribArray(positionLocation);
|
||||
gl.vertexAttribPointer(positionLocation, 2, gl.FLOAT, false, 0, 0);
|
||||
|
||||
const timeLocation = gl.getUniformLocation(program, 'time');
|
||||
const resolutionLocation = gl.getUniformLocation(program, 'resolution');
|
||||
const seedLocation = gl.getUniformLocation(program, 'seed');
|
||||
|
||||
// GUI-controlled uniform locations
|
||||
const timeScaleLocation = gl.getUniformLocation(program, 'timeScale');
|
||||
const bloomStrengthLocation = gl.getUniformLocation(program, 'bloomStrength');
|
||||
const saturationLocation = gl.getUniformLocation(program, 'saturation');
|
||||
const grainAmountLocation = gl.getUniformLocation(program, 'grainAmount');
|
||||
const colorTintLocation = gl.getUniformLocation(program, 'colorTint');
|
||||
const minCircleSizeLocation = gl.getUniformLocation(program, 'minCircleSize');
|
||||
const circleStrengthLocation = gl.getUniformLocation(program, 'circleStrength');
|
||||
const distortXLocation = gl.getUniformLocation(program, 'distortX');
|
||||
const distortYLocation = gl.getUniformLocation(program, 'distortY');
|
||||
|
||||
const patternAmpLocation = gl.getUniformLocation(program, 'patternAmp');
|
||||
const patternFreqLocation = gl.getUniformLocation(program, 'patternFreq');
|
||||
|
||||
// Initialize parameters object for dat.gui
|
||||
const params = {
|
||||
canvasWidth: startingWidth,
|
||||
canvasHeight: startingHeight,
|
||||
timeScale: .666,
|
||||
patternAmp: 2,
|
||||
patternFreq: 0.4,
|
||||
bloomStrength: 0.777,
|
||||
saturation: 1.74,
|
||||
grainAmount: 0.161,
|
||||
colorTintR: 1.5,
|
||||
colorTintG: 1.0,
|
||||
colorTintB: 1.0,
|
||||
minCircleSize: 2.8,
|
||||
circleStrength: 0,
|
||||
distortX: 1,
|
||||
distortY: 1,
|
||||
};
|
||||
|
||||
// Also refresh on page load
|
||||
window.addEventListener('load', refreshPattern);
|
||||
window.addEventListener('resize', updateCanvasSize);
|
||||
|
||||
// Initialize dat.gui
|
||||
const gui = new dat.GUI({ autoplace: false });
|
||||
gui.close();
|
||||
|
||||
// Add GUI controls with folders for organization
|
||||
const canvasFolder = gui.addFolder('Canvas Size');
|
||||
canvasFolder.add(params, 'canvasWidth', 100, 4000).step(10).name('Width').onChange(updateCanvasSize);
|
||||
canvasFolder.add(params, 'canvasHeight', 100, 4000).step(10).name('Height').onChange(updateCanvasSize);
|
||||
canvasFolder.open();
|
||||
|
||||
const timeFolder = gui.addFolder('Animation');
|
||||
timeFolder.add(params, 'timeScale', 0.1, 3.0).name('Speed').onChange(updateUniforms);
|
||||
timeFolder.open();
|
||||
|
||||
const patternFolder = gui.addFolder('Pattern');
|
||||
patternFolder.add(params, 'patternAmp', 1.0, 50.0).step(0.1).name('Pattern Amp').onChange(updateUniforms);
|
||||
patternFolder.add(params, 'patternFreq', 0.2, 10.0).step(0.1).name('Pattern Freq').onChange(updateUniforms);
|
||||
patternFolder.open();
|
||||
|
||||
const visualFolder = gui.addFolder('Visual Effects');
|
||||
visualFolder.add(params, 'bloomStrength', 0.0, 5.0).name('Bloom').onChange(updateUniforms);
|
||||
visualFolder.add(params, 'saturation', 0.0, 2.0).name('Saturation').onChange(updateUniforms);
|
||||
visualFolder.add(params, 'grainAmount', 0.0, 0.5).name('Grain').onChange(updateUniforms);
|
||||
visualFolder.add(params, 'minCircleSize', 0.0, 10.0).name('Circle Size').onChange(updateUniforms);
|
||||
visualFolder.add(params, 'circleStrength', 0.0, 3.0).name('Circle Strength').onChange(updateUniforms);
|
||||
visualFolder.add(params, 'distortX', 0.0, 50.0).name('Distort-X').onChange(updateUniforms);
|
||||
visualFolder.add(params, 'distortY', 0.0, 50.0).name('Distort-Y').onChange(updateUniforms);
|
||||
|
||||
visualFolder.open();
|
||||
|
||||
const colorFolder = gui.addFolder('Color Tint');
|
||||
colorFolder.add(params, 'colorTintR', 0.0, 1.5).name('Red').onChange(updateUniforms);
|
||||
colorFolder.add(params, 'colorTintG', 0.0, 1.5).name('Green').onChange(updateUniforms);
|
||||
colorFolder.add(params, 'colorTintB', 0.0, 1.5).name('Blue').onChange(updateUniforms);
|
||||
colorFolder.open();
|
||||
|
||||
// Function to update shader uniforms from GUI values
|
||||
function updateUniforms() {
|
||||
gl.uniform1f(timeScaleLocation, params.timeScale);
|
||||
gl.uniform1f(patternAmpLocation, params.patternAmp);
|
||||
gl.uniform1f(patternFreqLocation, params.patternFreq);
|
||||
gl.uniform1f(bloomStrengthLocation, params.bloomStrength);
|
||||
gl.uniform1f(saturationLocation, params.saturation);
|
||||
gl.uniform1f(grainAmountLocation, params.grainAmount);
|
||||
gl.uniform3f(colorTintLocation, params.colorTintR, params.colorTintG, params.colorTintB);
|
||||
gl.uniform1f(minCircleSizeLocation, params.minCircleSize);
|
||||
gl.uniform1f(circleStrengthLocation, params.circleStrength);
|
||||
gl.uniform1f(distortXLocation, params.distortX);
|
||||
gl.uniform1f(distortYLocation, params.distortY);
|
||||
}
|
||||
|
||||
function drawScene(){
|
||||
gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4);
|
||||
}
|
||||
|
||||
// Animation loop
|
||||
function render(timestamp) {
|
||||
if (isPlaying) {
|
||||
// Calculate adjusted time by subtracting the offset
|
||||
const adjustedTime = timestamp - timeOffset;
|
||||
time = timestamp;
|
||||
|
||||
const timeInSeconds = adjustedTime * 0.0035;
|
||||
gl.uniform1f(timeLocation, timeInSeconds);
|
||||
gl.uniform2f(resolutionLocation, canvas.width, canvas.height);
|
||||
|
||||
|
||||
// If video recording is ongoing, drawScene is called already
|
||||
if (!recordVideoState || useMobileRecord) {
|
||||
drawScene();
|
||||
}
|
||||
|
||||
animationID = requestAnimationFrame(render);
|
||||
}
|
||||
}
|
||||
|
||||
// Start the animation loop
|
||||
isPlaying = true;
|
||||
refreshPattern();
|
||||
updateUniforms();
|
||||
animationID = requestAnimationFrame(render);
|
||||
@@ -0,0 +1,21 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2023 Vanilagy
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
@@ -0,0 +1,358 @@
|
||||
# mp4-muxer - JavaScript MP4 multiplexer
|
||||
|
||||
[](https://www.npmjs.com/package/mp4-muxer)
|
||||
[](https://bundlephobia.com/package/mp4-muxer)
|
||||
[](https://www.npmjs.com/package/mp4-muxer)
|
||||
|
||||
The WebCodecs API provides low-level access to media codecs, but provides no way of actually packaging (multiplexing)
|
||||
the encoded media into a playable file. This project implements an MP4 multiplexer in pure TypeScript, which is
|
||||
high-quality, fast and tiny, and supports both video and audio as well as various internal layouts such as Fast Start or
|
||||
fragmented MP4.
|
||||
|
||||
[Demo: Muxing into a file](https://vanilagy.github.io/mp4-muxer/demo/)
|
||||
|
||||
[Demo: Live streaming](https://vanilagy.github.io/mp4-muxer/demo-streaming)
|
||||
|
||||
> **Note:** If you're looking to create **WebM** files, check out [webm-muxer](https://github.com/Vanilagy/webm-muxer),
|
||||
the sister library to mp4-muxer.
|
||||
|
||||
> Consider [donating](https://ko-fi.com/vanilagy) if you've found this library useful and wish to support it ❤️
|
||||
|
||||
## Quick start
|
||||
The following is an example for a common usage of this library:
|
||||
```js
|
||||
import { Muxer, ArrayBufferTarget } from 'mp4-muxer';
|
||||
|
||||
let muxer = new Muxer({
|
||||
target: new ArrayBufferTarget(),
|
||||
video: {
|
||||
codec: 'avc',
|
||||
width: 1280,
|
||||
height: 720
|
||||
},
|
||||
fastStart: 'in-memory'
|
||||
});
|
||||
|
||||
let videoEncoder = new VideoEncoder({
|
||||
output: (chunk, meta) => muxer.addVideoChunk(chunk, meta),
|
||||
error: e => console.error(e)
|
||||
});
|
||||
videoEncoder.configure({
|
||||
codec: 'avc1.42001f',
|
||||
width: 1280,
|
||||
height: 720,
|
||||
bitrate: 1e6
|
||||
});
|
||||
|
||||
/* Encode some frames... */
|
||||
|
||||
await videoEncoder.flush();
|
||||
muxer.finalize();
|
||||
|
||||
let { buffer } = muxer.target; // Buffer contains final MP4 file
|
||||
```
|
||||
|
||||
## Motivation
|
||||
After [webm-muxer](https://github.com/Vanilagy/webm-muxer) gained traction for its ease of use and integration with the
|
||||
WebCodecs API, this library was created to now also allow the creation of MP4 files while maintaining the same DX.
|
||||
While WebM is a more modern format, MP4 is an established standard and supported on way more devices.
|
||||
|
||||
## Installation
|
||||
Using NPM, simply install this package using
|
||||
```
|
||||
npm install mp4-muxer
|
||||
```
|
||||
You can import all exported classes like so:
|
||||
```js
|
||||
import * as Mp4Muxer from 'mp4-muxer';
|
||||
// Or, using CommonJS:
|
||||
const Mp4Muxer = require('mp4-muxer');
|
||||
```
|
||||
Alternatively, you can simply include the library as a script in your HTML, which will add an `Mp4Muxer` object,
|
||||
containing all the exported classes, to the global object, like so:
|
||||
```html
|
||||
<script src="build/mp4-muxer.js"></script>
|
||||
```
|
||||
|
||||
## Usage
|
||||
### Initialization
|
||||
For each MP4 file you wish to create, create an instance of `Muxer` like so:
|
||||
```js
|
||||
import { Muxer } from 'mp4-muxer';
|
||||
|
||||
let muxer = new Muxer(options);
|
||||
```
|
||||
The available options are defined by the following interface:
|
||||
```ts
|
||||
interface MuxerOptions {
|
||||
target:
|
||||
| ArrayBufferTarget
|
||||
| StreamTarget
|
||||
| FileSystemWritableFileStreamTarget,
|
||||
|
||||
video?: {
|
||||
codec: 'avc' | 'hevc' | 'vp9' | 'av1',
|
||||
width: number,
|
||||
height: number,
|
||||
|
||||
// Adds rotation metadata to the file
|
||||
rotation?: 0 | 90 | 180 | 270 | TransformationMatrix
|
||||
},
|
||||
|
||||
audio?: {
|
||||
codec: 'aac' | 'opus',
|
||||
numberOfChannels: number,
|
||||
sampleRate: number
|
||||
},
|
||||
|
||||
fastStart:
|
||||
| false
|
||||
| 'in-memory'
|
||||
| 'fragmented'
|
||||
| { expectedVideoChunks?: number, expectedAudioChunks?: number }
|
||||
|
||||
firstTimestampBehavior?: 'strict' | 'offset' | 'cross-track-offset'
|
||||
}
|
||||
```
|
||||
Codecs currently supported by this library are AVC/H.264, HEVC/H.265, VP9 and AV1 for video, and AAC and Opus for audio.
|
||||
#### `target` (required)
|
||||
This option specifies where the data created by the muxer will be written. The options are:
|
||||
- `ArrayBufferTarget`: The file data will be written into a single large buffer, which is then stored in the target.
|
||||
|
||||
```js
|
||||
import { Muxer, ArrayBufferTarget } from 'mp4-muxer';
|
||||
|
||||
let muxer = new Muxer({
|
||||
target: new ArrayBufferTarget(),
|
||||
fastStart: 'in-memory',
|
||||
// ...
|
||||
});
|
||||
|
||||
// ...
|
||||
|
||||
muxer.finalize();
|
||||
let { buffer } = muxer.target;
|
||||
```
|
||||
- `StreamTarget`: This target defines callbacks that will get called whenever there is new data available - this is
|
||||
useful if you want to stream the data, e.g. pipe it somewhere else. The constructor has the following signature:
|
||||
|
||||
```ts
|
||||
constructor(options: {
|
||||
onData?: (data: Uint8Array, position: number) => void,
|
||||
chunked?: boolean,
|
||||
chunkSize?: number
|
||||
});
|
||||
```
|
||||
|
||||
`onData` is called for each new chunk of available data. The `position` argument specifies the offset in bytes at
|
||||
which the data has to be written. Since the data written by the muxer is not always sequential, **make sure to
|
||||
respect this argument**.
|
||||
|
||||
When using `chunked: true`, data created by the muxer will first be accumulated and only written out once it has
|
||||
reached sufficient size. This is useful for reducing the total amount of writes, at the cost of latency. It using a
|
||||
default chunk size of 16 MiB, which can be overridden by manually setting `chunkSize` to the desired byte length.
|
||||
|
||||
If you want to use this target for *live-streaming*, i.e. playback before muxing has finished, you also need to set
|
||||
`fastStart: 'fragmented'`.
|
||||
|
||||
Usage example:
|
||||
```js
|
||||
import { Muxer, StreamTarget } from 'mp4-muxer';
|
||||
|
||||
let muxer = new Muxer({
|
||||
target: new StreamTarget({
|
||||
onData: (data, position) => { /* Do something with the data */ }
|
||||
}),
|
||||
fastStart: false,
|
||||
// ...
|
||||
});
|
||||
```
|
||||
- `FileSystemWritableFileStreamTarget`: This is essentially a wrapper around a chunked `StreamTarget` with the intention
|
||||
of simplifying the use of this library with the File System Access API. Writing the file directly to disk as it's
|
||||
being created comes with many benefits, such as creating files way larger than the available RAM.
|
||||
|
||||
You can optionally override the default `chunkSize` of 16 MiB.
|
||||
```ts
|
||||
constructor(
|
||||
stream: FileSystemWritableFileStream,
|
||||
options?: { chunkSize?: number }
|
||||
);
|
||||
```
|
||||
|
||||
Usage example:
|
||||
```js
|
||||
import { Muxer, FileSystemWritableFileStreamTarget } from 'mp4-muxer';
|
||||
|
||||
let fileHandle = await window.showSaveFilePicker({
|
||||
suggestedName: `video.mp4`,
|
||||
types: [{
|
||||
description: 'Video File',
|
||||
accept: { 'video/mp4': ['.mp4'] }
|
||||
}],
|
||||
});
|
||||
let fileStream = await fileHandle.createWritable();
|
||||
let muxer = new Muxer({
|
||||
target: new FileSystemWritableFileStreamTarget(fileStream),
|
||||
fastStart: false,
|
||||
// ...
|
||||
});
|
||||
|
||||
// ...
|
||||
|
||||
muxer.finalize();
|
||||
await fileStream.close(); // Make sure to close the stream
|
||||
```
|
||||
#### `fastStart` (required)
|
||||
By default, MP4 metadata (track info, sample timing, etc.) is stored at the end of the file - this makes writing the
|
||||
file faster and easier. However, placing this metadata at the _start_ of the file instead (known as "Fast Start")
|
||||
provides certain benefits: The file becomes easier to stream over the web without range requests, and sites like YouTube
|
||||
can start processing the video while it's uploading. This library provides full control over the placement of metadata
|
||||
setting `fastStart` to one of these options:
|
||||
- `false`: Disables Fast Start, placing all metadata at the end of the file. This option is the fastest and uses the
|
||||
least memory. This option is recommended for large, unbounded files that are streamed directly to disk.
|
||||
- `'in-memory'`: Produces a file with Fast Start by keeping all media chunks in memory until the file is finalized. This
|
||||
option produces the most compact output possible at the cost of a more expensive finalization step and higher memory
|
||||
requirements. This is the preferred option when using `ArrayBufferTarget` as it will result in a higher-quality
|
||||
output with no change in memory footprint.
|
||||
- `'fragmented'`: Produces a _fragmented MP4 (fMP4)_ file, evenly placing sample metadata throughout the file by
|
||||
grouping it into "fragments" (short sections of media), while placing general metadata at the beginning of the file.
|
||||
Fragmented files are ideal for streaming, as they are optimized for random access with minimal to no seeking.
|
||||
Furthermore, they remain lightweight to create no matter how large the file becomes, as they don't require media to
|
||||
be kept in memory for very long. While fragmented files are not as widely supported as regular MP4 files, this
|
||||
option provides powerful benefits with very little downsides. Further details
|
||||
[here](#additional-notes-about-fragmented-mp4-files).
|
||||
- `object`: Produces a file with Fast Start by reserving space for metadata when muxing begins. To know
|
||||
how many bytes need to be reserved to be safe, you'll have to provide the following data:
|
||||
```ts
|
||||
{
|
||||
expectedVideoChunks?: number,
|
||||
expectedAudioChunks?: number
|
||||
}
|
||||
```
|
||||
Note that the property `expectedVideoChunks` is _required_ if you have a video track - the same goes for audio. With
|
||||
this option set, you cannot mux more chunks than the number you've specified (although less is fine).
|
||||
|
||||
This option is faster than `'in-memory'` and uses no additional memory, but results in a slightly larger output,
|
||||
making it useful for when you want to stream the file to disk while still retaining Fast Start.
|
||||
#### `firstTimestampBehavior` (optional)
|
||||
Specifies how to deal with the first chunk in each track having a non-zero timestamp. In the default strict mode,
|
||||
timestamps must start with 0 to ensure proper playback. However, when directly piping video frames or audio data
|
||||
from a MediaTrackStream into the encoder and then the muxer, the timestamps are usually relative to the age of
|
||||
the document or the computer's clock, which is typically not what we want. Handling of these timestamps must be
|
||||
set explicitly:
|
||||
- Use `'offset'` to offset the timestamp of each track by that track's first chunk's timestamp. This way, it
|
||||
starts at 0.
|
||||
- Use `'cross-track-offset'` to offset the timestamp of each track by the _minimum of all tracks' first chunk timestamp_.
|
||||
This works like `'offset'`, but should be used when the all tracks use the same clock.
|
||||
|
||||
### Muxing media chunks
|
||||
Then, with VideoEncoder and AudioEncoder set up, send encoded chunks to the muxer using the following methods:
|
||||
```ts
|
||||
addVideoChunk(
|
||||
chunk: EncodedVideoChunk,
|
||||
meta?: EncodedVideoChunkMetadata,
|
||||
timestamp?: number,
|
||||
compositionTimeOffset?: number
|
||||
): void;
|
||||
|
||||
addAudioChunk(
|
||||
chunk: EncodedAudioChunk,
|
||||
meta?: EncodedAudioChunkMetadata,
|
||||
timestamp?: number
|
||||
): void;
|
||||
```
|
||||
|
||||
Both methods accept an optional, third argument `timestamp` (microseconds) which, if specified, overrides
|
||||
the `timestamp` property of the passed-in chunk.
|
||||
|
||||
The metadata comes from the second parameter of the `output` callback given to the
|
||||
VideoEncoder or AudioEncoder's constructor and needs to be passed into the muxer, like so:
|
||||
```js
|
||||
let videoEncoder = new VideoEncoder({
|
||||
output: (chunk, meta) => muxer.addVideoChunk(chunk, meta),
|
||||
error: e => console.error(e)
|
||||
});
|
||||
videoEncoder.configure(/* ... */);
|
||||
```
|
||||
|
||||
The optional field `compositionTimeOffset` can be used when the decode time of the chunk doesn't equal its presentation
|
||||
time; this is the case when [B-frames](https://en.wikipedia.org/wiki/Video_compression_picture_types) are present.
|
||||
B-frames don't occur when using the WebCodecs API for encoding. The decode time is calculated by subtracting
|
||||
`compositionTimeOffset` from `timestamp`, meaning `timestamp` dictates the presentation time.
|
||||
|
||||
Should you have obtained your encoded media data from a source other than the WebCodecs API, you can use these following
|
||||
methods to directly send your raw data to the muxer:
|
||||
```ts
|
||||
addVideoChunkRaw(
|
||||
data: Uint8Array,
|
||||
type: 'key' | 'delta',
|
||||
timestamp: number, // in microseconds
|
||||
duration: number, // in microseconds
|
||||
meta?: EncodedVideoChunkMetadata,
|
||||
compositionTimeOffset?: number // in microseconds
|
||||
): void;
|
||||
|
||||
addAudioChunkRaw(
|
||||
data: Uint8Array,
|
||||
type: 'key' | 'delta',
|
||||
timestamp: number, // in microseconds
|
||||
duration: number, // in microseconds
|
||||
meta?: EncodedAudioChunkMetadata
|
||||
): void;
|
||||
```
|
||||
|
||||
### Finishing up
|
||||
When encoding is finished and all the encoders have been flushed, call `finalize` on the `Muxer` instance to finalize
|
||||
the MP4 file:
|
||||
```js
|
||||
muxer.finalize();
|
||||
```
|
||||
When using an ArrayBufferTarget, the final buffer will be accessible through it:
|
||||
```js
|
||||
let { buffer } = muxer.target;
|
||||
```
|
||||
When using a FileSystemWritableFileStreamTarget, make sure to close the stream after calling `finalize`:
|
||||
```js
|
||||
await fileStream.close();
|
||||
```
|
||||
|
||||
## Details
|
||||
### Variable frame rate
|
||||
MP4 files support variable frame rate, however some players (such as QuickTime) have been observed not to behave well
|
||||
when the timestamps are irregular. Therefore, whenever possible, try aiming for a fixed frame rate.
|
||||
|
||||
### Additional notes about fragmented MP4 files
|
||||
By breaking up the media and related metadata into small fragments, fMP4 files optimize for random access and are ideal
|
||||
for streaming, while remaining cheap to write even for long files. However, you should keep these things in mind:
|
||||
- **Media chunk buffering:**
|
||||
When muxing a file with a video **and** an audio track, the muxer needs to wait for the chunks from _both_ media
|
||||
to finalize any given fragment. In other words, it must buffer chunks of one medium if the other medium has not yet
|
||||
encoded chunks up to that timestamp. For example, should you first encode all your video frames and then encode the
|
||||
audio afterward, the multiplexer will have to hold all those video frames in memory until the audio chunks start
|
||||
coming in. This might lead to memory exhaustion should your video be very long. When there is only one media track,
|
||||
this issue does not arise. So, when muxing a multimedia file, make sure it is somewhat limited in size or the chunks
|
||||
are encoded in a somewhat interleaved way (like is the case for live media). This will keep memory usage at a
|
||||
constant low.
|
||||
- **Video key frame frequency:**
|
||||
Every track's first sample in a fragment must be a key frame in order to be able to play said fragment without the
|
||||
knowledge of previous ones. However, this means that the muxer needs to wait for a video key frame to begin a new
|
||||
fragment. If these key frames are too infrequent, fragments become too large, harming random access. Therefore,
|
||||
every 5–10 seconds, you should force a video key frame like so:
|
||||
```js
|
||||
videoEncoder.encode(frame, { keyFrame: true });
|
||||
```
|
||||
|
||||
## Implementation & development
|
||||
MP4 files are based on the ISO Base Media Format, which structures its files as a hierarchy of boxes (or atoms). The
|
||||
standards used to implement this library were
|
||||
[ISO/IEC 14496-1](http://netmedia.zju.edu.cn/multimedia2013/mpeg-4/ISO%20IEC%2014496-1%20MPEG-4%20System%20Standard.pdf),
|
||||
[ISO/IEC 14496-12](https://web.archive.org/web/20231123030701/https://b.goeswhere.com/ISO_IEC_14496-12_2015.pdf)
|
||||
and
|
||||
[ISO/IEC 14496-14](https://github.com/OpenAnsible/rust-mp4/raw/master/docs/ISO_IEC_14496-14_2003-11-15.pdf).
|
||||
Additionally, the
|
||||
[QuickTime MP4 Specification](https://developer.apple.com/library/archive/documentation/QuickTime/QTFF/QTFFPreface/qtffPreface.html)
|
||||
was a very useful resource.
|
||||
|
||||
For development, clone this repository, install everything with `npm install`, then run `npm run watch` to bundle the
|
||||
code into the `build` directory. Run `npm run check` to run the TypeScript type checker, and `npm run lint` to run
|
||||
ESLint.
|
||||
@@ -0,0 +1,46 @@
|
||||
import * as esbuild from 'esbuild';
|
||||
|
||||
const baseConfig = {
|
||||
entryPoints: ['src/index.ts'],
|
||||
bundle: true,
|
||||
logLevel: 'info'
|
||||
};
|
||||
|
||||
const umdConfig = {
|
||||
...baseConfig,
|
||||
format: 'iife',
|
||||
|
||||
// The following are hacks to basically make this an UMD module. No native support for that in esbuild as of today
|
||||
globalName: 'Mp4Muxer',
|
||||
|
||||
footer: {
|
||||
js:
|
||||
`if (typeof module === "object" && typeof module.exports === "object") Object.assign(module.exports, Mp4Muxer)`
|
||||
}
|
||||
};
|
||||
|
||||
const esmConfig = {
|
||||
...baseConfig,
|
||||
format: 'esm'
|
||||
};
|
||||
|
||||
let ctxUmd = await esbuild.context({
|
||||
...umdConfig,
|
||||
outfile: 'build/mp4-muxer.js'
|
||||
});
|
||||
let ctxEsm = await esbuild.context({
|
||||
...esmConfig,
|
||||
outfile: 'build/mp4-muxer.mjs'
|
||||
});
|
||||
let ctxUmdMinified = await esbuild.context({
|
||||
...umdConfig,
|
||||
outfile: 'build/mp4-muxer.min.js',
|
||||
minify: true
|
||||
});
|
||||
let ctxEsmMinified = await esbuild.context({
|
||||
...esmConfig,
|
||||
outfile: 'build/mp4-muxer.min.mjs',
|
||||
minify: true
|
||||
});
|
||||
|
||||
await Promise.all([ctxUmd.watch(), ctxEsm.watch(), ctxUmdMinified.watch(), ctxEsmMinified.watch()]);
|
||||
@@ -0,0 +1,226 @@
|
||||
declare type TransformationMatrix = [number, number, number, number, number, number, number, number, number];
|
||||
|
||||
declare interface VideoOptions {
|
||||
/**
|
||||
* The codec of the encoded video chunks.
|
||||
*/
|
||||
codec: 'avc' | 'hevc' | 'vp9' | 'av1',
|
||||
/**
|
||||
* The width of the video in pixels.
|
||||
*/
|
||||
width: number,
|
||||
/**
|
||||
* The height of the video in pixels.
|
||||
*/
|
||||
height: number,
|
||||
/**
|
||||
* The clockwise rotation of the video in degrees, or a transformation matrix.
|
||||
*/
|
||||
rotation?: 0 | 90 | 180 | 270 | TransformationMatrix
|
||||
}
|
||||
|
||||
declare interface AudioOptions {
|
||||
/**
|
||||
* The codec of the encoded audio chunks.
|
||||
*/
|
||||
codec: 'aac' | 'opus',
|
||||
/**
|
||||
* The number of audio channels in the audio track.
|
||||
*/
|
||||
numberOfChannels: number,
|
||||
/**
|
||||
* The sample rate of the audio track in samples per second per channel.
|
||||
*/
|
||||
sampleRate: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Describes the properties used to configure an instance of `Muxer`.
|
||||
*/
|
||||
declare type MuxerOptions<T extends Target> = {
|
||||
/**
|
||||
* Specifies what happens with the data created by the muxer.
|
||||
*/
|
||||
target: T,
|
||||
|
||||
/**
|
||||
* When set, declares the existence of a video track in the MP4 file and configures that video track.
|
||||
*/
|
||||
video?: VideoOptions,
|
||||
|
||||
/**
|
||||
* When set, declares the existence of an audio track in the MP4 file and configures that audio track.
|
||||
*/
|
||||
audio?: AudioOptions,
|
||||
|
||||
/**
|
||||
* Controls the placement of metadata in the file. Placing metadata at the start of the file is known as "Fast
|
||||
* Start", which results in better playback at the cost of more required processing or memory.
|
||||
*
|
||||
* Use `false` to disable Fast Start, placing the metadata at the end of the file. Fastest and uses the least
|
||||
* memory.
|
||||
*
|
||||
* Use `'in-memory'` to produce a file with Fast Start by keeping all media chunks in memory until the file is
|
||||
* finalized. This produces a high-quality and compact output at the cost of a more expensive finalization step and
|
||||
* higher memory requirements.
|
||||
*
|
||||
* Use `'fragmented'` to place metadata at the start of the file by creating a fragmented "fMP4" file. In a
|
||||
* fragmented file, chunks of media and their metadata are written to the file in "fragments", eliminating the need
|
||||
* to put all metadata in one place. Fragmented files are useful for streaming, as they allow for better random
|
||||
* access. Furthermore, they remain lightweight to create even for very large files, as they don't require all media
|
||||
* to be kept in memory. However, fragmented files are not as widely supported as regular MP4 files.
|
||||
*
|
||||
* Use an object to produce a file with Fast Start by reserving space for metadata when muxing starts. In order to
|
||||
* know how much space needs to be reserved, you'll need to tell it the upper bound of how many media chunks will be
|
||||
* muxed. Do this by setting `expectedVideoChunks` and/or `expectedAudioChunks`.
|
||||
*/
|
||||
fastStart: false | 'in-memory' | 'fragmented' | {
|
||||
expectedVideoChunks?: number,
|
||||
expectedAudioChunks?: number
|
||||
},
|
||||
|
||||
/**
|
||||
* Specifies how to deal with the first chunk in each track having a non-zero timestamp. In the default strict mode,
|
||||
* timestamps must start with 0 to ensure proper playback. However, when directly piping video frames or audio data
|
||||
* from a MediaTrackStream into the encoder and then the muxer, the timestamps are usually relative to the age of
|
||||
* the document or the computer's clock, which is typically not what we want. Handling of these timestamps must be
|
||||
* set explicitly:
|
||||
*
|
||||
* Use `'offset'` to offset the timestamp of each video track by that track's first chunk's timestamp. This way, it
|
||||
* starts at 0.
|
||||
*
|
||||
* Use `'cross-track-offset'` to offset the timestamp of _both_ tracks by whichever track's first chunk timestamp is
|
||||
* earliest. This is designed for cases when both tracks' timestamps come from the same clock source.
|
||||
*/
|
||||
firstTimestampBehavior?: 'strict' | 'offset' | 'cross-track-offset'
|
||||
};
|
||||
|
||||
declare type Target = ArrayBufferTarget | StreamTarget | FileSystemWritableFileStreamTarget;
|
||||
|
||||
/** The file data will be written into a single large buffer, which is then stored in `buffer` upon finalization.. */
|
||||
declare class ArrayBufferTarget {
|
||||
buffer: ArrayBuffer;
|
||||
}
|
||||
|
||||
/**
|
||||
* This target defines callbacks that will get called whenever there is new data available - this is useful if
|
||||
* you want to stream the data, e.g. pipe it somewhere else.
|
||||
*
|
||||
* When using `chunked: true` in the options, data created by the muxer will first be accumulated and only written out
|
||||
* once it has reached sufficient size, using a default chunk size of 16 MiB. This is useful for reducing the total
|
||||
* amount of writes, at the cost of latency.
|
||||
*/
|
||||
declare class StreamTarget {
|
||||
constructor(options: {
|
||||
onData?: (data: Uint8Array, position: number) => void,
|
||||
chunked?: boolean,
|
||||
chunkSize?: number
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* This is essentially a wrapper around a chunked `StreamTarget` with the intention of simplifying the use of this
|
||||
* library with the File System Access API. Writing the file directly to disk as it's being created comes with many
|
||||
* benefits, such as creating files way larger than the available RAM.
|
||||
*/
|
||||
declare class FileSystemWritableFileStreamTarget {
|
||||
constructor(
|
||||
stream: FileSystemWritableFileStream,
|
||||
options?: { chunkSize?: number }
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Used to multiplex video and audio chunks into a single MP4 file. For each MP4 file you want to create, create
|
||||
* one instance of `Muxer`.
|
||||
*/
|
||||
declare class Muxer<T extends Target> {
|
||||
target: T;
|
||||
|
||||
/**
|
||||
* Creates a new instance of `Muxer`.
|
||||
* @param options Specifies configuration and metadata for the MP4 file.
|
||||
*/
|
||||
constructor(options: MuxerOptions<T>);
|
||||
|
||||
/**
|
||||
* Adds a new, encoded video chunk to the MP4 file.
|
||||
* @param chunk The encoded video chunk. Can be obtained through a `VideoEncoder`.
|
||||
* @param meta The metadata about the encoded video, also provided by `VideoEncoder`.
|
||||
* @param timestamp Optionally, the presentation timestamp to use for the video chunk. When not provided, it will
|
||||
* use the one specified in `chunk`.
|
||||
* @param compositionTimeOffset Optionally, the composition time offset (i.e. presentation timestamp minus decode
|
||||
* timestamp) to use for the video chunk. When not provided, it will be zero.
|
||||
*/
|
||||
addVideoChunk(
|
||||
chunk: EncodedVideoChunk,
|
||||
meta?: EncodedVideoChunkMetadata,
|
||||
timestamp?: number,
|
||||
compositionTimeOffset?: number
|
||||
): void;
|
||||
/**
|
||||
* Adds a new, encoded audio chunk to the MP4 file.
|
||||
* @param chunk The encoded audio chunk. Can be obtained through an `AudioEncoder`.
|
||||
* @param meta The metadata about the encoded audio, also provided by `AudioEncoder`.
|
||||
* @param timestamp Optionally, the timestamp to use for the audio chunk. When not provided, it will use the one
|
||||
* specified in `chunk`.
|
||||
*/
|
||||
addAudioChunk(chunk: EncodedAudioChunk, meta?: EncodedAudioChunkMetadata, timestamp?: number): void;
|
||||
|
||||
/**
|
||||
* Adds a raw video chunk to the MP4 file. This method should be used when the encoded video is not obtained
|
||||
* through a `VideoEncoder` but through some other means, where no instance of `EncodedVideoChunk`is available.
|
||||
* @param data The raw data of the video chunk.
|
||||
* @param type Whether the video chunk is a keyframe or delta frame.
|
||||
* @param timestamp The timestamp of the video chunk.
|
||||
* @param duration The duration of the video chunk.
|
||||
* @param meta Optionally, any encoder metadata.
|
||||
* @param compositionTimeOffset The composition time offset (i.e. presentation timestamp minus decode timestamp) of
|
||||
* the video chunk.
|
||||
*/
|
||||
addVideoChunkRaw(
|
||||
data: Uint8Array,
|
||||
type: 'key' | 'delta',
|
||||
timestamp: number,
|
||||
duration: number,
|
||||
meta?: EncodedVideoChunkMetadata,
|
||||
compositionTimeOffset?: number
|
||||
): void;
|
||||
/**
|
||||
* Adds a raw audio chunk to the MP4 file. This method should be used when the encoded audio is not obtained
|
||||
* through an `AudioEncoder` but through some other means, where no instance of `EncodedAudioChunk`is available.
|
||||
* @param data The raw data of the audio chunk.
|
||||
* @param type Whether the audio chunk is a keyframe or delta frame.
|
||||
* @param timestamp The timestamp of the audio chunk.
|
||||
* @param duration The duration of the audio chunk.
|
||||
* @param meta Optionally, any encoder metadata.
|
||||
*/
|
||||
addAudioChunkRaw(
|
||||
data: Uint8Array,
|
||||
type: 'key' | 'delta',
|
||||
timestamp: number,
|
||||
duration: number,
|
||||
meta?: EncodedAudioChunkMetadata
|
||||
): void;
|
||||
|
||||
/**
|
||||
* Is to be called after all media chunks have been added to the muxer. Make sure to call and await the `flush`
|
||||
* method on your `VideoEncoder` and/or `AudioEncoder` before calling this method to ensure all encoding has
|
||||
* finished. This method will then finish up the writing process of the MP4 file.
|
||||
*/
|
||||
finalize(): void;
|
||||
}
|
||||
|
||||
declare global {
|
||||
let Mp4Muxer: typeof Mp4Muxer;
|
||||
}
|
||||
|
||||
export {
|
||||
Muxer,
|
||||
MuxerOptions,
|
||||
ArrayBufferTarget,
|
||||
StreamTarget,
|
||||
FileSystemWritableFileStreamTarget,
|
||||
TransformationMatrix
|
||||
};
|
||||
export as namespace Mp4Muxer;
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,27 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en" translate="no">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta http-equiv="X-UA-Compatible" content="IE=edge">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>MP4 muxer streaming demo</title>
|
||||
<link rel="stylesheet" href="./style.css">
|
||||
<script src="../build/mp4-muxer.js" defer></script>
|
||||
<script src="./script.js" defer></script>
|
||||
</head>
|
||||
<body>
|
||||
<main>
|
||||
<h1>MP4 muxer streaming demo - draw something!</h1>
|
||||
<h2>The live canvas state and your microphone input will be recorded,<br> muxed into a fragmented MP4 stream and shown live in the <video> element.</h2>
|
||||
<div id="controls">
|
||||
<button id="start-recording">Start recording</button>
|
||||
<button id="end-recording" style="display: none;">End recording</button>
|
||||
</div>
|
||||
<div id="center">
|
||||
<canvas width="640" height="480"></canvas>
|
||||
<video id="stream-preview" width="640" height="360" controls></video>
|
||||
</div>
|
||||
<p id="recording-status"></p>
|
||||
</main>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,204 @@
|
||||
const canvas = document.querySelector('canvas');
|
||||
const ctx = canvas.getContext('2d', { desynchronized: true });
|
||||
const streamPreview = document.querySelector('#stream-preview');
|
||||
const startRecordingButton = document.querySelector('#start-recording');
|
||||
const endRecordingButton = document.querySelector('#end-recording');
|
||||
const recordingStatus = document.querySelector('#recording-status');
|
||||
|
||||
/** RECORDING & MUXING STUFF */
|
||||
|
||||
let muxer = null;
|
||||
let videoEncoder = null;
|
||||
let audioEncoder = null;
|
||||
let startTime = null;
|
||||
let recording = false;
|
||||
let audioTrack = null;
|
||||
let intervalId = null;
|
||||
let lastKeyFrame = null;
|
||||
let framesGenerated = 0;
|
||||
|
||||
const startRecording = async () => {
|
||||
// Check for VideoEncoder availability
|
||||
if (typeof VideoEncoder === 'undefined') {
|
||||
alert("Looks like your user agent doesn't support VideoEncoder / WebCodecs API yet.");
|
||||
return;
|
||||
}
|
||||
|
||||
startRecordingButton.style.display = 'none';
|
||||
|
||||
// Check for AudioEncoder availability
|
||||
if (typeof AudioEncoder !== 'undefined') {
|
||||
// Try to get access to the user's microphone
|
||||
try {
|
||||
let userMedia = await navigator.mediaDevices.getUserMedia({ video: false, audio: true });
|
||||
audioTrack = userMedia.getAudioTracks()[0];
|
||||
} catch (e) {}
|
||||
if (!audioTrack) console.warn("Couldn't acquire a user media audio track.");
|
||||
} else {
|
||||
console.warn('AudioEncoder not available; no need to acquire a user media audio track.');
|
||||
}
|
||||
|
||||
let mediaSource = new MediaSource();
|
||||
streamPreview.src = URL.createObjectURL(mediaSource);
|
||||
streamPreview.play();
|
||||
|
||||
await new Promise(resolve => mediaSource.onsourceopen = resolve);
|
||||
|
||||
// We'll append ArrayBuffers to this as the muxer starts to spit out chunks
|
||||
let sourceBuffer = mediaSource.addSourceBuffer('video/mp4; codecs="avc1.64001F, mp4a.40.2"');
|
||||
|
||||
endRecordingButton.style.display = 'block';
|
||||
|
||||
let audioSampleRate = audioTrack?.getCapabilities().sampleRate.max;
|
||||
|
||||
// Create an MP4 muxer with a video track and maybe an audio track
|
||||
muxer = new Mp4Muxer.Muxer({
|
||||
target: new Mp4Muxer.StreamTarget({
|
||||
onData: buffer => sourceBuffer.appendBuffer(buffer)
|
||||
}),
|
||||
|
||||
video: {
|
||||
codec: 'avc',
|
||||
width: canvas.width,
|
||||
height: canvas.height
|
||||
},
|
||||
audio: audioTrack ? {
|
||||
codec: 'aac',
|
||||
sampleRate: audioSampleRate,
|
||||
numberOfChannels: 1
|
||||
} : undefined,
|
||||
|
||||
// Puts metadata to the start of the file. Since we're using ArrayBufferTarget anyway, this makes no difference
|
||||
// to memory footprint.
|
||||
fastStart: 'fragmented',
|
||||
|
||||
// Because we're directly pumping a MediaStreamTrack's data into it, which doesn't start at timestamp = 0
|
||||
firstTimestampBehavior: 'offset'
|
||||
});
|
||||
|
||||
videoEncoder = new VideoEncoder({
|
||||
output: (chunk, meta) => muxer.addVideoChunk(chunk, meta),
|
||||
error: e => console.error(e)
|
||||
});
|
||||
videoEncoder.configure({
|
||||
codec: 'avc1.64001F',
|
||||
width: canvas.width,
|
||||
height: canvas.height,
|
||||
bitrate: 1e6
|
||||
});
|
||||
|
||||
if (audioTrack) {
|
||||
audioEncoder = new AudioEncoder({
|
||||
output: (chunk, meta) => muxer.addAudioChunk(chunk, meta),
|
||||
error: e => console.error(e)
|
||||
});
|
||||
audioEncoder.configure({
|
||||
codec: 'mp4a.40.2',
|
||||
numberOfChannels: 1,
|
||||
sampleRate: audioSampleRate,
|
||||
bitrate: 128000
|
||||
});
|
||||
|
||||
// Create a MediaStreamTrackProcessor to get AudioData chunks from the audio track
|
||||
let trackProcessor = new MediaStreamTrackProcessor({ track: audioTrack });
|
||||
let consumer = new WritableStream({
|
||||
write(audioData) {
|
||||
if (!recording) return;
|
||||
audioEncoder.encode(audioData);
|
||||
audioData.close();
|
||||
}
|
||||
});
|
||||
trackProcessor.readable.pipeTo(consumer);
|
||||
}
|
||||
|
||||
startTime = document.timeline.currentTime;
|
||||
recording = true;
|
||||
lastKeyFrame = -Infinity;
|
||||
framesGenerated = 0;
|
||||
|
||||
encodeVideoFrame();
|
||||
intervalId = setInterval(encodeVideoFrame, 1000/30);
|
||||
};
|
||||
startRecordingButton.addEventListener('click', startRecording);
|
||||
|
||||
const encodeVideoFrame = () => {
|
||||
let elapsedTime = document.timeline.currentTime - startTime;
|
||||
let frame = new VideoFrame(canvas, {
|
||||
timestamp: framesGenerated * 1e6 / 30, // Ensure equally-spaced frames every 1/30th of a second
|
||||
duration: 1e6 / 30
|
||||
});
|
||||
framesGenerated++;
|
||||
|
||||
// Ensure a video key frame at least every 0.5 seconds
|
||||
let needsKeyFrame = elapsedTime - lastKeyFrame >= 500;
|
||||
if (needsKeyFrame) lastKeyFrame = elapsedTime;
|
||||
|
||||
videoEncoder.encode(frame, { keyFrame: needsKeyFrame });
|
||||
frame.close();
|
||||
|
||||
recordingStatus.textContent =
|
||||
`${elapsedTime % 1000 < 500 ? '🔴' : '⚫'} Recording - ${(elapsedTime / 1000).toFixed(1)} s`;
|
||||
};
|
||||
|
||||
const endRecording = async () => {
|
||||
endRecordingButton.style.display = 'none';
|
||||
recordingStatus.textContent = '';
|
||||
recording = false;
|
||||
|
||||
clearInterval(intervalId);
|
||||
audioTrack?.stop();
|
||||
|
||||
await videoEncoder?.flush();
|
||||
await audioEncoder?.flush();
|
||||
muxer.finalize();
|
||||
|
||||
videoEncoder = null;
|
||||
audioEncoder = null;
|
||||
muxer = null;
|
||||
startTime = null;
|
||||
firstAudioTimestamp = null;
|
||||
|
||||
startRecordingButton.style.display = 'block';
|
||||
};
|
||||
endRecordingButton.addEventListener('click', endRecording);
|
||||
|
||||
/** CANVAS DRAWING STUFF */
|
||||
|
||||
ctx.fillStyle = 'white';
|
||||
ctx.fillRect(0, 0, canvas.width, canvas.height);
|
||||
|
||||
let drawing = false;
|
||||
let lastPos = { x: 0, y: 0 };
|
||||
|
||||
const getRelativeMousePos = (e) => {
|
||||
let rect = canvas.getBoundingClientRect();
|
||||
return { x: e.clientX - rect.x, y: e.clientY - rect.y };
|
||||
};
|
||||
|
||||
const drawLine = (from, to) => {
|
||||
ctx.beginPath();
|
||||
ctx.moveTo(from.x, from.y);
|
||||
ctx.lineTo(to.x, to.y);
|
||||
ctx.strokeStyle = 'black';
|
||||
ctx.lineWidth = 3;
|
||||
ctx.lineCap = 'round';
|
||||
ctx.stroke();
|
||||
};
|
||||
|
||||
canvas.addEventListener('pointerdown', (e) => {
|
||||
if (e.button !== 0) return;
|
||||
|
||||
drawing = true;
|
||||
lastPos = getRelativeMousePos(e);
|
||||
drawLine(lastPos, lastPos);
|
||||
});
|
||||
window.addEventListener('pointerup', () => {
|
||||
drawing = false;
|
||||
});
|
||||
window.addEventListener('mousemove', (e) => {
|
||||
if (!drawing) return;
|
||||
|
||||
let newPos = getRelativeMousePos(e);
|
||||
drawLine(lastPos, newPos);
|
||||
lastPos = newPos;
|
||||
});
|
||||
@@ -0,0 +1,68 @@
|
||||
html, body {
|
||||
margin: 0;
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
background: #120d17;
|
||||
color: white;
|
||||
font-family: monospace;
|
||||
}
|
||||
|
||||
body {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
}
|
||||
|
||||
* {
|
||||
user-select: none;
|
||||
}
|
||||
|
||||
main {
|
||||
width: 100%;
|
||||
}
|
||||
|
||||
h1 {
|
||||
margin: 0;
|
||||
font-weight: normal;
|
||||
text-align: center;
|
||||
margin-bottom: 10px;
|
||||
}
|
||||
|
||||
h2 {
|
||||
margin: 0;
|
||||
font-weight: normal;
|
||||
text-align: center;
|
||||
font-size: 14px;
|
||||
margin-bottom: 20px;
|
||||
}
|
||||
|
||||
canvas {
|
||||
border-radius: 10px;
|
||||
outline: 3px solid rgb(202, 202, 202);
|
||||
}
|
||||
|
||||
#controls {
|
||||
margin-bottom: 20px;
|
||||
display: flex;
|
||||
justify-content: center;
|
||||
height: 38px;
|
||||
}
|
||||
|
||||
#center {
|
||||
display: flex;
|
||||
justify-content: center;
|
||||
align-items: center;
|
||||
gap: 10px;
|
||||
}
|
||||
|
||||
button {
|
||||
font-size: 20px;
|
||||
padding: 5px 8px;
|
||||
}
|
||||
|
||||
p {
|
||||
margin: 0;
|
||||
text-align: center;
|
||||
margin-top: 20px;
|
||||
height: 20px;
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en" translate="no">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta http-equiv="X-UA-Compatible" content="IE=edge">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>MP4 muxer demo</title>
|
||||
<link rel="stylesheet" href="./style.css">
|
||||
<script src="../build/mp4-muxer.js" defer></script>
|
||||
<script src="./script.js" defer></script>
|
||||
</head>
|
||||
<body>
|
||||
<main>
|
||||
<h1>MP4 muxer demo - draw something!</h1>
|
||||
<h2>The live canvas state and your microphone input will be recorded<br>and muxed into an MP4 file.</h2>
|
||||
<div id="controls">
|
||||
<button id="start-recording">Start recording</button>
|
||||
<button id="end-recording" style="display: none;">End recording</button>
|
||||
</div>
|
||||
<canvas width="640" height="480"></canvas>
|
||||
<p id="recording-status"></p>
|
||||
</main>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,206 @@
|
||||
const canvas = document.querySelector('canvas');
|
||||
const ctx = canvas.getContext('2d', { desynchronized: true });
|
||||
const startRecordingButton = document.querySelector('#start-recording');
|
||||
const endRecordingButton = document.querySelector('#end-recording');
|
||||
const recordingStatus = document.querySelector('#recording-status');
|
||||
|
||||
/** RECORDING & MUXING STUFF */
|
||||
|
||||
let muxer = null;
|
||||
let videoEncoder = null;
|
||||
let audioEncoder = null;
|
||||
let startTime = null;
|
||||
let recording = false;
|
||||
let audioTrack = null;
|
||||
let intervalId = null;
|
||||
let lastKeyFrame = null;
|
||||
let framesGenerated = 0;
|
||||
|
||||
const startRecording = async () => {
|
||||
// Check for VideoEncoder availability
|
||||
if (typeof VideoEncoder === 'undefined') {
|
||||
alert("Looks like your user agent doesn't support VideoEncoder / WebCodecs API yet.");
|
||||
return;
|
||||
}
|
||||
|
||||
startRecordingButton.style.display = 'none';
|
||||
|
||||
// Check for AudioEncoder availability
|
||||
if (typeof AudioEncoder !== 'undefined') {
|
||||
// Try to get access to the user's microphone
|
||||
try {
|
||||
let userMedia = await navigator.mediaDevices.getUserMedia({ video: false, audio: true });
|
||||
audioTrack = userMedia.getAudioTracks()[0];
|
||||
} catch (e) {}
|
||||
if (!audioTrack) console.warn("Couldn't acquire a user media audio track.");
|
||||
} else {
|
||||
console.warn('AudioEncoder not available; no need to acquire a user media audio track.');
|
||||
}
|
||||
|
||||
endRecordingButton.style.display = 'block';
|
||||
|
||||
let audioSampleRate = audioTrack?.getCapabilities().sampleRate.max;
|
||||
|
||||
// Create an MP4 muxer with a video track and maybe an audio track
|
||||
muxer = new Mp4Muxer.Muxer({
|
||||
target: new Mp4Muxer.ArrayBufferTarget(),
|
||||
|
||||
video: {
|
||||
codec: 'avc',
|
||||
width: canvas.width,
|
||||
height: canvas.height
|
||||
},
|
||||
audio: audioTrack ? {
|
||||
codec: 'aac',
|
||||
sampleRate: audioSampleRate,
|
||||
numberOfChannels: 1
|
||||
} : undefined,
|
||||
|
||||
// Puts metadata to the start of the file. Since we're using ArrayBufferTarget anyway, this makes no difference
|
||||
// to memory footprint.
|
||||
fastStart: 'in-memory',
|
||||
|
||||
// Because we're directly pumping a MediaStreamTrack's data into it, which doesn't start at timestamp = 0
|
||||
firstTimestampBehavior: 'offset'
|
||||
});
|
||||
|
||||
videoEncoder = new VideoEncoder({
|
||||
output: (chunk, meta) => muxer.addVideoChunk(chunk, meta),
|
||||
error: e => console.error(e)
|
||||
});
|
||||
videoEncoder.configure({
|
||||
codec: 'avc1.42001f',
|
||||
width: canvas.width,
|
||||
height: canvas.height,
|
||||
bitrate: 1e6
|
||||
});
|
||||
|
||||
if (audioTrack) {
|
||||
audioEncoder = new AudioEncoder({
|
||||
output: (chunk, meta) => muxer.addAudioChunk(chunk, meta),
|
||||
error: e => console.error(e)
|
||||
});
|
||||
audioEncoder.configure({
|
||||
codec: 'mp4a.40.2',
|
||||
numberOfChannels: 1,
|
||||
sampleRate: audioSampleRate,
|
||||
bitrate: 128000
|
||||
});
|
||||
|
||||
// Create a MediaStreamTrackProcessor to get AudioData chunks from the audio track
|
||||
let trackProcessor = new MediaStreamTrackProcessor({ track: audioTrack });
|
||||
let consumer = new WritableStream({
|
||||
write(audioData) {
|
||||
if (!recording) return;
|
||||
audioEncoder.encode(audioData);
|
||||
audioData.close();
|
||||
}
|
||||
});
|
||||
trackProcessor.readable.pipeTo(consumer);
|
||||
}
|
||||
|
||||
startTime = document.timeline.currentTime;
|
||||
recording = true;
|
||||
lastKeyFrame = -Infinity;
|
||||
framesGenerated = 0;
|
||||
|
||||
encodeVideoFrame();
|
||||
intervalId = setInterval(encodeVideoFrame, 1000/30);
|
||||
};
|
||||
startRecordingButton.addEventListener('click', startRecording);
|
||||
|
||||
const encodeVideoFrame = () => {
|
||||
let elapsedTime = document.timeline.currentTime - startTime;
|
||||
let frame = new VideoFrame(canvas, {
|
||||
timestamp: framesGenerated * 1e6 / 30, // Ensure equally-spaced frames every 1/30th of a second
|
||||
duration: 1e6 / 30
|
||||
});
|
||||
framesGenerated++;
|
||||
|
||||
// Ensure a video key frame at least every 5 seconds for good scrubbing
|
||||
let needsKeyFrame = elapsedTime - lastKeyFrame >= 5000;
|
||||
if (needsKeyFrame) lastKeyFrame = elapsedTime;
|
||||
|
||||
videoEncoder.encode(frame, { keyFrame: needsKeyFrame });
|
||||
frame.close();
|
||||
|
||||
recordingStatus.textContent =
|
||||
`${elapsedTime % 1000 < 500 ? '🔴' : '⚫'} Recording - ${(elapsedTime / 1000).toFixed(1)} s`;
|
||||
};
|
||||
|
||||
const endRecording = async () => {
|
||||
endRecordingButton.style.display = 'none';
|
||||
recordingStatus.textContent = '';
|
||||
recording = false;
|
||||
|
||||
clearInterval(intervalId);
|
||||
audioTrack?.stop();
|
||||
|
||||
await videoEncoder?.flush();
|
||||
await audioEncoder?.flush();
|
||||
muxer.finalize();
|
||||
|
||||
let buffer = muxer.target.buffer;
|
||||
downloadBlob(new Blob([buffer]));
|
||||
|
||||
videoEncoder = null;
|
||||
audioEncoder = null;
|
||||
muxer = null;
|
||||
startTime = null;
|
||||
firstAudioTimestamp = null;
|
||||
|
||||
startRecordingButton.style.display = 'block';
|
||||
};
|
||||
endRecordingButton.addEventListener('click', endRecording);
|
||||
|
||||
const downloadBlob = (blob) => {
|
||||
let url = window.URL.createObjectURL(blob);
|
||||
let a = document.createElement('a');
|
||||
a.style.display = 'none';
|
||||
a.href = url;
|
||||
a.download = 'davinci.mp4';
|
||||
document.body.appendChild(a);
|
||||
a.click();
|
||||
window.URL.revokeObjectURL(url);
|
||||
};
|
||||
|
||||
/** CANVAS DRAWING STUFF */
|
||||
|
||||
ctx.fillStyle = 'white';
|
||||
ctx.fillRect(0, 0, canvas.width, canvas.height);
|
||||
|
||||
let drawing = false;
|
||||
let lastPos = { x: 0, y: 0 };
|
||||
|
||||
const getRelativeMousePos = (e) => {
|
||||
let rect = canvas.getBoundingClientRect();
|
||||
return { x: e.clientX - rect.x, y: e.clientY - rect.y };
|
||||
};
|
||||
|
||||
const drawLine = (from, to) => {
|
||||
ctx.beginPath();
|
||||
ctx.moveTo(from.x, from.y);
|
||||
ctx.lineTo(to.x, to.y);
|
||||
ctx.strokeStyle = 'black';
|
||||
ctx.lineWidth = 3;
|
||||
ctx.lineCap = 'round';
|
||||
ctx.stroke();
|
||||
};
|
||||
|
||||
canvas.addEventListener('pointerdown', (e) => {
|
||||
if (e.button !== 0) return;
|
||||
|
||||
drawing = true;
|
||||
lastPos = getRelativeMousePos(e);
|
||||
drawLine(lastPos, lastPos);
|
||||
});
|
||||
window.addEventListener('pointerup', () => {
|
||||
drawing = false;
|
||||
});
|
||||
window.addEventListener('mousemove', (e) => {
|
||||
if (!drawing) return;
|
||||
|
||||
let newPos = getRelativeMousePos(e);
|
||||
drawLine(lastPos, newPos);
|
||||
lastPos = newPos;
|
||||
});
|
||||
@@ -0,0 +1,61 @@
|
||||
html, body {
|
||||
margin: 0;
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
background: #120d17;
|
||||
color: white;
|
||||
font-family: monospace;
|
||||
}
|
||||
|
||||
body {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
}
|
||||
|
||||
* {
|
||||
user-select: none;
|
||||
}
|
||||
|
||||
main {
|
||||
width: 640px;
|
||||
}
|
||||
|
||||
h1 {
|
||||
margin: 0;
|
||||
font-weight: normal;
|
||||
text-align: center;
|
||||
margin-bottom: 10px;
|
||||
}
|
||||
|
||||
h2 {
|
||||
margin: 0;
|
||||
font-weight: normal;
|
||||
text-align: center;
|
||||
font-size: 14px;
|
||||
margin-bottom: 20px;
|
||||
}
|
||||
|
||||
canvas {
|
||||
border-radius: 10px;
|
||||
outline: 3px solid rgb(202, 202, 202);
|
||||
}
|
||||
|
||||
#controls {
|
||||
margin-bottom: 20px;
|
||||
display: flex;
|
||||
justify-content: center;
|
||||
height: 38px;
|
||||
}
|
||||
|
||||
button {
|
||||
font-size: 20px;
|
||||
padding: 5px 8px;
|
||||
}
|
||||
|
||||
p {
|
||||
margin: 0;
|
||||
text-align: center;
|
||||
margin-top: 20px;
|
||||
height: 20px;
|
||||
}
|
||||
+2051
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,59 @@
|
||||
{
|
||||
"name": "mp4-muxer",
|
||||
"version": "4.3.3",
|
||||
"description": "MP4 multiplexer in pure TypeScript with support for WebCodecs API, video & audio.",
|
||||
"main": "./build/mp4-muxer.js",
|
||||
"module": "./build/mp4-muxer.mjs",
|
||||
"types": "./build/mp4-muxer.d.ts",
|
||||
"exports": {
|
||||
"types": "./build/mp4-muxer.d.ts",
|
||||
"import": "./build/mp4-muxer.mjs",
|
||||
"require": "./build/mp4-muxer.js"
|
||||
},
|
||||
"files": [
|
||||
"README.md",
|
||||
"package.json",
|
||||
"LICENSE",
|
||||
"build/mp4-muxer.js",
|
||||
"build/mp4-muxer.mjs",
|
||||
"build/mp4-muxer.d.ts"
|
||||
],
|
||||
"scripts": {
|
||||
"watch": "node build.mjs",
|
||||
"check": "npx tsc --noEmit --skipLibCheck",
|
||||
"lint": "npx eslint src demo build"
|
||||
},
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "git+https://github.com/Vanilagy/mp4-muxer.git"
|
||||
},
|
||||
"author": "Vanilagy",
|
||||
"license": "MIT",
|
||||
"bugs": {
|
||||
"url": "https://github.com/Vanilagy/mp4-muxer/issues"
|
||||
},
|
||||
"homepage": "https://github.com/Vanilagy/mp4-muxer#readme",
|
||||
"dependencies": {
|
||||
"@types/dom-webcodecs": "^0.1.6",
|
||||
"@types/wicg-file-system-access": "^2020.9.5"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^18.15.0",
|
||||
"@typescript-eslint/eslint-plugin": "^5.54.1",
|
||||
"@typescript-eslint/parser": "^5.54.1",
|
||||
"esbuild": "^0.17.11",
|
||||
"eslint": "^8.36.0",
|
||||
"typescript": "^4.9.5"
|
||||
},
|
||||
"keywords": [
|
||||
"mp4",
|
||||
"fmp4",
|
||||
"muxer",
|
||||
"muxing",
|
||||
"multiplexer",
|
||||
"video",
|
||||
"audio",
|
||||
"media",
|
||||
"webcodecs"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,740 @@
|
||||
import {
|
||||
AudioTrack,
|
||||
GLOBAL_TIMESCALE,
|
||||
SUPPORTED_AUDIO_CODECS,
|
||||
SUPPORTED_VIDEO_CODECS,
|
||||
Sample,
|
||||
Track,
|
||||
VideoTrack
|
||||
} from './muxer';
|
||||
import {
|
||||
ascii,
|
||||
i16,
|
||||
i32,
|
||||
intoTimescale,
|
||||
last,
|
||||
lastPresentedSample,
|
||||
u16,
|
||||
u64,
|
||||
u8,
|
||||
u32,
|
||||
fixed_16_16,
|
||||
fixed_8_8,
|
||||
u24,
|
||||
IDENTITY_MATRIX,
|
||||
matrixToBytes,
|
||||
rotationMatrix,
|
||||
isU32,
|
||||
TransformationMatrix
|
||||
} from './misc';
|
||||
|
||||
export interface Box {
|
||||
type: string,
|
||||
contents?: Uint8Array,
|
||||
children?: Box[],
|
||||
size?: number,
|
||||
largeSize?: boolean
|
||||
}
|
||||
|
||||
type NestedNumberArray = (number | NestedNumberArray)[];
|
||||
|
||||
export const box = (type: string, contents?: NestedNumberArray, children?: Box[]): Box => ({
|
||||
type,
|
||||
contents: contents && new Uint8Array(contents.flat(10) as number[]),
|
||||
children
|
||||
});
|
||||
|
||||
/** A FullBox always starts with a version byte, followed by three flag bytes. */
|
||||
export const fullBox = (
|
||||
type: string,
|
||||
version: number,
|
||||
flags: number,
|
||||
contents?: NestedNumberArray,
|
||||
children?: Box[]
|
||||
) => box(
|
||||
type,
|
||||
[u8(version), u24(flags), contents ?? []],
|
||||
children
|
||||
);
|
||||
|
||||
/**
|
||||
* File Type Compatibility Box: Allows the reader to determine whether this is a type of file that the
|
||||
* reader understands.
|
||||
*/
|
||||
export const ftyp = (details: {
|
||||
holdsAvc: boolean,
|
||||
fragmented: boolean
|
||||
}) => {
|
||||
// You can find the full logic for this at
|
||||
// https://github.com/FFmpeg/FFmpeg/blob/de2fb43e785773738c660cdafb9309b1ef1bc80d/libavformat/movenc.c#L5518
|
||||
// Obviously, this lib only needs a small subset of that logic.
|
||||
|
||||
let minorVersion = 0x200;
|
||||
|
||||
if (details.fragmented) return box('ftyp', [
|
||||
ascii('iso5'), // Major brand
|
||||
u32(minorVersion), // Minor version
|
||||
// Compatible brands
|
||||
ascii('iso5'),
|
||||
ascii('iso6'),
|
||||
ascii('mp41')
|
||||
]);
|
||||
|
||||
return box('ftyp', [
|
||||
ascii('isom'), // Major brand
|
||||
u32(minorVersion), // Minor version
|
||||
// Compatible brands
|
||||
ascii('isom'),
|
||||
details.holdsAvc ? ascii('avc1') : [],
|
||||
ascii('mp41')
|
||||
]);
|
||||
};
|
||||
|
||||
/** Movie Sample Data Box. Contains the actual frames/samples of the media. */
|
||||
export const mdat = (reserveLargeSize: boolean): Box => ({ type: 'mdat', largeSize: reserveLargeSize });
|
||||
|
||||
/** Free Space Box: A box that designates unused space in the movie data file. */
|
||||
export const free = (size: number): Box => ({ type: 'free', size });
|
||||
|
||||
/**
|
||||
* Movie Box: Used to specify the information that defines a movie - that is, the information that allows
|
||||
* an application to interpret the sample data that is stored elsewhere.
|
||||
*/
|
||||
export const moov = (tracks: Track[], creationTime: number, fragmented = false) => box('moov', null, [
|
||||
mvhd(creationTime, tracks),
|
||||
...tracks.map(x => trak(x, creationTime)),
|
||||
fragmented ? mvex(tracks) : null
|
||||
]);
|
||||
|
||||
/** Movie Header Box: Used to specify the characteristics of the entire movie, such as timescale and duration. */
|
||||
export const mvhd = (
|
||||
creationTime: number,
|
||||
tracks: Track[]
|
||||
) => {
|
||||
let duration = intoTimescale(Math.max(
|
||||
0,
|
||||
...tracks.
|
||||
filter(x => x.samples.length > 0).
|
||||
map(x => {
|
||||
const lastSample = lastPresentedSample(x.samples);
|
||||
return lastSample.presentationTimestamp + lastSample.duration;
|
||||
})
|
||||
), GLOBAL_TIMESCALE);
|
||||
let nextTrackId = Math.max(...tracks.map(x => x.id)) + 1;
|
||||
|
||||
// Conditionally use u64 if u32 isn't enough
|
||||
let needsU64 = !isU32(creationTime) || !isU32(duration);
|
||||
let u32OrU64 = needsU64 ? u64 : u32;
|
||||
|
||||
return fullBox('mvhd', +needsU64, 0, [
|
||||
u32OrU64(creationTime), // Creation time
|
||||
u32OrU64(creationTime), // Modification time
|
||||
u32(GLOBAL_TIMESCALE), // Timescale
|
||||
u32OrU64(duration), // Duration
|
||||
fixed_16_16(1), // Preferred rate
|
||||
fixed_8_8(1), // Preferred volume
|
||||
Array(10).fill(0), // Reserved
|
||||
matrixToBytes(IDENTITY_MATRIX), // Matrix
|
||||
Array(24).fill(0), // Pre-defined
|
||||
u32(nextTrackId) // Next track ID
|
||||
]);
|
||||
};
|
||||
|
||||
/**
|
||||
* Track Box: Defines a single track of a movie. A movie may consist of one or more tracks. Each track is
|
||||
* independent of the other tracks in the movie and carries its own temporal and spatial information. Each Track Box
|
||||
* contains its associated Media Box.
|
||||
*/
|
||||
export const trak = (track: Track, creationTime: number) => box('trak', null, [
|
||||
tkhd(track, creationTime),
|
||||
mdia(track, creationTime)
|
||||
]);
|
||||
|
||||
/** Track Header Box: Specifies the characteristics of a single track within a movie. */
|
||||
export const tkhd = (
|
||||
track: Track,
|
||||
creationTime: number
|
||||
) => {
|
||||
let lastSample = lastPresentedSample(track.samples);
|
||||
let durationInGlobalTimescale = intoTimescale(
|
||||
lastSample ? lastSample.presentationTimestamp + lastSample.duration : 0,
|
||||
GLOBAL_TIMESCALE
|
||||
);
|
||||
|
||||
let needsU64 = !isU32(creationTime) || !isU32(durationInGlobalTimescale);
|
||||
let u32OrU64 = needsU64 ? u64 : u32;
|
||||
|
||||
let matrix: TransformationMatrix;
|
||||
if (track.info.type === 'video') {
|
||||
matrix = typeof track.info.rotation === 'number' ? rotationMatrix(track.info.rotation) : track.info.rotation;
|
||||
} else {
|
||||
matrix = IDENTITY_MATRIX;
|
||||
}
|
||||
|
||||
return fullBox('tkhd', +needsU64, 3, [
|
||||
u32OrU64(creationTime), // Creation time
|
||||
u32OrU64(creationTime), // Modification time
|
||||
u32(track.id), // Track ID
|
||||
u32(0), // Reserved
|
||||
u32OrU64(durationInGlobalTimescale), // Duration
|
||||
Array(8).fill(0), // Reserved
|
||||
u16(0), // Layer
|
||||
u16(0), // Alternate group
|
||||
fixed_8_8(track.info.type === 'audio' ? 1 : 0), // Volume
|
||||
u16(0), // Reserved
|
||||
matrixToBytes(matrix), // Matrix
|
||||
fixed_16_16(track.info.type === 'video' ? track.info.width : 0), // Track width
|
||||
fixed_16_16(track.info.type === 'video' ? track.info.height : 0) // Track height
|
||||
]);
|
||||
};
|
||||
|
||||
/** Media Box: Describes and define a track's media type and sample data. */
|
||||
export const mdia = (track: Track, creationTime: number) => box('mdia', null, [
|
||||
mdhd(track, creationTime),
|
||||
hdlr(track.info.type === 'video' ? 'vide' : 'soun'),
|
||||
minf(track)
|
||||
]);
|
||||
|
||||
/** Media Header Box: Specifies the characteristics of a media, including timescale and duration. */
|
||||
export const mdhd = (
|
||||
track: Track,
|
||||
creationTime: number
|
||||
) => {
|
||||
let lastSample = lastPresentedSample(track.samples);
|
||||
let localDuration = intoTimescale(
|
||||
lastSample ? lastSample.presentationTimestamp + lastSample.duration : 0,
|
||||
track.timescale
|
||||
);
|
||||
|
||||
let needsU64 = !isU32(creationTime) || !isU32(localDuration);
|
||||
let u32OrU64 = needsU64 ? u64 : u32;
|
||||
|
||||
return fullBox('mdhd', +needsU64, 0, [
|
||||
u32OrU64(creationTime), // Creation time
|
||||
u32OrU64(creationTime), // Modification time
|
||||
u32(track.timescale), // Timescale
|
||||
u32OrU64(localDuration), // Duration
|
||||
u16(0b01010101_11000100), // Language ("und", undetermined)
|
||||
u16(0) // Quality
|
||||
]);
|
||||
};
|
||||
|
||||
/** Handler Reference Box: Specifies the media handler component that is to be used to interpret the media's data. */
|
||||
export const hdlr = (componentSubtype: string) => fullBox('hdlr', 0, 0, [
|
||||
ascii('mhlr'), // Component type
|
||||
ascii(componentSubtype), // Component subtype
|
||||
u32(0), // Component manufacturer
|
||||
u32(0), // Component flags
|
||||
u32(0), // Component flags mask
|
||||
ascii('mp4-muxer-hdlr', true) // Component name
|
||||
]);
|
||||
|
||||
/**
|
||||
* Media Information Box: Stores handler-specific information for a track's media data. The media handler uses this
|
||||
* information to map from media time to media data and to process the media data.
|
||||
*/
|
||||
export const minf = (track: Track) => box('minf', null, [
|
||||
track.info.type === 'video' ? vmhd() : smhd(),
|
||||
dinf(),
|
||||
stbl(track)
|
||||
]);
|
||||
|
||||
/** Video Media Information Header Box: Defines specific color and graphics mode information. */
|
||||
export const vmhd = () => fullBox('vmhd', 0, 1, [
|
||||
u16(0), // Graphics mode
|
||||
u16(0), // Opcolor R
|
||||
u16(0), // Opcolor G
|
||||
u16(0) // Opcolor B
|
||||
]);
|
||||
|
||||
/** Sound Media Information Header Box: Stores the sound media's control information, such as balance. */
|
||||
export const smhd = () => fullBox('smhd', 0, 0, [
|
||||
u16(0), // Balance
|
||||
u16(0) // Reserved
|
||||
]);
|
||||
|
||||
/**
|
||||
* Data Information Box: Contains information specifying the data handler component that provides access to the
|
||||
* media data. The data handler component uses the Data Information Box to interpret the media's data.
|
||||
*/
|
||||
export const dinf = () => box('dinf', null, [
|
||||
dref()
|
||||
]);
|
||||
|
||||
/**
|
||||
* Data Reference Box: Contains tabular data that instructs the data handler component how to access the media's data.
|
||||
*/
|
||||
export const dref = () => fullBox('dref', 0, 0, [
|
||||
u32(1) // Entry count
|
||||
], [
|
||||
url()
|
||||
]);
|
||||
|
||||
export const url = () => fullBox('url ', 0, 1); // Self-reference flag enabled
|
||||
|
||||
/**
|
||||
* Sample Table Box: Contains information for converting from media time to sample number to sample location. This box
|
||||
* also indicates how to interpret the sample (for example, whether to decompress the video data and, if so, how).
|
||||
*/
|
||||
export const stbl = (track: Track) => {
|
||||
const needsCtts = track.compositionTimeOffsetTable.length > 1 ||
|
||||
track.compositionTimeOffsetTable.some((x) => x.sampleCompositionTimeOffset !== 0);
|
||||
|
||||
return box('stbl', null, [
|
||||
stsd(track),
|
||||
stts(track),
|
||||
stss(track),
|
||||
stsc(track),
|
||||
stsz(track),
|
||||
stco(track),
|
||||
needsCtts ? ctts(track) : null
|
||||
]);
|
||||
};
|
||||
|
||||
/**
|
||||
* Sample Description Box: Stores information that allows you to decode samples in the media. The data stored in the
|
||||
* sample description varies, depending on the media type.
|
||||
*/
|
||||
export const stsd = (track: Track) => fullBox('stsd', 0, 0, [
|
||||
u32(1) // Entry count
|
||||
], [
|
||||
track.info.type === 'video'
|
||||
? videoSampleDescription(
|
||||
VIDEO_CODEC_TO_BOX_NAME[track.info.codec],
|
||||
track as VideoTrack
|
||||
)
|
||||
: soundSampleDescription(
|
||||
AUDIO_CODEC_TO_BOX_NAME[track.info.codec],
|
||||
track as AudioTrack
|
||||
)
|
||||
]);
|
||||
|
||||
/** Video Sample Description Box: Contains information that defines how to interpret video media data. */
|
||||
export const videoSampleDescription = (
|
||||
compressionType: string,
|
||||
track: VideoTrack
|
||||
) => box(compressionType, [
|
||||
Array(6).fill(0), // Reserved
|
||||
u16(1), // Data reference index
|
||||
u16(0), // Pre-defined
|
||||
u16(0), // Reserved
|
||||
Array(12).fill(0), // Pre-defined
|
||||
u16(track.info.width), // Width
|
||||
u16(track.info.height), // Height
|
||||
u32(0x00480000), // Horizontal resolution
|
||||
u32(0x00480000), // Vertical resolution
|
||||
u32(0), // Reserved
|
||||
u16(1), // Frame count
|
||||
Array(32).fill(0), // Compressor name
|
||||
u16(0x0018), // Depth
|
||||
i16(0xffff) // Pre-defined
|
||||
], [
|
||||
VIDEO_CODEC_TO_CONFIGURATION_BOX[track.info.codec](track)
|
||||
]);
|
||||
|
||||
/** AVC Configuration Box: Provides additional information to the decoder. */
|
||||
export const avcC = (track: VideoTrack) => track.info.decoderConfig && box('avcC', [
|
||||
// For AVC, description is an AVCDecoderConfigurationRecord, so nothing else to do here
|
||||
...new Uint8Array(track.info.decoderConfig.description as ArrayBuffer)
|
||||
]);
|
||||
|
||||
/** HEVC Configuration Box: Provides additional information to the decoder. */
|
||||
export const hvcC = (track: VideoTrack) => track.info.decoderConfig && box('hvcC', [
|
||||
// For HEVC, description is a HEVCDecoderConfigurationRecord, so nothing else to do here
|
||||
...new Uint8Array(track.info.decoderConfig.description as ArrayBuffer)
|
||||
]);
|
||||
|
||||
/** VP9 Configuration Box: Provides additional information to the decoder. */
|
||||
export const vpcC = (track: VideoTrack) => {
|
||||
// Reference: https://www.webmproject.org/vp9/mp4/
|
||||
|
||||
if (!track.info.decoderConfig) {
|
||||
return null;
|
||||
}
|
||||
|
||||
let decoderConfig = track.info.decoderConfig;
|
||||
if (!decoderConfig.colorSpace) {
|
||||
throw new Error(`'colorSpace' is required in the decoder config for VP9.`);
|
||||
}
|
||||
|
||||
let parts = decoderConfig.codec.split('.');
|
||||
let profile = Number(parts[1]);
|
||||
let level = Number(parts[2]);
|
||||
|
||||
let bitDepth = Number(parts[3]);
|
||||
let chromaSubsampling = 0;
|
||||
let thirdByte = (bitDepth << 4) + (chromaSubsampling << 1) + Number(decoderConfig.colorSpace.fullRange);
|
||||
|
||||
// Set all to undetermined. We could determine them using the codec color space info, but there's no need.
|
||||
let colourPrimaries = 2;
|
||||
let transferCharacteristics = 2;
|
||||
let matrixCoefficients = 2;
|
||||
|
||||
return fullBox('vpcC', 1, 0, [
|
||||
u8(profile), // Profile
|
||||
u8(level), // Level
|
||||
u8(thirdByte), // Bit depth, chroma subsampling, full range
|
||||
u8(colourPrimaries), // Colour primaries
|
||||
u8(transferCharacteristics), // Transfer characteristics
|
||||
u8(matrixCoefficients), // Matrix coefficients
|
||||
u16(0) // Codec initialization data size
|
||||
]);
|
||||
};
|
||||
|
||||
/** AV1 Configuration Box: Provides additional information to the decoder. */
|
||||
export const av1C = () => {
|
||||
// Reference: https://aomediacodec.github.io/av1-isobmff/
|
||||
|
||||
let marker = 1;
|
||||
let version = 1;
|
||||
let firstByte = (marker << 7) + version;
|
||||
|
||||
// The box contents are not correct like this, but its length is. Getting the values for the last three bytes
|
||||
// requires peeking into the bitstream of the coded chunks. Might come back later.
|
||||
return box('av1C', [
|
||||
firstByte,
|
||||
0,
|
||||
0,
|
||||
0
|
||||
]);
|
||||
};
|
||||
|
||||
/** Sound Sample Description Box: Contains information that defines how to interpret sound media data. */
|
||||
export const soundSampleDescription = (
|
||||
compressionType: string,
|
||||
track: AudioTrack
|
||||
) => box(compressionType, [
|
||||
Array(6).fill(0), // Reserved
|
||||
u16(1), // Data reference index
|
||||
u16(0), // Version
|
||||
u16(0), // Revision level
|
||||
u32(0), // Vendor
|
||||
u16(track.info.numberOfChannels), // Number of channels
|
||||
u16(16), // Sample size (bits)
|
||||
u16(0), // Compression ID
|
||||
u16(0), // Packet size
|
||||
fixed_16_16(track.info.sampleRate) // Sample rate
|
||||
], [
|
||||
AUDIO_CODEC_TO_CONFIGURATION_BOX[track.info.codec](track)
|
||||
]);
|
||||
|
||||
/** MPEG-4 Elementary Stream Descriptor Box. */
|
||||
export const esds = (track: Track) => {
|
||||
let description = new Uint8Array(track.info.decoderConfig.description as ArrayBuffer);
|
||||
|
||||
return fullBox('esds', 0, 0, [
|
||||
// https://stackoverflow.com/a/54803118
|
||||
u32(0x03808080), // TAG(3) = Object Descriptor ([2])
|
||||
u8(0x20 + description.byteLength), // length of this OD (which includes the next 2 tags)
|
||||
u16(1), // ES_ID = 1
|
||||
u8(0x00), // flags etc = 0
|
||||
u32(0x04808080), // TAG(4) = ES Descriptor ([2]) embedded in above OD
|
||||
u8(0x12 + description.byteLength), // length of this ESD
|
||||
u8(0x40), // MPEG-4 Audio
|
||||
u8(0x15), // stream type(6bits)=5 audio, flags(2bits)=1
|
||||
u24(0), // 24bit buffer size
|
||||
u32(0x0001FC17), // max bitrate
|
||||
u32(0x0001FC17), // avg bitrate
|
||||
u32(0x05808080), // TAG(5) = ASC ([2],[3]) embedded in above OD
|
||||
u8(description.byteLength), // length
|
||||
...description,
|
||||
u32(0x06808080), // TAG(6)
|
||||
u8(0x01), // length
|
||||
u8(0x02) // data
|
||||
]);
|
||||
};
|
||||
|
||||
/** Opus Specific Box. */
|
||||
export const dOps = (track: AudioTrack) => box('dOps', [
|
||||
u8(0), // Version
|
||||
u8(track.info.numberOfChannels), // OutputChannelCount
|
||||
u16(3840), // PreSkip, should be at least 80 milliseconds worth of playback, measured in 48000 Hz samples
|
||||
u32(track.info.sampleRate), // InputSampleRate
|
||||
fixed_8_8(0), // OutputGain
|
||||
u8(0) // ChannelMappingFamily
|
||||
]);
|
||||
|
||||
/**
|
||||
* Time-To-Sample Box: Stores duration information for a media's samples, providing a mapping from a time in a media
|
||||
* to the corresponding data sample. The table is compact, meaning that consecutive samples with the same time delta
|
||||
* will be grouped.
|
||||
*/
|
||||
export const stts = (track: Track) => {
|
||||
return fullBox('stts', 0, 0, [
|
||||
u32(track.timeToSampleTable.length), // Number of entries
|
||||
track.timeToSampleTable.map(x => [ // Time-to-sample table
|
||||
u32(x.sampleCount), // Sample count
|
||||
u32(x.sampleDelta) // Sample duration
|
||||
])
|
||||
]);
|
||||
};
|
||||
|
||||
/** Sync Sample Box: Identifies the key frames in the media, marking the random access points within a stream. */
|
||||
export const stss = (track: Track) => {
|
||||
if (track.samples.every(x => x.type === 'key')) return null; // No stss box -> every frame is a key frame
|
||||
|
||||
let keySamples = [...track.samples.entries()].filter(([, sample]) => sample.type === 'key');
|
||||
return fullBox('stss', 0, 0, [
|
||||
u32(keySamples.length), // Number of entries
|
||||
keySamples.map(([index]) => u32(index + 1)) // Sync sample table
|
||||
]);
|
||||
};
|
||||
|
||||
/**
|
||||
* Sample-To-Chunk Box: As samples are added to a media, they are collected into chunks that allow optimized data
|
||||
* access. A chunk contains one or more samples. Chunks in a media may have different sizes, and the samples within a
|
||||
* chunk may have different sizes. The Sample-To-Chunk Box stores chunk information for the samples in a media, stored
|
||||
* in a compactly-coded fashion.
|
||||
*/
|
||||
export const stsc = (track: Track) => {
|
||||
return fullBox('stsc', 0, 0, [
|
||||
u32(track.compactlyCodedChunkTable.length), // Number of entries
|
||||
track.compactlyCodedChunkTable.map(x => [ // Sample-to-chunk table
|
||||
u32(x.firstChunk), // First chunk
|
||||
u32(x.samplesPerChunk), // Samples per chunk
|
||||
u32(1) // Sample description index
|
||||
])
|
||||
]);
|
||||
};
|
||||
|
||||
/** Sample Size Box: Specifies the byte size of each sample in the media. */
|
||||
export const stsz = (track: Track) => fullBox('stsz', 0, 0, [
|
||||
u32(0), // Sample size (0 means non-constant size)
|
||||
u32(track.samples.length), // Number of entries
|
||||
track.samples.map(x => u32(x.size)) // Sample size table
|
||||
]);
|
||||
|
||||
/** Chunk Offset Box: Identifies the location of each chunk of data in the media's data stream, relative to the file. */
|
||||
export const stco = (track: Track) => {
|
||||
if (track.finalizedChunks.length > 0 && last(track.finalizedChunks).offset >= 2**32) {
|
||||
// If the file is large, use the co64 box
|
||||
return fullBox('co64', 0, 0, [
|
||||
u32(track.finalizedChunks.length), // Number of entries
|
||||
track.finalizedChunks.map(x => u64(x.offset)) // Chunk offset table
|
||||
]);
|
||||
}
|
||||
|
||||
return fullBox('stco', 0, 0, [
|
||||
u32(track.finalizedChunks.length), // Number of entries
|
||||
track.finalizedChunks.map(x => u32(x.offset)) // Chunk offset table
|
||||
]);
|
||||
};
|
||||
|
||||
/** Composition Time to Sample Box: Stores composition time offset information (PTS-DTS) for a
|
||||
* media's samples. The table is compact, meaning that consecutive samples with the same time
|
||||
* composition time offset will be grouped. */
|
||||
export const ctts = (track: Track) => {
|
||||
return fullBox('ctts', 0, 0, [
|
||||
u32(track.compositionTimeOffsetTable.length), // Number of entries
|
||||
track.compositionTimeOffsetTable.map(x => [ // Time-to-sample table
|
||||
u32(x.sampleCount), // Sample count
|
||||
u32(x.sampleCompositionTimeOffset) // Sample offset
|
||||
])
|
||||
]);
|
||||
};
|
||||
|
||||
/**
|
||||
* Movie Extends Box: This box signals to readers that the file is fragmented. Contains a single Track Extends Box
|
||||
* for each track in the movie.
|
||||
*/
|
||||
export const mvex = (tracks: Track[]) => {
|
||||
return box('mvex', null, tracks.map(trex));
|
||||
};
|
||||
|
||||
/** Track Extends Box: Contains the default values used by the movie fragments. */
|
||||
export const trex = (track: Track) => {
|
||||
return fullBox('trex', 0, 0, [
|
||||
u32(track.id), // Track ID
|
||||
u32(1), // Default sample description index
|
||||
u32(0), // Default sample duration
|
||||
u32(0), // Default sample size
|
||||
u32(0) // Default sample flags
|
||||
]);
|
||||
};
|
||||
|
||||
/**
|
||||
* Movie Fragment Box: The movie fragments extend the presentation in time. They provide the information that would
|
||||
* previously have been in the Movie Box.
|
||||
*/
|
||||
export const moof = (sequenceNumber: number, tracks: Track[]) => {
|
||||
return box('moof', null, [
|
||||
mfhd(sequenceNumber),
|
||||
...tracks.map(traf)
|
||||
]);
|
||||
};
|
||||
|
||||
/** Movie Fragment Header Box: Contains a sequence number as a safety check. */
|
||||
export const mfhd = (sequenceNumber: number) => {
|
||||
return fullBox('mfhd', 0, 0, [
|
||||
u32(sequenceNumber) // Sequence number
|
||||
]);
|
||||
};
|
||||
|
||||
const fragmentSampleFlags = (sample: Sample) => {
|
||||
let byte1 = 0;
|
||||
let byte2 = 0;
|
||||
let byte3 = 0;
|
||||
let byte4 = 0;
|
||||
|
||||
let sampleIsDifferenceSample = sample.type === 'delta';
|
||||
byte2 |= +sampleIsDifferenceSample;
|
||||
|
||||
if (sampleIsDifferenceSample) {
|
||||
byte1 |= 1; // There is redundant coding in this sample
|
||||
} else {
|
||||
byte1 |= 2; // There is no redundant coding in this sample
|
||||
}
|
||||
|
||||
// Note that there are a lot of other flags to potentially set here, but most are irrelevant / non-necessary
|
||||
return byte1 << 24 | byte2 << 16 | byte3 << 8 | byte4;
|
||||
};
|
||||
|
||||
/** Track Fragment Box */
|
||||
export const traf = (track: Track) => {
|
||||
return box('traf', null, [
|
||||
tfhd(track),
|
||||
tfdt(track),
|
||||
trun(track)
|
||||
]);
|
||||
};
|
||||
|
||||
/** Track Fragment Header Box: Provides a reference to the extended track, and flags. */
|
||||
export const tfhd = (track: Track) => {
|
||||
let tfFlags = 0;
|
||||
tfFlags |= 0x00008; // Default sample duration present
|
||||
tfFlags |= 0x00010; // Default sample size present
|
||||
tfFlags |= 0x00020; // Default sample flags present
|
||||
tfFlags |= 0x20000; // Default base is moof
|
||||
|
||||
// Prefer the second sample over the first one, as the first one is a sync sample and therefore the "odd one out"
|
||||
let referenceSample = track.currentChunk.samples[1] ?? track.currentChunk.samples[0];
|
||||
let referenceSampleInfo = {
|
||||
duration: referenceSample.timescaleUnitsToNextSample,
|
||||
size: referenceSample.size,
|
||||
flags: fragmentSampleFlags(referenceSample)
|
||||
};
|
||||
|
||||
return fullBox('tfhd', 0, tfFlags, [
|
||||
u32(track.id), // Track ID
|
||||
u32(referenceSampleInfo.duration), // Default sample duration
|
||||
u32(referenceSampleInfo.size), // Default sample size
|
||||
u32(referenceSampleInfo.flags) // Default sample flags
|
||||
]);
|
||||
};
|
||||
|
||||
/**
|
||||
* Track Fragment Decode Time Box: Provides the absolute decode time of the first sample of the fragment. This is
|
||||
* useful for performing random access on the media file.
|
||||
*/
|
||||
export const tfdt = (track: Track) => {
|
||||
return fullBox('tfdt', 1, 0, [
|
||||
u64(intoTimescale(track.currentChunk.startTimestamp, track.timescale)) // Base Media Decode Time
|
||||
]);
|
||||
};
|
||||
|
||||
/** Track Run Box: Specifies a run of contiguous samples for a given track. */
|
||||
export const trun = (track: Track) => {
|
||||
let allSampleDurations = track.currentChunk.samples.map(x => x.timescaleUnitsToNextSample);
|
||||
let allSampleSizes = track.currentChunk.samples.map(x => x.size);
|
||||
let allSampleFlags = track.currentChunk.samples.map(fragmentSampleFlags);
|
||||
let allSampleCompositionTimeOffsets = track.currentChunk.samples.
|
||||
map(x => intoTimescale(x.presentationTimestamp - x.decodeTimestamp, track.timescale));
|
||||
|
||||
let uniqueSampleDurations = new Set(allSampleDurations);
|
||||
let uniqueSampleSizes = new Set(allSampleSizes);
|
||||
let uniqueSampleFlags = new Set(allSampleFlags);
|
||||
let uniqueSampleCompositionTimeOffsets = new Set(allSampleCompositionTimeOffsets);
|
||||
|
||||
let firstSampleFlagsPresent = uniqueSampleFlags.size === 2 && allSampleFlags[0] !== allSampleFlags[1];
|
||||
let sampleDurationPresent = uniqueSampleDurations.size > 1;
|
||||
let sampleSizePresent = uniqueSampleSizes.size > 1;
|
||||
let sampleFlagsPresent = !firstSampleFlagsPresent && uniqueSampleFlags.size > 1;
|
||||
let sampleCompositionTimeOffsetsPresent =
|
||||
uniqueSampleCompositionTimeOffsets.size > 1 || [...uniqueSampleCompositionTimeOffsets].some(x => x !== 0);
|
||||
|
||||
let flags = 0;
|
||||
flags |= 0x0001; // Data offset present
|
||||
flags |= 0x0004 * +firstSampleFlagsPresent; // First sample flags present
|
||||
flags |= 0x0100 * +sampleDurationPresent; // Sample duration present
|
||||
flags |= 0x0200 * +sampleSizePresent; // Sample size present
|
||||
flags |= 0x0400 * +sampleFlagsPresent; // Sample flags present
|
||||
flags |= 0x0800 * +sampleCompositionTimeOffsetsPresent; // Sample composition time offsets present
|
||||
|
||||
return fullBox('trun', 1, flags, [
|
||||
u32(track.currentChunk.samples.length), // Sample count
|
||||
u32(track.currentChunk.offset - track.currentChunk.moofOffset || 0), // Data offset
|
||||
firstSampleFlagsPresent ? u32(allSampleFlags[0]) : [],
|
||||
track.currentChunk.samples.map((_, i) => [
|
||||
sampleDurationPresent ? u32(allSampleDurations[i]) : [], // Sample duration
|
||||
sampleSizePresent ? u32(allSampleSizes[i]) : [], // Sample size
|
||||
sampleFlagsPresent ? u32(allSampleFlags[i]) : [], // Sample flags
|
||||
// Sample composition time offsets
|
||||
sampleCompositionTimeOffsetsPresent ? i32(allSampleCompositionTimeOffsets[i]) : []
|
||||
])
|
||||
]);
|
||||
};
|
||||
|
||||
/**
|
||||
* Movie Fragment Random Access Box: For each track, provides pointers to sync samples within the file
|
||||
* for random access.
|
||||
*/
|
||||
export const mfra = (tracks: Track[]) => {
|
||||
return box('mfra', null, [
|
||||
...tracks.map(tfra),
|
||||
mfro()
|
||||
]);
|
||||
};
|
||||
|
||||
/** Track Fragment Random Access Box: Provides pointers to sync samples within the file for random access. */
|
||||
export const tfra = (track: Track, trackIndex: number) => {
|
||||
let version = 1; // Using this version allows us to use 64-bit time and offset values
|
||||
|
||||
return fullBox('tfra', version, 0, [
|
||||
u32(track.id), // Track ID
|
||||
u32(0b111111), // This specifies that traf number, trun number and sample number are 32-bit ints
|
||||
u32(track.finalizedChunks.length), // Number of entries
|
||||
track.finalizedChunks.map(chunk => [
|
||||
u64(intoTimescale(chunk.startTimestamp, track.timescale)), // Time
|
||||
u64(chunk.moofOffset), // moof offset
|
||||
u32(trackIndex + 1), // traf number
|
||||
u32(1), // trun number
|
||||
u32(1) // Sample number
|
||||
])
|
||||
]);
|
||||
};
|
||||
|
||||
/**
|
||||
* Movie Fragment Random Access Offset Box: Provides the size of the enclosing mfra box. This box can be used by readers
|
||||
* to quickly locate the mfra box by searching from the end of the file.
|
||||
*/
|
||||
export const mfro = () => {
|
||||
return fullBox('mfro', 0, 0, [
|
||||
// This value needs to be overwritten manually from the outside, where the actual size of the enclosing mfra box
|
||||
// is known
|
||||
u32(0) // Size
|
||||
]);
|
||||
};
|
||||
|
||||
const VIDEO_CODEC_TO_BOX_NAME: Record<typeof SUPPORTED_VIDEO_CODECS[number], string> = {
|
||||
'avc': 'avc1',
|
||||
'hevc': 'hvc1',
|
||||
'vp9': 'vp09',
|
||||
'av1': 'av01'
|
||||
};
|
||||
|
||||
const VIDEO_CODEC_TO_CONFIGURATION_BOX: Record<typeof SUPPORTED_VIDEO_CODECS[number], (track: VideoTrack) => Box> = {
|
||||
'avc': avcC,
|
||||
'hevc': hvcC,
|
||||
'vp9': vpcC,
|
||||
'av1': av1C
|
||||
};
|
||||
|
||||
const AUDIO_CODEC_TO_BOX_NAME: Record<typeof SUPPORTED_AUDIO_CODECS[number], string> = {
|
||||
'aac': 'mp4a',
|
||||
'opus': 'Opus'
|
||||
};
|
||||
|
||||
const AUDIO_CODEC_TO_CONFIGURATION_BOX: Record<typeof SUPPORTED_AUDIO_CODECS[number], (track: AudioTrack) => Box> = {
|
||||
'aac': esds,
|
||||
'opus': dOps
|
||||
};
|
||||
@@ -0,0 +1,2 @@
|
||||
export { Muxer } from './muxer';
|
||||
export * from './target';
|
||||
@@ -0,0 +1,117 @@
|
||||
import { Sample } from './muxer';
|
||||
|
||||
let bytes = new Uint8Array(8);
|
||||
let view = new DataView(bytes.buffer);
|
||||
|
||||
export const u8 = (value: number) => {
|
||||
return [(value % 0x100 + 0x100) % 0x100];
|
||||
};
|
||||
|
||||
export const u16 = (value: number) => {
|
||||
view.setUint16(0, value, false);
|
||||
return [bytes[0], bytes[1]];
|
||||
};
|
||||
|
||||
export const i16 = (value: number) => {
|
||||
view.setInt16(0, value, false);
|
||||
return [bytes[0], bytes[1]];
|
||||
};
|
||||
|
||||
export const u24 = (value: number) => {
|
||||
view.setUint32(0, value, false);
|
||||
return [bytes[1], bytes[2], bytes[3]];
|
||||
};
|
||||
|
||||
export const u32 = (value: number) => {
|
||||
view.setUint32(0, value, false);
|
||||
return [bytes[0], bytes[1], bytes[2], bytes[3]];
|
||||
};
|
||||
|
||||
export const i32 = (value: number) => {
|
||||
view.setInt32(0, value, false);
|
||||
return [bytes[0], bytes[1], bytes[2], bytes[3]];
|
||||
};
|
||||
|
||||
export const u64 = (value: number) => {
|
||||
view.setUint32(0, Math.floor(value / 2**32), false);
|
||||
view.setUint32(4, value, false);
|
||||
return [bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], bytes[5], bytes[6], bytes[7]];
|
||||
};
|
||||
|
||||
export const fixed_8_8 = (value: number) => {
|
||||
view.setInt16(0, 2**8 * value, false);
|
||||
return [bytes[0], bytes[1]];
|
||||
};
|
||||
|
||||
export const fixed_16_16 = (value: number) => {
|
||||
view.setInt32(0, 2**16 * value, false);
|
||||
return [bytes[0], bytes[1], bytes[2], bytes[3]];
|
||||
};
|
||||
|
||||
export const fixed_2_30 = (value: number) => {
|
||||
view.setInt32(0, 2**30 * value, false);
|
||||
return [bytes[0], bytes[1], bytes[2], bytes[3]];
|
||||
};
|
||||
|
||||
export const ascii = (text: string, nullTerminated = false) => {
|
||||
let bytes = Array(text.length).fill(null).map((_, i) => text.charCodeAt(i));
|
||||
if (nullTerminated) bytes.push(0x00);
|
||||
return bytes;
|
||||
};
|
||||
|
||||
export const last = <T>(arr: T[]) => {
|
||||
return arr && arr[arr.length - 1];
|
||||
};
|
||||
|
||||
export const lastPresentedSample = (samples: Sample[]): Sample | undefined => {
|
||||
let result: Sample | undefined = undefined;
|
||||
|
||||
for (let sample of samples) {
|
||||
if (!result || sample.presentationTimestamp > result.presentationTimestamp) {
|
||||
result = sample;
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
};
|
||||
|
||||
export const intoTimescale = (timeInSeconds: number, timescale: number, round = true) => {
|
||||
let value = timeInSeconds * timescale;
|
||||
return round ? Math.round(value) : value;
|
||||
};
|
||||
|
||||
export type TransformationMatrix = [number, number, number, number, number, number, number, number, number];
|
||||
|
||||
export const rotationMatrix = (rotationInDegrees: number): TransformationMatrix => {
|
||||
let theta = rotationInDegrees * (Math.PI / 180);
|
||||
let cosTheta = Math.cos(theta);
|
||||
let sinTheta = Math.sin(theta);
|
||||
|
||||
// Matrices are post-multiplied in MP4, meaning this is the transpose of your typical rotation matrix
|
||||
return [
|
||||
cosTheta, sinTheta, 0,
|
||||
-sinTheta, cosTheta, 0,
|
||||
0, 0, 1
|
||||
];
|
||||
};
|
||||
|
||||
export const IDENTITY_MATRIX = rotationMatrix(0);
|
||||
|
||||
export const matrixToBytes = (matrix: TransformationMatrix) => {
|
||||
return [
|
||||
fixed_16_16(matrix[0]), fixed_16_16(matrix[1]), fixed_2_30(matrix[2]),
|
||||
fixed_16_16(matrix[3]), fixed_16_16(matrix[4]), fixed_2_30(matrix[5]),
|
||||
fixed_16_16(matrix[6]), fixed_16_16(matrix[7]), fixed_2_30(matrix[8])
|
||||
];
|
||||
};
|
||||
|
||||
export const deepClone = <T>(x: T): T => {
|
||||
if (!x) return x;
|
||||
if (typeof x !== 'object') return x;
|
||||
if (Array.isArray(x)) return x.map(deepClone) as T;
|
||||
return Object.fromEntries(Object.entries(x).map(([key, value]) => [key, deepClone(value)])) as T;
|
||||
};
|
||||
|
||||
export const isU32 = (value: number) => {
|
||||
return value >= 0 && value < 2**32;
|
||||
};
|
||||
@@ -0,0 +1,842 @@
|
||||
import { Box, free, ftyp, mdat, mfra, moof, moov } from './box';
|
||||
import { deepClone, intoTimescale, last, TransformationMatrix } from './misc';
|
||||
import { ArrayBufferTarget, FileSystemWritableFileStreamTarget, StreamTarget, Target } from './target';
|
||||
import {
|
||||
Writer,
|
||||
ArrayBufferTargetWriter,
|
||||
StreamTargetWriter,
|
||||
ChunkedStreamTargetWriter,
|
||||
FileSystemWritableFileStreamTargetWriter
|
||||
} from './writer';
|
||||
|
||||
export const GLOBAL_TIMESCALE = 1000;
|
||||
export const SUPPORTED_VIDEO_CODECS = ['avc', 'hevc', 'vp9', 'av1'] as const;
|
||||
export const SUPPORTED_AUDIO_CODECS = ['aac', 'opus'] as const;
|
||||
const TIMESTAMP_OFFSET = 2_082_844_800; // Seconds between Jan 1 1904 and Jan 1 1970
|
||||
const FIRST_TIMESTAMP_BEHAVIORS = ['strict', 'offset', 'cross-track-offset'] as const;
|
||||
|
||||
interface VideoOptions {
|
||||
codec: typeof SUPPORTED_VIDEO_CODECS[number],
|
||||
width: number,
|
||||
height: number,
|
||||
rotation?: 0 | 90 | 180 | 270 | TransformationMatrix
|
||||
}
|
||||
|
||||
interface AudioOptions {
|
||||
codec: typeof SUPPORTED_AUDIO_CODECS[number],
|
||||
numberOfChannels: number,
|
||||
sampleRate: number
|
||||
}
|
||||
|
||||
type Mp4MuxerOptions<T extends Target> = {
|
||||
target: T,
|
||||
video?: VideoOptions,
|
||||
audio?: AudioOptions,
|
||||
fastStart: false | 'in-memory' | 'fragmented' | {
|
||||
expectedVideoChunks?: number,
|
||||
expectedAudioChunks?: number
|
||||
},
|
||||
firstTimestampBehavior?: typeof FIRST_TIMESTAMP_BEHAVIORS[number]
|
||||
};
|
||||
|
||||
export interface Track {
|
||||
id: number,
|
||||
info: {
|
||||
type: 'video',
|
||||
codec: VideoOptions['codec'],
|
||||
width: number,
|
||||
height: number,
|
||||
rotation: 0 | 90 | 180 | 270 | TransformationMatrix,
|
||||
decoderConfig: VideoDecoderConfig
|
||||
} | {
|
||||
type: 'audio',
|
||||
codec: AudioOptions['codec'],
|
||||
numberOfChannels: number,
|
||||
sampleRate: number,
|
||||
decoderConfig: AudioDecoderConfig
|
||||
},
|
||||
timescale: number,
|
||||
samples: Sample[],
|
||||
|
||||
firstDecodeTimestamp: number,
|
||||
lastDecodeTimestamp: number,
|
||||
|
||||
timeToSampleTable: { sampleCount: number, sampleDelta: number }[];
|
||||
compositionTimeOffsetTable: { sampleCount: number, sampleCompositionTimeOffset: number }[];
|
||||
lastTimescaleUnits: number,
|
||||
lastSample: Sample,
|
||||
|
||||
finalizedChunks: Chunk[],
|
||||
currentChunk: Chunk,
|
||||
compactlyCodedChunkTable: {
|
||||
firstChunk: number,
|
||||
samplesPerChunk: number
|
||||
}[]
|
||||
}
|
||||
|
||||
export type VideoTrack = Track & { info: { type: 'video' } };
|
||||
export type AudioTrack = Track & { info: { type: 'audio' } };
|
||||
|
||||
export interface Sample {
|
||||
presentationTimestamp: number,
|
||||
decodeTimestamp: number,
|
||||
duration: number,
|
||||
data: Uint8Array,
|
||||
size: number,
|
||||
type: 'key' | 'delta',
|
||||
timescaleUnitsToNextSample: number
|
||||
}
|
||||
|
||||
interface Chunk {
|
||||
startTimestamp: number,
|
||||
samples: Sample[],
|
||||
offset?: number,
|
||||
// In the case of a fragmented file, this indicates the position of the moof box pointing to the data in this chunk
|
||||
moofOffset?: number
|
||||
}
|
||||
|
||||
export class Muxer<T extends Target> {
|
||||
target: T;
|
||||
|
||||
#options: Mp4MuxerOptions<T>;
|
||||
#writer: Writer;
|
||||
#ftypSize: number;
|
||||
#mdat: Box;
|
||||
|
||||
#videoTrack: Track = null;
|
||||
#audioTrack: Track = null;
|
||||
#creationTime = Math.floor(Date.now() / 1000) + TIMESTAMP_OFFSET;
|
||||
#finalizedChunks: Chunk[] = [];
|
||||
|
||||
// Fields for fragmented MP4:
|
||||
#nextFragmentNumber = 1;
|
||||
#videoSampleQueue: Sample[] = [];
|
||||
#audioSampleQueue: Sample[] = [];
|
||||
|
||||
#finalized = false;
|
||||
|
||||
constructor(options: Mp4MuxerOptions<T>) {
|
||||
this.#validateOptions(options);
|
||||
|
||||
// Don't want these to be modified from the outside while processing:
|
||||
options.video = deepClone(options.video);
|
||||
options.audio = deepClone(options.audio);
|
||||
options.fastStart = deepClone(options.fastStart);
|
||||
|
||||
this.target = options.target;
|
||||
this.#options = {
|
||||
firstTimestampBehavior: 'strict',
|
||||
...options
|
||||
};
|
||||
|
||||
if (options.target instanceof ArrayBufferTarget) {
|
||||
this.#writer = new ArrayBufferTargetWriter(options.target);
|
||||
} else if (options.target instanceof StreamTarget) {
|
||||
this.#writer = options.target.options?.chunked
|
||||
? new ChunkedStreamTargetWriter(options.target)
|
||||
: new StreamTargetWriter(options.target);
|
||||
} else if (options.target instanceof FileSystemWritableFileStreamTarget) {
|
||||
this.#writer = new FileSystemWritableFileStreamTargetWriter(options.target);
|
||||
} else {
|
||||
throw new Error(`Invalid target: ${options.target}`);
|
||||
}
|
||||
|
||||
this.#prepareTracks();
|
||||
this.#writeHeader();
|
||||
}
|
||||
|
||||
#validateOptions(options: Mp4MuxerOptions<T>) {
|
||||
if (options.video) {
|
||||
if (!SUPPORTED_VIDEO_CODECS.includes(options.video.codec)) {
|
||||
throw new Error(`Unsupported video codec: ${options.video.codec}`);
|
||||
}
|
||||
|
||||
const videoRotation = options.video.rotation;
|
||||
if (typeof videoRotation === 'number' && ![0, 90, 180, 270].includes(videoRotation)) {
|
||||
throw new Error(`Invalid video rotation: ${videoRotation}. Has to be 0, 90, 180 or 270.`);
|
||||
} else if (
|
||||
Array.isArray(videoRotation) &&
|
||||
(videoRotation.length !== 9 || videoRotation.some(value => typeof value !== 'number'))
|
||||
) {
|
||||
throw new Error(`Invalid video transformation matrix: ${videoRotation.join()}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (options.audio && !SUPPORTED_AUDIO_CODECS.includes(options.audio.codec)) {
|
||||
throw new Error(`Unsupported audio codec: ${options.audio.codec}`);
|
||||
}
|
||||
|
||||
if (options.firstTimestampBehavior && !FIRST_TIMESTAMP_BEHAVIORS.includes(options.firstTimestampBehavior)) {
|
||||
throw new Error(`Invalid first timestamp behavior: ${options.firstTimestampBehavior}`);
|
||||
}
|
||||
|
||||
if (typeof options.fastStart === 'object') {
|
||||
if (options.video && options.fastStart.expectedVideoChunks === undefined) {
|
||||
throw new Error(`'fastStart' is an object but is missing property 'expectedVideoChunks'.`);
|
||||
}
|
||||
|
||||
if (options.audio && options.fastStart.expectedAudioChunks === undefined) {
|
||||
throw new Error(`'fastStart' is an object but is missing property 'expectedAudioChunks'.`);
|
||||
}
|
||||
} else if (![false, 'in-memory', 'fragmented'].includes(options.fastStart)) {
|
||||
throw new Error(`'fastStart' option must be false, 'in-memory', 'fragmented' or an object.`);
|
||||
}
|
||||
}
|
||||
|
||||
#writeHeader() {
|
||||
this.#writer.writeBox(ftyp({
|
||||
holdsAvc: this.#options.video?.codec === 'avc',
|
||||
fragmented: this.#options.fastStart === 'fragmented'
|
||||
}));
|
||||
|
||||
this.#ftypSize = this.#writer.pos;
|
||||
|
||||
if (this.#options.fastStart === 'in-memory') {
|
||||
this.#mdat = mdat(false);
|
||||
} else if (this.#options.fastStart === 'fragmented') {
|
||||
// We write the moov box once we write out the first fragment to make sure we get the decoder configs
|
||||
} else {
|
||||
if (typeof this.#options.fastStart === 'object') {
|
||||
let moovSizeUpperBound = this.#computeMoovSizeUpperBound();
|
||||
this.#writer.seek(this.#writer.pos + moovSizeUpperBound);
|
||||
}
|
||||
|
||||
this.#mdat = mdat(true); // Reserve large size by default, can refine this when finalizing.
|
||||
this.#writer.writeBox(this.#mdat);
|
||||
}
|
||||
|
||||
this.#maybeFlushStreamingTargetWriter();
|
||||
}
|
||||
|
||||
#computeMoovSizeUpperBound() {
|
||||
if (typeof this.#options.fastStart !== 'object') return;
|
||||
|
||||
let upperBound = 0;
|
||||
let sampleCounts = [
|
||||
this.#options.fastStart.expectedVideoChunks,
|
||||
this.#options.fastStart.expectedAudioChunks
|
||||
];
|
||||
|
||||
for (let n of sampleCounts) {
|
||||
if (!n) continue;
|
||||
|
||||
// Given the max allowed sample count, compute the space they'll take up in the Sample Table Box, assuming
|
||||
// the worst case for each individual box:
|
||||
|
||||
// stts box - since it is compactly coded, the maximum length of this table will be 2/3n
|
||||
upperBound += (4 + 4) * Math.ceil(2/3 * n);
|
||||
// stss box - 1 entry per sample
|
||||
upperBound += 4 * n;
|
||||
// stsc box - since it is compactly coded, the maximum length of this table will be 2/3n
|
||||
upperBound += (4 + 4 + 4) * Math.ceil(2/3 * n);
|
||||
// stsz box - 1 entry per sample
|
||||
upperBound += 4 * n;
|
||||
// co64 box - we assume 1 sample per chunk and 64-bit chunk offsets
|
||||
upperBound += 8 * n;
|
||||
}
|
||||
|
||||
upperBound += 4096; // Assume a generous 4 kB for everything else: Track metadata, codec descriptors, etc.
|
||||
|
||||
return upperBound;
|
||||
}
|
||||
|
||||
#prepareTracks() {
|
||||
if (this.#options.video) {
|
||||
this.#videoTrack = {
|
||||
id: 1,
|
||||
info: {
|
||||
type: 'video',
|
||||
codec: this.#options.video.codec,
|
||||
width: this.#options.video.width,
|
||||
height: this.#options.video.height,
|
||||
rotation: this.#options.video.rotation ?? 0,
|
||||
decoderConfig: null
|
||||
},
|
||||
timescale: 11520, // Timescale used by FFmpeg, contains many common frame rates as factors
|
||||
samples: [],
|
||||
finalizedChunks: [],
|
||||
currentChunk: null,
|
||||
firstDecodeTimestamp: undefined,
|
||||
lastDecodeTimestamp: -1,
|
||||
timeToSampleTable: [],
|
||||
compositionTimeOffsetTable: [],
|
||||
lastTimescaleUnits: null,
|
||||
lastSample: null,
|
||||
compactlyCodedChunkTable: []
|
||||
};
|
||||
}
|
||||
|
||||
if (this.#options.audio) {
|
||||
// For the case that we don't get any further decoder details, we can still make a pretty educated guess:
|
||||
let guessedCodecPrivate = this.#generateMpeg4AudioSpecificConfig(
|
||||
2, // Object type for AAC-LC, since it's the most common
|
||||
this.#options.audio.sampleRate,
|
||||
this.#options.audio.numberOfChannels
|
||||
);
|
||||
|
||||
this.#audioTrack = {
|
||||
id: this.#options.video ? 2 : 1,
|
||||
info: {
|
||||
type: 'audio',
|
||||
codec: this.#options.audio.codec,
|
||||
numberOfChannels: this.#options.audio.numberOfChannels,
|
||||
sampleRate: this.#options.audio.sampleRate,
|
||||
decoderConfig: {
|
||||
codec: this.#options.audio.codec,
|
||||
description: guessedCodecPrivate,
|
||||
numberOfChannels: this.#options.audio.numberOfChannels,
|
||||
sampleRate: this.#options.audio.sampleRate
|
||||
}
|
||||
},
|
||||
timescale: this.#options.audio.sampleRate,
|
||||
samples: [],
|
||||
finalizedChunks: [],
|
||||
currentChunk: null,
|
||||
firstDecodeTimestamp: undefined,
|
||||
lastDecodeTimestamp: -1,
|
||||
timeToSampleTable: [],
|
||||
compositionTimeOffsetTable: [],
|
||||
lastTimescaleUnits: null,
|
||||
lastSample: null,
|
||||
compactlyCodedChunkTable: []
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// https://wiki.multimedia.cx/index.php/MPEG-4_Audio
|
||||
#generateMpeg4AudioSpecificConfig(objectType: number, sampleRate: number, numberOfChannels: number) {
|
||||
let frequencyIndices =
|
||||
[96000, 88200, 64000, 48000, 44100, 32000, 24000, 22050, 16000, 12000, 11025, 8000, 7350];
|
||||
let frequencyIndex = frequencyIndices.indexOf(sampleRate);
|
||||
let channelConfig = numberOfChannels;
|
||||
|
||||
let configBits = '';
|
||||
configBits += objectType.toString(2).padStart(5, '0');
|
||||
|
||||
configBits += frequencyIndex.toString(2).padStart(4, '0');
|
||||
if (frequencyIndex === 15) configBits += sampleRate.toString(2).padStart(24, '0');
|
||||
|
||||
configBits += channelConfig.toString(2).padStart(4, '0');
|
||||
|
||||
// Pad with 0 bits to fit into a multiple of bytes
|
||||
let paddingLength = Math.ceil(configBits.length / 8) * 8;
|
||||
configBits = configBits.padEnd(paddingLength, '0');
|
||||
|
||||
let configBytes = new Uint8Array(configBits.length / 8);
|
||||
for (let i = 0; i < configBits.length; i += 8) {
|
||||
configBytes[i / 8] = parseInt(configBits.slice(i, i + 8), 2);
|
||||
}
|
||||
|
||||
return configBytes;
|
||||
}
|
||||
|
||||
addVideoChunk(
|
||||
sample: EncodedVideoChunk,
|
||||
meta?: EncodedVideoChunkMetadata,
|
||||
timestamp?: number,
|
||||
compositionTimeOffset?: number
|
||||
) {
|
||||
let data = new Uint8Array(sample.byteLength);
|
||||
sample.copyTo(data);
|
||||
|
||||
this.addVideoChunkRaw(
|
||||
data, sample.type, timestamp ?? sample.timestamp, sample.duration, meta, compositionTimeOffset
|
||||
);
|
||||
}
|
||||
|
||||
addVideoChunkRaw(
|
||||
data: Uint8Array,
|
||||
type: 'key' | 'delta',
|
||||
timestamp: number,
|
||||
duration: number,
|
||||
meta?: EncodedVideoChunkMetadata,
|
||||
compositionTimeOffset?: number
|
||||
) {
|
||||
this.#ensureNotFinalized();
|
||||
if (!this.#options.video) throw new Error('No video track declared.');
|
||||
|
||||
if (
|
||||
typeof this.#options.fastStart === 'object' &&
|
||||
this.#videoTrack.samples.length === this.#options.fastStart.expectedVideoChunks
|
||||
) {
|
||||
throw new Error(`Cannot add more video chunks than specified in 'fastStart' (${
|
||||
this.#options.fastStart.expectedVideoChunks
|
||||
}).`);
|
||||
}
|
||||
|
||||
let videoSample = this.#createSampleForTrack(
|
||||
this.#videoTrack, data, type, timestamp, duration, meta, compositionTimeOffset
|
||||
);
|
||||
|
||||
// Check if we need to interleave the samples in the case of a fragmented file
|
||||
if (this.#options.fastStart === 'fragmented' && this.#audioTrack) {
|
||||
// Add all audio samples with a timestamp smaller than the incoming video sample
|
||||
while (
|
||||
this.#audioSampleQueue.length > 0 &&
|
||||
this.#audioSampleQueue[0].decodeTimestamp <= videoSample.decodeTimestamp
|
||||
) {
|
||||
let audioSample = this.#audioSampleQueue.shift();
|
||||
this.#addSampleToTrack(this.#audioTrack, audioSample);
|
||||
}
|
||||
|
||||
// Depending on the last audio sample, either add the video sample to the file or enqueue it
|
||||
if (videoSample.decodeTimestamp <= this.#audioTrack.lastDecodeTimestamp) {
|
||||
this.#addSampleToTrack(this.#videoTrack, videoSample);
|
||||
} else {
|
||||
this.#videoSampleQueue.push(videoSample);
|
||||
}
|
||||
} else {
|
||||
this.#addSampleToTrack(this.#videoTrack, videoSample);
|
||||
}
|
||||
}
|
||||
|
||||
addAudioChunk(sample: EncodedAudioChunk, meta?: EncodedAudioChunkMetadata, timestamp?: number) {
|
||||
let data = new Uint8Array(sample.byteLength);
|
||||
sample.copyTo(data);
|
||||
|
||||
this.addAudioChunkRaw(data, sample.type, timestamp ?? sample.timestamp, sample.duration, meta);
|
||||
}
|
||||
|
||||
addAudioChunkRaw(
|
||||
data: Uint8Array,
|
||||
type: 'key' | 'delta',
|
||||
timestamp: number,
|
||||
duration: number,
|
||||
meta?: EncodedAudioChunkMetadata
|
||||
) {
|
||||
this.#ensureNotFinalized();
|
||||
if (!this.#options.audio) throw new Error('No audio track declared.');
|
||||
|
||||
if (
|
||||
typeof this.#options.fastStart === 'object' &&
|
||||
this.#audioTrack.samples.length === this.#options.fastStart.expectedAudioChunks
|
||||
) {
|
||||
throw new Error(`Cannot add more audio chunks than specified in 'fastStart' (${
|
||||
this.#options.fastStart.expectedAudioChunks
|
||||
}).`);
|
||||
}
|
||||
|
||||
let audioSample = this.#createSampleForTrack(this.#audioTrack, data, type, timestamp, duration, meta);
|
||||
|
||||
// Check if we need to interleave the samples in the case of a fragmented file
|
||||
if (this.#options.fastStart === 'fragmented' && this.#videoTrack) {
|
||||
// Add all video samples with a timestamp smaller than the incoming audio sample
|
||||
while (
|
||||
this.#videoSampleQueue.length > 0 &&
|
||||
this.#videoSampleQueue[0].decodeTimestamp <= audioSample.decodeTimestamp
|
||||
) {
|
||||
let videoSample = this.#videoSampleQueue.shift();
|
||||
this.#addSampleToTrack(this.#videoTrack, videoSample);
|
||||
}
|
||||
|
||||
// Depending on the last video sample, either add the audio sample to the file or enqueue it
|
||||
if (audioSample.decodeTimestamp <= this.#videoTrack.lastDecodeTimestamp) {
|
||||
this.#addSampleToTrack(this.#audioTrack, audioSample);
|
||||
} else {
|
||||
this.#audioSampleQueue.push(audioSample);
|
||||
}
|
||||
} else {
|
||||
this.#addSampleToTrack(this.#audioTrack, audioSample);
|
||||
}
|
||||
}
|
||||
|
||||
#createSampleForTrack(
|
||||
track: Track,
|
||||
data: Uint8Array,
|
||||
type: 'key' | 'delta',
|
||||
timestamp: number,
|
||||
duration: number,
|
||||
meta?: EncodedVideoChunkMetadata | EncodedAudioChunkMetadata,
|
||||
compositionTimeOffset?: number
|
||||
) {
|
||||
let presentationTimestampInSeconds = timestamp / 1e6;
|
||||
let decodeTimestampInSeconds = (timestamp - (compositionTimeOffset ?? 0)) / 1e6;
|
||||
let durationInSeconds = duration / 1e6;
|
||||
|
||||
let adjusted = this.#validateTimestamp(presentationTimestampInSeconds, decodeTimestampInSeconds, track);
|
||||
presentationTimestampInSeconds = adjusted.presentationTimestamp;
|
||||
decodeTimestampInSeconds = adjusted.decodeTimestamp;
|
||||
|
||||
if (meta?.decoderConfig) {
|
||||
if (track.info.decoderConfig === null) {
|
||||
track.info.decoderConfig = meta.decoderConfig;
|
||||
} else {
|
||||
Object.assign(track.info.decoderConfig, meta.decoderConfig);
|
||||
}
|
||||
}
|
||||
|
||||
let sample: Sample = {
|
||||
presentationTimestamp: presentationTimestampInSeconds,
|
||||
decodeTimestamp: decodeTimestampInSeconds,
|
||||
duration: durationInSeconds,
|
||||
data: data,
|
||||
size: data.byteLength,
|
||||
type: type,
|
||||
// Will be refined once the next sample comes in
|
||||
timescaleUnitsToNextSample: intoTimescale(durationInSeconds, track.timescale)
|
||||
};
|
||||
|
||||
return sample;
|
||||
}
|
||||
|
||||
#addSampleToTrack(
|
||||
track: Track,
|
||||
sample: Sample
|
||||
) {
|
||||
if (this.#options.fastStart !== 'fragmented') {
|
||||
track.samples.push(sample);
|
||||
}
|
||||
|
||||
const sampleCompositionTimeOffset =
|
||||
intoTimescale(sample.presentationTimestamp - sample.decodeTimestamp, track.timescale);
|
||||
|
||||
if (track.lastTimescaleUnits !== null) {
|
||||
let timescaleUnits = intoTimescale(sample.decodeTimestamp, track.timescale, false);
|
||||
let delta = Math.round(timescaleUnits - track.lastTimescaleUnits);
|
||||
track.lastTimescaleUnits += delta;
|
||||
track.lastSample.timescaleUnitsToNextSample = delta;
|
||||
|
||||
if (this.#options.fastStart !== 'fragmented') {
|
||||
let lastTableEntry = last(track.timeToSampleTable);
|
||||
if (lastTableEntry.sampleCount === 1) {
|
||||
// If we hit this case, we're the second sample
|
||||
lastTableEntry.sampleDelta = delta;
|
||||
lastTableEntry.sampleCount++;
|
||||
} else if (lastTableEntry.sampleDelta === delta) {
|
||||
// Simply increment the count
|
||||
lastTableEntry.sampleCount++;
|
||||
} else {
|
||||
// The delta has changed, subtract one from the previous run and create a new run with the new delta
|
||||
lastTableEntry.sampleCount--;
|
||||
track.timeToSampleTable.push({
|
||||
sampleCount: 2,
|
||||
sampleDelta: delta
|
||||
});
|
||||
}
|
||||
|
||||
const lastCompositionTimeOffsetTableEntry = last(track.compositionTimeOffsetTable);
|
||||
if (lastCompositionTimeOffsetTableEntry.sampleCompositionTimeOffset === sampleCompositionTimeOffset) {
|
||||
// Simply increment the count
|
||||
lastCompositionTimeOffsetTableEntry.sampleCount++;
|
||||
} else {
|
||||
// The composition time offset has changed, so create a new entry with the new composition time
|
||||
// offset
|
||||
track.compositionTimeOffsetTable.push({
|
||||
sampleCount: 1,
|
||||
sampleCompositionTimeOffset: sampleCompositionTimeOffset
|
||||
});
|
||||
}
|
||||
}
|
||||
} else {
|
||||
track.lastTimescaleUnits = 0;
|
||||
|
||||
if (this.#options.fastStart !== 'fragmented') {
|
||||
track.timeToSampleTable.push({
|
||||
sampleCount: 1,
|
||||
sampleDelta: intoTimescale(sample.duration, track.timescale)
|
||||
});
|
||||
track.compositionTimeOffsetTable.push({
|
||||
sampleCount: 1,
|
||||
sampleCompositionTimeOffset: sampleCompositionTimeOffset
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
track.lastSample = sample;
|
||||
|
||||
let beginNewChunk = false;
|
||||
if (!track.currentChunk) {
|
||||
beginNewChunk = true;
|
||||
} else {
|
||||
let currentChunkDuration = sample.presentationTimestamp - track.currentChunk.startTimestamp;
|
||||
|
||||
if (this.#options.fastStart === 'fragmented') {
|
||||
let mostImportantTrack = this.#videoTrack ?? this.#audioTrack;
|
||||
if (track === mostImportantTrack && sample.type === 'key' && currentChunkDuration >= 1.0) {
|
||||
beginNewChunk = true;
|
||||
this.#finalizeFragment();
|
||||
}
|
||||
} else {
|
||||
beginNewChunk = currentChunkDuration >= 0.5; // Chunk is long enough, we need a new one
|
||||
}
|
||||
}
|
||||
|
||||
if (beginNewChunk) {
|
||||
if (track.currentChunk) {
|
||||
this.#finalizeCurrentChunk(track);
|
||||
}
|
||||
|
||||
track.currentChunk = {
|
||||
startTimestamp: sample.presentationTimestamp,
|
||||
samples: []
|
||||
};
|
||||
}
|
||||
|
||||
track.currentChunk.samples.push(sample);
|
||||
}
|
||||
|
||||
#validateTimestamp(presentationTimestamp: number, decodeTimestamp: number, track: Track) {
|
||||
// Check first timestamp behavior
|
||||
const strictTimestampBehavior = this.#options.firstTimestampBehavior === 'strict';
|
||||
const noLastDecodeTimestamp = track.lastDecodeTimestamp === -1;
|
||||
const timestampNonZero = decodeTimestamp !== 0;
|
||||
if (strictTimestampBehavior && noLastDecodeTimestamp && timestampNonZero) {
|
||||
throw new Error(
|
||||
`The first chunk for your media track must have a timestamp of 0 (received DTS=${decodeTimestamp}).` +
|
||||
`Non-zero first timestamps are often caused by directly piping frames or audio data from a ` +
|
||||
`MediaStreamTrack into the encoder. Their timestamps are typically relative to the age of the` +
|
||||
`document, which is probably what you want.\n\nIf you want to offset all timestamps of a track such ` +
|
||||
`that the first one is zero, set firstTimestampBehavior: 'offset' in the options.\n`
|
||||
);
|
||||
} else if (
|
||||
this.#options.firstTimestampBehavior === 'offset' ||
|
||||
this.#options.firstTimestampBehavior === 'cross-track-offset'
|
||||
) {
|
||||
if (track.firstDecodeTimestamp === undefined) {
|
||||
track.firstDecodeTimestamp = decodeTimestamp;
|
||||
}
|
||||
|
||||
let baseDecodeTimestamp: number;
|
||||
if (this.#options.firstTimestampBehavior === 'offset') {
|
||||
baseDecodeTimestamp = track.firstDecodeTimestamp;
|
||||
} else {
|
||||
// Since each track may have its firstDecodeTimestamp set independently, but the tracks' timestamps come
|
||||
// from the same clock, we should subtract the earlier of the (up to) two tracks' first timestamps to
|
||||
// ensure A/V sync.
|
||||
baseDecodeTimestamp = Math.min(
|
||||
this.#videoTrack?.firstDecodeTimestamp ?? Infinity,
|
||||
this.#audioTrack?.firstDecodeTimestamp ?? Infinity
|
||||
);
|
||||
}
|
||||
|
||||
decodeTimestamp -= baseDecodeTimestamp;
|
||||
presentationTimestamp -= baseDecodeTimestamp;
|
||||
}
|
||||
|
||||
if (decodeTimestamp < track.lastDecodeTimestamp) {
|
||||
throw new Error(
|
||||
`Timestamps must be monotonically increasing ` +
|
||||
`(DTS went from ${track.lastDecodeTimestamp * 1e6} to ${decodeTimestamp * 1e6}).`
|
||||
);
|
||||
}
|
||||
|
||||
track.lastDecodeTimestamp = decodeTimestamp;
|
||||
|
||||
return { presentationTimestamp, decodeTimestamp };
|
||||
}
|
||||
|
||||
#finalizeCurrentChunk(track: Track) {
|
||||
if (this.#options.fastStart === 'fragmented') {
|
||||
throw new Error("Can't finalize individual chunks 'fastStart' is set to 'fragmented'.");
|
||||
}
|
||||
|
||||
if (!track.currentChunk) return;
|
||||
|
||||
track.finalizedChunks.push(track.currentChunk);
|
||||
this.#finalizedChunks.push(track.currentChunk);
|
||||
|
||||
if (
|
||||
track.compactlyCodedChunkTable.length === 0
|
||||
|| last(track.compactlyCodedChunkTable).samplesPerChunk !== track.currentChunk.samples.length
|
||||
) {
|
||||
track.compactlyCodedChunkTable.push({
|
||||
firstChunk: track.finalizedChunks.length, // 1-indexed
|
||||
samplesPerChunk: track.currentChunk.samples.length
|
||||
});
|
||||
}
|
||||
|
||||
if (this.#options.fastStart === 'in-memory') {
|
||||
track.currentChunk.offset = 0; // We'll compute the proper offset when finalizing
|
||||
return;
|
||||
}
|
||||
|
||||
// Write out the data
|
||||
track.currentChunk.offset = this.#writer.pos;
|
||||
for (let sample of track.currentChunk.samples) {
|
||||
this.#writer.write(sample.data);
|
||||
sample.data = null; // Can be GC'd
|
||||
}
|
||||
|
||||
this.#maybeFlushStreamingTargetWriter();
|
||||
}
|
||||
|
||||
#finalizeFragment(flushStreamingWriter = true) {
|
||||
if (this.#options.fastStart !== 'fragmented') {
|
||||
throw new Error("Can't finalize a fragment unless 'fastStart' is set to 'fragmented'.");
|
||||
}
|
||||
|
||||
let tracks = [this.#videoTrack, this.#audioTrack].filter((track) => track && track.currentChunk);
|
||||
if (tracks.length === 0) return;
|
||||
|
||||
let fragmentNumber = this.#nextFragmentNumber++;
|
||||
|
||||
if (fragmentNumber === 1) {
|
||||
// Write the moov box now that we have all decoder configs
|
||||
let movieBox = moov(tracks, this.#creationTime, true);
|
||||
this.#writer.writeBox(movieBox);
|
||||
}
|
||||
|
||||
// Write out an initial moof box; will be overwritten later once actual chunk offsets are known
|
||||
let moofOffset = this.#writer.pos;
|
||||
let moofBox = moof(fragmentNumber, tracks);
|
||||
this.#writer.writeBox(moofBox);
|
||||
|
||||
// Create the mdat box
|
||||
{
|
||||
let mdatBox = mdat(false); // Initially assume no fragment is larger than 4 GiB
|
||||
let totalTrackSampleSize = 0;
|
||||
|
||||
// Compute the size of the mdat box
|
||||
for (let track of tracks) {
|
||||
for (let sample of track.currentChunk.samples) {
|
||||
totalTrackSampleSize += sample.size;
|
||||
}
|
||||
}
|
||||
|
||||
let mdatSize = this.#writer.measureBox(mdatBox) + totalTrackSampleSize;
|
||||
if (mdatSize >= 2**32) {
|
||||
// Fragment is larger than 4 GiB, we need to use the large size
|
||||
mdatBox.largeSize = true;
|
||||
mdatSize = this.#writer.measureBox(mdatBox) + totalTrackSampleSize;
|
||||
}
|
||||
|
||||
mdatBox.size = mdatSize;
|
||||
this.#writer.writeBox(mdatBox);
|
||||
}
|
||||
|
||||
// Write sample data
|
||||
for (let track of tracks) {
|
||||
track.currentChunk.offset = this.#writer.pos;
|
||||
track.currentChunk.moofOffset = moofOffset;
|
||||
|
||||
for (let sample of track.currentChunk.samples) {
|
||||
this.#writer.write(sample.data);
|
||||
sample.data = null; // Can be GC'd
|
||||
}
|
||||
}
|
||||
|
||||
// Now that we set the actual chunk offsets, fix the moof box
|
||||
let endPos = this.#writer.pos;
|
||||
this.#writer.seek(this.#writer.offsets.get(moofBox));
|
||||
let newMoofBox = moof(fragmentNumber, tracks);
|
||||
this.#writer.writeBox(newMoofBox);
|
||||
this.#writer.seek(endPos);
|
||||
|
||||
for (let track of tracks) {
|
||||
track.finalizedChunks.push(track.currentChunk);
|
||||
this.#finalizedChunks.push(track.currentChunk);
|
||||
track.currentChunk = null;
|
||||
}
|
||||
|
||||
if (flushStreamingWriter) {
|
||||
this.#maybeFlushStreamingTargetWriter();
|
||||
}
|
||||
}
|
||||
|
||||
#maybeFlushStreamingTargetWriter() {
|
||||
if (this.#writer instanceof StreamTargetWriter) {
|
||||
this.#writer.flush();
|
||||
}
|
||||
}
|
||||
|
||||
#ensureNotFinalized() {
|
||||
if (this.#finalized) {
|
||||
throw new Error('Cannot add new video or audio chunks after the file has been finalized.');
|
||||
}
|
||||
}
|
||||
|
||||
/** Finalizes the file, making it ready for use. Must be called after all video and audio chunks have been added. */
|
||||
finalize() {
|
||||
if (this.#finalized) {
|
||||
throw new Error('Cannot finalize a muxer more than once.');
|
||||
}
|
||||
|
||||
if (this.#options.fastStart === 'fragmented') {
|
||||
for (let videoSample of this.#videoSampleQueue) this.#addSampleToTrack(this.#videoTrack, videoSample);
|
||||
for (let audioSample of this.#audioSampleQueue) this.#addSampleToTrack(this.#audioTrack, audioSample);
|
||||
|
||||
this.#finalizeFragment(false); // Don't flush the last fragment as we will flush it with the mfra box soon
|
||||
} else {
|
||||
if (this.#videoTrack) this.#finalizeCurrentChunk(this.#videoTrack);
|
||||
if (this.#audioTrack) this.#finalizeCurrentChunk(this.#audioTrack);
|
||||
}
|
||||
|
||||
let tracks = [this.#videoTrack, this.#audioTrack].filter(Boolean);
|
||||
|
||||
if (this.#options.fastStart === 'in-memory') {
|
||||
let mdatSize: number;
|
||||
|
||||
// We know how many chunks there are, but computing the chunk positions requires an iterative approach:
|
||||
// In order to know where the first chunk should go, we first need to know the size of the moov box. But we
|
||||
// cannot write a proper moov box without first knowing all chunk positions. So, we generate a tentative
|
||||
// moov box with placeholder values (0) for the chunk offsets to be able to compute its size. If it then
|
||||
// turns out that appending all chunks exceeds 4 GiB, we need to repeat this process, now with the co64 box
|
||||
// being used in the moov box instead, which will make it larger. After that, we definitely know the final
|
||||
// size of the moov box and can compute the proper chunk positions.
|
||||
|
||||
for (let i = 0; i < 2; i++) {
|
||||
let movieBox = moov(tracks, this.#creationTime);
|
||||
let movieBoxSize = this.#writer.measureBox(movieBox);
|
||||
mdatSize = this.#writer.measureBox(this.#mdat);
|
||||
let currentChunkPos = this.#writer.pos + movieBoxSize + mdatSize;
|
||||
|
||||
for (let chunk of this.#finalizedChunks) {
|
||||
chunk.offset = currentChunkPos;
|
||||
for (let { data } of chunk.samples) {
|
||||
currentChunkPos += data.byteLength;
|
||||
mdatSize += data.byteLength;
|
||||
}
|
||||
}
|
||||
|
||||
if (currentChunkPos < 2**32) break;
|
||||
if (mdatSize >= 2**32) this.#mdat.largeSize = true;
|
||||
}
|
||||
|
||||
let movieBox = moov(tracks, this.#creationTime);
|
||||
this.#writer.writeBox(movieBox);
|
||||
|
||||
this.#mdat.size = mdatSize;
|
||||
this.#writer.writeBox(this.#mdat);
|
||||
|
||||
for (let chunk of this.#finalizedChunks) {
|
||||
for (let sample of chunk.samples) {
|
||||
this.#writer.write(sample.data);
|
||||
sample.data = null;
|
||||
}
|
||||
}
|
||||
} else if (this.#options.fastStart === 'fragmented') {
|
||||
// Append the mfra box to the end of the file for better random access
|
||||
let startPos = this.#writer.pos;
|
||||
let mfraBox = mfra(tracks);
|
||||
this.#writer.writeBox(mfraBox);
|
||||
|
||||
// Patch the 'size' field of the mfro box at the end of the mfra box now that we know its actual size
|
||||
let mfraBoxSize = this.#writer.pos - startPos;
|
||||
this.#writer.seek(this.#writer.pos - 4);
|
||||
this.#writer.writeU32(mfraBoxSize);
|
||||
} else {
|
||||
let mdatPos = this.#writer.offsets.get(this.#mdat);
|
||||
let mdatSize = this.#writer.pos - mdatPos;
|
||||
this.#mdat.size = mdatSize;
|
||||
this.#mdat.largeSize = mdatSize >= 2**32; // Only use the large size if we need it
|
||||
this.#writer.patchBox(this.#mdat);
|
||||
|
||||
let movieBox = moov(tracks, this.#creationTime);
|
||||
|
||||
if (typeof this.#options.fastStart === 'object') {
|
||||
this.#writer.seek(this.#ftypSize);
|
||||
this.#writer.writeBox(movieBox);
|
||||
|
||||
let remainingBytes = mdatPos - this.#writer.pos;
|
||||
this.#writer.writeBox(free(remainingBytes));
|
||||
} else {
|
||||
this.#writer.writeBox(movieBox);
|
||||
}
|
||||
}
|
||||
|
||||
this.#maybeFlushStreamingTargetWriter();
|
||||
this.#writer.finalize();
|
||||
|
||||
this.#finalized = true;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
export type Target = ArrayBufferTarget | StreamTarget | FileSystemWritableFileStreamTarget;
|
||||
|
||||
export class ArrayBufferTarget {
|
||||
buffer: ArrayBuffer = null;
|
||||
}
|
||||
|
||||
export class StreamTarget {
|
||||
constructor(public options: {
|
||||
onData?: (data: Uint8Array, position: number) => void,
|
||||
chunked?: boolean,
|
||||
chunkSize?: number
|
||||
}) {}
|
||||
}
|
||||
|
||||
export class FileSystemWritableFileStreamTarget {
|
||||
constructor(
|
||||
public stream: FileSystemWritableFileStream,
|
||||
public options?: { chunkSize?: number }
|
||||
) {}
|
||||
}
|
||||
@@ -0,0 +1,380 @@
|
||||
import { Box } from './box';
|
||||
import { ArrayBufferTarget, FileSystemWritableFileStreamTarget, StreamTarget } from './target';
|
||||
|
||||
export abstract class Writer {
|
||||
pos = 0;
|
||||
#helper = new Uint8Array(8);
|
||||
#helperView = new DataView(this.#helper.buffer);
|
||||
|
||||
/**
|
||||
* Stores the position from the start of the file to where boxes elements have been written. This is used to
|
||||
* rewrite/edit elements that were already added before, and to measure sizes of things.
|
||||
*/
|
||||
offsets = new WeakMap<Box, number>();
|
||||
|
||||
/** Writes the given data to the target, at the current position. */
|
||||
abstract write(data: Uint8Array): void;
|
||||
/** Called after muxing has finished. */
|
||||
abstract finalize(): void;
|
||||
|
||||
/** Sets the current position for future writes to a new one. */
|
||||
seek(newPos: number) {
|
||||
this.pos = newPos;
|
||||
}
|
||||
|
||||
writeU32(value: number) {
|
||||
this.#helperView.setUint32(0, value, false);
|
||||
this.write(this.#helper.subarray(0, 4));
|
||||
}
|
||||
|
||||
writeU64(value: number) {
|
||||
this.#helperView.setUint32(0, Math.floor(value / 2**32), false);
|
||||
this.#helperView.setUint32(4, value, false);
|
||||
this.write(this.#helper.subarray(0, 8));
|
||||
}
|
||||
|
||||
writeAscii(text: string) {
|
||||
for (let i = 0; i < text.length; i++) {
|
||||
this.#helperView.setUint8(i % 8, text.charCodeAt(i));
|
||||
if (i % 8 === 7) this.write(this.#helper);
|
||||
}
|
||||
|
||||
if (text.length % 8 !== 0) {
|
||||
this.write(this.#helper.subarray(0, text.length % 8));
|
||||
}
|
||||
}
|
||||
|
||||
writeBox(box: Box) {
|
||||
this.offsets.set(box, this.pos);
|
||||
|
||||
if (box.contents && !box.children) {
|
||||
this.writeBoxHeader(box, box.size ?? box.contents.byteLength + 8);
|
||||
this.write(box.contents);
|
||||
} else {
|
||||
let startPos = this.pos;
|
||||
this.writeBoxHeader(box, 0);
|
||||
|
||||
if (box.contents) this.write(box.contents);
|
||||
if (box.children) for (let child of box.children) if (child) this.writeBox(child);
|
||||
|
||||
let endPos = this.pos;
|
||||
let size = box.size ?? endPos - startPos;
|
||||
this.seek(startPos);
|
||||
this.writeBoxHeader(box, size);
|
||||
this.seek(endPos);
|
||||
}
|
||||
}
|
||||
|
||||
writeBoxHeader(box: Box, size: number) {
|
||||
this.writeU32(box.largeSize ? 1 : size);
|
||||
this.writeAscii(box.type);
|
||||
if (box.largeSize) this.writeU64(size);
|
||||
}
|
||||
|
||||
measureBoxHeader(box: Box) {
|
||||
return 8 + (box.largeSize ? 8 : 0);
|
||||
}
|
||||
|
||||
patchBox(box: Box) {
|
||||
let endPos = this.pos;
|
||||
this.seek(this.offsets.get(box));
|
||||
this.writeBox(box);
|
||||
this.seek(endPos);
|
||||
}
|
||||
|
||||
measureBox(box: Box) {
|
||||
if (box.contents && !box.children) {
|
||||
let headerSize = this.measureBoxHeader(box);
|
||||
return headerSize + box.contents.byteLength;
|
||||
} else {
|
||||
let result = this.measureBoxHeader(box);
|
||||
if (box.contents) result += box.contents.byteLength;
|
||||
if (box.children) for (let child of box.children) if (child) result += this.measureBox(child);
|
||||
|
||||
return result;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Writes to an ArrayBufferTarget. Maintains a growable internal buffer during the muxing process, which will then be
|
||||
* written to the ArrayBufferTarget once the muxing finishes.
|
||||
*/
|
||||
export class ArrayBufferTargetWriter extends Writer {
|
||||
#target: ArrayBufferTarget;
|
||||
#buffer = new ArrayBuffer(2**16);
|
||||
#bytes = new Uint8Array(this.#buffer);
|
||||
#maxPos = 0;
|
||||
|
||||
constructor(target: ArrayBufferTarget) {
|
||||
super();
|
||||
|
||||
this.#target = target;
|
||||
}
|
||||
|
||||
#ensureSize(size: number) {
|
||||
let newLength = this.#buffer.byteLength;
|
||||
while (newLength < size) newLength *= 2;
|
||||
|
||||
if (newLength === this.#buffer.byteLength) return;
|
||||
|
||||
let newBuffer = new ArrayBuffer(newLength);
|
||||
let newBytes = new Uint8Array(newBuffer);
|
||||
newBytes.set(this.#bytes, 0);
|
||||
|
||||
this.#buffer = newBuffer;
|
||||
this.#bytes = newBytes;
|
||||
}
|
||||
|
||||
write(data: Uint8Array) {
|
||||
this.#ensureSize(this.pos + data.byteLength);
|
||||
|
||||
this.#bytes.set(data, this.pos);
|
||||
this.pos += data.byteLength;
|
||||
|
||||
this.#maxPos = Math.max(this.#maxPos, this.pos);
|
||||
}
|
||||
|
||||
finalize() {
|
||||
this.#ensureSize(this.pos);
|
||||
this.#target.buffer = this.#buffer.slice(0, Math.max(this.#maxPos, this.pos));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Writes to a StreamTarget every time it is flushed, sending out all of the new data written since the
|
||||
* last flush. This is useful for streaming applications, like piping the output to disk.
|
||||
*/
|
||||
export class StreamTargetWriter extends Writer {
|
||||
#target: StreamTarget;
|
||||
#sections: {
|
||||
data: Uint8Array,
|
||||
start: number
|
||||
}[] = [];
|
||||
|
||||
constructor(target: StreamTarget) {
|
||||
super();
|
||||
|
||||
this.#target = target;
|
||||
}
|
||||
|
||||
write(data: Uint8Array) {
|
||||
this.#sections.push({
|
||||
data: data.slice(),
|
||||
start: this.pos
|
||||
});
|
||||
this.pos += data.byteLength;
|
||||
}
|
||||
|
||||
flush() {
|
||||
if (this.#sections.length === 0) return;
|
||||
|
||||
let chunks: {
|
||||
start: number,
|
||||
size: number,
|
||||
data?: Uint8Array
|
||||
}[] = [];
|
||||
let sorted = [...this.#sections].sort((a, b) => a.start - b.start);
|
||||
|
||||
chunks.push({
|
||||
start: sorted[0].start,
|
||||
size: sorted[0].data.byteLength
|
||||
});
|
||||
|
||||
// Figure out how many contiguous chunks we have
|
||||
for (let i = 1; i < sorted.length; i++) {
|
||||
let lastChunk = chunks[chunks.length - 1];
|
||||
let section = sorted[i];
|
||||
|
||||
if (section.start <= lastChunk.start + lastChunk.size) {
|
||||
lastChunk.size = Math.max(lastChunk.size, section.start + section.data.byteLength - lastChunk.start);
|
||||
} else {
|
||||
chunks.push({
|
||||
start: section.start,
|
||||
size: section.data.byteLength
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
for (let chunk of chunks) {
|
||||
chunk.data = new Uint8Array(chunk.size);
|
||||
|
||||
// Make sure to write the data in the correct order for correct overwriting
|
||||
for (let section of this.#sections) {
|
||||
// Check if the section is in the chunk
|
||||
if (chunk.start <= section.start && section.start < chunk.start + chunk.size) {
|
||||
chunk.data.set(section.data, section.start - chunk.start);
|
||||
}
|
||||
}
|
||||
|
||||
this.#target.options.onData?.(chunk.data, chunk.start);
|
||||
}
|
||||
|
||||
this.#sections.length = 0;
|
||||
}
|
||||
|
||||
finalize() {}
|
||||
}
|
||||
|
||||
const DEFAULT_CHUNK_SIZE = 2**24;
|
||||
const MAX_CHUNKS_AT_ONCE = 2;
|
||||
|
||||
interface Chunk {
|
||||
start: number,
|
||||
written: ChunkSection[],
|
||||
data: Uint8Array,
|
||||
shouldFlush: boolean
|
||||
}
|
||||
|
||||
interface ChunkSection {
|
||||
start: number,
|
||||
end: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Writes to a StreamTarget using a chunked approach: Data is first buffered in memory until it reaches a large enough
|
||||
* size, which is when it is piped to the StreamTarget. This is helpful for reducing the total amount of writes.
|
||||
*/
|
||||
export class ChunkedStreamTargetWriter extends Writer {
|
||||
#target: StreamTarget;
|
||||
#chunkSize: number;
|
||||
/**
|
||||
* The data is divided up into fixed-size chunks, whose contents are first filled in RAM and then flushed out.
|
||||
* A chunk is flushed if all of its contents have been written.
|
||||
*/
|
||||
#chunks: Chunk[] = [];
|
||||
|
||||
constructor(target: StreamTarget) {
|
||||
super();
|
||||
|
||||
this.#target = target;
|
||||
this.#chunkSize = target.options?.chunkSize ?? DEFAULT_CHUNK_SIZE;
|
||||
|
||||
if (!Number.isInteger(this.#chunkSize) || this.#chunkSize < 2**10) {
|
||||
throw new Error('Invalid StreamTarget options: chunkSize must be an integer not smaller than 1024.');
|
||||
}
|
||||
}
|
||||
|
||||
write(data: Uint8Array) {
|
||||
this.#writeDataIntoChunks(data, this.pos);
|
||||
this.#flushChunks();
|
||||
|
||||
this.pos += data.byteLength;
|
||||
}
|
||||
|
||||
#writeDataIntoChunks(data: Uint8Array, position: number) {
|
||||
// First, find the chunk to write the data into, or create one if none exists
|
||||
let chunkIndex = this.#chunks.findIndex(x => x.start <= position && position < x.start + this.#chunkSize);
|
||||
if (chunkIndex === -1) chunkIndex = this.#createChunk(position);
|
||||
let chunk = this.#chunks[chunkIndex];
|
||||
|
||||
// Figure out how much to write to the chunk, and then write to the chunk
|
||||
let relativePosition = position - chunk.start;
|
||||
let toWrite = data.subarray(0, Math.min(this.#chunkSize - relativePosition, data.byteLength));
|
||||
chunk.data.set(toWrite, relativePosition);
|
||||
|
||||
// Create a section describing the region of data that was just written to
|
||||
let section: ChunkSection = {
|
||||
start: relativePosition,
|
||||
end: relativePosition + toWrite.byteLength
|
||||
};
|
||||
this.#insertSectionIntoChunk(chunk, section);
|
||||
|
||||
// Queue chunk for flushing to target if it has been fully written to
|
||||
if (chunk.written[0].start === 0 && chunk.written[0].end === this.#chunkSize) {
|
||||
chunk.shouldFlush = true;
|
||||
}
|
||||
|
||||
// Make sure we don't hold too many chunks in memory at once to keep memory usage down
|
||||
if (this.#chunks.length > MAX_CHUNKS_AT_ONCE) {
|
||||
// Flush all but the last chunk
|
||||
for (let i = 0; i < this.#chunks.length-1; i++) {
|
||||
this.#chunks[i].shouldFlush = true;
|
||||
}
|
||||
this.#flushChunks();
|
||||
}
|
||||
|
||||
// If the data didn't fit in one chunk, recurse with the remaining datas
|
||||
if (toWrite.byteLength < data.byteLength) {
|
||||
this.#writeDataIntoChunks(data.subarray(toWrite.byteLength), position + toWrite.byteLength);
|
||||
}
|
||||
}
|
||||
|
||||
#insertSectionIntoChunk(chunk: Chunk, section: ChunkSection) {
|
||||
let low = 0;
|
||||
let high = chunk.written.length - 1;
|
||||
let index = -1;
|
||||
|
||||
// Do a binary search to find the last section with a start not larger than `section`'s start
|
||||
while (low <= high) {
|
||||
let mid = Math.floor(low + (high - low + 1) / 2);
|
||||
|
||||
if (chunk.written[mid].start <= section.start) {
|
||||
low = mid + 1;
|
||||
index = mid;
|
||||
} else {
|
||||
high = mid - 1;
|
||||
}
|
||||
}
|
||||
|
||||
// Insert the new section
|
||||
chunk.written.splice(index + 1, 0, section);
|
||||
if (index === -1 || chunk.written[index].end < section.start) index++;
|
||||
|
||||
// Merge overlapping sections
|
||||
while (index < chunk.written.length - 1 && chunk.written[index].end >= chunk.written[index + 1].start) {
|
||||
chunk.written[index].end = Math.max(chunk.written[index].end, chunk.written[index + 1].end);
|
||||
chunk.written.splice(index + 1, 1);
|
||||
}
|
||||
}
|
||||
|
||||
#createChunk(includesPosition: number) {
|
||||
let start = Math.floor(includesPosition / this.#chunkSize) * this.#chunkSize;
|
||||
let chunk: Chunk = {
|
||||
start,
|
||||
data: new Uint8Array(this.#chunkSize),
|
||||
written: [],
|
||||
shouldFlush: false
|
||||
};
|
||||
this.#chunks.push(chunk);
|
||||
this.#chunks.sort((a, b) => a.start - b.start);
|
||||
|
||||
return this.#chunks.indexOf(chunk);
|
||||
}
|
||||
|
||||
#flushChunks(force = false) {
|
||||
for (let i = 0; i < this.#chunks.length; i++) {
|
||||
let chunk = this.#chunks[i];
|
||||
if (!chunk.shouldFlush && !force) continue;
|
||||
|
||||
for (let section of chunk.written) {
|
||||
this.#target.options.onData?.(
|
||||
chunk.data.subarray(section.start, section.end),
|
||||
chunk.start + section.start
|
||||
);
|
||||
}
|
||||
this.#chunks.splice(i--, 1);
|
||||
}
|
||||
}
|
||||
|
||||
finalize() {
|
||||
this.#flushChunks(true);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Essentially a wrapper around ChunkedStreamTargetWriter, writing directly to disk using the File System Access API.
|
||||
* This is useful for large files, as available RAM is no longer a bottleneck.
|
||||
*/
|
||||
export class FileSystemWritableFileStreamTargetWriter extends ChunkedStreamTargetWriter {
|
||||
constructor(target: FileSystemWritableFileStreamTarget) {
|
||||
super(new StreamTarget({
|
||||
onData: (data, position) => target.stream.write({
|
||||
type: 'write',
|
||||
data,
|
||||
position
|
||||
}),
|
||||
chunkSize: target.options?.chunkSize
|
||||
}));
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@@ -0,0 +1,199 @@
|
||||
<!DOCTYPE html>
|
||||
|
||||
<style>
|
||||
body {
|
||||
font-family: sans-serif;
|
||||
}
|
||||
</style>
|
||||
|
||||
<input type="file">
|
||||
<button>Construct</button>
|
||||
<main></main>
|
||||
|
||||
<script>
|
||||
let contents;
|
||||
|
||||
const fileInput = document.querySelector('input');
|
||||
fileInput.addEventListener('change', async () => {
|
||||
let file = fileInput.files[0];
|
||||
let buffer = await file.arrayBuffer();
|
||||
|
||||
let totalBytes = new Uint8Array(buffer);
|
||||
|
||||
const isAlphanumeric = (charCode) => {
|
||||
return charCode && (
|
||||
(charCode >= 48 && charCode <= 57)
|
||||
|| (charCode >= 65 && charCode <= 90)
|
||||
|| (charCode >= 97 && charCode <= 122));
|
||||
};
|
||||
|
||||
const parseContents = (bytes) => {
|
||||
let totalContents = [];
|
||||
let view = new DataView(bytes.buffer);
|
||||
let lastIndex = 0;
|
||||
|
||||
for (let i = 0; i < bytes.byteLength; i++) {
|
||||
cond:
|
||||
if (
|
||||
isAlphanumeric(bytes[i+4])
|
||||
&& isAlphanumeric(bytes[i+5])
|
||||
&& isAlphanumeric(bytes[i+6])
|
||||
&& isAlphanumeric(bytes[i+7])
|
||||
) {
|
||||
let size = view.getUint32(i, false);
|
||||
if (size < 8) break cond;
|
||||
if (i + size > bytes.byteLength) break cond;
|
||||
|
||||
let tag = String.fromCharCode(bytes[i + 4])
|
||||
+ String.fromCharCode(bytes[i + 5])
|
||||
+ String.fromCharCode(bytes[i + 6])
|
||||
+ String.fromCharCode(bytes[i + 7]);
|
||||
if ((tag.toLowerCase() !== tag) && tag !== 'avcC' && tag !== 'avc1') break cond;
|
||||
|
||||
if (i - lastIndex > 0) {
|
||||
totalContents.push(bytes.slice(lastIndex, i));
|
||||
}
|
||||
|
||||
let contents = tag === 'mdat'
|
||||
? [bytes.slice(i + 8, i + size)]
|
||||
: parseContents(bytes.slice(i + 8, i + size));
|
||||
|
||||
totalContents.push({
|
||||
tag,
|
||||
contents
|
||||
});
|
||||
|
||||
lastIndex = i + size;
|
||||
i += size - 1;
|
||||
}
|
||||
}
|
||||
|
||||
if (bytes.byteLength - lastIndex > 1) {
|
||||
totalContents.push(bytes.slice(lastIndex));
|
||||
}
|
||||
|
||||
return totalContents;
|
||||
};
|
||||
contents = parseContents(totalBytes);
|
||||
|
||||
document.querySelector('main').append(...contents.map(dataToDiv));
|
||||
});
|
||||
|
||||
let crossedOut = new Set();
|
||||
let modified = new Map();
|
||||
|
||||
const dataToDiv = (data) => {
|
||||
if (data instanceof Uint8Array) {
|
||||
let div = document.createElement('div');
|
||||
div.setAttribute('contenteditable', true);
|
||||
div.textContent = [...data].map(x => x.toString(16).padStart(2, '0').toLowerCase()).join('');
|
||||
div.style.whiteSpace = 'nowrap';
|
||||
|
||||
div.addEventListener('keydown', () => {
|
||||
setTimeout(() => {
|
||||
modified.set(data, hexStringToUint8Array(div.textContent));
|
||||
|
||||
if (div.textContent.length % 2) {
|
||||
div.style.background = 'lime';
|
||||
} else {
|
||||
div.style.background = '';
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
return div;
|
||||
}
|
||||
|
||||
let div = document.createElement('div');
|
||||
let span = document.createElement('span');
|
||||
span.style.background = 'lightgray';
|
||||
span.textContent = data.tag;
|
||||
let children = document.createElement('div');
|
||||
children.style.paddingLeft = '10px';
|
||||
|
||||
div.append(span);
|
||||
div.append(children);
|
||||
children.append(...data.contents.map(dataToDiv));
|
||||
|
||||
span.addEventListener('click', (e) => {
|
||||
if (crossedOut.has(data)) {
|
||||
crossedOut.delete(data);
|
||||
span.style.textDecoration = '';
|
||||
children.style.opacity = 1;
|
||||
} else {
|
||||
crossedOut.add(data);
|
||||
span.style.textDecoration = 'line-through';
|
||||
children.style.opacity = 0.3;
|
||||
}
|
||||
|
||||
e.stopPropagation();
|
||||
});
|
||||
|
||||
return div;
|
||||
};
|
||||
|
||||
const hexStringToUint8Array = (hexString) => {
|
||||
if (hexString.length % 2 !== 0) {
|
||||
hexString += '0';
|
||||
}
|
||||
|
||||
const byteCount = hexString.length / 2;
|
||||
const uint8Array = new Uint8Array(byteCount);
|
||||
|
||||
for (let i = 0; i < byteCount; i++) {
|
||||
const hexByte = hexString.slice(i * 2, i * 2 + 2);
|
||||
uint8Array[i] = parseInt(hexByte, 16);
|
||||
}
|
||||
|
||||
return uint8Array;
|
||||
};
|
||||
|
||||
document.querySelector('button').addEventListener('click', () => {
|
||||
let constructed = construct(contents);
|
||||
downloadBlob(new Blob([new Uint8Array(constructed)]), 'edited.mp4');
|
||||
});
|
||||
|
||||
const u32 = (value) => {
|
||||
let bytes = new Uint8Array(4);
|
||||
let view = new DataView(bytes.buffer);
|
||||
view.setUint32(0, value, false);
|
||||
return [...bytes];
|
||||
};
|
||||
|
||||
const ascii = (text, nullTerminated = false) => {
|
||||
let bytes = Array(text.length).fill(null).map((_, i) => text.charCodeAt(i));
|
||||
if (nullTerminated) bytes.push(0x00);
|
||||
return bytes;
|
||||
};
|
||||
|
||||
const construct = (contents) => {
|
||||
if (contents instanceof Uint8Array) {
|
||||
if (modified.has(contents)) return [...modified.get(contents)];
|
||||
else return [...contents];
|
||||
} else if (Array.isArray(contents)) {
|
||||
return contents.flatMap(construct);
|
||||
} else {
|
||||
let constructedContents = construct(contents.contents);
|
||||
let size = constructedContents.length + 8;
|
||||
|
||||
return [
|
||||
...u32(size),
|
||||
...ascii(crossedOut.has(contents) ? 'free' : contents.tag),
|
||||
...constructedContents
|
||||
];
|
||||
}
|
||||
};
|
||||
|
||||
const downloadBlob = (blob, filename) => {
|
||||
const url = URL.createObjectURL(blob);
|
||||
const a = document.createElement('a');
|
||||
a.href = url;
|
||||
a.download = filename;
|
||||
document.body.appendChild(a);
|
||||
a.click();
|
||||
setTimeout(() => {
|
||||
document.body.removeChild(a);
|
||||
URL.revokeObjectURL(url);
|
||||
}, 0);
|
||||
};
|
||||
</script>
|
||||
@@ -0,0 +1,121 @@
|
||||
<script src="../build/mp4-muxer.js"></script>
|
||||
|
||||
<script type="module">
|
||||
const width = 1280;
|
||||
const height = 720;
|
||||
const sampleRate = 44100;
|
||||
const numberOfChannels = 1;
|
||||
|
||||
let fileHandle = await new Promise(resolve => {
|
||||
window.addEventListener('click', async () => {
|
||||
let fileHandle = await window.showSaveFilePicker({
|
||||
startIn: 'videos',
|
||||
suggestedName: `video.mp4`,
|
||||
types: [{
|
||||
description: 'Video File',
|
||||
accept: {'video/mp4' :['.mp4']}
|
||||
}],
|
||||
});
|
||||
resolve(fileHandle);
|
||||
}, { once: true });
|
||||
});
|
||||
let fileWritableStream = await fileHandle.createWritable();
|
||||
|
||||
let buf = new Uint8Array(2**24);
|
||||
let maxPos = 0;
|
||||
|
||||
let muxer = new Mp4Muxer.Muxer({
|
||||
//target: new Mp4Muxer.FileSystemWritableFileStreamTarget(fileWritableStream),
|
||||
target: new Mp4Muxer.ArrayBufferTarget(),
|
||||
|
||||
video: {
|
||||
codec: 'avc',
|
||||
width,
|
||||
height
|
||||
},
|
||||
audio: {
|
||||
codec: 'aac',
|
||||
numberOfChannels,
|
||||
sampleRate
|
||||
},
|
||||
|
||||
fastStart: false
|
||||
});
|
||||
|
||||
let canvas = document.createElement('canvas');
|
||||
canvas.width = width;
|
||||
canvas.height = height;
|
||||
let ctx = canvas.getContext('2d');
|
||||
|
||||
let videoEncoder = new VideoEncoder({
|
||||
output: (chunk, meta) => {
|
||||
//console.log(chunk, meta);
|
||||
muxer.addVideoChunk(chunk, meta);
|
||||
},
|
||||
error: (e) => console.error(e)
|
||||
});
|
||||
videoEncoder.configure({
|
||||
codec: 'avc1.640028',
|
||||
width: width,
|
||||
height: height,
|
||||
bitrate: 1e6,
|
||||
framerate: 10
|
||||
});
|
||||
|
||||
let audioEncoder = new AudioEncoder({
|
||||
output: (chunk, meta) => {
|
||||
//console.log(chunk, meta);
|
||||
muxer.addAudioChunk(chunk, meta);
|
||||
},
|
||||
error: (e) => console.error(e)
|
||||
});
|
||||
audioEncoder.configure({
|
||||
codec: 'mp4a.40.2',
|
||||
sampleRate,
|
||||
numberOfChannels,
|
||||
bitrate: 128000,
|
||||
});
|
||||
|
||||
for (let i = 0; i < 100; i++) {
|
||||
ctx.fillStyle = ['red', 'lime', 'blue', 'yellow'][Math.floor(Math.random() * 4)];
|
||||
ctx.fillRect(Math.random() * width, Math.random() * height, Math.random() * width, Math.random() * height);
|
||||
|
||||
let frame = new VideoFrame(canvas, { timestamp: 100000 * i });
|
||||
videoEncoder.encode(frame);
|
||||
}
|
||||
|
||||
let audioContext = new AudioContext();
|
||||
let audioBuffer = await audioContext.decodeAudioData(await (await fetch('./CantinaBand60.wav')).arrayBuffer());
|
||||
let length = 10;
|
||||
let data = new Float32Array(length * numberOfChannels * sampleRate);
|
||||
data.set(audioBuffer.getChannelData(0).subarray(0, data.length), 0);
|
||||
//data.set(audioBuffer.getChannelData(0).subarray(0, data.length/2), data.length/2);
|
||||
|
||||
let audioData = new AudioData({
|
||||
format: 'f32-planar',
|
||||
sampleRate,
|
||||
numberOfFrames: length * sampleRate,
|
||||
numberOfChannels,
|
||||
timestamp: 0,
|
||||
data
|
||||
});
|
||||
audioEncoder.encode(audioData);
|
||||
audioData.close();
|
||||
|
||||
await videoEncoder.flush();
|
||||
await audioEncoder.flush();
|
||||
|
||||
muxer.finalize();
|
||||
let buffer = muxer.target.buffer;
|
||||
console.log(buffer);
|
||||
|
||||
await fileWritableStream.close();
|
||||
|
||||
function download(blob, filename) {
|
||||
const a = document.createElement('a');
|
||||
a.href = URL.createObjectURL(blob);
|
||||
a.download = filename;
|
||||
a.click();
|
||||
}
|
||||
download(new Blob([buffer]), 't.mp4');
|
||||
</script>
|
||||
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2021",
|
||||
"strict": true,
|
||||
"strictNullChecks": false,
|
||||
"noImplicitAny": true,
|
||||
"noImplicitOverride": true,
|
||||
"types": ["@types/wicg-file-system-access", "@types/dom-webcodecs"]
|
||||
},
|
||||
"include": [
|
||||
"src/**/*",
|
||||
"build/**/*.ts"
|
||||
]
|
||||
}
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 17 KiB |
@@ -65,7 +65,7 @@
|
||||
mix-blend-mode: multiply;
|
||||
background-blend-mode: multiply;
|
||||
z-index: 1;
|
||||
opacity:0.333;
|
||||
opacity:0;
|
||||
}
|
||||
|
||||
/*=========Main*/
|
||||
|
||||
@@ -46,10 +46,12 @@
|
||||
position: fixed;
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
z-index: 0;
|
||||
z-index: 1;
|
||||
cursor: cell;
|
||||
scrollbar-color: var(--accent-color-2) transparent;
|
||||
scrollbar-width: thin;
|
||||
background-color: transparent;
|
||||
/*
|
||||
background-size: 100% 100%;
|
||||
background-position: 0px 0px,0px 0px,0px 0px,0px 0px,0px 0px,0px 0px;
|
||||
background-image:
|
||||
@@ -59,6 +61,7 @@
|
||||
radial-gradient(150% 100% at 80% 0%, var(--dark-color) 0%, var(--accent-color) 0%, var(--light-color) 27%, transparent 100%),
|
||||
radial-gradient(142% 91% at -6% 74%, var(--accent-color) 4%, transparent 99%),
|
||||
radial-gradient(142% 200% at 83% 77%, var(--dark-color) 20%, var(--accent-color) 39%, var(--light-color) 69%);
|
||||
*/
|
||||
}
|
||||
|
||||
/*===================Desact. Default*/
|
||||
|
||||
@@ -24,7 +24,7 @@
|
||||
<div id="contactField">Perf Nerds</div>
|
||||
</div>
|
||||
</a>
|
||||
<a href="https://signal.me/#eu/76w7wcHZpW1HtunwdjbIJZWpTGVMCqobn_qAzhGBrSRd8cGtdq9NZJondaOHJ9iM" target="_blank" class="contactLink">
|
||||
<a href="https://signal.me/#eu/jPBmn2MoCUWsKN0eJbWl_3w-HuPNu2SndVwaWZX3iC9HBnQEi4FPSZefIkfS86T8" target="_blank" class="contactLink">
|
||||
<div class="contactCard">
|
||||
<div id="contactName">Vega</div>
|
||||
<div id="contactField">Scéno</div>
|
||||
|
||||
Reference in New Issue
Block a user