record and set keyframes, introduce RecordingBuffer, avoid toDataUrl() for fft visualisation

This commit is contained in:
jrkb 2023-10-02 10:05:20 +02:00
parent 64b3d26384
commit 7f6ab572c7
6 changed files with 431 additions and 95 deletions

View file

@ -13,6 +13,8 @@ const Audio = function(tp, record) {
let started = false;
const mapping = {};
const canvass = [];
const canvasCtxs = [];
const addAudioOptions = (layer, propTitle) => {
const panelPropTitle = tp.getPanelPropTitle(propTitle);
@ -20,6 +22,10 @@ const Audio = function(tp, record) {
console.log('Audio::addAudioOptions::error',`cannot find panelPropTitle "${propTitle}"`);
return;
}
if (tp.getPanel().querySelector(`.audioOptions${propTitle}`) !== null) {
console.log('Audio::addAudioOptions::error',`audioOptions already exist for "${propTitle}"`);
return;
}
const container = tp.getPanelPropContainer(panelPropTitle);
const mappingOptions = mapping[layer.id()][propTitle];
const panel = tp.getPanel();
@ -30,10 +36,10 @@ const Audio = function(tp, record) {
audioOptions.style.position = 'relative';
audioOptions.style.width = '100%';
audioOptions.style.background = 'rgba(0,255,255,0.2)';
audioOptions.style.order = window.getComputedStyle(container).order;
audioOptions.style.order = parseInt(container.style.order) + 1;
mappingOptions.freq_min = 0;
mappingOptions.freq_max = 256 * 8 / 2;
mappingOptions.freq_max = config.audio.fftBandsUsed;
const updateMappingOptions = () => {
mappingOptions.min_out = parseFloat(panel.querySelector(`#audio_min${propTitle}`).value);
@ -107,7 +113,7 @@ const Audio = function(tp, record) {
audioOptions.append(sync_Dom);
const fft_Dom = document.createElement('div');
const fft_imgDom = document.createElement('img');
const fft_imgDom = document.createElement('canvas');
const fft_selectDom = document.createElement('div');
fft_Dom.style.position = 'relative';
fft_Dom.style.top = '0px';
@ -117,6 +123,8 @@ const Audio = function(tp, record) {
fft_imgDom.style.userDrag = 'none';
fft_imgDom.style.userSelect = 'none';
fft_imgDom.style.pointerEvents = 'none';
fft_imgDom.setAttribute('width', config.audio.fftBandsUsed);
fft_imgDom.setAttribute('height', config.audio.fftHeight);
fft_selectDom.style.position = 'absolute';
fft_selectDom.style.top = '0px';
fft_selectDom.style.left = '0px';
@ -138,18 +146,21 @@ const Audio = function(tp, record) {
setFrequency = true;
const bb = fft_Dom.getBoundingClientRect();
const x = e.clientX - bb.x;
freq_down = mapValue(e.clientX, bb.x, bb.x + bb.width, 0, 256 * 8 / 2, true);
freq_down = mapValue(e.clientX, bb.x, bb.x + bb.width, 0, config.audio.fftBandsUsed, true);
});
fft_Dom.addEventListener('mouseup', (e) => {
setFrequency = false;
const bb = fft_Dom.getBoundingClientRect();
const x = e.clientX - bb.x;
freq_down = mapValue(e.clientX, bb.x, bb.x + bb.width, 0, 256 * 8 / 2, true);
freq_down = mapValue(e.clientX, bb.x, bb.x + bb.width, 0, config.audio.fftBandsUsed, true);
});
//removeAudioOptions();
container.after(audioOptions);
canvass.push(fft_imgDom);
canvasCtxs.push(fft_imgDom.getContext("2d"));
updateMappingOptions();
mappingOptions.value = mappingOptions.min_out;
};
@ -346,7 +357,7 @@ const Audio = function(tp, record) {
const canvasCtx = canvas.getContext("2d");
const intendedWidth = audioDom.clientWidth;
canvas.setAttribute("width", 256 * 8 / 2);
canvas.setAttribute("width", config.audio.fftBandsUsed);
const visualSelect = audioDom.querySelector("#visual");
let drawVisual;
@ -428,13 +439,17 @@ const Audio = function(tp, record) {
draw();
} else if (visualSetting == "frequencybars") {
analyser.fftSize = 256 * 8;
analyser.fftSize = config.audio.fftBandsAnalysed;
const w = config.audio.fftBandsUsed;
const h = config.audio.fftHeight;
const bufferLengthAlt = analyser.frequencyBinCount / 2;
// See comment above for Float32Array()
const dataArrayAlt = new Uint8Array(bufferLengthAlt);
canvasCtx.clearRect(0, 0, WIDTH, HEIGHT);
for (let i = 0; i < canvasCtxs.length; i++) {
canvasCtxs[i].clearRect(0, 0, w, h);
}
let frameCount = 0;
const drawAlt = function() {
@ -442,10 +457,12 @@ const Audio = function(tp, record) {
analyser.getByteFrequencyData(dataArrayAlt);
canvasCtx.fillStyle = "rgb(0, 0, 0)";
canvasCtx.fillRect(0, 0, WIDTH, HEIGHT);
for (let i = 0; i < canvasCtxs.length; i++) {
canvasCtxs[i].fillStyle = "rgb(0, 0, 0)";
canvasCtxs[i].fillRect(0, 0, w, h);
}
const barWidth = (WIDTH / bufferLengthAlt) * 2.5;
const barWidth = (w / bufferLengthAlt) * 2.5;
let barHeight;
let x = 0;
@ -458,13 +475,15 @@ const Audio = function(tp, record) {
max_v = barHeight;
max_i = i;
}
canvasCtx.fillStyle = "rgb(" + (barHeight + 100) + ",50,50)";
canvasCtx.fillRect(
x,
HEIGHT - barHeight / 2,
barWidth,
barHeight / 2
);
for (let i = 0; i < canvasCtxs.length; i++) {
canvasCtxs[i].fillStyle = "rgb(" + (barHeight + 100) + ",50,50)";
canvasCtxs[i].fillRect(
x,
h - barHeight / 2,
barWidth,
barHeight / 2
);
}
x += barWidth + 1;
}
@ -478,6 +497,8 @@ const Audio = function(tp, record) {
let a = mapValue(max_v, 0, 255, m.min_out, m.max_out, true);
m.value = m.value * m.smoothing + (1.0 - m.smoothing) * a;
propsToSet.push({
layer,
id: layer.id(),
title: propTitle,
prop: layer.theatreObject.props[propTitle],
value: m.value,
@ -488,6 +509,8 @@ const Audio = function(tp, record) {
let a = mapValue(max_i, 0, bufferLengthAlt-1, m.min_out, m.max_out, true);
m.value = m.value * m.smoothing + (1.0 - m.smoothing) * a;
propsToSet.push({
layer,
id: layer.id(),
title: propTitle,
prop: layer.theatreObject.props[propTitle],
value: m.value,
@ -503,37 +526,57 @@ const Audio = function(tp, record) {
}
});
if (propsToSet.length > 0 && frameCount % 2 === 0) {
// this is when to monitor live
if (!record.isRecording()) {
tp.studio.transaction(({
set
}) => {
if (!tp.core.val(tp.sheet.sequence.pointer.playing)) {
if (typeof window.immediateUpdate !== 'function') {
window.immediateUpdate = (layer, values) => {
const v = {
...layer.theatreObject.value,
...values
};
const p = layer.values2cppProps(v);
if (p !== false) {
const id = layer.id();
if (id !== 'artboard') {
Module.setProps(p, layer.id());
} else {
Module.setArtboardProps(p, layer.id());
}
}
};
}
propsToSet.forEach((p) => {
set(p.prop, p.value, true);
immediateUpdate(p.layer, {
[p.title]: p.value
});
});
});
}
} else {
propsToSet.forEach((p) => {
// TODO: this does not have to be queried
// but we could store it in a map/set/dictionary/array/object
const inputElement = tp
.getPanelPropContainer(p.title)
.querySelector('input.recording');
const title = tp
.getPanelPropContainer(p.title);
if (inputElement !== null) {
inputElement.value = p.value;
inputElement.dispatchEvent(new Event('change'));
if (title !== null) {
const inputElement = title
.querySelector('input.recording');
if (inputElement !== null) {
inputElement.value = p.value;
inputElement.dispatchEvent(new Event('change'));
}
}
});
}
}
const panel = tp.getPanel();
const fft_images = panel.querySelectorAll('.audio_fft');
if (fft_images !== null) {
const src = canvas.toDataURL();
fft_images.forEach((e) => {
e.src = src;
});
}
//const panel = tp.getPanel();
//const fft_images = panel.querySelectorAll('.audio_fft');
//if (fft_images !== null) {
//const src = canvas.toDataURL();
//fft_images.forEach((e) => {
//e.src = src;
//});
//}
frameCount++;
};
drawAlt();

View file

@ -85,6 +85,9 @@ const config = {
audio: {
ignoreProps: ['transformOrigin', 'fontFamily', 'text', 'mirror_x', 'mirror_y', 'mirror_xy', 'fontVariationAxes', 'color'],
defaultSmoothing: 0.7,
fftBandsAnalysed: 256 * 8,
fftBandsUsed: 256 * 8 / 2,
fftHeight: 256 / 2,
},
record: {
ignoreProps: ['fontVariationAxes','letterDelays','color'],

View file

@ -188,6 +188,7 @@ window.onload = () => {
alert('Sorry, Variable Time is a tool currently designed to be used on desktop!');
}
window.addEventListener('panelEvent', (e) => {
console.log('debug panelEvent received', e);
clearTimeout(window.panelFinderTimeout);
let target = false;
if (e.detail.panelID === 'artboard') {
@ -203,6 +204,7 @@ window.onload = () => {
}
});
window.addEventListener('sequenceEvent', (e) => {
console.log('debug sequenceEvent received', e);
let target = false;
if (e.detail.panelID === 'artboard') {
target = artboard;

View file

@ -1,7 +1,163 @@
import {
clone,
sequencialPromises,
} from './utils.js';
const LiveBuffer = function() {
// private
//
// constants
const NO_TIME = -1;
// variables
/// @brief valueBuffer stores all values.
/// it is an object with layerIDs/artboard as key
const valueBuffer = {};
// functions
const register = (id) => {
if (!valueBuffer.hasOwnProperty(id)) {
valueBuffer[id] = new Map();
}
};
const deregister = (id) => {
if (valueBuffer.hasOwnProperty(id)) {
delete valueBuffer[id];
}
};
/// @brief addValues
// values are expected to be
// {
// x: 42.0,
// fontSize_px: 24,
// whatever: "something",
// }
const addValues = (id, values, time_s, start_time_s) => {
const subValueBuffer = valueBuffer[id];
// delete between start_time_s and time_s
if (start_time_s !== NO_TIME) {
subValueBuffer.forEach((value, value_time_s) => {
if (value_time_s > start_time_s && value_time_s <= time_s) {
const keys = Object.keys(values);
for (let k = 0; k < keys.length; k++) {
delete subValueBuffer.get(value_time_s)[keys[k]];
if (Object.keys(subValueBuffer.get(value_time_s)).length === 0) {
subValueBuffer.delete(value_time_s);
break;
}
};
}
});
}
if (subValueBuffer.has(time_s)) {
subValueBuffer.set(time_s, {...subValueBuffer.get(time_s), ...values});
} else {
subValueBuffer.set(time_s, clone(values));
}
};
// get values and merge with previous
// 0.42s: { x: 4 }
// 0.64s: { y: 7 }
// 0.82s: { y: 12, z: 24 }
//
// return for 0.82s = { x: 4, y: 12, x: 24 }
const getValues = (id, time_s) => {
if (valueBuffer[id].size === 0) {
return {};
} else {
valueBuffer[id] = new Map([...valueBuffer[id].entries()].sort());
let mergedValues = {};
let didMergeValues = {};
valueBuffer[id].forEach((value, value_time_s) => {
if (value_time_s < time_s) {
mergedValues = {...mergedValues, ...value};
} else {
if (Object.keys(didMergeValues).length === 0) {
didMergeValues = clone(mergedValues);
}
Object.keys(value).forEach((key) => {
if(!didMergeValues.hasOwnProperty(key)) {
mergedValues[key] = value[key];
}
});
}
});
return mergedValues;
}
};
// public
this.NO_TIME = NO_TIME;
this.addValues = addValues;
this.getValues = getValues;
this.register = register;
this.deregister = deregister;
};
const LiveUpdater = function(tp, buffy) {
const toUpdate = [];
const update = () => {
const time_s = tp.sheet.sequence.position;
}
this.add = (layer) => {
if (toUpdate.indexOf(layer) < 0) {
toUpdate.push(layer);
}
};
this.remove = (layer) => {
const index = toUpdate.indexOf(layer);
if (index >= 0) {
toUpdate.push(layer);
}
};
this.immediateUpdate = (layer, values) => {
const v = {...layer.theatreObject.value, ...values};
const p = layer.values2cppProps(v);
if (p !== false) {
const id = layer.id();
if (id !== 'artboard') {
Module.setProps(p, layer.id());
} else {
Module.setArtboardProps(p, layer.id());
}
}
};
};
const Record = function(tp) {
const hot = {};
let isRecording = false;
const buffy = new LiveBuffer();
const liveUpdater = new LiveUpdater(tp, buffy);
const isHot = (layerID, propTitle) => {
return hot.hasOwnProperty(layerID)
&& hot[layerID].hasOwnProperty(propTitle);
};
const makeHot = (layerID, propTitle) => {
if (!isHot(layerID, propTitle)) {
if (!hot.hasOwnProperty(layerID)) {
hot[layerID] = {};
}
hot[layerID][propTitle] = {
recording: [],
};
}
const button = tp
.getPanelPropContainer(propTitle)
.querySelector('.recordButton');
if (button !== null) {
button.classList.add('active');
}
};
//const makeNotHot = (layerID, propTitle) => {
//if (isHot(layerID, propTitle)) {
//// whatever
//}
//};
const addRecordButton = (layer, propTitle) => {
const panel = tp.getPanel();
@ -22,24 +178,24 @@ const Record = function(tp) {
button.innerHTML = `<img src="/web/assets/record.svg" alt="record" />`;
container.append(button);
button.addEventListener('click', () => {
if (!hot.hasOwnProperty(layer.id())) {
hot[layer.id()] = {};
}
if (!hot[layer.id()].hasOwnProperty(propTitle)) {
hot[layer.id()][propTitle] = {
recording: [],
};
button.classList.add('active');
startRecording();
} else {
if(isRecording) {
stopRecording();
delete hot[layer.id()][propTitle];
if (Object.keys(hot[layer.id()]).length === 0) {
delete hot[layer.id()];
}
button.classList.remove('active');
} else {
Object.keys(audio.mapping)
.forEach((layerID) => {
if (getLayer(layerID).isSelected()) {
Object.keys(audio.mapping[layerID])
.forEach((propTitle) => {
makeHot(layerID, propTitle);
});
buffy.register(layerID);
}
});
startRecording();
}
});
console.log("Record::addRecordButton",
`added a record button for ${propTitle}`);
}
} else {
console.log("Record::addRecordButton",
@ -127,12 +283,21 @@ const Record = function(tp) {
// NOTE: layerID is not actually used atm
// and should be the layer anyways
const input_clone = cloneInput(layerID, propTitle);
let lastPosition = buffy.NO_TIME;
if (input_clone !== null) {
input_clone.addEventListener('change', (e) => {
const position = tp.sheet.sequence.position;
const value = parseFloat(input_clone.value);
hot[layerID][propTitle].recording.push({
position: tp.sheet.sequence.position,
value: parseFloat(input_clone.value),
position,
value,
});
const recording = {
[propTitle]: value,
};
buffy.addValues(layerID, recording, position, lastPosition);
liveUpdater.immediateUpdate(layer, recording);
lastPosition = position;
});
} else {
console.log('whoops input_clone is null');
@ -144,32 +309,70 @@ const Record = function(tp) {
isRecording = true;
};
const stopRecording = () => {
console.log('stoprecording');
const layerKeys = Object.keys(hot);
console.log({layerKeys});
layerKeys.forEach((layerID) => {
console.log(layerID);
const layer = getLayer(layerID);
layer.updateValuesViaTheatre(true);
const propTitles = Object.keys(hot[layerID]);
const keyframes = [];
propTitles.forEach((propTitle) => {
console.log(propTitle);
// NOTE: layerID is not actually used atm
// and should be the layer anyways
uncloneInput(layerID, propTitle);
console.log('should have uncloned input for ' + propTitle);
keyframes.push({
path: [propTitle],
keyframes: hot[layerID][propTitle].recording,
return new Promise((resolve) => {
console.log('stoprecording');
const layerKeys = Object.keys(hot);
console.log('stopRecording', 'layerKeys', {
layerKeys
}, 'hot', JSON.stringify(hot));
const promises = [];
promises.push(() => {
return new Promise((subResolve) => {
const audioOptionsButtons = tp.getPanel()
.querySelectorAll(`.audioButton.active`);
if (audioOptionsButtons !== null) {
audioOptionsButtons.forEach((audioOptionsButton) => {
audioOptionsButton.click();
});
}
subResolve();
});
});
setTimeout(() => {
console.log('adding the keyframes now because we wnat it to happen right now please');
tp.addKeyframes(layer, keyframes);
}, 2000);
layerKeys.forEach((layerID) => {
console.log('stopRecording', layerID);
const layer = getLayer(layerID);
const propTitles = Object.keys(hot[layerID]);
const keyframes = [];
propTitles.forEach((propTitle) => {
console.log('stopRecording', propTitle);
// NOTE: layerID is not actually used atm
// and should be the layer anyways
uncloneInput(layerID, propTitle);
console.log('stopRecording', 'should have uncloned input for ' + propTitle);
keyframes.push({
path: [propTitle],
keyframes: hot[layerID][propTitle].recording,
});
});
//setTimeout(() => {
console.log('stopRecording', 'adding the keyframes now because we wnat it to happen right now please', keyframes);
promises.push(() => {
return new Promise((subResolve) => {
tp.setKeyframes(layer, keyframes).then(() => {
layer.updateValuesViaTheatre(true);
subResolve();
});
})
});
//}, 2000);
});
sequencialPromises(promises, () => {
Object.keys(hot).forEach((layerID) => {
buffy.deregister(layerID);
Object.keys(hot[layerID]).forEach((propTitle) => {
delete hot[layerID][propTitle];
if (Object.keys(hot[layerID]).length === 0) {
delete hot[layerID];
}
const button = tp.getPanel().querySelector(`.recordButton${propTitle}`);
button.classList.remove('active');
});
});
console.log('stopRecording', 'absolutely stopped recording');
isRecording = false;
resolve();
});
});
isRecording = false;
};
// public

View file

@ -7,6 +7,7 @@ import {
clone,
getParents,
arraysEqual,
sequencialPromises,
} from './utils.js';
//import {
@ -140,16 +141,53 @@ const TheatrePlay = function(autoInit = false) {
}
return t.parentElement.querySelector('[title="Sequence this prop"]');
};
// no idea how to delete keyframes
// so we can only add
const setSequenced = (propTitle, sequenced) => {
return new Promise((resolve) => {
const contextItem = sequenced ? 'sequence' : 'make static';
const antiContextItem = sequenced ? 'make static' : 'sequence';
const finishedSequencedEvent = (e) => {
tp.getPanel().removeEventListener('injected', finishedSequencedEvent);
console.log('debug FINISHED SEQUENCED EVENT', e, propTitle);
resolve(true);
};
const clickContextMenu = () => {
let done = false;
tp.getPanelPropTitle(propTitle).removeEventListener('contextmenu', clickContextMenu);
tp.shadowRoot.querySelectorAll('ul li span').forEach((s) => {
if (s.innerHTML.toLowerCase() === contextItem.toLowerCase()) {
tp.getPanel().addEventListener('injected', finishedSequencedEvent);
s.click();
console.log('debug click');
done = true;
} else if (s.innerHTML.toLowerCase() === antiContextItem.toLowerCase()) {
done = true;
resolve(false);
}
});
if (!done) {
setTimeout(() => {
clickContextMenu();
}, 100);
}
};
getPanelPropTitle(propTitle).addEventListener('contextmenu', clickContextMenu);
getPanelPropTitle(propTitle).dispatchEvent(new Event('contextmenu'));
});
};
const addKeyframes = (layer, keyframes) => {
return new Promise((resolve) => {
if (!Array.isArray(keyframes)) {
resolve(false);
return false;
}
const existingKeyframes = getKeyframes(layer);
const promises = [];
const ms = config.tp.addKeyframesTimeout_s * 1000;
const ms = 0;//config.tp.addKeyframesTimeout_s * 1000;
keyframes.forEach((k) => {
let prop = layer.theatreObject.props;
for (let i = 0; i < k.path.length; i++) {
@ -159,12 +197,23 @@ const TheatrePlay = function(autoInit = false) {
// NOTE: can we sequence values without pretend clicking?
const sequenceButton = getSequenceButton(k.path);
if (sequenceButton !== null) {
promises.push(new Promise((subResolve) => {
promises.push(() => { return new Promise((subResolve) => {
setTimeout(() => {
sequenceButton.click();
subResolve();
}, ms * promises.length);
}));
const detectSE = (e) => {
if (e.detail.panelID === layer.id()) {
window.removeEventListener('sequenceEvent',detectSE);
console.log('received sequenceEvent',e);
const f = (e) => {
tp.getPanel().removeEventListener('injected', f);
subResolve();
};
tp.getPanel().addEventListener('injected', f);
}
};
window.addEventListener('sequenceEvent', detectSE);
}, ms);// * promises.length);
})});
} else {
//console.error(k.path, 'did not find sequence button');
// is (probably) already sequenced
@ -191,7 +240,7 @@ const TheatrePlay = function(autoInit = false) {
});
}
if (!alreadyThere) {
promises.push(new Promise((subResolve) => {
promises.push(() => { return new Promise((subResolve) => {
setTimeout(() => {
tp.sheet.sequence.position = keyframe.position;
this.studio.transaction(({
@ -200,22 +249,46 @@ const TheatrePlay = function(autoInit = false) {
set(prop, keyframe.value);
subResolve();
});
}, ms * promises.length);
}));
}, ms);// * promises.length);
})});
}
});
promises.push(new Promise((subResolve) => {
promises.push(() => { return new Promise((subResolve) => {
setTimeout(() => {
tp.sheet.sequence.position = position;
subResolve();
}, ms * promises.length);
}));
});
Promise.all(promises).then(() => {
resolve();
}, ms);// * promises.length);
})});
});
sequencialPromises(promises, resolve);
//Promise.all(promises).then(() => {
//resolve();
//});
});
};
const setKeyframes = (layer, keyframes) => {
return new Promise((resolve) => {
if (!Array.isArray(keyframes)) {
resolve(false);
return false;
}
const promises = [];
keyframes.forEach((k) => {
promises.push(new Promise((subResolve) => {
const propTitle = k.path.join('.');
setSequenced(propTitle, false)
.then(subResolve);
}));
});
Promise
.all(promises)
.then(() => {
addKeyframes(layer, keyframes)
.then(resolve);
});
});
};
const friendlySequenceNames = () => {
const sequencePanelLeft = tp.getSequencePanelLeft();
let doItAgain = true;
@ -249,9 +322,11 @@ const TheatrePlay = function(autoInit = false) {
};
//public
this.setSequenced = setSequenced;
this.friendlySequenceNames = friendlySequenceNames;
this.getKeyframes = getKeyframes;
this.addKeyframes = addKeyframes;
this.setKeyframes = setKeyframes;
this.theatreObjects = theatreObjects;
this.addObject = (name, props, onValuesChange) => {
const obj = this.sheet.object(name, props);

View file

@ -395,6 +395,15 @@ const isMobile = () => {
return false;
};
const sequencialPromises = async (iterable, callback = false) => {
for (const x of iterable) {
await x();
}
if (callback !== false) {
callback();
}
};
/////////////////////////////////////
export {
@ -415,4 +424,5 @@ export {
arraysEqual,
mapValue,
isMobile,
sequencialPromises,
}