variablelab/variabletime
variablelab/variabletime
4
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Compare commits

base: variablelab:667447b9a2b7929cec86e400b6c63fa61d19e434
Branches Tags
variablelab:main
variablelab:audio
...
compare: variablelab:35ed6009e23ad760923a279e30f9827c82ca8364
Branches Tags
variablelab:audio
variablelab:main

2 commits
667447b9a2 ... 35ed6009e2

Author SHA1 Message Date
themancalledjakob
35ed6009e2 draw audio options 2023-10-08 21:26:22 +02:00
themancalledjakob
bb76bd09c6 fix remove color mapping & options 2023-10-08 17:28:15 +02:00
3 changed files with 188 additions and 69 deletions
Show stats Expand all files Collapse all files

249
bin/web/js/audio.js
View file

@ -11,8 +11,10 @@ window.mapValue = mapValue;
const AudioMappingOptions = function() { const AudioMappingOptions = function() {
this.freq_min = 0.0; this.min_freq = 0.0;
this.freq_max = config.audio.fftBandsUsed; this.max_freq = config.audio.fftBandsUsed;
this.min_in = 0.0;
this.max_in = 255.0 / 2;
this.min_out = 0.0; this.min_out = 0.0;
this.max_out = 1.0; this.max_out = 1.0;
this.smoothing = config.audio.defaultSmoothing; this.smoothing = config.audio.defaultSmoothing;
@ -43,14 +45,41 @@ const Audio = function(tp, record) {
let started = false; let started = false;
const mapping = {}; const mapping = {};
const canvass = []; //const canvass = [];
const canvasCtxs = []; let canvasCombos = {};
const mutationObserver = new MutationObserver(function(e) {
if (e[0].removedNodes) {
e[0].removedNodes.forEach((n) => {
if (n.hasAttribute('data-propTitle')) {
const propTitle = n.getAttribute('data-propTitle');
delete canvasCombos[propTitle];
} else {
const subProps = n.querySelectorAll('[data-propTitle]');
if (subProps.length > 0) {
subProps.forEach((sp) => {
const propTitle = sp.getAttribute('data-propTitle');
delete canvasCombos[propTitle];
});
}
}
});
}
});
let areMutationsObserved = false;
const isMapped = (layer, propTitle) => { const isMapped = (layer, propTitle) => {
if (!mapping.hasOwnProperty(layer.id())) { if (!mapping.hasOwnProperty(layer.id())) {
return false; return false;
} }
if (!mapping[layer.id()].hasOwnProperty(propTitle)) { if (!mapping[layer.id()].hasOwnProperty(propTitle)) {
if (propTitle === 'color' &&
config.audio.colorSeparateRGBA &&
mapping[layer.id()].hasOwnProperty('color.r') &&
mapping[layer.id()].hasOwnProperty('color.g') &&
mapping[layer.id()].hasOwnProperty('color.b') &&
mapping[layer.id()].hasOwnProperty('color.a')) {
return true;
}
return false; return false;
} }
return true; return true;
@ -121,7 +150,7 @@ const Audio = function(tp, record) {
if (!mapping[layer.id()].hasOwnProperty(propTitle)) { if (!mapping[layer.id()].hasOwnProperty(propTitle)) {
// no propTitle // no propTitle
// perhaps color? // perhaps color?
if (config.audio.separateRGBA && propTitle === 'color') { if (config.audio.colorSeparateRGBA && propTitle === 'color') {
let isGood = true; let isGood = true;
isGood = removeAudioMapping(layer, 'color.r'); isGood = removeAudioMapping(layer, 'color.r');
isGood = removeAudioMapping(layer, 'color.g'); isGood = removeAudioMapping(layer, 'color.g');
@ -141,33 +170,39 @@ const Audio = function(tp, record) {
const createAudioOptions = (layer, propTitle, container) => { const createAudioOptions = (layer, propTitle, container) => {
const mappingOptions = mapping[layer.id()][propTitle]; const mappingOptions = mapping[layer.id()][propTitle];
const panel = tp.getPanel(); const panel = tp.getPanel();
if (!areMutationsObserved) {
mutationObserver.observe(panel, { childList: true, subtree: true });
areMutationsObserved = true;
}
const audioOptions = document.createElement('div'); const audioOptions = document.createElement('div');
audioOptions.setAttribute('data-propTitle',propTitle);
audioOptions.classList.add('audioOptions'); audioOptions.classList.add('audioOptions');
audioOptions.classList.add('audioOptionsTypeDefault'); audioOptions.classList.add('audioOptionsTypeDefault');
audioOptions.classList.add(toCssClass(`audioOptions${propTitle}`)); audioOptions.classList.add(toCssClass(`audioOptions${propTitle}`));
audioOptions.style.position = 'relative'; audioOptions.style.position = 'relative';
audioOptions.style.width = '100%'; audioOptions.style.width = '100%';
if (propTitle.split('.')[0] === 'color' && propTitle.split('.').length > 1) { if (propTitle.split('.')[0] === 'color' && propTitle.split('.').length > 1) {
audioOptions.classList.add(toCssClass('audioOptionscolor'));
switch(propTitle.split('.')[1]) { switch(propTitle.split('.')[1]) {
case 'r': { case 'r': {
audioOptions.style.background = 'rgba(255,0,0,0.2)'; audioOptions.style.background = 'rgba(255,0,0,0.2)'; // AUDIO COLOR
break; break;
} }
case 'g': { case 'g': {
audioOptions.style.background = 'rgba(0,255,0,0.2)'; audioOptions.style.background = 'rgba(0,255,0,0.2)'; // AUDIO COLOR
break; break;
} }
case 'b': { case 'b': {
audioOptions.style.background = 'rgba(0,0,255,0.2)'; audioOptions.style.background = 'rgba(0,0,255,0.2)'; // AUDIO COLOR
break; break;
} }
case 'a': { case 'a': {
audioOptions.style.background = 'rgba(255,255,255,0.2)'; audioOptions.style.background = 'rgba(255,255,255,0.2)'; // AUDIO COLOR
break; break;
} }
} }
} else { } else {
audioOptions.style.background = 'rgba(0,255,255,0.2)'; audioOptions.style.background = 'rgba(0,255,255,0.2)'; // AUDIO COLOR
} }
audioOptions.style.order = parseInt(container.style.order) + 1; audioOptions.style.order = parseInt(container.style.order) + 1;
@ -248,6 +283,7 @@ const Audio = function(tp, record) {
fft_Dom.style.top = '0px'; fft_Dom.style.top = '0px';
fft_Dom.style.left = '0px'; fft_Dom.style.left = '0px';
fft_imgDom.classList.add('audio_fft'); fft_imgDom.classList.add('audio_fft');
fft_imgDom.classList.add(toCssClass(`audio_fft${propTitle}`));
fft_imgDom.style.width = '100%'; fft_imgDom.style.width = '100%';
fft_imgDom.style.userDrag = 'none'; fft_imgDom.style.userDrag = 'none';
fft_imgDom.style.userSelect = 'none'; fft_imgDom.style.userSelect = 'none';
@ -260,8 +296,8 @@ const Audio = function(tp, record) {
fft_selectDom.style.width = '100%'; fft_selectDom.style.width = '100%';
fft_selectDom.style.height = '100%'; fft_selectDom.style.height = '100%';
fft_selectDom.style.pointerEvents = 'none'; fft_selectDom.style.pointerEvents = 'none';
fft_selectDom.style.backgroundColor = 'rgba(0,255,0,0.2)'; fft_selectDom.style.backgroundColor = 'rgba(0,255,0,0.2)'; // AUDIO COLOR
fft_selectDom.style.border = '1px solid rgba(0,255,0,1.0)'; fft_selectDom.style.border = '1px solid rgba(0,255,0,1.0)'; // AUDIO COLOR
fft_Dom.append(fft_imgDom); fft_Dom.append(fft_imgDom);
fft_Dom.append(fft_selectDom); fft_Dom.append(fft_selectDom);
audioOptions.append(fft_Dom); audioOptions.append(fft_Dom);
@ -271,23 +307,53 @@ const Audio = function(tp, record) {
let setFrequency = false; let setFrequency = false;
let freq_down = 0; let freq_down = 0;
let freq_up = 0; let freq_up = 0;
let xy_start;
fft_Dom.addEventListener('mousedown', (e) => { fft_Dom.addEventListener('mousedown', (e) => {
setFrequency = true; setFrequency = true;
const bb = fft_Dom.getBoundingClientRect(); const bb = fft_imgDom.getBoundingClientRect();
const x = e.clientX - bb.x; const x = e.clientX - bb.x;
freq_down = mapValue(e.clientX, bb.x, bb.x + bb.width, 0, config.audio.fftBandsUsed, true); const y = e.clientY - bb.y;
xy_start = {x, y};
});
fft_Dom.addEventListener('mousemove', (e) => {
if (setFrequency) {
const bb = fft_imgDom.getBoundingClientRect();
const x_factor = config.audio.fftBandsUsed / bb.width;
const y_factor = 256.0 / bb.height;
const x = e.clientX - bb.x;
const y = e.clientY - bb.y;
let min_x, max_x, min_y, max_y;
if (x > xy_start.x) {
min_x = xy_start.x;
max_x = x;
} else {
min_x = x;
max_x = xy_start.x;
}
if (y > xy_start.y) {
min_y = xy_start.y;
max_y = y;
} else {
min_y = y;
max_y = xy_start.y;
}
mappingOptions.min_freq = min_x * x_factor;
mappingOptions.max_freq = max_x * x_factor;
mappingOptions.min_in = (bb.height - max_y) * y_factor;
mappingOptions.max_in = (bb.height - min_y) * y_factor;
}
}); });
fft_Dom.addEventListener('mouseup', (e) => { fft_Dom.addEventListener('mouseup', (e) => {
setFrequency = false; setFrequency = false;
const bb = fft_Dom.getBoundingClientRect();
const x = e.clientX - bb.x;
freq_down = mapValue(e.clientX, bb.x, bb.x + bb.width, 0, config.audio.fftBandsUsed, true);
}); });
//fft_Dom.addEventListener('mouseout', (e) => {
//setFrequency = false;
//});
container.after(audioOptions); container.after(audioOptions);
canvass.push(fft_imgDom); //canvass.push(fft_imgDom);
canvasCtxs.push(fft_imgDom.getContext("2d")); canvasCombos[propTitle] = [fft_imgDom, fft_imgDom.getContext("2d"), layer.id()];
updateMappingOptions(); updateMappingOptions();
return audioOptions; return audioOptions;
}; };
@ -331,6 +397,7 @@ const Audio = function(tp, record) {
allAudioOptions[i].remove(); allAudioOptions[i].remove();
} }
} }
canvasCombos = {};
panel.querySelectorAll('.audioButton').forEach((button) => { panel.querySelectorAll('.audioButton').forEach((button) => {
button.classList.remove('active'); button.classList.remove('active');
}); });
@ -338,6 +405,14 @@ const Audio = function(tp, record) {
// only selected layers have options // only selected layers have options
// otherwise the ui is not there // otherwise the ui is not there
if (layer.isSelected()) { if (layer.isSelected()) {
if (config.audio.colorSeparateRGBA && propTitle === 'color') {
delete canvasCombos[layer.id()]['color.r'];
delete canvasCombos[layer.id()]['color.g'];
delete canvasCombos[layer.id()]['color.b'];
delete canvasCombos[layer.id()]['color.a'];
} else {
delete canvasCombos[layer.id()][propTitle];
}
const audioOptions = panel.querySelectorAll(toCssClass(`audioOptions${propTitle}`,'.')); const audioOptions = panel.querySelectorAll(toCssClass(`audioOptions${propTitle}`,'.'));
if (audioOptions.length > 0) { if (audioOptions.length > 0) {
audioOptions.forEach((e) => { e.remove(); }); audioOptions.forEach((e) => { e.remove(); });
@ -513,7 +588,7 @@ const Audio = function(tp, record) {
convolver.buffer = buffer; convolver.buffer = buffer;
}, },
function(e) { function(e) {
console.log("Error with decoding audio data" + e.err); console.log("Audio::audioCtx.decodeAudioData", "Error with decoding audio data" + e.err);
} }
); );
}; };
@ -563,60 +638,63 @@ const Audio = function(tp, record) {
const visualSetting = visualSelect.value; const visualSetting = visualSelect.value;
if (visualSetting === "sinewave") { if (visualSetting === "sinewave") {
analyser.fftSize = 2048; //analyser.fftSize = 2048;
const bufferLength = analyser.fftSize; //const bufferLength = analyser.fftSize;
// We can use Float32Array instead of Uint8Array if we want higher precision //// We can use Float32Array instead of Uint8Array if we want higher precision
// const dataArray = new Float32Array(bufferLength); //// const dataArray = new Float32Array(bufferLength);
const dataArray = new Uint8Array(bufferLength); //const dataArray = new Uint8Array(bufferLength);
canvasCtx.clearRect(0, 0, WIDTH, HEIGHT); //canvasCtx.clearRect(0, 0, WIDTH, HEIGHT);
const draw = function() { //const draw = function() {
drawVisual = requestAnimationFrame(draw); //drawVisual = requestAnimationFrame(draw);
analyser.getByteTimeDomainData(dataArray); //analyser.getByteTimeDomainData(dataArray);
canvasCtx.fillStyle = "rgb(200, 200, 200)"; //canvasCtx.fillStyle = "rgb(200, 200, 200)";
canvasCtx.fillRect(0, 0, WIDTH, HEIGHT); //canvasCtx.fillRect(0, 0, WIDTH, HEIGHT);
canvasCtx.lineWidth = 2; //canvasCtx.lineWidth = 2;
canvasCtx.strokeStyle = "rgb(0, 0, 0)"; //canvasCtx.strokeStyle = "rgb(0, 0, 0)";
canvasCtx.beginPath(); //canvasCtx.beginPath();
const sliceWidth = (WIDTH * 1.0) / bufferLength; //const sliceWidth = (WIDTH * 1.0) / bufferLength;
let x = 0; //let x = 0;
for (let i = 0; i < bufferLength; i++) { //for (let i = 0; i < bufferLength; i++) {
let v = dataArray[i] / 128.0; //let v = dataArray[i] / 128.0;
let y = (v * HEIGHT) / 2; //let y = (v * HEIGHT) / 2;
if (i === 0) { //if (i === 0) {
canvasCtx.moveTo(x, y); //canvasCtx.moveTo(x, y);
} else { //} else {
canvasCtx.lineTo(x, y); //canvasCtx.lineTo(x, y);
} //}
x += sliceWidth; //x += sliceWidth;
} //}
canvasCtx.lineTo(canvas.width, canvas.height / 2); //canvasCtx.lineTo(canvas.width, canvas.height / 2);
canvasCtx.stroke(); //canvasCtx.stroke();
}; //};
draw(); //draw();
} else if (visualSetting == "frequencybars") { } else if (visualSetting == "frequencybars") {
analyser.fftSize = config.audio.fftBandsAnalysed; analyser.fftSize = config.audio.fftBandsAnalysed;
const w = config.audio.fftBandsUsed; const w = config.audio.fftBandsUsed;
const h = config.audio.fftHeight; const h = config.audio.fftHeight;
const verticalFactor = h / 256.0;
const bufferLengthAlt = analyser.frequencyBinCount / 2; const bufferLengthAlt = analyser.frequencyBinCount / 2;
// See comment above for Float32Array() // See comment above for Float32Array()
const dataArrayAlt = new Uint8Array(bufferLengthAlt); const dataArrayAlt = new Uint8Array(bufferLengthAlt);
for (let i = 0; i < canvasCtxs.length; i++) { const canvasKeys = Object.keys(canvasCombos);
canvasCtxs[i].clearRect(0, 0, w, h);
for (let i = 0; i < canvasKeys.length; i++) {
canvasCombos[canvasKeys[i]][1].clearRect(0, 0, w, h);
} }
let frameCount = 0; let frameCount = 0;
@ -625,35 +703,76 @@ const Audio = function(tp, record) {
analyser.getByteFrequencyData(dataArrayAlt); analyser.getByteFrequencyData(dataArrayAlt);
for (let i = 0; i < canvasCtxs.length; i++) { for (let i = 0; i < canvasKeys.length; i++) {
canvasCtxs[i].fillStyle = "rgb(0, 0, 0)"; canvasCombos[canvasKeys[i]][1].fillStyle = "rgb(0, 0, 0)"; // AUDIO COLOR
canvasCtxs[i].fillRect(0, 0, w, h); canvasCombos[canvasKeys[i]][1].fillRect(0, 0, w, h);
const layerID = canvasCombos[canvasKeys[i]][2];
const m = mapping[layerID][canvasKeys[i]];
if (m.sync === 'volume') {
const sx = m.min_freq;
const sw = m.max_freq - m.min_freq;
const sy = h - (m.max_in * verticalFactor);
const sh = (m.max_in - m.min_in) * verticalFactor;
canvasCombos[canvasKeys[i]][1].fillStyle = "rgb(80, 80, 80)"; // AUDIO COLOR
canvasCombos[canvasKeys[i]][1].fillRect(sx, sy, sw, sh);
} else if (m.sync === 'pitch') {
const sx = m.min_freq;
const sw = m.max_freq - m.min_freq;
const sy = 0;
const sh = h;
canvasCombos[canvasKeys[i]][1].fillStyle = "rgb(80, 80, 80)"; // AUDIO COLOR
canvasCombos[canvasKeys[i]][1].fillRect(sx, sy, sw, sh);
}
} }
const barWidth = (w / bufferLengthAlt) * 2.5; const barWidth = 1;//(w / bufferLengthAlt) * 2.5;
let barHeight; let barHeight;
let x = 0; let x = 0;
let max_i = 0; let max_i = 0;
let max_v = 0; let max_v = 0;
for (let i = 0; i < bufferLengthAlt; i++) { let min_v = 9999999;
for (let i = 0; i < w; i++) {
barHeight = dataArrayAlt[i]; barHeight = dataArrayAlt[i];
if (barHeight > max_v) { if (barHeight > max_v) {
max_v = barHeight; max_v = barHeight;
max_i = i; max_i = i;
} }
for (let i = 0; i < canvasCtxs.length; i++) { if (barHeight < min_v) {
canvasCtxs[i].fillStyle = "rgb(" + (barHeight + 100) + ",50,50)"; min_v = barHeight;
canvasCtxs[i].fillRect( }
for (let i = 0; i < canvasKeys.length; i++) {
canvasCombos[canvasKeys[i]][1].fillStyle = "rgb(200,200,200)"; // AUDIO COLOR
canvasCombos[canvasKeys[i]][1].fillRect(
x, x,
h - barHeight / 2, h - (barHeight * verticalFactor),
barWidth, barWidth,
barHeight / 2 (barHeight * verticalFactor)
); );
} }
x += barWidth + 1; x += barWidth;
}
for (let i = 0; i < canvasKeys.length; i++) {
const layerID = canvasCombos[canvasKeys[i]][2];
const m = mapping[layerID][canvasKeys[i]];
if (m.sync === 'volume') {
const sx = m.min_freq;
const sw = m.max_freq - m.min_freq;
const sy = h - (m.max_in * verticalFactor);
const sh = (m.max_in - m.min_in) * verticalFactor;
canvasCombos[canvasKeys[i]][1].strokeStyle = "rgb(255, 255, 255)"; // AUDIO COLOR
canvasCombos[canvasKeys[i]][1].strokeRect(sx, sy, sw, sh);
} else if (m.sync === 'pitch') {
const m = mapping[layerID][canvasKeys[i]];
const sx = m.min_freq;
const sw = m.max_freq - m.min_freq;
const sy = 0;
const sh = h;
canvasCombos[canvasKeys[i]][1].strokeStyle = "rgb(255, 255, 255)"; // AUDIO COLOR
canvasCombos[canvasKeys[i]][1].strokeRect(sx, sy, sw, sh);
}
} }
const propsToSet = []; const propsToSet = [];
getLayers().forEach((layer) => { getLayers().forEach((layer) => {
@ -662,7 +781,7 @@ const Audio = function(tp, record) {
const m = mapping[layer.id()][propTitle]; const m = mapping[layer.id()][propTitle];
switch (m.sync) { switch (m.sync) {
case 'volume': { case 'volume': {
let a = mapValue(max_v, 0, 255, m.min_out, m.max_out, true); let a = mapValue(max_v, m.min_in, m.max_in, m.min_out, m.max_out, true);
m.value = m.value * m.smoothing + (1.0 - m.smoothing) * a; m.value = m.value * m.smoothing + (1.0 - m.smoothing) * a;
propsToSet.push({ propsToSet.push({
layer, layer,
@ -674,7 +793,7 @@ const Audio = function(tp, record) {
break; break;
} }
case 'pitch': { case 'pitch': {
let a = mapValue(max_i, 0, bufferLengthAlt - 1, m.min_out, m.max_out, true); let a = mapValue(max_i, m.min_freq, m.max_freq, m.min_out, m.max_out, true);
m.value = m.value * m.smoothing + (1.0 - m.smoothing) * a; m.value = m.value * m.smoothing + (1.0 - m.smoothing) * a;
propsToSet.push({ propsToSet.push({
layer, layer,

4
bin/web/js/config.js
View file

@ -86,8 +86,8 @@ const config = {
ignoreProps: ['transformOrigin', 'fontFamily', 'text', 'mirror_x', 'mirror_y', 'mirror_xy'], ignoreProps: ['transformOrigin', 'fontFamily', 'text', 'mirror_x', 'mirror_y', 'mirror_xy'],
defaultSmoothing: 0.7, defaultSmoothing: 0.7,
fftBandsAnalysed: 256 * 8, fftBandsAnalysed: 256 * 8,
fftBandsUsed: 256 * 8 / 2, fftBandsUsed: 256 / 2,
fftHeight: 256 / 2, fftHeight: 256 / 4,
colorSeparateRGBA: true, colorSeparateRGBA: true,
}, },
record: { record: {

4
bin/web/js/main.js
View file

@ -465,14 +465,14 @@ const duplicateLayer = (originalLayer) => {
//delete originalValues[originalKeys[i]]; //delete originalValues[originalKeys[i]];
allKeysFound = false; allKeysFound = false;
} }
}; }
if (allKeysFound) { if (allKeysFound) {
tp.getPanel().removeEventListener("injected", addKeyframes); tp.getPanel().removeEventListener("injected", addKeyframes);
} }
tp.addKeyframes(newLayer, originalKeyframes).then(() => { tp.addKeyframes(newLayer, originalKeyframes).then(() => {
if (allKeysFound) { if (allKeysFound) {
resolve(); resolve();
}; }
}); });
}; };
tp.getPanel().addEventListener("injected", addKeyframes); tp.getPanel().addEventListener("injected", addKeyframes);