draw audio options

bin/web/js/audio.js

@@ -11,8 +11,10 @@ window.mapValue = mapValue;
 
 
 const AudioMappingOptions = function() {
-    this.freq_min = 0.0;
-    this.freq_max = config.audio.fftBandsUsed;
+    this.min_freq = 0.0;
+    this.max_freq = config.audio.fftBandsUsed;
+    this.min_in = 0.0;
+    this.max_in = 255.0 / 2;
     this.min_out = 0.0;
     this.max_out = 1.0;
     this.smoothing = config.audio.defaultSmoothing;
@@ -43,14 +45,41 @@ const Audio = function(tp, record) {
     let started = false;
 
     const mapping = {};
-    const canvass = [];
-    const canvasCtxs = [];
+    //const canvass = [];
+    let canvasCombos = {};
+    const mutationObserver = new MutationObserver(function(e) {
+        if (e[0].removedNodes) {
+            e[0].removedNodes.forEach((n) => {
+                if (n.hasAttribute('data-propTitle')) {
+                    const propTitle = n.getAttribute('data-propTitle');
+                    delete canvasCombos[propTitle];
+                } else {
+                    const subProps = n.querySelectorAll('[data-propTitle]');
+                    if (subProps.length > 0) {
+                        subProps.forEach((sp) => {
+                            const propTitle = sp.getAttribute('data-propTitle');
+                            delete canvasCombos[propTitle];
+                        });
+                    }
+                }
+            });
+        }
+    });
+    let areMutationsObserved = false;
 
     const isMapped = (layer, propTitle) => {
         if (!mapping.hasOwnProperty(layer.id())) {
             return false;
         }
         if (!mapping[layer.id()].hasOwnProperty(propTitle)) {
+            if (propTitle === 'color' &&
+                config.audio.colorSeparateRGBA &&
+                mapping[layer.id()].hasOwnProperty('color.r') &&
+                mapping[layer.id()].hasOwnProperty('color.g') &&
+                mapping[layer.id()].hasOwnProperty('color.b') &&
+                mapping[layer.id()].hasOwnProperty('color.a')) {
+                return true;
+            }
             return false;
         }
         return true;
@@ -121,7 +150,7 @@ const Audio = function(tp, record) {
         if (!mapping[layer.id()].hasOwnProperty(propTitle)) {
             // no propTitle
             // perhaps color?
-            if (config.audio.separateRGBA && propTitle === 'color') {
+            if (config.audio.colorSeparateRGBA && propTitle === 'color') {
                 let isGood = true;
                 isGood = removeAudioMapping(layer, 'color.r');
                 isGood = removeAudioMapping(layer, 'color.g');
@@ -141,33 +170,39 @@ const Audio = function(tp, record) {
     const createAudioOptions = (layer, propTitle, container) => {
         const mappingOptions = mapping[layer.id()][propTitle];
         const panel = tp.getPanel();
+        if (!areMutationsObserved) {
+            mutationObserver.observe(panel, { childList: true, subtree: true });
+            areMutationsObserved = true;
+        }
         const audioOptions = document.createElement('div');
+        audioOptions.setAttribute('data-propTitle',propTitle);
         audioOptions.classList.add('audioOptions');
         audioOptions.classList.add('audioOptionsTypeDefault');
         audioOptions.classList.add(toCssClass(`audioOptions${propTitle}`));
         audioOptions.style.position = 'relative';
         audioOptions.style.width = '100%';
         if (propTitle.split('.')[0] === 'color' && propTitle.split('.').length > 1) {
+            audioOptions.classList.add(toCssClass('audioOptionscolor'));
             switch(propTitle.split('.')[1]) {
                 case 'r': {
-                    audioOptions.style.background = 'rgba(255,0,0,0.2)';
+                    audioOptions.style.background = 'rgba(255,0,0,0.2)'; // AUDIO COLOR
                     break;
                 }
                 case 'g': {
-                    audioOptions.style.background = 'rgba(0,255,0,0.2)';
+                    audioOptions.style.background = 'rgba(0,255,0,0.2)'; // AUDIO COLOR
                     break;
                 }
                 case 'b': {
-                    audioOptions.style.background = 'rgba(0,0,255,0.2)';
+                    audioOptions.style.background = 'rgba(0,0,255,0.2)'; // AUDIO COLOR
                     break;
                 }
                 case 'a': {
-                    audioOptions.style.background = 'rgba(255,255,255,0.2)';
+                    audioOptions.style.background = 'rgba(255,255,255,0.2)'; // AUDIO COLOR
                     break;
                 }
             }
         } else {
-            audioOptions.style.background = 'rgba(0,255,255,0.2)';
+            audioOptions.style.background = 'rgba(0,255,255,0.2)'; // AUDIO COLOR
         }
         audioOptions.style.order = parseInt(container.style.order) + 1;
 
@@ -248,6 +283,7 @@ const Audio = function(tp, record) {
         fft_Dom.style.top = '0px';
         fft_Dom.style.left = '0px';
         fft_imgDom.classList.add('audio_fft');
+        fft_imgDom.classList.add(toCssClass(`audio_fft${propTitle}`));
         fft_imgDom.style.width = '100%';
         fft_imgDom.style.userDrag = 'none';
         fft_imgDom.style.userSelect = 'none';
@@ -260,8 +296,8 @@ const Audio = function(tp, record) {
         fft_selectDom.style.width = '100%';
         fft_selectDom.style.height = '100%';
         fft_selectDom.style.pointerEvents = 'none';
-        fft_selectDom.style.backgroundColor = 'rgba(0,255,0,0.2)';
-        fft_selectDom.style.border = '1px solid rgba(0,255,0,1.0)';
+        fft_selectDom.style.backgroundColor = 'rgba(0,255,0,0.2)'; // AUDIO COLOR
+        fft_selectDom.style.border = '1px solid rgba(0,255,0,1.0)'; // AUDIO COLOR
         fft_Dom.append(fft_imgDom);
         fft_Dom.append(fft_selectDom);
         audioOptions.append(fft_Dom);
@@ -271,23 +307,53 @@ const Audio = function(tp, record) {
         let setFrequency = false;
         let freq_down = 0;
         let freq_up = 0;
+        let xy_start;
         fft_Dom.addEventListener('mousedown', (e) => {
             setFrequency = true;
-            const bb = fft_Dom.getBoundingClientRect();
+            const bb = fft_imgDom.getBoundingClientRect();
             const x = e.clientX - bb.x;
-            freq_down = mapValue(e.clientX, bb.x, bb.x + bb.width, 0, config.audio.fftBandsUsed, true);
+            const y = e.clientY - bb.y;
+            xy_start = {x, y};
+        });
+        fft_Dom.addEventListener('mousemove', (e) => {
+            if (setFrequency) {
+                const bb = fft_imgDom.getBoundingClientRect();
+                const x_factor = config.audio.fftBandsUsed / bb.width;
+                const y_factor = 256.0 / bb.height;
+                const x = e.clientX - bb.x;
+                const y = e.clientY - bb.y;
+                let min_x, max_x, min_y, max_y;
+                if (x > xy_start.x) {
+                    min_x = xy_start.x;
+                    max_x = x;
+                } else {
+                    min_x = x;
+                    max_x = xy_start.x;
+                }
+                if (y > xy_start.y) {
+                    min_y = xy_start.y;
+                    max_y = y;
+                } else {
+                    min_y = y;
+                    max_y = xy_start.y;
+                }
+                mappingOptions.min_freq = min_x * x_factor;
+                mappingOptions.max_freq = max_x * x_factor;
+                mappingOptions.min_in = (bb.height - max_y) * y_factor;
+                mappingOptions.max_in = (bb.height - min_y) * y_factor;
+            }
         });
         fft_Dom.addEventListener('mouseup', (e) => {
             setFrequency = false;
-            const bb = fft_Dom.getBoundingClientRect();
-            const x = e.clientX - bb.x;
-            freq_down = mapValue(e.clientX, bb.x, bb.x + bb.width, 0, config.audio.fftBandsUsed, true);
         });
+        //fft_Dom.addEventListener('mouseout', (e) => {
+            //setFrequency = false;
+        //});
 
         container.after(audioOptions);
 
-        canvass.push(fft_imgDom);
-        canvasCtxs.push(fft_imgDom.getContext("2d"));
+        //canvass.push(fft_imgDom);
+        canvasCombos[propTitle] = [fft_imgDom, fft_imgDom.getContext("2d"), layer.id()];
         updateMappingOptions();
         return audioOptions;
     };
@@ -331,6 +397,7 @@ const Audio = function(tp, record) {
                     allAudioOptions[i].remove();
                 }
             }
+            canvasCombos = {};
             panel.querySelectorAll('.audioButton').forEach((button) => {
                 button.classList.remove('active');
             });
@@ -338,6 +405,14 @@ const Audio = function(tp, record) {
             // only selected layers have options
             // otherwise the ui is not there
             if (layer.isSelected()) {
+                if (config.audio.colorSeparateRGBA && propTitle === 'color') {
+                    delete canvasCombos[layer.id()]['color.r'];
+                    delete canvasCombos[layer.id()]['color.g'];
+                    delete canvasCombos[layer.id()]['color.b'];
+                    delete canvasCombos[layer.id()]['color.a'];
+                } else {
+                    delete canvasCombos[layer.id()][propTitle];
+                }
                 const audioOptions = panel.querySelectorAll(toCssClass(`audioOptions${propTitle}`,'.'));
                 if (audioOptions.length > 0) {
                     audioOptions.forEach((e) => { e.remove(); });
@@ -513,7 +588,7 @@ const Audio = function(tp, record) {
                         convolver.buffer = buffer;
                     },
                     function(e) {
-                        console.log("Error with decoding audio data" + e.err);
+                        console.log("Audio::audioCtx.decodeAudioData", "Error with decoding audio data" + e.err);
                     }
                 );
             };
@@ -563,60 +638,63 @@ const Audio = function(tp, record) {
                 const visualSetting = visualSelect.value;
 
                 if (visualSetting === "sinewave") {
-                    analyser.fftSize = 2048;
-                    const bufferLength = analyser.fftSize;
+                    //analyser.fftSize = 2048;
+                    //const bufferLength = analyser.fftSize;
 
-                    // We can use Float32Array instead of Uint8Array if we want higher precision
-                    // const dataArray = new Float32Array(bufferLength);
-                    const dataArray = new Uint8Array(bufferLength);
+                    //// We can use Float32Array instead of Uint8Array if we want higher precision
+                    //// const dataArray = new Float32Array(bufferLength);
+                    //const dataArray = new Uint8Array(bufferLength);
 
-                    canvasCtx.clearRect(0, 0, WIDTH, HEIGHT);
+                    //canvasCtx.clearRect(0, 0, WIDTH, HEIGHT);
 
-                    const draw = function() {
-                        drawVisual = requestAnimationFrame(draw);
+                    //const draw = function() {
+                        //drawVisual = requestAnimationFrame(draw);
 
-                        analyser.getByteTimeDomainData(dataArray);
+                        //analyser.getByteTimeDomainData(dataArray);
 
-                        canvasCtx.fillStyle = "rgb(200, 200, 200)";
-                        canvasCtx.fillRect(0, 0, WIDTH, HEIGHT);
+                        //canvasCtx.fillStyle = "rgb(200, 200, 200)";
+                        //canvasCtx.fillRect(0, 0, WIDTH, HEIGHT);
 
-                        canvasCtx.lineWidth = 2;
-                        canvasCtx.strokeStyle = "rgb(0, 0, 0)";
+                        //canvasCtx.lineWidth = 2;
+                        //canvasCtx.strokeStyle = "rgb(0, 0, 0)";
 
-                        canvasCtx.beginPath();
+                        //canvasCtx.beginPath();
 
-                        const sliceWidth = (WIDTH * 1.0) / bufferLength;
-                        let x = 0;
+                        //const sliceWidth = (WIDTH * 1.0) / bufferLength;
+                        //let x = 0;
 
-                        for (let i = 0; i < bufferLength; i++) {
-                            let v = dataArray[i] / 128.0;
-                            let y = (v * HEIGHT) / 2;
+                        //for (let i = 0; i < bufferLength; i++) {
+                            //let v = dataArray[i] / 128.0;
+                            //let y = (v * HEIGHT) / 2;
 
-                            if (i === 0) {
-                                canvasCtx.moveTo(x, y);
-                            } else {
-                                canvasCtx.lineTo(x, y);
-                            }
+                            //if (i === 0) {
+                                //canvasCtx.moveTo(x, y);
+                            //} else {
+                                //canvasCtx.lineTo(x, y);
+                            //}
 
-                            x += sliceWidth;
-                        }
+                            //x += sliceWidth;
+                        //}
 
-                        canvasCtx.lineTo(canvas.width, canvas.height / 2);
-                        canvasCtx.stroke();
-                    };
+                        //canvasCtx.lineTo(canvas.width, canvas.height / 2);
+                        //canvasCtx.stroke();
+                    //};
 
-                    draw();
+                    //draw();
                 } else if (visualSetting == "frequencybars") {
                     analyser.fftSize = config.audio.fftBandsAnalysed;
                     const w = config.audio.fftBandsUsed;
                     const h = config.audio.fftHeight;
+                    const verticalFactor = h / 256.0;
                     const bufferLengthAlt = analyser.frequencyBinCount / 2;
 
                     // See comment above for Float32Array()
                     const dataArrayAlt = new Uint8Array(bufferLengthAlt);
 
-                    for (let i = 0; i < canvasCtxs.length; i++) {
-                        canvasCtxs[i].clearRect(0, 0, w, h);
+                    const canvasKeys = Object.keys(canvasCombos);
+
+                    for (let i = 0; i < canvasKeys.length; i++) {
+                        canvasCombos[canvasKeys[i]][1].clearRect(0, 0, w, h);
                     }
 
                     let frameCount = 0;
@@ -625,35 +703,76 @@ const Audio = function(tp, record) {
 
                         analyser.getByteFrequencyData(dataArrayAlt);
 
-                        for (let i = 0; i < canvasCtxs.length; i++) {
-                            canvasCtxs[i].fillStyle = "rgb(0, 0, 0)";
-                            canvasCtxs[i].fillRect(0, 0, w, h);
+                        for (let i = 0; i < canvasKeys.length; i++) {
+                            canvasCombos[canvasKeys[i]][1].fillStyle = "rgb(0, 0, 0)"; // AUDIO COLOR
+                            canvasCombos[canvasKeys[i]][1].fillRect(0, 0, w, h);
+                            const layerID = canvasCombos[canvasKeys[i]][2];
+                            const m = mapping[layerID][canvasKeys[i]];
+                            if (m.sync === 'volume') {
+                                const sx = m.min_freq;
+                                const sw = m.max_freq - m.min_freq;
+                                const sy = h - (m.max_in * verticalFactor);
+                                const sh = (m.max_in - m.min_in) * verticalFactor;
+                                canvasCombos[canvasKeys[i]][1].fillStyle = "rgb(80, 80, 80)"; // AUDIO COLOR
+                                canvasCombos[canvasKeys[i]][1].fillRect(sx, sy, sw, sh);
+                            } else if (m.sync === 'pitch') {
+                                const sx = m.min_freq;
+                                const sw = m.max_freq - m.min_freq;
+                                const sy = 0;
+                                const sh = h;
+                                canvasCombos[canvasKeys[i]][1].fillStyle = "rgb(80, 80, 80)"; // AUDIO COLOR
+                                canvasCombos[canvasKeys[i]][1].fillRect(sx, sy, sw, sh);
+                            }
                         }
 
-                        const barWidth = (w / bufferLengthAlt) * 2.5;
+                        const barWidth = 1;//(w / bufferLengthAlt) * 2.5;
                         let barHeight;
                         let x = 0;
 
                         let max_i = 0;
                         let max_v = 0;
-                        for (let i = 0; i < bufferLengthAlt; i++) {
+                        let min_v = 9999999;
+                        for (let i = 0; i < w; i++) {
                             barHeight = dataArrayAlt[i];
 
                             if (barHeight > max_v) {
                                 max_v = barHeight;
                                 max_i = i;
                             }
-                            for (let i = 0; i < canvasCtxs.length; i++) {
-                                canvasCtxs[i].fillStyle = "rgb(" + (barHeight + 100) + ",50,50)";
-                                canvasCtxs[i].fillRect(
+                            if (barHeight < min_v) {
+                                min_v = barHeight;
+                            }
+                            for (let i = 0; i < canvasKeys.length; i++) {
+                                canvasCombos[canvasKeys[i]][1].fillStyle = "rgb(200,200,200)"; // AUDIO COLOR
+                                canvasCombos[canvasKeys[i]][1].fillRect(
                                     x,
-                                    h - barHeight / 2,
+                                    h - (barHeight * verticalFactor),
                                     barWidth,
-                                    barHeight / 2
+                                    (barHeight * verticalFactor)
                                 );
                             }
 
-                            x += barWidth + 1;
+                            x += barWidth;
+                        }
+                        for (let i = 0; i < canvasKeys.length; i++) {
+                            const layerID = canvasCombos[canvasKeys[i]][2];
+                            const m = mapping[layerID][canvasKeys[i]];
+                            if (m.sync === 'volume') {
+                                const sx = m.min_freq;
+                                const sw = m.max_freq - m.min_freq;
+                                const sy = h - (m.max_in * verticalFactor);
+                                const sh = (m.max_in - m.min_in) * verticalFactor;
+                                canvasCombos[canvasKeys[i]][1].strokeStyle = "rgb(255, 255, 255)"; // AUDIO COLOR
+                                canvasCombos[canvasKeys[i]][1].strokeRect(sx, sy, sw, sh);
+                            } else if (m.sync === 'pitch') {
+                                const m = mapping[layerID][canvasKeys[i]];
+                                const sx = m.min_freq;
+                                const sw = m.max_freq - m.min_freq;
+                                const sy = 0;
+                                const sh = h;
+                                canvasCombos[canvasKeys[i]][1].strokeStyle = "rgb(255, 255, 255)"; // AUDIO COLOR
+                                canvasCombos[canvasKeys[i]][1].strokeRect(sx, sy, sw, sh);
+                            }
                         }
                         const propsToSet = [];
                         getLayers().forEach((layer) => {
@@ -662,7 +781,7 @@ const Audio = function(tp, record) {
                                     const m = mapping[layer.id()][propTitle];
                                     switch (m.sync) {
                                         case 'volume': {
-                                            let a = mapValue(max_v, 0, 255, m.min_out, m.max_out, true);
+                                            let a = mapValue(max_v, m.min_in, m.max_in, m.min_out, m.max_out, true);
                                             m.value = m.value * m.smoothing + (1.0 - m.smoothing) * a;
                                             propsToSet.push({
                                                 layer,
@@ -674,7 +793,7 @@ const Audio = function(tp, record) {
                                             break;
                                         }
                                         case 'pitch': {
-                                            let a = mapValue(max_i, 0, bufferLengthAlt - 1, m.min_out, m.max_out, true);
+                                            let a = mapValue(max_i, m.min_freq, m.max_freq, m.min_out, m.max_out, true);
                                             m.value = m.value * m.smoothing + (1.0 - m.smoothing) * a;
                                             propsToSet.push({
                                                 layer,

bin/web/js/config.js

@@ -86,8 +86,8 @@ const config = {
         ignoreProps: ['transformOrigin', 'fontFamily', 'text', 'mirror_x', 'mirror_y', 'mirror_xy'],
         defaultSmoothing: 0.7,
         fftBandsAnalysed: 256 * 8,
-        fftBandsUsed: 256 * 8 / 2,
-        fftHeight: 256 / 2,
+        fftBandsUsed: 256 / 2,
+        fftHeight: 256 / 4,
         colorSeparateRGBA: true,
     },
     record: {

bin/web/js/main.js

@@ -465,14 +465,14 @@ const duplicateLayer = (originalLayer) => {
                     //delete originalValues[originalKeys[i]];
                     allKeysFound = false;
                 }
-            };
+            }
             if (allKeysFound) {
                 tp.getPanel().removeEventListener("injected", addKeyframes);
             }
             tp.addKeyframes(newLayer, originalKeyframes).then(() => {
                 if (allKeysFound) {
                     resolve();
-                };
+                }
             });
         };
         tp.getPanel().addEventListener("injected", addKeyframes);