add audio layer and audio player

dependencies hashes:
openFrameworks d78075f4bca6be2a2533c6e51a75cc1f18404501
ofxMsdfgen e14da13d02c4dff04fb69d7923469f606924e6c3
ofxGPUFont d482bb7cbdf6b296fa4ab5abcf73fb5ff8c8b239
ofxVariableLab 0b5f9bdebc1e5550621957e73c040c258ec6317b
ofxProfiler a868e34fa1a79189dd4fbdede2938e308535e5e8
theatre 86d3e07f6f2c75fd6e08fca8c97e3617c9e23b18

bin/em/variabletime/web/css/theatre.css

@@ -207,6 +207,7 @@ margin-bottom: 3px;
 .source_Dom_Cont {
     padding: 5px;
     display: flex;
+    flex-wrap: wrap;
     align-items: center;
     justify-content: space-between;
     border-top: 1px dashed #91919177;
@@ -370,14 +371,6 @@ input[type=checkbox]:checked+label {
     -webkit-tap-highlight-color: transparent;
 }
 
-/*input[type=checkbox]+label::after{
-    content: ' OFF';
-}
-
-input[type=checkbox]:checked+label::after {
-    content: ' ON';
-}*/
-
 label svg{
     width: 20px;
     height: 20px;

bin/em/variabletime/web/js/audio.js

@@ -13,8 +13,9 @@ import {
     getFileExtensionFromMimeType,
 } from './utils.js';
 
-window.mapValue = mapValue;
+import {
-
+    AudioPlayer
+} from './audioPlayer.js';
 
 const AudioMappingOptions = function() {
     this.min_freq = 0.0;
@@ -40,102 +41,6 @@ window.playAudioFile = (file) => {
     return audioElement;
 };
 
-const AudioPlayer = function() {
-    const audioElements = [];
-    let updateInterval = false;
-    let updateInterval_ms = 10;
-    this.add = (layer, propTitle, time, file) => {
-        const layerID = typeof layer === 'string' ? layer : layer.id();
-        console.log('AudioPlayer::add',{layerID, propTitle, time, file});
-        const index = audioElements.findIndex((e) => e.layerID === layerID && e.propTitle === propTitle);
-        if (index === -1) {
-            const audioElement = document.createElement('audio');
-            audioElement.classList.add('invisible');
-            audioElement.classList.add('audio_file');
-            audioElement.src = audio.audioSourceCombos[file].audioElement.src;
-            audioElements.push({
-                layerID, propTitle, audioElement, time, file
-            });
-        } else {
-            audioElements[index].src = audio.audioSourceCombos[file].audioElement.src;
-            audioElements[index].time = time;
-        }
-    };
-    this.update = () => {
-        audioElements.forEach((audioElement, i) => {
-            if (tp.isPlaying() && !record.isRecording()) {
-                const diff = tp.sheet.sequence.position - audioElement.time;
-                if (diff >= 0) {
-                    if (audioElement.audioElement.paused) {
-                        audioElement.audioElement.currentTime = diff;
-                        audioElement.audioElement.play();
-                        console.log('play audioElement ', audioElement.file, audioElement.propTitle, i);
-                    }
-                } else if(!audioElement.audioElement.paused) {
-                    audioElement.audioElement.pause();
-                    audioElement.audioElement.currentTime = 0;
-                        console.log('paus audioElement ', audioElement.file, audioElement.propTitle, i);
-                }
-            } else if (!audioElement.audioElement.paused) {
-                audioElement.audioElement.pause();
-                audioElement.audioElement.currentTime = 0;
-                        console.log('pausé audioElement ', audioElement.file, audioElement.propTitle, i);
-            }
-        });
-    };
-    this.audioElements = audioElements;
-    this.init = () => {
-        clearInterval(updateInterval);
-        updateInterval = setInterval(() => {
-            this.update();
-        }, updateInterval_ms);
-    };
-    this.listener = (event) => {
-        console.log('AUDIOPLAYORECEIVED', event);
-        let hot = false;
-        let time = false;
-        if (event.detail === record.possibleStates.RECORDING) {
-            hot = clone(record.getHot());
-            time = tp.sheet.sequence.position;
-            const layerIDs = Object.keys(hot);
-            layerIDs.forEach((layerID) => {
-                const propTitles = Object.keys(hot[layerID]);
-                propTitles.forEach((propTitle) => {
-                    const m = audio.getMapping()[layerID][propTitle];
-                    if (m.addToTimeline) {
-                        if (m.source === 'microphone') {
-                            const waitForMicrophoneListener = (event) => {
-                                if (event.detail.fileIsRead) {
-                                    this.add(layerID, propTitle, time, event.detail.filename);
-                                    window.removeEventListener('microphoneRecorder', waitForMicrophoneListener);
-                                }
-                            };
-                            window.addEventListener('microphoneRecorder', waitForMicrophoneListener);
-                        }
-                    }
-                });
-            });
-        }
-        if (event.detail === record.possibleStates.NOT_RECORDING) {
-            const layerIDs = Object.keys(hot);
-            layerIDs.forEach((layerID) => {
-                const propTitles = Object.keys(hot[layerID]);
-                propTitles.forEach((propTitle) => {
-                    const m = audio.getMapping()[layerID][propTitle];
-                    if (m.addToTimeline) {
-                        if (m.source === 'microphone') {
-                            // we already handled this above
-                        } else {
-                            this.add(layerID, propTitle, time, m.source);
-                        }
-                    }
-                });
-            });
-            window.removeEventListener('record', this.listener);
-        }
-    };
-};
-
 const MicrophoneRecorder = function() {
     let recorder = false;
     let buffy = [];
@@ -456,8 +361,14 @@ const Audio = function(tp, record) {
         audioOptions.style.order = parseInt(container.style.order) + 1;
 
         const updateMappingOptions = () => {
+            if (panel.querySelector(toCssClass(`audioOptions${propTitle}`,'.')) === null) {
+                // we may not have the audioOptions created, if we select a different layer
+                // so let's give up in case there are no audioOptions and
+                // return safely from our journey before anyone gets hurt.
+                return;
+            }
             isSequenced = tp.isSequenced(propTitle, layer);
-            isLetterDelayMapped = typeof mapping[layer.id()][`letterDelays.${propTitle}`] !== 'undefined'; // if the letterDelay is mapped itself, we don't do it
+            isLetterDelayMapped = typeof mapping[layer.id()] === 'object' && typeof mapping[layer.id()][`letterDelays.${propTitle}`] !== 'undefined'; // if the letterDelay is mapped itself, we don't do it
             if (propTitle === 'color') {
                 mappingOptions.min_out = hexaToRgba(panel.querySelector(toCssClass(`audio_min${propTitle}`,'#')).value);
                 const max_cssClass = toCssClass(`audio_max${propTitle}`,'#');
@@ -532,22 +443,25 @@ const Audio = function(tp, record) {
             {
                 const cssClass = toCssClass(`audio_addToTimeline${propTitle}`);
                 const checkboxDom = document.createElement('input');
+                checkboxDom.classList.add(cssClass);
                 checkboxDom.type = 'checkbox';
+                checkboxDom.title = 'add to timeline';
                 checkboxDom.checked = false;
                 mappingOptions.addToTimeline = checkboxDom.checked;
                 checkboxDom.addEventListener('change', (event) => {
                     mappingOptions.addToTimeline = event.currentTarget.checked;
                 });
-                record_Dom_Cont.append(checkboxDom);
+                const checkboxDom_label = document.createElement('label');
+                checkboxDom_label.innerHTML = 'add to timeline';
+                checkboxDom_label.append(checkboxDom);
+                record_Dom_Cont.append(checkboxDom_label);
             }
 
             recordSoloButton.addEventListener('click', () => {
                 if (!record.isRecording()) {
-                    console.log('SHOULD ADD LEAST AUDOPLAYOR');
                     // will be recording
                     window.addEventListener('record', audioPlayer.listener);
                     if (mappingOptions.source === 'microphone') {
-                        console.log('SHOULD ADD LEAST ALSO THE MICROPHONERECORDER');
                         window.addEventListener('record', microphoneRecorder.startListener);
                         window.addEventListener('record', microphoneRecorder.stopListener);
                     } else {
@@ -919,7 +833,6 @@ const Audio = function(tp, record) {
 
         canvasCombos[propTitle] = [fft_imgDom, fft_imgDom.getContext("2d"), layer.id()];
 
-
         tp.getPanel().addEventListener('updateAudioMapping', () => {
             updateMappingOptions();
         });
@@ -1630,6 +1543,5 @@ const Audio = function(tp, record) {
 };
 
 export {
-    Audio,
+    Audio
-    AudioLayer
 }

bin/em/variabletime/web/js/audioLayer.js

@@ -0,0 +1,123 @@
+const AudioLayer = function(tp, audioLayerID, values = false, autoInit = true) {
+    // private
+    let props = {
+        time: tp.core.types.boolean(false),
+        audioFile: tp.core.types.stringLiteral(
+            'no_audio_file',
+            {
+                no_audio_file: 'no audio file :(',
+            },
+        ),
+    };
+
+    // private functions
+    const onValuesChange = (values) => {
+        console.log(values);
+    };
+
+    const updateAudioFiles = () => {
+        console.log('update audio files');
+        return new Promise((resolve) => {
+            moduleFS.readdir(config.fs.idbfsAudioDir).then((files) => {
+                ;
+                console.log(files);
+                if (files.length > 0) {
+                    const audioFiles = {
+                        no_audio_file: 'no audio file :(',
+                    };
+                    files.forEach((file) => {
+                        audioFiles[file] = file;
+                    });
+                    props.audioFile = tp.core.types.stringLiteral(
+                        audioFiles[files[0]],
+                        audioFiles
+                    );
+                    tp.changeObject(this.id(), props);
+                    if (typeof this.theatreObject === 'object') {
+                        tp.studio.transaction(({
+                            set
+                        }) => {
+                            set(this.theatreObject.props.audioFile, files[0]);
+                        });
+                    }
+                }
+                resolve(files);
+            });
+        });
+    };
+
+    const findInjectPanel = () => {
+        console.log('find and inject');
+    };
+
+    const setTime = (start, stop) => {
+        const keyframes = [
+            {
+                path: ['time'],
+                keyframes: [ 
+                    {
+                        position: -1,
+                        value: false,
+                    },
+                    {
+                        position: start,
+                        value: true,
+                    },
+                    {
+                        position: stop,
+                        value: false,
+                    },
+                ],
+            }
+        ];
+        tp.setKeyframes(this, keyframes);
+    };
+
+    const init = () => {
+        return new Promise((resolve) => {
+            updateAudioFiles().then((files) => {
+                this.theatreObject = tp.addObject(audioLayerID, this.props, onValuesChange);
+                tp.changeObject(audioLayerID, this.props);
+                tp.studio.transaction(({
+                    set
+                }) => {
+                    const mergedValues = {...this.theatreObject.value, ...values};
+                    console.log({values, mergedValues}, this.props);
+                    set(this.theatreObject.props, values ? mergedValues : this.theatreObject.value);
+                });
+                // this has to be a timeout, because theatre is not ready otherwise
+
+                setTime(1,2);
+                resolve();
+            });
+        });
+    };
+
+    const remove = () => {
+    };
+
+    // public
+    this.props = props;
+
+    this.findInjectPanel = findInjectPanel;
+
+    this.setTime = setTime;
+
+    this.init = init;
+
+    this.id = () => {
+        return audioLayerID;
+    };
+
+    // well, I know this is a shitty name. My apologies. However, ..
+    this.prepareForDepartureFromThisBeautifulExperience = remove;
+
+    // action
+    if (autoInit) {
+        this.init();
+    }
+};
+
+export {
+    AudioLayer,
+}

bin/em/variabletime/web/js/audioPlayer.js

@@ -0,0 +1,108 @@
+import {
+    clone,
+} from './utils.js';
+
+const AudioPlayer = function() {
+    const audioElements = [];
+    let updateInterval = false;
+    let updateInterval_ms = 10;
+    this.add = (layer, propTitle, time, file) => {
+        const layerID = typeof layer === 'string' ? layer : layer.id();
+        propTitle = Array.isArray(propTitle) ? propTitle.join('.') : propTitle;
+        console.log('AudioPlayer::add',{layerID, propTitle, time, file});
+        const index = audioElements.findIndex((e) => e.layerID === layerID && e.propTitle === propTitle);
+        if (index === -1) {
+            const audioDomElement = document.createElement('audio');
+            audioDomElement.classList.add('invisible');
+            audioDomElement.classList.add('audio_file');
+            audioDomElement.src = audio.audioSourceCombos[file].audioElement.src;
+            audioElements.push({
+                layerID, propTitle, audioDomElement, time, file
+            });
+        } else {
+            audioElements[index].src = audio.audioSourceCombos[file].audioDomElement.src;
+            audioElements[index].time = time;
+        }
+    };
+    this.update = () => {
+        audioElements.forEach((audioElement, i) => {
+            if (tp.isPlaying() && !record.isRecording()) {
+                const diff = tp.sheet.sequence.position - audioElement.time;
+                if (diff >= 0) {
+                    if (audioElement.audioDomElement.paused) {
+                        audioElement.audioDomElement.currentTime = diff;
+                        audioElement.audioDomElement.play();
+                        console.log('play audioElement ', audioElement.file, audioElement.propTitle, i);
+                    }
+                } else if(!audioElement.audioDomElement.paused) {
+                    audioElement.audioDomElement.pause();
+                    audioElement.audioDomElement.currentTime = 0;
+                        console.log('paus audioElement ', audioElement.file, audioElement.propTitle, i);
+                }
+            } else if (!audioElement.audioDomElement.paused) {
+                audioElement.audioDomElement.pause();
+                audioElement.audioDomElement.currentTime = 0;
+                        console.log('pausé audioElement ', audioElement.file, audioElement.propTitle, i);
+            }
+        });
+    };
+    this.audioElements = audioElements;
+    this.init = () => {
+        clearInterval(updateInterval);
+        updateInterval = setInterval(() => {
+            this.update();
+        }, updateInterval_ms);
+    };
+    this.listener = (event) => {
+        let hot = false;
+        let time = false;
+        if (event.detail === record.possibleStates.RECORDING) {
+            hot = clone(record.getHot());
+            time = tp.sheet.sequence.position;
+            const layerIDs = Object.keys(hot);
+            layerIDs.forEach((layerID) => {
+                const propTitles = Object.keys(hot[layerID]);
+                propTitles.forEach((propTitle) => {
+                    const m = audio.getMapping()[layerID][propTitle];
+                    if (m.addToTimeline) {
+                        if (m.source === 'microphone') {
+                            const waitForMicrophoneListener = (event) => {
+                                if (event.detail.fileIsRead) {
+                                    this.add(layerID, propTitle, time, event.detail.filename);
+                                    addAudioLayer().then((audioLayer) => {
+                                        m.layer = audioLayer;
+                                    });
+                                    window.removeEventListener('microphoneRecorder', waitForMicrophoneListener);
+                                }
+                            };
+                            window.addEventListener('microphoneRecorder', waitForMicrophoneListener);
+                        }
+                    }
+                });
+            });
+        }
+        if (event.detail === record.possibleStates.NOT_RECORDING) {
+            const layerIDs = Object.keys(hot);
+            layerIDs.forEach((layerID) => {
+                const propTitles = Object.keys(hot[layerID]);
+                propTitles.forEach((propTitle) => {
+                    const m = audio.getMapping()[layerID][propTitle];
+                    if (m.addToTimeline) {
+                        if (m.source === 'microphone') {
+                        } else {
+                            this.add(layerID, propTitle, time, m.source);
+                            addAudioLayer().then((audioLayer) => {
+                                m.layer = audioLayer;
+                            });
+                        }
+                    }
+                });
+            });
+            window.removeEventListener('record', this.listener);
+        }
+    };
+};
+
+export {
+    AudioPlayer
+};

bin/em/variabletime/web/js/main.js

@@ -34,6 +34,9 @@ import {
     Record
 } from './record.js';
 
+import {
+    AudioLayer
+} from './audioLayer.js';
 //import {
     //MidiController
 //} from './midiController.js';
@@ -65,6 +68,7 @@ const interactor = new Interactor();
 const moduleFS = new ModuleFS();
 const record = new Record(tp);
 const audio = new Audio(tp, record);
+const audioLayers = [];
 
 // globally reachables
 // classes
@@ -250,7 +254,7 @@ window.onload = () => {
             }
             tp.studio.onSelectionChange((newSelection) => {
                 if (newSelection.length > 0) {
-                    [getArtboard(), getLayers(), getAudioLayer()].flat().forEach((e) => {
+                    [getArtboard(), getLayers(), getAudioLayers()].flat().forEach((e) => {
                         if (e.id() === newSelection[0].address.objectKey) {
                             if (e.id().indexOf('layer-') === 0) {
                                 e.findInjectPanel();
@@ -455,6 +459,10 @@ window.getLayers = () => {
     return layers;
 };
 
+window.getAudioLayers = () => {
+    return audioLayers;
+};
+
 window.getLayer = (layerID = tp.studio.selection[0].address.objectKey) => {
     if (layerID === 'artboard') {
         return artboard;
@@ -570,6 +578,51 @@ const deleteLayer = (id, saveProject = true) => {
     }
 };
 
+const addAudioLayer = (values = false) => {
+    return new Promise((resolve) => {
+        const audioLayerID = (() => {
+            let index = 0;
+            for (let i = 0; i < audioLayers.length; i++) {
+                if (audioLayers.findIndex((audioLayer) => audioLayer.id() !== `audio-${index}`) < 0) {
+                    break;
+                }
+                index++;
+            }
+            return `audio-${index}`;
+        })();
+        console.log('adding', audioLayerID);
+        const audioLayer = new AudioLayer(tp, audioLayerID, values, false);
+        layersById[audioLayerID] = audioLayer;
+        audioLayers.push(audioLayer);
+        audioLayer.init().then(() => {
+            resolve(audioLayer);
+        });
+    });
+};
+
+const addExistingAudioLayer = (audioLayerID, values) => {
+    return new Promise((resolve) => {
+        const audioLayer = new AudioLayer(tp, audioLayerID, values, false);
+        audioLayers.push(audioLayer);
+        layersById[audioLayerID] = audioLayer;
+        audioLayer.init().then(() => {
+            resolve();
+        });
+    });
+};
+
+const deleteAudioLayer = (audioLayerID) => {
+    tp.removeObject(audioLayerID);
+    let index = 0;
+    for (let i = 0; i < audioLayers.length; i++) {
+        if (audioLayers[i].id() === audioLayerID) {
+            index = i;
+        }
+    }
+    audioLayers.splice(index, 1);
+    //delete audioLayersById[id];
+};
+
 // TODO: better function names
 // because, come on. it may be funny for a second
 // but tolerance for fun is low when you're grumpy
@@ -594,6 +647,9 @@ window.duplicateLayer = (layer) => {
 window.addLayer = addLayer;
 window.addExistingLayer = addExistingLayer;
 window.deleteLayer = deleteLayer;
+window.addAudioLayer = addAudioLayer;
+window.addExistingAudioLayer = addExistingAudioLayer;
+window.deleteAudioLayer = deleteAudioLayer;
 window.renderFrames = exporter.renderFrames;
 
 const layer_panel = document.querySelector('#layer_panel');

bin/em/variabletime/web/js/moduleFS.js

@@ -38,6 +38,23 @@ const ModuleFS = function() {
         });
     };
 
+    this.readdir = (path) => {
+        return new Promise((resolve) => {
+            this.syncfs(MODE_READ_FROM_PERSISTENT)
+                .then(() => {
+                    const analyzed = FS.analyzePath(path);
+                    if (analyzed.exists && FS.isDir(analyzed.object.mode)) {
+                        const content = FS.readdir(path)
+                                            .filter((e) => ['.','..'].indexOf(e) < 0);
+                        resolve(content);
+                    } else {
+                        console.log('ModuleFS::readdir', `${path} is not a directory or does not exist`);
+                        resolve([]);
+                    }
+                });
+        });
+    };
+
     // check utils::uploadFile() for details of file
     this.save = (file) => {
         return new Promise((resolve) => {

bin/em/variabletime/web/js/theatre-play.js

@@ -228,11 +228,11 @@ const TheatrePlay = function(autoInit = false) {
             let waitify = false;
             keyframes.forEach((k) => {
                 const propTitle = k.path.join('.');
-                if (isSequenced(propTitle)) {
+                if (isSequenced(propTitle, layer)) {
                     waitify = true;
                     promises.push(() => {
                         return new Promise((subResolve) => {
-                            setSequenced(propTitle, false)
+                            setSequenced(propTitle, false, layer)
                                 .then(() => {
                                     subResolve();
                                 });
@@ -651,6 +651,7 @@ const TheatrePlay = function(autoInit = false) {
             project = projectJson === false ? core.getProject(projectId) : core.getProject(projectId, {
                 state: projectJson
             });
+            console.log({project, projectJson});
             window.setLoadingTask('setting up animation', 10);
             project.ready.then(() => {
                 this.sheet = project.sheet('mainSheet');
@@ -724,12 +725,13 @@ const TheatrePlay = function(autoInit = false) {
                         window.project_fontsHashMap = project.fontsHashMap;
                         layerPromises.push(window.addExistingLayer(layerId, objects[layerId]));
                     });
+                console.log(clone(objects));
                 Object.keys(objects)
                     .filter((e) => e.indexOf('audio-') === 0)
                     .forEach((layerId) => {
                         window.setLoadingTask(`setting up the sounds of ${layerId} to come`, 90);
                         window.project_fontsHashMap = project.fontsHashMap;
-                        layerPromises.push(window.addExistingLayer(layerId, objects[layerId]));
+                        layerPromises.push(window.addExistingAudioLayer(layerId, objects[layerId]));
                     });
                 Promise.all(layerPromises).then(() => {
                     window.layerOrder.set(project.layerOrder);