fix playing multiple recordings with audio and more

dependencies hashes:
openFrameworks d78075f4bca6be2a2533c6e51a75cc1f18404501
ofxMsdfgen e14da13d02c4dff04fb69d7923469f606924e6c3
ofxGPUFont d482bb7cbdf6b296fa4ab5abcf73fb5ff8c8b239
ofxVariableLab 0b5f9bdebc1e5550621957e73c040c258ec6317b
ofxProfiler a868e34fa1a79189dd4fbdede2938e308535e5e8
theatre 78a67ee6650d846fe5cc4770770b1511328033e6

.gitignore

@@ -2,8 +2,22 @@
 bin/*
 !bin/data/
 !bin/data/*
-!bin/web/
-!bin/web/*
+# ugly recursion https://stackoverflow.com/questions/3667986/git-ignore-exception-not-working-as-desired
+!bin/em/
+bin/em/*
+!bin/em/*/
+bin/em/*/*
+!bin/em/*/web/
+bin/em/*/web/*
+!bin/em/*/web/js/
+!bin/em/*/web/css/
+!bin/em/*/web/assets/
+bin/em/*/web/js/*
+bin/em/*/web/css/*
+bin/em/*/web/assets/*
+!bin/em/*/web/js/*.*
+!bin/em/*/web/css/*.*
+!bin/em/*/web/assets/*.*
 obj/
 bin/data/ofxMsdfgen
 bin/data/ofxGPUFont
@@ -39,3 +53,6 @@ bin/web/fonts/*
 # DIST
 # these are related to distributing variable time
 upload*
+
+# GENERATED
+.browserpid

bin/em/variabletime/web/css/theatre.css

@@ -207,6 +207,7 @@ margin-bottom: 3px;
 .source_Dom_Cont {
     padding: 5px;
     display: flex;
+    flex-wrap: wrap;
     align-items: center;
     justify-content: space-between;
     border-top: 1px dashed #91919177;
@@ -370,14 +371,6 @@ input[type=checkbox]:checked+label {
     -webkit-tap-highlight-color: transparent;
 }
 
-/*input[type=checkbox]+label::after{
-    content: ' OFF';
-}
-
-input[type=checkbox]:checked+label::after {
-    content: ' ON';
-}*/
-
 label svg{
     width: 20px;
     height: 20px;
@@ -734,3 +727,8 @@ li.layerMover div.selected svg circle {
 .color_preview{
     border: none !important;
 }
+
+input:disabled {
+    background: darkgrey;
+    color: lightgrey;
+}

bin/em/variabletime/web/js/audio.js

@@ -9,10 +9,13 @@ import {
     clone,
     rgbaToHexa,
     hexaToRgba,
+    getTimestamp,
+    getFileExtensionFromMimeType,
 } from './utils.js';
 
-window.mapValue = mapValue;
-
+import {
+    AudioPlayer
+} from './audioPlayer.js';
 
 const AudioMappingOptions = function() {
     this.min_freq = 0.0;
@@ -26,10 +29,102 @@ const AudioMappingOptions = function() {
     this.source = 'microphone';
     this.value = 0.0;
     this.muted = true;
+    this.addToTimeline = false;
+};
+
+window.playAudioFile = (file) => {
+    const audioElement = document.createElement('audio');
+    audioElement.classList.add('invisible');
+    audioElement.classList.add('audio_file');
+    audioElement.src = src;
+    document.querySelector('body').append(audioElement);
+    return audioElement;
+};
+
+const MicrophoneRecorder = function() {
+    let recorder = false;
+    let buffy = [];
+    let filenameWithoutExtension;
+    let fileExtension;
+
+    this.getLastFilename = () => {
+        return `${filenameWithoutExtension}.${fileExtension}`;
+    };
+
+    this.init = (stream) => {
+        recorder = new MediaRecorder(stream);
+        recorder.addEventListener('dataavailable', (event) => {
+            buffy.push(event.data);
+        });
+    };
+
+    this.start = (name = "") => {
+        if (name === "") {
+            filenameWithoutExtension = getTimestamp();
+        } else {
+            filenameWithoutExtension = name;
+        }
+        buffy = [];
+        recorder.start();
+    };
+
+    this.stop = () => {
+        return new Promise((resolve) => {
+            //save audio type to pass to set the Blob type
+            let mimeType = recorder.mimeType;
+
+            //listen to the stop event in order to create & return a single Blob object
+            recorder.addEventListener("stop", () => {
+                //create a single blob object, as we might have gathered a few Blob objects that needs to be joined as one
+                let blob = new Blob(buffy, { type: mimeType });
+                var arrayBuffer;
+                var fileReader = new FileReader();
+                fileReader.onload = function(event) {
+                    arrayBuffer = event.target.result;
+                    fileExtension = getFileExtensionFromMimeType(mimeType);
+                    if (fileExtension !== false) {
+                        const file = {
+                            name: `${filenameWithoutExtension}.${fileExtension}`,
+                            type: 'audio',
+                            arrayBuffer
+                        };
+                        moduleFS.save(file);
+                        window.dispatchEvent(new CustomEvent('microphoneRecorder', {detail: {fileIsRead: false, filename: file.name}}));
+                        resolve(file.name);
+                    } else {
+                        resolve(false);
+                    }
+                };
+                fileReader.readAsArrayBuffer(blob);
+
+                //resolve promise with the single audio blob representing the recorded audio
+            });
+
+            recorder.stop();
+        });
+    };
+
+    this.startListener = (event) => {
+        if (event.detail === record.possibleStates.RECORDING) {
+            this.start();
+            window.removeEventListener('record', this.startListener);
+        }
+    };
+
+    this.stopListener = (event) => {
+        if (event.detail === record.possibleStates.STOPPING_RECORDING) {
+            this.stop().then((filename) => {
+                // be happy
+            });
+            window.removeEventListener('record', this.stopListener);
+        }
+    };
 };
 
 const Audio = function(tp, record) {
 
+    const audioPlayer = new AudioPlayer(); 
+    this.audioPlayer = audioPlayer; // DEBUG TODO: remove me
     const audioDom = document.querySelector('.audioWrapper');
     let audioCtx = false;
     const heading = audioDom.querySelector("h1");
@@ -49,6 +144,8 @@ const Audio = function(tp, record) {
     //});
     let started = false;
     let allowedMicrophone = true; // TODO: actually use this
+    const microphoneRecorder = new MicrophoneRecorder();
+    this.microphoneRecorder = microphoneRecorder; // DEBUG TODO: remove me
 
     let mapping = {};
     let savedMapping = {};
@@ -167,6 +264,11 @@ const Audio = function(tp, record) {
         }
     };
 
+    const updateAudioMappingEvent = (type, layerID, propTitle, options) => {
+        const e = new CustomEvent('updateAudioMapping', {type, layerID, propTitle, options});
+        tp.getPanel().dispatchEvent(e);
+    };
+
     // potentially recursive
     const addAudioMapping = (layer, propTitle, options = false) => {
         if (!options) {
@@ -178,6 +280,7 @@ const Audio = function(tp, record) {
                     const subPropTitle = `${propTitle}.${subPropKey}`;
                     isGood = addAudioMapping(layer, subPropTitle, o[subPropKey]) ? isGood : false; 
                 });
+                updateAudioMappingEvent('addAudioMapping', layer.id(), propTitle, options);
                 return isGood;
             }
         }
@@ -186,6 +289,7 @@ const Audio = function(tp, record) {
         }
         if (!mapping[layer.id()].hasOwnProperty(propTitle)) {
             mapping[layer.id()][propTitle] = options;
+            updateAudioMappingEvent('addAudioMapping', layer.id(), propTitle, options);
             return true;
         } else {
             // already there
@@ -194,6 +298,7 @@ const Audio = function(tp, record) {
                 fixColor(mapping[layer.id()][propTitle].max_out);
                 fixColor(mapping[layer.id()][propTitle].value);
             }
+            updateAudioMappingEvent('addAudioMapping', layer.id(), propTitle, options);
             return false;
         }
     };
@@ -206,6 +311,7 @@ const Audio = function(tp, record) {
                 });
                 delete mapping[layerID];
             });
+            updateAudioMappingEvent('removeAudioMapping', false, false, false);
             return true;
         }
         if (!mapping.hasOwnProperty(layer.id())) {
@@ -220,6 +326,7 @@ const Audio = function(tp, record) {
         if (Object.keys(mapping[layer.id()]).length === 0) {
             delete mapping[layer.id()];
         }
+        updateAudioMappingEvent('removeAudioMapping', layer.id(), propTitle, false);
         return true;
     }
 
@@ -229,10 +336,10 @@ const Audio = function(tp, record) {
             config
             .layer.letterDelayProps
             .indexOf(propTitle.split('.')[0]) >= 0
-                && mapping[layer.id()][`letterDelays.${propTitle}`] === 'undefined' // if the letterDelay is mapped itself, we don't do it
                 //&& propTitle.indexOf('color') < 0
                 //&& !tp.isSequenced(propTitle)
         ;
+        let isLetterDelayMapped = typeof mapping[layer.id()][`letterDelays.${propTitle}`] === 'undefined'; // if the letterDelay is mapped itself, we don't do it
         let isSequenced = tp.isSequenced(propTitle, layer);
         const panel = tp.getPanel();
         if (!areMutationsObserved) {
@@ -250,6 +357,14 @@ const Audio = function(tp, record) {
         audioOptions.style.order = parseInt(container.style.order) + 1;
 
         const updateMappingOptions = () => {
+            if (panel.querySelector(toCssClass(`audioOptions${propTitle}`,'.')) === null) {
+                // we may not have the audioOptions created, if we select a different layer
+                // so let's give up in case there are no audioOptions and
+                // return safely from our journey before anyone gets hurt.
+                return;
+            }
+            isSequenced = tp.isSequenced(propTitle, layer);
+            isLetterDelayMapped = typeof mapping[layer.id()] === 'object' && typeof mapping[layer.id()][`letterDelays.${propTitle}`] !== 'undefined'; // if the letterDelay is mapped itself, we don't do it
             if (propTitle === 'color') {
                 mappingOptions.min_out = hexaToRgba(panel.querySelector(toCssClass(`audio_min${propTitle}`,'#')).value);
                 const max_cssClass = toCssClass(`audio_max${propTitle}`,'#');
@@ -267,14 +382,23 @@ const Audio = function(tp, record) {
             mappingOptions.smoothing = parseFloat(s);
             if (hasLetterDelay) {
                 const ld = panel.querySelector(toCssClass(`audio_letterDelay${propTitle}`, '#'));
+                if (isLetterDelayMapped) {
+                    ld.setAttribute('disabled', 'disabled');
+                    ld.setAttribute('title', 'disabled, as the letterDelay is audioreactive (scroll down)');
+                } else {
+                    ld.removeAttribute('title');
+                    ld.removeAttribute('disabled');
                     mappingOptions.letterDelay = typeof ld.value === 'number' ? ld.value : parseInt(ld.value);
                     if (isSequenced) {
                         const prop = getNestedProperty(layer.theatreObject.props.letterDelays, propTitle.split('.'));
-                    tp.studio.transaction(({set}) => {
+                        tp.studio.transaction(({
+                            set
+                        }) => {
                             set(prop, mappingOptions.letterDelay);
                         });
                     }
                 }
+            }
             mappingOptions.source = panel.querySelector(toCssClass(`audio_source${propTitle}`,'#')).value;
             mappingOptions.muted = panel.querySelector(toCssClass(`audio_mute${propTitle}`,'#')).checked;
 
@@ -312,7 +436,34 @@ const Audio = function(tp, record) {
             record_Dom_Cont.append(recordSoloButton);
             record_Dom_Cont.append(recordAllButton);
 
+            {
+                const cssClass = toCssClass(`audio_addToTimeline${propTitle}`);
+                const checkboxDom = document.createElement('input');
+                checkboxDom.classList.add(cssClass);
+                checkboxDom.type = 'checkbox';
+                checkboxDom.title = 'add to timeline';
+                checkboxDom.checked = false;
+                mappingOptions.addToTimeline = checkboxDom.checked;
+                checkboxDom.addEventListener('change', (event) => {
+                    mappingOptions.addToTimeline = event.currentTarget.checked;
+                });
+                const checkboxDom_label = document.createElement('label');
+                checkboxDom_label.innerHTML = 'add to timeline';
+                checkboxDom_label.append(checkboxDom);
+                record_Dom_Cont.append(checkboxDom_label);
+            }
+
             recordSoloButton.addEventListener('click', () => {
+                if (!record.isRecording()) {
+                    // will be recording
+                    window.addEventListener('record', audioPlayer.listener);
+                    if (mappingOptions.source === 'microphone') {
+                        window.addEventListener('record', microphoneRecorder.startListener);
+                        window.addEventListener('record', microphoneRecorder.stopListener);
+                    } else {
+                        audioSourceCombos[mappingOptions.source].audioElement.currentTime = 0;
+                    }
+                }
                 record.toggleRecording([[layer.id()].concat(propTitle.split('.'))]);
             });
             recordAllButton.addEventListener('click', () => {
@@ -677,6 +828,10 @@ const Audio = function(tp, record) {
         container.after(audioOptions);
 
         canvasCombos[propTitle] = [fft_imgDom, fft_imgDom.getContext("2d"), layer.id()];
+
+        tp.getPanel().addEventListener('updateAudioMapping', () => {
+            updateMappingOptions();
+        });
         updateMappingOptions();
         return audioOptions;
     };
@@ -832,23 +987,7 @@ const Audio = function(tp, record) {
                 audioElement.classList.add(toCssClass(`audio_file${file}`));
                 document.querySelector('body').append(audioElement);
 
-                const arr = FS.readFile(`${config.fs.idbfsAudioDir}/${file}`);
-                let type = 'audio/wav';
-                const filesplit = file.split('.');
-                const extension = filesplit[filesplit.length - 1];
-                if (extension === 'wav') {
-                    type = 'audio/wav';
-                } else if (extension === 'mp3') {
-                    type = 'audio/mpeg';
-                } else if (extension === 'ogg') {
-                    type = 'audio/ogg';
-                }
-
-                const src = URL.createObjectURL(
-                    new Blob([arr], {
-                        type
-                    })
-                );
+                const src = moduleFS.objectUrl(`${config.fs.idbfsAudioDir}/${file}`);
 
                 audioElement.src = src;
                 audioElement.loop = true;
@@ -876,6 +1015,14 @@ const Audio = function(tp, record) {
             }
         });
     };
+    window.addEventListener('microphoneRecorder', (event) => {
+        if (!event.detail.fileIsRead) {
+            readAudioFiles();
+            const newDetails = event.detail;
+            newDetails.fileIsRead = true;
+            window.dispatchEvent(new CustomEvent('microphoneRecorder', {detail: newDetails}));
+        }
+    });
 
 
     const init = () => {
@@ -891,6 +1038,7 @@ const Audio = function(tp, record) {
                     return;
                 }
             }
+            audioPlayer.init();
             heading.textContent = "Voice-change-O-matic";
             //document.body.removeEventListener("click", init);
 
@@ -998,6 +1146,8 @@ const Audio = function(tp, record) {
                         audioSourceCombos['microphone'].source = source;
                         audioSourceCombos['microphone'].gain = gain;
 
+                        microphoneRecorder.init(stream);
+
                         visualize();
                     })
                     .catch(function(err) {

bin/em/variabletime/web/js/audioLayer.js

@@ -0,0 +1,151 @@
+import {
+    sanitizeTheatreKey
+} from './utils.js';
+
+const AudioLayer = function(tp, audioLayerID, values = false, autoInit = true) {
+    // private
+    let props = { };
+    if (typeof values === 'string') {
+        const filename = sanitizeTheatreKey(values);
+        values = {};
+        values[filename] = false;
+    }
+    if (typeof values === 'object') {
+        props[Object.keys(values)[0]] = tp.core.types.boolean(false);
+    }
+
+    // private functions
+    //
+    const onValuesChange = (values) => {
+        console.log(values);
+    };
+
+    const findInjectPanel = () => {
+        console.log('find and inject');
+    };
+
+    const setTime = (filename, start, stop) => {
+        filename = sanitizeTheatreKey(filename);
+        return new Promise((resolve) => {
+            const keyframes = [{
+                path: [filename],
+                keyframes: [{
+                        position: -1,
+                        value: false,
+                        type: 'hold',
+                    },
+                    {
+                        position: start,
+                        value: true,
+                        type: 'hold',
+                    },
+                    {
+                        position: stop,
+                        value: false,
+                        type: 'hold',
+                    },
+                ],
+            }];
+            if (tp.getKeyframes(this, [filename]).length > 0) {
+                tp.studio.transaction(({
+                    __experimental_deleteKeyframes
+                }) => {
+                    __experimental_deleteKeyframes(this.theatreObject.props[filename]);
+                });
+            }
+            tp.addKeyframes(this, keyframes).then(() => {
+                resolve();
+            });
+        });
+    };
+
+    const init = () => {
+        return new Promise((resolve) => {
+            this.theatreObject = tp.addObject(audioLayerID, this.props, onValuesChange);
+            tp.studio.transaction(({
+                set
+            }) => {
+                const mergedValues = {
+                    ...this.theatreObject.value,
+                    ...values
+                };
+                set(this.theatreObject.props, values ? mergedValues : this.theatreObject.value);
+            });
+            resolve();
+        });
+    };
+
+    const remove = () => {
+    };
+
+    // public
+    this.props = props;
+
+    this.findInjectPanel = findInjectPanel;
+
+    this.setFile = (filename) => {
+        filename = sanitizeTheatreKey(filename);
+        const values = {};
+        values[filename] = false;
+        props = {};
+        props[filename] = tp.core.types.boolean(values[filename]);
+        this.props = props;
+        tp.changeObject(audioLayerID, this.props);
+        tp.studio.transaction(({
+            set
+        }) => {
+            set(this.theatreObject.props, values);
+        });
+    };
+    this.addFile = (filename) => {
+        return new Promise((resolve) => {
+            filename = sanitizeTheatreKey(filename);
+            const values = this.theatreObject.value;
+            values[filename] = false;
+            props[filename] = tp.core.types.boolean(values[filename]);
+            this.props = props;
+            tp.changeObject(audioLayerID, {
+                dummy: true
+            });
+            setTimeout(() => {
+                tp.changeObject(audioLayerID, this.props);
+                resolve();
+            }, 100);
+        });
+    };
+    this.removeFile = (filename) => {
+        return new Promise((resolve) => {
+            filename = sanitizeTheatreKey(filename);
+            const values = this.theatreObject.value;
+            delete values[filename];
+            delete props[filename];
+            this.props = props;
+            tp.changeObject(audioLayerID, {
+                dummy: true
+            });
+            setTimeout(() => {
+                tp.changeObject(audioLayerID, this.props);
+                resolve();
+            }, 100);
+        });
+    };
+    this.setTime = setTime;
+
+    this.init = init;
+
+    this.id = () => {
+        return audioLayerID;
+    };
+
+    // well, I know this is a shitty name. My apologies. However, ..
+    this.prepareForDepartureFromThisBeautifulExperience = remove;
+
+    // action
+    if (autoInit) {
+        this.init();
+    }
+};
+
+export {
+    AudioLayer,
+}

bin/em/variabletime/web/js/audioPlayer.js

@@ -0,0 +1,147 @@
+import {
+    clone,
+    sanitizeTheatreKey,
+} from './utils.js';
+
+const AudioPlayer = function() {
+    const audioElements = [];
+    let updateInterval = false;
+    let updateInterval_ms = 10;
+
+    const getAudioID = (filename) => {
+        let audioID = sanitizeTheatreKey(filename);
+        const duplicate = audioElements.findIndex((e) => e.audioID === audioID) !== -1;
+        if (duplicate) {
+            let index = 0;
+            let unique = false;
+            while (!unique) { // uuuh, while loops..
+                const newAudioID = `${audioID}_${index}`;
+                if (audioElements.findIndex((e) => e.audioID === newAudioID) === -1) {
+                    audioID = newAudioID;
+                    unique = true;
+                } else {
+                    index++;
+                }
+            }
+            return audioID
+        }
+        return audioID;
+    };
+
+    this.addMany = (manyAudioElements) => {
+        manyAudioElements.forEach((e) => {
+            this.add(e.audioID, e.layerID, e.propTitle, e.file);
+        });
+    };
+
+    this.add = (audioID, layer, propTitle, file, startTime) => {
+        const layerID = typeof layer === 'string' ? layer : layer.id();
+        propTitle = Array.isArray(propTitle) ? propTitle.join('.') : propTitle;
+
+        if (!audioID) {
+            audioID = getAudioID(file);
+        }
+        const audioDomElement = new Audio(moduleFS.objectUrl(`${config.fs.idbfsAudioDir}/${file}`));
+        audioDomElement.loop = true;
+        audioDomElement.load();
+        //const audioDomElement = document.createElement('audio');
+        //audioDomElement.classList.add('invisible');
+        //audioDomElement.classList.add('audio_file');
+        //audioDomElement.classList.add('audioPlayer');
+        //audioDomElement.src = moduleFS.objectUrl(`${config.fs.idbfsAudioDir}/${file}`);
+        audioDomElement.loop = true;
+        audioElements.push({
+            audioID,
+            layerID,
+            propTitle,
+            audioDomElement,
+            file,
+            startTime
+        });
+    };
+    this.update = () => {
+        audioElements.forEach((audioElement, i) => {
+            if (tp.isPlaying() && !record.isRecording()) {
+                const shouldBePlaying = getAudioLayer().theatreObject.value[audioElement.audioID];
+                if (shouldBePlaying && audioElement.audioDomElement.paused) {
+                    // sequence position is always greater than startTime
+                    // this is true, as it is written
+                    const diff = tp.sheet.sequence.position - audioElement.startTime;
+                        audioElement.audioDomElement.currentTime = diff;
+                        audioElement.audioDomElement.play();
+                } else if (!shouldBePlaying && !audioElement.audioDomElement.paused) {
+                    audioElement.audioDomElement.pause();
+                    audioElement.audioDomElement.currentTime = 0;
+                }
+            } else if (!audioElement.audioDomElement.paused) {
+                audioElement.audioDomElement.pause();
+                audioElement.audioDomElement.currentTime = 0;
+            }
+        });
+    };
+    this.audioElements = audioElements;
+    this.init = () => {
+        clearInterval(updateInterval);
+        updateInterval = setInterval(() => {
+            this.update();
+        }, updateInterval_ms);
+    };
+    let hot = false;
+    let startTime = false;
+    this.listener = (event) => {
+        if (event.detail === record.possibleStates.RECORDING) {
+            hot = clone(record.getHot());
+            startTime = tp.sheet.sequence.position;
+            const layerIDs = Object.keys(hot);
+            layerIDs.forEach((layerID) => {
+                const propTitles = Object.keys(hot[layerID]);
+                propTitles.forEach((propTitle) => {
+                    const m = audio.getMapping()[layerID][propTitle];
+                    if (m.addToTimeline) {
+                        if (m.source === 'microphone') {
+                            const waitForMicrophoneListener = (event) => {
+                                if (event.detail.fileIsRead) {
+                                    const filename = event.detail.filename;
+                                    const audioID = getAudioID(filename);
+                                    this.add(audioID, layerID, propTitle, filename, startTime);
+                                    addAudioLayerTrack(audioID, startTime, tp.sheet.sequence.position);
+                                    window.removeEventListener(
+                                        'microphoneRecorder',
+                                        waitForMicrophoneListener);
+                                }
+                            };
+                            window.addEventListener(
+                                'microphoneRecorder',
+                                waitForMicrophoneListener);
+                        }
+                    }
+                });
+            });
+        }
+        if (event.detail === record.possibleStates.NOT_RECORDING) {
+            const layerIDs = Object.keys(hot);
+            layerIDs.forEach((layerID) => {
+                const propTitles = Object.keys(hot[layerID]);
+                propTitles.forEach((propTitle) => {
+                    const m = audio.getSavedMapping()[layerID][propTitle];
+                    if (m.addToTimeline) {
+                        if (m.source === 'microphone') {
+                            // handled above, as we need to wait for the microphone recording to be written and read
+                            // and we rather start waiting, when it has definitely not happened yet
+                        } else {
+                            const filename = m.source;
+                            const audioID = getAudioID(filename);
+                            this.add(audioID, layerID, propTitle, filename, startTime);
+                            addAudioLayerTrack(audioID, startTime, tp.sheet.sequence.position);
+                        }
+                    }
+                });
+            });
+            window.removeEventListener('record', this.listener);
+        }
+    };
+};
+
+export {
+    AudioPlayer
+};

bin/em/variabletime/web/js/config.js

@@ -99,7 +99,7 @@ const config = {
             layer: ['transformOrigin', 'fontFamily', 'text', 'mirror_x', 'mirror_y', 'mirror_xy', 'height'],
         },
         maxFilenameLength: 24,
-        defaultSmoothing: 0.7,
+        defaultSmoothing: 0.5,
         analyser: {
             fftSize: 256 * 8,
             minDecibels: -90,

bin/em/variabletime/web/js/main.js

@@ -34,6 +34,9 @@ import {
     Record
 } from './record.js';
 
+import {
+    AudioLayer
+} from './audioLayer.js';
 //import {
     //MidiController
 //} from './midiController.js';
@@ -52,31 +55,35 @@ import {
     Config
 } from './config.js';
 
-window.uploadFile = uploadFile;
-window.downloadFile = downloadFile;
-window.isInitialized = false;
-window.hashFromString = hashFromString;
 
 const config = new Config();
-window.config = config;
 const tp = new TheatrePlay();
-window.tp = tp;
 const layers = [];
 const layersById = {};
 const layerOrder = new LayerOrder();
-window.layerOrder = layerOrder;
 const fontsAndAxes = [];
 let artboard;
 const exporter = new Exporter();
 const interactor = new Interactor();
 const moduleFS = new ModuleFS();
-window.moduleFS = moduleFS;
 const record = new Record(tp);
-window.record = record;
 const audio = new Audio(tp, record);
-window.audio = audio;
+const audioLayers = [];
 
+// globally reachables
+// classes
+window.config = config;
+window.tp = tp;
+window.layerOrder = layerOrder;
+window.moduleFS = moduleFS;
+window.record = record;
+window.audio = audio;
+// utilities
+window.uploadFile = uploadFile;
+window.downloadFile = downloadFile;
+window.isInitialized = false;
 window.panelFinderTimeout = false;
+
 const sequenceEventBuffer = {};
 
 //const midiController = new MidiController();
@@ -98,7 +105,7 @@ const getAbout = () => {
                 const buttonExp = textParent.querySelector(".expandText");
 
                 if (buttonExp === null) {
-                    console.error("Could not find .expandText within .textParent");
+                    console.error("Main::getAbout","Could not find .expandText within .textParent");
                     return;
                 }
 
@@ -247,7 +254,7 @@ window.onload = () => {
             }
             tp.studio.onSelectionChange((newSelection) => {
                 if (newSelection.length > 0) {
-                    [getArtboard(), getLayers()].flat().forEach((e) => {
+                    [getArtboard(), getLayers(), getAudioLayers()].flat().forEach((e) => {
                         if (e.id() === newSelection[0].address.objectKey) {
                             if (e.id().indexOf('layer-') === 0) {
                                 e.findInjectPanel();
@@ -257,6 +264,8 @@ window.onload = () => {
                                 }, 60);
                             } else if (e.id() === 'artboard') {
                                 e.findInjectPanel();
+                            } else if (e.id().indexOf('audio') === 0) {
+                                e.findInjectPanel();
                             }
                         }
                     });
@@ -450,11 +459,21 @@ window.getLayers = () => {
     return layers;
 };
 
+window.getAudioLayers = () => {
+    return audioLayers;
+};
+
+// for now we changed the design to have only one audio layer
+window.getAudioLayer = () => {
+    return audioLayers.length > 0 ? audioLayers[0] : false;
+};
+
 window.getLayer = (layerID = tp.studio.selection[0].address.objectKey) => {
     if (layerID === 'artboard') {
         return artboard;
     } else {
-        return layers.find((layer) => layer.id() === layerID);
+        //return layers.find((layer) => layer.id() === layerID);
+        return layersById[layerID];
     }
 };
 
@@ -564,6 +583,74 @@ const deleteLayer = (id, saveProject = true) => {
     }
 };
 
+const addAudioLayer = (filename = false) => {
+    return new Promise((resolve) => {
+        const audioLayerID = (() => {
+            let index = 0;
+            for (let i = 0; i < audioLayers.length; i++) {
+                if (audioLayers.findIndex((audioLayer) => audioLayer.id() === `audio-${index}`) < 0) {
+                    break;
+                }
+                index++;
+            }
+            return `audio-${index}`;
+        })();
+        const audioLayer = new AudioLayer(tp, audioLayerID, filename, false);
+        layersById[audioLayerID] = audioLayer;
+        audioLayers.push(audioLayer);
+        audioLayer.init().then(() => {
+            resolve(audioLayer);
+        });
+    });
+};
+
+const addExistingAudioLayer = (audioLayerID, values) => {
+    return new Promise((resolve) => {
+        const audioLayer = new AudioLayer(tp, audioLayerID, values, false);
+        audioLayers.push(audioLayer);
+        layersById[audioLayerID] = audioLayer;
+        audioLayer.init().then(() => {
+            resolve(audioLayer);
+        });
+    });
+};
+
+const deleteAudioLayer = (audioLayerID) => {
+    tp.removeObject(audioLayerID);
+    let index = 0;
+    for (let i = 0; i < audioLayers.length; i++) {
+        if (audioLayers[i].id() === audioLayerID) {
+            index = i;
+        }
+    }
+    audioLayers.splice(index, 1);
+    //delete audioLayersById[id];
+};
+
+const addAudioLayerTrack = (filename, start = 0, end = 1) => {
+    const addTrack = () => {
+        return new Promise((resolve) => {
+            audioLayers[0].addFile(filename).then(() => {
+                audioLayers[0].setTime(filename, start, end).then(() => {
+                    resolve();
+                });
+            });
+        });
+    };
+    return new Promise((resolve) => {
+        if (audioLayers.length < 1) {
+            addAudioLayer().then((audioLayer) => {
+                addTrack().then(() => {
+                    resolve();
+                });
+            });;
+        } else {
+            addTrack().then(() => {
+                resolve();
+            });
+        }
+    });
+};
 // TODO: better function names
 // because, come on. it may be funny for a second
 // but tolerance for fun is low when you're grumpy
@@ -588,6 +675,10 @@ window.duplicateLayer = (layer) => {
 window.addLayer = addLayer;
 window.addExistingLayer = addExistingLayer;
 window.deleteLayer = deleteLayer;
+window.addAudioLayer = addAudioLayer;
+window.addExistingAudioLayer = addExistingAudioLayer;
+window.deleteAudioLayer = deleteAudioLayer;
+window.addAudioLayerTrack = addAudioLayerTrack;
 window.renderFrames = exporter.renderFrames;
 
 const layer_panel = document.querySelector('#layer_panel');
@@ -629,7 +720,6 @@ const initPanels = () => {
                 moduleFS
                     .save(file)
                     .then(() => {
-                        console.log('ermh... done uploading?', file);
                         audio.readAudioFiles();
                     });
             });

bin/em/variabletime/web/js/moduleFS.js

@@ -38,12 +38,28 @@ const ModuleFS = function() {
         });
     };
 
+    this.readdir = (path) => {
+        return new Promise((resolve) => {
+            this.syncfs(MODE_READ_FROM_PERSISTENT)
+                .then(() => {
+                    const analyzed = FS.analyzePath(path);
+                    if (analyzed.exists && FS.isDir(analyzed.object.mode)) {
+                        const content = FS.readdir(path)
+                                            .filter((e) => ['.','..'].indexOf(e) < 0);
+                        resolve(content);
+                    } else {
+                        console.log('ModuleFS::readdir', `${path} is not a directory or does not exist`);
+                        resolve([]);
+                    }
+                });
+        });
+    };
+
     // check utils::uploadFile() for details of file
     this.save = (file) => {
         return new Promise((resolve) => {
             if (file.type.indexOf('font') >= 0 || file.hasOwnProperty('isFont') && file.isFont === true) {
                 var uint8View = new Uint8Array(file.arrayBuffer);
-                console.log('trying to save the font file, file, uint8View', file, uint8View);
                 if (!FS.analyzePath(`${config.fs.idbfsFontDir}/${file.name}`).exists) {
                     FS.createDataFile(config.fs.idbfsFontDir, file.name, uint8View, true, true);
                 }
@@ -54,7 +70,6 @@ const ModuleFS = function() {
             } else if (file.type.indexOf('zip') >= 0 || file.hasOwnProperty('isZip') && file.isZip === true) {
                 var uint8View = new Uint8Array(file.arrayBuffer);
                 var filePath = `${config.fs.idbfsTmpDir}/${file.name}`;
-                console.log(filePath);
                 if (!FS.analyzePath(filePath).exists) {
                     FS.createDataFile(config.fs.idbfsTmpDir, file.name, uint8View, true, true);
                 }
@@ -64,7 +79,6 @@ const ModuleFS = function() {
                     });
             } else if (file.type.indexOf('audio') === 0) {
                 var uint8View = new Uint8Array(file.arrayBuffer);
-                console.log('trying to save the audio file, file, uint8View', file, uint8View);
                 if (!FS.analyzePath(`${config.fs.idbfsAudioDir}/${file.name}`).exists) {
                     FS.createDataFile(config.fs.idbfsAudioDir, file.name, uint8View, true, true);
                 this.syncfs(MODE_WRITE_TO_PERSISTENT)
@@ -97,6 +111,33 @@ const ModuleFS = function() {
                 resolve(false);
             }
     };
+
+    const objectUrls = {};
+    this.objectUrl = (filePath) => {
+        if (typeof objectUrls[filePath] === 'undefined') {
+            const arr = FS.readFile(filePath);
+            let type = 'audio/wav';
+            const filesplit = filePath.split('.');
+            const extension = filesplit[filesplit.length - 1];
+
+            // is there a way to get mime type without guessing by extension?
+            if (extension === 'wav') {
+                type = 'audio/wav';
+            } else if (extension === 'mp3') {
+                type = 'audio/mpeg';
+            } else if (extension === 'ogg') {
+                type = 'audio/ogg';
+            } else if (extension === 'webm') {
+                type = 'audio/webm';
+            }
+            objectUrls[filePath] = URL.createObjectURL(
+                new Blob([arr], {
+                    type
+                })
+            );
+        }
+        return objectUrls[filePath];
+    };
 };
 
 export {

bin/em/variabletime/web/js/record.js

@@ -154,6 +154,28 @@ const Record = function(tp) {
     const RECORDING = 2;
     const STOPPING_RECORDING = 3;
 
+    this.possibleStates = {
+        NOT_RECORDING,
+        STARTING_RECORDING,
+        RECORDING,
+        STOPPING_RECORDING,
+    };
+
+    this.friendlyState = (state) => {
+        switch(state) {
+            case NOT_RECORDING: return 'NOT_RECORDING';
+            case STARTING_RECORDING: return 'STARTING_RECORDING';
+            case RECORDING: return 'RECORDING';
+            case STOPPING_RECORDING: return 'STOPPING_RECORDING';
+        }
+    };
+
+    const setIsRecording = (status) => {
+        isRecording = status;
+        window.dispatchEvent(new CustomEvent("record", {detail: status}));
+        console.log('setIsRecording', this.friendlyState(status));
+    };
+
     const hot = {}; 
     let isRecording = NOT_RECORDING;
     const buffy = new LiveBuffer();
@@ -290,6 +312,7 @@ const Record = function(tp) {
         if(isRecording === RECORDING) {
             stopRecording();
         } else {
+            // set microphone recording to false by default
             if (!propPaths) {
                 // make all mapped props hot and
                 Object.keys(audio.getMapping())
@@ -466,7 +489,7 @@ const Record = function(tp) {
     };
 
     const startRecording = () => {
-        isRecording = STARTING_RECORDING;
+        setIsRecording(STARTING_RECORDING);
         console.log('Record::startRecording');
         document.querySelector('#notice_recording')
             .classList.add('visible');
@@ -511,7 +534,7 @@ const Record = function(tp) {
             //tp.sheet.sequence.position = 0;
             tp.sheet.sequence.play();
         });
-        isRecording = RECORDING;
+        setIsRecording(RECORDING);
     };
     const stopRecording = () => {
         document.querySelector('#notice_recording')
@@ -520,7 +543,7 @@ const Record = function(tp) {
             .classList.add('imprenetrable');
         document.querySelector('#notice_recording .what p').innerHTML = 'digesting recording';
         document.querySelector('#notice_recording .details p').innerHTML = 'please wait';
-        isRecording = STOPPING_RECORDING;
+        setIsRecording(STOPPING_RECORDING);
         return new Promise((resolve) => {
             const layerKeys = Object.keys(hot);
             const promises = [];
@@ -584,7 +607,7 @@ const Record = function(tp) {
                 document.querySelector('#notice_recording')
                     .classList.remove('visible');
                 console.log('Record::stopRecording', 'stopped recording');
-                isRecording = NOT_RECORDING;
+                setIsRecording(NOT_RECORDING);
 
                 if (remember.isPlaying) {
                     tp.sheet.sequence.play();

bin/em/variabletime/web/js/theatre-play.js

@@ -228,11 +228,11 @@ const TheatrePlay = function(autoInit = false) {
             let waitify = false;
             keyframes.forEach((k) => {
                 const propTitle = k.path.join('.');
-                if (isSequenced(propTitle)) {
+                if (isSequenced(propTitle, layer)) {
                     waitify = true;
                     promises.push(() => {
                         return new Promise((subResolve) => {
-                            setSequenced(propTitle, false)
+                            setSequenced(propTitle, false, layer)
                                 .then(() => {
                                     subResolve();
                                 });
@@ -651,6 +651,7 @@ const TheatrePlay = function(autoInit = false) {
             project = projectJson === false ? core.getProject(projectId) : core.getProject(projectId, {
                 state: projectJson
             });
+            //console.log({project, projectJson});
             window.setLoadingTask('setting up animation', 10);
             project.ready.then(() => {
                 this.sheet = project.sheet('mainSheet');
@@ -724,6 +725,14 @@ const TheatrePlay = function(autoInit = false) {
                         window.project_fontsHashMap = project.fontsHashMap;
                         layerPromises.push(window.addExistingLayer(layerId, objects[layerId]));
                     });
+                //console.log(clone(objects));
+                Object.keys(objects)
+                    .filter((e) => e.indexOf('audio-') === 0)
+                    .forEach((layerId) => {
+                        window.setLoadingTask(`setting up the sounds of ${layerId} to come`, 90);
+                        window.project_fontsHashMap = project.fontsHashMap;
+                        layerPromises.push(window.addExistingAudioLayer(layerId, objects[layerId]));
+                    });
                 Promise.all(layerPromises).then(() => {
                     window.layerOrder.set(project.layerOrder);
                     this.duration = this.core.val(this.sheet.sequence.pointer.length);
@@ -812,6 +821,9 @@ const TheatrePlay = function(autoInit = false) {
             Module.setPlaying(playing);
         });
     };
+    this.isPlaying = () => {
+        return this.core.val(this.sheet.sequence.pointer.playing);
+    };
     this.studio = studio;
     this.core = core;
 

bin/em/variabletime/web/js/utils.js

@@ -34,6 +34,19 @@ const getUuid = () => {
     return uuid.getUuid();
 }
 
+const getTimestamp = () => {
+    const now = new Date();
+    const dd = String(now.getDate()).padStart(2, '0');
+    const mm = String(now.getMonth() + 1).padStart(2, '0'); //January is 0!
+    const yyyy = now.getFullYear();
+    const hours = String(now.getHours()).padStart(2, '0');
+    const minutes = String(now.getMinutes()).padStart(2, '0');
+    const seconds = String(now.getSeconds()).padStart(2, '0');
+
+    const timestamp = `${yyyy}.${mm}.${dd}.${hours}:${minutes}:${seconds}`;
+    return timestamp;
+}
+
 const makeEven = (n) => {
     const nr = Math.round(n);
     return nr - nr % 2;
@@ -433,16 +446,30 @@ const sequencialPromises = async (iterable, callback = false) => {
     }
 };
 
-// NOTE: this is not perfect,
-// but good enough for our use case
-// theoretically we would have to get
-// rid of all special characters
+// NOTE: this is perfect
 const toCssClass = (text, prefix = '') => {
-    return prefix + 'vt_' + text
-        .replaceAll('.','-dot-')
+    const cssClass = prefix + 'vt_' + text
+        .replaceAll('.', '-d-')
         .replaceAll(' ', '_')
-        .replaceAll('#','-hash-')
+        .replaceAll(':', '-c-')
+        .replaceAll('#', '-h-')
+        .replace(/[^a-zA-Z0-9_-]/g, "")
     ;
+    return cssClass;
+};
+const sanitizeTheatreKey = (key, removeExtension = true) => {
+    let theatreKey = key;
+    if (removeExtension) {
+        theatreKey = theatreKey.split('.');
+        if (theatreKey.length > 1) {
+            theatreKey.pop();
+        }
+        theatreKey = theatreKey.join('');
+    }
+    if (theatreKey.substr(0, 1).match(/^([0-9])$/i)) {
+        theatreKey = `t_${theatreKey}`;
+    }
+    return theatreKey.replace(/[^a-zA-Z0-9_]/g, "");
 };
 
 const renameProperty = (o, old_key, new_key) => {
@@ -527,6 +554,22 @@ const hexaToRgba = (hex_a) => {
     }
 };
 
+const getFileExtensionFromMimeType = (mimeType) => {
+    if (mimeType.toLowerCase().indexOf('audio/webm') >= 0) {
+        return "webm";
+    }
+    if (mimeType.toLowerCase().indexOf('audio/mpeg') >= 0) {
+        return "mp3";
+    }
+    if (mimeType.toLowerCase().indexOf('audio/ogg') >= 0) {
+        return "ogg";
+    }
+    if (mimeType.toLowerCase().indexOf('audio/wav') >= 0) {
+        return "wav";
+    }
+    return false;
+};
+
 
 /////////////////////////////////////
 // you can test these functions in
@@ -542,6 +585,7 @@ const hexaToRgba = (hex_a) => {
 
 export {
     getUuid,
+    getTimestamp,
     htmlToElement,
     downloadFile,
     uploadFile,
@@ -561,10 +605,12 @@ export {
     isMobile,
     sequencialPromises,
     toCssClass,
+    sanitizeTheatreKey,
     renameProperty,
     flattenObject,
     deFlattenObject,
     getNestedProperty,
     rgbaToHexa,
     hexaToRgba,
+    getFileExtensionFromMimeType,
 }

bin/em/variabletime/web/theatre_modules/core-and-studio.js

@@ -38466,7 +38466,6 @@ Instead found: ${devStringify(butFoundInstead)}` : "";
           callback: () => {
             getStudio().transaction(({ stateEditors: stateEditors2 }) => {
               const propAddress = __spreadProps(__spreadValues({}, obj.address), { pathToProp });
-              console.log({ propConfig, propAddress });
               stateEditors2.coreByProject.historic.sheetsById.sequence.setPrimitivePropAsSequenced(propAddress, propConfig);
             });
           }
@@ -50430,14 +50429,16 @@ ${content}</tr>
       max = Math.max(n3, max);
     }
     keyframes.forEach((cur, i3) => {
-      const curVal = valueInProp(cur.value, propConfig);
+      const cv = typeof cur.value === "boolean" ? cur.value ? 1 : 0 : cur.value;
+      const curVal = valueInProp(cv, propConfig);
       check(curVal);
       if (!cur.connectedRight)
         return;
       const next = keyframes[i3 + 1];
       if (!next)
         return;
-      const diff = (typeof next.value === "number" ? next.value : 1) - curVal;
+      const nv = typeof next.value === "boolean" ? next.value ? 1 : 0 : next.value;
+      const diff = (typeof nv === "number" ? nv : 1) - curVal;
       check(curVal + cur.handles[3] * diff);
       check(curVal + next.handles[1] * diff);
     });
@@ -50493,7 +50494,7 @@ ${content}</tr>
           };
         }, []);
         const extremumSpace = (0, import_react185.useMemo)(() => {
-          const extremums = propConfig.type === "number" ? calculateScalarExtremums(trackData.keyframes, propConfig) : calculateNonScalarExtremums(trackData.keyframes);
+          const extremums = propConfig.type === "number" || propConfig.type === "boolean" ? calculateScalarExtremums(trackData.keyframes, propConfig) : calculateNonScalarExtremums(trackData.keyframes);
           const fromValueSpace = (val3) => (val3 - extremums[0]) / (extremums[1] - extremums[0]);
           const toValueSpace = (ex) => extremums[0] + deltaToValueSpace(ex);
           const deltaToValueSpace = (ex) => ex * (extremums[1] - extremums[0]);
@@ -50518,7 +50519,7 @@ ${content}</tr>
           layoutP,
           sheetObject,
           trackId,
-          isScalar: propConfig.type === "number",
+          isScalar: propConfig.type === "number" || propConfig.type === "boolean",
           key: kf.id,
           extremumSpace: cachedExtremumSpace.current,
           color: color2
@@ -61771,7 +61772,6 @@ Note that it **is okay** to import '@theatre/core' multiple times. But those imp
                 detail: toVT
               });
               window.dispatchEvent(event);
-              console.log({ toVT, p: p3 });
             }
             sequence2.setPrimitivePropAsSequenced = setPrimitivePropAsSequenced;
             function setPrimitivePropAsStatic(p3) {
@@ -61840,7 +61840,6 @@ Note that it **is okay** to import '@theatre/core' multiple times. But those imp
                   };
                   tracks.trackData[trackId] = track;
                   tracks.trackIdByPropPath[encodedPropPath] = trackId;
-                  console.log({ trackId, encodedPropPath });
                 }
               }
               const toVT = {
@@ -62837,15 +62836,23 @@ Note that it **is okay** to import '@theatre/core' multiple times. But those imp
                   trackId,
                   keyframes
                 }));
+                console.log({
+                  a: __spreadValues({}, root3.address),
+                  objectKey,
+                  trackId,
+                  keyframes
+                });
               } else if (propConfig !== void 0) {
                 throw Error("hmmm");
               }
             };
             if (propConfig.type === "compound") {
               forEachPropDeep(defaultValue, (v6, pathToProp) => {
+                console.log("comp compoumnD");
                 addStaticOrKeyframeProp(v6, pathToProp);
               }, getPointerParts(prop).path);
             } else {
+              console.log("singlerer", { defaultValue, path });
               addStaticOrKeyframeProp(defaultValue, path);
             }
           } else {

grepJs.sh

@@ -1,3 +1,3 @@
 #!/bin/bash
 
-grep bin/web/js --exclude="*.min.*" --exclude="node_modules" --exclude="*.swp" --exclude="*.bundle.js" --exclude="script.js" --exclude="ffmpeg_modules" -nr -e $@
+grep bin/em/variabletime/web/js --exclude="*.min.*" --exclude="node_modules" --exclude="*.swp" --exclude="*.bundle.js" --exclude="script.js" --exclude="ffmpeg_modules" -nr -e "$@"

watch.sh

@@ -5,6 +5,10 @@ PREVIOUS_DIR=$(pwd)
 
 cd $DIR
 
+while true;
+do
     echo "$(git ls-files src && git ls-files bin/data && echo "assets/template.html")" | entr -d ./rebuild.sh
+    sleep 1
+done
 
 cd $PREVIOUS_DIR

watchJs.sh

@@ -5,6 +5,10 @@ PREVIOUS_DIR=$(pwd)
 
 cd $DIR
 
-echo "$(git ls-files bin/em/variabletime/web)" | entr -d ./reloadbrowser.sh
+while true;
+do
+    echo "$(git ls-files bin/em/variabletime/web && find "$(pwd)/bin/em/variabletime/web/theatre_modules" -iname "*.js")" | entr -d ./reloadbrowser.sh
+    sleep 1
+done
 
 cd $PREVIOUS_DIR