artboard audio, save audio with project json

not stable yet

assets/template.html

@@ -359,6 +359,7 @@
    <div class="content">
      <div class="what"><p></p></div>
      <div class="details"><p></p></div>
+     <div class="button"><p>OK</p></div>
    </div>
  </div>
  <div id="notice_recording">

bin/web/css/demo.css

@@ -217,12 +217,36 @@ body.debug div:not(.centerLine) {
 }
 
 #notice .content .what p {
+    font-size: 1.2em;
     color: black;
 }
 
 #notice .content .details p {
     color: black;
 }
+#notice .content .button {
+    display: none;
+    margin: 0px;
+    background-color: rgba(222, 222, 222, 0.97);
+    border: none;
+    cursor: pointer;
+    padding: 7px 15px;
+    border-radius: 10px;
+    font-size: 1.2em;
+    text-transform: uppercase;
+    font-variation-settings: 'wght' 800;
+    transition: all 0.125s;
+}
+#notice .content .button:hover {
+    background-color: rgba(0, 0, 0, 0.97);
+    color: rgba(222, 222, 222, 1.0);
+}
+#notice .content .button.visible {
+    display: flex;
+}
+#notice .content .button p {
+    user-select: none;
+}
 #notice_recording {
     position: fixed;
     top: 0px;

bin/web/js/artboard.js

@@ -11,7 +11,7 @@ const Artboard = function(tp, domElement = false, autoInit = true) {
     let x = 0;
     let y = 0;
     let props = {
-        backgroundColor: tp.core.types.rgba({
+        color: tp.core.types.rgba({
             r: 74,
             g: 94,
             b: 181,
@@ -100,25 +100,29 @@ const Artboard = function(tp, domElement = false, autoInit = true) {
     };
     const props2cppProps = (_props) => {
         let cppProps = JSON.parse(JSON.stringify(_props));
-        let bgIsArray = Array.isArray(cppProps.backgroundColor);
-        if (bgIsArray && cppProps.backgroundColor.length === 4) {
+        let bgIsArray = Array.isArray(cppProps.color);
+        if (bgIsArray && cppProps.color.length === 4) {
+            cppProps.backgroundColor = color;
+            delete cppProps.color;
             // nothing to do
-        } else if (!bgIsArray && cppProps.backgroundColor.hasOwnProperty('r')) {
+        } else if (!bgIsArray && cppProps.color.hasOwnProperty('r')) {
             cppProps.backgroundColor = [
-                cppProps.backgroundColor.r,
-                cppProps.backgroundColor.g,
-                cppProps.backgroundColor.b,
-                cppProps.backgroundColor.a
+                cppProps.color.r,
+                cppProps.color.g,
+                cppProps.color.b,
+                cppProps.color.a
             ];
-        } else if (!bgIsArray && cppProps.backgroundColor.default.hasOwnProperty('r')) {
+            delete cppProps.color;
+        } else if (!bgIsArray && cppProps.color.default.hasOwnProperty('r')) {
             cppProps.backgroundColor = [
-                cppProps.backgroundColor.default.r,
-                cppProps.backgroundColor.default.g,
-                cppProps.backgroundColor.default.b,
-                cppProps.backgroundColor.default.a
+                cppProps.color.default.r,
+                cppProps.color.default.g,
+                cppProps.color.default.b,
+                cppProps.color.default.a
             ];
+            delete cppProps.color;
         } else {
-            console.error('js::layer::props2cppProps', 'color could not be translated');
+            console.error('js::artboard::props2cppProps', 'color could not be translated');
         }
         return cppProps;
     };
@@ -146,6 +150,12 @@ const Artboard = function(tp, domElement = false, autoInit = true) {
                         panelPropTitle.innerHTML = friendlyName;
                     }
                 });
+                // should we have an audio object, let's inject the buttons, etc
+                if (typeof audio === 'object' && audio.hasOwnProperty('injectPanel')) {
+                    audio.injectPanel(this);
+                } else {
+                    console.log('Artboard::findInjectPanel', `cannot inject audio panel for ${this.id()} for some reason.`);
+                }
                 doItAgain = false;
             }
         }

bin/web/js/audio.js

@@ -46,7 +46,8 @@ const Audio = function(tp, record) {
     //document.body.addEventListener("click", init);
     let started = false;
 
-    const mapping = {};
+    let mapping = {};
+    let savedMapping = {};
     //const canvass = [];
     let canvasCombos = {};
     const mutationObserver = new MutationObserver(function(e) {
@@ -116,8 +117,8 @@ const Audio = function(tp, record) {
             ];
         } else if (propTitle.indexOf('letterDelay') === 0) {
             return [
-                config.audio.defaultRange.letterDelay[0],
-                config.audio.defaultRange.letterDelay[1]
+                config.audio.defaultRange.letterDelays[0],
+                config.audio.defaultRange.letterDelays[1]
             ];
         } else if (propTitle.split('.')[0] === 'fontVariationAxes') {
             return layer.props.fontVariationAxes
@@ -126,7 +127,9 @@ const Audio = function(tp, record) {
     };
 
     const getAudioMappingOptions = (layer, propTitle) => {
-        if (propTitle === 'color') {
+        if (savedMapping.hasOwnProperty(layer.id()) && savedMapping[layer.id()].hasOwnProperty(propTitle)) {
+            return savedMapping[layer.id()][propTitle];
+        } else if (propTitle === 'color') {
             const mm = getDefaultRange(layer, 'color');
             if (config.audio.colorSeparateRGBA) {
                 const r = new AudioMappingOptions();
@@ -222,8 +225,8 @@ const Audio = function(tp, record) {
         let hasLetterDelay = //false;
             config
             .layer.letterDelayProps
-            .indexOf(propTitle.split('.')[0]) >= 0 && propTitle.indexOf('color') < 0;
-            //&& tp.isSequenced([...[layer.id()], ...propTitle.split('.')]);
+            .indexOf(propTitle.split('.')[0]) >= 0 && propTitle.indexOf('color') < 0
+            && !tp.isSequenced(propTitle);
         const panel = tp.getPanel();
         if (!areMutationsObserved) {
             mutationObserver.observe(panel, { childList: true, subtree: true });
@@ -275,6 +278,11 @@ const Audio = function(tp, record) {
             }
             mappingOptions.source = panel.querySelector(toCssClass(`audio_source${propTitle}`,'#')).value;
             mappingOptions.muted = panel.querySelector(toCssClass(`audio_mute${propTitle}`,'#')).checked;
+
+            if (!savedMapping.hasOwnProperty(layer.id())) {
+                savedMapping[layer.id()] = {};
+            }
+            savedMapping[layer.id()][propTitle] = mappingOptions;
         };
         const source_Dom_Cont = document.createElement('div');
         source_Dom_Cont.classList.add('source_Dom_Cont');
@@ -615,11 +623,14 @@ const Audio = function(tp, record) {
     };
 
     const injectPanel = (layer) => {
+        console.log('injecting panel');
         const flatValues = clone(layer.theatreObject.value);
         flattenObject(flatValues, ['color']);
+        const layerType = layer.id().split('-')[0];
         const props = Object.keys(flatValues);
         props.forEach((propTitle) => {
-            if (config.audio.ignoreProps.indexOf(propTitle) < 0) {
+        console.log('injecting prop', propTitle);
+            if (config.audio.ignoreProps[layerType].indexOf(propTitle) < 0) {
                 let isActive = false;
                 if (mapping.hasOwnProperty(layer.id())) {
                     if (mapping[layer.id()].hasOwnProperty(propTitle)) {
@@ -691,9 +702,15 @@ const Audio = function(tp, record) {
         if (!started) {
             started = true;
             if (audioCtx !== false && audioCtx.state === 'suspended') {
+                if (confirm('It looks as if your project has audio.'
+                + 'Should we start audio now?'
+                + 'It is possible that you get a request that Variable Time may use your microphone.'
+                + 'Note: all data / microphone stream will stay on your device. If you don\'t believe us you can disconnect from the internet and it will still work. :-)')) {
                     audioCtx.resume();
+                } else {
                     return;
                 }
+            }
             heading.textContent = "Voice-change-O-matic";
             //document.body.removeEventListener("click", init);
 
@@ -1089,6 +1106,30 @@ const Audio = function(tp, record) {
                     };
                     drawAlt();
             }
+            if (audioCtx === false || audioCtx.state === 'suspended') {
+                const notice = document.querySelector('#notice');
+                const button = notice.querySelector('.button');
+                const buttonP = button.querySelector('p');
+                const whatP = notice.querySelector('.what p');
+                const detailsP = notice.querySelector('.details p');
+                button.classList.add('visible');
+                notice.classList.add('visible');
+
+                whatP.innerHTML = 'Start AudioContext';
+                detailsP.innerHTML = 'This project has audio. For audio to be allowed to start, we need a user interaction.<br>You are the user. If you click the button below, you interacted with Variable Time, and we can start audio.<br>Also, if you have not previously allowed Variable Time to use the microphone, there might be another notification asking you for permission.<br>Sounds good?';
+                buttonP.innerHTML = 'Yeah, absolutely, I love audio!';
+
+                const alright = () => {
+                    audioCtx.resume();
+                    button.classList.remove('visible');
+                    notice.classList.remove('visible');
+                    detailsP.innerHTML = '';
+                    whatP.innerHTML = '';
+                    buttonP.innerHTML = 'OK';
+                    button.removeEventListener('click', alright);
+                };
+                button.addEventListener('click', alright);
+            }
         }
     }
     const deinit = () => {
@@ -1106,7 +1147,10 @@ const Audio = function(tp, record) {
     this.init = init;
     this.deinit = deinit;
     this.injectPanel = injectPanel;
-    this.mapping = mapping;
+    this.getMapping = () => { return mapping; };
+    this.getSavedMapping = () => { return savedMapping };
+    this.setMapping = (m) => { mapping = m; };
+    this.setSavedMapping = (m) => { savedMapping = m; };
     this.addAudioMapping = addAudioMapping;
     this.removeAudioMapping = removeAudioMapping;
     this.addAudioOptions = addAudioOptions;

bin/web/js/config.js

@@ -6,7 +6,7 @@ const config = {
         maximumPixelDensity: 3.0,
         incrementPixelDensity: 0.01,
         friendlyNames: {
-            'backgroundColor': 'Background<br>Color',
+            'color': 'Background<br>Color',
             'x': 'Position X',
             'y': 'Position Y',
             'width': 'Artboard Width',
@@ -94,7 +94,10 @@ const config = {
             'color': [0, 1],
             'letterDelays': [0, 1000],
         },
-        ignoreProps: ['transformOrigin', 'fontFamily', 'text', 'mirror_x', 'mirror_y', 'mirror_xy', 'height'],
+        ignoreProps: { 
+            artboard: ['x', 'y', 'zoom', 'pixelDensity', 'width', 'height'],
+            layer: ['transformOrigin', 'fontFamily', 'text', 'mirror_x', 'mirror_y', 'mirror_xy', 'height'],
+        },
         maxFilenameLength: 24,
         defaultSmoothing: 0.7,
         analyser: {
@@ -112,7 +115,10 @@ const config = {
         rolloverResetLoop: true,
     },
     record: {
-        ignoreProps: ['transformOrigin', 'fontFamily', 'text', 'mirror_x', 'mirror_y', 'mirror_xy'],
+        ignoreProps: {
+            artboard: ['x', 'y', 'zoom', 'pixelDensity', 'width', 'height'],
+            layer: ['transformOrigin', 'fontFamily', 'text', 'mirror_x', 'mirror_y', 'mirror_xy'],
+        },
         recordMapped: true,
     },
     midi: {

bin/web/js/layer.js

@@ -463,11 +463,14 @@ const Layer = function(tp, layerID, fontsAndAxes, autoInit = true) {
             const panelProp= tp.getPanelPropTitle(propName);
             const panelPropMom = tp.getPanelPropContainer(panelProp);
             const newValue = active ? getArtboard().theatreObject.value.width : 0;
+            const textWrappingButton = tp.getPanel().querySelector('.textWrappingButton');
+
+            if (panelPropMom !== null && textWrappingButton !== null) {
             if (active) {
                 panelPropMom.style.display = 'flex';
-                tp.getPanel().querySelector('.textWrappingButton').classList.add('active');
+                textWrappingButton.classList.add('active');
             } else {
-                tp.getPanel().querySelector('.textWrappingButton').classList.remove('active');
+                textWrappingButton.classList.remove('active');
                 panelPropMom.style.display = 'none';
             }
 
@@ -479,6 +482,7 @@ const Layer = function(tp, layerID, fontsAndAxes, autoInit = true) {
                 });
             }
             }
+        }
     };
 
     // phew.. so we don't want to get into typescript and react atm
@@ -809,7 +813,7 @@ const Layer = function(tp, layerID, fontsAndAxes, autoInit = true) {
                         }
 
                         injectedPanel = true;
-                        const detail = {titles: Object.keys(panelPropTitles), containers: Object.keys(panelPropContainers)};
+                        const detail = {layerID: this.id(), titles: Object.keys(panelPropTitles), containers: Object.keys(panelPropContainers)};
                         const e = new CustomEvent('injected', {detail});
                         tp.getPanel().dispatchEvent(e);
                         if (typeof resolve === 'function') {

bin/web/js/main.js

@@ -261,6 +261,7 @@ window.onload = () => {
                 }
                 findInjectPanel();
                 tp.friendlySequenceNames();
+                console.log('Main::selectionChange',newSelection.length > 0 ? newSelection[0].address.objectKey : 'aah nothing');
             });
         });
     // ABOUT BEGIN

bin/web/js/record.js

@@ -257,10 +257,10 @@ const Record = function(tp) {
                     } else {
                         if (config.record.recordMapped) {
                             // make all mapped props hot and
-                            Object.keys(audio.mapping)
+                            Object.keys(audio.getMapping())
                                 .forEach((layerID) => {
                                     //if (getLayer(layerID).isSelected()) { // NOTE: multilayer recording
-                                        Object.keys(audio.mapping[layerID])
+                                        Object.keys(audio.getMapping()[layerID])
                                             .forEach((propTitle) => {
                                                 addHot(layerID, propTitle);
                                             });
@@ -347,8 +347,9 @@ const Record = function(tp) {
         const flatValues = clone(layer.theatreObject.value);
         flattenObject(flatValues, ['color']);
         const props = Object.keys(flatValues);
+        const layerType = layer.id().split('-')[0];
         props.forEach((propTitle) => {
-            if (config.record.ignoreProps.indexOf(propTitle) < 0) {
+            if (config.record.ignoreProps[layerType].indexOf(propTitle) < 0) {
                 addRecordButton(layer, propTitle);
             }
         });

bin/web/js/theatre-play.js

@@ -1143,6 +1143,8 @@ const TheatrePlay = function(autoInit = false) {
             layerOrder: window.layerOrder.get(),
             fontsHashMap,
             projectId,
+            audioSavedMapping: audio.getSavedMapping(),
+            audioMapping: audio.getMapping(),
         };
         const theatre = tp.studio.createContentOfSaveFile(currentProjectId);
         return this.theatre2variableTime(theatre, vt_params);
@@ -1251,6 +1253,11 @@ const TheatrePlay = function(autoInit = false) {
                     artboardValues['zoom'] = artboardValues['scale'];
                     delete artboardValues['scale'];
                 }
+                if (artboardValues.hasOwnProperty('backgroundColor')) {
+                    artboardValues['color'] = artboardValues['backgroundColor'];
+                    delete artboardValues['backgroundColor'];
+                }
+                // for backward compatibility merge with current values
                 // for backward compatibility merge with current values
                 const defaultArtboardValues = window.getArtboard().theatreObject.value;
                 const artboardProps = {...defaultArtboardValues, ...artboardValues};
@@ -1260,6 +1267,12 @@ const TheatrePlay = function(autoInit = false) {
                 }) => {
                     set(window.getArtboard().theatreObject.props, artboardProps);
                 });
+                    if (project.hasOwnProperty('audioMapping')) {
+                        audio.setMapping(project.audioMapping);
+                    }
+                    if (project.hasOwnProperty('audioSavedMapping')) {
+                        audio.setSavedMapping(project.audioSavedMapping);
+                    }
 
                 // load layers
                 const layerPromises = [];
@@ -1277,6 +1290,14 @@ const TheatrePlay = function(autoInit = false) {
                         getLayers().forEach((layer) => {
                             if (layer.id() === project.layerOrder[project.layerOrder.length - 1]) {
                                 layer.select();
+                                const addAudioMapping = (e) => {
+                                    //if (project.audioMapping.hasOwnProperty(layer.id())) {
+                                        //Object.keys(project.audioMapping[layer.id()]).forEach((propTitle) => {
+                                            //audio.addAudioOptions(layer, propTitle);
+                                        //});
+                                    //}
+                                };
+                                //tp.getPanel().addEventListener('injected', addAudioMapping);
                             }
                         });
                     }
@@ -1296,6 +1317,9 @@ const TheatrePlay = function(autoInit = false) {
                         });
                     });
                 } else {
+                    const layers = getLayers();
+                    if (layers.length > 0) {
+                    }
                     this.duration = this.core.val(this.sheet.sequence.pointer.length);
                     resolve();
                     this.isProjectLoaded = true;

src/main.cpp

@@ -137,6 +137,7 @@ shared_ptr <VariableEditor::ofApp> app;
         dir.listDir();
 
         ofDirectory userFonts("/idbfs/fonts");
+        ofDirectory userAudio("/idbfs/audio"); // possibly rename to media?
 
         nlohmann::json json;
         json["files"] = nlohmann::json::array();
@@ -157,6 +158,15 @@ shared_ptr <VariableEditor::ofApp> app;
                 if(!file.moveTo(userFonts)){
                     std::cout << "Module::importProjectAsZip" << " - cannot move font " << file.getFileName();
                 }
+            }else if(ofToLower(file.getExtension()) == "mp3"
+                     || ofToLower(file.getExtension()) == "m4a"
+                     || ofToLower(file.getExtension()) == "wav"
+                     || ofToLower(file.getExtension()) == "webm"
+                     || ofToLower(file.getExtension()) == "aac"
+                     || ofToLower(file.getExtension()) == "ogg"){
+                if(!file.moveTo(userAudio)){
+                    std::cout << "Module::importProjectAsZip" << " - cannot move audiofile " << file.getFileName();
+                }
             }else{
                 file.remove();
             }

src/ofApp.h

@@ -50,6 +50,7 @@ class ZipProjectSaver : public ofThread {
             this->projectName = projectName;
             this->projectJsonString = projectJsonString;
             this->userFontsPath = "/idbfs/fonts";
+            this->userAudioPath = "/idbfs/audio";
             this->timestamp = ofGetTimestampString();
             this->filename = projectName + "_project_" + timestamp + ".zip";
         }
@@ -93,6 +94,28 @@ class ZipProjectSaver : public ofThread {
                 ofBuffer buffy = file.readToBuffer();
                 zip.addBuffer(fontFilename, buffy.getData(), buffy.size());
             }
+            ofDirectory userAudio(userAudioPath);
+            userAudio.sort();
+            userAudio.allowExt("mp3");
+            userAudio.allowExt("ogg");
+            userAudio.allowExt("wav");
+            userAudio.allowExt("webm");
+            userAudio.allowExt("m4a");
+            userAudio.listDir();
+            for(int i = 0; i < userAudio.size(); i++){
+                std::string audioFilename = userAudio.getName(i);
+                std::string audioFilepath = userAudioPath + "/" + audioFilename;
+                ofFile file = userAudio.getFile(i);
+                ofStringReplace(audioFilepath, "/idbfs/", "idbfs/");
+                if(of::filesystem::exists(audioFilepath)){
+                    //cout << "huuurrayy " << audioFilepath << " exists" << endl;
+                }else{
+                    cout << "ofApp::downloadProject() trying to load " << audioFilepath << " but it does not exist." << endl;
+                }
+                file.open(audioFilepath);
+                ofBuffer buffy = file.readToBuffer();
+                zip.addBuffer(audioFilename, buffy.getData(), buffy.size());
+            }
             buffer = NULL;
             buffer_size = 0;
             zip.getOutputBuffer(&buffer, buffer_size);
@@ -107,6 +130,7 @@ class ZipProjectSaver : public ofThread {
         string projectName;
         string projectJsonString;
         string userFontsPath;
+        string userAudioPath;
         string timestamp;
         string filename;
         char * buffer;