From d35d949526ac298d263dda59f5ac1ad311e74f1d Mon Sep 17 00:00:00 2001 From: themancalledjakob Date: Thu, 11 Apr 2024 18:06:45 +0200 Subject: [PATCH] add audio layer and audio player dependencies hashes: openFrameworks d78075f4bca6be2a2533c6e51a75cc1f18404501 ofxMsdfgen e14da13d02c4dff04fb69d7923469f606924e6c3 ofxGPUFont d482bb7cbdf6b296fa4ab5abcf73fb5ff8c8b239 ofxVariableLab 0b5f9bdebc1e5550621957e73c040c258ec6317b ofxProfiler a868e34fa1a79189dd4fbdede2938e308535e5e8 theatre 86d3e07f6f2c75fd6e08fca8c97e3617c9e23b18 --- bin/em/variabletime/web/css/theatre.css | 9 +- bin/em/variabletime/web/js/audio.js | 122 +++----------------- bin/em/variabletime/web/js/audioLayer.js | 123 +++++++++++++++++++++ bin/em/variabletime/web/js/audioPlayer.js | 108 ++++++++++++++++++ bin/em/variabletime/web/js/main.js | 58 +++++++++- bin/em/variabletime/web/js/moduleFS.js | 17 +++ bin/em/variabletime/web/js/theatre-play.js | 8 +- 7 files changed, 328 insertions(+), 117 deletions(-) create mode 100644 bin/em/variabletime/web/js/audioLayer.js create mode 100644 bin/em/variabletime/web/js/audioPlayer.js diff --git a/bin/em/variabletime/web/css/theatre.css b/bin/em/variabletime/web/css/theatre.css index d8036a6..883ba64 100644 --- a/bin/em/variabletime/web/css/theatre.css +++ b/bin/em/variabletime/web/css/theatre.css @@ -207,6 +207,7 @@ margin-bottom: 3px; .source_Dom_Cont { padding: 5px; display: flex; + flex-wrap: wrap; align-items: center; justify-content: space-between; border-top: 1px dashed #91919177; @@ -370,14 +371,6 @@ input[type=checkbox]:checked+label { -webkit-tap-highlight-color: transparent; } -/*input[type=checkbox]+label::after{ - content: ' OFF'; -} - -input[type=checkbox]:checked+label::after { - content: ' ON'; -}*/ - label svg{ width: 20px; height: 20px; diff --git a/bin/em/variabletime/web/js/audio.js b/bin/em/variabletime/web/js/audio.js index fe186a6..fddfa90 100644 --- a/bin/em/variabletime/web/js/audio.js +++ b/bin/em/variabletime/web/js/audio.js @@ -13,8 +13,9 @@ import { getFileExtensionFromMimeType, } from './utils.js'; -window.mapValue = mapValue; - +import { + AudioPlayer +} from './audioPlayer.js'; const AudioMappingOptions = function() { this.min_freq = 0.0; @@ -40,102 +41,6 @@ window.playAudioFile = (file) => { return audioElement; }; -const AudioPlayer = function() { - const audioElements = []; - let updateInterval = false; - let updateInterval_ms = 10; - this.add = (layer, propTitle, time, file) => { - const layerID = typeof layer === 'string' ? layer : layer.id(); - console.log('AudioPlayer::add',{layerID, propTitle, time, file}); - const index = audioElements.findIndex((e) => e.layerID === layerID && e.propTitle === propTitle); - if (index === -1) { - const audioElement = document.createElement('audio'); - audioElement.classList.add('invisible'); - audioElement.classList.add('audio_file'); - audioElement.src = audio.audioSourceCombos[file].audioElement.src; - audioElements.push({ - layerID, propTitle, audioElement, time, file - }); - } else { - audioElements[index].src = audio.audioSourceCombos[file].audioElement.src; - audioElements[index].time = time; - } - }; - this.update = () => { - audioElements.forEach((audioElement, i) => { - if (tp.isPlaying() && !record.isRecording()) { - const diff = tp.sheet.sequence.position - audioElement.time; - if (diff >= 0) { - if (audioElement.audioElement.paused) { - audioElement.audioElement.currentTime = diff; - audioElement.audioElement.play(); - console.log('play audioElement ', audioElement.file, audioElement.propTitle, i); - } - } else if(!audioElement.audioElement.paused) { - audioElement.audioElement.pause(); - audioElement.audioElement.currentTime = 0; - console.log('paus audioElement ', audioElement.file, audioElement.propTitle, i); - } - } else if (!audioElement.audioElement.paused) { - audioElement.audioElement.pause(); - audioElement.audioElement.currentTime = 0; - console.log('pausé audioElement ', audioElement.file, audioElement.propTitle, i); - } - }); - }; - this.audioElements = audioElements; - this.init = () => { - clearInterval(updateInterval); - updateInterval = setInterval(() => { - this.update(); - }, updateInterval_ms); - }; - this.listener = (event) => { - console.log('AUDIOPLAYORECEIVED', event); - let hot = false; - let time = false; - if (event.detail === record.possibleStates.RECORDING) { - hot = clone(record.getHot()); - time = tp.sheet.sequence.position; - const layerIDs = Object.keys(hot); - layerIDs.forEach((layerID) => { - const propTitles = Object.keys(hot[layerID]); - propTitles.forEach((propTitle) => { - const m = audio.getMapping()[layerID][propTitle]; - if (m.addToTimeline) { - if (m.source === 'microphone') { - const waitForMicrophoneListener = (event) => { - if (event.detail.fileIsRead) { - this.add(layerID, propTitle, time, event.detail.filename); - window.removeEventListener('microphoneRecorder', waitForMicrophoneListener); - } - }; - window.addEventListener('microphoneRecorder', waitForMicrophoneListener); - } - } - }); - }); - } - if (event.detail === record.possibleStates.NOT_RECORDING) { - const layerIDs = Object.keys(hot); - layerIDs.forEach((layerID) => { - const propTitles = Object.keys(hot[layerID]); - propTitles.forEach((propTitle) => { - const m = audio.getMapping()[layerID][propTitle]; - if (m.addToTimeline) { - if (m.source === 'microphone') { - // we already handled this above - } else { - this.add(layerID, propTitle, time, m.source); - } - } - }); - }); - window.removeEventListener('record', this.listener); - } - }; -}; - const MicrophoneRecorder = function() { let recorder = false; let buffy = []; @@ -456,8 +361,14 @@ const Audio = function(tp, record) { audioOptions.style.order = parseInt(container.style.order) + 1; const updateMappingOptions = () => { + if (panel.querySelector(toCssClass(`audioOptions${propTitle}`,'.')) === null) { + // we may not have the audioOptions created, if we select a different layer + // so let's give up in case there are no audioOptions and + // return safely from our journey before anyone gets hurt. + return; + } isSequenced = tp.isSequenced(propTitle, layer); - isLetterDelayMapped = typeof mapping[layer.id()][`letterDelays.${propTitle}`] !== 'undefined'; // if the letterDelay is mapped itself, we don't do it + isLetterDelayMapped = typeof mapping[layer.id()] === 'object' && typeof mapping[layer.id()][`letterDelays.${propTitle}`] !== 'undefined'; // if the letterDelay is mapped itself, we don't do it if (propTitle === 'color') { mappingOptions.min_out = hexaToRgba(panel.querySelector(toCssClass(`audio_min${propTitle}`,'#')).value); const max_cssClass = toCssClass(`audio_max${propTitle}`,'#'); @@ -532,22 +443,25 @@ const Audio = function(tp, record) { { const cssClass = toCssClass(`audio_addToTimeline${propTitle}`); const checkboxDom = document.createElement('input'); + checkboxDom.classList.add(cssClass); checkboxDom.type = 'checkbox'; + checkboxDom.title = 'add to timeline'; checkboxDom.checked = false; mappingOptions.addToTimeline = checkboxDom.checked; checkboxDom.addEventListener('change', (event) => { mappingOptions.addToTimeline = event.currentTarget.checked; }); - record_Dom_Cont.append(checkboxDom); + const checkboxDom_label = document.createElement('label'); + checkboxDom_label.innerHTML = 'add to timeline'; + checkboxDom_label.append(checkboxDom); + record_Dom_Cont.append(checkboxDom_label); } recordSoloButton.addEventListener('click', () => { if (!record.isRecording()) { - console.log('SHOULD ADD LEAST AUDOPLAYOR'); // will be recording window.addEventListener('record', audioPlayer.listener); if (mappingOptions.source === 'microphone') { - console.log('SHOULD ADD LEAST ALSO THE MICROPHONERECORDER'); window.addEventListener('record', microphoneRecorder.startListener); window.addEventListener('record', microphoneRecorder.stopListener); } else { @@ -919,7 +833,6 @@ const Audio = function(tp, record) { canvasCombos[propTitle] = [fft_imgDom, fft_imgDom.getContext("2d"), layer.id()]; - tp.getPanel().addEventListener('updateAudioMapping', () => { updateMappingOptions(); }); @@ -1630,6 +1543,5 @@ const Audio = function(tp, record) { }; export { - Audio, - AudioLayer + Audio } diff --git a/bin/em/variabletime/web/js/audioLayer.js b/bin/em/variabletime/web/js/audioLayer.js new file mode 100644 index 0000000..8aa3dec --- /dev/null +++ b/bin/em/variabletime/web/js/audioLayer.js @@ -0,0 +1,123 @@ +const AudioLayer = function(tp, audioLayerID, values = false, autoInit = true) { + // private + let props = { + time: tp.core.types.boolean(false), + audioFile: tp.core.types.stringLiteral( + 'no_audio_file', + { + no_audio_file: 'no audio file :(', + }, + ), + }; + + // private functions + const onValuesChange = (values) => { + console.log(values); + }; + + const updateAudioFiles = () => { + console.log('update audio files'); + return new Promise((resolve) => { + moduleFS.readdir(config.fs.idbfsAudioDir).then((files) => { + ; + console.log(files); + if (files.length > 0) { + const audioFiles = { + no_audio_file: 'no audio file :(', + }; + files.forEach((file) => { + audioFiles[file] = file; + }); + props.audioFile = tp.core.types.stringLiteral( + audioFiles[files[0]], + audioFiles + ); + tp.changeObject(this.id(), props); + if (typeof this.theatreObject === 'object') { + tp.studio.transaction(({ + set + }) => { + set(this.theatreObject.props.audioFile, files[0]); + }); + } + } + resolve(files); + }); + }); + }; + + const findInjectPanel = () => { + console.log('find and inject'); + }; + + const setTime = (start, stop) => { + const keyframes = [ + { + path: ['time'], + keyframes: [ + { + position: -1, + value: false, + }, + { + position: start, + value: true, + }, + { + position: stop, + value: false, + }, + ], + } + ]; + tp.setKeyframes(this, keyframes); + }; + + const init = () => { + return new Promise((resolve) => { + updateAudioFiles().then((files) => { + this.theatreObject = tp.addObject(audioLayerID, this.props, onValuesChange); + tp.changeObject(audioLayerID, this.props); + tp.studio.transaction(({ + set + }) => { + const mergedValues = {...this.theatreObject.value, ...values}; + console.log({values, mergedValues}, this.props); + set(this.theatreObject.props, values ? mergedValues : this.theatreObject.value); + }); + // this has to be a timeout, because theatre is not ready otherwise + + setTime(1,2); + resolve(); + }); + }); + }; + + const remove = () => { + }; + + // public + this.props = props; + + this.findInjectPanel = findInjectPanel; + + this.setTime = setTime; + + this.init = init; + + this.id = () => { + return audioLayerID; + }; + + // well, I know this is a shitty name. My apologies. However, .. + this.prepareForDepartureFromThisBeautifulExperience = remove; + + // action + if (autoInit) { + this.init(); + } +}; + +export { + AudioLayer, +} diff --git a/bin/em/variabletime/web/js/audioPlayer.js b/bin/em/variabletime/web/js/audioPlayer.js new file mode 100644 index 0000000..0951f25 --- /dev/null +++ b/bin/em/variabletime/web/js/audioPlayer.js @@ -0,0 +1,108 @@ +import { + clone, +} from './utils.js'; + +const AudioPlayer = function() { + const audioElements = []; + let updateInterval = false; + let updateInterval_ms = 10; + this.add = (layer, propTitle, time, file) => { + const layerID = typeof layer === 'string' ? layer : layer.id(); + propTitle = Array.isArray(propTitle) ? propTitle.join('.') : propTitle; + console.log('AudioPlayer::add',{layerID, propTitle, time, file}); + const index = audioElements.findIndex((e) => e.layerID === layerID && e.propTitle === propTitle); + if (index === -1) { + const audioDomElement = document.createElement('audio'); + audioDomElement.classList.add('invisible'); + audioDomElement.classList.add('audio_file'); + audioDomElement.src = audio.audioSourceCombos[file].audioElement.src; + audioElements.push({ + layerID, propTitle, audioDomElement, time, file + }); + } else { + audioElements[index].src = audio.audioSourceCombos[file].audioDomElement.src; + audioElements[index].time = time; + } + }; + this.update = () => { + audioElements.forEach((audioElement, i) => { + if (tp.isPlaying() && !record.isRecording()) { + const diff = tp.sheet.sequence.position - audioElement.time; + if (diff >= 0) { + if (audioElement.audioDomElement.paused) { + audioElement.audioDomElement.currentTime = diff; + audioElement.audioDomElement.play(); + console.log('play audioElement ', audioElement.file, audioElement.propTitle, i); + } + } else if(!audioElement.audioDomElement.paused) { + audioElement.audioDomElement.pause(); + audioElement.audioDomElement.currentTime = 0; + console.log('paus audioElement ', audioElement.file, audioElement.propTitle, i); + } + } else if (!audioElement.audioDomElement.paused) { + audioElement.audioDomElement.pause(); + audioElement.audioDomElement.currentTime = 0; + console.log('pausé audioElement ', audioElement.file, audioElement.propTitle, i); + } + }); + }; + this.audioElements = audioElements; + this.init = () => { + clearInterval(updateInterval); + updateInterval = setInterval(() => { + this.update(); + }, updateInterval_ms); + }; + this.listener = (event) => { + let hot = false; + let time = false; + if (event.detail === record.possibleStates.RECORDING) { + hot = clone(record.getHot()); + time = tp.sheet.sequence.position; + const layerIDs = Object.keys(hot); + layerIDs.forEach((layerID) => { + const propTitles = Object.keys(hot[layerID]); + propTitles.forEach((propTitle) => { + const m = audio.getMapping()[layerID][propTitle]; + if (m.addToTimeline) { + if (m.source === 'microphone') { + const waitForMicrophoneListener = (event) => { + if (event.detail.fileIsRead) { + this.add(layerID, propTitle, time, event.detail.filename); + addAudioLayer().then((audioLayer) => { + m.layer = audioLayer; + }); + window.removeEventListener('microphoneRecorder', waitForMicrophoneListener); + } + }; + window.addEventListener('microphoneRecorder', waitForMicrophoneListener); + } + } + }); + }); + } + if (event.detail === record.possibleStates.NOT_RECORDING) { + const layerIDs = Object.keys(hot); + layerIDs.forEach((layerID) => { + const propTitles = Object.keys(hot[layerID]); + propTitles.forEach((propTitle) => { + const m = audio.getMapping()[layerID][propTitle]; + if (m.addToTimeline) { + if (m.source === 'microphone') { + } else { + this.add(layerID, propTitle, time, m.source); + addAudioLayer().then((audioLayer) => { + m.layer = audioLayer; + }); + } + } + }); + }); + window.removeEventListener('record', this.listener); + } + }; +}; + +export { + AudioPlayer +}; diff --git a/bin/em/variabletime/web/js/main.js b/bin/em/variabletime/web/js/main.js index 68bc313..35a2095 100644 --- a/bin/em/variabletime/web/js/main.js +++ b/bin/em/variabletime/web/js/main.js @@ -34,6 +34,9 @@ import { Record } from './record.js'; +import { + AudioLayer +} from './audioLayer.js'; //import { //MidiController //} from './midiController.js'; @@ -65,6 +68,7 @@ const interactor = new Interactor(); const moduleFS = new ModuleFS(); const record = new Record(tp); const audio = new Audio(tp, record); +const audioLayers = []; // globally reachables // classes @@ -250,7 +254,7 @@ window.onload = () => { } tp.studio.onSelectionChange((newSelection) => { if (newSelection.length > 0) { - [getArtboard(), getLayers(), getAudioLayer()].flat().forEach((e) => { + [getArtboard(), getLayers(), getAudioLayers()].flat().forEach((e) => { if (e.id() === newSelection[0].address.objectKey) { if (e.id().indexOf('layer-') === 0) { e.findInjectPanel(); @@ -455,6 +459,10 @@ window.getLayers = () => { return layers; }; +window.getAudioLayers = () => { + return audioLayers; +}; + window.getLayer = (layerID = tp.studio.selection[0].address.objectKey) => { if (layerID === 'artboard') { return artboard; @@ -570,6 +578,51 @@ const deleteLayer = (id, saveProject = true) => { } }; +const addAudioLayer = (values = false) => { + return new Promise((resolve) => { + const audioLayerID = (() => { + let index = 0; + for (let i = 0; i < audioLayers.length; i++) { + if (audioLayers.findIndex((audioLayer) => audioLayer.id() !== `audio-${index}`) < 0) { + break; + } + index++; + } + return `audio-${index}`; + })(); + console.log('adding', audioLayerID); + const audioLayer = new AudioLayer(tp, audioLayerID, values, false); + layersById[audioLayerID] = audioLayer; + audioLayers.push(audioLayer); + audioLayer.init().then(() => { + resolve(audioLayer); + }); + }); +}; + +const addExistingAudioLayer = (audioLayerID, values) => { + return new Promise((resolve) => { + const audioLayer = new AudioLayer(tp, audioLayerID, values, false); + audioLayers.push(audioLayer); + layersById[audioLayerID] = audioLayer; + audioLayer.init().then(() => { + resolve(); + }); + }); +}; + +const deleteAudioLayer = (audioLayerID) => { + tp.removeObject(audioLayerID); + let index = 0; + for (let i = 0; i < audioLayers.length; i++) { + if (audioLayers[i].id() === audioLayerID) { + index = i; + } + } + audioLayers.splice(index, 1); + //delete audioLayersById[id]; +}; + // TODO: better function names // because, come on. it may be funny for a second // but tolerance for fun is low when you're grumpy @@ -594,6 +647,9 @@ window.duplicateLayer = (layer) => { window.addLayer = addLayer; window.addExistingLayer = addExistingLayer; window.deleteLayer = deleteLayer; +window.addAudioLayer = addAudioLayer; +window.addExistingAudioLayer = addExistingAudioLayer; +window.deleteAudioLayer = deleteAudioLayer; window.renderFrames = exporter.renderFrames; const layer_panel = document.querySelector('#layer_panel'); diff --git a/bin/em/variabletime/web/js/moduleFS.js b/bin/em/variabletime/web/js/moduleFS.js index a3d872d..ff99589 100644 --- a/bin/em/variabletime/web/js/moduleFS.js +++ b/bin/em/variabletime/web/js/moduleFS.js @@ -38,6 +38,23 @@ const ModuleFS = function() { }); }; + this.readdir = (path) => { + return new Promise((resolve) => { + this.syncfs(MODE_READ_FROM_PERSISTENT) + .then(() => { + const analyzed = FS.analyzePath(path); + if (analyzed.exists && FS.isDir(analyzed.object.mode)) { + const content = FS.readdir(path) + .filter((e) => ['.','..'].indexOf(e) < 0); + resolve(content); + } else { + console.log('ModuleFS::readdir', `${path} is not a directory or does not exist`); + resolve([]); + } + }); + }); + }; + // check utils::uploadFile() for details of file this.save = (file) => { return new Promise((resolve) => { diff --git a/bin/em/variabletime/web/js/theatre-play.js b/bin/em/variabletime/web/js/theatre-play.js index 7e2dacc..c431e22 100644 --- a/bin/em/variabletime/web/js/theatre-play.js +++ b/bin/em/variabletime/web/js/theatre-play.js @@ -228,11 +228,11 @@ const TheatrePlay = function(autoInit = false) { let waitify = false; keyframes.forEach((k) => { const propTitle = k.path.join('.'); - if (isSequenced(propTitle)) { + if (isSequenced(propTitle, layer)) { waitify = true; promises.push(() => { return new Promise((subResolve) => { - setSequenced(propTitle, false) + setSequenced(propTitle, false, layer) .then(() => { subResolve(); }); @@ -651,6 +651,7 @@ const TheatrePlay = function(autoInit = false) { project = projectJson === false ? core.getProject(projectId) : core.getProject(projectId, { state: projectJson }); + console.log({project, projectJson}); window.setLoadingTask('setting up animation', 10); project.ready.then(() => { this.sheet = project.sheet('mainSheet'); @@ -724,12 +725,13 @@ const TheatrePlay = function(autoInit = false) { window.project_fontsHashMap = project.fontsHashMap; layerPromises.push(window.addExistingLayer(layerId, objects[layerId])); }); + console.log(clone(objects)); Object.keys(objects) .filter((e) => e.indexOf('audio-') === 0) .forEach((layerId) => { window.setLoadingTask(`setting up the sounds of ${layerId} to come`, 90); window.project_fontsHashMap = project.fontsHashMap; - layerPromises.push(window.addExistingLayer(layerId, objects[layerId])); + layerPromises.push(window.addExistingAudioLayer(layerId, objects[layerId])); }); Promise.all(layerPromises).then(() => { window.layerOrder.set(project.layerOrder);