From 32778138bad88960f9624cafe95c01f89ffd4cbf Mon Sep 17 00:00:00 2001 From: themancalledjakob Date: Fri, 12 Apr 2024 21:33:37 +0200 Subject: [PATCH] fix playing multiple recordings with audio and more dependencies hashes: openFrameworks d78075f4bca6be2a2533c6e51a75cc1f18404501 ofxMsdfgen e14da13d02c4dff04fb69d7923469f606924e6c3 ofxGPUFont d482bb7cbdf6b296fa4ab5abcf73fb5ff8c8b239 ofxVariableLab 0b5f9bdebc1e5550621957e73c040c258ec6317b ofxProfiler a868e34fa1a79189dd4fbdede2938e308535e5e8 theatre 78a67ee6650d846fe5cc4770770b1511328033e6 --- bin/em/variabletime/web/js/audio.js | 24 +----- bin/em/variabletime/web/js/audioLayer.js | 3 + bin/em/variabletime/web/js/audioPlayer.js | 95 ++++++++++++++++------- bin/em/variabletime/web/js/main.js | 9 ++- bin/em/variabletime/web/js/moduleFS.js | 30 ++++++- bin/em/variabletime/web/js/utils.js | 16 ++-- 6 files changed, 113 insertions(+), 64 deletions(-) diff --git a/bin/em/variabletime/web/js/audio.js b/bin/em/variabletime/web/js/audio.js index a261e68..8ee4375 100644 --- a/bin/em/variabletime/web/js/audio.js +++ b/bin/em/variabletime/web/js/audio.js @@ -105,18 +105,14 @@ const MicrophoneRecorder = function() { }; this.startListener = (event) => { - console.log('microphoneRecorder startl received', event); if (event.detail === record.possibleStates.RECORDING) { this.start(); - console.log('microphoneRecorder start', filenameWithoutExtension); window.removeEventListener('record', this.startListener); } }; this.stopListener = (event) => { - console.log('microphoneRecorder stopl received', event); if (event.detail === record.possibleStates.STOPPING_RECORDING) { - console.log('microphoneRecorder stop', filenameWithoutExtension); this.stop().then((filename) => { // be happy }); @@ -991,25 +987,7 @@ const Audio = function(tp, record) { audioElement.classList.add(toCssClass(`audio_file${file}`)); document.querySelector('body').append(audioElement); - const arr = FS.readFile(`${config.fs.idbfsAudioDir}/${file}`); - let type = 'audio/wav'; - const filesplit = file.split('.'); - const extension = filesplit[filesplit.length - 1]; - if (extension === 'wav') { - type = 'audio/wav'; - } else if (extension === 'mp3') { - type = 'audio/mpeg'; - } else if (extension === 'ogg') { - type = 'audio/ogg'; - } else if (extension === 'webm') { - type = 'audio/webm'; - } - - const src = URL.createObjectURL( - new Blob([arr], { - type - }) - ); + const src = moduleFS.objectUrl(`${config.fs.idbfsAudioDir}/${file}`); audioElement.src = src; audioElement.loop = true; diff --git a/bin/em/variabletime/web/js/audioLayer.js b/bin/em/variabletime/web/js/audioLayer.js index ce3ac76..2fc67b8 100644 --- a/bin/em/variabletime/web/js/audioLayer.js +++ b/bin/em/variabletime/web/js/audioLayer.js @@ -32,14 +32,17 @@ const AudioLayer = function(tp, audioLayerID, values = false, autoInit = true) { keyframes: [{ position: -1, value: false, + type: 'hold', }, { position: start, value: true, + type: 'hold', }, { position: stop, value: false, + type: 'hold', }, ], }]; diff --git a/bin/em/variabletime/web/js/audioPlayer.js b/bin/em/variabletime/web/js/audioPlayer.js index 87424c4..802b70a 100644 --- a/bin/em/variabletime/web/js/audioPlayer.js +++ b/bin/em/variabletime/web/js/audioPlayer.js @@ -1,48 +1,81 @@ import { clone, + sanitizeTheatreKey, } from './utils.js'; const AudioPlayer = function() { const audioElements = []; let updateInterval = false; let updateInterval_ms = 10; - this.add = (layer, propTitle, time, file) => { + + const getAudioID = (filename) => { + let audioID = sanitizeTheatreKey(filename); + const duplicate = audioElements.findIndex((e) => e.audioID === audioID) !== -1; + if (duplicate) { + let index = 0; + let unique = false; + while (!unique) { // uuuh, while loops.. + const newAudioID = `${audioID}_${index}`; + if (audioElements.findIndex((e) => e.audioID === newAudioID) === -1) { + audioID = newAudioID; + unique = true; + } else { + index++; + } + } + return audioID + } + return audioID; + }; + + this.addMany = (manyAudioElements) => { + manyAudioElements.forEach((e) => { + this.add(e.audioID, e.layerID, e.propTitle, e.file); + }); + }; + + this.add = (audioID, layer, propTitle, file, startTime) => { const layerID = typeof layer === 'string' ? layer : layer.id(); propTitle = Array.isArray(propTitle) ? propTitle.join('.') : propTitle; - console.log('AudioPlayer::add',{layerID, propTitle, time, file}); - const index = audioElements.findIndex((e) => e.layerID === layerID && e.propTitle === propTitle); - if (index === -1) { - const audioDomElement = document.createElement('audio'); - audioDomElement.classList.add('invisible'); - audioDomElement.classList.add('audio_file'); - audioDomElement.src = audio.audioSourceCombos[file].audioElement.src; - audioElements.push({ - layerID, propTitle, audioDomElement, time, file - }); - } else { - audioElements[index].src = audio.audioSourceCombos[file].audioDomElement.src; - audioElements[index].time = time; + + if (!audioID) { + audioID = getAudioID(file); } + const audioDomElement = new Audio(moduleFS.objectUrl(`${config.fs.idbfsAudioDir}/${file}`)); + audioDomElement.loop = true; + audioDomElement.load(); + //const audioDomElement = document.createElement('audio'); + //audioDomElement.classList.add('invisible'); + //audioDomElement.classList.add('audio_file'); + //audioDomElement.classList.add('audioPlayer'); + //audioDomElement.src = moduleFS.objectUrl(`${config.fs.idbfsAudioDir}/${file}`); + audioDomElement.loop = true; + audioElements.push({ + audioID, + layerID, + propTitle, + audioDomElement, + file, + startTime + }); }; this.update = () => { audioElements.forEach((audioElement, i) => { if (tp.isPlaying() && !record.isRecording()) { - const diff = tp.sheet.sequence.position - audioElement.time; - if (diff >= 0) { - if (audioElement.audioDomElement.paused) { + const shouldBePlaying = getAudioLayer().theatreObject.value[audioElement.audioID]; + if (shouldBePlaying && audioElement.audioDomElement.paused) { + // sequence position is always greater than startTime + // this is true, as it is written + const diff = tp.sheet.sequence.position - audioElement.startTime; audioElement.audioDomElement.currentTime = diff; audioElement.audioDomElement.play(); - console.log('play audioElement ', audioElement.file, audioElement.propTitle, i); - } - } else if(!audioElement.audioDomElement.paused) { + } else if (!shouldBePlaying && !audioElement.audioDomElement.paused) { audioElement.audioDomElement.pause(); audioElement.audioDomElement.currentTime = 0; - console.log('pause audioElement ', audioElement.file, audioElement.propTitle, i); } } else if (!audioElement.audioDomElement.paused) { audioElement.audioDomElement.pause(); audioElement.audioDomElement.currentTime = 0; - console.log('pausé audioElement ', audioElement.file, audioElement.propTitle, i); } }); }; @@ -54,11 +87,11 @@ const AudioPlayer = function() { }, updateInterval_ms); }; let hot = false; - let time = false; + let startTime = false; this.listener = (event) => { if (event.detail === record.possibleStates.RECORDING) { hot = clone(record.getHot()); - time = tp.sheet.sequence.position; + startTime = tp.sheet.sequence.position; const layerIDs = Object.keys(hot); layerIDs.forEach((layerID) => { const propTitles = Object.keys(hot[layerID]); @@ -68,8 +101,10 @@ const AudioPlayer = function() { if (m.source === 'microphone') { const waitForMicrophoneListener = (event) => { if (event.detail.fileIsRead) { - this.add(layerID, propTitle, time, event.detail.filename); - addAudioLayerTrack(event.detail.filename, time, tp.sheet.sequence.position); + const filename = event.detail.filename; + const audioID = getAudioID(filename); + this.add(audioID, layerID, propTitle, filename, startTime); + addAudioLayerTrack(audioID, startTime, tp.sheet.sequence.position); window.removeEventListener( 'microphoneRecorder', waitForMicrophoneListener); @@ -91,9 +126,13 @@ const AudioPlayer = function() { const m = audio.getSavedMapping()[layerID][propTitle]; if (m.addToTimeline) { if (m.source === 'microphone') { + // handled above, as we need to wait for the microphone recording to be written and read + // and we rather start waiting, when it has definitely not happened yet } else { - this.add(layerID, propTitle, time, m.source); - addAudioLayerTrack(m.source, time, tp.sheet.sequence.position); + const filename = m.source; + const audioID = getAudioID(filename); + this.add(audioID, layerID, propTitle, filename, startTime); + addAudioLayerTrack(audioID, startTime, tp.sheet.sequence.position); } } }); diff --git a/bin/em/variabletime/web/js/main.js b/bin/em/variabletime/web/js/main.js index 1eaa4ba..04d2c07 100644 --- a/bin/em/variabletime/web/js/main.js +++ b/bin/em/variabletime/web/js/main.js @@ -105,7 +105,7 @@ const getAbout = () => { const buttonExp = textParent.querySelector(".expandText"); if (buttonExp === null) { - console.error("Could not find .expandText within .textParent"); + console.error("Main::getAbout","Could not find .expandText within .textParent"); return; } @@ -463,6 +463,11 @@ window.getAudioLayers = () => { return audioLayers; }; +// for now we changed the design to have only one audio layer +window.getAudioLayer = () => { + return audioLayers.length > 0 ? audioLayers[0] : false; +}; + window.getLayer = (layerID = tp.studio.selection[0].address.objectKey) => { if (layerID === 'artboard') { return artboard; @@ -590,7 +595,6 @@ const addAudioLayer = (filename = false) => { } return `audio-${index}`; })(); - console.log('adding', audioLayerID); const audioLayer = new AudioLayer(tp, audioLayerID, filename, false); layersById[audioLayerID] = audioLayer; audioLayers.push(audioLayer); @@ -716,7 +720,6 @@ const initPanels = () => { moduleFS .save(file) .then(() => { - console.log('ermh... done uploading?', file); audio.readAudioFiles(); }); }); diff --git a/bin/em/variabletime/web/js/moduleFS.js b/bin/em/variabletime/web/js/moduleFS.js index ff99589..fbb56c2 100644 --- a/bin/em/variabletime/web/js/moduleFS.js +++ b/bin/em/variabletime/web/js/moduleFS.js @@ -60,7 +60,6 @@ const ModuleFS = function() { return new Promise((resolve) => { if (file.type.indexOf('font') >= 0 || file.hasOwnProperty('isFont') && file.isFont === true) { var uint8View = new Uint8Array(file.arrayBuffer); - console.log('trying to save the font file, file, uint8View', file, uint8View); if (!FS.analyzePath(`${config.fs.idbfsFontDir}/${file.name}`).exists) { FS.createDataFile(config.fs.idbfsFontDir, file.name, uint8View, true, true); } @@ -71,7 +70,6 @@ const ModuleFS = function() { } else if (file.type.indexOf('zip') >= 0 || file.hasOwnProperty('isZip') && file.isZip === true) { var uint8View = new Uint8Array(file.arrayBuffer); var filePath = `${config.fs.idbfsTmpDir}/${file.name}`; - console.log(filePath); if (!FS.analyzePath(filePath).exists) { FS.createDataFile(config.fs.idbfsTmpDir, file.name, uint8View, true, true); } @@ -81,7 +79,6 @@ const ModuleFS = function() { }); } else if (file.type.indexOf('audio') === 0) { var uint8View = new Uint8Array(file.arrayBuffer); - console.log('trying to save the audio file, file, uint8View', file, uint8View); if (!FS.analyzePath(`${config.fs.idbfsAudioDir}/${file.name}`).exists) { FS.createDataFile(config.fs.idbfsAudioDir, file.name, uint8View, true, true); this.syncfs(MODE_WRITE_TO_PERSISTENT) @@ -114,6 +111,33 @@ const ModuleFS = function() { resolve(false); } }; + + const objectUrls = {}; + this.objectUrl = (filePath) => { + if (typeof objectUrls[filePath] === 'undefined') { + const arr = FS.readFile(filePath); + let type = 'audio/wav'; + const filesplit = filePath.split('.'); + const extension = filesplit[filesplit.length - 1]; + + // is there a way to get mime type without guessing by extension? + if (extension === 'wav') { + type = 'audio/wav'; + } else if (extension === 'mp3') { + type = 'audio/mpeg'; + } else if (extension === 'ogg') { + type = 'audio/ogg'; + } else if (extension === 'webm') { + type = 'audio/webm'; + } + objectUrls[filePath] = URL.createObjectURL( + new Blob([arr], { + type + }) + ); + } + return objectUrls[filePath]; + }; }; export { diff --git a/bin/em/variabletime/web/js/utils.js b/bin/em/variabletime/web/js/utils.js index ea2eee9..93d8e98 100644 --- a/bin/em/variabletime/web/js/utils.js +++ b/bin/em/variabletime/web/js/utils.js @@ -457,17 +457,19 @@ const toCssClass = (text, prefix = '') => { ; return cssClass; }; -const sanitizeTheatreKey = (key) => { +const sanitizeTheatreKey = (key, removeExtension = true) => { let theatreKey = key; - theatreKey = theatreKey.split('.'); - if (theatreKey.length > 1) { - theatreKey.pop(); + if (removeExtension) { + theatreKey = theatreKey.split('.'); + if (theatreKey.length > 1) { + theatreKey.pop(); + } + theatreKey = theatreKey.join(''); } - theatreKey = theatreKey.join(''); - if (theatreKey.substr(0,1).match(/^([0-9])$/i)) { + if (theatreKey.substr(0, 1).match(/^([0-9])$/i)) { theatreKey = `t_${theatreKey}`; } - return theatreKey.replace(/[^a-zA-Z0-9_]/g,""); + return theatreKey.replace(/[^a-zA-Z0-9_]/g, ""); }; const renameProperty = (o, old_key, new_key) => {