From adcd1ce848540fbb3a91b53b58c77888d5140b3b Mon Sep 17 00:00:00 2001 From: Aria Minaei Date: Fri, 3 Sep 2021 14:02:02 +0200 Subject: [PATCH] Implemented Sequence.attachAudio() --- theatre/core/src/sequences/TheatreSequence.ts | 95 ++++++++++++++ .../AudioPlaybackController.ts | 121 ++++++------------ 2 files changed, 135 insertions(+), 81 deletions(-) diff --git a/theatre/core/src/sequences/TheatreSequence.ts b/theatre/core/src/sequences/TheatreSequence.ts index d7d98e6..d161902 100644 --- a/theatre/core/src/sequences/TheatreSequence.ts +++ b/theatre/core/src/sequences/TheatreSequence.ts @@ -3,6 +3,23 @@ import {privateAPI, setPrivateAPI} from '@theatre/core/privateAPIs' import {defer} from '@theatre/shared/utils/defer' import type Sequence from './Sequence' import type {IPlaybackDirection, IPlaybackRange} from './Sequence' +import AudioPlaybackController from './playbackControllers/AudioPlaybackController' +import coreTicker from '@theatre/core/coreTicker' + +interface IAttachAudioArgs { + /** + * Either a URL to the audio file (eg "https://localhost/audio.mp3") or an instance of AudioBuffer + */ + source: string | AudioBuffer + /** + * An optional AudioContext. If not provided, one will be created. + */ + audioContext?: AudioContext + /** + * An AudioDestinationNode to feed the audio into. One will be created if not provided. + */ + destinationNode?: AudioDestinationNode +} export interface ISequence { readonly type: 'Theatre_Sequence_PublicAPI' @@ -43,6 +60,12 @@ export interface ISequence { * In a time-based sequence, this represents the current time. */ position: number + + /** + * + * @param args + */ + attachAudio(args: IAttachAudioArgs): Promise } export default class TheatreSequence implements ISequence { @@ -98,4 +121,76 @@ export default class TheatreSequence implements ISequence { set position(position: number) { privateAPI(this).position = position } + + async attachAudio(args: IAttachAudioArgs): Promise { + const {audioContext, destinationNode, decodedBuffer} = + await resolveAudioBuffer(args) + + const playbackController = new AudioPlaybackController( + coreTicker, + decodedBuffer, + audioContext, + destinationNode, + ) + + privateAPI(this).replacePlaybackController(playbackController) + } +} + +async function resolveAudioBuffer(args: IAttachAudioArgs): Promise<{ + decodedBuffer: AudioBuffer + audioContext: AudioContext + destinationNode: AudioDestinationNode +}> { + const audioContext = args.audioContext || new AudioContext() + + const decodedBufferDeferred = defer() + if (args.source instanceof AudioBuffer) { + decodedBufferDeferred.resolve(args.source) + } else if (typeof args.source !== 'string') { + throw new Error( + `Error validating arguments to sequence.attachAudio(). ` + + `args.source must either be a string or an instance of AudioBuffer.`, + ) + } else { + let fetchResponse + try { + fetchResponse = await fetch(args.source) + } catch (e) { + console.error(e) + throw new Error( + `Could not fetch '${args.source}'. Network error logged above.`, + ) + } + + let buffer + try { + buffer = await fetchResponse.arrayBuffer() + } catch (e) { + console.error(e) + throw new Error(`Could not read '${args.source}' as an arrayBuffer.`) + } + + audioContext.decodeAudioData( + buffer, + decodedBufferDeferred.resolve, + decodedBufferDeferred.reject, + ) + } + + let decodedBuffer + try { + decodedBuffer = await decodedBufferDeferred.promise + } catch (e) { + console.error(e) + throw new Error(`Could not decode ${args.source} as an audio file.`) + } + + const destinationNode = args.destinationNode || audioContext.destination + + return { + destinationNode, + audioContext, + decodedBuffer, + } } diff --git a/theatre/core/src/sequences/playbackControllers/AudioPlaybackController.ts b/theatre/core/src/sequences/playbackControllers/AudioPlaybackController.ts index d5c7c9f..4657b7c 100644 --- a/theatre/core/src/sequences/playbackControllers/AudioPlaybackController.ts +++ b/theatre/core/src/sequences/playbackControllers/AudioPlaybackController.ts @@ -64,15 +64,8 @@ export default class AudioPlaybackController implements IPlaybackController { this.playing = true const ticker = this._ticker - let lastTickerTime = ticker.time - const dur = range.end - range.start - const prevTime = this.getCurrentPosition() - - if (rate !== 1.0) { - throw new InvalidArgumentError( - `Audio-controlled sequences can only have a playbackRate of 1.0. ${rate} given.`, - ) - } + const startPos = this.getCurrentPosition() + const iterationLength = range.end - range.start if (direction !== 'normal') { throw new InvalidArgumentError( @@ -81,98 +74,64 @@ export default class AudioPlaybackController implements IPlaybackController { ) } - if (prevTime < range.start || prevTime > range.end) { + if (iterationCount !== 1) { + throw new InvalidArgumentError( + `Audio-controlled sequences can only have an iterationCount of 1 ` + + `'${iterationCount}' given.`, + ) + } + + if (startPos < range.start || startPos > range.end) { // if we're currently out of the range this._updatePositionInState(range.start) - } else if (prevTime === range.end) { + } else if (startPos === range.end) { // if we're currently at the very end of the range this._updatePositionInState(range.start) } - let countSoFar = 1 - const deferred = defer() const currentSource = this._audioContext.createBufferSource() currentSource.buffer = this._decodedBuffer currentSource.connect(this._mainGain) + currentSource.playbackRate.value = rate + const audioStartTimeInSeconds = this._audioContext.currentTime const wait = 0 - const timeToRangeEnd = range.end - prevTime - - if (iterationCount > 1) { - currentSource.loop = true - currentSource.loopStart = range.start - currentSource.loopEnd = range.end - } + const timeToRangeEnd = range.end - startPos currentSource.start( audioStartTimeInSeconds + wait, - prevTime - wait, - iterationCount === 1 ? wait + timeToRangeEnd : undefined, + startPos - wait, + wait + timeToRangeEnd, ) + const initialTickerTime = ticker.time + let initialElapsedPos = this.getCurrentPosition() - range.start + const totalPlaybackLength = iterationLength * iterationCount - const tick = (tickerTime: number) => { - const lastTime = this.getCurrentPosition() - const timeDiff = (tickerTime - lastTickerTime) * rate - lastTickerTime = tickerTime - /* - * I don't know why exactly this happens, but every 10 times or so, the first sequence.play({iterationCount: 1}), - * the first call of tick() will have a timeDiff < 0. - * This might be because of Spectre mitigation (they randomize performance.now() a bit), or it could be that - * I'm using performance.now() the wrong way. - * Anyway, this seems like a working fix for it: - */ - if (timeDiff < 0) { + const tick = (currentTickerTime: number) => { + const elapsedTickerTime = Math.max( + currentTickerTime - initialTickerTime, + 0, + ) + const elapsedTickerTimeInSeconds = elapsedTickerTime / 1000 + + const elapsedPos = Math.min( + elapsedTickerTimeInSeconds * rate + initialElapsedPos, + totalPlaybackLength, + ) + + if (elapsedPos !== totalPlaybackLength) { + let currentIterationPos = + ((elapsedPos / iterationLength) % 1) * iterationLength + + this._updatePositionInState(currentIterationPos) requestNextTick() - return - } - const newTime = lastTime + timeDiff - - if (newTime < range.start) { - if (countSoFar === iterationCount) { - this._updatePositionInState(range.start) - this.playing = false - cleanup() - deferred.resolve(true) - return - } else { - countSoFar++ - const diff = (range.start - newTime) % dur - - this._updatePositionInState(range.start + diff) - requestNextTick() - return - } - } else if (newTime === range.end) { - this._updatePositionInState(range.end) - if (countSoFar === iterationCount) { - this.playing = false - cleanup() - deferred.resolve(true) - return - } - requestNextTick() - return - } else if (newTime > range.end) { - if (countSoFar === iterationCount) { - this._updatePositionInState(range.end) - this.playing = false - cleanup() - deferred.resolve(true) - return - } else { - countSoFar++ - const diff = (newTime - range.end) % dur - this._updatePositionInState(range.start + diff) - - requestNextTick() - return - } } else { - this._updatePositionInState(newTime) - requestNextTick() - return + this._updatePositionInState(range.end) + this.playing = false + cleanup() + deferred.resolve(true) } }