Implemented Sequence.attachAudio()

theatre/core/src/sequences/TheatreSequence.ts

@@ -3,6 +3,23 @@ import {privateAPI, setPrivateAPI} from '@theatre/core/privateAPIs'
 import {defer} from '@theatre/shared/utils/defer'
 import type Sequence from './Sequence'
 import type {IPlaybackDirection, IPlaybackRange} from './Sequence'
+import AudioPlaybackController from './playbackControllers/AudioPlaybackController'
+import coreTicker from '@theatre/core/coreTicker'
+
+interface IAttachAudioArgs {
+  /**
+   * Either a URL to the audio file (eg "https://localhost/audio.mp3") or an instance of AudioBuffer
+   */
+  source: string | AudioBuffer
+  /**
+   * An optional AudioContext. If not provided, one will be created.
+   */
+  audioContext?: AudioContext
+  /**
+   * An AudioDestinationNode to feed the audio into. One will be created if not provided.
+   */
+  destinationNode?: AudioDestinationNode
+}
 
 export interface ISequence {
   readonly type: 'Theatre_Sequence_PublicAPI'
@@ -43,6 +60,12 @@ export interface ISequence {
    * In a time-based sequence, this represents the current time.
    */
   position: number
+
+  /**
+   *
+   * @param args
+   */
+  attachAudio(args: IAttachAudioArgs): Promise<void>
 }
 
 export default class TheatreSequence implements ISequence {
@@ -98,4 +121,76 @@ export default class TheatreSequence implements ISequence {
   set position(position: number) {
     privateAPI(this).position = position
   }
+
+  async attachAudio(args: IAttachAudioArgs): Promise<void> {
+    const {audioContext, destinationNode, decodedBuffer} =
+      await resolveAudioBuffer(args)
+
+    const playbackController = new AudioPlaybackController(
+      coreTicker,
+      decodedBuffer,
+      audioContext,
+      destinationNode,
+    )
+
+    privateAPI(this).replacePlaybackController(playbackController)
+  }
+}
+
+async function resolveAudioBuffer(args: IAttachAudioArgs): Promise<{
+  decodedBuffer: AudioBuffer
+  audioContext: AudioContext
+  destinationNode: AudioDestinationNode
+}> {
+  const audioContext = args.audioContext || new AudioContext()
+
+  const decodedBufferDeferred = defer<AudioBuffer>()
+  if (args.source instanceof AudioBuffer) {
+    decodedBufferDeferred.resolve(args.source)
+  } else if (typeof args.source !== 'string') {
+    throw new Error(
+      `Error validating arguments to sequence.attachAudio(). ` +
+        `args.source must either be a string or an instance of AudioBuffer.`,
+    )
+  } else {
+    let fetchResponse
+    try {
+      fetchResponse = await fetch(args.source)
+    } catch (e) {
+      console.error(e)
+      throw new Error(
+        `Could not fetch '${args.source}'. Network error logged above.`,
+      )
+    }
+
+    let buffer
+    try {
+      buffer = await fetchResponse.arrayBuffer()
+    } catch (e) {
+      console.error(e)
+      throw new Error(`Could not read '${args.source}' as an arrayBuffer.`)
+    }
+
+    audioContext.decodeAudioData(
+      buffer,
+      decodedBufferDeferred.resolve,
+      decodedBufferDeferred.reject,
+    )
+  }
+
+  let decodedBuffer
+  try {
+    decodedBuffer = await decodedBufferDeferred.promise
+  } catch (e) {
+    console.error(e)
+    throw new Error(`Could not decode ${args.source} as an audio file.`)
+  }
+
+  const destinationNode = args.destinationNode || audioContext.destination
+
+  return {
+    destinationNode,
+    audioContext,
+    decodedBuffer,
+  }
 }

theatre/core/src/sequences/playbackControllers/AudioPlaybackController.ts

@@ -64,15 +64,8 @@ export default class AudioPlaybackController implements IPlaybackController {
     this.playing = true
 
     const ticker = this._ticker
-    let lastTickerTime = ticker.time
+    const startPos = this.getCurrentPosition()
-    const dur = range.end - range.start
+    const iterationLength = range.end - range.start
-    const prevTime = this.getCurrentPosition()
-
-    if (rate !== 1.0) {
-      throw new InvalidArgumentError(
-        `Audio-controlled sequences can only have a playbackRate of 1.0. ${rate} given.`,
-      )
-    }
 
     if (direction !== 'normal') {
       throw new InvalidArgumentError(
@@ -81,98 +74,64 @@ export default class AudioPlaybackController implements IPlaybackController {
       )
     }
 
-    if (prevTime < range.start || prevTime > range.end) {
+    if (iterationCount !== 1) {
+      throw new InvalidArgumentError(
+        `Audio-controlled sequences can only have an iterationCount of 1 ` +
+          `'${iterationCount}' given.`,
+      )
+    }
+
+    if (startPos < range.start || startPos > range.end) {
       // if we're currently out of the range
       this._updatePositionInState(range.start)
-    } else if (prevTime === range.end) {
+    } else if (startPos === range.end) {
       // if we're currently at the very end of the range
       this._updatePositionInState(range.start)
     }
 
-    let countSoFar = 1
-
     const deferred = defer<boolean>()
 
     const currentSource = this._audioContext.createBufferSource()
     currentSource.buffer = this._decodedBuffer
     currentSource.connect(this._mainGain)
+    currentSource.playbackRate.value = rate
+
     const audioStartTimeInSeconds = this._audioContext.currentTime
     const wait = 0
-    const timeToRangeEnd = range.end - prevTime
+    const timeToRangeEnd = range.end - startPos
-
-    if (iterationCount > 1) {
-      currentSource.loop = true
-      currentSource.loopStart = range.start
-      currentSource.loopEnd = range.end
-    }
 
     currentSource.start(
       audioStartTimeInSeconds + wait,
-      prevTime - wait,
+      startPos - wait,
-      iterationCount === 1 ? wait + timeToRangeEnd : undefined,
+      wait + timeToRangeEnd,
+    )
+    const initialTickerTime = ticker.time
+    let initialElapsedPos = this.getCurrentPosition() - range.start
+    const totalPlaybackLength = iterationLength * iterationCount
+
+    const tick = (currentTickerTime: number) => {
+      const elapsedTickerTime = Math.max(
+        currentTickerTime - initialTickerTime,
+        0,
+      )
+      const elapsedTickerTimeInSeconds = elapsedTickerTime / 1000
+
+      const elapsedPos = Math.min(
+        elapsedTickerTimeInSeconds * rate + initialElapsedPos,
+        totalPlaybackLength,
       )
 
-    const tick = (tickerTime: number) => {
+      if (elapsedPos !== totalPlaybackLength) {
-      const lastTime = this.getCurrentPosition()
+        let currentIterationPos =
-      const timeDiff = (tickerTime - lastTickerTime) * rate
+          ((elapsedPos / iterationLength) % 1) * iterationLength
-      lastTickerTime = tickerTime
-      /*
-       * I don't know why exactly this happens, but every 10 times or so, the first sequence.play({iterationCount: 1}),
-       * the first call of tick() will have a timeDiff < 0.
-       * This might be because of Spectre mitigation (they randomize performance.now() a bit), or it could be that
-       * I'm using performance.now() the wrong way.
-       * Anyway, this seems like a working fix for it:
-       */
-      if (timeDiff < 0) {
-        requestNextTick()
-        return
-      }
-      const newTime = lastTime + timeDiff
 
-      if (newTime < range.start) {
+        this._updatePositionInState(currentIterationPos)
-        if (countSoFar === iterationCount) {
+        requestNextTick()
-          this._updatePositionInState(range.start)
-          this.playing = false
-          cleanup()
-          deferred.resolve(true)
-          return
       } else {
-          countSoFar++
-          const diff = (range.start - newTime) % dur
-
-          this._updatePositionInState(range.start + diff)
-          requestNextTick()
-          return
-        }
-      } else if (newTime === range.end) {
-        this._updatePositionInState(range.end)
-        if (countSoFar === iterationCount) {
-          this.playing = false
-          cleanup()
-          deferred.resolve(true)
-          return
-        }
-        requestNextTick()
-        return
-      } else if (newTime > range.end) {
-        if (countSoFar === iterationCount) {
         this._updatePositionInState(range.end)
         this.playing = false
         cleanup()
         deferred.resolve(true)
-          return
-        } else {
-          countSoFar++
-          const diff = (newTime - range.end) % dur
-          this._updatePositionInState(range.start + diff)
-
-          requestNextTick()
-          return
-        }
-      } else {
-        this._updatePositionInState(newTime)
-        requestNextTick()
-        return
       }
     }