dev: Build playground files much faster and add watch

packages/playground/devEnv/.gitignore

@@ -0,0 +1 @@
+build.compiled.js

packages/playground/devEnv/build.ts

@@ -3,15 +3,17 @@ import {writeFile, readFile} from 'fs/promises'
 import path from 'path'
 import type {BuildOptions} from 'esbuild'
 import esbuild from 'esbuild'
-import type {IncomingMessage, ServerResponse} from 'http'
-import {definedGlobals} from '../../../theatre/devEnv/buildUtils'
+import {definedGlobals} from '../../../theatre/devEnv/definedGlobals'
 import {mapValues} from 'lodash-es'
-import {createServer, request} from 'http'
-import {spawn} from 'child_process'
 import React from 'react'
 import {renderToStaticMarkup} from 'react-dom/server'
 import {Home} from './Home'
-import type {Server} from 'net'
+import {timer} from './timer'
+import {openForOS} from './openForOS'
+import {tryMultiplePorts} from './tryMultiplePorts'
+import {createProxyServer} from './createProxyServer'
+import {createEsbuildLiveReloadTools} from './createEsbuildLiveReloadTools'
+import {createServerForceClose} from './createServerForceClose'
 
 const playgroundDir = (folder: string) => path.join(__dirname, '..', folder)
 const buildDir = playgroundDir('build')
@@ -19,278 +21,178 @@ const sharedDir = playgroundDir('src/shared')
 const personalDir = playgroundDir('src/personal')
 const testDir = playgroundDir('src/tests')
 
-const dev = process.argv.find((arg) => ['--dev', '-d'].includes(arg)) != null
-const defaultPort = 8080
+export async function start(options: {
+  dev: boolean
+  findAvailablePort: boolean
+  openBrowser: boolean
+  waitBeforeStartingServer?: Promise<void>
+  /** defaults to 8080 */
+  defaultPort?: number
+}): Promise<{stop(): Promise<void>}> {
+  const defaultPort = options.defaultPort ?? 8080
 
-const liveReload =
-  (dev || undefined) &&
-  ((): {
-    handleRequest(req: IncomingMessage, res: ServerResponse): boolean
-    hasOpenConnections(): boolean
-    esbuildBanner: esbuild.BuildOptions['banner']
-    esbuildWatch: esbuild.WatchMode
-  } => {
-    const openResponses: ServerResponse[] = []
-    return {
-      handleRequest(req, res) {
-        // If special /esbuild url requested, subscribe clients to changes
-        if (req.url === '/esbuild') {
-          openResponses.push(
-            res.writeHead(200, {
-              'Content-Type': 'text/event-stream',
-              'Cache-Control': 'no-cache',
-              Connection: 'keep-alive',
-            }),
-          )
-          return true
-        }
-        return false
-      },
-      hasOpenConnections() {
-        return openResponses.length > 0
-      },
-      esbuildBanner: {
-        js: ' (() => new EventSource("/esbuild").onmessage = () => location.reload())();',
-      },
-      esbuildWatch: {
-        onRebuild(error) {
-          if (!error) {
-            console.error('Reloading...')
-            // Notify clients on rebuild
-            openResponses.forEach((res) => res.write('data: update\n\n'))
-            openResponses.length = 0
-          } else {
-            console.error('Rebuild had errors...')
-          }
-        },
-      },
-    }
-  })()
+  const liveReload = options.dev ? createEsbuildLiveReloadTools() : undefined
 
-type Groups = {
-  [group: string]: {
-    [module: string]: {
-      entryDir: string
-      outDir: string
-    }
-  }
-}
-
-// Collect all entry directories per module per group
-const groups: Groups = Object.fromEntries(
-  [sharedDir, personalDir, testDir]
-    .map((groupDir) => {
-      try {
-        return [
-          path.basename(groupDir),
-          Object.fromEntries(
-            readdirSync(groupDir).map((moduleDirName) => [
-              path.basename(moduleDirName),
-              {
-                entryDir: path.join(groupDir, moduleDirName),
-                outDir: path.join(
-                  buildDir,
-                  path.basename(groupDir),
-                  moduleDirName,
-                ),
-              },
-            ]),
-          ),
-        ]
-      } catch (e) {
-        // If the group dir doesn't exist, we just set its entry to undefined
-        return [path.basename(groupDir), undefined]
+  type Groups = {
+    [group: string]: {
+      [module: string]: {
+        entryDir: string
+        outDir: string
       }
-    })
-    // and then filter it out.
-    .filter((entry) => entry[1] !== undefined),
-)
-
-// Collect all entry files
-const entryPoints = Object.values(groups)
-  .flatMap((group) => Object.values(group))
-  .map((module) => path.join(module.entryDir, 'index.tsx'))
-
-// Collect all output directories
-const outDirs = Object.values(groups).flatMap((group) =>
-  Object.values(group).map((module) => module.outDir),
-)
-
-// Render home page contents
-const homeHtml = renderToStaticMarkup(
-  React.createElement(Home, {
-    groups: mapValues(groups, (group) => Object.keys(group)),
-  }),
-)
-
-const config: BuildOptions = {
-  entryPoints,
-  bundle: true,
-  sourcemap: true,
-  outdir: playgroundDir('build'),
-  target: ['firefox88'],
-  loader: {
-    '.png': 'file',
-    '.glb': 'file',
-    '.gltf': 'file',
-    '.svg': 'dataurl',
-  },
-  define: {
-    ...definedGlobals,
-    'window.__IS_VISUAL_REGRESSION_TESTING': 'true',
-  },
-  banner: liveReload?.esbuildBanner,
-  watch: liveReload?.esbuildWatch,
-}
-
-esbuild
-  .build(config)
-  .catch((err) => {
-    // if in dev mode, permit continuing to watch even if there was an error
-    return dev ? Promise.resolve() : Promise.reject(err)
-  })
-  .then(async () => {
-    // Read index.html template
-    const index = await readFile(path.join(__dirname, 'index.html'), 'utf8')
-    await Promise.all([
-      // Write home page
-      writeFile(
-        path.join(buildDir, 'index.html'),
-        index.replace(/<body>[\s\S]*<\/body>/, `<body>${homeHtml}</body>`),
-      ),
-      // Write module pages
-      ...outDirs.map((outDir) =>
-        writeFile(
-          path.join(outDir, 'index.html'),
-          // Substitute %ENTRYPOINT% placeholder with the output file path
-          index.replace(
-            '%ENTRYPOINT%',
-            path.join('/', path.relative(buildDir, outDir), 'index.js'),
-          ),
-        ),
-      ),
-    ])
-  })
-  .catch((err) => {
-    console.log(err)
-    return process.exit(1)
-  })
-  .then(async () => {
-    // Only start dev server in dev, otherwise just run build and that's it
-    if (!dev) {
-      return
     }
-
-    // We start ESBuild serve with no build config because it doesn't need to build
-    // anything, we are already using ESBuild watch.
-    const {port: esbuildPort} = await esbuild.serve(
-      {servedir: playgroundDir('build')},
-      {},
-    )
-
-    const proxyServer = createServer((req, res) => {
-      const {url, method, headers} = req
-      if (liveReload?.handleRequest(req, res)) {
-        return
-      }
-
-      // Otherwise forward requests to ESBuild server
-      req.pipe(
-        request(
-          {
-            hostname: '0.0.0.0',
-            port: esbuildPort,
-            path: url,
-            method,
-            headers,
-          },
-          (prxRes) => {
-            res.writeHead(prxRes.statusCode!, prxRes.headers)
-            prxRes.pipe(res, {end: true})
-          },
-        ),
-        {end: true},
-      )
-    })
-
-    const isCI = process.env.CI
-    const portTries = isCI ? 1 : 10
-    const portChosen = await tryMultiplePorts(
-      defaultPort,
-      portTries,
-      proxyServer,
-    )
-
-    const hostedAt = `http://localhost:${portChosen}`
-
-    console.log('Playground running at', hostedAt)
-
-    // If not in CI, try to spawn a browser
-    if (!isCI) {
-      setTimeout(() => {
-        if (!liveReload?.hasOpenConnections()) openForOS(hostedAt)
-      }, 1000)
-    }
-  })
-
-function openForOS(hostedAt: string) {
-  const open = {
-    darwin: ['open'],
-    linux: ['xdg-open'],
-    win32: ['cmd', '/c', 'start'],
   }
-  const platform = process.platform as keyof typeof open
-  if (open[platform]) {
-    spawn(open[platform][0], [...open[platform].slice(1), hostedAt])
-  } else {
-    console.error(
-      `Failed to open (${hostedAt}) for unconfigured platform (${platform})`,
-    )
-  }
-}
 
-async function tryMultiplePorts(
-  port: number,
-  tries: number,
-  server: Server,
-): Promise<number> {
-  let portToTry = port
-  let firstError = null
-  let lastError = null
-
-  while (portToTry < port + tries) {
-    try {
-      await new Promise((res, rej) => {
-        const onListening = () => (rm(), res(true))
-        const onError = () => (rm(), rej())
-        const rm = () => {
-          server.off('error', onError)
-          server.off('listening', onListening)
+  // Collect all entry directories per module per group
+  const groups: Groups = Object.fromEntries(
+    [sharedDir, personalDir, testDir]
+      .map((groupDir) => {
+        try {
+          return [
+            path.basename(groupDir),
+            Object.fromEntries(
+              readdirSync(groupDir).map((moduleDirName) => [
+                path.basename(moduleDirName),
+                {
+                  entryDir: path.join(groupDir, moduleDirName),
+                  outDir: path.join(
+                    buildDir,
+                    path.basename(groupDir),
+                    moduleDirName,
+                  ),
+                },
+              ]),
+            ),
+          ]
+        } catch (e) {
+          // If the group dir doesn't exist, we just set its entry to undefined
+          return [path.basename(groupDir), undefined]
         }
-
-        server
-          .listen(portToTry)
-          .on('listening', onListening)
-          .on('error', onError)
       })
+      // and then filter it out.
+      .filter((entry) => entry[1] !== undefined),
+  )
 
-      firstError = null
-      lastError = null
-      break // found a working port
-    } catch (err) {
-      if (!firstError) firstError = err
-      lastError = err
-      portToTry += 1
+  // Collect all entry files
+  const entryPoints = Object.values(groups)
+    .flatMap((group) => Object.values(group))
+    .map((module) => path.join(module.entryDir, 'index.tsx'))
+
+  // Collect all output directories
+  const outDirs = Object.values(groups).flatMap((group) =>
+    Object.values(group).map((module) => module.outDir),
+  )
+
+  // Render home page contents
+  const homeHtml = renderToStaticMarkup(
+    React.createElement(Home, {
+      groups: mapValues(groups, (group) => Object.keys(group)),
+    }),
+  )
+
+  const _initialBuild = timer('esbuild initial playground entry point builds')
+
+  const esbuildConfig: BuildOptions = {
+    entryPoints,
+    bundle: true,
+    sourcemap: true,
+    outdir: buildDir,
+    target: ['firefox88'],
+    loader: {
+      '.png': 'file',
+      '.glb': 'file',
+      '.gltf': 'file',
+      '.svg': 'dataurl',
+    },
+    define: {
+      ...definedGlobals,
+      'window.__IS_VISUAL_REGRESSION_TESTING': 'true',
+    },
+    banner: liveReload?.esbuildBanner,
+    watch: liveReload?.esbuildWatch,
+  }
+
+  let esbuildWatchStop: undefined | (() => void)
+
+  await esbuild
+    .build(esbuildConfig)
+    .finally(() => _initialBuild.stop())
+    .catch((err) => {
+      // if in dev mode, permit continuing to watch even if there was an error
+      return options.dev ? Promise.resolve() : Promise.reject(err)
+    })
+    .then(async (buildResult) => {
+      esbuildWatchStop = buildResult?.stop
+      // Read index.html template
+      const index = await readFile(path.join(__dirname, 'index.html'), 'utf8')
+      await Promise.all([
+        // Write home page
+        writeFile(
+          path.join(buildDir, 'index.html'),
+          index.replace(/<body>[\s\S]*<\/body>/, `<body>${homeHtml}</body>`),
+          'utf-8',
+        ),
+        // Write module pages
+        ...outDirs.map((outDir) =>
+          writeFile(
+            path.join(outDir, 'index.html'),
+            // Substitute %ENTRYPOINT% placeholder with the output file path
+            index.replace(
+              '%ENTRYPOINT%',
+              path.join('/', path.relative(buildDir, outDir), 'index.js'),
+            ),
+            'utf-8',
+          ),
+        ),
+      ])
+    })
+    .catch((err) => {
+      console.error(err)
+      return process.exit(1)
+    })
+
+  // Only start dev server in dev, otherwise just run build and that's it
+  if (!options.dev) {
+    return {
+      stop() {
+        esbuildWatchStop?.()
+        return Promise.resolve()
+      },
     }
   }
 
-  if (firstError) {
-    console.error(firstError)
-    console.error(lastError)
-    throw new Error(
-      `Failed to find port starting at ${port} with ${tries} tries.`,
-    )
+  await options.waitBeforeStartingServer
+
+  // We start ESBuild serve with no build config because it doesn't need to build
+  // anything, we are already using ESBuild watch.
+  /** See https://esbuild.github.io/api/#serve-return-values */
+  const esbuildServe = await esbuild.serve({servedir: buildDir}, {})
+
+  const proxyServer = createProxyServer(liveReload?.handleRequest, {
+    hostname: '0.0.0.0',
+    port: esbuildServe.port,
+  })
+
+  const proxyForceExit = createServerForceClose(proxyServer)
+  const portTries = options.findAvailablePort ? 10 : 1
+  const portChosen = await tryMultiplePorts(defaultPort, portTries, proxyServer)
+
+  const hostedAt = `http://localhost:${portChosen}`
+
+  console.log('Playground running at', hostedAt)
+
+  if (options.openBrowser) {
+    setTimeout(() => {
+      if (!liveReload?.hasOpenConnections()) openForOS(hostedAt)
+    }, 1000)
   }
 
-  return portToTry
+  return {
+    stop() {
+      esbuildServe.stop()
+      esbuildWatchStop?.()
+      return Promise.all([proxyForceExit(), esbuildServe.wait]).then(() => {
+        // map to void for type defs
+      })
+    },
+  }
 }

packages/playground/devEnv/cli.js

@@ -0,0 +1,55 @@
+const {timer} = require('./timer')
+
+const dev = process.argv.find((arg) => ['--dev', '-d'].includes(arg)) != null
+const isCI = Boolean(process.env.CI)
+let current
+
+function onUpdatedBuildScript(rebuild) {
+  delete require.cache[require.resolve('./build.compiled')]
+  /** @type {import("./build")} */
+  const module = require('./build.compiled')
+  const _start = timer('build.compiled start')
+  try {
+    module
+      .start({
+        dev,
+        findAvailablePort: !isCI,
+        // If not in CI, try to spawn a browser
+        openBrowser: !isCI && !rebuild,
+        waitBeforeStartingServer: current?.stop(),
+      })
+      .then((running) => {
+        current = running
+      })
+      .catch((err) => {
+        console.error('cli.js calling start() in build.compiled.js', err)
+      })
+      .finally(() => _start.stop())
+  } catch (err) {
+    _start.stop()
+  }
+}
+
+timer('cli.js').wrap(() => {
+  timer('esbuild build.compiled.js').wrap(() => {
+    const {build} = require('esbuild')
+
+    // compile build files directly which is about 10x faster than esbuild-register
+    build({
+      entryPoints: [__dirname + '/build.ts'],
+      outfile: __dirname + '/build.compiled.js',
+      bundle: true,
+      platform: 'node',
+      external: ['esbuild', 'react', 'react-dom/server'],
+      watch: dev && {
+        onRebuild(err, res) {
+          if (!err) {
+            onUpdatedBuildScript(true)
+          }
+        },
+      },
+    }).then(() => {
+      onUpdatedBuildScript(false)
+    })
+  })
+})

packages/playground/devEnv/createEsbuildLiveReloadTools.ts

@@ -0,0 +1,75 @@
+import type esbuild from 'esbuild'
+import type {IncomingMessage, ServerResponse} from 'http'
+
+export function createEsbuildLiveReloadTools(): {
+  handleRequest(req: IncomingMessage, res: ServerResponse): boolean
+  hasOpenConnections(): boolean
+  esbuildBanner: esbuild.BuildOptions['banner']
+  esbuildWatch: esbuild.WatchMode
+} {
+  const openResponses = new Set<ServerResponse>()
+  return {
+    handleRequest(req, res) {
+      // If special /esbuild url requested, subscribe clients to changes
+      if (req.url === '/esbuild') {
+        res.writeHead(200, {
+          'Content-Type': 'text/event-stream',
+          'Cache-Control': 'no-cache',
+          Connection: 'keep-alive',
+        })
+        res.write('data: open\n\n')
+        openResponses.add(res)
+        res.on('close', () => openResponses.delete(res))
+        return true // handled
+      }
+      return false
+    },
+    hasOpenConnections() {
+      return openResponses.size > 0
+    },
+    esbuildBanner: {
+      // Below uses function toString to insert raw source code of the function into the JS source.
+      // This is being used so we can at least get a few type completions, but please understand that
+      // you cannot reference any non-global browser values from within the function.
+      js: `;(${function liveReloadClientSetup() {
+        // from packages/playground/devEnv/createEsbuildLiveReloadTools.ts
+        function connect() {
+          try {
+            const es = new EventSource('/esbuild')
+            es.onmessage = (evt) => {
+              switch (evt.data) {
+                case 'reload':
+                  location.reload()
+                  break
+                case 'open':
+                  console.log('%cLive reload ready', 'color: gray')
+                  break
+              }
+            }
+            es.onerror = attemptConnect
+          } catch (err) {
+            attemptConnect()
+          }
+        }
+        function attemptConnect() {
+          setTimeout(() => connect(), 1000)
+        }
+        attemptConnect()
+      }.toString()})();`,
+    },
+    esbuildWatch: {
+      onRebuild(error, res) {
+        if (!error) {
+          if (openResponses.size > 0) {
+            console.error(`Reloading for ${openResponses.size} clients...`)
+            // Notify clients on rebuild
+            openResponses.forEach((res) => res.write('data: reload\n\n'))
+            openResponses.clear()
+          }
+        } else {
+          console.error('Rebuild had errors...')
+        }
+      },
+    },
+  }
+}

packages/playground/devEnv/createProxyServer.ts

@@ -0,0 +1,34 @@
+import type {IncomingMessage, ServerResponse} from 'http'
+import {createServer, request} from 'http'
+
+// See example from https://esbuild.github.io/api/#customizing-server-behavior
+export function createProxyServer(
+  handleRequest:
+    | ((req: IncomingMessage, res: ServerResponse) => boolean)
+    | undefined,
+  target: {hostname: string; port: number},
+) {
+  return createServer((req, res) => {
+    const {url, method, headers} = req
+    if (handleRequest?.(req, res)) {
+      return
+    }
+
+    // Otherwise forward requests to target (e.g. ESBuild server)
+    req.pipe(
+      request(
+        {
+          ...target,
+          path: url,
+          method,
+          headers,
+        },
+        (prxRes) => {
+          res.writeHead(prxRes.statusCode!, prxRes.headers)
+          prxRes.pipe(res, {end: true})
+        },
+      ),
+      {end: true},
+    )
+  })
+}

packages/playground/devEnv/createServerForceClose.ts

@@ -0,0 +1,21 @@
+import type {Server, Socket} from 'net'
+
+export function createServerForceClose(server: Server) {
+  const openConnections = new Set<Socket>()
+  server.on('connection', (conn) => {
+    openConnections.add(conn)
+    conn.on('close', () => openConnections.delete(conn))
+  })
+
+  return function serverForceClose(): Promise<void> {
+    for (const openConnection of openConnections) {
+      openConnection.destroy()
+    }
+
+    return new Promise((res) => {
+      server.close(() => {
+        res()
+      })
+    })
+  }
+}

packages/playground/devEnv/openForOS.ts

@@ -0,0 +1,17 @@
+import {spawn} from 'child_process'
+
+export function openForOS(hostedAt: string) {
+  const open = {
+    darwin: ['open'],
+    linux: ['xdg-open'],
+    win32: ['cmd', '/c', 'start'],
+  }
+  const platform = process.platform as keyof typeof open
+  if (open[platform]) {
+    spawn(open[platform][0], [...open[platform].slice(1), hostedAt])
+  } else {
+    console.error(
+      `Failed to open (${hostedAt}) for unconfigured platform (${platform})`,
+    )
+  }
+}

packages/playground/devEnv/timer.d.ts

@@ -0,0 +1,5 @@
+/** Create timer */
+export function timer(name: string): {
+  wrap<T>(fn: () => T): T
+  stop(): void
+}

packages/playground/devEnv/timer.js

@@ -0,0 +1,25 @@
+/** @param {string} name */
+function timer(name) {
+  const startMs = Date.now()
+  console.group(`▶️ ${name}`)
+  let stopped = false
+  return {
+    /**
+     * @type {<T> (fn: () => T): T}
+     */
+    wrap(fn) {
+      const result = fn()
+      this.stop()
+      return result
+    },
+    stop() {
+      if (stopped) return
+      stopped = true
+      console.groupEnd()
+      console.log(
+        `✓ ${name} in ${((Date.now() - startMs) * 0.001).toFixed(3)}s`,
+      )
+    },
+  }
+}
+exports.timer = timer

packages/playground/devEnv/tryMultiplePorts.ts

@@ -0,0 +1,47 @@
+import type {Server} from 'net'
+
+export async function tryMultiplePorts(
+  port: number,
+  tries: number,
+  server: Server,
+): Promise<number> {
+  let portToTry = port
+  let firstError = null
+  let lastError = null
+
+  while (portToTry < port + tries) {
+    try {
+      await new Promise((res, rej) => {
+        const onListening = () => (rm(), res(true))
+        const onError = () => (rm(), rej())
+        const rm = () => {
+          server.off('error', onError)
+          server.off('listening', onListening)
+        }
+
+        server
+          .listen(portToTry)
+          .on('listening', onListening)
+          .on('error', onError)
+      })
+
+      firstError = null
+      lastError = null
+      break // found a working port
+    } catch (err) {
+      if (!firstError) firstError = err
+      lastError = err
+      portToTry += 1
+    }
+  }
+
+  if (firstError) {
+    console.error(firstError)
+    console.error(lastError)
+    throw new Error(
+      `Failed to find port starting at ${port} with ${tries} tries.`,
+    )
+  }
+
+  return portToTry
+}

packages/playground/package.json

@@ -8,8 +8,8 @@
     "dist/**/*"
   ],
   "scripts": {
-    "serve": "node -r esbuild-register devEnv/build.ts --dev",
-    "build": "node -r esbuild-register devEnv/build.ts",
+    "serve": "node devEnv/cli.js --dev",
+    "build": "node devEnv/cli.js",
     "build:static": "yarn build",
     "typecheck": "yarn run build",
     "test": "playwright test --config=devEnv/playwright.config.ts",

theatre/devEnv/bundle.ts

@@ -1,3 +1,3 @@
-import {createBundles} from './buildUtils'
+import {createBundles} from './createBundles'
 
 createBundles(false)

theatre/devEnv/createBundles.ts

@@ -1,14 +1,6 @@
 import path from 'path'
 import {build} from 'esbuild'
-
-export const definedGlobals = {
-  'process.env.THEATRE_VERSION': JSON.stringify(
-    require('../studio/package.json').version,
-  ),
-  // json-touch-patch (an unmaintained package) reads this value. We patch it to just 'Set', becauce
-  // this is only used in `@theatre/studio`, which only supports evergreen browsers
-  'global.Set': 'Set',
-}
+import {definedGlobals} from './definedGlobals'
 
 export function createBundles(watch: boolean) {
   for (const which of ['core', 'studio']) {

theatre/devEnv/definedGlobals.ts

@@ -0,0 +1,8 @@
+export const definedGlobals = {
+  'process.env.THEATRE_VERSION': JSON.stringify(
+    require('../studio/package.json').version,
+  ),
+  // json-touch-patch (an unmaintained package) reads this value. We patch it to just 'Set', becauce
+  // this is only used in `@theatre/studio`, which only supports evergreen browsers
+  'global.Set': 'Set',
+}

theatre/devEnv/watch.ts

@@ -1,3 +1,3 @@
-import {createBundles} from './buildUtils'
+import {createBundles} from './createBundles'
 
 createBundles(true)