dev: Build playground files much faster and add watch

packages/playground/devEnv/.gitignore

@@ -0,0 +1 @@
+build.compiled.js

packages/playground/devEnv/build.ts

@@ -3,15 +3,17 @@ import {writeFile, readFile} from 'fs/promises'
 import path from 'path'
 import type {BuildOptions} from 'esbuild'
 import esbuild from 'esbuild'
-import type {IncomingMessage, ServerResponse} from 'http'
-import {definedGlobals} from '../../../theatre/devEnv/buildUtils'
+import {definedGlobals} from '../../../theatre/devEnv/definedGlobals'
 import {mapValues} from 'lodash-es'
-import {createServer, request} from 'http'
-import {spawn} from 'child_process'
 import React from 'react'
 import {renderToStaticMarkup} from 'react-dom/server'
 import {Home} from './Home'
-import type {Server} from 'net'
+import {timer} from './timer'
+import {openForOS} from './openForOS'
+import {tryMultiplePorts} from './tryMultiplePorts'
+import {createProxyServer} from './createProxyServer'
+import {createEsbuildLiveReloadTools} from './createEsbuildLiveReloadTools'
+import {createServerForceClose} from './createServerForceClose'
 
 const playgroundDir = (folder: string) => path.join(__dirname, '..', folder)
 const buildDir = playgroundDir('build')
@@ -19,65 +21,29 @@ const sharedDir = playgroundDir('src/shared')
 const personalDir = playgroundDir('src/personal')
 const testDir = playgroundDir('src/tests')
 
-const dev = process.argv.find((arg) => ['--dev', '-d'].includes(arg)) != null
-const defaultPort = 8080
+export async function start(options: {
+  dev: boolean
+  findAvailablePort: boolean
+  openBrowser: boolean
+  waitBeforeStartingServer?: Promise<void>
+  /** defaults to 8080 */
+  defaultPort?: number
+}): Promise<{stop(): Promise<void>}> {
+  const defaultPort = options.defaultPort ?? 8080
 
-const liveReload =
-  (dev || undefined) &&
-  ((): {
-    handleRequest(req: IncomingMessage, res: ServerResponse): boolean
-    hasOpenConnections(): boolean
-    esbuildBanner: esbuild.BuildOptions['banner']
-    esbuildWatch: esbuild.WatchMode
-  } => {
-    const openResponses: ServerResponse[] = []
-    return {
-      handleRequest(req, res) {
-        // If special /esbuild url requested, subscribe clients to changes
-        if (req.url === '/esbuild') {
-          openResponses.push(
-            res.writeHead(200, {
-              'Content-Type': 'text/event-stream',
-              'Cache-Control': 'no-cache',
-              Connection: 'keep-alive',
-            }),
-          )
-          return true
-        }
-        return false
-      },
-      hasOpenConnections() {
-        return openResponses.length > 0
-      },
-      esbuildBanner: {
-        js: ' (() => new EventSource("/esbuild").onmessage = () => location.reload())();',
-      },
-      esbuildWatch: {
-        onRebuild(error) {
-          if (!error) {
-            console.error('Reloading...')
-            // Notify clients on rebuild
-            openResponses.forEach((res) => res.write('data: update\n\n'))
-            openResponses.length = 0
-          } else {
-            console.error('Rebuild had errors...')
-          }
-        },
-      },
-    }
-  })()
+  const liveReload = options.dev ? createEsbuildLiveReloadTools() : undefined
 
-type Groups = {
+  type Groups = {
     [group: string]: {
       [module: string]: {
         entryDir: string
         outDir: string
       }
     }
-}
+  }
 
-// Collect all entry directories per module per group
-const groups: Groups = Object.fromEntries(
+  // Collect all entry directories per module per group
+  const groups: Groups = Object.fromEntries(
     [sharedDir, personalDir, testDir]
       .map((groupDir) => {
         try {
@@ -104,30 +70,32 @@ const groups: Groups = Object.fromEntries(
       })
       // and then filter it out.
       .filter((entry) => entry[1] !== undefined),
-)
+  )
 
-// Collect all entry files
-const entryPoints = Object.values(groups)
+  // Collect all entry files
+  const entryPoints = Object.values(groups)
     .flatMap((group) => Object.values(group))
     .map((module) => path.join(module.entryDir, 'index.tsx'))
 
-// Collect all output directories
-const outDirs = Object.values(groups).flatMap((group) =>
+  // Collect all output directories
+  const outDirs = Object.values(groups).flatMap((group) =>
     Object.values(group).map((module) => module.outDir),
-)
+  )
 
-// Render home page contents
-const homeHtml = renderToStaticMarkup(
+  // Render home page contents
+  const homeHtml = renderToStaticMarkup(
     React.createElement(Home, {
       groups: mapValues(groups, (group) => Object.keys(group)),
     }),
-)
+  )
 
-const config: BuildOptions = {
+  const _initialBuild = timer('esbuild initial playground entry point builds')
+
+  const esbuildConfig: BuildOptions = {
     entryPoints,
     bundle: true,
     sourcemap: true,
-  outdir: playgroundDir('build'),
+    outdir: buildDir,
     target: ['firefox88'],
     loader: {
       '.png': 'file',
@@ -141,15 +109,19 @@ const config: BuildOptions = {
     },
     banner: liveReload?.esbuildBanner,
     watch: liveReload?.esbuildWatch,
-}
+  }
 
-esbuild
-  .build(config)
+  let esbuildWatchStop: undefined | (() => void)
+
+  await esbuild
+    .build(esbuildConfig)
+    .finally(() => _initialBuild.stop())
     .catch((err) => {
       // if in dev mode, permit continuing to watch even if there was an error
-    return dev ? Promise.resolve() : Promise.reject(err)
+      return options.dev ? Promise.resolve() : Promise.reject(err)
     })
-  .then(async () => {
+    .then(async (buildResult) => {
+      esbuildWatchStop = buildResult?.stop
       // Read index.html template
       const index = await readFile(path.join(__dirname, 'index.html'), 'utf8')
       await Promise.all([
@@ -157,6 +129,7 @@ esbuild
         writeFile(
           path.join(buildDir, 'index.html'),
           index.replace(/<body>[\s\S]*<\/body>/, `<body>${homeHtml}</body>`),
+          'utf-8',
         ),
         // Write module pages
         ...outDirs.map((outDir) =>
@@ -167,130 +140,59 @@ esbuild
               '%ENTRYPOINT%',
               path.join('/', path.relative(buildDir, outDir), 'index.js'),
             ),
+            'utf-8',
           ),
         ),
       ])
     })
     .catch((err) => {
-    console.log(err)
+      console.error(err)
       return process.exit(1)
     })
-  .then(async () => {
+
   // Only start dev server in dev, otherwise just run build and that's it
-    if (!dev) {
-      return
+  if (!options.dev) {
+    return {
+      stop() {
+        esbuildWatchStop?.()
+        return Promise.resolve()
+      },
     }
+  }
+
+  await options.waitBeforeStartingServer
 
   // We start ESBuild serve with no build config because it doesn't need to build
   // anything, we are already using ESBuild watch.
-    const {port: esbuildPort} = await esbuild.serve(
-      {servedir: playgroundDir('build')},
-      {},
-    )
+  /** See https://esbuild.github.io/api/#serve-return-values */
+  const esbuildServe = await esbuild.serve({servedir: buildDir}, {})
 
-    const proxyServer = createServer((req, res) => {
-      const {url, method, headers} = req
-      if (liveReload?.handleRequest(req, res)) {
-        return
-      }
-
-      // Otherwise forward requests to ESBuild server
-      req.pipe(
-        request(
-          {
+  const proxyServer = createProxyServer(liveReload?.handleRequest, {
     hostname: '0.0.0.0',
-            port: esbuildPort,
-            path: url,
-            method,
-            headers,
-          },
-          (prxRes) => {
-            res.writeHead(prxRes.statusCode!, prxRes.headers)
-            prxRes.pipe(res, {end: true})
-          },
-        ),
-        {end: true},
-      )
+    port: esbuildServe.port,
   })
 
-    const isCI = process.env.CI
-    const portTries = isCI ? 1 : 10
-    const portChosen = await tryMultiplePorts(
-      defaultPort,
-      portTries,
-      proxyServer,
-    )
+  const proxyForceExit = createServerForceClose(proxyServer)
+  const portTries = options.findAvailablePort ? 10 : 1
+  const portChosen = await tryMultiplePorts(defaultPort, portTries, proxyServer)
 
   const hostedAt = `http://localhost:${portChosen}`
 
   console.log('Playground running at', hostedAt)
 
-    // If not in CI, try to spawn a browser
-    if (!isCI) {
+  if (options.openBrowser) {
     setTimeout(() => {
       if (!liveReload?.hasOpenConnections()) openForOS(hostedAt)
     }, 1000)
   }
-  })
 
-function openForOS(hostedAt: string) {
-  const open = {
-    darwin: ['open'],
-    linux: ['xdg-open'],
-    win32: ['cmd', '/c', 'start'],
-  }
-  const platform = process.platform as keyof typeof open
-  if (open[platform]) {
-    spawn(open[platform][0], [...open[platform].slice(1), hostedAt])
-  } else {
-    console.error(
-      `Failed to open (${hostedAt}) for unconfigured platform (${platform})`,
-    )
+  return {
+    stop() {
+      esbuildServe.stop()
+      esbuildWatchStop?.()
+      return Promise.all([proxyForceExit(), esbuildServe.wait]).then(() => {
+        // map to void for type defs
+      })
+    },
   }
 }
-
-async function tryMultiplePorts(
-  port: number,
-  tries: number,
-  server: Server,
-): Promise<number> {
-  let portToTry = port
-  let firstError = null
-  let lastError = null
-
-  while (portToTry < port + tries) {
-    try {
-      await new Promise((res, rej) => {
-        const onListening = () => (rm(), res(true))
-        const onError = () => (rm(), rej())
-        const rm = () => {
-          server.off('error', onError)
-          server.off('listening', onListening)
-        }
-
-        server
-          .listen(portToTry)
-          .on('listening', onListening)
-          .on('error', onError)
-      })
-
-      firstError = null
-      lastError = null
-      break // found a working port
-    } catch (err) {
-      if (!firstError) firstError = err
-      lastError = err
-      portToTry += 1
-    }
-  }
-
-  if (firstError) {
-    console.error(firstError)
-    console.error(lastError)
-    throw new Error(
-      `Failed to find port starting at ${port} with ${tries} tries.`,
-    )
-  }
-
-  return portToTry
-}

packages/playground/devEnv/cli.js

@@ -0,0 +1,55 @@
+const {timer} = require('./timer')
+
+const dev = process.argv.find((arg) => ['--dev', '-d'].includes(arg)) != null
+const isCI = Boolean(process.env.CI)
+let current
+
+function onUpdatedBuildScript(rebuild) {
+  delete require.cache[require.resolve('./build.compiled')]
+  /** @type {import("./build")} */
+  const module = require('./build.compiled')
+  const _start = timer('build.compiled start')
+  try {
+    module
+      .start({
+        dev,
+        findAvailablePort: !isCI,
+        // If not in CI, try to spawn a browser
+        openBrowser: !isCI && !rebuild,
+        waitBeforeStartingServer: current?.stop(),
+      })
+      .then((running) => {
+        current = running
+      })
+      .catch((err) => {
+        console.error('cli.js calling start() in build.compiled.js', err)
+      })
+      .finally(() => _start.stop())
+  } catch (err) {
+    _start.stop()
+  }
+}
+
+timer('cli.js').wrap(() => {
+  timer('esbuild build.compiled.js').wrap(() => {
+    const {build} = require('esbuild')
+
+    // compile build files directly which is about 10x faster than esbuild-register
+    build({
+      entryPoints: [__dirname + '/build.ts'],
+      outfile: __dirname + '/build.compiled.js',
+      bundle: true,
+      platform: 'node',
+      external: ['esbuild', 'react', 'react-dom/server'],
+      watch: dev && {
+        onRebuild(err, res) {
+          if (!err) {
+            onUpdatedBuildScript(true)
+          }
+        },
+      },
+    }).then(() => {
+      onUpdatedBuildScript(false)
+    })
+  })
+})

packages/playground/devEnv/createEsbuildLiveReloadTools.ts

@@ -0,0 +1,75 @@
+import type esbuild from 'esbuild'
+import type {IncomingMessage, ServerResponse} from 'http'
+
+export function createEsbuildLiveReloadTools(): {
+  handleRequest(req: IncomingMessage, res: ServerResponse): boolean
+  hasOpenConnections(): boolean
+  esbuildBanner: esbuild.BuildOptions['banner']
+  esbuildWatch: esbuild.WatchMode
+} {
+  const openResponses = new Set<ServerResponse>()
+  return {
+    handleRequest(req, res) {
+      // If special /esbuild url requested, subscribe clients to changes
+      if (req.url === '/esbuild') {
+        res.writeHead(200, {
+          'Content-Type': 'text/event-stream',
+          'Cache-Control': 'no-cache',
+          Connection: 'keep-alive',
+        })
+        res.write('data: open\n\n')
+        openResponses.add(res)
+        res.on('close', () => openResponses.delete(res))
+        return true // handled
+      }
+      return false
+    },
+    hasOpenConnections() {
+      return openResponses.size > 0
+    },
+    esbuildBanner: {
+      // Below uses function toString to insert raw source code of the function into the JS source.
+      // This is being used so we can at least get a few type completions, but please understand that
+      // you cannot reference any non-global browser values from within the function.
+      js: `;(${function liveReloadClientSetup() {
+        // from packages/playground/devEnv/createEsbuildLiveReloadTools.ts
+        function connect() {
+          try {
+            const es = new EventSource('/esbuild')
+            es.onmessage = (evt) => {
+              switch (evt.data) {
+                case 'reload':
+                  location.reload()
+                  break
+                case 'open':
+                  console.log('%cLive reload ready', 'color: gray')
+                  break
+              }
+            }
+            es.onerror = attemptConnect
+          } catch (err) {
+            attemptConnect()
+          }
+        }
+        function attemptConnect() {
+          setTimeout(() => connect(), 1000)
+        }
+        attemptConnect()
+      }.toString()})();`,
+    },
+    esbuildWatch: {
+      onRebuild(error, res) {
+        if (!error) {
+          if (openResponses.size > 0) {
+            console.error(`Reloading for ${openResponses.size} clients...`)
+            // Notify clients on rebuild
+            openResponses.forEach((res) => res.write('data: reload\n\n'))
+            openResponses.clear()
+          }
+        } else {
+          console.error('Rebuild had errors...')
+        }
+      },
+    },
+  }
+}

packages/playground/devEnv/createProxyServer.ts

@@ -0,0 +1,34 @@
+import type {IncomingMessage, ServerResponse} from 'http'
+import {createServer, request} from 'http'
+
+// See example from https://esbuild.github.io/api/#customizing-server-behavior
+export function createProxyServer(
+  handleRequest:
+    | ((req: IncomingMessage, res: ServerResponse) => boolean)
+    | undefined,
+  target: {hostname: string; port: number},
+) {
+  return createServer((req, res) => {
+    const {url, method, headers} = req
+    if (handleRequest?.(req, res)) {
+      return
+    }
+
+    // Otherwise forward requests to target (e.g. ESBuild server)
+    req.pipe(
+      request(
+        {
+          ...target,
+          path: url,
+          method,
+          headers,
+        },
+        (prxRes) => {
+          res.writeHead(prxRes.statusCode!, prxRes.headers)
+          prxRes.pipe(res, {end: true})
+        },
+      ),
+      {end: true},
+    )
+  })
+}

packages/playground/devEnv/createServerForceClose.ts

@@ -0,0 +1,21 @@
+import type {Server, Socket} from 'net'
+
+export function createServerForceClose(server: Server) {
+  const openConnections = new Set<Socket>()
+  server.on('connection', (conn) => {
+    openConnections.add(conn)
+    conn.on('close', () => openConnections.delete(conn))
+  })
+
+  return function serverForceClose(): Promise<void> {
+    for (const openConnection of openConnections) {
+      openConnection.destroy()
+    }
+
+    return new Promise((res) => {
+      server.close(() => {
+        res()
+      })
+    })
+  }
+}

packages/playground/devEnv/openForOS.ts

@@ -0,0 +1,17 @@
+import {spawn} from 'child_process'
+
+export function openForOS(hostedAt: string) {
+  const open = {
+    darwin: ['open'],
+    linux: ['xdg-open'],
+    win32: ['cmd', '/c', 'start'],
+  }
+  const platform = process.platform as keyof typeof open
+  if (open[platform]) {
+    spawn(open[platform][0], [...open[platform].slice(1), hostedAt])
+  } else {
+    console.error(
+      `Failed to open (${hostedAt}) for unconfigured platform (${platform})`,
+    )
+  }
+}

packages/playground/devEnv/timer.d.ts

@@ -0,0 +1,5 @@
+/** Create timer */
+export function timer(name: string): {
+  wrap<T>(fn: () => T): T
+  stop(): void
+}

packages/playground/devEnv/timer.js

@@ -0,0 +1,25 @@
+/** @param {string} name */
+function timer(name) {
+  const startMs = Date.now()
+  console.group(`▶️ ${name}`)
+  let stopped = false
+  return {
+    /**
+     * @type {<T> (fn: () => T): T}
+     */
+    wrap(fn) {
+      const result = fn()
+      this.stop()
+      return result
+    },
+    stop() {
+      if (stopped) return
+      stopped = true
+      console.groupEnd()
+      console.log(
+        `✓ ${name} in ${((Date.now() - startMs) * 0.001).toFixed(3)}s`,
+      )
+    },
+  }
+}
+exports.timer = timer

packages/playground/devEnv/tryMultiplePorts.ts

@@ -0,0 +1,47 @@
+import type {Server} from 'net'
+
+export async function tryMultiplePorts(
+  port: number,
+  tries: number,
+  server: Server,
+): Promise<number> {
+  let portToTry = port
+  let firstError = null
+  let lastError = null
+
+  while (portToTry < port + tries) {
+    try {
+      await new Promise((res, rej) => {
+        const onListening = () => (rm(), res(true))
+        const onError = () => (rm(), rej())
+        const rm = () => {
+          server.off('error', onError)
+          server.off('listening', onListening)
+        }
+
+        server
+          .listen(portToTry)
+          .on('listening', onListening)
+          .on('error', onError)
+      })
+
+      firstError = null
+      lastError = null
+      break // found a working port
+    } catch (err) {
+      if (!firstError) firstError = err
+      lastError = err
+      portToTry += 1
+    }
+  }
+
+  if (firstError) {
+    console.error(firstError)
+    console.error(lastError)
+    throw new Error(
+      `Failed to find port starting at ${port} with ${tries} tries.`,
+    )
+  }
+
+  return portToTry
+}

packages/playground/package.json

@@ -8,8 +8,8 @@
     "dist/**/*"
   ],
   "scripts": {
-    "serve": "node -r esbuild-register devEnv/build.ts --dev",
-    "build": "node -r esbuild-register devEnv/build.ts",
+    "serve": "node devEnv/cli.js --dev",
+    "build": "node devEnv/cli.js",
     "build:static": "yarn build",
     "typecheck": "yarn run build",
     "test": "playwright test --config=devEnv/playwright.config.ts",

theatre/devEnv/bundle.ts

@@ -1,3 +1,3 @@
-import {createBundles} from './buildUtils'
+import {createBundles} from './createBundles'
 
 createBundles(false)

theatre/devEnv/createBundles.ts

@@ -1,14 +1,6 @@
 import path from 'path'
 import {build} from 'esbuild'
-
-export const definedGlobals = {
-  'process.env.THEATRE_VERSION': JSON.stringify(
-    require('../studio/package.json').version,
-  ),
-  // json-touch-patch (an unmaintained package) reads this value. We patch it to just 'Set', becauce
-  // this is only used in `@theatre/studio`, which only supports evergreen browsers
-  'global.Set': 'Set',
-}
+import {definedGlobals} from './definedGlobals'
 
 export function createBundles(watch: boolean) {
   for (const which of ['core', 'studio']) {

theatre/devEnv/definedGlobals.ts

@@ -0,0 +1,8 @@
+export const definedGlobals = {
+  'process.env.THEATRE_VERSION': JSON.stringify(
+    require('../studio/package.json').version,
+  ),
+  // json-touch-patch (an unmaintained package) reads this value. We patch it to just 'Set', becauce
+  // this is only used in `@theatre/studio`, which only supports evergreen browsers
+  'global.Set': 'Set',
+}

theatre/devEnv/watch.ts

@@ -1,3 +1,3 @@
-import {createBundles} from './buildUtils'
+import {createBundles} from './createBundles'
 
 createBundles(true)