Set up end-to-end tests (#85)

packages/playground/.gitignore

@@ -1 +1,3 @@
-/dist
+/dist
+/test-results/
+/playwright-report/

packages/playground/README.md

@@ -1,12 +1,58 @@
 # The playground
 
-The playground is the quickest way to hack on the internals of Theatre. It uses a simple ESBuild config that builds all the related packages in one go, so you _don't_ have to run a bunch of build commands separately to start developing.
+The playground is the quickest way to hack on the internals of Theatre. It also hosts our end-to-end tests. It uses a simple vite config that builds all the related packages in one go, so you _don't_ have to run a bunch of build commands separately to start developing.
 
-## How to use
+## Directory structure
+
+```
+src/
+  shared/                      <---- playgrounds shared with teammates.
+    [playground-name]/         <---- each playground has a name...
+      index.tsx                <---- and an entry file.
+
+  personal/                    <---- personal playgrounds (gitignored).
+    [playground-name]/         <---- personal playgrounds also have names,
+      index.tsx                <---- and an entry file.
+
+  tests/                       <---- playgrounds for e2e testing.
+    [playground-name]/         <---- the name of the test playground,
+      index.tsx                <---- and its entry file.
+      [test-file-name].e2e.ts  <---- The playwright test script that tests this particular playground.
+      [test2].e2e.ts           <---- We can have more than one test file per playground.
+```
+
+## How to use the playground
 
 Simply run `yarn run serve` in this folder to start the dev server.
 
-The first time you run `serve`, an `src/index.ts` file will be created. This file is the entry point, and it won't be comitted to the repo, so you're free to change it.
+There are some shared playgrounds in `src/shared` which are committed to the repo. You can make your own playgrounds in `src/personal` which will be `.gitignore`d. Each 
 
-There are some shared playgrounds in `src/shared` which are committed to the repo. You can make your own playgrounds in `src/personal` which will be `.gitignore`d.
+## How to write and run end-to-end tests
 
+The end-to-end tests are in the `src/tests` folder. Look at [directory structure](#directory-structure) to see how test files are organized.
+
+The end-to-end tests are made using [playwright](https://playwright.dev). You should refer to playwright's documentation 
+
+```bash
+$ cd playground
+$ yarn test # runs the end-to-end tests
+$ yarn test --project=firefox # only run the tests in firefox
+$ yarn test --project=firefox --headed # run the test in headed mode in firefox
+$ yarn test --debug # run in debug mode using the inspector: https://playwright.dev/docs/inspector
+```
+
+### Using playwright codegen
+
+To use [playwright's codegen tool](https://playwright.dev/docs/codegen), first serve the playground and then run the codegen on the a url that points to the playground you wish to test:
+
+```bash
+$ cd playground
+$ yarn serve # first serve the playground
+$ yarn playwright codegen http://localhost:8080/tests/[playground-name] # run the codegen for [playground-name]
+```
+
+## Visual regression testing
+
+We're currently using [percy](https://percy.io) for visual regression testing. These tests run only the the [CI](../../.github/workflows/main.yml) using [Github actions](https://github.com/theatre-js/theatre/actions). Look at the example at [`src/tests/setting-static-props/test.e2e.ts`](src/tests/setting-static-props/test.e2e.ts) for an example of recording and diffing a screenshot.
+
+Please note that we haven't figured out the best practices for visual regression testing yet, so if the setup isn't optimal, please let us know.

packages/playground/devEnv/playwright.config.ts

@@ -0,0 +1,77 @@
+import type {PlaywrightTestConfig} from '@playwright/test'
+import {devices} from '@playwright/test'
+
+/**
+ * Read environment variables from file.
+ * https://github.com/motdotla/dotenv
+ */
+// require('dotenv').config();
+
+/**
+ * See https://playwright.dev/docs/test-configuration.
+ */
+const config: PlaywrightTestConfig = {
+  testDir: '../src',
+  testMatch: /.*\.e2e\.ts/,
+  /* Maximum time one test can run for. */
+  timeout: 100000,
+  expect: {
+    /**
+     * Maximum time expect() should wait for the condition to be met.
+     * For example in `await expect(locator).toHaveText();`
+     */
+    timeout: 10000,
+  },
+  /* Fail the build on CI if you accidentally left test.only in the source code. */
+  forbidOnly: !!process.env.CI,
+  /* Retry on CI only */
+  retries: process.env.CI ? 0 : 0,
+  /* Opt out of parallel tests on CI. */
+  workers: process.env.CI ? 1 : undefined,
+  /* Reporter to use. See https://playwright.dev/docs/test-reporters */
+  reporter: process.env.CI ? 'github' : 'html',
+  /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
+  use: {
+    /* Maximum time each action such as `click()` can take. Defaults to 0 (no limit). */
+    // actionTimeout: 200,
+    /* Base URL to use in actions like `await page.goto('/')`. */
+    // baseURL: 'http://localhost:3000',
+
+    /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
+    trace: 'on-first-retry',
+  },
+
+  /* Configure projects for major browsers */
+  projects: [
+    {
+      name: 'chromium',
+      use: {
+        ...devices['Desktop Chrome'],
+      },
+    },
+
+    {
+      name: 'firefox',
+      use: {
+        ...devices['Desktop Firefox'],
+      },
+    },
+  ],
+
+  /* Folder for test artifacts such as screenshots, videos, traces, etc. */
+  outputDir: '../test-results/',
+
+  /*
+  This will serve the playground before running the tests, unless the playground is already running.
+
+  Note that if the playground is not running but some other server is serving at port 8080, this will fail.
+  TODO 👆
+  */
+  webServer: {
+    command: 'yarn run serve',
+    port: 8080,
+    reuseExistingServer: !process.env.CI,
+  },
+}
+
+export default config

packages/playground/devEnv/vite.config.ts

@@ -3,7 +3,6 @@ import react from '@vitejs/plugin-react'
 import path from 'path'
 import {getAliasesFromTsConfigForRollup} from '../../../devEnv/getAliasesFromTsConfig'
 import {definedGlobals} from '../../../theatre/devEnv/buildUtils'
-import {existsSync, writeFileSync} from 'fs'
 
 /*
 We're using vite instead of the older pure-esbuild setup. The tradeoff is
@@ -18,36 +17,10 @@ const playgroundDir = path.join(__dirname, '..')
 
 const port = 8080
 
-/**
- * Creates playground/src/index.ts, since that file isn't committed to the repo.
- */
-function createPlaygroundIndex() {
-  const playgroundIndexContent = `
-  /**
-   * This file is created automatically and won't be comitted to the repo.
-   * You can change the import statement and import your own playground code.
-   * 
-   * Your own playground code should reside in './personal', which is a folder
-   * that won't be committed to the repo.
-   * 
-   * The shared playgrounds which other contributors can use are in the './shared' folder,
-   * which are comitted to the repo.
-   * 
-   * Happy playing!
-   * */
-  import './shared/r3f-rocket'
-  `
-
-  const playgroundEntry = path.join(playgroundDir, 'src/index.ts')
-  if (!existsSync(playgroundEntry)) {
-    writeFileSync(playgroundEntry, playgroundIndexContent, {encoding: 'utf-8'})
-  }
-}
-createPlaygroundIndex()
-
 // https://vitejs.dev/config/
 export default defineConfig({
   root: path.join(playgroundDir, './src'),
+
   assetsInclude: ['**/*.gltf', '**/*.glb'],
   server: {
     port,
@@ -61,5 +34,5 @@ export default defineConfig({
     */
     alias: [...getAliasesFromTsConfigForRollup()],
   },
-  define: definedGlobals,
+  define: {...definedGlobals, 'window.__IS_VISUAL_REGRESSION_TESTING': 'true'},
 })

packages/playground/package.json

@@ -10,9 +10,14 @@
   "scripts": {
     "serve": "vite --config ./devEnv/vite.config.ts",
     "typecheck": "yarn run build",
+    "test": "playwright test --config=devEnv/playwright.config.ts",
+    "test:ci": "percy exec -- playwright test --reporter=dot --config=devEnv/playwright.config.ts --project=chromium",
     "build": "tsc --build ./tsconfig.json"
   },
   "devDependencies": {
+    "@percy/cli": "^1.0.0-beta.76",
+    "@percy/playwright": "^1.0.1",
+    "@playwright/test": "^1.19.1",
     "@react-three/drei": "^7.2.2",
     "@react-three/fiber": "^7.0.6",
     "@theatre/core": "workspace:*",

packages/playground/src/.gitignore

@@ -1,2 +1 @@
-personal
-index.ts
+personal

packages/playground/src/index.html

@@ -8,6 +8,13 @@
         padding: 0;
         height: 100%;
         background: black;
+        color: white;
+        font-family: sans-serif;
+        font-size: 12px;
+      }
+
+      a {
+        color: inherit;
       }
     </style>
   </head>
@@ -15,6 +22,6 @@
   <body>
     <div id="root"></div>
 
-    <script type="module" src="./index.ts"></script>
+    <script type="module" src="./index.tsx"></script>
   </body>
 </html>

packages/playground/src/index.tsx

@@ -0,0 +1,92 @@
+/**
+ * TODO explain this file
+ * */
+/// <reference types="vite/client" />
+
+import type {$FixMe} from '@theatre/shared/utils/types'
+import {mapKeys} from 'lodash-es'
+import React from 'react'
+import ReactDOM from 'react-dom'
+
+const groups = {
+  shared: mapKeys(import.meta.glob('./shared/*/index.tsx'), (_, path) =>
+    pathToModuleName(path),
+  ),
+  personal: mapKeys(import.meta.glob('./personal/*/index.tsx'), (_, path) =>
+    pathToModuleName(path),
+  ),
+  tests: mapKeys(import.meta.glob('./tests/*/index.tsx'), (_, path) =>
+    pathToModuleName(path),
+  ),
+}
+
+function pathToModuleName(path: string): string {
+  const matches = path.match(
+    /^\.\/(shared|personal|tests)\/([a-zA-Z0-9\-\s]+)\/index\.tsx$/,
+  )
+
+  if (!matches) {
+    throw new Error(
+      `module ${path} has invalid characters in its path. Valid names should match the regexp above this line.`,
+    )
+  }
+
+  return matches[2]
+}
+
+const Home = () => (
+  <ul>
+    {Object.entries(groups).map(([groupName, modules]) => (
+      <li key={`li-${groupName}`}>
+        <span>{groupName}</span>
+        <Group
+          key={`group-${groupName}`}
+          groupName={groupName}
+          modules={modules}
+        />
+      </li>
+    ))}
+  </ul>
+)
+
+const Group = (props: {groupName: string; modules: Record<string, $FixMe>}) => {
+  const {groupName, modules} = props
+  return (
+    <ul>
+      {Object.entries(modules).map(([moduleName, callback]) => (
+        <li key={`li-${moduleName}`}>
+          <a href={`/${groupName}/${moduleName}`}>{moduleName}</a>
+          {/* <Group key={`group-${group}`} modules={modules} /> */}
+        </li>
+      ))}
+    </ul>
+  )
+}
+
+const currentPathname = document.location.pathname
+
+if (currentPathname === '/') {
+  renderHome()
+} else {
+  const parts = currentPathname.match(
+    /^\/(shared|personal|tests)\/([a-zA-Z0-9\-]+)$/,
+  )
+  if (parts) {
+    const [, groupName, moduleName] = parts
+    const group = groups[groupName as 'shared' | 'personal']
+    if (!group) {
+      throw new Error(`Unknown group ${groupName}`)
+    }
+    const module = group[moduleName]
+    if (!module) {
+      throw new Error(`Unknown module ${moduleName}`)
+    }
+    module()
+  } else {
+    throw new Error(`Unknown path ${currentPathname}`)
+  }
+}
+
+function renderHome() {
+  ReactDOM.render(React.createElement(Home), document.getElementById('root'))
+}

packages/playground/src/tests/setting-static-props/index.tsx

@@ -0,0 +1,13 @@
+import studio from '@theatre/studio'
+import {getProject} from '@theatre/core'
+
+studio.initialize({usePersistentStorage: false})
+
+const project = getProject('sample project')
+const sheet = project.sheet('sample sheet')
+const obj = sheet.object('sample object', {
+  position: {
+    x: 0,
+    y: 0,
+  },
+})

packages/playground/src/tests/setting-static-props/test.e2e.ts

@@ -0,0 +1,47 @@
+import {test, expect} from '@playwright/test'
+import percySnapshot from '@percy/playwright'
+
+const isMac = process.platform === 'darwin'
+
+test.describe('setting-static-props', () => {
+  test.beforeEach(async ({page}) => {
+    // Go to the starting url before each test.
+    await page.goto('http://localhost:8080/tests/setting-static-props')
+  })
+
+  test('Undo/redo', async ({page}) => {
+    await page.locator('[data-testid="OutlinePanel-TriggerButton"]').click()
+
+    await page.locator('span:has-text("sample object")').first().click()
+
+    const detailPanel = page.locator('[data-testid="DetailPanel-Object"]')
+
+    const firstInput = detailPanel.locator('input[type="text"]').first()
+    // Click input[type="text"] >> nth=0
+    await firstInput.click()
+    // Fill input[type="text"] >> nth=0
+    await firstInput.fill('1')
+    // Press Enter
+    await firstInput.press('Enter')
+    const secondInput = detailPanel.locator('input[type="text"]').nth(1)
+    // Click input[type="text"] >> nth=1
+    await secondInput.click()
+    // Fill input[type="text"] >> nth=1
+    await secondInput.fill('2')
+    // Press Enter
+    await secondInput.press('Enter')
+
+    const metaKey = isMac ? 'Meta' : 'Control'
+
+    // Press z with modifiers
+    await page.locator('body').press(`${metaKey}+z`)
+    await expect(firstInput).toHaveAttribute('value', '1')
+    await expect(secondInput).toHaveAttribute('value', '0')
+    await page.locator('body').press(`${metaKey}+Shift+z`)
+    await expect(firstInput).toHaveAttribute('value', '1')
+    await expect(secondInput).toHaveAttribute('value', '2')
+
+    // Our first visual regression test
+    await percySnapshot(page, test.info().titlePath.join('/') + '/After redo')
+  })
+})