Set up end-to-end tests (#85)

.github/workflows/main.yml

@@ -37,3 +37,13 @@ jobs:
       - run: yarn typecheck
       - run: yarn lint:all
       - run: yarn test
+      - name: Download playwright
+        run: yarn workspace playground run playwright install
+      - name: Run e2e tests
+        run: yarn test:e2e
+      - name: Run e2e tests with percy
+        uses: percy/exec-action@v0.3.1
+        with:
+          custom-command: 'yarn test:e2e:ci'
+        env:
+          PERCY_TOKEN: ${{ secrets.PERCY_TOKEN }}

.gitignore

@@ -18,3 +18,4 @@
 !.yarn/releases
 !.yarn/sdks
 !.yarn/versions
+

CONTRIBUTING.md

@@ -41,11 +41,12 @@ The quickest way to start tweaking things is to run the `playground` package.
 ```sh
 $ cd ./packages/playground
 $ yarn serve
+# or, shortcut:
+$ cd root
+$ yarn playground
 ```
 
-The playground is a bunch of ready-made projects that you can run to experiment with Theatre.js.
+The playground is a bunch of ready-made projects that you can run to experiment with Theatre.js. It also contains the project's end-to-end tests.
-
-It uses a single ESBuild config to build all of the related packages in one go, so you don't have to run a bunch of build commands separately.
 
 Read more at [`./packages/playground/README.md`](./packages/playground/README.md).
 
@@ -65,7 +66,7 @@ $ cd examples/dom-cra
 $ yarn start
 ```
 
-### Running tests
+### Running unit/integration tests
 
 We use a single [jest](https://jestjs.io/) setup for the repo. The tests files have the `.test.ts` or `.test.tsx` extension.
 
@@ -78,6 +79,10 @@ $ yarn test
 $ yarn test --watch
 ```
 
+### Running end-to-end tests
+
+End-to-end tests are hosted in the playground package. More details [there](./packages/playground/README.md).
+
 ### Type checking
 
 The packages in this repo have full typescript coverage, so you should be able to get diagnostics and intellisense if your editor supports typescript.

package.json

@@ -9,6 +9,8 @@
   ],
   "scripts": {
     "playground": "yarn workspace playground run serve",
+    "test:e2e": "yarn workspace playground run test",
+    "test:e2e:ci": "yarn workspace playground run test:ci",
     "typecheck": "yarn run build:ts",
     "build": "zx scripts/build.mjs",
     "build:ts": "tsc --build ./devEnv/typecheck-all-projects/tsconfig.all.json",

packages/playground/.gitignore

@@ -1 +1,3 @@
 /dist
+/test-results/
+/playwright-report/

packages/playground/README.md

@@ -1,12 +1,58 @@
 # The playground
 
-The playground is the quickest way to hack on the internals of Theatre. It uses a simple ESBuild config that builds all the related packages in one go, so you _don't_ have to run a bunch of build commands separately to start developing.
+The playground is the quickest way to hack on the internals of Theatre. It also hosts our end-to-end tests. It uses a simple vite config that builds all the related packages in one go, so you _don't_ have to run a bunch of build commands separately to start developing.
 
-## How to use
+## Directory structure
+
+```
+src/
+  shared/                      <---- playgrounds shared with teammates.
+    [playground-name]/         <---- each playground has a name...
+      index.tsx                <---- and an entry file.
+
+  personal/                    <---- personal playgrounds (gitignored).
+    [playground-name]/         <---- personal playgrounds also have names,
+      index.tsx                <---- and an entry file.
+
+  tests/                       <---- playgrounds for e2e testing.
+    [playground-name]/         <---- the name of the test playground,
+      index.tsx                <---- and its entry file.
+      [test-file-name].e2e.ts  <---- The playwright test script that tests this particular playground.
+      [test2].e2e.ts           <---- We can have more than one test file per playground.
+```
+
+## How to use the playground
 
 Simply run `yarn run serve` in this folder to start the dev server.
 
-The first time you run `serve`, an `src/index.ts` file will be created. This file is the entry point, and it won't be comitted to the repo, so you're free to change it.
+There are some shared playgrounds in `src/shared` which are committed to the repo. You can make your own playgrounds in `src/personal` which will be `.gitignore`d. Each 
 
-There are some shared playgrounds in `src/shared` which are committed to the repo. You can make your own playgrounds in `src/personal` which will be `.gitignore`d.
+## How to write and run end-to-end tests
 
+The end-to-end tests are in the `src/tests` folder. Look at [directory structure](#directory-structure) to see how test files are organized.
+
+The end-to-end tests are made using [playwright](https://playwright.dev). You should refer to playwright's documentation 
+
+```bash
+$ cd playground
+$ yarn test # runs the end-to-end tests
+$ yarn test --project=firefox # only run the tests in firefox
+$ yarn test --project=firefox --headed # run the test in headed mode in firefox
+$ yarn test --debug # run in debug mode using the inspector: https://playwright.dev/docs/inspector
+```
+
+### Using playwright codegen
+
+To use [playwright's codegen tool](https://playwright.dev/docs/codegen), first serve the playground and then run the codegen on the a url that points to the playground you wish to test:
+
+```bash
+$ cd playground
+$ yarn serve # first serve the playground
+$ yarn playwright codegen http://localhost:8080/tests/[playground-name] # run the codegen for [playground-name]
+```
+
+## Visual regression testing
+
+We're currently using [percy](https://percy.io) for visual regression testing. These tests run only the the [CI](../../.github/workflows/main.yml) using [Github actions](https://github.com/theatre-js/theatre/actions). Look at the example at [`src/tests/setting-static-props/test.e2e.ts`](src/tests/setting-static-props/test.e2e.ts) for an example of recording and diffing a screenshot.
+
+Please note that we haven't figured out the best practices for visual regression testing yet, so if the setup isn't optimal, please let us know.

packages/playground/devEnv/playwright.config.ts

@@ -0,0 +1,77 @@
+import type {PlaywrightTestConfig} from '@playwright/test'
+import {devices} from '@playwright/test'
+
+/**
+ * Read environment variables from file.
+ * https://github.com/motdotla/dotenv
+ */
+// require('dotenv').config();
+
+/**
+ * See https://playwright.dev/docs/test-configuration.
+ */
+const config: PlaywrightTestConfig = {
+  testDir: '../src',
+  testMatch: /.*\.e2e\.ts/,
+  /* Maximum time one test can run for. */
+  timeout: 100000,
+  expect: {
+    /**
+     * Maximum time expect() should wait for the condition to be met.
+     * For example in `await expect(locator).toHaveText();`
+     */
+    timeout: 10000,
+  },
+  /* Fail the build on CI if you accidentally left test.only in the source code. */
+  forbidOnly: !!process.env.CI,
+  /* Retry on CI only */
+  retries: process.env.CI ? 0 : 0,
+  /* Opt out of parallel tests on CI. */
+  workers: process.env.CI ? 1 : undefined,
+  /* Reporter to use. See https://playwright.dev/docs/test-reporters */
+  reporter: process.env.CI ? 'github' : 'html',
+  /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
+  use: {
+    /* Maximum time each action such as `click()` can take. Defaults to 0 (no limit). */
+    // actionTimeout: 200,
+    /* Base URL to use in actions like `await page.goto('/')`. */
+    // baseURL: 'http://localhost:3000',
+
+    /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
+    trace: 'on-first-retry',
+  },
+
+  /* Configure projects for major browsers */
+  projects: [
+    {
+      name: 'chromium',
+      use: {
+        ...devices['Desktop Chrome'],
+      },
+    },
+
+    {
+      name: 'firefox',
+      use: {
+        ...devices['Desktop Firefox'],
+      },
+    },
+  ],
+
+  /* Folder for test artifacts such as screenshots, videos, traces, etc. */
+  outputDir: '../test-results/',
+
+  /*
+  This will serve the playground before running the tests, unless the playground is already running.
+
+  Note that if the playground is not running but some other server is serving at port 8080, this will fail.
+  TODO 👆
+  */
+  webServer: {
+    command: 'yarn run serve',
+    port: 8080,
+    reuseExistingServer: !process.env.CI,
+  },
+}
+
+export default config

packages/playground/devEnv/vite.config.ts

@@ -3,7 +3,6 @@ import react from '@vitejs/plugin-react'
 import path from 'path'
 import {getAliasesFromTsConfigForRollup} from '../../../devEnv/getAliasesFromTsConfig'
 import {definedGlobals} from '../../../theatre/devEnv/buildUtils'
-import {existsSync, writeFileSync} from 'fs'
 
 /*
 We're using vite instead of the older pure-esbuild setup. The tradeoff is
@@ -18,36 +17,10 @@ const playgroundDir = path.join(__dirname, '..')
 
 const port = 8080
 
-/**
- * Creates playground/src/index.ts, since that file isn't committed to the repo.
- */
-function createPlaygroundIndex() {
-  const playgroundIndexContent = `
-  /**
-   * This file is created automatically and won't be comitted to the repo.
-   * You can change the import statement and import your own playground code.
-   * 
-   * Your own playground code should reside in './personal', which is a folder
-   * that won't be committed to the repo.
-   * 
-   * The shared playgrounds which other contributors can use are in the './shared' folder,
-   * which are comitted to the repo.
-   * 
-   * Happy playing!
-   * */
-  import './shared/r3f-rocket'
-  `
-
-  const playgroundEntry = path.join(playgroundDir, 'src/index.ts')
-  if (!existsSync(playgroundEntry)) {
-    writeFileSync(playgroundEntry, playgroundIndexContent, {encoding: 'utf-8'})
-  }
-}
-createPlaygroundIndex()
-
 // https://vitejs.dev/config/
 export default defineConfig({
   root: path.join(playgroundDir, './src'),
+
   assetsInclude: ['**/*.gltf', '**/*.glb'],
   server: {
     port,
@@ -61,5 +34,5 @@ export default defineConfig({
     */
     alias: [...getAliasesFromTsConfigForRollup()],
   },
-  define: definedGlobals,
+  define: {...definedGlobals, 'window.__IS_VISUAL_REGRESSION_TESTING': 'true'},
 })

packages/playground/package.json

@@ -10,9 +10,14 @@
   "scripts": {
     "serve": "vite --config ./devEnv/vite.config.ts",
     "typecheck": "yarn run build",
+    "test": "playwright test --config=devEnv/playwright.config.ts",
+    "test:ci": "percy exec -- playwright test --reporter=dot --config=devEnv/playwright.config.ts --project=chromium",
     "build": "tsc --build ./tsconfig.json"
   },
   "devDependencies": {
+    "@percy/cli": "^1.0.0-beta.76",
+    "@percy/playwright": "^1.0.1",
+    "@playwright/test": "^1.19.1",
     "@react-three/drei": "^7.2.2",
     "@react-three/fiber": "^7.0.6",
     "@theatre/core": "workspace:*",

packages/playground/src/.gitignore

@@ -1,2 +1 @@
 personal
-index.ts

packages/playground/src/index.html

@@ -8,6 +8,13 @@
         padding: 0;
         height: 100%;
         background: black;
+        color: white;
+        font-family: sans-serif;
+        font-size: 12px;
+      }
+
+      a {
+        color: inherit;
       }
     </style>
   </head>
@@ -15,6 +22,6 @@
   <body>
     <div id="root"></div>
 
-    <script type="module" src="./index.ts"></script>
+    <script type="module" src="./index.tsx"></script>
   </body>
 </html>

packages/playground/src/index.tsx

@@ -0,0 +1,92 @@
+/**
+ * TODO explain this file
+ * */
+/// <reference types="vite/client" />
+
+import type {$FixMe} from '@theatre/shared/utils/types'
+import {mapKeys} from 'lodash-es'
+import React from 'react'
+import ReactDOM from 'react-dom'
+
+const groups = {
+  shared: mapKeys(import.meta.glob('./shared/*/index.tsx'), (_, path) =>
+    pathToModuleName(path),
+  ),
+  personal: mapKeys(import.meta.glob('./personal/*/index.tsx'), (_, path) =>
+    pathToModuleName(path),
+  ),
+  tests: mapKeys(import.meta.glob('./tests/*/index.tsx'), (_, path) =>
+    pathToModuleName(path),
+  ),
+}
+
+function pathToModuleName(path: string): string {
+  const matches = path.match(
+    /^\.\/(shared|personal|tests)\/([a-zA-Z0-9\-\s]+)\/index\.tsx$/,
+  )
+
+  if (!matches) {
+    throw new Error(
+      `module ${path} has invalid characters in its path. Valid names should match the regexp above this line.`,
+    )
+  }
+
+  return matches[2]
+}
+
+const Home = () => (
+  <ul>
+    {Object.entries(groups).map(([groupName, modules]) => (
+      <li key={`li-${groupName}`}>
+        <span>{groupName}</span>
+        <Group
+          key={`group-${groupName}`}
+          groupName={groupName}
+          modules={modules}
+        />
+      </li>
+    ))}
+  </ul>
+)
+
+const Group = (props: {groupName: string; modules: Record<string, $FixMe>}) => {
+  const {groupName, modules} = props
+  return (
+    <ul>
+      {Object.entries(modules).map(([moduleName, callback]) => (
+        <li key={`li-${moduleName}`}>
+          <a href={`/${groupName}/${moduleName}`}>{moduleName}</a>
+          {/* <Group key={`group-${group}`} modules={modules} /> */}
+        </li>
+      ))}
+    </ul>
+  )
+}
+
+const currentPathname = document.location.pathname
+
+if (currentPathname === '/') {
+  renderHome()
+} else {
+  const parts = currentPathname.match(
+    /^\/(shared|personal|tests)\/([a-zA-Z0-9\-]+)$/,
+  )
+  if (parts) {
+    const [, groupName, moduleName] = parts
+    const group = groups[groupName as 'shared' | 'personal']
+    if (!group) {
+      throw new Error(`Unknown group ${groupName}`)
+    }
+    const module = group[moduleName]
+    if (!module) {
+      throw new Error(`Unknown module ${moduleName}`)
+    }
+    module()
+  } else {
+    throw new Error(`Unknown path ${currentPathname}`)
+  }
+}
+
+function renderHome() {
+  ReactDOM.render(React.createElement(Home), document.getElementById('root'))
+}

packages/playground/src/tests/setting-static-props/index.tsx

@@ -0,0 +1,13 @@
+import studio from '@theatre/studio'
+import {getProject} from '@theatre/core'
+
+studio.initialize({usePersistentStorage: false})
+
+const project = getProject('sample project')
+const sheet = project.sheet('sample sheet')
+const obj = sheet.object('sample object', {
+  position: {
+    x: 0,
+    y: 0,
+  },
+})

packages/playground/src/tests/setting-static-props/test.e2e.ts

@@ -0,0 +1,47 @@
+import {test, expect} from '@playwright/test'
+import percySnapshot from '@percy/playwright'
+
+const isMac = process.platform === 'darwin'
+
+test.describe('setting-static-props', () => {
+  test.beforeEach(async ({page}) => {
+    // Go to the starting url before each test.
+    await page.goto('http://localhost:8080/tests/setting-static-props')
+  })
+
+  test('Undo/redo', async ({page}) => {
+    await page.locator('[data-testid="OutlinePanel-TriggerButton"]').click()
+
+    await page.locator('span:has-text("sample object")').first().click()
+
+    const detailPanel = page.locator('[data-testid="DetailPanel-Object"]')
+
+    const firstInput = detailPanel.locator('input[type="text"]').first()
+    // Click input[type="text"] >> nth=0
+    await firstInput.click()
+    // Fill input[type="text"] >> nth=0
+    await firstInput.fill('1')
+    // Press Enter
+    await firstInput.press('Enter')
+    const secondInput = detailPanel.locator('input[type="text"]').nth(1)
+    // Click input[type="text"] >> nth=1
+    await secondInput.click()
+    // Fill input[type="text"] >> nth=1
+    await secondInput.fill('2')
+    // Press Enter
+    await secondInput.press('Enter')
+
+    const metaKey = isMac ? 'Meta' : 'Control'
+
+    // Press z with modifiers
+    await page.locator('body').press(`${metaKey}+z`)
+    await expect(firstInput).toHaveAttribute('value', '1')
+    await expect(secondInput).toHaveAttribute('value', '0')
+    await page.locator('body').press(`${metaKey}+Shift+z`)
+    await expect(firstInput).toHaveAttribute('value', '1')
+    await expect(secondInput).toHaveAttribute('value', '2')
+
+    // Our first visual regression test
+    await percySnapshot(page, test.info().titlePath.join('/') + '/After redo')
+  })
+})

theatre/devEnv/buildUtils.ts

@@ -17,7 +17,7 @@ export function createBundles(watch: boolean) {
       loader: {'.png': 'file', '.svg': 'dataurl'},
       bundle: true,
       sourcemap: true,
-      define: definedGlobals,
+      define: {...definedGlobals, __IS_VISUAL_REGRESSION_TESTING: 'false'},
       watch,
       external: [
         '@theatre/dataverse',

theatre/globals.d.ts

@@ -1,5 +1,6 @@
 interface Window {
   __REDUX_DEVTOOLS_EXTENSION__?: $IntentionalAny
+  __IS_VISUAL_REGRESSION_TESTING?: boolean
 }
 
 interface NodeModule {

theatre/studio/src/UI.ts

@@ -25,12 +25,15 @@ export default class UI {
       pointer-events: none;
       z-index: 100;
     `
-    this.containerShadow = this.containerEl.attachShadow({
+    this.containerShadow =
-      mode: 'open',
+      window.__IS_VISUAL_REGRESSION_TESTING === true
-      // To see why I had to cast this value to HTMLElement, take a look at its
+        ? (document.getElementById('root') as $IntentionalAny)
-      // references of this prop. There are a few functions that actually work
+        : (this.containerEl.attachShadow({
-      // with a ShadowRoot but are typed to accept HTMLElement
+            mode: 'open',
-    }) as $IntentionalAny as ShadowRoot & HTMLElement
+            // To see why I had to cast this value to HTMLElement, take a look at its
+            // references of this prop. There are a few functions that actually work
+            // with a ShadowRoot but are typed to accept HTMLElement
+          }) as $IntentionalAny as ShadowRoot & HTMLElement)
   }
 
   render() {

theatre/studio/src/UIRoot/UIRoot.tsx

@@ -85,7 +85,11 @@ export default function UIRoot() {
     return !initialised ? null : (
       <StyleSheetManager
         disableVendorPrefixes
-        target={getStudio()!.ui.containerShadow}
+        target={
+          window.__IS_VISUAL_REGRESSION_TESTING === true
+            ? undefined
+            : getStudio()!.ui.containerShadow
+        }
       >
         <>
           <GlobalStyle />

theatre/studio/src/panels/DetailPanel/DetailPanel.tsx

@@ -109,7 +109,7 @@ const DetailPanel: React.FC<{}> = (props) => {
     if (obj) {
       return (
         <Container>
-          <Content>
+          <Content data-testid="DetailPanel-Object">
             <Header>
               <Title
                 title={`${obj.sheet.address.sheetId}: ${obj.sheet.address.sheetInstanceId} > ${obj.address.objectKey}`}

theatre/studio/src/panels/OutlinePanel/OutlinePanel.tsx

@@ -166,7 +166,10 @@ const OutlinePanel: React.FC<{}> = (props) => {
     <Container>
       <TriggerContainer>
         {triggerTooltip}
-        <TriggerButton ref={triggerButtonRef as $IntentionalAny}>
+        <TriggerButton
+          ref={triggerButtonRef as $IntentionalAny}
+          data-testid="OutlinePanel-TriggerButton"
+        >
           <VscListTree />
         </TriggerButton>
         {conflicts.length > 0 ? (
@@ -177,7 +180,7 @@ const OutlinePanel: React.FC<{}> = (props) => {
         {/* <Title>Outline</Title> */}
       </TriggerContainer>
       <Content>
-        <Body>
+        <Body data-testid="OutlinePanel-Content">
           <ProjectsList />
         </Body>
       </Content>