diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5b32f68..5ea7039 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -88,12 +88,8 @@ jobs: node-version: ${{ matrix.node-version }} - uses: ./.github/actions/yarn-nm-install - - name: Run e2e tests with percy - uses: percy/exec-action@v0.3.1 - with: - custom-command: 'yarn test:e2e:ci' - env: - PERCY_TOKEN: ${{ secrets.PERCY_TOKEN }} + - name: Run e2e tests + run: yarn test:e2e:ci Compatibility-Tests: runs-on: ubuntu-latest diff --git a/packages/playground/README.md b/packages/playground/README.md index 6e6ad76..4c919c1 100644 --- a/packages/playground/README.md +++ b/packages/playground/README.md @@ -53,6 +53,16 @@ $ yarn playwright codegen http://localhost:8080/tests/[playground-name] # run th ## Visual regression testing -We're currently using [percy](https://percy.io) for visual regression testing. These tests run only the the [CI](../../.github/workflows/main.yml) using [Github actions](https://github.com/theatre-js/theatre/actions). Look at the example at [`src/tests/setting-static-props/test.e2e.ts`](src/tests/setting-static-props/test.e2e.ts) for an example of recording and diffing a screenshot. +Some `.e2e.ts` files also contain visual regression tetst. These tests run only the the [CI](../../.github/workflows/main.yml) using [Github actions](https://github.com/theatre-js/theatre/actions). Look at the example at [`src/tests/setting-static-props/test.e2e.ts`](src/tests/setting-static-props/test.e2e.ts) for an example of recording and diffing a screenshot. -Please note that we haven't figured out the best practices for visual regression testing yet, so if the setup isn't optimal, please let us know. +Note that CI runs the visual regression tests in a linux VM, which is bound to produce a slightly different screenshot than a browser on Mac/Windows. Because of that, we have a `docker-compose.yml` file at the root of the repo which you can use to produce a screenshot in a linux vm. Here is how you can use it: + +```bash +$ cd repo +$ docker-compose up -d # start the linux vm +$ docker-compose exec -it node bash # ssh into the vm +$ yarn +$ yarn test:e2e:ci +``` + +If you're submitting a PR that breaks the visual regression tests and you're not familiar with Docker, simply ask the mainainers to update the screenshots for you. \ No newline at end of file diff --git a/packages/playground/package.json b/packages/playground/package.json index a7007ce..4dc4e97 100644 --- a/packages/playground/package.json +++ b/packages/playground/package.json @@ -17,8 +17,6 @@ "test:ci": "playwright test --reporter=dot --config=devEnv/playwright.config.ts --project=chromium" }, "devDependencies": { - "@percy/cli": "^1.16.0", - "@percy/playwright": "^1.0.4", "@playwright/test": "^1.36.2", "@react-three/drei": "^9.80.1", "@react-three/fiber": "^8.13.6", diff --git a/packages/playground/src/tests/setting-static-props/test.e2e.ts b/packages/playground/src/tests/setting-static-props/test.e2e.ts index ce2933f..d537d51 100644 --- a/packages/playground/src/tests/setting-static-props/test.e2e.ts +++ b/packages/playground/src/tests/setting-static-props/test.e2e.ts @@ -1,5 +1,4 @@ import {test, expect} from '@playwright/test' -import percySnapshot from '@percy/playwright' const isMac = process.platform === 'darwin' @@ -43,8 +42,6 @@ test.describe('setting-static-props', () => { await expect(firstInput).toHaveAttribute('value', '1') await expect(secondInput).toHaveAttribute('value', '2') - // Our first visual regression test - // @ts-ignore - probably percy uses a different version of playwright - await percySnapshot(page, test.info().titlePath.join('/') + '/After redo') + await expect(page).toHaveScreenshot() }) }) diff --git a/packages/playground/src/tests/setting-static-props/test.e2e.ts-snapshots/setting-static-props-Undo-redo-1-chromium-darwin.png b/packages/playground/src/tests/setting-static-props/test.e2e.ts-snapshots/setting-static-props-Undo-redo-1-chromium-darwin.png new file mode 100644 index 0000000..c4cc464 Binary files /dev/null and b/packages/playground/src/tests/setting-static-props/test.e2e.ts-snapshots/setting-static-props-Undo-redo-1-chromium-darwin.png differ diff --git a/packages/playground/src/tests/setting-static-props/test.e2e.ts-snapshots/setting-static-props-Undo-redo-1-chromium-linux.png b/packages/playground/src/tests/setting-static-props/test.e2e.ts-snapshots/setting-static-props-Undo-redo-1-chromium-linux.png new file mode 100644 index 0000000..a94d8f3 Binary files /dev/null and b/packages/playground/src/tests/setting-static-props/test.e2e.ts-snapshots/setting-static-props-Undo-redo-1-chromium-linux.png differ diff --git a/yarn.lock b/yarn.lock index 1044494..06e518e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6823,180 +6823,6 @@ __metadata: languageName: node linkType: hard -"@percy/cli-app@npm:1.16.0": - version: 1.16.0 - resolution: "@percy/cli-app@npm:1.16.0" - dependencies: - "@percy/cli-command": 1.16.0 - "@percy/cli-exec": 1.16.0 - checksum: 90566bf5ecc7ab113f068ec3c097d76d943f24933b52f06f4e47716eb897cbf04b0330594c6387cb1699896e2f473c8d3dd54576a26e1cdd2d7c8caf9cb676fb - languageName: node - linkType: hard - -"@percy/cli-build@npm:1.16.0": - version: 1.16.0 - resolution: "@percy/cli-build@npm:1.16.0" - dependencies: - "@percy/cli-command": 1.16.0 - checksum: 6ef171552ff61a687d91f4f7a2a190168e8ecb167ea5e7f1488510f0d9b570bb692b59bd7184664cd3155a409bc7cff655d2a124b166eef88c0469d12184b9bc - languageName: node - linkType: hard - -"@percy/cli-command@npm:1.16.0": - version: 1.16.0 - resolution: "@percy/cli-command@npm:1.16.0" - dependencies: - "@percy/config": 1.16.0 - "@percy/core": 1.16.0 - "@percy/logger": 1.16.0 - bin: - percy-cli-readme: bin/readme.js - checksum: 06833a9b31578e1985f284b072d0b98964b940c46d83ffa6a9bec4feec753521422b562e9c5f6fe941185567bb6fb3718a4559aa34eca6eeed931e083ad778bd - languageName: node - linkType: hard - -"@percy/cli-config@npm:1.16.0": - version: 1.16.0 - resolution: "@percy/cli-config@npm:1.16.0" - dependencies: - "@percy/cli-command": 1.16.0 - checksum: 187144b81d21af8724f9cfb33995b6ad9736d4fd50987c9223e616766efd44db31cb18e618e387d085e8281e893e5979c200dd0965b2bcd67b78eed26743213f - languageName: node - linkType: hard - -"@percy/cli-exec@npm:1.16.0": - version: 1.16.0 - resolution: "@percy/cli-exec@npm:1.16.0" - dependencies: - "@percy/cli-command": 1.16.0 - cross-spawn: ^7.0.3 - which: ^2.0.2 - checksum: 78b64f36683db44859a0a82bd49984adeb5fd7aaa5105bed4574cc4f7a3a6fcc88bb05c823e1736bf505cd73c04771c7908dcc938f42cfc23bde9ffec4d54172 - languageName: node - linkType: hard - -"@percy/cli-snapshot@npm:1.16.0": - version: 1.16.0 - resolution: "@percy/cli-snapshot@npm:1.16.0" - dependencies: - "@percy/cli-command": 1.16.0 - yaml: ^2.0.0 - checksum: 87461a6036d4e15299bea7cb6c4b64564eace7385eae48e708ed75322ed3fdd2b179a42437e675a025ee0191c943655b718a81d48a3fe5789f71fba0c88f183b - languageName: node - linkType: hard - -"@percy/cli-upload@npm:1.16.0": - version: 1.16.0 - resolution: "@percy/cli-upload@npm:1.16.0" - dependencies: - "@percy/cli-command": 1.16.0 - fast-glob: ^3.2.11 - image-size: ^1.0.0 - checksum: 8bab3fea7583abdc42403034e86e2d52821099749df0452751f24857ededf801ff7a69adf752a807e0d913e51d7503ad05608da7056cf77f2038a4f97bfb4232 - languageName: node - linkType: hard - -"@percy/cli@npm:^1.16.0": - version: 1.16.0 - resolution: "@percy/cli@npm:1.16.0" - dependencies: - "@percy/cli-app": 1.16.0 - "@percy/cli-build": 1.16.0 - "@percy/cli-command": 1.16.0 - "@percy/cli-config": 1.16.0 - "@percy/cli-exec": 1.16.0 - "@percy/cli-snapshot": 1.16.0 - "@percy/cli-upload": 1.16.0 - "@percy/client": 1.16.0 - "@percy/logger": 1.16.0 - bin: - percy: bin/run.cjs - checksum: fb4131670028399d0a4d76d1f6e35ac4691683a7d77ed8d69206ac6ca1e95272df2a5349c72fa4ff96506f8d0e0b74dd94cc1a5ea5fcd97043dba7d2d305772f - languageName: node - linkType: hard - -"@percy/client@npm:1.16.0": - version: 1.16.0 - resolution: "@percy/client@npm:1.16.0" - dependencies: - "@percy/env": 1.16.0 - "@percy/logger": 1.16.0 - checksum: c42ffd61cbd491101bed62d7c0f1dafd4ca40d8c8601cf6dacfcc8cabf626bb828d0ff2eefc9aa77454dd3b0bc79152f2f0895000a5c7d461c69280f68584b58 - languageName: node - linkType: hard - -"@percy/config@npm:1.16.0": - version: 1.16.0 - resolution: "@percy/config@npm:1.16.0" - dependencies: - "@percy/logger": 1.16.0 - ajv: ^8.6.2 - cosmiconfig: ^7.0.0 - yaml: ^2.0.0 - checksum: 652000a556c593a3ed539e21d094cca918342af899ea919b2722d50e5a1a51567637ed0618dc95a1f7f8e465dc5a68d69b9be02efc72f622ee37845d82b25599 - languageName: node - linkType: hard - -"@percy/core@npm:1.16.0": - version: 1.16.0 - resolution: "@percy/core@npm:1.16.0" - dependencies: - "@percy/client": 1.16.0 - "@percy/config": 1.16.0 - "@percy/dom": 1.16.0 - "@percy/logger": 1.16.0 - content-disposition: ^0.5.4 - cross-spawn: ^7.0.3 - extract-zip: ^2.0.1 - fast-glob: ^3.2.11 - micromatch: ^4.0.4 - mime-types: ^2.1.34 - path-to-regexp: ^6.2.0 - rimraf: ^3.0.2 - ws: ^8.0.0 - checksum: a8202db2a65b0dcb16d879f900cdddb5b5cc0138aec88867f54471f0ed52e5debb8b1f43c620a7dc1fdcfefd9f4b7b8457153f4b0bc2643c6b01a2cc2c4cbe24 - languageName: node - linkType: hard - -"@percy/dom@npm:1.16.0": - version: 1.16.0 - resolution: "@percy/dom@npm:1.16.0" - checksum: 433c2823c82ab4aa2636e3b17658df50765733fa6f645a80ede2dd8e3ee0199cbc62c265b5e77f75919cda2304ef071066f8d1be16b713048298bc66acfb94f0 - languageName: node - linkType: hard - -"@percy/env@npm:1.16.0": - version: 1.16.0 - resolution: "@percy/env@npm:1.16.0" - checksum: 88fba7c4be5ce5c5662d20c3d7d4b2c6df66c516a4e0c0c5e0327768d480eeda8490a9503f60dfd8720df0dfa34481633d3c284a12c25cda87cef3e5381a5e3f - languageName: node - linkType: hard - -"@percy/logger@npm:1.16.0": - version: 1.16.0 - resolution: "@percy/logger@npm:1.16.0" - checksum: 172d2271a6ffd04d1990d616067f1440fb0a236b65f1c52072722e1c6e9c308110bcce86f9ccf91266e1857e83a1d40aed2db2f6dfb947efaf5db59004001d99 - languageName: node - linkType: hard - -"@percy/playwright@npm:^1.0.4": - version: 1.0.4 - resolution: "@percy/playwright@npm:1.0.4" - dependencies: - "@percy/sdk-utils": ^1.0.0 - peerDependencies: - playwright-core: ">=1" - checksum: d0969b49208178a7221093517b50b5cee1e3094b89be301dc5f194172f4132b601570a89bdad617830dcfce7599cdbbe23cef4398bc58faeec05ef7b1f222802 - languageName: node - linkType: hard - -"@percy/sdk-utils@npm:^1.0.0": - version: 1.3.0 - resolution: "@percy/sdk-utils@npm:1.3.0" - checksum: 0d3fb3b452c3e008a23a80339b345540b0cfdab6a2db8214f4f72fa38b93806fd9c3dabfaea9bbf09183dee1cfceb4b932055b332bbd517dcdec5204a8f64689 - languageName: node - linkType: hard - "@playwright/test@npm:^1.36.2": version: 1.36.2 resolution: "@playwright/test@npm:1.36.2" @@ -8934,15 +8760,6 @@ __metadata: languageName: node linkType: hard -"@types/yauzl@npm:^2.9.1": - version: 2.9.2 - resolution: "@types/yauzl@npm:2.9.2" - dependencies: - "@types/node": "*" - checksum: dfb49abe82605615712fc694eaa4f7068fe30aa03f38c085e2c2e74408beaad30471d36da9654a811482ece2ea4405575fd99b19c0aa327ed2a9736b554bbf43 - languageName: node - linkType: hard - "@typescript-eslint/eslint-plugin@npm:^4.5.0": version: 4.29.0 resolution: "@typescript-eslint/eslint-plugin@npm:4.29.0" @@ -10313,18 +10130,6 @@ __metadata: languageName: node linkType: hard -"ajv@npm:^8.6.2": - version: 8.10.0 - resolution: "ajv@npm:8.10.0" - dependencies: - fast-deep-equal: ^3.1.1 - json-schema-traverse: ^1.0.0 - require-from-string: ^2.0.2 - uri-js: ^4.2.2 - checksum: 3594728ef1e31219ef97bfacb203d0d72db8ad5c35d6d0578e38ee453e4537c2bf927dad144bb84b0c893f661d71b58337d4643e8ee2f2a6e1d63b041c92fe82 - languageName: node - linkType: hard - "alphanum-sort@npm:^1.0.0": version: 1.0.2 resolution: "alphanum-sort@npm:1.0.2" @@ -12064,13 +11869,6 @@ __metadata: languageName: node linkType: hard -"buffer-crc32@npm:~0.2.3": - version: 0.2.13 - resolution: "buffer-crc32@npm:0.2.13" - checksum: 06252347ae6daca3453b94e4b2f1d3754a3b146a111d81c68924c22d91889a40623264e95e67955b1cb4a68cbedf317abeabb5140a9766ed248973096db5ce1c - languageName: node - linkType: hard - "buffer-equal-constant-time@npm:1.0.1": version: 1.0.1 resolution: "buffer-equal-constant-time@npm:1.0.1" @@ -13115,7 +12913,7 @@ __metadata: languageName: node linkType: hard -"content-disposition@npm:0.5.4, content-disposition@npm:^0.5.4": +"content-disposition@npm:0.5.4": version: 0.5.4 resolution: "content-disposition@npm:0.5.4" dependencies: @@ -17256,23 +17054,6 @@ __metadata: languageName: node linkType: hard -"extract-zip@npm:^2.0.1": - version: 2.0.1 - resolution: "extract-zip@npm:2.0.1" - dependencies: - "@types/yauzl": ^2.9.1 - debug: ^4.1.1 - get-stream: ^5.1.0 - yauzl: ^2.10.0 - dependenciesMeta: - "@types/yauzl": - optional: true - bin: - extract-zip: cli.js - checksum: 8cbda9debdd6d6980819cc69734d874ddd71051c9fe5bde1ef307ebcedfe949ba57b004894b585f758b7c9eeeea0e3d87f2dda89b7d25320459c2c9643ebb635 - languageName: node - linkType: hard - "extsprintf@npm:1.3.0": version: 1.3.0 resolution: "extsprintf@npm:1.3.0" @@ -17436,15 +17217,6 @@ __metadata: languageName: node linkType: hard -"fd-slicer@npm:~1.1.0": - version: 1.1.0 - resolution: "fd-slicer@npm:1.1.0" - dependencies: - pend: ~1.2.0 - checksum: c8585fd5713f4476eb8261150900d2cb7f6ff2d87f8feb306ccc8a1122efd152f1783bdb2b8dc891395744583436bfd8081d8e63ece0ec8687eeefea394d4ff2 - languageName: node - linkType: hard - "fetch-blob@npm:^3.1.2, fetch-blob@npm:^3.1.4": version: 3.1.5 resolution: "fetch-blob@npm:3.1.5" @@ -18082,7 +17854,7 @@ fsevents@^1.2.7: languageName: node linkType: hard -"get-stream@npm:^5.0.0, get-stream@npm:^5.1.0": +"get-stream@npm:^5.0.0": version: 5.2.0 resolution: "get-stream@npm:5.2.0" dependencies: @@ -19230,17 +19002,6 @@ fsevents@^1.2.7: languageName: node linkType: hard -"image-size@npm:^1.0.0": - version: 1.0.1 - resolution: "image-size@npm:1.0.1" - dependencies: - queue: 6.0.2 - bin: - image-size: bin/image-size.js - checksum: ffa74672dc7a1b6529c66255adbfe4e7865408004db88ed100855816f03175494ec21ef9dad199b8685b5b194996ebe83ab27803af152adb66a301172fdd622d - languageName: node - linkType: hard - "immediate@npm:~3.0.5": version: 3.0.6 resolution: "immediate@npm:3.0.6" @@ -23205,13 +22966,6 @@ fsevents@^1.2.7: languageName: node linkType: hard -"mime-db@npm:1.51.0": - version: 1.51.0 - resolution: "mime-db@npm:1.51.0" - checksum: 613b1ac9d6e725cc24444600b124a7f1ce6c60b1baa654f39a3e260d0995a6dffc5693190217e271af7e2a5612dae19f2a73f3e316707d797a7391165f7ef423 - languageName: node - linkType: hard - "mime-db@npm:1.52.0": version: 1.52.0 resolution: "mime-db@npm:1.52.0" @@ -23237,15 +22991,6 @@ fsevents@^1.2.7: languageName: node linkType: hard -"mime-types@npm:^2.1.34": - version: 2.1.34 - resolution: "mime-types@npm:2.1.34" - dependencies: - mime-db: 1.51.0 - checksum: 67013de9e9d6799bde6d669d18785b7e18bcd212e710d3e04a4727f92f67a8ad4e74aee24be28b685adb794944814bde649119b58ee3282ffdbee58f9278d9f3 - languageName: node - linkType: hard - "mime-types@npm:~2.1.17, mime-types@npm:~2.1.24": version: 2.1.32 resolution: "mime-types@npm:2.1.32" @@ -25035,13 +24780,6 @@ fsevents@^1.2.7: languageName: node linkType: hard -"path-to-regexp@npm:^6.2.0": - version: 6.2.0 - resolution: "path-to-regexp@npm:6.2.0" - checksum: a6aca74d2d6e2e7594d812f653cf85e9cb5054d3a8d80f099722a44ef6ad22639b02078e5ea83d11db16321c3e4359e3f1ab0274fa78dad0754a6e53f630b0fc - languageName: node - linkType: hard - "path-type@npm:^3.0.0": version: 3.0.0 resolution: "path-type@npm:3.0.0" @@ -25071,13 +24809,6 @@ fsevents@^1.2.7: languageName: node linkType: hard -"pend@npm:~1.2.0": - version: 1.2.0 - resolution: "pend@npm:1.2.0" - checksum: 6c72f5243303d9c60bd98e6446ba7d30ae29e3d56fdb6fae8767e8ba6386f33ee284c97efe3230a0d0217e2b1723b8ab490b1bbf34fcbb2180dbc8a9de47850d - languageName: node - linkType: hard - "performance-now@npm:^2.1.0": version: 2.1.0 resolution: "performance-now@npm:2.1.0" @@ -25302,8 +25033,6 @@ fsevents@^1.2.7: version: 0.0.0-use.local resolution: "playground@workspace:packages/playground" dependencies: - "@percy/cli": ^1.16.0 - "@percy/playwright": ^1.0.4 "@playwright/test": ^1.36.2 "@react-three/drei": ^9.80.1 "@react-three/fiber": ^8.13.6 @@ -27611,15 +27340,6 @@ fsevents@^1.2.7: languageName: node linkType: hard -"queue@npm:6.0.2": - version: 6.0.2 - resolution: "queue@npm:6.0.2" - dependencies: - inherits: ~2.0.3 - checksum: ebc23639248e4fe40a789f713c20548e513e053b3dc4924b6cb0ad741e3f264dcff948225c8737834dd4f9ec286dbc06a1a7c13858ea382d9379f4303bcc0916 - languageName: node - linkType: hard - "quick-format-unescaped@npm:^4.0.3": version: 4.0.4 resolution: "quick-format-unescaped@npm:4.0.4" @@ -34098,21 +33818,6 @@ fsevents@^1.2.7: languageName: node linkType: hard -"ws@npm:^8.0.0, ws@npm:^8.4.2": - version: 8.5.0 - resolution: "ws@npm:8.5.0" - peerDependencies: - bufferutil: ^4.0.1 - utf-8-validate: ^5.0.2 - peerDependenciesMeta: - bufferutil: - optional: true - utf-8-validate: - optional: true - checksum: 76f2f90e40344bf18fd544194e7067812fb1372b2a37865678d8f12afe4b478ff2ebc0c7c0aff82cd5e6b66fc43d889eec0f1865c2365d8f7a66d92da7744a77 - languageName: node - linkType: hard - "ws@npm:^8.11.0": version: 8.11.0 resolution: "ws@npm:8.11.0" @@ -34128,6 +33833,21 @@ fsevents@^1.2.7: languageName: node linkType: hard +"ws@npm:^8.4.2": + version: 8.5.0 + resolution: "ws@npm:8.5.0" + peerDependencies: + bufferutil: ^4.0.1 + utf-8-validate: ^5.0.2 + peerDependenciesMeta: + bufferutil: + optional: true + utf-8-validate: + optional: true + checksum: 76f2f90e40344bf18fd544194e7067812fb1372b2a37865678d8f12afe4b478ff2ebc0c7c0aff82cd5e6b66fc43d889eec0f1865c2365d8f7a66d92da7744a77 + languageName: node + linkType: hard + "xml-name-validator@npm:^3.0.0": version: 3.0.0 resolution: "xml-name-validator@npm:3.0.0" @@ -34191,13 +33911,6 @@ fsevents@^1.2.7: languageName: node linkType: hard -"yaml@npm:^2.0.0": - version: 2.1.0 - resolution: "yaml@npm:2.1.0" - checksum: 59323a8b51b10d9ad0eab951e7a1f315f1076c123b08ffe60441add1df4fa3433b1d5783b21c50a65536e9d853b23fa567921dbd4bc0d711be2dbf14a06be03b - languageName: node - linkType: hard - "yaml@npm:^2.0.1": version: 2.1.3 resolution: "yaml@npm:2.1.3" @@ -34320,16 +34033,6 @@ fsevents@^1.2.7: languageName: node linkType: hard -"yauzl@npm:^2.10.0": - version: 2.10.0 - resolution: "yauzl@npm:2.10.0" - dependencies: - buffer-crc32: ~0.2.3 - fd-slicer: ~1.1.0 - checksum: 7f21fe0bbad6e2cb130044a5d1d0d5a0e5bf3d8d4f8c4e6ee12163ce798fee3de7388d22a7a0907f563ac5f9d40f8699a223d3d5c1718da90b0156da6904022b - languageName: node - linkType: hard - "yocto-queue@npm:^0.1.0": version: 0.1.0 resolution: "yocto-queue@npm:0.1.0"