[web-php] master: Add visual regression tests (#994)

Author: Sergey Panteleev (saundefined) Committer: GitHub (web-flow) Pusher: saundefined Date: 2024-08-20T09:20:25+03:00 Commit: https://github.com/php/web-php/commit/c632d199d1a4916b815113fa81fd5a60deb1e626 Raw diff: https://github.com/php/web-php/commit/c632d199d1a4916b815113fa81fd5a60deb1e626.diff Add visual regression tests (#994) Co-authored-by: Roman Pronskiy <roman@pronskiy.com> Co-authored-by: Derick Rethans <derick@php.net> Changed paths: A .github/workflows/update-screenshots.yaml A package.json A playwright.config.ts A tests/Visual/SmokeTest.spec.ts A tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-archive-1998-php-chromium.png A tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-conferences-index-php-chromium.png A tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-index-php-chromium.png A tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-manual-index-php-chromium.png A tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-manual-php5-php-chromium.png A tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-releases-8-0-index-php-chromium.png A tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-releases-8-1-index-php-chromium.png A tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-releases-8-2-index-php-chromium.png A tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-releases-8-3-6-php-chromium.png A tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-releases-8-3-index-php-chromium.png A yarn.lock M .github/workflows/pr-closed.yml M .github/workflows/pr-preview.yml M .gitignore M .htaccess M Makefile Diff: diff --git a/.github/workflows/pr-closed.yml b/.github/workflows/pr-closed.yml index a745b6e0ef..a3823183b2 100644 --- a/.github/workflows/pr-closed.yml +++ b/.github/workflows/pr-closed.yml @@ -13,4 +13,6 @@ jobs: host: ${{ secrets.PREVIEW_REMOTE_HOST }} username: ${{ secrets.PREVIEW_REMOTE_USER }} key: ${{ secrets.PREVIEW_SSH_KEY }} - script: bash /home/thephpfoundation/scripts/pr_closed.sh web-php ${{ github.event.number }} + script: | + bash /home/thephpfoundation/scripts/pr_closed.sh web-php ${{ github.event.number }} + bash /home/thephpfoundation/scripts/pr_closed.sh web-php-regression-report ${{ github.event.number }} diff --git a/.github/workflows/pr-preview.yml b/.github/workflows/pr-preview.yml index 2c62f78f67..8292a59a2a 100644 --- a/.github/workflows/pr-preview.yml +++ b/.github/workflows/pr-preview.yml @@ -32,4 +32,99 @@ jobs: comment-id: ${{ steps.fc.outputs.comment-id }} edit-mode: 'replace' body: | - 🚀 Commit ${{ github.sha }} Deployed on https://web-php-pr-${{ github.event.number }}.preview.thephp.foundation + 🚀 Preview for commit ${{ github.sha }} can be found at https://web-php-pr-${{ github.event.number }}.preview.thephp.foundation + + tests_visual: + name: "Visual Tests" + + runs-on: "ubuntu-latest" + if: github.repository_owner == 'php' + + strategy: + matrix: + php-version: + - "8.2" + node-version: + - "22.x" + + env: + HTTP_HOST: "localhost:8080" + + steps: + - name: "Checkout" + uses: actions/checkout@v4 + with: + ref: "refs/pull/${{ github.event.number }}/merge" + + - name: "Set up PHP" + uses: "shivammathur/setup-php@v2" + with: + coverage: "none" + extensions: "none, curl, dom, json, mbstring, tokenizer, xml, xmlwriter" + php-version: "${{ matrix.php-version }}" + + - name: Use Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v3 + with: + node-version: ${{ matrix.node-version }} + + - name: "Set up problem matchers for PHP" + run: "echo \"::add-matcher::${{ runner.tool_cache }}/php.json\"" + + - name: "Set up problem matchers for phpunit/phpunit" + run: "echo \"::add-matcher::${{ runner.tool_cache }}/phpunit.json\"" + + - name: "Determine composer cache directory" + run: "echo \"COMPOSER_CACHE_DIR=$(composer config cache-dir)\" >> $GITHUB_ENV" + + - name: "Cache dependencies installed with composer" + uses: "actions/cache@v3" + with: + path: "${{ env.COMPOSER_CACHE_DIR }}" + key: "php-${{ matrix.php-version }}-composer-${{ hashFiles('composer.lock') }}" + restore-keys: "php-${{ matrix.php-version }}-composer-" + + - name: "Install dependencies with composer" + run: "composer install --ansi --no-interaction --no-progress" + + - name: "Install dependencies" + run: "yarn install" + + - name: "Install Playwright" + run: "npx playwright install" + + - name: "Run visual tests" + run: "make tests_visual" + + - uses: actions/upload-artifact@v4 + if: ${{ !cancelled() }} + with: + name: playwright-report + path: playwright-report/ + retention-days: 30 + + - uses: easingthemes/ssh-deploy@main + if: ${{ !cancelled() }} + with: + REMOTE_HOST: ${{ secrets.PREVIEW_REMOTE_HOST }} + REMOTE_USER: ${{ secrets.PREVIEW_REMOTE_USER }} + SSH_PRIVATE_KEY: ${{ secrets.PREVIEW_SSH_KEY }} + SOURCE: "playwright-report/" + TARGET: "/home/thephpfoundation/preview/web-php-regression-report-pr-${{ github.event.number }}/public" + SCRIPT_BEFORE: bash /home/thephpfoundation/scripts/pr_created_pre.sh web-php-regression-report ${{ github.event.number }} + + - uses: peter-evans/find-comment@v3 + if: ${{ !cancelled() }} + id: snapshot + with: + issue-number: ${{ github.event.number }} + comment-author: 'github-actions[bot]' + + - uses: peter-evans/create-or-update-comment@v4 + if: ${{ !cancelled() }} + with: + issue-number: ${{ github.event.number }} + comment-id: ${{ steps.snapshot.outputs.comment-id }} + edit-mode: 'replace' + body: | + 🚀 Regression report for commit ${{ github.sha }} is at https://web-php-regression-report-pr-${{ github.event.number }}.preview.thephp.foundation diff --git a/.github/workflows/update-screenshots.yaml b/.github/workflows/update-screenshots.yaml new file mode 100644 index 0000000000..16e03ccc8f --- /dev/null +++ b/.github/workflows/update-screenshots.yaml @@ -0,0 +1,74 @@ +# https://docs.github.com/en/actions + +name: "Update screenshots" + +on: + workflow_dispatch: + +jobs: + tests_update_snapshots: + name: "Update Tests snapshots" + + runs-on: "ubuntu-latest" + + strategy: + matrix: + php-version: + - "8.2" + node-version: + - "22.x" + + env: + HTTP_HOST: "localhost:8080" + + steps: + - name: "Checkout" + uses: "actions/checkout@v4" + + - name: "Set up PHP" + uses: "shivammathur/setup-php@v2" + with: + coverage: "none" + extensions: "none, curl, dom, json, mbstring, tokenizer, xml, xmlwriter" + php-version: "${{ matrix.php-version }}" + + - name: Use Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v3 + with: + node-version: ${{ matrix.node-version }} + + - name: "Set up problem matchers for PHP" + run: "echo \"::add-matcher::${{ runner.tool_cache }}/php.json\"" + + - name: "Set up problem matchers for phpunit/phpunit" + run: "echo \"::add-matcher::${{ runner.tool_cache }}/phpunit.json\"" + + - name: "Determine composer cache directory" + run: "echo \"COMPOSER_CACHE_DIR=$(composer config cache-dir)\" >> $GITHUB_ENV" + + - name: "Cache dependencies installed with composer" + uses: "actions/cache@v3" + with: + path: "${{ env.COMPOSER_CACHE_DIR }}" + key: "php-${{ matrix.php-version }}-composer-${{ hashFiles('composer.lock') }}" + restore-keys: "php-${{ matrix.php-version }}-composer-" + + - name: "Install dependencies with composer" + run: "composer install --ansi --no-interaction --no-progress" + + - name: "Install dependencies" + run: "yarn install" + + - name: "Install Playwright" + run: "npx playwright install" + + - name: "Run visual tests" + run: "make tests_update_snapshots" + + - name: Update snapshots + uses: test-room-7/action-update-file@v1 + if: ${{ !cancelled() }} + with: + file-path: tests/Visual/**/* + commit-msg: Update snapshots + github-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore index b491def39c..5fa5cd36ef 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,8 @@ backend/mirror.gif backend/mirror.png backend/mirror.jpg backend/GeoIP.dat +node_modules/ +/test-results/ +/playwright-report/ +/blob-report/ +/playwright/.cache/ diff --git a/.htaccess b/.htaccess index 912ca12580..918a3f818e 100644 --- a/.htaccess +++ b/.htaccess @@ -1,4 +1,4 @@ -<FilesMatch "\.(inc|sql)$"> +<FilesMatch "\.(inc|sql|lock)$"> Order allow,deny Deny from all </FilesMatch> diff --git a/Makefile b/Makefile index f70a2fe662..2d6e571924 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,7 @@ .EXPORT_ALL_VARIABLES: HTTP_HOST:=localhost:8080 +CORES?=$(shell (nproc || sysctl -n hw.ncpu) 2> /dev/null) .PHONY: it it: coding-standards tests ## Runs all the targets @@ -21,7 +22,19 @@ help: ## Displays this list of targets with descriptions tests: vendor ## Runs unit and end-to-end tests with phpunit/phpunit vendor/bin/phpunit --configuration=tests/phpunit.xml --testsuite=unit rm -rf tests/server.log - tests/server start; vendor/bin/phpunit --configuration=tests/phpunit.xml --testsuite=end-to-end; tests/server stop + tests/server start; + vendor/bin/phpunit --configuration=tests/phpunit.xml --testsuite=end-to-end; + tests/server stop + +tests_visual: + tests/server start; + npx playwright test --workers=$(CORES) + tests/server stop + +tests_update_snapshots: + tests/server start; + npx playwright test --update-snapshots + tests/server stop vendor: composer.json composer.lock composer validate --strict diff --git a/package.json b/package.json new file mode 100644 index 0000000000..6678d5ccce --- /dev/null +++ b/package.json @@ -0,0 +1,8 @@ +{ + "name": "php.net", + "devDependencies": { + "@playwright/test": "^1.44.0", + "@types/node": "^20.12.11" + }, + "scripts": {} +} diff --git a/playwright.config.ts b/playwright.config.ts new file mode 100644 index 0000000000..8951edeead --- /dev/null +++ b/playwright.config.ts @@ -0,0 +1,34 @@ +import {defineConfig, devices} from '@playwright/test'; + +/** + * See https://playwright.dev/docs/test-configuration. + */ +export default defineConfig({ + testDir: './tests/Visual', + /* Run tests in files in parallel */ + fullyParallel: true, + /* Fail the build on CI if you accidentally left test.only in the source code. */ + forbidOnly: !!process.env.CI, + /* Retry on CI only */ + retries: process.env.CI ? 2 : 0, + /* Opt out of parallel tests on CI. */ + workers: process.env.CI ? 1 : undefined, + /* Reporter to use. See https://playwright.dev/docs/test-reporters */ + reporter: 'html', + /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ + use: { + /* Base URL to use in actions like await page.goto('/'). */ + // baseURL: 'http://127.0.0.1:3000', + + /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */ + trace: 'on-first-retry', + }, + timeout: 0, + + projects: [ + { + name: 'chromium', + use: {...devices['Desktop Chrome']}, + }, + ], +}); diff --git a/tests/Visual/SmokeTest.spec.ts b/tests/Visual/SmokeTest.spec.ts new file mode 100644 index 0000000000..c475e26d8c --- /dev/null +++ b/tests/Visual/SmokeTest.spec.ts @@ -0,0 +1,63 @@ +import {expect, test} from '@playwright/test'; + +export type TestPageOptions = { + path: string + options?: object + evaluate?: () => any +} + +const items: TestPageOptions[] = [ + { + path: 'index.php', + evaluate: () => { + const selector = document.querySelector('.elephpants'); + selector.remove() + } + }, + { + path: 'archive/1998.php', + evaluate: () => { + const selector = document.querySelector('.elephpants'); + selector.remove() + } + }, + {path: 'releases/8_3_6.php'}, + {path: 'releases/8.0/index.php'}, + {path: 'releases/8.1/index.php'}, + {path: 'releases/8.2/index.php'}, + {path: 'releases/8.3/index.php'}, + {path: 'manual/index.php'}, + {path: 'manual/php5.php'}, + { + path: 'conferences/index.php', + options: { + fullPage: false, + } + }, +] + +for (const item of items) { + test(testing with ${item.path}, async ({page}, testInfo) => { + testInfo.snapshotSuffix = ''; + + const httpHost = process.env.HTTP_HOST + + if (typeof httpHost !== 'string') { + throw new Error('Environment variable "HTTP_HOST" is not set.') + } + + await page.goto(${httpHost}/${item.path}) + + if (typeof item.evaluate === 'function') { + await page.evaluate(item.evaluate) + } + + await expect(page).toHaveScreenshot( + tests/screenshots/${item.path}.png, + item.options ?? { + fullPage: true, + timeout: 10000, + } + ) + }) +} diff --git a/tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-archive-1998-php-chromium.png b/tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-archive-1998-php-chromium.png new file mode 100644 index 0000000000..9148879646 Binary files /dev/null and b/tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-archive-1998-php-chromium.png differ diff --git a/tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-conferences-index-php-chromium.png b/tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-conferences-index-php-chromium.png new file mode 100644 index 0000000000..87f7701330 Binary files /dev/null and b/tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-conferences-index-php-chromium.png differ diff --git a/tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-index-php-chromium.png b/tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-index-php-chromium.png new file mode 100644 index 0000000000..6bbd63cf22 Binary files /dev/null and b/tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-index-php-chromium.png differ diff --git a/tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-manual-index-php-chromium.png b/tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-manual-index-php-chromium.png new file mode 100644 index 0000000000..0a0668de56 Binary files /dev/null and b/tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-manual-index-php-chromium.png differ diff --git a/tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-manual-php5-php-chromium.png b/tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-manual-php5-php-chromium.png new file mode 100644 index 0000000000..0ba35d00dd Binary files /dev/null and b/tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-manual-php5-php-chromium.png differ diff --git a/tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-releases-8-0-index-php-chromium.png b/tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-releases-8-0-index-php-chromium.png new file mode 100644 index 0000000000..41c4149248 Binary files /dev/null and b/tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-releases-8-0-index-php-chromium.png differ diff --git a/tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-releases-8-1-index-php-chromium.png b/tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-releases-8-1-index-php-chromium.png new file mode 100644 index 0000000000..a8fa6a0645 Binary files /dev/null and b/tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-releases-8-1-index-php-chromium.png differ diff --git a/tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-releases-8-2-index-php-chromium.png b/tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-releases-8-2-index-php-chromium.png new file mode 100644 index 0000000000..fc649a81b9 Binary files /dev/null and b/tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-releases-8-2-index-php-chromium.png differ diff --git a/tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-releases-8-3-6-php-chromium.png b/tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-releases-8-3-6-php-chromium.png new file mode 100644 index 0000000000..438aafe319 Binary files /dev/null and b/tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-releases-8-3-6-php-chromium.png differ diff --git a/tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-releases-8-3-index-php-chromium.png b/tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-releases-8-3-index-php-chromium.png new file mode 100644 index 0000000000..b32afbee0a Binary files /dev/null and b/tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-releases-8-3-index-php-chromium.png differ diff --git a/yarn.lock b/yarn.lock new file mode 100644 index 0000000000..1f195cd592 --- /dev/null +++ b/yarn.lock @@ -0,0 +1,41 @@ +# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. +# yarn lockfile v1 + + +"@playwright/test@^1.44.0": + version "1.44.0" + resolved "https://registry.yarnpkg.com/@playwright/test/-/test-1.44.0.tgz#ac7a764b5ee6a80558bdc0fcbc525fcb81f83465" + integrity sha512-rNX5lbNidamSUorBhB4XZ9SQTjAqfe5M+p37Z8ic0jPFBMo5iCtQz1kRWkEMg+rYOKSlVycpQmpqjSFq7LXOfg== + dependencies: + playwright "1.44.0" + +"@types/node@^20.12.11": + version "20.12.11" + resolved "https://registry.yarnpkg.com/@types/node/-/node-20.12.11.tgz#c4ef00d3507000d17690643278a60dc55a9dc9be" + integrity sha512-vDg9PZ/zi+Nqp6boSOT7plNuthRugEKixDv5sFTIpkE89MmNtEArAShI4mxuX2+UrLEe9pxC1vm2cjm9YlWbJw== + dependencies: + undici-types "~5.26.4" + +fsevents@2.3.2: + version "2.3.2" + resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.2.tgz#8a526f78b8fdf4623b709e0b975c52c24c02fd1a" + integrity sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA== + +playwright-core@1.44.0: + version "1.44.0" + resolved "https://registry.yarnpkg.com/playwright-core/-/playwright-core-1.44.0.tgz#316c4f0bca0551ffb88b6eb1c97bc0d2d861b0d5" + integrity sha512-ZTbkNpFfYcGWohvTTl+xewITm7EOuqIqex0c7dNZ+aXsbrLj0qI8XlGKfPpipjm0Wny/4Lt4CJsWJk1stVS5qQ== + +playwright@1.44.0: + version "1.44.0" + resolved "https://registry.yarnpkg.com/playwright/-/playwright-1.44.0.tgz#22894e9b69087f6beb639249323d80fe2b5087ff" + integrity sha512-F9b3GUCLQ3Nffrfb6dunPOkE5Mh68tR7zN32L4jCk4FjQamgesGay7/dAAe1WaMEGV04DkdJfcJzjoCKygUaRQ== + dependencies: + playwright-core "1.44.0" + optionalDependencies: + fsevents "2.3.2" + +undici-types@~5.26.4: + version "5.26.5" + resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-5.26.5.tgz#bcd539893d00b56e964fd2657a4866b221a65617" + integrity sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==