feat: fix and migrate smoke tests to Playwright (#1961)

* feat: migrate e2e tests to Playwright * fix: OS dependency * fix: race condition * fix: increased launch timeout
5 years ago · 1dd0635db4
parent f6505d22f2
commit 1dd0635db4
6 changed files with 192 additions and 150 deletions
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@ -2,6 +2,9 @@ name: Build

 on:
  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
  schedule:
    - cron:  '0 8 * * *'

@ -12,6 +15,7 @@ jobs:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v1
+      - uses: microsoft/playwright-github-action@v1
      - name: Set up Node.js
        uses: actions/setup-node@v1
        with:
@ -40,10 +44,11 @@ jobs:
        run: |
          npm run bundle
      - name: Build Tests
-        run: |
-          npm run build-test
+        run: npm run build-test
      - name: Run Smoke Test
        run: |
          npm run ciserver &
          sleep 10
-          npm run test
+          BROWSER=chromium npm run test
+          BROWSER=firefox npm run test
+          BROWSER=webkit npm run test
--- a/.gitignore
+++ b/.gitignore
@ -6,4 +6,5 @@ dist/*.js
 dist/fonts/*
 out-ci/
 .DS_Store
-vscode
+vscode
+/typings-test
--- a/ci/core.test.ts
+++ b/ci/core.test.ts
@ -1,35 +1,48 @@
-import * as puppeteer from 'puppeteer';
+import * as playwright from 'playwright';
 import { assert } from 'chai';

 const APP = 'http://127.0.0.1:8080/dist/core.html';

-let browser: puppeteer.Browser;
-let page: puppeteer.Page;
-const width = 800;
-const height = 600;
+let browser: playwright.Browser;
+let page: playwright.Page;

-describe('Basic loading', function (): void {
-    this.timeout(20000);
+type BrowserType = "chromium" | "firefox" | "webkit"

-    after(() => {
-        browser.close();
-    });
+const browserType: BrowserType = process.env.BROWSER as BrowserType || "chromium"

-    it('should fail because page has an error', async () => {
-        browser = await puppeteer.launch({
-            headless: process.argv.indexOf('--headless') !== -1,
-            args: [`--window-size=${width},${height}`, `--no-sandbox`]
-        });
+before(async () => {
+    console.log(`Starting browser: ${browserType}`)
+    browser = await playwright[browserType].launch({
+        headless: process.argv.includes('--headless'),
+    });
+});
+after(async () => {
+    await browser.close();
+});
+beforeEach(async function () {
+    this.timeout(5 * 1000)
+    page = await browser.newPage({
+        viewport: {
+            width: 800,
+            height: 600
+        }
+    });
+});
+afterEach(async () => {
+    await page.close();
+});

-        page = (await browser.pages())[0];
+describe('Basic loading', function (): void {
+    this.timeout(20000);

+    it('should fail because page has an error', async () => {
        const pageErrors: any[] = [];
        page.on('pageerror', (e) => {
            console.log(e);
            pageErrors.push(e);
        });

-        page.on('error', (e) => {
+        page.on('pageerror', (e) => {
            console.log(e);
            pageErrors.push(e);
        });
@ -46,19 +59,6 @@ describe('Basic loading', function (): void {
 describe('API Integration Tests', function (): void {
    this.timeout(20000);

-    before(async function (): Promise<any> {
-        browser = await puppeteer.launch({
-            headless: process.argv.indexOf('--headless') !== -1,
-            args: [`--window-size=${width},${height}`, `--no-sandbox`]
-        });
-        page = (await browser.pages())[0];
-        await page.setViewport({ width, height });
-    });
-
-    after(() => {
-        browser.close();
-    });
-
    beforeEach(async () => {
        await page.goto(APP);
    });
@ -88,7 +88,7 @@ describe('API Integration Tests', function (): void {
            instance.trigger('keyboard', 'type', {
                text: 'a'
            });
-            instance.trigger('keyboard', 'undo');
+            instance.getModel().undo();
        })()
        `);
        assert.equal(await page.evaluate(`instance.getModel().getLineContent(1)`), 'from banana import *');
@ -110,7 +110,7 @@ describe('API Integration Tests', function (): void {
        })()
        `);

-        await page.waitFor(1000);
+        await page.waitForTimeout(1000);

        assert.deepEqual(await page.evaluate(`
            [
@ -123,13 +123,13 @@ describe('API Integration Tests', function (): void {
                instance.getModel().getLineContent(7),
            ]
        `), [
-                '# from banana import *',
-                '# ',
-                '# class Monkey:',
-                '# 	# Bananas the monkey can eat.',
-                '# 	capacity = 10',
-                '# 	def eat(self, N):',
-                '\t\t\'\'\'Make the monkey eat N bananas!\'\'\''
+            '# from banana import *',
+            '# ',
+            '# class Monkey:',
+            '# 	# Bananas the monkey can eat.',
+            '# 	capacity = 10',
+            '# 	def eat(self, N):',
+            '\t\t\'\'\'Make the monkey eat N bananas!\'\'\''
        ]);
    });
 });
--- a/ci/tsconfig.json
+++ b/ci/tsconfig.json
@ -7,7 +7,8 @@
        "rootDir": ".",
        "outDir": "../out-ci/",
        "types": [
-            "../node_modules/@types/mocha"
+            "../node_modules/@types/mocha",
+            "../node_modules/@types/node"
        ],
        "sourceMap": true,
        "removeComments": true,
--- a/package-lock.json
+++ b/package-lock.json
@ -26,13 +26,15 @@
      "version": "13.9.1",
      "resolved": "https://registry.npmjs.org/@types/node/-/node-13.9.1.tgz",
      "integrity": "sha512-E6M6N0blf/jiZx8Q3nb0vNaswQeEyn0XlupO+xN6DtJ6r6IT4nXrTry7zhIfYvFCl3/8Cu6WIysmUBKiqV0bqQ==",
-      "dev": true
+      "dev": true,
+      "optional": true
    },
-    "@types/puppeteer": {
-      "version": "1.20.4",
-      "resolved": "https://registry.npmjs.org/@types/puppeteer/-/puppeteer-1.20.4.tgz",
-      "integrity": "sha512-T/kFgyLnYWk0H94hxI0HbOLnqHvzBRpfS0F0oo9ESGI24oiC2fEjDcMbBjuK3wH7VLsaIsp740vVXVzR1dsMNg==",
+    "@types/yauzl": {
+      "version": "2.9.1",
+      "resolved": "https://registry.npmjs.org/@types/yauzl/-/yauzl-2.9.1.tgz",
+      "integrity": "sha512-A1b8SU4D10uoPjwb0lnHmmu8wZhR9d+9o2PKBQT2jU5YPTKsxac6M2qGAdY7VcL+dHHhARVUDmeg0rOrcd9EjA==",
      "dev": true,
+      "optional": true,
      "requires": {
        "@types/node": "*"
      }
@ -887,6 +889,12 @@
        "isarray": "^1.0.0"
      }
    },
+    "buffer-crc32": {
+      "version": "0.2.13",
+      "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz",
+      "integrity": "sha1-DTM+PwDqxQqhRUq9MO+MKl2ackI=",
+      "dev": true
+    },
    "buffer-equal": {
      "version": "1.0.0",
      "resolved": "https://registry.npmjs.org/buffer-equal/-/buffer-equal-1.0.0.tgz",
@ -2068,18 +2076,6 @@
        }
      }
    },
-    "extract-zip": {
-      "version": "1.6.7",
-      "resolved": "https://registry.npmjs.org/extract-zip/-/extract-zip-1.6.7.tgz",
-      "integrity": "sha1-qEC0uK9kAyZMjbV/Txp0Mz74H+k=",
-      "dev": true,
-      "requires": {
-        "concat-stream": "1.6.2",
-        "debug": "2.6.9",
-        "mkdirp": "0.5.1",
-        "yauzl": "2.4.1"
-      }
-    },
    "extsprintf": {
      "version": "1.3.0",
      "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz",
@ -2116,15 +2112,6 @@
      "integrity": "sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc=",
      "dev": true
    },
-    "fd-slicer": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.0.1.tgz",
-      "integrity": "sha1-i1vL2ewyfFBBv5qwI/1nUPEXfmU=",
-      "dev": true,
-      "requires": {
-        "pend": "~1.2.0"
-      }
-    },
    "figgy-pudding": {
      "version": "3.5.1",
      "resolved": "https://registry.npmjs.org/figgy-pudding/-/figgy-pudding-3.5.1.tgz",
@ -3317,33 +3304,6 @@
      "integrity": "sha1-7AbBDgo0wPL68Zn3/X/Hj//QPHM=",
      "dev": true
    },
-    "https-proxy-agent": {
-      "version": "2.2.4",
-      "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-2.2.4.tgz",
-      "integrity": "sha512-OmvfoQ53WLjtA9HeYP9RNrWMJzzAz1JGaSFr1nijg0PVR1JaD/xbJq1mdEIIlxGpXp9eSe/O2LgU9DJmTPd0Eg==",
-      "dev": true,
-      "requires": {
-        "agent-base": "^4.3.0",
-        "debug": "^3.1.0"
-      },
-      "dependencies": {
-        "debug": {
-          "version": "3.2.6",
-          "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz",
-          "integrity": "sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==",
-          "dev": true,
-          "requires": {
-            "ms": "^2.1.1"
-          }
-        },
-        "ms": {
-          "version": "2.1.2",
-          "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
-          "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==",
-          "dev": true
-        }
-      }
-    },
    "iconv-lite": {
      "version": "0.4.24",
      "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz",
@ -3714,6 +3674,12 @@
      "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=",
      "dev": true
    },
+    "jpeg-js": {
+      "version": "0.3.7",
+      "resolved": "https://registry.npmjs.org/jpeg-js/-/jpeg-js-0.3.7.tgz",
+      "integrity": "sha512-9IXdWudL61npZjvLuVe/ktHiA41iE8qFyLB+4VDTblEsWBzeg8WQTlktdUK4CdncUqtUgUg0bbOmTE2bKBKaBQ==",
+      "dev": true
+    },
    "jquery": {
      "version": "3.4.1",
      "resolved": "https://registry.npmjs.org/jquery/-/jquery-3.4.1.tgz",
@ -5139,6 +5105,121 @@
        }
      }
    },
+    "playwright": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.0.1.tgz",
+      "integrity": "sha512-kVTE7uvZ7OcDVOBx7MVArUm2nbzzzpauKV9tuVIAH6vWGsOWbGGALUoTWMzNDzsPPTBJXXmxzC4KgI2zN+kVhw==",
+      "dev": true,
+      "requires": {
+        "playwright-core": "=1.0.1"
+      }
+    },
+    "playwright-core": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.0.1.tgz",
+      "integrity": "sha512-a71FjUDRFqWLG3VBAojVen2TaZiXkuog+ZmI0Nh0+/QndFUbbW3kameOfUTMXFvLUGWx2ipERZx6EQTJMEQDMA==",
+      "dev": true,
+      "requires": {
+        "debug": "^4.1.1",
+        "extract-zip": "^2.0.0",
+        "https-proxy-agent": "^3.0.0",
+        "jpeg-js": "^0.3.7",
+        "mime": "^2.4.4",
+        "pngjs": "^5.0.0",
+        "progress": "^2.0.3",
+        "proxy-from-env": "^1.1.0",
+        "rimraf": "^3.0.2",
+        "ws": "^6.1.0"
+      },
+      "dependencies": {
+        "debug": {
+          "version": "4.1.1",
+          "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz",
+          "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==",
+          "dev": true,
+          "requires": {
+            "ms": "^2.1.1"
+          }
+        },
+        "extract-zip": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/extract-zip/-/extract-zip-2.0.0.tgz",
+          "integrity": "sha512-i42GQ498yibjdvIhivUsRslx608whtGoFIhF26Z7O4MYncBxp8CwalOs1lnHy21A9sIohWO2+uiE4SRtC9JXDg==",
+          "dev": true,
+          "requires": {
+            "@types/yauzl": "^2.9.1",
+            "debug": "^4.1.1",
+            "get-stream": "^5.1.0",
+            "yauzl": "^2.10.0"
+          }
+        },
+        "fd-slicer": {
+          "version": "1.1.0",
+          "resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.1.0.tgz",
+          "integrity": "sha1-JcfInLH5B3+IkbvmHY85Dq4lbx4=",
+          "dev": true,
+          "requires": {
+            "pend": "~1.2.0"
+          }
+        },
+        "get-stream": {
+          "version": "5.1.0",
+          "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.1.0.tgz",
+          "integrity": "sha512-EXr1FOzrzTfGeL0gQdeFEvOMm2mzMOglyiOXSTpPC+iAjAKftbr3jpCMWynogwYnM+eSj9sHGc6wjIcDvYiygw==",
+          "dev": true,
+          "requires": {
+            "pump": "^3.0.0"
+          }
+        },
+        "https-proxy-agent": {
+          "version": "3.0.1",
+          "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-3.0.1.tgz",
+          "integrity": "sha512-+ML2Rbh6DAuee7d07tYGEKOEi2voWPUGan+ExdPbPW6Z3svq+JCqr0v8WmKPOkz1vOVykPCBSuobe7G8GJUtVg==",
+          "dev": true,
+          "requires": {
+            "agent-base": "^4.3.0",
+            "debug": "^3.1.0"
+          },
+          "dependencies": {
+            "debug": {
+              "version": "3.2.6",
+              "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz",
+              "integrity": "sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==",
+              "dev": true,
+              "requires": {
+                "ms": "^2.1.1"
+              }
+            }
+          }
+        },
+        "ms": {
+          "version": "2.1.2",
+          "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
+          "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==",
+          "dev": true
+        },
+        "pump": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz",
+          "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==",
+          "dev": true,
+          "requires": {
+            "end-of-stream": "^1.1.0",
+            "once": "^1.3.1"
+          }
+        },
+        "yauzl": {
+          "version": "2.10.0",
+          "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.10.0.tgz",
+          "integrity": "sha1-x+sXyT4RLLEIb6bY5R+wZnt5pfk=",
+          "dev": true,
+          "requires": {
+            "buffer-crc32": "~0.2.3",
+            "fd-slicer": "~1.1.0"
+          }
+        }
+      }
+    },
    "plugin-error": {
      "version": "1.0.1",
      "resolved": "https://registry.npmjs.org/plugin-error/-/plugin-error-1.0.1.tgz",
@ -5157,6 +5238,12 @@
      "integrity": "sha512-2qHaIQr2VLRFoxe2nASzsV6ef4yOOH+Fi9FBOVH6cqeSgUnoyySPZkxzLuzd+RYOQTRpROA0ztTMqxROKSb/nA==",
      "dev": true
    },
+    "pngjs": {
+      "version": "5.0.0",
+      "resolved": "https://registry.npmjs.org/pngjs/-/pngjs-5.0.0.tgz",
+      "integrity": "sha512-40QW5YalBNfQo5yRYmiw7Yz6TKKVr3h6970B2YE+3fQpsWcrbj1PzJgxeJ19DRQjhMbKPIuMY8rFaXc8moolVw==",
+      "dev": true
+    },
    "posix-character-classes": {
      "version": "0.1.1",
      "resolved": "https://registry.npmjs.org/posix-character-classes/-/posix-character-classes-0.1.1.tgz",
@ -5327,48 +5414,6 @@
      "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==",
      "dev": true
    },
-    "puppeteer": {
-      "version": "1.20.0",
-      "resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-1.20.0.tgz",
-      "integrity": "sha512-bt48RDBy2eIwZPrkgbcwHtb51mj2nKvHOPMaSH2IsWiv7lOG9k9zhaRzpDZafrk05ajMc3cu+lSQYYOfH2DkVQ==",
-      "dev": true,
-      "requires": {
-        "debug": "^4.1.0",
-        "extract-zip": "^1.6.6",
-        "https-proxy-agent": "^2.2.1",
-        "mime": "^2.0.3",
-        "progress": "^2.0.1",
-        "proxy-from-env": "^1.0.0",
-        "rimraf": "^2.6.1",
-        "ws": "^6.1.0"
-      },
-      "dependencies": {
-        "debug": {
-          "version": "4.1.1",
-          "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz",
-          "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==",
-          "dev": true,
-          "requires": {
-            "ms": "^2.1.1"
-          }
-        },
-        "ms": {
-          "version": "2.1.2",
-          "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
-          "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==",
-          "dev": true
-        },
-        "rimraf": {
-          "version": "2.7.1",
-          "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz",
-          "integrity": "sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==",
-          "dev": true,
-          "requires": {
-            "glob": "^7.1.3"
-          }
-        }
-      }
-    },
    "qs": {
      "version": "6.5.2",
      "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz",
@ -7466,15 +7511,6 @@
      "resolved": "https://registry.npmjs.org/yaserver/-/yaserver-0.2.0.tgz",
      "integrity": "sha512-onsELrl7Y42M4P3T9R0N/ZJNJRu4cGwzhDyOWIFRMJvPUIrGKInYGh+DJBefrbr1qoyDu7DSCLl9BL5hSSVfDA==",
      "dev": true
-    },
-    "yauzl": {
-      "version": "2.4.1",
-      "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.4.1.tgz",
-      "integrity": "sha1-lSj0QtqxsihOWLQ3m7GU4i4MQAU=",
-      "dev": true,
-      "requires": {
-        "fd-slicer": "~1.0.1"
-      }
    }
  }
 }
--- a/package.json
+++ b/package.json
@ -24,7 +24,6 @@
  "devDependencies": {
    "@types/chai": "^3.4.34",
    "@types/mocha": "^2.2.33",
-    "@types/puppeteer": "^1.12.4",
    "chai": "^4.2.0",
    "clean-css": "^4.2.3",
    "css-loader": "^3.2.0",
@ -39,7 +38,7 @@
    "monaco-json": "2.9.0",
    "monaco-languages": "1.10.0",
    "monaco-typescript": "3.7.0",
-    "puppeteer": "^1.15.0",
+    "playwright": "^1.0.1",
    "rimraf": "^3.0.2",
    "style-loader": "^1.1.3",
    "typedoc": "^0.15.8",