Skip to content

Commit

Permalink
move usage script to playwright
Browse files Browse the repository at this point in the history
  • Loading branch information
zloirock committed Feb 11, 2022
1 parent b746e19 commit 4c4655d
Show file tree
Hide file tree
Showing 2 changed files with 8 additions and 13 deletions.
1 change: 0 additions & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,6 @@
"playwright": "^1.18.1",
"promises-aplus-tests": "^2.1.2",
"promises-es6-tests": "~0.5.0",
"puppeteer": "~13.3.1",
"qunit": "^2.17.2",
"semver": "^7.3.5",
"terser": "^5.10.0",
Expand Down
20 changes: 8 additions & 12 deletions scripts/usage.mjs
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import puppeteer from 'puppeteer';
import { chromium } from 'playwright';
import pTimeout from 'p-timeout';
import jszip from 'jszip';

Expand All @@ -20,15 +20,14 @@ const file = await archive.file('top-1m.csv').async('string');
const sites = file.split('\n').slice(0, limit).map(string => string.replace(/^\d+,(.+)$/, '$1')).reverse();

// run in parallel
await Promise.all(Array(Math.ceil(os.cpus().length / 2)).fill(0).map(async i => {
await Promise.all(Array(Math.ceil(os.cpus().length / 2)).fill(0).map(async () => {
let browser, site;

async function check() {
let errors = 0;
for (const protocol of protocols) for (const agent of agents) try {
const page = await browser.newPage();
page.setDefaultNavigationTimeout(12e4);
await page.setUserAgent(agent);
for (const protocol of protocols) for (const userAgent of agents) try {
const page = await browser.newPage({ userAgent });
page.setDefaultNavigationTimeout(6e4);
await page.goto(`${ protocol }://${ site }`);

// seems js hangs on some sites, so added a time limit
Expand All @@ -48,11 +47,8 @@ await Promise.all(Array(Math.ceil(os.cpus().length / 2)).fill(0).map(async i =>
}

while (site = sites.pop()) try {
// restart browser each some pages for prevent possible `puppeteer` crash and memory leaks
if (!(i++ % 8) || !browser) {
if (browser) await browser.close();
browser = await puppeteer.launch();
}
if (browser) await browser.close();
browser = await chromium.launch();

const { core, versions } = await check();

Expand All @@ -67,7 +63,7 @@ await Promise.all(Array(Math.ceil(os.cpus().length / 2)).fill(0).map(async i =>
} catch {
const attempting = (attempts.get(site) | 0) + 1;
attempts.set(site, attempting);
if (attempting < 4) sites.push(site);
if (attempting < 3) sites.push(site);
else console.log(red(`${ cyan(`${ site }:`) } problems with access`));
await sleep(3e3);
}
Expand Down

0 comments on commit 4c4655d

Please sign in to comment.