From 8418980e6ca345a2c5408b54bd3d2396dff601eb Mon Sep 17 00:00:00 2001 From: Mark Yen Date: Mon, 9 Sep 2024 13:41:56 -0700 Subject: [PATCH] postinstall: implement retries When fetching things during postinstall, actually retry downloads if they fail partway through. If nodejs/node/issues/43187 gets resolved, we may be able to switch to importing `node:undici` instead of adding another copy with possibly incompatible API. Signed-off-by: Mark Yen --- .github/actions/spelling/expect.txt | 1 + package.json | 1 + scripts/lib/download.ts | 75 ++++++++++++++++++++++++----- yarn.lock | 36 +++----------- 4 files changed, 74 insertions(+), 39 deletions(-) diff --git a/.github/actions/spelling/expect.txt b/.github/actions/spelling/expect.txt index 8833969113c..e689cfdf806 100644 --- a/.github/actions/spelling/expect.txt +++ b/.github/actions/spelling/expect.txt @@ -817,6 +817,7 @@ udhcpc UEFI Unauthed UNCONFIGURED +undici unexpose unexposing unfetch diff --git a/package.json b/package.json index e8995d73236..9f6412d1cab 100644 --- a/package.json +++ b/package.json @@ -168,6 +168,7 @@ "ts-node": "10.9.2", "tsconfig-paths": "4.2.0", "typescript": "5.5.4", + "undici": "^6.19.8", "vue-eslint-parser": "9.4.3", "vue-jest": "3.0.7", "vue-template-compiler": "2.7.16", diff --git a/scripts/lib/download.ts b/scripts/lib/download.ts index e717447b009..da6f5ade6a3 100644 --- a/scripts/lib/download.ts +++ b/scripts/lib/download.ts @@ -9,6 +9,8 @@ import os from 'os'; import path from 'path'; import stream from 'stream'; +import { Agent, RetryAgent } from 'undici'; + import { simpleSpawn } from 'scripts/simple_process'; type ChecksumAlgorithm = 'sha1' | 'sha256' | 'sha512'; @@ -30,18 +32,69 @@ export type ArchiveDownloadOptions = DownloadOptions & { }; async function fetchWithRetry(url: string) { - while (true) { - try { - return await fetch(url, { redirect: 'follow' }); - } catch (ex: any) { - if (ex && ex.errno === 'EAI_AGAIN') { - console.log(`Recoverable error downloading ${ url }, retrying...`); - continue; + const errorCodes = ['EAI_AGAIN', 'ECONNRESET', 'ECONNREFUSED', 'EHOSTDOWN', 'ENETDOWN', 'ENETUNREACH']; + const UndiciPrefix = 'UND_ERR_'; // spellcheck-ignore-line + + errorCodes.push(...['BODY_TIMEOUT', 'CONNECT_TIMEOUT', 'REQ_RETRY', 'SOCKET'].map(e => UndiciPrefix + e )); + const agent = new RetryAgent(new Agent({ bodyTimeout: 1_000 }), { + maxTimeout: 5_000, + errorCodes, + retry: (rawError, { state, opts }, cb) => { + const err: Error & { + code?: string; + statusCode?: number; + headers?: Record, + } = rawError; + const { + maxRetries, maxTimeout, errorCodes, statusCodes, + } = opts.retryOptions ?? {}; + + // Unfortunately, the Undici API doesn't allow us to just call the default + // retry handler without a bunch of hacky gymnastics; so we'll have to + // mostly duplicate it instead. + if (err.code && !(errorCodes ?? []).includes(err.code)) { + cb(err); // Unexpected error code. + + return null; } - console.dir(ex); - throw ex; - } - } + + if (err.statusCode && !(statusCodes ?? []).includes(err.statusCode)) { + cb(err); // Unexpected status code. + + return null; + } + + if (state.counter > (maxRetries ?? 5)) { + cb(err); // Maximum retries reached. + + return null; + } + + // retryAfterHeader is number of seconds, or a date string. + const retryAfterHeader = err.headers?.['retry-after'] || ''; + const retryAfter = (() => { + if (!retryAfterHeader) { + return 0; + } + + return Number(retryAfterHeader) * 1_000 || (Date.parse(retryAfterHeader) - Date.now()); + })(); + const retryTimeout = (() => { + if (retryAfter > 0) { + return Math.max(retryAfter, maxTimeout ?? 30_000); + } + + return 1_000; // Hard code one second; we have a ten minute total for the whole download. + })(); + + console.log(`Recoverable error ${ err } (${ err.code }) downloading ${ url }, retrying after ${ Math.floor(retryTimeout / 1_000) } seconds...`); + setTimeout(() => cb(null), retryTimeout); + + return null; + }, + }); + + return await fetch(url, { redirect: 'follow', dispatcher: agent } as any); } /** diff --git a/yarn.lock b/yarn.lock index 8540a7067a7..fdbbddce1f2 100644 --- a/yarn.lock +++ b/yarn.lock @@ -12259,16 +12259,7 @@ string-length@^5.0.1: char-regex "^2.0.0" strip-ansi "^7.0.1" -"string-width-cjs@npm:string-width@^4.2.0": - version "4.2.3" - resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" - integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== - dependencies: - emoji-regex "^8.0.0" - is-fullwidth-code-point "^3.0.0" - strip-ansi "^6.0.1" - -string-width@^2.1.1, string-width@^4, string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3, string-width@^5.0.1, string-width@^5.1.2: +"string-width-cjs@npm:string-width@^4.2.0", string-width@^2.1.1, string-width@^4, string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3, string-width@^5.0.1, string-width@^5.1.2: version "4.2.3" resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== @@ -12346,7 +12337,7 @@ string_decoder@~1.1.1: dependencies: safe-buffer "~5.1.0" -"strip-ansi-cjs@npm:strip-ansi@^6.0.1": +"strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@^6.0.0, strip-ansi@^6.0.1: version "6.0.1" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== @@ -12367,13 +12358,6 @@ strip-ansi@^4.0.0: dependencies: ansi-regex "^3.0.0" -strip-ansi@^6.0.0, strip-ansi@^6.0.1: - version "6.0.1" - resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" - integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== - dependencies: - ansi-regex "^5.0.1" - strip-ansi@^7.0.1: version "7.1.0" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-7.1.0.tgz#d5b6568ca689d8561370b0707685d22434faff45" @@ -13026,6 +13010,11 @@ undici-types@~5.26.4: resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-5.26.5.tgz#bcd539893d00b56e964fd2657a4866b221a65617" integrity sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA== +undici@^6.19.8: + version "6.19.8" + resolved "https://registry.yarnpkg.com/undici/-/undici-6.19.8.tgz#002d7c8a28f8cc3a44ff33c3d4be4d85e15d40e1" + integrity sha512-U8uCCl2x9TK3WANvmBavymRzxbfFYG+tAu+fgx3zxQy3qdagQqBLwJVrdyO1TBfUXvfKveMKJZhpvUYoOjM+4g== + unfetch@4.2.0: version "4.2.0" resolved "https://registry.yarnpkg.com/unfetch/-/unfetch-4.2.0.tgz#7e21b0ef7d363d8d9af0fb929a5555f6ef97a3be" @@ -13702,7 +13691,7 @@ word-wrap@~1.2.3: resolved "https://registry.yarnpkg.com/word-wrap/-/word-wrap-1.2.3.tgz#610636f6b1f703891bd34771ccb17fb93b47079c" integrity sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ== -"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0": +"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0", wrap-ansi@^7.0.0: version "7.0.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== @@ -13728,15 +13717,6 @@ wrap-ansi@^6.2.0: string-width "^4.1.0" strip-ansi "^6.0.0" -wrap-ansi@^7.0.0: - version "7.0.0" - resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" - integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== - dependencies: - ansi-styles "^4.0.0" - string-width "^4.1.0" - strip-ansi "^6.0.0" - wrap-ansi@^8.1.0: version "8.1.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-8.1.0.tgz#56dc22368ee570face1b49819975d9b9a5ead214"