diff --git a/lib/adBlockRustUtils.js b/lib/adBlockRustUtils.js index 64581a0e..699fde94 100644 --- a/lib/adBlockRustUtils.js +++ b/lib/adBlockRustUtils.js @@ -2,7 +2,6 @@ const { uBlockResources } = require('adblock-rs') const path = require('path') const fs = require('fs').promises -const request = require('request') const uBlockLocalRoot = 'submodules/uBlock' const uBlockWebAccessibleResources = path.join(uBlockLocalRoot, 'src/web_accessible_resources') @@ -29,19 +28,16 @@ const resourcesPubkey = 'MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA7Qk6xtml8Si * @param url The URL to fetch from * @return a promise that resolves with the content of the list or rejects with an error message. */ -const requestJSON = (url) => new Promise((resolve, reject) => { - request.get(url, function (error, response, body) { - if (error) { - reject(new Error(`Request error: ${error}`)) - return +const requestJSON = (url) => { + return fetch(url).then(response => { + if (response.status !== 200) { + throw new Error(`Error status ${response.status} ${response.statusText} returned for URL: ${url}`) } - if (response.statusCode !== 200) { - reject(new Error(`Error status code ${response.statusCode} returned for URL: ${url}`)) - return - } - resolve(JSON.parse(body)) + return response.json() + }).catch(error => { + throw new Error(`Error when fetching ${url}: ${error.cause}`) }) -}) +} const getDefaultLists = requestJSON.bind(null, defaultListsUrl) const getRegionalLists = requestJSON.bind(null, regionalListsUrl) @@ -54,24 +50,16 @@ const lazyInit = (fn) => { } } -const generateResources = lazyInit(() => new Promise((resolve, reject) => { +const generateResources = lazyInit(async () => { const resourceData = uBlockResources( uBlockWebAccessibleResources, uBlockRedirectEngine, uBlockScriptlets ) - request.get(braveResourcesUrl, function (error, response, body) { - if (error) { - reject(new Error(`Request error: ${error}`)) - } - if (response.statusCode !== 200) { - reject(new Error(`Error status code ${response.statusCode} returned for URL: ${braveResourcesUrl}`)) - } - const braveResources = JSON.parse(body) - resourceData.push(...braveResources) - resolve(JSON.stringify(resourceData)) - }) -})) + const braveResources = await requestJSON(braveResourcesUrl) + resourceData.push(...braveResources) + return JSON.stringify(resourceData) +}) /** * Returns a promise that generates a resources file from the uBlock Origin diff --git a/lib/ntpUtil.js b/lib/ntpUtil.js index 636f37cb..fa9127f1 100644 --- a/lib/ntpUtil.js +++ b/lib/ntpUtil.js @@ -6,7 +6,8 @@ const childProcess = require('child_process') const fs = require('fs') const path = require('path') const util = require('../lib/util') -const request = require('request') +const { Readable } = require('stream') +const { finished } = require('stream/promises') const jsonSchemaVersion = 1 @@ -98,13 +99,9 @@ const prepareAssets = (jsonFileUrl, targetResourceDir, targetJsonFileName) => { // Download and parse jsonFileUrl. // If it doesn't exist, create with empty object. - request(jsonFileUrl, async function (error, response, body) { - if (error) { - console.error(`Error from ${jsonFileUrl}:`, error) - return reject(error) - } - if (response && response.statusCode === 200) { - jsonFileBody = body + fetch(jsonFileUrl, async function (response) { + if (response.status === 200) { + jsonFileBody = await response.text() } let photoData = {} try { @@ -112,7 +109,7 @@ const prepareAssets = (jsonFileUrl, targetResourceDir, targetJsonFileName) => { photoData = JSON.parse(jsonFileBody) } catch (err) { console.error(`Invalid json file ${jsonFileUrl}`) - return reject(error) + return reject(err) } console.log(`Done - json file ${jsonFileUrl} parsing`) // Make sure the data has a schema version so that clients can opt to parse or not @@ -124,7 +121,8 @@ const prepareAssets = (jsonFileUrl, targetResourceDir, targetJsonFileName) => { // have a schema version. incomingSchemaVersion = jsonSchemaVersion } else if (!isValidSchemaVersion(incomingSchemaVersion)) { - console.error(`Error: Cannot parse JSON data at ${jsonFileUrl} since it has a schema version of ${incomingSchemaVersion} but we expected ${jsonSchemaVersion}! This region will not be updated.`) + const error = `Error: Cannot parse JSON data at ${jsonFileUrl} since it has a schema version of ${incomingSchemaVersion} but we expected ${jsonSchemaVersion}! This region will not be updated.` + console.error(error) return reject(error) } @@ -132,18 +130,18 @@ const prepareAssets = (jsonFileUrl, targetResourceDir, targetJsonFileName) => { // Download image files that specified in jsonFileUrl const imageFileNameList = getImageFileNameListFrom(photoData) - const downloadOps = imageFileNameList.map((imageFileName) => new Promise(resolve => { + const downloadOps = imageFileNameList.map(async (imageFileName) => { const targetImageFilePath = path.join(targetResourceDir, imageFileName) const targetImageFileUrl = new URL(imageFileName, jsonFileUrl).href - request(targetImageFileUrl) - .pipe(fs.createWriteStream(targetImageFilePath)) - .on('finish', () => { - console.log(targetImageFileUrl) - resolve() - }) - })) + const response = await fetch(targetImageFileUrl) + const ws = fs.createWriteStream(targetImageFilePath) + return finished(Readable.fromWeb(response.body).pipe(ws)) + .then(() => console.log(targetImageFileUrl)) + }) await Promise.all(downloadOps) resolve() + }).catch(error => { + throw new Error(`Error from ${jsonFileUrl}: ${error.cause}`) }) }) } diff --git a/lib/util.js b/lib/util.js index 78b6b604..f778d9b4 100644 --- a/lib/util.js +++ b/lib/util.js @@ -7,26 +7,20 @@ const crypto = require('crypto') const fs = require('fs') const mkdirp = require('mkdirp') const path = require('path') -const request = require('request') const s3 = require('s3-client') const unzip = require('unzip-crx-3') const AWS = require('aws-sdk') +const { Readable } = require('stream') +const { finished } = require('stream/promises') const DynamoDBTableName = 'Extensions' const FirstVersion = '1.0.0' -const downloadExtensionFromCWS = (componentId, chromiumVersion, outputPath) => { +const downloadExtensionFromCWS = async (componentId, chromiumVersion, outputPath) => { const url = `https://clients2.google.com/service/update2/crx?response=redirect&prodversion=${chromiumVersion}&acceptformat=crx3&x=id%3D${componentId}%26uc` - return new Promise((resolve, reject) => { - request(url) - .pipe(fs.createWriteStream(outputPath)) - .on('finish', () => { - resolve() - }) - .on('error', () => { - reject(new Error('Failed to make request to Chrome Web Store')) - }) - }) + const response = await fetch(url) + const ws = fs.createWriteStream(outputPath) + return finished(Readable.fromWeb(response.body).pipe(ws)) } const generateCRXFile = (binary, crxFile, privateKeyFile, publisherProofKey, diff --git a/package-lock.json b/package-lock.json index 96e8e5da..2454650c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -19,7 +19,6 @@ "https-everywhere-builder": "github:brave/https-everywhere-builder", "playlist-component": "github:brave/playlist-component", "recursive-readdir-sync": "1.0.6", - "request": "2.88.2", "s3-client": "4.4.2", "unzip-crx-3": "0.2.0" }, diff --git a/package.json b/package.json index f7fd6681..6b2cf733 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,6 @@ "https-everywhere-builder": "github:brave/https-everywhere-builder", "playlist-component": "github:brave/playlist-component", "recursive-readdir-sync": "1.0.6", - "request": "2.88.2", "s3-client": "4.4.2", "unzip-crx-3": "0.2.0" }, diff --git a/scripts/generateAdBlockRustDataFiles.js b/scripts/generateAdBlockRustDataFiles.js index e759b815..efe1eba9 100644 --- a/scripts/generateAdBlockRustDataFiles.js +++ b/scripts/generateAdBlockRustDataFiles.js @@ -6,7 +6,6 @@ const { Engine, FilterFormat, FilterSet, RuleTypes } = require('adblock-rs') const { generateResourcesFile, getDefaultLists, getRegionalLists, defaultPlaintextComponentId, resourcesComponentId, regionalCatalogComponentId } = require('../lib/adBlockRustUtils') const path = require('path') const fs = require('fs') -const request = require('request') /** * Returns a promise that which resolves with the list data @@ -15,18 +14,13 @@ const request = require('request') * @return a promise that resolves with the content of the list or rejects with an error message. */ const getListBufferFromURL = (listURL) => { - return new Promise((resolve, reject) => { - request.get(listURL, function (error, response, body) { - if (error) { - reject(new Error(`Request error for ${listURL}: ${error}`)) - return - } - if (response.statusCode !== 200) { - reject(new Error(`Error status code ${response.statusCode} returned for URL: ${listURL}`)) - return - } - resolve(body) - }) + return fetch(listURL).then(response => { + if (response.status !== 200) { + throw new Error(`Error status ${response.status} ${response.statusText} returned for URL: ${listURL}`) + } + return response.text() + }).catch(error => { + throw new Error(`Error when fetching ${listURL}: ${error.cause}`) }) } @@ -95,8 +89,7 @@ const generatePlaintextListFromLists = (listBuffers, outSubdir) => { } /** - * Convenience function that uses getListBufferFromURL and generateDataFileFromLists - * to construct a DAT file from a URL. + * Convenience function that prepares adblock components for a particular regional list. * * @param listURL the URL of the list to fetch. * @param the format of the filter list at the given URL. @@ -111,24 +104,15 @@ const generateDataFileFromRegionalCatalogEntry = (entry) => { const uuid = entry.uuid const outputDATFilename = `rs-${entry.uuid}.dat` const listTextComponent = entry.list_text_component - return new Promise((resolve, reject) => { - console.log(`${langs} ${listURL}...`) - request.get(listURL, function (error, response, body) { - if (error) { - reject(new Error(`Request error for ${listURL}: ${error}`)) - return - } - if (response.statusCode !== 200) { - reject(new Error(`Error status code ${response.statusCode} returned for URL: ${listURL}`)) - return - } - generateDataFileFromLists([{ title, format, data: body }], outputDATFilename, uuid) - if (listTextComponent !== undefined) { - const outPath = getOutPath('list.txt', listTextComponent.component_id) - fs.writeFileSync(outPath, enforceBraveDirectives(title, body)) - } - resolve() - }) + console.log(`${langs} ${listURL}...`) + return getListBufferFromURL(listURL).then(body => { + generateDataFileFromLists([{ title, format, data: body }], outputDATFilename, uuid) + if (listTextComponent !== undefined) { + const outPath = getOutPath('list.txt', listTextComponent.component_id) + fs.writeFileSync(outPath, enforceBraveDirectives(title, body)) + } + }).catch(error => { + throw new Error(`Error when fetching ${listURL}: ${error.cause}`) }) } @@ -145,7 +129,7 @@ const generateDataFilesForAllRegions = () => { fs.writeFileSync(getOutPath('regional_catalog.json', 'default'), catalogString) fs.writeFileSync(getOutPath('regional_catalog.json', regionalCatalogComponentId), catalogString) resolve() - }).then(Promise.all(regions.map(region => + }).then(() => Promise.all(regions.map(region => generateDataFileFromRegionalCatalogEntry(region) ))) }) @@ -171,7 +155,7 @@ const generateDefaultDataFiles = (lists) => { return p } -const generateDataFilesForResourcesComponent = async () => { +const generateDataFilesForResourcesComponent = () => { return generateResourcesFile(getOutPath('resources.json', resourcesComponentId)) } diff --git a/scripts/generateBraveAdsResourcesComponentInputFiles.js b/scripts/generateBraveAdsResourcesComponentInputFiles.js index 70b8062a..01295e3b 100644 --- a/scripts/generateBraveAdsResourcesComponentInputFiles.js +++ b/scripts/generateBraveAdsResourcesComponentInputFiles.js @@ -5,8 +5,9 @@ const path = require('path') const mkdirp = require('mkdirp') const fs = require('fs-extra') -const request = require('request') const commander = require('commander') +const { Readable } = require('stream') +const { finished } = require('stream/promises') const getComponentList = () => { return [ @@ -79,19 +80,15 @@ const getComponentList = () => { function downloadComponentInputFiles (manifestFileName, manifestUrl, outDir) { return new Promise(function (resolve, reject) { let manifestBody = '{}' - request(manifestUrl, async function (error, response, body) { - if (error) { - console.error(`Error from ${manifestUrl}:`, error) - return reject(error) - } - - if (response && response.statusCode === 200) { - manifestBody = body + fetch(manifestUrl, async function (response) { + if (response.status === 200) { + manifestBody = await response.text() } const manifestJson = JSON.parse(manifestBody) if (!manifestJson.schemaVersion) { - console.error('Error: Missing schema version') + const error = 'Error: Missing schema version' + console.error(error) return reject(error) } @@ -105,20 +102,20 @@ function downloadComponentInputFiles (manifestFileName, manifestUrl, outDir) { }) } - const downloadOps = fileList.map((fileName) => new Promise(resolve => { + const downloadOps = fileList.map(async (fileName) => { const resourceFileOutPath = path.join(outDir, fileName) const resourceFileUrl = new URL(fileName, manifestUrl).href - request(resourceFileUrl) - .pipe(fs.createWriteStream(resourceFileOutPath)) - .on('finish', () => { - console.log(resourceFileUrl) - resolve() - }) - })) + const response = await fetch(resourceFileUrl) + const ws = fs.createWriteStream(resourceFileOutPath) + return finished(Readable.fromWeb(response.body).pipe(ws)) + .then(() => console.log(resourceFileUrl)) + }) await Promise.all(downloadOps) resolve() + }).catch(error => { + throw new Error(`Error from ${manifestUrl}: ${error.cause}`) }) }) } diff --git a/scripts/generateNTPBackgroundImages.js b/scripts/generateNTPBackgroundImages.js index 5a0278d6..df6f6e10 100644 --- a/scripts/generateNTPBackgroundImages.js +++ b/scripts/generateNTPBackgroundImages.js @@ -7,7 +7,8 @@ const mkdirp = require('mkdirp') const fs = require('fs-extra') const commander = require('commander') const util = require('../lib/util') -const request = require('request') +const { Readable } = require('stream') +const { finished } = require('stream/promises') const jsonSchemaVersion = 1 @@ -55,33 +56,29 @@ const prepareAssets = (jsonFileUrl, targetResourceDir) => { let body = '{}' // Download and parse jsonFileUrl. - request(jsonFileUrl, async function (error, response, jsonFileBody) { - if (error) { - console.error(`Error from ${jsonFileUrl}:`, error) - return reject(error) - } - - if (response && response.statusCode !== 200) { - console.error(`Error from ${jsonFileUrl}:`, response.statusMessage) - return reject(error) + fetch(jsonFileUrl, async function (response) { + if (response.status !== 200) { + throw new Error(`Error from ${jsonFileUrl}: ${response.status} ${response.statusText}`) } - if (response && response.statusCode === 200) { - body = jsonFileBody + if (response.status === 200) { + body = await response.text() } let photoData = {} try { console.log(`Start - json file ${jsonFileUrl} parsing`) photoData = JSON.parse(body) } catch (err) { - console.error(`Invalid json file ${jsonFileUrl}`) + const error = `Invalid json file ${jsonFileUrl}` + console.error(error) return reject(error) } console.log(`Done - json file ${jsonFileUrl} parsing`) console.log(`Start - json file ${jsonFileUrl} validation`) if (!validatePhotoData(photoData)) { - console.error(`Failed to validate json file ${jsonFileUrl}`) + const error = `Failed to validate json file ${jsonFileUrl}` + console.error(error) return reject(error) } console.log(`Done - json file ${jsonFileUrl} validation`) @@ -90,18 +87,18 @@ const prepareAssets = (jsonFileUrl, targetResourceDir) => { // Download image files that specified in jsonFileUrl const imageFileNameList = getImageFileNameListFrom(photoData) - const downloadOps = imageFileNameList.map((imageFileName) => new Promise(resolve => { + const downloadOps = imageFileNameList.map(async (imageFileName) => { const targetImageFilePath = path.join(targetResourceDir, imageFileName) const targetImageFileUrl = new URL(imageFileName, jsonFileUrl).href - request(targetImageFileUrl) - .pipe(fs.createWriteStream(targetImageFilePath)) - .on('finish', () => { - console.log(`Downloaded ${targetImageFileUrl}`) - resolve() - }) - })) + const response = await fetch(targetImageFileUrl) + const ws = fs.createWriteStream(targetImageFilePath) + return finished(Readable.fromWeb(response.body).pipe(ws)) + .then(() => console.log(`Downloaded ${targetImageFileUrl}`)) + }) await Promise.all(downloadOps) resolve() + }).catch(error => { + throw new Error(`Error from ${jsonFileUrl}: ${error.cause}`) }) }) } diff --git a/scripts/generateNTPSuperReferrerMappingTable.js b/scripts/generateNTPSuperReferrerMappingTable.js index 305fdcc0..776b83c1 100644 --- a/scripts/generateNTPSuperReferrerMappingTable.js +++ b/scripts/generateNTPSuperReferrerMappingTable.js @@ -5,7 +5,6 @@ const path = require('path') const mkdirp = require('mkdirp') const fs = require('fs-extra') -const request = require('request') const commander = require('commander') const util = require('../lib/util') @@ -19,13 +18,9 @@ function downloadMappingTableJsonFile (jsonFileUrl, targetFilePath) { return new Promise(function (resolve, reject) { let jsonFileBody = '{}' - request(jsonFileUrl, async function (error, response, body) { - if (error) { - console.error(`Error from ${jsonFileUrl}:`, error) - return reject(error) - } - if (response && response.statusCode === 200) { - jsonFileBody = body + fetch(jsonFileUrl, async function (response) { + if (response.status === 200) { + jsonFileBody = await response.text() } const data = JSON.parse(jsonFileBody) @@ -38,13 +33,16 @@ function downloadMappingTableJsonFile (jsonFileUrl, targetFilePath) { data.schemaVersion = jsonSchemaVersion } else if (incomingSchemaVersion !== jsonSchemaVersion) { // We don't support this file format - console.error(`Error: Cannot parse JSON data at ${jsonFileUrl} since it has a schema version of ${incomingSchemaVersion} but we expected ${jsonSchemaVersion}! This region will not be updated.`) + const error = `Error: Cannot parse JSON data at ${jsonFileUrl} since it has a schema version of ${incomingSchemaVersion} but we expected ${jsonSchemaVersion}! This region will not be updated.` + console.error(error) return reject(error) } createDataJsonFile(targetFilePath, JSON.stringify(data)) resolve() + }).catch(error => { + throw new Error(`Error from ${jsonFileUrl}: ${error.cause}`) }) }) }