Merge pull request #528 from brave/replace-request-with-fetch

Replace `request` with `fetch`

lib/adBlockRustUtils.js

@@ -2,7 +2,6 @@ const { uBlockResources } = require('adblock-rs')
 
 const path = require('path')
 const fs = require('fs').promises
-const request = require('request')
 
 const uBlockLocalRoot = 'submodules/uBlock'
 const uBlockWebAccessibleResources = path.join(uBlockLocalRoot, 'src/web_accessible_resources')
@@ -29,19 +28,16 @@ const resourcesPubkey = 'MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA7Qk6xtml8Si
  * @param url The URL to fetch from
  * @return a promise that resolves with the content of the list or rejects with an error message.
  */
-const requestJSON = (url) => new Promise((resolve, reject) => {
-  request.get(url, function (error, response, body) {
-    if (error) {
-      reject(new Error(`Request error: ${error}`))
-      return
+const requestJSON = (url) => {
+  return fetch(url).then(response => {
+    if (response.status !== 200) {
+      throw new Error(`Error status ${response.status} ${response.statusText} returned for URL: ${url}`)
     }
-    if (response.statusCode !== 200) {
-      reject(new Error(`Error status code ${response.statusCode} returned for URL: ${url}`))
-      return
-    }
-    resolve(JSON.parse(body))
+    return response.json()
+  }).catch(error => {
+    throw new Error(`Error when fetching ${url}: ${error.cause}`)
   })
-})
+}
 
 const getDefaultLists = requestJSON.bind(null, defaultListsUrl)
 const getRegionalLists = requestJSON.bind(null, regionalListsUrl)
@@ -54,24 +50,16 @@ const lazyInit = (fn) => {
   }
 }
 
-const generateResources = lazyInit(() => new Promise((resolve, reject) => {
+const generateResources = lazyInit(async () => {
   const resourceData = uBlockResources(
     uBlockWebAccessibleResources,
     uBlockRedirectEngine,
     uBlockScriptlets
   )
-  request.get(braveResourcesUrl, function (error, response, body) {
-    if (error) {
-      reject(new Error(`Request error: ${error}`))
-    }
-    if (response.statusCode !== 200) {
-      reject(new Error(`Error status code ${response.statusCode} returned for URL: ${braveResourcesUrl}`))
-    }
-    const braveResources = JSON.parse(body)
-    resourceData.push(...braveResources)
-    resolve(JSON.stringify(resourceData))
-  })
-}))
+  const braveResources = await requestJSON(braveResourcesUrl)
+  resourceData.push(...braveResources)
+  return JSON.stringify(resourceData)
+})
 
 /**
  * Returns a promise that generates a resources file from the uBlock Origin

lib/ntpUtil.js

@@ -6,7 +6,8 @@ const childProcess = require('child_process')
 const fs = require('fs')
 const path = require('path')
 const util = require('../lib/util')
-const request = require('request')
+const { Readable } = require('stream')
+const { finished } = require('stream/promises')
 
 const jsonSchemaVersion = 1
 
@@ -98,21 +99,17 @@ const prepareAssets = (jsonFileUrl, targetResourceDir, targetJsonFileName) => {
 
     // Download and parse jsonFileUrl.
     // If it doesn't exist, create with empty object.
-    request(jsonFileUrl, async function (error, response, body) {
-      if (error) {
-        console.error(`Error from ${jsonFileUrl}:`, error)
-        return reject(error)
-      }
-      if (response && response.statusCode === 200) {
-        jsonFileBody = body
+    fetch(jsonFileUrl, async function (response) {
+      if (response.status === 200) {
+        jsonFileBody = await response.text()
       }
       let photoData = {}
       try {
         console.log(`Start - json file ${jsonFileUrl} parsing`)
         photoData = JSON.parse(jsonFileBody)
       } catch (err) {
         console.error(`Invalid json file ${jsonFileUrl}`)
-        return reject(error)
+        return reject(err)
       }
       console.log(`Done - json file ${jsonFileUrl} parsing`)
       // Make sure the data has a schema version so that clients can opt to parse or not
@@ -124,26 +121,27 @@ const prepareAssets = (jsonFileUrl, targetResourceDir, targetJsonFileName) => {
         // have a schema version.
         incomingSchemaVersion = jsonSchemaVersion
       } else if (!isValidSchemaVersion(incomingSchemaVersion)) {
-        console.error(`Error: Cannot parse JSON data at ${jsonFileUrl} since it has a schema version of ${incomingSchemaVersion} but we expected ${jsonSchemaVersion}! This region will not be updated.`)
+        const error = `Error: Cannot parse JSON data at ${jsonFileUrl} since it has a schema version of ${incomingSchemaVersion} but we expected ${jsonSchemaVersion}! This region will not be updated.`
+        console.error(error)
         return reject(error)
       }
 
       createPhotoJsonFile(path.join(targetResourceDir, 'photo.json'), JSON.stringify(photoData))
 
       // Download image files that specified in jsonFileUrl
       const imageFileNameList = getImageFileNameListFrom(photoData)
-      const downloadOps = imageFileNameList.map((imageFileName) => new Promise(resolve => {
+      const downloadOps = imageFileNameList.map(async (imageFileName) => {
         const targetImageFilePath = path.join(targetResourceDir, imageFileName)
         const targetImageFileUrl = new URL(imageFileName, jsonFileUrl).href
-        request(targetImageFileUrl)
-          .pipe(fs.createWriteStream(targetImageFilePath))
-          .on('finish', () => {
-            console.log(targetImageFileUrl)
-            resolve()
-          })
-      }))
+        const response = await fetch(targetImageFileUrl)
+        const ws = fs.createWriteStream(targetImageFilePath)
+        return finished(Readable.fromWeb(response.body).pipe(ws))
+          .then(() => console.log(targetImageFileUrl))
+      })
       await Promise.all(downloadOps)
       resolve()
+    }).catch(error => {
+      throw new Error(`Error from ${jsonFileUrl}: ${error.cause}`)
     })
   })
 }

lib/util.js

@@ -7,26 +7,20 @@ const crypto = require('crypto')
 const fs = require('fs')
 const mkdirp = require('mkdirp')
 const path = require('path')
-const request = require('request')
 const s3 = require('s3-client')
 const unzip = require('unzip-crx-3')
 const AWS = require('aws-sdk')
+const { Readable } = require('stream')
+const { finished } = require('stream/promises')
 
 const DynamoDBTableName = 'Extensions'
 const FirstVersion = '1.0.0'
 
-const downloadExtensionFromCWS = (componentId, chromiumVersion, outputPath) => {
+const downloadExtensionFromCWS = async (componentId, chromiumVersion, outputPath) => {
   const url = `https://clients2.google.com/service/update2/crx?response=redirect&prodversion=${chromiumVersion}&acceptformat=crx3&x=id%3D${componentId}%26uc`
-  return new Promise((resolve, reject) => {
-    request(url)
-      .pipe(fs.createWriteStream(outputPath))
-      .on('finish', () => {
-        resolve()
-      })
-      .on('error', () => {
-        reject(new Error('Failed to make request to Chrome Web Store'))
-      })
-  })
+  const response = await fetch(url)
+  const ws = fs.createWriteStream(outputPath)
+  return finished(Readable.fromWeb(response.body).pipe(ws))
 }
 
 const generateCRXFile = (binary, crxFile, privateKeyFile, publisherProofKey,

package.json

@@ -13,7 +13,6 @@
     "https-everywhere-builder": "github:brave/https-everywhere-builder",
     "playlist-component": "github:brave/playlist-component",
     "recursive-readdir-sync": "1.0.6",
-    "request": "2.88.2",
     "s3-client": "4.4.2",
     "unzip-crx-3": "0.2.0"
   },

scripts/generateAdBlockRustDataFiles.js

@@ -6,7 +6,6 @@ const { Engine, FilterFormat, FilterSet, RuleTypes } = require('adblock-rs')
 const { generateResourcesFile, getDefaultLists, getRegionalLists, defaultPlaintextComponentId, resourcesComponentId, regionalCatalogComponentId } = require('../lib/adBlockRustUtils')
 const path = require('path')
 const fs = require('fs')
-const request = require('request')
 
 /**
  * Returns a promise that which resolves with the list data
@@ -15,18 +14,13 @@ const request = require('request')
  * @return a promise that resolves with the content of the list or rejects with an error message.
  */
 const getListBufferFromURL = (listURL) => {
-  return new Promise((resolve, reject) => {
-    request.get(listURL, function (error, response, body) {
-      if (error) {
-        reject(new Error(`Request error for ${listURL}: ${error}`))
-        return
-      }
-      if (response.statusCode !== 200) {
-        reject(new Error(`Error status code ${response.statusCode} returned for URL: ${listURL}`))
-        return
-      }
-      resolve(body)
-    })
+  return fetch(listURL).then(response => {
+    if (response.status !== 200) {
+      throw new Error(`Error status ${response.status} ${response.statusText} returned for URL: ${listURL}`)
+    }
+    return response.text()
+  }).catch(error => {
+    throw new Error(`Error when fetching ${listURL}: ${error.cause}`)
   })
 }
 
@@ -95,8 +89,7 @@ const generatePlaintextListFromLists = (listBuffers, outSubdir) => {
 }
 
 /**
- * Convenience function that uses getListBufferFromURL and generateDataFileFromLists
- * to construct a DAT file from a URL.
+ * Convenience function that prepares adblock components for a particular regional list.
  *
  * @param listURL the URL of the list to fetch.
  * @param the format of the filter list at the given URL.
@@ -111,24 +104,15 @@ const generateDataFileFromRegionalCatalogEntry = (entry) => {
   const uuid = entry.uuid
   const outputDATFilename = `rs-${entry.uuid}.dat`
   const listTextComponent = entry.list_text_component
-  return new Promise((resolve, reject) => {
-    console.log(`${langs} ${listURL}...`)
-    request.get(listURL, function (error, response, body) {
-      if (error) {
-        reject(new Error(`Request error for ${listURL}: ${error}`))
-        return
-      }
-      if (response.statusCode !== 200) {
-        reject(new Error(`Error status code ${response.statusCode} returned for URL: ${listURL}`))
-        return
-      }
-      generateDataFileFromLists([{ title, format, data: body }], outputDATFilename, uuid)
-      if (listTextComponent !== undefined) {
-        const outPath = getOutPath('list.txt', listTextComponent.component_id)
-        fs.writeFileSync(outPath, enforceBraveDirectives(title, body))
-      }
-      resolve()
-    })
+  console.log(`${langs} ${listURL}...`)
+  return getListBufferFromURL(listURL).then(body => {
+    generateDataFileFromLists([{ title, format, data: body }], outputDATFilename, uuid)
+    if (listTextComponent !== undefined) {
+      const outPath = getOutPath('list.txt', listTextComponent.component_id)
+      fs.writeFileSync(outPath, enforceBraveDirectives(title, body))
+    }
+  }).catch(error => {
+    throw new Error(`Error when fetching ${listURL}: ${error.cause}`)
   })
 }
 
@@ -145,7 +129,7 @@ const generateDataFilesForAllRegions = () => {
       fs.writeFileSync(getOutPath('regional_catalog.json', 'default'), catalogString)
       fs.writeFileSync(getOutPath('regional_catalog.json', regionalCatalogComponentId), catalogString)
       resolve()
-    }).then(Promise.all(regions.map(region =>
+    }).then(() => Promise.all(regions.map(region =>
       generateDataFileFromRegionalCatalogEntry(region)
     )))
   })
@@ -171,7 +155,7 @@ const generateDefaultDataFiles = (lists) => {
   return p
 }
 
-const generateDataFilesForResourcesComponent = async () => {
+const generateDataFilesForResourcesComponent = () => {
   return generateResourcesFile(getOutPath('resources.json', resourcesComponentId))
 }
 

scripts/generateBraveAdsResourcesComponentInputFiles.js

@@ -5,8 +5,9 @@
 const path = require('path')
 const mkdirp = require('mkdirp')
 const fs = require('fs-extra')
-const request = require('request')
 const commander = require('commander')
+const { Readable } = require('stream')
+const { finished } = require('stream/promises')
 
 const getComponentList = () => {
   return [
@@ -79,19 +80,15 @@ const getComponentList = () => {
 function downloadComponentInputFiles (manifestFileName, manifestUrl, outDir) {
   return new Promise(function (resolve, reject) {
     let manifestBody = '{}'
-    request(manifestUrl, async function (error, response, body) {
-      if (error) {
-        console.error(`Error from ${manifestUrl}:`, error)
-        return reject(error)
-      }
-
-      if (response && response.statusCode === 200) {
-        manifestBody = body
+    fetch(manifestUrl, async function (response) {
+      if (response.status === 200) {
+        manifestBody = await response.text()
       }
 
       const manifestJson = JSON.parse(manifestBody)
       if (!manifestJson.schemaVersion) {
-        console.error('Error: Missing schema version')
+        const error = 'Error: Missing schema version'
+        console.error(error)
         return reject(error)
       }
 
@@ -105,20 +102,20 @@ function downloadComponentInputFiles (manifestFileName, manifestUrl, outDir) {
         })
       }
 
-      const downloadOps = fileList.map((fileName) => new Promise(resolve => {
+      const downloadOps = fileList.map(async (fileName) => {
         const resourceFileOutPath = path.join(outDir, fileName)
         const resourceFileUrl = new URL(fileName, manifestUrl).href
-        request(resourceFileUrl)
-          .pipe(fs.createWriteStream(resourceFileOutPath))
-          .on('finish', () => {
-            console.log(resourceFileUrl)
-            resolve()
-          })
-      }))
+        const response = await fetch(resourceFileUrl)
+        const ws = fs.createWriteStream(resourceFileOutPath)
+        return finished(Readable.fromWeb(response.body).pipe(ws))
+          .then(() => console.log(resourceFileUrl))
+      })
 
       await Promise.all(downloadOps)
 
       resolve()
+    }).catch(error => {
+      throw new Error(`Error from ${manifestUrl}: ${error.cause}`)
     })
   })
 }