From 0dffbd1f2a4509a1c99fb075a897cc74e2ce204a Mon Sep 17 00:00:00 2001 From: Mateo Silguero Date: Sat, 19 Oct 2019 10:16:37 -0300 Subject: [PATCH 1/6] eslint --- index.js | 27 ++--------- lib/format-events.js | 19 -------- lib/get-events.js | 7 --- lib/get-meetups.js | 17 ------- lib/make-request.js | 5 --- lib/scrape-events.js | 29 ++++++++++++ package-lock.json | 104 ++++++++++++++++++++++++++++++++++--------- package.json | 1 + 8 files changed, 117 insertions(+), 92 deletions(-) delete mode 100644 lib/format-events.js delete mode 100644 lib/get-events.js delete mode 100644 lib/get-meetups.js delete mode 100644 lib/make-request.js create mode 100644 lib/scrape-events.js diff --git a/index.js b/index.js index f3910e8..8058b35 100644 --- a/index.js +++ b/index.js @@ -7,40 +7,21 @@ if (process.env.NODE_ENV === 'development') { const cache = require('memory-cache') const microCors = require('micro-cors') const { send } = require('micro') +const url = require('url') -const getEvents = require('./lib/get-events') -const getMeetups = require('./lib/get-meetups') +const scrapeEvents = require('./lib/scrape-events') -const blackList = process.env.BLACK_LIST ? process.env.BLACK_LIST.split(',') : [] -const cacheExpiration = parseInt(process.env.CACHE_EXPIRATION) +const cacheExpiration = parseInt(process.env.CACHE_EXPIRATION || 1000) const cors = microCors({ allowMethods: ['GET'] }) -const whiteList = process.env.WHITE_LIST ? process.env.WHITE_LIST.split(',') : [] async function handler(req, res) { try { // si el resultado del API no fue previamente cacheado if (!cache.get('data')) { // obtenemos un array de meetups que corresponden al rango es búsqueda - const data = await getMeetups() - // filtramos los eventos que no queremos que aparezcan - .then(eventsList => - eventsList.filter(event => !blackList.includes(event.id.toString())) - ) - // agregamos los meetups que queremos que aparezcan que no entran en el rango de búsqueda - .then(meetups => meetups.concat(whiteList.map(m => ({ urlname: m })))) - // buscamos los eventos de esos meetups - .then(meetups => meetups.map(m => getEvents(m.urlname))) - // cuando obtengamos toda la información, vamos a tener un array de arrays, - // donde cada uno es una lista de eventos - .then(eventsProms => Promise.all(eventsProms)) - // generamos un array de 1 solo nivel por medio de un reduce - // que solo concatena todos los eventos de los diferentes meetups - .then(eventsLists => - eventsLists.reduce((output, events) => output.concat(events), []) - ) - + const data = await scrapeEvents(url.parse(req.url, true).query.date) // guardamos los datos en cache por el tiempo indicado por configuración cache.put('data', data, cacheExpiration) } diff --git a/lib/format-events.js b/lib/format-events.js deleted file mode 100644 index 029f2e1..0000000 --- a/lib/format-events.js +++ /dev/null @@ -1,19 +0,0 @@ -const moment = require('moment') - -// a partir de los que el API devolvió para el evento, generamos un array que tenga -// solo la información que necesitamos -module.exports = function formatEvents(meetups) { - return meetups.map(event => { - // horario de Argentina - const date = moment.utc(event.time).subtract(3, 'hours') - - return { - date, - eventName: `${event.group.name} - ${event.name}`, - eventLink: event.link, - // hay eventos donde la información del venue no va a estar porque el organizador - // consideró que esa info es privada y accesible solo para los miembros del meetup - place: event.venue ? event.venue.name : '' - } - }) -} diff --git a/lib/get-events.js b/lib/get-events.js deleted file mode 100644 index 570107a..0000000 --- a/lib/get-events.js +++ /dev/null @@ -1,7 +0,0 @@ -const formatEvents = require('./format-events') -const makeRequest = require('./make-request') - -// devuelve un array de arrays donde cada uno es una lista de eventos de un meetup en particular -module.exports = function getEvents(meetupID) { - return makeRequest(`https://api.meetup.com/${meetupID}/events`).then(formatEvents) -} diff --git a/lib/get-meetups.js b/lib/get-meetups.js deleted file mode 100644 index d63a452..0000000 --- a/lib/get-meetups.js +++ /dev/null @@ -1,17 +0,0 @@ -const makeRequest = require('./make-request') -const queryOptions = { - query: { - category: process.env.CATEGORIES, - country: 'ar', - key: process.env.MEETUP_API_KEY, - lat: process.env.LAT, - lon: process.env.LON, - radius: process.env.RADIUS, - sign: true - } -} - -// devuelve una promesa a un array de meetups -module.exports = function getMeetups() { - return makeRequest('https://api.meetup.com/find/groups', queryOptions) -} diff --git a/lib/make-request.js b/lib/make-request.js deleted file mode 100644 index 259957c..0000000 --- a/lib/make-request.js +++ /dev/null @@ -1,5 +0,0 @@ -const got = require('got') - -module.exports = function makeRequest(url, options) { - return got(url, options).then(res => JSON.parse(res.body)) -} diff --git a/lib/scrape-events.js b/lib/scrape-events.js new file mode 100644 index 0000000..391152d --- /dev/null +++ b/lib/scrape-events.js @@ -0,0 +1,29 @@ +const cheerio = require('cheerio') +const got = require('got') +const moment = require('moment') + +module.exports = async function scrapeEvents(date = moment().format('YYYY-MM-DD')) { + const { + RADIUS + } = process.env; + const { body } = await got( + `https://www.meetup.com/es-ES/find/events/tech/?allMeetups=false&radius=${RADIUS}&userFreeform=buenos+ai&mcId=c1000296&mcName=Buenos+Aires%2C+AR` + ) + const $ = cheerio.load(body) + const events = $(`li[data-uniqselector=".container-${date}"] ul.event-listing-container > li`) + const eventsArray = []; + events.each((_, li) => { + $(li).each((_, e) => { + eventsArray.push({ + date: + moment.utc(date + ' ' + $(e).find('div a').first().text().trim(), 'YYYY-MM-DD HH:mm'), + eventName: + `${$(e).find('div[itemprop="location"]').text().trim()} - ${$(e).find('a.event span[itemprop="name"]').text().trim()}`, + eventLink: $(e).find('a.event').prop('href'), + place: '', + attendeeCount: $(e).find('div.attendee-count').text().replace(/\n+/g, ' ') + }) + }) + }) + return eventsArray; +} diff --git a/package-lock.json b/package-lock.json index 2944413..58aff4d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -124,6 +124,11 @@ "defer-to-connect": "^1.0.1" } }, + "@types/node": { + "version": "12.11.1", + "resolved": "https://registry.npmjs.org/@types/node/-/node-12.11.1.tgz", + "integrity": "sha512-TJtwsqZ39pqcljJpajeoofYRfeZ7/I/OMUQ5pR4q5wOKf2ocrUvBAZUMhWsOvKx3dVc/aaV5GluBivt0sWqA5A==" + }, "@types/normalize-package-data": { "version": "2.4.0", "resolved": "https://registry.npmjs.org/@types/normalize-package-data/-/normalize-package-data-2.4.0.tgz", @@ -357,8 +362,7 @@ "boolbase": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", - "integrity": "sha1-aN/1++YMUes3cl6p4+0xDcwed24=", - "dev": true + "integrity": "sha1-aN/1++YMUes3cl6p4+0xDcwed24=" }, "boxen": { "version": "1.3.0", @@ -540,6 +544,68 @@ "integrity": "sha512-mT8iDcrh03qDGRRmoA2hmBJnxpllMR+0/0qlzjqZES6NdiWDcZkCNAk4rPFZ9Q85r27unkiNNg8ZOiwZXBHwcA==", "dev": true }, + "cheerio": { + "version": "1.0.0-rc.3", + "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.0.0-rc.3.tgz", + "integrity": "sha512-0td5ijfUPuubwLUu0OBoe98gZj8C/AA+RW3v67GPlGOrvxWjZmBXiBCRU+I8VEiNyJzjth40POfHiz2RB3gImA==", + "requires": { + "css-select": "~1.2.0", + "dom-serializer": "~0.1.1", + "entities": "~1.1.1", + "htmlparser2": "^3.9.1", + "lodash": "^4.15.0", + "parse5": "^3.0.1" + }, + "dependencies": { + "dom-serializer": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-0.1.1.tgz", + "integrity": "sha512-l0IU0pPzLWSHBcieZbpOKgkIn3ts3vAh7ZuFyXNwJxJXk/c4Gwj9xaTJwIDVQCXawWD0qb3IzMGH5rglQaO0XA==", + "requires": { + "domelementtype": "^1.3.0", + "entities": "^1.1.1" + } + }, + "domhandler": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-2.4.2.tgz", + "integrity": "sha512-JiK04h0Ht5u/80fdLMCEmV4zkNh2BcoMFBmZ/91WtYZ8qVXSKjiw7fXMgFPnHcSZgOo3XdinHvmnDUeMf5R4wA==", + "requires": { + "domelementtype": "1" + } + }, + "htmlparser2": { + "version": "3.10.1", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-3.10.1.tgz", + "integrity": "sha512-IgieNijUMbkDovyoKObU1DUhm1iwNYE/fuifEoEHfd1oZKZDaONBSkal7Y01shxsM49R4XaMdGez3WnF9UfiCQ==", + "requires": { + "domelementtype": "^1.3.1", + "domhandler": "^2.3.0", + "domutils": "^1.5.1", + "entities": "^1.1.1", + "inherits": "^2.0.1", + "readable-stream": "^3.1.1" + }, + "dependencies": { + "domelementtype": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-1.3.1.tgz", + "integrity": "sha512-BSKB+TSpMpFI/HOxCNr1O8aMOTZ8hT3pM3GQ0w/mWRmkhEDSFJkkyzz4XQsBV44BChwGkrDfMyjVD0eA2aFV3w==" + } + } + }, + "readable-stream": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.4.0.tgz", + "integrity": "sha512-jItXPLmrSR8jmTRmRWJXCnGJsfy85mB3Wd/uINMXA65yrnFo0cPClFIUWzo2najVNSl+mx7/4W8ttlLWJe99pQ==", + "requires": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + } + } + } + }, "chokidar": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-2.0.3.tgz", @@ -772,7 +838,6 @@ "version": "1.2.0", "resolved": "https://registry.npmjs.org/css-select/-/css-select-1.2.0.tgz", "integrity": "sha1-KzoRBTnFNV8c2NMUYj6HCxIeyFg=", - "dev": true, "requires": { "boolbase": "~1.0.0", "css-what": "2.1", @@ -783,8 +848,7 @@ "css-what": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/css-what/-/css-what-2.1.0.tgz", - "integrity": "sha1-lGfQMsOM+u+58teVASUwYvh/ob0=", - "dev": true + "integrity": "sha1-lGfQMsOM+u+58teVASUwYvh/ob0=" }, "debounce": { "version": "1.1.0", @@ -916,7 +980,6 @@ "version": "0.1.0", "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-0.1.0.tgz", "integrity": "sha1-BzxpdUbOB4DOI75KKOKT5AvDDII=", - "dev": true, "requires": { "domelementtype": "~1.1.1", "entities": "~1.1.1" @@ -925,16 +988,14 @@ "domelementtype": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-1.1.3.tgz", - "integrity": "sha1-vSh3PiZCiBrsUVRJJCmcXNgiGFs=", - "dev": true + "integrity": "sha1-vSh3PiZCiBrsUVRJJCmcXNgiGFs=" } } }, "domelementtype": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-1.3.0.tgz", - "integrity": "sha1-sXrtguirWeUt2cGbF1bg/BhyBMI=", - "dev": true + "integrity": "sha1-sXrtguirWeUt2cGbF1bg/BhyBMI=" }, "domhandler": { "version": "2.1.0", @@ -949,7 +1010,6 @@ "version": "1.5.1", "resolved": "https://registry.npmjs.org/domutils/-/domutils-1.5.1.tgz", "integrity": "sha1-3NhIiib1Y9YQeeSMn3t+Mjc2gs8=", - "dev": true, "requires": { "dom-serializer": "0", "domelementtype": "1" @@ -982,8 +1042,7 @@ "entities": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/entities/-/entities-1.1.1.tgz", - "integrity": "sha1-blwtClYhtdra7O+AuQ7ftc13cvA=", - "dev": true + "integrity": "sha1-blwtClYhtdra7O+AuQ7ftc13cvA=" }, "error-ex": { "version": "1.3.2", @@ -2641,8 +2700,7 @@ "lodash": { "version": "4.17.11", "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz", - "integrity": "sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg==", - "dev": true + "integrity": "sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg==" }, "lowercase-keys": { "version": "1.0.1", @@ -2902,7 +2960,6 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-1.0.1.tgz", "integrity": "sha1-mSms32KPwsQQmN6rgqxYDPFJquQ=", - "dev": true, "requires": { "boolbase": "~1.0.0" } @@ -3070,6 +3127,14 @@ "json-parse-better-errors": "^1.0.1" } }, + "parse5": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-3.0.3.tgz", + "integrity": "sha512-rgO9Zg5LLLkfJF9E6CCmXlSE4UVceloys8JrFqCcHloC3usd/kJCyPDwH2SOlzix2j3xaP9sUX3e8+kvkuleAA==", + "requires": { + "@types/node": "*" + } + }, "pascalcase": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/pascalcase/-/pascalcase-0.1.1.tgz", @@ -3458,8 +3523,7 @@ "safe-buffer": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", - "dev": true + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" }, "safe-regex": { "version": "1.1.0", @@ -3804,7 +3868,6 @@ "version": "1.1.1", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", - "dev": true, "requires": { "safe-buffer": "~5.1.0" } @@ -4118,8 +4181,7 @@ "util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", - "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=", - "dev": true + "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" }, "utila": { "version": "0.4.0", diff --git a/package.json b/package.json index f86aebb..737ca80 100644 --- a/package.json +++ b/package.json @@ -18,6 +18,7 @@ "micro-dev": "^3.0.0" }, "dependencies": { + "cheerio": "^1.0.0-rc.3", "dotenv": "^8.0.0", "got": "^9.6.0", "memory-cache": "^0.2.0", From 81857adc0a82feec1259c834012050821d4afc8b Mon Sep 17 00:00:00 2001 From: Mateo Silguero Date: Sat, 19 Oct 2019 10:16:52 -0300 Subject: [PATCH 2/6] eslint --- lib/scrape-events.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/scrape-events.js b/lib/scrape-events.js index 391152d..b7e0f1c 100644 --- a/lib/scrape-events.js +++ b/lib/scrape-events.js @@ -5,13 +5,13 @@ const moment = require('moment') module.exports = async function scrapeEvents(date = moment().format('YYYY-MM-DD')) { const { RADIUS - } = process.env; + } = process.env const { body } = await got( `https://www.meetup.com/es-ES/find/events/tech/?allMeetups=false&radius=${RADIUS}&userFreeform=buenos+ai&mcId=c1000296&mcName=Buenos+Aires%2C+AR` ) const $ = cheerio.load(body) const events = $(`li[data-uniqselector=".container-${date}"] ul.event-listing-container > li`) - const eventsArray = []; + const eventsArray = [] events.each((_, li) => { $(li).each((_, e) => { eventsArray.push({ @@ -25,5 +25,5 @@ module.exports = async function scrapeEvents(date = moment().format('YYYY-MM-DD' }) }) }) - return eventsArray; + return eventsArray } From e0e6260b929d8a1c299b06810b3b6cab2dd98818 Mon Sep 17 00:00:00 2001 From: Mateo Silguero Date: Sat, 19 Oct 2019 15:39:26 -0300 Subject: [PATCH 3/6] busca todos los eventos, no solo los de hoy --- index.js | 3 +-- lib/scrape-events.js | 49 +++++++++++++++++++++++++++++--------------- 2 files changed, 34 insertions(+), 18 deletions(-) diff --git a/index.js b/index.js index 8058b35..9f0b9df 100644 --- a/index.js +++ b/index.js @@ -7,7 +7,6 @@ if (process.env.NODE_ENV === 'development') { const cache = require('memory-cache') const microCors = require('micro-cors') const { send } = require('micro') -const url = require('url') const scrapeEvents = require('./lib/scrape-events') @@ -21,7 +20,7 @@ async function handler(req, res) { // si el resultado del API no fue previamente cacheado if (!cache.get('data')) { // obtenemos un array de meetups que corresponden al rango es búsqueda - const data = await scrapeEvents(url.parse(req.url, true).query.date) + const data = await scrapeEvents() // guardamos los datos en cache por el tiempo indicado por configuración cache.put('data', data, cacheExpiration) } diff --git a/lib/scrape-events.js b/lib/scrape-events.js index b7e0f1c..96a3de3 100644 --- a/lib/scrape-events.js +++ b/lib/scrape-events.js @@ -2,28 +2,45 @@ const cheerio = require('cheerio') const got = require('got') const moment = require('moment') -module.exports = async function scrapeEvents(date = moment().format('YYYY-MM-DD')) { - const { - RADIUS - } = process.env +module.exports = async function scrapeEvents() { + const { RADIUS } = process.env const { body } = await got( `https://www.meetup.com/es-ES/find/events/tech/?allMeetups=false&radius=${RADIUS}&userFreeform=buenos+ai&mcId=c1000296&mcName=Buenos+Aires%2C+AR` ) const $ = cheerio.load(body) - const events = $(`li[data-uniqselector=".container-${date}"] ul.event-listing-container > li`) + const days = $('li.event-listing-container-li') + const dayInfo = $('li.date-indicator') const eventsArray = [] - events.each((_, li) => { - $(li).each((_, e) => { - eventsArray.push({ - date: - moment.utc(date + ' ' + $(e).find('div a').first().text().trim(), 'YYYY-MM-DD HH:mm'), - eventName: - `${$(e).find('div[itemprop="location"]').text().trim()} - ${$(e).find('a.event span[itemprop="name"]').text().trim()}`, - eventLink: $(e).find('a.event').prop('href'), - place: '', - attendeeCount: $(e).find('div.attendee-count').text().replace(/\n+/g, ' ') + days.each((dayIndex, li) => { + const events = $(li).find('ul.event-listing-container > li') + events.each((_, li) => { + $(li).each((_, e) => { + const eventDateElement = dayInfo.get(dayIndex) + let eventDate + if (eventDateElement) { + const { + 'data-year': year, + 'data-month': month, + 'data-day': day + } = dayInfo.get(dayIndex).attribs + eventDate = `${year}-${month}-${day}` + } else { + eventDate = moment().format('YYYY-MM-DD') + } + eventsArray.push({ + date: + moment.utc( + `${eventDate} ${$(e).find('div a').first().text().trim()}`, + 'YYYY-MM-DD HH:mm' + ), + eventName: + `${$(e).find('div[itemprop="location"]').text().trim()} - ${$(e).find('a.event span[itemprop="name"]').text().trim()}`, + eventLink: $(e).find('a.event').prop('href'), + place: '', + attendeeCount: $(e).find('div.attendee-count').text().replace(/\n+/g, ' ') + }) }) }) }) return eventsArray -} +} \ No newline at end of file From df876a289b88e113bc7341daf6ac3ca9789739bc Mon Sep 17 00:00:00 2001 From: Mateo Silguero Date: Sat, 19 Oct 2019 15:40:42 -0300 Subject: [PATCH 4/6] cache-expiration --- index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/index.js b/index.js index 9f0b9df..ea9edea 100644 --- a/index.js +++ b/index.js @@ -10,7 +10,7 @@ const { send } = require('micro') const scrapeEvents = require('./lib/scrape-events') -const cacheExpiration = parseInt(process.env.CACHE_EXPIRATION || 1000) +const cacheExpiration = parseInt(process.env.CACHE_EXPIRATION) const cors = microCors({ allowMethods: ['GET'] }) From 637875d3f1f4f988c4c2bc5bcbd98c577d63e39e Mon Sep 17 00:00:00 2001 From: Mateo Silguero Date: Tue, 22 Oct 2019 09:53:49 -0300 Subject: [PATCH 5/6] requested changes --- .env-template | 20 +----- lib/scrape-events.js | 93 +++++++++++++++---------- package-lock.json | 160 ++++++++++++++++++++++++++++++++++--------- package.json | 5 +- 4 files changed, 186 insertions(+), 92 deletions(-) diff --git a/.env-template b/.env-template index 3de5698..4fdafc7 100644 --- a/.env-template +++ b/.env-template @@ -1,26 +1,8 @@ -# lista de eventos que no queremos que aparezcan (separados por coma) -# 19155277 = Infobyte (https://www.meetup.com/infobyte/) -BLACK_LIST=19155277 - # 15 minutos de tiempo de vida para los pedidos cacheados CACHE_EXPIRATION=900000 -# 34 = Tecnología -# en el caso de querer usar más de 1, separarlos por coma -CATEGORIES=34 - -# latitud y longitud que apuntan a Cabildo y Juan B. Justo -LAT=-34.578175 -LON=-58.426516 - -# https://secure.meetup.com/meetup_api/key/ -MEETUP_API_KEY=1234567890 - NODE_ENV=development # rango en millas (partiendo desde latitud y longitud) que se va a tener en cuenta # para buscar meetups -RADIUS=10 - -# lista de eventos a buscar que sabemos que no entran en el rango (separados por coma) -# WHITE_LIST= +RADIUS=10 \ No newline at end of file diff --git a/lib/scrape-events.js b/lib/scrape-events.js index 96a3de3..9bc0b6b 100644 --- a/lib/scrape-events.js +++ b/lib/scrape-events.js @@ -1,46 +1,65 @@ const cheerio = require('cheerio') const got = require('got') const moment = require('moment') +let yup = require('yup') + +let schema = yup.array().of( + yup.object().shape({ + date: yup.date().required(), + eventName: yup.string().required(), + eventLink: yup.string().url().required(), + place: yup.string() + }) +) module.exports = async function scrapeEvents() { - const { RADIUS } = process.env - const { body } = await got( - `https://www.meetup.com/es-ES/find/events/tech/?allMeetups=false&radius=${RADIUS}&userFreeform=buenos+ai&mcId=c1000296&mcName=Buenos+Aires%2C+AR` - ) - const $ = cheerio.load(body) - const days = $('li.event-listing-container-li') - const dayInfo = $('li.date-indicator') - const eventsArray = [] - days.each((dayIndex, li) => { - const events = $(li).find('ul.event-listing-container > li') - events.each((_, li) => { - $(li).each((_, e) => { - const eventDateElement = dayInfo.get(dayIndex) - let eventDate - if (eventDateElement) { - const { - 'data-year': year, - 'data-month': month, - 'data-day': day - } = dayInfo.get(dayIndex).attribs - eventDate = `${year}-${month}-${day}` - } else { - eventDate = moment().format('YYYY-MM-DD') - } - eventsArray.push({ - date: - moment.utc( - `${eventDate} ${$(e).find('div a').first().text().trim()}`, - 'YYYY-MM-DD HH:mm' - ), - eventName: - `${$(e).find('div[itemprop="location"]').text().trim()} - ${$(e).find('a.event span[itemprop="name"]').text().trim()}`, - eventLink: $(e).find('a.event').prop('href'), - place: '', - attendeeCount: $(e).find('div.attendee-count').text().replace(/\n+/g, ' ') + try { + const { RADIUS } = process.env + const { body } = await got( + `https://www.meetup.com/es-ES/find/events/tech/?allMeetups=false&radius=${RADIUS}&userFreeform=buenos+ai&mcId=c1000296&mcName=Buenos+Aires%2C+AR` + ) + const $ = cheerio.load(body) + const days = $('li.event-listing-container-li') + const dayInfo = $('li.date-indicator') + const eventsArray = [] + days.each((dayIndex, li) => { + const events = $(li).find('ul.event-listing-container > li') + events.each((_, li) => { + $(li).each((_, e) => { + const eventDateElement = dayInfo.get(dayIndex) + let eventDate + if (eventDateElement) { + const { + 'data-year': year, + 'data-month': month, + 'data-day': day + } = dayInfo.get(dayIndex).attribs + eventDate = `${year}-${month}-${day}` + } else { + eventDate = moment().format('YYYY-MM-DD') + } + eventsArray.push({ + date: + moment.utc( + `${eventDate} ${$(e).find('div a').first().text().trim()}`, + 'YYYY-MM-DD HH:mm' + ), + eventName: + `${$(e).find('div[itemprop="location"]').text().trim()} - ${$(e).find('a.event span[itemprop="name"]').text().trim()}`, + eventLink: $(e).find('a.event').prop('href'), + place: '' + }) }) }) }) - }) - return eventsArray + const isValid = schema.isValidSync(eventsArray) + console.log('schema valido: ', isValid) + if (isValid) { + return eventsArray + } else { + return [] + } + } catch (e) { + return [] + } } \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 58aff4d..613b788 100644 --- a/package-lock.json +++ b/package-lock.json @@ -72,6 +72,14 @@ "integrity": "sha512-9mUqkL1FF5T7f0WDFfAoDdiMVPWsdD1gZYzSnaXsxUCUqzuch/8of9G3VUSNiZmMBoRxT3neyVsqeiL/ZPcjew==", "dev": true }, + "@babel/runtime": { + "version": "7.6.3", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.6.3.tgz", + "integrity": "sha512-kq6anf9JGjW8Nt5rYfEuGRaEAaH1mkv3Bbu6rYvLOpPh/RusSJXuKPEAoZ7L7gybZkchE8+NV5g9vKF4AGAtsA==", + "requires": { + "regenerator-runtime": "^0.13.2" + } + }, "@babel/template": { "version": "7.4.4", "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.4.4.tgz", @@ -124,11 +132,6 @@ "defer-to-connect": "^1.0.1" } }, - "@types/node": { - "version": "12.11.1", - "resolved": "https://registry.npmjs.org/@types/node/-/node-12.11.1.tgz", - "integrity": "sha512-TJtwsqZ39pqcljJpajeoofYRfeZ7/I/OMUQ5pR4q5wOKf2ocrUvBAZUMhWsOvKx3dVc/aaV5GluBivt0sWqA5A==" - }, "@types/normalize-package-data": { "version": "2.4.0", "resolved": "https://registry.npmjs.org/@types/normalize-package-data/-/normalize-package-data-2.4.0.tgz", @@ -545,26 +548,32 @@ "dev": true }, "cheerio": { - "version": "1.0.0-rc.3", - "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.0.0-rc.3.tgz", - "integrity": "sha512-0td5ijfUPuubwLUu0OBoe98gZj8C/AA+RW3v67GPlGOrvxWjZmBXiBCRU+I8VEiNyJzjth40POfHiz2RB3gImA==", + "version": "0.22.0", + "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-0.22.0.tgz", + "integrity": "sha1-qbqoYKP5tZWmuBsahocxIe06Jp4=", "requires": { "css-select": "~1.2.0", - "dom-serializer": "~0.1.1", + "dom-serializer": "~0.1.0", "entities": "~1.1.1", "htmlparser2": "^3.9.1", - "lodash": "^4.15.0", - "parse5": "^3.0.1" + "lodash.assignin": "^4.0.9", + "lodash.bind": "^4.1.4", + "lodash.defaults": "^4.0.1", + "lodash.filter": "^4.4.0", + "lodash.flatten": "^4.2.0", + "lodash.foreach": "^4.3.0", + "lodash.map": "^4.4.0", + "lodash.merge": "^4.4.0", + "lodash.pick": "^4.2.1", + "lodash.reduce": "^4.4.0", + "lodash.reject": "^4.4.0", + "lodash.some": "^4.4.0" }, "dependencies": { - "dom-serializer": { - "version": "0.1.1", - "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-0.1.1.tgz", - "integrity": "sha512-l0IU0pPzLWSHBcieZbpOKgkIn3ts3vAh7ZuFyXNwJxJXk/c4Gwj9xaTJwIDVQCXawWD0qb3IzMGH5rglQaO0XA==", - "requires": { - "domelementtype": "^1.3.0", - "entities": "^1.1.1" - } + "domelementtype": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-1.3.1.tgz", + "integrity": "sha512-BSKB+TSpMpFI/HOxCNr1O8aMOTZ8hT3pM3GQ0w/mWRmkhEDSFJkkyzz4XQsBV44BChwGkrDfMyjVD0eA2aFV3w==" }, "domhandler": { "version": "2.4.2", @@ -585,13 +594,6 @@ "entities": "^1.1.1", "inherits": "^2.0.1", "readable-stream": "^3.1.1" - }, - "dependencies": { - "domelementtype": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-1.3.1.tgz", - "integrity": "sha512-BSKB+TSpMpFI/HOxCNr1O8aMOTZ8hT3pM3GQ0w/mWRmkhEDSFJkkyzz4XQsBV44BChwGkrDfMyjVD0eA2aFV3w==" - } } }, "readable-stream": { @@ -1438,6 +1440,11 @@ "integrity": "sha512-a1hQMktqW9Nmqr5aktAux3JMNqaucxGcjtjWnZLHX7yyPCmlSV3M54nGYbqT8K+0GhF3NBgmJCc3ma+WOgX8Jg==", "dev": true }, + "fn-name": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/fn-name/-/fn-name-2.0.1.tgz", + "integrity": "sha1-UhTXU3pNBqSjAcDMJi/rhBiAAuc=" + }, "for-in": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/for-in/-/for-in-1.0.2.tgz", @@ -2702,6 +2709,66 @@ "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz", "integrity": "sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg==" }, + "lodash.assignin": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/lodash.assignin/-/lodash.assignin-4.2.0.tgz", + "integrity": "sha1-uo31+4QesKPoBEIysOJjqNxqKKI=" + }, + "lodash.bind": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/lodash.bind/-/lodash.bind-4.2.1.tgz", + "integrity": "sha1-euMBfpOWIqwxt9fX3LGzTbFpDTU=" + }, + "lodash.defaults": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", + "integrity": "sha1-0JF4cW/+pN3p5ft7N/bwgCJ0WAw=" + }, + "lodash.filter": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/lodash.filter/-/lodash.filter-4.6.0.tgz", + "integrity": "sha1-ZosdSYFgOuHMWm+nYBQ+SAtMSs4=" + }, + "lodash.flatten": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/lodash.flatten/-/lodash.flatten-4.4.0.tgz", + "integrity": "sha1-8xwiIlqWMtK7+OSt2+8kCqdlph8=" + }, + "lodash.foreach": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.foreach/-/lodash.foreach-4.5.0.tgz", + "integrity": "sha1-Gmo16s5AEoDH8G3d7DUWWrJ+PlM=" + }, + "lodash.map": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/lodash.map/-/lodash.map-4.6.0.tgz", + "integrity": "sha1-dx7Hg540c9nEzeKLGTlMNWL09tM=" + }, + "lodash.merge": { + "version": "4.6.2", + "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", + "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==" + }, + "lodash.pick": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/lodash.pick/-/lodash.pick-4.4.0.tgz", + "integrity": "sha1-UvBWEP/53tQiYRRB7R/BI6AwAbM=" + }, + "lodash.reduce": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/lodash.reduce/-/lodash.reduce-4.6.0.tgz", + "integrity": "sha1-8atrg5KZrUj3hKu/R2WW8DuRTTs=" + }, + "lodash.reject": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/lodash.reject/-/lodash.reject-4.6.0.tgz", + "integrity": "sha1-gNZJLcFHCGS79YNTO2UfQqn1JBU=" + }, + "lodash.some": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/lodash.some/-/lodash.some-4.6.0.tgz", + "integrity": "sha1-G7nzFO9ri63tE7VJFpsqlF62jk0=" + }, "lowercase-keys": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-1.0.1.tgz", @@ -3127,14 +3194,6 @@ "json-parse-better-errors": "^1.0.1" } }, - "parse5": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/parse5/-/parse5-3.0.3.tgz", - "integrity": "sha512-rgO9Zg5LLLkfJF9E6CCmXlSE4UVceloys8JrFqCcHloC3usd/kJCyPDwH2SOlzix2j3xaP9sUX3e8+kvkuleAA==", - "requires": { - "@types/node": "*" - } - }, "pascalcase": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/pascalcase/-/pascalcase-0.1.1.tgz", @@ -3288,6 +3347,11 @@ "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==", "dev": true }, + "property-expr": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/property-expr/-/property-expr-1.5.1.tgz", + "integrity": "sha512-CGuc0VUTGthpJXL36ydB6jnbyOf/rAHFvmVrJlH+Rg0DqqLFQGAP6hIaxD/G0OAmBJPhXDHuEJigrp0e0wFV6g==" + }, "pseudomap": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz", @@ -3366,6 +3430,11 @@ "set-immediate-shim": "^1.0.1" } }, + "regenerator-runtime": { + "version": "0.13.3", + "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.3.tgz", + "integrity": "sha512-naKIZz2GQ8JWh///G7L3X6LaQUAMp2lvb1rvwwsURe/VXwD6VMfr+/1NuNw3ag8v2kY1aQ/go5SNn79O9JU7yw==" + }, "regex-not": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/regex-not/-/regex-not-1.0.2.tgz", @@ -3910,6 +3979,11 @@ "has-flag": "^3.0.0" } }, + "synchronous-promise": { + "version": "2.0.10", + "resolved": "https://registry.npmjs.org/synchronous-promise/-/synchronous-promise-2.0.10.tgz", + "integrity": "sha512-6PC+JRGmNjiG3kJ56ZMNWDPL8hjyghF5cMXIFOKg+NiwwEZZIvxTWd0pinWKyD227odg9ygF8xVhhz7gb8Uq7A==" + }, "table": { "version": "5.4.1", "resolved": "https://registry.npmjs.org/table/-/table-5.4.1.tgz", @@ -4033,6 +4107,11 @@ "repeat-string": "^1.6.1" } }, + "toposort": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/toposort/-/toposort-2.0.2.tgz", + "integrity": "sha1-riF2gXXRVZ1IvvNUILL0li8JwzA=" + }, "trim-right": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/trim-right/-/trim-right-1.0.1.tgz", @@ -4324,6 +4403,19 @@ "requires": { "camelcase": "^4.1.0" } + }, + "yup": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/yup/-/yup-0.27.0.tgz", + "integrity": "sha512-v1yFnE4+u9za42gG/b/081E7uNW9mUj3qtkmelLbW5YPROZzSH/KUUyJu9Wt8vxFJcT9otL/eZopS0YK1L5yPQ==", + "requires": { + "@babel/runtime": "^7.0.0", + "fn-name": "~2.0.1", + "lodash": "^4.17.11", + "property-expr": "^1.5.0", + "synchronous-promise": "^2.0.6", + "toposort": "^2.0.2" + } } } } diff --git a/package.json b/package.json index 737ca80..c5cfd6d 100644 --- a/package.json +++ b/package.json @@ -18,13 +18,14 @@ "micro-dev": "^3.0.0" }, "dependencies": { - "cheerio": "^1.0.0-rc.3", + "cheerio": "0.22.0", "dotenv": "^8.0.0", "got": "^9.6.0", "memory-cache": "^0.2.0", "micro": "^9.3.4", "micro-cors": "^0.1.1", - "moment": "^2.24.0" + "moment": "^2.24.0", + "yup": "^0.27.0" }, "repository": { "type": "git", From 5b42aea62ab98acbf32d66db787888a63592c5d6 Mon Sep 17 00:00:00 2001 From: Mateo Silguero Date: Tue, 22 Oct 2019 10:42:12 -0300 Subject: [PATCH 6/6] console.log error --- lib/scrape-events.js | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/scrape-events.js b/lib/scrape-events.js index 9bc0b6b..972f7ab 100644 --- a/lib/scrape-events.js +++ b/lib/scrape-events.js @@ -60,6 +60,7 @@ module.exports = async function scrapeEvents() { return [] } } catch (e) { + console.log(e) return [] } } \ No newline at end of file