Feature/scrape

.env-template

@@ -1,26 +1,8 @@
-# lista de eventos que no queremos que aparezcan (separados por coma)
-# 19155277 = Infobyte (https://www.meetup.com/infobyte/)
-BLACK_LIST=19155277
-
 # 15 minutos de tiempo de vida para los pedidos cacheados
 CACHE_EXPIRATION=900000
 
-# 34 = Tecnología
-# en el caso de querer usar más de 1, separarlos por coma
-CATEGORIES=34
-
-# latitud y longitud que apuntan a Cabildo y Juan B. Justo
-LAT=-34.578175
-LON=-58.426516
-
-# https://secure.meetup.com/meetup_api/key/
-MEETUP_API_KEY=1234567890
-
 NODE_ENV=development
 
 # rango en millas (partiendo desde latitud y longitud) que se va a tener en cuenta
 # para buscar meetups
-RADIUS=10
-
-# lista de eventos a buscar que sabemos que no entran en el rango (separados por coma)
-# WHITE_LIST=
+RADIUS=10

lib/scrape-events.js

@@ -1,46 +1,65 @@
 const cheerio = require('cheerio')
 const got = require('got')
 const moment = require('moment')
+let yup = require('yup')
+
+let schema = yup.array().of(
+    yup.object().shape({
+        date: yup.date().required(),
+        eventName: yup.string().required(),
+        eventLink: yup.string().url().required(),
+        place:  yup.string()
+    })
+)
 
 module.exports = async function scrapeEvents() {
-    const { RADIUS } = process.env
-    const { body } = await got(
-        `https://www.meetup.com/es-ES/find/events/tech/?allMeetups=false&radius=${RADIUS}&userFreeform=buenos+ai&mcId=c1000296&mcName=Buenos+Aires%2C+AR`
-    )
-    const $ = cheerio.load(body)
-    const days = $('li.event-listing-container-li')
-    const dayInfo = $('li.date-indicator')
-    const eventsArray = []
-    days.each((dayIndex, li) => {
-        const events = $(li).find('ul.event-listing-container > li')
-        events.each((_, li) => {
-            $(li).each((_, e) => {
-                const eventDateElement = dayInfo.get(dayIndex)
-                let eventDate
-                if (eventDateElement) {
-                    const {
-                        'data-year': year,
-                        'data-month': month,
-                        'data-day': day
-                    } = dayInfo.get(dayIndex).attribs
-                    eventDate = `${year}-${month}-${day}`
-                } else {
-                    eventDate = moment().format('YYYY-MM-DD')
-                }                
-                eventsArray.push({
-                    date:
-                        moment.utc(
-                            `${eventDate} ${$(e).find('div a').first().text().trim()}`,
-                            'YYYY-MM-DD HH:mm'
-                        ),
-                    eventName:
-                        `${$(e).find('div[itemprop="location"]').text().trim()} - ${$(e).find('a.event span[itemprop="name"]').text().trim()}`,
-                    eventLink: $(e).find('a.event').prop('href'),
-                    place: '',
-                    attendeeCount: $(e).find('div.attendee-count').text().replace(/\n+/g, ' ')        
+    try {
+        const { RADIUS } = process.env
+        const { body } = await got(
+            `https://www.meetup.com/es-ES/find/events/tech/?allMeetups=false&radius=${RADIUS}&userFreeform=buenos+ai&mcId=c1000296&mcName=Buenos+Aires%2C+AR`
+        )
+        const $ = cheerio.load(body)
+        const days = $('li.event-listing-container-li')
+        const dayInfo = $('li.date-indicator')
+        const eventsArray = []
+        days.each((dayIndex, li) => {
+            const events = $(li).find('ul.event-listing-container > li')
+            events.each((_, li) => {
+                $(li).each((_, e) => {
+                    const eventDateElement = dayInfo.get(dayIndex)
+                    let eventDate
+                    if (eventDateElement) {
+                        const {
+                            'data-year': year,
+                            'data-month': month,
+                            'data-day': day
+                        } = dayInfo.get(dayIndex).attribs
+                        eventDate = `${year}-${month}-${day}`
+                    } else {
+                        eventDate = moment().format('YYYY-MM-DD')
+                    }
+                    eventsArray.push({
+                        date:
+                            moment.utc(
+                                `${eventDate} ${$(e).find('div a').first().text().trim()}`,
+                                'YYYY-MM-DD HH:mm'
+                            ),
+                        eventName:
+                            `${$(e).find('div[itemprop="location"]').text().trim()} - ${$(e).find('a.event span[itemprop="name"]').text().trim()}`,
+                        eventLink: $(e).find('a.event').prop('href'),
+                        place: ''
+                    })
                 })
             })
         })
-    })
-    return eventsArray
+        const isValid = schema.isValidSync(eventsArray)
+        console.log('schema valido: ', isValid)
+        if (isValid) {
+            return eventsArray
+        } else {
+            return []
+        }
+    } catch (e) {
+        return []
+    }
 }

package.json

@@ -18,13 +18,14 @@
         "micro-dev": "^3.0.0"
     },
     "dependencies": {
-        "cheerio": "^1.0.0-rc.3",
+        "cheerio": "0.22.0",
         "dotenv": "^8.0.0",
         "got": "^9.6.0",
         "memory-cache": "^0.2.0",
         "micro": "^9.3.4",
         "micro-cors": "^0.1.1",
-        "moment": "^2.24.0"
+        "moment": "^2.24.0",
+        "yup": "^0.27.0"
     },
     "repository": {
         "type": "git",