diff --git a/src/js/app/server/bot.js b/src/js/app/server/bot.js index 95564a7..d459d9f 100644 --- a/src/js/app/server/bot.js +++ b/src/js/app/server/bot.js @@ -1,74 +1,42 @@ // run scraping process -var Model = require('./model'); -//var ParterScraper = require('./scrape-parter.js'); -//var DouScraper = require('./scrape-dou.js'); -var Scraper = require('./scraper.js'); -var request = require('request'), +var Model = require('./model'), + Scraper = require('./scraper.js'), + request = require('request'), debug = require('debug')('bot'), cheerio = require('cheerio'); +module.exports = function(){ + var sourses = ['http://parter.ua',"http://dou.ua/calendar"]; + sourses.forEach(runScraper); +}(); +function runScraper(site){ + Scraper(site).getEventUrls().then(function(eventsUrls) { + eventsUrls.forEach(processEventPage); + }) + .catch(function(err){ + console.error(err); + }); +} -var html2 = "http://dou.ua/calendar"; -var scraper2 = Scraper(html2); -var douUrls = scraper2.getEventUrls(html2); - - - -/* -var html3 = "http://dou.ua/calendar/9637/"; -var scraper2 = Scraper(html2); -scraper2.scrapeEventPage(html3).then(function(model){ - console.log("promised title", model); -}, function(error) {console.log("ERROR DOU SCRAPE", error, html3);})*/ - -/* -var html = 'http://parter.ua'; -getMainPageUrls(html).then(function(allUrls) { - console.log("start for each"); - allUrls.forEach(processEventPage); -}, function(error) {console.log("getMainPageUrls error ", error); -}); -*/ - -function processEventPage(url, index, array){ +function processEventPage(fullUrl, index, array){ debug("processEventPage"); - var fullUrl = html + url; debug("start of model creation", fullUrl); - var scraper = Scraper(html); - scraper.scrapeEventPage(fullUrl).then(addEventToDatabase, - function(error) {debug("processEventPage error ", error, fullUrl); -}); + var scraper = Scraper(fullUrl); + scraper.scrapeEventPage(fullUrl) + .then(addEventToDatabase) + .catch( function(error) { + debug("processEventPage error ", error, fullUrl); + }); }; function addEventToDatabase(eventData){ - var parterEvent = new Model(eventData); - debug("addEventToDatabase", parterEvent.eventTitle); - parterEvent.save(function(err) { + var singleEvent = new Model(eventData); + debug("addEventToDatabase", singleEvent.eventTitle); + singleEvent.save(function(err) { if (err) {debug('Database err saving: ' + url);} }); }; -function getMainPageUrls(html) { - var promise = new Promise(function(resolve, reject) { - debug("getMainPageUrls"); - var url, urls = []; - request(html, function(err,resp, body){ - if (!err && resp.statusCode == 200){ - var $ = cheerio.load(body); - $("div[class='event']").each(function(){ - url = $(this).find('a').attr('href'); - urls.push(url); - }); - debug("resolve", urls.length); - resolve(urls); - } - else{ - cdebug("ERROR in getMainPageUrls()"); - reject(err); - } - }); - }) - return promise; -}; + diff --git a/src/js/app/server/scrape-dou.js b/src/js/app/server/scrape-dou.js index 08839b2..ceee479 100644 --- a/src/js/app/server/scrape-dou.js +++ b/src/js/app/server/scrape-dou.js @@ -4,22 +4,29 @@ var request = require('request'), cheerio = require('cheerio'); //var html = 'http://dou.ua/calendar'; +//503 ERROR!!! function DouScraper(){ debug("DouScraper constructor"); + this.html = 'http://dou.ua/calendar'; } -DouScraper.prototype.getEventUrls = function(html){ - scrapeMainPage(html).then(function(pageQuantity, urls){ - scrapeAllSecondaryDouPages(html,pageQuantity).then(function(nextPagesUrls){ - urls = urls.concat(nextPagesUrls); - }, - function(error){console.log("scrapeAllSecondaryDouPages ERROR!",error);}); - - console.log("eventsQuantity ",urls.length); - return urls;}, - function(error){console.log(error);}); -} +DouScraper.prototype.getEventUrls = function(){ + debug("getEventUrls"); + //debug("this.html",this.html); + var html = this.html; + var promise = new Promise(function(resolve,reject){ + scrapeMainPage(html) + .then(scrapeSecondaryPages) + .then(function(allUrls){ + debug("allUrls, resolved getEventUrls",allUrls.length); + resolve(allUrls); + }) + .catch(function(err){ + reject(err);}); + }); + return promise; +}; DouScraper.prototype.scrapeEventPage = function(fullUrl) { debug("DouScraper scrapeEventPage begin"); @@ -60,7 +67,7 @@ DouScraper.prototype.scrapeEventPage = function(fullUrl) { resolve(model); } else{ - debug("ERROR in scrapeEventPage"); + debug("ERROR in scrapeEventPage", "response code:", resp.statusCode); reject(err); } }); @@ -70,6 +77,7 @@ DouScraper.prototype.scrapeEventPage = function(fullUrl) { }; function scrapeMainPage(html){ + debug("scrapeMainPage",html); var promise = new Promise(function(resolve, reject) { request(html, function(error, resp, body){ @@ -78,12 +86,17 @@ function scrapeMainPage(html){ var $ = cheerio.load(body); var pageQuantity = $('div.b-paging > span:last-of-type').prev().text(); var urls = findDouUrls($); - - - resolve(pageQuantity, urls); + debug("resolve of scrapeMainPage:", pageQuantity, urls.length); + var result = { + pageQuantity: pageQuantity, + urls: urls, + html: html + }; + resolve(result); } else{ + debug("error in scrapeMainPage"); reject(error); }; }); @@ -91,38 +104,56 @@ function scrapeMainPage(html){ return promise; }; -function scrapeAllSecondaryDouPages(html,quantity){ - var urls = []; +function scrapeSecondaryPages(result){ var promise = new Promise(function(resolve,reject){ - for (var i = 2; i <= quantity; i++){ - var pageUrl = html+'/page-'+i; - console.log(pageUrl); - scrapeSecondaryDouPage(pageUrl).then(function(nextUrls){ - urls = urls.concat(nextUrls); - console.log("urls", urls.length); - }, - function(error){ - - console.log("loadDouUrls ERROR!",error, " at ", pageUrl); - reject(error); - }) + var pageQuantity = result.pageQuantity, + html = result.html, + urls = result.urls; + debug("first batch: ", urls.length,pageQuantity); + var urlsArray = []; + //6 should be changed to pageQuantity + //now it throughs 503 error + for (var i = 2; i <= 6; i++){ + var pageUrl = html+'/page-'+i; + //debug("pageUrl:",pageUrl); + urlsArray.push(pageUrl); } - resolve(urls); + debug(urlsArray); + var arrayOfPromises = urlsArray.map(scrapeSingleSecondaryPage); + Promise.all(arrayOfPromises) + .then(function(arrayOfResults){ + var newUrls = [].concat.apply([], arrayOfResults); + urls = urls.concat(newUrls); + debug("FINAL!!!!!!!!",urls.length); + resolve(urls); + }).catch(function(err,pageUrl){ + // log that I have an error, return the entire array; + console.log('A promise failed to resolve', err, "at", pageUrl); + reject(err); + //return arrayOfPromises; + }); }); return promise; }; -function scrapeSecondaryDouPage(pageUrl){ +function scrapeSingleSecondaryPage(pageUrl){ + debug("scrapeSecondaryDouPage"); //console.log("pageUrl",pageUrl); var promise = new Promise(function(resolve,reject){ request(pageUrl, function(err, resp, body){ if (!err && resp.statusCode == 200){ var $ = cheerio.load(body); var urls = findDouUrls($); - console.log(pageUrl,"is loaded"); + debug(pageUrl,"is loaded, urls:", urls.length); resolve(urls); } - else{ reject(err);}; + else{ + console.log("scrapeSingleSecondaryPage ERROR at ",pageUrl, err,resp.statusCode); + if (err){ + reject(err); + } + + }; }); }); return promise; @@ -132,7 +163,7 @@ function findDouUrls($){ var urls = []; $('div.event > div.title > a').each(function(eventTag){ var url = $(this).attr('href'); - debug("url",url); + //debug("url in findDouUrls",url); urls.push(url); }); return urls; diff --git a/src/js/app/server/scrape-parter.js b/src/js/app/server/scrape-parter.js index 7c62057..a2ee13a 100644 --- a/src/js/app/server/scrape-parter.js +++ b/src/js/app/server/scrape-parter.js @@ -3,53 +3,16 @@ var request = require('request'), debug = require('debug')('bot:scrape-parter'), cheerio = require('cheerio'); - -// return model through promises -//separete scrapeEventPage into 2 functions - -function ParterScraper(){ - debug("ParterScraper constructor"); -} - -function getEventImage($) { - var eventImage = $('img[src*="/img/item/"]').attr('src'); - if (eventImage){ - return eventImage; - } - else{ - return ""; - } -}; - -function getEventDescription($) { - var eventDescription = $('p').text(); - return eventDescription; -}; - -function getEventPrice($) { - var eventPrice = $('tr:nth-child(3) > td[align="center"]').first().text(); - return eventPrice; -}; - -function getEventTime($) { - var eventTime = [] - $('tr:nth-child(1) > td[align="center"]').each(function(){ - var time = $(this).text(); - eventTime.push(time); - }); - return eventTime; -}; - ParterScraper.prototype.scrapeEventPage = function(fullUrl) { debug("scrapeEventPage begin"); var promise = new Promise(function(resolve, reject) { request(fullUrl,function(err,resp,body){ if (!err && resp.statusCode == 200){ - debug("scrapeEventPage request is successfull"); + //debug("scrapeEventPage request is successfull"); var $ = cheerio.load(body,{ decodeEntities: false }); - debug("createModel begin"); + //debug("createModel begin"); var eventTitle, eventLocation, eventDescription, eventTime = [], eventLink, eventImage, eventPrice; var eventPage = $('td.center'); @@ -71,7 +34,7 @@ ParterScraper.prototype.scrapeEventPage = function(fullUrl) { eventImage: eventImage } //console.log("createModel",model); - debug("createModel resolve"); + //debug("createModel resolve"); resolve(model); } else{ @@ -84,4 +47,64 @@ ParterScraper.prototype.scrapeEventPage = function(fullUrl) { return promise; }; +ParterScraper.prototype.getEventUrls = function(){ + var html = this.html; + var promise = new Promise(function(resolve, reject) { + debug("getMainPageUrls"); + var url, urls = []; + request(html, function(err,resp, body){ + if (!err && resp.statusCode == 200){ + var $ = cheerio.load(body); + $("div[class='event']").each(function(){ + url = $(this).find('a').attr('href'); + urls.push(html+url); + }); + debug("resolve", urls.length); + resolve(urls); + } + else{ + debug("ERROR in getMainPageUrls()"); + reject(err); + } + }); + }) + return promise; +}; + +function ParterScraper(){ + debug("ParterScraper constructor"); + this.html = 'http://parter.ua'; +} + +function getEventImage($) { + var eventImage = $('img[src*="/img/item/"]').attr('src'); + if (eventImage){ + return eventImage; + } + else{ + return ""; + } +}; + +function getEventDescription($) { + var eventDescription = $('p').text(); + return eventDescription; +}; + +function getEventPrice($) { + var eventPrice = $('tr:nth-child(3) > td[align="center"]').first().text(); + return eventPrice; +}; + +function getEventTime($) { + var eventTime = [] + $('tr:nth-child(1) > td[align="center"]').each(function(){ + var time = $(this).text(); + eventTime.push(time); + }); + return eventTime; +}; + + + module.exports = ParterScraper; diff --git a/src/js/app/server/scraper.js b/src/js/app/server/scraper.js index bf2f1ee..f8f1140 100644 --- a/src/js/app/server/scraper.js +++ b/src/js/app/server/scraper.js @@ -5,10 +5,10 @@ var DouScraper = require('./scrape-dou.js'), function makeScaraper(html){ debug("makeScraper"); - if (html === 'http://parter.ua'){ + if (html.indexOf('http://parter.ua') != -1){ return new ParterScraper(); } - if (html === 'http://dou.ua/calendar') { + if (html.indexOf('http://dou.ua/calendar') != -1) { return new DouScraper(); } }