From 185ad3bf3de2577194456cb879fc00a71846dd5e Mon Sep 17 00:00:00 2001 From: Adialia Date: Tue, 2 Feb 2016 00:44:53 +0200 Subject: [PATCH 1/5] Dou Scraper improved structure --- src/js/app/server/bot.js | 64 ++++++++---------------- src/js/app/server/scrape-dou.js | 78 ++++++++++++++++-------------- src/js/app/server/scrape-parter.js | 25 ++++++++++ src/js/app/server/scraper.js | 2 +- 4 files changed, 88 insertions(+), 81 deletions(-) diff --git a/src/js/app/server/bot.js b/src/js/app/server/bot.js index 95564a7..f9f25ff 100644 --- a/src/js/app/server/bot.js +++ b/src/js/app/server/bot.js @@ -1,42 +1,29 @@ // run scraping process var Model = require('./model'); -//var ParterScraper = require('./scrape-parter.js'); -//var DouScraper = require('./scrape-dou.js'); var Scraper = require('./scraper.js'); var request = require('request'), debug = require('debug')('bot'), cheerio = require('cheerio'); - -var html2 = "http://dou.ua/calendar"; -var scraper2 = Scraper(html2); -var douUrls = scraper2.getEventUrls(html2); +/*var parterScraper = Scraper('http://parter.ua'); +parterScraper.getEventUrls('http://parter.ua').then(function(parterEventsUrls) { + debug("parterEventsUrls"); + parterEventsUrls.forEach(processEventPage); +});*/ +var douScraper = Scraper("http://dou.ua/calendar"); +douScraper.getEventUrls(); -/* -var html3 = "http://dou.ua/calendar/9637/"; -var scraper2 = Scraper(html2); -scraper2.scrapeEventPage(html3).then(function(model){ - console.log("promised title", model); -}, function(error) {console.log("ERROR DOU SCRAPE", error, html3);})*/ -/* -var html = 'http://parter.ua'; -getMainPageUrls(html).then(function(allUrls) { - console.log("start for each"); - allUrls.forEach(processEventPage); -}, function(error) {console.log("getMainPageUrls error ", error); -}); -*/ -function processEventPage(url, index, array){ +function processEventPage(fullUrl, index, array){ debug("processEventPage"); - var fullUrl = html + url; + //var fullUrl = html + url; debug("start of model creation", fullUrl); - var scraper = Scraper(html); + var scraper = Scraper(fullUrl); scraper.scrapeEventPage(fullUrl).then(addEventToDatabase, function(error) {debug("processEventPage error ", error, fullUrl); }); @@ -50,25 +37,12 @@ function addEventToDatabase(eventData){ }); }; -function getMainPageUrls(html) { - var promise = new Promise(function(resolve, reject) { - debug("getMainPageUrls"); - var url, urls = []; - request(html, function(err,resp, body){ - if (!err && resp.statusCode == 200){ - var $ = cheerio.load(body); - $("div[class='event']").each(function(){ - url = $(this).find('a').attr('href'); - urls.push(url); - }); - debug("resolve", urls.length); - resolve(urls); - } - else{ - cdebug("ERROR in getMainPageUrls()"); - reject(err); - } - }); - }) - return promise; -}; + + + +/* +var html3 = "http://dou.ua/calendar/9637/"; +var scraper2 = Scraper(html2); +scraper2.scrapeEventPage(html3).then(function(model){ + console.log("promised title", model); +}, function(error) {console.log("ERROR DOU SCRAPE", error, html3);})*/ diff --git a/src/js/app/server/scrape-dou.js b/src/js/app/server/scrape-dou.js index 08839b2..c8566b2 100644 --- a/src/js/app/server/scrape-dou.js +++ b/src/js/app/server/scrape-dou.js @@ -7,19 +7,15 @@ var request = require('request'), function DouScraper(){ debug("DouScraper constructor"); + this.html = 'http://dou.ua/calendar'; } -DouScraper.prototype.getEventUrls = function(html){ - scrapeMainPage(html).then(function(pageQuantity, urls){ - scrapeAllSecondaryDouPages(html,pageQuantity).then(function(nextPagesUrls){ - urls = urls.concat(nextPagesUrls); - }, - function(error){console.log("scrapeAllSecondaryDouPages ERROR!",error);}); - - console.log("eventsQuantity ",urls.length); - return urls;}, - function(error){console.log(error);}); -} +DouScraper.prototype.getEventUrls = function(){ + debug("getEventUrls"); + //debug("this.html",this.html); + scrapeMainPage(this.html).then(scrapeSecondaryPages) + .catch(function(err){console.log(err);}); +}; DouScraper.prototype.scrapeEventPage = function(fullUrl) { debug("DouScraper scrapeEventPage begin"); @@ -70,6 +66,7 @@ DouScraper.prototype.scrapeEventPage = function(fullUrl) { }; function scrapeMainPage(html){ + debug("scrapeMainPage"); var promise = new Promise(function(resolve, reject) { request(html, function(error, resp, body){ @@ -78,12 +75,17 @@ function scrapeMainPage(html){ var $ = cheerio.load(body); var pageQuantity = $('div.b-paging > span:last-of-type').prev().text(); var urls = findDouUrls($); - - - resolve(pageQuantity, urls); + debug("resolve of scrapeMainPage:", pageQuantity, urls.length); + var result = { + pageQuantity: pageQuantity, + urls: urls, + html: html + }; + resolve(result); } else{ + debug("error in scrapeMainPage"); reject(error); }; }); @@ -91,35 +93,41 @@ function scrapeMainPage(html){ return promise; }; -function scrapeAllSecondaryDouPages(html,quantity){ - var urls = []; - var promise = new Promise(function(resolve,reject){ - for (var i = 2; i <= quantity; i++){ - var pageUrl = html+'/page-'+i; - console.log(pageUrl); - scrapeSecondaryDouPage(pageUrl).then(function(nextUrls){ - urls = urls.concat(nextUrls); - console.log("urls", urls.length); - }, - function(error){ - - console.log("loadDouUrls ERROR!",error, " at ", pageUrl); - reject(error); - }) +function scrapeSecondaryPages(result){ + var pageQuantity = result.pageQuantity, + html = result.html, + urls = result.urls; + debug("first batch: ", urls.length,pageQuantity); + var urlsArray = []; + for (var i = 2; i <= pageQuantity; i++){ + var pageUrl = html+'/page-'+i; + //debug("pageUrl:",pageUrl); + urlsArray.push(pageUrl); } - resolve(urls); - }); - return promise; + debug(urlsArray); + Promise.all(urlsArray.map(scrapeSingleSecondaryPage)) + .then(function(arrayOfResults){ + debug("problem in concat"); + var urls = [].concat.apply([], arrayOfResults); + //urls = urls.concat(nextUrls); + debug("urls", urls.length); + debug("FINAL!!!!!!!!",urls.length); + return urls; + }).catch(function(err){ + console.log(error); + }); }; -function scrapeSecondaryDouPage(pageUrl){ + +function scrapeSingleSecondaryPage(pageUrl){ + debug("scrapeSecondaryDouPage"); //console.log("pageUrl",pageUrl); var promise = new Promise(function(resolve,reject){ request(pageUrl, function(err, resp, body){ if (!err && resp.statusCode == 200){ var $ = cheerio.load(body); var urls = findDouUrls($); - console.log(pageUrl,"is loaded"); + debug(pageUrl,"is loaded, urls:", urls.length); resolve(urls); } else{ reject(err);}; @@ -132,7 +140,7 @@ function findDouUrls($){ var urls = []; $('div.event > div.title > a').each(function(eventTag){ var url = $(this).attr('href'); - debug("url",url); + //debug("url in findDouUrls",url); urls.push(url); }); return urls; diff --git a/src/js/app/server/scrape-parter.js b/src/js/app/server/scrape-parter.js index 7c62057..def345d 100644 --- a/src/js/app/server/scrape-parter.js +++ b/src/js/app/server/scrape-parter.js @@ -9,6 +9,7 @@ var request = require('request'), function ParterScraper(){ debug("ParterScraper constructor"); + this.html = 'http://parter.ua'; } function getEventImage($) { @@ -84,4 +85,28 @@ ParterScraper.prototype.scrapeEventPage = function(fullUrl) { return promise; }; +ParterScraper.prototype.getEventUrls = function(){ + var html = this.html; + var promise = new Promise(function(resolve, reject) { + debug("getMainPageUrls"); + var url, urls = []; + request(html, function(err,resp, body){ + if (!err && resp.statusCode == 200){ + var $ = cheerio.load(body); + $("div[class='event']").each(function(){ + url = $(this).find('a').attr('href'); + urls.push(html+url); + }); + debug("resolve", urls.length); + resolve(urls); + } + else{ + debug("ERROR in getMainPageUrls()"); + reject(err); + } + }); + }) + return promise; +}; + module.exports = ParterScraper; diff --git a/src/js/app/server/scraper.js b/src/js/app/server/scraper.js index bf2f1ee..04b90d1 100644 --- a/src/js/app/server/scraper.js +++ b/src/js/app/server/scraper.js @@ -5,7 +5,7 @@ var DouScraper = require('./scrape-dou.js'), function makeScaraper(html){ debug("makeScraper"); - if (html === 'http://parter.ua'){ + if (html.indexOf('http://parter.ua') != -1){ return new ParterScraper(); } if (html === 'http://dou.ua/calendar') { From a1f797f8f60d991e0511ad1338348cf2ed7e53dc Mon Sep 17 00:00:00 2001 From: Adialia Date: Tue, 2 Feb 2016 16:38:36 +0200 Subject: [PATCH 2/5] improved promises --- src/js/app/server/bot.js | 46 ++++++++++------- src/js/app/server/scrape-dou.js | 55 ++++++++++++++------ src/js/app/server/scrape-parter.js | 80 +++++++++++++++--------------- src/js/app/server/scraper.js | 2 +- 4 files changed, 107 insertions(+), 76 deletions(-) diff --git a/src/js/app/server/bot.js b/src/js/app/server/bot.js index f9f25ff..6aef551 100644 --- a/src/js/app/server/bot.js +++ b/src/js/app/server/bot.js @@ -1,22 +1,38 @@ // run scraping process -var Model = require('./model'); -var Scraper = require('./scraper.js'); -var request = require('request'), +var Model = require('./model'), + Scraper = require('./scraper.js'), + request = require('request'), debug = require('debug')('bot'), cheerio = require('cheerio'); - -/*var parterScraper = Scraper('http://parter.ua'); +var parterScraper = Scraper('http://parter.ua'); parterScraper.getEventUrls('http://parter.ua').then(function(parterEventsUrls) { debug("parterEventsUrls"); parterEventsUrls.forEach(processEventPage); -});*/ +}) +.catch(function(err){ + console.error(err); +}); var douScraper = Scraper("http://dou.ua/calendar"); -douScraper.getEventUrls(); +console.log("douScraper.html",douScraper.html); +douScraper.getEventUrls() +.then(function(douEventsUrls){ + debug("douEventsUrls", douEventsUrls.length); + douEventsUrls.forEach(processEventPage); +}) +.catch(function(err){ + console.error(err); +}); +/* +var html3 = "http://dou.ua/calendar/9637/"; +var scraper2 = Scraper(html2); +scraper2.scrapeEventPage(html3).then(function(model){ + console.log("promised title", model); +}, function(error) {console.log("ERROR DOU SCRAPE", error, html3);})*/ function processEventPage(fullUrl, index, array){ @@ -24,9 +40,11 @@ function processEventPage(fullUrl, index, array){ //var fullUrl = html + url; debug("start of model creation", fullUrl); var scraper = Scraper(fullUrl); - scraper.scrapeEventPage(fullUrl).then(addEventToDatabase, - function(error) {debug("processEventPage error ", error, fullUrl); -}); + scraper.scrapeEventPage(fullUrl) + .then(addEventToDatabase) + .catch( function(error) { + debug("processEventPage error ", error, fullUrl); + }); }; function addEventToDatabase(eventData){ @@ -38,11 +56,3 @@ function addEventToDatabase(eventData){ }; - - -/* -var html3 = "http://dou.ua/calendar/9637/"; -var scraper2 = Scraper(html2); -scraper2.scrapeEventPage(html3).then(function(model){ - console.log("promised title", model); -}, function(error) {console.log("ERROR DOU SCRAPE", error, html3);})*/ diff --git a/src/js/app/server/scrape-dou.js b/src/js/app/server/scrape-dou.js index c8566b2..ceee479 100644 --- a/src/js/app/server/scrape-dou.js +++ b/src/js/app/server/scrape-dou.js @@ -4,6 +4,7 @@ var request = require('request'), cheerio = require('cheerio'); //var html = 'http://dou.ua/calendar'; +//503 ERROR!!! function DouScraper(){ debug("DouScraper constructor"); @@ -13,8 +14,18 @@ function DouScraper(){ DouScraper.prototype.getEventUrls = function(){ debug("getEventUrls"); //debug("this.html",this.html); - scrapeMainPage(this.html).then(scrapeSecondaryPages) - .catch(function(err){console.log(err);}); + var html = this.html; + var promise = new Promise(function(resolve,reject){ + scrapeMainPage(html) + .then(scrapeSecondaryPages) + .then(function(allUrls){ + debug("allUrls, resolved getEventUrls",allUrls.length); + resolve(allUrls); + }) + .catch(function(err){ + reject(err);}); + }); + return promise; }; DouScraper.prototype.scrapeEventPage = function(fullUrl) { @@ -56,7 +67,7 @@ DouScraper.prototype.scrapeEventPage = function(fullUrl) { resolve(model); } else{ - debug("ERROR in scrapeEventPage"); + debug("ERROR in scrapeEventPage", "response code:", resp.statusCode); reject(err); } }); @@ -66,7 +77,7 @@ DouScraper.prototype.scrapeEventPage = function(fullUrl) { }; function scrapeMainPage(html){ - debug("scrapeMainPage"); + debug("scrapeMainPage",html); var promise = new Promise(function(resolve, reject) { request(html, function(error, resp, body){ @@ -94,31 +105,37 @@ function scrapeMainPage(html){ }; function scrapeSecondaryPages(result){ + var promise = new Promise(function(resolve,reject){ var pageQuantity = result.pageQuantity, html = result.html, urls = result.urls; debug("first batch: ", urls.length,pageQuantity); var urlsArray = []; - for (var i = 2; i <= pageQuantity; i++){ + //6 should be changed to pageQuantity + //now it throughs 503 error + for (var i = 2; i <= 6; i++){ var pageUrl = html+'/page-'+i; //debug("pageUrl:",pageUrl); urlsArray.push(pageUrl); } - debug(urlsArray); - Promise.all(urlsArray.map(scrapeSingleSecondaryPage)) + debug(urlsArray); + var arrayOfPromises = urlsArray.map(scrapeSingleSecondaryPage); + Promise.all(arrayOfPromises) .then(function(arrayOfResults){ - debug("problem in concat"); - var urls = [].concat.apply([], arrayOfResults); - //urls = urls.concat(nextUrls); - debug("urls", urls.length); + var newUrls = [].concat.apply([], arrayOfResults); + urls = urls.concat(newUrls); debug("FINAL!!!!!!!!",urls.length); - return urls; - }).catch(function(err){ - console.log(error); + resolve(urls); + }).catch(function(err,pageUrl){ + // log that I have an error, return the entire array; + console.log('A promise failed to resolve', err, "at", pageUrl); + reject(err); + //return arrayOfPromises; }); + }); + return promise; }; - function scrapeSingleSecondaryPage(pageUrl){ debug("scrapeSecondaryDouPage"); //console.log("pageUrl",pageUrl); @@ -130,7 +147,13 @@ function scrapeSingleSecondaryPage(pageUrl){ debug(pageUrl,"is loaded, urls:", urls.length); resolve(urls); } - else{ reject(err);}; + else{ + console.log("scrapeSingleSecondaryPage ERROR at ",pageUrl, err,resp.statusCode); + if (err){ + reject(err); + } + + }; }); }); return promise; diff --git a/src/js/app/server/scrape-parter.js b/src/js/app/server/scrape-parter.js index def345d..a2ee13a 100644 --- a/src/js/app/server/scrape-parter.js +++ b/src/js/app/server/scrape-parter.js @@ -3,54 +3,16 @@ var request = require('request'), debug = require('debug')('bot:scrape-parter'), cheerio = require('cheerio'); - -// return model through promises -//separete scrapeEventPage into 2 functions - -function ParterScraper(){ - debug("ParterScraper constructor"); - this.html = 'http://parter.ua'; -} - -function getEventImage($) { - var eventImage = $('img[src*="/img/item/"]').attr('src'); - if (eventImage){ - return eventImage; - } - else{ - return ""; - } -}; - -function getEventDescription($) { - var eventDescription = $('p').text(); - return eventDescription; -}; - -function getEventPrice($) { - var eventPrice = $('tr:nth-child(3) > td[align="center"]').first().text(); - return eventPrice; -}; - -function getEventTime($) { - var eventTime = [] - $('tr:nth-child(1) > td[align="center"]').each(function(){ - var time = $(this).text(); - eventTime.push(time); - }); - return eventTime; -}; - ParterScraper.prototype.scrapeEventPage = function(fullUrl) { debug("scrapeEventPage begin"); var promise = new Promise(function(resolve, reject) { request(fullUrl,function(err,resp,body){ if (!err && resp.statusCode == 200){ - debug("scrapeEventPage request is successfull"); + //debug("scrapeEventPage request is successfull"); var $ = cheerio.load(body,{ decodeEntities: false }); - debug("createModel begin"); + //debug("createModel begin"); var eventTitle, eventLocation, eventDescription, eventTime = [], eventLink, eventImage, eventPrice; var eventPage = $('td.center'); @@ -72,7 +34,7 @@ ParterScraper.prototype.scrapeEventPage = function(fullUrl) { eventImage: eventImage } //console.log("createModel",model); - debug("createModel resolve"); + //debug("createModel resolve"); resolve(model); } else{ @@ -109,4 +71,40 @@ ParterScraper.prototype.getEventUrls = function(){ return promise; }; +function ParterScraper(){ + debug("ParterScraper constructor"); + this.html = 'http://parter.ua'; +} + +function getEventImage($) { + var eventImage = $('img[src*="/img/item/"]').attr('src'); + if (eventImage){ + return eventImage; + } + else{ + return ""; + } +}; + +function getEventDescription($) { + var eventDescription = $('p').text(); + return eventDescription; +}; + +function getEventPrice($) { + var eventPrice = $('tr:nth-child(3) > td[align="center"]').first().text(); + return eventPrice; +}; + +function getEventTime($) { + var eventTime = [] + $('tr:nth-child(1) > td[align="center"]').each(function(){ + var time = $(this).text(); + eventTime.push(time); + }); + return eventTime; +}; + + + module.exports = ParterScraper; diff --git a/src/js/app/server/scraper.js b/src/js/app/server/scraper.js index 04b90d1..f8f1140 100644 --- a/src/js/app/server/scraper.js +++ b/src/js/app/server/scraper.js @@ -8,7 +8,7 @@ function makeScaraper(html){ if (html.indexOf('http://parter.ua') != -1){ return new ParterScraper(); } - if (html === 'http://dou.ua/calendar') { + if (html.indexOf('http://dou.ua/calendar') != -1) { return new DouScraper(); } } From 34107ebcd74a60160a40ce8a2cbdedcc11df5e52 Mon Sep 17 00:00:00 2001 From: Adialia Date: Tue, 2 Feb 2016 16:51:47 +0200 Subject: [PATCH 3/5] running abstract scrapers --- src/js/app/server/bot.js | 47 +++++++++++++++------------------------- 1 file changed, 18 insertions(+), 29 deletions(-) diff --git a/src/js/app/server/bot.js b/src/js/app/server/bot.js index 6aef551..8ea5ca0 100644 --- a/src/js/app/server/bot.js +++ b/src/js/app/server/bot.js @@ -6,38 +6,27 @@ var Model = require('./model'), debug = require('debug')('bot'), cheerio = require('cheerio'); -var parterScraper = Scraper('http://parter.ua'); -parterScraper.getEventUrls('http://parter.ua').then(function(parterEventsUrls) { - debug("parterEventsUrls"); - parterEventsUrls.forEach(processEventPage); -}) -.catch(function(err){ - console.error(err); -}); - - -var douScraper = Scraper("http://dou.ua/calendar"); -console.log("douScraper.html",douScraper.html); -douScraper.getEventUrls() -.then(function(douEventsUrls){ - debug("douEventsUrls", douEventsUrls.length); - douEventsUrls.forEach(processEventPage); -}) -.catch(function(err){ - console.error(err); -}); - -/* -var html3 = "http://dou.ua/calendar/9637/"; -var scraper2 = Scraper(html2); -scraper2.scrapeEventPage(html3).then(function(model){ - console.log("promised title", model); -}, function(error) {console.log("ERROR DOU SCRAPE", error, html3);})*/ - +module.exports = function(){ + + + Scraper('http://parter.ua').getEventUrls().then(function(parterEventsUrls) { + parterEventsUrls.forEach(processEventPage); + }) + .catch(function(err){ + console.error(err); + }); + + Scraper("http://dou.ua/calendar").getEventUrls() + .then(function(douEventsUrls){ + douEventsUrls.forEach(processEventPage); + }) + .catch(function(err){ + console.error(err); + }); +}(); function processEventPage(fullUrl, index, array){ debug("processEventPage"); - //var fullUrl = html + url; debug("start of model creation", fullUrl); var scraper = Scraper(fullUrl); scraper.scrapeEventPage(fullUrl) From adfe7356d6f920325361e215a9dc79d2884c2044 Mon Sep 17 00:00:00 2001 From: Adialia Date: Tue, 2 Feb 2016 16:55:06 +0200 Subject: [PATCH 4/5] run scraper on array of sourses --- src/js/app/server/bot.js | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/src/js/app/server/bot.js b/src/js/app/server/bot.js index 8ea5ca0..33a647c 100644 --- a/src/js/app/server/bot.js +++ b/src/js/app/server/bot.js @@ -7,23 +7,18 @@ var Model = require('./model'), cheerio = require('cheerio'); module.exports = function(){ + var sourses = ['http://parter.ua',"http://dou.ua/calendar"]; + sourses.forEach(runScraper); +}(); - - Scraper('http://parter.ua').getEventUrls().then(function(parterEventsUrls) { +function runScraper(site){ + Scraper(site).getEventUrls().then(function(parterEventsUrls) { parterEventsUrls.forEach(processEventPage); }) .catch(function(err){ console.error(err); }); - - Scraper("http://dou.ua/calendar").getEventUrls() - .then(function(douEventsUrls){ - douEventsUrls.forEach(processEventPage); - }) - .catch(function(err){ - console.error(err); - }); -}(); +} function processEventPage(fullUrl, index, array){ debug("processEventPage"); From 098ba3d55a0a7197a25aff4cad2e2d83d89d86f1 Mon Sep 17 00:00:00 2001 From: Adialia Date: Tue, 2 Feb 2016 17:03:01 +0200 Subject: [PATCH 5/5] correct names of functions and variables --- src/js/app/server/bot.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/js/app/server/bot.js b/src/js/app/server/bot.js index 33a647c..d459d9f 100644 --- a/src/js/app/server/bot.js +++ b/src/js/app/server/bot.js @@ -12,8 +12,8 @@ module.exports = function(){ }(); function runScraper(site){ - Scraper(site).getEventUrls().then(function(parterEventsUrls) { - parterEventsUrls.forEach(processEventPage); + Scraper(site).getEventUrls().then(function(eventsUrls) { + eventsUrls.forEach(processEventPage); }) .catch(function(err){ console.error(err); @@ -32,9 +32,9 @@ function processEventPage(fullUrl, index, array){ }; function addEventToDatabase(eventData){ - var parterEvent = new Model(eventData); - debug("addEventToDatabase", parterEvent.eventTitle); - parterEvent.save(function(err) { + var singleEvent = new Model(eventData); + debug("addEventToDatabase", singleEvent.eventTitle); + singleEvent.save(function(err) { if (err) {debug('Database err saving: ' + url);} }); };