Skip to content

Scraping bot module #14

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Adialia wants to merge 6 commits into
base: master
Choose a base branch
from
Open

Scraping bot module #14

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
185ad3b
Dou Scraper improved structure
Adialia Feb 1, 2016
b4d49fc
Merge branch 'master' of https://github.com/DiskoGoth/events-agrigator
Adialia Feb 1, 2016
a1f797f
improved promises
Adialia Feb 2, 2016
34107eb
running abstract scrapers
Adialia Feb 2, 2016
adfe735
run scraper on array of sourses
Adialia Feb 2, 2016
098ba3d
correct names of functions and variables
Adialia Feb 2, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
84 changes: 26 additions & 58 deletions src/js/app/server/bot.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,74 +1,42 @@
// run scraping process

var Model = require('./model');
//var ParterScraper = require('./scrape-parter.js');
//var DouScraper = require('./scrape-dou.js');
var Scraper = require('./scraper.js');
var request = require('request'),
var Model = require('./model'),
Scraper = require('./scraper.js'),
request = require('request'),
debug = require('debug')('bot'),
cheerio = require('cheerio');

module.exports = function(){
var sourses = ['http://parter.ua',"http://dou.ua/calendar"];
sourses.forEach(runScraper);
}();

function runScraper(site){
Scraper(site).getEventUrls().then(function(eventsUrls) {
eventsUrls.forEach(processEventPage);
})
.catch(function(err){
console.error(err);
});
}

var html2 = "http://dou.ua/calendar";
var scraper2 = Scraper(html2);
var douUrls = scraper2.getEventUrls(html2);



/*
var html3 = "http://dou.ua/calendar/9637/";
var scraper2 = Scraper(html2);
scraper2.scrapeEventPage(html3).then(function(model){
console.log("promised title", model);
}, function(error) {console.log("ERROR DOU SCRAPE", error, html3);})*/

/*
var html = 'http://parter.ua';
getMainPageUrls(html).then(function(allUrls) {
console.log("start for each");
allUrls.forEach(processEventPage);
}, function(error) {console.log("getMainPageUrls error ", error);
});
*/

function processEventPage(url, index, array){
function processEventPage(fullUrl, index, array){
debug("processEventPage");
var fullUrl = html + url;
debug("start of model creation", fullUrl);
var scraper = Scraper(html);
scraper.scrapeEventPage(fullUrl).then(addEventToDatabase,
function(error) {debug("processEventPage error ", error, fullUrl);
});
var scraper = Scraper(fullUrl);
scraper.scrapeEventPage(fullUrl)
.then(addEventToDatabase)
.catch( function(error) {
debug("processEventPage error ", error, fullUrl);
});
};

function addEventToDatabase(eventData){
var parterEvent = new Model(eventData);
debug("addEventToDatabase", parterEvent.eventTitle);
parterEvent.save(function(err) {
var singleEvent = new Model(eventData);
debug("addEventToDatabase", singleEvent.eventTitle);
singleEvent.save(function(err) {
if (err) {debug('Database err saving: ' + url);}
});
};

function getMainPageUrls(html) {
var promise = new Promise(function(resolve, reject) {
debug("getMainPageUrls");
var url, urls = [];
request(html, function(err,resp, body){
if (!err && resp.statusCode == 200){
var $ = cheerio.load(body);
$("div[class='event']").each(function(){
url = $(this).find('a').attr('href');
urls.push(url);
});
debug("resolve", urls.length);
resolve(urls);
}
else{
cdebug("ERROR in getMainPageUrls()");
reject(err);
}
});
})
return promise;
};

99 changes: 65 additions & 34 deletions src/js/app/server/scrape-dou.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,22 +4,29 @@ var request = require('request'),
cheerio = require('cheerio');

//var html = 'http://dou.ua/calendar';
//503 ERROR!!!

function DouScraper(){
debug("DouScraper constructor");
this.html = 'http://dou.ua/calendar';
}

DouScraper.prototype.getEventUrls = function(html){
scrapeMainPage(html).then(function(pageQuantity, urls){
scrapeAllSecondaryDouPages(html,pageQuantity).then(function(nextPagesUrls){
urls = urls.concat(nextPagesUrls);
},
function(error){console.log("scrapeAllSecondaryDouPages ERROR!",error);});

console.log("eventsQuantity ",urls.length);
return urls;},
function(error){console.log(error);});
}
DouScraper.prototype.getEventUrls = function(){
debug("getEventUrls");
//debug("this.html",this.html);
var html = this.html;
var promise = new Promise(function(resolve,reject){
scrapeMainPage(html)
.then(scrapeSecondaryPages)
.then(function(allUrls){
debug("allUrls, resolved getEventUrls",allUrls.length);
resolve(allUrls);
})
.catch(function(err){
reject(err);});
});
return promise;
};

DouScraper.prototype.scrapeEventPage = function(fullUrl) {
debug("DouScraper scrapeEventPage begin");
Expand Down Expand Up @@ -60,7 +67,7 @@ DouScraper.prototype.scrapeEventPage = function(fullUrl) {
resolve(model);
}
else{
debug("ERROR in scrapeEventPage");
debug("ERROR in scrapeEventPage", "response code:", resp.statusCode);
reject(err);
}
});
Expand All @@ -70,6 +77,7 @@ DouScraper.prototype.scrapeEventPage = function(fullUrl) {
};

function scrapeMainPage(html){
debug("scrapeMainPage",html);
var promise = new Promise(function(resolve, reject) {
request(html, function(error, resp, body){

Expand All @@ -78,51 +86,74 @@ function scrapeMainPage(html){
var $ = cheerio.load(body);
var pageQuantity = $('div.b-paging > span:last-of-type').prev().text();
var urls = findDouUrls($);


resolve(pageQuantity, urls);
debug("resolve of scrapeMainPage:", pageQuantity, urls.length);
var result = {
pageQuantity: pageQuantity,
urls: urls,
html: html
};
resolve(result);

}
else{
debug("error in scrapeMainPage");
reject(error);
};
});
});
return promise;
};

function scrapeAllSecondaryDouPages(html,quantity){
var urls = [];
function scrapeSecondaryPages(result){
var promise = new Promise(function(resolve,reject){
for (var i = 2; i <= quantity; i++){
var pageUrl = html+'/page-'+i;
console.log(pageUrl);
scrapeSecondaryDouPage(pageUrl).then(function(nextUrls){
urls = urls.concat(nextUrls);
console.log("urls", urls.length);
},
function(error){

console.log("loadDouUrls ERROR!",error, " at ", pageUrl);
reject(error);
})
var pageQuantity = result.pageQuantity,
html = result.html,
urls = result.urls;
debug("first batch: ", urls.length,pageQuantity);
var urlsArray = [];
//6 should be changed to pageQuantity
//now it throughs 503 error
for (var i = 2; i <= 6; i++){
var pageUrl = html+'/page-'+i;
//debug("pageUrl:",pageUrl);
urlsArray.push(pageUrl);
}
resolve(urls);
debug(urlsArray);
var arrayOfPromises = urlsArray.map(scrapeSingleSecondaryPage);
Promise.all(arrayOfPromises)
.then(function(arrayOfResults){
var newUrls = [].concat.apply([], arrayOfResults);
urls = urls.concat(newUrls);
debug("FINAL!!!!!!!!",urls.length);
resolve(urls);
}).catch(function(err,pageUrl){
// log that I have an error, return the entire array;
console.log('A promise failed to resolve', err, "at", pageUrl);
reject(err);
//return arrayOfPromises;
});
});
return promise;
};

function scrapeSecondaryDouPage(pageUrl){
function scrapeSingleSecondaryPage(pageUrl){
debug("scrapeSecondaryDouPage");
//console.log("pageUrl",pageUrl);
var promise = new Promise(function(resolve,reject){
request(pageUrl, function(err, resp, body){
if (!err && resp.statusCode == 200){
var $ = cheerio.load(body);
var urls = findDouUrls($);
console.log(pageUrl,"is loaded");
debug(pageUrl,"is loaded, urls:", urls.length);
resolve(urls);
}
else{ reject(err);};
else{
console.log("scrapeSingleSecondaryPage ERROR at ",pageUrl, err,resp.statusCode);
if (err){
reject(err);
}

};
});
});
return promise;
Expand All @@ -132,7 +163,7 @@ function findDouUrls($){
var urls = [];
$('div.event > div.title > a').each(function(eventTag){
var url = $(this).attr('href');
debug("url",url);
//debug("url in findDouUrls",url);
urls.push(url);
});
return urls;
Expand Down
103 changes: 63 additions & 40 deletions src/js/app/server/scrape-parter.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,53 +3,16 @@ var request = require('request'),
debug = require('debug')('bot:scrape-parter'),
cheerio = require('cheerio');


// return model through promises
//separete scrapeEventPage into 2 functions

function ParterScraper(){
debug("ParterScraper constructor");
}

function getEventImage($) {
var eventImage = $('img[src*="/img/item/"]').attr('src');
if (eventImage){
return eventImage;
}
else{
return "";
}
};

function getEventDescription($) {
var eventDescription = $('p').text();
return eventDescription;
};

function getEventPrice($) {
var eventPrice = $('tr:nth-child(3) > td[align="center"]').first().text();
return eventPrice;
};

function getEventTime($) {
var eventTime = []
$('tr:nth-child(1) > td[align="center"]').each(function(){
var time = $(this).text();
eventTime.push(time);
});
return eventTime;
};

ParterScraper.prototype.scrapeEventPage = function(fullUrl) {
debug("scrapeEventPage begin");
var promise = new Promise(function(resolve, reject) {
request(fullUrl,function(err,resp,body){
if (!err && resp.statusCode == 200){
debug("scrapeEventPage request is successfull");
//debug("scrapeEventPage request is successfull");
var $ = cheerio.load(body,{
decodeEntities: false
});
debug("createModel begin");
//debug("createModel begin");
var eventTitle, eventLocation, eventDescription, eventTime = [],
eventLink, eventImage, eventPrice;
var eventPage = $('td.center');
Expand All @@ -71,7 +34,7 @@ ParterScraper.prototype.scrapeEventPage = function(fullUrl) {
eventImage: eventImage
}
//console.log("createModel",model);
debug("createModel resolve");
//debug("createModel resolve");
resolve(model);
}
else{
Expand All @@ -84,4 +47,64 @@ ParterScraper.prototype.scrapeEventPage = function(fullUrl) {
return promise;
};

ParterScraper.prototype.getEventUrls = function(){
var html = this.html;
var promise = new Promise(function(resolve, reject) {
debug("getMainPageUrls");
var url, urls = [];
request(html, function(err,resp, body){
if (!err && resp.statusCode == 200){
var $ = cheerio.load(body);
$("div[class='event']").each(function(){
url = $(this).find('a').attr('href');
urls.push(html+url);
});
debug("resolve", urls.length);
resolve(urls);
}
else{
debug("ERROR in getMainPageUrls()");
reject(err);
}
});
})
return promise;
};

function ParterScraper(){
debug("ParterScraper constructor");
this.html = 'http://parter.ua';
}

function getEventImage($) {
var eventImage = $('img[src*="/img/item/"]').attr('src');
if (eventImage){
return eventImage;
}
else{
return "";
}
};

function getEventDescription($) {
var eventDescription = $('p').text();
return eventDescription;
};

function getEventPrice($) {
var eventPrice = $('tr:nth-child(3) > td[align="center"]').first().text();
return eventPrice;
};

function getEventTime($) {
var eventTime = []
$('tr:nth-child(1) > td[align="center"]').each(function(){
var time = $(this).text();
eventTime.push(time);
});
return eventTime;
};



module.exports = ParterScraper;
Loading