Dagmar is a deadly simple crawling/scraping package for Node.
It features:
- A clean, simple API
- Possible use of server-side DOM & automatic jQuery insertion with Cheerio
- node 0.10+ support
$ npm install dagmar
var crawler = new Crawler();
crawler.forEach(function(error, response, body) {
if (error || response.statusCode !== 200) {
console.log(error);
} else {
console.log(body);
}
});
crawler.end(function() {
console.log('Done.');
});
crawler.queue("http://www.google.com");
crawler.queue("http://www.yahoo.com");
crawler.queue("http://www.apple.com");
crawler.queue("http://www.twitter.com");
crawler.queue("http://www.facebook.com");
crawler.start();var crawler = new Crawler();
crawler.forEach(function(error, response, body) {
var $, list;
if (!error && response.statusCode === 200) {
$ = cheerio.load(body);
list = $('ul', '<ul id="fruits">...</ul>');
console.log(list);
} else {
console.log(error);
}
});
crawler.end(function() {
console.log('Done.');
});
crawler.queue("http://www.fruits.org");
crawler.start();var crawler = new Crawler();
crawler.forEach(function(error, response, body) {
if (!error && response.statusCode === 200) {
var $ = cheerio.load(body);
return $('a').each(function(index, a) {
var url = $(a).attr('href');
crawler.queue(url);
});
} else {
console.log(error);
}
});
crawler.end(function() {
console.log('Done.');
});
crawler.queue("http://www.fruits.org");
crawler.start();