Skip to content
Permalink

Comparing changes

Choose two branches to see what’s changed or to start a new pull request. If you need to, you can also or learn more about diff comparisons.

Open a pull request

Create a new pull request by comparing changes across two branches. If you need to, you can also . Learn more about diff comparisons here.
base repository: kanzure/papermonk-downloader-plosone
Failed to load repositories. Confirm that selected base ref is valid, then try again.
Loading
base: 6d4ca0f2e0a4
Choose a base ref
...
head repository: kanzure/papermonk-downloader-plosone
Failed to load repositories. Confirm that selected head ref is valid, then try again.
Loading
compare: f5218f512232
Choose a head ref
  • 4 commits
  • 4 files changed
  • 1 contributor

Commits on Aug 17, 2013

  1. replace request with hyperquest 

    Also, use concat-stream to buffer the response body for parsing.

Eventually, parsing will switch to some sort of stream-based html
parser and metadata will be emitted as it's parsed or something.
    @kanzure
    kanzure committed Aug 17, 2013

    Verified

    This commit was created on GitHub.com and signed with GitHub’s verified signature.
    GPG key ID: B5690EEEBB952194
    Verified
    Learn about vigilant mode
    Copy the full SHA
    7f6e54a View commit details

  2. also switch pdf downloading to hyperquest

    @kanzure
    kanzure committed Aug 17, 2013
    Copy the full SHA
    97d9e94 View commit details

  3. Merge branch 'use-hyperquest' into master 

    Replace request with hyperquest.
    @kanzure
    kanzure committed Aug 17, 2013
    Copy the full SHA
    9bff2d7 View commit details

  4. version bump to: v0.0.6

    @kanzure
    kanzure committed Aug 17, 2013
    Copy the full SHA
    f5218f5 View commit details
Showing with 17 additions and 13 deletions.
  1. +10 −7 index.js
  2. +3 −2 package.json
  3. +3 −3 src/parse.js
  4. +1 −1 tests.js
17 changes: 10 additions & 7 deletions index.js
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
var urlparser = require("url-parser");
var request = require("request");
var hyperquest = require("hyperquest");
var concat = require("concat-stream");
var parse = require("./src/parse.js");

module.exports.test = function test(url) {
@@ -21,13 +22,15 @@ module.exports.test = function test(url) {
};

module.exports.download = function download(url, options, callback) {
return request.get(url, function requestcallback(error, response, body) {
var metadata = null;
var request = hyperquest.get(url);

if (!error && response.statusCode == 200) {
metadata = parse(body);
}
request.pipe(concat(function(body) {
var metadata = parse(body);
callback(null, metadata, body);
}));

callback(error, metadata);
//request.on("response", function(response){});
request.on("error", function(error) {
callback(error, null, null);
});
};
5 changes: 3 additions & 2 deletions package.json
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"name": "papermonk-downloader-plosone",
"description": "plosone.org scraper",
"version": "0.0.5",
"version": "0.0.6",
"readmeFilename": "README.md",
"homepage": "https://github.com/kanzure/papermonk-downloader-plosone",
"author": {
@@ -43,9 +43,10 @@
},
"dependencies": {
"url-parser": "0.0.1",
"request": "~2.26.0",
"hyperquest": "~0.1.7",
"cheerio": "~0.12.1",
"nock": ">0.22.0",
"concat-stream": "1.0.0",
"tape": ">0"
},
"optionalDependencies": {
6 changes: 3 additions & 3 deletions src/parse.js
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
var request = require("request");
var hyperquest = require("hyperquest");
var cheerio = require("cheerio");

module.exports = function parse(body) {
@@ -35,8 +35,8 @@ module.exports = function parse(body) {
};

var makepipe = module.exports.makepipe = function makepipe(url) {
return function download(writestream) {
return request.get(url).pipe(writestream);
return function download(stream) {
return hyperquest.get(url).pipe(stream);
};
};

2 changes: 1 addition & 1 deletion tests.js
View file
Original file line number Diff line number Diff line change
@@ -70,7 +70,7 @@ test("extracts the correct html title", function(t) {
.replyWithFile(200, filepath);

// make sure the title is extracted correctly
downloader.download("http://example.com/paper", {}, function(error, metadata) {
downloader.download("http://example.com/paper", {}, function(error, metadata, body) {
t.equal(expected_title, metadata.html.title, "title matches expectations");
t.ok(scope.isDone(), "nock scope is done");
t.end();