Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #628 from metacpan/mickey/02packages
Addde type 'packages' to support 02packages.details info
- Loading branch information
Showing
9 changed files
with
241 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,25 @@ | ||
package MetaCPAN::Document::Packages; | ||
|
||
use MetaCPAN::Moose; | ||
|
||
use ElasticSearchX::Model::Document; | ||
use MetaCPAN::Types qw( Str ); | ||
|
||
has module_name => ( | ||
is => 'ro', | ||
isa => Str, | ||
required => 1, | ||
); | ||
|
||
has version => ( | ||
is => 'ro', | ||
isa => Str, | ||
); | ||
|
||
has file => ( | ||
is => 'ro', | ||
isa => Str, | ||
); | ||
|
||
__PACKAGE__->meta->make_immutable; | ||
1; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,39 @@ | ||
package MetaCPAN::Script::Mapping::CPAN::Packages; | ||
|
||
use strict; | ||
use warnings; | ||
|
||
sub mapping { | ||
'{ | ||
"dynamic" : false, | ||
"properties" : { | ||
"module_name" : { | ||
"ignore_above" : 2048, | ||
"index" : "not_analyzed", | ||
"type" : "string" | ||
}, | ||
"distribution" : { | ||
"ignore_above" : 2048, | ||
"index" : "not_analyzed", | ||
"type" : "string" | ||
}, | ||
"author" : { | ||
"ignore_above" : 2048, | ||
"index" : "not_analyzed", | ||
"type" : "string" | ||
}, | ||
"version" : { | ||
"ignore_above" : 2048, | ||
"index" : "not_analyzed", | ||
"type" : "string" | ||
}, | ||
"file" : { | ||
"ignore_above" : 2048, | ||
"index" : "not_analyzed", | ||
"type" : "string" | ||
} | ||
} | ||
}'; | ||
} | ||
|
||
1; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,98 @@ | ||
package MetaCPAN::Script::Packages; | ||
|
||
use Moose; | ||
|
||
use Log::Contextual qw( :log ); | ||
use MetaCPAN::Document::Packages (); | ||
use Parse::CPAN::Packages::Fast (); | ||
use IO::Uncompress::Gunzip (); | ||
use CPAN::DistnameInfo (); | ||
|
||
with 'MooseX::Getopt', 'MetaCPAN::Role::Script'; | ||
|
||
=head1 SYNOPSIS | ||
Loads 02packages.details info into db. | ||
=cut | ||
|
||
sub run { | ||
my $self = shift; | ||
$self->index_packages; | ||
$self->index->refresh; | ||
} | ||
|
||
sub _get_02packages_fh { | ||
my $self = shift; | ||
my $file | ||
= $self->cpan->file(qw(modules 02packages.details.txt.gz))->stringify; | ||
my $fh_uz = IO::Uncompress::Gunzip->new($file); | ||
return $fh_uz; | ||
} | ||
|
||
sub index_packages { | ||
my $self = shift; | ||
log_info {'Reading 02packages.details'}; | ||
|
||
my $fh = $self->_get_02packages_fh; | ||
|
||
# read first 9 lines (meta info) | ||
my $meta = "Meta info:\n"; | ||
for ( 0 .. 8 ) { | ||
chomp( my $line = <$fh> ); | ||
next unless $line; | ||
$meta .= "$line\n"; | ||
} | ||
log_debug {$meta}; | ||
|
||
my $bulk_helper = $self->es->bulk_helper( | ||
index => $self->index->name, | ||
type => 'packages', | ||
); | ||
|
||
# read the rest of the file line-by-line (too big to slurp) | ||
while ( my $line = <$fh> ) { | ||
next unless $line; | ||
chomp($line); | ||
|
||
my ( $name, $version, $file ) = split /\s+/ => $line; | ||
my $distinfo = CPAN::DistnameInfo->new($file); | ||
|
||
my $doc = +{ | ||
module_name => $name, | ||
version => $version, | ||
file => $file, | ||
author => $distinfo->cpanid, | ||
distribution => $distinfo->dist, | ||
}; | ||
|
||
$bulk_helper->update( | ||
{ | ||
id => $name, | ||
doc => $doc, | ||
doc_as_upsert => 1, | ||
} | ||
); | ||
} | ||
|
||
$bulk_helper->flush; | ||
log_info {'finished indexing 02packages.details'}; | ||
} | ||
|
||
__PACKAGE__->meta->make_immutable; | ||
1; | ||
|
||
=pod | ||
=head1 SYNOPSIS | ||
Parse out CPAN packages details. | ||
my $packages = MetaCPAN::Script::Packages->new; | ||
my $result = $packages->index_packages; | ||
=head2 index_packages | ||
Adds/updates all packages details in the CPAN index to Elasticsearch. | ||
=cut |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,11 @@ | ||
package MetaCPAN::Server::Controller::Packages; | ||
|
||
use Moose; | ||
use namespace::autoclean; | ||
|
||
BEGIN { extends 'MetaCPAN::Server::Controller' } | ||
|
||
with 'MetaCPAN::Server::Role::JSONP'; | ||
|
||
__PACKAGE__->meta->make_immutable; | ||
1; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,12 @@ | ||
use strict; | ||
use warnings; | ||
|
||
use Test::More; | ||
use MetaCPAN::Script::Runner; | ||
|
||
local @ARGV = ('packages'); | ||
|
||
# uses ./t/var/tmp/fakecpan/modules/02packages.details.txt | ||
ok( MetaCPAN::Script::Runner->run, 'runs' ); | ||
|
||
done_testing(); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,36 @@ | ||
use strict; | ||
use warnings; | ||
|
||
use Cpanel::JSON::XS qw( decode_json ); | ||
use MetaCPAN::Server::Test; | ||
use MetaCPAN::TestServer; | ||
use Test::More; | ||
|
||
my $server = MetaCPAN::TestServer->new; | ||
$server->index_packages; | ||
|
||
test_psgi app, sub { | ||
my $cb = shift; | ||
|
||
{ | ||
my $module_name = 'CPAN::Test::Dummy::Perl5::VersionBump'; | ||
ok( my $res = $cb->( GET "/packages/$module_name" ), | ||
"GET $module_name" ); | ||
is( $res->code, 200, '200 OK' ); | ||
|
||
is_deeply( | ||
decode_json( $res->content ), | ||
{ | ||
module_name => $module_name, | ||
version => '0.02', | ||
file => | ||
'M/MI/MIYAGAWA/CPAN-Test-Dummy-Perl5-VersionBump-0.02.tar.gz', | ||
author => 'MIYAGAWA', | ||
distribution => 'CPAN-Test-Dummy-Perl5-VersionBump', | ||
}, | ||
'Has the correct 02packages info' | ||
); | ||
} | ||
}; | ||
|
||
done_testing; |