Addde type 'packages' to support 02packages.details info

Added mapping, document, script & tests to support adding the contents of 02packages.details into its own new type in the index (in a very similar way to 06perms). Added modules: * Document::Packages document description of a package entry * Script::Mapping::CPAN::Packages type mapping for the index * Script::Packages a script to read the file & fill the index * Server::Controller::Packages an API controller to serve the data Added test files: * t/packages.t * t/server/controller/packages.t
metacpan · May 8, 2017 · 7344e4e · 7344e4e
1 parent c249022
commit 7344e4e
Show file tree

Hide file tree

Showing 9 changed files with 225 additions and 0 deletions.
diff --git a/lib/MetaCPAN/Document/Packages.pm b/lib/MetaCPAN/Document/Packages.pm
@@ -0,0 +1,25 @@
+package MetaCPAN::Document::Packages;
+
+use MetaCPAN::Moose;
+
+use ElasticSearchX::Model::Document;
+use MetaCPAN::Types qw( Str );
+
+has module_name => (
+    is       => 'ro',
+    isa      => Str,
+    required => 1,
+);
+
+has version => (
+    is  => 'ro',
+    isa => Str,
+);
+
+has file => (
+    is  => 'ro',
+    isa => Str,
+);
+
+__PACKAGE__->meta->make_immutable;
+1;
diff --git a/lib/MetaCPAN/Script/Mapping.pm b/lib/MetaCPAN/Script/Mapping.pm
@@ -13,6 +13,7 @@ use MetaCPAN::Script::Mapping::CPAN::Favorite     ();
 use MetaCPAN::Script::Mapping::CPAN::File         ();
 use MetaCPAN::Script::Mapping::CPAN::Mirror       ();
 use MetaCPAN::Script::Mapping::CPAN::Permission   ();
+use MetaCPAN::Script::Mapping::CPAN::Packages     ();
 use MetaCPAN::Script::Mapping::CPAN::Rating       ();
 use MetaCPAN::Script::Mapping::CPAN::Release      ();
 use MetaCPAN::Script::Mapping::DeployStatement    ();
@@ -410,6 +411,9 @@ sub deploy_mapping {
             permission =>
                 decode_json( MetaCPAN::Script::Mapping::CPAN::Permission::mapping
                 ),
+            packages =>
+                decode_json( MetaCPAN::Script::Mapping::CPAN::Packages::mapping
+                ),
             rating =>
                 decode_json(MetaCPAN::Script::Mapping::CPAN::Rating::mapping),
             release =>

diff --git a/lib/MetaCPAN/Script/Mapping/CPAN/Packages.pm b/lib/MetaCPAN/Script/Mapping/CPAN/Packages.pm
@@ -0,0 +1,29 @@
+package MetaCPAN::Script::Mapping::CPAN::Packages;
+
+use strict;
+use warnings;
+
+sub mapping {
+    '{
+        "dynamic" : false,
+        "properties" : {
+           "module_name" : {
+              "ignore_above" : 2048,
+              "index" : "not_analyzed",
+              "type" : "string"
+           },
+           "version" : {
+              "ignore_above" : 2048,
+              "index" : "not_analyzed",
+              "type" : "string"
+           },
+           "file" : {
+              "ignore_above" : 2048,
+              "index" : "not_analyzed",
+              "type" : "string"
+           }
+        }
+     }';
+}
+
+1;
diff --git a/lib/MetaCPAN/Script/Packages.pm b/lib/MetaCPAN/Script/Packages.pm
@@ -0,0 +1,94 @@
+package MetaCPAN::Script::Packages;
+
+use Moose;
+
+use Log::Contextual qw( :log );
+use MetaCPAN::Document::Packages ();
+use Parse::CPAN::Packages::Fast  ();
+use IO::Uncompress::Gunzip       ();
+
+with 'MooseX::Getopt', 'MetaCPAN::Role::Script';
+
+=head1 SYNOPSIS
+
+Loads 02packages.details info into db.
+
+=cut
+
+sub run {
+    my $self = shift;
+    $self->index_packages;
+    $self->index->refresh;
+}
+
+sub _get_02packages_fh {
+    my $self = shift;
+    my $file
+        = $self->cpan->file(qw(modules 02packages.details.txt.gz))->stringify;
+    my $fh_uz = IO::Uncompress::Gunzip->new($file);
+    return $fh_uz;
+}
+
+sub index_packages {
+    my $self = shift;
+    log_info {'Reading 02packages.details'};
+
+    my $fh = $self->_get_02packages_fh;
+
+    # read first 9 lines (meta info)
+    my $meta = "Meta info:\n";
+    for ( 0 .. 8 ) {
+        chomp( my $line = <$fh> );
+        next unless $line;
+        $meta .= "$line\n";
+    }
+    log_debug {$meta};
+
+    my $bulk_helper = $self->es->bulk_helper(
+        index => $self->index->name,
+        type  => 'packages',
+    );
+
+    # read the rest of the file line-by-line (too big to slurp)
+    while ( my $line = <$fh> ) {
+        next unless $line;
+        chomp($line);
+
+        my ( $name, $version, $file ) = split /\s+/ => $line;
+
+        my $doc = +{
+            module_name => $name,
+            version     => $version,
+            file        => $file,
+        };
+
+        $bulk_helper->update(
+            {
+                id            => $name,
+                doc           => $doc,
+                doc_as_upsert => 1,
+            }
+        );
+    }
+
+    $bulk_helper->flush;
+    log_info {'finished indexing 02packages.details'};
+}
+
+__PACKAGE__->meta->make_immutable;
+1;
+
+=pod
+
+=head1 SYNOPSIS
+
+Parse out CPAN packages details.
+
+    my $packages = MetaCPAN::Script::Packages->new;
+    my $result   = $packages->index_packages;
+
+=head2 index_packages
+
+Adds/updates all packages details in the CPAN index to Elasticsearch.
+
+=cut
diff --git a/lib/MetaCPAN/Server/Controller/Packages.pm b/lib/MetaCPAN/Server/Controller/Packages.pm
@@ -0,0 +1,11 @@
+package MetaCPAN::Server::Controller::Packages;
+
+use Moose;
+use namespace::autoclean;
+
+BEGIN { extends 'MetaCPAN::Server::Controller' }
+
+with 'MetaCPAN::Server::Role::JSONP';
+
+__PACKAGE__->meta->make_immutable;
+1;
diff --git a/t/00_setup.t b/t/00_setup.t
@@ -83,6 +83,7 @@ copy( $src_dir->file('author-1.0.json'),
 copy( $src_dir->file('bugs.tsv'), $fakecpan_dir->file('bugs.tsv') );
 
 $server->index_permissions;
+$server->index_packages;
 $server->index_releases;
 $server->set_latest;
 $server->set_first;

diff --git a/t/lib/MetaCPAN/TestServer.pm b/t/lib/MetaCPAN/TestServer.pm
@@ -8,6 +8,7 @@ use MetaCPAN::Script::CPANTesters ();
 use MetaCPAN::Script::First       ();
 use MetaCPAN::Script::Latest      ();
 use MetaCPAN::Script::Mapping     ();
+use MetaCPAN::Script::Packages    ();
 use MetaCPAN::Script::Permission  ();
 use MetaCPAN::Script::Release     ();
 use MetaCPAN::Server              ();
@@ -229,6 +230,20 @@ sub index_permissions {
     );
 }
 
+sub index_packages {
+    my $self = shift;
+
+    ok(
+        MetaCPAN::Script::Packages->new_with_options(
+            %{ $self->_config },
+
+            # Eventually maybe move this to use the DarkPAN 06perms
+            #cpan => MetaCPAN::DarkPAN->new->base_dir,
+            )->run,
+        'index packages'
+    );
+}
+
 sub prepare_user_test_data {
     my $self = shift;
     ok(

diff --git a/t/packages.t b/t/packages.t
@@ -0,0 +1,12 @@
+use strict;
+use warnings;
+
+use Test::More;
+use MetaCPAN::Script::Runner;
+
+local @ARGV = ('packages');
+
+# uses ./t/var/tmp/fakecpan/modules/02packages.details.txt
+ok( MetaCPAN::Script::Runner->run, 'runs' );
+
+done_testing();
diff --git a/t/server/controller/packages.t b/t/server/controller/packages.t
@@ -0,0 +1,34 @@
+use strict;
+use warnings;
+
+use Cpanel::JSON::XS qw( decode_json );
+use MetaCPAN::Server::Test;
+use MetaCPAN::TestServer;
+use Test::More;
+
+my $server = MetaCPAN::TestServer->new;
+$server->index_packages;
+
+test_psgi app, sub {
+    my $cb = shift;
+
+    {
+        my $module_name = 'CPAN::Test::Dummy::Perl5::VersionBump';
+        ok( my $res = $cb->( GET "/packages/$module_name" ),
+            "GET $module_name" );
+        is( $res->code, 200, '200 OK' );
+
+        is_deeply(
+            decode_json( $res->content ),
+            {
+                module_name => $module_name,
+                version     => '0.02',
+                file =>
+                    'M/MI/MIYAGAWA/CPAN-Test-Dummy-Perl5-VersionBump-0.02.tar.gz',
+            },
+            'Has the correct 02packages info'
+        );
+    }
+};
+
+done_testing;