Merge pull request #624 from metacpan/oalders/06perms-tiny

Adds indexing for 06perms.txt
metacpan · Apr 2, 2017 · b557c41 · b557c41
2 parents f9058f9 + 03f4af6
commit b557c41
Show file tree

Hide file tree

Showing 18 changed files with 338 additions and 67 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1,28 +1,24 @@
 # carton/local::lib
-/local/
-
-.DS_Store
-*.sw*
-*.kpf
+# coverage
+# generated by Makefile.PL (for instance when doing `cpanm --installdeps .`)
+# tidyall
 *.komodoproject
+*.kpf
 *.sqlite*
-/var
-/t/var/tmp/
-/t/var/darkpan/
-/t/var/log/
-/etc/metacpan_local.pl
-metacpan_server_local.conf
-
-# generated by Makefile.PL (for instance when doing `cpanm --installdeps .`)
+*.sw*
+.DS_Store
+.tidyall.d
+/MYMETA.*
 /Makefile
 /Makefile.old
-/MYMETA.*
-/pm_to_blib
 /blib
-
-# tidyall
-.tidyall.d
-perltidy.LOG
-
-# coverage
+/etc/metacpan_local.pl
+/local/
+/pm_to_blib
+/t/var/darkpan/
+/t/var/log/
+/t/var/tmp/
+/var
 cover_db/
+metacpan_server_local.conf
+perltidy.LOG
diff --git a/.tidyallrc b/.tidyallrc
@@ -57,3 +57,9 @@ ignore = lib/MetaCPAN/Server/View/JSON.pm
 ignore = lib/MetaCPAN/Server/View/Pod.pm
 ignore = lib/MetaCPAN/Util.pm
 ignore = lib/Plack/Session/Store/ElasticSearch.pm
+
+[SortLines]
+select = .gitignore
+
+[UniqueLines]
+select = .gitignore
diff --git a/.travis.yml b/.travis.yml
@@ -25,6 +25,9 @@ env:
     - USE_CPANFILE_SNAPSHOT=true
     - USE_CPANFILE_SNAPSHOT=false
 
+matrix:
+  allow_failures:
+    - env: USE_CPANFILE_SNAPSHOT=false
 
 before_install:
   - sudo service elasticsearch stop && curl -O -L https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-2.4.3.deb && sudo dpkg -i --force-confnew elasticsearch-2.4.3.deb && sudo service elasticsearch start

diff --git a/cpanfile b/cpanfile
@@ -195,4 +195,5 @@ test_requires 'Test::Routine::Util', '0';
 test_requires 'Test::Vars';
 
 author_requires 'Code::TidyAll', '>= 0.47';
+author_requires 'Code::TidyAll::Plugin::UniqueLines';
 author_requires 'Plack::Middleware::Rewrite';
diff --git a/cpanfile.snapshot b/cpanfile.snapshot
@@ -1166,6 +1166,19 @@ DISTRIBUTIONS
       strict 0
       vars 0
       warnings 0
+  Code-TidyAll-Plugin-UniqueLines-0.000002
+    pathname: O/OA/OALDERS/Code-TidyAll-Plugin-UniqueLines-0.000002.tar.gz
+    provides:
+      Code::TidyAll::Plugin::UniqueLines 0.000002
+    requirements:
+      Code::TidyAll::Plugin 0
+      ExtUtils::MakeMaker 0
+      List::Uniq 0
+      Module::Build 0.28
+      Moo 0
+      perl 5.006
+      strict 0
+      warnings 0
   Compress-Bzip2-2.24
     pathname: R/RU/RURBAN/Compress-Bzip2-2.24.tar.gz
     provides:
@@ -3887,6 +3900,11 @@ DISTRIBUTIONS
       perl 5.006
       strict 0
       warnings 0
+  List-Uniq-0.20
+    pathname: J/JF/JFITZ/List-Uniq-0.20.tar.gz
+    provides:
+      List::Uniq 0.20
+    requirements:
   Log-Any-1.040
     pathname: D/DA/DAGOLDEN/Log-Any-1.040.tar.gz
     provides:

diff --git a/lib/MetaCPAN/Document/Permission.pm b/lib/MetaCPAN/Document/Permission.pm
@@ -0,0 +1,25 @@
+package MetaCPAN::Document::Permission;
+
+use MetaCPAN::Moose;
+
+use ElasticSearchX::Model::Document;
+use MetaCPAN::Types qw( ArrayRef Str );
+
+has module_name => (
+    is       => 'ro',
+    isa      => Str,
+    required => 1,
+);
+
+has owner => (
+    is  => 'ro',
+    isa => Str,
+);
+
+has co_maintainers => (
+    is  => 'ro',
+    isa => ArrayRef,
+);
+
+__PACKAGE__->meta->make_immutable;
+1;
diff --git a/lib/MetaCPAN/Model.pm b/lib/MetaCPAN/Model.pm
@@ -1,10 +1,8 @@
 package MetaCPAN::Model;
 
-use strict;
-use warnings;
-
 # load order important
 use Moose;
+
 use ElasticSearchX::Model;
 
 analyzer lowercase => (

diff --git a/lib/MetaCPAN/Role/Script.pm b/lib/MetaCPAN/Role/Script.pm
@@ -201,6 +201,6 @@ __END__
 
 =head1 SYNOPSIS
 
-Roles which should be available to all modules
+Roles which should be available to all modules.
 
 =cut
diff --git a/lib/MetaCPAN/Script/Mapping.pm b/lib/MetaCPAN/Script/Mapping.pm
@@ -1,25 +1,25 @@
 package MetaCPAN::Script::Mapping;
 
-use strict;
-use warnings;
+use Moose;
 
+use Cpanel::JSON::XS qw( decode_json );
+use DateTime ();
+use IO::Interactive qw( is_interactive );
+use IO::Prompt qw( prompt );
 use Log::Contextual qw( :log );
-use Moose;
+use MetaCPAN::Script::Mapping::CPAN::Author       ();
+use MetaCPAN::Script::Mapping::CPAN::Distribution ();
+use MetaCPAN::Script::Mapping::CPAN::Favorite     ();
+use MetaCPAN::Script::Mapping::CPAN::File         ();
+use MetaCPAN::Script::Mapping::CPAN::Mirror       ();
+use MetaCPAN::Script::Mapping::CPAN::Permission   ();
+use MetaCPAN::Script::Mapping::CPAN::Rating       ();
+use MetaCPAN::Script::Mapping::CPAN::Release      ();
+use MetaCPAN::Script::Mapping::DeployStatement    ();
+use MetaCPAN::Script::Mapping::User::Account      ();
+use MetaCPAN::Script::Mapping::User::Identity     ();
+use MetaCPAN::Script::Mapping::User::Session      ();
 use MetaCPAN::Types qw( Bool Str );
-use Cpanel::JSON::XS qw( decode_json );
-use DateTime;
-
-use MetaCPAN::Script::Mapping::DeployStatement;
-use MetaCPAN::Script::Mapping::CPAN::Author;
-use MetaCPAN::Script::Mapping::CPAN::Distribution;
-use MetaCPAN::Script::Mapping::CPAN::Favorite;
-use MetaCPAN::Script::Mapping::CPAN::File;
-use MetaCPAN::Script::Mapping::CPAN::Mirror;
-use MetaCPAN::Script::Mapping::CPAN::Rating;
-use MetaCPAN::Script::Mapping::CPAN::Release;
-use MetaCPAN::Script::Mapping::User::Account;
-use MetaCPAN::Script::Mapping::User::Identity;
-use MetaCPAN::Script::Mapping::User::Session;
 
 use constant {
     EXPECTED     => 1,
@@ -407,6 +407,9 @@ sub deploy_mapping {
                 ),
             file =>
                 decode_json(MetaCPAN::Script::Mapping::CPAN::File::mapping),
+            permission =>
+                decode_json( MetaCPAN::Script::Mapping::CPAN::Permission::mapping
+                ),
             rating =>
                 decode_json(MetaCPAN::Script::Mapping::CPAN::Rating::mapping),
             release =>
@@ -448,6 +451,21 @@ sub deploy_mapping {
     1;
 }
 
+sub _prompt {
+    my ( $self, $msg ) = @_;
+
+    if (is_interactive) {
+        print colored( ['bold red'], "*** Warning ***: $msg" ), "\n";
+        my $answer = prompt
+            'Are you sure you want to do this (type "YES" to confirm) ? ';
+        if ( $answer ne 'YES' ) {
+            print "bye.\n";
+            exit 0;
+        }
+        print "alright then...\n";
+    }
+}
+
 __PACKAGE__->meta->make_immutable;
 1;
 

diff --git a/lib/MetaCPAN/Script/Mapping/CPAN/Permission.pm b/lib/MetaCPAN/Script/Mapping/CPAN/Permission.pm
@@ -0,0 +1,29 @@
+package MetaCPAN::Script::Mapping::CPAN::Permission;
+
+use strict;
+use warnings;
+
+sub mapping {
+    '{
+        "dynamic" : false,
+        "properties" : {
+           "module_name" : {
+              "ignore_above" : 2048,
+              "index" : "not_analyzed",
+              "type" : "string"
+           },
+           "owner" : {
+              "ignore_above" : 2048,
+              "index" : "not_analyzed",
+              "type" : "string"
+           },
+           "co_maintainers" : {
+              "ignore_above" : 2048,
+              "index" : "not_analyzed",
+              "type" : "string"
+           }
+        }
+     }';
+}
+
+1;
diff --git a/lib/MetaCPAN/Script/Permission.pm b/lib/MetaCPAN/Script/Permission.pm
@@ -0,0 +1,82 @@
+package MetaCPAN::Script::Permission;
+
+use Moose;
+
+use Log::Contextual qw( :log );
+use MetaCPAN::Document::Permission ();
+use PAUSE::Permissions             ();
+
+with 'MooseX::Getopt', 'MetaCPAN::Role::Script';
+
+=head1 SYNOPSIS
+
+Loads 06perms info into db. Does not require the presence of a local
+CPAN/minicpan.
+
+=cut
+
+sub run {
+    my $self = shift;
+    $self->index_permissions;
+    $self->index->refresh;
+}
+
+sub index_permissions {
+    my $self = shift;
+
+    my $file_path
+        = $self->cpan->subdir('modules')->file('06perms.txt')->absolute;
+    my $pp = PAUSE::Permissions->new( path => $file_path );
+
+    my $bulk_helper = $self->es->bulk_helper(
+        index => $self->index->name,
+        type  => 'permission',
+    );
+
+    my $iterator = $pp->module_iterator;
+    while ( my $perms = $iterator->next_module ) {
+
+        # This method does a "return sort @foo", so it can't be called in the
+        # ternary since it always returns false in that context.
+        # https://github.com/neilb/PAUSE-Permissions/pull/16
+
+        my @co_maints = $perms->co_maintainers;
+        my $doc       = {
+            @co_maints
+            ? ( co_maintainers => \@co_maints )
+            : (),
+            module_name => $perms->name,
+            owner       => $perms->owner,
+        };
+
+        $bulk_helper->update(
+            {
+                id            => $perms->name,
+                doc           => $doc,
+                doc_as_upsert => 1,
+            }
+        );
+    }
+
+    $bulk_helper->flush;
+    log_info {'finished indexing 06perms'};
+}
+
+__PACKAGE__->meta->make_immutable;
+1;
+
+=pod
+
+=head1 SYNOPSIS
+
+Parse out CPAN author permissions.
+
+    my $perms = MetaCPAN::Script::Permission->new;
+    my $result = $perms->index_permissions;
+
+=head2 index_authors
+
+Adds/updates all ownership and maintenance permissions in the CPAN index to
+Elasticsearch.
+
+=cut
diff --git a/lib/MetaCPAN/Server/Controller/Permission.pm b/lib/MetaCPAN/Server/Controller/Permission.pm
@@ -0,0 +1,11 @@
+package MetaCPAN::Server::Controller::Permission;
+
+use Moose;
+use namespace::autoclean;
+
+BEGIN { extends 'MetaCPAN::Server::Controller' }
+
+with 'MetaCPAN::Server::Role::JSONP';
+
+__PACKAGE__->meta->make_immutable;
+1;