Merge pull request #448 from CPAN-API/mi/es2

Elasticsearch 2.x
metacpan · Jun 14, 2016 · b67d768 · b67d768
2 parents 480d931 + 26c487f
commit b67d768
Show file tree

Hide file tree

Showing 226 changed files with 7,132 additions and 5,898 deletions.
diff --git a/.gitignore b/.gitignore
@@ -8,6 +8,8 @@
 *.sqlite*
 /var
 /t/var/tmp/
+/t/var/darkpan/
+/t/var/log/
 /etc/metacpan_local.pl
 metacpan_server_local.conf
 

diff --git a/.perlcriticrc b/.perlcriticrc
@@ -10,16 +10,18 @@ verbose = 11
 [-RegularExpressions::RequireDotMatchAnything]
 [-RegularExpressions::RequireExtendedFormatting]
 [-RegularExpressions::RequireLineBoundaryMatching]
+[-Subroutines::ProhibitExplicitReturnUndef]
+[-ValuesAndExpressions::ProhibitNoisyQuotes]
 [-Variables::ProhibitPunctuationVars]
 
 [CodeLayout::RequireTrailingCommas]
 severity = 4
 
 [TestingAndDebugging::RequireUseStrict]
-equivalent_modules = Test::Routine Mojo::Base
+equivalent_modules = MetaCPAN::Moose Mojo::Base Test::Routine
 
 [TestingAndDebugging::RequireUseWarnings]
-equivalent_modules = Test::Routine Mojo::Base
+equivalent_modules = MetaCPAN::Moose Mojo::Base Test::Routine
 
 [ValuesAndExpressions::ProhibitEmptyQuotes]
 severity = 4

diff --git a/.tidyallrc b/.tidyallrc
@@ -4,3 +4,56 @@ select = bin/daemon-control.pl
 select = app.psgi
 ignore = t/var/**/*
 argv = --profile=$ROOT/.perltidyrc
+
+[PerlCritic]
+select = {lib,t}/**/*.{pl,pm,t,psgi}
+ignore = t/var/**/*
+ignore = bin/build_test_CPAN_dir.pl
+ignore = bin/check_json.pl
+ignore = bin/convert_authors.pl
+ignore = bin/get_fields.pl
+ignore = bin/mirror_cpan_for_developers.pl
+ignore = bin/unlisted_prereqs.pl
+ignore = bin/write_config_json
+ignore = lib/Catalyst/Action/Deserialize/MetaCPANSanitizedJSON.pm
+ignore = lib/Catalyst/Authentication/Store/Proxy.pm
+ignore = lib/Catalyst/Plugin/OAuth2/Provider.pm
+ignore = lib/Catalyst/Plugin/Session/Store/ElasticSearch.pm
+ignore = lib/MetaCPAN/Document/Distribution.pm
+ignore = lib/MetaCPAN/Document/File.pm
+ignore = lib/MetaCPAN/Document/Rating.pm
+ignore = lib/MetaCPAN/Document/Release.pm
+ignore = lib/MetaCPAN/Model.pm
+ignore = lib/MetaCPAN/Pod/XHTML.pm
+ignore = lib/MetaCPAN/Role/Common.pm
+ignore = lib/MetaCPAN/Script/Author.pm
+ignore = lib/MetaCPAN/Script/Backup.pm
+ignore = lib/MetaCPAN/Script/CPANTesters.pm
+ignore = lib/MetaCPAN/Script/Check.pm
+ignore = lib/MetaCPAN/Script/First.pm
+ignore = lib/MetaCPAN/Script/Latest.pm
+ignore = lib/MetaCPAN/Script/Mapping.pm
+ignore = lib/MetaCPAN/Script/Pagerank.pm
+ignore = lib/MetaCPAN/Script/PerlMongers.pm
+ignore = lib/MetaCPAN/Script/Query.pm
+ignore = lib/MetaCPAN/Script/ReindexDist.pm
+ignore = lib/MetaCPAN/Script/Release.pm
+ignore = lib/MetaCPAN/Script/Runner.pm
+ignore = lib/MetaCPAN/Script/Tickets.pm
+ignore = lib/MetaCPAN/Script/Watcher.pm
+ignore = lib/MetaCPAN/Server/Controller.pm
+ignore = lib/MetaCPAN/Server/Controller/Changes.pm
+ignore = lib/MetaCPAN/Server/Controller/File.pm
+ignore = lib/MetaCPAN/Server/Controller/Login.pm
+ignore = lib/MetaCPAN/Server/Controller/Login/PAUSE.pm
+ignore = lib/MetaCPAN/Server/Controller/Login/Twitter.pm
+ignore = lib/MetaCPAN/Server/Controller/Root.pm
+ignore = lib/MetaCPAN/Server/Controller/Search/Autocomplete.pm
+ignore = lib/MetaCPAN/Server/Controller/Source.pm
+ignore = lib/MetaCPAN/Server/Controller/User/Favorite.pm
+ignore = lib/MetaCPAN/Server/Model/CPAN.pm
+ignore = lib/MetaCPAN/Server/Model/Source.pm
+ignore = lib/MetaCPAN/Server/View/JSON.pm
+ignore = lib/MetaCPAN/Server/View/Pod.pm
+ignore = lib/MetaCPAN/Util.pm
+ignore = lib/Plack/Session/Store/ElasticSearch.pm
diff --git a/.travis.yml b/.travis.yml
@@ -1,13 +1,6 @@
 language: perl
 perl:
   - "5.22"
-  - "5.20"
-  - "5.18"
-
-matrix:
-  allow_failures:
-    - perl: "5.20"
-    - perl: "5.22"
 
 notifications:
   email:
@@ -17,27 +10,28 @@ notifications:
     on_failure: always
   irc: "irc.perl.org#metacpan-travis"
 
-
 env:
   global:
     # We use a non-standard port to avoid trashing production
     # but travis will have it running on the standard port.
-    - METACPAN_ES_TEST_PORT=9200
+    - ES=localhost:9200
     # Carton --deployment only works on the same version of perl
     # that the snapshot was built from.
-    - DEPLOYMENT_PERL_VERSION=5.18
+    - DEPLOYMENT_PERL_VERSION=5.22
+
+    # Instantiate Catalyst models using metacpan_server_testing.conf
+    - METACPAN_SERVER_CONFIG_LOCAL_SUFFIX=testing
 
 
 before_install:
+  - sudo service elasticsearch stop && curl -O -L https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-2.3.0.deb && sudo dpkg -i --force-confnew elasticsearch-2.3.0.deb && sudo service elasticsearch start
+
   # Run update to make libgmp-dev findable (Required by Net::OpenID::Consumer)
   # postgresql-server-dev-all is required by DBD::Pg
-  - sudo apt-get update
   - sudo apt-get install libgmp-dev postgresql-server-dev-all
 
-  # We need to run a pre-1.0 instance of ES until we update everything.
-  - wget https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-0.90.13.deb
-  - sudo dpkg -i --force-confdef elasticsearch-0.90.13.deb
   - sudo service elasticsearch restart
+  - pwd
 
   - cpanm -n Devel::Cover::Report::Coveralls
   - cpanm -n Carton
@@ -50,19 +44,19 @@ install:
   - 'carton install `test "${TRAVIS_PERL_VERSION}" = "${DEPLOYMENT_PERL_VERSION}" && echo " --deployment"`'
 
 before_script:
-  # Show status info for ES to verify that it's working, what version, etc.
-  - "curl http://localhost:${METACPAN_ES_TEST_PORT}/"
-  - "perl -i -pe 's/(servers :)9900/$1$ENV{METACPAN_ES_TEST_PORT}/' metacpan_server_testing.conf"
+  - "perl -i -pe 's/(servers :)9900/localhost:9200/' metacpan_server_testing.conf"
 
 script:
   # Devel::Cover isn't in the cpanfile
   # but if it's installed into the global dirs this should work.
   # NOTE: No '-r' for prove; 't/fakecpan.t' does the recursion for us.
-  - HARNESS_PERL_SWITCHES=-MDevel::Cover=+ignore,local carton exec prove -lv t
+  - HARNESS_PERL_SWITCHES=-MDevel::Cover=+ignore,local carton exec prove -It/lib -lvr t
 
 after_success:
   - cover -report coveralls
 
+#after_failure:
+#  - cat ~/.cpanm/build.log
 
 services:
   - elasticsearch
diff --git a/README.md b/README.md
@@ -66,31 +66,31 @@ The test suite has to pass all tests.
 ## Create the ElasticSearch Index
 
 ```sh
-./bin/carton-exec bin/metacpan mapping --delete
+./bin/run bin/metacpan mapping --delete
 ```
 
 `--delete` will drop all indices first to clear the index from test data.
 
 ## Begin Indexing Your Modules
 
 ```sh
-./bin/carton-exec bin/metacpan release /path/to/cpan/authors/id/
+./bin/run bin/metacpan release /path/to/cpan/authors/id/
 ```
 You should note that you can index either your CPAN mirror or a minicpan mirror.  You can even index just parts of a mirror:
 ```sh
-./bin/carton-exec bin/metacpan release /path/to/cpan/authors/id/{A,B}
+./bin/run bin/metacpan release /path/to/cpan/authors/id/{A,B}
 ```
 
 ## Tag the Latest Releases
 
 ```sh
-./bin/carton-exec bin/metacpan latest --cpan /path/to/cpan/
+./bin/run bin/metacpan latest --cpan /path/to/cpan/
 ```
 
 ## Index Author Data
 
 ```sh
-./bin/carton-exec bin/metacpan author --cpan /path/to/cpan/
+./bin/run bin/metacpan author --cpan /path/to/cpan/
 ```
 Note that minicpan doesn't provide the 00whois.xml file which is used to generate the index; you will have to download it manually (it is in the authors/ directory) in order to index authors.
 
@@ -102,15 +102,15 @@ It also doesn't include author.json files, so that data will also be missing unl
 
 Start API server on port 5000
 ```sh
-./bin/carton-exec plackup -p 5000 -r
+./bin/run plackup -p 5000 -r
 ```
 This will start a single-threaded test server. If you need extra performance, use `Starman` instead.
 
 ## Notes
 
 For a full list of options:
 ```sh
-./bin/carton-exec bin/metacpan release --help
+./bin/run bin/metacpan release --help
 ```
 
 Contributing:

diff --git a/app.psgi b/app.psgi
@@ -3,6 +3,7 @@ use warnings;
 
 use FindBin;
 use lib "$FindBin::RealBin/lib";
+use Catalyst::Middleware::Stash 'stash';
 
 if ( $ENV{PLACK_ENV} eq 'development' ) {
 

diff --git a/bin/check_json.pl b/bin/check_json.pl
@@ -3,7 +3,7 @@
 use 5.010;
 
 use Data::Dumper;
-use JSON::XS;
+use Cpanel::JSON::XS;
 
 foreach my $file ( @ARGV ) {
     say "Processing $file";
@@ -15,4 +15,4 @@
     };
 
     if ( $@ ) { say "\terror in $file: $@" }
-}
+}
diff --git a/bin/convert_authors.pl b/bin/convert_authors.pl
@@ -2,7 +2,7 @@
 
 use strict;
 use warnings;
-use JSON;
+use Cpanel::JSON::XS;
 use File::Find;
 
 my @files;

diff --git a/bin/get_fields.pl b/bin/get_fields.pl
@@ -1,7 +1,7 @@
 #!/usr/bin/env perl
 # PODNAME: get_fields.pl
 use Data::Dumper;
-use JSON::XS;
+use Cpanel::JSON::XS;
 use File::Find::Rule;
 use File::Basename;
 use Path::Class;
@@ -15,11 +15,11 @@
 foreach my $file ( @files ) {
     warn "Processing $file";
     my $hash;
-	
-    eval { 
+
+    eval {
         $hash = decode_json( do { local( @ARGV, $/ ) = $file; <> } );
     } or print "\terror in $file: $@";
-    
+
     while ( my ($author, $info) = each %{$hash} ) {
         my @local_fields       = keys %{$info};
         @fields{@local_fields} = @local_fields;

diff --git a/bin/munin/monitor_minion_queue.pl b/bin/munin/monitor_minion_queue.pl
@@ -0,0 +1,56 @@
+#!/usr/bin/env perl
+
+use strict;
+use warnings;
+
+# Munin runs this as metacpan user, but with root's env
+# it's only for production so path is hard coded
+
+my $config_mode = 0;
+$config_mode = 1 if $ARGV[0] && $ARGV[0] eq 'config';
+
+if($config_mode) {
+
+# Dump this (though we supported dynamic below) so it's faster
+print <<'EOF';
+graph_title Minion Queue stats
+graph_vlabel count
+graph_category metacpan_api
+graph_info What's happening in the Minion queue
+workers_inactive.label Inactive workers
+workers_active.label Active workers
+jobs_inactive.label Inactive jobs
+jobs_active.label Active jobs
+jobs_failed.label Failed jobs
+jobs_finished.label Finished jobs
+EOF
+
+exit;
+}
+
+# Get the stats
+my $stats_report = `/home/metacpan/bin/metacpan-api-carton-exec bin/queue.pl minion job -s`;
+
+my @lines = split("\n", $stats_report);
+
+for my $line (@lines) {
+  my ($label, $num) = split ':', $line;
+
+  $num =~ s/\D//g;
+
+  my $key = lc($label); # Was 'Inactive jobs'
+
+  # Swap type and status around so idle_jobs becomes jobs_idle
+  $key =~ s/(\w+)\s+(\w+)/$2_$1/g;
+
+  if( $config_mode ) {
+     # config
+     print "${key}.label $label\n";
+
+  } else {
+     # results
+     print "${key}.value $num\n" if $num;
+  }
+
+
+}
diff --git a/bin/prove b/bin/prove
@@ -1,3 +1,7 @@
 #!/bin/sh
 
-`dirname "$0"`/run prove -lv "$@"
+export EMAIL_SENDER_TRANSPORT=Test
+export ES=localhost:9900
+export METACPAN_SERVER_CONFIG_LOCAL_SUFFIX=testing
+
+`dirname "$0"`/run prove -It/lib -lvr "$@"
diff --git a/bin/queue.pl b/bin/queue.pl
@@ -9,6 +9,10 @@ =head2 DESCRIPTION
 
     carton exec -- morbo bin/queue.pl
 
+Get status on jobs and workers:
+
+    sh /home/metacpan/bin/metacpan-api-carton-exec bin/queue.pl minion job -s
+
 =cut
 
 # for morbo

diff --git a/bin/write_config_json b/bin/write_config_json
@@ -3,9 +3,9 @@
 # PODNAME: write_config_json
 #
 # takes the root directory of an extracted distribution and outputs a JSON file
-# suitable for CPAN::Faker to STDOUT 
+# suitable for CPAN::Faker to STDOUT
 use strictures 1;
-use JSON;
+use Cpanel::JSON::XS;
 use YAML;
 
 use IO::All;