Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
script/latest: support queueing + code reuse
1. Moved queueing infra. to Role::Script for reuse.
2. Added support for queueing in Script::Latest
   (will be useful for later features)
  • Loading branch information
mickeyn committed Aug 10, 2016
1 parent 0f04a62 commit 8c0f18a
Show file tree
Hide file tree
Showing 5 changed files with 82 additions and 62 deletions.
64 changes: 36 additions & 28 deletions lib/MetaCPAN/Queue.pm
Expand Up @@ -29,35 +29,43 @@ sub startup {
$self->plugin( Minion => $helper->backend );

$self->minion->add_task(
index_release => sub {
my ( $job, @args ) = @_;

my @warnings;
local $SIG{__WARN__} = sub {
push @warnings, $_[0];
warn $_[0];
};

# @args could be ( '--latest', '/path/to/release' );
unshift @args, 'release';

# Runner expects to have been called via CLI
local @ARGV = @args;
try {
my $release = MetaCPAN::Script::Runner->run(@args);
$job->finish( @warnings ? { warnings => \@warnings } : () );
}
catch {
warn $_;
$job->fail(
{
message => $_,
@warnings ? ( warnings => \@warnings ) : (),
}
);
};
index_release => $self->_gen_index_task_sub('release') );

$self->minion->add_task(
index_latest => $self->_gen_index_task_sub('latest') );
}

sub _gen_index_task_sub {
my ( $self, $type ) = @_;

return sub {
my ( $job, @args ) = @_;

my @warnings;
local $SIG{__WARN__} = sub {
push @warnings, $_[0];
warn $_[0];
};

# @args could be ( '--latest', '/path/to/release' );
unshift @args, $type;

# Runner expects to have been called via CLI
local @ARGV = @args;
try {
MetaCPAN::Script::Runner->run(@args);
$job->finish( @warnings ? { warnings => \@warnings } : () );
}
catch {
warn $_;
$job->fail(
{
message => $_,
@warnings ? ( warnings => \@warnings ) : (),
}
);
};
}
);
}

1;
16 changes: 16 additions & 0 deletions lib/MetaCPAN/Role/Script.pm
Expand Up @@ -9,6 +9,7 @@ use Git::Helpers qw( checkout_root );
use Log::Contextual qw( :log :dlog );
use MetaCPAN::Model;
use MetaCPAN::Types qw(:all);
use MetaCPAN::Queue ();
use Moose::Role;
use Carp ();

Expand Down Expand Up @@ -67,6 +68,21 @@ has home => (
default => sub { checkout_root() },
);

has _minion => (
is => 'ro',
isa => 'Minion',
lazy => 1,
handles => { _add_to_queue => 'enqueue', stats => 'stats', },
default => sub { MetaCPAN::Queue->new->minion },
);

has queue => (
is => 'ro',
isa => Bool,
default => 0,
documentation => 'add indexing jobs to the minion queue',
);

with 'MetaCPAN::Role::Fastly', 'MetaCPAN::Role::HasConfig',
'MetaCPAN::Role::Logger';

Expand Down
39 changes: 30 additions & 9 deletions lib/MetaCPAN/Script/Latest.pm
Expand Up @@ -42,9 +42,17 @@ sub _build_packages {
shift->cpan->file(qw(modules 02packages.details.txt.gz))->stringify );
}

sub _queue_latest {
my $self = shift;
my $dist = shift || $self->distribution;

log_info { "queueing " . $dist };
$self->_add_to_queue( index_latest =>
[ ( $self->force ? '--force' : () ), '--distribution', $dist ] );
}

sub run {
my $self = shift;
my $modules = $self->index->type('file');
my $self = shift;

if ( $self->dry_run ) {
log_info {'Dry run: updates will not be written to ES'};
Expand All @@ -68,12 +76,18 @@ sub run {

return if ( !@filter && $self->distribution );

# if we are just queueing a single distribution
if ( $self->queue and $self->distribution ) {
$self->_queue_latest();
return;
}

my @module_filters = { term => { 'module.indexed' => 1 } };
push @module_filters, @filter
? { terms => { "module.name" => \@filter } }
: { exists => { field => "module.name" } };

my $scroll = $modules->filter(
my $scroll = $self->index->type('file')->filter(
{
bool => {
must => [
Expand All @@ -91,19 +105,18 @@ sub run {
]
}
}
)->source(
[
'module.name', 'author', 'release', 'distribution',
'date', 'status',
]
)->size(100)->raw->scroll;
)
->source(
[qw< author date distribution module.name release status >] )
->size(100)->raw->scroll;

my ( %downgrade, %upgrade );
log_debug { 'Found ' . $scroll->total . ' modules' };

my $i = 0;

my @modules_to_purge;
my %queued_distributions;

# For each file...
while ( my $file = $scroll->next ) {
Expand All @@ -125,6 +138,14 @@ sub run {
# Get P:C:P:F:Distribution (CPAN::DistnameInfo) object for package.
my $dist = $module->distribution;

if ( $self->queue ) {
my $d = $dist->dist;
$self->_queue_latest($d)
unless exists $queued_distributions{$d};
$queued_distributions{$d} = 1;
next;
}

# If 02packages has the same author/release for this package...

# NOTE: CPAN::DistnameInfo doesn't parse some weird uploads
Expand Down
9 changes: 0 additions & 9 deletions lib/MetaCPAN/Script/Queue.pm
Expand Up @@ -3,7 +3,6 @@ package MetaCPAN::Script::Queue;
use strict;
use warnings;

use MetaCPAN::Queue ();
use MetaCPAN::Types qw( Dir File );
use Moose;
use Path::Iterator::Rule ();
Expand All @@ -22,14 +21,6 @@ has file => (
coerce => 1,
);

has _minion => (
is => 'ro',
isa => 'Minion',
lazy => 1,
handles => { _add_to_queue => 'enqueue', stats => 'stats', },
default => sub { MetaCPAN::Queue->new->minion },
);

with 'MetaCPAN::Role::Script', 'MooseX::Getopt';

sub run {
Expand Down
16 changes: 0 additions & 16 deletions lib/MetaCPAN/Script/Release.pm
Expand Up @@ -16,7 +16,6 @@ use MetaCPAN::Util;
use MetaCPAN::Model::Release;
use MetaCPAN::Script::Runner;
use MetaCPAN::Types qw( Bool Dir HashRef Int Str );
use MetaCPAN::Queue ();
use Moose;
use PerlIO::gzip;
use Try::Tiny qw( catch try );
Expand All @@ -43,13 +42,6 @@ has skip => (
documentation => 'skip already indexed modules (0)',
);

has queue => (
is => 'ro',
isa => Bool,
default => 0,
documentation => 'add indexing jobs to the minion queue',
);

has status => (
is => 'ro',
isa => Str,
Expand Down Expand Up @@ -85,14 +77,6 @@ has _bulk_size => (
default => 10,
);

has _minion => (
is => 'ro',
isa => 'Minion',
lazy => 1,
handles => { _add_to_queue => 'enqueue', stats => 'stats', },
default => sub { MetaCPAN::Queue->new->minion },
);

sub run {
my $self = shift;
my ( undef, @args ) = @{ $self->extra_argv };
Expand Down

0 comments on commit 8c0f18a

Please sign in to comment.