Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
Merge pull request #626 from metacpan/mickey/script_author_bulk
script author: use proper bulk + bug fix
  • Loading branch information
oalders committed Apr 26, 2017
2 parents ee4e7c4 + 62dacc7 commit c249022
Show file tree
Hide file tree
Showing 2 changed files with 17 additions and 5 deletions.
2 changes: 1 addition & 1 deletion lib/MetaCPAN/Document/Author.pm
Expand Up @@ -89,7 +89,7 @@ has extra => (

has updated => (
is => 'ro',
isa => 'DateTime',
isa => Str,
);

has is_pause_custodial_account => (
Expand Down
20 changes: 16 additions & 4 deletions lib/MetaCPAN/Script/Author.pm
Expand Up @@ -61,7 +61,12 @@ sub index_authors {
} map { $_->{_source} } @{ $dates->{hits}->{hits} }
};

my $bulk = $self->model->bulk( size => 100 );
my $bulk = $self->es->bulk_helper(
index => $self->index->name,
type => 'author',
max_count => 250,
timeout => '25m',
);

my @author_ids_to_purge;

Expand Down Expand Up @@ -127,8 +132,15 @@ sub index_authors {
push @author_ids_to_purge, $put->{pauseid};

# Only try put if this is a valid format
$bulk->put($author);
$bulk->update(
{
id => $pauseid,
doc => $put,
doc_as_upsert => 1,
}
);
}
$bulk->flush;
$self->index->refresh;

$self->purge_author_key(@author_ids_to_purge);
Expand Down Expand Up @@ -158,7 +170,7 @@ sub author_config {

my $mtime = DateTime->from_epoch( epoch => $file->stat->mtime );

if ( $dates->{$pauseid} && $dates->{$pauseid} >= $mtime ) {
if ( $dates->{$pauseid} && $dates->{$pauseid} > $mtime ) {
log_debug {"Skipping $pauseid (newer version in index)"};
return undef;
}
Expand All @@ -175,7 +187,7 @@ sub author_config {
= { map { $_ => $author->{$_} }
qw(name asciiname profile blog perlmongers donation email website city region country location extra)
};
$author->{updated} = $mtime;
$author->{updated} = $mtime->iso8601;
return $author;
}

Expand Down

0 comments on commit c249022

Please sign in to comment.