Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
added a script to backfill/update suggest data
  • Loading branch information
mickeyn committed Dec 9, 2016
1 parent c343733 commit 5ae393a
Showing 1 changed file with 117 additions and 0 deletions.
117 changes: 117 additions & 0 deletions lib/MetaCPAN/Script/Suggest.pm
@@ -0,0 +1,117 @@
package MetaCPAN::Script::Suggest;

use strict;
use warnings;

use Log::Contextual qw( :log );
use Moose;
use MetaCPAN::Types qw( Bool Int );
use DateTime;

with 'MetaCPAN::Role::Script', 'MooseX::Getopt';

has age => (
is => 'ro',
isa => Int,
default => 1,
documentation => 'number of days back to cover.',
);

has all => (
is => 'ro',
isa => Bool,
default => 0,
documentation => 'update all records',
);

sub run {
my $self = shift;

if ( $self->all ) {
my $dt = DateTime->new( year => 1994, month => 1 );
my $end_time = DateTime->now()->add( months => 1 );

while ( $dt < $end_time ) {
my $gte = $dt->strftime("%Y-%m");
$dt->add( months => 1 );
my $lt = $dt->strftime("%Y-%m");

my $range = +{ range => { date => { gte => $gte, lt => $lt } } };
log_info {"updating suggest data for month: $gte"};
$self->_update_slice($range);
}
}
else {
my $gte = DateTime->now()->subtract( days => $self->age )
->strftime("%Y-%m-%d");
my $range = +{ range => { date => { gte => $gte } } };
log_info {"updating suggest data since: $gte "};
$self->_update_slice($range);
}

log_info {"done."};
}

sub _update_slice {
my ( $self, $range ) = @_;

my $files = $self->es->scroll_helper(
index => $self->index->name,
type => 'file',
search_type => 'scan',
scroll => '5m',
fields => [qw< id documentation >],
size => 500,
body => {
query => {
bool => {
must => [
{ exists => { field => "documentation" } }, $range
],
}
}
},
);

my $bulk = $self->es->bulk_helper(
index => $self->index->name,
type => 'file',
max_count => 250,
timeout => '5m',
);

while ( my $file = $files->next ) {
my $documentation = $file->{fields}{documentation}[0];
my $weight = 1000 - length($documentation);
$weight = 0 if $weight < 0;

$bulk->update(
{
id => $file->{fields}{id}[0],
doc => {
suggest => {
input => [$documentation],
payload => { doc_name => $documentation },
weight => $weight,
}
},
}
);
}

$bulk->flush;
}

__PACKAGE__->meta->make_immutable;
1;

__END__
=head1 SYNOPSIS
# bin/metacpan suggest
=head1 DESCRIPTION
After importing releases from cpan, this script will set the suggest
field for autocompletion searches.

0 comments on commit 5ae393a

Please sign in to comment.