Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
Merge pull request #601 from metacpan/mickey/autocomplete_suggester
[WIP] autocomplete suggester
  • Loading branch information
oalders committed Dec 13, 2016
2 parents d1f2134 + dd2493e commit c0fb968
Show file tree
Hide file tree
Showing 16 changed files with 674 additions and 560 deletions.
2 changes: 1 addition & 1 deletion cpanfile
Expand Up @@ -102,7 +102,7 @@ requires 'MooseX::Aliases';
requires 'MooseX::Attribute::Deflator', '2.1.5';
requires 'MooseX::ChainedAccessors';
requires 'MooseX::ClassAttribute';
requires 'MooseX::Getopt';
requires 'MooseX::Getopt', '0.71';
requires 'MooseX::Getopt::Dashes';
requires 'MooseX::Getopt::OptionTypeMap';
requires 'MooseX::Fastly::Role', '0.02';
Expand Down
28 changes: 14 additions & 14 deletions cpanfile.snapshot
Expand Up @@ -5220,20 +5220,20 @@ DISTRIBUTIONS
ExtUtils::MakeMaker 0
HTTP::Tiny 0
Moose::Role 0
MooseX-Getopt-0.70
pathname: D/DR/DROLSKY/MooseX-Getopt-0.70.tar.gz
provides:
MooseX::Getopt 0.70
MooseX::Getopt::Basic 0.70
MooseX::Getopt::Dashes 0.70
MooseX::Getopt::GLD 0.70
MooseX::Getopt::Meta::Attribute 0.70
MooseX::Getopt::Meta::Attribute::NoGetopt 0.70
MooseX::Getopt::Meta::Attribute::Trait 0.70
MooseX::Getopt::Meta::Attribute::Trait::NoGetopt 0.70
MooseX::Getopt::OptionTypeMap 0.70
MooseX::Getopt::ProcessedArgv 0.70
MooseX::Getopt::Strict 0.70
MooseX-Getopt-0.71
pathname: E/ET/ETHER/MooseX-Getopt-0.71.tar.gz
provides:
MooseX::Getopt 0.71
MooseX::Getopt::Basic 0.71
MooseX::Getopt::Dashes 0.71
MooseX::Getopt::GLD 0.71
MooseX::Getopt::Meta::Attribute 0.71
MooseX::Getopt::Meta::Attribute::NoGetopt 0.71
MooseX::Getopt::Meta::Attribute::Trait 0.71
MooseX::Getopt::Meta::Attribute::Trait::NoGetopt 0.71
MooseX::Getopt::OptionTypeMap 0.71
MooseX::Getopt::ProcessedArgv 0.71
MooseX::Getopt::Strict 0.71
requirements:
Carp 0
Getopt::Long 2.37
Expand Down
31 changes: 31 additions & 0 deletions lib/MetaCPAN/Document/File.pm
Expand Up @@ -363,6 +363,37 @@ sub _build_documentation {
return undef;
}

=head2 suggest
Autocomplete info for documentation.
=cut

has suggest => (
is => 'ro',

# isa => Maybe [HashRef], # remarked: breaks the suggester
lazy => 1,
builder => '_build_suggest',
);

sub _build_suggest {
my $self = shift;
my $doc = $self->documentation;

# return +{} unless $doc; # remarked because of 'isa'
return unless $doc;

my $weight = 1000 - length($doc);
$weight = 0 if $weight < 0;

return +{
input => [$doc],
payload => { doc_name => $doc },
weight => $weight,
};
}

=head2 indexed
B<Default 0>
Expand Down
58 changes: 58 additions & 0 deletions lib/MetaCPAN/Document/File/Set.pm
Expand Up @@ -475,5 +475,63 @@ sub autocomplete {
)->sort( [ '_score', 'documentation' ] );
}

# this method will replace 'sub autocomplete' after the
# mapping + data is fully deployed.
# -- Mickey
sub autocomplete_using_suggester {
my ( $self, @terms ) = @_;
my $query = join( q{ }, @terms );
return $self unless $query;

my $suggestions
= $self->search_type('dfs_query_then_fetch')->es->suggest(
{
index => $self->index->name,
body => {
documentation => {
text => $query,
completion => {
field => "suggest",
size => 50,
}
}
},
}
);

my @docs
= map { $_->{text} } @{ $suggestions->{documentation}[0]{options} };

my $data = $self->es->search(
{
index => $self->index->name,
type => 'file',
body => {
query => { match_all => {} },
filter => {
bool => {
must => [
{ term => { indexed => 1 } },
{ term => { authorized => 1 } },
{ term => { status => 'latest' } },
{ terms => { 'documentation' => \@docs } },
],
}
}
},
fields => ['documentation'],
size => 10
}
);

return +{
suggestions => [
sort { length($a) <=> length($b) || $a cmp $b }
map { $_->{fields}{documentation}[0] }
@{ $data->{hits}{hits} }
]
};
}

__PACKAGE__->meta->make_immutable;
1;
89 changes: 48 additions & 41 deletions lib/MetaCPAN/Script/Mapping.pm
Expand Up @@ -28,28 +28,32 @@ use constant {

with 'MetaCPAN::Role::Script', 'MooseX::Getopt';

has delete => (
has arg_deploy_mapping => (
init_arg => 'delete',
is => 'ro',
isa => Bool,
default => 0,
documentation => 'delete index if it exists already',
);

has list_types => (
has arg_list_types => (
init_arg => 'list_types',
is => 'ro',
isa => Bool,
default => 0,
documentation => 'list available index type names',
);

has create_index => (
has arg_create_index => (
init_arg => 'create_index',
is => 'ro',
isa => Str,
default => "",
documentation => 'create a new empty index (copy mappings)',
);

has update_index => (
has arg_update_index => (
init_arg => 'update_index',
is => 'ro',
isa => Str,
default => "",
Expand Down Expand Up @@ -77,7 +81,8 @@ has copy_to_index => (
documentation => 'index to copy type to',
);

has copy_type => (
has arg_copy_type => (
init_arg => 'copy_type',
is => 'ro',
isa => Str,
default => "",
Expand All @@ -99,7 +104,8 @@ has reindex => (
documentation => 'reindex data from source index for exact mapping types',
);

has delete_index => (
has arg_delete_index => (
init_arg => 'delete_index',
is => 'ro',
isa => Str,
default => "",
Expand All @@ -115,13 +121,13 @@ has delete_from_type => (

sub run {
my $self = shift;
$self->index_create if $self->create_index;
$self->index_delete if $self->delete_index;
$self->index_update if $self->update_index;
$self->type_copy if $self->copy_to_index;
$self->type_empty if $self->delete_from_type;
$self->types_list if $self->list_types;
$self->deploy_mapping if $self->delete;
$self->create_index if $self->arg_create_index;
$self->delete_index if $self->arg_delete_index;
$self->update_index if $self->arg_update_index;
$self->copy_type if $self->copy_to_index;
$self->empty_type if $self->delete_from_type;
$self->list_types if $self->arg_list_types;
$self->deploy_mapping if $self->arg_deploy_mapping;
}

sub _check_index_exists {
Expand All @@ -139,25 +145,25 @@ sub _check_index_exists {
}
}

sub index_delete {
sub delete_index {
my $self = shift;
my $name = $self->delete_index;
my $name = $self->arg_delete_index;

$self->_check_index_exists( $name, EXPECTED );
$self->are_you_sure("Index $name will be deleted !!!");

$self->_index_delete($name);
$self->_delete_index($name);
}

sub _index_delete {
sub _delete_index {
my ( $self, $name ) = @_;
log_info {"Deleting index: $name"};
$self->es->indices->delete( index => $name );
}

sub index_update {
sub update_index {
my $self = shift;
my $name = $self->update_index;
my $name = $self->arg_update_index;

$self->_check_index_exists( $name, EXPECTED );
$self->are_you_sure("Index $name will be updated !!!");
Expand Down Expand Up @@ -185,10 +191,10 @@ sub index_update {
log_info {"Done."};
}

sub index_create {
sub create_index {
my $self = shift;

my $dst_idx = $self->create_index;
my $dst_idx = $self->arg_create_index;
$self->_check_index_exists( $dst_idx, NOT_EXPECTED );

my $patch_mapping = decode_json $self->patch_mapping;
Expand Down Expand Up @@ -227,7 +233,7 @@ sub index_create {
)
{
log_info {"Re-indexing data to index $dst_idx from type: $type"};
$self->type_copy( $dst_idx, $type );
$self->copy_type( $dst_idx, $type );
}
}

Expand All @@ -238,12 +244,12 @@ sub index_create {
if @patch_types;
}

sub type_copy {
sub copy_type {
my ( $self, $index, $type ) = @_;
$index //= $self->copy_to_index;

$self->_check_index_exists( $index, EXPECTED );
$type //= $self->copy_type;
$type //= $self->arg_copy_type;
$type or die "can't copy without a type\n";

my $arg_query = $self->copy_query;
Expand Down Expand Up @@ -323,7 +329,7 @@ sub _copy_slice {
$bulk->flush;
}

sub type_empty {
sub empty_type {
my $self = shift;

my $bulk = $self->es->bulk_helper(
Expand Down Expand Up @@ -355,41 +361,42 @@ sub type_empty {
$bulk->flush;
}

sub types_list {
sub list_types {
my $self = shift;
print "$_\n" for sort keys %{ $self->index->types };
}

sub deploy_mapping {
my $self = shift;
my $es = $self->es;
my $idx_cpan = 'cpan_v1_01';
my $idx_user = 'user';
my $self = shift;
my $es = $self->es;
my $cpan_index = 'cpan_v1_01';
my $user_index = 'user';

$self->are_you_sure(
'this will delete EVERYTHING and re-create the (empty) indexes');

# delete cpan (aliased) + user indices

$self->_index_delete($idx_user)
if $es->indices->exists( index => $idx_user );
$self->_index_delete($idx_cpan)
if $es->indices->exists( index => $idx_cpan );
$self->_delete_index($user_index)
if $es->indices->exists( index => $user_index );
$self->_delete_index($cpan_index)
if $es->indices->exists( index => $cpan_index );

# create new indices

my $dep = decode_json MetaCPAN::Script::Mapping::DeployStatement::mapping;
my $dep
= decode_json(MetaCPAN::Script::Mapping::DeployStatement::mapping);

log_info {"Creating index: user"};
$es->indices->create( index => $idx_user, body => $dep );
$es->indices->create( index => $user_index, body => $dep );

log_info {"Creating index: $idx_cpan"};
$es->indices->create( index => $idx_cpan, body => $dep );
log_info {"Creating index: $cpan_index"};
$es->indices->create( index => $cpan_index, body => $dep );

# create type mappings

my %mappings = (
$idx_cpan => {
$cpan_index => {
author =>
decode_json(MetaCPAN::Script::Mapping::CPAN::Author::mapping),
distribution =>
Expand All @@ -406,7 +413,7 @@ sub deploy_mapping {
decode_json( MetaCPAN::Script::Mapping::CPAN::Release::mapping
),
},
$idx_user => {
$user_index => {
account =>
decode_json( MetaCPAN::Script::Mapping::User::Account::mapping
),
Expand All @@ -432,7 +439,7 @@ sub deploy_mapping {

# create alias
$es->indices->put_alias(
index => $idx_cpan,
index => $cpan_index,
name => 'cpan',
);

Expand Down

0 comments on commit c0fb968

Please sign in to comment.