Skip to content

Commit

Permalink
Merge pull request #599 from metacpan/mickey/mapping_simplified
Browse files Browse the repository at this point in the history 
mapping simplified
  • Loading branch information
@oalders
oalders committed Dec 13, 2016
2 parents 4e92414 + df52c67 commit d1f2134
Show file tree
Hide file tree
Showing 13 changed files with 1,471 additions and 5 deletions.
99 changes: 94 additions & 5 deletions lib/MetaCPAN/Script/Mapping.pm
View file
Expand Up @@ -9,6 +9,18 @@ use MetaCPAN::Types qw( Bool Str );
use Cpanel::JSON::XS qw( decode_json );
use DateTime;

use MetaCPAN::Script::Mapping::DeployStatement;
use MetaCPAN::Script::Mapping::CPAN::Author;
use MetaCPAN::Script::Mapping::CPAN::Distribution;
use MetaCPAN::Script::Mapping::CPAN::Favorite;
use MetaCPAN::Script::Mapping::CPAN::File;
use MetaCPAN::Script::Mapping::CPAN::Mirror;
use MetaCPAN::Script::Mapping::CPAN::Rating;
use MetaCPAN::Script::Mapping::CPAN::Release;
use MetaCPAN::Script::Mapping::User::Account;
use MetaCPAN::Script::Mapping::User::Identity;
use MetaCPAN::Script::Mapping::User::Session;

use constant {
EXPECTED => 1,
NOT_EXPECTED => 0,
Expand Down Expand Up @@ -109,7 +121,7 @@ sub run {
$self->type_copy if $self->copy_to_index;
$self->type_empty if $self->delete_from_type;
$self->types_list if $self->list_types;
$self->delete_mapping if $self->delete;
$self->deploy_mapping if $self->delete;
}

sub _check_index_exists {
Expand All @@ -134,6 +146,11 @@ sub index_delete {
$self->_check_index_exists( $name, EXPECTED );
$self->are_you_sure("Index $name will be deleted !!!");

$self->_index_delete($name);
}

sub _index_delete {
my ( $self, $name ) = @_;
log_info {"Deleting index: $name"};
$self->es->indices->delete( index => $name );
}
Expand Down Expand Up @@ -343,13 +360,85 @@ sub types_list {
print "$_\n" for sort keys %{ $self->index->types };
}

sub delete_mapping {
my $self = shift;
sub deploy_mapping {
my $self = shift;
my $es = $self->es;
my $idx_cpan = 'cpan_v1_01';
my $idx_user = 'user';

$self->are_you_sure(
'this will delete EVERYTHING and re-create the (empty) indexes');
log_info {"Putting mapping to ElasticSearch server"};
$self->model->deploy( delete => $self->delete );

# delete cpan (aliased) + user indices

$self->_index_delete($idx_user)
if $es->indices->exists( index => $idx_user );
$self->_index_delete($idx_cpan)
if $es->indices->exists( index => $idx_cpan );

# create new indices

my $dep = decode_json MetaCPAN::Script::Mapping::DeployStatement::mapping;

log_info {"Creating index: user"};
$es->indices->create( index => $idx_user, body => $dep );

log_info {"Creating index: $idx_cpan"};
$es->indices->create( index => $idx_cpan, body => $dep );

# create type mappings

my %mappings = (
$idx_cpan => {
author =>
decode_json(MetaCPAN::Script::Mapping::CPAN::Author::mapping),
distribution =>
decode_json( MetaCPAN::Script::Mapping::CPAN::Distribution::mapping
),
favorite =>
decode_json( MetaCPAN::Script::Mapping::CPAN::Favorite::mapping
),
file =>
decode_json(MetaCPAN::Script::Mapping::CPAN::File::mapping),
rating =>
decode_json(MetaCPAN::Script::Mapping::CPAN::Rating::mapping),
release =>
decode_json( MetaCPAN::Script::Mapping::CPAN::Release::mapping
),
},
$idx_user => {
account =>
decode_json( MetaCPAN::Script::Mapping::User::Account::mapping
),
identity =>
decode_json( MetaCPAN::Script::Mapping::User::Identity::mapping
),
session =>
decode_json( MetaCPAN::Script::Mapping::User::Session::mapping
),
},
);

for my $idx ( sort keys %mappings ) {
for my $type ( sort keys %{ $mappings{$idx} } ) {
log_info {"Adding mapping: $idx/$type"};
$es->indices->put_mapping(
index => $idx,
type => $type,
body => { $type => $mappings{$idx}{$type} },
);
}
}

# create alias
$es->indices->put_alias(
index => $idx_cpan,
name => 'cpan',
);

# done
log_info {"Done."};
1;
}

__PACKAGE__->meta->make_immutable;
Expand Down
163 changes: 163 additions & 0 deletions lib/MetaCPAN/Script/Mapping/CPAN/Author.pm
View file
@@ -0,0 +1,163 @@
package MetaCPAN::Script::Mapping::CPAN::Author;

use strict;
use warnings;

sub mapping {
'{
"dynamic" : false,
"properties" : {
"profile" : {
"include_in_root" : true,
"dynamic" : false,
"type" : "nested",
"properties" : {
"name" : {
"ignore_above" : 2048,
"index" : "not_analyzed",
"type" : "string"
},
"id" : {
"ignore_above" : 2048,
"index" : "not_analyzed",
"fields" : {
"analyzed" : {
"store" : true,
"fielddata" : {
"format" : "disabled"
},
"type" : "string",
"analyzer" : "simple"
}
},
"type" : "string"
}
}
},
"website" : {
"ignore_above" : 2048,
"index" : "not_analyzed",
"type" : "string"
},
"email" : {
"ignore_above" : 2048,
"index" : "not_analyzed",
"type" : "string"
},
"city" : {
"ignore_above" : 2048,
"index" : "not_analyzed",
"type" : "string"
},
"user" : {
"ignore_above" : 2048,
"index" : "not_analyzed",
"type" : "string"
},
"updated" : {
"format" : "strict_date_optional_time||epoch_millis",
"type" : "date"
},
"pauseid" : {
"ignore_above" : 2048,
"index" : "not_analyzed",
"type" : "string"
},
"country" : {
"ignore_above" : 2048,
"index" : "not_analyzed",
"type" : "string"
},
"gravatar_url" : {
"ignore_above" : 2048,
"index" : "not_analyzed",
"type" : "string"
},
"location" : {
"type" : "geo_point"
},
"donation" : {
"dynamic" : true,
"properties" : {
"name" : {
"ignore_above" : 2048,
"index" : "not_analyzed",
"type" : "string"
},
"id" : {
"ignore_above" : 2048,
"index" : "not_analyzed",
"type" : "string"
}
}
},
"asciiname" : {
"ignore_above" : 2048,
"index" : "not_analyzed",
"fields" : {
"analyzed" : {
"store" : true,
"fielddata" : {
"format" : "disabled"
},
"type" : "string",
"analyzer" : "standard"
}
},
"type" : "string"
},
"name" : {
"ignore_above" : 2048,
"index" : "not_analyzed",
"fields" : {
"analyzed" : {
"store" : true,
"fielddata" : {
"format" : "disabled"
},
"type" : "string",
"analyzer" : "standard"
}
},
"type" : "string"
},
"region" : {
"ignore_above" : 2048,
"index" : "not_analyzed",
"type" : "string"
},
"blog" : {
"dynamic" : true,
"properties" : {
"feed" : {
"ignore_above" : 2048,
"index" : "not_analyzed",
"type" : "string"
},
"url" : {
"ignore_above" : 2048,
"index" : "not_analyzed",
"type" : "string"
}
}
},
"perlmongers" : {
"dynamic" : true,
"properties" : {
"url" : {
"ignore_above" : 2048,
"index" : "not_analyzed",
"type" : "string"
},
"name" : {
"ignore_above" : 2048,
"index" : "not_analyzed",
"type" : "string"
}
}
}
}
}';
}

1;
91 changes: 91 additions & 0 deletions lib/MetaCPAN/Script/Mapping/CPAN/Distribution.pm
View file
@@ -0,0 +1,91 @@
package MetaCPAN::Script::Mapping::CPAN::Distribution;

use strict;
use warnings;

sub mapping {
'{
"dynamic" : false,
"properties" : {
"river" : {
"dynamic" : true,
"properties" : {
"immediate" : {
"type" : "integer"
},
"bucket" : {
"type" : "integer"
},
"total" : {
"type" : "integer"
}
}
},
"name" : {
"ignore_above" : 2048,
"index" : "not_analyzed",
"type" : "string"
},
"bugs" : {
"dynamic" : true,
"properties" : {
"rt" : {
"dynamic" : true,
"properties" : {
"source" : {
"ignore_above" : 2048,
"index" : "not_analyzed",
"type" : "string"
},
"closed" : {
"type" : "integer"
},
"rejected" : {
"type" : "integer"
},
"resolved" : {
"type" : "integer"
},
"active" : {
"type" : "integer"
},
"patched" : {
"type" : "integer"
},
"stalled" : {
"type" : "integer"
},
"open" : {
"type" : "integer"
},
"new" : {
"type" : "integer"
}
}
},
"github" : {
"dynamic" : true,
"properties" : {
"source" : {
"ignore_above" : 2048,
"index" : "not_analyzed",
"type" : "string"
},
"open" : {
"type" : "integer"
},
"closed" : {
"type" : "integer"
},
"active" : {
"type" : "integer"
}
}
}
}
}
}
}';
}

1;

0 comments on commit d1f2134

Please sign in to comment.