Added cleanup mode for script/permission

This will allow cleaning up removed records once in a while (it requires a longer run time so not suited for every cron run, and records don't get deleted often anyways). Usage: add --clean_up flag to the permission script execution.
metacpan · May 14, 2017 · c8806c4 · c8806c4
1 parent 705f491
commit c8806c4
Show file tree

Hide file tree

Showing 2 changed files with 56 additions and 11 deletions.
diff --git a/lib/MetaCPAN/Script/Permission.pm b/lib/MetaCPAN/Script/Permission.pm
@@ -4,7 +4,8 @@ use Moose;
 
 use Log::Contextual qw( :log );
 use MetaCPAN::Document::Permission ();
-use PAUSE::Permissions             ();
+use MetaCPAN::Types qw( Bool );
+use PAUSE::Permissions ();
 
 with 'MooseX::Getopt', 'MetaCPAN::Role::Script';
 
@@ -15,6 +16,12 @@ CPAN/minicpan.
 
 =cut
 
+has clean_up => (
+    is      => 'ro',
+    isa     => Bool,
+    default => 0,
+);
+
 sub run {
     my $self = shift;
     $self->index_permissions;
@@ -28,40 +35,77 @@ sub index_permissions {
         = $self->cpan->subdir('modules')->file('06perms.txt')->absolute;
     my $pp = PAUSE::Permissions->new( path => $file_path );
 
-    my $bulk_helper = $self->es->bulk_helper(
+    my $bulk = $self->es->bulk_helper(
         index => $self->index->name,
         type  => 'permission',
     );
 
+    my %seen;
+    log_debug {"building permission data to add"};
+
     my $iterator = $pp->module_iterator;
     while ( my $perms = $iterator->next_module ) {
 
         # This method does a "return sort @foo", so it can't be called in the
         # ternary since it always returns false in that context.
         # https://github.com/neilb/PAUSE-Permissions/pull/16
 
+        my $name = $perms->name;
+
         my @co_maints = $perms->co_maintainers;
         my $doc       = {
-            @co_maints
-            ? ( co_maintainers => \@co_maints )
-            : (),
-            module_name => $perms->name,
+            module_name => $name,
             owner       => $perms->owner,
+
+            # empty list means no co-maintainers
+            # and passing the empty arrayref will force
+            # deleting existingd values in the field.
+            co_maintainers => \@co_maints,
         };
 
-        $bulk_helper->update(
+        $bulk->update(
             {
-                id            => $perms->name,
+                id            => $name,
                 doc           => $doc,
                 doc_as_upsert => 1,
             }
         );
+
+        $seen{$name} = 1;
     }
+    $bulk->flush;
+
+    $self->run_cleanup( $bulk, \%seen ) if $self->clean_up;
 
-    $bulk_helper->flush;
     log_info {'finished indexing 06perms'};
 }
 
+sub run_cleanup {
+    my ( $self, $bulk, $seen ) = @_;
+
+    log_debug {"checking permission data to remove"};
+
+    my $scroll = $self->es->scroll_helper(
+        index  => $self->index->name,
+        type   => 'permission',
+        scroll => '30m',
+        body   => { query => { match_all => {} } },
+    );
+
+    my @remove;
+    my $count = $scroll->total;
+    while ( my $p = $scroll->next ) {
+        my $id = $p->{_id};
+        unless ( exists $seen->{$id} ) {
+            push @remove, $id;
+            log_debug {"removed $id"};
+        }
+        log_debug { $count . " left to check" } if --$count % 10000 == 0;
+    }
+    $bulk->delete_ids(@remove);
+    $bulk->flush;
+}
+
 __PACKAGE__->meta->make_immutable;
 1;
 

diff --git a/t/server/controller/permission.t b/t/server/controller/permission.t
@@ -39,8 +39,9 @@ test_psgi app, sub {
         is_deeply(
             decode_json( $res->content ),
             {
-                module_name => $module_name,
-                owner       => 'RWSTAUNER',
+                co_maintainers => [],
+                module_name    => $module_name,
+                owner          => 'RWSTAUNER',
             },
             'Owned by RWSTAUNER, no co-maint'
         );