removing values not upstream anymore + added logging

metacpan · May 13, 2017 · 2656288 · 2656288
1 parent 8296c45
commit 2656288
Showing 1 changed file with 37 additions and 0 deletions.
diff --git a/lib/MetaCPAN/Script/Debian.pm b/lib/MetaCPAN/Script/Debian.pm
@@ -8,6 +8,7 @@ use DBI                   ();
 use Email::Sender::Simple ();
 use Email::Simple         ();
 use List::MoreUtils qw( uniq );
+use Log::Contextual qw( :log );
 
 use MetaCPAN::Types qw( HashRef Str );
 
@@ -105,6 +106,27 @@ sub run {
         Email::Sender::Simple->send($email);
     }
 
+    my $scroll = $self->es->scroll_helper(
+        index => $self->index->name,
+        type  => 'distribution',
+        scroll => '10m',
+        body => {
+            query => { exists => { field => "external_package.debian" } }
+        },
+    );
+
+    my @to_remove;
+
+    while ( my $s = $scroll->next ) {
+        my $name = $s->{_source}{name};
+
+        if ( exists $dist{$name} ) {
+            delete $dist{$name} if $dist{$name} eq $s->{_source}{external_package}{debian};
+        } else {
+            push @to_remove => $name;
+        }
+    }
+
     my $bulk = $self->es->bulk_helper(
         index => $self->index->name,
         type  => 'distribution',
@@ -118,6 +140,7 @@ sub run {
         );
         next unless $exists;
 
+        log_debug {"adding $d"};
         $bulk->update(
             {
                 id  => $d,
@@ -130,6 +153,20 @@ sub run {
         );
     }
 
+    for my $d ( @to_remove ) {
+        log_debug {"removing $d"};
+        $bulk->update(
+            {
+                id  => $d,
+                doc => +{
+                    'external_package' => {
+                        debian => undef
+                    }
+                }
+            }
+        );
+    }
+
     $bulk->flush;
 }