Skip to content

Commit

Permalink
Merge pull request #516 from metacpan/mickey/bugs_remap
Browse files Browse the repository at this point in the history 
[WIP] distribution: bugs -> rt + github
  • Loading branch information
@oalders
oalders committed Nov 18, 2016
2 parents d7af4a4 + b2f4636 commit 6ab948c
Show file tree
Hide file tree
Showing 4 changed files with 118 additions and 58 deletions.
109 changes: 77 additions & 32 deletions lib/MetaCPAN/Script/Tickets.pm
View file
Expand Up @@ -53,6 +53,20 @@ has pithub => (
builder => '_build_pithub',
);

has _bulk => (
is => 'ro',
lazy => 1,
builder => '_build_bulk',
);

sub _build_bulk {
my $self = shift;
$self->es->bulk_helper(
index => $self->index->name,
type => 'distribution',
);
}

sub _build_github_token {
my $self = shift;
exists $self->config->{github_token}
Expand All @@ -71,48 +85,62 @@ sub _build_pithub {

sub run {
my $self = shift;
my $bugs = {};

# NOTE: Order is important here.
$self->check_all_distributions;

# Hash keys are distribution names.
# rt issues are counted for all dists (the download tsv contains everything).
# gh issues are counted for any dist with a github url in `resources.bugtracker.web`.
# Any dists in the second will overwrite the first.
foreach my $source ( @{ $self->source } ) {
if ( $source eq 'github' ) {
log_debug {'Fetching GitHub issues'};
$bugs = { %$bugs, %{ $self->retrieve_github_bugs } };
$self->index_github_bugs;
}
elsif ( $source eq 'rt' ) {
log_debug {'Fetching RT bugs'};
$bugs = { %$bugs, %{ $self->retrieve_rt_bugs } };
$self->index_rt_bugs;
}
}
$self->index_bug_summary($bugs);

return 1;
}

sub index_bug_summary {
my ( $self, $bugs ) = @_;
$self->index->refresh;
my $dists = $self->index->type('distribution');
my $bulk = $self->index->bulk( size => 300 );
for my $dist ( keys %$bugs ) {
my $doc = $dists->get($dist);
$doc ||= $dists->new_document( { name => $dist } );
$doc->_set_bugs( $bugs->{ $doc->name } );
$bulk->put($doc);
sub check_all_distributions {
my $self = shift;

# first: make sure all distributions have an entry
my $scroll = $self->es->scroll_helper(
size => 500,
scroll => '5m',
index => $self->index->name,
type => 'release',
fields => ['distribution'],
body => {
query => {
not => { term => { status => 'backpan' } }
}
},
);

my $dists = {};

while ( my $release = $scroll->next ) {
my $distribution = $release->{'fields'}{'distribution'}[0];
$distribution or next;
$dists->{$distribution} = {};
}
$bulk->commit;

$self->_bulk_update($dists);
}

sub retrieve_github_bugs {
sub index_github_bugs {
my $self = shift;
my %summary;

my $scroll
= $self->index->type('release')->find_github_based->scroll('5m');
log_debug { sprintf( "Found %s repos", $scroll->total ) };
my $summary = {};

while ( my $release = $scroll->next ) {
my $resources = $release->resources;
my ( $user, $repo, $source )
Expand All @@ -131,17 +159,16 @@ sub retrieve_github_bugs {
params => { state => 'closed' }
);
next unless ( $closed->success );
$summary->{ $release->{distribution} }
= { open => 0, closed => 0, source => $source, type => 'github' };
$summary->{ $release->{distribution} }->{open}++
while ( $open->next );
$summary->{ $release->{distribution} }->{closed}++
while ( $closed->next );
$summary->{ $release->{distribution} }->{active}
= $summary->{ $release->{distribution} }->{open};

my $rec = { open => 0, closed => 0, source => $source };
$rec->{open}++ while ( $open->next );
$rec->{closed}++ while ( $closed->next );
$rec->{active} = $rec->{open};
$summary{ $release->{'distribution'} }{'bugs'}{'github'} = $rec;
}
return $summary;

log_info {"writing github data"};
$self->_bulk_update( \%summary );
}

# Try (recursively) to find a github url in the resources hash.
Expand All @@ -164,15 +191,18 @@ sub github_user_repo_from_resources {
return ();
}

sub retrieve_rt_bugs {
sub index_rt_bugs {
my ($self) = @_;

my $resp = $self->ua->request( GET $self->rt_summary_url );

log_error { $resp->status_line } unless $resp->is_success;

# NOTE: This is sending a byte string.
return $self->parse_tsv( $resp->content );
my $summary = $self->parse_tsv( $resp->content );

log_info {"writing rt data"};
$self->_bulk_update($summary);
}

sub parse_tsv {
Expand All @@ -189,8 +219,7 @@ sub parse_tsv {

my %summary;
while ( my $row = $tsv_parser->fetch ) {
$summary{ $row->{dist} } = {
type => 'rt',
$summary{ $row->{dist} }{'bugs'}{'rt'} = {
source => $self->rt_dist_url( $row->{dist} ),
active => $row->{active},
closed => $row->{inactive},
Expand All @@ -209,6 +238,22 @@ sub rt_dist_url {
. uri_escape($dist);
}

sub _bulk_update {
my ( $self, $summary ) = @_;

for my $distribution ( keys %$summary ) {
$self->_bulk->update(
{
id => $distribution,
doc => $summary->{$distribution},
doc_as_upsert => 1,
}
);
}

$self->_bulk->flush;
}

__PACKAGE__->meta->make_immutable;

1;
Expand Down
19 changes: 16 additions & 3 deletions lib/MetaCPAN/Types/Internal.pm
View file
Expand Up @@ -25,6 +25,8 @@ use MooseX::Types -declare => [
Blog
PerlMongers
Tests
RTIssueStatus
GitHubIssueStatus
BugSummary
RiverSummary
)
Expand Down Expand Up @@ -93,16 +95,27 @@ coerce Profile, from HashRef,
subtype Tests,
as Dict [ fail => Int, na => Int, pass => Int, unknown => Int ];

subtype BugSummary,
subtype RTIssueStatus,
as Dict [
(
map { $_ => Optional [Int] }
qw(new open stalled patched resolved rejected active closed)
qw( active closed new open patched rejected resolved stalled )
),
type => Str,
source => Str
];

subtype GitHubIssueStatus,
as Dict [
( map { $_ => Optional [Int] } qw( active closed open ) ),
source => Str,
];

subtype BugSummary,
as Dict [
rt => Optional [RTIssueStatus],
github => Optional [GitHubIssueStatus],
];

subtype RiverSummary,
as Dict [ ( map { $_ => Optional [Int] } qw(total immediate bucket) ), ];

Expand Down
23 changes: 12 additions & 11 deletions t/release/bugs.t
View file
Expand Up @@ -9,17 +9,18 @@ test_distribution(
'Moose',
{
bugs => {
type => 'rt',
source =>
'https://rt.cpan.org/Public/Dist/Display.html?Name=Moose',
new => 15,
open => 20,
stalled => 4,
patched => 0,
resolved => 122,
rejected => 23,
active => 39,
closed => 145,
rt => {
source =>
'https://rt.cpan.org/Public/Dist/Display.html?Name=Moose',
new => 15,
open => 20,
stalled => 4,
patched => 0,
resolved => 122,
rejected => 23,
active => 39,
closed => 145,
},
},
},
'Test bug data for Moose dist',
Expand Down
25 changes: 13 additions & 12 deletions t/release/text-tabs-wrap.t
View file
Expand Up @@ -9,18 +9,19 @@ test_distribution(
'Text-Tabs+Wrap',
{
bugs => {
type => 'rt',
source =>
'https://rt.cpan.org/Public/Dist/Display.html?Name=Text-Tabs%2BWrap',
new => 2,
open => 0,
stalled => 0,
patched => 0,
resolved => 15,
rejected => 1,
active => 2,
closed => 16,
},
rt => {
source =>
'https://rt.cpan.org/Public/Dist/Display.html?Name=Text-Tabs%2BWrap',
new => 2,
open => 0,
stalled => 0,
patched => 0,
resolved => 15,
rejected => 1,
active => 2,
closed => 16,
},
}
},
'rt url is uri escaped',
);
Expand Down

0 comments on commit 6ab948c

Please sign in to comment.