Skip to content
Permalink

Comparing changes

Choose two branches to see what’s changed or to start a new pull request. If you need to, you can also or learn more about diff comparisons.

Open a pull request

Create a new pull request by comparing changes across two branches. If you need to, you can also . Learn more about diff comparisons here.
base repository: NixOS/hydra
Failed to load repositories. Confirm that selected base ref is valid, then try again.
Loading
base: 53c2fc2216c7
Choose a base ref
...
head repository: NixOS/hydra
Failed to load repositories. Confirm that selected head ref is valid, then try again.
Loading
compare: d0b3f2dac446
Choose a head ref
  • 2 commits
  • 9 files changed
  • 2 contributors

Commits on Jan 22, 2021

  1. Normalize nixexpr{input,path} from builds to jobsetevals. 

    Duplicating this data on every record of the builds table cost
approximately 4G of duplication.

Note that the database migration included took about 4h45m on an
untuned server which uses very slow rotational disks in a RAID5 setup,
with not a lot of RAM. I imagine in production it might take an hour
or two, but not 4. If this should become a chunked migration, I can do
that.

Note: Because of the question about chunked migrations, I have NOT
YET tested this migration thoroughly enough for merge.
    @grahamc
    grahamc committed Jan 22, 2021

    Verified

    This commit was signed with the committer’s verified signature.
    renovate-bot Mend Renovate
    GPG key ID: 81C63AE1C224182B
    Verified
    Learn about vigilant mode
    Copy the full SHA
    9516b25 View commit details

Commits on Jan 25, 2021

  1. Merge pull request #848 from grahamc/normalize-nixexprinputpath 

    Normalize nixexpr{input,path} from builds to jobsetevals.
    @edolstra
    edolstra authored Jan 25, 2021
    1
    Copy the full SHA
    d0b3f2d View commit details
18 changes: 2 additions & 16 deletions src/lib/Hydra/Schema/Builds.pm
View file
Original file line number Diff line number Diff line change
@@ -134,16 +134,6 @@ __PACKAGE__->table("builds");
default_value: 0
is_nullable: 1
=head2 nixexprinput
data_type: 'text'
is_nullable: 1
=head2 nixexprpath
data_type: 'text'
is_nullable: 1
=head2 priority
data_type: 'integer'
@@ -246,10 +236,6 @@ __PACKAGE__->add_columns(
{ data_type => "integer", default_value => 0, is_nullable => 0 },
"iscurrent",
{ data_type => "integer", default_value => 0, is_nullable => 1 },
"nixexprinput",
{ data_type => "text", is_nullable => 1 },
"nixexprpath",
{ data_type => "text", is_nullable => 1 },
"priority",
{ data_type => "integer", default_value => 0, is_nullable => 0 },
"globalpriority",
@@ -542,8 +528,8 @@ __PACKAGE__->many_to_many(
);


# Created by DBIx::Class::Schema::Loader v0.07049 @ 2020-05-27 17:40:41
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:RIKKFfcKXFWIUeM8ma++iw
# Created by DBIx::Class::Schema::Loader v0.07049 @ 2021-01-22 07:11:57
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Df5N0EByYJqoSUqA0dld/A

__PACKAGE__->has_many(
"dependents",
18 changes: 16 additions & 2 deletions src/lib/Hydra/Schema/JobsetEvals.pm
View file
Original file line number Diff line number Diff line change
@@ -89,6 +89,16 @@ __PACKAGE__->table("jobsetevals");
data_type: 'text'
is_nullable: 0
=head2 nixexprinput
data_type: 'text'
is_nullable: 1
=head2 nixexprpath
data_type: 'text'
is_nullable: 1
=head2 nrbuilds
data_type: 'integer'
@@ -132,6 +142,10 @@ __PACKAGE__->add_columns(
{ data_type => "integer", is_nullable => 0 },
"hash",
{ data_type => "text", is_nullable => 0 },
"nixexprinput",
{ data_type => "text", is_nullable => 1 },
"nixexprpath",
{ data_type => "text", is_nullable => 1 },
"nrbuilds",
{ data_type => "integer", is_nullable => 1 },
"nrsucceeded",
@@ -215,8 +229,8 @@ __PACKAGE__->belongs_to(
);


# Created by DBIx::Class::Schema::Loader v0.07049 @ 2021-01-21 11:13:38
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:zDBtAFc4HiFUcL/TpkuCcg
# Created by DBIx::Class::Schema::Loader v0.07049 @ 2021-01-22 07:11:57
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:hdu+0WWo2363dVvImMKxdA

__PACKAGE__->has_many(
"buildIds",
4 changes: 2 additions & 2 deletions src/lib/Hydra/Schema/Jobsets.pm
View file
Original file line number Diff line number Diff line change
@@ -375,8 +375,8 @@ __PACKAGE__->has_many(
);


# Created by DBIx::Class::Schema::Loader v0.07049 @ 2020-05-27 17:40:41
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:aDW78MCelU/ma953aTcHvA
# Created by DBIx::Class::Schema::Loader v0.07049 @ 2021-01-22 07:11:57
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:6P1qlC5oVSPRSgRBp6nmrw


=head2 builds
4 changes: 2 additions & 2 deletions src/lib/Hydra/Schema/Projects.pm
View file
Original file line number Diff line number Diff line change
@@ -258,8 +258,8 @@ Composing rels: L</projectmembers> -> username
__PACKAGE__->many_to_many("usernames", "projectmembers", "username");


# Created by DBIx::Class::Schema::Loader v0.07049 @ 2020-05-27 17:40:41
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:iBGJjFWiI9Wy9zwT7xGOEA
# Created by DBIx::Class::Schema::Loader v0.07049 @ 2021-01-22 07:11:57
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Ff5gJejFu+02b0lInobOoQ

my %hint = (
columns => [
6 changes: 3 additions & 3 deletions src/root/build.tt
View file
Original file line number Diff line number Diff line change
@@ -120,7 +120,7 @@ END;
<b class="caret"></b>
</a>
<ul class="dropdown-menu">
[% IF build.nixexprinput || eval.flake %]
[% IF eval.nixexprinput || eval.flake %]
<li><a href="#reproduce" data-toggle="modal">Reproduce locally</a></li>
[% END %]
[% IF c.user_exists %]
@@ -346,10 +346,10 @@ END;
<td>[% build.priority %]</td>
</tr>
[% END %]
[% IF build.nixexprinput %]
[% IF eval.nixexprinput %]
<tr>
<th>Nix expression:</th>
<td>file <tt>[% HTML.escape(build.nixexprpath) %]</tt> in input <tt>[% HTML.escape(build.nixexprinput) %]</tt></td>
<td>file <tt>[% HTML.escape(eval.nixexprpath) %]</tt> in input <tt>[% HTML.escape(eval.nixexprinput) %]</tt></td>
</tr>
[% END %]
<tr>
4 changes: 2 additions & 2 deletions src/root/reproduce.tt
View file
Original file line number Diff line number Diff line change
@@ -169,7 +169,7 @@ echo "$0: input ‘[% input.name %]’ has unsupported type ‘[% input.type %]
exit 1
[% END %]

[% IF input.name == build.nixexprinput +%]
[% IF input.name == eval.nixexprinput +%]
nixExprInputDir="$inputDir"
[%+ END %]

@@ -197,7 +197,7 @@ args+=(--option extra-binary-caches '[% c.uri_for('/') %]')
# when evaluating jobs that rely on builtins.currentSystem.
args+=(--option system x86_64-linux)

args+=("$nixExprInputDir/[% build.nixexprpath %]" -A '[% build.job.name %]')
args+=("$nixExprInputDir/[% eval.nixexprpath %]" -A '[% build.job.name %]')

if [ -n "$printFlags" ]; then
first=1
4 changes: 2 additions & 2 deletions src/script/hydra-eval-jobset
View file
Original file line number Diff line number Diff line change
@@ -460,8 +460,6 @@ sub checkBuild {
, nixname => $buildInfo->{nixName}
, drvpath => $drvPath
, system => $buildInfo->{system}
, nixexprinput => $jobset->nixexprinput
, nixexprpath => $jobset->nixexprpath
, priority => $buildInfo->{schedulingPriority}
, finished => 0
, iscurrent => 1
@@ -724,6 +722,8 @@ sub checkJobsetWrapped {
, hasnewbuilds => $jobsetChanged ? 1 : 0
, nrbuilds => $jobsetChanged ? scalar(keys %buildMap) : undef
, flake => $flakeRef
, nixexprinput => $jobset->nixexprinput
, nixexprpath => $jobset->nixexprpath
});

$db->storage->dbh->do("notify eval_added, ?", undef,
9 changes: 2 additions & 7 deletions src/sql/hydra.sql
View file
Original file line number Diff line number Diff line change
@@ -162,13 +162,6 @@ create table Builds (
isChannel integer not null default 0, -- meta.isHydraChannel
isCurrent integer default 0,

-- Copy of the nixExprInput/nixExprPath fields of the jobset that
-- instantiated this build. Needed if we want to reproduce this
-- build. FIXME: this should be stored in JobsetEvals, storing it
-- here is denormal.
nixExprInput text,
nixExprPath text,

-- Priority within a jobset, set via meta.schedulingPriority.
priority integer not null default 0,

@@ -466,6 +459,8 @@ create table JobsetEvals (
nrSucceeded integer, -- set lazily when all builds are finished

flake text, -- immutable flake reference
nixExprInput text, -- name of the jobsetInput containing the Nix or Guix expression
nixExprPath text, -- relative path of the Nix or Guix expression

foreign key (project) references Projects(name) on delete cascade on update cascade,
foreign key (project, jobset) references Jobsets(project, name) on delete cascade on update cascade
31 changes: 31 additions & 0 deletions src/sql/upgrade-71.sql
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
ALTER TABLE JobsetEvals
ADD COLUMN nixExprInput text,
ADD COLUMN nixExprPath text;


-- This migration took 4.5 hours on a server
-- with 5400RPM drives, against a copy of hydra's
-- production dataset. It might take a significantly
-- less amount of time there, and not justify a
-- batched migration.
UPDATE jobsetevals
SET (nixexprinput, nixexprpath) = (
SELECT builds.nixexprinput, builds.nixexprpath
FROM builds
LEFT JOIN jobsetevalmembers
ON jobsetevalmembers.build = builds.id
WHERE jobsetevalmembers.eval = jobsetevals.id
LIMIT 1
)
WHERE jobsetevals.id in (
SELECT jobsetevalsprime.id
FROM jobsetevals as jobsetevalsprime
WHERE jobsetevalsprime.nixexprinput IS NULL
-- AND jobsetevalsprime.id > ? --------- These are in case of a batched migration
ORDER BY jobsetevalsprime.id ASC -- /
-- LIMIT ? -- ----------------------
);

ALTER TABLE builds
DROP COLUMN nixexprinput,
DROP COLUMN nixexprpath;