Skip to content
Permalink

Comparing changes

Choose two branches to see what’s changed or to start a new pull request. If you need to, you can also or learn more about diff comparisons.

Open a pull request

Create a new pull request by comparing changes across two branches. If you need to, you can also . Learn more about diff comparisons here.
base repository: NixOS/ofborg
Failed to load repositories. Confirm that selected base ref is valid, then try again.
Loading
base: 0065a134617b
Choose a base ref
...
head repository: NixOS/ofborg
Failed to load repositories. Confirm that selected head ref is valid, then try again.
Loading
compare: d11552addb31
Choose a head ref

Commits on Mar 17, 2018

  1. Copy the full SHA
    9420cb5 View commit details
  2. Copy the full SHA
    29fadfa View commit details

Commits on Mar 18, 2018

  1. Require full logs everywhere

    grahamc committed Mar 18, 2018
    Copy the full SHA
    789986a View commit details
  2. Copy the full SHA
    cfa9547 View commit details
  3. fixup: full logs

    grahamc committed Mar 18, 2018
    Copy the full SHA
    bebe197 View commit details
  4. Copy the full SHA
    f971b7b View commit details
  5. Copy the full SHA
    69502ec View commit details
  6. Merge pull request #132 from NixOS/clarify-partial-log

    Clarify partial log
    grahamc authored Mar 18, 2018
    Copy the full SHA
    b46eea0 View commit details
  7. fixup tests

    grahamc committed Mar 18, 2018
    Copy the full SHA
    d64eb49 View commit details
  8. Copy the full SHA
    df10054 View commit details
  9. Copy the full SHA
    ec75dba View commit details
  10. Clean up result partition

    grahamc committed Mar 18, 2018
    Copy the full SHA
    a0e336b View commit details
  11. Bump version: 0.1.4

    grahamc committed Mar 18, 2018
    Copy the full SHA
    38c48da View commit details
  12. Merge pull request #134 from NixOS/log-instantiation-failures

    Log instantiation failures
    grahamc authored Mar 18, 2018
    Copy the full SHA
    41da488 View commit details
  13. Copy the full SHA
    b7a296e View commit details
  14. Simplify log_snippet

    Ekleog committed Mar 18, 2018
    Copy the full SHA
    ff71d83 View commit details
  15. Merge pull request #135 from Ekleog/next

    Simplify log_snippet
    grahamc authored Mar 18, 2018
    Copy the full SHA
    dc06ba1 View commit details
  16. Copy the full SHA
    2c5b55d View commit details
  17. Copy the full SHA
    e86fc6b View commit details

Commits on Mar 19, 2018

  1. Merge pull request #136 from NixOS/request-id

    Pass a request-id along with build jobs and results
    grahamc authored Mar 19, 2018
    Copy the full SHA
    022e209 View commit details
  2. Own the merge conflict label

    grahamc committed Mar 19, 2018
    Copy the full SHA
    5820df9 View commit details

Commits on Mar 20, 2018

  1. Handle capitalized user names

    Ekleog committed Mar 20, 2018
    Copy the full SHA
    c88d98b View commit details
  2. Merge pull request #140 from Ekleog/next

    Handle capitalized user names
    grahamc authored Mar 20, 2018
    Copy the full SHA
    d11552a View commit details
12 changes: 6 additions & 6 deletions nix/ofborg-carnix.nix
Original file line number Diff line number Diff line change
@@ -45,7 +45,7 @@ let kernel = buildPlatform.parsed.kernel.name;
) [] (builtins.attrNames feat);
in
rec {
ofborg = f: ofborg_0_1_3 { features = ofborg_0_1_3_features { ofborg_0_1_3 = f; }; };
ofborg = f: ofborg_0_1_5 { features = ofborg_0_1_5_features { ofborg_0_1_5 = f; }; };
aho_corasick_0_5_3_ = { dependencies?[], buildDependencies?[], features?[] }: buildRustCrate {
crateName = "aho-corasick";
version = "0.5.3";
@@ -413,9 +413,9 @@ rec {
sha256 = "1y6qnd9r8ga6y8mvlabdrr73nc8cshjjlzbvnanzyj9b8zzkfwk2";
inherit dependencies buildDependencies features;
};
ofborg_0_1_3_ = { dependencies?[], buildDependencies?[], features?[] }: buildRustCrate {
ofborg_0_1_5_ = { dependencies?[], buildDependencies?[], features?[] }: buildRustCrate {
crateName = "ofborg";
version = "0.1.3";
version = "0.1.5";
authors = [ "Graham Christensen <graham@grahamc.com>" ];
src = include [ "Cargo.toml" "Cargo.lock" "src" "test-srcs" "build.rs" ] ./../ofborg;
build = "build.rs";
@@ -1342,10 +1342,10 @@ rec {
libc_0_2_36.default = true;
num_cpus_1_8_0.default = (f.num_cpus_1_8_0.default or true);
}) [ libc_0_2_36_features ];
ofborg_0_1_3 = { features?(ofborg_0_1_3_features {}) }: ofborg_0_1_3_ {
ofborg_0_1_5 = { features?(ofborg_0_1_5_features {}) }: ofborg_0_1_5_ {
dependencies = mapFeatures features ([ amqp_0_1_0 either_1_4_0 env_logger_0_4_3 fs2_0_4_3 hubcaps_0_3_16 hyper_0_10_13 hyper_native_tls_0_2_4 log_0_3_8 lru_cache_0_1_1 md5_0_3_6 serde_1_0_27 serde_derive_1_0_27 serde_json_1_0_9 tempfile_2_2_0 uuid_0_4_0 ]);
};
ofborg_0_1_3_features = f: updateFeatures f (rec {
ofborg_0_1_5_features = f: updateFeatures f (rec {
amqp_0_1_0.default = true;
either_1_4_0.default = true;
env_logger_0_4_3.default = true;
@@ -1356,7 +1356,7 @@ rec {
log_0_3_8.default = true;
lru_cache_0_1_1.default = true;
md5_0_3_6.default = true;
ofborg_0_1_3.default = (f.ofborg_0_1_3.default or true);
ofborg_0_1_5.default = (f.ofborg_0_1_5.default or true);
serde_1_0_27.default = true;
serde_derive_1_0_27.default = true;
serde_json_1_0_9.default = true;
2 changes: 1 addition & 1 deletion ofborg/Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion ofborg/Cargo.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[package]
name = "ofborg"
version = "0.1.3"
version = "0.1.5"
authors = ["Graham Christensen <graham@grahamc.com>"]
include = ["Cargo.toml", "Cargo.lock", "src", "test-srcs", "build.rs"]
build = "build.rs"
8 changes: 7 additions & 1 deletion ofborg/src/acl.rs
Original file line number Diff line number Diff line change
@@ -6,7 +6,13 @@ pub struct ACL {
}

impl ACL {
pub fn new(repos: Vec<String>, trusted_users: Vec<String>, known_users: Vec<String>) -> ACL {
pub fn new(
repos: Vec<String>,
mut trusted_users: Vec<String>,
mut known_users: Vec<String>
) -> ACL {
trusted_users.iter_mut().map(|x| *x = x.to_lowercase()).last();
known_users.iter_mut().map(|x| *x = x.to_lowercase()).last();
return ACL {
trusted_users: trusted_users,
known_users: known_users,
1 change: 1 addition & 0 deletions ofborg/src/bin/build-faker.rs
Original file line number Diff line number Diff line change
@@ -53,6 +53,7 @@ fn main() {
attrs: vec!["success".to_owned()],
logs: Some((Some("logs".to_owned()), Some(logbackrk.to_lowercase()))),
statusreport: Some((None, Some("scratch".to_owned()))),
request_id: "bogus-request-id".to_owned(),
};

{
63 changes: 39 additions & 24 deletions ofborg/src/bin/builder.rs
Original file line number Diff line number Diff line change
@@ -25,18 +25,16 @@ fn main() {
let cloner = checkout::cached_cloner(Path::new(&cfg.checkout.root));
let nix = cfg.nix();

let full_logs: bool = match &cfg.feedback {
&Some(ref feedback) => feedback.full_logs,
&None => {
warn!("Please define feedback.full_logs in your configuration to true or false!");
warn!("feedback.full_logs when true will cause the full build log to be sent back");
warn!("to the server, and be viewable by everyone.");
warn!("I strongly encourage everybody turn this on!");
false
}
if cfg.feedback.full_logs != true {
warn!("Please define feedback.full_logs in your configuration to true!");
warn!("feedback.full_logs when true will cause the full build log to be sent back");
warn!("to the server, and be viewable by everyone.");
warn!("");
warn!("Builders are no longer allowed to operate with this off");
warn!("so your builder will no longer start.");
panic!();
};


let mut session = easyamqp::session_from_config(&cfg.rabbitmq).unwrap();
let mut channel = session.open_channel(1).unwrap();
channel.basic_prefetch(1).unwrap();
@@ -53,21 +51,38 @@ fn main() {
})
.unwrap();

channel
.declare_queue(easyamqp::QueueConfig {
queue: format!("build-inputs-{}", cfg.nix.system.clone()),
passive: false,
durable: true,
exclusive: false,
auto_delete: false,
no_wait: false,
arguments: None,
})
.unwrap();
let queue_name: String;
if cfg.runner.build_all_jobs != Some(true) {
queue_name = channel
.declare_queue(easyamqp::QueueConfig {
queue: format!("build-inputs-{}", cfg.nix.system.clone()),
passive: false,
durable: true,
exclusive: false,
auto_delete: false,
no_wait: false,
arguments: None,
})
.unwrap().queue;
} else {
warn!("Building all jobs, please don't use this unless you're");
warn!("developing and have Graham's permission!");
queue_name = channel
.declare_queue(easyamqp::QueueConfig {
queue: "".to_owned(),
passive: false,
durable: false,
exclusive: true,
auto_delete: true,
no_wait: false,
arguments: None,
})
.unwrap().queue;
}

channel
.bind_queue(easyamqp::BindQueueConfig {
queue: format!("build-inputs-{}", cfg.nix.system.clone()),
queue: queue_name.clone(),
exchange: "build-jobs".to_owned(),
routing_key: None,
no_wait: false,
@@ -82,10 +97,9 @@ fn main() {
nix,
cfg.nix.system.clone(),
cfg.runner.identity.clone(),
full_logs,
)),
easyamqp::ConsumeConfig {
queue: format!("build-inputs-{}", cfg.nix.system.clone()),
queue: queue_name.clone(),
consumer_tag: format!("{}-builder", cfg.whoami()),
no_local: false,
no_ack: false,
@@ -96,6 +110,7 @@ fn main() {
)
.unwrap();

println!("Fetching jobs from {}", &queue_name);
channel.start_consuming();
channel.close(200, "Bye").unwrap();
println!("Closed the channel");
1 change: 1 addition & 0 deletions ofborg/src/bin/log-message-generator.rs
Original file line number Diff line number Diff line change
@@ -44,6 +44,7 @@ fn main() {
Some(String::from("build.log")),
)),
statusreport: Some((Some(String::from("build-results")), None)),
request_id: "bogus-request-id".to_owned(),
};

loop {
10 changes: 9 additions & 1 deletion ofborg/src/config.rs
Original file line number Diff line number Diff line change
@@ -15,7 +15,7 @@ use ofborg::acl;
#[derive(Serialize, Deserialize, Debug)]
pub struct Config {
pub runner: RunnerConfig,
pub feedback: Option<FeedbackConfig>,
pub feedback: FeedbackConfig,
pub checkout: CheckoutConfig,
pub nix: NixConfig,
pub rabbitmq: RabbitMQConfig,
@@ -62,6 +62,14 @@ pub struct RunnerConfig {
pub repos: Option<Vec<String>>,
pub trusted_users: Option<Vec<String>>,
pub known_users: Option<Vec<String>>,

/// If true, will create its own queue attached to the build job
/// exchange. This means that builders with this enabled will
/// trigger duplicate replies to the request for this
/// architecture.
///
/// This should only be turned on for development.
pub build_all_jobs: Option<bool>
}

#[derive(Serialize, Deserialize, Debug)]
13 changes: 13 additions & 0 deletions ofborg/src/lib.rs
Original file line number Diff line number Diff line change
@@ -70,6 +70,19 @@ pub mod ofborg {
pub use easyamqp;

pub const VERSION: &'static str = env!("CARGO_PKG_VERSION");

pub fn partition_result<A,B>(results: Vec<Result<A,B>>) -> (Vec<A>, Vec<B>) {
let mut ok = Vec::new();
let mut err = Vec::new();
for result in results.into_iter() {
match result {
Ok(x) => { ok.push(x); }
Err(x) => { err.push(x); }
}
}

(ok, err)
}
}

pub fn setup_log() {
3 changes: 3 additions & 0 deletions ofborg/src/message/buildjob.rs
Original file line number Diff line number Diff line change
@@ -8,6 +8,7 @@ pub struct BuildJob {
pub pr: Pr,
pub subset: Option<Subset>,
pub attrs: Vec<String>,
pub request_id: String,
pub logs: Option<ExchangeQueue>, // (Exchange, Routing Key)
pub statusreport: Option<ExchangeQueue>, // (Exchange, Routing Key)
}
@@ -24,6 +25,7 @@ impl BuildJob {
attrs: Vec<String>,
logs: Option<ExchangeQueue>,
statusreport: Option<ExchangeQueue>,
request_id: String,
) -> BuildJob {

let logbackrk = format!(
@@ -41,6 +43,7 @@ impl BuildJob {
statusreport: Some(statusreport.unwrap_or(
(Some("build-results".to_owned()), None),
)),
request_id: request_id,
}
}
}
1 change: 1 addition & 0 deletions ofborg/src/message/buildresult.rs
Original file line number Diff line number Diff line change
@@ -7,6 +7,7 @@ pub struct BuildResult {
pub system: String,
pub output: Vec<String>,
pub attempt_id: String,
pub request_id: Option<String>,
pub success: Option<bool>,
pub skipped_attrs: Option<Vec<String>>,
pub attempted_attrs: Option<Vec<String>>,
55 changes: 36 additions & 19 deletions ofborg/src/nix.rs
Original file line number Diff line number Diff line change
@@ -6,6 +6,9 @@ use std::io::SeekFrom;
use std::path::Path;
use std::process::{Command, Stdio};
use tempfile::tempfile;
use std::io::BufReader;
use std::io::BufRead;
use ofborg::partition_result;

#[derive(Clone, Debug)]
pub enum Operation {
@@ -102,16 +105,23 @@ impl Nix {
nixpkgs: &Path,
file: &str,
attrs: Vec<String>,
) -> (Vec<String>, Vec<String>) {
attrs
) -> (Vec<String>, Vec<(String,Vec<String>)>) {
let attr_instantiations: Vec<Result<String, (String, Vec<String>)>> =
attrs
.into_iter()
.partition(|attr| {
self.safely_instantiate_attrs(
nixpkgs,
file,
vec![attr.clone()]
).is_ok()
})
.map(|attr|
match self.safely_instantiate_attrs(
nixpkgs,
file,
vec![attr.clone()]
) {
Ok(_) => Ok(attr.clone()),
Err(f) => Err((attr.clone(), lines_from_file(f)))
}
)
.collect();

partition_result(attr_instantiations)
}

pub fn safely_instantiate_attrs(
@@ -253,6 +263,15 @@ impl Nix {
}
}

fn lines_from_file(file: File) -> Vec<String> {
BufReader::new(file)
.lines()
.into_iter()
.filter(|line| line.is_ok())
.map(|line| line.unwrap())
.collect()
}

#[cfg(test)]
mod tests {
fn nix() -> Nix {
@@ -302,12 +321,7 @@ mod tests {
Err(file) => file,
};

let lines: Vec<String> = BufReader::new(file)
.lines()
.into_iter()
.filter(|line| line.is_ok())
.map(|line| line.unwrap())
.collect();
let lines = lines_from_file(file);

let buildlog = lines
.into_iter()
@@ -374,8 +388,6 @@ mod tests {
}

use super::*;
use std::io::BufReader;
use std::io::BufRead;
use std::path::PathBuf;
use std::env;

@@ -563,7 +575,7 @@ mod tests {
fn partition_instantiable_attributes() {
let nix = nix();

let ret: (Vec<String>, Vec<String>) = nix.safely_partition_instantiable_attrs(
let ret: (Vec<String>, Vec<(String, Vec<String>)>) = nix.safely_partition_instantiable_attrs(
individual_eval_path().as_path(),
"default.nix",
vec![
@@ -574,7 +586,12 @@ mod tests {
);

assert_eq!(ret.0, vec!["passes-instantiation"]);
assert_eq!(ret.1, vec!["fails-instantiation", "missing-attr"]);

assert_eq!(ret.1[0].0, "fails-instantiation");
assert_eq!(ret.1[0].1[0], "trace: You just can\'t frooble the frozz on this particular system.");

assert_eq!(ret.1[1].0, "missing-attr");
assert_eq!(ret.1[1].1[0], "error: attribute ‘missing-attr’ in selection path ‘missing-attr’ not found");
}

#[test]
Loading