jruby · Dec 1, 2016 · Dec 1, 2016 · Dec 1, 2016 · Dec 1, 2016 · Dec 1, 2016
Showing with 80 additions and 10 deletions.

+1 −0 .gitignore

+16 −10 ci.hocon

+63 −0 tool/truffle/fail-if-any-failed.rb
diff --git a/.gitignore b/.gitignore
@@ -23,6 +23,7 @@ Makefile
 cext-output.txt
 bench-results.json
 bench-results-processed.json
+failures
 
 .DS_Store
 .debug.properties

diff --git a/ci.hocon b/ci.hocon
@@ -271,18 +271,24 @@ bench: {
   ]
 }
 
+post-process-and-upload-results-wait: [
+  [tool/truffle/post-process-results-json.rb, bench-results.json, bench-results-processed.json],
+  [bench-uploader.py, bench-results-processed.json],
+  [tool/truffle/fail-if-any-failed.rb, bench-results-processed.json, --wait]
+]
+
 post-process-and-upload-results: [
   [tool/truffle/post-process-results-json.rb, bench-results.json, bench-results-processed.json],
-  [cat, bench-results-processed.json],
-  [bench-uploader.py, bench-results-processed.json]
+  [bench-uploader.py, bench-results-processed.json],
+  [tool/truffle/fail-if-any-failed.rb, bench-results-processed.json]
 ]
 
 metrics: {
   run: ${bench.setup} [
     ${bench.cmd} [allocation]
-  ] ${post-process-and-upload-results} [
+  ] ${post-process-and-upload-results-wait} [
     ${bench.cmd} [minheap]
-  ] ${post-process-and-upload-results} [
+  ] ${post-process-and-upload-results-wait} [
     ${bench.cmd} [time]
   ] ${post-process-and-upload-results}
   timelimit: "00:25:00"
@@ -291,9 +297,9 @@ metrics: {
 compiler-metrics: {
   run: ${bench.setup} [
     ${bench.cmd} ["allocation:compile-mandelbrot"]
-  ] ${post-process-and-upload-results} [
+  ] ${post-process-and-upload-results-wait} [
     ${bench.cmd} ["minheap:compile-mandelbrot"]
-  ] ${post-process-and-upload-results} [
+  ] ${post-process-and-upload-results-wait} [
     ${bench.cmd} ["time:compile-mandelbrot"]
   ] ${post-process-and-upload-results}
   timelimit: "00:50:00"
@@ -334,13 +340,13 @@ asciidoctor-benchmarks: {
 other-benchmarks: {
   run: ${bench.setup} [
     ${bench.cmd} [image-demo]
-  ] ${post-process-and-upload-results} [
+  ] ${post-process-and-upload-results-wait} [
     ${bench.cmd} [optcarrot]
-  ] ${post-process-and-upload-results} [
+  ] ${post-process-and-upload-results-wait} [
     ${bench.cmd} [savina]
-  ] ${post-process-and-upload-results} [
+  ] ${post-process-and-upload-results-wait} [
     ${bench.cmd} [synthetic]
-  ] ${post-process-and-upload-results} [
+  ] ${post-process-and-upload-results-wait} [
     ${bench.cmd} [micro]
   ] ${post-process-and-upload-results}
   timelimit: "00:40:00"

diff --git a/tool/truffle/fail-if-any-failed.rb b/tool/truffle/fail-if-any-failed.rb
@@ -0,0 +1,63 @@
+#!/usr/bin/env ruby
+# Copyright (c) 2016 Oracle and/or its affiliates. All rights reserved.
+# This code is released under a tri EPL/GPL/LGPL license. You can use it,
+# redistribute it and/or modify it under the terms of the:
+#
+# Eclipse Public License version 1.0
+# GNU General Public License version 2
+# GNU Lesser General Public License version 2.1
+
+require 'json'
+
+result = JSON.parse(File.read(ARGV[0]))
+wait = ARGV[1] == '--wait'
+
+failures = []
+
+known_failures = [
+  ["server", "graal-enterprise-no-om", "jruby", "truffle", "asciidoctor", "asciidoctor:load-file"],
+  ["server", "graal-enterprise-no-om", "jruby", "truffle", "server", "tcp-server"],
+  ["server", "graal-enterprise", "jruby", "truffle", "server", "tcp-server"],
+  ["server", "graal-enterprise", "jruby", "truffle", "asciidoctor", "asciidoctor:load-file"],
+  ["server", "graal-core", "jruby", "truffle", "server", "tcp-server"],
+  ["server", "graal-core", "jruby", "truffle", "asciidoctor", "asciidoctor:load-file"],
+  ["server", "graal-vm", "jruby", "truffle", "asciidoctor", "asciidoctor:load-file"],
+  ["server", "graal-vm", "jruby", "truffle", "server", "tcp-server"],
+  ["server", "graal-vm-snap", "jruby", "truffle", "server", "tcp-server"],
+  ["server", "graal-vm-snap", "jruby", "truffle", "asciidoctor", "asciidoctor:load-file"],
+  ["server", "graal-vm", "jruby", "truffle", "chunky", "chunky-decode-png-image-pass"],
+  ["server", "graal-vm-snap", "jruby", "truffle", "chunky", "chunky-decode-png-image-pass"]
+]
+
+if File.exist?('failures')
+  failures = File.read('failures').split("\n").map { |failure| eval(failure) }
+else
+  failures = []
+end
+
+(result['queries'] || []).any? do |q|
+  if q['extra.error'] == 'failed'
+    failures.push [q['host-vm'], q['host-vm-config'], q['guest-vm'], q['guest-vm-config'], q['bench-suite'], q['benchmark']]
+  end
+end
+
+known_failures.each do |known_failure|
+  if failures.delete(known_failure)
+    STDERR.puts "#{known_failure.inspect} failed, but we know about that"
+  end
+end
+
+if wait
+  if !failures.empty?
+    STDERR.puts 'waiting to return failure...'
+    File.write('failures', failures.map(&:inspect).join("\n"))
+  end
+else
+  if !failures.empty? || File.exist?('failures')
+    STDERR.puts 'these failed:'
+    failures.each do |failure|
+      STDERR.puts failure.inspect
+    end
+    exit 1
+  end
+end