Skip to content
Permalink

Comparing changes

Choose two branches to see what’s changed or to start a new pull request. If you need to, you can also or learn more about diff comparisons.

Open a pull request

Create a new pull request by comparing changes across two branches. If you need to, you can also . Learn more about diff comparisons here.
base repository: jruby/jruby
Failed to load repositories. Confirm that selected base ref is valid, then try again.
Loading
base: c8543eaec479
Choose a base ref
...
head repository: jruby/jruby
Failed to load repositories. Confirm that selected head ref is valid, then try again.
Loading
compare: 790a3f3dd5be
Choose a head ref
  • 2 commits
  • 5 files changed
  • 1 contributor

Commits on Mar 7, 2016

  1. Copy the full SHA
    74d3291 View commit details
  2. [Truffle] j+tr: Updates

    -   make option blocks avaliable in ci definition
    -   move --interpreter-path option to run
    -   ci command runs setup and run commands within the same process
        (ensures that it will execute on the same Ruby interpreter)
    -   ci definition can define its own options (see default)
    -   remove unused Ruby evaluation in Yaml
    -   ci command accepts --definition NAME option to use different ci defintion
        than default (or with different name than the one of the tested gem)
    -   default ci-definition takes option --version
    -   bundler helper method added to CIEnvironment
    -   bundler has to run it's own process (uses same ruby executable as the tool)
    pitr-ch committed Mar 7, 2016
    Copy the full SHA
    790a3f3 View commit details
3 changes: 2 additions & 1 deletion lib/ruby/truffle/jruby+truffle/bin/jruby+truffle
Original file line number Diff line number Diff line change
@@ -2,4 +2,5 @@

require_relative '../lib/runner.rb'

JRubyTruffleRunner.new
runner = JRubyTruffleRunner.new
exit runner.run
4 changes: 3 additions & 1 deletion lib/ruby/truffle/jruby+truffle/gem_ci/activesupport.rb
Original file line number Diff line number Diff line change
@@ -5,7 +5,9 @@
git_clone 'https://github.com/rails/rails.git', branch: '4-2-stable'
end

setup
use_only_https_git_paths!

has_to_succeed setup

result run(%w[--require-pattern test/**/*_test.rb -r exclude_tests -- -I test -e nil], raise: false)

11 changes: 9 additions & 2 deletions lib/ruby/truffle/jruby+truffle/gem_ci/default.rb
Original file line number Diff line number Diff line change
@@ -1,8 +1,15 @@
declare_options git: ['--git URL', 'Path to the gem\'s repository', STORE_NEW_VALUE, nil],
version: ['--version VERSION', 'Version of the gem', STORE_NEW_VALUE, nil]

unless File.exists? repository_dir
git_clone option(:git)
git_clone option(:git),
tag: get_git_tag(option(:version))
end

setup
delete_gemfile_lock!
use_only_https_git_paths!

has_to_succeed setup

result run(%w[-S rake], raise: false)

Original file line number Diff line number Diff line change
@@ -1,12 +1,4 @@
:setup:
:file:
"bundler/gem_tasks.rb": nil
:stored_commands:
:ci:
- :setup
- :test
:setup:
- "git clone git@github.com:lucasocon/openweather.git"
- "jruby+truffle --dir openweather setup"
:test: "jruby+truffle --dir openweather run --require-pattern 'test/*_test.rb' -I test -- -e nil"

315 changes: 197 additions & 118 deletions lib/ruby/truffle/jruby+truffle/lib/runner.rb
Original file line number Diff line number Diff line change
@@ -12,14 +12,8 @@
require 'fileutils'
require 'shellwords'
require 'pathname'

require 'rbconfig'
require 'rubygems'
begin
require 'bundler'
rescue LoadError => e
puts "Bundler has to be installed.\n"
raise e
end

class JRubyTruffleRunner
module Utils
@@ -78,12 +72,17 @@ def print_cmd(cmd, dir, print)
ROOT = Pathname(__FILE__).dirname.parent.expand_path
JRUBY_PATH = ROOT.join('../../../..').expand_path

module OptionBlocks
STORE_NEW_VALUE = -> (new, old, _) { new }
STORE_NEW_NEGATED_VALUE = -> (new, old, _) { !new }
ADD_TO_ARRAY = -> (new, old, _) { old << new }
MERGE_TO_HASH = -> ((k, v), old, _) { old.merge k => v }
end

include OptionBlocks

begin
assign_new_value = -> (new, old, _) { new }
assign_new_negated_value = -> (new, old, _) { !new }
add_to_array = -> (new, old, _) { old << new }
merge_hash = -> ((k, v), old, _) { old.merge k => v }
apply_pattern = -> (pattern, old, options) do
apply_pattern = -> (pattern, old, options) do
Dir.glob(pattern) do |file|
if options[:exclude_pattern].any? { |p| /#{p}/ =~ file }
puts "skipped: #{file}" if verbose?
@@ -100,57 +99,57 @@ def print_cmd(cmd, dir, print)
# -> (new_value, old_value) { result_of_this_block_is_stored },
# default_value]
# }
OPTION_DEFINITIONS = {
OPTION_DEFINITIONS = {
global: {
verbose: ['-v', '--verbose', 'Run verbosely (prints options)', assign_new_value, false],
help: ['-h', '--help', 'Show this message', assign_new_value, false],
debug_port: ['--debug-port PORT', 'Debug port', assign_new_value, '51819'],
debug_option: ['--debug-option OPTION', 'Debug JVM option', assign_new_value,
verbose: ['-v', '--verbose', 'Run verbosely (prints options)', STORE_NEW_VALUE, false],
help: ['-h', '--help', 'Show this message', STORE_NEW_VALUE, false],
debug_port: ['--debug-port PORT', 'Debug port', STORE_NEW_VALUE, '51819'],
debug_option: ['--debug-option OPTION', 'Debug JVM option', STORE_NEW_VALUE,
'-J-agentlib:jdwp=transport=dt_socket,server=y,address=%d,suspend=y'],
truffle_bundle_path: ['--truffle-bundle-path NAME', 'Bundle path', assign_new_value, '.jruby+truffle_bundle'],
interpreter_path: ['--interpreter-path PATH', "Path to #{BRANDING} interpreter executable", assign_new_value,
JRUBY_PATH.join('bin', 'jruby')],
graal_path: ['--graal-path PATH', 'Path to Graal', assign_new_value, (JRUBY_PATH + '../GraalVM-0.10/jre/bin/javao').to_s],
truffle_bundle_path: ['--truffle-bundle-path NAME', 'Bundle path', STORE_NEW_VALUE, '.jruby+truffle_bundle'],
graal_path: ['--graal-path PATH', 'Path to Graal', STORE_NEW_VALUE, (JRUBY_PATH + '../GraalVM-0.10/jre/bin/javao').to_s],
mock_load_path: ['--mock-load-path PATH',
'Path of mocks & monkey-patches (prepended in $:, relative to --truffle_bundle_path)',
assign_new_value, 'mocks'],
STORE_NEW_VALUE, 'mocks'],
use_fs_core: ['--[no-]use-fs-core', 'Use core from the filesystem rather than the JAR',
assign_new_value, true],
bundle_options: ['--bundle-options OPTIONS', 'bundle options separated by space', assign_new_value, ''],
configuration: ['--config GEM_NAME', 'Load configuration for specified gem', assign_new_value, nil],
dir: ['--dir DIRECTORY', 'Set working directory', assign_new_value, Dir.pwd],
STORE_NEW_VALUE, true],
bundle_options: ['--bundle-options OPTIONS', 'bundle options separated by space', STORE_NEW_VALUE, ''],
configuration: ['--config GEM_NAME', 'Load configuration for specified gem', STORE_NEW_VALUE, nil],
dir: ['--dir DIRECTORY', 'Set working directory', STORE_NEW_VALUE, nil],
},
setup: {
help: ['-h', '--help', 'Show this message', assign_new_value, false],
after: ['--after SH_CMD', 'Commands to execute after setup', add_to_array, []],
file: ['--file NAME,CONTENT', Array, 'Create file in truffle_bundle_path', merge_hash, {}],
without: ['--without GROUP', 'Do not install listed gem group by bundler', add_to_array, []]
help: ['-h', '--help', 'Show this message', STORE_NEW_VALUE, false],
after: ['--after SH_CMD', 'Commands to execute after setup', ADD_TO_ARRAY, []],
file: ['--file NAME,CONTENT', Array, 'Create file in truffle_bundle_path', MERGE_TO_HASH, {}],
without: ['--without GROUP', 'Do not install listed gem group by bundler', ADD_TO_ARRAY, []]
},
run: {
help: ['-h', '--help', 'Show this message', assign_new_value, false],
no_truffle: ['-n', '--no-truffle', "Use conventional JRuby instead of #{BRANDING}", assign_new_negated_value, false],
graal: ['-g', '--graal', 'Run on graal', assign_new_value, false],
build: ['-b', '--build', 'Run `jt build` using conventional JRuby', assign_new_value, false],
rebuild: ['--rebuild', 'Run `jt rebuild` using conventional JRuby', assign_new_value, false],
debug: ['-d', '--debug', 'JVM remote debugging', assign_new_value, false],
require: ['-r', '--require FILE', 'Files to require, same as Ruby\'s -r', add_to_array, []],
require_pattern: ['--require-pattern DIR_GLOB_PATTERN', 'Files matching the pattern will be required', apply_pattern, []],
exclude_pattern: ['--exclude-pattern REGEXP', 'Files matching the regexp will not be required by --require-pattern (applies to subsequent --require-pattern options)', add_to_array, []],
load_path: ['-I', '--load-path LOAD_PATH', 'Paths to add to load path, same as Ruby\'s -I', add_to_array, []],
executable: ['-S', '--executable NAME', 'finds and runs an executable of a gem', assign_new_value, nil],
jexception: ['--jexception', 'print Java exceptions', assign_new_value, false]
help: ['-h', '--help', 'Show this message', STORE_NEW_VALUE, false],
interpreter_path: ['--interpreter-path PATH', "Path to #{BRANDING} interpreter executable", STORE_NEW_VALUE,
JRUBY_PATH.join('bin', 'jruby')],
no_truffle: ['-n', '--no-truffle', "Use conventional JRuby instead of #{BRANDING}", STORE_NEW_NEGATED_VALUE, false],
graal: ['-g', '--graal', 'Run on graal', STORE_NEW_VALUE, false],
build: ['-b', '--build', 'Run `jt build` using conventional JRuby', STORE_NEW_VALUE, false],
rebuild: ['--rebuild', 'Run `jt rebuild` using conventional JRuby', STORE_NEW_VALUE, false],
debug: ['-d', '--debug', 'JVM remote debugging', STORE_NEW_VALUE, false],
require: ['-r', '--require FILE', 'Files to require, same as Ruby\'s -r', ADD_TO_ARRAY, []],
require_pattern: ['--require-pattern DIR_GLOB_PATTERN', 'Files matching the pattern will be required', apply_pattern, []],
exclude_pattern: ['--exclude-pattern REGEXP', 'Files matching the regexp will not be required by --require-pattern (applies to subsequent --require-pattern options)', ADD_TO_ARRAY, []],
load_path: ['-I', '--load-path LOAD_PATH', 'Paths to add to load path, same as Ruby\'s -I', ADD_TO_ARRAY, []],
executable: ['-S', '--executable NAME', 'finds and runs an executable of a gem', STORE_NEW_VALUE, nil],
jexception: ['--jexception', 'print Java exceptions', STORE_NEW_VALUE, false]
},
ci: {
git: ['--git URL', 'Path to the gem\'s repository', assign_new_value, nil],
batch: ['--batch FILE', 'Run batch of ci tests supplied in a file. One ci command options per line. If FILE is in or stdin it reads from $stdin.',
assign_new_value, nil],
help: ['-h', '--help', 'Show this message', assign_new_value, false]
batch: ['--batch FILE', 'Run batch of ci tests supplied in a file. One ci command options per line. If FILE is in or stdin it reads from $stdin.',
STORE_NEW_VALUE, nil],
definition: ['--definition NAME', 'Specify which definition file to use', STORE_NEW_VALUE, nil],
help: ['-h', '--help', 'Show this message', STORE_NEW_VALUE, false]
},
clean: {
help: ['-h', '--help', 'Show this message', assign_new_value, false]
help: ['-h', '--help', 'Show this message', STORE_NEW_VALUE, false]
},
readme: {
help: ['-h', '--help', 'Show this message', assign_new_value, false]
help: ['-h', '--help', 'Show this message', STORE_NEW_VALUE, false]
}
}
end
@@ -205,7 +204,7 @@ def print_cmd(cmd, dir, print)

ci_help = <<-TXT.gsub(/^ {6}/, '')
Usage: #{EXECUTABLE} [options] ci [subcommand-options] GEM_NAME
Usage: #{EXECUTABLE} [options] ci [subcommand-options] GEM_NAME [options-declared-in-CI-definition]
Runs CI tests for predefined gems or it uses default CI definition. CI Definitions
are stored in gem_ci directory. The CI definition files are evaluated in CIEnvironment
@@ -216,8 +215,12 @@ def print_cmd(cmd, dir, print)
Examples: #{EXECUTABLE} ci activesupport
(Runs gem_ci/activesupport.rb CI definition)
#{EXECUTABLE} ci --git https://github.com/ruby-concurrency/concurrent-ruby.git concurrent-ruby
(Runs gem_ci/default.rb CI definition which uses git option)
#{EXECUTABLE} ci concurrent-ruby --git https://github.com/ruby-concurrency/concurrent-ruby.git
(Runs gem_ci/default.rb CI definition which declared and uses git option)
#{EXECUTABLE} ci --help
(Shows options of ci command)
#{EXECUTABLE} ci concurrent-ruby --help
(Shows options defined by ci definition for concurrent-ruby)
TXT

@@ -226,32 +229,36 @@ def print_cmd(cmd, dir, print)


def initialize(argv = ARGV)
construct_default_options
build_option_parsers
@options = construct_default_options
@option_parsers = build_option_parsers

subcommand, *argv_after_global = @option_parsers[:global].order argv
@subcommand, *argv_after_global = @option_parsers[:global].order argv

Dir.chdir @options[:global][:dir] do
Dir.chdir dir do
puts "pwd: #{Dir.pwd}" if verbose?

load_gem_configuration
load_local_configuration

if subcommand.nil?
if @subcommand.nil?
print_options
help :global
end
help :global if @options[:global][:help]

subcommand = subcommand.to_sym
@subcommand = @subcommand.to_sym

subcommand_option_parser = @option_parsers[subcommand] || raise("unknown subcommand: #{subcommand}")
argv_after_subcommand = subcommand_option_parser.order argv_after_global
subcommand_option_parser = @option_parsers[@subcommand] || raise("unknown subcommand: #{@subcommand}")
@argv_after_subcommand = subcommand_option_parser.order argv_after_global

print_options
help subcommand if @options[subcommand][:help] && subcommand != :readme
help @subcommand if @options[@subcommand][:help] && @subcommand != :readme
end
end

send "subcommand_#{subcommand}", argv_after_subcommand
def run
Dir.chdir dir do
send "subcommand_#{@subcommand}", @argv_after_subcommand
end
end

@@ -264,29 +271,37 @@ def print_options

private

def dir
@options[:global][:dir] || Dir.pwd
end

def verbose?
@options[:global][:verbose]
end

def build_option_parsers
@option_parsers = OPTION_DEFINITIONS.each_with_object({}) do |(name, parser_options), parsers|
option_parsers = OPTION_DEFINITIONS.each_with_object({}) do |(name, parser_options), parsers|
parsers[name] = build_option_parser(parser_options, @options.fetch(name))
end

@option_parsers.each { |key, option_parser| option_parser.banner = HELP[key] }
option_parsers.each { |key, option_parser| option_parser.banner = HELP[key] }
end

def build_option_parser(parser_options, options_hash)
OptionParser.new do |option_parser|
parser_options.each do |option, data|
*args, description, block, default = data
def self.build_option_parser(parser_options, options_hash, option_parser: OptionParser.new)
parser_options.each do |option, data|
*args, description, block, default = data

option_parser.on(*args, description + " (default: #{default.inspect})") do |new_value|
old_value = options_hash[option]
options_hash[option] = instance_exec new_value, old_value, options_hash, &block
end
option_parser.on(*args, description + " (default: #{default.inspect})") do |new_value|
old_value = options_hash[option]
options_hash[option] = instance_exec new_value, old_value, options_hash, &block
end
end

option_parser
end

def build_option_parser(parser_options, options_hash)
self.class.build_option_parser(parser_options, options_hash)
end

def load_gem_configuration
@@ -314,14 +329,22 @@ def apply_yaml_to_configuration(yaml_path)
end

def construct_default_options
@options = OPTION_DEFINITIONS.each_with_object({}) do |(group, group_options), options|
group_options.each_with_object(options[group] = {}) do |(option, data), group_option_defaults|
*args, block, default = data
group_option_defaults[option] = default
end
OPTION_DEFINITIONS.each_with_object({}) do |(group, group_options), options|
options[group] = default_option_values(group_options)
end
end

def self.default_option_values(group_options)
group_options.each_with_object({}) do |(option, data), group_option_defaults|
*args, block, default = data
group_option_defaults[option] = default
end
end

def default_option_values(group_options)
self.class.default_option_values(group_options)
end

def help(key = nil)
parsers = if key
[@option_parsers[key]]
@@ -343,26 +366,13 @@ def deep_merge(a, b)

if Array === a
if Array === b
return a.concat b.map { |v| eval_yaml_strings v }
return a.concat b
else
return a
end
end

eval_yaml_strings b
end

def eval_yaml_strings(value)
if String === value
begin
eval('"' + value.gsub(/\\#|"|\\/, '\#' => '\#', '"' => '\"', '\\' => '\\\\') + '"')
rescue => e
p value
raise e
end
else
value
end
b
end

def subcommand_setup(rest)
@@ -401,18 +411,21 @@ def subcommand_setup(rest)
@options[:setup][:after].each do |cmd|
execute_cmd cmd
end

true
rescue => e
puts format('%s: %s\n%s', e.class, e.message, e.backtrace.join("\n"))
false
end

def bundle_cli(argv)
require 'bundler/friendly_errors'
Bundler.with_friendly_errors do
require 'bundler/cli'
Bundler::CLI.start(argv, :debug => true)
end
ruby = Pathname(RbConfig::CONFIG['bindir']).join('ruby')
execute_cmd [ruby.to_s, "#{Gem.bindir}/bundle", *argv]
end

def subcommand_run(rest)
jruby_path = Pathname("#{@options[:global][:interpreter_path]}/../..").expand_path
jruby_path = Pathname("#{@options[:run][:interpreter_path]}/../..").expand_path
raise unless jruby_path.absolute?
ruby_options, rest = if rest.include?('--')
split = rest.index('--')
[rest[0...split], rest[(split+1)..-1]]
@@ -424,8 +437,6 @@ def subcommand_run(rest)
jruby_path = jruby_path.relative_path_from(Pathname('.'))
end

jruby_path = jruby_path.to_s

if @options[:run][:build] || @options[:run][:rebuild]
Dir.chdir jruby_path do
execute_cmd "./tool/jt.rb #{'re' if @options[:run][:rebuild]}build"
@@ -438,7 +449,7 @@ def subcommand_run(rest)
raise "no executable with name '#{@options[:run][:executable]}' found"
end

core_load_path = "#{jruby_path}/truffle/src/main/ruby"
core_load_path = jruby_path.join 'truffle/src/main/ruby'

truffle_options = [
('-X+T'),
@@ -459,22 +470,24 @@ def subcommand_run(rest)
env['JAVACMD'] = @options[:global][:graal_path] if @options[:run][:graal]

cmd = [(env unless env.empty?),
@options[:global][:interpreter_path].to_s,
@options[:run][:interpreter_path].to_s,
*cmd_options,
executable,
*rest
].compact

execute_cmd(cmd, raise: false, print_always: true)
exit $?.exitstatus
$?.success?
end

def subcommand_clean(rest)
FileUtils.rm_rf @options[:global][:truffle_bundle_path]
true
end

def subcommand_readme(rest)
puts File.read(ROOT.join('README.md'))
true
end

def subcommand_ci(rest)
@@ -493,13 +506,14 @@ def subcommand_ci(rest)
rest = option_parser.order line.split

gem_name = rest.first
CIEnvironment.new(options, self.options[:global][:dir], gem_name).success?
CIEnvironment.new(dir, gem_name, rest[1..-1]).success?
end

exit results.all? ? 0 : 1
results.all?
else
gem_name = rest.first
CIEnvironment.new options[:ci], options[:global][:dir], gem_name
ci = CIEnvironment.new dir, gem_name, rest[1..-1], definition: options[:ci][:definition]
ci.success?
end
end

@@ -509,6 +523,7 @@ def execute_cmd(cmd, dir: nil, raise: true, print_always: false)

class CIEnvironment
include Utils
include OptionBlocks

def self.define_dsl_attr(*names, &conversion)
nothing = Object.new
@@ -531,20 +546,56 @@ def self.define_dsl_attr(*names, &conversion)
define_dsl_attr(:working_dir) { |v| Pathname(v) }
attr_reader :gem_name

def initialize(options, working_dir, gem_name)
@options = options
def initialize(working_dir, gem_name, rest, definition: nil)
@options = {}
@gem_name = gem_name

ci_file = Dir.glob(ROOT.join('gem_ci', "{#{@gem_name},default}.rb")).first

puts "Running #{ci_file}"
@rest = rest

@working_dir = Pathname(working_dir)
@repository_name = gem_name # TODO (pitr-ch 11-Feb-2016): gem name?
@repository_name = gem_name
@subdir = '.'
@result = nil

instance_eval File.read(ci_file), ci_file, 1
option_parser = @option_parser = OptionParser.new
@option_parser.banner = "\nUsage: #{EXECUTABLE} [options] ci [subcommand-options] #{gem_name} [options-declared-in-CI-definition]\n\n"
@option_parsed = false

declare_options parse_options: false, help: ['-h', '--help', 'Show this message', -> (new, old, _) { puts option_parser; exit }, false]
if definition
do_definition(definition)
else
do_definition(gem_name, raise: false) || do_definition('default')
end
end

def do_definition(name, raise: true)
ci_file = Dir.glob(ROOT.join('gem_ci', "{#{name}}.rb")).first
if ci_file.nil?
if raise
raise "no ci definition with name: #{name}"
else
return false
end
else
puts "Using: #{ci_file}"
catch :cancel_ci! do
instance_eval File.read(ci_file), ci_file, 1
end
end
end

def declare_options(parse_options: true, **parser_options)
raise 'cannot declare options after they were parsed' if @option_parsed

JRubyTruffleRunner.build_option_parser(parser_options, @options, option_parser: @option_parser)
@options.merge! JRubyTruffleRunner.default_option_values(parser_options)

if !@option_parsed && parse_options
@option_parsed = true
@option_parser.order @rest
end

nil
end

def repository_dir
@@ -567,18 +618,32 @@ def option(key)
@options.fetch(key)
end

def git_clone(url, branch: 'master')
def git_clone(url, branch: 'master', tag: nil, commit: nil)
execute_cmd "git clone -b #{branch} #{url} #{repository_name}", dir: working_dir, print: true
execute_cmd "git checkout #{tag || commit}", dir: working_dir, print: true if tag || commit
end

def get_git_tag(version)
tag = `git tags -l`.lines.find { |l| l.include? version }
raise "fetching tag for #{version} version failed" unless $?.success?
tag
end

def setup
execute_cmd [jruby_truffle_path.to_s, 'setup'], dir: testing_dir, print: true
Dir.chdir(testing_dir) { JRubyTruffleRunner.new(['setup']).run }
end

def cancel_ci!(result = false)
throw cancel_ci!, result
end

def has_to_succeed(result)
result or cancel_ci! result
end

def run(options, raise: true)
cmd = [jruby_truffle_path.to_s, 'run', *options]
cmd = cmd.join ' ' if options.is_a? String
execute_cmd cmd, dir: testing_dir, print: true, raise: raise
raise ArgumentError unless options.is_a? Array
Dir.chdir(testing_dir) { JRubyTruffleRunner.new(['run', *options]).run }
end

def execute(cmd, dir: testing_dir, raise: true)
@@ -597,5 +662,19 @@ def success?
@result
end

def use_only_https_git_paths!
gemfile_path = repository_dir.join('Gemfile')
File.write gemfile_path,
File.read(gemfile_path).
gsub(/github: ("|')/, 'git: \1https://github.com/').
gsub(/git:\/\//, 'https://')

end

def delete_gemfile_lock!
path = repository_dir.join('Gemfile.lock')
FileUtils.rm path if File.exists? path
end

end
end