Skip to content
Permalink

Comparing changes

Choose two branches to see what’s changed or to start a new pull request. If you need to, you can also or learn more about diff comparisons.

Open a pull request

Create a new pull request by comparing changes across two branches. If you need to, you can also . Learn more about diff comparisons here.
base repository: jruby/jruby
Failed to load repositories. Confirm that selected base ref is valid, then try again.
Loading
base: 3fc33531a9b0
Choose a base ref
...
head repository: jruby/jruby
Failed to load repositories. Confirm that selected head ref is valid, then try again.
Loading
compare: 63fc91fcaeb5
Choose a head ref
  • 10 commits
  • 2 files changed
  • 1 contributor

Commits on Jul 29, 2016

  1. [Truffle] Cleanup, refactor and improve JRuby build with mx.

    * mx build when already built takes 2s now.
    * Clearer dependencies and less dirty tricks.
    eregon committed Jul 29, 2016
    Copy the full SHA
    d77a9d7 View commit details
  2. Copy the full SHA
    50924c7 View commit details
  3. [Truffle] Mx: Implement mx ruby.

    * Add JRuby+Truffle Ruby core files to the jar.
    * Extract RUBY-ZIP in mxbuild.
    eregon committed Jul 29, 2016
    Copy the full SHA
    72c724f View commit details
  4. Copy the full SHA
    dfc5280 View commit details
  5. [Truffle] Mx: no need to install truffle in the local Maven repo.

    * Since it's built directly with mx.
    eregon committed Jul 29, 2016
    Copy the full SHA
    8c40733 View commit details
  6. Copy the full SHA
    0e5002e View commit details
  7. Copy the full SHA
    53ebf6b View commit details
  8. Copy the full SHA
    9aff41f View commit details
  9. Copy the full SHA
    0343095 View commit details
  10. Copy the full SHA
    63fc91f View commit details
Showing with 310 additions and 223 deletions.
  1. +210 −171 mx.jruby/mx_jruby.py
  2. +100 −52 mx.jruby/suite.py
381 changes: 210 additions & 171 deletions mx.jruby/mx_jruby.py
Original file line number Diff line number Diff line change
@@ -13,227 +13,266 @@
import shutil
import json
import time
import tarfile
import zipfile
from threading import Thread
from os.path import join, exists

import mx
import mx_benchmark

_suite = mx.suite('jruby')

def jt(args, suite=None, nonZeroIsFatal=True, out=None, err=None, timeout=None, env=None, cwd=None):
rubyDir = _suite.dir
jt = os.path.join(rubyDir, 'tool', 'jt.rb')
return mx.run(['ruby', jt] + args, nonZeroIsFatal=nonZeroIsFatal, out=out, err=err, timeout=timeout, env=env, cwd=cwd)

FNULL = open(os.devnull, 'w')

class BackgroundServerTask:
def __init__(self, args):
self.args = args

def __enter__(self):
preexec_fn, creationflags = mx._get_new_progress_group_args()
if mx._opts.verbose:
mx.log(' '.join(['(background)'] + map(pipes.quote, self.args)))
self.process = subprocess.Popen(self.args, preexec_fn=preexec_fn, creationflags=creationflags, stdout=FNULL, stderr=FNULL)
mx._addSubprocess(self.process, self.args)

def __exit__(self, type, value, traceback):
self.process.kill()
self.process.wait()

def is_running(self):
return self.process.poll() is None
TimeStampFile = mx.TimeStampFile

class BackgroundJT(BackgroundServerTask):
def __init__(self, args):
rubyDir = _suite.dir
jt = os.path.join(rubyDir, 'tool', 'jt.rb')
BackgroundServerTask.__init__(self, ['ruby', jt] + args)
# Project and BuildTask classes

class MavenProject(mx.Project):
class ArchiveProject(mx.ArchivableProject):
def __init__(self, suite, name, deps, workingSets, theLicense, **args):
mx.Project.__init__(self, suite, name, "", [], deps, workingSets, _suite.dir, theLicense)
self.javaCompliance = "1.7"
self.build = hasattr(args, 'build')
self.prefix = args['prefix']

def source_dirs(self):
return []
mx.ArchivableProject.__init__(self, suite, name, deps, workingSets, theLicense)
assert 'prefix' in args
assert 'outputDir' in args

def output_dir(self, relative=False):
dir = os.path.join(_suite.dir, self.prefix)
return dir.rstrip('/')
def output_dir(self):
return join(self.dir, self.outputDir)

def source_gen_dir(self):
return None

def getOutput(self, replaceVar=False):
return os.path.join(_suite.dir, "target")

def getResults(self, replaceVar=False):
return mx.Project.getResults(self, replaceVar=replaceVar)
def archive_prefix(self):
return self.prefix

def getBuildTask(self, args):
return MavenBuildTask(self, args, None, None)
def getResults(self):
return mx.ArchivableProject.walk(self.output_dir())

def isJavaProject(self):
return True
class LicensesProject(ArchiveProject):
license_files = ['BSDL', 'COPYING', 'LICENSE.RUBY']

def archive_prefix(self):
return self.prefix
def getResults(self):
return [join(_suite.dir, f) for f in self.license_files]

def annotation_processors(self):
return []
class AntlrProject(mx.Project):
def __init__(self, suite, name, deps, workingSets, theLicense, **args):
mx.Project.__init__(self, suite, name, "", [], deps, workingSets, _suite.dir, theLicense)
assert 'outputDir' in args
assert 'sourceDir' in args
assert 'grammars' in args
self.doNotArchive = True

def find_classes_with_matching_source_line(self, pkgRoot, function, includeInnerClasses=False):
return dict()
def getBuildTask(self, args):
return AntlrBuildTask(self, args)

class MavenBuildTask(mx.BuildTask):
def __init__(self, project, args, vmbuild, vm):
class AntlrBuildTask(mx.BuildTask):
def __init__(self, project, args):
mx.BuildTask.__init__(self, project, args, 1)
self.vm = vm
self.vmbuild = vmbuild

def __str__(self):
return 'Building Maven for {}'.format(self.subject)
return 'Generate Antlr grammar for {}'.format(self.subject)

def needsBuild(self, newestInput):
return (True, 'Let us re-build everytime')
sup = mx.BuildTask.needsBuild(self, newestInput)
if sup[0]:
return sup
newestOutput = self.newestOutput()
if not newestOutput:
return (True, "no class files yet")
src = join(_suite.dir, self.subject.sourceDir)
for root, _, files in os.walk(src):
for name in files:
file = TimeStampFile(join(root, name))
if file.isNewerThan(newestOutput):
return (True, file)
return (False, 'all files are up to date')

def newestOutput(self):
return None
out = join(_suite.dir, self.subject.outputDir)
newest = None
for root, _, files in os.walk(out):
for name in files:
file = TimeStampFile(join(root, name))
if not newest or file.isNewerThan(newest):
newest = file
return newest

def build(self):
if not self.subject.build:
mx.log("...skip build of {}".format(self.subject))
return
mx.log('...perform build of {}'.format(self.subject))
antlr4 = None
for lib in _suite.libs:
if lib.name == "ANTLR4_MAIN":
antlr4 = lib
assert antlr4

antlr4jar = antlr4.classpath_repr()
out = join(_suite.dir, self.subject.outputDir)
src = join(_suite.dir, self.subject.sourceDir)
for grammar in self.subject.grammars:
pkg = os.path.dirname(grammar).replace('/', '.')
mx.run_java(['-jar', antlr4jar, '-o', out, '-package', pkg, grammar], cwd=src)

rubyDir = _suite.dir
mavenDir = os.path.join(rubyDir, 'mxbuild', 'mvn')
def clean(self, forBuild=False):
pass

def mavenSetup():
mavenDir = join(_suite.dir, 'mxbuild', 'mvn')
maven_repo_arg = '-Dmaven.repo.local=' + mavenDir
env = os.environ.copy()
env['JRUBY_BUILD_MORE_QUIET'] = 'true'
# HACK: since the maven executable plugin does not configure the
# java executable that is used we unfortunately need to append it to the PATH
javaHome = os.getenv('JAVA_HOME')
if javaHome:
env["PATH"] = javaHome + '/bin' + os.pathsep + env["PATH"]
mx.logv('Setting PATH to {}'.format(os.environ["PATH"]))
mx.run(['java', '-version'])
return maven_repo_arg, env

# HACK: since the maven executable plugin does not configure the
# java executable that is used we unfortunately need to append it to the PATH
javaHome = os.getenv('JAVA_HOME')
if javaHome:
os.environ["PATH"] = os.environ["JAVA_HOME"] + '/bin' + os.pathsep + os.environ["PATH"]
class JRubyCoreMavenProject(mx.MavenProject):
def getBuildTask(self, args):
return MavenBuildTask(self, args, 1)

mx.logv('Setting PATH to {}'.format(os.environ["PATH"]))
mx.logv('Calling java -version')
mx.run(['java', '-version'])
def getResults(self):
return None

class MavenBuildTask(mx.BuildTask):
def __str__(self):
return 'Building {} with Maven'.format(self.subject)

# Truffle version
def newestOutput(self):
return TimeStampFile(join(_suite.dir, self.subject.jar))

truffle = mx.suite('truffle')
truffle_commit = truffle.vc.parent(truffle.dir)
maven_version_arg = '-Dtruffle.version=' + truffle_commit
maven_repo_arg = '-Dmaven.repo.local=' + mavenDir

mx.run_mx(['maven-install', '--repo', mavenDir, '--only', 'TRUFFLE_API,TRUFFLE_DEBUG,TRUFFLE_DSL_PROCESSOR,TRUFFLE_TCK'], suite=truffle)
def needsBuild(self, newestInput):
sup = mx.BuildTask.needsBuild(self, newestInput)
if sup[0]:
return sup

open(os.path.join(rubyDir, 'VERSION'), 'w').write('graal-vm\n')
jar = self.newestOutput()

# Build jruby-truffle

env = os.environ.copy()
env['JRUBY_BUILD_MORE_QUIET'] = 'true'
if not jar.exists():
return (True, 'no jar yet')

mx.run_maven(['-q', '--version', maven_repo_arg], nonZeroIsFatal=False, cwd=rubyDir, env=env)
jni_libs = join(_suite.dir, 'lib/jni')
if not exists(jni_libs) or not os.listdir(jni_libs):
return (True, jni_libs)

mx.log('Building without tests')
bundler = join(_suite.dir, 'lib/ruby/gems/shared/gems/bundler-1.10.6')
if not exists(bundler) or not os.listdir(bundler):
return (True, bundler)

mx.run_maven(['-q', '-DskipTests', maven_version_arg, maven_repo_arg], cwd=rubyDir, env=env)
for watched in self.subject.watch:
watched = join(_suite.dir, watched)
if not exists(watched):
return (True, watched + ' does not exist')
elif os.path.isfile(watched) and TimeStampFile(watched).isNewerThan(jar):
return (True, watched + ' is newer than the jar')
else:
for root, _, files in os.walk(watched):
for name in files:
source = join(root, name)
if TimeStampFile(source).isNewerThan(jar):
return (True, source + ' is newer than the jar')

mx.log('Building complete version')
return (False, 'all files are up to date')

mx.run_maven(['-q', '-Pcomplete', '-DskipTests', maven_version_arg, maven_repo_arg], cwd=rubyDir, env=env)
mx.run(['zip', '-d', 'maven/jruby-complete/target/jruby-complete-graal-vm.jar', 'META-INF/jruby.home/lib/*'], cwd=rubyDir)
mx.run(['bin/jruby', 'bin/gem', 'install', 'bundler', '-v', '1.10.6'], cwd=rubyDir)

mx.log('...finished build of {}'.format(self.subject))
def build(self):
cwd = _suite.dir
maven_repo_arg, env = mavenSetup()
mx.log("Building jruby-core with Maven")
mx.run_maven(['-q', '-DskipTests', maven_repo_arg, '-pl', 'core,lib'], cwd=cwd, env=env)
# Install Bundler
gem_home = join(_suite.dir, 'lib', 'ruby', 'gems', 'shared')
env['GEM_HOME'] = gem_home
env['GEM_PATH'] = gem_home
mx.run(['bin/jruby', 'bin/gem', 'install', 'bundler', '-v', '1.10.6'], cwd=cwd, env=env)

def clean(self, forBuild=False):
if forBuild:
return
rubyDir = _suite.dir
mx.run_maven(['-q', 'clean'], nonZeroIsFatal=False, cwd=rubyDir)

class LicensesProject(mx.Project):
def __init__(self, suite, name, deps, workingSets, theLicense, **args):
mx.Project.__init__(self, suite, name, "", [], deps, workingSets, _suite.dir, theLicense)
self.javaCompliance = "1.7"
self.build = hasattr(args, 'build')
self.prefix = args['prefix']

def source_dirs(self):
return []

def output_dir(self, relative=False):
dir = os.path.join(_suite.dir, self.prefix)
return dir.rstrip('/')

def source_gen_dir(self):
return None

def getOutput(self, replaceVar=False):
return os.path.join(_suite.dir, "target")

def getResults(self, replaceVar=False):
return mx.Project.getResults(self, replaceVar=replaceVar)

def getBuildTask(self, args):
return LicensesBuildTask(self, args, None, None)

def isJavaProject(self):
return True

def archive_prefix(self):
return self.prefix

def annotation_processors(self):
return []
mx.run_maven(['-q', 'clean'], nonZeroIsFatal=False, cwd=_suite.dir)
jar = self.newestOutput()
if jar.exists():
os.remove(jar.path)

# Commands

def extractArguments(args):
vmArgs = []
rubyArgs = []
for i in range(len(args)):
arg = args[i]
if arg.startswith('-J-'):
vmArgs.append(arg[2:])
elif arg.startswith('-X'):
vmArgs.append('-Djruby.' + arg[2:])
else:
rubyArgs.append(arg)
return vmArgs, rubyArgs

def extractTarball(file, target_dir):
if file.endswith('tar'):
with tarfile.open(file, 'r:') as tf:
tf.extractall(target_dir)
elif file.endswith('jar') or file.endswith('zip'):
with zipfile.ZipFile(file, "r") as zf:
zf.extractall(target_dir)
else:
mx.abort('Unsupported compressed file ' + file)

def setup_jruby_home():
rubyZip = mx.distribution('RUBY-ZIP').path
assert exists(rubyZip)
extractPath = join(_suite.dir, 'mxbuild', 'ruby-zip-extracted')
if TimeStampFile(extractPath).isOlderThan(rubyZip):
if exists(extractPath):
shutil.rmtree(extractPath)
extractTarball(rubyZip, extractPath)
env = os.environ.copy()
env['JRUBY_HOME'] = extractPath
return env

def ruby_command(args):
"""runs Ruby"""
vmArgs, rubyArgs = extractArguments(args)
vmArgs += ['-cp', mx.classpath(['TRUFFLE_API', 'RUBY'])]
vmArgs += ['org.jruby.Main', '-X+T']
env = setup_jruby_home()
mx.run_java(vmArgs + rubyArgs, env=env)

mx.update_commands(_suite, {
'ruby' : [ruby_command, '[ruby args|@VM options]'],
})

# Utilities

def find_classes_with_matching_source_line(self, pkgRoot, function, includeInnerClasses=False):
return dict()
def jt(args, suite=None, nonZeroIsFatal=True, out=None, err=None, timeout=None, env=None, cwd=None):
rubyDir = _suite.dir
jt = join(rubyDir, 'tool', 'jt.rb')
return mx.run(['ruby', jt] + args, nonZeroIsFatal=nonZeroIsFatal, out=out, err=err, timeout=timeout, env=env, cwd=cwd)

class LicensesBuildTask(mx.BuildTask):
def __init__(self, project, args, vmbuild, vm):
mx.BuildTask.__init__(self, project, args, 1)
self.vm = vm
self.vmbuild = vmbuild
class BackgroundServerTask:
FNULL = open(os.devnull, 'w')

def __str__(self):
return 'Building licences for {}'.format(self.subject)
def __init__(self, args):
self.args = args

def needsBuild(self, newestInput):
return (True, 'Let us re-build everytime')
def __enter__(self):
preexec_fn, creationflags = mx._get_new_progress_group_args()
if mx._opts.verbose:
mx.log(' '.join(['(background)'] + map(pipes.quote, self.args)))
self.process = subprocess.Popen(self.args, preexec_fn=preexec_fn, creationflags=creationflags, stdout=FNULL, stderr=FNULL)
mx._addSubprocess(self.process, self.args)

def newestOutput(self):
return None
def __exit__(self, type, value, traceback):
self.process.kill()
self.process.wait()

def build(self):
if not self.subject.build:
mx.log("...skip build of {}".format(self.subject))
return
mx.log('...perform build of {}'.format(self.subject))
def is_running(self):
return self.process.poll() is None

class BackgroundJT(BackgroundServerTask):
def __init__(self, args):
rubyDir = _suite.dir
licenses_dir = os.path.join(rubyDir, 'licenses')
if not os.path.exists(licenses_dir):
os.mkdir(licenses_dir)
for f in ['BSDL', 'COPYING', 'LICENSE.RUBY']:
shutil.copyfile(os.path.join(rubyDir, f), os.path.join(licenses_dir, f))

mx.log('...finished build of {}'.format(self.subject))
jt = join(rubyDir, 'tool', 'jt.rb')
BackgroundServerTask.__init__(self, ['ruby', jt] + args)

def clean(self, forBuild=False):
if forBuild:
return
rubyDir = _suite.dir
licenses_dir = os.path.join(rubyDir, 'licenses')
if os.path.exists(licenses_dir):
shutil.rmtree(licenses_dir)
##############
# BENCHMARKS #
##############

class RubyBenchmarkSuite(mx_benchmark.BenchmarkSuite):
def group(self):
@@ -602,10 +641,10 @@ def benchmarks(self):
jt(['where', 'repos', 'all-ruby-benchmarks'], out=out)
all_ruby_benchmarks = out.data.strip()
benchmarks = []
for root, dirs, files in os.walk(os.path.join(all_ruby_benchmarks, 'micro')):
for root, dirs, files in os.walk(join(all_ruby_benchmarks, 'micro')):
for name in files:
if name.endswith('.rb'):
benchmark_file = os.path.join(root, name)[len(all_ruby_benchmarks)+1:]
benchmark_file = join(root, name)[len(all_ruby_benchmarks)+1:]
out = mx.OutputCapture()
jt(['benchmark', 'list', benchmark_file], out=out)
benchmarks.extend([benchmark_file + ':' + b.strip() for b in out.data.split('\n') if len(b.strip()) > 0])
152 changes: 100 additions & 52 deletions mx.jruby/suite.py
Original file line number Diff line number Diff line change
@@ -6,8 +6,23 @@
# GNU General Public License version 2
# GNU Lesser General Public License version 2.1

def mavenLib(mavenDep, sha1):
groupId, artifactId, version = mavenDep.split(':')
args = (groupId.replace('.', '/'), artifactId, version, artifactId, version)
url = "https://search.maven.org/remotecontent?filepath=%s/%s/%s/%s-%s.jar" % args
return {
"urls": [ url ],
"sha1": sha1,
"maven": {
"groupId": groupId,
"artifactId": artifactId,
"version": version,
},
"license": "EPL", # fake
}

suite = {
"mxversion": "5.31.4",
"mxversion": "5.36.0",
"name": "jrubytruffle",
"defaultLicense": "EPL",

@@ -17,104 +32,137 @@
"name": "truffle",
"version": "47033f56665100fd5f7cbafd96d6c3112329f517",
"urls": [
{"url": "https://github.com/graalvm/truffle.git",
"kind": "git"},
{"url": "https://github.com/graalvm/truffle.git", "kind": "git"},
{"url": "https://curio.ssw.jku.at/nexus/content/repositories/snapshots", "kind": "binary"},
]
},
],
},
"libraries": {
"RUBY_COMPLETE": {
"path": "maven/jruby-complete/target/jruby-complete-graal-vm.jar",
"sha1": "NOCHECK",
"optional":"true",
"license": "EPL"
},
"RUBY_TRUFFLE": {
"path": "lib/jruby-truffle.jar",
"sha1": "NOCHECK",
"optional":"true",
"license": "EPL"

"licenses": {
"EPL": {
"name": "EPL",
"url": "https://opensource.org/licenses/EPL-1.0",
},
},

"libraries": {

# ------------- Libraries -------------

"ANTLR4_MAIN": mavenLib(
"org.antlr:antlr4:4.5.1-1",
"a8867c83a73791cf30e30de4cf5d0c9a5f0dfdab"),

"ANTLR4_RUNTIME": mavenLib(
"org.antlr:antlr4-runtime:4.5.1-1",
"66144204f9d6d7d3f3f775622c2dd7e9bd511d97"),

"SNAKEYAML": mavenLib(
"org.yaml:snakeyaml:1.14",
"c2df91929ed06a25001939929bff5120e0ea3fd4"),
},

"projects": {

# ------------- Projects -------------

"jruby-ruby": {
"subDir": "lib/ruby",
"class": "MavenProject",
"build": "true",
"prefix": "lib/ruby/",
"jruby-core": {
"class": "JRubyCoreMavenProject",
"sourceDirs": [ "core/src/main/java" ],
"watch": [ "core/src" ],
"jar": "lib/jruby.jar",
},

"jruby-antlr": {
"class": "AntlrProject",
"sourceDir": "truffle/src/main/antlr4",
"outputDir": "truffle/target/generated-sources/antlr4",
"grammars": [ "org/jruby/truffle/core/format/pack/Pack.g4" ],
"dependencies": [ "ANTLR4_RUNTIME" ],
},

"jruby-truffle": {
"dir": "truffle",
"sourceDirs": [
"src/main/java",
"target/generated-sources/antlr4",
],
"dependencies": [
"jruby-core",
"jruby-antlr",
"truffle:TRUFFLE_API",
"truffle:TRUFFLE_DEBUG",
"ANTLR4_RUNTIME",
"SNAKEYAML",
],
"annotationProcessors": ["truffle:TRUFFLE_DSL_PROCESSOR"],
"javaCompliance": "1.8",
"workingSets": "JRubyTruffle",
},

"jruby-truffle-ruby": {
"class": "ArchiveProject",
"outputDir": "truffle/src/main/ruby",
"prefix": "jruby-truffle",
},

# Depends on jruby-core extracting jni libs in lib/jni
"jruby-lib-jni": {
"subDir": "lib/jni",
"class": "MavenProject",
"prefix": "lib/jni/",
"dependencies": [
"truffle:TRUFFLE_API",
"truffle:TRUFFLE_DEBUG",
],
"class": "ArchiveProject",
"outputDir": "lib/jni",
"prefix": "lib/jni",
"dependencies": [ "jruby-core" ],
},

# Depends on jruby-core installing gems in lib/ruby
"jruby-lib-ruby": {
"class": "ArchiveProject",
"outputDir": "lib/ruby",
"prefix": "lib/ruby",
"dependencies": [ "jruby-core" ],
},

"jruby-licences": {
"subDir": "licenses",
"class": "LicensesProject",
"build": "true",
"prefix": "licenses",
"dependencies": [
"truffle:TRUFFLE_API",
"truffle:TRUFFLE_DEBUG",
],
},
},
"licenses": {
"EPL": {
"name": "EPL",
"url": "https://opensource.org/licenses/EPL-1.0",
"outputDir": "",
"prefix": "",
},
},

"distributions": {

# ------------- Distributions -------------

"RUBY": {
"mainClass": "org.jruby.Main",
"dependencies": [
"RUBY_COMPLETE",
"RUBY_TRUFFLE"
"jruby-core",
"jruby-truffle",
"jruby-truffle-ruby",
],
"exclude": [
"truffle:JLINE",
],
"distDependencies": [
"RUBY-ZIP",
"truffle:TRUFFLE_API",
"truffle:TRUFFLE_DEBUG",
],
"description": "JRuby+Truffle",
"license": "EPL"
},

# Set of extra files to extract to run Ruby
"RUBY-ZIP": {
"native": True, # Not Java
"relpath": True,
"dependencies": [
"jruby-ruby",
"jruby-lib-jni",
"jruby-lib-ruby",
"jruby-licences",
],
"exclude": [
"truffle:JLINE",
],
"distDependencies": [
"truffle:TRUFFLE_API",
"truffle:TRUFFLE_DEBUG",
],
"description": "JRuby+Truffle Native Libs",
"license": "EPL"
},

},
}