Skip to content

Commit

Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[Truffle] Add bundler workarounds files, jt test bundle command,
Browse files Browse the repository at this point in the history
Brandon Fish committed Oct 17, 2016
1 parent b607f52 commit 5ede6bf
Showing 5 changed files with 576 additions and 4 deletions.
1 change: 1 addition & 0 deletions .travis.yml
Original file line number Diff line number Diff line change
@@ -24,6 +24,7 @@ matrix:
- env: JT='test tck'
- env: JT='check_ambiguous_arguments' SKIP_BUILD=true V=1
- env: JT='test mri'
- env: JT='test bundle'
allow_failures:
- env: JT='test mri'
# Exlude the default job https://github.com/travis-ci/travis-ci/issues/4681
516 changes: 516 additions & 0 deletions lib/ruby/truffle/truffle/bundler-workarounds.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,516 @@
$quiet_workaround = false

workaround_header = <<~HEREDOC
==========================================
WORKING AROUND BUNDLE INSTALL ISSUES
Instructions:
1. Run bundle install command with workarounds. E.g.:
GEM_HOME=/Users/brandonfish/Documents/truffle_gem_home ../../jruby/bin/jruby -X+T -rbundler-workarounds -S bundle install
2. Try running gem tests, e.g.:
GEM_HOME=/Users/brandonfish/Documents/truffle_gem_home ../../jruby/bin/jruby -X+T -rbundler-workarounds -S bundle exec rake
HEREDOC
puts workaround_header unless $quiet_workaround



have_extensions = <<~HEREDOC
==========================================
Workaround: Add nil check Gem::BasicSpecification.have_extensions?
Error:
undefined method `empty?' for nil:NilClass
Called here:
lib/ruby/stdlib/rubygems/basic_specification.rb:313
HEREDOC
puts have_extensions unless $quiet_workaround
class Gem::BasicSpecification
def have_extensions?; !extensions.nil? && !extensions.empty?; end
end


remote_fetcher = <<~HEREDOC
==========================================
Workaround: Use wget in remote fetcher
HEREDOC
puts remote_fetcher unless $quiet_workaround

require 'rubygems/remote_fetcher'
class Gem::RemoteFetcher
def download(spec, source_uri, install_dir = Gem.dir)
cache_dir =
if Dir.pwd == install_dir then # see fetch_command
install_dir
elsif File.writable? install_dir then
File.join install_dir, "cache"
else
File.join Gem.user_dir, "cache"
end

gem_file_name = File.basename spec.cache_file
local_gem_path = File.join cache_dir, gem_file_name

FileUtils.mkdir_p cache_dir rescue nil unless File.exist? cache_dir

# Always escape URI's to deal with potential spaces and such
# It should also be considered that source_uri may already be
# a valid URI with escaped characters. e.g. "{DESede}" is encoded
# as "%7BDESede%7D". If this is escaped again the percentage
# symbols will be escaped.
unless source_uri.is_a?(URI::Generic)
begin
source_uri = URI.parse(source_uri)
rescue
source_uri = URI.parse(URI.const_defined?(:DEFAULT_PARSER) ?
URI::DEFAULT_PARSER.escape(source_uri.to_s) :
URI.escape(source_uri.to_s))
end
end

scheme = source_uri.scheme

# URI.parse gets confused by MS Windows paths with forward slashes.
scheme = nil if scheme =~ /^[a-z]$/i

# REFACTOR: split this up and dispatch on scheme (eg download_http)
# REFACTOR: be sure to clean up fake fetcher when you do this... cleaner
case scheme
when 'http', 'https', 's3' then
unless File.exist? local_gem_path then
begin
verbose "Downloading gem #{gem_file_name}"

remote_gem_path = source_uri + "gems/#{gem_file_name}"

# WORKAROUND STARTS HERE
cmd = "wget #{remote_gem_path} -O #{local_gem_path}"
`#{cmd}`
# self.cache_update_path remote_gem_path, local_gem_path
# WORKAROUND ENDS HERE
rescue Gem::RemoteFetcher::FetchError
raise if spec.original_platform == spec.platform

alternate_name = "#{spec.original_name}.gem"

verbose "Failed, downloading gem #{alternate_name}"

remote_gem_path = source_uri + "gems/#{alternate_name}"

self.cache_update_path remote_gem_path, local_gem_path
end
end
when 'file' then
begin
path = source_uri.path
path = File.dirname(path) if File.extname(path) == '.gem'

remote_gem_path = correct_for_windows_path(File.join(path, 'gems', gem_file_name))

FileUtils.cp(remote_gem_path, local_gem_path)
rescue Errno::EACCES
local_gem_path = source_uri.to_s
end

verbose "Using local gem #{local_gem_path}"
when nil then # TODO test for local overriding cache
source_path = if Gem.win_platform? && source_uri.scheme &&
!source_uri.path.include?(':') then
"#{source_uri.scheme}:#{source_uri.path}"
else
source_uri.path
end

source_path = Gem::UriFormatter.new(source_path).unescape

begin
FileUtils.cp source_path, local_gem_path unless
File.identical?(source_path, local_gem_path)
rescue Errno::EACCES
local_gem_path = source_uri.to_s
end

verbose "Using local gem #{local_gem_path}"
else
raise ArgumentError, "unsupported URI scheme #{source_uri.scheme}"
end

local_gem_path
end
end

ruby_version = <<~HEREDOC
==========================================
Workaround: Set RUBY_ENGINE to 'ruby'
Error:
RUBY_ENGINE value jruby+truffle is not recognized
Called here:
lib/bundler/ruby_version.rb:98
HEREDOC
puts ruby_version unless $quiet_workaround
RUBY_ENGINE = "ruby"


curl_https = <<~HEREDOC
==========================================
Workaround: Use curl when URIs are https
HEREDOC
puts curl_https unless $quiet_workaround
class CurlResponse < Net::HTTPSuccess
def body
@body
end
end
require 'rubygems/request'
class Gem::Request
def fetch
request = @request_class.new @uri.request_uri

unless @uri.nil? || @uri.user.nil? || @uri.user.empty? then
request.basic_auth Gem::UriFormatter.new(@uri.user).unescape,
Gem::UriFormatter.new(@uri.password).unescape
end

request.add_field 'User-Agent', @user_agent
request.add_field 'Connection', 'keep-alive'
request.add_field 'Keep-Alive', '30'

if @last_modified then
request.add_field 'If-Modified-Since', @last_modified.httpdate
end

yield request if block_given?

# WORKAROUND START
if @uri.scheme == "https"
resp = CurlResponse.new("1.1", 200, "OK")
resp_raw = `curl -i #{@uri}`
blank_line_idx = resp_raw.index("\r\n\r\n")
header = resp_raw[0, blank_line_idx]
resp.body = resp_raw[(blank_line_idx+4)..-1]
if m = /ETag: (\"[[:alnum:]]*\")/.match(header)
resp["ETag"] = m[1]
end
resp
else
perform_request request
end
# WORKAROUND END
end
end

rubygems_package = <<~HEREDOC
==========================================
Workarounds:
Gem::Package
- shell to tar for untarring
- skip some checksum/digest verification
HEREDOC
puts rubygems_package unless $quiet_workaround
require 'rubygems/package'
class Gem::Package
def extract_files destination_dir, pattern = "*"
verify unless @spec

FileUtils.mkdir_p destination_dir

# WORKAROUND START
extr_to = File.dirname(@gem.path) + '/' + File.basename(@gem.path, File.extname(@gem.path))
Dir.mkdir(extr_to)
`tar -C #{extr_to} -xf #{@gem.path}`
# WORKAROUND END

@gem.with_read_io do |io|
reader = Gem::Package::TarReader.new io

reader.each do |entry|
next unless entry.full_name == 'data.tar.gz'

# WORKAROUND START
`tar -C #{destination_dir} -xzf #{extr_to + '/' + entry.full_name}`
#extract_tar_gz entry, destination_dir, pattern
FileUtils.remove_dir(extr_to)
# WORKAROUND END

return # ignore further entries
end
end
end

def verify
@files = []
@spec = nil

@gem.with_read_io do |io|
Gem::Package::TarReader.new io do |reader|
read_checksums reader

verify_files reader
end
end

# WORKAROUND START
#verify_checksums @digests, @checksums
# WORKAROUND END

@security_policy.verify_signatures @spec, @digests, @signatures if
@security_policy

true
rescue Gem::Security::Exception
@spec = nil
@files = []
raise
rescue Errno::ENOENT => e
raise Gem::Package::FormatError.new e.message
rescue Gem::Package::TarInvalidError => e
raise Gem::Package::FormatError.new e.message, @gem
end

def verify_entry entry
file_name = entry.full_name
@files << file_name

case file_name
when /\.sig$/ then
@signatures[$`] = entry.read if @security_policy
return
else
# WORKAROUND START
#digest entry
# WORKAROUND END
end

case file_name
when /^metadata(.gz)?$/ then
load_spec entry
when 'data.tar.gz' then
# WORKAROUND START
#verify_gz entry
# WORKAROUND END
end
rescue => e
message = "package is corrupt, exception while verifying: " +
"#{e.message} (#{e.class})"
raise Gem::Package::FormatError.new message, @gem
end

end


zlib_crc = <<~HEREDOC
==========================================
Workaround: Disable zlib crc check
HEREDOC
puts zlib_crc unless $quiet_workaround
require "zlib"
module Zlib
class GzipFile
def gzfile_check_footer()
@gz.z.flags |= GZFILE_FLAG_FOOTER_FINISHED

if (!gzfile_read_raw_ensure(8))
raise NoFooter, "footer is not found"
end
crc = gzfile_get32(@gz.z.input)
length = gzfile_get32(@gz.z.input[4,4])
@gz.z.stream.total_in += 8
@gz.z.zstream_discard_input(8)
# WORKAROUND START
# if (@gz.crc != crc)
# raise CRCError, "invalid compressed data -- crc error"
# end
# WORKAROUND END
if (@gz.z.stream.total_out != length)
raise LengthError, "invalid compressed data -- length error"
end
end
end
end


bundler_updater = <<~HEREDOC
==========================================
Workaround: Shell to gunzip in bundler updater
HEREDOC
puts bundler_updater unless $quiet_workaround
require "bundler"
require "bundler/vendor/compact_index_client/lib/compact_index_client"
class Bundler::CompactIndexClient
class Updater
def update(local_path, remote_path, retrying = nil)
headers = {}

Dir.mktmpdir(local_path.basename.to_s, local_path.dirname) do |local_temp_dir|

local_temp_path = Pathname.new(local_temp_dir).join(local_path.basename)


# download new file if retrying
if retrying.nil? && local_path.file?
FileUtils.cp local_path, local_temp_path
headers["If-None-Match"] = etag_for(local_temp_path)
headers["Range"] = "bytes=#{local_temp_path.size}-"
else
# Fastly ignores Range when Accept-Encoding: gzip is set
headers["Accept-Encoding"] = "gzip"
end

response = @fetcher.call(remote_path, headers)
return if response.is_a?(Net::HTTPNotModified)

content = response.body

if response["Content-Encoding"] == "gzip"
IO.binwrite("#{local_temp_dir}/gzipped_versions", content)

content = Zlib::GzipReader.new(StringIO.new(content)).read
#content = `gunzip -c #{local_temp_dir}/gzipped_versions`
end

mode = response.is_a?(Net::HTTPPartialContent) ? "a" : "w"
local_temp_path.open(mode) {|f| f << content }


response_etag = response["ETag"]

if etag_for(local_temp_path) == response_etag
FileUtils.mv(local_temp_path, local_path)
return
end

if retrying.nil?
update(local_path, remote_path, :retrying)
else
# puts "ERROR: #{remote_path},#{local_path},#{local_temp_path}"
raise MisMatchedChecksumError.new(remote_path, response_etag, etag_for(local_temp_path))
end
end
end
end
end


module OpenSSL

module Cipher

cipher_new = <<~HEREDOC
==========================================
Workaround: Stub OpenSSL::Cipher.new
Called here:
lib/ruby/stdlib/rubygems/security.rb:372
HEREDOC
puts cipher_new unless $quiet_workaround

def self.new(enc)

end

end

verify_peer = <<~HEREDOC
==========================================
Workaround: Stub OpenSSL::SSL::VERIFY_PEER
Error:
uninitialized constant OpenSSL::SSL::VERIFY_PEER
Called here:
bundler/vendor/net/http/persistent.rb:519
HEREDOC
puts verify_peer unless $quiet_workaround
module SSL
VERIFY_PEER = 1
end

verify_none = <<~HEREDOC
==========================================
Workaround: Stub OpenSSL::SSL::VERIFY_NONE
Error:
uninitialized constant OpenSSL::SSL::VERIFY_NONE
Called here:
bundler/vendor/net/http/persistent.rb:1142
HEREDOC
puts verify_none unless $quiet_workaround
module SSL
VERIFY_NONE = 1
end

# module Digest
# def self.new(enc)
#
# end
# end

module X509
class Store
def set_default_paths

end
def add_file(f)

end
end

end

end

bundler_fetcher = <<~HEREDOC
==========================================
Workaround: Use curl in bundler downloader for https requests
HEREDOC
puts bundler_fetcher unless $quiet_workaround

require "openssl-stubs"
require "bundler/fetcher/downloader"

module Bundler
class Fetcher
class Downloader
def fetch(uri, options = {}, counter = 0)
raise HTTPError, "Too many redirects" if counter >= redirect_limit

# WORKAROUND START
response = if uri.scheme == "https"
resp = CurlResponse.new("1.1",200,"OK")
resp_raw = `curl -i #{uri}`
blank_line_idx = resp_raw.index("\r\n\r\n")
header = resp_raw[0, blank_line_idx]
resp.body = resp_raw[(blank_line_idx+4)..-1]
if m = /ETag: (\"[[:alnum:]]*\")/.match(header)
resp["ETag"] = m[1]
end
resp
else
response = request(uri, options)
end
# WORKAROUND END

Bundler.ui.debug("HTTP #{response.code} #{response.message}")

case response
when Net::HTTPSuccess, Net::HTTPNotModified
response
when Net::HTTPRedirection
new_uri = URI.parse(response["location"])
if new_uri.host == uri.host
new_uri.user = uri.user
new_uri.password = uri.password
end
fetch(new_uri, options, counter + 1)
when Net::HTTPRequestEntityTooLarge
raise FallbackError, response.body
when Net::HTTPUnauthorized
raise AuthenticationRequiredError, uri.host
when Net::HTTPNotFound
raise FallbackError, "Net::HTTPNotFound"
else
raise HTTPError, "#{response.class}#{": #{response.body}" unless response.body.empty?}"
end
end

end
end
end


#

puts "==========================================" unless $quiet_workaround
38 changes: 38 additions & 0 deletions tool/jt.rb
Original file line number Diff line number Diff line change
@@ -459,6 +459,7 @@ def help
jt test compiler run compiler tests (uses the same logic as --graal to find Graal)
jt test integration runs all integration tests
jt test integration [TESTS] runs the given integration tests
jt test bundle tests using bundler
jt test gems tests using gems
jt test ecosystem [TESTS] tests using the wider ecosystem such as bundler, Rails, etc
jt test cexts [--no-libxml --no-openssl --no-argon2] run C extension tests
@@ -732,6 +733,7 @@ def test(*args)
test_ecosystem 'HAS_REDIS' => 'true'
test_compiler
test_cexts
when 'bundle' then test_bundle(*rest)
when 'compiler' then test_compiler(*rest)
when 'cexts' then test_cexts(*rest)
when 'report' then test_report(*rest)
@@ -954,6 +956,42 @@ def test_ecosystem(env={}, *args)
end
private :test_ecosystem

def test_bundle(*args)
gems = [{:url => "https://github.com/sstephenson/hike", :commit => "3abf0b3feb47c26911f8cedf2cd409471fd26da1"}]
gems.each do |gem|
gem_url = gem[:url]
name = gem_url.split('/')[-1]
require 'tmpdir'
temp_dir = Dir.mktmpdir(name)
begin
Dir.chdir(temp_dir) do
puts "Cloning gem #{name} into temp directory: #{temp_dir}"
raw_sh(*['git', 'clone', gem_url])
end
gem_dir = File.join(temp_dir, name)
gem_home = if ENV['GEM_HOME']
ENV['GEM_HOME']
else
tmp_home = File.join(temp_dir, "gem_home")
Dir.mkdir(tmp_home)
puts "Using temporary GEM_HOME:#{tmp_home}"
tmp_home
end
Dir.chdir(gem_dir) do
if gem.has_key?(:commit)
raw_sh(*['git', 'checkout', gem[:commit]])
end
run({'GEM_HOME' => gem_home}, *["-rbundler-workarounds", "-S", "bundle", "install"])
# Need to workaround ruby_install name `jruby-truffle` in the gems binstubs (command can't be found )
# or figure out how to get it on the path to get `bundle exec rake` working
#run({'GEM_HOME' => gem_home}, *["-rbundler-workarounds", "-S", "bundle", "exec", "rake"])
end
ensure
FileUtils.remove_entry temp_dir
end
end
end

def test_specs(command, *args)
env_vars = {}
options = []
Original file line number Diff line number Diff line change
@@ -588,6 +588,10 @@ public DynamicObject protectedMethodError(String name, Object self, Object[] arg
public DynamicObject loadError(String message, String path, Node currentNode) {
DynamicObject messageString = StringOperations.createString(context, StringOperations.encodeRope(message, UTF8Encoding.INSTANCE));
DynamicObject loadError = ExceptionOperations.createRubyException(context.getCoreLibrary().getLoadErrorClass(), messageString, context.getCallStack().getBacktrace(currentNode));
if("openssl.so".equals(path)){
// This is a workaround for the rubygems/security.rb file expecting the error path to be openssl
path = "openssl";
}
loadError.define("@path", StringOperations.createString(context, StringOperations.encodeRope(path, UTF8Encoding.INSTANCE)), 0);
return loadError;
}
Original file line number Diff line number Diff line change
@@ -149,10 +149,14 @@
import java.io.InputStreamReader;
import java.lang.ProcessBuilder.Redirect;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
import java.util.Map;

import static org.jruby.truffle.core.array.ArrayHelpers.getStore;

@CoreClass("Kernel")
public abstract class KernelNodes {

@@ -710,12 +714,21 @@ public Object exec(VirtualFrame frame, Object command, Object[] args) {

@TruffleBoundary
private String[] buildCommandLine(Object command, Object[] args) {
final String[] commandLine = new String[1 + args.length];
commandLine[0] = command.toString();
final List<String> commandLine = new ArrayList<>(1 + args.length);
if (RubyGuards.isRubyArray(command)) {
final Object[] store = (Object[]) getStore((DynamicObject) command);
commandLine.add(store[0].toString());
} else {
commandLine.add(command.toString());
}
for (int n = 0; n < args.length; n++) {
commandLine[1 + n] = args[n].toString();
if (n == args.length - 1 && RubyGuards.isRubyHash(args[n])) {
break;
}
commandLine.add(args[n].toString());
}
return commandLine;
final String[] result = new String[commandLine.size()];
return commandLine.toArray(result);
}

@TruffleBoundary

0 comments on commit 5ede6bf

Please sign in to comment.