Skip to content
Permalink

Comparing changes

Choose two branches to see what’s changed or to start a new pull request. If you need to, you can also or learn more about diff comparisons.

Open a pull request

Create a new pull request by comparing changes across two branches. If you need to, you can also . Learn more about diff comparisons here.
base repository: jruby/jruby
Failed to load repositories. Confirm that selected base ref is valid, then try again.
Loading
base: d77153327884
Choose a base ref
...
head repository: jruby/jruby
Failed to load repositories. Confirm that selected head ref is valid, then try again.
Loading
compare: a3aae093a0d3
Choose a head ref
  • 2 commits
  • 3 files changed
  • 1 contributor

Commits on Sep 7, 2017

  1. Use pattern's encoding for Dir glob results. Fixes #4693.

    @headius
    headius committed Sep 7, 2017
    Copy the full SHA
    49d1eb3 View commit details

  2. Unexclude passing glob encoding tests. See #4693.

    @headius
    headius committed Sep 7, 2017
    Copy the full SHA
    a3aae09 View commit details
Showing with 34 additions and 44 deletions.
  1. +2 −11 core/src/main/java/org/jruby/RubyDir.java
  2. +32 −31 core/src/main/java/org/jruby/util/Dir.java
  3. +0 −2 test/mri/excludes/TestDir_M17N.rb
13 changes: 2 additions & 11 deletions core/src/main/java/org/jruby/RubyDir.java
View file
Original file line number Diff line number Diff line change
@@ -164,14 +164,9 @@ private static RubyArray asRubyStringList(Ruby runtime, List<ByteList> dirs) {
final int size = dirs.size();
if ( size == 0 ) return RubyArray.newEmptyArray(runtime);

Encoding enc = runtime.getDefaultExternalEncoding();
if (enc == null) {
enc = UTF8;
}

IRubyObject[] dirStrings = new IRubyObject[ size ];
for ( int i = 0; i < size; i++ ) {
dirStrings[i] = RubyString.newString(runtime, dirs.get(i), enc);
dirStrings[i] = RubyString.newStringNoCopy(runtime, dirs.get(i));
}
return RubyArray.newArrayMayCopy(runtime, dirStrings);
}
@@ -226,11 +221,7 @@ public static IRubyObject glob(ThreadContext context, IRubyObject recv, IRubyObj

if (block.isGiven()) {
for (int i = 0; i < dirs.size(); i++) {
Encoding enc = runtime.getDefaultExternalEncoding();
if (enc == null) {
enc = UTF8;
}
block.yield(context, RubyString.newString(runtime, dirs.get(i), enc));
block.yield(context, RubyString.newString(runtime, dirs.get(i)));
}

return runtime.getNil();
63 changes: 32 additions & 31 deletions core/src/main/java/org/jruby/util/Dir.java
View file
Original file line number Diff line number Diff line change
@@ -34,8 +34,11 @@

import jnr.posix.POSIX;

import org.jcodings.Encoding;
import org.jcodings.specific.USASCIIEncoding;
import org.jruby.Ruby;
import org.jruby.RubyEncoding;
import org.jruby.RubyString;
import org.jruby.platform.Platform;
import static org.jruby.util.ByteList.NULL_ARRAY;
import static org.jruby.util.StringSupport.EMPTY_STRING_ARRAY;
@@ -316,20 +319,22 @@ private static class GlobPattern {
final byte[] bytes;
final int begin;
final int end;
final Encoding enc;

private int index;

private final int flags;

GlobPattern(ByteList bytes, int flags) {
this(bytes.getUnsafeBytes(), bytes.getBegin(), bytes.getBegin() + bytes.getRealSize(), flags);
this(bytes.getUnsafeBytes(), bytes.getBegin(), bytes.getBegin() + bytes.getRealSize(), bytes.getEncoding(), flags);
}

GlobPattern(byte[] bytes, int index, int end, int flags) {
GlobPattern(byte[] bytes, int index, int end, Encoding enc, int flags) {
this.bytes = bytes;
this.index = index;
this.begin = index;
this.end = end;
this.enc = enc;
this.flags = flags;
}

@@ -389,8 +394,8 @@ public byte next() {

}

public interface GlobFunc<T> {
int call(byte[] ptr, int p, int len, T ary);
private interface GlobFunc<T> {
int call(byte[] ptr, int p, int len, Encoding enc, T ary);
}

private static class GlobArgs {
@@ -404,16 +409,16 @@ private static class GlobArgs {
}
}

final static GlobFunc<List<ByteList>> push_pattern = new GlobFunc<List<ByteList>>() {
public int call(byte[] ptr, int p, int len, List<ByteList> ary) {
ary.add(new ByteList(ptr, p, len));
private final static GlobFunc<List<ByteList>> push_pattern = new GlobFunc<List<ByteList>>() {
public int call(byte[] ptr, int p, int len, Encoding enc, List<ByteList> ary) {
ary.add(new ByteList(ptr, p, len, enc, true));
return 0;
}
};
private final static GlobFunc<GlobArgs> glob_caller = new GlobFunc<GlobArgs>() {
public int call(byte[] ptr, int p, int len, GlobArgs args) {
public int call(byte[] ptr, int p, int len, Encoding enc, GlobArgs args) {
args.c = p;
return args.func.call(ptr, args.c, len, args.arg);
return args.func.call(ptr, args.c, len, enc, args.arg);
}
};

@@ -430,6 +435,7 @@ private static int push_braces(Ruby runtime, String cwd, List<ByteList> result,
lbrace > 0 && pattern.bytes[lbrace-1] == '\\' ||
rbrace > 0 && pattern.bytes[rbrace-1] == '\\') {
ByteList unescaped = new ByteList(pattern.bytes.length - 1);
unescaped.setEncoding(pattern.enc);
for (int i = pattern.begin; i < pattern.end; i++) {
byte b = pattern.bytes[i];
if (b == '\\' && i < pattern.bytes.length - 1) {
@@ -447,6 +453,7 @@ private static int push_braces(Ruby runtime, String cwd, List<ByteList> result,
// Peel onion...make subpatterns out of outer layer of glob and recall with each subpattern
// Example: foo{a{c},b}bar -> fooa{c}bar, foobbar
final ByteList bytes = new ByteList(20);
bytes.setEncoding(pattern.enc);
int middleRegionIndex;
int i = lbrace;
while (pattern.bytes[i] != '}') {
@@ -656,8 +663,8 @@ private static boolean isSpecialFile(String name) {
return c == '.' && name.charAt(2) == '/';
}

private static int addToResultIfExists(Ruby runtime, String cwd, byte[] bytes, int begin, int end, int flags, GlobFunc<GlobArgs> func, GlobArgs arg) {
final String fileName = newStringFromUTF8(bytes, begin, end - begin);
private static int addToResultIfExists(Ruby runtime, String cwd, byte[] bytes, int begin, int end, Encoding enc, int flags, GlobFunc<GlobArgs> func, GlobArgs arg) {
final String fileName = new String(bytes, begin, end - begin, enc.getCharset());

// FIXME: Ultimately JRubyFile.createResource should do this but all 1.7.x is only selectively honoring raw
// paths and using system drive make it absolute. MRI does this on many methods we don't.
@@ -668,7 +675,7 @@ private static int addToResultIfExists(Ruby runtime, String cwd, byte[] bytes, i
}
FileResource file = JRubyFile.createResource(runtime, cwd, fileName);
if (file.exists()) {
return func.call(bytes, begin, end - begin, arg);
return func.call(bytes, begin, end - begin, enc, arg);
}

return 0;
@@ -677,11 +684,12 @@ private static int addToResultIfExists(Ruby runtime, String cwd, byte[] bytes, i
private static int glob_helper(Ruby runtime, String cwd, ByteList path, int sub, int flags, GlobFunc<GlobArgs> func, GlobArgs arg) {
final int begin = path.getBegin();
final int end = begin + path.getRealSize();
return glob_helper(runtime, cwd, path.getUnsafeBytes(), begin, end, sub, flags, func, arg);
final Encoding enc = path.getEncoding();
return glob_helper(runtime, cwd, path.getUnsafeBytes(), begin, end, enc, sub, flags, func, arg);
}

private static int glob_helper(Ruby runtime, String cwd,
byte[] path, int begin, int end, int sub,
byte[] path, int begin, int end, Encoding enc, int sub,
final int flags, GlobFunc<GlobArgs> func, GlobArgs arg) {
int status = 0;

@@ -703,9 +711,9 @@ private static int glob_helper(Ruby runtime, String cwd,

if (end > begin) {
if ( isAbsolutePath(path, begin, end) ) {
status = addToResultIfExists(runtime, null, path, begin, end, flags, func, arg);
status = addToResultIfExists(runtime, null, path, begin, end, enc, flags, func, arg);
} else {
status = addToResultIfExists(runtime, cwd, path, begin, end, flags, func, arg);
status = addToResultIfExists(runtime, cwd, path, begin, end, enc, flags, func, arg);
}
}

@@ -714,7 +722,9 @@ private static int glob_helper(Ruby runtime, String cwd,

final ArrayList<DirGlobber> links = new ArrayList<DirGlobber>();

ByteList buf = new ByteList(20); FileResource resource;
ByteList buf = new ByteList(20);
buf.setEncoding(enc);
FileResource resource;

mainLoop: while(ptr != -1 && status == 0) {
if ( path[ptr] == '/' ) ptr++;
@@ -727,7 +737,7 @@ private static int glob_helper(Ruby runtime, String cwd,
byte[] magic = extract_elem(path, ptr, end);
boolean recursive = false;

resource = JRubyFile.createResource(runtime, cwd, newStringFromUTF8(dir, 0, dir.length));
resource = JRubyFile.createResource(runtime, cwd, new String(dir, 0, dir.length, enc.getCharset()));
if ( resource.isDirectory() ) {
if ( SLASH_INDEX != -1 && Arrays.equals(magic, DOUBLE_STAR) ) {
final int lengthOfBase = base.length;
@@ -770,7 +780,7 @@ private static int glob_helper(Ruby runtime, String cwd,
buf.append(base);
buf.append( isRoot(base) ? EMPTY : SLASH );
buf.append( getBytesInUTF8(file) );
resource = JRubyFile.createResource(runtime, cwd, newStringFromUTF8(buf));
resource = JRubyFile.createResource(runtime, cwd, new String(buf.unsafeBytes(), buf.begin(), buf.length(), enc.getCharset()));
if ( !resource.isSymLink() && resource.isDirectory() && !".".equals(file) && !"..".equals(file) ) {
final int len = buf.getRealSize();
buf.append(SLASH);
@@ -787,12 +797,13 @@ private static int glob_helper(Ruby runtime, String cwd,
buf.append( isRoot(base) ? EMPTY : SLASH );
buf.append( getBytesInUTF8(file) );
if ( SLASH_INDEX == -1 ) {
status = func.call(buf.getUnsafeBytes(), 0, buf.getRealSize(), arg);
status = func.call(buf.getUnsafeBytes(), 0, buf.getRealSize(), enc, arg);
if ( status != 0 ) break;
continue;
}
links.add(new DirGlobber(buf));
buf = new ByteList(20);
buf.setEncoding(enc);
}
}
} while(false);
@@ -801,7 +812,7 @@ private static int glob_helper(Ruby runtime, String cwd,
for ( DirGlobber globber : links ) {
final ByteList link = globber.link;
if ( status == 0 ) {
resource = JRubyFile.createResource(runtime, cwd, newStringFromUTF8(link));
resource = JRubyFile.createResource(runtime, cwd, RubyString.byteListToString(link));
if ( resource.isDirectory() ) {
final int len = link.getRealSize();
buf.length(0);
@@ -824,14 +835,4 @@ private static byte[] getBytesInUTF8(final String str) {
return RubyEncoding.encodeUTF8(str);
}

private static String newStringFromUTF8(final ByteList bytes) {
final int offset = bytes.getBegin();
final int length = bytes.getRealSize();
return RubyEncoding.decodeUTF8(bytes.getUnsafeBytes(), offset, length);
}

private static String newStringFromUTF8(final byte[] bytes, int offset, int len) {
return RubyEncoding.decodeUTF8(bytes, offset, len);
}

}
2 changes: 0 additions & 2 deletions test/mri/excludes/TestDir_M17N.rb
View file
Original file line number Diff line number Diff line change
@@ -3,14 +3,12 @@
exclude :test_filename_bytes_euc_jp, "needs investigation"
exclude :test_filename_euc_jp, "needs investigation"
exclude :test_filename_ext_euc_jp_and_int_utf_8, "needs investigation"
exclude :test_filename_extutf8, "needs investigation"
exclude :test_filename_extutf8_inteucjp_representable, "needs investigation"
exclude :test_filename_extutf8_inteucjp_unrepresentable, "needs investigation"
exclude :test_filename_extutf8_invalid, "needs investigation"
exclude :test_filename_utf8_raw_jp_name, "broken subprocess logic in setup"
exclude :test_filename_utf8_raw_windows_1251_name, "broken subprocess logic in setup"
exclude :test_filename_utf8_raw_windows_1252_name, "broken subprocess logic in setup"
exclude :test_glob_encoding, "we always normalize to Unicode internally"
exclude :test_glob_incompatible, "needs investigation"
exclude :test_glob_warning_match_all, "missing warning"
exclude :test_glob_warning_match_dir, "missing warning"