string_decoder: add support for CESU-8

Fixes #3217.
nodejs · May 5, 2012 · ceb51dd · ceb51dd
1 parent eaf607e
commit ceb51dd
Show file tree

Hide file tree

Showing 2 changed files with 78 additions and 14 deletions.
diff --git a/lib/string_decoder.js b/lib/string_decoder.js
@@ -22,7 +22,7 @@
 var StringDecoder = exports.StringDecoder = function(encoding) {
   this.encoding = (encoding || 'utf8').toLowerCase().replace(/[-_]/, '');
   if (this.encoding === 'utf8') {
-    this.charBuffer = new Buffer(4);
+    this.charBuffer = new Buffer(6);
     this.charReceived = 0;
     this.charLength = 0;
   }
@@ -36,16 +36,18 @@ StringDecoder.prototype.write = function(buffer) {
   }
 
   var charStr = '';
+  var offset = 0;
   // if our last write ended with an incomplete multibyte character
-  if (this.charLength) {
+  while (this.charLength) {
     // determine how many remaining bytes this buffer has to offer for this char
     var i = (buffer.length >= this.charLength - this.charReceived) ?
                 this.charLength - this.charReceived :
                 buffer.length;
 
     // add the new bytes to the char buffer
-    buffer.copy(this.charBuffer, this.charReceived, 0, i);
-    this.charReceived += i;
+    buffer.copy(this.charBuffer, this.charReceived, offset, i);
+    this.charReceived += (i - offset);
+    offset = i;
 
     if (this.charReceived < this.charLength) {
       // still not enough chars in this buffer? wait for more ...
@@ -54,13 +56,24 @@ StringDecoder.prototype.write = function(buffer) {
 
     // get the character that was split
     charStr = this.charBuffer.slice(0, this.charLength).toString();
+
+    // lead surrogate (D800-DBFF) is also the incomplete character
+    if (this.charLength === 3) {
+      var charCode = charStr.charCodeAt(0);
+      if (charCode >= 0xD800 && charCode <= 0xDBFF) {
+        charStr = '';
+        this.charLength += 3; // size of trail surrogate (DC00-DFFF)
+        continue;
+      }
+    }
     this.charReceived = this.charLength = 0;
 
     // if there are no more bytes in this buffer, just emit our char
     if (i == buffer.length) return charStr;
 
     // otherwise cut off the characters end from the beginning of this buffer
     buffer = buffer.slice(i, buffer.length);
+    break;
   }
 
 
@@ -93,18 +106,26 @@ StringDecoder.prototype.write = function(buffer) {
     }
   }
 
-  if (!this.charLength) {
-    // no incomplete char at the end of this buffer, emit the whole thing
-    return charStr + buffer.toString();
+  var end = buffer.length;
+  if (this.charLength) {
+    // buffer the incomplete character bytes we got
+    buffer.copy(this.charBuffer, 0, buffer.length - i, buffer.length);
+    this.charReceived = i;
+    end -= i;
   }
 
-  // buffer the incomplete character bytes we got
-  buffer.copy(this.charBuffer, 0, buffer.length - i, buffer.length);
-  this.charReceived = i;
-
-  if (buffer.length - i > 0) {
-    // buffer had more bytes before the incomplete char, emit them
-    return charStr + buffer.toString('utf8', 0, buffer.length - i);
+  charStr += buffer.toString('utf8', 0, end);
+
+  // lead surrogate (D800-DBFF) is also the incomplete character
+  end = charStr.length - 1;
+  var charCode = charStr.charCodeAt(end);
+  if (charCode >= 0xD800 && charCode <= 0xDBFF) {
+    // CESU-8 represents each of Surrogate Pair by 3-bytes
+    this.charLength += 3
+    this.charReceived += 3
+    this.charBuffer.copy(this.charBuffer, 3, 0, 3);
+    this.charBuffer.write(charStr.charAt(end));
+    return charStr.substring(0, end);
   }
 
   // or just emit the charStr

diff --git a/test/simple/test-string-decoder.js b/test/simple/test-string-decoder.js
@@ -46,6 +46,49 @@ s += decoder.write(buffer.slice(2, 3));
 s += decoder.write(buffer.slice(3, 4));
 assert.ok(s.length > 0);
 
+// CESU-8
+buffer = new Buffer('EDA0BDEDB18D', 'hex'); // THUMBS UP SIGN (in CESU-8)
+var s = '';
+s += decoder.write(buffer.slice(0, 1));
+s += decoder.write(buffer.slice(1, 2));
+s += decoder.write(buffer.slice(2, 3)); // complete lead surrogate
+assert.equal(s, '');
+s += decoder.write(buffer.slice(3, 4));
+s += decoder.write(buffer.slice(4, 5));
+s += decoder.write(buffer.slice(5, 6)); // complete trail surrogate
+assert.equal(s, '\uD83D\uDC4D'); // THUMBS UP SIGN (in UTF-16)
+
+var s = '';
+s += decoder.write(buffer.slice(0, 2));
+s += decoder.write(buffer.slice(2, 4)); // complete lead surrogate
+assert.equal(s, '');
+s += decoder.write(buffer.slice(4, 6)); // complete trail surrogate
+assert.equal(s, '\uD83D\uDC4D'); // THUMBS UP SIGN (in UTF-16)
+
+var s = '';
+s += decoder.write(buffer.slice(0, 3)); // complete lead surrogate
+assert.equal(s, '');
+s += decoder.write(buffer.slice(3, 6)); // complete trail surrogate
+assert.equal(s, '\uD83D\uDC4D'); // THUMBS UP SIGN (in UTF-16)
+
+var s = '';
+s += decoder.write(buffer.slice(0, 4)); // complete lead surrogate
+assert.equal(s, '');
+s += decoder.write(buffer.slice(4, 5));
+s += decoder.write(buffer.slice(5, 6)); // complete trail surrogate
+assert.equal(s, '\uD83D\uDC4D'); // THUMBS UP SIGN (in UTF-16)
+
+var s = '';
+s += decoder.write(buffer.slice(0, 5)); // complete lead surrogate
+assert.equal(s, '');
+s += decoder.write(buffer.slice(5, 6)); // complete trail surrogate
+assert.equal(s, '\uD83D\uDC4D'); // THUMBS UP SIGN (in UTF-16)
+
+var s = '';
+s += decoder.write(buffer.slice(0, 6));
+assert.equal(s, '\uD83D\uDC4D'); // THUMBS UP SIGN (in UTF-16)
+
+
 // A mixed ascii and non-ascii string
 // Test stolen from deps/v8/test/cctest/test-strings.cc
 // U+02E4 -> CB A4