Merge pull request #163 from mustafaiman/fixEmojiStringSerialization

fix utf8 serialization/deserialization algorithm

Author: mustafaiman

src/serialization/ObjectData.ts

@@ -30,7 +30,7 @@ export class ObjectDataOutput implements DataOutput {
     private ensureAvailable(size: number): void {
         if (this.available() < size ) {
             var newBuffer = new Buffer(this.pos + size);
-            this.buffer.copy(newBuffer);
+            this.buffer.copy(newBuffer, 0, 0, this.pos);
             this.buffer = newBuffer;
         }
     }
@@ -62,7 +62,13 @@ export class ObjectDataOutput implements DataOutput {
     }
 
     toBuffer(): Buffer {
-        return this.buffer;
+        if (this.buffer == null || this.pos === 0) {
+            return new Buffer(0);
+        } else {
+            var snapBuffer = new Buffer(this.pos);
+            this.buffer.copy(snapBuffer, 0, 0, this.pos);
+            return snapBuffer;
+        }
     }
 
     write(byte: number|Buffer): void {
@@ -199,8 +205,19 @@ export class ObjectDataOutput implements DataOutput {
     writeUTF(val: string): void {
         var len = (val != null) ? val.length : BitsUtil.NULL_ARRAY_LENGTH;
         this.writeInt(len);
-        if (len > 0 ) {
-            this.write(new Buffer(val, 'utf8'));
+        this.ensureAvailable(len * 3);
+        for (let i = 0; i < len; i++) {
+            let ch = val.charCodeAt(i);
+            if (ch <= 0x007F) {
+                this.writeByte(ch);
+            } else if (ch <= 0x07FF) {
+                this.write(0xC0 | ch >> 6 & 0x1F);
+                this.write(0x80 | ch & 0x3F);
+            } else {
+                this.write(0xE0 | ch >> 12 & 0x0F);
+                this.write(0x80 | ch >> 6 & 0x3F);
+                this.write(0x80 | ch & 0x3F);
+            }
         }
     }
 
@@ -472,31 +489,52 @@ export class ObjectDataInput implements DataInput {
     }
 
     readUTF(pos?: number): string {
-        var len = this.readInt(pos);
-        var readingIndex = this.addOrUndefined(pos, 4);
+        let len = this.readInt(pos);
+        let readingIndex = this.addOrUndefined(pos, 4);
         if (len === BitsUtil.NULL_ARRAY_LENGTH) {
             return null;
-        } else {
-            var result: number[] = [];
-            var leadingByte: number;
-            var continuationByte: number;
-            for (var i = 0; i < len; i++) {
-                leadingByte = this.readByte(readingIndex) & MASK_1BYTE;
-                readingIndex = this.addOrUndefined(readingIndex, 1);
-                result.push(leadingByte);
-                if (leadingByte >= 128) {
-                    while (((leadingByte <<= 1) & MASK_1BYTE) >= 128) {
-                        continuationByte = this.readByte(readingIndex);
-                        readingIndex = this.addOrUndefined(readingIndex, 1);
-                        if (((continuationByte >> 6) & MASK_1BYTE) !== 2) {
-                            throw new Error('String is not properly UTF8 encoded');
-                        }
-                        result.push(continuationByte);
-                    }
-                }
+        }
+        let result: string = '';
+        let leadingByte: number;
+        for (let i = 0; i < len; i++) {
+            let charCode: number;
+            leadingByte = this.readByte(readingIndex) & MASK_1BYTE;
+            readingIndex = this.addOrUndefined(readingIndex, 1);
+
+            let b = leadingByte & 0xFF;
+            switch (b >> 4) {
+                /* tslint:disable:no-switch-case-fall-through */
+                case 0:
+                case 1:
+                case 2:
+                case 3:
+                case 4:
+                case 5:
+                case 6:
+                case 7:
+                    charCode = leadingByte;
+                    break;
+                case 12:
+                case 13:
+                    let first = (b & 0x1F) << 6;
+                    let second = this.readByte(readingIndex) & 0x3F;
+                    readingIndex = this.addOrUndefined(readingIndex, 1);
+                    charCode = first | second;
+                    break;
+                case 14:
+                    let first2 = (b & 0x0F) << 12;
+                    let second2 = (this.readByte(readingIndex) & 0x3F) << 6;
+                    readingIndex = this.addOrUndefined(readingIndex, 1);
+                    let third2 = this.readByte(readingIndex) & 0x3F;
+                    readingIndex = this.addOrUndefined(readingIndex, 1);
+                    charCode = (first2 | second2 | third2);
+                    break;
+                default:
+                    throw new Error('Malformed UTF8 string');
             }
-            return new Buffer(result).toString('utf8');
+            result += String.fromCharCode(charCode);
         }
+        return result;
     }
 
     readUTFArray(pos?: number): string[] {

test/serialization/DefaultSerializersLiveTest.js

@@ -48,6 +48,38 @@ describe('Default serializers with live instance', function() {
         })
     });
 
+    it('emoji string', function () {
+        return map.put('key', '1⚐中💦2😭‍🙆😔5').then(function () {
+            return RC.executeOnController(cluster.id, _generateGet('key'), 1);
+        }).then(function (response) {
+            return expect(response.result.toString()).to.equal('1⚐中💦2😭‍🙆😔5');
+        });
+    });
+
+    it('utf8 characters test', function() {
+        return map.put('key', '\u0040\u0041\u01DF\u06A0\u12E0\u{1D306}').then(function () {
+            return RC.executeOnController(cluster.id, _generateGet('key'), 1);
+        }).then(function (response) {
+            return expect(response.result.toString()).to.equal('\u0040\u0041\u01DF\u06A0\u12E0\u{1D306}');
+        });
+    });
+
+    it('utf8 characters test with surrogates', function() {
+        return map.put('key', '\u0040\u0041\u01DF\u06A0\u12E0\uD834\uDF06').then(function () {
+            return RC.executeOnController(cluster.id, _generateGet('key'), 1);
+        }).then(function (response) {
+            return expect(response.result.toString()).to.equal('\u0040\u0041\u01DF\u06A0\u12E0\u{1D306}');
+        });
+    });
+
+    it('utf8 sample string test', function() {
+        return map.put('key', 'Iñtërnâtiônàlizætiøn').then(function () {
+            return RC.executeOnController(cluster.id, _generateGet('key'), 1);
+        }).then(function (response) {
+            return expect(response.result.toString()).to.equal('Iñtërnâtiônàlizætiøn');
+        });
+    });
+
     it('number', function () {
         return map.put('a', 23).then(function () {
             return RC.executeOnController(cluster.id, _generateGet('a'), 1);

test/serialization/DefaultSerializersTest.js

@@ -23,6 +23,9 @@ describe('Default serializers Test', function() {
         [''],
         '',
         'client',
+        '1⚐中💦2😭‍🙆😔5',
+        'Iñtërnâtiônàlizætiøn',
+        '\u0040\u0041\u01DF\u06A0\u12E0\u{1D306}',
         [12, 56, 54, 12],
         [43546.6, 2343.4, 8988,4],
         [23545798.6],