utf16.js 5.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202
  1. // == UTF16-BE codec. ==========================================================
  2. exports.utf16be = function(options) {
  3. return {
  4. encoder: utf16beEncoder,
  5. decoder: utf16beDecoder,
  6. bom: new Buffer([0xFE, 0xFF]),
  7. };
  8. };
  9. // -- Encoding
  10. function utf16beEncoder(options) {
  11. return {
  12. write: utf16beEncoderWrite,
  13. end: function() {},
  14. }
  15. }
  16. function utf16beEncoderWrite(str) {
  17. var buf = new Buffer(str, 'ucs2');
  18. for (var i = 0; i < buf.length; i += 2) {
  19. var tmp = buf[i]; buf[i] = buf[i+1]; buf[i+1] = tmp;
  20. }
  21. return buf;
  22. }
  23. // -- Decoding
  24. function utf16beDecoder(options) {
  25. return {
  26. write: utf16beDecoderWrite,
  27. end: function() {},
  28. overflowByte: -1,
  29. };
  30. }
  31. function utf16beDecoderWrite(buf) {
  32. if (buf.length == 0)
  33. return '';
  34. var buf2 = new Buffer(buf.length + 1),
  35. i = 0, j = 0;
  36. if (this.overflowByte !== -1) {
  37. buf2[0] = buf[0];
  38. buf2[1] = this.overflowByte;
  39. i = 1; j = 2;
  40. }
  41. for (; i < buf.length-1; i += 2, j+= 2) {
  42. buf2[j] = buf[i+1];
  43. buf2[j+1] = buf[i];
  44. }
  45. this.overflowByte = (i == buf.length-1) ? buf[buf.length-1] : -1;
  46. return buf2.slice(0, j).toString('ucs2');
  47. }
  48. // == UTF-16 codec =============================================================
  49. // Decoder chooses automatically from UTF-16LE and UTF-16BE using BOM and space-based heuristic.
  50. // Defaults to UTF-16BE, according to RFC 2781, although it is against some industry practices, see
  51. // http://en.wikipedia.org/wiki/UTF-16 and http://encoding.spec.whatwg.org/#utf-16le
  52. // Decoder default can be changed: iconv.decode(buf, 'utf16', {default: 'utf-16le'});
  53. // Encoder prepends BOM and uses UTF-16BE.
  54. // Endianness can also be changed: iconv.encode(str, 'utf16', {use: 'utf-16le'});
  55. exports.utf16 = function(options) {
  56. return {
  57. encoder: utf16Encoder,
  58. decoder: utf16Decoder,
  59. getCodec: options.iconv.getCodec,
  60. };
  61. };
  62. // -- Encoding
  63. function utf16Encoder(options) {
  64. options = options || {};
  65. var codec = this.getCodec(options.use || 'utf-16be');
  66. if (!codec.bom)
  67. throw new Error("iconv-lite: in UTF-16 encoder, 'use' parameter should be either UTF-16BE or UTF16-LE.");
  68. return {
  69. write: utf16EncoderWrite,
  70. end: utf16EncoderEnd,
  71. bom: codec.bom,
  72. internalEncoder: codec.encoder(options),
  73. };
  74. }
  75. function utf16EncoderWrite(str) {
  76. var buf = this.internalEncoder.write(str);
  77. if (this.bom) {
  78. buf = Buffer.concat([this.bom, buf]);
  79. this.bom = null;
  80. }
  81. return buf;
  82. }
  83. function utf16EncoderEnd() {
  84. return this.internalEncoder.end();
  85. }
  86. // -- Decoding
  87. function utf16Decoder(options) {
  88. return {
  89. write: utf16DecoderWrite,
  90. end: utf16DecoderEnd,
  91. internalDecoder: null,
  92. initialBytes: [],
  93. initialBytesLen: 0,
  94. options: options || {},
  95. getCodec: this.getCodec,
  96. };
  97. }
  98. function utf16DecoderWrite(buf) {
  99. if (this.internalDecoder)
  100. return this.internalDecoder.write(buf);
  101. // Codec is not chosen yet. Accumulate initial bytes.
  102. this.initialBytes.push(buf);
  103. this.initialBytesLen += buf.length;
  104. if (this.initialBytesLen < 16) // We need > 2 bytes to use space heuristic (see below)
  105. return '';
  106. // We have enough bytes -> decide endianness.
  107. return utf16DecoderDecideEndianness.call(this);
  108. }
  109. function utf16DecoderEnd() {
  110. if (this.internalDecoder)
  111. return this.internalDecoder.end();
  112. var res = utf16DecoderDecideEndianness.call(this);
  113. var trail;
  114. if (this.internalDecoder)
  115. trail = this.internalDecoder.end();
  116. return (trail && trail.length > 0) ? (res + trail) : res;
  117. }
  118. function utf16DecoderDecideEndianness() {
  119. var buf = Buffer.concat(this.initialBytes);
  120. this.initialBytes.length = this.initialBytesLen = 0;
  121. if (buf.length < 2)
  122. return ''; // Not a valid UTF-16 sequence anyway.
  123. // Default encoding.
  124. var enc = this.options.default || 'utf-16be';
  125. // Check BOM.
  126. if (buf[0] == 0xFE && buf[1] == 0xFF) { // UTF-16BE BOM
  127. enc = 'utf-16be'; buf = buf.slice(2);
  128. }
  129. else if (buf[0] == 0xFF && buf[1] == 0xFE) { // UTF-16LE BOM
  130. enc = 'utf-16le'; buf = buf.slice(2);
  131. }
  132. else {
  133. // No BOM found. Try to deduce encoding from initial content.
  134. // Most of the time, the content has spaces (U+0020), but the opposite (U+2000) is very uncommon.
  135. // So, we count spaces as if it was LE or BE, and decide from that.
  136. var spaces = [0, 0], // Counts of space chars in both positions
  137. _len = Math.min(buf.length - (buf.length % 2), 64); // Len is always even.
  138. for (var i = 0; i < _len; i += 2) {
  139. if (buf[i] == 0x00 && buf[i+1] == 0x20) spaces[0]++;
  140. if (buf[i] == 0x20 && buf[i+1] == 0x00) spaces[1]++;
  141. }
  142. if (spaces[0] > 0 && spaces[1] == 0)
  143. enc = 'utf-16be';
  144. else if (spaces[0] == 0 && spaces[1] > 0)
  145. enc = 'utf-16le';
  146. }
  147. this.internalDecoder = this.getCodec(enc).decoder(this.options);
  148. return this.internalDecoder.write(buf);
  149. }