version bump 1.3.6: codepage cmd fixes

Author: SheetJSDev

NOTES.md

@@ -0,0 +1,67 @@
+# Verifying Codepages
+
+After installing every language pack in Windows 7, many codepages are available
+via the .NET System.Text.Encoding class.  The MakeEncoding.cs source included with
+the project generates a full manifest that can be parsed into a mapping table.
+
+The included `nls2tbl` script extracts data from the various `C_#####.NLS` files
+available in the system or system32 directories in various versions of Windows.
+
+Many codepages are also available in various iconv libraries, but there are some
+differences.  For example, some codepages break ASCII by using the Arabic percent
+sign ٪ U+066A but other libraries assume they preserve the ASCII space.  
+
+# Missing Codepages
+
+The following codepages are not implemented. Normative references may not be
+available in all cases.  Furthermore, other software packages are known to hack
+certain codepages (for example, Mozilla treats ASMO-708 as an alias of Arabic
+ISO-8869-6 when in fact there are many differences), so all implementations
+*should* be cleanroom when possible.
+
+- 709   Arabic (ASMO-449+, BCON V4)
+- 710   Arabic - Transparent Arabic
+- 50229 ISO 2022 Traditional Chinese
+- 50930 EBCDIC Japanese (Katakana) Extended
+- 50931 EBCDIC US-Canada and Japanese
+- 50933 EBCDIC Korean Extended and Korean
+- 50935 EBCDIC Simplified Chinese Extended and Simplified Chinese
+- 50936 EBCDIC Simplified Chinese
+- 50937 EBCDIC US-Canada and Traditional Chinese
+- 50939 EBCDIC Japanese (Latin) Extended and Japanese
+- 51950 EUC Traditional Chinese
+
+Each version of Windows adds a few and removes a few codepages, so the missing
+codepages most likely reside in a specific version that we may not be able to
+obtain.  These notes document our progress.
+
+## Arabic codepages 709-710
+
+These codepages are not available in the Arabic version of Windows XP.  They may
+be available in the Arabic versions of MS-DOS or Windows 3.1/95/98/2000.
+
+The "Code Page and Text Layout Conversion Utility" CONVTEXT.EXE ships with some
+versions of Office.  It can convert from the various codepages to ANSI.
+
+To produce a UTF16LE (1200) manifest, convert from the relevant codepage to ANSI
+and then convert from ANSI to "Unicode using Arabic ANSI Code Page".
+
+Since there is no way to convert directly to unicode using the tool, CONVTEXT is
+useful only for the characters which exist in both the relevant codepage and in
+codepage 1256.  There are various non-Microsoft sources which claim to document
+both codepages, but there is no way to verify the claim.
+
+## EUC Traditional Chinese 51950
+
+The raw NLS file C_51950.NLS supposedly exists, although there is no way for a US
+version of Windows to obtain the file.  As with the Arabic Codepages, most likely
+the manifest is only available in Chinese versions of Windows 95/98/2000
+
+### ISO 2022 Traditional Chinese 50229
+
+Some sources claim 50229 is ISO-2022-TW and others claim it is ISO-2022-CN.
+
+### EBCDIC Codepages 50930-50939
+
+WHATWG claims that the supposed-EBCDIC codepages are really hybrids of ASCII (even
+though the Microsoft name suggests they should be the same as the originals)

README.md

@@ -245,25 +245,9 @@ Note that MakeEncoding.cs deviates from unicode.org for some codepages.  In the
 case of direct conflicts, unicode.org takes precedence.  In cases where the
 unicode.org listing does not prescribe a value, MakeEncoding.cs value is used.
 
-## Missing Codepages
-
-The following codepages are not implemented. Normative references may not be
-available in all cases.  Furthermore, other software packages are known to hack
-certain codepages (for example, Mozilla treats ASMO-708 as an alias of Arabic
-ISO-8869-6 when in fact there are many differences), so all implementations
-*should* be cleanroom when possible.
-
-- 709   Arabic (ASMO-449+, BCON V4)
-- 710   Arabic - Transparent Arabic
-- 50229 ISO 2022 Traditional Chinese
-- 50930 EBCDIC Japanese (Katakana) Extended
-- 50931 EBCDIC US-Canada and Japanese
-- 50933 EBCDIC Korean Extended and Korean
-- 50935 EBCDIC Simplified Chinese Extended and Simplified Chinese
-- 50936 EBCDIC Simplified Chinese
-- 50937 EBCDIC US-Canada and Traditional Chinese
-- 50939 EBCDIC Japanese (Latin) Extended and Japanese
-- 51950 EUC Traditional Chinese
+NLS refers to the National Language Support files supplied in various versions of
+Windows.  In older versions of Windows (e.g. Windows 98) these files followed the
+pattern `CP_#.NLS`, but newer versions use the pattern `C_#.NLS`.
 
 ## Sources
 

bin/codepage.njs

@@ -10,6 +10,7 @@ program
 	.option('-t, --to-code <code>', 'codepage of output (default 65001 utf8)')
 	.option('-o, --output <file>', 'output file (<file>.<to> if specified)')
 	.option('-B, --bom', 'write BOM (for unicode codepages)')
+	.option('-F, --force', 'force writing to stdout for non-utf8 codepages')
 	.option('-l, --list', 'List supported codepages');
 
 program.on('--help', function() {
@@ -25,8 +26,8 @@ if(program.list) {
 	process.exit();
 }
 
-var fr = program.fromCode || 65001;
-var to = program.toCode || 65001;
+var fr = +program.fromCode || 65001;
+var to = +program.toCode || 65001;
 var f = program.args[0];
 var o = program.output;
 
@@ -53,10 +54,31 @@ function process_text(text) {
 		65001: new Buffer([0xEF, 0xBB, 0xBF])
 	}
 
-	if(!program.toCode && !o) console.log(dec.toString('utf8'));
-	else if(!program.bom || !bom[fr]) fs.writeFileSync(o || (f + "." + to), codepage.utils.encode(to, dec));
+	var mybom = (program.bom && bom[fr] ? bom[fr] : "");
+	var out = to === 65001 ? dec.toString('utf8') : codepage.utils.encode(to, dec);
+
+	/* if output file is specified */
+	if(o) writefile(o, out, mybom);
+	/* utf8 -> print to stdout */
+	else if(to === 65001) logit(out, mybom);
+	/* stdout piped to process -> print */
+	else if(!process.stdout.isTTY) logit(out, mybom);
+	/* forced */
+	else if(program.force) logit(out, mybom);
+	/* input file specified -> write to file */
+	else if(f !== "-") writefile(f + "." + to, out, mybom);
 	else {
-		fs.writeFileSync(o || (f + "." + to), bom[fr]);
-		fs.appendFileSync(o || (f + "." + to), codepage.utils.encode(to, dec));
+		console.error('codepage: use force (-F, --force) to print ' + to + ' codes');
+		process.exit(14);
 	}
 }
+
+function logit(out, bom) {
+	process.stdout.write(bom);
+	process.stdout.write(out);
+}
+
+function writefile(o, out, bom) {
+	fs.writeFileSync(o, bom);
+	fs.appendFileSync(o, out);
+}

codepage.md

@@ -719,7 +719,7 @@ describe('failures', function() {
 ```json>package.json
 {
   "name": "codepage",
-  "version": "1.3.5",
+  "version": "1.3.6",
   "author": "SheetJS",
   "description": "pure-JS library to handle codepages",
   "keywords": [ "codepage", "iconv", "convert", "strings" ],