Skip to content

Commit 6084d92

Browse files
joshgoebeljoshgoebel
committed
allow utf-8 input in repl
1 parent 6db2f2c commit 6084d92

File tree

5 files changed

+119
-22
lines changed

5 files changed

+119
-22
lines changed

‎src/module/io.wren‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -228,6 +228,14 @@ class Stdin {
228228
}
229229
}
230230

231+
static readCodePoint() {
232+
return read_ {
233+
var codepoint = __buffered[0]
234+
__buffered = __buffered[1..-1]
235+
return codepoint
236+
}
237+
}
238+
231239
static readLine() {
232240
return read_ {
233241
// TODO: Handle Windows line separators.

‎src/module/io.wren.inc‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -230,6 +230,14 @@ static const char* ioModuleSource =
230230
" }\n"
231231
" }\n"
232232
"\n"
233+
" static readCodePoint() {\n"
234+
" return read_ {\n"
235+
" var codepoint = __buffered[0]\n"
236+
" __buffered = __buffered[1..-1]\n"
237+
" return codepoint\n"
238+
" }\n"
239+
" }\n"
240+
"\n"
233241
" static readLine() {\n"
234242
" return read_ {\n"
235243
" // TODO: Handle Windows line separators.\n"

‎src/module/repl.wren‎

Lines changed: 49 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,30 @@ import "meta" for Meta
22
import "io" for Stdin, Stdout
33
import "os" for Platform
44

5+
class CodePointList {
6+
construct new(s) {
7+
_string = s
8+
}
9+
[x] {
10+
if (x is Num) {
11+
return _string.codePoints.skip(x).take(1).map { |x| String.fromCodePoint(x) }.join()
12+
} else {
13+
if (x.to == -1) {
14+
return _string.codePoints.skip(x.from).map { |x| String.fromCodePoint(x) }.join()
15+
} else {
16+
var n = x.to - x.from
17+
return _string.codePoints.skip(x.from).take(n).map { |x| String.fromCodePoint(x) }.join()
18+
}
19+
}
20+
}
21+
// +(other) {
22+
// _string = _string + other
23+
// }
24+
// toString {
25+
// return _string
26+
// }
27+
}
28+
529
/// Abstract base class for the REPL. Manages the input line and history, but
630
/// does not render.
731
class Repl {
@@ -23,13 +47,15 @@ class Repl {
2347
refreshLine(false)
2448

2549
while (true) {
26-
var byte = Stdin.readByte()
27-
if (handleChar(byte)) break
50+
var codepoint = Stdin.readCodePoint()
51+
if (handleChar(codepoint)) break
2852
refreshLine(true)
2953
}
3054
}
3155

32-
handleChar(byte) {
56+
handleChar(codepoint) {
57+
var byte = codepoint.bytes[0]
58+
var codepoints
3359
if (byte == Chars.ctrlC) {
3460
System.print()
3561
return true
@@ -70,14 +96,18 @@ class Repl {
7096
} else if (byte == Chars.delete) {
7197
deleteLeft()
7298
} else if (byte >= Chars.space && byte <= Chars.tilde) {
73-
insertChar(byte)
99+
insertCodePoint(codepoint)
100+
} else if (byte > 127) {
101+
insertCodePoint(codepoint)
74102
} else if (byte == Chars.ctrlW) { // Handle Ctrl+w
75103
// Delete trailing spaces
76-
while (_cursor != 0 && _line[_cursor - 1] == " ") {
104+
codepoints = CodePointList.new(_line)
105+
while (_cursor != 0 && codepoints[_cursor - 1] == " ") {
77106
deleteLeft()
78107
}
108+
codepoints = CodePointList.new(_line)
79109
// Delete until the next space
80-
while (_cursor != 0 && _line[_cursor - 1] != " ") {
110+
while (_cursor != 0 && codepoints[_cursor - 1] != " ") {
81111
deleteLeft()
82112
}
83113
} else {
@@ -88,6 +118,12 @@ class Repl {
88118
return false
89119
}
90120

121+
insertCodePoint(cp) {
122+
var codePoints = CodePointList.new(_line)
123+
_line = codePoints[0..._cursor] + cp + codePoints[_cursor..-1]
124+
_cursor = _cursor + 1
125+
}
126+
91127
/// Inserts the character with [byte] value at the current cursor position.
92128
insertChar(byte) {
93129
var char = String.fromCodePoint(byte)
@@ -100,7 +136,8 @@ class Repl {
100136
if (_cursor == 0) return
101137

102138
// Delete the character before the cursor.
103-
_line = _line[0...(_cursor - 1)] + _line[_cursor..-1]
139+
var codePoints = CodePointList.new(_line)
140+
_line = codePoints[0...(_cursor - 1)] + codePoints[_cursor..-1]
104141
_cursor = _cursor - 1
105142
}
106143

@@ -109,7 +146,8 @@ class Repl {
109146
if (_cursor == _line.count) return
110147

111148
// Delete the character after the cursor.
112-
_line = _line[0..._cursor] + _line[(_cursor + 1)..-1]
149+
var codePoints = CodePointList.new(_line)
150+
_line = codePoints[0..._cursor] + codePoints[(_cursor + 1)..-1]
113151
}
114152

115153
handleEscapeBracket(byte) {
@@ -295,7 +333,8 @@ class AnsiRepl is Repl {
295333
super()
296334
}
297335

298-
handleChar(byte) {
336+
handleChar(codepoint) {
337+
var byte = codepoint.bytes[0]
299338
if (byte == Chars.ctrlA) {
300339
cursor = 0
301340
} else if (byte == Chars.ctrlB) {
@@ -317,7 +356,7 @@ class AnsiRepl is Repl {
317356
// TODO: ESC H and F to move to beginning and end of line. (Both ESC
318357
// [ and ESC 0 sequences?)
319358
// TODO: Ctrl-W delete previous word.
320-
return super.handleChar(byte)
359+
return super.handleChar(codepoint)
321360
}
322361

323362
return false

‎src/module/repl.wren.inc‎

Lines changed: 49 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,30 @@ static const char* replModuleSource =
44
"import \"io\" for Stdin, Stdout\n"
55
"import \"os\" for Platform\n"
66
"\n"
7+
"class CodePointList {\n"
8+
" construct new(s) {\n"
9+
" _string = s\n"
10+
" }\n"
11+
" [x] {\n"
12+
" if (x is Num) {\n"
13+
" return _string.codePoints.skip(x).take(1).map { |x| String.fromCodePoint(x) }.join()\n"
14+
" } else {\n"
15+
" if (x.to == -1) {\n"
16+
" return _string.codePoints.skip(x.from).map { |x| String.fromCodePoint(x) }.join()\n"
17+
" } else {\n"
18+
" var n = x.to - x.from\n"
19+
" return _string.codePoints.skip(x.from).take(n).map { |x| String.fromCodePoint(x) }.join()\n"
20+
" }\n"
21+
" }\n"
22+
" }\n"
23+
" // +(other) {\n"
24+
" // _string = _string + other\n"
25+
" // }\n"
26+
" // toString {\n"
27+
" // return _string\n"
28+
" // }\n"
29+
"}\n"
30+
"\n"
731
"/// Abstract base class for the REPL. Manages the input line and history, but\n"
832
"/// does not render.\n"
933
"class Repl {\n"
@@ -25,13 +49,15 @@ static const char* replModuleSource =
2549
" refreshLine(false)\n"
2650
"\n"
2751
" while (true) {\n"
28-
" var byte = Stdin.readByte()\n"
29-
" if (handleChar(byte)) break\n"
52+
" var codepoint = Stdin.readCodePoint()\n"
53+
" if (handleChar(codepoint)) break\n"
3054
" refreshLine(true)\n"
3155
" }\n"
3256
" }\n"
3357
"\n"
34-
" handleChar(byte) {\n"
58+
" handleChar(codepoint) {\n"
59+
" var byte = codepoint.bytes[0]\n"
60+
" var codepoints\n"
3561
" if (byte == Chars.ctrlC) {\n"
3662
" System.print()\n"
3763
" return true\n"
@@ -72,14 +98,18 @@ static const char* replModuleSource =
7298
" } else if (byte == Chars.delete) {\n"
7399
" deleteLeft()\n"
74100
" } else if (byte >= Chars.space && byte <= Chars.tilde) {\n"
75-
" insertChar(byte)\n"
101+
" insertCodePoint(codepoint)\n"
102+
" } else if (byte > 127) {\n"
103+
" insertCodePoint(codepoint)\n"
76104
" } else if (byte == Chars.ctrlW) { // Handle Ctrl+w\n"
77105
" // Delete trailing spaces\n"
78-
" while (_cursor != 0 && _line[_cursor - 1] == \" \") {\n"
106+
" codepoints = CodePointList.new(_line)\n"
107+
" while (_cursor != 0 && codepoints[_cursor - 1] == \" \") {\n"
79108
" deleteLeft()\n"
80109
" }\n"
110+
" codepoints = CodePointList.new(_line)\n"
81111
" // Delete until the next space\n"
82-
" while (_cursor != 0 && _line[_cursor - 1] != \" \") {\n"
112+
" while (_cursor != 0 && codepoints[_cursor - 1] != \" \") {\n"
83113
" deleteLeft()\n"
84114
" }\n"
85115
" } else {\n"
@@ -90,6 +120,12 @@ static const char* replModuleSource =
90120
" return false\n"
91121
" }\n"
92122
"\n"
123+
" insertCodePoint(cp) {\n"
124+
" var codePoints = CodePointList.new(_line)\n"
125+
" _line = codePoints[0..._cursor] + cp + codePoints[_cursor..-1]\n"
126+
" _cursor = _cursor + 1\n"
127+
" }\n"
128+
"\n"
93129
" /// Inserts the character with [byte] value at the current cursor position.\n"
94130
" insertChar(byte) {\n"
95131
" var char = String.fromCodePoint(byte)\n"
@@ -102,7 +138,8 @@ static const char* replModuleSource =
102138
" if (_cursor == 0) return\n"
103139
"\n"
104140
" // Delete the character before the cursor.\n"
105-
" _line = _line[0...(_cursor - 1)] + _line[_cursor..-1]\n"
141+
" var codePoints = CodePointList.new(_line)\n"
142+
" _line = codePoints[0...(_cursor - 1)] + codePoints[_cursor..-1]\n"
106143
" _cursor = _cursor - 1\n"
107144
" }\n"
108145
"\n"
@@ -111,7 +148,8 @@ static const char* replModuleSource =
111148
" if (_cursor == _line.count) return\n"
112149
"\n"
113150
" // Delete the character after the cursor.\n"
114-
" _line = _line[0..._cursor] + _line[(_cursor + 1)..-1]\n"
151+
" var codePoints = CodePointList.new(_line)\n"
152+
" _line = codePoints[0..._cursor] + codePoints[(_cursor + 1)..-1]\n"
115153
" }\n"
116154
"\n"
117155
" handleEscapeBracket(byte) {\n"
@@ -297,7 +335,8 @@ static const char* replModuleSource =
297335
" super()\n"
298336
" }\n"
299337
"\n"
300-
" handleChar(byte) {\n"
338+
" handleChar(codepoint) {\n"
339+
" var byte = codepoint.bytes[0]\n"
301340
" if (byte == Chars.ctrlA) {\n"
302341
" cursor = 0\n"
303342
" } else if (byte == Chars.ctrlB) {\n"
@@ -319,7 +358,7 @@ static const char* replModuleSource =
319358
" // TODO: ESC H and F to move to beginning and end of line. (Both ESC\n"
320359
" // [ and ESC 0 sequences?)\n"
321360
" // TODO: Ctrl-W delete previous word.\n"
322-
" return super.handleChar(byte)\n"
361+
" return super.handleChar(codepoint)\n"
323362
" }\n"
324363
"\n"
325364
" return false\n"

‎test/io/stdin/read_byte.wren‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,18 @@
11
import "io" for Stdin
22

3-
for (i in 1...13) {
3+
for (i in 1...15) {
44
System.print(Stdin.readByte())
55
}
66

7-
// stdin: first
7+
8+
// stdin: firstÿ
89
// expect: 102
910
// expect: 105
1011
// expect: 114
1112
// expect: 115
1213
// expect: 116
14+
// expect: 195
15+
// expect: 191
1316
// expect: 10
1417

1518
// stdin: second

0 commit comments

Comments
 (0)