Skip to content

Commit

Permalink
feat(cli-repl): add support for bracketed paste in REPL MONGOSH-1909 (#…
Browse files Browse the repository at this point in the history
…2328)

Bracketed paste allows us to receive a copy-pasted piece of mongosh
as a single block, rather than interpreting it line-by-line.
For now, this requires some monkey-patching of Node.js internals, so
a follow-up ticket will include work to upstream support for this
into Node.js core.
  • Loading branch information
addaleax authored Jan 29, 2025
1 parent 03a4dfc commit 4232a98
Show file tree
Hide file tree
Showing 12 changed files with 272 additions and 16 deletions.
7 changes: 7 additions & 0 deletions .evergreen/setup-env.sh
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,13 @@ export MONGOSH_TEST_ONLY_MAX_LOG_FILE_COUNT=100000
export IS_MONGOSH_EVERGREEN_CI=1
export DEBUG="mongodb*,$DEBUG"

# This is, weirdly enough, specifically set on s390x hosts, but messes
# with our e2e tests.
if [ x"$TERM" = x"dumb" ]; then
unset TERM
fi
echo "TERM variable is set to '${TERM:-}'"

if [ "$OS" != "Windows_NT" ]; then
if which realpath; then # No realpath on macOS, but also not needed there
export HOME="$(realpath "$HOME")" # Needed to de-confuse nvm when /home is a symlink
Expand Down
2 changes: 1 addition & 1 deletion package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion packages/cli-repl/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@
"access": "public"
},
"engines": {
"node": ">=16.15.0"
"node": ">=18.19.0"
},
"mongosh": {
"ciRequiredOptionalDependencies": {
Expand Down
39 changes: 39 additions & 0 deletions packages/cli-repl/src/async-repl.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -313,4 +313,43 @@ describe('AsyncRepl', function () {
});
});
});

it('does not run pasted text immediately', async function () {
const { input, output } = createDefaultAsyncRepl({
terminal: true,
useColors: false,
});

output.read(); // Read prompt so it doesn't mess with further output
input.write('\x1b[200~1234\n*5678\n\x1b[201~');
await tick();
// ESC[nG is horizontal cursor movement, ESC[nJ is cursor display reset
expect(output.read()).to.equal(
'1234\r\n\x1B[1G\x1B[0J... \x1B[5G*5678\r\n\x1B[1G\x1B[0J... \x1B[5G'
);
input.write('\n');
await tick();
// Contains the expected result after hitting newline
expect(output.read()).to.equal('\r\n7006652\n\x1B[1G\x1B[0J> \x1B[3G');
});

it('allows using ctrl+c to avoid running pasted text', async function () {
const { input, output } = createDefaultAsyncRepl({
terminal: true,
useColors: false,
});

output.read(); // Read prompt so it doesn't mess with further output
input.write('\x1b[200~1234\n*5678\n\x1b[201~');
await tick();
expect(output.read()).to.equal(
'1234\r\n\x1B[1G\x1B[0J... \x1B[5G*5678\r\n\x1B[1G\x1B[0J... \x1B[5G'
);
input.write('\x03'); // Ctrl+C
await tick();
expect(output.read()).to.equal('\r\n\x1b[1G\x1b[0J> \x1b[3G');
input.write('"foo";\n'); // Write something else
await tick();
expect(output.read()).to.equal(`"foo";\r\n'foo'\n\x1B[1G\x1B[0J> \x1B[3G`);
});
});
21 changes: 20 additions & 1 deletion packages/cli-repl/src/async-repl.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import type { ReadLineOptions } from 'readline';
import type { ReplOptions, REPLServer } from 'repl';
import type { start as originalStart } from 'repl';
import { promisify } from 'util';
import type { KeypressKey } from './repl-paste-support';

// Utility, inverse of Readonly<T>
type Mutable<T> = {
Expand Down Expand Up @@ -75,7 +76,9 @@ function getPrompt(repl: any): string {
export function start(opts: AsyncREPLOptions): REPLServer {
// 'repl' is not supported in startup snapshots yet.
// eslint-disable-next-line @typescript-eslint/no-var-requires
const { Recoverable, start: originalStart } = require('repl');
const { Recoverable, start: originalStart } =
// eslint-disable-next-line @typescript-eslint/consistent-type-imports
require('repl') as typeof import('repl');
const { asyncEval, wrapCallbackError = (err) => err, onAsyncSigint } = opts;
if (onAsyncSigint) {
(opts as ReplOptions).breakEvalOnSigint = true;
Expand All @@ -96,12 +99,28 @@ export function start(opts: AsyncREPLOptions): REPLServer {
return wasInRawMode;
};

// TODO(MONGOSH-1911): Upstream this feature into Node.js core.
let isPasting = false;
repl.input.on('keypress', (s: string, key: KeypressKey) => {
if (key.name === 'paste-start') {
isPasting = true;
} else if (key.name === 'paste-end') {
isPasting = false;
}
});

(repl as Mutable<typeof repl>).eval = (
input: string,
context: any,
filename: string,
callback: (err: Error | null, result?: any) => void
): void => {
if (isPasting) {
return callback(
new Recoverable(new Error('recoverable because pasting in progress'))
);
}

async function _eval() {
let previouslyInRawMode;

Expand Down
5 changes: 2 additions & 3 deletions packages/cli-repl/src/cli-repl.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2612,8 +2612,7 @@ describe('CliRepl', function () {
for (const { version, deprecated } of [
{ version: 'v20.5.1', deprecated: false },
{ version: '20.0.0', deprecated: false },
{ version: '18.0.0', deprecated: true },
{ version: '16.20.3', deprecated: true },
{ version: '18.19.0', deprecated: true },
]) {
delete (process as any).version;
(process as any).version = version;
Expand All @@ -2639,7 +2638,7 @@ describe('CliRepl', function () {

it('does not print any deprecation warning when CLI is ran with --quiet flag', async function () {
// Setting all the possible situation for a deprecation warning
process.version = '16.20.3';
process.version = '18.20.0';
process.versions.openssl = '1.1.11';
cliRepl.getGlibcVersion = () => '1.27';

Expand Down
7 changes: 4 additions & 3 deletions packages/cli-repl/src/line-by-line-input.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import { Readable } from 'stream';
import { StringDecoder } from 'string_decoder';
import type { ReadStream } from 'tty';

const LINE_ENDING_RE = /\r?\n|\r(?!\n)/;
const CTRL_C = '\u0003';
Expand All @@ -22,14 +23,14 @@ const CTRL_D = '\u0004';
* the proxied `tty.ReadStream`, forwarding all the characters.
*/
export class LineByLineInput extends Readable {
private _originalInput: NodeJS.ReadStream;
private _originalInput: Readable & Partial<ReadStream>;
private _forwarding: boolean;
private _blockOnNewLineEnabled: boolean;
private _charQueue: (string | null)[];
private _decoder: StringDecoder;
private _insidePushCalls: number;

constructor(readable: NodeJS.ReadStream) {
constructor(readable: Readable & Partial<ReadStream>) {
super();
this._originalInput = readable;
this._forwarding = true;
Expand Down Expand Up @@ -64,7 +65,7 @@ export class LineByLineInput extends Readable {
);

const proxy = new Proxy(readable, {
get: (target: NodeJS.ReadStream, property: string): any => {
get: (target: typeof readable, property: string): any => {
if (
typeof property === 'string' &&
!property.startsWith('_') &&
Expand Down
10 changes: 8 additions & 2 deletions packages/cli-repl/src/mongosh-repl.ts
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ import type { FormatOptions } from './format-output';
import { markTime } from './startup-timing';
import type { Context } from 'vm';
import { Script, createContext, runInContext } from 'vm';
import { installPasteSupport } from './repl-paste-support';

declare const __non_webpack_require__: any;

Expand Down Expand Up @@ -135,6 +136,7 @@ class MongoshNodeRepl implements EvaluationListener {
input: Readable;
lineByLineInput: LineByLineInput;
output: Writable;
outputFinishString = ''; // Can add ANSI escape codes to reset state from previously written ones
bus: MongoshBus;
nodeReplOptions: Partial<ReplOptions>;
shellCliOptions: Partial<MongoshCliOptions>;
Expand Down Expand Up @@ -251,7 +253,7 @@ class MongoshNodeRepl implements EvaluationListener {
// 'repl' is not supported in startup snapshots yet.
// eslint-disable-next-line @typescript-eslint/no-var-requires
start: require('pretty-repl').start,
input: this.lineByLineInput as unknown as Readable,
input: this.lineByLineInput,
output: this.output,
prompt: '',
writer: this.writer.bind(this),
Expand Down Expand Up @@ -387,6 +389,8 @@ class MongoshNodeRepl implements EvaluationListener {
const { repl, instanceState } = this.runtimeState();
if (!repl) return;

this.outputFinishString += installPasteSupport(repl);

const origReplCompleter = promisify(repl.completer.bind(repl)); // repl.completer is callback-style
const mongoshCompleter = completer.bind(
null,
Expand Down Expand Up @@ -1079,7 +1083,9 @@ class MongoshNodeRepl implements EvaluationListener {
await once(rs.repl, 'exit');
}
await rs.instanceState.close(true);
await new Promise((resolve) => this.output.write('', resolve));
await new Promise((resolve) =>
this.output.write(this.outputFinishString, resolve)
);
}
}

Expand Down
91 changes: 91 additions & 0 deletions packages/cli-repl/src/repl-paste-support.spec.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
import type { ReplOptions, REPLServer } from 'repl';
import { start } from 'repl';
import type { Readable, Writable } from 'stream';
import { PassThrough } from 'stream';
import { tick } from '../test/repl-helpers';
import { installPasteSupport } from './repl-paste-support';
import { expect } from 'chai';

function createTerminalRepl(extraOpts: Partial<ReplOptions> = {}): {
input: Writable;
output: Readable;
repl: REPLServer;
} {
const input = new PassThrough();
const output = new PassThrough({ encoding: 'utf8' });

const repl = start({
input: input,
output: output,
prompt: '> ',
terminal: true,
useColors: false,
...extraOpts,
});
return { input, output, repl };
}

describe('installPasteSupport', function () {
it('does nothing for non-terminal REPL instances', async function () {
const { repl, output } = createTerminalRepl({ terminal: false });
const onFinish = installPasteSupport(repl);
await tick();
expect(output.read()).to.equal('> ');
expect(onFinish).to.equal('');
});

it('prints a control character sequence that indicates support for bracketed paste', async function () {
const { repl, output } = createTerminalRepl();
const onFinish = installPasteSupport(repl);
await tick();
expect(output.read()).to.include('\x1B[?2004h');
expect(onFinish).to.include('\x1B[?2004l');
});

it('echoes back control characters in the input by default', async function () {
const { repl, input, output } = createTerminalRepl();
installPasteSupport(repl);
await tick();
output.read(); // Ignore prompt etc.
input.write('foo\x1b[Dbar'); // ESC[D = 1 character to the left
await tick();
expect(output.read()).to.equal(
'foo\x1B[1D\x1B[1G\x1B[0J> fobo\x1B[6G\x1B[1G\x1B[0J> fobao\x1B[7G\x1B[1G\x1B[0J> fobaro\x1B[8G'
);
});

it('ignores control characters in the input while pasting', async function () {
const { repl, input, output } = createTerminalRepl();
installPasteSupport(repl);
await tick();
output.read(); // Ignore prompt etc.
input.write('\x1b[200~foo\x1b[Dbar\x1b[201~'); // ESC[D = 1 character to the left
await tick();
expect(output.read()).to.equal('foobar');
});

it('resets to accepting control characters in the input after pasting', async function () {
const { repl, input, output } = createTerminalRepl();
installPasteSupport(repl);
await tick();
output.read();
input.write('\x1b[200~foo\x1b[Dbar\x1b[201~'); // ESC[D = 1 character to the left
await tick();
output.read();
input.write('foo\x1b[Dbar');
await tick();
expect(output.read()).to.equal(
'foo\x1B[1D\x1B[1G\x1B[0J> foobarfobo\x1B[12G\x1B[1G\x1B[0J> foobarfobao\x1B[13G\x1B[1G\x1B[0J> foobarfobaro\x1B[14G'
);
});

it('allows a few special characters while pasting', async function () {
const { repl, input, output } = createTerminalRepl();
installPasteSupport(repl);
await tick();
output.read();
input.write('\x1b[200~12*34\n_*_\n\x1b[201~');
await tick();
expect(output.read()).to.include((12 * 34) ** 2);
});
});
67 changes: 67 additions & 0 deletions packages/cli-repl/src/repl-paste-support.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
import type { REPLServer } from 'repl';

// https://github.com/nodejs/node/blob/d9786109b2a0982677135f0c146f6b591a0e4961/lib/internal/readline/utils.js#L90
// https://nodejs.org/api/readline.html#readlineemitkeypresseventsstream-interface
export type KeypressKey = {
sequence: string | null;
name: string | undefined;
ctrl: boolean;
meta: boolean;
shift: boolean;
code?: string;
};

function* prototypeChain(obj: unknown): Iterable<unknown> {
if (!obj) return;
yield obj;
yield* prototypeChain(Object.getPrototypeOf(obj));
}

export function installPasteSupport(repl: REPLServer): string {
if (!repl.terminal || process.env.TERM === 'dumb') return ''; // No paste needed in non-terminal environments

// TODO(MONGOSH-1911): Upstream as much of this into Node.js core as possible,
// both because of the value to the wider community but also because this is
// messing with Node.js REPL internals to a very unfortunate degree.
repl.output.write('\x1b[?2004h'); // Indicate support for paste mode
const onEnd = '\x1b[?2004l'; // End of support for paste mode
// Find the symbol used for the (internal) _ttyWrite method of readline.Interface
// https://github.com/nodejs/node/blob/d9786109b2a0982677135f0c146f6b591a0e4961/lib/internal/readline/interface.js#L1056
const ttyWriteKey = [...prototypeChain(repl)]
.flatMap((proto) => Object.getOwnPropertySymbols(proto))
.find((s) => String(s).includes('(_ttyWrite)'));
if (!ttyWriteKey)
throw new Error('Could not find _ttyWrite key on readline instance');
repl.input.on('keypress', (s: string, key: KeypressKey) => {
if (key.name === 'paste-start') {
if (Object.prototype.hasOwnProperty.call(repl, ttyWriteKey))
throw new Error(
'Unexpected existing own _ttyWrite key on readline instance'
);
const origTtyWrite = (repl as any)[ttyWriteKey];
Object.defineProperty(repl as any, ttyWriteKey, {
value: function (s: string, key: KeypressKey) {
if (key.ctrl || key.meta || key.code) {
// Special character or escape code sequence, ignore while pasting
return;
}
if (
key.name &&
key.name !== key.sequence?.toLowerCase() &&
!['tab', 'return', 'enter', 'space'].includes(key.name)
) {
// Special character or escape code sequence, ignore while pasting
return;
}
return origTtyWrite.call(this, s, key);
},
enumerable: false,
writable: true,
configurable: true,
});
} else if (key.name === 'paste-end') {
delete (repl as any)[ttyWriteKey];
}
});
return onEnd;
}
Loading

0 comments on commit 4232a98

Please sign in to comment.