repl: add basic syntax highlighting

Signed-off-by: avivkeller <me@aviv.sh>
PR-URL: #64591
Reviewed-By: James M Snell <jasnell@gmail.com>

Author: avivkeller

lib/internal/readline/interface.js

@@ -157,13 +157,15 @@ const kPreviousCursorCols = Symbol('_previousCursorCols');
 const kMultilineMove = Symbol('_multilineMove');
 const kPreviousPrevRows = Symbol('_previousPrevRows');
 const kAddNewLineOnTTY = Symbol('_addNewLineOnTTY');
+const kColorize = Symbol('_colorize');
 
 function InterfaceConstructor(input, output, completer, terminal) {
   this[kSawReturnAt] = 0;
   // TODO(BridgeAR): Document this property. The name is not ideal, so we
   // might want to expose an alias and document that instead.
   this.isCompletionEnabled = true;
   this[kSawKeyPress] = false;
+  this[kColorize] = undefined;
   this[kPreviousKey] = null;
   this.escapeCodeTimeout = ESCAPE_CODE_TIMEOUT;
   this.tabSize = 8;
@@ -186,6 +188,7 @@ function InterfaceConstructor(input, output, completer, terminal) {
     const historySize = input.historySize;
     const history = input.history;
     const removeHistoryDuplicates = input.removeHistoryDuplicates;
+    this[kColorize] = input.colorize;
 
     if (input.tabSize !== undefined) {
       validateUint32(input.tabSize, 'tabSize', true);
@@ -523,15 +526,18 @@ class Interface extends InterfaceConstructor {
     if (this[kIsMultiline]) {
       const lines = StringPrototypeSplit(this.line, '\n');
       // Write first line with normal prompt
-      this[kWriteToOutput](this[kPrompt] + lines[0]);
+      this[kWriteToOutput](this[kPrompt] +
+                           (this[kColorize]?.(lines[0]) ?? lines[0]));
 
       // For continuation lines, add the "|" prefix
       for (let i = 1; i < lines.length; i++) {
-        this[kWriteToOutput](`\n${kMultilinePrompt.description}` + lines[i]);
+        this[kWriteToOutput](`\n${kMultilinePrompt.description}` +
+                             (this[kColorize]?.(lines[i]) ?? lines[i]));
       }
     } else {
       // Write the prompt and the current buffer content.
-      this[kWriteToOutput](line);
+      this[kWriteToOutput](this[kPrompt] +
+                           (this[kColorize]?.(this.line) ?? this.line));
     }
 
     // Force terminal to allocate a new line
@@ -687,7 +693,11 @@ class Interface extends InterfaceConstructor {
         this.line += c;
       }
       this.cursor += c.length;
-      this[kWriteToOutput](c);
+      if (this[kColorize] === undefined) {
+        this[kWriteToOutput](c);
+      } else {
+        this[kRefreshLine]();
+      }
       return;
     }
     if (this.cursor < this.line.length) {
@@ -706,7 +716,7 @@ class Interface extends InterfaceConstructor {
       this.cursor += c.length;
       const newPos = this.getCursorPos();
 
-      if (oldPos.rows < newPos.rows) {
+      if (oldPos.rows < newPos.rows || this[kColorize] !== undefined) {
         this[kRefreshLine]();
       } else {
         this[kWriteToOutput](c);

lib/internal/repl/highlight.js

@@ -0,0 +1,89 @@
+'use strict';
+
+const {
+  FunctionPrototypeBind,
+  StringPrototypeSlice,
+} = primordials;
+
+const { stylizeWithColor } = require('internal/util/inspect');
+const { Parser } = require('internal/deps/acorn/acorn/dist/acorn');
+
+const tokenizer = FunctionPrototypeBind(Parser.tokenizer, Parser);
+
+function tokenStyle(token) {
+  const { label, keyword } = token.type;
+
+  if (keyword !== undefined) {
+    if (label === 'true' || label === 'false') return 'boolean';
+    if (label === 'null') return 'null';
+    if (label === 'import' || label === 'export') return 'module';
+    return 'special';
+  }
+
+  switch (label) {
+    case 'name':
+      switch (token.value) {
+        case 'undefined':
+          return 'undefined';
+        case 'NaN':
+        case 'Infinity':
+          return 'number';
+        case 'Symbol':
+          return 'symbol';
+        case 'Date':
+          return 'date';
+        default:
+          return undefined;
+      }
+    case 'num':
+      return 'number';
+    case 'bigint':
+      return 'bigint';
+    case 'string':
+    case 'template':
+    case '`':
+      return 'string';
+    case 'regexp':
+      return 'regexp';
+    default:
+      return undefined;
+  }
+}
+
+function highlight(code) {
+  if (code.length === 0) return code;
+
+  let result = '';
+  let offset = 0;
+  function write(start, end, style) {
+    result +=
+      StringPrototypeSlice(code, offset, start) +
+      stylizeWithColor(StringPrototypeSlice(code, start, end), style);
+    offset = end;
+  }
+
+  try {
+    const iterator = tokenizer(code, {
+      __proto__: null,
+      allowHashBang: true,
+      ecmaVersion: 'latest',
+      onComment(_block, _text, start, end) {
+        write(start, end, 'undefined');
+      },
+    });
+
+    for (const token of iterator) {
+      const style = tokenStyle(token);
+      if (style !== undefined) {
+        write(token.start, token.end, style);
+      }
+    }
+  } catch {
+    // Acorn throws for unfinished strings, templates, and comments. Tokens and
+    // comments reported before that point are still useful while typing.
+  }
+
+  return result + StringPrototypeSlice(code, offset);
+}
+
+module.exports = { highlight };

lib/internal/util/inspect.js

@@ -3057,6 +3057,7 @@ module.exports = {
   identicalSequenceRange,
   inspect,
   inspectDefaultOptions,
+  stylizeWithColor,
   format,
   formatWithOptions,
   getStringWidth,

lib/repl.js

@@ -98,6 +98,7 @@ const {
 } = require('internal/readline/utils');
 const { Console } = require('console');
 const { shouldColorize } = require('internal/util/colors');
+const { highlight } = require('internal/repl/highlight');
 const CJSModule = require('internal/modules/cjs/loader').Module;
 const { AsyncLocalStorage } = require('async_hooks');
 let debug = require('internal/util/debuglog').debuglog('repl', (fn) => {
@@ -246,6 +247,7 @@ class REPLServer extends Interface {
       terminal: options.terminal,
       historySize: options.historySize,
       prompt,
+      colorize: options.useColors ? highlight : undefined,
     });
 
     ObjectDefineProperty(this, 'inputStream', {

test/parallel/test-repl-highlight.js

@@ -0,0 +1,61 @@
+// Flags: --expose-internals
+'use strict';
+
+const common = require('../common');
+const assert = require('assert');
+const { inspect, stripVTControlCharacters } = require('util');
+const ArrayStream = require('../common/arraystream');
+const { highlight } = require('internal/repl/highlight');
+const { REPLServer } = require('repl');
+
+common.skipIfInspectorDisabled();
+
+assert.strictEqual(stripVTControlCharacters(highlight(
+  'const /* comment */ answer = 42; // comment',
+)), 'const /* comment */ answer = 42; // comment');
+
+assert.strictEqual(
+  highlight('const value = "text"'),
+  `\u001b[${inspect.colors.cyan[0]}mconst\u001b[${inspect.colors.cyan[1]}m value = ` +
+  `\u001b[${inspect.colors.green[0]}m"text"\u001b[${inspect.colors.green[1]}m`,
+);
+
+for (const source of [
+  'undefined',
+  'NaN',
+  'Infinity',
+  'Symbol.iterator',
+  'Date.now()',
+  'true',
+  'null',
+  '1n',
+  '/regexp/u',
+  'import("node:fs")',
+]) {
+  assert.notStrictEqual(highlight(source), source);
+}
+
+// Incomplete source is expected while the user is typing.
+highlight('const value = "');
+
+{
+  const input = new ArrayStream();
+  const output = new ArrayStream();
+  let rendered = '';
+  output.write = (chunk) => {
+    rendered += chunk;
+  };
+
+  const repl = new REPLServer({
+    input,
+    output,
+    prompt: '',
+    terminal: true,
+    useColors: false,
+  });
+  rendered = '';
+  repl.write('const');
+
+  assert.strictEqual(rendered, 'const');
+  repl.close();
+}

test/parallel/test-repl-multiline.js

@@ -2,6 +2,7 @@
 const common = require('../common');
 const assert = require('assert');
 const { startNewREPLServer } = require('../common/repl');
+const { stripVTControlCharacters } = require('util');
 
 const input = ['const foo = {', '};', 'foo'];
 
@@ -10,16 +11,16 @@ async function run({ useColors }) {
 
   await write(input);
 
-  const actual = output.accumulator.split('\n');
+  // The output contains various escape codes, including
+  // screen clears and others, so we trim them all to
+  // simplify this test.
+  const actual = stripVTControlCharacters(output.accumulator);
 
-  // Validate the output, which contains terminal escape codes.
-  assert.strictEqual(actual.length, 6);
-  assert.ok(actual[0].endsWith(input[0]));
-  assert.ok(actual[1].includes('| '));
-  assert.ok(actual[1].endsWith(input[1]));
-  assert.ok(actual[2].includes('undefined'));
-  assert.ok(actual[3].endsWith(input[2]));
-  assert.strictEqual(actual[4], '{}');
+  const firstStatementIdx = actual.indexOf(input.slice(0, 1).join('\n| '));
+  assert(firstStatementIdx > -1);
+
+  assert(actual.slice(firstStatementIdx).includes('undefined'));
+  assert(actual.slice(firstStatementIdx).includes('foo'));
 
   replServer.on('exit', common.mustCall());
   replServer.close();

test/parallel/test-repl-preview-newlines.mjs

@@ -4,7 +4,14 @@ import { startNewREPLServer } from '../common/repl.js';
 
 common.skipIfInspectorDisabled();
 
-const { output, run, replServer } = startNewREPLServer({ useColors: true });
+// Ignore terminal settings so the preview remains active under TERM=dumb.
+process.env.TERM = '';
+
+// Keep syntax highlighting out of this test so it only covers preview layout.
+// Preview and result colors are enabled after readline is initialized.
+const { output, run, replServer } = startNewREPLServer({ useColors: false });
+replServer.useColors = true;
+replServer.writer.options.colors = true;
 
 for (const char of ['\\n', '\\v', '\\r']) {
   output.accumulator = '';

test/parallel/test-repl-preview.mjs

@@ -83,9 +83,13 @@ async function tests(options) {
     prompt: PROMPT,
     stream: new REPLStream(),
     ignoreUndefined: true,
-    useColors: true,
+    // Keep syntax highlighting out of these preview transcript assertions.
+    // Preview and result colors are enabled after readline is initialized.
+    useColors: false,
     ...options
   });
+  repl.useColors = true;
+  repl.writer.options.colors = true;
 
   await runAndWait([
     'function foo(x) { return x; } ' +

test/parallel/test-repl-reverse-search.js

@@ -324,7 +324,9 @@ function runTest() {
     }),
     completer: opts.completer,
     prompt,
-    useColors: opts.useColors || false,
+    // Keep syntax highlighting out of the reverse-search transcript. Result
+    // colors are enabled below after readline has been initialized.
+    useColors: false,
     terminal: true
   }, common.mustCall((err, repl) => {
     if (err) {
@@ -346,6 +348,11 @@ function runTest() {
       setImmediate(runTestWrap, true);
     }));
 
+    if (opts.useColors) {
+      repl.useColors = true;
+      repl.writer.options.colors = true;
+    }
+
     if (opts.columns) {
       Object.defineProperty(repl, 'columns', {
         value: opts.columns,