Introduces MATLAB language syntax highlighting for JupyterLab 4.

Author: philipc-mw

.github/workflows/publish-jupyter-matlab-proxy.yml

@@ -44,4 +44,4 @@ jobs:
         run: python3 -m hatch build -t wheel
 
       - name: Publish to PyPI.
-        uses: pypa/gh-action-pypi-publish@release/v1
+        uses: pypa/gh-action-pypi-publish@release/v1

.github/workflows/run-e2e-tests.yml

@@ -48,7 +48,7 @@ jobs:
         run: |
           python3 -m pip install --upgrade pip
           python3 -m pip install ".[dev]"
-          python3 -m pip install "jupyterlab>3.1.0,<4.0.0"
+          python3 -m pip install "jupyterlab>4.0.0,<5.0.0"
 
       - name: Install playwright browsers
         run: npx playwright install --with-deps

.github/workflows/run-unit-tests.yml

@@ -78,6 +78,32 @@ jobs:
           path: ./coverage.xml
           retention-days: 5
 
+  lezer_unit_tests:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest]
+
+    runs-on: ${{ matrix.os }}
+    defaults:
+      run:
+        working-directory: src/jupyter_matlab_labextension/src/lezer-matlab/test/
+    env:
+      NODE_VERSION: 18
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: ${{ env.NODE_VERSION }}
+
+      - name: Install node dependencies and build parser
+        run: npm install
+
+      - name: Run Lezer tests
+        run: npm test
+
   upload_code_coverage:
     name: "Upload Code Coverage using codecov"
     needs: [python_unit_tests]

.github/workflows/test-jupyter-matlab-proxy.yml

@@ -26,4 +26,4 @@ jobs:
   call-e2e-tests:
     # Run the end-to-end tests
     uses: ./.github/workflows/run-e2e-tests.yml
-    secrets: inherit
+    secrets: inherit

.gitignore

@@ -17,3 +17,6 @@ test-results
 jupyterlab.log
 jlab.pid
 licensing-logs
+.yarn
+src/jupyter_matlab_labextension/src/lezer-matlab/src/parser.js
+src/jupyter_matlab_labextension/src/lezer-matlab/src/parser.terms.js

README.md

@@ -107,15 +107,11 @@ Install Jupyter Notebook or JupyterLab:
   # For Jupyter Notebook
   python -m pip install notebook
 
-  # For JupyterLab 3
-  python -m pip install 'jupyterlab>=3.0.0,<4.0.0a0'
+  # For JupyterLab 4
+  python -m pip install jupyterlab
   ```
 
-Note: the package allows you to execute MATLAB code in both JupyterLab 3 and JupyterLab 4, but syntax highlighting and auto indentation are currently only supported on JupyterLab 3.
-
-> :warning: **Functionality being removed or changed.** </br>
-> Project Jupyter ended maintenance of JupyterLab 3 on May 15, 2024. For details, see [JupyterLab 3 End of Maintenance](https://blog.jupyter.org/jupyterlab-3-end-of-maintenance-879778927db2).
-> Starting from a future release of MATLAB Integration _for Jupyter_, editor features such as syntax highlighting and autoindentation will be removed from JupyterLab 3 and introduced for JupyterLab 4. Execution of MATLAB code will remain available with both JupyterLab 3 and JupyterLab 4.
+Note: you can use this package to execute MATLAB code in both JupyterLab 3 and 4, but syntax highlighting is only supported on JupyterLab 4.
 
 
 Open your Jupyter environment by starting Jupyter Notebook or JupyterLab.
@@ -186,7 +182,7 @@ You can also edit MATLAB `.m` files in JupyterLab. Click the `MATLAB File` butto
 
 <p align="center"><img width="300" src="https://github.com/mathworks/jupyter-matlab-proxy/raw/main/img/matlabfile-icon.png"></p>
 
-This opens an untitled `.m` file where you can write MATLAB code with syntax highlighting and auto indentation.
+This opens an untitled `.m` file where you can write MATLAB code with syntax highlighting.
 
 <p align="center"><img width="600" src="https://github.com/mathworks/jupyter-matlab-proxy/raw/main/img/new-matlab-file.png"></p>
 

pyproject.toml

@@ -1,7 +1,7 @@
 # Copyright 2023-2024 The MathWorks, Inc.
 
 [build-system]
-requires = ["jupyterlab>=3.1,<4.0.0", "hatchling"]
+requires = ["hatchling>=1.5.0", "jupyterlab>=4.0.0,<5", "hatch-nodejs-version>=0.3.2"]
 build-backend = "hatchling.build"
 
 [project]
@@ -63,6 +63,8 @@ matlab = "jupyter_matlab_proxy:setup_matlab"
 [project.entry-points.matlab_proxy_configs]
 Jupyter = "jupyter_matlab_proxy.jupyter_config:config"
 
+[tool.hatch.version]
+source = "nodejs"
 
 [tool.hatch.build.targets.wheel]
 packages = ["src/jupyter_matlab_kernel", "src/jupyter_matlab_proxy"]

src/jupyter_matlab_labextension/.yarnrc.yml

@@ -0,0 +1 @@
+nodeLinker: node-modules

src/jupyter_matlab_labextension/package.json

@@ -1,13 +1,13 @@
 {
   "name": "jupyter_matlab_labextension",
-  "version": "0.1.0",
+  "version": "1.0.0",
   "description": "A JupyterLab extension.",
   "keywords": [
     "jupyter",
     "jupyterlab",
     "jupyterlab-extension"
   ],
-  "license": "BSD-3-Clause",
+  "license": "SEE LICENSE IN LICENSE.md",
   "author": {
     "name": "The MathWorks Inc.",
     "email": "jupyter-support@mathworks.com"
@@ -28,7 +28,8 @@
     "build:prod": "jlpm clean && jlpm build:lib && jlpm build:labextension",
     "build:labextension": "jupyter labextension build .",
     "build:labextension:dev": "jupyter labextension build --development True .",
-    "build:lib": "tsc",
+    "build:lib": "jlpm build:lezer && tsc",
+    "build:lezer": "cd src/lezer-matlab && npm run build",
     "clean": "jlpm clean:lib",
     "clean:lib": "rimraf lib tsconfig.tsbuildinfo",
     "clean:lintcache": "rimraf .eslintcache .stylelintcache",
@@ -50,30 +51,39 @@
     "audit:fix": "npx yarn-audit-fix"
   },
   "dependencies": {
-    "@jupyterlab/application": "^3.1.0",
-    "@jupyterlab/launcher-extension": "",
-    "@jupyterlab/notebook": "",
-    "@types/codemirror": ""
+    "@codemirror/language": "^6.0.0",
+    "@jupyterlab/application": "^4.0.0",
+    "@jupyterlab/apputils": "^4.0.0",
+    "@jupyterlab/codemirror": "^4.0.0",
+    "@jupyterlab/coreutils": "^6.0.0",
+    "@jupyterlab/docregistry": "^4.0.0",
+    "@jupyterlab/launcher": "^4.0.0",
+    "@jupyterlab/notebook": "^4.0.0",
+    "@jupyterlab/ui-components": "^4.0.0",
+    "@lumino/coreutils": "^2.0.0",
+    "@lumino/disposable": "^2.0.0",
+    "@rollup/plugin-node-resolve": "^9.0.0"
   },
   "devDependencies": {
-    "@jupyterlab/builder": "^3.1.0",
-    "@typescript-eslint/eslint-plugin": "^5.0.0",
-    "@typescript-eslint/parser": "^5.30.7",
-    "eslint": "^8.0.1",
+    "@jupyterlab/builder": ">=4.0.0",
+    "@typescript-eslint/eslint-plugin": "^5.62.0",
+    "@typescript-eslint/parser": "^5.62.0",
+    "eslint": "^8.57.0",
     "eslint-config-standard": "^17.0.0",
     "eslint-config-standard-with-typescript": "^22.0.0",
     "eslint-plugin-import": "^2.25.2",
     "eslint-plugin-n": "^15.0.0",
     "eslint-plugin-node": "^11.1.0",
     "eslint-plugin-promise": "^6.0.0",
     "npm-run-all": "^4.1.5",
-    "prettier": "^2.1.1",
-    "rimraf": "^3.0.2",
+    "prettier": "^2.8.7",
+    "rimraf": "^4.4.1",
+    "rollup": "^2.52.2",
     "semver": ">=5.7.2",
-    "typescript": "~4.1.3",
+    "typescript": "~5.0.2",
+    "ws": "^7.5.10",
     "yarn-audit-fix": "^10.0.1",
-    "yarn-deduplicate": "^6.0.2",
-    "ws": "^7.5.10"
+    "yarn-deduplicate": "^6.0.2"
   },
   "sideEffects": [
     "style/*.css",
@@ -90,7 +100,7 @@
   "jupyter-releaser": {
     "hooks": {
       "before-build-npm": [
-        "python -m pip install jupyterlab~=3.1",
+        "python -m pip install jupyterlab~=4",
         "jlpm"
       ],
       "before-build-python": [

src/jupyter_matlab_labextension/src/codemirror-lang-matlab/codemirror-lang-matlab.ts

@@ -0,0 +1,19 @@
+// Copyright 2024 The MathWorks, Inc.
+
+import { parser } from '../lezer-matlab/dist/index';
+import { LRLanguage, LanguageSupport } from '@codemirror/language';
+
+// Define a CodeMirror language from the Lezer parser.
+// https://codemirror.net/docs/ref/#language.LRLanguage
+export const matlabLanguage = LRLanguage.define({
+    name: 'matlab',
+    parser,
+    languageData: {
+        commentTokens: { line: '%' }
+    }
+});
+
+// MATLAB language support
+export function matlab () {
+    return new LanguageSupport(matlabLanguage);
+}

src/jupyter_matlab_labextension/src/index.ts

@@ -1,9 +1,9 @@
-// Copyright 2023 The MathWorks, Inc.
+// Copyright 2023-2024 The MathWorks, Inc.
 
 import { JupyterFrontEndPlugin } from '@jupyterlab/application';
 import { matlabToolbarButtonPlugin } from './matlab_browser_button';
 import { matlabMFilesPlugin } from './matlab_files';
-import { matlabCodeMirrorPlugin } from './matlab_cm_mode';
+import { matlabCodeMirror6Plugin } from './matlab_cm6_mode';
 
-const plugins: JupyterFrontEndPlugin<any>[] = [matlabToolbarButtonPlugin, matlabMFilesPlugin, matlabCodeMirrorPlugin];
+const plugins: JupyterFrontEndPlugin<any>[] = [matlabToolbarButtonPlugin, matlabMFilesPlugin, matlabCodeMirror6Plugin];
 export default plugins;

src/jupyter_matlab_labextension/src/lezer-matlab/dist/index.d.cts

@@ -0,0 +1,4 @@
+// Copyright 2024 The MathWorks, Inc.
+import {LRParser} from "@lezer/lr"
+
+export const parser: LRParser

src/jupyter_matlab_labextension/src/lezer-matlab/dist/index.d.ts

@@ -0,0 +1,4 @@
+// Copyright 2024 The MathWorks, Inc.
+import { LRParser } from '@lezer/lr';
+
+export const parser: LRParser;

src/jupyter_matlab_labextension/src/lezer-matlab/package.json

@@ -0,0 +1,37 @@
+
+{
+  "name": "@lezer/matlab",
+  "version": "1.0.0",
+  "description": "Lezer-based MATLAB grammar",
+  "main": "dist/index.cjs",
+  "type": "module",
+  "exports": {
+    "import": "./dist/index.js",
+    "require": "./dist/index.cjs"
+  },
+  "module": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "author": {
+    "name": "The MathWorks Inc.",
+    "email": "jupyter-support@mathworks.com"
+  },
+  "license": "SEE LICENSE IN LICENSE.md",
+  "devDependencies": {
+    "@lezer/generator": "^1.0.0",
+    "mocha": "^10.2.0",
+    "rollup": "^2.52.2",
+    "@rollup/plugin-node-resolve": "^9.0.0"
+  },
+  "dependencies": {
+    "@lezer/common": "^1.2.0",
+    "@lezer/lr": "^1.0.0",
+    "@lezer/highlight": "^1.0.0"
+  },
+  "scripts": {
+    "build": "lezer-generator src/matlab.grammar -o src/parser && rollup -c && npm run copy-lezer-files-to-build-on-windows",
+    "build-debug": "lezer-generator src/matlab.grammar --names -o src/parser && rollup -c && npm run copy-lezer-files-to-build-on-windows",
+    "copy-lezer-files-to-build-on-windows": "cp src/*.js dist/",
+    "prepare": "npm run build",
+    "test": "mocha test/test-*.js"
+  }
+}

src/jupyter_matlab_labextension/src/lezer-matlab/rollup.config.js

@@ -0,0 +1,18 @@
+// Copyright 2024 The MathWorks, Inc.
+
+import { nodeResolve } from '@rollup/plugin-node-resolve';
+
+export default {
+  input: "./src/parser.js",
+  output: [{
+    format: "cjs",
+    file: "./dist/index.cjs"
+  }, {
+    format: "es",
+    file: "./dist/index.js"
+  }],
+  external(id) { return !/^[\.\/]/.test(id) },
+  plugins: [
+    nodeResolve()
+  ]
+}

src/jupyter_matlab_labextension/src/lezer-matlab/src/highlight.js

@@ -0,0 +1,16 @@
+// Copyright 2024 The MathWorks, Inc.
+
+import { styleTags, tags as t } from '@lezer/highlight';
+
+// Associate nodes in the Lezer tree with styles.
+// https://lezer.codemirror.net/docs/ref/#highlight.styleTags
+export const matlabHighlighting = styleTags({
+    Keyword: t.keyword,
+    VariableName: t.variableName,
+    LineComment: t.comment,
+    MultilineComment: t.comment,
+    String: t.string,
+    '( )': t.paren,
+    '[ ]': t.squareBracket,
+    '{ }': t.brace
+});

src/jupyter_matlab_labextension/src/lezer-matlab/src/matlab.grammar

@@ -0,0 +1,46 @@
+// Copyright 2024 The MathWorks, Inc.
+
+@top Script { expression* }
+
+expression {
+  VariableName |
+  String |
+  MultilineComment |
+  keyword |
+  Symbol |
+  SystemCommand
+}
+
+// See https://lezer.codemirror.net/docs/guide/ for documentation on syntax
+// specific to @tokens blocks, and how it differs from regular expression syntax.
+@tokens {
+  VariableName { $[a-zA-Z0-9_]+ $[a-zA-Z0-9_']? }
+  charVector { '"' (!["\n])* '"' }
+  stringArray { "'" (!['\n])* "'" }
+  SystemCommand { "!" (![\n])* }
+  Symbol { "+" | "-" | "*" | "=" | ";" | ":" | "(" | ")" | "{" | "}" | "[" | "]" }
+  space { @whitespace+ }
+  @precedence { SystemCommand, VariableName }
+  @precedence { SystemCommand, space }
+  @precedence { VariableName, charVector }
+  @precedence { VariableName, stringArray }
+}
+
+String { charVector | stringArray }
+
+// Once a string has been parsed and found to be a VariableName, it will then
+// be tested against its specialize table, to test if it is a keyword.
+// The keyword node name is "Keyword".
+// https://lezer.codemirror.net/docs/guide/#token-specialization
+keyword {
+  @specialize[@name=Keyword]<VariableName, "break" | "case" | "classdef" | "continue" | "global" | "otherwise" | "persistent" | "return" | "spmd" | "arguments" | "enumeration" | "events" | "for" | "function" | "if" | "methods" | "parfor" | "properties" | "try" | "while" | "elseif" | "else" | "end" | "switch" | "catch">
+}
+
+@skip { space | LineComment }
+
+@external propSource matlabHighlighting from "./highlight"
+
+// Call out to comment parser.
+@external tokens parseComments from "./parse_comments" { MultilineComment, LineComment }
+
+@detectDelim