Introduce tree-sitter queries for syntactic scopes (#629)

- Partially addresses #616
- Partially addresses
#436
- Depends on #1396

## Todo
- [x] **[DISCUSS]** What to do about fallback `iterationScope`? That's
the only thing that is a regression here.
- [x] File issues for FIXMEs
- [x] File issue for defining iteration scopes. Can probably reuse most
of the code from the regular scope handler other than creating the
target
- [x] File issue to add unit tests for scope handlers
- [x] File issue to add some Python scope types where multiple can end
at the same point (due to lack of closing brackets)
- [x] Add test that checks no scope types are duplicated between legacy
and new definition, or file issue to add test
- [x] File PR for my 7783da6 (Add support for domain, leading, trailing,
interior) #1427
- [x] Look through comments on this thread for anything worth filing /
doing
- [x] Open as new PR?
- [x] Remove extraneous test cases
- [x] Double check
#629 (comment);
a lot of those tests we already have for the generic modifier code
- [x] Make sure changes to parse-tree-extension are shipped
- [x] Close #785 if
we fix that
- [x] Comment on #484 saying the process has started and providing link
to example
- [x] Close #797 if
we fix that

---------

Co-authored-by: Pokey Rule <[email protected]>

Author: Will-Sommers

packages/cursorless-engine/src/cursorlessEngine.ts

@@ -9,6 +9,7 @@ import { MarkStageFactoryImpl } from "./processTargets/MarkStageFactoryImpl";
 import { ModifierStageFactoryImpl } from "./processTargets/ModifierStageFactoryImpl";
 import { ScopeHandlerFactoryImpl } from "./processTargets/modifiers/scopeHandlers";
 import { injectIde } from "./singletons/ide.singleton";
+import { LanguageDefinitions } from "./languages/LanguageDefinitions";
 
 export function createCursorlessEngine(
   treeSitter: TreeSitter,
@@ -36,7 +37,8 @@ export function createCursorlessEngine(
 
   const testCaseRecorder = new TestCaseRecorder(hatTokenMap);
 
-  const scopeHandlerFactory = new ScopeHandlerFactoryImpl();
+  const languageDefinitions = new LanguageDefinitions(treeSitter);
+  const scopeHandlerFactory = new ScopeHandlerFactoryImpl(languageDefinitions);
   const markStageFactory = new MarkStageFactoryImpl();
   const modifierStageFactory = new ModifierStageFactoryImpl(
     scopeHandlerFactory,

packages/cursorless-engine/src/languages/LanguageDefinition.ts

@@ -0,0 +1,69 @@
+import { ScopeType, SimpleScopeType } from "@cursorless/common";
+import { Query } from "web-tree-sitter";
+import { ide } from "../singletons/ide.singleton";
+import { join } from "path";
+import { TreeSitterScopeHandler } from "../processTargets/modifiers/scopeHandlers";
+import { TreeSitter } from "../typings/TreeSitter";
+import { existsSync, readFileSync } from "fs";
+import { LanguageId } from "./constants";
+
+/**
+ * Represents a language definition for a single language, including the
+ * tree-sitter query used to extract scopes for the given language
+ */
+export class LanguageDefinition {
+  private constructor(
+    private treeSitter: TreeSitter,
+    /**
+     * The tree-sitter query used to extract scopes for the given language.
+     * Note that this query contains patterns for all scope types that the
+     * language supports using new-style tree-sitter queries
+     */
+    private query: Query,
+  ) {}
+
+  /**
+   * Construct a language definition for the given language id, if the language
+   * has a new-style query definition, or return undefined if the language doesn't
+   *
+   * @param treeSitter The tree-sitter instance to use for parsing
+   * @param languageId The language id for which to create a language definition
+   * @returns A language definition for the given language id, or undefined if the given language
+   * id doesn't have a new-style query definition
+   */
+  static create(
+    treeSitter: TreeSitter,
+    languageId: LanguageId,
+  ): LanguageDefinition | undefined {
+    const queryPath = join(ide().assetsRoot, "queries", `${languageId}.scm`);
+
+    if (!existsSync(queryPath)) {
+      return undefined;
+    }
+
+    const rawLanguageQueryString = readFileSync(queryPath, "utf8");
+
+    return new LanguageDefinition(
+      treeSitter,
+      treeSitter.getLanguage(languageId)!.query(rawLanguageQueryString),
+    );
+  }
+
+  /**
+   * @param scopeType The scope type for which to get a scope handler
+   * @returns A scope handler for the given scope type and language id, or
+   * undefined if the given scope type / language id combination is still using
+   * legacy pathways
+   */
+  getScopeHandler(scopeType: ScopeType) {
+    if (!this.query.captureNames.includes(scopeType.type)) {
+      return undefined;
+    }
+
+    return new TreeSitterScopeHandler(
+      this.treeSitter,
+      this.query,
+      scopeType as SimpleScopeType,
+    );
+  }
+}

packages/cursorless-engine/src/languages/LanguageDefinitions.ts

@@ -0,0 +1,54 @@
+import { TreeSitter } from "..";
+import { LanguageDefinition } from "./LanguageDefinition";
+import { LanguageId } from "./constants";
+
+/**
+ * Sentinel value to indicate that a language doesn't have
+ * a new-style query definition file
+ */
+const LANGUAGE_UNDEFINED = Symbol("LANGUAGE_UNDEFINED");
+
+/**
+ * Keeps a map from language ids to  {@link LanguageDefinition} instances,
+ * constructing them as necessary
+ */
+export class LanguageDefinitions {
+  /**
+   * Maps from language id to {@link LanguageDefinition} or
+   * {@link LANGUAGE_UNDEFINED} if language doesn't have new-style definitions.
+   * We use a sentinel value instead of undefined so that we can distinguish
+   * between a situation where we haven't yet checked whether a language has a
+   * new-style query definition and a situation where we've checked and found
+   * that it doesn't.  The former case is represented by `undefined` (due to the
+   * semantics of {@link Map.get}), while the latter is represented by the
+   * sentinel value.
+   */
+  private languageDefinitions: Map<
+    string,
+    LanguageDefinition | typeof LANGUAGE_UNDEFINED
+  > = new Map();
+
+  constructor(private treeSitter: TreeSitter) {}
+
+  /**
+   * Get a language definition for the given language id, if the language
+   * has a new-style query definition, or return undefined if the language doesn't
+   *
+   * @param languageId The language id for which to get a language definition
+   * @returns A language definition for the given language id, or undefined if
+   * the given language id doesn't have a new-style query definition
+   */
+  get(languageId: string): LanguageDefinition | undefined {
+    let definition = this.languageDefinitions.get(languageId);
+
+    if (definition == null) {
+      definition =
+        LanguageDefinition.create(this.treeSitter, languageId as LanguageId) ??
+        LANGUAGE_UNDEFINED;
+
+      this.languageDefinitions.set(languageId, definition);
+    }
+
+    return definition === LANGUAGE_UNDEFINED ? undefined : definition;
+  }
+}

packages/cursorless-engine/src/languages/index.ts

@@ -3,5 +3,5 @@ import { SupportedLanguageId, supportedLanguageIds } from "./constants";
 export function isLanguageSupported(
   languageId: string,
 ): languageId is SupportedLanguageId {
-  return languageId in supportedLanguageIds;
+  return supportedLanguageIds.includes(languageId as SupportedLanguageId);
 }

packages/cursorless-engine/src/languages/ruby.ts

@@ -151,8 +151,6 @@ function blockFinder(node: SyntaxNode) {
 const nodeMatchers: Partial<
   Record<SimpleScopeTypeType, NodeMatcherAlternative>
 > = {
-  map: mapTypes,
-  list: listTypes,
   statement: cascadingMatcher(
     patternMatcher(...STATEMENT_TYPES),
     ancestorChainNodeMatcher(
@@ -164,26 +162,20 @@ const nodeMatchers: Partial<
     ),
   ),
   string: "string",
-  ifStatement: "if",
-  functionCall: "call",
-  comment: "comment",
-  namedFunction: ["method", "singleton_method"],
   functionName: ["method[name]", "singleton_method[name]"],
   anonymousFunction: cascadingMatcher(
     patternMatcher("lambda", "do_block"),
     matcher(blockFinder),
   ),
-  regularExpression: "regex",
   condition: conditionMatcher("*[condition]"),
   argumentOrParameter: argumentMatcher(
     "lambda_parameters",
     "method_parameters",
     "block_parameters",
     "argument_list",
   ),
-  class: "class",
-  className: "class[name]",
   collectionKey: trailingMatcher(["pair[key]"], [":"]),
+  className: "class[name]",
   name: [
     "assignment[left]",
     "operator_assignment[left]",

packages/cursorless-engine/src/processTargets/modifiers/scopeHandlers/ScopeHandlerFactoryImpl.ts

@@ -1,16 +1,17 @@
+import type { ScopeType } from "@cursorless/common";
 import {
   CharacterScopeHandler,
   DocumentScopeHandler,
   IdentifierScopeHandler,
   LineScopeHandler,
-  TokenScopeHandler,
-  WordScopeHandler,
   OneOfScopeHandler,
   ParagraphScopeHandler,
+  TokenScopeHandler,
+  WordScopeHandler,
 } from ".";
-import type { ScopeType } from "@cursorless/common";
-import type { ScopeHandler } from "./scopeHandler.types";
+import { LanguageDefinitions } from "../../../languages/LanguageDefinitions";
 import { ScopeHandlerFactory } from "./ScopeHandlerFactory";
+import type { ScopeHandler } from "./scopeHandler.types";
 
 /**
  * Returns a scope handler for the given scope type and language id, or
@@ -30,7 +31,7 @@ import { ScopeHandlerFactory } from "./ScopeHandlerFactory";
  * legacy pathways
  */
 export class ScopeHandlerFactoryImpl implements ScopeHandlerFactory {
-  constructor() {
+  constructor(private languageDefinitions: LanguageDefinitions) {
     this.create = this.create.bind(this);
   }
 
@@ -53,7 +54,9 @@ export class ScopeHandlerFactoryImpl implements ScopeHandlerFactory {
       case "paragraph":
         return new ParagraphScopeHandler(scopeType, languageId);
       default:
-        return undefined;
+        return this.languageDefinitions
+          .get(languageId)
+          ?.getScopeHandler(scopeType);
     }
   }
 }

packages/cursorless-engine/src/processTargets/modifiers/scopeHandlers/TreeSitterScopeHandler.ts

@@ -0,0 +1,141 @@
+import {
+  Direction,
+  Position,
+  ScopeType,
+  SimpleScopeType,
+  TextDocument,
+  TextEditor,
+} from "@cursorless/common";
+
+import { Point, Query, QueryMatch } from "web-tree-sitter";
+import { TreeSitter } from "../../..";
+import { getNodeRange } from "../../../util/nodeSelectors";
+import ScopeTypeTarget from "../../targets/ScopeTypeTarget";
+import BaseScopeHandler from "./BaseScopeHandler";
+import { compareTargetScopes } from "./compareTargetScopes";
+import { TargetScope } from "./scope.types";
+import { ScopeIteratorRequirements } from "./scopeHandler.types";
+
+/**
+ * Handles scopes that are implemented using tree-sitter.
+ */
+export class TreeSitterScopeHandler extends BaseScopeHandler {
+  protected isHierarchical: boolean = true;
+
+  constructor(
+    private treeSitter: TreeSitter,
+    private query: Query,
+    public scopeType: SimpleScopeType,
+  ) {
+    super();
+  }
+
+  public get iterationScopeType(): ScopeType {
+    throw Error("Not implemented");
+  }
+
+  *generateScopeCandidates(
+    editor: TextEditor,
+    position: Position,
+    direction: Direction,
+    hints: ScopeIteratorRequirements,
+  ): Iterable<TargetScope> {
+    const { document } = editor;
+
+    /** Narrow the range within which tree-sitter searches, for performance */
+    const { start, end } = getQueryRange(document, position, direction, hints);
+
+    yield* this.query
+      .matches(
+        this.treeSitter.getTree(document).rootNode,
+        positionToPoint(start),
+        positionToPoint(end),
+      )
+      .filter(({ captures }) =>
+        captures.some((capture) => capture.name === this.scopeType.type),
+      )
+      .map((match) => this.matchToScope(editor, match))
+      .sort((a, b) => compareTargetScopes(direction, position, a, b));
+  }
+
+  private matchToScope(editor: TextEditor, match: QueryMatch): TargetScope {
+    const contentRange = getNodeRange(
+      match.captures.find((capture) => capture.name === this.scopeType.type)!
+        .node,
+    );
+
+    return {
+      editor,
+      // FIXME: Actually get domain
+      domain: contentRange,
+      getTarget: (isReversed) =>
+        new ScopeTypeTarget({
+          scopeTypeType: this.scopeType.type,
+          editor,
+          isReversed,
+          contentRange,
+          // FIXME: Actually get removalRange
+          removalRange: contentRange,
+          // FIXME: Other fields here
+        }),
+    };
+  }
+}
+
+/**
+ * Constructs a range to pass to {@link Query.matches} to find scopes. Note
+ * that {@link Query.matches} will only return scopes that have non-empty
+ * intersection with this range.  Also note that the base
+ * {@link BaseScopeHandler.generateScopes} will filter out any extra scopes
+ * that we yield, so we don't need to be totally precise.
+ *
+ * @returns Range to pass to {@link Query.matches}
+ */
+function getQueryRange(
+  document: TextDocument,
+  position: Position,
+  direction: Direction,
+  { containment, distalPosition }: ScopeIteratorRequirements,
+) {
+  const offset = document.offsetAt(position);
+  const distalOffset =
+    distalPosition == null ? null : document.offsetAt(distalPosition);
+
+  if (containment === "required") {
+    // If containment is required, we smear the position left and right by one
+    // character so that we have a non-empty intersection with any scope that
+    // touches position
+    return {
+      start: document.positionAt(offset - 1),
+      end: document.positionAt(offset + 1),
+    };
+  }
+
+  // If containment is disallowed, we can shift the position forward by a character to avoid
+  // matching scopes that touch position.  Otherwise, we shift the position backward by a
+  // character to ensure we get scopes that touch position.
+  const proximalShift = containment === "disallowed" ? 1 : -1;
+
+  // FIXME: Don't go all the way to end of document when there is no distalPosition?
+  // Seems wasteful to query all the way to end of document for something like "next funk"
+  // Might be better to start smaller and exponentially grow
+  return direction === "forward"
+    ? {
+        start: document.positionAt(offset + proximalShift),
+        end:
+          distalOffset == null
+            ? document.range.end
+            : document.positionAt(distalOffset + 1),
+      }
+    : {
+        start:
+          distalOffset == null
+            ? document.range.start
+            : document.positionAt(distalOffset - 1),
+        end: document.positionAt(offset - proximalShift),
+      };
+}
+
+function positionToPoint(start: Position): Point | undefined {
+  return { row: start.line, column: start.character };
+}

packages/cursorless-engine/src/processTargets/modifiers/scopeHandlers/index.ts

@@ -12,6 +12,7 @@ export * from "./TokenScopeHandler";
 export { default as TokenScopeHandler } from "./TokenScopeHandler";
 export * from "./DocumentScopeHandler";
 export { default as DocumentScopeHandler } from "./DocumentScopeHandler";
+export * from "./TreeSitterScopeHandler";
 export * from "./OneOfScopeHandler";
 export { default as OneOfScopeHandler } from "./OneOfScopeHandler";
 export * from "./ParagraphScopeHandler";