Migrate Python to use next-gen scope handlers for text fragment extractors (#1862)

- Depends on #1863 

## Checklist

- [x] I have added
[tests](https://www.cursorless.org/docs/contributing/test-case-recorder/)
- [/] I have updated the
[docs](https://github.com/cursorless-dev/cursorless/tree/main/docs) and
[cheatsheet](https://github.com/cursorless-dev/cursorless/tree/main/cursorless-talon/src/cheatsheet)
- [/] I have not broken the cheatsheet
- [x] merge #1858 first
as seems required
- [x] @pokey to fix a bug in the "processSurroundingPairCore" function
atm that makes it not working atm (see commit message). It will be
handled in a different commit before we can take this PR into account
- [x] record a test for the fix by @pokey by saying "cursorless record"
and using "change string"

This PR also includes a few additional unit tests for Python strings.

---------

Co-authored-by: Cedric Halbronn <[email protected]>
Co-authored-by: Pokey Rule <[email protected]>
Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com>

Author: 4 people

data/playground/python/statements.py

@@ -0,0 +1,36 @@
+import sys
+
+
+def statements():
+    print(sys.path[0])  # comment 1
+    print("hello")  # comment 2
+
+    # the below statement has additional spaces after it
+    val = 1 == 2
+    if val is True:
+        return
+
+    # also the below has non-empty indentation
+
+    c = range(10)
+    c.append(100)
+    x = 1
+    x /= 2
+    for i in range(10):
+        if i == 0:
+            continue
+        print(i)
+        break
+
+    age = 120
+    if age > 90:
+        print("You are too old to party, granny.")
+    elif age < 0:
+        print("You're yet to be born")
+    elif age >= 18:
+        print("You are allowed to party")
+    else:
+        print("You're too young to party")
+
+
+statements()

data/playground/python/strings.py

@@ -0,0 +1,8 @@
+value = 3
+
+a = "single quote string"
+b = "double quote string"
+c = """triple single quote string"""
+d = """triple double quote string"""
+e = r"literal string"
+f = f"format string {value}"

packages/cursorless-engine/src/languages/getTextFragmentExtractor.ts

@@ -159,7 +159,6 @@ const textFragmentExtractors: Record<
     "php",
     phpStringTextFragmentExtractor,
   ),
-  python: constructDefaultTextFragmentExtractor("python"),
   ruby: constructDefaultTextFragmentExtractor(
     "ruby",
     rubyStringTextFragmentExtractor,

packages/cursorless-engine/src/languages/python.ts

@@ -45,7 +45,6 @@ function itemNodeFinder(
 const nodeMatchers: Partial<
   Record<SimpleScopeTypeType, NodeMatcherAlternative>
 > = {
-  string: "string",
   collectionItem: cascadingMatcher(
     matcher(
       itemNodeFinder("import_from_statement", "dotted_name", true),
@@ -61,7 +60,6 @@ const nodeMatchers: Partial<
   anonymousFunction: "lambda?.lambda",
   functionCall: "call",
   functionCallee: "call[function]",
-  comment: "comment",
   condition: cascadingMatcher(
     conditionMatcher("*[condition]"),
 

packages/cursorless-vscode-e2e/src/suite/fixtures/recorded/languages/python/takeString2.yml

@@ -0,0 +1,24 @@
+languageId: python
+command:
+  version: 1
+  spokenForm: take string
+  action: setSelection
+  targets:
+    - type: primitive
+      modifier: {type: containingScope, scopeType: string}
+spokenFormError: Scope type 'string'
+initialState:
+  documentContents: |
+
+    value = """hello world"""
+  selections:
+    - anchor: {line: 1, character: 17}
+      active: {line: 1, character: 17}
+  marks: {}
+finalState:
+  documentContents: |
+
+    value = """hello world"""
+  selections:
+    - anchor: {line: 1, character: 8}
+      active: {line: 1, character: 25}

packages/cursorless-vscode-e2e/src/suite/fixtures/recorded/languages/python/takeString3.yml

@@ -0,0 +1,24 @@
+languageId: python
+command:
+  version: 1
+  spokenForm: take string
+  action: setSelection
+  targets:
+    - type: primitive
+      modifier: {type: containingScope, scopeType: string}
+spokenFormError: Scope type 'string'
+initialState:
+  documentContents: |
+
+    value = r"hello world"
+  selections:
+    - anchor: {line: 1, character: 16}
+      active: {line: 1, character: 16}
+  marks: {}
+finalState:
+  documentContents: |
+
+    value = r"hello world"
+  selections:
+    - anchor: {line: 1, character: 8}
+      active: {line: 1, character: 22}

packages/cursorless-vscode-e2e/src/suite/fixtures/recorded/languages/python/takeString4.yml

@@ -0,0 +1,26 @@
+languageId: python
+command:
+  version: 1
+  spokenForm: take string
+  action: setSelection
+  targets:
+    - type: primitive
+      modifier: {type: containingScope, scopeType: string}
+spokenFormError: Scope type 'string'
+initialState:
+  documentContents: |
+
+    w = "world"
+    value = f"hello {w}"
+  selections:
+    - anchor: {line: 2, character: 16}
+      active: {line: 2, character: 16}
+  marks: {}
+finalState:
+  documentContents: |
+
+    w = "world"
+    value = f"hello {w}"
+  selections:
+    - anchor: {line: 2, character: 8}
+      active: {line: 2, character: 20}

queries/python.scm

@@ -27,6 +27,13 @@
   (with_statement)
 ] @statement
 
+(comment) @comment @textFragment
+
+(string
+  _ @textFragment.start.endOf
+  _ @textFragment.end.startOf
+) @string
+
 [
   (dictionary)
   (dictionary_comprehension)