From 8dd32a8648ca8036272f103df506700b2c192094 Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Thu, 26 Apr 2018 15:41:43 -0700 Subject: [PATCH 01/12] Fix for #259 --- pythonFiles/normalizeForInterpreter.py | 111 ++++++++++++++++++ src/client/terminals/codeExecution/helper.ts | 21 +++- .../terminalExec/sample1_normalized.py | 5 +- .../terminalExec/sample3_normalized.py | 1 + .../pythonFiles/terminalExec/sample3_raw.py | 1 + .../terminalExec/sample6_normalized.py | 15 +++ .../pythonFiles/terminalExec/sample6_raw.py | 12 ++ .../terminalExec/sample7_normalized.py | 8 ++ .../pythonFiles/terminalExec/sample7_raw.py | 9 ++ .../terminalExec/sample_normalized.py | 5 + .../pythonFiles/terminalExec/sample_raw.py | 8 ++ .../terminals/codeExecution/helper.test.ts | 40 +++++-- 12 files changed, 219 insertions(+), 17 deletions(-) create mode 100644 pythonFiles/normalizeForInterpreter.py create mode 100644 src/test/pythonFiles/terminalExec/sample6_normalized.py create mode 100644 src/test/pythonFiles/terminalExec/sample6_raw.py create mode 100644 src/test/pythonFiles/terminalExec/sample7_normalized.py create mode 100644 src/test/pythonFiles/terminalExec/sample7_raw.py create mode 100644 src/test/pythonFiles/terminalExec/sample_normalized.py create mode 100644 src/test/pythonFiles/terminalExec/sample_raw.py diff --git a/pythonFiles/normalizeForInterpreter.py b/pythonFiles/normalizeForInterpreter.py new file mode 100644 index 000000000000..2a1008e3a9a6 --- /dev/null +++ b/pythonFiles/normalizeForInterpreter.py @@ -0,0 +1,111 @@ +import ast +import io +import os +import sys +import token +import tokenize + + +class Visitor(ast.NodeVisitor): + def __init__(self, lines): + self._lines = lines + self.line_numbers_with_nodes = set() + self.line_numbers_with_statements = [] + + def generic_visit(self, node): + node_type = type(node).__name__ + if hasattr(node, 'col_offset') and hasattr(node, 'lineno') and node.col_offset == 0: + self.line_numbers_with_nodes.add(node.lineno) + if isinstance(node, ast.stmt): + self.line_numbers_with_statements.append(node.lineno) + + ast.NodeVisitor.generic_visit(self, node) + + +def _tokenize(source): + return tokenize.generate_tokens(io.StringIO(source).readline) + + +def _indent_size(line): + for index, char in enumerate(line): + if not char.isspace(): + return index + + +def _get_global_statement_blocks(source, lines): + """Gets a list of all global statement blocks. + + The list contains the start and end line numbers of block. + + """ + tree = ast.parse(source) + visitor = Visitor(lines) + visitor.visit(tree) + + statement_ranges = [] + for index, line_number in enumerate(visitor.line_numbers_with_statements): + remaining_line_numbers = visitor.line_numbers_with_statements[index + 1:] + end_line_number = len(lines) if len(remaining_line_numbers) == 0 else min(remaining_line_numbers) - 1 + current_statement_is_oneline = line_number == end_line_number + + if len(statement_ranges) == 0: + statement_ranges.append((line_number, end_line_number, current_statement_is_oneline)) + continue + + previous_statement = statement_ranges[-1] + previous_statement_is_oneline = previous_statement[2] + if previous_statement_is_oneline and current_statement_is_oneline: + statement_ranges[-1] = (previous_statement[0], end_line_number, True) + else: + statement_ranges.append((line_number, end_line_number, current_statement_is_oneline)) + + return statement_ranges + + +def normalize_lines(source): + """Normalize blank lines for sending to the terminal. + + Blank lines within a statement block are removed to prevent the REPL + from thinking the block is finished. Newlines are added to separate + top-level statements so that the REPL does not think there is a syntax + error. + + """ + lines = source.splitlines(False) + # Find out if we have any trailing blank lines + has_blank_lines = len(lines[-1].strip()) == 0 or source.endswith(os.linesep) + + # Step 1: Remove empty lines. + tokens = _tokenize(source) + newlines_indexes_to_remove = (spos[0] for (toknum, tokval, spos, epos, line) in tokens + if len(line.strip()) == 0 and token.tok_name[toknum] == 'NL' and spos[0] == epos[0]) + + for line_number in reversed(list(newlines_indexes_to_remove)): + del lines[line_number - 1] + + # Step 2: Add blank lines between each global statement block. + # A consequtive single lines blocks of code will be treated as a single statement, + # just to ensure we do not unnecessarily add too many blank lines. + source = os.linesep.join(lines) + tokens = _tokenize(source) + dedent_indexes = (spos[0] for (toknum, tokval, spos, epos, line) in tokens + if toknum == token.DEDENT and _indent_size(line) == 0) + + global_statement_ranges = _get_global_statement_blocks(source, lines) + + for line_number in (start_line for start_line, _, _ in reversed(global_statement_ranges) if start_line > 1): + lines.insert(line_number - 1, '') + + sys.stdout.write(os.linesep.join(lines) + (os.linesep if has_blank_lines else '')) + sys.stdout.flush() + + +if __name__ == '__main__': + contents = sys.argv[1] + if isinstance(contents, bytes): + contents = contents.decode('utf8') + # contents = open('/Users/donjayamanne/.vscode-insiders/extensions/pythonVSCode/src/test/pythonFiles/terminalExec/sample6_raw.py', 'r').read() + # sys.stdout.write('contents\n') + # sys.stdout.write(contents) + # sys.stdout.write('contents') + normalize_lines(contents) diff --git a/src/client/terminals/codeExecution/helper.ts b/src/client/terminals/codeExecution/helper.ts index 674d5cfba681..de31a8b94049 100644 --- a/src/client/terminals/codeExecution/helper.ts +++ b/src/client/terminals/codeExecution/helper.ts @@ -2,10 +2,14 @@ // Licensed under the MIT License. import { inject, injectable } from 'inversify'; +import * as path from 'path'; import { Range, TextEditor, Uri } from 'vscode'; import { IApplicationShell, IDocumentManager } from '../../common/application/types'; -import { PYTHON_LANGUAGE } from '../../common/constants'; +import { EXTENSION_ROOT_DIR, PYTHON_LANGUAGE } from '../../common/constants'; import '../../common/extensions'; +import { IProcessService } from '../../common/process/types'; +import { IConfigurationService } from '../../common/types'; +import { IEnvironmentVariablesProvider } from '../../common/variables/types'; import { IServiceContainer } from '../../ioc/types'; import { ICodeExecutionHelper } from '../types'; @@ -13,19 +17,26 @@ import { ICodeExecutionHelper } from '../types'; export class CodeExecutionHelper implements ICodeExecutionHelper { private readonly documentManager: IDocumentManager; private readonly applicationShell: IApplicationShell; + private readonly envVariablesProvider: IEnvironmentVariablesProvider; + private readonly processService: IProcessService; + private readonly configurationService: IConfigurationService; constructor(@inject(IServiceContainer) serviceContainer: IServiceContainer) { this.documentManager = serviceContainer.get(IDocumentManager); this.applicationShell = serviceContainer.get(IApplicationShell); + this.envVariablesProvider = serviceContainer.get(IEnvironmentVariablesProvider); + this.processService = serviceContainer.get(IProcessService); + this.configurationService = serviceContainer.get(IConfigurationService); } public async normalizeLines(code: string, resource?: Uri): Promise { try { if (code.trim().length === 0) { return ''; } - const regex = /(\n)([ \t]*\r?\n)([ \t]+\S+)/gm; - return code.replace(regex, (_, a, b, c) => { - return `${a}${c}`; - }); + const env = await this.envVariablesProvider.getEnvironmentVariables(resource); + const pythonPath = this.configurationService.getSettings(resource).pythonPath; + const args = [path.join(EXTENSION_ROOT_DIR, 'pythonFiles', 'normalizeForInterpreter.py'), code]; + const proc = await this.processService.exec(pythonPath, args, { env, throwOnStdErr: true }); + return proc.stdout; } catch (ex) { console.error(ex, 'Python: Failed to normalize code for execution in terminal'); return code; diff --git a/src/test/pythonFiles/terminalExec/sample1_normalized.py b/src/test/pythonFiles/terminalExec/sample1_normalized.py index 0896de65d22f..8591baeb6489 100644 --- a/src/test/pythonFiles/terminalExec/sample1_normalized.py +++ b/src/test/pythonFiles/terminalExec/sample1_normalized.py @@ -1,18 +1,21 @@ # Sample block 1 + def square(x): return x**2 print('hello') # Sample block 2 + a = 2 + if a < 2: print('less than 2') else: print('more than 2') print('hello') - # Sample block 3 + for i in range(5): print(i) print(i) diff --git a/src/test/pythonFiles/terminalExec/sample3_normalized.py b/src/test/pythonFiles/terminalExec/sample3_normalized.py index e4f028b0b778..4fa62091c66d 100644 --- a/src/test/pythonFiles/terminalExec/sample3_normalized.py +++ b/src/test/pythonFiles/terminalExec/sample3_normalized.py @@ -1,4 +1,5 @@ if True: print(1) print(2) + print(3) diff --git a/src/test/pythonFiles/terminalExec/sample3_raw.py b/src/test/pythonFiles/terminalExec/sample3_raw.py index 5865e6d2cbde..fee6c839aa89 100644 --- a/src/test/pythonFiles/terminalExec/sample3_raw.py +++ b/src/test/pythonFiles/terminalExec/sample3_raw.py @@ -2,4 +2,5 @@ print(1) print(2) + print(3) diff --git a/src/test/pythonFiles/terminalExec/sample6_normalized.py b/src/test/pythonFiles/terminalExec/sample6_normalized.py new file mode 100644 index 000000000000..242bf35abea8 --- /dev/null +++ b/src/test/pythonFiles/terminalExec/sample6_normalized.py @@ -0,0 +1,15 @@ +if True: + print(1) +else: print(2) + +print('🔨') +print(3) +print(3) + +if True: + print(1) +else: print(2) + +if True: + print(1) +else: print(2) diff --git a/src/test/pythonFiles/terminalExec/sample6_raw.py b/src/test/pythonFiles/terminalExec/sample6_raw.py new file mode 100644 index 000000000000..b064ca962070 --- /dev/null +++ b/src/test/pythonFiles/terminalExec/sample6_raw.py @@ -0,0 +1,12 @@ +if True: + print(1) +else: print(2) +print('🔨') +print(3) +print(3) +if True: + print(1) +else: print(2) +if True: + print(1) +else: print(2) diff --git a/src/test/pythonFiles/terminalExec/sample7_normalized.py b/src/test/pythonFiles/terminalExec/sample7_normalized.py new file mode 100644 index 000000000000..2288800fc985 --- /dev/null +++ b/src/test/pythonFiles/terminalExec/sample7_normalized.py @@ -0,0 +1,8 @@ +if True: + print(1) + print(1) +else: + print(2) + print(2) + +print(3) diff --git a/src/test/pythonFiles/terminalExec/sample7_raw.py b/src/test/pythonFiles/terminalExec/sample7_raw.py new file mode 100644 index 000000000000..62d01b9659c6 --- /dev/null +++ b/src/test/pythonFiles/terminalExec/sample7_raw.py @@ -0,0 +1,9 @@ +if True: + print(1) + + print(1) +else: + print(2) + + print(2) +print(3) diff --git a/src/test/pythonFiles/terminalExec/sample_normalized.py b/src/test/pythonFiles/terminalExec/sample_normalized.py new file mode 100644 index 000000000000..8ee9b90cdd27 --- /dev/null +++ b/src/test/pythonFiles/terminalExec/sample_normalized.py @@ -0,0 +1,5 @@ +import sys +print(sys.executable) +print("1234") +print(1) +print(2) diff --git a/src/test/pythonFiles/terminalExec/sample_raw.py b/src/test/pythonFiles/terminalExec/sample_raw.py new file mode 100644 index 000000000000..d1b32aaf606c --- /dev/null +++ b/src/test/pythonFiles/terminalExec/sample_raw.py @@ -0,0 +1,8 @@ +import sys + +print(sys.executable) + +print("1234") + +print(1) +print(2) diff --git a/src/test/terminals/codeExecution/helper.test.ts b/src/test/terminals/codeExecution/helper.test.ts index 1d0347c8d48b..6c6ddeb9ca8f 100644 --- a/src/test/terminals/codeExecution/helper.test.ts +++ b/src/test/terminals/codeExecution/helper.test.ts @@ -11,9 +11,15 @@ import * as TypeMoq from 'typemoq'; import { Range, Selection, TextDocument, TextEditor, TextLine, Uri } from 'vscode'; import { IApplicationShell, IDocumentManager } from '../../../client/common/application/types'; import { EXTENSION_ROOT_DIR, PYTHON_LANGUAGE } from '../../../client/common/constants'; +import { BufferDecoder } from '../../../client/common/process/decoder'; +import { ProcessService } from '../../../client/common/process/proc'; +import { IProcessService } from '../../../client/common/process/types'; +import { IConfigurationService, IPythonSettings } from '../../../client/common/types'; +import { IEnvironmentVariablesProvider } from '../../../client/common/variables/types'; import { IServiceContainer } from '../../../client/ioc/types'; import { CodeExecutionHelper } from '../../../client/terminals/codeExecution/helper'; import { ICodeExecutionHelper } from '../../../client/terminals/types'; +import { PYTHON_PATH } from '../../common'; const TEST_FILES_PATH = path.join(EXTENSION_ROOT_DIR, 'src', 'test', 'pythonFiles', 'terminalExec'); @@ -24,12 +30,24 @@ suite('Terminal - Code Execution Helper', () => { let helper: ICodeExecutionHelper; let document: TypeMoq.IMock; let editor: TypeMoq.IMock; + let processService: TypeMoq.IMock; + let configService: TypeMoq.IMock; setup(() => { const serviceContainer = TypeMoq.Mock.ofType(); documentManager = TypeMoq.Mock.ofType(); applicationShell = TypeMoq.Mock.ofType(); + const envVariablesProvider = TypeMoq.Mock.ofType(); + processService = TypeMoq.Mock.ofType(); + configService = TypeMoq.Mock.ofType(); + const pythonSettings = TypeMoq.Mock.ofType(); + pythonSettings.setup(p => p.pythonPath).returns(() => PYTHON_PATH); + configService.setup(c => c.getSettings(TypeMoq.It.isAny())).returns(() => pythonSettings.object); + envVariablesProvider.setup(e => e.getEnvironmentVariables(TypeMoq.It.isAny())).returns(() => Promise.resolve({})); serviceContainer.setup(c => c.get(TypeMoq.It.isValue(IDocumentManager), TypeMoq.It.isAny())).returns(() => documentManager.object); serviceContainer.setup(c => c.get(TypeMoq.It.isValue(IApplicationShell), TypeMoq.It.isAny())).returns(() => applicationShell.object); + serviceContainer.setup(c => c.get(TypeMoq.It.isValue(IEnvironmentVariablesProvider), TypeMoq.It.isAny())).returns(() => envVariablesProvider.object); + serviceContainer.setup(c => c.get(TypeMoq.It.isValue(IProcessService), TypeMoq.It.isAny())).returns(() => processService.object); + serviceContainer.setup(c => c.get(TypeMoq.It.isValue(IConfigurationService), TypeMoq.It.isAny())).returns(() => configService.object); helper = new CodeExecutionHelper(serviceContainer.object); document = TypeMoq.Mock.ofType(); @@ -38,18 +56,23 @@ suite('Terminal - Code Execution Helper', () => { }); async function ensureBlankLinesAreRemoved(source: string, expectedSource: string) { + const actualProcessService = new ProcessService(new BufferDecoder()); + processService.setup(p => p.exec(TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny())) + .returns((file, args, options) => { + return actualProcessService.exec.apply(actualProcessService, [file, args, options]); + }); const normalizedZCode = await helper.normalizeLines(source); expect(normalizedZCode).to.be.equal(expectedSource); } test('Ensure blank lines are NOT removed when code is not indented (simple)', async () => { - const code = ['import sys', '', 'print(sys.executable)', '', 'print("1234")', '', 'print(1)', 'print(2)']; - const expectedCode = code.join(EOL); + const code = ['import sys', '', '', '', 'print(sys.executable)', '', 'print("1234")', '', '', 'print(1)', 'print(2)']; + const expectedCode = code.filter(line => line.trim().length > 0).join(EOL); await ensureBlankLinesAreRemoved(code.join(EOL), expectedCode); }); - ['sample1', 'sample2', 'sample3', 'sample4', 'sample5'].forEach(fileName => { - test(`Ensure blank lines are removed (${fileName})`, async () => { - const code = await fs.readFile(path.join(TEST_FILES_PATH, `${fileName}_raw.py`), 'utf8'); - const expectedCode = await fs.readFile(path.join(TEST_FILES_PATH, `${fileName}_normalized.py`), 'utf8'); + ['', '1', '2', '3', '4', '5', '6', '7'].forEach(fileNameSuffix => { + test(`Ensure blank lines are removed (Sample${fileNameSuffix})`, async () => { + const code = await fs.readFile(path.join(TEST_FILES_PATH, `sample${fileNameSuffix}_raw.py`), 'utf8'); + const expectedCode = await fs.readFile(path.join(TEST_FILES_PATH, `sample${fileNameSuffix}_normalized.py`), 'utf8'); await ensureBlankLinesAreRemoved(code, expectedCode); }); // test(`Ensure blank lines are removed, including leading empty lines (${fileName})`, async () => { @@ -58,11 +81,6 @@ suite('Terminal - Code Execution Helper', () => { // await ensureBlankLinesAreRemoved(['', '', ''].join(EOL) + EOL + code, expectedCode); // }); }); - test('Ensure blank lines are removed (sample2)', async () => { - const code = await fs.readFile(path.join(TEST_FILES_PATH, 'sample2_raw.py'), 'utf8'); - const expectedCode = await fs.readFile(path.join(TEST_FILES_PATH, 'sample2_normalized.py'), 'utf8'); - await ensureBlankLinesAreRemoved(code, expectedCode); - }); test('Display message if there\s no active file', async () => { documentManager.setup(doc => doc.activeTextEditor).returns(() => undefined); From 80480e3e2373612628cb940d53bf2c311db5f64d Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Thu, 26 Apr 2018 15:43:21 -0700 Subject: [PATCH 02/12] Add news entry --- news/2 Fixes/259.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 news/2 Fixes/259.md diff --git a/news/2 Fixes/259.md b/news/2 Fixes/259.md new file mode 100644 index 000000000000..8579bc07f4cc --- /dev/null +++ b/news/2 Fixes/259.md @@ -0,0 +1 @@ +Add blank lines to seprate blocks of indented code (function defs, classes, and the like) to ensure the code can be run within a Python interactive prompt. From 608d5e8cf03fb9bd42760baa4bfac874568d9592 Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Thu, 26 Apr 2018 15:44:57 -0700 Subject: [PATCH 03/12] remove test code --- pythonFiles/normalizeForInterpreter.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pythonFiles/normalizeForInterpreter.py b/pythonFiles/normalizeForInterpreter.py index 2a1008e3a9a6..6299d6ef8ee1 100644 --- a/pythonFiles/normalizeForInterpreter.py +++ b/pythonFiles/normalizeForInterpreter.py @@ -104,8 +104,4 @@ def normalize_lines(source): contents = sys.argv[1] if isinstance(contents, bytes): contents = contents.decode('utf8') - # contents = open('/Users/donjayamanne/.vscode-insiders/extensions/pythonVSCode/src/test/pythonFiles/terminalExec/sample6_raw.py', 'r').read() - # sys.stdout.write('contents\n') - # sys.stdout.write(contents) - # sys.stdout.write('contents') normalize_lines(contents) From 058c828988f3a4a7ff4d9000be38a6516bf7b763 Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Thu, 26 Apr 2018 15:45:42 -0700 Subject: [PATCH 04/12] Fix typo --- news/2 Fixes/259.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/news/2 Fixes/259.md b/news/2 Fixes/259.md index 8579bc07f4cc..6460214099c3 100644 --- a/news/2 Fixes/259.md +++ b/news/2 Fixes/259.md @@ -1 +1 @@ -Add blank lines to seprate blocks of indented code (function defs, classes, and the like) to ensure the code can be run within a Python interactive prompt. +Add blank lines to separate blocks of indented code (function defs, classes, and the like) so as to ensure the code can be run within a Python interactive prompt. From 0340efd29e4a6ff139a902c18a19ada9ea74237f Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Mon, 30 Apr 2018 12:06:03 -0700 Subject: [PATCH 05/12] Fix code review comments --- pythonFiles/normalizeForInterpreter.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/pythonFiles/normalizeForInterpreter.py b/pythonFiles/normalizeForInterpreter.py index 6299d6ef8ee1..588dd169063f 100644 --- a/pythonFiles/normalizeForInterpreter.py +++ b/pythonFiles/normalizeForInterpreter.py @@ -13,7 +13,6 @@ def __init__(self, lines): self.line_numbers_with_statements = [] def generic_visit(self, node): - node_type = type(node).__name__ if hasattr(node, 'col_offset') and hasattr(node, 'lineno') and node.col_offset == 0: self.line_numbers_with_nodes.add(node.lineno) if isinstance(node, ast.stmt): @@ -23,6 +22,11 @@ def generic_visit(self, node): def _tokenize(source): + """Tokenize python source using undocumented api + + The documented api does not work in Python 2.7 + + """ return tokenize.generate_tokens(io.StringIO(source).readline) @@ -33,9 +37,10 @@ def _indent_size(line): def _get_global_statement_blocks(source, lines): - """Gets a list of all global statement blocks. + """Return a list of all global statement blocks. - The list contains the start and end line numbers of block. + The list comprises of 3-item tuples that contain the starting line number, + ending line number and whether the statement is a single line. """ tree = ast.parse(source) @@ -44,7 +49,7 @@ def _get_global_statement_blocks(source, lines): statement_ranges = [] for index, line_number in enumerate(visitor.line_numbers_with_statements): - remaining_line_numbers = visitor.line_numbers_with_statements[index + 1:] + remaining_line_numbers = visitor.line_numbers_with_statements[index+1:] end_line_number = len(lines) if len(remaining_line_numbers) == 0 else min(remaining_line_numbers) - 1 current_statement_is_oneline = line_number == end_line_number @@ -55,7 +60,7 @@ def _get_global_statement_blocks(source, lines): previous_statement = statement_ranges[-1] previous_statement_is_oneline = previous_statement[2] if previous_statement_is_oneline and current_statement_is_oneline: - statement_ranges[-1] = (previous_statement[0], end_line_number, True) + statement_ranges[-1] = previous_statement[0], end_line_number, True else: statement_ranges.append((line_number, end_line_number, current_statement_is_oneline)) @@ -81,11 +86,11 @@ def normalize_lines(source): if len(line.strip()) == 0 and token.tok_name[toknum] == 'NL' and spos[0] == epos[0]) for line_number in reversed(list(newlines_indexes_to_remove)): - del lines[line_number - 1] + del lines[line_number-1] # Step 2: Add blank lines between each global statement block. # A consequtive single lines blocks of code will be treated as a single statement, - # just to ensure we do not unnecessarily add too many blank lines. + # just to ensure we do not unnecessarily add too many blank lines. source = os.linesep.join(lines) tokens = _tokenize(source) dedent_indexes = (spos[0] for (toknum, tokval, spos, epos, line) in tokens @@ -93,8 +98,8 @@ def normalize_lines(source): global_statement_ranges = _get_global_statement_blocks(source, lines) - for line_number in (start_line for start_line, _, _ in reversed(global_statement_ranges) if start_line > 1): - lines.insert(line_number - 1, '') + for line_number in (filter(lambda x: x > 1, map(operator.itemgetter(0), reversed(global_statement_ranges))): + lines.insert(line_number-1, '') sys.stdout.write(os.linesep.join(lines) + (os.linesep if has_blank_lines else '')) sys.stdout.flush() From 2ae14b87820598f9f56e863447ebe6854168f6f6 Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Mon, 30 Apr 2018 12:22:49 -0700 Subject: [PATCH 06/12] fixed errors --- pythonFiles/normalizeForInterpreter.py | 3 ++- src/test/terminals/codeExecution/terminalCodeExec.test.ts | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pythonFiles/normalizeForInterpreter.py b/pythonFiles/normalizeForInterpreter.py index 588dd169063f..031395eef23b 100644 --- a/pythonFiles/normalizeForInterpreter.py +++ b/pythonFiles/normalizeForInterpreter.py @@ -1,5 +1,6 @@ import ast import io +import operator import os import sys import token @@ -98,7 +99,7 @@ def normalize_lines(source): global_statement_ranges = _get_global_statement_blocks(source, lines) - for line_number in (filter(lambda x: x > 1, map(operator.itemgetter(0), reversed(global_statement_ranges))): + for line_number in filter(lambda x: x > 1, map(operator.itemgetter(0), reversed(global_statement_ranges))): lines.insert(line_number-1, '') sys.stdout.write(os.linesep.join(lines) + (os.linesep if has_blank_lines else '')) diff --git a/src/test/terminals/codeExecution/terminalCodeExec.test.ts b/src/test/terminals/codeExecution/terminalCodeExec.test.ts index 2ada071c2f3c..8fec1ace6b8f 100644 --- a/src/test/terminals/codeExecution/terminalCodeExec.test.ts +++ b/src/test/terminals/codeExecution/terminalCodeExec.test.ts @@ -18,7 +18,7 @@ import { ICodeExecutionService } from '../../../client/terminals/types'; import { PYTHON_PATH } from '../../common'; // tslint:disable-next-line:max-func-body-length -suite('Terminal Code Execution', () => { +suite('Terminal - Code Execution', () => { // tslint:disable-next-line:max-func-body-length ['Terminal Execution', 'Repl Execution', 'Django Execution'].forEach(testSuiteName => { let terminalSettings: TypeMoq.IMock; From 0268dbf2bdc8268e2339a26c320a1e10645435fb Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Mon, 30 Apr 2018 12:43:40 -0700 Subject: [PATCH 07/12] Tweak comment [skip ci] --- pythonFiles/normalizeForInterpreter.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/pythonFiles/normalizeForInterpreter.py b/pythonFiles/normalizeForInterpreter.py index 031395eef23b..cc43df9c7dae 100644 --- a/pythonFiles/normalizeForInterpreter.py +++ b/pythonFiles/normalizeForInterpreter.py @@ -23,11 +23,9 @@ def generic_visit(self, node): def _tokenize(source): - """Tokenize python source using undocumented api - - The documented api does not work in Python 2.7 - - """ + """Tokenize Python source code.""" + # Using an undocumented API as the documented one in Python 2.7 does not work as needed + # cross-version. return tokenize.generate_tokens(io.StringIO(source).readline) From dffef7ced51d8b3c1aa9664ec191e1b1cb976698 Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Mon, 30 Apr 2018 12:50:21 -0700 Subject: [PATCH 08/12] Fix unicode errors --- src/test/pythonFiles/terminalExec/sample6_normalized.py | 2 +- src/test/pythonFiles/terminalExec/sample6_raw.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/pythonFiles/terminalExec/sample6_normalized.py b/src/test/pythonFiles/terminalExec/sample6_normalized.py index 242bf35abea8..d827c678a958 100644 --- a/src/test/pythonFiles/terminalExec/sample6_normalized.py +++ b/src/test/pythonFiles/terminalExec/sample6_normalized.py @@ -2,7 +2,7 @@ print(1) else: print(2) -print('🔨') +print('a') print(3) print(3) diff --git a/src/test/pythonFiles/terminalExec/sample6_raw.py b/src/test/pythonFiles/terminalExec/sample6_raw.py index b064ca962070..2570489630de 100644 --- a/src/test/pythonFiles/terminalExec/sample6_raw.py +++ b/src/test/pythonFiles/terminalExec/sample6_raw.py @@ -1,7 +1,7 @@ if True: print(1) else: print(2) -print('🔨') +print('a') print(3) print(3) if True: From 767e5e89d7fe842eca04e6ee5fd71c06c197533c Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Mon, 30 Apr 2018 13:58:27 -0700 Subject: [PATCH 09/12] Fix unicode error --- pythonFiles/normalizeForInterpreter.py | 7 ++++++- src/test/pythonFiles/terminalExec/sample6_normalized.py | 2 +- src/test/pythonFiles/terminalExec/sample6_raw.py | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/pythonFiles/normalizeForInterpreter.py b/pythonFiles/normalizeForInterpreter.py index cc43df9c7dae..e2180cbbdc7b 100644 --- a/pythonFiles/normalizeForInterpreter.py +++ b/pythonFiles/normalizeForInterpreter.py @@ -38,7 +38,7 @@ def _indent_size(line): def _get_global_statement_blocks(source, lines): """Return a list of all global statement blocks. - The list comprises of 3-item tuples that contain the starting line number, + The list comprises of 3-item tuples that contain the starting line number, ending line number and whether the statement is a single line. """ @@ -106,6 +106,11 @@ def normalize_lines(source): if __name__ == '__main__': contents = sys.argv[1] + try: + # In case content is not escaped. + contents = contents.encode('utf-8', 'surrogateescape').decode('utf-8', 'replace') + except Exception: + pass if isinstance(contents, bytes): contents = contents.decode('utf8') normalize_lines(contents) diff --git a/src/test/pythonFiles/terminalExec/sample6_normalized.py b/src/test/pythonFiles/terminalExec/sample6_normalized.py index d827c678a958..242bf35abea8 100644 --- a/src/test/pythonFiles/terminalExec/sample6_normalized.py +++ b/src/test/pythonFiles/terminalExec/sample6_normalized.py @@ -2,7 +2,7 @@ print(1) else: print(2) -print('a') +print('🔨') print(3) print(3) diff --git a/src/test/pythonFiles/terminalExec/sample6_raw.py b/src/test/pythonFiles/terminalExec/sample6_raw.py index 2570489630de..b064ca962070 100644 --- a/src/test/pythonFiles/terminalExec/sample6_raw.py +++ b/src/test/pythonFiles/terminalExec/sample6_raw.py @@ -1,7 +1,7 @@ if True: print(1) else: print(2) -print('a') +print('🔨') print(3) print(3) if True: From ee1d459ee034aea254953371cac4557d267a3caa Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Mon, 30 Apr 2018 14:36:14 -0700 Subject: [PATCH 10/12] Fix encoding errors --- pythonFiles/normalizeForInterpreter.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pythonFiles/normalizeForInterpreter.py b/pythonFiles/normalizeForInterpreter.py index e2180cbbdc7b..e78955d3e608 100644 --- a/pythonFiles/normalizeForInterpreter.py +++ b/pythonFiles/normalizeForInterpreter.py @@ -107,9 +107,9 @@ def normalize_lines(source): if __name__ == '__main__': contents = sys.argv[1] try: - # In case content is not escaped. - contents = contents.encode('utf-8', 'surrogateescape').decode('utf-8', 'replace') - except Exception: + default_encoding = sys.getdefaultencoding() + contents = contents.encode(default_encoding, 'surrogatepass').decode(default_encoding) + except UnicodeError, LookupError: pass if isinstance(contents, bytes): contents = contents.decode('utf8') From d402a8c9eb3f9ba17d783a801bed4250210267d3 Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Mon, 30 Apr 2018 14:45:46 -0700 Subject: [PATCH 11/12] syntax error --- pythonFiles/normalizeForInterpreter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pythonFiles/normalizeForInterpreter.py b/pythonFiles/normalizeForInterpreter.py index e78955d3e608..bf84dfecd594 100644 --- a/pythonFiles/normalizeForInterpreter.py +++ b/pythonFiles/normalizeForInterpreter.py @@ -109,7 +109,7 @@ def normalize_lines(source): try: default_encoding = sys.getdefaultencoding() contents = contents.encode(default_encoding, 'surrogatepass').decode(default_encoding) - except UnicodeError, LookupError: + except (UnicodeError, LookupError): pass if isinstance(contents, bytes): contents = contents.decode('utf8') From 375df2a8b048739e4472dc8a4add070c7c989460 Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Mon, 30 Apr 2018 14:58:31 -0700 Subject: [PATCH 12/12] More changes --- pythonFiles/normalizeForInterpreter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pythonFiles/normalizeForInterpreter.py b/pythonFiles/normalizeForInterpreter.py index bf84dfecd594..1bb1823b9f04 100644 --- a/pythonFiles/normalizeForInterpreter.py +++ b/pythonFiles/normalizeForInterpreter.py @@ -108,7 +108,7 @@ def normalize_lines(source): contents = sys.argv[1] try: default_encoding = sys.getdefaultencoding() - contents = contents.encode(default_encoding, 'surrogatepass').decode(default_encoding) + contents = contents.encode(default_encoding, 'surrogateescape').decode(default_encoding, 'replace') except (UnicodeError, LookupError): pass if isinstance(contents, bytes):