From fb9bc69ce7a159131afc38fa6eb59ea26ab4dc63 Mon Sep 17 00:00:00 2001 From: Som Snytt Date: Mon, 27 Jan 2025 18:42:18 -0800 Subject: [PATCH 1/3] Permit indent at width after colon arrow eol --- .../dotty/tools/dotc/parsing/Parsers.scala | 5 ++ .../dotty/tools/dotc/parsing/Scanners.scala | 30 +++++++--- .../src/dotty/tools/dotc/parsing/Tokens.scala | 5 +- tests/neg/i22193.scala | 44 ++++++++++++++ tests/pos/i22193.scala | 57 +++++++++++++++++++ 5 files changed, 131 insertions(+), 10 deletions(-) create mode 100644 tests/neg/i22193.scala create mode 100644 tests/pos/i22193.scala diff --git a/compiler/src/dotty/tools/dotc/parsing/Parsers.scala b/compiler/src/dotty/tools/dotc/parsing/Parsers.scala index 968dcccc3d00..e4a09382f3da 100644 --- a/compiler/src/dotty/tools/dotc/parsing/Parsers.scala +++ b/compiler/src/dotty/tools/dotc/parsing/Parsers.scala @@ -1090,6 +1090,7 @@ object Parsers { def isArrowIndent() = lookahead.isArrow && { + lookahead.observeArrowEOL() lookahead.nextToken() lookahead.token == INDENT || lookahead.token == EOF } @@ -2654,10 +2655,14 @@ object Parsers { def closureRest(start: Int, location: Location, params: List[Tree]): Tree = atSpan(start, in.offset) { + if location == Location.InColonArg then + in.observeArrowEOL() if in.token == CTXARROW then if params.isEmpty then syntaxError(em"context function literals require at least one formal parameter", Span(start, in.lastOffset)) in.nextToken() + else if in.token == ARROWeol then + in.nextToken() else accept(ARROW) val body = diff --git a/compiler/src/dotty/tools/dotc/parsing/Scanners.scala b/compiler/src/dotty/tools/dotc/parsing/Scanners.scala index 68c43dc52189..6d04122bee9e 100644 --- a/compiler/src/dotty/tools/dotc/parsing/Scanners.scala +++ b/compiler/src/dotty/tools/dotc/parsing/Scanners.scala @@ -92,7 +92,7 @@ object Scanners { || token == IDENTIFIER && isOperatorPart(name(name.length - 1)) def isArrow = - token == ARROW || token == CTXARROW + token == ARROW || token == CTXARROW || token == ARROWeol } abstract class ScannerCommon(source: SourceFile)(using Context) extends CharArrayReader with TokenData { @@ -612,7 +612,11 @@ object Scanners { insert(if (pastBlankLine) NEWLINES else NEWLINE, lineOffset) else if indentIsSignificant then if nextWidth < lastWidth - || nextWidth == lastWidth && (indentPrefix == MATCH || indentPrefix == CATCH) && token != CASE then + || nextWidth == lastWidth + && indentPrefix.match + case MATCH | CATCH => token != CASE + case _ => false + then if currentRegion.isOutermost then if nextWidth < lastWidth then currentRegion = topLevelRegion(nextWidth) else if !isLeadingInfixOperator(nextWidth) && !statCtdTokens.contains(lastToken) && lastToken != INDENT then @@ -638,9 +642,13 @@ object Scanners { insert(OUTDENT, offset) else if r.isInstanceOf[InBraces] && !closingRegionTokens.contains(token) then report.warning("Line is indented too far to the left, or a `}` is missing", sourcePos()) - else if lastWidth < nextWidth - || lastWidth == nextWidth && (lastToken == MATCH || lastToken == CATCH) && token == CASE then + || lastWidth == nextWidth + && lastToken.match + case MATCH | CATCH => token == CASE + case ARROWeol => true + case _ => false + then if canStartIndentTokens.contains(lastToken) then currentRegion = Indented(nextWidth, lastToken, currentRegion) insert(INDENT, offset) @@ -658,7 +666,7 @@ object Scanners { def spaceTabMismatchMsg(lastWidth: IndentWidth, nextWidth: IndentWidth): Message = em"""Incompatible combinations of tabs and spaces in indentation prefixes. |Previous indent : $lastWidth - |Latest indent : $nextWidth""" + |Latest indent : $nextWidth""" def observeColonEOL(inTemplate: Boolean): Unit = val enabled = @@ -672,6 +680,13 @@ object Scanners { reset() if atEOL then token = COLONeol + def observeArrowEOL(): Unit = + if indentSyntax && token == ARROW then + peekAhead() + val atEOL = isAfterLineEnd || token == EOF + reset() + if atEOL then token = ARROWeol + def observeIndented(): Unit = if indentSyntax && isNewLine then val nextWidth = indentWidth(next.offset) @@ -680,7 +695,6 @@ object Scanners { currentRegion = Indented(nextWidth, COLONeol, currentRegion) offset = next.offset token = INDENT - end observeIndented /** Insert an token if next token closes an indentation region. * Exception: continue if indentation region belongs to a `match` and next token is `case`. @@ -1100,7 +1114,7 @@ object Scanners { reset() next - class LookaheadScanner(val allowIndent: Boolean = false) extends Scanner(source, offset, allowIndent = allowIndent) { + class LookaheadScanner(allowIndent: Boolean = false) extends Scanner(source, offset, allowIndent = allowIndent) { override protected def initialCharBufferSize = 8 override def languageImportContext = Scanner.this.languageImportContext } @@ -1652,7 +1666,7 @@ object Scanners { case class InCase(outer: Region) extends Region(OUTDENT) /** A class describing an indentation region. - * @param width The principal indendation width + * @param width The principal indentation width * @param prefix The token before the initial of the region */ case class Indented(width: IndentWidth, prefix: Token, outer: Region | Null) extends Region(OUTDENT): diff --git a/compiler/src/dotty/tools/dotc/parsing/Tokens.scala b/compiler/src/dotty/tools/dotc/parsing/Tokens.scala index c78a336ecdf5..cc0e92953fb8 100644 --- a/compiler/src/dotty/tools/dotc/parsing/Tokens.scala +++ b/compiler/src/dotty/tools/dotc/parsing/Tokens.scala @@ -202,9 +202,10 @@ object Tokens extends TokensCommon { inline val COLONeol = 89; enter(COLONeol, ":", ": at eol") // A `:` recognized as starting an indentation block inline val SELFARROW = 90; enter(SELFARROW, "=>") // reclassified ARROW following self-type + inline val ARROWeol = 99; enter(ARROWeol, "=>", "=> at eol") // lambda ARROW at eol followed by indent /** XML mode */ - inline val XMLSTART = 99; enter(XMLSTART, "$XMLSTART$<") // TODO: deprecate + inline val XMLSTART = 100; enter(XMLSTART, "$XMLSTART$<") // TODO: deprecate final val alphaKeywords: TokenSet = tokenRange(IF, END) final val symbolicKeywords: TokenSet = tokenRange(USCORE, CTXARROW) @@ -282,7 +283,7 @@ object Tokens extends TokensCommon { final val closingRegionTokens = BitSet(RBRACE, RPAREN, RBRACKET, CASE) | statCtdTokens final val canStartIndentTokens: BitSet = - statCtdTokens | BitSet(COLONeol, WITH, EQUALS, ARROW, CTXARROW, LARROW, WHILE, TRY, FOR, IF, THROW, RETURN) + statCtdTokens | BitSet(COLONeol, WITH, EQUALS, ARROWeol, ARROW, CTXARROW, LARROW, WHILE, TRY, FOR, IF, THROW, RETURN) /** Faced with the choice between a type and a formal parameter, the following * tokens determine it's a formal parameter. diff --git a/tests/neg/i22193.scala b/tests/neg/i22193.scala new file mode 100644 index 000000000000..b8bad8d070f2 --- /dev/null +++ b/tests/neg/i22193.scala @@ -0,0 +1,44 @@ + +def fn2(arg: String, arg2: String)(f: String => Unit): Unit = f(arg) + +def fn3(arg: String, arg2: String)(f: => Unit): Unit = f + +def test1() = + + fn2(arg = "blue sleeps faster than tuesday", arg2 = "the quick brown fox jumped over the lazy dog"): env => + val x = env + println(x) + + fn2( // error not a legal formal parameter for a function literal + arg = "blue sleeps faster than tuesday", + arg2 = "the quick brown fox jumped over the lazy dog"): env => + val x = env // error + println(x) + + fn2( // error + arg = "blue sleeps faster than tuesday", + arg2 = "the quick brown fox jumped over the lazy dog"): env => + val x = env // error + println(x) + + fn2( + arg = "blue sleeps faster than tuesday", + arg2 = "the quick brown fox jumped over the lazy dog"): + env => // error indented definitions expected, identifier env found + val x = env + println(x) + +def test2() = + + fn2( + arg = "blue sleeps faster than tuesday", + arg2 = "the quick brown fox jumped over the lazy dog" + ): env => + val x = env + println(x) + + fn3( // error missing argument list for value of type (=> Unit) => Unit + arg = "blue sleeps faster than tuesday", + arg2 = "the quick brown fox jumped over the lazy dog"): + val x = "Hello" // error + println(x) // error diff --git a/tests/pos/i22193.scala b/tests/pos/i22193.scala new file mode 100644 index 000000000000..14d9359c4701 --- /dev/null +++ b/tests/pos/i22193.scala @@ -0,0 +1,57 @@ + +def fn2(arg: String, arg2: String)(f: String => Unit): Unit = f(arg) + +def fn3(arg: String, arg2: String)(f: => Unit): Unit = f + +def test() = + + fn2(arg = "blue sleeps faster than tuesday", arg2 = "the quick brown fox jumped over the lazy dog"): env => + val x = env + println(x) + + // doesn't compile + fn2( + arg = "blue sleeps faster than tuesday", + arg2 = "the quick brown fox jumped over the lazy dog"): env => + val x = env + println(x) + + fn2( + arg = "blue sleeps faster than tuesday", + arg2 = "the quick brown fox jumped over the lazy dog"): env => + val x = env + println(x) + + // does compile + fn2( + arg = "blue sleeps faster than tuesday", + arg2 = "the quick brown fox jumped over the lazy dog"): + env => + val x = env + println(x) + + // does compile + fn2( + arg = "blue sleeps faster than tuesday", + arg2 = "the quick brown fox jumped over the lazy dog" + ): env => + val x = env + println(x) + + fn3( + arg = "blue sleeps faster than tuesday", + arg2 = "the quick brown fox jumped over the lazy dog"): + val x = "Hello" + println(x) + + fn3( + arg = "blue sleeps faster than tuesday", + arg2 = "the quick brown fox jumped over the lazy dog"): + val x = "Hello" + println(x) + +// don't turn innocent empty cases into functions +def regress(x: Int) = + x match + case 42 => + case _ => From 5ad196e6cd285477931c4213719b59687060e0e4 Mon Sep 17 00:00:00 2001 From: Som Snytt Date: Wed, 29 Jan 2025 12:50:26 -0800 Subject: [PATCH 2/3] Fix indent insertion at arrow eol --- .../dotty/tools/dotc/parsing/Parsers.scala | 9 ++---- .../dotty/tools/dotc/parsing/Scanners.scala | 28 +++++++++---------- .../src/dotty/tools/dotc/parsing/Tokens.scala | 5 ++-- tests/neg/i22193.scala | 14 +--------- tests/pos/i22193.scala | 21 ++++++++++++++ 5 files changed, 40 insertions(+), 37 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/parsing/Parsers.scala b/compiler/src/dotty/tools/dotc/parsing/Parsers.scala index e4a09382f3da..ef07d477c303 100644 --- a/compiler/src/dotty/tools/dotc/parsing/Parsers.scala +++ b/compiler/src/dotty/tools/dotc/parsing/Parsers.scala @@ -34,6 +34,7 @@ import config.Feature.{sourceVersion, migrateTo3} import config.SourceVersion.* import config.SourceVersion import dotty.tools.dotc.config.MigrationVersion +import dotty.tools.dotc.util.chaining.* object Parsers { @@ -1087,11 +1088,11 @@ object Parsers { */ def followingIsLambdaAfterColon(): Boolean = val lookahead = in.LookaheadScanner(allowIndent = true) + .tap(_.currentRegion.knownWidth = in.currentRegion.indentWidth) def isArrowIndent() = lookahead.isArrow && { - lookahead.observeArrowEOL() - lookahead.nextToken() + lookahead.observeArrowIndented() lookahead.token == INDENT || lookahead.token == EOF } lookahead.nextToken() @@ -2655,14 +2656,10 @@ object Parsers { def closureRest(start: Int, location: Location, params: List[Tree]): Tree = atSpan(start, in.offset) { - if location == Location.InColonArg then - in.observeArrowEOL() if in.token == CTXARROW then if params.isEmpty then syntaxError(em"context function literals require at least one formal parameter", Span(start, in.lastOffset)) in.nextToken() - else if in.token == ARROWeol then - in.nextToken() else accept(ARROW) val body = diff --git a/compiler/src/dotty/tools/dotc/parsing/Scanners.scala b/compiler/src/dotty/tools/dotc/parsing/Scanners.scala index 6d04122bee9e..e5bba6c3b73b 100644 --- a/compiler/src/dotty/tools/dotc/parsing/Scanners.scala +++ b/compiler/src/dotty/tools/dotc/parsing/Scanners.scala @@ -92,7 +92,7 @@ object Scanners { || token == IDENTIFIER && isOperatorPart(name(name.length - 1)) def isArrow = - token == ARROW || token == CTXARROW || token == ARROWeol + token == ARROW || token == CTXARROW } abstract class ScannerCommon(source: SourceFile)(using Context) extends CharArrayReader with TokenData { @@ -612,11 +612,7 @@ object Scanners { insert(if (pastBlankLine) NEWLINES else NEWLINE, lineOffset) else if indentIsSignificant then if nextWidth < lastWidth - || nextWidth == lastWidth - && indentPrefix.match - case MATCH | CATCH => token != CASE - case _ => false - then + || nextWidth == lastWidth && (indentPrefix == MATCH || indentPrefix == CATCH) && token != CASE then if currentRegion.isOutermost then if nextWidth < lastWidth then currentRegion = topLevelRegion(nextWidth) else if !isLeadingInfixOperator(nextWidth) && !statCtdTokens.contains(lastToken) && lastToken != INDENT then @@ -643,12 +639,7 @@ object Scanners { else if r.isInstanceOf[InBraces] && !closingRegionTokens.contains(token) then report.warning("Line is indented too far to the left, or a `}` is missing", sourcePos()) else if lastWidth < nextWidth - || lastWidth == nextWidth - && lastToken.match - case MATCH | CATCH => token == CASE - case ARROWeol => true - case _ => false - then + || lastWidth == nextWidth && (lastToken == MATCH || lastToken == CATCH) && token == CASE then if canStartIndentTokens.contains(lastToken) then currentRegion = Indented(nextWidth, lastToken, currentRegion) insert(INDENT, offset) @@ -680,12 +671,19 @@ object Scanners { reset() if atEOL then token = COLONeol - def observeArrowEOL(): Unit = - if indentSyntax && token == ARROW then + // consume => and insert if applicable + def observeArrowIndented(): Unit = + if isArrow && indentSyntax then peekAhead() val atEOL = isAfterLineEnd || token == EOF reset() - if atEOL then token = ARROWeol + if atEOL then + val nextWidth = indentWidth(next.offset) + val lastWidth = currentRegion.indentWidth + if lastWidth < nextWidth then + currentRegion = Indented(nextWidth, COLONeol, currentRegion) + offset = next.offset + token = INDENT def observeIndented(): Unit = if indentSyntax && isNewLine then diff --git a/compiler/src/dotty/tools/dotc/parsing/Tokens.scala b/compiler/src/dotty/tools/dotc/parsing/Tokens.scala index cc0e92953fb8..c78a336ecdf5 100644 --- a/compiler/src/dotty/tools/dotc/parsing/Tokens.scala +++ b/compiler/src/dotty/tools/dotc/parsing/Tokens.scala @@ -202,10 +202,9 @@ object Tokens extends TokensCommon { inline val COLONeol = 89; enter(COLONeol, ":", ": at eol") // A `:` recognized as starting an indentation block inline val SELFARROW = 90; enter(SELFARROW, "=>") // reclassified ARROW following self-type - inline val ARROWeol = 99; enter(ARROWeol, "=>", "=> at eol") // lambda ARROW at eol followed by indent /** XML mode */ - inline val XMLSTART = 100; enter(XMLSTART, "$XMLSTART$<") // TODO: deprecate + inline val XMLSTART = 99; enter(XMLSTART, "$XMLSTART$<") // TODO: deprecate final val alphaKeywords: TokenSet = tokenRange(IF, END) final val symbolicKeywords: TokenSet = tokenRange(USCORE, CTXARROW) @@ -283,7 +282,7 @@ object Tokens extends TokensCommon { final val closingRegionTokens = BitSet(RBRACE, RPAREN, RBRACKET, CASE) | statCtdTokens final val canStartIndentTokens: BitSet = - statCtdTokens | BitSet(COLONeol, WITH, EQUALS, ARROWeol, ARROW, CTXARROW, LARROW, WHILE, TRY, FOR, IF, THROW, RETURN) + statCtdTokens | BitSet(COLONeol, WITH, EQUALS, ARROW, CTXARROW, LARROW, WHILE, TRY, FOR, IF, THROW, RETURN) /** Faced with the choice between a type and a formal parameter, the following * tokens determine it's a formal parameter. diff --git a/tests/neg/i22193.scala b/tests/neg/i22193.scala index b8bad8d070f2..f7ee5b1cf5e1 100644 --- a/tests/neg/i22193.scala +++ b/tests/neg/i22193.scala @@ -5,6 +5,7 @@ def fn3(arg: String, arg2: String)(f: => Unit): Unit = f def test1() = + // ok baseline fn2(arg = "blue sleeps faster than tuesday", arg2 = "the quick brown fox jumped over the lazy dog"): env => val x = env println(x) @@ -15,12 +16,6 @@ def test1() = val x = env // error println(x) - fn2( // error - arg = "blue sleeps faster than tuesday", - arg2 = "the quick brown fox jumped over the lazy dog"): env => - val x = env // error - println(x) - fn2( arg = "blue sleeps faster than tuesday", arg2 = "the quick brown fox jumped over the lazy dog"): @@ -30,13 +25,6 @@ def test1() = def test2() = - fn2( - arg = "blue sleeps faster than tuesday", - arg2 = "the quick brown fox jumped over the lazy dog" - ): env => - val x = env - println(x) - fn3( // error missing argument list for value of type (=> Unit) => Unit arg = "blue sleeps faster than tuesday", arg2 = "the quick brown fox jumped over the lazy dog"): diff --git a/tests/pos/i22193.scala b/tests/pos/i22193.scala index 14d9359c4701..588fc20cb785 100644 --- a/tests/pos/i22193.scala +++ b/tests/pos/i22193.scala @@ -22,6 +22,12 @@ def test() = val x = env println(x) + fn2( + arg = "blue sleeps faster than tuesday", + arg2 = "the quick brown fox jumped over the lazy dog"): env => + val x = env + println(x) + // does compile fn2( arg = "blue sleeps faster than tuesday", @@ -38,6 +44,13 @@ def test() = val x = env println(x) + fn2( + arg = "blue sleeps faster than tuesday", + arg2 = "the quick brown fox jumped over the lazy dog" + ): env => + val x = env + println(x) + fn3( arg = "blue sleeps faster than tuesday", arg2 = "the quick brown fox jumped over the lazy dog"): @@ -55,3 +68,11 @@ def regress(x: Int) = x match case 42 => case _ => + +// previously lookahead calculated indent width at the colon +def k(xs: List[Int]) = + xs.foldLeft( + 0) + : (acc, x) => + acc + x + From b57549e66fb3e63f18adc674fe1ae4a5fd4c1fa9 Mon Sep 17 00:00:00 2001 From: Som Snytt Date: Wed, 29 Jan 2025 15:32:15 -0800 Subject: [PATCH 3/3] More OG tests --- tests/pos/i22193.scala | 63 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/tests/pos/i22193.scala b/tests/pos/i22193.scala index 588fc20cb785..2ba7f920fbd7 100644 --- a/tests/pos/i22193.scala +++ b/tests/pos/i22193.scala @@ -63,6 +63,30 @@ def test() = val x = "Hello" println(x) + fn3( // arg at 3, body at 3 + arg = "blue sleeps faster than tuesday", + arg2 = "the quick brown fox jumped over the lazy dog"): + val x = "Hello" + println(x) + + fn3( // arg at 3, body at 1: not sure if sig indent of 1 is allowed, saw some comments from odersky + arg = "blue sleeps faster than tuesday", + arg2 = "the quick brown fox jumped over the lazy dog"): + val x = "Hello" + println(x) + + fn3( // arg at 3, body at 2: even if sig indent of 1 is not allowed, body is at fn3+2, not arg2-1 + arg = "blue sleeps faster than tuesday", + arg2 = "the quick brown fox jumped over the lazy dog"): + val x = "Hello" + println(x) + + fn3( // arg at 3, body at 4 + arg = "blue sleeps faster than tuesday", + arg2 = "the quick brown fox jumped over the lazy dog"): + val x = "Hello" + println(x) + // don't turn innocent empty cases into functions def regress(x: Int) = x match @@ -76,3 +100,42 @@ def k(xs: List[Int]) = : (acc, x) => acc + x +def `test kit`(xs: List[Int]): Unit = + def addOne(i: Int): Int = i + 1 + def isPositive(i: Int): Boolean = i > 0 + // doesn't compile but would be nice + // first body is indented "twice", or, rather, first outdent establishes an intermediate indentation level + xs.map: x => + x + 1 + .filter: x => + x > 0 + xs.map: + addOne + .filter: + isPositive + + // does compile + xs + .map: x => + x + 1 + .filter: x => + x > 0 + + // does compile but doesn't look good, at least, to some people + xs.map: x => + x + 1 + .filter: x => + x > 0 + +def `tested kit`(xs: List[Int]): Unit = + { + def addOne(i: Int): Int = i.+(1) + def isPositive(i: Int): Boolean = i.>(0) + xs.map[Int]((x: Int) => x.+(1)).filter((x: Int) => x.>(0)) + xs.map[Int]((i: Int) => addOne(i)).filter((i: Int) => isPositive(i)) + xs.map[Int]((x: Int) => x.+(1)).filter((x: Int) => x.>(0)) + { + xs.map[Int]((x: Int) => x.+(1)).filter((x: Int) => x.>(0)) + () + } + }