Correct parsing for f-string string literal concatenation

rewrite-python-remote/src/main/java/org/openrewrite/python/remote/PythonReceiver.java

@@ -411,6 +411,15 @@ public Py.Slice visitSlice(Py.Slice slice, ReceiverContext ctx) {
             return slice;
         }
 
+        @Override
+        public Py.StringLiteralConcatenation visitStringLiteralConcatenation(Py.StringLiteralConcatenation stringLiteralConcatenation, ReceiverContext ctx) {
+            stringLiteralConcatenation = stringLiteralConcatenation.withId(ctx.receiveNonNullValue(stringLiteralConcatenation.getId(), UUID.class));
+            stringLiteralConcatenation = stringLiteralConcatenation.withPrefix(ctx.receiveNonNullNode(stringLiteralConcatenation.getPrefix(), PythonReceiver::receiveSpace));
+            stringLiteralConcatenation = stringLiteralConcatenation.withMarkers(ctx.receiveNonNullNode(stringLiteralConcatenation.getMarkers(), ctx::receiveMarkers));
+            stringLiteralConcatenation = stringLiteralConcatenation.getPadding().withLiterals(ctx.receiveNonNullNodes(stringLiteralConcatenation.getPadding().getLiterals(), PythonReceiver::receiveRightPaddedTree));
+            return stringLiteralConcatenation;
+        }
+
         @Override
         public J.AnnotatedType visitAnnotatedType(J.AnnotatedType annotatedType, ReceiverContext ctx) {
             annotatedType = annotatedType.withId(ctx.receiveNonNullValue(annotatedType.getId(), UUID.class));
@@ -1449,6 +1458,15 @@ public <T> T create(Class<T> type, ReceiverContext ctx) {
                 );
             }
 
+            if (type == Py.StringLiteralConcatenation.class) {
+                return (T) new Py.StringLiteralConcatenation(
+                    ctx.receiveNonNullValue(null, UUID.class),
+                    ctx.receiveNonNullNode(null, PythonReceiver::receiveSpace),
+                    ctx.receiveNonNullNode(null, ctx::receiveMarkers),
+                    ctx.receiveNonNullNodes(null, PythonReceiver::receiveRightPaddedTree)
+                );
+            }
+
             if (type == J.AnnotatedType.class) {
                 return (T) new J.AnnotatedType(
                     ctx.receiveNonNullValue(null, UUID.class),

rewrite-python-remote/src/main/java/org/openrewrite/python/remote/PythonSender.java

@@ -394,6 +394,15 @@ public Py.Slice visitSlice(Py.Slice slice, SenderContext ctx) {
             return slice;
         }
 
+        @Override
+        public Py.StringLiteralConcatenation visitStringLiteralConcatenation(Py.StringLiteralConcatenation stringLiteralConcatenation, SenderContext ctx) {
+            ctx.sendValue(stringLiteralConcatenation, Py.StringLiteralConcatenation::getId);
+            ctx.sendNode(stringLiteralConcatenation, Py.StringLiteralConcatenation::getPrefix, PythonSender::sendSpace);
+            ctx.sendNode(stringLiteralConcatenation, Py.StringLiteralConcatenation::getMarkers, ctx::sendMarkers);
+            ctx.sendNodes(stringLiteralConcatenation, e -> e.getPadding().getLiterals(), PythonSender::sendRightPadded, e -> e.getElement().getId());
+            return stringLiteralConcatenation;
+        }
+
         @Override
         public J.AnnotatedType visitAnnotatedType(J.AnnotatedType annotatedType, SenderContext ctx) {
             ctx.sendValue(annotatedType, J.AnnotatedType::getId);

rewrite-python/src/main/java/org/openrewrite/python/PythonIsoVisitor.java

@@ -157,6 +157,11 @@ public Py.TrailingElseWrapper visitTrailingElseWrapper(Py.TrailingElseWrapper og
         return (Py.TrailingElseWrapper) super.visitTrailingElseWrapper(ogWrapper, p);
     }
 
+    @Override
+    public Py.StringLiteralConcatenation visitStringLiteralConcatenation(Py.StringLiteralConcatenation stringLiteralConcatenation, P p) {
+        return (Py.StringLiteralConcatenation) super.visitStringLiteralConcatenation(stringLiteralConcatenation, p);
+    }
+
     @Override
     public Py.TypeHint visitTypeHint(Py.TypeHint ogType, P p) {
         return (Py.TypeHint) super.visitTypeHint(ogType, p);

rewrite-python/src/main/java/org/openrewrite/python/PythonParser.java

@@ -172,8 +172,8 @@ private void initializeRemoting(ExecutionContext ctx) throws IOException {
                 processBuilder.redirectOutput(new File("NUL"));
                 processBuilder.redirectError(new File("NUL"));
             } else {
-                processBuilder.redirectOutput(new File("/dev/null"));
-                processBuilder.redirectError(new File("/dev/null"));
+                processBuilder.redirectOutput(new File("/tmp/out.txt"));
+                processBuilder.redirectError(new File("/tmp/err.txt"));
             }
             pythonProcess = processBuilder.start();
             for (int i = 0; i < 30 && pythonProcess.isAlive(); i++) {

rewrite-python/src/main/java/org/openrewrite/python/PythonVisitor.java

@@ -369,6 +369,20 @@ public J visitStatementExpression(Py.StatementExpression statementExpression, P
         return visitExpression(expr, p);
     }
 
+    public J visitStringLiteralConcatenation(Py.StringLiteralConcatenation stringLiteralConcatenation, P p) {
+        Py.StringLiteralConcatenation slc = stringLiteralConcatenation;
+        slc = slc.withPrefix(visitSpace(slc.getPrefix(), PySpace.Location.STRING_LITERAL_CONCATENATION_PREFIX, p));
+        slc = slc.withMarkers(visitMarkers(slc.getMarkers(), p));
+        Expression temp = (Expression) visitExpression(slc, p);
+        if (!(temp instanceof Py.StringLiteralConcatenation)) {
+            return temp;
+        } else {
+            slc = (Py.StringLiteralConcatenation) temp;
+        }
+        slc = slc.getPadding().withLiterals(ListUtils.map(slc.getPadding().getLiterals(), l -> visitRightPadded(l, PyRightPadded.Location.STRING_LITERAL_CONCATENATION_LITERAL, p)));
+        return slc;
+    }
+
     public J visitTypeHint(Py.TypeHint ogType, P p) {
         Py.TypeHint type = ogType;
         type = type.withPrefix(visitSpace(type.getPrefix(), PySpace.Location.EXCEPTION_TYPE_PREFIX, p));

rewrite-python/src/main/java/org/openrewrite/python/internal/PythonPrinter.java

@@ -628,6 +628,14 @@ public J visitTrailingElseWrapper(Py.TrailingElseWrapper wrapper, PrintOutputCap
         return wrapper;
     }
 
+    @Override
+    public J visitStringLiteralConcatenation(Py.StringLiteralConcatenation concatenation, PrintOutputCapture<P> p) {
+        beforeSyntax(concatenation, PySpace.Location.STRING_LITERAL_CONCATENATION_PREFIX, p);
+        visitRightPadded(concatenation.getPadding().getLiterals(), PyRightPadded.Location.STRING_LITERAL_CONCATENATION_LITERAL, "", p);
+        afterSyntax(concatenation, p);
+        return concatenation;
+    }
+
     @Override
     public J visitTypeHint(Py.TypeHint type, PrintOutputCapture<P> p) {
         beforeSyntax(type, PySpace.Location.TYPE_HINT_PREFIX, p);

rewrite-python/src/main/java/org/openrewrite/python/tree/Py.java

@@ -2253,4 +2253,86 @@ public Slice withStep(@Nullable JRightPadded<Expression> step) {
         }
     }
 
+    @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+    @EqualsAndHashCode(callSuper = false)
+    @RequiredArgsConstructor
+    @AllArgsConstructor(access = AccessLevel.PRIVATE)
+    final class StringLiteralConcatenation implements Py, Expression, TypedTree {
+        @Nullable
+        @NonFinal
+        transient WeakReference<Padding> padding;
+
+        @Getter
+        @With
+        @EqualsAndHashCode.Include
+        UUID id;
+
+        @Getter
+        @With
+        Space prefix;
+
+        @Getter
+        @With
+        Markers markers;
+
+        List<JRightPadded<Expression>> literals;
+
+        public List<Expression> getLiterals() {
+            return JRightPadded.getElements(literals);
+        }
+
+        public StringLiteralConcatenation withLiterals(List<Expression> literals) {
+            return getPadding().withLiterals(JRightPadded.withElements(this.literals, literals));
+        }
+
+        @Override
+        public JavaType getType() {
+            return JavaType.Primitive.String;
+        }
+
+        @Override
+        public <T extends J> T withType(@Nullable JavaType type) {
+            //noinspection unchecked
+            return (T) this;
+        }
+
+        @Override
+        public <P> J acceptPython(PythonVisitor<P> v, P p) {
+            return v.visitStringLiteralConcatenation(this, p);
+        }
+
+        @Override
+        @Transient
+        public CoordinateBuilder.Expression getCoordinates() {
+            return new CoordinateBuilder.Expression(this);
+        }
+
+        public Padding getPadding() {
+            Padding p;
+            if (this.padding == null) {
+                p = new Padding(this);
+                this.padding = new WeakReference<>(p);
+            } else {
+                p = this.padding.get();
+                if (p == null || p.t != this) {
+                    p = new Padding(this);
+                    this.padding = new WeakReference<>(p);
+                }
+            }
+            return p;
+        }
+
+        @RequiredArgsConstructor
+        public static class Padding {
+            private final StringLiteralConcatenation t;
+
+            public List<JRightPadded<Expression>> getLiterals() {
+                return t.literals;
+            }
+
+            public StringLiteralConcatenation withLiterals(List<JRightPadded<Expression>> literals) {
+                return t.literals == literals ? t : new StringLiteralConcatenation(t.id, t.prefix, t.markers, literals);
+            }
+        }
+    }
 }

rewrite-python/src/main/java/org/openrewrite/python/tree/PyRightPadded.java

@@ -37,6 +37,7 @@ public enum Location {
         SLICE_EXPRESSION_START(PySpace.Location.SLICE_START_SUFFIX),
         SLICE_EXPRESSION_STEP(PySpace.Location.SLICE_STEP_SUFFIX),
         SLICE_EXPRESSION_STOP(PySpace.Location.SLICE_STOP_SUFFIX),
+        STRING_LITERAL_CONCATENATION_LITERAL(PySpace.Location.STRING_LITERAL_CONCATENATION_LITERAL_SUFFIX),
         TOP_LEVEL_STATEMENT_SUFFIX(PySpace.Location.TOP_LEVEL_STATEMENT),
         UNION_TYPE_TYPE(PySpace.Location.UNION_ELEMENT_SUFFIX),
         VARIABLE_SCOPE_ELEMENT(PySpace.Location.VARIABLE_SCOPE_NAME_SUFFIX),

rewrite-python/src/main/java/org/openrewrite/python/tree/PySpace.java

@@ -366,6 +366,8 @@ public enum Location {
         SLICE_STOP_SUFFIX,
         SPECIAL_PARAM_PREFIX,
         STAR_PREFIX,
+        STRING_LITERAL_CONCATENATION_LITERAL_SUFFIX,
+        STRING_LITERAL_CONCATENATION_PREFIX,
         TOP_LEVEL_STATEMENT,
         TRAILING_ELSE_WRAPPER_PREFIX,
         TYPE_HINTED_EXPRESSION_PREFIX,

rewrite/rewrite/python/__init__.py

@@ -41,6 +41,7 @@
     'SpecialParameter',
     'Star',
     'StatementExpression',
+    'StringLiteralConcatenation',
     'TrailingElseWrapper',
     'TypeHint',
     'TypeHintedExpression',

rewrite/rewrite/python/_parser_visitor.py

@@ -1294,15 +1294,31 @@ def visit_Constant(self, node):
             else:
                 break
 
-        return j.Literal(
-            random_id(),
-            prefix,
-            Markers.EMPTY,
-            None if node.value is Ellipsis else node.value,
-            self._source[start:self._cursor],
-            None,
-            self.__map_type(node),
-        )
+        if isinstance(node.value, str) and '\n' in node.value:
+            return py.StringLiteralConcatenation(
+                random_id(),
+                prefix,
+                Markers.EMPTY,
+                [self.__pad_right(j.Literal(
+                    random_id(),
+                    Space.EMPTY,
+                    Markers.EMPTY,
+                    None if node.value is Ellipsis else node.value,
+                    self._source[start:self._cursor],
+                    None,
+                    self.__map_type(node),
+                ), Space.EMPTY)]
+            )
+        else:
+            return j.Literal(
+                random_id(),
+                prefix,
+                Markers.EMPTY,
+                None if node.value is Ellipsis else node.value,
+                self._source[start:self._cursor],
+                None,
+                self.__map_type(node),
+            )
 
 
     def visit_Dict(self, node):
@@ -2123,6 +2139,7 @@ def __map_fstring(self, node: ast.JoinedStr, prefix: Space, tok: TokenInfo, toke
 
         # tokenizer tokens: FSTRING_START, FSTRING_MIDDLE, OP, ..., OP, FSTRING_MIDDLE, FSTRING_END
         parts = []
+        literals = []
         for value in node.values:
             if tok.type == token.OP and tok.string == '{':
                 if not isinstance(value, ast.FormattedValue):
@@ -2201,29 +2218,37 @@ def __map_fstring(self, node: ast.JoinedStr, prefix: Space, tok: TokenInfo, toke
                     self._cursor += len(tok.string) + (1 if tok.string.endswith('{') or tok.string.endswith('}') else 0)
                     if (tok := next(tokens)).type != token.FSTRING_MIDDLE:
                         break
-                parts.append(j.Literal(
+                literals.append(self.__pad_right(j.Literal(
                     random_id(),
                     Space.EMPTY,
                     Markers.EMPTY,
                     cast(ast.Constant, value).s,
                     self._source[save_cursor:self._cursor],
                     None,
                     self.__map_type(value),
-                ))
+                ), Space.EMPTY))
 
         if consume_end_delim:
             self._cursor += len(tok.string)  # FSTRING_END token
             tok = next(tokens)
         elif tok.type == token.FSTRING_MIDDLE and len(tok.string) == 0:
             tok = next(tokens)
 
-        return (py.FormattedString(
-            random_id(),
-            prefix,
-            Markers.EMPTY,
-            delimiter,
-            parts
-        ), tok)
+        if literals:
+            return (py.StringLiteralConcatenation(
+                random_id(),
+                prefix,
+                Markers.EMPTY,
+                literals
+            ), tok)
+        else:
+            return (py.FormattedString(
+                random_id(),
+                prefix,
+                Markers.EMPTY,
+                delimiter,
+                parts
+            ), tok)
 
     def __cursor_at(self, s: str):
         return self._cursor < len(self._source) and (len(s) == 1 and self._source[self._cursor] == s or self._source.startswith(s, self._cursor))

rewrite/rewrite/python/remote/receiver.py

@@ -306,6 +306,13 @@ def visit_slice(self, slice: Slice, ctx: ReceiverContext) -> J:
             slice = slice.padding.with_step(ctx.receive_node(slice.padding.step, PythonReceiver.receive_right_padded_tree))
             return slice
 
+        def visit_string_literal_concatenation(self, string_literal_concatenation: StringLiteralConcatenation, ctx: ReceiverContext) -> J:
+            string_literal_concatenation = string_literal_concatenation.with_id(ctx.receive_value(string_literal_concatenation.id, UUID))
+            string_literal_concatenation = string_literal_concatenation.with_prefix(ctx.receive_node(string_literal_concatenation.prefix, PythonReceiver.receive_space))
+            string_literal_concatenation = string_literal_concatenation.with_markers(ctx.receive_node(string_literal_concatenation.markers, ctx.receive_markers))
+            string_literal_concatenation = string_literal_concatenation.padding.with_literals(ctx.receive_nodes(string_literal_concatenation.padding.literals, PythonReceiver.receive_right_padded_tree))
+            return string_literal_concatenation
+
         def visit_annotated_type(self, annotated_type: AnnotatedType, ctx: ReceiverContext) -> J:
             annotated_type = annotated_type.with_id(ctx.receive_value(annotated_type.id, UUID))
             annotated_type = annotated_type.with_prefix(ctx.receive_node(annotated_type.prefix, PythonReceiver.receive_space))
@@ -1213,6 +1220,14 @@ def create(self, type: str, ctx: ReceiverContext) -> Tree:
                     ctx.receive_node(None, PythonReceiver.receive_right_padded_tree)
                 )
 
+            if type in ["rewrite.python.tree.StringLiteralConcatenation", "org.openrewrite.python.tree.Py$StringLiteralConcatenation"]:
+                return StringLiteralConcatenation(
+                    ctx.receive_value(None, UUID),
+                    ctx.receive_node(None, PythonReceiver.receive_space),
+                    ctx.receive_node(None, ctx.receive_markers),
+                    ctx.receive_nodes(None, PythonReceiver.receive_right_padded_tree)
+                )
+
             if type in ["rewrite.python.tree.AnnotatedType", "org.openrewrite.java.tree.J$AnnotatedType"]:
                 return AnnotatedType(
                     ctx.receive_value(None, UUID),

rewrite/rewrite/python/remote/sender.py

@@ -300,6 +300,13 @@ def visit_slice(self, slice: Slice, ctx: SenderContext) -> J:
             ctx.send_node(slice, attrgetter('_step'), PythonSender.send_right_padded)
             return slice
 
+        def visit_string_literal_concatenation(self, string_literal_concatenation: StringLiteralConcatenation, ctx: SenderContext) -> J:
+            ctx.send_value(string_literal_concatenation, attrgetter('_id'))
+            ctx.send_node(string_literal_concatenation, attrgetter('_prefix'), PythonSender.send_space)
+            ctx.send_node(string_literal_concatenation, attrgetter('_markers'), ctx.send_markers)
+            ctx.send_nodes(string_literal_concatenation, attrgetter('_literals'), PythonSender.send_right_padded, lambda t: t.element.id)
+            return string_literal_concatenation
+
         def visit_annotated_type(self, annotated_type: AnnotatedType, ctx: SenderContext) -> J:
             ctx.send_value(annotated_type, attrgetter('_id'))
             ctx.send_node(annotated_type, attrgetter('_prefix'), PythonSender.send_space)

rewrite/rewrite/python/support_types.py

@@ -79,12 +79,14 @@ class Location(Enum):
         SLICE_STOP_SUFFIX = auto()
         SPECIAL_PARAMETER_PREFIX = auto()
         STAR_PREFIX = auto()
+        STRING_LITERAL_CONCATENATION_LITERAL_SUFFIX = auto()
+        STRING_LITERAL_CONCATENATION_PREFIX = auto()
         TOP_LEVEL_STATEMENT = auto()
         TRAILING_ELSE_WRAPPER_PREFIX = auto()
         TYPE_HINTED_EXPRESSION_PREFIX = auto()
         TYPE_HINT_PREFIX = auto()
         UNION_TYPE_PREFIX = auto()
-        UNION_TYPES_SUFFIX = auto()
+        UNION_TYPE_TYPES_SUFFIX = auto()
         VARIABLE_SCOPE_NAME_SUFFIX = auto()
         VARIABLE_SCOPE_PREFIX = auto()
         YIELD_EXPRESSION_SUFFIX = auto()
@@ -98,6 +100,7 @@ class Location(Enum):
 
 class PyRightPadded:
     class Location(Enum):
+        STRING_LITERAL_CONCATENATION_LITERALS = PySpace.Location.STRING_LITERAL_CONCATENATION_LITERAL_SUFFIX
         CHAINED_ASSIGNMENT_VARIABLES = PySpace.Location.CHAINED_ASSIGNMENT_VARIABLE_SUFFIX
         ASSERT_STATEMENT_EXPRESSIONS = PySpace.Location.ASSERT_STATEMENT_EXPRESSION_SUFFIX
         COLLECTION_LITERAL_ELEMENT = PySpace.Location.COLLECTION_LITERAL_ELEMENT_SUFFIX
@@ -115,7 +118,7 @@ class Location(Enum):
         SLICE_STEP = PySpace.Location.SLICE_STEP_SUFFIX
         SLICE_STOP = PySpace.Location.SLICE_STOP_SUFFIX
         TOP_LEVEL_STATEMENT_SUFFIX = PySpace.Location.TOP_LEVEL_STATEMENT
-        UNION_TYPE_TYPES = PySpace.Location.UNION_TYPES_SUFFIX
+        UNION_TYPE_TYPES = PySpace.Location.UNION_TYPE_TYPES_SUFFIX
         VARIABLE_SCOPE_NAMES = PySpace.Location.VARIABLE_SCOPE_NAME_SUFFIX
         YIELD_EXPRESSIONS = PySpace.Location.YIELD_EXPRESSION_SUFFIX