Fix handling of "\" escape character in identifier names

.github/workflows/publish_timestamped_release.yml

@@ -4,6 +4,7 @@ on:
   push:
     branches:
       - query-grouping-aggregation
+      - identifier_unescaping
   workflow_dispatch:
 
 jobs:

.github/workflows/pull_request_ubuntu_build.yml

@@ -15,6 +15,7 @@ on:
       - native-build
       - revert-client-decl-master
       - query-grouping-aggregation
+      - identifier_unescaping
 
 jobs:
   ubuntu_build:

compiler/ballerina-lang/src/main/java/org/wso2/ballerinalang/compiler/bir/codegen/JvmTypeGen.java

@@ -603,7 +603,7 @@ private void loadTableType(MethodVisitor mv, BTableType bType) {
                 mv.visitInsn(DUP);
                 mv.visitLdcInsn((long) i);
                 mv.visitInsn(L2I);
-                mv.visitLdcInsn(Utils.unescapeJava(fieldName));
+                mv.visitLdcInsn(fieldName);
                 mv.visitInsn(AASTORE);
                 i += 1;
             }

compiler/ballerina-lang/src/main/java/org/wso2/ballerinalang/compiler/desugar/ClosureGenerator.java

@@ -17,7 +17,6 @@
  */
 package org.wso2.ballerinalang.compiler.desugar;
 
-import io.ballerina.identifier.Utils;
 import io.ballerina.tools.diagnostics.Location;
 import org.ballerinalang.model.TreeBuilder;
 import org.ballerinalang.model.elements.Flag;
@@ -625,9 +624,9 @@ private void generateClosureForDefaultValues(String closureName, String paramNam
         if (symbol.getKind() == SymbolKind.INVOKABLE_TYPE) {
             BInvokableTypeSymbol invokableTypeSymbol = (BInvokableTypeSymbol) symbol;
             updateFunctionParams(function, invokableTypeSymbol.params, paramName);
-            invokableTypeSymbol.defaultValues.put(Utils.unescapeBallerina(paramName), varSymbol);
+            invokableTypeSymbol.defaultValues.put(paramName, varSymbol);
         } else {
-            ((BRecordTypeSymbol) symbol).defaultValues.put(Utils.unescapeBallerina(paramName), varSymbol);
+            ((BRecordTypeSymbol) symbol).defaultValues.put(paramName, varSymbol);
             lambdaFunction.function.flagSet.add(Flag.RECORD);
         }
         env.enclPkg.symbol.scope.define(function.symbol.name, function.symbol);

compiler/ballerina-lang/src/main/java/org/wso2/ballerinalang/compiler/desugar/Desugar.java

@@ -17,7 +17,6 @@
  */
 package org.wso2.ballerinalang.compiler.desugar;
 
-import io.ballerina.identifier.Utils;
 import io.ballerina.runtime.api.constants.RuntimeConstants;
 import io.ballerina.tools.diagnostics.Location;
 import org.ballerinalang.compiler.CompilerPhase;
@@ -1770,8 +1769,8 @@ private void createVarDefStmts(BLangRecordVariable parentRecordVariable, BLangBl
         List<BLangRecordVariableKeyValue> variableList = parentRecordVariable.variableList;
         for (BLangRecordVariableKeyValue recordFieldKeyValue : variableList) {
             BLangVariable variable = recordFieldKeyValue.valueBindingPattern;
-            BLangLiteral indexExpr = ASTBuilderUtil.createLiteral(variable.pos, symTable.stringType,
-                                                                  Utils.unescapeJava(recordFieldKeyValue.key.value));
+            BLangLiteral indexExpr =
+                    ASTBuilderUtil.createLiteral(variable.pos, symTable.stringType, recordFieldKeyValue.key.value);
 
             if (recordFieldKeyValue.valueBindingPattern.getKind() == NodeKind.VARIABLE) {
                 createSimpleVarDefStmt((BLangSimpleVariable) recordFieldKeyValue.valueBindingPattern, parentBlockStmt,
@@ -2826,7 +2825,7 @@ private void createVarRefAssignmentStmts(BLangRecordVarRef parentRecordVarRef, B
         for (BLangRecordVarRefKeyValue varRefKeyValue : variableRefList) {
             BLangExpression expression = varRefKeyValue.variableReference;
             BLangLiteral indexExpr = ASTBuilderUtil.createLiteral(expression.pos, symTable.stringType,
-                    Utils.unescapeJava(varRefKeyValue.variableName.getValue()));
+                    varRefKeyValue.variableName.getValue());
 
             if (NodeKind.SIMPLE_VARIABLE_REF == expression.getKind() ||
                     NodeKind.FIELD_BASED_ACCESS_EXPR == expression.getKind() ||
@@ -6118,12 +6117,12 @@ private List<String> getNamesOfUserSpecifiedRecordFields(List<RecordLiteralNode.
             if (field.isKeyValueField()) {
                 BLangExpression key = ((BLangRecordLiteral.BLangRecordKeyValueField) field).key.expr;
                 if (key.getKind() == NodeKind.LITERAL) {
-                    fieldNames.add(Utils.unescapeBallerina(((BLangLiteral) key).value.toString()));
+                    fieldNames.add(((BLangLiteral) key).value.toString());
                 } else if (key.getKind() == NodeKind.SIMPLE_VARIABLE_REF) {
-                    fieldNames.add(Utils.unescapeBallerina(((BLangSimpleVarRef) key).variableName.value));
+                    fieldNames.add(((BLangSimpleVarRef) key).variableName.value);
                 }
             } else if (field.getKind() == NodeKind.SIMPLE_VARIABLE_REF) {
-                fieldNames.add(Utils.unescapeBallerina(((BLangSimpleVarRef) field).variableName.value));
+                fieldNames.add(((BLangSimpleVarRef) field).variableName.value);
             } else {
                 addRequiredFieldsFromSpreadOperator(field, fieldNames);
             }
@@ -6140,7 +6139,7 @@ private void addRequiredFieldsFromSpreadOperator(RecordLiteralNode.RecordField f
             return;
         }
         for (BField bField : ((BRecordType) type).fields.values()) {
-            fieldNames.add(Utils.unescapeBallerina(bField.name.value));
+            fieldNames.add(bField.name.value);
         }
     }
 
@@ -6179,7 +6178,7 @@ private BLangRecordLiteral.BLangRecordKeyValueField createRecordKeyValueField(Lo
                                                                                   BLangExpression expression) {
         BLangRecordLiteral.BLangRecordKeyValueField member = new BLangRecordLiteral.BLangRecordKeyValueField();
         member.key = new BLangRecordLiteral.BLangRecordKey(ASTBuilderUtil.createLiteral(pos, symTable.stringType,
-                    Utils.unescapeJava(fieldName)));
+                    fieldName));
         member.valueExpr = types.addConversionExprIfRequired(expression, expression.getBType());
         return member;
     }
@@ -6349,8 +6348,7 @@ private void rewriteFieldBasedAccess(BLangFieldBasedAccess fieldAccessExpr) {
             fieldAccessExpr.expr = types.addConversionExprIfRequired(fieldAccessExpr.expr, varRefType);
         }
 
-        BLangLiteral stringLit = createStringLiteral(fieldAccessExpr.field.pos,
-                Utils.unescapeJava(fieldAccessExpr.field.value));
+        BLangLiteral stringLit = createStringLiteral(fieldAccessExpr.field.pos, fieldAccessExpr.field.value);
         BType refType = Types.getImpliedType(varRefType);
         int varRefTypeTag = refType.tag;
         if (varRefTypeTag == TypeTags.OBJECT ||
@@ -6780,7 +6778,7 @@ private BLangStatementExpression createStmtExpr(BLangInvocation invocation) {
                 continue;
             }
 
-            BInvokableSymbol invokableSymbol = defaultValues.get(Utils.unescapeBallerina(paramName));
+            BInvokableSymbol invokableSymbol = defaultValues.get(paramName);
             BLangInvocation closureInvocation = getFunctionPointerInvocation(invokableSymbol);
             for (int m = 0; m < invokableSymbol.params.size(); m++) {
                 String langLibFuncParam = invokableSymbol.params.get(m).name.value;
@@ -6829,8 +6827,8 @@ public void visit(BLangErrorConstructorExpr errorConstructorExpr) {
         } else {
             for (BLangNamedArgsExpression namedArg : errorConstructorExpr.namedArgs) {
                 BLangRecordLiteral.BLangRecordKeyValueField member = new BLangRecordLiteral.BLangRecordKeyValueField();
-                member.key = new BLangRecordLiteral.BLangRecordKey(ASTBuilderUtil.createLiteral(namedArg.name.pos,
-                        symTable.stringType, Utils.unescapeJava(namedArg.name.value)));
+                member.key = new BLangRecordLiteral.BLangRecordKey(
+                        ASTBuilderUtil.createLiteral(namedArg.name.pos, symTable.stringType, namedArg.name.value));
 
                 if (Types.getImpliedType(recordLiteral.getBType()).tag == TypeTags.RECORD) {
                     member.valueExpr = types.addConversionExprIfRequired(namedArg.expr, symTable.anyType);
@@ -10573,8 +10571,8 @@ private BLangRecordLiteral rewriteMappingConstructor(BLangRecordLiteral mappingC
                 if (key.computedKey) {
                     keyExpr = origKey;
                 } else {
-                    keyExpr = origKey.getKind() == NodeKind.SIMPLE_VARIABLE_REF ? createStringLiteral(pos,
-                            Utils.unescapeJava(((BLangSimpleVarRef) origKey).variableName.value)) :
+                    keyExpr = origKey.getKind() == NodeKind.SIMPLE_VARIABLE_REF ?
+                            createStringLiteral(pos, ((BLangSimpleVarRef) origKey).variableName.value) :
                             ((BLangLiteral) origKey);
                 }
 
@@ -10587,7 +10585,7 @@ private BLangRecordLiteral rewriteMappingConstructor(BLangRecordLiteral mappingC
             } else if (field.getKind() == NodeKind.SIMPLE_VARIABLE_REF) {
                 BLangSimpleVarRef varRefField = (BLangSimpleVarRef) field;
                 rewrittenFields.add(ASTBuilderUtil.createBLangRecordKeyValue(
-                        rewriteExpr(createStringLiteral(pos, Utils.unescapeJava(varRefField.variableName.value))),
+                        rewriteExpr(createStringLiteral(pos, varRefField.variableName.value)),
                         rewriteExpr(varRefField)));
             } else {
                 BLangRecordLiteral.BLangRecordSpreadOperatorField spreadOpField =

compiler/ballerina-lang/src/main/java/org/wso2/ballerinalang/compiler/parser/BLangNodeBuilder.java

@@ -913,7 +913,7 @@ private void createAnonymousTypeDefForConstantDeclaration(BLangConstant constant
 
         // Create a new anonymous type definition.
         BLangTypeDefinition typeDef = (BLangTypeDefinition) TreeBuilder.createTypeDefinition();
-        this.anonTypeNameSuffixes.push(constantNode.name.value);
+        this.anonTypeNameSuffixes.push(constantNode.name.originalValue);
         String genName = anonymousModelHelper.getNextAnonymousTypeKey(packageID, anonTypeNameSuffixes);
         this.anonTypeNameSuffixes.pop();
         IdentifierNode anonTypeGenName = createIdentifier(symTable.builtinPos, genName, constantNode.name.value);
@@ -979,7 +979,7 @@ public BLangNode transform(TypeDefinitionNode typeDefNode) {
         typeDef.markdownDocumentationAttachment =
                 createMarkdownDocumentationAttachment(getDocumentationString(typeDefNode.metadata()));
 
-        this.anonTypeNameSuffixes.push(typeDef.name.value);
+        this.anonTypeNameSuffixes.push(typeDef.name.originalValue);
         typeDef.typeNode = createTypeNode(typeDefNode.typeDescriptor());
         this.anonTypeNameSuffixes.pop();
 
@@ -1577,7 +1577,7 @@ private void populateFunctionNode(BLangIdentifier name, NodeList<Token> qualifie
         //Set method qualifiers
         setFunctionQualifiers(bLFunction, qualifierList);
         // Set function signature
-        this.anonTypeNameSuffixes.push(name.value);
+        this.anonTypeNameSuffixes.push(name.originalValue);
         populateFuncSignature(bLFunction, functionSignature);
         this.anonTypeNameSuffixes.pop();
 
@@ -1782,10 +1782,10 @@ public BLangNode transform(NamedWorkerDeclarationNode namedWorkerDeclNode) {
         String workerOriginalName = workerName;
         if (workerName.startsWith(IDENTIFIER_LITERAL_PREFIX)) {
             bLFunction.defaultWorkerName.setOriginalValue(workerName);
-            workerName = Utils.unescapeUnicodeCodepoints(workerName.substring(1));
+            workerName = workerName.substring(1);
         }
 
-        bLFunction.defaultWorkerName.value = workerName;
+        bLFunction.defaultWorkerName.value = Utils.unescapeBallerina(workerName);
         bLFunction.defaultWorkerName.pos = getPosition(namedWorkerDeclNode.workerName());
 
         NodeList<AnnotationNode> annotations = namedWorkerDeclNode.annotations();
@@ -3881,7 +3881,7 @@ public BLangConstant transformEnumMember(EnumMemberNode member, Boolean publicQu
 
         BLangIdentifier memberName = (BLangIdentifier) transform(member.identifier());
         bLangConstant.setName(memberName);
-        this.anonTypeNameSuffixes.push(memberName.value);
+        this.anonTypeNameSuffixes.push(memberName.originalValue);
 
         BLangExpression deepLiteral;
         if (member.constExprNode().isPresent()) {
@@ -3905,7 +3905,7 @@ public BLangConstant transformEnumMember(EnumMemberNode member, Boolean publicQu
                 literal.originalValue = null;
                 typeNodeAssociated.addValue(deepLiteral);
                 bLangConstant.associatedTypeDefinition = createTypeDefinitionWithTypeNode(typeNodeAssociated,
-                        memberName.value);
+                        memberName.originalValue);
             } else {
                 bLangConstant.associatedTypeDefinition = null;
             }
@@ -5489,7 +5489,8 @@ private BLangNode createActionOrExpression(Node actionOrExpression) {
             BLangSimpleVarRef bLVarRef = (BLangSimpleVarRef) TreeBuilder.createSimpleVariableReferenceNode();
             bLVarRef.pos = getPosition(actionOrExpression);
             bLVarRef.pkgAlias = this.createIdentifier(nameReference[0].getPosition(), nameReference[0].getValue());
-            bLVarRef.variableName = this.createIdentifier(nameReference[1].getPosition(), nameReference[1].getValue());
+            bLVarRef.variableName =
+                    this.createIdentifier(nameReference[1].getPosition(), nameReference[1].originalValue);
             return bLVarRef;
         } else if (actionOrExpression.kind() == SyntaxKind.BRACED_EXPRESSION) {
             BLangGroupExpr group = (BLangGroupExpr) TreeBuilder.createGroupExpressionNode();
@@ -6060,10 +6061,10 @@ private BLangIdentifier createIdentifier(Location pos, String value, String orig
         }
 
         if (value.startsWith(IDENTIFIER_LITERAL_PREFIX)) {
-            bLIdentifer.setValue(Utils.unescapeUnicodeCodepoints(value.substring(1)));
+            bLIdentifer.setValue(Utils.unescapeBallerina(value.substring(1)));
             bLIdentifer.setLiteral(true);
         } else {
-            bLIdentifer.setValue(Utils.unescapeUnicodeCodepoints(value));
+            bLIdentifer.setValue(Utils.unescapeBallerina(value));
             bLIdentifer.setLiteral(false);
         }
         bLIdentifer.setOriginalValue(originalValue);

compiler/ballerina-lang/src/main/java/org/wso2/ballerinalang/compiler/semantics/analyzer/CodeAnalyzer.java

@@ -17,7 +17,6 @@
 */
 package org.wso2.ballerinalang.compiler.semantics.analyzer;
 
-import io.ballerina.identifier.Utils;
 import io.ballerina.tools.diagnostics.Location;
 import org.ballerinalang.compiler.CompilerPhase;
 import org.ballerinalang.model.elements.Flag;
@@ -2424,8 +2423,7 @@ public void visit(BLangRecordLiteral recordLiteral, AnalyzerData data) {
                     }
                 }
 
-                LinkedHashMap<String, BField> fieldsInRecordType = getUnescapedFieldList(spreadExprRecordType.fields);
-
+                LinkedHashMap<String, BField> fieldsInRecordType = spreadExprRecordType.fields;
                 for (Object fieldName : names) {
                     if (!fieldsInRecordType.containsKey(fieldName) && !isSpreadExprRecordTypeSealed) {
                         this.dlog.error(spreadOpExpr.pos,
@@ -2476,24 +2474,21 @@ public void visit(BLangRecordLiteral recordLiteral, AnalyzerData data) {
 
                 if (keyExpr.getKind() == NodeKind.SIMPLE_VARIABLE_REF) {
                     String name = ((BLangSimpleVarRef) keyExpr).variableName.value;
-                    String unescapedName = Utils.unescapeJava(name);
-                    if (names.contains(unescapedName)) {
+                    if (names.contains(name)) {
                         this.dlog.error(keyExpr.pos, DiagnosticErrorCode.DUPLICATE_KEY_IN_MAPPING_CONSTRUCTOR,
-                                        Types.getImpliedType(recordLiteral.expectedType).getKind().typeName(),
-                                        unescapedName);
-                    } else if (inclusiveTypeSpreadField != null && !neverTypedKeys.contains(unescapedName)) {
+                                Types.getImpliedType(recordLiteral.expectedType).getKind().typeName(), name);
+                    } else if (inclusiveTypeSpreadField != null && !neverTypedKeys.contains(name)) {
                         this.dlog.error(keyExpr.pos,
-                                        DiagnosticErrorCode.POSSIBLE_DUPLICATE_OF_FIELD_SPECIFIED_VIA_SPREAD_OP,
-                                unescapedName, inclusiveTypeSpreadField);
+                                DiagnosticErrorCode.POSSIBLE_DUPLICATE_OF_FIELD_SPECIFIED_VIA_SPREAD_OP, name,
+                                inclusiveTypeSpreadField);
                     }
 
                     if (!isInferredRecordForMapCET && isOpenRecord
                             && !((BRecordType) referredType).fields.containsKey(name)) {
-                        dlog.error(keyExpr.pos, DiagnosticErrorCode.INVALID_RECORD_LITERAL_IDENTIFIER_KEY,
-                                unescapedName);
+                        dlog.error(keyExpr.pos, DiagnosticErrorCode.INVALID_RECORD_LITERAL_IDENTIFIER_KEY, name);
                     }
 
-                    names.add(unescapedName);
+                    names.add(name);
                 } else if (keyExpr.getKind() == NodeKind.LITERAL || keyExpr.getKind() == NodeKind.NUMERIC_LITERAL) {
                     Object name = ((BLangLiteral) keyExpr).value;
                     if (names.contains(name)) {
@@ -2519,16 +2514,7 @@ public void visit(BLangRecordLiteral recordLiteral, AnalyzerData data) {
     public void visit(BLangRecordLiteral.BLangRecordVarNameField node, AnalyzerData data) {
         visit((BLangSimpleVarRef) node, data);
     }
-
-    private LinkedHashMap<String, BField> getUnescapedFieldList(LinkedHashMap<String, BField> fieldMap) {
-        LinkedHashMap<String, BField> newMap = new LinkedHashMap<>();
-        for (String key : fieldMap.keySet()) {
-            newMap.put(Utils.unescapeJava(key), fieldMap.get(key));
-        }
-
-        return newMap;
-    }
-
+
     @Override
     public void visit(BLangSimpleVarRef varRefExpr, AnalyzerData data) {
         switch (varRefExpr.parent.getKind()) {

compiler/ballerina-lang/src/main/java/org/wso2/ballerinalang/compiler/semantics/analyzer/SymbolResolver.java

@@ -1725,7 +1725,7 @@ public BType transform(BLangUserDefinedType userDefinedTypeNode, AnalyzerData da
             if (!missingNodesHelper.isMissingNode(pkgAlias) && !missingNodesHelper.isMissingNode(typeName) &&
                     !symbolEnter.isUnknownTypeRef(userDefinedTypeNode)
                     && typeResolver.isNotUnknownTypeRef(userDefinedTypeNode)) {
-                dlog.error(userDefinedTypeNode.pos, data.diagCode, typeName);
+                dlog.error(userDefinedTypeNode.pos, data.diagCode, names.originalNameFromIdNode(typeNameIdentifier));
             }
             return symTable.semanticError;
         }

compiler/ballerina-lang/src/main/java/org/wso2/ballerinalang/compiler/semantics/analyzer/TypeChecker.java

@@ -17,7 +17,6 @@
  */
 package org.wso2.ballerinalang.compiler.semantics.analyzer;
 
-import io.ballerina.identifier.Utils;
 import io.ballerina.tools.diagnostics.DiagnosticCode;
 import io.ballerina.tools.diagnostics.Location;
 import org.ballerinalang.model.TreeBuilder;
@@ -3226,7 +3225,7 @@ public void visit(BLangSimpleVarRef varRefExpr, AnalyzerData data) {
                 }
             } else {
                 varRefExpr.symbol = symbol; // Set notFoundSymbol
-                logUndefinedSymbolError(varRefExpr.pos, varName.value);
+                logUndefinedSymbolError(varRefExpr.pos, names.originalNameFromIdNode(identifier).value);
             }
         }
 
@@ -9183,8 +9182,8 @@ private BType checkRecordIndexBasedAccess(BLangIndexBasedAccess accessExpr, BRec
             case TypeTags.STRING:
             case TypeTags.CHAR_STRING:
                 if (isConstExpr(indexExpr)) {
-                    String fieldName = Utils.escapeSpecialCharacters(getConstFieldName(indexExpr));
-                    actualType = checkRecordRequiredFieldAccess(accessExpr, Names.fromString(fieldName), record, data);
+                    String fieldName = getConstFieldName(indexExpr);
+                    actualType = checkRecordRequiredFieldAccess(accessExpr, names.fromString(fieldName), record, data);
                     if (actualType != symTable.semanticError) {
                         return actualType;
                     }