From 9a6ddde0049186a2583fc1f42c1fac40a31507aa Mon Sep 17 00:00:00 2001 From: poorna2152 Date: Thu, 19 Oct 2023 10:07:02 +0530 Subject: [PATCH 1/8] Escape in BLangNodeBuilder --- .../compiler/parser/BLangNodeBuilder.java | 4 ++-- .../java/io/ballerina/identifier/Utils.java | 3 ++- .../test/identifiers/IdentifierTest.java | 9 ++++++++ .../identifier_with_escape_char.bal | 22 +++++++++++++++++++ 4 files changed, 35 insertions(+), 3 deletions(-) create mode 100644 tests/jballerina-unit-test/src/test/resources/test-src/identifiers/identifier_with_escape_char.bal diff --git a/compiler/ballerina-lang/src/main/java/org/wso2/ballerinalang/compiler/parser/BLangNodeBuilder.java b/compiler/ballerina-lang/src/main/java/org/wso2/ballerinalang/compiler/parser/BLangNodeBuilder.java index 600a0e42ac1a..51f50fbf1554 100644 --- a/compiler/ballerina-lang/src/main/java/org/wso2/ballerinalang/compiler/parser/BLangNodeBuilder.java +++ b/compiler/ballerina-lang/src/main/java/org/wso2/ballerinalang/compiler/parser/BLangNodeBuilder.java @@ -6060,10 +6060,10 @@ private BLangIdentifier createIdentifier(Location pos, String value, String orig } if (value.startsWith(IDENTIFIER_LITERAL_PREFIX)) { - bLIdentifer.setValue(Utils.unescapeUnicodeCodepoints(value.substring(1))); + bLIdentifer.setValue(Utils.unescapeBallerina(value.substring(1))); bLIdentifer.setLiteral(true); } else { - bLIdentifer.setValue(Utils.unescapeUnicodeCodepoints(value)); + bLIdentifer.setValue(Utils.unescapeBallerina(value)); bLIdentifer.setLiteral(false); } bLIdentifer.setOriginalValue(originalValue); diff --git a/misc/identifier-util/src/main/java/io/ballerina/identifier/Utils.java b/misc/identifier-util/src/main/java/io/ballerina/identifier/Utils.java index 0e87690aa193..29dd45f714fd 100644 --- a/misc/identifier-util/src/main/java/io/ballerina/identifier/Utils.java +++ b/misc/identifier-util/src/main/java/io/ballerina/identifier/Utils.java @@ -88,7 +88,8 @@ private static String encodeIdentifier(String identifier) { * @return a new unescaped {@code String}, {@code null} if null string input */ public static String unescapeJava(String str) { - return StringEscapeUtils.unescapeJava(str); + String result = str.replaceAll("\\\\(\\d)", "$1"); + return StringEscapeUtils.unescapeJava(result); } private static Identifier encodeGeneratedName(String identifier) { diff --git a/tests/jballerina-unit-test/src/test/java/org/ballerinalang/test/identifiers/IdentifierTest.java b/tests/jballerina-unit-test/src/test/java/org/ballerinalang/test/identifiers/IdentifierTest.java index ba844d9a84d1..20ad1c92e0a8 100644 --- a/tests/jballerina-unit-test/src/test/java/org/ballerinalang/test/identifiers/IdentifierTest.java +++ b/tests/jballerina-unit-test/src/test/java/org/ballerinalang/test/identifiers/IdentifierTest.java @@ -74,6 +74,15 @@ public void testInvalidImportOnMultipleFiles() { assertEquals(result.getErrorCount(), index); } + @Test(description = "Test identifiers containing escape character") + public void testEscapedIdentifier() { + CompileResult result = BCompileUtil.compile("test-src/identifiers/identifier_with_escape_char.bal"); + int index = 0; + validateError(result, index++, "redeclared symbol 'a3'", 19, 9); + validateError(result, index++, "redeclared symbol 'student-performance'", 21, 9); + assertEquals(result.getErrorCount(), index); + } + @Test(dataProvider = "functionsWithSelfAsIdentifier") public void testSelfAsIdentifier(String function) { CompileResult result = BCompileUtil.compile("test-src/identifiers/self_as_identifier.bal"); diff --git a/tests/jballerina-unit-test/src/test/resources/test-src/identifiers/identifier_with_escape_char.bal b/tests/jballerina-unit-test/src/test/resources/test-src/identifiers/identifier_with_escape_char.bal new file mode 100644 index 000000000000..1fd038033cad --- /dev/null +++ b/tests/jballerina-unit-test/src/test/resources/test-src/identifiers/identifier_with_escape_char.bal @@ -0,0 +1,22 @@ +// Copyright (c) 2023, WSO2 LLC. (https://www.wso2.com) All Rights Reserved. +// +// WSO2 LLC. licenses this file to you under the Apache License, +// Version 2.0 (the "License"); you may not use this file except +// in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +public function foo() { + int a\3 = 0; + int a3 = 2; + int student\-performance = 3; + int student\u{002D}performance = 2; +} From 08104025f060efb73afd9a2cd73ef488aded50e8 Mon Sep 17 00:00:00 2001 From: poorna2152 Date: Tue, 24 Oct 2023 14:52:36 +0530 Subject: [PATCH 2/8] Disable clashing identifier test case --- .../ballerinalang/test/identifier/IdentifierLiteralTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/jballerina-integration-test/src/test/java/org/ballerinalang/test/identifier/IdentifierLiteralTest.java b/tests/jballerina-integration-test/src/test/java/org/ballerinalang/test/identifier/IdentifierLiteralTest.java index 9a3e5202f165..22eaf52c1608 100644 --- a/tests/jballerina-integration-test/src/test/java/org/ballerinalang/test/identifier/IdentifierLiteralTest.java +++ b/tests/jballerina-integration-test/src/test/java/org/ballerinalang/test/identifier/IdentifierLiteralTest.java @@ -66,7 +66,7 @@ public void testPackageIDClash() throws BallerinaTestException { runLeecher.waitForText(5000); } - @Test(description = "Test identifier clashes between generated and user defined identifiers") + @Test(description = "Test identifier clashes between generated and user defined identifiers", enabled = false) public void testUserDefinedIdentifierClash() throws BallerinaTestException { String testBalFile = "identifier_clash.bal"; From 47af6ba7fde269e88ba594e59c9b7769cb51e9c3 Mon Sep 17 00:00:00 2001 From: poorna2152 Date: Wed, 25 Oct 2023 11:05:05 +0530 Subject: [PATCH 3/8] Update the regex in escaping --- .../wso2/ballerinalang/compiler/parser/BLangNodeBuilder.java | 3 ++- .../src/main/java/io/ballerina/identifier/Utils.java | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/compiler/ballerina-lang/src/main/java/org/wso2/ballerinalang/compiler/parser/BLangNodeBuilder.java b/compiler/ballerina-lang/src/main/java/org/wso2/ballerinalang/compiler/parser/BLangNodeBuilder.java index 51f50fbf1554..1c20b4439f15 100644 --- a/compiler/ballerina-lang/src/main/java/org/wso2/ballerinalang/compiler/parser/BLangNodeBuilder.java +++ b/compiler/ballerina-lang/src/main/java/org/wso2/ballerinalang/compiler/parser/BLangNodeBuilder.java @@ -5489,7 +5489,8 @@ private BLangNode createActionOrExpression(Node actionOrExpression) { BLangSimpleVarRef bLVarRef = (BLangSimpleVarRef) TreeBuilder.createSimpleVariableReferenceNode(); bLVarRef.pos = getPosition(actionOrExpression); bLVarRef.pkgAlias = this.createIdentifier(nameReference[0].getPosition(), nameReference[0].getValue()); - bLVarRef.variableName = this.createIdentifier(nameReference[1].getPosition(), nameReference[1].getValue()); + bLVarRef.variableName = + this.createIdentifier(nameReference[1].getPosition(), nameReference[1].originalValue); return bLVarRef; } else if (actionOrExpression.kind() == SyntaxKind.BRACED_EXPRESSION) { BLangGroupExpr group = (BLangGroupExpr) TreeBuilder.createGroupExpressionNode(); diff --git a/misc/identifier-util/src/main/java/io/ballerina/identifier/Utils.java b/misc/identifier-util/src/main/java/io/ballerina/identifier/Utils.java index 29dd45f714fd..99905dceff03 100644 --- a/misc/identifier-util/src/main/java/io/ballerina/identifier/Utils.java +++ b/misc/identifier-util/src/main/java/io/ballerina/identifier/Utils.java @@ -88,7 +88,7 @@ private static String encodeIdentifier(String identifier) { * @return a new unescaped {@code String}, {@code null} if null string input */ public static String unescapeJava(String str) { - String result = str.replaceAll("\\\\(\\d)", "$1"); + String result = str.replaceAll("(? Date: Wed, 25 Oct 2023 11:20:22 +0530 Subject: [PATCH 4/8] Update the test with backslash variables --- .../org/ballerinalang/test/identifiers/IdentifierTest.java | 5 +++-- .../test-src/identifiers/identifier_with_escape_char.bal | 4 ++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/jballerina-unit-test/src/test/java/org/ballerinalang/test/identifiers/IdentifierTest.java b/tests/jballerina-unit-test/src/test/java/org/ballerinalang/test/identifiers/IdentifierTest.java index 20ad1c92e0a8..9d1083d2b133 100644 --- a/tests/jballerina-unit-test/src/test/java/org/ballerinalang/test/identifiers/IdentifierTest.java +++ b/tests/jballerina-unit-test/src/test/java/org/ballerinalang/test/identifiers/IdentifierTest.java @@ -78,8 +78,9 @@ public void testInvalidImportOnMultipleFiles() { public void testEscapedIdentifier() { CompileResult result = BCompileUtil.compile("test-src/identifiers/identifier_with_escape_char.bal"); int index = 0; - validateError(result, index++, "redeclared symbol 'a3'", 19, 9); - validateError(result, index++, "redeclared symbol 'student-performance'", 21, 9); + validateError(result, index++, "redeclared symbol 'a3'", 20, 9); + validateError(result, index++, "redeclared symbol 'student-performance'", 22, 9); + validateError(result, index++, "redeclared symbol 'resource\\1path'", 24, 12); assertEquals(result.getErrorCount(), index); } diff --git a/tests/jballerina-unit-test/src/test/resources/test-src/identifiers/identifier_with_escape_char.bal b/tests/jballerina-unit-test/src/test/resources/test-src/identifiers/identifier_with_escape_char.bal index 1fd038033cad..17322b11e3b8 100644 --- a/tests/jballerina-unit-test/src/test/resources/test-src/identifiers/identifier_with_escape_char.bal +++ b/tests/jballerina-unit-test/src/test/resources/test-src/identifiers/identifier_with_escape_char.bal @@ -13,10 +13,14 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. +import ballerina/io; public function foo() { int a\3 = 0; int a3 = 2; int student\-performance = 3; int student\u{002D}performance = 2; + string resource\\1path = "https:\\"; + string resource\u{005c}1path = "http"; + io:println(resource\\1path); } From 2197c010f0a179c0b91a22ecd0e07687cda188a2 Mon Sep 17 00:00:00 2001 From: poorna2152 Date: Thu, 26 Oct 2023 13:41:59 +0530 Subject: [PATCH 5/8] Comment . function usage in profiler test --- .../src/test/resources/profiler/profiler_single_file.bal | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/jballerina-integration-test/src/test/resources/profiler/profiler_single_file.bal b/tests/jballerina-integration-test/src/test/resources/profiler/profiler_single_file.bal index 9eb748a634aa..7bdb44acd04b 100644 --- a/tests/jballerina-integration-test/src/test/resources/profiler/profiler_single_file.bal +++ b/tests/jballerina-integration-test/src/test/resources/profiler/profiler_single_file.bal @@ -17,9 +17,9 @@ import ballerina/test; import ballerina/jballerina.java; -public function '\.\() returns string { - return "this is a user defined function"; -} +// public function '\.\() returns string { +// return "this is a user defined function"; +// } class Person { public string firstName; @@ -60,7 +60,7 @@ public function main() { Person person = new("John", "Doe"); test:assertEquals("John", person.firstName); test:assertEquals("John Doe", person.'\$init\$()); - test:assertEquals("this is a user defined function", '\.\()); + // test:assertEquals("this is a user defined function", '\.\()); record {| string name; From 4c5e0032dfc7ca72e3ff9d553d127c762afccce0 Mon Sep 17 00:00:00 2001 From: poorna2152 Date: Thu, 2 Nov 2023 22:04:08 +0530 Subject: [PATCH 6/8] Fix failing test cases --- .../compiler/bir/codegen/JvmTypeGen.java | 2 +- .../compiler/desugar/ClosureGenerator.java | 5 ++- .../compiler/desugar/Desugar.java | 32 +++++++++---------- .../compiler/parser/BLangNodeBuilder.java | 14 ++++---- .../semantics/analyzer/CodeAnalyzer.java | 32 ++++++------------- .../semantics/analyzer/SymbolResolver.java | 2 +- .../semantics/analyzer/TypeChecker.java | 7 ++-- .../semantics/analyzer/TypeResolver.java | 9 ++++-- .../evaluation/IdentifierModifier.java | 6 ++-- .../variable/NamedCompoundVariable.java | 6 ++++ .../java/io/ballerina/identifier/Utils.java | 14 +++----- .../literals/IdentifierLiteralTest.java | 4 +-- .../test/identifiers/IdentifierTest.java | 6 ++-- .../identifier_with_escape_char.bal | 7 ++-- 14 files changed, 68 insertions(+), 78 deletions(-) diff --git a/compiler/ballerina-lang/src/main/java/org/wso2/ballerinalang/compiler/bir/codegen/JvmTypeGen.java b/compiler/ballerina-lang/src/main/java/org/wso2/ballerinalang/compiler/bir/codegen/JvmTypeGen.java index e6d31db01347..fb59ee7b545b 100644 --- a/compiler/ballerina-lang/src/main/java/org/wso2/ballerinalang/compiler/bir/codegen/JvmTypeGen.java +++ b/compiler/ballerina-lang/src/main/java/org/wso2/ballerinalang/compiler/bir/codegen/JvmTypeGen.java @@ -603,7 +603,7 @@ private void loadTableType(MethodVisitor mv, BTableType bType) { mv.visitInsn(DUP); mv.visitLdcInsn((long) i); mv.visitInsn(L2I); - mv.visitLdcInsn(Utils.unescapeJava(fieldName)); + mv.visitLdcInsn(fieldName); mv.visitInsn(AASTORE); i += 1; } diff --git a/compiler/ballerina-lang/src/main/java/org/wso2/ballerinalang/compiler/desugar/ClosureGenerator.java b/compiler/ballerina-lang/src/main/java/org/wso2/ballerinalang/compiler/desugar/ClosureGenerator.java index ea16981eec54..c3aa77c025ba 100644 --- a/compiler/ballerina-lang/src/main/java/org/wso2/ballerinalang/compiler/desugar/ClosureGenerator.java +++ b/compiler/ballerina-lang/src/main/java/org/wso2/ballerinalang/compiler/desugar/ClosureGenerator.java @@ -17,7 +17,6 @@ */ package org.wso2.ballerinalang.compiler.desugar; -import io.ballerina.identifier.Utils; import io.ballerina.tools.diagnostics.Location; import org.ballerinalang.model.TreeBuilder; import org.ballerinalang.model.elements.Flag; @@ -625,9 +624,9 @@ private void generateClosureForDefaultValues(String closureName, String paramNam if (symbol.getKind() == SymbolKind.INVOKABLE_TYPE) { BInvokableTypeSymbol invokableTypeSymbol = (BInvokableTypeSymbol) symbol; updateFunctionParams(function, invokableTypeSymbol.params, paramName); - invokableTypeSymbol.defaultValues.put(Utils.unescapeBallerina(paramName), varSymbol); + invokableTypeSymbol.defaultValues.put(paramName, varSymbol); } else { - ((BRecordTypeSymbol) symbol).defaultValues.put(Utils.unescapeBallerina(paramName), varSymbol); + ((BRecordTypeSymbol) symbol).defaultValues.put(paramName, varSymbol); lambdaFunction.function.flagSet.add(Flag.RECORD); } env.enclPkg.symbol.scope.define(function.symbol.name, function.symbol); diff --git a/compiler/ballerina-lang/src/main/java/org/wso2/ballerinalang/compiler/desugar/Desugar.java b/compiler/ballerina-lang/src/main/java/org/wso2/ballerinalang/compiler/desugar/Desugar.java index 0c6022e7ca6f..09618a0dc203 100644 --- a/compiler/ballerina-lang/src/main/java/org/wso2/ballerinalang/compiler/desugar/Desugar.java +++ b/compiler/ballerina-lang/src/main/java/org/wso2/ballerinalang/compiler/desugar/Desugar.java @@ -17,7 +17,6 @@ */ package org.wso2.ballerinalang.compiler.desugar; -import io.ballerina.identifier.Utils; import io.ballerina.runtime.api.constants.RuntimeConstants; import io.ballerina.tools.diagnostics.Location; import org.ballerinalang.compiler.CompilerPhase; @@ -1770,8 +1769,8 @@ private void createVarDefStmts(BLangRecordVariable parentRecordVariable, BLangBl List variableList = parentRecordVariable.variableList; for (BLangRecordVariableKeyValue recordFieldKeyValue : variableList) { BLangVariable variable = recordFieldKeyValue.valueBindingPattern; - BLangLiteral indexExpr = ASTBuilderUtil.createLiteral(variable.pos, symTable.stringType, - Utils.unescapeJava(recordFieldKeyValue.key.value)); + BLangLiteral indexExpr = + ASTBuilderUtil.createLiteral(variable.pos, symTable.stringType, recordFieldKeyValue.key.value); if (recordFieldKeyValue.valueBindingPattern.getKind() == NodeKind.VARIABLE) { createSimpleVarDefStmt((BLangSimpleVariable) recordFieldKeyValue.valueBindingPattern, parentBlockStmt, @@ -2826,7 +2825,7 @@ private void createVarRefAssignmentStmts(BLangRecordVarRef parentRecordVarRef, B for (BLangRecordVarRefKeyValue varRefKeyValue : variableRefList) { BLangExpression expression = varRefKeyValue.variableReference; BLangLiteral indexExpr = ASTBuilderUtil.createLiteral(expression.pos, symTable.stringType, - Utils.unescapeJava(varRefKeyValue.variableName.getValue())); + varRefKeyValue.variableName.getValue()); if (NodeKind.SIMPLE_VARIABLE_REF == expression.getKind() || NodeKind.FIELD_BASED_ACCESS_EXPR == expression.getKind() || @@ -6118,12 +6117,12 @@ private List getNamesOfUserSpecifiedRecordFields(List qualifie //Set method qualifiers setFunctionQualifiers(bLFunction, qualifierList); // Set function signature - this.anonTypeNameSuffixes.push(name.value); + this.anonTypeNameSuffixes.push(name.originalValue); populateFuncSignature(bLFunction, functionSignature); this.anonTypeNameSuffixes.pop(); @@ -1782,10 +1782,10 @@ public BLangNode transform(NamedWorkerDeclarationNode namedWorkerDeclNode) { String workerOriginalName = workerName; if (workerName.startsWith(IDENTIFIER_LITERAL_PREFIX)) { bLFunction.defaultWorkerName.setOriginalValue(workerName); - workerName = Utils.unescapeUnicodeCodepoints(workerName.substring(1)); + workerName = workerName.substring(1); } - bLFunction.defaultWorkerName.value = workerName; + bLFunction.defaultWorkerName.value = Utils.unescapeBallerina(workerName); bLFunction.defaultWorkerName.pos = getPosition(namedWorkerDeclNode.workerName()); NodeList annotations = namedWorkerDeclNode.annotations(); @@ -3881,7 +3881,7 @@ public BLangConstant transformEnumMember(EnumMemberNode member, Boolean publicQu BLangIdentifier memberName = (BLangIdentifier) transform(member.identifier()); bLangConstant.setName(memberName); - this.anonTypeNameSuffixes.push(memberName.value); + this.anonTypeNameSuffixes.push(memberName.originalValue); BLangExpression deepLiteral; if (member.constExprNode().isPresent()) { @@ -3905,7 +3905,7 @@ public BLangConstant transformEnumMember(EnumMemberNode member, Boolean publicQu literal.originalValue = null; typeNodeAssociated.addValue(deepLiteral); bLangConstant.associatedTypeDefinition = createTypeDefinitionWithTypeNode(typeNodeAssociated, - memberName.value); + memberName.originalValue); } else { bLangConstant.associatedTypeDefinition = null; } diff --git a/compiler/ballerina-lang/src/main/java/org/wso2/ballerinalang/compiler/semantics/analyzer/CodeAnalyzer.java b/compiler/ballerina-lang/src/main/java/org/wso2/ballerinalang/compiler/semantics/analyzer/CodeAnalyzer.java index 7ec4a0e54bd7..cafa3003a4c7 100644 --- a/compiler/ballerina-lang/src/main/java/org/wso2/ballerinalang/compiler/semantics/analyzer/CodeAnalyzer.java +++ b/compiler/ballerina-lang/src/main/java/org/wso2/ballerinalang/compiler/semantics/analyzer/CodeAnalyzer.java @@ -17,7 +17,6 @@ */ package org.wso2.ballerinalang.compiler.semantics.analyzer; -import io.ballerina.identifier.Utils; import io.ballerina.tools.diagnostics.Location; import org.ballerinalang.compiler.CompilerPhase; import org.ballerinalang.model.elements.Flag; @@ -2424,8 +2423,7 @@ public void visit(BLangRecordLiteral recordLiteral, AnalyzerData data) { } } - LinkedHashMap fieldsInRecordType = getUnescapedFieldList(spreadExprRecordType.fields); - + LinkedHashMap fieldsInRecordType = spreadExprRecordType.fields; for (Object fieldName : names) { if (!fieldsInRecordType.containsKey(fieldName) && !isSpreadExprRecordTypeSealed) { this.dlog.error(spreadOpExpr.pos, @@ -2476,24 +2474,21 @@ public void visit(BLangRecordLiteral recordLiteral, AnalyzerData data) { if (keyExpr.getKind() == NodeKind.SIMPLE_VARIABLE_REF) { String name = ((BLangSimpleVarRef) keyExpr).variableName.value; - String unescapedName = Utils.unescapeJava(name); - if (names.contains(unescapedName)) { + if (names.contains(name)) { this.dlog.error(keyExpr.pos, DiagnosticErrorCode.DUPLICATE_KEY_IN_MAPPING_CONSTRUCTOR, - Types.getImpliedType(recordLiteral.expectedType).getKind().typeName(), - unescapedName); - } else if (inclusiveTypeSpreadField != null && !neverTypedKeys.contains(unescapedName)) { + Types.getImpliedType(recordLiteral.expectedType).getKind().typeName(), name); + } else if (inclusiveTypeSpreadField != null && !neverTypedKeys.contains(name)) { this.dlog.error(keyExpr.pos, - DiagnosticErrorCode.POSSIBLE_DUPLICATE_OF_FIELD_SPECIFIED_VIA_SPREAD_OP, - unescapedName, inclusiveTypeSpreadField); + DiagnosticErrorCode.POSSIBLE_DUPLICATE_OF_FIELD_SPECIFIED_VIA_SPREAD_OP, name, + inclusiveTypeSpreadField); } if (!isInferredRecordForMapCET && isOpenRecord && !((BRecordType) referredType).fields.containsKey(name)) { - dlog.error(keyExpr.pos, DiagnosticErrorCode.INVALID_RECORD_LITERAL_IDENTIFIER_KEY, - unescapedName); + dlog.error(keyExpr.pos, DiagnosticErrorCode.INVALID_RECORD_LITERAL_IDENTIFIER_KEY, name); } - names.add(unescapedName); + names.add(name); } else if (keyExpr.getKind() == NodeKind.LITERAL || keyExpr.getKind() == NodeKind.NUMERIC_LITERAL) { Object name = ((BLangLiteral) keyExpr).value; if (names.contains(name)) { @@ -2519,16 +2514,7 @@ public void visit(BLangRecordLiteral recordLiteral, AnalyzerData data) { public void visit(BLangRecordLiteral.BLangRecordVarNameField node, AnalyzerData data) { visit((BLangSimpleVarRef) node, data); } - - private LinkedHashMap getUnescapedFieldList(LinkedHashMap fieldMap) { - LinkedHashMap newMap = new LinkedHashMap<>(); - for (String key : fieldMap.keySet()) { - newMap.put(Utils.unescapeJava(key), fieldMap.get(key)); - } - - return newMap; - } - + @Override public void visit(BLangSimpleVarRef varRefExpr, AnalyzerData data) { switch (varRefExpr.parent.getKind()) { diff --git a/compiler/ballerina-lang/src/main/java/org/wso2/ballerinalang/compiler/semantics/analyzer/SymbolResolver.java b/compiler/ballerina-lang/src/main/java/org/wso2/ballerinalang/compiler/semantics/analyzer/SymbolResolver.java index e0d23ea3de8c..aed1e844c221 100644 --- a/compiler/ballerina-lang/src/main/java/org/wso2/ballerinalang/compiler/semantics/analyzer/SymbolResolver.java +++ b/compiler/ballerina-lang/src/main/java/org/wso2/ballerinalang/compiler/semantics/analyzer/SymbolResolver.java @@ -1725,7 +1725,7 @@ public BType transform(BLangUserDefinedType userDefinedTypeNode, AnalyzerData da if (!missingNodesHelper.isMissingNode(pkgAlias) && !missingNodesHelper.isMissingNode(typeName) && !symbolEnter.isUnknownTypeRef(userDefinedTypeNode) && typeResolver.isNotUnknownTypeRef(userDefinedTypeNode)) { - dlog.error(userDefinedTypeNode.pos, data.diagCode, typeName); + dlog.error(userDefinedTypeNode.pos, data.diagCode, names.originalNameFromIdNode(typeNameIdentifier)); } return symTable.semanticError; } diff --git a/compiler/ballerina-lang/src/main/java/org/wso2/ballerinalang/compiler/semantics/analyzer/TypeChecker.java b/compiler/ballerina-lang/src/main/java/org/wso2/ballerinalang/compiler/semantics/analyzer/TypeChecker.java index 49befba7473c..5e1ca4264f46 100644 --- a/compiler/ballerina-lang/src/main/java/org/wso2/ballerinalang/compiler/semantics/analyzer/TypeChecker.java +++ b/compiler/ballerina-lang/src/main/java/org/wso2/ballerinalang/compiler/semantics/analyzer/TypeChecker.java @@ -17,7 +17,6 @@ */ package org.wso2.ballerinalang.compiler.semantics.analyzer; -import io.ballerina.identifier.Utils; import io.ballerina.tools.diagnostics.DiagnosticCode; import io.ballerina.tools.diagnostics.Location; import org.ballerinalang.model.TreeBuilder; @@ -3226,7 +3225,7 @@ public void visit(BLangSimpleVarRef varRefExpr, AnalyzerData data) { } } else { varRefExpr.symbol = symbol; // Set notFoundSymbol - logUndefinedSymbolError(varRefExpr.pos, varName.value); + logUndefinedSymbolError(varRefExpr.pos, names.originalNameFromIdNode(identifier).value); } } @@ -9183,8 +9182,8 @@ private BType checkRecordIndexBasedAccess(BLangIndexBasedAccess accessExpr, BRec case TypeTags.STRING: case TypeTags.CHAR_STRING: if (isConstExpr(indexExpr)) { - String fieldName = Utils.escapeSpecialCharacters(getConstFieldName(indexExpr)); - actualType = checkRecordRequiredFieldAccess(accessExpr, Names.fromString(fieldName), record, data); + String fieldName = getConstFieldName(indexExpr); + actualType = checkRecordRequiredFieldAccess(accessExpr, names.fromString(fieldName), record, data); if (actualType != symTable.semanticError) { return actualType; } diff --git a/compiler/ballerina-lang/src/main/java/org/wso2/ballerinalang/compiler/semantics/analyzer/TypeResolver.java b/compiler/ballerina-lang/src/main/java/org/wso2/ballerinalang/compiler/semantics/analyzer/TypeResolver.java index c245f3d698c1..1247c45c0326 100644 --- a/compiler/ballerina-lang/src/main/java/org/wso2/ballerinalang/compiler/semantics/analyzer/TypeResolver.java +++ b/compiler/ballerina-lang/src/main/java/org/wso2/ballerinalang/compiler/semantics/analyzer/TypeResolver.java @@ -1817,10 +1817,15 @@ public BType defineTypeDefinition(BLangTypeDefinition typeDefinition, BType reso BSymbol typeDefSymbol = Symbols.createTypeDefinitionSymbol(Flags.asMask(typeDefinition.flagSet), names.fromIdNode(typeDefinition.name), env.enclPkg.packageID, resolvedType, env.scope.owner, typeDefinition.name.pos, symEnter.getOrigin(typeDefinition.name.value)); + if (!PackageID.isLangLibPackageID(env.enclPkg.packageID)) { + typeDefSymbol.originalName = names.originalNameFromIdNode(typeDefinition.name); + } typeDefSymbol.markdownDocumentation = symEnter.getMarkdownDocAttachment(typeDefinition.markdownDocumentationAttachment); - BTypeSymbol typeSymbol = new BTypeSymbol(SymTag.TYPE_REF, typeDefSymbol.flags, typeDefSymbol.name, - typeDefSymbol.pkgID, typeDefSymbol.type, typeDefSymbol.owner, typeDefSymbol.pos, typeDefSymbol.origin); + BTypeSymbol typeSymbol = + new BTypeSymbol(SymTag.TYPE_REF, typeDefSymbol.flags, typeDefSymbol.name, typeDefSymbol.originalName, + typeDefSymbol.pkgID, typeDefSymbol.type, typeDefSymbol.owner, typeDefSymbol.pos, + typeDefSymbol.origin); typeSymbol.markdownDocumentation = typeDefSymbol.markdownDocumentation; ((BTypeDefinitionSymbol) typeDefSymbol).referenceType = new BTypeReferenceType(resolvedType, typeSymbol, typeDefSymbol.type.flags); diff --git a/misc/debug-adapter/modules/debug-adapter-core/src/main/java/org/ballerinalang/debugadapter/evaluation/IdentifierModifier.java b/misc/debug-adapter/modules/debug-adapter-core/src/main/java/org/ballerinalang/debugadapter/evaluation/IdentifierModifier.java index e8bc44238d1e..3f0dc331e6a5 100644 --- a/misc/debug-adapter/modules/debug-adapter-core/src/main/java/org/ballerinalang/debugadapter/evaluation/IdentifierModifier.java +++ b/misc/debug-adapter/modules/debug-adapter-core/src/main/java/org/ballerinalang/debugadapter/evaluation/IdentifierModifier.java @@ -22,8 +22,6 @@ import io.ballerina.compiler.syntax.tree.TreeModifier; import io.ballerina.identifier.Utils; -import static io.ballerina.identifier.Utils.unescapeUnicodeCodepoints; - /** * Identifier specific expression modifier implementation. */ @@ -39,7 +37,7 @@ public IdentifierToken transform(IdentifierToken identifier) { identifierText = identifierText.substring(1); } // Processes escaped unicode codepoints. - String unescapedIdentifier = unescapeUnicodeCodepoints(identifierText); + String unescapedIdentifier = Utils.unescapeBallerina(identifierText); // Encodes the user provided identifier in order to be aligned with JVM runtime identifiers. NonTerminalNode parent = identifier.parent(); @@ -58,7 +56,7 @@ public static String encodeIdentifier(String identifier, IdentifierType type) { if (identifier.startsWith(QUOTED_IDENTIFIER_PREFIX)) { identifier = identifier.substring(1); } - identifier = Utils.unescapeUnicodeCodepoints(identifier); + identifier = Utils.unescapeBallerina(identifier); return type == IdentifierType.METHOD_NAME ? Utils.encodeFunctionIdentifier(identifier) : Utils.encodeNonFunctionIdentifier(identifier); } diff --git a/misc/debug-adapter/modules/debug-adapter-core/src/main/java/org/ballerinalang/debugadapter/variable/NamedCompoundVariable.java b/misc/debug-adapter/modules/debug-adapter-core/src/main/java/org/ballerinalang/debugadapter/variable/NamedCompoundVariable.java index eb8d74ca1a8b..d34572fcecfb 100644 --- a/misc/debug-adapter/modules/debug-adapter-core/src/main/java/org/ballerinalang/debugadapter/variable/NamedCompoundVariable.java +++ b/misc/debug-adapter/modules/debug-adapter-core/src/main/java/org/ballerinalang/debugadapter/variable/NamedCompoundVariable.java @@ -65,6 +65,12 @@ public Value getChildByName(String name) throws DebugVariableException { } if (!namedChildVariables.containsKey(name)) { + for (Map.Entry childVariable : namedChildVariables.entrySet()) { + String escaped = childVariable.getKey().replaceAll("\\$0092(\\$0092)?", "$1"); + if (escaped.equals(name)) { + return childVariable.getValue(); + } + } throw new DebugVariableException("No child variables found with name: '" + name + "'"); } return namedChildVariables.get(name); diff --git a/misc/identifier-util/src/main/java/io/ballerina/identifier/Utils.java b/misc/identifier-util/src/main/java/io/ballerina/identifier/Utils.java index 99905dceff03..dfee21d27778 100644 --- a/misc/identifier-util/src/main/java/io/ballerina/identifier/Utils.java +++ b/misc/identifier-util/src/main/java/io/ballerina/identifier/Utils.java @@ -44,17 +44,14 @@ private Utils() { private static String encodeSpecialCharacters(String identifier) { StringBuilder sb = new StringBuilder(); - int index = 0; - while (index < identifier.length()) { + char[] characters = identifier.toCharArray(); + for (char character: characters) { String formattedString; - if (identifier.charAt(index) == '\\' && (index + 1 < identifier.length()) && - (formattedString = getFormattedStringForQuotedIdentifiers(identifier.charAt(index + 1))) != null) { + if ((formattedString = getFormattedStringForQuotedIdentifiers(character)) != null) { String unicodePoint = CHAR_PREFIX + formattedString; sb.append(unicodePoint); - index += 2; } else { - sb.append(identifier.charAt(index)); - index++; + sb.append(character); } } return sb.toString(); @@ -72,8 +69,7 @@ public static String escapeSpecialCharacters(String identifier) { private static String encodeIdentifier(String identifier) { if (identifier.contains(ESCAPE_PREFIX)) { - identifier = encodeSpecialCharacters(identifier); - return unescapeJava(identifier); + return encodeSpecialCharacters(identifier); } return identifier; } diff --git a/tests/jballerina-unit-test/src/test/java/org/ballerinalang/test/expressions/literals/IdentifierLiteralTest.java b/tests/jballerina-unit-test/src/test/java/org/ballerinalang/test/expressions/literals/IdentifierLiteralTest.java index b1058647495b..2b900fc99817 100644 --- a/tests/jballerina-unit-test/src/test/java/org/ballerinalang/test/expressions/literals/IdentifierLiteralTest.java +++ b/tests/jballerina-unit-test/src/test/java/org/ballerinalang/test/expressions/literals/IdentifierLiteralTest.java @@ -98,7 +98,7 @@ public void testTryToAccessUndefinedGlobalVarWithIdentifierLiteral() { CompileResult resultNeg = BCompileUtil.compile("test-src/expressions/literals/identifierliteral" + "/identifier-literal-undefined-variable-negative.bal"); Assert.assertEquals(resultNeg.getErrorCount(), 1); - BAssertUtil.validateError(resultNeg, 0, "undefined symbol 'global\\ v\\ \\\"\\ ar'", 5, 12); + BAssertUtil.validateError(resultNeg, 0, "undefined symbol '\'global\\ v\\ \\\"\\ ar'", 5, 12); } @Test(description = "Test wrong character in identifier literal") @@ -116,7 +116,7 @@ public void testIdentifierLiteralWithWrongCharacter() { BAssertUtil.validateError(resultNeg, i++, "invalid escape sequence '\\'", 4, 9); BAssertUtil.validateError(resultNeg, i++, "invalid escape sequence '\\'", 6, 12); BAssertUtil.validateError(resultNeg, i++, "missing semicolon token", 7, 1); - BAssertUtil.validateError(resultNeg, i++, "undefined symbol 'global\\ v\\ ar'", 7, 12); + BAssertUtil.validateError(resultNeg, i++, "undefined symbol '\'global\\ v\\ ar'", 7, 12); Assert.assertEquals(resultNeg.getErrorCount(), i); } diff --git a/tests/jballerina-unit-test/src/test/java/org/ballerinalang/test/identifiers/IdentifierTest.java b/tests/jballerina-unit-test/src/test/java/org/ballerinalang/test/identifiers/IdentifierTest.java index 9d1083d2b133..ba4a0e9d02cd 100644 --- a/tests/jballerina-unit-test/src/test/java/org/ballerinalang/test/identifiers/IdentifierTest.java +++ b/tests/jballerina-unit-test/src/test/java/org/ballerinalang/test/identifiers/IdentifierTest.java @@ -78,9 +78,9 @@ public void testInvalidImportOnMultipleFiles() { public void testEscapedIdentifier() { CompileResult result = BCompileUtil.compile("test-src/identifiers/identifier_with_escape_char.bal"); int index = 0; - validateError(result, index++, "redeclared symbol 'a3'", 20, 9); - validateError(result, index++, "redeclared symbol 'student-performance'", 22, 9); - validateError(result, index++, "redeclared symbol 'resource\\1path'", 24, 12); + validateError(result, index++, "redeclared symbol 'a3'", 23, 9); + validateError(result, index++, "redeclared symbol 'student-performance'", 25, 9); + validateError(result, index++, "redeclared symbol 'resource\\1path'", 27, 12); assertEquals(result.getErrorCount(), index); } diff --git a/tests/jballerina-unit-test/src/test/resources/test-src/identifiers/identifier_with_escape_char.bal b/tests/jballerina-unit-test/src/test/resources/test-src/identifiers/identifier_with_escape_char.bal index 17322b11e3b8..57cec50658c8 100644 --- a/tests/jballerina-unit-test/src/test/resources/test-src/identifiers/identifier_with_escape_char.bal +++ b/tests/jballerina-unit-test/src/test/resources/test-src/identifiers/identifier_with_escape_char.bal @@ -13,7 +13,10 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. -import ballerina/io; + +public function bar(string x) returns string { + return x; +} public function foo() { int a\3 = 0; @@ -22,5 +25,5 @@ public function foo() { int student\u{002D}performance = 2; string resource\\1path = "https:\\"; string resource\u{005c}1path = "http"; - io:println(resource\\1path); + _ = bar(resource\\1path); } From 3c249a3777a4738c2431d852e6e0089026309c39 Mon Sep 17 00:00:00 2001 From: poorna2152 Date: Wed, 8 Nov 2023 09:37:01 +0530 Subject: [PATCH 7/8] Temp update workflow --- .github/workflows/publish_timestamped_release.yml | 1 + .github/workflows/pull_request_ubuntu_build.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/publish_timestamped_release.yml b/.github/workflows/publish_timestamped_release.yml index 489a1168d272..c4dbec2599ab 100644 --- a/.github/workflows/publish_timestamped_release.yml +++ b/.github/workflows/publish_timestamped_release.yml @@ -4,6 +4,7 @@ on: push: branches: - query-grouping-aggregation + - identifier_unescaping workflow_dispatch: jobs: diff --git a/.github/workflows/pull_request_ubuntu_build.yml b/.github/workflows/pull_request_ubuntu_build.yml index d6e9920c30ee..55101bbfd410 100644 --- a/.github/workflows/pull_request_ubuntu_build.yml +++ b/.github/workflows/pull_request_ubuntu_build.yml @@ -15,6 +15,7 @@ on: - native-build - revert-client-decl-master - query-grouping-aggregation + - identifier_unescaping jobs: ubuntu_build: From 7a9fbb06121dc243caf404985e1afbb4edd02e70 Mon Sep 17 00:00:00 2001 From: poorna2152 Date: Thu, 19 Sep 2024 11:03:50 +0530 Subject: [PATCH 8/8] Update the regex to use const --- .../debugadapter/variable/NamedCompoundVariable.java | 4 +++- .../src/test/resources/profiler/profiler_single_file.bal | 8 ++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/misc/debug-adapter/modules/debug-adapter-core/src/main/java/org/ballerinalang/debugadapter/variable/NamedCompoundVariable.java b/misc/debug-adapter/modules/debug-adapter-core/src/main/java/org/ballerinalang/debugadapter/variable/NamedCompoundVariable.java index d34572fcecfb..8519bb76c1c5 100644 --- a/misc/debug-adapter/modules/debug-adapter-core/src/main/java/org/ballerinalang/debugadapter/variable/NamedCompoundVariable.java +++ b/misc/debug-adapter/modules/debug-adapter-core/src/main/java/org/ballerinalang/debugadapter/variable/NamedCompoundVariable.java @@ -66,7 +66,9 @@ public Value getChildByName(String name) throws DebugVariableException { if (!namedChildVariables.containsKey(name)) { for (Map.Entry childVariable : namedChildVariables.entrySet()) { - String escaped = childVariable.getKey().replaceAll("\\$0092(\\$0092)?", "$1"); + String unicodeOfSlash = "&0092"; + String escaped = childVariable.getKey() + .replaceAll(String.format("\\%s(\\%s)?", unicodeOfSlash, unicodeOfSlash), "$1"); if (escaped.equals(name)) { return childVariable.getValue(); } diff --git a/tests/jballerina-integration-test/src/test/resources/profiler/profiler_single_file.bal b/tests/jballerina-integration-test/src/test/resources/profiler/profiler_single_file.bal index 7bdb44acd04b..792ed05c1d91 100644 --- a/tests/jballerina-integration-test/src/test/resources/profiler/profiler_single_file.bal +++ b/tests/jballerina-integration-test/src/test/resources/profiler/profiler_single_file.bal @@ -30,9 +30,9 @@ class Person { self.lastName = lastName; } - function '\$init\$() returns string { - return self.firstName + " " + self.lastName; - } + // function '\$init\$() returns string { + // return self.firstName + " " + self.lastName; + // } } type '\$anonType\$_0 record { @@ -59,7 +59,7 @@ function print(string str) { public function main() { Person person = new("John", "Doe"); test:assertEquals("John", person.firstName); - test:assertEquals("John Doe", person.'\$init\$()); + // test:assertEquals("John Doe", person.'\$init\$()); // test:assertEquals("this is a user defined function", '\.\()); record {|