DynamoDB: Validate duplicate path in projection expression (#8298)

moto/dynamodb/exceptions.py

@@ -305,6 +305,17 @@ def __init__(self, names: List[str]):
         )
 
 
+class InvalidProjectionExpression(MockValidationException):
+    msg = (
+        "Invalid ProjectionExpression: "
+        "Two document paths overlap with each other; must remove or rewrite one of these paths; "
+        "path one: [{paths[0]}], path two: [{paths[1]}]"
+    )
+
+    def __init__(self, paths: List[str]):
+        super().__init__(self.msg.format(paths=paths))
+
+
 class TooManyClauses(InvalidUpdateExpression):
     def __init__(self, _type: str) -> None:
         super().__init__(

moto/dynamodb/parsing/ast_nodes.py

@@ -10,6 +10,7 @@
     MockValidationException,
     TooManyClauses,
 )
+from ..utils import extract_duplicates
 
 
 class Node(metaclass=abc.ABCMeta):
@@ -38,8 +39,11 @@ def validate(self, limit_set_actions: bool = False) -> None:
             set_attributes = [s.children[0].to_str() for s in set_actions]
             # We currently only check for duplicates
             # We should also check for partial duplicates, i.e. [attr, attr.sub] is also invalid
-            if len(set_attributes) != len(set(set_attributes)):
-                raise DuplicateUpdateExpression(set_attributes)
+            duplicates = extract_duplicates(set_attributes)
+            if duplicates:
+                # There might be more than one attribute duplicated:
+                # they may get mixed up in the Error Message which is inline with actual boto3 Error Messages
+                raise DuplicateUpdateExpression(duplicates)
 
             set_clauses = self.find_clauses([UpdateExpressionSetClause])
             if limit_set_actions and len(set_clauses) > 1:

moto/dynamodb/responses.py

@@ -14,12 +14,14 @@
 from moto.utilities.aws_headers import amz_crc32
 
 from .exceptions import (
+    InvalidProjectionExpression,
     KeyIsEmptyStringException,
     MockValidationException,
     ProvidedKeyDoesNotExist,
     ResourceNotFoundException,
     UnknownKeyType,
 )
+from .utils import extract_duplicates
 
 TRANSACTION_MAX_ITEMS = 25
 
@@ -794,6 +796,9 @@ def _adjust(expression: str) -> str:
 
         if projection_expression:
             expressions = [x.strip() for x in projection_expression.split(",")]
+            duplicates = extract_duplicates(expressions)
+            if duplicates:
+                raise InvalidProjectionExpression(duplicates)
             for expression in expressions:
                 check_projection_expression(expression)
             return [

moto/dynamodb/utils.py

@@ -0,0 +1,5 @@
+from typing import List
+
+
+def extract_duplicates(input_list: List[str]) -> List[str]:
+    return [item for item in input_list if input_list.count(item) > 1]

tests/test_dynamodb/exceptions/test_dynamodb_exceptions.py

@@ -664,6 +664,148 @@ def test_update_item_with_duplicate_expressions(expression):
     assert item == {"pk": "example_id", "example_column": "example"}
 
 
+@mock_aws
+def test_query_item_with_duplicate_path_in_projection_expressions():
+    dynamodb = boto3.resource("dynamodb", region_name="us-east-1")
+
+    # Create the DynamoDB table.
+    dynamodb.create_table(
+        TableName="users",
+        KeySchema=[
+            {"AttributeName": "forum_name", "KeyType": "HASH"},
+            {"AttributeName": "subject", "KeyType": "RANGE"},
+        ],
+        AttributeDefinitions=[
+            {"AttributeName": "forum_name", "AttributeType": "S"},
+            {"AttributeName": "subject", "AttributeType": "S"},
+        ],
+        ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5},
+    )
+    table = dynamodb.Table("users")
+    table.put_item(
+        Item={"forum_name": "the-key", "subject": "123", "body": "some test message"}
+    )
+
+    with pytest.raises(ClientError) as exc:
+        _result = table.query(
+            KeyConditionExpression=Key("forum_name").eq("the-key"),
+            ProjectionExpression="body, subject, body",  # duplicate body
+        )["Items"][0]
+
+    err = exc.value.response["Error"]
+    assert err["Code"] == "ValidationException"
+    assert err["Message"] == (
+        "Invalid ProjectionExpression: "
+        "Two document paths overlap with each other; must remove or rewrite one of these paths; "
+        "path one: [body], path two: [body]"
+    )
+
+
+@mock_aws
+def test_get_item_with_duplicate_path_in_projection_expressions():
+    dynamodb = boto3.resource("dynamodb", region_name="us-east-1")
+
+    # Create the DynamoDB table.
+    table = dynamodb.create_table(
+        TableName="users",
+        KeySchema=[
+            {"AttributeName": "forum_name", "KeyType": "HASH"},
+            {"AttributeName": "subject", "KeyType": "RANGE"},
+        ],
+        AttributeDefinitions=[
+            {"AttributeName": "forum_name", "AttributeType": "S"},
+            {"AttributeName": "subject", "AttributeType": "S"},
+        ],
+        ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5},
+    )
+    table = dynamodb.Table("users")
+
+    table.put_item(
+        Item={
+            "forum_name": "the-key",
+            "subject": "123",
+            "body": "some test message",
+            "attachment": "something",
+        }
+    )
+
+    table.put_item(
+        Item={
+            "forum_name": "not-the-key",
+            "subject": "123",
+            "body": "some other test message",
+            "attachment": "something",
+        }
+    )
+    with pytest.raises(ClientError) as exc:
+        _result = table.get_item(
+            Key={"forum_name": "the-key", "subject": "123"},
+            ProjectionExpression="#rl, #rt, subject, subject",  # duplicate subject
+            ExpressionAttributeNames={"#rl": "body", "#rt": "attachment"},
+        )
+
+    err = exc.value.response["Error"]
+    assert err["Code"] == "ValidationException"
+    assert err["Message"] == (
+        "Invalid ProjectionExpression: "
+        "Two document paths overlap with each other; must remove or rewrite one of these paths; "
+        "path one: [subject], path two: [subject]"
+    )
+
+
+@mock_aws
+def test_scan_with_duplicate_path_in_projection_expressions():
+    dynamodb = boto3.resource("dynamodb", region_name="us-east-1")
+
+    # Create the DynamoDB table.
+    table = dynamodb.create_table(
+        TableName="users",
+        KeySchema=[
+            {"AttributeName": "forum_name", "KeyType": "HASH"},
+            {"AttributeName": "subject", "KeyType": "RANGE"},
+        ],
+        AttributeDefinitions=[
+            {"AttributeName": "forum_name", "AttributeType": "S"},
+            {"AttributeName": "subject", "AttributeType": "S"},
+        ],
+        ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5},
+    )
+    table = dynamodb.Table("users")
+
+    table.put_item(
+        Item={
+            "forum_name": "the-key",
+            "subject": "123",
+            "body": "some test message",
+            "attachment": "something",
+        }
+    )
+
+    table.put_item(
+        Item={
+            "forum_name": "not-the-key",
+            "subject": "123",
+            "body": "some other test message",
+            "attachment": "something",
+        }
+    )
+
+    with pytest.raises(ClientError) as exc:
+        _results = table.scan(
+            FilterExpression=Key("forum_name").eq("the-key"),
+            ProjectionExpression="#rl, #rt, subject, subject",  # duplicate subject
+            ExpressionAttributeNames={"#rl": "body", "#rt": "attachment"},
+        )
+
+    err = exc.value.response["Error"]
+    assert err["Code"] == "ValidationException"
+    assert err["Message"] == (
+        "Invalid ProjectionExpression: "
+        "Two document paths overlap with each other; must remove or rewrite one of these paths; "
+        "path one: [subject], path two: [subject]"
+    )
+
+
 @mock_aws
 def test_put_item_wrong_datatype():
     if settings.TEST_SERVER_MODE: