Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

chore(tests): Build in source sync temp folder #6166

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
170 changes: 148 additions & 22 deletions tests/integration/sync/test_sync_build_in_source.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
import pytest
import logging
import json
from unittest import skip
from parameterized import parameterized, parameterized_class

from tests.integration.sync.sync_integ_base import SyncIntegBase
Expand All @@ -15,7 +14,6 @@
LOG = logging.getLogger(__name__)


@skip("Building in source option is not exposed yet. Stop skipping once it is.")
class TestSyncInfra_BuildInSource_Makefile(SyncIntegBase):
dependency_layer = False

Expand All @@ -30,6 +28,7 @@ def setUp(self):

def tearDown(self):
super().tearDown()

for path in self.new_files_in_source:
if os.path.isfile(path):
os.remove(path)
Expand Down Expand Up @@ -57,7 +56,7 @@ def test_sync_builds_and_deploys_successfully(self, build_in_source, new_file_sh
tags="integ=true clarity=yes foo_bar=baz",
build_in_source=build_in_source,
)
sync_process_execute = run_command_with_input(sync_command_list, "y\n".encode())
sync_process_execute = run_command_with_input(sync_command_list, "y\n".encode(), cwd=self.test_data_path)
self.assertEqual(sync_process_execute.process.returncode, 0)
self.assertIn("Sync infra completed.", str(sync_process_execute.stderr))

Expand All @@ -74,7 +73,6 @@ def test_sync_builds_and_deploys_successfully(self, build_in_source, new_file_sh
)


@skip("Building in source option is not exposed yet. Stop skipping once it is.")
class TestSyncCode_BuildInSource_Makefile(TestSyncCodeBase):
dependency_layer = False
folder = "code"
Expand All @@ -87,10 +85,11 @@ def setUp(self):
Path("makefile_layer_create_new_file", "file-created-from-makefile-layer.txt"),
]
# When running tests, TestSyncCodeBase copies the source onto a temp directory
self.new_files_in_source = [TestSyncCodeBase.temp_dir.joinpath(path) for path in paths]
self.new_files_in_source = [Path(self.test_data_path, self.folder, "before", path) for path in paths]

def tearDown(self):
super().tearDown()

for path in self.new_files_in_source:
if os.path.isfile(path):
os.remove(path)
Expand All @@ -105,11 +104,10 @@ def tearDown(self):
def test_sync_code_builds_and_deploys_successfully(self, build_in_source, new_file_should_be_in_source):
# update layer to trigger rebuild
layer_path = "makefile_layer_create_new_file"
shutil.rmtree(TestSyncCodeBase.temp_dir.joinpath(layer_path), ignore_errors=True)
shutil.copytree(
self.test_data_path.joinpath(self.folder).joinpath("after").joinpath(layer_path),
TestSyncCodeBase.temp_dir.joinpath(layer_path),
)
code_uri = Path(self.test_data_path, self.folder)

shutil.rmtree(Path(code_uri, "before", layer_path), ignore_errors=True)
shutil.copytree(Path(code_uri, "after", layer_path), Path(code_uri, "before", layer_path))

# Run code sync
sync_command_list = self.get_sync_command_list(
Expand All @@ -123,7 +121,7 @@ def test_sync_code_builds_and_deploys_successfully(self, build_in_source, new_fi
tags="integ=true clarity=yes foo_bar=baz",
build_in_source=build_in_source,
)
sync_process_execute = run_command_with_input(sync_command_list, "y\n".encode())
sync_process_execute = run_command_with_input(sync_command_list, "y\n".encode(), cwd=self.test_data_path)
self.assertEqual(sync_process_execute.process.returncode, 0)

# check whether the new files were created in the source directory
Expand All @@ -139,16 +137,16 @@ def test_sync_code_builds_and_deploys_successfully(self, build_in_source, new_fi
)


@skip("Building in source option is not exposed yet. Stop skipping once it is.")
@parameterized_class([{"dependency_layer": True}, {"dependency_layer": False}])
class TestSyncInfra_BuildInSource_Esbuild(SyncIntegBase):
dependency_layer = False

def setUp(self):
super().setUp()
self.source_dependencies_paths = []

def tearDown(self):
super().tearDown()
# clean up dependencies installed in source directories

for path in self.source_dependencies_paths:
shutil.rmtree(path, ignore_errors=True)

Expand Down Expand Up @@ -182,7 +180,7 @@ def test_sync_builds_successfully_without_local_dependencies(
tags="integ=true clarity=yes foo_bar=baz",
build_in_source=build_in_source,
)
sync_process_execute = run_command_with_input(sync_command_list, "y\n".encode())
sync_process_execute = run_command_with_input(sync_command_list, "y\n".encode(), cwd=self.test_data_path)
self.assertEqual(sync_process_execute.process.returncode, 0)
self.assertIn("Sync infra completed.", str(sync_process_execute.stderr))

Expand Down Expand Up @@ -232,20 +230,19 @@ def test_sync_builds_successfully_with_local_dependency(self):
self.assertEqual(lambda_response.get("message"), "hello world")


@skip("Building in source option is not exposed yet. Stop skipping once it is.")
@parameterized_class([{"dependency_layer": True}, {"dependency_layer": False}])
class TestSyncCode_BuildInSource_Esbuild(TestSyncCodeBase):
dependency_layer = False
folder = "code"
template = "template-esbuild.yaml"

def setUp(self):
super().setUp()

self.source_dependencies_paths = []

def tearDown(self):
super().tearDown()
# clean up dependencies installed in source directories

for path in self.source_dependencies_paths:
shutil.rmtree(path, ignore_errors=True)

Expand All @@ -259,7 +256,19 @@ def tearDown(self):
def test_sync_code_builds_successfully_without_local_dependencies(
self, build_in_source, dependencies_expected_in_source
):
self.source_dependencies_paths = [TestSyncCodeBase.temp_dir.joinpath("esbuild_function", "node_modules")]
code_folder = "esbuild_function"

# make a code change (update message and add extra message in output)
shutil.rmtree(self.test_data_path.joinpath(self.folder, "before", code_folder))
shutil.copytree(
self.test_data_path.joinpath(self.folder, "after", code_folder),
self.test_data_path.joinpath(self.folder, "before", code_folder),
dirs_exist_ok=True,
)

self.source_dependencies_paths = [
Path.joinpath(self.test_data_path, self.folder, "before", code_folder, "node_modules")
]

# Run code sync
sync_command_list = self.get_sync_command_list(
Expand All @@ -273,7 +282,7 @@ def test_sync_code_builds_successfully_without_local_dependencies(
tags="integ=true clarity=yes foo_bar=baz",
build_in_source=build_in_source,
)
sync_process_execute = run_command_with_input(sync_command_list, "y\n".encode())
sync_process_execute = run_command_with_input(sync_command_list, "y\n".encode(), cwd=self.test_data_path)
self.assertEqual(sync_process_execute.process.returncode, 0)

# check whether dependencies were installed in the source directory
Expand All @@ -284,11 +293,12 @@ def test_sync_code_builds_successfully_without_local_dependencies(
lambda_functions = stack_resources.get(AWS_LAMBDA_FUNCTION)
for lambda_function in lambda_functions:
lambda_response = json.loads(self._get_lambda_response(lambda_function))
self.assertEqual(lambda_response.get("message"), "hello world")
self.assertEqual(lambda_response.get("message"), "Hello world!")
self.assertEqual(lambda_response.get("extra_message"), "banana")

def test_sync_code_builds_successfully_with_local_dependencies(self):
codeuri = "esbuild_function_with_local_dependency"
self.source_dependencies_paths = [TestSyncCodeBase.temp_dir.joinpath(codeuri, "node_modules")]
self.source_dependencies_paths = [Path(self.test_data_path, self.folder, "before", codeuri, "node_modules")]

# Run code sync
sync_command_list = self.get_sync_command_list(
Expand All @@ -315,3 +325,119 @@ def test_sync_code_builds_successfully_with_local_dependencies(self):
for lambda_function in lambda_functions:
lambda_response = json.loads(self._get_lambda_response(lambda_function))
self.assertEqual(lambda_response.get("message"), "hello world")


class TestSyncCode_BuildInSource_Nodejs_Without_Local_Dep(TestSyncCodeBase):
dependency_layer = False
folder = "code"
template = "template-nodejs.yaml"

def tearDown(self):
super().tearDown()

for path in self.source_dependencies_paths:
shutil.rmtree(path, ignore_errors=True)

@parameterized.expand(
[
(True, True), # build in source
(False, False), # don't build in source
(None, False), # use default for workflow (don't build in source)
]
)
def test_sync_code_builds_successfully_without_local_dependencies(
self, build_in_source, dependencies_expected_in_source
):
code_folder = "nodejs_function"

# make a code change (calling faker api method)
shutil.rmtree(self.test_data_path.joinpath(self.folder, "before", code_folder), ignore_errors=True)
shutil.copytree(
self.test_data_path.joinpath(self.folder, "after", code_folder),
self.test_data_path.joinpath(self.folder, "before", code_folder),
dirs_exist_ok=True,
)

self.source_dependencies_paths = [
Path.joinpath(self.test_data_path, self.folder, "before", code_folder, "node_modules")
]

# Run code sync
sync_command_list = self.get_sync_command_list(
template_file=TestSyncCodeBase.template_path,
stack_name=TestSyncCodeBase.stack_name,
code=True,
dependency_layer=self.dependency_layer,
image_repository=self.ecr_repo_name,
s3_prefix=self.s3_prefix,
kms_key_id=self.kms_key,
tags="integ=true clarity=yes foo_bar=baz",
build_in_source=build_in_source,
)
sync_process_execute = run_command_with_input(sync_command_list, "y\n".encode(), cwd=self.test_data_path)
self.assertEqual(sync_process_execute.process.returncode, 0)

# check whether dependencies were installed in the source directory
for path in self.source_dependencies_paths:
self.assertEqual(os.path.isdir(path), dependencies_expected_in_source)

stack_resources = self._get_stacks(TestSyncCodeBase.stack_name)
lambda_functions = stack_resources.get(AWS_LAMBDA_FUNCTION)
for lambda_function in lambda_functions:
lambda_response = json.loads(self._get_lambda_response(lambda_function))

self.assertEqual(lambda_response.get("message"), "Hello world!")
# extra message is new field containing random name
self.assertIn("extra_message", lambda_response)


class TestSyncCode_BuildInSource_Nodejs_Using_Local_Dep(TestSyncCodeBase):
dependency_layer = False
folder = "code"
template = "template_nodejs_local_dep.yaml"

def tearDown(self):
super().tearDown()

for path in self.source_dependencies_paths:
shutil.rmtree(path, ignore_errors=True)

def test_sync_code_builds_successfully_with_local_dependencies(self):
code_folder = "nodejs_local_dep"

# make a code change (installing local dep)
shutil.rmtree(self.test_data_path.joinpath(self.folder, "before", code_folder), ignore_errors=True)
shutil.copytree(
self.test_data_path.joinpath(self.folder, "after", code_folder),
self.test_data_path.joinpath(self.folder, "before", code_folder),
dirs_exist_ok=True,
)

self.source_dependencies_paths = [
Path(self.test_data_path, self.folder, "before", code_folder, "src", "node_modules")
]

# Run code sync
sync_command_list = self.get_sync_command_list(
template_file=TestSyncCodeBase.template_path,
stack_name=TestSyncCodeBase.stack_name,
code=True,
dependency_layer=self.dependency_layer,
image_repository=self.ecr_repo_name,
s3_prefix=self.s3_prefix,
kms_key_id=self.kms_key,
tags="integ=true clarity=yes foo_bar=baz",
build_in_source=True,
)
sync_process_execute = run_command_with_input(sync_command_list, "y\n".encode())
self.assertEqual(sync_process_execute.process.returncode, 0)

# check whether dependencies were installed in the source directory
for path in self.source_dependencies_paths:
self.assertEqual(os.path.isdir(path), True)

stack_resources = self._get_stacks(TestSyncCodeBase.stack_name)
lambda_functions = stack_resources.get(AWS_LAMBDA_FUNCTION)
for lambda_function in lambda_functions:
lambda_response = json.loads(self._get_lambda_response(lambda_function))
self.assertEqual(lambda_response.get("message"), 123)
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import * as faker from '@faker-js/faker';
const faker = require("@faker-js/faker");

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Curious why we needed to change the import syntax here?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The build in source test that reuses this project is complaining about using the module import keywords. Changed it back here and ran the entire suite of integration tests on my local machine to make sure that nothing broke. We can test again using the dedicated canaries too.

const name = faker.faker.name.firstName();
let response;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
const foo = 123;

exports.exported = () => {
return foo;
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
{
"name": "local-dep",
"version": "1.0.0",
"description": "",
"main": "index.js",
"author": ""
}

Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
const localDep = require("local-dep");

exports.handler = async (event, context) => {
return {
'statusCode': 200,
'body': JSON.stringify({
message: localDep.exported(),
})
};
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
{
"name": "src",
"version": "1.0.0",
"description": "",
"main": "index.js",
"author": "",
"dependencies": {
"local-dep": "file:../local-dep"
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
exports.handler = async (event, context) => {
return {
'statusCode': 200,
'body': JSON.stringify({
message: 'no local dep',
})
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
{
"name": "src",
"version": "1.0.0",
"description": "",
"main": "index.js",
"author": "",
"dependencies": {
"axios": "^0.27.2"
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ Resources:
Properties:
CodeUri: nodejs_function/
Handler: app.lambdaHandler
Runtime: nodejs16.x
Runtime: nodejs18.x
Tracing: Active

HelloWorldApi:
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
AWSTemplateFormatVersion : '2010-09-09'
Transform: AWS::Serverless-2016-10-31

Resources:
PrintLocalDep:
Type: AWS::Serverless::Function
Properties:
Handler: index.handler
Runtime: nodejs18.x
CodeUri: nodejs_local_dep/src
Loading