diff --git a/.github/sync-repo-settings.yaml b/.github/sync-repo-settings.yaml index acdbfcc..cf9ef2b 100644 --- a/.github/sync-repo-settings.yaml +++ b/.github/sync-repo-settings.yaml @@ -27,6 +27,11 @@ branchProtectionRules: requiredStatusCheckContexts: - "cla/google" - "lint" + - "integration-test-pr-py38 (prow-build-graybox)" + - "integration-test-pr-py39 (prow-build-graybox)" + - "integration-test-pr-py310 (prow-build-graybox)" + - "integration-test-pr-py311 (prow-build-graybox)" + - "integration-test-pr-py312 (prow-build-graybox)" - "conventionalcommits.org" - "header-check" # - Add required status checks like presubmit tests diff --git a/DEVELOPER.md b/DEVELOPER.md new file mode 100644 index 0000000..f4b2698 --- /dev/null +++ b/DEVELOPER.md @@ -0,0 +1,82 @@ +# DEVELOPER.md + +## Versioning + +This library follows [Semantic Versioning](http://semver.org/). + +## Processes + +### Conventional Commit messages + +This repository uses tool [Release Please](https://github.com/googleapis/release-please) to create GitHub and PyPi releases. It does so by parsing your +git history, looking for [Conventional Commit messages](https://www.conventionalcommits.org/), +and creating release PRs. + +Learn more by reading [How should I write my commits?](https://github.com/googleapis/release-please?tab=readme-ov-file#how-should-i-write-my-commits) + +## Testing + +### Run tests locally + +1. Set env vars: `DB_HOST`, `DB_PORT`, `DB_NAME`, `DB_USER`, `DB_PASSWORD` + +1. Run pytest to automatically run all tests: + + ```bash + pytest + ``` + +### CI Platform Setup + +Cloud Build is used to run tests against Google Cloud resources in test project: prow-build-graybox. +Each test has a corresponding Cloud Build trigger, see [all triggers][triggers]. +These tests are registered as required tests in `.github/sync-repo-settings.yaml`. + +#### Trigger Setup + +Cloud Build triggers (for Python versions 3.8 to 3.11) were created with the following specs: + +```YAML +name: integration-test-pr-py38 +description: Run integration tests on PR for Python 3.8 +filename: integration.cloudbuild.yaml +github: + name: langchain-google-firestore-python + owner: googleapis + pullRequest: + branch: .* + commentControl: COMMENTS_ENABLED_FOR_EXTERNAL_CONTRIBUTORS_ONLY +ignoredFiles: + - docs/** + - .kokoro/** + - .github/** + - "*.md" +substitutions: + _VERSION: "3.8" + _DB_HOST: <> + _DB_PORT: <> + _DB_NAME: <> + _DB_USER: <> +``` + +Use `gcloud builds triggers import --source=trigger.yaml` create triggers via the command line + +#### Project Setup + +1. Create an GKE Cluster with El Carro database +1. Setup Cloud Build triggers (above) + +#### Run tests with Cloud Build + +* Run integration test: + + ```bash + gcloud builds submit --config integration.cloudbuild.yaml --substitutions=,_DB_HOST=$DB_HOST,_DB_PORT=$DB_PORT,_DB_NAME=$DB_NAME,_DB_USER=$DB_USER,_DB_PASSWORD=$DB_PASSWORD + ``` + +#### Trigger + +To run Cloud Build tests on GitHub from external contributors, ie RenovateBot, comment: `/gcbrun`. + + +[triggers]: https://console.cloud.google.com/cloud-build/triggers?e=13802955&project=prow-build-graybox diff --git a/integration.cloudbuild.yaml b/integration.cloudbuild.yaml index aca5bf0..caff360 100644 --- a/integration.cloudbuild.yaml +++ b/integration.cloudbuild.yaml @@ -14,17 +14,17 @@ steps: - id: Install dependencies - name: python:3.11 + name: python:${_VERSION} entrypoint: pip args: ["install", "--user", "-r", "requirements.txt"] - id: Install module (and test requirements) - name: python:3.11 + name: python:${_VERSION} entrypoint: pip args: ["install", ".[test]", "--user"] - id: Run integration tests - name: python:3.11 + name: python:${_VERSION} entrypoint: python args: ["-m", "pytest"] env: @@ -33,6 +33,11 @@ steps: - "DB_NAME=$_DB_NAME" - "DB_USER=$_DB_USER" - "DB_PASSWORD=$_DB_PASSWORD" + +substitutions: + _VERSION: "3.8" + options: pool: name: "projects/prow-build-graybox/locations/us-central1/workerPools/langchain-el-carro-wp" + dynamicSubstitutions: true diff --git a/tests/test_chat_message_history.py b/tests/test_chat_message_history.py index cbe9613..5c39ac6 100644 --- a/tests/test_chat_message_history.py +++ b/tests/test_chat_message_history.py @@ -53,12 +53,18 @@ def temporary_table(memory_engine: ElCarroEngine) -> Generator: """ random_suffix = "".join(random.choices(string.ascii_lowercase, k=6)) table_name = f"message_store_{random_suffix}" - memory_engine.drop_chat_history_table(table_name) + try: + memory_engine.drop_chat_history_table(table_name) + except sqlalchemy.exc.NoSuchTableError as e: + pass yield table_name # Cleanup - memory_engine.drop_chat_history_table(table_name) + try: + memory_engine.drop_chat_history_table(table_name) + except sqlalchemy.exc.NoSuchTableError as e: + print(e) def test_chat_message_history( diff --git a/tests/test_loader.py b/tests/test_loader.py index 1be44d5..70e474b 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -49,7 +49,10 @@ def elcarro_engine() -> Generator: def sample_table_1(elcarro_engine: ElCarroEngine) -> Generator: random_suffix = "".join(random.choices(string.ascii_lowercase, k=6)) table_name = f"doc_{random_suffix}" - elcarro_engine.drop_document_table(table_name) + try: + elcarro_engine.drop_document_table(table_name) + except sqlalchemy.exc.NoSuchTableError as e: + pass create_table_query = f"""CREATE TABLE {table_name} ( fruit_id NUMBER GENERATED BY DEFAULT AS IDENTITY (START WITH 1), @@ -65,7 +68,10 @@ def sample_table_1(elcarro_engine: ElCarroEngine) -> Generator: yield table_name # Teardown - elcarro_engine.drop_document_table(table_name) + try: + elcarro_engine.drop_document_table(table_name) + except sqlalchemy.exc.NoSuchTableError as e: + print(e) # Simulate existing customer data table @@ -76,7 +82,10 @@ def sample_table_1(elcarro_engine: ElCarroEngine) -> Generator: def sample_table_2_clob(elcarro_engine: ElCarroEngine) -> Generator: random_suffix = "".join(random.choices(string.ascii_lowercase, k=6)) table_name = f"doc_{random_suffix}" - elcarro_engine.drop_document_table(table_name) + try: + elcarro_engine.drop_document_table(table_name) + except sqlalchemy.exc.NoSuchTableError as e: + pass create_table_query = f"""CREATE TABLE {table_name} ( fruit_id NUMBER GENERATED BY DEFAULT AS IDENTITY (START WITH 1), @@ -92,14 +101,20 @@ def sample_table_2_clob(elcarro_engine: ElCarroEngine) -> Generator: yield table_name # Teardown - elcarro_engine.drop_document_table(table_name) + try: + elcarro_engine.drop_document_table(table_name) + except sqlalchemy.exc.NoSuchTableError as e: + print(e) @pytest.fixture(name="sample_table_2_varchar") def sample_table_2_varchar(elcarro_engine: ElCarroEngine) -> Generator: random_suffix = "".join(random.choices(string.ascii_lowercase, k=6)) table_name = f"doc_{random_suffix}" - elcarro_engine.drop_document_table(table_name) + try: + elcarro_engine.drop_document_table(table_name) + except sqlalchemy.exc.NoSuchTableError as e: + pass create_table_query = f"""CREATE TABLE {table_name} ( fruit_id NUMBER GENERATED BY DEFAULT AS IDENTITY (START WITH 1), @@ -115,7 +130,10 @@ def sample_table_2_varchar(elcarro_engine: ElCarroEngine) -> Generator: yield table_name # Teardown - elcarro_engine.drop_document_table(table_name) + try: + elcarro_engine.drop_document_table(table_name) + except sqlalchemy.exc.NoSuchTableError as e: + print(e) # Generate random name but don't create a table @@ -123,11 +141,17 @@ def sample_table_2_varchar(elcarro_engine: ElCarroEngine) -> Generator: def custom_table(elcarro_engine: ElCarroEngine) -> Generator: random_suffix = "".join(random.choices(string.ascii_lowercase, k=6)) table_name = f"doc_{random_suffix}" - elcarro_engine.drop_document_table(table_name) + try: + elcarro_engine.drop_document_table(table_name) + except sqlalchemy.exc.NoSuchTableError as e: + pass yield table_name # Teardown - elcarro_engine.drop_document_table(table_name) + try: + elcarro_engine.drop_document_table(table_name) + except sqlalchemy.exc.NoSuchTableError as e: + print(e) def test_load_from_sample_table_1(elcarro_engine: ElCarroEngine, sample_table_1: str):