SOLR/API/IMPORTER - major infrastructure update (#271)

* SOLR/API/IMPORTER - major infrastructure update

Signed-off-by: Kial Jinnah <kialj876@gmail.com>

* cleanup

Signed-off-by: Kial Jinnah <kialj876@gmail.com>

* fix test setup attempt

Signed-off-by: Kial Jinnah <kialj876@gmail.com>

* downgrade deps to be the same as bor

Signed-off-by: Kial Jinnah <kialj876@gmail.com>

---------

Signed-off-by: Kial Jinnah <kialj876@gmail.com>

.gitignore

@@ -131,4 +131,8 @@ search-solr/test_solr
 
 .DS_Store
 
-.history
+.history
+
+__pycache__
+
+.python-version

search-api/.env.sample

@@ -5,69 +5,68 @@ FLASK_APP=wsgi.py
 # Namespace
 POD_NAMESPACE=local
 
-# third party Services
-LD_SDK_KEY=
+# Timeouts
+AUTH_API_TIMEOUT=20
+PAY_API_TIMEOUT=20
+BUSINESS_API_TIMEOUT=30
+
+# Sentry
 SENTRY_DSN=
 SENTRY_TSR=1.0
 
-# Registry Integration Services
-AUTH_SVC_URL=
-PAYMENT_SVC_URL=
-LEGAL_API_URL=
-LEGAL_API_VERSION_2=/api/v2
-
-# SQL Alchemy
-DATABASE_USERNAME=
-DATABASE_PASSWORD=
-DATABASE_NAME=
-DATABASE_HOST=
-DATABASE_PORT=
-
-# SOLR URL
-SOLR_SVC_URL=
-
-# TEST SOLR
-SOLR_SVC_TEST_URL=
+# LDarkly
+LD_SDK_KEY=
+OPS_LOGGER_LEVEL=ops-logger-level
 
-# Flag Names
-FF_QUEUE_DOC_REQUEST_NAME=
-OPS_LOGGER_LEVEL=
-PUBSUB_EMULATOR_HOST=fake
+# JWT Settings
+JWT_OIDC_JWKS_CACHE_TIMEOUT=300
+JWT_OIDC_ALGORITHMS=RS256
+JWT_OIDC_AUDIENCE=business-search-service
+JWT_OIDC_CACHING_ENABLED=True
 
-# Queue
-QUEUE_PROJECT_ID=
+# Test values
+SOLR_SVC_BUS_LEADER_TEST_URL=http://localhost:8873/solr
+SOLR_SVC_BUS_FOLLOWER_TEST_URL=http://localhost:8873/solr
 
-# TEST DB
 DATABASE_TEST_USERNAME=
 DATABASE_TEST_PASSWORD=
 DATABASE_TEST_NAME=
 DATABASE_TEST_HOST=localhost
 DATABASE_TEST_PORT=5432
 
-# Service Account
-ACCOUNT_SVC_AUTH_URL=
-ACCOUNT_SVC_CLIENT_ID=
-ACCOUNT_SVC_CLIENT_SECRET=
-
-# JWT Settings
-JWT_OIDC_WELL_KNOWN_CONFIG=
-JWT_OIDC_ALGORITHMS=RS256
-JWT_OIDC_AUDIENCE=
-JWT_OIDC_CLIENT_SECRET=
-JWT_OIDC_CACHING_ENABLED=True
-JWT_OIDC_JWKS_CACHE_TIMEOUT=300
-
-# SOLR REINDEX INFO
-# i.e. 02, 09, 21
-SOLR_REINDEX_DAY=
-SOLR_REINDEX_WEEKDAY=1  # 0-6 (0 = Monday with python3.8)
-SOLR_REINDEX_START_TIME=07:05:00+0000  # HH:MM:SS+HHMM
-SOLR_REINDEX_LENGTH=15  # minutes
-
-# INTEGRATION TESTS
 RUN_AUTHORIZATION_TESTS=True
 RUN_LD_TESTS=True
 RUN_NATS_TESTS=True
 RUN_PAYMENT_TESTS=True
 RUN_SOLR_TESTS=True
 NOT_GITHUB_CI=True
+
+## Local
+
+# SQL Alchemy
+DATABASE_USERNAME=
+DATABASE_PASSWORD=
+DATABASE_NAME=
+DATABASE_HOST=localhost
+DATABASE_PORT=5432
+
+# Solr
+SOLR_SVC_BUS_LEADER_URL=http://localhost:8873/solr
+SOLR_SVC_BUS_FOLLOWER_URL=http://localhost:8873/solr
+SOLR_SVC_BUS_LEADER_CORE=business
+SOLR_SVC_BUS_FOLLOWER_CORE=business
+
+## DEV
+AUTH_SVC_URL=https://auth-api-dev.apps.silver.devops.gov.bc.ca/api/v1
+PAYMENT_SVC_URL=https://pay-api-dev.apps.silver.devops.gov.bc.ca/api/v1/payment-requests
+LEGAL_API_URL=https://legal-api-dev.apps.silver.devops.gov.bc.ca
+LEGAL_API_VERSION_2=/api/v2
+
+JWT_OIDC_CLIENT_SECRET=
+JWT_OIDC_ISSUER=
+JWT_TOKEN_URL=
+JWT_OIDC_WELL_KNOWN_CONFIG=
+
+ACCOUNT_SVC_AUTH_URL=
+ACCOUNT_SVC_CLIENT_ID=
+ACCOUNT_SVC_CLIENT_SECRET=

search-api/migrations/alembic.ini

@@ -3,6 +3,7 @@
 [alembic]
 # template used to generate migration files
 # file_template = %%(rev)s_%%(slug)s
+file_template = %%(year)d%%(month).2d%%(day).2d_%%(hour).2d%%(minute).2d%%(second).2d_%%(rev)s_%%(slug)s
 
 # set to 'true' to run the environment during
 # the 'revision' command, regardless of autogenerate

search-api/migrations/versions/20240723_131259_eaf25b9a20bf_.py

@@ -0,0 +1,42 @@
+"""empty message
+
+Revision ID: eaf25b9a20bf
+Revises: c10f3d4b2262
+Create Date: 2024-07-23 13:12:59.736684
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'eaf25b9a20bf'
+down_revision = 'c10f3d4b2262'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('solr_doc_events', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('event_last_update', sa.DateTime(timezone=True), nullable=True))
+
+    with op.batch_alter_table('users', schema=None) as batch_op:
+        batch_op.drop_index('ix_user_idp_userid')
+        batch_op.drop_constraint('users_idp_userid_key', type_='unique')
+        batch_op.create_index(batch_op.f('ix_users_idp_userid'), ['idp_userid'], unique=True)
+
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('users', schema=None) as batch_op:
+        batch_op.drop_index(batch_op.f('ix_users_idp_userid'))
+        batch_op.create_unique_constraint('users_idp_userid_key', ['idp_userid'])
+        batch_op.create_index('ix_user_idp_userid', ['idp_userid'], unique=True)
+
+    with op.batch_alter_table('solr_doc_events', schema=None) as batch_op:
+        batch_op.drop_column('event_last_update')
+
+    # ### end Alembic commands ###

search-api/requirements.txt

@@ -1,68 +1,66 @@
-Babel==2.12.1
-Flask-Cors==3.0.10
+Babel==2.15.0
+Flask-Cors==4.0.0
 Flask-Migrate==4.0.4
-Flask-Moment==1.0.5
-Flask-SQLAlchemy==3.0.3
-Flask==2.2.3
-Jinja2==3.1.2
-Mako==1.2.4
-MarkupSafe==2.1.2
-SQLAlchemy==2.0.6
-Werkzeug==2.2.3
-alembic==1.10.2
-attrs==22.2.0
-blinker==1.5
-cachelib==0.10.2
-cachetools==5.3.0
-certifi==2022.12.7
-charset-normalizer==3.1.0
-click==8.1.3
+Flask-Moment==1.0.6
+Flask-SQLAlchemy==3.0.5
+Flask==2.3.2
+Jinja2==3.1.4
+Mako==1.3.5
+MarkupSafe==2.1.5
+SQLAlchemy==2.0.19
+Werkzeug==2.3.6
+alembic==1.13.2
+attrs==23.2.0
+blinker==1.8.2
+cachelib==0.13.0
+cachetools==5.4.0
+certifi==2024.7.4
+charset-normalizer==3.3.2
+click==8.1.7
 datedelta==1.4
-dpath==2.1.4
-ecdsa==0.18.0
+dpath==2.2.0
+ecdsa==0.19.0
 expiringdict==1.2.2
-flask-babel==3.0.1
+flask-babel==4.0.0
 flask-jwt-oidc==0.3.0
-google-api-core==2.11.0
-google-auth-oauthlib==1.0.0
-google-auth==2.16.2
-google-cloud-core==2.3.2
-google-cloud-pubsub==2.15.0
-google-cloud-storage==2.7.0
+google-api-core==2.19.1
+google-auth-oauthlib==1.2.1
+google-auth==2.32.0
+google-cloud-core==2.4.1
+google-cloud-pubsub==2.22.0
+google-cloud-storage==2.17.0
 google-crc32c==1.5.0
-google-resumable-media==2.4.1
-googleapis-common-protos==1.58.0
-greenlet==2.0.2
-grpc-google-iam-v1==0.12.6
-grpcio-status==1.51.3
-grpcio==1.51.3
-gunicorn==20.1.0
-idna==3.4
-itsdangerous==2.1.2
+google-resumable-media==2.7.1
+googleapis-common-protos==1.63.2
+grpc-google-iam-v1==0.13.1
+grpcio-status==1.65.1
+grpcio==1.65.1
+gunicorn==22.0.0
+idna==3.7
+itsdangerous==2.2.0
 jsonschema==3.2.0
-launchdarkly-server-sdk==8.1.1
+launchdarkly-server-sdk==8.1.4
 oauthlib==3.2.2
-packaging==23.0
-proto-plus==1.22.2
-protobuf==4.22.1
-psycopg2-binary==2.9.5
+packaging==24.1
+proto-plus==1.24.0
+protobuf==5.27.2
+psycopg2-binary==2.9.9
 pyRFC3339==1.1
-pyasn1-modules==0.2.8
-pyasn1==0.4.8
-pyrsistent==0.19.3
-python-dateutil==2.8.2
-python-dotenv==1.0.0
+pyasn1==0.6.0
+pyasn1_modules==0.4.0
+pyrsistent==0.20.0
+python-dateutil==2.9.0.post0
+python-dotenv==1.0.1
 python-jose==3.3.0
-pytz==2022.7.1
-requests-oauthlib==1.3.1
-requests==2.28.2
+pytz==2024.1
+requests-oauthlib==2.0.0
+requests==2.31.0
 rsa==4.9
 semver==2.13.0
-sentry-sdk==1.20.0
+sentry-sdk==2.10.0
 six==1.16.0
 strict-rfc3339==0.7
-typing_extensions==4.5.0
-urllib3==1.26.15
-git+https://github.com/bcgov/registry-schemas.git@1.5.3#egg=registry_schemas
+typing_extensions==4.12.2
+urllib3==1.26.16
 git+https://github.com/daxiom/simple-cloudevent.py.git@0.0.2
-git+https://github.com/daxiom/flask-pub.git@0.0.4
+git+https://github.com/daxiom/flask-pub.git@0.0.4

search-api/requirements/bcregistry-libraries.txt

@@ -1 +0,0 @@
-git+https://github.com/bcgov/registry-schemas.git@1.5.3#egg=registry_schemas

search-api/setup.cfg

@@ -41,7 +41,7 @@ application_import_names=search_api
 per-file-ignores =
     *.py:I001
     */__init__.py:F401
-    */solr_docs.py:N815
+    */doc_models/*.py:N815,F401
 
 [pycodestyle]
 max_line_length = 120

search-api/src/search_api/__init__.py

@@ -23,17 +23,14 @@
 
 import sentry_sdk  # noqa: I001; pylint: disable=ungrouped-imports; conflicts with Flake8
 from sentry_sdk.integrations.flask import FlaskIntegration  # noqa: I001
-from flask import redirect, url_for, Flask  # noqa: I001
-from flask_migrate import Migrate
-from registry_schemas import __version__ as registry_schemas_version
-from registry_schemas.flask import SchemaServices  # noqa: I001
+from flask import Flask, redirect  # noqa: I001
+from flask_migrate import Migrate  # noqa: I001
 
 from search_api import errorhandlers, models
 from search_api.config import config
 from search_api.models import db
 from search_api.resources import v1_endpoint
-from search_api.schemas import rsbc_schemas
-from search_api.services import Flags, queue, search_solr
+from search_api.services import Flags, business_solr, queue
 from search_api.translations import babel
 from search_api.utils.auth import jwt
 from search_api.utils.logging import set_log_level_by_flag, setup_logging
@@ -65,9 +62,8 @@ def create_app(config_name: str = os.getenv('APP_ENV') or 'production', **kwargs
 
     errorhandlers.init_app(app)
     db.init_app(app)
-    rsbc_schemas.init_app(app)
     queue.init_app(app)
-    search_solr.init_app(app)
+    business_solr.init_app(app)
     babel.init_app(app)
     migrate.init_app(app, db)
 
@@ -88,7 +84,6 @@ def be_nice_swagger_redirect():  # pylint: disable=unused-variable
     def add_version(response):  # pylint: disable=unused-variable
         version = get_run_version()
         response.headers['API'] = f'search_api/{version}'
-        response.headers['SCHEMAS'] = f'registry_schemas/{registry_schemas_version}'
         return response
 
     register_shellcontext(app)

search-api/src/search_api/config.py

@@ -35,9 +35,18 @@ class Config():  # pylint: disable=too-few-public-methods
 
     PROJECT_ROOT = os.path.abspath(os.path.dirname(__file__))
 
+    # Used by /sync endpoint
+    MAX_BATCH_UPDATE_NUM = int(os.getenv('MAX_BATCH_UPDATE_NUM', '1000'))
+    # Used by /sync heartbeat
+    LAST_REPLICATION_THRESHOLD = int(os.getenv('LAST_REPLICATION_THRESHOLD', '24'))  # hours
+
+    SOLR_SVC_BUS_LEADER_CORE = os.getenv('SOLR_SVC_BUS_LEADER_CORE', 'business')
+    SOLR_SVC_BUS_FOLLOWER_CORE = os.getenv('SOLR_SVC_BUS_FOLLOWER_CORE', 'business_follower')
+    SOLR_SVC_BUS_LEADER_URL = os.getenv('SOLR_SVC_BUS_LEADER_URL', 'http://localhost:8873/solr')
+    SOLR_SVC_BUS_FOLLOWER_URL = os.getenv('SOLR_SVC_BUS_FOLLOWER_URL', 'http://localhost:8873/solr')
+
     PAYMENT_SVC_URL = os.getenv('PAYMENT_SVC_URL', 'http://')
     AUTH_SVC_URL = os.getenv('AUTH_SVC_URL', 'http://')
-    SOLR_SVC_URL = os.getenv('SOLR_SVC_URL', 'http://')
     LEAR_SVC_URL = os.getenv('LEGAL_API_URL', 'http://') + os.getenv('LEGAL_API_VERSION_2', '/api/v2')
 
     # Flask-Pub
@@ -138,15 +147,6 @@ class Config():  # pylint: disable=too-few-public-methods
     GATEWAY_URL = os.getenv('GATEWAY_URL', 'https://bcregistry-dev.apigee.net')
     SUBSCRIPTION_API_KEY = os.getenv('SUBSCRIPTION_API_KEY')
 
-    # reindex times
-    SOLR_REINDEX_DAY = os.getenv('SOLR_REINDEX_DAY', None)
-    if os.getenv('SOLR_REINDEX_WEEKDAY'):
-        SOLR_REINDEX_WEEKDAY = int(os.getenv('SOLR_REINDEX_WEEKDAY'))
-    else:
-        SOLR_REINDEX_WEEKDAY = None
-    SOLR_REINDEX_START_TIME = os.getenv('SOLR_REINDEX_START_TIME', '')
-    SOLR_REINDEX_LENGTH = int(os.getenv('SOLR_REINDEX_LENGTH')) if os.getenv('SOLR_REINDEX_LENGTH', None) else 0
-
 
 class DevelopmentConfig(Config):  # pylint: disable=too-few-public-methods
     """Config object for development environment."""
@@ -163,7 +163,10 @@ class UnitTestingConfig(Config):  # pylint: disable=too-few-public-methods
     DEVELOPMENT = False
     TESTING = True
     # SOLR
-    SOLR_SVC_URL = os.getenv('SOLR_SVC_TEST_URL', 'http://')
+    SOLR_SVC_BUS_LEADER_CORE = os.getenv('SOLR_SVC_BUS_LEADER_TEST_CORE', 'business')
+    SOLR_SVC_BUS_FOLLOWER_CORE = os.getenv('SOLR_SVC_BUS_FOLLOWER_TEST_CORE', 'business')
+    SOLR_SVC_BUS_LEADER_URL = os.getenv('SOLR_SVC_BUS_LEADER_TEST_URL', 'http://localhost:8980/solr')
+    SOLR_SVC_BUS_FOLLOWER_URL = os.getenv('SOLR_SVC_BUS_FOLLOWER_TEST_URL', 'http://localhost:8980/solr')
     # POSTGRESQL
     DB_USER = os.getenv('DATABASE_TEST_USERNAME', '')
     DB_PASSWORD = os.getenv('DATABASE_TEST_PASSWORD', '')