From 00be82f6631809ec36219ccf302c952eb0d71001 Mon Sep 17 00:00:00 2001 From: wrigleyDan Date: Mon, 7 Mar 2022 13:58:25 +0100 Subject: [PATCH] Initial SMUI integration. Coverage: Python + shell scripts to push rules to Elasticsearch. Keycloak fixes for local Apple M1 development. See https://github.com/docker/for-mac/issues/5310#issuecomment-877653653 for additional information --- README.md | 5 +-- docker-compose.yml | 33 ++++++++++++++++-- quickstart.sh | 12 ++++--- smui/Dockerfile | 13 ++++++++ smui/README.md | 0 smui/conf/predefined_tags.json | 44 ++++++++++++++++++++++++ smui/conf/push_common_rules.py | 23 +++++++++++++ smui/conf/push_replace.py | 26 +++++++++++++++ smui/conf/smui2es.sh | 61 ++++++++++++++++++++++++++++++++++ 9 files changed, 208 insertions(+), 9 deletions(-) create mode 100644 smui/Dockerfile create mode 100644 smui/README.md create mode 100644 smui/conf/predefined_tags.json create mode 100755 smui/conf/push_common_rules.py create mode 100644 smui/conf/push_replace.py create mode 100755 smui/conf/smui2es.sh diff --git a/README.md b/README.md index ee557c4..19e881b 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ Chorus makes deploying powerful ecommerce search easier by shifting the **buy vs 3. ***Sharing Knowledge is a Must!*** It isn't enough to just have conference talks, we need sample code and sample data in order to share knowledge about improving ecommerce search. Chorus is that public environment that you can use to share your next great idea! -This is the project covering Elasticsearch as the search engine. As of now (January 2022), this project is under development. For those interested in the whole stack: You can visit the [Solr version of Chorus](https://github.com/querqy/chorus) +This is the project covering Elasticsearch as the search engine. As of now (March 2022), this project is under development. For those interested in the whole stack: You can visit the [Solr version of Chorus](https://github.com/querqy/chorus) Want to stay up-to-date with the community? Visit https://querqy.org/ to learn more, and join the [E-Commerce Search Slack](https://ecom-search.slack.com/) group for tips, tricks and news on what's new in the Chorus ecosystem. @@ -33,12 +33,13 @@ Want to stay up-to-date with the community? Visit https://querqy.org/ to learn m * The UI (Reactivesearch) runs at http://localhost:4001 * Elasticsearch runs at http://localhost:9200 * Kibana runs at http://localhost:5601 +* SMUI runs at http://localhost:9000 * Quepid runs at http://localhost:3000 * Prometheus runs at http://localhost:9090 * Grafana runs at http://localhost:9091 Working with macOS? Pop open all the tuning related web pages with one terminal command: -> open http://localhost:9200 http://localhost:5601 http://localhost:3000 http://localhost:7979 +> open http://localhost:4001 http://localhost:9200 http://localhost:5601 http://localhost:9000 http://localhost:3000 http://localhost:7979 # 5 Minutes to Run Chorus! diff --git a/docker-compose.yml b/docker-compose.yml index 902a7d3..6d9f841 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,4 @@ -version: '3' +version: '3.7' services: mysql: @@ -8,6 +8,7 @@ services: - 3306:3306 environment: - MYSQL_ROOT_PASSWORD=password + - MYSQL_DATABASE=smui volumes: - ./mysql/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d - mysql_data:/var/lib/mysql @@ -54,7 +55,7 @@ services: quepid: container_name: quepid - image: o19s/quepid:6.10.0 + image: o19s/quepid:6.10.1 ports: - 3000:3000 environment: @@ -82,6 +83,32 @@ services: - mysql - redis + smui: + container_name: smui + build: ./smui + ports: + - 9000:9000 + init: true + environment: + - SMUI_HEADLINE=Chorus SMUI + - SMUI_DB_URL=jdbc:mysql://mysql:3306/smui?autoReconnect=true&useSSL=false&allowPublicKeyRetrieval=true + - SMUI_DB_USER=root + - SMUI_DB_PASSWORD=password + - SMUI_DEPLOY_PRELIVE_SOLR_HOST=elasticsearch:9200 + - SMUI_2SOLR_SOLR_HOST=elasticsearch:9200 + - SMUI_TOGGLE_DEPL_PRELIVE=true + - SMUI_TOGGLE_SPELLING=true + - SMUI_TOGGLE_DEPL_CUSTOM_SCRIPT=true + - SMUI_TOGGLE_DEPL_CUSTOM_SCRIPT_PATH=/smui/conf/smui2es.sh + - SMUI_DEPLOY_PRELIVE_FN_RULES_TXT=/configs/ecommerce/rules.txt + - SMUI_DEPLOY_PRELIVE_FN_REPLACE_TXT=/configs/ecommerce/replace-rules.txt + - SMUI_TOGGLE_RULE_ID_LOGGING=true + - SMUI_TOGGLE_EVENTHISTORY=true + - SMUI_RULE_TAGGING_ACTIVE=true + - SMUI_PREDEFINED_TAGS_FILE=/smui/conf/predefined_tags.json + depends_on: + - mysql + prometheus: image: prom/prometheus:v2.32.1 container_name: prometheus @@ -117,7 +144,7 @@ services: - "9114:9114" keycloak: - image: quay.io/keycloak/keycloak:16.1.0 + image: jboss/keycloak:16.1.1 container_name: keycloak command: ["-b", "0.0.0.0", "-Dkeycloak.migration.action=import", "-Dkeycloak.migration.provider=dir", "-Dkeycloak.migration.dir=/opt/jboss/keycloak/realm-config", "-Dkeycloak.migration.strategy=OVERWRITE_EXISTING", "-Djboss.socket.binding.port-offset=1000", "-Dkeycloak.profile.feature.upload_scripts=enabled"] ports: diff --git a/quickstart.sh b/quickstart.sh index 042f858..bbb2414 100755 --- a/quickstart.sh +++ b/quickstart.sh @@ -60,7 +60,7 @@ do shift done -services="elasticsearch kibana chorus-ui" +services="elasticsearch kibana chorus-ui smui" if $observability; then services="${services} grafana elasticsearch-exporter" fi @@ -85,9 +85,8 @@ docker-compose up -d --build ${services} echo -e "${MAJOR}Waiting for Elasticsearch to start up and be online.${RESET}" ./elasticsearch/wait-for-es.sh # Wait for Elasticsearch to be online -#ToDo: This failed, no idea why. -#echo -e "${MINOR}waiting for Keycloak to be available${RESET}" -#./keycloak/wait-for-keycloak.sh +echo -e "${MINOR}waiting for Keycloak to be available${RESET}" +./keycloak/wait-for-keycloak.sh echo -e "${MAJOR}Creating ecommerce index and defining its mapping.\n${RESET}" curl -s -X PUT "localhost:9200/ecommerce/" -H 'Content-Type: application/json' --data-binary @./elasticsearch/schema.json @@ -153,6 +152,11 @@ curl -s --request PUT 'http://localhost:9200/_querqy/rewriter/replace_prelive' \ echo -e "${MAJOR}Setting up SMUI${RESET}" #TODO: Integrate SMUI +while [ $(curl -s http://localhost:9000/api/v1/solr-index | wc -c) -lt 2 ]; do + echo "Waiting 5s for SMUI to be ready..." + sleep 5 +done +curl -X PUT -H "Content-Type: application/json" -d '{"name":"ecommerce", "description":"Ecommerce Demo"}' http://localhost:9000/api/v1/solr-index if $offline_lab; then echo -e "${MAJOR}Setting up Quepid${RESET}" diff --git a/smui/Dockerfile b/smui/Dockerfile new file mode 100644 index 0000000..95b3f35 --- /dev/null +++ b/smui/Dockerfile @@ -0,0 +1,13 @@ +FROM querqy/smui:3.14 + +COPY conf/smui2es.sh /smui/conf/smui2es.sh +COPY conf/*.py /smui/conf/ +COPY conf/predefined_tags.json /smui/conf/predefined_tags.json + +USER root +RUN apt-get update -y +RUN apt-get install -y python3 +RUN apt-get install -y python3-pip +RUN chown -R smui:smui /smui/conf +USER smui +RUN python3 -m pip install requests \ No newline at end of file diff --git a/smui/README.md b/smui/README.md new file mode 100644 index 0000000..e69de29 diff --git a/smui/conf/predefined_tags.json b/smui/conf/predefined_tags.json new file mode 100644 index 0000000..78eff83 --- /dev/null +++ b/smui/conf/predefined_tags.json @@ -0,0 +1,44 @@ +[ + { + "property": "tenant", + "value": "MO" + }, + { + "property": "tenant", + "value": "HA" + }, + { + "solrIndexName": "de", + "property": "tenant", + "value": "MO_DE" + }, + { + "solrIndexName": "de", + "property": "tenant", + "value": "MO_AT" + }, + { + "solrIndexName": "de", + "property": "tenant", + "value": "MO_CH" + }, + { + "solrIndexName": "fr", + "property": "tenant", + "value": "MO_CH" + }, + { + "solrIndexName": "fr", + "property": "tenant", + "value": "MO_FR" + }, + { + "solrIndexName": "fr", + "property": "tenant", + "value": "MO_BE" + }, + { + "value": "zero results", + "exported": false + } +] \ No newline at end of file diff --git a/smui/conf/push_common_rules.py b/smui/conf/push_common_rules.py new file mode 100755 index 0000000..c8dcedd --- /dev/null +++ b/smui/conf/push_common_rules.py @@ -0,0 +1,23 @@ +import sys,json,requests + +if __name__ == "__main__": + + if len(sys.argv) != 3: + print('Usage: push_common_rules.py /_querqy/rewriter/') + sys.exit(1) + + rules_file = sys.argv[1] + rewriter_url = sys.argv[2] + + f = open(rules_file, "r") + + req = { + "class": "querqy.elasticsearch.rewriter.SimpleCommonRulesRewriterFactory", + "config": { + "rules" : f.read() + } + } + + resp = requests.put(rewriter_url, json=req) + if resp.status_code != 200: + sys.exit(2) \ No newline at end of file diff --git a/smui/conf/push_replace.py b/smui/conf/push_replace.py new file mode 100644 index 0000000..e396723 --- /dev/null +++ b/smui/conf/push_replace.py @@ -0,0 +1,26 @@ +import sys,json,requests + +if __name__ == "__main__": + + if len(sys.argv) != 3: + print('Usage: push_replace.py /_querqy/rewriter/') + sys.exit(1) + + rules_file = sys.argv[1] + rewriter_url = sys.argv[2] + + f = open(rules_file, "r") + + req = { + "class": "querqy.elasticsearch.rewriter.ReplaceRewriterFactory", + "config": { + "rules": f.read(), + "ignoreCase": True, + "inputDelimiter": ";", + "querqyParser": "querqy.rewrite.commonrules.WhiteSpaceQuerqyParserFactory" + } + } + + resp = requests.put(rewriter_url, json=req) + if resp.status_code != 200: + sys.exit(2) \ No newline at end of file diff --git a/smui/conf/smui2es.sh b/smui/conf/smui2es.sh new file mode 100755 index 0000000..1df580d --- /dev/null +++ b/smui/conf/smui2es.sh @@ -0,0 +1,61 @@ +#!/bin/bash + +set -euo pipefail + +SRC_TMP_FILE=$1 +DST_CP_FILE_TO=$2 +ES_HOST=$3 +SOLR_CORE_NAME=$4 +DECOMPOUND_DST_CP_FILE_TO=$5 +TARGET_SYSTEM=$6 +REPLACE_RULES_SRC_TMP_FILE=$7 +REPLACE_RULES_DST_CP_FILE_TO=$8 + +echo "In smui2es.sh - script deploying rules to Elasticsearch for Querqy" +echo "^-- SRC_TMP_FILE = $SRC_TMP_FILE" +echo "^-- DST_CP_FILE_TO = $DST_CP_FILE_TO" +echo "^-- ES_HOST = $ES_HOST" +echo "^-- SOLR_CORE_NAME = $SOLR_CORE_NAME" +echo "^-- DECOMPOUND_DST_CP_FILE_TO = $DECOMPOUND_DST_CP_FILE_TO" +echo "^-- TARGET_SYSTEM = $TARGET_SYSTEM" +echo "^-- REPLACE_RULES_SRC_TMP_FILE = $REPLACE_RULES_SRC_TMP_FILE" +echo "^-- REPLACE_RULES_DST_CP_FILE_TO = $REPLACE_RULES_DST_CP_FILE_TO" + +# DEPLOYMENT +##### + +echo "^-- Perform rules.txt deployment (decompound-rules.txt eventually)" + +function common_rules_rewriter_url() { + if [ $TARGET_SYSTEM == "LIVE" ]; then + URL="http://$ES_HOST/_querqy/rewriter/common_rules" + else + URL="http://$ES_HOST/_querqy/rewriter/common_rules_prelive" + fi +} +function replace_rewriter_url() { + if [ $TARGET_SYSTEM == "LIVE" ]; then + URL="http://$ES_HOST/_querqy/rewriter/replace" + else + URL="http://$ES_HOST/_querqy/rewriter/replace_prelive" + fi +} + +echo "^-- ... rules.txt" +common_rules_rewriter_url $1 + +python3 /smui/conf/push_common_rules.py $SRC_TMP_FILE $URL +if [ $? -ne 0 ]; then + exit 16 +fi + +echo "^-- ... replace-rules.txt" +replace_rewriter_url $1 +if ! [[ $REPLACE_RULES_SRC_TMP_FILE == "NONE" && $REPLACE_RULES_DST_CP_FILE_TO == "NONE" ]] +then + python3 /smui/conf/push_replace.py $REPLACE_RULES_SRC_TMP_FILE $URL +fi + +# all ok +echo "smui2es.sh - ok" +exit 0 \ No newline at end of file