Initial SMUI integration. Coverage: Python + shell scripts to push ru…

…les to Elasticsearch.

Keycloak fixes for local Apple M1 development. See docker/for-mac#5310 (comment) for additional information

README.md

@@ -16,7 +16,7 @@ Chorus makes deploying powerful ecommerce search easier by shifting the **buy vs
 
 3. ***Sharing Knowledge is a Must!*** It isn't enough to just have conference talks, we need sample code and sample data in order to share knowledge about improving ecommerce search. Chorus is that public environment that you can use to share your next great idea!
 
-This is the project covering Elasticsearch as the search engine. As of now (January 2022), this project is under development. For those interested in the whole stack: You can visit the [Solr version of Chorus](https://github.com/querqy/chorus)
+This is the project covering Elasticsearch as the search engine. As of now (March 2022), this project is under development. For those interested in the whole stack: You can visit the [Solr version of Chorus](https://github.com/querqy/chorus)
 
 Want to stay up-to-date with the community? Visit https://querqy.org/ to learn more, and join the [E-Commerce Search Slack](https://ecom-search.slack.com/) group for tips, tricks and news on what's new in the Chorus ecosystem.
 
@@ -33,12 +33,13 @@ Want to stay up-to-date with the community? Visit https://querqy.org/ to learn m
 * The UI (Reactivesearch) runs at http://localhost:4001 
 * Elasticsearch runs at http://localhost:9200
 * Kibana runs at http://localhost:5601
+* SMUI runs at http://localhost:9000
 * Quepid runs at http://localhost:3000
 * Prometheus runs at http://localhost:9090
 * Grafana runs at http://localhost:9091
 
 Working with macOS? Pop open all the tuning related web pages with one terminal command:
-> open http://localhost:9200 http://localhost:5601 http://localhost:3000 http://localhost:7979
+> open http://localhost:4001 http://localhost:9200 http://localhost:5601 http://localhost:9000 http://localhost:3000 http://localhost:7979
 
 # 5 Minutes to Run Chorus!
 

docker-compose.yml

@@ -1,4 +1,4 @@
-version: '3'
+version: '3.7'
 services:
 
   mysql:
@@ -8,6 +8,7 @@ services:
       - 3306:3306
     environment:
       - MYSQL_ROOT_PASSWORD=password
+      - MYSQL_DATABASE=smui
     volumes:
       - ./mysql/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d
       - mysql_data:/var/lib/mysql
@@ -54,7 +55,7 @@ services:
 
   quepid:
     container_name: quepid
-    image: o19s/quepid:6.10.0
+    image: o19s/quepid:6.10.1
     ports:
       - 3000:3000
     environment:
@@ -82,6 +83,32 @@ services:
       - mysql
       - redis
 
+  smui:
+    container_name: smui
+    build: ./smui
+    ports:
+      - 9000:9000
+    init: true
+    environment:
+      - SMUI_HEADLINE=Chorus SMUI
+      - SMUI_DB_URL=jdbc:mysql://mysql:3306/smui?autoReconnect=true&useSSL=false&allowPublicKeyRetrieval=true
+      - SMUI_DB_USER=root
+      - SMUI_DB_PASSWORD=password
+      - SMUI_DEPLOY_PRELIVE_SOLR_HOST=elasticsearch:9200
+      - SMUI_2SOLR_SOLR_HOST=elasticsearch:9200
+      - SMUI_TOGGLE_DEPL_PRELIVE=true
+      - SMUI_TOGGLE_SPELLING=true
+      - SMUI_TOGGLE_DEPL_CUSTOM_SCRIPT=true
+      - SMUI_TOGGLE_DEPL_CUSTOM_SCRIPT_PATH=/smui/conf/smui2es.sh
+      - SMUI_DEPLOY_PRELIVE_FN_RULES_TXT=/configs/ecommerce/rules.txt
+      - SMUI_DEPLOY_PRELIVE_FN_REPLACE_TXT=/configs/ecommerce/replace-rules.txt
+      - SMUI_TOGGLE_RULE_ID_LOGGING=true
+      - SMUI_TOGGLE_EVENTHISTORY=true
+      - SMUI_RULE_TAGGING_ACTIVE=true
+      - SMUI_PREDEFINED_TAGS_FILE=/smui/conf/predefined_tags.json
+    depends_on:
+      - mysql
+
   prometheus:
     image: prom/prometheus:v2.32.1
     container_name: prometheus
@@ -117,7 +144,7 @@ services:
       - "9114:9114"
 
   keycloak:
-    image: quay.io/keycloak/keycloak:16.1.0
+    image: jboss/keycloak:16.1.1
     container_name: keycloak
     command: ["-b", "0.0.0.0", "-Dkeycloak.migration.action=import", "-Dkeycloak.migration.provider=dir", "-Dkeycloak.migration.dir=/opt/jboss/keycloak/realm-config", "-Dkeycloak.migration.strategy=OVERWRITE_EXISTING", "-Djboss.socket.binding.port-offset=1000", "-Dkeycloak.profile.feature.upload_scripts=enabled"]
     ports:

quickstart.sh

@@ -60,7 +60,7 @@ do
 	shift
 done
 
-services="elasticsearch kibana chorus-ui"
+services="elasticsearch kibana chorus-ui smui"
 if $observability; then
   services="${services} grafana elasticsearch-exporter"
 fi
@@ -85,9 +85,8 @@ docker-compose up -d --build ${services}
 echo -e "${MAJOR}Waiting for Elasticsearch to start up and be online.${RESET}"
 ./elasticsearch/wait-for-es.sh # Wait for Elasticsearch to be online
 
-#ToDo: This failed, no idea why.
-#echo -e "${MINOR}waiting for Keycloak to be available${RESET}"
-#./keycloak/wait-for-keycloak.sh
+echo -e "${MINOR}waiting for Keycloak to be available${RESET}"
+./keycloak/wait-for-keycloak.sh
 
 echo -e "${MAJOR}Creating ecommerce index and defining its mapping.\n${RESET}"
 curl -s -X PUT "localhost:9200/ecommerce/" -H 'Content-Type: application/json' --data-binary @./elasticsearch/schema.json
@@ -153,6 +152,11 @@ curl -s --request PUT 'http://localhost:9200/_querqy/rewriter/replace_prelive' \
 
 echo -e "${MAJOR}Setting up SMUI${RESET}"
 #TODO: Integrate SMUI
+while [ $(curl -s http://localhost:9000/api/v1/solr-index | wc -c) -lt 2 ]; do
+    echo "Waiting 5s for SMUI to be ready..."
+    sleep 5
+done
+curl -X PUT -H "Content-Type: application/json" -d '{"name":"ecommerce", "description":"Ecommerce Demo"}' http://localhost:9000/api/v1/solr-index
 
 if $offline_lab; then
   echo -e "${MAJOR}Setting up Quepid${RESET}"

smui/Dockerfile

@@ -0,0 +1,13 @@
+FROM querqy/smui:3.14
+
+COPY conf/smui2es.sh /smui/conf/smui2es.sh
+COPY conf/*.py /smui/conf/
+COPY conf/predefined_tags.json /smui/conf/predefined_tags.json
+
+USER root
+RUN apt-get update -y
+RUN apt-get install -y python3
+RUN apt-get install -y python3-pip
+RUN chown -R smui:smui /smui/conf
+USER smui
+RUN python3 -m pip install requests

smui/conf/predefined_tags.json

@@ -0,0 +1,44 @@
+[
+  {
+    "property": "tenant",
+    "value": "MO"
+  },
+  {
+    "property": "tenant",
+    "value": "HA"
+  },
+  {
+    "solrIndexName": "de",
+    "property": "tenant",
+    "value": "MO_DE"
+  },
+  {
+    "solrIndexName": "de",
+    "property": "tenant",
+    "value": "MO_AT"
+  },
+  {
+    "solrIndexName": "de",
+    "property": "tenant",
+    "value": "MO_CH"
+  },
+  {
+    "solrIndexName": "fr",
+    "property": "tenant",
+    "value": "MO_CH"
+  },
+  {
+    "solrIndexName": "fr",
+    "property": "tenant",
+    "value": "MO_FR"
+  },
+  {
+    "solrIndexName": "fr",
+    "property": "tenant",
+    "value": "MO_BE"
+  },
+  {
+    "value": "zero results",
+    "exported": false
+  }
+]

smui/conf/push_common_rules.py

@@ -0,0 +1,23 @@
+import sys,json,requests
+
+if __name__ == "__main__":
+
+    if len(sys.argv) != 3:
+        print('Usage: push_common_rules.py <rules.txt> <http://<elasticsearch_host:port>/_querqy/rewriter/<rewriter_name>')
+        sys.exit(1)
+
+    rules_file = sys.argv[1]
+    rewriter_url = sys.argv[2]
+
+    f = open(rules_file, "r")
+
+    req = {
+        "class": "querqy.elasticsearch.rewriter.SimpleCommonRulesRewriterFactory",
+        "config": {
+            "rules" : f.read()
+        }
+    }
+
+    resp = requests.put(rewriter_url, json=req)
+    if resp.status_code != 200:
+        sys.exit(2)

smui/conf/push_replace.py

@@ -0,0 +1,26 @@
+import sys,json,requests
+
+if __name__ == "__main__":
+
+    if len(sys.argv) != 3:
+        print('Usage: push_replace.py <rules.txt> <http://<elasticsearch_host:port>/_querqy/rewriter/<rewriter_name>')
+        sys.exit(1)
+
+    rules_file = sys.argv[1]
+    rewriter_url = sys.argv[2]
+
+    f = open(rules_file, "r")
+
+    req = {
+        "class": "querqy.elasticsearch.rewriter.ReplaceRewriterFactory",
+        "config": {
+            "rules": f.read(),
+            "ignoreCase": True,
+            "inputDelimiter": ";",
+            "querqyParser": "querqy.rewrite.commonrules.WhiteSpaceQuerqyParserFactory"
+        }
+    }
+
+    resp = requests.put(rewriter_url, json=req)
+    if resp.status_code != 200:
+        sys.exit(2)

smui/conf/smui2es.sh

@@ -0,0 +1,61 @@
+#!/bin/bash
+
+set -euo pipefail
+
+SRC_TMP_FILE=$1
+DST_CP_FILE_TO=$2
+ES_HOST=$3
+SOLR_CORE_NAME=$4
+DECOMPOUND_DST_CP_FILE_TO=$5
+TARGET_SYSTEM=$6
+REPLACE_RULES_SRC_TMP_FILE=$7
+REPLACE_RULES_DST_CP_FILE_TO=$8
+
+echo "In smui2es.sh - script deploying rules to Elasticsearch for Querqy"
+echo "^-- SRC_TMP_FILE = $SRC_TMP_FILE"
+echo "^-- DST_CP_FILE_TO = $DST_CP_FILE_TO"
+echo "^-- ES_HOST = $ES_HOST"
+echo "^-- SOLR_CORE_NAME = $SOLR_CORE_NAME"
+echo "^-- DECOMPOUND_DST_CP_FILE_TO = $DECOMPOUND_DST_CP_FILE_TO"
+echo "^-- TARGET_SYSTEM = $TARGET_SYSTEM"
+echo "^-- REPLACE_RULES_SRC_TMP_FILE = $REPLACE_RULES_SRC_TMP_FILE"
+echo "^-- REPLACE_RULES_DST_CP_FILE_TO = $REPLACE_RULES_DST_CP_FILE_TO"
+
+# DEPLOYMENT
+#####
+
+echo "^-- Perform rules.txt deployment (decompound-rules.txt eventually)"
+
+function common_rules_rewriter_url() {
+  if [ $TARGET_SYSTEM == "LIVE" ]; then
+    URL="http://$ES_HOST/_querqy/rewriter/common_rules"
+  else
+    URL="http://$ES_HOST/_querqy/rewriter/common_rules_prelive"
+  fi
+}
+function replace_rewriter_url() {
+  if [ $TARGET_SYSTEM == "LIVE" ]; then
+    URL="http://$ES_HOST/_querqy/rewriter/replace"
+  else
+    URL="http://$ES_HOST/_querqy/rewriter/replace_prelive"
+  fi
+}
+
+echo "^-- ... rules.txt"
+common_rules_rewriter_url $1
+
+python3 /smui/conf/push_common_rules.py $SRC_TMP_FILE $URL
+if [ $? -ne 0 ]; then
+  exit 16
+fi
+
+echo "^-- ... replace-rules.txt"
+replace_rewriter_url $1
+if ! [[ $REPLACE_RULES_SRC_TMP_FILE == "NONE" && $REPLACE_RULES_DST_CP_FILE_TO == "NONE" ]]
+then
+    python3 /smui/conf/push_replace.py $REPLACE_RULES_SRC_TMP_FILE $URL
+fi
+
+# all ok
+echo "smui2es.sh - ok"
+exit 0