Fixing validator missing device error (#975)

.gencode_hash.txt

@@ -3,8 +3,8 @@
 fb876b8f7bfdccf156d751efddf2660a7a7ab5585e89be0f82c59f7a61e68d4c  gencode/docs/config.html
 22bfb3fd7c0e73dac4cbd830aa7983714917ec6da813f6bf5768ce3aa9786fb2  gencode/docs/config_mapping.html
 1ce30b7c554a0be09a9124707cb5602dce78753304cca17c06268ecb3337da51  gencode/docs/configuration_endpoint.html
-c2ad0293315ef1e8fd6a0326752b4ce4d7b0d54ae07cd1ebd07be351ac22f11f  gencode/docs/configuration_execution.html
-6a097f1f87ab7b9a82e4d3aa6f6cedf69499a95742863a7f53bb5fdf53fd6ff0  gencode/docs/configuration_pod.html
+136eeb98d4db5c2427498b0483e0fb8e73bc3dcb962f0ab942f7b98115b1b71f  gencode/docs/configuration_execution.html
+f4937abf377ef27e18e2dba77589fa08f2d955cf85a905f360ddaf4ad0bea0c3  gencode/docs/configuration_pod.html
 4d327975ae4d48a5dd920f298931fc5056a7239d677e61a9f4d129068316dd0a  gencode/docs/configuration_pubber.html
 f24d15e549f1b143b50d6011f2f7fd03286e1cd128a3e25591195b25f8efc472  gencode/docs/data_template.html
 45d39c61ce39afc0d14f3dc99a93947587d63999cd373cb44b63ce8f576c6e7e  gencode/docs/events.html
@@ -81,7 +81,7 @@ e256275d840d94c6e61316e756b0dad6d823db8900e35e785f6ef5001fdb5fc9  gencode/java/u
 a773f8271a99f96a3fb09ceb79bc4022a8b7ab941f5e6246e073703d33022b99  gencode/java/udmi/schema/GatewayConfig.java
 0f176bbd9748ac357ebcb7cf9ba66fca4f4768ce40a2a1436bb6d14093ee8101  gencode/java/udmi/schema/GatewayModel.java
 1f7f6809e143e9e3e13e6fa73776a893b62b0c05f8802087b1e276dc0ab83e19  gencode/java/udmi/schema/GatewayState.java
-d14fdea87e337eb8bbe44f6b1a435dd0bacaf71cf6384dfeaf28a0a121e07605  gencode/java/udmi/schema/IotAccess.java
+6ffcc0820c02c9f83787f3102389380b5d4da20c4d813219964151bdfebf335e  gencode/java/udmi/schema/IotAccess.java
 d0858bd7ba306176f4e3c19eeb69ef2467afec0832b6917b72d3ab87469e4974  gencode/java/udmi/schema/Jwt.java
 5a0cc53317592a868f7f23a67a7493bd571054bb25d9e28de51000ad6473dfeb  gencode/java/udmi/schema/Level.java
 6b4c29a682d0697d5420efb9e7788ec7358d94435da288938007205b8669e180  gencode/java/udmi/schema/LocalnetConfig.java

bin/pubber

@@ -13,7 +13,7 @@ function cleanup {
 }
 trap cleanup EXIT
 
-[[ $# == 0 ]] && usage SITE_{PATH/FILE} [PROJECT_ID] [DEVICE_ID] [SERIAL_NO] [options] ...
+[[ $# == 0 ]] && usage SITE_{PATH/FILE} [PROJECT_SPEC] [DEVICE_ID] [SERIAL_NO] [options] ...
 
 config_file=$(realpath $1)
 serial_no=NA

bin/test_combos

@@ -4,8 +4,11 @@
 #
 
 tests=tests/combos/*.json
+echo Testing $tests
 
 validator/bin/build
+jarfile=$(realpath validator/build/libs/validator-1.0-SNAPSHOT-all.jar)
+javarun="java -cp $jarfile com.google.daq.mqtt.validator.SchemaTester"
 
 failures=
 for test in $tests; do
@@ -14,7 +17,10 @@ for test in $tests; do
     site_model=$(jq -r .site_model $test)
     validator_config=$(jq -r .validator $test)
     if [[ $validator_config != null ]]; then
-        validator/bin/validate -- schema trace $traces $site_model
+        cmd="$javarun -p -- -s $site_model -r $traces --"
+        echo $cmd
+        $cmd
+
         check=$(echo $validator_config | jq -r .check)
         echo $check
         code=0

bin/test_schema

@@ -46,6 +46,7 @@ if [ "$build" == y ]; then
 fi
 
 jarfile=$(realpath validator/build/libs/validator-1.0-SNAPSHOT-all.jar)
+javarun="java -cp $jarfile com.google.daq.mqtt.validator.SchemaTester"
 
 if [[ -z $subsets ]]; then
     subsets=$(cd $testdir; ls -d *)
@@ -68,7 +69,7 @@ for subset in $subsets; do
         reldir=$(realpath --relative-to $schemadir $testdir)
         reltest=$reldir:$subset
     fi
-    (cd $schemadir; java -jar -Dnashorn.args=--no-deprecation-warning $jarfile -a $schemaname $upgrade -f $reltest) || true
+    (cd $schemadir; $javarun -a $schemaname $upgrade -f $reltest) || true
 
     for testfile in $testfiles; do
         outfile=${testfile%.json}.out

bin/test_trace

@@ -19,6 +19,9 @@ function redact() {
         -e 's/\\n.*\.java:[0-9]+\)/\\tREDACTED\\n/g'
 }
 
+jarfile=$(realpath validator/build/libs/validator-1.0-SNAPSHOT-all.jar)
+javarun="java -cp $jarfile com.google.daq.mqtt.validator.SchemaTester"
+validator/bin/build
 
 trace_in=tests/traces/${trace_name}
 trace_out=$trace_in/expected/
@@ -33,7 +36,9 @@ export UDMI_TOOLS=test_trace
 
 # Clean things out, including some potentially corrupt state.
 rm -rf $site_out $site_model/devices/XXX-1
-validator/bin/validate -- schema trace $trace_in $site_model
+cmd="$javarun -p -- -s $site_model -r $trace_in --"
+echo $cmd
+$cmd
 
 echo Redacting output files...
 for file in `find $site_out -type f`; do redact; done

bin/validator

@@ -0,0 +1 @@
+toolrun

common/src/main/java/com/google/udmi/util/Common.java

@@ -51,16 +51,16 @@ public abstract class Common {
   public static final char DETAIL_SEPARATOR_CHAR = ';';
   public static final String DETAIL_SEPARATOR = DETAIL_SEPARATOR_CHAR + " ";
   public static final Joiner DETAIL_JOINER = Joiner.on(DETAIL_SEPARATOR);
-  public static final String CONFIG_CATEGORY = "config";
-  public static final String COMMANDS_CATEGORY = "commands";
   public static final String CATEGORY_PROPERTY_KEY = "category";
   public static final Pattern DEVICE_ID_ALLOWABLE = Pattern.compile("^[-_a-zA-Z0-9]+$");
   public static final Pattern POINT_NAME_ALLOWABLE = DEVICE_ID_ALLOWABLE;
   public static final int SEC_TO_MS = 1000;
   public static final String SOURCE_SEPARATOR = "+";
+  public static final String SOURCE_SEPARATOR_REGEX = "\\" + SOURCE_SEPARATOR;
 
   private static final String PREFIX_SEPARATOR = "~";
   private static final String UDMI_VERSION_ENV = "UDMI_TOOLS";
+  public static final int EXIT_CODE_ERROR = 1;
 
   /**
    * Remove the next item from the list in an exception-safe way.

etc/validator.out

@@ -60,7 +60,7 @@ sites/udmi_site_model/out/devices/AHU-1/events_pointset.out
   "sub_type" : "events",
   "status" : {
     "message" : "Multiple validation errors",
-    "detail" : "Timestamp jitter REDACTED_DURATION (REDACTED_TIMESTAMP to REDACTED_TIMESTAMP) exceeds REDACTED_DURATION threshold; While converting to json node: 1 schema violations found",
+    "detail" : "Timestamp skew REDACTED_DURATION (REDACTED_TIMESTAMP to REDACTED_TIMESTAMP) exceeds REDACTED_DURATION threshold; While converting to json node: 1 schema violations found",
     "category" : "validation.device.multiple",
     "timestamp" : "REDACTED_TIMESTAMP",
     "level" : 500
@@ -70,8 +70,8 @@ sites/udmi_site_model/out/devices/AHU-1/events_pointset.out
     "extra" : [ ]
   },
   "errors" : [ {
-    "message" : "Timestamp jitter REDACTED_DURATION (REDACTED_TIMESTAMP to REDACTED_TIMESTAMP) exceeds REDACTED_DURATION threshold",
-    "detail" : "events_pointset: Timestamp jitter REDACTED_DURATION (REDACTED_TIMESTAMP to REDACTED_TIMESTAMP) exceeds REDACTED_DURATION threshold REDACTED_ERROR",
+    "message" : "Timestamp skew REDACTED_DURATION (REDACTED_TIMESTAMP to REDACTED_TIMESTAMP) exceeds REDACTED_DURATION threshold",
+    "detail" : "events_pointset: Timestamp skew REDACTED_DURATION (REDACTED_TIMESTAMP to REDACTED_TIMESTAMP) exceeds REDACTED_DURATION threshold REDACTED_ERROR",
     "category" : "validation.device.schema",
     "timestamp" : "REDACTED_TIMESTAMP",
     "level" : 500
@@ -92,15 +92,15 @@ sites/udmi_site_model/out/devices/AHU-1/events_system.out
   "sub_folder" : "system",
   "sub_type" : "events",
   "status" : {
-    "message" : "Timestamp jitter REDACTED_DURATION (REDACTED_TIMESTAMP to REDACTED_TIMESTAMP) exceeds REDACTED_DURATION threshold",
-    "detail" : "events_system: Timestamp jitter REDACTED_DURATION (REDACTED_TIMESTAMP to REDACTED_TIMESTAMP) exceeds REDACTED_DURATION threshold REDACTED_ERROR",
+    "message" : "Timestamp skew REDACTED_DURATION (REDACTED_TIMESTAMP to REDACTED_TIMESTAMP) exceeds REDACTED_DURATION threshold",
+    "detail" : "events_system: Timestamp skew REDACTED_DURATION (REDACTED_TIMESTAMP to REDACTED_TIMESTAMP) exceeds REDACTED_DURATION threshold REDACTED_ERROR",
     "category" : "validation.device.schema",
     "timestamp" : "REDACTED_TIMESTAMP",
     "level" : 500
   },
   "errors" : [ {
-    "message" : "Timestamp jitter REDACTED_DURATION (REDACTED_TIMESTAMP to REDACTED_TIMESTAMP) exceeds REDACTED_DURATION threshold",
-    "detail" : "events_system: Timestamp jitter REDACTED_DURATION (REDACTED_TIMESTAMP to REDACTED_TIMESTAMP) exceeds REDACTED_DURATION threshold REDACTED_ERROR",
+    "message" : "Timestamp skew REDACTED_DURATION (REDACTED_TIMESTAMP to REDACTED_TIMESTAMP) exceeds REDACTED_DURATION threshold",
+    "detail" : "events_system: Timestamp skew REDACTED_DURATION (REDACTED_TIMESTAMP to REDACTED_TIMESTAMP) exceeds REDACTED_DURATION threshold REDACTED_ERROR",
     "category" : "validation.device.schema",
     "timestamp" : "REDACTED_TIMESTAMP",
     "level" : 500
@@ -115,15 +115,15 @@ sites/udmi_site_model/out/devices/AHU-1/state_localnet.out
   "sub_folder" : "localnet",
   "sub_type" : "state",
   "status" : {
-    "message" : "Timestamp jitter REDACTED_DURATION (REDACTED_TIMESTAMP to REDACTED_TIMESTAMP) exceeds REDACTED_DURATION threshold",
-    "detail" : "state_localnet: Timestamp jitter REDACTED_DURATION (REDACTED_TIMESTAMP to REDACTED_TIMESTAMP) exceeds REDACTED_DURATION threshold REDACTED_ERROR",
+    "message" : "Timestamp skew REDACTED_DURATION (REDACTED_TIMESTAMP to REDACTED_TIMESTAMP) exceeds REDACTED_DURATION threshold",
+    "detail" : "state_localnet: Timestamp skew REDACTED_DURATION (REDACTED_TIMESTAMP to REDACTED_TIMESTAMP) exceeds REDACTED_DURATION threshold REDACTED_ERROR",
     "category" : "validation.device.schema",
     "timestamp" : "REDACTED_TIMESTAMP",
     "level" : 500
   },
   "errors" : [ {
-    "message" : "Timestamp jitter REDACTED_DURATION (REDACTED_TIMESTAMP to REDACTED_TIMESTAMP) exceeds REDACTED_DURATION threshold",
-    "detail" : "state_localnet: Timestamp jitter REDACTED_DURATION (REDACTED_TIMESTAMP to REDACTED_TIMESTAMP) exceeds REDACTED_DURATION threshold REDACTED_ERROR",
+    "message" : "Timestamp skew REDACTED_DURATION (REDACTED_TIMESTAMP to REDACTED_TIMESTAMP) exceeds REDACTED_DURATION threshold",
+    "detail" : "state_localnet: Timestamp skew REDACTED_DURATION (REDACTED_TIMESTAMP to REDACTED_TIMESTAMP) exceeds REDACTED_DURATION threshold REDACTED_ERROR",
     "category" : "validation.device.schema",
     "timestamp" : "REDACTED_TIMESTAMP",
     "level" : 500
@@ -139,14 +139,14 @@ sites/udmi_site_model/out/devices/AHU-1/state.out
   "sub_type" : "state",
   "status" : {
     "message" : "Multiple validation errors",
-    "detail" : "Timestamp jitter REDACTED_DURATION (REDACTED_TIMESTAMP to REDACTED_TIMESTAMP) exceeds REDACTED_DURATION threshold; While converting to json node: 1 schema violations found",
+    "detail" : "Timestamp skew REDACTED_DURATION (REDACTED_TIMESTAMP to REDACTED_TIMESTAMP) exceeds REDACTED_DURATION threshold; While converting to json node: 1 schema violations found",
     "category" : "validation.device.multiple",
     "timestamp" : "REDACTED_TIMESTAMP",
     "level" : 500
   },
   "errors" : [ {
-    "message" : "Timestamp jitter REDACTED_DURATION (REDACTED_TIMESTAMP to REDACTED_TIMESTAMP) exceeds REDACTED_DURATION threshold",
-    "detail" : "state_update: Timestamp jitter REDACTED_DURATION (REDACTED_TIMESTAMP to REDACTED_TIMESTAMP) exceeds REDACTED_DURATION threshold REDACTED_ERROR",
+    "message" : "Timestamp skew REDACTED_DURATION (REDACTED_TIMESTAMP to REDACTED_TIMESTAMP) exceeds REDACTED_DURATION threshold",
+    "detail" : "state_update: Timestamp skew REDACTED_DURATION (REDACTED_TIMESTAMP to REDACTED_TIMESTAMP) exceeds REDACTED_DURATION threshold REDACTED_ERROR",
     "category" : "validation.device.schema",
     "timestamp" : "REDACTED_TIMESTAMP",
     "level" : 500
@@ -167,8 +167,8 @@ sites/udmi_site_model/out/devices/AHU-1/state_pointset.out
   "sub_folder" : "pointset",
   "sub_type" : "state",
   "status" : {
-    "message" : "Timestamp jitter REDACTED_DURATION (REDACTED_TIMESTAMP to REDACTED_TIMESTAMP) exceeds REDACTED_DURATION threshold",
-    "detail" : "state_pointset: Timestamp jitter REDACTED_DURATION (REDACTED_TIMESTAMP to REDACTED_TIMESTAMP) exceeds REDACTED_DURATION threshold REDACTED_ERROR",
+    "message" : "Timestamp skew REDACTED_DURATION (REDACTED_TIMESTAMP to REDACTED_TIMESTAMP) exceeds REDACTED_DURATION threshold",
+    "detail" : "state_pointset: Timestamp skew REDACTED_DURATION (REDACTED_TIMESTAMP to REDACTED_TIMESTAMP) exceeds REDACTED_DURATION threshold REDACTED_ERROR",
     "category" : "validation.device.schema",
     "timestamp" : "REDACTED_TIMESTAMP",
     "level" : 500
@@ -178,8 +178,8 @@ sites/udmi_site_model/out/devices/AHU-1/state_pointset.out
     "extra" : [ ]
   },
   "errors" : [ {
-    "message" : "Timestamp jitter REDACTED_DURATION (REDACTED_TIMESTAMP to REDACTED_TIMESTAMP) exceeds REDACTED_DURATION threshold",
-    "detail" : "state_pointset: Timestamp jitter REDACTED_DURATION (REDACTED_TIMESTAMP to REDACTED_TIMESTAMP) exceeds REDACTED_DURATION threshold REDACTED_ERROR",
+    "message" : "Timestamp skew REDACTED_DURATION (REDACTED_TIMESTAMP to REDACTED_TIMESTAMP) exceeds REDACTED_DURATION threshold",
+    "detail" : "state_pointset: Timestamp skew REDACTED_DURATION (REDACTED_TIMESTAMP to REDACTED_TIMESTAMP) exceeds REDACTED_DURATION threshold REDACTED_ERROR",
     "category" : "validation.device.schema",
     "timestamp" : "REDACTED_TIMESTAMP",
     "level" : 500
@@ -194,15 +194,15 @@ sites/udmi_site_model/out/devices/AHU-1/state_system.out
   "sub_folder" : "system",
   "sub_type" : "state",
   "status" : {
-    "message" : "Timestamp jitter REDACTED_DURATION (REDACTED_TIMESTAMP to REDACTED_TIMESTAMP) exceeds REDACTED_DURATION threshold",
-    "detail" : "state_system: Timestamp jitter REDACTED_DURATION (REDACTED_TIMESTAMP to REDACTED_TIMESTAMP) exceeds REDACTED_DURATION threshold REDACTED_ERROR",
+    "message" : "Timestamp skew REDACTED_DURATION (REDACTED_TIMESTAMP to REDACTED_TIMESTAMP) exceeds REDACTED_DURATION threshold",
+    "detail" : "state_system: Timestamp skew REDACTED_DURATION (REDACTED_TIMESTAMP to REDACTED_TIMESTAMP) exceeds REDACTED_DURATION threshold REDACTED_ERROR",
     "category" : "validation.device.schema",
     "timestamp" : "REDACTED_TIMESTAMP",
     "level" : 500
   },
   "errors" : [ {
-    "message" : "Timestamp jitter REDACTED_DURATION (REDACTED_TIMESTAMP to REDACTED_TIMESTAMP) exceeds REDACTED_DURATION threshold",
-    "detail" : "state_system: Timestamp jitter REDACTED_DURATION (REDACTED_TIMESTAMP to REDACTED_TIMESTAMP) exceeds REDACTED_DURATION threshold REDACTED_ERROR",
+    "message" : "Timestamp skew REDACTED_DURATION (REDACTED_TIMESTAMP to REDACTED_TIMESTAMP) exceeds REDACTED_DURATION threshold",
+    "detail" : "state_system: Timestamp skew REDACTED_DURATION (REDACTED_TIMESTAMP to REDACTED_TIMESTAMP) exceeds REDACTED_DURATION threshold REDACTED_ERROR",
     "category" : "validation.device.schema",
     "timestamp" : "REDACTED_TIMESTAMP",
     "level" : 500
@@ -376,13 +376,19 @@ sites/udmi_site_model/out/devices/GAT-123/events_blobset.out
   "sub_folder" : "blobset",
   "sub_type" : "events",
   "status" : {
-    "message" : "Raw string message for GAT-123 events_blobset",
-    "detail" : "events_blobset: Raw string message for GAT-123 events_blobset REDACTED_ERROR",
-    "category" : "validation.device.schema",
+    "message" : "Multiple validation errors",
+    "detail" : "Unknown schema subFolder 'events_blobset' for GAT-123; Raw string message for GAT-123 events_blobset",
+    "category" : "validation.device.multiple",
     "timestamp" : "REDACTED_TIMESTAMP",
     "level" : 500
   },
   "errors" : [ {
+    "message" : "Unknown schema subFolder 'events_blobset' for GAT-123",
+    "detail" : "events_blobset: Unknown schema subFolder 'events_blobset' for GAT-123 REDACTED_ERROR",
+    "category" : "validation.device.schema",
+    "timestamp" : "REDACTED_TIMESTAMP",
+    "level" : 500
+  }, {
     "message" : "Raw string message for GAT-123 events_blobset",
     "detail" : "events_blobset: Raw string message for GAT-123 events_blobset REDACTED_ERROR",
     "category" : "validation.device.schema",