Merge branch 'release/v0.14.0'

Hurence · Jul 31, 2018 · 45486f1 · 45486f1
2 parents 8247c6c + 9dfedca
commit 45486f1
Show file tree

Hide file tree

Showing 507 changed files with 11,630 additions and 19,486 deletions.
diff --git a/README.rst b/README.rst
@@ -53,7 +53,7 @@ to build from the source just clone source and package with maven
     cd logisland
     mvn clean install
 
-the final package is available at `logisland-assembly/target/logisland-0.13.0-bin-hdp2.5.tar.gz`
+the final package is available at `logisland-assembly/target/logisland-0.14.0-bin-hdp2.5.tar.gz`
 
 You can also download the `latest release build <https://github.com/Hurence/logisland/releases>`_
 
@@ -76,9 +76,9 @@ Alternatively you can deploy **logisland** on any linux server from which Kafka
     curl -s http://d3kbcqa49mib13.cloudfront.net/spark-2.1.0-bin-hadoop2.7.tgz | tar -xz -C /usr/local/
     export SPARK_HOME=/usr/local/spark-2.1.0-bin-hadoop2.7
 
-    # install Logisland 0.13.0
-    curl -s https://github.com/Hurence/logisland/releases/download/v0.10.0/logisland-0.13.0-bin-hdp2.5.tar.gz  | tar -xz -C /usr/local/
-    cd /usr/local/logisland-0.13.0
+    # install Logisland 0.14.0
+    curl -s https://github.com/Hurence/logisland/releases/download/v0.10.0/logisland-0.14.0-bin-hdp2.5.tar.gz  | tar -xz -C /usr/local/
+    cd /usr/local/logisland-0.14.0
 
     # launch a logisland job
     bin/logisland.sh --conf conf/index-apache-logs.yml
@@ -107,9 +107,9 @@ Launching logisland streaming apps is just easy as unarchiving logisland distrib
 
 .. code-block:: sh
 
-    # install Logisland 0.13.0
-    curl -s https://github.com/Hurence/logisland/releases/download/v0.10.0/logisland-0.13.0-bin-hdp2.5.tar.gz  | tar -xz -C /usr/local/
-    cd /usr/local/logisland-0.13.0
+    # install Logisland 0.14.0
+    curl -s https://github.com/Hurence/logisland/releases/download/v0.10.0/logisland-0.14.0-bin-hdp2.5.tar.gz  | tar -xz -C /usr/local/
+    cd /usr/local/logisland-0.14.0
     bin/logisland.sh --conf conf/index-apache-logs.yml
 
 
@@ -130,7 +130,7 @@ The first part is the `ProcessingEngine` configuration (here a Spark streaming e
 
 .. code-block:: yaml
 
-    version: 0.13.0
+    version: 0.14.0
     documentation: LogIsland job config file
     engine:
       component: com.hurence.logisland.engine.spark.KafkaStreamProcessingEngine

diff --git a/launch-tuto.sh b/launch-tuto.sh
@@ -1,4 +1,4 @@
 #!/bin/bash
 
-logisland-assembly/target/logisland-0.13.0-bin-hdp2.5/logisland-0.13.0/bin/logisland.sh \
+logisland-assembly/target/logisland-0.14.0-bin-hdp2.5/logisland-0.14.0/bin/logisland.sh \
     --conf logisland-framework/logisland-resources/src/main/resources/conf/$1
diff --git a/logisland-api/pom.xml b/logisland-api/pom.xml
@@ -22,7 +22,7 @@
     <parent>
         <groupId>com.hurence.logisland</groupId>
         <artifactId>logisland</artifactId>
-        <version>0.13.0</version>
+        <version>0.14.0</version>
     </parent>
     <artifactId>logisland-api</artifactId>
     <packaging>jar</packaging>

diff --git a/logisland-api/src/main/java/com/hurence/logisland/component/AbstractConfiguredComponent.java b/logisland-api/src/main/java/com/hurence/logisland/component/AbstractConfiguredComponent.java
@@ -159,6 +159,8 @@ public Map<PropertyDescriptor, String> getProperties() {
 
     }
 
+
+
     @Override
     public String getProperty(final PropertyDescriptor property) {
         return properties.get(property);

diff --git a/logisland-api/src/main/java/com/hurence/logisland/component/AbstractPropertyValue.java b/logisland-api/src/main/java/com/hurence/logisland/component/AbstractPropertyValue.java
@@ -1,5 +1,5 @@
 /**
- * Copyright (C) 2016 Hurence (bailet.thomas@gmail.com)
+ * Copyright (C) 2016 Hurence (support@hurence.com)
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -87,8 +87,7 @@ public boolean isSet() {
     @Override
     public Record asRecord() {
         return (getRawValue() == null) ? null : new StandardRecord()
-                .setStringField(FieldDictionary.RECORD_VALUE,getRawValue().trim())
-                .setStringField(FieldDictionary.RECORD_RAW_VALUE,getRawValue().trim());
+                .setStringField(FieldDictionary.RECORD_VALUE,getRawValue().trim());
     }
 
     @Override

diff --git a/logisland-api/src/main/java/com/hurence/logisland/component/ComponentContext.java b/logisland-api/src/main/java/com/hurence/logisland/component/ComponentContext.java
@@ -81,4 +81,6 @@ public interface ComponentContext extends ConfiguredComponent {
      * @return the configured name of this processor
      */
     String getName();
+
+
 }
diff --git a/...-api/src/main/java/com/hurence/logisland/component/DecoratedInterpretedPropertyValue.java b/...-api/src/main/java/com/hurence/logisland/component/DecoratedInterpretedPropertyValue.java
@@ -1,5 +1,5 @@
 /**
- * Copyright (C) 2016 Hurence (bailet.thomas@gmail.com)
+ * Copyright (C) 2016 Hurence (support@hurence.com)
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

diff --git a/logisland-api/src/main/java/com/hurence/logisland/component/InterpretedPropertyValue.java b/logisland-api/src/main/java/com/hurence/logisland/component/InterpretedPropertyValue.java
@@ -1,5 +1,5 @@
 /**
- * Copyright (C) 2016 Hurence (bailet.thomas@gmail.com)
+ * Copyright (C) 2016 Hurence (support@hurence.com)
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

diff --git a/logisland-api/src/main/java/com/hurence/logisland/component/PropertyValueFactory.java b/logisland-api/src/main/java/com/hurence/logisland/component/PropertyValueFactory.java
@@ -1,5 +1,5 @@
 /**
- * Copyright (C) 2016 Hurence (bailet.thomas@gmail.com)
+ * Copyright (C) 2016 Hurence (support@hurence.com)
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

diff --git a/logisland-api/src/main/java/com/hurence/logisland/config/DefaultConfigValues.java b/logisland-api/src/main/java/com/hurence/logisland/config/DefaultConfigValues.java
@@ -1,12 +1,12 @@
 /**
  * Copyright (C) 2016 Hurence (support@hurence.com)
- * <p>
+ *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -26,7 +26,6 @@ public enum DefaultConfigValues {
     KAFKA_BROKERS("sandbox:9092"),
     ZK_QUORUM("sandbox:2181"),
     SOLR_CONNECTION("http://sandbox:8983/solr"),
-    LOGISLAND_AGENT_HOST("sandbox:8008"),
     MQTT_BROKER_URL("tcp://sandbox:1883");
 
 

diff --git a/logisland-api/src/main/java/com/hurence/logisland/engine/EngineContext.java b/logisland-api/src/main/java/com/hurence/logisland/engine/EngineContext.java
@@ -51,7 +51,9 @@ public interface EngineContext extends ComponentContext {
     /**
      * add a ControllerServiceConfiguration
      *
-     * @param config  to add
+     * @param config to add
      */
     void addControllerServiceConfiguration(ControllerServiceConfiguration config);
+
+
 }
diff --git a/logisland-api/src/main/java/com/hurence/logisland/engine/MockProcessingEngine.java b/logisland-api/src/main/java/com/hurence/logisland/engine/MockProcessingEngine.java
@@ -55,4 +55,13 @@ public void shutdown(EngineContext engineContext) {
         logger.info("engine shutdown");
     }
 
+    @Override
+    public void awaitTermination(EngineContext engineContext) {
+    }
+
+    @Override
+    public void reset(EngineContext engineContext) {
+        engineContext.getStreamContexts().clear();
+        engineContext.getControllerServiceConfigurations().clear();
+    }
 }
diff --git a/logisland-api/src/main/java/com/hurence/logisland/engine/ProcessingEngine.java b/logisland-api/src/main/java/com/hurence/logisland/engine/ProcessingEngine.java
@@ -31,8 +31,21 @@ public interface ProcessingEngine extends ConfigurableComponent {
 
     /**
      * shutdown the engine with a context
+     *
      * @param engineContext
      */
     void shutdown(EngineContext engineContext);
 
+    /**
+     * Await for termination.
+     * @param engineContext
+     */
+    void awaitTermination(EngineContext engineContext);
+
+    /**
+     * Reset the engine by stopping the streaming context.
+     * @param engineContext
+     */
+    void reset(EngineContext engineContext);
+
 }
diff --git a/logisland-api/src/main/java/com/hurence/logisland/engine/StandardEngineContext.java b/logisland-api/src/main/java/com/hurence/logisland/engine/StandardEngineContext.java
@@ -91,4 +91,5 @@ public Collection<ControllerServiceConfiguration> getControllerServiceConfigurat
     public void addControllerServiceConfiguration(ControllerServiceConfiguration config) {
         controllerServiceConfigurations.add(config);
     }
+
 }
diff --git a/logisland-api/src/main/java/com/hurence/logisland/expressionlanguage/InterpreterEngine.java b/logisland-api/src/main/java/com/hurence/logisland/expressionlanguage/InterpreterEngine.java
@@ -1,5 +1,5 @@
 /**
- * Copyright (C) 2016 Hurence (bailet.thomas@gmail.com)
+ * Copyright (C) 2016 Hurence (support@hurence.com)
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

diff --git a/...pi/src/main/java/com/hurence/logisland/expressionlanguage/InterpreterEngineException.java b/...pi/src/main/java/com/hurence/logisland/expressionlanguage/InterpreterEngineException.java
@@ -1,5 +1,5 @@
 /**
- * Copyright (C) 2016 Hurence (bailet.thomas@gmail.com)
+ * Copyright (C) 2016 Hurence (support@hurence.com)
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

diff --git a/...-api/src/main/java/com/hurence/logisland/expressionlanguage/InterpreterEngineFactory.java b/...-api/src/main/java/com/hurence/logisland/expressionlanguage/InterpreterEngineFactory.java
@@ -1,5 +1,5 @@
 /**
- * Copyright (C) 2016 Hurence (bailet.thomas@gmail.com)
+ * Copyright (C) 2016 Hurence (support@hurence.com)
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

diff --git a/logisland-api/src/main/java/com/hurence/logisland/expressionlanguage/Jsr223Context.java b/logisland-api/src/main/java/com/hurence/logisland/expressionlanguage/Jsr223Context.java
@@ -1,5 +1,5 @@
 /**
- * Copyright (C) 2016 Hurence (bailet.thomas@gmail.com)
+ * Copyright (C) 2016 Hurence (support@hurence.com)
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

diff --git a/...d-api/src/main/java/com/hurence/logisland/expressionlanguage/Jsr223InterpreterEngine.java b/...d-api/src/main/java/com/hurence/logisland/expressionlanguage/Jsr223InterpreterEngine.java
@@ -1,12 +1,12 @@
 /**
- * Copyright (C) 2016 Hurence (bailet.thomas@gmail.com)
- * <p>
+ * Copyright (C) 2016 Hurence (support@hurence.com)
+ *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

diff --git a/logisland-api/src/main/java/com/hurence/logisland/processor/ProcessContext.java b/logisland-api/src/main/java/com/hurence/logisland/processor/ProcessContext.java
@@ -20,6 +20,8 @@
 import com.hurence.logisland.component.InitializationException;
 import com.hurence.logisland.controller.ControllerServiceLookup;
 
+import java.io.Closeable;
+
 public interface ProcessContext extends ComponentContext {
 
     /**

diff --git a/logisland-api/src/main/java/com/hurence/logisland/processor/StandardProcessContext.java b/logisland-api/src/main/java/com/hurence/logisland/processor/StandardProcessContext.java
@@ -19,6 +19,8 @@
 import com.hurence.logisland.component.*;
 import com.hurence.logisland.controller.ControllerServiceLookup;
 
+import java.io.IOException;
+
 public class StandardProcessContext extends AbstractConfiguredComponent implements ProcessContext {
 
     private ControllerServiceLookup controllerServiceLookup;
@@ -64,4 +66,5 @@ public PropertyValue newPropertyValue(final String rawValue) {
     public void verifyModifiable() throws IllegalStateException {
 
     }
+
 }
diff --git a/logisland-api/src/main/java/com/hurence/logisland/record/Field.java b/logisland-api/src/main/java/com/hurence/logisland/record/Field.java
@@ -1,12 +1,12 @@
 /**
  * Copyright (C) 2016 Hurence (support@hurence.com)
- * <p>
+ *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

diff --git a/logisland-api/src/main/java/com/hurence/logisland/record/FieldDictionary.java b/logisland-api/src/main/java/com/hurence/logisland/record/FieldDictionary.java
@@ -1,12 +1,12 @@
 /**
  * Copyright (C) 2016 Hurence (support@hurence.com)
- * <p>
+ *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -28,12 +28,12 @@ public class FieldDictionary {
     public static String RECORD_DAYTIME = "record_daytime";
     public static String RECORD_KEY = "record_key";
     public static String RECORD_VALUE = "record_value";
-    public static String RECORD_RAW_KEY = "record_raw_key";
-    public static String RECORD_RAW_VALUE = "record_raw_value";
     public static String RECORD_NAME = "record_name";
     public static String PROCESSOR_NAME = "processor_name";
     public static String RECORD_POSITION = "record_position";
     public static String RECORD_BODY = "record_body";
+    public static String RECORD_COUNT = "record_count";
+    public static String RECORD_LAST_UPDATE_TIME = "record_last_update_time";
 
 
     public static String RECORD_POSITION_LATITUDE = "record_position_latitude";
@@ -59,8 +59,6 @@ public static List<String> asList() {
                 RECORD_DAYTIME,
                 RECORD_KEY,
                 RECORD_VALUE,
-                RECORD_RAW_KEY,
-                RECORD_RAW_VALUE,
                 PROCESSOR_NAME,
                 RECORD_POSITION,
                 RECORD_BODY,
@@ -72,7 +70,9 @@ public static List<String> asList() {
                 RECORD_POSITION_SATELLITES,
                 RECORD_POSITION_SPEED,
                 RECORD_POSITION_STATUS,
-                RECORD_POSITION_TIMESTAMP
+                RECORD_POSITION_TIMESTAMP,
+                RECORD_COUNT,
+                RECORD_LAST_UPDATE_TIME
         );
     }
 }
diff --git a/logisland-api/src/main/java/com/hurence/logisland/record/Position.java b/logisland-api/src/main/java/com/hurence/logisland/record/Position.java
@@ -1,12 +1,11 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
+/**
+ * Copyright (C) 2016 Hurence (support@hurence.com)
  *
- *     http://www.apache.org/licenses/LICENSE-2.0
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,

diff --git a/logisland-api/src/main/java/com/hurence/logisland/record/RecordDictionary.java b/logisland-api/src/main/java/com/hurence/logisland/record/RecordDictionary.java
@@ -22,4 +22,9 @@ public class RecordDictionary {
     public static String METRIC = "metric";
     public static String EVENT = "event";
     public static String MESSAGE = "message";
+    public static String ERROR = "error";
+    public static String TAG = "tag";
+    public static String COMPUTED_TAG = "computed_tag";
+    public static String THRESHOLD = "threshold";
+    public static String ALERT = "alert";
 }
diff --git a/logisland-api/src/main/java/com/hurence/logisland/record/StandardRecord.java b/logisland-api/src/main/java/com/hurence/logisland/record/StandardRecord.java
@@ -107,8 +107,8 @@ public int hashCode() {
 
     @Override
     public Position getPosition() {
-        if(hasPosition())
-            return (Position)getField(FieldDictionary.RECORD_POSITION).asRecord();
+        if (hasPosition())
+            return (Position) getField(FieldDictionary.RECORD_POSITION).asRecord();
         else return null;
     }
 
@@ -249,7 +249,12 @@ public Record setStringField(String fieldName, String value) {
      */
     @Override
     public Field removeField(String fieldName) {
-        return fields.remove(fieldName);
+        if (fieldName.equals(FieldDictionary.RECORD_TIME)) {
+            logger.debug("trying to remove record_time field. we won't let you do that !!");
+            return fields.get(FieldDictionary.RECORD_TIME);
+        } else {
+            return fields.remove(fieldName);
+        }
     }
 
     /**
-Original file line number
+Diff line change
@@ Expand Up / @@ -159,6 +159,8 @@ public Map<PropertyDescriptor, String> getProperties() { @@
         }
         @Override
         public String getProperty(final PropertyDescriptor property) {
             return properties.get(property);
@@ Expand Down @@