Added jars list, audit-kafka-unikernel file list, and jars that were …

…being taken from neo4j

docs/jar-list.csv

@@ -0,0 +1,42 @@
+Jar Name,Source Link,License Link,License
+akuma-1.9.jar,http://akuma.kohsuke.org/,,The MIT License
+avro-tools-1.8.1.jar,https://avro.apache.org/,,Apache License
+commons-codec-1.10.jar,http://commons.apache.org/proper/commons-codec/,,Apache License
+externalsortinginjava-0.1.9.jar,https://github.com/lemire/externalsortinginjava,https://github.com/lemire/externalsortinginjava/blob/master/LICENSE.txt,
+h2-1.4.187.jar,http://www.h2database.com/html/main.html,http://www.h2database.com/html/license.html,
+hamcrest-core-1.3.jar,https://github.com/hamcrest/JavaHamcrest/,https://github.com/hamcrest/JavaHamcrest/blob/master/LICENSE.txt,BSD License
+httpclient-4.2.6.jar,http://hc.apache.org/,,Apache License
+httpclient-cache-4.2.6.jar,http://hc.apache.org/,,Apache License
+httpcore-4.2.5.jar,http://hc.apache.org/,,Apache License
+iris-0.60.jar,http://www.iris-reasoner.org/,http://www.iris-reasoner.org/license/,LGPL License
+iris-parser-0.60.jar,http://www.iris-reasoner.org/,http://www.iris-reasoner.org/license/,LGPL License
+jackson-annotations-2.3.0.jar,https://github.com/FasterXML/jackson,https://github.com/FasterXML/jackson#paperwork,
+jackson-core-2.3.3.jar,https://github.com/FasterXML/jackson,https://github.com/FasterXML/jackson#paperwork,
+jackson-databind-2.3.3.jar,https://github.com/FasterXML/jackson,https://github.com/FasterXML/jackson#paperwork,
+je-7.3.7.jar,https://www.oracle.com/database/technologies/related/berkeleydb-downloads.html,https://www.oracle.com/technetwork/database/berkeleydb/downloads/licensing-098979.html,
+jedis-2.9.0.jar,https://github.com/xetorthio/jedis/releases,https://github.com/xetorthio/jedis/blob/master/LICENSE.txt,MIT License
+jena-arq-3.0.0.jar,https://jena.apache.org/,,Apache License 2.0
+jena-base-3.0.0.jar,https://jena.apache.org/,,Apache License 2.0
+jena-core-3.0.0.jar,https://jena.apache.org/,,Apache License 2.0
+jena-iri-3.0.0.jar,https://jena.apache.org/,,Apache License 2.0
+jena-shaded-guava-3.0.0.jar,https://jena.apache.org/,,Apache License 2.0
+jena-tdb-3.0.0.jar,https://jena.apache.org/,,Apache License 2.0
+jgrapht-core-0.9.2.jar,https://jgrapht.org/,https://github.com/jgrapht/jgrapht/blob/master/license-EPL.txt,
+jna-4.1.0.jar,https://github.com/java-native-access/jna,https://github.com/java-native-access/jna/blob/master/LICENSE,
+jsonld-java-0.5.1.jar,https://github.com/jsonld-java/jsonld-java,https://github.com/jsonld-java/jsonld-java/blob/master/LICENCE,
+junit-4.12.jar,https://junit.org/junit5/,https://github.com/junit-team/junit5/blob/master/LICENSE.md,
+kafkaclients-CDM20.jar,http://kafka.apache.org/,,Apache License 2.0
+leveldbjni-all-1.8.jar,https://github.com/google/leveldb/releases,https://github.com/google/leveldb/blob/master/LICENSE,
+mysql-connector-java-5.1.38-bin.jar,https://downloads.mysql.com/archives/c-j/,https://github.com/mysql/mysql-connector-j/blob/release/8.0/LICENSE,
+OoaBFilter_commit_31792eb.jar,https://github.com/jmhodges/opposite_of_a_bloom_filter/tree/31792eb58e55d67e7f8d7373f1b055a990241cb5,https://github.com/jmhodges/opposite_of_a_bloom_filter/blob/master/LICENSE,MIT License
+org.json-20120521.jar,https://github.com/stleary/JSON-java,https://github.com/stleary/JSON-java/blob/master/LICENSE,
+postgresql-9.4.1212.jar,https://jdbc.postgresql.org/,https://github.com/pgjdbc/pgjdbc/blob/master/LICENSE,
+restfb-1.6.14.jar,https://restfb.com/,,MIT License
+xercesImpl-2.11.0.jar,https://xerces.apache.org/xerces2-j/,,Apache License 2.0
+xml-apis-1.4.01.jar,http://xerces.apache.org/xml-commons/index.html,http://xerces.apache.org/xml-commons/licenses.html,
+commons-io-2.6.jar,https://commons.apache.org/proper/commons-io/download_io.cgi,,Apache License
+commons-lang-2.6.jar,http://commons.apache.org/proper/commons-lang/download_lang.cgi,,Apache License
+guava-16.0.1.jar,https://github.com/google/guava/wiki/Release16,https://github.com/google/guava/blob/master/COPYING,Apache License 2.0
+jackson-mapper-asl-1.9.13.jar,https://github.com/FasterXML/jackson,https://github.com/FasterXML/jackson#paperwork,
+slf4j-api-1.7.30.jar,http://www.slf4j.org/,http://www.slf4j.org/license.html,MIT License
+neo4j-community-3.4.4-unix.tar.gz,https://neo4j.com/,https://neo4j.com/licensing/,

docs/spade-audit-kafka-unikernel-files

@@ -0,0 +1,217 @@
+src/spade/core/MatrixFilter.java
+src/spade/core/AbstractVertex.java
+src/spade/core/AbstractStorage.java
+src/spade/core/Unikernel.java
+src/spade/core/Graph.java
+src/spade/core/Vertex.java
+src/spade/core/AbstractAnalyzer.java
+src/spade/core/AbstractFilter.java
+src/spade/core/Kernel.java
+src/spade/core/BloomFilter.java
+src/spade/core/AbstractEdge.java
+src/spade/core/Buffer.java
+src/spade/core/Settings.java
+src/spade/core/AbstractReporter.java
+src/spade/core/Edge.java
+src/spade/core/AbstractSketch.java
+src/spade/core/SPADEQuery.java
+src/spade/core/AbstractTransformer.java
+src/spade/reporter/spadeAuditBridge.c
+src/spade/reporter/audit/LinuxPathResolver.java
+src/spade/reporter/audit/MalformedAuditDataException.java
+src/spade/reporter/audit/AuditEventReader.java
+src/spade/reporter/audit/kernel-modules/Makefile
+src/spade/reporter/audit/kernel-modules/netio_controller.c
+src/spade/reporter/audit/kernel-modules/globals.h
+src/spade/reporter/audit/kernel-modules/netio.c
+src/spade/reporter/audit/SYSCALL.java
+src/spade/reporter/audit/PathRecord.java
+src/spade/reporter/audit/VertexIdentifier.java
+src/spade/reporter/audit/artifact/UnknownIdentifier.java
+src/spade/reporter/audit/artifact/MemoryIdentifier.java
+src/spade/reporter/audit/artifact/BlockDeviceIdentifier.java
+src/spade/reporter/audit/artifact/ArtifactState.java
+src/spade/reporter/audit/artifact/UnnamedPipeIdentifier.java
+src/spade/reporter/audit/artifact/FileIdentifier.java
+src/spade/reporter/audit/artifact/LinkIdentifier.java
+src/spade/reporter/audit/artifact/FdPairIdentifier.java
+src/spade/reporter/audit/artifact/PathIdentifier.java
+src/spade/reporter/audit/artifact/ArtifactManager.java
+src/spade/reporter/audit/artifact/NetworkSocketIdentifier.java
+src/spade/reporter/audit/artifact/UnnamedUnixSocketPairIdentifier.java
+src/spade/reporter/audit/artifact/DirectoryIdentifier.java
+src/spade/reporter/audit/artifact/UnnamedNetworkSocketPairIdentifier.java
+src/spade/reporter/audit/artifact/CharacterDeviceIdentifier.java
+src/spade/reporter/audit/artifact/ArtifactIdentifier.java
+src/spade/reporter/audit/artifact/UnixSocketIdentifier.java
+src/spade/reporter/audit/artifact/NamedPipeIdentifier.java
+src/spade/reporter/audit/artifact/ArtifactConfig.java
+src/spade/reporter/audit/OPMConstants.java
+src/spade/reporter/audit/Globals.java
+src/spade/reporter/audit/process/FileDescriptor.java
+src/spade/reporter/audit/process/ProcessIdentifier.java
+src/spade/reporter/audit/process/ProcessStateManager.java
+src/spade/reporter/audit/process/ProcessManager.java
+src/spade/reporter/audit/process/NamespaceIdentifier.java
+src/spade/reporter/audit/process/UnitIdentifier.java
+src/spade/reporter/audit/process/ProcessWithAgentState.java
+src/spade/reporter/audit/process/ProcessWithAgentManager.java
+src/spade/reporter/audit/process/AgentIdentifier.java
+src/spade/reporter/audit/process/ProcessWithoutAgentManager.java
+src/spade/reporter/audit/process/ProcessUnitState.java
+src/spade/reporter/Audit.java
+src/spade/edge/opm/WasGeneratedBy.java
+src/spade/edge/opm/WasControlledBy.java
+src/spade/edge/opm/Used.java
+src/spade/edge/opm/WasDerivedFrom.java
+src/spade/edge/opm/WasTriggeredBy.java
+src/spade/storage/kafka/JsonFileWriter.java
+src/spade/storage/kafka/ServerWriter.java
+src/spade/storage/kafka/FileWriter.java
+src/spade/storage/kafka/Vertex.java
+src/spade/storage/kafka/DataWriter.java
+src/spade/storage/kafka/GraphElement.java
+src/spade/storage/kafka/Edge.java
+src/spade/storage/Kafka.java
+src/spade/vertex/opm/Artifact.java
+src/spade/vertex/opm/Process.java
+src/spade/vertex/opm/Agent.java
+src/spade/filter/FinalCommitFilter.java
+src/spade/query/quickgrail/types/TypedValue.java
+src/spade/query/quickgrail/types/Type.java
+src/spade/query/quickgrail/types/TypeID.java
+src/spade/query/quickgrail/core/GraphDescription.java
+src/spade/query/quickgrail/core/GraphStats.java
+src/spade/query/quickgrail/core/QuickGrailQueryResolver.java
+src/spade/query/quickgrail/core/QueryInstructionExecutor.java
+src/spade/query/quickgrail/core/QueriedEdge.java
+src/spade/query/quickgrail/core/Program.java
+src/spade/query/quickgrail/core/AbstractQueryEnvironment.java
+src/spade/query/quickgrail/parser/ParseOperation.java
+src/spade/query/quickgrail/parser/ParseName.java
+src/spade/query/quickgrail/parser/ParseAssignment.java
+src/spade/query/quickgrail/parser/ParseCommand.java
+src/spade/query/quickgrail/parser/ParseStatement.java
+src/spade/query/quickgrail/parser/ParseTreeNode.java
+src/spade/query/quickgrail/parser/ParseString.java
+src/spade/query/quickgrail/parser/ParseLiteral.java
+src/spade/query/quickgrail/parser/ParseVariable.java
+src/spade/query/quickgrail/parser/ParseProgram.java
+src/spade/query/quickgrail/parser/ParseExpression.java
+src/spade/query/quickgrail/instruction/CreateEmptyGraph.java
+src/spade/query/quickgrail/instruction/GetSubgraph.java
+src/spade/query/quickgrail/instruction/GetAdjacentVertex.java
+src/spade/query/quickgrail/instruction/InsertLiteralVertex.java
+src/spade/query/quickgrail/instruction/EraseSymbols.java
+src/spade/query/quickgrail/instruction/OverwriteGraphMetadata.java
+src/spade/query/quickgrail/instruction/DistinctifyGraph.java
+src/spade/query/quickgrail/instruction/List.java
+src/spade/query/quickgrail/instruction/SubtractGraph.java
+src/spade/query/quickgrail/instruction/StatGraph.java
+src/spade/query/quickgrail/instruction/GetLink.java
+src/spade/query/quickgrail/instruction/CollapseEdge.java
+src/spade/query/quickgrail/instruction/DescribeGraph.java
+src/spade/query/quickgrail/instruction/GetLineage.java
+src/spade/query/quickgrail/instruction/PrintPredicate.java
+src/spade/query/quickgrail/instruction/GetEdgeEndpoint.java
+src/spade/query/quickgrail/instruction/UnionGraph.java
+src/spade/query/quickgrail/instruction/GetEdge.java
+src/spade/query/quickgrail/instruction/InsertLiteralEdge.java
+src/spade/query/quickgrail/instruction/GetPath.java
+src/spade/query/quickgrail/instruction/EvaluateQuery.java
+src/spade/query/quickgrail/instruction/LimitGraph.java
+src/spade/query/quickgrail/instruction/Instruction.java
+src/spade/query/quickgrail/instruction/ExportGraph.java
+src/spade/query/quickgrail/instruction/IntersectGraph.java
+src/spade/query/quickgrail/instruction/GetVertex.java
+src/spade/query/quickgrail/instruction/SetGraphMetadata.java
+src/spade/query/quickgrail/instruction/CreateEmptyGraphMetadata.java
+src/spade/query/quickgrail/instruction/GetShortestPath.java
+src/spade/query/quickgrail/entities/Graph.java
+src/spade/query/quickgrail/entities/EntityType.java
+src/spade/query/quickgrail/entities/GraphPredicate.java
+src/spade/query/quickgrail/entities/GraphMetadata.java
+src/spade/query/quickgrail/entities/Entity.java
+src/spade/query/quickgrail/utility/ResultTable.java
+src/spade/query/quickgrail/utility/Schema.java
+src/spade/query/quickgrail/utility/QuickGrailPredicateTree.java
+src/spade/query/quickgrail/utility/TreeStringSerializable.java
+src/spade/query/scaffold/Scaffold.java
+src/spade/query/scaffold/BerkeleyDB.java
+src/spade/query/scaffold/ScaffoldFactory.java
+src/spade/client/QueryMetaData.java
+src/spade/utility/HelperFunctions.java
+src/spade/utility/Series.java
+src/spade/utility/Serializable2ByteArrayConverter.java
+src/spade/utility/FileUtility.java
+src/spade/utility/LogManager.java
+src/spade/utility/Execute.java
+src/spade/utility/DoublyLinkedList.java
+src/spade/utility/Converter.java
+src/spade/utility/map/external/ExternalMap.java
+src/spade/utility/map/external/ExternalMapManager.java
+src/spade/utility/map/external/cache/LRUCacheArgument.java
+src/spade/utility/map/external/cache/CacheArgument.java
+src/spade/utility/map/external/cache/LRUCacheManager.java
+src/spade/utility/map/external/cache/Cache.java
+src/spade/utility/map/external/cache/LRUCache.java
+src/spade/utility/map/external/cache/CacheProfile.java
+src/spade/utility/map/external/cache/ProfiledCache.java
+src/spade/utility/map/external/cache/CacheManager.java
+src/spade/utility/map/external/cache/CacheName.java
+src/spade/utility/map/external/ExternalMapArgument.java
+src/spade/utility/map/external/ExternalMapTest.java
+src/spade/utility/map/external/screen/ScreenProfile.java
+src/spade/utility/map/external/screen/ProfiledScreen.java
+src/spade/utility/map/external/screen/ScreenManager.java
+src/spade/utility/map/external/screen/ScreenArgument.java
+src/spade/utility/map/external/screen/BloomFilterManager.java
+src/spade/utility/map/external/screen/ScreenName.java
+src/spade/utility/map/external/screen/BloomFilterScreen.java
+src/spade/utility/map/external/screen/BloomFilterArgument.java
+src/spade/utility/map/external/screen/Screen.java
+src/spade/utility/map/external/store/ProfiledStore.java
+src/spade/utility/map/external/store/StoreName.java
+src/spade/utility/map/external/store/DBStore.java
+src/spade/utility/map/external/store/db/DatabaseManager.java
+src/spade/utility/map/external/store/db/leveldb/LevelDBManager.java
+src/spade/utility/map/external/store/db/leveldb/LevelDBHandle.java
+src/spade/utility/map/external/store/db/leveldb/LevelDBArgument.java
+src/spade/utility/map/external/store/db/DatabaseHandle.java
+src/spade/utility/map/external/store/db/DatabaseArgument.java
+src/spade/utility/map/external/store/db/berkeleydb/BerkeleyDBArgument.java
+src/spade/utility/map/external/store/db/berkeleydb/BerkeleyDBHandle.java
+src/spade/utility/map/external/store/db/berkeleydb/BerkeleyDBEnvironmentHandle.java
+src/spade/utility/map/external/store/db/berkeleydb/BerkeleyDBManager.java
+src/spade/utility/map/external/store/StoreManager.java
+src/spade/utility/map/external/store/StoreArgument.java
+src/spade/utility/map/external/store/Store.java
+src/spade/utility/map/external/store/StoreProfile.java
+src/spade/utility/profile/TimeProfile.java
+src/spade/utility/profile/Intervaler.java
+src/spade/utility/profile/ReportingArgument.java
+src/spade/utility/HostInfo.java
+src/spade/utility/Result.java
+cfg/spade.core.Unikernel.config
+cfg/spade.reporter.audit.process.ProcessManager.config
+cfg/spade.reporter.Audit.config
+cfg/spade.storage.Kafka.config
+cfg/spade.core.AbstractStorage.config
+cfg/spade.core.AbstractAnalyzer.config
+cfg/spade.storage.Kafka.avsc
+cfg/spade.storage.kafka.ServerWriter.config
+cfg/spade.core.Kernel.config
+cfg/spade.reporter.audit.artifact.ArtifactManager.config
+cfg/spade.reporter.audit.AuditEventReader.config
+lib/slf4j-api-1.7.30.jar
+lib/je-7.3.7.jar
+lib/jackson-core-asl-1.9.13.jar
+lib/commons-io-2.6.jar
+lib/org.json-20120521.jar
+lib/commons-codec-1.10.jar
+lib/kafka-clients-2.5.0.jar
+lib/commons-lang-2.6.jar
+lib/avro-1.8.1.jar
+lib/leveldbjni-all-1.8.jar
+lib/jackson-mapper-asl-1.9.13.jar
+lib/guava-16.0.1.jar

src/spade/core/Unikernel.java

@@ -39,6 +39,7 @@ License, or (at your option) any later version.
 public class Unikernel{
 
 	private static volatile boolean shutdown = false;
+	private static volatile boolean executedExitFunction = false;
 
 	private static void setupSPADELogging() throws RuntimeException{
 		System.setProperty("java.util.logging.manager", spade.utility.LogManager.class.getName());
@@ -268,12 +269,7 @@ public void run(){
 				}
 				shutdown = true;
 				// Wait for storage thread to exit after the buffer is flushed
-				try{
-					storageThread.join();
-				}catch(Throwable t0){
-					logger.log(Level.SEVERE, "Failed to successfully wrap-up storage thread", t0);
-				}
-				LogManager.shutdownReset();
+				waitForThreadAndThenExit(logger, storageThread);
 			}
 		});
 
@@ -293,21 +289,24 @@ public void run(){
 
 			// Do cleanup of storage before exiting
 			shutdown = true;
-			try{
-				storageThread.join();
-			}catch(Throwable t0){
-				logger.log(Level.SEVERE, "Failed to successfully wrap-up storage thread", t0);
-			}
-
+			waitForThreadAndThenExit(logger, storageThread);
 			throw new RuntimeException(errorMessage, e);
 		}
 
 		// Continue running until the storage thread is running
-		try{
-			storageThread.join();
-		}catch(Throwable t0){
-			logger.log(Level.SEVERE, "Failed to successfully wrap-up storage thread", t0);
+		waitForThreadAndThenExit(logger, storageThread);
+	}
+
+	// Cleanup function
+	private synchronized static void waitForThreadAndThenExit(final Logger logger, final Thread thread){
+		if(!executedExitFunction){
+			try{
+				thread.join();
+			}catch(Throwable t0){
+				logger.log(Level.SEVERE, "Failed to successfully wrap-up storage thread", t0);
+			}
+			executedExitFunction = true;
+			LogManager.shutdownReset();
 		}
-		LogManager.shutdownReset();
 	}
 }