Feature / Reworked data pipeline framework (#188)

* Initial interface for DataPipeline

* New codec base classes to work with data pipelines

* Move arrow schema util into the main data package (it is used by all data components, not just the arrow codec)

* Update arrow codec to work with the data pipeline framework

* Update CSV codec to work with the data pipeline framework

* Update JSON codec to work with the data pipeline framework

* First implementation of data pipeline and supporting classes

* Update the codec test suite to work with data pipelines

* Update data round trip test in the orchestrator after data framework changes

* Update storage interface to use the new data framework, and use it in the main data service

* Remove DataBlock class which was part of the old data implementation

* Use a flow helper to handle mkdir before writing file in flat data storage

* Add some comments on the rationale for data pipelines

* Remove debug logging

* Try to flush arrow content stream in RT test

* Remove bad char sequence from doc comment that prevents compile in Java 11.

* Fix handling of arrow data encode/decode in end-to-end integration test

tracdap-libs/tracdap-lib-common/src/main/java/org/finos/tracdap/common/concurrent/Flows.java

@@ -64,6 +64,12 @@ Flow.Processor<T, T> interceptResult(Flow.Publisher<T> source, BiConsumer<T, Thr
         return interceptor;
     }
 
+    public static <T>
+    Flow.Subscriber<T> waitForSignal(Flow.Subscriber<T> target, CompletionStage<?> signal) {
+
+        return new DelayedSubscriber<>(target, signal);
+    }
+
     public static <T, U>
     Flow.Publisher<U> map(Flow.Publisher<T> source, Function<T, U> mapping) {
 

tracdap-libs/tracdap-lib-common/src/main/java/org/finos/tracdap/common/concurrent/flow/DelayedSubscriber.java

@@ -0,0 +1,62 @@
+/*
+ * Copyright 2022 Accenture Global Solutions Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.finos.tracdap.common.concurrent.flow;
+
+import java.util.concurrent.CompletionStage;
+import java.util.concurrent.Flow;
+
+
+public class DelayedSubscriber<T> implements Flow.Subscriber<T> {
+
+    private final Flow.Subscriber<T> subscriber;
+    private final CompletionStage<?> signal;
+
+    public DelayedSubscriber(Flow.Subscriber<T> subscriber, CompletionStage<?> signal) {
+        this.subscriber = subscriber;
+        this.signal = signal;
+    }
+
+    @Override
+    public void onSubscribe(Flow.Subscription subscription) {
+
+        signal.whenComplete((result, error) -> {
+
+            if (error == null)
+                subscriber.onSubscribe(subscription);
+
+            else {
+                subscription.cancel();
+                subscriber.onError(error);
+            }
+        });
+    }
+
+    @Override
+    public void onNext(T item) {
+        subscriber.onNext(item);
+    }
+
+    @Override
+    public void onError(Throwable throwable) {
+        subscriber.onError(throwable);
+    }
+
+    @Override
+    public void onComplete() {
+        subscriber.onComplete();
+    }
+}