Merge remote-tracking branch 'upstream/main' into fix-nightly-check

* upstream/main: Move slow tests (>= 10 seconds) to the nightly check, fixes deephaven#606 (deephaven#616) Set ownership of DevRel relevant files (deephaven#625) Fix context leak in RedirectedColumnSource.RedirectionFillFrom (deephaven#619) Followup to review comments. Followup to review comments. Removed some unused code. Ensure generated formula code handles prev correctly for i,ii in ticking tables. Fixes deephaven#544 Produce the right exception, instead of returning null, on invalid arguments for calls to SortedRanges.invert. Towards deephaven#544.
devinrsmith · May 18, 2021 · 3089bbb · 3089bbb
2 parents b5993ee + f02fa07
commit 3089bbb
Show file tree

Hide file tree

Showing 45 changed files with 310 additions and 39 deletions.
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -10,3 +10,11 @@
 #
 # See https://securitylab.github.com/research/github-actions-preventing-pwn-requests/
 /.github/workflows/ @devinrsmith
+
+CODE_OF_CONDUCT.md @chipkent
+CONTRIBUTING.md @chipkent
+LICENSE.md @chipkent
+NOTICE.md @chipkent
+README.md @chipkent
+TRIAGE.md @chipkent
+/licenses @chipkent
diff --git a/DB/src/main/java/io/deephaven/db/v2/select/DhFormulaColumn.java b/DB/src/main/java/io/deephaven/db/v2/select/DhFormulaColumn.java
@@ -322,8 +322,18 @@ private CodeGenerator generateAppropriateGetMethod(TypeAnalyzer ta, boolean useP
         final CodeGenerator g = CodeGenerator.create(
                 "@Override",
                 "public [[RETURN_TYPE]] [[GETTER_NAME]](final long k)", CodeGenerator.block(
-                        CodeGenerator.optional("maybeCreateI", "final int i = __intSize(__index.find(k));"),
-                        CodeGenerator.optional("maybeCreateII", "final long ii = __index.find(k);"),
+                        (usePrev
+                            ? CodeGenerator.optional("maybeCreateIorII",
+                                  "final long findResult;",
+                                  "try (final Index prev = __index.getPrevIndex())", CodeGenerator.block(
+                                       "findResult = prev.find(k);"
+                                  ))
+                            : CodeGenerator.optional("maybeCreateIorII",
+                                "final long findResult = __index.find(k);")),
+                        CodeGenerator.optional("maybeCreateI",
+                                "final int i = __intSize(findResult);"),
+                        CodeGenerator.optional("maybeCreateII",
+                                "final long ii = findResult;"),
                         CodeGenerator.repeated("cacheColumnSourceGet", "final [[TYPE]] [[VAR]] = [[GET_EXPRESSION]];"),
                         "if ([[LAZY_RESULT_CACHE_NAME]] != null)", CodeGenerator.block(
                                 "final Object __lazyKey = [[C14NUTIL_CLASSNAME]].maybeMakeSmartKey([[FORMULA_ARGS]]);",
@@ -345,6 +355,9 @@ private CodeGenerator generateAppropriateGetMethod(TypeAnalyzer ta, boolean useP
         g.replace("RESULT_TYPE", resultTypeString);
         g.replace("GETTER_NAME", getterName);
 
+        if (usesI || usesII) {
+            g.activateOptional("maybeCreateIorII");
+        }
         if (usesI) {
             g.activateOptional("maybeCreateI");
         }
@@ -472,7 +485,7 @@ private CodeGenerator generateFillChunk(boolean usePrev) {
                                 "final [[CHUNK_TYPE]] __chunk__col__[[COL_SOURCE_NAME]] = this.[[COL_SOURCE_NAME]].[[GET_CURR_OR_PREV_CHUNK]](" +
                                         "__typedContext.__subContext[[COL_SOURCE_NAME]], __orderedKeys).[[AS_CHUNK_METHOD]]();"
                         ),
-                        "fillChunkHelper(__typedContext, __destination, __orderedKeys[[ADDITIONAL_CHUNK_ARGS]]);"
+                        "fillChunkHelper(" + Boolean.toString(usePrev) + ", __typedContext, __destination, __orderedKeys[[ADDITIONAL_CHUNK_ARGS]]);"
                 )
         );
 
@@ -497,17 +510,22 @@ private CodeGenerator generateFillChunk(boolean usePrev) {
     @NotNull
     private CodeGenerator generateFillChunkHelper(TypeAnalyzer ta) {
         final CodeGenerator g = CodeGenerator.create(
-                "private void fillChunkHelper(final FormulaFillContext __context,", CodeGenerator.indent(
+                "private void fillChunkHelper(final boolean __usePrev, final FormulaFillContext __context,", CodeGenerator.indent(
                         "final WritableChunk<? super Attributes.Values> __destination,",
                         "final OrderedKeys __orderedKeys[[ADDITIONAL_CHUNK_ARGS]])"), CodeGenerator.block(
                         "final [[DEST_CHUNK_TYPE]] __typedDestination = __destination.[[DEST_AS_CHUNK_METHOD]]();",
-                        CodeGenerator.optional("maybeCreateI",
-                                "__context.__iChunk.setSize(0);",
-                                "__index.invert(__orderedKeys.asIndex()).forAllLongs(l -> __context.__iChunk.add(__intSize(l)));"
-                        ),
-                        CodeGenerator.optional("maybeCreateII",
-                                "__index.invert(__orderedKeys.asIndex()).fillKeyIndicesChunk(__context.__iiChunk);"
-                        ),
+                        CodeGenerator.optional("maybeCreateIOrII",
+                                "try (final Index prev = __usePrev ? __index.getPrevIndex() : null;", CodeGenerator.indent(
+                                     "final Index inverted = ((prev != null) ? prev : __index).invert(__orderedKeys.asIndex()))"), CodeGenerator.block(
+                                            CodeGenerator.optional("maybeCreateI",
+                                                    "__context.__iChunk.setSize(0);",
+                                                    "inverted.forAllLongs(l -> __context.__iChunk.add(__intSize(l)));"
+                                            ),
+                                            CodeGenerator.optional("maybeCreateII",
+                                                    "inverted.fillKeyIndicesChunk(__context.__iiChunk);"
+                                            )
+                                        )
+                                ),
                         CodeGenerator.repeated("getChunks",
                                 "final [[CHUNK_TYPE]] __chunk__col__[[COL_SOURCE_NAME]] = __sources[[[SOURCE_INDEX]]].[[AS_CHUNK_METHOD]]();"
                         ),
@@ -547,6 +565,9 @@ private CodeGenerator generateFillChunkHelper(TypeAnalyzer ta) {
                 null);
         final String additionalChunkArgs = chunkArgs.isEmpty() ? "" : ", " + makeCommaSeparatedList(chunkArgs);
         g.replace("ADDITIONAL_CHUNK_ARGS", additionalChunkArgs);
+        if (usesI || usesII) {
+            g.activateOptional("maybeCreateIOrII");
+        }
         if (usesI) {
             g.activateAllOptionals("maybeCreateI");
         }

diff --git a/DB/src/main/java/io/deephaven/db/v2/sources/RedirectedColumnSource.java b/DB/src/main/java/io/deephaven/db/v2/sources/RedirectedColumnSource.java
@@ -96,14 +96,21 @@ public void ensureCapacity(long capacity) {
 
     private class RedirectionFillFrom implements FillFromContext {
         final RedirectionIndex.FillContext redirectionFillContext;
-        final WritableLongChunk<KeyIndices> redirections;
         final FillFromContext innerFillFromContext;
+        final WritableLongChunk<KeyIndices> redirections;
 
         private RedirectionFillFrom(int chunkCapacity) {
             this.redirectionFillContext = redirectionIndex.makeFillContext(chunkCapacity, null);
             this.innerFillFromContext = ((WritableSource)innerSource).makeFillFromContext(chunkCapacity);
             this.redirections = WritableLongChunk.makeWritableChunk(chunkCapacity);
         }
+
+        @Override
+        public void close() {
+            redirectionFillContext.close();
+            innerFillFromContext.close();
+            redirections.close();
+        }
     }
 
     @Override

diff --git a/DB/src/main/java/io/deephaven/db/v2/utils/codegen/CodeGenerator.java b/DB/src/main/java/io/deephaven/db/v2/utils/codegen/CodeGenerator.java
@@ -371,7 +371,7 @@ final class Optional extends CodeGenerator {
         this(tag, inner, false);
     }
 
-    Optional(String tag, CodeGenerator inner, boolean active) {
+    private Optional(String tag, CodeGenerator inner, boolean active) {
         this.tag = tag;
         this.inner = inner;
         this.active = active;

diff --git a/DB/src/test/java/io/deephaven/db/tables/QueryTableHugeSortTest.java b/DB/src/test/java/io/deephaven/db/tables/QueryTableHugeSortTest.java
@@ -5,20 +5,24 @@
 import io.deephaven.db.tables.utils.TableTools;
 import io.deephaven.db.v2.sources.AbstractColumnSource;
 import io.deephaven.db.v2.utils.Index;
+import io.deephaven.test.types.OutOfBandTest;
 import org.junit.After;
 import org.junit.Test;
 
 import java.util.Arrays;
 import java.util.Comparator;
 import java.util.LinkedHashMap;
 import java.util.Map;
+import org.junit.experimental.categories.Category;
 
 import static io.deephaven.db.tables.utils.TableTools.show;
 import static io.deephaven.db.v2.TstUtils.*;
 import static io.deephaven.db.v2.TstUtils.assertTableEquals;
 
+@Category(OutOfBandTest.class)
 public class QueryTableHugeSortTest {
     @Test
+    @Category(OutOfBandTest.class)
     public void testHugeSort() {
         final int megaSortSize = SortHelpers.megaSortSize;
         final int sortChunkSize = SortHelpers.sortChunkSize;

diff --git a/DB/src/test/java/io/deephaven/db/tables/utils/TestTableTools.java b/DB/src/test/java/io/deephaven/db/tables/utils/TestTableTools.java
@@ -19,6 +19,7 @@
 import io.deephaven.db.v2.sources.chunk.Attributes;
 import io.deephaven.db.v2.sources.chunk.IntChunk;
 import io.deephaven.db.v2.utils.*;
+import io.deephaven.test.types.OutOfBandTest;
 import io.deephaven.util.ExceptionDetails;
 import io.deephaven.util.QueryConstants;
 import junit.framework.TestCase;
@@ -41,10 +42,12 @@
 import static io.deephaven.util.QueryConstants.NULL_DOUBLE;
 
 import java.nio.charset.Charset;
+import org.junit.experimental.categories.Category;
 
 /**
  * Unit tests for {@link TableTools}.
  */
+@Category(OutOfBandTest.class)
 public class TestTableTools extends TestCase implements UpdateErrorReporter {
 
     private static final boolean ENABLE_COMPILER_TOOLS_LOGGING = Configuration.getInstance().getBooleanForClassWithDefault(TestTableTools.class, "CompilerTools.logEnabled", false);
@@ -741,6 +744,7 @@ protected Table e() {
 
     // This merge should work out nicely, we'll end up collapsing it into a single broad merge.
     @Test
+    @Category(OutOfBandTest.class)
     public void testMergeRecursive() {
         Table result = null;
 
@@ -1143,6 +1147,7 @@ public void testEmptyTable() throws IOException {
     }
 
     @Test
+    @Category(OutOfBandTest.class)
     public void testMergeIndexShiftingPerformance() {
         final QueryTable testRefreshingTable = TstUtils.testRefreshingTable(i(0), intCol("IntCol", 0), charCol("CharCol", 'a'));
 

diff --git a/DB/src/test/java/io/deephaven/db/util/datastructures/intrusive/TestIntrusiveSoftLRU.java b/DB/src/test/java/io/deephaven/db/util/datastructures/intrusive/TestIntrusiveSoftLRU.java
@@ -1,12 +1,15 @@
 package io.deephaven.db.util.datastructures.intrusive;
 
+import io.deephaven.test.types.OutOfBandTest;
 import io.deephaven.util.datastructures.intrusive.IntrusiveSoftLRU;
 import junit.framework.TestCase;
 import org.junit.Test;
 
 import java.lang.ref.WeakReference;
 import java.util.*;
+import org.junit.experimental.categories.Category;
 
+@Category(OutOfBandTest.class)
 public class TestIntrusiveSoftLRU {
 
     class Obj extends IntrusiveSoftLRU.Node.Impl<Obj> {
@@ -78,6 +81,7 @@ public void testBasics() {
     }
 
     @Test
+    @Category(OutOfBandTest.class)
     public void testRobustness() {
         ArrayList<Obj> objects = new ArrayList<>(1000);
         ArrayList<WeakReference<Obj>> objRefs = new ArrayList<>(1000);

diff --git a/DB/src/test/java/io/deephaven/db/v2/IndexGroupingTest.java b/DB/src/test/java/io/deephaven/db/v2/IndexGroupingTest.java
@@ -16,18 +16,21 @@
 import io.deephaven.db.v2.tuples.TupleSourceFactory;
 import io.deephaven.db.v2.utils.Index;
 import io.deephaven.db.v2.utils.UpdatePerformanceTracker;
+import io.deephaven.test.types.OutOfBandTest;
 import org.apache.commons.lang3.mutable.MutableInt;
 import org.junit.After;
 import org.junit.Before;
 
 import java.io.IOException;
 import java.util.*;
 import java.util.stream.Collectors;
+import org.junit.experimental.categories.Category;
 
 import static io.deephaven.db.tables.utils.TableTools.show;
 import static io.deephaven.db.v2.TstUtils.getTable;
 
 @SuppressWarnings("ClassInitializerMayBeStatic")
+@Category(OutOfBandTest.class)
 public class IndexGroupingTest extends LiveTableTestCase {
 
     private static final boolean ENABLE_COMPILER_TOOLS_LOGGING = Configuration.getInstance().getBooleanForClassWithDefault(IndexGroupingTest.class, "CompilerTools.logEnabled", false);

diff --git a/DB/src/test/java/io/deephaven/db/v2/QueryTableAggregationTest.java b/DB/src/test/java/io/deephaven/db/v2/QueryTableAggregationTest.java
@@ -52,6 +52,7 @@
 import static io.deephaven.db.tables.utils.TableTools.*;
 import static io.deephaven.db.v2.TstUtils.*;
 
+@Category(OutOfBandTest.class)
 public class QueryTableAggregationTest {
     private JUnit4QueryTableTestBase base = new JUnit4QueryTableTestBase();
 
@@ -125,6 +126,7 @@ public void testStaticReinterpretableKeyByWithChunks() {
     }
 
     @Test
+    @Category(OutOfBandTest.class)
     public void testStaticByWithChunksAndAggressiveOverflow() {
         final AggregationControl control = new AggregationControl() {
             @Override
@@ -651,6 +653,7 @@ public Table e() {
     }
 
     @Test
+    @Category(OutOfBandTest.class)
     public void testFirstByLastByIncremental() {
         final Random random = new Random(0);
 
@@ -1183,6 +1186,7 @@ public void testApplyToAllBy() {
 
 
     @Test
+    @Category(OutOfBandTest.class)
     public void testSumByStatic() {
         final int[] sizes = {10, 100, 1000};
         for (final int size : sizes) {
@@ -1311,6 +1315,7 @@ private void testMinMaxByStatic(int size, boolean lotsOfStrings) {
     }
 
     @Test
+    @Category(OutOfBandTest.class)
     public void testAvgByStatic() {
         final int[] sizes = {10, 100, 1000};
         for (final int size : sizes) {
@@ -2002,6 +2007,7 @@ private void testWeightedAvgByIncremental(int size, int seed) {
     }
 
     @Test
+    @Category(OutOfBandTest.class)
     public void testWeightedSumByIncremental() {
         final int[] sizes = {10, 50, 200};
         for (int size : sizes) {
@@ -2084,6 +2090,7 @@ private void testCountByIncremental(int size) {
     }
 
     @Test
+    @Category(OutOfBandTest.class)
     public void testMinMaxByIncremental() {
         final int[] sizes = {10, 50, 200};
         for (final int size : sizes) {

diff --git a/DB/src/test/java/io/deephaven/db/v2/QueryTableAjTest.java b/DB/src/test/java/io/deephaven/db/v2/QueryTableAjTest.java
@@ -42,6 +42,7 @@
 import static junit.framework.TestCase.assertEquals;
 import static junit.framework.TestCase.assertNotNull;
 
+@Category(OutOfBandTest.class)
 public class QueryTableAjTest {
 
     private JUnit4QueryTableTestBase base = new JUnit4QueryTableTestBase();
@@ -560,6 +561,7 @@ public void testAjRandomLeftStaticRightIncremental() {
     }
 
     @Test
+    @Category(OutOfBandTest.class)
     public void testAjBothIncremental() {
         final int tableMultiplier = 10;
         final int initialTableSize = 10;

diff --git a/DB/src/test/java/io/deephaven/db/v2/QueryTableCrossJoinSmallRightBitsTest.java b/DB/src/test/java/io/deephaven/db/v2/QueryTableCrossJoinSmallRightBitsTest.java
@@ -5,21 +5,25 @@
 import io.deephaven.db.v2.utils.Index;
 import io.deephaven.db.v2.utils.IndexShiftData;
 import io.deephaven.db.v2.utils.OutOfKeySpaceException;
+import io.deephaven.test.types.OutOfBandTest;
 import org.apache.commons.lang3.mutable.MutableInt;
 
 import java.util.Arrays;
+import org.junit.experimental.categories.Category;
 
 import static io.deephaven.db.tables.utils.TableTools.intCol;
 import static io.deephaven.db.v2.TstUtils.c;
 import static io.deephaven.db.v2.TstUtils.i;
 import static io.deephaven.db.v2.TstUtils.testRefreshingTable;
 import static io.deephaven.db.v2.TstUtils.testTable;
 
+@Category(OutOfBandTest.class)
 public class QueryTableCrossJoinSmallRightBitsTest extends QueryTableCrossJoinTestBase {
    public QueryTableCrossJoinSmallRightBitsTest() {
        super(1);
    }
 
+    @Category(OutOfBandTest.class)
     public void testIncrementalWithKeyColumnsShallow() {
         final int size = 10;
 

diff --git a/DB/src/test/java/io/deephaven/db/v2/QueryTableCrossJoinTest.java b/DB/src/test/java/io/deephaven/db/v2/QueryTableCrossJoinTest.java
@@ -1,5 +1,9 @@
 package io.deephaven.db.v2;
 
+import io.deephaven.test.types.OutOfBandTest;
+import org.junit.experimental.categories.Category;
+
+@Category(OutOfBandTest.class)
 public class QueryTableCrossJoinTest extends QueryTableCrossJoinTestBase {
     public QueryTableCrossJoinTest() {
         super(16); // default

diff --git a/DB/src/test/java/io/deephaven/db/v2/QueryTableCrossJoinTestBase.java b/DB/src/test/java/io/deephaven/db/v2/QueryTableCrossJoinTestBase.java
@@ -16,6 +16,7 @@
 import io.deephaven.db.v2.sources.chunk.util.pools.ChunkPoolReleaseTracking;
 import io.deephaven.db.v2.utils.Index;
 import io.deephaven.db.v2.utils.IndexShiftData;
+import io.deephaven.test.types.OutOfBandTest;
 import org.apache.commons.lang3.mutable.MutableInt;
 import org.apache.commons.lang3.mutable.MutableLong;
 import org.apache.commons.lang3.mutable.MutableObject;
@@ -24,10 +25,12 @@
 import java.util.Arrays;
 import java.util.Map;
 import java.util.Random;
+import org.junit.experimental.categories.Category;
 
 import static io.deephaven.db.tables.utils.TableTools.longCol;
 import static io.deephaven.db.v2.TstUtils.*;
 
+@Category(OutOfBandTest.class)
 public abstract class QueryTableCrossJoinTestBase extends QueryTableTestBase {
 
     private final int numRightBitsToReserve;
@@ -188,6 +191,7 @@ public void testZeroKeyJoinCompoundShift() {
         TstUtils.validate(en);
     }
 
+    @Category(OutOfBandTest.class)
     public void testIncrementalZeroKeyJoin() {
         final int[] sizes = {10, 100, 1000};
         for (int size : sizes) {
@@ -249,6 +253,7 @@ public void testSmallStaticJoin() {
         }
     }
 
+    @Category(OutOfBandTest.class)
     public void testLargeStaticJoin() {
         final String[] types = new String[26];
         final int[] cardinality = new int[26];
@@ -454,6 +459,7 @@ int initialBuildSize() {
         }
     }
 
+    @Category(OutOfBandTest.class)
     public void testIncrementalWithKeyColumns() {
         final int[] sizes = {10, 100, 1000};
 
@@ -529,6 +535,7 @@ public void testColumnSourceCanReuseContextWithSmallerOrderedKeys() {
         }
     }
 
+    @Category(OutOfBandTest.class)
     public void testShiftingDuringRehash() {
         final int maxSteps = 2500;
         final MutableInt numSteps = new MutableInt();