-
Notifications
You must be signed in to change notification settings - Fork 141
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge branch 'feature/maximus-m1' into feature/maximus/statement
- Loading branch information
Showing
58 changed files
with
3,105 additions
and
2,663 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -43,3 +43,4 @@ gen | |
|
|
||
| /artifacts/ | ||
| /.pid.lock | ||
| /.prom.pid.lock | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
27 changes: 27 additions & 0 deletions
27
core/src/main/java/org/opensearch/sql/catalog/model/Catalog.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,27 @@ | ||
| /* | ||
| * | ||
| * * Copyright OpenSearch Contributors | ||
| * * SPDX-License-Identifier: Apache-2.0 | ||
| * | ||
| */ | ||
|
|
||
| package org.opensearch.sql.catalog.model; | ||
|
|
||
| import lombok.EqualsAndHashCode; | ||
| import lombok.Getter; | ||
| import lombok.RequiredArgsConstructor; | ||
| import org.opensearch.sql.storage.StorageEngine; | ||
|
|
||
| @Getter | ||
| @RequiredArgsConstructor | ||
| @EqualsAndHashCode | ||
| public class Catalog { | ||
|
|
||
| private final String name; | ||
|
|
||
| private final ConnectorType connectorType; | ||
|
|
||
| @EqualsAndHashCode.Exclude | ||
| private final StorageEngine storageEngine; | ||
|
|
||
| } |
76 changes: 76 additions & 0 deletions
76
core/src/main/java/org/opensearch/sql/executor/streaming/DefaultMetadataLog.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,76 @@ | ||
| /* | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| * | ||
| * The OpenSearch Contributors require contributions made to | ||
| * this file be licensed under the Apache-2.0 license or a | ||
| * compatible open source license. | ||
| */ | ||
|
|
||
| package org.opensearch.sql.executor.streaming; | ||
|
|
||
| import com.google.common.base.Preconditions; | ||
| import java.util.ArrayList; | ||
| import java.util.List; | ||
| import java.util.Optional; | ||
| import java.util.SortedMap; | ||
| import java.util.TreeMap; | ||
| import org.apache.commons.lang3.tuple.Pair; | ||
|
|
||
| /** | ||
| * In memory implementation of {@link MetadataLog}. Todo. Current implementation does not guarantee | ||
| * thread safe. We will re-evaluate it when adding pipeline execution. | ||
| * | ||
| * @param <T> type of metadata type. | ||
| */ | ||
| public class DefaultMetadataLog<T> implements MetadataLog<T> { | ||
|
|
||
| private static final long MIN_ACCEPTABLE_ID = 0L; | ||
|
|
||
| private SortedMap<Long, T> metadataMap = new TreeMap<>(); | ||
|
|
||
| @Override | ||
| public boolean add(Long batchId, T metadata) { | ||
| Preconditions.checkArgument(batchId >= MIN_ACCEPTABLE_ID, "batch id must large or equal 0"); | ||
|
|
||
| if (metadataMap.containsKey(batchId)) { | ||
| return false; | ||
| } | ||
| metadataMap.put(batchId, metadata); | ||
| return true; | ||
| } | ||
|
|
||
| @Override | ||
| public Optional<T> get(Long batchId) { | ||
| if (!metadataMap.containsKey(batchId)) { | ||
| return Optional.empty(); | ||
| } else { | ||
| return Optional.of(metadataMap.get(batchId)); | ||
| } | ||
| } | ||
|
|
||
| @Override | ||
| public List<T> get(Optional<Long> startBatchId, Optional<Long> endBatchId) { | ||
| if (startBatchId.isEmpty() && endBatchId.isEmpty()) { | ||
| return new ArrayList<>(metadataMap.values()); | ||
| } else { | ||
| Long s = startBatchId.orElse(MIN_ACCEPTABLE_ID); | ||
| Long e = endBatchId.map(i -> i + 1).orElse(Long.MAX_VALUE); | ||
| return new ArrayList<>(metadataMap.subMap(s, e).values()); | ||
| } | ||
| } | ||
|
|
||
| @Override | ||
| public Optional<Pair<Long, T>> getLatest() { | ||
| if (metadataMap.isEmpty()) { | ||
| return Optional.empty(); | ||
| } else { | ||
| Long latestId = metadataMap.lastKey(); | ||
| return Optional.of(Pair.of(latestId, metadataMap.get(latestId))); | ||
| } | ||
| } | ||
|
|
||
| @Override | ||
| public void purge(Long batchId) { | ||
| metadataMap.headMap(batchId).clear(); | ||
| } | ||
| } |
61 changes: 61 additions & 0 deletions
61
core/src/main/java/org/opensearch/sql/executor/streaming/MetadataLog.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,61 @@ | ||
| /* | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| * | ||
| * The OpenSearch Contributors require contributions made to | ||
| * this file be licensed under the Apache-2.0 license or a | ||
| * compatible open source license. | ||
| */ | ||
|
|
||
| package org.opensearch.sql.executor.streaming; | ||
|
|
||
| import java.util.List; | ||
| import java.util.Optional; | ||
| import org.apache.commons.lang3.tuple.Pair; | ||
|
|
||
| /** | ||
| * Write-ahead Log (WAL). Which allow client write metadata associate with id. | ||
| * | ||
| * @param <T> type of metadata type. | ||
| */ | ||
| public interface MetadataLog<T> { | ||
|
|
||
| /** | ||
| * add metadata to WAL. | ||
| * | ||
| * @param id metadata index in WAL. | ||
| * @param metadata metadata. | ||
| * @return true if add success, otherwise return false. | ||
| */ | ||
| boolean add(Long id, T metadata); | ||
|
|
||
| /** | ||
| * get metadata from WAL. | ||
| * | ||
| * @param id metadata index in WAL. | ||
| * @return metadata. | ||
| */ | ||
| Optional<T> get(Long id); | ||
|
|
||
| /** | ||
| * Return metadata for id between [startId, endId]. | ||
| * | ||
| * @param startId If startId is empty, return all metadata before endId (inclusive). | ||
| * @param endId If end is empty, return all batches after endId (inclusive). | ||
| * @return a list of metadata sorted by id (nature order). | ||
| */ | ||
| List<T> get(Optional<Long> startId, Optional<Long> endId); | ||
|
|
||
| /** | ||
| * Get latest batchId and metadata. | ||
| * | ||
| * @return pair of id and metadata if not empty. | ||
| */ | ||
| Optional<Pair<Long, T>> getLatest(); | ||
|
|
||
| /** | ||
| * Remove all the metadata less then id (exclusive). | ||
| * | ||
| * @param id smallest batchId should keep. | ||
| */ | ||
| void purge(Long id); | ||
| } |
Oops, something went wrong.