Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[Branch-2.0](Schema Change)light schema change support add key column #42915

Open
wants to merge 1 commit into
base: branch-2.0
Choose a base branch
from
Open

[Branch-2.0](Schema Change)light schema change support add key column #42915

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion be/src/vec/olap/block_reader.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -204,7 +204,8 @@ Status BlockReader::init(const ReaderParams& read_params) {
auto cid = read_params.origin_return_columns->at(i);
for (int j = 0; j < read_params.return_columns.size(); ++j) {
if (read_params.return_columns[j] == cid) {
if (j < _tablet->num_key_columns() || _tablet->keys_type() != AGG_KEYS) {
if (j < _tablet_schema->num_key_columns() ||
_tablet_schema->keys_type() != AGG_KEYS) {
_normal_columns_idx.emplace_back(j);
} else {
_agg_columns_idx.emplace_back(j);
Expand Down
84 changes: 69 additions & 15 deletions fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -183,8 +183,11 @@ private boolean processAddColumn(AddColumnClause alterClause, OlapTable olapTabl

Set<String> newColNameSet = Sets.newHashSet(column.getName());

return addColumnInternal(olapTable, column, columnPos, targetIndexId, baseIndexId, indexSchemaMap,
boolean lightSchemaChange = true;
addColumnInternal(olapTable, column, columnPos, targetIndexId, baseIndexId, indexSchemaMap,
newColNameSet, false, colUniqueIdSupplierMap);
lightSchemaChange = checkLightSchemaChange(olapTable, column, indexSchemaMap);
return lightSchemaChange;
}

private void processAddColumn(AddColumnClause alterClause, Table externalTable, List<Column> newSchema)
Expand Down Expand Up @@ -241,8 +244,9 @@ public boolean processAddColumns(AddColumnsClause alterClause, OlapTable olapTab

boolean lightSchemaChange = true;
for (Column column : columns) {
boolean result = addColumnInternal(olapTable, column, null, targetIndexId, baseIndexId, indexSchemaMap,
addColumnInternal(olapTable, column, null, targetIndexId, baseIndexId, indexSchemaMap,
newColNameSet, ignoreSameColumn, colUniqueIdSupplierMap);
boolean result = checkLightSchemaChange(olapTable, column, indexSchemaMap);
if (!result) {
lightSchemaChange = false;
}
Expand Down Expand Up @@ -887,18 +891,16 @@ private void addColumnInternal(Column newColumn, ColumnPosition columnPos, List<
* @param newColNameSet
* @param ignoreSameColumn
* @param colUniqueIdSupplierMap
* @return true: can light schema change, false: cannot
* @return void
* @throws DdlException
*/
private boolean addColumnInternal(OlapTable olapTable, Column newColumn, ColumnPosition columnPos,
private void addColumnInternal(OlapTable olapTable, Column newColumn, ColumnPosition columnPos,
long targetIndexId, long baseIndexId,
Map<Long, LinkedList<Column>> indexSchemaMap,
Set<String> newColNameSet, boolean ignoreSameColumn,
Map<Long, IntSupplier> colUniqueIdSupplierMap)
throws DdlException {

//only new table generate ColUniqueId, exist table do not.
boolean lightSchemaChange = olapTable.getEnableLightSchemaChange();
String newColName = newColumn.getName();

// check the validation of aggregation method on column.
Expand Down Expand Up @@ -963,12 +965,6 @@ private boolean addColumnInternal(OlapTable olapTable, Column newColumn, ColumnP
throw new DdlException("BITMAP_UNION must be used in AGG_KEYS");
}

//type key column do not allow light schema change.
if (newColumn.isKey()) {
LOG.debug("newColumn: {}, isKey()==true", newColumn);
lightSchemaChange = false;
}

// check if the new column already exist in base schema.
// do not support adding new column which already exist in base schema.
List<Column> baseSchema = olapTable.getBaseSchema(true);
Expand Down Expand Up @@ -1033,7 +1029,7 @@ private boolean addColumnInternal(OlapTable olapTable, Column newColumn, ColumnP
checkAndAddColumn(modIndexSchema, newColumn, columnPos, newColNameSet, true,
baseIndexNewColumnUniqueId);
if (targetIndexId == -1L) {
return lightSchemaChange;
return;
}
// 2. add to rollup
modIndexSchema = indexSchemaMap.get(targetIndexId);
Expand All @@ -1054,7 +1050,7 @@ private boolean addColumnInternal(OlapTable olapTable, Column newColumn, ColumnP
checkAndAddColumn(modIndexSchema, newColumn, columnPos, newColNameSet, true,
baseIndexNewColumnUniqueId);
// no specified target index. return
return lightSchemaChange;
return;
} else {
// add to rollup index
List<Column> modIndexSchema = indexSchemaMap.get(targetIndexId);
Expand Down Expand Up @@ -1088,7 +1084,7 @@ private boolean addColumnInternal(OlapTable olapTable, Column newColumn, ColumnP

if (targetIndexId == -1L) {
// no specified target index. return
return lightSchemaChange;
return;
}

// 2. add to rollup index
Expand All @@ -1098,6 +1094,64 @@ private boolean addColumnInternal(OlapTable olapTable, Column newColumn, ColumnP
modIndexSchema = indexSchemaMap.get(targetIndexId);
checkAndAddColumn(modIndexSchema, newColumn, columnPos, newColNameSet, false, rollUpNewColumnUniqueId);
}
return;
}

/**
* @param olapTable
* @param newColumn
* @param indexSchemaMap
* @return true: can light schema change, false: cannot
* @throws DdlException
*/
private boolean checkLightSchemaChange(OlapTable olapTable, Column newColumn,
Map<Long, LinkedList<Column>> indexSchemaMap) {
// only new table generate ColUniqueId, exist table do not.
boolean lightSchemaChange = olapTable.getEnableLightSchemaChange();
if (!lightSchemaChange || !newColumn.isKey()) {
return lightSchemaChange;
}

long baseIndexId = olapTable.getBaseIndexId();
Set<String> baseColumnNames = Sets.newHashSet();

// check light schema change with add key column
for (Long alterIndexId : indexSchemaMap.keySet()) {
List<Column> alterSchema = indexSchemaMap.get(alterIndexId);
int newColumnPos = -1;
for (int i = 0; i < alterSchema.size(); ++i) {
if (alterSchema.get(i).getName() == newColumn.getName()) {
newColumnPos = i;
}
}

if (newColumnPos >= 0) {
// add key column in short key columns
MaterializedIndexMeta currentIndexMeta = olapTable.getIndexMetaByIndexId(alterIndexId);
if (newColumnPos < currentIndexMeta.getShortKeyColumnCount()) {
return false;
}

// not support add key column for mv index
if (alterIndexId != baseIndexId) {
if (baseColumnNames.isEmpty()) {
for (Column col : olapTable.getBaseSchemaKeyColumns()) {
baseColumnNames.add(col.getName());
}
}
for (Column col : olapTable.getKeyColumnsByIndexId(alterIndexId)) {
if (null != col.getDefineExpr() || !baseColumnNames.contains(col.getName())) {
return false;
}
}
}

// unique key merge on write
if (olapTable.getEnableUniqueKeyMergeOnWrite()) {
return false;
}
}
}
return lightSchemaChange;
}

Expand Down
119 changes: 119 additions & 0 deletions regression-test/data/schema_change_p0/test_add_keys_light_schema_change.out
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,119 @@
-- This file is automatically generated. You should know what you did if you want to edit this
-- !sc --
1 2 2017-10-01 Beijing 10 1 1 20 \N \N

-- !sc --
1 2017-10-01 Beijing 10 1 20

-- !21_agg_multi_rowset --
1 3 40 3 3

-- !21_agg_multi_rowset --
1 2 30 2 2

-- !21_agg_multi_rowset --
1 1 40 1 1

-- !21_agg_compaction --
1 3 40 3 3

-- !21_agg_compaction --
1 2 30 2 2

-- !21_agg_compaction --
1 1 40 1 1

-- !22_agg_drop_multi_rowset --
1 3 40 3 3

-- !22_agg_drop_multi_rowset --
1 2 30 2 2

-- !22_agg_drop_multi_rowset --
1 1 40 1 1

-- !22_agg_drop_compaction --
1 3 40 3 3

-- !22_agg_drop_compaction --
1 2 30 2 2

-- !22_agg_drop_compaction --
1 1 40 1 1

-- !23_base_table_multi_rowset --
1 3 40 3 3

-- !23_base_table_multi_rowset --
1 2 30 2 2

-- !23_base_table_multi_rowset --
1 1 40 1 1

-- !23_base_table_compaction --
1 3 40 3 3

-- !23_base_table_compaction --
1 2 30 2 2

-- !23_base_table_compaction --
1 1 40 1 1

-- !23_rollup_multi_rowset --
1 3

-- !23_rollup_multi_rowset --
1 2

-- !23_rollup_multi_rowset --
1 1

-- !23_agg_rollup_compaction --
1 3

-- !23_agg_rollup_compaction --
1 2

-- !23_agg_rollup_compaction --
1 1

-- !31_duplicate_multi_rowset --
2017-10-01T10:00 1 0 2 10 2017-10-01T12:00
2017-10-01T10:00 1 0 2 none 10 2017-10-01T12:00
2017-10-01T10:00 1 0 1 none 10 2017-10-01T12:00

-- !31_duplicate_multi_rowset --
2017-10-01T10:00 1 0 2 10 2017-10-01T12:00
2017-10-01T10:00 1 0 2 none 10 2017-10-01T12:00

-- !31_duplicate_multi_rowset --
2017-10-01T10:00 1 0 1 none 10 2017-10-01T12:00

-- !31_duplicate_compaction --
2017-10-01T10:00 1 0 2 10 2017-10-01T12:00
2017-10-01T10:00 1 0 2 none 10 2017-10-01T12:00
2017-10-01T10:00 1 0 1 none 10 2017-10-01T12:00

-- !31_duplicate_compaction --
2017-10-01T10:00 1 0 2 10 2017-10-01T12:00
2017-10-01T10:00 1 0 2 none 10 2017-10-01T12:00

-- !31_duplicate_compaction --
2017-10-01T10:00 1 0 1 none 10 2017-10-01T12:00

-- !41_unique_multi_rowset --
1 Jone Beijing 1 10 1 10010 Haidian 2017-10-01T14:00

-- !41_unique_multi_rowset --

-- !41_unique_multi_rowset --
1 Jone Beijing 1 10 1 10010 Haidian 2017-10-01T14:00

-- !41_unique_compaction --
1 Jone Beijing 1 10 1 10010 Haidian 2017-10-01T14:00

-- !41_unique_compaction --

-- !41_unique_compaction --
1 Jone Beijing 1 10 1 10010 Haidian 2017-10-01T14:00

Loading
Loading