Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[DRAFT]set list support; #141

Draft
wants to merge 2 commits into
base: main
Choose a base branch
from
Draft

[DRAFT]set list support; #141

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
1e41dca
set list support;
ricki-epsilla Mar 18, 2024
7c0f20a
Merge branch 'main' into set-list-type
ricki-epsilla Mar 18, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 26 additions & 0 deletions engine/db/catalog/basic_meta_impl.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ constexpr const char* METRIC_TYPE = "metric_type";
constexpr const char* INDICES = "indices";
constexpr const char* MODEL = "model";
constexpr const char* DIMENSIONS = "dimensions";
constexpr const char* ELEMENT_TYPE = "element_type";

constexpr const char* DB_CATALOG_FILE_NAME = "catalog";

Expand All @@ -49,6 +50,10 @@ Status LoadFieldSchemaFromJson(const vectordb::Json& json, meta::FieldSchema& fi
field_schema.vector_dimension_ = json.GetInt(VECTOR_DIMENSION);
field_schema.metric_type_ = static_cast<meta::MetricType>(json.GetInt(METRIC_TYPE));
}
// Only set and list fields have element_type.
if (field_schema.field_type_ == meta::FieldType::SET || field_schema.field_type_ == meta::FieldType::LIST) {
field_schema.element_type = static_cast<meta::FieldType>(json.GetInt(ELEMENT_TYPE));
}
return Status::OK();
}

Expand Down Expand Up @@ -113,6 +118,10 @@ void DumpFieldSchemaToJson(const meta::FieldSchema& field_schema, vectordb::Json
json.SetInt(VECTOR_DIMENSION, field_schema.vector_dimension_);
json.SetInt(METRIC_TYPE, static_cast<int>(field_schema.metric_type_));
}
// Only set and list fields have element_type.
if (field_schema.field_type_ == meta::FieldType::SET || field_schema.field_type_ == meta::FieldType::LIST) {
json.SetInt(ELEMENT_TYPE, static_cast<int>(field_schema.element_type));
}
}

// Convert a TableSchema to a Json object
Expand Down Expand Up @@ -364,6 +373,23 @@ Status ValidateSchema(TableSchema& table_schema, std::vector<EmbeddingModel> &em
}
}

// 8. Set and list fields must have element type and should be supported data type.
if (field.field_type_ == FieldType::SET || field.field_type_ == FieldType::LIST) {
if (field.element_type == FieldType::UNKNOWN) {
return Status(DB_UNEXPECTED_ERROR, "Element type of " + field.name_ + " is not valid.");
}
if (field.element_type != FieldType::INT1 &&
field.element_type != FieldType::INT2 &&
field.element_type != FieldType::INT4 &&
field.element_type != FieldType::INT8 &&
field.element_type != FieldType::FLOAT &&
field.element_type != FieldType::DOUBLE &&
field.element_type != FieldType::STRING &&
field.element_type != FieldType::BOOL) {
return Status(DB_UNEXPECTED_ERROR, "Element type of " + field.name_ + " is not supported.");
}
}

if (has_primary_key && field.is_primary_key_) {
return Status(DB_UNEXPECTED_ERROR, "Cannot have more than 1 primary key fields.");
}
Expand Down
9 changes: 9 additions & 0 deletions engine/db/catalog/meta_types.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,9 @@ enum class FieldType {

GEO_POINT = 60,

SET = 70,
LIST = 71,

UNKNOWN = 999,
};

Expand All @@ -57,6 +60,7 @@ struct FieldSchema {
FieldType field_type_ = FieldType::INT4;
size_t vector_dimension_ = DEFAULT_VECTOR_DIMENSION;
MetricType metric_type_ = MetricType::EUCLIDEAN;
FieldType element_type = FieldType::INT4;
};

struct AutoEmbedding {
Expand Down Expand Up @@ -103,7 +107,12 @@ static const std::unordered_map<std::string, FieldType> fieldTypeMap = {
{"VECTOR_DOUBLE", FieldType::VECTOR_DOUBLE},
{"SPARSE_VECTOR_FLOAT", FieldType::SPARSE_VECTOR_FLOAT},
{"SPARSE_VECTOR_DOUBLE", FieldType::SPARSE_VECTOR_DOUBLE},
<<<<<<< HEAD
{"SET", FieldType::SET},
{"LIST", FieldType::LIST},
=======
{"GEO_POINT", FieldType::GEO_POINT},
>>>>>>> main
{"UNKNOWN", FieldType::UNKNOWN}};

static const std::unordered_map<std::string, MetricType> metricTypeMap = {
Expand Down
62 changes: 55 additions & 7 deletions engine/db/table_segment_mvp.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,11 @@ constexpr size_t FieldTypeSizeMVP(meta::FieldType type) {
// like the length of the string or the dimension of the vector. You might want
// to handle these cases differently.
return 0;
case meta::FieldType::SET:
case meta::FieldType::LIST:
// For these types, we can't determine the size without additional information
// like the length of the set or the list.
return 0;
case meta::FieldType::UNKNOWN:
default:
// Unknown type
Expand All @@ -54,13 +59,15 @@ Status TableSegmentMVP::Init(meta::TableSchema& table_schema, int64_t size_limit
primitive_offset_ = 0;
schema = table_schema;

// Get how many primitive, vectors, and variable-length attributes (string, sparse vectors).
// Get how many primitive, vectors, and variable-length attributes (string, sparse vectors, set, list).
for (auto& field_schema : table_schema.fields_) {
auto current_total_vec_num = dense_vector_num_ + sparse_vector_num_;
if (field_schema.field_type_ == meta::FieldType::STRING ||
field_schema.field_type_ == meta::FieldType::JSON ||
field_schema.field_type_ == meta::FieldType::SPARSE_VECTOR_DOUBLE ||
field_schema.field_type_ == meta::FieldType::SPARSE_VECTOR_FLOAT) {
field_schema.field_type_ == meta::FieldType::SPARSE_VECTOR_FLOAT ||
field_schema.field_type_ == meta::FieldType::SET ||
field_schema.field_type_ == meta::FieldType::LIST) {
field_id_mem_offset_map_[field_schema.id_] = var_len_attr_num_;
field_name_mem_offset_map_[field_schema.name_] = var_len_attr_num_;
if (field_schema.field_type_ == meta::FieldType::SPARSE_VECTOR_DOUBLE ||
Expand Down Expand Up @@ -265,6 +272,47 @@ TableSegmentMVP::TableSegmentMVP(meta::TableSchema& table_schema, const std::str
file.read(reinterpret_cast<char*>(v->data()), dataLen);
var_len_attr_table_[attrIdx][recordIdx] = std::move(v);
break;
case meta::FieldType::SET:
case meta::FieldType::LIST: {
// set or list contains elements of primitive data types
switch (field.element_type) {
case meta::FieldType::INT1:
case meta::FieldType::INT2:
case meta::FieldType::INT4:
case meta::FieldType::INT8: {
std::vector<int64_t> values(dataLen / sizeof(int64_t));
file.read(reinterpret_cast<char*>(values.data()), dataLen);
var_len_attr_table_[attrIdx][recordIdx] = std::move(values);
break;
}
case meta::FieldType::FLOAT: {
std::vector<float> values(dataLen / sizeof(float));
file.read(reinterpret_cast<char*>(values.data()), dataLen);
var_len_attr_table_[attrIdx][recordIdx] = std::move(values);
break;
}
case meta::FieldType::DOUBLE: {
std::vector<double> values(dataLen / sizeof(double));
file.read(reinterpret_cast<char*>(values.data()), dataLen);
var_len_attr_table_[attrIdx][recordIdx] = std::move(values);
break;
}
case meta::FieldType::STRING: {
std::string str(dataLen, '\0');
file.read(&str[0], dataLen);
var_len_attr_table_[attrIdx][recordIdx] = str;
break;
}
case meta::FieldType::BOOL: {
std::vector<bool> values(dataLen / sizeof(bool));
file.read(reinterpret_cast<char*>(values.data()), dataLen);
var_len_attr_table_[attrIdx][recordIdx] = std::move(values);
break;
}
default:
break;
}
}
}
}
}
Expand Down Expand Up @@ -663,7 +711,7 @@ Status TableSegmentMVP::Insert(meta::TableSchema& table_schema, Json& records, i
} else {
// std::cerr << "primary key [" << value << "] already exists, skipping." << std::endl;
skipped_entry++;
goto LOOP_END;
goto LOOP_END;
}
}
} else {
Expand All @@ -679,7 +727,7 @@ Status TableSegmentMVP::Insert(meta::TableSchema& table_schema, Json& records, i
} else {
// std::cerr << "primary key [" << value << "] already exists, skipping." << std::endl;
skipped_entry++;
goto LOOP_END;
goto LOOP_END;
}
}
break;
Expand All @@ -695,7 +743,7 @@ Status TableSegmentMVP::Insert(meta::TableSchema& table_schema, Json& records, i
} else {
// std::cerr << "primary key [" << value << "] already exists, skipping." << std::endl;
skipped_entry++;
goto LOOP_END;
goto LOOP_END;
}
}
break;
Expand All @@ -711,7 +759,7 @@ Status TableSegmentMVP::Insert(meta::TableSchema& table_schema, Json& records, i
} else {
// std::cerr << "primary key [" << value << "] already exists, skipping." << std::endl;
skipped_entry++;
goto LOOP_END;
goto LOOP_END;
}
}
break;
Expand All @@ -727,7 +775,7 @@ Status TableSegmentMVP::Insert(meta::TableSchema& table_schema, Json& records, i
} else {
// std::cerr << "primary key [" << value << "] already exists, skipping." << std::endl;
skipped_entry++;
goto LOOP_END;
goto LOOP_END;
}
}
break;
Expand Down
4 changes: 4 additions & 0 deletions engine/db/table_segment_mvp.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,10 @@ class TableSegmentMVP {
int64_t dense_vector_num_;
char* attribute_table_; // The attribute table in memory (exclude vector attributes and string attributes).
std::vector<VariableLenAttrColumnContainer> var_len_attr_table_; // The variable length attribute table in memory.

// TODO:
// vector for container attribute table

// std::vector<std::vector<std::string>> string_tables_; // Hold the string attributes.
std::vector<int64_t> vector_dims_;
float** vector_tables_; // The vector attribute tables. Each vector attribute has its own vector table.
Expand Down