Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add packed bit field support to Row Structure #493

Closed
wants to merge 4 commits into from
Closed
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
Add column type enum for dynamic mapping
Differential Revision: D43341286

fbshipit-source-id: b6a0e068caa5ad5da40729eceee3dad71858fe84
Tal Davidi authored and facebook-github-bot committed Feb 22, 2023

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature. The key has expired.
commit 62e9579e1ffac48818796a6cb4cf7905ad02e1e6
Original file line number Diff line number Diff line change
@@ -8,6 +8,7 @@
#pragma once

#include <memory>
#include <stdexcept>
#include "fbpcf/mpc_std_lib/unified_data_process/serialization/IColumnDefinition.h"

namespace fbpcf::mpc_std_lib::unified_data_process::serialization {
@@ -31,6 +32,23 @@ class FixedSizeArrayColumn : public IColumnDefinition<schedulerId> {
return length_ * innerType_->getColumnSizeBytes();
}

typename IColumnDefinition<schedulerId>::SupportedColumnTypes getColumnType()
const override {
auto innerColumnType = innerType_->getColumnType();

switch (innerColumnType) {
case IColumnDefinition<schedulerId>::SupportedColumnTypes::UInt32:
return IColumnDefinition<schedulerId>::SupportedColumnTypes::UInt32Vec;
case IColumnDefinition<schedulerId>::SupportedColumnTypes::Int32:
return IColumnDefinition<schedulerId>::SupportedColumnTypes::Int32Vec;
case IColumnDefinition<schedulerId>::SupportedColumnTypes::Int64:
return IColumnDefinition<schedulerId>::SupportedColumnTypes::Int64Vec;
default:
throw std::runtime_error(
"This code should be unreachable. Tried to get invalid Array Column");
}
}

size_t getLength() const {
return length_;
}
Original file line number Diff line number Diff line change
@@ -21,6 +21,17 @@ class IColumnDefinition {
using MPCTypes = frontend::MPCTypes<schedulerId, true /* usingBatch */>;

public:
enum SupportedColumnTypes {
Bit = 0,
PackedBitField = 1,
UInt32 = 2,
Int32 = 3,
Int64 = 4,
UInt32Vec = 5,
Int32Vec = 6,
Int64Vec = 7,
};

/* Possible return types for deserialization following UDP run */
using DeserializeType = std::variant<
typename MPCTypes::SecBool,
@@ -38,6 +49,8 @@ class IColumnDefinition {

virtual size_t getColumnSizeBytes() const = 0;

virtual SupportedColumnTypes getColumnType() const = 0;

/* Pass in a single value of the column to be serialized, sequentially write
* the bytes starting at the beginning of buf */
virtual void serializeColumnAsPlaintextBytes(
Original file line number Diff line number Diff line change
@@ -10,7 +10,11 @@
#include "fbpcf/frontend/Int.h"
#include "fbpcf/mpc_std_lib/unified_data_process/serialization/IColumnDefinition.h"

#include <stdexcept>
#include <string>

#include "folly/Format.h"

namespace fbpcf::mpc_std_lib::unified_data_process::serialization {

template <int schedulerId, bool isSigned, int8_t width>
@@ -51,6 +55,26 @@ class IntegerColumn : public IColumnDefinition<schedulerId> {
return width / 8;
}

typename IColumnDefinition<schedulerId>::SupportedColumnTypes getColumnType()
const override {
static_assert(
(isSigned && (width == 32 || width == 64)) || width == 32,
"For now only support int32, int64, uint64");
if constexpr (isSigned) {
if constexpr (width == 32) {
return IColumnDefinition<schedulerId>::SupportedColumnTypes::Int32;
} else if constexpr (width == 64) {
return IColumnDefinition<schedulerId>::SupportedColumnTypes::Int64;
}
} else if constexpr (width == 32) {
return IColumnDefinition<schedulerId>::SupportedColumnTypes::UInt32;
}
throw std::runtime_error(folly::sformat(
"This code should be unreachable. {}int{}_t column type is not supported",
isSigned ? "" : "u",
width));
}

void serializeColumnAsPlaintextBytes(
const void* inputData,
unsigned char* buf) const override {
Original file line number Diff line number Diff line change
@@ -42,6 +42,11 @@ class PackedBitFieldColumn : public IColumnDefinition<schedulerId> {
return 1;
}

typename IColumnDefinition<schedulerId>::SupportedColumnTypes getColumnType()
const override {
return IColumnDefinition<schedulerId>::SupportedColumnTypes::PackedBitField;
}

// input to this function is a pointer to a bool vector since memory layout
// is not guaranteed by compiler (i.e. can not get a bool* from a
// vector<bool>.data())
Original file line number Diff line number Diff line change
@@ -299,4 +299,39 @@ TEST(SerializationTest, PackedBitFieldColumnTest) {
testVectorEq(vals[j], rst[j]);
}
}

TEST(erializationTest, ColumnTypeTest) {
using ColType = IColumnDefinition<0>::SupportedColumnTypes;
std::unique_ptr<IColumnDefinition<0>> col0 =
std::make_unique<IntegerColumn<0, true, 32>>("col0");
EXPECT_EQ(col0->getColumnType(), ColType::Int32);

std::unique_ptr<IColumnDefinition<0>> col1 =
std::make_unique<IntegerColumn<0, true, 64>>("col1");
EXPECT_EQ(col1->getColumnType(), ColType::Int64);

std::unique_ptr<IColumnDefinition<0>> col2 =
std::make_unique<IntegerColumn<0, false, 32>>("col2");
EXPECT_EQ(col2->getColumnType(), ColType::UInt32);

std::vector<std::string> names{"bool1", "bool2"};
std::unique_ptr<IColumnDefinition<0>> col3 =
std::make_unique<PackedBitFieldColumn<0>>("col3", names);
EXPECT_EQ(col3->getColumnType(), ColType::PackedBitField);

std::unique_ptr<IColumnDefinition<0>> col4 = std::make_unique<
FixedSizeArrayColumn<0, frontend::MPCTypes<0>::Sec32Int>>(
"col4", std::make_unique<IntegerColumn<0, true, 32>>("test"), 4);
EXPECT_EQ(col4->getColumnType(), ColType::Int32Vec);

std::unique_ptr<IColumnDefinition<0>> col5 = std::make_unique<
FixedSizeArrayColumn<0, frontend::MPCTypes<0>::Sec64Int>>(
"col4", std::make_unique<IntegerColumn<0, true, 64>>("test"), 4);
EXPECT_EQ(col5->getColumnType(), ColType::Int64Vec);

std::unique_ptr<IColumnDefinition<0>> col6 = std::make_unique<
FixedSizeArrayColumn<0, frontend::MPCTypes<0>::SecUnsigned32Int>>(
"col4", std::make_unique<IntegerColumn<0, false, 32>>("test"), 4);
EXPECT_EQ(col6->getColumnType(), ColType::UInt32Vec);
}
} // namespace fbpcf::mpc_std_lib::unified_data_process::serialization