Skip to content

Commit

Permalink
[jit] jit support property.proto (PaddlePaddle#44337)
Browse files Browse the repository at this point in the history
* add property.proto, can compiled

* property get and deserilize

* support get float

* format code

* format code

* add unittest

* add more set method

* fix grammar error

* Update paddle/fluid/jit/property.h

Co-authored-by: Aurelius84 <zhangliujie@baidu.com>

* Update paddle/fluid/jit/property.cc

Co-authored-by: Aurelius84 <zhangliujie@baidu.com>

* Update paddle/fluid/jit/property.cc

Co-authored-by: Aurelius84 <zhangliujie@baidu.com>

* Update paddle/fluid/jit/property.cc

Co-authored-by: Aurelius84 <zhangliujie@baidu.com>

* fix comment

* fix error throw

* fix property save unit test

* fix error info

* fix copyright and header import

* reorder jit property tensor datatype

Co-authored-by: Aurelius84 <zhangliujie@baidu.com>
  • Loading branch information
zh794390558 and Aurelius84 committed Jul 29, 2022
1 parent db1f3d4 commit bc8426b
Show file tree
Hide file tree
Showing 10 changed files with 612 additions and 1 deletion.
7 changes: 7 additions & 0 deletions paddle/fluid/jit/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -52,3 +52,10 @@ if(WITH_TESTING AND NOT WIN32)
DEPS ${JIT_DEPS})
add_dependencies(layer_test jit_download_program)
endif()

proto_library(paddle_jit_property_proto SRCS property.proto)

cc_library(
jit_property
SRCS property.cc
DEPS paddle_jit_property_proto)
173 changes: 173 additions & 0 deletions paddle/fluid/jit/property.cc
Original file line number Diff line number Diff line change
@@ -0,0 +1,173 @@
/* Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */

#include "paddle/fluid/jit/property.h"
#include "glog/logging.h"
#include "paddle/phi/core/enforce.h"
#include "paddle/phi/core/errors.h"

namespace paddle {
namespace jit {

int Property::Size() const { return property_.entrys_size(); }

void Property::SetFloat(const float &f) {
auto type = proto::ValueProto::FLOAT;
auto entry = property_.add_entrys();
entry->set_type(type);
entry->set_f(f);
VLOG(3) << "Property: set_float " << f;
}

void Property::SetFloat(const std::string &name, const float &f) {
auto type = proto::ValueProto::FLOAT;
auto entry = property_.add_entrys();
entry->set_name(name);
entry->set_type(type);
entry->set_f(f);
VLOG(3) << "Property: set_float " << f << " name: " << name;
}

float Property::GetFloat(const std::string &name) const {
for (int i = 0; i < Size(); i++) {
auto e = property_.entrys(i);
if (e.has_name() && e.name() == name) {
return e.f();
}
}

PADDLE_THROW(phi::errors::NotFound(
"JIT::Property GetFloat: name: %s not found", name));
return 0;
}

float Property::GetFloat(const int &idx) const {
PADDLE_ENFORCE_EQ(
idx < Size() && idx >= 0,
true,
phi::errors::OutOfRange(
"JIT::Property GetFloat: idx=%d out of range %d", idx, Size()));

auto e = property_.entrys(idx);
if (e.has_f()) {
return e.f();
}

PADDLE_THROW(phi::errors::InvalidArgument(
"JIT::Property GetFloat: input idx (%d) element is not a float.", idx));
return 0;
}

void Property::SetFloats(const std::vector<float> &v) {
auto type = proto::ValueProto::FLOATS;
auto entry = property_.add_entrys();
entry->set_type(type);
for (auto i : v) {
entry->add_floats(i);
}
VLOG(3) << "Property: set_floats with length: " << v.size();
}

void Property::SetFloats(const std::string &name, const std::vector<float> &v) {
auto type = proto::ValueProto::FLOATS;
auto entry = property_.add_entrys();
entry->set_name(name);
entry->set_type(type);
for (auto i : v) {
entry->add_floats(i);
}
VLOG(3) << "Property: set_floats with length " << v.size()
<< " for name: " << name;
}

void Property::SetInt64(const int64_t &i) {
auto type = proto::ValueProto::INT;
auto entry = property_.add_entrys();
entry->set_type(type);
entry->set_i(i);
VLOG(3) << "Property: set_int " << i;
}

void Property::SetInt64(const std::string &name, const int64_t &i) {
auto type = proto::ValueProto::INT;
auto entry = property_.add_entrys();
entry->set_name(name);
entry->set_type(type);
entry->set_i(i);
VLOG(3) << "Property: set_int " << i << " name: " << name;
}

void Property::SetInt64s(const std::vector<int64_t> &v) {
auto type = proto::ValueProto::INTS;
auto entry = property_.add_entrys();
entry->set_type(type);
for (auto e : v) {
entry->add_ints(e);
}
VLOG(3) << "Property: set_ints " << v.size();
}

void Property::SetInt64s(const std::string &name,
const std::vector<int64_t> &v) {
auto type = proto::ValueProto::INTS;
auto entry = property_.add_entrys();
entry->set_name(name);
entry->set_type(type);
for (auto i : v) {
entry->add_ints(i);
}
VLOG(3) << "Property: set_ints " << v[0] << " name: " << name;
}

void Property::SetString(const std::string &s) {
auto type = proto::ValueProto::STRING;
auto entry = property_.add_entrys();
entry->set_type(type);
entry->set_s(s);
VLOG(3) << "Property: set_string with value : " << s;
}

void Property::SetString(const std::string &name, const std::string &s) {
auto type = proto::ValueProto::STRING;
auto entry = property_.add_entrys();
entry->set_name(name);
entry->set_type(type);
entry->set_s(s);
VLOG(3) << "Property: set_string " << s << " name: " << name;
}

void Property::SetStrings(const std::vector<std::string> &v) {
auto type = proto::ValueProto::STRINGS;
auto entry = property_.add_entrys();
entry->set_type(type);
for (auto i : v) {
entry->add_strings(i);
}
VLOG(3) << "Property: set_strings " << v.size();
}

void Property::SetStrings(const std::string &name,
const std::vector<std::string> &v) {
auto type = proto::ValueProto::STRINGS;
auto entry = property_.add_entrys();
entry->set_name(name);
entry->set_type(type);
for (auto i : v) {
entry->add_strings(i);
}
VLOG(3) << "Property: set_strings " << v[0] << " name: " << name;
}

} // namespace jit
} // namespace paddle
93 changes: 93 additions & 0 deletions paddle/fluid/jit/property.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#pragma once

#include <algorithm>
#include <atomic>
#include <string>
#include <vector>

#include "paddle/fluid/jit/property.pb.h"

namespace paddle {
namespace jit {

class Property {
public:
Property() {}

// Explicitly implement the copy constructor for auto parallel
explicit Property(const Property &other)
: property_(other.property_), original_id_(other.original_id_) {}

Property &operator=(const Property &other) {
property_ = other.property_;
original_id_ = other.original_id_;
return *this;
}

proto::PropertyVals *Proto() { return &property_; }

const proto::PropertyVals *Proto() const { return &property_; }

int Size() const;

void SetFloat(const float &f);
void SetFloat(const std::string &name, const float &f);

void SetFloats(const std::vector<float> &v);
void SetFloats(const std::string &name, const std::vector<float> &v);

float GetFloat(const std::string &name) const;
float GetFloat(const int &idx) const;

void SetInt64(const int64_t &i);
void SetInt64(const std::string &name, const int64_t &i);

void SetInt64s(const std::vector<int64_t> &v);
void SetInt64s(const std::string &name, const std::vector<int64_t> &v);

void SetString(const std::string &s);
void SetString(const std::string &name, const std::string &s);

void SetStrings(const std::vector<std::string> &v);
void SetStrings(const std::string &name, const std::vector<std::string> &v);

// The Id() and OriginalId() are only used for auto parallel.
uint64_t Id() const { return id_; }
uint64_t OriginalId() const { return original_id_; }
void SetOriginalId(uint64_t original_id) { original_id_ = original_id; }

private:
proto::PropertyVals property_;

// This thread-safe implementation seems to be redudent since the neural
// networks are usually constructed in a single thread.
static uint64_t GenerateId() {
static std::atomic<std::uint64_t> uid{0};
return ++uid;
}

// Note: the id_ is unique for all Property (only for auto parallel).
uint64_t id_ = GenerateId();
// Note: the orignal_id_ is used for referring to the original Property
// that the current Property is built from (only for auto parallel).
// The default original_id_ is same as the id_, which means the
// current Property is not built from the other one.
uint64_t original_id_ = id_;
};

} // namespace jit
} // namespace paddle
Loading

0 comments on commit bc8426b

Please sign in to comment.