Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

refactor: Move some common files out of rdsn directory #1079

Merged
merged 2 commits into from
Jul 26, 2022
Merged

refactor: Move some common files out of rdsn directory #1079

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
6519e3c
refactor: Move some common files out of rdsn directory
acelyc111 Jul 25, 2022
2f1896d
.gitignore for .DS_Store
acelyc111 Jul 25, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 11 additions & 6 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,14 @@
# under the License.

src/.matchfile
src/base/rrdb_types.cpp
src/include/rrdb/rrdb_types.h
src/rdsn/builder/
src/rdsn/include/dsn/cpp/serialization_helper/dsn.layer2_types.h
src/rdsn/src/runtime/dsn.layer2_types.cpp
src/runtime
src/builder

onebox/
.zk_install/
*.data/
Expand Down Expand Up @@ -263,9 +270,7 @@ package-lock.json
#go
go-client/bin

# rdsn
rdsn/builder
thirdparty/build
thirdparty/output
rdsn/cmake-build-debug
rdsn/test_reports
thirdparty/output/

#macOS
.DS_Store
8 changes: 4 additions & 4 deletions .licenserc.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,9 +94,11 @@ header:
- 'src/rdsn/src/utils/test/config-empty.ini'
# The MIT License (MIT), Copyright (c) 2015 Microsoft Corporation
- 'cmake_modules/BaseFunctions.cmake'
- 'docs/rdsn-README.md'
- 'idl/dsn.layer2.thrift'
- 'idl/dsn.thrift'
- 'scripts/compile_thrift.py'
- 'scripts/learn_stat.py'
- 'src/rdsn/README.md'
- 'src/rdsn/compile_thrift.py'
- 'src/rdsn/include/dsn/c/api_common.h'
- 'src/rdsn/include/dsn/c/api_layer1.h'
- 'src/rdsn/include/dsn/c/api_task.h'
Expand Down Expand Up @@ -234,8 +236,6 @@ header:
- 'src/rdsn/src/common/test/duplication_common_test.cpp'
- 'src/rdsn/src/common/test/main.cpp'
- 'src/rdsn/src/common/test/run.sh'
- 'src/rdsn/src/dsn.layer2.thrift'
- 'src/rdsn/src/dsn.thrift'
- 'src/rdsn/src/failure_detector/failure_detector.cpp'
- 'src/rdsn/src/failure_detector/failure_detector_multimaster.cpp'
- 'src/rdsn/src/failure_detector/fd.thrift'
Expand Down
0 src/rdsn/CONTRIBUTING.md → docs/CONTRIBUTING.md
View file
Open in desktop
File renamed without changes.
0 src/rdsn/README.md → docs/rdsn-README.md
View file
Open in desktop
File renamed without changes.
0 src/rdsn/resources/arch.png → docs/resources/arch.png
View file
Open in desktop
File renamed without changes
0 src/rdsn/resources/config.png → docs/resources/config.png
View file
Open in desktop
File renamed without changes
0 src/rdsn/resources/rdsn-layer1.jpg → docs/resources/rdsn-layer1.jpg
View file
Open in desktop
File renamed without changes
0 src/rdsn/resources/rdsn-layer2.jpg → docs/resources/rdsn-layer2.jpg
View file
Open in desktop
File renamed without changes
0 src/rdsn/resources/rdsn-layer3.jpg → docs/resources/rdsn-layer3.jpg
View file
Open in desktop
File renamed without changes
0 src/rdsn/resources/rdsn-state-all.jpg → docs/resources/rdsn-state-all.jpg
View file
Open in desktop
File renamed without changes
0 src/rdsn/resources/rdsn.jpg → docs/resources/rdsn.jpg
View file
Open in desktop
File renamed without changes
0 src/rdsn/resources/viz.png → docs/resources/viz.png
View file
Open in desktop
File renamed without changes
1 change: 1 addition & 0 deletions src/rdsn/src/dsn.layer2.thrift → idl/dsn.layer2.thrift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@

include "dsn.thrift"

# TODO(yingchun): reuse the idls for server and all client libs
namespace cpp dsn

struct partition_configuration
Expand Down
0 src/rdsn/src/dsn.thrift → idl/dsn.thrift
View file
Open in desktop
File renamed without changes.
0 src/idl/rrdb.thrift → idl/rrdb.thrift
View file
Open in desktop
File renamed without changes.
7 changes: 2 additions & 5 deletions run.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -249,9 +249,8 @@ function run_build()
fi

echo "INFO: Start build rdsn..."
cd $ROOT/src/rdsn
echo "Gen rdsn thrift"
python3 compile_thrift.py
python3 $ROOT/scripts/compile_thrift.py
BUILD_DIR="$ROOT/src/rdsn/builder"
if [ "$CLEAR" == "YES" ]; then
echo "Clear $BUILD_DIR ..."
Expand Down Expand Up @@ -285,9 +284,7 @@ function run_build()

echo "INFO: start build pegasus..."
echo "Gen pegasus thrift"
pushd $ROOT/src/idl
sh recompile_thrift.sh
popd
sh scripts/recompile_thrift.sh

cd "$ROOT/src"
PEGASUS_GIT_COMMIT="non-git-repo"
Expand Down
15 changes: 6 additions & 9 deletions src/rdsn/compile_thrift.py → scripts/compile_thrift.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@
thrift_description = [
{
"name": "dsn.layer2",
"path": "src",
"path": "idl",
"include_fix": {
"_types.h": {
"add": ["<dsn/cpp/serialization_helper/dsn_types.h>"],
Expand All @@ -48,8 +48,8 @@
}
},
"file_move": {
"_types.h": "include/dsn/cpp/serialization_helper",
"_types.cpp": "src/runtime"
"_types.h": "src/rdsn/include/dsn/cpp/serialization_helper",
"_types.cpp": "src/rdsn/src/runtime"
}
},
]
Expand Down Expand Up @@ -219,8 +219,8 @@ def add_hook(name, path, func, args):


if __name__ == "__main__":
thrift_exe = os.getcwd() + "/../../thirdparty/output/bin/thrift"
root_dir = os.getcwd()
thrift_exe = root_dir + "/thirdparty/output/bin/thrift"
print("thrift_exe = " + thrift_exe)
print("root_dir = " + root_dir)

Expand All @@ -230,11 +230,8 @@ def add_hook(name, path, func, args):

ctor_kv_pair = " kv_pair(const std::string& _key, const std::string& _val): key(_key), value(_val) {\n }"
ctor_configuration_proposal_action = " configuration_proposal_action(::dsn::rpc_address t, ::dsn::rpc_address n, config_type::type tp): target(t), node(n), type(tp) {}"
add_hook("simple_kv", "src/replica/storage/simple_kv", constructor_hook,
["simple_kv_types.h", "kv_pair", ctor_kv_pair])
add_hook("replication", "src/", constructor_hook,
["replication_types.h", "configuration_proposal_action", ctor_configuration_proposal_action])
add_hook("dsn.layer2", "src", replace_hook, ["dsn.layer2_types.h", {

add_hook("dsn.layer2", "idl", replace_hook, ["dsn.layer2_types.h", {
r"dsn\.layer2_TYPES_H": 'dsn_layer2_TYPES_H'}])

for i in thrift_description:
Expand Down
12 changes: 4 additions & 8 deletions src/idl/recompile_thrift.sh → scripts/recompile_thrift.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
# under the License.

cd `dirname $0`
THIRDPARTY_ROOT=../../thirdparty
THIRDPARTY_ROOT=../thirdparty

if [ ! -d "$THIRDPARTY_ROOT" ]; then
echo "ERROR: THIRDPARTY_ROOT not set"
Expand All @@ -28,16 +28,12 @@ TMP_DIR=./tmp
rm -rf $TMP_DIR

mkdir -p $TMP_DIR
$THIRDPARTY_ROOT/output/bin/thrift --gen cpp:moveable_types -out $TMP_DIR rrdb.thrift
$THIRDPARTY_ROOT/output/bin/thrift --gen cpp:moveable_types -out $TMP_DIR ../idl/rrdb.thrift

sed 's/#include "dsn_types.h"/#include <dsn\/service_api_cpp.h>/' $TMP_DIR/rrdb_types.h > ../include/rrdb/rrdb_types.h
sed 's/#include "rrdb_types.h"/#include <rrdb\/rrdb_types.h>/' $TMP_DIR/rrdb_types.cpp > ../base/rrdb_types.cpp
sed 's/#include "dsn_types.h"/#include <dsn\/service_api_cpp.h>/' $TMP_DIR/rrdb_types.h > ../src/include/rrdb/rrdb_types.h
sed 's/#include "rrdb_types.h"/#include <rrdb\/rrdb_types.h>/' $TMP_DIR/rrdb_types.cpp > ../src/base/rrdb_types.cpp

rm -rf $TMP_DIR

echo
echo "You should manually modify these files:"
echo " src/include/rrdb/rrdb.code.definition.h"
echo " src/include/rrdb/rrdb.client.h"
echo
echo "done"
28 changes: 0 additions & 28 deletions src/idl/dsn.thrift
View file
Open in desktop

This file was deleted.

4 changes: 2 additions & 2 deletions src/rdsn/src/common/backup.thrift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,8 @@
// specific language governing permissions and limitations
// under the License.

include "../dsn.thrift"
include "../dsn.layer2.thrift"
include "../../../../idl/dsn.thrift"
include "../../../../idl/dsn.layer2.thrift"

namespace cpp dsn.replication

Expand Down
4 changes: 2 additions & 2 deletions src/rdsn/src/common/bulk_load.thrift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,8 @@
// specific language governing permissions and limitations
// under the License.

include "../dsn.thrift"
include "../dsn.layer2.thrift"
include "../../../../idl/dsn.thrift"
include "../../../../idl/dsn.layer2.thrift"
include "metadata.thrift"

namespace cpp dsn.replication
Expand Down
4 changes: 2 additions & 2 deletions src/rdsn/src/common/consensus.thrift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,8 @@
* THE SOFTWARE.
*/

include "../dsn.thrift"
include "../dsn.layer2.thrift"
include "../../../../idl/dsn.thrift"
include "../../../../idl/dsn.layer2.thrift"
include "metadata.thrift"

namespace cpp dsn.replication
Expand Down
4 changes: 2 additions & 2 deletions src/rdsn/src/common/duplication.thrift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,8 @@
// specific language governing permissions and limitations
// under the License.

include "../dsn.thrift"
include "../dsn.layer2.thrift"
include "../../../../idl/dsn.thrift"
include "../../../../idl/dsn.layer2.thrift"

namespace cpp dsn.replication

Expand Down
4 changes: 2 additions & 2 deletions src/rdsn/src/common/meta_admin.thrift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,8 @@
* THE SOFTWARE.
*/

include "../dsn.thrift"
include "../dsn.layer2.thrift"
include "../../../../idl/dsn.thrift"
include "../../../../idl/dsn.layer2.thrift"
include "metadata.thrift"

namespace cpp dsn.replication
Expand Down
4 changes: 2 additions & 2 deletions src/rdsn/src/common/metadata.thrift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,8 @@
* THE SOFTWARE.
*/

include "../dsn.thrift"
include "../dsn.layer2.thrift"
include "../../../../idl/dsn.thrift"
include "../../../../idl/dsn.layer2.thrift"

namespace cpp dsn.replication

Expand Down
4 changes: 2 additions & 2 deletions src/rdsn/src/common/partition_split.thrift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,8 @@
// specific language governing permissions and limitations
// under the License.

include "../dsn.thrift"
include "../dsn.layer2.thrift"
include "../../../../idl/dsn.thrift"
include "../../../../idl/dsn.layer2.thrift"
include "metadata.thrift"

namespace cpp dsn.replication
Expand Down
4 changes: 2 additions & 2 deletions src/rdsn/src/common/replica_admin.thrift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,8 @@
* THE SOFTWARE.
*/

include "../dsn.thrift"
include "../dsn.layer2.thrift"
include "../../../../idl/dsn.thrift"
include "../../../../idl/dsn.layer2.thrift"
include "metadata.thrift"

namespace cpp dsn.replication
Expand Down
2 changes: 1 addition & 1 deletion src/rdsn/src/failure_detector/fd.thrift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
* THE SOFTWARE.
*/

include "../dsn.thrift"
include "../../../../idl/dsn.thrift"

namespace cpp dsn.fd

Expand Down
2 changes: 1 addition & 1 deletion src/rdsn/src/nfs/nfs.thrift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
* THE SOFTWARE.
*/

include "../dsn.thrift"
include "../../../../idl/dsn.thrift"

namespace cpp dsn.service

Expand Down
2 changes: 1 addition & 1 deletion src/rdsn/src/runtime/security/security.thrift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.

include "../../dsn.thrift"
include "../../../../../idl/dsn.thrift"
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

dsn.thrift in rdsn/src/ looks fine to me, why did you move it to an outer directory?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

dsn.thrift contains some common thrift structures which will be used for not only rdsn, but also client libs. Its name 'dsn.thrift' is not accurate, maybe 'base.thrift' would be better, like it named in client lib directories(e.g. https://github.com/apache/incubator-pegasus/blob/master/java-client/idl/base.thrift).

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Of course, we will refactor the code and only leave one unique base.thrift in Pegasus later.


namespace cpp dsn.security

Expand Down