feat: Merge branch import_braft

.github/workflows/import_braft.yml

@@ -0,0 +1,60 @@
+name: Import BRaft Actions (Temporary)
+
+on:
+  push:
+  pull_request:
+    branches: [ "import-braft" ]
+
+jobs:
+  check_format:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Build
+        run: bash ci/build.sh
+
+      - name: Check Format
+        working-directory: ${{ github.workspace }}/build
+        run: make check-format
+
+  build_on_macos:
+    runs-on: macos-latest
+    needs: check_format
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Build
+        env: 
+          CPLUS_INCLUDE_PATH: /opt/homebrew/include
+        run: |
+          brew install autoconf
+          brew install go
+          sh build.sh
+
+      - name: Run Go E2E Tests
+        working-directory: ${{ github.workspace }}/build
+        run: |
+          cd ../tests
+          go mod tidy
+          go test ./pikiwidb_suite_test.go ./consistency_test.go -v
+
+  build_on_ubuntu:
+    runs-on: ubuntu-latest
+    needs: check_format
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Build
+        run: |
+          bash build.sh
+
+      - name: Run Go E2E Tests
+        working-directory: ${{ github.workspace }}/build
+        run: |
+          cd ../tests
+          go mod tidy
+          go test ./pikiwidb_suite_test.go ./consistency_test.go -v

.github/workflows/pikiwidb.yml

@@ -27,6 +27,8 @@ jobs:
       - uses: actions/checkout@v4
 
       - name: Build
+        env:
+          CPLUS_INCLUDE_PATH: /opt/homebrew/include
         run: |
           brew install autoconf
           brew install go

CMakeLists.txt

@@ -1,7 +1,12 @@
+# Copyright (c) 2023-present, Qihoo, Inc.  All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree. An additional grant
+# of patent rights can be found in the PATENTS file in the same directory.
+
 CMAKE_MINIMUM_REQUIRED(VERSION 3.14)
 PROJECT(PikiwiDB)
 
-SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-deprecated -g -D'GIT_COMMIT_ID=\"${GIT_COMMIT_ID}\"'")
+SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror=return-type -D'GIT_COMMIT_ID=\"${GIT_COMMIT_ID}\"'")
 
 # Avoid warning about DOWNLOAD_EXTRACT_TIMESTAMP in CMake 3.24:
 IF (CMAKE_VERSION VERSION_GREATER_EQUAL "3.24.0")
@@ -35,7 +40,7 @@ EXECUTE_PROCESS(COMMAND git rev-parse HEAD OUTPUT_VARIABLE GIT_COMMIT_ID OUTPUT_
 ADD_DEFINITIONS(-DKPIKIWIDB_GIT_COMMIT_ID="${GIT_COMMIT_ID}")
 MESSAGE(STATUS "Git commit id: ${GIT_COMMIT_ID}")
 
-############# You should enable sanitizer if you are developing pika #############
+############# You should enable sanitizer if you are developing pikiwidb #############
 # Uncomment the following two lines to enable AddressSanitizer to detect memory leaks and other memory-related bugs.
 # SET(CMAKE_BUILD_TYPE "Debug")
 # SET(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -fsanitize=address  -O0 -fno-omit-frame-pointer -fno-optimize-sibling-calls")
@@ -78,7 +83,7 @@ ELSEIF (CMAKE_SYSTEM_NAME MATCHES "Linux")
         set(CMAKE_THREAD_LIBS_INIT "-lpthread")
     ELSEIF (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
         SET(CMAKE_EXE_LINKER_FLAGS "-static-libgcc -static-libstdc++")
-        SET(CMAKE_CXX_FLAGS "-pthread -Wl,--no-as-needed -ldl -Wno-restrict")
+        SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pthread -Wl,--no-as-needed -ldl")
     ENDIF ()
     ADD_DEFINITIONS(-DOS_LINUX)
 ELSE ()
@@ -101,28 +106,80 @@ SET(INSTALL_LIBDIR ${STAGED_INSTALL_PREFIX}/lib)
 SET(INSTALL_LIBDIR_64 ${STAGED_INSTALL_PREFIX}/lib64)
 SET(CMAKE_PREFIX_PATH ${CMAKE_PREFIX_PATH} ${STAGED_INSTALL_PREFIX})
 SET(BUILD_SUPPORT_DIR ${PROJECT_SOURCE_DIR}/build_support)
+# make sure we use the same compiler for all dependencies
+SET(CMAKE_POSITION_INDEPENDENT_CODE ON)
 
 MESSAGE(STATUS "${PROJECT_NAME} staged install: ${STAGED_INSTALL_PREFIX}")
 MESSAGE(STATUS "Current platform: ${OS_VERSION} ")
 CMAKE_HOST_SYSTEM_INFORMATION(RESULT CPU_CORE QUERY NUMBER_OF_LOGICAL_CORES)
 MESSAGE(STATUS "CPU core ${CPU_CORE}")
 
+#openssl
+FIND_PACKAGE(OpenSSL REQUIRED) 
+
+MESSAGE(STATUS "ssl:" ${OPENSSL_SSL_LIBRARY})
+MESSAGE(STATUS "crypto:" ${OPENSSL_CRYPTO_LIBRARY})
+
+ADD_LIBRARY(ssl SHARED IMPORTED GLOBAL)
+SET_PROPERTY(TARGET ssl PROPERTY IMPORTED_LOCATION ${OPENSSL_SSL_LIBRARY})
+
+ADD_LIBRARY(crypto SHARED IMPORTED GLOBAL)
+SET_PROPERTY(TARGET crypto PROPERTY IMPORTED_LOCATION ${OPENSSL_CRYPTO_LIBRARY})
+
+SET(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
+
+SET(THIRD_PARTY_PATH ${CMAKE_CURRENT_BINARY_DIR}/third-party)
+SET(THIRD_PARTY_BUILD_TYPE ${CMAKE_BUILD_TYPE})
+SET(EXTERNAL_PROJECT_LOG_ARGS
+        LOG_DOWNLOAD 0
+        LOG_UPDATE 1
+        LOG_CONFIGURE 0
+        LOG_BUILD 0
+        LOG_TEST 1
+        LOG_INSTALL 0)
+
+IF(CMAKE_SYSTEM_NAME STREQUAL "Linux")
+    SET(LIB rt)
+ELSEIF(CMAKE_SYSTEM_NAME STREQUAL "Darwin")
+    SET(LIB
+            pthread
+            "-framework CoreFoundation"
+            "-framework CoreGraphics"
+            "-framework CoreData"
+            "-framework CoreText"
+            "-framework Security"
+            "-framework Foundation"
+            "-Wl,-U,_MallocExtension_ReleaseFreeMemory"
+            "-Wl,-U,_ProfilerStart"
+            "-Wl,-U,_ProfilerStop"
+            "-Wl,-U,__Z13GetStackTracePPvii")
+ENDIF()
+SET(LIB ${LIB} CACHE INTERNAL "libs which should be linked for executable target")
 
 INCLUDE(FetchContent)
-#include(cmake/CmakeLists.txt)
 
-include(cmake/findTools.cmake)
-include(cmake/libevent.cmake)
-include(cmake/llhttp.cmake)
-include(cmake/fmt.cmake)
-include(cmake/spdlog.cmake)
-include(cmake/gtest.cmake)
-include(cmake/rocksdb.cmake)
+INCLUDE(cmake/gflags.cmake)
+INCLUDE(cmake/findTools.cmake)
+INCLUDE(cmake/leveldb.cmake)
+INCLUDE(cmake/libevent.cmake)
+INCLUDE(cmake/llhttp.cmake)
+INCLUDE(cmake/fmt.cmake)
+INCLUDE(cmake/spdlog.cmake)
+INCLUDE(cmake/gtest.cmake)
+INCLUDE(cmake/rocksdb.cmake)
+INCLUDE(cmake/zlib.cmake)
+INCLUDE(cmake/protobuf.cmake)
+INCLUDE(cmake/brpc.cmake)
+INCLUDE(cmake/braft.cmake)
+
+ENABLE_TESTING()
 
-enable_testing()
+SET(PROTO_OUTPUT_DIR "${CMAKE_BINARY_DIR}/generated_pb")
+FILE(MAKE_DIRECTORY "${PROTO_OUTPUT_DIR}")
 
 ADD_SUBDIRECTORY(src/pstd)
 ADD_SUBDIRECTORY(src/net)
+ADD_SUBDIRECTORY(src/praft)
 ADD_SUBDIRECTORY(src/storage)
 ADD_SUBDIRECTORY(src)
 
@@ -177,4 +234,3 @@ ADD_CUSTOM_TARGET(cpplint echo '${LINT_FILES}' | xargs -n12 -P8
   --linelength=120
   --filter=-legal/copyright,-build/header_guard,-runtime/references
 )
-

README_CN.md

@@ -69,7 +69,6 @@ PikiwiDB 可以和 Redis 之间进行复制，可以读取 Redis 的 rdb 文件
 这些特性 PikiwiDB 都有:-)
 
 ## 持久化：内存不再是上限
-
 RocksDB 可以配置为 PikiwiDB 的持久化存储引擎，可以存储更多的数据。
 
 ## 命令列表

build.sh

@@ -6,7 +6,13 @@ C_GREEN="\033[32m"
 
 C_END="\033[0m"
 
-BUILD_TYPE=release
+BUILD_TIME=$(git log -1 --format=%ai)
+BUILD_TIME=${BUILD_TIME: 0: 10}
+
+COMMIT_ID=$(git rev-parse HEAD)
+SHORT_COMMIT_ID=${COMMIT_ID: 0: 8}
+
+BUILD_TYPE=Release
 VERBOSE=0
 CMAKE_FLAGS=""
 MAKE_FLAGS=""

cmake/braft.cmake

@@ -0,0 +1,46 @@
+# Copyright (c) 2023-present, Qihoo, Inc.  All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree. An additional grant
+# of patent rights can be found in the PATENTS file in the same directory.
+
+INCLUDE(ExternalProject)
+
+SET(BRAFT_SOURCES_DIR ${THIRD_PARTY_PATH}/braft)
+SET(BRAFT_INSTALL_DIR ${THIRD_PARTY_PATH}/install/braft)
+SET(BRAFT_INCLUDE_DIR "${BRAFT_INSTALL_DIR}/include" CACHE PATH "braft include directory." FORCE)
+SET(BRAFT_LIBRARIES "${BRAFT_INSTALL_DIR}/lib/libbraft.a" CACHE FILEPATH "braft library." FORCE)
+
+SET(prefix_path "${THIRD_PARTY_PATH}/install/brpc|${CMAKE_CURRENT_BINARY_DIR}/_deps/gflags-build|${THIRD_PARTY_PATH}/install/protobuf|${THIRD_PARTY_PATH}/install/zlib|${CMAKE_CURRENT_BINARY_DIR}/_deps/leveldb-build|${CMAKE_CURRENT_BINARY_DIR}/_deps/leveldb-src")
+
+ExternalProject_Add(
+        extern_braft
+        ${EXTERNAL_PROJECT_LOG_ARGS}
+        DEPENDS brpc
+        GIT_REPOSITORY "https://github.com/pikiwidb/braft.git"
+        GIT_TAG master
+        PREFIX ${BRAFT_SOURCES_DIR}
+        UPDATE_COMMAND ""
+        CMAKE_ARGS -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
+        -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
+        -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS}
+        -DCMAKE_C_FLAGS=${CMAKE_C_FLAGS}
+        -DCMAKE_INSTALL_PREFIX=${BRAFT_INSTALL_DIR}
+        -DCMAKE_INSTALL_LIBDIR=${BRAFT_INSTALL_DIR}/lib
+        -DCMAKE_POSITION_INDEPENDENT_CODE=ON
+        -DCMAKE_BUILD_TYPE=${THIRD_PARTY_BUILD_TYPE}
+        -DCMAKE_PREFIX_PATH=${prefix_path}
+        -DBRPC_WITH_GLOG=OFF
+        ${EXTERNAL_OPTIONAL_ARGS}
+        LIST_SEPARATOR |
+        CMAKE_CACHE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${BRAFT_INSTALL_DIR}
+        -DCMAKE_INSTALL_LIBDIR:PATH=${BRAFT_INSTALL_DIR}/lib
+        -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON
+        -DCMAKE_BUILD_TYPE:STRING=${THIRD_PARTY_BUILD_TYPE}
+        BUILD_IN_SOURCE 1
+        BUILD_COMMAND $(MAKE) -j ${CPU_CORE} braft-static
+        INSTALL_COMMAND mkdir -p ${BRAFT_INSTALL_DIR}/lib/ COMMAND cp ${BRAFT_SOURCES_DIR}/src/extern_braft/output/lib/libbraft.a ${BRAFT_LIBRARIES} COMMAND cp -r ${BRAFT_SOURCES_DIR}/src/extern_braft/output/include ${BRAFT_INCLUDE_DIR}/
+)
+ADD_DEPENDENCIES(extern_braft brpc)
+ADD_LIBRARY(braft STATIC IMPORTED GLOBAL)
+SET_PROPERTY(TARGET braft PROPERTY IMPORTED_LOCATION ${BRAFT_LIBRARIES})
+ADD_DEPENDENCIES(braft extern_braft)

cmake/brpc.cmake

@@ -0,0 +1,49 @@
+# Copyright (c) 2023-present, Qihoo, Inc.  All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree. An additional grant
+# of patent rights can be found in the PATENTS file in the same directory.
+
+INCLUDE(ExternalProject)
+
+SET(BRPC_SOURCES_DIR ${THIRD_PARTY_PATH}/brpc)
+SET(BRPC_INSTALL_DIR ${THIRD_PARTY_PATH}/install/brpc)
+SET(BRPC_INCLUDE_DIR "${BRPC_INSTALL_DIR}/include" CACHE PATH "brpc include directory." FORCE)
+SET(BRPC_LIBRARIES "${BRPC_INSTALL_DIR}/lib/libbrpc.a" CACHE FILEPATH "brpc library." FORCE)
+
+# Reference https://stackoverflow.com/questions/45414507/pass-a-list-of-prefix-paths-to-externalproject-add-in-cmake-args
+SET(prefix_path "${CMAKE_CURRENT_BINARY_DIR}/_deps/gflags-build|${THIRD_PARTY_PATH}/install/protobuf|${THIRD_PARTY_PATH}/install/zlib|${CMAKE_CURRENT_BINARY_DIR}/_deps/leveldb-build|${CMAKE_CURRENT_BINARY_DIR}/_deps/leveldb-src")
+SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-deprecated-declarations")
+# If minimal .a is need, you can set  WITH_DEBUG_SYMBOLS=OFF
+EXTERNALPROJECT_ADD( 
+        extern_brpc
+        ${EXTERNAL_PROJECT_LOG_ARGS}
+        DEPENDS ssl crypto zlib protobuf leveldb gflags
+        URL https://github.com/apache/brpc/archive/refs/tags/1.8.0.tar.gz
+        URL_HASH SHA256=13ffb2f1f57c679379a20367c744b3e597614a793ec036cd7580aae90798019d
+        PREFIX ${BRPC_SOURCES_DIR}
+        UPDATE_COMMAND ""
+        CMAKE_ARGS -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
+        -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
+        -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS}
+        -DCMAKE_C_FLAGS=${CMAKE_C_FLAGS}
+        -DCMAKE_INSTALL_PREFIX=${BRPC_INSTALL_DIR}
+        -DCMAKE_INSTALL_LIBDIR=${BRPC_INSTALL_DIR}/lib
+        -DCMAKE_POSITION_INDEPENDENT_CODE=ON
+        -DCMAKE_BUILD_TYPE=${THIRD_PARTY_BUILD_TYPE}
+        -DCMAKE_PREFIX_PATH=${prefix_path}
+        -DWITH_GLOG=OFF
+        -DDOWNLOAD_GTEST=OFF
+        ${EXTERNAL_OPTIONAL_ARGS}
+        LIST_SEPARATOR |
+        CMAKE_CACHE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${BRPC_INSTALL_DIR}
+        -DCMAKE_INSTALL_LIBDIR:PATH=${BRPC_INSTALL_DIR}/lib
+        -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON
+        -DCMAKE_BUILD_TYPE:STRING=${THIRD_PARTY_BUILD_TYPE}
+        BUILD_IN_SOURCE 1
+        BUILD_COMMAND $(MAKE) -j ${CPU_CORE} brpc-static
+        INSTALL_COMMAND mkdir -p ${BRPC_INSTALL_DIR}/lib/ COMMAND cp ${BRPC_SOURCES_DIR}/src/extern_brpc/output/lib/libbrpc.a ${BRPC_LIBRARIES} COMMAND cp -r ${BRPC_SOURCES_DIR}/src/extern_brpc/output/include ${BRPC_INCLUDE_DIR}/
+)
+ADD_DEPENDENCIES(extern_brpc ssl crypto zlib protobuf leveldb gflags)
+ADD_LIBRARY(brpc STATIC IMPORTED GLOBAL)
+SET_PROPERTY(TARGET brpc PROPERTY IMPORTED_LOCATION ${BRPC_LIBRARIES})
+ADD_DEPENDENCIES(brpc extern_brpc)