Skip to content

Commit

Permalink
fix concurent dependency build, more cleanups
Browse files Browse the repository at this point in the history
  • Loading branch information
mirostauder committed Nov 29, 2023
1 parent 68e256c commit 8de4e0e
Show file tree
Hide file tree
Showing 8 changed files with 132 additions and 73 deletions.
16 changes: 11 additions & 5 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -31,14 +31,15 @@ O2 := -O2
O1 := -O1
O3 := -O3 -mtune=native

EXTRALINK := #-pg
ALL_DEBUG := -ggdb -DDEBUG
NO_DEBUG :=
DEBUG := ${ALL_DEBUG}
#EXTRALINK := #-pg
ALL_DEBUG := $(O0) -ggdb -DDEBUG
NO_DEBUG := $(O2) -ggdb
DEBUG := $(ALL_DEBUG)
CURVER ?= 2.6.0
#export DEBUG
#export EXTRALINK
export MAKE
export CURVER=${CURVER:-2.6.0}
export CURVER

### detect compiler support for c++11/17
CPLUSPLUS := $(shell ${CC} -std=c++17 -dM -E -x c++ /dev/null 2>/dev/null | grep -F __cplusplus | grep -Po '\d\d\d\d\d\dL')
Expand Down Expand Up @@ -80,6 +81,7 @@ GROUPCHECK := $(shell getent group proxysql)

### main targets

.DEFAULT: default
.PHONY: default
default: build_src

Expand Down Expand Up @@ -185,11 +187,13 @@ build_src_testall: build_lib_testall
build_lib_testall: build_deps_debug
cd lib && OPTZ="${O0} -ggdb -DDEBUG -DTEST_AURORA -DTEST_GALERA -DTEST_GROUPREP -DTEST_READONLY -DTEST_REPLICATIONLAG" CC=${CC} CXX=${CXX} ${MAKE}

.PHONY: build_tap_test
build_tap_test: build_tap_tests
.PHONY: build_tap_tests
build_tap_tests: build_src_clickhouse
cd test/tap && OPTZ="${O0} -ggdb" CC=${CC} CXX=${CXX} ${MAKE}

.PHONY: build_tap_test_debug
build_tap_test_debug: build_tap_tests_debug
.PHONY: build_tap_tests_debug
build_tap_tests_debug: build_src_debug_clickhouse
Expand Down Expand Up @@ -230,6 +234,8 @@ build_src_debug_clickhouse: build_lib_debug_clickhouse

### packaging targets

SYS_KERN := $(shell uname -s)
#SYS_DIST := $(shell source /etc/os-release &>/dev/null; if [ -z ${NAME} ]; then head -1 /etc/redhat-release; else echo ${NAME}; fi | awk '{ print $1 })
SYS_ARCH := $(shell uname -m)
REL_ARCH := $(subst x86_64,amd64,$(subst aarch64,arm64,$(SYS_ARCH)))
RPM_ARCH := .$(SYS_ARCH)
Expand Down
16 changes: 8 additions & 8 deletions deps/Makefile
Original file line number Diff line number Diff line change
@@ -1,15 +1,15 @@
#!/bin/make -f


UNAME_S := $(shell uname -s)
UNAME_MS := $(word 2, $(shell uname -ms))
SYS_KERN := $(shell uname -s)
SYS_ARCH := $(shell uname -m)

CENTOSVER := Unknown
ifneq (,$(wildcard /etc/system-release))
CENTOSVER := $(shell rpm --eval %rhel)
endif

IS_ARM := $(if $(findstring aarch64, $(arch)),true,false)
IS_ARM := $(if $(findstring aarch64, $(SYS_ARCH)),true,false)
IS_CENTOS := $(if $(findstring Unknown, $(CENTOSVER)),false,true)

# to compile libmariadb_client with support for valgrind enabled, run:
Expand Down Expand Up @@ -39,7 +39,7 @@ STDCPP := -std=c++$(shell echo $(CPLUSPLUS) | cut -c3-4) -DCXX$(shell echo $(CPL


targets := libconfig libdaemon jemalloc mariadb_client re2 sqlite3 pcre lz4 cityhash microhttpd curl ev libssl libhttpserver libinjection prometheus-cpp
ifeq ($(UNAME_S),Linux)
ifeq ($(SYS_KERN),Linux)
targets += coredumper
endif
ifeq ($(PROXYSQLCLICKHOUSE),1)
Expand All @@ -60,7 +60,7 @@ ifneq ($(CENTOSVER),6)
cd libinjection/libinjection && patch -p1 < ../update-build-py3.diff
cd libinjection/libinjection && patch -p1 < ../libinjection_sqli.c.patch
endif
ifeq ($(UNAME_S),Darwin)
ifeq ($(SYS_KERN),Darwin)
sed -i '' 's/CC=/CC?=/' libinjection/libinjection/src/Makefile
else
sed -i -e 's/CC=/CC?=/' libinjection/libinjection/src/Makefile
Expand Down Expand Up @@ -91,7 +91,7 @@ libhttpserver/libhttpserver/build/src/.libs/libhttpserver.a: libmicrohttpd/libmi
cd libhttpserver/libhttpserver && patch -p1 < ../final_val_post_process.patch
cd libhttpserver/libhttpserver && patch -p1 < ../empty_uri_log_crash.patch
cd libhttpserver/libhttpserver && sed -i -e 's/<microhttpd.h>/"microhttpd.h"/' src/httpserver/http_utils.hpp
ifeq ($(UNAME_S),FreeBSD)
ifeq ($(SYS_KERN),FreeBSD)
cd libhttpserver/libhttpserver && sed -i -e 's/\/bin\/bash/\/usr\/local\/bin\/bash/' bootstrap
endif
cd libhttpserver/libhttpserver && ./bootstrap && mkdir build
Expand Down Expand Up @@ -123,7 +123,7 @@ curl/curl/lib/.libs/libcurl.a: libssl/openssl/libssl.a
cd curl && tar -zxf curl-*.tar.gz
# cd curl/curl && ./configure --disable-debug --disable-ftp --disable-ldap --disable-ldaps --disable-rtsp --disable-proxy --disable-dict --disable-telnet --disable-tftp --disable-pop3 --disable-imap --disable-smb --disable-smtp --disable-gopher --disable-manual --disable-ipv6 --disable-sspi --disable-crypto-auth --disable-ntlm-wb --disable-tls-srp --without-nghttp2 --without-libidn2 --without-libssh2 --without-brotli --with-ssl=$(shell pwd)/../../libssl/openssl/ && CC=${CC} CXX=${CXX} ${MAKE}
cd curl/curl && autoreconf -fi
ifeq ($(UNAME_S),Darwin)
ifeq ($(SYS_KERN),Darwin)
cd curl/curl && patch configure < ../configure.patch
endif
cd curl/curl && CFLAGS=-fPIC ./configure --disable-debug --disable-ftp --disable-ldap --disable-ldaps --disable-rtsp --disable-proxy --disable-dict --disable-telnet --disable-tftp --disable-pop3 --disable-imap --disable-smb --disable-smtp --disable-gopher --disable-manual --disable-ipv6 --disable-sspi --disable-ntlm-wb --disable-tls-srp --without-nghttp2 --without-libidn2 --without-libssh2 --without-brotli --without-librtmp --without-libpsl --without-zstd --with-ssl=$(shell pwd)/libssl/openssl/ --enable-shared=yes && LD_LIBRARY_PATH=$(shell pwd)/libssl/openssl CC=${CC} CXX=${CXX} ${MAKE}
Expand Down Expand Up @@ -284,7 +284,7 @@ re2/re2/obj/libre2.a:
# cd re2/re2 && sed -i -e 's/-O3 -g /-O3 -fPIC /' Makefile
# cd re2/re2 && patch util/mutex.h < ../mutex.h.patch
cd re2/re2 && patch re2/onepass.cc < ../onepass.cc-multiplication-overflow.patch
ifeq ($(UNAME_S),Darwin)
ifeq ($(SYS_KERN),Darwin)
cd re2/re2 && sed -i '' -e 's/-O3 -g/-O3 -g -${STDCPP} -fPIC -DMEMORY_SANITIZER -DRE2_ON_VALGRIND /' Makefile
# cd re2/re2 && sed -i '' -e 's/RE2_CXXFLAGS?=-std=c++11 /RE2_CXXFLAGS?=-std=c++11 -fPIC /' Makefile
else
Expand Down
79 changes: 51 additions & 28 deletions test/Makefile
Original file line number Diff line number Diff line change
@@ -1,50 +1,73 @@
DEPS_PATH=../deps
#!/bin/make -f


MARIADB_PATH=$(DEPS_PATH)/mariadb-client-library/mariadb_client
MARIADB_IDIR=$(MARIADB_PATH)/include
PROXYSQL_PATH := $(shell while [ ! -f ./src/proxysql_global.cpp ]; do cd ..; done; pwd)
PROXYSQL_IDIR := $(PROXYSQL_PATH)/include

JEMALLOC_PATH=$(DEPS_PATH)/jemalloc/jemalloc
JEMALLOC_IDIR=$(JEMALLOC_PATH)/include/jemalloc
DEPS_PATH := $(PROXYSQL_PATH)/deps

LIBCONFIG_PATH=$(DEPS_PATH)/libconfig/libconfig
LIBCONFIG_IDIR=-I$(LIBCONFIG_PATH)/lib

#INJECTION_PATH=$(DEPS_PATH)/libinjection
#INJECTION_IDIR=$(INJECTION_PATH)
MARIADB_PATH := $(DEPS_PATH)/mariadb-client-library/mariadb_client
MARIADB_IDIR := $(MARIADB_PATH)/include

RE2_PATH=$(DEPS_PATH)/re2/re2
RE2_IDIR=$(RE2_PATH)
JEMALLOC_PATH := $(DEPS_PATH)/jemalloc/jemalloc
JEMALLOC_IDIR := $(JEMALLOC_PATH)/include/jemalloc

PCRE_PATH=$(DEPS_PATH)/pcre/pcre
LIBCONFIG_PATH := $(DEPS_PATH)/libconfig/libconfig
LIBCONFIG_IDIR := $(LIBCONFIG_PATH)/lib

SQLITE3_DIR=$(DEPS_PATH)/sqlite3/sqlite3
#INJECTION_PATH := $(DEPS_PATH)/libinjection
#INJECTION_IDIR := $(INJECTION_PATH)

CLICKHOUSE_CPP_DIR=$(DEPS_PATH)/clickhouse-cpp/clickhouse-cpp
RE2_PATH := $(DEPS_PATH)/re2/re2
RE2_IDIR := $(RE2_PATH)

LIBINJECTION_DIR=$(DEPS_PATH)/libinjection/libinjection
LIBINJECTION_IDIR=-I$(LIBINJECTION_DIR)/src
PCRE_PATH := $(DEPS_PATH)/pcre/pcre
PCRE_IDIR := $(PCRE_PATH)

LIBHTTPSERVER_DIR=$(DEPS_PATH)/libhttpserver/libhttpserver
LIBHTTPSERVER_IDIR=-I$(LIBHTTPSERVER_DIR)/src
SQLITE3_PATH := $(DEPS_PATH)/sqlite3/sqlite3
SQLITE3_IDIR := $(SQLITE3_PATH)

MICROHTTPD_DIR=$(DEPS_PATH)/libmicrohttpd/libmicrohttpd
MICROHTTPD_IDIR=-I$(MICROHTTPD_DIR) -I$(MICROHTTPD_DIR)/src/include
CLICKHOUSE_CPP_PATH := $(DEPS_PATH)/clickhouse-cpp/clickhouse-cpp
CLICKHOUSE_CPP_IDIR := $(CLICKHOUSE_CPP_PATH)

CURL_DIR=$(DEPS_PATH)/curl/curl
CURL_IDIR=-I$(CURL_DIR)/include
LIBINJECTION_PATH := $(DEPS_PATH)/libinjection/libinjection
LIBINJECTION_IDIR := -I$(LIBINJECTION_PATH)/src

SSL_DIR=$(DEPS_PATH)/libssl/openssl/
SSL_IDIR=$(SSL_DIR)/include
LIBHTTPSERVER_PATH := $(DEPS_PATH)/libhttpserver/libhttpserver
LIBHTTPSERVER_IDIR := -I$(LIBHTTPSERVER_PATH)/src

MICROHTTPD_PATH := $(DEPS_PATH)/libmicrohttpd/libmicrohttpd
MICROHTTPD_IDIR := $(MICROHTTPD_PATH)/src/include

EV_DIR=$(DEPS_PATH)/libev/libev/
EV_IDIR=$(EV_DIR)
CURL_PATH := $(DEPS_PATH)/curl/curl
CURL_IDIR := -I$(CURL_PATH)/include

IDIR=../include
SSL_PATH := $(DEPS_PATH)/libssl/openssl/
SSL_IDIR := $(SSL_PATH)/include

IDIRS=-I$(IDIR) -I$(JEMALLOC_IDIR) -I$(MARIADB_IDIR) $(LIBCONFIG_IDIR) -I$(RE2_IDIR) -I$(SQLITE3_DIR) -I$(PCRE_PATH) -I/usr/local/include -I$(CLICKHOUSE_CPP_DIR) $(MICROHTTPD_IDIR) $(LIBHTTPSERVER_IDIR) $(LIBINJECTION_IDIR) $(CURL_IDIR) -I$(EV_DIR) -I$(SSL_IDIR)
EV_PATH := $(DEPS_PATH)/libev/libev/
EV_IDIR := $(EV_PATH)

LOCAL_IDIR := /usr/local/include


IDIRS := -I$(PROXYSQL_IDIR)
IDIRS += -I$(JEMALLOC_IDIR)
IDIRS += -I$(MARIADB_IDIR)
IDIRS += -I$(LIBCONFIG_IDIR)
IDIRS += -I$(RE2_IDIR)
IDIRS += -I$(SQLITE3_IDIR)
IDIRS += -I$(PCRE_IDIR)
IDIRS += -I$(LOCAL_IDIR)
IDIRS += -I$(CLICKHOUSE_CPP_DIR)
IDIRS += -I$(MICROHTTPD_PATH)
IDIRS += -I$(MICROHTTPD_IDIR)
IDIRS += -I$(LIBHTTPSERVER_IDIR)
IDIRS += -I$(LIBINJECTION_IDIR)
IDIRS += -I$(CURL_IDIR)
IDIRS += -I$(EV_DIR)
IDIRS += -I$(SSL_IDIR)


sqlite_history_convert: sqlite_history_convert.cpp
Expand Down
11 changes: 6 additions & 5 deletions test/deps/Makefile
Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
#!/bin/make -f


PROXYSQL_PATH := $(shell while [ ! -f ./src/proxysql_global.cpp ]; do cd ..; done; pwd )
PROXYSQL_PATH := $(shell while [ ! -f ./src/proxysql_global.cpp ]; do cd ..; done; pwd)
DEPS_PATH := $(PROXYSQL_PATH)/deps


.DEFAULT: default
.PHONY: default
default: all

Expand Down Expand Up @@ -49,16 +50,16 @@ mysql_client: mysql-connector-c/mysql-connector-c/libmysql/libmysqlclient.a

### clean targets

.SILENT: cleanall
.PHONY: cleanall
.SILENT: cleanall
cleanall:
cd mariadb-connector-c && rm -rf mariadb-connector-c-*/ || true
cd mysql-connector-c && rm -rf mysql-5.7.*/ || true

.SILENT: clean
.PHONY: clean
.SILENT: clean
clean:
cd mariadb-connector-c/mariadb-connector-c && $(MAKE) clean || true
cd mariadb-connector-c/mariadb-connector-c && $(MAKE) --no-print-directory clean || true
cd mariadb-connector-c/mariadb-connector-c && rm -f CMakeCache.txt || true
cd mysql-connector-c/mysql-connector-c && $(MAKE) clean || true
cd mysql-connector-c/mysql-connector-c && $(MAKE) --no-print-directory clean || true
cd mysql-connector-c/mysql-connector-c && rm -f CMakeCache.txt || true
45 changes: 32 additions & 13 deletions test/tap/Makefile
Original file line number Diff line number Diff line change
@@ -1,20 +1,39 @@
#!/bin/make -f


.DEFAULT: all
.PHONY: all
all: tap tests tests_with_deps
all: tap test_deps
${MAKE} tests
${MAKE} tests_with_deps

.PHONY: debug
debug: tap test_deps
${MAKE} tests
${MAKE} tests_with_deps

.PHONY: clean
clean:
cd tap && ${MAKE} clean
cd tests && ${MAKE} clean
cd tests_with_deps && ${MAKE} clean

debug: tap tests tests_with_deps
.PHONY: test_deps
test_deps:
cd ../deps && CC=${CC} CXX=${CXX} ${MAKE}

.PHONY: tap
tap:
cd tap && CC=${CC} CXX=${CXX} ${MAKE} $(MAKECMDGOALS)
tap:
cd tap && CC=${CC} CXX=${CXX} ${MAKE}

tests: tap
cd tests && CC=${CC} CXX=${CXX} ${MAKE} $(MAKECMDGOALS)
.PHONY: tests
tests:
cd tests && CC=${CC} CXX=${CXX} ${MAKE}

tests_with_deps: tap tests
cd tests_with_deps && CC=${CC} CXX=${CXX} ${MAKE} $(MAKECMDGOALS)
.PHONY: tests_with_deps
tests_with_deps:
cd tests_with_deps && CC=${CC} CXX=${CXX} ${MAKE}


.PHONY: clean
.SILENT: clean
clean:
cd ../deps && ${MAKE} -s clean
cd tap && ${MAKE} -s clean
cd tests && ${MAKE} -s clean
cd tests_with_deps && ${MAKE} -s clean
5 changes: 3 additions & 2 deletions test/tap/tests/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -155,6 +155,7 @@ OPT := $(STDCPP) -O2 -ggdb -Wl,--no-as-needed -Wl,-rpath,$(TAP_LDIR) $(WGCOV) -D

### main targets

.DEFAULT: default
.PHONY: default
default: all

Expand All @@ -164,7 +165,7 @@ all: tests
debug: OPT := $(STDCPP) -O0 -DDEBUG -ggdb -Wl,--no-as-needed -Wl,-rpath,$(TAP_LDIR) $(WGCOV) $(WASAN) -DGITVERSION=\"$(GIT_VERSION)\"
debug: tests

tests: build_test_deps
#tests: build_test_deps
tests: tests-cpp \
tests-php \
tests-py \
Expand All @@ -190,7 +191,7 @@ testaurora: aurora

### test deps targets

build_test_deps: $(TAP_LDIR)/libtap.so $(TEST_MARIADB_LDIR)/libmariadbclient.a $(TEST_MYSQL_LDIR)/libmysqlclient.a
#build_test_deps: $(TAP_LDIR)/libtap.so $(TEST_MARIADB_LDIR)/libmariadbclient.a $(TEST_MYSQL_LDIR)/libmysqlclient.a

$(TAP_LDIR)/libtap.so:
cd $(TAP_PATH) && CC=${CC} CXX=${CXX} ${MAKE}
Expand Down
17 changes: 12 additions & 5 deletions test/tap/tests_with_deps/Makefile
Original file line number Diff line number Diff line change
@@ -1,13 +1,20 @@
#!/bin/make -f


.DEFAULT: all
.PHONY: all
all: deprecate_eof_support

.PHONY: clean
clean:
cd deprecate_eof_support && ${MAKE} clean

.PHONY: debug
debug: deprecate_eof_support


.PHONY: deprecate_eof_support
deprecate_eof_support:
cd deprecate_eof_support && CC=${CC} CXX=${CXX} ${MAKE} $(MAKECMDGOALS)
cd deprecate_eof_support && CC=${CC} CXX=${CXX} ${MAKE}


.PHONY: clean
.SILENT: clean
clean:
cd deprecate_eof_support && ${MAKE} clean
Loading

0 comments on commit 8de4e0e

Please sign in to comment.