Skip to content

Commit

Permalink
build: separate stdlib manifests by version (#47596)
Browse files Browse the repository at this point in the history
We install these by version, so it helps a lot if they reflect that in the
manifest name. Also some other mild cleanup.
  • Loading branch information
vtjnash authored Nov 20, 2022
1 parent bba41d4 commit 67b8ac0
Show file tree
Hide file tree
Showing 9 changed files with 59 additions and 47 deletions.
17 changes: 0 additions & 17 deletions Make.inc
Original file line number Diff line number Diff line change
Expand Up @@ -135,23 +135,6 @@ endif
export BUILDROOT
unexport O

# Make sure the user didn't try to specify a path that will confuse the shell / make
METACHARACTERS := ][?*{}() $$%:;&|!\#,\\`\":
ifneq (,$(findstring ',$(value BUILDROOT)))
$(error cowardly refusing to build into directory with a single-quote in the path)
endif
ifneq (,$(findstring ',$(value JULIAHOME)))
$(error cowardly refusing to build from source directory with a single-quote in the path)
endif
ifneq (,$(shell echo '$(value BUILDROOT)' | grep '[$(METACHARACTERS)]'))
$(error cowardly refusing to build into directory with a shell-metacharacter in the path\
(got: $(value BUILDROOT)))
endif
ifneq (,$(shell echo '$(value JULIAHOME)' | grep '[$(METACHARACTERS)]'))
$(error cowardly refusing to build from source directory with a shell-metacharacter in the path\
(got: $(value JULIAHOME)))
endif

# we include twice to pickup user definitions better
# include from JULIAHOME first so that BUILDROOT can override
MAYBE_HOST :=
Expand Down
38 changes: 26 additions & 12 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,23 @@ include $(JULIAHOME)/Make.inc
# import LLVM_SHARED_LIB_NAME
include $(JULIAHOME)/deps/llvm-ver.make

# Make sure the user didn't try to build in a path that will confuse the shell or make
METACHARACTERS := [][?*{}() $$%:;&|!\#,\\`\":]\|/\./\|/\.\./
ifneq (,$(findstring ',$(value BUILDROOT)))
$(error cowardly refusing to build into directory with a single-quote in the path)
endif
ifneq (,$(findstring ',$(value JULIAHOME)))
$(error cowardly refusing to build from source directory with a single-quote in the path)
endif
ifneq (,$(shell echo '$(value BUILDROOT)/' | grep '$(METACHARACTERS)'))
$(error cowardly refusing to build into directory with a shell-metacharacter in the path\
(got: $(value BUILDROOT)))
endif
ifneq (,$(shell echo '$(value JULIAHOME)/' | grep '$(METACHARACTERS)'))
$(error cowardly refusing to build from source directory with a shell-metacharacter in the path\
(got: $(value JULIAHOME)))
endif

VERSDIR := v`cut -d. -f1-2 < $(JULIAHOME)/VERSION`

default: $(JULIA_BUILD_MODE) # contains either "debug" or "release"
Expand All @@ -13,7 +30,7 @@ DIRS := $(sort $(build_bindir) $(build_depsbindir) $(build_libdir) $(build_priva
ifneq ($(BUILDROOT),$(JULIAHOME))
BUILDDIRS := $(BUILDROOT) $(addprefix $(BUILDROOT)/,base src src/flisp src/support src/clangsa cli doc deps stdlib test test/clangsa test/embedding test/llvmpasses)
BUILDDIRMAKE := $(addsuffix /Makefile,$(BUILDDIRS)) $(BUILDROOT)/sysimage.mk
DIRS := $(DIRS) $(BUILDDIRS)
DIRS += $(BUILDDIRS)
$(BUILDDIRMAKE): | $(BUILDDIRS)
@# add Makefiles to the build directories for convenience (pointing back to the source location of each)
@echo '# -- This file is automatically generated in julia/Makefile -- #' > $@
Expand Down Expand Up @@ -44,10 +61,6 @@ $(foreach link,base $(JULIAHOME)/test,$(eval $(call symlink_target,$(link),$$(bu
julia_flisp.boot.inc.phony: julia-deps
@$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/src julia_flisp.boot.inc.phony

# Build the HTML docs (skipped if already exists, notably in tarballs)
$(BUILDROOT)/doc/_build/html/en/index.html: $(shell find $(BUILDROOT)/base $(BUILDROOT)/doc \( -path $(BUILDROOT)/doc/_build -o -path $(BUILDROOT)/doc/deps -o -name *_constants.jl -o -name *_h.jl -o -name version_git.jl \) -prune -o -type f -print)
@$(MAKE) docs

julia-symlink: julia-cli-$(JULIA_BUILD_MODE)
ifeq ($(OS),WINNT)
echo '@"%~dp0/'"$$(echo '$(call rel_path,$(BUILDROOT),$(JULIA_EXECUTABLE))')"'" %*' | tr / '\\' > $(BUILDROOT)/julia.bat
Expand Down Expand Up @@ -236,7 +249,7 @@ define stringreplace
endef


install: $(build_depsbindir)/stringreplace $(BUILDROOT)/doc/_build/html/en/index.html
install: $(build_depsbindir)/stringreplace docs
@$(MAKE) $(QUIET_MAKE) $(JULIA_BUILD_MODE)
@for subdir in $(bindir) $(datarootdir)/julia/stdlib/$(VERSDIR) $(docdir) $(man1dir) $(includedir)/julia $(libdir) $(private_libdir) $(sysconfdir) $(libexecdir); do \
mkdir -p $(DESTDIR)$$subdir; \
Expand Down Expand Up @@ -484,7 +497,7 @@ app:
darwinframework:
$(MAKE) -C $(JULIAHOME)/contrib/mac/framework

light-source-dist.tmp: $(BUILDROOT)/doc/_build/html/en/index.html
light-source-dist.tmp: docs
ifneq ($(BUILDROOT),$(JULIAHOME))
$(error make light-source-dist does not work in out-of-tree builds)
endif
Expand Down Expand Up @@ -561,12 +574,13 @@ distcleanall: cleanall
@-$(MAKE) -C $(BUILDROOT)/deps distcleanall
@-$(MAKE) -C $(BUILDROOT)/doc cleanall

.PHONY: default debug release check-whitespace release-candidate \
.FORCE:
.PHONY: .FORCE default debug release check-whitespace release-candidate \
julia-debug julia-release julia-stdlib julia-deps julia-deps-libs \
julia-cli-release julia-cli-debug julia-src-release julia-src-debug \
julia-symlink julia-base julia-sysimg julia-sysimg-ji julia-sysimg-release julia-sysimg-debug \
test testall testall1 test test-* test-revise-* \
clean distcleanall cleanall clean-* \
test testall testall1 test \
clean distcleanall cleanall $(CLEAN_TARGETS) \
run-julia run-julia-debug run-julia-release run \
install binary-dist light-source-dist.tmp light-source-dist \
dist full-source-dist source-dist
Expand All @@ -583,12 +597,12 @@ testall: check-whitespace $(JULIA_BUILD_MODE)
testall1: check-whitespace $(JULIA_BUILD_MODE)
@env JULIA_CPU_THREADS=1 $(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/test all JULIA_BUILD_MODE=$(JULIA_BUILD_MODE)

test-%: check-whitespace $(JULIA_BUILD_MODE)
test-%: check-whitespace $(JULIA_BUILD_MODE) .FORCE
@([ $$(( $$(date +%s) - $$(date -r $(build_private_libdir)/sys.$(SHLIB_EXT) +%s) )) -le 100 ] && \
printf '\033[93m HINT The system image was recently rebuilt. Are you aware of the test-revise-* targets? See CONTRIBUTING.md. \033[0m\n') || true
@$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/test $* JULIA_BUILD_MODE=$(JULIA_BUILD_MODE)

test-revise-%:
test-revise-%: .FORCE
@$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/test revise-$* JULIA_BUILD_MODE=$(JULIA_BUILD_MODE)

# download target for some hardcoded windows dependencies
Expand Down
3 changes: 1 addition & 2 deletions cli/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -70,8 +70,7 @@ dump-trampolines: $(SRCDIR)/trampolines/trampolines_$(ARCH).S
$(CC) $(SHIPFLAGS) $(LOADER_CFLAGS) $< -S | sed -E 's/ ((%%)|;) /\n/g' | sed -E 's/.global/\n.global/g'

DIRS = $(build_bindir) $(build_libdir)
$(DIRS):
@mkdir -p $@
$(foreach dir,$(DIRS),$(eval $(call dir_target,$(dir))))

ifeq ($(OS),WINNT)
$(BUILDDIR)/julia_res.o: $(JULIAHOME)/contrib/windows/julia.rc $(JULIAHOME)/VERSION
Expand Down
15 changes: 10 additions & 5 deletions deps/tools/common.mk
Original file line number Diff line number Diff line change
Expand Up @@ -107,8 +107,8 @@ endif
DIRS := $(sort $(build_bindir) $(build_depsbindir) $(build_libdir) $(build_includedir) $(build_sysconfdir) $(build_datarootdir) $(build_staging) $(build_prefix)/manifest)

$(foreach dir,$(DIRS),$(eval $(call dir_target,$(dir))))

$(build_prefix): | $(DIRS)

$(eval $(call dir_target,$(SRCCACHE)))


Expand Down Expand Up @@ -174,6 +174,7 @@ $$(build_prefix)/manifest/$(strip $1): $$(build_staging)/$2.tar | $(build_prefix
$(UNTAR) $$< -C $$(build_prefix)
$6
echo '$$(UNINSTALL_$(strip $1))' > $$@
.PHONY: $(addsuffix -$(strip $1),stage install distclean uninstall reinstall)
endef

define staged-uninstaller
Expand All @@ -192,14 +193,18 @@ endef
define symlink_install # (target-name, rel-from, abs-to)
clean-$1: uninstall-$1
install-$1: $$(build_prefix)/manifest/$1
reinstall-$1: install-$1
reinstall-$1:
+$$(MAKE) uninstall-$1
+$$(MAKE) stage-$1
+$$(MAKE) install-$1
.PHONY: $(addsuffix -$1,clean install reinstall)

UNINSTALL_$(strip $1) := $2 symlink-uninstaller $3

$$(build_prefix)/manifest/$1: $$(BUILDDIR)/$2/build-compiled | $3 $$(build_prefix)/manifest
$$(build_prefix)/manifest/$1: $$(BUILDDIR)/$2/build-compiled | $$(abspath $$(dir $3/$1)) $$(abspath $$(dir $$(build_prefix)/manifest/$1))
-+[ ! \( -e $3/$1 -o -h $3/$1 \) ] || $$(MAKE) uninstall-$1
ifeq ($$(BUILD_OS), WINNT)
cmd //C mklink //J $$(call mingw_to_dos,$3/$1,cd $3 &&) $$(call mingw_to_dos,$$(BUILDDIR)/$2,)
cmd //C mklink //J $$(call mingw_to_dos,$3/$1,cd $3/$(dir $1) &&) $$(call mingw_to_dos,$$(BUILDDIR)/$2,)
else ifneq (,$$(findstring CYGWIN,$$(BUILD_OS)))
cmd /C mklink /J $$(call cygpath_w,$3/$1) $$(call cygpath_w,$$(BUILDDIR)/$2)
else ifdef JULIA_VAGRANT_BUILD
Expand All @@ -213,7 +218,7 @@ endef
define symlink-uninstaller
uninstall-$1:
ifeq ($$(BUILD_OS), WINNT)
-cmd //C rmdir $$(call mingw_to_dos,$3/$1,cd $3 &&)
-cmd //C rmdir $$(call mingw_to_dos,$3/$1,cd $3/$(dir $1) &&)
else
rm -rf $3/$1
endif
Expand Down
3 changes: 2 additions & 1 deletion deps/tools/git-external.mk
Original file line number Diff line number Diff line change
Expand Up @@ -68,11 +68,12 @@ $5/$$($2_SRC_DIR)/source-extracted: $$($2_SRC_FILE)
$(TAR) -C $$(dir $$@) --strip-components 1 -xf $$<
echo 1 > $$@

checksum-$(1): $$($2_SRC_FILE)
checksum-$1: $$($2_SRC_FILE)
$$(JLCHECKSUM) $$<
endif # DEPS_GIT

$$(build_prefix)/manifest/$1: $$(SRCDIR)/$1.version # make the manifest stale if the version file is touched (causing re-install for compliant targets)
distclean-$1:
rm -rf $5/$$($2_SRC_DIR) $$($2_SRC_FILE) $$(BUILDDIR)/$$($2_SRC_DIR)
.PHONY: $(addsuffix -$1,checksum distclean)
endef
9 changes: 7 additions & 2 deletions deps/tools/stdlib-external.mk
Original file line number Diff line number Diff line change
Expand Up @@ -16,12 +16,17 @@ $$(eval $$(call git-external,$1,$2,,,$$(BUILDDIR)))
$$(BUILDDIR)/$$($2_SRC_DIR)/build-compiled: $$(BUILDDIR)/$$($2_SRC_DIR)/source-extracted
@# no build steps
echo 1 > $$@
$$(eval $$(call symlink_install,$1,$$$$($2_SRC_DIR),$$$$(build_datarootdir)/julia/stdlib/$$$$(VERSDIR)))
$$(eval $$(call symlink_install,$$$$(VERSDIR)/$1,$$$$($2_SRC_DIR),$$$$(build_datarootdir)/julia/stdlib))
clean-$1:
-rm -f $$(BUILDDIR)/$$($2_SRC_DIR)/build-compiled
get-$1: $$($2_SRC_FILE)
extract-$1: $$(BUILDDIR)/$$($2_SRC_DIR)/source-extracted
configure-$1: extract-$1
compile-$1: $$(BUILDDIR)/$$($2_SRC_DIR)/build-compiled

install-$1: install-$$(VERSDIR)/$1
uninstall-$1: uninstall-$$(VERSDIR)/$1
reinstall-$1: reinstall-$$(VERSDIR)/$1
version-check-$1: version-check-$$(VERSDIR)/$1
clean-$1: clean-$$(VERSDIR)/$1
.PHONY: $(addsuffix -$1,get extract configure compile install uninstall reinstall clean)
endef
1 change: 1 addition & 0 deletions deps/tools/uninstallers.mk
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ else
uninstall-$1:
@echo "skipping uninstall: $1 not installed"
endif
.PHONY: uninstall-$1
endef
$(foreach dep,$(DEP_LIBS_STAGED_ALL),$(eval $(call define-uninstaller,$(dep))))

Expand Down
16 changes: 10 additions & 6 deletions stdlib/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,8 @@ include $(JULIAHOME)/deps/*.version


VERSDIR := v$(shell cut -d. -f1-2 < $(JULIAHOME)/VERSION)

$(build_datarootdir)/julia/stdlib/$(VERSDIR):
mkdir -p $@
DIRS := $(build_datarootdir)/julia/stdlib/$(VERSDIR) $(build_prefix)/manifest/$(VERSDIR)
$(foreach dir,$(DIRS),$(eval $(call dir_target,$(dir))))

JLLS = DSFMT GMP CURL LIBGIT2 LLVM LIBSSH2 LIBUV MBEDTLS MPFR NGHTTP2 \
BLASTRAMPOLINE OPENBLAS OPENLIBM P7ZIP PCRE LIBSUITESPARSE ZLIB \
Expand All @@ -43,14 +42,19 @@ $(foreach jll,$(JLLS),$(eval $(call download-artifacts-toml,$(jll))))

STDLIBS = Artifacts Base64 CRC32c Dates Distributed FileWatching \
Future InteractiveUtils LazyArtifacts Libdl LibGit2 LinearAlgebra Logging \
Markdown Mmap Printf Profile Random REPL Serialization SHA \
SharedArrays Sockets SparseArrays SuiteSparse Test TOML Unicode UUIDs \
Markdown Mmap Printf Profile Random REPL Serialization \
SharedArrays Sockets Test TOML Unicode UUIDs \
$(JLL_NAMES)

STDLIBS_EXT = Pkg Statistics LibCURL Downloads ArgTools Tar NetworkOptions SuiteSparse SparseArrays SHA

$(foreach module, $(STDLIBS_EXT), $(eval $(call stdlib-external,$(module),$(shell echo $(module) | tr a-z A-Z))))

ifneq ($(filter $(STDLIBS),$(STDLIBS_EXT)),)
$(error ERROR duplicated STDLIBS in list)
endif


# Generate symlinks to all stdlibs at usr/share/julia/stdlib/vX.Y/
$(foreach module, $(STDLIBS), $(eval $(call symlink_target,$$(JULIAHOME)/stdlib/$(module),$$(build_datarootdir)/julia/stdlib/$$(VERSDIR),$(module))))

Expand All @@ -68,5 +72,5 @@ clean: $(addprefix clean-, $(STDLIBS_EXT)) $(CLEAN_TARGETS) extstdlibclean
distclean: $(addprefix distclean-, $(STDLIBS_EXT)) clean
checksumall: $(addprefix checksum-, $(STDLIBS_EXT))

DEP_LIBS_STAGED_ALL := $(STDLIBS_EXT)
DEP_LIBS_STAGED_ALL := $(addprefix $(VERSDIR)/,$(STDLIBS_EXT))
include $(JULIAHOME)/deps/tools/uninstallers.mk
4 changes: 2 additions & 2 deletions sysimage.mk
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ sysimg-bc: $(build_private_libdir)/sys-bc.a
sysimg-release: $(build_private_libdir)/sys.$(SHLIB_EXT)
sysimg-debug: $(build_private_libdir)/sys-debug.$(SHLIB_EXT)

VERSDIR := v`cut -d. -f1-2 < $(JULIAHOME)/VERSION`
VERSDIR := v$(shell cut -d. -f1-2 < $(JULIAHOME)/VERSION)

$(build_private_libdir)/%.$(SHLIB_EXT): $(build_private_libdir)/%-o.a
@$(call PRINT_LINK, $(CXX) $(LDFLAGS) -shared $(fPIC) -L$(build_private_libdir) -L$(build_libdir) -L$(build_shlibdir) -o $@ \
Expand Down Expand Up @@ -54,7 +54,7 @@ COMPILER_SRCS += $(shell find $(JULIAHOME)/base/compiler -name \*.jl)
BASE_SRCS := $(sort $(shell find $(JULIAHOME)/base -name \*.jl -and -not -name sysimg.jl) \
$(shell find $(BUILDROOT)/base -name \*.jl -and -not -name sysimg.jl))
STDLIB_SRCS := $(JULIAHOME)/base/sysimg.jl $(shell find $(build_datarootdir)/julia/stdlib/$(VERSDIR)/*/src -name \*.jl) \
$(build_prefix)/manifest/Pkg
$(wildcard $(build_prefix)/manifest/$(VERSDIR)/*)
RELBUILDROOT := $(call rel_path,$(JULIAHOME)/base,$(BUILDROOT)/base)/ # <-- make sure this always has a trailing slash

$(build_private_libdir)/corecompiler.ji: $(COMPILER_SRCS)
Expand Down

0 comments on commit 67b8ac0

Please sign in to comment.