make: centralize wget/curl & unzip/7z feature test

With many open PRs that could benefit from loading SDKs when needed,
instead adding vast amounts of code to RIOTs master, this PR provides
the "functions" `$(DOWNLOAD_TO_STDOUT)`, `$(DOWNLOAD_TO_FILE)`, and
`$(UNZIP_HERE)`.

The first "function" takes one argument, the URL from where to download
the content. It is then piped to stdout. To be used e.g. with `tar xz`.

The second "function" taken two arguments, the destination file name,
and the source URL. If the previous invocation was interrupted, then the
download gets continued, if possible.

The last "function" takes one argument, the source ZIP file. The file
gets extracted into the cwd, so best use this "function" with
`cd $(SOME_WHERE) &&`.

The clumsy name `$(UNZIP_HERE)` is taken because the program "unzip"
takes the environment variable `UNZIP` as the source file, even if
another file name was given on the command line. The rationale for that
is that the hackers of "unzip" hate their users. Also they sacrifice
hamsters to Satan.

Makefile.buildtests

@@ -179,6 +179,10 @@ info-build:
 	@echo ''
 	@echo 'DEBUGGER:       $(DEBUGGER)'
 	@echo 'DEBUGGER_FLAGS: $(DEBUGGER_FLAGS)'
+	@echo
+	@echo 'DOWNLOAD_TO_FILE:   $(DOWNLOAD_TO_FILE)'
+	@echo 'DOWNLOAD_TO_STDOUT: $(DOWNLOAD_TO_STDOUT)'
+	@echo 'UNZIP_HERE:         $(UNZIP_HERE)'
 	@echo ''
 	@echo 'DEBUGSERVER:       $(DEBUGSERVER)'
 	@echo 'DEBUGSERVER_FLAGS: $(DEBUGSERVER_FLAGS)'

Makefile.include

@@ -35,6 +35,50 @@ endif
 
 BOARD := $(strip $(BOARD))
 
+# provide common external programs for `Makefile.include`s
+
+ifeq (,$(AXEL))
+  ifeq (0,$(shell which axel 2>&1 > /dev/null ; echo $$?))
+    AXEL := $(shell which axel)
+  endif
+endif
+ifeq (,$(WGET))
+  ifeq (0,$(shell which wget 2>&1 > /dev/null ; echo $$?))
+    WGET := $(shell which wget)
+  endif
+endif
+ifeq (,$(CURL))
+  ifeq (0,$(shell which curl 2>&1 > /dev/null ; echo $$?))
+    CURL := $(shell which curl)
+  endif
+endif
+ifeq (,$(WGET)$(CURL))
+  $(error Neither wget nor curl is installed!)
+endif
+
+ifeq (,$(DOWNLOAD_TO_STDOUT))
+  DOWNLOAD_TO_STDOUT := $(if $(CURL),$(CURL) -s,$(WGET) -q -O-)
+endif
+ifeq (,$(DOWNLOAD_TO_FILE))
+  ifneq (,$(AXEL))
+    DOWNLOAD_TO_FILE := $(AXEL) -n 4 -q -a -o
+  else
+    DOWNLOAD_TO_FILE := $(if $(WGET),$(WGET) -nv -c -O,$(CURL) -s -o)
+  endif
+endif
+
+ifeq (,$(UNZIP_HERE))
+  ifeq (0,$(shell which unzip 2>&1 > /dev/null ; echo $$?))
+    UNZIP_HERE := $(shell which unzip) -q
+  else
+    ifeq (0,$(shell which 7z 2>&1 > /dev/null ; echo $$?))
+      UNZIP_HERE := $(shell which 7z) x -bd
+    else
+      $(error Neither unzip nor 7z is installed.)
+    endif
+  endif
+endif
+
 # mandatory includes!
 include $(RIOTBASE)/Makefile.modules
 include $(RIOTBOARD)/$(BOARD)/Makefile.include

Makefile.vars

@@ -48,3 +48,7 @@ export DEBUGSERVER           # The command to call on "make debug-server", usual
 export DEBUGSERVER_FLAGS     # The parameters to supply to DEBUGSERVER.
 export RESET                 # The command to call on "make reset", this command resets/reboots the target.
 export RESET_FLAGS           # The parameters to supply to RESET.
+
+export DOWNLOAD_TO_FILE      # Use `$(DOWNLOAD_TO_FILE) $(DESTINATION) $(URL)` to download `$(URL)` to `$(DESTINATION)`.
+export DOWNLOAD_TO_STDOUT    # Use `$(DOWNLOAD_TO_STDOUT) $(URL)` to download `$(URL)` output `$(URL)` to stdout, e.g. to be piped into `tar xz`.
+export UNZIP_HERE            # Use `cd $(SOME_FOLDER) && $(UNZIP_HERE) $(SOME_FILE)` to extract the contents of the zip file `$(SOME_FILE)` into `$(SOME_FOLDER)`.

boards/x86-multiboot-common/Makefile.include

@@ -1,25 +1,15 @@
 ifeq (, $(NEWLIB_BASE))
   NEWLIB_BASE := $(RIOTBASE)/toolchain/x86/i586-none-elf
+endif
 
-  ifneq (0, $(shell test -e "$(NEWLIB_BASE)/lib/libc.a" && echo $$?))
-    NEWLIB_PRECOMPILED_NAME := i586-newlib_2.1.0_tlsf-1254.tar.bz2
-    NEWLIB_PRECOMPILED := http://download.riot-os.org/$(NEWLIB_PRECOMPILED_NAME)
-
-    $(warning "Precompiled newlib is missing in $(NEWLIB_BASE)")
-    $(warning "Downloading from $(NEWLIB_PRECOMPILED)")
+ifneq (0, $(shell test -e "$(NEWLIB_BASE)/lib/libc.a" && echo $$?))
+  NEWLIB_PRECOMPILED_NAME := i586-newlib_2.1.0_tlsf-1254.tar.bz2
+  NEWLIB_PRECOMPILED := http://download.riot-os.org/$(NEWLIB_PRECOMPILED_NAME)
 
-    ifeq (0, $(shell which wget 2>&1 > /dev/null ; echo $$?))
-      DOWNLOAD_TO_STDOUT := $(shell which wget) -qO-
-    else
-      ifeq (0, $(shell which curl 2>&1 > /dev/null ; echo $$?))
-        DOWNLOAD_TO_STDOUT := $(shell which curl) -s
-      else
-        $(error "Neither wget nor curl is installed!")
-      endif
-    endif
+  $(warning Precompiled newlib is missing in $(NEWLIB_BASE))
+  $(warning Downloading from $(NEWLIB_PRECOMPILED))
 
-    $(shell cd $(RIOTBASE) && $(DOWNLOAD_TO_STDOUT) "$(NEWLIB_PRECOMPILED)" | tar xj)
-  endif
+  $(shell cd $(RIOTBASE) && $(DOWNLOAD_TO_STDOUT) "$(NEWLIB_PRECOMPILED)" | tar xj)
 endif
 
 ifeq (,$(BUILD_INCLUDE_BASE))

pkg/Makefile.http

@@ -1,15 +1,7 @@
-PKG_NAME=		# name of the package
-PKG_URL=		# source url of the package e.g. a git repository
-PKG_VERSION=	# version of the package to use e.g. a git commit/ref
-PKG_EXT=		# extenison of this package
-
-FETCH=$(shell which wget &> /dev/null && echo "wget" || echo "curl")
-UNPACK=tar -xvf
-
-ifneq ($(RIOTBOARD),)
-include $(RIOTBOARD)/Makefile.base
-include $(RIOTBOARD)/$(BOARD)/Makefile.include
-endif
+PKG_NAME    = my_pkg                       # name of the package
+PKG_URL     = http://example.com/downloads # source url of the package e.g. a git repository
+PKG_VERSION = v1.2.3                       # version of the package to use e.g. a git commit/ref
+PKG_EXT     = zip                          # extension of this package
 
 .PHONY: all clean patch reset
 
@@ -27,11 +19,13 @@ $(CURDIR)/$(PKG_NAME)-$(PKG_VERSION)/Makefile: $(CURDIR)/$(PKG_NAME)-$(PKG_VERSI
 	cd $< && patch ../patch.txt
 
 $(CURDIR)/$(PKG_NAME)-$(PKG_VERSION)/: $(CURDIR)/$(PKG_NAME)-$(PKG_VERSION).$(PKG_EXT)
-	$(UNPACK) $<
+	# Here you unpack the file.
+	# This example assumes the common pattern that the archive contains its data in a subfolder with the same name as itself.
+	$(AD)$(UNZIP_HERE) $<
 
 $(CURDIR)/$(PKG_NAME)-$(PKG_VERSION).$(PKG_EXT):
 	# Get PKG_VERSION of package from PKG_URL
-	@$(FETCH) -O $@ $(PKG_URL)/$(PKG_NAME)-$(PKG_VERSION).$(PKG_EXT) || true
+	$(AD)$(DOWNLOAD_TO_FILE) $@ $(PKG_URL)/$(PKG_NAME)-$(PKG_VERSION).$(PKG_EXT)
 
 clean::
 	# Reset package to checkout state.

pkg/openwsn/Makefile

@@ -3,21 +3,6 @@ PKG_URL=https://codeload.github.com/openwsn-berkeley/openwsn-fw
 PKG_VERSION=1.4
 PKG_EXT=zip
 
-FETCH=$(shell which wget &> /dev/null && echo "wget" || echo "curl")
-#UNPACK=tar -xvf
-UNPACK=unzip
-
-ifeq ($(FETCH),curl )
-	FETCH_FLAGS += -z $(CURDIR)/$(PKG_NAME)-$(PKG_VERSION).$(PKG_EXT) -o
-else
-	FETCH_FLAGS += -nc -O
-endif
-
-ifneq ($(RIOTBOARD),)
-#include $(RIOTBOARD)/Makefile.base
-include $(RIOTBOARD)/$(BOARD)/Makefile.include
-endif
-
 .PHONY: all clean
 
 all: $(CURDIR)/$(PKG_NAME)-$(PKG_VERSION)/
@@ -26,13 +11,13 @@ all: $(CURDIR)/$(PKG_NAME)-$(PKG_VERSION)/
 
 $(CURDIR)/$(PKG_NAME)-$(PKG_VERSION)/: $(CURDIR)/$(PKG_NAME)-$(PKG_VERSION).$(PKG_EXT)
 	$(AD)rm -rf $(CURDIR)/$(PKG_NAME)-$(PKG_VERSION)
-	$(AD)$(UNPACK) -q $< -d $(PKG_NAME)-$(PKG_VERSION)
-	$(AD)cd $@ && sh ../structure_changes.sh
-	$(AD)cd $@ && sh ../apply_patches.sh
+	$(AD)$(UNZIP_HERE) $< -d $(PKG_NAME)-$(PKG_VERSION)
+	$(AD)cd $@ && sh $(CURDIR)/structure_changes.sh
+	$(AD)cd $@ && sh $(CURDIR)/apply_patches.sh
 
 $(CURDIR)/$(PKG_NAME)-$(PKG_VERSION).$(PKG_EXT):
 	# Get PKG_VERSION of package from PKG_URL
-	$(AD)$(FETCH) $(FETCH_FLAGS) $@ $(PKG_URL)/$(PKG_EXT)/$(PKG_NAME)-$(PKG_VERSION) || true
+	$(AD)$(DOWNLOAD_TO_FILE) $@ $(PKG_URL)/$(PKG_EXT)/$(PKG_NAME)-$(PKG_VERSION)
 
 clean::
 	rm -rf $(CURDIR)/$(PKG_NAME)-$(PKG_VERSION)

pkg/tlsf/Makefile

@@ -3,30 +3,6 @@ PKG_VERSION = 3.0
 PKG_FILE = tlsf-$(PKG_VERSION).zip
 PKG_URL = http://tlsf.baisoku.org/$(PKG_FILE)
 
-ifeq (, $(DOWNLOAD_TO))
-  ifeq (0, $(shell which wget 2>&1 > /dev/null ; echo $$?))
-    DOWNLOAD_TO := $(shell which wget) -nv -c -O
-  else
-    ifeq (0, $(shell which curl 2>&1 > /dev/null ; echo $$?))
-      DOWNLOAD_TO := $(shell which curl) -s -o
-    else
-      $(error "Neither wget nor curl is installed.")
-    endif
-  endif
-endif
-
-ifeq (, $(UNZIP))
-  ifeq (0, $(shell which unzip 2>&1 > /dev/null ; echo $$?))
-    UNZIP := $(shell which unzip)
-  else
-    ifeq (0, $(shell which 7z 2>&1 > /dev/null ; echo $$?))
-      UNZIP := $(shell which 7z) x
-    else
-      $(error "Neither unzip nor 7z is installed.")
-    endif
-  endif
-endif
-
 .PHONY: all clean distclean
 
 all: $(BINDIR)$(PKG_NAME).a
@@ -37,11 +13,11 @@ $(BINDIR)$(PKG_NAME).a: $(BINDIR)$(PKG_NAME)-src/Makefile
 $(BINDIR)$(PKG_NAME)-src/Makefile: $(CURDIR)/$(PKG_FILE) $(CURDIR)/patch.txt
 	@rm -rf $(@D)
 	@mkdir -p $(@D)
-	$(AD)cd $(@D) && $(UNZIP) $(CURDIR)/$(PKG_FILE)
+	$(AD)cd $(@D) && $(UNZIP_HERE) $(CURDIR)/$(PKG_FILE)
 	$(AD)cd $(@D) && patch --binary -p0 -N -i $(CURDIR)/patch.txt
 
 $(CURDIR)/$(PKG_FILE):
-	$(AD)$(DOWNLOAD_TO) $@ $(PKG_URL)
+	$(AD)$(DOWNLOAD_TO_FILE) $@ $(PKG_URL)
 
 clean::
 	rm -rf $(BINDIR)$(PKG_NAME)-src/