Skip to content

Commit

Permalink
added Windows script, download retries, and MSVC linking
Browse files Browse the repository at this point in the history
  • Loading branch information
jameslamb committed Mar 24, 2020
1 parent ef948ae commit e54954a
Show file tree
Hide file tree
Showing 6 changed files with 130 additions and 106 deletions.
3 changes: 3 additions & 0 deletions .appveyor.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,9 @@ environment:
- COMPILER: MINGW
TASK: r-package
R_WINDOWS_VERSION: 3.6.3
- COMPILER: MSVC
TASK: r-package
R_WINDOWS_VERSION: 3.6.3

clone_depth: 5

Expand Down
86 changes: 86 additions & 0 deletions .ci/test_r_package_windows.ps1
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
# Download a file and retry upon failure. This looks like
# an infinite looop
function Download-File-With-Retries {
param(
[string]$url,
[string]$destfile
)
do {
Write-Output "Downloading '${url}'"
sleep 5;
(New-Object System.Net.WebClient).DownloadFile($url, $destfile)
} while(!$?);
}

$env:R_LIB_PATH = "C:/RLibrary"
$env:PATH = "$env:R_LIB_PATH/Rtools/bin;" + "$env:R_LIB_PATH/R/bin/x64;" + "$env:R_LIB_PATH/miktex/texmfs/install/miktex/bin/x64;" + $env:PATH
$env:BINPREF = "C:/mingw-w64/x86_64-8.1.0-posix-seh-rt_v6-rev0/mingw64/bin/"

cd $env:BUILD_SOURCESDIRECTORY
tzutil /s "GMT Standard Time"
[Void][System.IO.Directory]::CreateDirectory($env:R_LIB_PATH)

if ($env:COMPILER -eq "MINGW") {
Write-Output "Telling R to use MinGW"
$install_libs = "$env:BUILD_SOURCESDIRECTORY\R-package\src\install.libs.R"
((Get-Content -path $install_libs -Raw) -replace 'use_mingw <- FALSE','use_mingw <- TRUE') | Set-Content -Path $install_libs
}

# set up R if it doesn't exist yet
if (!(Get-Command R.exe -errorAction SilentlyContinue)) {

Write-Output "Downloading R and Rtools"

# download R and RTools
Download-File-With-Retries -url "https://cloud.r-project.org/bin/windows/base/old/$env:R_WINDOWS_VERSION/R-$env:R_WINDOWS_VERSION-win.exe" -destfile "R-win.exe"
Download-File-With-Retries -url "https://cloud.r-project.org/bin/windows/Rtools/Rtools35.exe" -destfile "Rtools.exe"

# Install R
Write-Output "Installing R"
Start-Process -FilePath R-win.exe -NoNewWindow -Wait -ArgumentList "/VERYSILENT /DIR=$env:R_LIB_PATH/R /COMPONENTS=main,x64" ; Check-Output $?
Write-Output "Done installing R"

Write-Output "Installing Rtools"
Start-Process -FilePath Rtools.exe -NoNewWindow -Wait -ArgumentList "/VERYSILENT /DIR=$env:R_LIB_PATH/Rtools" ; Check-Output $?
Write-Output "Done installing Rtools"

# download Miktex
Write-Output "Downloading MiKTeX"
Download-File-With-Retries -url "https://miktex.org/download/win/miktexsetup-x64.zip" -destfile "miktexsetup-x64.zip"
Add-Type -AssemblyName System.IO.Compression.FileSystem
[System.IO.Compression.ZipFile]::ExtractToDirectory("miktexsetup-x64.zip", "miktex")
Write-Output "Setting up MiKTeX"
.\miktex\miktexsetup.exe --local-package-repository=./miktex/download --package-set=essential --quiet download ; Check-Output $?
Write-Output "Installing MiKTeX"
.\miktex\download\miktexsetup.exe --portable="$env:R_LIB_PATH/miktex" --quiet install ; Check-Output $?
Write-Output "Done installing R, Rtools, and MiKTeX"
}

initexmf --set-config-value [MPM]AutoInstall=1
conda install -y --no-deps pandoc

Add-Content .Renviron "R_LIBS=$env:R_LIB_PATH"
Add-Content .Rprofile "options(repos = 'https://cran.rstudio.com')"
Add-Content .Rprofile "options(pkgType = 'binary')"
Add-Content .Rprofile "options(install.packages.check.source = 'no')"

Write-Output "Installing dependencies"
Rscript.exe -e "install.packages(c('data.table', 'jsonlite', 'Matrix', 'R6', 'testthat'), dependencies = c('Imports', 'Depends', 'LinkingTo'), lib = '$env:R_LIB_PATH')" ; Check-Output $?

Write-Output "Building R package"
Rscript build_r.R ; Check-Output $?

$PKG_FILE_NAME = Get-Item *.tar.gz
$PKG_NAME = $PKG_FILE_NAME.BaseName.split("_")[0]
$LOG_FILE_NAME = "$PKG_NAME.Rcheck/00check.log"

Write-Output "Running R CMD check"
$env:_R_CHECK_FORCE_SUGGESTS_=0
R.exe CMD check "${PKG_FILE_NAME}" --as-cran --no-multiarch; Check-Output $?

if (Get-Content "$LOG_FILE_NAME" | Select-String -Pattern "WARNING" -Quiet) {
echo "WARNINGS have been found by R CMD check!"
Check-Output $False
}

Exit 0
100 changes: 13 additions & 87 deletions .ci/test_windows.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -11,16 +11,19 @@ if (Test-Path env:APPVEYOR) {
$env:BUILD_SOURCESDIRECTORY = $env:APPVEYOR_BUILD_FOLDER
}

# setup for Python
if ($env:TASK -ne "r-package") {
conda init powershell
conda activate
conda config --set always_yes yes --set changeps1 no
conda update -q -y conda
conda create -q -y -n $env:CONDA_ENV python=$env:PYTHON_VERSION joblib matplotlib numpy pandas psutil pytest python-graphviz "scikit-learn<=0.21.3" scipy wheel ; Check-Output $?
conda activate $env:CONDA_ENV
if ($env:TASK -eq "r-package") {
& $env:BUILD_SOURCESDIRECTORY\.ci\test_r_package_windows.ps1 ; Check-Output $?
Exit 0
}

# setup for Python
conda init powershell
conda activate
conda config --set always_yes yes --set changeps1 no
conda update -q -y conda
conda create -q -y -n $env:CONDA_ENV python=$env:PYTHON_VERSION joblib matplotlib numpy pandas psutil pytest python-graphviz "scikit-learn<=0.21.3" scipy wheel ; Check-Output $?
conda activate $env:CONDA_ENV

if ($env:TASK -eq "regular") {
mkdir $env:BUILD_SOURCESDIRECTORY/build; cd $env:BUILD_SOURCESDIRECTORY/build
cmake -A x64 .. ; cmake --build . --target ALL_BUILD --config Release ; Check-Output $?
Expand Down Expand Up @@ -57,10 +60,8 @@ elseif ($env:TASK -eq "bdist") {
}
}

if ($env:TASK -ne "r-package") {
$tests = $env:BUILD_SOURCESDIRECTORY + $(If (($env:TASK -eq "sdist") -or ($env:TASK -eq "appveyor-python")) {"/tests/python_package_test"} Else {"/tests"}) # cannot test C API with "sdist" task
pytest $tests ; Check-Output $?
}
$tests = $env:BUILD_SOURCESDIRECTORY + $(If (($env:TASK -eq "sdist") -or ($env:TASK -eq "appveyor-python")) {"/tests/python_package_test"} Else {"/tests"}) # cannot test C API with "sdist" task
pytest $tests ; Check-Output $?

if (($env:TASK -eq "regular") -or ($env:TASK -eq "appveyor-python")) {
cd $env:BUILD_SOURCESDIRECTORY/examples/python-guide
Expand All @@ -74,78 +75,3 @@ if (($env:TASK -eq "regular") -or ($env:TASK -eq "appveyor-python")) {
conda install -q -y -n $env:CONDA_ENV ipywidgets notebook
jupyter nbconvert --ExecutePreprocessor.timeout=180 --to notebook --execute --inplace *.ipynb ; Check-Output $? # run all notebooks
}

# test R package
if ($env:TASK -eq "r-package"){

$env:R_LIB_PATH = "C:/RLibrary"
$env:PATH = "$env:R_LIB_PATH/Rtools/bin;" + "$env:R_LIB_PATH/R/bin/x64;" + "$env:R_LIB_PATH/miktex/texmfs/install/miktex/bin/x64;" + $env:PATH
$env:BINPREF = "C:/mingw-w64/x86_64-8.1.0-posix-seh-rt_v6-rev0/mingw64/bin/"

cd $env:BUILD_SOURCESDIRECTORY
tzutil /s "GMT Standard Time"
[Void][System.IO.Directory]::CreateDirectory($env:R_LIB_PATH)

if ($env:COMPILER -eq "MINGW") {
Write-Output "Telling R to use MinGW"
$install_libs = "$env:BUILD_SOURCESDIRECTORY\R-package\src\install.libs.R"
((Get-Content -path $install_libs -Raw) -replace 'use_mingw <- FALSE','use_mingw <- TRUE') | Set-Content -Path $install_libs
}

# set up R if it doesn't exist yet
if (!(Get-Command R.exe -errorAction SilentlyContinue)) {

Write-Output "Downloading R and Rtools"

# download R and RTools
(New-Object System.Net.WebClient).DownloadFile("https://cloud.r-project.org/bin/windows/base/old/$env:R_WINDOWS_VERSION/R-$env:R_WINDOWS_VERSION-win.exe", "R-win.exe")
(New-Object System.Net.WebClient).DownloadFile("https://cloud.r-project.org/bin/windows/Rtools/Rtools35.exe", "Rtools.exe")

# Install R
Write-Output "Installing R"
Start-Process -FilePath R-win.exe -NoNewWindow -Wait -ArgumentList "/VERYSILENT /DIR=$env:R_LIB_PATH/R /COMPONENTS=main,x64" ; Check-Output $?
Write-Output "Done installing R"

Write-Output "Installing Rtools"
Start-Process -FilePath Rtools.exe -NoNewWindow -Wait -ArgumentList "/VERYSILENT /DIR=$env:R_LIB_PATH/Rtools" ; Check-Output $?
Write-Output "Done installing Rtools"

# download Miktex
Write-Output "Downloading MiKTeX"
(New-Object System.Net.WebClient).DownloadFile("https://miktex.org/download/win/miktexsetup-x64.zip", "miktexsetup-x64.zip")
Add-Type -AssemblyName System.IO.Compression.FileSystem
[System.IO.Compression.ZipFile]::ExtractToDirectory("miktexsetup-x64.zip", "miktex")
Write-Output "Setting up MiKTeX"
.\miktex\miktexsetup.exe --local-package-repository=./miktex/download --package-set=essential --quiet download ; Check-Output $?
Write-Output "Installing MiKTeX"
.\miktex\download\miktexsetup.exe --portable="$env:R_LIB_PATH/miktex" --quiet install ; Check-Output $?
Write-Output "Done installing R, Rtools, and MiKTeX"
}

initexmf --set-config-value [MPM]AutoInstall=1
conda install -y --no-deps pandoc

Add-Content .Renviron "R_LIBS=$env:R_LIB_PATH"
Add-Content .Rprofile "options(repos = 'https://cran.rstudio.com')"
Add-Content .Rprofile "options(pkgType = 'binary')"
Add-Content .Rprofile "options(install.packages.check.source = 'no')"

Write-Output "Installing dependencies"
Rscript.exe -e "install.packages(c('data.table', 'jsonlite', 'Matrix', 'R6', 'testthat'), dependencies = c('Imports', 'Depends', 'LinkingTo'), lib = '$env:R_LIB_PATH')" ; Check-Output $?

Write-Output "Building R package"
Rscript build_r.R ; Check-Output $?

$PKG_FILE_NAME = Get-Item *.tar.gz
$PKG_NAME = $PKG_FILE_NAME.BaseName.split("_")[0]
$LOG_FILE_NAME = "$PKG_NAME.Rcheck/00check.log"

Write-Output "Running R CMD check"
$env:_R_CHECK_FORCE_SUGGESTS_=0
R.exe CMD check "${PKG_FILE_NAME}" --as-cran --no-multiarch; Check-Output $?

if (Get-Content "$LOG_FILE_NAME" | Select-String -Pattern "WARNING" -Quiet) {
echo "WARNINGS have been found by R CMD check!"
Check-Output $False
}
}
6 changes: 6 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -308,8 +308,14 @@ if(WIN32 AND (MINGW OR CYGWIN))
endif()

if(BUILD_FOR_R)
if(MSVC)
# https://docs.microsoft.com/en-us/cpp/build/reference/link-input-files?redirectedfrom=MSDN&view=vs-2019
TARGET_LINK_LIBRARIES(lightgbm ${CMAKE_CURRENT_BINARY_DIR}/R.lib)
TARGET_LINK_LIBRARIES(_lightgbm ${CMAKE_CURRENT_BINARY_DIR}/R.lib)
else()
TARGET_LINK_LIBRARIES(lightgbm ${LIBR_CORE_LIBRARY})
TARGET_LINK_LIBRARIES(_lightgbm ${LIBR_CORE_LIBRARY})
endif()
endif(BUILD_FOR_R)

install(TARGETS lightgbm _lightgbm
Expand Down
40 changes: 21 additions & 19 deletions R-package/src/cmake/modules/FindLibR.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@
# LIBR_HOME
# LIBR_EXECUTABLE
# LIBR_INCLUDE_DIRS
# LIBR_LIB_DIR
# LIBR_CORE_LIBRARY
# and a CMake function to create R.lib for MSVC

Expand All @@ -36,8 +35,8 @@ function(create_rlib_for_msvc)
message(FATAL_ERROR "create_rlib_for_msvc() can only be used with MSVC")
endif()

if(NOT EXISTS "${LIBR_LIB_DIR}")
message(FATAL_ERROR "LIBR_LIB_DIR, '${LIBR_LIB_DIR}', not found")
if(NOT EXISTS "${LIBR_CORE_LIBRARY}")
message(FATAL_ERROR "LIBR_CORE_LIBRARY, '${LIBR_CORE_LIBRARY}', not found")
endif()

find_program(GENDEF_EXE gendef)
Expand All @@ -50,7 +49,8 @@ function(create_rlib_for_msvc)

# extract symbols from R.dll into R.def and R.lib import library
execute_process(COMMAND ${GENDEF_EXE}
"-" "${LIBR_LIB_DIR}/R.dll"
#"-" "${LIBR_LIB_DIR}/R.dll"
"-" "${LIBR_CORE_LIBRARY}"
OUTPUT_FILE "${CMAKE_CURRENT_BINARY_DIR}/R.def"
)
execute_process(COMMAND ${DLLTOOL_EXE}
Expand Down Expand Up @@ -168,25 +168,28 @@ execute_process(
OUTPUT_VARIABLE LIBR_INCLUDE_DIRS
)

# ask R for the lib dir
execute_process(
COMMAND ${LIBR_EXECUTABLE} "--slave" "--vanilla" "-e" "cat(normalizePath(R.home('lib'), winslash='/'))"
OUTPUT_VARIABLE LIBR_LIB_DIR
)

# look for the core R library
find_library(
LIBR_CORE_LIBRARY
NAMES R
HINTS "${CMAKE_CURRENT_BINARY_DIR}" "${LIBR_LIB_DIR}" "${LIBR_HOME}/bin" "${LIBR_LIBRARIES}"
)

set(LIBR_HOME ${LIBR_HOME} CACHE PATH "R home directory")
set(LIBR_EXECUTABLE ${LIBR_EXECUTABLE} CACHE PATH "R executable")
set(LIBR_INCLUDE_DIRS ${LIBR_INCLUDE_DIRS} CACHE PATH "R include directory")
set(LIBR_LIB_DIR ${LIBR_LIB_DIR} CACHE PATH "R shared libraries directory")

# look for the core R library
if(WIN32)
set(LIBR_CORE_LIBRARY ${LIBR_HOME}/bin/${R_ARCH}/R.dll)
else()
find_library(
LIBR_CORE_LIBRARY
NAMES R
HINTS "${CMAKE_CURRENT_BINARY_DIR}" "${LIBR_HOME}/lib" "${LIBR_HOME}/bin/${R_ARCH}" "${LIBR_HOME}/bin" "${LIBR_LIBRARIES}"
)
endif()

set(LIBR_CORE_LIBRARY ${LIBR_CORE_LIBRARY} CACHE PATH "R core shared library")

message(STATUS "LIBR_EXECUTABLE: ${LIBR_EXECUTABLE}")
message(STATUS "LIBR_HOME: ${LIBR_HOME}")
message(STATUS "LIBR_INCLUDE_DIRS: ${LIBR_INCLUDE_DIRS}")
message(STATUS "LIBR_CORE_LIBRARY: ${LIBR_CORE_LIBRARY}")

if(WIN32 AND MSVC)

# create a local R.lib import library for R.dll if it doesn't exist
Expand All @@ -203,6 +206,5 @@ find_package_handle_standard_args(LibR DEFAULT_MSG
LIBR_HOME
LIBR_EXECUTABLE
LIBR_INCLUDE_DIRS
LIBR_LIB_DIR
LIBR_CORE_LIBRARY
)
1 change: 1 addition & 0 deletions R-package/src/install.libs.R
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,7 @@ if (!use_precompile) {
# Check if Windows installation (for gcc vs Visual Studio)
if (WINDOWS) {
if (use_mingw) {
print("Trying to build with MinGW")
cmake_cmd <- paste0(cmake_cmd, " -G \"MinGW Makefiles\" ")
build_cmd <- "mingw32-make.exe _lightgbm"
system(paste0(cmake_cmd, " ..")) # Must build twice for Windows due sh.exe in Rtools
Expand Down

0 comments on commit e54954a

Please sign in to comment.