From e54954ad077389f019ea19add4829f23fb4aa7a0 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 23 Mar 2020 21:33:06 -0500 Subject: [PATCH] added Windows script, download retries, and MSVC linking --- .appveyor.yml | 3 + .ci/test_r_package_windows.ps1 | 86 ++++++++++++++++++ .ci/test_windows.ps1 | 100 +++------------------ CMakeLists.txt | 6 ++ R-package/src/cmake/modules/FindLibR.cmake | 40 +++++---- R-package/src/install.libs.R | 1 + 6 files changed, 130 insertions(+), 106 deletions(-) create mode 100644 .ci/test_r_package_windows.ps1 diff --git a/.appveyor.yml b/.appveyor.yml index 0fbd8d88f8c1..63651fb9f1b7 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -16,6 +16,9 @@ environment: - COMPILER: MINGW TASK: r-package R_WINDOWS_VERSION: 3.6.3 + - COMPILER: MSVC + TASK: r-package + R_WINDOWS_VERSION: 3.6.3 clone_depth: 5 diff --git a/.ci/test_r_package_windows.ps1 b/.ci/test_r_package_windows.ps1 new file mode 100644 index 000000000000..b416cae9eb11 --- /dev/null +++ b/.ci/test_r_package_windows.ps1 @@ -0,0 +1,86 @@ +# Download a file and retry upon failure. This looks like +# an infinite looop +function Download-File-With-Retries { + param( + [string]$url, + [string]$destfile + ) + do { + Write-Output "Downloading '${url}'" + sleep 5; + (New-Object System.Net.WebClient).DownloadFile($url, $destfile) + } while(!$?); +} + +$env:R_LIB_PATH = "C:/RLibrary" +$env:PATH = "$env:R_LIB_PATH/Rtools/bin;" + "$env:R_LIB_PATH/R/bin/x64;" + "$env:R_LIB_PATH/miktex/texmfs/install/miktex/bin/x64;" + $env:PATH +$env:BINPREF = "C:/mingw-w64/x86_64-8.1.0-posix-seh-rt_v6-rev0/mingw64/bin/" + +cd $env:BUILD_SOURCESDIRECTORY +tzutil /s "GMT Standard Time" +[Void][System.IO.Directory]::CreateDirectory($env:R_LIB_PATH) + +if ($env:COMPILER -eq "MINGW") { + Write-Output "Telling R to use MinGW" + $install_libs = "$env:BUILD_SOURCESDIRECTORY\R-package\src\install.libs.R" + ((Get-Content -path $install_libs -Raw) -replace 'use_mingw <- FALSE','use_mingw <- TRUE') | Set-Content -Path $install_libs +} + +# set up R if it doesn't exist yet +if (!(Get-Command R.exe -errorAction SilentlyContinue)) { + + Write-Output "Downloading R and Rtools" + + # download R and RTools + Download-File-With-Retries -url "https://cloud.r-project.org/bin/windows/base/old/$env:R_WINDOWS_VERSION/R-$env:R_WINDOWS_VERSION-win.exe" -destfile "R-win.exe" + Download-File-With-Retries -url "https://cloud.r-project.org/bin/windows/Rtools/Rtools35.exe" -destfile "Rtools.exe" + + # Install R + Write-Output "Installing R" + Start-Process -FilePath R-win.exe -NoNewWindow -Wait -ArgumentList "/VERYSILENT /DIR=$env:R_LIB_PATH/R /COMPONENTS=main,x64" ; Check-Output $? + Write-Output "Done installing R" + + Write-Output "Installing Rtools" + Start-Process -FilePath Rtools.exe -NoNewWindow -Wait -ArgumentList "/VERYSILENT /DIR=$env:R_LIB_PATH/Rtools" ; Check-Output $? + Write-Output "Done installing Rtools" + + # download Miktex + Write-Output "Downloading MiKTeX" + Download-File-With-Retries -url "https://miktex.org/download/win/miktexsetup-x64.zip" -destfile "miktexsetup-x64.zip" + Add-Type -AssemblyName System.IO.Compression.FileSystem + [System.IO.Compression.ZipFile]::ExtractToDirectory("miktexsetup-x64.zip", "miktex") + Write-Output "Setting up MiKTeX" + .\miktex\miktexsetup.exe --local-package-repository=./miktex/download --package-set=essential --quiet download ; Check-Output $? + Write-Output "Installing MiKTeX" + .\miktex\download\miktexsetup.exe --portable="$env:R_LIB_PATH/miktex" --quiet install ; Check-Output $? + Write-Output "Done installing R, Rtools, and MiKTeX" +} + +initexmf --set-config-value [MPM]AutoInstall=1 +conda install -y --no-deps pandoc + +Add-Content .Renviron "R_LIBS=$env:R_LIB_PATH" +Add-Content .Rprofile "options(repos = 'https://cran.rstudio.com')" +Add-Content .Rprofile "options(pkgType = 'binary')" +Add-Content .Rprofile "options(install.packages.check.source = 'no')" + +Write-Output "Installing dependencies" +Rscript.exe -e "install.packages(c('data.table', 'jsonlite', 'Matrix', 'R6', 'testthat'), dependencies = c('Imports', 'Depends', 'LinkingTo'), lib = '$env:R_LIB_PATH')" ; Check-Output $? + +Write-Output "Building R package" +Rscript build_r.R ; Check-Output $? + +$PKG_FILE_NAME = Get-Item *.tar.gz +$PKG_NAME = $PKG_FILE_NAME.BaseName.split("_")[0] +$LOG_FILE_NAME = "$PKG_NAME.Rcheck/00check.log" + +Write-Output "Running R CMD check" +$env:_R_CHECK_FORCE_SUGGESTS_=0 +R.exe CMD check "${PKG_FILE_NAME}" --as-cran --no-multiarch; Check-Output $? + +if (Get-Content "$LOG_FILE_NAME" | Select-String -Pattern "WARNING" -Quiet) { + echo "WARNINGS have been found by R CMD check!" + Check-Output $False +} + +Exit 0 diff --git a/.ci/test_windows.ps1 b/.ci/test_windows.ps1 index 741a1a5c6f3f..e724be4cc48b 100644 --- a/.ci/test_windows.ps1 +++ b/.ci/test_windows.ps1 @@ -11,16 +11,19 @@ if (Test-Path env:APPVEYOR) { $env:BUILD_SOURCESDIRECTORY = $env:APPVEYOR_BUILD_FOLDER } -# setup for Python -if ($env:TASK -ne "r-package") { - conda init powershell - conda activate - conda config --set always_yes yes --set changeps1 no - conda update -q -y conda - conda create -q -y -n $env:CONDA_ENV python=$env:PYTHON_VERSION joblib matplotlib numpy pandas psutil pytest python-graphviz "scikit-learn<=0.21.3" scipy wheel ; Check-Output $? - conda activate $env:CONDA_ENV +if ($env:TASK -eq "r-package") { + & $env:BUILD_SOURCESDIRECTORY\.ci\test_r_package_windows.ps1 ; Check-Output $? + Exit 0 } +# setup for Python +conda init powershell +conda activate +conda config --set always_yes yes --set changeps1 no +conda update -q -y conda +conda create -q -y -n $env:CONDA_ENV python=$env:PYTHON_VERSION joblib matplotlib numpy pandas psutil pytest python-graphviz "scikit-learn<=0.21.3" scipy wheel ; Check-Output $? +conda activate $env:CONDA_ENV + if ($env:TASK -eq "regular") { mkdir $env:BUILD_SOURCESDIRECTORY/build; cd $env:BUILD_SOURCESDIRECTORY/build cmake -A x64 .. ; cmake --build . --target ALL_BUILD --config Release ; Check-Output $? @@ -57,10 +60,8 @@ elseif ($env:TASK -eq "bdist") { } } -if ($env:TASK -ne "r-package") { - $tests = $env:BUILD_SOURCESDIRECTORY + $(If (($env:TASK -eq "sdist") -or ($env:TASK -eq "appveyor-python")) {"/tests/python_package_test"} Else {"/tests"}) # cannot test C API with "sdist" task - pytest $tests ; Check-Output $? -} +$tests = $env:BUILD_SOURCESDIRECTORY + $(If (($env:TASK -eq "sdist") -or ($env:TASK -eq "appveyor-python")) {"/tests/python_package_test"} Else {"/tests"}) # cannot test C API with "sdist" task +pytest $tests ; Check-Output $? if (($env:TASK -eq "regular") -or ($env:TASK -eq "appveyor-python")) { cd $env:BUILD_SOURCESDIRECTORY/examples/python-guide @@ -74,78 +75,3 @@ if (($env:TASK -eq "regular") -or ($env:TASK -eq "appveyor-python")) { conda install -q -y -n $env:CONDA_ENV ipywidgets notebook jupyter nbconvert --ExecutePreprocessor.timeout=180 --to notebook --execute --inplace *.ipynb ; Check-Output $? # run all notebooks } - -# test R package -if ($env:TASK -eq "r-package"){ - - $env:R_LIB_PATH = "C:/RLibrary" - $env:PATH = "$env:R_LIB_PATH/Rtools/bin;" + "$env:R_LIB_PATH/R/bin/x64;" + "$env:R_LIB_PATH/miktex/texmfs/install/miktex/bin/x64;" + $env:PATH - $env:BINPREF = "C:/mingw-w64/x86_64-8.1.0-posix-seh-rt_v6-rev0/mingw64/bin/" - - cd $env:BUILD_SOURCESDIRECTORY - tzutil /s "GMT Standard Time" - [Void][System.IO.Directory]::CreateDirectory($env:R_LIB_PATH) - - if ($env:COMPILER -eq "MINGW") { - Write-Output "Telling R to use MinGW" - $install_libs = "$env:BUILD_SOURCESDIRECTORY\R-package\src\install.libs.R" - ((Get-Content -path $install_libs -Raw) -replace 'use_mingw <- FALSE','use_mingw <- TRUE') | Set-Content -Path $install_libs - } - - # set up R if it doesn't exist yet - if (!(Get-Command R.exe -errorAction SilentlyContinue)) { - - Write-Output "Downloading R and Rtools" - - # download R and RTools - (New-Object System.Net.WebClient).DownloadFile("https://cloud.r-project.org/bin/windows/base/old/$env:R_WINDOWS_VERSION/R-$env:R_WINDOWS_VERSION-win.exe", "R-win.exe") - (New-Object System.Net.WebClient).DownloadFile("https://cloud.r-project.org/bin/windows/Rtools/Rtools35.exe", "Rtools.exe") - - # Install R - Write-Output "Installing R" - Start-Process -FilePath R-win.exe -NoNewWindow -Wait -ArgumentList "/VERYSILENT /DIR=$env:R_LIB_PATH/R /COMPONENTS=main,x64" ; Check-Output $? - Write-Output "Done installing R" - - Write-Output "Installing Rtools" - Start-Process -FilePath Rtools.exe -NoNewWindow -Wait -ArgumentList "/VERYSILENT /DIR=$env:R_LIB_PATH/Rtools" ; Check-Output $? - Write-Output "Done installing Rtools" - - # download Miktex - Write-Output "Downloading MiKTeX" - (New-Object System.Net.WebClient).DownloadFile("https://miktex.org/download/win/miktexsetup-x64.zip", "miktexsetup-x64.zip") - Add-Type -AssemblyName System.IO.Compression.FileSystem - [System.IO.Compression.ZipFile]::ExtractToDirectory("miktexsetup-x64.zip", "miktex") - Write-Output "Setting up MiKTeX" - .\miktex\miktexsetup.exe --local-package-repository=./miktex/download --package-set=essential --quiet download ; Check-Output $? - Write-Output "Installing MiKTeX" - .\miktex\download\miktexsetup.exe --portable="$env:R_LIB_PATH/miktex" --quiet install ; Check-Output $? - Write-Output "Done installing R, Rtools, and MiKTeX" - } - - initexmf --set-config-value [MPM]AutoInstall=1 - conda install -y --no-deps pandoc - - Add-Content .Renviron "R_LIBS=$env:R_LIB_PATH" - Add-Content .Rprofile "options(repos = 'https://cran.rstudio.com')" - Add-Content .Rprofile "options(pkgType = 'binary')" - Add-Content .Rprofile "options(install.packages.check.source = 'no')" - - Write-Output "Installing dependencies" - Rscript.exe -e "install.packages(c('data.table', 'jsonlite', 'Matrix', 'R6', 'testthat'), dependencies = c('Imports', 'Depends', 'LinkingTo'), lib = '$env:R_LIB_PATH')" ; Check-Output $? - - Write-Output "Building R package" - Rscript build_r.R ; Check-Output $? - - $PKG_FILE_NAME = Get-Item *.tar.gz - $PKG_NAME = $PKG_FILE_NAME.BaseName.split("_")[0] - $LOG_FILE_NAME = "$PKG_NAME.Rcheck/00check.log" - - Write-Output "Running R CMD check" - $env:_R_CHECK_FORCE_SUGGESTS_=0 - R.exe CMD check "${PKG_FILE_NAME}" --as-cran --no-multiarch; Check-Output $? - - if (Get-Content "$LOG_FILE_NAME" | Select-String -Pattern "WARNING" -Quiet) { - echo "WARNINGS have been found by R CMD check!" - Check-Output $False - } -} diff --git a/CMakeLists.txt b/CMakeLists.txt index 895f2dd26fce..b28fade32694 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -308,8 +308,14 @@ if(WIN32 AND (MINGW OR CYGWIN)) endif() if(BUILD_FOR_R) + if(MSVC) + # https://docs.microsoft.com/en-us/cpp/build/reference/link-input-files?redirectedfrom=MSDN&view=vs-2019 + TARGET_LINK_LIBRARIES(lightgbm ${CMAKE_CURRENT_BINARY_DIR}/R.lib) + TARGET_LINK_LIBRARIES(_lightgbm ${CMAKE_CURRENT_BINARY_DIR}/R.lib) + else() TARGET_LINK_LIBRARIES(lightgbm ${LIBR_CORE_LIBRARY}) TARGET_LINK_LIBRARIES(_lightgbm ${LIBR_CORE_LIBRARY}) + endif() endif(BUILD_FOR_R) install(TARGETS lightgbm _lightgbm diff --git a/R-package/src/cmake/modules/FindLibR.cmake b/R-package/src/cmake/modules/FindLibR.cmake index 20c0a974e1ff..9716f99ae3b3 100644 --- a/R-package/src/cmake/modules/FindLibR.cmake +++ b/R-package/src/cmake/modules/FindLibR.cmake @@ -10,7 +10,6 @@ # LIBR_HOME # LIBR_EXECUTABLE # LIBR_INCLUDE_DIRS -# LIBR_LIB_DIR # LIBR_CORE_LIBRARY # and a CMake function to create R.lib for MSVC @@ -36,8 +35,8 @@ function(create_rlib_for_msvc) message(FATAL_ERROR "create_rlib_for_msvc() can only be used with MSVC") endif() - if(NOT EXISTS "${LIBR_LIB_DIR}") - message(FATAL_ERROR "LIBR_LIB_DIR, '${LIBR_LIB_DIR}', not found") + if(NOT EXISTS "${LIBR_CORE_LIBRARY}") + message(FATAL_ERROR "LIBR_CORE_LIBRARY, '${LIBR_CORE_LIBRARY}', not found") endif() find_program(GENDEF_EXE gendef) @@ -50,7 +49,8 @@ function(create_rlib_for_msvc) # extract symbols from R.dll into R.def and R.lib import library execute_process(COMMAND ${GENDEF_EXE} - "-" "${LIBR_LIB_DIR}/R.dll" + #"-" "${LIBR_LIB_DIR}/R.dll" + "-" "${LIBR_CORE_LIBRARY}" OUTPUT_FILE "${CMAKE_CURRENT_BINARY_DIR}/R.def" ) execute_process(COMMAND ${DLLTOOL_EXE} @@ -168,25 +168,28 @@ execute_process( OUTPUT_VARIABLE LIBR_INCLUDE_DIRS ) -# ask R for the lib dir -execute_process( - COMMAND ${LIBR_EXECUTABLE} "--slave" "--vanilla" "-e" "cat(normalizePath(R.home('lib'), winslash='/'))" - OUTPUT_VARIABLE LIBR_LIB_DIR -) - -# look for the core R library -find_library( - LIBR_CORE_LIBRARY - NAMES R - HINTS "${CMAKE_CURRENT_BINARY_DIR}" "${LIBR_LIB_DIR}" "${LIBR_HOME}/bin" "${LIBR_LIBRARIES}" -) - set(LIBR_HOME ${LIBR_HOME} CACHE PATH "R home directory") set(LIBR_EXECUTABLE ${LIBR_EXECUTABLE} CACHE PATH "R executable") set(LIBR_INCLUDE_DIRS ${LIBR_INCLUDE_DIRS} CACHE PATH "R include directory") -set(LIBR_LIB_DIR ${LIBR_LIB_DIR} CACHE PATH "R shared libraries directory") + +# look for the core R library +if(WIN32) + set(LIBR_CORE_LIBRARY ${LIBR_HOME}/bin/${R_ARCH}/R.dll) +else() + find_library( + LIBR_CORE_LIBRARY + NAMES R + HINTS "${CMAKE_CURRENT_BINARY_DIR}" "${LIBR_HOME}/lib" "${LIBR_HOME}/bin/${R_ARCH}" "${LIBR_HOME}/bin" "${LIBR_LIBRARIES}" + ) +endif() + set(LIBR_CORE_LIBRARY ${LIBR_CORE_LIBRARY} CACHE PATH "R core shared library") +message(STATUS "LIBR_EXECUTABLE: ${LIBR_EXECUTABLE}") +message(STATUS "LIBR_HOME: ${LIBR_HOME}") +message(STATUS "LIBR_INCLUDE_DIRS: ${LIBR_INCLUDE_DIRS}") +message(STATUS "LIBR_CORE_LIBRARY: ${LIBR_CORE_LIBRARY}") + if(WIN32 AND MSVC) # create a local R.lib import library for R.dll if it doesn't exist @@ -203,6 +206,5 @@ find_package_handle_standard_args(LibR DEFAULT_MSG LIBR_HOME LIBR_EXECUTABLE LIBR_INCLUDE_DIRS - LIBR_LIB_DIR LIBR_CORE_LIBRARY ) diff --git a/R-package/src/install.libs.R b/R-package/src/install.libs.R index f6d2335b2cd8..bf4eb8ab1169 100644 --- a/R-package/src/install.libs.R +++ b/R-package/src/install.libs.R @@ -67,6 +67,7 @@ if (!use_precompile) { # Check if Windows installation (for gcc vs Visual Studio) if (WINDOWS) { if (use_mingw) { + print("Trying to build with MinGW") cmake_cmd <- paste0(cmake_cmd, " -G \"MinGW Makefiles\" ") build_cmd <- "mingw32-make.exe _lightgbm" system(paste0(cmake_cmd, " ..")) # Must build twice for Windows due sh.exe in Rtools