Skip to content

Commit

Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[R-package] make package installable with CRAN tooolchain (fixes micr…
Browse files Browse the repository at this point in the history
jameslamb committed Jun 23, 2020

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature. The key has expired.
1 parent f5e5164 commit d2bcf5a
Showing 17 changed files with 3,728 additions and 23 deletions.
18 changes: 16 additions & 2 deletions .ci/test_r_package.sh
Original file line number Diff line number Diff line change
@@ -54,6 +54,8 @@ fi

# Installing R precompiled for Mac OS 10.11 or higher
if [[ $OS_NAME == "macos" ]]; then

brew install automake
brew install qpdf
brew cask install basictex
export PATH="/Library/TeX/texbin:$PATH"
@@ -91,10 +93,22 @@ fi
Rscript --vanilla -e "install.packages(${packages}, repos = '${CRAN_MIRROR}', lib = '${R_LIB_PATH}', dependencies = c('Depends', 'Imports', 'LinkingTo'))" || exit -1

cd ${BUILD_DIRECTORY}
Rscript build_r.R --skip-install || exit -1

PKG_TARBALL="lightgbm_${LGB_VER}.tar.gz"
LOG_FILE_NAME="lightgbm.Rcheck/00check.log"
if [[ $R_BUILD_TYPE == "cmake" ]]; then
Rscript build_r.R --skip-install || exit -1
elif [[ $R_BUILD_TYPE == "cran" ]]; then
./build-cran-package.sh || exit -1
# Test CRAN source .tar.gz in a directory that is not this repo or below it.
# When people install.packages('lightgbm'), they won't have the LightGBM
# git repo around. This is to protect against the use of relative paths
# like ../../CMakeLists.txt that would only work if you are in the repoo
R_CMD_CHECK_DIR="${HOME}/tmp-r-cmd-check/"
mkdir -p ${R_CMD_CHECK_DIR}
mv ${PKG_TARBALL} ${R_CMD_CHECK_DIR}
cd ${R_CMD_CHECK_DIR}
fi

# suppress R CMD check warning from Suggests dependencies not being available
export _R_CHECK_FORCE_SUGGESTS_=0
@@ -119,7 +133,7 @@ while kill -0 ${CHECK_PID} >/dev/null 2>&1; do
done

echo "R CMD check build logs:"
cat ${BUILD_DIRECTORY}/lightgbm.Rcheck/00install.out
cat lightgbm.Rcheck/00install.out

if [[ $check_succeeded == "no" ]]; then
exit -1
57 changes: 40 additions & 17 deletions .ci/test_r_package_windows.ps1
Original file line number Diff line number Diff line change
@@ -44,9 +44,13 @@ function Run-R-Code-Redirect-Stderr {
Rscript --vanilla -e $decorated_code
}

# Rtools has to be installed at C:\Rtools\
# * https://stackoverflow.com/a/46619260/3986677
$RTOOLS_INSTALL_PATH = "C:\Rtools"

$env:R_LIB_PATH = "$env:BUILD_SOURCESDIRECTORY/RLibrary" -replace '[\\]', '/'
$env:R_LIBS = "$env:R_LIB_PATH"
$env:PATH = "$env:R_LIB_PATH/Rtools/bin;" + "$env:R_LIB_PATH/Rtools/usr/bin;" + "$env:R_LIB_PATH/R/bin/x64;" + "$env:R_LIB_PATH/miktex/texmfs/install/miktex/bin/x64;" + $env:PATH
$env:PATH = "$RTOOLS_INSTALL_PATH/bin;" + "$env:R_LIB_PATH/Rtools/usr/bin;" + "$env:R_LIB_PATH/R/bin/x64;" + "$env:R_LIB_PATH/miktex/texmfs/install/miktex/bin/x64;" + $env:PATH
$env:CRAN_MIRROR = "https://cloud.r-project.org/"
$env:CTAN_MIRROR = "https://ctan.math.illinois.edu/systems/win32/miktex"
$env:CTAN_MIKTEX_ARCHIVE = "$env:CTAN_MIRROR/setup/windows-x64/"
@@ -70,20 +74,21 @@ if ($env:R_MAJOR_VERSION -eq "3") {
Check-Output $false
}

if ($env:COMPILER -eq "MINGW") {
$env:CXX = "$env:RTOOLS_MINGW_BIN/g++.exe"
$env:CC = "$env:RTOOLS_MINGW_BIN/gcc.exe"
if (($env:COMPILER -eq "MINGW") -and ($env:R_BUILD_TYPE -eq "cmake")) {
$env:CXX = "$RTOOLS_INSTALL_PATH/mingw_64/bin/g++.exe"
$env:CC = "$RTOOLS_INSTALL_PATH/mingw_64/bin/gcc.exe"
}

cd $env:BUILD_SOURCESDIRECTORY
tzutil /s "GMT Standard Time"
[Void][System.IO.Directory]::CreateDirectory($env:R_LIB_PATH)
$env:LGB_VER = Get-Content -Path VERSION.txt -TotalCount 1

if ($env:TOOLCHAIN -eq "MINGW") {
if (($env:TOOLCHAIN -eq "MINGW") -and ($env:R_BUILD_TYPE -eq "cmake")) {
Write-Output "Telling R to use MinGW"
$install_libs = "$env:BUILD_SOURCESDIRECTORY/R-package/src/install.libs.R"
((Get-Content -Path $install_libs -Raw) -Replace 'use_mingw <- FALSE','use_mingw <- TRUE') | Set-Content -Path $install_libs
} elseif ($env:TOOLCHAIN -eq "MSYS") {
if (($env:TOOLCHAIN -eq "MSYS") -and ($env:R_BUILD_TYPE -eq "cmake")) {
Write-Output "Telling R to use MSYS"
$install_libs = "$env:BUILD_SOURCESDIRECTORY/R-package/src/install.libs.R"
((Get-Content -Path $install_libs -Raw) -Replace 'use_msys2 <- FALSE','use_msys2 <- TRUE') | Set-Content -Path $install_libs
@@ -105,16 +110,18 @@ Start-Process -FilePath R-win.exe -NoNewWindow -Wait -ArgumentList "/VERYSILENT
Write-Output "Done installing R"

Write-Output "Installing Rtools"
Start-Process -FilePath Rtools.exe -NoNewWindow -Wait -ArgumentList "/VERYSILENT /DIR=$env:R_LIB_PATH/Rtools" ; Check-Output $?
Start-Process -FilePath Rtools.exe -NoNewWindow -Wait -ArgumentList "/VERYSILENT /DIR=$RTOOLS_INSTALL_PATH" ; Check-Output $?
Write-Output "Done installing Rtools"

Write-Output "Installing dependencies"
$packages = "c('data.table', 'jsonlite', 'Matrix', 'processx', 'R6', 'testthat'), dependencies = c('Imports', 'Depends', 'LinkingTo')"
Run-R-Code-Redirect-Stderr "options(install.packages.check.source = 'no'); install.packages($packages, repos = '$env:CRAN_MIRROR', type = 'binary', lib = '$env:R_LIB_PATH')" ; Check-Output $?

# MiKTeX and pandoc can be skipped on MSVC builds, since we don't
# build the package documentation for those
if ($env:COMPILER -ne "MSVC") {
# MiKTeX and pandoc can be skipped on non-MinGW builds, since we don't
# build the package documentation for those.
#
# MiKTeX always needs to be built to test a CRAN package.
if (($env:COMPILER -eq "MINGW") -or ($env:R_BUILD_TYPE -eq "cran")) {
Download-Miktex-Setup "$env:CTAN_MIKTEX_ARCHIVE" "miktexsetup-x64.zip"
Add-Type -AssemblyName System.IO.Compression.FileSystem
[System.IO.Compression.ZipFile]::ExtractToDirectory("miktexsetup-x64.zip", "miktex")
@@ -134,16 +141,29 @@ Write-Output "Building R package"
if ($env:COMPILER -ne "MSVC") {
Run-R-Code-Redirect-Stderr "commandArgs <- function(...){'--skip-install'}; source('build_r.R')"; Check-Output $?

$PKG_FILE_NAME = Get-Item *.tar.gz
$PKG_FILE_NAME = $PKG_FILE_NAME -replace '[\\]', '/'
$PKG_FILE_NAME = "lightgbm_$env:LGB_VER.tar.gz"
$LOG_FILE_NAME = "lightgbm.Rcheck/00check.log"

if ($env:R_BUILD_TYPE -eq "cmake") {
Rscript build_r.R --skip-install ; Check-Output $?
} elseif ($env:R_BUILD_TYPE -eq "cran") {
sh build-cran-package.sh ; Check-Output $?
# Test CRAN source .tar.gz in a directory that is not this repo or below it.
# When people install.packages('lightgbm'), they won't have the LightGBM
# git repo around. This is to protect against the use of relative paths
# like ../../CMakeLists.txt that would only work if you are in the repoo
$R_CMD_CHECK_DIR="tmp-r-cmd-check"
New-Item -Path "C:\" -Name $R_CMD_CHECK_DIR -ItemType "directory" > $null
Move-Item -Path "$PKG_FILE_NAME" -Destination "C:\$R_CMD_CHECK_DIR\" > $null
cd "C:\$R_CMD_CHECK_DIR\"
}

$env:_R_CHECK_FORCE_SUGGESTS_ = 0
Write-Output "Running R CMD check as CRAN"
Run-R-Code-Redirect-Stderr "processx::run(command = 'R.exe', args = c('CMD', 'check', '--no-multiarch', '--as-cran', '$PKG_FILE_NAME'), windows_verbatim_args = FALSE)" ; $check_succeeded = $?

Write-Output "R CMD check build logs:"
$INSTALL_LOG_FILE_NAME = "$env:BUILD_SOURCESDIRECTORY\lightgbm.Rcheck\00install.out"
$INSTALL_LOG_FILE_NAME = "lightgbm.Rcheck\00install.out"
Get-Content -Path "$INSTALL_LOG_FILE_NAME"

Check-Output $check_succeeded
@@ -176,9 +196,12 @@ if ($env:COMPILER -ne "MSVC") {
# to fail back to MinGW if MSVC fails, but for CI builds we need to check that the correct
# compiler was used.
$checks = Select-String -Path "${INSTALL_LOG_FILE_NAME}" -Pattern "Check for working CXX compiler.*$env:COMPILER"
if ($checks.Matches.length -eq 0) {
Write-Output "The wrong compiler was used. Check the build logs."
Check-Output $False
if ($env:R_BUILD_TYPE -eq "cmake") {
$checks = Select-String -Path "${INSTALL_LOG_FILE_NAME}" -Pattern "Check for working CXX compiler.*$env:COMPILER"
if ($checks.Matches.length -eq 0) {
Write-Output "The wrong compiler was used. Check the build logs."
Check-Output $False
}
}

# Checking that we got the right toolchain for MinGW. If using MinGW, both
@@ -191,7 +214,7 @@ if ($env:COMPILER -eq "MINGW") {
}
}

if ($env:COMPILER -eq "MSVC") {
if (($env:COMPILER -eq "MSVC") -and ($env:R_BUILD_TYPE -eq "cmake")) {
Write-Output "Running tests with testthat.R"
cd R-package/tests
Run-R-Code-Redirect-Stderr "source('testthat.R')" ; Check-Output $?
33 changes: 33 additions & 0 deletions .github/workflows/main.yml
Original file line number Diff line number Diff line change
@@ -20,56 +20,87 @@ jobs:
task: r-package
compiler: gcc
r_version: 3.6
build_type: cmake
- os: ubuntu-latest
task: r-package
compiler: gcc
r_version: 4.0
build_type: cmake
- os: ubuntu-latest
task: r-package
compiler: clang
r_version: 3.6
build_type: cmake
- os: ubuntu-latest
task: r-package
compiler: clang
r_version: 4.0
build_type: cmake
- os: macOS-latest
task: r-package
compiler: gcc
r_version: 3.6
build_type: cmake
- os: macOS-latest
task: r-package
compiler: gcc
r_version: 4.0
build_type: cmake
- os: macOS-latest
task: r-package
compiler: clang
r_version: 3.6
build_type: cmake
- os: macOS-latest
task: r-package
compiler: clang
r_version: 4.0
build_type: cmake
- os: windows-latest
task: r-package
compiler: MINGW
toolchain: MINGW
r_version: 3.6
build_type: cmake
- os: windows-latest
task: r-package
compiler: MINGW
toolchain: MSYS
r_version: 4.0
build_type: cmake
# Visual Studio 2017
- os: windows-2016
task: r-package
compiler: MSVC
toolchain: MSVC
r_version: 3.6
build_type: cmake
# Visual Studio 2019
- os: windows-2019
task: r-package
compiler: MSVC
toolchain: MSVC
r_version: 4.0
build_type: cmake
###############
# CRAN builds #
###############
- os: windows-latest
task: r-package
compiler: MINGW
toolchain: MSYS
r_version: 4.0
build_type: cran
- os: ubuntu-latest
task: r-package
compiler: gcc
r_version: 4.0
build_type: cran
- os: macOS-latest
task: r-package
compiler: gcc
r_version: 4.0
build_type: cran
steps:
- name: Checkout repository
uses: actions/checkout@v1
@@ -96,6 +127,7 @@ jobs:
export PATH="$CONDA/bin:${HOME}/.local/bin:$PATH"
export LGB_VER=$(head -n 1 VERSION.txt)
export R_VERSION="${{ matrix.r_version }}"
export R_BUILD_TYPE="${{ matrix.build_type }}"
$GITHUB_WORKSPACE/.ci/setup.sh
$GITHUB_WORKSPACE/.ci/test.sh
- name: Use conda on Windows
@@ -110,6 +142,7 @@ jobs:
$env:BUILD_SOURCESDIRECTORY = $env:GITHUB_WORKSPACE
$env:TOOLCHAIN = "${{ matrix.toolchain }}"
$env:R_VERSION = "${{ matrix.r_version }}"
$env:R_BUILD_TYPE = "${{ matrix.build_type }}"
$env:COMPILER = "${{ matrix.compiler }}"
$env:GITHUB_ACTIONS = "true"
$env:TASK = "${{ matrix.task }}"
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -403,9 +403,11 @@ R-package/src/lib_lightgbm.so.dSYM/
R-package/src/src/
R-package/src-x64
R-package/src-i386
**/Makevars.win
lightgbm_r/*
lightgbm*.tar.gz
lightgbm.Rcheck/
**/autom4te.cache/
*.def

# Files created by R examples and tests
12 changes: 12 additions & 0 deletions .vsts-ci.yml
Original file line number Diff line number Diff line change
@@ -82,6 +82,14 @@ jobs:
PYTHON_VERSION: 3.5
bdist:
TASK: bdist
regular:
TASK: regular
PYTHON_VERSION: 3.7
sdist:
TASK: sdist
PYTHON_VERSION: 3.5
bdist:
TASK: bdist
steps:
- script: |
echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY"
@@ -120,6 +128,10 @@ jobs:
bdist:
TASK: bdist
steps:
- powershell: git config --global core.autocrlf false
displayName: $(System.JobName) git config core.autocrlf
- checkout: self
fetchDepth: 5
- powershell: |
Write-Host "##vso[task.prependpath]$env:CONDA\Scripts"
Write-Host "##vso[task.setvariable variable=AZURE]true"
1 change: 1 addition & 0 deletions R-package/.Rbuildignore
Original file line number Diff line number Diff line change
@@ -14,6 +14,7 @@
^src/CMakeLists.txt$
^Makefile$
^src/build/.*$
^autom4te.cache/.*$

# unnecessary files from submodules
^src/compute/.appveyor.yml$
Loading

0 comments on commit d2bcf5a

Please sign in to comment.