From a97c444b4cf9d2755bd888911ce65ace1fe13e4b Mon Sep 17 00:00:00 2001 From: James Lamb <jaylamb20@gmail.com> Date: Thu, 4 May 2023 17:06:11 -0500 Subject: [PATCH] [ci] [python-package] replace 'python setup.py' with a shell script (#5837) --- .appveyor.yml | 1 - .ci/test.sh | 73 +++++---- .ci/test_windows.ps1 | 22 +-- .gitignore | 1 + build-python.sh | 334 ++++++++++++++++++++++++++++++++++++++ docker/dockerfile-python | 2 +- docker/gpu/dockerfile.gpu | 2 +- docs/FAQ.rst | 4 + docs/GPU-Tutorial.rst | 4 +- python-package/README.rst | 25 ++- python-package/setup.py | 51 +----- 11 files changed, 413 insertions(+), 106 deletions(-) create mode 100755 build-python.sh diff --git a/.appveyor.yml b/.appveyor.yml index 274064fc56cd..a5cd02d69e23 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -23,7 +23,6 @@ clone_depth: 5 install: - git submodule update --init --recursive # get `external_libs` folder - - set PATH=%PATH:C:\Program Files\Git\usr\bin;=% # delete sh.exe from PATH (mingw32-make fix) - set PATH=C:\mingw-w64\x86_64-8.1.0-posix-seh-rt_v6-rev0\mingw64\bin;%PATH% - set PYTHON_VERSION=%CONFIGURATION% - set CONDA_ENV="test-env" diff --git a/.ci/test.sh b/.ci/test.sh index e8dff8ce3c0a..e4b6eb7fbae0 100755 --- a/.ci/test.sh +++ b/.ci/test.sh @@ -146,19 +146,21 @@ if [[ $OS_NAME == "macos" ]] && [[ $COMPILER == "clang" ]]; then fi if [[ $TASK == "sdist" ]]; then - cd $BUILD_DIRECTORY/python-package && python setup.py sdist || exit -1 - sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/python-package/dist || exit -1 - pip install --user $BUILD_DIRECTORY/python-package/dist/lightgbm-$LGB_VER.tar.gz -v || exit -1 + cd $BUILD_DIRECTORY && sh ./build-python.sh sdist || exit -1 + sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/dist || exit -1 + pip install --user $BUILD_DIRECTORY/dist/lightgbm-$LGB_VER.tar.gz -v || exit -1 if [[ $PRODUCES_ARTIFACTS == "true" ]]; then - cp $BUILD_DIRECTORY/python-package/dist/lightgbm-$LGB_VER.tar.gz $BUILD_ARTIFACTSTAGINGDIRECTORY + cp $BUILD_DIRECTORY/dist/lightgbm-$LGB_VER.tar.gz $BUILD_ARTIFACTSTAGINGDIRECTORY fi pytest $BUILD_DIRECTORY/tests/python_package_test || exit -1 exit 0 elif [[ $TASK == "bdist" ]]; then if [[ $OS_NAME == "macos" ]]; then - cd $BUILD_DIRECTORY/python-package && python setup.py bdist_wheel --plat-name=macosx --python-tag py3 || exit -1 - sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/python-package/dist || exit -1 - mv dist/lightgbm-$LGB_VER-py3-none-macosx.whl dist/lightgbm-$LGB_VER-py3-none-macosx_10_15_x86_64.macosx_11_6_x86_64.macosx_12_5_x86_64.whl + cd $BUILD_DIRECTORY && sh ./build-python.sh bdist_wheel || exit -1 + sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/dist || exit -1 + mv \ + dist/lightgbm-$LGB_VER-py3-none-macosx*.whl \ + dist/lightgbm-$LGB_VER-py3-none-macosx_10_15_x86_64.macosx_11_6_x86_64.macosx_12_5_x86_64.whl if [[ $PRODUCES_ARTIFACTS == "true" ]]; then cp dist/lightgbm-$LGB_VER-py3-none-macosx*.whl $BUILD_ARTIFACTSTAGINGDIRECTORY fi @@ -169,21 +171,22 @@ elif [[ $TASK == "bdist" ]]; then else PLATFORM="manylinux2014_$ARCH" fi - cd $BUILD_DIRECTORY/python-package && python setup.py bdist_wheel --integrated-opencl --plat-name=$PLATFORM --python-tag py3 || exit -1 - sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/python-package/dist || exit -1 + cd $BUILD_DIRECTORY && sh ./build-python.sh bdist_wheel --integrated-opencl || exit -1 + mv \ + ./dist/*.whl \ + ./dist/lightgbm-$LGB_VER-py3-none-$PLATFORM.whl + sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/dist || exit -1 if [[ $PRODUCES_ARTIFACTS == "true" ]]; then cp dist/lightgbm-$LGB_VER-py3-none-$PLATFORM.whl $BUILD_ARTIFACTSTAGINGDIRECTORY fi # Make sure we can do both CPU and GPU; see tests/python_package_test/test_dual.py export LIGHTGBM_TEST_DUAL_CPU_GPU=1 fi - pip install --user $BUILD_DIRECTORY/python-package/dist/*.whl || exit -1 + pip install --user $BUILD_DIRECTORY/dist/*.whl || exit -1 pytest $BUILD_DIRECTORY/tests || exit -1 exit 0 fi -mkdir $BUILD_DIRECTORY/build && cd $BUILD_DIRECTORY/build - # temporarily pin pip to versions that support 'pip install --install-option' # ref: https://github.com/microsoft/LightGBM/issues/5061#issuecomment-1510642287 if [[ $METHOD == "pip" ]]; then @@ -194,18 +197,20 @@ if [[ $TASK == "gpu" ]]; then sed -i'.bak' 's/std::string device_type = "cpu";/std::string device_type = "gpu";/' $BUILD_DIRECTORY/include/LightGBM/config.h grep -q 'std::string device_type = "gpu"' $BUILD_DIRECTORY/include/LightGBM/config.h || exit -1 # make sure that changes were really done if [[ $METHOD == "pip" ]]; then - cd $BUILD_DIRECTORY/python-package && python setup.py sdist || exit -1 - sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/python-package/dist || exit -1 - pip install --user $BUILD_DIRECTORY/python-package/dist/lightgbm-$LGB_VER.tar.gz -v --install-option=--gpu || exit -1 + cd $BUILD_DIRECTORY && sh ./build-python.sh sdist || exit -1 + sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/dist || exit -1 + pip install --user $BUILD_DIRECTORY/dist/lightgbm-$LGB_VER.tar.gz -v --install-option=--gpu || exit -1 pytest $BUILD_DIRECTORY/tests/python_package_test || exit -1 exit 0 elif [[ $METHOD == "wheel" ]]; then - cd $BUILD_DIRECTORY/python-package && python setup.py bdist_wheel --gpu || exit -1 - sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/python-package/dist || exit -1 - pip install --user $BUILD_DIRECTORY/python-package/dist/lightgbm-$LGB_VER*.whl -v || exit -1 + cd $BUILD_DIRECTORY && sh ./build-python.sh bdist_wheel --gpu || exit -1 + sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/dist || exit -1 + pip install --user $BUILD_DIRECTORY/dist/lightgbm-$LGB_VER*.whl -v || exit -1 pytest $BUILD_DIRECTORY/tests || exit -1 exit 0 elif [[ $METHOD == "source" ]]; then + mkdir $BUILD_DIRECTORY/build + cd $BUILD_DIRECTORY/build cmake -DUSE_GPU=ON .. fi elif [[ $TASK == "cuda" ]]; then @@ -215,43 +220,49 @@ elif [[ $TASK == "cuda" ]]; then sed -i'.bak' 's/gpu_use_dp = false;/gpu_use_dp = true;/' $BUILD_DIRECTORY/include/LightGBM/config.h grep -q 'gpu_use_dp = true' $BUILD_DIRECTORY/include/LightGBM/config.h || exit -1 # make sure that changes were really done if [[ $METHOD == "pip" ]]; then - cd $BUILD_DIRECTORY/python-package && python setup.py sdist || exit -1 - sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/python-package/dist || exit -1 - pip install --user $BUILD_DIRECTORY/python-package/dist/lightgbm-$LGB_VER.tar.gz -v --install-option=--cuda || exit -1 + cd $BUILD_DIRECTORY && sh ./build-python.sh sdist || exit -1 + sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/dist || exit -1 + pip install --user $BUILD_DIRECTORY/dist/lightgbm-$LGB_VER.tar.gz -v --install-option=--cuda || exit -1 pytest $BUILD_DIRECTORY/tests/python_package_test || exit -1 exit 0 elif [[ $METHOD == "wheel" ]]; then - cd $BUILD_DIRECTORY/python-package && python setup.py bdist_wheel --cuda || exit -1 - sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/python-package/dist || exit -1 - pip install --user $BUILD_DIRECTORY/python-package/dist/lightgbm-$LGB_VER*.whl -v || exit -1 + cd $BUILD_DIRECTORY && sh ./build-python.sh bdist_wheel --cuda || exit -1 + sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/dist || exit -1 + pip install --user $BUILD_DIRECTORY/dist/lightgbm-$LGB_VER*.whl -v || exit -1 pytest $BUILD_DIRECTORY/tests || exit -1 exit 0 elif [[ $METHOD == "source" ]]; then + mkdir $BUILD_DIRECTORY/build + cd $BUILD_DIRECTORY/build cmake -DUSE_CUDA=ON .. fi elif [[ $TASK == "mpi" ]]; then if [[ $METHOD == "pip" ]]; then - cd $BUILD_DIRECTORY/python-package && python setup.py sdist || exit -1 - sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/python-package/dist || exit -1 - pip install --user $BUILD_DIRECTORY/python-package/dist/lightgbm-$LGB_VER.tar.gz -v --install-option=--mpi || exit -1 + cd $BUILD_DIRECTORY && sh ./build-python.sh sdist || exit -1 + sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/dist || exit -1 + pip install --user $BUILD_DIRECTORY/dist/lightgbm-$LGB_VER.tar.gz -v --install-option=--mpi || exit -1 pytest $BUILD_DIRECTORY/tests/python_package_test || exit -1 exit 0 elif [[ $METHOD == "wheel" ]]; then - cd $BUILD_DIRECTORY/python-package && python setup.py bdist_wheel --mpi || exit -1 - sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/python-package/dist || exit -1 - pip install --user $BUILD_DIRECTORY/python-package/dist/lightgbm-$LGB_VER*.whl -v || exit -1 + cd $BUILD_DIRECTORY && sh ./build-python.sh bdist_wheel --mpi || exit -1 + sh $BUILD_DIRECTORY/.ci/check_python_dists.sh $BUILD_DIRECTORY/dist || exit -1 + pip install --user $BUILD_DIRECTORY/dist/lightgbm-$LGB_VER*.whl -v || exit -1 pytest $BUILD_DIRECTORY/tests || exit -1 exit 0 elif [[ $METHOD == "source" ]]; then + mkdir $BUILD_DIRECTORY/build + cd $BUILD_DIRECTORY/build cmake -DUSE_MPI=ON -DUSE_DEBUG=ON .. fi else + mkdir $BUILD_DIRECTORY/build + cd $BUILD_DIRECTORY/build cmake .. fi make _lightgbm -j4 || exit -1 -cd $BUILD_DIRECTORY/python-package && python setup.py install --precompile --user || exit -1 +cd $BUILD_DIRECTORY && sh ./build-python.sh install --precompile --user || exit -1 pytest $BUILD_DIRECTORY/tests || exit -1 if [[ $TASK == "regular" ]]; then diff --git a/.ci/test_windows.ps1 b/.ci/test_windows.ps1 index 4735de82902c..3d07496d855b 100644 --- a/.ci/test_windows.ps1 +++ b/.ci/test_windows.ps1 @@ -65,15 +65,15 @@ if ($env:TASK -ne "bdist") { if ($env:TASK -eq "regular") { mkdir $env:BUILD_SOURCESDIRECTORY/build; cd $env:BUILD_SOURCESDIRECTORY/build cmake -A x64 .. ; cmake --build . --target ALL_BUILD --config Release ; Check-Output $? - cd $env:BUILD_SOURCESDIRECTORY/python-package - python setup.py install --precompile ; Check-Output $? + cd $env:BUILD_SOURCESDIRECTORY + sh $env:BUILD_SOURCESDIRECTORY/build-python.sh install --precompile ; Check-Output $? cp $env:BUILD_SOURCESDIRECTORY/Release/lib_lightgbm.dll $env:BUILD_ARTIFACTSTAGINGDIRECTORY cp $env:BUILD_SOURCESDIRECTORY/Release/lightgbm.exe $env:BUILD_ARTIFACTSTAGINGDIRECTORY } elseif ($env:TASK -eq "sdist") { - cd $env:BUILD_SOURCESDIRECTORY/python-package - python setup.py sdist --formats gztar ; Check-Output $? - sh $env:BUILD_SOURCESDIRECTORY/.ci/check_python_dists.sh $env:BUILD_SOURCESDIRECTORY/python-package/dist ; Check-Output $? + cd $env:BUILD_SOURCESDIRECTORY + sh $env:BUILD_SOURCESDIRECTORY/build-python.sh sdist ; Check-Output $? + sh $env:BUILD_SOURCESDIRECTORY/.ci/check_python_dists.sh $env:BUILD_SOURCESDIRECTORY/dist ; Check-Output $? cd dist; pip install @(Get-ChildItem *.gz) -v ; Check-Output $? } elseif ($env:TASK -eq "bdist") { @@ -87,17 +87,17 @@ elseif ($env:TASK -eq "bdist") { Get-ItemProperty -Path Registry::HKEY_LOCAL_MACHINE\SOFTWARE\Khronos\OpenCL\Vendors conda activate $env:CONDA_ENV - cd $env:BUILD_SOURCESDIRECTORY/python-package - python setup.py bdist_wheel --integrated-opencl --plat-name=win-amd64 --python-tag py3 ; Check-Output $? - sh $env:BUILD_SOURCESDIRECTORY/.ci/check_python_dists.sh $env:BUILD_SOURCESDIRECTORY/python-package/dist ; Check-Output $? + cd $env:BUILD_SOURCESDIRECTORY + sh "build-python.sh" bdist_wheel --integrated-opencl ; Check-Output $? + sh $env:BUILD_SOURCESDIRECTORY/.ci/check_python_dists.sh $env:BUILD_SOURCESDIRECTORY/dist ; Check-Output $? cd dist; pip install --user @(Get-ChildItem *.whl) ; Check-Output $? cp @(Get-ChildItem *.whl) $env:BUILD_ARTIFACTSTAGINGDIRECTORY } elseif (($env:APPVEYOR -eq "true") -and ($env:TASK -eq "python")) { - cd $env:BUILD_SOURCESDIRECTORY\python-package + cd $env:BUILD_SOURCESDIRECTORY if ($env:COMPILER -eq "MINGW") { - python setup.py install --mingw ; Check-Output $? + sh $env:BUILD_SOURCESDIRECTORY/build-python.sh install --mingw ; Check-Output $? } else { - python setup.py install ; Check-Output $? + sh $env:BUILD_SOURCESDIRECTORY/build-python.sh install ; Check-Output $? } } diff --git a/.gitignore b/.gitignore index bb65ca426bba..d4045d9a4798 100644 --- a/.gitignore +++ b/.gitignore @@ -399,6 +399,7 @@ lightgbm.model /cmake-build-debug/ # Files from local Python install +lightgbm-python/ python-package/LICENSE python-package/build_cpp/ python-package/compile/ diff --git a/build-python.sh b/build-python.sh new file mode 100755 index 000000000000..a969dc390e9e --- /dev/null +++ b/build-python.sh @@ -0,0 +1,334 @@ +#!/bin/sh + +# [description] +# +# Prepare a source distribution (sdist) or built distribution (wheel) +# of the Python package, and optionally install it. +# +# [usage] +# +# # build sdist and put it in dist/ +# sh ./build-python.sh sdist +# +# # build wheel and put it in dist/ +# sh ./build-python.sh bdist_wheel [OPTIONS] +# +# # compile lib_lightgbm and install the Python package wrapping it +# sh ./build-python.sh install [OPTIONS] +# +# # install the Python package using a pre-compiled lib_lightgbm +# # (assumes lib_lightgbm.{dll,so} is located at the root of the repo) +# sh ./build-python.sh install --precompile +# +# [options] +# +# --boost-include-dir=FILEPATH +# Directory containing Boost headers. +# --boost-librarydir=FILEPATH +# Preferred Boost library directory. +# --boost-root=FILEPATH +# Boost preferred installation prefix. +# --opencl-include-dir=FILEPATH +# OpenCL include directory. +# --opencl-library=FILEPATH +# Path to OpenCL library. +# --bit32 +# Compile 32-bit version. +# --cuda +# Compile CUDA version. +# --gpu +# Compile GPU version. +# --hdfs +# Compile HDFS version. +# --integrated-opencl +# Compile integrated OpenCL version. +# --mingw +# Compile with MinGW. +# --mpi +# Compile MPI version. +# --nomp +# Compile version without OpenMP support. +# --precompile +# Use precompiled library. +# Only used with 'install' command. +# --time-costs +# Output time costs for different internal routines. +# --user +# Install into user-specific instead of global site-packages directory. +# Only used with 'install' command. + +set -e -u + +echo "building lightgbm" + +# Default values of arguments +INSTALL="false" +BUILD_SDIST="false" +BUILD_WHEEL="false" + +PIP_INSTALL_ARGS="" +BUILD_ARGS="" +PRECOMPILE="false" + +BOOST_INCLUDE_DIR="" +BOOST_LIBRARY_DIR="" +BOOST_ROOT="" +OPENCL_INCLUDE_DIR="" +OPENCL_LIBRARY="" + +while [ $# -gt 0 ]; do + case "$1" in + ############################ + # sub-commands of setup.py # + ############################ + install) + INSTALL="true" + ;; + sdist) + BUILD_SDIST="true" + ;; + bdist_wheel) + BUILD_WHEEL="true" + ;; + ############################ + # customized library paths # + ############################ + --boost-include-dir|--boost-include-dir=*) + if [[ "$1" != *=* ]]; + then shift; + fi + BOOST_INCLUDE_DIR="${1#*=}" + BUILD_ARGS="${BUILD_ARGS} --boost-include-dir='${BOOST_INCLUDE_DIR}'" + ;; + --boost-librarydir|--boost-librarydir=*) + if [[ "$1" != *=* ]]; + then shift; + fi + BOOST_LIBRARY_DIR="${1#*=}" + BUILD_ARGS="${BUILD_ARGS} --boost-librarydir='${BOOST_LIBRARY_DIR}'" + ;; + --boost-root|--boost-root=*) + if [[ "$1" != *=* ]]; + then shift; + fi + BOOST_ROOT="${1#*=}" + BUILD_ARGS="${BUILD_ARGS} --boost-root='${BOOST_ROOT}'" + ;; + --opencl-include-dir|--opencl-include-dir=*) + if [[ "$1" != *=* ]]; + then shift; + fi + OPENCL_INCLUDE_DIR="${1#*=}" + BUILD_ARGS="${BUILD_ARGS} --opencl-include-dir='${OPENCL_INCLUDE_DIR}'" + ;; + --opencl-library|--opencl-library=*) + if [[ "$1" != *=* ]]; + then shift; + fi + OPENCL_LIBRARY="${1#*=}" + BUILD_ARGS="${BUILD_ARGS} --opencl-library='${OPENCL_LIBRARY}'" + ;; + ######### + # flags # + ######### + --bit32) + BUILD_ARGS="${BUILD_ARGS} --bit32" + ;; + --cuda) + BUILD_ARGS="${BUILD_ARGS} --cuda" + ;; + --gpu) + BUILD_ARGS="${BUILD_ARGS} --gpu" + ;; + --hdfs) + BUILD_ARGS="${BUILD_ARGS} --hdfs" + ;; + --integrated-opencl) + BUILD_ARGS="${BUILD_ARGS} --integrated-opencl" + ;; + --mingw) + BUILD_ARGS="${BUILD_ARGS} --mingw" + ;; + --mpi) + BUILD_ARGS="${BUILD_ARGS} --mpi" + ;; + --nomp) + BUILD_ARGS="${BUILD_ARGS} --nomp" + ;; + --precompile) + PRECOMPILE="true" + ;; + --time-costs) + BUILD_ARGS="${PIP_INSTALL_ARGS} --time-costs" + ;; + --user) + PIP_INSTALL_ARGS="${PIP_INSTALL_ARGS} --user" + ;; + *) + echo "invalid argument '${1}'" + exit -1 + ;; + esac + shift +done + +# create a new directory that just contains the files needed +# to build the Python package +create_isolated_source_dir() { + rm -rf \ + ./lightgbm-python \ + ./lightgbm \ + ./python-package/build \ + ./python-package/build_cpp \ + ./python-package/compile \ + ./python-package/dist \ + ./python-package/lightgbm.egg-info + + cp -R ./python-package ./lightgbm-python + + cp LICENSE ./lightgbm-python/ + cp VERSION.txt ./lightgbm-python/lightgbm/VERSION.txt + + mkdir -p ./lightgbm-python/compile + cp -R ./cmake ./lightgbm-python/compile + cp CMakeLists.txt ./lightgbm-python/compile + cp -R ./include ./lightgbm-python/compile + cp -R ./src ./lightgbm-python/compile + cp -R ./swig ./lightgbm-python/compile + cp -R ./windows ./lightgbm-python/compile + + # include only specific files from external_libs, to keep the package + # small and avoid redistributing code with licenses incompatible with + # LightGBM's license + + ###################### + # fast_double_parser # + ###################### + mkdir -p ./lightgbm-python/compile/external_libs/fast_double_parser + cp \ + external_libs/fast_double_parser/CMakeLists.txt \ + ./lightgbm-python/compile/external_libs/fast_double_parser/CMakeLists.txt + cp \ + external_libs/fast_double_parser/LICENSE* \ + ./lightgbm-python/compile/external_libs/fast_double_parser/ + + mkdir -p ./lightgbm-python/compile/external_libs/fast_double_parser/include/ + cp \ + external_libs/fast_double_parser/include/fast_double_parser.h \ + ./lightgbm-python/compile/external_libs/fast_double_parser/include/ + + ####### + # fmt # + ####### + mkdir -p ./lightgbm-python/compile/external_libs/fmt + cp \ + external_libs/fast_double_parser/CMakeLists.txt \ + ./lightgbm-python/compile/external_libs/fmt/CMakeLists.txt + cp \ + external_libs/fmt/LICENSE* \ + ./lightgbm-python/compile/external_libs/fmt/ + + mkdir -p ./lightgbm-python/compile/external_libs/fmt/include/fmt + cp \ + external_libs/fmt/include/fmt/*.h \ + ./lightgbm-python/compile/external_libs/fmt/include/fmt/ + + ######### + # Eigen # + ######### + mkdir -p ./lightgbm-python/compile/external_libs/eigen/Eigen + cp \ + external_libs/eigen/CMakeLists.txt \ + ./lightgbm-python/compile/external_libs/eigen/CMakeLists.txt + + modules="Cholesky Core Dense Eigenvalues Geometry Householder Jacobi LU QR SVD" + for eigen_module in ${modules}; do + cp \ + external_libs/eigen/Eigen/${eigen_module} \ + ./lightgbm-python/compile/external_libs/eigen/Eigen/${eigen_module} + if [ ${eigen_module} != "Dense" ]; then + mkdir -p ./lightgbm-python/compile/external_libs/eigen/Eigen/src/${eigen_module}/ + cp \ + -R \ + external_libs/eigen/Eigen/src/${eigen_module}/* \ + ./lightgbm-python/compile/external_libs/eigen/Eigen/src/${eigen_module}/ + fi + done + + mkdir -p ./lightgbm-python/compile/external_libs/eigen/Eigen/misc + cp \ + -R \ + external_libs/eigen/Eigen/src/misc \ + ./lightgbm-python/compile/external_libs/eigen/Eigen/src/misc/ + + mkdir -p ./lightgbm-python/compile/external_libs/eigen/Eigen/plugins + cp \ + -R \ + external_libs/eigen/Eigen/src/plugins \ + ./lightgbm-python/compile/external_libs/eigen/Eigen/src/plugins/ + + ################### + # compute (Boost) # + ################### + mkdir -p ./lightgbm-python/compile/external_libs/compute + cp \ + external_libs/compute/CMakeLists.txt \ + ./lightgbm-python/compile/external_libs/compute/ + cp \ + -R \ + external_libs/compute/cmake \ + ./lightgbm-python/compile/external_libs/compute/cmake/ + cp \ + -R \ + external_libs/compute/include \ + ./lightgbm-python/compile/external_libs/compute/include/ + cp \ + -R \ + external_libs/compute/meta \ + ./lightgbm-python/compile/external_libs/compute/meta/ +} + +create_isolated_source_dir + +cd ./lightgbm-python + +# installation involves building the wheel + `pip install`-ing it +if test "${INSTALL}" = true; then + if test "${PRECOMPILE}" = true; then + echo "--- installing lightgbm (from precompiled lib_lightgbm) ---" + python setup.py install ${PIP_INSTALL_ARGS} --precompile + exit 0 + else + BUILD_SDIST="false" + BUILD_WHEEL="true" + fi +fi + +if test "${BUILD_SDIST}" = true; then + echo "--- building sdist ---" + rm -f ../dist/*.tar.gz + python ./setup.py sdist \ + --dist-dir ../dist +fi + +if test "${BUILD_WHEEL}" = true; then + echo "--- building wheel ---"# + rm -f ../dist/*.whl || true + python setup.py bdist_wheel \ + --dist-dir ../dist \ + ${BUILD_ARGS} +fi + +if test "${INSTALL}" = true; then + echo "--- installing lightgbm ---" + # ref for use of '--find-links': https://stackoverflow.com/a/52481267/3986677 + cd ../dist + pip install \ + ${PIP_INSTALL_ARGS} \ + --find-links=. \ + lightgbm + cd ../ +fi + +echo "cleaning up" +rm -rf ./lightgbm-python diff --git a/docker/dockerfile-python b/docker/dockerfile-python index 6c5ca6501ac3..541884811a0b 100644 --- a/docker/dockerfile-python +++ b/docker/dockerfile-python @@ -26,7 +26,7 @@ RUN apt-get update && \ # lightgbm conda install -q -y numpy scipy scikit-learn pandas && \ git clone --recursive --branch stable --depth 1 https://github.com/Microsoft/LightGBM && \ - cd LightGBM/python-package && python setup.py install && \ + sh ./build-python.sh install && \ # clean apt-get autoremove -y && apt-get clean && \ conda clean -a -y && \ diff --git a/docker/gpu/dockerfile.gpu b/docker/gpu/dockerfile.gpu index bac9d97b2c2b..74c301234020 100644 --- a/docker/gpu/dockerfile.gpu +++ b/docker/gpu/dockerfile.gpu @@ -88,7 +88,7 @@ RUN cd /usr/local/src && mkdir lightgbm && cd lightgbm && \ ENV PATH /usr/local/src/lightgbm/LightGBM:${PATH} -RUN /bin/bash -c "source activate py3 && cd /usr/local/src/lightgbm/LightGBM/python-package && python setup.py install --precompile && source deactivate" +RUN /bin/bash -c "source activate py3 && cd /usr/local/src/lightgbm/LightGBM && sh ./build-python.sh install --precompile && source deactivate" ################################################################################################################# # System CleanUp diff --git a/docs/FAQ.rst b/docs/FAQ.rst index 9f86b882e0a1..6ce86c257f65 100644 --- a/docs/FAQ.rst +++ b/docs/FAQ.rst @@ -277,6 +277,10 @@ Python-package 1. ``Error: setup script specifies an absolute path`` when installing from GitHub using ``python setup.py install``. -------------------------------------------------------------------------------------------------------------------- +.. note:: + As of v4.0.0, ``lightgbm`` does not support directly invoking ``setup.py``. + This answer refers only to versions of ``lightgbm`` prior to v4.0.0. + .. code-block:: console error: Error: setup script specifies an absolute path: diff --git a/docs/GPU-Tutorial.rst b/docs/GPU-Tutorial.rst index 1ca98784e3f6..836ab1add378 100644 --- a/docs/GPU-Tutorial.rst +++ b/docs/GPU-Tutorial.rst @@ -80,9 +80,7 @@ If you want to use the Python interface of LightGBM, you can install it now (alo sudo apt-get -y install python-pip sudo -H pip install setuptools numpy scipy scikit-learn -U - cd python-package/ - sudo python setup.py install --precompile - cd .. + sudo sh ./build-python.sh install --precompile You need to set an additional parameter ``"device" : "gpu"`` (along with your other options like ``learning_rate``, ``num_leaves``, etc) to use GPU in Python. diff --git a/python-package/README.rst b/python-package/README.rst index 9e8aaa13c8a3..8ca3cd31e50f 100644 --- a/python-package/README.rst +++ b/python-package/README.rst @@ -193,34 +193,33 @@ For **Windows** users, if you get any errors during installation and there is th .. code:: sh git clone --recursive https://github.com/microsoft/LightGBM.git - cd LightGBM/python-package # export CXX=g++-7 CC=gcc-7 # macOS users, if you decided to compile with gcc, don't forget to specify compilers (replace "7" with version of gcc installed on your machine) - python setup.py install + sh ./build-python.sh install Note: ``sudo`` (or administrator rights in **Windows**) may be needed to perform the command. -Run ``python setup.py install --nomp`` to disable **OpenMP** support. All requirements from `Build Threadless Version section <#build-threadless-version>`__ apply for this installation option as well. +Run ``sh ./build-python.sh install --nomp`` to disable **OpenMP** support. All requirements from `Build Threadless Version section <#build-threadless-version>`__ apply for this installation option as well. -Run ``python setup.py install --mpi`` to enable **MPI** support. All requirements from `Build MPI Version section <#build-mpi-version>`__ apply for this installation option as well. +Run ``sh ./build-python.sh install --mpi`` to enable **MPI** support. All requirements from `Build MPI Version section <#build-mpi-version>`__ apply for this installation option as well. -Run ``python setup.py install --mingw``, if you want to use **MinGW-w64** on **Windows** instead of **Visual Studio**. All requirements from `Build with MinGW-w64 on Windows section <#build-with-mingw-w64-on-windows>`__ apply for this installation option as well. +Run ``sh ./build-python.sh install --mingw``, if you want to use **MinGW-w64** on **Windows** instead of **Visual Studio**. All requirements from `Build with MinGW-w64 on Windows section <#build-with-mingw-w64-on-windows>`__ apply for this installation option as well. -Run ``python setup.py install --gpu`` to enable GPU support. All requirements from `Build GPU Version section <#build-gpu-version>`__ apply for this installation option as well. To pass additional options to **CMake** use the following syntax: ``python setup.py install --gpu --opencl-include-dir=/usr/local/cuda/include/``, see `Build GPU Version section <#build-gpu-version>`__ for the complete list of them. +Run ``sh ./build-python.sh install --gpu`` to enable GPU support. All requirements from `Build GPU Version section <#build-gpu-version>`__ apply for this installation option as well. To pass additional options to **CMake** use the following syntax: ``sh ./build-python.sh install --gpu --opencl-include-dir="/usr/local/cuda/include/"``, see `Build GPU Version section <#build-gpu-version>`__ for the complete list of them. -Run ``python setup.py install --cuda`` to enable CUDA support. All requirements from `Build CUDA Version section <#build-cuda-version>`__ apply for this installation option as well. +Run ``sh ./build-python.sh install --cuda`` to enable CUDA support. All requirements from `Build CUDA Version section <#build-cuda-version>`__ apply for this installation option as well. -Run ``python setup.py install --hdfs`` to enable HDFS support. All requirements from `Build HDFS Version section <#build-hdfs-version>`__ apply for this installation option as well. +Run ``sh ./build-python.sh install --hdfs`` to enable HDFS support. All requirements from `Build HDFS Version section <#build-hdfs-version>`__ apply for this installation option as well. -Run ``python setup.py install --bit32``, if you want to use 32-bit version. All requirements from `Build 32-bit Version with 32-bit Python section <#build-32-bit-version-with-32-bit-python>`__ apply for this installation option as well. +Run ``sh ./build-python.sh install --bit32``, if you want to use 32-bit version. All requirements from `Build 32-bit Version with 32-bit Python section <#build-32-bit-version-with-32-bit-python>`__ apply for this installation option as well. -Run ``python setup.py install --time-costs``, if you want to output time costs for different internal routines. All requirements from `Build with Time Costs Output section <#build-with-time-costs-output>`__ apply for this installation option as well. +Run ``sh ./build-python.sh install --time-costs``, if you want to output time costs for different internal routines. All requirements from `Build with Time Costs Output section <#build-with-time-costs-output>`__ apply for this installation option as well. -If you get any errors during installation or due to any other reasons, you may want to build dynamic library from sources by any method you prefer (see `Installation Guide <https://github.com/microsoft/LightGBM/blob/master/docs/Installation-Guide.rst>`__) and then just run ``python setup.py install --precompile``. +If you get any errors during installation or due to any other reasons, you may want to build dynamic library from sources by any method you prefer (see `Installation Guide <https://github.com/microsoft/LightGBM/blob/master/docs/Installation-Guide.rst>`__) and then just run ``sh ./build-python.sh install --precompile``. Build Wheel File **************** -You can use ``python setup.py bdist_wheel`` instead of ``python setup.py install`` to build wheel file and use it for installation later. This might be useful for systems with restricted or completely without network access. +You can use ``sh ./build-python.sh install bdist_wheel`` instead of ``sh ./build-python.sh install`` to build wheel file and use it for installation later. This might be useful for systems with restricted or completely without network access. Install Dask-package '''''''''''''''''''' @@ -235,7 +234,7 @@ To install all additional dependencies required for Dask-package, you can append pip install lightgbm[dask] -Or replace ``python setup.py install`` with ``pip install -e .[dask]`` if you are installing the package from source files. +Or replace ``sh ./build-python.sh install`` with ``pip install -e .[dask]`` if you are installing the package from source files. Troubleshooting --------------- diff --git a/python-package/setup.py b/python-package/setup.py index b1620929f816..1fde06d727bc 100644 --- a/python-package/setup.py +++ b/python-package/setup.py @@ -7,8 +7,8 @@ from os import chdir from pathlib import Path from platform import system -from shutil import copyfile, copytree, rmtree -from typing import List, Optional, Union +from shutil import rmtree +from typing import List, Optional from setuptools import find_packages, setup from setuptools.command.install import install @@ -46,41 +46,6 @@ def find_lib() -> List[str]: return LIB_PATH -def copy_files(integrated_opencl: bool = False, use_gpu: bool = False) -> None: - - def copy_files_helper(folder_name: Union[str, Path]) -> None: - src = CURRENT_DIR.parent / folder_name - if src.is_dir(): - dst = CURRENT_DIR / 'compile' / folder_name - if dst.is_dir(): - rmtree(dst) - copytree(src, dst) - else: - raise Exception(f'Cannot copy {src} folder') - - if not IS_SOURCE_FLAG_PATH.is_file(): - copy_files_helper('include') - copy_files_helper('src') - for submodule in (CURRENT_DIR.parent / 'external_libs').iterdir(): - submodule_stem = submodule.stem - if submodule_stem == 'compute' and not use_gpu: - continue - copy_files_helper(Path('external_libs') / submodule_stem) - (CURRENT_DIR / "compile" / "windows").mkdir(parents=True, exist_ok=True) - copyfile(CURRENT_DIR.parent / "windows" / "LightGBM.sln", - CURRENT_DIR / "compile" / "windows" / "LightGBM.sln") - copyfile(CURRENT_DIR.parent / "windows" / "LightGBM.vcxproj", - CURRENT_DIR / "compile" / "windows" / "LightGBM.vcxproj") - copyfile(CURRENT_DIR.parent / "LICENSE", - CURRENT_DIR / "LICENSE") - copyfile(CURRENT_DIR.parent / "CMakeLists.txt", - CURRENT_DIR / "compile" / "CMakeLists.txt") - if integrated_opencl: - (CURRENT_DIR / "compile" / "cmake").mkdir(parents=True, exist_ok=True) - copyfile(CURRENT_DIR.parent / "cmake" / "IntegratedOpenCL.cmake", - CURRENT_DIR / "compile" / "cmake" / "IntegratedOpenCL.cmake") - - def clear_path(path: Path) -> None: if path.is_dir(): for file_name in path.iterdir(): @@ -160,7 +125,8 @@ def compile_cpp( if use_mpi: raise Exception('MPI version cannot be compiled by MinGW due to the miss of MPI library in it') logger.info("Starting to compile with CMake and MinGW.") - silent_call(cmake_cmd + ["-G", "MinGW Makefiles"], raise_error=True, + # ref: https://stackoverflow.com/a/45104058/3986677 + silent_call(cmake_cmd + ["-G", "MinGW Makefiles", "-DCMAKE_SH=CMAKE_SH-NOTFOUND"], raise_error=True, error_msg='Please install CMake and all required dependencies first') silent_call(["mingw32-make.exe", "_lightgbm", f"-I{build_dir}", "-j4"], raise_error=True, error_msg='Please install MinGW first') @@ -254,7 +220,6 @@ def run(self) -> None: "please use 64-bit Python instead.") LOG_PATH.touch() if not self.precompile: - copy_files(integrated_opencl=self.integrated_opencl, use_gpu=self.gpu) compile_cpp(use_mingw=self.mingw, use_gpu=self.gpu, use_cuda=self.cuda, use_mpi=self.mpi, use_hdfs=self.hdfs, boost_root=self.boost_root, boost_dir=self.boost_dir, boost_include_dir=self.boost_include_dir, boost_librarydir=self.boost_librarydir, @@ -315,7 +280,6 @@ def finalize_options(self) -> None: class CustomSdist(sdist): def run(self) -> None: - copy_files(integrated_opencl=True, use_gpu=True) IS_SOURCE_FLAG_PATH.touch() rmtree(CURRENT_DIR / 'lightgbm' / 'Release', ignore_errors=True) rmtree(CURRENT_DIR / 'lightgbm' / 'windows' / 'x64', ignore_errors=True) @@ -332,11 +296,8 @@ def run(self) -> None: LOG_PATH = Path.home() / 'LightGBM_compilation.log' LOG_NOTICE = f"The full version of error log was saved into {LOG_PATH}" IS_SOURCE_FLAG_PATH = CURRENT_DIR / '_IS_SOURCE_PACKAGE.txt' - _version_src = CURRENT_DIR.parent / 'VERSION.txt' - _version_dst = CURRENT_DIR / 'lightgbm' / 'VERSION.txt' - if _version_src.is_file(): - copyfile(_version_src, _version_dst) - version = _version_dst.read_text(encoding='utf-8').strip() + _version_file = CURRENT_DIR / 'lightgbm' / 'VERSION.txt' + version = _version_file.read_text(encoding='utf-8').strip() readme = (CURRENT_DIR / 'README.rst').read_text(encoding='utf-8') sys.path.insert(0, str(CURRENT_DIR))