Merge pull request apache#14 from dato-code/build_script

Improve build script, add packaging step
LoneStarAI · Feb 25, 2016 · 00cfdec · 00cfdec
2 parents 635585b + 1377d18
commit 00cfdec
Show file tree

Hide file tree

Showing 4 changed files with 133 additions and 19 deletions.
diff --git a/Makefile b/Makefile
@@ -181,19 +181,13 @@ $(EXTRA_OPERATORS)/build/%_gpu.o: $(EXTRA_OPERATORS)/%.cu
 	$(NVCC) $(NVCCFLAGS) -Xcompiler "$(CFLAGS) -Isrc/operator" -M -MT $(EXTRA_OPERATORS)/build/$*_gpu.o $< >$(EXTRA_OPERATORS)/build/$*_gpu.d
 	$(NVCC) -c -o $@ $(NVCCFLAGS) -Xcompiler "$(CFLAGS) -Isrc/operator" $<
 
-copy_cuda_deps:
-ifdef CUDA_DEP
-	@mkdir -p lib
-	cp $(CUDA_DEP) lib
-endif
-
 # NOTE: to statically link libmxnet.a we need the option
 # --Wl,--whole-archive -lmxnet --Wl,--no-whole-archive
 lib/libmxnet.$(STATIC_LIB_EXT): $(ALL_DEP)
 	@mkdir -p $(@D)
 	ar crv $@ $(filter %.o, $?)
 
-lib/libmxnet.$(SHARED_LIB_EXT): $(ALL_DEP) copy_cuda_deps
+lib/libmxnet.$(SHARED_LIB_EXT): $(ALL_DEP)
 	@mkdir -p $(@D)
 	$(CXX) $(CFLAGS) -shared -o $@ $(filter %.o %.a, $^) $(LDFLAGS)
 

diff --git a/python/mxnet/base.py b/python/mxnet/base.py
@@ -7,6 +7,8 @@
 import ctypes
 import numpy as np
 import atexit
+import logging
+import os
 from . import libinfo
 
 __all__ = ['MXNetError']
@@ -31,8 +33,19 @@ class MXNetError(Exception):
 
 def _load_lib():
     """Load libary by searching possible path."""
-    lib_path = libinfo.find_lib_path()
-    lib = ctypes.cdll.LoadLibrary(lib_path[0])
+    lib_path = libinfo.find_lib_path()[0]
+    cuda_lib_path = lib_path.replace('libmxnet.', 'libmxnet.cuda.')
+    if os.path.exists(cuda_lib_path):
+        try:
+            lib = ctypes.cdll.LoadLibrary(cuda_lib_path)
+            logging.info("CUDA GPU support is activated.")
+        except Exception as e:
+            logging.warn("Fail loading CUDA library. Error: %s" % e)
+            logging.info("Please try adding the CUDA installation path to LD_LIBRARY_PATH. Running CPU only mode.")
+            lib = ctypes.cdll.LoadLibrary(lib_path)
+    else:
+        logging.info("CUDA support is currently not available on this platform. Running CPU only mode.")
+        lib = ctypes.cdll.LoadLibrary(lib_path)
     # DMatrix functions
     lib.MXGetLastError.restype = ctypes.c_char_p
     return lib

diff --git a/python/mxnet/libinfo.py b/python/mxnet/libinfo.py
@@ -13,8 +13,9 @@ def find_lib_path():
         List of all found path to the libraries
     """
     curr_path = os.path.dirname(os.path.abspath(os.path.expanduser(__file__)))
+    parent_path = os.path.join(curr_path, '../')
     api_path = os.path.join(curr_path, '../../lib/')
-    dll_path = [curr_path, api_path]
+    dll_path = [parent_path, curr_path, api_path]
     if os.name == 'nt':
         vs_configuration = 'Release'
         if platform.architecture()[0] == '64bit':

diff --git a/scripts/build.sh b/scripts/build.sh
@@ -1,29 +1,135 @@
 #!/bin/bash
 
+##########
+# Environment variables:
+# CUDA_PATH: set to build with cuda at particular location
+# BUILD_NUMBER: the build number of the final artifact
+##########
+
+if [[ -z "$BUILD_NUMBER" ]]; then
+  BUILD_NUMBER=0.1
+fi
+
 SCRIPT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
 WORKSPACE=${SCRIPT_DIR}/..
 
 cd ${WORKSPACE}
 source ${WORKSPACE}/scripts/python_env.sh
 
-if [ -d "/usr/local/cuda-7.5" ]; then
-	./configure --cleanup_if_invalid --yes --cuda_path=/usr/local/cuda-7.5
-	HAS_GPU=1
-else
-	./configure --cleanup_if_invalid --yes
-fi
+## Build 
+function build {
+echo "============= Build =============="
+./configure --cleanup_if_invalid --yes
+make clean_all
+make -j4
+echo "==================================="
+}
 
+## Build with cuda
+function build_with_cuda {
+echo "============= Build with CUDA =============="
+echo "CUDA path: ${CUDA_PATH}"
+./configure --cleanup_if_invalid --yes --cuda_path=${CUDA_PATH}
 make clean_all
 make -j4
+echo "==================================="
+}
 
+## Test
+function unittest {
+echo "============= UnitTest =============="
 if [[ $OSTYPE != msys ]]; then
   nosecmd="${PYTHON_EXECUTABLE} ${NOSETEST_EXECUTABLE}"
 else
   nosecmd="${NOSETEST_EXECUTABLE}"
 fi
+${nosecmd} -v --with-id ${WORKSPACE}/tests/python/unittest ${WORKSPACE}/tests/python/train --with-xunit --xunit-file=alltests.nosetests.xml
+echo "==================================="
+}
 
-if [ ! -z "$HAS_GPU" ]; then
-  ${nosecmd} -v --with-id ${WORKSPACE}/tests/python/unittest ${WORKSPACE}/tests/python/train ${WORKSPACE}/tests/python/gpu --with-xunit --xunit-file=alltests.nosetests.xml
+## Test with cuda
+function unittest_with_cuda {
+echo "============= UnitTest =============="
+if [[ $OSTYPE != msys ]]; then
+  nosecmd="${PYTHON_EXECUTABLE} ${NOSETEST_EXECUTABLE}"
 else
-  ${nosecmd} -v --with-id ${WORKSPACE}/tests/python/unittest ${WORKSPACE}/tests/python/train --with-xunit --xunit-file=alltests.nosetests.xml
+  nosecmd="${NOSETEST_EXECUTABLE}"
+fi
+${nosecmd} -v --with-id ${WORKSPACE}/tests/python/unittest ${WORKSPACE}/tests/python/gpu --with-xunit --xunit-file=alltests.nosetests.xml
+echo "==================================="
+}
+
+## Copy artifacts
+function copy_artifact {
+echo "============= Copy artifacts =============="
+TARGET_DIR=${WORKSPACE}/target/build
+if [ ! -d "$TARGET_DIR" ]; then
+  mkdir -p ${TARGET_DIR}
+  mkdir -p ${TARGET_DIR}/python
+fi
+
+if [[ $OSTYPE == linux* ]]; then
+  dll_ext='so'
+elif [[ $OSTYPE == darwin* ]]; then
+  dll_ext='so'
+elif [[ $OSTYPE == msys ]]; then
+  dll_ext='dll'
 fi
+
+if [[ -z "${LIB_NAME}" ]]; then
+  LIB_NAME='libmxnet'
+fi
+
+set -x
+cp -r python/mxnet ${TARGET_DIR}/python/
+cp -r lib/libmxnet.${dll_ext} ${TARGET_DIR}/python/${LIB_NAME}.${dll_ext}
+set +x
+echo "====================================="
+}
+
+# Package
+function package {
+echo "============= Package =============="
+echo "Build number: $BUILD_NUMBER"
+
+if [[ $OSTYPE == linux* ]]; then
+  PLATFORM='linux'
+elif [[ $OSTYPE == darwin* ]]; then
+  PLATFORM='mac'
+elif [[ $OSTYPE == msys ]]; then
+  PLATFORM='windows'
+fi
+
+TARGET_DIR=${WORKSPACE}/target
+archive_file_ext="tar.gz"
+cd ${TARGET_DIR}/build/python
+FNAME=${TARGET_DIR}/mxnet_${PLATFORM}_${BUILD_NUMBER}.${archive_file_ext}
+tar -czvf ${FNAME} mxnet/*.py libmxnet*
+echo "====================================="
+}
+
+function clean_target_dir {
+TARGET_DIR=${WORKSPACE}/target
+rm -rf ${TARGET_DIR}/build
+rm -rf ${TARGET_DIR}/*.tar.gz
+}
+
+# Cleanup previous build ##
+clean_target_dir
+
+## Standard build ##
+build
+unittest
+LIB_NAME='libmxnet'
+copy_artifact
+
+## CUDA build ##
+if [[ ! -z "$CUDA_PATH" ]]; then
+  build_with_cuda
+  unittest_with_cuda
+  LIB_NAME='libmxnet.cuda'
+  copy_artifact
+fi
+
+## Package everything into tarball
+package