update to v1.2

xdl/CMakeLists.txt

@@ -1,10 +1,12 @@
 project(xdl)
-cmake_minimum_required(VERSION 2.8)
+cmake_minimum_required(VERSION 3.0.0 FATAL_ERROR)
 
+SET(CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake")
 include(${PROJECT_SOURCE_DIR}/cmake/Utils.cmake)
 
 xdl_option(test      "Build all tests."          ON)
 xdl_option(USE_GPU "use gpu" off)
+xdl_option(USE_PS_PLUS "use ps-plus" ON)
 xdl_option(coverage  "Generate coverage analysis"  off)
 
 enable_testing()
@@ -46,12 +48,21 @@ include_directories(
 
 include(cmake/Dependencies.cmake)
 
+set(TBB_ROOT "${PROJECT_SOURCE_DIR}/third_party/tbb/")
+include(${TBB_ROOT}/cmake/TBBBuild.cmake)
+set(CMAKE_CXX_FLAGS "-std=c++1y ${CMAKE_CXX_FLAGS}")
+tbb_build(TBB_ROOT ${TBB_ROOT} CONFIG_DIR TBB_DIR MAKE_ARGS)
+find_package(TBB REQUIRED)
+include_directories("${TBB_ROOT}/include")
+
 IF (USE_GPU)
+   set(CUDA_PATH "/usr/local/cuda-9.0/")
    set(CUDA_TOOLKIT_ROOT_DIR ${CUDA_PATH})
    find_package(CUDA REQUIRED)
    message("-- CUDA_PATH = ${CUDA_PATH} ")
    include(${PROJECT_SOURCE_DIR}/cmake/Cuda.cmake)
    include_directories(
+     ${CUDA_PATH}/targets/x86_64-linux/include/
      ${CUDA_PATH}/include/
      )
    link_directories(
@@ -62,7 +73,7 @@ ENDIF ()
 
 # PS_PLUS
 set(SEASTAR_LIBRARYS -Wl,--whole-archive seastar_service ps_network_static seastar -Wl,--no-whole-archive -L/usr/local/lib64/boost -lboost_timer -lboost_chrono  -laio -lboost_program_options -lboost_system -lboost_filesystem -lm -lboost_thread -lcryptopp -lrt -lgnutls -lgnutlsxx -llz4 -ldl -lgcc_s -lunwind -lhwloc -lnuma -lpciaccess -lxml2 -lz -lcares-seastar libstdc++.a)
-set(PS_LIBRARYS -Wl,--whole-archive libzookeeper.a libhashtable.a ps_common ps_client ps_server ps_model_server ps_plugin_hdfs libevent_core.a glog -Wl,--no-whole-archive ${SEASTAR_LIBRARYS})
+set(PS_LIBRARYS -Wl,--whole-archive libzookeeper.a libhashtable.a ps_common ps_client ps_server ps_scheduler ps_model_server ps_plugin_hdfs libevent_core.a -Wl,--no-whole-archive ${SEASTAR_LIBRARYS})
 include_directories(${PROJECT_SOURCE_DIR}/ps-plus/)
 include_directories(${PROJECT_SOURCE_DIR}/third_party/zookeeper-client/include)
 include_directories(${PROJECT_SOURCE_DIR}/third_party/zookeeper-client/generated)
@@ -96,11 +107,12 @@ IF (MXNET_BACKEND)
 ENDIF()
 
 IF (USE_GPU)
-   set(XDL_CORE_DEPEND_LIB libprotobuf ${PS_LIBRARYS} ${BACKEND_LIB} python2.7 cudart dl)
+   set(XDL_CORE_DEPEND_LIB libprotobuf ${PS_LIBRARYS} ${BACKEND_LIB} python2.7 cudart dl ${TBB_IMPORTED_TARGETS})
 ELSE ()
-   set(XDL_CORE_DEPEND_LIB libprotobuf ${PS_LIBRARYS} ${BACKEND_LIB} python2.7 dl)
+   set(XDL_CORE_DEPEND_LIB libprotobuf ${PS_LIBRARYS} ${BACKEND_LIB} python2.7 dl ${TBB_IMPORTED_TARGETS})
 ENDIF ()
 
+set(XDL_IO_DEPEND_LIB dl rdkafka++ jsoncpp)
 set(XDL_CORE_LIB -Wl,--whole-archive xdl_core -Wl,--no-whole-archive ${XDL_CORE_DEPEND_LIB})
 set(XDL_IO_LIB -Wl,--whole-archive xdl_io -Wl,--no-whole-archive)
 

xdl/cmake/FindNumPy.cmake

@@ -0,0 +1,42 @@
+# Find the Python NumPy package
+# PYTHON_NUMPY_INCLUDE_DIR
+# PYTHON_NUMPY_FOUND
+# will be set by this script
+
+cmake_minimum_required(VERSION 2.6)
+
+if(NOT PYTHON_EXECUTABLE)
+  if(NumPy_FIND_QUIETLY)
+    find_package(PythonInterp QUIET)
+  else()
+    find_package(PythonInterp)
+    set(__numpy_out 1)
+  endif()
+endif()
+
+if (PYTHON_EXECUTABLE)
+  # Find out the include path
+  execute_process(
+    COMMAND "${PYTHON_EXECUTABLE}" -c
+            "from __future__ import print_function\ntry: import numpy; print(numpy.get_include(), end='')\nexcept:pass\n"
+            OUTPUT_VARIABLE __numpy_path)
+  # And the version
+  execute_process(
+    COMMAND "${PYTHON_EXECUTABLE}" -c
+            "from __future__ import print_function\ntry: import numpy; print(numpy.__version__, end='')\nexcept:pass\n"
+    OUTPUT_VARIABLE __numpy_version)
+elseif(__numpy_out)
+  message(STATUS "Python executable not found.")
+endif(PYTHON_EXECUTABLE)
+
+find_path(PYTHON_NUMPY_INCLUDE_DIR numpy/arrayobject.h
+  HINTS "${__numpy_path}" "${PYTHON_INCLUDE_PATH}" NO_DEFAULT_PATH)
+
+if(PYTHON_NUMPY_INCLUDE_DIR)
+  set(PYTHON_NUMPY_FOUND 1 CACHE INTERNAL "Python numpy found")
+endif(PYTHON_NUMPY_INCLUDE_DIR)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(NumPy REQUIRED_VARS PYTHON_NUMPY_INCLUDE_DIR
+                                        VERSION_VAR __numpy_version)
+

xdl/cmake/Utils.cmake

@@ -414,6 +414,26 @@ function(xdl_add_test dir extension)
   endforeach()
 endfunction()
 
+function(xdl_add_test_exclude dir extension exclude)
+  file(GLOB TEST_SOURCE "${dir}/*${extension}")
+  foreach(file ${TEST_SOURCE})
+    get_filename_component(file_name ${file} NAME)
+    get_filename_component(file_exe ${file} NAME_WE)
+    if ("${file_name}" STREQUAL "${exclude}" OR "${file_exe}" STREQUAL "${exclude}")
+            message(STATUS "Skip " ${exclude})
+        continue()
+    endif()
+    #message(${extension})
+    #message(${file})
+    #message(${file_exe})
+    #message(${exclude})
+    add_executable(${file_exe} ${file})
+    target_link_libraries(${file_exe} ${ARGN} gcov)
+    set_target_properties(${file_exe} PROPERTIES COMPILE_FLAGS "-g -O0 --coverage")
+    add_test(${file_exe} ${file_exe} COMMAND ${file_exe})
+  endforeach()
+endfunction()
+
 function(xdl_add_cuda_test dir extension)
   file(GLOB TEST_SOURCE "${dir}/*${extension}")
   foreach(file ${TEST_SOURCE})

xdl/distributed/install_xdl_submit.sh

@@ -22,4 +22,6 @@ fi
 
 cd $(dirname ${BASH_SOURCE[0]})
 cp ./xdl_submit/xdl_submit.py /usr/bin/xdl_submit.py
+chmod 777 /usr/bin/xdl_submit.py
 cp ./xdl_yarn_scheduler/bin/xdl-yarn-scheduler-1.0.0-SNAPSHOT-jar-with-dependencies.jar /usr/bin/xdl-yarn-scheduler-1.0.0-SNAPSHOT-jar-with-dependencies.jar
+chmod 777 /usr/bin/xdl-yarn-scheduler-1.0.0-SNAPSHOT-jar-with-dependencies.jar

xdl/distributed/xdl_yarn_scheduler/pom.xml

@@ -76,7 +76,7 @@
 		<dependency>
 			<groupId>com.alibaba</groupId>
 			<artifactId>fastjson</artifactId>
-			<version>1.2.28</version>
+			<version>1.2.58</version>
 		</dependency>
 	</dependencies>
 

xdl/examples/mnist/mnist.py

@@ -80,7 +80,7 @@ def model(images, labels):
 
 @xdl.tf_wrapper(is_training=False)
 def eval_model(images, labels):
-    with tf.variable_scope("train", reuse=True):
+    with tf.variable_scope("train", reuse=tf.AUTO_REUSE):
         eval_y = fc(images, [784, 10], [10])
         labels_test = tf.cast(labels, tf.int64)
         correct_prediction = tf.equal(tf.argmax(eval_y, 1), labels_test)

xdl/ps-plus/CMakeLists.txt

@@ -1,28 +1,90 @@
-cmake_minimum_required(VERSION 2.8)
+cmake_minimum_required(VERSION 3.0.0 FATAL_ERROR)
 project(ps-plus)
 
 if (DEBUG)
    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -fPIC -std=c++1y -D_GLIBCXX_USE_CXX11_ABI=0")
 else ()
-     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -fPIC -std=c++1y -g -DNDEBUG -D_GLIBCXX_USE_CXX11_ABI=0")
+   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -fPIC -std=c++1y -g -DNDEBUG -D_GLIBCXX_USE_CXX11_ABI=0")
 endif ()
 
 if (APPLE)
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
 elseif (UNIX)
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pthread")
-endif()
+endif ()
+
+include_directories(.)
+
+set(CMAKE_SKIP_BUILD_RPATH FALSE)
+set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
+set(CMAKE_INSTALL_RPATH "/usr/local/lib:/usr/local/lib64/boost:/usr/local/gcc-5.3.0/lib64")
+set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
+
+# build third_party libraries using cmake
+function (third_party_library_builder_cmake arg)
+list(LENGTH ARGV  argv_len)
+    set(i 0)
+    while( i LESS ${argv_len})
+        list(GET ARGV ${i} argv_value)
+        message(STATUS "start build third_party library:${argv_value}")
+        execute_process(COMMAND bash -c "cd ${PROJECT_SOURCE_DIR}/third_party/${argv_value}; mkdir -p build; cd build; cmake .. -DCMAKE_INSTALL_PREFIX=. -DCMAKE_CXX_FLAGS=-D_GLIBCXX_USE_CXX11_ABI=0; make; make install; cd ${PROJECT_SOURCE_DIR}")
+        IF(EXISTS "${PROJECT_SOURCE_DIR}/third_party/${argv_value}/build/include")
+          include_directories("${PROJECT_SOURCE_DIR}/third_party/${argv_value}/build/include")
+        ENDIF()
+        IF(EXISTS "${PROJECT_SOURCE_DIR}/third_party/${argv_value}/build/lib")
+            link_directories("${PROJECT_SOURCE_DIR}/third_party/${argv_value}/build/lib")
+        ENDIF()
+        IF(EXISTS "${PROJECT_SOURCE_DIR}/third_party/${argv_value}/build/lib64")
+            link_directories("${PROJECT_SOURCE_DIR}/third_party/${argv_value}/build/lib64")
+        ENDIF()
+        math(EXPR i "${i} + 1")
+    endwhile()
+endfunction ()
+
+# build third_party libraries using autotools
+function (third_party_library_builder_autotools arg)
+    list(LENGTH ARGV argv_len)
+    set(i 0)
+    while (i LESS ${argv_len})
+        list(GET ARGV ${i} argv_value)
+	message(STATUS "Start to build third_party library: ${argv_value}")
+	execute_process(COMMAND bash -c "
+	    cd ${PROJECT_SOURCE_DIR}/third_party/${argv_value};
+            mkdir -p build;
+            ./configure --prefix=$(pwd)/build CXXFLAGS='-D_GLIBCXX_USE_CXX11_ABI=0' LDFLAGS='-D_GLIBCXX_USE_CXX11_ABI=0';
+            make;
+            make install;
+            cd ${PROJECT_SOURCE_DIR}")
+	if (EXISTS "${PROJECT_SOURCE_DIR}/third_party/${argv_value}/build/include")
+    include_directories("${PROJECT_SOURCE_DIR}/third_party/${argv_value}/build/include")
+	endif ()
+	if (EXISTS "${PROJECT_SOURCE_DIR}/third_party/${argv_value}/build/lib")
+            link_directories("${PROJECT_SOURCE_DIR}/third_party/${argv_value}/build/lib")
+	endif ()
+	if (EXISTS "${PROJECT_SOURCE_DIR}/third_party/${argv_value}/build/lib64")
+            link_directories("${PROJECT_SOURCE_DIR}/third_party/${argv_value}/build/lib64")
+	endif ()
+	math(EXPR i "${i} + 1")
+    endwhile ()
+endfunction ()
 
 set(CMAKE_SHARED_LIBRARY_SUFFIX ".so")
+function (compile_protobuf dir)
+  file(GLOB pbs ${PROJECT_SOURCE_DIR}/${dir}/*.proto)
+  execute_process(COMMAND ${PROTOBUF_PROTOC_EXECUTABLE} ${pbs} --cpp_out=${PROJECT_SOURCE_DIR} --proto_path=${PROJECT_SOURCE_DIR})
+endfunction()
 
-include_directories(.)
+find_package(NumPy REQUIRED)
+find_package(PythonLibs REQUIRED)
 
 link_directories(${PROJECT_BINARY_DIR}/third_party/googletest/)
 link_directories(${PROJECT_BINARY_DIR}/third_party/zookeeper-client/)
-link_directories(${PROJECT_BINARY_DIR}/third_party/glog/)
+include_directories("${PYTHON_NUMPY_INCLUDE_DIR}")
+include_directories("${PYTHON_INCLUDE_DIRS}")
 
-include_directories(${PROJECT_SOURCE_DIR}/third_party/glog/)
 include_directories(${PROJECT_SOURCE_DIR}/third_party/hdfs/)
+include_directories(${PROJECT_SOURCE_DIR}/third_party/seastar/ ${PROJECT_SOURCE_DIR}/third_party/seastar/fmt ${PROJECT_SOURCE_DIR}/third_party/seastar/c-ares)
+link_directories("${PROJECT_SOURCE_DIR}/third_party/seastar/lib")
 
 aux_source_directory(ps-plus/service/seastar/lib SEASTAR_LIB)
 add_library(seastar_service STATIC ${SEASTAR_LIB})
@@ -35,7 +97,7 @@ if (USE_HDFS OR NOT DEFINED USE_HDFS)
   SET(PLUGINS ${PLUGINS} ps_plugin_hdfs)
 endif()
 
-set(LIBRARYS  -Wl,--whole-archive ps_common ps_server ps_model_server ps_client ps_scheduler libhashtable.a libzookeeper.a libevent_core.a ${PLUGINS} glog -Wl,--no-whole-archive ${PLUGINS_DEPENDENCY} ${SEASTAR_LIBRARYS})
+set(LIBRARYS  -Wl,--whole-archive ps_common ps_server ps_model_server ps_client ps_scheduler libhashtable.a libzookeeper.a libevent_core.a ${PLUGINS} -Wl,--no-whole-archive ${PLUGINS_DEPENDENCY} ${SEASTAR_LIBRARYS} ${PYTHON_LIBRARIES})
 
 aux_source_directory(ps-plus/common COMMON)
 aux_source_directory(ps-plus/common/initializer COMMON_INITIALIZER)
@@ -52,7 +114,6 @@ aux_source_directory(ps-plus/scheduler/test SCHEDULER_TEST)
 aux_source_directory(ps-plus/main MAIN)
 aux_source_directory(ps-plus/model_server MODEL_SERVER)
 aux_source_directory(ps-plus/model_server/test MODEL_SERVER_TEST)
-aux_source_directory(ps-plus/tool CLIENT_TOOL)
 aux_source_directory(ps-plus/profiler PROFILER)
 aux_source_directory(ps-plus/common/test COMMON_TEST)
 aux_source_directory(ps-plus/common/initializer/test COMMON_INITIALIZER_TEST)
@@ -72,7 +133,7 @@ add_library(ps_client STATIC ${CLIENT} ${CLIENT_PARTITIONER})
 add_library(ps_plugin_hdfs STATIC ${PLUGINS_HDFS})
 
 add_executable(ps ${MAIN})
-add_executable(tool ${CLIENT_TOOL})
+add_executable(tool ps-plus/tool/client_tool.cpp)
 
 # tests
 add_executable(ps_common_test ${COMMON_TEST} ${COMMON_INITIALIZER_TEST} ${SRC_TEST_UTIL})
@@ -85,15 +146,15 @@ add_executable(ps_scheduler_test ${SCHEDULER_TEST})
 # profiler
 add_executable(ps_profiler ${PROFILER})
 
-target_link_libraries(ps ${LIBRARYS} libjemalloc.a)
-target_link_libraries(tool ${LIBRARYS} libjemalloc.a)
-target_link_libraries(ps_common_test ${LIBRARYS} gtest gtest_main libjemalloc.a)
-target_link_libraries(ps_message_test ${LIBRARYS} gtest gtest_main libjemalloc.a)
-target_link_libraries(ps_model_server_test ${LIBRARYS} gtest gtest_main libjemalloc.a)
-target_link_libraries(ps_server_test ${LIBRARYS} gtest gtest_main libjemalloc.a)
-target_link_libraries(ps_client_test ${LIBRARYS} gtest gtest_main libjemalloc.a)
-target_link_libraries(ps_scheduler_test ${LIBRARYS} gtest gtest_main libjemalloc.a)
-target_link_libraries(ps_profiler ${LIBRARYS} libjemalloc.a)
+target_link_libraries(ps ${LIBRARYS} libjemalloc.a ${TBB_IMPORTED_TARGETS})
+target_link_libraries(tool ${LIBRARYS} libjemalloc.a ${TBB_IMPORTED_TARGETS})
+target_link_libraries(ps_common_test ${LIBRARYS} gtest gtest_main libjemalloc.a ${TBB_IMPORTED_TARGETS})
+target_link_libraries(ps_message_test ${LIBRARYS} gtest gtest_main libjemalloc.a ${TBB_IMPORTED_TARGETS})
+target_link_libraries(ps_model_server_test ${LIBRARYS} gtest gtest_main libjemalloc.a ${TBB_IMPORTED_TARGETS})
+target_link_libraries(ps_server_test ${LIBRARYS} gtest gtest_main libjemalloc.a ${TBB_IMPORTED_TARGETS})
+target_link_libraries(ps_client_test ${LIBRARYS} gtest gtest_main libjemalloc.a ${TBB_IMPORTED_TARGETS})
+target_link_libraries(ps_scheduler_test ${LIBRARYS} gtest gtest_main libjemalloc.a ${TBB_IMPORTED_TARGETS})
+target_link_libraries(ps_profiler ${LIBRARYS} libjemalloc.a ${TBB_IMPORTED_TARGETS})
 
 enable_testing()
 add_test(NAME ps_common_test COMMAND ps_common_test)

xdl/ps-plus/README.md

@@ -1,3 +1,18 @@
+/* Copyright (C) 2016-2018 Alibaba Group Holding Limited
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
 #ps-plus使用
 
 ps-plus是底层的ps组件，可单独介入到其他的业务系统中。简单实用如下：