Skip to content

Commit 4bc5bf9

Browse files
cxystevencxysteven
authored
Merge pull request #1 from PaddlePaddle/master
Master
2 parents f45b45e + 2c98bec commit 4bc5bf9

File tree

563 files changed

+26809
-6392
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

563 files changed

+26809
-6392
lines changed

‎.dockerignore‎

Lines changed: 0 additions & 1 deletion
This file was deleted.

‎.dockerignore‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
*.DS_Store
2+
build/
3+
*.user
4+
.vscode
5+
.idea
6+
.project
7+
.cproject
8+
.pydevproject
9+
Makefile
10+
.test_env/
11+
third_party/
12+
*~
13+
bazel-*
14+
15+
!build/*.deb

‎.gitignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,14 @@
11
*.DS_Store
22
build/
3+
build_doc/
34
*.user
45

56
.vscode
67
.idea
78
.project
89
.cproject
910
.pydevproject
11+
.settings/
1012
Makefile
1113
.test_env/
1214
third_party/

‎.pre-commit-config.yaml‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,21 @@
11
- repo: https://github.com/Lucas-C/pre-commit-hooks.git
2-
sha: c25201a00e6b0514370501050cf2a8538ac12270
2+
sha: v1.0.1
33
hooks:
44
- id: remove-crlf
5-
files: (?!.*third_party)^.*$
5+
files: (?!.*third_party)^.*$ | (?!.*book)^.*$
66
- repo: https://github.com/reyoung/mirrors-yapf.git
77
sha: v0.13.2
88
hooks:
9-
- id: yapf
10-
files: (.*\.(py|bzl)|BUILD|.*\.BUILD|WORKSPACE)$ # Bazel BUILD files follow Python syntax.
9+
- id: yapf
10+
files: (.*\.(py|bzl)|BUILD|.*\.BUILD|WORKSPACE)$
1111
- repo: https://github.com/pre-commit/pre-commit-hooks
12-
sha: 7539d8bd1a00a3c1bfd34cdb606d3a6372e83469
12+
sha: 5bf6c09bfa1297d3692cadd621ef95f1284e33c0
1313
hooks:
1414
- id: check-added-large-files
1515
- id: check-merge-conflict
1616
- id: check-symlinks
1717
- id: detect-private-key
18-
files: (?!.*third_party)^.*$
18+
files: (?!.*third_party)^.*$ | (?!.*book)^.*$
1919
- id: end-of-file-fixer
2020
- repo: https://github.com/PaddlePaddle/clang-format-pre-commit-hook.git
2121
sha: 28c0ea8a67a3e2dbbf4822ef44e85b63a0080a29

‎.travis.yml‎

Lines changed: 17 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,39 +1,37 @@
11
language: cpp
2-
cache: ccache
2+
cache:
3+
directories:
4+
- $HOME/third_party
5+
- $HOME/.ccache
6+
- $HOME/.cache/pip
37
sudo: required
48
dist: trusty
59
os:
610
- linux
7-
- osx
811
env:
912
- JOB=DOCS
1013
- JOB=BUILD_AND_TEST
1114
- JOB=PRE_COMMIT
12-
matrix:
13-
exclude:
14-
- os: osx
15-
env: JOB=DOCS # Only generate documentation in linux.
16-
- os: osx
17-
env: JOB=PRE_COMMIT # Only check pre-commit hook in linux
18-
1915
addons:
2016
apt:
2117
packages:
2218
- gcc-4.8
2319
- g++-4.8
24-
- wget
20+
- gfortran-4.8
2521
- git
2622
- build-essential
27-
- libatlas-base-dev
2823
- python
2924
- python-pip
3025
- python2.7-dev
26+
- python-numpy
27+
- python-wheel
3128
- curl
3229
- swig
3330
- graphviz
3431
- clang-format-3.8
3532
- automake
3633
- libtool
34+
- ccache
3735
before_install:
3836
- |
3937
if [ ${JOB} == "BUILD_AND_TEST" ]; then
@@ -46,12 +44,16 @@ before_install:
4644
fi
4745
fi
4846
fi
49-
- if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then paddle/scripts/travis/before_install.osx.sh; fi
5047
- if [[ "$JOB" == "PRE_COMMIT" ]]; then sudo ln -s /usr/bin/clang-format-3.8 /usr/bin/clang-format; fi
51-
- pip install --upgrade pip
52-
- pip install wheel protobuf sphinx recommonmark sphinx_rtd_theme virtualenv pre-commit requests==2.9.2 LinkChecker
48+
# Paddle is using protobuf 3.1 currently. Protobuf 3.2 breaks the compatibility. So we specify the python
49+
# protobuf version.
50+
- pip install numpy wheel 'protobuf==3.1' sphinx recommonmark sphinx-rtd-theme==0.1.9 virtualenv pre-commit requests==2.9.2 LinkChecker
51+
- |
52+
function timeout() { perl -e 'alarm shift; exec @ARGV' "$@"; }
5353
script:
54-
- paddle/scripts/travis/main.sh
54+
- |
55+
timeout 2580 paddle/scripts/travis/main.sh # 43min timeout
56+
RESULT=$?; if [ $RESULT -eq 0 ] || [ $RESULT -eq 142 ]; then true; else false; fi;
5557
notifications:
5658
email:
5759
on_success: change

‎CMakeLists.txt‎

Lines changed: 64 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -12,37 +12,70 @@
1212
# See the License for the specific language governing permissions and
1313
# limitations under the License
1414

15-
cmake_minimum_required(VERSION 3.0)
16-
17-
project(paddle CXX C)
18-
1915
set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake")
2016
set(PROJ_ROOT ${CMAKE_SOURCE_DIR})
2117

18+
include(system)
19+
20+
if(ANDROID)
21+
cmake_minimum_required(VERSION 3.7)
22+
else()
23+
cmake_minimum_required(VERSION 3.0)
24+
endif()
25+
26+
project(paddle CXX C)
27+
2228
find_package(Sphinx)
23-
find_package(CUDA QUIET)
29+
if(NOT CMAKE_CROSSCOMPILING)
30+
find_package(CUDA QUIET)
31+
endif(NOT CMAKE_CROSSCOMPILING)
2432
find_package(Git REQUIRED)
2533
find_package(Threads REQUIRED)
2634

27-
include(system)
2835
include(simd)
2936

30-
###################### Configurations ############################
31-
option(WITH_DSO "Compile PaddlePaddle with dynamic linked libraries" ON)
32-
option(WITH_GPU "Compile PaddlePaddle with gpu" ${CUDA_FOUND})
33-
option(WITH_DOUBLE "Compile PaddlePaddle with double precision, otherwise use single precision" OFF)
34-
option(WITH_AVX "Compile PaddlePaddle with avx intrinsics" ${AVX_FOUND})
35-
option(WITH_PYTHON "Compile PaddlePaddle with python interpreter" ON)
36-
option(WITH_STYLE_CHECK "Style Check for PaddlePaddle" ON)
37-
option(WITH_RDMA "Compile PaddlePaddle with rdma support" OFF)
38-
option(WITH_TIMER "Compile PaddlePaddle use timer" OFF)
39-
option(WITH_PROFILER "Compile PaddlePaddle use gpu profiler" OFF)
40-
option(WITH_TESTING "Compile and run unittest for PaddlePaddle" ON)
41-
option(WITH_DOC "Compile PaddlePaddle with documentation" OFF)
42-
option(WITH_SWIG_PY "Compile PaddlePaddle with py PaddlePaddle prediction api" ON)
43-
option(ON_TRAVIS "Running test on travis-ci or not." OFF)
44-
option(ON_COVERALLS "Generating code coverage data on coveralls or not." OFF)
45-
option(COVERALLS_UPLOAD "Uploading the generated coveralls json." ON)
37+
################################ Configurations #######################################
38+
option(WITH_GPU "Compile PaddlePaddle with NVIDIA GPU" ${CUDA_FOUND})
39+
option(WITH_AVX "Compile PaddlePaddle with AVX intrinsics" ${AVX_FOUND})
40+
option(WITH_DSO "Compile PaddlePaddle with dynamic linked CUDA" ON)
41+
option(WITH_TESTING "Compile PaddlePaddle with unit testing" ON)
42+
option(WITH_SWIG_PY "Compile PaddlePaddle with inference api" ON)
43+
option(WITH_STYLE_CHECK "Compile PaddlePaddle with style check" ON)
44+
option(WITH_PYTHON "Compile PaddlePaddle with python interpreter" ON)
45+
option(WITH_DOUBLE "Compile PaddlePaddle with double precision" OFF)
46+
option(WITH_RDMA "Compile PaddlePaddle with RDMA support" OFF)
47+
option(WITH_TIMER "Compile PaddlePaddle with stats timer" OFF)
48+
option(WITH_PROFILER "Compile PaddlePaddle with GPU profiler" OFF)
49+
option(WITH_DOC "Compile PaddlePaddle with documentation" OFF)
50+
option(WITH_COVERAGE "Compile PaddlePaddle with code coverage" OFF)
51+
option(COVERALLS_UPLOAD "Package code coverage data to coveralls" OFF)
52+
option(ON_TRAVIS "Exclude special unit test on Travis CI" OFF)
53+
54+
# CMAKE_BUILD_TYPE
55+
if(NOT CMAKE_BUILD_TYPE)
56+
set(CMAKE_BUILD_TYPE "RelWithDebInfo" CACHE STRING
57+
"Choose the type of build, options are: Debug Release RelWithDebInfo MinSizeRel"
58+
FORCE)
59+
endif()
60+
61+
if(ANDROID)
62+
if(${CMAKE_SYSTEM_VERSION} VERSION_LESS "21")
63+
message(FATAL_ERROR "Unsupport standalone toolchains with Android API level lower than 21")
64+
endif()
65+
66+
set(WITH_GPU OFF CACHE STRING
67+
"Disable GPU when cross-compiling for Android" FORCE)
68+
set(WITH_AVX OFF CACHE STRING
69+
"Disable AVX when cross-compiling for Android" FORCE)
70+
set(WITH_PYTHON OFF CACHE STRING
71+
"Disable PYTHON when cross-compiling for Android" FORCE)
72+
set(WITH_RDMA OFF CACHE STRING
73+
"Disable RDMA when cross-compiling for Android" FORCE)
74+
endif(ANDROID)
75+
76+
set(THIRD_PARTY_PATH "${PROJ_ROOT}/third_party" CACHE STRING
77+
"A path setting third party libraries download & build directories.")
78+
########################################################################################
4679

4780
include(external/zlib) # download, build, install zlib
4881
include(external/gflags) # download, build, install gflags
@@ -53,6 +86,7 @@ include(external/python) # download, build, install python
5386
include(external/openblas) # download, build, install openblas
5487
include(external/swig) # download, build, install swig
5588
include(external/warpctc) # download, build, install warpctc
89+
include(external/any) # download libn::any
5690

5791
include(package) # set paddle packages
5892
include(cpplint) # set paddle c++ style
@@ -63,23 +97,28 @@ include(flags) # set paddle compile flags
6397
include(cudnn) # set cudnn libraries
6498
include(version) # set PADDLE_VERSION
6599
include(coveralls) # set code coverage
66-
include(python_module) # set python module
67-
68100
include(configure) # add paddle env configuration
69101

70102
include_directories("${PROJ_ROOT}")
71103
include_directories("${PROJ_ROOT}/paddle/cuda/include")
72104
include_directories("${CMAKE_CURRENT_BINARY_DIR}/proto")
73105

74106
set(EXTERNAL_LIBS
75-
# have not include gtest here.
76107
${GFLAGS_LIBRARIES}
77108
${GLOG_LIBRARIES}
78109
${CBLAS_LIBRARIES}
79110
${PROTOBUF_LIBRARY}
80111
${ZLIB_LIBRARIES}
112+
${PYTHON_LIBRARIES}
81113
)
82114

115+
if(WITH_GPU)
116+
list(APPEND EXTERNAL_LIB ${CUDA_LIBRARIES} ${CUDA_rt_LIBRARY})
117+
if(NOT WITH_DSO)
118+
list(APPEND EXTERNAL_LIB ${CUDNN_LIBRARY} ${CUDA_CUBLAS_LIBRARIES} ${CUDA_curand_LIBRARY})
119+
endif(NOT WITH_DSO)
120+
endif(WITH_GPU)
121+
83122
add_subdirectory(proto)
84123
add_subdirectory(paddle)
85124
add_subdirectory(python)

‎Dockerfile‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
# A image for building paddle binaries
2+
# Use cuda devel base image for both cpu and gpu environment
3+
FROM nvidia/cuda:8.0-cudnn5-devel-ubuntu14.04
4+
MAINTAINER PaddlePaddle Authors <paddle-dev@baidu.com>
5+
6+
ARG UBUNTU_MIRROR
7+
RUN /bin/bash -c 'if [[ -n ${UBUNTU_MIRROR} ]]; then sed -i 's#http://archive.ubuntu.com/ubuntu#${UBUNTU_MIRROR}#g' /etc/apt/sources.list; fi'
8+
9+
# ENV variables
10+
ARG WITH_GPU
11+
ARG WITH_AVX
12+
ARG WITH_DOC
13+
ARG WITH_STYLE_CHECK
14+
15+
ENV WOBOQ OFF
16+
ENV WITH_GPU=${WITH_GPU:-OFF}
17+
ENV WITH_AVX=${WITH_AVX:-ON}
18+
ENV WITH_DOC=${WITH_DOC:-OFF}
19+
ENV WITH_STYLE_CHECK=${WITH_STYLE_CHECK:-OFF}
20+
21+
ENV HOME /root
22+
# Add bash enhancements
23+
COPY ./paddle/scripts/docker/root/ /root/
24+
25+
RUN apt-get update && \
26+
apt-get install -y git python-pip python-dev openssh-server bison && \
27+
apt-get install -y wget unzip tar xz-utils bzip2 gzip coreutils && \
28+
apt-get install -y curl sed grep graphviz libjpeg-dev zlib1g-dev && \
29+
apt-get install -y python-numpy python-matplotlib gcc g++ gfortran && \
30+
apt-get install -y automake locales clang-format-3.8 swig doxygen && \
31+
apt-get clean -y
32+
33+
# git credential to skip password typing
34+
RUN git config --global credential.helper store
35+
36+
# Fix locales to en_US.UTF-8
37+
RUN localedef -i en_US -f UTF-8 en_US.UTF-8
38+
39+
# FIXME: due to temporary ipykernel dependency issue, specify ipykernel jupyter
40+
# version util jupyter fixes this issue.
41+
RUN pip install --upgrade pip && \
42+
pip install -U 'protobuf==3.1.0' && \
43+
pip install -U wheel pillow BeautifulSoup && \
44+
pip install -U docopt PyYAML sphinx && \
45+
pip install -U sphinx-rtd-theme==0.1.9 recommonmark && \
46+
pip install pre-commit 'requests==2.9.2' 'ipython==5.3.0' && \
47+
pip install 'ipykernel==4.6.0' 'jupyter==1.0.0'
48+
49+
RUN curl -sSL https://cmake.org/files/v3.4/cmake-3.4.1.tar.gz | tar -xz && \
50+
cd cmake-3.4.1 && ./bootstrap && make -j `nproc` && make install && \
51+
cd .. && rm -rf cmake-3.4.1
52+
53+
VOLUME ["/woboq_out"]
54+
55+
# Configure OpenSSH server. c.f. https://docs.docker.com/engine/examples/running_ssh_service
56+
RUN mkdir /var/run/sshd
57+
RUN echo 'root:root' | chpasswd
58+
RUN sed -ri 's/^PermitRootLogin\s+.*/PermitRootLogin yes/' /etc/ssh/sshd_config
59+
RUN sed -ri 's/UsePAM yes/#UsePAM yes/g' /etc/ssh/sshd_config
60+
EXPOSE 22
61+
62+
# development image default do build work
63+
CMD ["bash", "/paddle/paddle/scripts/docker/build.sh"]

‎README.md‎

Lines changed: 24 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,8 @@
22

33

44
[![Build Status](https://travis-ci.org/PaddlePaddle/Paddle.svg?branch=develop)](https://travis-ci.org/PaddlePaddle/Paddle)
5-
[![Documentation Status](https://img.shields.io/badge/docs-latest-brightgreen.svg?style=flat)](http://www.paddlepaddle.org/)
6-
[![Documentation Status](https://img.shields.io/badge/中文文档-最新-brightgreen.svg)](http://www.paddlepaddle.org/cn/index.html)
5+
[![Documentation Status](https://img.shields.io/badge/docs-latest-brightgreen.svg?style=flat)](http://www.paddlepaddle.org/develop/doc/)
6+
[![Documentation Status](https://img.shields.io/badge/中文文档-最新-brightgreen.svg)](http://www.paddlepaddle.org/doc_cn/)
77
[![Coverage Status](https://coveralls.io/repos/github/PaddlePaddle/Paddle/badge.svg?branch=develop)](https://coveralls.io/github/PaddlePaddle/Paddle?branch=develop)
88
[![Release](https://img.shields.io/github/release/PaddlePaddle/Paddle.svg)](https://github.com/PaddlePaddle/Paddle/releases)
99
[![License](https://img.shields.io/badge/license-Apache%202-blue.svg)](LICENSE)
@@ -59,36 +59,36 @@ Please refer to our [release announcement](https://github.com/PaddlePaddle/Paddl
5959
the capability of PaddlePaddle to make a huge impact for your product.
6060

6161
## Installation
62-
Check out the [Install Guide](http://paddlepaddle.org/doc/build/) to install from
63-
pre-built packages (**docker image**, **deb package**) or
64-
directly build on **Linux** and **Mac OS X** from the source code.
62+
63+
It is recommended to check out the
64+
[Docker installation guide](http://www.paddlepaddle.org/develop/doc/getstarted/build_and_install/docker_install_en.html)
65+
before looking into the
66+
[build from source guide](http://www.paddlepaddle.org/develop/doc/getstarted/build_and_install/build_from_source_en.html)
6567

6668
## Documentation
67-
Both [English Docs](http://paddlepaddle.org/doc/) and [Chinese Docs](http://paddlepaddle.org/doc_cn/) are provided for our users and developers.
6869

69-
- [Quick Start](http://paddlepaddle.org/doc/demo/quick_start/index_en) <br>
70-
You can follow the quick start tutorial to learn how use PaddlePaddle
71-
step-by-step.
70+
We provide [English](http://www.paddlepaddle.org/develop/doc/) and
71+
[Chinese](http://www.paddlepaddle.org/doc_cn/) documentation.
72+
73+
- [Deep Learning 101](http://book.paddlepaddle.org/index.en.html)
74+
75+
You might want to start from the this online interactive book that can run in Jupyter Notebook.
76+
77+
- [Distributed Training](http://www.paddlepaddle.org/develop/doc/howto/usage/cluster/cluster_train_en.html)
78+
79+
You can run distributed training jobs on MPI clusters.
80+
81+
- [Distributed Training on Kubernetes](http://www.paddlepaddle.org/develop/doc/howto/usage/k8s/k8s_en.html)
7282

73-
- [Example and Demo](http://paddlepaddle.org/doc/demo/) <br>
74-
We provide five demos, including: image classification, sentiment analysis,
75-
sequence to sequence model, recommendation, semantic role labeling.
83+
You can also run distributed training jobs on Kubernetes clusters.
7684

77-
- [Distributed Training](http://paddlepaddle.org/doc/cluster) <br>
78-
This system supports training deep learning models on multiple machines
79-
with data parallelism.
85+
- [Python API](http://www.paddlepaddle.org/develop/doc/api/index_en.html)
8086

81-
- [Python API](http://paddlepaddle.org/doc/ui/) <br>
82-
PaddlePaddle supports using either Python interface or C++ to build your
83-
system. We also use SWIG to wrap C++ source code to create a user friendly
84-
interface for Python. You can also use SWIG to create interface for your
85-
favorite programming language.
87+
Our new API enables much shorter programs.
8688

87-
- [How to Contribute](http://paddlepaddle.org/doc/build/contribute_to_paddle.html) <br>
88-
We sincerely appreciate your interest and contributions. If you would like to
89-
contribute, please read the contribution guide.
89+
- [How to Contribute](http://www.paddlepaddle.org/develop/doc/howto/dev/contribute_to_paddle_en.html)
9090

91-
- [Source Code Documents](http://paddlepaddle.org/doc/source/) <br>
91+
We appreciate your contributions!
9292

9393
## Ask Questions
9494

0 commit comments

Comments
 (0)