From a19711bd5979740c44504b8d4d074e7e7e5d7486 Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Wed, 4 Jan 2017 09:51:31 +0800 Subject: [PATCH 1/5] add parameter server python api --- paddle/api/CMakeLists.txt | 3 ++- paddle/api/Paddle.swig | 2 ++ paddle/api/PaddleAPI.h | 21 +++++++++++++++++++++ paddle/api/PaddleAPIPrivate.h | 5 +++++ paddle/api/ParameterServer.cpp | 31 +++++++++++++++++++++++++++++++ 5 files changed, 61 insertions(+), 1 deletion(-) create mode 100644 paddle/api/ParameterServer.cpp diff --git a/paddle/api/CMakeLists.txt b/paddle/api/CMakeLists.txt index da6dad10cd8076..c060865764c4ea 100644 --- a/paddle/api/CMakeLists.txt +++ b/paddle/api/CMakeLists.txt @@ -10,7 +10,8 @@ set(API_SOURCES SequenceGenerator.cpp Trainer.cpp Util.cpp - Vector.cpp) + Vector.cpp + ParameterServer.cpp) set(API_HEADER PaddleAPI.h Internal.h) diff --git a/paddle/api/Paddle.swig b/paddle/api/Paddle.swig index 068ba286c07d88..84c50c30add877 100644 --- a/paddle/api/Paddle.swig +++ b/paddle/api/Paddle.swig @@ -179,6 +179,7 @@ namespace std { %newobject ParameterOptimizer::needSpecialTraversal; %newobject ParameterUpdater::createLocalUpdater; %newobject ParameterUpdater::createRemoteUpdater; +%newobject ParameterServer::createParameterServer; %feature("director") UpdateCallback; %feature("autodoc", 1); // To generate method stub, for code hint in ide @@ -197,5 +198,6 @@ namespace std { %ignore ParameterConfigPrivate; %ignore OptimizationConfigPrivate; %ignore ParameterTraverseCallbackPrivate; +%ignore ParameterServerPrivate; %include "utils/GlobalConstants.h" %include "api/PaddleAPI.h" diff --git a/paddle/api/PaddleAPI.h b/paddle/api/PaddleAPI.h index 81c9eed0bccd5a..c129a6c2279ef7 100644 --- a/paddle/api/PaddleAPI.h +++ b/paddle/api/PaddleAPI.h @@ -868,6 +868,27 @@ class ParameterUpdater { ParameterUpdaterPrivate* m; }; +struct ParameterServerPrivate; +class ParameterServer { +private: + ParameterServer(); + +public: + static ParameterServer* createParameterServer(); + + ~ParameterServer(); + + /** + * @brief initialize Parameter Server. + * @param gm + */ + void start(); + void join(); + +private: + ParameterServerPrivate* m; +}; + struct EvaluatorPrivate; class Evaluator { private: diff --git a/paddle/api/PaddleAPIPrivate.h b/paddle/api/PaddleAPIPrivate.h index f41352bfec7c33..74c96b8acef9c9 100644 --- a/paddle/api/PaddleAPIPrivate.h +++ b/paddle/api/PaddleAPIPrivate.h @@ -17,6 +17,7 @@ limitations under the License. */ #include "paddle/gserver/evaluators/Evaluator.h" #include "paddle/gserver/gradientmachines/GradientMachine.h" #include "paddle/parameter/ParameterUpdaterBase.h" +#include "paddle/pserver/PServerUtil.h" #include "paddle/trainer/TrainerConfigHelper.h" struct GradientMachinePrivate { @@ -72,6 +73,10 @@ struct ParameterUpdaterPrivate { std::unique_ptr updater; }; +struct ParameterServerPrivate { + std::unique_ptr pServerUtil; +}; + struct ParameterPrivate { std::shared_ptr sharedPtr; paddle::Parameter* rawPtr; // rawPtr only used in ParameterUpdater, diff --git a/paddle/api/ParameterServer.cpp b/paddle/api/ParameterServer.cpp new file mode 100644 index 00000000000000..f0acfd67bef355 --- /dev/null +++ b/paddle/api/ParameterServer.cpp @@ -0,0 +1,31 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "PaddleAPI.h" + +#include "PaddleAPIPrivate.h" + +ParameterServer::ParameterServer() : m(new ParameterServerPrivate()) {} + +ParameterServer* ParameterServer::createParameterServer() { + auto pServer = new ParameterServer(); + pServer->m->pServerUtil.reset(paddle::PServerUtil::createWithGflags()); + return pServer; +} + +ParameterServer::~ParameterServer() { delete m; } + +void ParameterServer::start() { m->pServerUtil->start(); } + +void ParameterServer::join() { m->pServerUtil->join(); } From 9b4ebd1817ab90c9e1a28d7ecce6f37456ba503c Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Wed, 4 Jan 2017 16:45:26 +0800 Subject: [PATCH 2/5] add swig api for parameter server --- demo/quick_start/cluster/pserver.py | 32 +++++++++++++++++++++++++++++ paddle/api/Paddle.swig | 3 ++- paddle/api/PaddleAPI.h | 3 ++- paddle/api/ParameterServer.cpp | 16 +++++++++++++-- paddle/py_paddle/util.py | 32 +++++++++++++++++++++++------ proto/ParameterServerConfig.proto | 18 ++++++++-------- 6 files changed, 85 insertions(+), 19 deletions(-) create mode 100644 demo/quick_start/cluster/pserver.py diff --git a/demo/quick_start/cluster/pserver.py b/demo/quick_start/cluster/pserver.py new file mode 100644 index 00000000000000..a09316729c7a52 --- /dev/null +++ b/demo/quick_start/cluster/pserver.py @@ -0,0 +1,32 @@ +# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from py_paddle import swig_paddle as api +import paddle.proto.ParameterServerConfig_pb2 as ParameterServerConfig + + +def main(): + api.initPaddle() + pServerConfig = ParameterServerConfig.ParameterServerConfig() + pServerConfig.ports_num = 1 + pServerConfig.nics = "lo0" + pServerConfig.num_gradient_servers = 1 + pServerConfig.port = 7164 + pserver = api.ParameterServer.createFromConfigProto(pServerConfig) + pserver.start() + pserver.join() + + +if __name__ == '__main__': + main() diff --git a/paddle/api/Paddle.swig b/paddle/api/Paddle.swig index 84c50c30add877..ca5d4fcefaf12b 100644 --- a/paddle/api/Paddle.swig +++ b/paddle/api/Paddle.swig @@ -179,7 +179,8 @@ namespace std { %newobject ParameterOptimizer::needSpecialTraversal; %newobject ParameterUpdater::createLocalUpdater; %newobject ParameterUpdater::createRemoteUpdater; -%newobject ParameterServer::createParameterServer; +%newobject ParameterServer::createByConfigProtoPtr; +%newobject ParameterServer::createByConfigProtoStr; %feature("director") UpdateCallback; %feature("autodoc", 1); // To generate method stub, for code hint in ide diff --git a/paddle/api/PaddleAPI.h b/paddle/api/PaddleAPI.h index c129a6c2279ef7..cb691cb35167b3 100644 --- a/paddle/api/PaddleAPI.h +++ b/paddle/api/PaddleAPI.h @@ -874,7 +874,8 @@ class ParameterServer { ParameterServer(); public: - static ParameterServer* createParameterServer(); + static ParameterServer* createByConfigProtoPtr(const void* confPtr); + static ParameterServer* createByConfigProtoStr(const std::string& protoStr); ~ParameterServer(); diff --git a/paddle/api/ParameterServer.cpp b/paddle/api/ParameterServer.cpp index f0acfd67bef355..1014648b95ba03 100644 --- a/paddle/api/ParameterServer.cpp +++ b/paddle/api/ParameterServer.cpp @@ -18,12 +18,24 @@ limitations under the License. */ ParameterServer::ParameterServer() : m(new ParameterServerPrivate()) {} -ParameterServer* ParameterServer::createParameterServer() { +ParameterServer* ParameterServer::createByConfigProtoPtr(const void* confPtr) { + auto& conf = *(const paddle::ParameterServerConfig*)(confPtr); auto pServer = new ParameterServer(); - pServer->m->pServerUtil.reset(paddle::PServerUtil::createWithGflags()); + pServer->m->pServerUtil.reset(paddle::PServerUtil::create(conf)); return pServer; } +ParameterServer* ParameterServer::createByConfigProtoStr( + const std::string& protoStr) { + paddle::ParameterServerConfig conf; + conf.ParseFromString(protoStr); + if (conf.IsInitialized()) { + return ParameterServer::createByConfigProtoPtr(&conf); + } else { + return nullptr; + } +} + ParameterServer::~ParameterServer() { delete m; } void ParameterServer::start() { m->pServerUtil->start(); } diff --git a/paddle/py_paddle/util.py b/paddle/py_paddle/util.py index ce105d249aaf3e..1e483fb7c41140 100644 --- a/paddle/py_paddle/util.py +++ b/paddle/py_paddle/util.py @@ -15,18 +15,20 @@ Some Useful method for py_paddle. """ -import swig_paddle import os -import paddle.trainer.PyDataProviderWrapper -import paddle.proto.ParameterConfig_pb2 -import paddle.proto.ModelConfig_pb2 -import paddle.proto.TrainerConfig_pb2 import weakref import numpy import struct import sys import copy +import swig_paddle +import paddle.trainer.PyDataProviderWrapper +import paddle.proto.ParameterConfig_pb2 +import paddle.proto.ModelConfig_pb2 +import paddle.proto.TrainerConfig_pb2 +import paddle.proto.ParameterServerConfig_pb2 + def initializePaddle(*args): """ @@ -558,11 +560,29 @@ def getForwardOutput(self): swig_paddle.Trainer.getForwardOutput = getForwardOutput +def __monkeypatch_parameter_server__(): + def createFromConfigProto(protoObj): + """ + Create Parameter Server From Proto object. + :param protoObj: ParameterServer Config + :type protoObj: proto.ParameterServerConfig_pb2.ParameterServerConfig + :return: paddle.ParameterServer + """ + assert isinstance( + protoObj, + paddle.proto.ParameterServerConfig_pb2.ParameterServerConfig) + return swig_paddle.ParameterServer.createByConfigProtoStr( + protoObj.SerializeToString()) + + swig_paddle.ParameterServer.createFromConfigProto = \ + staticmethod(createFromConfigProto) + + def monkeypatches(): patches = [ __monkeypatch_init_paddle__, __monkeypatch_gradient_machine__, __monkey_patch_protobuf_objects__, __monkey_patch_parameter__, - __monkey_patch_trainer__ + __monkey_patch_trainer__, __monkeypatch_parameter_server__ ] for patch in patches: patch() diff --git a/proto/ParameterServerConfig.proto b/proto/ParameterServerConfig.proto index b4fbf901c20cce..6632af8e2543c7 100644 --- a/proto/ParameterServerConfig.proto +++ b/proto/ParameterServerConfig.proto @@ -22,22 +22,22 @@ message ParameterClientConfig { message ParameterServerConfig { // The ports number for parameter send, // increment based on default port number - required int32 ports_num = 1 [default = 1]; + optional int32 ports_num = 1 [default = 1]; // The ports number for parameter send, // increment based on default (port + ports_num - required int32 ports_num_for_sparse = 2 [default = 0]; + optional int32 ports_num_for_sparse = 2 [default = 0]; // network device name for pservers - required string nics = 3 [default = "xgbe0,xgbe1"]; - required string rdma_tcp = 4 [default = "tcp"]; + optional string nics = 3 [default = "xgbe0,xgbe1"]; + optional string rdma_tcp = 4 [default = "tcp"]; // Listening port for pserver - required int32 port = 5 [default = 20134]; + optional int32 port = 5 [default = 20134]; // number of gradient servers - required int32 num_gradient_servers = 6 [default = 1]; + optional int32 num_gradient_servers = 6 [default = 1]; // number of threads for sync op exec - required int32 pserver_num_threads = 7 [default = 1]; + optional int32 pserver_num_threads = 7 [default = 1]; // control config_.async_lagged_grad_discard_ratio() min value - required double async_lagged_ratio_min = 8 [default = 1.0]; + optional double async_lagged_ratio_min = 8 [default = 1.0]; // if async_lagged_grad_discard_ratio is not set in trainer_config.conf // use it as defalut value - required double async_lagged_ratio_default = 9 [default = 1.5]; + optional double async_lagged_ratio_default = 9 [default = 1.5]; } \ No newline at end of file From 77c501cec45c0e833fd892bf026c539e05ba1ab1 Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Wed, 4 Jan 2017 19:46:57 +0800 Subject: [PATCH 3/5] PServerUtil -> PServerController after merge --- paddle/api/PaddleAPIPrivate.h | 4 ++-- paddle/api/ParameterServer.cpp | 6 +++--- paddle/pserver/PServerController.h | 10 +++++----- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/paddle/api/PaddleAPIPrivate.h b/paddle/api/PaddleAPIPrivate.h index 74c96b8acef9c9..1f92f587f40841 100644 --- a/paddle/api/PaddleAPIPrivate.h +++ b/paddle/api/PaddleAPIPrivate.h @@ -17,7 +17,7 @@ limitations under the License. */ #include "paddle/gserver/evaluators/Evaluator.h" #include "paddle/gserver/gradientmachines/GradientMachine.h" #include "paddle/parameter/ParameterUpdaterBase.h" -#include "paddle/pserver/PServerUtil.h" +#include "paddle/pserver/PServerController.h" #include "paddle/trainer/TrainerConfigHelper.h" struct GradientMachinePrivate { @@ -74,7 +74,7 @@ struct ParameterUpdaterPrivate { }; struct ParameterServerPrivate { - std::unique_ptr pServerUtil; + std::unique_ptr pServerController; }; struct ParameterPrivate { diff --git a/paddle/api/ParameterServer.cpp b/paddle/api/ParameterServer.cpp index 1014648b95ba03..6a4fcdbb431329 100644 --- a/paddle/api/ParameterServer.cpp +++ b/paddle/api/ParameterServer.cpp @@ -21,7 +21,7 @@ ParameterServer::ParameterServer() : m(new ParameterServerPrivate()) {} ParameterServer* ParameterServer::createByConfigProtoPtr(const void* confPtr) { auto& conf = *(const paddle::ParameterServerConfig*)(confPtr); auto pServer = new ParameterServer(); - pServer->m->pServerUtil.reset(paddle::PServerUtil::create(conf)); + pServer->m->pServerController.reset(paddle::PServerController::create(conf)); return pServer; } @@ -38,6 +38,6 @@ ParameterServer* ParameterServer::createByConfigProtoStr( ParameterServer::~ParameterServer() { delete m; } -void ParameterServer::start() { m->pServerUtil->start(); } +void ParameterServer::start() { m->pServerController->start(); } -void ParameterServer::join() { m->pServerUtil->join(); } +void ParameterServer::join() { m->pServerController->join(); } diff --git a/paddle/pserver/PServerController.h b/paddle/pserver/PServerController.h index cecf7290094a25..1cd78ec63ccfe4 100644 --- a/paddle/pserver/PServerController.h +++ b/paddle/pserver/PServerController.h @@ -26,7 +26,7 @@ class PServerController final { DISABLE_COPY(PServerController); /** - * @brief Ctor, Create a PServerUtil from ParameterServerConfig. + * @brief Ctor, Create a PServerController from ParameterServerConfig. */ explicit PServerController(const ParameterServerConfig& config); @@ -36,24 +36,24 @@ class PServerController final { ~PServerController(); /** - * @brief create PServerUtil from gflags, this is used for + * @brief create PServerController from gflags, this is used for * compatibility with the old usage of configuration by gflags. */ static PServerController* createByGflags(); /** - * @brief create PServerUtil with ParameterServerConfig, remove gflags + * @brief create PServerController with ParameterServerConfig, remove gflags * from ParameterServer. Init all pservers thread according to the config. */ static PServerController* create(const ParameterServerConfig& config); /** - * @brief start all pserver thread in this PServerUtil. + * @brief start all pserver thread in this PServerController. */ void start(); /** - * @brief join and wait for all pserver thread in this PServerUtil. + * @brief join and wait for all pserver thread in this PServerController. */ void join(); From 1134ecc5a2a224a8d47cc39aca2695547610b1be Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Mon, 9 Jan 2017 14:58:26 +0800 Subject: [PATCH 4/5] change pServerController to parameterServerController --- paddle/api/PaddleAPIPrivate.h | 4 ++-- paddle/api/ParameterServer.cpp | 7 ++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/paddle/api/PaddleAPIPrivate.h b/paddle/api/PaddleAPIPrivate.h index 1f92f587f40841..bb1a669fa19676 100644 --- a/paddle/api/PaddleAPIPrivate.h +++ b/paddle/api/PaddleAPIPrivate.h @@ -17,7 +17,7 @@ limitations under the License. */ #include "paddle/gserver/evaluators/Evaluator.h" #include "paddle/gserver/gradientmachines/GradientMachine.h" #include "paddle/parameter/ParameterUpdaterBase.h" -#include "paddle/pserver/PServerController.h" +#include "paddle/pserver/ParameterServerController.h" #include "paddle/trainer/TrainerConfigHelper.h" struct GradientMachinePrivate { @@ -74,7 +74,7 @@ struct ParameterUpdaterPrivate { }; struct ParameterServerPrivate { - std::unique_ptr pServerController; + std::unique_ptr parameterServerController; }; struct ParameterPrivate { diff --git a/paddle/api/ParameterServer.cpp b/paddle/api/ParameterServer.cpp index 6a4fcdbb431329..620ed293e401fd 100644 --- a/paddle/api/ParameterServer.cpp +++ b/paddle/api/ParameterServer.cpp @@ -21,7 +21,8 @@ ParameterServer::ParameterServer() : m(new ParameterServerPrivate()) {} ParameterServer* ParameterServer::createByConfigProtoPtr(const void* confPtr) { auto& conf = *(const paddle::ParameterServerConfig*)(confPtr); auto pServer = new ParameterServer(); - pServer->m->pServerController.reset(paddle::PServerController::create(conf)); + pServer->m->parameterServerController.reset( + paddle::ParameterServerController::create(conf)); return pServer; } @@ -38,6 +39,6 @@ ParameterServer* ParameterServer::createByConfigProtoStr( ParameterServer::~ParameterServer() { delete m; } -void ParameterServer::start() { m->pServerController->start(); } +void ParameterServer::start() { m->parameterServerController->start(); } -void ParameterServer::join() { m->pServerController->join(); } +void ParameterServer::join() { m->parameterServerController->join(); } From 55c8eafdbeed5bff6fb9e00035ebf49de4fb5085 Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Tue, 17 Jan 2017 17:03:49 +0800 Subject: [PATCH 5/5] change join to wait --- demo/quick_start/cluster/pserver.py | 2 +- paddle/api/PaddleAPI.h | 2 +- paddle/api/ParameterServer.cpp | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/demo/quick_start/cluster/pserver.py b/demo/quick_start/cluster/pserver.py index a09316729c7a52..b36f3749877e01 100644 --- a/demo/quick_start/cluster/pserver.py +++ b/demo/quick_start/cluster/pserver.py @@ -25,7 +25,7 @@ def main(): pServerConfig.port = 7164 pserver = api.ParameterServer.createFromConfigProto(pServerConfig) pserver.start() - pserver.join() + pserver.wait() if __name__ == '__main__': diff --git a/paddle/api/PaddleAPI.h b/paddle/api/PaddleAPI.h index 42561338c5271b..41c429490ea333 100644 --- a/paddle/api/PaddleAPI.h +++ b/paddle/api/PaddleAPI.h @@ -890,7 +890,7 @@ class ParameterServer { * @param gm */ void start(); - void join(); + void wait(); private: ParameterServerPrivate* m; diff --git a/paddle/api/ParameterServer.cpp b/paddle/api/ParameterServer.cpp index 620ed293e401fd..8ef01ce538417a 100644 --- a/paddle/api/ParameterServer.cpp +++ b/paddle/api/ParameterServer.cpp @@ -41,4 +41,4 @@ ParameterServer::~ParameterServer() { delete m; } void ParameterServer::start() { m->parameterServerController->start(); } -void ParameterServer::join() { m->parameterServerController->join(); } +void ParameterServer::wait() { m->parameterServerController->wait(); }