diff --git a/.github/workflows/deploy-github-page.yml b/.github/workflows/deploy-github-page.yml index 4f60b9f0..6929951a 100644 --- a/.github/workflows/deploy-github-page.yml +++ b/.github/workflows/deploy-github-page.yml @@ -33,7 +33,7 @@ jobs: environment-file: environment-wasm-build.yml init-shell: bash environment-name: xeus-python-wasm-build - + - name: Set ncpus run: echo "ncpus=$(nproc --all)" >> $GITHUB_ENV @@ -47,6 +47,12 @@ jobs: set -eux + + URL=https://github.com/DerThorsten/xeus-python-shell/archive/refs/heads/libuv.zip + curl -L $URL -o xeus-python-shell-libuv.zip + unzip xeus-python-shell-libuv.zip + + export PREFIX=$MAMBA_ROOT_PREFIX/envs/xeus-python-wasm-host echo "PREFIX=$PREFIX" >> $GITHUB_ENV @@ -60,13 +66,19 @@ jobs: emmake make -j${{ env.ncpus }} install + + + - name: Jupyter Lite integration shell: bash -l {0} run: | + THIS_DIR=$(pwd) jupyter lite build \ --XeusAddon.prefix=${{ env.PREFIX }} \ --contents notebooks/ \ - --output-dir dist + --output-dir dist \ + --XeusAddon.mounts=$THIS_DIR/xeus-python-shell-libuv/xeus_python_shell/:/lib/python3.13/site-packages/xeus_python_shell/ + - name: Upload artifact uses: actions/upload-pages-artifact@v4 diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9870f650..c79417e8 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -22,7 +22,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-22.04, ubuntu-24.04, macos-13, macos-14] + os: [ubuntu-22.04, ubuntu-24.04, macos-14] build_type: [static_build, shared_build] steps: diff --git a/CMakeLists.txt b/CMakeLists.txt index 92429df1..72053244 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -105,8 +105,14 @@ else () if (NOT TARGET xeus-zmq AND NOT TARGET xeus-zmq-static) find_package(xeus-zmq ${xeus-zmq_REQUIRED_VERSION} REQUIRED) endif () + + find_package(xeus-uv REQUIRED) + find_library(UVW_LIBRARY uvw) + find_path(UVW_INCLUDE_DIR uvw.hpp) endif() + + # Configuration # ============= @@ -163,6 +169,7 @@ set(XEUS_PYTHON_SRC src/xdebugpy_client.cpp src/xdisplay.cpp src/xdisplay.hpp + src/xhook.cpp src/xinput.cpp src/xinput.hpp src/xinspect.cpp @@ -188,6 +195,7 @@ set(XEUS_PYTHON_HEADERS include/xeus-python/xinterpreter_raw.hpp include/xeus-python/xtraceback.hpp include/xeus-python/xutils.hpp + include/xeus-python/xhook.hpp ) set(XPYTHON_SRC @@ -317,7 +325,10 @@ macro(xpyt_create_target target_name src headers linkage output_name) set(XPYT_XEUS_TARGET xeus-zmq-static) endif () - target_link_libraries(${target_name} PUBLIC ${XPYT_XEUS_TARGET} PRIVATE pybind11::pybind11 pybind11_json) + target_link_libraries(${target_name} + PUBLIC ${XPYT_XEUS_TARGET} ${UVW_LIBRARY} + PRIVATE pybind11::pybind11 pybind11_json + ) if (WIN32 OR CYGWIN) target_link_libraries(${target_name} PRIVATE ${PYTHON_LIBRARIES}) elseif (APPLE) @@ -371,7 +382,7 @@ endif () if (XPYT_BUILD_XPYTHON_EXECUTABLE) add_executable(xpython ${XPYTHON_SRC}) - target_link_libraries(xpython PRIVATE pybind11::embed) + target_link_libraries(xpython PRIVATE xeus-uv pybind11::embed ) xpyt_set_common_options(xpython) xpyt_set_kernel_options(xpython) diff --git a/environment-dev.yml b/environment-dev.yml index be0e6932..1cef1590 100644 --- a/environment-dev.yml +++ b/environment-dev.yml @@ -17,6 +17,7 @@ dependencies: - python <3.13 - xeus-python-shell>=0.6.3,<0.7 - debugpy>=1.6.5 + - libuvw - ipython # Test dependencies - pytest diff --git a/environment-wasm-build.yml b/environment-wasm-build.yml index 617bf918..59cec0e7 100644 --- a/environment-wasm-build.yml +++ b/environment-wasm-build.yml @@ -13,3 +13,5 @@ dependencies: - jupyterlite-core >0.6 - jupyter_server - jupyterlite-xeus + - # widgets + - ipywidgets \ No newline at end of file diff --git a/environment-wasm-host.yml b/environment-wasm-host.yml index 72e89111..9004a4a5 100644 --- a/environment-wasm-host.yml +++ b/environment-wasm-host.yml @@ -12,9 +12,11 @@ dependencies: - numpy - xeus-lite - xeus - - xeus-python-shell>=0.6.3 - pyjs >=2,<3 - libpython - zstd - openssl - xz + - ipywidgets + # - xeus-python-shell>=0.6.3 + diff --git a/include/xeus-python/xhook.hpp b/include/xeus-python/xhook.hpp new file mode 100644 index 00000000..033edc16 --- /dev/null +++ b/include/xeus-python/xhook.hpp @@ -0,0 +1,52 @@ +/*************************************************************************** +* Copyright (c) 2018, Martin Renou, Johan Mabille, Sylvain Corlay, and * +* Wolf Vollprecht * +* Copyright (c) 2018, QuantStack +* * +* Distributed under the terms of the BSD 3-Clause License. * +* * +* The full license is in the file LICENSE, distributed with this software. * +****************************************************************************/ + +#ifndef XPYT_HOOK_HPP +#define XPYT_HOOK_HPP + +// pybind11 code internally forces hidden visibility on all internal code, but +// if non-hidden (and thus exported) code attempts to include a pybind type +// this warning occurs: +// 'xpyt::hook' declared with greater visibility than the type of its +// field 'xpyt::hook::p_acquire' [-Wattributes] +#ifdef __GNUC__ + #pragma GCC diagnostic ignored "-Wattributes" +#endif + +#include "xeus-python/xeus_python_config.hpp" +#include "xeus-uv/xhook_base.hpp" + +#include "pybind11/embed.h" +#include "pybind11/pybind11.h" + +namespace py = pybind11; + +namespace xpyt +{ + XEUS_PYTHON_API + class hook : public xeus::xhook_base + { + public: + + hook() = default; + virtual ~hook(); + + private: + + void pre_hook_impl() override; + void post_hook_impl() override; + void run_impl(std::shared_ptr loop) override; + + py::gil_scoped_acquire* p_acquire{ nullptr}; + }; + +} + +#endif diff --git a/include/xeus-python/xinterpreter.hpp b/include/xeus-python/xinterpreter.hpp index 86727408..6ffdae81 100644 --- a/include/xeus-python/xinterpreter.hpp +++ b/include/xeus-python/xinterpreter.hpp @@ -93,7 +93,7 @@ namespace xpyt // is started, m_release_gil_at_startup has to be set to false to prevent // releasing it again in configure_impl(). // - bool m_release_gil_at_startup = true; + bool m_release_gil_at_startup = false; gil_scoped_release_ptr m_release_gil = nullptr; bool m_redirect_output_enabled; @@ -101,6 +101,20 @@ namespace xpyt private: + // helper methods: + void execute_request_impl_sync(send_reply_callback cb, + int execution_counter, + const std::string& code, + xeus::execute_request_config config, + nl::json user_expressions); + + void execute_request_impl_async(send_reply_callback cb, + int execution_counter, + const std::string& code, + xeus::execute_request_config config, + nl::json user_expressions); + + virtual void instanciate_ipython_shell(); virtual bool use_jedi_for_completion() const; }; diff --git a/notebooks/xeus-python.ipynb b/notebooks/xeus-python.ipynb index 12a0b75b..026d828c 100644 --- a/notebooks/xeus-python.ipynb +++ b/notebooks/xeus-python.ipynb @@ -19,7 +19,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, "outputs": [], "source": [ @@ -28,18 +28,40 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "3" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "a" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "7921" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "b = 89\n", "\n", @@ -51,9 +73,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "print" ] @@ -653,15 +686,18 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3.9 (XPython)", + "display_name": "Python 3.13 (XPython)", "language": "python", "name": "xpython" }, "language_info": { + "codemirror_mode": "{\"name\": \"ipython\", \"version\": 3}", "file_extension": ".py", "mimetype": "text/x-python", "name": "python", - "version": "3.9.1" + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.11" } }, "nbformat": 4, diff --git a/src/main.cpp b/src/main.cpp index 8862a319..f1bf3733 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -21,14 +21,24 @@ #include #endif +#ifndef UVW_AS_LIB +#define UVW_AS_LIB +#include +#endif + #include "xeus/xkernel.hpp" #include "xeus/xkernel_configuration.hpp" #include "xeus/xinterpreter.hpp" #include "xeus/xhelper.hpp" -#include "xeus-zmq/xserver_zmq_split.hpp" +#include "xeus-zmq/xserver_zmq.hpp" #include "xeus-zmq/xzmq_context.hpp" + +#include "xeus-uv/xserver_uv.hpp" +#include "xeus-uv/xhook_base.hpp" + + #include "pybind11/embed.h" #include "pybind11/pybind11.h" @@ -38,10 +48,10 @@ #include "xeus-python/xpaths.hpp" #include "xeus-python/xeus_python_config.hpp" #include "xeus-python/xutils.hpp" +#include "xeus-python/xhook.hpp" namespace py = pybind11; - int main(int argc, char* argv[]) { if (xeus::should_print_version(argc, argv)) @@ -88,11 +98,38 @@ int main(int argc, char* argv[]) config.home = const_cast(wstr.c_str()); xpyt::print_pythonhome(); - // Implicitly pre-initialize Python - status = PyConfig_SetBytesArgv(&config, argc, argv); - if (PyStatus_Exception(status)) { - std::cerr << "Error:" << status.err_msg << std::endl; + // Instantiating the Python interpreter + py::scoped_interpreter guard{}; + + uv_loop_t* uv_loop_ptr{ nullptr }; + + { + py::gil_scoped_acquire acquire; + + // Create a uvloop and get pointer to the loop + py::module asyncio = py::module::import("asyncio"); + py::module uvloop = py::module::import("uvloop"); + py::object loop = uvloop.attr("new_event_loop")(); + asyncio.attr("set_event_loop")(loop); + + std::cout<<"getting uv loop pointer from uvloop"<(raw_ptr); + } + + if (!uv_loop_ptr) + { + throw std::runtime_error("Failed to get libuv loop pointer"); } + auto loop_ptr = uvw::loop::create(uv_loop_ptr); // Setting argv wchar_t** argw = new wchar_t*[size_t(argc)]; @@ -113,8 +150,7 @@ int main(int argc, char* argv[]) } delete[] argw; - // Instantiating the Python interpreter - py::scoped_interpreter guard; + std::unique_ptr context = xeus::make_zmq_context(); @@ -146,6 +182,16 @@ int main(int argc, char* argv[]) nl::json debugger_config; debugger_config["python"] = executable; + auto py_hook = std::make_unique(); + + + + auto make_xserver = [&](xeus::xcontext& context, + const xeus::xconfiguration& config, + nl::json::error_handler_t eh) { + return xeus::make_xserver_uv(context, config, eh, loop_ptr, std::move(py_hook)); + }; + if (!connection_filename.empty()) { xeus::xconfiguration config = xeus::load_configuration(connection_filename); @@ -154,12 +200,12 @@ int main(int argc, char* argv[]) xeus::get_user_name(), std::move(context), std::move(interpreter), - xeus::make_xserver_shell_main, + make_xserver, std::move(hist), xeus::make_console_logger(xeus::xlogger::msg_type, - xeus::make_file_logger(xeus::xlogger::content, "xeus.log")), - xpyt::make_python_debugger, - debugger_config); + xeus::make_file_logger(xeus::xlogger::content, "xeus.log"))); + // xpyt::make_python_debugger, + // debugger_config); std::clog << "Starting xeus-python kernel...\n\n" @@ -175,7 +221,7 @@ int main(int argc, char* argv[]) xeus::xkernel kernel(xeus::get_user_name(), std::move(context), std::move(interpreter), - xeus::make_xserver_shell_main, + make_xserver, std::move(hist), nullptr, xpyt::make_python_debugger, @@ -201,7 +247,9 @@ int main(int argc, char* argv[]) "}\n```" << std::endl; + std::cout << "Starting kernel..." << std::endl; kernel.start(); + std::cout << "Kernel stopped." << std::endl; } return 0; diff --git a/src/xdebugger.cpp b/src/xdebugger.cpp index 700e4702..f6a6624b 100644 --- a/src/xdebugger.cpp +++ b/src/xdebugger.cpp @@ -26,6 +26,7 @@ #include "pybind11/stl.h" #include "xeus/xinterpreter.hpp" +#include "xeus/xeus_context.hpp" #include "xeus/xsystem.hpp" #include "xeus-zmq/xmiddleware.hpp" diff --git a/src/xdebugpy_client.cpp b/src/xdebugpy_client.cpp index f6825530..6924de19 100644 --- a/src/xdebugpy_client.cpp +++ b/src/xdebugpy_client.cpp @@ -10,6 +10,7 @@ #include "nlohmann/json.hpp" #include "xeus/xmessage.hpp" +#include "xeus/xeus_context.hpp" #include "xdebugpy_client.hpp" #include diff --git a/src/xdebugpy_client.hpp b/src/xdebugpy_client.hpp index 0f83e5e0..aec9e4e5 100644 --- a/src/xdebugpy_client.hpp +++ b/src/xdebugpy_client.hpp @@ -12,6 +12,7 @@ #define XPYT_DEBUGPY_CLIENT_HPP #include "xeus-zmq/xdap_tcp_client.hpp" +#include "xeus/xeus_context.hpp" namespace xpyt { diff --git a/src/xhook.cpp b/src/xhook.cpp new file mode 100644 index 00000000..5fcf2db1 --- /dev/null +++ b/src/xhook.cpp @@ -0,0 +1,51 @@ +/*************************************************************************** +* Copyright (c) 2018, Martin Renou, Johan Mabille, Sylvain Corlay, and * +* Wolf Vollprecht * +* Copyright (c) 2018, QuantStack +* * +* Distributed under the terms of the BSD 3-Clause License. * +* * +* The full license is in the file LICENSE, distributed with this software. * +****************************************************************************/ + +#ifndef UVW_AS_LIB +#define UVW_AS_LIB +#include +#endif + +#include "xeus-python/xhook.hpp" + +#include "pybind11/embed.h" +#include "pybind11/pybind11.h" + +namespace py = pybind11; + +namespace xpyt +{ + hook::~hook() + { + delete p_acquire; + } + + void hook::pre_hook_impl() + { + if (!p_acquire) + { + p_acquire = new py::gil_scoped_acquire(); + } + } + + void hook::post_hook_impl() + { + delete p_acquire; + p_acquire = nullptr; + } + + void hook::run_impl(std::shared_ptr /* loop */) + { + py::gil_scoped_acquire acquire; + py::module asyncio = py::module::import("asyncio"); + py::object loop = asyncio.attr("get_event_loop")(); + loop.attr("run_forever")(); + } +} diff --git a/src/xinterpreter.cpp b/src/xinterpreter.cpp index ff210b2d..e7402a38 100644 --- a/src/xinterpreter.cpp +++ b/src/xinterpreter.cpp @@ -58,6 +58,7 @@ namespace xpyt { if (m_release_gil_at_startup) { + std::cout << "Releasing GIL at startup." << std::endl; // The GIL is not held by default by the interpreter, so every time we need to execute Python code we // will need to acquire the GIL m_release_gil = gil_scoped_release_ptr(new py::gil_scoped_release()); @@ -114,6 +115,10 @@ namespace xpyt py::module context_module = get_request_context_module(); } + + + + void interpreter::execute_request_impl(send_reply_callback cb, int /*execution_count*/, const std::string& code, @@ -134,9 +139,52 @@ namespace xpyt std::string evalue; std::vector traceback; + + + + + auto when_done_callback_lambda = [this, cb, config, user_expressions]() { + py::gil_scoped_acquire acquire; + // Placeholder for any actions to perform when execution is done + + + + // Get payload + nl::json payload = this->m_ipython_shell.attr("payload_manager").attr("read_payload")(); + this->m_ipython_shell.attr("payload_manager").attr("clear_payload")(); + + // if(exception_occurred) + // { + // cb(xeus::create_error_reply(ename, evalue, traceback)); + // return; + // } + + if (this->m_ipython_shell.attr("last_error").is_none()) + { + nl::json user_exprs = this->m_ipython_shell.attr("user_expressions")(user_expressions); + cb(xeus::create_successful_reply(payload, user_exprs)); + } + else + { + py::list pyerror = this->m_ipython_shell.attr("last_error"); + + xerror error = extract_error(pyerror); + + if (!config.silent) + { + publish_execution_error(error.m_ename, error.m_evalue, error.m_traceback); + } + + cb(xeus::create_error_reply(error.m_ename, error.m_evalue, error.m_traceback)); + } + }; + std::function when_done_callback = when_done_callback_lambda; + + + try { - m_ipython_shell.attr("run_cell")(code, "store_history"_a=config.store_history, "silent"_a=config.silent); + m_ipython_shell.attr("run_cell_async")(code, when_done_callback, "store_history"_a=config.store_history, "silent"_a=config.silent); } catch(std::runtime_error& e) { @@ -171,34 +219,12 @@ namespace xpyt exception_occurred = true; } - // Get payload - nl::json payload = m_ipython_shell.attr("payload_manager").attr("read_payload")(); - m_ipython_shell.attr("payload_manager").attr("clear_payload")(); - if(exception_occurred) - { - cb(xeus::create_error_reply(ename, evalue, traceback)); - return; - } - if (m_ipython_shell.attr("last_error").is_none()) - { - nl::json user_exprs = m_ipython_shell.attr("user_expressions")(user_expressions); - cb(xeus::create_successful_reply(payload, user_exprs)); - } - else - { - py::list pyerror = m_ipython_shell.attr("last_error"); - xerror error = extract_error(pyerror); - if (!config.silent) - { - publish_execution_error(error.m_ename, error.m_evalue, error.m_traceback); - } - cb(xeus::create_error_reply(error.m_ename, error.m_evalue, error.m_traceback)); - } + } nl::json interpreter::complete_request_impl( @@ -266,8 +292,7 @@ namespace xpyt } nl::json interpreter::kernel_info_request_impl() - { - + { /* The jupyter-console banner for xeus-python is the following: __ _____ _ _ ___ \ \/ / _ \ | | / __| @@ -300,21 +325,39 @@ namespace xpyt {"url", "https://xeus-python.readthedocs.io"} }); - return xeus::create_info_reply( - "5.3", // protocol_version - "xeus-python", // implementation - XPYT_VERSION, // implementation_version - "python", // language_name - PY_VERSION, // language_version - "text/x-python", // language_mimetype - ".py", // language_file_extension - "ipython" + std::to_string(PY_MAJOR_VERSION), // pygments_lexer - R"({"name": "ipython", "version": )" + std::to_string(PY_MAJOR_VERSION) + "}", // language_codemirror_mode - "python", // language_nbconvert_exporter - banner, // banner - (PY_MAJOR_VERSION != 3) || (PY_MINOR_VERSION != 13), // debugger - help_links // help_links - ); + // return xeus::create_info_reply( + // "5.3", // protocol_version + // "xeus-python", // implementation + // XPYT_VERSION, // implementation_version + // "python", // language_name + // PY_VERSION, // language_version + // "text/x-python", // language_mimetype + // ".py", // language_file_extension + // "ipython" + std::to_string(PY_MAJOR_VERSION), // pygments_lexer + // R"({"name": "ipython", "version": )" + std::to_string(PY_MAJOR_VERSION) + "}", // language_codemirror_mode + // "python", // language_nbconvert_exporter + // banner, // banner + // (PY_MAJOR_VERSION != 3) || (PY_MINOR_VERSION != 13), // debugger + // help_links // help_links + // ); + + nl::json kernel_res; + kernel_res["status"] = "ok"; + kernel_res["protocol_version"] = "5.3"; + kernel_res["implementation"] = "xeus-python"; + kernel_res["implementation_version"] = XPYT_VERSION; + kernel_res["language_info"]["name"] = "python"; + kernel_res["language_info"]["version"] = PY_VERSION; + kernel_res["language_info"]["mimetype"] = "text/x-python"; + kernel_res["language_info"]["file_extension"] = ".py"; + kernel_res["language_info"]["pygments_lexer"] = "ipython" + std::to_string(PY_MAJOR_VERSION); + //kernel_res["language_info"]["codemirror_mode"] = R"({"name": "ipython", "version": )" + std::to_string(PY_MAJOR_VERSION) + "}"; + kernel_res["language_info"]["nbconvert_exporter"] = "python"; + kernel_res["banner"] = banner; + kernel_res["debugger"] = false; + kernel_res["help_links"] = help_links; + return kernel_res; + } void interpreter::shutdown_request_impl()