libuv based event loop / server

.github/workflows/main.yml

@@ -22,7 +22,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-22.04, ubuntu-24.04, macos-13, macos-14]
+        os: [ubuntu-22.04, ubuntu-24.04, macos-14]
         build_type: [static_build, shared_build]
 
     steps:

CMakeLists.txt

@@ -105,8 +105,14 @@ else ()
     if (NOT TARGET xeus-zmq AND NOT TARGET xeus-zmq-static)
         find_package(xeus-zmq ${xeus-zmq_REQUIRED_VERSION} REQUIRED)
     endif ()
+
+    find_package(xeus-uv REQUIRED)
+    find_library(UVW_LIBRARY uvw)
+    find_path(UVW_INCLUDE_DIR uvw.hpp)
 endif()
 
+
+
 # Configuration
 # =============
 
@@ -163,6 +169,7 @@ set(XEUS_PYTHON_SRC
     src/xdebugpy_client.cpp
     src/xdisplay.cpp
     src/xdisplay.hpp
+    src/xhook.cpp
     src/xinput.cpp
     src/xinput.hpp
     src/xinspect.cpp
@@ -188,6 +195,7 @@ set(XEUS_PYTHON_HEADERS
     include/xeus-python/xinterpreter_raw.hpp
     include/xeus-python/xtraceback.hpp
     include/xeus-python/xutils.hpp
+    include/xeus-python/xhook.hpp
 )
 
 set(XPYTHON_SRC
@@ -317,7 +325,10 @@ macro(xpyt_create_target target_name src headers linkage output_name)
         set(XPYT_XEUS_TARGET xeus-zmq-static)
     endif ()
 
-    target_link_libraries(${target_name} PUBLIC ${XPYT_XEUS_TARGET} PRIVATE pybind11::pybind11 pybind11_json)
+    target_link_libraries(${target_name}
+        PUBLIC ${XPYT_XEUS_TARGET}  ${UVW_LIBRARY}
+        PRIVATE pybind11::pybind11 pybind11_json
+    )
     if (WIN32 OR CYGWIN)
         target_link_libraries(${target_name} PRIVATE ${PYTHON_LIBRARIES})
     elseif (APPLE)
@@ -371,7 +382,7 @@ endif ()
 
 if (XPYT_BUILD_XPYTHON_EXECUTABLE)
     add_executable(xpython ${XPYTHON_SRC})
-    target_link_libraries(xpython PRIVATE pybind11::embed)
+    target_link_libraries(xpython PRIVATE xeus-uv pybind11::embed )
 
     xpyt_set_common_options(xpython)
     xpyt_set_kernel_options(xpython)

environment-dev.yml

@@ -17,6 +17,7 @@ dependencies:
   - python <3.13
   - xeus-python-shell>=0.6.3,<0.7
   - debugpy>=1.6.5
+  - libuvw
   - ipython
   # Test dependencies
   - pytest

include/xeus-python/xhook.hpp

@@ -0,0 +1,52 @@
+/***************************************************************************
+* Copyright (c) 2018, Martin Renou, Johan Mabille, Sylvain Corlay, and     *
+* Wolf Vollprecht                                                          *
+* Copyright (c) 2018, QuantStack
+*                                                                          *
+* Distributed under the terms of the BSD 3-Clause License.                 *
+*                                                                          *
+* The full license is in the file LICENSE, distributed with this software. *
+****************************************************************************/
+
+#ifndef XPYT_HOOK_HPP
+#define XPYT_HOOK_HPP
+
+// pybind11 code internally forces hidden visibility on all internal code, but
+// if non-hidden (and thus exported) code attempts to include a pybind type
+// this warning occurs:
+// 'xpyt::hook' declared with greater visibility than the type of its
+// field 'xpyt::hook::p_acquire' [-Wattributes]
+#ifdef __GNUC__
+    #pragma GCC diagnostic ignored "-Wattributes"
+#endif
+
+#include "xeus-python/xeus_python_config.hpp"
+#include "xeus-uv/xhook_base.hpp"
+
+#include "pybind11/embed.h"
+#include "pybind11/pybind11.h"
+
+namespace py = pybind11;
+
+namespace xpyt
+{
+    XEUS_PYTHON_API
+    class hook : public xeus::xhook_base
+    {
+    public:
+
+        hook() = default;
+        virtual ~hook();
+
+    private:
+
+        void pre_hook_impl() override;
+        void post_hook_impl() override;
+        void run_impl(std::shared_ptr<uvw::loop> loop) override;
+
+        py::gil_scoped_acquire* p_acquire{ nullptr};
+    };
+
+}
+
+#endif

include/xeus-python/xinterpreter.hpp

@@ -93,7 +93,7 @@ namespace xpyt
         // is started, m_release_gil_at_startup has to be set to false to prevent
         // releasing it again in configure_impl().
         //
-        bool m_release_gil_at_startup = true;
+        bool m_release_gil_at_startup = false;
         gil_scoped_release_ptr m_release_gil = nullptr;
 
         bool m_redirect_output_enabled;

src/main.cpp

@@ -21,14 +21,24 @@
 #include <unistd.h>
 #endif
 
+#ifndef UVW_AS_LIB
+#define UVW_AS_LIB
+#include <uvw.hpp>
+#endif
+
 #include "xeus/xkernel.hpp"
 #include "xeus/xkernel_configuration.hpp"
 #include "xeus/xinterpreter.hpp"
 #include "xeus/xhelper.hpp"
 
-#include "xeus-zmq/xserver_zmq_split.hpp"
+#include "xeus-zmq/xserver_zmq.hpp"
 #include "xeus-zmq/xzmq_context.hpp"
 
+
+#include "xeus-uv/xserver_uv.hpp"
+#include "xeus-uv/xhook_base.hpp"
+
+
 #include "pybind11/embed.h"
 #include "pybind11/pybind11.h"
 
@@ -38,10 +48,10 @@
 #include "xeus-python/xpaths.hpp"
 #include "xeus-python/xeus_python_config.hpp"
 #include "xeus-python/xutils.hpp"
+#include "xeus-python/xhook.hpp"
 
 namespace py = pybind11;
 
-
 int main(int argc, char* argv[])
 {
     if (xeus::should_print_version(argc, argv))
@@ -88,11 +98,35 @@ int main(int argc, char* argv[])
     config.home = const_cast<wchar_t*>(wstr.c_str());
     xpyt::print_pythonhome();
 
-    // Implicitly pre-initialize Python
-    status = PyConfig_SetBytesArgv(&config, argc, argv);
-    if (PyStatus_Exception(status)) {
-        std::cerr << "Error:" << status.err_msg << std::endl;
+    // Instantiating the Python interpreter
+    py::scoped_interpreter guard{};
+
+    uv_loop_t* uv_loop_ptr{ nullptr };
+
+    {
+        py::gil_scoped_acquire acquire;
+
+        // Create a uvloop and get pointer to the loop
+        py::module asyncio = py::module::import("asyncio");
+        py::module uvloop = py::module::import("uvloop");
+        py::object loop = uvloop.attr("new_event_loop")();
+        asyncio.attr("set_event_loop")(loop);
+        py::object py_loop_ptr = uvloop.attr("loop").attr("libuv_get_loop_t_ptr")(loop);
+
+        void* raw_ptr = PyCapsule_GetPointer(py_loop_ptr.ptr(), nullptr);
+        if (!raw_ptr)
+        {
+            throw std::runtime_error("Failed to get uvloop pointer");
+        }
+
+        uv_loop_ptr = static_cast<uv_loop_t*>(raw_ptr);
+    }
+
+    if (!uv_loop_ptr)
+    {
+        throw std::runtime_error("Failed to get libuv loop pointer");
     }
+    auto loop_ptr = uvw::loop::create(uv_loop_ptr);
 
     // Setting argv
     wchar_t** argw = new wchar_t*[size_t(argc)];
@@ -113,8 +147,7 @@ int main(int argc, char* argv[])
     }
     delete[] argw;
 
-    // Instantiating the Python interpreter
-    py::scoped_interpreter guard;
+
 
     std::unique_ptr<xeus::xcontext> context = xeus::make_zmq_context();
 
@@ -146,6 +179,16 @@ int main(int argc, char* argv[])
     nl::json debugger_config;
     debugger_config["python"] = executable;
 
+    auto py_hook = std::make_unique<xpyt::hook>();
+
+
+
+    auto make_xserver = [&](xeus::xcontext& context,
+                                   const xeus::xconfiguration& config,
+                                   nl::json::error_handler_t eh) {
+        return xeus::make_xserver_uv(context, config, eh, loop_ptr, std::move(py_hook));
+    };
+
     if (!connection_filename.empty())
     {
         xeus::xconfiguration config = xeus::load_configuration(connection_filename);
@@ -154,12 +197,12 @@ int main(int argc, char* argv[])
                              xeus::get_user_name(),
                              std::move(context),
                              std::move(interpreter),
-                             xeus::make_xserver_shell_main,
+                             make_xserver,
                              std::move(hist),
                              xeus::make_console_logger(xeus::xlogger::msg_type,
-                                                       xeus::make_file_logger(xeus::xlogger::content, "xeus.log")),
-                             xpyt::make_python_debugger,
-                             debugger_config);
+                                                       xeus::make_file_logger(xeus::xlogger::content, "xeus.log")));
+                            //  xpyt::make_python_debugger,
+                            //  debugger_config);
 
         std::clog <<
             "Starting xeus-python kernel...\n\n"
@@ -175,7 +218,7 @@ int main(int argc, char* argv[])
         xeus::xkernel kernel(xeus::get_user_name(),
                              std::move(context),
                              std::move(interpreter),
-                             xeus::make_xserver_shell_main,
+                             make_xserver,
                              std::move(hist),
                              nullptr,
                              xpyt::make_python_debugger,

src/xdebugger.cpp

@@ -26,6 +26,7 @@
 #include "pybind11/stl.h"
 
 #include "xeus/xinterpreter.hpp"
+#include "xeus/xeus_context.hpp"
 #include "xeus/xsystem.hpp"
 #include "xeus-zmq/xmiddleware.hpp"
 

src/xdebugpy_client.cpp

@@ -10,6 +10,7 @@
 
 #include "nlohmann/json.hpp"
 #include "xeus/xmessage.hpp"
+#include "xeus/xeus_context.hpp"
 #include "xdebugpy_client.hpp"
 
 #include <thread>

src/xdebugpy_client.hpp

@@ -12,6 +12,7 @@
 #define XPYT_DEBUGPY_CLIENT_HPP
 
 #include "xeus-zmq/xdap_tcp_client.hpp"
+#include "xeus/xeus_context.hpp"
 
 namespace xpyt
 {

src/xhook.cpp

@@ -0,0 +1,51 @@
+/***************************************************************************
+* Copyright (c) 2018, Martin Renou, Johan Mabille, Sylvain Corlay, and     *
+* Wolf Vollprecht                                                          *
+* Copyright (c) 2018, QuantStack
+*                                                                          *
+* Distributed under the terms of the BSD 3-Clause License.                 *
+*                                                                          *
+* The full license is in the file LICENSE, distributed with this software. *
+****************************************************************************/
+
+#ifndef UVW_AS_LIB
+#define UVW_AS_LIB
+#include <uvw.hpp>
+#endif
+
+#include "xeus-python/xhook.hpp"
+
+#include "pybind11/embed.h"
+#include "pybind11/pybind11.h"
+
+namespace py = pybind11;
+
+namespace xpyt
+{
+    hook::~hook()
+    {
+        delete p_acquire;
+    }
+
+    void hook::pre_hook_impl()
+    {
+        if (!p_acquire)
+        {
+            p_acquire = new py::gil_scoped_acquire();
+        }
+    }
+
+    void hook::post_hook_impl()
+    {
+        delete p_acquire;
+        p_acquire = nullptr;
+    }
+
+    void hook::run_impl(std::shared_ptr<uvw::loop> /* loop */)
+    {
+        py::gil_scoped_acquire acquire;
+        py::module asyncio = py::module::import("asyncio");
+        py::object loop = asyncio.attr("get_event_loop")();
+        loop.attr("run_forever")();
+    }
+}