diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6f201f03..218247b1 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -12,6 +12,8 @@ jobs: runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v2 + - name: Install dependencies + run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev - name: Build run: cd src && make -f Makefile.linux-x86.mk - name: Install external tools @@ -32,6 +34,8 @@ jobs: runs-on: ubuntu-24.04-arm steps: - uses: actions/checkout@v2 + - name: Install dependencies + run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev - name: Build run: cd src && make -f Makefile.linux-arm64.mk - name: Test @@ -57,8 +61,13 @@ jobs: steps: - uses: actions/checkout@v2 - uses: ilammy/msvc-dev-cmd@v1 + - name: Install dependencies + run: | + vcpkg install curl:x64-windows - name: Build run: cd src && nmake.exe /F Makefile.windows.mk + env: + VCPKG_ROOT: C:\vcpkg - name: Test run: ./loda.exe test build-windows-arm64: @@ -68,14 +77,22 @@ jobs: - uses: ilammy/msvc-dev-cmd@v1 with: arch: amd64_arm64 + - name: Install dependencies + run: | + vcpkg install curl:arm64-windows - name: Build run: cd src && nmake.exe /F Makefile.windows.mk + env: + VCPKG_ROOT: C:\vcpkg + VCPKG_ARCH: arm64-windows code-coverage: runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v2 + - name: Install dependencies + run: sudo apt-get update && sudo apt-get install -y lcov libcurl4-openssl-dev - name: Generate - run: sudo apt install lcov && cd src && make -f Makefile.linux-x86.mk coverage + run: cd src && make -f Makefile.linux-x86.mk coverage - name: Upload uses: actions/upload-artifact@v4 with: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b14e7ec3..b472b229 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -44,6 +44,8 @@ jobs: runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v2 + - name: Install dependencies + run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev - name: Build run: cd src && make -f Makefile.linux-x86.mk loda LODA_PLATFORM=linux-x86 LODA_VERSION=${{ needs.create-release.outputs.version }} - name: Upload @@ -60,6 +62,8 @@ jobs: runs-on: ubuntu-24.04-arm steps: - uses: actions/checkout@v2 + - name: Install dependencies + run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev - name: Build run: cd src && make -f Makefile.linux-arm64.mk loda LODA_PLATFORM=linux-arm64 LODA_VERSION=${{ needs.create-release.outputs.version }} - name: Upload @@ -109,8 +113,13 @@ jobs: steps: - uses: actions/checkout@v2 - uses: ilammy/msvc-dev-cmd@v1 + - name: Install dependencies + run: | + vcpkg install curl:x64-windows - name: Build run: cd src && nmake.exe /F Makefile.windows.mk loda LODA_PLATFORM=windows-x86 LODA_VERSION=${{ needs.create-release.outputs.version }} + env: + VCPKG_ROOT: C:\vcpkg - name: Upload uses: actions/upload-release-asset@v1 with: @@ -128,8 +137,14 @@ jobs: - uses: ilammy/msvc-dev-cmd@v1 with: arch: amd64_arm64 + - name: Install dependencies + run: | + vcpkg install curl:arm64-windows - name: Build run: cd src && nmake.exe /F Makefile.windows.mk loda LODA_PLATFORM=windows-arm64 LODA_VERSION=${{ needs.create-release.outputs.version }} + env: + VCPKG_ROOT: C:\vcpkg + VCPKG_ARCH: arm64-windows - name: Upload uses: actions/upload-release-asset@v1 with: diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b64b496..748bf722 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ * Use `v2/submissions/` API endpoint to fetch programs * Handle program submissions with `delete` mode +* Integrate libcurl for HTTP requests, replacing external curl/wget tools ## v25.11.9 diff --git a/src/Makefile b/src/Makefile index 364a4d25..4be09fe8 100644 --- a/src/Makefile +++ b/src/Makefile @@ -1,5 +1,8 @@ CXXFLAGS += -I. -g -Wall -Werror -fmessage-length=0 -std=c++17 +# Default curl libs for dynamic linking (can be overridden by platform-specific makefiles) +CURL_LIBS ?= -lcurl + ifdef LODA_VERSION CXXFLAGS += -DLODA_VERSION=$(LODA_VERSION) endif @@ -21,14 +24,14 @@ OBJS = base/uid.o \ loda: CXXFLAGS += -O2 loda: $(OBJS) - $(CXX) $(LDFLAGS) -o loda $(OBJS) + $(CXX) $(LDFLAGS) -o loda $(OBJS) $(CURL_LIBS) [ -L ../loda ] || ( cd .. && ln -s src/loda loda ) du -sh loda coverage: CXXFLAGS += -O0 -fprofile-arcs -ftest-coverage coverage: LDFLAGS += --coverage coverage: $(OBJS) - $(CXX) $(LDFLAGS) -o loda $(OBJS) + $(CXX) $(LDFLAGS) -o loda $(OBJS) $(CURL_LIBS) [ -L ../loda ] || ( cd .. && ln -s src/loda loda ) cd .. && ./loda test && cd src gcov $(OBJS:.o=.cpp) diff --git a/src/Makefile.linux-arm64.mk b/src/Makefile.linux-arm64.mk index 920d3972..5aae0303 100644 --- a/src/Makefile.linux-arm64.mk +++ b/src/Makefile.linux-arm64.mk @@ -2,7 +2,8 @@ # see https://stackoverflow.com/questions/35116327/when-g-static-link-pthread-cause-segmentation-fault-why CXX = aarch64-linux-gnu-g++ CXXFLAGS += -pthread -flto=auto -LDFLAGS = -static -static-libstdc++ -static-libgcc -lrt -pthread -Wl,--whole-archive -lpthread -Wl,--no-whole-archive -s -flto=auto +LDFLAGS = -static-libstdc++ -static-libgcc -pthread -s -flto=auto +CURL_LIBS = -lcurl # END PLATFORM CONFIG FOR LINUX ARM64 include Makefile diff --git a/src/Makefile.linux-x86.mk b/src/Makefile.linux-x86.mk index e03f414e..0ed81abc 100644 --- a/src/Makefile.linux-x86.mk +++ b/src/Makefile.linux-x86.mk @@ -2,7 +2,8 @@ # see https://stackoverflow.com/questions/35116327/when-g-static-link-pthread-cause-segmentation-fault-why CXX = x86_64-linux-gnu-g++ CXXFLAGS += -pthread -flto=auto -LDFLAGS = -static -static-libstdc++ -static-libgcc -lrt -pthread -Wl,--whole-archive -lpthread -Wl,--no-whole-archive -s -flto=auto +LDFLAGS = -static-libstdc++ -static-libgcc -pthread -s -flto=auto +CURL_LIBS = -lcurl # END PLATFORM CONFIG FOR LINUX X86 include Makefile diff --git a/src/Makefile.macos-arm64.mk b/src/Makefile.macos-arm64.mk index dec52581..e25ac6a4 100644 --- a/src/Makefile.macos-arm64.mk +++ b/src/Makefile.macos-arm64.mk @@ -2,6 +2,7 @@ CXX = clang++ CXXFLAGS = -target arm64-apple-macos14 -flto=thin LDFLAGS = -target arm64-apple-macos14 -flto=thin +CURL_LIBS = -lcurl # END PLATFORM CONFIG FOR MAC OS ARM64 include Makefile diff --git a/src/Makefile.macos-x86.mk b/src/Makefile.macos-x86.mk index d2944780..6a7d6e64 100644 --- a/src/Makefile.macos-x86.mk +++ b/src/Makefile.macos-x86.mk @@ -2,6 +2,7 @@ CXX = clang++ CXXFLAGS = -target x86_64-apple-macos14 -flto=thin LDFLAGS = -target x86_64-apple-macos14 -flto=thin +CURL_LIBS = -lcurl # END PLATFORM CONFIG FOR MAC OS X86 include Makefile diff --git a/src/Makefile.windows.mk b/src/Makefile.windows.mk index 20a6355c..97525ef7 100644 --- a/src/Makefile.windows.mk +++ b/src/Makefile.windows.mk @@ -1,6 +1,18 @@ CXXFLAGS = /I. /std:c++17 /O2 /GL LDFLAGS = /link /LTCG +# Check if VCPKG_ROOT is set (for vcpkg integration) +!IFDEF VCPKG_ROOT +# Default to x64-windows if VCPKG_ARCH is not specified +!IFNDEF VCPKG_ARCH +VCPKG_ARCH = x64-windows +!ENDIF +CXXFLAGS = $(CXXFLAGS) /I$(VCPKG_ROOT)\installed\$(VCPKG_ARCH)\include +LDFLAGS = $(LDFLAGS) /LIBPATH:$(VCPKG_ROOT)\installed\$(VCPKG_ARCH)\lib +!ENDIF + +CURL_LIBS = libcurl.lib + !IFDEF LODA_VERSION CXXFLAGS = $(CXXFLAGS) /DLODA_VERSION=$(LODA_VERSION) !ELSE @@ -23,7 +35,7 @@ SRCS = base/uid.cpp \ sys/csv.cpp sys/file.cpp sys/git.cpp sys/jute.cpp sys/log.cpp sys/metrics.cpp sys/process.cpp sys/setup.cpp sys/util.cpp sys/web_client.cpp loda: $(SRCS) - cl /EHsc /Feloda.exe $(CXXFLAGS) $(SRCS) $(LDFLAGS) + cl /EHsc /Feloda.exe $(CXXFLAGS) $(SRCS) $(LDFLAGS) $(CURL_LIBS) copy loda.exe .. clean: diff --git a/src/README.md b/src/README.md index 0c5642b0..341812a0 100644 --- a/src/README.md +++ b/src/README.md @@ -17,7 +17,13 @@ The C++ source code contain is organized into the following modules: ## Building -LODA supports Linux, macOS, and Windows. You need a standard C++ compiler and `make` (or `nmake` on Windows). No external libraries are required, but the `curl` and `gzip` command-line tools must be available. +LODA supports Linux, macOS, and Windows. You need a standard C++ compiler and `make` (or `nmake` on Windows). The `libcurl` development library and the `gzip` command-line tool must be available at build time. + +### Build Dependencies + +- **C++ compiler**: g++, clang++, or MSVC (C++17 support required) +- **libcurl**: Development headers and library (e.g., `libcurl4-openssl-dev` on Debian/Ubuntu) +- **gzip**: Command-line tool for compression To build from source, switch to the `src/` folder and run the appropriate command for your platform: diff --git a/src/sys/web_client.cpp b/src/sys/web_client.cpp index 6533c368..172f6c1e 100644 --- a/src/sys/web_client.cpp +++ b/src/sys/web_client.cpp @@ -1,67 +1,106 @@ #include "sys/web_client.hpp" +#include +#include + +#include + +// Undefine Windows macros that conflict with our code +#ifdef ERROR +#undef ERROR +#endif + #include "sys/file.hpp" #include "sys/log.hpp" -enum WebClientType { WC_UNKNOWN = 0, WC_CURL = 1, WC_WGET = 2 }; - -int64_t WebClient::WEB_CLIENT_TYPE = WC_UNKNOWN; +int64_t WebClient::WEB_CLIENT_TYPE = 0; void WebClient::initWebClient() { - if (WEB_CLIENT_TYPE == WC_UNKNOWN) { - const std::string curl_cmd = "curl --version " + getNullRedirect(); - const std::string wget_cmd = "wget --version " + getNullRedirect(); - if (system(curl_cmd.c_str()) == 0) { - WEB_CLIENT_TYPE = WC_CURL; - } else if (system(wget_cmd.c_str()) == 0) { - WEB_CLIENT_TYPE = WC_WGET; - } else { - Log::get().error("No web client found. Please install curl or wget.", - true); - } + if (WEB_CLIENT_TYPE == 0) { + curl_global_init(CURL_GLOBAL_DEFAULT); + WEB_CLIENT_TYPE = 1; + // Note: curl_global_cleanup() is intentionally not called as it's not + // thread-safe and the library remains usable for the lifetime of the + // program. This is a common pattern for libcurl usage. } } +namespace { + +// Callback function to write data to a file +size_t writeFileCallback(void* contents, size_t size, size_t nmemb, + void* userp) { + std::ofstream* file = static_cast(userp); + size_t total_size = size * nmemb; + file->write(static_cast(contents), total_size); + return total_size; +} + +// Callback function to write data to a string +size_t writeStringCallback(void* contents, size_t size, size_t nmemb, + void* userp) { + std::string* str = static_cast(userp); + size_t total_size = size * nmemb; + str->append(static_cast(contents), total_size); + return total_size; +} + +// Callback function to read data from a file +size_t readFileCallback(void* ptr, size_t size, size_t nmemb, void* userp) { + std::ifstream* file = static_cast(userp); + size_t total_size = size * nmemb; + file->read(static_cast(ptr), total_size); + return file->gcount(); +} + +} // namespace + bool WebClient::get(const std::string& url, const std::string& local_path, bool silent, bool fail_on_error, bool insecure) { - std::string url_processed; - size_t len = url.length(); - for (size_t pos = 0; pos < len; pos++) { - if (url[pos] == '&' || url[pos] == '|') { -#ifdef _WIN64 - url_processed.push_back('^'); -#else - url_processed.push_back('\\'); -#endif + initWebClient(); + + CURL* curl = curl_easy_init(); + if (!curl) { + if (fail_on_error) { + Log::get().error("Failed to initialize libcurl", true); } - url_processed.push_back(url[pos]); + return false; } - initWebClient(); - std::string cmd; - switch (WEB_CLIENT_TYPE) { - case WC_CURL: - cmd = "curl"; - if (insecure) { - cmd += " --insecure"; - } - cmd += " -fsSLo \"" + local_path + "\" " + url_processed; - break; - case WC_WGET: - cmd = "wget -q --no-use-server-timestamps -O \"" + local_path + "\" " + - url_processed; - break; - default: - Log::get().error("Unsupported web client for GET request", true); - } - if (system(cmd.c_str()) != 0) { + + std::ofstream file(local_path, std::ios::binary); + if (!file.is_open()) { + curl_easy_cleanup(curl); + if (fail_on_error) { + Log::get().error("Failed to open file for writing: " + local_path, true); + } + return false; + } + + curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writeFileCallback); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &file); + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); + curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1L); + if (insecure) { + curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L); + curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0L); + } + + CURLcode res = curl_easy_perform(curl); + file.close(); + + if (res != CURLE_OK) { std::remove(local_path.c_str()); + curl_easy_cleanup(curl); if (fail_on_error) { - Log::get().info(cmd); - Log::get().error("Error fetching " + url, true); - } else { - return false; + Log::get().error("Error fetching " + url + ": " + + std::string(curl_easy_strerror(res)), + true); } + return false; } + + curl_easy_cleanup(curl); if (!silent) { Log::get().info("Fetched " + url); } @@ -73,53 +112,94 @@ bool WebClient::postFile(const std::string& url, const std::string& file_path, const std::vector& headers, bool enable_debug) { initWebClient(); - std::string cmd; - switch (WEB_CLIENT_TYPE) { - case WC_CURL: { - cmd = "curl -fsSL"; - if (!auth.empty()) { - cmd += " -u " + auth; - } - if (file_path.empty()) { - cmd += " -X POST"; - } else { - cmd += " --data-binary \"@" + file_path + "\""; - } - for (const auto& header : headers) { - cmd += " -H \"" + header + "\""; - } - break; + + CURL* curl = curl_easy_init(); + if (!curl) { + if (enable_debug) { + Log::get().error("Failed to initialize libcurl", false); } - case WC_WGET: { - cmd = "wget -q"; - if (!auth.empty()) { - auto colon = auth.find(':'); - cmd += " --user '" + auth.substr(0, colon) + "' --password '" + - auth.substr(colon + 1) + "'"; - } - if (file_path.empty()) { - cmd += " --post-data \"\""; - } else { - cmd += " --post-file \"" + file_path + "\""; + return false; + } + + curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); + + if (!auth.empty()) { + curl_easy_setopt(curl, CURLOPT_USERPWD, auth.c_str()); + } + + struct curl_slist* header_list = nullptr; + for (const auto& header : headers) { + header_list = curl_slist_append(header_list, header.c_str()); + } + if (header_list) { + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, header_list); + } + + std::ifstream file; + if (!file_path.empty()) { + file.open(file_path, std::ios::binary); + if (!file.is_open()) { + curl_slist_free_all(header_list); + curl_easy_cleanup(curl); + if (enable_debug) { + Log::get().error("Failed to open file: " + file_path, false); } - for (const auto& header : headers) { - cmd += " --header \"" + header + "\""; + return false; + } + + file.seekg(0, std::ios::end); + std::streampos pos = file.tellg(); + if (pos == std::streampos(-1)) { + file.close(); + curl_slist_free_all(header_list); + curl_easy_cleanup(curl); + if (enable_debug) { + Log::get().error("Failed to determine file size: " + file_path, false); } - break; + return false; } - default: - Log::get().error("Unsupported web client for POST request", true); + size_t file_size = static_cast(pos); + file.seekg(0, std::ios::beg); + + curl_easy_setopt(curl, CURLOPT_POST, 1L); + curl_easy_setopt(curl, CURLOPT_READFUNCTION, readFileCallback); + curl_easy_setopt(curl, CURLOPT_READDATA, &file); + curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, static_cast(file_size)); + } else { + curl_easy_setopt(curl, CURLOPT_POST, 1L); + curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, 0L); } - cmd += " " + url; - const std::string msg = "Executing command: " + cmd; + + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); + curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1L); + + std::string response; + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writeStringCallback); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &response); + if (enable_debug) { - Log::get().info(msg); - } else { - Log::get().debug(msg); - cmd += " " + getNullRedirect(); + curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L); + Log::get().info("Posting to URL: " + url); } - auto exit_code = system(cmd.c_str()); - return (exit_code == 0); + + CURLcode res = curl_easy_perform(curl); + + if (file.is_open()) { + file.close(); + } + curl_slist_free_all(header_list); + curl_easy_cleanup(curl); + + if (res != CURLE_OK) { + if (enable_debug) { + Log::get().error("Error posting to " + url + ": " + + std::string(curl_easy_strerror(res)), + false); + } + return false; + } + + return true; } jute::jValue WebClient::getJson(const std::string& url) {