Merge bitcoin/bitcoin#32604: log: Mitigate disk filling attacks by rate limiting LogPrintf, LogInfo, LogWarning, LogError

4c772cb doc: add release notes for new rate limiting logging behavior (Eugene Siegel)
d541409 log: Add rate limiting to LogPrintf, LogInfo, LogWarning, LogError, LogPrintLevel (Eugene Siegel)
a6a35cc log: use std::source_location in place of __func__, __FILE__, __LINE__ (Eugene Siegel)
afb9e39 log: introduce LogRateLimiter, LogLimitStats, Status (Eugene Siegel)
df7972a test: Mark ~DebugLogHelper as noexcept(false) (Eugene Siegel)

Pull request description:

  This revives the work done by dergoegge in bitcoin/bitcoin#21603. The approach is similar — this PR uses `std::source_location` under the hood now that we can use c++20 features. It also resets the rate limiting statistics via the `CScheduler`. The logging functions have also changed slightly since that PR was opened, so work has been done to preserve the intent of the original rate limiting change. I have tried to give commit attribution where possible.

  **Approach:**
  Each source code location is given an hourly logging quota of 1MiB of logging per hour. Logging is only dropped from source locations that exceed the quota.
  - Only logging to disk is rate limited. Logging to console is not rate limited.
  - Printing with the category argument is not rate limited.
  - `UpdateTip: new best=[…]` is logged without rate limiting. High log volume is expected for that source location during IBD.
  - When logging is restarted a tally of how many bytes were dropped is printed.
  - All logs will be prefixed with [*] if there is at least one source location that is currently being suppressed.

  I've repurposed the old logging rpc mentioned in #21603 in another branch for testing [here](https://github.com/Crypt-iQ/bitcoin/tree/log_ratelimiting_05192025_rpc). This can be used to log from source locations and test out the new changes in logging behavior. Note that the `setmocktime` RPC needs to be used to set the mock time past the current clock time to reset the logging messages.

  Example usage:
  ```
  bitcoin-cli -regtest excessivelog 1 1048500 # log just under 1MiB
  bitcoin-cli -regtest excessivelog 1 100 # this should get the total amount logged above 1MiB
                                          # and the rate limiting logic should kick in
  bitcoin-cli -regtest excessivelog 2 1048500
  bitcoin-cli -regtest excessivelog 2 100 # trigger rate limiting from another location
  bitcoin-cli -regtest mockscheduler 3600 # fast-forward the scheduler
  bitcoin-cli -regtest excessivelog 1 100 # this should trigger logging to resume and will log the source locations that were reset
  ```

  Example output:
  ```
  2025-07-02T22:03:56Z [warning] Excessive logging detected from rpc/node.cpp:142 (RPCHelpMan excessivelog()): >1048576 bytes logged during the last time window of 3600s. Suppressing logging to disk from this source location until time window resets. Console logging unaffected. Last log entry.
  [*] 2025-07-02T22:03:56Z aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
  2025-07-02T22:04:58Z (mocktime: 2262-04-11T23:47:15Z) Restarting logging from rpc/node.cpp:142 (RPCHelpMan excessivelog()): 121 bytes were dropped during the last 3600s.
  2025-07-02T22:04:58Z (mocktime: 2262-04-11T23:47:15Z) Restarting logging from rpc/node.cpp:139 (RPCHelpMan excessivelog()): 121 bytes were dropped during the last 3600s.
  ```

ACKs for top commit:
  maflcko:
    re-ACK 4c772cb 🕚
  glozow:
    reACK 4c772cb
  stickies-v:
    re-ACK 4c772cb, no changes except release notes update

Tree-SHA512: d07087cd0f2b188100b51c9b8c3da376fa24ec3612a2a284bd83f650bba0ea409f9fa0acd5f3b10f45e664ef4fdf3abc97ed3da08098d2beb599cc83e3fc4504

Author: glozow

doc/release-notes-32604.md

@@ -0,0 +1,10 @@
+Logging
+-------
+Unconditional logging to disk is now rate limited by giving each source location
+a quota of 1MiB per hour. Unconditional logging is any logging with a log level
+higher than debug, that is `info`, `warning`, and `error`. All logs will be
+prefixed with `[*]` if there is at least one source location that is currently
+being suppressed. (#32604)
+
+When `-logsourcelocations` is enabled, the log output now contains the entire
+function signature instead of just the function name. (#32604)

src/init.cpp

@@ -1378,6 +1378,11 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info)
         }
     }, std::chrono::minutes{5});
 
+    LogInstance().SetRateLimiting(std::make_unique<BCLog::LogRateLimiter>(
+        [&scheduler](auto func, auto window) { scheduler.scheduleEvery(std::move(func), window); },
+        BCLog::RATELIMIT_MAX_BYTES,
+        1h));
+
     assert(!node.validation_signals);
     node.validation_signals = std::make_unique<ValidationSignals>(std::make_unique<SerialTaskRunner>(scheduler));
     auto& validation_signals = *node.validation_signals;

src/logging.cpp

@@ -12,8 +12,10 @@
 #include <util/time.h>
 
 #include <array>
+#include <cstring>
 #include <map>
 #include <optional>
+#include <utility>
 
 using util::Join;
 using util::RemovePrefixView;
@@ -73,12 +75,12 @@ bool BCLog::Logger::StartLogging()
     // dump buffered messages from before we opened the log
     m_buffering = false;
     if (m_buffer_lines_discarded > 0) {
-        LogPrintStr_(strprintf("Early logging buffer overflowed, %d log lines discarded.\n", m_buffer_lines_discarded), __func__, __FILE__, __LINE__, BCLog::ALL, Level::Info);
+        LogPrintStr_(strprintf("Early logging buffer overflowed, %d log lines discarded.\n", m_buffer_lines_discarded), std::source_location::current(), BCLog::ALL, Level::Info, /*should_ratelimit=*/false);
     }
     while (!m_msgs_before_open.empty()) {
         const auto& buflog = m_msgs_before_open.front();
         std::string s{buflog.str};
-        FormatLogStrInPlace(s, buflog.category, buflog.level, buflog.source_file, buflog.source_line, buflog.logging_function, buflog.threadname, buflog.now, buflog.mocktime);
+        FormatLogStrInPlace(s, buflog.category, buflog.level, buflog.source_loc, buflog.threadname, buflog.now, buflog.mocktime);
         m_msgs_before_open.pop_front();
 
         if (m_print_to_file) FileWriteStr(s, m_fileout);
@@ -364,17 +366,43 @@ std::string BCLog::Logger::GetLogPrefix(BCLog::LogFlags category, BCLog::Level l
 
 static size_t MemUsage(const BCLog::Logger::BufferedLog& buflog)
 {
-    return buflog.str.size() + buflog.logging_function.size() + buflog.source_file.size() + buflog.threadname.size() + memusage::MallocUsage(sizeof(memusage::list_node<BCLog::Logger::BufferedLog>));
+    return memusage::DynamicUsage(buflog.str) +
+           memusage::DynamicUsage(buflog.threadname) +
+           memusage::MallocUsage(sizeof(memusage::list_node<BCLog::Logger::BufferedLog>));
 }
 
-void BCLog::Logger::FormatLogStrInPlace(std::string& str, BCLog::LogFlags category, BCLog::Level level, std::string_view source_file, int source_line, std::string_view logging_function, std::string_view threadname, SystemClock::time_point now, std::chrono::seconds mocktime) const
+BCLog::LogRateLimiter::LogRateLimiter(
+    SchedulerFunction scheduler_func,
+    uint64_t max_bytes,
+    std::chrono::seconds reset_window) : m_max_bytes{max_bytes}, m_reset_window{reset_window}
+{
+    scheduler_func([this] { Reset(); }, reset_window);
+}
+
+BCLog::LogRateLimiter::Status BCLog::LogRateLimiter::Consume(
+    const std::source_location& source_loc,
+    const std::string& str)
+{
+    StdLockGuard scoped_lock(m_mutex);
+    auto& counter{m_source_locations.try_emplace(source_loc, m_max_bytes).first->second};
+    Status status{counter.GetDroppedBytes() > 0 ? Status::STILL_SUPPRESSED : Status::UNSUPPRESSED};
+
+    if (!counter.Consume(str.size()) && status == Status::UNSUPPRESSED) {
+        status = Status::NEWLY_SUPPRESSED;
+        m_suppression_active = true;
+    }
+
+    return status;
+}
+
+void BCLog::Logger::FormatLogStrInPlace(std::string& str, BCLog::LogFlags category, BCLog::Level level, const std::source_location& source_loc, std::string_view threadname, SystemClock::time_point now, std::chrono::seconds mocktime) const
 {
     if (!str.ends_with('\n')) str.push_back('\n');
 
     str.insert(0, GetLogPrefix(category, level));
 
     if (m_log_sourcelocations) {
-        str.insert(0, strprintf("[%s:%d] [%s] ", RemovePrefixView(source_file, "./"), source_line, logging_function));
+        str.insert(0, strprintf("[%s:%d] [%s] ", RemovePrefixView(source_loc.file_name(), "./"), source_loc.line(), source_loc.function_name()));
     }
 
     if (m_log_threadnames) {
@@ -384,28 +412,27 @@ void BCLog::Logger::FormatLogStrInPlace(std::string& str, BCLog::LogFlags catego
     str.insert(0, LogTimestampStr(now, mocktime));
 }
 
-void BCLog::Logger::LogPrintStr(std::string_view str, std::string_view logging_function, std::string_view source_file, int source_line, BCLog::LogFlags category, BCLog::Level level)
+void BCLog::Logger::LogPrintStr(std::string_view str, std::source_location&& source_loc, BCLog::LogFlags category, BCLog::Level level, bool should_ratelimit)
 {
     StdLockGuard scoped_lock(m_cs);
-    return LogPrintStr_(str, logging_function, source_file, source_line, category, level);
+    return LogPrintStr_(str, std::move(source_loc), category, level, should_ratelimit);
 }
 
-void BCLog::Logger::LogPrintStr_(std::string_view str, std::string_view logging_function, std::string_view source_file, int source_line, BCLog::LogFlags category, BCLog::Level level)
+// NOLINTNEXTLINE(misc-no-recursion)
+void BCLog::Logger::LogPrintStr_(std::string_view str, std::source_location&& source_loc, BCLog::LogFlags category, BCLog::Level level, bool should_ratelimit)
 {
     std::string str_prefixed = LogEscapeMessage(str);
 
     if (m_buffering) {
         {
             BufferedLog buf{
-                .now=SystemClock::now(),
-                .mocktime=GetMockTime(),
-                .str=str_prefixed,
-                .logging_function=std::string(logging_function),
-                .source_file=std::string(source_file),
-                .threadname=util::ThreadGetInternalName(),
-                .source_line=source_line,
-                .category=category,
-                .level=level,
+                .now = SystemClock::now(),
+                .mocktime = GetMockTime(),
+                .str = str_prefixed,
+                .threadname = util::ThreadGetInternalName(),
+                .source_loc = std::move(source_loc),
+                .category = category,
+                .level = level,
             };
             m_cur_buffer_memusage += MemUsage(buf);
             m_msgs_before_open.push_back(std::move(buf));
@@ -424,7 +451,29 @@ void BCLog::Logger::LogPrintStr_(std::string_view str, std::string_view logging_
         return;
     }
 
-    FormatLogStrInPlace(str_prefixed, category, level, source_file, source_line, logging_function, util::ThreadGetInternalName(), SystemClock::now(), GetMockTime());
+    FormatLogStrInPlace(str_prefixed, category, level, source_loc, util::ThreadGetInternalName(), SystemClock::now(), GetMockTime());
+    bool ratelimit{false};
+    if (should_ratelimit && m_limiter) {
+        auto status{m_limiter->Consume(source_loc, str_prefixed)};
+        if (status == BCLog::LogRateLimiter::Status::NEWLY_SUPPRESSED) {
+            // NOLINTNEXTLINE(misc-no-recursion)
+            LogPrintStr_(strprintf(
+                             "Excessive logging detected from %s:%d (%s): >%d bytes logged during "
+                             "the last time window of %is. Suppressing logging to disk from this "
+                             "source location until time window resets. Console logging "
+                             "unaffected. Last log entry.\n",
+                             source_loc.file_name(), source_loc.line(), source_loc.function_name(),
+                             m_limiter->m_max_bytes,
+                             Ticks<std::chrono::seconds>(m_limiter->m_reset_window)),
+                         std::source_location::current(), LogFlags::ALL, Level::Warning, /*should_ratelimit=*/false); // with should_ratelimit=false, this cannot lead to infinite recursion
+        }
+        ratelimit = status == BCLog::LogRateLimiter::Status::STILL_SUPPRESSED;
+        // To avoid confusion caused by dropped log messages when debugging an issue,
+        // we prefix log lines with "[*]" when there are any suppressed source locations.
+        if (m_limiter->SuppressionsActive()) {
+            str_prefixed.insert(0, "[*] ");
+        }
+    }
 
     if (m_print_to_console) {
         // print to console
@@ -434,7 +483,7 @@ void BCLog::Logger::LogPrintStr_(std::string_view str, std::string_view logging_
     for (const auto& cb : m_print_callbacks) {
         cb(str_prefixed);
     }
-    if (m_print_to_file) {
+    if (m_print_to_file && !ratelimit) {
         assert(m_fileout != nullptr);
 
         // reopen the log file, if requested
@@ -492,6 +541,37 @@ void BCLog::Logger::ShrinkDebugFile()
         fclose(file);
 }
 
+void BCLog::LogRateLimiter::Reset()
+{
+    decltype(m_source_locations) source_locations;
+    {
+        StdLockGuard scoped_lock(m_mutex);
+        source_locations.swap(m_source_locations);
+        m_suppression_active = false;
+    }
+    for (const auto& [source_loc, counter] : source_locations) {
+        uint64_t dropped_bytes{counter.GetDroppedBytes()};
+        if (dropped_bytes == 0) continue;
+        LogPrintLevel_(
+            LogFlags::ALL, Level::Info, /*should_ratelimit=*/false,
+            "Restarting logging from %s:%d (%s): %d bytes were dropped during the last %ss.\n",
+            source_loc.file_name(), source_loc.line(), source_loc.function_name(),
+            dropped_bytes, Ticks<std::chrono::seconds>(m_reset_window));
+    }
+}
+
+bool BCLog::LogLimitStats::Consume(uint64_t bytes)
+{
+    if (bytes > m_available_bytes) {
+        m_dropped_bytes += bytes;
+        m_available_bytes = 0;
+        return false;
+    }
+
+    m_available_bytes -= bytes;
+    return true;
+}
+
 bool BCLog::Logger::SetLogLevel(std::string_view level_str)
 {
     const auto level = GetLogLevel(level_str);