From 7596282a556accb84fbcec3f31a3e2017d8ade0c Mon Sep 17 00:00:00 2001 From: laanwj <126646+laanwj@users.noreply.github.com> Date: Sun, 27 Oct 2024 09:03:47 +0100 Subject: [PATCH 1/5] memusage: Allow counting usage of vectors with different allocators --- src/memusage.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/memusage.h b/src/memusage.h index 08be66172e3..43407507ed8 100644 --- a/src/memusage.h +++ b/src/memusage.h @@ -84,10 +84,10 @@ struct stl_shared_counter size_t weak_count; }; -template -static inline size_t DynamicUsage(const std::vector& v) +template +static inline size_t DynamicUsage(const std::vector& v) { - return MallocUsage(v.capacity() * sizeof(X)); + return MallocUsage(v.capacity() * sizeof(T)); } template From c6594c0b142133535c1d2d5b8d8084cf9e57592b Mon Sep 17 00:00:00 2001 From: laanwj <126646+laanwj@users.noreply.github.com> Date: Mon, 4 Nov 2024 18:19:00 +0100 Subject: [PATCH 2/5] memusage: Add DynamicUsage for std::string Add DynamicUsage(std::string) which Returns the dynamic allocation of a std::string, or 0 if none (in case of small string optimization). --- src/memusage.h | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/memusage.h b/src/memusage.h index 43407507ed8..9d9e549ef22 100644 --- a/src/memusage.h +++ b/src/memusage.h @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -90,6 +91,18 @@ static inline size_t DynamicUsage(const std::vector& v) return MallocUsage(v.capacity() * sizeof(T)); } +static inline size_t DynamicUsage(const std::string& s) +{ + const char* s_ptr = reinterpret_cast(&s); + // Don't count the dynamic memory used for string, if it resides in the + // "small string" optimization area (which stores data inside the object itself, up to some + // size; 15 bytes in modern libstdc++). + if (!std::less{}(s.data(), s_ptr) && !std::greater{}(s.data() + s.size(), s_ptr + sizeof(s))) { + return 0; + } + return MallocUsage(s.capacity()); +} + template static inline size_t DynamicUsage(const prevector& v) { From c3a6722f34a575834b282894d441d7bf802bd467 Mon Sep 17 00:00:00 2001 From: laanwj <126646+laanwj@users.noreply.github.com> Date: Mon, 4 Nov 2024 18:26:06 +0100 Subject: [PATCH 3/5] net: Use DynamicUsage(m_type) in CSerializedNetMsg::GetMemoryUsage Now that memusage correctly computes the dynamic size of a string, there is no need for special handling here. --- src/net.cpp | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/net.cpp b/src/net.cpp index d76b39f75d5..3e936b5f3de 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -121,10 +121,7 @@ std::string strSubVersion; size_t CSerializedNetMsg::GetMemoryUsage() const noexcept { - // Don't count the dynamic memory used for the m_type string, by assuming it fits in the - // "small string" optimization area (which stores data inside the object itself, up to some - // size; 15 bytes in modern libstdc++). - return sizeof(*this) + memusage::DynamicUsage(data); + return sizeof(*this) + memusage::DynamicUsage(m_type) + memusage::DynamicUsage(data); } void CConnman::AddAddrFetch(const std::string& strDest) From 047b5e2af1f03549d85926aa02fed0dfa00d449f Mon Sep 17 00:00:00 2001 From: laanwj <126646+laanwj@users.noreply.github.com> Date: Sun, 27 Oct 2024 09:04:54 +0100 Subject: [PATCH 4/5] streams: add DataStream::GetMemoryUsage --- src/streams.cpp | 6 ++++++ src/streams.h | 3 +++ 2 files changed, 9 insertions(+) diff --git a/src/streams.cpp b/src/streams.cpp index baa5ad7abed..cd496ee2be4 100644 --- a/src/streams.cpp +++ b/src/streams.cpp @@ -2,6 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or https://opensource.org/license/mit/. +#include #include #include #include @@ -110,3 +111,8 @@ bool AutoFile::Truncate(unsigned size) { return ::TruncateFile(m_file, size); } + +size_t DataStream::GetMemoryUsage() const noexcept +{ + return sizeof(*this) + memusage::DynamicUsage(vch); +} diff --git a/src/streams.h b/src/streams.h index e9f3562c6ce..64dbfbff145 100644 --- a/src/streams.h +++ b/src/streams.h @@ -277,6 +277,9 @@ public: { util::Xor(MakeWritableByteSpan(*this), MakeByteSpan(key)); } + + /** Compute total memory usage of this object (own memory + any dynamic memory). */ + size_t GetMemoryUsage() const noexcept; }; template From d22a234ed270286b483aec2db1e2f716b9756231 Mon Sep 17 00:00:00 2001 From: laanwj <126646+laanwj@users.noreply.github.com> Date: Sun, 27 Oct 2024 09:05:20 +0100 Subject: [PATCH 5/5] net: Use actual memory size in receive buffer accounting Add a method CNetMessage::GetMemoryUsage and use this for accounting of the size of the process receive queue instead of the raw message size. This ensures that allocation and deserialization overhead is taken into account. --- src/net.cpp | 9 +++++++-- src/net.h | 3 +++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/net.cpp b/src/net.cpp index 3e936b5f3de..f1ac681b998 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -124,6 +124,11 @@ size_t CSerializedNetMsg::GetMemoryUsage() const noexcept return sizeof(*this) + memusage::DynamicUsage(m_type) + memusage::DynamicUsage(data); } +size_t CNetMessage::GetMemoryUsage() const noexcept +{ + return sizeof(*this) + memusage::DynamicUsage(m_type) + m_recv.GetMemoryUsage(); +} + void CConnman::AddAddrFetch(const std::string& strDest) { LOCK(m_addr_fetches_mutex); @@ -3769,7 +3774,7 @@ void CNode::MarkReceivedMsgsForProcessing() for (const auto& msg : vRecvMsg) { // vRecvMsg contains only completed CNetMessage // the single possible partially deserialized message are held by TransportDeserializer - nSizeAdded += msg.m_raw_message_size; + nSizeAdded += msg.GetMemoryUsage(); } LOCK(m_msg_process_queue_mutex); @@ -3786,7 +3791,7 @@ std::optional> CNode::PollMessage() std::list msgs; // Just take one message msgs.splice(msgs.begin(), m_msg_process_queue, m_msg_process_queue.begin()); - m_msg_process_queue_size -= msgs.front().m_raw_message_size; + m_msg_process_queue_size -= msgs.front().GetMemoryUsage(); fPauseRecv = m_msg_process_queue_size > m_recv_flood_size; return std::make_pair(std::move(msgs.front()), !m_msg_process_queue.empty()); diff --git a/src/net.h b/src/net.h index 6b63e2cc0d1..20c654c34ab 100644 --- a/src/net.h +++ b/src/net.h @@ -245,6 +245,9 @@ public: CNetMessage(const CNetMessage&) = delete; CNetMessage& operator=(CNetMessage&&) = default; CNetMessage& operator=(const CNetMessage&) = delete; + + /** Compute total memory usage of this object (own memory + any dynamic memory). */ + size_t GetMemoryUsage() const noexcept; }; /** The Transport converts one connection's sent messages to wire bytes, and received bytes back. */