mirrors/bitcoin
1
0
Fork 0
mirror of https://github.com/bitcoin/bitcoin.git synced 2025-03-04 15:18:05 +01:00
Code Issues Projects Releases Wiki Activity

util: add CBufferedFile::SkipTo() to move ahead in the stream

Browse source 
SkipTo() reads data from the file into the CBufferedFile object
(memory), but, unlike this object's read() method, SkipTo() doesn't
transfer data into a caller's memory buffer. This is useful because
after skipping forward in the stream in this way, the user can, if
needed, rewind the stream (SetPos()) and access the object's memory
buffer including ranges that were skipped over (without needing to
read from the disk file).
This commit is contained in:
Larry Ruane 2020-07-13 17:11:37 -06:00
parent 48a68908ba
commit c72de9990a
2 changed files with 96 additions and 19 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

48
src/streams.h
View file

@ -612,7 +612,6 @@ private:
uint64_t nRewind; //!< how many bytes we guarantee to rewind
std::vector<std::byte> vchBuf; //!< the buffer
protected:
//! read data from the source to fill the buffer
bool Fill() {
unsigned int pos = nSrcPos % vchBuf.size();
@ -630,6 +629,28 @@ protected:
return true;
}
//! Advance the stream's read pointer (m_read_pos) by up to 'length' bytes,
//! filling the buffer from the file so that at least one byte is available.
//! Return a pointer to the available buffer data and the number of bytes
//! (which may be less than the requested length) that may be accessed
//! beginning at that pointer.
std::pair<std::byte*, size_t> AdvanceStream(size_t length)
{
assert(m_read_pos <= nSrcPos);
if (m_read_pos + length > nReadLimit) {
throw std::ios_base::failure("Attempt to position past buffer limit");
}
// If there are no bytes available, read from the file.
if (m_read_pos == nSrcPos && length > 0) Fill();
size_t buffer_offset{static_cast<size_t>(m_read_pos % vchBuf.size())};
size_t buffer_available{static_cast<size_t>(vchBuf.size() - buffer_offset)};
size_t bytes_until_source_pos{static_cast<size_t>(nSrcPos - m_read_pos)};
size_t advance{std::min({length, buffer_available, bytes_until_source_pos})};
m_read_pos += advance;
return std::make_pair(&vchBuf[buffer_offset], advance);
}
public:
CBufferedFile(FILE* fileIn, uint64_t nBufSize, uint64_t nRewindIn, int nTypeIn, int nVersionIn)
: nType(nTypeIn), nVersion(nVersionIn), nSrcPos(0), m_read_pos(0), nReadLimit(std::numeric_limits<uint64_t>::max()), nRewind(nRewindIn), vchBuf(nBufSize, std::byte{0})
@ -667,24 +688,21 @@ public:
//! read a number of bytes
void read(Span<std::byte> dst)
{
if (dst.size() + m_read_pos > nReadLimit) {
throw std::ios_base::failure("Read attempted past buffer limit");
}
while (dst.size() > 0) {
if (m_read_pos == nSrcPos)
Fill();
unsigned int pos = m_read_pos % vchBuf.size();
size_t nNow = dst.size();
if (nNow + pos > vchBuf.size())
nNow = vchBuf.size() - pos;
if (nNow + m_read_pos > nSrcPos)
nNow = nSrcPos - m_read_pos;
memcpy(dst.data(), &vchBuf[pos], nNow);
m_read_pos += nNow;
dst = dst.subspan(nNow);
auto [buffer_pointer, length]{AdvanceStream(dst.size())};
memcpy(dst.data(), buffer_pointer, length);
dst = dst.subspan(length);
}
}
//! Move the read position ahead in the stream to the given position.
//! Use SetPos() to back up in the stream, not SkipTo().
void SkipTo(const uint64_t file_pos)
{
assert(file_pos >= m_read_pos);
while (m_read_pos < file_pos) AdvanceStream(file_pos - m_read_pos);
}
//! return the current reading position
uint64_t GetPos() const {
return m_read_pos;

67
src/test/streams_tests.cpp
View file

@ -253,7 +253,7 @@ BOOST_AUTO_TEST_CASE(streams_buffered_file)
BOOST_CHECK(false);
} catch (const std::exception& e) {
BOOST_CHECK(strstr(e.what(),
"Read attempted past buffer limit") != nullptr);
"Attempt to position past buffer limit") != nullptr);
}
// The default argument removes the limit completely.
BOOST_CHECK(bf.SetLimit());
@ -322,7 +322,7 @@ BOOST_AUTO_TEST_CASE(streams_buffered_file)
BOOST_CHECK(!bf.SetPos(0));
// But we should now be positioned at least as far back as allowed
// by the rewind window (relative to our farthest read position, 40).
BOOST_CHECK(bf.GetPos() <= 30);
BOOST_CHECK(bf.GetPos() <= 30U);
// We can explicitly close the file, or the destructor will do it.
bf.fclose();
@ -330,6 +330,55 @@ BOOST_AUTO_TEST_CASE(streams_buffered_file)
fs::remove(streams_test_filename);
}
BOOST_AUTO_TEST_CASE(streams_buffered_file_skip)
{
fs::path streams_test_filename = m_args.GetDataDirBase() / "streams_test_tmp";
FILE* file = fsbridge::fopen(streams_test_filename, "w+b");
// The value at each offset is the byte offset (e.g. byte 1 in the file has the value 0x01).
for (uint8_t j = 0; j < 40; ++j) {
fwrite(&j, 1, 1, file);
}
rewind(file);
// The buffer is 25 bytes, allow rewinding 10 bytes.
CBufferedFile bf(file, 25, 10, 222, 333);
uint8_t i;
// This is like bf >> (7-byte-variable), in that it will cause data
// to be read from the file into memory, but it's not copied to us.
bf.SkipTo(7);
BOOST_CHECK_EQUAL(bf.GetPos(), 7U);
bf >> i;
BOOST_CHECK_EQUAL(i, 7);
// The bytes in the buffer up to offset 7 are valid and can be read.
BOOST_CHECK(bf.SetPos(0));
bf >> i;
BOOST_CHECK_EQUAL(i, 0);
bf >> i;
BOOST_CHECK_EQUAL(i, 1);
bf.SkipTo(11);
bf >> i;
BOOST_CHECK_EQUAL(i, 11);
// SkipTo() honors the transfer limit; we can't position beyond the limit.
bf.SetLimit(13);
try {
bf.SkipTo(14);
BOOST_CHECK(false);
} catch (const std::exception& e) {
BOOST_CHECK(strstr(e.what(), "Attempt to position past buffer limit") != nullptr);
}
// We can position exactly to the transfer limit.
bf.SkipTo(13);
BOOST_CHECK_EQUAL(bf.GetPos(), 13U);
bf.fclose();
fs::remove(streams_test_filename);
}
BOOST_AUTO_TEST_CASE(streams_buffered_file_rand)
{
// Make this test deterministic.
@ -361,7 +410,7 @@ BOOST_AUTO_TEST_CASE(streams_buffered_file_rand)
// sizes; the boundaries of the objects can interact arbitrarily
// with the CBufferFile's internal buffer. These first three
// cases simulate objects of various sizes (1, 2, 5 bytes).
switch (InsecureRandRange(5)) {
switch (InsecureRandRange(6)) {
case 0: {
uint8_t a[1];
if (currentPos + 1 > fileSize)
@ -399,6 +448,16 @@ BOOST_AUTO_TEST_CASE(streams_buffered_file_rand)
break;
}
case 3: {
// SkipTo is similar to the "read" cases above, except
// we don't receive the data.
size_t skip_length{static_cast<size_t>(InsecureRandRange(5))};
if (currentPos + skip_length > fileSize) continue;
bf.SetLimit(currentPos + skip_length);
bf.SkipTo(currentPos + skip_length);
currentPos += skip_length;
break;
}
case 4: {
// Find a byte value (that is at or ahead of the current position).
size_t find = currentPos + InsecureRandRange(8);
if (find >= fileSize)
@ -415,7 +474,7 @@ BOOST_AUTO_TEST_CASE(streams_buffered_file_rand)
currentPos++;
break;
}
case 4: {
case 5: {
size_t requestPos = InsecureRandRange(maxPos + 4);
bool okay = bf.SetPos(requestPos);
// The new position may differ from the requested position