Merge #13878: utils: Add fstream wrapper to allow to pass unicode filename on Windows

43c7fbb1e7 Make MSVC compiler read the source code using utf-8 (Chun Kuan Lee)
f86a571edb tests: Add test case for std::ios_base::ate (Chun Kuan Lee)
a554cc901a Move boost/std fstream to fsbridge (Chun Kuan Lee)
86eb3b3f1a utils: Add fsbridge fstream function wrapper (Chun Kuan Lee)

Pull request description:

  If compiled with mingw, use glibc++ extension `stdio_filebuf` to open the file by `FILE*` instead of filename.

  In other condition, we can use boost::fstream.

Tree-SHA512: b5dbd83e347fb9b2a0c8b1c2c7bd71a272e839ec0617883b2a0ec12506ae9e825373cf6e95b9bcc91d7edc85bf51580a7716b56a9ecaad776bc3ae61638cb3da

build_msvc/common.vcxproj

@@ -12,4 +12,9 @@
           Outputs="$(MSBuildThisFileDirectory)..\src\config\bitcoin-config.h">
       <Copy SourceFiles="$(MSBuildThisFileDirectory)bitcoin_config.h" DestinationFiles="$(MSBuildThisFileDirectory)..\src\config\bitcoin-config.h" />
   </Target>
+  <ItemDefinitionGroup>
+    <ClCompile>
+      <AdditionalOptions>/utf-8 %(AdditionalOptions)</AdditionalOptions>
+    </ClCompile>
+  </ItemDefinitionGroup>
 </Project>

src/Makefile.test.include

@@ -51,6 +51,7 @@ BITCOIN_TESTS =\
   test/cuckoocache_tests.cpp \
   test/denialofservice_tests.cpp \
   test/descriptor_tests.cpp \
+  test/fs_tests.cpp \
   test/getarg_tests.cpp \
   test/hash_tests.cpp \
   test/key_io_tests.cpp \

src/fs.cpp

@@ -113,4 +113,106 @@ std::string get_filesystem_error_message(const fs::filesystem_error& e)
 #endif
 }
 
+#ifdef WIN32
+#ifdef __GLIBCXX__
+
+// reference: https://github.com/gcc-mirror/gcc/blob/gcc-7_3_0-release/libstdc%2B%2B-v3/include/std/fstream#L270
+
+static std::string openmodeToStr(std::ios_base::openmode mode)
+{
+    switch (mode & ~std::ios_base::ate) {
+    case std::ios_base::out:
+    case std::ios_base::out | std::ios_base::trunc:
+        return "w";
+    case std::ios_base::out | std::ios_base::app:
+    case std::ios_base::app:
+        return "a";
+    case std::ios_base::in:
+        return "r";
+    case std::ios_base::in | std::ios_base::out:
+        return "r+";
+    case std::ios_base::in | std::ios_base::out | std::ios_base::trunc:
+        return "w+";
+    case std::ios_base::in | std::ios_base::out | std::ios_base::app:
+    case std::ios_base::in | std::ios_base::app:
+        return "a+";
+    case std::ios_base::out | std::ios_base::binary:
+    case std::ios_base::out | std::ios_base::trunc | std::ios_base::binary:
+        return "wb";
+    case std::ios_base::out | std::ios_base::app | std::ios_base::binary:
+    case std::ios_base::app | std::ios_base::binary:
+        return "ab";
+    case std::ios_base::in | std::ios_base::binary:
+        return "rb";
+    case std::ios_base::in | std::ios_base::out | std::ios_base::binary:
+        return "r+b";
+    case std::ios_base::in | std::ios_base::out | std::ios_base::trunc | std::ios_base::binary:
+        return "w+b";
+    case std::ios_base::in | std::ios_base::out | std::ios_base::app | std::ios_base::binary:
+    case std::ios_base::in | std::ios_base::app | std::ios_base::binary:
+        return "a+b";
+    default:
+        return std::string();
+    }
+}
+
+void ifstream::open(const fs::path& p, std::ios_base::openmode mode)
+{
+    close();
+    m_file = fsbridge::fopen(p, openmodeToStr(mode).c_str());
+    if (m_file == nullptr) {
+        return;
+    }
+    m_filebuf = __gnu_cxx::stdio_filebuf<char>(m_file, mode);
+    rdbuf(&m_filebuf);
+    if (mode & std::ios_base::ate) {
+        seekg(0, std::ios_base::end);
+    }
+}
+
+void ifstream::close()
+{
+    if (m_file != nullptr) {
+        m_filebuf.close();
+        fclose(m_file);
+    }
+    m_file = nullptr;
+}
+
+void ofstream::open(const fs::path& p, std::ios_base::openmode mode)
+{
+    close();
+    m_file = fsbridge::fopen(p, openmodeToStr(mode).c_str());
+    if (m_file == nullptr) {
+        return;
+    }
+    m_filebuf = __gnu_cxx::stdio_filebuf<char>(m_file, mode);
+    rdbuf(&m_filebuf);
+    if (mode & std::ios_base::ate) {
+        seekp(0, std::ios_base::end);
+    }
+}
+
+void ofstream::close()
+{
+    if (m_file != nullptr) {
+        m_filebuf.close();
+        fclose(m_file);
+    }
+    m_file = nullptr;
+}
+#else // __GLIBCXX__
+
+static_assert(sizeof(*fs::path().BOOST_FILESYSTEM_C_STR) == sizeof(wchar_t),
+    "Warning: This build is using boost::filesystem ofstream and ifstream "
+    "implementations which will fail to open paths containing multibyte "
+    "characters. You should delete this static_assert to ignore this warning, "
+    "or switch to a different C++ standard library like the Microsoft C++ "
+    "Standard Library (where boost uses non-standard extensions to construct "
+    "stream objects with wide filenames), or the GNU libstdc++ library (where "
+    "a more complicated workaround has been implemented above).");
+
+#endif // __GLIBCXX__
+#endif // WIN32
+
 } // fsbridge

src/fs.h

@@ -7,6 +7,9 @@
 
 #include <stdio.h>
 #include <string>
+#if defined WIN32 && defined __GLIBCXX__
+#include <ext/stdio_filebuf.h>
+#endif
 
 #include <boost/filesystem.hpp>
 #include <boost/filesystem/fstream.hpp>
@@ -39,6 +42,54 @@ namespace fsbridge {
     };
 
     std::string get_filesystem_error_message(const fs::filesystem_error& e);
+
+    // GNU libstdc++ specific workaround for opening UTF-8 paths on Windows.
+    //
+    // On Windows, it is only possible to reliably access multibyte file paths through
+    // `wchar_t` APIs, not `char` APIs. But because the C++ standard doesn't
+    // require ifstream/ofstream `wchar_t` constructors, and the GNU library doesn't
+    // provide them (in contrast to the Microsoft C++ library, see
+    // https://stackoverflow.com/questions/821873/how-to-open-an-stdfstream-ofstream-or-ifstream-with-a-unicode-filename/822032#822032),
+    // Boost is forced to fall back to `char` constructors which may not work properly.
+    //
+    // Work around this issue by creating stream objects with `_wfopen` in
+    // combination with `__gnu_cxx::stdio_filebuf`. This workaround can be removed
+    // with an upgrade to C++17, where streams can be constructed directly from
+    // `std::filesystem::path` objects.
+
+#if defined WIN32 && defined __GLIBCXX__
+    class ifstream : public std::istream
+    {
+    public:
+        ifstream() = default;
+        explicit ifstream(const fs::path& p, std::ios_base::openmode mode = std::ios_base::in) { open(p, mode); }
+        ~ifstream() { close(); }
+        void open(const fs::path& p, std::ios_base::openmode mode = std::ios_base::in);
+        bool is_open() { return m_filebuf.is_open(); }
+        void close();
+
+    private:
+        __gnu_cxx::stdio_filebuf<char> m_filebuf;
+        FILE* m_file = nullptr;
+    };
+    class ofstream : public std::ostream
+    {
+    public:
+        ofstream() = default;
+        explicit ofstream(const fs::path& p, std::ios_base::openmode mode = std::ios_base::out) { open(p, mode); }
+        ~ofstream() { close(); }
+        void open(const fs::path& p, std::ios_base::openmode mode = std::ios_base::out);
+        bool is_open() { return m_filebuf.is_open(); }
+        void close();
+
+    private:
+        __gnu_cxx::stdio_filebuf<char> m_filebuf;
+        FILE* m_file = nullptr;
+    };
+#else  // !(WIN32 && __GLIBCXX__)
+    typedef fs::ifstream ifstream;
+    typedef fs::ofstream ofstream;
+#endif // WIN32 && __GLIBCXX__
 };
 
 #endif // BITCOIN_FS_H

src/qt/guiutil.cpp

@@ -367,7 +367,7 @@ bool openBitcoinConf()
     fs::path pathConfig = GetConfigFile(gArgs.GetArg("-conf", BITCOIN_CONF_FILENAME));
 
     /* Create the file */
-    fs::ofstream configFile(pathConfig, std::ios_base::app);
+    fsbridge::ofstream configFile(pathConfig, std::ios_base::app);
 
     if (!configFile.good())
         return false;
@@ -611,7 +611,7 @@ fs::path static GetAutostartFilePath()
 
 bool GetStartOnSystemStartup()
 {
-    fs::ifstream optionFile(GetAutostartFilePath());
+    fsbridge::ifstream optionFile(GetAutostartFilePath());
     if (!optionFile.good())
         return false;
     // Scan through file for "Hidden=true":
@@ -642,7 +642,7 @@ bool SetStartOnSystemStartup(bool fAutoStart)
 
         fs::create_directories(GetAutostartDir());
 
-        fs::ofstream optionFile(GetAutostartFilePath(), std::ios_base::out|std::ios_base::trunc);
+        fsbridge::ofstream optionFile(GetAutostartFilePath(), std::ios_base::out | std::ios_base::trunc);
         if (!optionFile.good())
             return false;
         std::string chain = gArgs.GetChainName();

src/rpc/protocol.cpp

@@ -12,8 +12,6 @@
 #include <utiltime.h>
 #include <version.h>
 
-#include <fstream>
-
 /**
  * JSON-RPC protocol.  Bitcoin speaks version 1.0 for maximum compatibility,
  * but uses JSON-RPC 1.1/2.0 standards for parts of the 1.0 standard that were
@@ -85,9 +83,9 @@ bool GenerateAuthCookie(std::string *cookie_out)
     /** the umask determines what permissions are used to create this file -
      * these are set to 077 in init.cpp unless overridden with -sysperms.
      */
-    std::ofstream file;
+    fsbridge::ofstream file;
     fs::path filepath_tmp = GetAuthCookieFile(true);
-    file.open(filepath_tmp.string().c_str());
+    file.open(filepath_tmp);
     if (!file.is_open()) {
         LogPrintf("Unable to open cookie authentication file %s for writing\n", filepath_tmp.string());
         return false;
@@ -109,10 +107,10 @@ bool GenerateAuthCookie(std::string *cookie_out)
 
 bool GetAuthCookie(std::string *cookie_out)
 {
-    std::ifstream file;
+    fsbridge::ifstream file;
     std::string cookie;
     fs::path filepath = GetAuthCookieFile();
-    file.open(filepath.string().c_str());
+    file.open(filepath);
     if (!file.is_open())
         return false;
     std::getline(file, cookie);

src/test/fs_tests.cpp

@@ -0,0 +1,56 @@
+// Copyright (c) 2011-2018 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+//
+#include <fs.h>
+#include <test/test_bitcoin.h>
+
+#include <boost/test/unit_test.hpp>
+
+BOOST_FIXTURE_TEST_SUITE(fs_tests, BasicTestingSetup)
+
+BOOST_AUTO_TEST_CASE(fsbridge_fstream)
+{
+    fs::path tmpfolder = SetDataDir("fsbridge_fstream");
+    // tmpfile1 should be the same as tmpfile2
+    fs::path tmpfile1 = tmpfolder / "fs_tests_₿_🏃";
+    fs::path tmpfile2 = tmpfolder / L"fs_tests_₿_🏃";
+    {
+        fsbridge::ofstream file(tmpfile1);
+        file << "bitcoin";
+    }
+    {
+        fsbridge::ifstream file(tmpfile2);
+        std::string input_buffer;
+        file >> input_buffer;
+        BOOST_CHECK_EQUAL(input_buffer, "bitcoin");
+    }
+    {
+        fsbridge::ifstream file(tmpfile1, std::ios_base::in | std::ios_base::ate);
+        std::string input_buffer;
+        file >> input_buffer;
+        BOOST_CHECK_EQUAL(input_buffer, "");
+    }
+    {
+        fsbridge::ofstream file(tmpfile2, std::ios_base::out | std::ios_base::app);
+        file << "tests";
+    }
+    {
+        fsbridge::ifstream file(tmpfile1);
+        std::string input_buffer;
+        file >> input_buffer;
+        BOOST_CHECK_EQUAL(input_buffer, "bitcointests");
+    }
+    {
+        fsbridge::ofstream file(tmpfile2, std::ios_base::out | std::ios_base::trunc);
+        file << "bitcoin";
+    }
+    {
+        fsbridge::ifstream file(tmpfile1);
+        std::string input_buffer;
+        file >> input_buffer;
+        BOOST_CHECK_EQUAL(input_buffer, "bitcoin");
+    }
+}
+
+BOOST_AUTO_TEST_SUITE_END()

src/util.cpp

@@ -892,7 +892,7 @@ bool ArgsManager::ReadConfigFiles(std::string& error, bool ignore_invalid_keys)
     }
 
     const std::string confPath = GetArg("-conf", BITCOIN_CONF_FILENAME);
-    fs::ifstream stream(GetConfigFile(confPath));
+    fsbridge::ifstream stream(GetConfigFile(confPath));
 
     // ok to not have a config file
     if (stream.good()) {
@@ -925,7 +925,7 @@ bool ArgsManager::ReadConfigFiles(std::string& error, bool ignore_invalid_keys)
             }
 
             for (const std::string& to_include : includeconf) {
-                fs::ifstream include_config(GetConfigFile(to_include));
+                fsbridge::ifstream include_config(GetConfigFile(to_include));
                 if (include_config.good()) {
                     if (!ReadConfigStream(include_config, error, ignore_invalid_keys)) {
                         return false;

src/wallet/rpcdump.cpp

@@ -17,7 +17,6 @@
 
 #include <wallet/rpcwallet.h>
 
-#include <fstream>
 #include <stdint.h>
 
 #include <boost/algorithm/string.hpp>
@@ -540,8 +539,8 @@ UniValue importwallet(const JSONRPCRequest& request)
 
         EnsureWalletIsUnlocked(pwallet);
 
-        std::ifstream file;
-        file.open(request.params[0].get_str().c_str(), std::ios::in | std::ios::ate);
+        fsbridge::ifstream file;
+        file.open(request.params[0].get_str(), std::ios::in | std::ios::ate);
         if (!file.is_open()) {
             throw JSONRPCError(RPC_INVALID_PARAMETER, "Cannot open wallet dump file");
         }
@@ -717,8 +716,8 @@ UniValue dumpwallet(const JSONRPCRequest& request)
         throw JSONRPCError(RPC_INVALID_PARAMETER, filepath.string() + " already exists. If you are sure this is what you want, move it out of the way first");
     }
 
-    std::ofstream file;
-    file.open(filepath.string().c_str());
+    fsbridge::ofstream file;
+    file.open(filepath);
     if (!file.is_open())
         throw JSONRPCError(RPC_INVALID_PARAMETER, "Cannot open wallet dump file");