Implement a better copy_file.

This patch improves both the performance, and the safety of the
copy_file implementation.

The performance improvements are achieved by using sendfile on
Linux and copyfile on OS X when available.

The TOCTOU hardening is achieved by opening the source and
destination files and then using fstat to check their attributes to
see if we can copy them.

Unfortunately for the destination file, there is no way to open
it without accidentally creating it, so we first have to use
stat to determine if it exists, and if we should copy to it.
Then, once we're sure we should try to copy, we open the dest
file and ensure it names the same entity we previously stat'ed.

llvm-svn: 337649

2 files changed, 240 insertions, 144 deletions

diff --git a/libcxx/test/std/experimental/filesystem/fs.op.funcs/fs.op.copy_file/copy_file.pass.cpp b/libcxx/test/std/experimental/filesystem/fs.op.funcs/fs.op.copy_file/copy_file.pass.cpp
index 33b3ba9b586..3c2543ab942 100644
--- a/libcxx/test/std/experimental/filesystem/fs.op.funcs/fs.op.copy_file/copy_file.pass.cpp
+++ b/libcxx/test/std/experimental/filesystem/fs.op.funcs/fs.op.copy_file/copy_file.pass.cpp
@@ -26,167 +26,164 @@
 #include "rapid-cxx-test.hpp"
 #include "filesystem_test_helper.hpp"
 
+#include <iostream>
+
 using namespace fs;
 
 using CO = fs::copy_options;
 
 TEST_SUITE(filesystem_copy_file_test_suite)
 
-TEST_CASE(test_signatures)
-{
-    const path p; ((void)p);
-    const copy_options opts{}; ((void)opts);
-    std::error_code ec; ((void)ec);
-    ASSERT_SAME_TYPE(decltype(fs::copy_file(p, p)), bool);
-    ASSERT_SAME_TYPE(decltype(fs::copy_file(p, p, opts)), bool);
-    ASSERT_SAME_TYPE(decltype(fs::copy_file(p, p, ec)), bool);
-    ASSERT_SAME_TYPE(decltype(fs::copy_file(p, p, opts, ec)), bool);
-    ASSERT_NOT_NOEXCEPT(fs::copy_file(p, p));
-    ASSERT_NOT_NOEXCEPT(fs::copy_file(p, p, opts));
-    ASSERT_NOT_NOEXCEPT(fs::copy_file(p, p, ec));
-    ASSERT_NOT_NOEXCEPT(fs::copy_file(p, p, opts, ec));
+TEST_CASE(test_signatures) {
+  const path p;
+  ((void)p);
+  const copy_options opts{};
+  ((void)opts);
+  std::error_code ec;
+  ((void)ec);
+  ASSERT_SAME_TYPE(decltype(fs::copy_file(p, p)), bool);
+  ASSERT_SAME_TYPE(decltype(fs::copy_file(p, p, opts)), bool);
+  ASSERT_SAME_TYPE(decltype(fs::copy_file(p, p, ec)), bool);
+  ASSERT_SAME_TYPE(decltype(fs::copy_file(p, p, opts, ec)), bool);
+  ASSERT_NOT_NOEXCEPT(fs::copy_file(p, p));
+  ASSERT_NOT_NOEXCEPT(fs::copy_file(p, p, opts));
+  ASSERT_NOT_NOEXCEPT(fs::copy_file(p, p, ec));
+  ASSERT_NOT_NOEXCEPT(fs::copy_file(p, p, opts, ec));
 }
 
-TEST_CASE(test_error_reporting)
-{
-    auto checkThrow = [](path const& f, path const& t, const std::error_code& ec)
-    {
-#ifndef TEST_HAS_NO_EXCEPTIONS
-        try {
-            fs::copy_file(f, t);
-            return false;
-        } catch (filesystem_error const& err) {
-            return err.path1() == f
-                && err.path2() == t
-                && err.code() == ec;
-        }
-#else
-        ((void)f); ((void)t); ((void)ec);
-        return true;
-#endif
-    };
-
-    scoped_test_env env;
-    const path file = env.create_file("file1", 42);
-    const path file2 = env.create_file("file2", 55);
-    const path non_regular_file = env.create_fifo("non_reg");
-    const path dne = env.make_env_path("dne");
-    { // exists(to) && equivalent(to, from)
-        std::error_code ec;
-        TEST_CHECK(fs::copy_file(file, file, copy_options::overwrite_existing,
-                                 ec) == false);
-        TEST_REQUIRE(ec);
-        TEST_CHECK(ec == std::make_error_code(std::errc::file_exists));
-        TEST_CHECK(checkThrow(file, file, ec));
-    }
-    { // exists(to) && !(skip_existing | overwrite_existing | update_existing)
-        std::error_code ec;
-        TEST_CHECK(fs::copy_file(file, file2, ec) == false);
-        TEST_REQUIRE(ec);
-        TEST_CHECK(ec == std::make_error_code(std::errc::file_exists));
-        TEST_CHECK(checkThrow(file, file2, ec));
-    }
+TEST_CASE(test_error_reporting) {
+
+  scoped_test_env env;
+  const path file = env.create_file("file1", 42);
+  const path file2 = env.create_file("file2", 55);
+  const path non_regular_file = env.create_fifo("non_reg");
+  const path dne = env.make_env_path("dne");
+
+  { // exists(to) && equivalent(to, from)
+    std::error_code ec;
+    TEST_CHECK(fs::copy_file(file, file, copy_options::overwrite_existing,
+                             ec) == false);
+    TEST_CHECK(ErrorIs(ec, std::errc::file_exists));
+    ExceptionChecker Checker(file, file, std::errc::file_exists);
+    TEST_CHECK_THROW_RESULT(filesystem_error, Checker, copy_file(file, file, copy_options::overwrite_existing));
+
+  }
+  { // exists(to) && !(skip_existing | overwrite_existing | update_existing)
+    std::error_code ec;
+    TEST_CHECK(fs::copy_file(file, file2, ec) == false);
+    TEST_CHECK(ErrorIs(ec, std::errc::file_exists));
+    ExceptionChecker Checker(file, file, std::errc::file_exists);
+    TEST_CHECK_THROW_RESULT(filesystem_error, Checker, copy_file(file, file, copy_options::overwrite_existing));
+
+  }
+}
+
+TEST_CASE(non_regular_file_test) {
+  scoped_test_env env;
+  const path fifo = env.create_fifo("fifo");
+  const path dest = env.make_env_path("dest");
+  const path file = env.create_file("file", 42);
+
+  {
+    std::error_code ec = GetTestEC();
+    TEST_REQUIRE(fs::copy_file(fifo, dest, ec) == false);
+    TEST_CHECK(ErrorIs(ec, std::errc::not_supported));
+    TEST_CHECK(!exists(dest));
+  }
+  {
+    std::error_code ec = GetTestEC();
+    TEST_REQUIRE(fs::copy_file(file, fifo, copy_options::overwrite_existing,
+                               ec) == false);
+    TEST_CHECK(ErrorIs(ec, std::errc::not_supported));
+    TEST_CHECK(is_fifo(fifo));
+  }
+
 }
 
-TEST_CASE(copy_file)
-{
-    scoped_test_env env;
-    const path file = env.create_file("file1", 42);
-
-    { // !exists(to)
-        const path dest = env.make_env_path("dest1");
-        std::error_code ec;
-        TEST_REQUIRE(fs::copy_file(file, dest, ec) == true);
-        TEST_CHECK(!ec);
-        TEST_CHECK(file_size(dest) == 42);
-    }
-    { // exists(to) && overwrite_existing
-        const path dest = env.create_file("dest2", 55);
-        std::error_code ec;
-        TEST_REQUIRE(fs::copy_file(file, dest,
-                                   copy_options::overwrite_existing, ec) == true);
-        TEST_CHECK(!ec);
-        TEST_CHECK(file_size(dest) == 42);
-    }
-    { // exists(to) && update_existing
-        using Sec = std::chrono::seconds;
-        const path older = env.create_file("older_file", 1);
-
-        SleepFor(Sec(2));
-        const path from = env.create_file("update_from", 55);
-
-        SleepFor(Sec(2));
-        const path newer = env.create_file("newer_file", 2);
-
-        std::error_code ec;
-        TEST_REQUIRE(fs::copy_file(from, older, copy_options::update_existing, ec) == true);
-        TEST_CHECK(!ec);
-        TEST_CHECK(file_size(older) == 55);
-
-        TEST_REQUIRE(fs::copy_file(from, newer, copy_options::update_existing, ec) == false);
-        TEST_CHECK(!ec);
-        TEST_CHECK(file_size(newer) == 2);
-    }
-    { // skip_existing
-        const path file2 = env.create_file("file2", 55);
-        std::error_code ec;
-        TEST_REQUIRE(fs::copy_file(file, file2, copy_options::skip_existing, ec) == false);
-        TEST_CHECK(!ec);
-        TEST_CHECK(file_size(file2) == 55);
-    }
+TEST_CASE(test_attributes_get_copied) {
+  scoped_test_env env;
+  const path file = env.create_file("file1", 42);
+  const path dest = env.make_env_path("file2");
+  auto st = status(file);
+  perms new_perms = perms::owner_read;
+  permissions(file, new_perms);
+  std::error_code ec = GetTestEC();
+  TEST_REQUIRE(fs::copy_file(file, dest, ec) == true);
+  TEST_CHECK(!ec);
+  auto new_st = status(dest);
+  TEST_CHECK(new_st.permissions() == new_perms);
 }
 
-TEST_CASE(test_attributes_get_copied)
-{
-    scoped_test_env env;
-    const path file = env.create_file("file1", 42);
-    const path dest = env.make_env_path("file2");
-    auto st = status(file);
-    perms new_perms = perms::owner_read;
-    permissions(file, new_perms);
-    std::error_code ec;
+TEST_CASE(copy_dir_test) {
+  scoped_test_env env;
+  const path file = env.create_file("file1", 42);
+  const path dest = env.create_dir("dir1");
+  std::error_code ec = GetTestEC();
+  TEST_CHECK(fs::copy_file(file, dest, ec) == false);
+  TEST_CHECK(ec);
+  TEST_CHECK(ec != GetTestEC());
+  ec = GetTestEC();
+  TEST_CHECK(fs::copy_file(dest, file, ec) == false);
+  TEST_CHECK(ec);
+  TEST_CHECK(ec != GetTestEC());
+}
+
+TEST_CASE(copy_file) {
+  scoped_test_env env;
+  const path file = env.create_file("file1", 42);
+
+  { // !exists(to)
+    const path dest = env.make_env_path("dest1");
+    std::error_code ec = GetTestEC();
+
     TEST_REQUIRE(fs::copy_file(file, dest, ec) == true);
     TEST_CHECK(!ec);
-    auto new_st = status(dest);
-    TEST_CHECK(new_st.permissions() == new_perms);
-}
+    TEST_CHECK(file_size(dest) == 42);
+  }
+  { // exists(to) && overwrite_existing
+    const path dest = env.create_file("dest2", 55);
+    permissions(dest, perms::all);
+    permissions(file,
+                perms::group_write | perms::owner_write | perms::others_write,
+                perm_options::remove);
 
-TEST_CASE(copy_dir_test)
-{
-    scoped_test_env env;
-    const path file = env.create_file("file1", 42);
-    const path dest = env.create_dir("dir1");
     std::error_code ec = GetTestEC();
-    TEST_CHECK(fs::copy_file(file, dest, ec) == false);
-    TEST_CHECK(ec);
-    TEST_CHECK(ec != GetTestEC());
-    ec = GetTestEC();
-    TEST_CHECK(fs::copy_file(dest, file, ec) == false);
-    TEST_CHECK(ec);
-    TEST_CHECK(ec != GetTestEC());
-}
+    TEST_REQUIRE(fs::copy_file(file, dest, copy_options::overwrite_existing,
+                               ec) == true);
+    TEST_CHECK(!ec);
+    TEST_CHECK(file_size(dest) == 42);
+    TEST_CHECK(status(dest).permissions() == status(file).permissions());
+  }
+  { // exists(to) && update_existing
+    using Sec = std::chrono::seconds;
+    const path older = env.create_file("older_file", 1);
+
+    SleepFor(Sec(2));
+    const path from = env.create_file("update_from", 55);
+
+    SleepFor(Sec(2));
+    const path newer = env.create_file("newer_file", 2);
 
-TEST_CASE(non_regular_file_test)
-{
-    scoped_test_env env;
-    const path fifo = env.create_fifo("fifo");
-    const path dest = env.make_env_path("dest");
-    const path file = env.create_file("file", 42);
-    {
-        std::error_code ec = GetTestEC();
-        TEST_REQUIRE(fs::copy_file(fifo, dest, ec) == false);
-        TEST_CHECK(ec);
-        TEST_CHECK(ec != GetTestEC());
-        TEST_CHECK(!exists(dest));
-    }
-    {
-        std::error_code ec = GetTestEC();
-        TEST_REQUIRE(fs::copy_file(file, fifo, copy_options::overwrite_existing, ec) == false);
-        TEST_CHECK(ec);
-        TEST_CHECK(ec != GetTestEC());
-        TEST_CHECK(ec == std::make_error_code(std::errc::not_supported));
-        TEST_CHECK(is_fifo(fifo));
-    }
+    std::error_code ec = GetTestEC();
+    TEST_REQUIRE(
+        fs::copy_file(from, older, copy_options::update_existing, ec) == true);
+    TEST_CHECK(!ec);
+    TEST_CHECK(file_size(older) == 55);
+
+    TEST_REQUIRE(
+        fs::copy_file(from, newer, copy_options::update_existing, ec) == false);
+    TEST_CHECK(!ec);
+    TEST_CHECK(file_size(newer) == 2);
+  }
+  { // skip_existing
+    const path file2 = env.create_file("file2", 55);
+    std::error_code ec = GetTestEC();
+    TEST_REQUIRE(fs::copy_file(file, file2, copy_options::skip_existing, ec) ==
+                 false);
+    TEST_CHECK(!ec);
+    TEST_CHECK(file_size(file2) == 55);
+  }
 }
 
+
 TEST_SUITE_END()
diff --git a/libcxx/test/std/experimental/filesystem/fs.op.funcs/fs.op.copy_file/copy_file_large.pass.cpp b/libcxx/test/std/experimental/filesystem/fs.op.funcs/fs.op.copy_file/copy_file_large.pass.cpp
new file mode 100644
index 00000000000..46b5f2a6702
--- /dev/null
+++ b/libcxx/test/std/experimental/filesystem/fs.op.funcs/fs.op.copy_file/copy_file_large.pass.cpp
@@ -0,0 +1,99 @@
+//===----------------------------------------------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++98, c++03
+// REQUIRES: long_tests
+
+// <experimental/filesystem>
+
+// bool copy_file(const path& from, const path& to);
+// bool copy_file(const path& from, const path& to, error_code& ec) noexcept;
+// bool copy_file(const path& from, const path& to, copy_options options);
+// bool copy_file(const path& from, const path& to, copy_options options,
+//           error_code& ec) noexcept;
+
+#include "filesystem_include.hpp"
+#include <type_traits>
+#include <chrono>
+#include <cassert>
+
+#include "test_macros.h"
+#include "rapid-cxx-test.hpp"
+#include "filesystem_test_helper.hpp"
+
+using namespace fs;
+
+TEST_SUITE(filesystem_copy_file_test_suite)
+
+static std::string random_hex_chars(uintmax_t size) {
+  std::string data;
+  data.reserve(size);
+  for (uintmax_t I = 0; I < size; ++I)
+    data.push_back(random_utils::random_hex_char());
+  return data;
+}
+
+// This test is intended to test 'sendfile's 2gb limit for a single call, and
+// to ensure that libc++ correctly copies files larger than that limit.
+// However it requires allocating ~5GB of filesystem space. This might not
+// be acceptable on all systems.
+TEST_CASE(large_file) {
+  using namespace fs;
+  constexpr uintmax_t sendfile_size_limit = 2147479552ull;
+  constexpr uintmax_t additional_size = 1024;
+  constexpr uintmax_t test_file_size = sendfile_size_limit + additional_size;
+  static_assert(test_file_size > sendfile_size_limit, "");
+
+  scoped_test_env env;
+
+  // Check that we have more than sufficient room to create the files needed
+  // to perform the test.
+  if (space(env.test_root).available < 3 * test_file_size) {
+    TEST_UNSUPPORTED();
+  }
+
+  // Use python to create a file right at the size limit.
+  const path file = env.create_file("source", sendfile_size_limit);
+  // Create some random data that looks different than the data before the
+  // size limit.
+  const std::string additional_data = random_hex_chars(additional_size);
+  // Append this known data to the end of the source file.
+  {
+    std::ofstream outf(file.native(), std::ios_base::app);
+    TEST_REQUIRE(outf.good());
+    outf << additional_data;
+    TEST_REQUIRE(outf);
+  }
+  TEST_REQUIRE(file_size(file) == test_file_size);
+  const path dest = env.make_env_path("dest");
+
+  std::error_code ec = GetTestEC();
+  TEST_CHECK(copy_file(file, dest, ec));
+  TEST_CHECK(!ec);
+
+  TEST_REQUIRE(is_regular_file(dest));
+  TEST_CHECK(file_size(dest) == test_file_size);
+
+  // Read the data from the end of the destination file, and ensure it matches
+  // the data at the end of the source file.
+  std::string out_data;
+  out_data.reserve(additional_size);
+  {
+    std::ifstream dest_file(dest.native());
+    TEST_REQUIRE(dest_file);
+    dest_file.seekg(sendfile_size_limit);
+    TEST_REQUIRE(dest_file);
+    dest_file >> out_data;
+    TEST_CHECK(dest_file.eof());
+  }
+  TEST_CHECK(out_data.size() == additional_data.size());
+  TEST_CHECK(out_data == additional_data);
+}
+
+TEST_SUITE_END()