shifathosn
/
KytyPS5
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
KytyPS5/tests/ArchiveFileTests.cpp

401 lines
18 KiB
C++
Raw Blame History

#include "common/file.h"
#include "common/archive.h"
#include "common/stringUtils.h"
#include "ArchiveTestFixture.h"
#include <algorithm>
#include <array>
#include <atomic>
#include <chrono>
#include <cstdint>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <filesystem>
#include <limits>
#include <string>
#include <thread>
#include <vector>
#include <zarchive/zarchivecommon.h>
#if !defined(_WIN32) || defined(_GLIBCXX_GCC_GTHR_POSIX_H)
#include <pthread.h>
#endif
namespace {
void Check(bool value, const char *text) {
if (!value) {
std::fprintf(stderr, "ArchiveFileTests: failed: %s\n", text);
std::abort();
}
}
class TempDirectory {
public:
TempDirectory() {
const auto unique =
std::chrono::steady_clock::now().time_since_epoch().count();
m_path = std::filesystem::temp_directory_path() /
("kyty_zarchive_test_!_" + std::to_string(unique));
Check(std::filesystem::create_directories(m_path),
"create temporary directory");
}
~TempDirectory() {
std::error_code error;
std::filesystem::remove_all(m_path, error);
}
[[nodiscard]] const std::filesystem::path &Path() const { return m_path; }
KYTY_CLASS_NO_COPY(TempDirectory);
private:
std::filesystem::path m_path;
};
template <typename T>
void SetValue(std::vector<std::byte> &bytes, size_t offset, T value) {
value = _ZARCHIVE::_store(value);
Check(offset <= bytes.size() && sizeof(value) <= bytes.size() - offset,
"patch lies inside fixture");
std::memcpy(bytes.data() + offset, &value, sizeof(value));
}
void SetMemberNameOffset(std::vector<std::byte> &bytes,
const _ZARCHIVE::Footer &footer, uint32_t name_offset) {
const auto entry_offset = footer.sectionFileTree.offset + sizeof(_ZARCHIVE::FileDirectoryEntry);
uint32_t original = 0;
std::memcpy(&original, bytes.data() + entry_offset, sizeof(original));
SetValue(bytes, entry_offset, (_ZARCHIVE::_load(original) & 0x80000000u) | name_offset);
}
template <typename Edit>
std::filesystem::path MutateArchive(const std::filesystem::path &source,
const char *name, Edit edit) {
Common::File input(source, Common::File::Mode::Read);
auto bytes = input.ReadWholeBuffer();
_ZARCHIVE::Footer footer{};
Check(bytes.size() >= sizeof(footer), "fixture has a footer");
std::memcpy(&footer, bytes.data() + bytes.size() - sizeof(footer), sizeof(footer));
_ZARCHIVE::Footer::Deserialize(&footer, &footer);
edit(bytes, footer);
_ZARCHIVE::Footer::Serialize(&footer, &footer);
std::memcpy(bytes.data() + bytes.size() - sizeof(footer), &footer, sizeof(footer));
const auto destination = source.parent_path() / name;
Common::File output;
Check(output.Create(destination), "create malformed archive fixture");
uint32_t written = 0;
output.Write(bytes.data(), static_cast<uint32_t>(bytes.size()), &written);
Check(written == bytes.size(), "write malformed archive fixture");
return destination;
}
void CheckMalformedArchives(const std::filesystem::path &source,
const std::vector<uint8_t> &payload) {
const auto bad_name = MutateArchive(source, "bad-name.zar", [](auto &bytes, auto &footer) {
SetMemberNameOffset(bytes, footer, static_cast<uint32_t>(footer.sectionNames.size));
});
Check(!Common::OpenArchive(bad_name), "reject name offset at the end of name table");
const auto bad_length = MutateArchive(source, "bad-length.zar", [](auto &bytes, auto &footer) {
SetMemberNameOffset(bytes, footer, static_cast<uint32_t>(footer.sectionNames.size - 1));
bytes[footer.sectionNames.offset + footer.sectionNames.size - 1] = std::byte{0x80};
});
Check(!Common::OpenArchive(bad_length), "reject truncated extended name length");
const auto bad_children = MutateArchive(source, "bad-children.zar", [](auto &bytes, auto &footer) {
SetValue(bytes, footer.sectionFileTree.offset + sizeof(uint32_t), UINT32_MAX);
});
Check(!Common::OpenArchive(bad_children), "reject invalid directory children without throwing");
const auto bad_range = MutateArchive(source, "bad-range.zar", [](auto &, auto &footer) {
footer.sectionCompressedData.offset = std::numeric_limits<uint64_t>::max();
});
Check(!Common::OpenArchive(bad_range), "reject overflowing footer section range");
const auto truncated = source.parent_path() / "truncated.zar";
Check(std::filesystem::copy_file(source, truncated), "copy truncated archive fixture");
std::filesystem::resize_file(truncated, std::filesystem::file_size(truncated) - 1);
Check(!Common::OpenArchive(truncated), "reject missing footer byte");
const auto corrupted = MutateArchive(source, "corrupt-data.zar", [](auto &bytes, auto &footer) {
_ZARCHIVE::CompressionOffsetRecord record{};
std::memcpy(&record, bytes.data() + footer.sectionOffsetRecords.offset, sizeof(record));
_ZARCHIVE::CompressionOffsetRecord::Deserialize(&record, 1, &record);
const auto second_block = footer.sectionCompressedData.offset + record.baseOffset + record.size[0] + 1;
std::fill_n(bytes.data() + second_block, record.size[1] + 1, std::byte{});
});
Common::File damaged(Common::MakeArchivePath(corrupted, "assets/subdir/data.bin"),
Common::File::Mode::Read);
Check(!damaged.IsInvalid(), "open archive with damaged compressed data");
std::vector<uint8_t> bytes(payload.size());
uint32_t read = 0;
damaged.Read(bytes.data(), static_cast<uint32_t>(bytes.size()), &read);
Check(read > 0 && read < 64 * 1024 && damaged.Tell() == read &&
std::equal(bytes.begin(), bytes.begin() + read, payload.begin()),
"report the valid prefix before a corrupt compressed block");
Check(damaged.Seek(2 * 64 * 1024), "seek past corrupt compressed block");
damaged.Read(bytes.data(), 97, &read);
Check(read == 97 && std::equal(bytes.begin(), bytes.begin() + read,
payload.begin() + 2 * 64 * 1024),
"recover by reading an unaffected compressed block");
}
#if !defined(_WIN32) || defined(_GLIBCXX_GCC_GTHR_POSIX_H)
void CheckCancellation(const std::filesystem::path &path,
const std::vector<uint8_t> &payload) {
// Keep the file and destination on the parent thread: cancellation must only release the
// reader thread's request and staging buffer after the worker has finished using them.
struct CancelledRead {
Common::File file;
std::vector<uint8_t> data;
std::atomic_bool ready = false;
std::atomic_bool start = false;
uint32_t bytes = 0;
} read;
Check(read.file.Open(path, Common::File::Mode::Read), "open cancellation fixture");
read.data.resize(payload.size());
pthread_t thread;
Check(pthread_create(&thread, nullptr, [](void *opaque) -> void * {
auto &read = *static_cast<CancelledRead *>(opaque);
read.ready = true;
while (!read.start.load()) {}
read.file.Read(read.data.data(), static_cast<uint32_t>(read.data.size()), &read.bytes);
pthread_testcancel();
return nullptr;
}, &read) == 0, "start cancellable archive reader");
while (!read.ready.load()) {}
Check(pthread_cancel(thread) == 0, "queue cancellation before the archive worker wait");
read.start = true;
void *result = nullptr;
Check(pthread_join(thread, &result) == 0 && result == PTHREAD_CANCELED,
"restore cancellation after the archive read");
Check(read.bytes == payload.size() && read.data == payload,
"finish worker reads and copies before releasing a cancelled caller");
int previous = PTHREAD_CANCEL_ENABLE;
int observed = PTHREAD_CANCEL_ENABLE;
pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &previous);
read.file.Seek(0);
read.file.Read(read.data.data(), 97, &read.bytes);
pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &observed);
pthread_setcancelstate(previous, nullptr);
Check(observed == PTHREAD_CANCEL_DISABLE && read.bytes == 97,
"preserve a caller's disabled cancellation state");
}
#endif
} // namespace
int main() {
TempDirectory temporary;
const auto archive_path =
temporary.Path() / std::filesystem::path(u8"game-テスト.zar");
std::vector<uint8_t> payload(3 * 1024 * 1024 + 37);
for (size_t index = 0; index < payload.size(); index++) {
payload[index] = static_cast<uint8_t>((index * 37 + 11) & 0xff);
}
Check(ArchiveTests::CreateArchive(archive_path, payload), "create archive fixture");
const auto native_file = Common::File::GetInfo(archive_path);
const auto native_directory = Common::File::GetInfo(temporary.Path() / "");
Check(native_file && native_file->is_file &&
native_file->size == std::filesystem::file_size(archive_path),
"query native file type and size together");
Check(native_directory && !native_directory->is_file && native_directory->size == 0,
"query native directory with trailing separator");
Check(!Common::File::GetInfo(archive_path / "") && Common::File::Size(archive_path / "") == 0 &&
!Common::File::GetInfo(temporary.Path() / "missing.bin"),
"reject missing native paths and files with a trailing separator");
const auto root = Common::MakeArchivePath(archive_path);
auto reader_pin = Common::OpenArchive(archive_path);
Check(reader_pin && Common::OpenArchive(root) == reader_pin,
"mounted archive pin reuses the same reader and index");
const auto archive_file = Common::File::GetInfo(root / "assets/subdir/data.bin");
const auto archive_directory = Common::File::GetInfo(root / "assets/");
Check(archive_file && archive_file->is_file && archive_file->size == payload.size(),
"query archive member type and size together");
Check(archive_directory && !archive_directory->is_file && archive_directory->size == 0 &&
!Common::File::GetInfo(root / "missing.bin"),
"distinguish archive directories from missing members");
Check(Common::IsSupportedArchive(archive_path) &&
Common::IsSupportedArchive("GAME.ZAR") &&
!Common::IsSupportedArchive("game.zip"),
"select an archive backend by supported extension");
Check(!Common::IsArchivePath(temporary.Path() / "ordinary!" / "asset.bin"),
"ordinary host paths with exclamation marks stay host paths");
Check(Common::IsArchivePath(root), "recognize virtual archive root");
Check(Common::GetArchiveHostPath(root) == archive_path,
"recover host archive path");
Check(Common::File::IsDirectoryExisting(root), "find archive root directory");
Check(Common::File::IsFileExisting(root / "EBOOT.BIN"),
"lookup is case insensitive");
Check(!Common::File::IsFileExisting(root / "missing.bin"),
"reject missing member");
Check(Common::IsArchivePath(root / "../outside.bin"),
"retain archive classification for invalid traversal");
Check(!Common::File::IsFileExisting(root / "../outside.bin"),
"reject leading parent traversal");
Check(Common::File::IsFileExisting(root / "assets/../eboot.bin"),
"normalize internal parent traversal");
Check(Common::File::IsFileExisting(root / ArchiveTests::LongFilename),
"decode extended-length archive names");
Check(Common::File::IsFileExisting(root / Common::PathFromUtf8(ArchiveTests::UnicodeFilename)),
"lookup Unicode archive member");
const auto entries = Common::File::GetDirEntries(root);
Check(std::ranges::any_of(entries,
[](const auto &entry) {
return entry.is_file && entry.name == "eboot.bin";
}),
"enumerate root file");
Check(std::ranges::any_of(entries,
[](const auto &entry) {
return !entry.is_file && entry.name == "assets";
}),
"enumerate root directory");
Check(std::ranges::any_of(entries, [](const auto &entry) {
return entry.is_file && entry.name == ArchiveTests::UnicodeFilename;
}), "enumerate Unicode member names as UTF-8");
const auto data_path = root / "assets/subdir/data.bin";
Check(Common::File::Size(data_path) == payload.size(), "report member size");
Common::File file(data_path, Common::File::Mode::Read);
Check(!file.IsInvalid(), "open archive member");
Check(file.Remaining() == payload.size(), "report initial remaining bytes");
Check(file.Seek(64 * 1024 - 19), "seek before compression block boundary");
std::vector<uint8_t> slice(97);
uint32_t bytes_read = 0;
file.Read(slice.data(), static_cast<uint32_t>(slice.size()), &bytes_read);
Check(bytes_read == slice.size(), "read across compression block boundary");
Check(
std::equal(slice.begin(), slice.end(), payload.begin() + 64 * 1024 - 19),
"preserve data across compression block boundary");
Check(file.Seek(payload.size() - 9), "seek near member end");
file.Read(slice.data(), static_cast<uint32_t>(slice.size()), &bytes_read);
Check(bytes_read == 9, "truncate read at member end");
Check(file.IsEOF(), "report member end of file");
Check(file.Seek(payload.size() + 100), "allow seek past member end");
Check(file.Remaining() == 0, "report no bytes remaining past member end");
file.Read(slice.data(), static_cast<uint32_t>(slice.size()), &bytes_read);
Check(bytes_read == 0, "read past member end returns no data");
Check(file.Seek(0), "seek to member start");
std::vector<uint8_t> full(payload.size());
file.Read(full.data(), static_cast<uint32_t>(full.size()), &bytes_read);
Check(bytes_read == full.size(), "read a multi-chunk member");
Check(full == payload, "preserve a multi-chunk member");
std::atomic_bool concurrent_reads_ok = true;
std::vector<std::thread> readers;
for (uint32_t worker = 0; worker < 8; worker++) {
readers.emplace_back([&, worker] {
Common::File concurrent_file(data_path, Common::File::Mode::Read);
std::array<uint8_t, 4096> concurrent_data{};
for (uint32_t iteration = 0; iteration < 16; iteration++) {
const auto offset =
static_cast<size_t>(worker * 7919 + iteration * 12347) %
(payload.size() - concurrent_data.size());
uint32_t concurrent_bytes_read = 0;
if (concurrent_file.IsInvalid() || !concurrent_file.Seek(offset)) {
concurrent_reads_ok = false;
return;
}
concurrent_file.Read(concurrent_data.data(), concurrent_data.size(),
&concurrent_bytes_read);
if (concurrent_bytes_read != concurrent_data.size() ||
!std::equal(concurrent_data.begin(), concurrent_data.end(),
payload.begin() +
static_cast<std::ptrdiff_t>(offset))) {
concurrent_reads_ok = false;
return;
}
}
});
}
for (auto &reader : readers) {
reader.join();
}
Check(concurrent_reads_ok, "read concurrently through independent file handles");
#if !defined(_WIN32) || defined(_GLIBCXX_GCC_GTHR_POSIX_H)
CheckCancellation(data_path, payload);
#endif
#if !defined(_WIN32)
// Guest threads have much smaller stacks than std::thread's default stack.
pthread_attr_t attributes;
Check(pthread_attr_init(&attributes) == 0 &&
pthread_attr_setstacksize(&attributes, 64 * 1024) == 0,
"configure a guest-sized stack");
struct SmallStackRead {
const std::filesystem::path &path;
const std::vector<uint8_t> &expected;
bool passed = false;
} small_stack{data_path, payload};
pthread_t thread;
Check(pthread_create(&thread, &attributes, [](void *opaque) -> void * {
auto &test = *static_cast<SmallStackRead *>(opaque);
Common::File input(test.path, Common::File::Mode::Read);
auto data = input.ReadWholeBuffer();
test.passed = data.size() == test.expected.size() &&
std::memcmp(data.data(), test.expected.data(), data.size()) == 0;
return nullptr;
}, &small_stack) == 0, "start guest-sized-stack reader");
pthread_attr_destroy(&attributes);
Check(pthread_join(thread, nullptr) == 0 && small_stack.passed,
"decompress a multi-block file on a guest-sized stack");
#endif
Common::File metadata(root / "sce_sys/param.json", Common::File::Mode::Read);
const auto metadata_data = metadata.ReadWholeBuffer();
const std::string metadata_text(
reinterpret_cast<const char *>(metadata_data.data()),
metadata_data.size());
Check(metadata_text.find("Archive Test") != std::string::npos,
"read metadata member");
Common::File writable;
Check(!writable.Open(data_path, Common::File::Mode::ReadWrite),
"reject writable member open");
Check(!Common::File::DeleteFile(data_path), "reject member deletion");
Check(!Common::File::CreateDirectory(root / "new-dir"),
"reject member directory creation");
Check(!file.Truncate(0) && !file.Unlink(), "reject descriptor mutations");
const auto copied = temporary.Path() / "copied.bin";
Common::File host_fixture;
Check(host_fixture.Create(copied), "create host mutation fixture");
host_fixture.Close();
Check(!Common::File::CopyFile(data_path, copied) &&
!Common::File::CopyFile(copied, data_path) &&
!Common::File::RenameFile(data_path, copied) &&
!Common::File::RenameFile(copied, data_path),
"reject unsupported copy and rename operations involving archives");
CheckMalformedArchives(archive_path, payload);
file.Close();
metadata.Close();
Check(Common::OpenArchive(archive_path) == reader_pin,
"mount pin retains the archive reader after the last member closes");
reader_pin.reset();
std::error_code remove_error;
Check(std::filesystem::remove(archive_path, remove_error) && !remove_error,
"release archive handle after the final reader closes");
const std::array<uint8_t, 3> replacement{42, 7, 19};
Check(ArchiveTests::CreateArchive(archive_path, replacement), "replace a closed archive");
Common::File replaced(data_path, Common::File::Mode::Read);
const auto replaced_data = replaced.ReadWholeBuffer();
Check(replaced_data.size() == replacement.size() &&
std::memcmp(replaced_data.data(), replacement.data(), replacement.size()) == 0,
"reopening after the final reader closes sees replacement contents");
std::printf("ArchiveFileTests: all cases passed\n");
return 0;
}
Reference in New Issue View Git Blame Copy Permalink