diff --git a/.gitignore b/.gitignore index 15a3772cd8..38e675d875 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,7 @@ libraries/win64 libraries/source/cpp-httplib/* libraries/source/cxxtest-4.4/* libraries/source/fcollada/* +libraries/source/libzip/* libraries/source/nvtt/* libraries/source/premake-core/* libraries/source/spidermonkey/* diff --git a/build/premake/extern_libs5.lua b/build/premake/extern_libs5.lua index 4b916edca4..a772d797bc 100644 --- a/build/premake/extern_libs5.lua +++ b/build/premake/extern_libs5.lua @@ -68,6 +68,10 @@ if os.istarget("macosx") then pkgconfig.set_static_link_libs(true) end +if not _OPTIONS["with-system-libzip"] then + pkgconfig.add_pkg_config_path(libraries_source_dir .. "libzip/lib/pkgconfig/") +end + local function add_delayload(name, suffix, def) if def["no_delayload"] then @@ -532,6 +536,25 @@ extern_lib_defs = { end end, }, + libzip = { + compile_settings = function() + if os.istarget("windows") then + add_default_include_paths("libzip") + else + pkgconfig.add_includes("libzip") + end + end, + link_settings = function() + if os.istarget("windows") then + add_default_lib_paths("libzip") + add_default_links({ + win_names = { "zip" }, + }) + else + pkgconfig.add_links("libzip") + end + end, + }, miniupnpc = { compile_settings = function() if os.istarget("windows") then diff --git a/build/premake/premake5.lua b/build/premake/premake5.lua index dedc3713a2..5ef24180e0 100644 --- a/build/premake/premake5.lua +++ b/build/premake/premake5.lua @@ -52,6 +52,7 @@ newoption { category = "Pyrogenesis", trigger = "with-system-cpp-httplib", descr newoption { category = "Pyrogenesis", trigger = "with-system-cxxtest", description = "Search standard paths for cxxtest, instead of using bundled copy" } newoption { category = "Pyrogenesis", trigger = "with-lto", description = "Enable Link Time Optimization (LTO)" } newoption { category = "Pyrogenesis", trigger = "with-system-mozjs", description = "Search standard paths for libmozjs128, instead of using bundled copy" } +newoption { category = "Pyrogenesis", trigger = "with-system-libzip", description = "Search standard paths for libzip, instead of using bundled copy" } newoption { category = "Pyrogenesis", trigger = "with-system-nvtt", description = "Search standard paths for nvidia-texture-tools library, instead of using bundled copy" } newoption { category = "Pyrogenesis", trigger = "with-valgrind", description = "Enable Valgrind support (non-Windows only)" } newoption { category = "Pyrogenesis", trigger = "without-audio", description = "Disable use of OpenAL/Ogg/Vorbis APIs" } @@ -994,6 +995,7 @@ function setup_all_libs () "valgrind", "cxxtest", "fmt", + "libzip", } -- CPU architecture-specific @@ -1090,6 +1092,7 @@ used_extern_libs = { "libpng", "zlib", + "libzip", "spidermonkey", "libxml2", diff --git a/libraries/build-macos-libs.sh b/libraries/build-macos-libs.sh index c388de9ddb..f6c9a7be1f 100755 --- a/libraries/build-macos-libs.sh +++ b/libraries/build-macos-libs.sh @@ -1248,6 +1248,11 @@ export ARCH CXXFLAGS CFLAGS LDFLAGS CMAKE_FLAGS JOBS # shellcheck disable=SC2086 ./../source/fcollada/build.sh $build_sh_options || die "FCollada build failed" +# -------------------------------------------------------------- +# shellcheck disable=SC2086 +./../source/libzip/build.sh $build_sh_options || die "libzip build failed" +cp ./../source/libzip/lib/pkgconfig/* "$PC_PATH" + # -------------------------------------------------------------- # shellcheck disable=SC2086 ./../source/nvtt/build.sh $build_sh_options || die "NVTT build failed" diff --git a/libraries/build-source-libs.sh b/libraries/build-source-libs.sh index b411d4afb4..7d9c801118 100755 --- a/libraries/build-source-libs.sh +++ b/libraries/build-source-libs.sh @@ -33,6 +33,7 @@ options: --force-rebuild - rebuild all --without-nvtt - don't build nvtt --with-system-cxxtest - don't build cxxtest + --with-system-libzip - don't build libzip --with-system-nvtt - don't build nvtt --with-system-mozjs - don't build spidermonkey --with-system-premake - don't build premake @@ -44,6 +45,7 @@ EOF without_nvtt=false with_system_cpp_httplib=false with_system_cxxtest=false +with_system_libzip=false with_system_nvtt=false with_system_mozjs=false with_system_premake=false @@ -62,6 +64,7 @@ while [ "$#" -gt 0 ]; do --without-nvtt) without_nvtt=true ;; --with-system-cpp-httplib) with_system_cpp_httplib=true ;; --with-system-cxxtest) with_system_cxxtest=true ;; + --with-system-libzip) with_system_libzip=true ;; --with-system-nvtt) with_system_nvtt=true ;; --with-system-mozjs) with_system_mozjs=true ;; --with-system-premake) with_system_premake=true ;; @@ -102,6 +105,11 @@ if [ "$with_system_cxxtest" = "false" ]; then fi # shellcheck disable=SC2086 ./source/fcollada/build.sh $build_sh_options || die "FCollada build failed" +if [ "$with_system_libzip" = "false" ]; then + # shellcheck disable=SC2086 + ./source/libzip/build.sh $build_sh_options || die "libzip build failed" + cp source/libzip/lib/*so* ../binaries/system/ +fi if [ "$with_system_nvtt" = "false" ] && [ "$without_nvtt" = "false" ]; then # shellcheck disable=SC2086 ./source/nvtt/build.sh $build_sh_options || die "NVTT build failed" diff --git a/libraries/get-windows-libs.bat b/libraries/get-windows-libs.bat index 65bf3b35c8..49658c9be6 100644 --- a/libraries/get-windows-libs.bat +++ b/libraries/get-windows-libs.bat @@ -22,7 +22,7 @@ rem **Copy dependencies' binaries to binaries/system/** rem static libs: boost fmt rem wxwidgets isn't provided and needs to be built manually -set DIR_LIST=cpp-httplib enet fcollada freetype gloox iconv icu libcurl libpng libsodium libxml2 microsoft miniupnpc nvtt openal sdl2 spidermonkey vorbis zlib +set DIR_LIST=cpp-httplib enet fcollada freetype gloox iconv icu libcurl libpng libsodium libxml2 libzip microsoft miniupnpc nvtt openal sdl2 spidermonkey vorbis zlib for %%d in (%DIR_LIST%) do ( copy /y %LIBS_PATH%\%%d\bin\* ..\binaries\system\ || exit /b 1 ) diff --git a/libraries/source/libzip/build.sh b/libraries/source/libzip/build.sh new file mode 100755 index 0000000000..46bae1c4d8 --- /dev/null +++ b/libraries/source/libzip/build.sh @@ -0,0 +1,80 @@ +#!/bin/sh +set -e + +: "${OS:=$(uname -s)}" +: "${TAR:=tar}" + +cd "$(dirname "$0")" + +PV=1.11.4 +LIB_VERSION=${PV}+wfg0 + +fetch() +{ + curl -fLo "libzip-${PV}.tar.xz" \ + "https://libzip.org/download/libzip-${PV}.tar.xz" +} + +echo "Building libzip..." +while [ "$#" -gt 0 ]; do + case "$1" in + --fetch-only) + fetch + exit + ;; + --force-rebuild) rm -f .already-built ;; + *) + echo "Unknown option: $1" + exit 1 + ;; + esac + shift +done + +if [ -e .already-built ] && [ "$(cat .already-built || true)" = "${LIB_VERSION}" ]; then + echo "Skipping - already built (use --force-rebuild to override)" + exit +fi + +# fetch +if [ ! -e "libzip-${PV}.tar.xz" ]; then + fetch +fi + +# unpack +rm -Rf "libzip-${PV}" +"${TAR}" xf "libzip-${PV}.tar.xz" + +# configure +rm -rf build +shared_libs=ON +if [ "${OS}" = "Darwin" ]; then + shared_libs=OFF +fi +cmake -B build -S "libzip-${PV}" \ + -DCMAKE_INSTALL_PREFIX="$(realpath . || true)" \ + -DCMAKE_INSTALL_LIBDIR=lib \ + -DBUILD_SHARED_LIBS=${shared_libs} \ + -DENABLE_COMMONCRYPTO=OFF \ + -DENABLE_GNUTLS=OFF \ + -DENABLE_MBEDTLS=OFF \ + -DENABLE_OPENSSL=OFF \ + -DENABLE_WINDOWS_CRYPTO=OFF \ + -DENABLE_BZIP2=OFF \ + -DENABLE_LZMA=OFF \ + -DENABLE_ZSTD=OFF \ + -DENABLE_FDOPEN=OFF \ + -DBUILD_TOOLS=OFF \ + -DBUILD_REGRESS=OFF \ + -DBUILD_OSSFUZZ=OFF \ + -DBUILD_EXAMPLES=OFF \ + -DBUILD_DOC=OFF + +# build +cmake --build build + +# install +rm -Rf bin include lib +cmake --install build + +echo "${LIB_VERSION}" >.already-built diff --git a/source/lib/file/archive/archive_zip.cpp b/source/lib/file/archive/archive_zip.cpp index a6e011a51a..1d49b6246d 100644 --- a/source/lib/file/archive/archive_zip.cpp +++ b/source/lib/file/archive/archive_zip.cpp @@ -1,4 +1,4 @@ -/* Copyright (C) 2025 Wildfire Games. +/* Copyright (C) 2026 Wildfire Games. * * Permission is hereby granted, free of charge, to any person obtaining * a copy of this software and associated documentation files (the @@ -28,296 +28,20 @@ #include "archive_zip.h" -#include "lib/alignment.h" -#include "lib/allocators/dynarray.h" -#include "lib/allocators/pool.h" -#include "lib/bits.h" -#include "lib/byte_order.h" -#include "lib/code_annotation.h" #include "lib/debug.h" #include "lib/file/archive/archive.h" -#include "lib/file/archive/codec.h" -#include "lib/file/archive/codec_zlib.h" -#include "lib/file/archive/stream.h" -#include "lib/file/file.h" -#include "lib/file/file_system.h" -#include "lib/file/io/io.h" -#include "lib/lib.h" -#include "lib/path.h" -#include "lib/posix/posix_types.h" +#include "lib/os_path.h" #include "lib/status.h" #include "lib/types.h" -#include "lib/utf8.h" -#include -#include -#include -#include -#include -#include -#include -#include +#include +#include -//----------------------------------------------------------------------------- -// timestamp conversion: DOS FAT <-> Unix time_t -//----------------------------------------------------------------------------- - -static time_t time_t_from_FAT(u32 fat_timedate) -{ - const u32 fat_time = bits(fat_timedate, 0, 15); - const u32 fat_date = bits(fat_timedate, 16, 31); - - struct tm t; // struct tm format: - t.tm_sec = bits(fat_time, 0,4) * 2; // [0,59] - t.tm_min = bits(fat_time, 5,10); // [0,59] - t.tm_hour = bits(fat_time, 11,15); // [0,23] - t.tm_mday = bits(fat_date, 0,4); // [1,31] - t.tm_mon = bits(fat_date, 5,8) - 1; // [0,11] - t.tm_year = bits(fat_date, 9,15) + 80; // since 1900 - t.tm_isdst = -1; // unknown - let libc determine - - // otherwise: totally bogus, and at the limit of 32-bit time_t - ENSURE(t.tm_year < 138); - - time_t ret = mktime(&t); - ENSURE(ret != (time_t)-1); // mktime shouldn't fail - return ret; -} - - -static u32 FAT_from_time_t(time_t time) -{ - // (values are adjusted for DST) - struct tm* t = localtime(&time); - - const u16 fat_time = u16( - (t->tm_sec/2) | // 5 - (u16(t->tm_min) << 5) | // 6 - (u16(t->tm_hour) << 11) // 5 - ); - - const u16 fat_date = u16( - (t->tm_mday) | // 5 - (u16(t->tm_mon+1) << 5) | // 4 - (u16(t->tm_year-80) << 9) // 7 - ); - - u32 fat_timedate = u32_from_u16(fat_date, fat_time); - return fat_timedate; -} - - -//----------------------------------------------------------------------------- -// Zip archive definitions -//----------------------------------------------------------------------------- - -static const u32 cdfh_magic = FOURCC_LE('P','K','\1','\2'); -static const u32 lfh_magic = FOURCC_LE('P','K','\3','\4'); -static const u32 ecdr_magic = FOURCC_LE('P','K','\5','\6'); - -enum ZipMethod -{ - ZIP_METHOD_NONE = 0, - ZIP_METHOD_DEFLATE = 8 -}; - -#pragma pack(push, 1) - -class LFH +class ArchiveFile_Zip : public IArchiveFile { public: - void Init(const CFileInfo& fileInfo, off_t csize, ZipMethod method, u32 checksum, const Path& pathname) - { - const std::string pathnameUTF8 = utf8_from_wstring(pathname.string()); - const size_t pathnameSize = pathnameUTF8.length(); - - m_magic = lfh_magic; - m_x1 = to_le16(0); - m_flags = to_le16(0); - m_method = to_le16(u16_from_larger(method)); - m_fat_mtime = to_le32(FAT_from_time_t(fileInfo.MTime())); - m_crc = to_le32(checksum); - m_csize = to_le32(u32_from_larger(csize)); - m_usize = to_le32(u32_from_larger(fileInfo.Size())); - m_fn_len = to_le16(u16_from_larger(pathnameSize)); - m_e_len = to_le16(0); - - memcpy((char*)this + sizeof(LFH), pathnameUTF8.c_str(), pathnameSize); - } - - size_t Size() const - { - ENSURE(m_magic == lfh_magic); - size_t size = sizeof(LFH); - size += read_le16(&m_fn_len); - size += read_le16(&m_e_len); - // note: LFH doesn't have a comment field! - return size; - } - -private: - u32 m_magic; - u16 m_x1; // version needed - u16 m_flags; - u16 m_method; - u32 m_fat_mtime; // last modified time (DOS FAT format) - u32 m_crc; - u32 m_csize; - u32 m_usize; - u16 m_fn_len; - u16 m_e_len; -}; - -cassert(sizeof(LFH) == 30); - - -class CDFH -{ -public: - void Init(const CFileInfo& fileInfo, off_t ofs, off_t csize, ZipMethod method, u32 checksum, const Path& pathname, size_t slack) - { - const std::string pathnameUTF8 = utf8_from_wstring(pathname.string()); - const size_t pathnameLength = pathnameUTF8.length(); - - m_magic = cdfh_magic; - m_x1 = to_le32(0); - m_flags = to_le16(0); - m_method = to_le16(u16_from_larger(method)); - m_fat_mtime = to_le32(FAT_from_time_t(fileInfo.MTime())); - m_crc = to_le32(checksum); - m_csize = to_le32(u32_from_larger(csize)); - m_usize = to_le32(u32_from_larger(fileInfo.Size())); - m_fn_len = to_le16(u16_from_larger(pathnameLength)); - m_e_len = to_le16(0); - m_c_len = to_le16(u16_from_larger((size_t)slack)); - m_x2 = to_le32(0); - m_x3 = to_le32(0); - m_lfh_ofs = to_le32(u32_from_larger(ofs)); - - memcpy((char*)this + sizeof(CDFH), pathnameUTF8.c_str(), pathnameLength); - } - - Path Pathname() const - { - const size_t length = (size_t)read_le16(&m_fn_len); - const char* pathname = (const char*)this + sizeof(CDFH); // not 0-terminated! - return Path(std::string(pathname, length)); - } - - off_t HeaderOffset() const - { - return read_le32(&m_lfh_ofs); - } - - off_t USize() const - { - return (off_t)read_le32(&m_usize); - } - - off_t CSize() const - { - return (off_t)read_le32(&m_csize); - } - - ZipMethod Method() const - { - return (ZipMethod)read_le16(&m_method); - } - - u32 Checksum() const - { - return read_le32(&m_crc); - } - - time_t MTime() const - { - const u32 fat_mtime = read_le32(&m_fat_mtime); - return time_t_from_FAT(fat_mtime); - } - - size_t Size() const - { - size_t size = sizeof(CDFH); - size += read_le16(&m_fn_len); - size += read_le16(&m_e_len); - size += read_le16(&m_c_len); - return size; - } - -private: - u32 m_magic; - u32 m_x1; // versions - u16 m_flags; - u16 m_method; - u32 m_fat_mtime; // last modified time (DOS FAT format) - u32 m_crc; - u32 m_csize; - u32 m_usize; - u16 m_fn_len; - u16 m_e_len; - u16 m_c_len; - u32 m_x2; // spanning - u32 m_x3; // attributes - u32 m_lfh_ofs; -}; - -cassert(sizeof(CDFH) == 46); - - -class ECDR -{ -public: - void Init(size_t cd_numEntries, off_t cd_ofs, size_t cd_size) - { - m_magic = ecdr_magic; - m_diskNum = to_le16(0); - m_cd_diskNum = to_le16(0); - m_cd_numEntriesOnDisk = to_le16(u16_from_larger(cd_numEntries)); - m_cd_numEntries = m_cd_numEntriesOnDisk; - m_cd_size = to_le32(u32_from_larger(cd_size)); - m_cd_ofs = to_le32(u32_from_larger(cd_ofs)); - m_comment_len = to_le16(0); - } - - void Decompose(size_t& cd_numEntries, off_t& cd_ofs, size_t& cd_size) const - { - cd_numEntries = (size_t)read_le16(&m_cd_numEntries); - cd_ofs = (off_t)read_le32(&m_cd_ofs); - cd_size = (size_t)read_le32(&m_cd_size); - } - - off_t GetCommentLength() const - { - return static_cast(read_le16(&m_comment_len)); - } - -private: - u32 m_magic; - u16 m_diskNum; - u16 m_cd_diskNum; - u16 m_cd_numEntriesOnDisk; - u16 m_cd_numEntries; - u32 m_cd_size; - u32 m_cd_ofs; - u16 m_comment_len; -}; - -cassert(sizeof(ECDR) == 22); - -#pragma pack(pop) - - -//----------------------------------------------------------------------------- -// ArchiveFile_Zip -//----------------------------------------------------------------------------- - -class ArchiveFile_Zip final : public IArchiveFile -{ -public: - ArchiveFile_Zip(const PFile& file, off_t ofs, off_t csize, u32 checksum, ZipMethod method) - : m_file(file), m_ofs(ofs) - , m_csize(csize), m_checksum(checksum), m_method((u16)method) - , m_flags(NeedsFixup) + ArchiveFile_Zip(std::shared_ptr zip, zip_uint64_t index, OsPath path) + : m_Zip(std::move(zip)), m_Index(index), m_Path(path) { } @@ -333,286 +57,179 @@ public: const OsPath& Path() const override { - return m_file->Pathname(); + return m_Path; } Status Load(const OsPath& /*name*/, const std::shared_ptr& buf, size_t size) const override { - AdjustOffset(); - - PICodec codec; - switch(m_method) + zip_file_t* zipFile = zip_fopen_index(m_Zip.get(), m_Index, 0); + if (zipFile == nullptr) { - case ZIP_METHOD_NONE: - codec = CreateCodec_ZLibNone(); - break; - case ZIP_METHOD_DEFLATE: - codec = CreateDecompressor_ZLibDeflate(); - break; - default: - WARN_RETURN(ERR::ARCHIVE_UNKNOWN_METHOD); + debug_printf("Failed to open file at zip index '%" PRIu64 "'\n", m_Index); + return ERR::FAIL; } - Stream stream(codec); - stream.SetOutputBuffer(buf.get(), size); - io::Operation op(*m_file.get(), 0, m_csize, m_ofs); - StreamFeeder streamFeeder(stream); - RETURN_STATUS_IF_ERR(io::Run(op, io::Parameters(), streamFeeder)); - RETURN_STATUS_IF_ERR(stream.Finish()); -#if CODEC_COMPUTE_CHECKSUM - ENSURE(m_checksum == stream.Checksum()); -#endif + if (zip_fread(zipFile, buf.get(), size) < 0) + { + debug_printf("Failed to read file at zip index '%" PRIu64 "'\n", m_Index); + zip_fclose(zipFile); + return ERR::FAIL; + } + + if (zip_fclose(zipFile) != 0) + { + debug_printf("Failed to close file at zip index '%" PRIu64 "'\n", m_Index); + return ERR::FAIL; + } return INFO::OK; } private: - enum Flags - { - // indicates m_ofs points to a "local file header" instead of - // the file data. a fixup routine is called when reading the file; - // it skips past the LFH and clears this flag. - // this is somewhat of a hack, but vital to archive open performance. - // without it, we'd have to scan through the entire archive file, - // which can take *seconds*. - // (we cannot use the information in CDFH, because its 'extra' field - // has been observed to differ from that of the LFH) - // since we read the LFH right before the rest of the file, the block - // cache will absorb the IO cost. - NeedsFixup = 1 - }; - - struct LFH_Copier - { - LFH_Copier(u8* lfh_dst, size_t lfh_bytes_remaining) - : lfh_dst(lfh_dst), lfh_bytes_remaining(lfh_bytes_remaining) - { - } - - // this code grabs an LFH struct from file block(s) that are - // passed to the callback. usually, one call copies the whole thing, - // but the LFH may straddle a block boundary. - // - // rationale: this allows using temp buffers for zip_fixup_lfh, - // which avoids involving the file buffer manager and thus - // avoids cluttering the trace and cache contents. - Status operator()(const u8* block, size_t size) const - { - ENSURE(size <= lfh_bytes_remaining); - memcpy(lfh_dst, block, size); - lfh_dst += size; - lfh_bytes_remaining -= size; - - return INFO::OK; - } - - mutable u8* lfh_dst; - mutable size_t lfh_bytes_remaining; - }; - - /** - * fix up m_ofs (adjust it to point to cdata instead of the LFH). - * - * note: we cannot use CDFH filename and extra field lengths to skip - * past LFH since that may not mirror CDFH (has happened). - * - * this is called at file-open time instead of while mounting to - * reduce seeks: since reading the file will typically follow, the - * block cache entirely absorbs the IO cost. - **/ - void AdjustOffset() const - { - if(!(m_flags & NeedsFixup)) - return; - m_flags &= ~NeedsFixup; - - // performance note: this ends up reading one file block, which is - // only in the block cache if the file starts in the same block as a - // previously read file (i.e. both are small). - LFH lfh; - io::Operation op(*m_file.get(), 0, sizeof(LFH), m_ofs); - if(io::Run(op, io::Parameters(), LFH_Copier((u8*)&lfh, sizeof(LFH))) == INFO::OK) - m_ofs += (off_t)lfh.Size(); - } - - PFile m_file; - - // all relevant LFH/CDFH fields not covered by CFileInfo - mutable off_t m_ofs; - off_t m_csize; - u32 m_checksum; - u16 m_method; - mutable u16 m_flags; + std::shared_ptr m_Zip; + zip_uint64_t m_Index; + OsPath m_Path; }; -//----------------------------------------------------------------------------- -// ArchiveReader_Zip -//----------------------------------------------------------------------------- +struct ZipArchiveDeleter +{ + void operator()(zip_t* zip) const noexcept + { + if (zip_close(zip) < 0) + { + debug_printf("archive-deleter: cannot close archive : %s\n", zip_strerror(zip)); + zip_discard(zip); + } + } +}; + class ArchiveReader_Zip : public IArchiveReader { public: - ArchiveReader_Zip(const OsPath& pathname) - : m_file(new File(pathname, O_RDONLY)) + ArchiveReader_Zip(const OsPath& archivePath) { - CFileInfo fileInfo; - GetFileInfo(pathname, &fileInfo); - m_fileSize = fileInfo.Size(); - const size_t minFileSize = sizeof(LFH)+sizeof(CDFH)+sizeof(ECDR); - ENSURE(m_fileSize >= off_t(minFileSize)); + int err; + zip_t* zip = zip_open(archivePath.string8().c_str(), 0, &err); + if (zip == nullptr) + { + zip_error_t error; + zip_error_init_with_code(&error, err); + std::runtime_error exception{"archive-reader: cannot open input archive " + archivePath.string8() + ": " + zip_error_strerror(&error)}; + zip_error_fini(&error); + throw exception; + } + m_Zip = std::shared_ptr(zip, ZipArchiveDeleter()); } - virtual Status ReadEntries(ArchiveEntryCallback cb, uintptr_t cbData) + Status ReadEntries(ArchiveEntryCallback cb, uintptr_t cbData) override { - // locate and read Central Directory - off_t cd_ofs = 0; - size_t cd_numEntries = 0; - size_t cd_size = 0; - RETURN_STATUS_IF_ERR(LocateCentralDirectory(m_file, m_fileSize, cd_ofs, cd_numEntries, cd_size)); - io::BufferPtr buf(io::Allocate(cd_size)); + Status returnStatus = INFO::OK; - io::Operation op(*m_file.get(), buf.get(), cd_size, cd_ofs); - RETURN_STATUS_IF_ERR(io::Run(op)); - - // iterate over Central Directory - const u8* pos = buf.get(); - for(size_t i = 0; i < cd_numEntries; i++) + const zip_int64_t numEntries{zip_get_num_entries(m_Zip.get(), 0)}; + if (numEntries < 0) { - // scan for next CDFH - CDFH* cdfh = (CDFH*)FindRecord(buf.get(), cd_size, pos, cdfh_magic, sizeof(CDFH)); - if(!cdfh) - WARN_RETURN(ERR::CORRUPTED); + debug_printf("Can't get entries count for null zip"); + } - const Path relativePathname(cdfh->Pathname()); + const zip_uint64_t indices{static_cast>(numEntries)}; + for (zip_uint64_t index = 0; index < indices; ++index) + { + zip_stat_t zipStat; + if (zip_stat_index(m_Zip.get(), index, 0, &zipStat) < 0) + { + debug_printf("Can't get zip stat for index '%" PRIu64 "'", index); + returnStatus = ERR::FAIL; + continue; + } + const Path relativePathname(zipStat.name); if(!relativePathname.IsDirectory()) { const OsPath name = relativePathname.Filename(); - CFileInfo fileInfo(name, cdfh->USize(), cdfh->MTime()); - std::shared_ptr archiveFile = std::make_shared(m_file, cdfh->HeaderOffset(), cdfh->CSize(), cdfh->Checksum(), cdfh->Method()); + CFileInfo fileInfo(name, zipStat.size, zipStat.mtime); + std::shared_ptr archiveFile = std::make_shared(m_Zip, index, name); cb(relativePathname, fileInfo, archiveFile, cbData); } - - pos += cdfh->Size(); } - return INFO::OK; + return returnStatus; } private: - /** - * Scan buffer for a Zip file record. - * - * @param buf - * @param size - * @param start position within buffer - * @param magic signature of record - * @param recordSize size of record (including signature) - * @return pointer to record within buffer or 0 if not found. - **/ - static const u8* FindRecord(const u8* buf, size_t size, const u8* start, u32 magic, size_t recordSize) - { - // (don't use as the counter - otherwise we can't tell if - // scanning within the buffer was necessary.) - for(const u8* p = start; p <= buf+size-recordSize; p++) - { - // found it - if(*(u32*)p == magic) - { - ENSURE(p == start); // otherwise, the archive is a bit broken - return p; - } - } + std::shared_ptr m_Zip; +}; - // passed EOF, didn't find it. - // note: do not warn - this happens in the initial ECDR search at - // EOF if the archive contains a comment field. - return 0; + +class ArchiveWriter_Zip : public IArchiveWriter +{ +public: + ArchiveWriter_Zip(const OsPath& archivePath, bool noDeflate) + : m_NoDeflate(noDeflate) + { + int err; + zip_t* zip = zip_open(archivePath.string8().c_str(), ZIP_CREATE | ZIP_EXCL, &err); + if (zip == nullptr) + { + zip_error_t error; + zip_error_init_with_code(&error, err); + std::runtime_error exception{"archive-writer: cannot open output archive " + archivePath.string8() + ": " + zip_error_strerror(&error)}; + zip_error_fini(&error); + throw exception; + } + m_Zip = std::unique_ptr(zip); } - // search for ECDR in the last bytes of the file. - // if found, fill with a copy of the (little-endian) ECDR and - // return INFO::OK, otherwise IO error or ERR::CORRUPTED. - static Status ScanForEcdr(const PFile& file, off_t fileSize, u8* buf, size_t maxScanSize, size_t& cd_numEntries, off_t& cd_ofs, size_t& cd_size) + Status AddFile(const OsPath& pathname, const OsPath& pathnameInArchive) override { - // don't scan more than the entire file - const size_t scanSize = std::min(maxScanSize, size_t(fileSize)); - - // read desired chunk of file into memory - const off_t ofs = fileSize - off_t(scanSize); - io::Operation op(*file.get(), buf, scanSize, ofs); - RETURN_STATUS_IF_ERR(io::Run(op)); - - // Scanning for ECDR first assumes no comment exists - // (standard case), so ECDR structure exists right at - // end of file - off_t offsetInBlock = scanSize - sizeof(ECDR); - const ECDR* ecdr = nullptr; - - for (off_t commentSize = 0; commentSize <= offsetInBlock && !ecdr; ++commentSize) + zip_error_t error; + zip_source_t* zipSource = zip_source_file_create(pathname.string8().c_str(), 0, ZIP_LENGTH_TO_END, &error); + if (zipSource == nullptr) { - const u8 *pECDRTest = buf + offsetInBlock - commentSize; - if (*reinterpret_cast(pECDRTest) == ecdr_magic) - { - // Signature matches, test whether comment - // fills up the whole space following the - // ECDR - ecdr = reinterpret_cast(pECDRTest); - if (commentSize != ecdr->GetCommentLength()) - { - // Signature matches but there is some other data between - // header, comment and EOF. There are three possibilities - // for this: - // 1) Header file format and size differ from what we expect - // 2) File has been truncated - // 3) The magic id occurs inside a zip comment - ecdr = nullptr; - } - else - { - // Seems like a valid archive header before an archive-level - // comment - break; - } - } + debug_printf("Can't open zip source file '%s' : %s\n", pathname.string8().c_str(), zip_error_strerror(&error)); + zip_error_fini(&error); + return ERR::FAIL; } + return AddZipSource(zipSource, pathnameInArchive); + } - if(!ecdr) - return INFO::CANNOT_HANDLE; + Status AddMemory(const u8* data, size_t size, time_t /*mtime*/, const OsPath& pathnameInArchive) override + { + zip_error_t error; + zip_source_t* zipSource = zip_source_buffer_create(data, size, 0, &error); + if (zipSource == nullptr) + { + debug_printf("Can't open zip source buffer : %s\n", zip_error_strerror(&error)); + zip_error_fini(&error); + return ERR::FAIL; + } + return AddZipSource(zipSource, pathnameInArchive); + } - ecdr->Decompose(cd_numEntries, cd_ofs, cd_size); +private: + std::unique_ptr m_Zip; + bool m_NoDeflate; + + Status AddZipSource(zip_source_t* zipSource, const OsPath& pathnameInArchive) + { + const zip_int64_t index{zip_file_add(m_Zip.get(), pathnameInArchive.string8().c_str(), zipSource, ZIP_FL_ENC_UTF_8)}; + if (index < 0) + { + debug_printf("Failed to add zip source as '%s'\n", pathnameInArchive.string8().c_str()); + zip_source_free(zipSource); + return ERR::FAIL; + } + const zip_int32_t comp{m_NoDeflate ? ZIP_CM_STORE : ZIP_CM_DEFLATE}; + if (zip_set_file_compression(m_Zip.get(), index, comp, 0) < 0) + { + debug_printf("Failed to set compression for '%s'\n", pathnameInArchive.string8().c_str()); + return ERR::FAIL; + } return INFO::OK; } - - static Status LocateCentralDirectory(const PFile& file, off_t fileSize, off_t& cd_ofs, size_t& cd_numEntries, size_t& cd_size) - { - const size_t maxScanSize = 66000u; // see below - io::BufferPtr buf(io::Allocate(maxScanSize)); - Status ret = ScanForEcdr(file, fileSize, static_cast(buf.get()), maxScanSize, cd_numEntries, cd_ofs, cd_size); - if(ret == INFO::OK) - return INFO::OK; - - io::Operation op(*file.get(), buf.get(), sizeof(LFH)); - RETURN_STATUS_IF_ERR(io::Run(op)); - // the Zip file has an LFH but lacks an ECDR. this can happen if - // the user hard-exits while an archive is being written. - // notes: - // - return ERR::CORRUPTED so VFS will not include this file. - // - we could work around this by scanning all LFHs, but won't bother - // because it'd be slow. - // - do not warn - the corrupt archive will be deleted on next - // successful archive builder run anyway. - if(FindRecord(buf.get(), sizeof(LFH), buf.get(), lfh_magic, sizeof(LFH))) - return ERR::CORRUPTED; // NOWARN - // totally bogus - else - WARN_RETURN(ERR::ARCHIVE_UNKNOWN_FORMAT); - } - - PFile m_file; - off_t m_fileSize; }; + PIArchiveReader CreateArchiveReader_Zip(const OsPath& archivePathname) { try @@ -625,170 +242,6 @@ PIArchiveReader CreateArchiveReader_Zip(const OsPath& archivePathname) } } - -//----------------------------------------------------------------------------- -// ArchiveWriter_Zip -//----------------------------------------------------------------------------- - -class ArchiveWriter_Zip : public IArchiveWriter -{ -public: - ArchiveWriter_Zip(const OsPath& archivePathname, bool noDeflate) - : m_file(new File(archivePathname, O_WRONLY)), m_fileSize(0) - , m_numEntries(0), m_noDeflate(noDeflate) - { - THROW_STATUS_IF_ERR(pool_create(&m_cdfhPool, 10*MiB, 0)); - } - - ~ArchiveWriter_Zip() - { - // append an ECDR to the CDFH list (this allows us to - // write out both to the archive file in one burst) - const size_t cd_size = m_cdfhPool.da.pos; - ECDR* ecdr = (ECDR*)pool_alloc(&m_cdfhPool, sizeof(ECDR)); - if(!ecdr) - std::terminate(); - const off_t cd_ofs = m_fileSize; - ecdr->Init(m_numEntries, cd_ofs, cd_size); - - if(write(m_file->Descriptor(), m_cdfhPool.da.base, cd_size+sizeof(ECDR)) < 0) - DEBUG_WARN_ERR(ERR::IO); // no way to return error code - - (void)pool_destroy(&m_cdfhPool); - } - - Status AddFile(const OsPath& pathname, const OsPath& pathnameInArchive) - { - CFileInfo fileInfo; - RETURN_STATUS_IF_ERR(GetFileInfo(pathname, &fileInfo)); - - PFile file(new File); - RETURN_STATUS_IF_ERR(file->Open(pathname, O_RDONLY)); - - return AddFileOrMemory(fileInfo, pathnameInArchive, file, NULL); - } - - Status AddMemory(const u8* data, size_t size, time_t mtime, const OsPath& pathnameInArchive) - { - CFileInfo fileInfo(pathnameInArchive, size, mtime); - - return AddFileOrMemory(fileInfo, pathnameInArchive, PFile(), data); - } - - Status AddFileOrMemory(const CFileInfo& fileInfo, const OsPath& pathnameInArchive, const PFile& file, const u8* data) - { - ENSURE((file && !data) || (data && !file)); - - const off_t usize = fileInfo.Size(); - // skip 0-length files. - // rationale: zip.cpp needs to determine whether a CDFH entry is - // a file or directory (the latter are written by some programs but - // not needed - they'd only pollute the file table). - // it looks like checking for usize=csize=0 is the safest way - - // relying on file attributes (which are system-dependent!) is - // even less safe. - // we thus skip 0-length files to avoid confusing them with directories. - if(!usize) - return INFO::SKIPPED; - - const size_t pathnameLength = pathnameInArchive.string().length(); - - // choose method and the corresponding codec - ZipMethod method; - PICodec codec; - if(m_noDeflate || IsFileTypeIncompressible(pathnameInArchive)) - { - method = ZIP_METHOD_NONE; - codec = CreateCodec_ZLibNone(); - } - else - { - method = ZIP_METHOD_DEFLATE; - codec = CreateCompressor_ZLibDeflate(); - } - - // allocate memory - const size_t csizeMax = codec->MaxOutputSize(size_t(usize)); - io::BufferPtr buf(io::Allocate(sizeof(LFH) + pathnameLength + csizeMax)); - - // read and compress file contents - size_t csize; u32 checksum; - { - u8* cdata = buf.get() + sizeof(LFH) + pathnameLength; - Stream stream(codec); - stream.SetOutputBuffer(cdata, csizeMax); - StreamFeeder streamFeeder(stream); - if(file) - { - io::Operation op(*file.get(), 0, usize); - RETURN_STATUS_IF_ERR(io::Run(op, io::Parameters(), streamFeeder)); - } - else - { - RETURN_STATUS_IF_ERR(streamFeeder(data, usize)); - } - RETURN_STATUS_IF_ERR(stream.Finish()); - csize = stream.OutSize(); - checksum = stream.Checksum(); - } - - // build LFH - { - LFH* lfh = reinterpret_cast(buf.get()); - lfh->Init(fileInfo, (off_t)csize, method, checksum, pathnameInArchive); - } - - // append a CDFH to the central directory (in memory) - const off_t ofs = m_fileSize; - const size_t prev_pos = m_cdfhPool.da.pos; // (required to determine padding size) - const size_t cdfhSize = sizeof(CDFH) + pathnameLength; - CDFH* cdfh = (CDFH*)pool_alloc(&m_cdfhPool, cdfhSize); - if(!cdfh) - WARN_RETURN(ERR::NO_MEM); - const size_t slack = m_cdfhPool.da.pos - prev_pos - cdfhSize; - cdfh->Init(fileInfo, ofs, (off_t)csize, method, checksum, pathnameInArchive, slack); - m_numEntries++; - - // write LFH, pathname and cdata to file - const size_t packageSize = sizeof(LFH) + pathnameLength + csize; - if(write(m_file->Descriptor(), buf.get(), packageSize) < 0) - WARN_RETURN(ERR::IO); - m_fileSize += (off_t)packageSize; - - return INFO::OK; - } - -private: - static bool IsFileTypeIncompressible(const OsPath& pathname) - { - const OsPath extension = pathname.Extension(); - - // file extensions that we don't want to compress - static const wchar_t* incompressibleExtensions[] = - { - L".zip", L".rar", - L".jpg", L".jpeg", L".png", - L".ogg", L".mp3" - }; - - for(size_t i = 0; i < ARRAY_SIZE(incompressibleExtensions); i++) - { - if(extension == incompressibleExtensions[i]) - return true; - } - - return false; - } - - PFile m_file; - off_t m_fileSize; - - Pool m_cdfhPool; - size_t m_numEntries; - - bool m_noDeflate; -}; - PIArchiveWriter CreateArchiveWriter_Zip(const OsPath& archivePathname, bool noDeflate) { try diff --git a/source/lib/file/archive/codec.cpp b/source/lib/file/archive/codec.cpp deleted file mode 100644 index b32fdb2528..0000000000 --- a/source/lib/file/archive/codec.cpp +++ /dev/null @@ -1,28 +0,0 @@ -/* Copyright (C) 2010 Wildfire Games. - * - * Permission is hereby granted, free of charge, to any person obtaining - * a copy of this software and associated documentation files (the - * "Software"), to deal in the Software without restriction, including - * without limitation the rights to use, copy, modify, merge, publish, - * distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to - * the following conditions: - * - * The above copyright notice and this permission notice shall be included - * in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY - * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, - * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE - * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -#include "precompiled.h" -#include "lib/file/archive/codec.h" - -ICodec::~ICodec() -{ -} diff --git a/source/lib/file/archive/codec.h b/source/lib/file/archive/codec.h deleted file mode 100644 index b187f89d2f..0000000000 --- a/source/lib/file/archive/codec.h +++ /dev/null @@ -1,100 +0,0 @@ -/* Copyright (C) 2025 Wildfire Games. - * - * Permission is hereby granted, free of charge, to any person obtaining - * a copy of this software and associated documentation files (the - * "Software"), to deal in the Software without restriction, including - * without limitation the rights to use, copy, modify, merge, publish, - * distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to - * the following conditions: - * - * The above copyright notice and this permission notice shall be included - * in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY - * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, - * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE - * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -/* - * this layer allows for other compression methods/libraries - * besides ZLib. it also simplifies the interface for user code and - * does error checking, etc. - */ - -#ifndef INCLUDED_CODEC -#define INCLUDED_CODEC - -#include "lib/status.h" -#include "lib/types.h" - -#include -#include - -#define CODEC_COMPUTE_CHECKSUM 1 - -struct ICodec -{ -public: - /** - * note: the implementation should not check whether any data remains - - * codecs are sometimes destroyed without completing a transfer. - **/ - virtual ~ICodec(); - - /** - * @return an upper bound on the output size for the given amount of input. - * this is used when allocating a single buffer for the whole operation. - **/ - virtual size_t MaxOutputSize(size_t inSize) const = 0; - - /** - * clear all previous state and prepare for reuse. - * - * this is as if the object were destroyed and re-created, but more - * efficient since it avoids reallocating a considerable amount of - * memory (about 200KB for LZ). - **/ - virtual Status Reset() = 0; - - /** - * process (i.e. compress or decompress) data. - * - * @param in - * @param inSize - * @param out - * @param outSize Bytes remaining in the output buffer; shall not be zero. - * @param inConsumed,outProduced How many bytes in the input and - * output buffers were used. either or both of these can be zero if - * the input size is small or there's not enough output space. - **/ - virtual Status Process(const u8* in, size_t inSize, u8* out, size_t outSize, size_t& inConsumed, size_t& outProduced) = 0; - - /** - * Flush buffers and make sure all output has been produced. - * - * @param checksum Checksum over all input data. - * @param outProduced - * @return error status for the entire operation. - **/ - virtual Status Finish(u32& checksum, size_t& outProduced) = 0; - - /** - * update a checksum to reflect the contents of a buffer. - * - * @param checksum the initial value (must be 0 on first call) - * @param in - * @param inSize - * @return the new checksum. note: after all data has been seen, this is - * identical to the what Finish would return. - **/ - virtual u32 UpdateChecksum(u32 checksum, const u8* in, size_t inSize) const = 0; -}; - -typedef std::shared_ptr PICodec; - -#endif // #ifndef INCLUDED_CODEC diff --git a/source/lib/file/archive/codec_zlib.cpp b/source/lib/file/archive/codec_zlib.cpp deleted file mode 100644 index 06499b520a..0000000000 --- a/source/lib/file/archive/codec_zlib.cpp +++ /dev/null @@ -1,308 +0,0 @@ -/* Copyright (C) 2025 Wildfire Games. - * - * Permission is hereby granted, free of charge, to any person obtaining - * a copy of this software and associated documentation files (the - * "Software"), to deal in the Software without restriction, including - * without limitation the rights to use, copy, modify, merge, publish, - * distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to - * the following conditions: - * - * The above copyright notice and this permission notice shall be included - * in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY - * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, - * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE - * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -#include "precompiled.h" - -#include "codec_zlib.h" - -#include "lib/alignment.h" -#include "lib/debug.h" -#include "lib/external_libraries/zlib.h" -#include "lib/file/archive/codec.h" -#include "lib/status.h" -#include "lib/types.h" - -#include -#include - -class Codec_ZLib : public ICodec -{ -public: - u32 UpdateChecksum([[maybe_unused]] u32 checksum, [[maybe_unused]] const u8* in, - [[maybe_unused]] size_t inSize) const - { -#if CODEC_COMPUTE_CHECKSUM - return (u32)crc32(checksum, in, (uInt)inSize); -#else - return 0; -#endif - } - -protected: - u32 InitializeChecksum() - { -#if CODEC_COMPUTE_CHECKSUM - return crc32(0, 0, 0); -#else - return 0; -#endif - } -}; - - -//----------------------------------------------------------------------------- - -class Codec_ZLibNone : public Codec_ZLib -{ -public: - Codec_ZLibNone() - { - Reset(); - } - - virtual ~Codec_ZLibNone() - { - } - - virtual size_t MaxOutputSize(size_t inSize) const - { - return inSize; - } - - virtual Status Reset() - { - m_checksum = InitializeChecksum(); - return INFO::OK; - } - - virtual Status Process(const u8* in, size_t inSize, u8* out, size_t outSize, size_t& inConsumed, size_t& outProduced) - { - const size_t transferSize = std::min(inSize, outSize); - memcpy(out, in, transferSize); - inConsumed = outProduced = transferSize; - m_checksum = UpdateChecksum(m_checksum, out, outProduced); - return INFO::OK; - } - - virtual Status Finish(u32& checksum, size_t& outProduced) - { - outProduced = 0; - checksum = m_checksum; - return INFO::OK; - } - -private: - u32 m_checksum; -}; - - -//----------------------------------------------------------------------------- - -class CodecZLibStream : public Codec_ZLib -{ -protected: - CodecZLibStream() - { - memset(&m_zs, 0, sizeof(m_zs)); - m_checksum = InitializeChecksum(); - } - - static Status LibError_from_zlib(int zlib_ret) - { - switch(zlib_ret) - { - case Z_OK: - return INFO::OK; - case Z_STREAM_END: - WARN_RETURN(ERR::FAIL); - case Z_MEM_ERROR: - WARN_RETURN(ERR::NO_MEM); - case Z_DATA_ERROR: - WARN_RETURN(ERR::CORRUPTED); - case Z_STREAM_ERROR: - WARN_RETURN(ERR::INVALID_PARAM); - default: - WARN_RETURN(ERR::FAIL); - } - } - - static void WarnIfZLibError(int zlib_ret) - { - (void)LibError_from_zlib(zlib_ret); - } - - typedef int ZEXPORT (*ZLibFunc)(z_streamp strm, int flush); - - Status CallStreamFunc(ZLibFunc func, int flush, const u8* in, const size_t inSize, u8* out, const size_t outSize, size_t& inConsumed, size_t& outProduced) - { - m_zs.next_in = (Byte*)in; - m_zs.avail_in = (uInt)inSize; - m_zs.next_out = (Byte*)out; - m_zs.avail_out = (uInt)outSize; - - int ret = func(&m_zs, flush); - // sanity check: if ZLib reports end of stream, all input data - // must have been consumed. - if(ret == Z_STREAM_END) - { - ENSURE(m_zs.avail_in == 0); - ret = Z_OK; - } - - ENSURE(inSize >= m_zs.avail_in && outSize >= m_zs.avail_out); - inConsumed = inSize - m_zs.avail_in; - outProduced = outSize - m_zs.avail_out; - - return LibError_from_zlib(ret); - } - - mutable z_stream m_zs; - - // note: z_stream does contain an 'adler' checksum field, but that's - // not updated in streams lacking a gzip header, so we'll have to - // calculate a checksum ourselves. - // adler32 is somewhat weaker than CRC32, but a more important argument - // is that we should use the latter for compatibility with Zip archives. - mutable u32 m_checksum; -}; - - -//----------------------------------------------------------------------------- - -class Compressor_ZLib : public CodecZLibStream -{ -public: - Compressor_ZLib() - { - // note: with Z_BEST_COMPRESSION, 78% percent of - // archive builder CPU time is spent in ZLib, even though - // that is interleaved with IO; everything else is negligible. - // we prefer faster speed at the cost of 1.5% larger archives. - const int level = Z_BEST_SPEED; - const int windowBits = -MAX_WBITS; // max window size; omit ZLib header - const int memLevel = 9; // max speed; total mem ~= 384KiB - const int strategy = Z_DEFAULT_STRATEGY; // normal data - not RLE - const int ret = deflateInit2(&m_zs, level, Z_DEFLATED, windowBits, memLevel, strategy); - ENSURE(ret == Z_OK); - } - - virtual ~Compressor_ZLib() - { - const int ret = deflateEnd(&m_zs); - WarnIfZLibError(ret); - } - - virtual size_t MaxOutputSize(size_t inSize) const - { - return (size_t)deflateBound(&m_zs, (uLong)inSize); - } - - virtual Status Reset() - { - m_checksum = InitializeChecksum(); - const int ret = deflateReset(&m_zs); - return LibError_from_zlib(ret); - } - - virtual Status Process(const u8* in, size_t inSize, u8* out, size_t outSize, size_t& inConsumed, size_t& outProduced) - { - m_checksum = UpdateChecksum(m_checksum, in, inSize); - return CodecZLibStream::CallStreamFunc(deflate, 0, in, inSize, out, outSize, inConsumed, outProduced); - } - - virtual Status Finish(u32& checksum, size_t& outProduced) - { - const uInt availOut = m_zs.avail_out; - - // notify zlib that no more data is forthcoming and have it flush output. - // our output buffer has enough space due to use of deflateBound; - // therefore, deflate must return Z_STREAM_END. - const int ret = deflate(&m_zs, Z_FINISH); - ENSURE(ret == Z_STREAM_END); - - outProduced = size_t(availOut - m_zs.avail_out); - - checksum = m_checksum; - return INFO::OK; - } -}; - - -//----------------------------------------------------------------------------- - -class Decompressor_ZLib : public CodecZLibStream -{ -public: - Decompressor_ZLib() - { - const int windowBits = -MAX_WBITS; // max window size; omit ZLib header - const int ret = inflateInit2(&m_zs, windowBits); - ENSURE(ret == Z_OK); - } - - virtual ~Decompressor_ZLib() - { - const int ret = inflateEnd(&m_zs); - WarnIfZLibError(ret); - } - - virtual size_t MaxOutputSize(size_t inSize) const - { - // relying on an upper bound for the output is a really bad idea for - // large files. archive formats store the uncompressed file sizes, - // so callers should use that when allocating the output buffer. - ENSURE(inSize < 1*MiB); - - return inSize*1032; // see http://www.zlib.org/zlib_tech.html - } - - virtual Status Reset() - { - m_checksum = InitializeChecksum(); - const int ret = inflateReset(&m_zs); - return LibError_from_zlib(ret); - } - - virtual Status Process(const u8* in, size_t inSize, u8* out, size_t outSize, size_t& inConsumed, size_t& outProduced) - { - const Status ret = CodecZLibStream::CallStreamFunc(inflate, Z_SYNC_FLUSH, in, inSize, out, outSize, inConsumed, outProduced); - m_checksum = UpdateChecksum(m_checksum, out, outProduced); - return ret; - } - - virtual Status Finish(u32& checksum, size_t& outProduced) - { - // no action needed - decompression always flushes immediately. - outProduced = 0; - - checksum = m_checksum; - return INFO::OK; - } -}; - - -//----------------------------------------------------------------------------- - -PICodec CreateCodec_ZLibNone() -{ - return PICodec(new Codec_ZLibNone); -} - -PICodec CreateCompressor_ZLibDeflate() -{ - return PICodec(new Compressor_ZLib); -} - -PICodec CreateDecompressor_ZLibDeflate() -{ - return PICodec (new Decompressor_ZLib); -} diff --git a/source/lib/file/archive/codec_zlib.h b/source/lib/file/archive/codec_zlib.h deleted file mode 100644 index 14cd3d86b1..0000000000 --- a/source/lib/file/archive/codec_zlib.h +++ /dev/null @@ -1,32 +0,0 @@ -/* Copyright (C) 2010 Wildfire Games. - * - * Permission is hereby granted, free of charge, to any person obtaining - * a copy of this software and associated documentation files (the - * "Software"), to deal in the Software without restriction, including - * without limitation the rights to use, copy, modify, merge, publish, - * distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to - * the following conditions: - * - * The above copyright notice and this permission notice shall be included - * in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY - * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, - * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE - * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -#ifndef INCLUDED_CODEC_ZLIB -#define INCLUDED_CODEC_ZLIB - -#include "lib/file/archive/codec.h" - -extern PICodec CreateCodec_ZLibNone(); -extern PICodec CreateCompressor_ZLibDeflate(); -extern PICodec CreateDecompressor_ZLibDeflate(); - -#endif // INCLUDED_CODEC_ZLIB diff --git a/source/lib/file/archive/stream.cpp b/source/lib/file/archive/stream.cpp deleted file mode 100644 index d69157dce1..0000000000 --- a/source/lib/file/archive/stream.cpp +++ /dev/null @@ -1,139 +0,0 @@ -/* Copyright (C) 2025 Wildfire Games. - * - * Permission is hereby granted, free of charge, to any person obtaining - * a copy of this software and associated documentation files (the - * "Software"), to deal in the Software without restriction, including - * without limitation the rights to use, copy, modify, merge, publish, - * distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to - * the following conditions: - * - * The above copyright notice and this permission notice shall be included - * in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY - * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, - * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE - * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -#include "precompiled.h" - -#include "stream.h" - -#include "lib/allocators/shared_ptr.h" -#include "lib/debug.h" -#include "lib/file/archive/codec.h" -//#include "lib/timer.h" - -//TIMER_ADD_CLIENT(tc_stream); - - -OutputBufferManager::OutputBufferManager() -{ - Reset(); -} - -void OutputBufferManager::Reset() -{ - m_buffer = 0; - m_size = 0; - m_capacity = 0; -} - -void OutputBufferManager::SetBuffer(u8* buffer, size_t size) -{ - ENSURE(IsAllowableBuffer(buffer, size)); - - m_buffer = buffer; - m_size = size; -} - -void OutputBufferManager::AllocateBuffer(size_t size) -{ - // notes: - // - this implementation allows reusing previous buffers if they - // are big enough, which reduces the number of allocations. - // - no further attempts to reduce allocations (e.g. by doubling - // the current size) are made; this strategy is enough. - // - Pool etc. cannot be used because files may be huge (larger - // than the address space of 32-bit systems). - - // no buffer or the previous one wasn't big enough: reallocate - if(!m_mem || m_capacity < size) - { - AllocateAligned(m_mem, size); - m_capacity = size; - } - - SetBuffer(m_mem.get(), size); -} - -bool OutputBufferManager::IsAllowableBuffer(u8* buffer, size_t size) -{ - // none yet established - if(m_buffer == 0 && m_size == 0) - return true; - - // same as last time (happens with temp buffers) - if(m_buffer == buffer && m_size == size) - return true; - - // located after the last buffer (note: not necessarily after - // the entire buffer; a lack of input can cause the output buffer - // to only partially be used before the next call.) - if((unsigned)(buffer - m_buffer) <= m_size) - return true; - - return false; -} - - -//----------------------------------------------------------------------------- - - -Stream::Stream(const PICodec& codec) - : m_codec(codec) - , m_inConsumed(0), m_outProduced(0) -{ -} - - -void Stream::AllocateOutputBuffer(size_t outSizeMax) -{ - m_outputBufferManager.AllocateBuffer(outSizeMax); -} - - -void Stream::SetOutputBuffer(u8* out, size_t outSize) -{ - m_outputBufferManager.SetBuffer(out, outSize); -} - - -Status Stream::Feed(const u8* in, size_t inSize) -{ - if(m_outProduced == m_outputBufferManager.Size()) // output buffer full; must not call Process - return INFO::ALL_COMPLETE; - - size_t inConsumed, outProduced; - u8* const out = m_outputBufferManager.Buffer() + m_outProduced; - const size_t outSize = m_outputBufferManager.Size() - m_outProduced; - RETURN_STATUS_IF_ERR(m_codec->Process(in, inSize, out, outSize, inConsumed, outProduced)); - - m_inConsumed += inConsumed; - m_outProduced += outProduced; - return INFO::OK; -} - - -Status Stream::Finish() -{ - size_t outProduced; - RETURN_STATUS_IF_ERR(m_codec->Finish(m_checksum, outProduced)); - m_outProduced += outProduced; - return INFO::OK; -} diff --git a/source/lib/file/archive/stream.h b/source/lib/file/archive/stream.h deleted file mode 100644 index 60ea9210ef..0000000000 --- a/source/lib/file/archive/stream.h +++ /dev/null @@ -1,139 +0,0 @@ -/* Copyright (C) 2025 Wildfire Games. - * - * Permission is hereby granted, free of charge, to any person obtaining - * a copy of this software and associated documentation files (the - * "Software"), to deal in the Software without restriction, including - * without limitation the rights to use, copy, modify, merge, publish, - * distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to - * the following conditions: - * - * The above copyright notice and this permission notice shall be included - * in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY - * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, - * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE - * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -/* - * output buffer and 'stream' layered on top of a compression codec - */ - -#ifndef INCLUDED_STREAM -#define INCLUDED_STREAM - -#include "lib/code_annotation.h" -#include "lib/file/archive/codec.h" -#include "lib/status.h" -#include "lib/types.h" - -#include -#include - -// note: this is similar in function to std::vector, but we don't need -// iterators etc. and would prefer to avoid initializing each byte. -class OutputBufferManager -{ -public: - OutputBufferManager(); - - void Reset(); - void SetBuffer(u8* buffer, size_t size); - - /** - * allocate a new output buffer. - * - * @param size [bytes] to allocate. - * - * notes: - * - if a buffer had previously been allocated and is large enough, - * it is reused (this reduces the number of allocations). - * - this class manages the lifetime of the buffer. - **/ - void AllocateBuffer(size_t size); - - u8* Buffer() const - { - return m_buffer; - } - - size_t Size() const - { - return m_size; - } - -private: - bool IsAllowableBuffer(u8* buffer, size_t size); - - u8* m_buffer; - size_t m_size; - - std::shared_ptr m_mem; - // size of m_mem. allows reusing previously allocated buffers - // (user-specified buffers can't be reused because we have no control - // over their lifetime) - size_t m_capacity; -}; - - -class Stream -{ -public: - Stream(const PICodec& codec); - - void SetOutputBuffer(u8* out, size_t outSize); - - void AllocateOutputBuffer(size_t outSizeMax); - - /** - * 'feed' the codec with a data block. - **/ - Status Feed(const u8* in, size_t inSize); - - Status Finish(); - - size_t OutSize() const - { - return m_outProduced; - } - - u32 Checksum() const - { - return m_checksum; - } - -private: - PICodec m_codec; - OutputBufferManager m_outputBufferManager; - - size_t m_inConsumed; - size_t m_outProduced; - u32 m_checksum; -}; - -// avoids the need for std::bind (not supported on all compilers) and boost::bind (can't be -// used at work) -struct StreamFeeder -{ - NONCOPYABLE(StreamFeeder); -public: - StreamFeeder(Stream& stream) - : stream(stream) - { - } - - Status operator()(const u8* data, size_t size) const - { - return stream.Feed(data, size); - } - -private: - Stream& stream; -}; - -#endif // #ifndef INCLUDED_STREAM diff --git a/source/lib/file/archive/tests/test_archive_zip.h b/source/lib/file/archive/tests/test_archive_zip.h index 364829eb5b..1583b637d8 100644 --- a/source/lib/file/archive/tests/test_archive_zip.h +++ b/source/lib/file/archive/tests/test_archive_zip.h @@ -1,4 +1,4 @@ -/* Copyright (C) 2025 Wildfire Games. +/* Copyright (C) 2026 Wildfire Games. * * Permission is hereby granted, free of charge, to any person obtaining * a copy of this software and associated documentation files (the @@ -33,6 +33,7 @@ #include #include +#include #include namespace @@ -108,10 +109,44 @@ public: TS_ASSERT_EQUALS("buildzipwithcomment.sh", g_ResultBuffer); } + void test_round_trip() + { + OsPath testDir = MOD_PATH / "file" / "archive"; + OsPath testPath = testDir / "test.zip"; + + TS_ASSERT_EQUALS(INFO::OK, CreateDirectories(testDir, 0700, false)); + + { + u8 buf[5] = { 'h', 'e', 'l', 'l', 'o' }; + PIArchiveWriter zWriter = CreateArchiveWriter_Zip(testPath, false); + zWriter->AddMemory(buf, std::size(buf), 0, "world/hello"); + } + + { + PIArchiveReader zReader = CreateArchiveReader_Zip(testPath); + TS_ASSERT_DIFFERS(nullptr, zReader); + + TS_ASSERT_EQUALS(INFO::OK, zReader->ReadEntries(TestArchiveZip::ReadWorldCallback, 0)); + TS_ASSERT_EQUALS("hello", g_ResultBuffer); + } + } + + private: static void ArchiveEntryCallback(const VfsPath& path, const CFileInfo&, PIArchiveFile, uintptr_t /*cbData*/) { g_ResultBuffer = path.string8(); } + + static void ReadWorldCallback(const VfsPath& path, const CFileInfo&, PIArchiveFile file, + uintptr_t /*cbData*/) + { + if (path.string8() == "world/hello") + { + std::shared_ptr buf = std::make_shared(5); + file->Load("", buf, 5); + g_ResultBuffer = std::string(buf.get(), buf.get() + 5); + } + } }; diff --git a/source/ps/ArchiveBuilder.cpp b/source/ps/ArchiveBuilder.cpp index 138ea59b25..1a79731381 100644 --- a/source/ps/ArchiveBuilder.cpp +++ b/source/ps/ArchiveBuilder.cpp @@ -1,4 +1,4 @@ -/* Copyright (C) 2025 Wildfire Games. +/* Copyright (C) 2026 Wildfire Games. * This file is part of 0 A.D. * * 0 A.D. is free software: you can redistribute it and/or modify