diff --git a/source/graphics/TextureConverter.cpp b/source/graphics/TextureConverter.cpp index f88f73a2b2..98380fcd24 100644 --- a/source/graphics/TextureConverter.cpp +++ b/source/graphics/TextureConverter.cpp @@ -458,7 +458,8 @@ bool CTextureConverter::Poll(CTexturePtr& texture, VfsPath& dest, bool& ok) // Move output into a correctly-aligned buffer size_t size = result->output.buffer.size(); - shared_ptr file = io_Allocate(size); + shared_ptr file; + AllocateAligned(file, size, maxSectorSize); memcpy(file.get(), &result->output.buffer[0], size); if (m_VFS->CreateFile(result->dest, file, size) < 0) { diff --git a/source/graphics/tests/test_MeshManager.h b/source/graphics/tests/test_MeshManager.h index 5f56d0e937..7d1cac5595 100644 --- a/source/graphics/tests/test_MeshManager.h +++ b/source/graphics/tests/test_MeshManager.h @@ -20,6 +20,7 @@ #include "lib/file/file_system_util.h" #include "lib/file/vfs/vfs.h" #include "lib/file/io/io.h" +#include "lib/allocators/shared_ptr.h" #include "graphics/ColladaManager.h" #include "graphics/MeshManager.h" @@ -116,7 +117,8 @@ public: { copyFile(srcDAE, testDAE); //buildArchive(); - shared_ptr buf = io_Allocate(100); + shared_ptr buf; + AllocateAligned(buf, 100, maxSectorSize); strcpy_s((char*)buf.get(), 5, "Test"); g_VFS->CreateFile(testDAE, buf, 4); } @@ -178,7 +180,8 @@ public: TestLogger logger; copyFile(srcDAE, testDAE); - shared_ptr buf = io_Allocate(100); + shared_ptr buf; + AllocateAligned(buf, 100, maxSectorSize); strcpy_s((char*)buf.get(), 100, "Not valid XML"); g_VFS->CreateFile(testSkeletonDefs, buf, 13); @@ -192,7 +195,8 @@ public: TestLogger logger; copyFile(srcSkeletonDefs, testSkeletonDefs); - shared_ptr buf = io_Allocate(100); + shared_ptr buf; + AllocateAligned(buf, 100, maxSectorSize); strcpy_s((char*)buf.get(), 100, "Not valid XML"); g_VFS->CreateFile(testDAE, buf, 13); diff --git a/source/lib/alignment.h b/source/lib/alignment.h new file mode 100644 index 0000000000..e1619d0578 --- /dev/null +++ b/source/lib/alignment.h @@ -0,0 +1,105 @@ +#ifndef INCLUDED_ALIGNMENT +#define INCLUDED_ALIGNMENT + +#include "lib/sysdep/compiler.h" // MSC_VERSION + +template +inline bool IsAligned(T t, uintptr_t multiple) +{ + return (uintptr_t(t) % multiple) == 0; +} + +template +inline size_t Align(size_t n) +{ + cassert(multiple != 0 && ((multiple & (multiple-1)) == 0)); // is power of 2 + return (n + multiple-1) & ~(multiple-1); +} + + +// +// SIMD vector +// + +static const size_t vectorSize = 16; + +#define VERIFY_VECTOR_MULTIPLE(size)\ + VERIFY(IsAligned(size, vectorSize)) + +#define VERIFY_VECTOR_ALIGNED(pointer)\ + VERIFY_VECTOR_MULTIPLE(pointer);\ + ASSUME_ALIGNED(pointer, vectorSize) + + +// +// CPU cache +// + +static const size_t cacheLineSize = 64; // (L2) + +#if MSC_VERSION +#define CACHE_ALIGNED __declspec(align(64)) // align() requires a literal; keep in sync with cacheLineSize +#endif + + +// +// MMU pages +// + +static const size_t pageSize = 0x1000; // 4 KB +static const size_t largePageSize = 0x200000; // 2 MB + + +// waio opens files with FILE_FLAG_NO_BUFFERING, so Windows requires +// file offsets / buffers and sizes to be sector-aligned. querying the +// actual sector size via GetDiskFreeSpace is inconvenient and slow. +// we always request large blocks anyway, so just check whether inputs +// are aligned to a `maximum' sector size. this catches common mistakes +// before they cause scary "IO failed" errors. if the value turns out +// to be too low, the Windows APIs will still complain. +static const uintptr_t maxSectorSize = 0x1000; + +#endif // #ifndef INCLUDED_ALIGNMENT +#ifndef INCLUDED_ALIGNMENT +#define INCLUDED_ALIGNMENT + +template +inline bool IsAligned(T t, uintptr_t multiple) +{ + return (uintptr_t(t) % multiple) == 0; +} + + +// +// SIMD vector +// + +static const size_t vectorSize = 16; + +#define VERIFY_VECTOR_MULTIPLE(size)\ + VERIFY(IsAligned(size, vectorSize)) + +#define VERIFY_VECTOR_ALIGNED(pointer)\ + VERIFY_VECTOR_MULTIPLE(pointer);\ + ASSUME_ALIGNED(pointer, vectorSize) + + +// +// CPU cache +// + +static const size_t cacheLineSize = 64; // (L2) + +#if MSC_VERSION +#define CACHE_ALIGNED __declspec(align(64)) // align() requires a literal; keep in sync with cacheLineSize +#endif + + +// +// MMU pages +// + +static const size_t pageSize = 0x1000; // 4 KB +static const size_t largePageSize = 0x200000; // 2 MB + +#endif // #ifndef INCLUDED_ALIGNMENT diff --git a/source/lib/allocators/allocators.cpp b/source/lib/allocators/allocators.cpp index c5685d6935..09bfca804c 100644 --- a/source/lib/allocators/allocators.cpp +++ b/source/lib/allocators/allocators.cpp @@ -27,9 +27,8 @@ #include "precompiled.h" #include "lib/allocators/allocators.h" +#include "lib/alignment.h" #include "lib/sysdep/cpu.h" // cpu_CAS -#include "lib/bits.h" - #include "lib/allocators/mem_util.h" @@ -157,7 +156,7 @@ void single_free(void* storage, volatile intptr_t* in_use_flag, void* p) void* static_calloc(StaticStorage* ss, size_t size) { - void* p = (void*)round_up((uintptr_t)ss->pos, (uintptr_t)16u); + void* p = (void*)Align<16>((uintptr_t)ss->pos); ss->pos = (u8*)p+size; debug_assert(ss->pos <= ss->end); return p; diff --git a/source/lib/allocators/mem_util.cpp b/source/lib/allocators/mem_util.cpp index bbcc3a2228..83c676b88f 100644 --- a/source/lib/allocators/mem_util.cpp +++ b/source/lib/allocators/mem_util.cpp @@ -28,6 +28,7 @@ #include "lib/allocators/mem_util.h" #include "lib/bits.h" // round_up +#include "lib/alignment.h" #include "lib/posix/posix_mman.h" #include "lib/sysdep/os_cpu.h" // os_cpu_PageSize @@ -45,8 +46,7 @@ size_t mem_RoundUpToPage(size_t size) size_t mem_RoundUpToAlignment(size_t size) { // all allocators should align to at least this many bytes: - const size_t alignment = 8; - return round_up(size, alignment); + return Align<8>(size); } diff --git a/source/lib/allocators/pool.cpp b/source/lib/allocators/pool.cpp index 22731b2fdd..342cd6d8dd 100644 --- a/source/lib/allocators/pool.cpp +++ b/source/lib/allocators/pool.cpp @@ -29,6 +29,10 @@ #include "lib/allocators/mem_util.h" +#include "lib/timer.h" + +TIMER_ADD_CLIENT(tc_pool_alloc); + LibError pool_create(Pool* p, size_t max_size, size_t el_size) { @@ -66,6 +70,7 @@ bool pool_contains(const Pool* p, void* el) void* pool_alloc(Pool* p, size_t size) { + TIMER_ACCRUE(tc_pool_alloc); // if pool allows variable sizes, go with the size parameter, // otherwise the pool el_size setting. const size_t el_size = p->el_size? p->el_size : mem_RoundUpToAlignment(size); diff --git a/source/lib/allocators/shared_ptr.h b/source/lib/allocators/shared_ptr.h index 0e73abf6c6..7b95a887ef 100644 --- a/source/lib/allocators/shared_ptr.h +++ b/source/lib/allocators/shared_ptr.h @@ -23,7 +23,7 @@ #ifndef INCLUDED_SHARED_PTR #define INCLUDED_SHARED_PTR -#include "lib/sysdep/arch/x86_x64/cache.h" +#include "lib/alignment.h" #include "lib/sysdep/rtl.h" // rtl_AllocateAligned struct DummyDeleter @@ -61,6 +61,7 @@ struct FreeDeleter // (note: uses CheckedArrayDeleter) LIB_API shared_ptr Allocate(size_t size); + struct AlignedDeleter { template @@ -71,9 +72,13 @@ struct AlignedDeleter }; template -inline shared_ptr AllocateAligned(size_t size) +static inline LibError AllocateAligned(shared_ptr& p, size_t size, size_t alignment = cacheLineSize) { - return shared_ptr((T*)rtl_AllocateAligned(size, x86_x64_Caches(L2D)->entrySize), AlignedDeleter()); + void* mem = rtl_AllocateAligned(size, alignment); + if(!mem) + WARN_RETURN(ERR::NO_MEM); + p.reset((T*)mem, AlignedDeleter()); + return INFO::OK; } #endif // #ifndef INCLUDED_SHARED_PTR diff --git a/source/lib/allocators/unique_range.cpp b/source/lib/allocators/unique_range.cpp new file mode 100644 index 0000000000..48841a0870 --- /dev/null +++ b/source/lib/allocators/unique_range.cpp @@ -0,0 +1,40 @@ +#include "precompiled.h" +#include "lib/allocators/unique_range.h" + +#include "lib/sysdep/cpu.h" // cpu_AtomicAdd +#include "lib/sysdep/rtl.h" // rtl_FreeAligned + + +static void UniqueRangeDeleterNone(void* UNUSED(pointer), size_t UNUSED(size)) +{ + // (introducing this do-nothing function avoids having to check whether deleter != 0) +} + +static void UniqueRangeDeleterAligned(void* pointer, size_t UNUSED(size)) +{ + return rtl_FreeAligned(pointer); +} + + +static UniqueRangeDeleter deleters[idxDeleterBits+1] = { UniqueRangeDeleterNone, UniqueRangeDeleterAligned }; + +static IdxDeleter numDeleters = 2; + + +IdxDeleter AddUniqueRangeDeleter(UniqueRangeDeleter deleter) +{ + debug_assert(deleter); + IdxDeleter idxDeleter = cpu_AtomicAdd(&numDeleters, 1); + debug_assert(idxDeleter < (IdxDeleter)ARRAY_SIZE(deleters)); + deleters[idxDeleter] = deleter; + return idxDeleter; +} + + +void CallUniqueRangeDeleter(void* pointer, size_t size, IdxDeleter idxDeleter) throw() +{ + ASSERT(idxDeleter < numDeleters); + // (some deleters do not tolerate null pointers) + if(pointer) + deleters[idxDeleter](pointer, size); +} diff --git a/source/lib/allocators/unique_range.h b/source/lib/allocators/unique_range.h new file mode 100644 index 0000000000..51f4cb7129 --- /dev/null +++ b/source/lib/allocators/unique_range.h @@ -0,0 +1,185 @@ +#ifndef INCLUDED_UNIQUE_RANGE +#define INCLUDED_UNIQUE_RANGE + + +#define ASSERT debug_assert + + +#include "lib/lib_api.h" + +// we usually don't hold multiple references to allocations, so unique_ptr +// can be used instead of the more complex (ICC generated incorrect code on +// 2 occasions) and expensive shared_ptr. +// a custom deleter is required because allocators such as ReserveAddressSpace need to +// pass the size to their deleter. we want to mix pointers from various allocators, but +// unique_ptr's deleter is fixed at compile-time, so it would need to be general enough +// to handle all allocators. +// storing the size and a function pointer would be one such solution, with the added +// bonus of no longer requiring a complete type at the invocation of ~unique_ptr. +// however, this inflates the pointer size to 3 words. if only a few allocator types +// are needed, we can replace the function pointer with an index stashed into the +// lower bits of the pointer (safe because allocations are always aligned to the +// word size). +typedef intptr_t IdxDeleter; + +// no-op deleter (use when returning part of an existing allocation) +// must be zero because reset() sets address (which includes idxDeleter) to zero. +static const IdxDeleter idxDeleterNone = 0; + +static const IdxDeleter idxDeleterAligned = 1; + +// (temporary value to prevent concurrent calls to AddUniqueRangeDeleter) +static const IdxDeleter idxDeleterBusy = -IdxDeleter(1); + +// governs the maximum number of IdxDeleter and each pointer's alignment requirements +static const IdxDeleter idxDeleterBits = 0x7; + +typedef void (*UniqueRangeDeleter)(void* pointer, size_t size); + +/** + * @return the next available IdxDeleter and associate it with the deleter. + * halts the program if the idxDeleterBits limit has been reached. + * + * thread-safe, but no attempt is made to detect whether the deleter has already been + * registered (would require a mutex). each allocator must ensure they only call this once. + **/ +LIB_API IdxDeleter AddUniqueRangeDeleter(UniqueRangeDeleter deleter); + +LIB_API void CallUniqueRangeDeleter(void* pointer, size_t size, IdxDeleter idxDeleter) throw(); + + +// unfortunately, unique_ptr allows constructing without a custom deleter. to ensure callers can +// rely upon pointers being associated with a size, we introduce a `UniqueRange' replacement. +// its interface is identical to unique_ptr except for the constructors, the addition of +// size() and the removal of operator bool (which avoids implicit casts to int). +class UniqueRange +{ +public: + typedef void* pointer; + typedef void element_type; + + UniqueRange() + { + Set(0, 0, idxDeleterNone); + } + + UniqueRange(pointer p, size_t size, IdxDeleter deleter) + { + Set(p, size, deleter); + } + + UniqueRange(RVREF(UniqueRange) rvref) + { + UniqueRange& rhs = LVALUE(rvref); + address_ = rhs.address_; + size_ = rhs.size_; + rhs.address_ = 0; + } + + UniqueRange& operator=(RVREF(UniqueRange) rvref) + { + UniqueRange& rhs = LVALUE(rvref); + if(this != &rhs) + { + Delete(); + address_ = rhs.address_; + size_ = rhs.size_; + rhs.address_ = 0; + } + return *this; + } + + ~UniqueRange() + { + Delete(); + } + + pointer get() const + { + return pointer(address_ & ~idxDeleterBits); + } + + IdxDeleter get_deleter() const + { + return IdxDeleter(address_ & idxDeleterBits); + } + + size_t size() const + { + return size_; + } + + // side effect: subsequent get_deleter will return idxDeleterNone + pointer release() // relinquish ownership + { + pointer ret = get(); + address_ = 0; + return ret; + } + + void reset() + { + Delete(); + address_ = 0; + } + + void reset(pointer p, size_t size, IdxDeleter deleter) + { + Delete(); + Set(p, size, deleter); + } + + void swap(UniqueRange& rhs) + { + std::swap(address_, rhs.address_); + std::swap(size_, rhs.size_); + } + +private: + void Set(pointer p, size_t size, IdxDeleter deleter) + { + ASSERT((uintptr_t(p) & idxDeleterBits) == 0); + ASSERT(deleter <= idxDeleterBits); + + address_ = uintptr_t(p) | deleter; + size_ = size; + + ASSERT(get() == p); + ASSERT(get_deleter() == deleter); + ASSERT(this->size() == size); + } + + void Delete() + { + CallUniqueRangeDeleter(get(), size(), get_deleter()); + } + + // disallow construction and assignment from lvalue + UniqueRange(const UniqueRange&); + UniqueRange& operator=(const UniqueRange&); + + // (IdxDeleter is stored in the lower bits of address since size might not even be a multiple of 4.) + uintptr_t address_; + size_t size_; +}; + +namespace std { + +static inline void swap(UniqueRange& p1, UniqueRange& p2) +{ + p1.swap(p2); +} + +static inline void swap(RVREF(UniqueRange) p1, UniqueRange& p2) +{ + p2.swap(LVALUE(p1)); +} + +static inline void swap(UniqueRange& p1, RVREF(UniqueRange) p2) +{ + p1.swap(LVALUE(p2)); +} + +} + +#endif // #ifndef INCLUDED_UNIQUE_RANGE diff --git a/source/lib/bits.h b/source/lib/bits.h index 1a8f15d5c1..1d4b77ab39 100644 --- a/source/lib/bits.h +++ b/source/lib/bits.h @@ -230,13 +230,6 @@ inline T round_down(T n, T multiple) } -template -inline bool IsAligned(T t, uintptr_t multiple) -{ - return ((uintptr_t)t % multiple) == 0; -} - - template inline T MaxPowerOfTwoDivisor(T value) { diff --git a/source/lib/code_annotation.h b/source/lib/code_annotation.h index 51ae901494..e1feb5800c 100644 --- a/source/lib/code_annotation.h +++ b/source/lib/code_annotation.h @@ -28,6 +28,7 @@ #define INCLUDED_CODE_ANNOTATION #include "lib/sysdep/compiler.h" +#include "lib/sysdep/arch.h" // ARCH_AMD64 /** * mark a function local variable or parameter as unused and avoid @@ -78,6 +79,20 @@ #define UNREACHABLE // actually defined below.. this is for # undef UNREACHABLE // CppDoc's benefit only. +// this macro should not generate any fallback code; it is merely the +// compiler-specific backend for UNREACHABLE. +// #define it to nothing if the compiler doesn't support such a hint. +#define HAVE_ASSUME_UNREACHABLE 1 +#if MSC_VERSION && !ICC_VERSION // (ICC ignores this) +# define ASSUME_UNREACHABLE __assume(0) +#elif GCC_VERSION >= 450 +# define ASSUME_UNREACHABLE __builtin_unreachable() +#else +# define ASSUME_UNREACHABLE +# undef HAVE_ASSUME_UNREACHABLE +# define HAVE_ASSUME_UNREACHABLE 0 +#endif + // compiler supports ASSUME_UNREACHABLE => allow it to assume the code is // never reached (improves optimization at the cost of undefined behavior // if the annotation turns out to be incorrect). @@ -214,4 +229,150 @@ private:\ # define COMPILER_FENCE #endif + +// try to define _W64, if not already done +// (this is useful for catching pointer size bugs) +#ifndef _W64 +# if MSC_VERSION +# define _W64 __w64 +# elif GCC_VERSION +# define _W64 __attribute__((mode (__pointer__))) +# else +# define _W64 +# endif +#endif + + +// C99-like restrict (non-standard in C++, but widely supported in various forms). +// +// May be used on pointers. May also be used on member functions to indicate +// that 'this' is unaliased (e.g. "void C::m() RESTRICT { ... }"). +// Must not be used on references - GCC supports that but VC doesn't. +// +// We call this "RESTRICT" to avoid conflicts with VC's __declspec(restrict), +// and because it's not really the same as C99's restrict. +// +// To be safe and satisfy the compilers' stated requirements: an object accessed +// by a restricted pointer must not be accessed by any other pointer within the +// lifetime of the restricted pointer, if the object is modified. +// To maximise the chance of optimisation, any pointers that could potentially +// alias with the restricted one should be marked as restricted too. +// +// It would probably be a good idea to write test cases for any code that uses +// this in an even very slightly unclear way, in case it causes obscure problems +// in a rare compiler due to differing semantics. +// +// .. GCC +#if GCC_VERSION +# define RESTRICT __restrict__ +// .. VC8 provides __restrict +#elif MSC_VERSION >= 1400 +# define RESTRICT __restrict +// .. ICC supports the keyword 'restrict' when run with the /Qrestrict option, +// but it always also supports __restrict__ or __restrict to be compatible +// with GCC/MSVC, so we'll use the underscored version. One of {GCC,MSC}_VERSION +// should have been defined in addition to ICC_VERSION, so we should be using +// one of the above cases (unless it's an old VS7.1-emulating ICC). +#elif ICC_VERSION +# error ICC_VERSION defined without either GCC_VERSION or an adequate MSC_VERSION +// .. unsupported; remove it from code +#else +# define RESTRICT +#endif + + +// C99-style __func__ +// .. newer GCC already have it +#if GCC_VERSION >= 300 +// nothing need be done +// .. old GCC and MSVC have __FUNCTION__ +#elif GCC_VERSION >= 200 || MSC_VERSION +# define __func__ __FUNCTION__ +// .. unsupported +#else +# define __func__ "(unknown)" +#endif + + +// extern "C", but does the right thing in pure-C mode +#if defined(__cplusplus) +# define EXTERN_C extern "C" +#else +# define EXTERN_C extern +#endif + + +#if MSC_VERSION +# define INLINE __forceinline +#else +# define INLINE inline +#endif + + +#if MSC_VERSION +# define CALL_CONV __cdecl +#else +# define CALL_CONV +#endif + + +#if MSC_VERSION && !ARCH_AMD64 +# define DECORATED_NAME(name) _##name +#else +# define DECORATED_NAME(name) name +#endif + + +// workaround for preprocessor limitation: macro args aren't expanded +// before being pasted. +#define STRINGIZE2(id) # id +#define STRINGIZE(id) STRINGIZE2(id) + + +//----------------------------------------------------------------------------- +// partial emulation of C++0x rvalue references (required for UniqueRange) + +#if HAVE_CPP0X + +#define RVREF(T) T&& // the type of an rvalue reference +#define LVALUE(rvalue) rvalue // (a named rvalue reference is an lvalue) +#define RVALUE(lvalue) std::move(lvalue) +#define RVALUE_FROM_R(rvalue) RVALUE(rvalue) // (see above) + +#else + +// RVALUE wraps an lvalue reference in this class for later use by a +// "move ctor" that takes an RValue. +template +class RValue +{ +public: + explicit RValue(T& lvalue): lvalue(lvalue) {} + T& LValue() const { return lvalue; } + +private: + T& lvalue; +}; + +// from rvalue or const lvalue +template +static inline RValue ToRValue(const T& t) +{ + return RValue((T&)t); +} + +// from lvalue +template +static inline RValue ToRValue(T& t) +{ + return RValue(t); +} + +#define RVREF(T) const RValue& // the type of an rvalue reference +#define LVALUE(rvalue) rvalue.LValue() +#define RVALUE(lvalue) ToRValue(lvalue) +#define RVALUE_FROM_R(rvalue) rvalue + +#endif // #if !HAVE_CPP0X + #endif // #ifndef INCLUDED_CODE_ANNOTATION diff --git a/source/lib/fat_time.cpp b/source/lib/fat_time.cpp deleted file mode 100644 index d832e4e3c1..0000000000 --- a/source/lib/fat_time.cpp +++ /dev/null @@ -1,77 +0,0 @@ -/* Copyright (c) 2010 Wildfire Games - * - * Permission is hereby granted, free of charge, to any person obtaining - * a copy of this software and associated documentation files (the - * "Software"), to deal in the Software without restriction, including - * without limitation the rights to use, copy, modify, merge, publish, - * distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to - * the following conditions: - * - * The above copyright notice and this permission notice shall be included - * in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY - * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, - * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE - * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -/* - * timestamp conversion: DOS FAT <-> Unix time_t - */ - -#include "precompiled.h" -#include "lib/fat_time.h" - -#include - -#include "lib/bits.h" - - -time_t time_t_from_FAT(u32 fat_timedate) -{ - const u32 fat_time = bits(fat_timedate, 0, 15); - const u32 fat_date = bits(fat_timedate, 16, 31); - - struct tm t; // struct tm format: - t.tm_sec = bits(fat_time, 0,4) * 2; // [0,59] - t.tm_min = bits(fat_time, 5,10); // [0,59] - t.tm_hour = bits(fat_time, 11,15); // [0,23] - t.tm_mday = bits(fat_date, 0,4); // [1,31] - t.tm_mon = bits(fat_date, 5,8) - 1; // [0,11] - t.tm_year = bits(fat_date, 9,15) + 80; // since 1900 - t.tm_isdst = -1; // unknown - let libc determine - - // otherwise: totally bogus, and at the limit of 32-bit time_t - debug_assert(t.tm_year < 138); - - time_t ret = mktime(&t); - debug_assert(ret != (time_t)-1); // mktime shouldn't fail - return ret; -} - - -u32 FAT_from_time_t(time_t time) -{ - // (values are adjusted for DST) - struct tm* t = localtime(&time); - - const u16 fat_time = u16( - (t->tm_sec/2) | // 5 - (u16(t->tm_min) << 5) | // 6 - (u16(t->tm_hour) << 11) // 5 - ); - - const u16 fat_date = u16( - (t->tm_mday) | // 5 - (u16(t->tm_mon+1) << 5) | // 4 - (u16(t->tm_year-80) << 9) // 7 - ); - - u32 fat_timedate = u32_from_u16(fat_date, fat_time); - return fat_timedate; -} diff --git a/source/lib/fat_time.h b/source/lib/fat_time.h deleted file mode 100644 index 83af03cc1e..0000000000 --- a/source/lib/fat_time.h +++ /dev/null @@ -1,29 +0,0 @@ -/* Copyright (c) 2010 Wildfire Games - * - * Permission is hereby granted, free of charge, to any person obtaining - * a copy of this software and associated documentation files (the - * "Software"), to deal in the Software without restriction, including - * without limitation the rights to use, copy, modify, merge, publish, - * distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to - * the following conditions: - * - * The above copyright notice and this permission notice shall be included - * in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY - * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, - * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE - * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -#ifndef INCLUDED_FAT_TIME -#define INCLUDED_FAT_TIME - -extern time_t time_t_from_FAT(u32 fat_timedate); -extern u32 FAT_from_time_t(time_t time); - -#endif // INCLUDED_FAT_TIME diff --git a/source/lib/file/archive/archive_zip.cpp b/source/lib/file/archive/archive_zip.cpp index a16b6d042a..4fac70f202 100644 --- a/source/lib/file/archive/archive_zip.cpp +++ b/source/lib/file/archive/archive_zip.cpp @@ -33,7 +33,6 @@ #include "lib/utf8.h" #include "lib/bits.h" #include "lib/byte_order.h" -#include "lib/fat_time.h" #include "lib/allocators/pool.h" #include "lib/sysdep/filesystem.h" #include "lib/file/archive/archive.h" @@ -41,8 +40,54 @@ #include "lib/file/archive/stream.h" #include "lib/file/file.h" #include "lib/file/io/io.h" -#include "lib/file/io/io_align.h" // BLOCK_SIZE -#include "lib/file/io/write_buffer.h" + +//----------------------------------------------------------------------------- +// timestamp conversion: DOS FAT <-> Unix time_t +//----------------------------------------------------------------------------- + +static time_t time_t_from_FAT(u32 fat_timedate) +{ + const u32 fat_time = bits(fat_timedate, 0, 15); + const u32 fat_date = bits(fat_timedate, 16, 31); + + struct tm t; // struct tm format: + t.tm_sec = bits(fat_time, 0,4) * 2; // [0,59] + t.tm_min = bits(fat_time, 5,10); // [0,59] + t.tm_hour = bits(fat_time, 11,15); // [0,23] + t.tm_mday = bits(fat_date, 0,4); // [1,31] + t.tm_mon = bits(fat_date, 5,8) - 1; // [0,11] + t.tm_year = bits(fat_date, 9,15) + 80; // since 1900 + t.tm_isdst = -1; // unknown - let libc determine + + // otherwise: totally bogus, and at the limit of 32-bit time_t + debug_assert(t.tm_year < 138); + + time_t ret = mktime(&t); + debug_assert(ret != (time_t)-1); // mktime shouldn't fail + return ret; +} + + +static u32 FAT_from_time_t(time_t time) +{ + // (values are adjusted for DST) + struct tm* t = localtime(&time); + + const u16 fat_time = u16( + (t->tm_sec/2) | // 5 + (u16(t->tm_min) << 5) | // 6 + (u16(t->tm_hour) << 11) // 5 + ); + + const u16 fat_date = u16( + (t->tm_mday) | // 5 + (u16(t->tm_mon+1) << 5) | // 4 + (u16(t->tm_year-80) << 9) // 7 + ); + + u32 fat_timedate = u32_from_u16(fat_date, fat_time); + return fat_timedate; +} //----------------------------------------------------------------------------- @@ -288,7 +333,8 @@ public: Stream stream(codec); stream.SetOutputBuffer(buf.get(), size); - RETURN_ERR(io_Scan(m_file, m_ofs, m_csize, FeedStream, (uintptr_t)&stream)); + io::Operation op(*m_file.get(), 0, m_csize, m_ofs); + RETURN_ERR(io::Run(op, io::Parameters(), std::bind(&Stream::Feed, &stream, std::placeholders::_1, std::placeholders::_2))); RETURN_ERR(stream.Finish()); #if CODEC_COMPUTE_CHECKSUM debug_assert(m_checksum == stream.Checksum()); @@ -315,29 +361,32 @@ private: struct LFH_Copier { - u8* lfh_dst; - size_t lfh_bytes_remaining; + LFH_Copier(u8* lfh_dst, size_t lfh_bytes_remaining) + : lfh_dst(lfh_dst), lfh_bytes_remaining(lfh_bytes_remaining) + { + } + + // this code grabs an LFH struct from file block(s) that are + // passed to the callback. usually, one call copies the whole thing, + // but the LFH may straddle a block boundary. + // + // rationale: this allows using temp buffers for zip_fixup_lfh, + // which avoids involving the file buffer manager and thus + // avoids cluttering the trace and cache contents. + LibError operator()(const u8* block, size_t size) const + { + debug_assert(size <= lfh_bytes_remaining); + memcpy(lfh_dst, block, size); + lfh_dst += size; + lfh_bytes_remaining -= size; + + return INFO::CB_CONTINUE; + } + + mutable u8* lfh_dst; + mutable size_t lfh_bytes_remaining; }; - // this code grabs an LFH struct from file block(s) that are - // passed to the callback. usually, one call copies the whole thing, - // but the LFH may straddle a block boundary. - // - // rationale: this allows using temp buffers for zip_fixup_lfh, - // which avoids involving the file buffer manager and thus - // avoids cluttering the trace and cache contents. - static LibError lfh_copier_cb(uintptr_t cbData, const u8* block, size_t size) - { - LFH_Copier* p = (LFH_Copier*)cbData; - - debug_assert(size <= p->lfh_bytes_remaining); - memcpy(p->lfh_dst, block, size); - p->lfh_dst += size; - p->lfh_bytes_remaining -= size; - - return INFO::CB_CONTINUE; - } - /** * fix up m_ofs (adjust it to point to cdata instead of the LFH). * @@ -358,8 +407,8 @@ private: // only in the block cache if the file starts in the same block as a // previously read file (i.e. both are small). LFH lfh; - LFH_Copier params = { (u8*)&lfh, sizeof(LFH) }; - if(io_Scan(m_file, m_ofs, sizeof(LFH), lfh_copier_cb, (uintptr_t)¶ms) == INFO::OK) + io::Operation op(*m_file.get(), 0, sizeof(LFH), m_ofs); + if(io::Run(op, io::Parameters(), LFH_Copier((u8*)&lfh, sizeof(LFH))) == INFO::OK) m_ofs += (off_t)lfh.Size(); } @@ -382,7 +431,7 @@ class ArchiveReader_Zip : public IArchiveReader { public: ArchiveReader_Zip(const OsPath& pathname) - : m_file(new File(pathname, 'r')) + : m_file(new File(pathname, LIO_READ)) { FileInfo fileInfo; GetFileInfo(pathname, &fileInfo); @@ -398,16 +447,17 @@ public: size_t cd_numEntries = 0; size_t cd_size = 0; RETURN_ERR(LocateCentralDirectory(m_file, m_fileSize, cd_ofs, cd_numEntries, cd_size)); - shared_ptr buf = io_Allocate(cd_size, cd_ofs); - u8* cd; - RETURN_ERR(io_Read(m_file, cd_ofs, buf.get(), cd_size, cd)); + UniqueRange buf(io::Allocate(cd_size)); + + io::Operation op(*m_file.get(), buf.get(), cd_size, cd_ofs); + RETURN_ERR(io::Run(op)); // iterate over Central Directory - const u8* pos = cd; + const u8* pos = (const u8*)buf.get(); for(size_t i = 0; i < cd_numEntries; i++) { // scan for next CDFH - CDFH* cdfh = (CDFH*)FindRecord(cd, cd_size, pos, cdfh_magic, sizeof(CDFH)); + CDFH* cdfh = (CDFH*)FindRecord((const u8*)buf.get(), cd_size, pos, cdfh_magic, sizeof(CDFH)); if(!cdfh) WARN_RETURN(ERR::CORRUPTED); @@ -467,11 +517,11 @@ private: // read desired chunk of file into memory const off_t ofs = fileSize - off_t(scanSize); - u8* data; - RETURN_ERR(io_Read(file, ofs, buf, scanSize, data)); + io::Operation op(*file.get(), buf, scanSize, ofs); + RETURN_ERR(io::Run(op)); // look for ECDR in buffer - const ECDR* ecdr = (const ECDR*)FindRecord(data, scanSize, data, ecdr_magic, sizeof(ECDR)); + const ECDR* ecdr = (const ECDR*)FindRecord(buf, scanSize, buf, ecdr_magic, sizeof(ECDR)); if(!ecdr) return INFO::CANNOT_HANDLE; @@ -482,19 +532,20 @@ private: static LibError LocateCentralDirectory(const PFile& file, off_t fileSize, off_t& cd_ofs, size_t& cd_numEntries, size_t& cd_size) { const size_t maxScanSize = 66000u; // see below - shared_ptr buf = io_Allocate(maxScanSize, BLOCK_SIZE-1); // assume worst-case for alignment + UniqueRange buf(io::Allocate(maxScanSize)); // expected case: ECDR at EOF; no file comment - LibError ret = ScanForEcdr(file, fileSize, const_cast(buf.get()), sizeof(ECDR), cd_numEntries, cd_ofs, cd_size); + LibError ret = ScanForEcdr(file, fileSize, (u8*)buf.get(), sizeof(ECDR), cd_numEntries, cd_ofs, cd_size); if(ret == INFO::OK) return INFO::OK; // worst case: ECDR precedes 64 KiB of file comment - ret = ScanForEcdr(file, fileSize, const_cast(buf.get()), maxScanSize, cd_numEntries, cd_ofs, cd_size); + ret = ScanForEcdr(file, fileSize, (u8*)buf.get(), maxScanSize, cd_numEntries, cd_ofs, cd_size); if(ret == INFO::OK) return INFO::OK; // both ECDR scans failed - this is not a valid Zip file. - RETURN_ERR(io_ReadAligned(file, 0, const_cast(buf.get()), sizeof(LFH))); + io::Operation op(*file.get(), buf.get(), sizeof(LFH)); + RETURN_ERR(io::Run(op)); // the Zip file has an LFH but lacks an ECDR. this can happen if // the user hard-exits while an archive is being written. // notes: @@ -503,7 +554,7 @@ private: // because it'd be slow. // - do not warn - the corrupt archive will be deleted on next // successful archive builder run anyway. - if(FindRecord(buf.get(), sizeof(LFH), buf.get(), lfh_magic, sizeof(LFH))) + if(FindRecord((const u8*)buf.get(), sizeof(LFH), (const u8*)buf.get(), lfh_magic, sizeof(LFH))) return ERR::CORRUPTED; // NOWARN // totally bogus else @@ -528,8 +579,7 @@ class ArchiveWriter_Zip : public IArchiveWriter { public: ArchiveWriter_Zip(const OsPath& archivePathname, bool noDeflate) - : m_file(new File(archivePathname, 'w')), m_fileSize(0) - , m_unalignedWriter(new UnalignedWriter(m_file, 0)) + : m_file(new File(archivePathname, LIO_WRITE)), m_fileSize(0) , m_numEntries(0), m_noDeflate(noDeflate) { THROW_ERR(pool_create(&m_cdfhPool, 10*MiB, 0)); @@ -546,19 +596,9 @@ public: const off_t cd_ofs = m_fileSize; ecdr->Init(m_numEntries, cd_ofs, cd_size); - m_unalignedWriter->Append(m_cdfhPool.da.base, cd_size+sizeof(ECDR)); - m_unalignedWriter->Flush(); - m_unalignedWriter.reset(); + write(m_file->Descriptor(), m_cdfhPool.da.base, cd_size+sizeof(ECDR)); (void)pool_destroy(&m_cdfhPool); - - const OsPath pathname = m_file->Pathname(); // (must be retrieved before resetting m_file) - m_file.reset(); - - m_fileSize += off_t(cd_size+sizeof(ECDR)); - - // remove padding added by UnalignedWriter - wtruncate(pathname, m_fileSize); } LibError AddFile(const OsPath& pathname, const OsPath& pathnameInArchive) @@ -578,7 +618,7 @@ public: return INFO::SKIPPED; PFile file(new File); - RETURN_ERR(file->Open(pathname, 'r')); + RETURN_ERR(file->Open(pathname, LIO_READ)); const size_t pathnameLength = pathnameInArchive.string().length(); @@ -598,7 +638,7 @@ public: // allocate memory const size_t csizeMax = codec->MaxOutputSize(size_t(usize)); - shared_ptr buf = io_Allocate(sizeof(LFH) + pathnameLength + csizeMax); + UniqueRange buf(io::Allocate(sizeof(LFH) + pathnameLength + csizeMax)); // read and compress file contents size_t csize; u32 checksum; @@ -606,7 +646,8 @@ public: u8* cdata = (u8*)buf.get() + sizeof(LFH) + pathnameLength; Stream stream(codec); stream.SetOutputBuffer(cdata, csizeMax); - RETURN_ERR(io_Scan(file, 0, usize, FeedStream, (uintptr_t)&stream)); + io::Operation op(*file.get(), 0, usize); + RETURN_ERR(io::Run(op, io::Parameters(), std::bind(&Stream::Feed, &stream, std::placeholders::_1, std::placeholders::_2))); RETURN_ERR(stream.Finish()); csize = stream.OutSize(); checksum = stream.Checksum(); @@ -631,7 +672,8 @@ public: // write LFH, pathname and cdata to file const size_t packageSize = sizeof(LFH) + pathnameLength + csize; - RETURN_ERR(m_unalignedWriter->Append(buf.get(), packageSize)); + if(write(m_file->Descriptor(), buf.get(), packageSize) < 0) + WARN_RETURN(ERR::IO); m_fileSize += (off_t)packageSize; return INFO::OK; @@ -661,7 +703,6 @@ private: PFile m_file; off_t m_fileSize; - PUnalignedWriter m_unalignedWriter; Pool m_cdfhPool; size_t m_numEntries; diff --git a/source/lib/file/archive/stream.cpp b/source/lib/file/archive/stream.cpp index 28a4d4ea89..f312cf5470 100644 --- a/source/lib/file/archive/stream.cpp +++ b/source/lib/file/archive/stream.cpp @@ -136,12 +136,3 @@ LibError Stream::Finish() m_outProduced += outProduced; return INFO::OK; } - - -LibError FeedStream(uintptr_t cbData, const u8* in, size_t inSize) -{ -// TIMER_ACCRUE(tc_stream); - - Stream& stream = *(Stream*)cbData; - return stream.Feed(in, inSize); -} diff --git a/source/lib/file/archive/stream.h b/source/lib/file/archive/stream.h index ab1e80cdb0..8b6f923bb8 100644 --- a/source/lib/file/archive/stream.h +++ b/source/lib/file/archive/stream.h @@ -110,6 +110,4 @@ private: u32 m_checksum; }; -extern LibError FeedStream(uintptr_t cbData, const u8* in, size_t inSize); - #endif // #ifndef INCLUDED_STREAM diff --git a/source/lib/file/common/file_stats.cpp b/source/lib/file/common/file_stats.cpp index 53e6d33eb0..3c5d01e09c 100644 --- a/source/lib/file/common/file_stats.cpp +++ b/source/lib/file/common/file_stats.cpp @@ -189,23 +189,13 @@ ScopedIoMonitor::~ScopedIoMonitor() timer_reset(&m_startTime); } -void ScopedIoMonitor::NotifyOfSuccess(FileIOImplentation fi, wchar_t mode, off_t size) +void ScopedIoMonitor::NotifyOfSuccess(FileIOImplentation fi, int opcode, off_t size) { debug_assert(fi < FI_MAX_IDX); - debug_assert(mode == 'r' || mode == 'w'); - const FileOp op = (mode == 'r')? FO_READ : FO_WRITE; + debug_assert(opcode == LIO_READ || opcode == LIO_WRITE); - io_actual_size_total[fi][op] += size; - io_elapsed_time[fi][op] += timer_reset(&m_startTime); -} - -void stats_io_check_seek(BlockId& blockId) -{ - static BlockId lastBlockId; - - if(blockId != lastBlockId) - io_seeks++; - lastBlockId = blockId; + io_actual_size_total[fi][opcode == LIO_WRITE] += size; + io_elapsed_time[fi][opcode == LIO_WRITE] += timer_reset(&m_startTime); } @@ -320,11 +310,11 @@ void file_stats_dump() L"Average size = %g KB; seeks: %lu; total callback time: %g ms\n" L"Total data actually read from disk = %g MB\n", (unsigned long)user_ios, user_io_size_total/MB, -#define THROUGHPUT(impl, op) (io_elapsed_time[impl][op] == 0.0)? 0.0 : (io_actual_size_total[impl][op] / io_elapsed_time[impl][op] / MB) - THROUGHPUT(FI_LOWIO, FO_READ), THROUGHPUT(FI_LOWIO, FO_WRITE), - THROUGHPUT(FI_AIO , FO_READ), THROUGHPUT(FI_AIO , FO_WRITE), +#define THROUGHPUT(impl, opcode) (io_elapsed_time[impl][opcode == LIO_WRITE] == 0.0)? 0.0 : (io_actual_size_total[impl][opcode == LIO_WRITE] / io_elapsed_time[impl][opcode == LIO_WRITE] / MB) + THROUGHPUT(FI_LOWIO, LIO_READ), THROUGHPUT(FI_LOWIO, LIO_WRITE), + THROUGHPUT(FI_AIO , LIO_READ), THROUGHPUT(FI_AIO , LIO_WRITE), user_io_size_total/user_ios/KB, (unsigned long)io_seeks, io_process_time_total/ms, - (io_actual_size_total[FI_LOWIO][FO_READ]+io_actual_size_total[FI_AIO][FO_READ])/MB + (io_actual_size_total[FI_LOWIO][0]+io_actual_size_total[FI_AIO][0])/MB ); debug_printf( diff --git a/source/lib/file/common/file_stats.h b/source/lib/file/common/file_stats.h index 8e14f468d0..42351e54da 100644 --- a/source/lib/file/common/file_stats.h +++ b/source/lib/file/common/file_stats.h @@ -27,15 +27,14 @@ #ifndef INCLUDED_FILE_STATS #define INCLUDED_FILE_STATS +#include "lib/posix/posix_aio.h" // LIO_READ, LIO_WRITE + #define FILE_STATS_ENABLED 0 enum FileIOImplentation { FI_LOWIO, FI_AIO, FI_BCACHE, FI_MAX_IDX }; -enum FileOp { FO_READ, FO_WRITE }; enum CacheRet { CR_HIT, CR_MISS }; -#include "lib/file/io/block_cache.h" // BlockId - #if FILE_STATS_ENABLED // vfs @@ -69,13 +68,12 @@ class ScopedIoMonitor public: ScopedIoMonitor(); ~ScopedIoMonitor(); - void NotifyOfSuccess(FileIOImplentation fi, wchar_t mode, off_t size); + void NotifyOfSuccess(FileIOImplentation fi, int opcode, off_t size); private: double m_startTime; }; -extern void stats_io_check_seek(BlockId& blockId); extern void stats_cb_start(); extern void stats_cb_finish(); @@ -106,9 +104,8 @@ class ScopedIoMonitor public: ScopedIoMonitor() {} ~ScopedIoMonitor() {} - void NotifyOfSuccess(FileIOImplentation UNUSED(fi), wchar_t UNUSED(mode), off_t UNUSED(size)) {} + void NotifyOfSuccess(FileIOImplentation UNUSED(fi), int UNUSED(opcode), off_t UNUSED(size)) {} }; -#define stats_io_check_seek(blockId) #define stats_cb_start() #define stats_cb_finish() #define stats_cache(cr, size) diff --git a/source/lib/file/common/real_directory.cpp b/source/lib/file/common/real_directory.cpp index 6faad12e46..b6c03e37b6 100644 --- a/source/lib/file/common/real_directory.cpp +++ b/source/lib/file/common/real_directory.cpp @@ -48,33 +48,13 @@ RealDirectory::RealDirectory(const OsPath& path, size_t priority, size_t flags) /*virtual*/ LibError RealDirectory::Load(const OsPath& name, const shared_ptr& buf, size_t size) const { - const OsPath pathname = m_path / name; - - PFile file(new File); - RETURN_ERR(file->Open(pathname, 'r')); - - RETURN_ERR(io_ReadAligned(file, 0, buf.get(), size)); - return INFO::OK; + return io::Load(m_path / name, buf.get(), size); } LibError RealDirectory::Store(const OsPath& name, const shared_ptr& fileContents, size_t size) { - const OsPath pathname = m_path / name; - - { - PFile file(new File); - RETURN_ERR(file->Open(pathname, 'w')); - RETURN_ERR(io_WriteAligned(file, 0, fileContents.get(), size)); - } - - // io_WriteAligned pads the file; we need to truncate it to the actual - // length. ftruncate can't be used because Windows' FILE_FLAG_NO_BUFFERING - // only allows resizing to sector boundaries, so the file must first - // be closed. - wtruncate(pathname, size); - - return INFO::OK; + return io::Store(m_path / name, fileContents.get(), size); } diff --git a/source/lib/file/common/trace.cpp b/source/lib/file/common/trace.cpp index 6900d565f0..fd67cf06fb 100644 --- a/source/lib/file/common/trace.cpp +++ b/source/lib/file/common/trace.cpp @@ -31,7 +31,6 @@ #include #include "lib/allocators/pool.h" -#include "lib/bits.h" // round_up #include "lib/timer.h" // timer_Time #include "lib/sysdep/sysdep.h" // sys_OpenFile diff --git a/source/lib/file/file.cpp b/source/lib/file/file.cpp index f81060bd8d..494ae7a79d 100644 --- a/source/lib/file/file.cpp +++ b/source/lib/file/file.cpp @@ -27,34 +27,27 @@ #include "precompiled.h" #include "lib/file/file.h" -#include "lib/config2.h" #include "lib/sysdep/filesystem.h" // O_*, S_* -#include "lib/posix/posix_aio.h" #include "lib/file/common/file_stats.h" ERROR_ASSOCIATE(ERR::FILE_ACCESS, L"Insufficient access rights to open file", EACCES); -ERROR_ASSOCIATE(ERR::IO, L"Error during IO", EIO); -namespace FileImpl { - -LibError Open(const OsPath& pathname, wchar_t accessType, int& fd) +LibError FileOpen(const OsPath& pathname, int opcode, int& fd) { int oflag = 0; - switch(accessType) + switch(opcode) { - case 'r': + case LIO_READ: oflag = O_RDONLY; break; - case 'w': + case LIO_WRITE: oflag = O_WRONLY|O_CREAT|O_TRUNC; break; - case '+': - oflag = O_RDWR; - break; default: debug_assert(0); + break; } #if OS_WIN oflag |= O_BINARY_NP; @@ -71,7 +64,7 @@ LibError Open(const OsPath& pathname, wchar_t accessType, int& fd) } -void Close(int& fd) +void FileClose(int& fd) { if(fd >= 0) { @@ -79,88 +72,3 @@ void Close(int& fd) fd = -1; } } - - -LibError IO(int fd, wchar_t accessType, off_t ofs, u8* buf, size_t size) -{ - debug_assert(accessType == 'r' || accessType == 'w'); - - ScopedIoMonitor monitor; - - lseek(fd, ofs, SEEK_SET); - - errno = 0; - const ssize_t ret = (accessType == 'w')? write(fd, buf, size) : read(fd, buf, size); - if(ret < 0) - return LibError_from_errno(); - - const size_t totalTransferred = (size_t)ret; -#if CONFIG2_FILE_ENABLE_AIO - // we won't be called from Issue, i.e. size is always the exact - // value without padding and can be checked. - if(totalTransferred != size) - WARN_RETURN(ERR::IO); -#endif - - monitor.NotifyOfSuccess(FI_LOWIO, accessType, totalTransferred); - return INFO::OK; -} - - -LibError Issue(aiocb& req, int fd, wchar_t accessType, off_t alignedOfs, u8* alignedBuf, size_t alignedSize) -{ - memset(&req, 0, sizeof(req)); - req.aio_lio_opcode = (accessType == 'w')? LIO_WRITE : LIO_READ; - req.aio_buf = (volatile void*)alignedBuf; - req.aio_fildes = fd; - req.aio_offset = alignedOfs; - req.aio_nbytes = alignedSize; -#if CONFIG2_FILE_ENABLE_AIO - struct sigevent* sig = 0; // no notification signal - aiocb* const reqs = &req; - if(lio_listio(LIO_NOWAIT, &reqs, 1, sig) != 0) - return LibError_from_errno(); - return INFO::OK; -#else - // quick and dirty workaround (see CONFIG2_FILE_ENABLE_AIO) - return IO(fd, accessType, alignedOfs, alignedBuf, alignedSize); -#endif -} - - -LibError WaitUntilComplete(aiocb& req, u8*& alignedBuf, size_t& alignedSize) -{ -#if CONFIG2_FILE_ENABLE_AIO - const int err = aio_error(&req); - if(err == EINPROGRESS) - { -SUSPEND_AGAIN: - aiocb* const reqs = &req; - errno = 0; - const int ret = aio_suspend(&reqs, 1, (timespec*)0); // no timeout - if(ret != 0) - { - if(errno == EINTR) // interrupted by signal - goto SUSPEND_AGAIN; - return LibError_from_errno(); - } - } - else if(err != 0) - { - errno = err; - return LibError_from_errno(); - } - - const ssize_t bytesTransferred = aio_return(&req); - if(bytesTransferred == -1) // transfer failed - WARN_RETURN(ERR::IO); - - alignedSize = bytesTransferred; -#else - alignedSize = req.aio_nbytes; -#endif - alignedBuf = (u8*)req.aio_buf; // cast from volatile void* - return INFO::OK; -} - -} // namespace FileImpl diff --git a/source/lib/file/file.h b/source/lib/file/file.h index 0dbfb91c8d..5fa5f3227d 100644 --- a/source/lib/file/file.h +++ b/source/lib/file/file.h @@ -27,50 +27,28 @@ #ifndef INCLUDED_FILE #define INCLUDED_FILE -struct aiocb; - #include "lib/os_path.h" +#include "lib/posix/posix_aio.h" // opcode: LIO_READ or LIO_WRITE namespace ERR { const LibError FILE_ACCESS = -110300; - const LibError IO = -110301; -} - -namespace FileImpl -{ - LIB_API LibError Open(const OsPath& pathname, wchar_t mode, int& fd); - LIB_API void Close(int& fd); - LIB_API LibError IO(int fd, wchar_t mode, off_t ofs, u8* buf, size_t size); - LIB_API LibError Issue(aiocb& req, int fd, wchar_t mode, off_t alignedOfs, u8* alignedBuf, size_t alignedSize); - LIB_API LibError WaitUntilComplete(aiocb& req, u8*& alignedBuf, size_t& alignedSize); } +LIB_API LibError FileOpen(const OsPath& pathname, int opcode, int& fd); +LIB_API void FileClose(int& fd); class File { public: File() - : m_pathname(), m_fd(0) + : pathname(), fd(-1) { } - LibError Open(const OsPath& pathname, wchar_t mode) + File(const OsPath& pathname, int opcode) { - RETURN_ERR(FileImpl::Open(pathname, mode, m_fd)); - m_pathname = pathname; - m_mode = mode; - return INFO::OK; - } - - void Close() - { - FileImpl::Close(m_fd); - } - - File(const OsPath& pathname, wchar_t mode) - { - (void)Open(pathname, mode); + (void)Open(pathname, opcode); } ~File() @@ -78,35 +56,38 @@ public: Close(); } + LibError Open(const OsPath& pathname, int opcode) + { + RETURN_ERR(FileOpen(pathname, opcode, fd)); + this->pathname = pathname; + this->opcode = opcode; + return INFO::OK; + } + + void Close() + { + FileClose(fd); + } + const OsPath& Pathname() const { - return m_pathname; + return pathname; } - wchar_t Mode() const + int Descriptor() const { - return m_mode; + return fd; } - LibError Issue(aiocb& req, wchar_t mode, off_t alignedOfs, u8* alignedBuf, size_t alignedSize) const + int Opcode() const { - return FileImpl::Issue(req, m_fd, mode, alignedOfs, alignedBuf, alignedSize); - } - - LibError Write(off_t ofs, const u8* buf, size_t size) - { - return FileImpl::IO(m_fd, 'w', ofs, const_cast(buf), size); - } - - LibError Read(off_t ofs, u8* buf, size_t size) const - { - return FileImpl::IO(m_fd, 'r', ofs, buf, size); + return opcode; } private: - OsPath m_pathname; - int m_fd; - wchar_t m_mode; + OsPath pathname; + int fd; + int opcode; }; typedef shared_ptr PFile; diff --git a/source/lib/file/io/block_cache.cpp b/source/lib/file/io/block_cache.cpp deleted file mode 100644 index 8f320e5e9c..0000000000 --- a/source/lib/file/io/block_cache.cpp +++ /dev/null @@ -1,163 +0,0 @@ -/* Copyright (c) 2010 Wildfire Games - * - * Permission is hereby granted, free of charge, to any person obtaining - * a copy of this software and associated documentation files (the - * "Software"), to deal in the Software without restriction, including - * without limitation the rights to use, copy, modify, merge, publish, - * distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to - * the following conditions: - * - * The above copyright notice and this permission notice shall be included - * in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY - * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, - * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE - * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -/* - * cache for aligned I/O blocks. - */ - -#include "precompiled.h" -#include "lib/file/io/block_cache.h" - -#include "lib/config2.h" // CONFIG2_CACHE_READ_ONLY -#include "lib/posix/posix_mman.h" // mprotect -#include "lib/file/common/file_stats.h" -#include "lib/lockfree.h" -#include "lib/allocators/pool.h" -#include "lib/fnv_hash.h" -#include "lib/file/io/io_align.h" - - -//----------------------------------------------------------------------------- - -BlockId::BlockId() - : m_id(0) -{ -} - -BlockId::BlockId(const OsPath& pathname, off_t ofs) -{ - const Path::String& string = pathname.string(); - m_id = fnv_hash64(string.c_str(), string.length()*sizeof(string[0])); - const size_t indexBits = 16; - m_id <<= indexBits; - const off_t blockIndex = off_t(ofs / BLOCK_SIZE); - debug_assert(blockIndex < off_t(1) << indexBits); - m_id |= blockIndex; -} - -bool BlockId::operator==(const BlockId& rhs) const -{ - return m_id == rhs.m_id; -} - -bool BlockId::operator!=(const BlockId& rhs) const -{ - return !operator==(rhs); -} - - -//----------------------------------------------------------------------------- - -struct Block -{ - Block(BlockId id, const shared_ptr& buf) - { - this->id = id; - this->buf = buf; - } - - // block is "valid" and can satisfy Retrieve() requests if a - // (non-default-constructed) ID has been assigned. - BlockId id; - - // this block is "in use" if use_count != 1. - shared_ptr buf; -}; - - -//----------------------------------------------------------------------------- - -class BlockCache::Impl -{ -public: - Impl(size_t numBlocks) - : m_maxBlocks(numBlocks) - { - } - - void Add(BlockId id, const shared_ptr& buf) - { - if(m_blocks.size() > m_maxBlocks) - { -#if CONFIG2_CACHE_READ_ONLY - mprotect((void*)m_blocks.front().buf.get(), BLOCK_SIZE, PROT_READ); -#endif - m_blocks.pop_front(); // evict oldest block - } - -#if CONFIG2_CACHE_READ_ONLY - mprotect((void*)buf.get(), BLOCK_SIZE, PROT_WRITE|PROT_READ); -#endif - m_blocks.push_back(Block(id, buf)); - } - - bool Retrieve(BlockId id, shared_ptr& buf) - { - // (linear search is ok since we only expect to manage a few blocks) - for(size_t i = 0; i < m_blocks.size(); i++) - { - Block& block = m_blocks[i]; - if(block.id == id) - { - buf = block.buf; - return true; - } - } - - return false; - } - - void InvalidateAll() - { - // note: don't check whether any references are held etc. because - // this should only be called at the end of the (test) program. - m_blocks.clear(); - } - -private: - size_t m_maxBlocks; - typedef std::deque Blocks; - Blocks m_blocks; -}; - - -//----------------------------------------------------------------------------- - -BlockCache::BlockCache(size_t numBlocks) - : impl(new Impl(numBlocks)) -{ -} - -void BlockCache::Add(BlockId id, const shared_ptr& buf) -{ - impl->Add(id, buf); -} - -bool BlockCache::Retrieve(BlockId id, shared_ptr& buf) -{ - return impl->Retrieve(id, buf); -} - -void BlockCache::InvalidateAll() -{ - return impl->InvalidateAll(); -} diff --git a/source/lib/file/io/block_cache.h b/source/lib/file/io/block_cache.h deleted file mode 100644 index d5250f3c52..0000000000 --- a/source/lib/file/io/block_cache.h +++ /dev/null @@ -1,106 +0,0 @@ -/* Copyright (c) 2010 Wildfire Games - * - * Permission is hereby granted, free of charge, to any person obtaining - * a copy of this software and associated documentation files (the - * "Software"), to deal in the Software without restriction, including - * without limitation the rights to use, copy, modify, merge, publish, - * distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to - * the following conditions: - * - * The above copyright notice and this permission notice shall be included - * in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY - * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, - * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE - * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -/* - * cache for aligned I/O blocks. - */ - -#ifndef INCLUDED_BLOCK_CACHE -#define INCLUDED_BLOCK_CACHE - -#include "lib/os_path.h" - -/** - * ID that uniquely identifies a block within a file - **/ -class BlockId -{ -public: - BlockId(); - BlockId(const OsPath& pathname, off_t ofs); - bool operator==(const BlockId& rhs) const; - bool operator!=(const BlockId& rhs) const; - -private: - u64 m_id; -}; - - -/** - * cache of (aligned) file blocks with support for zero-copy IO. - * absorbs the overhead of rounding up archive IOs to the nearest block - * boundaries by keeping the last few blocks in memory. - * - * the interface is somewhat similar to FileCache; see the note there. - * - * not thread-safe (each thread is intended to have its own cache). - **/ -class BlockCache -{ -public: - /** - * @param numBlocks (the default value is enough to support temp buffers - * and absorb the cost of unaligned reads from archives.) - **/ - BlockCache(size_t numBlocks = 16); - - /** - * Add a block to the cache. - * - * @param id Key that will be used to Retrieve the block. - * @param buf - * - * Call this when the block's IO has completed; its data will - * satisfy subsequent Retrieve calls for the same id. - * If CONFIG2_CACHE_READ_ONLY, the memory is made read-only. - **/ - void Add(BlockId id, const shared_ptr& buf); - - /** - * Attempt to retrieve a block's contents. - * - * @return whether the block is in cache. - * - * if successful, a shared pointer to the contents is returned. - * they remain valid until all references are removed and the block - * is evicted. - **/ - bool Retrieve(BlockId id, shared_ptr& buf); - - /** - * Invalidate the contents of the cache. - * - * this effectively discards the contents of existing blocks - * (more specifically: prevents them from satisfying Retrieve calls - * until a subsequent Add with the same id). - * - * useful for self-tests: multiple independent IO tests run in the same - * process and must not influence each other via the cache. - **/ - void InvalidateAll(); - -private: - class Impl; - shared_ptr impl; -}; - -#endif // #ifndef INCLUDED_BLOCK_CACHE diff --git a/source/lib/file/io/io.cpp b/source/lib/file/io/io.cpp index c19ea4c724..0c58c9da03 100644 --- a/source/lib/file/io/io.cpp +++ b/source/lib/file/io/io.cpp @@ -1,4 +1,4 @@ -/* Copyright (c) 2010 Wildfire Games +/* Copyright (c) 2011 Wildfire Games * * Permission is hereby granted, free of charge, to any person obtaining * a copy of this software and associated documentation files (the @@ -23,19 +23,14 @@ #include "precompiled.h" #include "lib/file/io/io.h" -#include "lib/posix/posix_aio.h" -#include "lib/allocators/allocators.h" // AllocatorChecker -#include "lib/file/file.h" -#include "lib/file/common/file_stats.h" -#include "lib/file/io/block_cache.h" -#include "lib/file/io/io_align.h" +#include "lib/sysdep/rtl.h" -static const size_t ioDepth = 16; +ERROR_ASSOCIATE(ERR::IO, L"Error during IO", EIO); +namespace io { -// the underlying aio implementation likes buffer and offset to be -// sector-aligned; if not, the transfer goes through an align buffer, -// and requires an extra memcpy. +// the Windows aio implementation requires buffer and offset to be +// sector-aligned. // // if the user specifies an unaligned buffer, there's not much we can // do - we can't assume the buffer contains padding. therefore, @@ -51,14 +46,14 @@ static const size_t ioDepth = 16; // AIO wrapper. rationale: // - aligning the transfer isn't possible here since we have no control // over the buffer, i.e. we cannot read more data than requested. -// instead, this is done in io_manager. +// instead, this is done in manager. // - transfer sizes here are arbitrary (i.e. not block-aligned); // that means the cache would have to handle this or also split them up -// into blocks, which would duplicate the abovementioned work. +// into blocks, which would duplicate the above mentioned work. // - if caching here, we'd also have to handle "forwarding" (i.e. // desired block has been issued but isn't yet complete). again, it -// is easier to let the synchronous io_manager handle this. -// - finally, io_manager knows more about whether the block should be cached +// is easier to let the synchronous manager handle this. +// - finally, manager knows more about whether the block should be cached // (e.g. whether another block request will follow), but we don't // currently make use of this. // @@ -71,271 +66,73 @@ static const size_t ioDepth = 16; // this is a bit more complicated than just using the cache as storage. -//----------------------------------------------------------------------------- -// allocator -//----------------------------------------------------------------------------- - -#ifndef NDEBUG -static AllocatorChecker allocatorChecker; -#endif - -class IoDeleter +UniqueRange Allocate(size_t size, size_t alignment) { -public: - IoDeleter(size_t paddedSize) - : m_paddedSize(paddedSize) - { - } + debug_assert(is_pow2(alignment)); + if(alignment <= idxDeleterBits) + alignment = idxDeleterBits+1; - void operator()(u8* mem) - { - debug_assert(m_paddedSize != 0); -#ifndef NDEBUG - allocatorChecker.OnDeallocate(mem, m_paddedSize); -#endif - page_aligned_free(mem, m_paddedSize); - m_paddedSize = 0; - } - -private: - size_t m_paddedSize; -}; - - -shared_ptr io_Allocate(size_t size, off_t ofs) -{ - debug_assert(size != 0); - - const size_t paddedSize = (size_t)PaddedSize(size, ofs); - u8* mem = (u8*)page_aligned_alloc(paddedSize); - if(!mem) - throw std::bad_alloc(); - -#ifndef NDEBUG - allocatorChecker.OnAllocate(mem, paddedSize); -#endif - - return shared_ptr(mem, IoDeleter(paddedSize)); + const size_t alignedSize = round_up(size, alignment); + const UniqueRange::pointer p = rtl_AllocateAligned(alignedSize, alignment); + return UniqueRange(p, size, idxDeleterAligned); } -//----------------------------------------------------------------------------- -// BlockIo -//----------------------------------------------------------------------------- - -class BlockIo +LibError Issue(aiocb& cb, size_t queueDepth) { -public: - LibError Issue(const PFile& file, off_t alignedOfs, u8* alignedBuf) +#if CONFIG2_FILE_ENABLE_AIO + if(queueDepth > 1) { - m_blockId = BlockId(file->Pathname(), alignedOfs); - if(file->Mode() == 'r') - { - if(s_blockCache.Retrieve(m_blockId, m_cachedBlock)) - { - stats_block_cache(CR_HIT); + const int ret = (cb.aio_lio_opcode == LIO_WRITE)? aio_write(&cb): aio_read(&cb); + RETURN_ERR(LibError_from_posix(ret)); + } + else +#endif + { + debug_assert(lseek(cb.aio_fildes, cb.aio_offset, SEEK_SET) == cb.aio_offset); - // copy from cache into user buffer - if(alignedBuf) - { - memcpy(alignedBuf, m_cachedBlock.get(), BLOCK_SIZE); - m_alignedBuf = alignedBuf; - } - // return cached block - else - { - m_alignedBuf = const_cast(m_cachedBlock.get()); - } + void* buf = (void*)cb.aio_buf; // cast from volatile void* + const ssize_t bytesTransferred = (cb.aio_lio_opcode == LIO_WRITE)? write(cb.aio_fildes, buf, cb.aio_nbytes) : read(cb.aio_fildes, buf, cb.aio_nbytes); + if(bytesTransferred < 0) + return LibError_from_errno(); - return INFO::OK; - } - else - { - stats_block_cache(CR_MISS); - // fall through to the actual issue.. - } - } - - stats_io_check_seek(m_blockId); - - // transfer directly to/from user buffer - if(alignedBuf) - { - m_alignedBuf = alignedBuf; - } - // transfer into newly allocated temporary block - else - { - m_tempBlock = io_Allocate(BLOCK_SIZE); - m_alignedBuf = const_cast(m_tempBlock.get()); - } - - return file->Issue(m_req, file->Mode(), alignedOfs, m_alignedBuf, BLOCK_SIZE); + cb.aio_nbytes = (size_t)bytesTransferred; } - LibError WaitUntilComplete(const u8*& block, size_t& blockSize) - { - if(m_cachedBlock) - { - block = m_alignedBuf; - blockSize = BLOCK_SIZE; - return INFO::OK; - } - - RETURN_ERR(FileImpl::WaitUntilComplete(m_req, const_cast(block), blockSize)); - - if(m_tempBlock) - s_blockCache.Add(m_blockId, m_tempBlock); - - return INFO::OK; - } - -private: - static BlockCache s_blockCache; - - BlockId m_blockId; - - // the address that WaitUntilComplete will return - // (cached or temporary block, or user buffer) - u8* m_alignedBuf; - - shared_ptr m_cachedBlock; - shared_ptr m_tempBlock; - - aiocb m_req; -}; - -BlockCache BlockIo::s_blockCache; - - -//----------------------------------------------------------------------------- -// IoSplitter -//----------------------------------------------------------------------------- - -class IoSplitter -{ - NONCOPYABLE(IoSplitter); -public: - IoSplitter(off_t ofs, u8* alignedBuf, off_t size) - : m_ofs(ofs), m_alignedBuf(alignedBuf), m_size(size) - , m_totalIssued(0), m_totalTransferred(0) - { - m_alignedOfs = AlignedOffset(ofs); - m_alignedSize = PaddedSize(size, ofs); - m_misalignment = size_t(ofs - m_alignedOfs); - } - - LibError Run(const PFile& file, IoCallback cb = 0, uintptr_t cbData = 0) - { - ScopedIoMonitor monitor; - - // (issue even if cache hit because blocks must be processed in order) - std::deque pendingIos; - for(;;) - { - while(pendingIos.size() < ioDepth && m_totalIssued < m_alignedSize) - { - pendingIos.push_back(BlockIo()); - const off_t alignedOfs = m_alignedOfs + m_totalIssued; - u8* const alignedBuf = m_alignedBuf? m_alignedBuf+m_totalIssued : 0; - RETURN_ERR(pendingIos.back().Issue(file, alignedOfs, alignedBuf)); - m_totalIssued += BLOCK_SIZE; - } - - if(pendingIos.empty()) - break; - - Process(pendingIos.front(), cb, cbData); - pendingIos.pop_front(); - } - - debug_assert(m_totalIssued >= m_totalTransferred && m_totalTransferred >= m_size); - - monitor.NotifyOfSuccess(FI_AIO, file->Mode(), m_totalTransferred); - return INFO::OK; - } - - off_t AlignedOfs() const - { - return m_alignedOfs; - } - -private: - LibError Process(BlockIo& blockIo, IoCallback cb, uintptr_t cbData) const - { - const u8* block; size_t blockSize; - RETURN_ERR(blockIo.WaitUntilComplete(block, blockSize)); - - // first block: skip past alignment - if(m_totalTransferred == 0) - { - block += m_misalignment; - blockSize -= m_misalignment; - } - - // last block: don't include trailing padding - if(m_totalTransferred + (off_t)blockSize > m_size) - blockSize = size_t(m_size - m_totalTransferred); - - m_totalTransferred += (off_t)blockSize; - - if(cb) - { - stats_cb_start(); - LibError ret = cb(cbData, block, blockSize); - stats_cb_finish(); - CHECK_ERR(ret); - } - - return INFO::OK; - } - - off_t m_ofs; - u8* m_alignedBuf; - off_t m_size; - - size_t m_misalignment; - off_t m_alignedOfs; - off_t m_alignedSize; - - // (useful, raw data: possibly compressed, but doesn't count padding) - mutable off_t m_totalIssued; - mutable off_t m_totalTransferred; -}; - - -LibError io_Scan(const PFile& file, off_t ofs, off_t size, IoCallback cb, uintptr_t cbData) -{ - u8* alignedBuf = 0; // use temporary block buffers - IoSplitter splitter(ofs, alignedBuf, size); - return splitter.Run(file, cb, cbData); -} - - -LibError io_Read(const PFile& file, off_t ofs, u8* alignedBuf, off_t size, u8*& data, IoCallback cb, uintptr_t cbData) -{ - IoSplitter splitter(ofs, alignedBuf, size); - RETURN_ERR(splitter.Run(file, cb, cbData)); - data = alignedBuf + ofs - splitter.AlignedOfs(); return INFO::OK; } -LibError io_WriteAligned(const PFile& file, off_t alignedOfs, const u8* alignedData, off_t size, IoCallback cb, uintptr_t cbData) +LibError WaitUntilComplete(aiocb& cb, size_t queueDepth) { - debug_assert(IsAligned_Offset(alignedOfs)); - debug_assert(IsAligned_Data(alignedData)); +#if CONFIG2_FILE_ENABLE_AIO + if(queueDepth > 1) + { + aiocb* const cbs = &cb; + timespec* const timeout = 0; // infinite +SUSPEND_AGAIN: + errno = 0; + const int ret = aio_suspend(&cbs, 1, timeout); + if(ret != 0) + { + if(errno == EINTR) // interrupted by signal + goto SUSPEND_AGAIN; + return LibError_from_errno(); + } - IoSplitter splitter(alignedOfs, const_cast(alignedData), size); - return splitter.Run(file, cb, cbData); + const int err = aio_error(&cb); + debug_assert(err != EINPROGRESS); // else aio_return is undefined + ssize_t bytesTransferred = aio_return(&cb); + if(bytesTransferred == -1) // transfer failed + { + errno = err; + return LibError_from_errno(); + } + cb.aio_nbytes = (size_t)bytesTransferred; + } +#endif + + return INFO::OK; } - -LibError io_ReadAligned(const PFile& file, off_t alignedOfs, u8* alignedBuf, off_t size, IoCallback cb, uintptr_t cbData) -{ - debug_assert(IsAligned_Offset(alignedOfs)); - debug_assert(IsAligned_Data(alignedBuf)); - - IoSplitter splitter(alignedOfs, alignedBuf, size); - return splitter.Run(file, cb, cbData); -} +} // namespace io diff --git a/source/lib/file/io/io.h b/source/lib/file/io/io.h index ef563a4042..0350e1908d 100644 --- a/source/lib/file/io/io.h +++ b/source/lib/file/io/io.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2010 Wildfire Games +/* Copyright (c) 2011 Wildfire Games * * Permission is hereby granted, free of charge, to any person obtaining * a copy of this software and associated documentation files (the @@ -21,36 +21,314 @@ */ /* - * IO manager: splits requests into asynchronously issued blocks, - * thus simplifying caching and enabling periodic callbacks. + * provide asynchronous and synchronous I/O with hooks to allow + * overlapped processing or progress reporting. */ #ifndef INCLUDED_IO #define INCLUDED_IO +#include "lib/config2.h" +#include "lib/alignment.h" +#include "lib/bits.h" #include "lib/file/file.h" +#include "lib/sysdep/filesystem.h" // wtruncate -// memory will be allocated from the heap, not the (limited) file cache. -// this makes sense for write buffers that are never used again, -// because we avoid having to displace some other cached items. -LIB_API shared_ptr io_Allocate(size_t size, off_t ofs = 0); +#include "lib/allocators/unique_range.h" -/** - * called after a block IO has completed. - * - * @return INFO::CB_CONTINUE to continue; any other value will cause the - * IO splitter to abort immediately and return that. - * - * this is useful for user progress notification or processing data while - * waiting for the next I/O to complete (without the complexity of threads). - **/ -typedef LibError (*IoCallback)(uintptr_t cbData, const u8* block, size_t blockSize); +namespace ERR +{ + const LibError IO = -110301; +} -LIB_API LibError io_Scan(const PFile& file, off_t ofs, off_t size, IoCallback cb, uintptr_t cbData); +namespace io { -LIB_API LibError io_Read(const PFile& file, off_t ofs, u8* alignedBuf, off_t size, u8*& data, IoCallback cb = 0, uintptr_t cbData = 0); +// @return memory suitable for use as an I/O buffer (address is a +// multiple of alignment, size is rounded up to a multiple of alignment) +// +// use this instead of the file cache for write buffers that are +// never reused (avoids displacing other items). +LIB_API UniqueRange Allocate(size_t size, size_t alignment = maxSectorSize); -LIB_API LibError io_WriteAligned(const PFile& file, off_t alignedOfs, const u8* alignedData, off_t size, IoCallback cb = 0, uintptr_t cbData = 0); -LIB_API LibError io_ReadAligned(const PFile& file, off_t alignedOfs, u8* alignedBuf, off_t size, IoCallback cb = 0, uintptr_t cbData = 0); + +#pragma pack(push, 1) + +// required information for any I/O (this is basically the same as aiocb, +// but also applies to synchronous I/O and has shorter/nicer names.) +struct Operation +{ + // @param buf can be 0, in which case temporary block buffers are allocated. + // otherwise, it must be padded to the I/O alignment, e.g. via io::Allocate. + Operation(const File& file, void* buf, off_t size, off_t offset = 0) + : fd(file.Descriptor()), opcode(file.Opcode()) + , offset(offset), size(size), buf((void*)buf) + { + } + + void Validate() const + { + debug_assert(fd >= 0); + debug_assert(opcode == LIO_READ || opcode == LIO_WRITE); + + debug_assert(offset >= 0); + debug_assert(size >= 0); + // buf can legitimately be 0 (see above) + } + + int fd; + int opcode; + + off_t offset; + off_t size; + void* buf; +}; + + +// optional information how an Operation is to be carried out +struct Parameters +{ + // default to single blocking I/Os + Parameters() + : alignment(1) // no alignment requirements + // use one huge "block" truncated to the requested size. + // (this value is a power of two as required by Validate and + // avoids overflowing off_t in DivideRoundUp) + , blockSize((SIZE_MAX/2)+1) + , queueDepth(1) // disable aio + { + } + + // parameters for asynchronous I/O that maximize throughput on current drives + struct OverlappedTag {}; + Parameters(OverlappedTag) + : alignment(maxSectorSize), blockSize(128*KiB), queueDepth(32) + { + } + + Parameters(size_t blockSize, size_t queueDepth, off_t alignment = maxSectorSize) + : alignment(alignment), blockSize(blockSize), queueDepth(queueDepth) + { + } + + void Validate(const Operation& op) const + { + debug_assert(is_pow2(alignment)); + debug_assert(alignment > 0); + + debug_assert(is_pow2(blockSize)); + debug_assert(pageSize <= blockSize); // no upper limit needed + + debug_assert(1 <= queueDepth && queueDepth <= maxQueueDepth); + + debug_assert(IsAligned(op.offset, alignment)); + // op.size doesn't need to be aligned + debug_assert(IsAligned(op.buf, alignment)); + } + + // (ATTO only allows 10, which improves upon 8) + static const size_t maxQueueDepth = 32; + + off_t alignment; + + size_t blockSize; + + size_t queueDepth; +}; + +#define IO_OVERLAPPED io::Parameters(io::Parameters::OverlappedTag()) + + +struct DefaultCompletedHook +{ + /** + * called after a block I/O has completed. + * + * @return INFO::CB_CONTINUE to proceed; any other value will + * be immediately returned by Run. + * + * allows progress notification and processing data while waiting for + * previous I/Os to complete. + **/ + LibError operator()(const u8* UNUSED(block), size_t UNUSED(blockSize)) const + { + return INFO::CB_CONTINUE; + } +}; + + +struct DefaultIssueHook +{ + /** + * called before a block I/O is issued. + * + * @return INFO::CB_CONTINUE to proceed; any other value will + * be immediately returned by Run. + * + * allows generating the data to write while waiting for + * previous I/Os to complete. + **/ + LibError operator()(aiocb& UNUSED(cb)) const + { + return INFO::CB_CONTINUE; + } +}; + + +// ring buffer of partially initialized aiocb that can be passed +// directly to aio_write etc. after setting offset and buffer. +class ControlBlockRingBuffer +{ +public: + ControlBlockRingBuffer(const Operation& op, const Parameters& p) + : controlBlocks() // zero-initialize + { + // (default p.blockSize is "infinity", so clamp to the total size) + const size_t blockSize = (size_t)std::min((off_t)p.blockSize, op.size); + + const bool temporaryBuffersRequested = (op.buf == 0); + if(temporaryBuffersRequested) + buffers = RVALUE(io::Allocate(blockSize * p.queueDepth, p.alignment)); + + for(size_t i = 0; i < ARRAY_SIZE(controlBlocks); i++) + { + aiocb& cb = operator[](i); + cb.aio_fildes = op.fd; + cb.aio_nbytes = blockSize; + cb.aio_lio_opcode = op.opcode; + if(temporaryBuffersRequested) + cb.aio_buf = (volatile void*)(uintptr_t(buffers.get()) + i * blockSize); + } + } + + INLINE aiocb& operator[](size_t counter) + { + return controlBlocks[counter % ARRAY_SIZE(controlBlocks)]; + } + +private: + UniqueRange buffers; + aiocb controlBlocks[Parameters::maxQueueDepth]; +}; + +#pragma pack(pop) + + +LIB_API LibError Issue(aiocb& cb, size_t queueDepth); +LIB_API LibError WaitUntilComplete(aiocb& cb, size_t queueDepth); + + +//----------------------------------------------------------------------------- +// Run + +// (hooks must be passed by const reference to allow passing rvalues. +// functors with non-const member data can mark them as mutable.) +template +static inline LibError Run(const Operation& op, const Parameters& p = Parameters(), const CompletedHook& completedHook = CompletedHook(), const IssueHook& issueHook = IssueHook()) +{ + op.Validate(); + p.Validate(op); + + ControlBlockRingBuffer controlBlockRingBuffer(op, p); + + const off_t numBlocks = DivideRoundUp(op.size, (off_t)p.blockSize); + for(off_t blocksIssued = 0, blocksCompleted = 0; blocksCompleted < numBlocks; blocksCompleted++) + { + for(; blocksIssued != numBlocks && blocksIssued < blocksCompleted + (off_t)p.queueDepth; blocksIssued++) + { + aiocb& cb = controlBlockRingBuffer[blocksIssued]; + cb.aio_offset = op.offset + blocksIssued * p.blockSize; + if(op.buf) + cb.aio_buf = (volatile void*)(uintptr_t(op.buf) + blocksIssued * p.blockSize); + if(blocksIssued == numBlocks-1) + cb.aio_nbytes = round_up(size_t(op.size - blocksIssued * p.blockSize), size_t(p.alignment)); + + RETURN_IF_NOT_CONTINUE(issueHook(cb)); + + RETURN_ERR(Issue(cb, p.queueDepth)); + } + + aiocb& cb = controlBlockRingBuffer[blocksCompleted]; + RETURN_ERR(WaitUntilComplete(cb, p.queueDepth)); + + RETURN_IF_NOT_CONTINUE(completedHook((u8*)cb.aio_buf, cb.aio_nbytes)); + } + + return INFO::OK; +} + +// (overloads allow omitting parameters without requiring a template argument list) +template +static inline LibError Run(const Operation& op, const Parameters& p = Parameters(), const CompletedHook& completedHook = CompletedHook()) +{ + return Run(op, p, completedHook, DefaultIssueHook()); +} + +static inline LibError Run(const Operation& op, const Parameters& p = Parameters()) +{ + return Run(op, p, DefaultCompletedHook(), DefaultIssueHook()); +} + + +//----------------------------------------------------------------------------- +// Store + +// efficient writing requires preallocation, and the resulting file is +// padded to the sector size and needs to be truncated afterwards. +// this function takes care of both. +template +static inline LibError Store(const OsPath& pathname, const void* data, size_t size, const Parameters& p = Parameters(), const CompletedHook& completedHook = CompletedHook(), const IssueHook& issueHook = IssueHook()) +{ + File file(pathname, LIO_WRITE); + io::Operation op(file, (void*)data, size); + +#if OS_WIN && CONFIG2_FILE_ENABLE_AIO + (void)waio_Preallocate(op.fd, (off_t)size, p.alignment); +#endif + + RETURN_ERR(io::Run(op, p, completedHook, issueHook)); + + file.Close(); // (required by wtruncate) + + RETURN_ERR(wtruncate(pathname, size)); + + return INFO::OK; +} + +template +static inline LibError Store(const OsPath& pathname, const void* data, size_t size, const Parameters& p = Parameters(), const CompletedHook& completedHook = CompletedHook()) +{ + return Store(pathname, data, size, p, completedHook, DefaultIssueHook()); +} + +static inline LibError Store(const OsPath& pathname, const void* data, size_t size, const Parameters& p = Parameters()) +{ + return Store(pathname, data, size, p, DefaultCompletedHook(), DefaultIssueHook()); +} + + +//----------------------------------------------------------------------------- +// Load + +// convenience function provided for symmetry with Store +template +static inline LibError Load(const OsPath& pathname, void* buf, size_t size, const Parameters& p = Parameters(), const CompletedHook& completedHook = CompletedHook(), const IssueHook& issueHook = IssueHook()) +{ + File file(pathname, LIO_READ); + io::Operation op(file, buf, size); + return io::Run(op, p, completedHook, issueHook); +} + +template +static inline LibError Load(const OsPath& pathname, void* buf, size_t size, const Parameters& p = Parameters(), const CompletedHook& completedHook = CompletedHook()) +{ + return Load(pathname, buf, size, p, completedHook, DefaultIssueHook()); +} + +static inline LibError Load(const OsPath& pathname, void* buf, size_t size, const Parameters& p = Parameters()) +{ + return Load(pathname, buf, size, p, DefaultCompletedHook(), DefaultIssueHook()); +} + +} // namespace io #endif // #ifndef INCLUDED_IO diff --git a/source/lib/file/io/io_align.cpp b/source/lib/file/io/io_align.cpp deleted file mode 100644 index ecf418bd3b..0000000000 --- a/source/lib/file/io/io_align.cpp +++ /dev/null @@ -1,24 +0,0 @@ -/* Copyright (c) 2010 Wildfire Games - * - * Permission is hereby granted, free of charge, to any person obtaining - * a copy of this software and associated documentation files (the - * "Software"), to deal in the Software without restriction, including - * without limitation the rights to use, copy, modify, merge, publish, - * distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to - * the following conditions: - * - * The above copyright notice and this permission notice shall be included - * in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY - * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, - * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE - * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -#include "precompiled.h" -#include "lib/file/io/io_align.h" diff --git a/source/lib/file/io/io_align.h b/source/lib/file/io/io_align.h deleted file mode 100644 index 04b4d19a63..0000000000 --- a/source/lib/file/io/io_align.h +++ /dev/null @@ -1,71 +0,0 @@ -/* Copyright (c) 2010 Wildfire Games - * - * Permission is hereby granted, free of charge, to any person obtaining - * a copy of this software and associated documentation files (the - * "Software"), to deal in the Software without restriction, including - * without limitation the rights to use, copy, modify, merge, publish, - * distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to - * the following conditions: - * - * The above copyright notice and this permission notice shall be included - * in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY - * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, - * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE - * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -#ifndef INCLUDED_IO_ALIGN -#define INCLUDED_IO_ALIGN - -#include "lib/bits.h" // IsAligned, round_up - -/** - * block := power-of-two sized chunk of a file. - * all transfers are expanded to naturally aligned, whole blocks. - * (this makes caching parts of files feasible; it is also much faster - * for some aio implementations, e.g. wposix.) - * (blocks are also thereby page-aligned, which allows write-protecting - * file buffers without worrying about their boundaries.) - **/ -static const size_t BLOCK_SIZE = 1024*KiB; - -// note: *sizes* and *offsets* are aligned to blocks to allow zero-copy block cache. -// that the *buffer* need only be sector-aligned (we assume 4kb for simplicity) -// (this is a requirement of the underlying Windows OS) -static const size_t SECTOR_SIZE = 4*KiB; - - -template -inline bool IsAligned_Data(T* address) -{ - return IsAligned((uintptr_t)address, SECTOR_SIZE); -} - -inline bool IsAligned_Offset(off_t ofs) -{ - return IsAligned(ofs, BLOCK_SIZE); -} - - -inline off_t AlignedOffset(off_t ofs) -{ - return (off_t)round_down(size_t(ofs), BLOCK_SIZE); -} - -inline off_t AlignedSize(off_t size) -{ - return (off_t)round_up(size_t(size), BLOCK_SIZE); -} - -inline off_t PaddedSize(off_t size, off_t ofs) -{ - return (off_t)round_up(size_t(size + ofs - AlignedOffset(ofs)), BLOCK_SIZE); -} - -#endif // #ifndef INCLUDED_IO_ALIGN diff --git a/source/lib/file/io/write_buffer.cpp b/source/lib/file/io/write_buffer.cpp index 34e2ca6b16..c3e9a86f79 100644 --- a/source/lib/file/io/write_buffer.cpp +++ b/source/lib/file/io/write_buffer.cpp @@ -25,12 +25,15 @@ #include "lib/bits.h" // IsAligned #include "lib/sysdep/cpu.h" +#include "lib/allocators/shared_ptr.h" #include "lib/file/io/io.h" -#include "lib/file/io/io_align.h" + + +static const size_t BLOCK_SIZE = 512*KiB; WriteBuffer::WriteBuffer() - : m_capacity(4096), m_data(io_Allocate(m_capacity)), m_size(0) + : m_capacity(pageSize), m_data((u8*)rtl_AllocateAligned(m_capacity, maxSectorSize), AlignedDeleter()), m_size(0) { } @@ -40,7 +43,8 @@ void WriteBuffer::Append(const void* data, size_t size) if(m_size + size > m_capacity) { m_capacity = round_up_to_pow2(m_size + size); - shared_ptr newData = io_Allocate(m_capacity); + shared_ptr newData; + AllocateAligned(newData, m_capacity, maxSectorSize); memcpy(newData.get(), m_data.get(), m_size); m_data = newData; } @@ -62,12 +66,15 @@ void WriteBuffer::Overwrite(const void* data, size_t size, size_t offset) //----------------------------------------------------------------------------- UnalignedWriter::UnalignedWriter(const PFile& file, off_t ofs) - : m_file(file), m_alignedBuf(io_Allocate(BLOCK_SIZE)) + : m_file(file), m_alignedBuf((u8*)rtl_AllocateAligned(BLOCK_SIZE, maxSectorSize), AlignedDeleter()) { - m_alignedOfs = AlignedOffset(ofs); + m_alignedOfs = round_down(ofs, (off_t)BLOCK_SIZE); const size_t misalignment = (size_t)(ofs - m_alignedOfs); if(misalignment) - io_ReadAligned(m_file, m_alignedOfs, m_alignedBuf.get(), BLOCK_SIZE); + { + io::Operation op(*m_file.get(), m_alignedBuf.get(), BLOCK_SIZE, m_alignedOfs); + THROW_ERR(io::Run(op)); + } m_bytesUsed = misalignment; } @@ -84,9 +91,10 @@ LibError UnalignedWriter::Append(const u8* data, size_t size) const { // optimization: write directly from the input buffer, if possible const size_t alignedSize = (size / BLOCK_SIZE) * BLOCK_SIZE; - if(m_bytesUsed == 0 && IsAligned(data, SECTOR_SIZE) && alignedSize != 0) + if(m_bytesUsed == 0 && IsAligned(data, maxSectorSize) && alignedSize != 0) { - RETURN_ERR(io_WriteAligned(m_file, m_alignedOfs, data, alignedSize)); + io::Operation op(*m_file.get(), (void*)data, alignedSize, m_alignedOfs); + RETURN_ERR(io::Run(op)); m_alignedOfs += (off_t)alignedSize; data += alignedSize; size -= alignedSize; @@ -118,7 +126,8 @@ void UnalignedWriter::Flush() const LibError UnalignedWriter::WriteBlock() const { - RETURN_ERR(io_WriteAligned(m_file, m_alignedOfs, m_alignedBuf.get(), BLOCK_SIZE)); + io::Operation op(*m_file.get(), m_alignedBuf.get(), BLOCK_SIZE, m_alignedOfs); + RETURN_ERR(io::Run(op)); m_alignedOfs += BLOCK_SIZE; m_bytesUsed = 0; return INFO::OK; diff --git a/source/lib/file/vfs/file_cache.cpp b/source/lib/file/vfs/file_cache.cpp index 48353a9fc2..8d0ee4df15 100644 --- a/source/lib/file/vfs/file_cache.cpp +++ b/source/lib/file/vfs/file_cache.cpp @@ -30,7 +30,6 @@ #include "lib/external_libraries/suppress_boost_warnings.h" #include "lib/file/common/file_stats.h" -#include "lib/file/io/io_align.h" // BLOCK_SIZE #include "lib/cache_adt.h" // Cache #include "lib/bits.h" // round_up #include "lib/allocators/allocators.h" @@ -98,7 +97,7 @@ public: shared_ptr Allocate(size_t size, const PAllocator& pthis) { - const size_t alignedSize = round_up(size, BLOCK_SIZE); + const size_t alignedSize = Align(size); u8* mem = (u8*)m_allocator.Allocate(alignedSize); if(!mem) @@ -114,7 +113,7 @@ public: void Deallocate(u8* mem, size_t size) { - const size_t alignedSize = round_up(size, BLOCK_SIZE); + const size_t alignedSize = Align(size); // (re)allow writes in case the buffer was made read-only. it would // be nice to unmap the buffer, but this is not possible because diff --git a/source/lib/file/vfs/vfs.cpp b/source/lib/file/vfs/vfs.cpp index 711409b24d..0a971565af 100644 --- a/source/lib/file/vfs/vfs.cpp +++ b/source/lib/file/vfs/vfs.cpp @@ -150,7 +150,7 @@ public: fileContents = DummySharedPtr((u8*)0); else if(size > m_cacheSize) { - fileContents = io_Allocate(size); + RETURN_ERR(AllocateAligned(fileContents, size, maxSectorSize)); RETURN_ERR(file->Loader()->Load(file->Name(), fileContents, file->Size())); } else diff --git a/source/lib/lib_errors.h b/source/lib/lib_errors.h index ee717fdc11..88055809b1 100644 --- a/source/lib/lib_errors.h +++ b/source/lib/lib_errors.h @@ -316,6 +316,15 @@ STMT(\ }\ ) + +// if expression evaluates to a negative error code, return 0. +#define RETURN_IF_NOT_CONTINUE(expression)\ +STMT(\ + i64 err64__ = (i64)(expression);\ + if(err64__ != INFO::CB_CONTINUE)\ + return err64__;\ +) + // return an error and warn about it (replaces debug_warn+return) #define WARN_RETURN(err)\ STMT(\ @@ -375,8 +384,8 @@ STMT(\ STMT(\ if(!(ok))\ {\ - debug_warn(L"FYI: WARN_RETURN_IF_FALSE reports that a function failed."\ - L"feel free to ignore or suppress this warning.");\ + debug_warn(L"FYI: WARN_RETURN_IF_FALSE reports that a function failed. "\ + L"Feel free to ignore or suppress this warning.");\ return ERR::FAIL;\ }\ ) @@ -392,8 +401,8 @@ STMT(\ #define WARN_IF_FALSE(ok)\ STMT(\ if(!(ok))\ - debug_warn(L"FYI: WARN_IF_FALSE reports that a function failed."\ - L"feel free to ignore or suppress this warning.");\ + debug_warn(L"FYI: WARN_IF_FALSE reports that a function failed. "\ + L"Feel free to ignore or suppress this warning.");\ ) diff --git a/source/lib/res/sound/ogg.cpp b/source/lib/res/sound/ogg.cpp index 8806ce2b27..8f519654ef 100644 --- a/source/lib/res/sound/ogg.cpp +++ b/source/lib/res/sound/ogg.cpp @@ -5,7 +5,7 @@ #include "lib/external_libraries/vorbis.h" #include "lib/byte_order.h" -#include "lib/file/file.h" +#include "lib/file/io/io.h" #include "lib/file/file_system_util.h" @@ -64,7 +64,8 @@ public: const off_t sizeRemaining = adapter->size - adapter->offset; const size_t sizeToRead = (size_t)std::min(sizeRequested, sizeRemaining); - if(adapter->file->Read(adapter->offset, (u8*)bufferToFill, sizeToRead) == INFO::OK) + io::Operation op(*adapter->file.get(), bufferToFill, sizeToRead, adapter->offset); + if(io::Run(op) == INFO::OK) { adapter->offset += sizeToRead; return sizeToRead; diff --git a/source/lib/sysdep/compiler.h b/source/lib/sysdep/compiler.h index a021c28b72..9d2a946bba 100644 --- a/source/lib/sysdep/compiler.h +++ b/source/lib/sysdep/compiler.h @@ -27,10 +27,6 @@ #ifndef INCLUDED_COMPILER #define INCLUDED_COMPILER -#include "lib/sysdep/arch.h" // ARCH_AMD64 -#include "lib/config.h" // CONFIG_OMIT_FP - - // detect compiler and its version (0 if not present, otherwise // major*100 + minor). note that more than one *_VERSION may be // non-zero due to interoperability (e.g. ICC with MSC). @@ -60,18 +56,6 @@ #endif -// pass "omit frame pointer" setting on to the compiler -#if MSC_VERSION && !ARCH_AMD64 -# if CONFIG_OMIT_FP -# pragma optimize("y", on) -# else -# pragma optimize("y", off) -# endif -#elif GCC_VERSION -// TODO -#endif - - // are PreCompiled Headers supported? #if MSC_VERSION # define HAVE_PCH 1 @@ -82,19 +66,6 @@ #endif -// try to define _W64, if not already done -// (this is useful for catching pointer size bugs) -#ifndef _W64 -# if MSC_VERSION -# define _W64 __w64 -# elif GCC_VERSION -# define _W64 __attribute__((mode (__pointer__))) -# else -# define _W64 -# endif -#endif - - // check if compiling in pure C mode (not C++) with support for C99. // (this is more convenient than testing __STDC_VERSION__ directly) // @@ -118,7 +89,7 @@ #endif -// (at least rudimentary) support for C++0x +// do we have (at least rudimentary) support for C++0x? #ifndef HAVE_CPP0X # if defined(__GXX_EXPERIMENTAL_CPP0X__) || MSC_VERSION >= 1600 || ICC_VERSION >= 1200 # define HAVE_CPP0X 1 @@ -127,104 +98,4 @@ # endif #endif - -// C99-like restrict (non-standard in C++, but widely supported in various forms). -// -// May be used on pointers. May also be used on member functions to indicate -// that 'this' is unaliased (e.g. "void C::m() RESTRICT { ... }"). -// Must not be used on references - GCC supports that but VC doesn't. -// -// We call this "RESTRICT" to avoid conflicts with VC's __declspec(restrict), -// and because it's not really the same as C99's restrict. -// -// To be safe and satisfy the compilers' stated requirements: an object accessed -// by a restricted pointer must not be accessed by any other pointer within the -// lifetime of the restricted pointer, if the object is modified. -// To maximise the chance of optimisation, any pointers that could potentially -// alias with the restricted one should be marked as restricted too. -// -// It would probably be a good idea to write test cases for any code that uses -// this in an even very slightly unclear way, in case it causes obscure problems -// in a rare compiler due to differing semantics. -// -// .. GCC -#if GCC_VERSION -# define RESTRICT __restrict__ -// .. VC8 provides __restrict -#elif MSC_VERSION >= 1400 -# define RESTRICT __restrict -// .. ICC supports the keyword 'restrict' when run with the /Qrestrict option, -// but it always also supports __restrict__ or __restrict to be compatible -// with GCC/MSVC, so we'll use the underscored version. One of {GCC,MSC}_VERSION -// should have been defined in addition to ICC_VERSION, so we should be using -// one of the above cases (unless it's an old VS7.1-emulating ICC). -#elif ICC_VERSION -# error ICC_VERSION defined without either GCC_VERSION or an adequate MSC_VERSION -// .. unsupported; remove it from code -#else -# define RESTRICT -#endif - - -// C99-style __func__ -// .. newer GCC already have it -#if GCC_VERSION >= 300 - // nothing need be done -// .. old GCC and MSVC have __FUNCTION__ -#elif GCC_VERSION >= 200 || MSC_VERSION -# define __func__ __FUNCTION__ -// .. unsupported -#else -# define __func__ "(unknown)" -#endif - - -// tell the compiler that the code at/following this macro invocation is -// unreachable. this can improve optimization and avoid warnings. -// -// this macro should not generate any fallback code; it is merely the -// compiler-specific backend for lib.h's UNREACHABLE. -// #define it to nothing if the compiler doesn't support such a hint. -#define HAVE_ASSUME_UNREACHABLE 1 -#if MSC_VERSION && !ICC_VERSION // (ICC ignores this) -# define ASSUME_UNREACHABLE __assume(0) -#elif GCC_VERSION >= 450 -# define ASSUME_UNREACHABLE __builtin_unreachable() -#else -# define ASSUME_UNREACHABLE -# undef HAVE_ASSUME_UNREACHABLE -# define HAVE_ASSUME_UNREACHABLE 0 -#endif - - -// extern "C", but does the right thing in pure-C mode -#if defined(__cplusplus) -# define EXTERN_C extern "C" -#else -# define EXTERN_C extern -#endif - -#if MSC_VERSION -# define INLINE __forceinline -#else -# define INLINE inline -#endif - -#if MSC_VERSION -# define CALL_CONV __cdecl -#else -# define CALL_CONV -#endif - -#if MSC_VERSION && !ARCH_AMD64 -# define DECORATED_NAME(name) _##name -#else -# define DECORATED_NAME(name) name -#endif - -// workaround for preprocessor limitation: macro args aren't expanded -// before being pasted. -#define STRINGIZE2(id) # id -#define STRINGIZE(id) STRINGIZE2(id) - #endif // #ifndef INCLUDED_COMPILER diff --git a/source/lib/sysdep/filesystem.h b/source/lib/sysdep/filesystem.h index 2d627ed4a9..2935b4b817 100644 --- a/source/lib/sysdep/filesystem.h +++ b/source/lib/sysdep/filesystem.h @@ -86,6 +86,11 @@ extern int wclose(int fd); // unistd.h // +// waio requires offsets and sizes to be multiples of the sector size. +// to allow arbitrarily sized files, we truncate them after I/O. +// however, ftruncate cannot be used since it is also subject to the +// sector-alignment requirement. instead, the file must be closed and +// this function called. LIB_API int wtruncate(const OsPath& pathname, off_t length); LIB_API int wunlink(const OsPath& pathname); diff --git a/source/lib/sysdep/os/win/wdir_watch.cpp b/source/lib/sysdep/os/win/wdir_watch.cpp index b162ededef..387b1b21d3 100644 --- a/source/lib/sysdep/os/win/wdir_watch.cpp +++ b/source/lib/sysdep/os/win/wdir_watch.cpp @@ -32,6 +32,7 @@ #include "lib/sysdep/os/win/win.h" #include "lib/sysdep/os/win/winit.h" #include "lib/sysdep/os/win/wutil.h" +#include "lib/sysdep/os/win/wiocp.h" WINIT_REGISTER_MAIN_INIT(wdir_watch_Init); @@ -290,84 +291,22 @@ struct DirWatch }; -//----------------------------------------------------------------------------- -// CompletionPort - -// this appears to be the best solution for IO notification. -// there are three alternatives: -// - multiple threads with blocking I/O. this is rather inefficient when -// many directories (e.g. mods) are being watched. -// - normal overlapped I/O: build a contiguous array of the hEvents -// in all OVERLAPPED structures, and WaitForMultipleObjects. -// it would be cumbersome to update this array when adding/removing watches. -// - callback notification: a notification function is called when the thread -// that initiated the I/O (ReadDirectoryChangesW) enters an alertable -// wait state. it is desirable for notifications to arrive at a single -// known point - see dir_watch_Poll. unfortunately there doesn't appear to -// be a reliable and non-blocking means of entering AWS - SleepEx(1) may -// wait for 10..15 ms if the system timer granularity is low. even worse, -// it was noted in a previous project that APCs are sometimes delivered from -// within APIs without having used SleepEx (it seems threads sometimes enter -// a semi-AWS when calling the kernel). -class CompletionPort -{ -public: - CompletionPort() - { - m_hIOCP = 0; // CreateIoCompletionPort requires 0, not INVALID_HANDLE_VALUE - } - - ~CompletionPort() - { - CloseHandle(m_hIOCP); - m_hIOCP = INVALID_HANDLE_VALUE; - } - - void Attach(HANDLE hFile, uintptr_t key) - { - WinScopedPreserveLastError s; // CreateIoCompletionPort - - // (when called for the first time, ends up creating m_hIOCP) - m_hIOCP = CreateIoCompletionPort(hFile, m_hIOCP, (ULONG_PTR)key, 0); - debug_assert(wutil_IsValidHandle(m_hIOCP)); - } - - LibError Poll(size_t& bytesTransferred, uintptr_t& key, OVERLAPPED*& ovl) - { - if(m_hIOCP == 0) - return ERR::INVALID_HANDLE; // NOWARN (happens if called before the first Attach) - for(;;) // don't return abort notifications to caller - { - DWORD dwBytesTransferred = 0; - ULONG_PTR ulKey = 0; - ovl = 0; - const DWORD timeout = 0; - const BOOL gotPacket = GetQueuedCompletionStatus(m_hIOCP, &dwBytesTransferred, &ulKey, &ovl, timeout); - bytesTransferred = size_t(dwBytesTransferred); - key = uintptr_t(ulKey); - if(gotPacket) - return INFO::OK; - - if(GetLastError() == WAIT_TIMEOUT) - return ERR::AGAIN; // NOWARN (nothing pending) - else if(GetLastError() == ERROR_OPERATION_ABORTED) - continue; // watch was canceled - ignore - else - return LibError_from_GLE(); // actual error - } - } - -private: - HANDLE m_hIOCP; -}; - - //----------------------------------------------------------------------------- // DirWatchManager class DirWatchManager { public: + DirWatchManager() + : hIOCP(0) + { + } + + ~DirWatchManager() + { + CloseHandle(hIOCP); + } + LibError Add(const OsPath& path, PDirWatch& dirWatch) { debug_assert(path.IsDirectory()); @@ -386,7 +325,7 @@ public: } PDirWatchRequest request(new DirWatchRequest(path)); - m_completionPort.Attach(request->GetDirHandle(), (uintptr_t)request.get()); + AttachToCompletionPort(request->GetDirHandle(), hIOCP, (uintptr_t)request.get()); RETURN_ERR(request->Issue()); dirWatch.reset(new DirWatch(&m_sentinel, request)); return INFO::OK; @@ -394,8 +333,17 @@ public: LibError Poll(DirWatchNotifications& notifications) { - size_t bytesTransferred; uintptr_t key; OVERLAPPED* ovl; - RETURN_ERR(m_completionPort.Poll(bytesTransferred, key, ovl)); + DWORD bytesTransferred; ULONG_PTR key; OVERLAPPED* ovl; + for(;;) // skip notifications of canceled watches + { + const LibError ret = PollCompletionPort(hIOCP, 0, bytesTransferred, key, ovl); + if(ret == INFO::OK) + break; + if(GetLastError() == ERROR_OPERATION_ABORTED) + continue; // watch was canceled - ignore + return ret; + } + DirWatchRequest* request = (DirWatchRequest*)key; request->RetrieveNotifications(notifications); RETURN_ERR(request->Issue()); // re-issue @@ -404,7 +352,7 @@ public: private: IntrusiveLink m_sentinel; - CompletionPort m_completionPort; + HANDLE hIOCP; }; static DirWatchManager* s_dirWatchManager; diff --git a/source/lib/sysdep/os/win/wfirmware.cpp b/source/lib/sysdep/os/win/wfirmware.cpp index bca17a5996..95ff2bc777 100644 --- a/source/lib/sysdep/os/win/wfirmware.cpp +++ b/source/lib/sysdep/os/win/wfirmware.cpp @@ -17,7 +17,7 @@ TableIds GetTableIDs(Provider provider) debug_assert(tableIdsSize % sizeof(TableId) == 0); TableIds tableIDs(DivideRoundUp(tableIdsSize, sizeof(TableId)), 0); - const size_t bytesWritten = pEnumSystemFirmwareTables(provider, &tableIDs[0], tableIdsSize); + const size_t bytesWritten = pEnumSystemFirmwareTables(provider, &tableIDs[0], (DWORD)tableIdsSize); debug_assert(bytesWritten == tableIdsSize); return tableIDs; @@ -39,7 +39,7 @@ Table GetTable(Provider provider, TableId tableId) } Table table(tableSize, 0); - const size_t bytesWritten = pGetSystemFirmwareTable(provider, tableId, &table[0], tableSize); + const size_t bytesWritten = pGetSystemFirmwareTable(provider, tableId, &table[0], (DWORD)tableSize); debug_assert(bytesWritten == tableSize); return table; diff --git a/source/lib/sysdep/os/win/whrt/counter.cpp b/source/lib/sysdep/os/win/whrt/counter.cpp index f948625fe3..ab70d696b5 100644 --- a/source/lib/sysdep/os/win/whrt/counter.cpp +++ b/source/lib/sysdep/os/win/whrt/counter.cpp @@ -27,7 +27,7 @@ #include "precompiled.h" #include "lib/sysdep/os/win/whrt/counter.h" -#include "lib/bits.h" +#include "lib/alignment.h" #include "lib/sysdep/cpu.h" // cpu_CAS #include "lib/sysdep/os/win/whrt/tsc.h" @@ -98,7 +98,7 @@ ICounter* CreateCounter(size_t id) static const size_t memSize = 200; static u8 mem[memSize]; - u8* alignedMem = (u8*)round_up((uintptr_t)mem, (uintptr_t)16u); + u8* alignedMem = (u8*)Align<16>((uintptr_t)mem); const size_t bytesLeft = mem+memSize - alignedMem; ICounter* counter = ConstructCounterAt(id, alignedMem, bytesLeft); diff --git a/source/lib/sysdep/os/win/whrt/whrt.cpp b/source/lib/sysdep/os/win/whrt/whrt.cpp index d0b6604165..a3ca536c23 100644 --- a/source/lib/sysdep/os/win/whrt/whrt.cpp +++ b/source/lib/sysdep/os/win/whrt/whrt.cpp @@ -33,7 +33,7 @@ #include "lib/sysdep/os/win/win.h" #include "lib/sysdep/os/win/winit.h" #include "lib/sysdep/acpi.h" -#include "lib/adts.h" +//#include "lib/adts.h" #include "lib/bits.h" #include "lib/sysdep/os/win/whrt/counter.h" diff --git a/source/lib/sysdep/os/win/wiocp.cpp b/source/lib/sysdep/os/win/wiocp.cpp new file mode 100644 index 0000000000..55c438a73c --- /dev/null +++ b/source/lib/sysdep/os/win/wiocp.cpp @@ -0,0 +1,32 @@ +#include "precompiled.h" +#include "lib/sysdep/os/win/wiocp.h" + +#include "lib/file/file.h" // ERR::IO +#include "lib/sysdep/os/win/wutil.h" + + +void AttachToCompletionPort(HANDLE hFile, HANDLE& hIOCP, ULONG_PTR key, DWORD numConcurrentThreads) +{ + WinScopedPreserveLastError s; // CreateIoCompletionPort + + // (when called for the first time, ends up creating hIOCP) + hIOCP = CreateIoCompletionPort(hFile, hIOCP, key, numConcurrentThreads); + debug_assert(wutil_IsValidHandle(hIOCP)); +} + + +LibError PollCompletionPort(HANDLE hIOCP, DWORD timeout, DWORD& bytesTransferred, ULONG_PTR& key, OVERLAPPED*& ovl) +{ + if(hIOCP == 0) + return ERR::INVALID_HANDLE; // NOWARN (happens if called before the first Attach) + + bytesTransferred = 0; + key = 0; + ovl = 0; + if(GetQueuedCompletionStatus(hIOCP, &bytesTransferred, &key, &ovl, timeout)) + return INFO::OK; + if(GetLastError() == WAIT_TIMEOUT) + return ERR::AGAIN; // NOWARN (nothing pending) + else + return ERR::FAIL; // NOWARN (let caller decide what to do) +} diff --git a/source/lib/sysdep/os/win/wiocp.h b/source/lib/sysdep/os/win/wiocp.h new file mode 100644 index 0000000000..60486ca7e6 --- /dev/null +++ b/source/lib/sysdep/os/win/wiocp.h @@ -0,0 +1,50 @@ +/* Copyright (c) 2010 Wildfire Games + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY + * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, + * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +/* + * I/O completion port + */ + +#ifndef INCLUDED_WIOCP +#define INCLUDED_WIOCP + +#include "lib/sysdep/os/win/win.h" + +// this appears to be the best solution for IO notification. +// there are three alternatives: +// - multiple threads with blocking I/O. this is rather inefficient when +// many directories (e.g. mods) are being watched. +// - normal overlapped I/O: build a contiguous array of the hEvents +// in all OVERLAPPED structures, and WaitForMultipleObjects. +// it would be cumbersome to update this array when adding/removing watches. +// - callback notification: a notification function is called when the thread +// that initiated the I/O (ReadDirectoryChangesW) enters an alertable +// wait state. it is desirable for notifications to arrive at a single +// known point - see dir_watch_Poll. however, other APIs might also +// trigger APC delivery. + +// @param hIOCP 0 to create a new port +extern void AttachToCompletionPort(HANDLE hFile, HANDLE& hIOCP, ULONG_PTR key, DWORD numConcurrentThreads = 0); + +extern LibError PollCompletionPort(HANDLE hIOCP, DWORD timeout, DWORD& bytesTransferred, ULONG_PTR& key, OVERLAPPED*& ovl); + +#endif // #ifndef INCLUDED_WIOCP diff --git a/source/lib/sysdep/os/win/wnuma.cpp b/source/lib/sysdep/os/win/wnuma.cpp index 6950e5adea..1eedab4bbb 100644 --- a/source/lib/sysdep/os/win/wnuma.cpp +++ b/source/lib/sysdep/os/win/wnuma.cpp @@ -23,7 +23,7 @@ #include "precompiled.h" #include "lib/sysdep/numa.h" -#include "lib/bits.h" // round_up, PopulationCount +#include "lib/bits.h" // PopulationCount #include "lib/timer.h" #include "lib/module_init.h" #include "lib/allocators/allocators.h" // page_aligned_alloc diff --git a/source/lib/sysdep/os/win/wposix/waio.cpp b/source/lib/sysdep/os/win/wposix/waio.cpp index 6e8be9689c..bb0ea21b90 100644 --- a/source/lib/sysdep/os/win/wposix/waio.cpp +++ b/source/lib/sysdep/os/win/wposix/waio.cpp @@ -1,4 +1,4 @@ -/* Copyright (c) 2010 Wildfire Games +/* Copyright (c) 2011 Wildfire Games * * Permission is hereby granted, free of charge, to any person obtaining * a copy of this software and associated documentation files (the @@ -24,105 +24,312 @@ * emulate POSIX asynchronous I/O on Windows. */ +// NB: this module is significantly faster than Intel's aio library, +// which also returns ERROR_INVALID_PARAMETER from aio_error if the +// file is opened with FILE_FLAG_OVERLAPPED. (it looks like they are +// using threaded blocking IO) + #include "precompiled.h" #include "lib/sysdep/os/win/wposix/waio.h" -#include - -#include "lib/sysdep/os/win/wposix/crt_posix.h" // correct definitions of _open() etc. +#include "lib/alignment.h" // IsAligned +#include "lib/module_init.h" +#include "lib/sysdep/cpu.h" // cpu_AtomicAdd #include "lib/sysdep/filesystem.h" // O_NO_AIO_NP +#include "lib/sysdep/os/win/wutil.h" // wutil_SetPrivilege +#include "lib/sysdep/os/win/wiocp.h" +#include "lib/sysdep/os/win/winit.h" -#include "lib/bits.h" // IsAligned -#include "lib/sysdep/os/win/wposix/wposix_internal.h" -#include "lib/sysdep/os/win/wposix/wtime.h" // timespec - - -WINIT_REGISTER_MAIN_INIT(waio_Init); WINIT_REGISTER_MAIN_SHUTDOWN(waio_Shutdown); -// note: we assume sector sizes no larger than a page. -// (GetDiskFreeSpace allows querying the actual size, but we'd -// have to do so for all drives, and that may change depending on whether -// there is a DVD in the drive or not) -// sector size is relevant because Windows aio requires all IO -// buffers, offsets and lengths to be a multiple of it. this requirement -// is also carried over into the vfs / file.cpp interfaces for efficiency -// (avoids the need for copying to/from align buffers). -const uintptr_t sectorSize = 0x1000; +// (dynamic linking preserves compatibility with previous Windows versions) +static WUTIL_FUNC(pSetFileCompletionNotificationModes, BOOL, (HANDLE, UCHAR)); +static WUTIL_FUNC(pSetFileIoOverlappedRange, BOOL, (HANDLE, PUCHAR, ULONG)); +static WUTIL_FUNC(pSetFileValidData, BOOL, (HANDLE, LONGLONG)); + +// (there must be one global IOCP because aio_suspend might be called for +// requests from different files) +static HANDLE hIOCP; + //----------------------------------------------------------------------------- +// OvlAllocator -// note: the Windows lowio file descriptor limit is currrently 2048. - -/** - * association between POSIX file descriptor and Win32 HANDLE. - * NB: callers must ensure thread safety. - **/ -class HandleManager +// allocator for OVERLAPPED (enables a special optimization, see Associate) +struct OvlAllocator // POD { -public: - /** - * associate an aio handle with a file descriptor. - **/ - void Associate(int fd, HANDLE hFile) + // freelist entries for (padded) OVERLAPPED from our storage + struct Entry { - debug_assert(fd > 2); - debug_assert(GetFileSize(hFile, 0) != INVALID_FILE_SIZE); - std::pair ret = m_map.insert(std::make_pair(fd, hFile)); - debug_assert(ret.second); // fd better not already have been associated + SLIST_ENTRY entry; + OVERLAPPED ovl; + }; + + LibError Init() + { + // the allocation must be naturally aligned to ensure it doesn't + // overlap another page, which might prevent SetFileIoOverlappedRange + // from pinning the pages if one of them is PAGE_NOACCESS. + storage = _mm_malloc(storageSize, storageSize); + if(!storage) + WARN_RETURN(ERR::NO_MEM); + memset(storage, 0, storageSize); + + InitializeSListHead(&freelist); + + // storageSize provides more than enough OVERLAPPED, so we + // pad them to the cache line size to maybe avoid a few RFOs. + const size_t size = Align(sizeof(Entry)); + for(uintptr_t offset = 0; offset+size <= storageSize; offset += size) + { + Entry* entry = (Entry*)(uintptr_t(storage) + offset); + debug_assert(IsAligned(entry, MEMORY_ALLOCATION_ALIGNMENT)); + InterlockedPushEntrySList(&freelist, &entry->entry); + } + + extant = 0; + + return INFO::OK; } - void Dissociate(int fd) + void Shutdown() { - const size_t numRemoved = m_map.erase(fd); - debug_assert(numRemoved == 1); + debug_assert(extant == 0); + + InterlockedFlushSList(&freelist); + + _mm_free(storage); + storage = 0; } - bool IsAssociated(int fd) const + // irrevocably enable a special optimization for all I/Os requests + // concerning this file, ending when the file is closed. has no effect + // unless Vista+ and SeLockMemoryPrivilege are available. + void Associate(HANDLE hFile) { - return m_map.find(fd) != m_map.end(); + debug_assert(extant == 0); + + // pin the page in kernel address space, which means our thread + // won't have to be the one to service the I/O, thus avoiding an APC. + // ("thread agnostic I/O") + if(pSetFileIoOverlappedRange) + WARN_IF_FALSE(pSetFileIoOverlappedRange(hFile, (PUCHAR)storage, storageSize)); } - /** - * @return aio handle associated with file descriptor or - * INVALID_HANDLE_VALUE if there is none. - **/ - HANDLE Get(int fd) const + // @return OVERLAPPED initialized for I/O starting at offset, + // or 0 if all available structures have already been allocated. + OVERLAPPED* Allocate(off_t offset) { - Map::const_iterator it = m_map.find(fd); - if(it == m_map.end()) - return INVALID_HANDLE_VALUE; - return it->second; + Entry* entry = (Entry*)InterlockedPopEntrySList(&freelist); + if(!entry) + return 0; + + OVERLAPPED& ovl = entry->ovl; + ovl.Internal = 0; + ovl.InternalHigh = 0; + ovl.Offset = u64_lo(offset); + ovl.OffsetHigh = u64_hi(offset); + ovl.hEvent = 0; // (notification is via IOCP and/or polling) + + cpu_AtomicAdd(&extant, +1); + + return &ovl; } -private: - typedef std::map Map; - Map m_map; + void Deallocate(OVERLAPPED* ovl) + { + cpu_AtomicAdd(&extant, -1); + + const uintptr_t address = uintptr_t(ovl); + debug_assert(uintptr_t(storage) <= address && address < uintptr_t(storage)+storageSize); + InterlockedPushEntrySList(&freelist, (PSLIST_ENTRY)(address - offsetof(Entry, ovl))); + } + + // one 4 KiB page is enough for 64 OVERLAPPED per file (i.e. plenty). + static const size_t storageSize = pageSize; + + void* storage; + +#if MSC_VERSION +# pragma warning(push) +# pragma warning(disable:4324) // structure was padded due to __declspec(align()) +#endif + __declspec(align(MEMORY_ALLOCATION_ALIGNMENT)) SLIST_HEADER freelist; +#if MSC_VERSION +# pragma warning(pop) +#endif + + volatile intptr_t extant; }; -static HandleManager* handleManager; +//----------------------------------------------------------------------------- +// FileControlBlock -// do we want to open a second aio-capable handle? -static bool IsAioPossible(int fd, bool is_com_port, int oflag) +// (must correspond to static zero-initialization of fd) +static const intptr_t FD_AVAILABLE = 0; + +// information required to start asynchronous I/Os from a file +// (aiocb stores a pointer to the originating FCB) +struct FileControlBlock // POD { - // stdin/stdout/stderr - if(fd <= 2) - return false; + // search key, indicates the file descriptor with which this + // FCB was associated (or FD_AVAILABLE if none). + volatile intptr_t fd; - // COM port - we don't currently need aio access for those, and - // aio_reopen's CreateFileW would fail with "access denied". - if(is_com_port) - return false; + // second aio-enabled handle from waio_reopen + HANDLE hFile; - // caller is requesting we skip it (see open()) - if(oflag & O_NO_AIO_NP) - return false; + OvlAllocator ovl; - return true; + LibError Init() + { + fd = FD_AVAILABLE; + hFile = INVALID_HANDLE_VALUE; + return ovl.Init(); + } + + void Shutdown() + { + debug_assert(fd == FD_AVAILABLE); + debug_assert(hFile == INVALID_HANDLE_VALUE); + ovl.Shutdown(); + } +}; + + +// NB: the Windows lowio file descriptor limit is 2048, but +// our applications rarely open more than a few files at a time. +static FileControlBlock fileControlBlocks[16]; + + +static FileControlBlock* AssociateFileControlBlock(int fd, HANDLE hFile) +{ + for(size_t i = 0; i < ARRAY_SIZE(fileControlBlocks); i++) + { + FileControlBlock& fcb = fileControlBlocks[i]; + if(cpu_CAS(&fcb.fd, FD_AVAILABLE, fd)) // the FCB is now ours + { + fcb.hFile = hFile; + fcb.ovl.Associate(hFile); + + AttachToCompletionPort(hFile, hIOCP, (ULONG_PTR)&fcb); + + // minor optimization: avoid posting to IOCP in rare cases + // where the I/O completes synchronously + if(pSetFileCompletionNotificationModes) + { + // (for reasons as yet unknown, this fails when the file is + // opened for read-only access) + (void)pSetFileCompletionNotificationModes(fcb.hFile, FILE_SKIP_COMPLETION_PORT_ON_SUCCESS); + } + + return &fcb; + } + } + + return 0; } +static void DissociateFileControlBlock(FileControlBlock* fcb) +{ + fcb->hFile = INVALID_HANDLE_VALUE; + fcb->fd = FD_AVAILABLE; +} + + +static FileControlBlock* FindFileControlBlock(int fd) +{ + debug_assert(fd != FD_AVAILABLE); + + for(size_t i = 0; i < ARRAY_SIZE(fileControlBlocks); i++) + { + FileControlBlock& fcb = fileControlBlocks[i]; + if(fcb.fd == fd) + return &fcb; + } + + return 0; +} + + +//----------------------------------------------------------------------------- +// init/shutdown + +static ModuleInitState waio_initState; + +static LibError waio_Init() +{ + for(size_t i = 0; i < ARRAY_SIZE(fileControlBlocks); i++) + fileControlBlocks[i].Init(); + + WUTIL_IMPORT_KERNEL32(SetFileCompletionNotificationModes, pSetFileCompletionNotificationModes); + + // NB: using these functions when the privileges are not available would + // trigger warnings. since callers have to check the function pointers + // anyway, just refrain from setting them in such cases. + + if(wutil_SetPrivilege(L"SeLockMemoryPrivilege", true) == INFO::OK) + WUTIL_IMPORT_KERNEL32(SetFileIoOverlappedRange, pSetFileIoOverlappedRange); + + if(wutil_SetPrivilege(L"SeManageVolumePrivilege", true) == INFO::OK) + WUTIL_IMPORT_KERNEL32(SetFileValidData, pSetFileValidData); + + return INFO::OK; +} + + +static LibError waio_Shutdown() +{ + if(waio_initState == 0) // we were never initialized + return INFO::OK; + + for(size_t i = 0; i < ARRAY_SIZE(fileControlBlocks); i++) + fileControlBlocks[i].Shutdown(); + + WARN_IF_FALSE(CloseHandle(hIOCP)); + + return INFO::OK; +} + + +//----------------------------------------------------------------------------- +// OpenFile + +static DWORD DesiredAccess(int oflag) +{ + switch(oflag & (O_RDONLY|O_WRONLY|O_RDWR)) + { + case O_RDONLY: + // (WinXP x64 requires FILE_WRITE_ATTRIBUTES for SetFileCompletionNotificationModes) + return GENERIC_READ|FILE_WRITE_ATTRIBUTES; + case O_WRONLY: + return GENERIC_WRITE; + case O_RDWR: + return GENERIC_READ|GENERIC_WRITE; + default: + DEBUG_WARN_ERR(ERR::INVALID_PARAM); + return 0; + } +} + +static DWORD ShareMode(int oflag) +{ + switch(oflag & (O_RDONLY|O_WRONLY|O_RDWR)) + { + case O_RDONLY: + return FILE_SHARE_READ; + case O_WRONLY: + return FILE_SHARE_WRITE; + case O_RDWR: + return FILE_SHARE_READ|FILE_SHARE_WRITE; + default: + DEBUG_WARN_ERR(ERR::INVALID_PARAM); + return 0; + } +} + static DWORD CreationDisposition(int oflag) { if(oflag & O_CREAT) @@ -134,68 +341,68 @@ static DWORD CreationDisposition(int oflag) return OPEN_EXISTING; } - -// (re)open file in asynchronous mode and associate handle with fd. -// (this works because the files default to DENY_NONE sharing) -LibError waio_reopen(int fd, const OsPath& pathname, int oflag, ...) +static DWORD FlagsAndAttributes() { - WinScopedPreserveLastError s; // CreateFile + // - FILE_FLAG_SEQUENTIAL_SCAN is ignored when FILE_FLAG_NO_BUFFERING + // is set (c.f. "Windows via C/C++", p. 324) + // - FILE_FLAG_WRITE_THROUGH is ~5% slower (diskspd.cpp suggests it + // disables hardware caching; the overhead may also be due to the + // Windows cache manager) + const DWORD flags = FILE_FLAG_OVERLAPPED|FILE_FLAG_NO_BUFFERING; + const DWORD attributes = FILE_ATTRIBUTE_NORMAL; + return flags|attributes; +} - debug_assert(!(oflag & O_APPEND)); // not supported - if(!IsAioPossible(fd, false, oflag)) - return INFO::SKIPPED; +static LibError OpenFile(const OsPath& pathname, int oflag, HANDLE& hFile) +{ + WinScopedPreserveLastError s; - DWORD flags = FILE_FLAG_OVERLAPPED|FILE_FLAG_NO_BUFFERING|FILE_FLAG_SEQUENTIAL_SCAN; - - // decode file access mode - DWORD access, share; - switch(oflag & (O_RDONLY|O_WRONLY|O_RDWR)) - { - case O_RDONLY: - access = GENERIC_READ; - share = FILE_SHARE_READ; - break; - - case O_WRONLY: - access = GENERIC_WRITE; - share = FILE_SHARE_WRITE; - flags |= FILE_FLAG_WRITE_THROUGH; - break; - - case O_RDWR: - access = GENERIC_READ|GENERIC_WRITE; - share = FILE_SHARE_READ|FILE_SHARE_WRITE; - flags |= FILE_FLAG_WRITE_THROUGH; - break; - - default: - WARN_RETURN(ERR::INVALID_PARAM); - } - - // open file + const DWORD access = DesiredAccess(oflag); + const DWORD share = ShareMode(oflag); const DWORD create = CreationDisposition(oflag); - const HANDLE hFile = CreateFileW(OsString(pathname).c_str(), access, share, 0, create, FILE_ATTRIBUTE_NORMAL|flags, 0); + const DWORD flags = FlagsAndAttributes(); + hFile = CreateFileW(OsString(pathname).c_str(), access, share, 0, create, flags, 0); if(hFile == INVALID_HANDLE_VALUE) return LibError_from_GLE(); + return INFO::OK; +} + + +//----------------------------------------------------------------------------- +// Windows-only APIs + +LibError waio_reopen(int fd, const OsPath& pathname, int oflag, ...) +{ + debug_assert(fd > 2); + debug_assert(!(oflag & O_APPEND)); // not supported + + if(oflag & O_NO_AIO_NP) + return INFO::SKIPPED; + + RETURN_ERR(ModuleInit(&waio_initState, waio_Init)); + + HANDLE hFile; + RETURN_ERR(OpenFile(pathname, oflag, hFile)); + + if(!AssociateFileControlBlock(fd, hFile)) { - WinScopedLock lock(WAIO_CS); - handleManager->Associate(fd, hFile); + CloseHandle(hFile); + WARN_RETURN(ERR::LIMIT); } + return INFO::OK; } LibError waio_close(int fd) { - HANDLE hFile; - { - WinScopedLock lock(WAIO_CS); - if(!handleManager->IsAssociated(fd)) // wasn't opened for aio - return INFO::SKIPPED; - hFile = handleManager->Get(fd); - handleManager->Dissociate(fd); - } + FileControlBlock* fcb = FindFileControlBlock(fd); + if(!fcb) + WARN_RETURN(ERR::INVALID_HANDLE); + const HANDLE hFile = fcb->hFile; + + DissociateFileControlBlock(fcb); if(!CloseHandle(hFile)) WARN_RETURN(ERR::INVALID_HANDLE); @@ -204,266 +411,106 @@ LibError waio_close(int fd) } -// we don't want to #define read to _read, since that's a fairly common -// identifier. therefore, translate from MS CRT names via thunk functions. -// efficiency is less important, and the overhead could be optimized away. - -int read(int fd, void* buf, size_t nbytes) +LibError waio_Preallocate(int fd, off_t alignedSize, off_t alignment) { - return _read(fd, buf, (int)nbytes); -} + debug_assert(IsAligned(alignedSize, alignment)); -int write(int fd, void* buf, size_t nbytes) -{ - return _write(fd, buf, (int)nbytes); -} - -off_t lseek(int fd, off_t ofs, int whence) -{ - return _lseeki64(fd, ofs, whence); -} - - -//----------------------------------------------------------------------------- - -class aiocb::Impl -{ -public: - Impl() - { - m_hFile = INVALID_HANDLE_VALUE; - - // (hEvent is initialized below and the rest in Issue(), but clear out - // any subsequently added fields) - memset(&m_overlapped, 0, sizeof(m_overlapped)); - - const BOOL manualReset = TRUE; - const BOOL initialState = FALSE; - m_overlapped.hEvent = CreateEvent(0, manualReset, initialState, 0); - } - - ~Impl() - { - CloseHandle(m_overlapped.hEvent); - } - - LibError Issue(HANDLE hFile, off_t ofs, void* buf, size_t size, bool isWrite) - { - WinScopedPreserveLastError s; - - m_hFile = hFile; - - // note: Read-/WriteFile reset m_overlapped.hEvent, so we don't have to. - m_overlapped.Internal = m_overlapped.InternalHigh = 0; - m_overlapped.Offset = u64_lo(ofs); - m_overlapped.OffsetHigh = u64_hi(ofs); - - DWORD bytesTransferred; - BOOL ok; - if(isWrite) - ok = WriteFile(hFile, buf, u64_lo(size), &bytesTransferred, &m_overlapped); - else - ok = ReadFile(hFile, buf, u64_lo(size), &bytesTransferred, &m_overlapped); - if(!ok && GetLastError() == ERROR_IO_PENDING) // "pending" isn't an error - { - ok = TRUE; - SetLastError(0); - } - return LibError_from_win32(ok); - } - - bool HasCompleted() const - { - // NB: .Internal "was originally reserved for system use and its behavior may change". - // besides 0 and STATUS_PENDING, I have seen the address of a pointer to a buffer. - return HasOverlappedIoCompleted(&m_overlapped); - } - - // required for WaitForMultipleObjects - HANDLE Event() const - { - return m_overlapped.hEvent; - } - - LibError GetResult(size_t* pBytesTransferred) - { - DWORD bytesTransferred; - const BOOL wait = FALSE; // callers should wait until HasCompleted - if(!GetOverlappedResult(m_hFile, &m_overlapped, &bytesTransferred, wait)) - { - *pBytesTransferred = 0; - return LibError_from_GLE(); - } - else - { - *pBytesTransferred = bytesTransferred; - return INFO::OK; - } - } - -private: - OVERLAPPED m_overlapped; - HANDLE m_hFile; -}; - - -// called by aio_read, aio_write, and lio_listio. -// cb->aio_lio_opcode specifies desired operation. -// @return LibError, also setting errno in case of failure. -static LibError aio_issue(struct aiocb* cb) -{ - // no-op (probably from lio_listio) - if(!cb || cb->aio_lio_opcode == LIO_NOP) - return INFO::SKIPPED; - - // extract aiocb fields for convenience - const bool isWrite = (cb->aio_lio_opcode == LIO_WRITE); - const int fd = cb->aio_fildes; - const size_t size = cb->aio_nbytes; - const off_t ofs = cb->aio_offset; - void* const buf = (void*)cb->aio_buf; // from volatile void* - - // Win32 requires transfers to be sector-aligned. - if(!IsAligned(ofs, sectorSize) || !IsAligned(buf, sectorSize) || !IsAligned(size, sectorSize)) - { - errno = EINVAL; - WARN_RETURN(ERR::INVALID_PARAM); - } - - HANDLE hFile; - { - WinScopedLock lock(WAIO_CS); - hFile = handleManager->Get(fd); - } - if(hFile == INVALID_HANDLE_VALUE) - { - errno = EINVAL; + FileControlBlock* fcb = FindFileControlBlock(fd); + if(!fcb) WARN_RETURN(ERR::INVALID_HANDLE); - } + const HANDLE hFile = fcb->hFile; - debug_assert(!cb->impl); // SUSv3 requires that the aiocb not be in use - cb->impl.reset(new aiocb::Impl); + // allocate all space up front to reduce fragmentation + LARGE_INTEGER size64; size64.QuadPart = alignedSize; + WARN_IF_FALSE(SetFilePointerEx(hFile, size64, 0, FILE_BEGIN)); + WARN_IF_FALSE(SetEndOfFile(hFile)); - LibError ret = cb->impl->Issue(hFile, ofs, buf, size, isWrite); - if(ret < 0) - { - LibError_set_errno(ret); - return ret; - } + // avoid synchronous zero-fill (see discussion in header) + if(pSetFileValidData) + WARN_IF_FALSE(pSetFileValidData(hFile, alignedSize)); return INFO::OK; } -// return status of transfer -int aio_error(const struct aiocb* cb) -{ - return cb->impl->HasCompleted()? 0 : EINPROGRESS; -} +//----------------------------------------------------------------------------- +// helper functions - -// get bytes transferred. call exactly once for each issued request. -ssize_t aio_return(struct aiocb* cb) +// called by aio_read, aio_write, and lio_listio. +// cb->aio_lio_opcode specifies desired operation. +// @return -1 on failure (having also set errno) +static int Issue(aiocb* cb) { - // SUSv3 says we mustn't be callable before the request has completed - debug_assert(cb->impl); - debug_assert(cb->impl->HasCompleted()); - size_t bytesTransferred; - LibError ret = cb->impl->GetResult(&bytesTransferred); - cb->impl.reset(); // disallow calling again, as required by SUSv3 - if(ret < 0) + debug_assert(IsAligned(cb->aio_offset, maxSectorSize)); + debug_assert(IsAligned(cb->aio_buf, maxSectorSize)); + debug_assert(IsAligned(cb->aio_nbytes, maxSectorSize)); + + FileControlBlock* fcb = FindFileControlBlock(cb->aio_fildes); + if(!fcb || fcb->hFile == INVALID_HANDLE_VALUE) { - LibError_set_errno(ret); - return (ssize_t)-1; - } - return (ssize_t)bytesTransferred; -} - - -int aio_suspend(const struct aiocb* const cbs[], int n, const struct timespec* ts) -{ - if(n <= 0 || n > MAXIMUM_WAIT_OBJECTS) - { - WARN_ERR(ERR::INVALID_PARAM); + DEBUG_WARN_ERR(ERR::INVALID_HANDLE); errno = EINVAL; return -1; } - // build array of event handles - HANDLE hEvents[MAXIMUM_WAIT_OBJECTS]; - size_t numPendingIos = 0; + debug_assert(!cb->fcb && !cb->ovl); // SUSv3: aiocb must not be in use + cb->fcb = fcb; + cb->ovl = fcb->ovl.Allocate(cb->aio_offset); + if(!cb->ovl) + { + DEBUG_WARN_ERR(ERR::LIMIT); + errno = EMFILE; + return -1; + } + + WinScopedPreserveLastError s; + + const HANDLE hFile = fcb->hFile; + void* const buf = (void*)cb->aio_buf; // from volatile void* + const DWORD size = u64_lo(cb->aio_nbytes); + debug_assert(u64_hi(cb->aio_nbytes) == 0); + OVERLAPPED* ovl = (OVERLAPPED*)cb->ovl; + // (there is no point in using WriteFileGather/ReadFileScatter here + // because the IO manager still needs to lock pages and translate them + // into an MDL, and we'd just be increasing the number of addresses) + const BOOL ok = (cb->aio_lio_opcode == LIO_WRITE)? WriteFile(hFile, buf, size, 0, ovl) : ReadFile(hFile, buf, size, 0, ovl); + if(ok || GetLastError() == ERROR_IO_PENDING) + return 0; // success + + LibError_set_errno(LibError_from_GLE()); + return -1; +} + + +static bool AreAnyComplete(const struct aiocb* const cbs[], int n) +{ for(int i = 0; i < n; i++) { - if(!cbs[i]) // SUSv3 says NULL entries are to be ignored + if(!cbs[i]) // SUSv3: must ignore NULL entries continue; - aiocb::Impl* impl = cbs[i]->impl.get(); - debug_assert(impl); - if(!impl->HasCompleted()) - hEvents[numPendingIos++] = impl->Event(); + if(HasOverlappedIoCompleted((OVERLAPPED*)cbs[i]->ovl)) + return true; } - if(!numPendingIos) // done, don't need to suspend. - return 0; - const BOOL waitAll = FALSE; - // convert timespec to milliseconds (ts == 0 => no timeout) - const DWORD timeout = ts? (DWORD)(ts->tv_sec*1000 + ts->tv_nsec/1000000) : INFINITE; - const DWORD result = WaitForMultipleObjects((DWORD)numPendingIos, hEvents, waitAll, timeout); - - for(size_t i = 0; i < numPendingIos; i++) - ResetEvent(hEvents[i]); - - switch(result) - { - case WAIT_FAILED: - WARN_ERR(ERR::FAIL); - errno = EIO; - return -1; - - case WAIT_TIMEOUT: - errno = EAGAIN; - return -1; - - default: - return 0; - } + return false; } -int aio_cancel(int fd, struct aiocb* cb) -{ - // Win32 limitation: can't cancel single transfers - - // all pending reads on this file are canceled. - UNUSED2(cb); - - HANDLE hFile; - { - WinScopedLock lock(WAIO_CS); - hFile = handleManager->Get(fd); - } - if(hFile == INVALID_HANDLE_VALUE) - { - WARN_ERR(ERR::INVALID_HANDLE); - errno = EINVAL; - return -1; - } - - WARN_IF_FALSE(CancelIo(hFile)); - return AIO_CANCELED; -} - +//----------------------------------------------------------------------------- +// API int aio_read(struct aiocb* cb) { cb->aio_lio_opcode = LIO_READ; - return (aio_issue(cb) < 0)? 0 : -1; + return Issue(cb); } int aio_write(struct aiocb* cb) { cb->aio_lio_opcode = LIO_WRITE; - return (aio_issue(cb) < 0)? 0 : -1; + return Issue(cb); } @@ -474,7 +521,10 @@ int lio_listio(int mode, struct aiocb* const cbs[], int n, struct sigevent* se) for(int i = 0; i < n; i++) { - if(aio_issue(cbs[i]) < 0) + if(cbs[i] == 0 || cbs[i]->aio_lio_opcode == LIO_NOP) + continue; + + if(Issue(cbs[i]) == -1) return -1; } @@ -485,24 +535,107 @@ int lio_listio(int mode, struct aiocb* const cbs[], int n, struct sigevent* se) } +int aio_suspend(const struct aiocb* const cbs[], int n, const struct timespec* timeout) +{ + // consume all pending notifications to prevent them from piling up if + // requests are always complete by the time we're called + DWORD bytesTransferred; ULONG_PTR key; OVERLAPPED* ovl; + while(PollCompletionPort(hIOCP, 0, bytesTransferred, key, ovl) == INFO::OK) {} + + // avoid blocking if already complete (synchronous requests don't post notifications) + if(AreAnyComplete(cbs, n)) + return 0; + + // caller doesn't want to block, and no requests are complete + if(timeout && timeout->tv_sec == 0 && timeout->tv_nsec == 0) + { + errno = EAGAIN; + return -1; + } + + // reduce CPU usage by blocking until a notification arrives or a + // brief timeout elapses (necessary because other threads - or even + // the above poll - might have consumed our notification). note that + // re-posting notifications that don't concern the respective requests + // is not desirable because POSIX doesn't require aio_suspend to be + // called, which means notifications might pile up. + const DWORD milliseconds = 1; // as short as possible (don't oversleep) + const LibError ret = PollCompletionPort(hIOCP, milliseconds, bytesTransferred, key, ovl); + if(ret != INFO::OK && ret != ERR::AGAIN) // failed + { + debug_assert(0); + return -1; + } + + // scan again (even if we got a notification, it might not concern THESE requests) + if(AreAnyComplete(cbs, n)) + return 0; + + // none completed, must repeat the above steps. provoke being called again by + // claiming to have been interrupted by a signal. + errno = EINTR; + return -1; +} + + +int aio_error(const struct aiocb* cb) +{ + const OVERLAPPED* ovl = (const OVERLAPPED*)cb->ovl; + if(!ovl) // called after aio_return + return EINVAL; + if(!HasOverlappedIoCompleted(ovl)) + return EINPROGRESS; + if(ovl->Internal != ERROR_SUCCESS) + return EIO; + return 0; +} + + +ssize_t aio_return(struct aiocb* cb) +{ + FileControlBlock* fcb = (FileControlBlock*)cb->fcb; + OVERLAPPED* ovl = (OVERLAPPED*)cb->ovl; + if(!fcb || !ovl) + { + errno = EINVAL; + return -1; + } + + const ULONG_PTR status = ovl->Internal; + const ULONG_PTR bytesTransferred = ovl->InternalHigh; + + cb->ovl = 0; // prevent further calls to aio_error/aio_return + COMPILER_FENCE; + fcb->ovl.Deallocate(ovl); + cb->fcb = 0; // allow reuse + + return (status == ERROR_SUCCESS)? bytesTransferred : -1; +} + + +int aio_cancel(int UNUSED(fd), struct aiocb* cb) +{ + // (faster than calling FindFileControlBlock) + const HANDLE hFile = ((const FileControlBlock*)cb->fcb)->hFile; + if(hFile == INVALID_HANDLE_VALUE) + { + WARN_ERR(ERR::INVALID_HANDLE); + errno = EINVAL; + return -1; + } + + // cancel all I/Os this thread issued for the given file + // (CancelIoEx can cancel individual operations, but is only + // available starting with Vista) + WARN_IF_FALSE(CancelIo(hFile)); + + return AIO_CANCELED; +} + + int aio_fsync(int, struct aiocb*) { WARN_ERR(ERR::NOT_IMPLEMENTED); errno = ENOSYS; return -1; } - - -//----------------------------------------------------------------------------- - -static LibError waio_Init() -{ - handleManager = new HandleManager; - return INFO::OK; -} - -static LibError waio_Shutdown() -{ - delete handleManager; - return INFO::OK; -} diff --git a/source/lib/sysdep/os/win/wposix/waio.h b/source/lib/sysdep/os/win/wposix/waio.h index 0ee5716489..6030cca7ed 100644 --- a/source/lib/sysdep/os/win/wposix/waio.h +++ b/source/lib/sysdep/os/win/wposix/waio.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2010 Wildfire Games +/* Copyright (c) 2011 Wildfire Games * * Permission is hereby granted, free of charge, to any person obtaining * a copy of this software and associated documentation files (the @@ -29,13 +29,12 @@ #include "lib/lib_errors.h" #include "lib/os_path.h" +#include "lib/posix/posix_time.h" // timespec #include "lib/sysdep/os/win/wposix/wposix_types.h" -#include "lib/sysdep/os/win/wposix/no_crt_posix.h" - // Note: transfer buffers, offsets, and lengths must be sector-aligned -// (we don't bother copying to an align buffer because the file cache -// already requires splitting IOs into aligned blocks) +// (we don't bother copying to an align buffer because our block cache +// already requires splitting IOs into naturally-aligned blocks) // @@ -57,15 +56,6 @@ struct sigevent // unused }; -// -// -// - -extern int read (int fd, void* buf, size_t nbytes); // thunk -extern int write(int fd, void* buf, size_t nbytes); // thunk -extern off_t lseek(int fd, off_t ofs, int whence); // thunk - - // // // @@ -80,8 +70,11 @@ struct aiocb struct sigevent aio_sigevent; // Signal number and value. (unused) int aio_lio_opcode; // Operation to be performed. - class Impl; - shared_ptr impl; + // internal use only; must be zero-initialized before + // calling the first aio_read/aio_write/lio_listio (aio_return also + // zero-initializes them) + void* fcb; + void* ovl; }; enum @@ -101,21 +94,55 @@ enum LIO_WRITE }; -extern int aio_cancel(int, struct aiocb*); -extern int aio_error(const struct aiocb*); -extern int aio_fsync(int, struct aiocb*); extern int aio_read(struct aiocb*); -extern ssize_t aio_return(struct aiocb*); -struct timespec; -extern int aio_suspend(const struct aiocb* const[], int, const struct timespec*); extern int aio_write(struct aiocb*); extern int lio_listio(int, struct aiocb* const[], int, struct sigevent*); -// for use by wfilesystem's wopen/wclose: +// (if never called, IOCP notifications will pile up.) +extern int aio_suspend(const struct aiocb* const[], int, const struct timespec*); -// (re)open file in asynchronous mode and associate handle with fd. -// (this works because the files default to DENY_NONE sharing) +// @return status of transfer (0 or an errno) +extern int aio_error(const struct aiocb*); + +// @return bytes transferred or -1 on error. +// frees internal storage related to the request and MUST be called +// exactly once for each aiocb after aio_error != EINPROGRESS. +extern ssize_t aio_return(struct aiocb*); + +extern int aio_cancel(int, struct aiocb*); + +extern int aio_fsync(int, struct aiocb*); + +// Windows doesn't allow aio unless the file is opened in asynchronous mode, +// which is not possible with _wsopen_s. since we don't want to have to +// provide a separate File class for aio-enabled files, our wopen wrapper +// will also call this function to open a SECOND handle to the file (works +// because CRT open() defaults to DENY_NONE sharing). the CRT's lowio +// descriptor table remains unaffected, but our [w]aio_* functions are +// notified of the file descriptor, which means e.g. read and aio_read can +// both be used. this function must have been called before any +// other [w]aio_* functions are used. extern LibError waio_reopen(int fd, const OsPath& pathname, int oflag, ...); + +// close our second aio-enabled handle to the file (called from wclose). extern LibError waio_close(int fd); +// call this before writing a large file to preallocate clusters, thus +// reducing fragmentation. +// +// @param alignedSize must be a multiple of alignment (SetEndOfFile requires +// sector alignment; this could be avoided by using the undocumented +// NtSetInformationFile or SetFileInformationByHandle on Vista and later). +// use wtruncate after I/O is complete to chop off any excess padding. +// +// NB: writes that extend a file (i.e. ALL WRITES when creating new files) +// are synchronous, which prevents overlapping I/O and other work. +// (http://support.microsoft.com/default.aspx?scid=kb%3Ben-us%3B156932) +// if Windows XP and the SE_MANAGE_VOLUME_NAME privileges are available, +// this function sets the valid data length to avoid the synchronous zero-fill. +// note that this exposes the previous disk contents (possibly even to +// other users since the waio_reopen design cannot deny file sharing) until +// the application successfully writes to the file. +LIB_API LibError waio_Preallocate(int fd, off_t alignedSize, off_t alignment); + #endif // #ifndef INCLUDED_WAIO diff --git a/source/lib/sysdep/os/win/wposix/wfilesystem.cpp b/source/lib/sysdep/os/win/wposix/wfilesystem.cpp index 7b53dd0168..23addce965 100644 --- a/source/lib/sysdep/os/win/wposix/wfilesystem.cpp +++ b/source/lib/sysdep/os/win/wposix/wfilesystem.cpp @@ -373,6 +373,26 @@ int wclose(int fd) // unistd.h //----------------------------------------------------------------------------- +// we don't want to #define read to _read, since that's a fairly common +// identifier. therefore, translate from MS CRT names via thunk functions. +// efficiency is less important, and the overhead could be optimized away. + +int read(int fd, void* buf, size_t nbytes) +{ + return _read(fd, buf, (int)nbytes); +} + +int write(int fd, void* buf, size_t nbytes) +{ + return _write(fd, buf, (int)nbytes); +} + +off_t lseek(int fd, off_t ofs, int whence) +{ + return _lseeki64(fd, ofs, whence); +} + + int wtruncate(const OsPath& pathname, off_t length) { HANDLE hFile = CreateFileW(OsString(pathname).c_str(), GENERIC_WRITE, 0, 0, OPEN_EXISTING, 0, 0); diff --git a/source/lib/sysdep/os/win/wposix/wfilesystem.h b/source/lib/sysdep/os/win/wposix/wfilesystem.h index fd1abf4be5..76cd3c1313 100644 --- a/source/lib/sysdep/os/win/wposix/wfilesystem.h +++ b/source/lib/sysdep/os/win/wposix/wfilesystem.h @@ -52,4 +52,13 @@ typedef unsigned int mode_t; // defined by MinGW but not VC #define S_ISDIR(m) (m & S_IFDIR) #define S_ISREG(m) (m & S_IFREG) + +// +// +// + +extern int read (int fd, void* buf, size_t nbytes); // thunk +extern int write(int fd, void* buf, size_t nbytes); // thunk +extern off_t lseek(int fd, off_t ofs, int whence); // thunk + #endif // #ifndef INCLUDED_WFILESYSTEM diff --git a/source/lib/sysdep/os/win/wsysdep.cpp b/source/lib/sysdep/os/win/wsysdep.cpp index 730dac3aec..a08cec2ce3 100644 --- a/source/lib/sysdep/os/win/wsysdep.cpp +++ b/source/lib/sysdep/os/win/wsysdep.cpp @@ -66,7 +66,7 @@ std::wstring sys_WideFromArgv(const char* argv_i) const int inputSize = -1; // null-terminated std::vector buf(strlen(argv_i)+1); // (upper bound on number of characters) // NB: avoid mbstowcs because it may specify another locale - const int ret = MultiByteToWideChar(cp, flags, argv_i, inputSize, &buf[0], buf.size()); + const int ret = MultiByteToWideChar(cp, flags, argv_i, (int)inputSize, &buf[0], (int)buf.size()); debug_assert(ret != 0); return std::wstring(&buf[0]); } diff --git a/source/lib/sysdep/os/win/wutil.cpp b/source/lib/sysdep/os/win/wutil.cpp index 710f1377d4..8ff0abb71a 100644 --- a/source/lib/sysdep/os/win/wutil.cpp +++ b/source/lib/sysdep/os/win/wutil.cpp @@ -438,6 +438,32 @@ WinScopedDisableWow64Redirection::~WinScopedDisableWow64Redirection() } +//----------------------------------------------------------------------------- + +LibError wutil_SetPrivilege(const wchar_t* privilege, bool enable) +{ + WinScopedPreserveLastError s; + + HANDLE hToken; + if(!OpenProcessToken(GetCurrentProcess(), TOKEN_ADJUST_PRIVILEGES|TOKEN_QUERY, &hToken)) + return ERR::_1; + + TOKEN_PRIVILEGES tp; + if (!LookupPrivilegeValueW(NULL, privilege, &tp.Privileges[0].Luid)) + return ERR::_2; + tp.PrivilegeCount = 1; + tp.Privileges[0].Attributes = enable? SE_PRIVILEGE_ENABLED : 0; + + SetLastError(0); + const BOOL ok = AdjustTokenPrivileges(hToken, FALSE, &tp, 0, 0, 0); + if(!ok || GetLastError() != 0) + return ERR::_3; + + WARN_IF_FALSE(CloseHandle(hToken)); + return INFO::OK; +} + + //----------------------------------------------------------------------------- // module handle diff --git a/source/lib/sysdep/os/win/wutil.h b/source/lib/sysdep/os/win/wutil.h index b3a1f72e95..eca2046358 100644 --- a/source/lib/sysdep/os/win/wutil.h +++ b/source/lib/sysdep/os/win/wutil.h @@ -77,7 +77,6 @@ extern void wutil_Free(void* p); // critical sections used by win-specific code enum WinLockId { - WAIO_CS, WDBG_SYM_CS, // protects (non-reentrant) dbghelp.dll WDIR_WATCH_CS, @@ -191,6 +190,8 @@ private: //----------------------------------------------------------------------------- +LIB_API LibError wutil_SetPrivilege(const wchar_t* privilege, bool enable); + /** * @return module handle of lib code (that of the main EXE if * linked statically, otherwise the DLL). diff --git a/source/lib/sysdep/smbios.cpp b/source/lib/sysdep/smbios.cpp index 22199a1f8f..464734be55 100644 --- a/source/lib/sysdep/smbios.cpp +++ b/source/lib/sysdep/smbios.cpp @@ -647,7 +647,7 @@ void FieldStringizer::operator()(size_t flags, const char*& value, if(lastChar == std::string::npos) // nothing but spaces return; string.resize(lastChar+1); // strip trailing spaces - if(string == "To Be Filled By O.E.M.") + if(!stricmp(value, "To Be Filled By O.E.M.")) return; WriteName(name); diff --git a/source/lib/tex/tex.cpp b/source/lib/tex/tex.cpp index 82fa437b20..595302ca78 100644 --- a/source/lib/tex/tex.cpp +++ b/source/lib/tex/tex.cpp @@ -33,6 +33,7 @@ #include "lib/timer.h" #include "lib/bits.h" +#include "lib/allocators/shared_ptr.h" #include "lib/sysdep/cpu.h" #include "tex_codec.h" @@ -255,7 +256,8 @@ static LibError add_mipmaps(Tex* t, size_t w, size_t h, size_t bpp, void* newDat WARN_RETURN(ERR::TEX_INVALID_SIZE); t->flags |= TEX_MIPMAPS; // must come before tex_img_size! const size_t mipmap_size = tex_img_size(t); - shared_ptr mipmapData = io_Allocate(mipmap_size, 0); + shared_ptr mipmapData; + AllocateAligned(mipmapData, mipmap_size); CreateLevelData cld = { bpp/8, w, h, (const u8*)newData, data_size }; tex_util_foreach_mipmap(w, h, bpp, mipmapData.get(), 0, 1, create_level, &cld); t->data = mipmapData; @@ -332,7 +334,8 @@ TIMER_ACCRUE(tc_plain_transform); // // this is necessary even when not flipping because the initial data // is read-only. - shared_ptr newData = io_Allocate(new_data_size); + shared_ptr newData; + AllocateAligned(newData, new_data_size); // setup row source/destination pointers (simplifies outer loop) u8* dst = (u8*)newData.get(); diff --git a/source/lib/tex/tex_dds.cpp b/source/lib/tex/tex_dds.cpp index 8f5f72d3a5..706975fc6e 100644 --- a/source/lib/tex/tex_dds.cpp +++ b/source/lib/tex/tex_dds.cpp @@ -27,12 +27,15 @@ #include "precompiled.h" #include "lib/byte_order.h" -#include "tex_codec.h" #include "lib/bits.h" #include "lib/timer.h" +#include "lib/allocators/shared_ptr.h" +#include "tex_codec.h" + // NOTE: the convention is bottom-up for DDS, but there's no way to tell. + //----------------------------------------------------------------------------- // S3TC decompression //----------------------------------------------------------------------------- @@ -279,7 +282,8 @@ static LibError s3tc_decompress(Tex* t) const size_t dxt = t->flags & TEX_DXT; const size_t out_bpp = (dxt != 1)? 32 : 24; const size_t out_size = tex_img_size(t) * out_bpp / t->bpp; - shared_ptr decompressedData = io_Allocate(out_size); + shared_ptr decompressedData; + AllocateAligned(decompressedData, out_size, pageSize); const size_t s3tc_block_size = (dxt == 3 || dxt == 5)? 16 : 8; S3tcDecompressInfo di = { dxt, s3tc_block_size, out_bpp/8, decompressedData.get() }; diff --git a/source/lib/tex/tex_internal.h b/source/lib/tex/tex_internal.h index 03631c5019..641beb4ee5 100644 --- a/source/lib/tex/tex_internal.h +++ b/source/lib/tex/tex_internal.h @@ -28,7 +28,7 @@ #define INCLUDED_TEX_INTERNAL #include "lib/allocators/dynarray.h" -#include "lib/file/io/io.h" // io_Allocate +#include "lib/file/io/io.h" // io::Allocate /** * check if the given texture format is acceptable: 8bpp grey, diff --git a/source/lib/tex/tex_jpg.cpp b/source/lib/tex/tex_jpg.cpp index d8bf479b94..78ae0b0c89 100644 --- a/source/lib/tex/tex_jpg.cpp +++ b/source/lib/tex/tex_jpg.cpp @@ -26,10 +26,12 @@ #include "precompiled.h" +#include + #include "lib/external_libraries/libjpeg.h" +#include "lib/allocators/shared_ptr.h" #include "tex_codec.h" -#include // squelch "dtor / setjmp interaction" warnings. @@ -478,7 +480,8 @@ static LibError jpg_decode_impl(DynArray* da, jpeg_decompress_struct* cinfo, Tex // alloc destination buffer const size_t pitch = w * bpp / 8; const size_t img_size = pitch * h; // for allow_rows - shared_ptr data = io_Allocate(img_size); + shared_ptr data; + AllocateAligned(data, img_size, pageSize); // read rows shared_ptr rows = tex_codec_alloc_rows(data.get(), h, pitch, TEX_TOP_DOWN, 0); diff --git a/source/lib/tex/tex_png.cpp b/source/lib/tex/tex_png.cpp index f121cfa81d..270e0805a8 100644 --- a/source/lib/tex/tex_png.cpp +++ b/source/lib/tex/tex_png.cpp @@ -30,6 +30,7 @@ #include "lib/byte_order.h" #include "tex_codec.h" +#include "lib/allocators/shared_ptr.h" #include "lib/timer.h" #if MSC_VERSION @@ -111,7 +112,8 @@ static LibError png_decode_impl(DynArray* da, png_structp png_ptr, png_infop inf WARN_RETURN(ERR::TEX_INVALID_COLOR_TYPE); const size_t img_size = pitch * h; - shared_ptr data = io_Allocate(img_size); + shared_ptr data; + AllocateAligned(data, img_size, pageSize); shared_ptr rows = tex_codec_alloc_rows(data.get(), h, pitch, TEX_TOP_DOWN, 0); png_read_image(png_ptr, (png_bytepp)rows.get()); diff --git a/source/ps/ConfigDB.cpp b/source/ps/ConfigDB.cpp index cf321fa9a9..4173f75bf1 100644 --- a/source/ps/ConfigDB.cpp +++ b/source/ps/ConfigDB.cpp @@ -17,17 +17,18 @@ #include "precompiled.h" +#include + #include "Pyrogenesis.h" #include "Parser.h" #include "ConfigDB.h" #include "CLogger.h" #include "Filesystem.h" #include "scripting/ScriptingHost.h" +#include "lib/allocators/shared_ptr.h" #include "scriptinterface/ScriptInterface.h" -#include - typedef std::map TConfigMap; TConfigMap CConfigDB::m_Map[CFG_LAST]; VfsPath CConfigDB::m_ConfigFile[CFG_LAST]; @@ -425,7 +426,8 @@ bool CConfigDB::WriteFile(EConfigNamespace ns, const VfsPath& path) return false; } - shared_ptr buf = io_Allocate(1*MiB); + shared_ptr buf; + AllocateAligned(buf, 1*MiB, maxSectorSize); char* pos = (char*)buf.get(); TConfigMap &map=m_Map[ns]; for(TConfigMap::const_iterator it = map.begin(); it != map.end(); ++it) diff --git a/source/ps/Util.cpp b/source/ps/Util.cpp index e52df583ca..7a6668f9a0 100644 --- a/source/ps/Util.cpp +++ b/source/ps/Util.cpp @@ -33,7 +33,6 @@ #include "lib/sysdep/arch/x86_x64/topology.h" #include "lib/sysdep/smbios.h" #include "lib/tex/tex.h" -#include "lib/file/io/io_align.h" // BLOCK_SIZE #include "ps/GameSetup/Config.h" #include "ps/GameSetup/GameSetup.h" @@ -207,7 +206,6 @@ LibError tex_write(Tex* t, const VfsPath& filename) // write to disk LibError ret = INFO::OK; { - (void)da_set_size(&da, round_up(da.cur_size, BLOCK_SIZE)); shared_ptr file = DummySharedPtr(da.base); const ssize_t bytes_written = g_VFS->CreateFile(filename, file, da.pos); if(bytes_written > 0) @@ -254,7 +252,8 @@ void WriteScreenshot(const VfsPath& extension) const size_t img_size = w * h * bpp/8; const size_t hdr_size = tex_hdr_size(filename); - shared_ptr buf = io_Allocate(hdr_size+img_size); + shared_ptr buf; + AllocateAligned(buf, hdr_size+img_size, maxSectorSize); GLvoid* img = buf.get() + hdr_size; Tex t; if(tex_wrap(w, h, bpp, flags, buf, hdr_size, &t) < 0) @@ -311,7 +310,8 @@ void WriteBigScreenshot(const VfsPath& extension, int tiles) void* tile_data = malloc(tile_size); if(!tile_data) WARN_ERR_RETURN(ERR::NO_MEM); - shared_ptr img_buf = io_Allocate(hdr_size+img_size); + shared_ptr img_buf; + AllocateAligned(img_buf, hdr_size+img_size, maxSectorSize); Tex t; GLvoid* img = img_buf.get() + hdr_size; diff --git a/source/ps/XML/XMLWriter.cpp b/source/ps/XML/XMLWriter.cpp index 8182b7f608..25f0e02be1 100644 --- a/source/ps/XML/XMLWriter.cpp +++ b/source/ps/XML/XMLWriter.cpp @@ -23,6 +23,7 @@ #include "ps/Filesystem.h" #include "ps/XML/Xeromyces.h" #include "lib/utf8.h" +#include "lib/allocators/shared_ptr.h" #include "lib/sysdep/cpu.h" #include "maths/Fixed.h" @@ -95,7 +96,8 @@ bool XMLWriter_File::StoreVFS(const PIVFS& vfs, const VfsPath& pathname) if (m_LastElement) debug_warn(L"ERROR: Saving XML while an element is still open"); const size_t size = m_Data.length(); - shared_ptr data = io_Allocate(size); + shared_ptr data; + AllocateAligned(data, size, maxSectorSize); memcpy(data.get(), m_Data.data(), size); LibError ret = vfs->CreateFile(pathname, data, size); if (ret < 0) diff --git a/source/renderer/Renderer.cpp b/source/renderer/Renderer.cpp index f801ad9067..591e1e762b 100644 --- a/source/renderer/Renderer.cpp +++ b/source/renderer/Renderer.cpp @@ -32,6 +32,7 @@ #include "lib/bits.h" // is_pow2 #include "lib/res/graphics/ogl_tex.h" +#include "lib/allocators/shared_ptr.h" #include "maths/Matrix3D.h" #include "maths/MathUtil.h" #include "ps/CLogger.h" @@ -1773,7 +1774,8 @@ int CRenderer::LoadAlphaMaps() size_t tile_w = 2+base+2; // 2 pixel border (avoids bilinear filtering artifacts) size_t total_w = round_up_to_pow2(tile_w * NumAlphaMaps); size_t total_h = base; debug_assert(is_pow2(total_h)); - shared_ptr data = io_Allocate(total_w*total_h*3); + shared_ptr data; + AllocateAligned(data, total_w*total_h*3, maxSectorSize); // for each tile on row for(size_t i=0;i buf = io_Allocate(hdr_size+img_size); + shared_ptr buf; + AllocateAligned(buf, hdr_size+img_size, maxSectorSize); Tex t; if (tex_wrap(w, h, bpp, flags, buf, hdr_size, &t) < 0) return;