Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Use mmap to load USD file for faster loading. #193

Merged
merged 3 commits into from
Oct 13, 2024
Merged

Use mmap to load USD file for faster loading. #193

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
680d24a
Use mmap to load USD file for faster loading.
syoyo Oct 13, 2024
53dbc2b
Fix build on non-win32 environment.
syoyo Oct 13, 2024
a11ab34
fix build on 32bit linux & windows.
syoyo Oct 13, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
245 changes: 199 additions & 46 deletions src/io-util.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
// SPDX-License-Identifier: Apache 2.0
// Copyright 2022 - 2023, Syoyo Fujita.
// Copyright 2023 - Present, Light Transport Entertainment Inc.
//
//
#include <algorithm>
#include <fstream>

Expand All @@ -17,14 +17,13 @@
#define WIN32_LEAN_AND_MEAN
#endif

#include <windows.h> // include API for expanding a file path
#include <io.h>
#include <windows.h> // include API for expanding a file path

#ifndef TINYUSDZ_MMAP_SUPPORTED
#define TINYUSDZ_MMAP_SUPPORTED (1)
#endif


#ifdef _MSC_VER
#undef NOMINMAX
#endif
Expand Down Expand Up @@ -52,16 +51,14 @@
#else

// Assume Posix
#include <wordexp.h>

#include <sys/stat.h>
#include <sys/mman.h>
#include <sys/stat.h>
#include <wordexp.h>

#ifndef TINYUSDZ_MMAP_SUPPORTED
#define TINYUSDZ_MMAP_SUPPORTED (1)
#endif


#endif

#endif // _WIN32
Expand Down Expand Up @@ -90,11 +87,52 @@
namespace tinyusdz {
namespace io {

#if defined(_WIN32)
namespace {

// from llama.cpp ----
// MIT license
std::string GetErrorMessageWin32(DWORD error_code) {
std::string ret;
LPSTR lpMsgBuf = NULL;
DWORD bufLen = FormatMessageA(
FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM |
FORMAT_MESSAGE_IGNORE_INSERTS,
NULL, error_code, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),
(LPSTR)&lpMsgBuf, 0, NULL);
if (!bufLen) {
ret = "Win32 error code: " + std::to_string(error_code);
} else {
ret = lpMsgBuf;
LocalFree(lpMsgBuf);
}

return ret;
}

static std::string llama_format_win_err(DWORD err) {
LPSTR buf;
size_t size = FormatMessageA(
FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM |
FORMAT_MESSAGE_IGNORE_INSERTS,
NULL, err, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), (LPSTR)&buf, 0,
NULL);
if (!size) {
return "FormatMessageA failed";
}
std::string ret(buf, size);
LocalFree(buf);
return ret;
}
// ----

} // namespace
#endif

#ifdef TINYUSDZ_ANDROID_LOAD_FROM_ASSETS
AAssetManager *asset_manager = nullptr;
#endif


bool IsMMapSupported() {
#if TINYUSDZ_MMAP_SUPPORTED
return true;
Expand All @@ -103,86 +141,198 @@ bool IsMMapSupported() {
#endif
}

bool MMapFile(const std::string &filepath, MMapFileHandle *handle, bool writable) {
(void)filepath;
(void)handle;
(void)writable;
bool MMapFile(const std::string &filepath, MMapFileHandle *handle,
bool writable, std::string *err) {

if (!FileExists(filepath, /* userdata */nullptr)) {
if (err) {
(*err) += "File not found: " + filepath + "\n";
}
return false;
}

#if TINYUSDZ_MMAP_SUPPORTED
#if defined(_WIN32)
#if 0 // TODO
int fd = open(filepath.c_str(), writable ? O_RDWR : O_RDONLY);
HANDLE hFile = _get_ofhandle(fd);
HANDLE hMapping = CreateFileMapping(hFile, nullptr, PAGE_READWRITE, 0, 0, nullptr);
if (hMapping == nullptr) {
return false;
// int fd = open(filepath.c_str(), writable ? O_RDWR : O_RDONLY);
HANDLE hFile =
CreateFile(filepath.c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr,
OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr);
if (hFile == INVALID_HANDLE_VALUE) {
if (err) {
(*err) += "Failed to open file.";
}
return false;
}

uint64_t size{0};
{
LARGE_INTEGER sz{};
if (!GetFileSizeEx(hFile, &sz)) {
if (err) {
(*err) +=
"GetFileSizeEx failed: " + llama_format_win_err(GetLastError());
}
void *data = MapViewOfFile(hMapping, FILE_MAP_READ, 0, 0, 0);
if (!data) {
return false;
return false;
}

size = sz.QuadPart;
}

HANDLE hMapping = CreateFileMapping(
hFile, nullptr, writable ? PAGE_READWRITE : PAGE_READONLY, 0, 0, nullptr);
if (hMapping == nullptr) {
if (err) {
(*err) +=
"CreateFileMapping failed: " + llama_format_win_err(GetLastError());
}
return false;
}
void *addr = MapViewOfFile(hMapping, FILE_MAP_READ, 0, 0, 0);
DWORD lastError = GetLastError();
CloseHandle(hMapping);
if (!addr) {
if (err) {
(*err) += "MapViewOfFile failed: " + llama_format_win_err(lastError);
}
return false;
}

size_t prefetch = 0; // TODO
if (prefetch > 0) {
#if _WIN32_WINNT >= 0x602
// PrefetchVirtualMemory is only present on Windows 8 and above, so we
// dynamically load it
BOOL(WINAPI * pPrefetchVirtualMemory)
(HANDLE, ULONG_PTR, PWIN32_MEMORY_RANGE_ENTRY, ULONG);
HMODULE hKernel32 = GetModuleHandleW(L"kernel32.dll");

// may fail on pre-Windows 8 systems
pPrefetchVirtualMemory = reinterpret_cast<decltype(pPrefetchVirtualMemory)>(
GetProcAddress(hKernel32, "PrefetchVirtualMemory"));

if (pPrefetchVirtualMemory) {
// advise the kernel to preload the mapped memory
WIN32_MEMORY_RANGE_ENTRY range;
range.VirtualAddress = addr;
range.NumberOfBytes = static_cast<SIZE_T>((std::min)(size_t(size), prefetch));
if (!pPrefetchVirtualMemory(GetCurrentProcess(), 1, &range, 0)) {
// warn
if (err) {
(*err) += "warning: PrefetchVirtualMemory failed: " +
llama_format_win_err(GetLastError());
}
}
CloseHandle(hMapping);
}
#else
return false;
throw std::runtime_error("PrefetchVirtualMemory unavailable");
if (err) {
(*err) += "PrefetchVirtualMemory unavailable";
}
return false;
#endif
#else // !WIN32
}

handle->addr = reinterpret_cast<uint8_t *>(addr);
handle->size = size;
handle->writable = writable;
handle->filename = filepath;

return true;

#else // !WIN32
// assume posix
FILE *fp = fopen(filepath.c_str(), writable ? "rw" : "r");
if (!fp) {
if (err) {
(*err) += "fopen failed.";
}
return false;
}

int ret = std::fseek(fp, 0, SEEK_END);
if (ret != 0) {
if (err) {
(*err) += "Failed to fseek.";
}
fclose(fp);
return false;
}
}

size_t size = size_t(std::ftell(fp));
std::fseek(fp, 0, SEEK_SET);

if (size == 0) {
if (err) {
(*err) += "File size is zero.";
}
return false;
}

int fd = fileno(fp);

int flags = MAP_PRIVATE; // delayed access
void *addr = mmap(nullptr, size, writable ? PROT_READ|PROT_WRITE : PROT_READ, flags, fd, 0);

int flags = MAP_PRIVATE; // delayed access
void *addr =
mmap(nullptr, size, writable ? PROT_READ | PROT_WRITE : PROT_READ, flags,
fd, 0);
if (addr == MAP_FAILED) {
if (err) {
(*err) += "mmap failed.";
}
return false;
}

handle->addr = reinterpret_cast<uint8_t *>(addr);
handle->size = size;
handle->size = uint64_t(size);
handle->writable = writable;
handle->filename = filepath;
close(fd);

return true;
#endif // !WIN32
#else // !TINYUSDZ_MMAP_SUPPORTED
#endif // !WIN32
#else // !TINYUSDZ_MMAP_SUPPORTED
(void)filepath;
(void)handle;
(void)writable;
(void)err;
return false;
#endif
}

bool UnmapFile(const MMapFileHandle &handle) {
(void)handle;
bool UnmapFile(const MMapFileHandle &handle, std::string *err) {
#if TINYUSDZ_MMAP_SUPPORTED
#if defined(_WIN32)
// TODO
if (handle.addr && handle.size) {
if (!UnmapViewOfFile(handle.addr)) {
if (err) {
(*err) += "warning: UnmapViewOfFile failed: " +
llama_format_win_err(GetLastError());
}
// May ok for now
return true;
}
}

return false;
#else // !WIN32
#else // !WIN32
if (handle.addr && handle.size) {
int ret = munmap(reinterpret_cast<void *>(handle.addr), handle.size);
int ret = munmap(reinterpret_cast<void *>(handle.addr), size_t(handle.size));
if (!ret) {
if (err) {
(*err) += "warning: munmap failed.";
}
}
// ignore return code for now
(void)ret;
return true;
}
return false;
#endif
#else // !TINYUSDZ_MMAP_SUPPORTED
#else // !TINYUSDZ_MMAP_SUPPORTED
(void)handle;
(void)err;
return false;
#endif
}


std::string ExpandFilePath(const std::string &_filepath, void *) {
std::string filepath = _filepath;
if (filepath.size() > 2048) {
Expand Down Expand Up @@ -289,10 +439,10 @@ bool ReadWholeFile(std::vector<uint8_t> *out, std::string *err,
size_t size = size_t(len);

if (size >= filesize_max) {
(*err) += "File size exceeds filesize_max : " + filepath +
" (filesize_max " + std::to_string(filesize_max) + ")";
(*err) += "File size exceeds filesize_max : " + filepath +
" (filesize_max " + std::to_string(filesize_max) + ")";

return false;
return false;
}

out->resize(size);
Expand Down Expand Up @@ -336,7 +486,9 @@ bool ReadWholeFile(std::vector<uint8_t> *out, std::string *err,
(void)buf;
if (!f) {
if (err) {
(*err) += "File read error. Maybe empty file or invalid file : " + filepath + "\n";
(*err) +=
"File read error. Maybe empty file or invalid file : " + filepath +
"\n";
}
return false;
}
Expand Down Expand Up @@ -462,12 +614,13 @@ bool ReadFileHeader(std::vector<uint8_t> *out, std::string *err,
(void)buf;
if (!f) {
if (err) {
(*err) += "File read error. Maybe empty file or invalid file : " + filepath + "\n";
(*err) +=
"File read error. Maybe empty file or invalid file : " + filepath +
"\n";
}
return false;
}


f.seekg(0, f.end);
size_t sz = static_cast<size_t>(f.tellg());
f.seekg(0, f.beg);
Expand Down
10 changes: 7 additions & 3 deletions src/io-util.hh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -133,16 +133,20 @@ struct MMapFileHandle
std::string filename;
bool writable{false};
uint8_t *addr{nullptr};
size_t size{0};
uint64_t size{0};
};

///
/// memory-map file.
/// Returns false when file is not found, invalid, or mmap feature is not available.
/// err = warning message when the API returns true.
///
bool MMapFile(const std::string &filepath, MMapFileHandle *handle, bool writable = false);
bool MMapFile(const std::string &filepath, MMapFileHandle *handle, bool writable, std::string *err);

bool UnmapFile(const MMapFileHandle &handle);
///
/// err = warning message when the API returns true.
///
bool UnmapFile(const MMapFileHandle &handle, std::string *err);

///
/// Filepath is treated as WideChar(UNICODE) on Windows.
Expand Down
Loading
Loading