diff --git a/examples/file_format/file-format-example.cc b/examples/file_format/file-format-example.cc index a95ee6696..039e4f34f 100644 --- a/examples/file_format/file-format-example.cc +++ b/examples/file_format/file-format-example.cc @@ -15,14 +15,38 @@ std::map g_map; // -// We provide two APIs -// -// - File system API(for AssetResolution::openAsset) -// - Create on-memory file(data) +// To read asset in custom format(and in custom file system), this example provides +// +// - AssetResolution handler(for AssetResolution::open_asset) +// - Create on-memory asset system // - File format API(read/write data with custom format) +// - Simple 4 byte binary(float value) +// -// File system APIs -static int MyFSizeData(const char *asset_name, uint64_t *nbytes, std::string *err, void *userdata) { +static int MyARResolve(const char *asset_name, const std::vector &search_paths, std::string *resolved_asset_name, std::string *err, void *userdata) { + (void)err; + (void)userdata; + (void)search_paths; + + if (!asset_name) { + return -2; // err + } + + if (!resolved_asset_name) { + return -2; // err + } + + if (g_map.count(asset_name)) { + (*resolved_asset_name) = asset_name; + return 0; // OK + } + + return -1; // failed to resolve. +} + + +// AssetResoltion handlers +static int MyARSize(const char *asset_name, uint64_t *nbytes, std::string *err, void *userdata) { (void)userdata; if (!asset_name) { @@ -43,7 +67,7 @@ static int MyFSizeData(const char *asset_name, uint64_t *nbytes, std::string *er return 0; // OK } -int MyFSReadData(const char *asset_name, uint64_t req_nbytes, uint8_t *out_buf, +int MyARRead(const char *asset_name, uint64_t req_nbytes, uint8_t *out_buf, uint64_t *nbytes, std::string *err, void *userdata) { if (!asset_name) { @@ -72,30 +96,26 @@ int MyFSReadData(const char *asset_name, uint64_t req_nbytes, uint8_t *out_buf, return 0; } - // + // return -1; } // // custom File-format APIs. // -static bool MyCheck(const uint8_t *addr, const size_t nbytes, std::string *warn, std::string *err, void *user_data) { +static bool MyCheck(const tinyusdz::Asset &asset, std::string *warn, std::string *err, void *user_data) { return true; } -static bool MyRead(const uint8_t *addr, const size_t nbytes, tinyusdz::PrimSpec &ps/* inout */, std::string *warn, std::string *err, void *user_data) { +static bool MyRead(const tinyusdz::Asset &asset, tinyusdz::PrimSpec &ps/* inout */, std::string *warn, std::string *err, void *user_data) { - if (!addr) { + if (asset.size() != 4) { return false; } - if (nbytes != 4) { - return false; - } - float val; - memcpy(&val, addr, 4); + memcpy(&val, asset.data(), 4); tinyusdz::Attribute attr; attr.set_value(val); @@ -107,7 +127,7 @@ static bool MyRead(const uint8_t *addr, const size_t nbytes, tinyusdz::PrimSpec return true; } -static bool MyWrite(const tinyusdz::PrimSpec &ps, std::vector *data_out, std::string *warn, std::string *err, void *user_data) { +static bool MyWrite(const tinyusdz::PrimSpec &ps, tinyusdz::Asset *asset_out, std::string *warn, std::string *err, void *user_data) { // TOOD return false; } @@ -146,8 +166,15 @@ int main(int argc, char **argv) { exit(-1); } - // dummy Asset resolver. tinyusdz::AssetResolutionResolver resolver; + // Register filesystem handler for `.my` asset. + tinyusdz::AssetResolutionHandler ar_handler; + ar_handler.resolve_fun = MyARResolve; + ar_handler.size_fun = MyARSize; + ar_handler.read_fun = MyARRead; + ar_handler.write_fun = nullptr; // not used in this example. + ar_handler.userdata = nullptr; // not used in this example; + resolver.register_asset_resolution_handler("my", ar_handler); tinyusdz::ReferencesCompositionOptions options; options.fileformats["my"] = my_handler; @@ -158,8 +185,8 @@ int main(int argc, char **argv) { std::cerr << "Failed to composite `references`: " << err << "\n"; return -1; } - - + + // Print USD scene as Ascii. std::cout << composited_layer << "\n"; diff --git a/src/asset-resolution.cc b/src/asset-resolution.cc index 853c71a02..02ac43780 100644 --- a/src/asset-resolution.cc +++ b/src/asset-resolution.cc @@ -1,11 +1,13 @@ // SPDX-License-Identifier: Apache 2.0 // Copyright 2022 - Present, Light Transport Entertainment, Inc. +#include #include #include "asset-resolution.hh" #include "common-macros.inc" #include "io-util.hh" #include "value-pprint.hh" +#include "str-util.hh" namespace tinyusdz { @@ -27,6 +29,35 @@ std::string AssetResolutionResolver::search_paths_str() const { bool AssetResolutionResolver::find(const std::string &assetPath) const { DCOUT("search_paths = " << _search_paths); DCOUT("assetPath = " << assetPath); + + std::string ext = io::GetFileExtension(assetPath); + + if (_asset_resolution_handlers.count(ext)) { + if (_asset_resolution_handlers.at(ext).resolve_fun && _asset_resolution_handlers.at(ext).size_fun) { + std::string resolvedPath; + std::string err; + + // Use custom handler's userdata + void *userdata = _asset_resolution_handlers.at(ext).userdata; + + int ret = _asset_resolution_handlers.at(ext).resolve_fun(assetPath.c_str(), _search_paths, &resolvedPath, &err, userdata); + if (ret != 0) { + return false; + } + + uint64_t sz{0}; + ret = _asset_resolution_handlers.at(ext).size_fun(resolvedPath.c_str(), &sz, &err, userdata); + if (ret != 0) { + return false; + } + + return sz > 0; + + } else { + DCOUT("Either Resolve function or Size function is nullptr. Fallback to built-in file handler."); + } + } + // TODO: Cache resolition. std::string fpath = io::FindFile(assetPath, _search_paths); return fpath.size(); @@ -34,10 +65,98 @@ bool AssetResolutionResolver::find(const std::string &assetPath) const { std::string AssetResolutionResolver::resolve( const std::string &assetPath) const { + + std::string ext = io::GetFileExtension(assetPath); + + if (_asset_resolution_handlers.count(ext)) { + if (_asset_resolution_handlers.at(ext).resolve_fun) { + std::string resolvedPath; + std::string err; + + // Use custom handler's userdata + void *userdata = _asset_resolution_handlers.at(ext).userdata; + + int ret = _asset_resolution_handlers.at(ext).resolve_fun(assetPath.c_str(), _search_paths, &resolvedPath, &err, userdata); + if (ret != 0) { + return std::string(); + } + + return resolvedPath; + + } else { + DCOUT("Resolve function is nullptr. Fallback to built-in file handler."); + } + } + DCOUT("search_paths = " << _search_paths); DCOUT("assetPath = " << assetPath); // TODO: Cache resolition. return io::FindFile(assetPath, _search_paths); } +bool AssetResolutionResolver::open_asset(const std::string &resolvedPath, const std::string &assetPath, + Asset *asset_out, std::string *warn, std::string *err) { + + + if (!asset_out) { + if (err) { + (*err) = "`asset` arg is nullptr."; + } + return false; + } + + (void)assetPath; + (void)warn; + + std::string ext = io::GetFileExtension(resolvedPath); + + if (_asset_resolution_handlers.count(ext)) { + if (_asset_resolution_handlers.at(ext).size_fun && _asset_resolution_handlers.at(ext).read_fun) { + + // Use custom handler's userdata + void *userdata = _asset_resolution_handlers.at(ext).userdata; + + // Get asset size. + uint64_t sz{0}; + int ret = _asset_resolution_handlers.at(ext).size_fun(resolvedPath.c_str(), &sz, err, userdata); + if (ret != 0) { + return false; + } + + tinyusdz::Asset asset; + asset.resize(sz); + + uint64_t read_size{0}; + + ret = _asset_resolution_handlers.at(ext).read_fun(resolvedPath.c_str(), /* req_size */asset.size(), asset.data(), &read_size, err, userdata); + if (ret != 0) { + return false; + } + + if (read_size < sz) { + asset.resize(read_size); + // May optimize memory usage + asset.shrink_to_fit(); + } + + (*asset_out) = std::move(asset); + + return true; + } else { + DCOUT("Resolve function is nullptr. Fallback to built-in file handler."); + } + } + + std::vector data; + size_t max_bytes = 1024 * 1024 * _max_asset_bytes_in_mb; + if (!io::ReadWholeFile(&data, err, resolvedPath, max_bytes, + /* userdata */ nullptr)) { + return false; + } + + asset_out->set_data(std::move(data)); + + return true; +} + } // namespace tinyusdz diff --git a/src/asset-resolution.hh b/src/asset-resolution.hh index 74b527ac1..696092614 100644 --- a/src/asset-resolution.hh +++ b/src/asset-resolution.hh @@ -29,6 +29,14 @@ class Asset { uint8_t *data() { return buf_.data(); } + void resize(uint64_t sz) { buf_.resize(sz); } + + void shrink_to_fit() { buf_.shrink_to_fit(); } + + void set_data(const std::vector &&rhs) { + buf_ = rhs; + } + #if 0 /// /// Read asset data to `buffer` @@ -59,16 +67,28 @@ struct ResolverAssetInfo { /// approach. /// +// Resolve asset. +// // @param[in] asset_name Asset name or filepath +// @param[in] search_paths Search paths. +// @param[out] resolved_asset_name Resolved asset name. +// @param[out] err Error message. +// @param[inout] userdata Userdata. +// +// @return 0 upon success. -1 = asset cannot be resolved(not found). other negative value = error +typedef int (*FSResolveAsset)(const char *asset_name, const std::vector &search_paths, std::string *resolved_asset_name, + std::string *err, void *userdata); + +// @param[in] resolved_asset_name Resolved Asset name or filepath // @param[out] nbytes Bytes of this asset. // @param[out] err Error message. // @param[inout] userdata Userdata. // // @return 0 upon success. negative value = error -typedef int (*FSSizeData)(const char *asset_name, uint64_t *nbytes, +typedef int (*FSSizeAsset)(const char *resolved_asset_name, uint64_t *nbytes, std::string *err, void *userdata); -// @param[in] asset_name Asset name or filepath +// @param[in] resolved_asset_name Resolved Asset name or filepath // @param[in] req_nbytes Required bytes for output buffer. // @param[out] out_buf Output buffer. Memory should be allocated before calling this functione(`req_nbytes` or more) // @param[out] nbytes Read bytes. 0 <= nbytes <= `req_nbytes` @@ -76,26 +96,29 @@ typedef int (*FSSizeData)(const char *asset_name, uint64_t *nbytes, // @param[inout] userdata Userdata. // // @return 0 upon success. negative value = error -typedef int (*FSReadData)(const char *asset_name, uint64_t req_nbytes, uint8_t *out_buf, +typedef int (*FSReadAsset)(const char *resolved_asset_name, uint64_t req_nbytes, uint8_t *out_buf, uint64_t *nbytes, std::string *err, void *userdata); -// @param[in] asset_name Asset name or filepath +// @param[in] asset_name Asset name or filepath(could be empty) +// @param[in] resolved_asset_name Resolved Asset name or filepath // @param[in] buffer Data. // @param[in] nbytes Data bytes. // @param[out] err Error message. // @param[inout] userdata Userdata. // // @return 0 upon success. negative value = error -typedef int (*FSWriteData)(const char *asset_name, const uint8_t *buffer, +typedef int (*FSWriteAsset)(const char *asset_name, const char *resolved_asset_name, const uint8_t *buffer, const uint64_t nbytes, std::string *err, void *userdata); -struct FileSystemHandler { - FSSizeData size_fun{nullptr}; - FSReadData read_fun{nullptr}; - FSWriteData write_fun{nullptr}; +struct AssetResolutionHandler { + FSResolveAsset resolve_fun{nullptr}; + FSSizeAsset size_fun{nullptr}; + FSReadAsset read_fun{nullptr}; + FSWriteAsset write_fun{nullptr}; void *userdata{nullptr}; }; +#if 0 // deprecated. /// /// @param[in] path Path string to be resolved. /// @param[in] assetInfo nullptr when no `assetInfo` assigned to this path. @@ -107,6 +130,7 @@ typedef bool (*ResolvePathHandler)(const std::string &path, const ResolverAssetInfo *assetInfo, void *userdata, std::string *resolvedPath, std::string *err); +#endif class AssetResolutionResolver { public: @@ -115,7 +139,8 @@ class AssetResolutionResolver { AssetResolutionResolver(const AssetResolutionResolver &rhs) { if (this != &rhs) { - _resolve_path_handler = rhs._resolve_path_handler; + //_resolve_path_handler = rhs._resolve_path_handler; + _asset_resolution_handlers = rhs._asset_resolution_handlers; _userdata = rhs._userdata; _search_paths = rhs._search_paths; } @@ -123,7 +148,8 @@ class AssetResolutionResolver { AssetResolutionResolver &operator=(const AssetResolutionResolver &rhs) { if (this != &rhs) { - _resolve_path_handler = rhs._resolve_path_handler; + // _resolve_path_handler = rhs._resolve_path_handler; + _asset_resolution_handlers = rhs._asset_resolution_handlers; _userdata = rhs._userdata; _search_paths = rhs._search_paths; } @@ -132,7 +158,8 @@ class AssetResolutionResolver { AssetResolutionResolver &operator=(AssetResolutionResolver &&rhs) { if (this != &rhs) { - _resolve_path_handler = rhs._resolve_path_handler; + //_resolve_path_handler = rhs._resolve_path_handler; + _asset_resolution_handlers = rhs._asset_resolution_handlers; _userdata = rhs._userdata; _search_paths = std::move(rhs._search_paths); } @@ -156,15 +183,30 @@ class AssetResolutionResolver { std::string search_paths_str() const; /// - /// Register user defined filesystem handler. - /// Default = use file handler(FILE/ifstream) + /// Register user defined AssetResolution handler per file extension. + /// Default = use built-in file handler(FILE/ifstream) + /// This handler is used in resolve(), find() and open_asset() /// - void register_filesystem_handler(FileSystemHandler handler) { - _filesystem_handler = handler; + void register_asset_resolution_handler(const std::string &ext_name, AssetResolutionHandler handler) { + if (ext_name.empty()) { + return; + } + _asset_resolution_handlers[ext_name] = handler; + } + + bool unregister_asset_resolution_handler(const std::string &ext_name) { + if (_asset_resolution_handlers.count(ext_name)) { + _asset_resolution_handlers.erase(ext_name); + } + return false; } - void unregister_filesystem_handler() { _filesystem_handler.reset(); } + bool has_asset_resolution_handler(const std::string &ext_name) { + return _asset_resolution_handlers.count(ext_name); + } + +#if 0 /// /// Register user defined asset path resolver. /// Default = find file from search paths. @@ -174,6 +216,7 @@ class AssetResolutionResolver { } void unregister_resolve_path_handler() { _resolve_path_handler = nullptr; } +#endif /// /// Check if input asset exists(do asset resolution inside the function). @@ -193,7 +236,7 @@ class AssetResolutionResolver { /// Open asset from the resolved Path. /// /// @param[in] resolvedPath Resolved path(through `resolve()`) - /// @param[in] assetPath Asset path of resolved path. + /// @param[in] assetPath Asset path(could be empty) /// @param[out] asset Asset. /// @param[out] warn Warning. /// @param[out] err Error message. @@ -207,16 +250,84 @@ class AssetResolutionResolver { void *get_userdata() { return _userdata; } const void *get_userdata() const { return _userdata; } + void set_max_asset_bytes_in_mb(size_t megabytes) { + if (megabytes > 0) { + _max_asset_bytes_in_mb = megabytes; + } + } + + size_t get_max_asset_bytes_in_mb() const { + return _max_asset_bytes_in_mb; + } + private: - ResolvePathHandler _resolve_path_handler{nullptr}; + //ResolvePathHandler _resolve_path_handler{nullptr}; void *_userdata{nullptr}; std::vector _search_paths; + size_t _max_asset_bytes_in_mb{1024*1024}; // default 1 TB - nonstd::optional _filesystem_handler; + std::map _asset_resolution_handlers; // TODO: Cache resolution result // mutable _dirty{true}; // mutable std::map _cached_resolved_paths; }; +// forward decl +class PrimSpec; + +// +// Fileformat plugin(callback) interface. +// For fileformat which is used in `subLayers`, `reference` or `payload`. +// +// TinyUSDZ uses C++ callback interface for security. +// (On the contrary, pxrUSD uses `plugInfo.json` + dll). +// +// Texture image/Shader file(e.g. glsl) is not handled in this API. +// (Plese refer T.B.D. for texture/shader) +// +// TODO: Move to another header file? + +// Check if given data is a expectected file format +// +// @param[in] asset Asset data. +// @param[out] warn Warning message +// @param[out] err Error message(when the fuction returns false) +// @param[inout] user_data Userdata. can be nullptr. +// @return true when the given data is expected file format. +typedef bool (*FileFormatCheckFunction)(const Asset &asset, std::string *warn, std::string *err, void *user_data); + + +// Read content of data into PrimSpec(metadatum, properties, primChildren/variantChildren). +// +// @param[in] asset Asset data +// @param[inout] ps PrimSpec which references/payload this asset. +// @param[out] warn Warning message +// @param[out] err Error message(when the fuction returns false) +// @param[inout] user_data Userdata. can be nullptr. +// @return true when reading data succeeds. +typedef bool (*FileFormatReadFunction)(const Asset &asset, PrimSpec &ps/* inout */, std::string *warn, std::string *err, void *user_data); + +// Write corresponding content of PrimSpec to a binary data +// +// @param[in] ps PrimSpec which refers this asset. +// @param[out] out_asset Output asset data. +// @param[out] warn Warning message +// @param[out] err Error message(when the fuction returns false) +// @param[inout] user_data Userdata. can be nullptr. +// @return true upon data write success. +typedef bool (*FileFormatWriteFunction)(const PrimSpec &ps, Asset *out_data, std::string *warn, std::string *err, void *user_data); + +struct FileFormatHandler +{ + std::string extension; // fileformat extension. + std::string description; // Description of this fileformat. can be empty. + + FileFormatCheckFunction checker{nullptr}; + FileFormatReadFunction reader{nullptr}; + FileFormatWriteFunction writer{nullptr}; + void *userdata{nullptr}; +}; + + } // namespace tinyusdz diff --git a/src/prim-types.hh b/src/prim-types.hh index a7fdfa145..66cb930e6 100644 --- a/src/prim-types.hh +++ b/src/prim-types.hh @@ -3587,61 +3587,6 @@ struct Layer { mutable bool _has_class_primspec{true}; }; -// -// Fileformat plugin(callback) interface. -// For fileformat which is used in `subLayers`, `reference` or `payload`. -// -// TinyUSDZ uses C++ callback interface for security. -// (On the contrary, pxrUSD uses `plugInfo.json` + dll). -// -// Texture image/Shader file(e.g. glsl) is not handled in this API. -// (Plese refer T.B.D. for texture/shader) -// -// TODO: Move to another header file? - -// Check if given data is a valid file format -// -// @param[in] addr Data address -// @param[in] nbytes Data bytes -// @param[out] warn Warning message -// @param[out] err Error message(when the fuction returns false) -// @param[inout] user_data Userdata. can be nullptr. -// @return true when the given data is expected file format. -typedef bool (*FileFormatCheckFunction)(const uint8_t *addr, const size_t nbytes, std::string *warn, std::string *err, void *user_data); - - -// Read content of data into PrimSpec(metadatum, properties, primChildren/variantChildren). -// -// @param[in] addr Data address -// @param[in] nbytes Data bytes -// @param[inout] ps PrimSpec which references/payload this asset. -// @param[out] warn Warning message -// @param[out] err Error message(when the fuction returns false) -// @param[inout] user_data Userdata. can be nullptr. -// @return true when reading data succeeds. -typedef bool (*FileFormatReadFunction)(const uint8_t *addr, const size_t nbytes, PrimSpec &ps/* inout */, std::string *warn, std::string *err, void *user_data); - -// Write corresponding content of PrimSpec to a binary data -// -// @param[in] ps PrimSpec which refers this asset. -// @param[out] out_data Data. -// @param[out] warn Warning message -// @param[out] err Error message(when the fuction returns false) -// @param[inout] user_data Userdata. can be nullptr. -// @return true upon data write success. -typedef bool (*FileFormatWriteFunction)(const PrimSpec &ps, std::vector *out_data, std::string *warn, std::string *err, void *user_data); - -struct FileFormatHandler -{ - std::string extension; // fileformat extension. - std::string description; // Description of this fileformat. can be empty. - - FileFormatCheckFunction checker{nullptr}; - FileFormatReadFunction reader{nullptr}; - FileFormatWriteFunction writer{nullptr}; - void *userdata{nullptr}; -}; - nonstd::optional InterpolationFromString(const std::string &v); nonstd::optional OrientationFromString(const std::string &v);