OpenVDB: nanovdb/util/IO.h Source File

OpenVDB 11.0.0

Loading...
Searching...
No Matches
IO.h
Go to the documentation of this file.
1// Copyright Contributors to the OpenVDB Project
2// SPDX-License-Identifier: MPL-2.0
3
4/*!
5 \file IO.h
6
7 \author Ken Museth
8
9 \date May 1, 2020
10
11 \brief Implements I/O for NanoVDB grids. Features optional BLOSC and ZIP
12 file compression, support for multiple grids per file as well as
13 multiple grid types.
14
15 \note This file does NOT depend on OpenVDB, but optionally on ZIP and BLOSC
16
17 \details NanoVDB files take on of two formats:
18 1) multiple segments each with multiple grids (segments have easy to access metadata about its grids)
19 2) starting with verion 32.6.0 nanovdb files also support a raw buffer with one or more grids (just a
20 dump of a raw grid buffer, so no new metadata).
21
22 // 1: Segment: FileHeader, MetaData0, gridName0...MetaDataN, gridNameN, compress Grid0,...compressed GridN
23 // 2: Raw: Grid0,...GridN
24*/
25
26#ifndef NANOVDB_IO_H_HAS_BEEN_INCLUDED
27#define NANOVDB_IO_H_HAS_BEEN_INCLUDED
28
29#include <nanovdb/NanoVDB.h>
30#include "GridHandle.h"
31#include "GridChecksum.h"// for updateGridCount
32
33#include <fstream> // for std::ifstream
34#include <iostream> // for std::cerr/cout
35#include <string> // for std::string
36#include <sstream> // for std::stringstream
37#include <cstring> // for std::strcmp
38#include <memory> // for std::unique_ptr
39#include <vector> // for std::vector
40#ifdef NANOVDB_USE_ZIP
41#include <zlib.h> // for ZIP compression
42#endif
43#ifdef NANOVDB_USE_BLOSC
44#include <blosc.h> // for BLOSC compression
45#endif
46
47// Due to a bug in older versions of gcc, including fstream might
48// define "major" and "minor" which are used as member data below.
49// See https://bugzilla.redhat.com/show_bug.cgi?id=130601
50#if defined(major) || defined(minor)
51#undef major
52#undef minor
53#endif
54
55namespace nanovdb {
56
57namespace io {
58
59// --------------------------> writeGrid(s) <------------------------------------
60
61/// @brief Write a single grid to file (over-writing existing content of the file)
62template<typename BufferT>
63void writeGrid(const std::string& fileName, const GridHandle<BufferT>& handle, io::Codec codec = io::Codec::NONE, int verbose = 0);
64
65/// @brief Write multiple grids to file (over-writing existing content of the file)
66template<typename BufferT = HostBuffer, template<typename...> class VecT = std::vector>
67void writeGrids(const std::string& fileName, const VecT<GridHandle<BufferT>>& handles, Codec codec = Codec::NONE, int verbose = 0);
68
69// --------------------------> readGrid(s) <------------------------------------
70
71/// @brief Read and return one or all grids from a file into a single GridHandle
72/// @tparam BufferT Type of buffer used memory allocation
73/// @param fileName string name of file to be read from
74/// @param n zero-based signed index of the grid to be read.
75/// The default value of 0 means read only first grid.
76/// A negative value of n means read all grids in the file.
77/// @param verbose specify verbosity level. Default value of zero means quiet.
78/// @param buffer optional buffer used for memory allocation
79/// @return return a single GridHandle with one or all grids found in the file
80/// @throw will throw a std::runtime_error if the file does not contain a grid with index n
81template<typename BufferT = HostBuffer>
82GridHandle<BufferT> readGrid(const std::string& fileName, int n = 0, int verbose = 0, const BufferT& buffer = BufferT());
83
84/// @brief Read and return the first grid with a specific name from a file
85/// @tparam BufferT Type of buffer used memory allocation
86/// @param fileName string name of file to be read from
87/// @param gridName string name of the grid to be read
88/// @param verbose specify verbosity level. Default value of zero means quiet.
89/// @param buffer optional buffer used for memory allocation
90/// @return return a single GridHandle containing the grid with the specific name
91/// @throw will throw a std::runtime_error if the file does not contain a grid with the specific name
92template<typename BufferT = HostBuffer>
93GridHandle<BufferT> readGrid(const std::string& fileName, const std::string& gridName, int verbose = 0, const BufferT& buffer = BufferT());
94
95/// @brief Read all the grids in the file and return them as a vector of multiple GridHandles, each containing
96/// all grids encoded in the same segment of the file (i.e. they where written together)
97/// @tparam BufferT Type of buffer used memory allocation
98/// @param fileName string name of file to be read from
99/// @param verbose specify verbosity level. Default value of zero means quiet.
100/// @param buffer optional buffer used for memory allocation
101/// @return Return a vector of GridHandles each containing all grids encoded
102/// in the same segment of the file (i.e. they where written together).
103template<typename BufferT = HostBuffer, template<typename...> class VecT = std::vector>
104VecT<GridHandle<BufferT>> readGrids(const std::string& fileName, int verbose = 0, const BufferT& buffer = BufferT());
105
106// -----------------------------------------------------------------------
107
108/// We fix a specific size for counting bytes in files so that they
109/// are saved the same regardless of machine precision. (Note there are
110/// still little/bigendian issues, however)
111 using fileSize_t = uint64_t;
112
113/// @brief Internal functions for compressed read/write of a NanoVDB GridHandle into a stream
114///
115/// @warning These functions should never be called directly by client code
116 namespace Internal {
117 static constexpr fileSize_t MAX_SIZE = 1UL << 30; // size is 1 GB
118
119template<typename BufferT>
120 static fileSize_t write(std::ostream& os, const GridHandle<BufferT>& handle, Codec codec, uint32_t n);
121
122template<typename BufferT>
123 static void read(std::istream& is, BufferT& buffer, Codec codec);
124
125static void read(std::istream& is, char* data, fileSize_t size, Codec codec);
126} // namespace Internal
127
128/// @brief Standard hash function to use on strings; std::hash may vary by
129/// platform/implementation and is know to produce frequent collisions.
130uint64_t stringHash(const char* cstr);
131
132/// @brief Return a uint64_t hash key of a std::string
133 inline uint64_t stringHash(const std::string& str){return stringHash(str.c_str());}
134
135/// @brief Return a uint64_t with its bytes reversed so we can check for endianness
136 inline uint64_t reverseEndianness(uint64_t val)
137{
138 return (((val) >> 56) & 0x00000000000000FF) | (((val) >> 40) & 0x000000000000FF00) |
139 (((val) >> 24) & 0x0000000000FF0000) | (((val) >> 8) & 0x00000000FF000000) |
140 (((val) << 8) & 0x000000FF00000000) | (((val) << 24) & 0x0000FF0000000000) |
141 (((val) << 40) & 0x00FF000000000000) | (((val) << 56) & 0xFF00000000000000);
142}
143
144/// @brief This class defines the meta data stored for each grid in a segment
145///
146/// @details A segment consists of a FileHeader followed by a list of FileGridMetaData
147/// each followed by grid names and then finally the grids themselves.
148///
149/// @note This class should not be confused with nanovdb::GridMetaData defined in NanoVDB.h
150/// Also, FileMetaData is defined in NanoVDB.h.
151 struct FileGridMetaData : public FileMetaData
152{
153 static_assert(sizeof(FileMetaData) == 176, "Unexpected sizeof(FileMetaData)");
154 std::string gridName;
155 void read(std::istream& is);
156 void write(std::ostream& os) const;
157 FileGridMetaData() {}
158 template<typename ValueT>
159 FileGridMetaData(uint64_t size, Codec c, const NanoGrid<ValueT>& grid);
160 uint64_t memUsage() const { return sizeof(FileMetaData) + nameSize; }
161}; // FileGridMetaData
162
163/// @brief This class defines all the data stored in segment of a file
164///
165/// @details A segment consists of a FileHeader followed by a list of FileGridMetaData
166/// each followed by grid names and then finally the grids themselves.
167 struct Segment
168{
169 // Check assumptions made during read and write of FileHeader and FileMetaData
170 static_assert(sizeof(FileHeader) == 16u, "Unexpected sizeof(FileHeader)");
171 FileHeader header;// defined in NanoVDB.h
172 std::vector<FileGridMetaData> meta;// defined in NanoVDB.h
173 Segment(Codec c = Codec::NONE)
174#ifdef NANOVDB_USE_NEW_MAGIC_NUMBERS
175 : header{NANOVDB_MAGIC_FILE, Version(), 0u, c}
176#else
177 : header{NANOVDB_MAGIC_NUMBER, Version(), 0u, c}
178#endif
179 , meta()
180 {
181 }
182 template<typename BufferT>
183 void add(const GridHandle<BufferT>& h);
184 bool read(std::istream& is);
185 void write(std::ostream& os) const;
186 uint64_t memUsage() const;
187}; // Segment
188
189/// @brief Return true if the file contains a grid with the specified name
190bool hasGrid(const std::string& fileName, const std::string& gridName);
191
192/// @brief Return true if the stream contains a grid with the specified name
193bool hasGrid(std::istream& is, const std::string& gridName);
194
195/// @brief Reads and returns a vector of meta data for all the grids found in the specified file
196std::vector<FileGridMetaData> readGridMetaData(const std::string& fileName);
197
198/// @brief Reads and returns a vector of meta data for all the grids found in the specified stream
199std::vector<FileGridMetaData> readGridMetaData(std::istream& is);
200
201// --------------------------> Implementations for Internal <------------------------------------
202
203template<typename BufferT>
204 fileSize_t Internal::write(std::ostream& os, const GridHandle<BufferT>& handle, Codec codec, unsigned int n)
205{
206 const char* data = reinterpret_cast<const char*>(handle.gridData(n));
207 fileSize_t total = 0, residual = handle.gridSize(n);
208
209 switch (codec) {
210 case Codec::ZIP: {
211#ifdef NANOVDB_USE_ZIP
212 uLongf size = compressBound(residual); // Get an upper bound on the size of the compressed data.
213 std::unique_ptr<Bytef[]> tmp(new Bytef[size]);
214 const int status = compress(tmp.get(), &size, reinterpret_cast<const Bytef*>(data), residual);
215 if (status != Z_OK)
216 std::runtime_error("Internal write error in ZIP");
217 if (size > residual)
218 std::cerr << "\nWarning: Unexpected ZIP compression from " << residual << " to " << size << " bytes\n";
219 const fileSize_t outBytes = size;
220 os.write(reinterpret_cast<const char*>(&outBytes), sizeof(fileSize_t));
221 os.write(reinterpret_cast<const char*>(tmp.get()), outBytes);
222 total += sizeof(fileSize_t) + outBytes;
223#else
224 throw std::runtime_error("ZIP compression codec was disabled during build");
225#endif
226 break;
227 }
228 case Codec::BLOSC: {
229#ifdef NANOVDB_USE_BLOSC
230 do {
231 fileSize_t chunk = residual < MAX_SIZE ? residual : MAX_SIZE, size = chunk + BLOSC_MAX_OVERHEAD;
232 std::unique_ptr<char[]> tmp(new char[size]);
233 const int count = blosc_compress_ctx(9, 1, sizeof(float), chunk, data, tmp.get(), size, BLOSC_LZ4_COMPNAME, 1 << 18, 1);
234 if (count <= 0)
235 std::runtime_error("Internal write error in BLOSC");
236 const fileSize_t outBytes = count;
237 os.write(reinterpret_cast<const char*>(&outBytes), sizeof(fileSize_t));
238 os.write(reinterpret_cast<const char*>(tmp.get()), outBytes);
239 total += sizeof(fileSize_t) + outBytes;
240 data += chunk;
241 residual -= chunk;
242 } while (residual > 0);
243#else
244 throw std::runtime_error("BLOSC compression codec was disabled during build");
245#endif
246 break;
247 }
248 default:
249 os.write(data, residual);
250 total += residual;
251 }
252 if (!os) throw std::runtime_error("Failed to write Tree to file");
253 return total;
254} // Internal::write
255
256template<typename BufferT>
257void Internal::read(std::istream& is, BufferT& buffer, Codec codec)
258{
259 Internal::read(is, reinterpret_cast<char*>(buffer.data()), buffer.size(), codec);
260} // Internal::read
261
262/// @brief read compressed grid from stream
263/// @param is input stream to read from
264/// @param data data buffer to write into
265/// @param residual expected size of uncompressed data
266/// @param codec mode of compression
267 void Internal::read(std::istream& is, char* data, fileSize_t residual, Codec codec)
268{
269 // read tree using optional compression
270 switch (codec) {
271 case Codec::ZIP: {
272#ifdef NANOVDB_USE_ZIP
273 fileSize_t size;
274 is.read(reinterpret_cast<char*>(&size), sizeof(fileSize_t));
275 std::unique_ptr<Bytef[]> tmp(new Bytef[size]);
276 is.read(reinterpret_cast<char*>(tmp.get()), size);
277 uLongf numBytes = residual;
278 int status = uncompress(reinterpret_cast<Bytef*>(data), &numBytes, tmp.get(), static_cast<uLongf>(size));
279 if (status != Z_OK) std::runtime_error("Internal read error in ZIP");
280 if (fileSize_t(numBytes) != residual) throw std::runtime_error("UNZIP failed on byte size");
281#else
282 throw std::runtime_error("ZIP compression codec was disabled during build");
283#endif
284 break;
285 }
286 case Codec::BLOSC: {
287#ifdef NANOVDB_USE_BLOSC
288 do {
289 fileSize_t size;
290 is.read(reinterpret_cast<char*>(&size), sizeof(fileSize_t));
291 std::unique_ptr<char[]> tmp(new char[size]);
292 is.read(reinterpret_cast<char*>(tmp.get()), size);
293 const fileSize_t chunk = residual < MAX_SIZE ? residual : MAX_SIZE;
294 const int count = blosc_decompress_ctx(tmp.get(), data, size_t(chunk), 1); //fails with more threads :(
295 if (count < 1)
296 std::runtime_error("Internal read error in BLOSC");
297 if (count != int(chunk))
298 throw std::runtime_error("BLOSC failed on byte size");
299 data += size_t(chunk);
300 residual -= chunk;
301 } while (residual > 0);
302#else
303 throw std::runtime_error("BLOSC compression codec was disabled during build");
304#endif
305 break;
306 }
307 default:
308 is.read(data, residual);// read uncompressed data
309 }
310 if (!is) throw std::runtime_error("Failed to read Tree from file");
311} // Internal::read
312
313// --------------------------> Implementations for FileGridMetaData <------------------------------------
314
315template<typename ValueT>
316 inline FileGridMetaData::FileGridMetaData(uint64_t size, Codec c, const NanoGrid<ValueT>& grid)
317 : FileMetaData{size, // gridSize
318 size, // fileSize (will typically be redefined)
319 0u, // nameKey
320 grid.activeVoxelCount(), // voxelCount
321 grid.gridType(), // gridType
322 grid.gridClass(), // gridClass
323 grid.worldBBox(), // worldBBox
324 grid.tree().bbox(), // indexBBox
325 grid.voxelSize(), // voxelSize
326 0, // nameSize
327 {0, 0, 0, 1}, // nodeCount[4]
328 {0, 0, 0}, // tileCount[3]
329 c, // codec
330 0, // padding
331 Version()}// version
332 , gridName(grid.gridName())
333{
334 nameKey = stringHash(gridName);
335 nameSize = static_cast<uint32_t>(gridName.size() + 1); // include '\0'
336 const uint32_t* ptr = reinterpret_cast<const TreeData*>(&grid.tree())->mNodeCount;
337 for (int i = 0; i < 3; ++i) FileMetaData::nodeCount[i] = *ptr++;
338 for (int i = 0; i < 3; ++i) FileMetaData::tileCount[i] = *ptr++;
339}// FileGridMetaData::FileGridMetaData
340
341 inline void FileGridMetaData::write(std::ostream& os) const
342{
343 os.write(reinterpret_cast<const char*>(this), sizeof(FileMetaData));
344 os.write(gridName.c_str(), nameSize);
345 if (!os) throw std::runtime_error("Failed writing FileGridMetaData");
346}// FileGridMetaData::write
347
348 inline void FileGridMetaData::read(std::istream& is)
349{
350 is.read(reinterpret_cast<char*>(this), sizeof(FileMetaData));
351 std::unique_ptr<char[]> tmp(new char[nameSize]);
352 is.read(reinterpret_cast<char*>(tmp.get()), nameSize);
353 gridName.assign(tmp.get());
354 if (!is) throw std::runtime_error("Failed reading FileGridMetaData");
355}// FileGridMetaData::read
356
357// --------------------------> Implementations for Segment <------------------------------------
358
359 inline uint64_t Segment::memUsage() const
360{
361 uint64_t sum = sizeof(FileHeader);
362 for (auto& m : meta) sum += m.memUsage();// includes FileMetaData + grid name
363 return sum;
364}// Segment::memUsage
365
366template<typename BufferT>
367 inline void Segment::add(const GridHandle<BufferT>& h)
368{
369 for (uint32_t i = 0; i < h.gridCount(); ++i) {
370 if (auto* grid = h.template grid<float>(i)) { // most common
371 meta.emplace_back(h.gridSize(i), header.codec, *grid);
372 } else if (auto* grid = h.template grid<Vec3f>(i)) {
373 meta.emplace_back(h.gridSize(i), header.codec, *grid);
374 } else if (auto* grid = h.template grid<double>(i)) {
375 meta.emplace_back(h.gridSize(i), header.codec, *grid);
376 } else if (auto* grid = h.template grid<int32_t>(i)) {
377 meta.emplace_back(h.gridSize(i), header.codec, *grid);
378 } else if (auto* grid = h.template grid<uint32_t>(i)) {
379 meta.emplace_back(h.gridSize(i), header.codec, *grid);
380 } else if (auto* grid = h.template grid<int64_t>(i)) {
381 meta.emplace_back(h.gridSize(i), header.codec, *grid);
382 } else if (auto* grid = h.template grid<int16_t>(i)) {
383 meta.emplace_back(h.gridSize(i), header.codec, *grid);
384 } else if (auto* grid = h.template grid<Vec3d>(i)) {
385 meta.emplace_back(h.gridSize(i), header.codec, *grid);
386 } else if (auto* grid = h.template grid<ValueMask>(i)) {
387 meta.emplace_back(h.gridSize(i), header.codec, *grid);
388 } else if (auto* grid = h.template grid<ValueIndex>(i)) {
389 meta.emplace_back(h.gridSize(i), header.codec, *grid);
390 } else if (auto* grid = h.template grid<ValueIndexMask>(i)) {
391 meta.emplace_back(h.gridSize(i), header.codec, *grid);
392 } else if (auto* grid = h.template grid<ValueOnIndex>(i)) {
393 meta.emplace_back(h.gridSize(i), header.codec, *grid);
394 } else if (auto* grid = h.template grid<ValueOnIndexMask>(i)) {
395 meta.emplace_back(h.gridSize(i), header.codec, *grid);
396 } else if (auto* grid = h.template grid<bool>(i)) {
397 meta.emplace_back(h.gridSize(i), header.codec, *grid);
398 } else if (auto* grid = h.template grid<Rgba8>(i)) {
399 meta.emplace_back(h.gridSize(i), header.codec, *grid);
400 } else if (auto* grid = h.template grid<Fp4>(i)) {
401 meta.emplace_back(h.gridSize(i), header.codec, *grid);
402 } else if (auto* grid = h.template grid<Fp8>(i)) {
403 meta.emplace_back(h.gridSize(i), header.codec, *grid);
404 } else if (auto* grid = h.template grid<Fp16>(i)) {
405 meta.emplace_back(h.gridSize(i), header.codec, *grid);
406 } else if (auto* grid = h.template grid<FpN>(i)) {
407 meta.emplace_back(h.gridSize(i), header.codec, *grid);
408 } else if (auto* grid = h.template grid<Vec4f>(i)) {
409 meta.emplace_back(h.gridSize(i), header.codec, *grid);
410 } else if (auto* grid = h.template grid<Vec4d>(i)) {
411 meta.emplace_back(h.gridSize(i), header.codec, *grid);
412 } else {
413 std::stringstream ss;
414 ss << "nanovdb::io::Segment::add: Cannot write grid of unknown type \""<<toStr(h.gridType(i));
415 throw std::runtime_error(ss.str() + "\" to file");
416 }
417 }
418 header.gridCount += h.gridCount();
419}// Segment::add
420
421 inline void Segment::write(std::ostream& os) const
422{
423 if (header.gridCount == 0) {
424 throw std::runtime_error("Segment contains no grids");
425 } else if (!os.write(reinterpret_cast<const char*>(&header), sizeof(FileHeader))) {
426 throw std::runtime_error("Failed to write FileHeader of Segment");
427 }
428 for (auto& m : meta) m.write(os);
429}// Segment::write
430
431 inline bool Segment::read(std::istream& is)
432{
433 is.read(reinterpret_cast<char*>(&header), sizeof(FileHeader));
434 if (is.eof()) {// The EOF flag is only set once a read tries to read past the end of the file
435 is.clear(std::ios_base::eofbit);// clear eof flag so we can rewind and read again
436 return false;
437 }
438 if (!header.isValid()) {
439 // first check for byte-swapped header magic.
440 if (header.magic == reverseEndianness(NANOVDB_MAGIC_NUMBER) ||
441 header.magic == reverseEndianness(NANOVDB_MAGIC_FILE)) {
442 throw std::runtime_error("This nvdb file has reversed endianness");
443 } else {
444 throw std::runtime_error("Magic number error: This is not a valid nvdb file");
445 }
446 } else if ( !header.version.isCompatible()) {
447 std::stringstream ss;
448 Version v;
449 is.read(reinterpret_cast<char*>(&v), sizeof(Version));// read GridData::mVersion located at byte 16=sizeof(FileHeader) is stream
450 if ( v.getMajor() == NANOVDB_MAJOR_VERSION_NUMBER) {
451 ss << "This file looks like it contains a raw grid buffer and not a standard file with meta data";
452 } else if ( header.version.getMajor() < NANOVDB_MAJOR_VERSION_NUMBER) {
453 ss << "The file contains an older version of NanoVDB: " << std::string(header.version.c_str()) << "!\n\t"
454 << "Recommendation: Re-generate this NanoVDB file with this version: " << NANOVDB_MAJOR_VERSION_NUMBER << ".X of NanoVDB";
455 } else {
456 ss << "This tool was compiled against an older version of NanoVDB: " << NANOVDB_MAJOR_VERSION_NUMBER << ".X!\n\t"
457 << "Recommendation: Re-compile this tool against the newer version: " << header.version.getMajor() << ".X of NanoVDB";
458 }
459 throw std::runtime_error("An unrecoverable error in nanovdb::Segment::read:\n\tIncompatible file format: " + ss.str());
460 }
461 meta.resize(header.gridCount);
462 for (auto& m : meta) {
463 m.read(is);
464 m.version = header.version;
465 }
466 return true;
467}// Segment::read
468
469// --------------------------> writeGrid <------------------------------------
470
471template<typename BufferT>
472 void writeGrid(std::ostream& os, const GridHandle<BufferT>& handle, Codec codec)
473{
474 Segment seg(codec);
475 seg.add(handle);
476 const auto start = os.tellp();
477 seg.write(os); // write header without the correct fileSize (so it's allocated)
478 for (uint32_t i = 0; i < handle.gridCount(); ++i) {
479 seg.meta[i].fileSize = Internal::write(os, handle, codec, i);
480 }
481 os.seekp(start);
482 seg.write(os);// re-write header with the correct fileSize
483 os.seekp(0, std::ios_base::end);// skip to end
484}// writeGrid
485
486template<typename BufferT>
487 void writeGrid(const std::string& fileName, const GridHandle<BufferT>& handle, Codec codec, int verbose)
488{
489 std::ofstream os(fileName, std::ios::out | std::ios::binary | std::ios::trunc);
490 if (!os.is_open()) {
491 throw std::ios_base::failure("Unable to open file named \"" + fileName + "\" for output");
492 }
493 writeGrid<BufferT>(os, handle, codec);
494 if (verbose) {
495 std::cout << "Wrote nanovdb::Grid to file named \"" << fileName << "\"" << std::endl;
496 }
497}// writeGrid
498
499// --------------------------> writeGrids <------------------------------------
500
501template<typename BufferT = HostBuffer, template<typename...> class VecT = std::vector>
502 void writeGrids(std::ostream& os, const VecT<GridHandle<BufferT>>& handles, Codec codec = Codec::NONE)
503{
504 for (auto& h : handles) writeGrid(os, h, codec);
505}// writeGrids
506
507template<typename BufferT, template<typename...> class VecT>
508 void writeGrids(const std::string& fileName, const VecT<GridHandle<BufferT>>& handles, Codec codec, int verbose)
509{
510 std::ofstream os(fileName, std::ios::out | std::ios::binary | std::ios::trunc);
511 if (!os.is_open()) throw std::ios_base::failure("Unable to open file named \"" + fileName + "\" for output");
512 writeGrids<BufferT, VecT>(os, handles, codec);
513 if (verbose) std::cout << "Wrote " << handles.size() << " nanovdb::Grid(s) to file named \"" << fileName << "\"" << std::endl;
514}// writeGrids
515
516// --------------------------> readGrid <------------------------------------
517
518template<typename BufferT>
519 GridHandle<BufferT> readGrid(std::istream& is, int n, const BufferT& pool)
520{
521 GridHandle<BufferT> handle;
522 if (n<0) {// read all grids into the same buffer
523 try {//first try to read a raw grid buffer
524 handle.read(is, pool);
525 } catch(const std::logic_error&) {
526 Segment seg;
527 uint64_t bufferSize = 0u;
528 uint32_t gridCount = 0u, gridIndex = 0u;
529 const auto start = is.tellg();
530 while (seg.read(is)) {
531 std::streamoff skipSize = 0;
532 for (auto& m : seg.meta) {
533 ++gridCount;
534 bufferSize += m.gridSize;
535 skipSize += m.fileSize;
536 }// loop over grids in segment
537 is.seekg(skipSize, std::ios_base::cur); // skip forward from the current position
538 }// loop over segments
539 auto buffer = BufferT::create(bufferSize, &pool);
540 char *ptr = (char*)buffer.data();
541 is.seekg(start);// rewind
542 while (seg.read(is)) {
543 for (auto& m : seg.meta) {
544 Internal::read(is, ptr, m.gridSize, seg.header.codec);
545 updateGridCount((GridData*)ptr, gridIndex++, gridCount);
546 ptr += m.gridSize;
547 }// loop over grids in segment
548 }// loop over segments
549 return GridHandle<BufferT>(std::move(buffer));
550 }
551 } else {// read a specific grid
552 try {//first try to read a raw grid buffer
553 handle.read(is, uint32_t(n), pool);
554 updateGridCount((GridData*)handle.data(), 0u, 1u);
555 } catch(const std::logic_error&) {
556 Segment seg;
557 int counter = -1;
558 while (seg.read(is)) {
559 std::streamoff seek = 0;
560 for (auto& m : seg.meta) {
561 if (++counter == n) {
562 auto buffer = BufferT::create(m.gridSize, &pool);
563 Internal::read(is, buffer, seg.header.codec);
564 updateGridCount((GridData*)buffer.data(), 0u, 1u);
565 return GridHandle<BufferT>(std::move(buffer));
566 } else {
567 seek += m.fileSize;
568 }
569 }// loop over grids in segment
570 is.seekg(seek, std::ios_base::cur); // skip forward from the current position
571 }// loop over segments
572 if (n != counter) throw std::runtime_error("stream does not contain a #" + std::to_string(n) + " grid");
573 }
574 }
575 return handle;
576}// readGrid
577
578/// @brief Read the n'th grid
579template<typename BufferT>
580 GridHandle<BufferT> readGrid(const std::string& fileName, int n, int verbose, const BufferT& buffer)
581{
582 std::ifstream is(fileName, std::ios::in | std::ios::binary);
583 if (!is.is_open()) throw std::ios_base::failure("Unable to open file named \"" + fileName + "\" for input");
584 auto handle = readGrid<BufferT>(is, n, buffer);
585 if (verbose) {
586 if (n<0) {
587 std::cout << "Read all NanoGrids from the file named \"" << fileName << "\"" << std::endl;
588 } else {
589 std::cout << "Read NanoGrid # " << n << " from the file named \"" << fileName << "\"" << std::endl;
590 }
591 }
592 return handle; // is converted to r-value and return value is move constructed.
593}// readGrid
594
595/// @brief Read a specific grid from an input stream given the name of the grid
596/// @tparam BufferT Buffer type used for allocation
597/// @param is input stream from which to read the grid
598/// @param gridName string name of the (first) grid to be returned
599/// @param pool optional memory pool from which to allocate the grid buffer
600/// @return Return the first grid in the input stream with a specific name
601/// @throw std::runtime_error with no grid exists with the specified name
602template<typename BufferT>
603 GridHandle<BufferT> readGrid(std::istream& is, const std::string& gridName, const BufferT& pool)
604{
605 try {
606 GridHandle<BufferT> handle;
607 handle.read(is, gridName, pool);
608 return handle;
609 } catch(const std::logic_error&) {
610 const auto key = stringHash(gridName);
611 Segment seg;
612 while (seg.read(is)) {// loop over all segments in stream
613 std::streamoff seek = 0;
614 for (auto& m : seg.meta) {// loop over all grids in segment
615 if ((m.nameKey == 0u || m.nameKey == key) && m.gridName == gridName) { // check for hash key collision
616 auto buffer = BufferT::create(m.gridSize, &pool);
617 is.seekg(seek, std::ios_base::cur); // rewind
618 Internal::read(is, buffer, seg.header.codec);
619 updateGridCount((GridData*)buffer.data(), 0u, 1u);
620 return GridHandle<BufferT>(std::move(buffer));
621 } else {
622 seek += m.fileSize;
623 }
624 }
625 is.seekg(seek, std::ios_base::cur); // skip forward from the current position
626 }
627 }
628 throw std::runtime_error("Grid name '" + gridName + "' not found in file");
629}// readGrid
630
631/// @brief Read the first grid with a specific name
632template<typename BufferT>
633 GridHandle<BufferT> readGrid(const std::string& fileName, const std::string& gridName, int verbose, const BufferT& buffer)
634{
635 std::ifstream is(fileName, std::ios::in | std::ios::binary);
636 if (!is.is_open()) throw std::ios_base::failure("Unable to open file named \"" + fileName + "\" for input");
637 auto handle = readGrid<BufferT>(is, gridName, buffer);
638 if (verbose) {
639 if (handle) {
640 std::cout << "Read NanoGrid named \"" << gridName << "\" from the file named \"" << fileName << "\"" << std::endl;
641 } else {
642 std::cout << "File named \"" << fileName << "\" does not contain a grid named \"" + gridName + "\"" << std::endl;
643 }
644 }
645 return handle; // is converted to r-value and return value is move constructed.
646}// readGrid
647
648// --------------------------> readGrids <------------------------------------
649
650template<typename BufferT = HostBuffer, template<typename...> class VecT = std::vector>
651VecT<GridHandle<BufferT>> readGrids(std::istream& is, const BufferT& pool = BufferT())
652{
653 VecT<GridHandle<BufferT>> handles;
654 Segment seg;
655 while (seg.read(is)) {
656 uint64_t bufferSize = 0;
657 for (auto& m : seg.meta) bufferSize += m.gridSize;
658 auto buffer = BufferT::create(bufferSize, &pool);
659 uint64_t bufferOffset = 0;
660 for (uint16_t i = 0; i < seg.header.gridCount; ++i) {
661 auto *data = reinterpret_cast<GridData*>(buffer.data() + bufferOffset);
662 Internal::read(is, (char*)data, seg.meta[i].gridSize, seg.header.codec);
663 updateGridCount(data, uint32_t(i), uint32_t(seg.header.gridCount));
664 bufferOffset += seg.meta[i].gridSize;
665 }// loop over grids in segment
666 handles.emplace_back(std::move(buffer)); // force move copy assignment
667 }// loop over segments
668 return handles; // is converted to r-value and return value is move constructed.
669}// readGrids
670
671/// @brief Read all the grids
672template<typename BufferT, template<typename...> class VecT>
673VecT<GridHandle<BufferT>> readGrids(const std::string& fileName, int verbose, const BufferT& buffer)
674{
675 std::ifstream is(fileName, std::ios::in | std::ios::binary);
676 if (!is.is_open()) throw std::ios_base::failure("Unable to open file named \"" + fileName + "\" for input");
677 auto handles = readGrids<BufferT, VecT>(is, buffer);
678 if (verbose) std::cout << "Read " << handles.size() << " NanoGrid(s) from the file named \"" << fileName << "\"" << std::endl;
679 return handles; // is converted to r-value and return value is move constructed.
680}// readGrids
681
682// --------------------------> readGridMetaData <------------------------------------
683
684 inline std::vector<FileGridMetaData> readGridMetaData(const std::string& fileName)
685{
686 std::ifstream is(fileName, std::ios::in | std::ios::binary);
687 if (!is.is_open()) throw std::ios_base::failure("Unable to open file named \"" + fileName + "\" for input");
688 return readGridMetaData(is); // is converted to r-value and return value is move constructed.
689}// readGridMetaData
690
691 inline std::vector<FileGridMetaData> readGridMetaData(std::istream& is)
692{
693 Segment seg;
694 std::vector<FileGridMetaData> meta;
695 try {
696 GridHandle<> handle;// if stream contains a raw grid buffer we unfortunately have to load everything
697 handle.read(is);
698 seg.add(handle);
699 meta = std::move(seg.meta);
700 } catch(const std::logic_error&) {
701 while (seg.read(is)) {
702 std::streamoff skip = 0;
703 for (auto& m : seg.meta) {
704 meta.push_back(m);
705 skip += m.fileSize;
706 }// loop over grid meta data in segment
707 is.seekg(skip, std::ios_base::cur);
708 }// loop over segments
709 }
710 return meta; // is converted to r-value and return value is move constructed.
711}// readGridMetaData
712
713// --------------------------> hasGrid <------------------------------------
714
715 inline bool hasGrid(const std::string& fileName, const std::string& gridName)
716{
717 std::ifstream is(fileName, std::ios::in | std::ios::binary);
718 if (!is.is_open()) throw std::ios_base::failure("Unable to open file named \"" + fileName + "\" for input");
719 return hasGrid(is, gridName);
720}// hasGrid
721
722 inline bool hasGrid(std::istream& is, const std::string& gridName)
723{
724 const auto key = stringHash(gridName);
725 Segment seg;
726 while (seg.read(is)) {
727 std::streamoff seek = 0;
728 for (auto& m : seg.meta) {
729 if (m.nameKey == key && m.gridName == gridName) return true; // check for hash key collision
730 seek += m.fileSize;
731 }// loop over grid meta data in segment
732 is.seekg(seek, std::ios_base::cur);
733 }// loop over segments
734 return false;
735}// hasGrid
736
737// --------------------------> stringHash <------------------------------------
738
739 inline uint64_t stringHash(const char* c_str)
740{
741 uint64_t hash = 0;// zero is returned when cstr = nullptr or "\0"
742 if (c_str) {
743 for (auto* str = reinterpret_cast<const unsigned char*>(c_str); *str; ++str) {
744 uint64_t overflow = hash >> (64 - 8);
745 hash *= 67; // Next-ish prime after 26 + 26 + 10
746 hash += *str + overflow;
747 }
748 }
749 return hash;
750}// stringHash
751
752} // namespace io
753
754template<typename T>
755 inline std::ostream&
756operator<<(std::ostream& os, const BBox<Vec3<T>>& b)
757{
758 os << "(" << b[0][0] << "," << b[0][1] << "," << b[0][2] << ") -> "
759 << "(" << b[1][0] << "," << b[1][1] << "," << b[1][2] << ")";
760 return os;
761}
762
763 inline std::ostream&
764operator<<(std::ostream& os, const CoordBBox& b)
765{
766 os << "(" << b[0][0] << "," << b[0][1] << "," << b[0][2] << ") -> "
767 << "(" << b[1][0] << "," << b[1][1] << "," << b[1][2] << ")";
768 return os;
769}
770
771 inline std::ostream&
772operator<<(std::ostream& os, const Coord& ijk)
773{
774 os << "(" << ijk[0] << "," << ijk[1] << "," << ijk[2] << ")";
775 return os;
776}
777
778template<typename T>
779 inline std::ostream&
780operator<<(std::ostream& os, const Vec3<T>& v)
781{
782 os << "(" << v[0] << "," << v[1] << "," << v[2] << ")";
783 return os;
784}
785
786template<typename T>
787 inline std::ostream&
788operator<<(std::ostream& os, const Vec4<T>& v)
789{
790 os << "(" << v[0] << "," << v[1] << "," << v[2] << "," << v[3] << ")";
791 return os;
792}
793
794} // namespace nanovdb
795
796#endif // NANOVDB_IO_H_HAS_BEEN_INCLUDED
GridChecksum.h
Computes a pair of 32bit checksums, of a Grid, by means of Cyclic Redundancy Check (CRC)
GridHandle.h
Defines GridHandle, which manages a host, and possibly a device, memory buffer containing one or more...
NanoVDB.h
Implements a light-weight self-contained VDB data-structure in a single file! In other words,...
NANOVDB_MAGIC_FILE
#define NANOVDB_MAGIC_FILE
Definition NanoVDB.h:128
NANOVDB_MAJOR_VERSION_NUMBER
#define NANOVDB_MAJOR_VERSION_NUMBER
Definition NanoVDB.h:133
NANOVDB_MAGIC_NUMBER
#define NANOVDB_MAGIC_NUMBER
Definition NanoVDB.h:126
nanovdb::Coord
Signed (i, j, k) 32-bit integer coordinate class, similar to openvdb::math::Coord.
Definition NanoVDB.h:1302
nanovdb::GridHandle
This class serves to manage a buffer containing one or more NanoVDB Grids.
Definition GridHandle.h:38
nanovdb::GridHandle::gridSize
uint64_t gridSize(uint32_t n=0) const
Return the grid size of the n'th grid in this GridHandle.
Definition GridHandle.h:185
nanovdb::GridHandle::gridType
GridType gridType(uint32_t n=0) const
Return the GridType of the n'th grid in this GridHandle.
Definition GridHandle.h:190
nanovdb::GridHandle::data
uint8_t * data()
Returns a non-const pointer to the data.
Definition GridHandle.h:103
nanovdb::GridHandle::gridCount
uint32_t gridCount() const
Return the total number of grids contained in this buffer.
Definition GridHandle.h:180
nanovdb::GridHandle::read
void read(std::istream &is, const BufferT &pool=BufferT())
Read an entire raw grid buffer from an input stream.
Definition GridHandle.h:363
nanovdb::Grid
Highest level of the data structure. Contains a tree and a world->index transform (that currently onl...
Definition NanoVDB.h:3699
nanovdb::HostBuffer
This is a buffer that contains a shared or private pool to either externally or internally managed ho...
Definition HostBuffer.h:115
nanovdb::Vec3
A simple vector class with three components, similar to openvdb::math::Vec3.
Definition NanoVDB.h:1530
nanovdb::Vec4
A simple vector class with four components, similar to openvdb::math::Vec4.
Definition NanoVDB.h:1728
nanovdb::Version
Bit-compacted representation of all three version numbers.
Definition NanoVDB.h:948
nanovdb::Version::c_str
const char * c_str() const
Definition NanoVDB.h:981
nanovdb::Version::getMajor
uint32_t getMajor() const
Definition NanoVDB.h:971
nanovdb::Version::isCompatible
bool isCompatible() const
Definition NanoVDB.h:974
hash
__hostdev__ uint32_t hash(uint32_t x)
Definition common.h:14
nanovdb::io::Internal::read
static void read(std::istream &is, BufferT &buffer, Codec codec)
nanovdb::io::Internal::write
static fileSize_t write(std::ostream &os, const GridHandle< BufferT > &handle, Codec codec, uint32_t n)
nanovdb::io::Internal::MAX_SIZE
static constexpr fileSize_t MAX_SIZE
Definition IO.h:117
nanovdb::io::readGrids
VecT< GridHandle< BufferT > > readGrids(const std::string &fileName, int verbose=0, const BufferT &buffer=BufferT())
Read all the grids in the file and return them as a vector of multiple GridHandles,...
Definition IO.h:673
nanovdb::io::writeGrids
void writeGrids(const std::string &fileName, const VecT< GridHandle< BufferT > > &handles, Codec codec=Codec::NONE, int verbose=0)
Write multiple grids to file (over-writing existing content of the file)
Definition IO.h:508
nanovdb::io::fileSize_t
uint64_t fileSize_t
Definition IO.h:111
nanovdb::io::readGridMetaData
std::vector< FileGridMetaData > readGridMetaData(const std::string &fileName)
Reads and returns a vector of meta data for all the grids found in the specified file.
Definition IO.h:684
nanovdb::io::stringHash
uint64_t stringHash(const char *cstr)
Standard hash function to use on strings; std::hash may vary by platform/implementation and is know t...
Definition IO.h:739
nanovdb::io::reverseEndianness
uint64_t reverseEndianness(uint64_t val)
Return a uint64_t with its bytes reversed so we can check for endianness.
Definition IO.h:136
nanovdb::io::writeGrid
void writeGrid(const std::string &fileName, const GridHandle< BufferT > &handle, io::Codec codec=io::Codec::NONE, int verbose=0)
Write a single grid to file (over-writing existing content of the file)
Definition IO.h:487
nanovdb::io::readGrid
GridHandle< BufferT > readGrid(const std::string &fileName, int n=0, int verbose=0, const BufferT &buffer=BufferT())
Read and return one or all grids from a file into a single GridHandle.
Definition IO.h:580
nanovdb::io::Codec
Codec
Define compression codecs.
Definition NanoVDB.h:7839
nanovdb::io::Codec::ZIP
@ ZIP
nanovdb::io::Codec::BLOSC
@ BLOSC
nanovdb::io::Codec::NONE
@ NONE
nanovdb::io::hasGrid
bool hasGrid(const std::string &fileName, const std::string &gridName)
Return true if the file contains a grid with the specified name.
Definition IO.h:715
nanovdb
Convert a base-pointer to an openvdb grid, denoted srcGrid, to a nanovdb grid of the same type,...
Definition NanoVDB.h:247
nanovdb::toStr
const char * toStr(GridType gridType)
Maps a GridType to a c-string.
Definition NanoVDB.h:349
nanovdb::updateGridCount
bool updateGridCount(GridData *data, uint32_t gridIndex, uint32_t gridCount)
Updates the ground index and count, as well as the partial checksum if needed.
Definition GridChecksum.h:447
nanovdb::BBox
Definition NanoVDB.h:2295
nanovdb::GridData
Struct with all the member data of the Grid (useful during serialization of an openvdb grid)
Definition NanoVDB.h:3512
nanovdb::io::FileGridMetaData
This class defines the meta data stored for each grid in a segment.
Definition IO.h:152
nanovdb::io::FileGridMetaData::write
void write(std::ostream &os) const
Definition IO.h:341
nanovdb::io::FileGridMetaData::memUsage
uint64_t memUsage() const
Definition IO.h:160
nanovdb::io::FileGridMetaData::gridName
std::string gridName
Definition IO.h:154
nanovdb::io::FileGridMetaData::FileGridMetaData
FileGridMetaData()
Definition IO.h:157
nanovdb::io::FileGridMetaData::read
void read(std::istream &is)
Definition IO.h:348
nanovdb::io::FileHeader
Data encoded at the head of each segment of a file or stream.
Definition NanoVDB.h:7848
nanovdb::io::FileHeader::gridCount
uint16_t gridCount
Definition NanoVDB.h:7851
nanovdb::io::FileHeader::isValid
bool isValid() const
Definition NanoVDB.h:7853
nanovdb::io::FileHeader::codec
Codec codec
Definition NanoVDB.h:7852
nanovdb::io::FileHeader::magic
uint64_t magic
Definition NanoVDB.h:7849
nanovdb::io::FileHeader::version
Version version
Definition NanoVDB.h:7850
nanovdb::io::FileMetaData
Definition NanoVDB.h:7874
nanovdb::io::FileMetaData::nameSize
uint32_t nameSize
Definition NanoVDB.h:7881
nanovdb::io::Segment
This class defines all the data stored in segment of a file.
Definition IO.h:168
nanovdb::io::Segment::read
bool read(std::istream &is)
Definition IO.h:431
nanovdb::io::Segment::add
void add(const GridHandle< BufferT > &h)
Definition IO.h:367
nanovdb::io::Segment::write
void write(std::ostream &os) const
Definition IO.h:421
nanovdb::io::Segment::meta
std::vector< FileGridMetaData > meta
Definition IO.h:172
nanovdb::io::Segment::memUsage
uint64_t memUsage() const
Definition IO.h:359
nanovdb::io::Segment::Segment
Segment(Codec c=Codec::NONE)
Definition IO.h:173
nanovdb::io::Segment::header
FileHeader header
Definition IO.h:171