diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..c6c8b36 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,9 @@ +root = true + +[*] +indent_style = space +indent_size = 2 +end_of_line = lf +charset = utf-8 +trim_trailing_whitespace = true +insert_final_newline = true diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..348c2bb --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,13 @@ +# To get started with Dependabot version updates, you'll need to specify which +# package ecosystems to update and where the package manifests are located. +# Please see the documentation for all configuration options: +# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file + +version: 2 +updates: + - package-ecosystem: "github-actions" + # Workflow files stored in the default location of `.github/workflows` + # You don't need to specify `/.github/workflows` for `directory`. You can use `directory: "/"`. + directory: "/" + schedule: + interval: "monthly" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 41309bd..71493b3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -5,6 +5,7 @@ on: branches: [ main ] pull_request: branches: [ main ] + workflow_dispatch: jobs: build_and_test: @@ -12,13 +13,13 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Setup emsdk - uses: mymindstorm/setup-emsdk@v11 + uses: mymindstorm/setup-emsdk@v14 with: # Make sure to set a version number! - version: 3.1.10 + version: 3.1.68 # This is the name of the cache folder. # The cache folder will be placed in the build directory, # so make sure it doesn't conflict with anything! @@ -26,14 +27,19 @@ jobs: - name: build wasm run: make - - uses: actions/setup-node@v3 + - uses: actions/setup-node@v4 with: - node-version: 16 + node-version: 20 - name: build js run: | npm i npm run build + - name: check environment + run: | + pwd + ls -al ./test + - name: test run: npm test diff --git a/.gitignore b/.gitignore index ca2912c..3279083 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ build** .vscode dist** h5wasm*.tgz +experiment** diff --git a/.npmignore b/.npmignore index 2d62bd2..188182e 100644 --- a/.npmignore +++ b/.npmignore @@ -3,3 +3,4 @@ build** .cache .vscode h5wasm*.tgz +experiment** diff --git a/CHANGELOG.md b/CHANGELOG.md index 8206bc2..4aad82c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,325 @@ # Changelog +## v0.7.9 2025-01-09 +### Fixed +- Added `total_size` to metadata for VLEN datasets before further processing, fixes #89 +## v0.7.8 2024-09-06 +### Added +- Add `close_file` method to Module API: `close_file(file_id: bigint): number;` +### Fixed +- Fix accessing attributes of committed datatype with `my_datafile.attrs`. +- Fix calling `get_attribute_names` method of Module API on committed datatype. +### Changed +- Mark optional parameters as such in the TypeScript declarations of the following `H5Module` methods: `open`, `create_dataset`, `create_group`, `create_vlen_str_dataset` and `get_keys_vector`. +## v0.7.7 2024-08-28 +### Added +- A method to read metadata from committed datatypes in an HDF5 file + - `Datatype.metadata` is a getter that returns the dtype metadata, in the same format (`Metadata` TypeScript interface) returned from `Dataset.metadata` +- Enable "track order" mode of HDF5 for attributes of `Dataset`, and (fields and attributes) of `Group`. When `track_order == true` for a `Group`, both fields and attributes keep their insertion order on read. For a `Dataset`, the attributes keep their insertion order when the flag is set. + + Note that the flag is not inherited, i.e. if an `h5wasm.Group` object is created with `track_order = true`, its sub-groups and contained `Dataset` objects will not have the flag enabled by default. The flag is added to the `File` constructor just to track order of immediate children of the root group. + + Changes three signatures in the `h5wasm` API: + - `h5wasm.File(filename: string, mode: ACCESS_MODESTRING = "r", track_order: boolean = false)` + - `h5wasm.Group.create_group(name: string, track_order: boolean = false)` + - ``` + h5wasm.Group.create_dataset(args: { + name: string, + data: GuessableDataTypes, + shape?: number[] | null, + dtype?: string | null, + maxshape?: (number | null)[] | null, + chunks?: number[] | null, + compression?: (number | 'gzip'), + compression_opts?: number | number[], + track_order?: boolean, + }) + ``` + + Also adds one method to the Module API: + - `open(filename: string, mode?: number, track_order?: boolean): bigint;` + (the default arguments are `mode = H5F_ACC_RDONLY` and `track_order = false`) + + and modifies the signatures of three existing methods in the Module API: + - `create_dataset` + - `create_group` + - `create_vlen_str_dataset` +## v0.7.6 2024-08-06 +### Added + - Full support for reading VLEN datasets (numeric variable-length datasets), fixing #78 + + - previously the library made no attempt to read the stored values and returned the raw bytes (corresponding to intermediate`[length, data_pointer]` pairs) + - now returns an `Array` of `TypedArray` objects containing the stored values (or an `Array` of `Array` objects if `json_value` is requested) + - `vlen_type?: Metadata` key added to `Metadata` interface, and populated with type information about the base type in a VLEN dataset +### Changed + - Retrieve `value` of boolean dataset as `Int8Array` instead of plain JS boolean array. To retrieve a plain JS boolean array, use `json_value` instead: + + ```ts + // v0.7.5 and earlier + bool_dset.value; // -> [false, true] + bool_dset.json_value; // -> [false, true] + + // v0.7.6 onwards + bool_dset.value; // -> Int8Array(2) [0, 1] + bool_dset.json_value; // -> [false, true] + ``` +## v0.7.5 2024-06-03 +### Added + - added `virtual_sources?: { file_name: string, dset_name: string }` to `Dataset.metadata` when dataset is virtual. + - added `strpad` enum to metadata (type of string padding) - PR #72 @axelboc +## v0.7.4 2024-04-03 +### Fixed + - added missing `.js` extension in type imports + ( see issue comment https://github.com/usnistgov/h5wasm/issues/31#issuecomment-2032301488 and fix in #71 ) +## v0.7.3 2024-04-02 +### Changed + - removed references to `import.meta` in the compiled files, using `USE_ES6_IMPORT_META=0` compilation flag for emscripten. It shouldn't matter in any case, as it is compiled in single-file mode and there are no external imports at runtime. + + *Note that this enables bundling `h5wasm` into a worker without requiring `{type: "module"}`* +## v0.7.2 2024-02-29 +### Added + - new optional HDF5 error handler that throws Javascript error (old handler printed **HDF5-DIAG** error strings using `console.error()`, but did not throw a JS `Error`) + - when activated, the thrown `Error.message` is the same string that was previously printed to the console + +To enable: +```javascript +const Module = await h5wasm.ready; +Module.activate_throwing_error_handler(); +// to deactivate, restoring console.error behavior: +// Module.deactivate_throwing_error_handler(); +``` +## v0.7.1 2023-12-18 +### Added + - Support for object and region references + - create object reference with `Dataset.create_reference()` or `Group.create_reference()` + - dereference against any object in the same file or the root `File` object with e.g. `obj = File.dereference(ref)` + - create region references with `Dataset.create_region_reference([[start_axis_0, end_axis_0, step_axis_0], [start_axis_1, ...], ...])` + - retrieve values of region reference with `File.dereference(region_ref)` + - store references in a dataset using `dtype='Reference'` or `dtype='RegionReference'` or if data is an array of references, it will be guessed + - retrieve references from dataset using `slice` or `value` as usual, and dereference as above. +## v0.7.0 2023-12-05 +### Changed + - `package.json` modified so that exports are defined in a way that works better with bundlers + - (should now work with e.g. NextJS) + - **Breaking change** the `nodejs` import is now 'h5wasm/node' rather than 'h5wasm'. The `nodejs` examples in the README have been updated, as well as the tests. + - tests modified to use smaller compressed array (reduce package size) +## v0.6.10 2023-11-09 +### Added + - export more symbols to support `blosc2 ` filter +## v0.6.9 2023-11-06 +### Fixed + - added missing FileSystem API function `mkdirTree` to Emscripten typescript interface +### Added + - Functions for working with dimension scales in typescript interface: +```typescript +// convert dataset to dimension scale: +Dataset.make_scale(scale_name: string) +// attach a dimension scale to the "index" dimension of this dataset: +Dataset.attach_scale(index: number, scale_dset_path: string) +// detach a dimension scale from "index" dimension +Dataset.detach_scale(index: number, scale_dset_path: string) +// get full paths to all datasets that are attached as dimension scales +// to the specified dimension (at "index") of this dataset: +Dataset.get_attached_scales(index: number) +// if this dataset is a dimension scale, returns name as string +// (returns empty string if no name defined, but it is a dimension scale) +// else returns null if it is not set as a dimension scale: +Dataset.get_scale_name() +``` + - Functions for working with dimension labels (not related to dimension scales) +```typescript +// label dimension at "index" of this dataset with string "label": +Dataset.set_dimension_label(index: number, label: string) +// fetch labels for all dimensions of this dataset (null if label not defined): +Dataset.get_dimension_labels() +``` + +## v0.6.8 2023-11-02 +### Added + - Functions for creating, attaching and detaching dimension scales (no support for reading yet) + - h5wasm.Module.set_scale(file_id: bigint, dataset_name: string, dim_name: string) -> number // error code, zero on success + - h5wasm.Module.attach_scale(file_id: bigint, target_dataset_name: string, dimscale_dataset_name: string, index: number) -> number + - h5wasm.Module.detach_scale(file_id: bigint, target_dataset_name: string, dimscale_dataset_name: string, index: number) -> number +Usage: +```js +import h5wasm from "h5wasm"; +await h5wasm.ready; + +f = new h5wasm.File('dimscales.h5', 'w'); +f.create_dataset({name: "data", data: [0,1,2,3,4,5], shape: [2,3], dtype: ' 0 on success +h5wasm.Module.attach_scale(f.file_id, 'data', 'dimscale', 1); // -> 0 on success +f.close(); +``` +## v0.6.7 2023-10-24 +### Added + - Utility functions on the Module for manipulating/reading the plugin search path +```ts +get_plugin_search_paths(): string[], +insert_plugin_search_path(search_path: string, index: number): number, +remove_plugin_search_path(index: number): number, +``` +e.g. +```js +import h5wasm from "h5wasm"; +await h5wasm.ready; + +h5wasm.Module.get_plugin_search_paths(); +// [ '/usr/local/hdf5/lib/plugin' ] +h5wasm.Module.insert_plugin_search_path('/tmp/h5wasm-plugins', 0); +h5wasm.Module.get_plugin_search_paths(); +// [ '/tmp/h5wasm-plugins', '/usr/local/hdf5/lib/plugin' ] +h5wasm.Module.remove_plugin_search_path(1); +// 0, success +> h5wasm.Module.get_plugin_search_paths() +// [ '/tmp/h5wasm-plugins' ] +``` +## v0.6.6 2023-10-23 +### Added + - outputs filter settings when querying `Dataset.filters` + - e.g. for default gzip compression from h5py, returns `{id: 1, name: 'deflate', cd_values: [4]}` +## v0.6.4 2023-10-23 +### Added + - builtin support for `SZIP` compression (in addition to previously-included `GZIP`) + - a lot of extra symbols to support plugins `zstd`, `zfp`, `bzip2`, `SZ3`, etc. +### Changed + - uses HDF5 version 1.14.2 (instead of 1.12.2) +## v0.6.3 2023-09-15 +### Added + - extra symbols used by the LZ4 plugin added to EXPORTED_FUNCTIONS: + - from : `htonl`, `htons`, `ntohl`, `ntohs` + - from HDF5: `H5allocate_memory`, `H5free_memory` + - node.js library compiled as MAIN_MODULE, allowing plugin use (if they are installed in `/usr/local/hdf5/lib/plugin`) +## v0.6.2 2023-08-28 +### Added + - From PR #59 (thanks to @TheLartians !): allows slicing datasets with a specified step size. This is extremely useful if we want to query a high resolution dataset using a lower sample rate than originally recorded, e.g. for performance / bandwidth reasons. Also can be used with `Dataset.write_slice` + +Example: + +```ts + const dset = write_file.create_dataset({ + name: "data", + data: [0,1,2,3,4,5,6,7,8,9], + shape: [10], + dtype: " [0,2,4,6,8] + + dset.write_slice([[null, 7, 2]], [-2,-3,-4,-5]); + dset.value; // -> Float32Array(10) [-2, 1, -3, 3, -4, 5, -5, 7, 8, 9]; +``` + +## v0.6.1 2023-07-13 +### Fixed + - memory error from double-free when creating vlen str datasets (no need to call H5Treclaim when memory for vector will be automatically garbage-collected) + +## v0.6.0 2023-07-03 +### Added + - `compression: number | 'gzip'` and `compression_opts: number[]` arguments to `create_dataset`. + - **Must specify chunks when using `compression`** + - **Can not specify `compression` with VLEN datasets** + - if `compression` is supplied as a number without `compression_opts`, the 'gzip' (DEFLATE, filter_id=1) filter is applied, and the compression level used is taken from the value of `compression`. An integer value of 0-9 is required for the compression level of the 'gzip' filter. + - if `compression` and `compression_opts` are supplied, the `compression` value is passed as the filter_id (or 1 if 'gzip' is supplied), and the value of `compression_opts` is promoted to a list if it is a number, or passed as-is if it is a list. *Use both compression (numeric filter_id) and compression_opts when specifying any filter other than 'gzip', e.g. with a filter plugin* + - if `compression === 'gzip'` and `compression_opts === undefined`, the default gzip compression level of 4 is applied. + +#### Example usage: + ```js + // short form: uses default compression (DEFLATE, filter_id=1) + // with compression level specified (here, 9) + Group.create_dataset({..., compression=9}); + + // specify gzip and compression level: + Group.create_dataset({..., compression='gzip', compression_opts=[4]}); + + // specify another filter, e.g. BLOSC plugin: + Group.create_dataset({..., compression=32001, compression_opts=[]}); + ``` + +### Changed + - **BREAKING**: for `create_dataset`, all arguments are supplied in a single object, as + ```ts + Group.create_dataset(args: { + name: string, + data: GuessableDataTypes, + shape?: number[] | null, + dtype?: string | null, + maxshape?: (number | null)[] | null, + chunks?: number[] | null, + compression?: (number | 'gzip'), + compression_opts?: number | number[] + }): Dataset {} + ``` +## v0.5.2 2023-05-17 +(v0.5.1 release was removed as incomplete) +### Added + - Support TS 5's moduleResolution: "bundler" (thanks, @axelboc) +## v0.5.0 2023-05-15 +### Fixed + - with emscripten >= 3.1.28, the shim for ES6 builds in nodejs is no longer needed (see https://github.com/emscripten-core/emscripten/pull/17915) + - added `malloc` and `free` to the list of explicit exports for the `Module` (newer emscripten was optimizing them away) +### Changed + - **POSSIBLY BREAKING**: building as MAIN_MODULE=2 with `POSITION_INDEPENDENT_CODE ON` (to allow dynamic linking) + - Using newer HDF5 version 1.12.2 libraries + - compiled using Emscripten 3.1.28 + - simplified imports for tests to use "h5wasm" directly +### Added + - Plugins can now be used if they are compiled as SIDE_MODULE, by loading them into the expected plugin folder `/usr/local/hdf5/lib/plugin` in the emscripten virtual file system (might need the name of the plugin file to end in .so, even if it is WASM) + +## v0.4.11 2023-04-19 +### Fixed + - all datasets and attributes are read out in little-endian order (closes #49) +### Added + - New method to overwrite slice of an existing dataset: `Dataset.write_slice(ranges: Array>, data: any): void;` + - Method to delete an attribute from a dataset or group (can be used to update an attribute by deleting it then re-creating it) + - `Dataset.delete_attribute(name: str): number` (returns non-zero value if it fails for some reason) + - `Group.delete_attribute(name: str): number` + - Ability to specify chunks and maxshape when creating dataset (to make a resizable dataset): `Group.create_dataset(name: string, data: GuessableDataTypes, shape?: number[] | null, dtype?: string | null, maxshape?: (number | null)[] | null, chunks?: number[] | null): Dataset` + - New method to resize datasets (only works if chunks and maxshape were defined): `Dataset.resize(new_shape: number[]): void` + ### Changed + - Metadata now includes `chunks: Array | null,` information and `maxshape: Array | null` + +## v0.4.10 2023-02-19 +### Added + - Group.paths(): string[]; // returns a list of all link paths found below the group in the tree. (Use on root object to get all paths in the file) +## v0.4.9 2022-12-21 +### Added + - Group.create_soft_link(target: string, name: string): number; // creates a soft link in a group with name: name (target must be absolute path) + - Group.create_hard_link(target:string, name: string): number; // + - Group.create_external_link(file_name: string, target: string, name: string): number; + +All of these return non-zero values on error. To create links with absolute path, just use e.g. File.create_soft_link(target_path, link_path); +## v0.4.8 2022-12-05 +### Added + - IIFE build at `./dist/iife/h5wasm.js`, to support use in Firefox workers (which don't currently support ESM) + - WORKERFS support in ESM/IIFE builds, for loading local files with random access instead of copying whole file into memory + +## v0.4.7 2022-11-01 +### Added + - basic support for reading datatypes from files (see PR #34) + - basic filter information made available (see PR #35) +## v0.4.6 2022-08-04 +### Changed + - removed ```Dataset.auto_refresh``` and replaced with ```.refresh()``` method (allows consistent access to metadata and data between refreshes) +## v0.4.5 2022-08-04 +### Fixed + - H5Create should only be called with access modes H5F_ACC_TRUNC (w) and H5F_ACC_EXCL (x) +### Added + - support for SWMR read with refresh on a dataset: e.g. +```js +const file = new hdf5.File("swmr.h5", "Sr"); +let ds=file.get("data"); +ds.auto_refresh=true; +ds.shape; +// returns 12 +ds.shape; +// returns 16 because dataset was updated with SWMR write +ds.value +// has size=16 +``` +## v0.4.4 2022-05-25 +### Fixed + - error in ```isIterable``` when called on non-object (affects ```to_array``` method) ## v0.4.3 2022-05-24 ### Added - ```to_array``` method on Dataset and Attribute classes: returns nested array of values with dimensions matching ```shape```. Auto-converts all values to JSON-compatible types (BigInt -> Number, TypedArray -> Array) @@ -75,7 +396,7 @@ ## v0.2.1 2022-02-04 ### Fixed - Writing of Float64 datasets was broken, and is now working - - Guessed shape of simple string values is fixed so that this will work: + - Guessed shape of simple string values is fixed so that this will work: ```js f.create_attribute("my_attr", "a string value"); ``` @@ -87,7 +408,7 @@ - Support for reading and processing HDF5 Array datatype - esm and nodejs modules both offer ```ready``` Promise, which can be awaited before working with the module (addresses #5) - minimal set of tests (```npm test```) - + ### Changed - **POSSIBLY BREAKING**: local paths to modules are changed - esm: ./dist/esm/hdf5_hl.js @@ -121,7 +442,7 @@ ### Changed - added documentation for finding libhdf5.a -### Fixed +### Fixed - documentation for creating new File object in nodejs - checking for name existing before getting type in wasm lib @@ -130,4 +451,4 @@ - download/upload helpers for web (file_handlers.js) ### Changed - - use ZLIB from emscripten ports \ No newline at end of file + - use ZLIB from emscripten ports diff --git a/CMakeLists.txt b/CMakeLists.txt index 254522f..b346880 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.14) +cmake_minimum_required(VERSION 3.24) include(FetchContent) project(H5WASM @@ -6,40 +6,73 @@ project(H5WASM LANGUAGES CXX C ) +set (BASE_URL "https://github.com/usnistgov/libhdf5-wasm/releases/download/v0.4.6_3.1.68" CACHE STRING "") +# set (BASE_URL "$ENV{HOME}/dev/libhdf5-wasm" CACHE STRING "") + FetchContent_Declare( libhdf5-wasm -# URL file:///home/brian/dev/libhdf5-wasm/libhdf5-wasm.tar.gz - URL https://github.com/usnistgov/libhdf5-wasm/releases/download/v0.1.1/libhdf5-1_12_1-wasm.tar.gz - URL_HASH SHA256=e9bb11d89c4f26fa79b9cf1dab6159640c7b184ebf00dc97b098cd4f6de49bfe + URL ${BASE_URL}/HDF5-1.14.6-Emscripten.tar.gz + URL_HASH SHA256=0c5de36a3c81e3854e57593c55274d89a43db3f8a7bbe6a0d9d5c560e9c222b1 ) -FetchContent_MakeAvailable(libhdf5-wasm) +if (NOT libhdf5-wasm_POPULATED) + FetchContent_MakeAvailable(libhdf5-wasm) +endif() + +set(HDF5_DIR ${libhdf5-wasm_SOURCE_DIR}/cmake) +find_package(HDF5 REQUIRED CONFIG) add_executable(hdf5_util src/hdf5_util.cc) -target_link_libraries(hdf5_util hdf5-wasm) +target_link_libraries(hdf5_util hdf5_hl-static) + +set (EXPORTED_FUNCTIONS) +list (APPEND EXPORTED_FUNCTIONS + H5Fopen H5Fclose H5Fcreate H5open + malloc calloc free memset memcpy memmove + htonl htons ntohl ntohs + H5allocate_memory H5free_memory + pthread_mutex_init posix_memalign strcmp getenv + stdin stdout stderr + H5Epush2 siprintf getTempRet0 __wasm_setjmp + H5E_ERR_CLS_g H5E_PLINE_g H5E_CANTINIT_g H5E_CANTGET_g H5E_CANTFILTER_g + H5E_BADTYPE_g H5E_BADVALUE_g H5E_ARGS_g H5E_CALLBACK_g H5E_CANTREGISTER_g + H5E_RESOURCE_g H5E_NOSPACE_g H5E_OVERFLOW_g H5E_READERROR_g + H5T_NATIVE_UINT_g H5T_STD_U32BE_g H5T_STD_U32LE_g H5T_NATIVE_UINT32_g + H5T_STD_U64BE_g H5T_NATIVE_UINT64_g H5T_STD_U64LE_g H5P_CLS_DATASET_CREATE_ID_g + ldexpf __THREW__ __threwValue +) +list (TRANSFORM EXPORTED_FUNCTIONS PREPEND "'_") +list (TRANSFORM EXPORTED_FUNCTIONS APPEND "'") +list(JOIN EXPORTED_FUNCTIONS ", " EXPORTED_FUNCTIONS_STRING) + # Optional flags to set when building your project set_target_properties(hdf5_util PROPERTIES LINK_FLAGS "-O3 --bind \ -lidbfs.js \ + -lworkerfs.js \ + -s MAIN_MODULE=2 \ -s ALLOW_TABLE_GROWTH=1 \ -s ALLOW_MEMORY_GROWTH=1 \ -s WASM_BIGINT \ -s ENVIRONMENT=web,worker \ -s SINGLE_FILE \ -s EXPORT_ES6=1 \ + -s USE_ES6_IMPORT_META=0 \ -s FORCE_FILESYSTEM=1 \ - -s USE_ZLIB=1 \ -s EXPORTED_RUNTIME_METHODS=\"['ccall', 'cwrap', 'FS', 'AsciiToString', 'UTF8ToString']\" \ - -s EXPORTED_FUNCTIONS=\"['_H5Fopen', '_H5Fclose', '_H5Fcreate']\"" + -s EXPORTED_FUNCTIONS=\"${EXPORTED_FUNCTIONS_STRING}\"" RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/dist/esm RUNTIME_OUTPUT_NAME hdf5_util + POSITION_INDEPENDENT_CODE ON ) add_executable(hdf5_util_node src/hdf5_util.cc) -target_link_libraries(hdf5_util_node hdf5-wasm) +target_link_libraries(hdf5_util_node hdf5_hl-static) set_target_properties(hdf5_util_node PROPERTIES LINK_FLAGS "-O3 --bind \ - --extern-pre-js ${CMAKE_CURRENT_SOURCE_DIR}/src/node_esm_shim.js \ + -s MAIN_MODULE=2 \ + -s NODEJS_CATCH_EXIT=0 \ + -s NODEJS_CATCH_REJECTION=0 \ -s ALLOW_TABLE_GROWTH=1 \ -s ALLOW_MEMORY_GROWTH=1 \ -s WASM_BIGINT \ @@ -48,10 +81,10 @@ set_target_properties(hdf5_util_node PROPERTIES -s ENVIRONMENT=node \ -s SINGLE_FILE \ -s EXPORT_ES6=1 \ - -s USE_ZLIB=1 \ -s ASSERTIONS=1 \ -s EXPORTED_RUNTIME_METHODS=\"['ccall', 'cwrap', 'FS', 'AsciiToString', 'UTF8ToString']\" \ - -s EXPORTED_FUNCTIONS=\"['_H5Fopen', '_H5Fclose', '_H5Fcreate']\"" + -s EXPORTED_FUNCTIONS=\"${EXPORTED_FUNCTIONS_STRING}\"" RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/dist/node RUNTIME_OUTPUT_NAME hdf5_util + POSITION_INDEPENDENT_CODE ON ) diff --git a/DEVELOPER.md b/DEVELOPER.md index a6d4635..eeced7e 100644 --- a/DEVELOPER.md +++ b/DEVELOPER.md @@ -17,5 +17,7 @@ which are then used by ```./dist/esm/hdf5_hl.js``` (ESM) (and also ```./dist/node/hdf5_util.js``` which is used by ```./dist/node/hdf5_hl.js```, the nodejs entry point) +an additional build ```./dist/iife/h5wasm.js``` is created from the ESM browser build, as an [IIFE](https://developer.mozilla.org/en-US/docs/Glossary/IIFE) which can be directly used in a Firefox web worker, with `importScripts()`. The exported names are exposed under the global name `h5wasm` after import. + ## distributable files The main library entrypoints are generated from ```./src/hdf5_hl.ts``` with the Typescript compiler by running ```npm run build```. This also generates the Typescript typings file ```./src/hdf5_hl.d.ts``` diff --git a/README.md b/README.md index 15e8498..96fbd14 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ The Emscripten filesystem is important for operations, and it can be accessed af ## Browser (no-build) ```js -import h5wasm from "https://cdn.jsdelivr.net/npm/h5wasm@0.4.0/dist/esm/hdf5_hl.js"; +import h5wasm from "https://cdn.jsdelivr.net/npm/h5wasm@0.4.9/dist/esm/hdf5_hl.js"; // the WASM loads asychronously, and you can get the module like this: const Module = await h5wasm.ready; @@ -45,6 +45,26 @@ let f = new h5wasm.File("sans59510.nxs.ngv", "r"); // File {path: "/", file_id: 72057594037927936n, filename: "data.h5", mode: "r"} ``` +### Worker usage +Since ESM is not supported in all web worker contexts (e.g. Firefox), an additional ```./dist/iife/h5wasm.js``` is provided in the package for `h5wasm>=0.4.8`; it can be loaded in a worker and used as in the example below (which uses the WORKERFS file system for random access on local files): +```js +// worker.js +onmessage = async function(e) { + const { FS } = await h5wasm.ready; + + // send in a file opened from an + const f_in = e.data[0]; + + FS.mkdir('/work'); + FS.mount(FS.filesystems.WORKERFS, { files: [f_in] }, '/work'); + + const f = new h5wasm.File(`/work/${f_in.name}`, 'r'); + console.log(f); +} + +self.importScripts('../dist/iife/h5wasm.js'); +``` + ## Browser target (build system) ```npm i h5wasm``` or ```yarn add h5wasm``` then in your file @@ -54,7 +74,7 @@ import h5wasm from "h5wasm"; const { FS } = await h5wasm.ready; let f = new h5wasm.File("test.h5", "w"); -f.create_dataset("text_data", ["this", "that"]); +f.create_dataset({name: "text_data", data: ["this", "that"]}); // ... ``` __note__: you must configure your build system to target >= ES2020 (for bigint support) @@ -70,7 +90,7 @@ node --experimental-wasm-bigint ``` ```js -const h5wasm = await import("h5wasm"); +const h5wasm = await import("h5wasm/node"); await h5wasm.ready; let f = new h5wasm.File("/home/brian/Downloads/sans59510.nxs.ngv", "r"); @@ -104,7 +124,6 @@ data.metadata /* { "signed": true, - "cset": -1, "vlen": false, "littleEndian": true, "type": 0, @@ -168,6 +187,19 @@ data.to_array() */ ``` +### SWMR Read +(single writer multiple readers) +```js +const swmr_file = new h5wasm.File("swmr.h5", "Sr"); +let dset = swmr_file.get("data"); +dset.shape; +// 12 +// ...later +dset.refresh(); +dset.shape; +// 16 +``` + ### Writing ```js let new_file = new h5wasm.File("myfile.h5", "w"); @@ -175,7 +207,7 @@ let new_file = new h5wasm.File("myfile.h5", "w"); new_file.create_group("entry"); // shape and dtype will match input if omitted -new_file.get("entry").create_dataset("auto", [3.1, 4.1, 0.0, -1.0]); +new_file.get("entry").create_dataset({name: "auto", data: [3.1, 4.1, 0.0, -1.0]}); new_file.get("entry/auto").shape // [4] new_file.get("entry/auto").dtype @@ -184,7 +216,7 @@ new_file.get("entry/auto").value // Float64Array(4) [3.1, 4.1, 0, -1] // make float array instead of double (shape will still match input if it is set to null) -new_file.get("entry").create_dataset("data", [3.1, 4.1, 0.0, -1.0], null, ' i); +new_file.get("entry").create_dataset({name: "compressed", data: long_data, shape: [1000, 1000], dtype: '=18" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.25.0.tgz", + "integrity": "sha512-PTyWCYYiU0+1eJKmw21lWtC+d08JDZPQ5g+kFyxP0V+es6VPPSUhM6zk8iImp2jbV6GwjX4pap0JFbUQN65X1g==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.25.0.tgz", + "integrity": "sha512-grvv8WncGjDSyUBjN9yHXNt+cq0snxXbDxy5pJtzMKGmmpPxeAmAhWxXI+01lU5rwZomDgD3kJwulEnhTRUd6g==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.25.0.tgz", + "integrity": "sha512-m/ix7SfKG5buCnxasr52+LI78SQ+wgdENi9CqyCXwjVR2X4Jkz+BpC3le3AoBPYTC9NHklwngVXvbJ9/Akhrfg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.25.0.tgz", + "integrity": "sha512-mVwdUb5SRkPayVadIOI78K7aAnPamoeFR2bT5nszFUZ9P8UpK4ratOdYbZZXYSqPKMHfS1wdHCJk1P1EZpRdvw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.25.0.tgz", + "integrity": "sha512-DgDaYsPWFTS4S3nWpFcMn/33ZZwAAeAFKNHNa1QN0rI4pUjgqf0f7ONmXf6d22tqTY+H9FNdgeaAa+YIFUn2Rg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.25.0.tgz", + "integrity": "sha512-VN4ocxy6dxefN1MepBx/iD1dH5K8qNtNe227I0mnTRjry8tj5MRk4zprLEdG8WPyAPb93/e4pSgi1SoHdgOa4w==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.25.0.tgz", + "integrity": "sha512-mrSgt7lCh07FY+hDD1TxiTyIHyttn6vnjesnPoVDNmDfOmggTLXRv8Id5fNZey1gl/V2dyVK1VXXqVsQIiAk+A==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.25.0.tgz", + "integrity": "sha512-vkB3IYj2IDo3g9xX7HqhPYxVkNQe8qTK55fraQyTzTX/fxaDtXiEnavv9geOsonh2Fd2RMB+i5cbhu2zMNWJwg==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.25.0.tgz", + "integrity": "sha512-9QAQjTWNDM/Vk2bgBl17yWuZxZNQIF0OUUuPZRKoDtqF2k4EtYbpyiG5/Dk7nqeK6kIJWPYldkOcBqjXjrUlmg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.25.0.tgz", + "integrity": "sha512-43ET5bHbphBegyeqLb7I1eYn2P/JYGNmzzdidq/w0T8E2SsYL1U6un2NFROFRg1JZLTzdCoRomg8Rvf9M6W6Gg==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.25.0.tgz", + "integrity": "sha512-fC95c/xyNFueMhClxJmeRIj2yrSMdDfmqJnyOY4ZqsALkDrrKJfIg5NTMSzVBr5YW1jf+l7/cndBfP3MSDpoHw==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.25.0.tgz", + "integrity": "sha512-nkAMFju7KDW73T1DdH7glcyIptm95a7Le8irTQNO/qtkoyypZAnjchQgooFUDQhNAy4iu08N79W4T4pMBwhPwQ==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.25.0.tgz", + "integrity": "sha512-NhyOejdhRGS8Iwv+KKR2zTq2PpysF9XqY+Zk77vQHqNbo/PwZCzB5/h7VGuREZm1fixhs4Q/qWRSi5zmAiO4Fw==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.25.0.tgz", + "integrity": "sha512-5S/rbP5OY+GHLC5qXp1y/Mx//e92L1YDqkiBbO9TQOvuFXM+iDqUNG5XopAnXoRH3FjIUDkeGcY1cgNvnXp/kA==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.25.0.tgz", + "integrity": "sha512-XM2BFsEBz0Fw37V0zU4CXfcfuACMrppsMFKdYY2WuTS3yi8O1nFOhil/xhKTmE1nPmVyvQJjJivgDT+xh8pXJA==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.25.0.tgz", + "integrity": "sha512-9yl91rHw/cpwMCNytUDxwj2XjFpxML0y9HAOH9pNVQDpQrBxHy01Dx+vaMu0N1CKa/RzBD2hB4u//nfc+Sd3Cw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-arm64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.25.0.tgz", + "integrity": "sha512-RuG4PSMPFfrkH6UwCAqBzauBWTygTvb1nxWasEJooGSJ/NwRw7b2HOwyRTQIU97Hq37l3npXoZGYMy3b3xYvPw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.25.0.tgz", + "integrity": "sha512-jl+qisSB5jk01N5f7sPCsBENCOlPiS/xptD5yxOx2oqQfyourJwIKLRA2yqWdifj3owQZCL2sn6o08dBzZGQzA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-arm64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.25.0.tgz", + "integrity": "sha512-21sUNbq2r84YE+SJDfaQRvdgznTD8Xc0oc3p3iW/a1EVWeNj/SdUCbm5U0itZPQYRuRTW20fPMWMpcrciH2EJw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.25.0.tgz", + "integrity": "sha512-2gwwriSMPcCFRlPlKx3zLQhfN/2WjJ2NSlg5TKLQOJdV0mSxIcYNTMhk3H3ulL/cak+Xj0lY1Ym9ysDV1igceg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.25.0.tgz", + "integrity": "sha512-bxI7ThgLzPrPz484/S9jLlvUAHYMzy6I0XiU1ZMeAEOBcS0VePBFxh1JjTQt3Xiat5b6Oh4x7UC7IwKQKIJRIg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.25.0.tgz", + "integrity": "sha512-ZUAc2YK6JW89xTbXvftxdnYy3m4iHIkDtK3CLce8wg8M2L+YZhIvO1DKpxrd0Yr59AeNNkTiic9YLf6FTtXWMw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.25.0.tgz", + "integrity": "sha512-eSNxISBu8XweVEWG31/JzjkIGbGIJN/TrRoiSVZwZ6pkC6VX4Im/WV2cz559/TXLcYbcrDN8JtKgd9DJVIo8GA==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.25.0.tgz", + "integrity": "sha512-ZENoHJBxA20C2zFzh6AI4fT6RraMzjYw4xKWemRTRmRVtN9c5DcH9r/f2ihEkMjOW5eGgrwCslG/+Y/3bL+DHQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/esbuild": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.25.0.tgz", + "integrity": "sha512-BXq5mqc8ltbaN34cDqWuYKyNhX8D/Z0J1xdtdQ8UcIIIyJyz+ZMKUt58tF3SrZ85jcfN/PZYhjR5uDQAYNVbuw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.25.0", + "@esbuild/android-arm": "0.25.0", + "@esbuild/android-arm64": "0.25.0", + "@esbuild/android-x64": "0.25.0", + "@esbuild/darwin-arm64": "0.25.0", + "@esbuild/darwin-x64": "0.25.0", + "@esbuild/freebsd-arm64": "0.25.0", + "@esbuild/freebsd-x64": "0.25.0", + "@esbuild/linux-arm": "0.25.0", + "@esbuild/linux-arm64": "0.25.0", + "@esbuild/linux-ia32": "0.25.0", + "@esbuild/linux-loong64": "0.25.0", + "@esbuild/linux-mips64el": "0.25.0", + "@esbuild/linux-ppc64": "0.25.0", + "@esbuild/linux-riscv64": "0.25.0", + "@esbuild/linux-s390x": "0.25.0", + "@esbuild/linux-x64": "0.25.0", + "@esbuild/netbsd-arm64": "0.25.0", + "@esbuild/netbsd-x64": "0.25.0", + "@esbuild/openbsd-arm64": "0.25.0", + "@esbuild/openbsd-x64": "0.25.0", + "@esbuild/sunos-x64": "0.25.0", + "@esbuild/win32-arm64": "0.25.0", + "@esbuild/win32-ia32": "0.25.0", + "@esbuild/win32-x64": "0.25.0" + } + }, "node_modules/typescript": { "version": "4.5.4", "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.5.4.tgz", @@ -27,6 +494,214 @@ } }, "dependencies": { + "@esbuild/aix-ppc64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.25.0.tgz", + "integrity": "sha512-O7vun9Sf8DFjH2UtqK8Ku3LkquL9SZL8OLY1T5NZkA34+wG3OQF7cl4Ql8vdNzM6fzBbYfLaiRLIOZ+2FOCgBQ==", + "dev": true, + "optional": true + }, + "@esbuild/android-arm": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.25.0.tgz", + "integrity": "sha512-PTyWCYYiU0+1eJKmw21lWtC+d08JDZPQ5g+kFyxP0V+es6VPPSUhM6zk8iImp2jbV6GwjX4pap0JFbUQN65X1g==", + "dev": true, + "optional": true + }, + "@esbuild/android-arm64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.25.0.tgz", + "integrity": "sha512-grvv8WncGjDSyUBjN9yHXNt+cq0snxXbDxy5pJtzMKGmmpPxeAmAhWxXI+01lU5rwZomDgD3kJwulEnhTRUd6g==", + "dev": true, + "optional": true + }, + "@esbuild/android-x64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.25.0.tgz", + "integrity": "sha512-m/ix7SfKG5buCnxasr52+LI78SQ+wgdENi9CqyCXwjVR2X4Jkz+BpC3le3AoBPYTC9NHklwngVXvbJ9/Akhrfg==", + "dev": true, + "optional": true + }, + "@esbuild/darwin-arm64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.25.0.tgz", + "integrity": "sha512-mVwdUb5SRkPayVadIOI78K7aAnPamoeFR2bT5nszFUZ9P8UpK4ratOdYbZZXYSqPKMHfS1wdHCJk1P1EZpRdvw==", + "dev": true, + "optional": true + }, + "@esbuild/darwin-x64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.25.0.tgz", + "integrity": "sha512-DgDaYsPWFTS4S3nWpFcMn/33ZZwAAeAFKNHNa1QN0rI4pUjgqf0f7ONmXf6d22tqTY+H9FNdgeaAa+YIFUn2Rg==", + "dev": true, + "optional": true + }, + "@esbuild/freebsd-arm64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.25.0.tgz", + "integrity": "sha512-VN4ocxy6dxefN1MepBx/iD1dH5K8qNtNe227I0mnTRjry8tj5MRk4zprLEdG8WPyAPb93/e4pSgi1SoHdgOa4w==", + "dev": true, + "optional": true + }, + "@esbuild/freebsd-x64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.25.0.tgz", + "integrity": "sha512-mrSgt7lCh07FY+hDD1TxiTyIHyttn6vnjesnPoVDNmDfOmggTLXRv8Id5fNZey1gl/V2dyVK1VXXqVsQIiAk+A==", + "dev": true, + "optional": true + }, + "@esbuild/linux-arm": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.25.0.tgz", + "integrity": "sha512-vkB3IYj2IDo3g9xX7HqhPYxVkNQe8qTK55fraQyTzTX/fxaDtXiEnavv9geOsonh2Fd2RMB+i5cbhu2zMNWJwg==", + "dev": true, + "optional": true + }, + "@esbuild/linux-arm64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.25.0.tgz", + "integrity": "sha512-9QAQjTWNDM/Vk2bgBl17yWuZxZNQIF0OUUuPZRKoDtqF2k4EtYbpyiG5/Dk7nqeK6kIJWPYldkOcBqjXjrUlmg==", + "dev": true, + "optional": true + }, + "@esbuild/linux-ia32": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.25.0.tgz", + "integrity": "sha512-43ET5bHbphBegyeqLb7I1eYn2P/JYGNmzzdidq/w0T8E2SsYL1U6un2NFROFRg1JZLTzdCoRomg8Rvf9M6W6Gg==", + "dev": true, + "optional": true + }, + "@esbuild/linux-loong64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.25.0.tgz", + "integrity": "sha512-fC95c/xyNFueMhClxJmeRIj2yrSMdDfmqJnyOY4ZqsALkDrrKJfIg5NTMSzVBr5YW1jf+l7/cndBfP3MSDpoHw==", + "dev": true, + "optional": true + }, + "@esbuild/linux-mips64el": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.25.0.tgz", + "integrity": "sha512-nkAMFju7KDW73T1DdH7glcyIptm95a7Le8irTQNO/qtkoyypZAnjchQgooFUDQhNAy4iu08N79W4T4pMBwhPwQ==", + "dev": true, + "optional": true + }, + "@esbuild/linux-ppc64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.25.0.tgz", + "integrity": "sha512-NhyOejdhRGS8Iwv+KKR2zTq2PpysF9XqY+Zk77vQHqNbo/PwZCzB5/h7VGuREZm1fixhs4Q/qWRSi5zmAiO4Fw==", + "dev": true, + "optional": true + }, + "@esbuild/linux-riscv64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.25.0.tgz", + "integrity": "sha512-5S/rbP5OY+GHLC5qXp1y/Mx//e92L1YDqkiBbO9TQOvuFXM+iDqUNG5XopAnXoRH3FjIUDkeGcY1cgNvnXp/kA==", + "dev": true, + "optional": true + }, + "@esbuild/linux-s390x": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.25.0.tgz", + "integrity": "sha512-XM2BFsEBz0Fw37V0zU4CXfcfuACMrppsMFKdYY2WuTS3yi8O1nFOhil/xhKTmE1nPmVyvQJjJivgDT+xh8pXJA==", + "dev": true, + "optional": true + }, + "@esbuild/linux-x64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.25.0.tgz", + "integrity": "sha512-9yl91rHw/cpwMCNytUDxwj2XjFpxML0y9HAOH9pNVQDpQrBxHy01Dx+vaMu0N1CKa/RzBD2hB4u//nfc+Sd3Cw==", + "dev": true, + "optional": true + }, + "@esbuild/netbsd-arm64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.25.0.tgz", + "integrity": "sha512-RuG4PSMPFfrkH6UwCAqBzauBWTygTvb1nxWasEJooGSJ/NwRw7b2HOwyRTQIU97Hq37l3npXoZGYMy3b3xYvPw==", + "dev": true, + "optional": true + }, + "@esbuild/netbsd-x64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.25.0.tgz", + "integrity": "sha512-jl+qisSB5jk01N5f7sPCsBENCOlPiS/xptD5yxOx2oqQfyourJwIKLRA2yqWdifj3owQZCL2sn6o08dBzZGQzA==", + "dev": true, + "optional": true + }, + "@esbuild/openbsd-arm64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.25.0.tgz", + "integrity": "sha512-21sUNbq2r84YE+SJDfaQRvdgznTD8Xc0oc3p3iW/a1EVWeNj/SdUCbm5U0itZPQYRuRTW20fPMWMpcrciH2EJw==", + "dev": true, + "optional": true + }, + "@esbuild/openbsd-x64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.25.0.tgz", + "integrity": "sha512-2gwwriSMPcCFRlPlKx3zLQhfN/2WjJ2NSlg5TKLQOJdV0mSxIcYNTMhk3H3ulL/cak+Xj0lY1Ym9ysDV1igceg==", + "dev": true, + "optional": true + }, + "@esbuild/sunos-x64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.25.0.tgz", + "integrity": "sha512-bxI7ThgLzPrPz484/S9jLlvUAHYMzy6I0XiU1ZMeAEOBcS0VePBFxh1JjTQt3Xiat5b6Oh4x7UC7IwKQKIJRIg==", + "dev": true, + "optional": true + }, + "@esbuild/win32-arm64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.25.0.tgz", + "integrity": "sha512-ZUAc2YK6JW89xTbXvftxdnYy3m4iHIkDtK3CLce8wg8M2L+YZhIvO1DKpxrd0Yr59AeNNkTiic9YLf6FTtXWMw==", + "dev": true, + "optional": true + }, + "@esbuild/win32-ia32": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.25.0.tgz", + "integrity": "sha512-eSNxISBu8XweVEWG31/JzjkIGbGIJN/TrRoiSVZwZ6pkC6VX4Im/WV2cz559/TXLcYbcrDN8JtKgd9DJVIo8GA==", + "dev": true, + "optional": true + }, + "@esbuild/win32-x64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.25.0.tgz", + "integrity": "sha512-ZENoHJBxA20C2zFzh6AI4fT6RraMzjYw4xKWemRTRmRVtN9c5DcH9r/f2ihEkMjOW5eGgrwCslG/+Y/3bL+DHQ==", + "dev": true, + "optional": true + }, + "esbuild": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.25.0.tgz", + "integrity": "sha512-BXq5mqc8ltbaN34cDqWuYKyNhX8D/Z0J1xdtdQ8UcIIIyJyz+ZMKUt58tF3SrZ85jcfN/PZYhjR5uDQAYNVbuw==", + "dev": true, + "requires": { + "@esbuild/aix-ppc64": "0.25.0", + "@esbuild/android-arm": "0.25.0", + "@esbuild/android-arm64": "0.25.0", + "@esbuild/android-x64": "0.25.0", + "@esbuild/darwin-arm64": "0.25.0", + "@esbuild/darwin-x64": "0.25.0", + "@esbuild/freebsd-arm64": "0.25.0", + "@esbuild/freebsd-x64": "0.25.0", + "@esbuild/linux-arm": "0.25.0", + "@esbuild/linux-arm64": "0.25.0", + "@esbuild/linux-ia32": "0.25.0", + "@esbuild/linux-loong64": "0.25.0", + "@esbuild/linux-mips64el": "0.25.0", + "@esbuild/linux-ppc64": "0.25.0", + "@esbuild/linux-riscv64": "0.25.0", + "@esbuild/linux-s390x": "0.25.0", + "@esbuild/linux-x64": "0.25.0", + "@esbuild/netbsd-arm64": "0.25.0", + "@esbuild/netbsd-x64": "0.25.0", + "@esbuild/openbsd-arm64": "0.25.0", + "@esbuild/openbsd-x64": "0.25.0", + "@esbuild/sunos-x64": "0.25.0", + "@esbuild/win32-arm64": "0.25.0", + "@esbuild/win32-ia32": "0.25.0", + "@esbuild/win32-x64": "0.25.0" + } + }, "typescript": { "version": "4.5.4", "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.5.4.tgz", diff --git a/package.json b/package.json index 5e82215..bf9fde5 100644 --- a/package.json +++ b/package.json @@ -1,19 +1,27 @@ { "name": "h5wasm", - "version": "0.4.3", + "version": "0.8.1", "description": "A high-level library for reading and writing HDF5 files from Javascript, using a wasm-compiled version of the HDF5 C library", - "types": "./src/hdf5_hl.d.ts", "type": "module", - "main": "./dist/node/hdf5_hl.js", + "main": "./dist/iife/hdf5_hl.js", "module": "./dist/esm/hdf5_hl.js", + "browser": "./dist/iife/h5wasm.js", + "types": "./src/hdf5_hl.d.ts", "exports": { - "node": "./dist/node/hdf5_hl.js", - "import": "./dist/esm/hdf5_hl.js" + ".": { + "types": "./src/hdf5_hl.d.ts", + "import": "./dist/esm/hdf5_hl.js" + }, + "./node": { + "types": "./src/hdf5_hl.d.ts", + "import": "./dist/node/hdf5_hl.js" + } }, "scripts": { - "build": "npm run build_esm && npm run build_node && npm run build_types", + "build": "npm run build_esm && npm run build_node && npm run build_types && npm run build_iife", "build_esm": "tsc src/hdf5_hl.ts src/file_handlers.ts --strict --outDir dist/esm --target es2020", "build_node": "tsc src/hdf5_hl.ts --outDir dist/node --target es2020 --allowJs --esModuleInterop", + "build_iife": "esbuild --bundle dist/esm/hdf5_hl.js --outfile=dist/iife/h5wasm.js --format=iife --global-name=h5wasm", "build_types": "tsc --declaration --strict --emitDeclarationOnly src/hdf5_hl.ts --target es2020", "test": "node ./test/test.mjs" }, @@ -31,6 +39,7 @@ "author": "Brian B. Maranville", "license": "SEE LICENSE IN LICENSE.txt", "devDependencies": { + "esbuild": "^0.25.0", "typescript": "^4.5.4" } } diff --git a/src/emscripten.d.ts b/src/emscripten.d.ts index ba2b7b5..99b438e 100644 --- a/src/emscripten.d.ts +++ b/src/emscripten.d.ts @@ -144,6 +144,7 @@ declare namespace FS { interface ErrnoError {} interface FileSystemType { + filesystems: any, ignorePermissions: boolean, trackingDelegate: any, tracking: any, @@ -185,6 +186,7 @@ declare namespace FS { unmount(mountpoint: string): void, mkdir(path: string, mode?: number): any, + mkdirTree(path: string, mode?: number): void, mkdev(path: string, mode?: number, dev?: number): any, symlink(oldpath: string, newpath: string): any, rename(old_path: string, new_path: string): void, diff --git a/src/hdf5_hl.d.ts b/src/hdf5_hl.d.ts index d36e4b9..cab8905 100644 --- a/src/hdf5_hl.d.ts +++ b/src/hdf5_hl.d.ts @@ -1,4 +1,4 @@ -import type { Status, Metadata, H5Module, CompoundTypeMetadata } from "./hdf5_util_helpers"; +import type { Status, Metadata, H5Module, CompoundMember, CompoundTypeMetadata, EnumTypeMetadata, Filter } from "./hdf5_util_helpers.js"; export declare var Module: H5Module; export declare var FS: (FS.FileSystemType | null); declare const ready: Promise; @@ -8,27 +8,38 @@ export declare const ACCESS_MODES: { readonly a: "H5F_ACC_RDWR"; readonly w: "H5F_ACC_TRUNC"; readonly x: "H5F_ACC_EXCL"; - readonly c: "H5F_ACC_CREAT"; readonly Sw: "H5F_ACC_SWMR_WRITE"; readonly Sr: "H5F_ACC_SWMR_READ"; }; declare type ACCESS_MODESTRING = keyof typeof ACCESS_MODES; -export declare type OutputData = TypedArray | string | number | bigint | boolean | OutputData[]; +export declare type OutputData = TypedArray | string | number | bigint | boolean | Reference | RegionReference | OutputData[]; export declare type JSONCompatibleOutputData = string | number | boolean | JSONCompatibleOutputData[]; export declare type Dtype = string | { compound_type: CompoundTypeMetadata; } | { array_type: Metadata; }; -export type { Metadata }; +export type { Metadata, Filter, CompoundMember, CompoundTypeMetadata, EnumTypeMetadata }; declare type TypedArray = Int8Array | Uint8Array | Uint8ClampedArray | Int16Array | Uint16Array | Int32Array | Uint32Array | BigInt64Array | BigUint64Array | Float32Array | Float64Array; -export declare type GuessableDataTypes = TypedArray | number | number[] | string | string[]; +/** + * Describes an array slice. + * `[]` - all data + * `[i0]` - select all data starting from the index `i0` + * `[i0, i1]` - select all data in the range `i0` to `i1` + * `[i0, i1, s]` - select every `s` values in the range `i0` to `i1` + **/ +declare type Slice = [] | [number | null] | [number | null, number | null] | [number | null, number | null, number | null]; +export declare type GuessableDataTypes = TypedArray | number | number[] | string | string[] | Reference | Reference[] | RegionReference | RegionReference[]; declare enum OBJECT_TYPE { DATASET = "Dataset", GROUP = "Group", BROKEN_SOFT_LINK = "BrokenSoftLink", - EXTERNAL_LINK = "ExternalLink" + EXTERNAL_LINK = "ExternalLink", + DATATYPE = "Datatype", + REFERENCE = "Reference", + REGION_REFERENCE = "RegionReference" } +export declare type Entity = Dataset | Group | BrokenSoftLink | ExternalLink | Datatype | Reference | RegionReference; export declare class BrokenSoftLink { target: string; type: OBJECT_TYPE; @@ -40,77 +51,131 @@ export declare class ExternalLink { type: OBJECT_TYPE; constructor(filename: string, obj_path: string); } +export declare class Reference { + ref_data: Uint8Array; + constructor(ref_data: Uint8Array); +} +export declare class RegionReference extends Reference { +} export declare class Attribute { file_id: bigint; path: string; name: string; metadata: Metadata; dtype: Dtype; - shape: number[]; + shape: number[] | null; private _value?; private _json_value?; constructor(file_id: bigint, path: string, name: string); - get value(): OutputData; - get json_value(): JSONCompatibleOutputData; - to_array(): string | number | boolean | JSONCompatibleOutputData[]; + get value(): OutputData | null; + get json_value(): JSONCompatibleOutputData | null; + to_array(): JSONCompatibleOutputData | null; } declare abstract class HasAttrs { file_id: bigint; path: string; type: OBJECT_TYPE; - get attrs(): { - [key: string]: Attribute; - }; + get attrs(): Record; + get root(): Group; + get parent(): Group; get_attribute(name: string, json_compatible: true): JSONCompatibleOutputData; get_attribute(name: string, json_compatible: false): OutputData; create_attribute(name: string, data: GuessableDataTypes, shape?: number[] | null, dtype?: string | null): void; + delete_attribute(name: string): number; + create_reference(): Reference; + dereference(ref: RegionReference): DatasetRegion; + dereference(ref: Reference | RegionReference): DatasetRegion | Entity | null; +} +export declare class Datatype extends HasAttrs { + constructor(file_id: bigint, path: string); + get metadata(): Metadata; } export declare class Group extends HasAttrs { constructor(file_id: bigint, path: string); - keys(): Array; - values(): Generator; - items(): Generator<(string | BrokenSoftLink | ExternalLink | Group | Dataset | null)[], void, unknown>; + keys(): string[]; + values(): Generator; + items(): Generator<[string, Entity | null], void, never>; get_type(obj_path: string): number; get_link(obj_path: string): string; get_external_link(obj_path: string): { filename: string; obj_path: string; }; - get(obj_name: string): BrokenSoftLink | ExternalLink | Group | Dataset | null; - create_group(name: string): Group; - create_dataset(name: string, data: GuessableDataTypes, shape?: number[] | null, dtype?: string | null): Dataset; + get(obj_name: string): Entity | null; + create_group(name: string, track_order?: boolean): Group; + create_dataset(args: { + name: string; + data: GuessableDataTypes; + shape?: number[] | null; + dtype?: string | null; + maxshape?: (number | null)[] | null; + chunks?: number[] | null; + compression?: (number | 'gzip'); + compression_opts?: number | number[]; + track_order?: boolean; + }): Dataset; + create_soft_link(target: string, name: string): number; + create_hard_link(target: string, name: string): number; + create_external_link(file_name: string, target: string, name: string): number; toString(): string; + paths(): string[]; } export declare class File extends Group { filename: string; mode: ACCESS_MODESTRING; - constructor(filename: string, mode?: ACCESS_MODESTRING); + constructor(filename: string, mode?: ACCESS_MODESTRING, track_order?: boolean); flush(): void; close(): Status; } export declare class Dataset extends HasAttrs { private _metadata?; constructor(file_id: bigint, path: string); + refresh(): void; get metadata(): Metadata; get dtype(): Dtype; - get shape(): number[]; - get value(): OutputData; - get json_value(): JSONCompatibleOutputData; - slice(ranges: Array>): OutputData; - to_array(): string | number | boolean | JSONCompatibleOutputData[]; - _value_getter(json_compatible?: boolean): OutputData; + get shape(): number[] | null; + get filters(): Filter[]; + get value(): OutputData | null; + get json_value(): JSONCompatibleOutputData | null; + slice(ranges: Slice[]): OutputData | null; + write_slice(ranges: Slice[], data: any): void; + create_region_reference(ranges: Slice[]): RegionReference; + to_array(): JSONCompatibleOutputData | null; + resize(new_shape: number[]): number; + make_scale(scale_name?: string): void; + attach_scale(index: number, scale_dset_path: string): void; + detach_scale(index: number, scale_dset_path: string): void; + get_attached_scales(index: number): string[]; + get_scale_name(): string | null; + set_dimension_label(index: number, label: string): void; + get_dimension_labels(): (string | null)[]; + _value_getter(json_compatible?: false): OutputData | null; + _value_getter(json_compatible: true): JSONCompatibleOutputData | null; + _value_getter(json_compatible: boolean): OutputData | JSONCompatibleOutputData | null; +} +export declare class DatasetRegion { + source_dataset: Dataset; + region_reference: RegionReference; + private _metadata?; + constructor(source_dataset: Dataset, region_reference: RegionReference); + get metadata(): Metadata; + get value(): OutputData | null; + _value_getter(json_compatible?: false): OutputData | null; + _value_getter(json_compatible: true): JSONCompatibleOutputData | null; + _value_getter(json_compatible: boolean): OutputData | JSONCompatibleOutputData | null; } export declare const h5wasm: { File: typeof File; Group: typeof Group; Dataset: typeof Dataset; + Datatype: typeof Datatype; + DatasetRegion: typeof DatasetRegion; ready: Promise; ACCESS_MODES: { readonly r: "H5F_ACC_RDONLY"; readonly a: "H5F_ACC_RDWR"; readonly w: "H5F_ACC_TRUNC"; readonly x: "H5F_ACC_EXCL"; - readonly c: "H5F_ACC_CREAT"; readonly Sw: "H5F_ACC_SWMR_WRITE"; readonly Sr: "H5F_ACC_SWMR_READ"; }; diff --git a/src/hdf5_hl.ts b/src/hdf5_hl.ts index 7bee948..84ef00c 100644 --- a/src/hdf5_hl.ts +++ b/src/hdf5_hl.ts @@ -1,4 +1,4 @@ -import type {Status, Metadata, H5Module, CompoundTypeMetadata, EnumTypeMetadata} from "./hdf5_util_helpers"; +import type {Status, Metadata, H5Module, CompoundMember, CompoundTypeMetadata, EnumTypeMetadata, Filter} from "./hdf5_util_helpers.js"; import ModuleFactory from './hdf5_util.js'; @@ -13,7 +13,6 @@ export const ACCESS_MODES = { "a": "H5F_ACC_RDWR", "w": "H5F_ACC_TRUNC", "x": "H5F_ACC_EXCL", - "c": "H5F_ACC_CREAT", "Sw": "H5F_ACC_SWMR_WRITE", "Sr": "H5F_ACC_SWMR_READ" } as const; @@ -29,11 +28,27 @@ function normalizePath(path: string) { return path; } -function get_attr(file_id: bigint, obj_name: string, attr_name: string, json_compatible: true): JSONCompatibleOutputData; -function get_attr(file_id: bigint, obj_name: string, attr_name: string, json_compatible: false): OutputData; -function get_attr(file_id: bigint, obj_name: string, attr_name: string, json_compatible: boolean): OutputData | JSONCompatibleOutputData; -function get_attr(file_id: bigint, obj_name: string, attr_name: string, json_compatible: boolean = false) { +function dirname(path: string) { + // adapted from dirname function in posixpath.py + const sep = "/"; + const sep_index = path.lastIndexOf(sep) + 1; + let head = path.slice(0, sep_index); + if (head && head !== sep.repeat(head.length)) { + // strip end slashes + head = head.replace(/(\/)+$/, ''); + } + return head; +} + +function get_attr(file_id: bigint, obj_name: string, attr_name: string, json_compatible: true): JSONCompatibleOutputData | null; +function get_attr(file_id: bigint, obj_name: string, attr_name: string, json_compatible: false): OutputData | null; +function get_attr(file_id: bigint, obj_name: string, attr_name: string, json_compatible: boolean): OutputData | JSONCompatibleOutputData | null; +function get_attr(file_id: bigint, obj_name: string, attr_name: string, json_compatible: boolean = false): OutputData | JSONCompatibleOutputData | null { let metadata = Module.get_attribute_metadata(file_id, obj_name, attr_name); + if (!metadata.shape) { + return null; + } + let nbytes = metadata.size * metadata.total_size; let data_ptr = Module._malloc(nbytes); var processed; @@ -81,10 +96,10 @@ function getAccessor(type: 0 | 1, size: Metadata["size"], signed: Metadata["sign } } -export type OutputData = TypedArray | string | number | bigint | boolean | OutputData[]; +export type OutputData = TypedArray | string | number | bigint | boolean | Reference | RegionReference | OutputData[]; export type JSONCompatibleOutputData = string | number | boolean | JSONCompatibleOutputData[]; export type Dtype = string | {compound_type: CompoundTypeMetadata} | {array_type: Metadata}; -export type { Metadata }; +export type { Metadata, Filter, CompoundMember, CompoundTypeMetadata, EnumTypeMetadata }; function process_data(data: Uint8Array, metadata: Metadata, json_compatible: true): JSONCompatibleOutputData; function process_data(data: Uint8Array, metadata: Metadata, json_compatible: false): OutputData; @@ -95,11 +110,12 @@ function process_data(data: Uint8Array, metadata: Metadata, json_compatible: boo // but otherwise returns Uint8Array raw bytes as loaded. let output_data: OutputData; let { shape, type } = metadata; + let known_type = true; // let length: number; if (type === Module.H5T_class_t.H5T_STRING.value) { if (metadata.vlen) { - let output = []; + let output: string[] = []; let reader = (metadata.cset == 1) ? Module.UTF8ToString : Module.AsciiToString; let ptrs = new Uint32Array(data.buffer); for (let ptr of ptrs) { @@ -113,7 +129,7 @@ function process_data(data: Uint8Array, metadata: Metadata, json_compatible: boo let decoder = new TextDecoder(encoding); let size = metadata.size; let n = Math.floor(data.byteLength / size); - let output = []; + let output: string[] = []; for (let i = 0; i < n; i++) { let bytes = data.slice(i * size, (i + 1) * size); // truncate at first null @@ -142,9 +158,9 @@ function process_data(data: Uint8Array, metadata: Metadata, json_compatible: boo else if (type === Module.H5T_class_t.H5T_COMPOUND.value) { const { size, compound_type } = <{size: Metadata["size"], compound_type: CompoundTypeMetadata}>metadata; let n = Math.floor(data.byteLength / size); - let output = []; + let output: (OutputData | JSONCompatibleOutputData)[] = []; for (let i = 0; i < n; i++) { - let row = []; + let row: (OutputData | JSONCompatibleOutputData)[] = []; let row_data = data.slice(i * size, (i + 1) * size); for (let member of compound_type.members) { let member_data = row_data.slice(member.offset, member.offset + member.size); @@ -157,7 +173,7 @@ function process_data(data: Uint8Array, metadata: Metadata, json_compatible: boo else if (type === Module.H5T_class_t.H5T_ARRAY.value) { const { array_type } = <{array_type: Metadata}>metadata; - shape = shape.concat(array_type.shape); + shape = (shape).concat(array_type.shape); array_type.shape = shape; // always convert ARRAY types to base JS types: output_data = process_data(data, array_type, true); @@ -172,7 +188,7 @@ function process_data(data: Uint8Array, metadata: Metadata, json_compatible: boo output_data = process_data(data, base_metadata, json_compatible); // Following the convention of h5py, treat all enum datasets where the // enum members are ["FALSE", "TRUE"] as boolean arrays - if (isH5PYBooleanEnum(metadata.enum_type as EnumTypeMetadata)) { + if (json_compatible && isH5PYBooleanEnum(metadata.enum_type as EnumTypeMetadata)) { if (isIterable(output_data)) { output_data = [...output_data].map((x) => !!x); } @@ -182,6 +198,40 @@ function process_data(data: Uint8Array, metadata: Metadata, json_compatible: boo } } + else if (type === Module.H5T_class_t.H5T_REFERENCE.value) { + const { ref_type, size } = metadata; // as { ref_type: 'object' | 'region', size: number }; + const cls = (ref_type === 'object') ? Reference : RegionReference; + output_data = Array.from({ length: metadata.total_size }).map((_, i) => { + const ref_data = data.slice(i*size, (i+1)*size); + return new cls(ref_data); + }); + return output_data; + } + + else if (type === Module.H5T_class_t.H5T_VLEN.value) { + // Data buffer holds HDF5 `hvl_t` struct + // https://docs.hdfgroup.org/hdf5/v1_12/structhvl__t.html + const ptr_pairs = new Uint32Array(data.buffer); // both `size_t` length and pointer are 4-byte-long + const ptr_pairs_length = ptr_pairs.length; + const vlen_type = metadata.vlen_type as Metadata; + const { size } = vlen_type; + + let output: (OutputData | JSONCompatibleOutputData)[] = []; + for (let p = 0; p < ptr_pairs_length; p += 2) { + const length = ptr_pairs[p]; // number of elements in this vlen array + const data_ptr = ptr_pairs[p + 1]; // pointer to this vlen array's data + + // Read vlen array data from memory + const data_nbytes = length * size; + const data = Module.HEAPU8.slice(data_ptr, data_ptr + data_nbytes); + + // Process this vlen array's data according to base datatype + output.push(process_data(data, { ...vlen_type, shape: [length], total_size: length }, json_compatible)); + } + + output_data = output; + } + else { known_type = false; output_data = data; @@ -199,7 +249,7 @@ function process_data(data: Uint8Array, metadata: Metadata, json_compatible: boo } function isIterable(x: any): x is Iterable { - return Symbol.iterator in x; + return typeof x === 'object' && x !== null && Symbol.iterator in x; } function isH5PYBooleanEnum(enum_type: EnumTypeMetadata) { @@ -208,24 +258,35 @@ function isH5PYBooleanEnum(enum_type: EnumTypeMetadata) { enum_type.members["TRUE"] === 1; } -function prepare_data(data: any, metadata: Metadata, shape?: Array | null): {data: Uint8Array | string[], shape: number[]} { +const H5S_UNLIMITED = 18446744073709551615n; // not exportable by emscripten_bindings because it's outside int32 range + +function prepare_data(data: any, metadata: Metadata, shape?: number[] | bigint[] | null, maxshape?: (number | null)[] | null | undefined): {data: Uint8Array | string[], shape: bigint[], maxshape: (bigint | null)[] } { // for data being sent to Module // set shape to size of array if it is not specified: - let final_shape: number[]; + let final_shape: bigint[]; + let final_maxshape: (bigint | null)[] | null; if (shape === undefined || shape === null) { if (data != null && data.length != null && !(typeof data === 'string')) { - final_shape = [data.length]; + final_shape = [BigInt(data.length)]; } else { final_shape = []; } } else { - final_shape = shape; + final_shape = shape.map(BigInt); } + + if (maxshape === undefined || maxshape === null) { + final_maxshape = final_shape; + } + else { + final_maxshape = maxshape.map((dim) => ((dim === null) ? H5S_UNLIMITED : BigInt(dim))); + } + data = (Array.isArray(data) || ArrayBuffer.isView(data)) ? data : [data]; - let total_size = final_shape.reduce((previous, current) => current * previous, 1); + let total_size = Number(final_shape.reduce((previous, current) => current * previous, 1n)); if (data.length != total_size) { throw `Error: shape ${final_shape} does not match number of elements in data`; @@ -258,17 +319,21 @@ function prepare_data(data: any, metadata: Metadata, shape?: Array | nul } else { // convert... - if (metadata.size > 4) { + if (metadata.size > 4 && metadata.type === Module.H5T_class_t.H5T_INTEGER.value) { data = data.map(BigInt); } typed_array = new accessor(data); } output = new Uint8Array(typed_array.buffer); } + else if (metadata.type === Module.H5T_class_t.H5T_REFERENCE.value) { + output = new Uint8Array(metadata.size * total_size); + (data as Reference[]).forEach((r, i) => (output as Uint8Array).set(r.ref_data, i*metadata.size)); + } else { throw new Error(`data with type ${metadata.type} can not be prepared for write`); } - return {data: output, shape: final_shape} + return {data: output, shape: final_shape, maxshape: final_maxshape} } function map_reverse(map: Map): Map { @@ -306,35 +371,45 @@ function metadata_to_dtype(metadata: Metadata): Dtype { else if (type === Module.H5T_class_t.H5T_ARRAY.value ) { return { array_type: array_type as Metadata } } + else if (type === Module.H5T_class_t.H5T_REFERENCE.value) { + return (metadata.ref_type === 'object') ? "Reference" : "RegionReference"; + } else { return "unknown"; } } function dtype_to_metadata(dtype_str: string) { - let match = dtype_str.match(/^([<>|]?)([bhiqefdsBHIQS])([0-9]*)$/); - if (match == null) { - throw dtype_str + " is not a recognized dtype" - } - let [full, endianness, typestr, length] = match; let metadata = { vlen: false, signed: false } as Metadata; - metadata.littleEndian = (endianness != '>'); - if (fmts_int.has(typestr.toLowerCase())) { - metadata.type = Module.H5T_class_t.H5T_INTEGER.value; - metadata.size = (fmts_int.get(typestr.toLowerCase()) as number); - metadata.signed = (typestr.toLowerCase() == typestr); - } - else if (fmts_float.has(typestr)) { - metadata.type = Module.H5T_class_t.H5T_FLOAT.value; - metadata.size = (fmts_float.get(typestr) as number); - } - else if (typestr.toUpperCase() == 'S') { - metadata.type = Module.H5T_class_t.H5T_STRING.value; - metadata.size = (length == "") ? 4 : parseInt(length, 10); - metadata.vlen = (length == ""); + if (dtype_str === "Reference" || dtype_str === "RegionReference") { + metadata.type = Module.H5T_class_t.H5T_REFERENCE.value; + metadata.size = (dtype_str === "Reference") ? Module.SIZEOF_OBJ_REF : Module.SIZEOF_DSET_REGION_REF; + metadata.littleEndian = true; } else { - throw "should never happen" + let match = dtype_str.match(/^([<>|]?)([bhiqefdsBHIQS])([0-9]*)$/); + if (match == null) { + throw dtype_str + " is not a recognized dtype" + } + let [full, endianness, typestr, length] = match; + metadata.littleEndian = (endianness != '>'); + if (fmts_int.has(typestr.toLowerCase())) { + metadata.type = Module.H5T_class_t.H5T_INTEGER.value; + metadata.size = (fmts_int.get(typestr.toLowerCase()) as number); + metadata.signed = (typestr.toLowerCase() == typestr); + } + else if (fmts_float.has(typestr)) { + metadata.type = Module.H5T_class_t.H5T_FLOAT.value; + metadata.size = (fmts_float.get(typestr) as number); + } + else if (typestr.toUpperCase() === 'S') { + metadata.type = Module.H5T_class_t.H5T_STRING.value; + metadata.size = (length == "") ? 4 : parseInt(length, 10); + metadata.vlen = (length == ""); + } + else { + throw "should never happen" + } } return metadata } @@ -352,7 +427,7 @@ type TypedArray = | Float32Array | Float64Array; -type TypedArrayConstructor = +type TypedArrayConstructor = | Int8ArrayConstructor | Uint8ArrayConstructor | Uint8ClampedArrayConstructor @@ -367,6 +442,7 @@ type TypedArrayConstructor = const TypedArray_to_dtype = new Map([ ['Uint8Array', ' (typeof d == 'string'))) { - return 'S' + return 'S'; + } + else if (arr_data.every((d) => d instanceof RegionReference)) { + return 'RegionReference'; + } + else if (arr_data.every((d) => d instanceof Reference)) { + return 'Reference'; } } throw new Error("unguessable type for data"); @@ -408,9 +499,14 @@ enum OBJECT_TYPE { DATASET = "Dataset", GROUP = "Group", BROKEN_SOFT_LINK = "BrokenSoftLink", - EXTERNAL_LINK = "ExternalLink" + EXTERNAL_LINK = "ExternalLink", + DATATYPE = 'Datatype', + REFERENCE = 'Reference', + REGION_REFERENCE = 'RegionReference', } +export type Entity = Dataset | Group | BrokenSoftLink | ExternalLink | Datatype | Reference | RegionReference; + export class BrokenSoftLink { // only used for broken links... target: string; @@ -430,15 +526,25 @@ export class ExternalLink { } } +export class Reference { + ref_data: Uint8Array; + constructor(ref_data: Uint8Array) { + this.ref_data = ref_data; + } +} + +export class RegionReference extends Reference { +} + export class Attribute { file_id: bigint; path: string; name: string; metadata: Metadata; dtype: Dtype; - shape: number[]; - private _value?: OutputData; - private _json_value?: JSONCompatibleOutputData; + shape: number[] | null; + private _value?: OutputData | null; + private _json_value?: JSONCompatibleOutputData | null; constructor(file_id: bigint, path: string, name: string) { this.file_id = file_id; @@ -450,27 +556,27 @@ export class Attribute { this.shape = metadata.shape; } - get value() { + get value(): OutputData | null { if (typeof this._value === "undefined") { this._value = get_attr(this.file_id, this.path, this.name, false); } return this._value; } - get json_value() { + get json_value(): JSONCompatibleOutputData | null { if (typeof this._json_value === "undefined") { this._json_value = get_attr(this.file_id, this.path, this.name, true); } return this._json_value; } - to_array() { + to_array(): JSONCompatibleOutputData | null { const { json_value, metadata } = this; const { shape } = metadata; if (!isIterable(json_value) || typeof json_value === "string") { return json_value; } - return create_nested_array(json_value, shape); + return create_nested_array(json_value, shape); } } @@ -481,7 +587,7 @@ abstract class HasAttrs { get attrs() { let attr_names = Module.get_attribute_names(this.file_id, this.path) as string[]; - let attrs: {[key: string]: Attribute} = {}; + let attrs: Record = {}; const { file_id, path } = this; for (let name of attr_names) { Object.defineProperty(attrs, name, { @@ -490,7 +596,14 @@ abstract class HasAttrs { }); } return attrs; + } + + get root() { + return new Group(this.file_id, '/'); + } + get parent(): Group { + return this.root.get(dirname(this.path)) as Group; } get_attribute(name: string, json_compatible: true): JSONCompatibleOutputData; @@ -499,19 +612,22 @@ abstract class HasAttrs { return get_attr(this.file_id, this.path, name, json_compatible); } - + create_attribute(name: string, data: GuessableDataTypes, shape?: number[] | null, dtype?: string | null) { const final_dtype = dtype ?? guess_dtype(data); let metadata = dtype_to_metadata(final_dtype); - let {data: prepared_data, shape: guessed_shape} = prepare_data(data, metadata, shape); - const final_shape = shape ?? guessed_shape; + if (!metadata.littleEndian) { + throw new Error("create_attribute with big-endian dtype is not supported"); + } + const {data: prepared_data, shape: guessed_shape} = prepare_data(data, metadata, shape, shape); + const final_shape = shape?.map(BigInt) ?? guessed_shape; if (metadata.vlen) { Module.create_vlen_str_attribute( this.file_id, this.path, name, prepared_data as string[], - final_shape.map(BigInt), + final_shape, metadata.type, metadata.size, metadata.signed, @@ -527,7 +643,7 @@ abstract class HasAttrs { this.path, name, BigInt(data_ptr), - final_shape.map(BigInt), + final_shape, metadata.type, metadata.size, metadata.signed, @@ -539,6 +655,37 @@ abstract class HasAttrs { } } + delete_attribute(name: string): number { + // returns non-zero value if delete failed. + return Module.delete_attribute(this.file_id, this.path, name); + } + + create_reference(): Reference { + const ref_data = Module.create_object_reference(this.file_id, this.path); + return new Reference(ref_data); + } + + dereference(ref: RegionReference): DatasetRegion; + dereference(ref: Reference | RegionReference): DatasetRegion | Entity | null; + dereference(ref: Reference | RegionReference): DatasetRegion | Entity | null { + const is_region = (ref instanceof RegionReference); + const name = Module.get_referenced_name(this.file_id, ref.ref_data, !is_region); + const target = this.root.get(name); + return (is_region) ? new DatasetRegion(target as Dataset, ref) : target; + } +} + +export class Datatype extends HasAttrs { + constructor(file_id: bigint, path: string) { + super(); + this.file_id = file_id; + this.path = path; + this.type = OBJECT_TYPE.DATATYPE; + } + + get metadata() { + return Module.get_datatype_metadata(this.file_id, this.path); + } } export class Group extends HasAttrs { @@ -549,22 +696,22 @@ export class Group extends HasAttrs { this.type = OBJECT_TYPE.GROUP; } - keys(): Array { - return Module.get_names(this.file_id, this.path) as string[]; + keys(): string[] { + return Module.get_names(this.file_id, this.path, false); } - * values() { + * values(): Generator { for (let name of this.keys()) { yield this.get(name); } - return + return; } - * items() { + * items(): Generator<[string, Entity | null], void, never> { for (let name of this.keys()) { yield [name, this.get(name)]; } - return + return; } get_type(obj_path: string) { @@ -579,7 +726,7 @@ export class Group extends HasAttrs { return Module.get_external_link(this.file_id, obj_path); } - get(obj_name: string) { + get(obj_name: string): Entity | null { let fullpath = (/^\//.test(obj_name)) ? obj_name : this.path + "/" + obj_name; fullpath = normalizePath(fullpath); @@ -600,30 +747,86 @@ export class Group extends HasAttrs { let {filename, obj_path} = this.get_external_link(fullpath) as {filename: string, obj_path: string}; return new ExternalLink(filename, obj_path); } + else if (type === Module.H5G_TYPE) { + return new Datatype(this.file_id, fullpath); + } // unknown type or object not found return null } - create_group(name: string): Group { - Module.create_group(this.file_id, this.path + "/" + name); + create_group(name: string, track_order: boolean = false): Group { + Module.create_group(this.file_id, this.path + "/" + name, track_order); return this.get(name) as Group; } - create_dataset(name: string, data: GuessableDataTypes, shape?: number[] | null, dtype?: string | null): Dataset { + create_dataset(args: { + name: string, + data: GuessableDataTypes, + shape?: number[] | null, + dtype?: string | null, + maxshape?: (number | null)[] | null, + chunks?: number[] | null, + compression?: (number | 'gzip'), + compression_opts?: number | number[], + track_order?: boolean, + }): Dataset { + const { name, data, shape, dtype, maxshape, chunks, compression, compression_opts, track_order } = args; const final_dtype = dtype ?? guess_dtype(data); let metadata = dtype_to_metadata(final_dtype); - let {data: prepared_data, shape: guessed_shape} = prepare_data(data, metadata, shape); - const final_shape: number[] = shape ?? guessed_shape; + if (compression && !chunks) { + throw new Error("cannot specify compression without chunks"); + } + if (compression && metadata.vlen) { + throw new Error("cannot specify compression with VLEN data"); + } + if (!metadata.littleEndian) { + throw new Error("create_dataset with big-endian dtype is not supported"); + } + const {data: prepared_data, shape: guessed_shape, maxshape: final_maxshape} = prepare_data(data, metadata, shape, maxshape); + const final_shape: bigint[] = shape?.map(BigInt) ?? guessed_shape; + const final_chunks = (chunks) ? chunks.map(BigInt) : null; + let compression_opts_out: number[]; + let compression_id: number = 0; + if (compression && typeof compression === 'number') { + if (typeof compression_opts === 'undefined') { + // handle special case where no opts are given, use 'gzip' + // and numeric value of compression as compression_opts + compression_id = 1; + compression_opts_out = [compression]; + } + else { + // promote single number to opts list. + compression_id = compression; + compression_opts_out = (typeof compression_opts === 'number') ? [compression_opts] : compression_opts; + } + } + else if (compression === 'gzip') { + compression_id = 1; + if (compression_opts === undefined) { + compression_opts_out = [4]; // default compression level + } + else { + compression_opts_out = (typeof compression_opts === 'number') ? [compression_opts] : compression_opts; + } + } + else { + compression_id = 0; + compression_opts_out = []; + } + if (metadata.vlen) { Module.create_vlen_str_dataset( this.file_id, this.path + "/" + name, prepared_data as string[], - final_shape.map(BigInt), + final_shape, + final_maxshape, + final_chunks, metadata.type, metadata.size, metadata.signed, - metadata.vlen + metadata.vlen, + track_order ?? false, ); } else { @@ -634,11 +837,16 @@ export class Group extends HasAttrs { this.file_id, this.path + "/" + name, BigInt(data_ptr), - final_shape.map(BigInt), + final_shape, + final_maxshape, + final_chunks, metadata.type, metadata.size, metadata.signed, - metadata.vlen + metadata.vlen, + compression_id, + compression_opts_out, + track_order ?? false ); } finally { Module._free(data_ptr); @@ -646,28 +854,44 @@ export class Group extends HasAttrs { } return this.get(name) as Dataset; } + + create_soft_link(target: string, name: string) { + // create a soft link in this group named *name* to *target* (absolute path) + const link_name = this.path + '/' + name; + return Module.create_soft_link(this.file_id, target, link_name); + } + + create_hard_link(target: string, name: string) { + // create a hard link in this group named *name* to *target* (absolute path) + const link_name = this.path + '/' + name; + return Module.create_hard_link(this.file_id, target, link_name); + } + + create_external_link(file_name: string, target: string, name: string) { + // create a soft link in this group named *name* to *target* (absolute path) + const link_name = this.path + '/' + name; + return Module.create_external_link(this.file_id, file_name, target, link_name); + } + toString() { return `Group(file_id=${this.file_id}, path=${this.path})`; } + + paths() { + // get all paths below this group in the tree + return Module.get_names(this.file_id, this.path, true); + } } export class File extends Group { filename: string; mode: ACCESS_MODESTRING; - constructor(filename: string, mode: ACCESS_MODESTRING = "r") { - let file_id: bigint; - let access_mode = ACCESS_MODES[mode]; - let h5_mode = Module[access_mode]; - if (['H5F_ACC_RDWR', 'H5F_ACC_RDONLY'].includes(access_mode)) { - // then it's an existing file... - file_id = Module.ccall("H5Fopen", "bigint", ["string", "number", "bigint"], [filename, h5_mode, 0n]); - } - else { - file_id = Module.ccall("H5Fcreate", "bigint", ["string", "number", "bigint", "bigint"], [filename, h5_mode, 0n, 0n]); - } + constructor(filename: string, mode: ACCESS_MODESTRING = "r", track_order: boolean = false) { + const access_mode = ACCESS_MODES[mode]; + const h5_mode = Module[access_mode]; + const file_id = Module.open(filename, h5_mode, track_order); super(file_id, "/"); this.filename = filename; - this.mode = mode; } @@ -676,10 +900,21 @@ export class File extends Group { } close(): Status { - return Module.ccall("H5Fclose", "number", ["bigint"], [this.file_id]); + return Module.close_file(this.file_id); } } +const calculateHyperslabParams = (shape: number[],ranges: Slice[]) => { + const strides = shape.map((s, i) => BigInt(ranges?.[i]?.[2] ?? 1)); + const count = shape.map((s, i) => { + const N = BigInt((Math.min(s, ranges?.[i]?.[1] ?? s) - Math.max(0, ranges?.[i]?.[0] ?? 0))); + const st = strides[i]; + return N / st + ((N % st) + st - 1n)/st + }); + const offset = shape.map((s, i) => BigInt(Math.min(s, Math.max(0, ranges?.[i]?.[0] ?? 0)))); + return {strides, count, offset} +} + export class Dataset extends HasAttrs { private _metadata?: Metadata; @@ -690,6 +925,14 @@ export class Dataset extends HasAttrs { this.type = OBJECT_TYPE.DATASET; } + refresh() { + const status = Module.refresh_dataset(this.file_id, this.path); + if (status < 0) { + throw new Error(`Could not refresh. Error code: ${status}`); + } + delete this._metadata; + } + get metadata() { if (typeof this._metadata === "undefined") { this._metadata = Module.get_dataset_metadata(this.file_id, this.path); @@ -705,31 +948,38 @@ export class Dataset extends HasAttrs { return this.metadata.shape; } - get value() { + get filters(): Filter[] { + return Module.get_dataset_filters(this.file_id, this.path); + } + + get value(): OutputData | null { return this._value_getter(false); } - get json_value(): JSONCompatibleOutputData { - return this._value_getter(true) as JSONCompatibleOutputData; + get json_value(): JSONCompatibleOutputData | null { + return this._value_getter(true); } - - slice(ranges: Array>) { + + slice(ranges: Slice[]): OutputData | null { // interpret ranges as [start, stop], with one per dim. - let metadata = this.metadata; + const metadata = this.metadata; + // if auto_refresh is on, getting the metadata has triggered a refresh of the dataset_id; const { shape } = metadata; - let ndims = shape.length; - let count = shape.map((s, i) => BigInt(Math.min(s, ranges?.[i]?.[1] ?? s) - Math.max(0, ranges?.[i]?.[0] ?? 0))); - let offset = shape.map((s, i) => BigInt(Math.min(s, Math.max(0, ranges?.[i]?.[0] ?? 0)))); - let total_size = count.reduce((previous, current) => current * previous, 1n); - let nbytes = metadata.size * Number(total_size); - let data_ptr = Module._malloc(nbytes); - var processed; + if (!shape) { + return null; + } + + const {strides, count, offset} = calculateHyperslabParams(shape, ranges); + const total_size = count.reduce((previous, current) => current * previous, 1n); + const nbytes = metadata.size * Number(total_size); + const data_ptr = Module._malloc(nbytes); + let processed: OutputData; try { - Module.get_dataset_data(this.file_id, this.path, count, offset, BigInt(data_ptr)); + Module.get_dataset_data(this.file_id, this.path, count, offset, strides, BigInt(data_ptr)); let data = Module.HEAPU8.slice(data_ptr, data_ptr + nbytes); processed = process_data(data, metadata, false); } finally { - if (metadata.vlen) { + if (metadata.vlen || metadata.type === Module.H5T_class_t.H5T_VLEN.value) { Module.reclaim_vlen_memory(this.file_id, this.path, "", BigInt(data_ptr)); } Module._free(data_ptr); @@ -737,23 +987,114 @@ export class Dataset extends HasAttrs { return processed; } - to_array() { + write_slice(ranges: Slice[], data: any) { + // interpret ranges as [start, stop], with one per dim. + let metadata = this.metadata; + if (!metadata.shape) { + throw new Error("cannot write to a slice of an empty dataset"); + } + if (metadata.vlen) { + throw new Error("writing to a slice of vlen dtype is not implemented"); + } + const { shape } = metadata; + // if auto_refresh is on, getting the metadata has triggered a refresh of the dataset_id; + const {strides, count, offset} = calculateHyperslabParams(shape, ranges); + + const { data: prepared_data, shape: guessed_shape } = prepare_data(data, metadata, count); + let data_ptr = Module._malloc((prepared_data as Uint8Array).byteLength); + Module.HEAPU8.set(prepared_data as Uint8Array, data_ptr); + + try { + Module.set_dataset_data(this.file_id, this.path, count, offset, strides, BigInt(data_ptr)); + } + finally { + Module._free(data_ptr); + } + } + + create_region_reference(ranges: Slice[]) { + const metadata = this.metadata; + if (!metadata.shape) { + throw new Error("cannot create region reference from empty dataset"); + } + + // interpret ranges as [start, stop], with one per dim. + const { shape } = metadata; + const {strides, count, offset} = calculateHyperslabParams(shape, ranges); + const ref_data = Module.create_region_reference(this.file_id, this.path, count, offset, strides); + return new RegionReference(ref_data); + } + + to_array(): JSONCompatibleOutputData | null { const { json_value, metadata } = this; const { shape } = metadata; if (!isIterable(json_value) || typeof json_value === "string") { return json_value; } - let nested = create_nested_array(json_value, shape); + let nested = create_nested_array(json_value, shape); return nested; } - _value_getter(json_compatible=false) { + resize(new_shape: number[]) { + const result = Module.resize_dataset(this.file_id, this.path, new_shape.map(BigInt)); + // reset metadata, pull from file on next read. + this._metadata = undefined; + return result; + } + + make_scale(scale_name: string = "") { + // convert dataset to dimension scale + Module.set_scale(this.file_id, this.path, scale_name); + } + + attach_scale(index: number, scale_dset_path: string) { + // attach a dimension scale to the "index" dimension of this dataset + Module.attach_scale(this.file_id, this.path, scale_dset_path, index); + } + + detach_scale(index: number, scale_dset_path: string) { + // detach a dimension scale from the "index" dimension of this dataset + Module.detach_scale(this.file_id, this.path, scale_dset_path, index); + } + + get_attached_scales(index: number) { + // get full paths to all datasets that are attached as dimension scales + // to the specified dimension (at "index") of this dataset. + return Module.get_attached_scales(this.file_id, this.path, index); + } + + get_scale_name() { + // if this dataset is a dimension scale, returns name as string + // (returns empty string if no name defined, but it is a dimension scale) + // else returns null if it is not set as a dimension scale + return Module.get_scale_name(this.file_id, this.path); + } + + set_dimension_label(index: number, label: string) { + // label dimension at "index" of this dataset with string "label" + Module.set_dimension_label(this.file_id, this.path, index, label) + } + + get_dimension_labels() { + // fetch labels for all dimensions of this dataset (null if label not defined) + return Module.get_dimension_labels(this.file_id, this.path); + } + + _value_getter(json_compatible?: false): OutputData | null; + _value_getter(json_compatible: true): JSONCompatibleOutputData | null; + _value_getter(json_compatible: boolean): OutputData | JSONCompatibleOutputData | null; + _value_getter(json_compatible=false): OutputData | JSONCompatibleOutputData | null { let metadata = this.metadata; + if (!metadata.shape) { + return null + } + + // if auto_refresh is on, getting the metadata has triggered a refresh of the dataset_id; let nbytes = metadata.size * metadata.total_size; let data_ptr = Module._malloc(nbytes); let processed: OutputData; try { - Module.get_dataset_data(this.file_id, this.path, null, null, BigInt(data_ptr)); + Module.get_dataset_data(this.file_id, this.path, null, null, null, BigInt(data_ptr)); let data = Module.HEAPU8.slice(data_ptr, data_ptr + nbytes); processed = process_data(data, metadata, json_compatible); } finally { @@ -767,6 +1108,54 @@ export class Dataset extends HasAttrs { } +export class DatasetRegion { + source_dataset: Dataset; + region_reference: RegionReference; + private _metadata?: Metadata; + + constructor(source_dataset: Dataset, region_reference: RegionReference) { + this.source_dataset = source_dataset; + this.region_reference = region_reference; + } + + get metadata() { + if (typeof this._metadata === "undefined") { + this._metadata = Module.get_region_metadata(this.source_dataset.file_id, this.region_reference.ref_data); + } + return this._metadata; + } + + get value(): OutputData | null { + return this._value_getter(false); + } + + _value_getter(json_compatible?: false): OutputData | null; + _value_getter(json_compatible: true): JSONCompatibleOutputData | null; + _value_getter(json_compatible: boolean): OutputData | JSONCompatibleOutputData | null; + _value_getter(json_compatible=false): OutputData | JSONCompatibleOutputData | null { + let metadata = this.metadata; + if (!metadata.shape) { + return null; + } + + // if auto_refresh is on, getting the metadata has triggered a refresh of the dataset_id; + let nbytes = metadata.size * metadata.total_size; + let data_ptr = Module._malloc(nbytes); + let processed: OutputData; + try { + Module.get_region_data(this.source_dataset.file_id, this.region_reference.ref_data, BigInt(data_ptr)); + let data = Module.HEAPU8.slice(data_ptr, data_ptr + nbytes); + processed = process_data(data, metadata, json_compatible); + } finally { + if (metadata.vlen) { + Module.reclaim_vlen_memory(this.source_dataset.file_id, this.source_dataset.path, "", BigInt(data_ptr)); + } + Module._free(data_ptr); + } + return processed; + } +} + function create_nested_array(value: JSONCompatibleOutputData[], shape: number[]) { // check that shapes match: const total_length = value.length; @@ -780,7 +1169,7 @@ function create_nested_array(value: JSONCompatibleOutputData[], shape: number[]) const subdims = shape.slice(1).reverse(); for (let dim of subdims) { // in each pass, replace input with array of slices of input - const new_output = []; + const new_output: JSONCompatibleOutputData[][] = []; const { length } = output; let cursor = 0; while (cursor < length) { @@ -795,8 +1184,10 @@ export const h5wasm = { File, Group, Dataset, + Datatype, + DatasetRegion, ready, ACCESS_MODES } -export default h5wasm; \ No newline at end of file +export default h5wasm; diff --git a/src/hdf5_util.cc b/src/hdf5_util.cc index 7679dca..cdfc20c 100644 --- a/src/hdf5_util.cc +++ b/src/hdf5_util.cc @@ -1,8 +1,11 @@ #include +#include #include +#include #include "hdf5.h" #include "hdf5_hl.h" +#include "H5PLextern.h" #include #include @@ -13,7 +16,7 @@ using namespace emscripten; EM_JS(void, throw_error, (const char *string_error), { - throw(UTF8ToString(string_error)); + throw new Error(UTF8ToString(string_error)); }); // void throw_error(const char *string_error) { @@ -24,6 +27,37 @@ EM_JS(void, throw_error, (const char *string_error), { // // pass // } +int64_t open(const std::string& filename_string, unsigned int h5_mode = H5F_ACC_RDONLY, bool track_order = false) +{ + const char *filename = filename_string.c_str(); + hid_t file_id; + hid_t fcpl_id = H5Pcreate(H5P_FILE_CREATE); + + if (track_order) + { + H5Pset_link_creation_order(fcpl_id, H5P_CRT_ORDER_TRACKED | H5P_CRT_ORDER_INDEXED); + H5Pset_attr_creation_order(fcpl_id, H5P_CRT_ORDER_TRACKED | H5P_CRT_ORDER_INDEXED); + } + + if (h5_mode == H5F_ACC_TRUNC || h5_mode == H5F_ACC_EXCL) + { + file_id = H5Fcreate(filename, h5_mode, fcpl_id, H5P_DEFAULT); + } + else + { + // then it is an existing file... + file_id = H5Fopen(filename, h5_mode, H5P_DEFAULT); + } + herr_t status = H5Pclose(fcpl_id); + return (int64_t)file_id; +} + +int close_file(hid_t file_id) +{ + herr_t status = H5Fclose(file_id); + return (int)status; +} + herr_t link_name_callback(hid_t loc_id, const char *name, const H5L_info_t *linfo, void *opdata) { std::vector *namelist = reinterpret_cast *>(opdata); @@ -39,7 +73,7 @@ std::vector get_keys_vector(hid_t group_id, H5_index_t index = H5_I return namelist; } -val get_child_names(hid_t loc_id, const std::string& group_name_string) +val get_child_names(hid_t loc_id, const std::string& group_name_string, bool recursive) { hid_t gcpl_id; unsigned crt_order_flags; @@ -59,7 +93,12 @@ val get_child_names(hid_t loc_id, const std::string& group_name_string) H5_index_t index = (crt_order_flags & H5P_CRT_ORDER_INDEXED) ? H5_INDEX_CRT_ORDER : H5_INDEX_NAME; std::vector names_vector; - herr_t idx = H5Literate(grp, index, H5_ITER_INC, NULL, link_name_callback, &names_vector); + if (recursive) { + status = H5Lvisit2(grp, H5_INDEX_NAME, H5_ITER_INC, link_name_callback, &names_vector); + } + else { + status = H5Literate2(grp, index, H5_ITER_INC, NULL, link_name_callback, &names_vector); + } val names = val::array(); size_t numObjs = names_vector.size(); @@ -213,6 +252,10 @@ val get_attribute_names(hid_t loc_id, const std::string& obj_name_string) { ocpl_id = H5Gget_create_plist(obj_id); } + else if (obj_type == H5O_TYPE_NAMED_DATATYPE) + { + ocpl_id = H5Tget_create_plist(obj_id); + } else { ocpl_id = H5Dget_create_plist(obj_id); @@ -253,10 +296,7 @@ val get_dtype_metadata(hid_t dtype) if (dtype_class == H5T_STRING) { attr.set("cset", (int)(H5Tget_cset(dtype))); - } - else - { - attr.set("cset", -1); + attr.set("strpad", (int)(H5Tget_strpad(dtype))); } if (dtype_class == H5T_COMPOUND) @@ -298,6 +338,12 @@ val get_dtype_metadata(hid_t dtype) array_type.set("total_size", total_size); attr.set("array_type", array_type); } + else if (dtype_class == H5T_VLEN) { + hid_t base_dtype = H5Tget_super(dtype); + val vlen_type = get_dtype_metadata(base_dtype); + H5Tclose(base_dtype); + attr.set("vlen_type", vlen_type); + } else if (dtype_class == H5T_ENUM) { val enum_type = val::object(); val members = val::object(); @@ -319,6 +365,11 @@ val get_dtype_metadata(hid_t dtype) enum_type.set("members", members); attr.set("enum_type", enum_type); } + else if (dtype_class == H5T_REFERENCE) + { + std::string ref_type = (H5Tequal(dtype, H5T_STD_REF_OBJ)) ? "object" : "region"; + attr.set("ref_type", ref_type); + } bool littleEndian = (order == H5T_ORDER_LE); attr.set("vlen", (bool)H5Tis_variable_str(dtype)); @@ -328,22 +379,94 @@ val get_dtype_metadata(hid_t dtype) return attr; } -val get_abstractDS_metadata(hid_t dspace, hid_t dtype) +val get_datatype_metadata(hid_t loc_id, const std::string& dtype_name_string) +{ + hid_t dtype_id; + herr_t status; + const char *dtype_name = dtype_name_string.c_str(); + + dtype_id = H5Topen2(loc_id, dtype_name, H5P_DEFAULT); + if (dtype_id < 0) + { + throw_error("error - name not defined!"); + return val::null(); + } + val metadata = get_dtype_metadata(dtype_id); + + H5Tclose(dtype_id); + return metadata; +} + +val get_abstractDS_metadata(hid_t dspace, hid_t dtype, hid_t dcpl) { val attr = get_dtype_metadata(dtype); - int rank = H5Sget_simple_extent_ndims(dspace); + int type = H5Sget_simple_extent_type(dspace); int total_size = H5Sget_simple_extent_npoints(dspace); - hsize_t dims_out[rank]; - int ndims = H5Sget_simple_extent_dims(dspace, dims_out, nullptr); + attr.set("total_size", total_size); + + if (type == H5S_NULL) { + attr.set("shape", val::null()); + attr.set("maxshape", val::null()); + attr.set("chunks", val::null()); + return attr; + } + + int rank = H5Sget_simple_extent_ndims(dspace); + std::vector dims_out(rank); + std::vector maxdims_out(rank); + + int ndims = H5Sget_simple_extent_dims(dspace, dims_out.data(), maxdims_out.data()); + val shape = val::array(); - for (int d = 0; d < ndims; d++) - { - shape.set(d, (uint)dims_out[d]); + val maxshape = val::array(); + for (int d = 0; d < ndims; d++) { + shape.set(d, (uint)dims_out.at(d)); + maxshape.set(d, (uint)maxdims_out.at(d)); } attr.set("shape", shape); - attr.set("total_size", total_size); + attr.set("maxshape", maxshape); + attr.set("chunks", val::null()); + + if (dcpl) { + H5D_layout_t layout = H5Pget_layout(dcpl); + + if (layout == H5D_CHUNKED) { + std::vector chunk_dims_out(ndims); + H5Pget_chunk(dcpl, ndims, chunk_dims_out.data()); + + val chunks = val::array(); + for (int c = 0; c < ndims; c++) { + chunks.set(c, (uint)chunk_dims_out.at(c)); + } + + attr.set("chunks", chunks); + } + + else if (layout == H5D_VIRTUAL) { + val virtual_sources = val::array(); + size_t virtual_count; + ssize_t file_name_size; + ssize_t dset_name_size; + H5Pget_virtual_count(dcpl, &virtual_count); + for (size_t i = 0; i < virtual_count; i++) { + val virtual_source = val::object(); + file_name_size = H5Pget_virtual_filename(dcpl, i, NULL, 0); + dset_name_size = H5Pget_virtual_dsetname(dcpl, i, NULL, 0); + char * file_name = (char *)malloc(file_name_size + 1); + char * dset_name = (char *)malloc(dset_name_size + 1); + H5Pget_virtual_filename(dcpl, i, file_name, file_name_size + 1); + H5Pget_virtual_dsetname(dcpl, i, dset_name, dset_name_size + 1); + virtual_source.set("file_name", std::string(file_name)); + virtual_source.set("dset_name", std::string(dset_name)); + free(file_name); + free(dset_name); + virtual_sources.set(i, virtual_source); + } + attr.set("virtual_sources", virtual_sources); + } + } return attr; } @@ -366,7 +489,7 @@ val get_attribute_metadata(hid_t loc_id, const std::string& group_name_string, c attr_id = H5Aopen_by_name(loc_id, group_name, attribute_name, H5P_DEFAULT, H5P_DEFAULT); dtype = H5Aget_type(attr_id); dspace = H5Aget_space(attr_id); - val metadata = get_abstractDS_metadata(dspace, dtype); + val metadata = get_abstractDS_metadata(dspace, dtype, NULL); H5Aclose(attr_id); H5Sclose(dspace); @@ -374,11 +497,28 @@ val get_attribute_metadata(hid_t loc_id, const std::string& group_name_string, c return metadata; } +int refresh_dataset(hid_t loc_id, const std::string& dataset_name_string) +{ + hid_t ds_id; + herr_t status; + const char *dataset_name = dataset_name_string.c_str(); + + ds_id = H5Dopen2(loc_id, dataset_name, H5P_DEFAULT); + if (ds_id < 0) + { + throw_error("error - name not defined!"); + return -1; + } + status = H5Drefresh(ds_id); + return (int)status; +} + val get_dataset_metadata(hid_t loc_id, const std::string& dataset_name_string) { hid_t ds_id; hid_t dspace; hid_t dtype; + hid_t dcpl; herr_t status; const char *dataset_name = dataset_name_string.c_str(); @@ -390,23 +530,77 @@ val get_dataset_metadata(hid_t loc_id, const std::string& dataset_name_string) } dtype = H5Dget_type(ds_id); dspace = H5Dget_space(ds_id); - val metadata = get_abstractDS_metadata(dspace, dtype); + dcpl = H5Dget_create_plist(ds_id); + val metadata = get_abstractDS_metadata(dspace, dtype, dcpl); H5Dclose(ds_id); H5Sclose(dspace); H5Tclose(dtype); + H5Pclose(dcpl); return metadata; } -int get_dataset_data(hid_t loc_id, const std::string& dataset_name_string, val count_out, val offset_out, uint64_t rdata_uint64) +val get_dataset_filters(hid_t loc_id, const std::string& dataset_name_string) +{ + hid_t ds_id; + hid_t plist_id; + herr_t status; + const char *dataset_name = dataset_name_string.c_str(); + + ds_id = H5Dopen2(loc_id, dataset_name, H5P_DEFAULT); + if (ds_id < 0) + { + throw_error("error - name not defined!"); + return val::null(); + } + + plist_id = H5Dget_create_plist(ds_id); + + val filters = val::array(); + int nfilters = H5Pget_nfilters(plist_id); + for (size_t i = 0; i < nfilters; i++) + { + unsigned int flags; + char name[257]; + size_t nelements = 16; + unsigned int cd_values[16]; + H5Z_filter_t filter_id = H5Pget_filter2(plist_id, i, &flags, &nelements, cd_values, 256, name, NULL); + val cd_values_out = val::array(); + if (nelements > 16) { + unsigned int * full_cd_values = (unsigned int *)malloc(nelements); + H5Pget_filter2(plist_id, i, &flags, &nelements, full_cd_values, 256, name, NULL); + for (size_t i = 0; i < nelements; i++) { + cd_values_out.set(i, full_cd_values[i]); + } + free(full_cd_values); + } + else { + for (size_t i = 0; i < nelements; i++) { + cd_values_out.set(i, cd_values[i]); + } + } + val filter = val::object(); + filter.set("id", filter_id); + filter.set("name", name); + filter.set("cd_values", cd_values_out); + filters.set(i, filter); + } + + H5Dclose(ds_id); + H5Pclose(plist_id); + return filters; +} + +int read_write_dataset_data(hid_t loc_id, const std::string& dataset_name_string, val count_out, val offset_out, val stride_out, uint64_t rwdata_uint64, bool write=false) { hid_t ds_id; hid_t dspace; hid_t dtype; + hid_t memtype; hid_t memspace; herr_t status; const char *dataset_name = dataset_name_string.c_str(); - void *rdata = (void *)rdata_uint64; + void *rwdata = (void *)rwdata_uint64; ds_id = H5Dopen2(loc_id, dataset_name, H5P_DEFAULT); if (ds_id < 0) @@ -415,12 +609,24 @@ int get_dataset_data(hid_t loc_id, const std::string& dataset_name_string, val c return -1; } dspace = H5Dget_space(ds_id); + dtype = H5Dget_type(ds_id); + // assumes that data to write will match type of dataset (exept endianness) + memtype = H5Tcopy(dtype); + // inputs and outputs from javascript will always be little-endian + H5T_order_t dorder = H5Tget_order(dtype); + if (dorder == H5T_ORDER_BE || dorder == H5T_ORDER_VAX) + { + status = H5Tset_order(memtype, H5T_ORDER_LE); + } + if (count_out != val::null() && offset_out != val::null()) { - std::vector count = vecFromJSArray(count_out); - std::vector offset = vecFromJSArray(offset_out); + std::vector count = convertJSArrayToNumberVector(count_out); + std::vector offset = convertJSArrayToNumberVector(offset_out); + std::vector strides = convertJSArrayToNumberVector(stride_out); + memspace = H5Screate_simple(count.size(), &count[0], nullptr); - status = H5Sselect_hyperslab(dspace, H5S_SELECT_SET, &offset[0], NULL, &count[0], NULL); + status = H5Sselect_hyperslab(dspace, H5S_SELECT_SET, &offset[0], &strides[0], &count[0], NULL); status = H5Sselect_all(memspace); } else @@ -429,17 +635,31 @@ int get_dataset_data(hid_t loc_id, const std::string& dataset_name_string, val c memspace = H5Scopy(dspace); } - dtype = H5Dget_type(ds_id); - - status = H5Dread(ds_id, dtype, memspace, dspace, H5P_DEFAULT, rdata); + if (write) { + status = H5Dwrite(ds_id, memtype, memspace, dspace, H5P_DEFAULT, rwdata); + } + else { + status = H5Dread(ds_id, memtype, memspace, dspace, H5P_DEFAULT, rwdata); + } H5Dclose(ds_id); H5Sclose(dspace); H5Sclose(memspace); H5Tclose(dtype); + H5Tclose(memtype); return (int)status; } +int get_dataset_data(hid_t loc_id, const std::string& dataset_name_string, val count_out, val offset_out, val stride_out, uint64_t rdata_uint64) +{ + return read_write_dataset_data(loc_id, dataset_name_string, count_out, offset_out, stride_out, rdata_uint64, false); +} + +int set_dataset_data(hid_t loc_id, const std::string& dataset_name_string, val count_out, val offset_out, val stride_out, uint64_t wdata_uint64) +{ + return read_write_dataset_data(loc_id, dataset_name_string, count_out, offset_out, stride_out, wdata_uint64, true); +} + int reclaim_vlen_memory(hid_t loc_id, const std::string& object_name_string, const std::string& attribute_name_string, uint64_t rdata_uint64) { hid_t ds_id; @@ -475,8 +695,8 @@ int reclaim_vlen_memory(hid_t loc_id, const std::string& object_name_string, con int get_attribute_data(hid_t loc_id, const std::string& group_name_string, const std::string& attribute_name_string, uint64_t rdata_uint64) { hid_t attr_id; - hid_t dspace; hid_t dtype; + hid_t memtype; herr_t status; const char *group_name = &group_name_string[0]; const char *attribute_name = &attribute_name_string[0]; @@ -490,36 +710,75 @@ int get_attribute_data(hid_t loc_id, const std::string& group_name_string, const } attr_id = H5Aopen_by_name(loc_id, group_name, attribute_name, H5P_DEFAULT, H5P_DEFAULT); dtype = H5Aget_type(attr_id); - dspace = H5Aget_space(attr_id); + memtype = H5Tcopy(dtype); + // inputs and outputs from javascript will always be little-endian + H5T_order_t dorder = H5Tget_order(dtype); + if (dorder == H5T_ORDER_BE || dorder == H5T_ORDER_VAX) + { + status = H5Tset_order(memtype, H5T_ORDER_LE); + } - status = H5Sselect_all(dspace); - status = H5Aread(attr_id, dtype, rdata); + status = H5Aread(attr_id, memtype, rdata); H5Aclose(attr_id); - H5Sclose(dspace); H5Tclose(dtype); + H5Tclose(memtype); return (int)status; } -int create_group(hid_t loc_id, std::string grp_name_string) +int create_group(hid_t loc_id, std::string grp_name_string, const bool track_order=false) { - hid_t grp_id = H5Gcreate2(loc_id, grp_name_string.c_str(), H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); - herr_t status = H5Gclose(grp_id); + hid_t gcpl_id = H5Pcreate(H5P_GROUP_CREATE); + if (track_order) + { + H5Pset_link_creation_order(gcpl_id, H5P_CRT_ORDER_TRACKED | H5P_CRT_ORDER_INDEXED); + H5Pset_attr_creation_order(gcpl_id, H5P_CRT_ORDER_TRACKED | H5P_CRT_ORDER_INDEXED); + } + hid_t grp_id = H5Gcreate2(loc_id, grp_name_string.c_str(), H5P_DEFAULT, gcpl_id, H5P_DEFAULT); + herr_t status = H5Pclose(gcpl_id); + status = H5Gclose(grp_id); return (int)status; } -herr_t setup_dataset(val dims_in, int dtype, int dsize, bool is_signed, bool is_vlstr, hid_t *filetype, hid_t *space) +herr_t setup_dataset(val dims_in, val maxdims_in, val chunks_in, int dtype, int dsize, bool is_signed, bool is_vlstr, int compression, val compression_opts, bool track_order, hid_t *filetype, hid_t *space, hid_t *dcpl) { herr_t status; std::vector dims_vec = vecFromJSArray(dims_in); int ndims = dims_vec.size(); hsize_t *dims = dims_vec.data(); + + std::vector maxdims_vec = vecFromJSArray(maxdims_in); + hsize_t *maxdims = maxdims_vec.data(); + /* * Create dataspace. Setting maximum size to NULL sets the maximum * size to be the current size. */ - *space = H5Screate_simple(ndims, dims, NULL); + *space = H5Screate_simple(ndims, dims, maxdims); + + /* + * Create dataset creation property list (dcpl), + * defining chunking if chunks_in is not null + */ + *dcpl = H5Pcreate(H5P_DATASET_CREATE); + + if (chunks_in != val::null()) { + std::vector chunks_vec = vecFromJSArray(chunks_in); + hsize_t *chunks = chunks_vec.data(); + int nchunks = chunks_vec.size(); + H5Pset_chunk(*dcpl, nchunks, chunks); + if (compression != 0) { + std::vector compression_opts_vec = vecFromJSArray(compression_opts); + size_t cd_nelmts = compression_opts_vec.size(); + uint *cd_values = compression_opts_vec.data(); + H5Pset_filter(*dcpl, compression, H5Z_FLAG_MANDATORY, cd_nelmts, cd_values); + } + } + + if (track_order) { + H5Pset_attr_creation_order(*dcpl, H5P_CRT_ORDER_TRACKED | H5P_CRT_ORDER_INDEXED); + } if (dtype == H5T_STRING) { @@ -548,6 +807,15 @@ herr_t setup_dataset(val dims_in, int dtype, int dsize, bool is_signed, bool is_ throw_error("data type not supported"); } } + else if (dtype == H5T_REFERENCE) + { + if (dsize == sizeof(hobj_ref_t)) { + *filetype = H5Tcopy(H5T_STD_REF_OBJ); + } + else if (dsize == sizeof(hdset_reg_ref_t)) { + *filetype = H5Tcopy(H5T_STD_REF_DSETREG); + } + } else { throw_error("data type not supported"); @@ -557,7 +825,7 @@ herr_t setup_dataset(val dims_in, int dtype, int dsize, bool is_signed, bool is_ int create_attribute(hid_t loc_id, std::string obj_name_string, std::string attr_name_string, uint64_t wdata_uint64, val dims_in, int dtype, int dsize, bool is_signed, bool is_vlstr) { - hid_t filetype, space, dset, attr, obj_id; + hid_t filetype, space, dset, attr, obj_id, dcpl; herr_t status; // data is pointer to raw bytes void *wdata = (void *)wdata_uint64; @@ -600,7 +868,7 @@ int create_attribute(hid_t loc_id, std::string obj_name_string, std::string attr // throw_error("data type not supported"); // } - status = setup_dataset(dims_in, dtype, dsize, is_signed, is_vlstr, &filetype, &space); + status = setup_dataset(dims_in, dims_in, val::null(), dtype, dsize, is_signed, is_vlstr, 0, val::null(), false, &filetype, &space, &dcpl); /* * Create the attribute and write the data to it. */ @@ -608,9 +876,6 @@ int create_attribute(hid_t loc_id, std::string obj_name_string, std::string attr attr = H5Acreate(obj_id, attr_name, filetype, space, H5P_DEFAULT, H5P_DEFAULT); status = H5Awrite(attr, filetype, wdata); - if (is_vlstr) { - status = H5Treclaim(filetype, space, H5P_DEFAULT, wdata); - } /* * Close and release resources. */ @@ -618,11 +883,18 @@ int create_attribute(hid_t loc_id, std::string obj_name_string, std::string attr status = H5Sclose(space); status = H5Tclose(filetype); + status = H5Pclose(dcpl); //status = H5Tclose(memtype); status = H5Oclose(obj_id); return (int)status; } +int delete_attribute(hid_t loc_id, const std::string obj_name_string, const std::string attr_name_string) +{ + herr_t status = H5Adelete_by_name(loc_id, obj_name_string.c_str(), attr_name_string.c_str(), H5P_DEFAULT); + return (int) status; +} + int create_vlen_str_attribute(hid_t loc_id, std::string obj_name_string, std::string attr_name_string, val data, val dims_in, int dtype, int dsize, bool is_signed, bool is_vlstr) { uint64_t wdata_uint64; // ptr as uint64_t (webassembly will be 64-bit someday) @@ -649,27 +921,26 @@ int create_vlen_str_attribute(hid_t loc_id, std::string obj_name_string, std::st return create_attribute(loc_id, obj_name_string, attr_name_string, wdata_uint64, dims_in, dtype, dsize, is_signed, is_vlstr); } -int create_dataset(hid_t loc_id, std::string dset_name_string, uint64_t wdata_uint64, val dims_in, int dtype, int dsize, bool is_signed, bool is_vlstr) +int create_dataset(hid_t loc_id, std::string dset_name_string, uint64_t wdata_uint64, val dims_in, val maxdims_in, val chunks_in, int dtype, int dsize, bool is_signed, bool is_vlstr, int compression, val compression_opts, bool track_order=false) { - hid_t filetype, space, dset; + hid_t filetype, space, dset, dcpl; herr_t status; // data is pointer to raw bytes void *wdata = (void *)wdata_uint64; const char *dset_name = dset_name_string.c_str(); - status = setup_dataset(dims_in, dtype, dsize, is_signed, is_vlstr, &filetype, &space); - dset = H5Dcreate2(loc_id, dset_name, filetype, space, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + status = setup_dataset(dims_in, maxdims_in, chunks_in, dtype, dsize, is_signed, is_vlstr, compression, compression_opts, track_order, &filetype, &space, &dcpl); + dset = H5Dcreate2(loc_id, dset_name, filetype, space, H5P_DEFAULT, dcpl, H5P_DEFAULT); status = H5Dwrite(dset, filetype, space, space, H5P_DEFAULT, wdata); - if (is_vlstr) { - status = H5Treclaim(filetype, space, H5P_DEFAULT, wdata); - } + status = H5Dclose(dset); status = H5Sclose(space); status = H5Tclose(filetype); + status = H5Pclose(dcpl); return (int)status; } -int create_vlen_str_dataset(hid_t loc_id, std::string dset_name_string, val data, val dims_in, int dtype, int dsize, bool is_signed, bool is_vlstr) { +int create_vlen_str_dataset(hid_t loc_id, std::string dset_name_string, val data, val dims_in, val maxdims_in, val chunks_in, int dtype, int dsize, bool is_signed, bool is_vlstr, bool track_order=false) { uint64_t wdata_uint64; // ptr as uint64_t (webassembly will be 64-bit someday) std::vector data_string_vec = vecFromJSArray(data); @@ -681,16 +952,390 @@ int create_vlen_str_dataset(hid_t loc_id, std::string dset_name_string, val data } // pass the pointer as an int... wdata_uint64 = (uint64_t)data_char_vec.data(); - return create_dataset(loc_id, dset_name_string, wdata_uint64, dims_in, dtype, dsize, is_signed, is_vlstr); + return create_dataset(loc_id, dset_name_string, wdata_uint64, dims_in, maxdims_in, chunks_in, dtype, dsize, is_signed, is_vlstr, 0, val::null(), track_order); +} + +int resize_dataset(hid_t loc_id, const std::string dset_name_string, val new_size_in) +{ + const char *dset_name = dset_name_string.c_str(); + hid_t dset_id = H5Dopen2(loc_id, dset_name, H5P_DEFAULT); + + std::vector new_size_vec = vecFromJSArray(new_size_in); + hsize_t *new_size = new_size_vec.data(); + + herr_t status = H5Dset_extent(dset_id, new_size); + H5Dclose(dset_id); + return (int) status; +} + +int create_soft_link(hid_t loc_id, std::string link_target_string, std::string link_name_string) { + const char *link_target = link_target_string.c_str(); + const char *link_name = link_name_string.c_str(); + + return H5Lcreate_soft(link_target, loc_id, link_name, H5P_DEFAULT, H5P_DEFAULT); +} + +int create_hard_link(hid_t loc_id, std::string link_target_string, std::string link_name_string) { + // only supports linking target with absolute paths (relative to root) + // will return non-zero value if target does not already exist. + const char *link_target = link_target_string.c_str(); + const char *link_name = link_name_string.c_str(); + + return H5Lcreate_hard(loc_id, link_target, loc_id, link_name, H5P_DEFAULT, H5P_DEFAULT); } +int create_external_link(hid_t loc_id, std::string file_name_string, std::string link_target_string, std::string link_name_string) { + const char *file_name = file_name_string.c_str(); + const char *link_target = link_target_string.c_str(); + const char *link_name = link_name_string.c_str(); + + return H5Lcreate_external(file_name, link_target, loc_id, link_name, H5P_DEFAULT, H5P_DEFAULT); +} + + int flush(hid_t file_id) { herr_t status = H5Fflush(file_id, H5F_SCOPE_GLOBAL); return (int)status; } +val get_plugin_search_paths() +{ + herr_t status; + unsigned int num_paths; + ssize_t path_length; + char *initial_path_buf = {}; + status = H5PLsize(&num_paths); + + val paths = val::array(); + for (unsigned int i = 0; i < num_paths; i++) + { + path_length = H5PLget(i, initial_path_buf, 0); + char * path_buf = (char *)malloc(path_length + 1); + H5PLget(i, path_buf, path_length + 1); + paths.set(i, std::string(path_buf)); + free(path_buf); + } + return paths; +} + +int insert_plugin_search_path(const std::string search_path_string, unsigned int index) +{ + const char *search_path = search_path_string.c_str(); + herr_t status = H5PLinsert(search_path, index); + return (int)status; +} + +// Dimension scales +int set_scale(hid_t loc_id, const std::string& dset_name_string, const std::string& dim_name_string) +{ + const char *dset_name = dset_name_string.c_str(); + const char *dim_name = dim_name_string.c_str(); + hid_t dset_id = H5Dopen2(loc_id, dset_name, H5P_DEFAULT); + herr_t status = H5DSset_scale(dset_id, dim_name); + status = H5Dclose(dset_id); + return (int)status; +} + +int attach_scale(hid_t loc_id, const std::string& target_dset_name_string, const std::string& dimscale_dset_name_string, const unsigned int index) +{ + const char *target_dset_name = target_dset_name_string.c_str(); + const char *dimscale_dset_name = dimscale_dset_name_string.c_str(); + hid_t target_dset_id = H5Dopen2(loc_id, target_dset_name, H5P_DEFAULT); + hid_t dimscale_dset_id = H5Dopen2(loc_id, dimscale_dset_name, H5P_DEFAULT); + herr_t status = H5DSattach_scale(target_dset_id, dimscale_dset_id, index); + status = H5Dclose(target_dset_id); + status = H5Dclose(dimscale_dset_id); + return (int)status; +} + +int detach_scale(hid_t loc_id, const std::string& target_dset_name_string, const std::string& dimscale_dset_name_string, const unsigned int index) +{ + const char *target_dset_name = target_dset_name_string.c_str(); + const char *dimscale_dset_name = dimscale_dset_name_string.c_str(); + hid_t target_dset_id = H5Dopen2(loc_id, target_dset_name, H5P_DEFAULT); + hid_t dimscale_dset_id = H5Dopen2(loc_id, dimscale_dset_name, H5P_DEFAULT); + herr_t status = H5DSdetach_scale(target_dset_id, dimscale_dset_id, index); + status = H5Dclose(target_dset_id); + status = H5Dclose(dimscale_dset_id); + return (int)status; +} + +val get_scale_name(hid_t loc_id, const std::string& dimscale_dset_name_string) +{ + const char *dimscale_dset_name = dimscale_dset_name_string.c_str(); + hid_t dimscale_dset_id = H5Dopen2(loc_id, dimscale_dset_name, H5P_DEFAULT); + htri_t is_scale = H5DSis_scale(dimscale_dset_id); + val output = val::null(); + if (is_scale) + { + ssize_t namesize = H5DSget_scale_name(dimscale_dset_id, nullptr, 0); + if (namesize > 0) + { + char *name = new char[namesize + 1]; + H5DSget_scale_name(dimscale_dset_id, name, namesize + 1); + output = val(std::string(name)); + delete[] name; + } + else + { + output = val(""); + } + } + herr_t status = H5Dclose(dimscale_dset_id); + return output; +} + +herr_t scale_path_callback(hid_t dset_id, unsigned dim, hid_t dimscale_dset_id, void* opdata) +{ + ssize_t pathsize = H5Iget_name(dimscale_dset_id, nullptr, 0); + char *path = new char[pathsize + 1]; + H5Iget_name(dimscale_dset_id, path, pathsize + 1); + std::vector *pathlist = reinterpret_cast *>(opdata); + (*pathlist).push_back(path); + delete[] path; + return 0; +} + +val get_attached_scales(hid_t loc_id, const std::string& target_dset_name_string, const unsigned int index) +{ + // returns paths to all attached scales + const char *target_dset_name = target_dset_name_string.c_str(); + hid_t target_dset_id = H5Dopen2(loc_id, target_dset_name, H5P_DEFAULT); + std::vector paths_vector; + herr_t status = H5DSiterate_scales(target_dset_id, index, nullptr, &scale_path_callback, &paths_vector); + status = H5Dclose(target_dset_id); + + val pathlist = val::array(); + size_t numObjs = paths_vector.size(); + for (size_t i = 0; i < numObjs; i++) + { + pathlist.set(i, paths_vector.at(i)); + } + return pathlist; +} + +int set_dimension_label(hid_t loc_id, const std::string& target_dset_name_string, const unsigned int index, const std::string& label_string) +{ + const char *target_dset_name = target_dset_name_string.c_str(); + const char *label = label_string.c_str(); + hid_t target_dset_id = H5Dopen2(loc_id, target_dset_name, H5P_DEFAULT); + herr_t status = H5DSset_label(target_dset_id, index, label); + H5Dclose(target_dset_id); + return (int)status; +} + +val get_dimension_labels(hid_t loc_id, const std::string& target_dset_name_string) +{ + const char *target_dset_name = target_dset_name_string.c_str(); + hid_t target_dset_id = H5Dopen2(loc_id, target_dset_name, H5P_DEFAULT); + hid_t dspace = H5Dget_space(target_dset_id); + int ndims = H5Sget_simple_extent_dims(dspace, nullptr, nullptr); + val dim_labels = val::array(); + for (int d = 0; d < ndims; d++) + { + ssize_t labelsize = H5DSget_label(target_dset_id, d, nullptr, 0); + if (labelsize > 0) + { + char *label = new char[labelsize + 1]; + H5DSget_label(target_dset_id, d, label, labelsize + 1); + dim_labels.set(d, val(std::string(label))); + delete[] label; + } + else + { + dim_labels.set(d, val::null()); + } + } + herr_t status = H5Dclose(target_dset_id); + status = H5Sclose(dspace); + return dim_labels; +} + +// References +val create_object_reference(hid_t loc_id, const std::string& obj_name_string) +{ + const char *obj_name = obj_name_string.c_str(); + std::vector ref(sizeof(hobj_ref_t)); + herr_t status = H5Rcreate(ref.data(), loc_id, obj_name, H5R_OBJECT, (hid_t)-1); + return val::array(ref); +} + +val create_region_reference(hid_t loc_id, const std::string& dataset_name_string, val count_out, val offset_out, val stride_out) +{ + hid_t ds_id; + hid_t dspace; + std::vector ref(sizeof(hdset_reg_ref_t)); + herr_t status; + const char *dataset_name = dataset_name_string.c_str(); + + ds_id = H5Dopen2(loc_id, dataset_name, H5P_DEFAULT); + dspace = H5Dget_space(ds_id); + if (count_out != val::null() && offset_out != val::null()) + { + std::vector count = convertJSArrayToNumberVector(count_out); + std::vector offset = convertJSArrayToNumberVector(offset_out); + std::vector strides = convertJSArrayToNumberVector(stride_out); + status = H5Sselect_hyperslab(dspace, H5S_SELECT_SET, &offset[0], &strides[0], &count[0], NULL); + } + else + { + status = H5Sselect_all(dspace); + } + status = H5Rcreate(ref.data(), loc_id, dataset_name, H5R_DATASET_REGION, dspace); + H5Sclose(dspace); + H5Dclose(ds_id); + return val::array(ref); +} + +val get_referenced_name(hid_t loc_id, const val ref_data_in, const bool is_object) +{ + ssize_t namesize = 0; + std::vector ref_data_vec = convertJSArrayToNumberVector(ref_data_in); + const hobj_ref_t *ref_ptr = (hobj_ref_t *)ref_data_vec.data(); + val output = val::null(); + const H5R_type_t ref_type = (is_object) ? H5R_OBJECT : H5R_DATASET_REGION; + hid_t object_id = H5Rdereference2(loc_id, H5P_DEFAULT, ref_type, ref_ptr); + namesize = H5Iget_name(object_id, nullptr, 0); + if (namesize > 0) + { + char *name = new char[namesize + 1]; + H5Iget_name(object_id, name, namesize + 1); + + output = val::u8string(name); + delete[] name; + } + H5Oclose(object_id); + return output; +} + +val get_region_metadata(hid_t loc_id, const val ref_data_in) +{ + hid_t dspace; + hid_t dtype; + hid_t dcpl; + herr_t status; + const std::vector ref_data_vec = convertJSArrayToNumberVector(ref_data_in); + const hdset_reg_ref_t *ref_ptr = (hdset_reg_ref_t *)ref_data_vec.data(); + hid_t ds_id = H5Rdereference2(loc_id, H5P_DEFAULT, H5R_DATASET_REGION, ref_ptr); + + dtype = H5Dget_type(ds_id); + dspace = H5Rget_region(ds_id, H5R_DATASET_REGION, ref_ptr); + dcpl = H5Dget_create_plist(ds_id); + // fill in shape, maxshape, chunks, total_size + val metadata = get_abstractDS_metadata(dspace, dtype, dcpl); + // then override the ones that are specific to a region: + int total_size = H5Sget_select_npoints(dspace); + metadata.set("total_size", total_size); + + int rank = H5Sget_simple_extent_ndims(dspace); + // shape will be null if the selection is not a regular hyperslab + val shape = val::null(); + htri_t is_regular = H5Sis_regular_hyperslab(dspace); + if (is_regular > 0) + { + std::vector count(rank); + std::vector block(rank); + htri_t success = H5Sget_regular_hyperslab(dspace, nullptr, nullptr, count.data(), block.data()); + shape = val::array(); + for (int d = 0; d < rank; d++) + { + int blocksize = (block.at(d) == NULL) ? 1 : block.at(d); + shape.set(d, (uint)(count.at(d) * blocksize)); + } + } + metadata.set("shape", shape); + H5Dclose(ds_id); + H5Sclose(dspace); + H5Tclose(dtype); + H5Pclose(dcpl); + return metadata; +} + +int get_region_data(hid_t loc_id, val ref_data_in, uint64_t rdata_uint64) +{ + hid_t ds_id; + hid_t dspace; + hid_t dtype; + hid_t memtype; + hid_t memspace; + herr_t status; + void *rdata = (void *)rdata_uint64; + const std::vector ref_data_vec = convertJSArrayToNumberVector(ref_data_in); + const hdset_reg_ref_t *ref_ptr = (hdset_reg_ref_t *)ref_data_vec.data(); + ds_id = H5Rdereference2(loc_id, H5P_DEFAULT, H5R_DATASET_REGION, ref_ptr); + dspace = H5Rget_region(ds_id, H5R_DATASET_REGION, ref_ptr); + dtype = H5Dget_type(ds_id); + // assumes that data to write will match type of dataset (exept endianness) + memtype = H5Tcopy(dtype); + // inputs and outputs from javascript will always be little-endian + H5T_order_t dorder = H5Tget_order(dtype); + if (dorder == H5T_ORDER_BE || dorder == H5T_ORDER_VAX) + { + status = H5Tset_order(memtype, H5T_ORDER_LE); + } + int rank = H5Sget_simple_extent_ndims(dspace); + htri_t is_regular = H5Sis_regular_hyperslab(dspace); + if (is_regular > 0) + { + std::vector count(rank); + std::vector block(rank); + std::vector shape_out(rank); + htri_t success = H5Sget_regular_hyperslab(dspace, nullptr, nullptr, count.data(), block.data()); + for (int d = 0; d < rank; d++) + { + int blocksize = (block.at(d) == NULL) ? 1 : block.at(d); + shape_out.at(d) = (count.at(d) * blocksize); + } + memspace = H5Screate_simple(shape_out.size(), &shape_out[0], nullptr); + } + else + { + hsize_t total_size = H5Sget_select_npoints(dspace); + memspace = H5Screate_simple(1, &total_size, nullptr); + } + status = H5Dread(ds_id, memtype, memspace, dspace, H5P_DEFAULT, rdata); + H5Dclose(ds_id); + H5Sclose(dspace); + H5Sclose(memspace); + H5Tclose(dtype); + H5Tclose(memtype); + return (int)status; +} + +herr_t throwing_error_handler(hid_t estack, void *client_data) +{ + FILE *error_file = tmpfile(); + herr_t status = H5Eprint2(estack, error_file); + rewind(error_file); + std::stringstream output_stream; + char line[256]; + while (fgets(line, sizeof(line), error_file) != NULL) { + output_stream << line; + } + std::string error_message = output_stream.str(); + throw_error(error_message.c_str()); + fclose(error_file); + return 0; +} + +H5E_auto2_t default_error_handler; +void *default_error_handler_client_data; +herr_t error_handler_get_result = H5Eget_auto2(H5E_DEFAULT, &default_error_handler, &default_error_handler_client_data); + +int activate_throwing_error_handler() { + herr_t error_handler_set_result = H5Eset_auto2(H5E_DEFAULT, throwing_error_handler, NULL); + return (int)error_handler_set_result; +} + +int deactivate_throwing_error_handler() { + herr_t error_handler_set_result = H5Eset_auto2(H5E_DEFAULT, default_error_handler, default_error_handler_client_data); + return (int)error_handler_set_result; +} + EMSCRIPTEN_BINDINGS(hdf5) { + function("open", &open); + function("close_file", &close_file); function("get_keys", &get_keys_vector); function("get_names", &get_child_names); function("get_types", &get_child_types); @@ -700,15 +1345,41 @@ EMSCRIPTEN_BINDINGS(hdf5) function("get_attribute_names", &get_attribute_names); function("get_attribute_metadata", &get_attribute_metadata); function("get_dataset_metadata", &get_dataset_metadata); + function("get_datatype_metadata", &get_datatype_metadata); + function("get_dataset_filters", &get_dataset_filters); + function("refresh_dataset", &refresh_dataset); function("get_dataset_data", &get_dataset_data); + function("set_dataset_data", &set_dataset_data); function("get_attribute_data", &get_attribute_data); function("reclaim_vlen_memory", &reclaim_vlen_memory); function("create_group", &create_group); function("create_dataset", &create_dataset); + function("resize_dataset", &resize_dataset); function("create_attribute", &create_attribute, allow_raw_pointers()); + function("delete_attribute", &delete_attribute); function("create_vlen_str_attribute", &create_vlen_str_attribute); function("create_vlen_str_dataset", &create_vlen_str_dataset); + function("create_soft_link", &create_soft_link); + function("create_hard_link", &create_hard_link); + function("create_external_link", &create_external_link); function("flush", &flush); + function("get_plugin_search_paths", &get_plugin_search_paths); + function("insert_plugin_search_path", &insert_plugin_search_path); + function("remove_plugin_search_path", &H5PLremove); + function("set_scale", &set_scale); + function("attach_scale", &attach_scale); + function("detach_scale", &detach_scale); + function("get_scale_name", &get_scale_name); + function("get_attached_scales", &get_attached_scales); + function("get_dimension_labels", &get_dimension_labels); + function("set_dimension_label", &set_dimension_label); + function("create_object_reference", &create_object_reference); + function("create_region_reference", &create_region_reference); + function("get_referenced_name", &get_referenced_name); + function("get_region_metadata", &get_region_metadata); + function("get_region_data", &get_region_data); + function("activate_throwing_error_handler", &activate_throwing_error_handler); + function("deactivate_throwing_error_handler", &deactivate_throwing_error_handler); class_("H5L_info2_t") .constructor<>() @@ -746,6 +1417,7 @@ EMSCRIPTEN_BINDINGS(hdf5) constant("H5F_ACC_EXCL", H5F_ACC_EXCL); constant("H5F_ACC_CREAT", H5F_ACC_CREAT); constant("H5F_ACC_SWMR_WRITE", H5F_ACC_SWMR_WRITE); + constant("H5F_ACC_SWMR_READ", H5F_ACC_SWMR_READ); constant("H5G_GROUP", (int)H5G_GROUP); // 0 Object is a group. @@ -758,6 +1430,18 @@ EMSCRIPTEN_BINDINGS(hdf5) constant("H5O_TYPE_GROUP", (int)H5O_TYPE_GROUP); constant("H5O_TYPE_DATASET", (int)H5O_TYPE_DATASET); constant("H5O_TYPE_NAMED_DATATYPE", (int)H5O_TYPE_NAMED_DATATYPE); + constant("SIZEOF_OBJ_REF", (int)(sizeof(hobj_ref_t))); + constant("SIZEOF_DSET_REGION_REF", (int)(sizeof(hdset_reg_ref_t))); + + constant("H5Z_FILTER_NONE", H5Z_FILTER_NONE); + constant("H5Z_FILTER_DEFLATE", H5Z_FILTER_DEFLATE); + constant("H5Z_FILTER_SHUFFLE", H5Z_FILTER_SHUFFLE); + constant("H5Z_FILTER_FLETCHER32", H5Z_FILTER_FLETCHER32); + constant("H5Z_FILTER_SZIP", H5Z_FILTER_SZIP); + constant("H5Z_FILTER_NBIT", H5Z_FILTER_NBIT); + constant("H5Z_FILTER_SCALEOFFSET", H5Z_FILTER_SCALEOFFSET); + constant("H5Z_FILTER_RESERVED", H5Z_FILTER_RESERVED); + constant("H5Z_FILTER_MAX", H5Z_FILTER_MAX); register_vector("vector"); } diff --git a/src/hdf5_util_helpers.d.ts b/src/hdf5_util_helpers.d.ts index 956635b..bcf7fe0 100644 --- a/src/hdf5_util_helpers.d.ts +++ b/src/hdf5_util_helpers.d.ts @@ -19,15 +19,21 @@ export interface H5T_class_t { export interface Metadata { array_type?: Metadata, + chunks: number[] | null, compound_type?: CompoundTypeMetadata, - cset: number, + cset?: number, enum_type?: EnumTypeMetadata, + vlen_type?: Metadata, littleEndian: boolean, - shape: Array, + maxshape: number[] | null, + ref_type?: 'object' | 'region', + shape: number[] | null, signed: boolean, size: number, + strpad?: number, total_size: number, type: number, + virtual_sources?: VirtualSource[], vlen: boolean, } @@ -37,7 +43,7 @@ export interface CompoundMember extends Metadata { } export interface CompoundTypeMetadata { - members: Array + members: CompoundMember[] nmembers: number; } @@ -47,13 +53,25 @@ export interface EnumTypeMetadata { type: number; } +export interface VirtualSource { + file_name: string; + dset_name: string; +} + export interface H5Module extends EmscriptenModule { - create_dataset(file_id: bigint, arg1: string, arg2: bigint, arg3: bigint[], type: number, size: number, signed: boolean, vlen: boolean): number; + open(filename: string, mode?: number, track_order?: boolean): bigint; + close_file(file_id: bigint): number; + create_dataset(file_id: bigint, arg1: string, arg2: bigint, shape: bigint[], maxshape: (bigint | null)[], chunks: bigint[] | null, type: number, size: number, signed: boolean, vlen: boolean, compression_id: number, compression_opts: number[], track_order?: boolean): number; + create_soft_link(file_id: bigint, link_target: string, link_name: string): number; + create_hard_link(file_id: bigint, link_target: string, link_name: string): number; + create_external_link(file_id: bigint, file_name: string, link_target: string, link_name: string): number; get_type(file_id: bigint, obj_path: string): number; get_symbolic_link(file_id: bigint, obj_path: string): string; get_external_link(file_id: bigint, obj_path: string): {filename: string, obj_path: string}; H5O_TYPE_DATASET: number; H5O_TYPE_GROUP: number; + SIZEOF_OBJ_REF: number; + SIZEOF_DSET_REGION_REF: number; H5G_GROUP: number; H5G_DATASET: number; H5G_TYPE: number; @@ -66,15 +84,30 @@ export interface H5Module extends EmscriptenModule { H5F_ACC_CREAT: 16; H5F_ACC_SWMR_WRITE: 32; H5F_ACC_SWMR_READ: 64; - create_group(file_id: bigint, name: string): number; - create_vlen_str_dataset(file_id: bigint, dset_name: string, prepared_data: any, arg3: any, type: number, size: number, signed: boolean, vlen: boolean): number; - get_dataset_data(file_id: bigint, path: string, arg2: bigint[] | null, arg3: bigint[] | null, arg4: bigint): number; + H5Z_FILTER_NONE: 0; + H5Z_FILTER_DEFLATE: 1; + H5Z_FILTER_SHUFFLE: 2; + H5Z_FILTER_FLETCHER32: 3; + H5Z_FILTER_SZIP: 4; + H5Z_FILTER_NBIT: 5; + H5Z_FILTER_SCALEOFFSET: 6; + H5Z_FILTER_RESERVED: 256; + H5Z_FILTER_MAX: 65535; + create_group(file_id: bigint, name: string, track_order?: boolean): number; + create_vlen_str_dataset(file_id: bigint, dset_name: string, prepared_data: any, shape: bigint[], maxshape: (bigint | null)[], chunks: bigint[] | null, type: number, size: number, signed: boolean, vlen: boolean, track_order?: boolean): number; + get_dataset_data(file_id: bigint, path: string, count: bigint[] | null, offset: bigint[] | null, strides: bigint[] | null, rdata_ptr: bigint): number; + set_dataset_data(file_id: bigint, path: string, count: bigint[] | null, offset: bigint[] | null, strides: bigint[] | null, wdata_ptr: bigint): number; + refresh_dataset(file_id: bigint, path: string): number; + resize_dataset(file_id: bigint, path: string, new_size: bigint[]): number; get_dataset_metadata(file_id: bigint, path: string): Metadata; + get_datatype_metadata(file_id: bigint, path: string): Metadata; + get_dataset_filters(file_id: bigint, path: string): Filter[]; flush(file_id: bigint): number; ccall: typeof ccall; - get_names(file_id: bigint, path: string): string[]; + get_names(file_id: bigint, path: string, recursive: boolean): string[]; create_attribute(file_id: bigint, path: string, name: any, arg3: bigint, arg4: any, type: number, size: number, signed: boolean, vlen: boolean): number; - create_vlen_str_attribute(file_id: bigint, path: string, name: any, prepared_data: any, arg4: any, type: number, size: number, signed: boolean, vlen: boolean): number; + delete_attribute(file_id: bigint, path: string, name: string): number; + create_vlen_str_attribute(file_id: bigint, path: string, name: any, prepared_data: any, shape: bigint[], type: number, size: number, signed: boolean, vlen: boolean): number; get_attribute_names(file_id: any, path: any): string[]; // things from ModuleFactory: UTF8ToString(ptr: number): string, @@ -83,6 +116,29 @@ export interface H5Module extends EmscriptenModule { reclaim_vlen_memory(file_id: BigInt, obj_name: string, attr_name: string, data_ptr: bigint): Status; get_attribute_data(file_id: BigInt, obj_name: string, attr_name: string, arg3: bigint): Status; FS: FS.FileSystemType, - get_keys_vector(group_id: bigint, H5_index_t: number): Array, - get_attribute_metadata(loc_id: bigint, group_name_string: string, attribute_name_string: string): Metadata + get_keys_vector(group_id: bigint, H5_index_t?: number): string[], + get_attribute_metadata(loc_id: bigint, group_name_string: string, attribute_name_string: string): Metadata, + get_plugin_search_paths(): string[], + insert_plugin_search_path(search_path: string, index: number): number, + remove_plugin_search_path(index: number): number, + set_scale(loc_id: bigint, dset_name: string, dim_name: string): number, + attach_scale(loc_id: bigint, target_dset_name: string, dimscale_dset_name: string, index: number): number, + detach_scale(loc_id: bigint, target_dset_name: string, dimscale_dset_name: string, index: number): number, + get_scale_name(loc_id: bigint, dimscale_dset_name: string): string | null, + get_attached_scales(loc_id: bigint, target_dset_name: string, index: number): string[], + set_dimension_label(loc_id: bigint, target_dset_name: string, index: number, label: string): number, + get_dimension_labels(loc_id: bigint, target_dset_name: string): (string | null)[], + create_object_reference(loc_id: bigint, target_name: string): Uint8Array, + create_region_reference(file_id: bigint, path: string, count: bigint[] | null, offset: bigint[] | null, strides: bigint[] | null): Uint8Array, + get_referenced_name(loc_id: bigint, ref_ptr: Uint8Array, is_object: boolean): string; + get_region_metadata(loc_id: bigint, ref_ptr: Uint8Array): Metadata; + get_region_data(loc_id: bigint, ref_data: Uint8Array, rdata_ptr: bigint): number; + activate_throwing_error_handler(): number; + deactivate_throwing_error_handler(): number; +} + +export declare type Filter = { + id: number; + name: string; + cd_values: number[]; } diff --git a/test/array.h5 b/test/array.h5 index a719a08..28148ce 100644 Binary files a/test/array.h5 and b/test/array.h5 differ diff --git a/test/bigendian_read.mjs b/test/bigendian_read.mjs new file mode 100644 index 0000000..bf287d7 --- /dev/null +++ b/test/bigendian_read.mjs @@ -0,0 +1,25 @@ +#!/usr/bin/env node + +import { strict as assert } from 'assert'; +import h5wasm from 'h5wasm/node'; + +async function bigendian_read() { + await h5wasm.ready; + var f = new h5wasm.File('./test/array.h5', 'r'); + + const dset = f.get('bigendian'); + assert.equal(dset.metadata.littleEndian, false); + assert.deepEqual([...dset.value].map(Number), [3,2,1]); + + const attr = dset.attrs['bigendian_attr']; + assert.equal(attr.metadata.littleEndian, false); + assert.deepEqual([...attr.value].map(Number), [3,2,1]); +} + +export const tests = [ + { + description: 'Read big-endian dataset', + test: bigendian_read, + }, +]; +export default tests; diff --git a/test/bool_test.mjs b/test/bool_test.mjs index 4c92367..d77e68e 100644 --- a/test/bool_test.mjs +++ b/test/bool_test.mjs @@ -1,7 +1,7 @@ #!/usr/bin/env node import { strict as assert } from 'assert'; -import h5wasm from '../dist/node/hdf5_hl.js'; +import h5wasm from 'h5wasm/node'; async function bool_test() { await h5wasm.ready; @@ -9,11 +9,16 @@ async function bool_test() { assert.deepEqual( f.get('bool').value, + new Int8Array([ 0, 1, 1, 0 ]) + ); + + assert.deepEqual( + f.get('bool').json_value, [ false, true, true, false ] ); assert.deepEqual(f.get('bool').metadata, { - cset: -1, + chunks: null, enum_type: { type: 0, nmembers: 2, @@ -23,6 +28,7 @@ async function bool_test() { } }, littleEndian: true, + maxshape: [2, 2], shape: [2, 2], signed: true, size: 1, diff --git a/test/chunks_resize.mjs b/test/chunks_resize.mjs new file mode 100644 index 0000000..b0ca03e --- /dev/null +++ b/test/chunks_resize.mjs @@ -0,0 +1,49 @@ +#!/usr/bin/env node + +import { strict as assert } from 'assert'; +import { existsSync, mkdirSync, unlinkSync } from 'fs'; +import { join } from 'path'; +import h5wasm from "h5wasm/node"; + +async function readwrite_chunked() { + + await h5wasm.ready; + const PATH = join(".", "test", "tmp"); + const FILEPATH = join(PATH, "chunked.h5"); + const DSET_NAME = "chunked"; + const DTYPE = " read_file.dereference(ref)); + assert(obj_0 instanceof h5wasm.Group); + assert.strictEqual(obj_0.path, `/${DATASET_GROUP}`); + assert(obj_1 instanceof h5wasm.Dataset); + assert.strictEqual(obj_1.path, `/${DATASET_GROUP}/${DATASET_NAME}`); + + const region_refs = refs_group.get(REGION_REF_DATASET_NAME).value; + const [region_0, region_1] = region_refs.map((ref) => read_file.dereference(ref)); + assert(region_0 instanceof h5wasm.DatasetRegion); + assert.deepEqual(region_0.value, new Float32Array(REGION_REF_DATA_0.flat())); + assert(region_1 instanceof h5wasm.DatasetRegion); + assert.deepEqual(region_1.value, new Float32Array(REGION_REF_DATA_1.flat())); + // assert.deepEqual(hard_link_dataset.value, DATA); + + read_file.close() + } + + // cleanup file when finished: + unlinkSync(FILEPATH); + +} + +export const tests = [ + { + description: "Create and read object and region references", + test: test_refs + }, +]; + +export default tests; diff --git a/test/datatype_test.mjs b/test/datatype_test.mjs new file mode 100644 index 0000000..857770c --- /dev/null +++ b/test/datatype_test.mjs @@ -0,0 +1,33 @@ +#!/usr/bin/env node + +import { strict as assert } from 'assert'; +import h5wasm from 'h5wasm/node'; + +async function datatype_test() { + await h5wasm.ready; + var f = new h5wasm.File('./test/array.h5', 'r'); + + const datatype = f.get('datatype/value'); + assert(datatype instanceof h5wasm.Datatype); + assert.deepEqual(datatype.metadata, { + signed: false, + type: 3, + cset: 0, + strpad: 1, + vlen: false, + littleEndian: false, + size: 10 + }); + + assert.deepEqual(Object.keys(datatype.attrs), ['named_dtype_attr']); + assert.deepEqual(datatype.attrs['named_dtype_attr'].value, + 'An attribute of a named datatype'); +} + +export const tests = [ + { + description: 'Read datatypes', + test: datatype_test, + }, +]; +export default tests; diff --git a/test/dimension_labels.mjs b/test/dimension_labels.mjs new file mode 100644 index 0000000..634f5c8 --- /dev/null +++ b/test/dimension_labels.mjs @@ -0,0 +1,60 @@ +#!/usr/bin/env node + +import { strict as assert } from 'assert'; +import { existsSync, mkdirSync, unlinkSync } from 'fs'; +import { join } from 'path'; +import h5wasm from "h5wasm/node"; + +async function test_dimension_labels() { + + await h5wasm.ready; + const PATH = join(".", "test", "tmp"); + const FILEPATH = join(PATH, "dimension_labels.h5"); + const VALUES = [6,5,4,3,2,1]; + const DATA = new Float32Array(VALUES); + const SHAPE = [2,3]; + const DATASET_NAME = "data"; + const DIM_INDEX = 1; + const DIM_LABEL = "y"; + + if (!(existsSync(PATH))) { + mkdirSync(PATH); + } + + // write + { + const write_file = new h5wasm.File(FILEPATH, "w"); + + const dataset = write_file.create_dataset({name: DATASET_NAME, data: DATA, shape: SHAPE}); + dataset.set_dimension_label(DIM_INDEX, DIM_LABEL); + + write_file.flush(); + write_file.close(); + } + + // read + { + const read_file = new h5wasm.File(FILEPATH, "r"); + + const dataset = read_file.get(DATASET_NAME); + assert(dataset instanceof h5wasm.Dataset); + + const labels = dataset.get_dimension_labels(); + assert.deepEqual(labels, [null, DIM_LABEL]); + + read_file.close() + } + + // cleanup file when finished: + unlinkSync(FILEPATH); + +} + +export const tests = [ + { + description: "Create and read dimension labels", + test: test_dimension_labels + }, +]; + +export default tests; diff --git a/test/dimension_scales.mjs b/test/dimension_scales.mjs new file mode 100644 index 0000000..77601e6 --- /dev/null +++ b/test/dimension_scales.mjs @@ -0,0 +1,79 @@ +#!/usr/bin/env node + +import { strict as assert } from 'assert'; +import { existsSync, mkdirSync, unlinkSync } from 'fs'; +import { join } from 'path'; +import h5wasm from "h5wasm/node"; + +async function test_dimension_scales() { + + await h5wasm.ready; + const PATH = join(".", "test", "tmp"); + const FILEPATH = join(PATH, "dimension_labels.h5"); + const VALUES = [6,5,4,3,2,1]; + const DATA = new Float32Array(VALUES); + const DIMSCALE_DATA = [10, 12, 14]; + const SHAPE = [2,3]; + const DATASET_NAME = "data"; + const DIMSCALE_NAME = "y"; + const DIMSCALE_DATASET_NAME = "dimscale"; + const DIM_INDEX = 1; + + if (!(existsSync(PATH))) { + mkdirSync(PATH); + } + + // write + { + const write_file = new h5wasm.File(FILEPATH, "w"); + + const dataset = write_file.create_dataset({name: DATASET_NAME, data: DATA, shape: SHAPE}); + const dimscale_dataset = write_file.create_dataset({name: DIMSCALE_DATASET_NAME, data: DIMSCALE_DATA}); + + dimscale_dataset.make_scale(DIMSCALE_NAME); + dataset.attach_scale(DIM_INDEX, DIMSCALE_DATASET_NAME); + + write_file.flush(); + write_file.close(); + } + + // read + { + const read_file = new h5wasm.File(FILEPATH, "r"); + + const dataset = read_file.get(DATASET_NAME); + assert(dataset instanceof h5wasm.Dataset); + + // dataset is not a dimension scale: + const not_scale_name = dataset.get_scale_name(); + assert.strictEqual(not_scale_name, null); + + // no scales attached to dimension 0: + assert.deepEqual(dataset.get_attached_scales(0), []); + + const attached_scales = dataset.get_attached_scales(DIM_INDEX); + // returns full path, including leading slash + assert.deepEqual(attached_scales, [`/${DIMSCALE_DATASET_NAME}`]); + + const dimscale_dataset = read_file.get(attached_scales[0]); + assert(dimscale_dataset instanceof h5wasm.Dataset); + + const scale_name = dimscale_dataset.get_scale_name(); + assert.strictEqual(scale_name, DIMSCALE_NAME); + + read_file.close() + } + + // cleanup file when finished: + unlinkSync(FILEPATH); + +} + +export const tests = [ + { + description: "Create and read dimension scales", + test: test_dimension_scales + }, +]; + +export default tests; diff --git a/test/empty.h5 b/test/empty.h5 new file mode 100644 index 0000000..2be2aaa Binary files /dev/null and b/test/empty.h5 differ diff --git a/test/empty_dataset_and_attrs.mjs b/test/empty_dataset_and_attrs.mjs new file mode 100644 index 0000000..852a5f0 --- /dev/null +++ b/test/empty_dataset_and_attrs.mjs @@ -0,0 +1,40 @@ +#!/usr/bin/env node + +import { strict as assert } from 'assert'; +import { existsSync, mkdirSync, unlinkSync } from 'fs'; +import { join } from 'path'; +import h5wasm from "h5wasm/node"; + +async function test_empty() { + + await h5wasm.ready; + const PATH = join(".", "test"); + const FILEPATH = join(PATH, "empty.h5"); + const DATASET_NAME = "empty_dataset"; + const ATTR_NAME = "empty_attr"; + + // read + { + const read_file = new h5wasm.File(FILEPATH, "r"); + + const dataset = read_file.get(DATASET_NAME); + assert(dataset instanceof h5wasm.Dataset); + assert.deepStrictEqual(dataset.shape, null); + assert.deepStrictEqual(dataset.value, null); + + assert(ATTR_NAME in read_file.attrs); + assert.deepStrictEqual(read_file.get_attribute(ATTR_NAME), null); + + read_file.close() + } + +} + +export const tests = [ + { + description: "Read empty datasets and attributes", + test: test_empty + }, +]; + +export default tests; diff --git a/test/filters_test.mjs b/test/filters_test.mjs new file mode 100644 index 0000000..25f0e7e --- /dev/null +++ b/test/filters_test.mjs @@ -0,0 +1,32 @@ +#!/usr/bin/env node + +import { strict as assert } from "assert"; +import h5wasm from "h5wasm/node"; + +async function filters_test() { + await h5wasm.ready; + const f = new h5wasm.File("./test/compressed.h5", "r"); + + assert.deepEqual(f.get("gzip").filters, [{ id: 1, name: "deflate", cd_values: [4] }]); + + assert.deepEqual(f.get("gzip_shuffle").filters, [ + { id: 2, name: "shuffle", cd_values: [8] }, + { id: 1, name: "deflate", cd_values: [4] }, + ]); + + assert.deepEqual(f.get("scaleoffset").filters, [ + { + id: 6, + name: "scaleoffset", + cd_values: [0, 4, 1250, 1, 8, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + }, + ]); +} + +export const tests = [ + { + description: "Read dataset compression filters", + test: filters_test, + }, +]; +export default tests; diff --git a/test/make_test_files.py b/test/make_test_files.py index eb2cbdc..56c0f84 100644 --- a/test/make_test_files.py +++ b/test/make_test_files.py @@ -1,22 +1,45 @@ import h5py import numpy as np -array_type = h5py.h5t.array_create(h5py.h5t.py_create(' { /*global*/ tests = tests.concat(tests_in)} @@ -13,7 +28,23 @@ add_tests(bool_test); add_tests(compound_and_array_tests); add_tests(create_dataset); add_tests(read_to_array); +add_tests(datatype_test); +add_tests(filters_test); +add_tests(test_links); +add_tests(test_attributes); +add_tests(test_overwrite_dataset); +add_tests(create_chunked); +add_tests(bigendian_read); +add_tests(create_compressed); +add_tests(dimension_labels); +add_tests(dimension_scales); +add_tests(references); +add_tests(test_throwing_error_handler); +add_tests(test_empty); +add_tests(vlen_test); +add_tests(track_order); +let passed = true; async function run_test(test) { try { await test.test(); @@ -22,6 +53,7 @@ async function run_test(test) { catch (error) { console.log('x', test.description); console.log(error.stack); + passed = false; } } @@ -32,3 +64,6 @@ async function run_tests(tests) { } await run_tests(tests); +if (!passed) { + throw new Error("Tests did not complete successfuly"); +} diff --git a/test/test_throwing_error_handler.mjs b/test/test_throwing_error_handler.mjs new file mode 100644 index 0000000..53a68fc --- /dev/null +++ b/test/test_throwing_error_handler.mjs @@ -0,0 +1,58 @@ +#!/usr/bin/env node +import { strict as assert } from 'assert'; +import { join } from 'path'; +import { spawnSync } from 'child_process'; +import h5wasm from "h5wasm/node"; + +async function test_throwing_error_handler() { + const Module = await h5wasm.ready; + const error_handler_set_result = Module.activate_throwing_error_handler(); + assert.equal(error_handler_set_result, 0); // success + + const PATH = join(".", "test", "tmp"); + const FILEPATH = join(PATH, "nonexistant_file.h5"); + + function open_nonexistant_file() { + const file = new h5wasm.File(FILEPATH, "r"); + return file; + } + + let error_message = "no error"; + try { + const file = open_nonexistant_file(); + } + catch (e) { + error_message = e.toString(); + } + const lines = error_message.split("\n"); + assert(lines.length > 2, "error message should have more than 2 lines"); + const expected_error_message = "H5Fopen(): unable to synchronously open file"; + assert(lines[1].endsWith(expected_error_message), "unexpected error message: " + lines[1]); +} + +async function test_default_error_handler() { + // verify that the default error handler does not throw, + // and that it prints an error message to stderr + const child_process = spawnSync("/usr/bin/env", ["node", "test/nonthrowing_error_handler.mjs"]); + const { stderr, status } = child_process; + // status should be zero, no error thrown: + assert.strictEqual(status, 0); + const error_message = stderr.toString(); + const lines = error_message.split("\n"); + assert(lines.length > 2, "error message should have more than 2 lines"); + const expected_error_message = "H5Fopen(): unable to synchronously open file"; + assert(lines[1].endsWith(expected_error_message), "unexpected error message: " + lines[1]); +} + +export const tests = [ + { + description: "test throwing error handler", + test: test_throwing_error_handler + }, + { + description: "test default error handler", + test: test_default_error_handler + } +]; + +export default tests; \ No newline at end of file diff --git a/test/to_array_test.mjs b/test/to_array_test.mjs index 0e82778..2c209c7 100644 --- a/test/to_array_test.mjs +++ b/test/to_array_test.mjs @@ -1,11 +1,11 @@ #!/usr/bin/env node import { strict as assert } from 'assert'; -import h5wasm from '../dist/node/hdf5_hl.js'; +import h5wasm from 'h5wasm/node'; async function to_array_test() { await h5wasm.ready; - var f = new h5wasm.File('./test/array.h5', 'r'); + const f = new h5wasm.File('./test/array.h5', 'r'); assert.deepEqual( f.get('bigint').to_array(), diff --git a/test/track_order.mjs b/test/track_order.mjs new file mode 100644 index 0000000..4e9e058 --- /dev/null +++ b/test/track_order.mjs @@ -0,0 +1,115 @@ +#!/usr/bin/env node + +import { strict as assert } from 'assert'; +import { existsSync, mkdirSync, unlinkSync } from 'fs'; +import { join } from 'path'; +import h5wasm from "h5wasm/node"; + +async function test_track_order() { + + await h5wasm.ready; + const PATH = join(".", "test", "tmp"); + const TRACKED_FILEPATH = join(PATH, "track_order.h5"); + const UNTRACKED_FILEPATH = join(PATH, "untrack_order.h5"); + + const VALUES = [3,2,1]; + const DATA = new Float32Array(VALUES); + + if (!(existsSync(PATH))) { + mkdirSync(PATH); + } + + // write with tracking on + { + const write_file = new h5wasm.File(TRACKED_FILEPATH, "w", true); + + // create attributes with names in reverse alphabetical order: + write_file.create_attribute("c", "first attribute"); + write_file.create_attribute("b", "second attribute"); + write_file.create_attribute("a", "third attribute"); + + // create datasets with names in reverse alphabetical order: + write_file.create_dataset({name: "c_data", data: DATA}); + write_file.create_dataset({name: "b_data", data: DATA}); + const a_data = write_file.create_dataset({name: "a_data", data: DATA, track_order: true}); + + // create attributes with names in reverse alphabetical order: + a_data.create_attribute("c", "first attribute"); + a_data.create_attribute("b", "second attribute"); + a_data.create_attribute("a", "third attribute"); + + write_file.flush(); + write_file.close(); + } + + // write with tracking off + { + const write_file = new h5wasm.File(UNTRACKED_FILEPATH, "w", false); + + // create attributes with names in reverse alphabetical order: + write_file.create_attribute("c", "first attribute"); + write_file.create_attribute("b", "second attribute"); + write_file.create_attribute("a", "third attribute"); + + // create datasets with names in reverse alphabetical order: + write_file.create_dataset({name: "c_data", data: DATA}); + write_file.create_dataset({name: "b_data", data: DATA}); + const a_data = write_file.create_dataset({name: "a_data", data: DATA, track_order: false}); + + // create attributes with names in reverse alphabetical order: + a_data.create_attribute("c", "first attribute"); + a_data.create_attribute("b", "second attribute"); + a_data.create_attribute("a", "third attribute"); + + write_file.flush(); + write_file.close(); + } + + // read with tracking on + { + const read_file = new h5wasm.File(TRACKED_FILEPATH, "r"); + + // check that attrs are in original order: + assert.deepEqual(Object.keys(read_file.attrs), ["c", "b", "a"]); + + // check that datasets are in original order: + assert.deepEqual(read_file.keys(), ["c_data", "b_data", "a_data"]); + + // check that attrs of dataset are in original order: + const dataset_attrs = read_file.get("a_data").attrs; + assert.deepEqual(Object.keys(dataset_attrs), ["c", "b", "a"]); + + read_file.close() + } + + // read with tracking off + { + const read_file = new h5wasm.File(UNTRACKED_FILEPATH, "r"); + + // check that attrs are in alphabetical (not original) order: + assert.deepEqual(Object.keys(read_file.attrs), ["a", "b", "c"]); + + // check that datasets are in alphabetical (not original) order: + assert.deepEqual(read_file.keys(), ["a_data", "b_data", "c_data"]); + + // check that attrs of dataset are in alphabetical (not original) order: + const dataset_attrs = read_file.get("a_data").attrs; + assert.deepEqual(Object.keys(dataset_attrs), ["a", "b", "c"]); + + read_file.close() + } + + // cleanup file when finished: + unlinkSync(TRACKED_FILEPATH); + unlinkSync(UNTRACKED_FILEPATH); + +} + +export const tests = [ + { + description: "Create and read attrs and datasets with and without track_order", + test: test_track_order + }, +]; + +export default tests; diff --git a/test/vlen.h5 b/test/vlen.h5 new file mode 100644 index 0000000..4fabfdb Binary files /dev/null and b/test/vlen.h5 differ diff --git a/test/vlen_test.mjs b/test/vlen_test.mjs new file mode 100644 index 0000000..a6e552a --- /dev/null +++ b/test/vlen_test.mjs @@ -0,0 +1,65 @@ +#!/usr/bin/env node + +import { strict as assert } from 'assert'; +import h5wasm from 'h5wasm/node'; + +async function vlen_test() { + await h5wasm.ready; + var f = new h5wasm.File('./test/vlen.h5', 'r'); + + assert.deepEqual(f.get('int8_scalar').metadata, { + type: 9, + shape: [], + maxshape: [], + chunks: null, + size: 8, + total_size: 1, + signed: true, + littleEndian: true, + vlen: false, + vlen_type: { + type: 0, + size: 1, + signed: true, + littleEndian: true, + vlen: false, + }, + }); + + assert.deepEqual(f.get('float32_oneD').metadata, { + type: 9, + shape: [3], + maxshape: [3], + chunks: null, + size: 8, + total_size: 3, + signed: false, + littleEndian: true, + vlen: false, + vlen_type: { + type: 1, + size: 4, + signed: false, + littleEndian: true, + vlen: false, + }, + }); + + assert.deepEqual(f.get('int8_scalar').value, new Int8Array([0, 1])); + assert.deepEqual( + f.get('float32_oneD').value, + [ + new Float32Array([0]), + new Float32Array([0, 1]), + new Float32Array([0, 1, 2]) + ] + ); +} + +export const tests = [ + { + description: 'Read variable-length datasets', + test: vlen_test, + }, +]; +export default tests;