From 4d6db210e8cb6a6f195c8e61a1a2a82635761675 Mon Sep 17 00:00:00 2001 From: mkuehbach Date: Fri, 21 Mar 2025 10:37:30 +0100 Subject: [PATCH 01/23] Tracking specific defs branch to test bugfixes in preparation for pynxtools release v0.10.1 --- src/pynxtools/definitions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pynxtools/definitions b/src/pynxtools/definitions index a85e10cd0..ece88b30d 160000 --- a/src/pynxtools/definitions +++ b/src/pynxtools/definitions @@ -1 +1 @@ -Subproject commit a85e10cd0289f4e44b0fec011ff54703e6705383 +Subproject commit ece88b30db5dda14baca9a73ce61048a1534bd85 From 1f76f700a2b6c452427652d3523f6f1d2922d549 Mon Sep 17 00:00:00 2001 From: mkuehbach Date: Fri, 21 Mar 2025 13:47:30 +0100 Subject: [PATCH 02/23] Deactivated topology visualizer as it does not recongnize the atoms for the new formatting, fix later, downgraded severity of warning for not isfinite to info --- dev-requirements.txt | 225 ---------------------------------- pyproject.toml | 4 + src/pynxtools/nomad/parser.py | 4 +- src/pynxtools/nomad/schema.py | 27 +++- 4 files changed, 27 insertions(+), 233 deletions(-) delete mode 100644 dev-requirements.txt diff --git a/dev-requirements.txt b/dev-requirements.txt deleted file mode 100644 index a61dad8ee..000000000 --- a/dev-requirements.txt +++ /dev/null @@ -1,225 +0,0 @@ -# This file was autogenerated by uv via the following command: -# uv pip compile --universal -p 3.11 --extra=dev --extra=docs --output-file=dev-requirements.txt pyproject.toml -anytree==2.12.1 - # via pynxtools (pyproject.toml) -ase==3.23.0 - # via pynxtools (pyproject.toml) -babel==2.16.0 - # via mkdocs-material -certifi==2024.8.30 - # via requests -cfgv==3.4.0 - # via pre-commit -charset-normalizer==3.4.0 - # via requests -click==8.1.7 - # via - # pynxtools (pyproject.toml) - # click-default-group - # mkdocs - # mkdocs-click -click-default-group==1.2.4 - # via pynxtools (pyproject.toml) -colorama==0.4.6 - # via - # click - # mkdocs - # mkdocs-material - # pytest -contourpy==1.3.1 - # via matplotlib -coverage==7.6.9 - # via pytest-cov -cycler==0.12.1 - # via matplotlib -distlib==0.3.9 - # via virtualenv -filelock==3.16.1 - # via virtualenv -fonttools==4.55.2 - # via matplotlib -ghp-import==2.1.0 - # via mkdocs -h5py==3.12.1 - # via pynxtools (pyproject.toml) -hjson==3.1.0 - # via - # mkdocs-macros-plugin - # super-collections -identify==2.6.3 - # via pre-commit -idna==3.10 - # via requests -importlib-metadata==8.5.0 - # via pynxtools (pyproject.toml) -iniconfig==2.0.0 - # via pytest -jinja2==3.1.4 - # via - # mkdocs - # mkdocs-macros-plugin - # mkdocs-material -kiwisolver==1.4.7 - # via matplotlib -lxml==5.3.0 - # via pynxtools (pyproject.toml) -markdown==3.7 - # via - # markdown-include - # mkdocs - # mkdocs-click - # mkdocs-material - # pymdown-extensions -markdown-include==0.8.1 - # via pynxtools (pyproject.toml) -markupsafe==3.0.2 - # via - # jinja2 - # mkdocs -matplotlib==3.9.3 - # via ase -mergedeep==1.3.4 - # via - # pynxtools (pyproject.toml) - # mkdocs - # mkdocs-get-deps -mkdocs==1.6.1 - # via - # pynxtools (pyproject.toml) - # mkdocs-macros-plugin - # mkdocs-material -mkdocs-click==0.8.1 - # via pynxtools (pyproject.toml) -mkdocs-get-deps==0.2.0 - # via mkdocs -mkdocs-macros-plugin==1.3.7 - # via pynxtools (pyproject.toml) -mkdocs-material==9.5.48 - # via pynxtools (pyproject.toml) -mkdocs-material-extensions==1.3.1 - # via - # pynxtools (pyproject.toml) - # mkdocs-material -mypy==1.13.0 - # via pynxtools (pyproject.toml) -mypy-extensions==1.0.0 - # via mypy -nodeenv==1.9.1 - # via pre-commit -numpy==1.26.4 - # via - # pynxtools (pyproject.toml) - # ase - # contourpy - # h5py - # matplotlib - # pandas - # scipy - # xarray -packaging==24.2 - # via - # matplotlib - # mkdocs - # mkdocs-macros-plugin - # pytest - # xarray -paginate==0.5.7 - # via mkdocs-material -pandas==2.2.3 - # via - # pynxtools (pyproject.toml) - # xarray -pathspec==0.12.1 - # via - # mkdocs - # mkdocs-macros-plugin -pillow==11.0.0 - # via matplotlib -platformdirs==4.3.6 - # via - # mkdocs-get-deps - # virtualenv -pluggy==1.5.0 - # via pytest -pre-commit==4.0.1 - # via pynxtools (pyproject.toml) -pygments==2.18.0 - # via mkdocs-material -pymdown-extensions==10.12 - # via mkdocs-material -pyparsing==3.2.0 - # via matplotlib -pytest==8.3.4 - # via - # pynxtools (pyproject.toml) - # pytest-cov - # pytest-timeout -pytest-cov==6.0.0 - # via pynxtools (pyproject.toml) -pytest-timeout==2.3.1 - # via pynxtools (pyproject.toml) -python-dateutil==2.9.0.post0 - # via - # ghp-import - # matplotlib - # mkdocs-macros-plugin - # pandas -pytz==2024.2 - # via pandas -pyyaml==6.0.2 - # via - # pynxtools (pyproject.toml) - # mkdocs - # mkdocs-get-deps - # mkdocs-macros-plugin - # pre-commit - # pymdown-extensions - # pyyaml-env-tag -pyyaml-env-tag==0.1 - # via mkdocs -regex==2024.11.6 - # via mkdocs-material -requests==2.32.3 - # via mkdocs-material -ruff==0.9.6 - # via pynxtools (pyproject.toml) -scipy==1.14.1 - # via ase -six==1.17.0 - # via - # anytree - # python-dateutil -structlog==24.4.0 - # via pynxtools (pyproject.toml) -super-collections==0.5.3 - # via mkdocs-macros-plugin -termcolor==2.5.0 - # via mkdocs-macros-plugin -tomli==2.2.1 ; python_full_version <= '3.11' - # via coverage -toposort==1.10 - # via pynxtools (pyproject.toml) -types-pytz==2024.2.0.20241003 - # via pynxtools (pyproject.toml) -types-pyyaml==6.0.12.20240917 - # via pynxtools (pyproject.toml) -types-requests==2.32.0.20241016 - # via pynxtools (pyproject.toml) -typing-extensions==4.12.2 - # via mypy -tzdata==2024.2 - # via pandas -urllib3==2.2.3 - # via - # requests - # types-requests -uv==0.5.7 - # via pynxtools (pyproject.toml) -virtualenv==20.28.0 - # via pre-commit -watchdog==6.0.0 - # via mkdocs -xarray==2024.11.0 - # via pynxtools (pyproject.toml) -zipp==3.21.0 - # via importlib-metadata diff --git a/pyproject.toml b/pyproject.toml index cc59d830e..046e6adf1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,6 +65,10 @@ dev = [ "uv", "pre-commit", ] +nomad = [ + "urllib3<=1.26.20", + "filelock==3.3.1" +] convert = [ "pynxtools[apm,ellips,em,igor,mpes,raman,stm,xps,xrd]", ] diff --git a/src/pynxtools/nomad/parser.py b/src/pynxtools/nomad/parser.py index 930cf5967..9349113ac 100644 --- a/src/pynxtools/nomad/parser.py +++ b/src/pynxtools/nomad/parser.py @@ -289,7 +289,7 @@ def _populate_data( ] field = field_stats[0] if not np.isfinite(field): - self._logger.warning( + self._logger.info( "set NaN for field of an array", target_name=field_name + "[" + data_instance_name + "]", ) @@ -298,7 +298,7 @@ def _populate_data( return else: if field.size == 1 and not np.isfinite(field): - self._logger.warning( + self._logger.info( "set NaN for field of a scalar", target_name=field_name + "[" + data_instance_name + "]", ) diff --git a/src/pynxtools/nomad/schema.py b/src/pynxtools/nomad/schema.py index 1eb7ad19f..2394e846c 100644 --- a/src/pynxtools/nomad/schema.py +++ b/src/pynxtools/nomad/schema.py @@ -31,6 +31,7 @@ import h5py import numpy as np import pandas as pd +import pint from ase import Atoms from ase.data import atomic_numbers from nomad.datamodel.metainfo.plot import PlotlyFigure, PlotSection @@ -39,7 +40,6 @@ from nomad.normalizing.common import nomad_atoms_from_ase_atoms from nomad.normalizing.topology import add_system, add_system_info from scipy.spatial import cKDTree -import pint try: from nomad import utils @@ -1258,6 +1258,7 @@ def normalize_atom_probe(self, archive, logger): ].section_cls super(current_cls, self).normalize(archive, logger) # temporarily disable extra normalisation step + return def plot_3d_plotly(df, palette="Set1"): import plotly.express as px @@ -1318,6 +1319,17 @@ def plot_3d_plotly(df, palette="Set1"): os.path.join(archive.m_context.raw_path(), self.m_attributes["m_nx_data_file"]), "r", ) as fp: + if ( + f"{data_path}/reconstruction/reconstructed_positions" not in fp + or f"{data_path}/mass_to_charge_conversion/mass_to_charge" not in fp + ): + return + if not hasattr(self, "ranging"): + return + if not hasattr(self.ranging, "peak_identification"): + return + if not hasattr(self.ranging.peak_identification, "ionID"): + return # Load the reconstructed positions and mass-to-charge values positions = fp[f"{data_path}/reconstruction/reconstructed_positions"][:] mass_to_charge_values = fp[ @@ -1373,7 +1385,7 @@ def create_ase_atoms(df): positions = df[["x", "y", "z"]].values return Atoms(symbols=symbols, positions=positions) - atoms_lamela = create_ase_atoms(df_sampled) + apt_tip = create_ase_atoms(df_sampled) # **Build NOMAD Topology Structure with Individual Element Systems** def build_nomad_topology(archive): @@ -1382,19 +1394,22 @@ def build_nomad_topology(archive): if not archive.results.material: archive.results.material = Material() - elements = list(set(atoms_lamela.get_chemical_symbols())) + # @Pepe-Marquez, can just be checked for existence, elements have at this point + # along the processing pipeline been already been populated + elements = list(set(apt_tip.get_chemical_symbols())) if not archive.results.material.elements: archive.results.material.elements = elements else: + # @Pepe-Marquez why does the storage order of the elements matter here? for i in range(len(elements)): if archive.results.material.elements[i] != elements[i]: print("WARNING: elements are swapped") topology = {} system = System( - atoms=nomad_atoms_from_ase_atoms(atoms_lamela), - label="Atom Probe Tomography - Lamella", - description="Reconstructed 3D atom probe tomography dataset.", + atoms=nomad_atoms_from_ase_atoms(apt_tip), + label="Reconstruction", + description="Reconstruction of the original atom positions", structural_type="bulk", dimensionality="3D", system_relation=Relation(type="root"), From 60c79fddb838af8063f2942a293349bbfa0ba527 Mon Sep 17 00:00:00 2001 From: sanbrock Date: Fri, 21 Mar 2025 16:03:04 +0100 Subject: [PATCH 03/23] identifiers; also handle variadic quantities --- src/pynxtools/nomad/schema.py | 56 +++++++++++++++++++++++------------ 1 file changed, 37 insertions(+), 19 deletions(-) diff --git a/src/pynxtools/nomad/schema.py b/src/pynxtools/nomad/schema.py index 2394e846c..b9aac1688 100644 --- a/src/pynxtools/nomad/schema.py +++ b/src/pynxtools/nomad/schema.py @@ -175,8 +175,8 @@ def get_entry_reference(archive, f_name): self.reference = get_entry_reference(archive, f_name) -class NexusReferences(ArchiveSection): - AnchoredReferences = SubSection( +class NexusIdentifiers(ArchiveSection): + NXidentifiers = SubSection( section_def=AnchoredReference, repeats=True, description="These are the NOMAD references correspond to NeXus identifierNAME fields.", @@ -191,25 +191,43 @@ def normalize(self, archive, logger): ] if not identifiers: return - self.AnchoredReferences = [] + self.NXidentifiers = [] for identifier in identifiers: if not (val := getattr(self, identifier)): continue - # identifier_path = f"{self.m_def.name}_{identifier.split('__field')[0]}" - field_n = identifier.split("__field")[0] - logger.info(f"Lab id {val} to be created") - nx_id = AnchoredReference(lab_id=val, name=field_n) - nx_id.m_set_section_attribute( - "m_nx_data_path", - self.m_get_quantity_attribute(identifier, "m_nx_data_path"), - ) - nx_id.m_set_section_attribute( - "m_nx_data_file", - self.m_get_quantity_attribute(identifier, "m_nx_data_file"), - ) + if isinstance(val, dict): + for idname, idobj in val: + # identifier_path = f"{self.m_def.name}_{identifier.split('__field')[0]}" + field_n = idname.split("__field")[0] + logger.info(f"Lab id {val} to be created") + nx_id = AnchoredReference(lab_id=val, name=field_n) + nx_id.m_set_section_attribute( + "m_nx_data_path", + self.m_get_quantity_attribute(idname, "m_nx_data_path"), + ) + nx_id.m_set_section_attribute( + "m_nx_data_file", + self.m_get_quantity_attribute(idname, "m_nx_data_file"), + ) + + self.NXientifiers.append(nx_id) + nx_id.normalize(archive, logger) + else: + # identifier_path = f"{self.m_def.name}_{identifier.split('__field')[0]}" + field_n = identifier.split("__field")[0] + logger.info(f"Lab id {val} to be created") + nx_id = AnchoredReference(lab_id=val, name=field_n) + nx_id.m_set_section_attribute( + "m_nx_data_path", + self.m_get_quantity_attribute(identifier, "m_nx_data_path"), + ) + nx_id.m_set_section_attribute( + "m_nx_data_file", + self.m_get_quantity_attribute(identifier, "m_nx_data_file"), + ) - self.AnchoredReferences.append(nx_id) - nx_id.normalize(archive, logger) + self.NXientifiers.append(nx_id) + nx_id.normalize(archive, logger) super().normalize(archive, logger) @@ -229,7 +247,7 @@ class NexusActivityResult(ActivityResult): "NXfabrication": [basesections.Instrument], "NXsample": [CompositeSystem], "NXsample_component": [Component], - "NXobject": [NexusReferences], + "NXobject": [NexusIdentifiers], "NXentry": [NexusActivityStep], "NXprocess": [NexusActivityStep], "NXdata": [NexusActivityResult], @@ -1148,7 +1166,7 @@ def init_nexus_metainfo(): nexus_metainfo_package.section_definitions.append(NexusActivityResult.m_def) nexus_metainfo_package.section_definitions.append(NexusBaseSection.m_def) nexus_metainfo_package.section_definitions.append(AnchoredReference.m_def) - nexus_metainfo_package.section_definitions.append(NexusReferences.m_def) + nexus_metainfo_package.section_definitions.append(NexusIdentifiers.m_def) # We need to initialize the metainfo definitions. This is usually done automatically, # when the metainfo schema is defined though MSection Python classes. From f87e9fbfa2060a78afd656c16fa2cf6c31ba386c Mon Sep 17 00:00:00 2001 From: sanbrock Date: Fri, 21 Mar 2025 16:05:27 +0100 Subject: [PATCH 04/23] fix --- src/pynxtools/nomad/schema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pynxtools/nomad/schema.py b/src/pynxtools/nomad/schema.py index b9aac1688..a28f3398e 100644 --- a/src/pynxtools/nomad/schema.py +++ b/src/pynxtools/nomad/schema.py @@ -196,7 +196,7 @@ def normalize(self, archive, logger): if not (val := getattr(self, identifier)): continue if isinstance(val, dict): - for idname, idobj in val: + for idname, idobj in val.items(): # identifier_path = f"{self.m_def.name}_{identifier.split('__field')[0]}" field_n = idname.split("__field")[0] logger.info(f"Lab id {val} to be created") From 98283b71b463e4fbf146f1618bcc07052fd4a9ab Mon Sep 17 00:00:00 2001 From: mkuehbach Date: Fri, 21 Mar 2025 18:38:57 +0100 Subject: [PATCH 05/23] Fix not val but idobj.val --- src/pynxtools/nomad/schema.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pynxtools/nomad/schema.py b/src/pynxtools/nomad/schema.py index a28f3398e..517be3b68 100644 --- a/src/pynxtools/nomad/schema.py +++ b/src/pynxtools/nomad/schema.py @@ -199,8 +199,8 @@ def normalize(self, archive, logger): for idname, idobj in val.items(): # identifier_path = f"{self.m_def.name}_{identifier.split('__field')[0]}" field_n = idname.split("__field")[0] - logger.info(f"Lab id {val} to be created") - nx_id = AnchoredReference(lab_id=val, name=field_n) + logger.info(f"Lab id {idobj.val} to be created") + nx_id = AnchoredReference(lab_id=idobj.val, name=field_n) nx_id.m_set_section_attribute( "m_nx_data_path", self.m_get_quantity_attribute(idname, "m_nx_data_path"), From 8b16e01f486ba409cb2f571ccf5bad7f9e461862 Mon Sep 17 00:00:00 2001 From: mkuehbach Date: Fri, 21 Mar 2025 20:22:52 +0100 Subject: [PATCH 06/23] val to value --- src/pynxtools/nomad/schema.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pynxtools/nomad/schema.py b/src/pynxtools/nomad/schema.py index 517be3b68..a2800381f 100644 --- a/src/pynxtools/nomad/schema.py +++ b/src/pynxtools/nomad/schema.py @@ -199,8 +199,8 @@ def normalize(self, archive, logger): for idname, idobj in val.items(): # identifier_path = f"{self.m_def.name}_{identifier.split('__field')[0]}" field_n = idname.split("__field")[0] - logger.info(f"Lab id {idobj.val} to be created") - nx_id = AnchoredReference(lab_id=idobj.val, name=field_n) + logger.info(f"Lab id {idobj.value} to be created") + nx_id = AnchoredReference(lab_id=idobj.value, name=field_n) nx_id.m_set_section_attribute( "m_nx_data_path", self.m_get_quantity_attribute(idname, "m_nx_data_path"), From a287577ac795770152548539455def426ed18e13 Mon Sep 17 00:00:00 2001 From: mkuehbach Date: Sat, 22 Mar 2025 09:05:47 +0100 Subject: [PATCH 07/23] Bump definitions to take advantage of group consistent rotations --- src/pynxtools/definitions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pynxtools/definitions b/src/pynxtools/definitions index ece88b30d..bb4ce8c22 160000 --- a/src/pynxtools/definitions +++ b/src/pynxtools/definitions @@ -1 +1 @@ -Subproject commit ece88b30db5dda14baca9a73ce61048a1534bd85 +Subproject commit bb4ce8c22d32094f1f315cd4ecc1b0a1f37b1f2a From aa28b12f8ba83e1d8e0f4e3464d0ba6fa6b2a494 Mon Sep 17 00:00:00 2001 From: mkuehbach Date: Sat, 22 Mar 2025 10:24:14 +0100 Subject: [PATCH 08/23] Update to profit from bugfix that now all AXISNAME_indices are NX_UINT --- src/pynxtools/definitions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pynxtools/definitions b/src/pynxtools/definitions index bb4ce8c22..28968c1dd 160000 --- a/src/pynxtools/definitions +++ b/src/pynxtools/definitions @@ -1 +1 @@ -Subproject commit bb4ce8c22d32094f1f315cd4ecc1b0a1f37b1f2a +Subproject commit 28968c1dd2afb814780f2423cce6800f14e521f4 From 4a842fd9271ca45dacd87cbb02e5129062cdc237 Mon Sep 17 00:00:00 2001 From: mkuehbach Date: Tue, 25 Mar 2025 09:47:32 +0100 Subject: [PATCH 09/23] Using the final definitions for this feature branch --- src/pynxtools/definitions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pynxtools/definitions b/src/pynxtools/definitions index 28968c1dd..8eea0d4d4 160000 --- a/src/pynxtools/definitions +++ b/src/pynxtools/definitions @@ -1 +1 @@ -Subproject commit 28968c1dd2afb814780f2423cce6800f14e521f4 +Subproject commit 8eea0d4d4dab58874b8f5aa266ec2fdb04fa6cfb From 42af07ab461850ab6a30a7f465d9f84540759356 Mon Sep 17 00:00:00 2001 From: mkuehbach Date: Tue, 25 Mar 2025 10:00:30 +0100 Subject: [PATCH 10/23] Reverted changes in pyproject toml and dev-requirements, fixed NXidentifiers typo --- dev-requirements.txt | 228 ++++++++++++++++++++++++++++++++++ pyproject.toml | 4 - src/pynxtools/nomad/schema.py | 4 +- 3 files changed, 230 insertions(+), 6 deletions(-) create mode 100644 dev-requirements.txt diff --git a/dev-requirements.txt b/dev-requirements.txt new file mode 100644 index 000000000..261e7d34c --- /dev/null +++ b/dev-requirements.txt @@ -0,0 +1,228 @@ +# This file was autogenerated by uv via the following command: +# uv pip compile --universal -p 3.11 --extra=dev --extra=docs --output-file=dev-requirements.txt pyproject.toml +anytree==2.12.1 + # via pynxtools (pyproject.toml) +ase==3.24.0 + # via pynxtools (pyproject.toml) +babel==2.17.0 + # via mkdocs-material +backrefs==5.8 + # via mkdocs-material +certifi==2025.1.31 + # via requests +cfgv==3.4.0 + # via pre-commit +charset-normalizer==3.4.1 + # via requests +click==8.1.8 + # via + # pynxtools (pyproject.toml) + # click-default-group + # mkdocs + # mkdocs-click +click-default-group==1.2.4 + # via pynxtools (pyproject.toml) +colorama==0.4.6 + # via + # click + # mkdocs + # mkdocs-material + # pytest +contourpy==1.3.1 + # via matplotlib +coverage==7.7.1 + # via pytest-cov +cycler==0.12.1 + # via matplotlib +distlib==0.3.9 + # via virtualenv +filelock==3.18.0 + # via virtualenv +fonttools==4.56.0 + # via matplotlib +ghp-import==2.1.0 + # via mkdocs +h5py==3.13.0 + # via pynxtools (pyproject.toml) +hjson==3.1.0 + # via + # mkdocs-macros-plugin + # super-collections +identify==2.6.9 + # via pre-commit +idna==3.10 + # via requests +importlib-metadata==8.6.1 + # via pynxtools (pyproject.toml) +iniconfig==2.1.0 + # via pytest +jinja2==3.1.6 + # via + # mkdocs + # mkdocs-macros-plugin + # mkdocs-material +kiwisolver==1.4.8 + # via matplotlib +lxml==5.3.1 + # via pynxtools (pyproject.toml) +markdown==3.7 + # via + # markdown-include + # mkdocs + # mkdocs-click + # mkdocs-material + # pymdown-extensions +markdown-include==0.8.1 + # via pynxtools (pyproject.toml) +markupsafe==3.0.2 + # via + # jinja2 + # mkdocs +matplotlib==3.10.1 + # via ase +mergedeep==1.3.4 + # via + # pynxtools (pyproject.toml) + # mkdocs + # mkdocs-get-deps +mkdocs==1.6.1 + # via + # pynxtools (pyproject.toml) + # mkdocs-macros-plugin + # mkdocs-material +mkdocs-click==0.8.1 + # via pynxtools (pyproject.toml) +mkdocs-get-deps==0.2.0 + # via mkdocs +mkdocs-macros-plugin==1.3.7 + # via pynxtools (pyproject.toml) +mkdocs-material==9.6.9 + # via pynxtools (pyproject.toml) +mkdocs-material-extensions==1.3.1 + # via + # pynxtools (pyproject.toml) + # mkdocs-material +mypy==1.15.0 + # via pynxtools (pyproject.toml) +mypy-extensions==1.0.0 + # via mypy +nodeenv==1.9.1 + # via pre-commit +numpy==1.26.4 + # via + # pynxtools (pyproject.toml) + # ase + # contourpy + # h5py + # matplotlib + # pandas + # scipy + # xarray +packaging==24.2 + # via + # matplotlib + # mkdocs + # mkdocs-macros-plugin + # pint + # pytest + # xarray +paginate==0.5.7 + # via mkdocs-material +pandas==2.2.3 + # via + # pynxtools (pyproject.toml) + # xarray +pathspec==0.12.1 + # via + # mkdocs + # mkdocs-macros-plugin +pillow==11.1.0 + # via matplotlib +pint==0.17 + # via pynxtools (pyproject.toml) +platformdirs==4.3.7 + # via + # mkdocs-get-deps + # virtualenv +pluggy==1.5.0 + # via pytest +pre-commit==4.2.0 + # via pynxtools (pyproject.toml) +pygments==2.19.1 + # via mkdocs-material +pymdown-extensions==10.14.3 + # via mkdocs-material +pyparsing==3.2.3 + # via matplotlib +pytest==8.3.5 + # via + # pynxtools (pyproject.toml) + # pytest-cov + # pytest-timeout +pytest-cov==6.0.0 + # via pynxtools (pyproject.toml) +pytest-timeout==2.3.1 + # via pynxtools (pyproject.toml) +python-dateutil==2.9.0.post0 + # via + # ghp-import + # matplotlib + # mkdocs-macros-plugin + # pandas +pytz==2025.2 + # via pandas +pyyaml==6.0.2 + # via + # pynxtools (pyproject.toml) + # mkdocs + # mkdocs-get-deps + # mkdocs-macros-plugin + # pre-commit + # pymdown-extensions + # pyyaml-env-tag +pyyaml-env-tag==0.1 + # via mkdocs +requests==2.32.3 + # via mkdocs-material +ruff==0.11.2 + # via pynxtools (pyproject.toml) +scipy==1.15.2 + # via ase +six==1.17.0 + # via + # anytree + # python-dateutil +structlog==25.2.0 + # via pynxtools (pyproject.toml) +super-collections==0.5.3 + # via mkdocs-macros-plugin +termcolor==2.5.0 + # via mkdocs-macros-plugin +tomli==2.2.1 ; python_full_version <= '3.11' + # via coverage +toposort==1.10 + # via pynxtools (pyproject.toml) +types-pytz==2025.1.0.20250318 + # via pynxtools (pyproject.toml) +types-pyyaml==6.0.12.20241230 + # via pynxtools (pyproject.toml) +types-requests==2.32.0.20250306 + # via pynxtools (pyproject.toml) +typing-extensions==4.12.2 + # via mypy +tzdata==2025.2 + # via pandas +urllib3==2.3.0 + # via + # requests + # types-requests +uv==0.6.9 + # via pynxtools (pyproject.toml) +virtualenv==20.29.3 + # via pre-commit +watchdog==6.0.0 + # via mkdocs +xarray==2025.3.0 + # via pynxtools (pyproject.toml) +zipp==3.21.0 + # via importlib-metadata diff --git a/pyproject.toml b/pyproject.toml index 046e6adf1..cc59d830e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,10 +65,6 @@ dev = [ "uv", "pre-commit", ] -nomad = [ - "urllib3<=1.26.20", - "filelock==3.3.1" -] convert = [ "pynxtools[apm,ellips,em,igor,mpes,raman,stm,xps,xrd]", ] diff --git a/src/pynxtools/nomad/schema.py b/src/pynxtools/nomad/schema.py index a2800381f..1f73bc6c9 100644 --- a/src/pynxtools/nomad/schema.py +++ b/src/pynxtools/nomad/schema.py @@ -210,7 +210,7 @@ def normalize(self, archive, logger): self.m_get_quantity_attribute(idname, "m_nx_data_file"), ) - self.NXientifiers.append(nx_id) + self.NXidentifiers.append(nx_id) nx_id.normalize(archive, logger) else: # identifier_path = f"{self.m_def.name}_{identifier.split('__field')[0]}" @@ -226,7 +226,7 @@ def normalize(self, archive, logger): self.m_get_quantity_attribute(identifier, "m_nx_data_file"), ) - self.NXientifiers.append(nx_id) + self.NXidentifiers.append(nx_id) nx_id.normalize(archive, logger) super().normalize(archive, logger) From 6f6a45da15b132c2b774748ea665e90c70be3d59 Mon Sep 17 00:00:00 2001 From: mkuehbach Date: Tue, 25 Mar 2025 10:19:28 +0100 Subject: [PATCH 11/23] Updated nexus version artifact --- src/pynxtools/nexus-version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pynxtools/nexus-version.txt b/src/pynxtools/nexus-version.txt index c137d59d6..fc6a08845 100644 --- a/src/pynxtools/nexus-version.txt +++ b/src/pynxtools/nexus-version.txt @@ -1 +1 @@ -v2024.02-1913-ga85e10cd \ No newline at end of file +v2024.02-1913-g8eea0d4 \ No newline at end of file From d3091a55126068913ab3ca086135bc0c9bc70803 Mon Sep 17 00:00:00 2001 From: mkuehbach Date: Tue, 25 Mar 2025 10:21:36 +0100 Subject: [PATCH 12/23] Updated nexus version artifact --- src/pynxtools/nexus-version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pynxtools/nexus-version.txt b/src/pynxtools/nexus-version.txt index fc6a08845..648482d47 100644 --- a/src/pynxtools/nexus-version.txt +++ b/src/pynxtools/nexus-version.txt @@ -1 +1 @@ -v2024.02-1913-g8eea0d4 \ No newline at end of file +v2024.02-1918-g8eea0d4 \ No newline at end of file From 5eda1234311c8252637d8c6f55bdb7469bebb698 Mon Sep 17 00:00:00 2001 From: mkuehbach Date: Tue, 25 Mar 2025 10:22:56 +0100 Subject: [PATCH 13/23] Updated nexus version artifact --- src/pynxtools/nexus-version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pynxtools/nexus-version.txt b/src/pynxtools/nexus-version.txt index 648482d47..c9035a941 100644 --- a/src/pynxtools/nexus-version.txt +++ b/src/pynxtools/nexus-version.txt @@ -1 +1 @@ -v2024.02-1918-g8eea0d4 \ No newline at end of file +v2024.02-1918-g8eea0d4d From 8e9ff81884e5341694fc2ed05ba85fe406b9c4d0 Mon Sep 17 00:00:00 2001 From: mkuehbach Date: Tue, 25 Mar 2025 10:26:25 +0100 Subject: [PATCH 14/23] Updated nexus version artifact --- src/pynxtools/nexus-version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pynxtools/nexus-version.txt b/src/pynxtools/nexus-version.txt index c9035a941..526a242b3 100644 --- a/src/pynxtools/nexus-version.txt +++ b/src/pynxtools/nexus-version.txt @@ -1 +1 @@ -v2024.02-1918-g8eea0d4d +v2024.02-1918-g8eea0d4d \ No newline at end of file From 1709a87275606e4399540f9784ce4bf8ef827adf Mon Sep 17 00:00:00 2001 From: mkuehbach Date: Tue, 25 Mar 2025 12:45:52 +0100 Subject: [PATCH 15/23] Bumped definitions --- src/pynxtools/definitions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pynxtools/definitions b/src/pynxtools/definitions index 8eea0d4d4..ecbe9b27a 160000 --- a/src/pynxtools/definitions +++ b/src/pynxtools/definitions @@ -1 +1 @@ -Subproject commit 8eea0d4d4dab58874b8f5aa266ec2fdb04fa6cfb +Subproject commit ecbe9b27a9aeefaf562470ec688297228d10e839 From 11eb5e455d8a35e6cbcb1ab6f17f27298c6145c5 Mon Sep 17 00:00:00 2001 From: mkuehbach Date: Tue, 25 Mar 2025 12:48:06 +0100 Subject: [PATCH 16/23] Version text artifact --- src/pynxtools/nexus-version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pynxtools/nexus-version.txt b/src/pynxtools/nexus-version.txt index 526a242b3..8316aa823 100644 --- a/src/pynxtools/nexus-version.txt +++ b/src/pynxtools/nexus-version.txt @@ -1 +1 @@ -v2024.02-1918-g8eea0d4d \ No newline at end of file +v2024.02-1920-gecbe9b27 From a1ac20a997e8407583165fc8613c6da760c9e5a6 Mon Sep 17 00:00:00 2001 From: mkuehbach Date: Tue, 25 Mar 2025 12:50:05 +0100 Subject: [PATCH 17/23] Remove newline from the text artifact --- src/pynxtools/nexus-version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pynxtools/nexus-version.txt b/src/pynxtools/nexus-version.txt index 8316aa823..4da4a7d42 100644 --- a/src/pynxtools/nexus-version.txt +++ b/src/pynxtools/nexus-version.txt @@ -1 +1 @@ -v2024.02-1920-gecbe9b27 +v2024.02-1920-gecbe9b27 \ No newline at end of file From 04d9800a63a4b1ecb4a18d0436fca1eabe4a026c Mon Sep 17 00:00:00 2001 From: mkuehbach Date: Tue, 25 Mar 2025 13:23:03 +0100 Subject: [PATCH 18/23] Deactivated test for spm and xrd to remove blocker, deactivated the combined testing of all pynx plugins with pynxtools itself against nomad in favor for these tests should better be done in each plugin --- .github/workflows/nomad-requirements.yml | 4 ++-- .github/workflows/plugin_test.yaml | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/nomad-requirements.yml b/.github/workflows/nomad-requirements.yml index 14c93f2e8..567d82c04 100644 --- a/.github/workflows/nomad-requirements.yml +++ b/.github/workflows/nomad-requirements.yml @@ -32,7 +32,7 @@ jobs: working-directory: ./nomad run: | echo "" >> test_plugins.txt - echo "pynxtools[convert]@git+https://github.com/FAIRmat-NFDI/pynxtools.git@${{ github.head_ref || github.ref_name }}" >> test_plugins.txt + echo "pynxtools@git+https://github.com/FAIRmat-NFDI/pynxtools.git@${{ github.head_ref || github.ref_name }}" >> test_plugins.txt - name: Install uv and set the python version to ${{ env.python-version }} uses: astral-sh/setup-uv@v5 with: @@ -48,4 +48,4 @@ jobs: run: | uv pip install -r requirements-plugins.txt env: - PYTHONPATH: "" # Ensure no pre-installed packages interfere with the test \ No newline at end of file + PYTHONPATH: "" # Ensure no pre-installed packages interfere with the test diff --git a/.github/workflows/plugin_test.yaml b/.github/workflows/plugin_test.yaml index 5a8654912..e4be14f6e 100644 --- a/.github/workflows/plugin_test.yaml +++ b/.github/workflows/plugin_test.yaml @@ -38,15 +38,15 @@ jobs: - plugin: pynxtools-raman branch: main tests_to_run: tests/. - - plugin: pynxtools-spm - branch: main - tests_to_run: tests/. + # - plugin: pynxtools-spm + # branch: main + # tests_to_run: tests/. - plugin: pynxtools-xps branch: main tests_to_run: tests/. - - plugin: pynxtools-xrd - branch: main - tests_to_run: tests/. + # - plugin: pynxtools-xrd + # branch: main + # tests_to_run: tests/. steps: - uses: actions/checkout@v4 From 7a16338fb200f6a0e9f12706c9935dd34f45262b Mon Sep 17 00:00:00 2001 From: mkuehbach Date: Tue, 25 Mar 2025 13:55:10 +0100 Subject: [PATCH 19/23] Refactoring based on suggestion @lukaspie, @RubelMozumder inner function --- src/pynxtools/nomad/schema.py | 43 +++++++++++++++-------------------- 1 file changed, 18 insertions(+), 25 deletions(-) diff --git a/src/pynxtools/nomad/schema.py b/src/pynxtools/nomad/schema.py index 1f73bc6c9..7528d7a6d 100644 --- a/src/pynxtools/nomad/schema.py +++ b/src/pynxtools/nomad/schema.py @@ -184,6 +184,21 @@ class NexusIdentifiers(ArchiveSection): def normalize(self, archive, logger): # Consider multiple identifiers exists in the same group/section + def generate_anchored_reference_and_normalize(value, idname): + """Generate anchored reference, connect to m quantities, and normalize.""" + field_n = idname.split("__field")[0] + logger.info(f"Lab id {value} to be created") + nx_id = AnchoredReference(lab_id=value, name=field_n) + nx_id.m_set_section_attribute( + "m_nx_data_path", + self.m_get_quantity_attribute(idname, "m_nx_data_path"), + ) + nx_id.m_set_section_attribute( + "m_nx_data_file", + self.m_get_quantity_attribute(idname, "m_nx_data_file"), + ) + return nx_id + identifiers = [ key for key in self.__dict__.keys() @@ -197,35 +212,13 @@ def normalize(self, archive, logger): continue if isinstance(val, dict): for idname, idobj in val.items(): - # identifier_path = f"{self.m_def.name}_{identifier.split('__field')[0]}" - field_n = idname.split("__field")[0] - logger.info(f"Lab id {idobj.value} to be created") - nx_id = AnchoredReference(lab_id=idobj.value, name=field_n) - nx_id.m_set_section_attribute( - "m_nx_data_path", - self.m_get_quantity_attribute(idname, "m_nx_data_path"), - ) - nx_id.m_set_section_attribute( - "m_nx_data_file", - self.m_get_quantity_attribute(idname, "m_nx_data_file"), + nx_id = generate_anchored_reference_and_normalize( + idobj.value, idname ) - self.NXidentifiers.append(nx_id) nx_id.normalize(archive, logger) else: - # identifier_path = f"{self.m_def.name}_{identifier.split('__field')[0]}" - field_n = identifier.split("__field")[0] - logger.info(f"Lab id {val} to be created") - nx_id = AnchoredReference(lab_id=val, name=field_n) - nx_id.m_set_section_attribute( - "m_nx_data_path", - self.m_get_quantity_attribute(identifier, "m_nx_data_path"), - ) - nx_id.m_set_section_attribute( - "m_nx_data_file", - self.m_get_quantity_attribute(identifier, "m_nx_data_file"), - ) - + nx_id = generate_anchored_reference_and_normalize(val, identifier) self.NXidentifiers.append(nx_id) nx_id.normalize(archive, logger) From 7fec6369917248838823efdede31cfdb4aaa5871 Mon Sep 17 00:00:00 2001 From: mkuehbach Date: Tue, 25 Mar 2025 14:08:02 +0100 Subject: [PATCH 20/23] Updated individual plugins in preparation v0.10.1 --- dev-requirements.txt | 2 +- pyproject.toml | 20 ++++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 261e7d34c..588be7184 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -198,7 +198,7 @@ super-collections==0.5.3 # via mkdocs-macros-plugin termcolor==2.5.0 # via mkdocs-macros-plugin -tomli==2.2.1 ; python_full_version <= '3.11' +tomli==2.2.1 ; python_full_version == '3.11' # via coverage toposort==1.10 # via pynxtools (pyproject.toml) diff --git a/pyproject.toml b/pyproject.toml index cc59d830e..a988dda8a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,32 +66,32 @@ dev = [ "pre-commit", ] convert = [ - "pynxtools[apm,ellips,em,igor,mpes,raman,stm,xps,xrd]", + "pynxtools[apm,ellips,em,igor,mpes,raman,spm,xps,xrd]", ] apm = [ - "pynxtools-apm>=0.2.2", + "pynxtools-apm>=0.2.3", ] ellips = [ - "pynxtools-ellips>=0.0.7", + "pynxtools-ellips>=0.0.8", ] em = [ - "pynxtools-em>=0.3.1", + "pynxtools-em>=0.3.2", ] igor = [ - "pynxtools-igor>=0.1.0", + "pynxtools-igor>=0.1.1", ] mpes = [ - "pynxtools-mpes>=0.2.1", + "pynxtools-mpes>=0.2.2", ] raman = [ - "pynxtools-raman>=0.0.5", + "pynxtools-raman>=0.0.9", ] -stm = [ - "pynxtools-spm>=0.0.0", +spm = [ + "pynxtools-spm>=0.0.8", ] xps = [ - "pynxtools-xps>=0.4.8", + "pynxtools-xps>=0.5.0", ] xrd = [ "pynxtools-xrd>=0.0.3", From ac52c43492f4b2377b1309078baf25c6b9045c75 Mon Sep 17 00:00:00 2001 From: mkuehbach Date: Tue, 25 Mar 2025 15:15:24 +0100 Subject: [PATCH 21/23] Updated citation, dev-requirements.txt --- CITATION.cff | 11 ++++++++--- dev-requirements.txt | 2 +- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/CITATION.cff b/CITATION.cff index cf46d9c00..1752a5f69 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -4,7 +4,7 @@ message: If you use this software, please cite it using the metadata from this file. type: software -version: 0.10.0 +version: 0.10.1 authors: - given-names: Sherjeel family-names: Shabih @@ -60,7 +60,12 @@ authors: - given-names: Heiko B. family-names: Weber orcid: 'https://orcid.org/0000-0002-6403-9022' - + - given-names: Christoph T. + family-names: Koch + orcid: 'https://orcid.org/0000-0002-3984-1523' + - given-names: Claudia + family-names: Draxl + orcid: 'https://orcid.org/0000-0003-3523-6657' doi: 10.5281/zenodo.13862041 repository-code: 'https://github.com/FAIRmat-NFDI/pynxtools' url: https://doi.org/10.5281/zenodo.13862042 @@ -84,4 +89,4 @@ abstract: [FAIRmat page](https://www.fairmat-nfdi.eu/fairmat/about-fairmat/consortium-fairmat) for more info. -license: Apache-2.0 \ No newline at end of file +license: Apache-2.0 diff --git a/dev-requirements.txt b/dev-requirements.txt index 588be7184..261e7d34c 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -198,7 +198,7 @@ super-collections==0.5.3 # via mkdocs-macros-plugin termcolor==2.5.0 # via mkdocs-macros-plugin -tomli==2.2.1 ; python_full_version == '3.11' +tomli==2.2.1 ; python_full_version <= '3.11' # via coverage toposort==1.10 # via pynxtools (pyproject.toml) From 1de8f088eab3acc29e4a0bd906956e47afe9a012 Mon Sep 17 00:00:00 2001 From: Lukas Pielsticker <50139597+lukaspie@users.noreply.github.com> Date: Tue, 25 Mar 2025 15:47:45 +0100 Subject: [PATCH 22/23] do not return early in identifier normalization --- src/pynxtools/nomad/schema.py | 33 +++++++++++++-------------------- 1 file changed, 13 insertions(+), 20 deletions(-) diff --git a/src/pynxtools/nomad/schema.py b/src/pynxtools/nomad/schema.py index 7528d7a6d..2d1be9afb 100644 --- a/src/pynxtools/nomad/schema.py +++ b/src/pynxtools/nomad/schema.py @@ -176,7 +176,7 @@ def get_entry_reference(archive, f_name): class NexusIdentifiers(ArchiveSection): - NXidentifiers = SubSection( + NeXus_identifiers = SubSection( section_def=AnchoredReference, repeats=True, description="These are the NOMAD references correspond to NeXus identifierNAME fields.", @@ -197,31 +197,24 @@ def generate_anchored_reference_and_normalize(value, idname): "m_nx_data_file", self.m_get_quantity_attribute(idname, "m_nx_data_file"), ) - return nx_id + nx_id.normalize(archive, logger) + self.NeXus_identifiers.append(nx_id) identifiers = [ key for key in self.__dict__.keys() if key.startswith("identifier") and key.endswith("__field") ] - if not identifiers: - return - self.NXidentifiers = [] - for identifier in identifiers: - if not (val := getattr(self, identifier)): - continue - if isinstance(val, dict): - for idname, idobj in val.items(): - nx_id = generate_anchored_reference_and_normalize( - idobj.value, idname - ) - self.NXidentifiers.append(nx_id) - nx_id.normalize(archive, logger) - else: - nx_id = generate_anchored_reference_and_normalize(val, identifier) - self.NXidentifiers.append(nx_id) - nx_id.normalize(archive, logger) - + if identifiers: + self.NeXus_identifiers = [] + for identifier in identifiers: + if not (val := getattr(self, identifier)): + continue + if isinstance(val, dict): + for idname, idobj in val.items(): + generate_anchored_reference_and_normalize(idobj.value, idname) + else: + generate_anchored_reference_and_normalize(val, identifier) super().normalize(archive, logger) From f30a5f91962cdc2094f5a5fdba12e9eff07233c1 Mon Sep 17 00:00:00 2001 From: Lukas Pielsticker <50139597+lukaspie@users.noreply.github.com> Date: Tue, 25 Mar 2025 15:48:03 +0100 Subject: [PATCH 23/23] reactivate spm and xrd plugins --- .github/workflows/plugin_test.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/plugin_test.yaml b/.github/workflows/plugin_test.yaml index e4be14f6e..5a8654912 100644 --- a/.github/workflows/plugin_test.yaml +++ b/.github/workflows/plugin_test.yaml @@ -38,15 +38,15 @@ jobs: - plugin: pynxtools-raman branch: main tests_to_run: tests/. - # - plugin: pynxtools-spm - # branch: main - # tests_to_run: tests/. + - plugin: pynxtools-spm + branch: main + tests_to_run: tests/. - plugin: pynxtools-xps branch: main tests_to_run: tests/. - # - plugin: pynxtools-xrd - # branch: main - # tests_to_run: tests/. + - plugin: pynxtools-xrd + branch: main + tests_to_run: tests/. steps: - uses: actions/checkout@v4