FAIRmat-NFDI · sanbrock · Aug 10, 2022 · Aug 2, 2022 · Aug 2, 2022 · Aug 3, 2022
diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml
@@ -22,6 +22,7 @@ jobs:
         python -m pip install --upgrade pip
         pip install nomad-lab==1.1.1 --extra-index-url https://gitlab.mpcdf.mpg.de/api/v4/projects/2187/packages/pypi/simple
         if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+        pip install astroid==2.5.1
     - name: Install package
       run: |
         python -m pip install .

diff --git a/nexusparser/definitions b/nexusparser/definitions
diff --git a/nexusparser/tools/dataconverter/Readme.md b/nexusparser/tools/dataconverter/Readme.md
@@ -42,7 +42,7 @@ For now, I have made sure that this tools works on this branch.
 
 ## Writing a Reader
 
-This converter allows extending support to other data formats by allowing extensions called readers.  The converter provides a dev platform to build a Nexus compatible reader by providing checking against a chosen Nexus Application Definition.
+This converter allows extending support to other data formats by allowing extensions called readers. The converter provides a dev platform to build a Nexus compatible reader by providing checking against a chosen Nexus Application Definition.
 
 Readers have to be placed in the **readers** folder in there own subfolder. The reader folder should be named with the reader's name and contain a `reader.py`.\
 For example: The reader `Example Reader` is placed under `dataconverter/readers/example/reader.py`.
@@ -53,15 +53,20 @@ Then implement the reader function:
 
 ```python
 """MyDataReader implementation for the DataConverter to convert mydata to Nexus."""
-from typing import Tuple
+from typing import Tuple, Any
 
-from nexusparser.tools.dataconverter.readers.base_reader import BaseReader
+from nexusparser.tools.dataconverter.readers.base
8000
.reader import BaseReader
 
 
 class MyDataReader(BaseReader):
     """MyDataReader implementation for the DataConverter to convert mydata to Nexus."""
 
-    def read(self, template: dict = None, file_paths: Tuple[str] = None) -> dict:
+    def read(
+        self,
+        template: dict = None,
+        file_paths: Tuple[str] = None,
+        objects: Tuple[Any] = None
+    ) -> dict:
         """Reads data from given file and returns a filled template dictionary"""
         # Fill the template
         for path in file_paths:
@@ -81,8 +86,9 @@ The read function takes a template dictionary based on the provided NXDL file (s
 The returned dictionary should contain keys that exist in the template as defined below. The values of these keys have to be data objects to be populated in the output Nexus file. They can be lists, numpy arrays, numpy bytes, numpy floats, numpy ints. Practically you can pass any value that can be handled by `h5py` package.
 
 Then you can then call this using:
+
 ```console
-user@box:~$ python convert.py --reader mydata --nxdl NXmynxdl --output path_to_output.nxs
+user@box:~$ dataconverter --reader mydata --nxdl NXmynxdl --output path_to_output.nxs
 ```
 
 ### The reader template dictionary
@@ -91,16 +97,16 @@ Example:
 
 ```json
 {
-    "/entry/instrument/source/type": "None"
+  "/entry/instrument/source/type": "None"
 }
 ```
 
 **Units**: If there is a field defined in the NXDL, the converter expects a filled in /data/@units entry in the template dictionary corresponding to the right /data field unless it is specified as NX_UNITLESS in the NXDL. Otherwise, you will get an exception.
 
 ```json
 {
-    "/ENTRY[my_entry]/INSTRUMENT[my_instrument]/SOURCE[my_source]/data": "None",
-    "/ENTRY[my_entry]/INSTRUMENT[my_instrument]/SOURCE[my_source]/data/@units": "Should be set to a string value"
+  "/ENTRY[my_entry]/INSTRUMENT[my_instrument]/SOURCE[my_source]/data": "None",
+  "/ENTRY[my_entry]/INSTRUMENT[my_instrument]/SOURCE[my_source]/data/@units": "Should be set to a string value"
 }
 ```
 
@@ -109,15 +115,15 @@ You can choose any name you prefer instead of the suggested name. This allows th
 
 ```json
 {
-    "/ENTRY[my_entry]/INSTRUMENT[my_instrument]/SOURCE[my_source]/type": "None"
+  "/ENTRY[my_entry]/INSTRUMENT[my_instrument]/SOURCE[my_source]/type": "None"
 }
 ```
 
 For attributes defined in the NXDL, the reader template dictionary will have the assosciated key with a "@" prefix to the attributes name at the end of the path:
 
 ```json
 {
-    "/entry/instrument/source/@attribute": "None"
+  "/entry/instrument/source/@attribute": "None"
 }
 ```
 

diff --git a/nexusparser/tools/dataconverter/readers/mpes/reader.py b/nexusparser/tools/dataconverter/readers/mpes/reader.py
@@ -19,7 +19,6 @@
 import errno
 import json
 import os
 import collections
 from functools import reduce
 from typing import Any
 from typing import Tuple
@@ -29,6 +28,7 @@
 import yaml
 
 from nexusparser.tools.dataconverter.readers.base.reader import BaseReader
+from nexusparser.tools.dataconverter.readers.utils import flatten_and_replace
 
 DEFAULT_UNITS = {
     "X": "step",
@@ -172,32 +172,6 @@ def iterate_dictionary(dic, key_string):
 }
 
 
-def flatten_and_replace(dic, parent_key='/ENTRY[entry]', sep='/'):
-    """Flatten a nested dictionary, and replace the keys with the appropriate
-    paths in the nxs file.
-    Args:
-        d (dict): dictionary to flatten
-        parent_key (str): parent key of the dictionary
-        sep (str): separator for the keys
-    Returns:
-        dict: flattened dictionary
-    """
-    items = []
-    for key, val in dic.items():
-        new_key = parent_key + sep + CONVERT_DICT.get(key, key)
-        if isinstance(val, collections.Mapping):
-            items.extend(flatten_and_replace(val, new_key, sep=sep).items())
-        else:
-            for old, new in REPLACE_NESTED.items():
-                new_key = new_key.replace(old, new)
-
-            if new_key.endswith('/value'):
-                items.append((new_key[:-6], val))
-            else:
-                items.append((new_key, val))
-    return dict(items)
-
-
 def handle_h5_and_json_file(file_paths, objects):
     """Handle h5 or json input files."""
     x_array_loaded = xr.DataArray()
@@ -250,7 +224,9 @@ def handle_h5_and_json_file(file_paths, objects):
                 config_file_dict = json.load(file)
         elif file_extension in [".yaml", ".yml"]:
             with open(file_path) as feln:
-                eln_data_dict = flatten_and_replace(yaml.safe_load(feln))
+                eln_data_dict = flatten_and_replace(
+                    yaml.safe_load(feln), CONVERT_DICT, REPLACE_NESTED
+                )
 
     if objects is not None:
         # For the case of a single object

diff --git a/nexusparser/tools/dataconverter/readers/transmission/__init__.py b/nexusparser/tools/dataconverter/readers/transmission/__init__.py
diff --git a/nexusparser/tools/dataconverter/readers/transmission/metadata_parsers.py b/nexusparser/tools/dataconverter/readers/transmission/metadata_parsers.py
@@ -0,0 +1,64 @@
+#
+# Copyright The NOMAD Authors.
+#
+# This file is part of NOMAD. See https://nomad-lab.eu for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""Functions for reading metadata values from Perkin Ellmer files"""
+from datetime import datetime
+
+# The min & max wavelength the instrument can measure
+MIN_WAVELENGTH = 190.
+MAX_WAVELENGTH = 3350.
+
+
+def read_start_date(metadata: list) -> str:
+    """Reads the start date from the metadata"""
+    century = str(datetime.now().year // 100)
+    formated_date = metadata[3].replace("/", "-")
+    return f"{century}{formated_date}T{metadata[4]}000Z"
+
+
+def read_sample_attenuator(metadata: list) -> int:
+    """Reads the sample attenuator from the metadata"""
+    return int(metadata[47].split()[0].split(":")[1])
+
+
+def read_ref_attenuator(metadata: list) -> int:
+    """Reads the sample attenuator from the metadata"""
+    return int(metadata[47].split()[1].split(":")[1])
+
+
+def read_uv_monochromator_range(metadata: list) -> list:
+    """Reads the uv monochromator range from the metadata"""
+    monochromator_change = float(metadata[41])
+    return [MIN_WAVELENGTH, monochromator_change]
+
+
+def read_visir_monochromator_range(metadata: list) -> list:
+    """Reads the visir monochromator range from the metadata"""
+    monochromator_change = float(metadata[41])
+    return [monochromator_change, MAX_WAVELENGTH]
+
+
+def get_d2_range(metadata: list) -> list:
+    """Reads the D2 lamp range from the metadata"""
+    lamp_change = float(metadata[42])
+    return [MIN_WAVELENGTH, lamp_change]
+
+
+def get_halogen_range(metadata: list) -> list:
+    """Reads the halogen lamp range from the metadata"""
+    lamp_change = float(metadata[42])
+    return [lamp_change, MAX_WAVELENGTH]
+1 −1		Makefile
+0 −189		contributed_definitions/NXIVtemp.nxdl.xml
+2 −1		contributed_definitions/NXbeam.nxdl.xml
+386 −0		contributed_definitions/NXiv_temp.nxdl.xml
+327 −0		contributed_definitions/NXtransmission.nxdl.xml
+18 −12		manual/source/apm-structure.rst
+1 −21		manual/source/classes/applications/index.rst
+3 −79		manual/source/classes/base_classes/index.rst
+241 −5		manual/source/classes/contributed_definitions/index.rst
+100 −11		manual/source/em-structure.rst
+40 −0		manual/source/epics-structure.rst
+100 −58		manual/source/fairmat-cover.rst
+1 −0		manual/source/index.rst
+1 −0		manual/source/sed/landing-page.rst