MetOffice · hiker · Aug 14, 2024 · Aug 14, 2024 · Aug 16, 2024 · Aug 16, 2024
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -1,14 +1,6 @@
 name: Build
 
-    on:
-  push:
-    branches:
-      - main
-      - v*.*
-  pull_request:
-    branches:
-      - main
-      - v*.*
+on: [push, pull_request]
 
 jobs:
   build:
@@ -17,7 +9,7 @@ jobs:
     strategy:
       max-parallel: 4
       matrix:
-        python-version: ['3.9', '3.10', '3.11', '3.12', '3.13']
+        python-version: ['3.9', '3.10']
 
     steps:
       # Should this step use a cache?
@@ -56,10 +48,10 @@ jobs:
 
       - name: Unit tests with Pytest
         run: |
-          pip install .[tests]
+          pip install .[dev]
           python -m pytest --cov=fab tests/unit_tests
 
       - name: System tests with Pytest
         run: |
-          pip install .[tests]
+          pip install .[dev]
           python -m pytest --cov=fab tests/system_tests
diff --git a/Documentation/source/advanced_config.rst b/Documentation/source/advanced_config.rst
@@ -11,29 +11,9 @@ aspects of software building may require more esoteric Fab features.
 
 Understanding the Environment
 =============================
-
-Fab uses well-known environment variables to identify tools and configure them.
-
-
-.. list-table:: Environment variables
-
-   * - FPP
-     - Fortran preprocessor, e.g ``fpp`` or ``cpp -traditional-cpp -P``.
-       Fab imposes the ``-P``.
-   * - FC
-     - Fortran compiler, e.g ``gfortran`` or ``ifort -c``.
-       Fab imposes the ``-c``.
-   * - FFLAGS
-     - Fortran compiler flags..
-   * - CC
-     - C compiler.
-   * - CFLAGS
-     - C compiler flags.
-       * - LD
-     - Linker, e.g ``ld``.
-   * - LFLAGS
-     - Linker flags.
-
+Fab itself does not support any environment variables. But a user
+script can obviously query the environment and make use of environment
+variables, and provide their values to Fab.
 
 Configuration Reuse
 ===================
@@ -176,6 +156,137 @@ notification, as it needs to use this flag to control the output location.
 
 .. _Advanced Flags:
 
+Compilation Profiles
+====================
+Fab supports compilation profiles. A compilation profile is essentially a simple
+string that represents a set of compilation and linking flags to be used.
+For example, an application might have profiles for `full-debug`, `fast-debug`,
+and `production`. Compilation profiles can inherit settings, for example
+`fast-debug` might inherit from `full-debug`, but add optimisations.
+
+Any flag for any tool can make use of a profile, but in many cases this is
+not necessary (think of options for `rsync`, `git`, `svn`, ...). Fab will
+internally create a dummy profile, indicated by an empty string `""`. If no
+profile is specified, this default profile will be used.
+
+A profile is defined as follows:
+
+.. code-block::
+    :linenos:
+
+    tr = ToolRepository()
+    gfortran = tr.get_tool(Category.FORTRAN_COMPILER, "gfortran")
+
+    gfortran.define_profile("base")
+    gfortran.define_profile("fast-debug", inherit_from="base")
+    gfortran.define_profile("full-debug", inherit_from="fast-debug")
+
+    gfortran.add_flags(["-g", '-std=f2008'], "base")
+    gfortran.add_flags(["-O2], "fast-debug")
+    gfortran.add_flags(["-O0", "-fcheck=all"], "full-debug")
+
+Line 3 defines a profile called ``base``, which does not inherit from any
+other profile. Next, a profile ``fast-debug`` is defined, which is based
+on ``base``. It will add the flags ``-O2`` to the command line, together
+with the inherited flags from base, it will be using ``-g -std=f2008 -O2``
+Finally, a ``full-debug`` profile is declared, based on ``fast-debug``.
+Due to the inheritance, it will be using the options
+``-g -std=f2008 -O2 -O0 -fcheck=all``. Note that because of the precedence
+of compiler flags, the no-optimisation flag ``-O0`` will overwrite the
+valued of ``-O2``.
+
+Tools that do not want to use a profile just do not specify the
+profile parameter when defining flags:
+
+.. code-block::
+    :linenos:
+
+    git = config.tool_box[Category.GIT]
+    git.add_flags(["-c", "foo.bar=123"])
+
+This will internally add these flags to the dummy profile ``""``, and
+this will be used by various Fab functions.
+
+By default, the dummy profile ``""`` is not used as a base class for
+any other profile. But it can be convenient to set this up to make
+user scripts slightly easier. Here is an example of the usage
+in LFRic, where at startup time a consistent set of profile modes are
+defined for any compiler and linker:
+
+.. code-block::
+    :linenos:
+
+    tr = ToolRepository()
+    for compiler in (tr[Category.C_COMPILER] +
+                     tr[Category.FORTRAN_COMPILER] +
+                     tr[Category.LINKER]):
+        compiler.define_profile("base", inherit_from="")
+        for profile in ["full-debug", "fast-debug", "production"]:
+            compiler.define_profile(profile, inherit_from="base")
+
+Line 5 defines a ``base`` profile, which inherits from the dummy
+profile ``""``. Then a set of three profiles are defined, each
+inheriting from ``base`` (and therefore in turn from ``""``).
+
+Later, the Intel Fortran compiler and linker ``ifort`` are setup as follows:
+
+.. code-block::
+    :linenos:
+
+    tr = ToolRepository()
+    ifort = tr.get_tool(Category.FORTRAN_COMPILER, "ifort")
+    ifort.add_flags(["-stand", "f08"],           "base")
+    ifort.add_flags(["-g", "-traceback"],        "base")
+    ifort.add_flags(["-O0", "-ftrapuv"],         "full-debug")
+    ifort.add_flags(["-O2", "-fp-model=strict"], "fast-debug")
+    ifort.add_flags(["-O3", "-xhost"],           "production")
+
+    linker = tr.get_tool(Category.LINKER, "linker-ifort")
+    linker.add_lib_flags("yaxt", ["-lyaxt", "-lyaxt_c"])
+    linker.add_post_lib_flags(["-lstdc++"])
+
+The setup of the compiler does not use the dummy profile ``""`` at all,
+so it will stay empty. It is up to the user to decide how to use the
+profiles, it would be entirely valid not to use the ``base`` profile, but
+instead to use ``""``. But when setting up the linker, no profile is specified.
+So line 10 and 11 will set these flags for ``""``. Because of ``base``
+inheriting from ``""``, and any other profile inheriting from ``base``,
+this means these linker flags will be used for all profiles. It would
+be equally valid to define these flags for the ``base`` profile:
+
+.. code-block::
+    :linenos:
+
+    linker = tr.get_tool(Category.LINKER, "linker-ifort")
+    linker.add_lib_flags("yaxt", ["-lyaxt", "-lyaxt_c"], "base")
+    linker.add_post_lib_flags(["-lstdc++"], "base")
+
+This design was chosen, since the most common use case for compiler
+profiles is in changing compiler flags, linker flags are typically
+not affected. So it is more intuitive for a user not to specify
+the profile modes for linker.
+
+The advantage of supporting the profile modes for linker is that
+you can specify profile modes that require additional linking options.
+One example is GNU's address sanitizer, which requires to
+add the compilation option ``-fsanitize=address``, and the linker option
+``-static-libasan``.
+
+.. code-block::
+    :linenos:
+
+    tr = ToolRepository()
+    gfortran = tr.get_tool(Category.FORTRAN_COMPILER, "gfortran")
+    ...
+    gfortran.define_profile("memory-debug", "full-debug")
+    gfortran.add_flags(["-fsanitize=address"], "memory-debug")
+    linker = tr.get_tool(Category.LINKER, "linker-gfortran")
+    linker.add_post_lib_flags(["-static-libasan"], "memory-debug")
+
+This way, by just changing the profile, compilation and linking
+will be affected consistently.
+
+
 Tool arguments
 ============== 
 

diff --git a/Documentation/source/site-specific-config.rst b/Documentation/source/site-specific-config.rst
@@ -60,24 +60,6 @@ New tools can easily be created, look at
 created by providing a different set of parameters in the
 constructor.
 
-This also allows compiler wrappers to be easily defined. For example,
-if you want to use `mpif90` as compiler, which is a MPI-specific
-wrapper for `ifort`, you can create this class as follows:
-
-.. code-block::
-    :linenos:
-    :caption: Compiler wrapper
-
-    from fab.tools import Ifort
-
-    class MpiF90(Ifort):
-        '''A simple compiler wrapper'''
-        def __init__(self):
-            super().__init__(name="mpif90-intel",
-                             exec_name="mpif90")
-
-.. note:: In `ticket 312 <https://github.com/metomi/fab/issues/312>`_ a better
-        implementation of compiler wrapper will be implemented.
 
 Tool Repository
 ===============

diff --git a/Documentation/source/writing_config.rst b/Documentation/source/writing_config.rst
@@ -102,9 +102,6 @@ It filters just the (uppercase) ``.F90`` files.
 
     Uppercase ``.F90`` are preprocessed into lowercase ``.f90``.
 
-The Fortran preprocessor will read the :ref:`FPP<env_vars>` environment variable to determine which tool to call.
-
-
 .. code-block::
     :linenos:
     :caption: build_it.py

diff --git a/pyproject.toml b/pyproject.toml
@@ -6,7 +6,7 @@ authors = [
 ]
 license = {file = 'LICENSE.txt'}
 dynamic = ['version', 'readme']
-requires-python = '>=3.7, <4'
+requires-python = '>=3.9, <4'
 dependencies = ['fparser >= 0.2']
 classifiers = [
     'Development Status :: 1 - Planning',
@@ -21,13 +21,19 @@ classifiers = [
 [project.optional-dependencies]
 c-language = ['libclang']
 plots = ['matplotlib']
-tests = ['pytest', 'pytest-cov', 'pytest-mock']
-checks = ['flake8>=5.0.4', 'mypy']
 docs = ['sphinx',
         'pydata-sphinx-theme>=0.13.3',
         'sphinx-autodoc-typehints',
         'sphinx-copybutton']
-dev = ['sci-fab[plots, tests, checks, docs]']
+dev = [
+    'pytest >= 8.3.0',
+    'pytest-cov',
+    'pytest-mock',
+    'pytest-subprocess >= 1.5.3',
+    'pyfakefs',
+    'flake8 >= 5.0.4',
+    'mypy'
+]
 
 [project.scripts]
 fab = 'fab.cli:cli_fab'

diff --git a/source/fab/artefacts.py b/source/fab/artefacts.py
@@ -82,13 +82,16 @@ def add(self, collection: Union[str, ArtefactSet],
         self[collection].update(files)
 
     def update_dict(self, collection: Union[str, ArtefactSet],
-                    key: str, values: Union[str, Iterable]):
-        '''For ArtefactSets that are a dictionary of sets: update
-        the set with the specified values.
        :param collection: the name of the collection to add this to.
-        :param key: the key in the dictionary to update.
-        :param values: the values to update with.
-        '''
+                    values: Union[str, Iterable],
+                    key: Optional[str] = None):
+        """
+        Modifies data associated with artefact set.
+
+        :param collection: Name or enumeration of set to modify.
+        :param values: New data for set.
+        :param key: Executable name associated with data. Do not specify for
+                    libraries.
+        """
         self[collection][key].update([values] if isinstance(values, str)
                                      else values)
 

diff --git a/source/fab/build_config.py b/source/fab/build_config.py
@@ -44,6 +44,7 @@ def __init__(self, project_label: str,
                  tool_box: ToolBox,
                  mpi: bool = False,
                  openmp: bool = False,
+                 profile: Optional[str] = None,
                  multiprocessing: bool = True,
                  n_procs: Optional[int] = None,
                  reuse_artefacts: bool = False,
@@ -63,6 +64,7 @@ def __init__(self, project_label: str,
             (if none is explicitly set in the ToolBox). The compiler-specific
             flag to enable OpenMP will automatically be added when compiling
             and linking.
+        :param profile: the name of a compiler profile to use.
         :param multiprocessing:
             An option to disable multiprocessing to aid debugging.
         :param n_procs:
@@ -87,6 +89,11 @@ def __init__(self, project_label: str,
         self._tool_box = tool_box
         self._mpi = mpi
         self._openmp = openmp
+        if profile is None:
+            # An empty string is used for non-profiled flags
+            self._profile = ""
+        else:
+            self._profile = profile
         self.two_stage = two_stage
         self.verbose = verbose
         compiler = tool_box.get_tool(Category.FORTRAN_COMPILER, mpi=mpi,
@@ -205,6 +212,11 @@ def openmp(self) -> bool:
         ''':returns: whether OpenMP is requested or not in this config.'''
         return self._openmp
 
+    @property
+    def profile(self) -> str:
+        ''':returns: the name of the compiler profile to use.'''
+        return self._profile
+
     def add_current_prebuilds(self, artefacts: Iterable[Path]):
         """
         Mark the given file paths as being current prebuilds, not to be

diff --git a/source/fab/dep_tree.py b/source/fab/dep_tree.py
@@ -12,7 +12,7 @@
 from abc import ABC
 import logging
 from pathlib import Path
-from typing import Set, Dict, Iterable, List, Union, Optional, Any
+from typing import Any, Dict, Iterable, List, Optional, Set, Union
 
 from fab.parse import AnalysedFile
 
@@ -98,7 +98,8 @@ def from_dict(cls, d):
         return result
 
 
-def extract_sub_tree(source_tree: Dict[Path, AnalysedDependent], root: Path, verbose=False)\
+def extract_sub_tree(source_tree: Dict[Path, AnalysedDependent],
+                     root: Path, verbose=False)\
         -> Dict[Path, AnalysedDependent]:
     """
     Extract the subtree required to build the target, from the full source tree of all analysed source files.
@@ -114,16 +115,24 @@ def extract_sub_tree(source_tree: Dict[Path, AnalysedDependent], root: Path, ver
     result: Dict[Path, AnalysedDependent] = dict()
     missing: Set[Path] = set()
 
-    _extract_sub_tree(src_tree=source_tree, key=root, dst_tree=result, missing=missing, verbose=verbose)
+    _extract_sub_tree(src_tree=source_tree,
+                      key=root,
+                      dst_tree=result,
+                      missing=missing,
+                      verbose=verbose)
 
     if missing:
         logger.warning(f"{root} has missing deps: {missing}")
 
     return result
 
 
-def _extract_sub_tree(src_tree: Dict[Path, AnalysedDependent], key: Path,
-                      dst_tree: Dict[Path, AnalysedDependent], missing: Set[Path], verbose: bool, indent: int = 0):
+def _extract_sub_tree(src_tree: Dict[Path, AnalysedDependent],
+                      key: Path,
+                      dst_tree: Dict[Path, AnalysedDependent],
+                      missing: Set[Path],
+                      verbose: bool,
+                      indent: int = 0):
     # is this node already in the sub tree?
     if key in dst_tree:
         return

diff --git a/source/fab/parse/x90.py b/source/fab/parse/x90.py
@@ -12,7 +12,6 @@
 from fab.parse import AnalysedFile
 from fab.build_config import BuildConfig
 from fab.parse.fortran_common import FortranAnalyserBase, logger, _typed_child
-
 from fab.util import by_type