From 410aa688947ec964febc9c722d49e2a12939f906 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Sun, 24 Oct 2021 21:14:03 +0200
Subject: [PATCH 1/4] Store system matrix in any case on device and allow
 update

---
 .../IOGKOMatrixHandler/IOGKOMatrixHandler.C   | 25 +++++++++----------
 .../IOGKOMatrixHandler/IOGKOMatrixHandler.H   |  5 ++++
 lduLduBase/lduLduBase.H                       |  7 ++++--
 3 files changed, 22 insertions(+), 15 deletions(-)

diff --git a/IOHandler/IOGKOMatrixHandler/IOGKOMatrixHandler.C b/IOHandler/IOGKOMatrixHandler/IOGKOMatrixHandler.C
index 5505f0310..8f2782260 100644
--- a/IOHandler/IOGKOMatrixHandler/IOGKOMatrixHandler.C
+++ b/IOHandler/IOGKOMatrixHandler/IOGKOMatrixHandler.C
@@ -32,11 +32,11 @@ namespace Foam {
 void IOGKOMatrixHandler::init_device_matrix(
     const objectRegistry &db, std::vector<scalar> &values_host,
     std::vector<label> &col_idxs_host, std::vector<label> &row_idxs_host,
-    const label nElems, const label nCells, const bool store) const
+    const label nElems, const label nCells, const bool update) const
 {
     std::shared_ptr<gko::Executor> device_exec = get_device_executor();
 
-    if (sys_matrix_stored_) {
+    if (sys_matrix_stored_ && !update) {
         gkomatrix_ptr_ = &db.lookupObjectRef<GKOCSRIOPtr>(sys_matrix_name_);
         return;
     }
@@ -79,20 +79,19 @@ void IOGKOMatrixHandler::init_device_matrix(
 
 
     // if updating system matrix is not needed store ptr in obj registry
-    if (store) {
-        const fileName path = sys_matrix_name_;
+    const fileName path = sys_matrix_name_;
+    if (!sys_matrix_stored_) {
         gkomatrix_ptr_ = new GKOCSRIOPtr(IOobject(path, db), gkomatrix);
+        // in any case store sparsity pattern
+        const fileName path_col = sparsity_pattern_name_cols_;
+        const fileName path_row = sparsity_pattern_name_rows_;
+        io_col_idxs_ptr_ = new GKOIDXIOPtr(IOobject(path_col, db), col_idx);
+        io_row_idxs_ptr_ = new GKOIDXIOPtr(IOobject(path_row, db), row_idx);
     } else {
-        gkomatrix_ptr_ = NULL;
-        gkomatrix_ = gkomatrix;
+        Info << "!!! matrix has been updated " << endl;
+        // TODO delet old pointer
+        gkomatrix_ptr_ = new GKOCSRIOPtr(IOobject(path, db), gkomatrix);
     }
-
-
-    // in any case store sparsity pattern
-    const fileName path_col = sparsity_pattern_name_cols_;
-    const fileName path_row = sparsity_pattern_name_rows_;
-    io_col_idxs_ptr_ = new GKOIDXIOPtr(IOobject(path_col, db), col_idx);
-    io_row_idxs_ptr_ = new GKOIDXIOPtr(IOobject(path_row, db), row_idx);
 };
 
 
diff --git a/IOHandler/IOGKOMatrixHandler/IOGKOMatrixHandler.H b/IOHandler/IOGKOMatrixHandler/IOGKOMatrixHandler.H
index 780af25e4..9d968a9de 100644
--- a/IOHandler/IOGKOMatrixHandler/IOGKOMatrixHandler.H
+++ b/IOHandler/IOGKOMatrixHandler/IOGKOMatrixHandler.H
@@ -99,6 +99,11 @@ public:
 
     bool get_sys_matrix_stored() const { return sys_matrix_stored_; };
 
+    bool get_sparsity_pattern_stored() const
+    {
+        return sparsity_pattern_stored_;
+    };
+
     void init_device_matrix(const objectRegistry &db,
                             std::vector<scalar> &values_host,
                             std::vector<label> &col_idxs_host,
diff --git a/lduLduBase/lduLduBase.H b/lduLduBase/lduLduBase.H
index 3dc782457..38524036e 100644
--- a/lduLduBase/lduLduBase.H
+++ b/lduLduBase/lduLduBase.H
@@ -94,8 +94,11 @@ public:
         // initially
         bool stored = get_sys_matrix_stored();
         if (!stored) {
-            SIMPLE_TIME(verbose_, init_host_sparsity_pattern,
-                        this->init_host_sparsity_pattern();)
+            if (!this->sparsity_pattern_stored) {
+                SIMPLE_TIME(verbose_, init_host_sparsity_pattern,
+                            this->init_host_sparsity_pattern();)
+            }
+
             SIMPLE_LOG(verbose_, "matrix not stored update host matrix")
             SIMPLE_TIME(verbose_, update_host_mtx,
                         this->update_host_matrix_data();)

From 5cbb65e4ed89125297d970a92cfa063db2ac7bca Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Sun, 24 Oct 2021 21:26:20 +0200
Subject: [PATCH 2/4] FIX init sparsity pattern if not stored

---
 lduLduBase/lduLduBase.H | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/lduLduBase/lduLduBase.H b/lduLduBase/lduLduBase.H
index 38524036e..6a53ffda4 100644
--- a/lduLduBase/lduLduBase.H
+++ b/lduLduBase/lduLduBase.H
@@ -94,10 +94,8 @@ public:
         // initially
         bool stored = get_sys_matrix_stored();
         if (!stored) {
-            if (!this->sparsity_pattern_stored) {
-                SIMPLE_TIME(verbose_, init_host_sparsity_pattern,
-                            this->init_host_sparsity_pattern();)
-            }
+            SIMPLE_TIME(verbose_, init_host_sparsity_pattern,
+                        this->init_host_sparsity_pattern();)
 
             SIMPLE_LOG(verbose_, "matrix not stored update host matrix")
             SIMPLE_TIME(verbose_, update_host_mtx,

From 29cfe9663bd0c0fb25bfef86c74530e89467c2b9 Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Sun, 24 Oct 2021 21:28:44 +0200
Subject: [PATCH 3/4] Add resorting of host matrix

---
 lduLduBase/lduLduBase.H | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/lduLduBase/lduLduBase.H b/lduLduBase/lduLduBase.H
index 6a53ffda4..c2a31ed3d 100644
--- a/lduLduBase/lduLduBase.H
+++ b/lduLduBase/lduLduBase.H
@@ -109,6 +109,10 @@ public:
                     "matrix is stored and update of host matrix requested")
                 SIMPLE_TIME(verbose_, exp_update_host_mtx,
                             this->update_host_matrix_data();)
+                SIMPLE_LOG(verbose_, "host matrix needs to be resorted")
+                SIMPLE_TIME(
+                    verbose_, sort_host_mtx,
+                    this->sort_host_matrix_data(this->get_sorting_idxs());)
             }
         }
 

From 22a9c8ee8e6f5acd697cc48a89fb9ab44cf9d89b Mon Sep 17 00:00:00 2001
From: Gregor Olenik <gregor.olenik@web.de>
Date: Sun, 24 Oct 2021 22:09:12 +0200
Subject: [PATCH 4/4] add timings and verbosity

---
 IOHandler/IOGKOMatrixHandler/IOGKOMatrixHandler.C | 6 ++++--
 IOHandler/IOGKOMatrixHandler/IOGKOMatrixHandler.H | 7 +++++--
 lduLduBase/lduLduBase.H                           | 2 +-
 3 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/IOHandler/IOGKOMatrixHandler/IOGKOMatrixHandler.C b/IOHandler/IOGKOMatrixHandler/IOGKOMatrixHandler.C
index 8f2782260..60f34a3dc 100644
--- a/IOHandler/IOGKOMatrixHandler/IOGKOMatrixHandler.C
+++ b/IOHandler/IOGKOMatrixHandler/IOGKOMatrixHandler.C
@@ -26,6 +26,7 @@ SourceFiles
 \*---------------------------------------------------------------------------*/
 #include <ginkgo/ginkgo.hpp>
 #include "IOGKOMatrixHandler.H"
+#include "common.H"
 
 namespace Foam {
 
@@ -75,7 +76,9 @@ void IOGKOMatrixHandler::init_device_matrix(
     auto gkomatrix =
         gko::share(mtx::create(device_exec, gko::dim<2>(nCells, nCells)));
 
+    SIMPLE_TIME(verbose_, convert_coo_to_csr,
     coo_mtx->convert_to(gkomatrix.get());
+    )
 
 
     // if updating system matrix is not needed store ptr in obj registry
@@ -88,8 +91,7 @@ void IOGKOMatrixHandler::init_device_matrix(
         io_col_idxs_ptr_ = new GKOIDXIOPtr(IOobject(path_col, db), col_idx);
         io_row_idxs_ptr_ = new GKOIDXIOPtr(IOobject(path_row, db), row_idx);
     } else {
-        Info << "!!! matrix has been updated " << endl;
-        // TODO delet old pointer
+        SIMPLE_LOG(verbose_, "Matrix has been updated ");
         gkomatrix_ptr_ = new GKOCSRIOPtr(IOobject(path, db), gkomatrix);
     }
 };
diff --git a/IOHandler/IOGKOMatrixHandler/IOGKOMatrixHandler.H b/IOHandler/IOGKOMatrixHandler/IOGKOMatrixHandler.H
index 9d968a9de..9e22b6ff8 100644
--- a/IOHandler/IOGKOMatrixHandler/IOGKOMatrixHandler.H
+++ b/IOHandler/IOGKOMatrixHandler/IOGKOMatrixHandler.H
@@ -61,6 +61,8 @@ private:
 
     const bool update_init_guess_vector_;
 
+    const bool verbose_;
+
     const bool export_;
 
     mutable label prev_solve_iters_ = 0;
@@ -94,7 +96,8 @@ public:
               db.foundObject<regIOobject>(init_guess_vector_name_)),
           update_init_guess_vector_(
               controlDict.lookupOrDefault<Switch>("updateInitVector", false)),
-
+          verbose_(
+              controlDict.lookupOrDefault<Switch>("verbose", false)),
           export_(controlDict.lookupOrDefault<Switch>("export", false)){};
 
     bool get_sys_matrix_stored() const { return sys_matrix_stored_; };
@@ -109,7 +112,7 @@ public:
                             std::vector<label> &col_idxs_host,
                             std::vector<label> &row_idxs_host,
                             const label nElems, const label nCells,
-                            const bool store) const;
+                            const bool update) const;
 
     std::shared_ptr<mtx> get_gkomatrix() const
     {
diff --git a/lduLduBase/lduLduBase.H b/lduLduBase/lduLduBase.H
index c2a31ed3d..a45e8e942 100644
--- a/lduLduBase/lduLduBase.H
+++ b/lduLduBase/lduLduBase.H
@@ -138,7 +138,7 @@ public:
 
         init_device_matrix(this->matrix().mesh().thisDb(), this->values(),
                            this->col_idxs(), this->row_idxs(), this->nElems(),
-                           this->nCells(), !get_update_sys_matrix());
+                           this->nCells(), get_update_sys_matrix());
     }