ogs monitor

ogs6py/log_parser/common_ogs_analyses.py

@@ -49,7 +49,7 @@ def wrapped_f(df):
 
 
 def analysis_time_step(df):
-    interest1 = ['output_time', 'time_step_solution_time', 'step_size']
+    interest1 = ['output_time', 'step_start_time', 'time_step_solution_time', 'step_size']
     interest2 = ['assembly_time', 'linear_solver_time', 'dirichlet_time']
     interest = [*interest1, *interest2]
     context = ['mpi_process', 'time_step']

ogs6py/log_parser/log_parser.py

@@ -40,7 +40,7 @@ def mpi_processes(file_name):
         return processes
 
 
-def parse_file(file_name, maximum_lines=None, force_parallel=False):
+def parse_file(file_name, maximum_lines=None, force_parallel=False, start_line=0):
     ogs_res = ogs_regexes()
     parallel_log = force_parallel or mpi_processes(file_name) > 1
 
@@ -61,16 +61,19 @@ def compile_re_fn(mpi_process_regex):
     with open(file_name) as file:
         lines = iter(file)
         records = list()
+        last_line = 0
         for line in lines:
-            number_of_lines_read += 1
+            last_line = last_line + 1
+            if last_line > start_line:
+                number_of_lines_read += 1
 
-            if (maximum_lines is not None) and (maximum_lines > number_of_lines_read):
-                break
-
-            for key, value in patterns:
-                if r := try_match(line, number_of_lines_read, key, value):
-                    records.append(value(*r))
+                if (maximum_lines is not None) and (maximum_lines > number_of_lines_read):
                     break
 
-    return records
+                for key, value in patterns:
+                    if r := try_match(line.replace("-nan","nan"), number_of_lines_read, key, value):
+                        records.append(value(*r))
+                        break
+
+    return records, last_line
 

ogs6py/ogs.py

@@ -1018,7 +1018,8 @@ def write_input(self, keep_includes: bool = False) -> None:
     def parse_out(self, logfile: str | None = None,
                   filter: str | None = None,
                   maximum_lines: int | None = None,
-                  reset_index: bool = True) -> pd.DataFrame:
+                  reset_index: bool = True,
+                  return_records: bool = False) -> pd.DataFrame:
         """Parses the logfile
 
         Parameters
@@ -1035,7 +1036,9 @@ def parse_out(self, logfile: str | None = None,
         """
         if logfile is None:
             logfile = self.logfile
-        records = parser.parse_file(logfile, maximum_lines=maximum_lines, force_parallel=False)
+        records, _ = parser.parse_file(logfile, maximum_lines=maximum_lines, force_parallel=False)
+        if return_records is True:
+            return records
         df = pd.DataFrame(records)
 
         df = parse_fcts.fill_ogs_context(df)

ogs6py/ogs_regexes/ogs_regexes.py

@@ -125,13 +125,28 @@ class ComponentConvergenceCriterion(MPIProcess,Info):
     x: float
     dx_x: float
 
+@dataclass
+class ComponentConvergenceCriterionResidual(MPIProcess,Info):
+    component: int
+    dr: float
+    r0: float
+    dr_r0: float
 
 @dataclass
 class TimeStepConvergenceCriterion(MPIProcess,Info):
     dx: float
     x: float
     dx_x: float
 
+@dataclass
+class ConvergenceCriterionResidual(MPIProcess,Info):
+    dr: float
+    r0: float
+    dr_r0: float
+
+@dataclass
+class Residual(MPIProcess,Info):
+    r: float
 
 @dataclass
 class CouplingIterationConvergence(MPIProcess,Info):
@@ -186,6 +201,12 @@ def ogs_regexes():
             (
                 "info: Convergence criterion, component (\d+): \|dx\|=([\d\.e+-]+), \|x\|=([\d\.e+-]+), \|dx\|/\|x\|=([\d\.e+-]+|nan|inf)$",
                 ComponentConvergenceCriterion),
+            (
+                "info: Convergence criterion, component (\d+): \|dr\|=([\d\.e+-]+), \|r0\|=([\d\.e+-]+), \|dr\|/\|r0\|=([\d\.e+-]+|nan|inf)$",
+                ComponentConvergenceCriterionResidual),
+            ("info: Convergence criterion: \|dr\|=([\d\.e+-]+) \|r0\|=([\d\.e+-]+) \|dr\|/\|r0\|=([\d\.e+-]+|nan|inf)",
+             ConvergenceCriterionResidual),
+            ("info: \[time\] Residual norm: ([\d\.e+-]+|nan|inf)", Residual),
             ("critical: (.*)", CriticalMessage),
             ("error: (.*)", ErrorMessage),
             ("warning: (.*)", WarningMessage)

setup.py

@@ -55,5 +55,5 @@ def find_version(*file_paths):
       include_package_data=True,
       python_requires='>=3.8',
       install_requires=["lxml","pandas"],
-      py_modules=["ogs6py/ogs","ogs6py/log_parser/log_parser", "ogs6py/log_parser/common_ogs_analyses", "ogs6py/ogs_regexes/ogs_regexes"],
+      py_modules=["ogs6py/ogs","ogs6py/log_parser/log_parser", "ogs6py/log_parser/common_ogs_analyses", "ogs6py/ogs_regexes/ogs_regexes", "ogs-monitor"],
       packages=["ogs6py/classes","ogs6py/log_parser","ogs6py/ogs_regexes"])

static_log_vis.py

@@ -0,0 +1,163 @@
+# pylint: disable=C0301
+import os
+import sys
+
+import numpy as np
+import matplotlib.pyplot as plt
+
+import ogs6py
+import ogs6py.log_parser.log_parser as parser
+import ogs6py.log_parser.common_ogs_analyses as parse_fcts
+
+class PLOTLOG(object):
+    def __init__(self,
+                 logfilelist,
+                 maximum_lines,
+                 convergence_metric,
+                 convergence_component,
+                 convergence_type,
+                 show_quadrant):
+        self.log_file = logfilelist
+        self.crit = convergence_metric
+        self.crit_comp = convergence_component
+        if maximum_lines is None:
+            self.maximum_lines = maximum_lines
+        else:
+            self.maximum_lines = int(maximum_lines)
+        self.running = True
+        self.show_quadrant = int(show_quadrant)
+        self.convergence_type = convergence_type
+        self.criteria_types = {"PerComponentDeltaX": ogs6py.ogs_regexes.ogs_regexes.ComponentConvergenceCriterion,
+                          "DeltaX": ogs6py.ogs_regexes.ogs_regexes.TimeStepConvergenceCriterion,
+                          "Residual": ogs6py.ogs_regexes.ogs_regexes.ConvergenceCriterionResidual,
+                          "Res": ogs6py.ogs_regexes.ogs_regexes.Residual,
+                          "PerComponentResidual": ogs6py.ogs_regexes.ogs_regexes.ComponentConvergenceCriterionResidual}
+        self.filters = ["time_step_vs_iterations","by_time_step", "convergence_newton_iteration","analysis_simulation"]
+        # plot data from records actual
+        self.last_record = {file:0 for file in self.log_file}
+        self.last_line = {file:0 for file in self.log_file}
+        self.timesteps = {file:[] for file in self.log_file}
+        self.ts_simtime = {file:[] for file in self.log_file}
+        self.iterations_per_ts = {file:[] for file in self.log_file}
+        self.assembly_solver_time = {file:[] for file in self.log_file}
+        self.criterion_x = {file:[] for file in self.log_file}
+        self.criterion_label = {file:[] for file in self.log_file}
+        self.criterion_y = {file:[] for file in self.log_file}
+        self.crit_timesteppointer = {file:[] for file in self.log_file}
+        self.time_step_solution_time = {file:[] for file in self.log_file}
+        self.walltime = {file:0 for file in self.log_file}
+        self.records = {}
+        for file in logfilelist:
+            records_ref, _ = parser.parse_file(file, maximum_lines=self.maximum_lines, force_parallel=False)
+            self.records[file] =  records_ref
+            self.parse_records(self.records[file], dataset=file)
+        self.plot()
+
+    def plot(self):
+        if (self.show_quadrant == 1):
+            for file in self.log_file:
+                plt.plot(np.array(self.timesteps[file]), np.array(self.iterations_per_ts[file]), label=file)
+            plt.xlabel("time step")
+            plt.ylabel("iterations")
+            plt.title("iterations per timestep")
+            plt.legend()
+            plt.show()
+        elif self.show_quadrant == 0:
+            for file in self.log_file:
+                plt.plot(np.array(self.timesteps[file]), np.array(self.ts_simtime[file])[:,1], label=file)
+            plt.xlabel("time step")
+            plt.ylabel("step size")
+            plt.title("step size per timestep")
+            plt.legend()
+            plt.show()
+        elif self.show_quadrant == 2:
+            for file in self.log_file:
+                plt.plot(np.array(self.timesteps[file]), np.array(self.ts_simtime[file])[:,0], label=file)
+            plt.xlabel("time step")
+            plt.ylabel("time step start time")
+            plt.title("start time vs timestep")
+            plt.legend()
+            plt.show()
+
+
+
+    def parse_records(self, records, dataset="actual"):
+        #len_records = len(records)
+        for record in records: #[self.last_record[dataset]:len_records]:
+            if isinstance(record, ogs6py.ogs_regexes.ogs_regexes.TimeStepStartTime):
+                self.timesteps[dataset].append(int(record.time_step))
+                self.ts_simtime[dataset].append((float(record.step_start_time), float(record.step_size)))
+                self.iterations_per_ts[dataset].append(1)
+                self.assembly_solver_time[dataset].append([0, 0])
+
+            elif isinstance(record, ogs6py.ogs_regexes.ogs_regexes.IterationTime):
+                self.iterations_per_ts[dataset][-1] = int(record.iteration_number)+1
+            elif isinstance(record, ogs6py.ogs_regexes.ogs_regexes.AssemblyTime):
+                self.assembly_solver_time[dataset][-1][0] = (self.assembly_solver_time[dataset][-1][0] +
+                                                    float(record.assembly_time)) / (self.iterations_per_ts[dataset][-1])
+            elif isinstance(record, ogs6py.ogs_regexes.ogs_regexes.LinearSolverTime):
+                self.assembly_solver_time[dataset][-1][1] = (self.assembly_solver_time[dataset][-1][1] +
+                                                   float(record.linear_solver_time)) / (self.iterations_per_ts[dataset][-1])
+            elif isinstance(record, self.criteria_types[self.convergence_type]):
+                if self.convergence_type in ["PerComponentResidual", "PerComponentDeltaX"]:
+                    comp = record.component
+                else:
+                    comp = int(self.crit_comp)
+                if comp == int(self.crit_comp):
+                    if int(self.iterations_per_ts[dataset][-1]) == 1:
+                        self.criterion_x[dataset].append(len(self.criterion_x[dataset])+1)
+                        self.criterion_label[dataset].append(f"{self.iterations_per_ts[dataset][-1]} \n{self.timesteps[dataset][-1]}")
+                        self.criterion_y[dataset].append(float(getattr(record, self.crit)))
+                        self.crit_timesteppointer[dataset].append(len(self.criterion_x[dataset])-1)
+                    else:
+                        self.criterion_x[dataset].append(len(self.criterion_x[dataset])+1)
+                        self.criterion_label[dataset].append(f"{self.iterations_per_ts[dataset][-1]} \n")
+                        self.criterion_y[dataset].append(float(getattr(record, self.crit)))
+            elif isinstance(record, ogs6py.ogs_regexes.ogs_regexes.TimeStepSolutionTime):
+                self.time_step_solution_time[dataset].append(float(record.time_step_solution_time))
+            elif isinstance(record, ogs6py.ogs_regexes.ogs_regexes.SimulationExecutionTime):
+                self.walltime[dataset] = float(record.execution_time)
+
+if __name__ == '__main__':
+    input_files = []
+    for entry in sys.argv:
+        if ".log" in entry:
+            input_files.append(entry)
+    window_length = 10
+    maximum_lines = None
+    convergence_metric = "dx"
+    convergence_component = 0
+    convergence_type = "PerComponentDeltaX"
+    show_quadrant = 0
+    norun = False
+    for i, arg in enumerate(sys.argv):
+        if arg == "-r":
+            ref_file = sys.argv[i+1]
+        elif arg == "-wl":
+            window_length = sys.argv[i+1]
+        elif arg == "-ml":
+            maximum_lines = sys.argv[i+1]
+        elif arg == "-cm":
+            convergence_metric = sys.argv[i+1]
+        elif arg == "-cc":
+            convergence_component = sys.argv[i+1]
+        elif arg == "-ct":
+            convergence_type = sys.argv[i+1]
+        elif arg == "-ui":
+            update_interval = sys.argv[i+1]
+        elif arg == "-q":
+            show_quadrant = sys.argv[i+1]
+        elif arg == "-h":
+            norun = True
+            print("OGS monitor help\n")
+            print("default usage: ogs-monitor.py [logfile.log ...] args")
+            print("args:")
+            print(" -ml [max lines]  : provide maximum lines to read in")
+            print(" -cm [convergence metric]  : provide convergence metric, could be dx, r, dx/x")
+            print(" -cc [convergence component]  : provide convergence component, default: 0")
+            print(" -ct [convergence type] : provide a type could be DeltaX, PerComponentDeltaX (default), PerComponentResidual, Residual")
+            print(" -q [X] : show only quadrant X, default: all four are shown")
+            print(" -h : display this help")
+    if norun is False:
+        ogs_monitor = PLOTLOG(input_files, maximum_lines, convergence_metric, convergence_component, convergence_type, show_quadrant)
+

tests/test_ogs6py.py

@@ -683,19 +683,19 @@ def test_model_run(self):
     def test_parallel_1_compare_serial_info(self):
         filename_p = 'tests/parser/parallel_1_info.txt'
         # Only for MPI execution with 1 process we need to tell the log parser by force_parallel=True!
-        records_p = parse_file(filename_p, force_parallel=True)
+        records_p, _ = parse_file(filename_p, force_parallel=True)
         num_of_record_type_p = [len(i) for i in log_types(records_p).values()]
 
         filename_s = 'tests/parser/serial_info.txt'
-        records_s = parse_file(filename_s)
+        records_s, _  = parse_file(filename_s)
         num_of_record_type_s = [len(i) for i in log_types(records_s).values()]
 
         self.assertSequenceEqual(num_of_record_type_s, num_of_record_type_p,
                                  'The number of logs for each type must be equal for parallel log and serial log')
 
     def test_parallel_3_debug(self):
         filename = 'tests/parser/parallel_3_debug.txt'
-        records = parse_file(filename)
+        records, _ = parse_file(filename)
         mpi_processes = 3
 
         self.assertEqual(len(records) % mpi_processes, 0,
@@ -723,7 +723,7 @@ def test_parallel_3_debug(self):
 
     def test_serial_convergence_newton_iteration_long(self):
         filename = 'tests/parser/serial_convergence_long.txt'
-        records = parse_file(filename)
+        records, _ = parse_file(filename)
         df = pd.DataFrame(records)
         df = fill_ogs_context(df)
         dfe = analysis_convergence_newton_iteration(df)
@@ -740,7 +740,7 @@ def test_serial_convergence_newton_iteration_long(self):
 
     def test_serial_convergence_coupling_iteration_long(self):
         filename = 'tests/parser/serial_convergence_long.txt'
-        records = parse_file(filename)
+        records, _ = parse_file(filename)
         df = pd.DataFrame(records)
         dfe = analysis_simulation_termination(df)
         status = dfe.empty  # No errors assumed
@@ -763,7 +763,7 @@ def test_serial_convergence_coupling_iteration_long(self):
 
     def test_serial_critical(self):
         filename = 'tests/parser/serial_critical.txt'
-        records = parse_file(filename)
+        records, _ = parse_file(filename)
         self.assertEqual(len(records),4)
         df = pd.DataFrame(records)
         self.assertEqual(len(df), 4)
@@ -776,7 +776,7 @@ def test_serial_critical(self):
 
     def test_serial_warning_only(self):
         filename = 'tests/parser/serial_warning_only.txt'
-        records = parse_file(filename)
+        records, _ = parse_file(filename)
         self.assertEqual(len(records),1)
         df = pd.DataFrame(records)
         self.assertEqual(len(df), 1)
@@ -789,7 +789,7 @@ def test_serial_warning_only(self):
 
     def test_serial_time_vs_iterations(self):
         filename = 'tests/parser/serial_convergence_long.txt'
-        records = parse_file(filename)
+        records, _ = parse_file(filename)
         df = pd.DataFrame(records)
         df = fill_ogs_context(df)
         dfe = time_step_vs_iterations(df)