Merge branch 'main' into 183_default_precision

Author: jedbakerMO

README.md

@@ -66,14 +66,10 @@ make test
 The HTML Doxygen documentation is built by default when building the project and 
 the generated files can be found in `<build>/html`.
 
-To build only the Doxygen docs use `make doxygen-docs`. On installation the
-HTML pages are copied to `<installation_root>/share/docs`.
+To build only the Doxygen docs, configure cmake as above and use `make doxygendocs`.   
+On installation the HTML pages are copied to `<installation_root>/doxygen/html`.   
+These can be viewed with `<browser> index.html`.   
 
-The user guide can be built using `make sphinxdocs`.
-
-#### Unit Test Coverage
-
-The unit test coverage is generated using `gcovr` which wraps `gcov` with additional
-functionality, such as multiple output format options.
-
-A detailed coverage report can be found [here](https://metoffice.github.io/profiler/coverage/).
+The user guide can be built using `make sphinxdocs`.   
+On installation the docs are copied to `<installation_root>/sphinx`.   
+These can be viewed with `<browser> index.html`.   

documentation/Sphinx/user_guide/output/post-processing.rst

@@ -53,13 +53,15 @@ script, these are given in the table below.
     * - ``-d/ --decimals``
       - 3
       - Number of decimal places to report calculated values to
-
-    * - ``-b/ --basic_output``
+    * - ``-m/ --max_only``
+      - False
+      - If used will only calculate maximum values across MPI ranks
+    * - ``-f/ --full_info``
       - False
-      - If used will only output mean values across MPI ranks
+      - Enables full Vernier output to be merged and output
 
+Below is an example of the current script output without additional flags.
 
-Below is an example of the current script output.
 
 .. code-block:: text
 

post-processing/post-process.py

@@ -30,12 +30,12 @@ def parse_cli_arguments(input_arguments: list[str] = None,
 
     parser = argparse.ArgumentParser(description="This script is for merging the outputs from a test that uses Vernier callipers into one file. For full documentation please see the post-processing section of the user guide.")
 
-    parser.add_argument("-p", "--path",         type=Path,  default=(os.getcwd()),                 help="Path to Vernier output files")
-    parser.add_argument("-o", "--output_name",   type=str,   default=str("vernier-merged-output"), help="Name of file to write to")
-    parser.add_argument("-i", "--input_name",    type=str,   default=str("vernier-output-"),       help="Vernier files to read from")
-    parser.add_argument("-d", "--decimals",     type=int,   default=3,                             help="Number of decimal places calculated results will be reported to")
-    parser.add_argument("-b", "--basic_output", action="store_true", default=False,                help="Outputs only mean values across MPI ranks")
-
+    parser.add_argument("-p", "--path",         type=Path,  default=(os.getcwd()),                help="Path to Vernier output files")
+    parser.add_argument("-o", "--output_name",  type=str,   default=str("vernier-merged-output"), help="Name of file to write to")
+    parser.add_argument("-i", "--input_name",   type=str,   default=str("vernier-output-"),       help="Vernier files to read from")
+    parser.add_argument("-m", "--max_only",     action="store_true", default=False,               help="Only calculates the maximum value across all ranks")
+    parser.add_argument("-f", "--full_info",    action="store_true", default=False,               help="Enables merging and displaying of all information Vernier records")
+    parser.add_argument("-d", "--decimals",     type=int,   default=3,                            help="Number of decimal places calculated results will be reported to")
 
     return parser.parse_args(args=input_arguments)
 
@@ -61,17 +61,19 @@ def read_mpi_ranks(directory_path: Path,
 def read_and_pre_process(file_path: Path,
                          rank: int, 
                          input_name: str,
+                         full_info_bool: bool,
                      ) -> pd.DataFrame:
     """ Reads a vernier-output and processes it 
 
     Reads in the current vernier-output file for a given rank before removing
     whitespace and formatting into a pandas dataframe.
 
     Args:
-        file_path:  The path where the vernier outputs are located.
-        rank:       The current output file to open, as different output files 
-                    are ordered according to MPI rank.
-        input_name: The name of the vernier output files without the rank.
+        file_path:      The path where the vernier outputs are located.
+        rank:           The current output file to open, as different output files 
+                        are ordered according to MPI rank.
+        input_name:     The name of the vernier output files without the rank.
+        full_info_bool: A boolean which if set to True will give merge all vernier recordings for final output.
 
     Returns:
         A Pandas dataframe containing the processed vernier output data.
@@ -94,14 +96,25 @@ def read_and_pre_process(file_path: Path,
     """ Organises the new dataframe """
     dataframe = dataframe.sort_values(by="Routine")
     dataframe = dataframe.reset_index(drop=True)
-    temp_dataframe = dataframe[["Total", "Self", "Routine"]]
+    dataframe = dataframe.drop(columns=["index"])
+
+    """ If the user wants the full information then it will be returned, 
+    otherwise the pruned information will be pruned """
+    if full_info_bool:
+
+        return dataframe
+
+    else:
 
-    return temp_dataframe
+        temp_dataframe = dataframe[["Total", "Self", "Routine"]]
+
+        return temp_dataframe
 
 def merge_and_analyse(file_path: Path,
                       mpiranks: int,
                       input_name: str,
-                      basic_output_bool: bool
+                      max_only_bool: bool,
+                      full_info_bool: bool,
                   ) -> pd.DataFrame:
     """ Reads in the files and merges them 
 
@@ -110,10 +123,14 @@ def merge_and_analyse(file_path: Path,
     before averaging them.
 
     Args:
-        file_path:         The path where the vernier outputs are located.
-        mpiranks:          The number of mpi ranks (equivalent to the number of files) to iterate through.
-        input_name:        The name of the vernier output files without the rank.
-        basic_output_bool: A boolean which if set to True will not calculate minimum/ maximum values. 
+        file_path:      The path where the vernier outputs are located.
+        mpiranks:       The number of mpi ranks (equivalent to the number of files) to iterate through.
+        input_name:     The name of the vernier output files without the rank.
+        max_only_bool:  A boolean which if set to True will not calculate minimum/ maximum values. 
+        file_path:      The path where the vernier outputs are located.
+        mpiranks:       The number of mpi ranks (equivalent to the number of files) to iterate through.
+        input_name:     The name of the vernier output files without the rank.
+        full_info_bool: A boolean which if set to True will give merge all vernier recordings for final output.
 
     Returns:
         The merged dataframe, containing the routine names and the mean 'Self' and 'Total' values across all outputs.
@@ -125,14 +142,15 @@ def merge_and_analyse(file_path: Path,
     for rank in range(0,mpiranks):
 
         """ Open the file, read it, workout where it actually starts """
-        dataframe = read_and_pre_process(file_path, rank, input_name)
+        dataframe = read_and_pre_process(file_path, rank, input_name, full_info_bool)
 
         if rank == 0:     
 
+            """ Creates the initial dataframe for future calculations """
             prev_df = dataframe.copy()
-            if not(basic_output_bool):
+            if not(max_only_bool):
                 min_df  = dataframe.copy()
-                max_df  = dataframe.copy()
+            max_df  = dataframe.copy()
 
         else:
 
@@ -141,10 +159,10 @@ def merge_and_analyse(file_path: Path,
             new_df["Routine"] = prev_df["Routine"]
 
             """ Calculates new min/ max values """
-            if not(basic_output_bool):
-                for column in dataframe.columns:
+            for column in dataframe.columns:
+                if not(max_only_bool):
                     min_df[column] = min_df[column].where(min_df[column] < dataframe[column], dataframe[column])
-                    max_df[column] = max_df[column].where(max_df[column] > dataframe[column], dataframe[column])
+                max_df[column] = max_df[column].where(max_df[column] > dataframe[column], dataframe[column])
 
             prev_df = new_df.copy()
 
@@ -154,14 +172,19 @@ def merge_and_analyse(file_path: Path,
 
     """ Adds the min/ max values to the mean dataframe and renames columns """
 
-    for column in mean_df.drop(columns=["Routine"]):
-        mean_df[f"Mean_{column}"] = mean_df[column]
-        if not(basic_output_bool):
-            mean_df[f"Min_{column}"]  = min_df[column]
-            mean_df[f"Max_{column}"]  = max_df[column]
-        mean_df = mean_df.drop(columns=[f"{column}"])  
-
-    return mean_df
+    if full_info_bool:
+        output_df = mean_df.drop(columns=["Calls"])
+    else:
+        output_df = mean_df.copy()
+    for column in output_df.drop(columns=["Routine"]):
+        if not(max_only_bool):
+            output_df[f"Min_{column}"]  = min_df[column]
+            output_df[f"Mean_{column}"] = mean_df[column]
+        output_df[f"Max_{column}"]  = max_df[column]
+        output_df = output_df.drop(columns=[f"{column}"])  
+    if full_info_bool:
+        output_df["Calls"] = max_df["Calls"]
+    return output_df
 
 def main():
 
@@ -170,8 +193,10 @@ def main():
     file_path = args.path
     merged_file_name = args.output_name
     input_name = args.input_name
-    basic_output_bool = args.basic_output
+    max_only_bool = args.max_only
+    full_info_bool = args.full_info
     decimals = args.decimals
+
     mpiranks = read_mpi_ranks(file_path, input_name)
 
     if decimals >= 4:
@@ -188,7 +213,7 @@ def main():
         print("\nReading and Merging...")
 
 
-        merged_frame = merge_and_analyse(file_path, int(mpiranks), input_name, basic_output_bool)
+        merged_frame = merge_and_analyse(file_path, int(mpiranks), input_name, max_only_bool, full_info_bool)
 
         thread_string = "@0" 
         merged_frame["Routine"] = merged_frame["Routine"].str.replace(thread_string, '')