Add runtime call tracing

CMakeLists.txt

@@ -23,6 +23,7 @@ option(MFC_OpenMP        "Build with OpenMP"                                 OFF
 option(MFC_GCov          "Build with GCov"                                   OFF)
 option(MFC_Unified       "Build with unified CPU & GPU memory (GH-200 only)" OFF)
 option(MFC_Fastmath      "Build with -gpu=fastmath on NV GPUs"               OFF)
+option(MFC_TRACE_CALLS   "Build with generated runtime call tracing"          OFF)
 option(MFC_PRE_PROCESS   "Build pre_process"                                 OFF)
 option(MFC_SIMULATION    "Build simulation"                                  OFF)
 option(MFC_POST_PROCESS  "Build post_process"                                OFF)
@@ -108,6 +109,17 @@ endif()
 
 find_program(FYPP_EXE fypp REQUIRED)
 
+set(MFC_TRACE_SUPPORTED_COMPILERS GNU Intel IntelLLVM NVHPC PGI)
+
+if (MFC_TRACE_CALLS)
+    if (NOT CMAKE_Fortran_COMPILER_ID IN_LIST MFC_TRACE_SUPPORTED_COMPILERS)
+        message(FATAL_ERROR "ERROR: Runtime call tracing is supported with GNU, Intel/IntelLLVM, and NVHPC/PGI Fortran compilers. CMake detected ${CMAKE_Fortran_COMPILER_ID}.")
+    endif()
+
+    find_package(Python3 REQUIRED COMPONENTS Interpreter)
+    find_package(Threads REQUIRED)
+endif()
+
 
 # Miscellaneous Configuration:
 # *  Explicitly link to -ldl (or system equivalent)
@@ -133,6 +145,16 @@ if (CMAKE_Fortran_COMPILER_ID STREQUAL "GNU")
         $<$<COMPILE_LANGUAGE:Fortran>:-ffree-line-length-none>
     )
 
+    if (MFC_TRACE_CALLS)
+        add_compile_options(
+            $<$<COMPILE_LANGUAGE:Fortran>:-finstrument-functions>
+            $<$<COMPILE_LANGUAGE:Fortran>:-g>
+            $<$<COMPILE_LANGUAGE:Fortran>:-fno-inline>
+            $<$<COMPILE_LANGUAGE:Fortran>:-fno-optimize-sibling-calls>
+        )
+        add_link_options("-rdynamic")
+    endif()
+
     if (MFC_GCov)
 
         # Warning present in gcc versions >= 12 that is treated as an error
@@ -241,14 +263,23 @@ elseif (CMAKE_Fortran_COMPILER_ID STREQUAL "Flang")
     if (CMAKE_BUILD_TYPE STREQUAL "Debug" OR CMAKE_BUILD_TYPE STREQUAL "RelDebug")
         add_compile_options($<$<COMPILE_LANGUAGE:Fortran>:-O1> $<$<COMPILE_LANGUAGE:Fortran>:-g>)
     endif()
-elseif (CMAKE_Fortran_COMPILER_ID STREQUAL "Intel")
+elseif ((CMAKE_Fortran_COMPILER_ID STREQUAL "Intel") OR (CMAKE_Fortran_COMPILER_ID STREQUAL "IntelLLVM"))
     add_compile_options($<$<COMPILE_LANGUAGE:Fortran>:-free>)
 
     if (CMAKE_BUILD_TYPE STREQUAL "Debug")
         add_compile_options(-g -Og -traceback -debug -check all)
     elseif (CMAKE_BUILD_TYPE STREQUAL "RelDebug")
         add_compile_options(-g -Og -traceback -check bounds)
     endif()
+
+    if (MFC_TRACE_CALLS)
+        add_compile_options(
+            $<$<COMPILE_LANGUAGE:Fortran>:-finstrument-functions>
+            $<$<COMPILE_LANGUAGE:Fortran>:-g>
+            $<$<COMPILE_LANGUAGE:Fortran>:-O0>
+        )
+        add_link_options("-Wl,-export-dynamic")
+    endif()
 elseif ((CMAKE_Fortran_COMPILER_ID STREQUAL "NVHPC") OR (CMAKE_Fortran_COMPILER_ID STREQUAL "PGI"))
     add_compile_options(
         $<$<COMPILE_LANGUAGE:Fortran>:-Mfreeform>
@@ -275,12 +306,28 @@ elseif ((CMAKE_Fortran_COMPILER_ID STREQUAL "NVHPC") OR (CMAKE_Fortran_COMPILER_
         )
     endif()
 
+    if (MFC_TRACE_CALLS)
+        add_compile_options(
+            $<$<COMPILE_LANGUAGE:Fortran>:-Minstrument=functions>
+            $<$<COMPILE_LANGUAGE:Fortran>:-g>
+        )
+        add_link_options("-Wl,-export-dynamic")
+    endif()
+
     if (DEFINED ENV{MFC_CUDA_CC})
         string(REGEX MATCHALL "[0-9]+" MFC_CUDA_CC $ENV{MFC_CUDA_CC})
         message(STATUS "Found $MFC_CUDA_CC specified. GPU code will be generated for compute capability(ies) ${MFC_CUDA_CC}.")
     endif()
 endif()
 
+if (MFC_TRACE_CALLS)
+    if (CMAKE_Fortran_COMPILER_ID STREQUAL "GNU")
+        set_source_files_properties("${CMAKE_SOURCE_DIR}/src/common/m_trace.f90" PROPERTIES COMPILE_OPTIONS "-fno-instrument-functions")
+    elseif ((CMAKE_Fortran_COMPILER_ID STREQUAL "NVHPC") OR (CMAKE_Fortran_COMPILER_ID STREQUAL "PGI"))
+        set_source_files_properties("${CMAKE_SOURCE_DIR}/src/common/m_trace.f90" PROPERTIES COMPILE_OPTIONS "-Minstrument-exclude-func-list=s_trace_point_begin;-Minstrument-exclude-func-list=s_trace_point_end")
+    endif()
+endif()
+
 if (CMAKE_BUILD_TYPE STREQUAL "Release")
     # Processor tuning: Check if we can target the host's native CPU's ISA.
     # Skip for gcov builds — -march=native on newer CPUs (e.g. Granite Rapids)
@@ -306,7 +353,9 @@ if (CMAKE_BUILD_TYPE STREQUAL "Release")
 
     # Enable LTO/IPO if supported (skip for gcov — LTO interferes with coverage
     # instrumentation and can trigger assembler errors on newer architectures).
-    if (MFC_GCov)
+    if (MFC_TRACE_CALLS)
+        message(STATUS "LTO/IPO disabled for trace build")
+    elseif (MFC_GCov)
         message(STATUS "LTO/IPO disabled for gcov build")
     elseif (CMAKE_Fortran_COMPILER_ID STREQUAL "NVHPC")
         if (MFC_Unified)
@@ -396,6 +445,7 @@ macro(HANDLE_SOURCES target useCommon)
     set(${target}_SRCs ${${target}_F90s})
     if (${useCommon})
         file(GLOB common_F90s CONFIGURE_DEPENDS "${common_DIR}/*.f90")
+        list(FILTER common_F90s EXCLUDE REGEX ".*/m_trace\.f90$")
         list(APPEND ${target}_SRCs ${common_F90s})
     endif()
 
@@ -458,6 +508,48 @@ macro(HANDLE_SOURCES target useCommon)
 
         list(APPEND ${target}_SRCs ${f90})
     endforeach()
+
+    if (MFC_TRACE_CALLS)
+        list(FIND ${target}_SRCs "${common_DIR}/m_trace.f90" _trace_module_index)
+        if (_trace_module_index EQUAL -1)
+            list(APPEND ${target}_SRCs "${common_DIR}/m_trace.f90")
+        endif()
+        list(APPEND ${target}_SRCs "${common_DIR}/m_trace_runtime.c")
+
+        set(_trace_SRCs "")
+        set(_trace_instrument_options "")
+        if (MFC_OpenACC OR MFC_OpenMP)
+            list(APPEND _trace_instrument_options "--no-point-traces")
+        endif()
+        file(MAKE_DIRECTORY "${CMAKE_BINARY_DIR}/trace/${target}")
+
+        foreach(src ${${target}_SRCs})
+            cmake_path(GET src FILENAME src_filename)
+
+            if(src_filename STREQUAL "m_trace.f90" OR src_filename STREQUAL "m_trace_runtime.c" OR src_filename STREQUAL "m_thermochem.f90")
+                list(APPEND _trace_SRCs "${src}")
+            else()
+                set(trace_src "${CMAKE_BINARY_DIR}/trace/${target}/${src_filename}")
+
+                add_custom_command(
+                    OUTPUT  "${trace_src}"
+                    COMMAND "${Python3_EXECUTABLE}"
+                            "${CMAKE_SOURCE_DIR}/toolchain/instrument_fortran_trace.py"
+                            ${_trace_instrument_options}
+                            "${src}"
+                            "${trace_src}"
+                    DEPENDS "${src}"
+                            "${CMAKE_SOURCE_DIR}/toolchain/instrument_fortran_trace.py"
+                    COMMENT "Instrumenting (trace) ${src_filename}"
+                    VERBATIM
+                )
+
+                list(APPEND _trace_SRCs "${trace_src}")
+            endif()
+        endforeach()
+
+        set(${target}_SRCs ${_trace_SRCs})
+    endif()
 endmacro()
 
 
@@ -559,6 +651,10 @@ exit 0
                                 MFC_${${ARGS_TARGET}_UPPER}
         )
 
+        if (MFC_TRACE_CALLS)
+            target_link_libraries(${a_target} PRIVATE Threads::Threads)
+        endif()
+
         if (MFC_MPI AND ARGS_MPI)
             find_package(MPI COMPONENTS Fortran REQUIRED)
 
@@ -1039,4 +1135,4 @@ site_name(SITE_NAME)
 
 configure_file(
     "${CMAKE_CURRENT_SOURCE_DIR}/toolchain/cmake/configuration.cmake.in"
-    "${CMAKE_CURRENT_BINARY_DIR}/configuration.txt")
+    "${CMAKE_CURRENT_BINARY_DIR}/configuration.txt")

docs/module_categories.json

@@ -71,7 +71,8 @@
             "m_checker",
             "m_checker_common",
             "m_sim_helpers",
-            "m_derived_variables"
+            "m_derived_variables",
+            "m_trace"
         ]
     },
     {

src/common/m_trace.f90

@@ -0,0 +1,122 @@
+!>
+!! @file
+!! @brief Contains module m_trace
+
+!> @brief Simple runtime call tracing helpers.
+module m_trace
+
+    implicit none
+
+    private
+    public :: s_trace_point_begin, s_trace_point_end
+
+    logical, private :: trace_initialized = .false.
+    logical, private :: trace_enabled = .false.
+    logical, private :: trace_point_enabled = .false.
+    logical, private :: trace_point_middle = .false.
+    integer, private :: trace_j = 0
+    integer, private :: trace_k = 0
+    integer, private :: trace_l = 0
+    integer, private :: trace_point_depth = 0
+
+    interface
+        subroutine c_trace_point_begin() bind(C, name="mfc_trace_point_begin")
+
+        end subroutine c_trace_point_begin
+
+        subroutine c_trace_point_end() bind(C, name="mfc_trace_point_end")
+
+        end subroutine c_trace_point_end
+    end interface
+
+contains
+
+    !> Initialize trace settings from environment variables.
+    subroutine s_initialize_trace
+
+        character(len=64) :: env_value
+        integer           :: env_len
+        integer           :: first_comma
+        integer           :: second_comma
+        integer           :: read_status
+
+        if (trace_initialized) return
+
+        call get_environment_variable('MFC_TRACE', env_value, length=env_len)
+        trace_enabled = env_len > 0 .and. trim(env_value) /= '0'
+
+        call get_environment_variable('MFC_TRACE_POINT', env_value, length=env_len)
+        if (env_len > 0) then
+            if (trim(env_value(:env_len)) == 'middle') then
+                trace_point_enabled = .true.
+                trace_point_middle = .true.
+            else
+                first_comma = index(env_value, ',')
+                second_comma = 0
+                if (first_comma > 0) second_comma = index(env_value(first_comma + 1:), ',')
+                if (second_comma > 0) second_comma = second_comma + first_comma
+
+                if (first_comma > 0 .and. second_comma > first_comma) then
+                    read (env_value(:first_comma - 1), *, iostat=read_status) trace_j
+                    trace_point_enabled = read_status == 0
+                    read (env_value(first_comma + 1:second_comma - 1), *, iostat=read_status) trace_k
+                    trace_point_enabled = trace_point_enabled .and. read_status == 0
+                    read (env_value(second_comma + 1:env_len), *, iostat=read_status) trace_l
+                    trace_point_enabled = trace_point_enabled .and. read_status == 0
+                end if
+            end if
+        end if
+
+        trace_initialized = .true.
+
+    end subroutine s_initialize_trace
+
+    !> Enable nested routine-entry tracing while a call at MFC_TRACE_POINT executes.
+    subroutine s_trace_point_begin(j, k, l, mid_j, mid_k, mid_l)
+
+        integer, intent(in) :: j
+        integer, intent(in) :: k
+        integer, intent(in) :: l
+        integer, intent(in) :: mid_j
+        integer, intent(in) :: mid_k
+        integer, intent(in) :: mid_l
+        integer             :: target_j
+        integer             :: target_k
+        integer             :: target_l
+
+        call s_initialize_trace()
+
+        if (.not. trace_enabled) return
+        if (.not. trace_point_enabled) return
+        if (trace_point_middle) then
+            target_j = mid_j
+            target_k = mid_k
+            target_l = mid_l
+        else
+            target_j = trace_j
+            target_k = trace_k
+            target_l = trace_l
+        end if
+
+        if (j /= target_j .or. k /= target_k .or. l /= target_l) return
+
+        trace_point_depth = trace_point_depth + 1
+        call c_trace_point_begin()
+
+    end subroutine s_trace_point_begin
+
+    !> Disable nested point tracing after a grid-point call returns.
+    subroutine s_trace_point_end
+
+        call s_initialize_trace()
+
+        if (.not. trace_enabled) return
+        if (.not. trace_point_enabled) return
+        if (trace_point_depth > 0) then
+            call c_trace_point_end()
+            trace_point_depth = trace_point_depth - 1
+        end if
+
+    end subroutine s_trace_point_end
+
+end module m_trace