AMDFlang compiler for MFC

.github/file-filter.yml

@@ -24,6 +24,7 @@ scripts: &scripts
 yml: &yml
   - '.github/workflows/phoenix/**'
   - '.github/workflows/frontier/**'
+  - '.github/workflows/frontier_amd/**'
   - '.github/workflows/bench.yml'
   - '.github/workflows/test.yml'
   - '.github/workflows/formatting.yml'

.github/workflows/bench.yml

@@ -70,6 +70,14 @@ jobs:
             device: gpu
             interface: omp
             build_script: "bash .github/workflows/frontier/build.sh gpu omp bench"
+          - cluster: frontier_amd
+            name: Oak Ridge | Frontier (AMD)
+            group: phoenix
+            labels: frontier
+            flag: famd
+            device: gpu
+            interface: omp
+            build_script: "bash .github/workflows/frontier_amd/build.sh gpu omp bench"
     runs-on:
       group: ${{ matrix.group }}
       labels: ${{ matrix.labels }}

.github/workflows/frontier_amd/bench.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+n_ranks=12
+device_opts=""
+if [ "$job_device" = "gpu" ]; then
+    gpus=$(rocm-smi --showid | awk '{print $1}' | grep -Eo '[0-9]+' | uniq | tr '\n' ' ')
+    n_ranks=$(echo "$gpus" | wc -w)         # number of GPUs on node
+    gpu_ids=$(echo "$gpus" | tr ' ' '\n' | tr '\n' ' ' | sed 's/ $//')  # GPU IDs from rocm-smi
+    device_opts+="--gpu"
+    if [ "$job_interface" = "acc" ]; then
+        device_opts+=" acc"
+    elif [ "$job_interface" = "omp" ]; then
+        device_opts+=" mp"
+    fi
+    device_opts+=" -g $gpu_ids"
+fi
+
+if [ "$job_device" = "gpu" ]; then
+    ./mfc.sh bench --mem 12 -j $n_ranks -o "$job_slug.yaml" -- -c frontier_amd $device_opts -n $n_ranks
+else
+    ./mfc.sh bench --mem 1 -j $(nproc) -o "$job_slug.yaml" -- -c frontier_amd $device_opts -n $n_ranks
+fi

.github/workflows/frontier_amd/build.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+job_device=$1
+job_interface=$2
+run_bench=$3
+build_opts=""
+if [ "$job_device" = "gpu" ]; then
+  build_opts+="--gpu"
+  if [ "$job_interface" = "acc" ]; then
+      build_opts+=" acc"
+  elif [ "$job_interface" = "omp" ]; then
+      build_opts+=" mp"
+  fi
+fi
+
+. ./mfc.sh load -c famd -m g
+
+if [ "$run_bench" == "bench" ]; then
+    for dir in benchmarks/*/; do
+        dirname=$(basename "$dir")
+        ./mfc.sh run "$dir/case.py" --case-optimization -j 8 --dry-run $build_opts
+    done
+else
+    ./mfc.sh test -a --dry-run -j 8 $build_opts
+fi
+

.github/workflows/frontier_amd/submit-bench.sh

@@ -0,0 +1,54 @@
+#!/bin/bash
+
+set -e
+
+usage() {
+    echo "Usage: $0 [script.sh] [cpu|gpu]"
+}
+
+if [ ! -z "$1" ]; then
+    sbatch_script_contents=`cat $1`
+else
+    usage
+    exit 1
+fi
+
+if [ "$2" = "cpu" ]; then
+    sbatch_device_opts="\
+#SBATCH -n 32                       # Number of cores required"
+elif [ "$2" = "gpu" ]; then
+    sbatch_device_opts="\
+#SBATCH -n 8                       # Number of cores required"
+else
+    usage; exit 1
+fi
+
+
+job_slug="`basename "$1" | sed 's/\.sh$//' | sed 's/[^a-zA-Z0-9]/-/g'`-$2-$3"
+
+sbatch <<EOT
+#!/bin/bash
+#SBATCH -JMFC-$job_slug            # Job name
+#SBATCH -A CFD154                  # charge account
+#SBATCH -N 1                       # Number of nodes required
+$sbatch_device_opts
+#SBATCH -t 02:59:00                # Duration of the job (Ex: 15 mins)
+#SBATCH -o$job_slug.out            # Combined output and error messages file
+#SBATCH -p extended                # Extended partition for shorter queues
+
+set -e
+set -x
+
+cd "\$SLURM_SUBMIT_DIR"
+echo "Running in $(pwd):"
+
+job_slug="$job_slug"
+job_device="$2"
+job_interface="$3"
+
+. ./mfc.sh load -c famd -m g
+
+$sbatch_script_contents
+
+EOT
+

.github/workflows/frontier_amd/submit.sh

@@ -0,0 +1,56 @@
+#!/bin/bash
+
+set -e
+
+usage() {
+    echo "Usage: $0 [script.sh] [cpu|gpu]"
+}
+
+if [ ! -z "$1" ]; then
+    sbatch_script_contents=`cat $1`
+else
+    usage
+    exit 1
+fi
+
+if [ "$2" = "cpu" ]; then
+    sbatch_device_opts="\
+#SBATCH -n 32                       # Number of cores required"
+elif [ "$2" = "gpu" ]; then
+    sbatch_device_opts="\
+#SBATCH -n 8                       # Number of cores required"
+else
+    usage
+    exit 1
+fi
+
+
+job_slug="`basename "$1" | sed 's/\.sh$//' | sed 's/[^a-zA-Z0-9]/-/g'`-$2-$3"
+
+sbatch <<EOT
+#!/bin/bash
+#SBATCH -J MFC-$job_slug            # Job name
+#SBATCH -A CFD154                  # charge account
+#SBATCH -N 1                       # Number of nodes required
+$sbatch_device_opts
+#SBATCH -t 05:59:00                # Duration of the job (Ex: 15 mins)
+#SBATCH -o$job_slug.out            # Combined output and error messages file
+#SBATCH -p extended                # Extended partition for shorter queues
+#SBATCH -W                         # Do not exit until the submitted job terminates.
+
+set -e
+set -x
+
+cd "\$SLURM_SUBMIT_DIR"
+echo "Running in $(pwd):"
+
+job_slug="$job_slug"
+job_device="$2"
+job_interface="$3"
+
+. ./mfc.sh load -c famd -m $([ "$2" = "gpu" ] && echo "g" || echo "c")
+
+$sbatch_script_contents
+
+EOT
+

.github/workflows/frontier_amd/test.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+gpus=`rocm-smi --showid | awk '{print $1}' | grep -Eo '[0-9]+' | uniq | tr '\n' ' '`
+ngpus=`echo "$gpus" | tr -d '[:space:]' | wc -c`
+
+device_opts=""
+if [ "$job_device" = "gpu" ]; then
+    device_opts+="--gpu"
+    if [ "$job_interface" = "acc" ]; then
+        device_opts+=" acc"
+    elif [ "$job_interface" = "omp" ]; then
+        device_opts+=" mp"
+    fi
+fi
+
+if [ "$job_device" = "gpu" ]; then
+    ./mfc.sh test -a  --max-attempts 3 -j $ngpus $device_opts -- -c frontier_amd
+else
+    ./mfc.sh test -a --max-attempts 3 -j 32 --no-gpu -- -c frontier_amd
+fi

.github/workflows/test.yml

@@ -103,30 +103,46 @@ jobs:
         include:
           # Phoenix (GT)
           - lbl: 'gt'
+            lbl_comp: 'gt'
             cluster_name: 'Georgia Tech | Phoenix'
             device: 'gpu'
             interface: 'acc'
           - lbl: 'gt'
+            lbl_comp: 'gt'
             cluster_name: 'Georgia Tech | Phoenix'
             device: 'gpu'
             interface: 'omp'
           - lbl: 'gt'
+            lbl_comp: 'gt'
             cluster_name: 'Georgia Tech | Phoenix'
             device: 'cpu'
             interface: 'none'
           # Frontier (ORNL)
           - lbl: 'frontier'
+            lbl_comp: 'frontier'
             cluster_name: 'Oak Ridge | Frontier'
             device: 'gpu'
             interface: 'acc'
           - lbl: 'frontier'
+            lbl_comp: 'frontier'
             cluster_name: 'Oak Ridge | Frontier'
             device: 'gpu'
             interface: 'omp'
           - lbl: 'frontier'
+            lbl_comp: 'frontier'
             cluster_name: 'Oak Ridge | Frontier'
             device: 'cpu'
             interface: 'none'
+          - lbl: 'frontier'
+            lbl_comp: 'frontier_amd'
+            cluster_name: 'Oak Ridge | Frontier (AMD)'
+            device: 'gpu'
+            interface: 'omp'
+          - lbl: 'frontier'
+            lbl_comp: 'frontier_amd'
+            cluster_name: 'Oak Ridge | Frontier (AMD)'
+            device: 'cpu'
+            interface: 'none'
     runs-on:
       group:  phoenix
       labels: ${{ matrix.lbl }}
@@ -139,24 +155,39 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Build & Test
-        if:   matrix.lbl == 'gt'
+        if:   matrix.lbl_comp == 'gt'
         run:  bash .github/workflows/phoenix/submit.sh .github/workflows/phoenix/test.sh ${{ matrix.device }} ${{ matrix.interface }}
 
       - name: Build
-        if:   matrix.lbl == 'frontier'
+        if:   matrix.lbl_comp == 'frontier'
         run:  bash .github/workflows/frontier/build.sh ${{ matrix.device }} ${{ matrix.interface }}
 
       - name: Test
-        if:   matrix.lbl == 'frontier'
+        if:   matrix.lbl_comp == 'frontier'
         run:  bash .github/workflows/frontier/submit.sh .github/workflows/frontier/test.sh ${{matrix.device}} ${{ matrix.interface }}
 
+      - name: Build
+        if:   matrix.lbl_comp == 'frontier_amd'
+        run:  bash .github/workflows/frontier_amd/build.sh ${{ matrix.device }} ${{ matrix.interface }}
+
+      - name: Test
+        if:   matrix.lbl_comp == 'frontier_amd'
+        run:  bash .github/workflows/frontier_amd/submit.sh .github/workflows/frontier_amd/test.sh ${{matrix.device}} ${{ matrix.interface }}  
+
       - name: Print Logs
         if:   always()
         run:  cat test-${{ matrix.device }}-${{ matrix.interface }}.out
 
       - name: Archive Logs
         uses: actions/upload-artifact@v4
-        if:   matrix.lbl == 'frontier'
+        if:   matrix.lbl_comp == 'frontier'
+        with:
+          name: logs-${{ strategy.job-index }}-${{ matrix.device }}-${{ matrix.interface }}
+          path: test-${{ matrix.device }}-${{ matrix.interface }}.out
+
+      - name: Archive Logs
+        uses: actions/upload-artifact@v4
+        if:   matrix.lbl_comp == 'frontier_amd'
         with:
           name: logs-${{ strategy.job-index }}-${{ matrix.device }}-${{ matrix.interface }}
           path: test-${{ matrix.device }}-${{ matrix.interface }}.out

CMakeLists.txt

@@ -311,6 +311,7 @@ endif()
 # files that will eventually be used to compile <target>.
 
 macro(HANDLE_SOURCES target useCommon)
+
     set(${target}_DIR "${CMAKE_SOURCE_DIR}/src/${target}")
     set(common_DIR    "${CMAKE_SOURCE_DIR}/src/common")
 
@@ -371,7 +372,7 @@ macro(HANDLE_SOURCES target useCommon)
                                  -D MFC_${CMAKE_Fortran_COMPILER_ID}
                                  -D MFC_${${target}_UPPER}
                                  -D MFC_COMPILER="${CMAKE_Fortran_COMPILER_ID}"
-                                 -D MFC_CASE_OPTIMIZATION=False
+				                 -D MFC_CASE_OPTIMIZATION=False
                                  -D chemistry=False
                                  --line-numbering
                                  --no-folding
@@ -393,7 +394,6 @@ HANDLE_SOURCES(simulation ON)
 HANDLE_SOURCES(post_process ON)
 HANDLE_SOURCES(syscheck OFF)
 
-
 # MFC_SETUP_TARGET: Given a target (herein <target>), this macro creates a new
 # executable <target> with the appropriate sources, compiler definitions, and
 # linked libraries (assuming HANDLE_SOURCES was called on <target>).
@@ -430,6 +430,7 @@ function(MFC_SETUP_TARGET)
 
     foreach (a_target ${IPO_TARGETS})
         set_target_properties(${a_target} PROPERTIES Fortran_PREPROCESS ON)
+	    message(STATUS ${CMAKE_Fortran_COMPILER_ID})
 
         target_include_directories(${a_target} PRIVATE
             "${CMAKE_SOURCE_DIR}/src/common"
@@ -449,9 +450,14 @@ function(MFC_SETUP_TARGET)
 
         if (MFC_MPI AND ARGS_MPI)
             find_package(MPI COMPONENTS Fortran REQUIRED)
-
+	    
             target_compile_definitions(${a_target} PRIVATE MFC_MPI)
-            target_link_libraries     (${a_target} PRIVATE MPI::MPI_Fortran)
+            if(CMAKE_Fortran_COMPILER_ID STREQUAL "LLVMFlang")
+                target_compile_options(${a_target} PRIVATE "$ENV{CRAY_MPICH_INC}")
+                target_link_libraries(${a_target} PRIVATE $ENV{CRAY_MPICH_LIB})
+            else() 
+                target_link_libraries(${a_target} PRIVATE MPI::MPI_Fortran)
+            endif()
         endif()
 
         if (ARGS_SILO)
@@ -469,9 +475,11 @@ function(MFC_SETUP_TARGET)
                 if (CMAKE_Fortran_COMPILER_ID STREQUAL "NVHPC" OR CMAKE_Fortran_COMPILER_ID STREQUAL "PGI")
                     find_package(CUDAToolkit REQUIRED)
                     target_link_libraries(${a_target} PRIVATE CUDA::cudart CUDA::cufft)
-                else()
+                elseif(CMAKE_Fortran_COMPILER_ID STREQUAL "LLVMFlang")
+                    target_link_libraries(${a_target} PRIVATE $ENV{CRAY_HIPFORT_LIB})
+                else() 
                     find_package(hipfort COMPONENTS hipfft CONFIG REQUIRED)
-                    target_link_libraries(${a_target} PRIVATE hipfort::hipfft)
+                    target_link_libraries(${a_target} PRIVATE hipfort::hipfft)                    
                 endif()
             else()
                 find_package(FFTW REQUIRED)
@@ -584,7 +592,7 @@ function(MFC_SETUP_TARGET)
                         PRIVATE -DFRONTIER_UNIFIED)
                 endif()
 
-    		find_package(hipfort COMPONENTS hip CONFIG REQUIRED)
+                find_package(hipfort COMPONENTS hip CONFIG REQUIRED)
                 target_link_libraries(${a_target} PRIVATE hipfort::hip hipfort::hipfort-amdgcn flang_rt.hostdevice)
             endif()
         elseif (CMAKE_Fortran_COMPILER_ID STREQUAL "Cray")