Skip to content

fix gpu templates for real though #1077

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
sbryngelson merged 2 commits into from
Dec 5, 2025
Merged

fix gpu templates for real though #1077

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
831a8ca
fix gpu templates for real though
sbryngelson Dec 5, 2025
5d6f9ae
fix ups on formatting and some defaults
sbryngelson Dec 5, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 11 additions & 2 deletions toolchain/mfc/run/run.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@

from ..build import get_targets, build, REQUIRED_TARGETS, SIMULATION
from ..printer import cons
from ..state import ARG, ARGS, CFG
from ..state import ARG, ARGS, CFG, gpuConfigOptions
from ..common import MFCException, isspace, file_read, does_command_exist
from ..common import MFC_TEMPLATE_DIR, file_write, system, MFC_ROOT_DIR
from ..common import format_list_to_string, file_dump_yaml
Expand Down Expand Up @@ -99,6 +99,12 @@ def __generate_job_script(targets, case: input.MFCInputFile):
'HIP_VISIBLE_DEVICES': gpu_ids
})

# Compute GPU mode booleans for templates
gpu_mode = ARG('gpu')
gpu_enabled = (gpu_mode != gpuConfigOptions.NONE.value)
gpu_acc = (gpu_mode == gpuConfigOptions.ACC.value)
gpu_mp = (gpu_mode == gpuConfigOptions.MP.value)

content = __get_template().render(
**{**ARGS(), 'targets': targets},
ARG=ARG,
Expand All @@ -107,7 +113,10 @@ def __generate_job_script(targets, case: input.MFCInputFile):
MFC_ROOT_DIR=MFC_ROOT_DIR,
SIMULATION=SIMULATION,
qsystem=queues.get_system(),
profiler=shlex.join(__profiler_prepend())
profiler=shlex.join(__profiler_prepend()),
gpu_enabled=gpu_enabled,
gpu_acc=gpu_acc,
gpu_mp=gpu_mp
)

file_write(__job_script_filepath(), content)
Expand Down
4 changes: 2 additions & 2 deletions toolchain/templates/bridges2.mako
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
% if account:
#SBATCH --account="${account}"
% endif
% if gpu:
% if gpu_enabled:
#SBATCH --gpu-bind=verbose,closest
#SBATCH --gres=gpu:v100-16:${tasks_per_node}
% endif
Expand All @@ -31,7 +31,7 @@ ${helpers.template_prologue()}

ok ":) Loading modules:\n"
cd "${MFC_ROOT_DIR}"
. ./mfc.sh load -c b -m ${'g' if gpu else 'c'}
. ./mfc.sh load -c b -m ${'g' if gpu_enabled else 'c'}
cd - > /dev/null
echo

Expand Down
2 changes: 1 addition & 1 deletion toolchain/templates/carpenter-cray.mako
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ ${helpers.template_prologue()}

ok ":) Loading modules:\n"
cd "${MFC_ROOT_DIR}"
. ./mfc.sh load -c cc -m ${'g' if gpu else 'c'}
. ./mfc.sh load -c cc -m ${'g' if gpu_enabled else 'c'}
cd - > /dev/null
echo

Expand Down
2 changes: 1 addition & 1 deletion toolchain/templates/carpenter.mako
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ ${helpers.template_prologue()}

ok ":) Loading modules:\n"
cd "${MFC_ROOT_DIR}"
. ./mfc.sh load -c c -m ${'g' if gpu else 'c'}
. ./mfc.sh load -c c -m ${'g' if gpu_enabled else 'c'}
cd - > /dev/null
echo

Expand Down
2 changes: 1 addition & 1 deletion toolchain/templates/default.mako
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ if engine == 'batch':
(set -x; ${profiler} \
jsrun --nrs ${tasks_per_node*nodes} \
--cpu_per_rs 1 \
--gpu_per_rs ${1 if gpu else 0} \
--gpu_per_rs ${1 if gpu_enabled else 0} \
--tasks_per_rs 1 \
"${target.get_install_binpath(case)}")
elif [ "$binary" == "srun" ]; then
Expand Down
4 changes: 2 additions & 2 deletions toolchain/templates/delta.mako
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
% if account:
#SBATCH --account="${account}"
% endif
% if gpu:
% if gpu_enabled:
#SBATCH --gpus-per-node=${tasks_per_node}
#SBATCH --mem=208G
#SBATCH --gpu-bind=closest
Expand All @@ -32,7 +32,7 @@ ${helpers.template_prologue()}

ok ":) Loading modules:\n"
cd "${MFC_ROOT_DIR}"
. ./mfc.sh load -c d -m ${'g' if gpu else 'c'}
. ./mfc.sh load -c d -m ${'g' if gpu_enabled else 'c'}
cd - > /dev/null
echo

Expand Down
4 changes: 2 additions & 2 deletions toolchain/templates/deltaai.mako
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
% if account:
#SBATCH --account="${account}"
% endif
% if gpu:
% if gpu_enabled:
#SBATCH --gpus-per-node=${tasks_per_node}
#SBATCH --mem=208G
#SBATCH --gpu-bind=closest
Expand All @@ -32,7 +32,7 @@ ${helpers.template_prologue()}

ok ":) Loading modules:\n"
cd "${MFC_ROOT_DIR}"
. ./mfc.sh load -c dai -m ${'g' if gpu else 'c'}
. ./mfc.sh load -c dai -m ${'g' if gpu_enabled else 'c'}
cd - > /dev/null
echo

Expand Down
8 changes: 4 additions & 4 deletions toolchain/templates/frontier.mako
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
#SBATCH --time=${walltime}
#SBATCH --cpus-per-task=7
#SBATCH -C nvme
% if gpu != 'no':
% if gpu_enabled:
#SBATCH --gpus-per-task=1
#SBATCH --gpu-bind=closest
% endif
Expand All @@ -34,12 +34,12 @@ ${helpers.template_prologue()}
ok ":) Loading modules:\n"
cd "${MFC_ROOT_DIR}"
% if engine == 'batch':
. ./mfc.sh load -c f -m ${'g' if gpu != 'no' else 'c'}
. ./mfc.sh load -c f -m ${'g' if gpu_enabled else 'c'}
% endif
cd - > /dev/null
echo

% if gpu != 'no':
% if gpu_enabled:
export MPICH_GPU_SUPPORT_ENABLED=1
% else:
export MPICH_GPU_SUPPORT_ENABLED=0
Expand All @@ -66,7 +66,7 @@ ulimit -s unlimited
% if engine == 'interactive':
--unbuffered --nodes ${nodes} --ntasks-per-node ${tasks_per_node} \
--cpus-per-task 7 \
% if gpu != 'no':
% if gpu_enabled:
--gpus-per-task 1 --gpu-bind closest \
% endif
${profiler} "${target.get_install_binpath(case)}")
Expand Down
4 changes: 2 additions & 2 deletions toolchain/templates/hipergator.mako
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
#SBATCH --output="${name}.out"
#SBATCH --time=${walltime}
#SBATCH --cpus-per-task=7
% if gpu:
% if gpu_enabled:
#SBATCH --gpus-per-task=1
#SBATCH --gpu-bind=closest
% endif
Expand All @@ -35,7 +35,7 @@ ${helpers.template_prologue()}
ok ":) Loading modules:\n"
cd "${MFC_ROOT_DIR}"
% if engine == 'batch':
. ./mfc.sh load -c h -m ${'g' if gpu else 'c'}
. ./mfc.sh load -c h -m ${'g' if gpu_enabled else 'c'}
% endif
cd - > /dev/null
echo
Expand Down
4 changes: 2 additions & 2 deletions toolchain/templates/nautilus.mako
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
% if account:
#SBATCH --account="${account}"
% endif
% if gpu:
% if gpu_enabled:
#SBATCH --gpu-bind=verbose,closest
#SBATCH --gres=gpu:v100-16:${tasks_per_node}
% endif
Expand All @@ -31,7 +31,7 @@ ${helpers.template_prologue()}

ok ":) Loading modules:\n"
cd "${MFC_ROOT_DIR}"
. ./mfc.sh load -c n -m ${'g' if gpu else 'c'}
. ./mfc.sh load -c n -m ${'g' if gpu_enabled else 'c'}
cd - > /dev/null
echo

Expand Down
4 changes: 2 additions & 2 deletions toolchain/templates/oscar.mako
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
% if account:
#SBATCH --account="${account}"
% endif
% if gpu:
% if gpu_enabled:
#SBATCH --gpus-per-node=${tasks_per_node}
#SBATCH --mem=64G
#SBATCH --gpu-bind=closest
Expand All @@ -32,7 +32,7 @@ ${helpers.template_prologue()}

ok ":) Loading modules:\n"
cd "${MFC_ROOTDIR}"
. ./mfc.sh load -c o -m ${'g' if gpu else 'c'}
. ./mfc.sh load -c o -m ${'g' if gpu_enabled else 'c'}
cd - > /dev/null
Comment on lines 34 to 36
Copy link
Contributor

@qodo-code-review qodo-code-review bot Dec 5, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggestion: Correct the typo in the environment variable from ${MFC_ROOTDIR} to ${MFC_ROOT_DIR} in the oscar.mako template. [possible issue, importance: 9]

Suggested change
cd "${MFC_ROOTDIR}"
. ./mfc.sh load -c o -m ${'g' if gpu else 'c'}
. ./mfc.sh load -c o -m ${'g' if gpu_enabled else 'c'}
cd - > /dev/null
cd "${MFC_ROOT_DIR}"
. ./mfc.sh load -c o -m ${'g' if gpu_enabled else 'c'}
cd - > /dev/null

echo

Expand Down
4 changes: 2 additions & 2 deletions toolchain/templates/phoenix-bench.mako
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
% if quality_of_service:
#SBATCH --qos=${quality_of_service}
% endif
% if gpu:
% if gpu_enabled:
#SBATCH --gres=gpu:V100:${tasks_per_node}
#SBATCH --mem-per-gpu=16G\
% endif
Expand All @@ -31,7 +31,7 @@ ${helpers.template_prologue()}

ok ":) Loading modules:\n"
cd "${MFC_ROOT_DIR}"
. ./mfc.sh load -c p -m ${'g' if gpu else 'c'}
. ./mfc.sh load -c p -m ${'g' if gpu_enabled else 'c'}
cd - > /dev/null
echo

Expand Down
4 changes: 2 additions & 2 deletions toolchain/templates/phoenix.mako
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
% if quality_of_service:
#SBATCH --qos=${quality_of_service}
% endif
% if gpu:
% if gpu_enabled:
#SBATCH --gres=gpu:V100:${tasks_per_node}
#SBATCH --mem-per-gpu=16G\
% endif
Expand All @@ -31,7 +31,7 @@ ${helpers.template_prologue()}

ok ":) Loading modules:\n"
cd "${MFC_ROOT_DIR}"
. ./mfc.sh load -c p -m ${'g' if gpu else 'c'}
. ./mfc.sh load -c p -m ${'g' if gpu_enabled else 'c'}
cd - > /dev/null
echo

Expand Down
4 changes: 2 additions & 2 deletions toolchain/templates/santis.mako
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ ${helpers.template_prologue()}
ok ":) Loading modules:\n"
cd "${MFC_ROOT_DIR}"
% if engine == 'batch':
. ./mfc.sh load -c san -m ${'g' if gpu else 'c'}
. ./mfc.sh load -c san -m ${'g' if gpu_enabled else 'c'}
% endif
cd - > /dev/null
echo
Expand All @@ -74,7 +74,7 @@ echo
--ntasks=${nodes*tasks_per_node} \
--cpus-per-task 72 \
--cpu-bind=none \
% if gpu:
% if gpu_enabled:
--gpus-per-task 1 \
% endif
--wait 200 --bcast=/tmp/${target.name} \
Expand Down
14 changes: 7 additions & 7 deletions toolchain/templates/summit.mako
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ ${helpers.template_prologue()}

ok ":) Loading modules:\n"
cd "${MFC_ROOT_DIR}"
. ./mfc.sh load -c s -m ${'g' if gpu else 'c'}
. ./mfc.sh load -c s -m ${'g' if gpu_enabled else 'c'}
cd - > /dev/null
echo

Expand All @@ -27,12 +27,12 @@ echo
(set -x; ${rofiler} "${target.get_install_binpath(case)}")
% else:
(set -x; ${profiler} \
jsrun \
${'--smpiargs="-gpu"' if gpu else ''} \
--nrs ${tasks_per_node*nodes} \
--cpu_per_rs 1 \
--gpu_per_rs ${1 if gpu else 0} \
--tasks_per_rs 1 \
jsrun \
${'--smpiargs="-gpu"' if gpu_enabled else ''} \
--nrs ${tasks_per_node*nodes} \
--cpu_per_rs 1 \
--gpu_per_rs ${1 if gpu_enabled else 0} \
--tasks_per_rs 1 \
"${target.get_install_binpath(case)}")
% endif

Expand Down
Loading