port to tacc vista system

ESMCI · Jul 29, 2024 · 452642c · 452642c
1 parent 1d99a89
commit 452642c
Show file tree

Hide file tree

Showing 4 changed files with 73 additions and 8 deletions.
diff --git a/config/cesm/machines/config_batch.xml b/config/cesm/machines/config_batch.xml
@@ -706,6 +706,18 @@
     </queues>
   </batch_system>
 
+  <batch_system MACH="vista" type="slurm" >
+    <batch_submit>ssh vista.tacc.utexas.edu cd $CASEROOT ; sbatch</batch_submit>
+    <submit_args>
+      <arg flag="--time" name="$JOB_WALLCLOCK_TIME"/>
+      <arg flag="-p" name="$JOB_QUEUE"/>
+      <arg flag="--account" name="$PROJECT"/>
+    </submit_args>
+    <queues>
+      <queue walltimemax="2:00:00" nodemin="1" nodemax="30">gh-dev</queue>
+      <queue walltimemax="2:00:00" nodemin="1" nodemax="250" default="true">gh-dev-64k</queue>
+    </queues>
+  </batch_system>
   <batch_system MACH="zeus" type="lsf">
     <batch_env>-env</batch_env>
     <submit_args>

diff --git a/config/cesm/machines/config_compilers.xml b/config/cesm/machines/config_compilers.xml
@@ -311,7 +311,7 @@ using a fortran linker.
   <SFC> nagfor </SFC>
 </compiler>
 
-<compiler COMPILER="pgi">
+<compiler COMPILER="nvhpc">
   <CFLAGS>
     <base> -gopt  -time </base>
     <append compile_threaded="true"> -mp </append>
@@ -615,19 +615,23 @@ using a fortran linker.
   </CFLAGS>
 </compiler>
 
-<compiler MACH="bluewaters" COMPILER="pgi">
+<compiler MACH="vista" COMPILER="nvhpc">
   <CFLAGS>
     <append DEBUG="FALSE"> -O2 </append>
     <append> -nofma </append>
   </CFLAGS>
   <CXX_LIBS>
-    <base> -lmpichf90_pgi $ENV{PGI_PATH}/linux86-64/$ENV{PGI_VERSION}/lib/f90main.o </base>
+    <base>  </base>
   </CXX_LIBS>
   <FFLAGS>
     <append DEBUG="FALSE"> -O2 </append>
     <append> -nofma </append>
   </FFLAGS>
   <SUPPORTS_CXX>TRUE</SUPPORTS_CXX>
+  <SLIBS>
+    <append> -L$ENV{NETCDF_PATH}/lib -lnetcdff -lnetcdf </append>
+    <append MPILIB="openmpi"> -L$ENV{PNETCDF_PATH}/lib -lpnetcdf</append>
+  </SLIBS>
 </compiler>
 
 

diff --git a/config/cesm/machines/config_machines.xml b/config/cesm/machines/config_machines.xml
@@ -2890,6 +2890,55 @@ This allows using a different mpirun command to launch unit tests
     </module_system>
   </machine>
 
+  <machine MACH="vista">
+    <DESC>Grace Hopper ARM TACC , batch system is SLURM</DESC>
+    <NODENAME_REGEX>.*.vista.tacc.utexas.edu</NODENAME_REGEX>
+    <OS>LINUX</OS>
+    <COMPILERS>nvhpc,gnu</COMPILERS>
+    <MPILIBS>openmpi</MPILIBS>
+    <CIME_OUTPUT_ROOT>$ENV{SCRATCH}</CIME_OUTPUT_ROOT>
+    <DIN_LOC_ROOT>/work2/02503/edwardsj/CESM/inputdata</DIN_LOC_ROOT>
+    <DIN_LOC_ROOT_CLMFORC>/work2/02503/edwardsj/CESM/inputdata/lmwg</DIN_LOC_ROOT_CLMFORC>
+    <DOUT_S_ROOT>$CIME_OUTPUT_ROOT/archive/$CASE</DOUT_S_ROOT>
+    <BASELINE_ROOT>/work2/02503/edwardsj/CESM/cesm_baselines</BASELINE_ROOT>
+    <CCSM_CPRNC>/work2/02503/edwardsj/CESM/cime/tools/cprnc/cprnc.vista</CCSM_CPRNC>
+    <GMAKE_J>4</GMAKE_J>
+    <BATCH_SYSTEM>slurm</BATCH_SYSTEM>
+    <SUPPORTED_BY></SUPPORTED_BY>
+    <MAX_TASKS_PER_NODE>72</MAX_TASKS_PER_NODE>
+    <MAX_MPITASKS_PER_NODE>72</MAX_MPITASKS_PER_NODE>
+    <mpirun mpilib="openmpi">
+      <executable>mpirun</executable>
+      <arguments>
+	<arg name="num_tasks"> -np $TOTALPES</arg>
+      </arguments>
+    </mpirun>
+    <module_system type="module">
+      <init_path lang="perl">/opt/apps/lmod/lmod/init/perl</init_path>
+      <init_path lang="python">/opt/apps/lmod/lmod/init/env_modules_python.py</init_path>
+      <init_path lang="sh">/opt/apps/lmod/lmod/init/sh</init_path>
+      <init_path lang="csh">/opt/apps/lmod/lmod/init/csh</init_path>
+      <cmd_path lang="perl">/opt/apps/lmod/lmod/libexec/lmod perl</cmd_path>
+      <cmd_path lang="python">/opt/apps/lmod/lmod/libexec/lmod python</cmd_path>
+      <cmd_path lang="sh">module</cmd_path>
+      <cmd_path lang="csh">module</cmd_path>
+      <modules>
+	<command name="purge"></command>
+	<command name="load">TACC</command>
+	<command name="load">nvidia</command>
+      </modules>
+      <modules mpilib="openmpi">
+	<command name="load">openmpi/5.0.3</command>
+      </modules>
+    </module_system>
+    <environment_variables>
+      <env name="OMP_STACKSIZE">256M</env>
+      <env name="NETCDF_PATH">/scratch/00422/cazes/vista_netcdf_install/netcdf_4.9.2_nvhpc/</env>
+      <env name="PNETCDF_PATH">/scratch/00422/cazes/vista_netcdf_install/pnetcdf_1.12.3_nvhpc/</env>
+    </environment_variables>
+  </machine>
+
+
   <machine MACH="zeus">
     <DESC> CMCC Lenovo ThinkSystem SD530, os is Linux, 36 pes/node, batch system is LSF</DESC>
     <NODENAME_REGEX>(login[1,2]-ib|n[0-9][0-9][0-9]-ib)</NODENAME_REGEX>

diff --git a/scripts/tests/scripts_regression_tests.py b/scripts/tests/scripts_regression_tests.py
@@ -303,21 +303,21 @@ def test_a_createnewcase(self):
         self.assertTrue(os.path.exists(os.path.join(testdir, "case.setup")))
 
         run_cmd_assert_result(self, "./case.setup", from_dir=testdir)
-        run_cmd_assert_result(self, "./case.build", from_dir=testdir)
+        run_cmd_assert_result(self, "./case.build --skip-provenance-check", from_dir=testdir)
 
         with Case(testdir, read_only=False) as case:
             ntasks = case.get_value("NTASKS_ATM")
             case.set_value("NTASKS_ATM", ntasks+1)
         # this should fail with a locked file issue
-        run_cmd_assert_result(self, "./case.build",
+        run_cmd_assert_result(self, "./case.build --skip-provenance-check",
                               from_dir=testdir, expected_stat=1)
 
         run_cmd_assert_result(self, "./case.setup --reset", from_dir=testdir)
-        run_cmd_assert_result(self, "./case.build", from_dir=testdir)
+        run_cmd_assert_result(self, "./case.build --skip-provenance-check", from_dir=testdir)
         with Case(testdir, read_only=False) as case:
             case.set_value("CHARGE_ACCOUNT", "fred")
         # this should not fail with a locked file issue
-        run_cmd_assert_result(self, "./case.build",from_dir=testdir, expected_stat=0)
+        run_cmd_assert_result(self, "./case.build --skip-provenance-check",from_dir=testdir, expected_stat=0)
 
         run_cmd_assert_result(self, "./case.st_archive --test-all", from_dir=testdir)
 
@@ -488,7 +488,7 @@ def test_f_createnewcase_with_user_compset(self):
 
         run_cmd_assert_result(self, "%s/create_newcase %s"%(SCRIPT_DIR, args), from_dir=SCRIPT_DIR)
         run_cmd_assert_result(self, "./case.setup", from_dir=testdir)
-        run_cmd_assert_result(self, "./case.build", from_dir=testdir)
+        run_cmd_assert_result(self, "./case.build --skip-provenance-check", from_dir=testdir)
 
         cls._do_teardown.append(testdir)