Merge pull request #596 from LLNL/add-kokkos-nodal-accum

This PR adds a Kokkos variant for NODAL_ACCUMULATION_3D.

Author: MrBurmark

src/CMakeLists.txt

@@ -12,6 +12,7 @@ add_subdirectory(common)
 add_subdirectory(basic)
 add_subdirectory(basic-kokkos)
 add_subdirectory(apps)
+add_subdirectory(apps-kokkos)
 add_subdirectory(lcals)
 add_subdirectory(lcals-kokkos)
 add_subdirectory(polybench)
@@ -23,6 +24,7 @@ add_subdirectory(comm)
 set(RAJA_PERFSUITE_LIBS
     common
     apps
+    apps-kokkos
     basic
     basic-kokkos
     lcals

src/apps-kokkos/CMakeLists.txt

@@ -0,0 +1,16 @@
+###############################################################################
+# Copyright (c) 2017-25, Lawrence Livermore National Security, LLC
+# and RAJA Performance Suite project contributors.
+# See the RAJAPerf/LICENSE file for details.
+#
+# SPDX-License-Identifier: (BSD-3-Clause)
+###############################################################################
+
+
+blt_add_library(
+  NAME apps-kokkos
+  SOURCES
+          NODAL_ACCUMULATION_3D-Kokkos.cpp
+  INCLUDES ${CMAKE_CURRENT_SOURCE_DIR}/../apps
+  DEPENDS_ON common ${RAJA_PERFSUITE_DEPENDS}
+  )

src/apps-kokkos/NODAL_ACCUMULATION_3D-Kokkos.cpp

@@ -0,0 +1,78 @@
+//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~//
+// Copyright (c) 2017-25, Lawrence Livermore National Security, LLC
+// and RAJA Performance Suite project contributors.
+// See the RAJAPerf/LICENSE file for details.
+//
+// SPDX-License-Identifier: (BSD-3-Clause)
+//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~//
+
+#include "NODAL_ACCUMULATION_3D.hpp"
+
+#if defined(RUN_KOKKOS)
+#include "common/KokkosViewUtils.hpp"
+#include <iostream>
+#include "AppsData.hpp"
+
+namespace rajaperf
+{
+namespace apps
+{
+
+void NODAL_ACCUMULATION_3D::runKokkosVariant(VariantID vid)
+{
+  const Index_type run_reps = getRunReps();
+  const Index_type ibegin = 0;
+  const Index_type iend = getActualProblemSize(); //m_domain->n_real_zones;
+
+  NODAL_ACCUMULATION_3D_DATA_SETUP;
+
+  if ( vid == Kokkos_Lambda ) {
+    int jp = m_domain->jp;
+    int kp = m_domain->kp;
+    auto real_zones_v = getViewFromPointer(real_zones, iend);
+    auto vol_v = getViewFromPointer(vol, m_nodal_array_length);
+    auto x_v = getViewFromPointer(x, m_nodal_array_length);
+   
+    using view_t = decltype(x_v);
+    view_t x0_v = x_v;
+    view_t x1_v(x_v.data() + 1, m_nodal_array_length - 1); 
+    view_t x2_v(x_v.data() + jp, m_nodal_array_length - jp); 
+    view_t x3_v(x_v.data() + jp, m_nodal_array_length - jp); 
+    view_t x4_v(x_v.data() + jp, m_nodal_array_length - jp); 
+    view_t x5_v(x_v.data() + kp, m_nodal_array_length - kp); 
+    view_t x6_v(x_v.data() + kp, m_nodal_array_length - kp); 
+    view_t x7_v(x_v.data() + kp, m_nodal_array_length - kp); 
+
+    Kokkos::fence();
+    startTimer();
+    // Awkward expression for loop counter quiets C++20 compiler warning
+    for (RepIndex_type irep = 0; irep < run_reps; ((irep = irep + 1), 0)) {
+      Kokkos::parallel_for("NODAL_ACCUMULATION_3D", iend, KOKKOS_LAMBDA(Index_type ii) {
+        Index_type i = real_zones_v(ii);
+        Real_type val = 0.125 * vol_v(i);
+        Kokkos::atomic_add(&x0_v(i), val);
+        Kokkos::atomic_add(&x1_v(i), val);
+        Kokkos::atomic_add(&x2_v(i), val);
+        Kokkos::atomic_add(&x3_v(i), val);
+        Kokkos::atomic_add(&x4_v(i), val);
+        Kokkos::atomic_add(&x5_v(i), val);
+        Kokkos::atomic_add(&x6_v(i), val);
+        Kokkos::atomic_add(&x7_v(i), val);
+      });
+    }
+    Kokkos::fence();
+    stopTimer();
+
+    moveDataToHostFromKokkosView(x, x_v, m_nodal_array_length);
+
+  } else {
+     getCout() << "\n  NODAL_ACCUMULATION_3D : Unknown Kokkos variant id = " << vid << std::endl;
+  }
+}
+
+RAJAPERF_DEFAULT_TUNING_DEFINE_BOILERPLATE(NODAL_ACCUMULATION_3D, Kokkos, Kokkos_Lambda)
+
+} // end namespace apps
+} // end namespace rajaperf
+
+#endif  // RAJA_ENABLE_CUDA

src/apps/NODAL_ACCUMULATION_3D.hpp

@@ -85,10 +85,12 @@ class NODAL_ACCUMULATION_3D : public KernelBase
   void defineOpenMPTargetVariantTunings();
   void defineCudaVariantTunings();
   void defineHipVariantTunings();
+  void defineKokkosVariantTunings();
 
   void runSeqVariant(VariantID vid);
   void runOpenMPVariant(VariantID vid);
   void runOpenMPTargetVariant(VariantID vid);
+  void runKokkosVariant(VariantID vid);
 
   template < size_t block_size >
   void runCudaVariantImpl(VariantID vid);

test/CMakeLists.txt

@@ -9,6 +9,7 @@
 set(RAJA_PERFSUITE_TEST_EXECUTABLE_DEPENDS
     common
     apps
+    apps-kokkos
     basic
     basic-kokkos
     lcals