Swap to using median value with outliers removed for deciding on best solution to run (#3515)

* Swap to using modified Z when deciding on best solution to run

* Update comments

* Code review comments swap to using mean, and removing only positive-z scores

* Code review comments and warnings clean-up

* Code review comments change names

* Code review comments add static_assert

* Update copyright date

Author: BrianHarrisonAMD

src/conv/solver_finders.cpp

@@ -32,6 +32,7 @@
 #include <miopen/perf_field.hpp>
 #include <miopen/conv/problem_description.hpp>
 #include <miopen/solution.hpp>
+#include <miopen/utility/modified_z.hpp>
 
 MIOPEN_DECLARE_ENV_VAR_BOOL(MIOPEN_DEBUG_CONV_GEMM)
 MIOPEN_DECLARE_ENV_VAR_BOOL(MIOPEN_DEBUG_CONV_DIRECT)
@@ -230,6 +231,7 @@ static std::vector<Solution> EvaluateInvokers(const Handle& handle,
     auto best         = std::numeric_limits<float>::max();
     auto best_invoker = Invoker{};
     auto ret          = std::vector<Solution>{};
+    std::vector<float> samples;
 
     for(const auto& sol : solutions)
     {
@@ -261,31 +263,40 @@ static std::vector<Solution> EvaluateInvokers(const Handle& handle,
 
         try
         {
-            // Run invoker max 6 times, with ~5 sec time limit.
+            // Run invoker max 8 times, with ~5 sec time limit.
             using elapsed_t                 = decltype(handle.GetKernelTime());
             constexpr elapsed_t TIME_MS_MAX = 5000.0;
             constexpr int N_RUNS_MAX        = 8;
-            constexpr int N_RUNS_DISCARD    = 3;
             auto elapsed                    = static_cast<elapsed_t>(0);
             auto first_elapsed              = static_cast<elapsed_t>(0);
             int i                           = 0;
+            samples.clear();
             while(i < N_RUNS_MAX && elapsed < TIME_MS_MAX)
             {
                 invoker(handle, invoke_ctx);
-                elapsed += handle.GetKernelTime();
-                if(i == (N_RUNS_DISCARD - 1))
-                    first_elapsed = elapsed;
+
+                // don't include warm-up run in our samples.
+                if(i > 0)
+                {
+                    samples.push_back(handle.GetKernelTime());
+                }
+                else
+                {
+                    // Keep first run just in case we go over the limit, and have no samples.
+                    first_elapsed = handle.GetKernelTime();
+                }
                 ++i;
             }
-            // If the execution time was not too long,
-            // then the 1st run is not counted (assume it's warm-up):
-            if(i > N_RUNS_DISCARD)
+
+            if(samples.size() > 0)
             {
-                elapsed = (elapsed - first_elapsed) / static_cast<elapsed_t>(i - N_RUNS_DISCARD);
+                // Remove outliers that are more than 2 positive modified z-score's away, and get
+                // the mean.
+                elapsed = miopen::removeHighOutliersAndGetMean(samples, 2.0f);
             }
-            else if(i > 0)
+            else
             {
-                elapsed /= i;
+                elapsed = first_elapsed;
             }
 
             MIOPEN_THROW_IF(elapsed <= 0, "Invalid elapsed time detected in EvaluateInvokers");

src/include/miopen/generic_search.hpp

@@ -38,6 +38,7 @@
 #include <miopen/timer.hpp>
 #include <miopen/mt_queue.hpp>
 #include <miopen/generic_search_controls.hpp>
+#include <miopen/utility/modified_z.hpp>
 
 #include <algorithm>
 #include <vector>
@@ -400,10 +401,11 @@ auto GenericSearch(const Solver s,
         }
     }
 
-    bool is_passed  = false; // left false only if all iterations failed.
-    float best_time = std::numeric_limits<float>::max();
-    size_t n_failed = 0;
-    size_t n_best   = 0;
+    bool is_passed   = false; // left false only if all iterations failed.
+    float best_time  = std::numeric_limits<float>::max();
+    float worst_time = std::numeric_limits<float>::max();
+    size_t n_failed  = 0;
+    size_t n_best    = 0;
     HeartBeat<PerformanceConfig> heartbeat;
     heartbeat.Start();
 
@@ -429,6 +431,7 @@ auto GenericSearch(const Solver s,
         size_t n_current       = 0;
         size_t last_imprv      = 0;
         auto threads_remaining = total_threads;
+        std::vector<float> samples;
         while(true)
         {
             if(n_current >= n_runs_total)
@@ -461,6 +464,7 @@ auto GenericSearch(const Solver s,
                 }
             }
 
+            samples.clear();
             float elapsed_time = 0.0f;
             int ret            = 0;
             MIOPEN_LOG_I2('#' << n_current << '/' << n_failed << '/' << n_runs_total << ' '
@@ -481,8 +485,18 @@ auto GenericSearch(const Solver s,
 
                 invoker = profile_h.PrepareInvoker(*current_solution.invoker_factory,
                                                    current_solution.construction_params);
+
+                // Warm-up run for first time invoker is used
+                if(n_current == 0)
+                {
+                    invoker(profile_h, invoke_ctx);
+                    profile_h.ResetKernelTime();
+                }
+
                 invoker(profile_h, invoke_ctx);
                 elapsed_time = profile_h.GetKernelTime();
+                samples.push_back(elapsed_time);
+                profile_h.ResetKernelTime();
             }
             catch(const std::exception& e)
             {
@@ -503,11 +517,11 @@ auto GenericSearch(const Solver s,
             if(ret == 0)
             {
                 // Smooth the jitter of measurements:
-                // If the 1st probe is NOT too bad (measured time <= 1.10 * best known time),
-                // then re-run it 9 times more and compute average time,
-                // and decide using average of all 10 attempts vs. the best.
+                // If the 1st probe is NOT too bad (measured time <= 1.10 * worst sample of the best
+                // config), then gather 9 more samples, and remove positive z-score outliers. Use
+                // the mean value with outliers removed for calculating best config.
                 constexpr int N_RUNS = 10;
-                if(elapsed_time / best_time < 1.10f)
+                if(elapsed_time / worst_time < 1.10f)
                 {
                     MIOPEN_LOG_I2("Finding average for: " << elapsed_time << " / " << best_time
                                                           << " = " << (elapsed_time / best_time));
@@ -517,7 +531,8 @@ auto GenericSearch(const Solver s,
                         for(int i = 1; i < N_RUNS; ++i)
                         {
                             invoker(profile_h, invoke_ctx);
-                            elapsed_time += profile_h.GetKernelTime();
+                            samples.push_back(profile_h.GetKernelTime());
+                            profile_h.ResetKernelTime();
                         }
                     }
                     catch(...)
@@ -528,21 +543,28 @@ auto GenericSearch(const Solver s,
                     if(ret == 0)
                     {
                         is_passed = true;
-                        elapsed_time /= N_RUNS;
+
+                        // Remove outliers that are more than 2 positive modified z-score's away,
+                        // and get the mean.
+                        elapsed_time = miopen::removeHighOutliersAndGetMean(samples, 2.0f);
                         if(elapsed_time < best_time)
                         {
                             MIOPEN_LOG_I('#' << n_current << '/' << n_failed << '/' << n_runs_total
                                              << ' ' << elapsed_time << " < " << best_time << ' '
                                              << current_config);
                             best_config = current_config;
                             best_time   = elapsed_time;
-                            n_best      = n_current;
-                            last_imprv  = 0;
+
+                            // Samples gets sorted by the RemoveOutliers call so the last element
+                            // will be the slowest.
+                            worst_time = samples.back();
+                            n_best     = n_current;
+                            last_imprv = 0;
                         }
                         else
                         {
-                            MIOPEN_LOG_I2("Average is not better: " << elapsed_time
-                                                                    << " >= " << best_time);
+                            MIOPEN_LOG_I2("Mean is not better: " << elapsed_time
+                                                                 << " >= " << best_time);
                         }
                     }
                 }

src/include/miopen/utility/modified_z.hpp

@@ -0,0 +1,134 @@
+/*******************************************************************************
+ *
+ * MIT License
+ *
+ * Copyright (c) 2025 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ *******************************************************************************/
+
+#pragma once
+
+#include <vector>
+#include <algorithm>
+#include <numeric>
+#include <miopen/errors.hpp>
+
+namespace miopen {
+
+template <typename T>
+T mean(const std::vector<T>& data)
+{
+    static_assert(std::is_floating_point_v<T>);
+    MIOPEN_THROW_IF(data.size() == 0, "Cannot find Mean of 0 length data");
+
+    T sumOfValues = std::accumulate(data.begin(), data.end(), 0.0);
+    return sumOfValues / data.size();
+}
+
+template <typename T>
+T medianOfSortedData(const std::vector<T>& sortedData)
+{
+    static_assert(std::is_floating_point_v<T>);
+    MIOPEN_THROW_IF(sortedData.size() == 0, "Cannot find Median of 0 length data");
+
+    size_t size = sortedData.size();
+
+    T median = (size % 2 == 0) ? (sortedData[size / 2 - 1] + sortedData[size / 2]) / 2.0
+                               : sortedData[size / 2];
+
+    return median;
+}
+
+template <typename T>
+T median(std::vector<T>& data)
+{
+    static_assert(std::is_floating_point_v<T>);
+    // Note: The data needs to be sorted for other parts of the algorthim
+    std::sort(data.begin(), data.end());
+
+    return medianOfSortedData(data);
+}
+
+template <typename T>
+std::vector<T> medianAbsoluteDeviation(const std::vector<T>& sortedData)
+{
+    static_assert(std::is_floating_point_v<T>);
+    T median = medianOfSortedData(sortedData);
+
+    std::vector<T> absDeviation;
+    absDeviation.reserve(sortedData.size());
+
+    std::transform(sortedData.begin(),
+                   sortedData.end(),
+                   std::back_inserter(absDeviation),
+                   [&](auto& value) { return std::abs(value - median); });
+
+    return absDeviation;
+}
+
+template <typename T>
+std::vector<T> modifiedZScores(const std::vector<T>& sortedData)
+{
+    static_assert(std::is_floating_point_v<T>);
+    T medianValue = medianOfSortedData(sortedData);
+
+    std::vector<T> absolute_deviation = medianAbsoluteDeviation(sortedData);
+    T mad                             = median(absolute_deviation);
+
+    // If MAD is 0, then we cannot calcualte the ModifiedZScore
+    if(mad == T{0})
+    {
+        return std::vector<T>(sortedData.size(), 0);
+    }
+    else
+    {
+        std::vector<T> modZScores;
+        modZScores.reserve(sortedData.size());
+
+        std::transform(sortedData.begin(),
+                       sortedData.end(),
+                       std::back_inserter(modZScores),
+                       [&](auto& value) { return 0.6745 * (value - medianValue) / mad; });
+
+        return modZScores;
+    }
+}
+
+template <typename T>
+T removeHighOutliersAndGetMean(std::vector<T>& data, T z_threshold)
+{
+    static_assert(std::is_floating_point_v<T>);
+    std::sort(data.begin(), data.end());
+
+    std::vector<T> modZScores = modifiedZScores(data);
+    std::vector<T> filteredData;
+
+    for(size_t i = 0; i < data.size(); ++i)
+    {
+        if(modZScores[i] <= z_threshold)
+        {
+            filteredData.push_back(data[i]);
+        }
+    }
+
+    return mean(filteredData);
+}
+} // namespace miopen