Refactor: Exclude selected prop keys in API instead of in FE

Feedback from original PR: #3719 (comment)

The FE approach would run into problems if you have >25 custom props, so excluding
in the BE allows avoiding problems

Author: macobo

assets/js/dashboard/components/combobox.js

@@ -92,9 +92,7 @@ export default function PlausibleCombobox(props) {
   }
 
   function isOptionDisabled(option) {
-    const optionAlreadySelected = props.values.some((val) => val.value === option.value)
-    const optionDisabled = (props.disabledOptions || []).some((val) => val?.value === option.value)
-    return optionAlreadySelected || optionDisabled
+    return props.values.some((val) => val.value === option.value)
   }
 
   function fetchOptions(query) {

assets/js/dashboard/stats/modals/prop-filter-modal.js

@@ -34,7 +34,7 @@ function PropFilterModal(props) {
   const query = parseQuery(props.location.search, props.site)
   const [formState, setFormState] = useState(getFormState(query))
 
-  const selectedPropKeys = useMemo(() => Object.values(formState.values).map((value) => value.propKey), [formState])
+  const selectedPropKeys = useMemo(() => Object.values(formState.values).map((value) => value.propKey).filter((value) => value), [formState])
 
   function onPropKeySelect(id, selectedOptions) {
     const newPropKey = selectedOptions.length === 0 ? null : selectedOptions[0]

assets/js/dashboard/stats/modals/prop-filter-row.js

@@ -23,7 +23,8 @@ function PropFilterRow({
 }) {
   function fetchPropKeyOptions() {
     return (input) => {
-      return api.get(apiPath(site, "/suggestions/prop_key"), query, { q: input.trim() })
+      const exclude = selectedPropKeys.map((key) => key.value)
+      return api.get(apiPath(site, "/suggestions/prop_key"), query, { q: input.trim(), exclude: JSON.stringify(exclude) })
     }
   }
 
@@ -50,7 +51,6 @@ function PropFilterRow({
           values={propKey ? [propKey] : []}
           onSelect={(value) => onPropKeySelect(id, value)}
           placeholder={'Property'}
-          disabledOptions={selectedPropKeys}
         />
       </div>
       <div className="col-span-3 mx-2">

lib/plausible/stats.ex

@@ -38,8 +38,8 @@ defmodule Plausible.Stats do
     end
   end
 
-  def filter_suggestions(site, query, filter_name, filter_search) do
+  def filter_suggestions(site, query, filter_name, filter_search, exclude \\ %{}) do
     include_sentry_replay_info()
-    FilterSuggestions.filter_suggestions(site, query, filter_name, filter_search)
+    FilterSuggestions.filter_suggestions(site, query, filter_name, filter_search, exclude)
   end
 end

lib/plausible/stats/filter_suggestions.ex

@@ -5,7 +5,7 @@ defmodule Plausible.Stats.FilterSuggestions do
   import Plausible.Stats.Base
   alias Plausible.Stats.Query
 
-  def filter_suggestions(site, query, "country", filter_search) do
+  def filter_suggestions(site, query, "country", filter_search, _exclude) do
     matches = Location.search_country(filter_search)
 
     q =
@@ -28,7 +28,7 @@ defmodule Plausible.Stats.FilterSuggestions do
     end)
   end
 
-  def filter_suggestions(site, query, "region", "") do
+  def filter_suggestions(site, query, "region", "", _exclude) do
     from(
       e in query_sessions(site, query),
       group_by: e.subdivision1_code,
@@ -48,7 +48,7 @@ defmodule Plausible.Stats.FilterSuggestions do
     end)
   end
 
-  def filter_suggestions(site, query, "region", filter_search) do
+  def filter_suggestions(site, query, "region", filter_search, _exclude) do
     matches = Location.search_subdivision(filter_search)
 
     q =
@@ -71,7 +71,7 @@ defmodule Plausible.Stats.FilterSuggestions do
     end)
   end
 
-  def filter_suggestions(site, query, "city", "") do
+  def filter_suggestions(site, query, "city", "", _exclude) do
     from(
       e in query_sessions(site, query),
       group_by: e.city_geoname_id,
@@ -91,7 +91,7 @@ defmodule Plausible.Stats.FilterSuggestions do
     end)
   end
 
-  def filter_suggestions(site, query, "city", filter_search) do
+  def filter_suggestions(site, query, "city", filter_search, _exclude) do
     filter_search = String.downcase(filter_search)
 
     q =
@@ -118,7 +118,7 @@ defmodule Plausible.Stats.FilterSuggestions do
     end)
   end
 
-  def filter_suggestions(site, _query, "goal", filter_search) do
+  def filter_suggestions(site, _query, "goal", filter_search, _exclude) do
     site
     |> Plausible.Goals.for_site()
     |> Enum.map(fn x -> if x.event_name, do: x.event_name, else: "Visit #{x.page_path}" end)
@@ -131,14 +131,15 @@ defmodule Plausible.Stats.FilterSuggestions do
     |> wrap_suggestions()
   end
 
-  def filter_suggestions(site, query, "prop_key", filter_search) do
+  def filter_suggestions(site, query, "prop_key", filter_search, exclude) do
     filter_query = if filter_search == nil, do: "%", else: "%#{filter_search}%"
 
     from(e in base_event_query(site, query),
       array_join: meta in "meta",
       as: :meta,
       select: meta.key,
       where: fragment("? ilike ?", meta.key, ^filter_query),
+      where: meta.key not in ^exclude,
       group_by: meta.key,
       order_by: [desc: fragment("count(*)")],
       limit: 25
@@ -148,7 +149,7 @@ defmodule Plausible.Stats.FilterSuggestions do
     |> wrap_suggestions()
   end
 
-  def filter_suggestions(site, query, "prop_value", filter_search) do
+  def filter_suggestions(site, query, "prop_value", filter_search, _exclude) do
     filter_query = if filter_search == nil, do: "%", else: "%#{filter_search}%"
 
     {"event:props:" <> key, _filter} = Query.get_filter_by_prefix(query, "event:props")
@@ -180,7 +181,7 @@ defmodule Plausible.Stats.FilterSuggestions do
     |> wrap_suggestions()
   end
 
-  def filter_suggestions(site, query, filter_name, filter_search) do
+  def filter_suggestions(site, query, filter_name, filter_search, _exclude) do
     filter_search = if filter_search == nil, do: "", else: filter_search
 
     filter_query =

lib/plausible_web/controllers/api/stats_controller.ex

@@ -1273,10 +1273,11 @@ defmodule PlausibleWeb.Api.StatsController do
     site = conn.assigns[:site]
 
     query = Query.from(site, params)
+    exclude = Jason.decode!(params["exclude"] || "[]")
 
     json(
       conn,
-      Stats.filter_suggestions(site, query, params["filter_name"], params["q"])
+      Stats.filter_suggestions(site, query, params["filter_name"], params["q"], exclude)
     )
   end
 

test/plausible_web/controllers/api/stats_controller/suggestions_test.exs

@@ -282,6 +282,51 @@ defmodule PlausibleWeb.Api.StatsController.SuggestionsTest do
              ]
     end
 
+    test "allowing excluding keys", %{conn: conn, site: site} do
+      populate_stats(site, [
+        build(:pageview,
+          "meta.key": ["author"],
+          "meta.value": ["Uku Taht"],
+          timestamp: ~N[2022-01-01 00:00:00]
+        ),
+        build(:pageview,
+          "meta.key": ["author"],
+          "meta.value": ["Uku Taht"],
+          timestamp: ~N[2022-01-01 00:00:00]
+        ),
+        build(:pageview,
+          "meta.key": ["author"],
+          "meta.value": ["Uku Taht"],
+          timestamp: ~N[2022-01-01 00:00:00]
+        ),
+        build(:pageview,
+          "meta.key": ["logged_in"],
+          "meta.value": ["false"],
+          timestamp: ~N[2022-01-01 00:00:00]
+        ),
+        build(:pageview,
+          "meta.key": ["logged_in"],
+          "meta.value": ["false"],
+          timestamp: ~N[2022-01-01 00:00:00]
+        ),
+        build(:pageview,
+          "meta.key": ["dark_mode"],
+          "meta.value": ["true"],
+          timestamp: ~N[2022-01-01 00:00:00]
+        )
+      ])
+
+      conn =
+        get(
+          conn,
+          "/api/stats/#{site.domain}/suggestions/prop_key?period=day&date=2022-01-01&exclude=[\"author\",\"logged_in\"]"
+        )
+
+      assert json_response(conn, 200) == [
+               %{"label" => "dark_mode", "value" => "dark_mode"}
+             ]
+    end
+
     test "returns suggestions found in time frame", %{
       conn: conn,
       site: site