Support contains/matches for custom properties (#3687)

* Allow `matches` operator to work in BE for custom props

Note: No FE support yet, needs further testing

* feat: allow choosing `contains` for property filters in the UI

* no autocomplete on prop values if `contains` for consistency

* CHANGELOG.md

* Fix: Handle (none) property in property breakdowns when using matching

When matching we should always exclude (none)

Author: macobo

CHANGELOG.md

@@ -2,6 +2,7 @@
 All notable changes to this project will be documented in this file.
 
 ### Added
+- Allow filtering with `contains`/`matches` operator for custom properties
 - Add `referrers.csv` to CSV export
 - Add a new Properties section in the dashboard to break down by custom properties
 - Add `custom_props.csv` to CSV export (almost the same as the old `prop_breakdown.csv`, but has different column headers, and includes props for pageviews too, not only custom events)

assets/js/dashboard/stats/modals/prop-filter-modal.js

@@ -40,11 +40,15 @@ function PropFilterModal(props) {
 
   const fetchPropValueOptions = useCallback(() => {
     return (input) => {
+      if (formState.prop_value?.type === FILTER_TYPES.contains) {
+        return Promise.resolve([])
+      }
+
       const propKey = formState.prop_key?.value
       const updatedQuery = { ...query, filters: { ...query.filters, props: {[propKey]: '!(none)'} } }
       return api.get(apiPath(props.site, "/suggestions/prop_value"), updatedQuery, { q: input.trim() })
     }
-  }, [formState.prop_key])
+  }, [formState.prop_key, formState.prop_value])
 
   function onPropKeySelect() {
     return (selectedOptions) => {
@@ -86,7 +90,14 @@ function PropFilterModal(props) {
           <FilterTypeSelector isDisabled={!formState.prop_key} forFilter={'prop_value'} onSelect={onFilterTypeSelect()} selectedType={selectedFilterType()} />
         </div>
         <div className="col-span-4">
-          <Combobox isDisabled={!formState.prop_key} fetchOptions={fetchPropValueOptions()} values={formState.prop_value.clauses} onSelect={onPropValueSelect()} placeholder={'Value'} />
+          <Combobox
+            isDisabled={!formState.prop_key}
+            fetchOptions={fetchPropValueOptions()}
+            values={formState.prop_value.clauses}
+            onSelect={onPropValueSelect()}
+            placeholder={'Value'}
+            freeChoice={selectedFilterType() == FILTER_TYPES.contains}
+          />
         </div>
       </div>
     )
@@ -167,4 +178,4 @@ function PropFilterModal(props) {
   )
 }
 
-export default withRouter(PropFilterModal)
+export default withRouter(PropFilterModal)

assets/js/dashboard/util/filters.js

@@ -10,6 +10,10 @@ export const FILTER_GROUPS = {
   'props': ['prop_key', 'prop_value']
 }
 
+export const ALLOW_FREE_CHOICE = new Set(
+  FILTER_GROUPS['page'].concat(FILTER_GROUPS['utm']).concat(['prop_value'])
+)
+
 export const FILTER_TYPES = {
   isNot: 'is not',
   contains: 'contains',
@@ -27,7 +31,7 @@ export function supportsIsNot(filterName) {
 }
 
 export function isFreeChoiceFilter(filterName) {
-  return FILTER_GROUPS['page'].concat(FILTER_GROUPS['utm']).includes(filterName)
+  return ALLOW_FREE_CHOICE.has(filterName)
 }
 
 // As of March 2023, Safari does not support negative lookbehind regexes. In case it throws an error, falls back to plain | matching. This means

lib/plausible/stats/base.ex

@@ -142,6 +142,16 @@ defmodule Plausible.Stats.Base do
             )
           end
 
+        {"event:props:" <> prop_name, {:matches, value}} ->
+          regex = page_regex(value)
+
+          from(
+            e in q,
+            left_array_join: meta in "meta",
+            as: :meta,
+            where: meta.key == ^prop_name and fragment("match(?, ?)", meta.value, ^regex)
+          )
+
         {"event:props:" <> prop_name, {:member, values}} ->
           none_value_included = Enum.member?(values, "(none)")
 

lib/plausible/stats/breakdown.ex

@@ -189,6 +189,7 @@ defmodule Plausible.Stats.Breakdown do
   defp include_none_result?({:is_not, "(none)"}), do: false
   defp include_none_result?({:member, values}), do: Enum.member?(values, "(none)")
   defp include_none_result?({:not_member, values}), do: !Enum.member?(values, "(none)")
+  defp include_none_result?({:matches, _}), do: false
   defp include_none_result?(_), do: true
 
   defp breakdown_sessions(_, _, _, [], _), do: []

test/plausible_web/controllers/api/stats_controller/custom_prop_breakdown_test.exs

@@ -1046,6 +1046,42 @@ defmodule PlausibleWeb.Api.StatsController.CustomPropBreakdownTest do
                }
              ]
     end
+
+    test "returns prop-breakdown with a prop_value matching filter", %{
+      conn: conn,
+      site: site
+    } do
+      populate_stats(site, [
+        build(:pageview, "meta.key": ["key"], "meta.value": ["foo"]),
+        build(:pageview, "meta.key": ["key"], "meta.value": ["bar"]),
+        build(:pageview, "meta.key": ["key"], "meta.value": ["bar"]),
+        build(:pageview, "meta.key": ["key"], "meta.value": ["foobar"]),
+        build(:pageview)
+      ])
+
+      filters = Jason.encode!(%{props: %{key: "~bar"}})
+
+      conn =
+        get(
+          conn,
+          "/api/stats/#{site.domain}/custom-prop-values/key?period=day&filters=#{filters}"
+        )
+
+      assert json_response(conn, 200) == [
+               %{
+                 "visitors" => 2,
+                 "name" => "bar",
+                 "events" => 2,
+                 "percentage" => 66.7
+               },
+               %{
+                 "visitors" => 1,
+                 "name" => "foobar",
+                 "events" => 1,
+                 "percentage" => 33.3
+               }
+             ]
+    end
   end
 
   describe "GET /api/stats/:domain/custom-prop-values/:prop_key - for a Growth subscription" do

test/plausible_web/controllers/api/stats_controller/pages_test.exs

@@ -75,6 +75,43 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
              ]
     end
 
+    test "returns top pages with :matches filter on custom pageview props", %{
+      conn: conn,
+      site: site
+    } do
+      populate_stats(site, [
+        build(:pageview,
+          pathname: "/1",
+          "meta.key": ["prop"],
+          "meta.value": ["bar"]
+        ),
+        build(:pageview,
+          pathname: "/2",
+          "meta.key": ["prop"],
+          "meta.value": ["foobar"]
+        ),
+        build(:pageview,
+          pathname: "/3",
+          "meta.key": ["prop"],
+          "meta.value": ["baar"]
+        ),
+        build(:pageview,
+          pathname: "/4",
+          "meta.key": ["another"],
+          "meta.value": ["bar"]
+        ),
+        build(:pageview, pathname: "/5")
+      ])
+
+      filters = Jason.encode!(%{props: %{"prop" => "~bar"}})
+      conn = get(conn, "/api/stats/#{site.domain}/pages?period=day&filters=#{filters}")
+
+      assert json_response(conn, 200) == [
+               %{"visitors" => 1, "name" => "/1"},
+               %{"visitors" => 1, "name" => "/2"}
+             ]
+    end
+
     test "calculates bounce_rate and time_on_page with :is filter on custom pageview props",
          %{conn: conn, site: site} do
       populate_stats(site, [