🌈 adding marker props (#148)

* ✨ first draft

* 🧹 cleaning up

* 💪 adding tests

* 🔍

* 🙈

* 🖊️ review code

* 🔆 fix sub attribute assignment

This commit now allows sub-attribute assignment via hf_data_container

* 🗽 add recursive dict attrib. assignment

* 🖊️ review

* 😅 updating readme

* 🙈 improving replace functionality

* 🙈 add gap handler to serialization

* 🔥 add marker size to basic example

* 🙈 fix linting

* refactor: improve QOL of serialization tests

* 🔍 final review

---------

Co-authored-by: Jeroen Van Der Donckt <[email protected]>
Co-authored-by: Jeroen Van Der Donckt <[email protected]>

Author: jonasvdd

README.md

@@ -158,7 +158,7 @@ Paper (preprint): https://arxiv.org/abs/2206.08703
 ## Future work 🔨
 
 - [x] Support `.add_traces()` (currently only `.add_trace` is supported)
-- [ ] Support `hf_color` and `hf_markersize`, see [#148](https://github.com/predict-idlab/plotly-resampler/pull/148)
+- [x] Support `hf_color` and `hf_markersize`, see [#148](https://github.com/predict-idlab/plotly-resampler/pull/148)
 - [x] Integrate with [tsdownsample](https://github.com/predict-idlab/tsdownsample) :racehorse:
 
 <br>

examples/README.md

@@ -17,6 +17,7 @@ The [basic example notebook](basic_example.ipynb) covers most use-cases in which
 
 Additionally, this notebook also shows some more advanced functionalities, such as:
 * Retaining (a static) plotly-resampler figure in your notebook
+* Showing how to style the marker color and size of plotly-resampler figures
 * Adjusting trace data of plotly-resampler figures at runtime
 * How to add (shaded) confidence bounds to your time series
 * The flexibility of configuring different aggregation-algorithms and number of shown samples per trace

examples/basic_example.ipynb

@@ -57,7 +57,9 @@ def plot_graph(n_clicks):
     if len(ctx.triggered) and "plot-button" in ctx.triggered[0]["prop_id"]:
         # Note how the replace method is used here on the global figure object
         global fig
-        fig.replace(go.Figure())
+        if len(fig.data):
+            # Replace the figure with an empty one to clear the graph
+            fig.replace(go.Figure())
         fig.add_trace(go.Scattergl(name="log"), hf_x=x, hf_y=noisy_sin * 0.9999995**x)
         fig.add_trace(go.Scattergl(name="exp"), hf_x=x, hf_y=noisy_sin * 1.000002**x)
         return fig

examples/dash_apps/01_minimal_global.py

@@ -31,7 +31,14 @@
 from ..aggregation.plotly_aggregator_parser import PlotlyAggregatorParser
 from .utils import round_number_str, round_td_str
 
-_hf_data_container = namedtuple("DataContainer", ["x", "y", "text", "hovertext"])
+# A high-frequency data container
+# NOTE: the attributes must all be valid trace attributes, with attribute levels
+# separated by an '_' (e.g., 'marker_color' is valid) as the
+# `_hf_data_container._asdict()` function is used in
+#  `AbstractFigureAggregator._construct_hf_data_dict`.
+_hf_data_container = namedtuple(
+    "DataContainer", ["x", "y", "text", "hovertext", "marker_size", "marker_color"]
+)
 
 
 class AbstractFigureAggregator(BaseFigure, ABC):
@@ -355,14 +362,22 @@ def _check_update_trace_data(
             hf_trace_data, end_idx - start_idx, agg_x
         )
 
+        def _nest_dict_rec(k: str, v: any, out: dict) -> None:
+            """Recursively nest a dict based on the key whose '_' indicates level."""
+            k, *rest = k.split("_", 1)
+            if rest:
+                _nest_dict_rec(rest[0], v, out.setdefault(k, {}))
+            else:
+                out[k] = v
+
         # Check if (hover)text also needs to be downsampled
-        for k in ["text", "hovertext"]:
+        for k in ["text", "hovertext", "marker_size", "marker_color"]:
             k_val = hf_trace_data.get(k)
             if isinstance(k_val, (np.ndarray, pd.Series)):
                 assert isinstance(
                     hf_trace_data["downsampler"], DataPointSelector
                 ), "Only DataPointSelector can downsample non-data trace array props."
-                trace[k] = k_val[start_idx + indices]
+                _nest_dict_rec(k, k_val[start_idx + indices], trace)
             elif k_val is not None:
                 trace[k] = k_val
 
@@ -535,6 +550,8 @@ def _parse_get_trace_props(
         hf_y: Iterable = None,
         hf_text: Iterable = None,
         hf_hovertext: Iterable = None,
+        hf_marker_size: Iterable = None,
+        hf_marker_color: Iterable = None,
         check_nans: bool = True,
     ) -> _hf_data_container:
         """Parse and capture the possibly high-frequency trace-props in a datacontainer.
@@ -601,6 +618,26 @@ def _parse_get_trace_props(
             else None
         )
 
+        hf_marker_size = (
+            trace["marker"]["size"]
+            if (
+                hf_marker_size is None
+                and hasattr(trace, "marker")
+                and "size" in trace["marker"]
+            )
+            else hf_marker_size
+        )
+
+        hf_marker_color = (
+            trace["marker"]["color"]
+            if (
+                hf_marker_color is None
+                and hasattr(trace, "marker")
+                and "color" in trace["marker"]
+            )
+            else hf_marker_color
+        )
+
         if trace["type"].lower() in self._high_frequency_traces:
             if hf_x is None:  # if no data as x or hf_x is passed
                 if hf_y.ndim != 0:  # if hf_y is an array
@@ -687,8 +724,15 @@ def _parse_get_trace_props(
 
             if hasattr(trace, "hovertext"):
                 trace["hovertext"] = hf_hovertext
-
-        return _hf_data_container(hf_x, hf_y, hf_text, hf_hovertext)
+            if hasattr(trace, "marker"):
+                if hasattr(trace.marker, "size"):
+                    trace.marker.size = hf_marker_size
+                if hasattr(trace.marker, "color"):
+                    trace.marker.color = hf_marker_color
+
+        return _hf_data_container(
+            hf_x, hf_y, hf_text, hf_hovertext, hf_marker_size, hf_marker_color
+        )
 
     def _construct_hf_data_dict(
         self,
@@ -800,6 +844,8 @@ def add_trace(
         hf_y: Iterable = None,
         hf_text: Union[str, Iterable] = None,
         hf_hovertext: Union[str, Iterable] = None,
+        hf_marker_size: Union[str, Iterable] = None,
+        hf_marker_color: Union[str, Iterable] = None,
         check_nans: bool = True,
         **trace_kwargs,
     ):
@@ -850,6 +896,12 @@ def add_trace(
         hf_hovertext: Iterable, optional
             The original high frequency hovertext. If set, this has priority over the
             trace its ```hovertext`` argument.
+        hf_marker_size: Iterable, optional
+            The original high frequency marker size. If set, this has priority over the
+            trace its ``marker.size`` argument.
+        hf_marker_color: Iterable, optional
+            The original high frequency marker color. If set, this has priority over the
+            trace its ``marker.color`` argument.
         check_nans: boolean, optional
             If set to True, the trace's data will be checked for NaNs - which will be
             removed. By default True.
@@ -929,7 +981,14 @@ def add_trace(
         # construct the hf_data_container
         # TODO in future version -> maybe regex on kwargs which start with `hf_`
         dc = self._parse_get_trace_props(
-            trace, hf_x, hf_y, hf_text, hf_hovertext, check_nans
+            trace,
+            hf_x,
+            hf_y,
+            hf_text,
+            hf_hovertext,
+            hf_marker_size,
+            hf_marker_color,
+            check_nans,
         )
 
         # These traces will determine the autoscale its RANGE!
@@ -955,6 +1014,8 @@ def add_trace(
                 # We copy (by reference) all the non-data properties of the trace in
                 # the new trace.
                 trace = trace._props  # convert the trace into a dict
+                # NOTE: there is no need to store `marker` property here.
+                # If needed, it will be added  to `trace` via `check_update_trace_data`
                 trace = {
                     k: trace[k] for k in set(trace.keys()).difference(set(dc._fields))
                 }
@@ -970,12 +1031,12 @@ def add_trace(
                 )
             else:
                 self._print(f"[i] NOT resampling {trace['name']} - len={n_samples}")
-                for k in dc._fields:
-                    setattr(trace, k, getattr(dc, k))
+                trace._process_kwargs(**{k: getattr(dc, k) for k in dc._fields})
                 return super(AbstractFigureAggregator, self).add_traces(
                     [trace], **self._add_trace_to_add_traces_kwargs(trace_kwargs)
                 )
-        return super(AbstractFigureAggregator, self).add_traces(
+
+        return super(self._figure_class, self).add_traces(
             [trace], **self._add_trace_to_add_traces_kwargs(trace_kwargs)
         )
 
@@ -1184,7 +1245,10 @@ def replace(self, figure: go.Figure, convert_existing_traces: bool = True):
             convert_existing_traces=convert_existing_traces,
             default_n_shown_samples=self._global_n_shown_samples,
             default_downsampler=self._global_downsampler,
+            default_gap_handler=self._global_gap_handler,
             resampled_trace_prefix_suffix=(self._prefix, self._suffix),
+            show_mean_aggregation_size=self._show_mean_aggregation_size,
+            verbose=self._print_verbose,
         )
 
     def _parse_relayout(self, relayout_dict: dict) -> dict:
@@ -1299,7 +1363,7 @@ def construct_update_data(
         layout_traces_list: List[dict] = [relayout_data]
 
         # 2. Create the additional trace data for the frond-end
-        relevant_keys = list(_hf_data_container._fields) + ["name"]
+        relevant_keys = list(_hf_data_container._fields) + ["name", "marker"]
         # Note that only updated trace-data will be sent to the client
         for idx in updated_trace_indices:
             trace = current_graph["data"][idx]
@@ -1355,6 +1419,7 @@ def _get_pr_props_keys(self) -> List[str]:
             "_prefix",
             "_suffix",
             "_global_downsampler",
+            "_global_gap_handler",
         ]
 
     def __reduce__(self):

plotly_resampler/figure_resampler/figure_resampler_interface.py

@@ -17,7 +17,7 @@
 from selenium.webdriver.common.by import By
 
 from plotly_resampler import LTTB, EveryNthPoint, FigureResampler
-from plotly_resampler.aggregation import PlotlyAggregatorParser
+from plotly_resampler.aggregation import NoGapHandler, PlotlyAggregatorParser
 
 # Note: this will be used to skip / alter behavior when running browser tests on
 # non-linux platforms.
@@ -329,6 +329,30 @@ def test_replace_figure(float_series):
     assert len(go_fig.data[0]["x"]) == len(float_series)
 
 
+def test_replace_properties(float_series):
+    resampled_trace_prefix_suffix = ("a", "b")
+    verbose = True
+    default_n_shown_samples = 1050
+    default_gap_handler = NoGapHandler()
+    default_downsampler = EveryNthPoint()
+    fr_fig = FigureResampler(
+        default_n_shown_samples=default_n_shown_samples,
+        verbose=verbose,
+        resampled_trace_prefix_suffix=resampled_trace_prefix_suffix,
+        default_gap_handler=default_gap_handler,
+        default_downsampler=default_downsampler,
+    )
+
+    fr_fig.add_trace(go.Scattergl(x=float_series.index, y=float_series, name="fs"))
+    fr_fig.replace(go.Figure())
+
+    assert fr_fig._global_n_shown_samples == default_n_shown_samples
+    assert fr_fig._print_verbose == verbose
+    assert (fr_fig._prefix, fr_fig._suffix) == resampled_trace_prefix_suffix
+    assert fr_fig._global_gap_handler == default_gap_handler
+    assert fr_fig._global_downsampler == default_downsampler
+
+
 def test_nan_removed_input(float_series):
     # see: https://plotly.com/python/subplots/#custom-sized-subplot-with-subplot-titles
     base_fig = make_subplots(
@@ -1559,3 +1583,84 @@ def test_fr_copy_grid():
     assert fr._grid_ref == f_dict.get("_grid_ref")
     assert fr._grid_str is not None
     assert fr._grid_str == f_dict.get("_grid_str")
+
+
+# Testing HF marker_size and color arguments
+def test_hf_marker_size_hf_args():
+    # create dummy data
+    n = 100_000
+    y = np.sin(np.arange(n) / 2_000) + np.random.randn(n) / 10
+
+    # construct the figure via hf kwargs
+    fr = FigureResampler()
+    fr.add_trace(
+        go.Scattergl(mode="markers"),
+        hf_y=y,
+        hf_marker_size=(3 + 20 * np.abs(y)).astype(int),
+        hf_marker_color=np.abs(y) / np.max(np.abs(y)),
+    )
+
+    # Perform asserts on the hf_data part of the figure
+    hf_trace = fr.hf_data[0]
+    assert "marker_size" in hf_trace
+    assert "marker_color" in hf_trace
+
+    assert len(hf_trace["marker_size"] == len(y))
+    assert len(hf_trace["marker_color"] == len(y))
+
+    # perform some asserts on the to-be constructed update data
+    update_trace = fr.construct_update_data(
+        {"xaxis.autorange": True, "xaxis.showspikes": True}
+    )[1]
+
+    assert all(k in update_trace for k in ["x", "y", "name", "marker", "index"])
+
+    # check whether the marker size and marker color are available
+    assert all(k in update_trace["marker"] for k in ["size", "color"])
+    assert len(update_trace["marker"]["size"]) == len(update_trace["x"])
+    assert np.allclose(
+        update_trace["marker"]["color"],
+        (np.abs(update_trace["y"]) / np.max(np.abs(y))),
+        rtol=1e-3,
+    )
+
+
+def test_hf_marker_size_plotly_args():
+    # create dummy data
+    n = 100_000
+    y = np.sin(np.arange(n) / 2_000) + np.random.randn(n) / 10
+
+    # construct the figure via hf kwargs
+    fr = FigureResampler()
+    fr.add_trace(
+        go.Scattergl(
+            mode="markers",
+            marker_size=(3 + 20 * np.abs(y)).astype(int),
+            marker_color=np.abs(y) / np.max(np.abs(y)),
+        ),
+        hf_y=y,
+    )
+
+    # Perform asserts on the hf_data part of the figure
+    hf_trace = fr.hf_data[0]
+    assert "marker_size" in hf_trace
+    assert "marker_color" in hf_trace
+
+    assert len(hf_trace["marker_size"] == len(y))
+    assert len(hf_trace["marker_color"] == len(y))
+
+    # perform some asserts on the to-be constructed update data
+    update_trace = fr.construct_update_data(
+        {"xaxis.autorange": True, "xaxis.showspikes": True}
+    )[1]
+
+    assert all(k in update_trace for k in ["x", "y", "name", "marker", "index"])
+
+    # check whether the marker size and marker color are available
+    assert all(k in update_trace["marker"] for k in ["size", "color"])
+    assert len(update_trace["marker"]["size"]) == len(update_trace["x"])
+    assert np.allclose(
+        update_trace["marker"]["color"],
+        (np.abs(update_trace["y"]) / np.max(np.abs(y))),
+        rtol=1e-3,
+    )