Improve job stat history retention policy

Previously a delete query was run on the `bgw_job_stat_history` table to
remove job history. This could be slow once the job history table became
very large. We now use a temporary table to keep recent history and
truncate the job history table instead. We also introduce a
`job_history_bsearch` function which finds the ids of the jobs that need
to be deleted using binary search.

This PR is based on experiments done
previously (timescale/timescaledb-extras#47).

Author: kpan2034

.unreleased/pr_8494

@@ -0,0 +1 @@
+Implements: #8494 Improve job stat history retention policy

sql/job_stat_history_log_retention.sql

@@ -4,23 +4,110 @@
 
 -- A retention policy is set up for the table _timescaledb_internal.job_errors (Error Log Retention Policy [2])
 -- By default, it will run once a month and and drop rows older than a month.
+
+-- We use binary search on the id column to figure out which rows should be retained from the job history table
+-- Doing it this way allows us to use the index on the `id` column, which (empirically) we found is faster than querying on the `execution_finish` column directly without an index
+-- This works because `execution_finish` is always ordered.
+-- We can consider alternative approaches to simplify this in the future.
+CREATE OR REPLACE FUNCTION _timescaledb_functions.job_history_bsearch(search_point TIMESTAMPTZ) RETURNS BIGINT
+AS
+$$
+DECLARE
+  id_lower BIGINT;
+  id_upper BIGINT;
+  id_middle BIGINT DEFAULT 0;
+  target_tz TIMESTAMPTZ;
+BEGIN
+
+  SELECT COALESCE(min(id), 0), COALESCE(max(id), 0)
+  INTO id_lower, id_upper
+  FROM _timescaledb_internal.bgw_job_stat_history;
+
+  IF id_lower = 0 AND id_upper = 0 THEN
+    RETURN NULL;
+  END IF;
+
+  -- We want the first entry in the table where execution_finish is >= search_point
+  WHILE id_lower < id_upper LOOP
+    id_middle := id_lower + (id_upper - id_lower) / 2;
+
+    SELECT execution_finish
+    INTO target_tz
+    FROM _timescaledb_internal.bgw_job_stat_history
+    WHERE id = id_middle;
+
+    -- If the id_middle is not found, shift to a previous id that's still in the search space
+    IF NOT FOUND THEN
+      SELECT execution_finish, id
+      INTO target_tz, id_middle
+      FROM _timescaledb_internal.bgw_job_stat_history
+      WHERE id <= id_middle AND id >= id_lower
+      ORDER BY id LIMIT 1;
+
+      IF NOT FOUND THEN
+        id_middle := id_lower;
+      END IF;
+
+    END IF;
+
+    IF target_tz >= search_point THEN
+      id_upper := id_middle;
+    ELSE
+      id_lower := id_middle + 1;
+    END IF;
+  END LOOP;
+
+  -- Handle the case where no ids need to be deleted and return NULL instead
+  SELECT execution_finish
+  INTO target_tz
+  FROM _timescaledb_internal.bgw_job_stat_history
+  WHERE id = id_lower;
+
+  IF target_tz < search_point THEN
+    RETURN NULL;
+  END IF;
+
+  RETURN id_lower;
+END
+$$
+LANGUAGE plpgsql SET search_path TO pg_catalog, pg_temp;
+
 CREATE OR REPLACE FUNCTION _timescaledb_functions.policy_job_stat_history_retention(job_id integer, config JSONB) RETURNS integer
 LANGUAGE PLPGSQL AS
 $BODY$
 DECLARE
-    drop_after INTERVAL;
     numrows INTEGER;
+    search_point TIMESTAMPTZ;
+    id_found BIGINT;
 BEGIN
-    drop_after := config->>'drop_after';
+  PERFORM set_config('lock_timeout', coalesce(config->>'lock_timeout', '5s'), true /* is local */);
 
-    DELETE
-    FROM _timescaledb_internal.bgw_job_stat_history
-    WHERE execution_finish < (now() - drop_after);
+  -- We need to prevent concurrent changes on this table when running this retention job
+  -- We take an AccessExclusiveLock at the start since we TRUNCATE later
+  LOCK TABLE _timescaledb_internal.bgw_job_stat_history IN ACCESS EXCLUSIVE MODE;
+
+  search_point := now() - (config->>'drop_after')::interval;
+
+  id_found := _timescaledb_functions.job_history_bsearch(search_point);
+
+  IF id_found IS NULL THEN
+    RETURN 0;
+  END IF;
+
+  CREATE TEMP TABLE __tmp_bgw_job_stat_history ON COMMIT DROP AS
+  SELECT * FROM _timescaledb_internal.bgw_job_stat_history
+  WHERE id >= id_found
+  ORDER BY id;
+
+  TRUNCATE _timescaledb_internal.bgw_job_stat_history;
+
+  INSERT INTO _timescaledb_internal.bgw_job_stat_history
+  SELECT * FROM __tmp_bgw_job_stat_history;
 
-    GET DIAGNOSTICS numrows = ROW_COUNT;
+  GET DIAGNOSTICS numrows = ROW_COUNT;
 
-    RETURN numrows;
-END;
+  RETURN numrows;
+END
 $BODY$ SET search_path TO pg_catalog, pg_temp;
 
 CREATE OR REPLACE FUNCTION _timescaledb_functions.policy_job_stat_history_retention_check(config JSONB) RETURNS VOID
@@ -34,7 +121,7 @@ BEGIN
     IF config->>'drop_after' IS NULL THEN
         RAISE EXCEPTION 'drop_after interval not provided';
     END IF ;
-END;
+END
 $BODY$ SET search_path TO pg_catalog, pg_temp;
 
 INSERT INTO _timescaledb_config.bgw_job (

sql/updates/reverse-dev.sql

@@ -252,3 +252,4 @@ ALTER EXTENSION timescaledb DROP TABLE _timescaledb_catalog.continuous_aggs_mate
 
 DROP TABLE IF EXISTS _timescaledb_catalog.continuous_aggs_materialization_ranges;
 
+DROP FUNCTION _timescaledb_functions.job_history_bsearch(TIMESTAMPTZ);