Skip to content

Commit ee6d266

Browse files
svenklemmtimescale-automation
svenklemm
authored and
timescale-automation
committed
Disable decompression limit during cagg refresh
We dont want continuous aggregate refresh to fail because of decompression limit so disable limit for the refresh job. (cherry picked from commit ff85c5e)
1 parent caee1bb commit ee6d266

File tree

5 files changed

+86
-13
lines changed

5 files changed

+86
-13
lines changed

‎.unreleased/pr_8171‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Fixes: #8171 Disable decompression limit during continuous aggregate refresh

‎tsl/src/continuous_aggs/refresh.c‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -805,6 +805,17 @@ continuous_agg_refresh_internal(const ContinuousAgg *cagg,
805805
* prevent transaction blocks. */
806806
PreventInTransactionBlock(nonatomic, REFRESH_FUNCTION_NAME);
807807

808+
/*
809+
* We don't cagg refresh to fail because of decompression limit. So disable
810+
* the decompression limit for the duration of the refresh.
811+
*/
812+
const char *old_decompression_limit =
813+
GetConfigOption("timescaledb.max_tuples_decompressed_per_dml_transaction", false, false);
814+
SetConfigOption("timescaledb.max_tuples_decompressed_per_dml_transaction",
815+
"0",
816+
PGC_USERSET,
817+
PGC_S_SESSION);
818+
808819
/* Connect to SPI manager due to the underlying SPI calls */
809820
int rc = SPI_connect_ext(SPI_OPT_NONATOMIC);
810821
if (rc != SPI_OK_CONNECT)
@@ -920,6 +931,11 @@ continuous_agg_refresh_internal(const ContinuousAgg *cagg,
920931
/* Restore search_path */
921932
AtEOXact_GUC(false, save_nestlevel);
922933

934+
SetConfigOption("timescaledb.max_tuples_decompressed_per_dml_transaction",
935+
old_decompression_limit,
936+
PGC_USERSET,
937+
PGC_S_SESSION);
938+
923939
rc = SPI_finish();
924940
if (rc != SPI_OK_FINISH)
925941
elog(ERROR, "SPI_finish failed: %s", SPI_result_code_string(rc));

‎tsl/test/expected/cagg_refresh.out‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -600,3 +600,25 @@ EXECUTE FUNCTION refresh_cagg_trigger_fun();
600600
INSERT INTO refresh_cagg_trigger_table VALUES(1);
601601
psql:include/cagg_refresh_common.sql:407: ERROR: refresh_continuous_aggregate() cannot be executed from a function
602602
\set ON_ERROR_STOP 1
603+
-- check that cagg refresh is not blocked by decompression limit
604+
CREATE TABLE conditions_decompress_limit (time timestamptz NOT NULL, device text, temp float) WITH (tsdb.hypertable, tsdb.partition_column='time');
605+
INSERT INTO conditions_decompress_limit SELECT '2020-01-01','d' || i::text, 1.0 FROM generate_series(1,100) g(i);
606+
CREATE MATERIALIZED VIEW daily_temp_decompress_limit WITH (tsdb.continuous) AS SELECT time_bucket('1 day', time) AS day, device, avg(temp) AS avg_temp FROM conditions_decompress_limit GROUP BY 1,2 WITH NO DATA;
607+
ALTER MATERIALIZED VIEW daily_temp_decompress_limit SET (tsdb.columnstore,tsdb.segmentby = 'device');
608+
psql:include/cagg_refresh_common.sql:415: NOTICE: defaulting compress_orderby to day
609+
CALL refresh_continuous_aggregate('daily_temp_decompress_limit', NULL, NULL);
610+
SELECT compress_chunk(show_chunks('daily_temp_decompress_limit'));
611+
compress_chunk
612+
------------------------------------------
613+
_timescaledb_internal._hyper_17_26_chunk
614+
(1 row)
615+
616+
INSERT INTO conditions_decompress_limit SELECT '2020-01-01','d' || i::text, 2.0 FROM generate_series(1,100) g(i);
617+
SET timescaledb.max_tuples_decompressed_per_dml_transaction TO 1;
618+
CALL refresh_continuous_aggregate('daily_temp_decompress_limit', NULL, NULL);
619+
SHOW timescaledb.max_tuples_decompressed_per_dml_transaction;
620+
timescaledb.max_tuples_decompressed_per_dml_transaction
621+
---------------------------------------------------------
622+
1
623+
(1 row)
624+

‎tsl/test/expected/cagg_refresh_using_merge.out‎

Lines changed: 35 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -601,6 +601,28 @@ EXECUTE FUNCTION refresh_cagg_trigger_fun();
601601
INSERT INTO refresh_cagg_trigger_table VALUES(1);
602602
psql:include/cagg_refresh_common.sql:407: ERROR: refresh_continuous_aggregate() cannot be executed from a function
603603
\set ON_ERROR_STOP 1
604+
-- check that cagg refresh is not blocked by decompression limit
605+
CREATE TABLE conditions_decompress_limit (time timestamptz NOT NULL, device text, temp float) WITH (tsdb.hypertable, tsdb.partition_column='time');
606+
INSERT INTO conditions_decompress_limit SELECT '2020-01-01','d' || i::text, 1.0 FROM generate_series(1,100) g(i);
607+
CREATE MATERIALIZED VIEW daily_temp_decompress_limit WITH (tsdb.continuous) AS SELECT time_bucket('1 day', time) AS day, device, avg(temp) AS avg_temp FROM conditions_decompress_limit GROUP BY 1,2 WITH NO DATA;
608+
ALTER MATERIALIZED VIEW daily_temp_decompress_limit SET (tsdb.columnstore,tsdb.segmentby = 'device');
609+
psql:include/cagg_refresh_common.sql:415: NOTICE: defaulting compress_orderby to day
610+
CALL refresh_continuous_aggregate('daily_temp_decompress_limit', NULL, NULL);
611+
SELECT compress_chunk(show_chunks('daily_temp_decompress_limit'));
612+
compress_chunk
613+
------------------------------------------
614+
_timescaledb_internal._hyper_17_26_chunk
615+
(1 row)
616+
617+
INSERT INTO conditions_decompress_limit SELECT '2020-01-01','d' || i::text, 2.0 FROM generate_series(1,100) g(i);
618+
SET timescaledb.max_tuples_decompressed_per_dml_transaction TO 1;
619+
CALL refresh_continuous_aggregate('daily_temp_decompress_limit', NULL, NULL);
620+
SHOW timescaledb.max_tuples_decompressed_per_dml_transaction;
621+
timescaledb.max_tuples_decompressed_per_dml_transaction
622+
---------------------------------------------------------
623+
1
624+
(1 row)
625+
604626
-- Additional tests for MERGE refresh
605627
DROP TABLE conditions CASCADE;
606628
NOTICE: drop cascades to 10 other objects
@@ -646,7 +668,7 @@ WITH NO DATA;
646668
SET client_min_messages TO LOG;
647669
CALL refresh_continuous_aggregate('conditions_daily', NULL, '2018-11-01 23:59:59-08');
648670
LOG: statement: CALL refresh_continuous_aggregate('conditions_daily', NULL, '2018-11-01 23:59:59-08');
649-
LOG: inserted 5 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_16"
671+
LOG: inserted 5 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_20"
650672
SELECT * FROM conditions_daily ORDER BY 1, 2, 3 NULLS LAST, 4 NULLS LAST, 5 NULLS LAST;
651673
LOG: statement: SELECT * FROM conditions_daily ORDER BY 1, 2, 3 NULLS LAST, 4 NULLS LAST, 5 NULLS LAST;
652674
bucket | location | avg | max | min
@@ -661,7 +683,7 @@ LOG: statement: SELECT * FROM conditions_daily ORDER BY 1, 2, 3 NULLS LAST, 4 N
661683
-- Second refresh using MERGE should also fall back to INSERT since there's no data in the materialization hypertable
662684
CALL refresh_continuous_aggregate('conditions_daily', '2018-11-01', NULL);
663685
LOG: statement: CALL refresh_continuous_aggregate('conditions_daily', '2018-11-01', NULL);
664-
LOG: inserted 2 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_16"
686+
LOG: inserted 2 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_20"
665687
SELECT * FROM conditions_daily ORDER BY 1, 2, 3 NULLS LAST, 4 NULLS LAST, 5 NULLS LAST;
666688
LOG: statement: SELECT * FROM conditions_daily ORDER BY 1, 2, 3 NULLS LAST, 4 NULLS LAST, 5 NULLS LAST;
667689
bucket | location | avg | max | min
@@ -698,8 +720,8 @@ LOG: statement: UPDATE conditions SET humidity = humidity + 100 WHERE time = '2
698720
-- Shoudn't affect the materialization hypertable (merged=0 and deleted=0)
699721
CALL refresh_continuous_aggregate('conditions_daily', NULL, NULL);
700722
LOG: statement: CALL refresh_continuous_aggregate('conditions_daily', NULL, NULL);
701-
LOG: merged 0 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_16"
702-
LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_16"
723+
LOG: merged 0 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_20"
724+
LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_20"
703725
SELECT * FROM conditions_daily ORDER BY 1, 2, 3 NULLS LAST, 4 NULLS LAST, 5 NULLS LAST;
704726
LOG: statement: SELECT * FROM conditions_daily ORDER BY 1, 2, 3 NULLS LAST, 4 NULLS LAST, 5 NULLS LAST;
705727
bucket | location | avg | max | min
@@ -727,7 +749,7 @@ VALUES
727749
-- There's no data in the affected range so the refresh should fall back to INSERT
728750
CALL refresh_continuous_aggregate('conditions_daily', NULL, NULL);
729751
LOG: statement: CALL refresh_continuous_aggregate('conditions_daily', NULL, NULL);
730-
LOG: inserted 3 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_16"
752+
LOG: inserted 3 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_20"
731753
SELECT * FROM conditions_daily ORDER BY 1, 2, 3 NULLS LAST, 4 NULLS LAST, 5 NULLS LAST;
732754
LOG: statement: SELECT * FROM conditions_daily ORDER BY 1, 2, 3 NULLS LAST, 4 NULLS LAST, 5 NULLS LAST;
733755
bucket | location | avg | max | min
@@ -750,8 +772,8 @@ LOG: statement: UPDATE conditions SET temperature = temperature + 100 WHERE tim
750772
-- Should merge 1 bucket (merged=1 and deleted=0)
751773
CALL refresh_continuous_aggregate('conditions_daily', NULL, NULL);
752774
LOG: statement: CALL refresh_continuous_aggregate('conditions_daily', NULL, NULL);
753-
LOG: merged 1 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_16"
754-
LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_16"
775+
LOG: merged 1 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_20"
776+
LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_20"
755777
SELECT * FROM conditions_daily ORDER BY 1, 2, 3 NULLS LAST, 4 NULLS LAST, 5 NULLS LAST;
756778
LOG: statement: SELECT * FROM conditions_daily ORDER BY 1, 2, 3 NULLS LAST, 4 NULLS LAST, 5 NULLS LAST;
757779
bucket | location | avg | max | min
@@ -774,8 +796,8 @@ LOG: statement: DELETE FROM conditions WHERE time >= '2018-11-02' AND time < '2
774796
-- Should not merge any bucket but delete one bucket (merged=0 and deleted=1)
775797
CALL refresh_continuous_aggregate('conditions_daily', NULL, NULL);
776798
LOG: statement: CALL refresh_continuous_aggregate('conditions_daily', NULL, NULL);
777-
LOG: merged 0 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_16"
778-
LOG: deleted 1 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_16"
799+
LOG: merged 0 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_20"
800+
LOG: deleted 1 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_20"
779801
SELECT * FROM conditions_daily ORDER BY 1, 2, 3 NULLS LAST, 4 NULLS LAST, 5 NULLS LAST;
780802
LOG: statement: SELECT * FROM conditions_daily ORDER BY 1, 2, 3 NULLS LAST, 4 NULLS LAST, 5 NULLS LAST;
781803
bucket | location | avg | max | min
@@ -856,7 +878,7 @@ SET client_min_messages TO LOG;
856878
LOG: statement: SET client_min_messages TO LOG;
857879
CALL refresh_continuous_aggregate('conditions_nullable_daily', NULL, '2018-11-01 23:59:59-08');
858880
LOG: statement: CALL refresh_continuous_aggregate('conditions_nullable_daily', NULL, '2018-11-01 23:59:59-08');
859-
LOG: inserted 2 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_18"
881+
LOG: inserted 2 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_22"
860882
RESET client_min_messages;
861883
LOG: statement: RESET client_min_messages;
862884
SELECT * FROM conditions_nullable_daily ORDER BY 1, 2 NULLS LAST, 3 NULLS LAST;
@@ -875,8 +897,8 @@ VALUES
875897
SET client_min_messages TO LOG;
876898
CALL refresh_continuous_aggregate('conditions_nullable_daily', NULL, '2018-11-01 23:59:59-08');
877899
LOG: statement: CALL refresh_continuous_aggregate('conditions_nullable_daily', NULL, '2018-11-01 23:59:59-08');
878-
LOG: merged 2 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_18"
879-
LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_18"
900+
LOG: merged 2 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_22"
901+
LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_22"
880902
RESET client_min_messages;
881903
LOG: statement: RESET client_min_messages;
882904
SELECT * FROM conditions_nullable_daily ORDER BY 1, 2 NULLS LAST, 3 NULLS LAST;
@@ -889,5 +911,5 @@ SELECT * FROM conditions_nullable_daily ORDER BY 1, 2 NULLS LAST, 3 NULLS LAST;
889911
(4 rows)
890912

891913
DROP MATERIALIZED VIEW conditions_nullable_daily;
892-
NOTICE: drop cascades to table _timescaledb_internal._hyper_18_32_chunk
914+
NOTICE: drop cascades to table _timescaledb_internal._hyper_22_35_chunk
893915
DROP TABLE conditions CASCADE;

‎tsl/test/sql/include/cagg_refresh_common.sql‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -407,3 +407,15 @@ EXECUTE FUNCTION refresh_cagg_trigger_fun();
407407
INSERT INTO refresh_cagg_trigger_table VALUES(1);
408408

409409
\set ON_ERROR_STOP 1
410+
411+
-- check that cagg refresh is not blocked by decompression limit
412+
CREATE TABLE conditions_decompress_limit (time timestamptz NOT NULL, device text, temp float) WITH (tsdb.hypertable, tsdb.partition_column='time');
413+
INSERT INTO conditions_decompress_limit SELECT '2020-01-01','d' || i::text, 1.0 FROM generate_series(1,100) g(i);
414+
CREATE MATERIALIZED VIEW daily_temp_decompress_limit WITH (tsdb.continuous) AS SELECT time_bucket('1 day', time) AS day, device, avg(temp) AS avg_temp FROM conditions_decompress_limit GROUP BY 1,2 WITH NO DATA;
415+
ALTER MATERIALIZED VIEW daily_temp_decompress_limit SET (tsdb.columnstore,tsdb.segmentby = 'device');
416+
CALL refresh_continuous_aggregate('daily_temp_decompress_limit', NULL, NULL);
417+
SELECT compress_chunk(show_chunks('daily_temp_decompress_limit'));
418+
INSERT INTO conditions_decompress_limit SELECT '2020-01-01','d' || i::text, 2.0 FROM generate_series(1,100) g(i);
419+
SET timescaledb.max_tuples_decompressed_per_dml_transaction TO 1;
420+
CALL refresh_continuous_aggregate('daily_temp_decompress_limit', NULL, NULL);
421+
SHOW timescaledb.max_tuples_decompressed_per_dml_transaction;

0 commit comments

Comments
 (0)