Add gapfill query support

This patch adds first level support for gap fill queries, including
support for LOCF (last observation carried forward) and interpolation, without
requiring to join against `generate_series`. This makes it easier to join
timeseries with different or irregular sampling intervals.

Author: svenklemm

sql/CMakeLists.txt

@@ -44,6 +44,7 @@ set(SOURCE_FILES
   bgw_scheduler.sql
   installation_metadata.sql
   views.sql
+  gapfill.sql
 )
 
 # These files should be pre-pended to update scripts so that they are

sql/gapfill.sql

@@ -0,0 +1,43 @@
+-- Copyright (c) 2016-2018  Timescale, Inc. All Rights Reserved.
+--
+-- This file is licensed under the Apache License, see LICENSE-APACHE
+-- at the top level directory of the TimescaleDB distribution.
+
+CREATE OR REPLACE FUNCTION time_bucket_gapfill(bucket_width SMALLINT, ts SMALLINT, "start" SMALLINT, "end" SMALLINT) RETURNS SMALLINT
+	AS '@MODULE_PATHNAME@', 'ts_gapfill_int16_bucket' LANGUAGE C IMMUTABLE PARALLEL RESTRICTED STRICT;
+
+CREATE OR REPLACE FUNCTION time_bucket_gapfill(bucket_width INT, ts INT, "start" INT, "end" INT) RETURNS INT
+	AS '@MODULE_PATHNAME@', 'ts_gapfill_int32_bucket' LANGUAGE C IMMUTABLE PARALLEL RESTRICTED STRICT;
+
+CREATE OR REPLACE FUNCTION time_bucket_gapfill(bucket_width BIGINT, ts BIGINT, "start" BIGINT, "end" BIGINT) RETURNS BIGINT
+	AS '@MODULE_PATHNAME@', 'ts_gapfill_int64_bucket' LANGUAGE C IMMUTABLE PARALLEL RESTRICTED STRICT;
+
+CREATE OR REPLACE FUNCTION time_bucket_gapfill(bucket_width INTERVAL, ts DATE, "start" DATE, "end" DATE) RETURNS DATE
+	AS '@MODULE_PATHNAME@', 'ts_gapfill_date_bucket' LANGUAGE C IMMUTABLE PARALLEL RESTRICTED STRICT;
+
+CREATE OR REPLACE FUNCTION time_bucket_gapfill(bucket_width INTERVAL, ts TIMESTAMP, "start" TIMESTAMP, "end" TIMESTAMP) RETURNS TIMESTAMP
+	AS '@MODULE_PATHNAME@', 'ts_gapfill_timestamp_bucket' LANGUAGE C IMMUTABLE PARALLEL RESTRICTED STRICT;
+
+CREATE OR REPLACE FUNCTION time_bucket_gapfill(bucket_width INTERVAL, ts TIMESTAMPTZ, "start" TIMESTAMPTZ, "end" TIMESTAMPTZ) RETURNS TIMESTAMPTZ
+	AS '@MODULE_PATHNAME@', 'ts_gapfill_timestamptz_bucket' LANGUAGE C IMMUTABLE PARALLEL RESTRICTED STRICT;
+
+-- locf function
+CREATE OR REPLACE FUNCTION locf(value ANYELEMENT, prev ANYELEMENT=NULL) RETURNS ANYELEMENT
+	AS '@MODULE_PATHNAME@', 'ts_gapfill_marker' LANGUAGE C IMMUTABLE PARALLEL RESTRICTED;
+
+-- interpolate functions
+CREATE OR REPLACE FUNCTION interpolate(value SMALLINT,prev RECORD=NULL,next RECORD=NULL) RETURNS SMALLINT
+	AS '@MODULE_PATHNAME@', 'ts_gapfill_marker' LANGUAGE C IMMUTABLE PARALLEL RESTRICTED;
+
+CREATE OR REPLACE FUNCTION interpolate(value INT,prev RECORD=NULL,next RECORD=NULL) RETURNS INT
+	AS '@MODULE_PATHNAME@', 'ts_gapfill_marker' LANGUAGE C IMMUTABLE PARALLEL RESTRICTED;
+
+CREATE OR REPLACE FUNCTION interpolate(value BIGINT,prev RECORD=NULL,next RECORD=NULL) RETURNS BIGINT
+	AS '@MODULE_PATHNAME@', 'ts_gapfill_marker' LANGUAGE C IMMUTABLE PARALLEL RESTRICTED;
+
+CREATE OR REPLACE FUNCTION interpolate(value REAL,prev RECORD=NULL,next RECORD=NULL) RETURNS REAL
+	AS '@MODULE_PATHNAME@', 'ts_gapfill_marker' LANGUAGE C IMMUTABLE PARALLEL RESTRICTED;
+
+CREATE OR REPLACE FUNCTION interpolate(value FLOAT,prev RECORD=NULL,next RECORD=NULL) RETURNS FLOAT
+	AS '@MODULE_PATHNAME@', 'ts_gapfill_marker' LANGUAGE C IMMUTABLE PARALLEL RESTRICTED;
+

src/CMakeLists.txt

@@ -20,6 +20,7 @@ set(SOURCES
   dimension_vector.c
   event_trigger.c
   extension.c
+  gapfill.c
   guc.c
   histogram.c
   hypercube.c

src/cross_module_fn.c

@@ -7,6 +7,7 @@
 #include <postgres.h>
 #include <fmgr.h>
 #include <utils/timestamp.h>
+#include <utils/lsyscache.h>
 
 #include "export.h"
 #include "cross_module_fn.h"
@@ -90,6 +91,18 @@ bgw_policy_job_execute_default_fn(BgwJob *job)
 	return false;
 }
 
+static Datum
+error_no_default_fn_pg_function(PG_FUNCTION_ARGS)
+{
+	ereport(ERROR,
+			(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+			 errmsg("function \"%s\" is not supported under the current license \"%s\"",
+					get_func_name(fcinfo->flinfo->fn_oid),
+					ts_guc_license_key),
+			 errhint("Buy a Timescale license to enable the functionality")));
+	PG_RETURN_NULL();
+}
+
 /*
  * Define cross-module functions' default values:
  * If the submodule isn't activated, using one of the cm functions will throw an
@@ -106,6 +119,14 @@ TSDLLEXPORT CrossModuleFunctions ts_cm_functions_default = {
 	.add_recluster_policy = error_no_default_fn_pg,
 	.remove_drop_chunks_policy = error_no_default_fn_pg,
 	.remove_recluster_policy = error_no_default_fn_pg,
+	.create_upper_paths_hook = NULL,
+	.gapfill_marker = error_no_default_fn_pg_function,
+	.gapfill_int16_time_bucket = error_no_default_fn_pg_function,
+	.gapfill_int32_time_bucket = error_no_default_fn_pg_function,
+	.gapfill_int64_time_bucket = error_no_default_fn_pg_function,
+	.gapfill_date_time_bucket = error_no_default_fn_pg_function,
+	.gapfill_timestamp_time_bucket = error_no_default_fn_pg_function,
+	.gapfill_timestamptz_time_bucket = error_no_default_fn_pg_function,
 };
 
 TSDLLEXPORT CrossModuleFunctions *ts_cm_functions = &ts_cm_functions_default;

src/cross_module_fn.h

@@ -9,6 +9,7 @@
 
 #include <c.h>
 #include <utils/timestamp.h>
+#include <optimizer/planner.h>
 
 #include "export.h"
 #include "bgw/job.h"
@@ -35,6 +36,14 @@ typedef struct CrossModuleFunctions
 	Datum		(*add_recluster_policy) (PG_FUNCTION_ARGS);
 	Datum		(*remove_drop_chunks_policy) (PG_FUNCTION_ARGS);
 	Datum		(*remove_recluster_policy) (PG_FUNCTION_ARGS);
+	void		(*create_upper_paths_hook) (PlannerInfo *, UpperRelationKind, RelOptInfo *, RelOptInfo *);
+	PGFunction	gapfill_marker;
+	PGFunction	gapfill_int16_time_bucket;
+	PGFunction	gapfill_int32_time_bucket;
+	PGFunction	gapfill_int64_time_bucket;
+	PGFunction	gapfill_date_time_bucket;
+	PGFunction	gapfill_timestamp_time_bucket;
+	PGFunction	gapfill_timestamptz_time_bucket;
 } CrossModuleFunctions;
 
 extern TSDLLEXPORT CrossModuleFunctions *ts_cm_functions;

src/export.h

@@ -58,7 +58,7 @@
 #endif
 
 #define TS_FUNCTION_INFO_V1(fn) \
-	PGDLLEXPORT Datum fn(PG_FUNCTION_ARGS); \
+	TSDLLEXPORT Datum fn(PG_FUNCTION_ARGS); \
 	PG_FUNCTION_INFO_V1(fn)
 
 #endif							/* TIMESCALEDB_EXPORT_H */

src/gapfill.c

@@ -0,0 +1,42 @@
+/*
+ * Copyright (c) 2016-2018  Timescale, Inc. All Rights Reserved.
+ *
+ * This file is licensed under the Apache License,
+ * see LICENSE-APACHE at the top level directory.
+ */
+#include <postgres.h>
+#include <fmgr.h>
+
+#include "license_guc.h"
+#include "cross_module_fn.h"
+#include "compat.h"
+#include "export.h"
+
+/*
+ * stub function to trigger locf and interpolate in gapfill node
+ * the calls to these functions will be removed from the final plan
+ * for a valid gapfill query. This function will only be called if
+ * no timescale license is available or when they are used outside of
+ * a valid gapfill query.
+ */
+TS_FUNCTION_INFO_V1(ts_gapfill_marker);
+Datum
+ts_gapfill_marker(PG_FUNCTION_ARGS)
+{
+	PG_RETURN_DATUM(ts_cm_functions->gapfill_marker(fcinfo));
+}
+
+#define GAPFILL_TIMEBUCKET_WRAPPER(datatype) \
+TS_FUNCTION_INFO_V1(ts_gapfill_ ## datatype ## _bucket); \
+Datum \
+ts_gapfill_ ## datatype ## _bucket(PG_FUNCTION_ARGS) \
+{ \
+	return ts_cm_functions->gapfill_ ## datatype ## _time_bucket(fcinfo); \
+}
+
+GAPFILL_TIMEBUCKET_WRAPPER(int16);
+GAPFILL_TIMEBUCKET_WRAPPER(int32);
+GAPFILL_TIMEBUCKET_WRAPPER(int64);
+GAPFILL_TIMEBUCKET_WRAPPER(date);
+GAPFILL_TIMEBUCKET_WRAPPER(timestamp);
+GAPFILL_TIMEBUCKET_WRAPPER(timestamptz);

src/planner.c

@@ -37,6 +37,8 @@
 #endif
 #include "compat-msvc-exit.h"
 
+#include "cross_module_fn.h"
+#include "license_guc.h"
 #include "hypertable_cache.h"
 #include "extension.h"
 #include "utils.h"
@@ -515,6 +517,9 @@ timescale_create_upper_paths_hook(PlannerInfo *root,
 	if (!ts_extension_is_loaded())
 		return;
 
+	if (ts_cm_functions->create_upper_paths_hook != NULL)
+		ts_cm_functions->create_upper_paths_hook(root, stage, input_rel, output_rel);
+
 	/* Modify for INSERTs on a hypertable */
 	if (output_rel != NULL && output_rel->pathlist != NIL)
 		output_rel->pathlist = replace_hypertable_insert_paths(root, output_rel->pathlist);

src/time_bucket.h

@@ -0,0 +1,20 @@
+/*
+ * Copyright (c) 2016-2018  Timescale, Inc. All Rights Reserved.
+ *
+ * This file is licensed under the Apache License,
+ * see LICENSE-APACHE at the top level directory.
+ */
+#ifndef TIMESCALEDB_TIME_BUCKET_H
+#define TIMESCALEDB_TIME_BUCKET_H
+
+#include <postgres.h>
+#include <fmgr.h>
+
+extern Datum ts_int16_bucket(PG_FUNCTION_ARGS);
+extern Datum ts_int32_bucket(PG_FUNCTION_ARGS);
+extern Datum ts_int64_bucket(PG_FUNCTION_ARGS);
+extern Datum ts_date_bucket(PG_FUNCTION_ARGS);
+extern Datum ts_timestamp_bucket(PG_FUNCTION_ARGS);
+extern Datum ts_timestamptz_bucket(PG_FUNCTION_ARGS);
+
+#endif							/* TIMESCALEDB_TIME_BUCKET_H */

test/expected/extension.out

@@ -33,7 +33,9 @@ ORDER BY proname;
  hypertable_relation_size_pretty
  indexes_relation_size
  indexes_relation_size_pretty
+ interpolate
  last
+ locf
  remove_drop_chunks_policy
  remove_recluster_policy
  set_adaptive_chunking
@@ -42,5 +44,6 @@ ORDER BY proname;
  show_chunks
  show_tablespaces
  time_bucket
-(27 rows)
+ time_bucket_gapfill
+(30 rows)