release: 2024-02-15

.github/workflows/snowflake-ded.yml

@@ -5,7 +5,7 @@ on:
     types: [closed, unlabeled, labeled, synchronize]
 
 env:
-  NODE_VERSION: 16
+  NODE_VERSION: 18.16
 
 jobs:
 

.github/workflows/snowflake.yml

@@ -15,7 +15,7 @@ on:
   workflow_call:
 
 env:
-  NODE_VERSION: 16
+  NODE_VERSION: 18.16
   PYTHON3_VERSION: 3.8.10
   VIRTUALENV_VERSION: 20.21.1
   GCLOUD_VERSION: 290.0.1

CHANGELOG.md

@@ -4,6 +4,14 @@ CARTO Analytics Toolbox Core.
 
 All notable commits to this project will be documented in this file.
 
+## 2024-02-15
+
+- chore(sf): add additional tables to native apps (#473)
+- docs(bs,sf,rs|transformations): fix ST_DESTINATION bearing parameter description (#475)
+- fix(sf|quadbin): QUADBIN_TOPARENT not working with views (#476)
+- docs(bq,sf,rs|constructors): fix angle parameter description in ST_MAKEELLIPSE (#477)
+- fix(sf|random): ST_GENERATEPOINTS was not accepting column names (#480)
+
 ## 2024-01-17
 
 - chore(bq): increase tests timeout to 200000 (#455)

clouds/bigquery/modules/doc/constructors/ST_MAKEELLIPSE.md

@@ -11,7 +11,7 @@ Takes a Point and calculates the ellipse polygon given two semi-axes expressed i
 * `center`: `GEOGRAPHY` center point.
 * `xSemiAxis`: `FLOAT64` semi (major) axis of the ellipse along the x-axis.
 * `ySemiAxis`: `FLOAT64` semi (minor) axis of the ellipse along the y-axis.
-* `angle`: `FLOAT64`|`NULL` angle of rotation (along the vertical axis), from North in decimal degrees, negative clockwise. If `NULL` the default value `0` is used.
+* `angle`: `FLOAT64`|`NULL` angle of rotation (along the vertical axis), from North in decimal degrees, positive clockwise. If `NULL` the default value `0` is used.
 * `units`: `STRING`|`NULL` units of length, the supported options are: miles, kilometers, and degrees. If `NULL`the default value `kilometers` is used.
 * `steps`: `INT64`|`NULL` number of steps. If `NULL` the default value `64` is used.
 

clouds/bigquery/modules/doc/transformations/ST_DESTINATION.md

@@ -10,7 +10,7 @@ Takes a Point and calculates the location of a destination point given a distanc
 
 * `origin`: `GEOGRAPHY` starting point.
 * `distance`: `FLOAT64` distance from the origin point in the units specified.
-* `bearing`: `FLOAT64` counter-clockwise angle from East, ranging from -180 to 180 (e.g. 0 is East, 90 is North, 180 is West, -90 is South).
+* `bearing`: `FLOAT64` ranging from -180 to 180 (e.g. 0 is North, 90 is East, 180 is South, -90 is West).
 * `units`: `STRING`|`NULL` units of length, the supported options are: `miles`, `kilometers`, `degrees` or `radians`. If `NULL`the default value `kilometers` is used.
 
 **Return type**

clouds/redshift/modules/doc/constructors/ST_MAKEELLIPSE.md

@@ -11,7 +11,7 @@ Takes a Point as input and calculates the ellipse polygon given two semi-axes ex
 * `center`: `GEOMETRY` center point.
 * `xSemiAxis`: `FLOAT8` semi (major) axis of the ellipse along the x-axis.
 * `ySemiAxis`: `FLOAT8` semi (minor) axis of the ellipse along the y-axis.
-* `angle` (optional): `FLOAT8` angle of rotation (along the vertical axis), from North in decimal degrees, negative clockwise. If not specified, the default value of `0` will be used.
+* `angle` (optional): `FLOAT8` angle of rotation (along the vertical axis), from North in decimal degrees, positive clockwise. If not specified, the default value of `0` will be used.
 * `units` (optional): `VARCHAR(10)` units of length. The supported options are: miles, kilometers, meters, and degrees. If not specified, `kilometers` will be used.
 * `steps` (optional): `INT` number of steps. If not specified, the default value of `64` will be used.
 

clouds/redshift/modules/doc/transformations/ST_DESTINATION.md

@@ -10,7 +10,7 @@ Takes a Point and calculates the location of a destination point given a distanc
 
 * `geom`: `GEOMETRY` starting point.
 * `distance`: `FLOAT8` distance from the origin point in the units specified.
-* `bearing`: `FLOAT8` counter-clockwise angle from East, ranging from -180 to 180 (e.g. 0 is East, 90 is North, 180 is West, -90 is South).
+* `bearing`: `FLOAT8` ranging from -180 to 180 (e.g. 0 is North, 90 is East, 180 is South, -90 is West).
 * `units` (optional): `VARCHAR(15)` units of length. The supported options are: `miles`, `kilometers`, `degrees` or `radians`. If `NULL`the default value `kilometers` is used.
 
 **Return type**

clouds/snowflake/CHANGELOG.md

@@ -4,6 +4,12 @@ CARTO Analytics Toolbox Core for Snowflake.
 
 All notable commits to this project will be documented in this file.
 
+## [1.2.1] - 2024-02-15
+
+- chore: add additional tables to native apps (#473)
+- fix(quadbin): QUADBIN_TOPARENT not working with views (#476)
+- fix(random): ST_GENERATEPOINTS was not accepting column names (#480)
+
 ## [1.2.0] - 2024-01-17
 
 - feat(quadbin): add function QUADBIN_DISTANCE (#457)

clouds/snowflake/common/build_native_app_setup_script.js

@@ -15,6 +15,7 @@ const argv = require('minimist')(process.argv.slice(2));
 const inputDirs = argv._[0] && argv._[0].split(',');
 const outputDir = argv.output || 'build';
 const libsBuildDir = argv.libs_build_dir || '../libraries/javascript/build';
+const nativeAppDir = argv.native_app_dir || '../native_app';
 const diff = argv.diff || [];
 const nodeps = argv.nodeps;
 let modulesFilter = (argv.modules && argv.modules.split(',')) || [];
@@ -232,8 +233,13 @@ const header = `CREATE OR REPLACE APPLICATION ROLE @@APP_ROLE@@;
 CREATE OR ALTER VERSIONED SCHEMA @@SF_SCHEMA@@;
 GRANT USAGE ON SCHEMA @@SF_SCHEMA@@ TO APPLICATION ROLE @@APP_ROLE@@;\n`;
 
+let additionalTables = '';
+if (argv.production) {
+    additionalTables = fs.readFileSync(path.resolve(nativeAppDir, 'ADDITIONAL_TABLES.sql')).toString() + separator;
+}
+
 const footer = fetchPermissionsGrant (fs.readFileSync(path.resolve(__dirname, 'VERSION.sql')).toString());
-content = header + separator + content + separator + footer;
+content = header + separator + additionalTables + content + separator + footer;
 
 content = apply_replacements(content);
 

clouds/snowflake/common/test-utils.js

@@ -56,6 +56,13 @@ async function deleteTable (tablename) {
     await runQuery(query);
 }
 
+async function deleteView (tablename) {
+    const query = `
+      DROP VIEW IF EXISTS ${tablename}
+    `;
+    await runQuery(query);
+}
+
 function sortByKey (list, key) {
     return list.sort((a, b) => (a[key] > b[key]) ? 1 : -1);
 }
@@ -113,6 +120,7 @@ module.exports = {
     runQuery,
     createTable,
     deleteTable,
+    deleteView,
     sortByKey,
     sortByKeyAndRound,
     existsTable,

clouds/snowflake/libraries/javascript/libs/random.js

@@ -1,5 +1,8 @@
-import { bbox } from '@turf/turf';
+import {
+    generateRandomPointsInPolygon
+
+} from '../src/random';
 
 export default {
-    bbox
+    generateRandomPointsInPolygon
 };

clouds/snowflake/libraries/javascript/src/random.js

@@ -0,0 +1,12 @@
+import { bbox, booleanPointInPolygon, randomPoint } from '@turf/turf';
+
+export function generateRandomPointsInPolygon (polygon, numPoints) {
+    const randomPoints = [];
+    while (randomPoints.length < numPoints) {
+        const point = randomPoint(1, { bbox: bbox(polygon) }).features[0];
+        if (booleanPointInPolygon(point, polygon)) {
+            randomPoints.push(JSON.stringify(point.geometry));
+        }
+    }
+    return randomPoints;
+}

clouds/snowflake/libraries/javascript/test/random.test.js

@@ -1,5 +1,5 @@
 const randomLib = require('../build/random');
 
 test('random library defined', () => {
-    expect(randomLib.bbox).toBeDefined();
+    expect(randomLib.generateRandomPointsInPolygon).toBeDefined();
 });

clouds/snowflake/modules/Makefile

@@ -9,6 +9,7 @@ SQL_DIR ?= $(ROOT_DIR)/sql
 ESLINTRC_DIR ?= $(ROOT_DIR)/../../..
 COMMON_DIR = $(ROOT_DIR)/../common
 LIBS_BUILD_DIR ?= $(ROOT_DIR)/../libraries/javascript/build
+NATIVE_APP_DIR ?= $(ROOT_DIR)/../native_app
 
 MODULES_DIRS ?= $(ROOT_DIR)
 export SF_VERSION_FUNCTION ?= VERSION_CORE
@@ -75,7 +76,7 @@ build-native-app-setup-script: $(NODE_MODULES_DEV)
 	SF_SCHEMA=$(SF_UNQUALIFIED_SCHEMA) APP_ROLE=app_public \
 	REPLACEMENTS=$(REPLACEMENTS)" "$(REPLACEMENTS_EXTRA) \
 	$(COMMON_DIR)/build_native_app_setup_script.js $(MODULES_DIRS) \
-		--output=$(BUILD_DIR) --libs_build_dir=$(LIBS_BUILD_DIR) --diff="$(diff)" \
+		--output=$(BUILD_DIR) --libs_build_dir=$(LIBS_BUILD_DIR) --native_app_dir=$(NATIVE_APP_DIR) --diff="$(diff)" \
 		--modules=$(modules) --functions=$(functions) --production=$(production) --nodeps=$(nodeps) --dropfirst=1
 
 deploy: check build

clouds/snowflake/modules/doc/constructors/ST_MAKEELLIPSE.md

@@ -11,7 +11,7 @@ Takes a Point and calculates the ellipse polygon given two semi-axes expressed i
 * `center`: `GEOGRAPHY` center point.
 * `xSemiAxis`: `DOUBLE` semi (major) axis of the ellipse along the x-axis.
 * `ySemiAxis`: `DOUBLE` semi (minor) axis of the ellipse along the y-axis.
-* `angle` (optional): `DOUBLE` angle of rotation (along the vertical axis), from North in decimal degrees, negative clockwise. By default `angle` is `0`.
+* `angle` (optional): `DOUBLE` angle of rotation (along the vertical axis), from North in decimal degrees, positive clockwise. By default `angle` is `0`.
 * `units` (optional): `STRING` units of length, the supported options are: miles, kilometers, and degrees. By default `units` is `kilometers`.
 * `steps` (optional): `INT` number of steps. By default `steps` is `64`.
 

clouds/snowflake/modules/doc/transformations/ST_DESTINATION.md

@@ -10,7 +10,7 @@ Takes a Point and calculates the location of a destination point given a distanc
 
 * `origin`: `GEOGRAPHY` starting point.
 * `distance`: `DOUBLE` distance from the origin point in the units specified.
-* `bearing`: `DOUBLE` counter-clockwise angle from East, ranging from -180 to 180 (e.g. 0 is East, 90 is North, 180 is West, -90 is South).
+* `bearing`: `DOUBLE` ranging from -180 to 180 (e.g. 0 is North, 90 is East, 180 is South, -90 is West).
 * `units` (optional): `STRING` units of length, the supported options are: `miles`, `kilometers`, `degrees` or `radians`. If `NULL`the default value `kilometers` is used.
 
 **Return type**

clouds/snowflake/modules/sql/quadbin/QUADBIN_RESOLUTION.sql

@@ -7,5 +7,5 @@ CREATE OR REPLACE SECURE FUNCTION @@SF_SCHEMA@@.QUADBIN_RESOLUTION
 RETURNS INT
 IMMUTABLE
 AS $$
-    SELECT BITAND(BITSHIFTRIGHT(quadbin, 52), 31)
+    BITAND(BITSHIFTRIGHT(quadbin, 52), 31)
 $$;

clouds/snowflake/modules/sql/quadbin/QUADBIN_TOPARENT.sql

@@ -9,22 +9,20 @@ IMMUTABLE
 AS $$
     IFF(quadbin IS NULL OR resolution IS NULL OR resolution < 0 OR resolution > 26,
         NULL,
-        (
-        SELECT bitor(
-                    bitor(
-                        bitand(
-                            quadbin,
-                            bitnot(
-                                bitshiftleft(31, 52)
-                            )
-                        ),
-                        bitshiftleft(resolution, 52)
-                    ),
-                    bitshiftright(
-                        4503599627370495,
-                        resolution * 2
+        bitor(
+            bitor(
+                bitand(
+                    quadbin,
+                    bitnot(
+                        bitshiftleft(31, 52)
                     )
-                )
+                ),
+                bitshiftleft(resolution, 52)
+            ),
+            bitshiftright(
+                4503599627370495,
+                resolution * 2
+            )
         )
     )
 $$;

clouds/snowflake/modules/sql/random/ST_GENERATEPOINTS.sql

@@ -1,53 +1,25 @@
-----------------------------
--- Copyright (C) 2021 CARTO
-----------------------------
+--------------------------------
+-- Copyright (C) 2021-2024 CARTO
+--------------------------------
+
+CREATE OR REPLACE FUNCTION @@SF_SCHEMA@@._ST_GENERATEPOINTS
+(geojson STRING, npoints DOUBLE)
+RETURNS ARRAY
+LANGUAGE JAVASCRIPT
+IMMUTABLE
+AS $$
+    @@SF_LIBRARY_RANDOM@@
+
+    return randomLib.generateRandomPointsInPolygon(JSON.parse(GEOJSON), NPOINTS);
+$$;
 
 CREATE OR REPLACE SECURE FUNCTION @@SF_SCHEMA@@.ST_GENERATEPOINTS
 (geog GEOGRAPHY, npoints INT)
 RETURNS ARRAY
-VOLATILE
-AS $$(
-    WITH bbox AS (
-    -- compute the bounding box of the polygon
-        SELECT @@SF_SCHEMA@@._BBOX_FROM_GEOJSON(ST_ASGEOJSON(GEOG)::STRING) AS box
-    ),
-    bbox_coords AS (
-        -- break down the bbox array into minx, miny, maxx, maxy
-        SELECT
-            GET(box, 0) AS minx, GET(box, 1) AS miny,
-            GET(box, 2) AS maxx, GET(box, 3) AS maxy
-        FROM bbox
-    ),
-    bbox_data AS (
-        -- compute area of bbox and put some handy values here too
-        SELECT minx, miny, maxx, maxy,
-            1.2 AS k, -- security factor to make it more likely that at least npoints fall within the polygon
-            ST_AREA(@@SF_SCHEMA@@.ST_MAKEENVELOPE(minx, miny, maxx, maxy)) AS bbox_area,
-            CEIL(k*NPOINTS*bbox_area/ST_AREA(GEOG)) AS nRows
-        FROM bbox_coords
-    ),
-    point_seeds AS (
-        -- generate enough values so that we will hopefully have at least npoints of them randomly placed inside geog
-        SELECT SPLIT(lpad('', (nRows - 1), '0'),'0') as rowsArray,
-        SIN(miny*PI()/180.0) AS minsin,
-        SIN(maxy*PI()/180.0) AS maxsin,
-        180.0/PI() AS radtodeg
-        FROM bbox_data
-    ),
-    bbox_points AS (
-        -- compute the random points uniformly in the bbox;
-        SELECT
-            ST_POINT(minx+UNIFORM(0::FLOAT, 1::FLOAT, random())*(maxx-minx), radtodeg*ASIN(minsin+UNIFORM(0::FLOAT, 1::FLOAT, random())*(maxsin-minsin))) AS point
-        FROM bbox_coords, point_seeds, lateral FLATTEN(input => rowsArray)
-    ),
-    poly_points AS (
-        -- now we need to select the points inside the polygon and number them, so we can limit
-        -- the end result to npoints (note that we can't have a dynamic LIMIT)
-        SELECT ROW_NUMBER() OVER(ORDER BY (SELECT NULL)) AS rn, point
-        FROM bbox_points
-        WHERE ST_WITHIN(point, GEOG)
+IMMUTABLE
+AS $$
+    @@SF_SCHEMA@@._ST_GENERATEPOINTS(
+        ST_ASGEOJSON(GEOG)::STRING,
+        NPOINTS
     )
-    -- finally select at most npoints  and return them in an array
-    SELECT ARRAY_AGG(ST_ASGEOJSON(point)::STRING)
-    FROM poly_points WHERE rn <= NPOINTS
-)$$;
+$$;

clouds/snowflake/modules/test/processing/ST_DELAUNAYLINES.test.js

@@ -12,7 +12,7 @@ test('ST_DELAUNAYLINES should work', async () => {
 
     const rows = await runQuery(query);
     expect(rows.length).toEqual(fixturesOut.expectedTriangles1.length);
-    expect(rows.map(item => item.GEOM)).toEqual(fixturesOut.expectedTriangles1);
+    expect(rows.map(item => item.GEOM)).toEqual(expect.arrayContaining(fixturesOut.expectedTriangles1));
 });
 
 test('ST_DELAUNAYLINES should return an empty array if passed an empty array geometry', async () => {

clouds/snowflake/modules/test/processing/ST_DELAUNAYPOLYGONS.test.js

@@ -12,7 +12,7 @@ test('ST_DELAUNAYPOLYGONS should work', async () => {
 
     const rows = await runQuery(query);
     expect(rows.length).toEqual(fixturesOut.expectedPolygons1.length);
-    expect(rows.map(item => item.GEOM)).toEqual(fixturesOut.expectedPolygons1);
+    expect(rows.map(item => item.GEOM)).toEqual(expect.arrayContaining(fixturesOut.expectedPolygons1));
 });
 
 test('ST_DELAUNAYPOLYGONS should return an empty array if passed null geometry', async () => {

clouds/snowflake/modules/test/processing/ST_POLYGONIZE.test.js

@@ -12,7 +12,7 @@ test('ST_POLYGONIZE should work', async () => {
 
     const rows = await runQuery(query);
     expect(rows.length).toEqual(fixturesOut.expectedPolygons.length);
-    expect(rows.map(item => item.GEOM)).toEqual(fixturesOut.expectedPolygons);
+    expect(rows.map(item => item.GEOM)).toEqual(expect.arrayContaining(fixturesOut.expectedPolygons));
 });
 
 test('ST_POLYGONIZE should fail if a degenerated line path is received', async () => {

clouds/snowflake/modules/test/processing/ST_VORONOILINES.test.js

@@ -18,7 +18,7 @@ test('ST_VORONOILINES should work', async () => {
         FROM voronoi, LATERAL FLATTEN(input => voronoi.geomArray) as unnestedFeatures`;
     const rows = await runQuery(query);
     expect(rows.length).toEqual(fixturesOut.expectedLines1.length);
-    expect(rows.map(item => item.GEOM)).toEqual(fixturesOut.expectedLines1);
+    expect(rows.map(item => item.GEOM)).toEqual(expect.arrayContaining(fixturesOut.expectedLines1));
 });
 
 

clouds/snowflake/modules/test/processing/ST_VORONOIPOLYGONS.test.js

@@ -18,7 +18,7 @@ test('ST_VORONOIPOLYGONS should work', async () => {
         FROM voronoi, LATERAL FLATTEN(input => voronoi.geomArray) as unnestedFeatures`;
     const rows = await runQuery(query);
     expect(rows.length).toEqual(fixturesOut.expectedPoly1.length);
-    expect(rows.map(item => item.GEOM)).toEqual(fixturesOut.expectedPoly1);
+    expect(rows.map(item => item.GEOM)).toEqual(expect.arrayContaining(fixturesOut.expectedPoly1));
 });
 
 

clouds/snowflake/modules/test/quadbin/QUADBIN_TOPARENT.test.js

@@ -1,8 +1,24 @@
-const { runQuery } = require('../../../common/test-utils');
+const { runQuery, deleteTable, deleteView } = require('../../../common/test-utils');
 
 test('QUADBIN_TOPARENT should work', async () => {
     const query = 'SELECT CAST(QUADBIN_TOPARENT(5209574053332910079, 3) AS STRING) AS OUTPUT';
     const rows = await runQuery(query);
     expect(rows.length).toEqual(1);
     expect(rows[0].OUTPUT).toEqual('5205105638077628415');
+});
+
+test('QUADBIN_TOPARENT should work with nested functions when readin data from views', async () => {
+    const inputTable = '@@SF_SCHEMA@@.coords_sample';
+    const inputTableView = '@@SF_SCHEMA@@.test_quadbin_toparent_view';
+
+    query = `CREATE VIEW IF NOT EXISTS ${inputTableView} AS
+        SELECT * FROM ${inputTable};`;
+    await runQuery(query);
+
+    query = `SELECT CAST(QUADBIN_TOPARENT(QUADBIN_FROMLONGLAT(long, lat, zoom), 6) AS STRING) AS OUTPUT
+        FROM ${inputTableView};`;
+    const rows = await runQuery(query);
+    expect(rows.length).toEqual(120);
+
+    deleteView(inputTableView);
 });