flatten uuid strings #13094

aardgoose · 2018-01-11T20:35:45Z

Workaround for #13069 issues

mrdoob · 2018-01-11T20:46:04Z

var _Math = {

	generateUUID: ( function () {

		// http://stackoverflow.com/questions/105034/how-to-create-a-guid-uuid-in-javascript/21963136#21963136

		var lut = [];

		for ( var i = 0; i < 256; i ++ ) {

			lut[ i ] = ( i < 16 ? '0' : '' ) + ( i ).toString( 16 ).toUpperCase();

		}

		return function generateUUID() {

			var d0 = Math.random() * 0xffffffff | 0;
			var d1 = Math.random() * 0xffffffff | 0;
			var d2 = Math.random() * 0xffffffff | 0;
			var d3 = Math.random() * 0xffffffff | 0;
			return lut[ d0 & 0xff ] + lut[ d0 >> 8 & 0xff ] + lut[ d0 >> 16 & 0xff ] + lut[ d0 >> 24 & 0xff ] + '-' +
				lut[ d1 & 0xff ] + lut[ d1 >> 8 & 0xff ] + '-' + lut[ d1 >> 16 & 0x0f | 0x40 ] + lut[ d1 >> 24 & 0xff ] + '-' +
				lut[ d2 & 0x3f | 0x80 ] + lut[ d2 >> 8 & 0xff ] + '-' + lut[ d2 >> 16 & 0xff ] + lut[ d2 >> 24 & 0xff ] +
				lut[ d3 & 0xff ] + lut[ d3 >> 8 & 0xff ] + lut[ d3 >> 16 & 0xff ] + lut[ d3 >> 24 & 0xff ];

		};

	} )(),

	generateUUID2: ( function () {

		// http://stackoverflow.com/questions/105034/how-to-create-a-guid-uuid-in-javascript/21963136#21963136

		var lut = [];

		for ( var i = 0; i < 256; i ++ ) {

			lut[ i ] = ( i < 16 ? '0' : '' ) + ( i ).toString( 16 );

		}

		return function generateUUID() {

			var d0 = Math.random() * 0xffffffff | 0;
			var d1 = Math.random() * 0xffffffff | 0;
			var d2 = Math.random() * 0xffffffff | 0;
			var d3 = Math.random() * 0xffffffff | 0;

			return String( lut[ d0 & 0xff ] + lut[ d0 >> 8 & 0xff ] + lut[ d0 >> 16 & 0xff ] + lut[ d0 >> 24 & 0xff ] + '-' +
				lut[ d1 & 0xff ] + lut[ d1 >> 8 & 0xff ] + '-' + lut[ d1 >> 16 & 0x0f | 0x40 ] + lut[ d1 >> 24 & 0xff ] + '-' +
				lut[ d2 & 0x3f | 0x80 ] + lut[ d2 >> 8 & 0xff ] + '-' + lut[ d2 >> 16 & 0xff ] + lut[ d2 >> 24 & 0xff ] +
				lut[ d3 & 0xff ] + lut[ d3 >> 8 & 0xff ] + lut[ d3 >> 16 & 0xff ] + lut[ d3 >> 24 & 0xff ] ).toUpperCase();

		};

	} )()

};

function run( func ) {

	var loop = 0x100000;

	var startTime = performance.now();

	for ( var i = 0; i < loop; i ++ ) {

		func();

	}

	var endTime = performance.now();

	console.log( ( endTime - startTime ).toFixed( 2 ) + 'ms, loop=' + loop );

}

run(_Math.generateUUID);
run(_Math.generateUUID);
run(_Math.generateUUID2);
run(_Math.generateUUID2);

mrdoob · 2018-01-11T20:47:26Z

Goes from ~350ms to ~740ms here... 😕

makc · 2018-01-11T20:50:19Z

why does 3js use uuids at all? can't it just use counters?

mrdoob · 2018-01-11T20:51:40Z

why does 3js use uuids at all? can't it just use counters?

Mainly for serialising/deserialising.

makc · 2018-01-11T20:53:17Z

deserialising

ok, and why can't you do geometries[123] instead of geometries['XXX-YYY-SOMETHING'] while deserialising?

mrdoob · 2018-01-11T20:53:29Z

I'm wondering if uppercasing the uuid is worth it though. As far as I understand, uppercase uuid consumes 10x more memory than lowercase?

mrdoob · 2018-01-11T20:54:39Z

ok, and why can't you do geometries[123] instead of geometries['XXX-YYY-SOMETHING'] while deserialising?

Lets not derail this thread.

makc · 2018-01-11T20:54:54Z

in the context of #13069 .toLowerCase() would probably have the same effect

aardgoose · 2018-01-11T20:56:48Z

the performance hit is why i looked at removing the requirement for the uuid's with the lazy_uuid PR.

The WeakMap.get( object ) is probably a simpler lookup just on the object itself rather than having to get an object property as the uuid lookup in WebGLProperties & Attributes so that is probably a gain too.

@mrdoob the case of the uuid doesn't matter. It just gets the VM to create a more compact copy as a side effect - hence the hacky nature.

mrdoob · 2018-01-11T20:56:56Z

How about keeping the toUpperCase() for the LUT and doing a String() in the return?

( function () {

	// http://stackoverflow.com/questions/105034/how-to-create-a-guid-uuid-in-javascript/21963136#21963136

	var lut = [];

	for ( var i = 0; i < 256; i ++ ) {

		lut[ i ] = ( i < 16 ? '0' : '' ) + ( i ).toString( 16 ).toUpperCase();

	}

	return function generateUUID() {

		var d0 = Math.random() * 0xffffffff | 0;
		var d1 = Math.random() * 0xffffffff | 0;
		var d2 = Math.random() * 0xffffffff | 0;
		var d3 = Math.random() * 0xffffffff | 0;

		return String( lut[ d0 & 0xff ] + lut[ d0 >> 8 & 0xff ] + lut[ d0 >> 16 & 0xff ] + lut[ d0 >> 24 & 0xff ] + '-' +
			lut[ d1 & 0xff ] + lut[ d1 >> 8 & 0xff ] + '-' + lut[ d1 >> 16 & 0x0f | 0x40 ] + lut[ d1 >> 24 & 0xff ] + '-' +
			lut[ d2 & 0x3f | 0x80 ] + lut[ d2 >> 8 & 0xff ] + '-' + lut[ d2 >> 16 & 0xff ] + lut[ d2 >> 24 & 0xff ] +
			lut[ d3 & 0xff ] + lut[ d3 >> 8 & 0xff ] + lut[ d3 >> 16 & 0xff ] + lut[ d3 >> 24 & 0xff ] );

	};

} )()

No performance changes with that code here. Does this solve the memory issue?

mrdoob · 2018-01-11T21:00:35Z

For testing...

var _Math = {

	generateUUID: ( function () {

		// http://stackoverflow.com/questions/105034/how-to-create-a-guid-uuid-in-javascript/21963136#21963136

		var lut = [];

		for ( var i = 0; i < 256; i ++ ) {

			lut[ i ] = ( i < 16 ? '0' : '' ) + ( i ).toString( 16 ).toUpperCase();

		}

		return function generateUUID() {

			var d0 = Math.random() * 0xffffffff | 0;
			var d1 = Math.random() * 0xffffffff | 0;
			var d2 = Math.random() * 0xffffffff | 0;
			var d3 = Math.random() * 0xffffffff | 0;
			return lut[ d0 & 0xff ] + lut[ d0 >> 8 & 0xff ] + lut[ d0 >> 16 & 0xff ] + lut[ d0 >> 24 & 0xff ] + '-' +
				lut[ d1 & 0xff ] + lut[ d1 >> 8 & 0xff ] + '-' + lut[ d1 >> 16 & 0x0f | 0x40 ] + lut[ d1 >> 24 & 0xff ] + '-' +
				lut[ d2 & 0x3f | 0x80 ] + lut[ d2 >> 8 & 0xff ] + '-' + lut[ d2 >> 16 & 0xff ] + lut[ d2 >> 24 & 0xff ] +
				lut[ d3 & 0xff ] + lut[ d3 >> 8 & 0xff ] + lut[ d3 >> 16 & 0xff ] + lut[ d3 >> 24 & 0xff ];

		};

	} )(),

	generateUUID2: ( function () {

		// http://stackoverflow.com/questions/105034/how-to-create-a-guid-uuid-in-javascript/21963136#21963136

		var lut = [];

		for ( var i = 0; i < 256; i ++ ) {

			lut[ i ] = ( i < 16 ? '0' : '' ) + ( i ).toString( 16 ).toUpperCase();

		}

		return function generateUUID() {

			var d0 = Math.random() * 0xffffffff | 0;
			var d1 = Math.random() * 0xffffffff | 0;
			var d2 = Math.random() * 0xffffffff | 0;
			var d3 = Math.random() * 0xffffffff | 0;

			return String(
				lut[ d0 & 0xff ] + lut[ d0 >> 8 & 0xff ] + lut[ d0 >> 16 & 0xff ] + lut[ d0 >> 24 & 0xff ] + '-' +
				lut[ d1 & 0xff ] + lut[ d1 >> 8 & 0xff ] + '-' + lut[ d1 >> 16 & 0x0f | 0x40 ] + lut[ d1 >> 24 & 0xff ] + '-' +
				lut[ d2 & 0x3f | 0x80 ] + lut[ d2 >> 8 & 0xff ] + '-' + lut[ d2 >> 16 & 0xff ] + lut[ d2 >> 24 & 0xff ] +
				lut[ d3 & 0xff ] + lut[ d3 >> 8 & 0xff ] + lut[ d3 >> 16 & 0xff ] + lut[ d3 >> 24 & 0xff ]
			);

		};

	} )()

};

function run( func ) {

	var loop = 0x100000;

	var startTime = performance.now();

	for ( var i = 0; i < loop; i ++ ) {

		func();

	}

	var endTime = performance.now();

	console.log( ( endTime - startTime ).toFixed( 2 ) + 'ms, loop=' + loop );

}

run(_Math.generateUUID);
run(_Math.generateUUID);
run(_Math.generateUUID2);
run(_Math.generateUUID2);

aardgoose · 2018-01-11T21:02:52Z

Do a heap dump in Chrome and look at the (concatenated string)

section. You can see the way it stores the strings in lots of small sections.

I think just String() alone just produces a reference to the bloated version hence no performance change and no reduction of memory.

Just checked - just String() still leaves the strings in the fragmented form. I thought I had tried that myself.

makc · 2018-01-11T22:09:14Z

out of interest just tried to do this with Uint8Array and TextDecoder, and it's only 15 times slower :D then I thought if I inline setHex, I can gain something, but profiler says -23% max

compared to this, toUpperCase x2 perfrmance drop does not look that bad at all.

takahirox · 2018-01-12T00:11:42Z

BTW, Array.join( '' ) (array includes pointers to one character in another array) used in older generateUUID() seems to allocate one consecutive memory space. You can try #13069 (comment) by replacing r89 with r87. (Seems like this hack with new generateUUID() is still faster than the older one tho. #12432 (comment))

So, this uuid memory consumption issue is new to Three.js(>=r88).

mcollina · 2018-01-12T07:40:15Z

If UUIDs are not needed, I would recommend using an approach similar to https://github.com/mcollina/hyperid/blob/master/hyperid.js: generate something unique and then concatenate with a counter.

Also https://www.npmjs.com/package/flatstr.

takahirox · 2018-01-12T14:45:52Z

Number tech is interesting...
But sometimes it doesn't work well so we need to carefully use it?

I tried four patterns with the following code on my Windows10 + Chrome.

<script src="https://rawgit.com/mrdoob/three.js/r89/build/three.js"></script>
<script>
var num = 0x10000;

function flatstr( s ) {

    Number( s );
    return s;

}

var array = Array( num );

for ( var i = 0; i < num; i ++ ) {

    // 1.
    //array[ i ] = THREE.Math.generateUUID();

    // 2.
    //array[ i ] = THREE.Math.generateUUID().toUpperCase();

    // 3.
    //array[ i ] = flatstr( THREE.Math.generateUUID() );

    // 4.
    //var uuid = THREE.Math.generateUUID();
    //Number( uuid );
    //array[ i ] = uuid;

}
</script>

	concatenated string count	shallow size	retained size	total heap snapshot size
1.	921,259	36,850,360	39,507,080	42.3MB
2.	3,755	150,200	185,480	8.8MB
3.	809,262	32,370,480	34,776,728	38.2MB
4.	52,893	2,115,720	5,295,832	10.7MB

I expected 3.'s and 4.'s result are same as the 2.'s result but seems like a lot of concatenated strings are left in heap.

aardgoose · 2018-01-12T15:41:00Z

I was half expecting this to be a recent issue, since I hadn't seen the problem before when looking at heap size in general.

I would check what you get out from Number( uuid ). console.log( Number ( new THREE.Object3D().uuid ) ); produces NaN.

takahirox · 2018-01-13T00:16:36Z

Probably what Number( uuid ) produces doesn't matter in this tech if I'm right.

makc · 2018-01-13T02:35:15Z

@takahirox and from 42 vs 38 mb it does not seem it produces anything at all

takahirox · 2018-03-13T22:07:54Z

3 works now by replacing flatstr with

function flatstr( s ) {

    return isNaN( Number( s ) ) ? s :  '';

}

Probably we need to let Number( s ) not be removed by JS engine optimization.

Bad news, Number( s ) tech isn't fast. A bit slower than .toUpperCase(). I speculate String::Flatten method itself isn't fast regardless how we trigger.

takahirox · 2018-03-14T21:21:42Z

Would you reopen this because of #13069 (comment)?

takahirox · 2018-03-14T21:23:28Z

src/math/Math.js

 			var d3 = Math.random() * 0xffffffff | 0;
-			return lut[ d0 & 0xff ] + lut[ d0 >> 8 & 0xff ] + lut[ d0 >> 16 & 0xff ] + lut[ d0 >> 24 & 0xff ] + '-' +
+
+			return String( lut[ d0 & 0xff ] + lut[ d0 >> 8 & 0xff ] + lut[ d0 >> 16 & 0xff ] + lut[ d0 >> 24 & 0xff ] + '-' +


I prefer removing "String", not strong preference tho.

mrdoob · 2018-03-14T23:16:59Z

I can't reopen this PR because @aardgoose deleted the branch.

takahirox · 2018-03-14T23:48:56Z

Oops, then I make a new one.

flatten uuid strings

18306ab

makc mentioned this pull request Jan 29, 2018

memory consumption of UUID strings. #13069

Closed

11 tasks

aardgoose closed this Mar 1, 2018

aardgoose deleted the uuid-flatten branch March 1, 2018 14:43

takahirox reviewed Mar 14, 2018

View reviewed changes

takahirox mentioned this pull request Mar 15, 2018

Flatten UUID strings with .toUpperCase() to save heap memory space. #13595

Merged

Uh oh!

flatten uuid strings #13094

flatten uuid strings #13094

Uh oh!

Conversation

aardgoose commented Jan 11, 2018

Uh oh!

mrdoob commented Jan 11, 2018

Uh oh!

mrdoob commented Jan 11, 2018

Uh oh!

makc commented Jan 11, 2018

Uh oh!

mrdoob commented Jan 11, 2018

Uh oh!

makc commented Jan 11, 2018

Uh oh!

mrdoob commented Jan 11, 2018

Uh oh!

mrdoob commented Jan 11, 2018

Uh oh!

makc commented Jan 11, 2018

Uh oh!

aardgoose commented Jan 11, 2018

Uh oh!

mrdoob commented Jan 11, 2018 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Uh oh!

mrdoob commented Jan 11, 2018 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Uh oh!

aardgoose commented Jan 11, 2018 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Uh oh!

makc commented Jan 11, 2018

Uh oh!

takahirox commented Jan 12, 2018 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Uh oh!

mcollina commented Jan 12, 2018

Uh oh!

takahirox commented Jan 12, 2018 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Uh oh!

aardgoose commented Jan 12, 2018

Uh oh!

takahirox commented Jan 13, 2018

Uh oh!

makc commented Jan 13, 2018

Uh oh!

takahirox commented Mar 13, 2018

Uh oh!

takahirox commented Mar 14, 2018

Uh oh!

takahirox Mar 14, 2018

Choose a reason for hiding this comment

Uh oh!

mrdoob commented Mar 14, 2018

Uh oh!

takahirox commented Mar 14, 2018

Uh oh!

Reviewers

Assignees

Labels

Projects

Milestone

Development

Uh oh!

5 participants

mrdoob commented Jan 11, 2018 •

edited

Loading

mrdoob commented Jan 11, 2018 •

edited

Loading

aardgoose commented Jan 11, 2018 •

edited

Loading

takahirox commented Jan 12, 2018 •

edited

Loading

takahirox commented Jan 12, 2018 •

edited

Loading