Implement support for group layer blending

Translations/Translations.pot

@@ -2143,6 +2143,10 @@ msgstr ""
 msgid "Blend mode:"
 msgstr ""
 
+#. Found in the layer's section of the timeline, as a blend mode option, only available for group layers. If enabled, group blending is disabled and the group simply acts as a way to organize layers instead of affecting blending.
+msgid "Pass through"
+msgstr ""
+
 #. Adjective, refers to something usual/regular, such as the normal blend mode.
 msgid "Normal"
 msgstr ""

src/Autoload/DrawingAlgos.gd

@@ -53,8 +53,18 @@ func blend_layers(
 		if DisplayServer.get_name() == "headless":
 			blend_layers_headless(image, project, layer, cel, origin)
 		else:
-			var cel_image := layer.display_effects(cel)
-			textures.append(cel_image)
+			if layer is GroupLayer and layer.blend_mode != BaseLayer.BlendModes.PASS_THROUGH:
+				var cel_image := (layer as GroupLayer).blend_children(frame)
+				textures.append(cel_image)
+			else:
+				var cel_image := layer.display_effects(cel)
+				textures.append(cel_image)
+			if (
+				layer.is_blended_by_ancestor()
+				and not only_selected_cels
+				and not only_selected_layers
+			):
+				include = false
 			set_layer_metadata_image(layer, cel, metadata_image, ordered_index, include)
 	if DisplayServer.get_name() != "headless":
 		var texture_array := Texture2DArray.new()
@@ -84,9 +94,14 @@ func set_layer_metadata_image(
 		image.set_pixel(index, 1, Color())
 	# Store the clipping mask boolean
 	if layer.clipping_mask:
-		image.set_pixel(index, 3, Color.WHITE)
+		image.set_pixel(index, 3, Color.RED)
 	else:
 		image.set_pixel(index, 3, Color.BLACK)
+	if not include:
+		# Store a small red value as a way to indicate that this layer should be skipped
+		# Used for layers such as child layers of a group, so that the group layer itself can
+		# sucessfuly be used as a clipping mask with the layer below it.
+		image.set_pixel(index, 3, Color(0.2, 0.0, 0.0, 0.0))
 
 
 func blend_layers_headless(

src/Classes/Layers/BaseLayer.gd

@@ -1,3 +1,4 @@
+# gdlint: ignore=max-public-methods
 class_name BaseLayer
 extends RefCounted
 ## Base class for layer properties. Different layer types extend from this class.
@@ -9,7 +10,8 @@ signal visibility_changed  ## Emits when [member visible] is changed.
 ## is the blend layer, and the bottom layer is the base layer.
 ## For more information, refer to: [url]https://en.wikipedia.org/wiki/Blend_modes[/url]
 enum BlendModes {
-	NORMAL,  ## The blend layer colors are simply placed on top of the base colors.
+	PASS_THROUGH = -2,  ## Only for group layers. Ignores group blending, like it doesn't exist.
+	NORMAL = 0,  ## The blend layer colors are simply placed on top of the base colors.
 	DARKEN,  ## Keeps the darker colors between the blend and the base layers.
 	MULTIPLY,  ## Multiplies the numerical values of the two colors, giving a darker result.
 	COLOR_BURN,  ## Darkens by increasing the contrast between the blend and base colors.
@@ -124,6 +126,17 @@ func is_locked_in_hierarchy() -> bool:
 	return locked
 
 
+## Returns [code]true[/code] if the layer has at least one ancestor
+## that does not have its blend mode set to pass through.
+func is_blended_by_ancestor() -> bool:
+	var is_blended := false
+	for ancestor in get_ancestors():
+		if ancestor.blend_mode != BlendModes.PASS_THROUGH:
+			is_blended = true
+			break
+	return is_blended
+
+
 ## Returns an [Array] of [BaseLayer]s that are ancestors of this layer.
 ## If there are no ancestors, returns an empty array.
 func get_ancestors() -> Array[BaseLayer]:
@@ -141,6 +154,20 @@ func get_hierarchy_depth() -> int:
 	return 0
 
 
+## Returns the layer's top most parent that is responsible for its blending.
+## For example, if a layer belongs in a group with its blend mode set to anything but pass through,
+## and that group has no parents of its own, then that group gets returned.
+## If that group is a child of another non-pass through group,
+## then the grandparent group is returned, and so on.
+## If the layer has no ancestors, or if they are set to pass through mode, it returns self.
+func get_blender_ancestor() -> BaseLayer:
+	var blender := self
+	for ancestor in get_ancestors():
+		if ancestor.blend_mode != BlendModes.PASS_THROUGH:
+			blender = ancestor
+	return blender
+
+
 ## Returns the path of the layer in the timeline as a [String].
 func get_layer_path() -> String:
 	if is_instance_valid(parent):
@@ -189,9 +216,12 @@ func link_cel(cel: BaseCel, link_set = null) -> void:
 ## Returns a copy of the [param cel]'s [Image] with all of the effects applied to it.
 ## This method is not destructive as it does NOT change the data of the image,
 ## it just returns a copy.
-func display_effects(cel: BaseCel) -> Image:
+func display_effects(cel: BaseCel, image_override: Image = null) -> Image:
 	var image := Image.new()
-	image.copy_from(cel.get_image())
+	if is_instance_valid(image_override):
+		image.copy_from(image_override)
+	else:
+		image.copy_from(cel.get_image())
 	if not effects_enabled:
 		return image
 	var image_size := image.get_size()
@@ -200,8 +230,10 @@ func display_effects(cel: BaseCel) -> Image:
 			continue
 		var shader_image_effect := ShaderImageEffect.new()
 		shader_image_effect.generate_image(image, effect.shader, effect.params, image_size)
-	# Inherit effects from the parents
+	# Inherit effects from the parents, if their blend mode is set to pass through
 	for ancestor in get_ancestors():
+		if ancestor.blend_mode != BlendModes.PASS_THROUGH:
+			break
 		if not ancestor.effects_enabled:
 			continue
 		for effect in ancestor.effects:

src/Classes/Layers/GroupLayer.gd

@@ -11,44 +11,130 @@ func _init(_project: Project, _name := "") -> void:
 
 
 ## Blends all of the images of children layer of the group layer into a single image.
-func blend_children(frame: Frame, origin := Vector2i.ZERO) -> Image:
+func blend_children(frame: Frame, origin := Vector2i.ZERO, apply_effects := true) -> Image:
 	var image := Image.create(project.size.x, project.size.y, false, Image.FORMAT_RGBA8)
 	var children := get_children(false)
 	if children.size() <= 0:
 		return image
-	var blend_rect := Rect2i(Vector2i.ZERO, project.size)
 	var textures: Array[Image] = []
-	var metadata_image := Image.create(children.size(), 4, false, Image.FORMAT_R8)
+	var metadata_image := Image.create(children.size(), 4, false, Image.FORMAT_RG8)
+	var current_child_index := 0
 	for i in children.size():
 		var layer := children[i]
 		if not layer.is_visible_in_hierarchy():
+			current_child_index += 1
 			continue
-		var cel := frame.cels[layer.index]
 		if layer is GroupLayer:
-			var blended_children: Image = layer.blend_children(frame, origin)
-			if DisplayServer.get_name() == "headless":
-				image.blend_rect(blended_children, blend_rect, origin)
-			else:
-				textures.append(blended_children)
-				DrawingAlgos.set_layer_metadata_image(layer, cel, metadata_image, i)
+			current_child_index = _blend_child_group(
+				image,
+				layer,
+				frame,
+				textures,
+				metadata_image,
+				current_child_index,
+				origin,
+				apply_effects
+			)
 		else:
-			if DisplayServer.get_name() == "headless":
-				DrawingAlgos.blend_layers_headless(image, project, layer, cel, origin)
-			else:
-				textures.append(layer.display_effects(cel))
-				DrawingAlgos.set_layer_metadata_image(layer, cel, metadata_image, i)
-
-	if DisplayServer.get_name() != "headless":
+			_include_child_in_blending(
+				image,
+				layer,
+				frame,
+				textures,
+				metadata_image,
+				current_child_index,
+				origin,
+				apply_effects
+			)
+		current_child_index += 1
+
+	if DisplayServer.get_name() != "headless" and textures.size() > 0:
 		var texture_array := Texture2DArray.new()
 		texture_array.create_from_images(textures)
 		var params := {
-			"layers": texture_array, "metadata": ImageTexture.create_from_image(metadata_image)
+			"layers": texture_array,
+			"metadata": ImageTexture.create_from_image(metadata_image),
+			"origin_x_positive": origin.x > 0,
+			"origin_y_positive": origin.y > 0,
 		}
 		var gen := ShaderImageEffect.new()
 		gen.generate_image(image, DrawingAlgos.blend_layers_shader, params, project.size)
+		if apply_effects:
+			image = display_effects(frame.cels[index], image)
 	return image
 
 
+func _include_child_in_blending(
+	image: Image,
+	layer: BaseLayer,
+	frame: Frame,
+	textures: Array[Image],
+	metadata_image: Image,
+	i: int,
+	origin: Vector2i,
+	apply_effects: bool
+) -> void:
+	var cel := frame.cels[layer.index]
+	if DisplayServer.get_name() == "headless":
+		DrawingAlgos.blend_layers_headless(image, project, layer, cel, origin)
+	else:
+		var cel_image: Image
+		if apply_effects:
+			cel_image = layer.display_effects(cel)
+		else:
+			cel_image = cel.get_image()
+		textures.append(cel_image)
+		DrawingAlgos.set_layer_metadata_image(layer, cel, metadata_image, i)
+		if origin != Vector2i.ZERO:
+			# Only used as a preview for the move tool, when used on a group's children
+			var test_array := [project.frames.find(frame), project.layers.find(layer)]
+			if test_array in project.selected_cels:
+				var origin_fixed := Vector2(origin).abs() / Vector2(cel_image.get_size())
+				metadata_image.set_pixel(i, 2, Color(origin_fixed.x, origin_fixed.y, 0.0, 0.0))
+
+
+## Include a child group in the blending process.
+## If the child group is set to pass through mode, loop through its children
+## and include them as separate images, instead of blending them all together.
+## Gets called recursively if the child group has children groups of its own,
+## and they are also set to pass through mode.
+func _blend_child_group(
+	image: Image,
+	layer: BaseLayer,
+	frame: Frame,
+	textures: Array[Image],
+	metadata_image: Image,
+	i: int,
+	origin: Vector2i,
+	apply_effects: bool
+) -> int:
+	var new_i := i
+	var blend_rect := Rect2i(Vector2i.ZERO, project.size)
+	var cel := frame.cels[layer.index]
+	if layer.blend_mode == BlendModes.PASS_THROUGH:
+		var children := layer.get_children(false)
+		for j in children.size():
+			var child := children[j]
+			if child is GroupLayer:
+				new_i = _blend_child_group(
+					image, child, frame, textures, metadata_image, i + j, origin, apply_effects
+				)
+			else:
+				new_i += j
+				metadata_image.crop(metadata_image.get_width() + 1, metadata_image.get_height())
+				_include_child_in_blending(
+					image, child, frame, textures, metadata_image, new_i, origin, apply_effects
+				)
+	else:
+		var blended_children := (layer as GroupLayer).blend_children(frame, origin)
+		if DisplayServer.get_name() == "headless":
+			image.blend_rect(blended_children, blend_rect, origin)
+		else:
+			textures.append(blended_children)
+			DrawingAlgos.set_layer_metadata_image(layer, cel, metadata_image, i)
+	return new_i
+
+
 # Overridden Methods:
 
 

src/Shaders/BlendLayers.gdshader

@@ -144,6 +144,20 @@ float border_trim(vec4 color, vec2 uv) {
 }
 
 
+int find_previous_opaque_layer(int index, vec2 metadata_size_float) {
+	for (int i = index - 1; i > 0; i--) {
+		float layer_index = float(i) / metadata_size_float.x;
+		float clipping_mask = texture(metadata, vec2(layer_index, 3.0 / metadata_size_float.y)).r;
+		// If the red value is ~0.2, it means that it should be skipped.
+		// Otherwise, return the index.
+		if (clipping_mask > 0.25 || clipping_mask < 0.15) {
+			return i;
+		}
+	}
+	return 0;
+}
+
+
 void fragment() {
 	ivec2 metadata_size = textureSize(metadata, 0) - 1;
 	vec2 metadata_size_float = vec2(metadata_size);
@@ -173,17 +187,18 @@ void fragment() {
 			current_origin.y = -current_origin.y;
 		}
 		// get origin of previous layer (used for clipping masks to work correctly)
-		vec2 prev_origin = texture(metadata, vec2(float(i - 1), 2.0 / metadata_size_float.y)).rg;
+		float clipping_mask_index = float(find_previous_opaque_layer(i, metadata_size_float));
+		vec2 prev_origin = texture(metadata, vec2(clipping_mask_index, 2.0 / metadata_size_float.y)).rg;
 		if (!origin_x_positive) {
-		  prev_origin.x = -prev_origin.x;
+			prev_origin.x = -prev_origin.x;
 		}
 		if (!origin_y_positive) {
-		  prev_origin.y = -prev_origin.y;
+			prev_origin.y = -prev_origin.y;
 		}
 		float current_opacity = texture(metadata, vec2(layer_index, 1.0 / metadata_size_float.y)).r;
 		vec2 uv = UV - current_origin;
 		vec4 layer_color = texture(layers, vec3(uv, float(i)));
-		vec4 prev_layer_color = texture(layers, vec3(UV - prev_origin, float(i - 1)));
+		vec4 prev_layer_color = texture(layers, vec3(UV - prev_origin, clipping_mask_index));
 		float clipping_mask = texture(metadata, vec2(layer_index, 3.0 / metadata_size_float.y)).r;
 		layer_color.a *= prev_layer_color.a * step(0.5, clipping_mask) + 1.0 * step(clipping_mask, 0.5);
 		layer_color.a = border_trim(layer_color, uv);