Hi all,
I was looking into how to clip or mask one layer by another using a script. Here’s some r&d I’m passing along. Three prior threads on the subject are: one, two, three. If you can get what you want with the magic wand tool, replace color, alpha lock ink or other built-in tool, no need to read on.
The script has a bunch of limitations, such as not handling group layers, tilemaps or blend modes, only working in RGB color mode, only 2 layers at a time, etc.
The idea is to find an intersection – if any – between two cel images in layers n and n-1.
The blue rectangle in layer n is the mask, the red rectangle in layer n-1 is the source. The purple intersection is at the maximum of the top-left corners and the minimum of the bottom-right corners. The script creates a new cel at the intersection where non-transparent pixels from the blue rectangle adopt the color of the red.
The Rectangle class has methods to find the intersection. Also, this is why trimming cel images of excess alpha is helpful.
local targets = { "ACTIVE", "ALL", "RANGE" }
local defaults = {
target = "RANGE",
delOverLayer = false,
delUnderLayer = false,
pullFocus = false
}
local dlg = Dialog { title = "Layer Mask" }
dlg:combobox {
id = "target",
label = "Target:",
option = defaults.target,
options = targets
}
dlg:newrow { always = false }
dlg:check {
id = "delOverLayer",
label = "Delete:",
text = "Mask",
selected = defaults.delOverLayer
}
dlg:check {
id = "delUnderLayer",
text = "Source",
selected = defaults.delUnderLayer
}
dlg:newrow { always = false }
dlg:button {
id = "confirm",
text = "&OK",
focus = defaults.pullFocus,
onclick = function()
local activeSprite = app.activeSprite
if not activeSprite then
app.alert("There is no active sprite.")
return
end
local colorMode = activeSprite.colorMode
if colorMode ~= ColorMode.RGB then
app.alert("Only RGB color mode is supported.")
return
end
local overLayer = app.activeLayer
if not overLayer then
app.alert("There is no active layer.")
return
end
local overIndex = overLayer.stackIndex
if overIndex < 2 then
app.alert("There must be a layer beneath the active layer.")
return
end
-- A parent may be a sprite or a group layer.
-- Over and under layer should belong to same group.
local parent = overLayer.parent
local underIndex = overIndex - 1
local underLayer = parent.layers[underIndex]
if overLayer.isGroup or underLayer.isGroup then
app.alert("Group layers are not supported.")
return
end
-- Cache global functions used in loop.
local min = math.min
local max = math.max
-- Unpack arguments.
local args = dlg.data
local target = args.target or defaults.target
local delOverLayer = args.delOverLayer
and (not overLayer.isReference)
local delUnderLayer = args.delUnderLayer
and (not underLayer.isBackground)
and (not underLayer.isReference)
local frames = {}
if target == "ACTIVE" then
local activeFrame = app.activeFrame
if activeFrame then
frames[1] = activeFrame
end
elseif target == "RANGE" then
local appRange = app.range
local rangeFrames = appRange.frames
local rangeFramesLen = #rangeFrames
for i = 1, rangeFramesLen, 1 do
frames[i] = rangeFrames[i]
end
else
local activeFrames = activeSprite.frames
local activeFramesLen = #activeFrames
for i = 1, activeFramesLen, 1 do
frames[i] = activeFrames[i]
end
end
-- Unpack layer opacity.
local overLyrOpacity = overLayer.opacity
local underLyrOpacity = underLayer.opacity
-- Create new layer.
local compLayer = activeSprite:newLayer()
compLayer.name = string.format("Comp.%s.%s",
overLayer.name, underLayer.name)
compLayer.parent = parent
local framesLen = #frames
app.transaction(function()
for i = 1, framesLen, 1 do
local frame = frames[i]
local overCel = overLayer:cel(frame)
local underCel = underLayer:cel(frame)
if overCel and underCel then
local overImg = overCel.image
local overPos = overCel.position
local xTlOver = overPos.x
local yTlOver = overPos.y
local widthOver = overImg.width
local heightOver = overImg.height
local xBrOver = xTlOver + widthOver
local yBrOver = yTlOver + heightOver
local underImg = underCel.image
local underPos = underCel.position
local xTlUnder = underPos.x
local yTlUnder = underPos.y
local widthUnder = underImg.width
local heightUnder = underImg.height
local xBrUnder = xTlUnder + widthUnder
local yBrUnder = yTlUnder + heightUnder
-- Find intersection of over and under.
local xTlTarget = max(xTlOver, xTlUnder)
local yTlTarget = max(yTlOver, yTlUnder)
local xBrTarget = min(xBrOver, xBrUnder)
local yBrTarget = min(yBrOver, yBrUnder)
-- Intersection may be empty (invalid).
if xBrTarget > xTlTarget and yBrTarget > yTlTarget then
local overCelOpacity = overCel.opacity
local underCelOpacity = underCel.opacity
local overCompOpacity = (overLyrOpacity * overCelOpacity) // 0xff
local underCompOpacity = (underLyrOpacity * underCelOpacity) // 0xff
local widthTarget = xBrTarget - xTlTarget
local heightTarget = yBrTarget - yTlTarget
local trgImage = Image(widthTarget, heightTarget)
local trgPos = Point(xTlTarget, yTlTarget)
local trgItr = trgImage:pixels()
for elm in trgItr do
local xSprite = elm.x + xTlTarget
local ySprite = elm.y + yTlTarget
local xOver = xSprite - xTlOver
local yOver = ySprite - yTlOver
local overHex = overImg:getPixel(xOver, yOver)
local overAlpha = (overHex >> 0x18) & 0xff
overAlpha = (overAlpha * overCompOpacity) // 0xff
if overAlpha > 0 then
local xUnder = xSprite - xTlUnder
local yUnder = ySprite - yTlUnder
local underHex = underImg:getPixel(xUnder, yUnder)
local underAlpha = (underHex >> 0x18) & 0xff
underAlpha = (underAlpha * underCompOpacity) // 0xff
local compAlpha = (overAlpha * underAlpha) // 0xff
local compHex = (compAlpha << 0x18)
| (underHex & 0x00ffffff)
elm(compHex)
end
end
activeSprite:newCel(
compLayer, frame,
trgImage, trgPos)
end
end
end
end)
if delOverLayer then activeSprite:deleteLayer(overLayer) end
if delUnderLayer then activeSprite:deleteLayer(underLayer) end
app.activeLayer = compLayer
app.refresh()
end
}
dlg:button {
id = "cancel",
text = "&CANCEL",
onclick = function()
dlg:close()
end
}
dlg:show { wait = false }
I usually regret posting a duplicate script because a week later I realize there’s some overlooked problem. 
But anyway, the benefit is someone doesn’t have to download a lot of Lua scripts to use and/or develop one script. The longer-term version is on Github.
Here is a use case:
The first image from the left is the gradient with the color I want clipped. The second is a radial gradient from transparent to opaque. The third is a hexagon. In the fourth image, the layer mask script is used to cut out the hexagon from the radial gradient. The intersection is then used to mask out the color gradient for the final image.
The script was also handy when applying a gradient or pattern to text with anti-aliasing (from Edit > Insert Text).
I found it could improve the visuals of a baked onion skin as well.
I used an animation of Makoto and Ibuki from I assume Street Fighter III. The original was found here. In the background, I created a linear gradient from #8ffffe on the left to #7b0214 on the right. The onion skin – with no tinting – was used to cut out silhouettes from the gradient. The onion skin was then placed over the gradient silhouettes and its layer opacity reduced.
You could probably modify this so the script reads a black-to-white scale from the mask instead of alpha. However, I find that gets bogged down in the question of which greyscale conversion is preferred. I found it easy enough to invert a mask by going to Edit > Invert ... and selecting only the alpha channel; so I didn’t consider adding it to the script.
Best,
Jeremy