How To Find TopMost Visible Layer Under Cursor

Scripts & Extensions
1

Hi all,

Below is a Lua script to find the topmost visible layer under the cursor. [Edit: To be clear, this is for people like me who have issues with the Move Tool’s Auto Select Layer.]

Scripts can be installed under File > Scripts > Open Script Folder. They can be assigned shortcuts under Edit > Keyboard Shortcuts.

A longer term version of the script can be found in this repo. That version has a bit more code to deal with tile maps (which may have flipped or rotated tiles).

local function appendLeaves(layer, array)
    if layer.isVisible then
        if layer.isGroup then
            local childLayers = layer.layers
            if childLayers then
                local lenChildLayers = #childLayers
                local i = 0
                while i < lenChildLayers do
                    i = i + 1
                    appendLeaves(childLayers[i], array)
                end
            end
        elseif not layer.isReference then
            array[#array + 1] = layer
        end
    end
    return array
end

local function getLayerHierarchy(sprite)
    local array = {}
    local layers = sprite.layers
    local lenLayers = #layers
    local i = 0
    while i < lenLayers do
        i = i + 1
        appendLeaves(layers[i], array)
    end
    return array
end

local sprite = app.sprite
if not sprite then return end

local frObj = app.frame
if not frObj then return end

local editor = app.editor
if not editor then return end

local mouse = editor.spritePos
local xMouse = mouse.x
local yMouse = mouse.y

if xMouse < 0 or yMouse < 0 then return end

local specSprite = sprite.spec
local wSprite = specSprite.width
local hSprite = specSprite.height
local colorMode = specSprite.colorMode
local alphaIndex = specSprite.transparentColor

if xMouse >= wSprite or yMouse >= hSprite then return end

local palette = sprite.palettes[1]
local lenPalette = #palette

local layers = getLayerHierarchy(sprite)
local lenLayers = #layers

local i = lenLayers + 1
while i > 1 do
    i = i - 1
    local layer = layers[i]
    if layer.isBackground then
        app.layer = layer
        local appRange = app.range
        if appRange.sprite == sprite then
            appRange.layers = { layer }
        end
        app.refresh()
        return
    end

    local cel = layer:cel(frObj)
    if cel then
        local celBounds = cel.bounds

        local xtlCel = celBounds.x
        local ytlCel = celBounds.y

        local wCel = celBounds.width
        local hCel = celBounds.height

        local xbrCel = xtlCel + wCel - 1
        local ybrCel = ytlCel + hCel - 1

        if xMouse >= xtlCel and yMouse >= ytlCel
            and xMouse <= xbrCel and yMouse <= ybrCel then
            local image = cel.image

            local xLocal = xMouse - xtlCel
            local yLocal = yMouse - ytlCel

            local aLayer8 = layer.opacity or 255
            local aCel8 = cel.opacity
            local aComp01 = (aLayer8 / 255.0) * (aCel8 / 255.0)

            local isNonZero = false
            local isTileMap = layer.isTilemap
            if isTileMap then
                local tileSet = layer.tileset
                if tileSet then
                    local tileGrid = tileSet.grid
                    local tileSize = tileGrid.tileSize

                    local wTile = tileSize.width
                    local hTile = tileSize.height

                    local xMap = xLocal // wTile
                    local yMap = yLocal // hTile

                    local tileEntry = image:getPixel(xMap, yMap)
                    local tileIndex = app.pixelColor.tileI(tileEntry)
                    local lenTileSet = #tileSet

                    if tileIndex > 0 and tileIndex < lenTileSet then
                        -- Simplified.
                        isNonZero = aComp01 > 0.0
                    end
                end
            else
                local dataInt = image:getPixel(xLocal, yLocal)

                if colorMode == ColorMode.RGB then
                    local a8 = app.pixelColor.rgbaA(dataInt)
                    local a01 = aComp01 * (a8 / 255.0)
                    isNonZero = a01 > 0.0
                elseif colorMode == ColorMode.GRAY then
                    local a8 = app.pixelColor.grayaA(dataInt)
                    local a01 = aComp01 * (a8 / 255.0)
                    isNonZero = a01 > 0.0
                elseif colorMode == ColorMode.INDEXED then
                    if dataInt ~= alphaIndex
                        and dataInt >= 0 and dataInt < lenPalette then
                        local aseColor = palette:getColor(dataInt)
                        local a8 = aseColor.alpha
                        local a01 = aComp01 * (a8 / 255.0)
                        isNonZero = a01 > 0.0
                    end
                end
            end

            if isNonZero then
                app.layer = layer
                local appRange = app.range
                if appRange.sprite == sprite then
                    appRange.layers = { layer }
                end
                app.refresh()
                return
            end -- End pixel is non zero.
        end     -- End mouse within bounds.
    end         -- End cel is not nil.
end             -- End layers loop.

local appRange = app.range
if appRange.sprite == sprite then
    appRange:clear()
end
app.refresh()

This script ignores non-visible layers and reference layers. Because background layers treat colors as opaque that would be transparent on other layers, there’s an early return for that case.

For some, setting the layer to active is enough; others might want a range around the layer in the timeline. I included the range here because it makes it easier to see the script working.

1 Like