[Script] 4color per image checkup and overflow heatmap

Scripts & Extensions
1

Hello, this is a script, that checks if more than 4 colors are used on sprite work. Works in RGB mode, not sure about others.

It generates image from all layers combined (if visible), analyzes colors. If there more than 4 colors present, picks the least common ones and marks them onto heatmaps, afterwards dumping them onto separate new layer.

Displays basic app alerts to tell if something was done or nothing was done.

As it works per pixel basis and is no way optimized for performance, might chug a bit on bigger pictures.

Created to work with PokeTCG sprites that use 4 colors and are 64x48. (image size restrictions removed)

-----------------------------------------------------------------------
-- const
-----------------------------------------------------------------------
local DEFAULT_DUMPED_LAYER_NAME = "ERROR_Heatmap"
-----------------------------------------------------------------------
-- bb checks
-----------------------------------------------------------------------
local function isSpriteReady(sprite, frame)
	return sprite ~= nil and frame ~= nil
end
-----------------------------------------------------------------------
-- prime img funcs
-----------------------------------------------------------------------
local function drawToLayer(layer, frame, outputImage)
	if not layer.isVisible then return end

	if layer.isGroup then
		for _, child in ipairs(layer.layers) do
			drawToLayer(child, frame, outputImage) -- recursive call
		end
		return
	end

	local cel = layer:cel(frame)
	if not cel then return end

	outputImage:drawImage(cel.image, cel.position, layer.opacity, layer.blendMode)
end

local function drawToImage(sprite, frame)
	local result = Image(sprite.width, sprite.height, sprite.colorMode)
	for _, layer in ipairs(sprite.layers) do
		drawToLayer(layer, frame, result)
	end
	return result
end

local function dumpImageToNewLayer(sprite, frame, image, layerName)
	local layer = sprite:newLayer()
	layer.name = layerName

	sprite:newCel(layer, frame, image)

	return layer
end
-----------------------------------------------------------------------
-- manip img funcs
-----------------------------------------------------------------------
local function genEmpty(image)
	local empty = Image(image.width, image.height, image.colorMode)
	local el = Color { r = 0, g = 0, b = 0, a = 0 }
	empty:clear(el)
	return empty
end

local function genAnyPWhiteImage(pa, image)
	pa = math.max(0, math.min(255, pa))
	local overlay = Image(image.width, image.height, image.colorMode)
	local anyPWhite = Color { r = 255, g = 255, b = 255, a = pa }
	overlay:clear(anyPWhite)
	return overlay
end

local function genHeatMap(sourceImage, targetPx)
	local w, h = sourceImage.width, sourceImage.height
	local heatmap = Image(w, h, sourceImage.colorMode)

	local red = Color { r = 255, g = 0, b = 0, a = 255 }
	local transparent = Color { r = 0, g = 0, b = 0, a = 0 }

	for y = 0, h - 1 do
		for x = 0, w - 1 do
			if sourceImage:getPixel(x, y) == targetPx then
				heatmap:putPixel(x, y, red)
			else
				heatmap:putPixel(x, y, transparent)
			end
		end
	end

	return heatmap
end

local function mergeImages(sourceImage, targetImage)
	local raw = Image(sourceImage.width, sourceImage.height, sourceImage.colorMode)
	local pos = Point(0, 0)
	raw:drawImage(sourceImage, pos)
	raw:drawImage(targetImage, pos)
	return raw
end
-----------------------------------------------------------------------
-- analysis
-----------------------------------------------------------------------
local function calcOccurrence(image)
	local seen = {}

	for y = 0, image.height - 1 do
	  for x = 0, image.width - 1 do
		local px = image:getPixel(x, y)
		seen[px] = (seen[px] or 0) + 1
	  end
	end

	return seen
end

local function rowCount(dt)
	local ct = 0
	for _ in pairs(dt) do
		ct = ct + 1
	end
	return ct
end

local function filterLeastCommonColors(seen)
	local arr = {} 
	for px, ct in pairs(seen) do 
		arr[#arr + 1] = { px = px, ct = ct } -- transform
	end

	table.sort(arr, -- sort desc
		function(a, b)
			return a.ct > b.ct
		end
	)
	
	local fr = {} -- final result
	for i = 5, #arr do -- collect all colors above 4 that are lowest
		fr[#fr + 1] = arr[i].px
	end
	
	return fr
end
-----------------------------------------------------------------------
-- main func
-----------------------------------------------------------------------
local function main()
	local sprite = app.activeSprite
	local frame = app.activeFrame
	if isSpriteReady(sprite, frame) then
		local combinedImage = drawToImage(sprite, frame) -- generate combined pic for analysis
	
		-- analyze stuff
		local seen = calcOccurrence(combinedImage)
		local rowCount = rowCount(seen)
		if rowCount > 4 then -- apply only if more than 4 colors
			app.transaction -- for single undo
			(
				function()
					local bleach = genAnyPWhiteImage(128, combinedImage)

					local defaultHeatmap = genEmpty(combinedImage) -- initial dump
					local leastCommonColors = filterLeastCommonColors(seen)
					for i = 1, #leastCommonColors do
						local px = leastCommonColors[i]
						local heatmap = genHeatMap(combinedImage, px) -- gen heatmap for color
						defaultHeatmap = mergeImages(defaultHeatmap, heatmap) -- merge new heatmap onto old
					end
					
					local finalHeatmap = mergeImages(bleach, defaultHeatmap)
					dumpImageToNewLayer(sprite, frame, finalHeatmap, DEFAULT_DUMPED_LAYER_NAME) -- bleached + heatmap onto new layer for visual aid
				end
			)

			app.alert("Had some extra stuff.")
		else
			app.alert("Looks good.")
		end
	else
		app.alert("Bad image.")
	end
end
-----------------------------------------------------------------------
-----------------------------------------------------------------------
-----------------------------------------------------------------------
main()