Aseprite: export tilemap to GBASM format

export-tilemap-to-gbasm.lua

--  Author: Matcha, with some work from Anon of the referenced Pastebin
--  Author URL: https://hamac303.eu/
--  Put in your .config/aseprite/scripts folder
--  Use by selecting your tilemap layer (not your tileset)
--  This targets the clipboard, for simplicity
--  License: WTFPL

local debugArray = {}
local ERROR_COLOR = 536870911 -- this is a color value returned in some error cases
local enableDebug = false
local function addToDebug(s)
  table.insert(debugArray, s)
end

local function formatDebug()
  if enableDebug then return "; DEBUG: " .. table.concat(debugArray, "\n; DEBUG:") end
  return ""
end
local function toPascalCase(s)
  local function titleCase(first, rest)
    return first:upper() .. rest:lower()
  end

  return string.gsub(string.gsub(s, "(%a)([%w_']*)", titleCase), " ", "")
end

local function rgbds_row(items, leftOffset, padWith, padCols)
  local formattedAry, rightPaddingTable, leftPaddingTable = {}, {}, {}

  local padfmt = string.format("$%02x", padWith)
  while leftOffset > 0 do
    table.insert(leftPaddingTable, padfmt)
    leftOffset = leftOffset - 1
  end
  for _, tile in ipairs(items) do
    table.insert(formattedAry, string.format("$%02x", tile))
  end


  if padCols and #leftPaddingTable + #rightPaddingTable + #formattedAry < 32 then
    while #leftPaddingTable + #rightPaddingTable + #formattedAry < 32 do
      table.insert(rightPaddingTable, padfmt)
    end
  end
  local finalTable = {}
  table.move(leftPaddingTable, 1, #leftPaddingTable, #finalTable + 1, finalTable)
  table.move(formattedAry, 1, #formattedAry, #finalTable + 1, finalTable)
  table.move(rightPaddingTable, 1, #rightPaddingTable, #finalTable + 1, finalTable)

  return string.format("db %s", table.concat(finalTable, ", ")
  )
end

local function export_to_gbset(app)
  if TilesetMode == nil then return app.alert "Use Aseprite 1.3" end
  local spr = app.activeSprite
  addToDebug(string.format("Sprite: %s", spr))
  if not spr then return end

  local d = Dialog("Export Tilemap as (GB)ASM")
  d:label { id = "lab1", label = "", text = "Export Tilemap to clipboard" }
      :number { id = "cols", label = "Columns in active map: ", text = "20" }
      :number { id = "rows", label = "Rows in active map: ", text = "18" }
      :check { id = "autodetect", label = "Autodetect cols and rows", text = "Enable", selected = true }
      :check { id = "padRows", label = "Pad to 32 rows (wasteful)", text = "Enable", selected = false }
      :check { id = "padCols", label = "Pad to 32 cols (recommended)", text = "Enable", selected = true }
      :number { id = "padTileNum", label = "Pad with this tile ID: ", text = "0" }
      :label { id = "lab2", label = "", text = "This will export to clipboard in GBASM format (RGBDS)." }
      :label { id = "lab3", label = "", text = "If columns or rows less than 32, we will pad with $00 the remainder for each line." }
      :label { id = "lab4", label = "", text = "The Gameboy viewport is 160x144 (20x18 tiles), making a maximum of 21x19 tiles visible (with some scrolling)." }
      :number { id = "tilewidth", label = "Tile width (if tgt != GB): ", text = "8" }
      :number { id = "tileheight", label = "Tile width (if tgt != GB): ", text = "8" }

      :separator {}
      :button { id = "ok", text = "&OK", focus = true }
      :button { text = "&Cancel" }
      :show()

  local spr_width, spr_height = spr.width, spr.height
  addToDebug(string.format(";; Sprite width: %d, height: %d", spr_width, spr_height))
  local data = d.data
  if not data.ok then app.alert("cannot proceed, no proper data") end
  local lay = app.activeLayer
  if not lay.isTilemap then return app.alert("Layer is not tilemap") end
  pc = app.pixelColor
  local nCols = data.cols
  local nRows = data.rows
  local tilewidth, tileheight = data.tilewidth, data.tileheight
  local padWith = data.padTileNum
  local fbR <const> = 32
  local fbC <const> = 32
  local outputData = {}
  for celNum, c in ipairs(lay.cels) do
    local img = c.image
    local cel_pos = c.bounds
    local cel_offset_left, cel_offset_top = cel_pos.x / tilewidth, cel_pos.y / tileheight
    local cel_offset_bottom = (spr_height - (cel_pos.height + cel_pos.y)) / tileheight
    local cel_offset_right = (spr_width - (cel_pos.width + cel_pos.x)) / tilewidth
    if cel_offset_top > 0 then
      local to_i = cel_offset_top
      addToDebug(string.format(";; Compensating top offset of %d", to_i))
      while to_i > 0 do
        to_i = to_i - 1
        table.insert(outputData, rgbds_row({}, cel_offset_left, padWith, data.padCols))
      end
    end
    if img.colorMode ~= ColorMode.TILEMAP then
      return app.alert("Cannot work with a non-tilemap")
    end
    if data.autodetect then
      nCols = img.width  -- careful: this is in tiles!
      nRows = img.height -- likewise, this is in tiles
      addToDebug(string.format("Autodetected %d rows and %d cols", nRows, nCols))
    end
    local outputRow = {}
    for y = cel_offset_top, nRows + cel_offset_top - 1, 1 do
      for x = cel_offset_left, nCols + cel_offset_left - 1, 1 do
        local px = img:getPixel(x, y)
        local tileIdx = pc.tileI(px)
        if tileIdx == ERROR_COLOR then tileIdx = padWith end -- handle special missing color
        table.insert(outputRow, tileIdx)
      end
      addToDebug(string.format("Generated row of length %d", #outputRow))
      table.insert(outputData, rgbds_row(outputRow, cel_offset_left, padWith, data.padCols))
      outputRow = {}
    end
    if data.padRows and nRows < 32 then
      while #outputData < fbC do
        table.insert(outputData, rgbds_row({}, padWith, data.padCols))
      end
    end
    local labelName = toPascalCase(lay.name)
    app.clipboard.text = string.format(
      [[
; generated using export-tilemap-to-gbasm.lua
; source layer: %s cel %d
; parameters
;; pad to 32 rows: %s
;; autodetect rows and cols: %s
;; rows: %d
;; cols: %d
;; pad with tile ID: $%02x
;; %s
%sTM:
%s
End%sTM:
;; end generated]],
      lay.name, celNum, data.padRows, data.autodetect,
      nRows, nCols, padWith, formatDebug(),
      labelName,
      table.concat(outputData, "\n"),
      labelName)
  end -- end cel
end

export_to_gbset(app)