db2gen.lua

--[[

Build per-theme *FilePath.db2 (WDC1) tables from the community listfile.

The listfile is `FileDataID;path` per line (';' delimited, lowercase forward-slash paths).
Each theme collects the paths whose extension is in its set into one WDC1 table keyed by
FileDataID, with the path stored inline as a string. The WraithEngine DB2 reader decodes each
record to {uint32 id; uint32 pathOffset} and resolves the path through the table's string block.

Edit THEMES to add or restrict tables.

Usage:
  luajit db2gen.lua [--listfile PATH] [--out DIR] [--sep backslash|slash]
]]

-- Make sibling wdc1.lua importable regardless of the current working directory.
do
  local dir = (arg and arg[0] or ""):match("^(.*)[/\\]")
  if dir and dir ~= "" then
    package.path = dir .. "/?.lua;" .. package.path
  end
end

local wdc1 = require("wdc1")

local sfind, ssub = string.find, string.sub
local smatch, slower, sgsub = string.match, string.lower, string.gsub
local sformat = string.format

local IS_WINDOWS = package.config:sub(1, 1) == "\\"

local THEMES = {
  { name = "TextureFilePath", exts = { blp = true } },
  { name = "ModelFilePath",   exts = { m2 = true, wmo = true, skin = true, anim = true, phys = true, bone = true, skel = true } },
}

local DEFAULT_LISTFILE = [[.\listfile.csv]]

local function build_ext_to_theme()
  local ext_to_theme = {}
  for _, theme in ipairs(THEMES) do
    for ext in pairs(theme.exts) do
      ext_to_theme[ext] = theme.name
    end
  end
  return ext_to_theme
end

local function read_listfile(listfile_path, ext_to_theme, use_backslash)
  local buckets = {}
  for _, theme in ipairs(THEMES) do
    buckets[theme.name] = {}
  end

  local total = 0
  local kept = 0

  local f = assert(io.open(listfile_path, "r"))
  for line in f:lines() do
    total = total + 1

    local sep = sfind(line, ";", 1, true)
    if sep then
      local path = ssub(line, sep + 1)
      path = sgsub(path, "\r$", "")

      local ext = smatch(path, "%.([^.]+)$")
      if ext then
        local theme = ext_to_theme[slower(ext)]
        if theme then
          local fdid = tonumber(ssub(line, 1, sep - 1))
          if fdid then
            if use_backslash then
              path = sgsub(path, "/", "\\")
            end
            local bucket = buckets[theme]
            bucket[#bucket + 1] = { fdid, path }
            kept = kept + 1
          end
        end
      end
    end
  end
  f:close()

  return buckets, total, kept
end

local function default_out()
  local script = (arg and arg[0]) or "."
  local dir = smatch(script, "^(.*)[/\\]")
  if not dir or dir == "" then dir = "." end
  return dir .. "/out"
end

local function makedirs(path)
  if IS_WINDOWS then
    os.execute('if not exist "' .. path .. '" mkdir "' .. path .. '"')
  else
    os.execute('mkdir -p "' .. path .. '"')
  end
end

local function file_size(path)
  local f = assert(io.open(path, "rb"))
  local size = f:seek("end")
  f:close()
  return size
end

local function parse_args(argv)
  local opts = { listfile = DEFAULT_LISTFILE, out = nil, sep = "backslash" }
  local i = 1
  while argv and i <= #argv do
    local a = argv[i]
    if a == "--listfile" then
      i = i + 1; opts.listfile = argv[i]
    elseif a == "--out" then
      i = i + 1; opts.out = argv[i]
    elseif a == "--sep" then
      i = i + 1; opts.sep = argv[i]
      if opts.sep ~= "backslash" and opts.sep ~= "slash" then
        error("--sep must be 'backslash' or 'slash'")
      end
    else
      error("unknown argument: " .. tostring(a))
    end
    i = i + 1
  end
  return opts
end

local function main()
  local opts = parse_args(arg)
  local out = opts.out or default_out()
  makedirs(out)

  local ext_to_theme = build_ext_to_theme()
  local use_backslash = (opts.sep == "backslash")

  local t0 = os.clock()
  local buckets, total, kept = read_listfile(opts.listfile, ext_to_theme, use_backslash)
  print(sformat("read %d lines, kept %d in %.1fs", total, kept, os.clock() - t0))

  for _, theme in ipairs(THEMES) do
    local rows = buckets[theme.name]
    local out_path = out .. "/" .. theme.name .. ".db2"
    local n, string_size = wdc1.write_wdc1(out_path, rows)
    local fsize = file_size(out_path)
    print(sformat("  %s.db2: %d rows, string table %d bytes, file %d bytes", theme.name, n, string_size, fsize))

    local sample = wdc1.read_wdc1_sample(out_path, 3)
    for _, rec in ipairs(sample) do
      print(sformat("      verify id=%d -> %s", rec[1], rec[2]))
    end
  end
end

main()