From 89a8df5060b6f6e609952556992b17f0c833a8ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bagel=E2=84=A2?= Date: Fri, 25 Nov 2022 08:13:57 -0500 Subject: [PATCH 1/3] Coroutine uploading --- lua/d3bot/1_navmesh_sv.lua | 44 +++++++++++++++++++++------------ lua/d3bot/2_mapnavmeshui_cl.lua | 3 +++ 2 files changed, 31 insertions(+), 16 deletions(-) diff --git a/lua/d3bot/1_navmesh_sv.lua b/lua/d3bot/1_navmesh_sv.lua index 23f9f9d5..fb3a93b3 100644 --- a/lua/d3bot/1_navmesh_sv.lua +++ b/lua/d3bot/1_navmesh_sv.lua @@ -17,27 +17,39 @@ return function(lib) util.AddNetworkString(lib.MapNavMeshNetworkStr) function lib.UploadMapNavMesh(plOrPls) - local rawData = util.Compress(lib.MapNavMesh:Serialize()) or "" - local dataLen = rawData:len() - local maxChunkSize = 2^16 - 10 -- Leave 10 bytes for other stuff than the data. + local co = coroutine.create(function() + local rawData = util.Compress(lib.MapNavMesh:Serialize()) or "" + local dataLen = rawData:len() + local maxChunkSize = 2^16 - 10 -- Leave 10 bytes for other stuff than the data. + + for i = 1, dataLen, maxChunkSize do + local dataLeft = dataLen + 1 - i + local chunkSize = math.min(maxChunkSize, dataLeft) + local subDataComp = string.sub(rawData, i, i + chunkSize - 1) + + net.Start(lib.MapNavMeshNetworkStr, false) + net.WriteBool(false) + net.WriteUInt(chunkSize, 16) + net.WriteData(subDataComp, chunkSize) + net.Send(plOrPls) - for i = 1, dataLen, maxChunkSize do - local dataLeft = dataLen + 1 - i - local chunkSize = math.min(maxChunkSize, dataLeft) - local subDataComp = string.sub(rawData, i, i + chunkSize - 1) + print(chunkSize, dataLeft) + coroutine.yield() + end + + -- Finish the transfer. net.Start(lib.MapNavMeshNetworkStr, false) - net.WriteBool(false) - net.WriteUInt(chunkSize, 16) - net.WriteData(subDataComp, chunkSize) + net.WriteBool(true) + net.WriteUInt(0, 16) net.Send(plOrPls) - end - -- Finish the transfer. - net.Start(lib.MapNavMeshNetworkStr, false) - net.WriteBool(true) - net.WriteUInt(0, 16) - net.Send(plOrPls) + hook.Remove("Think", "d3bot.MapNavMeshUpload") + end) + + hook.Add("Think", "d3bot.MapNavMeshUpload", function() + coroutine.resume(co) + end) end file.CreateDir(lib.MapNavMeshDir) diff --git a/lua/d3bot/2_mapnavmeshui_cl.lua b/lua/d3bot/2_mapnavmeshui_cl.lua index d8222d83..799afb61 100644 --- a/lua/d3bot/2_mapnavmeshui_cl.lua +++ b/lua/d3bot/2_mapnavmeshui_cl.lua @@ -60,6 +60,7 @@ return function(lib) if isEnabled then nodeVisualProperties = {} hook.Add("Think", hooksId, function() + if not lib.MapNavMesh then return end cursoredItemOrNil = lib.MapNavMesh:GetCursoredItemOrNil(LocalPlayer()) local smartDraw = D3bot.Convar_Navmeshing_SmartDraw:GetBool() @@ -96,6 +97,7 @@ return function(lib) end) hook.Add("PostDrawOpaqueRenderables", hooksId, function(bDrawingDepth, bDrawingSkybox) + if not lib.MapNavMesh then return end if forceDrawInSkyboxCounter > 1 then forceDrawInSkybox = true print("D3bot: Force drawing of navmesh in skybox, it won't draw correctly otherwise.") @@ -408,6 +410,7 @@ return function(lib) end end) hook.Add("HUDPaint", hooksId, function() + if not lib.MapNavMesh then return end local smartDraw = D3bot.Convar_Navmeshing_SmartDraw:GetBool() local maxDrawingDistanceSqr = math.pow(math.min(D3bot.Convar_Navmeshing_DrawDistance:GetInt(), 500), 2) if maxDrawingDistanceSqr <= 0 then maxDrawingDistanceSqr = 500*500 end From cb5f2dd827de5a2d2165b9a728b1ad4a6d8f605a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bagel=E2=84=A2?= Date: Sun, 27 Nov 2022 10:06:00 -0500 Subject: [PATCH 2/3] Async improvements, race condition fix --- lua/d3bot/1_navmesh_sv.lua | 44 ++++++++++++++++------- lua/d3bot/sh_async.lua | 74 +++++++++++++++++++++++++++++++++----- 2 files changed, 97 insertions(+), 21 deletions(-) diff --git a/lua/d3bot/1_navmesh_sv.lua b/lua/d3bot/1_navmesh_sv.lua index fb3a93b3..8644fe80 100644 --- a/lua/d3bot/1_navmesh_sv.lua +++ b/lua/d3bot/1_navmesh_sv.lua @@ -15,13 +15,19 @@ return function(lib) return file.Exists(lib.GetMapNavMeshPath(mapName), "DATA") end - util.AddNetworkString(lib.MapNavMeshNetworkStr) - function lib.UploadMapNavMesh(plOrPls) - local co = coroutine.create(function() + local uploadWorker = nil + local uploading = false + + local NextTick = 0 + local function MapNavMeshUpload() + if NextTick > CurTime() then return end + NextTick = CurTime() + 0.2 + + local running, err = D3bot.Async.Run(uploadWorker, function(worker) local rawData = util.Compress(lib.MapNavMesh:Serialize()) or "" local dataLen = rawData:len() local maxChunkSize = 2^16 - 10 -- Leave 10 bytes for other stuff than the data. - + for i = 1, dataLen, maxChunkSize do local dataLeft = dataLen + 1 - i local chunkSize = math.min(maxChunkSize, dataLeft) @@ -31,9 +37,7 @@ return function(lib) net.WriteBool(false) net.WriteUInt(chunkSize, 16) net.WriteData(subDataComp, chunkSize) - net.Send(plOrPls) - - print(chunkSize, dataLeft) + net.Send(worker:GetData()) coroutine.yield() end @@ -42,14 +46,30 @@ return function(lib) net.Start(lib.MapNavMeshNetworkStr, false) net.WriteBool(true) net.WriteUInt(0, 16) - net.Send(plOrPls) + net.Send(worker:GetData()) + end) + if err then + print(string.format("D3bot: Navmesh upload worker failed: %s", err)) + end + + if not running then hook.Remove("Think", "d3bot.MapNavMeshUpload") - end) + uploading = false + end + end - hook.Add("Think", "d3bot.MapNavMeshUpload", function() - coroutine.resume(co) - end) + util.AddNetworkString(lib.MapNavMeshNetworkStr) + + function lib.UploadMapNavMesh(plOrPls) + if not uploading then + uploadWorker = D3bot.Async.CreateWorker() + uploadWorker:SetData(plOrPls) + hook.Add("Think", "d3bot.MapNavMeshUpload", MapNavMeshUpload) + uploading = true + elseif uploadWorker then + uploadWorker:SetData(plOrPls) + end end file.CreateDir(lib.MapNavMeshDir) diff --git a/lua/d3bot/sh_async.lua b/lua/d3bot/sh_async.lua index 9fd2eac6..6dbb0b23 100644 --- a/lua/d3bot/sh_async.lua +++ b/lua/d3bot/sh_async.lua @@ -5,30 +5,86 @@ local ASYNC = D3bot.Async ---Runs the given function asynchronously. ---This can be used to run blocking functions *seemingly* parallel to other code. ---There is no parallelism, so ASYNC.Run has to be called until it returns false, which means the function has ended, or has panicked. ----@param state table @Contains the coroutine and its state. Initialize and reuse an empty table for this. +---@param worker table @Contains the coroutine and its state. Use a table created with D3bot.CreateWorker() ---@param func function @The function to call asynchronously. ---@return boolean running ---@return string | nil err @The error message, if there was any error. -function ASYNC.Run(state, func) +function ASYNC.Run(worker, func) -- Start coroutine on the first call. - local cr = state[1] + local cr = worker:GetCoroutine() if not cr then - cr = coroutine.create(func) - state[1] = cr + cr = coroutine.create(function(used_worker) + local graceful, _ = pcall(func, used_worker) + if graceful then + worker:SetFinished(true) + end + end) + worker:SetCoroutine(cr) end -- Resume coroutine, catch and print any error. - local succ, msg = coroutine.resume(cr) - if not succ then + local succ, msg = coroutine.resume(cr, worker) + if not succ and not worker:GetFinished() then -- Coroutine ended unexpectedly. - print(string.format("D3bot: %s failed: %s.", cr, msg)) + worker = nil return false, string.format("%s failed: %s", cr, msg) end -- Check if the coroutine finished. We will never encounter "running", as we don't call coroutine.status from inside the coroutine. - if coroutine.status(cr) ~= "suspended" then + if coroutine.status(cr) ~= "suspended" or worker:GetFinished() then + worker = nil return false, nil end + worker:SetActive(true) + return true, nil end + +---Defers the given function to be later run asynchronously. This does not immediately start running +---This can be used to run blocking functions *seemingly* parallel to other code. +---There is no parallelism, so ASYNC.Run has to be called until it returns false, which means the function has ended, or has panicked. +---@param worker table @Contains the coroutine and its state. Use a table created with D3bot.CreateWorker() +---@param func function @The function to call asynchronously. +---@return boolean created +function ASYNC.Defer(worker, func) + -- Start coroutine on the first call. + local cr = worker:GetCoroutine() + if not cr then + cr = coroutine.create(function(used_worker) + local graceful, err = pcall(func, used_worker) + if graceful then + worker:SetFinished(true) + elseif err then + print(err) + end + end) + worker:SetCoroutine(cr) + end + + -- Check if the coroutine finished. We will never encounter "running", as we don't call coroutine.status from inside the coroutine. + if coroutine.status(cr) ~= "suspended" or worker:GetFinished() then + worker = nil + return false + end + + return true +end + +local WORKER = {} +WORKER.__index = WORKER +AccessorFunc(WORKER, "cr", "Coroutine") +AccessorFunc(WORKER, "active", "Active") +AccessorFunc(WORKER, "finished", "Finished") +AccessorFunc(WORKER, "data", "Data") + +---Creates a new worker structure for use with +---@return table worker +function ASYNC.CreateWorker() + -- cr the coroutine + -- active shows if the worker has started, it will be set to nil when finished by ASYNC.Run + -- finished shows if the worker has finished, it will be true if the couroutine finishes successfully + -- data ambiguous persistent data (for organization) + local newWorker = { cr = nil, active = false, finished = false, data = nil } + return setmetatable(newWorker, WORKER) +end From ee66101d51da3af1a17accaf1d81070ac7c38dbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bagel=E2=84=A2?= Date: Sun, 27 Nov 2022 11:02:21 -0500 Subject: [PATCH 3/3] Fixes and removed redundant code --- lua/d3bot/1_navmesh_sv.lua | 17 ++++++++++++----- lua/d3bot/sh_async.lua | 28 ++++++++-------------------- 2 files changed, 20 insertions(+), 25 deletions(-) diff --git a/lua/d3bot/1_navmesh_sv.lua b/lua/d3bot/1_navmesh_sv.lua index 8644fe80..1ac75f8a 100644 --- a/lua/d3bot/1_navmesh_sv.lua +++ b/lua/d3bot/1_navmesh_sv.lua @@ -15,6 +15,7 @@ return function(lib) return file.Exists(lib.GetMapNavMeshPath(mapName), "DATA") end + local uploadQueue = {} local uploadWorker = nil local uploading = false @@ -54,21 +55,27 @@ return function(lib) end if not running then - hook.Remove("Think", "d3bot.MapNavMeshUpload") - uploading = false + if not uploadQueue[1] then + hook.Remove("Think", "d3bot.MapNavMeshUpload") + uploading = false + elseif uploadWorker:GetFinished() then + uploadWorker = D3bot.Async.CreateWorker() + uploadWorker:SetData(table.remove(uploadQueue, 1)) + uploading = true + end end end util.AddNetworkString(lib.MapNavMeshNetworkStr) function lib.UploadMapNavMesh(plOrPls) + uploadQueue[#uploadQueue + 1] = plOrPls + if not uploading then uploadWorker = D3bot.Async.CreateWorker() - uploadWorker:SetData(plOrPls) + uploadWorker:SetData(table.remove(uploadQueue, 1)) hook.Add("Think", "d3bot.MapNavMeshUpload", MapNavMeshUpload) uploading = true - elseif uploadWorker then - uploadWorker:SetData(plOrPls) end end diff --git a/lua/d3bot/sh_async.lua b/lua/d3bot/sh_async.lua index 6dbb0b23..456864af 100644 --- a/lua/d3bot/sh_async.lua +++ b/lua/d3bot/sh_async.lua @@ -13,26 +13,21 @@ function ASYNC.Run(worker, func) -- Start coroutine on the first call. local cr = worker:GetCoroutine() if not cr then - cr = coroutine.create(function(used_worker) - local graceful, _ = pcall(func, used_worker) - if graceful then - worker:SetFinished(true) - end - end) + cr = coroutine.create(func) worker:SetCoroutine(cr) end -- Resume coroutine, catch and print any error. local succ, msg = coroutine.resume(cr, worker) - if not succ and not worker:GetFinished() then + if not succ then -- Coroutine ended unexpectedly. - worker = nil + worker:SetFinished() return false, string.format("%s failed: %s", cr, msg) end -- Check if the coroutine finished. We will never encounter "running", as we don't call coroutine.status from inside the coroutine. - if coroutine.status(cr) ~= "suspended" or worker:GetFinished() then - worker = nil + if not cr or coroutine.status(cr) ~= "suspended" then + worker:SetFinished(true) return false, nil end @@ -51,20 +46,13 @@ function ASYNC.Defer(worker, func) -- Start coroutine on the first call. local cr = worker:GetCoroutine() if not cr then - cr = coroutine.create(function(used_worker) - local graceful, err = pcall(func, used_worker) - if graceful then - worker:SetFinished(true) - elseif err then - print(err) - end - end) + cr = coroutine.create(func) worker:SetCoroutine(cr) end -- Check if the coroutine finished. We will never encounter "running", as we don't call coroutine.status from inside the coroutine. - if coroutine.status(cr) ~= "suspended" or worker:GetFinished() then - worker = nil + if not cr or coroutine.status(cr) ~= "suspended" then + worker:SetFinished(true) return false end