From b4741e6665385b39baaf7ca19c1b20b85d7ac9e3 Mon Sep 17 00:00:00 2001 From: Eric Pohl <31418444+ericpohl@users.noreply.github.com> Date: Tue, 7 Apr 2026 14:09:17 -0400 Subject: [PATCH 1/4] Performance refactorings (#267) * Construct DateTime objects with Local DateTimeKind * Consolidate "first day of 1980" fields * More messing with timezones * Cleanup * Performance refactorings * .gitignore JetBrains * Renaming --------- Co-authored-by: Eric Pohl --- .gitignore | 3 + .../{ExtensionMethods.cs => Extensions.cs} | 17 +++-- ISOv4Plugin/ISOModels/ISO11783_TaskData.cs | 61 +++++++++++------ ISOv4Plugin/ISOModels/ISOTimeLog.cs | 14 +++- .../ObjectModel/DeviceElementHierarchy.cs | 46 ++++++------- ISOv4Plugin/Plugin.cs | 66 +++++++++++-------- .../Representation/RepresentationMapper.cs | 28 ++++++-- 7 files changed, 152 insertions(+), 83 deletions(-) rename ISOv4Plugin/ExtensionMethods/{ExtensionMethods.cs => Extensions.cs} (95%) diff --git a/.gitignore b/.gitignore index 45a53ff9..5c4f875e 100644 --- a/.gitignore +++ b/.gitignore @@ -195,3 +195,6 @@ FakesAssemblies/ # Visual Studio 6 workspace options file *.opt + +# Jetbrains Rider +.idea/ diff --git a/ISOv4Plugin/ExtensionMethods/ExtensionMethods.cs b/ISOv4Plugin/ExtensionMethods/Extensions.cs similarity index 95% rename from ISOv4Plugin/ExtensionMethods/ExtensionMethods.cs rename to ISOv4Plugin/ExtensionMethods/Extensions.cs index 09490240..6dd7c0f2 100644 --- a/ISOv4Plugin/ExtensionMethods/ExtensionMethods.cs +++ b/ISOv4Plugin/ExtensionMethods/Extensions.cs @@ -3,6 +3,7 @@ */ using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; using System.Linq; @@ -20,9 +21,10 @@ namespace AgGateway.ADAPT.ISOv4Plugin.ExtensionMethods { - public static class ExtensionMethods + public static class Extensions { private static readonly Regex IsoIdPattern = new Regex("^[A-Z]{3,4}-?[0-9]+$", RegexOptions.Compiled); + private static readonly ConcurrentDictionary _directoryFilesCache = new ConcurrentDictionary(); public static string WithTaskDataPath(this string dataPath) { @@ -306,16 +308,23 @@ public static IEnumerable GetDirectoryFiles(this string dataPath, string { if (Directory.Exists(dataPath)) { - //Note! We need to iterate through all files and do a ToLower for this to work in .Net Core in Linux since that filesystem - //is case sensitive and the NetStandard interface for Directory.GetFiles doesn't account for that yet. var fileNameToFind = searchPath.ToLower(); - var allFiles = Directory.GetFiles(dataPath, "*.*", searchOption); + var cacheKey = string.Concat(dataPath, "\0", (int)searchOption); + var allFiles = _directoryFilesCache.GetOrAdd(cacheKey, _ => Directory.GetFiles(dataPath, "*.*", searchOption)); var matchedFiles = allFiles.Where(file => file.ToLower().EndsWith(fileNameToFind)); return matchedFiles; } return new List(); } + /// + /// Clears the cached directory file listings. Call when directory contents may have changed. + /// + public static void ClearDirectoryFilesCache() + { + _directoryFilesCache.Clear(); + } + /// /// Case-insensitive comparison of two strings /// diff --git a/ISOv4Plugin/ISOModels/ISO11783_TaskData.cs b/ISOv4Plugin/ISOModels/ISO11783_TaskData.cs index e3d96094..b9736f90 100644 --- a/ISOv4Plugin/ISOModels/ISO11783_TaskData.cs +++ b/ISOv4Plugin/ISOModels/ISO11783_TaskData.cs @@ -84,16 +84,35 @@ public static ISO11783_TaskData ReadXML(XmlNode taskDataNode, string baseFolder) taskData.DataTransferOriginInt = taskDataNode.GetXmlNodeValueAsInt("@DataTransferOrigin"); taskData.DataTransferLanguage = taskDataNode.GetXmlNodeValue("@DataTransferLanguage"); - //-------------- - //Child Elements - //-------------- + //External file references - select all XFR nodes once and group by prefix + var allXfrNodes = taskDataNode.SelectNodes("XFR"); + var xfrByPrefix = new Dictionary>(); + if (allXfrNodes != null) + { + for (int i = 0; i < allXfrNodes.Count; i++) + { + var xfrNode = allXfrNodes[i]; + var fileName = xfrNode.GetXmlNodeValue("@A"); + if (fileName != null && fileName.Length >= 3) + { + var prefix = fileName.Substring(0, 3); + if (!xfrByPrefix.TryGetValue(prefix, out var list)) + { + list = new List(); + xfrByPrefix[prefix] = list; + } + list.Add(xfrNode); + } + } + } + //Attached Files XmlNodeList afeNodes = taskDataNode.SelectNodes("AFE"); if (afeNodes != null) { taskData.ChildElements.AddRange(ISOAttachedFile.ReadXML(afeNodes)); } - ProcessExternalNodes(taskDataNode, "AFE", baseFolder, taskData, ISOAttachedFile.ReadXML); + ProcessExternalNodes(xfrByPrefix, "AFE", baseFolder, taskData, ISOAttachedFile.ReadXML); //Coded Comments XmlNodeList cctNodes = taskDataNode.SelectNodes("CCT"); @@ -101,7 +120,7 @@ public static ISO11783_TaskData ReadXML(XmlNode taskDataNode, string baseFolder) { taskData.ChildElements.AddRange(ISOCodedComment.ReadXML(cctNodes)); } - ProcessExternalNodes(taskDataNode, "CCT", baseFolder, taskData, ISOCodedComment.ReadXML); + ProcessExternalNodes(xfrByPrefix, "CCT", baseFolder, taskData, ISOCodedComment.ReadXML); //Crop Types XmlNodeList ctpNodes = taskDataNode.SelectNodes("CTP"); @@ -109,7 +128,7 @@ public static ISO11783_TaskData ReadXML(XmlNode taskDataNode, string baseFolder) { taskData.ChildElements.AddRange(ISOCropType.ReadXML(ctpNodes)); } - ProcessExternalNodes(taskDataNode, "CTP", baseFolder, taskData, ISOCropType.ReadXML); + ProcessExternalNodes(xfrByPrefix, "CTP", baseFolder, taskData, ISOCropType.ReadXML); //Cultural Practices XmlNodeList cpcNodes = taskDataNode.SelectNodes("CPC"); @@ -117,7 +136,7 @@ public static ISO11783_TaskData ReadXML(XmlNode taskDataNode, string baseFolder) { taskData.ChildElements.AddRange(ISOCulturalPractice.ReadXML(cpcNodes)); } - ProcessExternalNodes(taskDataNode, "CPC", baseFolder, taskData, ISOCulturalPractice.ReadXML); + ProcessExternalNodes(xfrByPrefix, "CPC", baseFolder, taskData, ISOCulturalPractice.ReadXML); //Customers XmlNodeList ctrNodes = taskDataNode.SelectNodes("CTR"); @@ -125,7 +144,7 @@ public static ISO11783_TaskData ReadXML(XmlNode taskDataNode, string baseFolder) { taskData.ChildElements.AddRange(ISOCustomer.ReadXML(ctrNodes)); } - ProcessExternalNodes(taskDataNode, "CTR", baseFolder, taskData, ISOCustomer.ReadXML); + ProcessExternalNodes(xfrByPrefix, "CTR", baseFolder, taskData, ISOCustomer.ReadXML); //Devices XmlNodeList dvcNodes = taskDataNode.SelectNodes("DVC"); @@ -133,7 +152,7 @@ public static ISO11783_TaskData ReadXML(XmlNode taskDataNode, string baseFolder) { taskData.ChildElements.AddRange(ISODevice.ReadXML(dvcNodes)); } - ProcessExternalNodes(taskDataNode, "DVC", baseFolder, taskData, ISODevice.ReadXML); + ProcessExternalNodes(xfrByPrefix, "DVC", baseFolder, taskData, ISODevice.ReadXML); //Farms XmlNodeList frmNodes = taskDataNode.SelectNodes("FRM"); @@ -141,7 +160,7 @@ public static ISO11783_TaskData ReadXML(XmlNode taskDataNode, string baseFolder) { taskData.ChildElements.AddRange(ISOFarm.ReadXML(frmNodes)); } - ProcessExternalNodes(taskDataNode, "FRM", baseFolder, taskData, ISOFarm.ReadXML); + ProcessExternalNodes(xfrByPrefix, "FRM", baseFolder, taskData, ISOFarm.ReadXML); //Operation Techniques XmlNodeList otqNodes = taskDataNode.SelectNodes("OTQ"); @@ -149,7 +168,7 @@ public static ISO11783_TaskData ReadXML(XmlNode taskDataNode, string baseFolder) { taskData.ChildElements.AddRange(ISOOperationTechnique.ReadXML(otqNodes)); } - ProcessExternalNodes(taskDataNode, "OTQ", baseFolder, taskData, ISOOperationTechnique.ReadXML); + ProcessExternalNodes(xfrByPrefix, "OTQ", baseFolder, taskData, ISOOperationTechnique.ReadXML); //Partfields XmlNodeList pfdNodes = taskDataNode.SelectNodes("PFD"); @@ -157,7 +176,7 @@ public static ISO11783_TaskData ReadXML(XmlNode taskDataNode, string baseFolder) { taskData.ChildElements.AddRange(ISOPartfield.ReadXML(pfdNodes)); } - ProcessExternalNodes(taskDataNode, "PFD", baseFolder, taskData, ISOPartfield.ReadXML); + ProcessExternalNodes(xfrByPrefix, "PFD", baseFolder, taskData, ISOPartfield.ReadXML); //Products XmlNodeList pdtNodes = taskDataNode.SelectNodes("PDT"); @@ -165,7 +184,7 @@ public static ISO11783_TaskData ReadXML(XmlNode taskDataNode, string baseFolder) { taskData.ChildElements.AddRange(ISOProduct.ReadXML(pdtNodes)); } - ProcessExternalNodes(taskDataNode, "PDT", baseFolder, taskData, ISOProduct.ReadXML); + ProcessExternalNodes(xfrByPrefix, "PDT", baseFolder, taskData, ISOProduct.ReadXML); //Product Groups XmlNodeList pgpNodes = taskDataNode.SelectNodes("PGP"); @@ -173,7 +192,7 @@ public static ISO11783_TaskData ReadXML(XmlNode taskDataNode, string baseFolder) { taskData.ChildElements.AddRange(ISOProductGroup.ReadXML(pgpNodes)); } - ProcessExternalNodes(taskDataNode, "PGP", baseFolder, taskData, ISOProductGroup.ReadXML); + ProcessExternalNodes(xfrByPrefix, "PGP", baseFolder, taskData, ISOProductGroup.ReadXML); //Task Controller Capabilities XmlNodeList tccNodes = taskDataNode.SelectNodes("TCC"); @@ -181,7 +200,7 @@ public static ISO11783_TaskData ReadXML(XmlNode taskDataNode, string baseFolder) { taskData.ChildElements.AddRange(ISOTaskControllerCapabilities.ReadXML(tccNodes)); } - ProcessExternalNodes(taskDataNode, "TCC", baseFolder, taskData, ISOTaskControllerCapabilities.ReadXML); + ProcessExternalNodes(xfrByPrefix, "TCC", baseFolder, taskData, ISOTaskControllerCapabilities.ReadXML); //Tasks XmlNodeList tskNodes = taskDataNode.SelectNodes("TSK"); @@ -189,7 +208,7 @@ public static ISO11783_TaskData ReadXML(XmlNode taskDataNode, string baseFolder) { taskData.ChildElements.AddRange(ISOTask.ReadXML(tskNodes)); } - ProcessExternalNodes(taskDataNode, "TSK", baseFolder, taskData, ISOTask.ReadXML); + ProcessExternalNodes(xfrByPrefix, "TSK", baseFolder, taskData, ISOTask.ReadXML); //Value Presentations XmlNodeList vpnNodes = taskDataNode.SelectNodes("VPN"); @@ -197,7 +216,7 @@ public static ISO11783_TaskData ReadXML(XmlNode taskDataNode, string baseFolder) { taskData.ChildElements.AddRange(ISOValuePresentation.ReadXML(vpnNodes)); } - ProcessExternalNodes(taskDataNode, "VPN", baseFolder, taskData, ISOValuePresentation.ReadXML); + ProcessExternalNodes(xfrByPrefix, "VPN", baseFolder, taskData, ISOValuePresentation.ReadXML); //Workers XmlNodeList wkrNodes = taskDataNode.SelectNodes("WKR"); @@ -205,7 +224,7 @@ public static ISO11783_TaskData ReadXML(XmlNode taskDataNode, string baseFolder) { taskData.ChildElements.AddRange(ISOWorker.ReadXML(wkrNodes)); } - ProcessExternalNodes(taskDataNode, "WKR", baseFolder, taskData, ISOWorker.ReadXML); + ProcessExternalNodes(xfrByPrefix, "WKR", baseFolder, taskData, ISOWorker.ReadXML); //LinkList ISOAttachedFile linkListFile = taskData.ChildElements.OfType().SingleOrDefault(afe => afe.FileType == 1); @@ -240,9 +259,11 @@ public override List Validate(List errors) return errors; } - private static void ProcessExternalNodes(XmlNode node, string xmlPrefix, string baseFolder, ISO11783_TaskData taskData, Func> readDelegate) + private static void ProcessExternalNodes(Dictionary> xfrByPrefix, string xmlPrefix, string baseFolder, ISO11783_TaskData taskData, Func> readDelegate) { - var externalNodes = node.SelectNodes($"XFR[starts-with(@A, '{xmlPrefix}')]"); + if (!xfrByPrefix.TryGetValue(xmlPrefix, out var externalNodes)) + return; + for (int i = 0; i < externalNodes.Count; i++) { var inputNodes = externalNodes[i].LoadActualNodes("XFR", baseFolder); diff --git a/ISOv4Plugin/ISOModels/ISOTimeLog.cs b/ISOv4Plugin/ISOModels/ISOTimeLog.cs index f10b85c5..d9d60814 100644 --- a/ISOv4Plugin/ISOModels/ISOTimeLog.cs +++ b/ISOv4Plugin/ISOModels/ISOTimeLog.cs @@ -22,6 +22,9 @@ public class ISOTimeLog : ISOElement public uint? Filelength { get; set; } public byte TimeLogType { get; set; } + private ISOTime _cachedTimeElement; + private string _cachedDataPath; + public override XmlWriter WriteXML(XmlWriter xmlBuilder) { xmlBuilder.WriteStartElement("TLG"); @@ -53,6 +56,11 @@ public static IEnumerable ReadXML(XmlNodeList nodes) public ISOTime GetTimeElement(string dataPath) { + if (_cachedTimeElement != null && _cachedDataPath == dataPath) + { + return _cachedTimeElement; + } + string xmlName = string.Concat(Filename, ".xml"); string filePath = dataPath.GetDirectoryFiles(xmlName, SearchOption.TopDirectoryOnly).FirstOrDefault(); if (filePath != null) @@ -61,10 +69,14 @@ public ISOTime GetTimeElement(string dataPath) document.Load(filePath); XmlNode rootNode = document.SelectSingleNode("TIM"); - return ISOTime.ReadXML(rootNode); + var timeElement = ISOTime.ReadXML(rootNode); + _cachedTimeElement = timeElement; + _cachedDataPath = dataPath; + return timeElement; } else { + _cachedDataPath = dataPath; return null; } } diff --git a/ISOv4Plugin/ObjectModel/DeviceElementHierarchy.cs b/ISOv4Plugin/ObjectModel/DeviceElementHierarchy.cs index 2b7fc59e..ecc566e4 100644 --- a/ISOv4Plugin/ObjectModel/DeviceElementHierarchy.cs +++ b/ISOv4Plugin/ObjectModel/DeviceElementHierarchy.cs @@ -247,31 +247,27 @@ public DeviceHierarchyElement(ISODeviceElement deviceElement, //DeviceProperty assigned Widths & Offsets //DeviceProcessData assigned values will be assigned as the SectionMapper reads timelog data. + //Build a lookup of DeviceProperties by DDI for O(1) access + var propertiesByDDI = deviceElement.DeviceProperties + .GroupBy(dpt => dpt.DDI) + .ToDictionary(g => g.Key, g => g.First()); + //Width - ISODeviceProperty widthProperty = deviceElement.DeviceProperties.FirstOrDefault(dpt => dpt.DDI == "0046"); //Max width - if (widthProperty != null) + ISODeviceProperty widthProperty; + if (propertiesByDDI.TryGetValue("0046", out widthProperty)) //Max width { Width = widthProperty.Value; WidthDDI = "0046"; } - else + else if (propertiesByDDI.TryGetValue("0044", out widthProperty)) //Default working width { - widthProperty = deviceElement.DeviceProperties.FirstOrDefault(dpt => dpt.DDI == "0044"); //Default working width - if (widthProperty != null) - { - Width = widthProperty.Value; - WidthDDI = "0044"; - } - - if (widthProperty == null) - { - widthProperty = deviceElement.DeviceProperties.FirstOrDefault(dpt => dpt.DDI == "0043"); //Actual working width - if (widthProperty != null) - { - Width = widthProperty.Value; - WidthDDI = "0043"; - } - } + Width = widthProperty.Value; + WidthDDI = "0044"; + } + else if (propertiesByDDI.TryGetValue("0043", out widthProperty)) //Actual working width + { + Width = widthProperty.Value; + WidthDDI = "0043"; } if (Width == null) @@ -281,8 +277,8 @@ public DeviceHierarchyElement(ISODeviceElement deviceElement, } //Offsets - ISODeviceProperty xOffsetProperty = deviceElement.DeviceProperties.FirstOrDefault(dpt => dpt.DDI == "0086"); - if (xOffsetProperty != null) + ISODeviceProperty xOffsetProperty; + if (propertiesByDDI.TryGetValue("0086", out xOffsetProperty)) { XOffset = xOffsetProperty.Value; } @@ -291,8 +287,8 @@ public DeviceHierarchyElement(ISODeviceElement deviceElement, AddMissingGeometryDefinition(missingGeometryDefinitions, deviceElement.DeviceElementId, "0086"); } - ISODeviceProperty yOffsetProperty = deviceElement.DeviceProperties.FirstOrDefault(dpt => dpt.DDI == "0087"); - if (yOffsetProperty != null) + ISODeviceProperty yOffsetProperty; + if (propertiesByDDI.TryGetValue("0087", out yOffsetProperty)) { YOffset = yOffsetProperty.Value; } @@ -301,8 +297,8 @@ public DeviceHierarchyElement(ISODeviceElement deviceElement, AddMissingGeometryDefinition(missingGeometryDefinitions, deviceElement.DeviceElementId, "0087"); } - ISODeviceProperty zOffsetProperty = deviceElement.DeviceProperties.FirstOrDefault(dpt => dpt.DDI == "0088"); - if (zOffsetProperty != null) + ISODeviceProperty zOffsetProperty; + if (propertiesByDDI.TryGetValue("0088", out zOffsetProperty)) { ZOffset = zOffsetProperty.Value; } diff --git a/ISOv4Plugin/Plugin.cs b/ISOv4Plugin/Plugin.cs index eb46d627..7ce1c4d1 100644 --- a/ISOv4Plugin/Plugin.cs +++ b/ISOv4Plugin/Plugin.cs @@ -34,44 +34,58 @@ public Plugin() public void Export(ApplicationDataModel.ADM.ApplicationDataModel dataModel, string exportPath, Properties properties) { - //Convert the ADAPT model into the ISO model - string outputPath = exportPath.WithTaskDataPath(); - TaskDataMapper taskDataMapper = new TaskDataMapper(outputPath, properties); - Errors = taskDataMapper.Errors; - ISO11783_TaskData taskData = taskDataMapper.Export(dataModel); - - //Serialize the ISO model to XML - using (TaskDocumentWriter writer = new TaskDocumentWriter()) + try { - writer.WriteTaskData(outputPath, taskData); - - //Serialize the Link List - if (taskData.Version > 3) + //Convert the ADAPT model into the ISO model + string outputPath = exportPath.WithTaskDataPath(); + TaskDataMapper taskDataMapper = new TaskDataMapper(outputPath, properties); + Errors = taskDataMapper.Errors; + ISO11783_TaskData taskData = taskDataMapper.Export(dataModel); + + //Serialize the ISO model to XML + using (TaskDocumentWriter writer = new TaskDocumentWriter()) { - writer.WriteLinkList(outputPath, taskData.LinkList); + writer.WriteTaskData(outputPath, taskData); + + //Serialize the Link List + if (taskData.Version > 3) + { + writer.WriteLinkList(outputPath, taskData.LinkList); + } } } + finally + { + Extensions.ClearDirectoryFilesCache(); + } } public IList Import(string dataPath, Properties properties = null) { - var taskDataObjects = ReadDataCard(dataPath); - if (taskDataObjects == null) - return null; - - var adms = new List(); - foreach (var taskData in taskDataObjects) + try { - //Convert the ISO model to ADAPT - TaskDataMapper taskDataMapper = new TaskDataMapper(taskData.DataFolder, properties, taskData.VersionMajor); - ApplicationDataModel.ADM.ApplicationDataModel dataModel = taskDataMapper.Import(taskData); - foreach (var error in taskDataMapper.Errors) + var taskDataObjects = ReadDataCard(dataPath); + if (taskDataObjects == null) + return null; + + var adms = new List(); + foreach (var taskData in taskDataObjects) { - Errors.Add(error); + //Convert the ISO model to ADAPT + TaskDataMapper taskDataMapper = new TaskDataMapper(taskData.DataFolder, properties, taskData.VersionMajor); + ApplicationDataModel.ADM.ApplicationDataModel dataModel = taskDataMapper.Import(taskData); + foreach (var error in taskDataMapper.Errors) + { + Errors.Add(error); + } + adms.Add(dataModel); } - adms.Add(dataModel); + return adms; + } + finally + { + Extensions.ClearDirectoryFilesCache(); } - return adms; } Properties _properties = null; diff --git a/ISOv4Plugin/Representation/RepresentationMapper.cs b/ISOv4Plugin/Representation/RepresentationMapper.cs index 0f6f2719..3ddab4f5 100644 --- a/ISOv4Plugin/Representation/RepresentationMapper.cs +++ b/ISOv4Plugin/Representation/RepresentationMapper.cs @@ -27,32 +27,46 @@ public interface IRepresentationMapper public class RepresentationMapper : IRepresentationMapper { private readonly Dictionary _ddis; + private readonly Dictionary _ddiToRepresentationCache; public RepresentationMapper() { _ddis = DdiLoader.Ddis; + _ddiToRepresentationCache = BuildDdiToRepresentationCache(); } - public AdaptRepresentation Map(int ddi) + private Dictionary BuildDdiToRepresentationCache() { - if (_ddis.ContainsKey(ddi)) + var cache = new Dictionary(); + foreach (var kvp in _ddis) { - var matchingDdi = _ddis[ddi]; + var ddi = kvp.Key; + var matchingDdi = kvp.Value; var representations = RepresentationManager.Instance.Representations.Where(x => x.Ddi.GetValueOrDefault() == matchingDdi.Id); if (representations.Any()) { - //Default the representation mapping approprately on import var representation = representations.FirstOrDefault(r => r.IsDefaultRepresentationForDDI) ?? representations.First(); - - AdaptRepresentation adaptRep = GetADAPTRepresentation(representation); + var adaptRep = GetADAPTRepresentation(representation); if (adaptRep != null) { - return adaptRep; + cache[ddi] = adaptRep; } } + } + return cache; + } + + public AdaptRepresentation Map(int ddi) + { + if (_ddiToRepresentationCache.TryGetValue(ddi, out var cached)) + { + return cached; + } + if (_ddis.ContainsKey(ddi)) + { return new ApplicationDataModel.Representations.NumericRepresentation { Code = ddi.ToString("X4"), CodeSource = RepresentationCodeSourceEnum.ISO11783_DDI }; } return null; From 2dd3ab8dfbad45a6a11a3a7fe68b3409d7d293ca Mon Sep 17 00:00:00 2001 From: Andrew Vardeman Date: Tue, 7 Apr 2026 15:08:18 -0500 Subject: [PATCH 2/4] ReadImplementGeometryValues: prefer the most common value for each DLV (#268) * Change ReadImplementGeometryValues to prefer the most common value for each dlv rather than the farthest from zero Signed-off-by: Andrew Vardeman * Remove debugging code Signed-off-by: Andrew Vardeman --------- Signed-off-by: Andrew Vardeman --- ISOv4Plugin/Mappers/TimeLogMapper.cs | 76 +++++++++++++++++++++------- 1 file changed, 57 insertions(+), 19 deletions(-) diff --git a/ISOv4Plugin/Mappers/TimeLogMapper.cs b/ISOv4Plugin/Mappers/TimeLogMapper.cs index 65bf980c..4d39cda4 100644 --- a/ISOv4Plugin/Mappers/TimeLogMapper.cs +++ b/ISOv4Plugin/Mappers/TimeLogMapper.cs @@ -767,7 +767,11 @@ protected class BinaryReader { public static Dictionary ReadImplementGeometryValues(string filePath, ISOTime templateTime, IEnumerable desiredDLVIndices, int version, IList errors) { - Dictionary output = new Dictionary(); + List>[] valuesByDLV = new List>[256]; + foreach (byte dlv in desiredDLVIndices) + { + valuesByDLV[dlv] = new List>(); + } List desiredIndexes = desiredDLVIndices.ToList(); //Determine the number of header bytes in each position @@ -775,7 +779,7 @@ public static Dictionary ReadImplementGeometryValues(string filePath, bool overrideTimelogAttributeChecks = DetermineTimelogAttributeValidity(filePath, version); SkipBytes(overrideTimelogAttributeChecks || (templateTime.HasStart && templateTime.Start == null), 6, ref headerCount); ISOPosition templatePosition = templateTime.Positions.FirstOrDefault(); - + if (templatePosition != null) { SkipBytes(overrideTimelogAttributeChecks || (templatePosition.HasPositionNorth && templatePosition.PositionNorth == null), 4, ref headerCount); @@ -789,6 +793,7 @@ public static Dictionary ReadImplementGeometryValues(string filePath, SkipBytes(overrideTimelogAttributeChecks || (templatePosition.HasGpsUtcDate && templatePosition.GpsUtcDate == null), 2, ref headerCount); } + int recordIndex = 0; using (var binaryReader = new System.IO.BinaryReader(File.Open(filePath, FileMode.Open, FileAccess.Read, FileShare.Read))) { while (ContinueReading(binaryReader)) @@ -807,23 +812,7 @@ public static Dictionary ReadImplementGeometryValues(string filePath, { //A desired DLV is reported here int value = ReadInt32(null, true, false, binaryReader).GetValueOrDefault(); - try - { - if (!output.ContainsKey(dlvIndex)) - { - output.Add(dlvIndex, value); - } - else if (Math.Abs(value) > Math.Abs(output[dlvIndex])) - { - //Values should be all the same, but prefer the furthest from 0 - output[dlvIndex] = value; - } - } - catch (OverflowException ex) - { - // If value == int.MinValue, Math.Abs(value) will throw System.OverflowException: Negating the minimum value of a twos complement number is invalid. - errors.Add(new Error() { Description = ex.Message, Id = ex.GetType().ToString(), Source = ex.Source, StackTrace = ex.StackTrace }); - } + valuesByDLV[dlvIndex].Add(new Tuple(recordIndex, value)); } else { @@ -833,9 +822,58 @@ public static Dictionary ReadImplementGeometryValues(string filePath, } } + recordIndex++; + } + + } + + int recordCount = recordIndex; + Dictionary output = new Dictionary(); + foreach (byte dlvIndex in desiredIndexes) + { + var vals = valuesByDLV[dlvIndex]; + if (vals.Count == 0) + { + continue; + } + + if (vals.Count == 1) + { + output[dlvIndex] = vals[0].Item2; } + else + { + var counts = new Dictionary(); + int lastVal = vals[0].Item2; + void AddCount(int key, int theCount) + { + if (counts.ContainsKey(key)) + { + counts[lastVal] += theCount; + } + else + { + counts[lastVal] = theCount; + } + } + + int count; + for (int i = 1; i < vals.Count; i++) + { + count = vals[i].Item1 - vals[i - 1].Item1; + AddCount(lastVal, count); + lastVal = vals[i].Item2; + } + + count = recordCount - vals[vals.Count - 1].Item1; + AddCount(lastVal, count); + + // set the most common value for this DLV as the value to return, with the count of that value as a tie-breaker if needed + output[dlvIndex] = counts.OrderByDescending(x => x.Value).ThenByDescending(x => x.Key).First().Key; + } } + return output; } From 7405b4d69146d325cef96c9c8035ddc3cda278eb Mon Sep 17 00:00:00 2001 From: Eric Pohl <31418444+ericpohl@users.noreply.github.com> Date: Wed, 22 Apr 2026 12:44:35 -0400 Subject: [PATCH 3/4] Validate inferred time zone offsets (#270) * Construct DateTime objects with Local DateTimeKind * Consolidate "first day of 1980" fields * More messing with timezones * Cleanup * Performance refactorings * .gitignore JetBrains * Renaming * Remove .vscode files from tracking * Validate inferred timezone offsets --------- Co-authored-by: Eric Pohl --- .gitignore | 3 +++ .vscode/launch.json | 14 ------------- .../Import/SpatialRecordMapper.cs | 16 +++++++++----- ISOv4Plugin/Mappers/TaskDataMapper.cs | 21 +++++++++++++++++++ ISOv4Plugin/Mappers/TimeLogMapper.cs | 14 +++++++++---- 5 files changed, 45 insertions(+), 23 deletions(-) delete mode 100644 .vscode/launch.json diff --git a/.gitignore b/.gitignore index 5c4f875e..52a44daf 100644 --- a/.gitignore +++ b/.gitignore @@ -198,3 +198,6 @@ FakesAssemblies/ # Jetbrains Rider .idea/ + +# VSCode +.vscode/ diff --git a/.vscode/launch.json b/.vscode/launch.json deleted file mode 100644 index a6e4859c..00000000 --- a/.vscode/launch.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - // Use IntelliSense to find out which attributes exist for C# debugging - // Use hover for the description of the existing attributes - // For further information visit https://github.com/OmniSharp/omnisharp-vscode/blob/master/debugger-launchjson.md - "version": "0.2.0", - "configurations": [ - { - "name": ".NET Core Attach", - "type": "coreclr", - "request": "attach", - "processId": "${command:pickProcess}" - } - ] -} diff --git a/ISOv4Plugin/Mappers/LoggedDataMappers/Import/SpatialRecordMapper.cs b/ISOv4Plugin/Mappers/LoggedDataMappers/Import/SpatialRecordMapper.cs index a77f8967..e745e917 100644 --- a/ISOv4Plugin/Mappers/LoggedDataMappers/Import/SpatialRecordMapper.cs +++ b/ISOv4Plugin/Mappers/LoggedDataMappers/Import/SpatialRecordMapper.cs @@ -49,7 +49,11 @@ public IEnumerable Map(IEnumerable isoSpatialRows, pan.AllocationStamp.Start.Value.Minute == firstSpatialRow.TimeStart.Minute && pan.AllocationStamp.Start.Value.Second == firstSpatialRow.TimeStart.Second) { - _effectiveTimeZoneOffset = firstSpatialRow.TimeStart - pan.AllocationStamp.Start.Value; + _effectiveTimeZoneOffset = TaskDataMapper.ValidateTimezoneOffset(firstSpatialRow.TimeStart, pan.AllocationStamp.Start.Value); + if (!_effectiveTimeZoneOffset.HasValue) + { + _taskDataMapper.AddError($"Unable to determine effective timezone offset from comparison of spatial record and product allocation timestamps. Monitor date/time setting may be invalid."); + } } } } @@ -202,22 +206,24 @@ private bool GovernsTimestamp(ISOProductAllocation p, SpatialRecord spatialRecor // Comparing DateTime values with different Kind values leads to inaccurate results. // Convert DateTimes to UTC if possible before comparing them - private DateTime? ToUtc(DateTime? nullableDateTime, TimeSpan? timezoneOffset) + private static DateTime? ToUtc(DateTime? nullableDateTime, TimeSpan? timezoneOffset) { return nullableDateTime.HasValue ? ToUtc(nullableDateTime.Value, timezoneOffset) : nullableDateTime; } - private DateTime ToUtc(DateTime dateTime, TimeSpan? timezoneOffset) + private static DateTime ToUtc(DateTime dateTime, TimeSpan? timezoneOffset) { if (dateTime.Kind == DateTimeKind.Utc) return dateTime; - if (_taskDataMapper.TimezoneOffset.HasValue) + if (timezoneOffset.HasValue) { // Convert from local time to UTC using the timezone offset. + // We're relying on the upstream guard ensuring the timezone offset is + // within 14 hours var localTime = new DateTimeOffset(dateTime.Year, dateTime.Month, dateTime.Day, dateTime.Hour, dateTime.Minute, dateTime.Second, dateTime.Millisecond, - _taskDataMapper.TimezoneOffset.Value); + timezoneOffset.Value); DateTime utc = localTime.UtcDateTime; return utc; } diff --git a/ISOv4Plugin/Mappers/TaskDataMapper.cs b/ISOv4Plugin/Mappers/TaskDataMapper.cs index 99517d7c..5116f37f 100644 --- a/ISOv4Plugin/Mappers/TaskDataMapper.cs +++ b/ISOv4Plugin/Mappers/TaskDataMapper.cs @@ -157,6 +157,27 @@ public void AddError(string error, string id = null, string source = null, strin Errors.Add(new Error() { Description = error, Id = id, Source = source, StackTrace = stackTrace }); } + /// + /// Validates and processes a timezone offset calculated from local and UTC times. + /// Calculates offset = localTime - utcTime, rounds to nearest minute, and validates it's within ±14 hours. + /// + /// The validated TimeSpan offset, or null if the offset is outside the acceptable ±14 hour range. + public static TimeSpan? ValidateTimezoneOffset(DateTime localTime, DateTime utcTime) + { + TimeSpan offset = localTime - utcTime; + // Round offset to nearest minute for use in timezone offset + offset = TimeSpan.FromMinutes(Math.Round(offset.TotalMinutes)); + // DateTimeOffset requires the offset to be within ±14 hours + if (Math.Abs(offset.TotalHours) <= 14) + { + return offset; + } + else + { + return null; + } + } + public ISO11783_TaskData Export(ApplicationDataModel.ADM.ApplicationDataModel adm) { AdaptDataModel = adm; diff --git a/ISOv4Plugin/Mappers/TimeLogMapper.cs b/ISOv4Plugin/Mappers/TimeLogMapper.cs index 4d39cda4..47430c8a 100644 --- a/ISOv4Plugin/Mappers/TimeLogMapper.cs +++ b/ISOv4Plugin/Mappers/TimeLogMapper.cs @@ -330,11 +330,17 @@ protected IEnumerable ImportTimeLog(ISOTask loggedTask, ISOTimeLo var firstRecord = isoRecords.FirstOrDefault(r => r.GpsUtcDateTime.HasValue && r.GpsUtcDate != ushort.MaxValue && r.GpsUtcDate != 0); if (firstRecord != null) { - //Local - UTC = Delta. This value will be rough based on the accuracy of the clock settings + // Local - UTC = Delta. This value will be rough based on the accuracy of the clock settings // but will expose the ability to derive the UTC times from the exported local times. - TimeSpan offset = firstRecord.TimeStart - firstRecord.GpsUtcDateTime.Value; - // Round offset to nearest minute for use in timezone offset - TaskDataMapper.TimezoneOffset = TimeSpan.FromMinutes(Math.Round(offset.TotalMinutes)); + TimeSpan? offset = TaskDataMapper.ValidateTimezoneOffset(firstRecord.TimeStart, firstRecord.GpsUtcDateTime.Value); + if (offset.HasValue) + { + TaskDataMapper.TimezoneOffset = offset.Value; + } + else + { + TaskDataMapper.AddError($"GPS time offset of {firstRecord.TimeStart - firstRecord.GpsUtcDateTime.Value} is outside the acceptable range. Monitor date/time setting is probably invalid. Product allocation logic may be impacted."); + } } } } From 58e3d0991d101402b2e34cbe9c44c7646165623f Mon Sep 17 00:00:00 2001 From: Andrew Vardeman Date: Mon, 15 Jun 2026 08:21:44 -0500 Subject: [PATCH 4/4] Make GetOperationTypeFromLoggingDevices respect just the device under consideration rather than all devices for a time log. Signed-off-by: Andrew Vardeman --- ISOv4Plugin/Mappers/TimeLogMapper.cs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/ISOv4Plugin/Mappers/TimeLogMapper.cs b/ISOv4Plugin/Mappers/TimeLogMapper.cs index 47430c8a..617357f7 100644 --- a/ISOv4Plugin/Mappers/TimeLogMapper.cs +++ b/ISOv4Plugin/Mappers/TimeLogMapper.cs @@ -418,7 +418,8 @@ protected IEnumerable ImportTimeLog(ISOTask loggedTask, ISOTimeLo operationData.DeviceElementUses = sectionMapper.ConvertToBaseTypes(sections.ToList()); operationData.GetDeviceElementUses = x => operationData.DeviceElementUses.Where(s => s.Depth == x).ToList(); operationData.PrescriptionId = prescriptionID; - operationData.OperationType = GetOperationType(productIDs, time, workingDatas); + var adaptDeviceModelId = TaskDataMapper.InstanceIDMap.GetADAPTID(dvc.DeviceId); + operationData.OperationType = GetOperationType(productIDs, time, workingDatas, adaptDeviceModelId); operationData.ProductIds = productIDs; if (!useDeferredExecution) { @@ -662,7 +663,7 @@ private void AddProductAllocationsForDeviceElement(Dictionary productIds, ISOTime time, List workingDatas) + private OperationTypeEnum GetOperationType(List productIds, ISOTime time, List workingDatas, int? adaptDeviceModelId) { var productCategories = productIds .Select(x => TaskDataMapper.AdaptDataModel.Catalog.Products.FirstOrDefault(y => y.Id.ReferenceId == x)) @@ -670,7 +671,7 @@ private OperationTypeEnum GetOperationType(List productIds, ISOTime time, L .Select(x => x.Category) .ToList(); - var deviceOperationType = GetOperationTypeFromLoggingDevices(time); + var deviceOperationType = GetOperationTypeFromLoggingDevices(time, adaptDeviceModelId); // Prefer product category to determine operation type where possible switch (productCategories.FirstOrDefault()) @@ -718,7 +719,7 @@ private OperationTypeEnum GetOperationType(List productIds, ISOTime time, L } } - private OperationTypeEnum GetOperationTypeFromLoggingDevices(ISOTime time) + private OperationTypeEnum GetOperationTypeFromLoggingDevices(ISOTime time, int? adaptDeviceModelId) { HashSet representedTypes = new HashSet(); IEnumerable distinctDeviceElementIDs = time.DataLogValues.Select(d => d.DeviceElementIdRef).Distinct(); @@ -728,7 +729,7 @@ private OperationTypeEnum GetOperationTypeFromLoggingDevices(ISOTime time) if (deviceElementID.HasValue) { DeviceElement deviceElement = DataModel.Catalog.DeviceElements.FirstOrDefault(d => d.Id.ReferenceId == deviceElementID.Value); - if (deviceElement != null && deviceElement.DeviceClassification != null) + if (deviceElement != null && deviceElement.DeviceClassification != null && deviceElement.DeviceModelId == adaptDeviceModelId) { DeviceOperationType deviceOperationType = DeviceOperationTypes.FirstOrDefault(d => d.MachineEnumerationMember.ToModelEnumMember().Value == deviceElement.DeviceClassification.Value.Value); if (deviceOperationType != null)