#455 evaluator precompiler

src/CSScriptLib/src/CSScriptLib/CSScript.cs

@@ -62,6 +62,8 @@ public static string GetCacheDirectory(string file)
     /// </summary>
     public class Settings
     {
+        internal const string DefaultEncodingName = "default";
+
         /// <summary>
         /// Loads and returns the settings instance.
         /// </summary>

src/CSScriptLib/src/CSScriptLib/CSScriptLib.csproj

@@ -122,6 +122,7 @@ CS-Script now uses separate temporary directories for better isolation:
     <Compile Include="..\..\..\cscs\CodeDom\CompilerResults.cs" Link="CodeDom\CompilerResults.cs" />
     <Compile Include="..\..\..\cscs\CodeDom\CompilerError.cs" Link="CodeDom\CompilerError.cs" />
     <Compile Include="..\..\..\cscs\fileparser.cs" Link="fileparser.cs" />
+    <Compile Include="..\..\..\cscs\Precompiler.cs" Link="Precompiler.cs" />
     <Compile Include="..\..\..\cscs\Project.cs" Link="Project.cs" />
     <Compile Include="..\..\..\cscs\ScriptParser.cs" Link="ScriptParser.cs" />
     <Compile Include="..\..\..\cscs\ScriptParsingResult.cs" Link="ScriptParsingResult.cs" />

src/CSScriptLib/src/CSScriptLib/Evaluator.CodeDom.cs

@@ -220,6 +220,49 @@ override protected (byte[] asm, byte[] pdb, Project project) Compile(string scri
                         return scriptCache[scriptHash];
                 }
 
+                /////////////////////////////////////
+                var lightParser = new CSharpParser(scriptFile ?? tempScriptFile, true);
+
+                if (lightParser.Precompilers.Any())
+                {
+                    var actualSources = sources.ToList();
+
+                    var precompiliationResult = base.PrecompileScript(scriptFile ?? tempScriptFile, lightParser);
+
+                    if (precompiliationResult != null)
+                    {
+                        tempScriptFile ??= CSScript.GetScriptTempFile();
+                        File.WriteAllText(tempScriptFile, precompiliationResult.Content);
+                        actualSources[0] = tempScriptFile;
+                    }
+
+                    int index = 1;
+                    foreach (string file in sources.Skip(1))
+                    {
+                        if (!file.EndsWith(Globals.InjectedAttributesPrefix))
+                        {
+                            var code = File.ReadAllText(file);
+
+                            var precompResult = base.PrecompileImportedScript(code, lightParser);
+                            if (precompResult != null)
+                            {
+                                var newFile = file.ChangeExtension(".g.cs");
+                                File.WriteAllText(newFile, precompResult.Content);
+                                actualSources[index] = newFile;
+                            }
+                        }
+                        index++;
+                    }
+
+                    actualSources.AddRange(precompiliationResult?.NewIncludes ?? []);
+                    sources = actualSources.Distinct().ToArray();
+
+                    if (precompiliationResult?.NewIncludes?.Any() == true)
+                    {
+                        refs = refs.Concat(precompiliationResult.NewReferences).ToArray();
+                    }
+                }
+
                 (byte[] asm, byte[] pdb) result = CompileAssemblyFromFileBatch_with_Csc(sources, refs, info?.AssemblyFile, this.IsDebug, info);
 
                 if (IsCachingEnabled)

src/CSScriptLib/src/CSScriptLib/Evaluator.Roslyn.cs

@@ -68,30 +68,6 @@
 // </summary>
 namespace CSScriptLib
 {
-    static class localExtensions
-    {
-        public static (string file, int line) Translate(this Dictionary<(int, int), (string, int)> mapping, int line)
-        {
-            foreach ((int start, int end) range in mapping.Keys)
-                if (range.start <= line && line <= range.end)
-                {
-                    (string file, int lineOffset) = mapping[range];
-                    return (file, line - range.start + lineOffset);
-                }
-
-            return ("", 0);
-        }
-
-        static public string[] SeparateUsingsFromCode(this string code)
-        {
-            SyntaxTree tree = CSharpSyntaxTree.ParseText(code);
-            CompilationUnitSyntax root = tree.GetCompilationUnitRoot();
-            int pos = root.Usings.FullSpan.End;
-
-            return new[] { code.Substring(0, pos).TrimEnd(), code.Substring(pos) };
-        }
-    }
-
     /// <summary>
     /// </summary>
     /// <seealso cref="CSScriptLib.IEvaluator"/>
@@ -208,13 +184,32 @@ override protected (byte[] asm, byte[] pdb, Project project) Compile(string scri
                 ////////////////////////////////////////
 
                 var mapping = new Dictionary<(int, int), (string, int)>();
+                var lightParser = new CSharpParser(scriptText, false);
+                List<string> extraRefs = [];
 
-                if (scriptFile == null && new CSharpParser(scriptText, false).Imports.Any())
+                if (scriptFile == null && (lightParser.Imports.Any() || lightParser.Precompilers.Any()))
                 {
                     tempScriptFile = CSScript.GetScriptTempFile();
                     File.WriteAllText(tempScriptFile, scriptText);
                 }
 
+                if (lightParser.Precompilers.Any())
+                {
+                    var precompiliationResult = base.PrecompileScript(scriptFile ?? tempScriptFile, lightParser);
+
+                    if (precompiliationResult != null)
+                    {
+                        tempScriptFile ??= CSScript.GetScriptTempFile();
+                        File.WriteAllText(tempScriptFile, precompiliationResult.Content);
+                        scriptText = precompiliationResult.Content;
+
+                        if (precompiliationResult?.NewIncludes?.Any() == true)
+                        {
+                            extraRefs.AddRange(precompiliationResult.NewReferences);
+                        }
+                    }
+                }
+
                 if (scriptFile == null && tempScriptFile == null)
                 {
                     // if (!DisableReferencingFromCode && info?.CodeKind != SourceCodeKind.Script)
@@ -247,9 +242,17 @@ override protected (byte[] asm, byte[] pdb, Project project) Compile(string scri
                     {
                         var parts = File.ReadAllText(file).SeparateUsingsFromCode();
                         var usings = parts[0].GetLines();
-                        var code = parts[1].GetLines();
+                        var code = parts[1];
+
+                        if (!file.EndsWith(Globals.InjectedAttributesPrefix) && lightParser.Precompilers.Any())
+                        {
+                            var precompResult = base.PrecompileImportedScript(code, lightParser);
 
-                        importedSources[file] = (usings.Count(), code);
+                            if (precompResult != null)
+                                code = precompResult.Content;
+                        }
+
+                        importedSources[file] = (usings.Count(), code.GetLines());
                         add_code(file, usings, 0);
                     }
 
@@ -320,7 +323,7 @@ void add_code(string file, string[] codeLines, int lineOffset)
 
                 var explicitRefs = this.refAssemblies.Except(refs); // from code
 
-                foreach (var asm in refs.Concat(explicitRefs))
+                foreach (var asm in refs.Concat(explicitRefs).Concat(extraRefs.Select(x => Assembly.LoadFrom(x))))
                 {
                     var metadata = ToMetadata(asm);
                     if (metadata != null)
@@ -671,4 +674,28 @@ public override IEvaluator Reset(bool referenceDomainAssemblies = true)
             return this;
         }
     }
+
+    static class localExtensions
+    {
+        public static (string file, int line) Translate(this Dictionary<(int, int), (string, int)> mapping, int line)
+        {
+            foreach ((int start, int end) range in mapping.Keys)
+                if (range.start <= line && line <= range.end)
+                {
+                    (string file, int lineOffset) = mapping[range];
+                    return (file, line - range.start + lineOffset);
+                }
+
+            return ("", 0);
+        }
+
+        static public string[] SeparateUsingsFromCode(this string code)
+        {
+            SyntaxTree tree = CSharpSyntaxTree.ParseText(code);
+            CompilationUnitSyntax root = tree.GetCompilationUnitRoot();
+            int pos = root.Usings.FullSpan.End;
+
+            return new[] { code.Substring(0, pos).TrimEnd(), code.Substring(pos) };
+        }
+    }
 }

src/CSScriptLib/src/CSScriptLib/EvaluatorBase.cs

@@ -33,17 +33,22 @@
 //using Microsoft.CodeAnalysis;
 //using Microsoft.CodeAnalysis.CSharp.Scripting
 using System;
+using System.Collections;
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.IO;
 using System.Linq;
 using System.Reflection;
+using System.Reflection.Metadata;
 using System.Runtime.Loader;
 using System.Runtime.Serialization;
 using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
+using System.Xml.Linq;
 using Microsoft.CodeAnalysis;
 using Microsoft.CodeAnalysis.CSharp;
+using Microsoft.CodeAnalysis.Scripting;
 using csscript;
 using CSScripting;
 using CSScripting.CodeDom;
@@ -1028,6 +1033,141 @@ public IEvaluator ReferenceAssembliesFromCode(string code, params string[] searc
             return this;
         }
 
+        Dictionary<string, (MethodInfo method, MethodInfo legacyMethod)> precompilersCache = new();
+
+        internal PrecompilationContext PrecompileImportedScript(string scriptCode, CSharpParser primaryScript)
+        {
+            string[] searchDirs = [this.GetType().Assembly.Location().GetDirName(), .. primaryScript.ExtraSearchDirs];
+
+            var retval = new PrecompilationContext { SearchDirs = searchDirs };
+
+            Hashtable contextData = new()
+            {
+                ["NewDependencies"] = retval.NewDependencies,
+                ["NewSearchDirs"] = retval.NewSearchDirs,
+                ["NewReferences"] = retval.NewReferences,
+                ["NewIncludes"] = retval.NewIncludes,
+                ["NewCompilerOptions"] = "",
+                ["SearchDirs"] = retval.SearchDirs,
+            };
+
+            var content = scriptCode;
+            var modified = false;
+
+            foreach (string file in primaryScript.Precompilers)
+            {
+                var precompilerPath = Precompiler.FindImlementationFile(file, searchDirs);
+                if (!precompilersCache.ContainsKey(precompilerPath))
+                    throw new Exception("Precompiler " + file + " cache cannot be loaded."); // but it must exist since the script precompilers have been processed already
+
+                var precompiler = precompilersCache[precompilerPath];
+
+                bool result = ApplyPrecompilation(null, retval, contextData, ref content, precompilerPath);
+
+                if (result)
+                {
+                    retval.Content = content;
+                    retval.NewDependencies.Add(file);
+                    modified = true;
+                }
+            }
+            return modified ? retval : null;
+        }
+
+        internal PrecompilationContext PrecompileScript(string script, CSharpParser parser)
+        {
+            string[] searchDirs = [this.GetType().Assembly.Location().GetDirName(), .. parser.ExtraSearchDirs];
+
+            var retval = new PrecompilationContext { SearchDirs = searchDirs };
+
+            Hashtable contextData = new()
+            {
+                ["NewDependencies"] = retval.NewDependencies,
+                ["NewSearchDirs"] = retval.NewSearchDirs,
+                ["NewReferences"] = retval.NewReferences,
+                ["NewIncludes"] = retval.NewIncludes,
+                ["NewCompilerOptions"] = "",
+                ["SearchDirs"] = retval.SearchDirs,
+            };
+
+            var content = parser.Code;
+            var modified = false;
+
+            foreach (string file in parser.Precompilers)
+            {
+                var precompilerPath = Precompiler.FindImlementationFile(file, searchDirs);
+
+                using (SimpleAsmProbing.For(searchDirs))
+                {
+                    if (!precompilersCache.ContainsKey(precompilerPath))
+                    {
+                        (byte[] asm, byte[] pdb, Project project) precompilerInfo = Compile(null, file, null);
+
+                        var precompilerAsm = Assembly.Load(precompilerInfo.asm, precompilerInfo.pdb);
+
+                        var precompilerType = precompilerAsm.GetTypes().FirstOrDefault(x => x.Name.EndsWith("Precompiler"));
+                        if (precompilerType == null)
+                            throw new Exception("Precompiler " + file + " cannot be loaded. CreateInstance returned null.");
+
+                        var methods = precompilerType.GetMethods(BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Static | BindingFlags.Instance)
+                                                     .Where(x => x.Name == "Compile");
+
+                        precompilersCache[precompilerPath] =
+                            (
+                                method: methods.FirstOrDefault(x => x.GetParameters().Count() == 1),
+                                legacyMethod: methods.FirstOrDefault(x => x.GetParameters().Count() == 4)
+                            );
+                    }
+
+                    bool result = ApplyPrecompilation(script, retval, contextData, ref content, precompilerPath);
+
+                    if (result)
+                    {
+                        retval.Content = content;
+                        retval.NewDependencies.Add(file);
+                        modified = true;
+                    }
+                }
+            }
+            return modified ? retval : null;
+        }
+
+        private bool ApplyPrecompilation(string script, PrecompilationContext retval, Hashtable contextData, ref string content, string precompilerPath)
+        {
+            var (method, legacyMethod) = precompilersCache[precompilerPath];
+
+            bool result;
+
+            if (method != null)
+            {
+                // bool Compile(dynamic context) bool
+                // Compile(PrecompilationContext context)
+                object compiler = null;
+                if (!method.IsStatic)
+                    compiler = Activator.CreateInstance(method.DeclaringType);
+
+                retval.Content = content;
+
+                result = (bool)method.Invoke(compiler, [retval]);
+
+                if (result)
+                    content = retval.Content;
+            }
+            else
+            {
+                // public static bool Compile(ref string scriptCode, string
+                // scriptFile, bool isPrimaryScript, Hashtable context)
+                var compile = (Precompiler.CompileMethod)Delegate.CreateDelegate(typeof(Precompiler.CompileMethod), legacyMethod);
+
+                result = compile(ref content,
+                                     script,
+                                     IsPrimaryScript: script != null,
+                                     contextData);
+            }
+
+            return result;
+        }
+
         /// <summary>
         /// References the given assembly by the assembly path.
         /// <para>

src/Tests.CSScriptLib/Evaluator.Api.Test.cs

@@ -65,13 +65,13 @@ public API_Roslyn()
         string testTempFile(string fileName, [CallerMemberName] string caller = null)
         {
             var rootDir = "TestData".PathJoin(nameof(API_Roslyn), caller).GetFullPath().EnsureDir();
-            if (fileName == "asm.hidden-from-xunit-dll-file")
-            {
-                File.AppendAllText(
-                    @"D:\dev\cs-script\src\Tests.CSScriptLib\bin\Debug\net10.0\TestData\test-error.log",
-                    rootDir.PathJoin(fileName) + Environment.NewLine);
-                Debugger.Launch(); // asm.dll is a special file name that xUnit locks just because it was present in the local dir. So avoid using it in the tests.
-            }
+            // if (fileName == "asm.hidden-from-xunit-dll-file")
+            // {
+            //     File.AppendAllText(
+            //         @"D:\dev\cs-script\src\Tests.CSScriptLib\bin\Debug\net10.0\TestData\test-error.log",
+            //         rootDir.PathJoin(fileName) + Environment.NewLine);
+            //     Debugger.Launch(); // asm.dll is a special file name that xUnit locks just because it was present in the local dir. So avoid using it in the tests.
+            // }
 
             return Path.Combine(rootDir, fileName);
         }