c#：实现动态编译，并实现动态MultiProcess功能（来自python multiprocess的想法）

　　由于之前一直遇到一些关于并行进行数据处理的时效果往往不好，不管是c#还是java程序都是一样，但是在Python中通过multiprocess实现同样的功能时，却发现确实可以提高程序运行的性能，及服务器资源使用提高。python具体性能及multiprocess用法，请参考：《Python：使用pymssql批量插入csv文件到数据库测试》

　　如有转载请标明原文地址：http://www.cnblogs.com/yy3b2007com/p/7228337.html

　　很久之前就设想如何在c#中实现多进程的方案，如今终于设计出了C#中动态实现多进程的方案：

具体代码：

 using Microsoft.CSharp;
 using System;
 using System.CodeDom;
 using System.CodeDom.Compiler;
 using System.Collections.Generic;
 using System.Linq;
 using System.Reflection;
 using System.Text;

 namespace MutilProcessDemo
 {
     public class Program
     {
         static System.Timers.Timer timer = null;

         public static void Main(string[] args)
         {
             GenerateExe();

             timer = new System.Timers.Timer();
             timer.AutoReset = true;
             timer.Enabled = true;
             timer.Interval = ;
             timer.Elapsed += Timer_Elapsed;

             Console.WriteLine("Press Enter key to exit...");
             Console.ReadKey();
         }

         private static void DynamicCompiler()
         {
             // 1.CSharpCodePrivoder
             CSharpCodeProvider objCSharpCodePrivoder = new CSharpCodeProvider();

             // 2.ICodeComplier
             ICodeCompiler objICodeCompiler = objCSharpCodePrivoder.CreateCompiler();

             // 3.CompilerParameters
             CompilerParameters objCompilerParameters = new CompilerParameters();
             objCompilerParameters.ReferencedAssemblies.Add("System.dll");
             objCompilerParameters.GenerateExecutable = false;
             objCompilerParameters.GenerateInMemory = true;

             // 4.CompilerResults
             CompilerResults cr = objICodeCompiler.CompileAssemblyFromSource(objCompilerParameters, GenerateCode());

             if (cr.Errors.HasErrors)
             {
                 Console.WriteLine("编译错误：");
                 foreach (CompilerError err in cr.Errors)
                 {
                     Console.WriteLine(err.ErrorText);
                 }
             }
             else
             {
                 // 通过反射，调用HelloWorld的实例
                 Assembly objAssembly = cr.CompiledAssembly;
                 object objHelloWorld = objAssembly.CreateInstance("DynamicCodeGenerate.HelloWorld");
                 MethodInfo objMI = objHelloWorld.GetType().GetMethod("OutPut");

                 Console.WriteLine(objMI.Invoke(objHelloWorld, null));
             }

             Console.ReadLine();
         }

         static string GenerateCode()
         {
             StringBuilder sb = new StringBuilder();
             sb.Append("using System;");
             sb.Append(Environment.NewLine);
             sb.Append("namespace DynamicCodeGenerate");
             sb.Append(Environment.NewLine);
             sb.Append("{");
             sb.Append(Environment.NewLine);
             sb.Append(" public class HelloWorld");
             sb.Append(Environment.NewLine);
             sb.Append(" {");
             sb.Append(Environment.NewLine);
             sb.Append(" public string OutPut()");
             sb.Append(Environment.NewLine);
             sb.Append(" {");
             sb.Append(Environment.NewLine);
             sb.Append(" return \"Hello world!\";");
             sb.Append(Environment.NewLine);
             sb.Append(" }");
             sb.Append(Environment.NewLine);
             sb.Append(" }");
             sb.Append(Environment.NewLine);
             sb.Append("}");

             string code = sb.ToString();
             Console.WriteLine(code);
             Console.WriteLine();

             return code;
         }

         private static void Timer_Elapsed(object sender, System.Timers.ElapsedEventArgs e)
         {
             System.Diagnostics.Process[] processes = System.Diagnostics.Process.GetProcessesByName("HelloWorld");

             int maxProcessCount = ;
             // 如果超过了最大允许任务数据不再触发新的进程任务
             if (processes.Length >= maxProcessCount)
                 return;

             List<int> tasks = new List<int>();
             Random random = new Random();
             for (int i = ; i < maxProcessCount - processes.Length; i++)
             {
                 tasks.Add(random.Next());
             }

             System.Threading.Tasks.Task[] taskItems = new System.Threading.Tasks.Task[tasks.Count];

             for (int i = ; i < tasks.Count; i++)
             {
                 //TaskBase taskSubmit = new ParseMrToHdfsFileItemTask();
                 //taskItems[i] = System.Threading.Tasks.Task.Factory.StartNew(taskSubmit.Submit, new ParseMrToHdfsFileItemTaskArgument() { Task = tasks[i], ComputeNode = this.ComputeNode }, TaskCreationOptions.PreferFairness);
                 taskItems[i] = System.Threading.Tasks.Task.Factory.StartNew((object state) =>
                 {
                     System.Diagnostics.Process process = new System.Diagnostics.Process();
                 //System.Diagnostics.Process process = System.Diagnostics.Process.Start("HelloWorld.exe", state.ToString());
                 process.StartInfo = new System.Diagnostics.ProcessStartInfo();
                     process.StartInfo.CreateNoWindow = true;
                     process.StartInfo.UseShellExecute = false;
                     process.StartInfo.WindowStyle = System.Diagnostics.ProcessWindowStyle.Hidden;
                     process.StartInfo.Arguments = state.ToString();
                     process.StartInfo.FileName = "HelloWorld.exe";
                     process.Start();
                     process.WaitForExit();
                     process.Close();
                     process.Dispose();
                 }, tasks[i]);
             }

             System.Threading.Tasks.Task.WaitAll(taskItems);
         }

         private static void GenerateExe()
         {
             // 创建编译器对象
             CSharpCodeProvider p = new CSharpCodeProvider();
             ICodeCompiler cc = p.CreateCompiler();

             // 设置编译参数
             CompilerParameters options = new CompilerParameters();
             options.ReferencedAssemblies.Add("System.dll");
             options.ReferencedAssemblies.Add("MutilProcessDemo.exe");
             options.GenerateExecutable = true;
             options.OutputAssembly = "HelloWorld.exe";

             //options.ReferencedAssemblies.Add("System.Windows.Forms.dll");
             //options.EmbeddedResources.Add("Data.xml"); // 添加内置资源
             //options.CompilerOptions += " /target:winexe";
             //options.CompilerOptions += " /res:Resource1.res";
             //options.CompilerOptions += " /win32icon:test.ico";

             // 创建源码

             // 1. 使用CodeDom创建源码
             //CodeCompileUnit cu = new CodeCompileUnit();
             //CodeNamespace Samples = new CodeNamespace("Samples");
             //cu.Namespaces.Add(Samples);
             //Samples.Imports.Add(new CodeNamespaceImport("System"));
             //CodeTypeDeclaration Class1 = new CodeTypeDeclaration("Class1");
             //Samples.Types.Add(Class1);
             //CodeEntryPointMethod Start = new CodeEntryPointMethod();
             //CodeMethodInvokeExpression cs1 = new CodeMethodInvokeExpression(
             // new CodeTypeReferenceExpression("System.Console"), "WriteLine",
             // new CodePrimitiveExpression("Hello World!")
             // );
             //Start.Statements.Add(new CodeExpressionStatement(cs1));
             //Class1.Members.Add(Start);

             // 2. 直接指定源码字符串
             string code = @"
 using System;
 using MutilProcessDemo;

 namespace Samples
 {
     public class Class1
     {
         static void Main(string[] args)
         {
             Console.WriteLine(""Hello, World!"");
             MutilProcessDemo.Program.DoMethod(args);
             Console.WriteLine(DateTime.Now.ToString());
         }
     }
 }";
             CodeSnippetCompileUnit codeSnippetCompileUnit = new CodeSnippetCompileUnit(code);

             // 开始编译
             CompilerResults compilerResults = cc.CompileAssemblyFromDom(options, codeSnippetCompileUnit);

             // 显示编译信息
             if (compilerResults.Errors.Count == )
                 Console.WriteLine("{0}compiled ok!", compilerResults.CompiledAssembly.Location);
             else
             {
                 Console.WriteLine("Complie Error:");
                 foreach (CompilerError error in compilerResults.Errors)
                     Console.WriteLine(" {0}", error);
             }
         }

         public static void DoMethod(string[] args)
         {
             System.Console.WriteLine("begin ..." + args[]);

             for (int i = ; i < int.Parse(args[]); i++)
             {
                 System.Threading.Thread.Sleep();
             }

             System.Console.WriteLine("end..." + args[]);
         }
     }
 }

上边的程序运行之后会在\MutilProcessDemo\bin\Debug\下出现两个可执行的.exe：HelloWorld.exe、MutilProcessDemo.exe。

实战：

 public class Program_ForInsertATU : BaseDao
 {
     static readonly BuildingsBO buildBO = new BuildingsBO();
     static readonly FingerPrintDatabaseBO fingerPrintDatabaseBO = new FingerPrintDatabaseBO();
     static readonly GridMappingBuildingBO gridMappingBuildingBO = new GridMappingBuildingBO();
     static readonly SiteBO siteBO = new SiteBO();
     static NLog.Logger logger = new NLog.LogFactory().GetCurrentClassLogger();

     private static void GenerateExe()
     {
         // 创建编译器对象
         CSharpCodeProvider p = new CSharpCodeProvider();
         ICodeCompiler cc = p.CreateCompiler();

         // 设置编译参数
         CompilerParameters options = new CompilerParameters();
         options.ReferencedAssemblies.Add("System.dll");
         options.ReferencedAssemblies.Add("System.Data.Linq.dll");
         options.ReferencedAssemblies.Add("System.Data.dll");
         options.ReferencedAssemblies.Add("System.Core.dll");

         options.ReferencedAssemblies.Add("Dx.Business.dll");
         options.ReferencedAssemblies.Add("Dx.Model.dll");
         options.ReferencedAssemblies.Add("Dx.Utility.dll");
         options.ReferencedAssemblies.Add("Dx.Configuration.dll");
         options.ReferencedAssemblies.Add("Dx.IDAL.dll");
         options.ReferencedAssemblies.Add("Dx.DataAccess.dll");
         options.ReferencedAssemblies.Add("Dx.Data.dll");
         options.ReferencedAssemblies.Add("Dx.Framework.dll");
         options.ReferencedAssemblies.Add("ICSharpCode.SharpZipLib.dll");
         options.ReferencedAssemblies.Add("Microsoft.CSharp.dll");

         options.EmbeddedResources.Add("HelloWorld.exe.config");
         //options.CompilerOptions += " /target:winexe";
         //options.CompilerOptions += " /res:Resource1.res";
         //options.CompilerOptions += " /win32icon:test.ico";

         options.GenerateExecutable = true;
         options.OutputAssembly = "HelloWorld.exe";

         // 2. 直接指定源码字符串
         string code = @"
 using System;
 using System.Collections.Generic;
 using System.Linq;
 using System.Text;
 using System.IO;
 using System.Data;
 using System.Data.SqlClient;
 using System.Threading;
 using System.Threading.Tasks;

 using Dx.Data;
 using Dx.Business.Signal;
 using Dx.Model.FingerprintDatabase;
 using Dx.Business;
 using Dx.Model;
 using Dx.Utility;
 using Dx.Configuration;
 using Dx.Business.FingerprintDatabase.BO;
 using Dx.Business.ATU.Parse;
 using Dx.Framework.Utils;
 using System.Text.RegularExpressions;
 using ICSharpCode.SharpZipLib.GZip;
 using System.Diagnostics;
 using Dx.Business.Scheme;
 using Dx.Business.ATU;
 using Dx.Business.SiChuan;
 using Dx.Model.SiChuan;
 using System.CodeDom;
 using System.CodeDom.Compiler;
 using Microsoft.CSharp;

 namespace Samples
 {
 public class Class1
 {
     static void Main(string[] args)
     {
         ConfigurationManger.Init(ConfigurationType.WindowsService);
         EnumsRegister.Register();

         Dx.Business.SiChuan.GridInnerDoorAnalysis.GridInnerDoorAnalysis bo=new Dx.Business.SiChuan.GridInnerDoorAnalysis.GridInnerDoorAnalysis();
         bo.ComputeGridInnerDoor(args[0]);
     }
 }
 }";
         CodeSnippetCompileUnit codeSnippetCompileUnit = new CodeSnippetCompileUnit(code);

         // 开始编译
         CompilerResults compilerResults = cc.CompileAssemblyFromDom(options, codeSnippetCompileUnit);

         // 显示编译信息
         if (compilerResults.Errors.Count == )
             Console.WriteLine("{0} compiled ok!", compilerResults.CompiledAssembly.Location);
         else
         {
             Console.WriteLine("Complie Error:");
             foreach (CompilerError error in compilerResults.Errors)
                 Console.WriteLine(" {0}", error);
         }
     }

     public static void SubmitTask(object state)
     {
         List<string> ids = state as List<string>;

         System.Threading.Tasks.Task[] taskItems = new System.Threading.Tasks.Task[ids.Count];

         for (int i = ; i < ids.Count; i++)
         {
             taskItems[i] = System.Threading.Tasks.Task.Factory.StartNew((object argument) =>
             {
                 System.Diagnostics.Process process = new System.Diagnostics.Process();

                 process.StartInfo = new System.Diagnostics.ProcessStartInfo();
                 process.StartInfo.CreateNoWindow = true;
                 process.StartInfo.UseShellExecute = false;
                 process.StartInfo.WindowStyle = System.Diagnostics.ProcessWindowStyle.Hidden;
                 process.StartInfo.Arguments = argument.ToString();
                 process.StartInfo.FileName = "HelloWorld.exe";

                 process.Start();

                 //Console.WriteLine(process.ProcessName);
                 process.WaitForExit();
                 process.Close();
                 process.Dispose();
             }, ids[i]);
         }

         System.Threading.Tasks.Task.WaitAll(taskItems);
     }

     public static void Main(string[] args)
     {
         ConfigurationManger.Init(ConfigurationType.WindowsService);
         EnumsRegister.Register();

         Console.WriteLine(DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));

         GenerateExe();

         List<BuildingVector> allBuildingVectors = buildBO.GetAllBuildingsAsGridInnerDoorAnalysis();

         // 将 指纹库数据拷贝到GridMappingBuildings
         // gridMappingBuildingBO.CopyFingerPrinterDatabase();

         List<string> tasks = new List<string>();
         int take = ;
         for (var i = ; i <= allBuildingVectors.Count / take; i++)
         {
             tasks.Add(string.Join(",", allBuildingVectors.Skip(i * take).Take(take).Select(m => m.BuildingID).ToArray()));
         }

         int skipCount = ;

         while (true)
         {
             System.Diagnostics.Process[] processes2 = System.Diagnostics.Process.GetProcessesByName("HelloWorld");

             int maxProcessCount = ;
             // 如果超过了最大允许任务数据不再触发新的进程任务
             if (processes2.Length >= maxProcessCount)
             {
                 Thread.Sleep();
                 continue;
             }

             // submit task
             int submitTaskCount = maxProcessCount - processes2.Length;
             List<string> submitTasks = tasks.Skip(skipCount).Take(submitTaskCount).ToList();

             if(submitTasks.Count==){
                 break;
             }

             System.Threading.ThreadPool.QueueUserWorkItem(SubmitTask, submitTasks);
             Thread.Sleep();
             skipCount += submitTaskCount;

             Console.WriteLine("complete task count:{0}*25,total task count:{1}*25", skipCount, tasks.Count);
         }

         Console.WriteLine("Complete...");
         Console.WriteLine(DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
         Console.ReadKey();
      }
     }

参考文章：

c#动态编译，程序集的动态创建（http://www.360doc.com/content/14/1014/11/5054188_416763069.shtml）