快速批量插入sqlserver方法之我见

--------------------------------------------------------------------------------------------------------------------------
大容量插入三种方法及优劣分析
--------------------------------------------------------------------------------------------------------------------------
/// <summary>
/// bulkcopy类——大容量插入 方法一 使用方便，不牵扯跨域问题，不牵扯数据类型等问题，推荐使用，效率高
/// </summary>
/// <param name="dt">datatable</param>
/// <param name="list">源</param>
/// <param name="tableName">插入表格名称</param>
/// <returns>返回结果</returns>
public static bool PerformBulkCopy(DataTable dt, IList list, string tableName)
{

using (SqlConnection destinationConnection = new SqlConnection(CONN_STRING))
{

destinationConnection.Open();
//开启事务
SqlTransaction sqlbulkTransaction = destinationConnection.BeginTransaction();
using (SqlBulkCopy bulkCopy = new SqlBulkCopy(destinationConnection.ConnectionString))
{
bulkCopy.BatchSize = 500;
bulkCopy.DestinationTableName = tableName;
//根据datatable循环添加映射
for (int i = 0; i < dt.Columns.Count; i++)
{

bulkCopy.ColumnMappings.Add(dt.Columns[i].ColumnName, dt.Columns[i].ColumnName);
}

try
{
//写入数据库
bulkCopy.WriteToServer(dt);
return true;
}
catch (Exception)
{
//回滚操作
sqlbulkTransaction.Rollback();

return false;
}

}

}
}

--------------------------------------------------------------------------------------------------------------------------
/// <summary>
/// 表值参数——大容量插入 方法二 效率最低，在1000条及以下数据时，效率不错，使用较方便，牵扯到数据类型问题
/// </summary>
/// <param name="dt">datatable</param>

private void PerformTableParam(DataTable dt)
{
string connectionString = @"data source=211.144.151.150;initial catalog=sctkNew;persist security info=True;user id=wjh;password=P@ssw0rd";

using (var conn = new SqlConnection(connectionString))
{
conn.Open();
//// Invokes the stored procedure.
using (var cmd = new SqlCommand("sp_insert_jk_users", conn))
{
cmd.CommandType = CommandType.StoredProcedure;

//// Adding a "structured" parameter allows you to insert tons of data with low overhead
var param = new SqlParameter("@usersTable", SqlDbType.Structured) { Value = dt };
cmd.Parameters.Add(param);
cmd.ExecuteNonQuery();
}
}

}

sql语句：

//1.创建表值参数
CREATE TYPE jk_users_bulk_insert AS TABLE (

[VideoTeacher] [nvarchar](100) NULL,
[ImageStype] [nvarchar](100) NULL,
[ImageBig] [nvarchar](1000) NULL
...
)

//2.创建存储过程 参数为表
CREATE PROCEDURE sp_insert_jk_users
@usersTable jk_users_bulk_insert READONLY
AS

INSERT INTO dbo.tb_E_Question(--字段)

SELECT --字段
FROM @usersTable

--------------------------------------------------------------------------------------------------------------------------
///<summary>
///bulk insert ——大容量插入 方法三 效率最高，牵扯到跨域的问题，使用不方便
///</summary>
public void InsertByBlukInsert()
{
//存入sb中，存储试题
StringBuilder shitiSB = new StringBuilder();
foreach (tb_E_Question Question in ListQuestionModel)
{
if (!String.IsNullOrEmpty(Question.QuestionTypeID))
{

//拼写字符串
string shitiModelStr = Question.QuestionID + "|@" +Question.FullFolderId+"|@\\";
shitiSB.Append(shitiModelStr);
}
else
{
if (Question.QuestionTypeID == null)
Ever += "第" + iic + "题在系统里面没有该题型，请确定系统里面有该题型。";
else if (ssid != "true")
Ever += "第" + iic + "题在系统里面已有该题。";

}
}
//不用去掉最后的两个字符
//shitiSB= shitiSB.Remove(shitiSB.Length-2,2);
//写入临时txt文本文件
string fileName=AppDomain.CurrentDomain.BaseDirectory+"\\AdminShiTiImportTemp.txt";
using (FileStream file = new FileStream(fileName, FileMode.Create, FileAccess.Write))
{
using (StreamWriter writer = new StreamWriter(file, Encoding.Unicode))
{
writer.Write(shitiSB);
}
}
//调用Bulk Insert方法插入
tb_E_QuestionService questionServiceBll=new tb_E_QuestionService();
int questionCount= questionServiceBll.BulkInsert(AppDomain.CurrentDomain.BaseDirectory+fileName);
//销毁或清空该文本文件，文件名命名规则：当前用户+文件名

}

sql bluk insert语句：
BULK INSERT dbo.course
FROM '\\127.0.0.1\D_TestFiles\E:\A_yanzhinan_Work\test.txt'
WITH
(
--DATAFILETYPE = 'widenative' ,
FIELDTERMINATOR = ','
,ROWTERMINATOR = ';'

)

---------------------------------------------------附加将集合类转换成DataTable的方法--------------------------------------------------------------

/// <summary>
/// 将集合类转换成DataTable——方法一 没有经过判断null值
/// </summary>
/// <param name="list">集合</param>
/// <returns></returns>
public DataTable ToDataTableOne(IList list)
{
DataTable result = new DataTable();
if (list.Count > 0)
{
PropertyInfo[] propertys = list[0].GetType().GetProperties();
foreach (PropertyInfo prop in propertys )
{
Type t = GetCoreType(prop.PropertyType);

tb.Columns.Add(prop.Name, t);
}

for (int i = 0; i < list.Count; i++)
{
ArrayList tempList = new ArrayList();
foreach (PropertyInfo pi in propertys)
{
object obj = pi.GetValue(list[i], null);
tempList.Add(obj);
}
object[] array = tempList.ToArray();
result.LoadDataRow(array, true);
}
}
return result;
}

/// <summary>
/// 将集合类转换成DataTable——方法二 推荐方案，转换可控性强，null值经过处理
/// </summary>
private DataTable ToDataTableTwo<T>(List<T> items)
{
var tb = new DataTable(typeof(T).Name);

PropertyInfo[] props = typeof(T).GetProperties(BindingFlags.Public | BindingFlags.Instance);

foreach (PropertyInfo prop in props)
{
//排除null值
Type t = GetCoreType(prop.PropertyType);
//可以使用add的第二个参数，类型参数
tb.Columns.Add(prop.Name, t);
}

foreach (T item in items)
{
var values = new object[props.Length];
int k = 0;
for (int i = 0; i < props.Length; i++)
{

values[i] = props[i].GetValue(item, null);
}

tb.Rows.Add(values);

}
//去除相关无用属性，具体情况具体判断
tb.Columns.Remove("UserAnser");
tb.Columns.Remove("Marked");
tb.Columns.Remove("SelectAnswerList");
tb.Columns.Remove("QuestionType");
return tb;
}

/// <summary>
/// Determine of specified type is nullable
/// </summary>
public static bool IsNullable(Type t)
{
return !t.IsValueType || (t.IsGenericType && t.GetGenericTypeDefinition() == typeof(Nullable<>));
}

/// <summary>
/// Return underlying type if type is Nullable otherwise return the type
/// </summary>
public static Type GetCoreType(Type t)
{
if (t != null && IsNullable(t))
{
if (!t.IsValueType)
{
return t;
}
else
{
return Nullable.GetUnderlyingType(t);
}
}
else
{
return t;
}
}

/// <summary>
///将集合类转换成DataTable——方法三 利用IEnumerable 较简洁，但过程不方便控制，null值没有经过判断
/// </summary>
public static DataTable ToDataTableThree<T>(IEnumerable<T> collection)
{
var props = typeof(T).GetProperties();
var dt = new DataTable();
dt.Columns.AddRange(props.Select(p => new DataColumn(p.Name)).ToArray());
object[] pos = new object[dt.Columns.Count];
if (collection.Count() > 0)
{
for (int i = 0; i < collection.Count(); i++)
{

ArrayList tempList = new ArrayList();
foreach (PropertyInfo pi in props)
{

object obj = pi.GetValue(collection.ElementAt(i), pos);

tempList.Add(obj);
}
object[] array = tempList.ToArray();
dt.LoadDataRow(array, true);
}
}
return dt;
}

推荐相关网址：

http://www.cnblogs.com/wlb/archive/2010/03/02/1676136.html

http://blog.csdn.net/zhoufoxcn/article/details/1871514

http://www.cnblogs.com/rush/archive/2012/08/31/2666090.html