C#大数据文本高效去重

C#大数据文本高效去重

转载请注明出处 http://www.cnblogs.com/Huerye/

TextReader reader = File.OpenText(@"C:\Users\Administrator\Desktop\原始数据.txt");
string[] files = new string[2];
files[0] = @"C:\Users\Administrator\Desktop"+ @"\不重复数据.txt";
files[1] = @"C:\Users\Administrator\Desktop"+ @"\重复数据.txt";
TextWriter writer1 = File.CreateText(files[0]);
TextWriter writer2 = File.CreateText(files[1]);
string currentLine;
int idx = 0;
HashSet<string> previousLines = new HashSet<string>();
while ((currentLine = reader.ReadLine()) != null)
{
if ((++idx % 10000) == 0)
currentLine = currentLine.TrimEnd();
if (previousLines.Add(currentLine))
{
writer1.WriteLine(currentLine);
}
else
{
writer2.WriteLine("第"+idx+"行\t"+currentLine+"出现重复！");
}
}
reader.Close();
writer1.Close();
writer2.Close();
reader.Dispose();
writer1.Dispose();
writer2.Dispose();