删除行并替换文本文件中的字符串

我有一个文本文件，里面有超过500万行。 我需要逐行运行并删除某些行并替换某个字符串。 我用 C# 编写了一些"有效"的东西，但可能需要将近一天的时间才能完成，这似乎很疯狂，因为在记事本++中进行搜索和替换可以在几分钟内完成。 但是，我们需要将其自动化。

文件可以任意包含一行

"<-comment 1: (more text on the line here)"

和

"<-Another line (more text on the line here)"

我想删除任何以注释 1 开头的行或另一行......

还有一个字符串

<tag>&#x2014;</tag> 

我想用下划线替换。 这应该只出现在以"LINK："开头的行上

到目前为止，我的代码是：

static void Main()
{
    const Int32 BufferSize = 128;
    int count = 0;
    int count2 = 0;
    string filename = @"C:'test'test.txt";
    string output = @"C:'text'output.txt";
    string Startcomment = @"<-comment 1:";
    string Startmoretext= @"<-Another line";
    string othercit = @"LINK:";
    string sub = @"<tag>&#x2014;</tag>";
    string subrepalce = @"_";
    string line;
    using (var filestream = File.OpenRead(filename))
    {
        Console.WriteLine("Start time: " + DateTime.Now.ToString());
        using (var streamreader = new StreamReader(filestream, Encoding.UTF8, true, BufferSize))
        {
            File.WriteAllText(output, "Clean text file" + Environment.NewLine);                    
            while ((line = streamreader.ReadLine()) != null)
            {
                count++;
                if(count % 10000 == 0)
                {
                    Console.WriteLine("Batch complete: " + DateTime.Now.ToString());
                    Console.WriteLine(count);
                }
                if(!line.StartsWith(Startcomment) && !line.StartsWith(Startmoretext))
                {
                    count2++;
                    if (line.StartsWith(othercit))
                    {
                        line = line.Replace(sub, subrepalce);
                    }
                    File.AppendAllText(output, line + Environment.NewLine);
                }
            }                    
        }                
        Console.WriteLine(DateTime.Now.ToString());
        Console.WriteLine(count + " Lines processed");
        Console.WriteLine(count2 + " Lines written back");
        Console.WriteLine("Finished!!!!!!");
        Console.Read();
    }
}

运行时是不可行的。

我想让它在一个正则表达式下运行，如果我们需要添加新的异常，该表达式将使用我们可以在脚本外部维护的配置文件，但似乎也永远运行。

static void Main()
{
    const Int32 BufferSize = 128;
    string filename = @"C:'test'test.txt";
    XmlDocument xdoc = new XmlDocument();
    xdoc.Load(@"C:'test'RegexConfig.xml");
    XmlElement xmlRoot = xdoc.DocumentElement;
    XmlNodeList xmlNodes = xmlRoot.SelectNodes("/root/line");
    int count = 0;
    string line;
    using (var filestream = File.OpenRead(filename))
    {
        Console.WriteLine(DateTime.Now.ToString());
        using (var streamreader = new StreamReader(filestream, Encoding.UTF8, true, BufferSize))
        {
            File.WriteAllText(@"C:'test'output.txt", "Clean file" + Environment.NewLine);
            while ((line = streamreader.ReadLine()) != null)
            {
                string output = line;
                foreach (XmlNode node in xmlNodes)
                {
                    string pattern = node["pattern"].InnerText;
                    string replacement = node["replacement"].InnerText;                           
                    Regex rgx = new Regex(pattern);
                    output = rgx.Replace(output, replacement);
                    rgx = null;
                }
                if (output.Length > 0)
                {
                    count++;
                    if (count % 10000 == 0)
                    {
                        Console.WriteLine(count);
                        Console.WriteLine(DateTime.Now.ToString());
                    }
                    File.AppendAllText(@"C:'test'test.txt", output + Environment.NewLine);
                }
            }
        }
        Console.WriteLine(DateTime.Now.ToString());
        Console.WriteLine("Finished!!!!!!");
        Console.Read();
    }
}

XML 配置文件

<?xml version="1.0" encoding="UTF-8"?>
<root>
    <line>
        <pattern><![CDATA[<-comment 1:.*]]></pattern>
        <replacement><![CDATA[]]></replacement>
    </line> 
    <line>
        <pattern><![CDATA[<-Another line.*]]></pattern>
        <replacement><![CDATA[]]></replacement>
    </line> 
    <line>
        <pattern><![CDATA[<tag>&#x2014;</tag>]]></pattern>
        <replacement>_</replacement>
    </line> 
</root>

应该如何做这样的事情才能最有效地工作？

using (FileStream fs = File.Open(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
using (BufferedStream bs = new BufferedStream(fs))
using (StreamReader sr = new StreamReader(bs))
using (StreamWriter writer = new StreamWriter("C:'test'test.txt"))
{
    string line;
    while ((line = sr.ReadLine()) != null)
    {
        string output = line;
        foreach (XmlNode node in xmlNodes)
        {
             string pattern = node["pattern"].InnerText;
             string replacement = node["replacement"].InnerText;                           
             Regex rgx = new Regex(pattern);
             output = rgx.Replace(output, replacement);
             rgx = null;
        }
        if (output.Length > 0)
        {
             count++;
             if (count % 10000 == 0)
             {
                  Console.WriteLine(count);
                  Console.WriteLine(DateTime.Now.ToString());
             }
             writer.WriteLine(output);
         }
    }
    writer.Close();
}

    const Int32 BufferSize = 128;
    int count = 0;
    int count2 = 0;
    string filename = @"C:'test'test.txt";
    string output = @"C:'text'output.txt";
    string Startcomment = @"<-comment 1:";
    string Startmoretext= @"<-Another line";
    string othercit = @"LINK:";
    string sub = @"<tag>&#x2014;</tag>";
    string subrepalce = @"_";

    string line;
    string[] fileText = File.ReadAllLines(filename);
        Console.WriteLine("Start time: " + DateTime.Now.ToString());
    Parallel.For(0, fileText.Length, i=>{
      if(!fileText[i].StartsWith(Startcomment) && !fileText[i].StartsWith(Startmoretext))
                {
                    count2++;
                    if (fileText[i].StartsWith(othercit))
                    {
                        fileText[i]= fileText[i].Replace(sub, subrepalce);
                    }
                    File.WriteAllLines(yourPath, fileText);
                }
            }                    
        }                
        Console.WriteLine(DateTime.Now.ToString());
        Console.WriteLine(count + " Lines processed");
        Console.WriteLine(count2 + " Lines written back");
        Console.WriteLine("Finished!!!!!!");
        Console.Read();
    });