如何使用file . readlines (fileName). getenumerator()逐行读取20到200gb

我们正在尝试下面的代码。

 public static int SplitFile(string fileName, string tmpFolder, List<string> queue, int splitSize = 100000)
    {
        int chunk = 0;
        if (!Directory.Exists(tmpFolder))
            Directory.CreateDirectory(tmpFolder);
        using (var lineIterator = File.ReadLines(fileName).GetEnumerator())
        {
            bool stillGoing = true;
            for (chunk = 0; stillGoing; chunk++)
            {
                stillGoing = WriteChunk(lineIterator, splitSize, chunk, tmpFolder, queue);
            }
        }
        return chunk;
    }
    private static bool WriteChunk(IEnumerator<string> lineIterator,
                                   int splitSize, int chunk, string tmpFolder, List<string> queue)
    {
        try
        {
            //int tmpChunkSize = 1000;
            //int tmpChunkInc = 0;
            string splitFile = Path.Combine(tmpFolder, "file" + chunk + ".txt");
            using (var writer = File.CreateText(splitFile))
            {
                queue.Add(splitFile);
                for (int i = 0; i < splitSize; i++)
                {
                    if (!lineIterator.MoveNext())
                    {
                        return false;
                    }
                    writer.WriteLine(lineIterator.Current);
                }
            }
            return true;
        }
        catch (Exception)
        {
            throw;
        }
    }

它创建了大约36个文本文件(大约800 MB)，但在lineIterator.MoveNext()创建第37个文件时开始抛出"内存不足异常"。

。当前显示调试器中的值。