c# - 解压缩字节时抛出 System.OutOfMemory 异常

Question

我正在压缩字节，然后在解压缩时再次出现 OOM 异常。当我有足够的内存来存储它时，我无法理解为什么会出现此错误。

数据压缩后约20MB左右，待解压。但我总是遇到 OutOfMemory 异常。

下面是相同的代码。

public byte[] Compress(byte[] data)
{
    byte[] compressArray = null;
    try
    {
        using (MemoryStream memoryStream = new MemoryStream())
        {
            using (DeflateStream deflateStream = new DeflateStream(memoryStream, CompressionMode.Compress))
            {
                deflateStream.Write(data, 0, data.Length);
                deflateStream.Close();
            }
            compressArray = memoryStream.GetBuffer();
            memoryStream.Dispose();
        }
    }
    catch (Exception exception)
    {
        LogManager.LogEvent(EventLogEntryType.Error, exception.Message);
        return data;
    }
    finally { GC.Collect(); }
    return compressArray;
}

public static byte[] Decompress_Bytes(byte[] data)// Around 20MB data
{
    byte[] decompressedArray = null;
    try
    {
        using (MemoryStream decompressedStream = new MemoryStream())
        {
            using (MemoryStream compressStream = new MemoryStream(data))
            {
                using (DeflateStream deflateStream = new DeflateStream(compressStream, CompressionMode.Decompress))
                {
                    deflateStream.CopyTo(decompressedStream);// Exception thrown at this line.
                    deflateStream.Close();
                }
                compressStream.Dispose();
            }
            decompressedArray = decompressedStream.GetBuffer();
            decompressedStream.Dispose();
        }
    }
    catch (Exception exception)
    {
        return data;
    }
    finally { GC.Collect(); }

    return decompressedArray;
}

为了更好地理解，下面是堆栈跟踪。

at System.IO.MemoryStream.set_Capacity(Int32 value)
at System.IO.MemoryStream.EnsureCapacity(Int32 value)
at System.IO.MemoryStream.Write(Byte[] buffer, Int32 offset, Int32 count)
at System.IO.Stream.InternalCopyTo(Stream destination, Int32 bufferSize)
at System.IO.Stream.CopyTo(Stream destination)
at Symtrax.SQConsole.ConsoleConnectClass.Decompress_Bytes(Byte[] data) in c:\Developement\BI\branch_5.0\MapDesignerUNICODE\ConsoleConnector\SQConsole\ConsoleConnectClass.cs:line 3710

我发现了很多与此相关的问题，但似乎没有一个能解决我的问题。

由于我的声望点数较少，我无法发表评论。因此不得不发布问题。提前致谢。

Answer 1

如评论中所述，您将使用具有不同长度特征的 GetBuffer 获取内部缓冲区，然后只需调用 ToArray。

我在您的代码中添加了一些转储语句，以便 LINQPad 可以揭示正在发生的事情:

public byte[] Compress(byte[] data)
{
    byte[] compressArray = null;
    data.Length.Dump("initial array length");
    try
    {
        using (MemoryStream memoryStream = new MemoryStream())
        {
            using (DeflateStream deflateStream = new DeflateStream(memoryStream, CompressionMode.Compress))
            {
                deflateStream.Write(data, 0, data.Length);
                deflateStream.Close();
            }
            memoryStream.GetBuffer().Length.Dump("buffer compress len");
            compressArray = memoryStream.ToArray();
            compressArray.Length.Dump("compress array len");
            // no need to call Dispose, using does that for you
            //memoryStream.Dispose();
        }
    }
    catch (Exception exception)
    {
        exception.Dump();
        return data;
    }
    finally { GC.Collect(); }
    return compressArray;
}

public static byte[] Decompress_Bytes(byte[] data)// Around 20MB data
{
    byte[] decompressedArray = null;
    try
    {
        using (MemoryStream decompressedStream = new MemoryStream())
        {
            using (MemoryStream compressStream = new MemoryStream(data))
            {
                using (DeflateStream deflateStream = new DeflateStream(compressStream, CompressionMode.Decompress))
                {
                    deflateStream.CopyTo(decompressedStream);// Exception thrown at this line.
                    deflateStream.Close();
                }
                // no need, using does that
                //compressStream.Dispose();
            }
            decompressedStream.GetBuffer().Length.Dump("buffer decompress len");
            decompressedArray = decompressedStream.ToArray();
            decompressedArray.Length.Dump("decompress array len");
            // no need, using does that
            decompressedStream.Dispose();
        }
    }
    catch (Exception exception)
    {
        exception.Dump();
        return data;
    }
    finally { GC.Collect(); }

    return decompressedArray;
}

这是输出:

初始数组长度248404

缓冲区压缩长度262144

压缩数组长度189849

缓冲区解压长度327680

解压数组len248404

正如您从这些数字中看到的，您的长度计数会有很大不同。如果 Deflate 协议(protocol)允许具有额外字节的字节流，那么您可以摆脱那些额外的字节。

使用 GetBuffer 而不是 ToArray 似乎是有益的，但我希望复制最终数组所需的内存分配和 CPU 滴答可以忽略不计，特别是如果无论如何处理内存流。这个事实实际上减少了一点内存占用。

如果您仍然坚持重新使用内存流缓冲区，请确保也返回并提供缓冲区中的实际长度:

public byte[] Compress(byte[] data, out int len)
{
    byte[] compressArray = null;
    data.Length.Dump("initial array length");
    try
    {
        using (MemoryStream memoryStream = new MemoryStream())
        {
            // keep the stream open, we need the length!
            using (DeflateStream deflateStream = new DeflateStream(
                                       memoryStream,
                                       CompressionMode.Compress, 
                                       true))
            {
                deflateStream.Write(data, 0, data.Length);
                deflateStream.Close();
            }
            // output length
            len = (int) memoryStream.Length;                
            compressArray = memoryStream.GetBuffer();
        }
    }
    catch (Exception exception)
    {
        exception.Dump();
        len =-1;
        return data;
    }
    finally { GC.Collect(); }
    return compressArray;
}

public static byte[] Decompress_Bytes(byte[] data, ref int len)// Around 20MB data
{
    byte[] decompressedArray = null;
    try
    {
        using (MemoryStream decompressedStream = new MemoryStream())
        {
            // use the overload that let us limit the memorystream buffer
            using (MemoryStream compressStream = new MemoryStream(data,0, len))
            {
                // keep the stream open
                using (DeflateStream deflateStream = new DeflateStream(
                                compressStream, 
                                CompressionMode.Decompress, 
                                true))
                {
                    deflateStream.CopyTo(decompressedStream);// Exception thrown at this line.
                    deflateStream.Close();
                }
            }
            // output length
            decompressedArray = decompressedStream.GetBuffer();
            len = (int) decompressedStream.Length;
        }
    }
    catch (Exception exception)
    {
        exception.Dump();
        return data;
    }
    finally { GC.Collect(); }

    return decompressedArray;
}

如果你使用上面的代码，你可以这样调用它:

int len;
var cmp = Compress(Encoding.UTF8.GetBytes(sb.ToString()), out len);
var dec = Decompress_Bytes(cmp,ref len);

注意使用 dec 中的字节，您只需要考虑第一个 len 字节数。实际上，这是通过使用 Array.Copy 来完成的，它打败了这个解决方案并将我们带回到调用 ToArray 的那个...