Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ public ComponentProcessor(MemoryAllocator memoryAllocator, JpegFrame frame, Size
this.Component = component;

this.BlockAreaSize = component.SubSamplingDivisors * blockSize;
this.ColorBuffer = memoryAllocator.Allocate2DOveraligned<float>(
this.ColorBuffer = memoryAllocator.Allocate2DOverAligned<float>(
postProcessorBufferSize.Width,
postProcessorBufferSize.Height,
this.BlockAreaSize.Height);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ public ComponentProcessor(MemoryAllocator memoryAllocator, Component component,
this.blockAreaSize = component.SubSamplingDivisors * 8;

// alignment of 8 so each block stride can be sampled from a single 'ref pointer'
this.ColorBuffer = memoryAllocator.Allocate2DOveraligned<float>(
this.ColorBuffer = memoryAllocator.Allocate2DOverAligned<float>(
postProcessorBufferSize.Width,
postProcessorBufferSize.Height,
8,
Expand Down
3 changes: 3 additions & 0 deletions src/ImageSharp/Formats/Png/PngDecoderCore.cs
Original file line number Diff line number Diff line change
Expand Up @@ -1968,6 +1968,9 @@ private IMemoryOwner<byte> ReadChunkData(int length)
}

// We rent the buffer here to return it afterwards in Decode()
// We don't want to throw a degenerated memory exception here as we want to allow partial decoding
// so limit the length.
length = (int)Math.Min(length, this.currentStream.Length - this.currentStream.Position);
IMemoryOwner<byte> buffer = this.configuration.MemoryAllocator.Allocate<byte>(length, AllocationOptions.Clean);

this.currentStream.Read(buffer.GetSpan(), 0, length);
Expand Down
84 changes: 83 additions & 1 deletion src/ImageSharp/Memory/Allocators/MemoryAllocator.cs
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@
// Licensed under the Six Labors Split License.

using System.Buffers;
using System.Runtime.CompilerServices;
using SixLabors.ImageSharp.PixelFormats;

namespace SixLabors.ImageSharp.Memory;

Expand All @@ -10,6 +12,16 @@ namespace SixLabors.ImageSharp.Memory;
/// </summary>
public abstract class MemoryAllocator
{
/// <summary>
/// Gets the default max allocatable size of a 1D buffer in bytes.
/// </summary>
public static readonly int DefaultMaxAllocatableSize1DInBytes = GetDefaultMaxAllocatableSize1DInBytes();

/// <summary>
/// Gets the default max allocatable size of a 2D buffer in bytes.
/// </summary>
public static readonly ulong DefaultMaxAllocatableSize2DInBytes = GetDefaultMaxAllocatableSize2DInBytes();

/// <summary>
/// Gets the default platform-specific global <see cref="MemoryAllocator"/> instance that
/// serves as the default value for <see cref="Configuration.MemoryAllocator"/>.
Expand All @@ -20,6 +32,18 @@ public abstract class MemoryAllocator
/// </summary>
public static MemoryAllocator Default { get; } = Create();

/// <summary>
/// Gets or sets the maximum allowable allocatable size of a 2 dimensional buffer.
/// Defaults to <value><see cref="DefaultMaxAllocatableSize2DInBytes"/>.</value>
/// </summary>
public ulong MaxAllocatableSize2DInBytes { get; set; } = DefaultMaxAllocatableSize2DInBytes;

/// <summary>
/// Gets or sets the maximum allowable allocatable size of a 1 dimensional buffer.
/// </summary>
/// Defaults to <value><see cref="GetDefaultMaxAllocatableSize1DInBytes"/>.</value>
public int MaxAllocatableSize1DInBytes { get; set; } = DefaultMaxAllocatableSize1DInBytes;
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is in fact a limit for a single contigous buffer.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Well, yes, but you have a different internal value for that which is used for contiguous chunks within a 2D buffer. I had to create something new.


/// <summary>
/// Gets the length of the largest contiguous buffer that can be handled by this allocator instance in bytes.
/// </summary>
Expand All @@ -42,7 +66,7 @@ public static MemoryAllocator Create(MemoryAllocatorOptions options) =>
new UniformUnmanagedMemoryPoolMemoryAllocator(options.MaximumPoolSizeMegabytes);

/// <summary>
/// Allocates an <see cref="IMemoryOwner{T}" />, holding a <see cref="Memory{T}"/> of length <paramref name="length"/>.
/// Allocates an <see cref="IMemoryOwner{T}"/>, holding a <see cref="Memory{T}"/> of length <paramref name="length"/>.
/// </summary>
/// <typeparam name="T">Type of the data stored in the buffer.</typeparam>
/// <param name="length">Size of the buffer to allocate.</param>
Expand All @@ -64,6 +88,7 @@ public virtual void ReleaseRetainedResources()
/// <summary>
/// Allocates a <see cref="MemoryGroup{T}"/>.
/// </summary>
/// <typeparam name="T">Type of the data stored in the buffer.</typeparam>
/// <param name="totalLength">The total length of the buffer.</param>
/// <param name="bufferAlignment">The expected alignment (eg. to make sure image rows fit into single buffers).</param>
/// <param name="options">The <see cref="AllocationOptions"/>.</param>
Expand All @@ -75,4 +100,61 @@ internal virtual MemoryGroup<T> AllocateGroup<T>(
AllocationOptions options = AllocationOptions.None)
where T : struct
=> MemoryGroup<T>.Allocate(this, totalLength, bufferAlignment, options);

internal void MemoryGuardAllocation2D<T>(Size value, string paramName)
where T : struct
{
if (value.Width < 0 || value.Height < 0)
{
throw new InvalidMemoryOperationException($"An allocation was attempted that exceeded allowable limits; \"{paramName}\" at {value.Width}x{value.Height}");
}

ulong typeSizeInBytes = (ulong)Unsafe.SizeOf<T>();
ulong valueInBytes = (ulong)value.Width * typeSizeInBytes * (ulong)value.Height;

if (valueInBytes <= this.MaxAllocatableSize2DInBytes)
{
return;
}

throw new InvalidMemoryOperationException(
$"An allocation was attempted that exceeded allowable limits; \"{paramName}\" at {value.Width}x{value.Height} must be less than or equal to {this.MaxAllocatableSize2DInBytes}, was {valueInBytes}");
}

internal void MemoryGuardAllocation1D<T>(int value, string paramName)
where T : struct
{
if (value < 0)
{
throw new InvalidMemoryOperationException($"An allocation was attempted that exceeded allowable limits; {paramName} must be greater than or equal to zero, was {value}");
}

ulong typeSizeInBytes = (ulong)Unsafe.SizeOf<T>();
ulong valueInBytes = (ulong)value * typeSizeInBytes;

if (valueInBytes <= (ulong)this.MaxAllocatableSize1DInBytes)
{
return;
}

throw new InvalidMemoryOperationException(
$"An allocation was attempted that exceeded allowable limits; \"{paramName}\" must be less than or equal {this.MaxAllocatableSize1DInBytes}, was {valueInBytes}");
}

private static ulong GetDefaultMaxAllocatableSize2DInBytes()
{
// Limit dimensions to 32767x32767 and 16383x16383 @ 4 bytes per pixel for 64 and 32 bit processes respectively.
ulong maxLength = Environment.Is64BitProcess ? ushort.MaxValue / 2 : (ulong)short.MaxValue / 4;
return maxLength * (ulong)Unsafe.SizeOf<Rgba32>() * maxLength;
}

private static int GetDefaultMaxAllocatableSize1DInBytes()
{
// It's possible to require buffers that are not related to image dimensions.
// For example, when we need to allocate buffers for IDAT chunks in PNG files or when allocating
// cache buffers for image quantization.
// Limit the maximum buffer size to 64MB for 64-bit processes and 32MB for 32-bit processes.
int limitInMB = Environment.Is64BitProcess ? 64 : 32;
return limitInMB * 1024 * 1024;
}
}
8 changes: 4 additions & 4 deletions src/ImageSharp/Memory/Allocators/SimpleGcMemoryAllocator.cs
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright (c) Six Labors.
// Copyright (c) Six Labors.
// Licensed under the Six Labors Split License.

using System.Buffers;
Expand All @@ -7,17 +7,17 @@
namespace SixLabors.ImageSharp.Memory;

/// <summary>
/// Implements <see cref="MemoryAllocator"/> by newing up managed arrays on every allocation request.
/// Implements <see cref="MemoryAllocator"/> by creating new managed arrays on every allocation request.
/// </summary>
public sealed class SimpleGcMemoryAllocator : MemoryAllocator
{
/// <inheritdoc />
protected internal override int GetBufferCapacityInBytes() => int.MaxValue;
protected internal override int GetBufferCapacityInBytes() => this.MaxAllocatableSize1DInBytes;

/// <inheritdoc />
public override IMemoryOwner<T> Allocate<T>(int length, AllocationOptions options = AllocationOptions.None)
{
Guard.MustBeGreaterThanOrEqualTo(length, 0, nameof(length));
this.MemoryGuardAllocation1D<T>(length, nameof(length));

return new BasicArrayBuffer<T>(new T[length]);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -79,16 +79,14 @@ internal UniformUnmanagedMemoryPoolMemoryAllocator(
protected internal override int GetBufferCapacityInBytes() => this.poolBufferSizeInBytes;

/// <inheritdoc />
public override IMemoryOwner<T> Allocate<T>(
int length,
AllocationOptions options = AllocationOptions.None)
public override IMemoryOwner<T> Allocate<T>(int length, AllocationOptions options = AllocationOptions.None)
{
Guard.MustBeGreaterThanOrEqualTo(length, 0, nameof(length));
this.MemoryGuardAllocation1D<T>(length, nameof(length));
int lengthInBytes = length * Unsafe.SizeOf<T>();

if (lengthInBytes <= this.sharedArrayPoolThresholdInBytes)
{
var buffer = new SharedArrayPoolBuffer<T>(length);
SharedArrayPoolBuffer<T> buffer = new(length);
if (options.Has(AllocationOptions.Clean))
{
buffer.GetSpan().Clear();
Expand All @@ -102,8 +100,7 @@ public override IMemoryOwner<T> Allocate<T>(
UnmanagedMemoryHandle mem = this.pool.Rent();
if (mem.IsValid)
{
UnmanagedBuffer<T> buffer = this.pool.CreateGuardedBuffer<T>(mem, length, options.Has(AllocationOptions.Clean));
return buffer;
return this.pool.CreateGuardedBuffer<T>(mem, length, options.Has(AllocationOptions.Clean));
}
}

Expand All @@ -124,7 +121,7 @@ internal override MemoryGroup<T> AllocateGroup<T>(

if (totalLengthInBytes <= this.sharedArrayPoolThresholdInBytes)
{
var buffer = new SharedArrayPoolBuffer<T>((int)totalLength);
SharedArrayPoolBuffer<T> buffer = new((int)totalLength);
return MemoryGroup<T>.CreateContiguous(buffer, options.Has(AllocationOptions.Clean));
}

Expand Down
4 changes: 3 additions & 1 deletion src/ImageSharp/Memory/Allocators/UnmanagedMemoryAllocator.cs
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,9 @@ internal class UnmanagedMemoryAllocator : MemoryAllocator

public override IMemoryOwner<T> Allocate<T>(int length, AllocationOptions options = AllocationOptions.None)
{
var buffer = UnmanagedBuffer<T>.Allocate(length);
this.MemoryGuardAllocation1D<T>(length, nameof(length));

UnmanagedBuffer<T> buffer = UnmanagedBuffer<T>.Allocate(length);
if (options.Has(AllocationOptions.Clean))
{
buffer.GetSpan().Clear();
Expand Down
19 changes: 12 additions & 7 deletions src/ImageSharp/Memory/MemoryAllocatorExtensions.cs
Original file line number Diff line number Diff line change
Expand Up @@ -18,20 +18,22 @@ public static class MemoryAllocatorExtensions
/// <param name="memoryAllocator">The memory allocator.</param>
/// <param name="width">The buffer width.</param>
/// <param name="height">The buffer height.</param>
/// <param name="preferContiguosImageBuffers">A value indicating whether the allocated buffer should be contiguous, unless bigger than <see cref="int.MaxValue"/>.</param>
/// <param name="preferContiguousImageBuffers">A value indicating whether the allocated buffer should be contiguous, unless bigger than <see cref="int.MaxValue"/>.</param>
/// <param name="options">The allocation options.</param>
/// <returns>The <see cref="Buffer2D{T}"/>.</returns>
public static Buffer2D<T> Allocate2D<T>(
this MemoryAllocator memoryAllocator,
int width,
int height,
bool preferContiguosImageBuffers,
bool preferContiguousImageBuffers,
AllocationOptions options = AllocationOptions.None)
where T : struct
{
memoryAllocator.MemoryGuardAllocation2D<T>(new Size(width, height), "size");

long groupLength = (long)width * height;
MemoryGroup<T> memoryGroup;
if (preferContiguosImageBuffers && groupLength < int.MaxValue)
if (preferContiguousImageBuffers && groupLength < int.MaxValue)
{
IMemoryOwner<T> buffer = memoryAllocator.Allocate<T>((int)groupLength, options);
memoryGroup = MemoryGroup<T>.CreateContiguous(buffer, false);
Expand Down Expand Up @@ -69,16 +71,16 @@ public static Buffer2D<T> Allocate2D<T>(
/// <typeparam name="T">The type of buffer items to allocate.</typeparam>
/// <param name="memoryAllocator">The memory allocator.</param>
/// <param name="size">The buffer size.</param>
/// <param name="preferContiguosImageBuffers">A value indicating whether the allocated buffer should be contiguous, unless bigger than <see cref="int.MaxValue"/>.</param>
/// <param name="preferContiguousImageBuffers">A value indicating whether the allocated buffer should be contiguous, unless bigger than <see cref="int.MaxValue"/>.</param>
/// <param name="options">The allocation options.</param>
/// <returns>The <see cref="Buffer2D{T}"/>.</returns>
public static Buffer2D<T> Allocate2D<T>(
this MemoryAllocator memoryAllocator,
Size size,
bool preferContiguosImageBuffers,
bool preferContiguousImageBuffers,
AllocationOptions options = AllocationOptions.None)
where T : struct =>
Allocate2D<T>(memoryAllocator, size.Width, size.Height, preferContiguosImageBuffers, options);
Allocate2D<T>(memoryAllocator, size.Width, size.Height, preferContiguousImageBuffers, options);

/// <summary>
/// Allocates a buffer of value type objects interpreted as a 2D region
Expand All @@ -96,14 +98,16 @@ public static Buffer2D<T> Allocate2D<T>(
where T : struct =>
Allocate2D<T>(memoryAllocator, size.Width, size.Height, false, options);

internal static Buffer2D<T> Allocate2DOveraligned<T>(
internal static Buffer2D<T> Allocate2DOverAligned<T>(
this MemoryAllocator memoryAllocator,
int width,
int height,
int alignmentMultiplier,
AllocationOptions options = AllocationOptions.None)
where T : struct
{
memoryAllocator.MemoryGuardAllocation2D<T>(new Size(width, height), "size");

long groupLength = (long)width * height;
MemoryGroup<T> memoryGroup = memoryAllocator.AllocateGroup<T>(
groupLength,
Expand All @@ -127,6 +131,7 @@ internal static IMemoryOwner<byte> AllocatePaddedPixelRowBuffer(
int paddingInBytes)
{
int length = (width * pixelSizeInBytes) + paddingInBytes;
memoryAllocator.MemoryGuardAllocation1D<byte>(length, nameof(length));
return memoryAllocator.Allocate<byte>(length);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ private ResizeKernelMap(
this.sourceLength = sourceLength;
this.DestinationLength = destinationLength;
this.MaxDiameter = (radius * 2) + 1;
this.data = memoryAllocator.Allocate2D<float>(this.MaxDiameter, bufferHeight, preferContiguosImageBuffers: true, AllocationOptions.Clean);
this.data = memoryAllocator.Allocate2D<float>(this.MaxDiameter, bufferHeight, preferContiguousImageBuffers: true, AllocationOptions.Clean);
this.pinHandle = this.data.DangerousGetSingleMemory().Pin();
this.kernels = new ResizeKernel[destinationLength];
this.tempValues = new double[this.MaxDiameter];
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ public ResizeWorker(
this.transposedFirstPassBuffer = configuration.MemoryAllocator.Allocate2D<Vector4>(
this.workerHeight,
targetWorkingRect.Width,
preferContiguosImageBuffers: true,
preferContiguousImageBuffers: true,
options: AllocationOptions.Clean);

this.tempRowBuffer = configuration.MemoryAllocator.Allocate<Vector4>(this.sourceRectangle.Width);
Expand Down
9 changes: 9 additions & 0 deletions tests/ImageSharp.Tests/Formats/Bmp/BmpDecoderTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -558,4 +558,13 @@ public void BmpDecoder_CanDecode_Os2BitmapArray<TPixel>(TestImageProvider<TPixel
// Compare to reference output instead.
image.CompareToReferenceOutput(provider, extension: "png");
}

[Theory]
[WithFile(Issue2696, PixelTypes.Rgba32)]
public void BmpDecoder_ThrowsException_Issue2696<TPixel>(TestImageProvider<TPixel> provider)
where TPixel : unmanaged, IPixel<TPixel>
=> Assert.Throws<InvalidImageContentException>(() =>
{
using Image<TPixel> image = provider.GetImage(BmpDecoder.Instance);
});
}
16 changes: 12 additions & 4 deletions tests/ImageSharp.Tests/Formats/Bmp/BmpEncoderTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@

using SixLabors.ImageSharp.Formats;
using SixLabors.ImageSharp.Formats.Bmp;
using SixLabors.ImageSharp.Memory;
using SixLabors.ImageSharp.Metadata;
using SixLabors.ImageSharp.PixelFormats;
using SixLabors.ImageSharp.Processing;
Expand Down Expand Up @@ -348,14 +349,21 @@ public void Encode_PreservesColorProfile<TPixel>(TestImageProvider<TPixel> provi
Assert.Equal(expectedProfileBytes, actualProfileBytes);
}

public static TheoryData<Size> Encode_WorksWithSizeGreaterThen65k_Data { get; set; } = new()
{
{ new Size(65535, 1) },
{ new Size(1, 65535) },
};

[Theory]
[InlineData(1, 66535)]
[InlineData(66535, 1)]
public void Encode_WorksWithSizeGreaterThen65k(int width, int height)
[MemberData(nameof(Encode_WorksWithSizeGreaterThen65k_Data))]
public void Encode_WorksWithSizeGreaterThen65k(Size size)
{
Exception exception = Record.Exception(() =>
{
using Image image = new Image<Rgba32>(width, height);
Configuration configuration = Configuration.CreateDefaultInstance();
configuration.MemoryAllocator = new UniformUnmanagedMemoryPoolMemoryAllocator(null) { MaxAllocatableSize2DInBytes = ulong.MaxValue };
using Image image = new Image<L8>(configuration, size.Width, size.Height);
using MemoryStream memStream = new();
image.Save(memStream, BmpEncoder);
});
Expand Down
17 changes: 3 additions & 14 deletions tests/ImageSharp.Tests/Formats/Jpg/JpegDecoderTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -338,21 +338,10 @@ public void Issue2564_DecodeWorks<TPixel>(TestImageProvider<TPixel> provider)
}

[Theory]
[WithFile(TestImages.Jpeg.Issues.HangBadScan, PixelTypes.L8)]
public void DecodeHang<TPixel>(TestImageProvider<TPixel> provider)
[WithFile(TestImages.Jpeg.Issues.HangBadScan, PixelTypes.Rgba32)]
public void DecodeHang_ThrowsException<TPixel>(TestImageProvider<TPixel> provider)
where TPixel : unmanaged, IPixel<TPixel>
{
if (TestEnvironment.IsWindows &&
TestEnvironment.RunsOnCI)
{
// Windows CI runs consistently fail with OOM.
return;
}

using Image<TPixel> image = provider.GetImage(JpegDecoder.Instance);
Assert.Equal(65503, image.Width);
Assert.Equal(65503, image.Height);
}
=> Assert.Throws<InvalidImageContentException>(() => { using Image<TPixel> image = provider.GetImage(JpegDecoder.Instance); });

// https://github.com/SixLabors/ImageSharp/issues/2517
[Theory]
Expand Down
Loading