Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Expose and conserve the color palette for indexed png images. #2485

Merged
merged 14 commits into from
Oct 3, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 11 additions & 1 deletion src/ImageSharp/Color/Color.cs
Original file line number Diff line number Diff line change
Expand Up @@ -251,7 +251,17 @@ public Color WithAlpha(float alpha)
/// </summary>
/// <returns>A hexadecimal string representation of the value.</returns>
[MethodImpl(InliningOptions.ShortMethod)]
public string ToHex() => this.data.ToRgba32().ToHex();
public string ToHex()
{
if (this.boxedHighPrecisionPixel is not null)
{
Rgba32 rgba = default;
this.boxedHighPrecisionPixel.ToRgba32(ref rgba);
return rgba.ToHex();
}

return this.data.ToRgba32().ToHex();
}

/// <inheritdoc />
public override string ToString() => this.ToHex();
Expand Down
8 changes: 4 additions & 4 deletions src/ImageSharp/Formats/Png/PngDecoder.cs
Original file line number Diff line number Diff line change
Expand Up @@ -61,24 +61,24 @@ protected override Image Decode(DecoderOptions options, Stream stream, Cancellat
case PngColorType.Grayscale:
if (bits == PngBitDepth.Bit16)
{
return !meta.HasTransparency
return !meta.TransparentColor.HasValue
? this.Decode<L16>(options, stream, cancellationToken)
: this.Decode<La32>(options, stream, cancellationToken);
}

return !meta.HasTransparency
return !meta.TransparentColor.HasValue
? this.Decode<L8>(options, stream, cancellationToken)
: this.Decode<La16>(options, stream, cancellationToken);

case PngColorType.Rgb:
if (bits == PngBitDepth.Bit16)
{
return !meta.HasTransparency
return !meta.TransparentColor.HasValue
? this.Decode<Rgb48>(options, stream, cancellationToken)
: this.Decode<Rgba64>(options, stream, cancellationToken);
}

return !meta.HasTransparency
return !meta.TransparentColor.HasValue
? this.Decode<Rgb24>(options, stream, cancellationToken)
: this.Decode<Rgba32>(options, stream, cancellationToken);

Expand Down
103 changes: 62 additions & 41 deletions src/ImageSharp/Formats/Png/PngDecoderCore.cs
Original file line number Diff line number Diff line change
Expand Up @@ -172,21 +172,20 @@ public Image<TPixel> Decode<TPixel>(BufferedReadStream stream, CancellationToken
if (image is null)
{
this.InitializeImage(metadata, out image);

// Both PLTE and tRNS chunks, if present, have been read at this point as per spec.
AssignColorPalette(this.palette, this.paletteAlpha, pngMetadata);
}

this.ReadScanlines(chunk, image.Frames.RootFrame, pngMetadata, cancellationToken);

break;
case PngChunkType.Palette:
byte[] pal = new byte[chunk.Length];
chunk.Data.GetSpan().CopyTo(pal);
this.palette = pal;
this.palette = chunk.Data.GetSpan().ToArray();
break;
case PngChunkType.Transparency:
byte[] alpha = new byte[chunk.Length];
chunk.Data.GetSpan().CopyTo(alpha);
this.paletteAlpha = alpha;
this.AssignTransparentMarkers(alpha, pngMetadata);
this.paletteAlpha = chunk.Data.GetSpan().ToArray();
this.AssignTransparentMarkers(this.paletteAlpha, pngMetadata);
break;
case PngChunkType.Text:
this.ReadTextChunk(metadata, pngMetadata, chunk.Data.GetSpan());
Expand Down Expand Up @@ -292,12 +291,15 @@ public ImageInfo Identify(BufferedReadStream stream, CancellationToken cancellat

this.SkipChunkDataAndCrc(chunk);
break;
case PngChunkType.Palette:
this.palette = chunk.Data.GetSpan().ToArray();
break;

case PngChunkType.Transparency:
byte[] alpha = new byte[chunk.Length];
chunk.Data.GetSpan().CopyTo(alpha);
this.paletteAlpha = alpha;
this.AssignTransparentMarkers(alpha, pngMetadata);
this.paletteAlpha = chunk.Data.GetSpan().ToArray();
this.AssignTransparentMarkers(this.paletteAlpha, pngMetadata);

// Spec says tRNS must be after PLTE so safe to exit.
if (this.colorMetadataOnly)
{
goto EOF;
Expand Down Expand Up @@ -370,6 +372,9 @@ public ImageInfo Identify(BufferedReadStream stream, CancellationToken cancellat
PngThrowHelper.ThrowNoHeader();
}

// Both PLTE and tRNS chunks, if present, have been read at this point as per spec.
AssignColorPalette(this.palette, this.paletteAlpha, pngMetadata);

return new ImageInfo(new PixelTypeInfo(this.CalculateBitsPerPixel()), new(this.header.Width, this.header.Height), metadata);
}
finally
Expand Down Expand Up @@ -766,9 +771,7 @@ private void ProcessDefilteredScanline<TPixel>(ReadOnlySpan<byte> defilteredScan
this.header,
scanlineSpan,
rowSpan,
pngMetadata.HasTransparency,
pngMetadata.TransparentL16.GetValueOrDefault(),
pngMetadata.TransparentL8.GetValueOrDefault());
pngMetadata.TransparentColor);

break;

Expand All @@ -787,8 +790,7 @@ private void ProcessDefilteredScanline<TPixel>(ReadOnlySpan<byte> defilteredScan
this.header,
scanlineSpan,
rowSpan,
this.palette,
this.paletteAlpha);
pngMetadata.ColorTable);

break;

Expand All @@ -800,9 +802,7 @@ private void ProcessDefilteredScanline<TPixel>(ReadOnlySpan<byte> defilteredScan
rowSpan,
this.bytesPerPixel,
this.bytesPerSample,
pngMetadata.HasTransparency,
pngMetadata.TransparentRgb48.GetValueOrDefault(),
pngMetadata.TransparentRgb24.GetValueOrDefault());
pngMetadata.TransparentColor);

break;

Expand Down Expand Up @@ -860,9 +860,7 @@ private void ProcessInterlacedDefilteredScanline<TPixel>(ReadOnlySpan<byte> defi
rowSpan,
(uint)pixelOffset,
(uint)increment,
pngMetadata.HasTransparency,
pngMetadata.TransparentL16.GetValueOrDefault(),
pngMetadata.TransparentL8.GetValueOrDefault());
pngMetadata.TransparentColor);

break;

Expand All @@ -885,8 +883,7 @@ private void ProcessInterlacedDefilteredScanline<TPixel>(ReadOnlySpan<byte> defi
rowSpan,
(uint)pixelOffset,
(uint)increment,
this.palette,
this.paletteAlpha);
pngMetadata.ColorTable);

break;

Expand All @@ -899,9 +896,7 @@ private void ProcessInterlacedDefilteredScanline<TPixel>(ReadOnlySpan<byte> defi
(uint)increment,
this.bytesPerPixel,
this.bytesPerSample,
pngMetadata.HasTransparency,
pngMetadata.TransparentRgb48.GetValueOrDefault(),
pngMetadata.TransparentRgb24.GetValueOrDefault());
pngMetadata.TransparentColor);

break;

Expand All @@ -924,10 +919,44 @@ private void ProcessInterlacedDefilteredScanline<TPixel>(ReadOnlySpan<byte> defi
}
}

/// <summary>
/// Decodes and assigns the color palette to the metadata
/// </summary>
/// <param name="palette">The palette buffer.</param>
/// <param name="alpha">The alpha palette buffer.</param>
/// <param name="pngMetadata">The png metadata.</param>
private static void AssignColorPalette(ReadOnlySpan<byte> palette, ReadOnlySpan<byte> alpha, PngMetadata pngMetadata)
{
if (palette.Length == 0)
{
return;
}

Color[] colorTable = new Color[palette.Length / Unsafe.SizeOf<Rgb24>()];
ReadOnlySpan<Rgb24> rgbTable = MemoryMarshal.Cast<byte, Rgb24>(palette);
for (int i = 0; i < colorTable.Length; i++)
{
colorTable[i] = new Color(rgbTable[i]);
}
Comment on lines +936 to +940
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nit: I see several loops like this one in codecs now. We can maybe define a public static Color.FromPixel<TPixel>(Span<TPixel>, Span<Color>) analogous to the bulk ToPixel<TPixel>(...) method.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Without knowing the component precision of the TPixel we don't know which boxed pixel to assign to. We could do it in v4 though if we added additional information to PixelTypeInfo #2534 (comment)

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah yeah this indeed goes well beyond this PR, nevermind then.


if (alpha.Length > 0)
{
// The alpha chunk may contain as many transparency entries as there are palette entries
// (more than that would not make any sense) or as few as one.
for (int i = 0; i < alpha.Length; i++)
{
ref Color color = ref colorTable[i];
color = color.WithAlpha(alpha[i] / 255F);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If perf is important here, then

Suggested change
color = color.WithAlpha(alpha[i] / 255F);
color = color.WithAlpha(alpha[i] * (1F / 255F));

will produce a mul instead of div.

Even if perf isn't critical here I think that I'd change the code, as

  • it's almost as readable
  • it's not only about the perf of this piece of code, but rather for the whole system, as the cpu-ports can be utilized better when there's no division clogging it

Note: it's Roslyn to blame here for not being able to emit 1 / 255F automatically -- though the JIT could do that optimization too, but it has more stringent timing constraints...

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This would not be a valid optimization for either compiler to make, as the results are not the same (due to 1/255f not being precisely representable in float). See https://sharplab.io/#v2:EYLgxg9gTgpgtADwGwBYA0AXEUCuA7AHwAEAmARgFgAoIgBgAIiyA6AGQEs8BHAbmqIDMjEvQDC9AN7V6M4SKlVZS+gDNoMAIZgAFvQAUnDPQBu9TvQCieHAFsYUDcAA2MZgCUNeAOYw9tNPQkAKxBAJShktLK0ewq+qYA9IEhcQCEALwm9ABU+mQJwUEq4VHRZYxkAJx6ACQARBLGAL4gkonJRU30GW05eQUpoU11oXyKZU2l9JNUTUA===

For that reason, unless it's a hot path, this optimization loses precision for no measurable benefit.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

as the results are not the same

Yep, but the error is in the region of 1e-8 or less (demo)

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In that value range, yes, but the magnitude of the error is relative to the magnitude of the value -- that's how floating point works. Whether the precision loss is acceptable in any given case is a question of the trade-off between performance (~12 cycles for divss vs ~4 for mulss) and precision.

I'm not familiar enough with the code to know whether that trade-off is worthwhile here (though it looks questionable). Mostly, I was commenting on your suggestion that Roslyn and RyuJIT were somehow missing an optimization one of them could have made automatically, which is incorrect.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

which is incorrect

You are right (and I have to apologize here by Roslyn).
I had a look what GCC does (before writing the first comment on this) and it emitted the mul -- but I had -Ofast compiler switch set which disregards strict standards compliance (as I re-read just now). With -O{2,3} it's a div.

Thanks for the heads up!

}
}

pngMetadata.ColorTable = colorTable;
}

/// <summary>
/// Decodes and assigns marker colors that identify transparent pixels in non indexed images.
/// </summary>
/// <param name="alpha">The alpha tRNS array.</param>
/// <param name="alpha">The alpha tRNS buffer.</param>
/// <param name="pngMetadata">The png metadata.</param>
private void AssignTransparentMarkers(ReadOnlySpan<byte> alpha, PngMetadata pngMetadata)
{
Expand All @@ -941,16 +970,14 @@ private void AssignTransparentMarkers(ReadOnlySpan<byte> alpha, PngMetadata pngM
ushort gc = BinaryPrimitives.ReadUInt16LittleEndian(alpha.Slice(2, 2));
ushort bc = BinaryPrimitives.ReadUInt16LittleEndian(alpha.Slice(4, 2));

pngMetadata.TransparentRgb48 = new Rgb48(rc, gc, bc);
pngMetadata.HasTransparency = true;
pngMetadata.TransparentColor = new(new Rgb48(rc, gc, bc));
return;
}

byte r = ReadByteLittleEndian(alpha, 0);
byte g = ReadByteLittleEndian(alpha, 2);
byte b = ReadByteLittleEndian(alpha, 4);
pngMetadata.TransparentRgb24 = new Rgb24(r, g, b);
pngMetadata.HasTransparency = true;
pngMetadata.TransparentColor = new(new Rgb24(r, g, b));
}
}
else if (this.pngColorType == PngColorType.Grayscale)
Expand All @@ -959,20 +986,14 @@ private void AssignTransparentMarkers(ReadOnlySpan<byte> alpha, PngMetadata pngM
{
if (this.header.BitDepth == 16)
{
pngMetadata.TransparentL16 = new L16(BinaryPrimitives.ReadUInt16LittleEndian(alpha[..2]));
pngMetadata.TransparentColor = Color.FromPixel(new L16(BinaryPrimitives.ReadUInt16LittleEndian(alpha[..2])));
}
else
{
pngMetadata.TransparentL8 = new L8(ReadByteLittleEndian(alpha, 0));
pngMetadata.TransparentColor = Color.FromPixel(new L8(ReadByteLittleEndian(alpha, 0)));
}

pngMetadata.HasTransparency = true;
}
}
else if (this.pngColorType == PngColorType.Palette && alpha.Length > 0)
{
pngMetadata.HasTransparency = true;
}
}

/// <summary>
Expand Down Expand Up @@ -1461,7 +1482,7 @@ private bool TryReadChunk(Span<byte> buffer, out PngChunk chunk)

// If we're reading color metadata only we're only interested in the IHDR and tRNS chunks.
// We can skip all other chunk data in the stream for better performance.
if (this.colorMetadataOnly && type != PngChunkType.Header && type != PngChunkType.Transparency)
if (this.colorMetadataOnly && type != PngChunkType.Header && type != PngChunkType.Transparency && type != PngChunkType.Palette)
{
chunk = new PngChunk(length, type);

Expand Down
11 changes: 11 additions & 0 deletions src/ImageSharp/Formats/Png/PngEncoder.cs
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
#nullable disable

using SixLabors.ImageSharp.Advanced;
using SixLabors.ImageSharp.Processing.Processors.Quantization;

namespace SixLabors.ImageSharp.Formats.Png;

Expand All @@ -11,6 +12,16 @@ namespace SixLabors.ImageSharp.Formats.Png;
/// </summary>
public class PngEncoder : QuantizingImageEncoder
{
/// <summary>
/// Initializes a new instance of the <see cref="PngEncoder"/> class.
/// </summary>
public PngEncoder()

// Hack. TODO: Investigate means to fix/optimize the Wu quantizer.
// The Wu quantizer does not handle the default sampling strategy well for some larger images.
// It's expensive and the results are not better than the extensive strategy.
=> this.PixelSamplingStrategy = new ExtensivePixelSamplingStrategy();

/// <summary>
/// Gets the number of bits per sample or per palette index (not per pixel).
/// Not all values are allowed for all <see cref="ColorType" /> values.
Expand Down
40 changes: 27 additions & 13 deletions src/ImageSharp/Formats/Png/PngEncoderCore.cs
Original file line number Diff line number Diff line change
Expand Up @@ -875,7 +875,7 @@ private void WriteGammaChunk(Stream stream)
// 4-byte unsigned integer of gamma * 100,000.
uint gammaValue = (uint)(this.gamma * 100_000F);

BinaryPrimitives.WriteUInt32BigEndian(this.chunkDataBuffer.Span.Slice(0, 4), gammaValue);
BinaryPrimitives.WriteUInt32BigEndian(this.chunkDataBuffer.Span[..4], gammaValue);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

FYI: the Slice here was IIRC intentional, as it produces better code than the slice. There are issues in Roslyn and runtime to care about that, but for .NET 8 these aren't resolved (when I'm not mistaken).

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We're just taking the hit here. One day Roslyn will catch up, but we need greater consistency to ease the (massive) maintenance burden. I'm hoping come v4/.NET8 with the SIMD migration improvements I have planned we won't notice codegen issues.


this.WriteChunk(stream, PngChunkType.Gamma, this.chunkDataBuffer.Span, 0, 4);
}
Expand All @@ -889,27 +889,27 @@ private void WriteGammaChunk(Stream stream)
/// <param name="pngMetadata">The image metadata.</param>
private void WriteTransparencyChunk(Stream stream, PngMetadata pngMetadata)
{
if (!pngMetadata.HasTransparency)
if (pngMetadata.TransparentColor is null)
{
return;
}

Span<byte> alpha = this.chunkDataBuffer.Span;
if (pngMetadata.ColorType == PngColorType.Rgb)
{
if (pngMetadata.TransparentRgb48.HasValue && this.use16Bit)
if (this.use16Bit)
{
Rgb48 rgb = pngMetadata.TransparentRgb48.Value;
Rgb48 rgb = pngMetadata.TransparentColor.Value.ToPixel<Rgb48>();
BinaryPrimitives.WriteUInt16LittleEndian(alpha, rgb.R);
BinaryPrimitives.WriteUInt16LittleEndian(alpha.Slice(2, 2), rgb.G);
BinaryPrimitives.WriteUInt16LittleEndian(alpha.Slice(4, 2), rgb.B);

this.WriteChunk(stream, PngChunkType.Transparency, this.chunkDataBuffer.Span, 0, 6);
}
else if (pngMetadata.TransparentRgb24.HasValue)
else
{
alpha.Clear();
Rgb24 rgb = pngMetadata.TransparentRgb24.Value;
Rgb24 rgb = pngMetadata.TransparentColor.Value.ToRgb24();
alpha[1] = rgb.R;
alpha[3] = rgb.G;
alpha[5] = rgb.B;
Expand All @@ -918,15 +918,17 @@ private void WriteTransparencyChunk(Stream stream, PngMetadata pngMetadata)
}
else if (pngMetadata.ColorType == PngColorType.Grayscale)
{
if (pngMetadata.TransparentL16.HasValue && this.use16Bit)
if (this.use16Bit)
{
BinaryPrimitives.WriteUInt16LittleEndian(alpha, pngMetadata.TransparentL16.Value.PackedValue);
L16 l16 = pngMetadata.TransparentColor.Value.ToPixel<L16>();
BinaryPrimitives.WriteUInt16LittleEndian(alpha, l16.PackedValue);
this.WriteChunk(stream, PngChunkType.Transparency, this.chunkDataBuffer.Span, 0, 2);
}
else if (pngMetadata.TransparentL8.HasValue)
else
{
L8 l8 = pngMetadata.TransparentColor.Value.ToPixel<L8>();
alpha.Clear();
alpha[1] = pngMetadata.TransparentL8.Value.PackedValue;
alpha[1] = l8.PackedValue;
this.WriteChunk(stream, PngChunkType.Transparency, this.chunkDataBuffer.Span, 0, 2);
}
}
Expand Down Expand Up @@ -1175,7 +1177,7 @@ private void WriteChunk(Stream stream, PngChunkType type, Span<byte> data, int o

stream.Write(buffer);

uint crc = Crc32.Calculate(buffer.Slice(4)); // Write the type buffer
uint crc = Crc32.Calculate(buffer[4..]); // Write the type buffer
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same here.


if (data.Length > 0 && length > 0)
{
Expand Down Expand Up @@ -1290,8 +1292,20 @@ private static IndexedImageFrame<TPixel> CreateQuantizedFrame<TPixel>(
}

// Use the metadata to determine what quantization depth to use if no quantizer has been set.
IQuantizer quantizer = encoder.Quantizer
?? new WuQuantizer(new QuantizerOptions { MaxColors = ColorNumerics.GetColorCountForBitDepth(bitDepth) });
IQuantizer quantizer = encoder.Quantizer;
if (quantizer is null)
{
PngMetadata metadata = image.Metadata.GetPngMetadata();
if (metadata.ColorTable is not null)
{
// Use the provided palette in total. The caller is responsible for setting values.
quantizer = new PaletteQuantizer(metadata.ColorTable.Value);
}
else
{
quantizer = new WuQuantizer(new QuantizerOptions { MaxColors = ColorNumerics.GetColorCountForBitDepth(bitDepth) });
}
}

// Create quantized frame returning the palette and set the bit depth.
using IQuantizer<TPixel> frameQuantizer = quantizer.CreatePixelSpecificQuantizer<TPixel>(image.GetConfiguration());
Expand Down
Loading
Loading