ofz#829 ensure palette can fulfil largest index used

Change-Id: I95aa4796875ee71e8b06a2bb8985845f270b8817
Reviewed-on: https://gerrit.libreoffice.org/35103
Reviewed-by: Caolán McNamara <caolanm@redhat.com>
Tested-by: Caolán McNamara <caolanm@redhat.com>

1 files changed, 42 insertions, 25 deletions

diff --git a/filter/source/graphicfilter/itiff/itiff.cxx b/filter/source/graphicfilter/itiff/itiff.cxx
index 68d838482d67..8ef3e8846f0a 100644
--- a/filter/source/graphicfilter/itiff/itiff.cxx
+++ b/filter/source/graphicfilter/itiff/itiff.cxx
@@ -52,6 +52,7 @@ private:
     Bitmap                  aBitmap;
     BitmapWriteAccess*      pAcc;
     sal_uInt16              nDstBitsPerPixel;
+    int                     nLargestPixelIndex;
     std::unique_ptr<AlphaMask>
                             pAlphaMask;
     BitmapWriteAccess*      pMaskAcc;
@@ -121,6 +122,9 @@ private:
         // converts a Scanline to the Windows-BMP format
 
     bool HasAlphaChannel() const;
+
+    void SetPixelIndex(BitmapWriteAccess *pAcc, long nY, long nX, sal_uInt8 cIndex);
+
 public:
 
     TIFFReader()
@@ -128,6 +132,7 @@ public:
         , pTIFF(nullptr)
         , pAcc(nullptr)
         , nDstBitsPerPixel(0)
+        , nLargestPixelIndex(-1)
         , pAlphaMask(nullptr)
         , pMaskAcc(nullptr)
         , nOrigPos(0)
@@ -785,6 +790,11 @@ sal_uInt32 TIFFReader::GetBits( const sal_uInt8 * pSrc, sal_uInt32 nBitsPos, sal
     return nRes;
 }
 
+void TIFFReader::SetPixelIndex(BitmapWriteAccess *pWriteAcc, long nY, long nX, sal_uInt8 cIndex)
+{
+    pWriteAcc->SetPixelIndex(nY, nX, cIndex);
+    nLargestPixelIndex = std::max<int>(nLargestPixelIndex, cIndex);
+}
 
 bool TIFFReader::ConvertScanline(sal_Int32 nY)
 {
@@ -947,7 +957,7 @@ bool TIFFReader::ConvertScanline(sal_Int32 nY)
                             for (sal_Int32 nx = 0; nx < nImageWidth; ++nx)
                             {
                                 nLast += nx == 0 ? BYTESWAP( *pt++ ) : *pt++;
-                                pAcc->SetPixelIndex( nY, nx, nLast );
+                                SetPixelIndex(pAcc, nY, nx, nLast);
                             }
                         }
                         else
@@ -955,7 +965,7 @@ bool TIFFReader::ConvertScanline(sal_Int32 nY)
                             for (sal_Int32 nx = 0; nx < nImageWidth; ++nx)
                             {
                                 sal_uInt8 nLast = *pt++;
-                                pAcc->SetPixelIndex( nY, nx, static_cast<sal_uInt8>( (BYTESWAP((sal_uInt32)nLast) - nMinSampleValue) * nMinMax ) );
+                                SetPixelIndex(pAcc, nY, nx, static_cast<sal_uInt8>( (BYTESWAP((sal_uInt32)nLast) - nMinSampleValue) * nMinMax ));
                             }
                         }
                     }
@@ -967,15 +977,14 @@ bool TIFFReader::ConvertScanline(sal_Int32 nY)
                             for (sal_Int32 nx = 0; nx < nImageWidth; ++nx)
                             {
                                 nLast += *pt++;
-                                pAcc->SetPixelIndex( nY, nx, nLast );
+                                SetPixelIndex(pAcc, nY, nx, nLast);
                             }
                         }
                         else
                         {
                             for (sal_Int32 nx = 0; nx < nImageWidth; ++nx)
                             {
-                                pAcc->SetPixelIndex( nY, nx, static_cast<sal_uInt8>( ((sal_uInt32)*pt++ - nMinSampleValue) * nMinMax ) );
-
+                                SetPixelIndex(pAcc, nY, nx, static_cast<sal_uInt8>( ((sal_uInt32)*pt++ - nMinSampleValue) * nMinMax ));
                             }
                         }
                     }
@@ -992,7 +1001,7 @@ bool TIFFReader::ConvertScanline(sal_Int32 nY)
                     for (sal_Int32 nx = 0; nx < nImageWidth; ++nx)
                     {
                         nVal = ( GetBits( pt, nx * nBitsPerSample, nBitsPerSample ) - nMinSampleValue ) * nMinMax;
-                        pAcc->SetPixelIndex( nY, nx, static_cast<sal_uInt8>(nVal));
+                        SetPixelIndex(pAcc, nY, nx, static_cast<sal_uInt8>(nVal));
                     }
                 }
                 break;
@@ -1013,28 +1022,28 @@ bool TIFFReader::ConvertScanline(sal_Int32 nY)
                         while (nByteCount--)
                         {
                             nByteVal = *pt++;
-                            pAcc->SetPixelIndex( nY, nx++, nByteVal & 1 );
+                            SetPixelIndex(pAcc, nY, nx++, nByteVal & 1);
                             nByteVal >>= 1;
-                            pAcc->SetPixelIndex( nY, nx++, nByteVal & 1 );
+                            SetPixelIndex(pAcc, nY, nx++, nByteVal & 1);
                             nByteVal >>= 1;
-                            pAcc->SetPixelIndex( nY, nx++, nByteVal & 1 );
+                            SetPixelIndex(pAcc, nY, nx++, nByteVal & 1);
                             nByteVal >>= 1;
-                            pAcc->SetPixelIndex( nY, nx++, nByteVal & 1 );
+                            SetPixelIndex(pAcc, nY, nx++, nByteVal & 1);
                             nByteVal >>= 1;
-                            pAcc->SetPixelIndex( nY, nx++, nByteVal & 1 );
+                            SetPixelIndex(pAcc, nY, nx++, nByteVal & 1);
                             nByteVal >>= 1;
-                            pAcc->SetPixelIndex( nY, nx++, nByteVal & 1 );
+                            SetPixelIndex(pAcc, nY, nx++, nByteVal & 1);
                             nByteVal >>= 1;
-                            pAcc->SetPixelIndex( nY, nx++, nByteVal & 1 );
+                            SetPixelIndex(pAcc, nY, nx++, nByteVal & 1);
                             nByteVal >>= 1;
-                            pAcc->SetPixelIndex( nY, nx++, nByteVal );
+                            SetPixelIndex(pAcc, nY, nx++, nByteVal);
                         }
                         if ( nImageWidth & 7 )
                         {
                             nByteVal = *pt++;
                             while ( nx < nImageWidth )
                             {
-                                pAcc->SetPixelIndex( nY, nx++, nByteVal & 1 );
+                                SetPixelIndex(pAcc, nY, nx++, nByteVal & 1);
                                 nByteVal >>= 1;
                             }
                         }
@@ -1045,21 +1054,21 @@ bool TIFFReader::ConvertScanline(sal_Int32 nY)
                         while (nByteCount--)
                         {
                             nByteVal = *pt++;
-                            pAcc->SetPixelIndex( nY, nx, nByteVal & 1 );
+                            SetPixelIndex(pAcc, nY, nx, nByteVal & 1);
                             nByteVal >>= 1;
-                            pAcc->SetPixelIndex( nY, --nx, nByteVal & 1 );
+                            SetPixelIndex(pAcc, nY, --nx, nByteVal & 1);
                             nByteVal >>= 1;
-                            pAcc->SetPixelIndex( nY, --nx, nByteVal & 1 );
+                            SetPixelIndex(pAcc, nY, --nx, nByteVal & 1);
                             nByteVal >>= 1;
-                            pAcc->SetPixelIndex( nY, --nx, nByteVal & 1 );
+                            SetPixelIndex(pAcc, nY, --nx, nByteVal & 1);
                             nByteVal >>= 1;
-                            pAcc->SetPixelIndex( nY, --nx, nByteVal & 1 );
+                            SetPixelIndex(pAcc, nY, --nx, nByteVal & 1);
                             nByteVal >>= 1;
-                            pAcc->SetPixelIndex( nY, --nx, nByteVal & 1 );
+                            SetPixelIndex(pAcc, nY, --nx, nByteVal & 1);
                             nByteVal >>= 1;
-                            pAcc->SetPixelIndex( nY, --nx, nByteVal & 1 );
+                            SetPixelIndex(pAcc, nY, --nx, nByteVal & 1);
                             nByteVal >>= 1;
-                            pAcc->SetPixelIndex( nY, --nx, nByteVal );
+                            SetPixelIndex(pAcc, nY, --nx, nByteVal);
                             nx += 15;
                         }
                         if ( nImageWidth & 7 )
@@ -1069,7 +1078,7 @@ bool TIFFReader::ConvertScanline(sal_Int32 nY)
                             nShift = 7;
                             while ( nx < nImageWidth )
                             {
-                                pAcc->SetPixelIndex( nY, nx++, ( nByteVal >> nShift ) & 1);
+                                SetPixelIndex(pAcc, nY, nx++, ( nByteVal >> nShift ) & 1);
                             }
                         }
                     }
@@ -1090,7 +1099,7 @@ bool TIFFReader::ConvertScanline(sal_Int32 nY)
             sal_uInt8*  pt = pMap[ 0 ];
             for (sal_Int32 nx = 0; nx < nImageWidth; nx++, pt += 2 )
             {
-                pAcc->SetPixelIndex( nY, nx, static_cast<sal_uInt8>( ((sal_uInt32)*pt - nMinSampleValue) * nMinMax) );
+                SetPixelIndex(pAcc, nY, nx, static_cast<sal_uInt8>( ((sal_uInt32)*pt - nMinSampleValue) * nMinMax));
             }
         }
     }
@@ -1111,6 +1120,13 @@ void TIFFReader::MakePalCol()
             nNumColors = (sal_uInt32)1 << nBitsPerSample;
             if ( nNumColors > 256 )
                 nNumColors = 256;
+
+            if (nLargestPixelIndex >= static_cast<int>(nNumColors))
+            {
+                SAL_WARN("filter.tiff", "palette has less entries that largest index used. Expanding palette to match");
+                nNumColors = nLargestPixelIndex + 1;
+            }
+
             pAcc->SetPaletteEntryCount(nNumColors);
             for (sal_uInt32 i = 0; i < nNumColors; ++i)
             {
@@ -1337,6 +1353,7 @@ bool TIFFReader::ReadTIFF(SvStream & rTIFF, Graphic & rGraphic )
                 bStatus = false;
             if ( bStatus )
             {
+                nLargestPixelIndex = -1;
                 if ( nMaxSampleValue == 0 )
                 {
                     if ( nBitsPerSample == 32 )         // sj: i93300, compiler bug, 1 << 32 gives 1 one 32bit windows platforms,