Re-factoring to make re-use in a next-gen iOS app easier

Change-Id: I1656d38fb8ad4213417b8c00c0c84540e0eacdbe
Reviewed-on: https://gerrit.libreoffice.org/c/online/+/98499
Tested-by: Jenkins
Tested-by: Jenkins CollaboraOffice <jenkinscollaboraoffice@gmail.com>
Reviewed-by: Tor Lillqvist <tml@collabora.com>

3 files changed, 617 insertions, 538 deletions

diff --git a/Makefile.am b/Makefile.am
index ce77b29265..d75de4b49f 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -256,6 +256,7 @@ shared_headers = common/Common.hpp \
                  common/MobileApp.hpp \
                  common/Png.hpp \
                  common/Rectangle.hpp \
+                 common/RenderTiles.hpp \
                  common/SigUtil.hpp \
                  common/security.h \
                  common/SpookyV2.h \
diff --git a/common/RenderTiles.hpp b/common/RenderTiles.hpp
new file mode 100644
index 0000000000..4c5ede5cd8
--- /dev/null
+++ b/common/RenderTiles.hpp
@@ -0,0 +1,595 @@
+/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4; fill-column: 100 -*- */
+/*
+ * This file is part of the LibreOffice project.
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ */
+
+#pragma once
+
+#include <cassert>
+#include <thread>
+
+#include "Png.hpp"
+#include "TileDesc.hpp"
+
+#if ENABLE_DEBUG
+#  define ADD_DEBUG_RENDERID (" renderid=" + Util::UniqueId() + '\n')
+#else
+#  define ADD_DEBUG_RENDERID ("\n")
+#endif
+
+/// A quick & dirty cache of the last few PNGs
+/// and their hashes to avoid re-compression
+/// wherever possible.
+class PngCache
+{
+public:
+    typedef std::shared_ptr< std::vector< char > > CacheData;
+private:
+    struct CacheEntry {
+    private:
+        size_t    _hitCount;
+        TileWireId _wireId;
+        CacheData _data;
+    public:
+        CacheEntry(const CacheData &data, TileWireId id) :
+            _hitCount(1),   // Every entry is used at least once; prevent removal at birth.
+            _wireId(id),
+            _data(data)
+        {
+        }
+
+        size_t getHitCount() const
+        {
+            return _hitCount;
+        }
+
+        void incrementHitCount()
+        {
+            ++_hitCount;
+        }
+
+        void decrementHitCount()
+        {
+            --_hitCount;
+        }
+
+        const CacheData& getData() const
+        {
+            return _data;
+        }
+
+        TileWireId getWireId() const
+        {
+            return _wireId;
+        }
+    } ;
+    size_t _cacheSize;
+    static const size_t CacheSizeSoftLimit = (1024 * 4 * 32); // 128k of cache
+    static const size_t CacheSizeHardLimit = CacheSizeSoftLimit * 2;
+    static const size_t CacheWidHardLimit = 4096;
+    size_t _cacheHits;
+    size_t _cacheTests;
+    TileWireId _nextId;
+    DeltaGenerator _deltaGen;
+
+    std::unordered_map< TileBinaryHash, CacheEntry > _cache;
+    // This uses little storage so can be much larger
+    std::unordered_map< TileBinaryHash, TileWireId > _hashToWireId;
+
+    void clearCache(bool logStats = false)
+    {
+        if (logStats)
+            LOG_DBG("cache clear " << _cache.size() << " items total size " <<
+                    _cacheSize << " current hits " << _cacheHits);
+        _cache.clear();
+        _hashToWireId.clear();
+        _cacheSize = 0;
+        _cacheHits = 0;
+        _cacheTests = 0;
+        _nextId = 1;
+    }
+
+    // Keep these ids small and wrap them.
+    TileWireId createNewWireId()
+    {
+        TileWireId id = ++_nextId;
+        // FIXME: if we wrap - we should flush the clients too really ...
+        if (id < 1)
+            clearCache(true);
+        return id;
+    }
+
+public:
+    // Performed only after a complete combinetiles
+    void balanceCache()
+    {
+        // A normalish PNG image size for text in a writer document is
+        // around 4k for a content tile, and sub 1k for a background one.
+        if (_cacheSize > CacheSizeHardLimit)
+        {
+            size_t avgHits = 0;
+            for (auto it = _cache.begin(); it != _cache.end(); ++it)
+                avgHits += it->second.getHitCount();
+
+            LOG_DBG("PNG cache has " << _cache.size() << " items, total size " <<
+                    _cacheSize << ", current hits " << avgHits << ", total hit rate " <<
+                    (_cacheHits * 100. / _cacheTests) << "% at balance start.");
+            avgHits /= _cache.size();
+
+            for (auto it = _cache.begin(); it != _cache.end();)
+            {
+                if ((_cacheSize > CacheSizeSoftLimit && it->second.getHitCount() == 0) ||
+                    (_cacheSize > CacheSizeHardLimit && it->second.getHitCount() > 0 && it->second.getHitCount() <= avgHits))
+                {
+                    // Shrink cache when we exceed the size to maximize
+                    // the chance of hitting these entries in the future.
+                    _cacheSize -= it->second.getData()->size();
+                    it = _cache.erase(it);
+                }
+                else
+                {
+                    if (it->second.getHitCount() > 0)
+                        it->second.decrementHitCount();
+                    ++it;
+                }
+            }
+
+            LOG_DBG("PNG cache has " << _cache.size() << " items with total size of " <<
+                    _cacheSize << " bytes after balance.");
+        }
+
+        if (_hashToWireId.size() > CacheWidHardLimit)
+        {
+            LOG_DBG("Clear half of wid cache of size " << _hashToWireId.size());
+            TileWireId max = _nextId - CacheWidHardLimit/2;
+            for (auto it = _hashToWireId.begin(); it != _hashToWireId.end();)
+            {
+                if (it->second < max)
+                    it = _hashToWireId.erase(it);
+                else
+                    ++it;
+            }
+            LOG_DBG("Wid cache is now size " << _hashToWireId.size());
+        }
+    }
+
+    /// Lookup an entry in the cache and store the data in output.
+    /// Returns true on success, otherwise false.
+    bool copyFromCache(const TileBinaryHash hash, std::vector<char>& output, size_t &imgSize)
+    {
+        if (hash)
+        {
+            ++_cacheTests;
+            auto it = _cache.find(hash);
+            if (it != _cache.end())
+            {
+                ++_cacheHits;
+                LOG_DBG("PNG cache with hash " << hash << " hit.");
+                output.insert(output.end(),
+                              it->second.getData()->begin(),
+                              it->second.getData()->end());
+                it->second.incrementHitCount();
+                imgSize = it->second.getData()->size();
+
+                return true;
+            }
+        }
+
+        LOG_DBG("PNG cache with hash " << hash << " missed.");
+        return false;
+    }
+
+    void addToCache(const CacheData &data, TileWireId wid, const TileBinaryHash hash)
+    {
+        CacheEntry newEntry(data, wid);
+
+        if (hash)
+        {
+            // Adding duplicates causes grim wid mixups
+            assert(hashToWireId(hash) == wid);
+            assert(_cache.find(hash) == _cache.end());
+
+            data->shrink_to_fit();
+            _cache.emplace(hash, newEntry);
+            _cacheSize += data->size();
+        }
+    }
+
+    PngCache()
+    {
+        clearCache();
+    }
+
+    TileWireId hashToWireId(TileBinaryHash hash)
+    {
+        TileWireId wid;
+        if (hash == 0)
+            return 0;
+        auto it = _hashToWireId.find(hash);
+        if (it != _hashToWireId.end())
+            wid = it->second;
+        else
+        {
+            wid = createNewWireId();
+            _hashToWireId.emplace(hash, wid);
+        }
+        return wid;
+    }
+};
+
+class ThreadPool {
+    std::mutex _mutex;
+    std::condition_variable _cond;
+    std::condition_variable _complete;
+    typedef std::function<void()> ThreadFn;
+    std::queue<ThreadFn> _work;
+    std::vector<std::thread> _threads;
+    size_t _working;
+    bool   _shutdown;
+public:
+    ThreadPool()
+        : _working(0),
+          _shutdown(false)
+    {
+        int maxConcurrency = 2;
+#if MOBILEAPP && !defined(GTKAPP)
+        maxConcurrency = std::max<int>(std::thread::hardware_concurrency(), 2);
+#else
+        const char *max = getenv("MAX_CONCURRENCY");
+        if (max)
+            maxConcurrency = atoi(max);
+#endif
+        LOG_TRC("PNG compression thread pool size " << maxConcurrency);
+        for (int i = 1; i < maxConcurrency; ++i)
+            _threads.push_back(std::thread(&ThreadPool::work, this));
+    }
+    ~ThreadPool()
+    {
+        {
+            std::unique_lock< std::mutex > lock(_mutex);
+            assert(_working == 0);
+            _shutdown = true;
+        }
+        _cond.notify_all();
+        for (auto &it : _threads)
+            it.join();
+    }
+
+    size_t count() const
+    {
+        return _work.size();
+    }
+
+    void pushWorkUnlocked(const ThreadFn &fn)
+    {
+        _work.push(fn);
+    }
+
+    void runOne(std::unique_lock< std::mutex >& lock)
+    {
+        assert(!_work.empty());
+
+        ThreadFn fn = _work.front();
+        _work.pop();
+        _working++;
+        lock.unlock();
+
+        fn();
+
+        lock.lock();
+        _working--;
+        if (_work.empty() && _working == 0)
+            _complete.notify_all();
+    }
+
+    void run()
+    {
+        std::unique_lock< std::mutex > lock(_mutex);
+        assert(_working == 0);
+
+        // Avoid notifying threads if we don't need to.
+        bool useThreads = _threads.size() > 1 && _work.size() > 1;
+        if (useThreads)
+            _cond.notify_all();
+
+        while(!_work.empty())
+            runOne(lock);
+
+        if (useThreads && (_working > 0 || !_work.empty()))
+            _complete.wait(lock, [this]() { return _working == 0 && _work.empty(); } );
+
+        assert(_working==0);
+        assert(_work.empty());
+    }
+
+    void work()
+    {
+        std::unique_lock< std::mutex > lock(_mutex);
+        while (!_shutdown)
+        {
+            _cond.wait(lock);
+            if (!_shutdown && !_work.empty())
+                runOne(lock);
+        }
+    }
+};
+
+namespace RenderTiles
+{
+    struct Buffer {
+        unsigned char *_data;
+        Buffer()
+        {
+            _data = nullptr;
+        }
+        Buffer(size_t x, size_t y) :
+            Buffer()
+        {
+            allocate(x, y);
+        }
+        void allocate(size_t x, size_t y)
+        {
+            assert(!_data);
+            _data = static_cast<unsigned char *>(calloc(x * y, 4));
+        }
+        ~Buffer()
+        {
+            if (_data)
+                free (_data);
+        }
+        unsigned char *data() { return _data; }
+    };
+
+    class WatermarkBlender
+    {
+    public:
+        virtual void blendWatermark(TileCombined &tileCombined,
+                                    unsigned char *data,
+                                    int offsetX, int offsetY,
+                                    size_t pixmapWidth, size_t pixmapHeight,
+                                    int pixelWidth, int pixelHeight,
+                                    LibreOfficeKitTileMode mode) = 0;
+    };
+
+    static void pushRendered(std::vector<TileDesc> &renderedTiles,
+                             const TileDesc &desc, TileWireId wireId, size_t imgSize)
+    {
+        renderedTiles.push_back(desc);
+        renderedTiles.back().setWireId(wireId);
+        renderedTiles.back().setImgSize(imgSize);
+    }
+
+    bool doRender(std::shared_ptr<lok::Document> document,
+                  TileCombined &tileCombined,
+                  WatermarkBlender &watermarkBlender,
+                  std::unique_ptr<char[]> &response,
+                  size_t &responseSize,
+                  PngCache &pngCache,
+                  ThreadPool &pngPool,
+                  bool combined)
+    {
+        auto& tiles = tileCombined.getTiles();
+
+        // Calculate the area we cover
+        Util::Rectangle renderArea;
+        std::vector<Util::Rectangle> tileRecs;
+        tileRecs.reserve(tiles.size());
+
+        for (auto& tile : tiles)
+        {
+            Util::Rectangle rectangle(tile.getTilePosX(), tile.getTilePosY(),
+                                      tileCombined.getTileWidth(), tileCombined.getTileHeight());
+
+            if (tileRecs.empty())
+            {
+                renderArea = rectangle;
+            }
+            else
+            {
+                renderArea.extend(rectangle);
+            }
+
+            tileRecs.push_back(rectangle);
+        }
+
+        const size_t tilesByX = renderArea.getWidth() / tileCombined.getTileWidth();
+        const size_t tilesByY = renderArea.getHeight() / tileCombined.getTileHeight();
+        const size_t pixmapWidth = tilesByX * tileCombined.getWidth();
+        const size_t pixmapHeight = tilesByY * tileCombined.getHeight();
+
+        if (pixmapWidth > 4096 || pixmapHeight > 4096)
+            LOG_WRN("Unusual extremely large tile combine of size " << pixmapWidth << 'x' << pixmapHeight);
+
+        RenderTiles::Buffer pixmap(pixmapWidth, pixmapHeight);
+
+        const size_t pixmapSize = 4 * pixmapWidth * pixmapHeight;
+
+        // Render the whole area
+        const double area = pixmapWidth * pixmapHeight;
+        auto start = std::chrono::system_clock::now();
+        LOG_TRC("Calling paintPartTile(" << (void*)pixmap.data() << ')');
+        document->paintPartTile(pixmap.data(),
+                                tileCombined.getPart(),
+                                pixmapWidth, pixmapHeight,
+                                renderArea.getLeft(), renderArea.getTop(),
+                                renderArea.getWidth(), renderArea.getHeight());
+        auto duration = std::chrono::system_clock::now() - start;
+        auto elapsed = std::chrono::duration_cast<std::chrono::microseconds>(duration).count();
+        double totalTime = elapsed/1000.;
+        LOG_DBG("paintPartTile at (" << renderArea.getLeft() << ", " << renderArea.getTop() << "), (" <<
+                renderArea.getWidth() << ", " << renderArea.getHeight() << ") " <<
+                " rendered in " << totalTime << " ms (" << area / elapsed << " MP/s).");
+
+        const auto mode = static_cast<LibreOfficeKitTileMode>(document->getTileMode());
+
+        std::vector<char> output;
+        output.reserve(pixmapSize);
+
+        // Compress the area as tiles
+        const int pixelWidth = tileCombined.getWidth();
+        const int pixelHeight = tileCombined.getHeight();
+
+        std::vector<TileDesc> renderedTiles;
+        std::vector<TileDesc> duplicateTiles;
+        std::vector<TileBinaryHash> duplicateHashes;
+        std::vector<TileWireId> renderingIds;
+
+        size_t tileIndex = 0;
+
+        std::mutex pngMutex;
+
+        for (Util::Rectangle& tileRect : tileRecs)
+        {
+            const size_t positionX = (tileRect.getLeft() - renderArea.getLeft()) / tileCombined.getTileWidth();
+            const size_t positionY = (tileRect.getTop() - renderArea.getTop()) / tileCombined.getTileHeight();
+
+            const int offsetX = positionX * pixelWidth;
+            const int offsetY = positionY * pixelHeight;
+            watermarkBlender.blendWatermark(tileCombined,
+                                            pixmap.data(), offsetX, offsetY,
+                                            pixmapWidth, pixmapHeight,
+                                            pixelWidth, pixelHeight,
+                                            mode);
+
+            const uint64_t hash = Png::hashSubBuffer(pixmap.data(), offsetX, offsetY,
+                                                     pixelWidth, pixelHeight, pixmapWidth, pixmapHeight);
+
+            TileWireId wireId = pngCache.hashToWireId(hash);
+            TileWireId oldWireId = tiles[tileIndex].getOldWireId();
+            if (hash != 0 && oldWireId == wireId)
+            {
+                // The tile content is identical to what the client already has, so skip it
+                LOG_TRC("Match for tile #" << tileIndex << " at (" << positionX << ',' <<
+                        positionY << ") oldhash==hash (" << hash << "), wireId: " << wireId << " skipping");
+                tileIndex++;
+                continue;
+            }
+
+            bool skipCompress = false;
+            size_t imgSize = -1;
+            if (pngCache.copyFromCache(hash, output, imgSize))
+            {
+                pushRendered(renderedTiles, tiles[tileIndex], wireId, imgSize);
+                skipCompress = true;
+            }
+            else
+            {
+                LOG_DBG("PNG cache with hash " << hash << " missed.");
+
+                // Don't re-compress the same thing multiple times.
+                for (auto id : renderingIds)
+                {
+                    if (wireId == id)
+                    {
+                        pushRendered(duplicateTiles, tiles[tileIndex], wireId, 0);
+                        duplicateHashes.push_back(hash);
+                        skipCompress = true;
+                        LOG_TRC("Rendering duplicate tile #" << tileIndex << " at (" << positionX << ',' <<
+                                positionY << ") oldhash==hash (" << hash << "), wireId: " << wireId << " skipping");
+                        break;
+                    }
+                }
+            }
+
+            if (!skipCompress)
+            {
+                renderingIds.push_back(wireId);
+
+                // Queue to be executed later in parallel inside 'run'
+                pngPool.pushWorkUnlocked([=,&output,&pixmap,&tiles,&renderedTiles,&pngCache,&pngMutex](){
+
+                        PngCache::CacheData data(new std::vector< char >() );
+                        data->reserve(pixmapWidth * pixmapHeight * 1);
+
+                        /*
+                         * Disable for now - pushed in error.
+                         *
+                         if (_deltaGen.createDelta(pixmap, startX, startY, width, height,
+                                                   bufferWidth, bufferHeight,
+                                                   output, wid, oldWid))
+                         else ...
+                        */
+
+                        LOG_DBG("Encode a new png for tile #" << tileIndex);
+                        if (!Png::encodeSubBufferToPNG(pixmap.data(), offsetX, offsetY, pixelWidth, pixelHeight,
+                                                       pixmapWidth, pixmapHeight, *data, mode))
+                        {
+                            // FIXME: Return error.
+                            // sendTextFrameAndLogError("error: cmd=tile kind=failure");
+                            LOG_ERR("Failed to encode tile into PNG.");
+                            return;
+                        }
+
+                        LOG_DBG("Tile " << tileIndex << " is " << data->size() << " bytes.");
+                        std::unique_lock<std::mutex> pngLock(pngMutex);
+                        output.insert(output.end(), data->begin(), data->end());
+                        pngCache.addToCache(data, wireId, hash);
+                        pushRendered(renderedTiles, tiles[tileIndex], wireId, data->size());
+                    });
+            }
+
+            LOG_TRC("Encoded tile #" << tileIndex << " at (" << positionX << ',' << positionY << ") with oldWireId=" <<
+                    tiles[tileIndex].getOldWireId() << ", hash=" << hash << " wireId: " << wireId << " in " << imgSize << " bytes.");
+            tileIndex++;
+        }
+
+        pngPool.run();
+
+        for (auto &i : renderedTiles)
+        {
+            if (i.getImgSize() == 0)
+            {
+                LOG_ERR("Encoded 0-sized tile!");
+                assert(!"0-sized tile enocded!");
+            }
+        }
+
+        // FIXME: append duplicates - tragically for now as real duplicates
+        // we should append these as
+        {
+            size_t imgSize = -1;
+            assert(duplicateTiles.size() == duplicateHashes.size());
+            for (size_t i = 0; i < duplicateTiles.size(); ++i)
+            {
+                if (pngCache.copyFromCache(duplicateHashes[i], output, imgSize))
+                    pushRendered(renderedTiles, duplicateTiles[i],
+                                 duplicateTiles[i].getWireId(), imgSize);
+                else
+                    LOG_ERR("Horror - tile disappeared while rendering! " << duplicateHashes[i]);
+            }
+        }
+
+        pngCache.balanceCache();
+
+        duration = std::chrono::system_clock::now() - start;
+        elapsed = std::chrono::duration_cast<std::chrono::microseconds>(duration).count();
+        totalTime = elapsed/1000.;
+        LOG_DBG("rendering tiles at (" << renderArea.getLeft() << ", " << renderArea.getTop() << "), (" <<
+                renderArea.getWidth() << ", " << renderArea.getHeight() << ") " <<
+                " took " << totalTime << " ms (including the paintPartTile).");
+
+        if (tileIndex == 0)
+            return false;
+
+        std::string tileMsg;
+        if (combined)
+            tileMsg = tileCombined.serialize("tilecombine:", ADD_DEBUG_RENDERID, renderedTiles);
+        else
+            tileMsg = tiles[0].serialize("tile:", ADD_DEBUG_RENDERID);
+
+        LOG_TRC("Sending back painted tiles for " << tileMsg << " of size " << output.size() << " bytes) for: " << tileMsg);
+
+        responseSize = tileMsg.size() + output.size();
+        response.reset(new char[responseSize]);
+        std::copy(tileMsg.begin(), tileMsg.end(), response.get());
+        std::copy(output.begin(), output.end(), response.get() + tileMsg.size());
+
+        return true;
+    }
+}
+
+/* vim:set shiftwidth=4 softtabstop=4 expandtab: */
diff --git a/kit/Kit.cpp b/kit/Kit.cpp
index d14a28a810..0124946763 100644
--- a/kit/Kit.cpp
+++ b/kit/Kit.cpp
@@ -67,6 +67,7 @@
 #include <Util.hpp>
 #include "Delta.hpp"
 #include "Watermark.hpp"
+#include "RenderTiles.hpp"
 #include "SetupKitEnvironment.hpp"
 
 #if !MOBILEAPP
@@ -112,12 +113,6 @@ static bool AnonymizeUserData = false;
 static uint64_t AnonymizationSalt = 82589933;
 #endif
 
-#if ENABLE_DEBUG
-#  define ADD_DEBUG_RENDERID (" renderid=" + Util::UniqueId() + '\n')
-#else
-#  define ADD_DEBUG_RENDERID ("\n")
-#endif
-
 #if !MOBILEAPP
 
 static LokHookFunction2* initFunction = nullptr;
@@ -419,304 +414,6 @@ namespace
 
 #endif
 
-/// A quick & dirty cache of the last few PNGs
-/// and their hashes to avoid re-compression
-/// wherever possible.
-class PngCache
-{
-public:
-    typedef std::shared_ptr< std::vector< char > > CacheData;
-private:
-    struct CacheEntry {
-    private:
-        size_t    _hitCount;
-        TileWireId _wireId;
-        CacheData _data;
-    public:
-        CacheEntry(const CacheData &data, TileWireId id) :
-            _hitCount(1),   // Every entry is used at least once; prevent removal at birth.
-            _wireId(id),
-            _data(data)
-        {
-        }
-
-        size_t getHitCount() const
-        {
-            return _hitCount;
-        }
-
-        void incrementHitCount()
-        {
-            ++_hitCount;
-        }
-
-        void decrementHitCount()
-        {
-            --_hitCount;
-        }
-
-        const CacheData& getData() const
-        {
-            return _data;
-        }
-
-        TileWireId getWireId() const
-        {
-            return _wireId;
-        }
-    } ;
-    size_t _cacheSize;
-    static const size_t CacheSizeSoftLimit = (1024 * 4 * 32); // 128k of cache
-    static const size_t CacheSizeHardLimit = CacheSizeSoftLimit * 2;
-    static const size_t CacheWidHardLimit = 4096;
-    size_t _cacheHits;
-    size_t _cacheTests;
-    TileWireId _nextId;
-    DeltaGenerator _deltaGen;
-
-    std::unordered_map< TileBinaryHash, CacheEntry > _cache;
-    // This uses little storage so can be much larger
-    std::unordered_map< TileBinaryHash, TileWireId > _hashToWireId;
-
-    void clearCache(bool logStats = false)
-    {
-        if (logStats)
-            LOG_DBG("cache clear " << _cache.size() << " items total size " <<
-                    _cacheSize << " current hits " << _cacheHits);
-        _cache.clear();
-        _hashToWireId.clear();
-        _cacheSize = 0;
-        _cacheHits = 0;
-        _cacheTests = 0;
-        _nextId = 1;
-    }
-
-    // Keep these ids small and wrap them.
-    TileWireId createNewWireId()
-    {
-        TileWireId id = ++_nextId;
-        // FIXME: if we wrap - we should flush the clients too really ...
-        if (id < 1)
-            clearCache(true);
-        return id;
-    }
-
-public:
-    // Performed only after a complete combinetiles
-    void balanceCache()
-    {
-        // A normalish PNG image size for text in a writer document is
-        // around 4k for a content tile, and sub 1k for a background one.
-        if (_cacheSize > CacheSizeHardLimit)
-        {
-            size_t avgHits = 0;
-            for (auto it = _cache.begin(); it != _cache.end(); ++it)
-                avgHits += it->second.getHitCount();
-
-            LOG_DBG("PNG cache has " << _cache.size() << " items, total size " <<
-                    _cacheSize << ", current hits " << avgHits << ", total hit rate " <<
-                    (_cacheHits * 100. / _cacheTests) << "% at balance start.");
-            avgHits /= _cache.size();
-
-            for (auto it = _cache.begin(); it != _cache.end();)
-            {
-                if ((_cacheSize > CacheSizeSoftLimit && it->second.getHitCount() == 0) ||
-                    (_cacheSize > CacheSizeHardLimit && it->second.getHitCount() > 0 && it->second.getHitCount() <= avgHits))
-                {
-                    // Shrink cache when we exceed the size to maximize
-                    // the chance of hitting these entries in the future.
-                    _cacheSize -= it->second.getData()->size();
-                    it = _cache.erase(it);
-                }
-                else
-                {
-                    if (it->second.getHitCount() > 0)
-                        it->second.decrementHitCount();
-                    ++it;
-                }
-            }
-
-            LOG_DBG("PNG cache has " << _cache.size() << " items with total size of " <<
-                    _cacheSize << " bytes after balance.");
-        }
-
-        if (_hashToWireId.size() > CacheWidHardLimit)
-        {
-            LOG_DBG("Clear half of wid cache of size " << _hashToWireId.size());
-            TileWireId max = _nextId - CacheWidHardLimit/2;
-            for (auto it = _hashToWireId.begin(); it != _hashToWireId.end();)
-            {
-                if (it->second < max)
-                    it = _hashToWireId.erase(it);
-                else
-                    ++it;
-            }
-            LOG_DBG("Wid cache is now size " << _hashToWireId.size());
-        }
-    }
-
-    /// Lookup an entry in the cache and store the data in output.
-    /// Returns true on success, otherwise false.
-    bool copyFromCache(const TileBinaryHash hash, std::vector<char>& output, size_t &imgSize)
-    {
-        if (hash)
-        {
-            ++_cacheTests;
-            auto it = _cache.find(hash);
-            if (it != _cache.end())
-            {
-                ++_cacheHits;
-                LOG_DBG("PNG cache with hash " << hash << " hit.");
-                output.insert(output.end(),
-                              it->second.getData()->begin(),
-                              it->second.getData()->end());
-                it->second.incrementHitCount();
-                imgSize = it->second.getData()->size();
-
-                return true;
-            }
-        }
-
-        LOG_DBG("PNG cache with hash " << hash << " missed.");
-        return false;
-    }
-
-    void addToCache(const CacheData &data, TileWireId wid, const TileBinaryHash hash)
-    {
-        CacheEntry newEntry(data, wid);
-
-        if (hash)
-        {
-            // Adding duplicates causes grim wid mixups
-            assert(hashToWireId(hash) == wid);
-            assert(_cache.find(hash) == _cache.end());
-
-            data->shrink_to_fit();
-            _cache.emplace(hash, newEntry);
-            _cacheSize += data->size();
-        }
-    }
-
-    PngCache()
-    {
-        clearCache();
-    }
-
-    TileWireId hashToWireId(TileBinaryHash hash)
-    {
-        TileWireId wid;
-        if (hash == 0)
-            return 0;
-        auto it = _hashToWireId.find(hash);
-        if (it != _hashToWireId.end())
-            wid = it->second;
-        else
-        {
-            wid = createNewWireId();
-            _hashToWireId.emplace(hash, wid);
-        }
-        return wid;
-    }
-};
-
-
-class ThreadPool {
-    std::mutex _mutex;
-    std::condition_variable _cond;
-    std::condition_variable _complete;
-    typedef std::function<void()> ThreadFn;
-    std::queue<ThreadFn> _work;
-    std::vector<std::thread> _threads;
-    size_t _working;
-    bool   _shutdown;
-public:
-    ThreadPool()
-        : _working(0),
-          _shutdown(false)
-    {
-        int maxConcurrency = 2;
-#if MOBILEAPP && !defined(GTKAPP)
-        maxConcurrency = std::max<int>(std::thread::hardware_concurrency(), 2);
-#else
-        const char *max = getenv("MAX_CONCURRENCY");
-        if (max)
-            maxConcurrency = atoi(max);
-#endif
-        LOG_TRC("PNG compression thread pool size " << maxConcurrency);
-        for (int i = 1; i < maxConcurrency; ++i)
-            _threads.push_back(std::thread(&ThreadPool::work, this));
-    }
-    ~ThreadPool()
-    {
-        {
-            std::unique_lock< std::mutex > lock(_mutex);
-            assert(_working == 0);
-            _shutdown = true;
-        }
-        _cond.notify_all();
-        for (auto &it : _threads)
-            it.join();
-    }
-
-    size_t count() const
-    {
-        return _work.size();
-    }
-
-    void pushWorkUnlocked(const ThreadFn &fn)
-    {
-        _work.push(fn);
-    }
-
-    void runOne(std::unique_lock< std::mutex >& lock)
-    {
-        assert(!_work.empty());
-
-        ThreadFn fn = _work.front();
-        _work.pop();
-        _working++;
-        lock.unlock();
-
-        fn();
-
-        lock.lock();
-        _working--;
-        if (_work.empty() && _working == 0)
-            _complete.notify_all();
-    }
-
-    void run()
-    {
-        std::unique_lock< std::mutex > lock(_mutex);
-        assert(_working == 0);
-
-        // Avoid notifying threads if we don't need to.
-        bool useThreads = _threads.size() > 1 && _work.size() > 1;
-        if (useThreads)
-            _cond.notify_all();
-
-        while(!_work.empty())
-            runOne(lock);
-
-        if (useThreads && (_working > 0 || !_work.empty()))
-            _complete.wait(lock, [this]() { return _working == 0 && _work.empty(); } );
-
-        assert(_working==0);
-        assert(_work.empty());
-    }
-
-    void work()
-    {
-        std::unique_lock< std::mutex > lock(_mutex);
-        while (!_shutdown)
-        {
-            _cond.wait(lock);
-            if (!_shutdown && !_work.empty())
-                runOne(lock);
-        }
-    }
-};
-
 /// A document container.
 /// Owns LOKitDocument instance and connections.
 /// Manages the lifetime of a document.
@@ -724,7 +421,7 @@ public:
 /// per process. But for security reasons don't.
 /// However, we could have a loolkit instance
 /// per user or group of users (a trusted circle).
-class Document final : public DocumentManagerInterface
+class Document final : public DocumentManagerInterface, public RenderTiles::WatermarkBlender
 {
 public:
     /// We have two types of password protected documents
@@ -934,65 +631,30 @@ public:
         renderTiles(tileCombined, true);
     }
 
-    static void pushRendered(std::vector<TileDesc> &renderedTiles,
-                             const TileDesc &desc, TileWireId wireId, size_t imgSize)
+    void blendWatermark(TileCombined &tileCombined,
+                        unsigned char *data, int offsetX, int offsetY,
+                        size_t pixmapWidth, size_t pixmapHeight,
+                        int pixelWidth, int pixelHeight,
+                        LibreOfficeKitTileMode mode) override
     {
-        renderedTiles.push_back(desc);
-        renderedTiles.back().setWireId(wireId);
-        renderedTiles.back().setImgSize(imgSize);
+        const auto session = _sessions.findByCanonicalId(tileCombined.getNormalizedViewId());
+        if (session->hasWatermark())
+            session->_docWatermark->blending(data, offsetX, offsetY,
+                                             pixmapWidth, pixmapHeight,
+                                             pixelWidth, pixelHeight,
+                                             mode);
     }
 
-    struct RenderBuffer {
-        unsigned char *_data;
-        RenderBuffer(size_t x, size_t y)
-        {
-            _data = static_cast<unsigned char *>(calloc(x * y, 4));
-        }
-        ~RenderBuffer()
-        {
-            if (_data)
-                free (_data);
-        }
-        unsigned char *data() { return _data; }
-    };
-
     void renderTiles(TileCombined &tileCombined, bool combined)
     {
-        auto& tiles = tileCombined.getTiles();
-
-        // Calculate the area we cover
-        Util::Rectangle renderArea;
-        std::vector<Util::Rectangle> tileRecs;
-        tileRecs.reserve(tiles.size());
-
-        for (auto& tile : tiles)
+        // Find a session matching our view / render settings.
+        const auto session = _sessions.findByCanonicalId(tileCombined.getNormalizedViewId());
+        if (!session)
         {
-            Util::Rectangle rectangle(tile.getTilePosX(), tile.getTilePosY(),
-                                      tileCombined.getTileWidth(), tileCombined.getTileHeight());
-
-            if (tileRecs.empty())
-            {
-                renderArea = rectangle;
-            }
-            else
-            {
-                renderArea.extend(rectangle);
-            }
-
-            tileRecs.push_back(rectangle);
+            LOG_ERR("Session is not found. Maybe exited after rendering request.");
+            return;
         }
 
-        const size_t tilesByX = renderArea.getWidth() / tileCombined.getTileWidth();
-        const size_t tilesByY = renderArea.getHeight() / tileCombined.getTileHeight();
-        const size_t pixmapWidth = tilesByX * tileCombined.getWidth();
-        const size_t pixmapHeight = tilesByY * tileCombined.getHeight();
-
-        if (pixmapWidth > 4096 || pixmapHeight > 4096)
-            LOG_WRN("Unusual extremely large tile combine of size " << pixmapWidth << 'x' << pixmapHeight);
-
-        const size_t pixmapSize = 4 * pixmapWidth * pixmapHeight;
-        RenderBuffer pixmap(pixmapWidth, pixmapHeight);
-
         if (!_loKitDocument)
         {
             LOG_ERR("Tile rendering requested before loading document.");
@@ -1005,198 +667,20 @@ public:
             return;
         }
 
-        // Find a session matching our view / render settings.
-        const auto session = _sessions.findByCanonicalId(tileCombined.getNormalizedViewId());
-        if (!session)
-        {
-            LOG_ERR("Session is not found. Maybe exited after rendering request.");
-            return;
-        }
-
 #ifdef FIXME_RENDER_SETTINGS
         // if necessary select a suitable rendering view eg. with 'show non-printing chars'
         if (tileCombined.getNormalizedViewId())
             _loKitDocument->setView(session->getViewId());
 #endif
 
-        // Render the whole area
-        const double area = pixmapWidth * pixmapHeight;
-        auto start = std::chrono::system_clock::now();
-        LOG_TRC("Calling paintPartTile(" << (void*)pixmap.data() << ')');
-        _loKitDocument->paintPartTile(pixmap.data(),
-                                      tileCombined.getPart(),
-                                      pixmapWidth, pixmapHeight,
-                                      renderArea.getLeft(), renderArea.getTop(),
-                                      renderArea.getWidth(), renderArea.getHeight());
-        auto duration = std::chrono::system_clock::now() - start;
-        auto elapsed = std::chrono::duration_cast<std::chrono::microseconds>(duration).count();
-        double totalTime = elapsed/1000.;
-        LOG_DBG("paintPartTile at (" << renderArea.getLeft() << ", " << renderArea.getTop() << "), (" <<
-                renderArea.getWidth() << ", " << renderArea.getHeight() << ") " <<
-                " rendered in " << totalTime << " ms (" << area / elapsed << " MP/s).");
-        const auto mode = static_cast<LibreOfficeKitTileMode>(_loKitDocument->getTileMode());
-
-        std::vector<char> output;
-        output.reserve(pixmapSize);
-
-        // Compress the area as tiles
-        const int pixelWidth = tileCombined.getWidth();
-        const int pixelHeight = tileCombined.getHeight();
-
-        std::vector<TileDesc> renderedTiles;
-        std::vector<TileDesc> duplicateTiles;
-        std::vector<TileBinaryHash> duplicateHashes;
-        std::vector<TileWireId> renderingIds;
-
-        size_t tileIndex = 0;
-        for (Util::Rectangle& tileRect : tileRecs)
-        {
-            const size_t positionX = (tileRect.getLeft() - renderArea.getLeft()) / tileCombined.getTileWidth();
-            const size_t positionY = (tileRect.getTop() - renderArea.getTop()) / tileCombined.getTileHeight();
-
-            const int offsetX = positionX * pixelWidth;
-            const int offsetY = positionY * pixelHeight;
-            if (session->hasWatermark())
-                    session->_docWatermark->blending(pixmap.data(), offsetX, offsetY,
-                                            pixmapWidth, pixmapHeight,
-                                            pixelWidth, pixelHeight,
-                                            mode);
-
-            const uint64_t hash = Png::hashSubBuffer(pixmap.data(), offsetX, offsetY,
-                                                     pixelWidth, pixelHeight, pixmapWidth, pixmapHeight);
-
-            TileWireId wireId = _pngCache.hashToWireId(hash);
-            TileWireId oldWireId = tiles[tileIndex].getOldWireId();
-            if (hash != 0 && oldWireId == wireId)
-            {
-                // The tile content is identical to what the client already has, so skip it
-                LOG_TRC("Match for tile #" << tileIndex << " at (" << positionX << ',' <<
-                        positionY << ") oldhash==hash (" << hash << "), wireId: " << wireId << " skipping");
-                tileIndex++;
-                continue;
-            }
-
-            bool skipCompress = false;
-            size_t imgSize = -1;
-            if (_pngCache.copyFromCache(hash, output, imgSize))
-            {
-                pushRendered(renderedTiles, tiles[tileIndex], wireId, imgSize);
-                skipCompress = true;
-            }
-            else
-            {
-                LOG_DBG("PNG cache with hash " << hash << " missed.");
-
-                // Don't re-compress the same thing multiple times.
-                for (auto id : renderingIds)
-                {
-                    if (wireId == id)
-                    {
-                        pushRendered(duplicateTiles, tiles[tileIndex], wireId, 0);
-                        duplicateHashes.push_back(hash);
-                        skipCompress = true;
-                        LOG_TRC("Rendering duplicate tile #" << tileIndex << " at (" << positionX << ',' <<
-                                positionY << ") oldhash==hash (" << hash << "), wireId: " << wireId << " skipping");
-                        break;
-                    }
-                }
-            }
-
-            if (!skipCompress)
-            {
-                renderingIds.push_back(wireId);
-
-                // Queue to be executed later in parallel inside 'run'
-                _pngPool.pushWorkUnlocked([=,&output,&pixmap,&tiles,&renderedTiles](){
-                        PngCache::CacheData data(new std::vector< char >() );
-                        data->reserve(pixmapWidth * pixmapHeight * 1);
-
-                        /*
-                         * Disable for now - pushed in error.
-                         *
-                         if (_deltaGen.createDelta(pixmap, startX, startY, width, height,
-                                                   bufferWidth, bufferHeight,
-                                                   output, wid, oldWid))
-                         else ...
-                        */
-
-                        LOG_DBG("Encode a new png for tile #" << tileIndex);
-                        if (!Png::encodeSubBufferToPNG(pixmap.data(), offsetX, offsetY, pixelWidth, pixelHeight,
-                                                       pixmapWidth, pixmapHeight, *data, mode))
-                        {
-                            // FIXME: Return error.
-                            // sendTextFrameAndLogError("error: cmd=tile kind=failure");
-                            LOG_ERR("Failed to encode tile into PNG.");
-                            return;
-                        }
-
-                        LOG_DBG("Tile " << tileIndex << " is " << data->size() << " bytes.");
-                        std::unique_lock<std::mutex> pngLock(_pngMutex);
-                        output.insert(output.end(), data->begin(), data->end());
-                        _pngCache.addToCache(data, wireId, hash);
-                        pushRendered(renderedTiles, tiles[tileIndex], wireId, data->size());
-                    });
-            }
-
-            LOG_TRC("Encoded tile #" << tileIndex << " at (" << positionX << ',' << positionY << ") with oldWireId=" <<
-                    tiles[tileIndex].getOldWireId() << ", hash=" << hash << " wireId: " << wireId << " in " << imgSize << " bytes.");
-            tileIndex++;
-        }
-
-        _pngPool.run();
-
-        for (auto &i : renderedTiles)
-        {
-            if (i.getImgSize() == 0)
-            {
-                LOG_ERR("Encoded 0-sized tile!");
-                assert(!"0-sized tile enocded!");
-            }
-        }
-
-        // FIXME: append duplicates - tragically for now as real duplicates
-        // we should append these as
-        {
-            size_t imgSize = -1;
-            assert(duplicateTiles.size() == duplicateHashes.size());
-            for (size_t i = 0; i < duplicateTiles.size(); ++i)
-            {
-                if (_pngCache.copyFromCache(duplicateHashes[i], output, imgSize))
-                    pushRendered(renderedTiles, duplicateTiles[i],
-                                 duplicateTiles[i].getWireId(), imgSize);
-                else
-                    LOG_ERR("Horror - tile disappeared while rendering! " << duplicateHashes[i]);
-            }
-        }
-
-        _pngCache.balanceCache();
-
-        duration = std::chrono::system_clock::now() - start;
-        elapsed = std::chrono::duration_cast<std::chrono::microseconds>(duration).count();
-        totalTime = elapsed/1000.;
-        LOG_DBG("renderTiles at (" << renderArea.getLeft() << ", " << renderArea.getTop() << "), (" <<
-                renderArea.getWidth() << ", " << renderArea.getHeight() << ") " <<
-                " took " << totalTime << " ms (including the paintPartTile).");
-
-        if (tileIndex == 0)
+        std::unique_ptr<char[]> response;
+        size_t responseSize;
+        if (!RenderTiles::doRender(_loKitDocument, tileCombined, *this, response, responseSize, _pngCache, _pngPool, combined))
         {
             LOG_DBG("All tiles skipped, not producing empty tilecombine: message");
             return;
         }
 
-        std::string tileMsg;
-        if (combined)
-            tileMsg = tileCombined.serialize("tilecombine:", ADD_DEBUG_RENDERID, renderedTiles);
-        else
-            tileMsg = tiles[0].serialize("tile:", ADD_DEBUG_RENDERID);
-
-        LOG_TRC("Sending back painted tiles for " << tileMsg << " of size " << output.size() << " bytes) for: " << tileMsg);
-
-        size_t responseSize = tileMsg.size() + output.size();
-        std::unique_ptr<char[]> response(new char[responseSize]);
-        std::copy(tileMsg.begin(), tileMsg.end(), response.get());
-        std::copy(output.begin(), output.end(), response.get() + tileMsg.size());
-
         postMessage(response.get(), responseSize, WSOpCode::Binary);
     }
 
@@ -2081,7 +1565,6 @@ private:
     std::shared_ptr<TileQueue> _tileQueue;
     std::shared_ptr<WebSocketHandler> _websocketHandler;
 
-    std::mutex _pngMutex;
     PngCache _pngCache;
 
     // Document password provided