New POST service to render a search result + unit and integ. tests

This adds a new service render-search-result, which renders an
image of the position where, the search fund a hit. The search
result contains the information in which object or paragraph,
which is just the node id and node type from the document model.

The service takes the document and the search result (xml file)
as the input and returns a PNG image as the output.

Signed-off-by: Tomaž Vajngerl <tomaz.vajngerl@collabora.co.uk>

Change-Id: Iffc158c5b0a3c9a63f8d05830314c9bc1616b3b1
Signed-off-by: Tomaž Vajngerl <tomaz.vajngerl@collabora.co.uk>

4 files changed, 315 insertions, 36 deletions

diff --git a/wsd/ClientSession.cpp b/wsd/ClientSession.cpp
index f12f5fac09..b9af19a171 100644
--- a/wsd/ClientSession.cpp
+++ b/wsd/ClientSession.cpp
@@ -986,7 +986,8 @@ bool ClientSession::_handleInput(const char *buffer, int length)
              tokens[0] == "exportsignanduploaddocument" ||
              tokens[0] == "rendershapeselection" ||
              tokens[0] == "resizewindow" ||
-             tokens[0] == "removetextcontext")
+             tokens[0] == "removetextcontext" ||
+             tokens[0] == "rendersearchresult")
     {
         if (tokens.equals(0, "key"))
             _keyEvents++;
@@ -1260,7 +1261,7 @@ bool ClientSession::filterMessage(const std::string& message) const
     {
         // By default, don't allow anything
         allowed = false;
-        if (tokens.equals(0, "userinactive") || tokens.equals(0, "useractive") || tokens.equals(0, "saveas"))
+        if (tokens.equals(0, "userinactive") || tokens.equals(0, "useractive") || tokens.equals(0, "saveas") || tokens.equals(0, "rendersearchresult"))
         {
             allowed = true;
         }
diff --git a/wsd/DocumentBroker.cpp b/wsd/DocumentBroker.cpp
index ab2d93cf10..c8f1bd2e0e 100644
--- a/wsd/DocumentBroker.cpp
+++ b/wsd/DocumentBroker.cpp
@@ -2927,21 +2927,31 @@ void DocumentBroker::getIOStats(uint64_t &sent, uint64_t &recv)
 }
 
 #if !MOBILEAPP
-static std::atomic<std::size_t> NumConverters;
+
+void StatelessBatchBroker::removeFile(const std::string &uriOrig)
+{
+    // Remove and report errors on failure.
+    FileUtil::removeFile(uriOrig);
+    const std::string dir = Poco::Path(uriOrig).parent().toString();
+    if (FileUtil::isEmptyDirectory(dir))
+        FileUtil::removeFile(dir);
+}
+
+static std::atomic<std::size_t> gConvertToBrokerInstanceCouter;
 
 std::size_t ConvertToBroker::getInstanceCount()
 {
-    return NumConverters;
+    return gConvertToBrokerInstanceCouter;
 }
 
 ConvertToBroker::ConvertToBroker(const std::string& uri,
                                  const Poco::URI& uriPublic,
                                  const std::string& docKey,
                                  const std::string& format,
-                                 const std::string& sOptions) :
-    DocumentBroker(ChildType::Batch, uri, uriPublic, docKey),
-    _format(format),
-    _sOptions(sOptions)
+                                 const std::string& sOptions)
+    : StatelessBatchBroker(uri, uriPublic, docKey)
+    , _format(format)
+    , _sOptions(sOptions)
 {
     LOG_TRC("Created ConvertToBroker: uri: [" << uri << "], uriPublic: [" << uriPublic.toString()
                                               << "], docKey: [" << docKey << "], format: ["
@@ -2950,9 +2960,12 @@ ConvertToBroker::ConvertToBroker(const std::string& uri,
     static const std::chrono::seconds limit_convert_secs(
         LOOLWSD::getConfigValue<int>("per_document.limit_convert_secs", 100));
     _limitLifeSeconds = limit_convert_secs;
-    ++NumConverters;
+    ++gConvertToBrokerInstanceCouter;
 }
 
+ConvertToBroker::~ConvertToBroker()
+{}
+
 bool ConvertToBroker::startConversion(SocketDisposition &disposition, const std::string &id)
 {
     std::shared_ptr<ConvertToBroker> docBroker = std::static_pointer_cast<ConvertToBroker>(shared_from_this());
@@ -2993,28 +3006,12 @@ void ConvertToBroker::dispose()
 {
     if (!_uriOrig.empty())
     {
-        NumConverters--;
+        gConvertToBrokerInstanceCouter--;
         removeFile(_uriOrig);
         _uriOrig.clear();
     }
 }
 
-ConvertToBroker::~ConvertToBroker()
-{
-    // Calling a virtual function from a dtor
-    // is only valid if there are no inheritors.
-    dispose();
-}
-
-void ConvertToBroker::removeFile(const std::string &uriOrig)
-{
-    // Remove and report errors on failure.
-    FileUtil::removeFile(uriOrig);
-    const std::string dir = Poco::Path(uriOrig).parent().toString();
-    if (FileUtil::isEmptyDirectory(dir))
-        FileUtil::removeFile(dir);
-}
-
 void ConvertToBroker::setLoaded()
 {
     DocumentBroker::setLoaded();
@@ -3039,6 +3036,122 @@ void ConvertToBroker::setLoaded()
 
     _clientSession->handleMessage(saveasRequest);
 }
+
+
+static std::atomic<std::size_t> gRenderSearchResultBrokerInstanceCouter;
+
+std::size_t RenderSearchResultBroker::getInstanceCount()
+{
+    return gRenderSearchResultBrokerInstanceCouter;
+}
+
+RenderSearchResultBroker::RenderSearchResultBroker(
+                            std::string const& uri,
+                            Poco::URI const& uriPublic,
+                            std::string const& docKey,
+                            std::shared_ptr<std::vector<char>> const& pSearchResultContent)
+    : StatelessBatchBroker(uri, uriPublic, docKey)
+    , _pSearchResultContent(pSearchResultContent)
+{
+    LOG_TRC("Created RenderSearchResultBroker: uri: [" << uri << "], uriPublic: [" << uriPublic.toString()
+                                              << "], docKey: [" << docKey << "].");
+    gConvertToBrokerInstanceCouter++;
+}
+
+RenderSearchResultBroker::~RenderSearchResultBroker()
+{}
+
+bool RenderSearchResultBroker::executeCommand(SocketDisposition& disposition, std::string const& id)
+{
+    std::shared_ptr<RenderSearchResultBroker> docBroker = std::static_pointer_cast<RenderSearchResultBroker>(shared_from_this());
+
+    const bool isReadOnly = true;
+
+    std::shared_ptr<ProtocolHandlerInterface> emptyProtocolHandler;
+    RequestDetails requestDetails("render-search-result");
+    _clientSession = std::make_shared<ClientSession>(emptyProtocolHandler, id, docBroker, getPublicUri(), isReadOnly, requestDetails);
+    _clientSession->construct();
+
+    if (!_clientSession)
+        return false;
+
+    docBroker->setupTransfer(disposition, [docBroker] (std::shared_ptr<Socket>const & moveSocket)
+    {
+        docBroker->setResponseSocket(std::static_pointer_cast<StreamSocket>(moveSocket));
+
+        // First add and load the session.
+        docBroker->addSession(docBroker->_clientSession);
+
+        // Load the document manually.
+        std::string encodedFrom;
+        Poco::URI::encode(docBroker->getPublicUri().getPath(), "", encodedFrom);
+        // add batch mode, no interactive dialogs
+        const std::string _load = "load url=" + encodedFrom + " batch=true";
+        std::vector<char> loadRequest(_load.begin(), _load.end());
+        docBroker->_clientSession->handleMessage(loadRequest);
+    });
+
+    return true;
+}
+
+void RenderSearchResultBroker::setLoaded()
+{
+    DocumentBroker::setLoaded();
+
+    // Send the rendersearchresult request ...
+    const std::string renderSearchResultCmd = "rendersearchresult ";
+    std::vector<char> renderSearchResultRequest(renderSearchResultCmd.begin(), renderSearchResultCmd.end());
+    renderSearchResultRequest.resize(renderSearchResultCmd.size() + _pSearchResultContent->size());
+    std::copy(_pSearchResultContent->begin(), _pSearchResultContent->end(), renderSearchResultRequest.begin() + renderSearchResultCmd.size());
+    _clientSession->handleMessage(renderSearchResultRequest);
+}
+
+void RenderSearchResultBroker::dispose()
+{
+    if (!_uriOrig.empty())
+    {
+        gRenderSearchResultBrokerInstanceCouter--;
+        removeFile(_uriOrig);
+        _uriOrig.clear();
+    }
+}
+
+bool RenderSearchResultBroker::handleInput(const std::vector<char>& payload)
+{
+    bool bResult = DocumentBroker::handleInput(payload);
+
+    if (bResult)
+    {
+        auto message = std::make_shared<Message>(payload.data(), payload.size(), Message::Dir::Out);
+        auto const& messageData = message->data();
+
+        static std::string commandString = "rendersearchresult:";
+        static std::vector<char> commandStringVector(commandString.begin(), commandString.end());
+
+        if (messageData.size() >= commandStringVector.size())
+        {
+           bool bEquals = std::equal(commandStringVector.begin(), commandStringVector.end(),
+                                      messageData.begin());
+            if (bEquals)
+            {
+                _aResposeData.resize(messageData.size() - commandStringVector.size());
+                std::copy(messageData.begin() + commandStringVector.size(), messageData.end(), _aResposeData.begin());
+
+                std::string aDataString(_aResposeData.data(), _aResposeData.size());
+                // really not ideal that the response works only with std::string
+                http::Response httpResponse(http::StatusLine(200));
+                httpResponse.setBody(aDataString, "image/png");
+                httpResponse.set("Connection", "close");
+                _socket->sendAndShutdown(httpResponse);
+
+                removeSession(_clientSession->getId());
+                stop("Finished RenderSearchResult handler.");
+            }
+        }
+    }
+    return bResult;
+}
+
 #endif
 
 std::vector<std::shared_ptr<ClientSession>> DocumentBroker::getSessionsTestOnlyUnsafe()
diff --git a/wsd/DocumentBroker.hpp b/wsd/DocumentBroker.hpp
index 686a4e9690..04ab8b5fb2 100644
--- a/wsd/DocumentBroker.hpp
+++ b/wsd/DocumentBroker.hpp
@@ -403,7 +403,7 @@ public:
 
     bool isMarkedToDestroy() const { return _docState.isMarkedToDestroy() || _stop; }
 
-    bool handleInput(const std::vector<char>& payload);
+    virtual bool handleInput(const std::vector<char>& payload);
 
     /// Forward a message from client session to its respective child session.
     bool forwardToChild(const std::string& viewId, const std::string& message);
@@ -1126,11 +1126,29 @@ private:
 };
 
 #if !MOBILEAPP
-class ConvertToBroker final : public DocumentBroker
+class StatelessBatchBroker : public DocumentBroker
+{
+protected:
+    std::shared_ptr<ClientSession> _clientSession;
+
+public:
+    StatelessBatchBroker(const std::string& uri,
+                   const Poco::URI& uriPublic,
+                   const std::string& docKey)
+        : DocumentBroker(ChildType::Batch, uri, uriPublic, docKey)
+    {}
+
+    virtual ~StatelessBatchBroker()
+    {}
+
+    /// Cleanup path and its parent
+    static void removeFile(const std::string &uri);
+};
+
+class ConvertToBroker final : public StatelessBatchBroker
 {
     const std::string _format;
     const std::string _sOptions;
-    std::shared_ptr<ClientSession> _clientSession;
 
 public:
     /// Construct DocumentBroker with URI and docKey
@@ -1144,21 +1162,54 @@ public:
     /// Move socket to this broker for response & do conversion
     bool startConversion(SocketDisposition &disposition, const std::string &id);
 
-    /// Called when removed from the DocBrokers list
-    void dispose() override;
-
     /// When the load completes - lets start saving
     void setLoaded() override;
 
+    /// Called when removed from the DocBrokers list
+    void dispose() override;
+
     /// How many live conversions are running.
     static std::size_t getInstanceCount();
 
-    /// Cleanup path and its parent
-    static void removeFile(const std::string &uri);
-
 private:
     bool isConvertTo() const override { return true; }
 };
+
+class RenderSearchResultBroker final : public StatelessBatchBroker
+{
+    std::shared_ptr<std::vector<char>> _pSearchResultContent;
+    std::vector<char> _aResposeData;
+    std::shared_ptr<StreamSocket> _socket;
+
+public:
+    RenderSearchResultBroker(std::string const& uri,
+                             Poco::URI const& uriPublic,
+                             std::string const& docKey,
+                             std::shared_ptr<std::vector<char>> const& pSearchResultContent);
+
+    virtual ~RenderSearchResultBroker();
+
+    void setResponseSocket(std::shared_ptr<StreamSocket> const & socket)
+    {
+        _socket = socket;
+    }
+
+    /// Execute command(s) and move the socket to this broker
+    bool executeCommand(SocketDisposition& disposition, std::string const& id);
+
+    /// Override method to start executing when the document is loaded
+    void setLoaded() override;
+
+    /// Called when removed from the DocBrokers list
+    void dispose() override;
+
+    /// Override to filter out the data that is returned by a command
+    bool handleInput(const std::vector<char>& payload) override;
+
+    /// How many instances are running.
+    static std::size_t getInstanceCount();
+};
+
 #endif
 
 /* vim:set shiftwidth=4 softtabstop=4 expandtab: */
diff --git a/wsd/LOOLWSD.cpp b/wsd/LOOLWSD.cpp
index 710a547276..27b09895e0 100644
--- a/wsd/LOOLWSD.cpp
+++ b/wsd/LOOLWSD.cpp
@@ -636,7 +636,7 @@ public:
         if (!_filename.empty())
         {
             LOG_TRC("Remove un-handled temporary file '" << _filename << '\'');
-            ConvertToBroker::removeFile(_filename);
+            StatelessBatchBroker::removeFile(_filename);
         }
     }
 
@@ -679,6 +679,84 @@ public:
     }
 };
 
+class RenderSearchResultPartHandler : public PartHandler
+{
+private:
+    std::string _filename;
+    std::shared_ptr<std::vector<char>> _pSearchResultContent;
+
+public:
+    std::string getFilename() const { return _filename; }
+
+    /// Afterwards someone else is responsible for cleaning that up.
+    void takeFile() { _filename.clear(); }
+
+    const std::shared_ptr<std::vector<char>>& getSearchResultContent() const
+    {
+        return _pSearchResultContent;
+    }
+
+    RenderSearchResultPartHandler() = default;
+
+    virtual ~RenderSearchResultPartHandler()
+    {
+        if (!_filename.empty())
+        {
+            LOG_TRC("Remove un-handled temporary file '" << _filename << '\'');
+            StatelessBatchBroker::removeFile(_filename);
+        }
+    }
+
+    virtual void handlePart(const MessageHeader& header, std::istream& stream) override
+    {
+        // Extract filename and put it to a temporary directory.
+        std::string label;
+        NameValueCollection content;
+        if (header.has("Content-Disposition"))
+        {
+            MessageHeader::splitParameters(header.get("Content-Disposition"), label, content);
+        }
+
+        std::string name = content.get("name", "");
+        if (name == "document")
+        {
+            std::string filename = content.get("filename", "");
+
+            const Path filenameParam(filename);
+
+            // The temporary directory is child-root/<JAIL_TMP_INCOMING_PATH>.
+            // Always create a random sub-directory to avoid file-name collision.
+            Path tempPath = Path::forDirectory(
+                FileUtil::createRandomTmpDir(LOOLWSD::ChildRoot + JailUtil::JAIL_TMP_INCOMING_PATH)
+                + '/');
+
+            LOG_TRC("Created temporary render-search-result file path: " << tempPath.toString());
+
+            // Prevent user inputting anything funny here.
+            // A "filename" should always be a filename, not a path
+
+            if (filenameParam.getFileName() == "callback:")
+                tempPath.setFileName("incoming_file"); // A sensible name.
+            else
+                tempPath.setFileName(filenameParam.getFileName()); //TODO: Sanitize.
+            _filename = tempPath.toString();
+
+            // Copy the stream to _filename.
+            std::ofstream fileStream;
+            fileStream.open(_filename);
+            StreamCopier::copyStream(stream, fileStream);
+            fileStream.close();
+        }
+        else if (name == "result")
+        {
+            // Copy content from the stream into a std::vector<char>
+            _pSearchResultContent = std::make_shared<std::vector<char>>(
+                                       std::istreambuf_iterator<char>(stream),
+                                       std::istreambuf_iterator<char>());
+        }
+    }
+};
+
 namespace
 {
 
@@ -3335,6 +3413,42 @@ private:
             }
             return;
         }
+        else if (requestDetails.equals(1, "render-search-result"))
+        {
+            RenderSearchResultPartHandler handler;
+            HTMLForm form(request, message, handler);
+
+            const std::string fromPath = handler.getFilename();
+
+            LOG_INF("Create render-search-result POST command handler");
+
+            if (fromPath.empty())
+                return;
+
+            Poco::URI uriPublic = DocumentBroker::sanitizeURI(fromPath);
+            const std::string docKey = DocumentBroker::getDocKey(uriPublic);
+
+            // This lock could become a bottleneck.
+            // In that case, we can use a pool and index by publicPath.
+            std::unique_lock<std::mutex> docBrokersLock(DocBrokersMutex);
+
+            LOG_DBG("New DocumentBroker for docKey [" << docKey << "].");
+            auto docBroker = std::make_shared<RenderSearchResultBroker>(fromPath, uriPublic, docKey, handler.getSearchResultContent());
+            handler.takeFile();
+
+            cleanupDocBrokers();
+
+            DocBrokers.emplace(docKey, docBroker);
+            LOG_TRC("Have " << DocBrokers.size() << " DocBrokers after inserting [" << docKey << "].");
+
+            if (!docBroker->executeCommand(disposition, _id))
+            {
+                LOG_WRN("Failed to create Client Session with id [" << _id << "] on docKey [" << docKey << "].");
+                cleanupDocBrokers();
+            }
+
+            return;
+        }
 
         throw BadRequestException("Invalid or unknown request.");
     }