Add a remote font download feature

The coolwsd.xml file can now contain a URI of a JSON file on some
server that contains URIs of fonts. These fonts are downloaded to the
coolwsd server. Just like the remote configuration thing, the URIs are
checked once a minute and the JSON or the fonts mentioned in it are
re-downloaded if their contents has changed.

If a font has been removed from the JSON file then the corresponding
downloaded could be removed, too. But there is no way to remove it
from core without restarting the whole COOL server, so we don't
bother.

We need to put the font in such a place so that its pathname is the
same both in the ForKit process (outside any chroot jail) and in a Kit
process (inside its own jail(), because even if it is in the ForKit
process that we call the LO core vcl API to load a "temporary" font,
code elsewhere in LO core re-opens the font file later, naturally
using the same pathname, when it is needed (see
FreetypeFontFile::Map() in vcl/unx/generic/glyphs/freetype_glyphcache.cxx).

Signed-off-by: Tor Lillqvist <tml@collabora.com>
Change-Id: If78058ddff5ed05c7a82d7ea465a7a414fd0d861

7 files changed, 172 insertions, 13 deletions

diff --git a/coolwsd-systemplate-setup b/coolwsd-systemplate-setup
index 3434ea9b1e..74e5802878 100755
--- a/coolwsd-systemplate-setup
+++ b/coolwsd-systemplate-setup
@@ -5,7 +5,10 @@ test $# -eq 2 || { echo "Usage: $0 <chroot template directory for system libs to
 
 # No provision for spaces or other weird characters in pathnames. So sue me.
 
+# First parameter is the pathname where this script will create the "systemplate" tree
 CHROOT=$1
+
+# Second parameter is the instdir directory of the LibreOffice installation to be used
 INSTDIR=$2
 
 test -d "$INSTDIR" || { echo "$0: No such directory: $INSTDIR"; exit 1; }
@@ -81,7 +84,7 @@ mkdir -p $CHROOT/etc/
 for file in hosts nsswitch.conf resolv.conf passwd group host.conf timezone localtime
 do
     # echo "Linking/Copying /etc/$file"
-    # Prefer hard-linking, fallback to just copying (do *not* use soft-linking because that would be relative to the jail).
+    # Prefer hard-linking, fallback to just copying (do *not* use symlinking because that would be relative to the jail).
     # When copying, we must make sure that we copy the source and not a symlink. Otherwise, the source won't be accessible from the jail.
     # In addition, we flag that at least one file is copied by creating the 'copied' file, so that we do check for updates.
     ln -f `${REALPATH} /etc/$file` $CHROOT/etc/$file 2> /dev/null || (${CP} --dereference --preserve=all /etc/$file $CHROOT/etc/$file && touch $CHROOT/etc/copied) || echo "$0: Failed to link or copy /etc/$file"
@@ -93,7 +96,7 @@ mkdir -p $CHROOT/dev
 mkdir -p $CHROOT/tmp/dev
 for file in random urandom
 do
-    # This link is relative anyway, so can be soft.
+    # This link is relative anyway, so can be symbolic.
     ln -f ../tmp/dev/$file $CHROOT/dev/ 2> /dev/null || ln -f -s ../tmp/dev/$file $CHROOT/dev/ || echo "$0: Failed to link dev/$file"
 done
 
@@ -104,12 +107,20 @@ mkdir -p $CHROOT/lo
 # In case the original path is different from
 for path in $INSTDIR $INSTDIR_LOGICAL
 do
-    # Create a soft-link, as it's a relative directory path (can't be a hard-link).
+    # Create a symlink, as it's a relative directory path (can't be a hard-link).
     INSTDIR_PARENT="$(dirname "$CHROOT/$path")"
     mkdir -p $INSTDIR_PARENT
     ln -f -s `${REALPATH} --relative-to=$INSTDIR_PARENT $CHROOT/lo` $CHROOT/$path
 done
 
+# A similar dance also for the directory where "temporary" fonts are
+# stored. Such are downloaded from other servers based on a separate
+# configuration file. They need to be accessible using the same
+# pathname both in the ForKit process and in the Kit processes.
+mkdir -p $CHROOT/tmpfonts
+mkdir -p $CHROOT$CHROOT
+ln -f -s `${REALPATH} --relative-to=$CHROOT$CHROOT $CHROOT/tmpfonts` $CHROOT$CHROOT
+
 # /usr/share/fonts needs to be taken care of separately because the
 # directory time stamps must be preserved for fontconfig to trust
 # its cache.
diff --git a/coolwsd.xml.in b/coolwsd.xml.in
index 4784db15bd..314c311108 100644
--- a/coolwsd.xml.in
+++ b/coolwsd.xml.in
@@ -241,6 +241,10 @@
         <remote_url desc="remote server to which you will send resquest to get remote config in response" type="string" default=""></remote_url>
     </remote_config>
 
+    <remote_font_config>
+        <url desc="URL of optional JSON file that lists fonts to be included in Online" type="string" default=""></url>
+    </remote_font_config>
+
     @LOCK_CONFIGURATION@
 
     @FEATURE_RESTRICTION_CONFIGURATION@
diff --git a/kit/ForKit.cpp b/kit/ForKit.cpp
index 910e969cc7..6fed576b60 100644
--- a/kit/ForKit.cpp
+++ b/kit/ForKit.cpp
@@ -170,6 +170,14 @@ protected:
                 LOG_ERR("Unknown setconfig command: " << message);
             }
         }
+        else if (tokens.size() == 2 && tokens.equals(0, "addfont"))
+        {
+            // Tell core to use that font file
+            std::string fontFile = tokens[1];
+
+            assert(loKitPtr);
+            loKitPtr->pClass->setOption(loKitPtr, "addfont", Poco::URI(Poco::Path(fontFile)).toString().c_str());
+        }
         else if (tokens.equals(0, "exit"))
         {
             LOG_INF("Setting TerminationFlag due to 'exit' command from parent.");
diff --git a/kit/Kit.cpp b/kit/Kit.cpp
index c97713a706..23d590f94f 100644
--- a/kit/Kit.cpp
+++ b/kit/Kit.cpp
@@ -120,6 +120,8 @@ class Document;
 static Document *singletonDocument = nullptr;
 #endif
 
+_LibreOfficeKit* loKitPtr = nullptr;
+
 /// Used for test code to accelerating waiting until idle and to
 /// flush sockets with a 'processtoidle' -> 'idle' reply.
 static std::chrono::steady_clock::time_point ProcessToIdleDeadline;
@@ -2446,8 +2448,6 @@ void copyCertificateDatabaseToTmp(Poco::Path const& jailPath)
 #endif
 }
 
-
-
 void lokit_main(
 #if !MOBILEAPP
                 const std::string& childRoot,
@@ -2967,13 +2967,12 @@ bool globalPreinit(const std::string &loTemplate)
         }
     }
 
-    LokHookPreInit* preInit = reinterpret_cast<LokHookPreInit *>(dlsym(handle, "lok_preinit"));
+    LokHookPreInit2* preInit = reinterpret_cast<LokHookPreInit2 *>(dlsym(handle, "lok_preinit_2"));
     if (!preInit)
     {
-        LOG_FTL("No lok_preinit symbol in " << loadedLibrary << ": " << dlerror());
+        LOG_FTL("No lok_preinit_2 symbol in " << loadedLibrary << ": " << dlerror());
         return false;
     }
-
     initFunction = reinterpret_cast<LokHookFunction2 *>(dlsym(handle, "libreofficekit_hook_2"));
     if (!initFunction)
     {
@@ -2992,14 +2991,16 @@ bool globalPreinit(const std::string &loTemplate)
              "javaloader javavm jdbc rpt rptui rptxml ",
              0 /* no overwrite */);
 
-    LOG_TRC("Invoking lok_preinit(" << loTemplate << "/program\", \"file:///tmp/user\")");
+    LOG_TRC("Invoking lok_preinit_2(" << loTemplate << "/program\", \"file:///tmp/user\")");
     const auto start = std::chrono::steady_clock::now();
-    if (preInit((loTemplate + "/program").c_str(), "file:///tmp/user") != 0)
+    if (preInit((loTemplate + "/program").c_str(), "file:///tmp/user", &loKitPtr) != 0)
     {
         LOG_FTL("lok_preinit() in " << loadedLibrary << " failed");
         return false;
     }
 
+    LOG_DBG("After lok_preinit_2: loKitPtr=" << loKitPtr);
+
     LOG_TRC("Finished lok_preinit(" << loTemplate << "/program\", \"file:///tmp/user\") in "
                                     << std::chrono::duration_cast<std::chrono::milliseconds>(
                                            std::chrono::steady_clock::now() - start));
diff --git a/kit/Kit.hpp b/kit/Kit.hpp
index 792e2965f1..ee7b2ec4e4 100644
--- a/kit/Kit.hpp
+++ b/kit/Kit.hpp
@@ -144,4 +144,6 @@ std::string anonymizeUsername(const std::string& username);
 std::shared_ptr<lok::Document> getLOKDocumentForAndroidOnly();
 #endif
 
+extern _LibreOfficeKit* loKitPtr;
+
 /* vim:set shiftwidth=4 softtabstop=4 expandtab: */
diff --git a/wsd/COOLWSD.cpp b/wsd/COOLWSD.cpp
index 0db00648b0..9640177e44 100644
--- a/wsd/COOLWSD.cpp
+++ b/wsd/COOLWSD.cpp
@@ -868,6 +868,7 @@ std::string COOLWSD::ChildRoot;
 std::string COOLWSD::ServerName;
 std::string COOLWSD::FileServerRoot;
 std::string COOLWSD::ServiceRoot;
+std::string COOLWSD::TmpFontDir;
 std::string COOLWSD::LOKitVersion;
 std::string COOLWSD::ConfigFile = COOLWSD_CONFIGDIR "/coolwsd.xml";
 std::string COOLWSD::ConfigDir = COOLWSD_CONFIGDIR "/conf.d";
@@ -1031,11 +1032,14 @@ public:
 
     virtual void handleJSON(Poco::JSON::Object::Ptr json) = 0;
 
+    virtual void handleUnchangedJSON()
+    { }
+
     void start()
     {
         if (remoteServerURI.empty())
         {
-            LOG_INF("Remote config url is not specified in coolwsd.xml");
+            LOG_INF("Remote " << expectedKind << " is not specified in coolwsd.xml");
             return; // no remote config server setup.
         }
 #if !ENABLE_DEBUG
@@ -1100,6 +1104,7 @@ public:
                 {
                     LOG_WRN("Remote config server has response status code: " +
                             std::to_string(statusCode) + ", JSON has not been changed");
+                    handleUnchangedJSON();
                 }
                 else
                 {
@@ -1117,6 +1122,8 @@ public:
 
 protected:
     LayeredConfiguration& conf;
+
+private:
     Poco::URI remoteServerURI;
     std::string expectedKind;
     std::string eTagValue;
@@ -1360,6 +1367,115 @@ public:
         return booleanFlag.toString();
     }
 };
+
+class RemoteFontConfigPoll : public RemoteJSONPoll
+{
+public:
+    RemoteFontConfigPoll(LayeredConfiguration& config)
+        : RemoteJSONPoll(config, Poco::URI(config.getString("remote_font_config.url")), "remotefontconfig_poll", "fontconfiguration")
+    {
+    }
+
+    virtual ~RemoteFontConfigPoll() { }
+
+    void handleJSON(Poco::JSON::Object::Ptr remoteJson) override
+    {
+        // First mark all fonts we have downloaded previously as "inactive" to be able to check if
+        // some font gets deleted from the list in the JSON file.
+        for (auto& it : fonts)
+            it.second.active = false;
+
+        // Just pick up new fonts.
+        for (auto& fontPtr : *remoteJson->getArray("fonts"))
+        {
+            if (!fontPtr.isString())
+                LOG_WRN("Element in fonts array is not a string");
+            else
+            {
+                fonts[std::string(fontPtr)].active = true;
+            }
+        }
+
+        // Any font that has been deleted from the JSON needs to be removed on this side, too.
+        for (auto it = fonts.begin(); it != fonts.end();)
+        {
+            if (!it->second.active)
+            {
+                LOG_DBG("Font no longer mentioned in the remote font config: " << it->first);
+                // FIXME: Removing the font file will make it unusable, but sadly it is not possible
+                // for ForKit to remove knowledge of it from core, so it would still show up in the
+                // font menu. Not sure what to do here.
+                // COOLWSD::sendMessageToForKit("removefont " + it->second.active);
+                // Oh well, at least remove it from our map.
+                it = fonts.erase(it);
+            }
+            else
+                ++it;
+        }
+    }
+
+    void handleUnchangedJSON() override
+    {
+        // Iterate over the fonts that were mentioned in the JSON file when it was last downloaded.
+        for (auto& it : fonts)
+        {
+            const Poco::URI fontURI{it.first};
+            std::shared_ptr<http::Session> httpSession(StorageBase::getHttpSession(fontURI));
+            http::Request request(fontURI.getPathAndQuery());
+
+            if (!it.second.eTag.empty())
+            {
+                request.set("If-None-Match", it.second.eTag);
+            }
+
+            request.set("User-Agent", WOPI_AGENT_STRING);
+
+            const std::shared_ptr<const http::Response> httpResponse
+                = httpSession->syncRequest(request);
+
+            unsigned int statusCode = httpResponse->statusLine().statusCode();
+
+            if (statusCode == Poco::Net::HTTPResponse::HTTP_NOT_MODIFIED)
+                LOG_DBG("Not modified since last time: " << it.first);
+            else if (statusCode != Poco::Net::HTTPResponse::HTTP_OK)
+                LOG_WRN("Could not fetch " << it.first);
+            else
+            {
+                it.second.eTag = httpResponse->get("ETag");
+
+                const std::string body = httpResponse->getBody();
+
+                const std::string fontFile = COOLWSD::TmpFontDir + "/" + Util::encodeId(Util::rng::getNext()) + ".ttf";
+                std::ofstream fontStream(fontFile);
+                fontStream.write(body.data(), body.size());
+                if (fontStream.good())
+                {
+                    LOG_DBG("Got " << body.size() << " bytes for " << it.first << " and wrote to " << fontFile);
+
+                    it.second.pathName = fontFile;
+                    COOLWSD::sendMessageToForKit("addfont " + fontFile);
+                }
+                else
+                    LOG_ERR("Could not write to " << fontFile);
+            }
+        }
+    }
+
+private:
+    struct FontData
+    {
+        std::string eTag;
+
+        // Where the font has been stored
+        std::string pathName;
+
+        // Flag that tells whether the font is mentioned in the JSON file that is being handled.
+        bool active;
+    };
+
+    // The key of this map is the download URI of the font.
+    std::map<std::string, FontData> fonts;
+};
 #endif
 
 void COOLWSD::innerInitialize(Application& self)
@@ -4590,13 +4706,12 @@ int COOLWSD::innerMain()
 
     initializeSSL();
 
-    // Fetch remote settings from server if configured
 #if !MOBILEAPP
+    // Fetch remote settings from server if configured
     RemoteConfigPoll remoteConfigThread(config());
     remoteConfigThread.start();
 #endif
 
-
 #ifndef IOS
     // We can open files with non-ASCII names just fine on iOS without this, and this code is
     // heavily Linux-specific anyway.
@@ -4644,6 +4759,8 @@ int COOLWSD::innerMain()
     assert(Server && "The COOLWSDServer instance does not exist.");
     Server->findClientPort();
 
+    TmpFontDir = SysTemplate + "/tmpfonts";
+
     // Start the internal prisoner server and spawn forkit,
     // which in turn forks first child.
     Server->startPrisoners();
@@ -4716,6 +4833,21 @@ int COOLWSD::innerMain()
     std::cerr << "Ready to accept connections on port " << ClientPortNumber <<  ".\n" << std::endl;
 #endif
 
+#if !MOBILEAPP
+    // Start the remote font downloading polling thread.
+    std::unique_ptr<RemoteFontConfigPoll> remoteFontConfigThread;
+    try
+    {
+        // Fetch font settings from server if configured
+        remoteFontConfigThread = Util::make_unique<RemoteFontConfigPoll>(config());
+        remoteFontConfigThread->start();
+    }
+    catch (const Poco::Exception&)
+    {
+        LOG_DBG("No remote_font_config");
+    }
+#endif
+
     const auto startStamp = std::chrono::steady_clock::now();
 
     while (!SigUtil::getTerminationFlag() && !SigUtil::getShutdownRequestFlag())
diff --git a/wsd/COOLWSD.hpp b/wsd/COOLWSD.hpp
index d513201351..abb5c82d15 100644
--- a/wsd/COOLWSD.hpp
+++ b/wsd/COOLWSD.hpp
@@ -246,6 +246,7 @@ public:
     static std::string ServerName;
     static std::string FileServerRoot;
     static std::string ServiceRoot; ///< There are installations that need prefixing every page with some path.
+    static std::string TmpFontDir;
     static std::string LOKitVersion;
     static bool EnableTraceEventLogging;
     static FILE *TraceEventFile;