vcl: fix hangs in SvpSalInstance

Since commit cb8bfa9a32799bcde4e960fa56e388d5f7b2a928 the main thread
will read from the timer pipe until it is empty. But evidently this
introduces the problem that the poll() in another thread will not
return, as the file descriptor will no longer be readable; see
https://paste.debian.net/1011306/ for a reproducer of that rather
under-documented poll behaviour.  So other threads can get stuck
forever in poll, and then the main thread can block in poll too with
no other thread to wake it up.  This is the problem that plagues
UITest_writerperfect_epubexport.

The timer pipe is difficult to fix, since the main thread can block on
either the poll or the subsequent AcquireYieldMutex().

So replace the timer pipe with a condition etc. that is mostly
copied from the OSX AquaSalInstance/SalYieldMutex implementation.

The main thread now does something different than the other threads,
and blocks on a condition_variable with a timeout, while other
threads still block on acquiring the mutex.

Non-main threads can poke the main thread to do a DoYield()
on their behalf, and then get the result back with a blocking
read from a pipe, all while holding the YieldMutex.  This
requires some fudging of the YieldMutex so that the main
thread can borrow the ownership temporarily.

Unfortunately SvpSalInstance, in addition to being directly
instantiable for --headless, has a whole bunch of subclasses:

* HeadlessSalInstance
* AndroidSalInstance
* IosSalInstance
* GtkInstance (in the gtk3 case)
* KDE5SalInstance

Of these GtkInstance overrides everything related to the
DoYield/SalYieldMutex implementation, but the others will be
affected by the change.

This commit will probably break IOS due to me not understanding the
point of the undocumented random #ifdef IOS in svpinst.cxx.

Change-Id: I1bbb143952dda89579e97ac32cd147e5b987573c
Reviewed-on: https://gerrit.libreoffice.org/50237
Tested-by: Jenkins <ci@libreoffice.org>
Reviewed-by: Michael Stahl <mstahl@redhat.com>

7 files changed, 260 insertions, 80 deletions

diff --git a/vcl/android/androidinst.cxx b/vcl/android/androidinst.cxx
index ae0399518c56..6400deea11c0 100644
--- a/vcl/android/androidinst.cxx
+++ b/vcl/android/androidinst.cxx
@@ -189,7 +189,7 @@ SalData::~SalData()
 SalInstance *CreateSalInstance()
 {
     LOGI("Android: CreateSalInstance!");
-    AndroidSalInstance* pInstance = new AndroidSalInstance( new SalYieldMutex() );
+    AndroidSalInstance* pInstance = new AndroidSalInstance( new SvpSalYieldMutex() );
     new AndroidSalData( pInstance );
     pInstance->AcquireYieldMutex();
     return pInstance;
diff --git a/vcl/headless/headlessinst.cxx b/vcl/headless/headlessinst.cxx
index a5f1c6ebd8ea..b49859c03c20 100644
--- a/vcl/headless/headlessinst.cxx
+++ b/vcl/headless/headlessinst.cxx
@@ -90,7 +90,7 @@ SalData::~SalData()
 // This is our main entry point:
 SalInstance *CreateSalInstance()
 {
-    HeadlessSalInstance* pInstance = new HeadlessSalInstance( new SalYieldMutex() );
+    HeadlessSalInstance* pInstance = new HeadlessSalInstance( new SvpSalYieldMutex() );
     new HeadlessSalData( pInstance );
     pInstance->AcquireYieldMutex();
     return pInstance;
diff --git a/vcl/headless/svpdata.cxx b/vcl/headless/svpdata.cxx
index 3bc2807c8cc5..ce72305f85d9 100644
--- a/vcl/headless/svpdata.cxx
+++ b/vcl/headless/svpdata.cxx
@@ -21,7 +21,7 @@ public:
 // plugin factory function
 SalInstance* svp_create_SalInstance()
 {
-    SvpSalInstance* pInstance = new SvpSalInstance( new SalYieldMutex() );
+    SvpSalInstance* pInstance = new SvpSalInstance( new SvpSalYieldMutex() );
     new SvpSalData( pInstance );
     return pInstance;
 }
diff --git a/vcl/headless/svpinst.cxx b/vcl/headless/svpinst.cxx
index 9912186c729c..dc71f63b0b37 100644
--- a/vcl/headless/svpinst.cxx
+++ b/vcl/headless/svpinst.cxx
@@ -62,15 +62,15 @@ static void atfork_child()
 
 #endif
 
-SvpSalInstance::SvpSalInstance( SalYieldMutex *pMutex ) :
-    SalGenericInstance( pMutex )
+SvpSalInstance::SvpSalInstance( SalYieldMutex *pMutex )
+    : SalGenericInstance( pMutex )
 {
     m_aTimeout.tv_sec       = 0;
     m_aTimeout.tv_usec      = 0;
     m_nTimeoutMS            = 0;
 
 #ifndef IOS
-    m_pTimeoutFDS[0] = m_pTimeoutFDS[1] = -1;
+    m_MainThread = osl::Thread::getCurrentIdentifier();
     CreateWakeupPipe(true);
 #endif
     if( s_pDefaultInstance == nullptr )
@@ -93,60 +93,56 @@ SvpSalInstance::~SvpSalInstance()
 
 void SvpSalInstance::CloseWakeupPipe(bool log)
 {
-    if (m_pTimeoutFDS[0] != -1)
+    SvpSalYieldMutex *const pMutex(dynamic_cast<SvpSalYieldMutex*>(mpSalYieldMutex.get()));
+    if (!pMutex)
+        return;
+    if (pMutex->m_FeedbackFDs[0] != -1)
     {
         if (log)
         {
-            SAL_INFO("vcl.headless", "CloseWakeupPipe: Closing inherited wakeup pipe: [" << m_pTimeoutFDS[0] << "," << m_pTimeoutFDS[1] << "]");
+            SAL_INFO("vcl.headless", "CloseWakeupPipe: Closing inherited feedback pipe: [" << pMutex->m_FeedbackFDs[0] << "," << pMutex->m_FeedbackFDs[1] << "]");
         }
-        close (m_pTimeoutFDS[0]);
-        close (m_pTimeoutFDS[1]);
-        m_pTimeoutFDS[0] = m_pTimeoutFDS[1] = -1;
+        close (pMutex->m_FeedbackFDs[0]);
+        close (pMutex->m_FeedbackFDs[1]);
+        pMutex->m_FeedbackFDs[0] = pMutex->m_FeedbackFDs[1] = -1;
     }
 }
 
 void SvpSalInstance::CreateWakeupPipe(bool log)
 {
-    if (pipe (m_pTimeoutFDS) == -1)
+    SvpSalYieldMutex *const pMutex(dynamic_cast<SvpSalYieldMutex*>(mpSalYieldMutex.get()));
+    if (!pMutex)
+        return;
+    if (pipe (pMutex->m_FeedbackFDs) == -1)
     {
         if (log)
         {
-            SAL_WARN("vcl.headless", "Could not create wakeup pipe: " << strerror(errno));
+            SAL_WARN("vcl.headless", "Could not create feedback pipe: " << strerror(errno));
+            std::abort();
         }
     }
     else
     {
         if (log)
         {
-            SAL_INFO("vcl.headless", "CreateWakeupPipe: Created wakeup pipe: [" << m_pTimeoutFDS[0] << "," << m_pTimeoutFDS[1] << "]");
+            SAL_INFO("vcl.headless", "CreateWakeupPipe: Created feedback pipe: [" << pMutex->m_FeedbackFDs[0] << "," << pMutex->m_FeedbackFDs[1] << "]");
         }
 
-        // initialize 'wakeup' pipe.
         int flags;
 
         // set close-on-exec descriptor flag.
-        if ((flags = fcntl (m_pTimeoutFDS[0], F_GETFD)) != -1)
+        if ((flags = fcntl (pMutex->m_FeedbackFDs[0], F_GETFD)) != -1)
         {
             flags |= FD_CLOEXEC;
-            (void)fcntl(m_pTimeoutFDS[0], F_SETFD, flags);
+            (void) fcntl(pMutex->m_FeedbackFDs[0], F_SETFD, flags);
         }
-        if ((flags = fcntl (m_pTimeoutFDS[1], F_GETFD)) != -1)
+        if ((flags = fcntl (pMutex->m_FeedbackFDs[1], F_GETFD)) != -1)
         {
             flags |= FD_CLOEXEC;
-            (void)fcntl(m_pTimeoutFDS[1], F_SETFD, flags);
+            (void) fcntl(pMutex->m_FeedbackFDs[1], F_SETFD, flags);
         }
 
-        // set non-blocking I/O flag.
-        if ((flags = fcntl(m_pTimeoutFDS[0], F_GETFL)) != -1)
-        {
-            flags |= O_NONBLOCK;
-            (void)fcntl(m_pTimeoutFDS[0], F_SETFL, flags);
-        }
-        if ((flags = fcntl(m_pTimeoutFDS[1], F_GETFL)) != -1)
-        {
-            flags |= O_NONBLOCK;
-            (void)fcntl(m_pTimeoutFDS[1], F_SETFL, flags);
-        }
+        // retain the default blocking I/O for feedback pipe
     }
 }
 
@@ -157,10 +153,29 @@ void SvpSalInstance::TriggerUserEventProcessing()
     Wakeup();
 }
 
-void SvpSalInstance::Wakeup()
+#ifndef NDEBUG
+static bool g_CheckedMutex = false;
+#endif
+
+void SvpSalInstance::Wakeup(SvpRequest const request)
 {
+#ifndef NDEBUG
+    if (!g_CheckedMutex)
+    {
+        assert(dynamic_cast<SvpSalYieldMutex*>(mpSalYieldMutex.get()) != nullptr
+            && "This SvpSalInstance function requires use of SvpSalYieldMutex");
+        g_CheckedMutex = true;
+    }
+#endif
 #ifndef IOS
-    OSL_VERIFY(write (m_pTimeoutFDS[1], "", 1) == 1);
+    SvpSalYieldMutex *const pMutex(static_cast<SvpSalYieldMutex*>(mpSalYieldMutex.get()));
+    std::unique_lock<std::mutex> g(pMutex->m_WakeUpMainMutex);
+    if (request != SvpRequest::NONE)
+    {
+        pMutex->m_Request = request;
+    }
+    pMutex->m_wakeUpMain = true;
+    pMutex->m_WakeUpMainCond.notify_one();
 #endif
 }
 
@@ -259,72 +274,199 @@ void SvpSalInstance::ProcessEvent( SalUserEvent aEvent )
         const SvpSalFrame* pSvpFrame = static_cast<const SvpSalFrame*>( aEvent.m_pFrame);
         pSvpFrame->PostPaint();
     }
+#ifndef NDEBUG
+    if (!g_CheckedMutex)
+    {
+        assert(dynamic_cast<SvpSalYieldMutex*>(mpSalYieldMutex.get()) != nullptr
+            && "This SvpSalInstance function requires use of SvpSalYieldMutex");
+        g_CheckedMutex = true;
+    }
+#endif
+    SvpSalYieldMutex *const pMutex(static_cast<SvpSalYieldMutex*>(mpSalYieldMutex.get()));
+    pMutex->m_NonMainWaitingYieldCond.set();
 }
 
-
-bool SvpSalInstance::DoYield(bool bWait, bool bHandleAllCurrentEvents)
+SvpSalYieldMutex::SvpSalYieldMutex()
 {
-    // first, process current user events
-    bool bEvent = DispatchUserEvents( bHandleAllCurrentEvents );
-    if ( !bHandleAllCurrentEvents && bEvent )
-        return true;
+    m_FeedbackFDs[0] = m_FeedbackFDs[1] = -1;
+}
 
-    bEvent = CheckTimeout() || bEvent;
+SvpSalYieldMutex::~SvpSalYieldMutex()
+{
+}
 
-    if (bWait && ! bEvent )
+void SvpSalYieldMutex::doAcquire(sal_uInt32 const nLockCount)
+{
+    SvpSalInstance *const pInst = static_cast<SvpSalInstance *>(GetSalData()->m_pInstance);
+    if (pInst && pInst->IsMainThread())
     {
-        int nTimeoutMS = 0;
-        if (m_aTimeout.tv_sec) // Timer is started.
+        if (m_bNoYieldLock)
+            return;
+
+        do
         {
-            timeval Timeout;
-            // determine remaining timeout.
-            gettimeofday (&Timeout, nullptr);
-            if ( m_aTimeout > Timeout )
+            SvpRequest request = SvpRequest::NONE;
+            {
+                std::unique_lock<std::mutex> g(m_WakeUpMainMutex);
+                if (m_aMutex.tryToAcquire()) {
+                    // if there's a request, the other thread holds m_aMutex
+                    assert(m_Request == SvpRequest::NONE);
+                    m_wakeUpMain = false;
+                    break;
+                }
+                m_WakeUpMainCond.wait(g, [this]() { return m_wakeUpMain; });
+                m_wakeUpMain = false;
+                std::swap(m_Request, request);
+            }
+            if (request != SvpRequest::NONE)
             {
-                int nTimeoutMicroS = m_aTimeout.tv_usec - Timeout.tv_usec;
-                nTimeoutMS = (m_aTimeout.tv_sec - Timeout.tv_sec) * 1000
-                           + nTimeoutMicroS / 1000;
-                if ( nTimeoutMicroS % 1000 )
-                    nTimeoutMS += 1;
+                // nested Yield on behalf of another thread
+                assert(!m_bNoYieldLock);
+                m_bNoYieldLock = true;
+                bool const bEvents = pInst->DoYield(false, request == SvpRequest::MainThreadDispatchAllEvents);
+                m_bNoYieldLock = false;
+                write(m_FeedbackFDs[1], &bEvents, sizeof(bool));
             }
         }
+        while (true);
+    }
+    else
+    {
+        m_aMutex.acquire();
+    }
+    ++m_nCount;
+    SalYieldMutex::doAcquire(nLockCount - 1);
+}
+
+sal_uInt32 SvpSalYieldMutex::doRelease(bool const bUnlockAll)
+{
+    SvpSalInstance *const pInst = static_cast<SvpSalInstance *>(GetSalData()->m_pInstance);
+    if (pInst && pInst->IsMainThread())
+    {
+        if (m_bNoYieldLock)
+            return 1;
         else
-            nTimeoutMS = -1; // wait until something happens
+            return SalYieldMutex::doRelease(bUnlockAll);
+    }
+    sal_uInt32 nCount;
+    {
+        // read m_nCount before doRelease
+        bool const isReleased(bUnlockAll || m_nCount == 1);
+        nCount = comphelper::GenericSolarMutex::doRelease( bUnlockAll );
+        if (isReleased) {
+            std::unique_lock<std::mutex> g(m_WakeUpMainMutex);
+            m_wakeUpMain = true;
+            m_WakeUpMainCond.notify_one();
+        }
+    }
+    return nCount;
+}
 
-        DoReleaseYield(nTimeoutMS);
+bool SvpSalYieldMutex::IsCurrentThread() const
+{
+    if (GetSalData()->m_pInstance->IsMainThread() && m_bNoYieldLock)
+    {
+        return true;
     }
-    else if ( bEvent )
+    else
     {
-        // Drain the wakeup pipe
-        int buffer;
-        while (read (m_pTimeoutFDS[0], &buffer, sizeof(buffer)) > 0);
+        return SalYieldMutex::IsCurrentThread();
     }
+}
 
-    return bEvent;
+bool SvpSalInstance::IsMainThread() const
+{
+    return osl::Thread::getCurrentIdentifier() == m_MainThread;
 }
 
-void SvpSalInstance::DoReleaseYield( int nTimeoutMS )
+bool SvpSalInstance::DoYield(bool bWait, bool bHandleAllCurrentEvents)
 {
-    // poll
-    struct pollfd aPoll;
-    aPoll.fd = m_pTimeoutFDS[0];
-    aPoll.events = POLLIN;
-    aPoll.revents = 0;
+#ifndef NDEBUG
+    if (!g_CheckedMutex)
+    {
+        assert(dynamic_cast<SvpSalYieldMutex*>(mpSalYieldMutex.get()) != nullptr
+            && "This SvpSalInstance function requires use of SvpSalYieldMutex");
+        g_CheckedMutex = true;
+    }
+#endif
 
-    // release yield mutex
-    sal_uInt32 nAcquireCount = ReleaseYieldMutexAll();
+    // first, process current user events
+    bool bEvent = DispatchUserEvents( bHandleAllCurrentEvents );
+    if ( !bHandleAllCurrentEvents && bEvent )
+        return true;
 
-    (void)poll( &aPoll, 1, nTimeoutMS );
+    bEvent = CheckTimeout() || bEvent;
 
-    // acquire yield mutex again
-    AcquireYieldMutex( nAcquireCount );
+    SvpSalYieldMutex *const pMutex(static_cast<SvpSalYieldMutex*>(mpSalYieldMutex.get()));
 
-    // clean up pipe
-    if( (aPoll.revents & POLLIN) != 0 )
+    if (IsMainThread())
+    {
+        if (bWait && ! bEvent)
+        {
+            int nTimeoutMS = 0;
+            if (m_aTimeout.tv_sec) // Timer is started.
+            {
+                timeval Timeout;
+                // determine remaining timeout.
+                gettimeofday (&Timeout, nullptr);
+                if (m_aTimeout > Timeout)
+                {
+                    int nTimeoutMicroS = m_aTimeout.tv_usec - Timeout.tv_usec;
+                    nTimeoutMS = (m_aTimeout.tv_sec - Timeout.tv_sec) * 1000
+                               + nTimeoutMicroS / 1000;
+                    if ( nTimeoutMicroS % 1000 )
+                        nTimeoutMS += 1;
+                }
+            }
+            else
+                nTimeoutMS = -1; // wait until something happens
+
+            sal_uInt32 nAcquireCount = ReleaseYieldMutexAll();
+            {
+                std::unique_lock<std::mutex> g(pMutex->m_WakeUpMainMutex);
+                // wait for doRelease() or Wakeup() to set the condition
+                if (nTimeoutMS == -1)
+                {
+                    pMutex->m_WakeUpMainCond.wait(g,
+                            [pMutex]() { return pMutex->m_wakeUpMain; });
+                }
+                else
+                {
+                    pMutex->m_WakeUpMainCond.wait_for(g,
+                            std::chrono::milliseconds(nTimeoutMS),
+                            [pMutex]() { return pMutex->m_wakeUpMain; });
+                }
+                // here no need to check m_Request because Acquire will do it
+            }
+            AcquireYieldMutex( nAcquireCount );
+        }
+        else if (bEvent)
+        {
+            pMutex->m_NonMainWaitingYieldCond.set(); // wake up other threads
+        }
+    }
+    else // !IsMainThread()
     {
-        int buffer;
-        while (read (m_pTimeoutFDS[0], &buffer, sizeof(buffer)) > 0);
+        Wakeup(bHandleAllCurrentEvents
+                ? SvpRequest::MainThreadDispatchAllEvents
+                : SvpRequest::MainThreadDispatchOneEvent);
+
+        bool bDidWork(false);
+        // blocking read (for synchronisation)
+        auto const nRet = read(pMutex->m_FeedbackFDs[0], &bDidWork, sizeof(bool));
+        assert(nRet == 1); (void) nRet;
+
+        if (!bDidWork && bWait)
+        {
+            // block & release YieldMutex until the main thread does something
+            pMutex->m_NonMainWaitingYieldCond.reset();
+            sal_uInt32 nAcquireCount = ReleaseYieldMutexAll();
+            pMutex->m_NonMainWaitingYieldCond.wait();
+            AcquireYieldMutex( nAcquireCount );
+        }
     }
+
+    return bEvent;
 }
 
 bool SvpSalInstance::AnyInput( VclInputFlags nType )
diff --git a/vcl/inc/headless/svpinst.hxx b/vcl/inc/headless/svpinst.hxx
index ba80960326dc..6027b2b4d683 100644
--- a/vcl/inc/headless/svpinst.hxx
+++ b/vcl/inc/headless/svpinst.hxx
@@ -22,6 +22,7 @@
 
 #include <osl/mutex.hxx>
 #include <osl/thread.hxx>
+#include <osl/conditn.hxx>
 #include <salinst.hxx>
 #include <salwtype.hxx>
 #include <saltimer.hxx>
@@ -30,6 +31,7 @@
 #include <unx/genprn.h>
 
 #include <list>
+#include <condition_variable>
 
 #include <time.h>
 
@@ -56,19 +58,55 @@ public:
 class SvpSalFrame;
 class GenPspGraphics;
 
+enum class SvpRequest
+{
+    NONE,
+    MainThreadDispatchOneEvent,
+    MainThreadDispatchAllEvents,
+};
+
+class SvpSalYieldMutex : public SalYieldMutex
+{
+private:
+    // note: these members might as well live in SvpSalInstance, but there is
+    // at least one subclass of SvpSalInstance (GTK3) that doesn't use them.
+    friend class SvpSalInstance;
+    // members for communication from main thread to non-main thread
+    int                     m_FeedbackFDs[2];
+    osl::Condition          m_NonMainWaitingYieldCond;
+    // members for communication from non-main thread to main thread
+    bool                    m_bNoYieldLock = false; // accessed only on main thread
+    std::mutex              m_WakeUpMainMutex; // guard m_wakeUpMain & m_Request
+    std::condition_variable m_WakeUpMainCond;
+    bool                    m_wakeUpMain = false;
+    SvpRequest              m_Request = SvpRequest::NONE;
+
+protected:
+    virtual void            doAcquire( sal_uInt32 nLockCount ) override;
+    virtual sal_uInt32      doRelease( bool bUnlockAll ) override;
+
+public:
+    SvpSalYieldMutex();
+    virtual ~SvpSalYieldMutex() override;
+
+    virtual bool IsCurrentThread() const override;
+
+};
+
 SalInstance* svp_create_SalInstance();
 
+// NOTE: the functions IsMainThread, DoYield and Wakeup *require* the use of
+// SvpSalYieldMutex; if a subclass uses something else it must override these
+// (Wakeup is only called by SvpSalTimer and SvpSalFrame)
 class VCL_DLLPUBLIC SvpSalInstance : public SalGenericInstance, public SalUserEventList
 {
     timeval                 m_aTimeout;
     sal_uLong               m_nTimeoutMS;
-    int                     m_pTimeoutFDS[2];
-
-    void                    DoReleaseYield( int nTimeoutMS );
+    oslThreadIdentifier     m_MainThread;
 
     virtual void            TriggerUserEventProcessing() override;
     virtual void            ProcessEvent( SalUserEvent aEvent ) override;
-    void                    Wakeup();
+    void                    Wakeup(SvpRequest request = SvpRequest::NONE);
 
 public:
     static SvpSalInstance*  s_pDefaultInstance;
@@ -132,7 +170,7 @@ public:
     // and timer
     virtual bool            DoYield(bool bWait, bool bHandleAllCurrentEvents) override;
     virtual bool            AnyInput( VclInputFlags nType ) override;
-    virtual bool            IsMainThread() const override { return true; }
+    virtual bool            IsMainThread() const override;
 
     // may return NULL to disable session management
     virtual SalSession*     CreateSalSession() override;
diff --git a/vcl/ios/iosinst.cxx b/vcl/ios/iosinst.cxx
index 7fed7a485cd2..749541bf964a 100644
--- a/vcl/ios/iosinst.cxx
+++ b/vcl/ios/iosinst.cxx
@@ -170,7 +170,7 @@ SalData::~SalData()
 // This is our main entry point:
 SalInstance *CreateSalInstance()
 {
-    IosSalInstance* pInstance = new IosSalInstance( new SalYieldMutex() );
+    IosSalInstance* pInstance = new IosSalInstance( new SvpSalYieldMutex() );
     new IosSalData( pInstance );
     pInstance->AcquireYieldMutex();
     return pInstance;
diff --git a/vcl/unx/kde5/main.cxx b/vcl/unx/kde5/main.cxx
index 654a9d654ee8..d1376945f4ce 100644
--- a/vcl/unx/kde5/main.cxx
+++ b/vcl/unx/kde5/main.cxx
@@ -72,7 +72,7 @@ extern "C" {
         }
 #endif
 
-        KDE5SalInstance* pInstance = new KDE5SalInstance( new SalYieldMutex() );
+        KDE5SalInstance* pInstance = new KDE5SalInstance( new SvpSalYieldMutex() );
         SAL_INFO( "vcl.kde5", "created KDE5SalInstance " << &pInstance );
 
         // initialize SalData