tdf#170961: unlock guard during UnoControl::createPeer

A deadlock seen with the following call stacks.

Main thread is trying to paint in a timer event, and is waiting for
the UnoControl's mutex:

  ntdll.dll!NtWaitForAlertByThreadId()
  ntdll.dll!RtlpWaitOnCriticalSection()
  ntdll.dll!RtlpEnterCriticalSectionContended()
  ntdll.dll!RtlEnterCriticalSection()
  sal3.dll!osl_acquireMutex(_oslMutexImpl * Mutex) Line 65
  tklo.dll!osl::Mutex::acquire() Line 63
  tklo.dll!osl::Guard<osl::Mutex>::Guard<osl::Mutex>(osl::Mutex & t) Line 144
• tklo.dll!UnoControl::getPosSize() Line 745
  tklo.dll!UnoDialogControl::getPosSize() Line 104
  svxcorelo.dll!sdr::contact::`anonymous namespace'::ControlHolder::getPosSize() Line 234
  svxcorelo.dll!sdr::contact::`anonymous namespace'::ControlHolder::setPosSize(const tools::Rectangle & _rPosSize) Line 220
  svxcorelo.dll!sdr::contact::UnoControlContactHelper::adjustControlGeometry_throw(const sdr::contact::`anonymous-namespace'::ControlHolder & _rControl, const tools::Rectangle & _rLogicBoundingRect, const basegfx::B2DHomMatrix & _rViewTransformation, const basegfx::B2DHomMatrix & _rZoomLevelNormalization) Line 316
  svxcorelo.dll!sdr::contact::ViewObjectContactOfUnoControl_Impl::positionAndZoomControl(const basegfx::B2DHomMatrix & _rViewTransformation) Line 940
  svxcorelo.dll!sdr::contact::ViewObjectContactOfUnoControl::isPrimitiveVisible(const sdr::contact::DisplayInfo & _rDisplayInfo) Line 1717
  svxcorelo.dll!sdr::contact::ViewObjectContact::getPrimitive2DSequenceHierarchy(sdr::contact::DisplayInfo & rDisplayInfo, drawinglayer::primitive2d::Primitive2DDecompositionVisitor & rVisitor) Line 559
  svxcorelo.dll!sdr::contact::ViewContact::getPrimitive2DSequenceHierarchyOfIndex(unsigned long a, sdr::contact::DisplayInfo & rDisplayInfo, sdr::contact::ObjectContact & rObjectContact, drawinglayer::primitive2d::Primitive2DDecompositionVisitor & rVisitor) Line 324
  svxcorelo.dll!sdr::contact::ViewObjectContact::getPrimitive2DSequenceSubHierarchy(sdr::contact::DisplayInfo & rDisplayInfo, drawinglayer::primitive2d::Primitive2DDecompositionVisitor & rVisitor) Line 594
  svxcorelo.dll!sdr::contact::ViewObjectContactOfPageHierarchy::getPrimitive2DSequenceHierarchy(sdr::contact::DisplayInfo & rDisplayInfo, drawinglayer::primitive2d::Primitive2DDecompositionVisitor & rVisitor) Line 371
  svxcorelo.dll!sdr::contact::ViewContact::getPrimitive2DSequenceHierarchyOfIndex(unsigned long a, sdr::contact::DisplayInfo & rDisplayInfo, sdr::contact::ObjectContact & rObjectContact, drawinglayer::primitive2d::Primitive2DDecompositionVisitor & rVisitor) Line 324
  svxcorelo.dll!sdr::contact::ViewObjectContact::getPrimitive2DSequenceSubHierarchy(sdr::contact::DisplayInfo & rDisplayInfo, drawinglayer::primitive2d::Primitive2DDecompositionVisitor & rVisitor) Line 594
  svxcorelo.dll!sdr::contact::ViewObjectContactOfSdrPage::getPrimitive2DSequenceHierarchy(sdr::contact::DisplayInfo & rDisplayInfo, drawinglayer::primitive2d::Primitive2DDecompositionVisitor & rVisitor) Line 569
  svxcorelo.dll!sdr::contact::ObjectContactOfPageView::DoProcessDisplay(sdr::contact::DisplayInfo & rDisplayInfo) Line 252
  svxcorelo.dll!sdr::contact::ObjectContactOfPageView::ProcessDisplay(sdr::contact::DisplayInfo & rDisplayInfo) Line 128
  svxcorelo.dll!SdrPageWindow::RedrawLayer(const o3tl::strong_int<short,SdrLayerIDTag> * pId, sdr::contact::ViewObjectContactRedirector * pRedirector, const basegfx::B2IRange * pPageFrame) Line 422
  svxcorelo.dll!SdrPageView::DrawLayer(o3tl::strong_int<short,SdrLayerIDTag> nID, OutputDevice * pGivenTarget, sdr::contact::ViewObjectContactRedirector * pRedirector, const tools::Rectangle & rRect, const basegfx::B2IRange * pPageFrame) Line 304
  svxcorelo.dll!SdrPaintView::ImpFormLayerDrawing(SdrPaintWindow & rPaintWindow, sdr::contact::ViewObjectContactRedirector * pRedirector) Line 798
  svxcorelo.dll!SdrPaintView::EndCompleteRedraw(SdrPaintWindow & rPaintWindow, bool bPaintFormLayer, sdr::contact::ViewObjectContactRedirector * pRedirector) Line 653
  svxcorelo.dll!SdrPaintView::EndDrawLayers(SdrPaintWindow & rPaintWindow, bool bPaintFormLayer, sdr::contact::ViewObjectContactRedirector * pRedirector) Line 726
  basctllo.dll!basctl::DlgEditor::Paint(OutputDevice & rRenderContext, const tools::Rectangle & rRect) Line 564
  basctllo.dll!basctl::DialogWindow::Paint(OutputDevice & rRenderContext, const tools::Rectangle & rRect) Line 124
  vcllo.dll!PaintHelper::DoPaint(const vcl::Region * pRegion) Line 316
  vcllo.dll!vcl::Window::ImplCallPaint(const vcl::Region * pRegion, ImplPaintFlags nPaintFlags) Line 618
  vcllo.dll!PaintHelper::~PaintHelper() Line 553
  vcllo.dll!vcl::Window::ImplCallPaint(const vcl::Region * pRegion, ImplPaintFlags nPaintFlags) Line 624
  vcllo.dll!PaintHelper::~PaintHelper() Line 553
  vcllo.dll!vcl::Window::ImplCallPaint(const vcl::Region * pRegion, ImplPaintFlags nPaintFlags) Line 624
  vcllo.dll!PaintHelper::~PaintHelper() Line 553
  vcllo.dll!vcl::Window::ImplCallPaint(const vcl::Region * pRegion, ImplPaintFlags nPaintFlags) Line 624
  vcllo.dll!PaintHelper::~PaintHelper() Line 553
  vcllo.dll!vcl::Window::ImplCallPaint(const vcl::Region * pRegion, ImplPaintFlags nPaintFlags) Line 624
  vcllo.dll!PaintHelper::~PaintHelper() Line 553
  vcllo.dll!vcl::Window::ImplCallPaint(const vcl::Region * pRegion, ImplPaintFlags nPaintFlags) Line 624
  vcllo.dll!PaintHelper::~PaintHelper() Line 553
  vcllo.dll!vcl::Window::ImplCallPaint(const vcl::Region * pRegion, ImplPaintFlags nPaintFlags) Line 624
  vcllo.dll!vcl::Window::ImplCallOverlapPaint() Line 645
  vcllo.dll!vcl::Window::ImplHandlePaintHdl(Timer * __formal) Line 668
  vcllo.dll!vcl::Window::LinkStubImplHandlePaintHdl(void * instance, Timer * data) Line 649
  vcllo.dll!Link<Timer *,void>::Call(Timer * data) Line 105
  vcllo.dll!Timer::Invoke() Line 75
  vcllo.dll!Scheduler::CallbackTaskScheduling() Line 615
  vcllo.dll!SalTimer::CallCallback() Line 53
  vclplug_winlo.dll!WinSalTimer::ImplHandleElapsedTimer() Line 169
  vclplug_winlo.dll!ImplSalYield(bool bWait, bool bHandleAllCurrentEvents) Line 443
  vclplug_winlo.dll!WinSalInstance::DoYield(bool bWait, bool bHandleAllCurrentEvents) Line 499
  vcllo.dll!InnerYield(bool i_bWait, bool i_bAllEvents) Line 389
  vcllo.dll!Application::Yield() Line 502
  vcllo.dll!Application::Execute() Line 365
  sofficeapp.dll!desktop::Desktop::Main() Line 1681
  vcllo.dll!ImplSVMain() Line 230
  vcllo.dll!SVMain() Line 249
  sofficeapp.dll!soffice_main() Line 122
  soffice.bin!sal_main() Line 51
  soffice.bin!main(int argc, char * * argv) Line 49
  soffice.bin!invoke_main() Line 79
  soffice.bin!__scrt_common_main_seh() Line 288
  soffice.bin!__scrt_common_main() Line 331
  soffice.bin!mainCRTStartup(void * __formal) Line 17
  kernel32.dll!BaseThreadInitThunk()
  ntdll.dll!RtlUserThreadStart()

while the worker thread is in UnoControl::createPeer, holding the mutex
(and its calling UnoControlContainer::createPeer is holding the same
mutex), and is sending a message to the main thread synchronously:

  win32u.dll!NtUserMessageCall()
  user32.dll!SendMessageWorker()
  user32.dll!SendMessageInternal(struct HWND__ *,unsigned int,unsigned __int64,__int64,int)
  user32.dll!SendMessageW()
  vclplug_winlo.dll!WinSalInstance::SendWndMessage_impl(HWND__ * hWnd, unsigned int Msg, unsigned __int64 wParam, __int64 lParam) Line 689
  vclplug_winlo.dll!WinSalInstance::SendComWndMessage(unsigned int Msg, unsigned __int64 wParam, __int64 lParam) Line 700
  vclplug_winlo.dll!WinSalFrame::AcquireGraphics() Line 1015
  vcllo.dll!vcl::WindowOutputDevice::AcquireGraphics() Line 838
  vcllo.dll!OutputDevice::IsNativeControlSupported(ControlType nType, ControlPart nPart) Line 140
  vcllo.dll!vcl::Window::IsNativeControlSupported(ControlType nType, ControlPart nPart) Line 78
  vcllo.dll!Dialog::ImplInitSettings() Line 526
  vcllo.dll!Dialog::ImplInitDialog(vcl::Window * pParent, __int64 nStyle, Dialog::InitFlag eFlag) Line 497
  vcllo.dll!Dialog::Dialog(vcl::Window * pParent, __int64 nStyle, Dialog::InitFlag eFlag) Line 588
  tklo.dll!toolkit::ScrollableDialog::ScrollableDialog(vcl::Window * pParent, __int64 nStyle, Dialog::InitFlag eFlag) Line 33
  tklo.dll!VclPtr<toolkit::ScrollableDialog>::Create<vcl::Window * &,__int64 &>(vcl::Window * & <arg_0>, __int64 & <arg_1>) Line 153
  tklo.dll!`anonymous namespace'::VCLXToolkit::ImplCreateWindow(rtl::Reference<VCLXWindow> * ppNewComp, const com::sun::star::awt::WindowDescriptor & rDescriptor, vcl::Window * pParent, __int64 nWinBits, MessBoxStyle nMessBoxStyle) Line 1501
  tklo.dll!`anonymous namespace'::VCLXToolkit::ImplCreateWindow(const com::sun::star::awt::WindowDescriptor & rDescriptor, MessBoxStyle nForceMessBoxStyle) Line 1822
  tklo.dll!`anonymous namespace'::VCLXToolkit::createWindow(const com::sun::star::awt::WindowDescriptor & rDescriptor) Line 965
• tklo.dll!UnoControl::createPeer(const com::sun::star::uno::Reference<com::sun::star::awt::XToolkit> & rxToolkit, const com::sun::star::uno::Reference<com::sun::star::awt::XWindowPeer> & rParentPeer) Line 1262
  tklo.dll!UnoControlContainer::createPeer(const com::sun::star::uno::Reference<com::sun::star::awt::XToolkit> & rxToolkit, const com::sun::star::uno::Reference<com::sun::star::awt::XWindowPeer> & rParent) Line 727
  tklo.dll!UnoDialogControl::createPeer(const com::sun::star::uno::Reference<com::sun::star::awt::XToolkit> & rxToolkit, const com::sun::star::uno::Reference<com::sun::star::awt::XWindowPeer> & rParentPeer) Line 357
  tklo.dll!UnoControl::ImplModelPropertiesChanged(const com::sun::star::uno::Sequence<com::sun::star::beans::PropertyChangeEvent> & rEvents) Line 610
  tklo.dll!ControlContainerBase::ImplModelPropertiesChanged(const com::sun::star::uno::Sequence<com::sun::star::beans::PropertyChangeEvent> & rEvents) Line 1567
  tklo.dll!UnoDialogControl::ImplModelPropertiesChanged(const com::sun::star::uno::Sequence<com::sun::star::beans::PropertyChangeEvent> & rEvents) Line 650
  tklo.dll!UnoControl::propertiesChange(const com::sun::star::uno::Sequence<com::sun::star::beans::PropertyChangeEvent> & rEvents) Line 420
  cppuhelper3MSC.dll!cppu::OPropertySetHelper::fire(long * pnHandles, const com::sun::star::uno::Any * pNewValues, const com::sun::star::uno::Any * pOldValues, long nHandles, unsigned char bVetoable) Line 779
  comphelper.dll!comphelper::OPropertySetAggregationHelper::propertiesChange(const com::sun::star::uno::Sequence<com::sun::star::beans::PropertyChangeEvent> & _rEvents) Line 398
  comphelper.dll!comphelper::OPropertySetHelper::fire(std::unique_lock<std::mutex> & rGuard, const long * pnHandles, const com::sun::star::uno::Any * pNewValues, const com::sun::star::uno::Any * pOldValues, long nHandles, bool bVetoable) Line 594
  comphelper.dll!comphelper::OPropertySetHelper::impl_fireAll(std::unique_lock<std::mutex> & rGuard, long * i_handles, const com::sun::star::uno::Any * i_newValues, const com::sun::star::uno::Any * i_oldValues, long i_count) Line 487
  comphelper.dll!comphelper::OPropertySetHelper::setFastPropertyValues(std::unique_lock<std::mutex> & rGuard, long nSeqLen, long * pHandles, const com::sun::star::uno::Any * pValues, long nHitCount) Line 748
  tklo.dll!UnoControlModel::setFastPropertyValueImpl(std::unique_lock<std::mutex> & rGuard, long nPropId, const com::sun::star::uno::Any & rValue) Line 1247
  comphelper.dll!comphelper::OPropertySetHelper::setFastPropertyValue(long nHandle, const com::sun::star::uno::Any & rValue) Line 393
  comphelper.dll!comphelper::OPropertySetAggregationHelper::setFastPropertyValue(long _nHandle, const com::sun::star::uno::Any & _rValue) Line 528
  cppuhelper3MSC.dll!cppu::OPropertySetHelper::setPropertyValue(const rtl::OUString & rPropertyName, const com::sun::star::uno::Any & rValue) Line 264
  pcrlo.dll!pcr::EditPropertyHandler::setPropertyValue(const rtl::OUString & _rPropertyName, const com::sun::star::uno::Any & _rValue) Line 147
  pcrlo.dll!pcr::OPropertyBrowserController::Commit(const rtl::OUString & rName, const com::sun::star::uno::Any & _rValue) Line 1326
  pcrlo.dll!pcr::OBrowserListBox::valueChanged(const com::sun::star::uno::Reference<com::sun::star::inspection::XPropertyControl> & _rxControl) Line 637
  pcrlo.dll!pcr::PropertyControlContext_Impl::impl_processEvent_throw(const comphelper::AnyEvent & _rEvent) Line 291
  pcrlo.dll!pcr::PropertyControlContext_Impl::processEvent(const comphelper::AnyEvent & _rEvent) Line 272
  comphelper.dll!comphelper::AsyncEventNotifierBase::execute() Line 138
  comphelper.dll!comphelper::AsyncEventNotifier::execute() Line 155
  salhelper3MSC.dll!salhelper::Thread::run() Line 40
  salhelper3MSC.dll!threadFunc(void * param) Line 190
  sal3.dll!oslWorkerWrapperFunction(void * pData) Line 67
  ucrtbased.dll!thread_start<unsigned int (__cdecl*)(void *),1>(void * const parameter) Line 97
  kernel32.dll!BaseThreadInitThunk()
  ntdll.dll!RtlUserThreadStart()

Change-Id: Ie36fe63b7de9cd04570a3ca625ff5ffa13806d91
Reviewed-on: https://gerrit.libreoffice.org/c/core/+/202298
Tested-by: Mike Kaganski <mike.kaganski@collabora.com>
Reviewed-by: Mike Kaganski <mike.kaganski@collabora.com>

toolkit/source/controls/unocontrol.cxx

@@ -1056,7 +1056,7 @@ void UnoControl::peerCreated()
 
 void UnoControl::createPeer( const Reference< XToolkit >& rxToolkit, const Reference< XWindowPeer >& rParentPeer )
 {
-    ::osl::ClearableMutexGuard aGuard( GetMutex() );
+    osl::ResettableMutexGuard aGuard(GetMutex());
     if ( !mxModel.is() )
     {
         throw RuntimeException(u"createPeer: no model!"_ustr, getXWeak());
@@ -1259,7 +1259,13 @@ void UnoControl::createPeer( const Reference< XToolkit >& rxToolkit, const Refer
     PrepareWindowDescriptor(aDescr);
 
     // create the peer
-    Reference<XWindowPeer> xTemp = xToolkit->createWindow( aDescr );
+    // tdf#170961: release mutex during createWindow, which may call into the main thread, while
+    // the main thread may need this mutex
+    Reference<XWindowPeer> xTemp;
+    {
+        osl::ResettableMutexGuardScopedReleaser aRelease(aGuard);
+        xTemp = xToolkit->createWindow(aDescr);
+    }
     mxVclWindowPeer.set(xTemp, UNO_QUERY);
     assert(mxVclWindowPeer);
 

toolkit/source/controls/unocontrolcontainer.cxx

@@ -714,7 +714,7 @@ void UnoControlContainer::removeTabController( const uno::Reference< awt::XTabCo
 // awt::XControl
 void UnoControlContainer::createPeer( const uno::Reference< awt::XToolkit >& rxToolkit, const uno::Reference< awt::XWindowPeer >& rParent )
 {
-    ::osl::Guard< ::osl::Mutex > aGuard( GetMutex() );
+    osl::ResettableMutexGuard aGuard(GetMutex());
 
     if( getPeer().is() )
         return;
@@ -724,7 +724,12 @@ void UnoControlContainer::createPeer( const uno::Reference< awt::XToolkit >& rxT
         UnoControl::setVisible( false );
 
     // Create a new peer
-    UnoControl::createPeer( rxToolkit, rParent );
+    // tdf#170961: release mutex around inherited createPeer, which may call to the main thread,
+    // while the main thread may need this mutex
+    {
+        osl::ResettableMutexGuardScopedReleaser aRelease(aGuard);
+        UnoControl::createPeer(rxToolkit, rParent);
+    }
 
     // Create all children's peers
     if ( !mbCreatingCompatiblePeer )