Enable the Terminal to tell ConPTY who the owner is (#12526)

## Window shenanigans, part the first:

This PR enables terminals to tell ConPTY what the owning window for the
pseudo window should be. This allows thigs like MessageBoxes created by
console applications to work. It also enables console apps to use
`GetAncestor(GetConsoleWindow(), GA_ROOT)`  to get directly at the HWND
of the Terminal (but _don't please_).

This is tested with our internal partners and seems to work for their
scenario. 

See #2988, #12799, #12515, #12570.

## PR Checklist
This is 1/3 of #2988.

.github/actions/spelling/allow/apis.txt

@@ -145,6 +145,7 @@ REGCLS
 RETURNCMD
 rfind
 roundf
+ROOTOWNER
 RSHIFT
 SACL
 schandle

src/cascadia/TerminalApp/TerminalPage.cpp

@@ -60,6 +60,12 @@ namespace winrt::TerminalApp::implementation
 
     // Method Description:
     // - implements the IInitializeWithWindow interface from shobjidl_core.
+    // - We're going to use this HWND as the owner for the ConPTY windows, via
+    //   ConptyConnection::ReparentWindow. We need this for applications that
+    //   call GetConsoleWindow, and attempt to open a MessageBox for the
+    //   console. By marking the conpty windows as owned by the Terminal HWND,
+    //   the message box will be owned by the Terminal window as well.
+    //   - see GH#2988
     HRESULT TerminalPage::Initialize(HWND hwnd)
     {
         _hostingHwnd = hwnd;
@@ -2409,6 +2415,11 @@ namespace winrt::TerminalApp::implementation
         // create here.
         // TermControl will copy the settings out of the settings passed to it.
         TermControl term{ settings.DefaultSettings(), settings.UnfocusedSettings(), connection };
+
+        if (_hostingHwnd.has_value())
+        {
+            term.OwningHwnd(reinterpret_cast<uint64_t>(*_hostingHwnd));
+        }
         return term;
     }
 

src/cascadia/TerminalConnection/ConptyConnection.cpp

@@ -310,6 +310,12 @@ namespace winrt::Microsoft::Terminal::TerminalConnection::implementation
             }
 
             THROW_IF_FAILED(_CreatePseudoConsoleAndPipes(dimensions, flags, &_inPipe, &_outPipe, &_hPC));
+
+            if (_initialParentHwnd != 0)
+            {
+                THROW_IF_FAILED(ConptyReparentPseudoConsole(_hPC.get(), reinterpret_cast<HWND>(_initialParentHwnd)));
+            }
+
             THROW_IF_FAILED(_LaunchAttachedClient());
         }
         // But if it was an inbound handoff... attempt to synchronize the size of it with what our connection
@@ -327,6 +333,7 @@ namespace winrt::Microsoft::Terminal::TerminalConnection::implementation
                 TelemetryPrivacyDataTag(PDT_ProductAndServicePerformance));
 
             THROW_IF_FAILED(ConptyResizePseudoConsole(_hPC.get(), dimensions));
+            THROW_IF_FAILED(ConptyReparentPseudoConsole(_hPC.get(), reinterpret_cast<HWND>(_initialParentHwnd)));
         }
 
         _startTime = std::chrono::high_resolution_clock::now();
@@ -482,6 +489,22 @@ namespace winrt::Microsoft::Terminal::TerminalConnection::implementation
         }
     }
 
+    void ConptyConnection::ReparentWindow(const uint64_t newParent)
+    {
+        // If we haven't started connecting at all, stash this HWND to use once we have started.
+        if (!_isStateAtOrBeyond(ConnectionState::Connecting))
+        {
+            _initialParentHwnd = newParent;
+        }
+        // Otherwise, just inform the conpty of the new owner window handle.
+        // This shouldn't be hittable until GH#5000 / GH#1256, when it's
+        // possible to reparent terminals to different windows.
+        else if (_isConnected())
+        {
+            THROW_IF_FAILED(ConptyReparentPseudoConsole(_hPC.get(), reinterpret_cast<HWND>(newParent)));
+        }
+    }
+
     void ConptyConnection::Close() noexcept
     try
     {

src/cascadia/TerminalConnection/ConptyConnection.h

@@ -35,6 +35,7 @@ namespace winrt::Microsoft::Terminal::TerminalConnection::implementation
         void Resize(uint32_t rows, uint32_t columns);
         void Close() noexcept;
         void ClearBuffer();
+        void ReparentWindow(const uint64_t newParent);
 
         winrt::guid Guid() const noexcept;
         winrt::hstring Commandline() const;
@@ -65,6 +66,7 @@ namespace winrt::Microsoft::Terminal::TerminalConnection::implementation
 
         uint32_t _initialRows{};
         uint32_t _initialCols{};
+        uint64_t _initialParentHwnd{ 0 };
         hstring _commandline{};
         hstring _startingDirectory{};
         hstring _startingTitle{};

src/cascadia/TerminalConnection/ConptyConnection.idl

@@ -13,6 +13,7 @@ namespace Microsoft.Terminal.TerminalConnection
         Guid Guid { get; };
         String Commandline { get; };
         void ClearBuffer();
+        void ReparentWindow(UInt64 newParent);
 
         static event NewConnectionHandler NewConnection;
         static void StartInboundListener();

src/cascadia/TerminalControl/ControlCore.cpp

@@ -262,6 +262,14 @@ namespace winrt::Microsoft::Terminal::Control::implementation
             const auto height = vp.Height();
             _connection.Resize(height, width);
 
+            if (_OwningHwnd != 0)
+            {
+                if (auto conpty{ _connection.try_as<TerminalConnection::ConptyConnection>() })
+                {
+                    conpty.ReparentWindow(_OwningHwnd);
+                }
+            }
+
             // Override the default width and height to match the size of the swapChainPanel
             _settings->InitialCols(width);
             _settings->InitialRows(height);

src/cascadia/TerminalControl/ControlCore.h

@@ -168,6 +168,11 @@ namespace winrt::Microsoft::Terminal::Control::implementation
 
         void AdjustOpacity(const double opacity, const bool relative);
 
+        // TODO:GH#1256 - When a tab can be torn out or otherwise reparented to
+        // another window, this value will need a custom setter, so that we can
+        // also update the connection.
+        WINRT_PROPERTY(uint64_t, OwningHwnd, 0);
+
         RUNTIME_SETTING(double, Opacity, _settings->Opacity());
         RUNTIME_SETTING(bool, UseAcrylic, _settings->UseAcrylic());
 

src/cascadia/TerminalControl/ICoreState.idl

@@ -24,5 +24,8 @@ namespace Microsoft.Terminal.Control
         Microsoft.Terminal.TerminalConnection.ConnectionState ConnectionState { get; };
 
         Microsoft.Terminal.Core.Scheme ColorScheme { get; set; };
+
+        UInt64 OwningHwnd;
+
     };
 }

src/cascadia/TerminalControl/TermControl.cpp

@@ -2794,4 +2794,14 @@ namespace winrt::Microsoft::Terminal::Control::implementation
         }
     }
 
+    void TermControl::OwningHwnd(uint64_t owner)
+    {
+        _core.OwningHwnd(owner);
+    }
+
+    uint64_t TermControl::OwningHwnd()
+    {
+        return _core.OwningHwnd();
+    }
+
 }

src/cascadia/TerminalControl/TermControl.h

@@ -59,6 +59,9 @@ namespace winrt::Microsoft::Terminal::Control::implementation
         bool BracketedPasteEnabled() const noexcept;
 
         double BackgroundOpacity() const;
+
+        uint64_t OwningHwnd();
+        void OwningHwnd(uint64_t owner);
 #pragma endregion
 
         void ScrollViewport(int viewTop);

src/cascadia/WindowsTerminal/IslandWindow.cpp

@@ -41,6 +41,11 @@ IslandWindow::~IslandWindow()
     _source.Close();
 }
 
+HWND IslandWindow::GetInteropHandle() const
+{
+    return _interopWindowHandle;
+}
+
 // Method Description:
 // - Create the actual window that we'll use for the application.
 // Arguments:

src/cascadia/WindowsTerminal/IslandWindow.h

@@ -23,6 +23,7 @@ class IslandWindow :
     virtual void MakeWindow() noexcept;
     void Close();
     virtual void OnSize(const UINT width, const UINT height);
+    HWND GetInteropHandle() const;
 
     [[nodiscard]] virtual LRESULT MessageHandler(UINT const message, WPARAM const wparam, LPARAM const lparam) noexcept override;
     void OnResize(const UINT width, const UINT height) override;

src/host/PtySignalInputThread.cpp

@@ -56,6 +56,10 @@ DWORD WINAPI PtySignalInputThread::StaticThreadProc(_In_ LPVOID lpParameter)
 //      (in and screen buffers) haven't yet been initialized.
 // - NOTE: Call under LockConsole() to ensure other threads have an opportunity
 //         to set early-work state.
+// - We need to do this specifically on the thread with the message pump. If the
+//   window is created on another thread, then the window won't have a message
+//   pump associated with it, and a DPI change in the connected terminal could
+//   end up HANGING THE CONPTY (for example).
 // Arguments:
 // - <none>
 // Return Value:
@@ -67,6 +71,12 @@ void PtySignalInputThread::ConnectConsole() noexcept
     {
         _DoResizeWindow(*_earlyResize);
     }
+
+    // If we were given a owner HWND, then manually start the pseudo window now.
+    if (_earlyReparent)
+    {
+        _DoSetWindowParent(*_earlyReparent);
+    }
 }
 
 // Method Description:
@@ -119,6 +129,28 @@ void PtySignalInputThread::ConnectConsole() noexcept
 
             break;
         }
+        case PtySignal::SetParent:
+        {
+            SetParentData reparentMessage = { 0 };
+            _GetData(&reparentMessage, sizeof(reparentMessage));
+
+            LockConsole();
+            auto Unlock = wil::scope_exit([&] { UnlockConsole(); });
+
+            // If the client app hasn't yet connected, stash the new owner.
+            // We'll later (PtySignalInputThread::ConnectConsole) use the value
+            // to set up the owner of the conpty window.
+            if (!_consoleConnected)
+            {
+                _earlyReparent = reparentMessage;
+            }
+            else
+            {
+                _DoSetWindowParent(reparentMessage);
+            }
+
+            break;
+        }
         default:
         {
             THROW_HR(E_UNEXPECTED);
@@ -147,6 +179,20 @@ void PtySignalInputThread::_DoClearBuffer()
     _pConApi->ClearBuffer();
 }
 
+// Method Description:
+// - Update the owner of the pseudo-window we're using for the conpty HWND. This
+//   allows to mark the pseudoconsole windows as "owner" by the terminal HWND
+//   that's actually hosting them.
+// - Refer to GH#2988
+// Arguments:
+// - data - Packet information containing owner HWND information
+// Return Value:
+// - <none>
+void PtySignalInputThread::_DoSetWindowParent(const SetParentData& data)
+{
+    _pConApi->ReparentWindow(data.handle);
+}
+
 // Method Description:
 // - Retrieves bytes from the file stream and exits or throws errors should the pipe state
 //   be compromised.

src/host/PtySignalInputThread.hpp

@@ -41,6 +41,7 @@ namespace Microsoft::Console
         enum class PtySignal : unsigned short
         {
             ClearBuffer = 2,
+            SetParent = 3,
             ResizeWindow = 8
         };
 
@@ -49,10 +50,15 @@ namespace Microsoft::Console
             unsigned short sx;
             unsigned short sy;
         };
+        struct SetParentData
+        {
+            uint64_t handle;
+        };
 
         [[nodiscard]] HRESULT _InputThread();
         bool _GetData(_Out_writes_bytes_(cbBuffer) void* const pBuffer, const DWORD cbBuffer);
         void _DoResizeWindow(const ResizeWindowData& data);
+        void _DoSetWindowParent(const SetParentData& data);
         void _DoClearBuffer();
         void _Shutdown();
 
@@ -62,5 +68,8 @@ namespace Microsoft::Console
         bool _consoleConnected;
         std::optional<ResizeWindowData> _earlyResize;
         std::unique_ptr<Microsoft::Console::VirtualTerminal::ConGetSet> _pConApi;
+
+    public:
+        std::optional<SetParentData> _earlyReparent;
     };
 }

src/host/VtIo.cpp

@@ -296,6 +296,11 @@ bool VtIo::IsUsingVt() const
 
     if (_pPtySignalInputThread)
     {
+        // IMPORTANT! Start the pseudo window on this thread. This thread has a
+        // message pump. If you DON'T, then a DPI change in the owning hwnd will
+        // cause us to get a dpi change as well, which we'll never deque and
+        // handle, effectively HANGING THE OWNER HWND.
+        //
         // Let the signal thread know that the console is connected
         _pPtySignalInputThread->ConnectConsole();
     }

src/host/outputStream.cpp

@@ -892,3 +892,17 @@ void ConhostInternalGetSet::UpdateSoftFont(const gsl::span<const uint16_t> bitPa
         pRender->UpdateSoftFont(bitPattern, cellSize, centeringHint);
     }
 }
+
+void ConhostInternalGetSet::ReparentWindow(const uint64_t handle)
+{
+    // This will initialize s_interactivityFactory for us. It will also
+    // conveniently return 0 when we're on OneCore.
+    //
+    // If the window hasn't been created yet, by some other call to
+    // LocatePseudoWindow, then this will also initialize the owner of the
+    // window.
+    if (const auto psuedoHwnd{ ServiceLocator::LocatePseudoWindow(reinterpret_cast<HWND>(handle)) })
+    {
+        LOG_LAST_ERROR_IF_NULL(::SetParent(psuedoHwnd, reinterpret_cast<HWND>(handle)));
+    }
+}

src/host/outputStream.hpp

@@ -116,6 +116,8 @@ class ConhostInternalGetSet final : public Microsoft::Console::VirtualTerminal::
                         const SIZE cellSize,
                         const size_t centeringHint) override;
 
+    void ReparentWindow(const uint64_t handle);
+
 private:
     void _modifyLines(const size_t count, const bool insert);
 

src/inc/conpty-static.h

@@ -25,6 +25,8 @@ HRESULT WINAPI ConptyResizePseudoConsole(HPCON hPC, COORD size);
 
 HRESULT WINAPI ConptyClearPseudoConsole(HPCON hPC);
 
+HRESULT WINAPI ConptyReparentPseudoConsole(HPCON hPC, HWND newParent);
+
 VOID WINAPI ConptyClosePseudoConsole(HPCON hPC);
 
 HRESULT WINAPI ConptyPackPseudoConsole(HANDLE hServerProcess, HANDLE hRef, HANDLE hSignal, HPCON* phPC);