Enable Passthrough for VT Input Mode in ConPty (microsoft#4856)

This commit enables passthrough mode for VT Input Mode in ConPty. This
will be used to pass VT Input from Mouse Mode directly to the app on the
other side.

## References
microsoft#545 - VT Mouse Mode (Terminal)
microsoft#376 - VT Mouse Mode (ConPty)

## Detailed Description of the Pull Request / Additional comments

### ConHost
- Set the callback for the InputEngine.
- Retrieve `IsInVirtualTerminalInputMode` from the InputBuffer

### Adapter (Dispatch)
Retrieve `VTInputMode` setting from ConHost

### Parser
- Add a callback to passthrough unknown input sequences directly to the
  input queue.
- If we're in VTInputMode, use the callback

## Validation Steps Performed
Tests should still pass.

src/host/VtInputThread.cpp

@@ -43,7 +43,13 @@ VtInputThread::VtInputThread(_In_ wil::unique_hfile hPipe,
 
     auto engine = std::make_unique<InputStateMachineEngine>(std::move(dispatch), inheritCursor);
 
+    auto engineRef = engine.get();
+
     _pInputStateMachine = std::make_unique<StateMachine>(std::move(engine));
+
+    // we need this callback to be able to flush an unknown input sequence to the app
+    auto flushCallback = std::bind(&StateMachine::FlushToTerminal, _pInputStateMachine.get());
+    engineRef->SetFlushToInputQueueCallback(flushCallback);
 }
 
 // Method Description:

src/host/outputStream.cpp

@@ -891,3 +891,16 @@ bool ConhostInternalGetSet::PrivateScrollRegion(const SMALL_RECT scrollRect,
                                               destinationOrigin,
                                               standardFillAttrs));
 }
+
+// Routine Description:
+// - Checks if the InputBuffer is willing to accept VT Input directly
+//   PrivateIsVtInputEnabled is an internal-only "API" call that the vt commands can execute,
+//    but it is not represented as a function call on our public API surface.
+// Arguments:
+// - <none>
+// Return value:
+// - true if enabled (see IsInVirtualTerminalInputMode). false otherwise.
+bool ConhostInternalGetSet::PrivateIsVtInputEnabled() const
+{
+    return _io.GetActiveInputBuffer()->IsInVirtualTerminalInputMode();
+}

src/host/outputStream.hpp

@@ -166,6 +166,8 @@ class ConhostInternalGetSet final : public Microsoft::Console::VirtualTerminal::
                              const COORD destinationOrigin,
                              const bool standardFillAttrs) noexcept override;
 
+    bool PrivateIsVtInputEnabled() const override;
+
 private:
     Microsoft::Console::IIoProvider& _io;
 };

src/terminal/adapter/IInteractDispatch.hpp

@@ -39,5 +39,7 @@ namespace Microsoft::Console::VirtualTerminal
 
         virtual bool MoveCursor(const size_t row,
                                 const size_t col) = 0;
+
+        virtual bool IsVtInputEnabled() const = 0;
     };
 }

src/terminal/adapter/InteractDispatch.cpp

@@ -203,3 +203,14 @@ bool InteractDispatch::MoveCursor(const size_t row, const size_t col)
 
     return success;
 }
+
+// Routine Description:
+// - Checks if the InputBuffer is willing to accept VT Input directly
+// Arguments:
+// - <none>
+// Return value:
+// - true if enabled (see IsInVirtualTerminalInputMode). false otherwise.
+bool InteractDispatch::IsVtInputEnabled() const
+{
+    return _pConApi->PrivateIsVtInputEnabled();
+}

src/terminal/adapter/InteractDispatch.hpp

@@ -32,6 +32,8 @@ namespace Microsoft::Console::VirtualTerminal
                                 const std::basic_string_view<size_t> parameters) override; // DTTERM_WindowManipulation
         bool MoveCursor(const size_t row, const size_t col) override;
 
+        bool IsVtInputEnabled() const override;
+
     private:
         std::unique_ptr<ConGetSet> _pConApi;
     };

src/terminal/adapter/adaptDispatch.cpp

@@ -1039,7 +1039,18 @@ bool AdaptDispatch::ResetPrivateModes(const std::basic_string_view<DispatchTypes
 // - True if handled successfully. False otherwise.
 bool AdaptDispatch::SetKeypadMode(const bool fApplicationMode)
 {
-    return _pConApi->PrivateSetKeypadMode(fApplicationMode);
+    bool success = true;
+    success = _pConApi->PrivateSetKeypadMode(fApplicationMode);
+
+    // If we're a conpty, always return false
+    bool isPty = false;
+    _pConApi->IsConsolePty(isPty);
+    if (isPty)
+    {
+        return false;
+    }
+
+    return success;
 }
 
 // - DECCKM - Sets the cursor keys input mode to either Application mode or Normal mode (true, false respectively)
@@ -1049,7 +1060,18 @@ bool AdaptDispatch::SetKeypadMode(const bool fApplicationMode)
 // - True if handled successfully. False otherwise.
 bool AdaptDispatch::SetCursorKeysMode(const bool applicationMode)
 {
-    return _pConApi->PrivateSetCursorKeysMode(applicationMode);
+    bool success = true;
+    success = _pConApi->PrivateSetCursorKeysMode(applicationMode);
+
+    // If we're a conpty, always return false
+    bool isPty = false;
+    _pConApi->IsConsolePty(isPty);
+    if (isPty)
+    {
+        return false;
+    }
+
+    return success;
 }
 
 // - att610 - Enables or disables the cursor blinking.
@@ -1688,7 +1710,18 @@ bool AdaptDispatch::EnableDECCOLMSupport(const bool enabled) noexcept
 // True if handled successfully. False otherwise.
 bool AdaptDispatch::EnableVT200MouseMode(const bool enabled)
 {
-    return _pConApi->PrivateEnableVT200MouseMode(enabled);
+    bool success = true;
+    success = _pConApi->PrivateEnableVT200MouseMode(enabled);
+
+    // If we're a conpty, always return false
+    bool isPty = false;
+    _pConApi->IsConsolePty(isPty);
+    if (isPty)
+    {
+        return false;
+    }
+
+    return success;
 }
 
 //Routine Description:
@@ -1700,7 +1733,18 @@ bool AdaptDispatch::EnableVT200MouseMode(const bool enabled)
 // True if handled successfully. False otherwise.
 bool AdaptDispatch::EnableUTF8ExtendedMouseMode(const bool enabled)
 {
-    return _pConApi->PrivateEnableUTF8ExtendedMouseMode(enabled);
+    bool success = true;
+    success = _pConApi->PrivateEnableUTF8ExtendedMouseMode(enabled);
+
+    // If we're a conpty, always return false
+    bool isPty = false;
+    _pConApi->IsConsolePty(isPty);
+    if (isPty)
+    {
+        return false;
+    }
+
+    return success;
 }
 
 //Routine Description:
@@ -1712,7 +1756,18 @@ bool AdaptDispatch::EnableUTF8ExtendedMouseMode(const bool enabled)
 // True if handled successfully. False otherwise.
 bool AdaptDispatch::EnableSGRExtendedMouseMode(const bool enabled)
 {
-    return _pConApi->PrivateEnableSGRExtendedMouseMode(enabled);
+    bool success = true;
+    success = _pConApi->PrivateEnableSGRExtendedMouseMode(enabled);
+
+    // If we're a conpty, always return false
+    bool isPty = false;
+    _pConApi->IsConsolePty(isPty);
+    if (isPty)
+    {
+        return false;
+    }
+
+    return success;
 }
 
 //Routine Description:
@@ -1723,7 +1778,18 @@ bool AdaptDispatch::EnableSGRExtendedMouseMode(const bool enabled)
 // True if handled successfully. False otherwise.
 bool AdaptDispatch::EnableButtonEventMouseMode(const bool enabled)
 {
-    return _pConApi->PrivateEnableButtonEventMouseMode(enabled);
+    bool success = true;
+    success = _pConApi->PrivateEnableButtonEventMouseMode(enabled);
+
+    // If we're a conpty, always return false
+    bool isPty = false;
+    _pConApi->IsConsolePty(isPty);
+    if (isPty)
+    {
+        return false;
+    }
+
+    return success;
 }
 
 //Routine Description:
@@ -1735,7 +1801,18 @@ bool AdaptDispatch::EnableButtonEventMouseMode(const bool enabled)
 // True if handled successfully. False otherwise.
 bool AdaptDispatch::EnableAnyEventMouseMode(const bool enabled)
 {
-    return _pConApi->PrivateEnableAnyEventMouseMode(enabled);
+    bool success = true;
+    success = _pConApi->PrivateEnableAnyEventMouseMode(enabled);
+
+    // If we're a conpty, always return false
+    bool isPty = false;
+    _pConApi->IsConsolePty(isPty);
+    if (isPty)
+    {
+        return false;
+    }
+
+    return success;
 }
 
 //Routine Description:
@@ -1747,7 +1824,18 @@ bool AdaptDispatch::EnableAnyEventMouseMode(const bool enabled)
 // True if handled successfully. False otherwise.
 bool AdaptDispatch::EnableAlternateScroll(const bool enabled)
 {
-    return _pConApi->PrivateEnableAlternateScroll(enabled);
+    bool success = true;
+    success = _pConApi->PrivateEnableAlternateScroll(enabled);
+
+    // If we're a conpty, always return false
+    bool isPty = false;
+    _pConApi->IsConsolePty(isPty);
+    if (isPty)
+    {
+        return false;
+    }
+
+    return success;
 }
 
 //Routine Description:

src/terminal/adapter/conGetSet.hpp

@@ -35,6 +35,8 @@ namespace Microsoft::Console::VirtualTerminal
         virtual bool SetConsoleCursorPosition(const COORD position) = 0;
         virtual bool SetConsoleTextAttribute(const WORD attr) = 0;
 
+        virtual bool PrivateIsVtInputEnabled() const = 0;
+
         virtual bool PrivateSetLegacyAttributes(const WORD attr,
                                                 const bool foreground,
                                                 const bool background,

src/terminal/adapter/ut_adapter/adapterTest.cpp

@@ -214,6 +214,11 @@ class TestGetSet final : public ConGetSet
         return _setConsoleTextAttributeResult;
     }
 
+    bool PrivateIsVtInputEnabled() const override
+    {
+        return false;
+    }
+
     bool PrivateSetLegacyAttributes(const WORD attr, const bool foreground, const bool background, const bool meta) override
     {
         Log::Comment(L"PrivateSetLegacyAttributes MOCK called...");

src/terminal/parser/InputStateMachineEngine.cpp

@@ -92,7 +92,8 @@ InputStateMachineEngine::InputStateMachineEngine(std::unique_ptr<IInteractDispat
 
 InputStateMachineEngine::InputStateMachineEngine(std::unique_ptr<IInteractDispatch> pDispatch, const bool lookingForDSR) :
     _pDispatch(std::move(pDispatch)),
-    _lookingForDSR(lookingForDSR)
+    _lookingForDSR(lookingForDSR),
+    _pfnFlushToInputQueue(nullptr)
 {
     THROW_HR_IF_NULL(E_INVALIDARG, _pDispatch.get());
 }
@@ -256,6 +257,27 @@ bool InputStateMachineEngine::ActionPrintString(const std::wstring_view string)
 // - true iff we successfully dispatched the sequence.
 bool InputStateMachineEngine::ActionPassThroughString(const std::wstring_view string)
 {
+    if (_pDispatch->IsVtInputEnabled())
+    {
+        // Synthesize string into key events that we'll write to the buffer
+        // similar to TerminalInput::_SendInputSequence
+        if (!string.empty())
+        {
+            try
+            {
+                std::deque<std::unique_ptr<IInputEvent>> inputEvents;
+                for (const auto& wch : string)
+                {
+                    inputEvents.push_back(std::make_unique<KeyEvent>(true, 1ui16, 0ui16, 0ui16, wch, 0));
+                }
+                return _pDispatch->WriteInput(inputEvents);
+            }
+            catch (...)
+            {
+                LOG_HR(wil::ResultFromCaughtException());
+            }
+        }
+    }
     return ActionPrintString(string);
 }
 
@@ -271,6 +293,11 @@ bool InputStateMachineEngine::ActionPassThroughString(const std::wstring_view st
 bool InputStateMachineEngine::ActionEscDispatch(const wchar_t wch,
                                                 const std::basic_string_view<wchar_t> /*intermediates*/)
 {
+    if (_pDispatch->IsVtInputEnabled() && _pfnFlushToInputQueue)
+    {
+        return _pfnFlushToInputQueue();
+    }
+
     bool success = false;
 
     // 0x7f is DEL, which we treat effectively the same as a ctrl character.
@@ -309,6 +336,11 @@ bool InputStateMachineEngine::ActionCsiDispatch(const wchar_t wch,
                                                 const std::basic_string_view<wchar_t> intermediates,
                                                 const std::basic_string_view<size_t> parameters)
 {
+    if (_pDispatch->IsVtInputEnabled() && _pfnFlushToInputQueue)
+    {
+        return _pfnFlushToInputQueue();
+    }
+
     DWORD modifierState = 0;
     short vkey = 0;
     unsigned int function = 0;
@@ -440,6 +472,11 @@ bool InputStateMachineEngine::ActionCsiDispatch(const wchar_t wch,
 bool InputStateMachineEngine::ActionSs3Dispatch(const wchar_t wch,
                                                 const std::basic_string_view<size_t> /*parameters*/)
 {
+    if (_pDispatch->IsVtInputEnabled() && _pfnFlushToInputQueue)
+    {
+        return _pfnFlushToInputQueue();
+    }
+
     // Ss3 sequence keys aren't modified.
     // When F1-F4 *are* modified, they're sent as CSI sequences, not SS3's.
     const DWORD modifierState = 0;
@@ -1043,6 +1080,22 @@ bool InputStateMachineEngine::DispatchIntermediatesFromEscape() const noexcept
     return true;
 }
 
+// Method Description:
+// - Sets us up for vt input passthrough.
+//      We'll set a couple members, and if they aren't null, when we get a
+//      sequence we don't understand, we'll pass it along to the app
+//      instead of eating it ourselves.
+// Arguments:
+// - pfnFlushToInputQueue: This is a callback to the underlying state machine to
+//      trigger it to call ActionPassThroughString with whatever sequence it's
+//      currently processing.
+// Return Value:
+// - <none>
+void InputStateMachineEngine::SetFlushToInputQueueCallback(std::function<bool()> pfnFlushToInputQueue)
+{
+    _pfnFlushToInputQueue = pfnFlushToInputQueue;
+}
+
 // Method Description:
 // - Retrieves the type of window manipulation operation from the parameter pool
 //      stored during Param actions.

src/terminal/parser/InputStateMachineEngine.hpp

@@ -168,8 +168,11 @@ namespace Microsoft::Console::VirtualTerminal
         bool DispatchControlCharsFromEscape() const noexcept override;
         bool DispatchIntermediatesFromEscape() const noexcept override;
 
+        void SetFlushToInputQueueCallback(std::function<bool()> pfnFlushToInputQueue);
+
     private:
         const std::unique_ptr<IInteractDispatch> _pDispatch;
+        std::function<bool()> _pfnFlushToInputQueue;
         bool _lookingForDSR;
         DWORD _mouseButtonState = 0;
 
@@ -180,7 +183,6 @@ namespace Microsoft::Console::VirtualTerminal
 
         bool _IsModified(const size_t paramCount) noexcept;
         DWORD _GetModifier(const size_t parameter) noexcept;
-        DWORD _GetSGRModifier(const size_t parameter) noexcept;
 
         bool _UpdateSGRMouseButtonState(const wchar_t wch,
                                         const std::basic_string_view<size_t> parameters,

src/terminal/parser/ut_parser/InputEngineTest.cpp

@@ -310,6 +310,8 @@ class Microsoft::Console::VirtualTerminal::TestInteractDispatch final : public I
     virtual bool MoveCursor(const size_t row,
                             const size_t col) override;
 
+    virtual bool IsVtInputEnabled() const override;
+
 private:
     std::function<void(std::deque<std::unique_ptr<IInputEvent>>&)> _pfnWriteInputCallback;
     TestState* _testState; // non-ownership pointer
@@ -376,6 +378,11 @@ bool TestInteractDispatch::MoveCursor(const size_t row, const size_t col)
     return true;
 }
 
+bool TestInteractDispatch::IsVtInputEnabled() const
+{
+    return true;
+}
+
 void InputEngineTest::C0Test()
 {
     auto pfn = std::bind(&TestState::TestInputCallback, &testState, std::placeholders::_1);