console: consume APC, DM, DCS

JIRA: https://issues.redhat.com/browse/RHEL-24205

commit 3a2b2eb55681158d3e3ef464fbf47574cf0c517c
Author: nick black <dankamongmen@gmail.com>
Date:   Mon Aug 30 04:56:15 2021 -0400

    console: consume APC, DM, DCS

    The Linux console's VT102 implementation already consumes OSC
    ("Operating System Command") sequences, probably because that's how
    palette changes are transmitted.

    In addition to OSC, there are three other major clases of ANSI control
    strings: APC ("Application Program Command"), PM ("Privacy Message"),
    and DCS ("Device Control String").  They are handled similarly to OSC in
    terms of termination.

    Source: vt100.net

    Add three new enumerated states, one for each of these types.  All three
    are handled the same way right now--they simply consume input until
    terminated.  I hope to expand upon this firmament in the future.  Add
    new predicate ansi_control_string(), returning true for any of these
    states.  Replace explicit checks against ESosc with calls to this
    function.  Transition to these states appropriately from the escape
    initiation (ESesc) state.

    This was motivated by the following Notcurses bugs:

     dankamongmen/notcurses#2050
     dankamongmen/notcurses#1828
     dankamongmen/notcurses#2069

    where standard VT sequences are not consumed by the Linux console.  It's
    not necessary that the Linux console *support* these sequences, but it
    ought *consume* these well-specified classes of sequences.

    Tested by sending a variety of escape sequences to the console, and
    verifying that they still worked, or were now properly consumed.
    Verified that the escapes were properly terminated at a generic level.
    Verified that the Notcurses tools continued to show expected output on
    the Linux console, except now without escape bleedthrough.

    Link: https://lore.kernel.org/lkml/YSydL0q8iaUfkphg@schwarzgerat.orthanc/
    Signed-off-by: nick black <dankamongmen@gmail.com>
    Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
    Cc: Jiri Slaby <jirislaby@kernel.org>
    Cc: Tetsuo Handa <penguin-kernel@i-love.sakura.ne.jp>
    Cc: Daniel Vetter <daniel.vetter@ffwll.ch>
    Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

Signed-off-by: Andrew Halaney <ahalaney@redhat.com>

drivers/tty/vt/vt.c

@@ -2076,7 +2076,7 @@ static void restore_cur(struct vc_data *vc)
 
 enum { ESnormal, ESesc, ESsquare, ESgetpars, ESfunckey,
 	EShash, ESsetG0, ESsetG1, ESpercent, EScsiignore, ESnonstd,
-	ESpalette, ESosc };
+	ESpalette, ESosc, ESapc, ESpm, ESdcs };
 
 /* console_lock is held (except via vc_init()) */
 static void reset_terminal(struct vc_data *vc, int do_clear)
@@ -2150,20 +2150,28 @@ static void vc_setGx(struct vc_data *vc, unsigned int which, int c)
 		vc->vc_translate = set_translate(*charset, vc);
 }
 
+/* is this state an ANSI control string? */
+static bool ansi_control_string(unsigned int state)
+{
+	if (state == ESosc || state == ESapc || state == ESpm || state == ESdcs)
+		return true;
+	return false;
+}
+
 /* console_lock is held */
 static void do_con_trol(struct tty_struct *tty, struct vc_data *vc, int c)
 {
 	/*
 	 *  Control characters can be used in the _middle_
-	 *  of an escape sequence.
+	 *  of an escape sequence, aside from ANSI control strings.
 	 */
-	if (vc->vc_state == ESosc && c>=8 && c<=13) /* ... except for OSC */
+	if (ansi_control_string(vc->vc_state) && c >= 8 && c <= 13)
 		return;
 	switch (c) {
 	case 0:
 		return;
 	case 7:
-		if (vc->vc_state == ESosc)
+		if (ansi_control_string(vc->vc_state))
 			vc->vc_state = ESnormal;
 		else if (vc->vc_bell_duration)
 			kd_mksound(vc->vc_bell_pitch, vc->vc_bell_duration);
@@ -2224,6 +2232,12 @@ static void do_con_trol(struct tty_struct *tty, struct vc_data *vc, int c)
 		case ']':
 			vc->vc_state = ESnonstd;
 			return;
+		case '_':
+			vc->vc_state = ESapc;
+			return;
+		case '^':
+			vc->vc_state = ESpm;
+			return;
 		case '%':
 			vc->vc_state = ESpercent;
 			return;
@@ -2241,6 +2255,9 @@ static void do_con_trol(struct tty_struct *tty, struct vc_data *vc, int c)
 			if (vc->state.x < VC_TABSTOPS_COUNT)
 				set_bit(vc->state.x, vc->vc_tab_stop);
 			return;
+		case 'P':
+			vc->vc_state = ESdcs;
+			return;
 		case 'Z':
 			respond_ID(tty);
 			return;
@@ -2537,8 +2554,14 @@ static void do_con_trol(struct tty_struct *tty, struct vc_data *vc, int c)
 		vc_setGx(vc, 1, c);
 		vc->vc_state = ESnormal;
 		return;
+	case ESapc:
+		return;
 	case ESosc:
 		return;
+	case ESpm:
+		return;
+	case ESdcs:
+		return;
 	default:
 		vc->vc_state = ESnormal;
 	}