obs-outputs: Adds AV1, HEVC via RTMP to YouTube.

Implements Enhanced RTMP spec at https://github.com/veovera/enhanced-rtmp,
specifically adding AV1 and HEVC support, and enabling streaming to
YouTube. Note that this feature is in beta at YouTube.
Additionally adds flag-guarded HDR metadata frame support, according to
the spec.

UI/data/locale/en-US.ini

@@ -402,6 +402,7 @@ Output.ConnectFail.Title="Failed to connect"
 Output.ConnectFail.BadPath="Invalid Path or Connection URL. Please check your settings to confirm that they are valid."
 Output.ConnectFail.ConnectFailed="Failed to connect to server"
 Output.ConnectFail.InvalidStream="Could not access the specified channel or stream key, please double-check your stream key. If it is correct, there may be a problem connecting to the server."
+Output.ConnectFail.HdrDisabled="This color configuration over RTMP is currently disabled for this codec"
 Output.ConnectFail.Error="An unexpected error occurred when trying to connect to the server. More information in the log file."
 Output.ConnectFail.Disconnected="Disconnected from server."
 

UI/window-basic-main.cpp

@@ -7252,6 +7252,10 @@ void OBSBasic::StreamingStop(int code, QString last_error)
 		encode_error = true;
 		break;
 
+	case OBS_OUTPUT_HDR_DISABLED:
+		errorDescription = Str("Output.ConnectFail.HdrDisabled");
+		break;
+
 	default:
 	case OBS_OUTPUT_ERROR:
 		use_last_error = true;

libobs/CMakeLists.txt

@@ -30,6 +30,7 @@ target_sources(
           obs-audio-controls.h
           obs-avc.c
           obs-avc.h
+          obs-bits.h
           obs-data.c
           obs-data.h
           obs-defs.h

libobs/obs-bits.h

@@ -0,0 +1,79 @@
+#pragma once
+
+#if defined(__GNUC__) || defined(__clang__)
+
+#define clz32(x) __builtin_clz(x)
+#define ctz32(x) __builtin_ctz(x)
+
+#elif defined(_MSC_VER) && _MSC_VER >= 1400
+
+static inline int clz32(unsigned long val)
+{
+	uint32_t zeros = 0;
+	_BitScanReverse(&zeros, val);
+	return 31 - zeros;
+}
+static inline int ctz32(unsigned long val)
+{
+	uint32_t zeros = 0;
+	_BitScanForward(&zeros, val);
+	return zeros;
+}
+
+#else
+
+static uint32_t popcnt(uint32_t x)
+{
+	x -= ((x >> 1) & 0x55555555);
+	x = (((x >> 2) & 0x33333333) + (x & 0x33333333));
+	x = (((x >> 4) + x) & 0x0f0f0f0f);
+	x += (x >> 8);
+	x += (x >> 16);
+	return x & 0x0000003f;
+}
+static uint32_t clz32(uint32_t x)
+{
+	x |= (x >> 1);
+	x |= (x >> 2);
+	x |= (x >> 4);
+	x |= (x >> 8);
+	x |= (x >> 16);
+	return 32 - popcnt(x);
+}
+static uint32_t ctz32(uint32_t x)
+{
+	return popcnt((x & -x) - 1);
+}
+
+#endif
+
+#ifdef MIN
+#undef MIN
+#endif
+static inline int MIN(int x, int y)
+{
+	return x < y ? x : y;
+}
+
+#ifdef MAX
+#undef MAX
+#endif
+static inline int MAX(int x, int y)
+{
+	return x > y ? x : y;
+}
+
+static inline uint32_t RB32(const uint8_t *ptr)
+{
+	return (ptr[0] << 24) + (ptr[1] << 16) + (ptr[2] << 8) + ptr[3];
+}
+
+static inline uint32_t av_bswap32(uint32_t x)
+{
+	return (((x << 8) & 0xff00) | ((x >> 8) & 0x00ff));
+}
+static inline uint64_t av_bswap64(uint64_t x)
+{
+	return ((uint64_t)av_bswap32((uint32_t)x) << 32) |
+	       av_bswap32((uint32_t)(x >> 32));
+}

libobs/obs-defs.h

@@ -42,6 +42,7 @@
 #define OBS_OUTPUT_UNSUPPORTED -6
 #define OBS_OUTPUT_NO_SPACE -7
 #define OBS_OUTPUT_ENCODE_ERROR -8
+#define OBS_OUTPUT_HDR_DISABLED -9
 
 #define OBS_VIDEO_SUCCESS 0
 #define OBS_VIDEO_FAIL -1

libobs/obs-hevc.c

@@ -21,50 +21,6 @@
 #include "obs-nal.h"
 #include "util/array-serializer.h"
 
-enum {
-	OBS_HEVC_NAL_TRAIL_N = 0,
-	OBS_HEVC_NAL_TRAIL_R = 1,
-	OBS_HEVC_NAL_TSA_N = 2,
-	OBS_HEVC_NAL_TSA_R = 3,
-	OBS_HEVC_NAL_STSA_N = 4,
-	OBS_HEVC_NAL_STSA_R = 5,
-	OBS_HEVC_NAL_RADL_N = 6,
-	OBS_HEVC_NAL_RADL_R = 7,
-	OBS_HEVC_NAL_RASL_N = 8,
-	OBS_HEVC_NAL_RASL_R = 9,
-	OBS_HEVC_NAL_VCL_N10 = 10,
-	OBS_HEVC_NAL_VCL_R11 = 11,
-	OBS_HEVC_NAL_VCL_N12 = 12,
-	OBS_HEVC_NAL_VCL_R13 = 13,
-	OBS_HEVC_NAL_VCL_N14 = 14,
-	OBS_HEVC_NAL_VCL_R15 = 15,
-	OBS_HEVC_NAL_BLA_W_LP = 16,
-	OBS_HEVC_NAL_BLA_W_RADL = 17,
-	OBS_HEVC_NAL_BLA_N_LP = 18,
-	OBS_HEVC_NAL_IDR_W_RADL = 19,
-	OBS_HEVC_NAL_IDR_N_LP = 20,
-	OBS_HEVC_NAL_CRA_NUT = 21,
-	OBS_HEVC_NAL_RSV_IRAP_VCL22 = 22,
-	OBS_HEVC_NAL_RSV_IRAP_VCL23 = 23,
-	OBS_HEVC_NAL_RSV_VCL24 = 24,
-	OBS_HEVC_NAL_RSV_VCL25 = 25,
-	OBS_HEVC_NAL_RSV_VCL26 = 26,
-	OBS_HEVC_NAL_RSV_VCL27 = 27,
-	OBS_HEVC_NAL_RSV_VCL28 = 28,
-	OBS_HEVC_NAL_RSV_VCL29 = 29,
-	OBS_HEVC_NAL_RSV_VCL30 = 30,
-	OBS_HEVC_NAL_RSV_VCL31 = 31,
-	OBS_HEVC_NAL_VPS = 32,
-	OBS_HEVC_NAL_SPS = 33,
-	OBS_HEVC_NAL_PPS = 34,
-	OBS_HEVC_NAL_AUD = 35,
-	OBS_HEVC_NAL_EOS_NUT = 36,
-	OBS_HEVC_NAL_EOB_NUT = 37,
-	OBS_HEVC_NAL_FD_NUT = 38,
-	OBS_HEVC_NAL_SEI_PREFIX = 39,
-	OBS_HEVC_NAL_SEI_SUFFIX = 40,
-};
-
 bool obs_hevc_keyframe(const uint8_t *data, size_t size)
 {
 	const uint8_t *nal_start, *nal_end;

libobs/obs-hevc.h

@@ -25,6 +25,50 @@ extern "C" {
 
 struct encoder_packet;
 
+enum {
+	OBS_HEVC_NAL_TRAIL_N = 0,
+	OBS_HEVC_NAL_TRAIL_R = 1,
+	OBS_HEVC_NAL_TSA_N = 2,
+	OBS_HEVC_NAL_TSA_R = 3,
+	OBS_HEVC_NAL_STSA_N = 4,
+	OBS_HEVC_NAL_STSA_R = 5,
+	OBS_HEVC_NAL_RADL_N = 6,
+	OBS_HEVC_NAL_RADL_R = 7,
+	OBS_HEVC_NAL_RASL_N = 8,
+	OBS_HEVC_NAL_RASL_R = 9,
+	OBS_HEVC_NAL_VCL_N10 = 10,
+	OBS_HEVC_NAL_VCL_R11 = 11,
+	OBS_HEVC_NAL_VCL_N12 = 12,
+	OBS_HEVC_NAL_VCL_R13 = 13,
+	OBS_HEVC_NAL_VCL_N14 = 14,
+	OBS_HEVC_NAL_VCL_R15 = 15,
+	OBS_HEVC_NAL_BLA_W_LP = 16,
+	OBS_HEVC_NAL_BLA_W_RADL = 17,
+	OBS_HEVC_NAL_BLA_N_LP = 18,
+	OBS_HEVC_NAL_IDR_W_RADL = 19,
+	OBS_HEVC_NAL_IDR_N_LP = 20,
+	OBS_HEVC_NAL_CRA_NUT = 21,
+	OBS_HEVC_NAL_RSV_IRAP_VCL22 = 22,
+	OBS_HEVC_NAL_RSV_IRAP_VCL23 = 23,
+	OBS_HEVC_NAL_RSV_VCL24 = 24,
+	OBS_HEVC_NAL_RSV_VCL25 = 25,
+	OBS_HEVC_NAL_RSV_VCL26 = 26,
+	OBS_HEVC_NAL_RSV_VCL27 = 27,
+	OBS_HEVC_NAL_RSV_VCL28 = 28,
+	OBS_HEVC_NAL_RSV_VCL29 = 29,
+	OBS_HEVC_NAL_RSV_VCL30 = 30,
+	OBS_HEVC_NAL_RSV_VCL31 = 31,
+	OBS_HEVC_NAL_VPS = 32,
+	OBS_HEVC_NAL_SPS = 33,
+	OBS_HEVC_NAL_PPS = 34,
+	OBS_HEVC_NAL_AUD = 35,
+	OBS_HEVC_NAL_EOS_NUT = 36,
+	OBS_HEVC_NAL_EOB_NUT = 37,
+	OBS_HEVC_NAL_FD_NUT = 38,
+	OBS_HEVC_NAL_SEI_PREFIX = 39,
+	OBS_HEVC_NAL_SEI_SUFFIX = 40,
+};
+
 EXPORT bool obs_hevc_keyframe(const uint8_t *data, size_t size);
 EXPORT void obs_parse_hevc_packet(struct encoder_packet *hevc_packet,
 				  const struct encoder_packet *src);

libobs/obs-nal.c

@@ -65,3 +65,15 @@ const uint8_t *obs_nal_find_startcode(const uint8_t *p, const uint8_t *end)
 		out--;
 	return out;
 }
+
+const uint8_t *obs_nal_find_next_startcode(const uint8_t *p, const uint8_t *end)
+{
+	if (p + 3 >= end)
+		return end;
+
+	for (end -= 3; p < end; p++) {
+		if (p[0] == 0 && p[1] == 0 && p[2] == 1)
+			return p;
+	}
+	return end + 3;
+}

libobs/obs-nal.h

@@ -32,6 +32,8 @@ enum {
 
 EXPORT const uint8_t *obs_nal_find_startcode(const uint8_t *p,
 					     const uint8_t *end);
+EXPORT const uint8_t *obs_nal_find_next_startcode(const uint8_t *p,
+						  const uint8_t *end);
 
 #ifdef __cplusplus
 }

plugins/obs-outputs/CMakeLists.txt

@@ -28,6 +28,8 @@ target_sources(
           rtmp-stream.c
           rtmp-stream.h
           rtmp-windows.c
+          rtmp-av1.c
+          rtmp-av1.h
           librtmp/amf.c
           librtmp/amf.h
           librtmp/bytes.h
@@ -45,6 +47,10 @@ target_sources(
           librtmp/rtmp.h
           librtmp/rtmp_sys.h)
 
+if(ENABLE_HEVC)
+  target_sources(libobs PRIVATE rtmp-hevc.c rtmp-hevc.h)
+endif()
+
 target_link_libraries(obs-outputs PRIVATE OBS::libobs)
 
 set_target_properties(obs-outputs PROPERTIES FOLDER "plugins" PREFIX "")