[SR] Support multi-touch gestures

sentry-android-replay/src/main/java/io/sentry/android/replay/DefaultReplayBreadcrumbConverter.kt

@@ -22,6 +22,8 @@ public open class DefaultReplayBreadcrumbConverter : ReplayBreadcrumbConverter {
         )
     }
 
+    private var lastConnectivityState: String? = null
+
     override fun convert(breadcrumb: Breadcrumb): RRWebEvent? {
         var breadcrumbMessage: String? = null
         var breadcrumbCategory: String? = null
@@ -79,6 +81,13 @@ public open class DefaultReplayBreadcrumbConverter : ReplayBreadcrumbConverter {
 
                     else -> return null
                 }
+
+                if (lastConnectivityState == breadcrumbData["state"]) {
+                    // debounce same state
+                    return null
+                }
+
+                lastConnectivityState = breadcrumbData["state"] as? String
             }
 
             breadcrumb.data["action"] == "BATTERY_CHANGED" -> {

sentry-android-replay/src/main/java/io/sentry/android/replay/capture/BaseCaptureStrategy.kt

@@ -60,7 +60,7 @@ internal abstract class BaseCaptureStrategy(
 
     protected val currentEvents = LinkedList<RRWebEvent>()
     private val currentEventsLock = Any()
-    private val currentPositions = mutableListOf<Position>()
+    private val currentPositions = LinkedHashMap<Int, ArrayList<Position>>(10)
     private var touchMoveBaseline = 0L
     private var lastCapturedMoveEvent = 0L
 
@@ -227,10 +227,10 @@ internal abstract class BaseCaptureStrategy(
     }
 
     override fun onTouchEvent(event: MotionEvent) {
-        val rrwebEvent = event.toRRWebIncrementalSnapshotEvent()
-        if (rrwebEvent != null) {
+        val rrwebEvents = event.toRRWebIncrementalSnapshotEvent()
+        if (rrwebEvents != null) {
             synchronized(currentEventsLock) {
-                currentEvents += rrwebEvent
+                currentEvents += rrwebEvents
             }
         }
     }
@@ -284,9 +284,9 @@ internal abstract class BaseCaptureStrategy(
         }
     }
 
-    private fun MotionEvent.toRRWebIncrementalSnapshotEvent(): RRWebIncrementalSnapshotEvent? {
+    private fun MotionEvent.toRRWebIncrementalSnapshotEvent(): List<RRWebIncrementalSnapshotEvent>? {
         val event = this
-        return when (val action = event.actionMasked) {
+        return when (event.actionMasked) {
             MotionEvent.ACTION_MOVE -> {
                 // we only throttle move events as those can be overwhelming
                 val now = dateProvider.currentTimeMillis
@@ -295,48 +295,109 @@ internal abstract class BaseCaptureStrategy(
                 }
                 lastCapturedMoveEvent = now
 
-                // idk why but rrweb does it like dis
-                if (touchMoveBaseline == 0L) {
-                    touchMoveBaseline = now
-                }
+                currentPositions.keys.forEach { pId ->
+                    val pIndex = event.findPointerIndex(pId)
+
+                    if (pIndex == -1) {
+                        // no data for this pointer
+                        return@forEach
+                    }
+
+                    // idk why but rrweb does it like dis
+                    if (touchMoveBaseline == 0L) {
+                        touchMoveBaseline = now
+                    }
 
-                currentPositions += Position().apply {
-                    x = event.x * recorderConfig.scaleFactorX
-                    y = event.y * recorderConfig.scaleFactorY
-                    id = 0 // html node id, but we don't have it, so hardcode to 0 to align with FE
-                    timeOffset = now - touchMoveBaseline
+                    currentPositions[pId]!! += Position().apply {
+                        x = event.getX(pIndex) * recorderConfig.scaleFactorX
+                        y = event.getY(pIndex) * recorderConfig.scaleFactorY
+                        id = 0 // html node id, but we don't have it, so hardcode to 0 to align with FE
+                        timeOffset = now - touchMoveBaseline
+                    }
                 }
 
                 val totalOffset = now - touchMoveBaseline
                 return if (totalOffset > CAPTURE_MOVE_EVENT_THRESHOLD) {
-                    RRWebInteractionMoveEvent().apply {
-                        timestamp = now
-                        positions = currentPositions.map { pos ->
-                            pos.timeOffset -= totalOffset
-                            pos
+                    val moveEvents = mutableListOf<RRWebInteractionMoveEvent>()
+                    for ((pointerId, positions) in currentPositions) {
+                        if (positions.isNotEmpty()) {
+                            moveEvents += RRWebInteractionMoveEvent().apply {
+                                this.timestamp = now
+                                this.positions = positions.map { pos ->
+                                    pos.timeOffset -= totalOffset
+                                    pos
+                                }
+                                this.pointerId = pointerId
+                            }
+                            currentPositions[pointerId]!!.clear()
                         }
-                    }.also {
-                        currentPositions.clear()
-                        touchMoveBaseline = 0L
                     }
+                    touchMoveBaseline = 0L
+                    moveEvents
                 } else {
                     null
                 }
             }
 
-            MotionEvent.ACTION_DOWN, MotionEvent.ACTION_UP, MotionEvent.ACTION_CANCEL -> {
-                RRWebInteractionEvent().apply {
-                    timestamp = dateProvider.currentTimeMillis
-                    x = event.x * recorderConfig.scaleFactorX
-                    y = event.y * recorderConfig.scaleFactorY
-                    id = 0 // html node id, but we don't have it, so hardcode to 0 to align with FE
-                    interactionType = when (action) {
-                        MotionEvent.ACTION_UP -> InteractionType.TouchEnd
-                        MotionEvent.ACTION_DOWN -> InteractionType.TouchStart
-                        MotionEvent.ACTION_CANCEL -> InteractionType.TouchCancel
-                        else -> InteractionType.TouchMove_Departed // should not happen
+            MotionEvent.ACTION_DOWN,
+            MotionEvent.ACTION_POINTER_DOWN -> {
+                val pId = event.getPointerId(event.actionIndex)
+                val pIndex = event.findPointerIndex(pId)
+
+                if (pIndex == -1) {
+                    // no data for this pointer
+                    return null
+                }
+
+                // new finger down - add a new pointer for tracking movement
+                currentPositions[pId] = ArrayList()
+                listOf(
+                    RRWebInteractionEvent().apply {
+                        timestamp = dateProvider.currentTimeMillis
+                        x = event.getX(pIndex) * recorderConfig.scaleFactorX
+                        y = event.getY(pIndex) * recorderConfig.scaleFactorY
+                        id = 0 // html node id, but we don't have it, so hardcode to 0 to align with FE
+                        pointerId = pId
+                        interactionType = InteractionType.TouchStart
                     }
+                )
+            }
+            MotionEvent.ACTION_UP,
+            MotionEvent.ACTION_POINTER_UP -> {
+                val pId = event.getPointerId(event.actionIndex)
+                val pIndex = event.findPointerIndex(pId)
+
+                if (pIndex == -1) {
+                    // no data for this pointer
+                    return null
                 }
+
+                // finger lift up - remove the pointer from tracking
+                currentPositions.remove(pId)
+                listOf(
+                    RRWebInteractionEvent().apply {
+                        timestamp = dateProvider.currentTimeMillis
+                        x = event.getX(pIndex) * recorderConfig.scaleFactorX
+                        y = event.getY(pIndex) * recorderConfig.scaleFactorY
+                        id = 0 // html node id, but we don't have it, so hardcode to 0 to align with FE
+                        pointerId = pId
+                        interactionType = InteractionType.TouchEnd
+                    }
+                )
+            }
+            MotionEvent.ACTION_CANCEL -> {
+                // gesture cancelled - remove all pointers from tracking
+                currentPositions.clear()
+                listOf(
+                    RRWebInteractionEvent().apply {
+                        timestamp = dateProvider.currentTimeMillis
+                        x = event.x * recorderConfig.scaleFactorX
+                        y = event.y * recorderConfig.scaleFactorY
+                        id = 0 // html node id, but we don't have it, so hardcode to 0 to align with FE
+                        pointerId = 0 // the pointerId is not used for TouchCancel, so just set it to 0
+                        interactionType = InteractionType.TouchCancel
+                    }
+                )
             }
 
             else -> null

sentry-android-replay/src/main/java/io/sentry/android/replay/capture/SessionCaptureStrategy.kt

@@ -129,7 +129,7 @@ internal class SessionCaptureStrategy(
         val now = dateProvider.currentTimeMillis
         val currentSegmentTimestamp = segmentTimestamp.get()
         val segmentId = currentSegment.get()
-        val duration = now - currentSegmentTimestamp.time
+        val duration = now - (currentSegmentTimestamp?.time ?: 0)
         val replayId = currentReplayId.get()
         val height = recorderConfig.recordingHeight
         val width = recorderConfig.recordingWidth

sentry/api/sentry.api

@@ -5210,6 +5210,7 @@ public final class io/sentry/rrweb/RRWebInteractionEvent : io/sentry/rrweb/RRWeb
 	public fun getDataUnknown ()Ljava/util/Map;
 	public fun getId ()I
 	public fun getInteractionType ()Lio/sentry/rrweb/RRWebInteractionEvent$InteractionType;
+	public fun getPointerId ()I
 	public fun getPointerType ()I
 	public fun getUnknown ()Ljava/util/Map;
 	public fun getX ()F
@@ -5218,6 +5219,7 @@ public final class io/sentry/rrweb/RRWebInteractionEvent : io/sentry/rrweb/RRWeb
 	public fun setDataUnknown (Ljava/util/Map;)V
 	public fun setId (I)V
 	public fun setInteractionType (Lio/sentry/rrweb/RRWebInteractionEvent$InteractionType;)V
+	public fun setPointerId (I)V
 	public fun setPointerType (I)V
 	public fun setUnknown (Ljava/util/Map;)V
 	public fun setX (F)V
@@ -5256,6 +5258,7 @@ public final class io/sentry/rrweb/RRWebInteractionEvent$InteractionType$Deseria
 public final class io/sentry/rrweb/RRWebInteractionEvent$JsonKeys {
 	public static final field DATA Ljava/lang/String;
 	public static final field ID Ljava/lang/String;
+	public static final field POINTER_ID Ljava/lang/String;
 	public static final field POINTER_TYPE Ljava/lang/String;
 	public static final field TYPE Ljava/lang/String;
 	public static final field X Ljava/lang/String;
@@ -5266,10 +5269,12 @@ public final class io/sentry/rrweb/RRWebInteractionEvent$JsonKeys {
 public final class io/sentry/rrweb/RRWebInteractionMoveEvent : io/sentry/rrweb/RRWebIncrementalSnapshotEvent, io/sentry/JsonSerializable, io/sentry/JsonUnknown {
 	public fun <init> ()V
 	public fun getDataUnknown ()Ljava/util/Map;
+	public fun getPointerId ()I
 	public fun getPositions ()Ljava/util/List;
 	public fun getUnknown ()Ljava/util/Map;
 	public fun serialize (Lio/sentry/ObjectWriter;Lio/sentry/ILogger;)V
 	public fun setDataUnknown (Ljava/util/Map;)V
+	public fun setPointerId (I)V
 	public fun setPositions (Ljava/util/List;)V
 	public fun setUnknown (Ljava/util/Map;)V
 }
@@ -5282,6 +5287,7 @@ public final class io/sentry/rrweb/RRWebInteractionMoveEvent$Deserializer : io/s
 
 public final class io/sentry/rrweb/RRWebInteractionMoveEvent$JsonKeys {
 	public static final field DATA Ljava/lang/String;
+	public static final field POINTER_ID Ljava/lang/String;
 	public static final field POSITIONS Ljava/lang/String;
 	public fun <init> ()V
 }

sentry/src/main/java/io/sentry/rrweb/RRWebInteractionEvent.java

@@ -57,6 +57,8 @@ public static final class Deserializer implements JsonDeserializer<InteractionTy
 
   private int pointerType = POINTER_TYPE_TOUCH;
 
+  private int pointerId;
+
   // to support unknown json attributes with nesting, we have to have unknown map for each of the
   // nested object in json: { ..., "data": { ... } }
   private @Nullable Map<String, Object> unknown;
@@ -107,6 +109,14 @@ public void setPointerType(final int pointerType) {
     this.pointerType = pointerType;
   }
 
+  public int getPointerId() {
+    return pointerId;
+  }
+
+  public void setPointerId(final int pointerId) {
+    this.pointerId = pointerId;
+  }
+
   @Nullable
   public Map<String, Object> getDataUnknown() {
     return dataUnknown;
@@ -136,6 +146,7 @@ public static final class JsonKeys {
     public static final String X = "x";
     public static final String Y = "y";
     public static final String POINTER_TYPE = "pointerType";
+    public static final String POINTER_ID = "pointerId";
   }
 
   @Override
@@ -163,6 +174,7 @@ private void serializeData(final @NotNull ObjectWriter writer, final @NotNull IL
     writer.name(JsonKeys.X).value(x);
     writer.name(JsonKeys.Y).value(y);
     writer.name(JsonKeys.POINTER_TYPE).value(pointerType);
+    writer.name(JsonKeys.POINTER_ID).value(pointerId);
     if (dataUnknown != null) {
       for (String key : dataUnknown.keySet()) {
         Object value = dataUnknown.get(key);
@@ -235,6 +247,9 @@ private void deserializeData(
           case JsonKeys.POINTER_TYPE:
             event.pointerType = reader.nextInt();
             break;
+          case JsonKeys.POINTER_ID:
+            event.pointerId = reader.nextInt();
+            break;
           default:
             if (!baseEventDeserializer.deserializeValue(event, nextName, reader, logger)) {
               if (dataUnknown == null) {

sentry/src/main/java/io/sentry/rrweb/RRWebInteractionMoveEvent.java

@@ -142,6 +142,7 @@ public static final class Deserializer implements JsonDeserializer<Position> {
     // endregion json
   }
 
+  private int pointerId;
   private @Nullable List<Position> positions;
   // to support unknown json attributes with nesting, we have to have unknown map for each of the
   // nested object in json: { ..., "data": { ... } }
@@ -180,12 +181,21 @@ public void setPositions(final @Nullable List<Position> positions) {
     this.positions = positions;
   }
 
+  public int getPointerId() {
+    return pointerId;
+  }
+
+  public void setPointerId(final int pointerId) {
+    this.pointerId = pointerId;
+  }
+
   // region json
 
   // rrweb uses camelCase hence the json keys are in camelCase here
   public static final class JsonKeys {
     public static final String DATA = "data";
     public static final String POSITIONS = "positions";
+    public static final String POINTER_ID = "pointerId";
   }
 
   @Override
@@ -211,6 +221,7 @@ private void serializeData(final @NotNull ObjectWriter writer, final @NotNull IL
     if (positions != null && !positions.isEmpty()) {
       writer.name(JsonKeys.POSITIONS).value(logger, positions);
     }
+    writer.name(JsonKeys.POINTER_ID).value(pointerId);
     if (dataUnknown != null) {
       for (String key : dataUnknown.keySet()) {
         Object value = dataUnknown.get(key);
@@ -271,6 +282,9 @@ private void deserializeData(
           case JsonKeys.POSITIONS:
             event.positions = reader.nextListOrNull(logger, new Position.Deserializer());
             break;
+          case JsonKeys.POINTER_ID:
+            event.pointerId = reader.nextInt();
+            break;
           default:
             if (!baseEventDeserializer.deserializeValue(event, nextName, reader, logger)) {
               if (dataUnknown == null) {

sentry/src/test/java/io/sentry/rrweb/RRWebInteractionEventSerializationTest.kt

@@ -17,6 +17,7 @@ class RRWebInteractionEventSerializationTest {
             x = 1.0f
             y = 2.0f
             interactionType = TouchStart
+            pointerId = 1
         }
     }
 

sentry/src/test/java/io/sentry/rrweb/RRWebInteractionMoveEventSerializationTest.kt

@@ -21,6 +21,7 @@ class RRWebInteractionMoveEventSerializationTest {
                     timeOffset = 100
                 }
             )
+            pointerId = 1
         }
     }
 

sentry/src/test/resources/json/replay_recording.json

@@ -1,2 +1,2 @@
 {"segment_id":0}
-[{"type":4,"timestamp":1234567890,"data":{"href":"https://sentry.io","height":1920,"width":1080}},{"type":5,"timestamp":12345678901,"data":{"tag":"video","payload":{"segmentId":0,"size":4000000,"duration":5000,"encoding":"h264","container":"mp4","height":1920,"width":1080,"frameCount":5,"frameRate":1,"frameRateType":"constant","left":100,"top":100}}},{"type":5,"timestamp":12345678901,"data":{"tag":"breadcrumb","payload":{"type":"default","timestamp":12345678.901,"category":"navigation","message":"message","level":"info","data":{"screen":"MainActivity","state":"resumed"}}}},{"type":5,"timestamp":12345678901,"data":{"tag":"performanceSpan","payload":{"op":"resource.http","description":"https://api.github.com/users/getsentry/repos","startTimestamp":12345678.901,"endTimestamp":12345679.901,"data":{"status_code":200,"method":"POST"}}}},{"type":3,"timestamp":12345678901,"data":{"source":2,"type":7,"id":1,"x":1.0,"y":2.0,"pointerType":2}},{"type":3,"timestamp":12345678901,"data":{"source":6,"positions":[{"id":1,"x":1.0,"y":2.0,"timeOffset":100}]}}]
+[{"type":4,"timestamp":1234567890,"data":{"href":"https://sentry.io","height":1920,"width":1080}},{"type":5,"timestamp":12345678901,"data":{"tag":"video","payload":{"segmentId":0,"size":4000000,"duration":5000,"encoding":"h264","container":"mp4","height":1920,"width":1080,"frameCount":5,"frameRate":1,"frameRateType":"constant","left":100,"top":100}}},{"type":5,"timestamp":12345678901,"data":{"tag":"breadcrumb","payload":{"type":"default","timestamp":12345678.901,"category":"navigation","message":"message","level":"info","data":{"screen":"MainActivity","state":"resumed"}}}},{"type":5,"timestamp":12345678901,"data":{"tag":"performanceSpan","payload":{"op":"resource.http","description":"https://api.github.com/users/getsentry/repos","startTimestamp":12345678.901,"endTimestamp":12345679.901,"data":{"status_code":200,"method":"POST"}}}},{"type":3,"timestamp":12345678901,"data":{"source":2,"type":7,"id":1,"x":1.0,"y":2.0,"pointerType":2,"pointerId":1}},{"type":3,"timestamp":12345678901,"data":{"source":6,"positions":[{"id":1,"x":1.0,"y":2.0,"timeOffset":100}],"pointerId":1}}]

sentry/src/test/resources/json/rrweb_interaction_event.json

@@ -7,6 +7,7 @@
     "id": 1,
     "x": 1.0,
     "y": 2.0,
-    "pointerType": 2
+    "pointerType": 2,
+    "pointerId": 1
   }
 }

sentry/src/test/resources/json/rrweb_interaction_move_event.json

@@ -10,6 +10,7 @@
         "y": 2.0,
         "timeOffset": 100
       }
-    ]
+    ],
+    "pointerId": 1
   }
 }