diff --git a/library/src/main/java/com/google/android/exoplayer2/ExoPlayerImplInternal.java b/library/src/main/java/com/google/android/exoplayer2/ExoPlayerImplInternal.java
index 85f365e51e7..56b862073a7 100644
--- a/library/src/main/java/com/google/android/exoplayer2/ExoPlayerImplInternal.java
+++ b/library/src/main/java/com/google/android/exoplayer2/ExoPlayerImplInternal.java
@@ -538,16 +538,23 @@ private long seekToPeriodPosition(int periodIndex, long periodPositionUs)
       periodIndex = C.INDEX_UNSET;
     }
 
-    // Clear the timeline, but keep the requested period if it is already prepared.
-    MediaPeriodHolder<T> periodHolder = playingPeriodHolder;
     MediaPeriodHolder<T> newPlayingPeriodHolder = null;
-    while (periodHolder != null) {
-      if (periodHolder.index == periodIndex && periodHolder.prepared) {
-        newPlayingPeriodHolder = periodHolder;
-      } else {
-        periodHolder.release();
+    if (playingPeriodHolder == null) {
+      // We're still waiting for the first period to be prepared.
+      if (loadingPeriodHolder != null) {
+        loadingPeriodHolder.release();
+      }
+    } else {
+      // Clear the timeline, but keep the requested period if it is already prepared.
+      MediaPeriodHolder<T> periodHolder = playingPeriodHolder;
+      while (periodHolder != null) {
+        if (periodHolder.index == periodIndex && periodHolder.prepared) {
+          newPlayingPeriodHolder = periodHolder;
+        } else {
+          periodHolder.release();
+        }
+        periodHolder = periodHolder.next;
       }
-      periodHolder = periodHolder.next;
     }
 
     // Disable all the renderers if the period is changing.
@@ -892,7 +899,8 @@ private void attemptRestart(Timeline newTimeline, Timeline oldTimeline,
     }
 
     // Release all loaded periods.
-    releasePeriodHoldersFrom(playingPeriodHolder);
+    releasePeriodHoldersFrom(playingPeriodHolder != null ? playingPeriodHolder
+        : loadingPeriodHolder);
     bufferAheadPeriodCount = 0;
     playingPeriodHolder = null;
     readingPeriodHolder = null;