Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Try adding delay between requests to avoid bot detection #813

Closed
wants to merge 2 commits into from
Closed

Try adding delay between requests to avoid bot detection #813

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
62ac988
Try adding delay between requests to avoid bot detection
Tyrrrz Sep 18, 2024
4ebfc11
Try longer delays
Tyrrrz Sep 18, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 12 additions & 30 deletions YoutubeExplode.Converter.Tests/GeneralSpecs.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@

namespace YoutubeExplode.Converter.Tests;

public class GeneralSpecs(ITestOutputHelper testOutput) : IAsyncLifetime
public class GeneralSpecs(ITestOutputHelper testOutput) : SpecsBase, IAsyncLifetime
{
public async Task InitializeAsync() => await FFmpeg.InitializeAsync();

Expand All @@ -22,13 +22,11 @@
public async Task I_can_download_a_video_as_a_single_mp4_file()
{
// Arrange
var youtube = new YoutubeClient();

using var dir = TempDir.Create();
var filePath = Path.Combine(dir.Path, "video.mp4");

// Act
await youtube.Videos.DownloadAsync("9bZkp7q19f0", filePath);
await Youtube.Videos.DownloadAsync("9bZkp7q19f0", filePath);

// Assert
MediaFormat.IsMp4File(filePath).Should().BeTrue();
Expand All @@ -38,13 +36,11 @@
public async Task I_can_download_a_video_as_a_single_webm_file()
{
// Arrange
var youtube = new YoutubeClient();

using var dir = TempDir.Create();
var filePath = Path.Combine(dir.Path, "video.webm");

// Act
await youtube.Videos.DownloadAsync("9bZkp7q19f0", filePath);
await Youtube.Videos.DownloadAsync("9bZkp7q19f0", filePath);

// Assert
MediaFormat.IsWebMFile(filePath).Should().BeTrue();
Expand All @@ -54,13 +50,11 @@
public async Task I_can_download_a_video_as_a_single_mp3_file()
{
// Arrange
var youtube = new YoutubeClient();

using var dir = TempDir.Create();
var filePath = Path.Combine(dir.Path, "video.mp3");

// Act
await youtube.Videos.DownloadAsync("9bZkp7q19f0", filePath);
await Youtube.Videos.DownloadAsync("9bZkp7q19f0", filePath);

// Assert
MediaFormat.IsMp3File(filePath).Should().BeTrue();
Expand All @@ -70,13 +64,11 @@
public async Task I_can_download_a_video_as_a_single_ogg_file()
{
// Arrange
var youtube = new YoutubeClient();

using var dir = TempDir.Create();
var filePath = Path.Combine(dir.Path, "video.ogg");

// Act
await youtube.Videos.DownloadAsync("9bZkp7q19f0", filePath);
await Youtube.Videos.DownloadAsync("9bZkp7q19f0", filePath);

// Assert
MediaFormat.IsOggFile(filePath).Should().BeTrue();
Expand All @@ -86,13 +78,11 @@
public async Task I_can_download_a_video_as_a_single_mp4_file_with_multiple_streams()
{
// Arrange
var youtube = new YoutubeClient();

using var dir = TempDir.Create();

Check failure on line 81 in YoutubeExplode.Converter.Tests/GeneralSpecs.cs

View workflow job for this annotation

GitHub Actions / main / test (macos-latest)

I can download a video as a single mp4 file with multiple streams 

YoutubeExplode.Exceptions.VideoUnplayableException : Video '9bZkp7q19f0' is unplayable. Reason: 'Sign in to confirm you’re not a bot'.
var filePath = Path.Combine(dir.Path, "video.mp4");

// Act
var manifest = await youtube.Videos.Streams.GetManifestAsync("9bZkp7q19f0");
var manifest = await Youtube.Videos.Streams.GetManifestAsync("9bZkp7q19f0");

var audioStreamInfos = manifest
.GetAudioOnlyStreams()
Expand All @@ -109,7 +99,7 @@
.Take(3)
.ToArray();

await youtube.Videos.DownloadAsync(
await Youtube.Videos.DownloadAsync(
videoStreamInfos.Concat<IStreamInfo>(audioStreamInfos).ToArray(),
new ConversionRequestBuilder(filePath).Build()
);
Expand All @@ -130,13 +120,11 @@
public async Task I_can_download_a_video_as_a_single_webm_file_with_multiple_streams()
{
// Arrange
var youtube = new YoutubeClient();

using var dir = TempDir.Create();
var filePath = Path.Combine(dir.Path, "video.webm");

// Act
var manifest = await youtube.Videos.Streams.GetManifestAsync("9bZkp7q19f0");
var manifest = await Youtube.Videos.Streams.GetManifestAsync("9bZkp7q19f0");

var audioStreamInfos = manifest
.GetAudioOnlyStreams()
Expand All @@ -153,7 +141,7 @@
.Take(3)
.ToArray();

await youtube.Videos.DownloadAsync(
await Youtube.Videos.DownloadAsync(
videoStreamInfos.Concat<IStreamInfo>(audioStreamInfos).ToArray(),
new ConversionRequestBuilder(filePath).Build()
);
Expand All @@ -174,13 +162,11 @@
public async Task I_can_download_a_video_using_custom_conversion_settings()
{
// Arrange
var youtube = new YoutubeClient();

using var dir = TempDir.Create();
var filePath = Path.Combine(dir.Path, "video.mp3");

// Act
await youtube.Videos.DownloadAsync(
await Youtube.Videos.DownloadAsync(
"9bZkp7q19f0",
filePath,
o =>
Expand All @@ -197,15 +183,13 @@
public async Task I_can_try_to_download_a_video_and_get_an_error_if_the_conversion_settings_are_invalid()
{
// Arrange
var youtube = new YoutubeClient();

using var dir = TempDir.Create();
var filePath = Path.Combine(dir.Path, "video.mp4");

// Act & assert
var ex = await Assert.ThrowsAnyAsync<Exception>(
async () =>
await youtube.Videos.DownloadAsync(
await Youtube.Videos.DownloadAsync(
"9bZkp7q19f0",
filePath,
o =>
Expand All @@ -224,15 +208,13 @@
public async Task I_can_download_a_video_while_tracking_progress()
{
// Arrange
var youtube = new YoutubeClient();

using var dir = TempDir.Create();

Check failure on line 211 in YoutubeExplode.Converter.Tests/GeneralSpecs.cs

View workflow job for this annotation

GitHub Actions / main / test (macos-latest)

I can download a video while tracking progress 

YoutubeExplode.Exceptions.VideoUnplayableException : Video '9bZkp7q19f0' is unplayable. Reason: 'Sign in to confirm you’re not a bot'.
var filePath = Path.Combine(dir.Path, "video.mp3");

var progress = new ProgressCollector<double>();

// Act
await youtube.Videos.DownloadAsync("9bZkp7q19f0", filePath, progress);
await Youtube.Videos.DownloadAsync("9bZkp7q19f0", filePath, progress);

// Assert
var progressValues = progress.GetValues();
Expand Down
14 changes: 14 additions & 0 deletions YoutubeExplode.Converter.Tests/SpecsBase.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
using System;
using System.Net.Http;
using YoutubeExplode.Converter.Tests.Utils;

namespace YoutubeExplode.Converter.Tests;

public abstract class SpecsBase
{
// Static handler to apply rate limiting to all tests
public static HttpMessageHandler HttpHandler { get; } =
new RateLimitedHttpHandler(TimeSpan.FromSeconds(1));

public YoutubeClient Youtube { get; } = new(new HttpClient(HttpHandler, false));
}
18 changes: 7 additions & 11 deletions YoutubeExplode.Converter.Tests/SubtitleSpecs.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@

namespace YoutubeExplode.Converter.Tests;

public class SubtitleSpecs : IAsyncLifetime
public class SubtitleSpecs : SpecsBase, IAsyncLifetime
{
public async Task InitializeAsync() => await FFmpeg.InitializeAsync();

Expand All @@ -19,24 +19,22 @@
public async Task I_can_download_a_video_as_a_single_mp4_file_with_subtitles()
{
// Arrange
var youtube = new YoutubeClient();

using var dir = TempDir.Create();

Check failure on line 22 in YoutubeExplode.Converter.Tests/SubtitleSpecs.cs

View workflow job for this annotation

GitHub Actions / main / test (macos-latest)

I can download a video as a single mp4 file with subtitles 

YoutubeExplode.Exceptions.VideoUnplayableException : Video 'NtQkz0aRDe8' is unplayable. Reason: 'Sign in to confirm you’re not a bot'.
var filePath = Path.Combine(dir.Path, "video.mp4");

var streamManifest = await youtube.Videos.Streams.GetManifestAsync("NtQkz0aRDe8");
var streamManifest = await Youtube.Videos.Streams.GetManifestAsync("NtQkz0aRDe8");
var streamInfos = streamManifest
.GetVideoStreams()
.Where(s => s.Container == Container.Mp4)
.OrderBy(s => s.Size)
.Take(1)
.ToArray();

var trackManifest = await youtube.Videos.ClosedCaptions.GetManifestAsync("NtQkz0aRDe8");
var trackManifest = await Youtube.Videos.ClosedCaptions.GetManifestAsync("NtQkz0aRDe8");
var trackInfos = trackManifest.Tracks;

// Act
await youtube.Videos.DownloadAsync(
await Youtube.Videos.DownloadAsync(
streamInfos,
trackInfos,
new ConversionRequestBuilder(filePath).Build()
Expand All @@ -58,24 +56,22 @@
public async Task I_can_download_a_video_as_a_single_webm_file_with_subtitles()
{
// Arrange
var youtube = new YoutubeClient();

using var dir = TempDir.Create();

Check failure on line 59 in YoutubeExplode.Converter.Tests/SubtitleSpecs.cs

View workflow job for this annotation

GitHub Actions / main / test (macos-latest)

I can download a video as a single webm file with subtitles 

YoutubeExplode.Exceptions.VideoUnplayableException : Video 'NtQkz0aRDe8' is unplayable. Reason: 'Sign in to confirm you’re not a bot'.
var filePath = Path.Combine(dir.Path, "video.webm");

var streamManifest = await youtube.Videos.Streams.GetManifestAsync("NtQkz0aRDe8");
var streamManifest = await Youtube.Videos.Streams.GetManifestAsync("NtQkz0aRDe8");
var streamInfos = streamManifest
.GetVideoStreams()
.Where(s => s.Container == Container.WebM)
.OrderBy(s => s.Size)
.Take(1)
.ToArray();

var trackManifest = await youtube.Videos.ClosedCaptions.GetManifestAsync("NtQkz0aRDe8");
var trackManifest = await Youtube.Videos.ClosedCaptions.GetManifestAsync("NtQkz0aRDe8");
var trackInfos = trackManifest.Tracks;

// Act
await youtube.Videos.DownloadAsync(
await Youtube.Videos.DownloadAsync(
streamInfos,
trackInfos,
new ConversionRequestBuilder(filePath).Build()
Expand Down
46 changes: 46 additions & 0 deletions YoutubeExplode.Converter.Tests/Utils/RateLimitedHttpHandler.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
using System;
using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;

namespace YoutubeExplode.Converter.Tests.Utils;

internal class RateLimitedHttpHandler(TimeSpan requestInterval) : HttpClientHandler
{
private readonly SemaphoreSlim _lock = new(1, 1);
private DateTimeOffset _lastRequestTimestamp = DateTimeOffset.MinValue;

protected override async Task<HttpResponseMessage> SendAsync(
HttpRequestMessage request,
CancellationToken cancellationToken
)
{
await _lock.WaitAsync(cancellationToken);

try
{
// Wait until the request interval has passed since the last request
var timeToWait = requestInterval - (DateTimeOffset.Now - _lastRequestTimestamp);
if (timeToWait > TimeSpan.Zero)
await Task.Delay(timeToWait, cancellationToken);

// Send the request
var response = await base.SendAsync(request, cancellationToken);

// Update the last request timestamp
_lastRequestTimestamp = DateTimeOffset.Now;

return response;
}
finally
{
_lock.Release();
}
}

protected override void Dispose(bool disposing)
{
base.Dispose(disposing);
_lock.Dispose();
}
}
2 changes: 1 addition & 1 deletion YoutubeExplode.Tests/ChannelHandleSpecs.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@

namespace YoutubeExplode.Tests;

public class ChannelHandleSpecs
public class ChannelHandleSpecs : SpecsBase
{
[Theory]
[InlineData("BeauMiles")]
Expand Down
2 changes: 1 addition & 1 deletion YoutubeExplode.Tests/ChannelIdSpecs.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@

namespace YoutubeExplode.Tests;

public class ChannelIdSpecs
public class ChannelIdSpecs : SpecsBase
{
[Theory]
[InlineData("UCEnBXANsKmyj2r9xVyKoDiQ")]
Expand Down
2 changes: 1 addition & 1 deletion YoutubeExplode.Tests/ChannelSlugSpecs.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@

namespace YoutubeExplode.Tests;

public class ChannelSlugSpecs
public class ChannelSlugSpecs : SpecsBase
{
[Theory]
[InlineData("Tyrrrz")]
Expand Down
Loading
Loading