Merge pull request #240 from pelmered/performance-optimizations-for-b…

…ackups-check

Performance optimizations for backups check

docs/available-checks/backups.md

@@ -86,3 +86,41 @@ Health::checks([
         ->atLeastSizeInMb(20),
 ]);
 ```
+
+### Check backups on external filesystems
+
+You can use the `onDisk` method to specify any disk you have configured in Laravel.
+This is useful when the backups are stored on an external filesystem.
+
+```php
+use Spatie\Health\Facades\Health;
+use Spatie\Health\Checks\Checks\BackupsCheck;
+
+Health::checks([
+    BackupsCheck::new()
+        ->onDisk('backups')
+        ->locatedAt('backups'),
+]);
+```
+
+Checking backup files on external filesystems can be slow if you have a lot of backup files.
+* You can use the `parseModifiedFormat` method to get the modified date of the file from the name instead of reaching out to the file and read its metadata. This strips out the file folder and file extension and uses the remaining string to parse the date with `Carbon::createFromFormat`.
+* You can also limit the size check to only the first and last backup files by using the `onlyCheckSizeOnFirstAndLast` method. Otherwise the check needs to reach out to all files and check the file sizes.
+
+These two things can speed up the check of ~200 files on an S3 bucket from about 30 seconds to about 1 second.
+
+```php
+use Spatie\Health\Facades\Health;
+use Spatie\Health\Checks\Checks\BackupsCheck;
+
+Health::checks([
+    BackupsCheck::new()
+        ->onDisk('backups')
+        ->parseModifiedFormat('Y-m-d_H-i-s'),
+        ->atLeastSizeInMb(20),
+        ->onlyCheckSizeOnFirstAndLast()
+]);
+```
+
+For files that contains more than just the date you can use something like parseModifiedFormat('\b\a\c\k\u\p_Ymd_His')
+which would parse a file with the name similar to `backup_20240101_120000.sql.zip`.

src/Checks/Checks/BackupsCheck.php

@@ -22,8 +22,12 @@ class BackupsCheck extends Check
 
     protected ?Carbon $oldestShouldHaveBeenMadeAfter = null;
 
+    protected ?string $parseModifiedUsing = null;
+
     protected int $minimumSizeInMegabytes = 0;
 
+    protected bool $onlyCheckSizeOnFirstAndLast = false;
+
     protected ?int $minimumNumberOfBackups = null;
 
     protected ?int $maximumNumberOfBackups = null;
@@ -35,13 +39,20 @@ public function locatedAt(string $globPath): self
         return $this;
     }
 
-    public function onDisk($disk)
+    public function onDisk(string $disk): static
     {
         $this->disk = Storage::disk($disk);
 
         return $this;
     }
 
+    public function parseModifiedFormat(string $parseModifiedFormat = 'Y-m-d_H-i-s'): self
+    {
+        $this->parseModifiedUsing = $parseModifiedFormat;
+
+        return $this;
+    }
+
     public function youngestBackShouldHaveBeenMadeBefore(Carbon $date): self
     {
         $this->youngestShouldHaveBeenMadeBefore = $date;
@@ -56,9 +67,17 @@ public function oldestBackShouldHaveBeenMadeAfter(Carbon $date): self
         return $this;
     }
 
-    public function atLeastSizeInMb(int $minimumSizeInMegabytes): self
+    public function atLeastSizeInMb(int $minimumSizeInMegabytes, bool $onlyCheckFirstAndLast = false): self
     {
         $this->minimumSizeInMegabytes = $minimumSizeInMegabytes;
+        $this->onlyCheckSizeOnFirstAndLast = $onlyCheckFirstAndLast;
+
+        return $this;
+    }
+
+    public function onlyCheckSizeOnFirstAndLast(bool $onlyCheckSizeOnFirstAndLast = true): self
+    {
+        $this->onlyCheckSizeOnFirstAndLast = $onlyCheckSizeOnFirstAndLast;
 
         return $this;
     }
@@ -73,80 +92,100 @@ public function numberOfBackups(?int $min = null, ?int $max = null): self
 
     public function run(): Result
     {
-        $files = collect($this->disk ? $files = $this->disk->files($this->locatedAt) : File::glob($this->locatedAt));
+        $eligibleBackups = $this->getBackupFiles();
 
-        if ($files->isEmpty()) {
-            return Result::make()->failed('No backups found');
-        }
+        $backupCount = $eligibleBackups->count();
 
-        $eligableBackups = $files
-            ->map(function (string $path) {
-                return new BackupFile($path, $this->disk);
-            })
-            ->filter(function (BackupFile $file) {
-                return $file->size() >= $this->minimumSizeInMegabytes * 1024 * 1024;
-            });
+        $result = Result::make()->meta([
+            'minimum_size' => $this->minimumSizeInMegabytes.'MB',
+            'backup_count' => $backupCount,
+        ]);
 
-        if ($eligableBackups->isEmpty()) {
-            return Result::make()->failed('No backups found that are large enough');
+        if ($backupCount === 0) {
+            return $result->failed('No backups found');
         }
 
-        if ($this->minimumNumberOfBackups) {
-            if ($eligableBackups->count() < $this->minimumNumberOfBackups) {
-                return Result::make()->failed('Not enough backups found');
-            }
+        if ($this->minimumNumberOfBackups && $backupCount < $this->minimumNumberOfBackups) {
+            return $result->failed('Not enough backups found');
         }
 
-        if ($this->maximumNumberOfBackups) {
-            if ($eligableBackups->count() > $this->maximumNumberOfBackups) {
-                return Result::make()->failed('Too many backups found');
-            }
+        if ($this->maximumNumberOfBackups && $backupCount > $this->maximumNumberOfBackups) {
+            return $result->failed('Too many backups found');
         }
 
-        if ($this->youngestShouldHaveBeenMadeBefore) {
-            if ($this->youngestBackupIsToolOld($eligableBackups)) {
-                return Result::make()
-                    ->failed('Youngest backup was too old');
-            }
+        $youngestBackup = $this->getYoungestBackup($eligibleBackups);
+        $oldestBackup = $this->getOldestBackup($eligibleBackups);
+
+        $result->appendMeta([
+            'youngest_backup' => $youngestBackup ? Carbon::createFromTimestamp($youngestBackup->lastModified())->toDateTimeString() : null,
+            'oldest_backup'   => $oldestBackup ? Carbon::createFromTimestamp($oldestBackup->lastModified())->toDateTimeString() : null,
+        ]);
+
+        if ($this->youngestBackupIsToolOld($youngestBackup)) {
+            return $result->failed('The youngest backup was too old');
         }
 
-        if ($this->oldestShouldHaveBeenMadeAfter) {
-            if ($this->oldestBackupIsTooYoung($eligableBackups)) {
-                return Result::make()
-                    ->failed('Oldest backup was too young');
-            }
+        if ($this->oldestBackupIsTooYoung($oldestBackup)) {
+            return $result->failed('The oldest backup was too young');
         }
 
-        return Result::make()->ok();
+        $backupsToCheckSizeOn = $this->onlyCheckSizeOnFirstAndLast
+            ? collect([$youngestBackup, $oldestBackup])
+            : $eligibleBackups;
+
+        if ($backupsToCheckSizeOn->filter(
+            fn(BackupFile $file) => $file->size() >= $this->minimumSizeInMegabytes * 1024 * 1024
+        )->isEmpty()) {
+            return $result->failed('Backups are not large enough');
+        }
+
+        return $result->ok();
     }
 
-    /**
-     * @param  Collection<SymfonyFile>  $backups
-     */
-    protected function youngestBackupIsToolOld(Collection $backups): bool
+    protected function getBackupFiles(): Collection
     {
-        /** @var SymfonyFile|null $youngestBackup */
-        $youngestBackup = $backups
+        return collect(
+            $this->disk
+                ? $this->disk->files($this->locatedAt)
+                : File::glob($this->locatedAt ?? '')
+        )->map(function (string $path) {
+            return new BackupFile($path, $this->disk, $this->parseModifiedUsing);
+        });
+    }
+
+    protected function getYoungestBackup(Collection $backups): ?BackupFile
+    {
+        return $backups
             ->sortByDesc(fn (BackupFile $file) => $file->lastModified())
             ->first();
+    }
+
+    protected function youngestBackupIsToolOld(?BackupFile $youngestBackup): bool
+    {
+        if ($this->youngestShouldHaveBeenMadeBefore === null) {
+            return false;
+        }
 
         $threshold = $this->youngestShouldHaveBeenMadeBefore->getTimestamp();
 
-        return $youngestBackup->lastModified() <= $threshold;
+        return !$youngestBackup || $youngestBackup->lastModified() <= $threshold;
     }
 
-    /**
-     * @param  Collection<SymfonyFile>  $backups
-     */
-    protected function oldestBackupIsTooYoung(Collection $backups): bool
+    protected function getOldestBackup(Collection $backups): ?BackupFile
     {
-        /** @var SymfonyFile|null $oldestBackup */
-        $oldestBackup = $backups
+        return $backups
             ->sortBy(fn (BackupFile $file) => $file->lastModified())
             ->first();
 
+    }
+    protected function oldestBackupIsTooYoung(?BackupFile $oldestBackup): bool
+    {
+        if ($this->oldestShouldHaveBeenMadeAfter === null) {
+            return false;
+        }
+
         $threshold = $this->oldestShouldHaveBeenMadeAfter->getTimestamp();
 
-        return $oldestBackup->lastModified() >= $threshold;
+        return !$oldestBackup || $oldestBackup->lastModified() >= $threshold;
     }
 }

src/Checks/Result.php

@@ -103,6 +103,13 @@ public function meta(array $meta): self
         return $this;
     }
 
+    public function appendMeta($meta): self
+    {
+        $this->meta = array_merge($this->meta, $meta);
+
+        return $this;
+    }
+
     public function endedAt(CarbonInterface $carbon): self
     {
         $this->ended_at = $carbon;

src/Support/BackupFile.php

@@ -2,7 +2,10 @@
 
 namespace Spatie\Health\Support;
 
+use Carbon\Exceptions\InvalidFormatException;
 use Illuminate\Contracts\Filesystem\Filesystem;
+use Illuminate\Support\Carbon;
+use Illuminate\Support\Str;
 use Symfony\Component\HttpFoundation\File\File as SymfonyFile;
 
 class BackupFile
@@ -12,6 +15,7 @@ class BackupFile
     public function __construct(
         protected string $path,
         protected ?Filesystem $disk = null,
+        protected ?string $parseModifiedUsing = null,
     ) {
         if (! $disk) {
             $this->file = new SymfonyFile($path);
@@ -28,8 +32,18 @@ public function size(): int
         return $this->file ? $this->file->getSize() : $this->disk->size($this->path);
     }
 
-    public function lastModified(): int
+    public function lastModified(): ?int
     {
+        if ($this->parseModifiedUsing) {
+            $filename = Str::of($this->path)->afterLast('/')->before('.');
+
+            try {
+                return (int) Carbon::createFromFormat($this->parseModifiedUsing, $filename)->timestamp;
+            } catch (InvalidFormatException $e) {
+                return null;
+            }
+        }
+
         return $this->file ? $this->file->getMTime() : $this->disk->lastModified($this->path);
     }
 }

tests/Checks/BackupsCheckTest.php

@@ -1,9 +1,11 @@
 <?php
 
+use Carbon\Carbon;
 use Illuminate\Support\Facades\Storage;
 use Spatie\Health\Checks\Checks\BackupsCheck;
 use Spatie\Health\Enums\Status;
 use Spatie\Health\Facades\Health;
+use Spatie\Health\Support\BackupFile;
 use Spatie\TemporaryDirectory\TemporaryDirectory;
 
 use function Spatie\PestPluginTestTime\testTime;
@@ -221,3 +223,68 @@
         ->run();
     expect($result)->status->toBe(Status::failed());
 });
+
+it('can parse modified time from file name', function ($format) {
+    Storage::fake('backups');
+
+    $now = now();
+    Storage::disk('backups')->put('backups/'.$now->format($format).'.zip', 'content');
+
+    $result1 = $this->backupsCheck
+        ->onDisk('backups')
+        ->locatedAt('backups')
+        ->parseModifiedFormat($format)
+        ->oldestBackShouldHaveBeenMadeAfter($now->subMinutes(5))
+        ->run();
+
+    testTime()->addMinutes(6);
+
+    $backupFile = new BackupFile('backups/'.$now->format($format).'.zip', Storage::disk('backups'), $format);
+
+    expect($backupFile->lastModified())->toBe($now->timestamp);
+
+    $result2 = $this->backupsCheck
+        ->onDisk('backups')
+        ->locatedAt('backups')
+        ->parseModifiedFormat($format)
+        ->oldestBackShouldHaveBeenMadeAfter(now()->subMinutes(5))
+        ->run();
+
+    expect($result1)->status->toBe(Status::failed())
+        ->and($result2)->status->toBe(Status::ok());
+
+    testTime()->addMinutes(2);
+
+    $result = $this->backupsCheck
+        ->locatedAt($this->temporaryDirectory->path('*.zip'))
+        ->oldestBackShouldHaveBeenMadeAfter(now()->subMinutes(5))
+        ->run();
+    expect($result)->status->toBe(Status::failed());
+})->with([
+    ['Y-m-d_H-i-s'],
+    ['Ymd_His'],
+    ['YmdHis'],
+    ['\B\a\c\k\u\p_Ymd_His'],
+]);
+
+it('can check the size of only the first and last backup files', function () {
+    $now = now()->startOfMinute();
+
+    addTestFile($this->temporaryDirectory->path('hey1.zip'), date: $now, sizeInMb: 5);
+    addTestFile($this->temporaryDirectory->path('hey2.zip'), date: $now->addMinutes(10), sizeInMb: 10);
+    addTestFile($this->temporaryDirectory->path('hey3.zip'), date: $now->addMinutes(20), sizeInMb: 5);
+
+    $result1 = $this->backupsCheck
+        ->locatedAt($this->temporaryDirectory->path('*.zip'))
+        ->atLeastSizeInMb(9)
+        ->run();
+
+    $result2 = $this->backupsCheck
+        ->locatedAt($this->temporaryDirectory->path('*.zip'))
+        ->atLeastSizeInMb(9)
+        ->onlyCheckSizeOnFirstAndLast()
+        ->run();
+
+    expect($result1)->status->toBe(Status::ok())
+        ->and($result2)->status->toBe(Status::failed());
+});