From 2a8c6e4c3832fc3d8eed79831c8275b232ad8e4a Mon Sep 17 00:00:00 2001 From: Marco Szulik Date: Tue, 23 Apr 2024 10:56:49 +0200 Subject: [PATCH 01/21] add an api url to the users table at which the client can receive notifications of all sorts --- app/Classes/MediaHandler/ImageHandler.php | 3 +- app/Classes/MediaHandler/VideoHandler.php | 17 +++----- app/Classes/Transcode.php | 10 +++-- app/Console/Commands/CreateUser.php | 11 ++++- app/Enums/ClientNotification.php | 9 ++++ app/Enums/ResponseState.php | 1 - .../Controllers/V1/UploadSlotController.php | 21 ++++++---- app/Http/Controllers/V1/VersionController.php | 5 ++- app/Http/Requests/V1/SetVersionRequest.php | 4 -- ...dSlotRequest.php => UploadSlotRequest.php} | 4 +- .../Requests/V1/VideoUploadSlotRequest.php | 33 --------------- app/Interfaces/MediaHandlerInterface.php | 3 +- app/Interfaces/TranscodeInterface.php | 3 +- app/Jobs/TranscodeVideo.php | 6 +-- app/Models/UploadSlot.php | 5 +-- app/Models/User.php | 3 +- app/Models/Version.php | 41 ++++++++++++------- ...4_23_095200_add_api_url_to_users_table.php | 24 +++++++++++ lang/en/responses.php | 1 - 19 files changed, 107 insertions(+), 97 deletions(-) create mode 100644 app/Enums/ClientNotification.php rename app/Http/Requests/V1/{ImageUploadSlotRequest.php => UploadSlotRequest.php} (84%) delete mode 100644 app/Http/Requests/V1/VideoUploadSlotRequest.php create mode 100644 database/migrations/2024_04_23_095200_add_api_url_to_users_table.php diff --git a/app/Classes/MediaHandler/ImageHandler.php b/app/Classes/MediaHandler/ImageHandler.php index 04c1fd4c..1a5a8afa 100644 --- a/app/Classes/MediaHandler/ImageHandler.php +++ b/app/Classes/MediaHandler/ImageHandler.php @@ -74,10 +74,9 @@ public function invalidateCdnCache(string $basePath): bool * @param Version $version * @param int $oldVersionNumber * @param bool $wasProcessed - * @param string $callbackUrl * @return array */ - public function setVersion(User $user, Version $version, int $oldVersionNumber, bool $wasProcessed, string $callbackUrl): array + public function setVersion(User $user, Version $version, int $oldVersionNumber, bool $wasProcessed): array { // Token and valid_until will be set in the 'saving' event. // By creating an upload slot, a currently active upload will be canceled. diff --git a/app/Classes/MediaHandler/VideoHandler.php b/app/Classes/MediaHandler/VideoHandler.php index c5f13137..7fa6e50d 100644 --- a/app/Classes/MediaHandler/VideoHandler.php +++ b/app/Classes/MediaHandler/VideoHandler.php @@ -61,21 +61,16 @@ public function invalidateCdnCache(string $basePath): bool * @param Version $version * @param int $oldVersionNumber * @param bool $wasProcessed - * @param string $callbackUrl * @return array */ - public function setVersion(User $user, Version $version, int $oldVersionNumber, bool $wasProcessed, string $callbackUrl): array + public function setVersion(User $user, Version $version, int $oldVersionNumber, bool $wasProcessed): array { - if ($callbackUrl) { - // Token and valid_until will be set in the 'saving' event. - // By creating an upload slot, currently active uploading or transcoding will be canceled. - $uploadSlot = $user->UploadSlots()->withoutGlobalScopes()->updateOrCreate(['identifier' => $version->Media->identifier], ['callback_url' => $callbackUrl, 'media_type' => MediaType::VIDEO]); + // Token and valid_until will be set in the 'saving' event. + // By creating an upload slot, currently active uploading or transcoding will be canceled. + $uploadSlot = $user->UploadSlots()->withoutGlobalScopes()->updateOrCreate(['identifier' => $version->Media->identifier], ['media_type' => MediaType::VIDEO]); - $success = Transcode::createJobForVersionUpdate($version, $uploadSlot, $oldVersionNumber, $wasProcessed); - $responseState = $success ? ResponseState::VIDEO_VERSION_SET : ResponseState::TRANSCODING_JOB_DISPATCH_FAILED; - } else { - $responseState = ResponseState::NO_CALLBACK_URL_PROVIDED; - } + $success = Transcode::createJobForVersionUpdate($version, $uploadSlot, $oldVersionNumber, $wasProcessed); + $responseState = $success ? ResponseState::VIDEO_VERSION_SET : ResponseState::TRANSCODING_JOB_DISPATCH_FAILED; return [ $responseState, diff --git a/app/Classes/Transcode.php b/app/Classes/Transcode.php index 87a06309..540f6a0e 100644 --- a/app/Classes/Transcode.php +++ b/app/Classes/Transcode.php @@ -2,6 +2,7 @@ namespace App\Classes; +use App\Enums\ClientNotification; use App\Enums\MediaType; use App\Enums\ResponseState; use App\Helpers\SodiumHelper; @@ -64,14 +65,13 @@ public function createJobForVersionUpdate(Version $version, UploadSlot $uploadSl * Inform client package about the transcoding result. * * @param ResponseState $responseState - * @param string $callbackUrl * @param string $uploadToken * @param Media $media * @param int $versionNumber * * @return void */ - public function callback(ResponseState $responseState, string $callbackUrl, string $uploadToken, Media $media, int $versionNumber): void + public function callback(ResponseState $responseState, string $uploadToken, Media $media, int $versionNumber): void { $response = [ 'state' => $responseState->getState()->value, @@ -79,11 +79,13 @@ public function callback(ResponseState $responseState, string $callbackUrl, stri 'identifier' => $media->identifier, 'version' => $versionNumber, 'upload_token' => $uploadToken, - 'public_path' => implode(DIRECTORY_SEPARATOR, array_filter([MediaType::VIDEO->prefix(), $media->baseDirectory()])) + 'public_path' => implode(DIRECTORY_SEPARATOR, array_filter([MediaType::VIDEO->prefix(), $media->baseDirectory()])), + 'hash' => Version::whereNumber($versionNumber)->first()?->hash, + 'type' => ClientNotification::VIDEO_TRANSCODING, ]; $signedResponse = SodiumHelper::sign(json_encode($response)); - Http::post($callbackUrl, ['signed_response' => $signedResponse]); + Http::post($media->User->api_url, ['signed_response' => $signedResponse]); } } diff --git a/app/Console/Commands/CreateUser.php b/app/Console/Commands/CreateUser.php index 1a0b38fe..c1813c5b 100644 --- a/app/Console/Commands/CreateUser.php +++ b/app/Console/Commands/CreateUser.php @@ -16,7 +16,8 @@ class CreateUser extends Command */ protected $signature = 'create:user {name : The name of the user.} - {email : The E-Mail of the user.}'; + {email : The E-Mail of the user.} + {api_url : The URL at which the client can receive notifications.}'; /** * The console command description. @@ -34,6 +35,7 @@ public function handle(): int { $name = $this->argument('name'); $email = $this->argument('email'); + $apiUrl = $this->argument('api_url'); if (!filter_var($email, FILTER_VALIDATE_EMAIL)) { $this->error('The provided email is not valid!'); @@ -56,13 +58,18 @@ public function handle(): int return Command::INVALID; } + if (!filter_var($apiUrl, FILTER_VALIDATE_URL, FILTER_FLAG_PATH_REQUIRED)) { + $this->error(sprintf('The API URL must be a valid URL and include a path.')); + return Command::INVALID; + } + /* * Laravel passwords are usually not nullable, so we will need to set something when creating the user. * Since we do not want to create a Password for the user, but need to store something secure, * we will just generate a string of random bytes. * This needs to be encoded to base64 because null bytes are not accepted anymore (PHP 8.3). */ - $user = User::create(['name' => $name, 'email' => $email, 'password' => Hash::make(base64_encode(random_bytes(300)))]); + $user = User::create(['name' => $name, 'email' => $email, 'api_url' => $apiUrl, 'password' => Hash::make(base64_encode(random_bytes(300)))]); $this->info(sprintf('Successfully created new user %s: %s (%s)', $user->getKey(), $user->name, $user->email)); $this->newLine(); diff --git a/app/Enums/ClientNotification.php b/app/Enums/ClientNotification.php new file mode 100644 index 00000000..2924f29d --- /dev/null +++ b/app/Enums/ClientNotification.php @@ -0,0 +1,9 @@ +updateOrCreateUploadSlot($request->user(), $request->merge(['media_type' => MediaType::IMAGE->value])->all()); + return $this->reserveUploadSlot($request, MediaType::IMAGE); } /** * Handle the incoming request. * - * @param VideoUploadSlotRequest $request + * @param UploadSlotRequest $request * * @return JsonResponse */ - public function reserveVideoUploadSlot(VideoUploadSlotRequest $request): JsonResponse + public function reserveVideoUploadSlot(UploadSlotRequest $request): JsonResponse { - return $this->updateOrCreateUploadSlot($request->user(), $request->merge(['media_type' => MediaType::VIDEO->value])->all()); + return $this->reserveUploadSlot($request, MediaType::VIDEO); + } + + protected function reserveUploadSlot(UploadSlotRequest $request, MediaType $mediaType): JsonResponse + { + return $this->updateOrCreateUploadSlot($request->user(), $request->merge(['media_type' => $mediaType->value])->all()); } /** @@ -118,7 +124,8 @@ protected function saveFile(UploadedFile $uploadedFile, UploadSlot $uploadSlot, 'version' => $versionNumber, // Base path is only passed for images since the video is not available at this path yet. 'public_path' => $type->isInstantlyAvailable() ? implode(DIRECTORY_SEPARATOR, array_filter([$type->prefix(), $basePath])) : null, - 'upload_token' => $uploadSlot->token + 'upload_token' => $uploadSlot->token, + 'hash' => $type->isInstantlyAvailable() ? $version?->hash : null, ], 201); } diff --git a/app/Http/Controllers/V1/VersionController.php b/app/Http/Controllers/V1/VersionController.php index 4e4f8ec3..9aaa3244 100644 --- a/app/Http/Controllers/V1/VersionController.php +++ b/app/Http/Controllers/V1/VersionController.php @@ -51,7 +51,7 @@ public function setVersion(SetVersionRequest $request, Media $media, Version $ve $version->update(['number' => $newVersionNumber, 'processed' => 0]); - [$responseState, $uploadToken] = $media->type->handler()->setVersion($user, $version, $oldVersionNumber, $wasProcessed, $request->get('callback_url')); + [$responseState, $uploadToken] = $media->type->handler()->setVersion($user, $version, $oldVersionNumber, $wasProcessed); return response()->json([ 'state' => $responseState->getState()->value, @@ -62,7 +62,8 @@ public function setVersion(SetVersionRequest $request, Media $media, Version $ve 'public_path' => $media->type->isInstantlyAvailable() ? implode(DIRECTORY_SEPARATOR, array_filter([$media->type->prefix(), $media->baseDirectory()])) : null, - 'upload_token' => $uploadToken + 'upload_token' => $uploadToken, + 'hash' => $media->type->isInstantlyAvailable() ? $version->hash : null, ]); } diff --git a/app/Http/Requests/V1/SetVersionRequest.php b/app/Http/Requests/V1/SetVersionRequest.php index c433e2ba..43a26559 100644 --- a/app/Http/Requests/V1/SetVersionRequest.php +++ b/app/Http/Requests/V1/SetVersionRequest.php @@ -23,9 +23,5 @@ public function authorize(): bool */ public function rules(): array { - // Nullable because this information is only used for videos. Validation is happening inside the VersionController. - return [ - 'callback_url' => ['nullable', 'string', 'url'] - ]; } } diff --git a/app/Http/Requests/V1/ImageUploadSlotRequest.php b/app/Http/Requests/V1/UploadSlotRequest.php similarity index 84% rename from app/Http/Requests/V1/ImageUploadSlotRequest.php rename to app/Http/Requests/V1/UploadSlotRequest.php index b3f8e2a0..18eb976d 100644 --- a/app/Http/Requests/V1/ImageUploadSlotRequest.php +++ b/app/Http/Requests/V1/UploadSlotRequest.php @@ -5,7 +5,7 @@ use App\Enums\ValidationRegex; use Illuminate\Foundation\Http\FormRequest; -class ImageUploadSlotRequest extends FormRequest +class UploadSlotRequest extends FormRequest { /** * Determine if the user is authorized to make this request. @@ -14,7 +14,7 @@ class ImageUploadSlotRequest extends FormRequest */ public function authorize(): bool { - return $this->user()->tokenCan('transmorpher:reserve-image-upload-slot'); + return $this->user()->tokenCan('transmorpher:reserve-upload-slot'); } /** diff --git a/app/Http/Requests/V1/VideoUploadSlotRequest.php b/app/Http/Requests/V1/VideoUploadSlotRequest.php deleted file mode 100644 index f6793fc5..00000000 --- a/app/Http/Requests/V1/VideoUploadSlotRequest.php +++ /dev/null @@ -1,33 +0,0 @@ -user()->tokenCan('transmorpher:reserve-video-upload-slot'); - } - - /** - * Get the validation rules that apply to the request. - * - * @return array - */ - public function rules(): array - { - return [ - // Identifier is used in file paths and URLs, therefore only lower/uppercase characters, numbers, underscores and hyphens are allowed. - 'identifier' => ['required', 'string', sprintf('regex:%s', ValidationRegex::IDENTIFIER->get())], - 'callback_url' => ['required', 'string', 'url'] - ]; - } -} diff --git a/app/Interfaces/MediaHandlerInterface.php b/app/Interfaces/MediaHandlerInterface.php index 3cd04a69..83abcffb 100644 --- a/app/Interfaces/MediaHandlerInterface.php +++ b/app/Interfaces/MediaHandlerInterface.php @@ -36,10 +36,9 @@ public function invalidateCdnCache(string $basePath): bool; * @param Version $version * @param int $oldVersionNumber * @param bool $wasProcessed - * @param string $callbackUrl * @return array */ - public function setVersion(User $user, Version $version, int $oldVersionNumber, bool $wasProcessed, string $callbackUrl): array; + public function setVersion(User $user, Version $version, int $oldVersionNumber, bool $wasProcessed): array; /** * @return Filesystem diff --git a/app/Interfaces/TranscodeInterface.php b/app/Interfaces/TranscodeInterface.php index 63b66ce3..6c27e49a 100644 --- a/app/Interfaces/TranscodeInterface.php +++ b/app/Interfaces/TranscodeInterface.php @@ -34,12 +34,11 @@ public function createJobForVersionUpdate(Version $version, UploadSlot $uploadSl * Inform client package about the transcoding result. * * @param ResponseState $responseState - * @param string $callbackUrl * @param string $uploadToken * @param Media $media * @param int $versionNumber * * @return void */ - public function callback(ResponseState $responseState, string $callbackUrl, string $uploadToken, Media $media, int $versionNumber): void; + public function callback(ResponseState $responseState, string $uploadToken, Media $media, int $versionNumber): void; } diff --git a/app/Jobs/TranscodeVideo.php b/app/Jobs/TranscodeVideo.php index dbf178cb..8a2180a5 100644 --- a/app/Jobs/TranscodeVideo.php +++ b/app/Jobs/TranscodeVideo.php @@ -47,7 +47,6 @@ class TranscodeVideo implements ShouldQueue protected Filesystem $localDisk; protected string $originalFilePath; - protected string $callbackUrl; protected string $uploadToken; // Derivatives are saved to a temporary folder first, else race conditions could cause newer versions to be overwritten. @@ -74,7 +73,6 @@ public function __construct( { $this->onQueue('video-transcoding'); $this->originalFilePath = $version->originalFilePath(); - $this->callbackUrl = $this->uploadSlot->callback_url; $this->uploadToken = $this->uploadSlot->token; } @@ -99,7 +97,7 @@ public function handle(): void } match ($this->responseState) { - ResponseState::TRANSCODING_SUCCESSFUL => Transcode::callback($this->responseState, $this->callbackUrl, $this->uploadToken, $this->version->Media, $this->version->number), + ResponseState::TRANSCODING_SUCCESSFUL => Transcode::callback($this->responseState, $this->uploadToken, $this->version->Media, $this->version->number), ResponseState::TRANSCODING_ABORTED => $this->failed(null), }; } @@ -132,7 +130,7 @@ public function failed(?Throwable $exception): void $versionNumber = $this->oldVersionNumber; } - Transcode::callback($this->responseState ?? ResponseState::TRANSCODING_FAILED, $this->callbackUrl, $this->uploadToken, $this->version->Media, $versionNumber); + Transcode::callback($this->responseState ?? ResponseState::TRANSCODING_FAILED, $this->uploadToken, $this->version->Media, $versionNumber); } /** diff --git a/app/Models/UploadSlot.php b/app/Models/UploadSlot.php index e9d8a4af..2909d923 100644 --- a/app/Models/UploadSlot.php +++ b/app/Models/UploadSlot.php @@ -16,7 +16,6 @@ * @property int $id * @property string|null $token * @property string $identifier - * @property string|null $callback_url * @property string|null $validation_rules * @property string|null $valid_until * @property MediaType $media_type @@ -28,7 +27,6 @@ * @method static Builder|UploadSlot newModelQuery() * @method static Builder|UploadSlot newQuery() * @method static Builder|UploadSlot query() - * @method static Builder|UploadSlot whereCallbackUrl($value) * @method static Builder|UploadSlot whereCreatedAt($value) * @method static Builder|UploadSlot whereId($value) * @method static Builder|UploadSlot whereIdentifier($value) @@ -51,9 +49,8 @@ class UploadSlot extends Model */ protected $fillable = [ 'identifier', - 'callback_url', - 'validation_rules', 'media_type', + 'validation_rules', ]; /** diff --git a/app/Models/User.php b/app/Models/User.php index 9fa2502f..9ae60af6 100644 --- a/app/Models/User.php +++ b/app/Models/User.php @@ -55,8 +55,9 @@ class User extends Authenticatable * @var array */ protected $fillable = [ - 'name', + 'api_url', 'email', + 'name', 'password', ]; diff --git a/app/Models/Version.php b/app/Models/Version.php index 40ef8107..34b2e326 100644 --- a/app/Models/Version.php +++ b/app/Models/Version.php @@ -4,9 +4,13 @@ use App\Enums\MediaStorage; use App\Enums\Transformation; +use Eloquent; +use Illuminate\Database\Eloquent\Builder; +use Illuminate\Database\Eloquent\Casts\Attribute; use Illuminate\Database\Eloquent\Factories\HasFactory; use Illuminate\Database\Eloquent\Model; use Illuminate\Database\Eloquent\Relations\BelongsTo; +use Illuminate\Support\Carbon; /** * App\Models\Version @@ -16,20 +20,20 @@ * @property string|null $filename * @property int $processed * @property int $media_id - * @property \Illuminate\Support\Carbon|null $created_at - * @property \Illuminate\Support\Carbon|null $updated_at - * @property-read \App\Models\Media $Media - * @method static \Illuminate\Database\Eloquent\Builder|Version newModelQuery() - * @method static \Illuminate\Database\Eloquent\Builder|Version newQuery() - * @method static \Illuminate\Database\Eloquent\Builder|Version query() - * @method static \Illuminate\Database\Eloquent\Builder|Version whereCreatedAt($value) - * @method static \Illuminate\Database\Eloquent\Builder|Version whereFilename($value) - * @method static \Illuminate\Database\Eloquent\Builder|Version whereId($value) - * @method static \Illuminate\Database\Eloquent\Builder|Version whereMediaId($value) - * @method static \Illuminate\Database\Eloquent\Builder|Version whereNumber($value) - * @method static \Illuminate\Database\Eloquent\Builder|Version whereProcessed($value) - * @method static \Illuminate\Database\Eloquent\Builder|Version whereUpdatedAt($value) - * @mixin \Eloquent + * @property Carbon|null $created_at + * @property Carbon|null $updated_at + * @property-read Media $Media + * @method static Builder|Version newModelQuery() + * @method static Builder|Version newQuery() + * @method static Builder|Version query() + * @method static Builder|Version whereCreatedAt($value) + * @method static Builder|Version whereFilename($value) + * @method static Builder|Version whereId($value) + * @method static Builder|Version whereMediaId($value) + * @method static Builder|Version whereNumber($value) + * @method static Builder|Version whereProcessed($value) + * @method static Builder|Version whereUpdatedAt($value) + * @mixin Eloquent */ class Version extends Model { @@ -44,8 +48,8 @@ class Version extends Model * @var array */ protected $fillable = [ - 'number', 'filename', + 'number', 'processed', ]; @@ -142,4 +146,11 @@ public function imageDerivativeDirectoryPath(): string { return sprintf('%s/%s', $this->Media->baseDirectory(), $this->getKey()); } + + public function hash(): Attribute + { + return Attribute::make( + get: fn(): string => md5(sprintf('%s-%s', $this->number, $this->created_at)) + ); + } } diff --git a/database/migrations/2024_04_23_095200_add_api_url_to_users_table.php b/database/migrations/2024_04_23_095200_add_api_url_to_users_table.php new file mode 100644 index 00000000..6daa0d22 --- /dev/null +++ b/database/migrations/2024_04_23_095200_add_api_url_to_users_table.php @@ -0,0 +1,24 @@ +string('api_url')->comment('The URL at which the client can receive notifications.'); + }); + + Schema::table('upload_slots', function (Blueprint $table) { + $table->dropColumn('callback_url'); + }); + } +}; diff --git a/lang/en/responses.php b/lang/en/responses.php index b8987c92..d4a0f163 100644 --- a/lang/en/responses.php +++ b/lang/en/responses.php @@ -14,7 +14,6 @@ 'deletion_successful' => 'Successfully deleted media.', 'image_upload_successful' => 'Successfully uploaded new image version.', 'image_version_set' => 'Successfully set image version.', - 'no_callback_url_provided' => 'A callback URL is needed for this identifier.', 'transcoding_aborted' => 'Transcoding process aborted due to a new version or upload.', 'transcoding_failed' => 'Video transcoding failed, version has been removed.', 'transcoding_job_dispatch_failed' => 'There was an error when trying to dispatch the transcoding job.', From e4b4d6cfc13aadc4b886a2142c0914b46e5f94ba Mon Sep 17 00:00:00 2001 From: Marco Szulik Date: Tue, 23 Apr 2024 13:24:49 +0200 Subject: [PATCH 02/21] add a command to purge derivatives and notify clients --- app/Classes/MediaHandler/ImageHandler.php | 13 ++++ app/Classes/MediaHandler/VideoHandler.php | 29 ++++++++ app/Classes/Transcode.php | 8 +-- app/Console/Commands/PurgeDerivatives.php | 55 +++++++++++++++ app/Enums/MediaStorage.php | 10 +++ .../ClientNotificationFailedException.php | 26 +++++++ .../Controllers/V1/UploadSlotController.php | 2 +- app/Http/Requests/V1/SetVersionRequest.php | 1 + app/Interfaces/MediaHandlerInterface.php | 7 +- app/Jobs/ClientPurgeNotification.php | 70 +++++++++++++++++++ app/Models/Media.php | 10 +++ routes/api/v1.php | 2 + 12 files changed, 226 insertions(+), 7 deletions(-) create mode 100644 app/Console/Commands/PurgeDerivatives.php create mode 100644 app/Exceptions/ClientNotificationFailedException.php create mode 100644 app/Jobs/ClientPurgeNotification.php diff --git a/app/Classes/MediaHandler/ImageHandler.php b/app/Classes/MediaHandler/ImageHandler.php index 1a5a8afa..d71ba05f 100644 --- a/app/Classes/MediaHandler/ImageHandler.php +++ b/app/Classes/MediaHandler/ImageHandler.php @@ -119,4 +119,17 @@ public function getVersions(Media $media): array 'versions' => $processedVersions->pluck('created_at', 'number')->map(fn($date) => strtotime($date)), ]; } + + /** + * @return array + */ + public function purgeDerivatives(): array + { + $success = $this->getDerivativesDisk()->deleteDirectory(''); + + return [ + 'success' => $success, + 'message' => $success ? 'Deleted image derivatives.' : 'Failed to delete image derivatives.', + ]; + } } diff --git a/app/Classes/MediaHandler/VideoHandler.php b/app/Classes/MediaHandler/VideoHandler.php index 7fa6e50d..a93efb39 100644 --- a/app/Classes/MediaHandler/VideoHandler.php +++ b/app/Classes/MediaHandler/VideoHandler.php @@ -5,6 +5,7 @@ use App\Enums\MediaStorage; use App\Enums\MediaType; use App\Enums\ResponseState; +use App\Enums\UploadState; use App\Interfaces\MediaHandlerInterface; use App\Models\Media; use App\Models\UploadSlot; @@ -100,4 +101,32 @@ public function getVersions(Media $media): array 'versions' => $versions->pluck('created_at', 'number')->map(fn($date) => strtotime($date)), ]; } + + /** + * @return array + */ + public function purgeDerivatives(): array + { + $failedMediaIds = []; + + foreach (Media::whereType(MediaType::VIDEO)->get() as $media) { + // Restore latest version to (re-)generate derivatives. + $version = $media->latestVersion; + + $oldVersionNumber = $version->number; + $wasProcessed = $version->processed; + + $version->update(['number' => $media->latestVersion->number + 1, 'processed' => 0]); + [$responseState, $uploadToken] = $this->setVersion($media->User, $version, $oldVersionNumber, $wasProcessed); + + if ($responseState->getState() === UploadState::ERROR) { + $failedMediaIds[] = $media->getKey(); + } + } + + return [ + 'success' => $success = !count($failedMediaIds), + 'message' => $success ? 'Restored versions for all video media.' : sprintf('Failed to restore versions for media ids: %s.', implode(', ', $failedMediaIds)), + ]; + } } diff --git a/app/Classes/Transcode.php b/app/Classes/Transcode.php index 540f6a0e..60a99444 100644 --- a/app/Classes/Transcode.php +++ b/app/Classes/Transcode.php @@ -73,7 +73,7 @@ public function createJobForVersionUpdate(Version $version, UploadSlot $uploadSl */ public function callback(ResponseState $responseState, string $uploadToken, Media $media, int $versionNumber): void { - $response = [ + $notification = [ 'state' => $responseState->getState()->value, 'message' => $responseState->getMessage(), 'identifier' => $media->identifier, @@ -81,11 +81,11 @@ public function callback(ResponseState $responseState, string $uploadToken, Medi 'upload_token' => $uploadToken, 'public_path' => implode(DIRECTORY_SEPARATOR, array_filter([MediaType::VIDEO->prefix(), $media->baseDirectory()])), 'hash' => Version::whereNumber($versionNumber)->first()?->hash, - 'type' => ClientNotification::VIDEO_TRANSCODING, + 'notification_type' => ClientNotification::VIDEO_TRANSCODING->value, ]; - $signedResponse = SodiumHelper::sign(json_encode($response)); + $signedNotification = SodiumHelper::sign(json_encode($notification)); - Http::post($media->User->api_url, ['signed_response' => $signedResponse]); + Http::post($media->User->api_url, ['signed_notification' => $signedNotification]); } } diff --git a/app/Console/Commands/PurgeDerivatives.php b/app/Console/Commands/PurgeDerivatives.php new file mode 100644 index 00000000..5c4a4d88 --- /dev/null +++ b/app/Console/Commands/PurgeDerivatives.php @@ -0,0 +1,55 @@ +option('all') || $this->option($mediaType->value)) { + ['success' => $success, 'message' => $message] = $mediaType->handler()->purgeDerivatives(); + $success ? $this->info($message) : $this->error($message); + } + } + + $originalsDisk = MediaStorage::ORIGINALS->getDisk(); + $cacheInvalidationFilePath = MediaStorage::getCacheInvalidationFilePath(); + + if (!$originalsDisk->put($cacheInvalidationFilePath, $originalsDisk->get($cacheInvalidationFilePath) + 1)) { + $this->error(sprintf('Failed to update cache invalidation revision at path %s on disk %s', $cacheInvalidationFilePath, MediaStorage::ORIGINALS->value)); + } + + foreach (User::get() as $user) { + ClientPurgeNotification::dispatch($user, $originalsDisk->get($cacheInvalidationFilePath)); + } + + return Command::SUCCESS; + } +} diff --git a/app/Enums/MediaStorage.php b/app/Enums/MediaStorage.php index f72b0038..c90458e9 100644 --- a/app/Enums/MediaStorage.php +++ b/app/Enums/MediaStorage.php @@ -20,4 +20,14 @@ public function getDisk(): Filesystem { return Storage::disk(config(sprintf('transmorpher.disks.%s', $this->value))); } + + /** + * Returns the file path to the cache invalidation file in which the current revision is stored. + * + * @return string + */ + public static function getCacheInvalidationFilePath(): string + { + return 'cacheInvalidationRevision'; + } } diff --git a/app/Exceptions/ClientNotificationFailedException.php b/app/Exceptions/ClientNotificationFailedException.php new file mode 100644 index 00000000..c7e1f856 --- /dev/null +++ b/app/Exceptions/ClientNotificationFailedException.php @@ -0,0 +1,26 @@ +validateUploadFile($uploadedFile, $type->handler()->getValidationRules()); $media->save(); - $versionNumber = $media->Versions()->max('number') + 1; + $versionNumber = $media->latestVersion?->number + 1; $version = $media->Versions()->create(['number' => $versionNumber]); $basePath = $media->baseDirectory(); diff --git a/app/Http/Requests/V1/SetVersionRequest.php b/app/Http/Requests/V1/SetVersionRequest.php index 43a26559..866caf7e 100644 --- a/app/Http/Requests/V1/SetVersionRequest.php +++ b/app/Http/Requests/V1/SetVersionRequest.php @@ -23,5 +23,6 @@ public function authorize(): bool */ public function rules(): array { + return []; } } diff --git a/app/Interfaces/MediaHandlerInterface.php b/app/Interfaces/MediaHandlerInterface.php index 83abcffb..1386e998 100644 --- a/app/Interfaces/MediaHandlerInterface.php +++ b/app/Interfaces/MediaHandlerInterface.php @@ -2,9 +2,7 @@ namespace App\Interfaces; -use App\Enums\MediaStorage; use App\Enums\ResponseState; -use App\Models\Media; use App\Models\UploadSlot; use App\Models\User; use App\Models\Version; @@ -44,4 +42,9 @@ public function setVersion(User $user, Version $version, int $oldVersionNumber, * @return Filesystem */ public function getDerivativesDisk(): Filesystem; + + /** + * @return array + */ + public function purgeDerivatives(): array; } diff --git a/app/Jobs/ClientPurgeNotification.php b/app/Jobs/ClientPurgeNotification.php new file mode 100644 index 00000000..fcbb7776 --- /dev/null +++ b/app/Jobs/ClientPurgeNotification.php @@ -0,0 +1,70 @@ +onQueue('client-notifications'); + } + + /** + * Execute the job. + * @throws ClientNotificationFailedException + */ + public function handle(): void + { + $notification = [ + 'notification_type' => $this->notificationType, + 'cache_invalidation_revision' => $this->cacheInvalidationRevision + ]; + + $signedNotification = SodiumHelper::sign(json_encode($notification)); + + $response = Http::post($this->user->api_url, ['signed_notification' => $signedNotification]); + + if (!$response->ok()) { + throw new ClientNotificationFailedException($this->user->name, $this->notificationType->value, $response->status(), $response->reason()); + } + } +} diff --git a/app/Models/Media.php b/app/Models/Media.php index e7dc54bb..a2e18424 100644 --- a/app/Models/Media.php +++ b/app/Models/Media.php @@ -163,6 +163,16 @@ public function currentVersion(): Attribute ); } + public function latestVersion(): Attribute + { + return Attribute::make( + get: function (): ?Version { + $versions = $this->Versions(); + return $versions->whereNumber($versions->max('number'))->first(); + } + ); + } + /** * Get the base path for files. * Path structure: {username}/{identifier}/ diff --git a/routes/api/v1.php b/routes/api/v1.php index 7ba7fd70..41652785 100644 --- a/routes/api/v1.php +++ b/routes/api/v1.php @@ -1,5 +1,6 @@ name('upload'); Route::get('publickey', fn(): string => SodiumHelper::getPublicKey())->name('getPublicKey'); + Route::get('cacheInvalidationRevision', fn(): string => MediaStorage::ORIGINALS->getDisk()->get(MediaStorage::getCacheInvalidationFilePath()) ?? 0)->name('getCacheInvalidationRevision'); }); From a4c75f0a064d9c705ba864f60bd6f4ed74bdb25c Mon Sep 17 00:00:00 2001 From: Marco Szulik Date: Tue, 23 Apr 2024 13:45:12 +0200 Subject: [PATCH 03/21] add worker for client notifications to production docker image --- docker/workers.conf | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docker/workers.conf b/docker/workers.conf index 69b87f64..a2418a4b 100644 --- a/docker/workers.conf +++ b/docker/workers.conf @@ -13,3 +13,19 @@ redirect_stderr=true stdout_logfile=/dev/stdout ; Timeout of the longest running job (video transcoding with 10800) plus 30. stopwaitsecs=10830 + +[program:client-notification-worker] +process_name=%(program_name)s_%(process_num)02d +; Supervisor starts programs as root by default, which might lead to permission problems when the webserver tries to access files or similar. +user=application +environment=HOME="/home/application",USER="application" +command=php /var/www/html/artisan queue:work --queue=purge-notifications +autostart=true +autorestart=true +stopasgroup=true +killasgroup=true +numprocs=1 +redirect_stderr=true +stdout_logfile=/dev/stdout +; Timeout of the longest running job (video transcoding with 10800) plus 30. +stopwaitsecs=330 From 54ddfd1bb719301fd3fca42a3d2ef089783eb4e2 Mon Sep 17 00:00:00 2001 From: Marco Szulik Date: Wed, 24 Apr 2024 15:53:32 +0200 Subject: [PATCH 04/21] update tests --- database/factories/UserFactory.php | 1 + tests/Unit/CreateUserCommandTest.php | 101 +++++++++++++++++++++------ tests/Unit/VideoTest.php | 4 +- 3 files changed, 83 insertions(+), 23 deletions(-) diff --git a/database/factories/UserFactory.php b/database/factories/UserFactory.php index e033548d..de653d06 100644 --- a/database/factories/UserFactory.php +++ b/database/factories/UserFactory.php @@ -29,6 +29,7 @@ public function definition(): array 'email_verified_at' => now(), 'password' => static::$password ??= Hash::make('password'), 'remember_token' => Str::random(10), + 'api_url' => 'http://example.com/transmorpher/notifications', ]; } diff --git a/tests/Unit/CreateUserCommandTest.php b/tests/Unit/CreateUserCommandTest.php index 823b609a..02b56f52 100644 --- a/tests/Unit/CreateUserCommandTest.php +++ b/tests/Unit/CreateUserCommandTest.php @@ -18,13 +18,14 @@ class CreateUserCommandTest extends TestCase protected const NAME = 'Oswald'; protected const EMAIL = 'oswald@example.com'; + protected const API_URL = 'http://example.com/transmorpher/notifications'; #[Test] public function ensureUserCanBeCreated() { $this->assertDatabaseMissing(User::getModel()->getTable(), ['name' => self::NAME, 'email' => self::EMAIL]); - $exitStatus = Artisan::call(CreateUser::class, ['name' => self::NAME, 'email' => self::EMAIL]); + $exitStatus = $this->createUser(); $this->assertEquals(Command::SUCCESS, $exitStatus); $this->assertDatabaseHas(User::getModel()->getTable(), ['name' => self::NAME, 'email' => self::EMAIL]); @@ -33,7 +34,7 @@ public function ensureUserCanBeCreated() #[Test] public function ensureUserHasSanctumToken() { - Artisan::call(CreateUser::class, ['name' => self::NAME, 'email' => self::EMAIL]); + $this->createUser(); $this->assertNotEmpty(User::whereName(self::NAME)->first()->tokens); } @@ -42,19 +43,20 @@ public function ensureUserHasSanctumToken() #[DataProvider('duplicateEntryDataProvider')] public function failOnDuplicateEntry(string $name, string $email) { - Artisan::call(CreateUser::class, ['name' => self::NAME, 'email' => self::EMAIL]); - $exitStatus = Artisan::call(CreateUser::class, ['name' => $name, 'email' => $email]); + $this->createUser(); + $exitStatus = $this->createUser($name, $email); $this->assertEquals(Command::INVALID, $exitStatus); } #[Test] #[DataProvider('missingArgumentsDataProvider')] - public function failOnMissingArguments(?string $name, ?string $email) + public function failOnMissingArguments(?string $name, ?string $email, ?string $apiUrl) { $arguments = []; $name && $arguments['name'] = $name; $email && $arguments['email'] = $email; + $apiUrl && $arguments['api_url'] = $apiUrl; $this->expectException(RuntimeException::class); Artisan::call(CreateUser::class, $arguments); @@ -62,9 +64,9 @@ public function failOnMissingArguments(?string $name, ?string $email) #[Test] #[DataProvider('invalidArgumentsDataProvider')] - public function failOnInvalidArguments(string $name, string $email) + public function failOnInvalidArguments(string $name, string $email, string $apiUrl) { - $exitStatus = Artisan::call(CreateUser::class, ['name' => $name, 'email' => $email]); + $exitStatus = $this->createUser($name, $email, $apiUrl); $this->assertEquals(Command::INVALID, $exitStatus); } @@ -92,15 +94,33 @@ public static function missingArgumentsDataProvider(): array return [ 'missing name' => [ 'name' => null, - 'email' => self::EMAIL + 'email' => self::EMAIL, + 'apiUrl' => self::API_URL ], 'missing email' => [ 'name' => self::NAME, - 'email' => null + 'email' => null, + 'apiUrl' => self::API_URL + ], + 'missing api url' => [ + 'name' => self::NAME, + 'email' => self::EMAIL, + 'apiUrl' => null ], 'missing name and email' => [ 'name' => null, - 'email' => null + 'email' => null, + 'apiUrl' => self::API_URL + ], + 'missing name and api url' => [ + 'name' => null, + 'email' => self::EMAIL, + 'apiUrl' => null + ], + 'missing email and api url' => [ + 'name' => self::NAME, + 'email' => null, + 'apiUrl' => null ] ]; } @@ -110,44 +130,85 @@ public static function invalidArgumentsDataProvider(): array return [ 'invalid name with slash' => [ 'name' => 'invalid/name', - 'email' => self::EMAIL + 'email' => self::EMAIL, + 'apiUrl' => self::API_URL ], 'invalid name with backslash' => [ 'name' => 'invalid\name', - 'email' => self::EMAIL + 'email' => self::EMAIL, + 'apiUrl' => self::API_URL ], 'invalid name with dot' => [ 'name' => 'invalid.name', - 'email' => self::EMAIL + 'email' => self::EMAIL, + 'apiUrl' => self::API_URL ], 'invalid name with hyphen' => [ 'name' => 'invalid--name', - 'email' => self::EMAIL + 'email' => self::EMAIL, + 'apiUrl' => self::API_URL ], 'invalid name with trailing hyphen' => [ 'name' => 'invalidName-', - 'email' => self::EMAIL + 'email' => self::EMAIL, + 'apiUrl' => self::API_URL ], 'invalid name with special character' => [ 'name' => 'invalidName!', - 'email' => self::EMAIL + 'email' => self::EMAIL, + 'apiUrl' => self::API_URL ], 'invalid name with umlaut' => [ 'name' => 'invalidNäme', - 'email' => self::EMAIL + 'email' => self::EMAIL, + 'apiUrl' => self::API_URL ], 'invalid name with space' => [ 'name' => 'invalid name', - 'email' => self::EMAIL + 'email' => self::EMAIL, + 'apiUrl' => self::API_URL ], 'invalid email' => [ 'name' => self::NAME, - 'email' => 'invalidEmail' + 'email' => 'invalidEmail', + 'apiUrl' => self::API_URL ], 'invalid name and email' => [ 'name' => 'invalid/name', - 'email' => 'invalidEmail' + 'email' => 'invalidEmail', + 'apiUrl' => self::API_URL + ], + 'invalid api url no scheme' => [ + 'name' => self::NAME, + 'email' => self::EMAIL, + 'apiUrl' => 'example.com/transmorpher/notifications' + ], + 'invalid api url no path' => [ + 'name' => self::NAME, + 'email' => self::EMAIL, + 'apiUrl' => 'https://example.com' + ], + 'invalid api url no scheme and no path' => [ + 'name' => self::NAME, + 'email' => self::EMAIL, + 'apiUrl' => 'example.com' + ], + 'invalid name and email and apiUrl' => [ + 'name' => 'invalid/name', + 'email' => 'invalidEmail', + 'apiUrl' => 'example.com' ] ]; } + + /** + * @param string|null $name + * @param string|null $email + * @param string|null $apiUrl + * @return int + */ + protected function createUser(?string $name = null, ?string $email = null, ?string $apiUrl = null): int + { + return Artisan::call(CreateUser::class, ['name' => $name ?? self::NAME, 'email' => $email ?? self::EMAIL, 'api_url' => $apiUrl ?? self::API_URL]); + } } diff --git a/tests/Unit/VideoTest.php b/tests/Unit/VideoTest.php index fa0d9ef0..4c1cd541 100644 --- a/tests/Unit/VideoTest.php +++ b/tests/Unit/VideoTest.php @@ -18,7 +18,6 @@ class VideoTest extends MediaTest { protected const IDENTIFIER = 'testVideo'; protected const VIDEO_NAME = 'video.mp4'; - protected const CALLBACK_URL = 'http://example.com/callback'; protected Filesystem $videoDerivativesDisk; protected function setUp(): void @@ -33,7 +32,6 @@ public function ensureVideoUploadSlotCanBeReserved() { $reserveUploadSlotResponse = $this->json('POST', route('v1.reserveVideoUploadSlot'), [ 'identifier' => self::IDENTIFIER, - 'callback_url' => self::CALLBACK_URL ]); $reserveUploadSlotResponse->assertOk(); @@ -56,7 +54,7 @@ public function ensureTranscodingIsAbortedWhenNewerVersionExists(string $uploadT TranscodeVideo::dispatch($outdatedVersion, $uploadSlot); $request = Http::recorded()[0][0]; - $transcodingResult = json_decode(SodiumHelper::decrypt($request->data()['signed_response']), true); + $transcodingResult = json_decode(SodiumHelper::decrypt($request->data()['signed_notification']), true); $this->assertEquals(ResponseState::TRANSCODING_ABORTED->getState()->value, $transcodingResult['state']); $this->assertEquals(ResponseState::TRANSCODING_ABORTED->getMessage(), $transcodingResult['message']); From 73083ccf273bf9c81ed8db8e2495dc216bd0f557 Mon Sep 17 00:00:00 2001 From: Marco Szulik Date: Wed, 24 Apr 2024 16:06:00 +0200 Subject: [PATCH 05/21] re-generate ide-helper --- _ide_helper.php | 15 ++++++++++++--- app/Models/Media.php | 1 + app/Models/User.php | 2 ++ app/Models/Version.php | 3 ++- 4 files changed, 17 insertions(+), 4 deletions(-) diff --git a/_ide_helper.php b/_ide_helper.php index df37af47..259252ca 100644 --- a/_ide_helper.php +++ b/_ide_helper.php @@ -18012,6 +18012,16 @@ * */ class TranscodeFacade { /** + * Returns the class which handles the actual transcoding. + * + * @return string + * @static + */ public static function getJobClass() + { + /** @var \App\Classes\Transcode $instance */ + return $instance->getJobClass(); + } + /** * Creates a job which handles the transcoding of a video. * * @param \App\Models\Version $version @@ -18041,16 +18051,15 @@ * Inform client package about the transcoding result. * * @param \App\Enums\ResponseState $responseState - * @param string $callbackUrl * @param string $uploadToken * @param \App\Models\Media $media * @param int $versionNumber * @return void * @static - */ public static function callback($responseState, $callbackUrl, $uploadToken, $media, $versionNumber) + */ public static function callback($responseState, $uploadToken, $media, $versionNumber) { /** @var \App\Classes\Transcode $instance */ - $instance->callback($responseState, $callbackUrl, $uploadToken, $media, $versionNumber); + $instance->callback($responseState, $uploadToken, $media, $versionNumber); } } /** diff --git a/app/Models/Media.php b/app/Models/Media.php index a2e18424..f1dbca2a 100644 --- a/app/Models/Media.php +++ b/app/Models/Media.php @@ -28,6 +28,7 @@ * @property-read \Illuminate\Database\Eloquent\Collection $Versions * @property-read int|null $versions_count * @property-read \App\Models\Version $current_version + * @property-read \App\Models\Version|null $latest_version * @method static \Illuminate\Database\Eloquent\Builder|Media newModelQuery() * @method static \Illuminate\Database\Eloquent\Builder|Media newQuery() * @method static \Illuminate\Database\Eloquent\Builder|Media query() diff --git a/app/Models/User.php b/app/Models/User.php index 9ae60af6..3ccdec65 100644 --- a/app/Models/User.php +++ b/app/Models/User.php @@ -22,6 +22,7 @@ * @property \Illuminate\Support\Carbon|null $created_at * @property \Illuminate\Support\Carbon|null $updated_at * @property string|null $protector_public_key The sodium public key for the Protector package. + * @property string $api_url The URL at which the client can receive notifications. * @property-read \Illuminate\Database\Eloquent\Collection $Media * @property-read int|null $media_count * @property-read \Illuminate\Database\Eloquent\Collection $UploadSlots @@ -34,6 +35,7 @@ * @method static \Illuminate\Database\Eloquent\Builder|User newModelQuery() * @method static \Illuminate\Database\Eloquent\Builder|User newQuery() * @method static \Illuminate\Database\Eloquent\Builder|User query() + * @method static \Illuminate\Database\Eloquent\Builder|User whereApiUrl($value) * @method static \Illuminate\Database\Eloquent\Builder|User whereCreatedAt($value) * @method static \Illuminate\Database\Eloquent\Builder|User whereEmail($value) * @method static \Illuminate\Database\Eloquent\Builder|User whereEmailVerifiedAt($value) diff --git a/app/Models/Version.php b/app/Models/Version.php index 34b2e326..b0d06853 100644 --- a/app/Models/Version.php +++ b/app/Models/Version.php @@ -22,7 +22,8 @@ * @property int $media_id * @property Carbon|null $created_at * @property Carbon|null $updated_at - * @property-read Media $Media + * @property-read \App\Models\Media $Media + * @property-read string $hash * @method static Builder|Version newModelQuery() * @method static Builder|Version newQuery() * @method static Builder|Version query() From e9323505aca124afa3baf8fe252349e466e7d12f Mon Sep 17 00:00:00 2001 From: Marco Szulik Date: Wed, 24 Apr 2024 16:20:18 +0200 Subject: [PATCH 06/21] update readme --- README.md | 142 ++++++++++++++++++++++++++++-------------------------- 1 file changed, 73 insertions(+), 69 deletions(-) diff --git a/README.md b/README.md index a3f8a292..f707754a 100644 --- a/README.md +++ b/README.md @@ -29,15 +29,16 @@ To not accidentally upgrade to a new major version, attach the major version you #### Configuration options -There needs to be at least 1 Laravel worker to transcode videos. The following variable specifies how many workers should be running in the container: +There needs to be at least 1 Laravel worker to transcode videos. +The following variable specifies how many workers should be running in the container: ```dotenv VIDEO_TRANSCODING_WORKERS_AMOUNT=1 ``` > [!CAUTION] -> Using the database queue connection does neither guarantee FIFO nor prevent duplicate runs. It is recommended to use a queue which can guarantee these aspects, such as AWS SQS -> FIFO. +> Using the database queue connection does neither guarantee FIFO nor prevent duplicate runs. +> It is recommended to use a queue which can guarantee these aspects, such as AWS SQS FIFO. > To prevent duplicate runs with database, use only one worker process. This environment variable has to be passed to the app container in your docker-compose.yml: @@ -49,7 +50,7 @@ environment: ### Cloning the repository -To clone the repository and get your media server running use: +To clone the repository and get your media server running, use: ```bash git clone --branch release/v0 --single-branch https://github.com/cybex-gmbh/transmorpher.git @@ -64,13 +65,13 @@ Image manipulation: - [ImageMagick](https://imagemagick.org/index.php) - [php-imagick](https://www.php.net/manual/en/book.imagick.php) -> Optionally you can use GD, which can be configured in the Intervention Image configuration file. +> Optionally, you can use GD, which can be configured in the Intervention Image configuration file. Image optimization: - [JpegOptim](https://github.com/tjko/jpegoptim) - [Optipng](https://optipng.sourceforge.net/) -- [Pngquant 2](https://pngquant.org/) +- [Pngquant](https://pngquant.org/) - [Gifsicle](https://www.lcdf.org/gifsicle/) - [cwebp](https://developers.google.com/speed/webp/docs/precompiled) @@ -78,6 +79,10 @@ To use video transcoding: - [FFmpeg](https://ffmpeg.org/) +#### Generic workers + +Client notifications will be pushed on the queue `client-notifications`. You will need to set up 1 worker for this queue. + #### Scheduling There may be some cases (e.g. failed uploads) where chunk files are not deleted and stay on the local disk. @@ -85,38 +90,57 @@ To keep the local disk clean, a command is scheduled hourly to delete chunk file See the [`chunk-upload` configuration file](config/chunk-upload.php) for more information. -To run the scheduler you will need to add a cron job that runs the `schedule:run` command on your server: +To run the scheduler, you will need to add a cron job that runs the `schedule:run` command on your server: ``` * * * * * cd /path-to-your-project && php artisan schedule:run >> /dev/null 2>&1 ``` -For more information about scheduling check the [Laravel Docs](https://laravel.com/docs/11.x/scheduling). +For more information about scheduling, check the [Laravel Docs](https://laravel.com/docs/11.x/scheduling). ## General configuration #### Disks -The media server uses 3 separate Laravel disks to store originals, image derivatives and video derivatives. Use the provided `.env` keys to select any of the disks in -the `filesystems.php` config file. +The media server uses 3 separate Laravel disks to store originals, image derivatives and video derivatives. +Use the provided `.env` keys to select any of the disks in the `filesystems.php` config file. > [!NOTE] > > 1. The root folder, like images/, of the configured derivatives disks has to always match the prefix provided by the `MediaType` enum. > 1. If this prefix would be changed after initially launching your media server, clients would no longer be able to retrieve their previously uploaded media. +#### Sodium Keypair + +A signed request is used to notify clients about finished transcodings and when derivatives are purged. +For this, a [Sodium](https://www.php.net/manual/en/book.sodium.php) keypair has to be configured. + +To create a keypair, use the provided command: + +```bash +php artisan create:keypair +``` + +The newly created keypair has to be written in the `.env` file: + +```dotenv +TRANSMORPHER_SIGNING_KEYPAIR= +``` + +The public key of the media server is available under the `/api/v*/publickey` endpoint and can be requested by any client. + ### Cloud Setup The Transmorpher media server is not dependent on a specific cloud service provider, but only provides classes for AWS services out of the box. #### Prerequisites for video functionality -- A file storage, for example AWS S3 -- A routing capable service, for example a Content Delivery Network, like AWS CloudFront +- A file storage, for example, AWS S3 +- A routing-capable service, for example, a Content Delivery Network, like AWS CloudFront #### IAM -Create an IAM user with programmatic access. For more information check the documentation for the corresponding service. +Create an IAM user with programmatic access. For more information, check the documentation for the corresponding service. Permissions: @@ -165,13 +189,13 @@ Configure your CloudFront-Distribution-ID: AWS_CLOUDFRONT_DISTRIBUTION_ID= ``` -Changes to media will automatically trigger a cache invalidation, therefore the CDN cache duration can be set to a long time. +Changes to media will automatically trigger a cache invalidation. Therefore, the CDN cache duration can be set to a long time. To forward incoming requests from the CDN to your media server, configure your Transmorpher media server as the main origin. For more information on configuring origins in CloudFront see the [documentation page](https://docs.aws.amazon.com/AmazonCloudFront/latest/DeveloperGuide/DownloadDistS3AndCustomOrigins.html). -In order to properly use the API you need to either: +To properly use the API, you need to either: 1. add a rule to not cache anything under `/api/*` 1. publish the Transmorpher media server under an additional domain that is not behind the CDN @@ -180,31 +204,13 @@ In order to properly use the API you need to either: *Content Delivery Network* -In the CDN routing create a new behavior which points requests starting with "/videos/*" to a new origin, which is the video derivatives S3 bucket. -\ -*Sodium Keypair* - -A signed request is used to notify clients about finished transcodings. For this, a [Sodium](https://www.php.net/manual/en/book.sodium.php) keypair has to be configured. - -To create a keypair, simply use the provided command: - -```bash -php artisan transmorpher:keypair -``` - -The newly created keypair has to be written in the `.env` file: - -```dotenv -TRANSMORPHER_SIGNING_KEYPAIR= -``` +In the CDN routing create a new behavior which points requests starting with "/videos/*" to a new origin, which is the video derivatives S3 bucket. -The public key of the media server is available under the `/api/v*/publickey` endpoint and can be requested -by any client. -\ *Queue* -Transcoding jobs are dispatched onto the "video-transcoding" queue. You can have these jobs processed on the main server or dedicated workers. For more information check -the [Laravel Queue Documentation](https://laravel.com/docs/10.x/queues). +Transcoding jobs are dispatched onto the "video-transcoding" queue. +You can have these jobs processed on the main server or dedicated workers. +For more information, check the [Laravel Queue Documentation](https://laravel.com/docs/11.x/queues). > [!NOTE] > Since queues are not generally FIFO, it is recommended to use a queue which guarantees FIFO and also prevents @@ -246,29 +252,11 @@ To access public derivatives for videos, generate a symlink from the Laravel sto php artisan storage:link ``` -*Sodium Keypair* - -A signed request is used to notify clients about finished transcodings. For this, a [Sodium](https://www.php.net/manual/en/book.sodium.php) keypair has to be configured. - -To create a keypair, simply use the provided command: - -```bash -php artisan transmorpher:keypair -``` - -The newly created keypair has to be written in the `.env` file: - -```dotenv -TRANSMORPHER_SIGNING_KEYPAIR= -``` - -The public key of the media server is available under the `/api/v*/publickey` endpoint and can be requested -by any client. -\ *Queue* -Transcoding jobs are dispatched onto the "video-transcoding" queue. You can have these jobs processed on the main server or dedicated workers. For more information check -the [Laravel Queue Documentation](https://laravel.com/docs/10.x/queues). +Transcoding jobs are dispatched onto the "video-transcoding" queue. +You can have these jobs processed on the main server or dedicated workers. +For more information, check the [Laravel Queue Documentation](https://laravel.com/docs/11.x/queues). You can define your queue connection in the `.env` file: @@ -302,7 +290,7 @@ Media always belongs to a user. To easily create one, use the provided command: php artisan create:user ``` -This command will provide you with a [Laravel Sanctum](https://laravel.com/docs/10.x/sanctum) token, which has to be +This command will provide you with a [Laravel Sanctum](https://laravel.com/docs/11.x/sanctum) token, which has to be written in the `.env` file of a client system. > The token will be passed for all API requests for authorization and is connected to the corresponding user. @@ -318,7 +306,7 @@ Media is identified by a string which is passed when uploading media. This "iden When media is uploaded on the same identifier by the same user, a new version for the same media will be created. -The media server provides following features for media: +The media server provides the following features for media: - upload - get derivative @@ -330,7 +318,8 @@ The media server provides following features for media: ## Image transformation -Images will always be optimized and transformed on the Transmorpher media server. Requests for derivatives will also be directly answered by the media server. +Images will always be optimized and transformed on the Transmorpher media server. +The media server will also directly answer requests for derivatives. The media server provides the following transformations for images: @@ -360,7 +349,8 @@ transformations. Video transcoding is handled as an asynchronous task. The client will receive the information about the transcoded video as soon as it completes. For this, a signed request is sent to the client. -Since video transcoding is a complex task it may take some time to complete. The client will also be notified about failed attempts. +Since video transcoding is a complex task, it may take some time to complete. +The client will also be notified about failed attempts. To publicly access a video, the client name, the identifier and a format have to be specified. There are different formats available: @@ -389,9 +379,8 @@ You will also have to adjust the `transmorpher.php` configuration value for the The class to transform images as well as the classes to convert images to different formats are interchangeable. This provides the ability to add additional image manipulation libraries or logic in a modular way. -To add a class for image transformation, simply create a new class which implements the `TransformInterface`. An example -implementation can be found -at `App\Classes\Intervention\Transform`. +To add a class for image transformation, create a new class which implements the `TransformInterface`. +An example implementation can be found at `App\Classes\Intervention\Transform`. Additionally, the newly created class has to be specified in the `transmorpher.php` configuration file: ```php @@ -416,7 +405,7 @@ You will also have to adjust the configuration values: The `image-optimizer.php` configuration file specifies which optimizers should be used. Here you can configure options for each optimizer and add new or remove optimizers. -For more information on adding custom optimizers check the documentation of +For more information on adding custom optimizers, check the documentation of the [Laravel Image Optimizer](https://github.com/spatie/laravel-image-optimizer#adding-your-own-optimizers) package. ### Video Transcoding @@ -424,7 +413,7 @@ the [Laravel Image Optimizer](https://github.com/spatie/laravel-image-optimizer# By default, the Transmorpher uses FFmpeg and Laravel jobs for transcoding videos. This can be changed similar to the image transformation classes. -To interchange the class, which is responsible for initiating transcoding, simply create a new class which implements +To interchange the class, which is responsible for initiating transcoding, create a new class which implements the `TranscodeInterface`. An example implementation, which dispatches a job, can be found at `App\Classes\Transcode.php`. You will also have to adjust the configuration value: @@ -433,13 +422,28 @@ You will also have to adjust the configuration value: 'transcode_class' => App\Classes\YourTranscodeClass::class, ``` +## Purging derivatives + +Adjusting the way derivatives are generated will not be reflected on already existing derivatives. Therefore, you might want to delete all existing derivatives or re-generate them. + +We provide a command which will additionally notify clients with a signed request about a new derivatives revision, so they can react accordingly (e.g. update cache buster). + +```bash +php artisan purge:derivatives +``` + +The command accepts the options `--image`, `--video` and `--all` (or `-a`) for purging the respective derivatives. +Image derivatives will be deleted, for video derivatives we dispatch a new transcoding job for the current version. + +The derivatives revision is available on the route `/api/v*/cacheInvalidationRevision`. + ## Development ### [Pullpreview](https://github.com/pullpreview/action) -For more information take a look at the PullPreview section of the [github-workflow repository](https://github.com/cybex-gmbh/github-workflows#pullpreview). +For more information, take a look at the PullPreview section of the [github-workflow repository](https://github.com/cybex-gmbh/github-workflows#pullpreview). -App specific GitHub Secrets: +App-specific GitHub Secrets: - APP_KEY - TRANSMORPHER_SIGNING_KEYPAIR From 0963bfcd3ff1183aa3465ebf8b3de6d223574c7e Mon Sep 17 00:00:00 2001 From: Marco Szulik Date: Thu, 25 Apr 2024 13:31:32 +0200 Subject: [PATCH 07/21] add tests --- app/Classes/Transcode.php | 10 +++ app/Interfaces/TranscodeInterface.php | 7 ++ tests/MediaTest.php | 4 +- tests/Unit/ImageTest.php | 60 ++++++++++--- tests/Unit/VideoTest.php | 118 +++++++++++++++++++++++--- 5 files changed, 174 insertions(+), 25 deletions(-) diff --git a/app/Classes/Transcode.php b/app/Classes/Transcode.php index 60a99444..49712199 100644 --- a/app/Classes/Transcode.php +++ b/app/Classes/Transcode.php @@ -16,6 +16,16 @@ class Transcode implements TranscodeInterface { + /** + * Returns the class which handles the actual transcoding. + * + * @return string + */ + public function getJobClass(): string + { + return TranscodeVideo::class; + } + /** * Creates a job which handles the transcoding of a video. * diff --git a/app/Interfaces/TranscodeInterface.php b/app/Interfaces/TranscodeInterface.php index 6c27e49a..adc56c75 100644 --- a/app/Interfaces/TranscodeInterface.php +++ b/app/Interfaces/TranscodeInterface.php @@ -9,6 +9,13 @@ interface TranscodeInterface { + /** + * Returns the class which handles the actual transcoding. + * + * @return string + */ + public function getJobClass(): string; + /** * Creates a job which handles the transcoding of a video. * diff --git a/tests/MediaTest.php b/tests/MediaTest.php index 332bd464..b97e9930 100644 --- a/tests/MediaTest.php +++ b/tests/MediaTest.php @@ -10,7 +10,7 @@ class MediaTest extends TestCase { - protected static User $user; + protected User $user; protected Filesystem $originalsDisk; protected function setUp(): void @@ -20,7 +20,7 @@ protected function setUp(): void $this->originalsDisk ??= Storage::persistentFake(config(sprintf('transmorpher.disks.%s', MediaStorage::ORIGINALS->value))); Sanctum::actingAs( - self::$user ??= User::factory()->create(), + $this->user ??= User::first() ?: User::factory()->create(), ['*'] ); } diff --git a/tests/Unit/ImageTest.php b/tests/Unit/ImageTest.php index 52b664fe..8b9e467c 100644 --- a/tests/Unit/ImageTest.php +++ b/tests/Unit/ImageTest.php @@ -2,15 +2,21 @@ namespace Tests\Unit; +use App\Console\Commands\PurgeDerivatives; +use App\Enums\ClientNotification; use App\Enums\MediaStorage; use App\Enums\Transformation; use App\Exceptions\InvalidTransformationFormatException; use App\Exceptions\InvalidTransformationValueException; use App\Exceptions\TransformationNotFoundException; +use App\Helpers\SodiumHelper; use App\Models\Media; use App\Models\UploadSlot; use App\Models\Version; +use Artisan; +use Http; use Illuminate\Contracts\Filesystem\Filesystem; +use Illuminate\Http\Client\Request; use Illuminate\Http\UploadedFile; use Illuminate\Testing\TestResponse; use PHPUnit\Framework\Attributes\DataProvider; @@ -61,6 +67,7 @@ protected function uploadImage(string $uploadToken): TestResponse ]); } + #[Test] #[Depends('ensureImageUploadSlotCanBeReserved')] public function ensureImageCanBeUploaded(string $uploadToken) @@ -79,7 +86,7 @@ public function ensureImageCanBeUploaded(string $uploadToken) protected function createDerivativeForVersion(Version $version): TestResponse { - return $this->get(route('getDerivative', [self::$user->name, $version->Media])); + return $this->get(route('getDerivative', [$this->user->name, $version->Media])); } #[Test] @@ -96,7 +103,7 @@ public function ensureProcessedFilesAreAvailable(Version $version) public function ensureUnprocessedFilesAreNotAvailable(Version $version) { $version->update(['processed' => 0]); - $getDerivativeResponse = $this->get(route('getDerivative', [self::$user->name, $version->Media])); + $getDerivativeResponse = $this->get(route('getDerivative', [$this->user->name, $version->Media])); $getDerivativeResponse->assertNotFound(); } @@ -114,8 +121,8 @@ protected function assertMediaDirectoryExists(Media $media): void protected function assertUserDirectoryExists(): void { - $this->originalsDisk->assertExists(self::$user->name); - $this->imageDerivativesDisk->assertExists(self::$user->name); + $this->originalsDisk->assertExists($this->user->name); + $this->imageDerivativesDisk->assertExists($this->user->name); } protected function assertVersionFilesMissing(Version $version): void @@ -132,11 +139,11 @@ protected function assertMediaDirectoryMissing(Media $media): void protected function assertUserDirectoryMissing(): void { - $this->originalsDisk->assertMissing(self::$user->name); - $this->imageDerivativesDisk->assertMissing(self::$user->name); + $this->originalsDisk->assertMissing($this->user->name); + $this->imageDerivativesDisk->assertMissing($this->user->name); } - protected function setupDeletionTest(): void + protected function setupMediaAndVersion(): void { $this->uploadToken = $this->reserveUploadSlot()->json()['upload_token']; $this->version = Media::whereIdentifier(self::IDENTIFIER)->first()->Versions()->whereNumber($this->uploadImage($this->uploadToken)['version'])->first(); @@ -148,7 +155,7 @@ protected function setupDeletionTest(): void #[Test] public function ensureVersionDeletionMethodsWork() { - $this->setupDeletionTest(); + $this->setupMediaAndVersion(); $this->assertVersionFilesExist($this->version); @@ -160,7 +167,7 @@ public function ensureVersionDeletionMethodsWork() #[Test] public function ensureMediaDeletionMethodsWork() { - $this->setupDeletionTest(); + $this->setupMediaAndVersion(); $this->assertVersionFilesExist($this->version); $this->assertMediaDirectoryExists($this->media); @@ -175,17 +182,46 @@ public function ensureMediaDeletionMethodsWork() $this->assertModelMissing($this->uploadSlot); } + #[Test] + public function ensureImageDerivativesArePurged() + { + $this->setupMediaAndVersion(); + + $this->assertVersionFilesExist($this->version); + + $cacheRevisionBeforeCommand = $this->originalsDisk->get(MediaStorage::getCacheInvalidationFilePath()); + + Http::fake([ + $this->user->api_url => Http::response() + ]); + + Artisan::call(PurgeDerivatives::class, ['--image' => true]); + + $cacheRevisionAfterCommand = $this->originalsDisk->get(MediaStorage::getCacheInvalidationFilePath()); + + Http::assertSent(function (Request $request) use ($cacheRevisionAfterCommand) { + $decryptedNotification = json_decode(SodiumHelper::decrypt($request['signed_notification']), true); + + return $request->url() == $this->user->api_url + && $decryptedNotification['notification_type'] == ClientNotification::CACHE_INVALIDATION->value + && $decryptedNotification['cache_invalidation_revision'] == $cacheRevisionAfterCommand; + }); + + $this->assertTrue(++$cacheRevisionBeforeCommand == $cacheRevisionAfterCommand); + $this->imageDerivativesDisk->assertMissing($this->version->imageDerivativeFilePath()); + } + #[Test] public function ensureUserDeletionMethodsWork() { - $this->setupDeletionTest(); + $this->setupMediaAndVersion(); $this->assertVersionFilesExist($this->version); $this->assertMediaDirectoryExists($this->media); $this->assertUserDirectoryExists(); - $this->runProtectedMethod(self::$user, 'deleteRelatedModels'); - $this->runProtectedMethod(self::$user, 'deleteMediaDirectories'); + $this->runProtectedMethod($this->user, 'deleteRelatedModels'); + $this->runProtectedMethod($this->user, 'deleteMediaDirectories'); $this->assertVersionFilesMissing($this->version); $this->assertMediaDirectoryMissing($this->media); diff --git a/tests/Unit/VideoTest.php b/tests/Unit/VideoTest.php index 4c1cd541..385dd19c 100644 --- a/tests/Unit/VideoTest.php +++ b/tests/Unit/VideoTest.php @@ -2,20 +2,32 @@ namespace Tests\Unit; +use App\Console\Commands\PurgeDerivatives; +use App\Enums\ClientNotification; use App\Enums\MediaStorage; use App\Enums\ResponseState; use App\Helpers\SodiumHelper; -use App\Jobs\TranscodeVideo; +use App\Jobs\ClientPurgeNotification; +use App\Models\Media; use App\Models\UploadSlot; +use Artisan; +use File; use Http; use Illuminate\Contracts\Filesystem\Filesystem; -use PHPUnit\Framework\Attributes\Depends; +use Illuminate\Foundation\Testing\RefreshDatabase; +use Illuminate\Http\Client\Request; +use Illuminate\Http\UploadedFile; +use Illuminate\Testing\TestResponse; use PHPUnit\Framework\Attributes\Test; +use Queue; use Storage; use Tests\MediaTest; +use Transcode; class VideoTest extends MediaTest { + use RefreshDatabase; + protected const IDENTIFIER = 'testVideo'; protected const VIDEO_NAME = 'video.mp4'; protected Filesystem $videoDerivativesDisk; @@ -27,31 +39,65 @@ protected function setUp(): void $this->videoDerivativesDisk ??= Storage::persistentFake(config(sprintf('transmorpher.disks.%s', MediaStorage::VIDEO_DERIVATIVES->value))); } - #[Test] - public function ensureVideoUploadSlotCanBeReserved() + /** + * @return TestResponse + */ + protected function reserveUploadSlot(): TestResponse { - $reserveUploadSlotResponse = $this->json('POST', route('v1.reserveVideoUploadSlot'), [ + return $this->json('POST', route('v1.reserveVideoUploadSlot'), [ 'identifier' => self::IDENTIFIER, ]); + } + + #[Test] + public function ensureVideoUploadSlotCanBeReserved() + { + $reserveUploadSlotResponse = $this->reserveUploadSlot(); $reserveUploadSlotResponse->assertOk(); - return $reserveUploadSlotResponse->json()['upload_token']; + return $reserveUploadSlotResponse->json('upload_token'); + } + + protected function uploadVideo(): TestResponse + { + $uploadToken = $this->reserveUploadSlot()->json('upload_token'); + + return $this->post(route('v1.upload', [$uploadToken]), [ + 'file' => UploadedFile::fake()->createWithContent('video.mp4', File::get(base_path('tests/data/test.mp4'))), + 'identifier' => self::IDENTIFIER + ]); } #[Test] - #[Depends('ensureVideoUploadSlotCanBeReserved')] - public function ensureTranscodingIsAbortedWhenNewerVersionExists(string $uploadToken) + public function ensureVideoCanBeUploaded() { - Http::fake(); + Queue::fake(); + + $uploadResponse = $this->uploadVideo(); + + $uploadResponse->assertCreated(); + Queue::assertPushed(Transcode::getJobClass()); + + $media = Media::whereIdentifier($uploadResponse['identifier'])->first(); + $version = $media->Versions()->whereNumber($uploadResponse['version'])->first(); + $this->originalsDisk->assertExists($version->originalFilePath()); + } + + #[Test] + public function ensureTranscodingIsAbortedWhenNewerVersionExists() + { + $uploadToken = $this->reserveUploadSlot()->json('upload_token'); $uploadSlot = UploadSlot::firstWhere('token', $uploadToken); - $media = self::$user->Media()->create(['identifier' => self::IDENTIFIER, 'type' => $uploadSlot->media_type]); + $media = $this->user->Media()->create(['identifier' => self::IDENTIFIER, 'type' => $uploadSlot->media_type]); $outdatedVersion = $media->Versions()->create(['number' => 1, 'filename' => sprintf('1-%s', self::VIDEO_NAME)]); $media->Versions()->create(['number' => 2, 'filename' => sprintf('2-%s', self::VIDEO_NAME)]); - TranscodeVideo::dispatch($outdatedVersion, $uploadSlot); + Http::fake(); + + Transcode::createJob($outdatedVersion, $uploadSlot); $request = Http::recorded()[0][0]; $transcodingResult = json_decode(SodiumHelper::decrypt($request->data()['signed_notification']), true); @@ -59,4 +105,54 @@ public function ensureTranscodingIsAbortedWhenNewerVersionExists(string $uploadT $this->assertEquals(ResponseState::TRANSCODING_ABORTED->getState()->value, $transcodingResult['state']); $this->assertEquals(ResponseState::TRANSCODING_ABORTED->getMessage(), $transcodingResult['message']); } + + #[Test] + public function ensureTranscodingWorks() + { + $uploadResponse = $this->uploadVideo(); + + $media = Media::whereIdentifier($uploadResponse['identifier'])->first(); + $version = $media->Versions()->whereNumber($uploadResponse['version'])->first(); + $uploadSlot = UploadSlot::withoutGlobalScopes()->whereToken($uploadResponse['upload_token'])->first(); + + Http::fake([ + $this->user->api_url => Http::response() + ]); + + $this->assertTrue(Transcode::createJob($version, $uploadSlot)); + + Http::assertSent(function (Request $request) { + $decryptedNotification = json_decode(SodiumHelper::decrypt($request['signed_notification']), true); + + return $request->url() == $this->user->api_url + && $decryptedNotification['notification_type'] == ClientNotification::VIDEO_TRANSCODING->value + && $decryptedNotification['state'] == ResponseState::TRANSCODING_SUCCESSFUL->getState()->value; + }); + + $this->videoDerivativesDisk->assertExists($media->videoDerivativeFilePath('mp4') . '.mp4'); + $this->videoDerivativesDisk->assertExists($media->videoDerivativeFilePath('hls') . '.m3u8'); + $this->videoDerivativesDisk->assertExists($media->videoDerivativeFilePath('dash') . '.mpd'); + } + + #[Test] + public function ensureVideoDerivativesArePurged() + { + $uploadResponse = $this->uploadVideo(); + + $media = Media::whereIdentifier($uploadResponse['identifier'])->first(); + $version = $media->currentVersion; + + $versionNumberBeforePurging = $version->number; + + Queue::fake(); + Http::fake([ + $this->user->api_url => Http::response() + ]); + + Artisan::call(PurgeDerivatives::class, ['--video' => true]); + + $this->assertTrue($versionNumberBeforePurging + 1 == $version->refresh()->number); + Queue::assertPushed(Transcode::getJobClass()); + Queue::assertPushed(ClientPurgeNotification::class); + } } From 4986e0e54110ab77a70e098f63284c496172c850 Mon Sep 17 00:00:00 2001 From: Marco Szulik Date: Fri, 26 Apr 2024 09:35:26 +0200 Subject: [PATCH 08/21] install necessary packages in tests workflow --- .github/workflows/docker.yml | 1 + .github/workflows/pullpreview.yml | 1 + .github/workflows/tests.yml | 2 ++ 3 files changed, 4 insertions(+) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 4aa3212f..87802124 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -7,6 +7,7 @@ on: jobs: build-push-docker-image: name: Build, test and push docker image + # https://github.com/cybex-gmbh/github-workflows/blob/main/.github/workflows/docker-build-push.yml uses: cybex-gmbh/github-workflows/.github/workflows/docker-build-push.yml@main with: DOCKER_REPOSITORY: cybexwebdev/transmorpher diff --git a/.github/workflows/pullpreview.yml b/.github/workflows/pullpreview.yml index d05af41f..9a2dfe21 100644 --- a/.github/workflows/pullpreview.yml +++ b/.github/workflows/pullpreview.yml @@ -13,6 +13,7 @@ jobs: statuses: write # to create commit status name: Deploy PullPreview staging environment + # https://github.com/cybex-gmbh/github-workflows/blob/main/.github/workflows/pullpreview.yml uses: cybex-gmbh/github-workflows/.github/workflows/pullpreview.yml@main with: PULLPREVIEW_ADMINS: jheusinger, gael-connan-cybex, holyfabi, lupinitylabs, mszulik diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 800d2646..f99ea360 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -11,6 +11,7 @@ on: jobs: execute-tests: name: Setup testing environment and execute tests + # https://github.com/cybex-gmbh/github-workflows/blob/main/.github/workflows/tests.yml uses: cybex-gmbh/github-workflows/.github/workflows/tests.yml@main strategy: fail-fast: true @@ -23,3 +24,4 @@ jobs: LARAVEL_VERSION: ${{ matrix.laravel }} DEPENDENCY_VERSION: ${{ matrix.dependency-version }} MYSQL_DATABASE: transmorpher_test + LINUX_PACKAGES: imagemagick jpegoptim optipng pngquant gifsicle webp ffmpeg From 36af54df7126420a0fb32da092325cc685e5fc05 Mon Sep 17 00:00:00 2001 From: Marco Szulik Date: Mon, 29 Apr 2024 11:42:13 +0200 Subject: [PATCH 09/21] test whether upload slots are invalidated after an upload --- tests/Unit/ImageTest.php | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/Unit/ImageTest.php b/tests/Unit/ImageTest.php index 8b9e467c..b59e068a 100644 --- a/tests/Unit/ImageTest.php +++ b/tests/Unit/ImageTest.php @@ -84,6 +84,14 @@ public function ensureImageCanBeUploaded(string $uploadToken) return $version; } + #[Test] + #[Depends('ensureImageUploadSlotCanBeReserved')] + #[Depends('ensureImageCanBeUploaded')] + public function ensureUploadTokenIsInvalidatedAfterUpload(string $uploadToken) + { + $this->uploadImage($uploadToken)->assertNotFound(); + } + protected function createDerivativeForVersion(Version $version): TestResponse { return $this->get(route('getDerivative', [$this->user->name, $version->Media])); From 2b9753cc56dd7dddf0154d24d6cf3da54cda3837 Mon Sep 17 00:00:00 2001 From: Marco Szulik Date: Mon, 29 Apr 2024 11:49:30 +0200 Subject: [PATCH 10/21] fix an issue where clients were notified despite not purging any derivatives --- app/Console/Commands/PurgeDerivatives.php | 5 +++++ app/Jobs/ClientPurgeNotification.php | 2 +- docker/workers.conf | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/app/Console/Commands/PurgeDerivatives.php b/app/Console/Commands/PurgeDerivatives.php index 5c4a4d88..c941c75f 100644 --- a/app/Console/Commands/PurgeDerivatives.php +++ b/app/Console/Commands/PurgeDerivatives.php @@ -32,6 +32,11 @@ class PurgeDerivatives extends Command */ public function handle(): int { + if (!$this->option('image') && !$this->option('video') && !$this->option('all')) { + $this->warn(sprintf('No options provided. Call "php artisan %s --help" for a list of all options.', $this->name)); + return Command::SUCCESS; + } + foreach (MediaType::cases() as $mediaType) { if ($this->option('all') || $this->option($mediaType->value)) { ['success' => $success, 'message' => $message] = $mediaType->handler()->purgeDerivatives(); diff --git a/app/Jobs/ClientPurgeNotification.php b/app/Jobs/ClientPurgeNotification.php index fcbb7776..349aa298 100644 --- a/app/Jobs/ClientPurgeNotification.php +++ b/app/Jobs/ClientPurgeNotification.php @@ -29,7 +29,7 @@ class ClientPurgeNotification implements ShouldQueue * * @var int */ - public int $timeout = 300; + public int $timeout = 10; /** * The number of seconds to wait before retrying the job. diff --git a/docker/workers.conf b/docker/workers.conf index a2418a4b..ac680dd5 100644 --- a/docker/workers.conf +++ b/docker/workers.conf @@ -28,4 +28,4 @@ numprocs=1 redirect_stderr=true stdout_logfile=/dev/stdout ; Timeout of the longest running job (video transcoding with 10800) plus 30. -stopwaitsecs=330 +stopwaitsecs=40 From 02960a187a0c620c9ac18f3b3f7133eac9921c10 Mon Sep 17 00:00:00 2001 From: Marco Szulik Date: Mon, 29 Apr 2024 12:15:09 +0200 Subject: [PATCH 11/21] clarify some parts in the readme --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index f707754a..9bd1ec8c 100644 --- a/README.md +++ b/README.md @@ -102,8 +102,8 @@ For more information about scheduling, check the [Laravel Docs](https://laravel. #### Disks -The media server uses 3 separate Laravel disks to store originals, image derivatives and video derivatives. -Use the provided `.env` keys to select any of the disks in the `filesystems.php` config file. +The media server must use 3 separate Laravel disks to store originals, image derivatives and video derivatives. +Use the provided `.env` keys to select the according disks in the `filesystems.php` config file. > [!NOTE] > @@ -158,7 +158,7 @@ AWS_DEFAULT_REGION=eu-central-1 #### File Storage -By default, AWS S3 disks are configured in the `.env`: +To use AWS S3 disks set the according `.env` values: ```dotenv TRANSMORPHER_DISK_ORIGINALS=s3Originals From 16f94b0c45a322b80c7100a332178954929e9625 Mon Sep 17 00:00:00 2001 From: Marco Szulik Date: Mon, 29 Apr 2024 15:04:11 +0200 Subject: [PATCH 12/21] fix pullpreview seeder --- database/seeders/PullpreviewSeeder.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/database/seeders/PullpreviewSeeder.php b/database/seeders/PullpreviewSeeder.php index ed8287e0..233693da 100644 --- a/database/seeders/PullpreviewSeeder.php +++ b/database/seeders/PullpreviewSeeder.php @@ -15,7 +15,7 @@ class PullpreviewSeeder extends Seeder */ public function run(): void { - Artisan::call('create:user pullpreview pullpreview@example.com'); + Artisan::call('create:user pullpreview pullpreview@example.com http://pullpreview.test/transmorpher/notifications'); DB::table('personal_access_tokens')->where('id', 1)->update(['token' => env('PULLPREVIEW_TRANSMORPHER_AUTH_TOKEN_HASH')]); } From cc396536b42da55c7904f74089ee2f88c2e8d692 Mon Sep 17 00:00:00 2001 From: Marco Szulik Date: Mon, 29 Apr 2024 16:07:10 +0200 Subject: [PATCH 13/21] add "PULLPREVIEW" prefix to github secrets used for pullpreview --- .github/workflows/pullpreview.yml | 6 +++--- README.md | 6 +++--- database/seeders/PullpreviewSeeder.php | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/pullpreview.yml b/.github/workflows/pullpreview.yml index 9a2dfe21..e690677e 100644 --- a/.github/workflows/pullpreview.yml +++ b/.github/workflows/pullpreview.yml @@ -20,8 +20,8 @@ jobs: INSTANCE_TYPE: medium secrets: ENV_VARS: | - APP_KEY="${{ secrets.APP_KEY }}" - TRANSMORPHER_SIGNING_KEYPAIR="${{ secrets.TRANSMORPHER_SIGNING_KEYPAIR }}" - PULLPREVIEW_TRANSMORPHER_AUTH_TOKEN_HASH="${{ secrets.PULLPREVIEW_TRANSMORPHER_AUTH_TOKEN_HASH }}" + APP_KEY="${{ secrets.PULLPREVIEW_APP_KEY }}" + TRANSMORPHER_SIGNING_KEYPAIR="${{ secrets.PULLPREVIEW_TRANSMORPHER_SIGNING_KEYPAIR }}" + TRANSMORPHER_AUTH_TOKEN_HASH="${{ secrets.PULLPREVIEW_TRANSMORPHER_AUTH_TOKEN_HASH }}" PULLPREVIEW_AWS_ACCESS_KEY_ID: ${{ secrets.PULLPREVIEW_AWS_ACCESS_KEY_ID }} PULLPREVIEW_AWS_SECRET_ACCESS_KEY: ${{ secrets.PULLPREVIEW_AWS_SECRET_ACCESS_KEY }} diff --git a/README.md b/README.md index 9bd1ec8c..e8b77722 100644 --- a/README.md +++ b/README.md @@ -445,8 +445,8 @@ For more information, take a look at the PullPreview section of the [github-work App-specific GitHub Secrets: -- APP_KEY -- TRANSMORPHER_SIGNING_KEYPAIR +- PULLPREVIEW_APP_KEY +- PULLPREVIEW_TRANSMORPHER_SIGNING_KEYPAIR - PULLPREVIEW_TRANSMORPHER_AUTH_TOKEN_HASH #### Auth Token Hash @@ -454,7 +454,7 @@ App-specific GitHub Secrets: The environment is seeded with a user with an auth token. To get access, you will have to locally create a token and use this token and its hash. ```bash -php artisan create:user pullpreview pullpreview@example.com +php artisan create:user pullpreview pullpreview@example.com http://pullpreview.test/transmorpher/notifications ``` Take the hash of the token from the `personal_access_tokens` table and save it to GitHub secrets. The command also provides a `TRANSMORPHER_AUTH_TOKEN`, which should be stored diff --git a/database/seeders/PullpreviewSeeder.php b/database/seeders/PullpreviewSeeder.php index 233693da..aff5f5ae 100644 --- a/database/seeders/PullpreviewSeeder.php +++ b/database/seeders/PullpreviewSeeder.php @@ -17,6 +17,6 @@ public function run(): void { Artisan::call('create:user pullpreview pullpreview@example.com http://pullpreview.test/transmorpher/notifications'); - DB::table('personal_access_tokens')->where('id', 1)->update(['token' => env('PULLPREVIEW_TRANSMORPHER_AUTH_TOKEN_HASH')]); + DB::table('personal_access_tokens')->where('id', 1)->update(['token' => env('TRANSMORPHER_AUTH_TOKEN_HASH')]); } } From 475cd4c99af2fa26d5bca7b6fe1540f3eb3ef29b Mon Sep 17 00:00:00 2001 From: Marco Szulik Date: Tue, 30 Apr 2024 08:57:23 +0200 Subject: [PATCH 14/21] remove unused import --- app/Http/Controllers/V1/UploadSlotController.php | 2 -- 1 file changed, 2 deletions(-) diff --git a/app/Http/Controllers/V1/UploadSlotController.php b/app/Http/Controllers/V1/UploadSlotController.php index 92194ec5..af31198e 100644 --- a/app/Http/Controllers/V1/UploadSlotController.php +++ b/app/Http/Controllers/V1/UploadSlotController.php @@ -7,10 +7,8 @@ use App\Enums\ResponseState; use App\Enums\UploadState; use App\Http\Controllers\Controller; -use App\Http\Requests\V1\ImageUploadSlotRequest; use App\Http\Requests\V1\UploadRequest; use App\Http\Requests\V1\UploadSlotRequest; -use App\Http\Requests\V1\VideoUploadSlotRequest; use App\Models\UploadSlot; use App\Models\User; use File; From ea1dc20a42a0fc930094e8c9054ab76626605e71 Mon Sep 17 00:00:00 2001 From: Marco Szulik Date: Tue, 30 Apr 2024 12:57:24 +0200 Subject: [PATCH 15/21] add a recovery section in the readme --- README.md | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/README.md b/README.md index e8b77722..c3ccdcac 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,12 @@ To clone the repository and get your media server running, use: git clone --branch release/v0 --single-branch https://github.com/cybex-gmbh/transmorpher.git ``` +Install composer dependencies: + +```bash +composer install --no-dev +``` + #### Required software See the Dockerfiles for details. @@ -100,6 +106,22 @@ For more information about scheduling, check the [Laravel Docs](https://laravel. ## General configuration +#### Basics + +1. Create an app key: + +```bash +php artisan key:generate +``` + +2. Configure the database in the `.env` file. + +3. Migrate the database: + +```bash +php artisan migrate +``` + #### Disks The media server must use 3 separate Laravel disks to store originals, image derivatives and video derivatives. @@ -437,6 +459,24 @@ Image derivatives will be deleted, for video derivatives we dispatch a new trans The derivatives revision is available on the route `/api/v*/cacheInvalidationRevision`. +## Recovery + +To restore operation of the server, restore the following: + +- database +- the `originals` disk +- `.env` file* +- the `image derivatives` disk* +- the `video derivatives` disk* + +> Marked with * are optional, but recommended. + +If the `.env` file is lost follow the setup instructions above, including creating a new signing keypair. + +If video derivatives are lost, use the [purge command](#purging-derivatives) to restore them. + +Lost image derivatives will automatically be re-generated on demand. + ## Development ### [Pullpreview](https://github.com/pullpreview/action) From 7f49f4df8a067939e57ad011b35bdc0e3a37e68e Mon Sep 17 00:00:00 2001 From: Marco Szulik Date: Tue, 30 Apr 2024 15:24:44 +0200 Subject: [PATCH 16/21] move cache invalidation file path to a config value --- .env.example | 1 + _ide_helper.php | 58 ++++++++++++++++++++++- app/Console/Commands/PurgeDerivatives.php | 2 +- app/Enums/MediaStorage.php | 10 ---- config/transmorpher.php | 10 ++++ routes/api/v1.php | 2 +- tests/Unit/ImageTest.php | 4 +- 7 files changed, 71 insertions(+), 16 deletions(-) diff --git a/.env.example b/.env.example index 5a87eeb1..74e218ca 100644 --- a/.env.example +++ b/.env.example @@ -21,6 +21,7 @@ TRANSMORPHER_SIGNING_KEYPAIR= TRANSMORPHER_OPTIMIZER_TIMEOUT=10 # More information: https://github.com/cybex-gmbh/transmorpher/tree/release/v0#configuration-options VIDEO_TRANSCODING_WORKERS_AMOUNT=1 +#CACHE_INVALIDATION_FILE_PATH="cacheInvalidationRevision" # AWS AWS_ACCESS_KEY_ID= diff --git a/_ide_helper.php b/_ide_helper.php index 259252ca..1c360ae7 100644 --- a/_ide_helper.php +++ b/_ide_helper.php @@ -5,7 +5,7 @@ /** * A helper file for Laravel, to provide autocomplete information to your IDE - * Generated for Laravel 11.4.0. + * Generated for Laravel 11.5.0. * * This file should not be included in your code, only analyzed by your IDE! * @@ -2889,6 +2889,33 @@ { /** @var \Illuminate\Broadcasting\BroadcastManager $instance */ return $instance->socket($request); + } + /** + * Begin sending an anonymous broadcast to the given channels. + * + * @static + */ public static function on($channels) + { + /** @var \Illuminate\Broadcasting\BroadcastManager $instance */ + return $instance->on($channels); + } + /** + * Begin sending an anonymous broadcast to the given private channels. + * + * @static + */ public static function private($channel) + { + /** @var \Illuminate\Broadcasting\BroadcastManager $instance */ + return $instance->private($channel); + } + /** + * Begin sending an anonymous broadcast to the given presence channels. + * + * @static + */ public static function presence($channel) + { + /** @var \Illuminate\Broadcasting\BroadcastManager $instance */ + return $instance->presence($channel); } /** * Begin broadcasting an event. @@ -10507,6 +10534,19 @@ { /** @var \Illuminate\Cache\RateLimiter $instance */ return $instance->increment($key, $decaySeconds, $amount); + } + /** + * Decrement the counter for a given key for a given decay time by a given amount. + * + * @param string $key + * @param int $decaySeconds + * @param int $amount + * @return int + * @static + */ public static function decrement($key, $decaySeconds = 60, $amount = 1) + { + /** @var \Illuminate\Cache\RateLimiter $instance */ + return $instance->decrement($key, $decaySeconds, $amount); } /** * Get the number of attempts for the given key. @@ -11113,7 +11153,7 @@ return $instance->mergeIfMissing($input); } /** - * Replace the input for the current request. + * Replace the input values for the current request. * * @param array $input * @return \Illuminate\Http\Request @@ -16219,6 +16259,20 @@ { /** @var \Illuminate\Routing\UrlGenerator $instance */ return $instance->to($path, $extra, $secure); + } + /** + * Generate an absolute URL with the given query parameters. + * + * @param string $path + * @param array $query + * @param mixed $extra + * @param bool|null $secure + * @return string + * @static + */ public static function query($path, $query = [], $extra = [], $secure = null) + { + /** @var \Illuminate\Routing\UrlGenerator $instance */ + return $instance->query($path, $query, $extra, $secure); } /** * Generate a secure, absolute URL to the given path. diff --git a/app/Console/Commands/PurgeDerivatives.php b/app/Console/Commands/PurgeDerivatives.php index c941c75f..928100a7 100644 --- a/app/Console/Commands/PurgeDerivatives.php +++ b/app/Console/Commands/PurgeDerivatives.php @@ -45,7 +45,7 @@ public function handle(): int } $originalsDisk = MediaStorage::ORIGINALS->getDisk(); - $cacheInvalidationFilePath = MediaStorage::getCacheInvalidationFilePath(); + $cacheInvalidationFilePath = config('transmorpher.cache_invalidation_file_path'); if (!$originalsDisk->put($cacheInvalidationFilePath, $originalsDisk->get($cacheInvalidationFilePath) + 1)) { $this->error(sprintf('Failed to update cache invalidation revision at path %s on disk %s', $cacheInvalidationFilePath, MediaStorage::ORIGINALS->value)); diff --git a/app/Enums/MediaStorage.php b/app/Enums/MediaStorage.php index c90458e9..f72b0038 100644 --- a/app/Enums/MediaStorage.php +++ b/app/Enums/MediaStorage.php @@ -20,14 +20,4 @@ public function getDisk(): Filesystem { return Storage::disk(config(sprintf('transmorpher.disks.%s', $this->value))); } - - /** - * Returns the file path to the cache invalidation file in which the current revision is stored. - * - * @return string - */ - public static function getCacheInvalidationFilePath(): string - { - return 'cacheInvalidationRevision'; - } } diff --git a/config/transmorpher.php b/config/transmorpher.php index 508aa2ac..8ec48f88 100644 --- a/config/transmorpher.php +++ b/config/transmorpher.php @@ -182,4 +182,14 @@ 'image' => App\Classes\MediaHandler\ImageHandler::class, 'video' => App\Classes\MediaHandler\VideoHandler::class ], + + /* + |-------------------------------------------------------------------------- + | Cache Invalidation File Path + |-------------------------------------------------------------------------- + | + | The path to a file on the originals disk that stores the current cache invalidation revision number. + | + */ + 'cache_invalidation_file_path' => env('CACHE_INVALIDATION_FILE_PATH', 'cacheInvalidationRevision'), ]; diff --git a/routes/api/v1.php b/routes/api/v1.php index 41652785..3ea3e243 100644 --- a/routes/api/v1.php +++ b/routes/api/v1.php @@ -37,5 +37,5 @@ function () { Route::post('/upload/{uploadSlot}', [UploadSlotController::class, 'receiveFile'])->name('upload'); Route::get('publickey', fn(): string => SodiumHelper::getPublicKey())->name('getPublicKey'); - Route::get('cacheInvalidationRevision', fn(): string => MediaStorage::ORIGINALS->getDisk()->get(MediaStorage::getCacheInvalidationFilePath()) ?? 0)->name('getCacheInvalidationRevision'); + Route::get('cacheInvalidationRevision', fn(): string => MediaStorage::ORIGINALS->getDisk()->get(config('transmorpher.cache_invalidation_file_path')) ?? 0)->name('getCacheInvalidationRevision'); }); diff --git a/tests/Unit/ImageTest.php b/tests/Unit/ImageTest.php index b59e068a..834878db 100644 --- a/tests/Unit/ImageTest.php +++ b/tests/Unit/ImageTest.php @@ -197,7 +197,7 @@ public function ensureImageDerivativesArePurged() $this->assertVersionFilesExist($this->version); - $cacheRevisionBeforeCommand = $this->originalsDisk->get(MediaStorage::getCacheInvalidationFilePath()); + $cacheRevisionBeforeCommand = $this->originalsDisk->get(config('transmorpher.cache_invalidation_file_path')); Http::fake([ $this->user->api_url => Http::response() @@ -205,7 +205,7 @@ public function ensureImageDerivativesArePurged() Artisan::call(PurgeDerivatives::class, ['--image' => true]); - $cacheRevisionAfterCommand = $this->originalsDisk->get(MediaStorage::getCacheInvalidationFilePath()); + $cacheRevisionAfterCommand = $this->originalsDisk->get(config('transmorpher.cache_invalidation_file_path')); Http::assertSent(function (Request $request) use ($cacheRevisionAfterCommand) { $decryptedNotification = json_decode(SodiumHelper::decrypt($request['signed_notification']), true); From 4e375f8179c048e612ca621cfc5eaaa0a7747b01 Mon Sep 17 00:00:00 2001 From: Marco Szulik Date: Tue, 30 Apr 2024 15:31:19 +0200 Subject: [PATCH 17/21] rename cache invalidation revision to cache invalidation counter --- .env.example | 2 +- README.md | 2 +- app/Console/Commands/PurgeDerivatives.php | 10 +++++----- app/Jobs/ClientPurgeNotification.php | 4 ++-- config/transmorpher.php | 6 +++--- routes/api/v1.php | 2 +- tests/Unit/ImageTest.php | 10 +++++----- 7 files changed, 18 insertions(+), 18 deletions(-) diff --git a/.env.example b/.env.example index 74e218ca..fd1f8214 100644 --- a/.env.example +++ b/.env.example @@ -21,7 +21,7 @@ TRANSMORPHER_SIGNING_KEYPAIR= TRANSMORPHER_OPTIMIZER_TIMEOUT=10 # More information: https://github.com/cybex-gmbh/transmorpher/tree/release/v0#configuration-options VIDEO_TRANSCODING_WORKERS_AMOUNT=1 -#CACHE_INVALIDATION_FILE_PATH="cacheInvalidationRevision" +#CACHE_INVALIDATION_COUNTER_FILE_PATH="cacheInvalidationCounter" # AWS AWS_ACCESS_KEY_ID= diff --git a/README.md b/README.md index c3ccdcac..ee064e71 100644 --- a/README.md +++ b/README.md @@ -457,7 +457,7 @@ php artisan purge:derivatives The command accepts the options `--image`, `--video` and `--all` (or `-a`) for purging the respective derivatives. Image derivatives will be deleted, for video derivatives we dispatch a new transcoding job for the current version. -The derivatives revision is available on the route `/api/v*/cacheInvalidationRevision`. +The derivatives revision is available on the route `/api/v*/cacheInvalidationCounter`. ## Recovery diff --git a/app/Console/Commands/PurgeDerivatives.php b/app/Console/Commands/PurgeDerivatives.php index 928100a7..300660bd 100644 --- a/app/Console/Commands/PurgeDerivatives.php +++ b/app/Console/Commands/PurgeDerivatives.php @@ -25,7 +25,7 @@ class PurgeDerivatives extends Command * * @var string */ - protected $description = 'Purge all derivatives, increment the cacheInvalidationRevision and notify all clients'; + protected $description = 'Purge all derivatives, increment the cache invalidation counter and notify all clients'; /** * Execute the console command. @@ -45,14 +45,14 @@ public function handle(): int } $originalsDisk = MediaStorage::ORIGINALS->getDisk(); - $cacheInvalidationFilePath = config('transmorpher.cache_invalidation_file_path'); + $cacheInvalidationCounterFilePath = config('transmorpher.cache_invalidation_counter_file_path'); - if (!$originalsDisk->put($cacheInvalidationFilePath, $originalsDisk->get($cacheInvalidationFilePath) + 1)) { - $this->error(sprintf('Failed to update cache invalidation revision at path %s on disk %s', $cacheInvalidationFilePath, MediaStorage::ORIGINALS->value)); + if (!$originalsDisk->put($cacheInvalidationCounterFilePath, $originalsDisk->get($cacheInvalidationCounterFilePath) + 1)) { + $this->error(sprintf('Failed to update cache invalidation counter at path %s on disk %s', $cacheInvalidationCounterFilePath, MediaStorage::ORIGINALS->value)); } foreach (User::get() as $user) { - ClientPurgeNotification::dispatch($user, $originalsDisk->get($cacheInvalidationFilePath)); + ClientPurgeNotification::dispatch($user, $originalsDisk->get($cacheInvalidationCounterFilePath)); } return Command::SUCCESS; diff --git a/app/Jobs/ClientPurgeNotification.php b/app/Jobs/ClientPurgeNotification.php index 349aa298..2c39dea2 100644 --- a/app/Jobs/ClientPurgeNotification.php +++ b/app/Jobs/ClientPurgeNotification.php @@ -43,7 +43,7 @@ class ClientPurgeNotification implements ShouldQueue /** * Create a new job instance. */ - public function __construct(protected User $user, protected int $cacheInvalidationRevision) + public function __construct(protected User $user, protected int $cacheInvalidationCounter) { $this->onQueue('client-notifications'); } @@ -56,7 +56,7 @@ public function handle(): void { $notification = [ 'notification_type' => $this->notificationType, - 'cache_invalidation_revision' => $this->cacheInvalidationRevision + 'cache_invalidation_counter' => $this->cacheInvalidationCounter ]; $signedNotification = SodiumHelper::sign(json_encode($notification)); diff --git a/config/transmorpher.php b/config/transmorpher.php index 8ec48f88..20a88396 100644 --- a/config/transmorpher.php +++ b/config/transmorpher.php @@ -185,11 +185,11 @@ /* |-------------------------------------------------------------------------- - | Cache Invalidation File Path + | Cache Invalidation Counter File Path |-------------------------------------------------------------------------- | - | The path to a file on the originals disk that stores the current cache invalidation revision number. + | The path to a file on the originals disk that stores the cache invalidation counter. | */ - 'cache_invalidation_file_path' => env('CACHE_INVALIDATION_FILE_PATH', 'cacheInvalidationRevision'), + 'cache_invalidation_counter_file_path' => env('CACHE_INVALIDATION_COUNTER_FILE_PATH', 'cacheInvalidationCounter'), ]; diff --git a/routes/api/v1.php b/routes/api/v1.php index 3ea3e243..6d66ae26 100644 --- a/routes/api/v1.php +++ b/routes/api/v1.php @@ -37,5 +37,5 @@ function () { Route::post('/upload/{uploadSlot}', [UploadSlotController::class, 'receiveFile'])->name('upload'); Route::get('publickey', fn(): string => SodiumHelper::getPublicKey())->name('getPublicKey'); - Route::get('cacheInvalidationRevision', fn(): string => MediaStorage::ORIGINALS->getDisk()->get(config('transmorpher.cache_invalidation_file_path')) ?? 0)->name('getCacheInvalidationRevision'); + Route::get('cacheInvalidationCounter', fn(): string => MediaStorage::ORIGINALS->getDisk()->get(config('transmorpher.cache_invalidation_counter_file_path')) ?? 0)->name('getCacheInvalidationCounter'); }); diff --git a/tests/Unit/ImageTest.php b/tests/Unit/ImageTest.php index 834878db..318cd7a8 100644 --- a/tests/Unit/ImageTest.php +++ b/tests/Unit/ImageTest.php @@ -197,7 +197,7 @@ public function ensureImageDerivativesArePurged() $this->assertVersionFilesExist($this->version); - $cacheRevisionBeforeCommand = $this->originalsDisk->get(config('transmorpher.cache_invalidation_file_path')); + $cacheCounterBeforeCommand = $this->originalsDisk->get(config('transmorpher.cache_invalidation_counter_file_path')); Http::fake([ $this->user->api_url => Http::response() @@ -205,17 +205,17 @@ public function ensureImageDerivativesArePurged() Artisan::call(PurgeDerivatives::class, ['--image' => true]); - $cacheRevisionAfterCommand = $this->originalsDisk->get(config('transmorpher.cache_invalidation_file_path')); + $cacheCounterAfterCommand = $this->originalsDisk->get(config('transmorpher.cache_invalidation_counter_file_path')); - Http::assertSent(function (Request $request) use ($cacheRevisionAfterCommand) { + Http::assertSent(function (Request $request) use ($cacheCounterAfterCommand) { $decryptedNotification = json_decode(SodiumHelper::decrypt($request['signed_notification']), true); return $request->url() == $this->user->api_url && $decryptedNotification['notification_type'] == ClientNotification::CACHE_INVALIDATION->value - && $decryptedNotification['cache_invalidation_revision'] == $cacheRevisionAfterCommand; + && $decryptedNotification['cache_invalidation_counter'] == $cacheCounterAfterCommand; }); - $this->assertTrue(++$cacheRevisionBeforeCommand == $cacheRevisionAfterCommand); + $this->assertTrue(++$cacheCounterBeforeCommand == $cacheCounterAfterCommand); $this->imageDerivativesDisk->assertMissing($this->version->imageDerivativeFilePath()); } From d51ecf41dd6c5e48105a5143f6ad8bac089a33ed Mon Sep 17 00:00:00 2001 From: Marco Szulik Date: Tue, 30 Apr 2024 15:36:57 +0200 Subject: [PATCH 18/21] rename route to be comprehensible from a client perspective --- README.md | 2 +- routes/api/v1.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index ee064e71..6e3c92b9 100644 --- a/README.md +++ b/README.md @@ -457,7 +457,7 @@ php artisan purge:derivatives The command accepts the options `--image`, `--video` and `--all` (or `-a`) for purging the respective derivatives. Image derivatives will be deleted, for video derivatives we dispatch a new transcoding job for the current version. -The derivatives revision is available on the route `/api/v*/cacheInvalidationCounter`. +The derivatives revision is available on the route `/api/v*/cacheInvalidator`. ## Recovery diff --git a/routes/api/v1.php b/routes/api/v1.php index 6d66ae26..6503fd8a 100644 --- a/routes/api/v1.php +++ b/routes/api/v1.php @@ -37,5 +37,5 @@ function () { Route::post('/upload/{uploadSlot}', [UploadSlotController::class, 'receiveFile'])->name('upload'); Route::get('publickey', fn(): string => SodiumHelper::getPublicKey())->name('getPublicKey'); - Route::get('cacheInvalidationCounter', fn(): string => MediaStorage::ORIGINALS->getDisk()->get(config('transmorpher.cache_invalidation_counter_file_path')) ?? 0)->name('getCacheInvalidationCounter'); + Route::get('cacheInvalidator', fn(): string => MediaStorage::ORIGINALS->getDisk()->get(config('transmorpher.cache_invalidation_counter_file_path')) ?? 0)->name('getCacheInvalidator'); }); From b9c7bbc222b4ec7c85c6c2ec55846fbcd2f547ab Mon Sep 17 00:00:00 2001 From: Marco Szulik Date: Tue, 30 Apr 2024 15:48:03 +0200 Subject: [PATCH 19/21] rename response key for cache invalidator --- app/Jobs/ClientPurgeNotification.php | 2 +- tests/Unit/ImageTest.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/Jobs/ClientPurgeNotification.php b/app/Jobs/ClientPurgeNotification.php index 2c39dea2..7e1a6dce 100644 --- a/app/Jobs/ClientPurgeNotification.php +++ b/app/Jobs/ClientPurgeNotification.php @@ -56,7 +56,7 @@ public function handle(): void { $notification = [ 'notification_type' => $this->notificationType, - 'cache_invalidation_counter' => $this->cacheInvalidationCounter + 'cache_invalidator' => $this->cacheInvalidationCounter ]; $signedNotification = SodiumHelper::sign(json_encode($notification)); diff --git a/tests/Unit/ImageTest.php b/tests/Unit/ImageTest.php index 318cd7a8..10372d33 100644 --- a/tests/Unit/ImageTest.php +++ b/tests/Unit/ImageTest.php @@ -212,7 +212,7 @@ public function ensureImageDerivativesArePurged() return $request->url() == $this->user->api_url && $decryptedNotification['notification_type'] == ClientNotification::CACHE_INVALIDATION->value - && $decryptedNotification['cache_invalidation_counter'] == $cacheCounterAfterCommand; + && $decryptedNotification['cache_invalidator'] == $cacheCounterAfterCommand; }); $this->assertTrue(++$cacheCounterBeforeCommand == $cacheCounterAfterCommand); From d07c14b9212ec7d1b61a162d325a5c78d7c1fd36 Mon Sep 17 00:00:00 2001 From: mszulik <69617961+mszulik@users.noreply.github.com> Date: Tue, 30 Apr 2024 15:53:51 +0200 Subject: [PATCH 20/21] adjust comment for client notifications worker Co-authored-by: Gael Connan --- docker/workers.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/workers.conf b/docker/workers.conf index ac680dd5..930d1fde 100644 --- a/docker/workers.conf +++ b/docker/workers.conf @@ -27,5 +27,5 @@ killasgroup=true numprocs=1 redirect_stderr=true stdout_logfile=/dev/stdout -; Timeout of the longest running job (video transcoding with 10800) plus 30. +; Timeout of the longest running job (purge notifications with 10) plus 30. stopwaitsecs=40 From 4e0f98705a3dda6a724ed0957b80a43179ca1a2f Mon Sep 17 00:00:00 2001 From: Marco Szulik Date: Fri, 3 May 2024 12:49:50 +0200 Subject: [PATCH 21/21] fix an issue where the client notifications worker used the wrong queue --- docker/workers.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/workers.conf b/docker/workers.conf index 930d1fde..b5825b6a 100644 --- a/docker/workers.conf +++ b/docker/workers.conf @@ -19,7 +19,7 @@ process_name=%(program_name)s_%(process_num)02d ; Supervisor starts programs as root by default, which might lead to permission problems when the webserver tries to access files or similar. user=application environment=HOME="/home/application",USER="application" -command=php /var/www/html/artisan queue:work --queue=purge-notifications +command=php /var/www/html/artisan queue:work --queue=client-notifications autostart=true autorestart=true stopasgroup=true