diff --git a/.env.example b/.env.example index 5a87eeb1..fd1f8214 100644 --- a/.env.example +++ b/.env.example @@ -21,6 +21,7 @@ TRANSMORPHER_SIGNING_KEYPAIR= TRANSMORPHER_OPTIMIZER_TIMEOUT=10 # More information: https://github.com/cybex-gmbh/transmorpher/tree/release/v0#configuration-options VIDEO_TRANSCODING_WORKERS_AMOUNT=1 +#CACHE_INVALIDATION_COUNTER_FILE_PATH="cacheInvalidationCounter" # AWS AWS_ACCESS_KEY_ID= diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 4aa3212f..87802124 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -7,6 +7,7 @@ on: jobs: build-push-docker-image: name: Build, test and push docker image + # https://github.com/cybex-gmbh/github-workflows/blob/main/.github/workflows/docker-build-push.yml uses: cybex-gmbh/github-workflows/.github/workflows/docker-build-push.yml@main with: DOCKER_REPOSITORY: cybexwebdev/transmorpher diff --git a/.github/workflows/pullpreview.yml b/.github/workflows/pullpreview.yml index d05af41f..e690677e 100644 --- a/.github/workflows/pullpreview.yml +++ b/.github/workflows/pullpreview.yml @@ -13,14 +13,15 @@ jobs: statuses: write # to create commit status name: Deploy PullPreview staging environment + # https://github.com/cybex-gmbh/github-workflows/blob/main/.github/workflows/pullpreview.yml uses: cybex-gmbh/github-workflows/.github/workflows/pullpreview.yml@main with: PULLPREVIEW_ADMINS: jheusinger, gael-connan-cybex, holyfabi, lupinitylabs, mszulik INSTANCE_TYPE: medium secrets: ENV_VARS: | - APP_KEY="${{ secrets.APP_KEY }}" - TRANSMORPHER_SIGNING_KEYPAIR="${{ secrets.TRANSMORPHER_SIGNING_KEYPAIR }}" - PULLPREVIEW_TRANSMORPHER_AUTH_TOKEN_HASH="${{ secrets.PULLPREVIEW_TRANSMORPHER_AUTH_TOKEN_HASH }}" + APP_KEY="${{ secrets.PULLPREVIEW_APP_KEY }}" + TRANSMORPHER_SIGNING_KEYPAIR="${{ secrets.PULLPREVIEW_TRANSMORPHER_SIGNING_KEYPAIR }}" + TRANSMORPHER_AUTH_TOKEN_HASH="${{ secrets.PULLPREVIEW_TRANSMORPHER_AUTH_TOKEN_HASH }}" PULLPREVIEW_AWS_ACCESS_KEY_ID: ${{ secrets.PULLPREVIEW_AWS_ACCESS_KEY_ID }} PULLPREVIEW_AWS_SECRET_ACCESS_KEY: ${{ secrets.PULLPREVIEW_AWS_SECRET_ACCESS_KEY }} diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 800d2646..f99ea360 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -11,6 +11,7 @@ on: jobs: execute-tests: name: Setup testing environment and execute tests + # https://github.com/cybex-gmbh/github-workflows/blob/main/.github/workflows/tests.yml uses: cybex-gmbh/github-workflows/.github/workflows/tests.yml@main strategy: fail-fast: true @@ -23,3 +24,4 @@ jobs: LARAVEL_VERSION: ${{ matrix.laravel }} DEPENDENCY_VERSION: ${{ matrix.dependency-version }} MYSQL_DATABASE: transmorpher_test + LINUX_PACKAGES: imagemagick jpegoptim optipng pngquant gifsicle webp ffmpeg diff --git a/README.md b/README.md index a3f8a292..6e3c92b9 100644 --- a/README.md +++ b/README.md @@ -29,15 +29,16 @@ To not accidentally upgrade to a new major version, attach the major version you #### Configuration options -There needs to be at least 1 Laravel worker to transcode videos. The following variable specifies how many workers should be running in the container: +There needs to be at least 1 Laravel worker to transcode videos. +The following variable specifies how many workers should be running in the container: ```dotenv VIDEO_TRANSCODING_WORKERS_AMOUNT=1 ``` > [!CAUTION] -> Using the database queue connection does neither guarantee FIFO nor prevent duplicate runs. It is recommended to use a queue which can guarantee these aspects, such as AWS SQS -> FIFO. +> Using the database queue connection does neither guarantee FIFO nor prevent duplicate runs. +> It is recommended to use a queue which can guarantee these aspects, such as AWS SQS FIFO. > To prevent duplicate runs with database, use only one worker process. This environment variable has to be passed to the app container in your docker-compose.yml: @@ -49,12 +50,18 @@ environment: ### Cloning the repository -To clone the repository and get your media server running use: +To clone the repository and get your media server running, use: ```bash git clone --branch release/v0 --single-branch https://github.com/cybex-gmbh/transmorpher.git ``` +Install composer dependencies: + +```bash +composer install --no-dev +``` + #### Required software See the Dockerfiles for details. @@ -64,13 +71,13 @@ Image manipulation: - [ImageMagick](https://imagemagick.org/index.php) - [php-imagick](https://www.php.net/manual/en/book.imagick.php) -> Optionally you can use GD, which can be configured in the Intervention Image configuration file. +> Optionally, you can use GD, which can be configured in the Intervention Image configuration file. Image optimization: - [JpegOptim](https://github.com/tjko/jpegoptim) - [Optipng](https://optipng.sourceforge.net/) -- [Pngquant 2](https://pngquant.org/) +- [Pngquant](https://pngquant.org/) - [Gifsicle](https://www.lcdf.org/gifsicle/) - [cwebp](https://developers.google.com/speed/webp/docs/precompiled) @@ -78,6 +85,10 @@ To use video transcoding: - [FFmpeg](https://ffmpeg.org/) +#### Generic workers + +Client notifications will be pushed on the queue `client-notifications`. You will need to set up 1 worker for this queue. + #### Scheduling There may be some cases (e.g. failed uploads) where chunk files are not deleted and stay on the local disk. @@ -85,38 +96,73 @@ To keep the local disk clean, a command is scheduled hourly to delete chunk file See the [`chunk-upload` configuration file](config/chunk-upload.php) for more information. -To run the scheduler you will need to add a cron job that runs the `schedule:run` command on your server: +To run the scheduler, you will need to add a cron job that runs the `schedule:run` command on your server: ``` * * * * * cd /path-to-your-project && php artisan schedule:run >> /dev/null 2>&1 ``` -For more information about scheduling check the [Laravel Docs](https://laravel.com/docs/11.x/scheduling). +For more information about scheduling, check the [Laravel Docs](https://laravel.com/docs/11.x/scheduling). ## General configuration +#### Basics + +1. Create an app key: + +```bash +php artisan key:generate +``` + +2. Configure the database in the `.env` file. + +3. Migrate the database: + +```bash +php artisan migrate +``` + #### Disks -The media server uses 3 separate Laravel disks to store originals, image derivatives and video derivatives. Use the provided `.env` keys to select any of the disks in -the `filesystems.php` config file. +The media server must use 3 separate Laravel disks to store originals, image derivatives and video derivatives. +Use the provided `.env` keys to select the according disks in the `filesystems.php` config file. > [!NOTE] > > 1. The root folder, like images/, of the configured derivatives disks has to always match the prefix provided by the `MediaType` enum. > 1. If this prefix would be changed after initially launching your media server, clients would no longer be able to retrieve their previously uploaded media. +#### Sodium Keypair + +A signed request is used to notify clients about finished transcodings and when derivatives are purged. +For this, a [Sodium](https://www.php.net/manual/en/book.sodium.php) keypair has to be configured. + +To create a keypair, use the provided command: + +```bash +php artisan create:keypair +``` + +The newly created keypair has to be written in the `.env` file: + +```dotenv +TRANSMORPHER_SIGNING_KEYPAIR= +``` + +The public key of the media server is available under the `/api/v*/publickey` endpoint and can be requested by any client. + ### Cloud Setup The Transmorpher media server is not dependent on a specific cloud service provider, but only provides classes for AWS services out of the box. #### Prerequisites for video functionality -- A file storage, for example AWS S3 -- A routing capable service, for example a Content Delivery Network, like AWS CloudFront +- A file storage, for example, AWS S3 +- A routing-capable service, for example, a Content Delivery Network, like AWS CloudFront #### IAM -Create an IAM user with programmatic access. For more information check the documentation for the corresponding service. +Create an IAM user with programmatic access. For more information, check the documentation for the corresponding service. Permissions: @@ -134,7 +180,7 @@ AWS_DEFAULT_REGION=eu-central-1 #### File Storage -By default, AWS S3 disks are configured in the `.env`: +To use AWS S3 disks set the according `.env` values: ```dotenv TRANSMORPHER_DISK_ORIGINALS=s3Originals @@ -165,13 +211,13 @@ Configure your CloudFront-Distribution-ID: AWS_CLOUDFRONT_DISTRIBUTION_ID= ``` -Changes to media will automatically trigger a cache invalidation, therefore the CDN cache duration can be set to a long time. +Changes to media will automatically trigger a cache invalidation. Therefore, the CDN cache duration can be set to a long time. To forward incoming requests from the CDN to your media server, configure your Transmorpher media server as the main origin. For more information on configuring origins in CloudFront see the [documentation page](https://docs.aws.amazon.com/AmazonCloudFront/latest/DeveloperGuide/DownloadDistS3AndCustomOrigins.html). -In order to properly use the API you need to either: +To properly use the API, you need to either: 1. add a rule to not cache anything under `/api/*` 1. publish the Transmorpher media server under an additional domain that is not behind the CDN @@ -180,31 +226,13 @@ In order to properly use the API you need to either: *Content Delivery Network* -In the CDN routing create a new behavior which points requests starting with "/videos/*" to a new origin, which is the video derivatives S3 bucket. -\ -*Sodium Keypair* - -A signed request is used to notify clients about finished transcodings. For this, a [Sodium](https://www.php.net/manual/en/book.sodium.php) keypair has to be configured. - -To create a keypair, simply use the provided command: - -```bash -php artisan transmorpher:keypair -``` +In the CDN routing create a new behavior which points requests starting with "/videos/*" to a new origin, which is the video derivatives S3 bucket. -The newly created keypair has to be written in the `.env` file: - -```dotenv -TRANSMORPHER_SIGNING_KEYPAIR= -``` - -The public key of the media server is available under the `/api/v*/publickey` endpoint and can be requested -by any client. -\ *Queue* -Transcoding jobs are dispatched onto the "video-transcoding" queue. You can have these jobs processed on the main server or dedicated workers. For more information check -the [Laravel Queue Documentation](https://laravel.com/docs/10.x/queues). +Transcoding jobs are dispatched onto the "video-transcoding" queue. +You can have these jobs processed on the main server or dedicated workers. +For more information, check the [Laravel Queue Documentation](https://laravel.com/docs/11.x/queues). > [!NOTE] > Since queues are not generally FIFO, it is recommended to use a queue which guarantees FIFO and also prevents @@ -246,29 +274,11 @@ To access public derivatives for videos, generate a symlink from the Laravel sto php artisan storage:link ``` -*Sodium Keypair* - -A signed request is used to notify clients about finished transcodings. For this, a [Sodium](https://www.php.net/manual/en/book.sodium.php) keypair has to be configured. - -To create a keypair, simply use the provided command: - -```bash -php artisan transmorpher:keypair -``` - -The newly created keypair has to be written in the `.env` file: - -```dotenv -TRANSMORPHER_SIGNING_KEYPAIR= -``` - -The public key of the media server is available under the `/api/v*/publickey` endpoint and can be requested -by any client. -\ *Queue* -Transcoding jobs are dispatched onto the "video-transcoding" queue. You can have these jobs processed on the main server or dedicated workers. For more information check -the [Laravel Queue Documentation](https://laravel.com/docs/10.x/queues). +Transcoding jobs are dispatched onto the "video-transcoding" queue. +You can have these jobs processed on the main server or dedicated workers. +For more information, check the [Laravel Queue Documentation](https://laravel.com/docs/11.x/queues). You can define your queue connection in the `.env` file: @@ -302,7 +312,7 @@ Media always belongs to a user. To easily create one, use the provided command: php artisan create:user ``` -This command will provide you with a [Laravel Sanctum](https://laravel.com/docs/10.x/sanctum) token, which has to be +This command will provide you with a [Laravel Sanctum](https://laravel.com/docs/11.x/sanctum) token, which has to be written in the `.env` file of a client system. > The token will be passed for all API requests for authorization and is connected to the corresponding user. @@ -318,7 +328,7 @@ Media is identified by a string which is passed when uploading media. This "iden When media is uploaded on the same identifier by the same user, a new version for the same media will be created. -The media server provides following features for media: +The media server provides the following features for media: - upload - get derivative @@ -330,7 +340,8 @@ The media server provides following features for media: ## Image transformation -Images will always be optimized and transformed on the Transmorpher media server. Requests for derivatives will also be directly answered by the media server. +Images will always be optimized and transformed on the Transmorpher media server. +The media server will also directly answer requests for derivatives. The media server provides the following transformations for images: @@ -360,7 +371,8 @@ transformations. Video transcoding is handled as an asynchronous task. The client will receive the information about the transcoded video as soon as it completes. For this, a signed request is sent to the client. -Since video transcoding is a complex task it may take some time to complete. The client will also be notified about failed attempts. +Since video transcoding is a complex task, it may take some time to complete. +The client will also be notified about failed attempts. To publicly access a video, the client name, the identifier and a format have to be specified. There are different formats available: @@ -389,9 +401,8 @@ You will also have to adjust the `transmorpher.php` configuration value for the The class to transform images as well as the classes to convert images to different formats are interchangeable. This provides the ability to add additional image manipulation libraries or logic in a modular way. -To add a class for image transformation, simply create a new class which implements the `TransformInterface`. An example -implementation can be found -at `App\Classes\Intervention\Transform`. +To add a class for image transformation, create a new class which implements the `TransformInterface`. +An example implementation can be found at `App\Classes\Intervention\Transform`. Additionally, the newly created class has to be specified in the `transmorpher.php` configuration file: ```php @@ -416,7 +427,7 @@ You will also have to adjust the configuration values: The `image-optimizer.php` configuration file specifies which optimizers should be used. Here you can configure options for each optimizer and add new or remove optimizers. -For more information on adding custom optimizers check the documentation of +For more information on adding custom optimizers, check the documentation of the [Laravel Image Optimizer](https://github.com/spatie/laravel-image-optimizer#adding-your-own-optimizers) package. ### Video Transcoding @@ -424,7 +435,7 @@ the [Laravel Image Optimizer](https://github.com/spatie/laravel-image-optimizer# By default, the Transmorpher uses FFmpeg and Laravel jobs for transcoding videos. This can be changed similar to the image transformation classes. -To interchange the class, which is responsible for initiating transcoding, simply create a new class which implements +To interchange the class, which is responsible for initiating transcoding, create a new class which implements the `TranscodeInterface`. An example implementation, which dispatches a job, can be found at `App\Classes\Transcode.php`. You will also have to adjust the configuration value: @@ -433,16 +444,49 @@ You will also have to adjust the configuration value: 'transcode_class' => App\Classes\YourTranscodeClass::class, ``` +## Purging derivatives + +Adjusting the way derivatives are generated will not be reflected on already existing derivatives. Therefore, you might want to delete all existing derivatives or re-generate them. + +We provide a command which will additionally notify clients with a signed request about a new derivatives revision, so they can react accordingly (e.g. update cache buster). + +```bash +php artisan purge:derivatives +``` + +The command accepts the options `--image`, `--video` and `--all` (or `-a`) for purging the respective derivatives. +Image derivatives will be deleted, for video derivatives we dispatch a new transcoding job for the current version. + +The derivatives revision is available on the route `/api/v*/cacheInvalidator`. + +## Recovery + +To restore operation of the server, restore the following: + +- database +- the `originals` disk +- `.env` file* +- the `image derivatives` disk* +- the `video derivatives` disk* + +> Marked with * are optional, but recommended. + +If the `.env` file is lost follow the setup instructions above, including creating a new signing keypair. + +If video derivatives are lost, use the [purge command](#purging-derivatives) to restore them. + +Lost image derivatives will automatically be re-generated on demand. + ## Development ### [Pullpreview](https://github.com/pullpreview/action) -For more information take a look at the PullPreview section of the [github-workflow repository](https://github.com/cybex-gmbh/github-workflows#pullpreview). +For more information, take a look at the PullPreview section of the [github-workflow repository](https://github.com/cybex-gmbh/github-workflows#pullpreview). -App specific GitHub Secrets: +App-specific GitHub Secrets: -- APP_KEY -- TRANSMORPHER_SIGNING_KEYPAIR +- PULLPREVIEW_APP_KEY +- PULLPREVIEW_TRANSMORPHER_SIGNING_KEYPAIR - PULLPREVIEW_TRANSMORPHER_AUTH_TOKEN_HASH #### Auth Token Hash @@ -450,7 +494,7 @@ App specific GitHub Secrets: The environment is seeded with a user with an auth token. To get access, you will have to locally create a token and use this token and its hash. ```bash -php artisan create:user pullpreview pullpreview@example.com +php artisan create:user pullpreview pullpreview@example.com http://pullpreview.test/transmorpher/notifications ``` Take the hash of the token from the `personal_access_tokens` table and save it to GitHub secrets. The command also provides a `TRANSMORPHER_AUTH_TOKEN`, which should be stored diff --git a/_ide_helper.php b/_ide_helper.php index df37af47..1c360ae7 100644 --- a/_ide_helper.php +++ b/_ide_helper.php @@ -5,7 +5,7 @@ /** * A helper file for Laravel, to provide autocomplete information to your IDE - * Generated for Laravel 11.4.0. + * Generated for Laravel 11.5.0. * * This file should not be included in your code, only analyzed by your IDE! * @@ -2889,6 +2889,33 @@ { /** @var \Illuminate\Broadcasting\BroadcastManager $instance */ return $instance->socket($request); + } + /** + * Begin sending an anonymous broadcast to the given channels. + * + * @static + */ public static function on($channels) + { + /** @var \Illuminate\Broadcasting\BroadcastManager $instance */ + return $instance->on($channels); + } + /** + * Begin sending an anonymous broadcast to the given private channels. + * + * @static + */ public static function private($channel) + { + /** @var \Illuminate\Broadcasting\BroadcastManager $instance */ + return $instance->private($channel); + } + /** + * Begin sending an anonymous broadcast to the given presence channels. + * + * @static + */ public static function presence($channel) + { + /** @var \Illuminate\Broadcasting\BroadcastManager $instance */ + return $instance->presence($channel); } /** * Begin broadcasting an event. @@ -10507,6 +10534,19 @@ { /** @var \Illuminate\Cache\RateLimiter $instance */ return $instance->increment($key, $decaySeconds, $amount); + } + /** + * Decrement the counter for a given key for a given decay time by a given amount. + * + * @param string $key + * @param int $decaySeconds + * @param int $amount + * @return int + * @static + */ public static function decrement($key, $decaySeconds = 60, $amount = 1) + { + /** @var \Illuminate\Cache\RateLimiter $instance */ + return $instance->decrement($key, $decaySeconds, $amount); } /** * Get the number of attempts for the given key. @@ -11113,7 +11153,7 @@ return $instance->mergeIfMissing($input); } /** - * Replace the input for the current request. + * Replace the input values for the current request. * * @param array $input * @return \Illuminate\Http\Request @@ -16219,6 +16259,20 @@ { /** @var \Illuminate\Routing\UrlGenerator $instance */ return $instance->to($path, $extra, $secure); + } + /** + * Generate an absolute URL with the given query parameters. + * + * @param string $path + * @param array $query + * @param mixed $extra + * @param bool|null $secure + * @return string + * @static + */ public static function query($path, $query = [], $extra = [], $secure = null) + { + /** @var \Illuminate\Routing\UrlGenerator $instance */ + return $instance->query($path, $query, $extra, $secure); } /** * Generate a secure, absolute URL to the given path. @@ -18012,6 +18066,16 @@ * */ class TranscodeFacade { /** + * Returns the class which handles the actual transcoding. + * + * @return string + * @static + */ public static function getJobClass() + { + /** @var \App\Classes\Transcode $instance */ + return $instance->getJobClass(); + } + /** * Creates a job which handles the transcoding of a video. * * @param \App\Models\Version $version @@ -18041,16 +18105,15 @@ * Inform client package about the transcoding result. * * @param \App\Enums\ResponseState $responseState - * @param string $callbackUrl * @param string $uploadToken * @param \App\Models\Media $media * @param int $versionNumber * @return void * @static - */ public static function callback($responseState, $callbackUrl, $uploadToken, $media, $versionNumber) + */ public static function callback($responseState, $uploadToken, $media, $versionNumber) { /** @var \App\Classes\Transcode $instance */ - $instance->callback($responseState, $callbackUrl, $uploadToken, $media, $versionNumber); + $instance->callback($responseState, $uploadToken, $media, $versionNumber); } } /** diff --git a/app/Classes/MediaHandler/ImageHandler.php b/app/Classes/MediaHandler/ImageHandler.php index 04c1fd4c..d71ba05f 100644 --- a/app/Classes/MediaHandler/ImageHandler.php +++ b/app/Classes/MediaHandler/ImageHandler.php @@ -74,10 +74,9 @@ public function invalidateCdnCache(string $basePath): bool * @param Version $version * @param int $oldVersionNumber * @param bool $wasProcessed - * @param string $callbackUrl * @return array */ - public function setVersion(User $user, Version $version, int $oldVersionNumber, bool $wasProcessed, string $callbackUrl): array + public function setVersion(User $user, Version $version, int $oldVersionNumber, bool $wasProcessed): array { // Token and valid_until will be set in the 'saving' event. // By creating an upload slot, a currently active upload will be canceled. @@ -120,4 +119,17 @@ public function getVersions(Media $media): array 'versions' => $processedVersions->pluck('created_at', 'number')->map(fn($date) => strtotime($date)), ]; } + + /** + * @return array + */ + public function purgeDerivatives(): array + { + $success = $this->getDerivativesDisk()->deleteDirectory(''); + + return [ + 'success' => $success, + 'message' => $success ? 'Deleted image derivatives.' : 'Failed to delete image derivatives.', + ]; + } } diff --git a/app/Classes/MediaHandler/VideoHandler.php b/app/Classes/MediaHandler/VideoHandler.php index c5f13137..a93efb39 100644 --- a/app/Classes/MediaHandler/VideoHandler.php +++ b/app/Classes/MediaHandler/VideoHandler.php @@ -5,6 +5,7 @@ use App\Enums\MediaStorage; use App\Enums\MediaType; use App\Enums\ResponseState; +use App\Enums\UploadState; use App\Interfaces\MediaHandlerInterface; use App\Models\Media; use App\Models\UploadSlot; @@ -61,21 +62,16 @@ public function invalidateCdnCache(string $basePath): bool * @param Version $version * @param int $oldVersionNumber * @param bool $wasProcessed - * @param string $callbackUrl * @return array */ - public function setVersion(User $user, Version $version, int $oldVersionNumber, bool $wasProcessed, string $callbackUrl): array + public function setVersion(User $user, Version $version, int $oldVersionNumber, bool $wasProcessed): array { - if ($callbackUrl) { - // Token and valid_until will be set in the 'saving' event. - // By creating an upload slot, currently active uploading or transcoding will be canceled. - $uploadSlot = $user->UploadSlots()->withoutGlobalScopes()->updateOrCreate(['identifier' => $version->Media->identifier], ['callback_url' => $callbackUrl, 'media_type' => MediaType::VIDEO]); - - $success = Transcode::createJobForVersionUpdate($version, $uploadSlot, $oldVersionNumber, $wasProcessed); - $responseState = $success ? ResponseState::VIDEO_VERSION_SET : ResponseState::TRANSCODING_JOB_DISPATCH_FAILED; - } else { - $responseState = ResponseState::NO_CALLBACK_URL_PROVIDED; - } + // Token and valid_until will be set in the 'saving' event. + // By creating an upload slot, currently active uploading or transcoding will be canceled. + $uploadSlot = $user->UploadSlots()->withoutGlobalScopes()->updateOrCreate(['identifier' => $version->Media->identifier], ['media_type' => MediaType::VIDEO]); + + $success = Transcode::createJobForVersionUpdate($version, $uploadSlot, $oldVersionNumber, $wasProcessed); + $responseState = $success ? ResponseState::VIDEO_VERSION_SET : ResponseState::TRANSCODING_JOB_DISPATCH_FAILED; return [ $responseState, @@ -105,4 +101,32 @@ public function getVersions(Media $media): array 'versions' => $versions->pluck('created_at', 'number')->map(fn($date) => strtotime($date)), ]; } + + /** + * @return array + */ + public function purgeDerivatives(): array + { + $failedMediaIds = []; + + foreach (Media::whereType(MediaType::VIDEO)->get() as $media) { + // Restore latest version to (re-)generate derivatives. + $version = $media->latestVersion; + + $oldVersionNumber = $version->number; + $wasProcessed = $version->processed; + + $version->update(['number' => $media->latestVersion->number + 1, 'processed' => 0]); + [$responseState, $uploadToken] = $this->setVersion($media->User, $version, $oldVersionNumber, $wasProcessed); + + if ($responseState->getState() === UploadState::ERROR) { + $failedMediaIds[] = $media->getKey(); + } + } + + return [ + 'success' => $success = !count($failedMediaIds), + 'message' => $success ? 'Restored versions for all video media.' : sprintf('Failed to restore versions for media ids: %s.', implode(', ', $failedMediaIds)), + ]; + } } diff --git a/app/Classes/Transcode.php b/app/Classes/Transcode.php index 87a06309..49712199 100644 --- a/app/Classes/Transcode.php +++ b/app/Classes/Transcode.php @@ -2,6 +2,7 @@ namespace App\Classes; +use App\Enums\ClientNotification; use App\Enums\MediaType; use App\Enums\ResponseState; use App\Helpers\SodiumHelper; @@ -15,6 +16,16 @@ class Transcode implements TranscodeInterface { + /** + * Returns the class which handles the actual transcoding. + * + * @return string + */ + public function getJobClass(): string + { + return TranscodeVideo::class; + } + /** * Creates a job which handles the transcoding of a video. * @@ -64,26 +75,27 @@ public function createJobForVersionUpdate(Version $version, UploadSlot $uploadSl * Inform client package about the transcoding result. * * @param ResponseState $responseState - * @param string $callbackUrl * @param string $uploadToken * @param Media $media * @param int $versionNumber * * @return void */ - public function callback(ResponseState $responseState, string $callbackUrl, string $uploadToken, Media $media, int $versionNumber): void + public function callback(ResponseState $responseState, string $uploadToken, Media $media, int $versionNumber): void { - $response = [ + $notification = [ 'state' => $responseState->getState()->value, 'message' => $responseState->getMessage(), 'identifier' => $media->identifier, 'version' => $versionNumber, 'upload_token' => $uploadToken, - 'public_path' => implode(DIRECTORY_SEPARATOR, array_filter([MediaType::VIDEO->prefix(), $media->baseDirectory()])) + 'public_path' => implode(DIRECTORY_SEPARATOR, array_filter([MediaType::VIDEO->prefix(), $media->baseDirectory()])), + 'hash' => Version::whereNumber($versionNumber)->first()?->hash, + 'notification_type' => ClientNotification::VIDEO_TRANSCODING->value, ]; - $signedResponse = SodiumHelper::sign(json_encode($response)); + $signedNotification = SodiumHelper::sign(json_encode($notification)); - Http::post($callbackUrl, ['signed_response' => $signedResponse]); + Http::post($media->User->api_url, ['signed_notification' => $signedNotification]); } } diff --git a/app/Console/Commands/CreateUser.php b/app/Console/Commands/CreateUser.php index 1a0b38fe..c1813c5b 100644 --- a/app/Console/Commands/CreateUser.php +++ b/app/Console/Commands/CreateUser.php @@ -16,7 +16,8 @@ class CreateUser extends Command */ protected $signature = 'create:user {name : The name of the user.} - {email : The E-Mail of the user.}'; + {email : The E-Mail of the user.} + {api_url : The URL at which the client can receive notifications.}'; /** * The console command description. @@ -34,6 +35,7 @@ public function handle(): int { $name = $this->argument('name'); $email = $this->argument('email'); + $apiUrl = $this->argument('api_url'); if (!filter_var($email, FILTER_VALIDATE_EMAIL)) { $this->error('The provided email is not valid!'); @@ -56,13 +58,18 @@ public function handle(): int return Command::INVALID; } + if (!filter_var($apiUrl, FILTER_VALIDATE_URL, FILTER_FLAG_PATH_REQUIRED)) { + $this->error(sprintf('The API URL must be a valid URL and include a path.')); + return Command::INVALID; + } + /* * Laravel passwords are usually not nullable, so we will need to set something when creating the user. * Since we do not want to create a Password for the user, but need to store something secure, * we will just generate a string of random bytes. * This needs to be encoded to base64 because null bytes are not accepted anymore (PHP 8.3). */ - $user = User::create(['name' => $name, 'email' => $email, 'password' => Hash::make(base64_encode(random_bytes(300)))]); + $user = User::create(['name' => $name, 'email' => $email, 'api_url' => $apiUrl, 'password' => Hash::make(base64_encode(random_bytes(300)))]); $this->info(sprintf('Successfully created new user %s: %s (%s)', $user->getKey(), $user->name, $user->email)); $this->newLine(); diff --git a/app/Console/Commands/PurgeDerivatives.php b/app/Console/Commands/PurgeDerivatives.php new file mode 100644 index 00000000..300660bd --- /dev/null +++ b/app/Console/Commands/PurgeDerivatives.php @@ -0,0 +1,60 @@ +option('image') && !$this->option('video') && !$this->option('all')) { + $this->warn(sprintf('No options provided. Call "php artisan %s --help" for a list of all options.', $this->name)); + return Command::SUCCESS; + } + + foreach (MediaType::cases() as $mediaType) { + if ($this->option('all') || $this->option($mediaType->value)) { + ['success' => $success, 'message' => $message] = $mediaType->handler()->purgeDerivatives(); + $success ? $this->info($message) : $this->error($message); + } + } + + $originalsDisk = MediaStorage::ORIGINALS->getDisk(); + $cacheInvalidationCounterFilePath = config('transmorpher.cache_invalidation_counter_file_path'); + + if (!$originalsDisk->put($cacheInvalidationCounterFilePath, $originalsDisk->get($cacheInvalidationCounterFilePath) + 1)) { + $this->error(sprintf('Failed to update cache invalidation counter at path %s on disk %s', $cacheInvalidationCounterFilePath, MediaStorage::ORIGINALS->value)); + } + + foreach (User::get() as $user) { + ClientPurgeNotification::dispatch($user, $originalsDisk->get($cacheInvalidationCounterFilePath)); + } + + return Command::SUCCESS; + } +} diff --git a/app/Enums/ClientNotification.php b/app/Enums/ClientNotification.php new file mode 100644 index 00000000..2924f29d --- /dev/null +++ b/app/Enums/ClientNotification.php @@ -0,0 +1,9 @@ +updateOrCreateUploadSlot($request->user(), $request->merge(['media_type' => MediaType::IMAGE->value])->all()); + return $this->reserveUploadSlot($request, MediaType::IMAGE); } /** * Handle the incoming request. * - * @param VideoUploadSlotRequest $request + * @param UploadSlotRequest $request * * @return JsonResponse */ - public function reserveVideoUploadSlot(VideoUploadSlotRequest $request): JsonResponse + public function reserveVideoUploadSlot(UploadSlotRequest $request): JsonResponse { - return $this->updateOrCreateUploadSlot($request->user(), $request->merge(['media_type' => MediaType::VIDEO->value])->all()); + return $this->reserveUploadSlot($request, MediaType::VIDEO); + } + + protected function reserveUploadSlot(UploadSlotRequest $request, MediaType $mediaType): JsonResponse + { + return $this->updateOrCreateUploadSlot($request->user(), $request->merge(['media_type' => $mediaType->value])->all()); } /** @@ -91,7 +95,7 @@ protected function saveFile(UploadedFile $uploadedFile, UploadSlot $uploadSlot, $media->validateUploadFile($uploadedFile, $type->handler()->getValidationRules()); $media->save(); - $versionNumber = $media->Versions()->max('number') + 1; + $versionNumber = $media->latestVersion?->number + 1; $version = $media->Versions()->create(['number' => $versionNumber]); $basePath = $media->baseDirectory(); @@ -118,7 +122,8 @@ protected function saveFile(UploadedFile $uploadedFile, UploadSlot $uploadSlot, 'version' => $versionNumber, // Base path is only passed for images since the video is not available at this path yet. 'public_path' => $type->isInstantlyAvailable() ? implode(DIRECTORY_SEPARATOR, array_filter([$type->prefix(), $basePath])) : null, - 'upload_token' => $uploadSlot->token + 'upload_token' => $uploadSlot->token, + 'hash' => $type->isInstantlyAvailable() ? $version?->hash : null, ], 201); } diff --git a/app/Http/Controllers/V1/VersionController.php b/app/Http/Controllers/V1/VersionController.php index 4e4f8ec3..9aaa3244 100644 --- a/app/Http/Controllers/V1/VersionController.php +++ b/app/Http/Controllers/V1/VersionController.php @@ -51,7 +51,7 @@ public function setVersion(SetVersionRequest $request, Media $media, Version $ve $version->update(['number' => $newVersionNumber, 'processed' => 0]); - [$responseState, $uploadToken] = $media->type->handler()->setVersion($user, $version, $oldVersionNumber, $wasProcessed, $request->get('callback_url')); + [$responseState, $uploadToken] = $media->type->handler()->setVersion($user, $version, $oldVersionNumber, $wasProcessed); return response()->json([ 'state' => $responseState->getState()->value, @@ -62,7 +62,8 @@ public function setVersion(SetVersionRequest $request, Media $media, Version $ve 'public_path' => $media->type->isInstantlyAvailable() ? implode(DIRECTORY_SEPARATOR, array_filter([$media->type->prefix(), $media->baseDirectory()])) : null, - 'upload_token' => $uploadToken + 'upload_token' => $uploadToken, + 'hash' => $media->type->isInstantlyAvailable() ? $version->hash : null, ]); } diff --git a/app/Http/Requests/V1/SetVersionRequest.php b/app/Http/Requests/V1/SetVersionRequest.php index c433e2ba..866caf7e 100644 --- a/app/Http/Requests/V1/SetVersionRequest.php +++ b/app/Http/Requests/V1/SetVersionRequest.php @@ -23,9 +23,6 @@ public function authorize(): bool */ public function rules(): array { - // Nullable because this information is only used for videos. Validation is happening inside the VersionController. - return [ - 'callback_url' => ['nullable', 'string', 'url'] - ]; + return []; } } diff --git a/app/Http/Requests/V1/ImageUploadSlotRequest.php b/app/Http/Requests/V1/UploadSlotRequest.php similarity index 84% rename from app/Http/Requests/V1/ImageUploadSlotRequest.php rename to app/Http/Requests/V1/UploadSlotRequest.php index b3f8e2a0..18eb976d 100644 --- a/app/Http/Requests/V1/ImageUploadSlotRequest.php +++ b/app/Http/Requests/V1/UploadSlotRequest.php @@ -5,7 +5,7 @@ use App\Enums\ValidationRegex; use Illuminate\Foundation\Http\FormRequest; -class ImageUploadSlotRequest extends FormRequest +class UploadSlotRequest extends FormRequest { /** * Determine if the user is authorized to make this request. @@ -14,7 +14,7 @@ class ImageUploadSlotRequest extends FormRequest */ public function authorize(): bool { - return $this->user()->tokenCan('transmorpher:reserve-image-upload-slot'); + return $this->user()->tokenCan('transmorpher:reserve-upload-slot'); } /** diff --git a/app/Http/Requests/V1/VideoUploadSlotRequest.php b/app/Http/Requests/V1/VideoUploadSlotRequest.php deleted file mode 100644 index f6793fc5..00000000 --- a/app/Http/Requests/V1/VideoUploadSlotRequest.php +++ /dev/null @@ -1,33 +0,0 @@ -user()->tokenCan('transmorpher:reserve-video-upload-slot'); - } - - /** - * Get the validation rules that apply to the request. - * - * @return array - */ - public function rules(): array - { - return [ - // Identifier is used in file paths and URLs, therefore only lower/uppercase characters, numbers, underscores and hyphens are allowed. - 'identifier' => ['required', 'string', sprintf('regex:%s', ValidationRegex::IDENTIFIER->get())], - 'callback_url' => ['required', 'string', 'url'] - ]; - } -} diff --git a/app/Interfaces/MediaHandlerInterface.php b/app/Interfaces/MediaHandlerInterface.php index 3cd04a69..1386e998 100644 --- a/app/Interfaces/MediaHandlerInterface.php +++ b/app/Interfaces/MediaHandlerInterface.php @@ -2,9 +2,7 @@ namespace App\Interfaces; -use App\Enums\MediaStorage; use App\Enums\ResponseState; -use App\Models\Media; use App\Models\UploadSlot; use App\Models\User; use App\Models\Version; @@ -36,13 +34,17 @@ public function invalidateCdnCache(string $basePath): bool; * @param Version $version * @param int $oldVersionNumber * @param bool $wasProcessed - * @param string $callbackUrl * @return array */ - public function setVersion(User $user, Version $version, int $oldVersionNumber, bool $wasProcessed, string $callbackUrl): array; + public function setVersion(User $user, Version $version, int $oldVersionNumber, bool $wasProcessed): array; /** * @return Filesystem */ public function getDerivativesDisk(): Filesystem; + + /** + * @return array + */ + public function purgeDerivatives(): array; } diff --git a/app/Interfaces/TranscodeInterface.php b/app/Interfaces/TranscodeInterface.php index 63b66ce3..adc56c75 100644 --- a/app/Interfaces/TranscodeInterface.php +++ b/app/Interfaces/TranscodeInterface.php @@ -9,6 +9,13 @@ interface TranscodeInterface { + /** + * Returns the class which handles the actual transcoding. + * + * @return string + */ + public function getJobClass(): string; + /** * Creates a job which handles the transcoding of a video. * @@ -34,12 +41,11 @@ public function createJobForVersionUpdate(Version $version, UploadSlot $uploadSl * Inform client package about the transcoding result. * * @param ResponseState $responseState - * @param string $callbackUrl * @param string $uploadToken * @param Media $media * @param int $versionNumber * * @return void */ - public function callback(ResponseState $responseState, string $callbackUrl, string $uploadToken, Media $media, int $versionNumber): void; + public function callback(ResponseState $responseState, string $uploadToken, Media $media, int $versionNumber): void; } diff --git a/app/Jobs/ClientPurgeNotification.php b/app/Jobs/ClientPurgeNotification.php new file mode 100644 index 00000000..7e1a6dce --- /dev/null +++ b/app/Jobs/ClientPurgeNotification.php @@ -0,0 +1,70 @@ +onQueue('client-notifications'); + } + + /** + * Execute the job. + * @throws ClientNotificationFailedException + */ + public function handle(): void + { + $notification = [ + 'notification_type' => $this->notificationType, + 'cache_invalidator' => $this->cacheInvalidationCounter + ]; + + $signedNotification = SodiumHelper::sign(json_encode($notification)); + + $response = Http::post($this->user->api_url, ['signed_notification' => $signedNotification]); + + if (!$response->ok()) { + throw new ClientNotificationFailedException($this->user->name, $this->notificationType->value, $response->status(), $response->reason()); + } + } +} diff --git a/app/Jobs/TranscodeVideo.php b/app/Jobs/TranscodeVideo.php index dbf178cb..8a2180a5 100644 --- a/app/Jobs/TranscodeVideo.php +++ b/app/Jobs/TranscodeVideo.php @@ -47,7 +47,6 @@ class TranscodeVideo implements ShouldQueue protected Filesystem $localDisk; protected string $originalFilePath; - protected string $callbackUrl; protected string $uploadToken; // Derivatives are saved to a temporary folder first, else race conditions could cause newer versions to be overwritten. @@ -74,7 +73,6 @@ public function __construct( { $this->onQueue('video-transcoding'); $this->originalFilePath = $version->originalFilePath(); - $this->callbackUrl = $this->uploadSlot->callback_url; $this->uploadToken = $this->uploadSlot->token; } @@ -99,7 +97,7 @@ public function handle(): void } match ($this->responseState) { - ResponseState::TRANSCODING_SUCCESSFUL => Transcode::callback($this->responseState, $this->callbackUrl, $this->uploadToken, $this->version->Media, $this->version->number), + ResponseState::TRANSCODING_SUCCESSFUL => Transcode::callback($this->responseState, $this->uploadToken, $this->version->Media, $this->version->number), ResponseState::TRANSCODING_ABORTED => $this->failed(null), }; } @@ -132,7 +130,7 @@ public function failed(?Throwable $exception): void $versionNumber = $this->oldVersionNumber; } - Transcode::callback($this->responseState ?? ResponseState::TRANSCODING_FAILED, $this->callbackUrl, $this->uploadToken, $this->version->Media, $versionNumber); + Transcode::callback($this->responseState ?? ResponseState::TRANSCODING_FAILED, $this->uploadToken, $this->version->Media, $versionNumber); } /** diff --git a/app/Models/Media.php b/app/Models/Media.php index e7dc54bb..f1dbca2a 100644 --- a/app/Models/Media.php +++ b/app/Models/Media.php @@ -28,6 +28,7 @@ * @property-read \Illuminate\Database\Eloquent\Collection $Versions * @property-read int|null $versions_count * @property-read \App\Models\Version $current_version + * @property-read \App\Models\Version|null $latest_version * @method static \Illuminate\Database\Eloquent\Builder|Media newModelQuery() * @method static \Illuminate\Database\Eloquent\Builder|Media newQuery() * @method static \Illuminate\Database\Eloquent\Builder|Media query() @@ -163,6 +164,16 @@ public function currentVersion(): Attribute ); } + public function latestVersion(): Attribute + { + return Attribute::make( + get: function (): ?Version { + $versions = $this->Versions(); + return $versions->whereNumber($versions->max('number'))->first(); + } + ); + } + /** * Get the base path for files. * Path structure: {username}/{identifier}/ diff --git a/app/Models/UploadSlot.php b/app/Models/UploadSlot.php index e9d8a4af..2909d923 100644 --- a/app/Models/UploadSlot.php +++ b/app/Models/UploadSlot.php @@ -16,7 +16,6 @@ * @property int $id * @property string|null $token * @property string $identifier - * @property string|null $callback_url * @property string|null $validation_rules * @property string|null $valid_until * @property MediaType $media_type @@ -28,7 +27,6 @@ * @method static Builder|UploadSlot newModelQuery() * @method static Builder|UploadSlot newQuery() * @method static Builder|UploadSlot query() - * @method static Builder|UploadSlot whereCallbackUrl($value) * @method static Builder|UploadSlot whereCreatedAt($value) * @method static Builder|UploadSlot whereId($value) * @method static Builder|UploadSlot whereIdentifier($value) @@ -51,9 +49,8 @@ class UploadSlot extends Model */ protected $fillable = [ 'identifier', - 'callback_url', - 'validation_rules', 'media_type', + 'validation_rules', ]; /** diff --git a/app/Models/User.php b/app/Models/User.php index 9fa2502f..3ccdec65 100644 --- a/app/Models/User.php +++ b/app/Models/User.php @@ -22,6 +22,7 @@ * @property \Illuminate\Support\Carbon|null $created_at * @property \Illuminate\Support\Carbon|null $updated_at * @property string|null $protector_public_key The sodium public key for the Protector package. + * @property string $api_url The URL at which the client can receive notifications. * @property-read \Illuminate\Database\Eloquent\Collection $Media * @property-read int|null $media_count * @property-read \Illuminate\Database\Eloquent\Collection $UploadSlots @@ -34,6 +35,7 @@ * @method static \Illuminate\Database\Eloquent\Builder|User newModelQuery() * @method static \Illuminate\Database\Eloquent\Builder|User newQuery() * @method static \Illuminate\Database\Eloquent\Builder|User query() + * @method static \Illuminate\Database\Eloquent\Builder|User whereApiUrl($value) * @method static \Illuminate\Database\Eloquent\Builder|User whereCreatedAt($value) * @method static \Illuminate\Database\Eloquent\Builder|User whereEmail($value) * @method static \Illuminate\Database\Eloquent\Builder|User whereEmailVerifiedAt($value) @@ -55,8 +57,9 @@ class User extends Authenticatable * @var array */ protected $fillable = [ - 'name', + 'api_url', 'email', + 'name', 'password', ]; diff --git a/app/Models/Version.php b/app/Models/Version.php index 40ef8107..b0d06853 100644 --- a/app/Models/Version.php +++ b/app/Models/Version.php @@ -4,9 +4,13 @@ use App\Enums\MediaStorage; use App\Enums\Transformation; +use Eloquent; +use Illuminate\Database\Eloquent\Builder; +use Illuminate\Database\Eloquent\Casts\Attribute; use Illuminate\Database\Eloquent\Factories\HasFactory; use Illuminate\Database\Eloquent\Model; use Illuminate\Database\Eloquent\Relations\BelongsTo; +use Illuminate\Support\Carbon; /** * App\Models\Version @@ -16,20 +20,21 @@ * @property string|null $filename * @property int $processed * @property int $media_id - * @property \Illuminate\Support\Carbon|null $created_at - * @property \Illuminate\Support\Carbon|null $updated_at + * @property Carbon|null $created_at + * @property Carbon|null $updated_at * @property-read \App\Models\Media $Media - * @method static \Illuminate\Database\Eloquent\Builder|Version newModelQuery() - * @method static \Illuminate\Database\Eloquent\Builder|Version newQuery() - * @method static \Illuminate\Database\Eloquent\Builder|Version query() - * @method static \Illuminate\Database\Eloquent\Builder|Version whereCreatedAt($value) - * @method static \Illuminate\Database\Eloquent\Builder|Version whereFilename($value) - * @method static \Illuminate\Database\Eloquent\Builder|Version whereId($value) - * @method static \Illuminate\Database\Eloquent\Builder|Version whereMediaId($value) - * @method static \Illuminate\Database\Eloquent\Builder|Version whereNumber($value) - * @method static \Illuminate\Database\Eloquent\Builder|Version whereProcessed($value) - * @method static \Illuminate\Database\Eloquent\Builder|Version whereUpdatedAt($value) - * @mixin \Eloquent + * @property-read string $hash + * @method static Builder|Version newModelQuery() + * @method static Builder|Version newQuery() + * @method static Builder|Version query() + * @method static Builder|Version whereCreatedAt($value) + * @method static Builder|Version whereFilename($value) + * @method static Builder|Version whereId($value) + * @method static Builder|Version whereMediaId($value) + * @method static Builder|Version whereNumber($value) + * @method static Builder|Version whereProcessed($value) + * @method static Builder|Version whereUpdatedAt($value) + * @mixin Eloquent */ class Version extends Model { @@ -44,8 +49,8 @@ class Version extends Model * @var array */ protected $fillable = [ - 'number', 'filename', + 'number', 'processed', ]; @@ -142,4 +147,11 @@ public function imageDerivativeDirectoryPath(): string { return sprintf('%s/%s', $this->Media->baseDirectory(), $this->getKey()); } + + public function hash(): Attribute + { + return Attribute::make( + get: fn(): string => md5(sprintf('%s-%s', $this->number, $this->created_at)) + ); + } } diff --git a/config/transmorpher.php b/config/transmorpher.php index 508aa2ac..20a88396 100644 --- a/config/transmorpher.php +++ b/config/transmorpher.php @@ -182,4 +182,14 @@ 'image' => App\Classes\MediaHandler\ImageHandler::class, 'video' => App\Classes\MediaHandler\VideoHandler::class ], + + /* + |-------------------------------------------------------------------------- + | Cache Invalidation Counter File Path + |-------------------------------------------------------------------------- + | + | The path to a file on the originals disk that stores the cache invalidation counter. + | + */ + 'cache_invalidation_counter_file_path' => env('CACHE_INVALIDATION_COUNTER_FILE_PATH', 'cacheInvalidationCounter'), ]; diff --git a/database/factories/UserFactory.php b/database/factories/UserFactory.php index e033548d..de653d06 100644 --- a/database/factories/UserFactory.php +++ b/database/factories/UserFactory.php @@ -29,6 +29,7 @@ public function definition(): array 'email_verified_at' => now(), 'password' => static::$password ??= Hash::make('password'), 'remember_token' => Str::random(10), + 'api_url' => 'http://example.com/transmorpher/notifications', ]; } diff --git a/database/migrations/2024_04_23_095200_add_api_url_to_users_table.php b/database/migrations/2024_04_23_095200_add_api_url_to_users_table.php new file mode 100644 index 00000000..6daa0d22 --- /dev/null +++ b/database/migrations/2024_04_23_095200_add_api_url_to_users_table.php @@ -0,0 +1,24 @@ +string('api_url')->comment('The URL at which the client can receive notifications.'); + }); + + Schema::table('upload_slots', function (Blueprint $table) { + $table->dropColumn('callback_url'); + }); + } +}; diff --git a/database/seeders/PullpreviewSeeder.php b/database/seeders/PullpreviewSeeder.php index ed8287e0..aff5f5ae 100644 --- a/database/seeders/PullpreviewSeeder.php +++ b/database/seeders/PullpreviewSeeder.php @@ -15,8 +15,8 @@ class PullpreviewSeeder extends Seeder */ public function run(): void { - Artisan::call('create:user pullpreview pullpreview@example.com'); + Artisan::call('create:user pullpreview pullpreview@example.com http://pullpreview.test/transmorpher/notifications'); - DB::table('personal_access_tokens')->where('id', 1)->update(['token' => env('PULLPREVIEW_TRANSMORPHER_AUTH_TOKEN_HASH')]); + DB::table('personal_access_tokens')->where('id', 1)->update(['token' => env('TRANSMORPHER_AUTH_TOKEN_HASH')]); } } diff --git a/docker/workers.conf b/docker/workers.conf index 69b87f64..b5825b6a 100644 --- a/docker/workers.conf +++ b/docker/workers.conf @@ -13,3 +13,19 @@ redirect_stderr=true stdout_logfile=/dev/stdout ; Timeout of the longest running job (video transcoding with 10800) plus 30. stopwaitsecs=10830 + +[program:client-notification-worker] +process_name=%(program_name)s_%(process_num)02d +; Supervisor starts programs as root by default, which might lead to permission problems when the webserver tries to access files or similar. +user=application +environment=HOME="/home/application",USER="application" +command=php /var/www/html/artisan queue:work --queue=client-notifications +autostart=true +autorestart=true +stopasgroup=true +killasgroup=true +numprocs=1 +redirect_stderr=true +stdout_logfile=/dev/stdout +; Timeout of the longest running job (purge notifications with 10) plus 30. +stopwaitsecs=40 diff --git a/lang/en/responses.php b/lang/en/responses.php index b8987c92..d4a0f163 100644 --- a/lang/en/responses.php +++ b/lang/en/responses.php @@ -14,7 +14,6 @@ 'deletion_successful' => 'Successfully deleted media.', 'image_upload_successful' => 'Successfully uploaded new image version.', 'image_version_set' => 'Successfully set image version.', - 'no_callback_url_provided' => 'A callback URL is needed for this identifier.', 'transcoding_aborted' => 'Transcoding process aborted due to a new version or upload.', 'transcoding_failed' => 'Video transcoding failed, version has been removed.', 'transcoding_job_dispatch_failed' => 'There was an error when trying to dispatch the transcoding job.', diff --git a/routes/api/v1.php b/routes/api/v1.php index 7ba7fd70..6503fd8a 100644 --- a/routes/api/v1.php +++ b/routes/api/v1.php @@ -1,5 +1,6 @@ name('upload'); Route::get('publickey', fn(): string => SodiumHelper::getPublicKey())->name('getPublicKey'); + Route::get('cacheInvalidator', fn(): string => MediaStorage::ORIGINALS->getDisk()->get(config('transmorpher.cache_invalidation_counter_file_path')) ?? 0)->name('getCacheInvalidator'); }); diff --git a/tests/MediaTest.php b/tests/MediaTest.php index 332bd464..b97e9930 100644 --- a/tests/MediaTest.php +++ b/tests/MediaTest.php @@ -10,7 +10,7 @@ class MediaTest extends TestCase { - protected static User $user; + protected User $user; protected Filesystem $originalsDisk; protected function setUp(): void @@ -20,7 +20,7 @@ protected function setUp(): void $this->originalsDisk ??= Storage::persistentFake(config(sprintf('transmorpher.disks.%s', MediaStorage::ORIGINALS->value))); Sanctum::actingAs( - self::$user ??= User::factory()->create(), + $this->user ??= User::first() ?: User::factory()->create(), ['*'] ); } diff --git a/tests/Unit/CreateUserCommandTest.php b/tests/Unit/CreateUserCommandTest.php index 823b609a..02b56f52 100644 --- a/tests/Unit/CreateUserCommandTest.php +++ b/tests/Unit/CreateUserCommandTest.php @@ -18,13 +18,14 @@ class CreateUserCommandTest extends TestCase protected const NAME = 'Oswald'; protected const EMAIL = 'oswald@example.com'; + protected const API_URL = 'http://example.com/transmorpher/notifications'; #[Test] public function ensureUserCanBeCreated() { $this->assertDatabaseMissing(User::getModel()->getTable(), ['name' => self::NAME, 'email' => self::EMAIL]); - $exitStatus = Artisan::call(CreateUser::class, ['name' => self::NAME, 'email' => self::EMAIL]); + $exitStatus = $this->createUser(); $this->assertEquals(Command::SUCCESS, $exitStatus); $this->assertDatabaseHas(User::getModel()->getTable(), ['name' => self::NAME, 'email' => self::EMAIL]); @@ -33,7 +34,7 @@ public function ensureUserCanBeCreated() #[Test] public function ensureUserHasSanctumToken() { - Artisan::call(CreateUser::class, ['name' => self::NAME, 'email' => self::EMAIL]); + $this->createUser(); $this->assertNotEmpty(User::whereName(self::NAME)->first()->tokens); } @@ -42,19 +43,20 @@ public function ensureUserHasSanctumToken() #[DataProvider('duplicateEntryDataProvider')] public function failOnDuplicateEntry(string $name, string $email) { - Artisan::call(CreateUser::class, ['name' => self::NAME, 'email' => self::EMAIL]); - $exitStatus = Artisan::call(CreateUser::class, ['name' => $name, 'email' => $email]); + $this->createUser(); + $exitStatus = $this->createUser($name, $email); $this->assertEquals(Command::INVALID, $exitStatus); } #[Test] #[DataProvider('missingArgumentsDataProvider')] - public function failOnMissingArguments(?string $name, ?string $email) + public function failOnMissingArguments(?string $name, ?string $email, ?string $apiUrl) { $arguments = []; $name && $arguments['name'] = $name; $email && $arguments['email'] = $email; + $apiUrl && $arguments['api_url'] = $apiUrl; $this->expectException(RuntimeException::class); Artisan::call(CreateUser::class, $arguments); @@ -62,9 +64,9 @@ public function failOnMissingArguments(?string $name, ?string $email) #[Test] #[DataProvider('invalidArgumentsDataProvider')] - public function failOnInvalidArguments(string $name, string $email) + public function failOnInvalidArguments(string $name, string $email, string $apiUrl) { - $exitStatus = Artisan::call(CreateUser::class, ['name' => $name, 'email' => $email]); + $exitStatus = $this->createUser($name, $email, $apiUrl); $this->assertEquals(Command::INVALID, $exitStatus); } @@ -92,15 +94,33 @@ public static function missingArgumentsDataProvider(): array return [ 'missing name' => [ 'name' => null, - 'email' => self::EMAIL + 'email' => self::EMAIL, + 'apiUrl' => self::API_URL ], 'missing email' => [ 'name' => self::NAME, - 'email' => null + 'email' => null, + 'apiUrl' => self::API_URL + ], + 'missing api url' => [ + 'name' => self::NAME, + 'email' => self::EMAIL, + 'apiUrl' => null ], 'missing name and email' => [ 'name' => null, - 'email' => null + 'email' => null, + 'apiUrl' => self::API_URL + ], + 'missing name and api url' => [ + 'name' => null, + 'email' => self::EMAIL, + 'apiUrl' => null + ], + 'missing email and api url' => [ + 'name' => self::NAME, + 'email' => null, + 'apiUrl' => null ] ]; } @@ -110,44 +130,85 @@ public static function invalidArgumentsDataProvider(): array return [ 'invalid name with slash' => [ 'name' => 'invalid/name', - 'email' => self::EMAIL + 'email' => self::EMAIL, + 'apiUrl' => self::API_URL ], 'invalid name with backslash' => [ 'name' => 'invalid\name', - 'email' => self::EMAIL + 'email' => self::EMAIL, + 'apiUrl' => self::API_URL ], 'invalid name with dot' => [ 'name' => 'invalid.name', - 'email' => self::EMAIL + 'email' => self::EMAIL, + 'apiUrl' => self::API_URL ], 'invalid name with hyphen' => [ 'name' => 'invalid--name', - 'email' => self::EMAIL + 'email' => self::EMAIL, + 'apiUrl' => self::API_URL ], 'invalid name with trailing hyphen' => [ 'name' => 'invalidName-', - 'email' => self::EMAIL + 'email' => self::EMAIL, + 'apiUrl' => self::API_URL ], 'invalid name with special character' => [ 'name' => 'invalidName!', - 'email' => self::EMAIL + 'email' => self::EMAIL, + 'apiUrl' => self::API_URL ], 'invalid name with umlaut' => [ 'name' => 'invalidNäme', - 'email' => self::EMAIL + 'email' => self::EMAIL, + 'apiUrl' => self::API_URL ], 'invalid name with space' => [ 'name' => 'invalid name', - 'email' => self::EMAIL + 'email' => self::EMAIL, + 'apiUrl' => self::API_URL ], 'invalid email' => [ 'name' => self::NAME, - 'email' => 'invalidEmail' + 'email' => 'invalidEmail', + 'apiUrl' => self::API_URL ], 'invalid name and email' => [ 'name' => 'invalid/name', - 'email' => 'invalidEmail' + 'email' => 'invalidEmail', + 'apiUrl' => self::API_URL + ], + 'invalid api url no scheme' => [ + 'name' => self::NAME, + 'email' => self::EMAIL, + 'apiUrl' => 'example.com/transmorpher/notifications' + ], + 'invalid api url no path' => [ + 'name' => self::NAME, + 'email' => self::EMAIL, + 'apiUrl' => 'https://example.com' + ], + 'invalid api url no scheme and no path' => [ + 'name' => self::NAME, + 'email' => self::EMAIL, + 'apiUrl' => 'example.com' + ], + 'invalid name and email and apiUrl' => [ + 'name' => 'invalid/name', + 'email' => 'invalidEmail', + 'apiUrl' => 'example.com' ] ]; } + + /** + * @param string|null $name + * @param string|null $email + * @param string|null $apiUrl + * @return int + */ + protected function createUser(?string $name = null, ?string $email = null, ?string $apiUrl = null): int + { + return Artisan::call(CreateUser::class, ['name' => $name ?? self::NAME, 'email' => $email ?? self::EMAIL, 'api_url' => $apiUrl ?? self::API_URL]); + } } diff --git a/tests/Unit/ImageTest.php b/tests/Unit/ImageTest.php index 52b664fe..10372d33 100644 --- a/tests/Unit/ImageTest.php +++ b/tests/Unit/ImageTest.php @@ -2,15 +2,21 @@ namespace Tests\Unit; +use App\Console\Commands\PurgeDerivatives; +use App\Enums\ClientNotification; use App\Enums\MediaStorage; use App\Enums\Transformation; use App\Exceptions\InvalidTransformationFormatException; use App\Exceptions\InvalidTransformationValueException; use App\Exceptions\TransformationNotFoundException; +use App\Helpers\SodiumHelper; use App\Models\Media; use App\Models\UploadSlot; use App\Models\Version; +use Artisan; +use Http; use Illuminate\Contracts\Filesystem\Filesystem; +use Illuminate\Http\Client\Request; use Illuminate\Http\UploadedFile; use Illuminate\Testing\TestResponse; use PHPUnit\Framework\Attributes\DataProvider; @@ -61,6 +67,7 @@ protected function uploadImage(string $uploadToken): TestResponse ]); } + #[Test] #[Depends('ensureImageUploadSlotCanBeReserved')] public function ensureImageCanBeUploaded(string $uploadToken) @@ -77,9 +84,17 @@ public function ensureImageCanBeUploaded(string $uploadToken) return $version; } + #[Test] + #[Depends('ensureImageUploadSlotCanBeReserved')] + #[Depends('ensureImageCanBeUploaded')] + public function ensureUploadTokenIsInvalidatedAfterUpload(string $uploadToken) + { + $this->uploadImage($uploadToken)->assertNotFound(); + } + protected function createDerivativeForVersion(Version $version): TestResponse { - return $this->get(route('getDerivative', [self::$user->name, $version->Media])); + return $this->get(route('getDerivative', [$this->user->name, $version->Media])); } #[Test] @@ -96,7 +111,7 @@ public function ensureProcessedFilesAreAvailable(Version $version) public function ensureUnprocessedFilesAreNotAvailable(Version $version) { $version->update(['processed' => 0]); - $getDerivativeResponse = $this->get(route('getDerivative', [self::$user->name, $version->Media])); + $getDerivativeResponse = $this->get(route('getDerivative', [$this->user->name, $version->Media])); $getDerivativeResponse->assertNotFound(); } @@ -114,8 +129,8 @@ protected function assertMediaDirectoryExists(Media $media): void protected function assertUserDirectoryExists(): void { - $this->originalsDisk->assertExists(self::$user->name); - $this->imageDerivativesDisk->assertExists(self::$user->name); + $this->originalsDisk->assertExists($this->user->name); + $this->imageDerivativesDisk->assertExists($this->user->name); } protected function assertVersionFilesMissing(Version $version): void @@ -132,11 +147,11 @@ protected function assertMediaDirectoryMissing(Media $media): void protected function assertUserDirectoryMissing(): void { - $this->originalsDisk->assertMissing(self::$user->name); - $this->imageDerivativesDisk->assertMissing(self::$user->name); + $this->originalsDisk->assertMissing($this->user->name); + $this->imageDerivativesDisk->assertMissing($this->user->name); } - protected function setupDeletionTest(): void + protected function setupMediaAndVersion(): void { $this->uploadToken = $this->reserveUploadSlot()->json()['upload_token']; $this->version = Media::whereIdentifier(self::IDENTIFIER)->first()->Versions()->whereNumber($this->uploadImage($this->uploadToken)['version'])->first(); @@ -148,7 +163,7 @@ protected function setupDeletionTest(): void #[Test] public function ensureVersionDeletionMethodsWork() { - $this->setupDeletionTest(); + $this->setupMediaAndVersion(); $this->assertVersionFilesExist($this->version); @@ -160,7 +175,7 @@ public function ensureVersionDeletionMethodsWork() #[Test] public function ensureMediaDeletionMethodsWork() { - $this->setupDeletionTest(); + $this->setupMediaAndVersion(); $this->assertVersionFilesExist($this->version); $this->assertMediaDirectoryExists($this->media); @@ -175,17 +190,46 @@ public function ensureMediaDeletionMethodsWork() $this->assertModelMissing($this->uploadSlot); } + #[Test] + public function ensureImageDerivativesArePurged() + { + $this->setupMediaAndVersion(); + + $this->assertVersionFilesExist($this->version); + + $cacheCounterBeforeCommand = $this->originalsDisk->get(config('transmorpher.cache_invalidation_counter_file_path')); + + Http::fake([ + $this->user->api_url => Http::response() + ]); + + Artisan::call(PurgeDerivatives::class, ['--image' => true]); + + $cacheCounterAfterCommand = $this->originalsDisk->get(config('transmorpher.cache_invalidation_counter_file_path')); + + Http::assertSent(function (Request $request) use ($cacheCounterAfterCommand) { + $decryptedNotification = json_decode(SodiumHelper::decrypt($request['signed_notification']), true); + + return $request->url() == $this->user->api_url + && $decryptedNotification['notification_type'] == ClientNotification::CACHE_INVALIDATION->value + && $decryptedNotification['cache_invalidator'] == $cacheCounterAfterCommand; + }); + + $this->assertTrue(++$cacheCounterBeforeCommand == $cacheCounterAfterCommand); + $this->imageDerivativesDisk->assertMissing($this->version->imageDerivativeFilePath()); + } + #[Test] public function ensureUserDeletionMethodsWork() { - $this->setupDeletionTest(); + $this->setupMediaAndVersion(); $this->assertVersionFilesExist($this->version); $this->assertMediaDirectoryExists($this->media); $this->assertUserDirectoryExists(); - $this->runProtectedMethod(self::$user, 'deleteRelatedModels'); - $this->runProtectedMethod(self::$user, 'deleteMediaDirectories'); + $this->runProtectedMethod($this->user, 'deleteRelatedModels'); + $this->runProtectedMethod($this->user, 'deleteMediaDirectories'); $this->assertVersionFilesMissing($this->version); $this->assertMediaDirectoryMissing($this->media); diff --git a/tests/Unit/VideoTest.php b/tests/Unit/VideoTest.php index fa0d9ef0..385dd19c 100644 --- a/tests/Unit/VideoTest.php +++ b/tests/Unit/VideoTest.php @@ -2,23 +2,34 @@ namespace Tests\Unit; +use App\Console\Commands\PurgeDerivatives; +use App\Enums\ClientNotification; use App\Enums\MediaStorage; use App\Enums\ResponseState; use App\Helpers\SodiumHelper; -use App\Jobs\TranscodeVideo; +use App\Jobs\ClientPurgeNotification; +use App\Models\Media; use App\Models\UploadSlot; +use Artisan; +use File; use Http; use Illuminate\Contracts\Filesystem\Filesystem; -use PHPUnit\Framework\Attributes\Depends; +use Illuminate\Foundation\Testing\RefreshDatabase; +use Illuminate\Http\Client\Request; +use Illuminate\Http\UploadedFile; +use Illuminate\Testing\TestResponse; use PHPUnit\Framework\Attributes\Test; +use Queue; use Storage; use Tests\MediaTest; +use Transcode; class VideoTest extends MediaTest { + use RefreshDatabase; + protected const IDENTIFIER = 'testVideo'; protected const VIDEO_NAME = 'video.mp4'; - protected const CALLBACK_URL = 'http://example.com/callback'; protected Filesystem $videoDerivativesDisk; protected function setUp(): void @@ -28,37 +39,120 @@ protected function setUp(): void $this->videoDerivativesDisk ??= Storage::persistentFake(config(sprintf('transmorpher.disks.%s', MediaStorage::VIDEO_DERIVATIVES->value))); } - #[Test] - public function ensureVideoUploadSlotCanBeReserved() + /** + * @return TestResponse + */ + protected function reserveUploadSlot(): TestResponse { - $reserveUploadSlotResponse = $this->json('POST', route('v1.reserveVideoUploadSlot'), [ + return $this->json('POST', route('v1.reserveVideoUploadSlot'), [ 'identifier' => self::IDENTIFIER, - 'callback_url' => self::CALLBACK_URL ]); + } + + #[Test] + public function ensureVideoUploadSlotCanBeReserved() + { + $reserveUploadSlotResponse = $this->reserveUploadSlot(); $reserveUploadSlotResponse->assertOk(); - return $reserveUploadSlotResponse->json()['upload_token']; + return $reserveUploadSlotResponse->json('upload_token'); + } + + protected function uploadVideo(): TestResponse + { + $uploadToken = $this->reserveUploadSlot()->json('upload_token'); + + return $this->post(route('v1.upload', [$uploadToken]), [ + 'file' => UploadedFile::fake()->createWithContent('video.mp4', File::get(base_path('tests/data/test.mp4'))), + 'identifier' => self::IDENTIFIER + ]); } #[Test] - #[Depends('ensureVideoUploadSlotCanBeReserved')] - public function ensureTranscodingIsAbortedWhenNewerVersionExists(string $uploadToken) + public function ensureVideoCanBeUploaded() { - Http::fake(); + Queue::fake(); + + $uploadResponse = $this->uploadVideo(); + + $uploadResponse->assertCreated(); + Queue::assertPushed(Transcode::getJobClass()); + + $media = Media::whereIdentifier($uploadResponse['identifier'])->first(); + $version = $media->Versions()->whereNumber($uploadResponse['version'])->first(); + $this->originalsDisk->assertExists($version->originalFilePath()); + } + + #[Test] + public function ensureTranscodingIsAbortedWhenNewerVersionExists() + { + $uploadToken = $this->reserveUploadSlot()->json('upload_token'); $uploadSlot = UploadSlot::firstWhere('token', $uploadToken); - $media = self::$user->Media()->create(['identifier' => self::IDENTIFIER, 'type' => $uploadSlot->media_type]); + $media = $this->user->Media()->create(['identifier' => self::IDENTIFIER, 'type' => $uploadSlot->media_type]); $outdatedVersion = $media->Versions()->create(['number' => 1, 'filename' => sprintf('1-%s', self::VIDEO_NAME)]); $media->Versions()->create(['number' => 2, 'filename' => sprintf('2-%s', self::VIDEO_NAME)]); - TranscodeVideo::dispatch($outdatedVersion, $uploadSlot); + Http::fake(); + + Transcode::createJob($outdatedVersion, $uploadSlot); $request = Http::recorded()[0][0]; - $transcodingResult = json_decode(SodiumHelper::decrypt($request->data()['signed_response']), true); + $transcodingResult = json_decode(SodiumHelper::decrypt($request->data()['signed_notification']), true); $this->assertEquals(ResponseState::TRANSCODING_ABORTED->getState()->value, $transcodingResult['state']); $this->assertEquals(ResponseState::TRANSCODING_ABORTED->getMessage(), $transcodingResult['message']); } + + #[Test] + public function ensureTranscodingWorks() + { + $uploadResponse = $this->uploadVideo(); + + $media = Media::whereIdentifier($uploadResponse['identifier'])->first(); + $version = $media->Versions()->whereNumber($uploadResponse['version'])->first(); + $uploadSlot = UploadSlot::withoutGlobalScopes()->whereToken($uploadResponse['upload_token'])->first(); + + Http::fake([ + $this->user->api_url => Http::response() + ]); + + $this->assertTrue(Transcode::createJob($version, $uploadSlot)); + + Http::assertSent(function (Request $request) { + $decryptedNotification = json_decode(SodiumHelper::decrypt($request['signed_notification']), true); + + return $request->url() == $this->user->api_url + && $decryptedNotification['notification_type'] == ClientNotification::VIDEO_TRANSCODING->value + && $decryptedNotification['state'] == ResponseState::TRANSCODING_SUCCESSFUL->getState()->value; + }); + + $this->videoDerivativesDisk->assertExists($media->videoDerivativeFilePath('mp4') . '.mp4'); + $this->videoDerivativesDisk->assertExists($media->videoDerivativeFilePath('hls') . '.m3u8'); + $this->videoDerivativesDisk->assertExists($media->videoDerivativeFilePath('dash') . '.mpd'); + } + + #[Test] + public function ensureVideoDerivativesArePurged() + { + $uploadResponse = $this->uploadVideo(); + + $media = Media::whereIdentifier($uploadResponse['identifier'])->first(); + $version = $media->currentVersion; + + $versionNumberBeforePurging = $version->number; + + Queue::fake(); + Http::fake([ + $this->user->api_url => Http::response() + ]); + + Artisan::call(PurgeDerivatives::class, ['--video' => true]); + + $this->assertTrue($versionNumberBeforePurging + 1 == $version->refresh()->number); + Queue::assertPushed(Transcode::getJobClass()); + Queue::assertPushed(ClientPurgeNotification::class); + } }