Feature/resource upload size (#352)

* Don't process ttl change if value has not changed * Optimize garbage collection on db connection * Transfer with speed declines doens't cancel * Add k6 load testing * Don't process ttl change if value has not changed * Optimize garbage collection on db connection * Transfer with speed declines doens't cancel * Add k6 load testing * keep connection alive for 1 hour * some changes to the test * update readme * format js file * Readme + some cleanup * Can change max filetransfer size on resource level * file transfer size should be nullable * Some readme changes * Fix spelling error and wrong error code * Remove inheritance from Repository --------- Co-authored-by: Hammerbeck <andreas.hammerbeck@digdir.no>
Altinn · Mar 12, 2024 · a3d90b8 · a3d90b8
1 parent a7f5ed2
commit a3d90b8
Show file tree

Hide file tree

Showing 21 changed files with 362 additions and 11 deletions.
diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -18,7 +18,8 @@
                 "pattern": "\\bNow listening on:\\s+(https?://\\S+)"
             },
             "env": {
-                "ASPNETCORE_ENVIRONMENT": "Development"
+                "ASPNETCORE_ENVIRONMENT": "Development",
+                "MAX_FILE_UPLOAD_SIZE": "2147483648"
             },
             "sourceFileMap": {
                 "/Views": "${workspaceFolder}/Views"

diff --git a/Dockerfile b/Dockerfile
@@ -17,6 +17,7 @@ FROM mcr.microsoft.com/dotnet/aspnet:8.0.2-alpine3.18 AS final
 WORKDIR /app
 EXPOSE 2525
 ENV ASPNETCORE_URLS=http://+:2525
+ENV MAX_FILE_UPLOAD_SIZE=2147483648
 
 COPY --from=build /app/out .
 #COPY src/Altinn.Broker.Persistence/Migration ./Migration

diff --git a/README.md b/README.md
@@ -13,6 +13,8 @@ Finally, you need to register a resource in the Resource Registry. First set the
 
 ## Local Development
 
+The start.ps1 script runs all neccassary commands to run the project. If you want to run the commands seperate, you can follow the steps below: 
+
 The services required to support local development are run using docker compose:
 ```docker compose up -d```
 
@@ -47,7 +49,7 @@ Formatting of the code base is handled by Dotnet format. [See how to configure i
 The build and push workflow produces a docker image that is pushed to Github packages. This image is then used by the release action found in the [altinn-broker-infra repository](https://github.com/Altinn/altinn-broker-infra).
 
 
-### Load testing with k6
+## Load testing with k6
 Before running tests you should mock the following:
 - AltinnAuthorization by setting the function CheckUserAccess to return true
 - AltinnRegisterService to return a string 

diff --git a/Test/Altinn.Broker.Tests/Helpers/CustomWebApplicationFactory.cs b/Test/Altinn.Broker.Tests/Helpers/CustomWebApplicationFactory.cs
@@ -108,6 +108,8 @@ protected override void ConfigureWebHost(
 
             var eventBus = new Mock<IEventBus>();
             services.AddSingleton(eventBus.Object);
+
+            Environment.SetEnvironmentVariable("MAX_FILE_UPLOAD_SIZE", "2147483648");
         });
     }
 

diff --git a/Test/Altinn.Broker.Tests/ResourceControllerTests.cs b/Test/Altinn.Broker.Tests/ResourceControllerTests.cs
@@ -0,0 +1,77 @@
+using System.Net;
+using System.Net.Http.Json;
+using System.Text.Json;
+
+using Altinn.Broker.Models;
+using Altinn.Broker.Tests.Helpers;
+
+using Xunit;
+
+namespace Altinn.Broker.Tests;
+public class ResourceControllerTests : IClassFixture<CustomWebApplicationFactory>
+{
+    private readonly CustomWebApplicationFactory _factory;
+    private readonly HttpClient _serviceOwnerClient;
+    private readonly HttpClient _senderClient;
+    private readonly HttpClient _recipientClient;
+    private readonly JsonSerializerOptions _responseSerializerOptions;
+
+    public ResourceControllerTests(CustomWebApplicationFactory factory)
+    {
+        _factory = factory;
+        _serviceOwnerClient = _factory.CreateClientWithAuthorization(TestConstants.DUMMY_SERVICE_OWNER_TOKEN);
+        _senderClient = _factory.CreateClientWithAuthorization(TestConstants.DUMMY_SENDER_TOKEN);
+        _recipientClient = _factory.CreateClientWithAuthorization(TestConstants.DUMMY_RECIPIENT_TOKEN);
+
+        _responseSerializerOptions = new JsonSerializerOptions(new JsonSerializerOptions()
+        {
+            PropertyNameCaseInsensitive = true
+        });
+        _responseSerializerOptions.Converters.Add(new System.Text.Json.Serialization.JsonStringEnumConverter());
+
+    }
+
+    [Fact]
+    public async Task Update_Resource_Max_Upload_Size()
+    {
+        var response = await _serviceOwnerClient.PutAsJsonAsync<ResourceExt>($"broker/api/v1/resource/MaxFileTransferSize", new ResourceExt
+        {
+            ResourceId = "123",
+            MaxFileTransferSize = 99999
+        });
+        Assert.True(response.IsSuccessStatusCode, await response.Content.ReadAsStringAsync());
+    }
+    [Fact]
+    public async Task Update_Resource_Max_Upload_Size_Over_Global_Should_Fail()
+    {
+        var response = await _serviceOwnerClient.PutAsJsonAsync<ResourceExt>($"broker/api/v1/resource/MaxFileTransferSize", new ResourceExt
+        {
+            ResourceId = "123",
+            MaxFileTransferSize = 999999999999999
+        });
+        Assert.True(response.StatusCode == HttpStatusCode.BadRequest, await response.Content.ReadAsStringAsync());
+    }
+
+    [Fact]
+    public async Task Update_Resource_Should_Fail_For_Sender()
+    {
+        var response = await _senderClient.PutAsJsonAsync<ResourceExt>($"broker/api/v1/resource/MaxFileTransferSize", new ResourceExt
+        {
+            ResourceId = "123",
+            MaxFileTransferSize = 1000000
+        });
+        Assert.True(response.StatusCode == HttpStatusCode.Forbidden, await response.Content.ReadAsStringAsync());
+    }
+
+    [Fact]
+    public async Task Update_Resource_Should_Fail_For_Receiver()
+    {
+        var response = await _recipientClient.PutAsJsonAsync<ResourceExt>($"broker/api/v1/resource/MaxFileTransferSize", new ResourceExt
+        {
+            ResourceId = "123",
+            MaxFileTransferSize = 1000000
+        });
+        Assert.True(response.StatusCode == HttpStatusCode.Forbidden, await response.Content.ReadAsStringAsync());
+    }
+
+}
diff --git a/Test/Altinn.Broker.Tests/ServiceOwnerControllerTests.cs b/Test/Altinn.Broker.Tests/ServiceOwnerControllerTests.cs
@@ -58,7 +58,7 @@ public async Task Update_FileRetention_For_ServiceOwner()
         };
 
         var retentionResponse = await _serviceOwnerClient.PutAsJsonAsync($"broker/api/v1/serviceowner/fileretention", serviceOwnerUpdateFileRetentionExt);
-        Assert.Equal(System.Net.HttpStatusCode.OK, retentionResponse.StatusCode);
+        Assert.True(HttpStatusCode.OK == retentionResponse.StatusCode || HttpStatusCode.Conflict == retentionResponse.StatusCode);
         var response = await _serviceOwnerClient.GetFromJsonAsync<ServiceOwnerOverviewExt>($"broker/api/v1/serviceowner", _responseSerializerOptions);
         Assert.Equal(TimeSpan.FromDays(90), response.FileTransferTimeToLive);
     }

diff --git a/altinn3-broker-postman-collection.json b/altinn3-broker-postman-collection.json
@@ -800,6 +800,51 @@
 				}
 			]
 		},
+		{
+			"name": "Resource",
+			"item": [
+				{
+					"name": "Update Max File Size",
+					"request": {
+						"auth": {
+							"type": "bearer",
+							"bearer": [
+								{
+									"key": "token",
+									"value": "{{serviceowner_token}}",
+									"type": "string"
+								}
+							]
+						},
+						"method": "PUT",
+						"header": [],
+						"body": {
+							"mode": "raw",
+							"raw": "{\r\n    \"resourceId\": \"{{resourceId}}\",\r\n    \"maxFileTransferSize\": \"219239123\"\r\n}",
+							"options": {
+								"raw": {
+									"language": "json"
+								}
+							}
+						},
+						"url": {
+							"raw": "{{baseUrl}}/broker/api/v1/resource/maxfiletransfersize",
+							"host": [
+								"{{baseUrl}}"
+							],
+							"path": [
+								"broker",
+								"api",
+								"v1",
+								"resource",
+								"maxfiletransfersize"
+							]
+						}
+					},
+					"response": []
+				}
+			]
+		},
 		{
 			"name": "Service Owner",
 			"item": [

diff --git a/src/Altinn.Broker.API/Controllers/FileTransferController.cs b/src/Altinn.Broker.API/Controllers/FileTransferController.cs
@@ -32,11 +32,15 @@ public class FileTransferController : Controller
     {
         private readonly ILogger<FileTransferController> _logger;
         private readonly IIdempotencyEventRepository _idempotencyEventRepository;
+        private readonly IResourceRepository _resourceRepository;
+        private readonly IFileTransferRepository _fileTransferRepository;
 
-        public FileTransferController(ILogger<FileTransferController> logger, IIdempotencyEventRepository idempotencyEventRepository)
+        public FileTransferController(ILogger<FileTransferController> logger, IIdempotencyEventRepository idempotencyEventRepository, IResourceRepository resourceRepository, IFileTransferRepository fileTransferRepository)
         {
             _logger = logger;
             _idempotencyEventRepository = idempotencyEventRepository;
+            _resourceRepository = resourceRepository;
+            _fileTransferRepository = fileTransferRepository;
         }
 
         /// <summary>
@@ -77,6 +81,14 @@ CancellationToken cancellationToken
             LogContextHelpers.EnrichLogsWithToken(token);
             _logger.LogInformation("Uploading file for file transfer {fileTransferId}", fileTransferId.ToString());
             Request.EnableBuffering();
+
+            var fileTransfer = await _fileTransferRepository.GetFileTransfer(fileTransferId, cancellationToken);
+            var resource = await _resourceRepository.GetResource(fileTransfer.ResourceId, cancellationToken);
+            var max_upload_size = resource?.MaxFileTransferSize ?? long.Parse(Environment.GetEnvironmentVariable("MAX_FILE_UPLOAD_SIZE"));
+            if (Request.ContentLength > max_upload_size || Request.Body.Length > max_upload_size)
+            {
+                return BadRequest($"File size exceeds maximum allowed size of {max_upload_size} bytes");
+            }
             var commandResult = await handler.Process(new UploadFileCommandRequest()
             {
                 FileTransferId = fileTransferId,

diff --git a/src/Altinn.Broker.API/Controllers/ResourceController.cs b/src/Altinn.Broker.API/Controllers/ResourceController.cs
@@ -0,0 +1,67 @@
+using Altinn.Broker.API.Configuration;
+using Altinn.Broker.Core.Domain;
+using Altinn.Broker.Core.Domain.Enums;
+using Altinn.Broker.Core.Repositories;
+using Altinn.Broker.Core.Services;
+using Altinn.Broker.Middlewares;
+using Altinn.Broker.Models;
+
+using Microsoft.AspNetCore.Authentication.JwtBearer;
+using Microsoft.AspNetCore.Authorization;
+using Microsoft.AspNetCore.Mvc;
+
+namespace Altinn.Broker.Controllers;
+
+[ApiController]
+[Route("broker/api/v1/resource")]
+[Authorize(AuthenticationSchemes = JwtBearerDefaults.AuthenticationScheme)]
+[Authorize(Policy = AuthorizationConstants.ServiceOwner)]
+public class ResourceController : Controller
+{
+    private readonly IResourceRepository _resourceRepository;
+    private readonly Core.Repositories.IAuthorizationService _resourceRightsRepository;
+    private readonly IResourceManager _resourceManager;
+
+    public ResourceController(IResourceRepository resourceRepository, IResourceManager resourceManager, Core.Repositories.IAuthorizationService resourceRightsRepository)
+    {
+        _resourceRepository = resourceRepository;
+        _resourceManager = resourceManager;
+        _resourceRightsRepository = resourceRightsRepository;
+
+    }
+
+    [HttpPut]
+    [Route("maxfiletransfersize")]
+    public async Task<ActionResult> UpdateMaxFileTransferSize([FromBody] ResourceExt resourceExt, [ModelBinder(typeof(MaskinportenModelBinder))] CallerIdentity token, CancellationToken cancellationToken)
+    {
+        var hasAccess = await _resourceRightsRepository.CheckUserAccess(resourceExt.ResourceId, token.ClientId, [ResourceAccessLevel.Write], false, cancellationToken);
+        if (!hasAccess)
+        {
+            return Unauthorized();
+        }
+        var resource = await _resourceRepository.GetResource(resourceExt.ResourceId, cancellationToken);
+
+        if (resource is null)
+        {
+            return NotFound();
+        }
+        if (resourceExt.MaxFileTransferSize < 0)
+        {
+            return BadRequest("Max upload size cannot be negative");
+        }
+        if (resourceExt.MaxFileTransferSize == resource.MaxFileTransferSize)
+        {
+            return BadRequest("Max upload size is already set to the requested value");
+        }
+        long maxFileTransferSize = long.Parse(Environment.GetEnvironmentVariable("MAX_FILE_UPLOAD_SIZE"));
+
+        if (resourceExt.MaxFileTransferSize > maxFileTransferSize)
+        {
+            return BadRequest("Max upload size cannot exceed the global maximum allowed size");
+        }
+        await _resourceRepository.UpdateMaxFileTransferSize(resourceExt.ResourceId, resourceExt.MaxFileTransferSize, cancellationToken);
+        return Ok();
+    }
+
+}
+
diff --git a/src/Altinn.Broker.API/Controllers/ServiceOwnerController.cs b/src/Altinn.Broker.API/Controllers/ServiceOwnerController.cs
@@ -82,7 +82,7 @@ public async Task<ActionResult> UpdateFileRetention([FromBody] ServiceOwnerUpdat
         var fileTimeToLive = XmlConvert.ToTimeSpan(serviceOwnerUpdateFileRetentionExt.FileTransferTimeToLive);
         if (fileTimeToLive == serviceOwner.FileTransferTimeToLive)
         {
-            return Problem(detail: "The file transfer already has the requested retention time", statusCode: (int)HttpStatusCode.Conflict);
+            return Problem(detail: "The file transfer already has the requested retention time", statusCode: (int)HttpStatusCode.BadRequest);
         }
         await _serviceOwnerRepository.UpdateFileRetention(token.Consumer, fileTimeToLive);
         await updateFileRetentionHandler.Process(new UpdateFileRetentionRequest

diff --git a/src/Altinn.Broker.API/Models/ResourceExt.cs b/src/Altinn.Broker.API/Models/ResourceExt.cs
@@ -0,0 +1,29 @@
+using System.ComponentModel.DataAnnotations;
+using System.Numerics;
+using System.Text.Json.Serialization;
+
+namespace Altinn.Broker.Models
+{
+    /// <summary>
+    /// API input model for file initialization.
+    /// </summary>
+    public class ResourceExt
+    {
+        /// <summary>
+        /// The Altinn resource ID
+        /// </summary>
+        [JsonPropertyName("resourceId")]
+        [StringLength(255, MinimumLength = 1)]
+        [Required]
+        public string ResourceId { get; set; } = string.Empty;
+
+        /// <summary>
+        /// The max upload size for the resource in bytes
+        /// </summary>
+        [JsonPropertyName("maxFileTransferSize")]
+        [Required]
+        public long MaxFileTransferSize { get; set; } = 0;
+
+
+    }
+}
diff --git a/src/Altinn.Broker.API/Properties/launchSettings.json b/src/Altinn.Broker.API/Properties/launchSettings.json
@@ -16,7 +16,8 @@
       "launchUrl": "swagger",
       "applicationUrl": "http://localhost:5096",
       "environmentVariables": {
-        "ASPNETCORE_ENVIRONMENT": "Development"
+        "ASPNETCORE_ENVIRONMENT": "Development",
+        "MAX_FILE_UPLOAD_SIZE": "2147483648"
       }
     },
     "https": {
@@ -26,16 +27,18 @@
       "launchUrl": "swagger",
       "applicationUrl": "https://localhost:7241;http://localhost:5096",
       "environmentVariables": {
-        "ASPNETCORE_ENVIRONMENT": "Development"
+        "ASPNETCORE_ENVIRONMENT": "Development",
+        "MAX_FILE_UPLOAD_SIZE": "2147483648"
       }
     },
     "IIS Express": {
       "commandName": "IISExpress",
       "launchBrowser": true,
       "launchUrl": "swagger",
       "environmentVariables": {
-        "ASPNETCORE_ENVIRONMENT": "Development"
+        "ASPNETCORE_ENVIRONMENT": "Development",
+        "MAX_FILE_UPLOAD_SIZE": "2147483648"
       }
     }
   }
-}
+}
diff --git a/src/Altinn.Broker.Core/Domain/ResourceEntity.cs b/src/Altinn.Broker.Core/Domain/ResourceEntity.cs
@@ -6,4 +6,5 @@ public class ResourceEntity
     public DateTimeOffset? Created { get; set; }
     public string? OrganizationNumber { get; set; }
     public string ServiceOwnerId { get; set; }
+    public long? MaxFileTransferSize { get; set; }
 }
diff --git a/src/Altinn.Broker.Core/Repositories/IAltinnResourceRepoistory.cs b/src/Altinn.Broker.Core/Repositories/IAltinnResourceRepoistory.cs
@@ -0,0 +1,10 @@
+
+using System.Numerics;
+
+using Altinn.Broker.Core.Domain;
+
+namespace Altinn.Broker.Core.Repositories;
+public interface IAltinnResourceRepository
+{
+    Task<ResourceEntity?> GetResource(string resourceId, CancellationToken cancellationToken = default);
+}
diff --git a/src/Altinn.Broker.Core/Repositories/IResourceRepository.cs b/src/Altinn.Broker.Core/Repositories/IResourceRepository.cs
@@ -1,8 +1,12 @@
 
+using System.Numerics;
+
 using Altinn.Broker.Core.Domain;
 
 namespace Altinn.Broker.Core.Repositories;
 public interface IResourceRepository
 {
     Task<ResourceEntity?> GetResource(string resourceId, CancellationToken cancellationToken = default);
+    Task UpdateMaxFileTransferSize(string resourceId, long maxSize, CancellationToken cancellationToken = default);
+    Task CreateResource(ResourceEntity resource, CancellationToken cancellationToken = default);
 }
diff --git a/src/Altinn.Broker.Integrations/Altinn/ResourceRegistry/AltinnResourceRegistryRepository.cs b/src/Altinn.Broker.Integrations/Altinn/ResourceRegistry/AltinnResourceRegistryRepository.cs
@@ -10,7 +10,7 @@
 using Microsoft.Extensions.Options;
 
 namespace Altinn.Broker.Integrations.Altinn.ResourceRegistry;
-public class AltinnResourceRegistryRepository : IResourceRepository
+public class AltinnResourceRegistryRepository : IAltinnResourceRepository
 {
     private readonly HttpClient _client;
     private readonly ILogger<AltinnResourceRegistryRepository> _logger;
-Original file line number
+Diff line change
@@ Expand Up / @@ -108,6 +108,8 @@ protected override void ConfigureWebHost( @@
                 var eventBus = new Mock<IEventBus>();
                 services.AddSingleton(eventBus.Object);
+                Environment.SetEnvironmentVariable("MAX_FILE_UPLOAD_SIZE", "2147483648");
             });
         }
@@ Expand Down @@