Checksum validation

Test/Altinn.Broker.Tests/FileControllerTests.cs

@@ -247,6 +247,105 @@ public async Task SendFile_UsingUnregisteredUser_Fails()
         Assert.Equal(Errors.NoAccessToResource.Message, parsedError.Detail);
     }
 
+    [Fact]
+    public async Task UploadFile_ChecksumCorrect_Succeeds()
+    {
+        // Arrange
+        var fileId = await InitializeAndAssertBasicFile();
+        var fileContent = "This is the contents of the uploaded file";
+        var fileContentBytes = Encoding.UTF8.GetBytes(fileContent);
+        var checksum = CalculateChecksum(fileContentBytes);
+        var file = FileInitializeExtTestFactory.BasicFile();
+        file.Checksum = checksum;
+
+        // Act
+        var initializeFileResponse = await _senderClient.PostAsJsonAsync("broker/api/v1/file", file);
+        Assert.True(initializeFileResponse.IsSuccessStatusCode, await initializeFileResponse.Content.ReadAsStringAsync());
+        var uploadResponse = await UploadTextFile(fileId, fileContent);
+        Assert.True(uploadResponse.IsSuccessStatusCode, await uploadResponse.Content.ReadAsStringAsync());
+    }
+
+    [Fact]
+    public async Task UploadFile_MismatchChecksum_Fails()
+    {
+        // Arrange
+        var fileContent = "This is the contents of the uploaded file";
+        var incorrectChecksumContent = "NOT THE CONTENTS OF UPLOADED FILE";
+        var incorrectChecksumContentBytes = Encoding.UTF8.GetBytes(incorrectChecksumContent);
+        var incorrectChecksum = CalculateChecksum(incorrectChecksumContentBytes);
+        var file = FileInitializeExtTestFactory.BasicFile();
+        file.Checksum = incorrectChecksum;
+
+        // Act
+        var initializeFileResponse = await _senderClient.PostAsJsonAsync("broker/api/v1/file", file);
+        Assert.True(initializeFileResponse.IsSuccessStatusCode, await initializeFileResponse.Content.ReadAsStringAsync());
+        var fileId = await initializeFileResponse.Content.ReadAsStringAsync();
+        var uploadResponse = await UploadTextFile(fileId, fileContent);
+
+        // Assert
+        Assert.False(uploadResponse.IsSuccessStatusCode);
+        Assert.True(uploadResponse.StatusCode == HttpStatusCode.BadRequest);
+    }
+
+    [Fact]
+    public async Task UploadFile_NoChecksumSetWhenInitialized_ChecksumSetAfterUpload()
+    {
+        // Arrange
+        var fileId = await InitializeAndAssertBasicFile();
+        var fileContent = "This is the contents of the uploaded file";
+        var fileContentBytes = Encoding.UTF8.GetBytes(fileContent);
+        var checksum = CalculateChecksum(fileContentBytes);
+
+        // Act
+        var uploadResponse = await UploadTextFile(fileId, fileContent);
+
+        // Assert
+        Assert.True(uploadResponse.IsSuccessStatusCode, await uploadResponse.Content.ReadAsStringAsync());
+
+        // Check if the checksum is set in the file details
+        var fileDetails = await _senderClient.GetFromJsonAsync<FileOverviewExt>($"broker/api/v1/file/{fileId}", _responseSerializerOptions);
+        Assert.NotNull(fileDetails);
+        Assert.NotNull(fileDetails.Checksum);
+        Assert.Equal(checksum, fileDetails.Checksum);
+    }
+
+    [Fact]
+    public async Task UploadFile_ChecksumSetWhenInitialized_SameChecksumSetAfterUpload()
+    {
+        // Arrange
+        var fileContent = "This is the contents of the uploaded file";
+        var fileContentBytes = Encoding.UTF8.GetBytes(fileContent);
+        var checksum = CalculateChecksum(fileContentBytes);
+        var file = FileInitializeExtTestFactory.BasicFile();
+        file.Checksum = checksum;
+
+        // Act
+        var initializeFileResponse = await _senderClient.PostAsJsonAsync("broker/api/v1/file", file);
+        Assert.True(initializeFileResponse.IsSuccessStatusCode, await initializeFileResponse.Content.ReadAsStringAsync());
+        var fileId = await initializeFileResponse.Content.ReadAsStringAsync();
+        var uploadResponse = await UploadTextFile(fileId, fileContent);
+
+        // Assert
+        Assert.True(uploadResponse.IsSuccessStatusCode, await uploadResponse.Content.ReadAsStringAsync());
+
+        // Check if the checksum is unchanged in the file details
+        var fileDetails = await _senderClient.GetFromJsonAsync<FileOverviewExt>($"broker/api/v1/file/{fileId}", _responseSerializerOptions);
+        Assert.NotNull(fileDetails);
+        Assert.NotNull(fileDetails.Checksum);
+        Assert.Equal(checksum, fileDetails.Checksum);
+    }
+
+    private async Task<HttpResponseMessage> UploadTextFile(string fileId, string fileContent)
+    {
+        var fileContents = Encoding.UTF8.GetBytes(fileContent);
+        using (var content = new ByteArrayContent(fileContents))
+        {
+            content.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream");
+            var uploadResponse = await _senderClient.PostAsync($"broker/api/v1/file/{fileId}/upload", content);
+            return uploadResponse;
+        }
+    }
+
     private async Task UploadDummyFileAsync(string fileId)
     {
         var fileContents = Encoding.UTF8.GetBytes("This is the contents of the uploaded file");
@@ -264,4 +363,13 @@ private async Task<string> InitializeAndAssertBasicFile()
         Assert.True(initializeFileResponse.IsSuccessStatusCode, await initializeFileResponse.Content.ReadAsStringAsync());
         return await initializeFileResponse.Content.ReadAsStringAsync();
     }
+
+    private string CalculateChecksum(byte[] data)
+    {
+        using (var md5 = System.Security.Cryptography.MD5.Create())
+        {
+            byte[] hash = md5.ComputeHash(data);
+            return BitConverter.ToString(hash).Replace("-", "").ToLower();
+        }
+    }
 }

src/Altinn.Broker.API/Models/FileInitializeExt.cs

@@ -81,6 +81,10 @@ protected override ValidationResult IsValid(object value, ValidationContext vali
             {
                 return new ValidationResult("The checksum, if used, must be a MD5 hash with a length of 32 characters");
             }
+            if (stringValue.ToLowerInvariant() != stringValue)
+            {
+                return new ValidationResult("The checksum, if used, must be a MD5 hash in lower case");
+            }
             return ValidationResult.Success;
         }
     }

src/Altinn.Broker.Application/Altinn.Broker.Application.csproj

@@ -14,6 +14,7 @@
   </ItemGroup>
 
   <ItemGroup>
+    <PackageReference Include="Azure.Storage.Blobs" Version="12.19.1" />
     <PackageReference Include="Hangfire.Core" Version="1.8.9" />
     <PackageReference Include="Microsoft.AspNetCore.Mvc.Core" Version="2.2.5" />
     <PackageReference Include="Microsoft.Extensions.Configuration.Abstractions" Version="8.0.0" />

src/Altinn.Broker.Application/Errors.cs

@@ -15,4 +15,6 @@ public static class Errors
     public static Error ResourceNotConfigured = new Error(7, "Resource needs to be configured to use the broker API", HttpStatusCode.Unauthorized);
     public static Error NoAccessToResource = new Error(8, "You must use a bearer token that represents a system user with access to the resource in the Resource Rights Registry", HttpStatusCode.Unauthorized);
     public static Error FileNotAvailable = new Error(9, "The requested file is not ready for download. See file status.", HttpStatusCode.Forbidden);
+    public static Error UploadFailed = new Error(10, "Error occurred while uploading file. See /details for more information.", HttpStatusCode.InternalServerError);
+    public static Error ChecksumMismatch = new Error(12, "The checksum of uploaded file did not match the checksum specified in initialize call.", HttpStatusCode.BadRequest);
 }

src/Altinn.Broker.Application/UploadFileCommand/UploadFileCommandHandler.cs

@@ -2,6 +2,8 @@
 using Altinn.Broker.Core.Domain.Enums;
 using Altinn.Broker.Core.Repositories;
 
+using Hangfire;
+
 using Microsoft.Extensions.Logging;
 
 using OneOf;
@@ -16,16 +18,18 @@ public class UploadFileCommandHandler : IHandler<UploadFileCommandRequest, Guid>
     private readonly IFileRepository _fileRepository;
     private readonly IFileStatusRepository _fileStatusRepository;
     private readonly IBrokerStorageService _brokerStorageService;
+    private readonly IBackgroundJobClient _backgroundJobClient;
     private readonly ILogger<UploadFileCommandHandler> _logger;
 
-    public UploadFileCommandHandler(IResourceRightsRepository resourceRightsRepository, IResourceRepository resourceRepository, IResourceOwnerRepository resourceOwnerRepository, IFileRepository fileRepository, IFileStatusRepository fileStatusRepository, IBrokerStorageService brokerStorageService, ILogger<UploadFileCommandHandler> logger)
+    public UploadFileCommandHandler(IResourceRightsRepository resourceRightsRepository, IResourceRepository resourceRepository, IResourceOwnerRepository resourceOwnerRepository, IFileRepository fileRepository, IFileStatusRepository fileStatusRepository, IBrokerStorageService brokerStorageService, IBackgroundJobClient backgroundJobClient, ILogger<UploadFileCommandHandler> logger)
     {
         _resourceRightsRepository = resourceRightsRepository;
         _resourceRepository = resourceRepository;
         _resourceOwnerRepository = resourceOwnerRepository;
         _fileRepository = fileRepository;
         _fileStatusRepository = fileStatusRepository;
         _brokerStorageService = brokerStorageService;
+        _backgroundJobClient = backgroundJobClient;
         _logger = logger;
     }
 
@@ -61,7 +65,26 @@ public async Task<OneOf<Guid, Error>> Process(UploadFileCommandRequest request)
         };
 
         await _fileStatusRepository.InsertFileStatus(request.FileId, FileStatus.UploadStarted);
-        await _brokerStorageService.UploadFile(resourceOwner, file, request.Filestream);
+        try
+        {
+            var checksum = await _brokerStorageService.UploadFile(resourceOwner, file, request.Filestream);
+            if (string.IsNullOrWhiteSpace(file.Checksum))
+            {
+                await _fileRepository.SetChecksum(request.FileId, checksum);
+            }
+            else if (!string.Equals(checksum, file.Checksum, StringComparison.InvariantCultureIgnoreCase))
+            {
+                await _fileStatusRepository.InsertFileStatus(request.FileId, FileStatus.Failed, "Checksum mismatch");
+                _backgroundJobClient.Enqueue(() => _brokerStorageService.DeleteFile(resourceOwner, file));
+                return Errors.ChecksumMismatch;
+            }
+        }
+        catch (Exception e)
+        {
+            _logger.LogError("Unexpected error occurred while uploading file: {errorMessage} \nStack trace: {stackTrace}", e.Message, e.StackTrace);
+            await _fileStatusRepository.InsertFileStatus(request.FileId, FileStatus.Failed, "Error occurred while uploading file.");
+            return Errors.UploadFailed;
+        }
         await _fileRepository.SetStorageDetails(request.FileId, resourceOwner.StorageProvider.Id, request.FileId.ToString(), request.Filestream.Length);
         await _fileStatusRepository.InsertFileStatus(request.FileId, FileStatus.UploadProcessing);
         // TODO, async jobs

src/Altinn.Broker.Core/Repositories/IFileRepository.cs

@@ -19,6 +19,7 @@ Task<Guid> AddFile(
     Task<List<Guid>> GetFilesAssociatedWithActor(FileSearchEntity fileSearch);
     Task<List<Guid>> GetFilesForRecipientWithRecipientStatus(FileSearchEntity fileSearch);
     Task<List<Guid>> LegacyGetFilesForRecipientsWithRecipientStatus(LegacyFileSearchEntity fileSearch);
+    Task SetChecksum(Guid fileId, string checksum);
     Task SetStorageDetails(
         Guid fileId,
         long storageProviderId,

src/Altinn.Broker.Core/Repositories/IFileStore.cs

@@ -3,7 +3,7 @@ namespace Altinn.Broker.Repositories
     public interface IFileStore
     {
         Task<Stream> GetFileStream(Guid fileId, string connectionString);
-        Task UploadFile(Stream filestream, Guid fileId, string connectionString);
+        Task<string> UploadFile(Stream filestream, Guid fileId, string connectionString);
         Task DeleteFile(Guid fileId, string connectionString);
     }
 }

src/Altinn.Broker.Core/Services/IBrokerStorageService.cs

@@ -8,10 +8,10 @@ public interface IBrokerStorageService
     /// <summary>
     /// Looks up the correct storage account to use for service owner and upload the file
     /// </summary>
-    /// <param name="resourceOwnerEntity"></param>
-    /// <param name="stream"></param>
-    /// <returns></returns>
-    Task UploadFile(ResourceOwnerEntity resourceOwnerEntity, FileEntity fileEntity, Stream stream);
+    /// <param name="resourceOwnerEntity">The resource owner entity.</param>
+    /// <param name="stream">The stream to upload.</param>
+    /// <returns>A string containing the MD5 checksum</returns>
+    Task<string> UploadFile(ResourceOwnerEntity resourceOwnerEntity, FileEntity fileEntity, Stream stream);
 
     Task<Stream> DownloadFile(ResourceOwnerEntity resourceOwnerEntity, FileEntity file);
     Task DeleteFile(ResourceOwnerEntity resourceOwnerEntity, FileEntity fileEntity);

src/Altinn.Broker.Integrations/Azure/AzureBrokerStorageService.cs

@@ -21,10 +21,10 @@ public AzureBrokerStorageService(IFileStore fileStore, IResourceManager resource
         _logger = logger;
     }
 
-    public async Task UploadFile(ResourceOwnerEntity resourceOwnerEntity, FileEntity fileEntity, Stream stream)
+    public async Task<string> UploadFile(ResourceOwnerEntity resourceOwnerEntity, FileEntity fileEntity, Stream stream)
     {
         var connectionString = await GetConnectionString(resourceOwnerEntity);
-        await _fileStore.UploadFile(stream, fileEntity.FileId, connectionString);
+        return await _fileStore.UploadFile(stream, fileEntity.FileId, connectionString);
     }
 
     public async Task<Stream> DownloadFile(ResourceOwnerEntity resourceOwnerEntity, FileEntity fileEntity)

src/Altinn.Broker.Integrations/Azure/BlobService.cs

@@ -1,3 +1,5 @@
+using Altinn.Broker.Repositories;
+
 using Azure;
 using Azure.Storage;
 using Azure.Storage.Blobs;
@@ -8,7 +10,7 @@
 
 namespace Altinn.Broker.Integrations.Azure;
 
-public class BlobService : Repositories.IFileStore
+public class BlobService : IFileStore
 {
 
     private readonly ILogger<BlobService> _logger;
@@ -40,7 +42,7 @@ public async Task<Stream> GetFileStream(Guid fileId, string connectionString)
         }
     }
 
-    public async Task UploadFile(Stream stream, Guid fileId, string connectionString)
+    public async Task<string> UploadFile(Stream stream, Guid fileId, string connectionString)
     {
         BlobClient blobClient = GetBlobClient(fileId, connectionString);
         var blobLeaseClient = blobClient.GetBlobLeaseClient();
@@ -59,7 +61,10 @@ public async Task UploadFile(Stream stream, Guid fileId, string connectionString
                 },
                 TransferValidation = new UploadTransferValidationOptions { ChecksumAlgorithm = StorageChecksumAlgorithm.MD5 },
             };
-            await blobClient.UploadAsync(stream, options);
+            var blobMetadata = await blobClient.UploadAsync(stream, options);
+            var metadata = blobMetadata.Value;
+            var hash = Convert.ToHexString(metadata.ContentHash).ToLowerInvariant();
+            return hash;
         }
         catch (RequestFailedException requestFailedException)
         {

src/Altinn.Broker.Persistence/Altinn.Broker.Persistence.csproj

@@ -10,6 +10,10 @@
     <Compile Remove="Migrations\**" />
   </ItemGroup>
 
+  <ItemGroup>
+    <Content Include="Migrations\**" />
+  </ItemGroup>
+
   <ItemGroup>
     <PackageReference Include="Azure.Identity" Version="1.10.4" />
     <PackageReference Include="Azure.Storage.Blobs" Version="12.19.1" />

src/Altinn.Broker.Persistence/Repositories/FileRepository.cs

@@ -470,4 +470,18 @@ private async Task SetMetadata(Guid fileId, Dictionary<string, string> property)
             throw;
         }
     }
+
+    public async Task SetChecksum(Guid fileId, string checksum)
+    {
+        await using (var command = await _connectionProvider.CreateCommand(
+            "UPDATE broker.file " +
+            "SET " +
+                "checksum = @checksum " +
+            "WHERE file_id_pk = @fileId"))
+        {
+            command.Parameters.AddWithValue("@fileId", fileId);
+            command.Parameters.AddWithValue("@checksum", checksum);
+            command.ExecuteNonQuery();
+        }
+    }
 }