From f4ee9342ee4bcc0f63e9913d22df19626c98b11d Mon Sep 17 00:00:00 2001 From: Maksym Sadovnychyy Date: Sun, 3 Nov 2024 01:14:42 +0100 Subject: [PATCH] (feature): changed files checksum algorythm, padded files descriptor with encryption and integrity check, handling secret in env var or text file fallback --- src/MaksIT.LTO.Backup/Application.cs | 204 +++++++++++------- .../MaksIT.LTO.Backup.csproj | 4 + src/MaksIT.LTO.Core/MaksIT.LTO.Core.csproj | 1 + .../Utilities/AESGCMUtility.cs | 57 +++++ .../Utilities/ChecksumUtility.cs | 130 +++++++++++ 5 files changed, 320 insertions(+), 76 deletions(-) create mode 100644 src/MaksIT.LTO.Core/Utilities/AESGCMUtility.cs create mode 100644 src/MaksIT.LTO.Core/Utilities/ChecksumUtility.cs diff --git a/src/MaksIT.LTO.Backup/Application.cs b/src/MaksIT.LTO.Backup/Application.cs index dc5e917..cb611c7 100644 --- a/src/MaksIT.LTO.Backup/Application.cs +++ b/src/MaksIT.LTO.Backup/Application.cs @@ -9,6 +9,8 @@ using MaksIT.LTO.Core; using MaksIT.LTO.Backup.Entities; using MaksIT.LTO.Core.MassStorage; using MaksIT.LTO.Core.Networking; +using MaksIT.LTO.Core.Utilities; +using MaksIT.LTO.Core.Helpers; namespace MaksIT.LTO.Backup; @@ -16,16 +18,19 @@ namespace MaksIT.LTO.Backup; public class Application { private const string _descriptoFileName = "descriptor.json"; + private const string _secretFileName = "secret.txt"; private readonly string appPath = AppDomain.CurrentDomain.BaseDirectory; private readonly string _tapePath; private readonly string _descriptorFilePath; + private readonly string _secretFilePath; private readonly ILogger _logger; private readonly ILogger _tapeDeviceLogger; private readonly ILogger _networkConnectionLogger; private readonly Configuration _configuration; + private readonly string _secret; public Application( ILogger logger, @@ -37,9 +42,26 @@ public class Application { _networkConnectionLogger = loggerFactory.CreateLogger(); _descriptorFilePath = Path.Combine(appPath, _descriptoFileName); + _secretFilePath = Path.Combine(appPath, _secretFileName); _configuration = configuration.Value; _tapePath = _configuration.TapePath; + + var secret = Environment.GetEnvironmentVariable("LTO_BACKUP_SECRET") + ?? Environment.GetEnvironmentVariable("LTO_BACKUP_SECRET", EnvironmentVariableTarget.Machine); + + if (!string.IsNullOrWhiteSpace(secret)) + _secret = secret; + else if (!File.Exists(_secretFilePath)) { + _secret = AESGCMUtility.GenerateKeyBase64(); + File.WriteAllText(_secretFilePath, _secret); + } + else + _secret = File.ReadAllText(_secretFilePath); + + if (string.IsNullOrWhiteSpace(_secret)) { + throw new InvalidOperationException("Secret is required for encryption."); + } } public void Run() { @@ -95,31 +117,31 @@ public class Application { } } - public void LoadTape() { + private void LoadTape() { using var handler = new TapeDeviceHandler(_tapeDeviceLogger, _tapePath); LoadTape(handler); } - public void LoadTape(TapeDeviceHandler handler) { + private void LoadTape(TapeDeviceHandler handler) { handler.Prepare(TapeDeviceHandler.TAPE_LOAD); Thread.Sleep(2000); _logger.LogInformation("Tape loaded."); } - public void EjectTape() { + private void EjectTape() { using var handler = new TapeDeviceHandler(_tapeDeviceLogger, _tapePath); EjectTape(handler); } - public void EjectTape(TapeDeviceHandler handler) { + private void EjectTape(TapeDeviceHandler handler) { handler.Prepare(TapeDeviceHandler.TAPE_UNLOAD); Thread.Sleep(2000); _logger.LogInformation("Tape ejected."); } - public void TapeErase() { + private void TapeErase() { using var handler = new TapeDeviceHandler(_tapeDeviceLogger, _tapePath); LoadTape(handler); @@ -143,12 +165,12 @@ public class Application { _logger.LogInformation("Tape erased."); } - public void GetDeviceStatus() { + private void GetDeviceStatus() { using var handler = new TapeDeviceHandler(_tapeDeviceLogger, _tapePath); handler.GetStatus(); } - public void PathAccessWrapper(WorkingFolder workingFolder, Action myAction) { + private void PathAccessWrapper(WorkingFolder workingFolder, Action myAction) { if (workingFolder.LocalPath != null) { var localPath = workingFolder.LocalPath.Path; @@ -182,7 +204,7 @@ public class Application { } } - public void CreateDescriptor(WorkingFolder workingFolder, string descriptorFilePath, uint blockSize) { + private void CreateDescriptor(WorkingFolder workingFolder, string descriptorFilePath, uint blockSize) { PathAccessWrapper(workingFolder, (directoryPath) => { var files = Directory.GetFiles(directoryPath, "*.*", SearchOption.AllDirectories); @@ -196,18 +218,8 @@ public class Application { var relativePath = Path.GetRelativePath(directoryPath, filePath); var numberOfBlocks = (uint)((fileInfo.Length + blockSize - 1) / blockSize); - // Optional: Calculate a simple hash for file integrity (e.g., MD5) - using var md5 = System.Security.Cryptography.MD5.Create(); - using var fileStream = new FileStream(filePath, FileMode.Open, FileAccess.Read); - using var bufferedStream = new BufferedStream(fileStream, (int)blockSize); - - byte[] buffer = new byte[blockSize]; - int bytesRead; - while ((bytesRead = bufferedStream.Read(buffer, 0, buffer.Length)) > 0) { - md5.TransformBlock(buffer, 0, bytesRead, null, 0); - } - md5.TransformFinalBlock(Array.Empty(), 0, 0); - string fileHash = BitConverter.ToString(md5.Hash).Replace("-", "").ToLower(); + // Calculate CRC32 checksum for file integrity + string fileHash = ChecksumUtility.CalculateCRC32ChecksumFromFileInChunks(filePath, (int)blockSize); descriptor.Add(new FileDescriptor { StartBlock = currentTapeBlock, // Position of the file on the tape @@ -247,7 +259,17 @@ public class Application { } } - public void WriteFilesToTape(WorkingFolder workingFolder, string descriptorFilePath, uint blockSize) { + private byte[] AddPadding(byte[] data, int blockSize) { + int paddingSize = blockSize - (data.Length % blockSize); + byte[] paddedData = new byte[data.Length + paddingSize]; + Array.Copy(data, paddedData, data.Length); + for (int i = data.Length; i < paddedData.Length; i++) { + paddedData[i] = (byte)paddingSize; + } + return paddedData; + } + + private void WriteFilesToTape(WorkingFolder workingFolder, string descriptorFilePath, uint blockSize) { PathAccessWrapper(workingFolder, (directoryPath) => { _logger.LogInformation($"Writing files to tape from: {directoryPath}."); _logger.LogInformation($"Block Size: {blockSize}."); @@ -279,9 +301,6 @@ public class Application { var currentTapeBlock = (descriptorJson.Length + blockSize - 1) / blockSize; - - int writeError = 0; - foreach (var file in descriptor.Files) { var filePath = Path.Combine(directoryPath, file.FilePath); using var fileStream = new FileStream(filePath, FileMode.Open, FileAccess.Read); @@ -296,7 +315,7 @@ public class Application { Array.Clear(buffer, bytesRead, buffer.Length - bytesRead); } - writeError = handler.WriteData(buffer); + var writeError = handler.WriteData(buffer); if (writeError != 0) { _logger.LogInformation($"Failed to write file: {filePath}"); return; @@ -314,14 +333,35 @@ public class Application { // write descriptor to tape var descriptorData = Encoding.UTF8.GetBytes(descriptorJson); - var descriptorBlocks = (descriptorData.Length + blockSize - 1) / blockSize; - for (int i = 0; i < descriptorBlocks; i++) { + descriptorData = AESGCMUtility.EncryptData(descriptorData, _secret); + + // calculate the padding size + var paddingSize = blockSize - (descriptorData.Length % blockSize); + if (paddingSize == blockSize) { + paddingSize = 0; + } + + // add padding to the descriptor data + var paddedDescriptorData = new byte[descriptorData.Length + paddingSize + 1]; + Array.Copy(descriptorData, paddedDescriptorData, descriptorData.Length); + + // fill the padding with a specific value (e.g., 0x00) + for (int i = descriptorData.Length; i < paddedDescriptorData.Length - 1; i++) { + paddedDescriptorData[i] = 0x00; + } + + // append the padding size at the end + paddedDescriptorData[paddedDescriptorData.Length - 1] = (byte)paddingSize; + + // calculate the number of blocks needed + var descriptorBlocks = (paddedDescriptorData.Length + blockSize - 1) / blockSize; + for (var i = 0; i < descriptorBlocks; i++) { var startIndex = i * blockSize; - var length = Math.Min(blockSize, descriptorData.Length - startIndex); - byte[] block = new byte[blockSize]; // Initialized with zeros by default - Array.Copy(descriptorData, startIndex, block, 0, length); - - writeError = handler.WriteData(block); + var length = Math.Min(blockSize, paddedDescriptorData.Length - startIndex); + var block = new byte[blockSize]; // Initialized with zeros by default + Array.Copy(paddedDescriptorData, startIndex, block, 0, length); + + var writeError = handler.WriteData(block); if (writeError != 0) return; @@ -329,8 +369,9 @@ public class Application { Thread.Sleep(_configuration.WriteDelay); // Small delay between blocks } - // write 3 0 filled blocks to indicate end of backup - ZeroFillBlocks(handler, 3, blockSize); + // write mark to indicate end of files + handler.WriteMarks(TapeDeviceHandler.TAPE_FILEMARKS, 2); + Thread.Sleep(_configuration.WriteDelay); handler.Prepare(TapeDeviceHandler.TAPE_UNLOCK); Thread.Sleep(2000); @@ -341,7 +382,7 @@ public class Application { }); } - public BackupDescriptor? FindDescriptor(uint blockSize) { + private BackupDescriptor? FindDescriptor(uint blockSize) { _logger.LogInformation("Searching for descriptor on tape..."); _logger.LogInformation($"Block Size: {blockSize}."); @@ -357,34 +398,55 @@ public class Application { handler.SetPosition(TapeDeviceHandler.TAPE_SPACE_FILEMARKS, 0, 1); Thread.Sleep(2000); - handler.WaitForTapeReady(); + var position = handler.GetPosition(TapeDeviceHandler.TAPE_ABSOLUTE_BLOCK); + if (position.Error != null) + return null; - // Read data from tape until 3 zero-filled blocks are found + var desctiptorBlocks = position.OffsetLow; + + handler.SetPosition(TapeDeviceHandler.TAPE_SPACE_FILEMARKS, 0, 2); + Thread.Sleep(2000); + + position = handler.GetPosition(TapeDeviceHandler.TAPE_ABSOLUTE_BLOCK); + if (position.Error != null) + return null; + + desctiptorBlocks = position.OffsetLow - desctiptorBlocks; + + + var padding = handler.ReadData(blockSize); + + handler.SetPosition(TapeDeviceHandler.TAPE_SPACE_FILEMARKS, 0, 1); + Thread.Sleep(2000); + + // read data from descriptorBlocks var buffer = new List(); - byte[] data; - var zeroBlocks = 0; - do { - data = handler.ReadData(blockSize); + for (var i = 0; i < desctiptorBlocks; i++) { + var data = handler.ReadData(blockSize); buffer.AddRange(data); - if (data.All(b => b == 0)) { - zeroBlocks++; - } - else { - zeroBlocks = 0; - } - } while (zeroBlocks < 3); - - // Remove the last 3 zero-filled blocks from the buffer - var totalZeroBlocksSize = (int)(3 * blockSize); - if (buffer.Count >= totalZeroBlocksSize) { - buffer.RemoveRange(buffer.Count - totalZeroBlocksSize, totalZeroBlocksSize); } - // Convert buffer to byte array - var byteArray = buffer.ToArray(); + // Convert buffer to array + var paddedData = buffer.ToArray(); + + // Retrieve the padding size from the last byte + int paddingSize = paddedData[^1]; + + // Calculate the length of the original data + int originalDataLength = paddedData.Length - paddingSize - 1; + + // Ensure the padding size is valid + if (paddingSize < 0 || paddingSize >= paddedData.Length || originalDataLength < 0) + return null; + + // Create a new array for the original data + var descriptorData = new byte[originalDataLength]; + Array.Copy(paddedData, descriptorData, originalDataLength); + + descriptorData = AESGCMUtility.DecryptData(descriptorData, _secret); // Convert byte array to string and trim ending zeros - var json = Encoding.UTF8.GetString(byteArray).TrimEnd('\0'); + var json = Encoding.UTF8.GetString(descriptorData); try { var descriptor = JsonSerializer.Deserialize(json); @@ -395,9 +457,9 @@ public class Application { } catch (JsonException ex) { _logger.LogInformation($"Failed to parse descriptor JSON: {ex.Message}"); + return null; } - handler.Prepare(TapeDeviceHandler.TAPE_UNLOCK); Thread.Sleep(2000); @@ -407,7 +469,7 @@ public class Application { return null; } - public void RestoreDirectory(BackupDescriptor descriptor, WorkingFolder workingFolder) { + private void RestoreDirectory(BackupDescriptor descriptor, WorkingFolder workingFolder) { PathAccessWrapper(workingFolder, (restoreDirectoryPath) => { _logger.LogInformation("Restoring files to directory: " + restoreDirectoryPath); @@ -448,20 +510,11 @@ public class Application { } } - // check md5 checksum of restored file with the one in descriptor - using (var md5 = System.Security.Cryptography.MD5.Create()) { - using (var fileStreamRead = new FileStream(filePath, FileMode.Open, FileAccess.Read)) { - var fileHash = md5.ComputeHash(fileStreamRead); - var fileHashString = BitConverter.ToString(fileHash).Replace("-", "").ToLower(); - - if (fileHashString != file.FileHash) { - _logger.LogInformation($"Checksum mismatch for file: {filePath}"); - } - else { - _logger.LogInformation($"Restored file: {filePath}"); - } - } - } + // check checksum of restored file with the one in descriptor + if (ChecksumUtility.VerifyCRC32ChecksumFromFileInChunks(filePath, file.FileHash, (int)descriptor.BlockSize)) + _logger.LogInformation($"Restored file: {filePath}"); + else + _logger.LogInformation($"Checksum mismatch for file: {filePath}"); } handler.SetPosition(TapeDeviceHandler.TAPE_REWIND); @@ -469,7 +522,7 @@ public class Application { }); } - public int CheckMediaSize(string ltoGen) { + private int CheckMediaSize(string ltoGen) { var descriptor = JsonSerializer.Deserialize(File.ReadAllText(_descriptorFilePath)); if (descriptor == null) { _logger.LogInformation("Failed to read descriptor."); @@ -498,7 +551,7 @@ public class Application { return 0; } - public void Backup() { + private void Backup() { while (true) { _logger.LogInformation("\nSelect a backup to perform:"); for (int i = 0; i < _configuration.Backups.Count; i++) { @@ -539,7 +592,7 @@ public class Application { } } - public void Restore() { + private void Restore() { while (true) { _logger.LogInformation("\nSelect a backup to restore:"); for (int i = 0; i < _configuration.Backups.Count; i++) { @@ -582,4 +635,3 @@ public class Application { } } } - diff --git a/src/MaksIT.LTO.Backup/MaksIT.LTO.Backup.csproj b/src/MaksIT.LTO.Backup/MaksIT.LTO.Backup.csproj index a76cc2f..a722edc 100644 --- a/src/MaksIT.LTO.Backup/MaksIT.LTO.Backup.csproj +++ b/src/MaksIT.LTO.Backup/MaksIT.LTO.Backup.csproj @@ -5,6 +5,7 @@ net8.0 enable enable + 0.0.2 @@ -23,6 +24,9 @@ PreserveNewest + + PreserveNewest + diff --git a/src/MaksIT.LTO.Core/MaksIT.LTO.Core.csproj b/src/MaksIT.LTO.Core/MaksIT.LTO.Core.csproj index e60b572..a01e4fb 100644 --- a/src/MaksIT.LTO.Core/MaksIT.LTO.Core.csproj +++ b/src/MaksIT.LTO.Core/MaksIT.LTO.Core.csproj @@ -5,6 +5,7 @@ enable enable NTDDI_VERSION_05010000;NTDDI_WINXP_05010000 + 0.0.2 diff --git a/src/MaksIT.LTO.Core/Utilities/AESGCMUtility.cs b/src/MaksIT.LTO.Core/Utilities/AESGCMUtility.cs new file mode 100644 index 0000000..c85c9cd --- /dev/null +++ b/src/MaksIT.LTO.Core/Utilities/AESGCMUtility.cs @@ -0,0 +1,57 @@ +using System.Security.Cryptography; + + +namespace MaksIT.LTO.Core.Utilities; + +public static class AESGCMUtility { + private const int IvLength = 12; // 12 bytes for AES-GCM IV + private const int TagLength = 16; // 16 bytes for AES-GCM Tag + + public static byte[] EncryptData(byte[] data, string base64Key) { + var key = Convert.FromBase64String(base64Key); + using (AesGcm aesGcm = new AesGcm(key, AesGcm.TagByteSizes.MaxSize)) { + var iv = new byte[IvLength]; + RandomNumberGenerator.Fill(iv); + + var cipherText = new byte[data.Length]; + var tag = new byte[TagLength]; + + aesGcm.Encrypt(iv, data, cipherText, tag); + + // Concatenate cipherText, tag, and iv + var result = new byte[cipherText.Length + tag.Length + iv.Length]; + Buffer.BlockCopy(cipherText, 0, result, 0, cipherText.Length); + Buffer.BlockCopy(tag, 0, result, cipherText.Length, tag.Length); + Buffer.BlockCopy(iv, 0, result, cipherText.Length + tag.Length, iv.Length); + + return result; + } + } + + public static byte[] DecryptData(byte[] data, string base64Key) { + var key = Convert.FromBase64String(base64Key); + + // Extract cipherText, tag, and iv + var cipherTextLength = data.Length - IvLength - TagLength; + + var cipherText = new byte[cipherTextLength]; + var tag = new byte[TagLength]; + var iv = new byte[IvLength]; + + Buffer.BlockCopy(data, 0, cipherText, 0, cipherTextLength); + Buffer.BlockCopy(data, cipherTextLength, tag, 0, TagLength); + Buffer.BlockCopy(data, cipherTextLength + TagLength, iv, 0, IvLength); + + using (AesGcm aesGcm = new AesGcm(key, AesGcm.TagByteSizes.MaxSize)) { + var decryptedData = new byte[cipherText.Length]; + aesGcm.Decrypt(iv, cipherText, tag, decryptedData); + return decryptedData; + } + } + + public static string GenerateKeyBase64() { + var key = new byte[32]; // 256-bit key for AES-256 + RandomNumberGenerator.Fill(key); + return Convert.ToBase64String(key); + } +} diff --git a/src/MaksIT.LTO.Core/Utilities/ChecksumUtility.cs b/src/MaksIT.LTO.Core/Utilities/ChecksumUtility.cs new file mode 100644 index 0000000..861f30c --- /dev/null +++ b/src/MaksIT.LTO.Core/Utilities/ChecksumUtility.cs @@ -0,0 +1,130 @@ +using System.IO; +using System.Security.Cryptography; + +namespace MaksIT.LTO.Core.Helpers; + +public static class ChecksumUtility { + public static string CalculateCRC32Checksum(byte[] data) { + using var crc32 = new Crc32(); + byte[] hashBytes = crc32.ComputeHash(data); + return BitConverter.ToString(hashBytes).Replace("-", "").ToLower(); + } + + public static string CalculateCRC32ChecksumFromFile(string filePath) { + using var crc32 = new Crc32(); + using var stream = File.OpenRead(filePath); + byte[] hashBytes = crc32.ComputeHash(stream); + return BitConverter.ToString(hashBytes).Replace("-", "").ToLower(); + } + + public static string CalculateCRC32ChecksumFromFileInChunks(string filePath, int chunkSize = 8192) { + using var crc32 = new Crc32(); + using var stream = File.OpenRead(filePath); + var buffer = new byte[chunkSize]; + int bytesRead; + while ((bytesRead = stream.Read(buffer, 0, buffer.Length)) > 0) { + crc32.TransformBlock(buffer, 0, bytesRead, null, 0); + } + crc32.TransformFinalBlock(buffer, 0, 0); + byte[] hashBytes = crc32.Hash; + return BitConverter.ToString(hashBytes).Replace("-", "").ToLower(); + } + + public static bool VerifyCRC32Checksum(byte[] data, string expectedChecksum) { + string calculatedChecksum = CalculateCRC32Checksum(data); + return string.Equals(calculatedChecksum, expectedChecksum, StringComparison.OrdinalIgnoreCase); + } + + public static bool VerifyCRC32ChecksumFromFile(string filePath, string expectedChecksum) { + string calculatedChecksum = CalculateCRC32ChecksumFromFile(filePath); + return string.Equals(calculatedChecksum, expectedChecksum, StringComparison.OrdinalIgnoreCase); + } + + public static bool VerifyCRC32ChecksumFromFileInChunks(string filePath, string expectedChecksum, int chunkSize = 8192) { + string calculatedChecksum = CalculateCRC32ChecksumFromFileInChunks(filePath, chunkSize); + return string.Equals(calculatedChecksum, expectedChecksum, StringComparison.OrdinalIgnoreCase); + } +} + +public class Crc32 : HashAlgorithm { + public const uint DefaultPolynomial = 0xedb88320; + public const uint DefaultSeed = 0xffffffff; + + private static uint[]? defaultTable; + + private readonly uint seed; + private readonly uint[] table; + private uint hash; + + public Crc32() + : this(DefaultPolynomial, DefaultSeed) { + } + + public Crc32(uint polynomial, uint seed) { + table = InitializeTable(polynomial); + this.seed = hash = seed; + } + + public override void Initialize() { + hash = seed; + } + + protected override void HashCore(byte[] buffer, int start, int length) { + hash = CalculateHash(table, hash, buffer, start, length); + } + + protected override byte[] HashFinal() { + var hashBuffer = UInt32ToBigEndianBytes(~hash); + HashValue = hashBuffer; + return hashBuffer; + } + + public override int HashSize => 32; + + public static uint Compute(byte[] buffer) { + return Compute(DefaultPolynomial, DefaultSeed, buffer); + } + + public static uint Compute(uint seed, byte[] buffer) { + return Compute(DefaultPolynomial, seed, buffer); + } + + public static uint Compute(uint polynomial, uint seed, byte[] buffer) { + return ~CalculateHash(InitializeTable(polynomial), seed, buffer, 0, buffer.Length); + } + + private static uint[] InitializeTable(uint polynomial) { + if (polynomial == DefaultPolynomial && defaultTable != null) + return defaultTable; + + var createTable = new uint[256]; + for (var i = 0; i < 256; i++) { + var entry = (uint)i; + for (var j = 0; j < 8; j++) + if ((entry & 1) == 1) + entry = (entry >> 1) ^ polynomial; + else + entry >>= 1; + createTable[i] = entry; + } + + if (polynomial == DefaultPolynomial) + defaultTable = createTable; + + return createTable; + } + + private static uint CalculateHash(uint[] table, uint seed, byte[] buffer, int start, int size) { + var crc = seed; + for (var i = start; i < size - start; i++) + crc = (crc >> 8) ^ table[buffer[i] ^ crc & 0xff]; + return crc; + } + + private static byte[] UInt32ToBigEndianBytes(uint x) => [ + (byte)((x >> 24) & 0xff), + (byte)((x >> 16) & 0xff), + (byte)((x >> 8) & 0xff), + (byte)(x & 0xff) + ]; +}