diff --git a/src/Chksum/Chksum.csproj b/src/Chksum/Chksum.csproj index cfc5d7f..1c180c7 100644 --- a/src/Chksum/Chksum.csproj +++ b/src/Chksum/Chksum.csproj @@ -22,6 +22,7 @@ + diff --git a/src/Chksum/Program.cs b/src/Chksum/Program.cs index 43a118a..adcaf54 100644 --- a/src/Chksum/Program.cs +++ b/src/Chksum/Program.cs @@ -38,14 +38,15 @@ public class Program { Console.ForegroundColor = ConsoleColor.Green; Console.WriteLine("Checksum process finished"); break; + case "saveToSqlite": + Console.ResetColor(); + utils.saveToSqlite(); + break; case "compareDatabases": Console.ResetColor(); utils.compareDatabases(args[1]); break; - case "createDB": - utils.initializeDB(); - break; case "checkIfFileWasDeleted": Console.ResetColor(); utils.checkIfFileWasDeleted(); @@ -68,7 +69,7 @@ public class Program { "checksum - MD5, Murmur and XxHash", "compareDatabases", "compareChecksums", - "createDB", + "saveToSqlite", "checkIfFileWasDeleted", "help" }; diff --git a/src/Chksum/chksum.cs b/src/Chksum/chksum.cs index 5ce92c7..7212865 100644 --- a/src/Chksum/chksum.cs +++ b/src/Chksum/chksum.cs @@ -6,6 +6,7 @@ using Serilog; using Serilog.Events; using MurmurHash.Net; using Standart.Hash.xxHash; +using StackExchange.Redis; namespace Chksum.Utils; public class ChksumUtils { @@ -52,7 +53,7 @@ public class ChksumUtils { } } - public void initializeDB() { + private void initializeDB() { if (File.Exists("chksum.db")) { logger.Information("A database already exits"); return; @@ -73,6 +74,14 @@ public class ChksumUtils { ); "; command.ExecuteNonQuery(); + + var walCommand = connection.CreateCommand(); + walCommand.CommandText = + @" + PRAGMA journal_mode = 'wal' + "; + walCommand.ExecuteNonQuery(); + logger.Information("Database was successfully created"); } } @@ -105,7 +114,6 @@ public class ChksumUtils { } }); - logger.Debug("All files were checksummed"); return new Dictionary(checksums); } @@ -121,7 +129,6 @@ public class ChksumUtils { } }); - logger.Debug("All files were checksummed"); return new Dictionary(checksums); } @@ -172,61 +179,95 @@ public class ChksumUtils { public void doTheThing(string hashalgo, int bufferSize) { + + ConnectionMultiplexer redis = ConnectionMultiplexer.Connect("localhost"); + IDatabase db = redis.GetDatabase(); + + + if (getTotalFileCount() < 1) { + logger.Information("There were no files to checksum"); + return; + } + + Dictionary fileHashes; + Dictionary fileHashesXxHash3; + Dictionary fileHashesMurmur; + Dictionary fileHashesMD5; + + switch (hashalgo) { + case "MD5": + fileHashesMD5 = CalculateChecksums(indexFiles()); + fileHashes = fileHashesMD5.ToDictionary(kv => kv.Key, kv => (object)kv.Value); + break; + case "Murmur": + fileHashesMurmur = CalculateChecksumsWithMurmur(indexFiles(), bufferSize); + fileHashes = fileHashesMurmur.ToDictionary(kv => kv.Key, kv => (object)kv.Value); + break; + case "XxHash": + fileHashesXxHash3 = CalculateChecksumsWithXxHash3(indexFiles(), bufferSize); + fileHashes = fileHashesXxHash3.ToDictionary(kv => kv.Key, kv => (object)kv.Value); + break; + default: + logger.Error("No valid hash algorithm was selected"); + throw new Exception($"{hashalgo} is not a valid option. Valid options are MD5, Murmur and XxHash"); + } + + logger.Information("All files were checksummed"); + + HashEntry[] hashEntries = fileHashes.Select(kv => new HashEntry(kv.Key, kv.Value.ToString())).ToArray(); + string hashKey = "fileHashes"; + db.HashSet(hashKey, hashEntries); + logger.Information("Dictionary inserted into Redis."); + } + + public void saveToSqlite() { + + initializeDB(); + + ConnectionMultiplexer redis = ConnectionMultiplexer.Connect("localhost"); + IDatabase db = redis.GetDatabase(); + + HashEntry[] fileHashes = db.HashGetAll("fileHashes"); + logger.Information("Retrived all values from redis"); + using (var connection = new SqliteConnection("Data Source=" + DatabaseRoot + "chksum.db;Mode=ReadWrite")) { - if (getTotalFileCount() < 1) { - logger.Information("There were no files to checksum"); - return; - } - connection.Open(); - - Dictionary fileHashes; - Dictionary fileHashesXxHash3; - Dictionary fileHashesMurmur; - Dictionary fileHashesMD5; - - switch (hashalgo) { - case "MD5": - fileHashesMD5 = CalculateChecksums(indexFiles()); - fileHashes = fileHashesMD5.ToDictionary(kv => kv.Key, kv => (object)kv.Value); - break; - case "Murmur": - fileHashesMurmur = CalculateChecksumsWithMurmur(indexFiles(), bufferSize); - fileHashes = fileHashesMurmur.ToDictionary(kv => kv.Key, kv => (object)kv.Value); - break; - case "XxHash": - fileHashesXxHash3 = CalculateChecksumsWithXxHash3(indexFiles(), bufferSize); - fileHashes = fileHashesXxHash3.ToDictionary(kv => kv.Key, kv => (object)kv.Value); - break; - default: - logger.Error("No valid hash algorithm was selected"); - throw new Exception($"{hashalgo} is not a valid option. Valid options are MD5, Murmur and XxHash"); - } - foreach (var file in fileHashes) { - string absolutePathToFile = file.Key; - string fileName = Path.GetFileName(absolutePathToFile); - string pathToFile = Path.GetRelativePath(DatabaseRoot, absolutePathToFile); - var fileHash = file.Value; - if (checkIfFileMovedAndUpdatePathToFile(fileHash, fileName, pathToFile) == false && checkIfFileAlreadyExistsInDatabase(fileHash, fileName) == false) { - var command = connection.CreateCommand(); - command.CommandText = + var absolutePathToFile = file.Name.ToString(); + string fileName = Path.GetFileName(absolutePathToFile.ToString()); + string pathToFile = Path.GetRelativePath(DatabaseRoot, absolutePathToFile.ToString()); + var fileHash = file.Value.ToString(); + + if (!checkIfFileMovedAndUpdatePathToFile(fileHash, fileName, pathToFile) && !checkIfFileAlreadyExistsInDatabase(fileHash, fileName)) { + connection.Open(); + var InsertCommand = connection.CreateCommand(); + InsertCommand.CommandText = @" INSERT INTO file (filehash, filename, pathtofile) VALUES ($filehash, $filename, $pathtofile) "; - command.Parameters.AddWithValue("$filehash", fileHash); - command.Parameters.AddWithValue("$filename", fileName); - command.Parameters.AddWithValue("$pathtofile", pathToFile); - command.ExecuteNonQuery(); + InsertCommand.Parameters.AddWithValue("$filehash", fileHash); + InsertCommand.Parameters.AddWithValue("$filename", fileName); + InsertCommand.Parameters.AddWithValue("$pathtofile", pathToFile); + InsertCommand.ExecuteNonQuery(); logger.Verbose("{fileName} which is located at {pathToFile} relative to the database with the hash {fileHash} was successfully inserted into the database", fileName, pathToFile, fileHash); } } - logger.Information("All files were successfully written to the database"); } + logger.Information("All filehashes were successfully inserted into the database"); + + var keys = db.Execute("KEYS", "*"); + if (keys == null) { + logger.Error("No values found in redis"); + return; + } + foreach (var key in (RedisValue[])keys) { + db.KeyDelete((RedisKey)key.ToString()); + } + logger.Information("Redis was successfully cleared of any remaining data"); } - private bool checkIfFileAlreadyExistsInDatabase(object fileHash, string pathToFile) { + private bool checkIfFileAlreadyExistsInDatabase(string fileHash, string pathToFile) { string filehash = string.Empty; string pathtofile = string.Empty; bool doesExist = false; @@ -239,7 +280,7 @@ public class ChksumUtils { @" SELECT filehash, pathtofile FROM file WHERE filehash = $filehash "; - command.Parameters.AddWithValue("$filehash", fileHash.ToString()); + command.Parameters.AddWithValue("$filehash", fileHash); using (var reader = command.ExecuteReader()) { while (reader.Read()) { @@ -247,17 +288,17 @@ public class ChksumUtils { pathtofile = reader.GetString(1); } } - logger.Verbose("{pathToFile} with the hash {fileHash} was successfully loaded", pathToFile, fileHash.ToString()); + logger.Verbose("{pathToFile} with the hash {fileHash} was successfully loaded", pathToFile, fileHash); } - if (fileHash.ToString() == filehash) { + if (fileHash == filehash) { logger.Verbose("File with filehash {filehash} already exists in the database", filehash); doesExist = true; } return doesExist; } - private bool checkIfFileMovedAndUpdatePathToFile(object fileHash, string fileName, string pathToFile) { + private bool checkIfFileMovedAndUpdatePathToFile(string fileHash, string fileName, string pathToFile) { string pathtofile = string.Empty; bool wasMoved = false; @@ -269,7 +310,7 @@ public class ChksumUtils { @" SELECT pathtofile FROM file WHERE filehash = $filehash "; - command.Parameters.AddWithValue("$filehash", fileHash.ToString()); + command.Parameters.AddWithValue("$filehash", fileHash); using (var reader = command.ExecuteReader()) { while (reader.Read()) { @@ -286,7 +327,7 @@ public class ChksumUtils { WHERE filehash = $filehash "; command2.Parameters.AddWithValue("$newpathtofile", pathToFile); - command2.Parameters.AddWithValue("$filehash", fileHash.ToString()); + command2.Parameters.AddWithValue("$filehash", fileHash); command2.ExecuteNonQuery(); //Console.WriteLine("File moved or is a duplicate:"); @@ -294,12 +335,15 @@ public class ChksumUtils { //Console.WriteLine($"\tto \t{pathtofile}\n"); wasMoved = true; } - logger.Verbose("{fileName} which is located at {pathToFile} relative to the database with the hash {fileHash} was successfully checked", fileName, pathToFile, fileHash.ToString()); + logger.Verbose("{fileName} which is located at {pathToFile} relative to the database with the hash {fileHash} was successfully checked", fileName, pathToFile, fileHash); } return wasMoved; } public void checkIfFileWasDeleted() { + + saveToSqlite(); + string pathToFile = string.Empty; using (var connection = new SqliteConnection("Data Source=" + DatabaseRoot + "chksum.db;Mode=ReadWrite")) { @@ -364,6 +408,9 @@ public class ChksumUtils { } public void compareDatabases(string filePathToOtherDatabase) { + + saveToSqlite(); + if (!File.Exists(filePathToOtherDatabase)) { logger.Error("No database could be found at {filePathToOtherDatabase}", filePathToOtherDatabase); throw new Exception("No database could be found at " + filePathToOtherDatabase);