Add new hashing algorithms

Add XxHash
Code cleanup

README.md

@@ -43,4 +43,19 @@ Run executable
 
 ```bash
 ./Chksum
-```
+```
+
+## Enabling verbose output for troubleshooting
+
+1. Open the file called chksum.cs with your editor of choice.
+2. At the top there will be the logger configuration which you can change. Should look like this.
+```cs
+private ILogger logger = new LoggerConfiguration()
+            .MinimumLevel.Debug()
+            .WriteTo.Console(restrictedToMinimumLevel: LogEventLevel.Error)
+            .WriteTo.File("chksum.log")
+            .CreateLogger();
+```
+3. Change the minimum level of the logger to Verbose.
+4. Compile the program
+5. Profit. Now you will be able to see how what the program is doing in detail.

src/Chksum/Chksum.csproj

@@ -18,6 +18,11 @@
 
   <ItemGroup>
     <PackageReference Include="Microsoft.Data.Sqlite" Version="7.0.8" />
+    <PackageReference Include="MurmurHash.Net" Version="0.0.2" />
+    <PackageReference Include="Serilog" Version="3.0.1" />
+    <PackageReference Include="Serilog.Sinks.Console" Version="4.1.0" />
+    <PackageReference Include="Serilog.Sinks.File" Version="5.0.0" />
+    <PackageReference Include="Standart.Hash.xxHash" Version="4.0.5" />
   </ItemGroup>
 
 </Project>

src/Chksum/Program.cs

@@ -58,6 +58,7 @@ public class Program {
     static void PrintAvailableOptions() {
         String[] options = {
             "checksum",
+            "compareDatabases",
             "compareChecksums",
             "createDB",
             "checkIfFileWasDeleted",

src/Chksum/chksum.cs

@@ -1,16 +1,38 @@
+using System.Collections.Concurrent;
 using System.Reflection;
+using System.Security.Cryptography;
 using Microsoft.Data.Sqlite;
+using Serilog;
+using Serilog.Events;
+using MurmurHash.Net;
+using Standart.Hash.xxHash;
+
 namespace Chksum.Utils;
 public class ChksumUtils {
+    private ILogger logger = new LoggerConfiguration()
+            .MinimumLevel.Debug()
+            .WriteTo.Console(restrictedToMinimumLevel: LogEventLevel.Error)
+            .WriteTo.File("chksum.log")
+            .CreateLogger();
 
-    private int getFileCount() {
+    private int getTotalFileCount() {
-        int fileCount = Directory.GetFiles(Directory.GetCurrentDirectory()).Length; // Get file count in current directory
+        int totalFileCount = Directory.GetFiles(Directory.GetCurrentDirectory(), "*", SearchOption.AllDirectories).Length;
-        return fileCount;
+        logger.Debug("Total file count is {totalFileCount}", totalFileCount);
+        return totalFileCount - 4; // Remove the program, datbase, log and library from the totalFileCount
+    }
+
+    private string[] indexFiles() {
+        string[] indexedFiles = Directory.GetFiles(Directory.GetCurrentDirectory(), "*", SearchOption.AllDirectories);
+        string[] filesToExclude = { "Chksum", "chksum.db", "libe_sqlite3.so" };
+        indexedFiles = indexedFiles.Where(file => !filesToExclude.Contains(Path.GetFileName(file))).ToArray();
+        logger.Information("All files were indexed");
+        return indexedFiles;
     }
 
     public string DatabaseRoot { get; set; } = string.Empty;
     public void getBaseDir() {
         DatabaseRoot = AppDomain.CurrentDomain.BaseDirectory;
+        logger.Debug("DatabaseRoot is {DatabaseRoot}", DatabaseRoot);
     }
 
     public string libraryPath { get; set; } = string.Empty;
@@ -22,7 +44,9 @@ public class ChksumUtils {
                 byte[] buffer = new byte[resourceStream.Length];
                 resourceStream.Read(buffer, 0, buffer.Length);
                 File.WriteAllBytes(libraryPath, buffer);
+                logger.Debug("libe_sqlite3.so was successfully created");
             } else {
+                logger.Error("libe_sqlite3.so could not be loaded");
                 throw new Exception(libraryPath + " could not be loaded");
             }
         }
@@ -30,6 +54,7 @@ public class ChksumUtils {
 
     public void initializeDB() {
         if (File.Exists("chksum.db")) {
+            logger.Information("A database already exits");
             return;
         }
 
@@ -48,6 +73,7 @@ public class ChksumUtils {
                 );
             ";
             command.ExecuteNonQuery();
+            logger.Information("Database was successfully created");
         }
     }
 
@@ -59,52 +85,148 @@ public class ChksumUtils {
                 vacuum;
             ";
             command.ExecuteNonQuery();
+            logger.Debug("Database was successfully vacuumed");
         }
     }
 
-    private string CalculateMD5(string filename) {
+    private Dictionary<string, string> CalculateChecksums(string[] filenames) {
-        using (var md5 = System.Security.Cryptography.MD5.Create()) {
+        ConcurrentDictionary<string, string> checksums = new ConcurrentDictionary<string, string>();
-            using (var stream = File.OpenRead(filename)) {
-                var hash = md5.ComputeHash(stream);
-                return BitConverter.ToString(hash).Replace("-", "").ToLowerInvariant();
-            }
-        }
-    }
 
-    public void doTheThing() {
+        Parallel.ForEach(filenames, (filename, state) => {
-        foreach (var directory in Directory.GetDirectories(Directory.GetCurrentDirectory())) 
+            using (var md5 = MD5.Create()) {
-        using (var connection = new SqliteConnection("Data Source=" + DatabaseRoot + "chksum.db;Mode=ReadWrite")) {
+                using (var stream = File.OpenRead(filename)) {
-            Directory.SetCurrentDirectory(directory); // Set new root
+                    var hash = md5.ComputeHash(stream);
-            if (getFileCount() >= 1) {
+                    var checksum = BitConverter.ToString(hash).Replace("-", "").ToLowerInvariant();
-                DirectoryInfo dir = new DirectoryInfo(Directory.GetCurrentDirectory());
-                FileInfo[] files = dir.GetFiles();
-                foreach (FileInfo file in files) {
-                    string fileName = file.Name;
-                    string absolutePathToFile = Path.GetFullPath(fileName);
-                    string pathToFile = Path.GetRelativePath(DatabaseRoot, absolutePathToFile);
-                    string fileHash = CalculateMD5(fileName);
 
-                    if (checkIfFileMovedAndUpdatePathToFile(fileHash, fileName, pathToFile) == false && checkIfFileAlreadyExistsInDatabase(fileHash, fileName) == false) {
+                    lock (checksums) {
-                        connection.Open();
+                        checksums.TryAdd(filename, checksum);
-
-                        var command = connection.CreateCommand();
-                        command.CommandText =
-                        @"
-                            INSERT INTO file (filehash, filename, pathtofile)
-                            VALUES ($filehash, $filename, $pathtofile)
-                        ";
-                        command.Parameters.AddWithValue("$filehash", fileHash);
-                        command.Parameters.AddWithValue("$filename", fileName);
-                        command.Parameters.AddWithValue("$pathtofile", pathToFile);
-                        command.ExecuteNonQuery();
                     }
                 }
             }
-            doTheThing();
+        });
+
+        logger.Debug("All files were checksummed");
+        return new Dictionary<string, string>(checksums);
+    }
+
+    private Dictionary<string, uint> CalculateChecksumsWithMurmur(string[] filenames) {
+        ConcurrentDictionary<string, uint> checksums = new ConcurrentDictionary<string, uint>();
+
+        Parallel.ForEach(filenames, (filename, state) => {
+                using (var stream = File.OpenRead(filename)) {
+                    var hash = CalculateMurmurHash32(stream);
+                    lock (checksums) {
+                        checksums.TryAdd(filename, hash);
+                    }
+            }
+        });
+
+        logger.Debug("All files were checksummed");
+        return new Dictionary<string, uint>(checksums);
+    }
+
+    private uint CalculateMurmurHash32(Stream stream) {
+        const int bufferSize = 4096;
+        const uint seed = 123456U;
+        
+        var buffer = new byte[bufferSize];
+        uint hash = seed;
+
+        int bytesRead;
+        ReadOnlySpan<byte> span = buffer;
+
+        while ((bytesRead = stream.Read(buffer, 0, bufferSize)) > 0) {
+            hash = MurmurHash3.Hash32(bytes: span, seed: 123456U);
+        }
+        return hash;
+    }
+
+    private Dictionary<string, ulong> CalculateChecksumsWithXxHash3(string[] filenames) {
+        ConcurrentDictionary<string, ulong> checksums = new ConcurrentDictionary<string, ulong>();
+
+        Parallel.ForEach(filenames, (filename, state) => {
+            using (var stream = File.OpenRead(filename)) {
+                var hash = CalculateXxHash3(stream);
+                checksums.TryAdd(filename, hash);
+            }
+        });
+
+        return new Dictionary<string, ulong>(checksums);
+    }
+
+    private ulong CalculateXxHash3(Stream stream) {
+        const int bufferSize = 4096;
+        const ulong seed = 123456U;
+
+        var buffer = new byte[bufferSize];
+        ulong hash = seed;
+        
+        int bytesRead;
+
+        while ((bytesRead = stream.Read(buffer, 0, buffer.Length)) > 0) {
+            hash = xxHash3.ComputeHash(buffer, buffer.Length);
+        }
+
+        return hash;
+    }
+
+
+    public void doTheThing(string hashalgo, int bufferSize) {
+        using (var connection = new SqliteConnection("Data Source=" + DatabaseRoot + "chksum.db;Mode=ReadWrite")) {
+            if (getTotalFileCount() < 1) {
+                logger.Information("There were no files to checksum");
+                return;
+            }
+            connection.Open();
+
+            Dictionary<string, object> fileHashes;
+            Dictionary<string, ulong>  fileHashesXxHash3;
+            Dictionary<string, uint>  fileHashesMurmur;
+            Dictionary<string, string> fileHashesMD5;
+
+            switch (hashalgo) {
+                case "MD5":
+                    fileHashesMD5 = CalculateChecksums(indexFiles());
+                    fileHashes = fileHashesMD5.ToDictionary(kv => kv.Key, kv => (object)kv.Value);
+                    break;
+                case "Murmur":
+                    fileHashesMurmur = CalculateChecksumsWithMurmur(indexFiles());
+                    fileHashes = fileHashesMurmur.ToDictionary(kv => kv.Key, kv => (object)kv.Value);
+                    break;
+                case "XxHash":
+                    fileHashesXxHash3 = CalculateChecksumsWithXxHash3(indexFiles());
+                    fileHashes = fileHashesXxHash3.ToDictionary(kv => kv.Key, kv => (object)kv.Value);
+                    break;
+                default:
+                    logger.Error("No valid hash algorithm was selected");
+                    throw new Exception($"{hashalgo} is not a valid option. Valid options are MD5, Murmur and XxHash");
+            }
+            
+            foreach (var file in fileHashes) {
+                string absolutePathToFile = file.Key;
+                string fileName = Path.GetFileName(absolutePathToFile);
+                string pathToFile = Path.GetRelativePath(DatabaseRoot, absolutePathToFile);
+                var fileHash = file.Value;
+                
+                if (checkIfFileMovedAndUpdatePathToFile(fileHash, fileName, pathToFile) == false && checkIfFileAlreadyExistsInDatabase(fileHash, fileName) == false) {
+                    var command = connection.CreateCommand();
+                    command.CommandText =
+                    @"
+                        INSERT INTO file (filehash, filename, pathtofile)
+                        VALUES ($filehash, $filename, $pathtofile)
+                    ";
+                    command.Parameters.AddWithValue("$filehash", fileHash);
+                    command.Parameters.AddWithValue("$filename", fileName);
+                    command.Parameters.AddWithValue("$pathtofile", pathToFile);
+                    command.ExecuteNonQuery();
+                    logger.Verbose("{fileName} which is located at {pathToFile} relative to the database with the hash {fileHash} was successfully inserted into the database", fileName, pathToFile, fileHash);
+                }
+            }
+            logger.Information("All files were successfully written to the database");
         }
     }
 
-    private bool checkIfFileAlreadyExistsInDatabase(string fileHash, string pathToFile) {
+    private bool checkIfFileAlreadyExistsInDatabase(object fileHash, string pathToFile) {
         string filehash = string.Empty;
         string pathtofile = string.Empty;
         bool doesExist = false;
@@ -117,7 +239,7 @@ public class ChksumUtils {
             @"
                 SELECT filehash, pathtofile FROM file WHERE filehash = $filehash
             ";
-            command.Parameters.AddWithValue("$filehash", fileHash);
+            command.Parameters.AddWithValue("$filehash", fileHash.ToString());
 
             using (var reader = command.ExecuteReader()) {
                 while (reader.Read()) {
@@ -125,15 +247,17 @@ public class ChksumUtils {
                     pathtofile = reader.GetString(1);
                 }
             }
+            logger.Verbose("{pathToFile} with the hash {fileHash} was successfully loaded", pathToFile, fileHash.ToString());
         }
 
-        if (fileHash == filehash) {
+        if (fileHash.ToString() == filehash) {
+            logger.Verbose("File with filehash {filehash} already exists in the database", filehash);
             doesExist = true;
         }
         return doesExist;
     }
 
-    private bool checkIfFileMovedAndUpdatePathToFile(string fileHash, string fileName, string pathToFile) {
+    private bool checkIfFileMovedAndUpdatePathToFile(object fileHash, string fileName, string pathToFile) {
         string pathtofile = string.Empty;
         bool wasMoved = false;
 
@@ -145,7 +269,7 @@ public class ChksumUtils {
             @"
                 SELECT pathtofile FROM file WHERE filehash = $filehash
             ";
-            command.Parameters.AddWithValue("$filehash", fileHash);
+            command.Parameters.AddWithValue("$filehash", fileHash.ToString());
 
             using (var reader = command.ExecuteReader()) {
                 while (reader.Read()) {
@@ -162,16 +286,17 @@ public class ChksumUtils {
                     WHERE filehash = $filehash
                 ";
                 command2.Parameters.AddWithValue("$newpathtofile", pathToFile);
-                command2.Parameters.AddWithValue("$filehash", fileHash);
+                command2.Parameters.AddWithValue("$filehash", fileHash.ToString());
                 command2.ExecuteNonQuery();
 
-                Console.WriteLine("File moved:");
+                //Console.WriteLine("File moved or is a duplicate:");
-                Console.WriteLine($"\tfrom\t{pathToFile}");
+                //Console.WriteLine($"\tfrom\t{pathToFile}");
-                Console.WriteLine($"\tto  \t{pathtofile}\n");
+                //Console.WriteLine($"\tto  \t{pathtofile}\n");
                 wasMoved = true;
             }
-            return wasMoved;
+            logger.Verbose("{fileName} which is located at {pathToFile} relative to the database with the hash {fileHash} was successfully checked", fileName, pathToFile, fileHash.ToString());
         }
+        return wasMoved;
     }
 
     public void checkIfFileWasDeleted() {
@@ -190,27 +315,31 @@ public class ChksumUtils {
                 while (reader.Read()) {
                     pathToFile = reader.GetString(0);
                     
-                    if (!File.Exists(pathToFile)) {
+                    if (File.Exists(pathToFile)) {
-                        var deleteCommand = connection.CreateCommand();
+                        logger.Verbose("{pathToFile} exists", pathToFile);
-                        deleteCommand.CommandText =
+                        continue;
-                        @"
-                            DELETE FROM file
-                            WHERE pathtofile = $pathtofile
-                        ";
-                        deleteCommand.Parameters.AddWithValue("$pathtofile", pathToFile);
-                        deleteCommand.ExecuteNonQuery();
-
-                        Console.WriteLine("File deleted:");
-                        Console.WriteLine($"\t{pathToFile}\n");
                     }
+                    var deleteCommand = connection.CreateCommand();
+                    deleteCommand.CommandText =
+                    @"
+                        DELETE FROM file
+                        WHERE pathtofile = $pathtofile
+                    ";
+                    deleteCommand.Parameters.AddWithValue("$pathtofile", pathToFile);
+                    deleteCommand.ExecuteNonQuery();
+
+                    //Console.WriteLine("File deleted:");
+                    //Console.WriteLine($"\t{pathToFile}\n");
+                    logger.Information("File deleted: {pathToFile}", pathToFile);
                 }
             }
+            logger.Information("All deleted files were successfully removed from the database");
         }
     }
 
     private List<string> getFilehashesFromDatabase(string connectionString) {
         List<string> filehashesFromDatabase = new List<string>();
-
+        
         using (var connection = new SqliteConnection(connectionString)) {
             string filehash = string.Empty;
             
@@ -230,12 +359,17 @@ public class ChksumUtils {
             }
         }
 
+        logger.Debug("All filehashes were successfully retrived from the database");
         return filehashesFromDatabase;
     }
 
     public void compareDatabases(string filePathToOtherDatabase) {
+        if (!File.Exists(filePathToOtherDatabase)) {
+            logger.Error("No database could be found at {filePathToOtherDatabase}", filePathToOtherDatabase);
+            throw new Exception("No database could be found at " + filePathToOtherDatabase);
+        }
+
         List<string> filesThatDoNotExistsInTheRemote = getFilehashesFromDatabase("Data Source=" + DatabaseRoot + "chksum.db;Mode=ReadOnly").Except(getFilehashesFromDatabase("Data Source=" + filePathToOtherDatabase + ";Mode=ReadOnly")).ToList();
-        //List<string> filesThatDoNotExistsInTheOrigin = filehashesOfRemoteDatabase.Except(filehashesOfOriginDatabase).ToList();
 
         foreach (string file in filesThatDoNotExistsInTheRemote) {
             using (var connection = new SqliteConnection("Data Source=" + DatabaseRoot + "chksum.db;Mode=ReadOnly")) {
@@ -256,13 +390,16 @@ public class ChksumUtils {
                         
                         Console.WriteLine("File not found in remote:");
                         Console.WriteLine($"\t{filename}\n");
+                        logger.Information("{filename} could not be found in the remote database", filename);
                     }
                 }
             }
         }
+        logger.Information("Compared both databases successfully");
     }
 
     public void cleanup() {
         File.Delete(libraryPath);
+        logger.Debug("Successfully deleted libe_sqlite3.so");
     }
 }