Merge pull request 'feature/multi-threading' (#2) from feature/multi-threading into main

Reviewed-on: #2

README.md

@@ -39,19 +39,8 @@ Go to the publish folder
 cd src/Chksum/bin/Release/net7.0/linux-x64/publish
 ```
 
-Copy the libe_sqlite3.so to your /usr/local/lib or /usr/lib
-```bash
-cp libe_sqlite3.so /usr/local/lib
-```
-
 Run executable
 
 ```bash
-LD_LIBRARY_PATH=/usr/local/lib ./Chksum
+./Chksum
 ```
-
-Info
-
-LD_LIBRARY_PATH=/usr/local/lib is needed to tell the executable where the library is located
-
-Alternately you can put the libe_sqlite3.so into the same folder as the executable

src/Chksum/Program.cs

@@ -8,7 +8,7 @@ public class Program {
             Console.WriteLine("Please specify an option.");
             PrintAvailableOptions();
             return;
-        } else if (args.Length > 1) {
+        } else if (args.Length > 1 && args[0] != "compareDatabases") {
             Console.WriteLine("Too many options.");
             return;
         }
@@ -30,15 +30,18 @@ public class Program {
                 Console.ForegroundColor = ConsoleColor.Green;
                 Console.WriteLine("Checksum process finished");
                 break;
-            case "compareChecksums":
-                Console.WriteLine("Comparing all md5 checksum files. If there is none, creating one.");
+            case "compareDatabases":
                 Console.ResetColor();
 
-                utils.compareChecksums();
+                utils.compareDatabases(args[1]);
                 break;
             case "createDB":
                 utils.initializeDB();
                 break;
+            case "checkIfFileWasDeleted":
+                Console.ResetColor();
+                utils.checkIfFileWasDeleted();
+                break;
             case "help":
                 PrintAvailableOptions();
                 break;
@@ -57,6 +60,7 @@ public class Program {
             "checksum",
             "compareChecksums",
             "createDB",
+            "checkIfFileWasDeleted",
             "help"
         };
 

src/Chksum/chksum.cs

@@ -1,11 +1,20 @@
+using System.Collections.Concurrent;
 using System.Reflection;
+using System.Security.Cryptography;
 using Microsoft.Data.Sqlite;
 namespace Chksum.Utils;
 public class ChksumUtils {
 
-    private int getFileCount() {
-        int fileCount = Directory.GetFiles(Directory.GetCurrentDirectory()).Length; // Get file count in current directory
-        return fileCount;
+    private int getTotalFileCount() {
+        int totalFileCount = Directory.GetFiles(Directory.GetCurrentDirectory(), "*", SearchOption.AllDirectories).Length;
+        return totalFileCount - 3; // Remove the program, datbase and library from the totalFileCount
+    }
+
+    private string[] indexFiles() {
+        string[] indexedFiles = Directory.GetFiles(Directory.GetCurrentDirectory(), "*", SearchOption.AllDirectories);
+        string[] filesToExclude = { "Chksum", "chksum.db", "libe_sqlite3.so" };
+        indexedFiles = indexedFiles.Where(file => !filesToExclude.Contains(Path.GetFileName(file))).ToArray();
+        return indexedFiles;
     }
 
     public string DatabaseRoot { get; set; } = string.Empty;
@@ -62,54 +71,61 @@ public class ChksumUtils {
         }
     }
 
-    private string CalculateMD5(string filename) {
-        using (var md5 = System.Security.Cryptography.MD5.Create()) {
-            using (var stream = File.OpenRead(filename)) {
-                var hash = md5.ComputeHash(stream);
-                return BitConverter.ToString(hash).Replace("-", "").ToLowerInvariant();
-            }
-        }
-    }
+    private Dictionary<string, string> CalculateChecksums(string[] filenames) {
+        ConcurrentDictionary<string, string> checksums = new ConcurrentDictionary<string, string>();
 
-    public void doTheThing() {
-        foreach (var directory in Directory.GetDirectories(Directory.GetCurrentDirectory())) 
-        using (var connection = new SqliteConnection("Data Source=" + DatabaseRoot + "chksum.db;Mode=ReadWrite")) {
-            Directory.SetCurrentDirectory(directory); // Set new root
-            if (getFileCount() >= 1) {
-                DirectoryInfo dir = new DirectoryInfo(Directory.GetCurrentDirectory());
-                FileInfo[] files = dir.GetFiles();
-                foreach (FileInfo file in files) {
-                    string fileName = file.Name;
-                    string absolutePathToFile = Path.GetFullPath(fileName);
-                    string pathToFile = Path.GetRelativePath(DatabaseRoot, absolutePathToFile);
-                    string fileHash = CalculateMD5(fileName);
+        Parallel.ForEach(filenames, (filename, state) => {
+            using (var md5 = MD5.Create()) {
+                using (var stream = File.OpenRead(filename)) {
+                    var hash = md5.ComputeHash(stream);
+                    var checksum = BitConverter.ToString(hash).Replace("-", "").ToLowerInvariant();
 
-                    if (checkIfFileMovedAndUpdatePathToFile(fileHash, fileName, pathToFile) == false && checkIfFileAlreadyExists(fileHash, fileName) == false) {
-                        connection.Open();
-
-                        var command = connection.CreateCommand();
-                        command.CommandText =
-                        @"
-                            INSERT INTO file (filehash, filename, pathtofile)
-                            VALUES ($filehash, $filename, $pathtofile)
-                        ";
-                        command.Parameters.AddWithValue("$filehash", fileHash);
-                        command.Parameters.AddWithValue("$filename", fileName);
-                        command.Parameters.AddWithValue("$pathtofile", pathToFile);
-                        command.ExecuteNonQuery();
+                    lock (checksums) {
+                        checksums.TryAdd(filename, checksum);
                     }
                 }
             }
-            doTheThing();
+        });
+
+        return new Dictionary<string, string>(checksums);
+    }
+
+    public void doTheThing() {
+        using (var connection = new SqliteConnection("Data Source=" + DatabaseRoot + "chksum.db;Mode=ReadWrite")) {
+            if (getTotalFileCount() < 1) {
+                return;
+            }
+            connection.Open();
+            Dictionary<string, string> fileHashes = CalculateChecksums(indexFiles());
+            
+            foreach (var file in fileHashes) {
+                string absolutePathToFile = file.Key;
+                string fileName = Path.GetFileName(absolutePathToFile);
+                string pathToFile = Path.GetRelativePath(DatabaseRoot, absolutePathToFile);
+                string fileHash = file.Value;
+                
+                if (checkIfFileMovedAndUpdatePathToFile(fileHash, fileName, pathToFile) == false && checkIfFileAlreadyExistsInDatabase(fileHash, fileName) == false) {
+                    var command = connection.CreateCommand();
+                    command.CommandText =
+                    @"
+                        INSERT INTO file (filehash, filename, pathtofile)
+                        VALUES ($filehash, $filename, $pathtofile)
+                    ";
+                    command.Parameters.AddWithValue("$filehash", fileHash);
+                    command.Parameters.AddWithValue("$filename", fileName);
+                    command.Parameters.AddWithValue("$pathtofile", pathToFile);
+                    command.ExecuteNonQuery();
+                }
+            }
         }
     }
 
-    private bool checkIfFileAlreadyExists(string fileHash, string pathToFile) {
+    private bool checkIfFileAlreadyExistsInDatabase(string fileHash, string pathToFile) {
         string filehash = string.Empty;
         string pathtofile = string.Empty;
         bool doesExist = false;
 
-        using (var connection = new SqliteConnection("Data Source=" + DatabaseRoot + "chksum.db;Mode=ReadWrite")) {
+        using (var connection = new SqliteConnection("Data Source=" + DatabaseRoot + "chksum.db;Mode=ReadOnly")) {
             connection.Open();
 
             var command = connection.CreateCommand();
@@ -128,9 +144,6 @@ public class ChksumUtils {
         }
 
         if (fileHash == filehash) {
-            Console.WriteLine("Duplicate files found:");
-            Console.WriteLine($"\toriginal\t{pathToFile}");
-            Console.WriteLine($"\tduplicate\t{pathtofile}\n");
             doesExist = true;
         }
         return doesExist;
@@ -168,47 +181,100 @@ public class ChksumUtils {
                 command2.Parameters.AddWithValue("$filehash", fileHash);
                 command2.ExecuteNonQuery();
 
-                Console.WriteLine("File moved:");
+                Console.WriteLine("File moved or is a duplicate:");
                 Console.WriteLine($"\tfrom\t{pathToFile}");
                 Console.WriteLine($"\tto  \t{pathtofile}\n");
                 wasMoved = true;
             }
-            return wasMoved;
+        }
+        return wasMoved;
+    }
+
+    public void checkIfFileWasDeleted() {
+        string pathToFile = string.Empty;
+
+        using (var connection = new SqliteConnection("Data Source=" + DatabaseRoot + "chksum.db;Mode=ReadWrite")) {
+            connection.Open();
+
+            var selectCommand = connection.CreateCommand();
+            selectCommand.CommandText =
+            @"
+                Select pathtofile FROM file
+            ";
+
+            using (var reader = selectCommand.ExecuteReader()) {
+                while (reader.Read()) {
+                    pathToFile = reader.GetString(0);
+                    
+                    if (File.Exists(pathToFile)) {
+                        continue;
+                    }
+                    var deleteCommand = connection.CreateCommand();
+                    deleteCommand.CommandText =
+                    @"
+                        DELETE FROM file
+                        WHERE pathtofile = $pathtofile
+                    ";
+                    deleteCommand.Parameters.AddWithValue("$pathtofile", pathToFile);
+                    deleteCommand.ExecuteNonQuery();
+
+                    Console.WriteLine("File deleted:");
+                    Console.WriteLine($"\t{pathToFile}\n");
+                }
+            }
         }
     }
 
-    public void compareChecksums() {
-        foreach (var directory in Directory.GetDirectories(Directory.GetCurrentDirectory())) {
-            Directory.SetCurrentDirectory(directory); // Set new root
-            if (getFileCount() >= 1) {
-                DirectoryInfo dir = new DirectoryInfo(Directory.GetCurrentDirectory());
-                FileInfo[] files = dir.GetFiles();
-                // files.ToList().ForEach(i => Console.WriteLine(i.ToString())); // Print all files in files array
-                foreach (FileInfo file in files) {
-                    string fileName = file.Name;
-                    string fileNameWithoutExtension = Path.GetFileNameWithoutExtension(fileName);
-                    string checksumFile = Directory.GetCurrentDirectory() + "/" + fileNameWithoutExtension + ".md5";
-                    string fileMd5Checksum = fileNameWithoutExtension + ".md5";
-                    if (File.Exists(fileMd5Checksum)) {
-                        string newFileChecksum = CalculateMD5(fileName) + "  " + fileName;
-                        string existingFileChecksum = File.ReadAllText(fileMd5Checksum);
-                        string newFileName = newFileChecksum.Substring(34);
-                        string existingFileName = existingFileChecksum.Substring(34);
-                        if (newFileChecksum.Equals(existingFileChecksum)) {
-                            Console.WriteLine(newFileName + " and " + existingFileName + " are the same.");
-                        } else {
-                            Console.WriteLine(newFileName + " and " + existingFileName + " are not the same.");
-                            Console.WriteLine("The checksum of " + newFileName + " is " + newFileChecksum);
-                            Console.WriteLine("The checksum of the already exting file " + existingFileName + " is " + existingFileChecksum);
-                            // TODO Tell the user to check which file is the correct one
-                        }
-                    } else {
-                        File.AppendAllText(checksumFile, CalculateMD5(fileName) + "  " + fileName);
-                        Console.WriteLine("Calculated checksum for: " + checksumFile);
+    private List<string> getFilehashesFromDatabase(string connectionString) {
+        List<string> filehashesFromDatabase = new List<string>();
+
+        using (var connection = new SqliteConnection(connectionString)) {
+            string filehash = string.Empty;
+            
+            connection.Open();
+
+            var selectCommand = connection.CreateCommand();
+            selectCommand.CommandText =
+            @"
+                Select filehash FROM file
+            ";
+
+            using (var reader = selectCommand.ExecuteReader()) {
+                while (reader.Read()) {
+                    filehash = reader.GetString(0);
+                    filehashesFromDatabase.Add(filehash);
+                }
+            }
+        }
+
+        return filehashesFromDatabase;
+    }
+
+    public void compareDatabases(string filePathToOtherDatabase) {
+        List<string> filesThatDoNotExistsInTheRemote = getFilehashesFromDatabase("Data Source=" + DatabaseRoot + "chksum.db;Mode=ReadOnly").Except(getFilehashesFromDatabase("Data Source=" + filePathToOtherDatabase + ";Mode=ReadOnly")).ToList();
+
+        foreach (string file in filesThatDoNotExistsInTheRemote) {
+            using (var connection = new SqliteConnection("Data Source=" + DatabaseRoot + "chksum.db;Mode=ReadOnly")) {
+                string filename = string.Empty;
+                
+                connection.Open();
+
+                var selectCommand = connection.CreateCommand();
+                selectCommand.CommandText =
+                @"
+                    Select filename FROM file WHERE filehash = $filehash
+                ";
+                selectCommand.Parameters.AddWithValue("$filehash", file);
+
+                using (var reader = selectCommand.ExecuteReader()) {
+                    while (reader.Read()) {
+                        filename = reader.GetString(0);
+                        
+                        Console.WriteLine("File not found in remote:");
+                        Console.WriteLine($"\t{filename}\n");
                     }
                 }
             }
-            compareChecksums();
         }
     }