Merge branch 'dev'

ObsidianTools.Test/TestPluginCleanupAssets.cs

@@ -0,0 +1,33 @@
+using System;
+using System.IO;
+using System.Linq;
+using NUnit.Framework;
+using ObsidianTools.plugins;
+
+namespace ObsidianTools.Test
+{
+    public class TestPluginCleanupAssets : BaseTest
+    {
+        [ Test ]
+        public void Test()
+        {
+            PrepareFile(Path.Join(VaultDirectory, "index.md"), "this is a test for ![[image askdl - 27.png]] image embedding");
+            PrepareFile(Path.Join(VaultDirectory, "image askdl - 27.png"), "a");
+            PrepareFile(Path.Join(VaultDirectory, "image2.png"), "b");
+            PrepareFile(Path.Join(VaultDirectory, "image 3 more.png"), "d");
+            PrepareFile(Path.Join(VaultDirectory, "style.css"), "d");
+            new PluginCleanupAssets().Execute(new[]
+                {
+                    VaultDirectory
+                }
+                , VaultDirectory);
+
+            String targetDir = Path.Join(VaultDirectory, "_UNUSED");
+            Assert.IsTrue(Directory.Exists(targetDir));
+            String[] files = Directory.GetFiles(targetDir);
+            Assert.AreEqual(2, files.Length);
+            Assert.IsTrue(files.Contains(Path.Join(targetDir, "image2.png")));
+            Assert.IsFalse(files.Contains(Path.Join(targetDir, "style.css")));
+        }
+    }
+}

ObsidianTools.Test/TestPluginCreateReferences.cs

@@ -15,13 +15,13 @@ public void Test()
             List<(String Name, String Before, String After)> files = new List<(String, String, String)>
             {
                 ("Index", "das hier ist der Index. ich habe Projekte und Management Aufgaben"
-                    , "das hier ist der [[Index]]. ich habe [[Projekt]]e und Management Aufgaben")
+                    , "das hier ist der Index. ich habe [[Projekt]]e und Management Aufgaben")
                 , ("Projekt", "ein Unterfangen, AnotherDocument welches vom Manager oder der Managerin geleitet wird"
                     , "ein Unterfangen, [[AnotherDocument]] welches vom [[Manager]] oder der [[Manager]]in geleitet wird")
                 , ("Manager", "chef, siehe Index, leitet Projekte, Projektmanager auch Projektmanagement, Seniormanager"
-                    , "chef, siehe [[Index]], leitet [[Projekt]]e, [[Projektmanager]] auch [[Projekt]]management, Senior[[Manager|manager]]")
-                , ("AnotherDocument", "chef, siehe Index,[[ leitet Projekte]], auch Projektmanagement, Seniormanager"
-                    , "chef, siehe [[Index]],[[ leitet Projekte]], auch [[Projekt]]management, Senior[[Manager|manager]]")
+                    , "chef, siehe [[Index]], leitet [[Projekt]]e, [[Projektmanager]] auch [[Projekt]]management, Seniormanager")
+                , ("AnotherDocument", "chef, siehe Index,[[ leitet Projekte]], auch Projektmanagement AnotherDocument, Seniormanager"
+                    , "chef, siehe [[Index]],[[ leitet Projekte]], auch [[Projekt]]management AnotherDocument, Senior[[Manager|manager]]")
                 , ("Projektmanager", "chef, siehe [[Index, leitet Projekte, auch Projektmanagement]], SENIORMANAGER"
                     , "chef, siehe [[Index, leitet Projekte, auch Projektmanagement]], SENIOR[[Manager|MANAGER]]")
             };

ObsidianTools.Test/TestPluginIdentifyHotspots.cs

@@ -0,0 +1,36 @@
+using System;
+using System.IO;
+using NUnit.Framework;
+using ObsidianTools.plugins;
+
+namespace ObsidianTools.Test
+{
+    public class TestPluginIdentifyHotspots : BaseTest
+    {
+        [ Test ]
+        public void Test()
+        {
+            PrepareFile(GetFilePathForName("Index"), "[[a1]] [[a2]] [[b]]");
+            PrepareFile(GetFilePathForName("a1"), "[[a1.1]] [[a1.2]] [[a1.3]], siehe auch: [[a2]]");
+            PrepareFile(GetFilePathForName("a2"), "[[a2.1]] [[a2.2]]");
+            PrepareFile(GetFilePathForName("b"), "[[b1]] [[b2]] [[b3]] [[b4]] [[b5]]");
+            PrepareFile(GetFilePathForName("b1"), "[[b]] [[b1.1]]");
+            PrepareFile(GetFilePathForName("b2"), "[[b]] [[b2.1]]");
+            PrepareFile(GetFilePathForName("b3"), "[[b]] [[b3.1]]");
+            PrepareFile(GetFilePathForName("b4"), "[[b]] [[b4.1]]");
+            PrepareFile(GetFilePathForName("b5"), "[[b]] [[b5.1]]");
+            PrepareFile(GetFilePathForName("b5.1"), "[[b5]] [[b5.1]]");
+            new PluginIdentifyHotspots().Execute(new[]
+                {
+                    VaultDirectory
+                }
+                , VaultDirectory);
+
+            MarkdownFile file = GetPluginOutputFile();
+            Assert.IsTrue(file.IsHealthy());
+            Assert.AreEqual(10, file.Links.Count);
+            Assert.IsTrue(file.Content.Contains("6x [[b]]"));
+            Assert.IsTrue(file.Content.Contains("2x [[b5.1]]"));
+        }
+    }
+}

ObsidianTools.Test/TestPluginReduceNoise.cs

@@ -0,0 +1,35 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using NUnit.Framework;
+using ObsidianTools.plugins;
+
+namespace ObsidianTools.Test
+{
+    public class TestPluginReduceNoise : BaseTest
+    {
+        [ Test ]
+        public void Test()
+        {
+            PrepareFile(GetFilePathForName("Index"), "[[a1]] [[a2]] [[b]]");
+            PrepareFile(GetFilePathForName("a1"), "[[a1.1]] [[a1.2]] [[a1.3]], siehe auch: [[a2]]");
+            PrepareFile(GetFilePathForName("a2"), "[[a2.1]] [[a2.2]]");
+            PrepareFile(GetFilePathForName("b"), "[[b1]] [[b2]] [[b3]] [[b4]] [[b5]]");
+            PrepareFile(GetFilePathForName("b1"), "[[b]] [[b1.1]]");
+            PrepareFile(GetFilePathForName("b2"), "[[b]] [[b2.1]]");
+            PrepareFile(GetFilePathForName("b3"), "[[b]] [[b3.1]]");
+            PrepareFile(GetFilePathForName("b4"), "[[b]] [[b4.1]]");
+            PrepareFile(GetFilePathForName("b5"), "[[b]] [[b5.1]]");
+            PrepareFile(GetFilePathForName("b5.1"), "[[b5]] [[b5.1]]");
+
+            new PluginReduceNoise().Execute(new[]
+                {
+                    VaultDirectory,
+                     "b", "b"
+                }
+                , VaultDirectory);
+
+
+        }
+    }
+}

ObsidianTools/MarkdownFile.cs

@@ -8,6 +8,7 @@ namespace ObsidianTools
     public class MarkdownFile
     {
         private List<MarkdownLink> _links;
+        private List<MarkdownLink> _linksEmbedded;
 
         public MarkdownFile(String path)
         {
@@ -36,6 +37,20 @@ public List<MarkdownLink> Links
                 return _links ??= MarkdownLink.ForContent(Content);
             }
         }
+        public String FileName
+        {
+            get
+            {
+                return Info?.Name.Replace(Info?.Extension, "");
+            }
+        }
+        public List<MarkdownLink> LinksInklEmbedded
+        {
+            get
+            {
+                return _linksEmbedded ??= MarkdownLink.ForContent(Content, false);
+            }
+        }
 
         public Boolean IsHealthy()
         {
@@ -61,6 +76,7 @@ public void SetContent(String content)
 
             HasChanged = true;
             Content = content;
+            _links = null;
         }
 
         public override String ToString()

ObsidianTools/MarkdownLink.cs

@@ -15,14 +15,14 @@ private MarkdownLink() { }
         public String ChapterName { get; set; }
         public String DisplayName { get; set; }
 
-        public static List<MarkdownLink> ForContent(String content)
+        public static List<MarkdownLink> ForContent(String content, Boolean ignoreEmbedded = true)
         {
             List<MarkdownLink> links = new List<MarkdownLink>();
 
             MatchCollection matches = Regex.Matches(content, REGEX_MARKDOWN_LINK_PATTERN, RegexOptions.IgnoreCase);
             foreach (Match match in matches)
             {
-                if (match.Value.StartsWith("!"))
+                if (ignoreEmbedded && match.Value.StartsWith("!"))
                 {
                     // ignore embedded links
                     continue;

ObsidianTools/Program.cs

@@ -17,8 +17,11 @@ internal class Program
             , new PluginFindReferences()
             , new PluginCreateReferences()
             , new PluginCleanupDead()
+            , new PluginCleanupAssets()
+            , new PluginIdentifyHotspots()
             , new PluginListDead()
             , new PluginCreateDead()
+            , new PluginReduceNoise()
         };
 
         private Program(String[] args)

ObsidianTools/plugins/Plugin.cs

@@ -29,6 +29,11 @@ protected IEnumerable<String> GetMarkdownFilePaths(String directory)
             return Directory.EnumerateFiles(directory, "*.md", SearchOption.AllDirectories);
         }
 
+        protected IEnumerable<String> GetVaultFilePaths(String directory)
+        {
+            return Directory.EnumerateFiles(directory, "*.*", SearchOption.AllDirectories);
+        }
+
         protected abstract void Handle(PluginPayload payload);
 
         protected List<MarkdownFile> ReadFiles(IEnumerable<String> paths)

ObsidianTools/plugins/PluginCleanupAssets.cs

@@ -0,0 +1,74 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+
+namespace ObsidianTools.plugins
+{
+    public class PluginCleanupAssets : Plugin
+    {
+        public PluginCleanupAssets() : base("--cleanup-assets", "Move all unreferenced [jpg|jpeg|pdf|png] files into subfolder .\\_UNUSED\\") { }
+
+        protected override void Handle(PluginPayload payload)
+        {
+            Console.WriteLine("Starting to cleanup-assets...");
+            List<String> unusedVaultFiles = GetVaultFilePaths(payload.VaultDirectory).ToList();
+            unusedVaultFiles.RemoveAll(p => p.EndsWith(".md"));
+            List<String> markdownFiles = GetMarkdownFilePaths(payload.VaultDirectory).ToList();
+            List<MarkdownFile> files = ReadFiles(markdownFiles);
+            Int32 movementsTotal = 0;
+            List<String> relevantFileExtensions = new List<String>
+            {
+                ".jpg"
+                , ".jpeg"
+                , ".pdf"
+                , ".png"
+            };
+            unusedVaultFiles.RemoveAll(f =>
+            {
+                String lowerName = f.ToLower();
+                return !relevantFileExtensions.Any(e => lowerName.EndsWith(e));
+            });
+            foreach (MarkdownFile file in files)
+            {
+                try
+                {
+                    if (!file.IsHealthy() && FileHelper.TryDeleteEmptyFile(file.Info.FullName))
+                    {
+                        continue;
+                    }
+
+                    // get all links
+                    IEnumerable<String> referencedFiles = file.LinksInklEmbedded.Select(l => FileHelper.GetAbsoluteFileName(payload.VaultDirectory, l.FileName));
+                    unusedVaultFiles = unusedVaultFiles.Except(referencedFiles).ToList();
+                }
+                catch (Exception x)
+                {
+                    LogHelper.LogException($"An error occurred cleaning up in path '{file.Info.FullName}'", x);
+                }
+            }
+
+            String targetDirectory = Path.Join(payload.VaultDirectory, "_UNUSED");
+            if (!Directory.Exists(targetDirectory))
+            {
+                Console.WriteLine("Creating \\_UNUSED target directory...");
+                Directory.CreateDirectory(targetDirectory);
+                Console.WriteLine("Creating \\_UNUSED target directory done");
+            }
+
+            Console.WriteLine($"Moving assets ({unusedVaultFiles.Count}x) assets:");
+
+            foreach (String unusedVaultFile in unusedVaultFiles)
+            {
+                String fileName = Path.GetFileName(unusedVaultFile);
+                String newFileName = Path.Join(targetDirectory, fileName);
+                Console.WriteLine($" > Moving asset {unusedVaultFiles} -> {newFileName}...");
+                File.Move(unusedVaultFile, newFileName);
+                Console.WriteLine($" < Moving asset done");
+            }
+
+            Console.WriteLine("Cleanup of unreferenced assets done");
+        }
+    }
+}

ObsidianTools/plugins/PluginCreateReferences.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Collections.Generic;
 using System.IO;
+using System.Linq;
 
 namespace ObsidianTools.plugins
 {
@@ -19,7 +20,7 @@ protected override void Handle(PluginPayload payload)
                 FileInfo info = file.Info;
                 String word = info.Name.Replace(info.Extension, String.Empty);
                 Console.WriteLine($"Processing '{word}'...");
-                files.ForEach(f =>
+                files.Except(new []{file}).ToList().ForEach(f =>
                 {
                     ProcessFile(f, word, word);
                     ProcessFile(f, word, word.ToLower());

ObsidianTools/plugins/PluginIdentifyHotspots.cs

@@ -0,0 +1,75 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+
+namespace ObsidianTools.plugins
+{
+    public class PluginIdentifyHotspots : Plugin
+    {
+        public PluginIdentifyHotspots() : base("--identify-hotspots", "Find the top nodes which are most often linked to") { }
+
+        protected override void Handle(PluginPayload payload)
+        {
+            Console.WriteLine("Starting to identify-hotspots...");
+            List<String> markdownFiles = GetMarkdownFilePaths(payload.VaultDirectory).ToList();
+            List<MarkdownFile> files = ReadFiles(markdownFiles);
+            Dictionary<String, HashSet<String>> referenceCount = CreateReferenceDict(files);
+
+            Console.WriteLine($"Found the following reference counds:");
+            List<KeyValuePair<String, HashSet<String>>> orderedResult = referenceCount.OrderByDescending(kv => kv.Value.Count).ToList();
+
+            String outputPath = $"obsidiantools-output-hotspots-{DateTime.Now:yyyyMMdd-HHmmss}.md";
+            using (StreamWriter writer = File.CreateText(outputPath))
+            {
+                writer.WriteLine("### Identified Hotspots");
+                writer.WriteLine("The following were counted:");
+                foreach ((String source, HashSet<String> linkedTo) in orderedResult.Take(10))
+                {
+                    writer.WriteLine($" - {linkedTo.Count}x [[{source}]]");
+                    Console.WriteLine($" - {linkedTo.Count}x {source}");
+                }
+            }
+
+            Console.WriteLine("Identify-hotspots done");
+            Console.WriteLine($"Creating result file done, you can find it here: {outputPath}");
+        }
+
+        public static Dictionary<String, HashSet<String>> CreateReferenceDict(List<MarkdownFile> files)
+        {
+            Dictionary<String, HashSet<String>> referenceCount = new Dictionary<String, HashSet<String>>();
+            foreach (MarkdownFile file in files)
+            {
+                try
+                {
+                    if (!file.IsHealthy() && FileHelper.TryDeleteEmptyFile(file.Info.FullName))
+                    {
+                        continue;
+                    }
+
+                    foreach (MarkdownLink link in file.Links)
+                    {
+                        if (referenceCount.ContainsKey(link.FileName))
+                        {
+                            referenceCount[link.FileName].Add(file.Info.Name);
+                        }
+                        else
+                        {
+                            referenceCount[link.FileName] = new HashSet<String>
+                            {
+                                file.Info.Name
+                            };
+                        }
+                    }
+                }
+                catch (Exception x)
+                {
+                    LogHelper.LogException($"An error occurred cleaning up in path '{file.Info.FullName}'", x);
+                }
+            }
+
+            return referenceCount;
+        }
+    }
+}