Add initial support for structured diffs

KustoSchemaTools/Changes/ClusterChanges.cs

@@ -41,7 +41,7 @@ public static ClusterChangeSet GenerateChanges(Cluster oldCluster, Cluster newCl
             // Run Kusto code diagnostics
             foreach (var script in changeSet.Scripts)
             {
-                var code = KustoCode.Parse(script.Text);
+                var code = KustoCode.Parse(script.Script.Text);
                 var diagnostics = code.GetDiagnostics();
                 script.IsValid = !diagnostics.Any();
             }
@@ -244,7 +244,7 @@ private static string GenerateClusterMarkdown(ClusterChangeSet changeSet)
                 sb.AppendLine("```kql");
                 foreach (var script in changeSet.Scripts)
                 {
-                    sb.AppendLine(script.Text);
+                    sb.AppendLine(script.Script.Text);
                     sb.AppendLine();
                 }
                 sb.AppendLine("```");

KustoSchemaTools/Changes/DatabaseChanges.cs

@@ -261,7 +261,7 @@ .. GenerateFollowerCachingChanges(oldState, newState, db => db.MaterializedViews
 
             foreach(var script in result.SelectMany(itm => itm.Scripts))
             {
-                var code = KustoCode.Parse(script.Text);
+                var code = KustoCode.Parse(script.Script.Text);
                 var diagnostics = code.GetDiagnostics();
                 script.IsValid = diagnostics.Any() == false;
             }

KustoSchemaTools/Changes/DatabaseScriptContainer.cs

@@ -1,4 +1,6 @@
-using KustoSchemaTools.Model;
+using System.Collections.Generic;
+using KustoSchemaTools.Model;
+using Newtonsoft.Json;
 
 namespace KustoSchemaTools.Changes
 {
@@ -23,11 +25,19 @@ public DatabaseScriptContainer(string kind, int order, string script, bool isAsy
             IsAsync = isAsync;
         }
 
+        [JsonProperty("script")]
         public DatabaseScript Script { get; set; }
-        public string Kind{ get; set; }
+
+        [JsonProperty("kind")]
+        public string Kind { get; set; }
+
+        [JsonProperty("isValid")]
         public bool? IsValid { get; set; }
-        public string Text => Script.Text;
-        public int Order => Script.Order;
-        public bool IsAsync { get;set; }
+
+        [JsonProperty("isAsync")]
+        public bool IsAsync { get; set; }
+
+        [JsonProperty("diagnostics", NullValueHandling = NullValueHandling.Ignore)]
+        public List<ScriptDiagnostic>? Diagnostics { get; set; }
     }
 }

KustoSchemaTools/Changes/DeletionChange.cs

@@ -22,7 +22,7 @@ public List<DatabaseScriptContainer> Scripts
             get
             {
                 var sc = new DatabaseScriptContainer("Deletion", 0, $".drop {EntityType} {Entity}");
-                var code = KustoCode.Parse(sc.Text);
+                var code = KustoCode.Parse(sc.Script.Text);
                 var diagnostics = code.GetDiagnostics();
                 sc.IsValid = diagnostics.Any() == false;
                 return new List<DatabaseScriptContainer> { sc };

KustoSchemaTools/Changes/EntityGroupChange.cs

@@ -34,7 +34,7 @@ private void Init()
             }
 
             Scripts.Add(toScript);
-            var code = KustoCode.Parse(toScript.Text);
+            var code = KustoCode.Parse(toScript.Script.Text);
             var diagnostics = code.GetDiagnostics();
             toScript.IsValid = diagnostics.Any() == false;
             var logo = toScript.IsValid.Value ? ":green_circle:" : ":red_circle:";

KustoSchemaTools/Changes/ScriptCompareChange.cs

@@ -7,12 +7,13 @@
 using System.Data;
 using DiffPlex.DiffBuilder.Model;
 using Kusto.Language.Editor;
+using System.Linq;
 
 namespace KustoSchemaTools.Changes
 {
     public class ScriptCompareChange : BaseChange<IKustoBaseEntity>
     {
         public ScriptCompareChange(string entity, IKustoBaseEntity? from, IKustoBaseEntity to) : base(to.GetType().Name, entity, from, to)
         {
             Init();
         }
@@ -32,8 +33,8 @@
             foreach (var change in to)
             {
                 var before = from.ContainsKey(change.Kind) ? from[change.Kind] : null;
-                var beforeText = before?.Text ?? "";
-                var afterText = change.Text;
+                var beforeText = before?.Script.Text ?? string.Empty;
+                var afterText = change.Script.Text;
 
                 var singleLinebeforeText = new KustoCodeService(KustoCode.Parse(beforeText)).GetMinimalText(MinimalTextKind.SingleLine);
                 var singleLineafterText = new KustoCodeService(KustoCode.Parse(afterText)).GetMinimalText(MinimalTextKind.SingleLine);
@@ -51,10 +52,19 @@
                 var diff = InlineDiffBuilder.Diff(beforeText, afterText, true);
                 if (diff.Lines.All(itm => itm.Type == ChangeType.Unchanged)) continue;
 
-                var code = KustoCode.Parse(change.Text);                
+                var code = KustoCode.Parse(change.Script.Text);                
 
                 var diagnostics = code.GetDiagnostics();
-                change.IsValid = diagnostics.Any() == false || change.Order == -1;
+                var hasDiagnostics = diagnostics.Any();
+                change.IsValid = hasDiagnostics == false || change.Script.Order == -1;
+                change.Diagnostics = hasDiagnostics
+                    ? diagnostics.Select(diagnostic => new ScriptDiagnostic
+                    {
+                        Start = diagnostic.Start,
+                        End = diagnostic.End,
+                        Description = diagnostic.Description
+                    }).ToList()
+                    : null;
                 Scripts.Add(change);     
 
 
@@ -102,7 +112,7 @@
                 }
                 sb.AppendLine("<tr>");
                 sb.AppendLine($"    <td colspan=\"2\">Script:</td>");
-                sb.AppendLine($"    <td colspan=\"10\"><pre lang=\"kql\">{change.Text.PrettifyKql()}</pre></td>");
+                sb.AppendLine($"    <td colspan=\"10\"><pre lang=\"kql\">{change.Script.Text.PrettifyKql()}</pre></td>");
                 sb.AppendLine("</tr>");
 
                 if (change.IsValid == false)

KustoSchemaTools/Changes/StructuredChangeExtensions.cs

@@ -0,0 +1,225 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using DiffPlex;
+using DiffPlex.DiffBuilder;
+using DiffPlex.DiffBuilder.Model;
+using KustoSchemaTools.Model;
+
+namespace KustoSchemaTools.Changes
+{
+    public static class StructuredChangeExtensions
+    {
+        public static StructuredChange ToStructuredChange(this IChange change)
+        {
+            if (change == null)
+            {
+                throw new ArgumentNullException(nameof(change));
+            }
+
+            var structuredChange = new StructuredChange
+            {
+                EntityType = change.EntityType,
+                Entity = change.Entity,
+                Scripts = change.Scripts?.Select(CloneScript).ToList() ?? new List<DatabaseScriptContainer>(),
+                Comment = StructuredComment.From(change.Comment)
+            };
+
+            switch (change)
+            {
+                case Heading heading:
+                    structuredChange.ChangeType = "Heading";
+                    structuredChange.HeadingText = heading.Entity;
+                    structuredChange.Scripts.Clear();
+                    break;
+                case DeletionChange deletion:
+                    structuredChange.ChangeType = "Delete";
+                    structuredChange.DeletedEntities = new List<string> { deletion.Entity };
+                    break;
+                case ScriptCompareChange scriptCompare:
+                    structuredChange.ChangeType = scriptCompare.From == null ? "Create" : "Update";
+                    structuredChange.ScriptComparison = scriptCompare.ToStructuredScriptComparison();
+                    structuredChange.DiffMarkdown = BuildDiffMarkdown(scriptCompare);
+                    break;
+                default:
+                    structuredChange.ChangeType = "Update";
+                    break;
+            }
+
+            structuredChange.DeletedEntities ??= new List<string>();
+
+            return structuredChange;
+        }
+
+        private static StructuredScriptComparison? ToStructuredScriptComparison(this ScriptCompareChange change)
+        {
+            var comparison = new StructuredScriptComparison
+            {
+                NewScripts = change.Scripts?.Select(CloneScript).ToList() ?? new List<DatabaseScriptContainer>()
+            };
+
+            if (change.From != null)
+            {
+                var previousScripts = change.From
+                    .CreateScripts(change.Entity, false)
+                    .GroupBy(script => script.Kind)
+                    .Select(group => group.First())
+                    .ToDictionary(script => script.Kind, script => script);
+
+                foreach (var script in comparison.NewScripts)
+                {
+                    if (previousScripts.TryGetValue(script.Kind, out var previous))
+                    {
+                        comparison.OldScripts.Add(CloneScript(previous));
+                    }
+                }
+
+                var validationPayload = BuildValidationPayload(previousScripts, comparison.NewScripts);
+                if (validationPayload.Count > 0)
+                {
+                    comparison.ValidationResults = validationPayload;
+                }
+            }
+
+            foreach (var script in comparison.OldScripts.Where(s => !s.IsValid.HasValue))
+            {
+                script.IsValid = true;
+            }
+
+            return comparison;
+        }
+
+        private static DatabaseScriptContainer CloneScript(DatabaseScriptContainer source)
+        {
+            var clone = new DatabaseScriptContainer(new DatabaseScript(source.Script.Text, source.Script.Order), source.Kind, source.IsAsync)
+            {
+                IsValid = source.IsValid
+            };
+
+            if (source.Diagnostics != null && source.Diagnostics.Count > 0)
+            {
+                clone.Diagnostics = source.Diagnostics
+                    .Select(diagnostic => new ScriptDiagnostic
+                    {
+                        Start = diagnostic.Start,
+                        End = diagnostic.End,
+                        Description = diagnostic.Description
+                    })
+                    .ToList();
+            }
+
+            return clone;
+        }
+
+        private static Dictionary<string, object?> BuildValidationPayload(Dictionary<string, DatabaseScriptContainer> previousScripts, List<DatabaseScriptContainer> newScripts)
+        {
+            var payload = new Dictionary<string, object?>();
+            foreach (var script in newScripts)
+            {
+                var oldScript = previousScripts.ContainsKey(script.Kind) ? previousScripts[script.Kind] : null;
+                var diffPreview = BuildDiffPreview(oldScript, script);
+                if (diffPreview.Count > 0)
+                {
+                    var keyName = string.IsNullOrWhiteSpace(script.Kind) ? "diff" : $"diff::{script.Kind}";
+                    payload[keyName] = diffPreview;
+                }
+            }
+
+            return payload;
+        }
+
+        private static List<string> BuildDiffPreview(DatabaseScriptContainer? oldScript, DatabaseScriptContainer? newScript)
+        {
+            var before = GetScriptText(oldScript);
+            var after = GetScriptText(newScript);
+
+            if (string.Equals(before, after, StringComparison.Ordinal))
+            {
+                return new List<string>();
+            }
+
+            var differ = new Differ();
+            var diff = InlineDiffBuilder.Diff(before, after, false);
+
+            var preview = diff.Lines
+                .Where(line => line.Type != ChangeType.Unchanged)
+                .Select(line =>
+                {
+                    var prefix = line.Type switch
+                    {
+                        ChangeType.Inserted => "+",
+                        ChangeType.Deleted => "-",
+                        ChangeType.Modified => "~",
+                        _ => " "
+                    };
+                    return $"{prefix}{line.Text?.TrimEnd()}";
+                })
+                .Where(line => !string.IsNullOrWhiteSpace(line))
+                .Take(10)
+                .ToList();
+
+            return preview;
+        }
+
+        private static string GetScriptText(DatabaseScriptContainer? script)
+        {
+            return script?.Script?.Text ?? string.Empty;
+        }
+
+        private static string? BuildDiffMarkdown(ScriptCompareChange change)
+        {
+            var previousScripts = change.From?
+                .CreateScripts(change.Entity, false)
+                .GroupBy(script => script.Kind)
+                .Select(group => group.First())
+                .ToDictionary(script => script.Kind, script => script)
+                ?? new Dictionary<string, DatabaseScriptContainer>();
+
+            var sb = new StringBuilder();
+            var differ = new Differ();
+            foreach (var script in change.Scripts)
+            {
+                var before = previousScripts.TryGetValue(script.Kind, out var prior)
+                    ? prior.Script?.Text ?? string.Empty
+                    : string.Empty;
+                var after = script.Script?.Text ?? string.Empty;
+
+                if (string.Equals(before, after, StringComparison.Ordinal))
+                {
+                    continue;
+                }
+
+                var diff = InlineDiffBuilder.Diff(before, after, false);
+                var hasMeaningfulDiff = diff.Lines.Any(line => line.Type != ChangeType.Unchanged);
+                if (!hasMeaningfulDiff)
+                {
+                    continue;
+                }
+
+                if (!string.IsNullOrWhiteSpace(script.Kind))
+                {
+                    sb.AppendLine($"// {script.Kind}");
+                }
+
+                sb.AppendLine("```diff");
+                foreach (var line in diff.Lines)
+                {
+                    var prefix = line.Type switch
+                    {
+                        ChangeType.Inserted => "+",
+                        ChangeType.Deleted => "-",
+                        ChangeType.Modified => "~",
+                        _ => " "
+                    };
+                    sb.AppendLine($"{prefix}{line.Text}");
+                }
+                sb.AppendLine("```");
+                sb.AppendLine();
+            }
+
+            var diffContent = sb.ToString().Trim();
+            return string.IsNullOrEmpty(diffContent) ? null : diffContent;
+        }
+    }
+}