diff --git a/KustoSchemaTools.Tests/Parser/YamlDatabaseHandlerTests.cs b/KustoSchemaTools.Tests/Parser/YamlDatabaseHandlerTests.cs index b7ed477..c5c4b71 100644 --- a/KustoSchemaTools.Tests/Parser/YamlDatabaseHandlerTests.cs +++ b/KustoSchemaTools.Tests/Parser/YamlDatabaseHandlerTests.cs @@ -21,7 +21,7 @@ public async Task GetDatabase() .WithPlugin(new TablePlugin()) .WithPlugin(new FunctionPlugin()) .WithPlugin(new DatabaseCleanup()); - var loader = factory.Create(Path.Combine(BasePath, Deployment), Database); + var loader = factory.Create(Path.Join(BasePath, Deployment), Database); var db = await loader.LoadAsync(); @@ -51,7 +51,7 @@ public async Task VerifyFunctionPreformatted() .WithPlugin(new TablePlugin()) .WithPlugin(new FunctionPlugin()); // DatabaseCleanup intentionally omitted - var loaderWithoutCleanup = factoryWithoutCleanup.Create(Path.Combine(BasePath, Deployment), Database); + var loaderWithoutCleanup = factoryWithoutCleanup.Create(Path.Join(BasePath, Deployment), Database); var dbWithoutCleanup = await loaderWithoutCleanup.LoadAsync(); // with the DatabaseCleanup plugin @@ -60,7 +60,7 @@ public async Task VerifyFunctionPreformatted() .WithPlugin(new FunctionPlugin()) .WithPlugin(new MaterializedViewsPlugin()) .WithPlugin(new DatabaseCleanup()); - var loaderWithCleanup = factoryWithCleanup.Create(Path.Combine(BasePath, Deployment), Database); + var loaderWithCleanup = factoryWithCleanup.Create(Path.Join(BasePath, Deployment), Database); var dbWithCleanup = await loaderWithCleanup.LoadAsync(); // Assert @@ -113,7 +113,7 @@ public async Task VerifyMaterializedView() .WithPlugin(new TablePlugin()) .WithPlugin(new MaterializedViewsPlugin()); // DatabaseCleanup intentionally omitted - var loaderWithoutCleanup = factoryWithoutCleanup.Create(Path.Combine(BasePath, Deployment), Database); + var loaderWithoutCleanup = factoryWithoutCleanup.Create(Path.Join(BasePath, Deployment), Database); var dbWithoutCleanup = await loaderWithoutCleanup.LoadAsync(); // with the DatabaseCleanup plugin @@ -121,7 +121,7 @@ public async Task VerifyMaterializedView() .WithPlugin(new TablePlugin()) .WithPlugin(new MaterializedViewsPlugin()) .WithPlugin(new DatabaseCleanup()); - var loaderWithCleanup = factoryWithCleanup.Create(Path.Combine(BasePath, Deployment), Database); + var loaderWithCleanup = factoryWithCleanup.Create(Path.Join(BasePath, Deployment), Database); var dbWithCleanup = await loaderWithCleanup.LoadAsync(); // Assert @@ -158,7 +158,7 @@ public async Task VerifyFunctionWithCommentAtEnd() .WithPlugin(new TablePlugin()) .WithPlugin(new FunctionPlugin()) .WithPlugin(new DatabaseCleanup()); - var loader = factory.Create(Path.Combine(BasePath, Deployment), Database); + var loader = factory.Create(Path.Join(BasePath, Deployment), Database); // Act - Load the database var db = await loader.LoadAsync(); diff --git a/KustoSchemaTools/Changes/ClusterChanges.cs b/KustoSchemaTools/Changes/ClusterChanges.cs index 3cd5299..dee21ba 100644 --- a/KustoSchemaTools/Changes/ClusterChanges.cs +++ b/KustoSchemaTools/Changes/ClusterChanges.cs @@ -41,9 +41,17 @@ public static ClusterChangeSet GenerateChanges(Cluster oldCluster, Cluster newCl // Run Kusto code diagnostics foreach (var script in changeSet.Scripts) { - var code = KustoCode.Parse(script.Text); + var code = KustoCode.Parse(script.Script.Text); var diagnostics = code.GetDiagnostics(); script.IsValid = !diagnostics.Any(); + script.Diagnostics = diagnostics.Any() + ? diagnostics.Select(diagnostic => new ScriptDiagnostic + { + Start = diagnostic.Start, + End = diagnostic.End, + Description = diagnostic.Description + }).ToList() + : null; } changeSet.Markdown = GenerateClusterMarkdown(changeSet); @@ -244,7 +252,7 @@ private static string GenerateClusterMarkdown(ClusterChangeSet changeSet) sb.AppendLine("```kql"); foreach (var script in changeSet.Scripts) { - sb.AppendLine(script.Text); + sb.AppendLine(script.Script.Text); sb.AppendLine(); } sb.AppendLine("```"); diff --git a/KustoSchemaTools/Changes/DatabaseChanges.cs b/KustoSchemaTools/Changes/DatabaseChanges.cs index 59017ef..f6ec18c 100644 --- a/KustoSchemaTools/Changes/DatabaseChanges.cs +++ b/KustoSchemaTools/Changes/DatabaseChanges.cs @@ -261,9 +261,17 @@ .. GenerateFollowerCachingChanges(oldState, newState, db => db.MaterializedViews foreach(var script in result.SelectMany(itm => itm.Scripts)) { - var code = KustoCode.Parse(script.Text); + var code = KustoCode.Parse(script.Script.Text); var diagnostics = code.GetDiagnostics(); - script.IsValid = diagnostics.Any() == false; + script.IsValid = !diagnostics.Any(); + script.Diagnostics = diagnostics.Any() + ? diagnostics.Select(diagnostic => new ScriptDiagnostic + { + Start = diagnostic.Start, + End = diagnostic.End, + Description = diagnostic.Description + }).ToList() + : null; } return result; diff --git a/KustoSchemaTools/Changes/DatabaseScriptContainer.cs b/KustoSchemaTools/Changes/DatabaseScriptContainer.cs index 30068ce..f92a8bd 100644 --- a/KustoSchemaTools/Changes/DatabaseScriptContainer.cs +++ b/KustoSchemaTools/Changes/DatabaseScriptContainer.cs @@ -1,4 +1,6 @@ -using KustoSchemaTools.Model; +using System.Collections.Generic; +using KustoSchemaTools.Model; +using Newtonsoft.Json; namespace KustoSchemaTools.Changes { @@ -23,11 +25,19 @@ public DatabaseScriptContainer(string kind, int order, string script, bool isAsy IsAsync = isAsync; } + [JsonProperty("script")] public DatabaseScript Script { get; set; } - public string Kind{ get; set; } + + [JsonProperty("kind")] + public string Kind { get; set; } + + [JsonProperty("isValid")] public bool? IsValid { get; set; } - public string Text => Script.Text; - public int Order => Script.Order; - public bool IsAsync { get;set; } + + [JsonProperty("isAsync")] + public bool IsAsync { get; set; } + + [JsonProperty("diagnostics", NullValueHandling = NullValueHandling.Ignore)] + public List? Diagnostics { get; set; } } } diff --git a/KustoSchemaTools/Changes/DeletionChange.cs b/KustoSchemaTools/Changes/DeletionChange.cs index 6328729..b74ed7a 100644 --- a/KustoSchemaTools/Changes/DeletionChange.cs +++ b/KustoSchemaTools/Changes/DeletionChange.cs @@ -1,5 +1,7 @@ -using KustoSchemaTools.Parser; +using KustoSchemaTools.Model; +using KustoSchemaTools.Parser; using System.Text; +using System.Linq; using Kusto.Language; @@ -22,9 +24,17 @@ public List Scripts get { var sc = new DatabaseScriptContainer("Deletion", 0, $".drop {EntityType} {Entity}"); - var code = KustoCode.Parse(sc.Text); + var code = KustoCode.Parse(sc.Script.Text); var diagnostics = code.GetDiagnostics(); - sc.IsValid = diagnostics.Any() == false; + sc.IsValid = !diagnostics.Any(); + sc.Diagnostics = diagnostics.Any() + ? diagnostics.Select(diagnostic => new ScriptDiagnostic + { + Start = diagnostic.Start, + End = diagnostic.End, + Description = diagnostic.Description + }).ToList() + : null; return new List { sc }; } } diff --git a/KustoSchemaTools/Changes/EntityGroupChange.cs b/KustoSchemaTools/Changes/EntityGroupChange.cs index 759f5ad..40e30f3 100644 --- a/KustoSchemaTools/Changes/EntityGroupChange.cs +++ b/KustoSchemaTools/Changes/EntityGroupChange.cs @@ -28,15 +28,23 @@ private void Init() var toEntityArr = string.Join(",", toEntityStrings); var toScript = new DatabaseScriptContainer("EntityGroup", 3, $".create-or-alter entity_group {Entity} ({toEntityArr})"); - if (added.Any() == false && removed.Any() == false) + if (!added.Any() && !removed.Any()) { return; } Scripts.Add(toScript); - var code = KustoCode.Parse(toScript.Text); + var code = KustoCode.Parse(toScript.Script.Text); var diagnostics = code.GetDiagnostics(); - toScript.IsValid = diagnostics.Any() == false; + toScript.IsValid = !diagnostics.Any(); + toScript.Diagnostics = diagnostics.Any() + ? diagnostics.Select(diagnostic => new ScriptDiagnostic + { + Start = diagnostic.Start, + End = diagnostic.End, + Description = diagnostic.Description + }).ToList() + : null; var logo = toScript.IsValid.Value ? ":green_circle:" : ":red_circle:"; diff --git a/KustoSchemaTools/Changes/ScriptCompareChange.cs b/KustoSchemaTools/Changes/ScriptCompareChange.cs index 74af2a8..8d63465 100644 --- a/KustoSchemaTools/Changes/ScriptCompareChange.cs +++ b/KustoSchemaTools/Changes/ScriptCompareChange.cs @@ -7,6 +7,7 @@ using System.Data; using DiffPlex.DiffBuilder.Model; using Kusto.Language.Editor; +using System.Linq; namespace KustoSchemaTools.Changes { @@ -23,7 +24,7 @@ private void Init() var to = To.CreateScripts(Entity, From == null); Markdown = string.Empty; - if (to.Any() == false) return; + if (!to.Any()) return; StringBuilder sb = new StringBuilder($"## {Entity}"); sb.AppendLine(); @@ -32,8 +33,8 @@ private void Init() foreach (var change in to) { var before = from.ContainsKey(change.Kind) ? from[change.Kind] : null; - var beforeText = before?.Text ?? ""; - var afterText = change.Text; + var beforeText = before?.Script.Text ?? string.Empty; + var afterText = change.Script.Text; var singleLinebeforeText = new KustoCodeService(KustoCode.Parse(beforeText)).GetMinimalText(MinimalTextKind.SingleLine); var singleLineafterText = new KustoCodeService(KustoCode.Parse(afterText)).GetMinimalText(MinimalTextKind.SingleLine); @@ -43,7 +44,7 @@ private void Init() var zipped = reducedBefore.GetLexicalTokens().Zip(reducedAfter.GetLexicalTokens()).ToList(); var diffs = zipped.Where(itm => itm.First.Text != itm.Second.Text).ToList(); - if(diffs.Any() == false) continue; + if (!diffs.Any()) continue; if (singleLinebeforeText.Equals(singleLineafterText)) continue; @@ -51,10 +52,19 @@ private void Init() var diff = InlineDiffBuilder.Diff(beforeText, afterText, true); if (diff.Lines.All(itm => itm.Type == ChangeType.Unchanged)) continue; - var code = KustoCode.Parse(change.Text); + var code = KustoCode.Parse(change.Script.Text); var diagnostics = code.GetDiagnostics(); - change.IsValid = diagnostics.Any() == false || change.Order == -1; + var hasDiagnostics = diagnostics.Any(); + change.IsValid = !hasDiagnostics || change.Script.Order == -1; + change.Diagnostics = hasDiagnostics + ? diagnostics.Select(diagnostic => new ScriptDiagnostic + { + Start = diagnostic.Start, + End = diagnostic.End, + Description = diagnostic.Description + }).ToList() + : null; Scripts.Add(change); @@ -102,10 +112,10 @@ private void Init() } sb.AppendLine(""); sb.AppendLine($" Script:"); - sb.AppendLine($"
{change.Text.PrettifyKql()}
"); + sb.AppendLine($"
{change.Script.Text.PrettifyKql()}
"); sb.AppendLine(""); - if (change.IsValid == false) + if (change.IsValid is false) { foreach (var diagnostic in diagnostics) { diff --git a/KustoSchemaTools/Changes/StructuredChangeExtensions.cs b/KustoSchemaTools/Changes/StructuredChangeExtensions.cs new file mode 100644 index 0000000..bc32909 --- /dev/null +++ b/KustoSchemaTools/Changes/StructuredChangeExtensions.cs @@ -0,0 +1,227 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using DiffPlex; +using DiffPlex.DiffBuilder; +using DiffPlex.DiffBuilder.Model; +using KustoSchemaTools.Model; + +namespace KustoSchemaTools.Changes +{ + public static class StructuredChangeExtensions + { + public static StructuredChange ToStructuredChange(this IChange change) + { + ArgumentNullException.ThrowIfNull(change); + + var structuredChange = new StructuredChange + { + EntityType = change.EntityType, + Entity = change.Entity, + Scripts = change.Scripts?.Select(CloneScript).ToList() ?? new List(), + Comment = StructuredComment.From(change.Comment) + }; + + switch (change) + { + case Heading heading: + structuredChange.ChangeType = "Heading"; + structuredChange.HeadingText = heading.Entity; + structuredChange.Scripts.Clear(); + break; + case DeletionChange deletion: + structuredChange.ChangeType = "Delete"; + structuredChange.DeletedEntities = new List { deletion.Entity }; + break; + case ScriptCompareChange scriptCompare: + structuredChange.ChangeType = scriptCompare.From == null ? "Create" : "Update"; + structuredChange.ScriptComparison = scriptCompare.ToStructuredScriptComparison(); + structuredChange.DiffMarkdown = BuildDiffMarkdown(scriptCompare); + break; + default: + structuredChange.ChangeType = "Update"; + break; + } + + structuredChange.DeletedEntities ??= new List(); + + return structuredChange; + } + + private static StructuredScriptComparison? ToStructuredScriptComparison(this ScriptCompareChange change) + { + var comparison = new StructuredScriptComparison + { + NewScripts = change.Scripts?.Select(CloneScript).ToList() ?? new List() + }; + + if (change.From != null) + { + var previousScripts = BuildPreviousScripts(change.From, change.Entity); + + foreach (var script in comparison.NewScripts) + { + if (previousScripts.TryGetValue(script.Kind, out var previous)) + { + comparison.OldScripts.Add(CloneScript(previous)); + } + } + + var validationPayload = BuildValidationPayload(previousScripts, comparison.NewScripts); + if (validationPayload.Count > 0) + { + comparison.ValidationResults = validationPayload; + } + } + + foreach (var script in comparison.OldScripts.Where(s => !s.IsValid.HasValue)) + { + script.IsValid = true; + } + + return comparison; + } + + private static DatabaseScriptContainer CloneScript(DatabaseScriptContainer source) + { + var clone = new DatabaseScriptContainer(new DatabaseScript(source.Script.Text, source.Script.Order), source.Kind, source.IsAsync) + { + IsValid = source.IsValid + }; + + if (source.Diagnostics != null && source.Diagnostics.Count > 0) + { + clone.Diagnostics = source.Diagnostics + .Select(diagnostic => new ScriptDiagnostic + { + Start = diagnostic.Start, + End = diagnostic.End, + Description = diagnostic.Description + }) + .ToList(); + } + + return clone; + } + + private static Dictionary BuildValidationPayload(Dictionary previousScripts, List newScripts) + { + var payload = new Dictionary(); + foreach (var script in newScripts) + { + previousScripts.TryGetValue(script.Kind, out var oldScript); + var diffPreview = BuildDiffPreview(oldScript, script); + if (diffPreview.Count > 0) + { + var keyName = string.IsNullOrWhiteSpace(script.Kind) ? "diff" : $"diff::{script.Kind}"; + payload[keyName] = diffPreview; + } + } + + return payload; + } + + private static List BuildDiffPreview(DatabaseScriptContainer? oldScript, DatabaseScriptContainer? newScript) + { + var before = GetScriptText(oldScript); + var after = GetScriptText(newScript); + + if (string.Equals(before, after, StringComparison.Ordinal)) + { + return new List(); + } + + var differ = new Differ(); + var diff = InlineDiffBuilder.Diff(before, after, false); + + var preview = diff.Lines + .Where(line => line.Type != ChangeType.Unchanged) + .Select(line => + { + var prefix = line.Type switch + { + ChangeType.Inserted => "+", + ChangeType.Deleted => "-", + ChangeType.Modified => "~", + _ => " " + }; + return $"{prefix}{line.Text?.TrimEnd()}"; + }) + .Where(line => !string.IsNullOrWhiteSpace(line)) + .Take(10) + .ToList(); + + return preview; + } + + private static string GetScriptText(DatabaseScriptContainer? script) + { + return script?.Script?.Text ?? string.Empty; + } + + private static string? BuildDiffMarkdown(ScriptCompareChange change) + { + var previousScripts = BuildPreviousScripts(change.From, change.Entity); + + var sb = new StringBuilder(); + var differ = new Differ(); + foreach (var script in change.Scripts) + { + var before = previousScripts.TryGetValue(script.Kind, out var prior) + ? prior.Script?.Text ?? string.Empty + : string.Empty; + var after = script.Script?.Text ?? string.Empty; + + if (string.Equals(before, after, StringComparison.Ordinal)) + { + continue; + } + + var diff = InlineDiffBuilder.Diff(before, after, false); + var hasMeaningfulDiff = diff.Lines.Any(line => line.Type != ChangeType.Unchanged); + if (!hasMeaningfulDiff) + { + continue; + } + + if (!string.IsNullOrWhiteSpace(script.Kind)) + { + sb.AppendLine($"// {script.Kind}"); + } + + sb.AppendLine("```diff"); + foreach (var line in diff.Lines) + { + var prefix = line.Type switch + { + ChangeType.Inserted => "+", + ChangeType.Deleted => "-", + ChangeType.Modified => "~", + _ => " " + }; + sb.AppendLine($"{prefix}{line.Text}"); + } + sb.AppendLine("```"); + sb.AppendLine(); + } + + var diffContent = sb.ToString().Trim(); + return string.IsNullOrEmpty(diffContent) ? null : diffContent; + } + + private static Dictionary BuildPreviousScripts(IKustoBaseEntity? entity, string changeEntity) + { + if (entity == null) + { + return new Dictionary(); + } + + return entity + .CreateScripts(changeEntity, false) + .GroupBy(script => script.Kind) + .Select(group => group.First()) + .ToDictionary(script => script.Kind, script => script); + } + } +} diff --git a/KustoSchemaTools/KustoSchemaHandler.cs b/KustoSchemaTools/KustoSchemaHandler.cs index c59d6e5..c5be0d9 100644 --- a/KustoSchemaTools/KustoSchemaHandler.cs +++ b/KustoSchemaTools/KustoSchemaHandler.cs @@ -5,8 +5,10 @@ using KustoSchemaTools.Parser.KustoLoader; using Microsoft.Extensions.Logging; using System.Collections.Concurrent; +using System.Collections.Generic; using System.Data; using System.Diagnostics.Metrics; +using System.Linq; using System.Text; using System.Threading.Channels; @@ -27,29 +29,137 @@ public KustoSchemaHandler(ILogger> schemaHandlerLogger, Ya public async Task<(string markDown, bool isValid)> GenerateDiffMarkdown(string path, string databaseName) { + var diffData = await BuildDiffComputationResult(path, databaseName); + return BuildMarkdownOutput(diffData, path, databaseName, logDetails: true); + } + + public async Task GenerateStructuredDiff(string path, string databaseName) + { + var diffData = await BuildDiffComputationResult(path, databaseName); + + var structuredDiffs = new List(); + + foreach (var clusterDiff in diffData.ClusterDiffs) + { + structuredDiffs.Add(ConvertToStructuredDiff(clusterDiff.Cluster.Name, clusterDiff.Cluster.Url, databaseName, clusterDiff.Changes)); + } + + foreach (var followerDiff in diffData.FollowerDiffs) + { + structuredDiffs.Add(ConvertToStructuredDiff(followerDiff.ConnectionKey, followerDiff.ConnectionKey, followerDiff.DatabaseName, followerDiff.Changes)); + } + + var result = new StructuredDiffResult + { + Diffs = structuredDiffs, + IsValid = structuredDiffs.All(diff => diff.IsValid) + }; + + var markdownPreview = BuildMarkdownOutput(diffData, path, databaseName, logDetails: false); + if (markdownPreview.markDown.Length > 50000) + { + result.Message = "The generated output is too long to be posted. Please get the results of the planning from the logs in the actions run."; + } + + return result; + } - var clustersFile = File.ReadAllText(Path.Combine(path, "clusters.yml")); + public async Task Import(string path, string databaseName, bool includeColumns) + { + var clustersFile = File.ReadAllText(Path.Join(path, "clusters.yml")); + var clusters = Serialization.YamlPascalCaseDeserializer.Deserialize(clustersFile); + + var dbHandler = KustoDatabaseHandlerFactory.Create(clusters.Connections[0].Url, databaseName); + + var db = await dbHandler.LoadAsync(); + if (!includeColumns) + { + foreach(var table in db.Tables.Values) + { + table.Columns = new Dictionary(); + } + } + + var fileHandler = YamlDatabaseHandlerFactory.Create(path, databaseName); + await fileHandler.WriteAsync(db); + } + + + public async Task> Apply(string path, string databaseName) + { + var clustersFile = File.ReadAllText(Path.Join(path, "clusters.yml")); var clusters = Serialization.YamlPascalCaseDeserializer.Deserialize(clustersFile); - var sb = new StringBuilder(); - bool isValid = true; var yamlHandler = YamlDatabaseHandlerFactory.Create(path, databaseName); var yamlDb = await yamlHandler.LoadAsync(); - foreach (var cluster in clusters.Connections) + var results = new ConcurrentDictionary(); + + await Parallel.ForEachAsync(clusters.Connections, async (cluster, token) => { - Log.LogInformation($"Generating diff markdown for {Path.Combine(path, databaseName)} => {cluster}/{databaseName}"); + try + { + Log.LogInformation($"Generating and applying script for {Path.Join(path, databaseName)} => {cluster}/{databaseName}"); + var dbHandler = KustoDatabaseHandlerFactory.Create(cluster.Url, databaseName); + await dbHandler.WriteAsync(yamlDb); + results.TryAdd(cluster.Url, null!); + } + catch (Exception ex) + { + results.TryAdd(cluster.Url, ex); + } + }); + return results; + } + + private async Task BuildDiffComputationResult(string path, string databaseName) + { + var clustersFile = File.ReadAllText(Path.Join(path, "clusters.yml")); + var clusters = Serialization.YamlPascalCaseDeserializer.Deserialize(clustersFile); + + var yamlHandler = YamlDatabaseHandlerFactory.Create(path, databaseName); + var yamlDb = await yamlHandler.LoadAsync(); + + var clusterDiffs = new List(); + foreach (var cluster in clusters.Connections) + { var dbHandler = KustoDatabaseHandlerFactory.Create(cluster.Url, databaseName); var kustoDb = await dbHandler.LoadAsync(); var changes = DatabaseChanges.GenerateChanges(kustoDb, yamlDb, databaseName, Log); + clusterDiffs.Add(new ClusterDiffContext(cluster, changes)); + } + + var followerDiffs = new List(); + foreach (var follower in yamlDb.Followers) + { + var followerClient = new KustoClient(follower.Key); + var oldModel = FollowerLoader.LoadFollower(follower.Value.DatabaseName, followerClient); + var changes = DatabaseChanges.GenerateFollowerChanges(oldModel, follower.Value, Log); + followerDiffs.Add(new FollowerDiffContext(follower.Key, follower.Value.DatabaseName, changes)); + } - var comments = changes.Select(itm => itm.Comment).Where(itm => itm != null).ToList(); + return new DiffComputationResult(clusterDiffs, followerDiffs); + } - - isValid &= changes.All(itm => itm.Scripts.All(itm => itm.IsValid != false)) && comments.All(itm => itm.FailsRollout == false); + private (string markDown, bool isValid) BuildMarkdownOutput(DiffComputationResult diffData, string path, string databaseName, bool logDetails) + { + var sb = new StringBuilder(); + bool isValid = true; - sb.AppendLine($"# {cluster.Name}/{databaseName} ({cluster.Url})"); + foreach (var clusterDiff in diffData.ClusterDiffs) + { + if (logDetails) + { + Log.LogInformation($"Generating diff markdown for {Path.Join(path, databaseName)} => {clusterDiff.Cluster.Name}/{databaseName}"); + } + + var changes = clusterDiff.Changes; + var comments = changes.Select(change => change.Comment).OfType().ToList(); + var clusterValid = IsDiffValid(changes); + isValid &= clusterValid; + + sb.AppendLine($"# {clusterDiff.Cluster.Name}/{databaseName} ({clusterDiff.Cluster.Url})"); foreach (var comment in comments) { @@ -70,86 +180,114 @@ public KustoSchemaHandler(ILogger> schemaHandlerLogger, Ya sb.AppendLine(); } - var scriptSb = new StringBuilder(); - foreach(var script in changes.SelectMany(itm => itm.Scripts).Where(itm => itm.IsValid == true).OrderBy(itm => itm.Order)) + if (logDetails) { - scriptSb.AppendLine(script.Text); - } + var scriptSb = new StringBuilder(); + foreach (var script in changes.SelectMany(itm => itm.Scripts).Where(itm => itm.IsValid is true).OrderBy(itm => itm.Script.Order)) + { + scriptSb.AppendLine(script.Script.Text); + } - Log.LogInformation($"Following scripts will be applied:\n{scriptSb}"); + Log.LogInformation($"Following scripts will be applied:\n{scriptSb}"); + } } - foreach(var follower in yamlDb.Followers) + foreach (var followerDiff in diffData.FollowerDiffs) { + if (logDetails) + { + Log.LogInformation($"Generating diff markdown for {Path.Join(path, databaseName)} => {followerDiff.ConnectionKey}/{followerDiff.DatabaseName}"); + } - Log.LogInformation($"Generating diff markdown for {Path.Combine(path, databaseName)} => {follower.Key}/{follower.Value.DatabaseName}"); - - - var followerClient = new KustoClient(follower.Key); - var oldModel = FollowerLoader.LoadFollower(follower.Value.DatabaseName, followerClient); - - var newModel = follower.Value; - - var changes = DatabaseChanges.GenerateFollowerChanges(oldModel, newModel, Log); - - sb.AppendLine($"# Changes for follower database {follower.Key}/{follower.Value.DatabaseName}"); + sb.AppendLine($"# Changes for follower database {followerDiff.ConnectionKey}/{followerDiff.DatabaseName}"); sb.AppendLine(); - foreach (var change in changes) + foreach (var change in followerDiff.Changes) { sb.AppendLine(change.Markdown); - sb.AppendLine(); + sb.AppendLine(); } + + var followerValid = IsDiffValid(followerDiff.Changes); + isValid &= followerValid; } + return (sb.ToString(), isValid); } - public async Task Import(string path, string databaseName, bool includeColumns) + private StructuredDiff ConvertToStructuredDiff(string clusterName, string clusterUrl, string databaseName, List changes) { - var clustersFile = File.ReadAllText(Path.Combine(path, "clusters.yml")); - var clusters = Serialization.YamlPascalCaseDeserializer.Deserialize(clustersFile); + var structuredChanges = changes.Select(change => change.ToStructuredChange()).ToList(); + var comments = changes + .Select(change => StructuredComment.From(change.Comment)) + .OfType() + .ToList(); + + var validScripts = changes + .SelectMany(change => change.Scripts) + .Where(script => script.IsValid is true) + .OrderBy(script => script.Script.Order) + .ToList(); + + return new StructuredDiff + { + ClusterName = clusterName, + ClusterUrl = clusterUrl, + DatabaseName = databaseName, + IsValid = IsDiffValid(changes), + Comments = comments, + Changes = structuredChanges, + ValidScripts = validScripts + }; + } - var dbHandler = KustoDatabaseHandlerFactory.Create(clusters.Connections[0].Url, databaseName); + private static bool IsDiffValid(IEnumerable changes) + { + var changeList = changes?.ToList() ?? new List(); + var scriptsHealthy = changeList.All(change => change.Scripts.All(script => script.IsValid is not false)); + var commentsHealthy = changeList + .Select(change => change.Comment) + .OfType() + .All(comment => comment.FailsRollout is false); + + return scriptsHealthy && commentsHealthy; + } - var db = await dbHandler.LoadAsync(); - if (includeColumns == false) + private sealed class DiffComputationResult + { + public DiffComputationResult(List clusterDiffs, List followerDiffs) { - foreach(var table in db.Tables.Values) - { - table.Columns = new Dictionary(); - } + ClusterDiffs = clusterDiffs; + FollowerDiffs = followerDiffs; } - var fileHandler = YamlDatabaseHandlerFactory.Create(path, databaseName); - await fileHandler.WriteAsync(db); + public List ClusterDiffs { get; } + public List FollowerDiffs { get; } } - - public async Task> Apply(string path, string databaseName) + private sealed class ClusterDiffContext { - var clustersFile = File.ReadAllText(Path.Combine(path, "clusters.yml")); - var clusters = Serialization.YamlPascalCaseDeserializer.Deserialize(clustersFile); - - var yamlHandler = YamlDatabaseHandlerFactory.Create(path, databaseName); - var yamlDb = await yamlHandler.LoadAsync(); + public ClusterDiffContext(Cluster cluster, List changes) + { + Cluster = cluster; + Changes = changes; + } - var results = new ConcurrentDictionary(); + public Cluster Cluster { get; } + public List Changes { get; } + } - await Parallel.ForEachAsync(clusters.Connections, async (cluster, token) => + private sealed class FollowerDiffContext + { + public FollowerDiffContext(string connectionKey, string databaseName, List changes) { - try - { - Log.LogInformation($"Generating and applying script for {Path.Combine(path, databaseName)} => {cluster}/{databaseName}"); - var dbHandler = KustoDatabaseHandlerFactory.Create(cluster.Url, databaseName); - await dbHandler.WriteAsync(yamlDb); - results.TryAdd(cluster.Url, null!); - } - catch (Exception ex) - { - results.TryAdd(cluster.Url, ex); - } - }); + ConnectionKey = connectionKey; + DatabaseName = databaseName; + Changes = changes; + } - return results; + public string ConnectionKey { get; } + public string DatabaseName { get; } + public List Changes { get; } } } } diff --git a/KustoSchemaTools/Model/DatabaseScript.cs b/KustoSchemaTools/Model/DatabaseScript.cs index 545e449..4e47e98 100644 --- a/KustoSchemaTools/Model/DatabaseScript.cs +++ b/KustoSchemaTools/Model/DatabaseScript.cs @@ -1,4 +1,6 @@ -namespace KustoSchemaTools.Model +using Newtonsoft.Json; + +namespace KustoSchemaTools.Model { public class DatabaseScript { @@ -12,7 +14,10 @@ public DatabaseScript() { } + [JsonProperty("text")] public string Text { get; set; } + + [JsonProperty("order")] public int Order { get; set; } } diff --git a/KustoSchemaTools/Model/ScriptDiagnostic.cs b/KustoSchemaTools/Model/ScriptDiagnostic.cs new file mode 100644 index 0000000..c37788f --- /dev/null +++ b/KustoSchemaTools/Model/ScriptDiagnostic.cs @@ -0,0 +1,16 @@ +using Newtonsoft.Json; + +namespace KustoSchemaTools.Model +{ + public class ScriptDiagnostic + { + [JsonProperty("start")] + public int Start { get; set; } + + [JsonProperty("end")] + public int End { get; set; } + + [JsonProperty("description")] + public string Description { get; set; } = string.Empty; + } +} diff --git a/KustoSchemaTools/Model/StructuredDiff.cs b/KustoSchemaTools/Model/StructuredDiff.cs new file mode 100644 index 0000000..b1f0461 --- /dev/null +++ b/KustoSchemaTools/Model/StructuredDiff.cs @@ -0,0 +1,111 @@ +using System.Collections.Generic; +using KustoSchemaTools.Changes; +using Newtonsoft.Json; + +namespace KustoSchemaTools.Model +{ + public class StructuredDiffResult + { + [JsonProperty("isValid")] + public bool IsValid { get; set; } + + [JsonProperty("diffs")] + public List Diffs { get; set; } = new List(); + + [JsonProperty("message", NullValueHandling = NullValueHandling.Ignore)] + public string? Message { get; set; } + } + + public class StructuredDiff + { + [JsonProperty("clusterName")] + public string ClusterName { get; set; } = string.Empty; + + [JsonProperty("clusterUrl")] + public string ClusterUrl { get; set; } = string.Empty; + + [JsonProperty("databaseName")] + public string DatabaseName { get; set; } = string.Empty; + + [JsonProperty("isValid")] + public bool IsValid { get; set; } + + [JsonProperty("comments")] + public List Comments { get; set; } = new List(); + + [JsonProperty("changes")] + public List Changes { get; set; } = new List(); + + [JsonProperty("validScripts")] + public List ValidScripts { get; set; } = new List(); + } + + public class StructuredChange + { + [JsonProperty("entityType")] + public string EntityType { get; set; } = string.Empty; + + [JsonProperty("entity")] + public string Entity { get; set; } = string.Empty; + + [JsonProperty("changeType")] + public string ChangeType { get; set; } = string.Empty; + + [JsonProperty("scripts")] + public List Scripts { get; set; } = new List(); + + [JsonProperty("comment", NullValueHandling = NullValueHandling.Ignore)] + public StructuredComment? Comment { get; set; } + + [JsonProperty("scriptComparison", NullValueHandling = NullValueHandling.Ignore)] + public StructuredScriptComparison? ScriptComparison { get; set; } + + [JsonProperty("deletedEntities")] + public List DeletedEntities { get; set; } = new List(); + + [JsonProperty("headingText", NullValueHandling = NullValueHandling.Ignore)] + public string? HeadingText { get; set; } + + [JsonProperty("diffMarkdown", NullValueHandling = NullValueHandling.Ignore)] + public string? DiffMarkdown { get; set; } + } + + public class StructuredScriptComparison + { + [JsonProperty("oldScripts")] + public List OldScripts { get; set; } = new List(); + + [JsonProperty("newScripts")] + public List NewScripts { get; set; } = new List(); + + [JsonProperty("validationResults", NullValueHandling = NullValueHandling.Ignore)] + public Dictionary? ValidationResults { get; set; } + } + + public class StructuredComment + { + [JsonProperty("kind")] + public string Kind { get; set; } = string.Empty; + + [JsonProperty("text")] + public string Text { get; set; } = string.Empty; + + [JsonProperty("failsRollout")] + public bool FailsRollout { get; set; } + + public static StructuredComment? From(Comment? source) + { + if (source == null) + { + return null; + } + + return new StructuredComment + { + Kind = source.Kind.ToString(), + Text = source.Text, + FailsRollout = source.FailsRollout + }; + } + } +} diff --git a/KustoSchemaTools/Parser/KustoClusterHandler.cs b/KustoSchemaTools/Parser/KustoClusterHandler.cs index 9d2c76b..ccd1c48 100644 --- a/KustoSchemaTools/Parser/KustoClusterHandler.cs +++ b/KustoSchemaTools/Parser/KustoClusterHandler.cs @@ -71,9 +71,9 @@ public virtual async Task> WriteAsync(ClusterCh { var scripts = changeSet.Changes .SelectMany(itm => itm.Scripts) - .Where(itm => itm.Order >= 0) + .Where(itm => itm.Script.Order >= 0) .Where(itm => itm.IsValid == true) - .OrderBy(itm => itm.Order) + .OrderBy(itm => itm.Script.Order) .ToList(); var result = await ExecuteClusterScriptAsync(scripts); @@ -88,7 +88,7 @@ private async Task> ExecuteClusterScriptAsync(L return new List(); } - var scriptTexts = scripts.Select(script => script.Text); + var scriptTexts = scripts.Select(script => script.Script.Text); var script = ".execute cluster script with(ContinueOnErrors = true) <|" + Environment.NewLine + string.Join(Environment.NewLine, scriptTexts); diff --git a/KustoSchemaTools/Parser/KustoWriter/DefaultDatabaseWriter.cs b/KustoSchemaTools/Parser/KustoWriter/DefaultDatabaseWriter.cs index 52b8db6..5968a30 100644 --- a/KustoSchemaTools/Parser/KustoWriter/DefaultDatabaseWriter.cs +++ b/KustoSchemaTools/Parser/KustoWriter/DefaultDatabaseWriter.cs @@ -62,9 +62,9 @@ private async Task> ApplyChangesToDatabase(stri { var scripts = changes .SelectMany(itm => itm.Scripts) - .Where(itm => itm.Order >= 0) + .Where(itm => itm.Script.Order >= 0) .Where(itm => itm.IsValid == true) - .OrderBy(itm => itm.Order) + .OrderBy(itm => itm.Script.Order) .ToList(); var results = new List(); @@ -94,7 +94,7 @@ private async Task ExecuteAsyncCommand(string databa { var interval = TimeSpan.FromSeconds(5); var iterations = (int)(TimeSpan.FromHours(1) / interval); - var result = await client.AdminClient.ExecuteControlCommandAsync(databaseName, sc.Text); + var result = await client.AdminClient.ExecuteControlCommandAsync(databaseName, sc.Script.Text); var operationId = result.ToScalar(); var finalState = false; string monitoringCommand = $".show operations | where OperationId == '{operationId}' " + @@ -114,7 +114,7 @@ private async Task ExecuteAsyncCommand(string databa if (operationState != null && operationState?.IsFinal() == true) { - operationState.CommandText = sc.Text; + operationState.CommandText = sc.Script.Text; return operationState; } await Task.Delay(interval); @@ -132,7 +132,7 @@ private static async Task> ExecutePendingSync(s sb.AppendLine(".execute script with(ContinueOnErrors = true) <|"); foreach (var sc in scripts) { - sb.AppendLine(sc.Text); + sb.AppendLine(sc.Script.Text); } var script = sb.ToString(); diff --git a/KustoSchemaTools/Parser/YamlDatabaseHandler.cs b/KustoSchemaTools/Parser/YamlDatabaseHandler.cs index a5ac9e2..b9b11e3 100644 --- a/KustoSchemaTools/Parser/YamlDatabaseHandler.cs +++ b/KustoSchemaTools/Parser/YamlDatabaseHandler.cs @@ -20,14 +20,14 @@ public YamlDatabaseHandler(string deployment, string database, List> Plugins { get; } public virtual async Task LoadAsync() { - var folder = Path.Combine(Deployment, Database); - var dbFileName = Path.Combine(folder, "database.yml"); + var folder = Path.Join(Deployment, Database); + var dbFileName = Path.Join(folder, "database.yml"); var dbYaml = File.ReadAllText(dbFileName); var db = Serialization.YamlPascalCaseDeserializer.Deserialize(dbYaml); db.Name = Database; foreach (var plugin in Plugins) { - await plugin.OnLoad(db, Path.Combine(Deployment, Database)); + await plugin.OnLoad(db, Path.Join(Deployment, Database)); } return db; } @@ -35,7 +35,7 @@ public virtual async Task LoadAsync() public virtual async Task WriteAsync(T database) { var clone = database.Clone(); - var path = Path.Combine(Deployment, Database); + var path = Path.Join(Deployment, Database); if(!Directory.Exists(path)) { Directory.CreateDirectory(path); @@ -46,7 +46,7 @@ public virtual async Task WriteAsync(T database) await plugin.OnWrite(clone, path); } var yaml = Serialization.YamlPascalCaseSerializer.Serialize(clone); - File.WriteAllText(Path.Combine(path, "database.yml"), yaml); + File.WriteAllText(Path.Join(path, "database.yml"), yaml); } } } diff --git a/KustoSchemaTools/Plugins/EntityPlugin.cs b/KustoSchemaTools/Plugins/EntityPlugin.cs index 2499611..df7f22c 100644 --- a/KustoSchemaTools/Plugins/EntityPlugin.cs +++ b/KustoSchemaTools/Plugins/EntityPlugin.cs @@ -19,7 +19,7 @@ public EntityPlugin(Func> selector, string subFo public async override Task OnLoad(Database existingDatabase, string basePath) { var dict = Selector(existingDatabase); - var path = Path.Combine(basePath, SubFolder); + var path = Path.Join(basePath, SubFolder); if (Directory.Exists(path) == false) return; var files = Directory.GetFiles(path, "*.yml"); @@ -49,9 +49,9 @@ public async override Task OnWrite(Database existingDatabase, string path) var yaml = Serialization.YamlPascalCaseSerializer.Serialize(entity.Value); if (yaml.RowLength() >= MinFileLength) { - var entitySubfolderPath = Path.Combine(path, SubFolder); + var entitySubfolderPath = Path.Join(path, SubFolder); Directory.CreateDirectory(entitySubfolderPath); - await File.WriteAllTextAsync(Path.Combine(entitySubfolderPath, $"{entity.Key}.yml"), yaml); + await File.WriteAllTextAsync(Path.Join(entitySubfolderPath, $"{entity.Key}.yml"), yaml); dict.Remove(entity.Key); } }