Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
79 changes: 79 additions & 0 deletions KustoSchemaTools.Tests/Changes/FollowerPermissionChangeTests.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
using KustoSchemaTools.Changes;
using KustoSchemaTools.Model;
using Microsoft.Extensions.Logging;
using Moq;

namespace KustoSchemaTools.Tests.Changes
{
public class FollowerPermissionChangeTests
{
private readonly Mock<ILogger> _logger = new();

private static FollowerDatabase BuildFollower(params (string role, string id, string name)[] principals)
{
var follower = new FollowerDatabase
{
DatabaseName = "DDoSNeuralAnalysis",
Permissions = new FollowerPermissions { ModificationKind = FollowerModificationKind.Union }
};

foreach (var (role, id, name) in principals)
{
var obj = new AADObject { Id = id, Name = name };
if (string.Equals(role, "admin", StringComparison.OrdinalIgnoreCase))
{
follower.Permissions.Admins.Add(obj);
}
else
{
follower.Permissions.Viewers.Add(obj);
}
}

return follower;
}

[Fact]
public void GeneratesFollowerAdd_WithLeaderName()
{
var oldFollower = BuildFollower();
var newFollower = BuildFollower(("viewer", "aadapp=64decea3-723a-4fbf-b2ec-9faaf852cfdc;398a6654-997b-47e9-b12b-9515b896b4de", "spn-dev-spam-slam"));
newFollower.Permissions.LeaderName = "leader-cluster";

var changes = DatabaseChanges.GenerateFollowerChanges(oldFollower, newFollower, _logger.Object);

var permChange = Assert.Single(changes.OfType<FollowerPermissionChange>());
var script = Assert.Single(permChange.Scripts).Script!.Text;

Assert.Equal(".add follower database DDoSNeuralAnalysis viewers (\"aadapp=64decea3-723a-4fbf-b2ec-9faaf852cfdc;398a6654-997b-47e9-b12b-9515b896b4de\") 'leader-cluster'", script);
}

[Fact]
public void GeneratesFollowerAdd_WithoutLeaderName()
{
var oldFollower = BuildFollower();
var newFollower = BuildFollower(("admin", "aaduser=foo;tenant", "Foo"));

var changes = DatabaseChanges.GenerateFollowerChanges(oldFollower, newFollower, _logger.Object);

var permChange = Assert.Single(changes.OfType<FollowerPermissionChange>());
var script = Assert.Single(permChange.Scripts).Script!.Text;

Assert.Equal(".add follower database DDoSNeuralAnalysis admins (\"aaduser=foo;tenant\")", script);
}

[Fact]
public void EmitsDrop_WhenRemovingPrincipals()
{
var oldFollower = BuildFollower(("admin", "aadapp=1;tenant", "v1"));
var newFollower = BuildFollower();

var changes = DatabaseChanges.GenerateFollowerChanges(oldFollower, newFollower, _logger.Object);

var permChange = Assert.Single(changes.OfType<FollowerPermissionChange>());
var script = Assert.Single(permChange.Scripts).Script!.Text;

Assert.Equal(".drop follower database DDoSNeuralAnalysis admins (\"aadapp=1;tenant\")", script);
}
}
}
13 changes: 13 additions & 0 deletions KustoSchemaTools/Changes/DatabaseChanges.cs
Original file line number Diff line number Diff line change
Expand Up @@ -224,6 +224,19 @@ .. GenerateFollowerCachingChanges(oldState, newState, db => db.MaterializedViews

];

var permissionChanges = new List<IChange>
{
new FollowerPermissionChange(newState.DatabaseName, "Admins", oldState.Permissions.Admins, newState.Permissions.Admins, newState.Permissions.LeaderName),
new FollowerPermissionChange(newState.DatabaseName, "Viewers", oldState.Permissions.Viewers, newState.Permissions.Viewers, newState.Permissions.LeaderName)
}.Where(itm => itm.Scripts.Any()).ToList();

if (permissionChanges.Any())
{
log.LogInformation($"Detected {permissionChanges.Count} follower permission changes");
permissionChanges.Insert(0, new Heading("Permissions (Follower)"));
result.AddRange(permissionChanges);
}

if (oldState.Permissions.ModificationKind != newState.Permissions.ModificationKind)
{
var kind = newState.Permissions.ModificationKind.ToString().ToLower();
Expand Down
95 changes: 95 additions & 0 deletions KustoSchemaTools/Changes/FollowerPermissionChange.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
using Kusto.Language;
using KustoSchemaTools.Model;
using KustoSchemaTools.Parser;
using System.Text;

namespace KustoSchemaTools.Changes
{
// Generates follower-safe permission commands (add/drop) for admins/viewers.
public class FollowerPermissionChange : BaseChange<List<AADObject>>
{
public FollowerPermissionChange(string db, string entity, List<AADObject> from, List<AADObject> to, string? leaderName)
: base("FollowerPermissions", entity, from ?? new List<AADObject>(), to)
{
Db = db;
LeaderName = leaderName;
Init();
}

public string Db { get; }
public string? LeaderName { get; }

private static IEnumerable<string> PrincipalStrings(IEnumerable<AADObject> principals) =>
principals.OrderBy(p => p.Id, StringComparer.OrdinalIgnoreCase).Select(a => "\"" + a.Id + "\"");

private string BuildAdd(IEnumerable<AADObject> principals)
{
var ids = string.Join(",", PrincipalStrings(principals));
var leaderSuffix = string.IsNullOrWhiteSpace(LeaderName) ? string.Empty : $" '{LeaderName}'";
return $".add follower database {Db.BracketIfIdentifier()} {Entity.ToLower()} ({ids}){leaderSuffix}";
}

private string BuildDrop(IEnumerable<AADObject> principals)
{
var ids = string.Join(",", PrincipalStrings(principals));
return $".drop follower database {Db.BracketIfIdentifier()} {Entity.ToLower()} ({ids})";
}

private void Init()
{
var added = To.Where(itm => From.All(t => t.Id != itm.Id)).ToList();
var removed = From.Where(itm => To.All(t => t.Id != itm.Id)).ToList();
var changed = From.Join(To, f => f.Id, t => t.Id, (f, t) => new { f, t })
.Where(x => x.f.Name != x.t.Name)
.ToList();

if (removed.Any())
{
// Execute drops before adds; keep non-negative so they aren't filtered out.
var script = new DatabaseScript { Text = BuildDrop(removed), Order = 0 };
var container = new DatabaseScriptContainer(script, "FollowerPermissionChange");
container.IsValid = !KustoCode.Parse(script.Text).GetDiagnostics().Any();
Scripts.Add(container);
}

if (added.Any())
{
var script = new DatabaseScript { Text = BuildAdd(added), Order = removed.Any() ? 1 : 0 };
var container = new DatabaseScriptContainer(script, "FollowerPermissionChange");
container.IsValid = !KustoCode.Parse(script.Text).GetDiagnostics().Any();
Scripts.Add(container);
}

if (changed.Any())
{
Scripts.Add(new DatabaseScriptContainer("FollowerPermissionRenamed", -1, "// No Database Change"));
}

var sb = new StringBuilder();
sb.AppendLine($"## {Entity} (Follower)");
sb.AppendLine();
sb.AppendLine("<table>");
sb.AppendLine("<tr></tr>");

if (added.Any())
{
sb.AppendLine("<tr><td colspan=\"2\">Added:</td><td colspan=\"10\">" + string.Join("<br>", added.Select(t => $"{t.Name} ({t.Id})")) + "</td></tr>");
}
if (removed.Any())
{
sb.AppendLine("<tr><td colspan=\"2\">Removed:</td><td colspan=\"10\">" + string.Join("<br>", removed.Select(t => $"{t.Name} ({t.Id})")) + "</td></tr>");
}
if (changed.Any())
{
sb.AppendLine("<tr><td colspan=\"2\">Changed:</td><td colspan=\"10\">" + string.Join("<br>", changed.Select(t => $"{t.f.Name} => {t.t.Name} ({t.t.Id})")) + "</td></tr>");
}

var logo = Scripts.Any() && Scripts.First().IsValid == false ? ":red_circle:" : ":green_circle:";
var displayCmd = Scripts.FirstOrDefault()?.Script?.Text ?? "// No change";
sb.AppendLine($"<tr><td colspan=\"2\">{logo}</td><td colspan=\"10\"><pre lang=\"kql\">{displayCmd.PrettifyKql()}</pre></td></tr>");
sb.AppendLine("</table>");

Markdown = sb.ToString();
}
}
}
6 changes: 6 additions & 0 deletions KustoSchemaTools/Model/FollowerDatabase.cs
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,12 @@ public class FollowerDatabase
public FollowerCache Cache { get; set; } = new FollowerCache();
// TODO: No logic to load data / roll out changes implemented yet!
public FollowerPermissions Permissions { get; set; } = new FollowerPermissions();

// True when follower metadata was returned by .show follower database
public bool IsFollower { get; set; }

// Populated when available from follower metadata (e.g., Data Share followers)
public string? LeaderClusterMetadataPath { get; set; }
}

}
3 changes: 3 additions & 0 deletions KustoSchemaTools/Model/FollowerPermissions.cs
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,9 @@ public class FollowerPermissions
public FollowerModificationKind ModificationKind { get; set; }
public List<AADObject> Viewers { get; set; } = new List<AADObject>();
public List<AADObject> Admins { get; set; } = new List<AADObject>();

// Optional leader name (as known to the follower) used by some follower commands
public string? LeaderName { get; set; }
}

}
63 changes: 59 additions & 4 deletions KustoSchemaTools/Parser/KustoLoader/FollowerLoader.cs
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
using KustoSchemaTools.Model;
using KustoSchemaTools.Parser;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using System;
Expand All @@ -21,15 +22,24 @@ public class FollowerLoader
| where isnotempty(Timespan)
| limit 1
)
| summarize CachingPolicies=make_bag(bag_pack(Table,Timespan))
)
";
| summarize CachingPolicies=make_bag(bag_pack(Table,Timespan)),
DatabaseName=any(DatabaseName),
LeaderClusterMetadataPath=any(LeaderClusterMetadataPath),
CachingPolicyOverride=any(CachingPolicyOverride),
AuthorizedPrincipalsOverride=any(AuthorizedPrincipalsOverride),
AuthorizedPrincipalsModificationKind=any(AuthorizedPrincipalsModificationKind),
CachingPoliciesModificationKind=any(CachingPoliciesModificationKind),
ChildEntities=any(ChildEntities),
OriginalDatabaseName=any(OriginalDatabaseName),
IsAutoPrefetchEnabled=any(IsAutoPrefetchEnabled),
LeaderName=any(LeaderName)
)";

public static FollowerDatabase LoadFollower(string databaseName, KustoClient client)
{
var follower = new FollowerDatabase { DatabaseName = databaseName };
// Execute the query and handle the case where no rows are returned (e.g., database is not a follower)
var queryResult = client.Client.ExecuteQuery(string.Format(FollowerMetadataQuery, databaseName));
var queryResult = client.Client.ExecuteQuery(string.Format(FollowerMetadataQuery, databaseName.BracketIfIdentifier()));
var metdaData = queryResult.As<FollowerMetadata>().FirstOrDefault();

if (metdaData == null)
Expand All @@ -38,6 +48,8 @@ public static FollowerDatabase LoadFollower(string databaseName, KustoClient cli
return follower;
}

follower.IsFollower = true;

switch (metdaData.AuthorizedPrincipalsModificationKind)
{
case "Union":
Expand Down Expand Up @@ -73,6 +85,48 @@ public static FollowerDatabase LoadFollower(string databaseName, KustoClient cli
target.Add(key, kvp.Value.Days+"d");
}

follower.Permissions.LeaderName = metdaData.LeaderName;

if (!string.IsNullOrWhiteSpace(metdaData.AuthorizedPrincipalsOverride))
{
try
{
var arr = JArray.Parse(metdaData.AuthorizedPrincipalsOverride);
foreach (var principalObj in arr)
{
var role = principalObj["Role"]?.Value<int?>();
var principal = principalObj["Principal"]?["FullyQualifiedName"]?.Value<string>()
?? principalObj["Principal"]?["Id"]?.Value<string>();
var displayName = principalObj["Principal"]?["DisplayName"]?.Value<string>()
?? principal;

if (string.IsNullOrWhiteSpace(principal) || role == null)
{
continue;
}

var aadObj = new AADObject
{
Id = principal,
Name = displayName
};

if (role == 0)
{
follower.Permissions.Admins.Add(aadObj);
}
else if (role == 2)
{
follower.Permissions.Viewers.Add(aadObj);
}
}
}
catch (Exception)
{
// Ignore parse errors; treat as empty and skip permission diffs
}
}

return follower;
}
}
Expand All @@ -81,6 +135,7 @@ public class FollowerMetadata
{
public string? DatabaseName { get; set; }
public string? LeaderClusterMetadataPath { get; set; }
public string? LeaderName { get; set; }
public string? CachingPolicyOverride { get; set; }
public string? AuthorizedPrincipalsOverride { get; set; }
public string? AuthorizedPrincipalsModificationKind { get; set; }
Expand Down
29 changes: 28 additions & 1 deletion KustoSchemaTools/Parser/KustoWriter/DefaultDatabaseWriter.cs
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,34 @@ public class DefaultDatabaseWriter : IDBEntityWriter
{
public async Task WriteAsync(Database sourceDb, Database targetDb, KustoClient client, ILogger logger)
{
var changes = DatabaseChanges.GenerateChanges(targetDb, sourceDb, targetDb.Name, logger);
var followerMeta = FollowerLoader.LoadFollower(targetDb.Name, client);
// Treat as follower only when metadata exists (.show follower database returned a row)
var isFollower = followerMeta.IsFollower;

List<IChange> changes;
if (isFollower)
{
// Build desired follower from YAML/source DB
var desiredFollower = new FollowerDatabase
{
DatabaseName = targetDb.Name,
Permissions = new FollowerPermissions
{
ModificationKind = followerMeta.Permissions.ModificationKind,
Admins = sourceDb.Admins,
Viewers = sourceDb.Viewers,
LeaderName = followerMeta.Permissions.LeaderName
},
Cache = followerMeta.Cache
};

changes = DatabaseChanges.GenerateFollowerChanges(followerMeta, desiredFollower, logger);
}
else
{
changes = DatabaseChanges.GenerateChanges(targetDb, sourceDb, targetDb.Name, logger);
}

var results = await ApplyChangesToDatabase(targetDb.Name, changes, client, logger);

foreach (var result in results)
Expand Down
Loading