Skip to content

Commit

Permalink
Fix delete order (#13)
Browse files Browse the repository at this point in the history 
* Delete databases in pool/server before deleting pool/server
* Delete app service slots before deleting the app
* Delete apps whose plan name matches the format
* Delete apps whose environment name matches the format
* Delete matching container app environments
* Delete whole kubernetes clusters that match the format
* Share logic for checking name patterns
  • Loading branch information
@mburumaxwell
mburumaxwell authored Apr 6, 2023
1 parent 0bcee97 commit e99b1a9
Show file tree
Hide file tree
Showing 2 changed files with 135 additions and 39 deletions.
38 changes: 37 additions & 1 deletion Tingle.AzdoCleaner.Tests/AzdoEventHandlerTests.cs
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
using Microsoft.Extensions.Caching.Memory;
using Azure.Core;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using System.Text.Json;
Expand Down Expand Up @@ -47,6 +48,41 @@ public void TryFindProject_Works()
Assert.Equal("123456789", token);
}

[Fact]
public void MakePossibleNames_Works()
{
Assert.Equal(new[] { "review-app-23765", "ra-23765", "ra23765", },
AzdoEventHandler.MakePossibleNames(new[] { 23765, }));
Assert.Equal(new[] { "review-app-23765", "ra-23765", "ra23765", "review-app-50", "ra-50", "ra50", },
AzdoEventHandler.MakePossibleNames(new[] { 23765, 50, }));
}

[Fact]
public void NameMatchesExpectedFormat_Works()
{
var possibleNames = AzdoEventHandler.MakePossibleNames(new[] { 23765, });

// works for all in exact format
var modified = possibleNames;
Assert.All(modified, pn => AzdoEventHandler.NameMatchesExpectedFormat(possibleNames, pn));

// works when prefixed
modified = possibleNames.Select(pn => $"bla:{pn}").ToList();
Assert.All(modified, pn => AzdoEventHandler.NameMatchesExpectedFormat(possibleNames, pn));

// works when suffixed
modified = possibleNames.Select(pn => $"{pn}:bla").ToList();
Assert.All(modified, pn => AzdoEventHandler.NameMatchesExpectedFormat(possibleNames, pn));

// works for AppServicePlan
var planId = new ResourceIdentifier($"/subscriptions/{Guid.Empty}/resourceGroups/FABRIKAM/providers/Microsoft.Web/serverfarms/fabrikam-sites-ra23765");
Assert.True(AzdoEventHandler.NameMatchesExpectedFormat(possibleNames, planId));

// works for ManagedEnvironment
var envId = new ResourceIdentifier($"/subscriptions/{Guid.Empty}/resourceGroups/FABRIKAM/providers/Microsoft.App/managedEnvironments/fabrikam-sites-ra-23765");
Assert.True(AzdoEventHandler.NameMatchesExpectedFormat(possibleNames, envId));
}

[Fact]
public async Task HandleAsync_Works()
{
Expand Down
136 changes: 98 additions & 38 deletions Tingle.AzdoCleaner/AzdoEventHandler.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@ protected virtual async Task DeleteReviewAppResourcesAsync(AzdoProjectUrl url, s
var credential = new DefaultAzureCredential();
var client = new ArmClient(credential);

var possibleNames = prIds.SelectMany(prId => new[] { $"review-app-{prId}", $"ra-{prId}", $"ra{prId}", }).ToHashSet().ToList();
var possibleNames = MakePossibleNames(prIds);
if (token is not null)
{
await DeleteReviewAppsEnvironmentsAsync(url, token, possibleNames, cancellationToken);
Expand Down Expand Up @@ -143,24 +143,33 @@ protected virtual async Task DeleteReviewAppResourcesAsync(AzdoProjectUrl url, s
}
}
}
protected virtual async Task DeleteAzureResourceGroupsAsync(SubscriptionResource sub, List<string> possibleNames, CancellationToken cancellationToken)
protected virtual async Task DeleteAzureResourceGroupsAsync(SubscriptionResource sub, IReadOnlyCollection<string> possibleNames, CancellationToken cancellationToken)
{
var groups = sub.GetResourceGroups();
await foreach (var group in groups)
{
var name = group.Data.Name;
if (possibleNames.Any(n => name.EndsWith(n) || name.StartsWith(n)))
if (NameMatchesExpectedFormat(possibleNames, name))
{
logger.LogInformation("Deleting resource group '{ResourceGroupName}' at '{ResourceId}'", name, group.Data.Id);
await group.DeleteAsync(Azure.WaitUntil.Completed, cancellationToken: cancellationToken);
}
}
}
protected virtual async Task DeleteAzureKubernetesNamespacesAsync(SubscriptionResource sub, IReadOnlyList<string> possibleNames, CancellationToken cancellationToken)
protected virtual async Task DeleteAzureKubernetesNamespacesAsync(SubscriptionResource sub, IReadOnlyCollection<string> possibleNames, CancellationToken cancellationToken)
{
var clusters = sub.GetContainerServiceManagedClustersAsync(cancellationToken);
await foreach (var cluster in clusters)
{
// delete matching clusters
var name = cluster.Data.Name;
if (NameMatchesExpectedFormat(possibleNames, name))
{
logger.LogInformation("Deleting AKS cluster '{ClusterName}' at '{ResourceId}'", name, cluster.Data.Id);
await cluster.DeleteAsync(Azure.WaitUntil.Completed, cancellationToken);
continue; // nothing more for the cluster
}

// skip stopped clusters
if (cluster.Data.PowerStateCode == ContainerServiceStateCode.Stopped) continue;

Expand All @@ -172,7 +181,7 @@ protected virtual async Task DeleteAzureKubernetesNamespacesAsync(SubscriptionRe
possibleNames.Count,
string.Join(",", possibleNames));
var namespaces = await kubeClient.ListNamespaceAsync(cancellationToken: cancellationToken); // using labelSelector causes problems, no idea why
var found = namespaces.Items.Where(ns => possibleNames.Any(n => ns.Metadata.Name.EndsWith(n) || ns.Metadata.Name.StartsWith(n))).ToList();
var found = namespaces.Items.Where(ns => NameMatchesExpectedFormat(possibleNames, ns.Metadata.Name)).ToList();
if (found.Count > 0)
{
var names = found.Select(n => n.Metadata.Name).ToList();
Expand All @@ -189,29 +198,17 @@ protected virtual async Task DeleteAzureKubernetesNamespacesAsync(SubscriptionRe
}
}
}
protected virtual async Task DeleteAzureWebsitesAsync(SubscriptionResource sub, List<string> possibleNames, CancellationToken cancellationToken)
protected virtual async Task DeleteAzureWebsitesAsync(SubscriptionResource sub, IReadOnlyCollection<string> possibleNames, CancellationToken cancellationToken)
{
var sites = sub.GetWebSitesAsync(cancellationToken);
await foreach (var site in sites)
{
// delete matching sites
var name = site.Data.Name;
if (possibleNames.Any(n => name.EndsWith(n) || name.StartsWith(n)))
{
logger.LogInformation("Deleting website '{WebsiteName}' in Plan '{ResourceId}'", name, site.Data.AppServicePlanId);
await site.DeleteAsync(Azure.WaitUntil.Completed,
deleteMetrics: true,
deleteEmptyServerFarm: false,
cancellationToken: cancellationToken);
continue; // nothing more for the site
}

// delete matching slots
var slots = site.GetWebSiteSlots().GetAllAsync(cancellationToken);
await foreach (var slot in slots)
{
var slotName = slot.Data.Name;
if (possibleNames.Contains(slotName, StringComparer.OrdinalIgnoreCase))
if (NameMatchesExpectedFormat(possibleNames, slotName))
{
logger.LogInformation("Deleting slot '{SlotName}' in Website '{ResourceId}'", slotName, site.Data.Id);
await slot.DeleteAsync(Azure.WaitUntil.Completed,
Expand All @@ -220,16 +217,29 @@ await slot.DeleteAsync(Azure.WaitUntil.Completed,
cancellationToken: cancellationToken);
}
}

// delete matching sites (either the name or the plan indicates a reviewapp)
var name = site.Data.Name;
var planName = site.Data.AppServicePlanId.Name;
if (NameMatchesExpectedFormat(possibleNames, name) || NameMatchesExpectedFormat(possibleNames, planName))
{
//site.Data.AppServicePlanId
logger.LogInformation("Deleting website '{WebsiteName}' in Plan '{ResourceId}'", name, site.Data.AppServicePlanId);
await site.DeleteAsync(Azure.WaitUntil.Completed,
deleteMetrics: true,
deleteEmptyServerFarm: false,
cancellationToken: cancellationToken);
}
}
}
protected virtual async Task DeleteAzureStaticWebAppsAsync(SubscriptionResource sub, List<string> possibleNames, CancellationToken cancellationToken)
protected virtual async Task DeleteAzureStaticWebAppsAsync(SubscriptionResource sub, IReadOnlyCollection<string> possibleNames, CancellationToken cancellationToken)
{
var sites = sub.GetStaticSitesAsync(cancellationToken);
await foreach (var site in sites)
{
// delete matching sites
var name = site.Data.Name;
if (possibleNames.Any(n => name.EndsWith(n) || name.StartsWith(n)))
if (NameMatchesExpectedFormat(possibleNames, name))
{
logger.LogInformation("Deleting static site '{WebsiteName}'", name);
await site.DeleteAsync(Azure.WaitUntil.Completed, cancellationToken);
Expand All @@ -247,50 +257,76 @@ protected virtual async Task DeleteAzureStaticWebAppsAsync(SubscriptionResource
await foreach (var build in builds)
{
var buildName = build.Data.Name;
if (possibleNames.Contains(buildName, StringComparer.OrdinalIgnoreCase))
if (NameMatchesExpectedFormat(possibleNames, buildName))
{
logger.LogInformation("Deleting build '{BuildName}' in Static WebApp '{ResourceId}'", buildName, site.Data.Id);
await build.DeleteAsync(Azure.WaitUntil.Completed, cancellationToken);
}
}
}
}
protected virtual async Task DeleteAzureContainerAppsAsync(SubscriptionResource sub, List<string> possibleNames, CancellationToken cancellationToken)
protected virtual async Task DeleteAzureContainerAppsAsync(SubscriptionResource sub, IReadOnlyCollection<string> possibleNames, CancellationToken cancellationToken)
{
// delete matching container apps (either the name or the environment indicates a reviewapp)
var apps = sub.GetContainerAppsAsync(cancellationToken);
await foreach (var app in apps)
{
var name = app.Data.Name;
if (possibleNames.Any(n => name.EndsWith(n) || name.StartsWith(n)))
var envName = app.Data.EnvironmentId.Name;
if (NameMatchesExpectedFormat(possibleNames, name) || NameMatchesExpectedFormat(possibleNames, envName))
{
logger.LogInformation("Deleting app '{ContainerAppName}' in Environment '{ResourceId}'", name, app.Data.ManagedEnvironmentId);
await app.DeleteAsync(Azure.WaitUntil.Completed, cancellationToken);
}
}

// delete matching environments
var envs = sub.GetContainerAppManagedEnvironmentsAsync(cancellationToken);
await foreach (var env in envs)
{
var name = env.Data.Name;
if (NameMatchesExpectedFormat(possibleNames, name))
{
logger.LogInformation("Deleting environment '{EnvironmentName}' at '{ResourceId}'", name, env.Data.Id);
await env.DeleteAsync(Azure.WaitUntil.Completed, cancellationToken);
}
}
}
protected virtual async Task DeleteAzureContainerInstancesAsync(SubscriptionResource sub, List<string> possibleNames, CancellationToken cancellationToken)
protected virtual async Task DeleteAzureContainerInstancesAsync(SubscriptionResource sub, IReadOnlyCollection<string> possibleNames, CancellationToken cancellationToken)
{
var groups = sub.GetContainerGroupsAsync(cancellationToken);
await foreach (var group in groups)
{
var name = group.Data.Name;
if (possibleNames.Any(n => name.EndsWith(n) || name.StartsWith(n)))
if (NameMatchesExpectedFormat(possibleNames, name))
{
logger.LogInformation("Deleting app '{ContainerGroupName}' at '{ResourceId}'", name, group.Data.Id);
await group.DeleteAsync(Azure.WaitUntil.Completed, cancellationToken);
}
}
}
protected virtual async Task DeleteAzureSqlAsync(SubscriptionResource sub, List<string> possibleNames, CancellationToken cancellationToken)
protected virtual async Task DeleteAzureSqlAsync(SubscriptionResource sub, IReadOnlyCollection<string> possibleNames, CancellationToken cancellationToken)
{
var servers = sub.GetSqlServersAsync(cancellationToken: cancellationToken);
await foreach (var server in servers)
{
// delete matching servers
var name = server.Data.Name;
if (possibleNames.Any(n => name.EndsWith(n) || name.StartsWith(n)))
if (NameMatchesExpectedFormat(possibleNames, name))
{
logger.LogInformation("Deleting SQL Server '{SqlServerName}'", name);
// delete databases in the server
logger.LogInformation("Deleting databases for SQL Server '{SqlServerName}' at '{ResourceId}'", name, server.Data.Id);
var serverDatabases = server.GetSqlDatabases().GetAllAsync(cancellationToken: cancellationToken);
await foreach (var database in serverDatabases)
{
var databaseName = database.Data.Name;
if (databaseName.Equals("master")) continue;
logger.LogInformation("Deleting database '{DatabaseName}' at '{ResourceId}'", databaseName, database.Data.Id);
await database.DeleteAsync(Azure.WaitUntil.Completed, cancellationToken: cancellationToken);
}

// delete the actual server
logger.LogInformation("Deleting SQL Server '{SqlServerName}' at '{ResourceId}'", name, server.Data.Id);
await server.DeleteAsync(Azure.WaitUntil.Completed, cancellationToken: cancellationToken);
continue; // nothing more for the server
}
Expand All @@ -300,9 +336,21 @@ protected virtual async Task DeleteAzureSqlAsync(SubscriptionResource sub, List<
await foreach (var pool in pools)
{
var poolName = pool.Data.Name;
if (possibleNames.Any(n => name.EndsWith(n) || name.StartsWith(n)))
if (NameMatchesExpectedFormat(possibleNames, poolName))
{
logger.LogInformation("Deleting elastic pool '{DatabaseName}' in Website '{ResourceId}'", poolName, pool.Data.Id);
// delete databases in the pool
logger.LogInformation("Deleting databases for elastic pool '{ElasticPoolName}' at '{ResourceId}'", poolName, pool.Data.Id);
var poolDatabases = pool.GetDatabasesAsync(cancellationToken);
await foreach (var database in poolDatabases)
{
var databaseName = database.Data.Name;
if (databaseName.Equals("master")) continue;
logger.LogInformation("Deleting database '{DatabaseName}' at '{ResourceId}'", databaseName, database.Data.Id);
await database.DeleteAsync(Azure.WaitUntil.Completed, cancellationToken: cancellationToken);
}

// delete the actual pool
logger.LogInformation("Deleting elastic pool '{ElasticPoolName}' at '{ResourceId}'", poolName, pool.Data.Id);
await pool.DeleteAsync(Azure.WaitUntil.Completed, cancellationToken: cancellationToken);
}
}
Expand All @@ -312,19 +360,19 @@ protected virtual async Task DeleteAzureSqlAsync(SubscriptionResource sub, List<
await foreach (var database in databases)
{
var databaseName = database.Data.Name;
if (possibleNames.Any(n => name.EndsWith(n) || name.StartsWith(n)))
if (databaseName.Equals("master")) continue;
if (NameMatchesExpectedFormat(possibleNames, databaseName))
{
logger.LogInformation("Deleting database '{DatabaseName}' in Website '{ResourceId}'", databaseName, database.Data.Id);
logger.LogInformation("Deleting database '{DatabaseName}' at '{ResourceId}'", databaseName, database.Data.Id);
await database.DeleteAsync(Azure.WaitUntil.Completed, cancellationToken: cancellationToken);
}
}
}
}

protected virtual async Task DeleteReviewAppsEnvironmentsAsync(AzdoProjectUrl url, string token, IReadOnlyList<string> names, CancellationToken cancellationToken)
protected virtual async Task DeleteReviewAppsEnvironmentsAsync(AzdoProjectUrl url, string token, IReadOnlyCollection<string> possibleNames, CancellationToken cancellationToken)
{
var connection = CreateVssConnection(url, token);

var client = await connection.GetClientAsync<TaskAgentHttpClient>(cancellationToken);

// iterate through all environments and resources
Expand All @@ -343,7 +391,7 @@ protected virtual async Task DeleteReviewAppsEnvironmentsAsync(AzdoProjectUrl ur

foreach (var resource in environment.Resources)
{
if (names.Contains(resource.Name, StringComparer.OrdinalIgnoreCase))
if (NameMatchesExpectedFormat(possibleNames, resource.Name))
{
logger.LogInformation("Deleting resource '{EnvironmentName}/{ResourceName}' in '{ProjectUrl}'", environment.Name, resource.Name, url);
await client.DeleteKubernetesResourceAsync(url.ProjectIdOrName, environment.Id, resource.Id, cancellationToken: cancellationToken);
Expand All @@ -360,12 +408,24 @@ protected virtual async Task<KubernetesClientConfiguration> GetAzureKubernetesCl
var response = await cluster.GetClusterAdminCredentialsAsync(cancellationToken: cancellationToken);
var credentials = response.Value;
var kubeConfig = FindConfig(credentials, "admin")
?? FindConfig(credentials, "clusterAdmin")
?? throw new InvalidOperationException("Unable to get the cluster credentials");
?? FindConfig(credentials, "clusterAdmin")
?? throw new InvalidOperationException("Unable to get the cluster credentials");
using var ms = new MemoryStream(kubeConfig.Value);
return await KubernetesClientConfiguration.BuildConfigFromConfigFileAsync(ms);
}

internal static IReadOnlyCollection<string> MakePossibleNames(IEnumerable<int> ids)
{
return ids.SelectMany(prId => new[] { $"review-app-{prId}", $"ra-{prId}", $"ra{prId}", })
.ToHashSet();
}

internal static bool NameMatchesExpectedFormat(IReadOnlyCollection<string> possibleNames, Azure.Core.ResourceIdentifier id)
=> NameMatchesExpectedFormat(possibleNames, id.Name);

internal static bool NameMatchesExpectedFormat(IReadOnlyCollection<string> possibleNames, string name)
=> possibleNames.Any(n => name.EndsWith(n) || name.StartsWith(n));

protected virtual VssConnection CreateVssConnection(AzdoProjectUrl url, string token)
{
static string hash(string v)
Expand Down

0 comments on commit e99b1a9

Please sign in to comment.