storage - back-port storage data plane workarounds (#27819)

* replace on branched fixes for base layer to allow testing

* squash of experimental commits to clean up

* make generate

* add changelog for manual release

* changes to match review on related PR

* review feedback

* update changelog in readiness for manual release

* restore return read

* prep changelog for manual release

.github/labeler-issue-triage.yml

@@ -103,7 +103,7 @@ service/database-migration:
   - '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_database_migration_((.|\n)*)###'
 
 service/databox-edge:
-  - '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_databox_edge_((.|\n)*)###'
+  - '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_databox_edge_device((.|\n)*)###'
 
 service/databricks:
   - '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_databricks_((.|\n)*)###'
@@ -121,7 +121,6 @@ service/digital-twins:
   - '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_digital_twins_((.|\n)*)###'
 
 service/disks:
-  - '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_disk_pool((.|\n)*)###'
 
 service/dns:
   - '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_(dns_|private_dns_a_record\W+|private_dns_aaaa_record\W+|private_dns_cname_record\W+|private_dns_mx_record\W+|private_dns_ptr_record\W+|private_dns_soa_record\W+|private_dns_srv_record\W+|private_dns_txt_record\W+|private_dns_zone\W+|private_dns_zone_virtual_network_link\W+)((.|\n)*)###'
@@ -154,7 +153,7 @@ service/frontdoor:
   - '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_frontdoor((.|\n)*)###'
 
 service/graph:
-  - '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_graph_((.|\n)*)###'
+  - '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_graph_services_account((.|\n)*)###'
 
 service/hdinsight:
   - '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_hdinsight_((.|\n)*)###'
@@ -166,7 +165,7 @@ service/hsm:
   - '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_dedicated_hardware_security_module((.|\n)*)###'
 
 service/hybrid-compute:
-  - '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_(arc_machine\W+|arc_machine_extension\W+|arc_private_link_scope\W+|hybrid_compute_machine)((.|\n)*)###'
+  - '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_(arc_machine\W+|arc_machine_extension\W+|arc_private_link_scope\W+)((.|\n)*)###'
 
 service/iot-central:
   - '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_iotcentral_((.|\n)*)###'
@@ -175,7 +174,6 @@ service/iot-hub:
   - '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_iothub((.|\n)*)###'
 
 service/iot-time-series:
-  - '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_iot_time_series_insights_((.|\n)*)###'
 
 service/key-vault:
   - '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_(key_vault\W+|key_vault_access_policy\W+|key_vault_certificate\W+|key_vault_certificate_contacts\W+|key_vault_certificate_data\W+|key_vault_certificate_issuer\W+|key_vault_certificates\W+|key_vault_encrypted_value\W+|key_vault_key\W+|key_vault_managed_storage_account\W+|key_vault_managed_storage_account_sas_token_definition\W+|key_vault_secret\W+|key_vault_secrets\W+)((.|\n)*)###'
@@ -184,7 +182,6 @@ service/kusto:
   - '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_kusto_((.|\n)*)###'
 
 service/labservice:
-  - '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_lab_service_((.|\n)*)###'
 
 service/lighthouse:
   - '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_lighthouse_((.|\n)*)###'
@@ -199,10 +196,9 @@ service/log-analytics:
   - '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_log_analytics_((.|\n)*)###'
 
 service/logic:
-  - '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_(integration_service_environment|logic_app_)((.|\n)*)###'
+  - '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_logic_app_((.|\n)*)###'
 
 service/logz:
-  - '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_logz_((.|\n)*)###'
 
 service/machine-learning:
   - '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_machine_learning_((.|\n)*)###'
@@ -226,7 +222,7 @@ service/maria-db:
   - '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_mariadb_((.|\n)*)###'
 
 service/media:
-  - '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_media_((.|\n)*)###'
+  - '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_media_services_account_filter((.|\n)*)###'
 
 service/mixed-reality:
   - '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_spatial_anchors_account((.|\n)*)###'
@@ -244,7 +240,7 @@ service/mssqlmanagedinstance:
   - '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_mssql_managed_((.|\n)*)###'
 
 service/mysql:
-  - '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_mysql_((.|\n)*)###'
+  - '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_mysql_flexible_((.|\n)*)###'
 
 service/netapp:
   - '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_netapp_((.|\n)*)###'
@@ -271,7 +267,7 @@ service/policy:
   - '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_(management_group_policy_|policy_|resource_group_policy_assignment\W+|resource_group_policy_exemption\W+|resource_group_policy_remediation\W+|resource_policy_assignment\W+|resource_policy_exemption\W+|resource_policy_remediation\W+|subscription_policy_)((.|\n)*)###'
 
 service/portal:
-  - '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_(dashboard\W+|portal_)((.|\n)*)###'
+  - '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_portal_((.|\n)*)###'
 
 service/postgresql:
   - '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_postgresql_((.|\n)*)###'
@@ -328,10 +324,9 @@ service/spring:
   - '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_(spring_cloud_accelerator\W+|spring_cloud_active_deployment\W+|spring_cloud_api_portal\W+|spring_cloud_api_portal_custom_domain\W+|spring_cloud_app\W+|spring_cloud_app_cosmosdb_association\W+|spring_cloud_app_dynamics_application_performance_monitoring\W+|spring_cloud_app_mysql_association\W+|spring_cloud_app_redis_association\W+|spring_cloud_application_insights_application_performance_monitoring\W+|spring_cloud_application_live_view\W+|spring_cloud_build_deployment\W+|spring_cloud_build_pack_binding\W+|spring_cloud_builder\W+|spring_cloud_certificate\W+|spring_cloud_configuration_service\W+|spring_cloud_container_deployment\W+|spring_cloud_custom_domain\W+|spring_cloud_customized_accelerator\W+|spring_cloud_dev_tool_portal\W+|spring_cloud_dynatrace_application_performance_monitoring\W+|spring_cloud_elastic_application_performance_monitoring\W+|spring_cloud_gateway\W+|spring_cloud_gateway_custom_domain\W+|spring_cloud_gateway_route_config\W+|spring_cloud_java_deployment\W+|spring_cloud_new_relic_application_performance_monitoring\W+|spring_cloud_service\W+|spring_cloud_storage\W+)((.|\n)*)###'
 
 service/sql:
-  - '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_sql_((.|\n)*)###'
 
 service/storage:
-  - '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_(storage_account\W+|storage_account_blob_container_sas\W+|storage_account_customer_managed_key\W+|storage_account_local_user\W+|storage_account_network_rules\W+|storage_account_sas\W+|storage_blob\W+|storage_blob_inventory_policy\W+|storage_container\W+|storage_container_immutability_policy\W+|storage_containers\W+|storage_data_lake_gen2_filesystem\W+|storage_data_lake_gen2_path\W+|storage_encryption_scope\W+|storage_management_policy\W+|storage_object_replication\W+|storage_queue\W+|storage_share\W+|storage_share_directory\W+|storage_share_file\W+|storage_sync\W+|storage_sync_cloud_endpoint\W+|storage_sync_group\W+|storage_sync_server_endpoint\W+|storage_table\W+|storage_table\W+|storage_table_entities\W+|storage_table_entity\W+)((.|\n)*)###'
+  - '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_(storage_account\W+|storage_account_blob_container_sas\W+|storage_account_customer_managed_key\W+|storage_account_local_user\W+|storage_account_network_rules\W+|storage_account_queue_properties\W+|storage_account_sas\W+|storage_account_static_website\W+|storage_blob\W+|storage_blob_inventory_policy\W+|storage_container\W+|storage_container_immutability_policy\W+|storage_containers\W+|storage_data_lake_gen2_filesystem\W+|storage_data_lake_gen2_path\W+|storage_encryption_scope\W+|storage_management_policy\W+|storage_object_replication\W+|storage_queue\W+|storage_share\W+|storage_share_directory\W+|storage_share_file\W+|storage_sync\W+|storage_sync_cloud_endpoint\W+|storage_sync_group\W+|storage_sync_server_endpoint\W+|storage_table\W+|storage_table\W+|storage_table_entities\W+|storage_table_entity\W+)((.|\n)*)###'
 
 service/storagemover:
   - '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_storage_mover((.|\n)*)###'
@@ -352,7 +347,6 @@ service/traffic-manager:
   - '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_traffic_manager_((.|\n)*)###'
 
 service/video-analyzer:
-  - '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_video_analyzer((.|\n)*)###'
 
 service/virtual-desktops:
   - '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_virtual_desktop_((.|\n)*)###'

CHANGELOG.md

@@ -1,3 +1,17 @@
+## 3.117.0 (November 7, 2024)
+
+SPECIAL NOTES: This 3.x release is a special, one-off, back-port of functionality for `azurerm_storage_account` to enable users to deploy this resource in environments which block / are restrictive of Data Plane access, thus preventing the resource being created and/or managed.  This functionality is back-ported from the `v4.9.0` release. Users migrating from this release to the 4.x line, should upgrade directly to `v4.9.0` or later, as these features are not compatible with earlier releases of 4.x.
+
+FEATURES:
+
+* **New Resource:** `azurerm_storage_account_queue_properties` ([#27819](https://github.com/hashicorp/terraform-provider-azurerm/pull/27819))
+* **New Resource:** `azurerm_storage_account_static_website`  ([#27819](https://github.com/hashicorp/terraform-provider-azurerm/pull/27819))
+* New Provider Feature - storage `data_plane_available` feature flag ([#27819](https://github.com/hashicorp/terraform-provider-azurerm/pull/27819))
+
+ENHANCEMENTS:
+
+* `azurerm_storage_account` - can now be created and managed if Data Plane endpoints are blocked by a firewall ([#27819](https://github.com/hashicorp/terraform-provider-azurerm/pull/27819)) 
+
 ## 3.116.0 (August 16, 2024)
 
 DEPRECATIONS:

go.mod

@@ -18,7 +18,7 @@ require (
 	github.com/google/uuid v1.6.0
 	github.com/hashicorp/go-azure-helpers v0.70.1
 	github.com/hashicorp/go-azure-sdk/resource-manager v0.20240731.1212841
-	github.com/hashicorp/go-azure-sdk/sdk v0.20240731.1212841
+	github.com/hashicorp/go-azure-sdk/sdk v0.20241025.1143247 // out of sync with rm due to drift of versions but required network fixes
 	github.com/hashicorp/go-hclog v1.6.3
 	github.com/hashicorp/go-multierror v1.1.1
 	github.com/hashicorp/go-uuid v1.0.3

go.sum

@@ -95,8 +95,8 @@ github.com/hashicorp/go-azure-helpers v0.70.1 h1:7hlnRrZobMZxpOzdlNEsayzAayj/KRG
 github.com/hashicorp/go-azure-helpers v0.70.1/go.mod h1:BmbF4JDYXK5sEmFeU5hcn8Br21uElcqLfdQxjatwQKw=
 github.com/hashicorp/go-azure-sdk/resource-manager v0.20240731.1212841 h1:H7BkxZl0qitdWq7sEGzNqkn5/11YTamwq2XTI8/7Jq0=
 github.com/hashicorp/go-azure-sdk/resource-manager v0.20240731.1212841/go.mod h1:/4Ly9Gppp/Nu9AaWDfod6atYQ4n2OqN0ERpE2xZQz8A=
-github.com/hashicorp/go-azure-sdk/sdk v0.20240731.1212841 h1:RzWuy96j/7q3Vi2aZoiIrokm8yotUNX1UGD3pbWi5Ck=
-github.com/hashicorp/go-azure-sdk/sdk v0.20240731.1212841/go.mod h1:dMKF6bXrgGmy1d3pLzkmBpG2JIHgSAV2/OMSCEgyMwE=
+github.com/hashicorp/go-azure-sdk/sdk v0.20241025.1143247 h1:NoYFgxtEsxHhE6TyJ6DRXqHLcxZ0cmrpGxNPio0lT84=
+github.com/hashicorp/go-azure-sdk/sdk v0.20241025.1143247/go.mod h1:dMKF6bXrgGmy1d3pLzkmBpG2JIHgSAV2/OMSCEgyMwE=
 github.com/hashicorp/go-checkpoint v0.5.0 h1:MFYpPZCnQqQTE18jFwSII6eUQrD/oxMFp3mlgcqk5mU=
 github.com/hashicorp/go-checkpoint v0.5.0/go.mod h1:7nfLNL10NsxqO4iWuW6tWW0HjZuDrwkBuEQsVcpCOgg=
 github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=

internal/clients/graph/client.go

@@ -41,7 +41,7 @@ type directoryObjectModel struct {
 }
 
 func graphClient(authorizer auth.Authorizer, environment environments.Environment) (*msgraph.Client, error) {
-	client, err := msgraph.NewMsGraphClient(environment.MicrosoftGraph, "Graph", msgraph.VersionOnePointZero)
+	client, err := msgraph.NewClient(environment.MicrosoftGraph, "Graph", msgraph.VersionOnePointZero)
 	if err != nil {
 		return nil, fmt.Errorf("building client: %+v", err)
 	}

internal/features/defaults.go

@@ -62,6 +62,9 @@ func Default() UserFeatures {
 			RollInstancesWhenRequired: true,
 			ScaleToZeroOnDelete:       true,
 		},
+		Storage: StorageFeatures{
+			DataPlaneAvailable: true,
+		},
 		Subscription: SubscriptionFeatures{
 			PreventCancellationOnDestroy: false,
 		},

internal/features/user_flags.go

@@ -16,6 +16,7 @@ type UserFeatures struct {
 	ResourceGroup            ResourceGroupFeatures
 	RecoveryServicesVault    RecoveryServicesVault
 	ManagedDisk              ManagedDiskFeatures
+	Storage                  StorageFeatures
 	Subscription             SubscriptionFeatures
 	PostgresqlFlexibleServer PostgresqlFlexibleServerFeatures
 	MachineLearning          MachineLearningFeatures
@@ -84,6 +85,10 @@ type AppConfigurationFeatures struct {
 	RecoverSoftDeleted       bool
 }
 
+type StorageFeatures struct {
+	DataPlaneAvailable bool
+}
+
 type SubscriptionFeatures struct {
 	PreventCancellationOnDestroy bool
 }

internal/provider/features.go

@@ -309,6 +309,21 @@ func schemaFeatures(supportLegacyTestSuite bool) *pluginsdk.Schema {
 			},
 		},
 
+		"storage": {
+			Type:     pluginsdk.TypeList,
+			Optional: true,
+			MaxItems: 1,
+			Elem: &pluginsdk.Resource{
+				Schema: map[string]*schema.Schema{
+					"data_plane_available": {
+						Type:     pluginsdk.TypeBool,
+						Optional: true,
+						Default:  true,
+					},
+				},
+			},
+		},
+
 		"subscription": {
 			Type:     pluginsdk.TypeList,
 			Optional: true,
@@ -581,6 +596,16 @@ func expandFeatures(input []interface{}) features.UserFeatures {
 		}
 	}
 
+	if raw, ok := val["storage"]; ok {
+		items := raw.([]interface{})
+		if len(items) > 0 {
+			storageRaw := items[0].(map[string]interface{})
+			if v, ok := storageRaw["data_plane_available"]; ok {
+				featuresMap.Storage.DataPlaneAvailable = v.(bool)
+			}
+		}
+	}
+
 	if raw, ok := val["subscription"]; ok {
 		items := raw.([]interface{})
 		if len(items) > 0 {

internal/provider/features_test.go

@@ -75,6 +75,9 @@ func TestExpandFeatures(t *testing.T) {
 				RecoveryServicesVault: features.RecoveryServicesVault{
 					RecoverSoftDeletedBackupProtectedVM: true,
 				},
+				Storage: features.StorageFeatures{
+					DataPlaneAvailable: true,
+				},
 				Subscription: features.SubscriptionFeatures{
 					PreventCancellationOnDestroy: false,
 				},
@@ -156,6 +159,11 @@ func TestExpandFeatures(t *testing.T) {
 							"recover_soft_deleted_backup_protected_vm": true,
 						},
 					},
+					"storage": []interface{}{
+						map[string]interface{}{
+							"data_plane_available": true,
+						},
+					},
 					"subscription": []interface{}{
 						map[string]interface{}{
 							"prevent_cancellation_on_destroy": true,
@@ -235,6 +243,9 @@ func TestExpandFeatures(t *testing.T) {
 				RecoveryServicesVault: features.RecoveryServicesVault{
 					RecoverSoftDeletedBackupProtectedVM: true,
 				},
+				Storage: features.StorageFeatures{
+					DataPlaneAvailable: true,
+				},
 				Subscription: features.SubscriptionFeatures{
 					PreventCancellationOnDestroy: true,
 				},
@@ -331,6 +342,11 @@ func TestExpandFeatures(t *testing.T) {
 							"recover_soft_deleted_backup_protected_vm": false,
 						},
 					},
+					"storage": []interface{}{
+						map[string]interface{}{
+							"data_plane_available": false,
+						},
+					},
 					"subscription": []interface{}{
 						map[string]interface{}{
 							"prevent_cancellation_on_destroy": false,
@@ -410,6 +426,9 @@ func TestExpandFeatures(t *testing.T) {
 				RecoveryServicesVault: features.RecoveryServicesVault{
 					RecoverSoftDeletedBackupProtectedVM: false,
 				},
+				Storage: features.StorageFeatures{
+					DataPlaneAvailable: false,
+				},
 				Subscription: features.SubscriptionFeatures{
 					PreventCancellationOnDestroy: false,
 				},
@@ -1431,6 +1450,54 @@ func TestExpandFeaturesManagedDisk(t *testing.T) {
 	}
 }
 
+func TestExpandFeaturesStorage(t *testing.T) {
+	testData := []struct {
+		Name     string
+		Input    []interface{}
+		EnvVars  map[string]interface{}
+		Expected features.UserFeatures
+	}{
+		{
+			Name: "Empty Block",
+			Input: []interface{}{
+				map[string]interface{}{
+					"storage": []interface{}{},
+				},
+			},
+			Expected: features.UserFeatures{
+				Storage: features.StorageFeatures{
+					DataPlaneAvailable: true,
+				},
+			},
+		},
+		{
+			Name: "Storage Data Plane on Create is Disabled",
+			Input: []interface{}{
+				map[string]interface{}{
+					"storage": []interface{}{
+						map[string]interface{}{
+							"data_plane_available": false,
+						},
+					},
+				},
+			},
+			Expected: features.UserFeatures{
+				Storage: features.StorageFeatures{
+					DataPlaneAvailable: false,
+				},
+			},
+		},
+	}
+
+	for _, testCase := range testData {
+		t.Logf("[DEBUG] Test Case: %q", testCase.Name)
+		result := expandFeatures(testCase.Input)
+		if !reflect.DeepEqual(result.Storage, testCase.Expected.Storage) {
+			t.Fatalf("Expected %+v but got %+v", result.Storage, testCase.Expected.Storage)
+		}
+	}
+}
+
 func TestExpandFeaturesSubscription(t *testing.T) {
 	testData := []struct {
 		Name     string

internal/provider/framework/config.go

@@ -409,6 +409,19 @@ func (p *ProviderConfig) Load(ctx context.Context, data *ProviderModel, tfVersio
 			f.ManagedDisk.ExpandWithoutDowntime = true
 		}
 
+		if !features.Storage.IsNull() && !features.Storage.IsUnknown() {
+			var feature []Storage
+			d := features.Storage.ElementsAs(ctx, &feature, true)
+			diags.Append(d...)
+			if diags.HasError() {
+				return
+			}
+			f.Storage.DataPlaneAvailable = true
+			if !feature[0].DataPlaneAvailable.IsNull() && !feature[0].DataPlaneAvailable.IsUnknown() {
+				f.Storage.DataPlaneAvailable = feature[0].DataPlaneAvailable.ValueBool()
+			}
+		}
+
 		if !features.Subscription.IsNull() && !features.Subscription.IsUnknown() {
 			var feature []Subscription
 			d := features.Subscription.ElementsAs(ctx, &feature, true)