Skip to content

Commit

Permalink
[DataFacotry]Added support copyComputeScale And pipelineExternalCompu…
Browse files Browse the repository at this point in the history 
…teScale in Set-AzDataFactoryV2IntegrationRuntime Command (#23321)

* [DataFacotry]Added support copyComputeScale And pipelineExternalComputeScale in Set-AzDataFactoryV2IntegrationRuntime Command

* fix

* change

* fix

* Update ChangeLog.md

---------

Co-authored-by: Yabo Hu <[email protected]>
  • Loading branch information
@Jingshu918 @VeryEarly
Jingshu918 and VeryEarly committed Nov 10, 2023
1 parent 8226ac0 commit 8289620
Show file tree
Hide file tree
Showing 6 changed files with 242 additions and 5 deletions.
1 change: 1 addition & 0 deletions src/DataFactory/DataFactoryV2/ChangeLog.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
- Additional information about change #1
-->
## Upcoming Release
* Added support copyComputeScale And pipelineExternalComputeScale in `Set-AzDataFactoryV2IntegrationRuntime` Command
* Added ParquetReadSettings in ADF
* Fixed minor issues

Expand Down
12 changes: 12 additions & 0 deletions src/DataFactory/DataFactoryV2/Constants.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -156,6 +156,18 @@ internal static class Constants

public const string HelpIntegrationRuntimeDataFlowTimeToLive = "Time to live (in minutes) setting of the data flow cluster which will execute data flow job.";

public const string HelpIntegrationRuntimeManagedVNetCopyComputeScaleDataIntegrationUnit = "DIU number setting reserved for copy activity execution.";

public const string HelpIntegrationRuntimeManagedVNetCopyComputeScaleTimeToLive = "Time to live (in minutes) setting of integration runtime which will execute copy activity.";

public const string HelpIntegrationRuntimeManagedVNetPipelineExternalComputeScaleTimeToLive = "Time to live (in minutes) setting of integration runtime which will execute pipeline and external activity.";

public const string HelpIntegrationRuntimeManagedVNetNumberOfPipelineNodes = "Number of the pipeline nodes, which should be greater than 0 and less than 11.";

public const string HelpIntegrationRuntimeManagedVNetNumberOfExternalNodes = "Number of the the external nodes, which should be greater than 0 and less than 11.";

public const string HelpIntegrationRuntimeSelfContainedInteractiveAuthoringEnabled = "An alternative option to ensure interactive authoring function when your self-hosted integration runtime is unable to establish a connection with Azure Relay.";

public const string HelpIntegrationRuntimeSetupScriptContainerSasUri = "The SAS URI of the Azure blob container that contains the custom setup script.";

public const string HelpIntegrationRuntimeEdition = "The edition for SSIS integration runtime which could be Standard or Enterprise, default is Standard if it is not specified.";
Expand Down
132 changes: 127 additions & 5 deletions ...Factory/DataFactoryV2/IntegrationRuntimes/SetAzureDataFactoryIntegrationRuntimeCommand.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@

namespace Microsoft.Azure.Commands.DataFactoryV2
{
[Cmdlet("Set", ResourceManager.Common.AzureRMConstants.AzureRMPrefix + "DataFactoryV2IntegrationRuntime",DefaultParameterSetName = ParameterSetNames.ByIntegrationRuntimeName,SupportsShouldProcess = true),OutputType(typeof(PSIntegrationRuntime))]
[Cmdlet("Set", ResourceManager.Common.AzureRMConstants.AzureRMPrefix + "DataFactoryV2IntegrationRuntime", DefaultParameterSetName = ParameterSetNames.ByIntegrationRuntimeName, SupportsShouldProcess = true), OutputType(typeof(PSIntegrationRuntime))]
public class SetAzureDataFactoryIntegrationRuntimeCommand : IntegrationRuntimeCmdlet
{
[Parameter(
Expand Down Expand Up @@ -286,8 +286,8 @@ public class SetAzureDataFactoryIntegrationRuntimeCommand : IntegrationRuntimeCm
ParameterSetName = ParameterSetNames.ByIntegrationRuntimeObject,
Mandatory = false,
HelpMessage = Constants.HelpIntegrationRuntimeDataFlowComputeType)]
[PSArgumentCompleter(Management.DataFactory.Models.DataFlowComputeType.General,
Management.DataFactory.Models.DataFlowComputeType.MemoryOptimized,
[PSArgumentCompleter(Management.DataFactory.Models.DataFlowComputeType.General,
Management.DataFactory.Models.DataFlowComputeType.MemoryOptimized,
Management.DataFactory.Models.DataFlowComputeType.ComputeOptimized)]
[ValidateNotNullOrEmpty]
public string DataFlowComputeType { get; set; }
Expand Down Expand Up @@ -334,6 +334,90 @@ public class SetAzureDataFactoryIntegrationRuntimeCommand : IntegrationRuntimeCm
HelpMessage = Constants.HelpIntegrationRuntimeDataFlowTimeToLive)]
public int? DataFlowTimeToLive { get; set; }

[Parameter(
ParameterSetName = ParameterSetNames.ByIntegrationRuntimeName,
Mandatory = false,
HelpMessage = Constants.HelpIntegrationRuntimeManagedVNetCopyComputeScaleDataIntegrationUnit)]
[Parameter(
ParameterSetName = ParameterSetNames.ByResourceId,
Mandatory = false,
HelpMessage = Constants.HelpIntegrationRuntimeManagedVNetCopyComputeScaleDataIntegrationUnit)]
[Parameter(
ParameterSetName = ParameterSetNames.ByIntegrationRuntimeObject,
Mandatory = false,
HelpMessage = Constants.HelpIntegrationRuntimeManagedVNetCopyComputeScaleDataIntegrationUnit)]
public int? ManagedVNetCopyComputeScaleDataIntegrationUnit { get; set; }

[Parameter(
ParameterSetName = ParameterSetNames.ByIntegrationRuntimeName,
Mandatory = false,
HelpMessage = Constants.HelpIntegrationRuntimeManagedVNetCopyComputeScaleTimeToLive)]
[Parameter(
ParameterSetName = ParameterSetNames.ByResourceId,
Mandatory = false,
HelpMessage = Constants.HelpIntegrationRuntimeManagedVNetCopyComputeScaleTimeToLive)]
[Parameter(
ParameterSetName = ParameterSetNames.ByIntegrationRuntimeObject,
Mandatory = false,
HelpMessage = Constants.HelpIntegrationRuntimeManagedVNetCopyComputeScaleTimeToLive)]
public int? ManagedVNetCopyComputeScaleTimeToLive { get; set; }

[Parameter(
ParameterSetName = ParameterSetNames.ByIntegrationRuntimeName,
Mandatory = false,
HelpMessage = Constants.HelpIntegrationRuntimeManagedVNetPipelineExternalComputeScaleTimeToLive)]
[Parameter(
ParameterSetName = ParameterSetNames.ByResourceId,
Mandatory = false,
HelpMessage = Constants.HelpIntegrationRuntimeManagedVNetPipelineExternalComputeScaleTimeToLive)]
[Parameter(
ParameterSetName = ParameterSetNames.ByIntegrationRuntimeObject,
Mandatory = false,
HelpMessage = Constants.HelpIntegrationRuntimeManagedVNetPipelineExternalComputeScaleTimeToLive)]
public int? ManagedVNetPipelineExternalComputeScaleTimeToLive { get; set; }

[Parameter(
ParameterSetName = ParameterSetNames.ByIntegrationRuntimeName,
Mandatory = false,
HelpMessage = Constants.HelpIntegrationRuntimeManagedVNetNumberOfPipelineNodes)]
[Parameter(
ParameterSetName = ParameterSetNames.ByResourceId,
Mandatory = false,
HelpMessage = Constants.HelpIntegrationRuntimeManagedVNetNumberOfPipelineNodes)]
[Parameter(
ParameterSetName = ParameterSetNames.ByIntegrationRuntimeObject,
Mandatory = false,
HelpMessage = Constants.HelpIntegrationRuntimeManagedVNetNumberOfPipelineNodes)]
public int? ManagedVNetNumberOfPipelineNodeCount { get; set; }

[Parameter(
ParameterSetName = ParameterSetNames.ByIntegrationRuntimeName,
Mandatory = false,
HelpMessage = Constants.HelpIntegrationRuntimeManagedVNetNumberOfExternalNodes)]
[Parameter(
ParameterSetName = ParameterSetNames.ByResourceId,
Mandatory = false,
HelpMessage = Constants.HelpIntegrationRuntimeManagedVNetNumberOfExternalNodes)]
[Parameter(
ParameterSetName = ParameterSetNames.ByIntegrationRuntimeObject,
Mandatory = false,
HelpMessage = Constants.HelpIntegrationRuntimeManagedVNetNumberOfExternalNodes)]
public int? ManagedVNetNumberOfExternalNodeCount { get; set; }

[Parameter(
ParameterSetName = ParameterSetNames.ByIntegrationRuntimeName,
Mandatory = false,
HelpMessage = Constants.HelpIntegrationRuntimeSelfContainedInteractiveAuthoringEnabled)]
[Parameter(
ParameterSetName = ParameterSetNames.ByResourceId,
Mandatory = false,
HelpMessage = Constants.HelpIntegrationRuntimeSelfContainedInteractiveAuthoringEnabled)]
[Parameter(
ParameterSetName = ParameterSetNames.ByIntegrationRuntimeObject,
Mandatory = false,
HelpMessage = Constants.HelpIntegrationRuntimeSelfContainedInteractiveAuthoringEnabled)]
public SwitchParameter SelfContainedInteractiveAuthoringEnabled { get; set; }

[Parameter(
ParameterSetName = ParameterSetNames.ByIntegrationRuntimeName,
Mandatory = false,
Expand Down Expand Up @@ -613,7 +697,10 @@ public override void ExecuteCmdlet()
var authKey = ConvertToUnsecureString(AuthKey);
selfHosted.LinkedInfo = new LinkedIntegrationRuntimeKeyAuthorization(new SecureString(authKey));
}

if (SelfContainedInteractiveAuthoringEnabled.IsPresent)
{
selfHosted.SelfContainedInteractiveAuthoringEnabled = true;
}
resource.Properties = selfHosted;
}
}
Expand All @@ -629,6 +716,10 @@ public override void ExecuteCmdlet()
if (selfHostedIr != null)
{
selfHostedIr.LinkedInfo = new LinkedIntegrationRuntimeRbacAuthorization(SharedIntegrationRuntimeResourceId);
if (SelfContainedInteractiveAuthoringEnabled.IsPresent)
{
selfHostedIr.SelfContainedInteractiveAuthoringEnabled = true;
}
}
else
{
Expand Down Expand Up @@ -790,7 +881,7 @@ private void HandleManagedIntegrationRuntime(ManagedIntegrationRuntime integrati
if (DataFlowEnableQuickReuse.IsPresent)
{
integrationRuntime.ComputeProperties.DataFlowProperties.Cleanup = false;
}
}
else
{
// setting it as null as the default value for the cleanup variable is false, and the backend endpoint treats null value as true.
Expand All @@ -799,6 +890,37 @@ private void HandleManagedIntegrationRuntime(ManagedIntegrationRuntime integrati

}

if (ManagedVNetCopyComputeScaleDataIntegrationUnit != null || ManagedVNetCopyComputeScaleTimeToLive != null)
{
if (integrationRuntime.ComputeProperties == null)
{
integrationRuntime.ComputeProperties = new IntegrationRuntimeComputeProperties();
}
if (integrationRuntime.ComputeProperties.CopyComputeScaleProperties == null)
{
integrationRuntime.ComputeProperties.CopyComputeScaleProperties = new CopyComputeScaleProperties();
}

integrationRuntime.ComputeProperties.CopyComputeScaleProperties.DataIntegrationUnit = ManagedVNetCopyComputeScaleDataIntegrationUnit ?? integrationRuntime.ComputeProperties.CopyComputeScaleProperties.DataIntegrationUnit;
integrationRuntime.ComputeProperties.CopyComputeScaleProperties.TimeToLive = ManagedVNetCopyComputeScaleTimeToLive ?? integrationRuntime.ComputeProperties.CopyComputeScaleProperties.TimeToLive;
}

if (ManagedVNetPipelineExternalComputeScaleTimeToLive != null || ManagedVNetNumberOfPipelineNodeCount != null || ManagedVNetNumberOfExternalNodeCount != null)
{
if (integrationRuntime.ComputeProperties == null)
{
integrationRuntime.ComputeProperties = new IntegrationRuntimeComputeProperties();
}
if (integrationRuntime.ComputeProperties.PipelineExternalComputeScaleProperties == null)
{
integrationRuntime.ComputeProperties.PipelineExternalComputeScaleProperties = new PipelineExternalComputeScaleProperties();
}

integrationRuntime.ComputeProperties.PipelineExternalComputeScaleProperties.TimeToLive = ManagedVNetPipelineExternalComputeScaleTimeToLive ?? integrationRuntime.ComputeProperties.PipelineExternalComputeScaleProperties.TimeToLive;
integrationRuntime.ComputeProperties.PipelineExternalComputeScaleProperties.NumberOfPipelineNodes = ManagedVNetNumberOfPipelineNodeCount ?? integrationRuntime.ComputeProperties.PipelineExternalComputeScaleProperties.NumberOfPipelineNodes;
integrationRuntime.ComputeProperties.PipelineExternalComputeScaleProperties.NumberOfExternalNodes = ManagedVNetNumberOfExternalNodeCount ?? integrationRuntime.ComputeProperties.PipelineExternalComputeScaleProperties.NumberOfExternalNodes;
}

if (PublicIPs != null)
{
if (string.IsNullOrWhiteSpace(VNetId) && string.IsNullOrWhiteSpace(SubnetId))
Expand Down
10 changes: 10 additions & 0 deletions src/DataFactory/DataFactoryV2/Models/PSManagedIntegrationRuntime.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,16 @@ public PSManagedIntegrationRuntime(

public bool? DataFlowEnableCleanUp => ManagedIntegrationRuntime.ComputeProperties?.DataFlowProperties.Cleanup;

public int? CopyComputeScaleDataIntegrationUnit => ManagedIntegrationRuntime.ComputeProperties?.CopyComputeScaleProperties?.DataIntegrationUnit;

public int? CopyComputeScaleTimeToLive => ManagedIntegrationRuntime.ComputeProperties?.CopyComputeScaleProperties?.TimeToLive;

public int? PipelineExternalComputeScaleTimeToLive => ManagedIntegrationRuntime.ComputeProperties?.PipelineExternalComputeScaleProperties?.TimeToLive;

public int? PipelineExternalComputeScaleNumberOfPipelineNodes => ManagedIntegrationRuntime.ComputeProperties?.PipelineExternalComputeScaleProperties?.NumberOfPipelineNodes;

public int? PipelineExternalComputeScaleNumberOfExternalNodes => ManagedIntegrationRuntime.ComputeProperties?.PipelineExternalComputeScaleProperties?.NumberOfExternalNodes;

public string State => ManagedIntegrationRuntime.State;

public string LicenseType => ManagedIntegrationRuntime.SsisProperties?.LicenseType;
Expand Down
2 changes: 2 additions & 0 deletions src/DataFactory/DataFactoryV2/Models/PSSelfHostedIntegrationRuntime.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,8 @@ public PSSelfHostedIntegrationRuntime(
}
}

public bool? SelfContainedInteractiveAuthoringEnabled => SelfHostedIntegrationRuntime.SelfContainedInteractiveAuthoringEnabled;

protected SelfHostedIntegrationRuntime SelfHostedIntegrationRuntime => IntegrationRuntime.Properties as SelfHostedIntegrationRuntime;
}
}
90 changes: 90 additions & 0 deletions src/DataFactory/DataFactoryV2/help/Set-AzDataFactoryV2IntegrationRuntime.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -289,6 +289,96 @@ Accept pipeline input: False
Accept wildcard characters: False
```
### -ManagedVNetCopyComputeScaleDataIntegrationUnit
DIU number setting reserved for copy activity execution.
```yaml
Type: System.Nullable`1[System.Int32]
Parameter Sets: ByIntegrationRuntimeName, ByResourceId, ByIntegrationRuntimeObject
Aliases:

Required: False
Position: Named
Default value: None
Accept pipeline input: False
Accept wildcard characters: False
```
### -ManagedVNetCopyComputeScaleTimeToLive
Time to live (in minutes) setting of integration runtime which will execute copy activity.
```yaml
Type: System.Nullable`1[System.Int32]
Parameter Sets: ByIntegrationRuntimeName, ByResourceId, ByIntegrationRuntimeObject
Aliases:

Required: False
Position: Named
Default value: None
Accept pipeline input: False
Accept wildcard characters: False
```
### -ManagedVNetPipelineExternalComputeScaleTimeToLive
Time to live (in minutes) setting of integration runtime which will execute pipeline and external activity.
```yaml
Type: System.Nullable`1[System.Int32]
Parameter Sets: ByIntegrationRuntimeName, ByResourceId, ByIntegrationRuntimeObject
Aliases:

Required: False
Position: Named
Default value: None
Accept pipeline input: False
Accept wildcard characters: False
```
### -ManagedVNetNumberOfPipelineNodeCount
Number of the pipeline nodes, which should be greater than 0 and less than 11.
```yaml
Type: System.Nullable`1[System.Int32]
Parameter Sets: ByIntegrationRuntimeName, ByResourceId, ByIntegrationRuntimeObject
Aliases:

Required: False
Position: Named
Default value: None
Accept pipeline input: False
Accept wildcard characters: False
```
### -ManagedVNetNumberOfExternalNodeCount
Number of the the external nodes, which should be greater than 0 and less than 11.
```yaml
Type: System.Nullable`1[System.Int32]
Parameter Sets: ByIntegrationRuntimeName, ByResourceId, ByIntegrationRuntimeObject
Aliases:

Required: False
Position: Named
Default value: None
Accept pipeline input: False
Accept wildcard characters: False
```
### -SelfContainedInteractiveAuthoringEnabled
An alternative option to ensure interactive authoring function when your self-hosted integration runtime is unable to establish a connection with Azure Relay.
```yaml
Type: System.Management.Automation.SwitchParameter
Parameter Sets: (All)
Aliases:

Required: False
Position: Named
Default value: None
Accept pipeline input: False
Accept wildcard characters: False
```
### -DataProxyIntegrationRuntimeName
The Self-Hosted Integration Runtime name which is used as a proxy
Expand Down

0 comments on commit 8289620

Please sign in to comment.