sdk/python/resources/compute/attach_manage_spark_pools.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Attach and manage a Synapse Spark pool in Azure Machine Learning (preview)\n",
    "Please see documentation page: [Attach and manage a Synapse Spark pool in Azure Machine Learning (preview)](https://learn.microsoft.com/azure/machine-learning/how-to-manage-synapse-spark-pool) for more details related to the code samples in this notebook."
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Attach a Synapse Spark pool"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import required libraries\n",
    "from azure.ai.ml import MLClient\n",
    "from azure.ai.ml.entities import SynapseSparkCompute\n",
    "from azure.identity import DefaultAzureCredential\n",
    "\n",
    "subscription_id = \"<SUBSCRIPTION_ID>\"\n",
    "resource_group = \"<RESOURCE_GROUP>\"\n",
    "workspace_name = \"<AML_WORKSPACE_NAME>\"\n",
    "synapse_workspace_name = \"<SYNAPSE_WORKSPACE_NAME>\"\n",
    "\n",
    "ml_client = MLClient(\n",
    "    DefaultAzureCredential(), subscription_id, resource_group, workspace_name\n",
    ")\n",
    "\n",
    "synapse_name = \"<ATTACHED_SPARK_POOL_NAME>\"\n",
    "spark_pool_name = \"<SPARK_POOL_NAME>\"\n",
    "\n",
    "synapse_resource = f\"/subscriptions/{subscription_id}/resourceGroups/{resource_group}/providers/Microsoft.Synapse/workspaces/{synapse_workspace_name}/bigDataPools/{spark_pool_name}\"\n",
    "\n",
    "synapse_comp = SynapseSparkCompute(name=synapse_name, resource_id=synapse_resource)\n",
    "ml_client.begin_create_or_update(synapse_comp).result()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Attach a Synapse Spark pool with system-assigned identity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import required libraries\n",
    "from azure.ai.ml import MLClient\n",
    "from azure.ai.ml.entities import SynapseSparkCompute, IdentityConfiguration\n",
    "from azure.identity import DefaultAzureCredential\n",
    "\n",
    "subscription_id = \"<SUBSCRIPTION_ID>\"\n",
    "resource_group = \"<RESOURCE_GROUP>\"\n",
    "workspace_name = \"<AML_WORKSPACE_NAME>\"\n",
    "synapse_workspace_name = \"<SYNAPSE_WORKSPACE_NAME>\"\n",
    "\n",
    "ml_client = MLClient(\n",
    "    DefaultAzureCredential(), subscription_id, resource_group, workspace_name\n",
    ")\n",
    "\n",
    "synapse_name = \"<ATTACHED_SPARK_POOL_NAME>\"\n",
    "spark_pool_name = \"<SPARK_POOL_NAME>\"\n",
    "\n",
    "synapse_resource = f\"/subscriptions/{subscription_id}/resourceGroups/{resource_group}/providers/Microsoft.Synapse/workspaces/{synapse_workspace_name}/bigDataPools/{spark_pool_name}\"\n",
    "synapse_identity = IdentityConfiguration(type=\"SystemAssigned\")\n",
    "\n",
    "synapse_comp = SynapseSparkCompute(\n",
    "    name=synapse_name, resource_id=synapse_resource, identity=synapse_identity\n",
    ")\n",
    "ml_client.begin_create_or_update(synapse_comp).result()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Attach a Synapse Spark pool with user-assigned identity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import required libraries\n",
    "from azure.ai.ml import MLClient\n",
    "from azure.ai.ml.entities import (\n",
    "    SynapseSparkCompute,\n",
    "    IdentityConfiguration,\n",
    "    ManagedIdentityConfiguration,\n",
    ")\n",
    "from azure.identity import DefaultAzureCredential\n",
    "\n",
    "subscription_id = \"<SUBSCRIPTION_ID>\"\n",
    "resource_group = \"<RESOURCE_GROUP>\"\n",
    "workspace_name = \"<AML_WORKSPACE_NAME>\"\n",
    "synapse_workspace_name = \"<SYNAPSE_WORKSPACE_NAME>\"\n",
    "\n",
    "# Define name of user-assigned managed identity\n",
    "user_managed_id = \"<AML_USER_MANAGED_ID>\"\n",
    "# Define Client ID for user-assigned managed identity\n",
    "user_managed_client_id = \"<USER_ASSIGNED_IDENTITY_CLIENT_ID>\"\n",
    "\n",
    "ml_client = MLClient(\n",
    "    DefaultAzureCredential(), subscription_id, resource_group, workspace_name\n",
    ")\n",
    "\n",
    "synapse_name = \"<ATTACHED_SPARK_POOL_NAME_UAI>\"\n",
    "spark_pool_name = \"<SPARK_POOL_NAME>\"\n",
    "synapse_resource = f\"/subscriptions/{subscription_id}/resourceGroups/{resource_group}/providers/Microsoft.Synapse/workspaces/{synapse_workspace_name}/bigDataPools/{spark_pool_name}\"\n",
    "synapse_identity = IdentityConfiguration(\n",
    "    type=\"UserAssigned\",\n",
    "    user_assigned_identities=[\n",
    "        ManagedIdentityConfiguration(\n",
    "            client_id=f\"{user_managed_client_id}\",\n",
    "            resource_id=f\"/subscriptions/{subscription_id}/resourceGroups/{resource_group}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{user_managed_id}\",\n",
    "        )\n",
    "    ],\n",
    ")\n",
    "\n",
    "synapse_comp = SynapseSparkCompute(\n",
    "    name=synapse_name, resource_id=synapse_resource, identity=synapse_identity\n",
    ")\n",
    "ml_client.begin_create_or_update(synapse_comp).result()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Detach an attached Synapse Spark pool"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import required libraries\n",
    "from azure.ai.ml import MLClient\n",
    "from azure.ai.ml.entities import SynapseSparkCompute\n",
    "from azure.identity import DefaultAzureCredential\n",
    "\n",
    "subscription_id = \"<SUBSCRIPTION_ID>\"\n",
    "resource_group = \"<RESOURCE_GROUP>\"\n",
    "workspace_name = \"<AML_WORKSPACE_NAME>\"\n",
    "\n",
    "ml_client = MLClient(\n",
    "    DefaultAzureCredential(), subscription_id, resource_group, workspace_name\n",
    ")\n",
    "\n",
    "synapse_name = \"<ATTACHED_SPARK_POOL_NAME>\"\n",
    "ml_client.compute.begin_delete(name=synapse_name, action=\"Detach\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.10 - SDK V2",
   "language": "python",
   "name": "python310-sdkv2"
  },
  "language_info": {
   "name": "python",
   "version": "3.7.10"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "6aeff17a1aa7735c2f7cb3a6d691fe1b4d4c3b8d2d650f644ad0f24e1b8e3f3f"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}