Merge branch 'latest' into ea/fix_jax

notebooks/paddle-to-openvino/paddle-to-openvino-classification.ipynb

@@ -59,11 +59,9 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "import platform\n",
-    "\n",
     "%pip install -q \"paddlepaddle>=2.5.1,<2.6.0\"\n",
     "%pip install -q \"paddleclas>=2.5.2\" --no-deps\n",
-    "%pip install -q \"prettytable\" \"ujson\" \"visualdl>=2.5.3\" \"faiss-cpu>=1.7.1\" Pillow tqdm \"matplotlib>=3.4\" \"opencv-python\"\n",
+    "%pip install -q \"prettytable\" \"ujson\" \"visualdl>=2.5.3\" \"faiss-cpu>=1.7.1\" Pillow tqdm \"matplotlib>=3.4\" \"opencv-python\" \"scikit-learn\"\n",
     "# Install openvino package\n",
     "%pip install -q \"openvino>=2023.1.0\""
    ]

notebooks/text-to-image-genai/README.md

@@ -1,6 +1,10 @@
 # Text to Image pipeline and OpenVINO with Generate API
 
-OpenVINO GenAI is a new flavor of OpenVINO, aiming to simplify running inference of generative AI models. It hides the complexity of the generation process and minimizes the amount of code required. You can now provide a model and input context directly to OpenVINO, which performs tokenization of the input text, executes the generation loop on the selected device, and returns the generated results. For a quick start guide, refer to the [GenAI API Guide](https://docs.openvino.ai/2024/learn-openvino/llm_inference_guide/genai-guide.html).
+[OpenVINO™ GenAI](https://github.com/openvinotoolkit/openvino.genai) is a library of the most popular Generative AI model pipelines, optimized execution methods, and samples that run on top of highly performant OpenVINO Runtime.
+
+This library is friendly to PC and laptop execution, and optimized for resource consumption. It requires no external dependencies to run generative models as it already includes all the core functionality (e.g. tokenization via openvino-tokenizers).
+
+In this tutorial we consider how to use OpenVINO GenAI for image generation scenario.
 
 ## Notebook Contents
 

notebooks/text-to-image-genai/gradio_helper.py

@@ -1,7 +1,7 @@
 import gradio as gr
 import numpy as np
 
-import openvino_genai
+import openvino_genai as ov_genai
 
 
 MAX_SEED = np.iinfo(np.int32).max
@@ -40,7 +40,7 @@ def infer(prompt, negative_prompt, seed, randomize_seed, width, height, num_infe
             adapters=adapter_config if use_lora else openvino_genai.AdapterConfig(),
         )
 
-        return image_tensor.data[0]
+        return image_tensor.data[0], seed
 
     with gr.Blocks(css=css) as demo:
         with gr.Column(elem_id="col-container"):
@@ -112,7 +112,7 @@ def infer(prompt, negative_prompt, seed, randomize_seed, width, height, num_infe
             triggers=[run_button.click, prompt.submit, negative_prompt.submit],
             fn=infer,
             inputs=[prompt, negative_prompt, seed, randomize_seed, width, height, num_inference_steps, use_lora],
-            outputs=[result],
+            outputs=[result, seed],
         )
 
     return demo

notebooks/text-to-image-genai/text-to-image-genai.ipynb

@@ -8,7 +8,9 @@
    "source": [
     "# Text to Image pipeline and OpenVINO with Generate API\n",
     "\n",
-    "OpenVINO GenAI is a new flavor of OpenVINO, aiming to simplify running inference of generative AI models. It hides the complexity of the generation process and minimizes the amount of code required. You can now provide a model and input context directly to OpenVINO, which performs tokenization of the input text, executes the generation loop on the selected device, and returns the generated results. For a quick start guide, refer to the [GenAI API Guide](https://docs.openvino.ai/2024/learn-openvino/llm_inference_guide/genai-guide.html).\n",
+    "[OpenVINO™ GenAI](https://github.com/openvinotoolkit/openvino.genai) is a library of the most popular Generative AI model pipelines, optimized execution methods, and samples that run on top of highly performant OpenVINO Runtime.\n",
+    "\n",
+    "This library is friendly to PC and laptop execution, and optimized for resource consumption. It requires no external dependencies to run generative models as it already includes all the core functionality (e.g. tokenization via openvino-tokenizers).\n",
     "\n",
     "In this notebook we will demonstrate how to use text to image models like Stable Diffusion 1.5, 2.1, LCM using [Dreamlike Anime 1.0](https://huggingface.co/dreamlike-art/dreamlike-anime-1.0) as an example. All it takes is two steps: \n",
     "1. Export OpenVINO IR format model using the [Hugging Face Optimum](https://huggingface.co/docs/optimum/installation) library accelerated by OpenVINO integration.\n",
@@ -90,7 +92,7 @@
     "optimum-cli export openvino --model <model_id_or_path> --task <task> <out_dir>\n",
     "```\n",
     "\n",
-    "where `--model` argument is model id from HuggingFace Hub or local directory with model (saved using `.save_pretrained` method), `--task ` is one of [supported task](https://huggingface.co/docs/optimum/exporters/task_manager) that exported model should solve. For LLMs it will be `text-generation-with-past`. If model initialization requires to use remote code, `--trust-remote-code` flag additionally should be passed.\n",
+    "where `--model` argument is model id from HuggingFace Hub or local directory with model (saved using `.save_pretrained` method), `--task ` is one of [supported task](https://huggingface.co/docs/optimum/exporters/task_manager) that exported model should solve. For image generation models, `text-to-image` should be used. If model initialization requires to use remote code, `--trust-remote-code` flag additionally should be passed.\n",
     "You can also apply fp16, 8-bit or 4-bit weight compression on the Linear, Convolutional and Embedding layers when exporting your model with the CLI by setting `--weight-format` to respectively fp16, int8 or int4. This type of optimization allows to reduce the memory footprint and inference latency.\n",
     "\n",
     "We will use `optimum_cli` from our helper `cmd_helper.py` that is a wrapper over cli-command."
@@ -172,24 +174,27 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "import openvino_genai\n",
+    "import openvino_genai as ov_genai\n",
+    "import openvino as ov\n",
     "from PIL import Image\n",
-    "import numpy as np\n",
+    "import torch\n",
     "\n",
     "\n",
-    "class Generator(openvino_genai.Generator):\n",
-    "    def __init__(self, seed, mu=0.0, sigma=1.0):\n",
-    "        openvino_genai.Generator.__init__(self)\n",
-    "        np.random.seed(seed)\n",
-    "        self.mu = mu\n",
-    "        self.sigma = sigma\n",
+    "class Generator(ov_genai.Generator):\n",
+    "    def __init__(self, seed):\n",
+    "        ov_genai.Generator.__init__(self)\n",
+    "        self.generator = torch.Generator(device=\"cpu\").manual_seed(seed)\n",
     "\n",
     "    def next(self):\n",
-    "        return np.random.normal(self.mu, self.sigma)\n",
+    "        return torch.randn(1, generator=self.generator, dtype=torch.float32).item()\n",
+    "\n",
+    "    def randn_tensor(self, shape: ov.Shape):\n",
+    "        torch_tensor = torch.randn(list(shape), generator=self.generator, dtype=torch.float32)\n",
+    "        return ov.Tensor(torch_tensor.numpy())\n",
     "\n",
     "\n",
     "random_generator = Generator(42)  # openvino_genai.CppStdGenerator can be used to have same images as C++ sample\n",
-    "pipe = openvino_genai.Text2ImagePipeline(model_dir, device.value)\n",
+    "pipe = ov_genai.Text2ImagePipeline(model_dir, device.value)\n",
     "prompt = \"anime, masterpiece, high quality, a green snowman with a happy smiling face in the snows\"\n",
     "\n",
     "image_tensor = pipe.generate(prompt, width=512, height=512, num_inference_steps=20, num_images_per_prompt=1, generator=random_generator)\n",
@@ -256,11 +261,11 @@
    "outputs": [],
    "source": [
     "def prepare_adapter_config(adapters):\n",
-    "    adapter_config = openvino_genai.AdapterConfig()\n",
+    "    adapter_config = ov_genai.AdapterConfig()\n",
     "\n",
     "    # Multiple LoRA adapters applied simultaneously are supported, parse them all and corresponding alphas from cmd parameters:\n",
     "    for i in range(int(len(adapters) / 2)):\n",
-    "        adapter = openvino_genai.Adapter(adapters[2 * i])\n",
+    "        adapter = ov_genai.Adapter(adapters[2 * i])\n",
     "        alpha = float(adapters[2 * i + 1])\n",
     "        adapter_config.add(adapter, alpha)\n",
     "\n",
@@ -269,7 +274,7 @@
     "\n",
     "adapter_config = prepare_adapter_config([\"soulcard.safetensors\", 0.5])\n",
     "\n",
-    "pipe = openvino_genai.Text2ImagePipeline(model_dir, device.value, adapters=adapter_config)\n",
+    "pipe = ov_genai.Text2ImagePipeline(model_dir, device.value, adapters=adapter_config)\n",
     "\n",
     "image_tensor = pipe.generate(prompt, generator=Generator(42), width=512, height=512, num_inference_steps=20)\n",
     "image = Image.fromarray(image_tensor.data[0])"
@@ -298,6 +303,15 @@
     "image"
    ]
   },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "id": "3b5da375",
+   "metadata": {},
+   "source": [
+    "You can find more detailed tutorial for running inference with multiple LoRA adapters in this [notebook](../multilora-image-generation/multilora-image-generation.ipynb)"
+   ]
+  },
   {
    "attachments": {},
    "cell_type": "markdown",
@@ -352,10 +366,13 @@
    "imageUrl": "https://github.com/user-attachments/assets/eb9f17ac-3a4b-459f-988d-c29b25b6a220",
    "tags": {
     "categories": [
-     "Model Demos"
+     "Model Demos",
+     "AI Trends"
     ],
     "libraries": [],
-    "other": [],
+    "other": [
+     "Stable Diffusion"
+    ],
     "tasks": [
      "Text-to-Image"
     ]