From bf5251d2858a330bad3480bfd7602264ef1ef11e Mon Sep 17 00:00:00 2001 From: Dmytro Parfeniuk Date: Thu, 12 Sep 2024 13:05:37 +0300 Subject: [PATCH 1/3] =?UTF-8?q?=F0=9F=9A=A7=20WIP?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/guidellm/backend/base.py | 1 + src/guidellm/backend/openai.py | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/src/guidellm/backend/base.py b/src/guidellm/backend/base.py index d71c5f6..8a0f1fe 100644 --- a/src/guidellm/backend/base.py +++ b/src/guidellm/backend/base.py @@ -309,6 +309,7 @@ def _cachable_default_model(backend: Backend) -> str: :rtype: str :raises ValueError: If no models are available. """ + logger.debug("Getting default model for backend: {}", backend) models = backend.available_models() if models: diff --git a/src/guidellm/backend/openai.py b/src/guidellm/backend/openai.py index 8a49d5e..a2c9d95 100644 --- a/src/guidellm/backend/openai.py +++ b/src/guidellm/backend/openai.py @@ -103,6 +103,7 @@ async def make_request( request_args.update(self._request_args) + print(">>> Creating stream object for OpenAI server ") stream = await self._async_client.chat.completions.create( model=self.model, messages=[ @@ -111,8 +112,10 @@ async def make_request( stream=True, **request_args, ) + token_count = 0 async for chunk in stream: + print(f"Getting chunk: {chunk}") choice = chunk.choices[0] token = choice.delta.content or "" @@ -145,6 +148,9 @@ def available_models(self) -> List[str]: :raises openai.OpenAIError: If an error occurs while retrieving models. """ + # TODO: Remove this line + return ["Meta-Llama-3-8B.Q4_K_M.gguf"] + try: return [model.id for model in self._client.models.list().data] except Exception as error: From b8ac9c17992a98e2a220a73a2cc488fd7f4f73ca Mon Sep 17 00:00:00 2001 From: Dmytro Parfeniuk Date: Thu, 12 Sep 2024 17:12:19 +0300 Subject: [PATCH 2/3] =?UTF-8?q?=F0=9F=9A=A7=20WIP?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/guidellm/core/report.py | 66 +++++++++++++++++++++++++++++++++++-- src/guidellm/main.py | 33 ++++++++++++++----- 2 files changed, 89 insertions(+), 10 deletions(-) diff --git a/src/guidellm/core/report.py b/src/guidellm/core/report.py index b6791e4..9c823cc 100644 --- a/src/guidellm/core/report.py +++ b/src/guidellm/core/report.py @@ -1,6 +1,7 @@ import time from datetime import datetime -from typing import List, Optional +from pathlib import Path +from typing import List, Literal, Optional, Union, get_args from loguru import logger from pydantic import Field @@ -10,11 +11,14 @@ from rich.table import Table from guidellm.core.result import TextGenerationBenchmark, TextGenerationBenchmarkReport -from guidellm.core.serializable import Serializable +from guidellm.core.serializable import Serializable, SerializableFileType __all__ = ["GuidanceReport"] +FlatFileType = Literal["csv"] + + def _create_benchmark_report_details(report: TextGenerationBenchmarkReport) -> str: """ Create a detailed string representation of a benchmark report. @@ -319,3 +323,61 @@ def print( console.print(report_viz) logger.info("Guidance report printing completed.") + + def _get_data(self): + """ + Select the data from the report and return it + in a flat format to be saved to the CSV file. + """ + + raise NotImplementedError("Work in progress...") + + def save_data( + self, + path: Union[str, Path], + type_: FlatFileType = "csv", + ) -> str: + """ + Save the data to a file in CSV format. + + :param path: Path to the exact file or the containing directory. + If it is a directory, the file name will be inferred from the class name. + :param type_: Optional type to save ('csv' is only supported). + :return: The path to the saved file. + """ + + logger.debug("Saving to file... {} with format: {}", path, type_) + + if isinstance(path, str): + path = Path(path) + + if path.suffix: + # is a file + ext = path.suffix[1:].lower() + if type_ not in get_args(FlatFileType): + raise ValueError( + f"Unsupported file extension: {type_}. " + f"Expected one of {SerializableFileType} " + f"for {path}" + ) + type_ = ext # type: ignore # noqa: PGH003 + else: + # is a directory + file_name = f"{self.__class__.__name__.lower()}.{type_}" + path = path / file_name + + path.parent.mkdir(parents=True, exist_ok=True) + + with path.open("w") as file: + if type_ == "csv": + file.write(self._get_data()) + else: + raise ValueError( + f"Unsupported file extension: {type_}" + f"Expected one of {SerializableFileType} " + f"for {path}" + ) + + logger.info("Successfully saved {} to {}", self.__class__.__name__, path) + + return str(path) diff --git a/src/guidellm/main.py b/src/guidellm/main.py index c0bea83..6b9cb77 100644 --- a/src/guidellm/main.py +++ b/src/guidellm/main.py @@ -132,7 +132,7 @@ ), ) @click.option( - "--output-path", + "--output-report-path", type=str, default=None, help=( @@ -142,6 +142,16 @@ "printed to the console." ), ) +@click.option( + "--output-data-path", + type=str, + default=None, + help=( + "The output path to save flat data results. " + "Ex: --output-data-path=data.csv" + "The default is None, meaning a file won't be generated." + ), +) @click.option( "--enable-continuous-refresh", is_flag=True, @@ -162,7 +172,8 @@ def generate_benchmark_report_cli( rate: Optional[float], max_seconds: Optional[int], max_requests: Optional[int], - output_path: str, + output_report_path: str, + output_data_path: str, enable_continuous_refresh: bool, ): """ @@ -179,7 +190,8 @@ def generate_benchmark_report_cli( rate=rate, max_seconds=max_seconds, max_requests=max_requests, - output_path=output_path, + output_report_path=output_report_path, + output_data_path=output_data_path, cont_refresh_table=enable_continuous_refresh, ) @@ -195,7 +207,8 @@ def generate_benchmark_report( rate: Optional[float], max_seconds: Optional[int], max_requests: Optional[int], - output_path: str, + output_report_path: str, + output_data_path: str, cont_refresh_table: bool, ) -> GuidanceReport: """ @@ -216,6 +229,7 @@ def generate_benchmark_report( :param max_seconds: Maximum duration for each benchmark run in seconds. :param max_requests: Maximum number of requests per benchmark run. :param output_path: Path to save the output report file. + :param output_csv_path: Path to save the flat output data. :param cont_refresh_table: Continually refresh the table in the CLI until the user exits. """ @@ -224,7 +238,7 @@ def generate_benchmark_report( ) # Create backend - backend_inst = Backend.create( + backend_inst: Backend = Backend.create( backend_type=backend, target=target, model=model, @@ -284,11 +298,14 @@ def generate_benchmark_report( guidance_report = GuidanceReport() guidance_report.benchmarks.append(report) - if output_path: - guidance_report.save_file(output_path) + if output_report_path: + guidance_report.save_file(output_report_path) + + if output_data_path: + guidance_report.save_file(output_report_path) guidance_report.print( - save_path=output_path if output_path is not None else "stdout", + save_path=output_report_path if output_report_path is not None else "stdout", continual_refresh=cont_refresh_table, ) From 943d264154868f3db53ae7e4477197194ca9090f Mon Sep 17 00:00:00 2001 From: Dmytro Parfeniuk Date: Thu, 12 Sep 2024 17:16:06 +0300 Subject: [PATCH 3/3] removed unused prints --- src/guidellm/backend/openai.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/guidellm/backend/openai.py b/src/guidellm/backend/openai.py index a2c9d95..8ab872b 100644 --- a/src/guidellm/backend/openai.py +++ b/src/guidellm/backend/openai.py @@ -103,7 +103,6 @@ async def make_request( request_args.update(self._request_args) - print(">>> Creating stream object for OpenAI server ") stream = await self._async_client.chat.completions.create( model=self.model, messages=[ @@ -115,7 +114,6 @@ async def make_request( token_count = 0 async for chunk in stream: - print(f"Getting chunk: {chunk}") choice = chunk.choices[0] token = choice.delta.content or ""