data_preperation_for_neuralnetwoks.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "865fa9a3-7cda-4cf0-a15e-a95973331a3c",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import os\n",
    "import glob\n",
    "from functions import *\n",
    "import csv\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e0839c57",
   "metadata": {},
   "source": [
    "# 1 Folder Data loading\n",
    "The code is loading multiple CSV files from a specified folder path and converting them into pandas DataFrames."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d9dbaa46",
   "metadata": {},
   "outputs": [],
   "source": [
    "folder_path = \"data/e production=ep sanyo ep sanyo 005\" \n",
    "csv_files = glob.glob(os.path.join(folder_path, \"*.csv\"))\n",
    "\n",
    "dataframes = []\n",
    "dfarrnames = []\n",
    "\n",
    "dataframesCU = []\n",
    "dfarrnamesCU = []\n",
    "\n",
    "# Loop through the CSV files and load only those ending with \"Format01=Kreis 5-064\" and \"TBA_CU\"\n",
    "for csv_file in csv_files:\n",
    "    if \"Format01=Kreis\" in csv_file:\n",
    "        if \"TBA_Zyk\" in csv_file:\n",
    "            dfarr = pd.read_csv(csv_file, skiprows=[1])\n",
    "            dfarr['Zeit'] = dfarr['Zeit'].apply(convert_to_linear_time)\n",
    "            dfarr['Zeit'] = dfarr['Zeit'] - dfarr['Zeit'].iloc[0]\n",
    "            dfarr['Zeit'] = dfarr['Zeit'] / 3600\n",
    "            dataframes.append(dfarr)\n",
    "            dfarrnames.append(csv_file)\n",
    "        if \"TBA_CU\" in csv_file:\n",
    "            dfarr = pd.read_csv(csv_file, skiprows=[1])\n",
    "            dfarr['Zeit'] = dfarr['Zeit'].apply(convert_to_linear_time)\n",
    "            dfarr['Zeit'] = dfarr['Zeit'] - dfarr['Zeit'].iloc[0]\n",
    "            dfarr['Zeit'] = dfarr['Zeit'] / 3600\n",
    "            dataframesCU.append(dfarr)\n",
    "            dfarrnamesCU.append(csv_file)\n",
    "\n",
    "noFiles = 0\n",
    "for i, dfarr in enumerate(dataframes):\n",
    "    num_rows, num_columns = dfarr.shape\n",
    "    print(f\"DataFrame {i + 1} - Rows: {num_rows}, Columns: {num_columns}\")\n",
    "    noFiles += 1\n",
    "print(noFiles)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6afb0987",
   "metadata": {},
   "source": [
    "Check for test that data is loaded succesfully"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6de4da17",
   "metadata": {},
   "outputs": [],
   "source": [
    "num_rows, num_columns = dataframes[1].shape\n",
    "\n",
    "print(f\"Number of rows (length): {num_rows}\")\n",
    "print(f\"Number of columns (width): {num_columns}\")\n",
    "# dataframes[0].head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8115467c",
   "metadata": {},
   "source": [
    "Deleting Invalid data for `TBA_CU`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "82969f46",
   "metadata": {},
   "outputs": [],
   "source": [
    "valid_dataframes = []\n",
    "valid_dataframesCU = []\n",
    "\n",
    "for i, (df, dfCu) in enumerate(zip(dataframes, dataframesCU)):\n",
    "    # Check if the 16th column is named 'Spannung' (Python uses zero-based indexing)\n",
    "    if len(df.columns) > 15 and df.columns[15] == 'Spannung':\n",
    "        valid_dataframes.append(df)\n",
    "        valid_dataframesCU.append(dfCu)\n",
    "    else:\n",
    "        print('deleted:', i)\n",
    "\n",
    "# Update dataframes with valid dataframes\n",
    "dataframes = valid_dataframes\n",
    "dataframesCU = valid_dataframesCU\n",
    "\n",
    "noFiles = 0\n",
    "for i, dfarr in enumerate(dataframes):\n",
    "    num_rows, num_columns = dfarr.shape\n",
    "    print(f\"DataFrame {i + 1} - Rows: {num_rows}, Columns: {num_columns}\")\n",
    "    noFiles += 1\n",
    "print(noFiles)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7ce80ef2",
   "metadata": {},
   "source": [
    "Deleting Invalid data for `Cycling test`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "88f863e7",
   "metadata": {},
   "outputs": [],
   "source": [
    "valid_dataframes = []\n",
    "valid_dataframesCU = []\n",
    "\n",
    "for i, (df, dfCu) in enumerate(zip(dataframes, dataframesCU)):\n",
    "    # Check if the 16th column is named 'Spannung' (Python uses zero-based indexing)\n",
    "    if len(dfCu.columns) > 15 and dfCu.columns[15] == 'Spannung':\n",
    "        valid_dataframes.append(df)\n",
    "        valid_dataframesCU.append(dfCu)\n",
    "    else:\n",
    "        print('deleted:', i)\n",
    "\n",
    "# Update dataframes with valid dataframes\n",
    "dataframes = valid_dataframes\n",
    "dataframesCU = valid_dataframesCU\n",
    "\n",
    "noFiles = 0\n",
    "for i, dfarr in enumerate(dataframes):\n",
    "    num_rows, num_columns = dfarr.shape\n",
    "    print(f\"DataFrame {i + 1} - Rows: {num_rows}, Columns: {num_columns}\")\n",
    "    noFiles += 1\n",
    "print(noFiles)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b575364b",
   "metadata": {},
   "outputs": [],
   "source": [
    "for df in dataframes:\n",
    "    print(df.shape)\n",
    "for df in dataframesCU:\n",
    "    print(df.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5e0e2521",
   "metadata": {},
   "source": [
    "### Extracting capacity\n",
    "The code iterates through a list of DataFrames, filters rows based on specific conditions related to time and step number, and stores the filtered DataFrames in a new list `filtered_dataframes`.\n",
    "\n",
    "This process seems to be part of data preprocessing or analysis related to battery tests, where the code is extracting capacity test data within a certain time frame after the battery is fully charged.\n",
    "\n",
    "![Image Description](dcb03b19-5872-479c-b955-75536c9e13a7.png)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2b827506",
   "metadata": {},
   "outputs": [],
   "source": [
    "filtered_dataframes = []\n",
    "\n",
    "for df in dataframesCU:\n",
    "    schritt_mask = ((df['Schritt'] == 4) | (df['Schritt'] == 5)) & (df['Zeit'] >= 10)\n",
    "    start_index = df[schritt_mask]\n",
    "    start_time = start_index.iloc[0]['Zeit']\n",
    "    end_time = start_time + 4\n",
    "    print(start_index.iloc[0]['Zeit'])\n",
    "\n",
    "    time_mask = (df['Zeit'] >= start_time) & (df['Zeit'] <= end_time) & ((df['Schritt'] == 4) | (df['Schritt'] == 5))\n",
    "\n",
    "    filtered_df = df[time_mask]\n",
    "    filtered_dataframes.append(filtered_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "25492f71",
   "metadata": {},
   "source": [
    "The code iterates through each DataFrame (`df`) in the `filtered_dataframes`. For each DataFrame, it extracts `maximum` and `minimum` values based on specified columns using the functions `max_threshold()` and `min_threshold()`. \n",
    "\n",
    "For each DataFrame, the maximum and minimum values are appended to `max_values_list` and `min_values_list` respectively. The capacity, calculated as the difference between the maximum and minimum values, is stored in `capacity_values_list`.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ce443081",
   "metadata": {},
   "outputs": [],
   "source": [
    "columns_to_extract = [9]\n",
    "\n",
    "max_values_list = []\n",
    "min_values_list = []\n",
    "capacity_values_list = []\n",
    "iteration_range = range(len(dataframes))\n",
    "\n",
    "for df in filtered_dataframes:\n",
    "    max_values = max_threshold(columns_to_extract, df, 0, 35)\n",
    "    min_values = min_threshold(columns_to_extract, df, 0, 35)\n",
    "    \n",
    "    max_values_list .append(max_values)\n",
    "    min_values_list.append(min_values)\n",
    "    capacity = max_values[0]-min_values[0]\n",
    "    capacity_values_list.append(capacity)\n",
    "\n",
    "print(max_values_list)\n",
    "print(min_values_list)\n",
    "print(capacity_values_list)\n",
    "print(iteration_range)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cfa9dd91",
   "metadata": {},
   "outputs": [],
   "source": [
    "skip_indices = []\n",
    "\n",
    "extracted_capacity_values = [value for i, value in enumerate(capacity_values_list) if i not in skip_indices]\n",
    "\n",
    "iteration_range = range(len(extracted_capacity_values))\n",
    "\n",
    "print(extracted_capacity_values)\n",
    "print(iteration_range)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "098649cc",
   "metadata": {},
   "source": [
    "### Crop The Values"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ea765d20",
   "metadata": {},
   "source": [
    "\n",
    "This script iterates through a list of dataframes and performs the following operations:\n",
    "\n",
    "1. Seprate the data farame of each cycle or `Zyklus`.\n",
    "2. Extracts a subset of the filtered dataframe from `3.65-3.85`.\n",
    "3. Filters the subset based on the `Spannung` column values.\n",
    "4. Calculates the maximum and minimum values of `AhAkku` column in the filtered dataframe.\n",
    "5. Interpolates 50 values for `AhAkku` column.\n",
    "6. Writes the interpolated values along with the difference between maximum and minimum `AhAkku` values to a CSV file.\n",
    "\n",
    "CSV File Format:\n",
    "- Each row corresponds to a set of interpolated values along with the difference between max and min AhAkku values.\n",
    "- The columns represent the interpolated values (Interpolated_1 to 'Interpolated_50') followed by `Max_Min_Ahakku_Difference`.\n",
    "\n",
    "Note: Ensure that the 'dataframes' variable is a list containing pandas DataFrames.\n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a285f2e3",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('temptest.csv', 'w', newline='') as csvfile:\n",
    "    writer = csv.writer(csvfile)\n",
    "    # ['Interpolated_1', 'Interpolated_2', ..., 'Interpolated_50', 'Max_Min_Ahakku_Difference']\n",
    "\n",
    "    I = 0\n",
    "    for i, df in enumerate(dataframes):\n",
    "        filtered_df = df[(df['Zyklus'] == 1)]\n",
    "        quarter_len = len(filtered_df) // 4\n",
    "        filtered_df = filtered_df.iloc[quarter_len:]\n",
    "\n",
    "        filtered_df_ahakku = filtered_df[(filtered_df['Spannung'] >= 3.65) & (filtered_df['Spannung'] <= 3.85)]\n",
    "\n",
    "        ahakku_values = filtered_df_ahakku['AhAkku'].values\n",
    "        \n",
    "        if len(ahakku_values) > 0:\n",
    "            # Interpolate 50 values\n",
    "            interpolated_values = np.interp(np.linspace(0, len(ahakku_values) - 1, 50), np.arange(len(ahakku_values)), ahakku_values)\n",
    "\n",
    "            writer.writerow(list(interpolated_values) + [extracted_capacity_values[I]])\n",
    "\n",
    "        I += 1"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}