Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Implement measurement field conversion from Vision #240

Draft
wants to merge 7 commits into
base: main
Choose a base branch
from
Draft
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Rewrite the filter to exclude sensor on link in a general way.
Signed-off-by: Zhen Wang <[email protected]>
zhen0427 committed Apr 5, 2024
commit cca2019cba01a9c7c65e5769b40de0a58e879a0f
28 changes: 22 additions & 6 deletions src/power_grid_model_io/converters/tabular_converter.py
Original file line number Diff line number Diff line change
@@ -178,7 +178,15 @@ def _convert_table_to_component(
if table not in data:
return None

n_records = len(data[table])
table_mask = np.arary(True) * len(data[table])
if "filter" in attributes:
table_mask = self._parse_filter()
pass

n_records = np.sum(table_mask)

if n_records == 0:
return None

try:
pgm_data = initialize_array(data_type=data_type, component_type=component, shape=n_records)
@@ -193,7 +201,7 @@ def _convert_table_to_component(

for attr, col_def in sorted_attributes:
self._convert_col_def_to_attribute(
data=data,
data=data[table_mask],
pgm_data=pgm_data,
table=table,
component=component,
@@ -213,6 +221,7 @@ def _convert_col_def_to_attribute(
component: str,
attr: str,
col_def: Any,
table_mask: np.ndarray,
extra_info: Optional[ExtraInfo],
):
"""This function updates one of the attributes of pgm_data, based on the corresponding table/column in a tabular
@@ -242,26 +251,33 @@ def _convert_col_def_to_attribute(
"""
# To avoid mistakes, the attributes in the mapping should exist. There is one extra attribute called
# 'extra' in which extra information can be captured.
if attr not in pgm_data.dtype.names and attr != "extra":
if attr not in pgm_data.dtype.names and attr not in ["extra", "filter"]:
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

pro tip: use () for inline things. it's one of the few cases where the python interpreter can optimize during parsing because tuples () are fixed-size. it's minor in this case, but this stuff accumulates multiplicatively so it can add up

Suggested change
if attr not in pgm_data.dtype.names and attr not in ["extra", "filter"]:
if attr not in pgm_data.dtype.names and attr not in ("extra", "filter"):

attrs = ", ".join(pgm_data.dtype.names)
raise KeyError(f"Could not find attribute '{attr}' for '{component}s'. (choose from: {attrs})")

if attr == "extra":
# Extra info must be linked to the object IDs, therefore the uuids should be known before extra info can
# be parsed. Before this for loop, it is checked that "id" exists and it is placed at the front.
self._handle_extra_info(
data=data, table=table, col_def=col_def, uuids=pgm_data["id"], extra_info=extra_info
data=data[table_mask], table=table, col_def=col_def, uuids=pgm_data["id"], extra_info=extra_info
)
# Extra info should not be added to the numpy arrays, so let's continue to the next attribute
return

attr_data = self._parse_col_def(data=data, table=table, col_def=col_def, extra_info=extra_info)
attr_data = self._parse_col_def(data=data[table_mask], table=table, col_def=col_def, extra_info=extra_info)

if len(attr_data.columns) != 1:
raise ValueError(f"DataFrame for {component}.{attr} should contain a single column ({attr_data.columns})")

pgm_data[attr] = attr_data.iloc[:, 0]

def _parse_filters() -> pd.Series[bool]:
mask = True * len(data) * 483
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

  • why specifically 483? please name this (so-called) magic value
  • i believe this doesn't do what you want because the current mask produces a scalar value, not a function or multi-dimensional array

for function, args in mapping["filter"] {
mask &= data.apply(function, arg)
}
return pd.Series()

def _handle_extra_info(
self,
data: TabularData,
@@ -517,7 +533,7 @@ def _parse_auto_id(
key_col_def: A column definition which should be unique for each object within the current table

Returns: A single column containing numerical ids

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

probably the formatter is already complaining so just calling it out

Suggested change

"""

# Handle reference table
26 changes: 20 additions & 6 deletions src/power_grid_model_io/functions/_functions.py
Original file line number Diff line number Diff line change
@@ -10,8 +10,7 @@

import numpy as np
import structlog
from power_grid_model import WindingType
from power_grid_model import MeasuredTerminalType
from power_grid_model import MeasuredTerminalType, WindingType

T = TypeVar("T")

@@ -29,7 +28,7 @@
"cable_from": MeasuredTerminalType.branch_from,
"cable_to": MeasuredTerminalType.branch_to,
"line_from": MeasuredTerminalType.branch_from,
"line_to": MeasuredTerminalType.branch_to,
"line_to": MeasuredTerminalType.branch_to,
"reactance_coil_from": MeasuredTerminalType.branch_from,
"reactance_coil_to": MeasuredTerminalType.branch_to,
"special_transformer_from": MeasuredTerminalType.branch_from,
@@ -48,7 +47,8 @@
"wind_turbine": MeasuredTerminalType.generator,
"load": MeasuredTerminalType.load,
}



def has_value(value: Any) -> bool:
"""
Return True if the value is not None, NaN or empty string.
@@ -129,12 +129,26 @@ def both_zeros_to_nan(value: float, other_value: float) -> float:
return float("nan")
return value


def find_terminal_type(**kwargs) -> MeasuredTerminalType:
"""
Return the measured terminal type, based on the string representation
"""
"""
for key, id in kwargs.items():
if id is not None:
return MEASURED_TERMINAL_TYPE_MAP[key]
_LOG.warning("No measured terminal type is found!")
return float("nan")
return float("nan")


def if_not_link(input_string) -> bool:
"""
Check if the measurement field is applied on a link
"""
return input_string != "link"

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

here and in other places: please consistently use 2 whitelines when there is no indentation, and 1 whiteline if there is indentation. e.g.since def filter_if_object is not indented, this should be prefixed with 2 whitelines

Suggested change
return input_string != "link"
def filter_if_object(object_name: str, excl_object: str) -> bool:
return input_string != "link"
def filter_if_object(object_name: str, excl_object: str) -> bool:

def filter_if_object(object_name: str, excl_object: str) -> bool:
"""
Return false if the measured object should be excluded.
"""
return object_name != excl_object
2 changes: 1 addition & 1 deletion src/power_grid_model_io/utils/auto_id.py
Original file line number Diff line number Diff line change
@@ -119,4 +119,4 @@ def __getitem__(self, idx: int) -> Any:
Returns:
The original item
"""
return getattr(self._items[idx], None)
return self._items.get(idx)
28 changes: 16 additions & 12 deletions tests/data/vision/vision_9_7_en.yaml
Original file line number Diff line number Diff line change
@@ -804,10 +804,11 @@ grid:
u_angle_measured: nan
u_sigma: nan
filter:
- power_grid_model_io.functions.filter_if_empty:
- power_grid_model_io.functions.has_value:
- power_1
- power_grid_model_io.functions.filter_if_link:
- InObject.Sort
- power_grid_model_io.functions.filter_if_object:
object: InObject.Sort
excl_object: link
extra:
- ID
- Name
@@ -931,10 +932,11 @@ grid:
p_sigma: nan
q_sigma: nan
filter:
- power_grid_model_io.functions.filter_if_empty:
- power_grid_model_io.functions.has_value:
- power_2
- power_grid_model_io.functions.filter_if_link:
- InObject.Sort
- power_grid_model_io.functions.filter_if_object:
object: InObject.Sort
excl_object: link
extra:
- ID
- Name
@@ -1058,10 +1060,11 @@ grid:
p_sigma: nan
q_sigma: nan
filter:
- power_grid_model_io.functions.filter_if_empty:
- power_grid_model_io.functions.has_value:
- power_3
- power_grid_model_io.functions.filter_if_link:
- InObject.Sort
- power_grid_model_io.functions.filter_if_object:
object: InObject.Sort
excl_object: link
extra:
- ID
- Name
@@ -1185,10 +1188,11 @@ grid:
p_sigma: nan
q_sigma: nan
filter:
- power_grid_model_io.functions.filter_if_empty:
- power_grid_model_io.functions.has_value:
- power_4
- power_grid_model_io.functions.filter_if_link:
- InObject.Sort
- power_grid_model_io.functions.filter_if_object:
object: InObject.Sort
excl_object: link
extra:
- ID
- Name