diff --git a/.gitignore b/.gitignore index 78683a6..f3e8265 100644 --- a/.gitignore +++ b/.gitignore @@ -400,3 +400,4 @@ FodyWeavers.xsd output **/_trial_temp-[0-9]/ **/_trial_temp-[0-9].lock +database diff --git a/APSToolkitPython/src/aps_toolkit/PropReader.py b/APSToolkitPython/src/aps_toolkit/PropReader.py index dc081be..3dcd0d5 100644 --- a/APSToolkitPython/src/aps_toolkit/PropReader.py +++ b/APSToolkitPython/src/aps_toolkit/PropReader.py @@ -21,6 +21,8 @@ import requests from .Derivative import Derivative from .ManifestItem import ManifestItem +import pandas as pd +from typing import List class PropReader: @@ -172,6 +174,55 @@ def get_internal_ref(self, id) -> list: if prop.category == "__internalref__": reference.append(int(prop.value)) + # TODO : It too slow, need find another way to get all data with less time and cover all format : dwg, rvt, nwd, ifc, ... + # def get_all_data(self) -> pd.DataFrame: + # db_index_ids = [i for i in range(len(self.offsets))] + # return self.get_recursive_ids(db_index_ids) + + def get_recursive_ids(self, db_ids: List[int]) -> pd.DataFrame: + dataframe = pd.DataFrame() + props_ignore = ['parent', 'instanceof_objid', 'child', "viewable_in"] + if len(db_ids) == 0: + return dataframe + for id in db_ids: + props = self.enumerate_properties(id) + properties = {} + for prop in props: + if prop.name not in props_ignore: + properties[prop.name] = prop.value + db_id = id + external_id = self.ids[id] + properties['dbId'] = db_id + properties['external_id'] = external_id + ins = self.get_instance(id) + if len(ins) > 0: + for instance in ins: + types = self.get_properties(instance) + properties = {**properties, **types} + singleDF = pd.DataFrame(properties, index=[0]) + dataframe = pd.concat([dataframe, singleDF], ignore_index=True) + ids = self.get_children(id) + dataframe = pd.concat([dataframe, self.get_recursive_ids(ids)], ignore_index=True) + if 'dbId' in dataframe.columns and 'external_id' in dataframe.columns: + dataframe = dataframe[ + ['dbId', 'external_id'] + [col for col in dataframe.columns if col not in ['dbId', 'external_id']]] + return dataframe + + def read_all_properties_name(self) -> List[str]: + props_names = [] + for i in range(len(self.offsets)): + av_start = 2 * self.offsets[i] + av_end = len(self.avs) if i == len(self.offsets) - 1 else 2 * self.offsets[i + 1] + for j in range(av_start, av_end, 2): + attr_offset = self.avs[j] + attr_obj = self.attrs[attr_offset] + if isinstance(attr_obj, list) and len(attr_obj) >= 2: + name = attr_obj[0] + props_names.append(name) + props_names = list(set(props_names)) + props_names.sort() + return props_names + class Property(): def __init__(self, id=None, name=None, category=None, data_type=None, data_type_context=None, description=None, diff --git a/APSToolkitPython/src/test/test_prop_reader.py b/APSToolkitPython/src/test/test_prop_reader.py index f42702f..30882e7 100644 --- a/APSToolkitPython/src/test/test_prop_reader.py +++ b/APSToolkitPython/src/test/test_prop_reader.py @@ -6,16 +6,24 @@ class TestPropDbReader(TestCase): def setUp(self): - self.urn = "dXJuOmFkc2sud2lwcHJvZDpmcy5maWxlOnZmLlhUOFFRSk53UXhpTFE2VE1QbmZRTkE_dmVyc2lvbj0x" + self.urn = "dXJuOmFkc2sud2lwcHJvZDpmcy5maWxlOnZmLm5KaEpjQkQ1UXd1bjlIV1ktNWViQmc_dmVyc2lvbj0x" self.token = Auth().auth2leg() self.prop_reader = PropReader(self.urn, self.token) def test_enumerate_properties(self): - properties = self.prop_reader.enumerate_properties(1) + properties = self.prop_reader.enumerate_properties(14) self.assertNotEquals(properties, 0) + def test_get_recursive_ids(self): + ids = self.prop_reader.get_recursive_ids([14, 15]) + self.assertNotEquals(len(ids), 0) + + def test_read_all_properties_name(self): + properties = self.prop_reader.read_all_properties_name() + self.assertNotEquals(len(properties), 0) + def test_get_properties(self): - properties = self.prop_reader.get_properties(1) + properties = self.prop_reader.get_properties(14) self.assertNotEquals(properties, 0) def test_get_all_properties(self): diff --git a/APSToolkitPython/src/test/test_svf_reader.py b/APSToolkitPython/src/test/test_svf_reader.py index 302e39f..97e97ed 100644 --- a/APSToolkitPython/src/test/test_svf_reader.py +++ b/APSToolkitPython/src/test/test_svf_reader.py @@ -7,12 +7,12 @@ class TestSVFReader(TestCase): def setUp(self): - self.urn = "dXJuOmFkc2sud2lwcHJvZDpmcy5maWxlOnZmLk9kOHR4RGJLU1NlbFRvVmcxb2MxVkE_dmVyc2lvbj0yNA" + self.urn = "dXJuOmFkc2sud2lwcHJvZDpmcy5maWxlOnZmLm5KaEpjQkQ1UXd1bjlIV1ktNWViQmc_dmVyc2lvbj0x" self.token = Auth().auth2leg() self.reader = SVFReader(self.urn, self.token) def test_read_contents(self): - contents = self.reader._read_contents() + contents = self.reader.read_contents() self.assertTrue(len(contents) > 0) def test_read_contents_manifest(self):