From ff4f3913ea43daf95ea8c102ad143677522c8d4a Mon Sep 17 00:00:00 2001 From: Daniel Harding Date: Wed, 12 May 2021 14:56:38 +0300 Subject: [PATCH] Make TokenList.value a property not an attribute. The fact that a new value was being computed each time TokenList.group_tokens() was called caused supra-linear runtime when token grouping was enabled. Address by making TokenList.value a dynamically-computed property rather than a static attribute. --- sqlparse/sql.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/sqlparse/sql.py b/sqlparse/sql.py index 7afcc8df..1d38a8e7 100644 --- a/sqlparse/sql.py +++ b/sqlparse/sql.py @@ -48,6 +48,9 @@ class TokenBase: def __init__(self): self.parent = None + def __str__(self): + return self.value + # Pending tokenlist __len__ bug fix # def __len__(self): # return len(self.value) @@ -151,9 +154,6 @@ def __init__(self, ttype, value): self.is_whitespace = ttype in T.Whitespace self.normalized = value.upper() if self.is_keyword else value - def __str__(self): - return self.value - def _get_repr_name(self): return str(self.ttype).split('.')[-1] @@ -165,11 +165,11 @@ def flatten(self): class TokenList(TokenBase): """A group of tokens. - It has two additional instance attributes, ``value``, which is the value of - the token list, and ``tokens``, which holds a list of child-tokens. + It has an additional instance attribute ``tokens`` which holds a + list of child-tokens. """ - __slots__ = ('tokens', 'value') + __slots__ = 'tokens' is_group = True ttype = None @@ -179,10 +179,10 @@ class TokenList(TokenBase): def __init__(self, tokens=None): super().__init__() self.tokens = tokens or [] - self.value = str(self) [setattr(token, 'parent', self) for token in self.tokens] - def __str__(self): + @property + def value(self): return ''.join(token.value for token in self.flatten()) @property @@ -347,7 +347,6 @@ def group_tokens(self, grp_cls, start, end, include_end=True, grp = start grp.tokens.extend(subtokens) del self.tokens[start_idx + 1:end_idx] - grp.value = str(start) else: subtokens = self.tokens[start_idx:end_idx] grp = grp_cls(subtokens)