Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

chore(metrics-summaries): Remove metrics summaries queries and endpoints #81133

Open
wants to merge 6 commits into
base: master
Choose a base branch
from
+75 −2,551
Open

chore(metrics-summaries): Remove metrics summaries queries and endpoints #81133

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
31df43b
chore(metrics-summaries): Remove metrics summaries queries and endpoints
phacops Nov 21, 2024
79c54e2
Remove more irrelevant code
phacops Nov 22, 2024
6c17333
Merge branch 'master' into pierre/remove-metrics-summaries-queries
phacops Nov 22, 2024
2c82b8c
:hammer_and_wrench: apply pre-commit fixes
getsantry[bot] Nov 22, 2024
6cc5cb7
Remove more irrelevant metrics samples related code
phacops Nov 22, 2024
1be01f2
Remove another mri related test
phacops Nov 22, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
107 changes: 0 additions & 107 deletions src/sentry/api/endpoints/organization_metrics_samples.py
View file
Open in desktop

This file was deleted.

196 changes: 27 additions & 169 deletions src/sentry/api/endpoints/organization_traces.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
from rest_framework.exceptions import ParseError, ValidationError
from rest_framework.request import Request
from rest_framework.response import Response
from snuba_sdk import And, BooleanCondition, BooleanOp, Column, Condition, Function, Op, Or
from snuba_sdk import BooleanCondition, BooleanOp, Column, Condition, Function, Op
from urllib3.exceptions import ReadTimeoutError

from sentry import features, options
Expand All @@ -35,7 +35,6 @@
)
from sentry.search.events.constants import TIMEOUT_SPAN_ERROR_MESSAGE
from sentry.search.events.types import QueryBuilderConfig, SnubaParams, WhereType
from sentry.sentry_metrics.querying.samples_list import SpanKey, get_sample_list_executor_cls
from sentry.snuba import discover, spans_indexed
from sentry.snuba.dataset import Dataset
from sentry.snuba.referrer import Referrer
Expand Down Expand Up @@ -311,7 +310,9 @@ def get(self, request: Request, organization: Organization) -> Response:
zerofill = not (
request.GET.get("withoutZerofill") == "1"
and features.get(
"organizations:performance-chart-interpolation", organization, actor=request.user
"organizations:performance-chart-interpolation",
organization,
actor=request.user,
)
)

Expand Down Expand Up @@ -499,76 +500,8 @@ def get_traces_matching_conditions(
self,
snuba_params: SnubaParams,
) -> tuple[datetime, datetime, list[str]]:
if self.mri is not None:
sentry_sdk.set_tag("mri", self.mri)
return self.get_traces_matching_metric_conditions(snuba_params)

return self.get_traces_matching_span_conditions(snuba_params)

def get_traces_matching_metric_conditions(
self,
snuba_params: SnubaParams,
) -> tuple[datetime, datetime, list[str]]:
assert self.mri is not None

executor_cls = get_sample_list_executor_cls(self.mri)
if executor_cls is None:
raise ParseError(detail=f"Unsupported MRI: {self.mri}")

executor = executor_cls(
mri=self.mri,
snuba_params=snuba_params,
fields=["trace"],
max=self.metrics_max,
min=self.metrics_min,
operation=self.metrics_operation,
query=self.metrics_query,
referrer=Referrer.API_TRACE_EXPLORER_METRICS_SPANS_LIST,
)

trace_ids, timestamps = executor.get_matching_traces(MAX_SNUBA_RESULTS)

min_timestamp = snuba_params.end
max_timestamp = snuba_params.start
assert min_timestamp is not None
assert max_timestamp is not None

for timestamp in timestamps:
min_timestamp = min(min_timestamp, timestamp)
max_timestamp = max(max_timestamp, timestamp)

if not trace_ids or min_timestamp > max_timestamp:
return min_timestamp, max_timestamp, []

self.refine_params(min_timestamp, max_timestamp)

if self.user_queries:
# If there are user queries, further refine the trace ids by applying them
# leaving us with only traces where the metric exists and matches the user
# queries.
(
min_timestamp,
max_timestamp,
trace_ids,
) = self.get_traces_matching_span_conditions_in_traces(snuba_params, trace_ids)

if not trace_ids:
return min_timestamp, max_timestamp, []
else:
# No user queries so take the first N trace ids as our list
min_timestamp = snuba_params.end
max_timestamp = snuba_params.start
assert min_timestamp is not None
assert max_timestamp is not None

trace_ids = trace_ids[: self.limit]
timestamps = timestamps[: self.limit]
for timestamp in timestamps:
min_timestamp = min(min_timestamp, timestamp)
max_timestamp = max(max_timestamp, timestamp)

return min_timestamp, max_timestamp, trace_ids

def get_traces_matching_span_conditions(
self,
snuba_params: SnubaParams,
Expand Down Expand Up @@ -1248,54 +1181,20 @@ def __init__(

def execute(self, offset: int, limit: int):
with handle_span_query_errors():
span_keys = self.get_metrics_span_keys()

with handle_span_query_errors():
spans = self.get_user_spans(
return self.get_user_spans(
self.snuba_params,
span_keys,
offset=offset,
limit=limit,
)

return spans

def get_metrics_span_keys(self) -> list[SpanKey] | None:
if self.mri is None:
return None

executor_cls = get_sample_list_executor_cls(self.mri)
if executor_cls is None:
raise ParseError(detail=f"Unsupported MRI: {self.mri}")

executor = executor_cls(
mri=self.mri,
snuba_params=self.snuba_params,
fields=["trace"],
max=self.metrics_max,
min=self.metrics_min,
operation=self.metrics_operation,
query=self.metrics_query,
referrer=Referrer.API_TRACE_EXPLORER_METRICS_SPANS_LIST,
)

span_keys = executor.get_matching_spans_from_traces(
[self.trace_id],
MAX_SNUBA_RESULTS,
)

return span_keys

def get_user_spans(
self,
snuba_params: SnubaParams,
span_keys: list[SpanKey] | None,
limit: int,
offset: int,
):
user_spans_query = self.get_user_spans_query(
snuba_params,
span_keys,
limit=limit,
offset=offset,
)
Expand All @@ -1313,15 +1212,14 @@ def get_user_spans(
def get_user_spans_query(
self,
snuba_params: SnubaParams,
span_keys: list[SpanKey] | None,
limit: int,
offset: int,
) -> BaseQueryBuilder:
if self.dataset == Dataset.EventsAnalyticsPlatform:
# span_keys is not supported in EAP mode because that's a legacy
# code path to support metrics that no longer exists
return self.get_user_spans_query_eap(snuba_params, limit, offset)
return self.get_user_spans_query_indexed(snuba_params, span_keys, limit, offset)
return self.get_user_spans_query_indexed(snuba_params, limit, offset)

def get_user_spans_query_eap(
self,
Expand Down Expand Up @@ -1383,7 +1281,6 @@ def get_user_spans_query_eap(
def get_user_spans_query_indexed(
self,
snuba_params: SnubaParams,
span_keys: list[SpanKey] | None,
limit: int,
offset: int,
) -> BaseQueryBuilder:
Expand Down Expand Up @@ -1411,69 +1308,30 @@ def get_user_spans_query_indexed(

conditions = []

if span_keys is None:
# Next we have to turn the user queries into the appropriate conditions in
# the SnQL that we produce.
# Next we have to turn the user queries into the appropriate conditions in
# the SnQL that we produce.

# There are multiple sets of user conditions that needs to be satisfied
# and if a span satisfy any of them, it should be considered.
#
# To handle this use case, we want to OR all the user specified
# conditions together in this query.
for where in user_conditions:
if len(where) > 1:
conditions.append(BooleanCondition(op=BooleanOp.AND, conditions=where))
elif len(where) == 1:
conditions.append(where[0])

if len(conditions) > 1:
# More than 1 set of conditions were specified, we want to show
# spans that match any 1 of them so join the conditions with `OR`s.
user_spans_query.add_conditions(
[BooleanCondition(op=BooleanOp.OR, conditions=conditions)]
)
elif len(conditions) == 1:
# Only 1 set of user conditions were specified, simply insert them into
# the final query.
user_spans_query.add_conditions([conditions[0]])
else:
# Next if there are known span_keys, we only try to fetch those spans
# This are the additional conditions to better take advantage of the ORDER BY
# on the spans table. This creates a list of conditions to be `OR`ed together
# that can will be used by ClickHouse to narrow down the granules.
#
# The span ids are not in this condition because they are more effective when
# specified within the `PREWHERE` clause. So, it's in a separate condition.
conditions = [
And(
[
Condition(user_spans_query.column("span.group"), Op.EQ, key.group),
Condition(
user_spans_query.column("timestamp"),
Op.EQ,
datetime.fromisoformat(key.timestamp),
),
]
)
for key in span_keys
]
# There are multiple sets of user conditions that needs to be satisfied
# and if a span satisfy any of them, it should be considered.
#
# To handle this use case, we want to OR all the user specified
# conditions together in this query.
for where in user_conditions:
if len(where) > 1:
conditions.append(BooleanCondition(op=BooleanOp.AND, conditions=where))
elif len(where) == 1:
conditions.append(where[0])

if len(conditions) == 1:
order_by_condition = conditions[0]
else:
order_by_condition = Or(conditions)

# Using `IN` combined with putting the list in a SnQL "tuple" triggers an optimizer
# in snuba where it
# 1. moves the condition into the `PREWHERE` clause
# 2. maps the ids to the underlying UInt64 and uses the bloom filter index
span_id_condition = Condition(
user_spans_query.column("id"),
Op.IN,
Function("tuple", [key.span_id for key in span_keys]),
if len(conditions) > 1:
# More than 1 set of conditions were specified, we want to show
# spans that match any 1 of them so join the conditions with `OR`s.
user_spans_query.add_conditions(
[BooleanCondition(op=BooleanOp.OR, conditions=conditions)]
)

user_spans_query.add_conditions([order_by_condition, span_id_condition])
elif len(conditions) == 1:
# Only 1 set of user conditions were specified, simply insert them into
# the final query.
user_spans_query.add_conditions([conditions[0]])

return user_spans_query

Expand Down
Loading
Loading