upstream/grafana
1
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2026-02-18 18:20:52 -05:00
Code Issues Projects Releases Packages Wiki Activity Actions 71

Alerting: Tweak Loki queries generated by Historian app. (#118368)

Browse source 
When searching Loki for a specific rule UID, it's much faster to search for the
raw UID string to prevent parsing lines into JSON unecessarily.

Also improve the request logging to make it easier to spot slow requests.
This commit is contained in:
Steve Simpson 2026-02-18 17:47:14 +01:00 committed by GitHub
parent a905e1f569
commit ef4c56d2d8
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 34 additions and 11 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
apps/alerting/historian/pkg/app/notification/lokireader.go
View file

@ -115,7 +115,14 @@ func buildQuery(query Query) (string, error) {
fmt.Sprintf(`%s=%q`, historian.LabelFrom, historian.LabelFromValue),
}
logql := fmt.Sprintf(`{%s} | json`, strings.Join(selectors, `,`))
logql := fmt.Sprintf(`{%s}`, strings.Join(selectors, `,`))
// Searching for ruleUID before JSON parsing can dramatically improve performance.
if query.RuleUID != nil && *query.RuleUID != "" {
logql += fmt.Sprintf(` |= %q`, *query.RuleUID)
}
logql += ` | json`
// Add ruleUID filter as JSON line filter if specified.
if query.RuleUID != nil && *query.RuleUID != "" {

12
apps/alerting/historian/pkg/app/notification/lokireader_test.go
View file

@ -135,7 +135,7 @@ func TestBuildQuery(t *testing.T) {
query: Query{
RuleUID: stringPtr("test-rule-uid"),
},
expected: fmt.Sprintf(`{%s=%q} | json | alert_labels___alert_rule_uid__ = "test-rule-uid"`,
expected: fmt.Sprintf(`{%s=%q} |= "test-rule-uid" | json | alert_labels___alert_rule_uid__ = "test-rule-uid"`,
historian.LabelFrom, historian.LabelFromValue),
},
{
@ -144,7 +144,7 @@ func TestBuildQuery(t *testing.T) {
RuleUID: stringPtr("test-rule-uid"),
Receiver: stringPtr("email-receiver"),
},
expected: fmt.Sprintf(`{%s=%q} | json | alert_labels___alert_rule_uid__ = "test-rule-uid" | receiver = "email-receiver"`,
expected: fmt.Sprintf(`{%s=%q} |= "test-rule-uid" | json | alert_labels___alert_rule_uid__ = "test-rule-uid" | receiver = "email-receiver"`,
historian.LabelFrom, historian.LabelFromValue),
},
{
@ -153,7 +153,7 @@ func TestBuildQuery(t *testing.T) {
RuleUID: stringPtr("test-rule-uid"),
Status: createStatusPtr(v0alpha1.CreateNotificationqueryRequestNotificationStatusFiring),
},
expected: fmt.Sprintf(`{%s=%q} | json | alert_labels___alert_rule_uid__ = "test-rule-uid" | status = "firing"`,
expected: fmt.Sprintf(`{%s=%q} |= "test-rule-uid" | json | alert_labels___alert_rule_uid__ = "test-rule-uid" | status = "firing"`,
historian.LabelFrom, historian.LabelFromValue),
},
{
@ -162,7 +162,7 @@ func TestBuildQuery(t *testing.T) {
RuleUID: stringPtr("test-rule-uid"),
Outcome: outcomePtr(v0alpha1.CreateNotificationqueryRequestNotificationOutcomeSuccess),
},
expected: fmt.Sprintf(`{%s=%q} | json | alert_labels___alert_rule_uid__ = "test-rule-uid" | error = ""`,
expected: fmt.Sprintf(`{%s=%q} |= "test-rule-uid" | json | alert_labels___alert_rule_uid__ = "test-rule-uid" | error = ""`,
historian.LabelFrom, historian.LabelFromValue),
},
{
@ -171,7 +171,7 @@ func TestBuildQuery(t *testing.T) {
RuleUID: stringPtr("test-rule-uid"),
Outcome: outcomePtr(v0alpha1.CreateNotificationqueryRequestNotificationOutcomeError),
},
expected: fmt.Sprintf(`{%s=%q} | json | alert_labels___alert_rule_uid__ = "test-rule-uid" | error != ""`,
expected: fmt.Sprintf(`{%s=%q} |= "test-rule-uid" | json | alert_labels___alert_rule_uid__ = "test-rule-uid" | error != ""`,
historian.LabelFrom, historian.LabelFromValue),
},
{
@ -182,7 +182,7 @@ func TestBuildQuery(t *testing.T) {
Status: createStatusPtr(v0alpha1.CreateNotificationqueryRequestNotificationStatusResolved),
Outcome: outcomePtr(v0alpha1.CreateNotificationqueryRequestNotificationOutcomeSuccess),
},
expected: fmt.Sprintf(`{%s=%q} | json | alert_labels___alert_rule_uid__ = "test-rule-uid" | receiver = "email-receiver" | status = "resolved" | error = ""`,
expected: fmt.Sprintf(`{%s=%q} |= "test-rule-uid" | json | alert_labels___alert_rule_uid__ = "test-rule-uid" | receiver = "email-receiver" | status = "resolved" | error = ""`,
historian.LabelFrom, historian.LabelFromValue),
},
{

24
apps/alerting/historian/pkg/app/notification/notification.go
View file

@ -4,7 +4,9 @@ import (
"context"
"encoding/json"
"errors"
"fmt"
"net/http"
"time"
"github.com/grafana/grafana-app-sdk/app"
"github.com/grafana/grafana-app-sdk/logging"
@ -33,44 +35,58 @@ func New(cfg config.NotificationConfig, reg prometheus.Registerer, logger loggin
}
func (n *Notification) QueryHandler(ctx context.Context, writer app.CustomRouteResponseWriter, request *app.CustomRouteRequest) error {
start := time.Now()
if n.loki == nil {
const msg = "Notification history query whilst disabled"
n.logger.Debug(msg)
return &apierrors.StatusError{
ErrStatus: metav1.Status{
Status: metav1.StatusFailure,
Code: http.StatusUnprocessableEntity,
Message: "notification history disabled",
Message: msg,
}}
}
var body v0alpha1.CreateNotificationqueryRequestBody
err := json.NewDecoder(request.Body).Decode(&body)
if err != nil {
const msg = "Notification history query malformed"
n.logger.Debug(msg, "err", err)
return &apierrors.StatusError{
ErrStatus: metav1.Status{
Status: metav1.StatusFailure,
Code: http.StatusBadRequest,
Message: err.Error(),
Message: fmt.Sprintf("%s: %s", msg, err.Error()),
}}
}
response, err := n.loki.Query(ctx, body)
if err != nil {
if errors.Is(err, ErrInvalidQuery) {
const msg = "Notification history query invalid"
n.logger.Debug(msg, "err", err)
return &apierrors.StatusError{
ErrStatus: metav1.Status{
Status: metav1.StatusFailure,
Code: http.StatusBadRequest,
Message: err.Error(),
Message: fmt.Sprintf("%s: %s", msg, err.Error()),
}}
}
const msg = "Notification history query failed"
n.logger.Error(msg, "err", err, "duration", time.Since(start))
return &apierrors.StatusError{
ErrStatus: metav1.Status{
Status: metav1.StatusFailure,
Code: http.StatusInternalServerError,
Message: err.Error(),
Message: fmt.Sprintf("%s: %s", msg, err.Error()),
}}
}
n.logger.Debug("Notification history query success",
"entries", len(response.Entries),
"duration", time.Since(start))
writer.Header().Add("Content-Type", "application/json")
writer.WriteHeader(http.StatusOK)
return json.NewEncoder(writer).Encode(response)