diff --git a/docs/querying/api.md b/docs/querying/api.md index 34fc823cb6..18bbff216f 100644 --- a/docs/querying/api.md +++ b/docs/querying/api.md @@ -384,19 +384,19 @@ $ curl http://localhost:9090/api/v1/alertmanagers ## TSDB Admin APIs -In 2.0, there are new APIs that expose database functionalities for the advanced user. These APIs are not enabled unless the `--web.enable-admin-api` is set. All the APIs below are experimental and might change in the future. +These are APIs that expose database functionalities for the advanced user. These APIs are not enabled unless the `--web.enable-admin-api` is set. -We also expose a gRPC API whose definition can be found [here](https://github.com/prometheus/prometheus/blob/master/prompb/rpc.proto). +We also expose a gRPC API whose definition can be found [here](https://github.com/prometheus/prometheus/blob/master/prompb/rpc.proto). This is experimental and might change in the future. ### Snapshot Snapshot creates a snapshot of all current data into `snapshots/-` under the TSDB's data directory and returns the directory as response. ``` -POST /api/v2/admin/tsdb/snapshot +POST /api/v1/admin/tsdb/snapshot ``` ```json -$ curl -XPOST http://localhost:9090/api/v2/admin/tsdb/snapshot +$ curl -XPOST http://localhost:9090/api/v1/admin/tsdb/snapshot { "name": "2017-11-30T15:31:59Z-2366f0a55106d6e1" } @@ -409,45 +409,20 @@ The snapshot now exists at `/snapshots/2017-11-30T15:31:59Z-2366f0a551 DeleteSeries deletes data for a selection of series in a time range. The actual data still exists on disk and is cleaned up in future compactions or can be explicitly cleaned up by hitting the Clean Tombstones endpoint. ``` -POST /api/v2/admin/tsdb/delete_series +DELETE /api/v1/admin/tsdb/delete_series ``` -Parameters (body): +URL query parameters: -```json -{ - "min_time": "date-time(iso-8601)" // optional, defaults to minmum-time. - "max_time": "date-time(iso-8601)" // optional, defaults to maximum-time - "matchers": [LabelMatcher] -} -``` - -```json -LabelMatchers: Matcher specifies a rule, which can match or set of labels or not. - -{ - "type": "string", // One of: EQ, NEQ, RE, NRE defaults to EQ. - "name": "label-name", - "value": "label-value" -} -``` +- `match[]=`: Repeated label matcher argument that selects the series to delete. At least one `match[]` argument must be provided. +- `start=`: Start timestamp. +- `end=`: End timestamp. Example: ```json -$ curl -X POST \ - http://localhost:9090/api/v2/admin/tsdb/delete_series \ - -H 'content-type: application/json' \ - -d '{ - "max_time": "'2017-11-30T20:18:30+05:30", - "matchers": [{ - "type": "EQ", - "name": "__name__", - "value": "up" - }] - }' - -{} +$ curl -X DELETE \ + -g 'http://localhost:9090/api/v1/series?match[]=up&match[]=process_start_time_seconds{job="prometheus"}' ``` @@ -456,12 +431,11 @@ $ curl -X POST \ CleanTombstones removes the deleted data from disk and cleans up the existing tombstones. This can be used after deleting series to free up space. ``` -POST /api/v2/admin/tsdb/clean_tombstones +POST /api/v1/admin/tsdb/clean_tombstones ``` This takes no parameters or body. ```json -$ curl -XPOST http://localhost:9090/api/v2/admin/tsdb/clean_tombstones -{} +$ curl -XPOST http://localhost:9090/api/v1/admin/tsdb/clean_tombstones ``` \ No newline at end of file diff --git a/web/api/v1/api.go b/web/api/v1/api.go index 238ba8ff6e..474b10b438 100644 --- a/web/api/v1/api.go +++ b/web/api/v1/api.go @@ -19,8 +19,11 @@ import ( "errors" "fmt" "math" + "math/rand" "net/http" "net/url" + "os" + "path/filepath" "sort" "strconv" "time" @@ -28,6 +31,7 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" "github.com/prometheus/common/route" + "github.com/prometheus/tsdb" "github.com/prometheus/prometheus/config" "github.com/prometheus/prometheus/pkg/labels" @@ -39,6 +43,7 @@ import ( "github.com/prometheus/prometheus/storage/remote" "github.com/prometheus/prometheus/util/httputil" "github.com/prometheus/prometheus/util/stats" + tsdbLabels "github.com/prometheus/tsdb/labels" ) type status string @@ -51,12 +56,13 @@ const ( type errorType string const ( - errorNone errorType = "" - errorTimeout = "timeout" - errorCanceled = "canceled" - errorExec = "execution" - errorBadData = "bad_data" - errorInternal = "internal" + errorNone errorType = "" + errorTimeout = "timeout" + errorCanceled = "canceled" + errorExec = "execution" + errorBadData = "bad_data" + errorInternal = "internal" + errorUnavailable = "unavailable" ) var corsHeaders = map[string]string{ @@ -111,6 +117,9 @@ type API struct { now func() time.Time config func() config.Config ready func(http.HandlerFunc) http.HandlerFunc + + db func() *tsdb.DB + enableAdmin bool } // NewAPI returns an initialized API type. @@ -121,15 +130,19 @@ func NewAPI( ar alertmanagerRetriever, configFunc func() config.Config, readyFunc func(http.HandlerFunc) http.HandlerFunc, + db func() *tsdb.DB, + enableAdmin bool, ) *API { return &API{ QueryEngine: qe, Queryable: q, targetRetriever: tr, alertmanagerRetriever: ar, - now: time.Now, - config: configFunc, - ready: readyFunc, + now: time.Now, + config: configFunc, + ready: readyFunc, + db: db, + enableAdmin: enableAdmin, } } @@ -168,6 +181,11 @@ func (api *API) Register(r *route.Router) { r.Get("/status/config", instr("config", api.serveConfig)) r.Post("/read", api.ready(prometheus.InstrumentHandler("read", http.HandlerFunc(api.remoteRead)))) + + // Admin APIs + r.Del("/admin/tsdb/delete_series", instr("delete_series", api.deleteSeries)) + r.Post("/admin/tsdb/clean_tombstones", instr("clean_tombstones", api.cleanTombstones)) + r.Post("/admin/tsdb/snapshot", instr("snapshot", api.snapshot)) } type queryData struct { @@ -555,6 +573,128 @@ func (api *API) remoteRead(w http.ResponseWriter, r *http.Request) { } } +func (api *API) deleteSeries(r *http.Request) (interface{}, *apiError) { + if !api.enableAdmin { + return nil, &apiError{errorUnavailable, errors.New("Admin APIs disabled")} + } + db := api.db() + if db == nil { + return nil, &apiError{errorUnavailable, errors.New("TSDB not ready")} + } + + r.ParseForm() + if len(r.Form["match[]"]) == 0 { + return nil, &apiError{errorBadData, fmt.Errorf("no match[] parameter provided")} + } + + var start time.Time + if t := r.FormValue("start"); t != "" { + var err error + start, err = parseTime(t) + if err != nil { + return nil, &apiError{errorBadData, err} + } + } else { + start = minTime + } + + var end time.Time + if t := r.FormValue("end"); t != "" { + var err error + end, err = parseTime(t) + if err != nil { + return nil, &apiError{errorBadData, err} + } + } else { + end = maxTime + } + + for _, s := range r.Form["match[]"] { + matchers, err := promql.ParseMetricSelector(s) + if err != nil { + return nil, &apiError{errorBadData, err} + } + + var selector tsdbLabels.Selector + for _, m := range matchers { + selector = append(selector, convertMatcher(m)) + } + + if err := db.Delete(timestamp.FromTime(start), timestamp.FromTime(end), selector...); err != nil { + return nil, &apiError{errorInternal, err} + } + } + + return nil, nil +} + +func (api *API) snapshot(r *http.Request) (interface{}, *apiError) { + if !api.enableAdmin { + return nil, &apiError{errorUnavailable, errors.New("Admin APIs disabled")} + } + db := api.db() + if db == nil { + return nil, &apiError{errorUnavailable, errors.New("TSDB not ready")} + } + + var ( + snapdir = filepath.Join(db.Dir(), "snapshots") + name = fmt.Sprintf("%s-%x", time.Now().UTC().Format(time.RFC3339), rand.Int()) + dir = filepath.Join(snapdir, name) + ) + if err := os.MkdirAll(dir, 0777); err != nil { + return nil, &apiError{errorInternal, fmt.Errorf("create snapshot directory: %s", err)} + } + if err := db.Snapshot(dir); err != nil { + return nil, &apiError{errorInternal, fmt.Errorf("create snapshot: %s", err)} + } + + return struct { + Name string `json:"name"` + }{name}, nil +} + +func (api *API) cleanTombstones(r *http.Request) (interface{}, *apiError) { + if !api.enableAdmin { + return nil, &apiError{errorUnavailable, errors.New("Admin APIs disabled")} + } + db := api.db() + if db == nil { + return nil, &apiError{errorUnavailable, errors.New("TSDB not ready")} + } + + if err := db.CleanTombstones(); err != nil { + return nil, &apiError{errorInternal, err} + } + + return nil, nil +} + +func convertMatcher(m *labels.Matcher) tsdbLabels.Matcher { + switch m.Type { + case labels.MatchEqual: + return tsdbLabels.NewEqualMatcher(m.Name, m.Value) + + case labels.MatchNotEqual: + return tsdbLabels.Not(tsdbLabels.NewEqualMatcher(m.Name, m.Value)) + + case labels.MatchRegexp: + res, err := tsdbLabels.NewRegexpMatcher(m.Name, "^(?:"+m.Value+")$") + if err != nil { + panic(err) + } + return res + + case labels.MatchNotRegexp: + res, err := tsdbLabels.NewRegexpMatcher(m.Name, "^(?:"+m.Value+")$") + if err != nil { + panic(err) + } + return tsdbLabels.Not(res) + } + panic("storage.convertMatcher: invalid matcher type") +} + // mergeLabels merges two sets of sorted proto labels, preferring those in // primary to those in secondary when there is an overlap. func mergeLabels(primary, secondary []*prompb.Label) []*prompb.Label { diff --git a/web/api/v2/api.go b/web/api/v2/api.go index 508bdc1564..41c641de91 100644 --- a/web/api/v2/api.go +++ b/web/api/v2/api.go @@ -79,7 +79,7 @@ func (api *API) RegisterGRPC(srv *grpc.Server) { if api.enableAdmin { pb.RegisterAdminServer(srv, NewAdmin(api.db)) } else { - pb.RegisterAdminServer(srv, &adminDisabled{}) + pb.RegisterAdminServer(srv, &AdminDisabled{}) } } @@ -134,23 +134,23 @@ func labelsToProto(lset labels.Labels) pb.Labels { return r } -// adminDisabled implements the administration interface that informs +// AdminDisabled implements the administration interface that informs // that the API endpoints are disbaled. -type adminDisabled struct { +type AdminDisabled struct { } // TSDBSnapshot implements pb.AdminServer. -func (s *adminDisabled) TSDBSnapshot(_ old_ctx.Context, _ *pb.TSDBSnapshotRequest) (*pb.TSDBSnapshotResponse, error) { +func (s *AdminDisabled) TSDBSnapshot(_ old_ctx.Context, _ *pb.TSDBSnapshotRequest) (*pb.TSDBSnapshotResponse, error) { return nil, status.Error(codes.Unavailable, "Admin APIs are disabled") } // TSDBCleanTombstones implements pb.AdminServer. -func (s *adminDisabled) TSDBCleanTombstones(_ old_ctx.Context, _ *pb.TSDBCleanTombstonesRequest) (*pb.TSDBCleanTombstonesResponse, error) { +func (s *AdminDisabled) TSDBCleanTombstones(_ old_ctx.Context, _ *pb.TSDBCleanTombstonesRequest) (*pb.TSDBCleanTombstonesResponse, error) { return nil, status.Error(codes.Unavailable, "Admin APIs are disabled") } // DeleteSeries imeplements pb.AdminServer. -func (s *adminDisabled) DeleteSeries(_ old_ctx.Context, r *pb.SeriesDeleteRequest) (*pb.SeriesDeleteResponse, error) { +func (s *AdminDisabled) DeleteSeries(_ old_ctx.Context, r *pb.SeriesDeleteRequest) (*pb.SeriesDeleteResponse, error) { return nil, status.Error(codes.Unavailable, "Admin APIs are disabled") } diff --git a/web/web.go b/web/web.go index 3ad90dd69e..f344fa5872 100644 --- a/web/web.go +++ b/web/web.go @@ -188,6 +188,8 @@ func New(logger log.Logger, o *Options) *Handler { return *h.config }, h.testReady, + h.options.TSDB, + h.options.EnableAdminAPI, ) if o.RoutePrefix != "/" {