diff --git a/docs/content/observe/logs-and-access-logs.md b/docs/content/observe/logs-and-access-logs.md index 51feeb557e..5d12829b89 100644 --- a/docs/content/observe/logs-and-access-logs.md +++ b/docs/content/observe/logs-and-access-logs.md @@ -175,6 +175,7 @@ When using the `json` format, you can customize which fields are included in you - **Request Fields:** You can choose to `keep`, `drop`, or `redact` any of the standard request fields. A complete list of available fields like `ClientHost`, `RequestMethod`, and `Duration` can be found in the [reference documentation](../reference/install-configuration/observability/logs-and-accesslogs.md#json-format-fields). - **Request Headers:** You can also specify which request headers should be included in the logs, and whether their values should be `kept`, `dropped`, or `redacted`. +- **Request Query Parameters:** You can choose to `keep` or `drop` the query parameters for a request. !!! info For detailed configuration options, refer to the [reference documentation](../reference/install-configuration/observability/logs-and-accesslogs.md). diff --git a/docs/content/reference/install-configuration/configuration-options.md b/docs/content/reference/install-configuration/configuration-options.md index 73b6af5560..50732d098a 100644 --- a/docs/content/reference/install-configuration/configuration-options.md +++ b/docs/content/reference/install-configuration/configuration-options.md @@ -14,6 +14,7 @@ THIS FILE MUST NOT BE EDITED BY HAND | accesslog.fields.headers.defaultmode | Default mode for fields: keep | drop | redact | drop | | accesslog.fields.headers.names._name_ | Override mode for headers | | | accesslog.fields.names._name_ | Override mode for fields | | +| accesslog.fields.queryparameters.defaultmode | Default mode for query parameters: keep | drop | keep | | accesslog.filepath | Access log file path. Stdout is used when omitted or empty. | | | accesslog.filters.minduration | Keep access logs when request took longer than the specified duration. | 0 | | accesslog.filters.retryattempts | Keep access logs when at least one retry happened. | false | diff --git a/docs/content/reference/install-configuration/observability/logs-and-accesslogs.md b/docs/content/reference/install-configuration/observability/logs-and-accesslogs.md index 147826fc00..4987e14ec4 100644 --- a/docs/content/reference/install-configuration/observability/logs-and-accesslogs.md +++ b/docs/content/reference/install-configuration/observability/logs-and-accesslogs.md @@ -168,6 +168,9 @@ accessLog: User-Agent: redact # Drop the Authorization header value Authorization: drop + queryParameters: + # Drop all query parameters + defaultMode: drop ``` ```toml tab="File (TOML)" @@ -191,6 +194,9 @@ accessLog: [accessLog.fields.headers.names] User-Agent = "redact" Authorization = "drop" + + [accessLog.fields.queryParameters] + defaultMode = "drop" ``` ```sh tab="CLI" @@ -204,6 +210,7 @@ accessLog: --accesslog.fields.headers.defaultmode=keep --accesslog.fields.headers.names.User-Agent=redact --accesslog.fields.headers.names.Authorization=drop +--accesslog.fields.queryparameters.defaultmode=drop ``` ### Configuration Options @@ -223,6 +230,7 @@ The section below describes how to configure Traefik access logs using the stati | `accesslog.fields.names` | Set the fields list to display in the access logs (format `name:mode`).
Available fields list [here](#json-format-fields). | [ ] | No | | `accesslog.fields.headers.defaultMode` | Mode to apply by default to the access logs headers (`keep`, `redact` or `drop`). | drop | No | | `accesslog.fields.headers.names` | Set the headers list to display in the access logs (format `name:mode`). | [ ] | No | +| `accesslog.fields.queryParameters.defaultMode` | Mode to apply by default to the access logs query parameters (`keep` or `drop`) | keep | No | ### OpenTelemetry diff --git a/pkg/middlewares/accesslog/logger.go b/pkg/middlewares/accesslog/logger.go index 2290cd4c06..fe69f33440 100644 --- a/pkg/middlewares/accesslog/logger.go +++ b/pkg/middlewares/accesslog/logger.go @@ -219,12 +219,18 @@ func (h *Handler) ServeHTTP(rw http.ResponseWriter, req *http.Request, next http core[RequestAddr] = req.Host core[RequestHost], core[RequestPort] = silentSplitHostPort(req.Host) } + + queryParameters := "" + if h.config.Fields.KeepQueryParameters() { + queryParameters = req.URL.RawQuery + } + // copy the URL without the scheme, hostname etc urlCopy := &url.URL{ Path: req.URL.Path, RawPath: req.URL.RawPath, - RawQuery: req.URL.RawQuery, - ForceQuery: req.URL.ForceQuery, + RawQuery: queryParameters, + ForceQuery: req.URL.ForceQuery && h.config.Fields.KeepQueryParameters(), Fragment: req.URL.Fragment, } urlCopyString := urlCopy.String() diff --git a/pkg/middlewares/accesslog/logger_test.go b/pkg/middlewares/accesslog/logger_test.go index 78bf2ebe68..f6cd87d07e 100644 --- a/pkg/middlewares/accesslog/logger_test.go +++ b/pkg/middlewares/accesslog/logger_test.go @@ -37,23 +37,25 @@ import ( const delta float64 = 1e-10 var ( - logFileNameSuffix = "/traefik/logger/test.log" - testContent = "Hello, World" - testServiceName = "http://127.0.0.1/testService" - testRouterName = "testRouter" - testStatus = 123 - testContentSize int64 = 12 - testHostname = "TestHost" - testUsername = "TestUser" - testPath = "testpath" - testPort = 8181 - testProto = "HTTP/0.0" - testScheme = "http" - testMethod = http.MethodPost - testReferer = "testReferer" - testUserAgent = "testUserAgent" - testRetryAttempts = 2 - testStart = time.Now() + logFileNameSuffix = "/traefik/logger/test.log" + testContent = "Hello, World" + testServiceName = "http://127.0.0.1/testService" + testRouterName = "testRouter" + testStatus = 123 + testContentSize int64 = 12 + testHostname = "TestHost" + testUsername = "TestUser" + testPath = "testpath" + testQueryParams = "param1=test1¶m2=test2" + testPathWithQueryParams = testPath + "?" + testQueryParams + testPort = 8181 + testProto = "HTTP/0.0" + testScheme = "http" + testMethod = http.MethodPost + testReferer = "testReferer" + testUserAgent = "testUserAgent" + testRetryAttempts = 2 + testStart = time.Now() ) func TestOTelAccessLogWithBody(t *testing.T) { @@ -393,7 +395,7 @@ func TestCommonLogger(t *testing.T) { logData, err := os.ReadFile(logFilePath) require.NoError(t, err) - expectedLog := ` TestHost - TestUser [13/Apr/2016:07:14:19 -0700] "POST testpath HTTP/0.0" 123 12 "testReferer" "testUserAgent" 1 "testRouter" "http://127.0.0.1/testService" 1ms` + expectedLog := ` TestHost - TestUser [13/Apr/2016:07:14:19 -0700] "POST testpath?param1=test1¶m2=test2 HTTP/0.0" 123 12 "testReferer" "testUserAgent" 1 "testRouter" "http://127.0.0.1/testService" 1ms` assertValidCommonLogData(t, expectedLog, logData) } @@ -408,6 +410,23 @@ func TestCommonLoggerWithBufferingSize(t *testing.T) { logData, err := os.ReadFile(logFilePath) require.NoError(t, err) + expectedLog := ` TestHost - TestUser [13/Apr/2016:07:14:19 -0700] "POST testpath?param1=test1¶m2=test2 HTTP/0.0" 123 12 "testReferer" "testUserAgent" 1 "testRouter" "http://127.0.0.1/testService" 1ms` + assertValidCommonLogData(t, expectedLog, logData) +} + +func TestCommonLoggerDropQueryParameters(t *testing.T) { + logFilePath := filepath.Join(t.TempDir(), logFileNameSuffix) + fieldConfig := &otypes.AccessLogFields{ + QueryParameters: &otypes.FieldQueryParameters{ + DefaultMode: "drop", + }, + } + config := &otypes.AccessLog{FilePath: logFilePath, Format: CommonFormat, Fields: fieldConfig} + doLogging(t, config, false) + + logData, err := os.ReadFile(logFilePath) + require.NoError(t, err) + expectedLog := ` TestHost - TestUser [13/Apr/2016:07:14:19 -0700] "POST testpath HTTP/0.0" 123 12 "testReferer" "testUserAgent" 1 "testRouter" "http://127.0.0.1/testService" 1ms` assertValidCommonLogData(t, expectedLog, logData) } @@ -420,7 +439,7 @@ func TestLoggerGenericCLF(t *testing.T) { logData, err := os.ReadFile(logFilePath) require.NoError(t, err) - expectedLog := ` TestHost - TestUser [13/Apr/2016:07:14:19 -0700] "POST testpath HTTP/0.0" 123 12 "testReferer" "testUserAgent"` + expectedLog := ` TestHost - TestUser [13/Apr/2016:07:14:19 -0700] "POST testpath?param1=test1¶m2=test2 HTTP/0.0" 123 12 "testReferer" "testUserAgent"` assertValidGenericCLFLogData(t, expectedLog, logData) } @@ -435,6 +454,23 @@ func TestLoggerGenericCLFWithBufferingSize(t *testing.T) { logData, err := os.ReadFile(logFilePath) require.NoError(t, err) + expectedLog := ` TestHost - TestUser [13/Apr/2016:07:14:19 -0700] "POST testpath?param1=test1¶m2=test2 HTTP/0.0" 123 12 "testReferer" "testUserAgent"` + assertValidGenericCLFLogData(t, expectedLog, logData) +} + +func TestLoggerGenericCLFDropQueryParameters(t *testing.T) { + logFilePath := filepath.Join(t.TempDir(), logFileNameSuffix) + fieldConfig := &otypes.AccessLogFields{ + QueryParameters: &otypes.FieldQueryParameters{ + DefaultMode: "drop", + }, + } + config := &otypes.AccessLog{FilePath: logFilePath, Format: GenericCLFFormat, Fields: fieldConfig} + doLogging(t, config, false) + + logData, err := os.ReadFile(logFilePath) + require.NoError(t, err) + expectedLog := ` TestHost - TestUser [13/Apr/2016:07:14:19 -0700] "POST testpath HTTP/0.0" 123 12 "testReferer" "testUserAgent"` assertValidGenericCLFLogData(t, expectedLog, logData) } @@ -490,7 +526,7 @@ func TestLoggerJSON(t *testing.T) { RequestHost: assertString(testHostname), RequestAddr: assertString(testHostname), RequestMethod: assertString(testMethod), - RequestPath: assertString(testPath), + RequestPath: assertString(testPathWithQueryParams), RequestProtocol: assertString(testProto), RequestScheme: assertString(testScheme), RequestPort: assertString("-"), @@ -530,7 +566,7 @@ func TestLoggerJSON(t *testing.T) { RequestHost: assertString(testHostname), RequestAddr: assertString(testHostname), RequestMethod: assertString(testMethod), - RequestPath: assertString(testPath), + RequestPath: assertString(testPathWithQueryParams), RequestProtocol: assertString(testProto), RequestScheme: assertString(testScheme), RequestPort: assertString("-"), @@ -574,7 +610,7 @@ func TestLoggerJSON(t *testing.T) { RequestHost: assertString(testHostname), RequestAddr: assertString(testHostname), RequestMethod: assertString(testMethod), - RequestPath: assertString(testPath), + RequestPath: assertString(testPathWithQueryParams), RequestProtocol: assertString(testProto), RequestScheme: assertString("https"), RequestPort: assertString("-"), @@ -714,6 +750,94 @@ func TestLoggerJSON(t *testing.T) { RequestRefererHeader: assertString(testReferer), }, }, + { + desc: "default config, drop query parameters", + config: &otypes.AccessLog{ + FilePath: "", + Format: JSONFormat, + Fields: &otypes.AccessLogFields{ + QueryParameters: &otypes.FieldQueryParameters{ + DefaultMode: "drop", + }, + }, + }, + expected: map[string]func(t *testing.T, value any){ + RequestContentSize: assertFloat64(0), + RequestHost: assertString(testHostname), + RequestAddr: assertString(testHostname), + RequestMethod: assertString(testMethod), + RequestPath: assertString(testPath), + RequestProtocol: assertString(testProto), + RequestScheme: assertString(testScheme), + RequestPort: assertString("-"), + DownstreamStatus: assertFloat64(float64(testStatus)), + DownstreamContentSize: assertFloat64(float64(len(testContent))), + OriginContentSize: assertFloat64(float64(len(testContent))), + OriginStatus: assertFloat64(float64(testStatus)), + RequestRefererHeader: assertString(testReferer), + RequestUserAgentHeader: assertString(testUserAgent), + RouterName: assertString(testRouterName), + ServiceURL: assertString(testServiceName), + ClientUsername: assertString(testUsername), + ClientHost: assertString(testHostname), + ClientPort: assertString(strconv.Itoa(testPort)), + ClientAddr: assertString(fmt.Sprintf("%s:%d", testHostname, testPort)), + "level": assertString("info"), + "msg": assertString(""), + "downstream_Content-Type": assertString("text/plain; charset=utf-8"), + RequestCount: assertFloat64NotZero(), + Duration: assertFloat64NotZero(), + Overhead: assertFloat64NotZero(), + RetryAttempts: assertFloat64(float64(testRetryAttempts)), + "time": assertNotEmpty(), + "StartLocal": assertNotEmpty(), + "StartUTC": assertNotEmpty(), + }, + }, + { + desc: "default config, keep query parameters", + config: &otypes.AccessLog{ + FilePath: "", + Format: JSONFormat, + Fields: &otypes.AccessLogFields{ + QueryParameters: &otypes.FieldQueryParameters{ + DefaultMode: "keep", + }, + }, + }, + expected: map[string]func(t *testing.T, value any){ + RequestContentSize: assertFloat64(0), + RequestHost: assertString(testHostname), + RequestAddr: assertString(testHostname), + RequestMethod: assertString(testMethod), + RequestPath: assertString(testPathWithQueryParams), + RequestProtocol: assertString(testProto), + RequestScheme: assertString(testScheme), + RequestPort: assertString("-"), + DownstreamStatus: assertFloat64(float64(testStatus)), + DownstreamContentSize: assertFloat64(float64(len(testContent))), + OriginContentSize: assertFloat64(float64(len(testContent))), + OriginStatus: assertFloat64(float64(testStatus)), + RequestRefererHeader: assertString(testReferer), + RequestUserAgentHeader: assertString(testUserAgent), + RouterName: assertString(testRouterName), + ServiceURL: assertString(testServiceName), + ClientUsername: assertString(testUsername), + ClientHost: assertString(testHostname), + ClientPort: assertString(strconv.Itoa(testPort)), + ClientAddr: assertString(fmt.Sprintf("%s:%d", testHostname, testPort)), + "level": assertString("info"), + "msg": assertString(""), + "downstream_Content-Type": assertString("text/plain; charset=utf-8"), + RequestCount: assertFloat64NotZero(), + Duration: assertFloat64NotZero(), + Overhead: assertFloat64NotZero(), + RetryAttempts: assertFloat64(float64(testRetryAttempts)), + "time": assertNotEmpty(), + "StartLocal": assertNotEmpty(), + "StartUTC": assertNotEmpty(), + }, + }, } for _, test := range testCases { @@ -815,7 +939,7 @@ func TestNewLogHandlerOutputStdout(t *testing.T) { FilePath: "", Format: CommonFormat, }, - expectedLog: `TestHost - TestUser [13/Apr/2016:07:14:19 -0700] "POST testpath HTTP/0.0" 123 12 "testReferer" "testUserAgent" 23 "testRouter" "http://127.0.0.1/testService" 1ms`, + expectedLog: `TestHost - TestUser [13/Apr/2016:07:14:19 -0700] "POST testpath?param1=test1¶m2=test2 HTTP/0.0" 123 12 "testReferer" "testUserAgent" 23 "testRouter" "http://127.0.0.1/testService" 1ms`, }, { desc: "default config with empty filters", @@ -824,7 +948,7 @@ func TestNewLogHandlerOutputStdout(t *testing.T) { Format: CommonFormat, Filters: &otypes.AccessLogFilters{}, }, - expectedLog: `TestHost - TestUser [13/Apr/2016:07:14:19 -0700] "POST testpath HTTP/0.0" 123 12 "testReferer" "testUserAgent" 23 "testRouter" "http://127.0.0.1/testService" 1ms`, + expectedLog: `TestHost - TestUser [13/Apr/2016:07:14:19 -0700] "POST testpath?param1=test1¶m2=test2 HTTP/0.0" 123 12 "testReferer" "testUserAgent" 23 "testRouter" "http://127.0.0.1/testService" 1ms`, }, { desc: "Status code filter not matching", @@ -846,7 +970,7 @@ func TestNewLogHandlerOutputStdout(t *testing.T) { StatusCodes: []string{"123"}, }, }, - expectedLog: `TestHost - TestUser [13/Apr/2016:07:14:19 -0700] "POST testpath HTTP/0.0" 123 12 "testReferer" "testUserAgent" 23 "testRouter" "http://127.0.0.1/testService" 1ms`, + expectedLog: `TestHost - TestUser [13/Apr/2016:07:14:19 -0700] "POST testpath?param1=test1¶m2=test2 HTTP/0.0" 123 12 "testReferer" "testUserAgent" 23 "testRouter" "http://127.0.0.1/testService" 1ms`, }, { desc: "Duration filter not matching", @@ -868,7 +992,7 @@ func TestNewLogHandlerOutputStdout(t *testing.T) { MinDuration: ptypes.Duration(1 * time.Millisecond), }, }, - expectedLog: `TestHost - TestUser [13/Apr/2016:07:14:19 -0700] "POST testpath HTTP/0.0" 123 12 "testReferer" "testUserAgent" 23 "testRouter" "http://127.0.0.1/testService" 1ms`, + expectedLog: `TestHost - TestUser [13/Apr/2016:07:14:19 -0700] "POST testpath?param1=test1¶m2=test2 HTTP/0.0" 123 12 "testReferer" "testUserAgent" 23 "testRouter" "http://127.0.0.1/testService" 1ms`, }, { desc: "Retry attempts filter matching", @@ -879,7 +1003,7 @@ func TestNewLogHandlerOutputStdout(t *testing.T) { RetryAttempts: true, }, }, - expectedLog: `TestHost - TestUser [13/Apr/2016:07:14:19 -0700] "POST testpath HTTP/0.0" 123 12 "testReferer" "testUserAgent" 23 "testRouter" "http://127.0.0.1/testService" 1ms`, + expectedLog: `TestHost - TestUser [13/Apr/2016:07:14:19 -0700] "POST testpath?param1=test1¶m2=test2 HTTP/0.0" 123 12 "testReferer" "testUserAgent" 23 "testRouter" "http://127.0.0.1/testService" 1ms`, }, { desc: "Default mode keep", @@ -890,7 +1014,7 @@ func TestNewLogHandlerOutputStdout(t *testing.T) { DefaultMode: "keep", }, }, - expectedLog: `TestHost - TestUser [13/Apr/2016:07:14:19 -0700] "POST testpath HTTP/0.0" 123 12 "testReferer" "testUserAgent" 23 "testRouter" "http://127.0.0.1/testService" 1ms`, + expectedLog: `TestHost - TestUser [13/Apr/2016:07:14:19 -0700] "POST testpath?param1=test1¶m2=test2 HTTP/0.0" 123 12 "testReferer" "testUserAgent" 23 "testRouter" "http://127.0.0.1/testService" 1ms`, }, { desc: "Default mode keep with override", @@ -904,7 +1028,7 @@ func TestNewLogHandlerOutputStdout(t *testing.T) { }, }, }, - expectedLog: `- - TestUser [13/Apr/2016:07:14:19 -0700] "POST testpath HTTP/0.0" 123 12 "testReferer" "testUserAgent" 23 "testRouter" "http://127.0.0.1/testService" 1ms`, + expectedLog: `- - TestUser [13/Apr/2016:07:14:19 -0700] "POST testpath?param1=test1¶m2=test2 HTTP/0.0" 123 12 "testReferer" "testUserAgent" 23 "testRouter" "http://127.0.0.1/testService" 1ms`, }, { desc: "Default mode drop", @@ -1110,8 +1234,10 @@ func doLoggingTLSOpt(t *testing.T, config *otypes.AccessLog, enableTLS, tracing Method: testMethod, RemoteAddr: fmt.Sprintf("%s:%d", testHostname, testPort), URL: &url.URL{ - User: url.UserPassword(testUsername, ""), - Path: testPath, + User: url.UserPassword(testUsername, ""), + Path: testPath, + RawQuery: testQueryParams, + ForceQuery: true, }, Body: io.NopCloser(bytes.NewReader([]byte("bar"))), } diff --git a/pkg/observability/types/logs.go b/pkg/observability/types/logs.go index 8bc108ddc3..37314751b9 100644 --- a/pkg/observability/types/logs.go +++ b/pkg/observability/types/logs.go @@ -92,9 +92,15 @@ type FieldHeaders struct { // AccessLogFields holds configuration for access log fields. type AccessLogFields struct { - DefaultMode string `description:"Default mode for fields: keep | drop" json:"defaultMode,omitempty" toml:"defaultMode,omitempty" yaml:"defaultMode,omitempty" export:"true"` - Names map[string]string `description:"Override mode for fields" json:"names,omitempty" toml:"names,omitempty" yaml:"names,omitempty" export:"true"` - Headers *FieldHeaders `description:"Headers to keep, drop or redact" json:"headers,omitempty" toml:"headers,omitempty" yaml:"headers,omitempty" export:"true"` + DefaultMode string `description:"Default mode for fields: keep | drop" json:"defaultMode,omitempty" toml:"defaultMode,omitempty" yaml:"defaultMode,omitempty" export:"true"` + Names map[string]string `description:"Override mode for fields" json:"names,omitempty" toml:"names,omitempty" yaml:"names,omitempty" export:"true"` + Headers *FieldHeaders `description:"Headers to keep, drop or redact" json:"headers,omitempty" toml:"headers,omitempty" yaml:"headers,omitempty" export:"true"` + QueryParameters *FieldQueryParameters `description:"Keep or drop all query parameters" json:"queryParameters,omitempty" toml:"queryParameters,omitempty" yaml:"queryParameters,omitempty" export:"true"` +} + +// FieldQueryParameters holds configuration for access log query parameters. +type FieldQueryParameters struct { + DefaultMode string `description:"Default mode for query parameters: keep | drop" json:"defaultMode,omitempty" toml:"defaultMode,omitempty" yaml:"defaultMode,omitempty" export:"true"` } // SetDefaults sets the default values. @@ -103,6 +109,9 @@ func (f *AccessLogFields) SetDefaults() { f.Headers = &FieldHeaders{ DefaultMode: AccessLogDrop, } + f.QueryParameters = &FieldQueryParameters{ + DefaultMode: AccessLogKeep, + } } // Keep check if the field need to be kept or dropped. @@ -131,6 +140,15 @@ func (f *AccessLogFields) KeepHeader(header string) string { return defaultValue } +// KeepQueryParameters checks if the query parameters need to be kept or dropped. +func (f *AccessLogFields) KeepQueryParameters() bool { + defaultKeep := true + if f == nil || f.QueryParameters == nil { + return defaultKeep + } + return checkFieldValue(f.QueryParameters.DefaultMode, defaultKeep) +} + func checkFieldValue(value string, defaultKeep bool) bool { switch value { case AccessLogKeep: