diff --git a/src/51d.c b/src/51d.c index 16fd6809c..e96e0e436 100644 --- a/src/51d.c +++ b/src/51d.c @@ -2,12 +2,13 @@ #include #include +#include #include #include +#include #include #include #include - #include struct _51d_property_names { @@ -15,8 +16,12 @@ struct _51d_property_names { char *name; }; +#ifdef FIFTYONEDEGREES_H_PATTERN_INCLUDED +#define _51DEGREES_CONV_CACHE_KEY "_51d_conv" +#define _51DEGREES_FETCH_CACHE_KEY "_51d_fetch" static struct lru64_head *_51d_lru_tree = NULL; static unsigned long long _51d_lru_seed; +#endif static int _51d_data_file(char **args, int section_type, struct proxy *curpx, struct proxy *defpx, const char *file, int line, @@ -37,8 +42,8 @@ static int _51d_data_file(char **args, int section_type, struct proxy *curpx, } static int _51d_property_name_list(char **args, int section_type, struct proxy *curpx, - struct proxy *defpx, const char *file, int line, - char **err) + struct proxy *defpx, const char *file, int line, + char **err) { int cur_arg = 1; struct _51d_property_names *name; @@ -104,38 +109,271 @@ static int _51d_cache_size(char **args, int section_type, struct proxy *curpx, return 0; } -static int _51d_conv_check(struct arg *arg, struct sample_conv *conv, - const char *file, int line, char **err) +static int _51d_fetch_check(struct arg *arg, char **err_msg) { if (global._51degrees.data_file_path) return 1; - memprintf(err, "51Degrees data file is not specified (parameter '51degrees-data-file')"); + memprintf(err_msg, "51Degrees data file is not specified (parameter '51degrees-data-file')"); return 0; } +static int _51d_conv_check(struct arg *arg, struct sample_conv *conv, + const char *file, int line, char **err_msg) +{ + if (global._51degrees.data_file_path) + return 1; + + memprintf(err_msg, "51Degrees data file is not specified (parameter '51degrees-data-file')"); + return 0; +} + +#ifdef FIFTYONEDEGREES_H_PATTERN_INCLUDED +/* Sets the important HTTP headers ahead of the detection + */ +static void _51d_set_headers(struct sample *smp, fiftyoneDegreesWorkset *ws) +{ + struct hdr_idx *idx; + struct hdr_ctx ctx; + const struct http_msg *msg; + int i; + + idx = &smp->strm->txn->hdr_idx; + msg = &smp->strm->txn->req; + + ws->importantHeadersCount = 0; + + for (i = 0; i < global._51degrees.header_count; i++) { + ctx.idx = 0; + if (http_find_full_header2( + (global._51degrees.header_names + i)->str, + (global._51degrees.header_names + i)->len, + msg->chn->buf->p, idx, &ctx) == 1) { + ws->importantHeaders[ws->importantHeadersCount].header = ws->dataSet->httpHeaders + i; + ws->importantHeaders[ws->importantHeadersCount].headerValue = ctx.line + ctx.val; + ws->importantHeaders[ws->importantHeadersCount].headerValueLength = ctx.vlen; + ws->importantHeadersCount++; + } + } +} +#endif + +#ifdef FIFTYONEDEGREES_H_TRIE_INCLUDED +static void _51d_set_device_offsets(struct sample *smp) +{ + struct hdr_idx *idx; + struct hdr_ctx ctx; + const struct http_msg *msg; + int index; + fiftyoneDegreesDeviceOffsets *offsets = &global._51degrees.device_offsets; + + idx = &smp->strm->txn->hdr_idx; + msg = &smp->strm->txn->req; + offsets->size = 0; + + for (index = 0; index < global._51degrees.header_count; index++) { + ctx.idx = 0; + if (http_find_full_header2( + (global._51degrees.header_names + index)->str, + (global._51degrees.header_names + index)->len, + msg->chn->buf->p, idx, &ctx) == 1) { + (offsets->firstOffset + offsets->size)->httpHeaderOffset = *(global._51degrees.header_offsets + index); + (offsets->firstOffset + offsets->size)->deviceOffset = fiftyoneDegreesGetDeviceOffset(ctx.line + ctx.val); + offsets->size++; + } + } +} +#endif + +#ifdef FIFTYONEDEGREES_H_PATTERN_INCLUDED +/* Provides a hash code for the important HTTP headers. + */ +unsigned long long _51d_req_hash(const struct arg *args, fiftyoneDegreesWorkset* ws) +{ + unsigned long long seed = _51d_lru_seed ^ (long)args; + unsigned long long hash = 0; + int i; + for(i = 0; i < ws->importantHeadersCount; i++) { + hash ^= ws->importantHeaders[i].header->headerNameOffset; + hash ^= XXH64(ws->importantHeaders[i].headerValue, + ws->importantHeaders[i].headerValueLength, + seed); + } + return hash; +} +#endif + +#ifdef FIFTYONEDEGREES_H_PATTERN_INCLUDED +static void _51d_process_match(const struct arg *args, struct sample *smp, fiftyoneDegreesWorkset* ws) +{ + char *methodName; +#endif +#ifdef FIFTYONEDEGREES_H_TRIE_INCLUDED +static void _51d_process_match(const struct arg *args, struct sample *smp) +{ + char valuesBuffer[1024]; + char **requiredProperties = fiftyoneDegreesGetRequiredPropertiesNames(); + int requiredPropertiesCount = fiftyoneDegreesGetRequiredPropertiesCount(); + fiftyoneDegreesDeviceOffsets *deviceOffsets = &global._51degrees.device_offsets; + +#endif + + char no_data[] = "NoData"; /* response when no data could be found */ + struct chunk *temp = get_trash_chunk(); + int j, i = 0, found; + const char* property_name; + + /* Loop through property names passed to the filter and fetch them from the dataset. */ + while (args[i].data.str.str) { + /* Try to find request property in dataset. */ + found = 0; +#ifdef FIFTYONEDEGREES_H_PATTERN_INCLUDED + if (strcmp("Method", args[i].data.str.str) == 0) { + switch(ws->method) { + case EXACT: methodName = "Exact"; break; + case NUMERIC: methodName = "Numeric"; break; + case NEAREST: methodName = "Nearest"; break; + case CLOSEST: methodName = "Closest"; break; + default: + case NONE: methodName = "None"; break; + } + chunk_appendf(temp, "%s", methodName); + found = 1; + } + else if (strcmp("Difference", args[i].data.str.str) == 0) { + chunk_appendf(temp, "%d", ws->difference); + found = 1; + } + else if (strcmp("Rank", args[i].data.str.str) == 0) { + chunk_appendf(temp, "%d", fiftyoneDegreesGetSignatureRank(ws)); + found = 1; + } + else { + for (j = 0; j < ws->dataSet->requiredPropertyCount; j++) { + property_name = fiftyoneDegreesGetPropertyName(ws->dataSet, ws->dataSet->requiredProperties[j]); + if (strcmp(property_name, args[i].data.str.str) == 0) { + found = 1; + fiftyoneDegreesSetValues(ws, j); + chunk_appendf(temp, "%s", fiftyoneDegreesGetValueName(ws->dataSet, *ws->values)); + break; + } + } + } +#endif +#ifdef FIFTYONEDEGREES_H_TRIE_INCLUDED + found = 0; + for (j = 0; j < requiredPropertiesCount; j++) { + property_name = requiredProperties[j]; + if (strcmp(property_name, args[i].data.str.str) == 0 && + fiftyoneDegreesGetValueFromOffsets(deviceOffsets, j, valuesBuffer, 1024) > 0) { + found = 1; + chunk_appendf(temp, "%s", valuesBuffer); + break; + } + } +#endif + if (!found) { + chunk_appendf(temp, "%s", no_data); + } + /* Add separator. */ + chunk_appendf(temp, "%c", global._51degrees.property_separator); + ++i; + } + + if (temp->len) { + --temp->len; + temp->str[temp->len] = '\0'; + } + + smp->data.u.str.str = temp->str; + smp->data.u.str.len = strlen(temp->str); +} + +static int _51d_fetch(const struct arg *args, struct sample *smp, const char *kw, void *private) +{ +#ifdef FIFTYONEDEGREES_H_PATTERN_INCLUDED + fiftyoneDegreesWorkset* ws; /* workset for detection */ + struct lru64 *lru = NULL; +#endif + + /* Needed to ensure that the HTTP message has been fully recieved when + * used with TCP operation. Not required for HTTP operation. + * Data type has to be reset to ensure the string output is processed + * correctly. + */ + CHECK_HTTP_MESSAGE_FIRST(); + smp->data.type = SMP_T_STR; + +#ifdef FIFTYONEDEGREES_H_PATTERN_INCLUDED + + /* Get only the headers needed for device detection so they can be used + * with the cache to return previous results. Pattern is slower than + * Trie so caching will help improve performance. + */ + + /* Get a workset from the pool which will later contain detection results. */ + ws = fiftyoneDegreesWorksetPoolGet(global._51degrees.pool); + if (!ws) + return 0; + + /* Set the important HTTP headers for this request in the workset. */ + _51d_set_headers(smp, ws); + + /* Check the cache to see if there's results for these headers already. */ + if (_51d_lru_tree) { + lru = lru64_get(_51d_req_hash(args, ws), + _51d_lru_tree, _51DEGREES_FETCH_CACHE_KEY, 0); + if (lru && lru->domain) { + smp->flags |= SMP_F_CONST; + smp->data.u.str.str = lru->data; + smp->data.u.str.len = strlen(lru->data); + fiftyoneDegreesWorksetPoolRelease(global._51degrees.pool, ws); + return 1; + } + } + + fiftyoneDegreesMatchForHttpHeaders(ws); + + _51d_process_match(args, smp, ws); + +#endif + +#ifdef FIFTYONEDEGREES_H_TRIE_INCLUDED + + /* Trie is very fast so all the headers can be passed in and the result + * returned faster than the hashing algorithm process. + */ + _51d_set_device_offsets(smp); + _51d_process_match(args, smp); + +#endif + +#ifdef FIFTYONEDEGREES_H_PATTERN_INCLUDED + fiftyoneDegreesWorksetPoolRelease(global._51degrees.pool, ws); + if (lru) { + smp->flags |= SMP_F_CONST; + lru64_commit(lru, strdup(smp->data.u.str.str), _51DEGREES_FETCH_CACHE_KEY, 0, free); + } +#endif + + return 1; +} + static int _51d_conv(const struct arg *args, struct sample *smp, void *private) { - int i; - char no_data[] = "NoData"; /* response when no data could be found */ - struct chunk *temp; #ifdef FIFTYONEDEGREES_H_PATTERN_INCLUDED - int j, found; - const char* property_name; fiftyoneDegreesWorkset* ws; /* workset for detection */ -#endif -#ifdef FIFTYONEDEGREES_H_TRIE_INCLUDED - int device_offset; - int property_index; -#endif struct lru64 *lru = NULL; +#endif + +#ifdef FIFTYONEDEGREES_H_PATTERN_INCLUDED /* Look in the list. */ if (_51d_lru_tree) { unsigned long long seed = _51d_lru_seed ^ (long)args; lru = lru64_get(XXH64(smp->data.u.str.str, smp->data.u.str.len, seed), - _51d_lru_tree, global._51degrees.data_file_path, 0); + _51d_lru_tree, _51DEGREES_CONV_CACHE_KEY, 0); if (lru && lru->domain) { smp->flags |= SMP_F_CONST; smp->data.u.str.str = lru->data; @@ -144,9 +382,8 @@ static int _51d_conv(const struct arg *args, struct sample *smp, void *private) } } -#ifdef FIFTYONEDEGREES_H_PATTERN_INCLUDED /* Create workset. This will later contain detection results. */ - ws = fiftyoneDegreesCreateWorkset(&global._51degrees.data_set); + ws = fiftyoneDegreesWorksetPoolGet(global._51degrees.pool); if (!ws) return 0; #endif @@ -160,66 +397,60 @@ static int _51d_conv(const struct arg *args, struct sample *smp, void *private) /* Perform detection. */ #ifdef FIFTYONEDEGREES_H_PATTERN_INCLUDED fiftyoneDegreesMatch(ws, smp->data.u.str.str); + _51d_process_match(args, smp, ws); #endif #ifdef FIFTYONEDEGREES_H_TRIE_INCLUDED - device_offset = fiftyoneDegreesGetDeviceOffset(smp->data.u.str.str); + global._51degrees.device_offsets.firstOffset->deviceOffset = fiftyoneDegreesGetDeviceOffset(smp->data.u.str.str); + global._51degrees.device_offsets.size = 1; + _51d_process_match(args, smp); #endif - i = 0; - temp = get_trash_chunk(); - - /* Loop through property names passed to the filter and fetch them from the dataset. */ - while (args[i].data.str.str) { - /* Try to find request property in dataset. */ -#ifdef FIFTYONEDEGREES_H_PATTERN_INCLUDED - found = 0; - for (j = 0; j < ws->dataSet->requiredPropertyCount; j++) { - property_name = fiftyoneDegreesGetPropertyName(ws->dataSet, ws->dataSet->requiredProperties[j]); - if (strcmp(property_name, args[i].data.str.str) == 0) { - found = 1; - fiftyoneDegreesSetValues(ws, j); - chunk_appendf(temp, "%s", fiftyoneDegreesGetValueName(ws->dataSet, *ws->values)); - break; - } - } - if (!found) { - chunk_appendf(temp, "%s", no_data); - } -#endif -#ifdef FIFTYONEDEGREES_H_TRIE_INCLUDED - property_index = fiftyoneDegreesGetPropertyIndex(args[i].data.str.str); - if (property_index > 0) { - chunk_appendf(temp, "%s", fiftyoneDegreesGetValue(device_offset, property_index)); - } - else { - chunk_appendf(temp, "%s", no_data); - } -#endif - /* Add separator. */ - chunk_appendf(temp, "%c", global._51degrees.property_separator); - ++i; - } - - if (temp->len) { - --temp->len; - temp->str[temp->len] = '\0'; - } - - smp->data.u.str.str = temp->str; - smp->data.u.str.len = strlen(smp->data.u.str.str); - #ifdef FIFTYONEDEGREES_H_PATTERN_INCLUDED - fiftyoneDegreesFreeWorkset(ws); -#endif - + fiftyoneDegreesWorksetPoolRelease(global._51degrees.pool, ws); if (lru) { smp->flags |= SMP_F_CONST; - lru64_commit(lru, strdup(smp->data.u.str.str), global._51degrees.data_file_path, 0, free); + lru64_commit(lru, strdup(smp->data.u.str.str), _51DEGREES_CONV_CACHE_KEY, 0, free); } +#endif return 1; } +#ifdef FIFTYONEDEGREES_H_PATTERN_INCLUDED +void _51d_init_http_headers() +{ + int index = 0; + const fiftyoneDegreesAsciiString *headerName; + fiftyoneDegreesDataSet *ds = &global._51degrees.data_set; + global._51degrees.header_count = ds->httpHeadersCount; + global._51degrees.header_names = (struct chunk*)malloc(global._51degrees.header_count * sizeof(struct chunk)); + for (index = 0; index < global._51degrees.header_count; index++) { + headerName = fiftyoneDegreesGetString(ds, ds->httpHeaders[index].headerNameOffset); + (global._51degrees.header_names + index)->str = (char*)&headerName->firstByte; + (global._51degrees.header_names + index)->len = headerName->length - 1; + (global._51degrees.header_names + index)->size = (global._51degrees.header_names + index)->len; + } +} +#endif + +#ifdef FIFTYONEDEGREES_H_TRIE_INCLUDED +void _51d_init_http_headers() +{ + int index = 0; + global._51degrees.header_count = fiftyoneDegreesGetHttpHeaderCount(); + global._51degrees.device_offsets.firstOffset = (fiftyoneDegreesDeviceOffset*)malloc( + global._51degrees.header_count * sizeof(fiftyoneDegreesDeviceOffset)); + global._51degrees.header_names = (struct chunk*)malloc(global._51degrees.header_count * sizeof(struct chunk)); + global._51degrees.header_offsets = (int32_t*)malloc(global._51degrees.header_count * sizeof(int32_t)); + for (index = 0; index < global._51degrees.header_count; index++) { + global._51degrees.header_offsets[index] = fiftyoneDegreesGetHttpHeaderNameOffset(index); + global._51degrees.header_names[index].str = fiftyoneDegreesGetHttpHeaderNamePointer(index); + global._51degrees.header_names[index].len = strlen(global._51degrees.header_names[index].str); + global._51degrees.header_names[index].size = global._51degrees.header_names[index].len; + } +} +#endif + int init_51degrees(void) { int i = 0; @@ -254,6 +485,13 @@ int init_51degrees(void) switch (_51d_dataset_status) { case DATA_SET_INIT_STATUS_SUCCESS: +#ifdef FIFTYONEDEGREES_H_PATTERN_INCLUDED + /* only 1 workset in the pool because HAProxy is currently single threaded + * this value should be set to the number of threads in future versions. + */ + global._51degrees.pool = fiftyoneDegreesWorksetPoolCreate(&global._51degrees.data_set, NULL, 1); +#endif + _51d_init_http_headers(); break; case DATA_SET_INIT_STATUS_INSUFFICIENT_MEMORY: chunk_printf(temp, "Insufficient memory."); @@ -290,9 +528,12 @@ int init_51degrees(void) } free(_51d_property_list); +#ifdef FIFTYONEDEGREES_H_PATTERN_INCLUDED _51d_lru_seed = random(); - if (global._51degrees.cache_size) + if (global._51degrees.cache_size) { _51d_lru_tree = lru64_new(global._51degrees.cache_size); + } +#endif return 0; } @@ -301,10 +542,14 @@ void deinit_51degrees(void) { struct _51d_property_names *_51d_prop_name, *_51d_prop_nameb; + free(global._51degrees.header_names); #ifdef FIFTYONEDEGREES_H_PATTERN_INCLUDED - fiftyoneDegreesDestroy(&global._51degrees.data_set); + fiftyoneDegreesWorksetPoolFree(global._51degrees.pool); + fiftyoneDegreesDataSetFree(&global._51degrees.data_set); #endif #ifdef FIFTYONEDEGREES_H_TRIE_INCLUDED + free(global._51degrees.device_offsets.firstOffset); + free(global._51degrees.header_offsets); fiftyoneDegreesDestroy(); #endif @@ -314,7 +559,9 @@ void deinit_51degrees(void) free(_51d_prop_name); } +#ifdef FIFTYONEDEGREES_H_PATTERN_INCLUDED while (lru64_destroy(_51d_lru_tree)); +#endif } static struct cfg_kw_list _51dcfg_kws = {{ }, { @@ -325,16 +572,23 @@ static struct cfg_kw_list _51dcfg_kws = {{ }, { { 0, NULL, NULL }, }}; +/* Note: must not be declared as its list will be overwritten */ +static struct sample_fetch_kw_list sample_fetch_keywords = {ILH, { + { "51d.all", _51d_fetch, ARG5(1,STR,STR,STR,STR,STR), _51d_fetch_check, SMP_T_STR, SMP_USE_HRQHV }, + { NULL, NULL, 0, 0, 0 }, +}}; + /* Note: must not be declared as its list will be overwritten */ static struct sample_conv_kw_list conv_kws = {ILH, { - { "51d", _51d_conv, ARG5(1,STR,STR,STR,STR,STR), _51d_conv_check, SMP_T_STR, SMP_T_STR }, + { "51d.single", _51d_conv, ARG5(1,STR,STR,STR,STR,STR), _51d_conv_check, SMP_T_STR, SMP_T_STR }, { NULL, NULL, 0, 0, 0 }, }}; __attribute__((constructor)) static void __51d_init(void) { - /* register sample fetch and format conversion keywords */ + /* register sample fetch and conversion keywords */ + sample_register_fetches(&sample_fetch_keywords); sample_register_convs(&conv_kws); cfg_register_keywords(&_51dcfg_kws); }