diff --git a/include/proxy/hdrs/HdrToken.h b/include/proxy/hdrs/HdrToken.h index 9d862bb4eae..fac37ef96ec 100644 --- a/include/proxy/hdrs/HdrToken.h +++ b/include/proxy/hdrs/HdrToken.h @@ -108,10 +108,13 @@ extern HdrTokenInfoFlags hdrtoken_str_flags[]; // //////////////////////////////////////////////////////////////////////////// -extern void hdrtoken_init(); -extern int hdrtoken_tokenize_dfa(const char *string, int string_len, const char **wks_string_out = nullptr); -extern int hdrtoken_tokenize(const char *string, int string_len, const char **wks_string_out = nullptr); -extern int hdrtoken_method_tokenize(const char *string, int string_len); +extern void hdrtoken_init(); +extern int hdrtoken_tokenize_dfa(const char *string, int string_len, const char **wks_string_out = nullptr); +extern int hdrtoken_tokenize(const char *string, int string_len, const char **wks_string_out = nullptr, + uint32_t *hash_out = nullptr); +extern int hdrtoken_tokenize_prehashed(const char *string, int string_len, uint32_t hash, const char **wks_string_out = nullptr); +extern int hdrtoken_field_name_scan(const char *string, int maxlen, uint32_t *hash_out, bool *all_valid_out); +extern int hdrtoken_method_tokenize(const char *string, int string_len); extern const char *hdrtoken_string_to_wks(const char *string); extern const char *hdrtoken_string_to_wks(const char *string, int length); extern c_str_view hdrtoken_string_to_wks_sv(const char *string); diff --git a/include/proxy/hdrs/MIME.h b/include/proxy/hdrs/MIME.h index 7605fc0640b..bc3254b87db 100644 --- a/include/proxy/hdrs/MIME.h +++ b/include/proxy/hdrs/MIME.h @@ -491,9 +491,24 @@ MIMEScanner::clear() struct MIMEParser { MIMEScanner m_scanner; - int32_t m_field; - int m_field_flags; - int m_value; + + // Parse-local duplicate filter for non-well-known field names. A Bloom of + // name hashes lets field attach skip its O(n) duplicate search when a name is + // provably new (bit clear => no earlier field can share it). Seeded lazily + // from the fields already in the header the first time the parser touches it, + // so it stays correct when parsing into a non-empty header and when a parser + // is reused on a different header without an intervening clear. m_dup_seed_mh + // records which header the Bloom currently reflects. 128 bits (two words) to + // keep the false-positive rate low; the low 7 hash bits pick word and bit. + uint64_t m_dup_bloom[2]; + MIMEHdrImpl *m_dup_seed_mh; + + // Most-recently attached field, used to splice a duplicate of the + // immediately-preceding field directly onto its dup chain's tail in O(1), + // skipping attach's O(n) duplicate search. Persists across CONT re-entry; + // reset per parse. The tail-append predicate is self-validating, so a stale + // value can never cause incorrect linkage (it just falls back to attach). + MIMEField *m_last_attached; }; /*********************************************************************** diff --git a/src/proxy/hdrs/HTTP.cc b/src/proxy/hdrs/HTTP.cc index e00c9d13e89..f36345ddb4a 100644 --- a/src/proxy/hdrs/HTTP.cc +++ b/src/proxy/hdrs/HTTP.cc @@ -1118,29 +1118,15 @@ http_parser_parse_req(HTTPParser *parser, HdrHeap *heap, HTTPHdrImpl *hh, const ParseResult validate_hdr_request_target(int method_wk_idx, URLImpl *url) { - ParseResult ret = ParseResult::DONE; - auto host{url->get_host()}; - auto path{url->get_path()}; - auto scheme{url->get_scheme()}; - - if (host.empty()) { - if (path == "*"sv) { // asterisk-form - // Skip this check for now because URLImpl can't distinguish '*' and '/*' - // if (method_wk_idx != HTTP_WKSIDX_OPTIONS) { - // ret = ParseResult::ERROR; - // } - } else { // origin-form - // Nothing to check here - } - } else if (scheme.empty() && !host.empty()) { // authority-form - if (method_wk_idx != HTTP_WKSIDX_CONNECT) { - ret = ParseResult::ERROR; - } - } else { // absolute-form - // Nothing to check here + // The only rejected request-target is authority-form (host present, scheme + // absent) with a method other than CONNECT. get_scheme() is empty iff no + // well-known scheme index is set and the inline scheme length is zero; the + // asterisk-form check is intentionally disabled (URLImpl can't distinguish + // '*' from '/*'), so origin-, asterisk-, and absolute-form all accept. + if (url->m_len_host != 0 && url->m_scheme_wks_idx < 0 && url->m_len_scheme == 0 && method_wk_idx != HTTP_WKSIDX_CONNECT) { + return ParseResult::ERROR; } - - return ret; + return ParseResult::DONE; } bool diff --git a/src/proxy/hdrs/HdrToken.cc b/src/proxy/hdrs/HdrToken.cc index 6d2beabfec9..2a072e621d4 100644 --- a/src/proxy/hdrs/HdrToken.cc +++ b/src/proxy/hdrs/HdrToken.cc @@ -25,7 +25,9 @@ #include "tscore/HashFNV.h" #include "tscore/Diags.h" #include "tscore/ink_memory.h" +#include #include +#include #include "tscore/Allocator.h" #include "proxy/hdrs/HTTP.h" #include "proxy/hdrs/HdrToken.h" @@ -53,7 +55,7 @@ DbgCtl dbg_ctl_hdr_token{"hdr_token"}; */ -const char *const _hdrtoken_strs[] = { +constexpr const char *const _hdrtoken_strs[] = { // MIME Field names "Accept-Charset", "Accept-Encoding", "Accept-Language", "Accept-Ranges", "Accept", "Age", "Allow", "Approved", // NNTP @@ -310,20 +312,84 @@ HdrTokenHashBucket hdrtoken_hash_table[HDRTOKEN_HASH_TABLE_SIZE]; **/ #define TINY_MASK(x) (((uint32_t)1 << (x)) - 1) -inline uint32_t +constexpr uint32_t hash_to_slot(uint32_t hash) { return ((hash >> 15) ^ hash) & TINY_MASK(15); } -inline uint32_t +// Branchless ASCII upper-fold: 'a'..'z' -> 'A'..'Z', every other byte (including +// 0x80-0xFF) unchanged. Matches libc toupper() under the C locale for all 256 +// byte values but is locale-independent, which is the correct behavior for +// case-insensitive matching of ASCII HTTP tokens. +static constexpr unsigned char +hdrtoken_ascii_toupper(unsigned char c) +{ + unsigned char const is_lower = static_cast((static_cast(c) - 'a') < 26u); + return static_cast(c - (is_lower << 5)); +} + +// FNV-1a 32-bit over ASCII-case-folded field-name bytes. Both hdrtoken_hash and +// the single-pass field-name scan fold their bytes through hdrtoken_hash_step so +// the seed and per-byte step live in one place. The fold matches toupper() under +// the C locale for all byte values, so the well-known-string table's hash values +// and collision pattern are identical to the previous ATSHash32FNV1a + +// ATSHash::nocase implementation, but locale-independent and call-free. +static constexpr uint32_t HDRTOKEN_HASH_SEED = 0x811c9dc5u; // FNV-1a 32-bit offset basis + +static constexpr uint32_t +hdrtoken_hash_step(uint32_t hval, unsigned char c) +{ + return (hval ^ hdrtoken_ascii_toupper(c)) * 0x01000193u; // fold byte, xor in, multiply by the FNV-1a prime +} + +constexpr uint32_t hdrtoken_hash(const unsigned char *string, unsigned int length) { - ATSHash32FNV1a fnv; - fnv.update(string, length, ATSHash::nocase()); - fnv.final(); - return fnv.get(); + uint32_t hval = HDRTOKEN_HASH_SEED; + for (unsigned int i = 0; i < length; ++i) { + hval = hdrtoken_hash_step(hval, string[i]); + } + return hval; +} + +// Compile-time slot of a NUL-terminated well-known string. Walking to the NUL +// (rather than reinterpret_cast + length, which a constant +// expression cannot do) lets this share hdrtoken_hash_step with hdrtoken_hash, so +// the slot it computes is exactly the one hdrtoken_hash_init assigns. +static constexpr uint32_t +hdrtoken_literal_slot(const char *s) +{ + uint32_t hval = HDRTOKEN_HASH_SEED; + for (; *s != '\0'; ++s) { + hval = hdrtoken_hash_step(hval, static_cast(*s)); + } + return hash_to_slot(hval); +} + +// Every well-known string must map to a distinct hash slot, or hdrtoken_hash_init +// would overwrite one bucket with another and silently lose a token. Enforce it at +// build time so a colliding table -- from editing the string list or the hash -- +// is a compile error rather than a startup abort. +static constexpr bool +hdrtoken_wks_slots_unique() +{ + // hash_to_slot yields a 15-bit slot: mark each well-known string's slot, and a + // repeat is a collision. Kept within the core-language constexpr subset -- + // std::sort/std::adjacent_find are only constexpr in libstdc++ 12+, and ATS + // still builds against older standard libraries. + std::array seen{}; + for (const char *const s : _hdrtoken_strs) { + uint32_t const slot = hdrtoken_literal_slot(s); + if (seen[slot]) { + return false; + } + seen[slot] = true; + } + return true; } +static_assert(hdrtoken_wks_slots_unique(), + "two well-known strings hash to the same slot; hdrtoken_hash_init would drop one -- change the table or the hash"); /*------------------------------------------------------------------------- -------------------------------------------------------------------------*/ @@ -331,13 +397,12 @@ hdrtoken_hash(const unsigned char *string, unsigned int length) void hdrtoken_hash_init() { - uint32_t i; - int num_collisions; - memset(hdrtoken_hash_table, 0, sizeof(hdrtoken_hash_table)); - num_collisions = 0; - for (i = 0; i < static_cast SIZEOF(_hdrtoken_strs); i++) { + // Slot uniqueness across the well-known strings is guaranteed at build time by + // static_assert(hdrtoken_wks_slots_unique()), so no bucket is ever assigned + // twice below. + for (uint32_t i = 0; i < static_cast SIZEOF(_hdrtoken_strs); i++) { // convert the common string to the well-known token unsigned const char *wks; int wks_idx = @@ -347,18 +412,9 @@ hdrtoken_hash_init() uint32_t hash = hdrtoken_hash(wks, hdrtoken_str_lengths[wks_idx]); uint32_t slot = hash_to_slot(hash); - if (hdrtoken_hash_table[slot].wks) { - printf("ERROR: hdrtoken_hash_table[%u] collision: '%s' replacing '%s'\n", slot, reinterpret_cast(wks), - hdrtoken_hash_table[slot].wks); - ++num_collisions; - } hdrtoken_hash_table[slot].wks = reinterpret_cast(wks); hdrtoken_hash_table[slot].hash = hash; } - - if (num_collisions > 0) { - abort(); - } } /*********************************************************************** @@ -531,36 +587,104 @@ hdrtoken_method_tokenize(const char *string, int string_len) /*------------------------------------------------------------------------- -------------------------------------------------------------------------*/ +// WKS lookup for a name whose FNV-1a hash the caller has already computed +// (e.g. fused into the field-name scan). Does the slot/length narrowing plus +// the exact ASCII-case-insensitive byte compare, but no hashing and no +// interned-pointer test, so it is only valid for a non-interned `string`. int -hdrtoken_tokenize(const char *string, int string_len, const char **wks_string_out) +hdrtoken_tokenize_prehashed(const char *string, int string_len, uint32_t hash, const char **wks_string_out) { - int wks_idx; - HdrTokenHashBucket *bucket; + uint32_t slot = hash_to_slot(hash); + HdrTokenHashBucket *bucket = &(hdrtoken_hash_table[slot]); + if ((bucket->wks != nullptr) && (bucket->hash == hash) && (hdrtoken_wks_to_length(bucket->wks) == string_len)) { + // hash + length narrow to a single WKS candidate, but a 32-bit hash collision + // of equal length would otherwise mis-intern an arbitrary name (giving it the + // WKS's presence bits / slot accelerators). Confirm with an exact + // ASCII-case-insensitive byte comparison before accepting the WKS. + // Browsers send canonical-case field names that match the WKS bytes + // exactly, so try a fast exact compare first; memcmp-equal implies + // fold-equal. The per-byte fold still runs on a miss to catch case + // variants (e.g. "HOST", "host"). + bool matches = (memcmp(string, bucket->wks, string_len) == 0); + if (!matches) { + matches = true; + for (int i = 0; i < string_len; ++i) { + if (hdrtoken_ascii_toupper(static_cast(string[i])) != + hdrtoken_ascii_toupper(static_cast(bucket->wks[i]))) { + matches = false; + break; + } + } + } + if (matches) { + int wks_idx = hdrtoken_wks_to_index(bucket->wks); + if (wks_string_out) { + *wks_string_out = bucket->wks; + } + return wks_idx; + } + } + + Dbg(dbg_ctl_hdr_token, "Did not find a WKS for '%.*s'", string_len, string); + return -1; +} + +// Single-pass field-name scan for the MIME parser. Scans up to `maxlen` bytes +// of `string` for the ':' delimiter while, in the same pass, accumulating the +// FNV-1a name hash (identical to hdrtoken_hash) and tracking whether every byte +// before ':' is a valid HTTP field-name char. Returns the index of ':' (i.e. +// the field-name length) or -1 if no ':' appears within `maxlen`. `*hash_out` +// and `*all_valid_out` describe the bytes scanned before ':' (or all `maxlen` +// bytes when ':' is absent). +int +hdrtoken_field_name_scan(const char *string, int maxlen, uint32_t *hash_out, bool *all_valid_out) +{ + uint32_t hval = HDRTOKEN_HASH_SEED; // same FNV-1a name hash as hdrtoken_hash + bool all_valid = true; + int i = 0; + + for (; i < maxlen; ++i) { + unsigned char const uc = static_cast(string[i]); + if (uc == ':') { + break; + } + hval = hdrtoken_hash_step(hval, uc); + all_valid &= (ParseRules::is_http_field_name(static_cast(uc)) != 0); + } + + *hash_out = hval; + *all_valid_out = all_valid; + return (i < maxlen) ? i : -1; +} + +int +hdrtoken_tokenize(const char *string, int string_len, const char **wks_string_out, uint32_t *hash_out) +{ ink_assert(string != nullptr); if (hdrtoken_is_wks(string)) { - wks_idx = hdrtoken_wks_to_index(string); + int wks_idx = hdrtoken_wks_to_index(string); if (wks_string_out) { *wks_string_out = string; } + // Keep hash_out total: an interned name is matched by its WKS index, not by + // hash, so none is computed here. A caller that needs a name hash for an + // already-interned string must call hdrtoken_hash() directly. + if (hash_out) { + *hash_out = 0; + } return wks_idx; } uint32_t hash = hdrtoken_hash(reinterpret_cast(string), static_cast(string_len)); - uint32_t slot = hash_to_slot(hash); - - bucket = &(hdrtoken_hash_table[slot]); - if ((bucket->wks != nullptr) && (bucket->hash == hash) && (hdrtoken_wks_to_length(bucket->wks) == string_len)) { - wks_idx = hdrtoken_wks_to_index(bucket->wks); - if (wks_string_out) { - *wks_string_out = bucket->wks; - } - return wks_idx; + // Hand the caller the name hash it can reuse (e.g. a parse-local duplicate + // filter) so it need not recompute it. + if (hash_out) { + *hash_out = hash; } - Dbg(dbg_ctl_hdr_token, "Did not find a WKS for '%.*s'", string_len, string); - return -1; + return hdrtoken_tokenize_prehashed(string, string_len, hash, wks_string_out); } /*------------------------------------------------------------------------- diff --git a/src/proxy/hdrs/MIME.cc b/src/proxy/hdrs/MIME.cc index f8e25de040b..0d24fd6a3f5 100644 --- a/src/proxy/hdrs/MIME.cc +++ b/src/proxy/hdrs/MIME.cc @@ -2365,9 +2365,28 @@ MIMEScanner::get(TextView &input, TextView &output, bool &output_shares_input, b void _mime_parser_init(MIMEParser *parser) { - parser->m_field = 0; - parser->m_field_flags = 0; - parser->m_value = -1; + parser->m_dup_bloom[0] = 0; + parser->m_dup_bloom[1] = 0; + parser->m_dup_seed_mh = nullptr; + parser->m_last_attached = nullptr; +} + +// Seed the parser's non-WKS duplicate Bloom from any fields already in the +// header. A no-op for the common fresh-header parse; keeps the skip-the-search +// optimization correct if parsing appends to a header that already has fields. +static void +mime_parser_seed_dup_bloom(MIMEParser *parser, MIMEHdrImpl *mh) +{ + for (MIMEFieldBlockImpl *fblock = &mh->m_first_fblock; fblock != nullptr; fblock = fblock->m_next) { + for (uint32_t i = 0; i < fblock->m_freetop; ++i) { + MIMEField *f = &fblock->m_field_slots[i]; + if (f->is_live() && f->m_wks_idx < 0) { + uint32_t h = 0; + hdrtoken_tokenize(f->m_ptr_name, f->m_len_name, nullptr, &h); + parser->m_dup_bloom[(h >> 6) & 1] |= (static_cast(1) << (h & 63)); + } + } + } } ////////////////////////////////////////////////////// // init first time structure setup // @@ -2436,11 +2455,23 @@ mime_parser_parse(MIMEParser *parser, HdrHeap *heap, MIMEHdrImpl *mh, const char } // find name last - auto field_value = parsed; // need parsed as is later on. - auto field_name = field_value.split_prefix_at(':'); - if (field_name.empty()) { + // + // Fuse the colon scan, FNV-1a name hash, and per-byte field-name validation + // into one pass over the name bytes. hdrtoken_field_name_scan returns the + // colon index (the name length) and, for those bytes, the hash reused below + // by the WKS lookup and the duplicate Bloom, plus whether every byte is a + // valid HTTP field-name char. + auto field_value = parsed; // need parsed as is later on. + uint32_t field_name_hash; + bool name_all_valid; + int colon_idx = hdrtoken_field_name_scan(parsed.data(), static_cast(parsed.size()), &field_name_hash, &name_all_valid); + if (colon_idx <= 0) { + // colon_idx < 0: no colon; colon_idx == 0: empty name. Both are garbage, + // matching the old empty-field_name toss. continue; // toss away garbage line } + auto field_name = parsed.prefix(colon_idx); + field_value.remove_prefix(colon_idx + 1); // RFC7230 section 3.2.4: // No whitespace is allowed between the header field-name and colon. In @@ -2452,12 +2483,15 @@ mime_parser_parse(MIMEParser *parser, HdrHeap *heap, MIMEHdrImpl *mh, const char // A proxy MUST remove any such whitespace from a response message before // forwarding the message downstream. bool raw_print_field = true; + bool name_scan_stale = false; if (is_ws(field_name.back())) { if (!remove_ws_from_field_name) { return ParseResult::ERROR; } field_name.rtrim_if(&ParseRules::is_ws); raw_print_field = false; + // The fused scan hashed and validated the untrimmed name; recompute below. + name_scan_stale = true; } else if (parsed.suffix(2) != "\r\n") { raw_print_field = false; } @@ -2487,14 +2521,24 @@ mime_parser_parse(MIMEParser *parser, HdrHeap *heap, MIMEHdrImpl *mh, const char // tokenize the name // /////////////////////// - int field_name_wks_idx = hdrtoken_tokenize(field_name.data(), field_name.size()); - - if (field_name_wks_idx < 0) { - for (auto i : field_name) { - if (!ParseRules::is_http_field_name(i)) { - return ParseResult::ERROR; + int field_name_wks_idx; + if (name_scan_stale) { + // BWS trimming shortened the name after the fused scan; redo the hash, + // WKS lookup, and byte validation over the trimmed name. + field_name_hash = 0; + field_name_wks_idx = hdrtoken_tokenize(field_name.data(), field_name.size(), nullptr, &field_name_hash); + if (field_name_wks_idx < 0) { + for (auto i : field_name) { + if (!ParseRules::is_http_field_name(i)) { + return ParseResult::ERROR; + } } } + } else { + field_name_wks_idx = hdrtoken_tokenize_prehashed(field_name.data(), static_cast(field_name.size()), field_name_hash); + if ((field_name_wks_idx < 0) && !name_all_valid) { + return ParseResult::ERROR; + } } // RFC 9110 Section 5.5. Field Values @@ -2511,7 +2555,91 @@ mime_parser_parse(MIMEParser *parser, HdrHeap *heap, MIMEHdrImpl *mh, const char MIMEField *field = mime_field_create(heap, mh); mime_field_name_value_set(heap, mh, field, field_name_wks_idx, field_name, field_value, raw_print_field, parsed.size(), false); - mime_hdr_field_attach(mh, field, 1, nullptr); + + // For a non-WKS name, consult the parse-local Bloom: if its hash bit is + // clear no earlier field can share the name, so skip attach's O(n) duplicate + // search (check_for_dups=0 takes the same no-dup path a null find would). + // For a WKS name the presence bit gives the same guarantee for free: a clear + // bit is exactly the first negative test mime_hdr_field_find performs for an + // interned name, so skipping the walk is byte-for-byte equivalent to a null + // find. Mask-zero WKS names (no presence bit) must keep the search on. + int check_for_dups = 1; + if (field_name_wks_idx < 0) { + // Seed (or reseed) the Bloom the first time this parser touches a given + // header. Keying on the header pointer keeps a reused parser correct when + // it is pointed at a different header without an intervening clear. + if (parser->m_dup_seed_mh != mh) { + parser->m_dup_bloom[0] = 0; + parser->m_dup_bloom[1] = 0; + mime_parser_seed_dup_bloom(parser, mh); + parser->m_dup_seed_mh = mh; + } + unsigned const word = (field_name_hash >> 6) & 1; + uint64_t const bit = static_cast(1) << (field_name_hash & 63); + if ((parser->m_dup_bloom[word] & bit) == 0) { + check_for_dups = 0; + } + parser->m_dup_bloom[word] |= bit; + } else { + uint64_t const mask = hdrtoken_index_to_mask(field_name_wks_idx); + if (mask != 0 && (mh->m_presence_bits & mask) == 0) { + check_for_dups = 0; + } + } + + // O(1) tail append for a duplicate of the immediately-preceding attached + // field (e.g. consecutive Set-Cookie). mime_field_create always hands out + // the highest slot, so a duplicate of the last-attached field belongs at + // its dup chain's tail. When that field is still its chain tail + // (m_next_dup == nullptr), is live, is in this header, and shares this + // field's name, splice directly instead of running attach's O(n) duplicate + // search. Under the "one chain per name, in slot order" invariant such a + // field is necessarily that name's chain tail, so the splice is correct + // regardless of how m_last_attached was set -- it stays safe across CONT + // re-entry and parser reuse (a mismatch just falls back to attach). + bool fast_tail_append = false; + MIMEField *const last = parser->m_last_attached; + + if (last != nullptr && last->is_live() && last->m_next_dup == nullptr) { + bool name_matches; + + if (field_name_wks_idx >= 0) { + name_matches = (last->m_wks_idx == field_name_wks_idx); + } else { + name_matches = + (last->m_wks_idx < 0) && + ts::iequals(std::string_view{last->m_ptr_name, static_cast(last->m_len_name)}, field_name); + } + + if (name_matches) { + bool in_mh = false; + + for (MIMEFieldBlockImpl *fb = &mh->m_first_fblock; fb != nullptr; fb = fb->m_next) { + if (fb->contains(last)) { + in_mh = true; + break; + } + } + + if (in_mh) { + field->m_readiness = MIME_FIELD_SLOT_READINESS_LIVE; + field->m_flags = (field->m_flags & ~MIME_FIELD_SLOT_FLAGS_DUP_HEAD); + field->m_next_dup = nullptr; + last->m_next_dup = field; + // Presence bit and slot accelerator were set by the chain head; a tail + // dup leaves them untouched, matching attach's patch-after-prev branch. + if (field->m_ptr_value && field->is_cooked()) { + mh->recompute_cooked_stuff(field); + } + fast_tail_append = true; + } + } + } + + if (!fast_tail_append) { + mime_hdr_field_attach(mh, field, check_for_dups, nullptr); + } + parser->m_last_attached = field; } } diff --git a/src/proxy/hdrs/URL.cc b/src/proxy/hdrs/URL.cc index 68d84b5d481..f8c2bc70f12 100644 --- a/src/proxy/hdrs/URL.cc +++ b/src/proxy/hdrs/URL.cc @@ -1172,17 +1172,18 @@ url_is_strictly_compliant(const char *start, const char *end) bool url_is_mostly_compliant(const char *start, const char *end) { + // Mode 2 accepts exactly the printable, non-space ASCII range 0x21..0x7E -- + // equivalent to the previous isspace()/isprint() pair, but locale-independent + // and call-free. This runs on every request target under the default + // strict_uri_parsing=2. OR-reducing an out-of-range flag over the whole target + // (no early exit, no data-dependent branch) lets the compiler auto-vectorize + // the scan to the build's SIMD; ATS builds -O3, where clang and GCC both do. + unsigned char bad = 0; for (const char *i = start; i < end; ++i) { - if (isspace(*i)) { - Dbg(dbg_ctl_http, "Whitespace character [0x%.2X] found in URL", static_cast(*i)); - return false; - } - if (!isprint(*i)) { - Dbg(dbg_ctl_http, "Non-printable character [0x%.2X] found in URL", static_cast(*i)); - return false; - } + unsigned char const c = static_cast(*i); + bad |= static_cast((c < 0x21) | (c > 0x7E)); } - return true; + return bad == 0; } } // namespace UrlImpl diff --git a/src/proxy/hdrs/unit_tests/test_URL.cc b/src/proxy/hdrs/unit_tests/test_URL.cc index dc5ff4ade74..7b2d9853e75 100644 --- a/src/proxy/hdrs/unit_tests/test_URL.cc +++ b/src/proxy/hdrs/unit_tests/test_URL.cc @@ -170,6 +170,63 @@ TEST_CASE("ParseRulesMostlyStrictURI", "[proxy][parseuri]") CHECK(url_is_mostly_compliant(i.uri, i.uri + strlen(i.uri)) == i.valid); } +namespace +{ +// Scalar reference: mode-2 compliance accepts exactly bytes 0x21..0x7E. +bool +url_mostly_compliant_reference(const char *start, const char *end) +{ + for (const char *p = start; p < end; ++p) { + unsigned char const c = static_cast(*p); + if (static_cast(c - 0x21u) > 0x5Du) { + return false; + } + } + return true; +} +} // namespace + +TEST_CASE("MostlyCompliantVsScalar", "[proxy][parseuri]") +{ + // The auto-vectorized url_is_mostly_compliant must agree with the scalar + // reference for every input. Exhaustively inject each of the 256 byte values + // at each position across lengths 1..40, which span a sub-vector target, a + // whole vector, and the scalar remainder for both 128- and 256-bit builds. + char buf[64]; + int mismatches = 0, first_len = 0, first_pos = 0, first_byte = 0; + + for (int len = 1; len <= 40; ++len) { + for (int pos = 0; pos < len; ++pos) { + for (int b = 0; b < 256; ++b) { + memset(buf, 'a', len); // 'a' (0x61) is compliant + buf[pos] = static_cast(b); + bool const got = url_is_mostly_compliant(buf, buf + len); + bool const want = url_mostly_compliant_reference(buf, buf + len); + if (got != want) { + if (mismatches == 0) { + first_len = len; + first_pos = pos; + first_byte = b; + } + ++mismatches; + } + } + } + } + CAPTURE(mismatches, first_len, first_pos, first_byte); + CHECK(mismatches == 0); + + // Boundary extremes at every length: all-0x7E accepts, all-0x7F rejects. + for (int len = 0; len <= 40; ++len) { + memset(buf, 0x7E, len); + CHECK(url_is_mostly_compliant(buf, buf + len) == true); + if (len > 0) { + memset(buf, 0x7F, len); + CHECK(url_is_mostly_compliant(buf, buf + len) == false); + } + } +} + struct url_parse_test_case { const std::string input_uri; const std::string expected_printed_url; diff --git a/src/proxy/hdrs/unit_tests/test_mime.cc b/src/proxy/hdrs/unit_tests/test_mime.cc index 857f008ae0e..e5b3d89be5f 100644 --- a/src/proxy/hdrs/unit_tests/test_mime.cc +++ b/src/proxy/hdrs/unit_tests/test_mime.cc @@ -22,8 +22,12 @@ */ #include +#include +#include +#include #include +#include using namespace std::literals; @@ -31,6 +35,8 @@ using namespace std::literals; #include #include #include "tscore/ink_platform.h" +#include "tscore/ParseRules.h" +#include "proxy/hdrs/HdrToken.h" #include "proxy/hdrs/MIME.h" TEST_CASE("Mime", "[proxy][mime]") @@ -69,6 +75,186 @@ TEST_CASE("Mime", "[proxy][mime]") hdr.destroy(); } +TEST_CASE("MimeParserReuseAcrossHeaders", "[proxy][mimeparser]") +{ + // A parser reused on a different header without an intervening clear must + // still detect duplicates of fields already present in that header. The + // dup-skip Bloom is keyed to the header it was seeded from; without that, a + // stale seed latch would skip reseeding and a wire duplicate of a pre-existing + // custom field would be attached as an independent head rather than joining + // the dup chain. + MIMEParser parser; + mime_parser_init(&parser); + + // First header: parse a non-WKS field so the parser seeds its dup state. + MIMEHdr hdrA; + hdrA.create(nullptr); + { + std::string_view text = "X-Foo: 1\r\n\r\n"sv; + const char *start = text.data(); + REQUIRE(hdrA.parse(&parser, &start, text.data() + text.size(), true, false, false) == ParseResult::DONE); + } + + // Second header (different mh) already carries a live non-WKS field; reuse the + // same parser WITHOUT clearing it and parse a duplicate of that field. + MIMEHdr hdrB; + hdrB.create(nullptr); + MIMEField *pre = hdrB.field_create("X-Baz"sv); + pre->value_set(hdrB.m_heap, hdrB.m_mime, "a"sv); + hdrB.field_attach(pre); + { + std::string_view text = "X-Baz: b\r\n\r\n"sv; + const char *start = text.data(); + REQUIRE(hdrB.parse(&parser, &start, text.data() + text.size(), true, false, false) == ParseResult::DONE); + } + + // The wire field must have joined the pre-existing field's dup chain: exactly + // two X-Baz values reachable from the head. + MIMEField *head = hdrB.field_find("X-Baz"sv); + REQUIRE(head != nullptr); + int count = 0; + for (MIMEField *f = head; f != nullptr; f = f->m_next_dup) { + ++count; + } + CHECK(count == 2); + + mime_parser_clear(&parser); + hdrA.destroy(); + hdrB.destroy(); +} + +TEST_CASE("MimeParserTailAppendEquivalence", "[proxy][mimeparser]") +{ + // The O(1) adjacent-duplicate tail append must produce the same field/dup + // structure as attach's full duplicate search. Build the same field sequence + // two ways -- via the parser (which takes the tail-append path) and via + // explicit create+attach (the reference full-attach path) -- and compare the + // dup chain of every name. + struct Field { + const char *name; + const char *value; + }; + auto scenario = GENERATE(from_range(std::vector>{ + {{"X-A", "1"}, {"X-A", "2"}, {"X-A", "3"}}, // consecutive custom dups + {{"X-A", "1"}, {"X-B", "2"}, {"X-A", "3"}}, // interleaved + {{"X-A", "1"}, {"X-A", "2"}, {"X-B", "3"}, {"X-B", "4"}}, // two adjacent runs + {{"Set-Cookie", "a"}, {"Set-Cookie", "b"}, {"Set-Cookie", "c"}, {"Set-Cookie", "d"}}, // well-known dups + {{"X-A", "1"}, {"X-B", "2"}, {"X-C", "3"}}, // no dups + })); + + // Parser-built header (tail-append path). + std::string raw; + for (auto const &f : scenario) { + raw += f.name; + raw += ": "; + raw += f.value; + raw += "\r\n"; + } + raw += "\r\n"; + MIMEParser parser; + mime_parser_init(&parser); + MIMEHdr hdrA; + hdrA.create(nullptr); + { + const char *start = raw.data(); + REQUIRE(hdrA.parse(&parser, &start, raw.data() + raw.size(), true, false, false) == ParseResult::DONE); + } + mime_parser_clear(&parser); + + // Reference header via explicit create+attach (attach's full path, no tail append). + MIMEHdr hdrB; + hdrB.create(nullptr); + for (auto const &f : scenario) { + MIMEField *fld = hdrB.field_create(std::string_view{f.name}); + fld->value_set(hdrB.m_heap, hdrB.m_mime, std::string_view{f.value}); + hdrB.field_attach(fld); + } + + auto collect = [](MIMEHdr &h, std::string_view n) { + std::vector vals; + for (MIMEField *fld = h.field_find(n); fld != nullptr; fld = fld->m_next_dup) { + auto v = fld->value_get(); + vals.emplace_back(v.data(), v.size()); + } + return vals; + }; + + std::set names; + for (auto const &f : scenario) { + names.insert(f.name); + } + for (auto const &name : names) { + std::vector va = collect(hdrA, name); + std::vector vb = collect(hdrB, name); + CAPTURE(name, va.size(), vb.size()); + CHECK(va == vb); + } + + hdrA.destroy(); + hdrB.destroy(); +} + +TEST_CASE("HdrTokenFusedNameScanParity", "[proxy][hdrtoken]") +{ + // opt2 fused the colon scan, FNV hash, and field-name validation into + // hdrtoken_field_name_scan + hdrtoken_tokenize_prehashed. Verify the fused + // path agrees with references: the colon position, per-byte validity, and -- + // via the prehashed lookup fed the fused hash -- the well-known index the + // standalone tokenizer returns (which is a proxy for hash parity). + struct Case { + const char *name; + const char *tail; + }; + static const std::vector cases = { + {"Content-Length", ": 5" }, + {"content-length", ":5" }, + {"CONTENT-LENGTH", ":5" }, + {"Host", ": x" }, + {"hOsT", ":x" }, + {"Set-Cookie", ": a=b" }, + {"Cache-Control", ":no" }, + {"Transfer-Encoding", ":chunk" }, + {"@Ats-Internal", ":z" }, + {"X-Custom-Header", ": v" }, + {"sec-ch-ua", ": \"x\""}, + {"sec-fetch-mode", ":cors" }, + {"priority", ":u=1" }, + {"X-My-Header", ":v" }, + {"a", ":b" }, + }; + + for (auto const &c : cases) { + std::string const buf = std::string(c.name) + c.tail; + int const name_len = static_cast(strlen(c.name)); + uint32_t hash = 0; + bool valid = false; + int const colon = hdrtoken_field_name_scan(buf.data(), static_cast(buf.size()), &hash, &valid); + CAPTURE(c.name); + CHECK(colon == name_len); + + bool ref_valid = true; + for (int i = 0; i < name_len; ++i) { + if (!ParseRules::is_http_field_name(c.name[i])) { + ref_valid = false; + break; + } + } + CHECK(valid == ref_valid); + CHECK(hdrtoken_tokenize_prehashed(c.name, name_len, hash) == hdrtoken_tokenize(c.name, name_len)); + } + + // Edge cases: no colon, empty name, and an invalid byte in the name. + uint32_t h = 0; + bool v = false; + CHECK(hdrtoken_field_name_scan("NoColon", 7, &h, &v) < 0); + CHECK(hdrtoken_field_name_scan(":value", 6, &h, &v) == 0); + { + const char bad[] = {'X', '\x01', 'Y', ':', 'v'}; + CHECK(hdrtoken_field_name_scan(bad, 5, &h, &v) == 3); + CHECK(v == false); + } +} + TEST_CASE("MimeGetHostPortValues", "[proxy][mimeport]") { MIMEHdr hdr; diff --git a/tools/benchmark/CMakeLists.txt b/tools/benchmark/CMakeLists.txt index e58e8dcf26b..047e512e971 100644 --- a/tools/benchmark/CMakeLists.txt +++ b/tools/benchmark/CMakeLists.txt @@ -51,3 +51,9 @@ target_link_libraries( add_executable(benchmark_HuffmanDecode benchmark_HuffmanDecode.cc) target_link_libraries(benchmark_HuffmanDecode PRIVATE Catch2::Catch2WithMain lshpack) target_include_directories(benchmark_HuffmanDecode PRIVATE ${CMAKE_SOURCE_DIR}/lib) + +add_executable(benchmark_HdrParse benchmark_HdrParse.cc) +target_link_libraries( + benchmark_HdrParse PRIVATE Catch2::Catch2 ts::hdrs ts::inkevent libswoc::libswoc lshpack configmanager +) +target_include_directories(benchmark_HdrParse PRIVATE ${CMAKE_SOURCE_DIR}/lib) diff --git a/tools/benchmark/benchmark_HdrParse.cc b/tools/benchmark/benchmark_HdrParse.cc new file mode 100644 index 00000000000..39849b429e4 --- /dev/null +++ b/tools/benchmark/benchmark_HdrParse.cc @@ -0,0 +1,772 @@ +/** @file + + Micro-benchmark for HTTP header parsing (src/proxy/hdrs). + + Two modes in one binary: + + * A/B stats mode (default): Catch2 BENCHMARK cases produce mean/median/ + stddev per (target x corpus) so an optimization can be measured before + and after and guarded against regression. Run e.g.: + benchmark_HdrParse "[bench]" --benchmark-samples 100 + + * Profiling mode (--profile ): a tight fixed-count loop with no + Catch2 harness overhead, meant to be wrapped by perf/vtune: + perf stat -e cycles,instructions \ + benchmark_HdrParse --profile request --iters 5000000 + + Targets: request, response, mime, url, wks. + + A realistic + adversarial corpus is built in. Real captured header blocks can + be supplied with --corpus-file FILE or --corpus-dir DIR (blocks split on a + blank line, classified request vs response by the first line); these are added + to both modes. + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include "proxy/hdrs/HTTP.h" +#include "proxy/hdrs/MIME.h" +#include "proxy/hdrs/URL.h" +#include "proxy/hdrs/HdrToken.h" +#include "proxy/hdrs/HdrHeap.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define CATCH_CONFIG_ENABLE_BENCHMARKING +#include +#include +#include + +// Defined in tscore; disables thread-local proxy allocators so no event-system +// thread setup is required (mirrors the hdrs unit-test main). +extern int cmd_disable_pfreelist; + +namespace +{ +// ---------------------------------------------------------------------------- +// Corpus +// ---------------------------------------------------------------------------- + +struct HeaderCase { + std::string label; + std::string data; // full block including the start line, ends "\r\n\r\n" + bool is_response = false; +}; + +struct Corpus { + std::vector cases; + std::vector urls; // bare request targets for the url target + std::vector wks; // field names (owned) for the wks target +}; + +// A representative modern browser request and typical responses, alongside the +// classic fixtures used by the hdrs unit tests. +constexpr std::string_view REQ_REALISTIC = + "GET /assets/app.9f2c.js HTTP/1.1\r\n" + "Host: www.example.com\r\n" + "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/126.0.0.0 Safari/537.36\r\n" + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8\r\n" + "Accept-Encoding: gzip, deflate, br, zstd\r\n" + "Accept-Language: en-US,en;q=0.9\r\n" + "Cookie: session=8f14e45fceea167a5a36dedd4bea2543; theme=dark; region=us-west-2; ab_bucket=37\r\n" + "Referer: https://www.example.com/\r\n" + "Connection: keep-alive\r\n" + "\r\n"; + +// A 2026 Chrome navigation request. Unlike REQ_REALISTIC (all classic WKS +// names), this carries the modern client-hint / fetch-metadata headers +// (sec-ch-ua*, sec-fetch-*, priority, upgrade-insecure-requests) that are NOT +// well-known strings, so it exercises the non-WKS name paths (tokenize miss, +// name validation, duplicate filter) the way real traffic does. +constexpr std::string_view REQ_MODERN = + "GET /app/feed?tab=home HTTP/1.1\r\n" + "Host: www.example.com\r\n" + "Connection: keep-alive\r\n" + "sec-ch-ua: \"Chromium\";v=\"126\", \"Google Chrome\";v=\"126\", \"Not-A.Brand\";v=\"99\"\r\n" + "sec-ch-ua-mobile: ?0\r\n" + "sec-ch-ua-platform: \"Windows\"\r\n" + "upgrade-insecure-requests: 1\r\n" + "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/126.0.0.0 Safari/537.36\r\n" + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8\r\n" + "Sec-Fetch-Site: same-origin\r\n" + "Sec-Fetch-Mode: navigate\r\n" + "Sec-Fetch-User: ?1\r\n" + "Sec-Fetch-Dest: document\r\n" + "Accept-Encoding: gzip, deflate, br, zstd\r\n" + "Accept-Language: en-US,en;q=0.9\r\n" + "Cookie: session=8f14e45fceea167a5a36dedd4bea2543; theme=dark; region=us-west-2; ab_bucket=37\r\n" + "Priority: u=0, i\r\n" + "\r\n"; + +constexpr std::string_view REQ_CLASSIC = "GET http://www.news.com:80/ HTTP/1.0\r\n" + "Proxy-Connection: Keep-Alive\r\n" + "User-Agent: Mozilla/4.04 [en] (X11; I; Linux 2.0.33 i586)\r\n" + "Pragma: no-cache\r\n" + "Host: www.news.com\r\n" + "Accept: image/gif, image/x-xbitmap, image/jpeg, image/pjpeg, image/png, */*\r\n" + "Accept-Language: en\r\n" + "Accept-Charset: iso-8859-1, *, utf-8\r\n" + "\r\n"; + +constexpr std::string_view RESP_REALISTIC = "HTTP/1.1 200 OK\r\n" + "Server: ATS/10.1.0\r\n" + "Date: Mon, 21 Oct 2013 20:13:21 GMT\r\n" + "Content-Type: text/html; charset=utf-8\r\n" + "Content-Length: 12345\r\n" + "Cache-Control: max-age=31536000, public, immutable\r\n" + "Vary: Accept-Encoding\r\n" + "Age: 42\r\n" + "\r\n"; + +constexpr std::string_view RESP_304 = "HTTP/1.1 304 Not Modified\r\n" + "Date: Mon, 21 Oct 2013 20:13:21 GMT\r\n" + "Etag: \"6f2c9a1b\"\r\n" + "Cache-Control: max-age=31536000\r\n" + "\r\n"; + +constexpr std::string_view URL_REALISTIC = "http://www.example.com/images/2026/06/some-article/hero.webp?w=1200&q=75"; + +// Build the adversarial cases programmatically so their worst-case sizes are +// obvious and easy to tune. +std::string +gen_many_fields(int n) +{ + std::string s = "GET /many HTTP/1.1\r\nHost: h\r\n"; + for (int i = 0; i < n; ++i) { + s += "X-Custom-Header-" + std::to_string(i) + ": value-" + std::to_string(i) + "\r\n"; + } + s += "\r\n"; + return s; +} + +std::string +gen_long_value(int len) +{ + std::string s = "GET /long HTTP/1.1\r\nHost: h\r\nX-Blob: "; + s.append(len, 'a'); + s += "\r\n\r\n"; + return s; +} + +// Field names that are guaranteed not to be well-known, forcing the +// hdrtoken_tokenize hash miss + per-char field-name validation path. +std::string +gen_wks_miss(int n) +{ + std::string s = "GET /miss HTTP/1.1\r\nHost: h\r\n"; + for (int i = 0; i < n; ++i) { + s += "X-Zzq-Nonstandard-Field-" + std::to_string(i) + ": v\r\n"; + } + s += "\r\n"; + return s; +} + +// Many duplicates of a well-known, commonly-repeated field: stresses +// mime_hdr_field_attach duplicate chaining. +std::string +gen_dup_fields(int n) +{ + std::string s = "HTTP/1.1 200 OK\r\nDate: Mon, 21 Oct 2013 20:13:21 GMT\r\n"; + for (int i = 0; i < n; ++i) { + s += "Set-Cookie: c" + std::to_string(i) + "=v" + std::to_string(i) + "; Path=/; HttpOnly\r\n"; + } + s += "\r\n"; + return s; +} + +std::string +gen_long_uri(int len) +{ + std::string s = "GET http://www.example.com/"; + s.append(len, 'x'); + s += " HTTP/1.1\r\nHost: www.example.com\r\n\r\n"; + return s; +} + +// The MIME-only target parses a field block with the start line removed. +std::string_view +strip_start_line(std::string_view block) +{ + auto pos = block.find('\n'); + return pos == std::string_view::npos ? block : block.substr(pos + 1); +} + +// Collect the field names in a block (for the wks target). Skips the start line, +// continuation lines, and lines without a colon. +void +collect_field_names(std::string_view block, std::vector &out) +{ + std::string_view rest = strip_start_line(block); + size_t pos = 0; + while (pos < rest.size()) { + size_t eol = rest.find('\n', pos); + std::string_view line = rest.substr(pos, (eol == std::string_view::npos ? rest.size() : eol) - pos); + pos = (eol == std::string_view::npos) ? rest.size() : eol + 1; + if (line.empty() || line == "\r" || line.front() == ' ' || line.front() == '\t') { + continue; // blank terminator or folded continuation + } + size_t colon = line.find(':'); + if (colon != std::string_view::npos && colon > 0) { + out.emplace_back(line.substr(0, colon)); + } + } +} + +// Load raw header blocks from a file. Multiple blocks may be separated by a +// blank line. Line endings are normalized and each block re-terminated with a +// canonical "\r\n\r\n". +void +load_file(const std::filesystem::path &path, std::vector &out) +{ + std::ifstream in(path, std::ios::binary); + if (!in) { + std::fprintf(stderr, "warning: cannot open corpus file %s\n", path.c_str()); + return; + } + std::stringstream ss; + ss << in.rdbuf(); + std::string content = ss.str(); + + // Normalize CRLF -> LF, then split blocks on a blank line ("\n\n"). + std::string norm; + norm.reserve(content.size()); + for (char ch : content) { + if (ch != '\r') { + norm += ch; + } + } + + int idx = 0; + size_t pos = 0; + while (pos < norm.size()) { + size_t sep = norm.find("\n\n", pos); + size_t end = (sep == std::string::npos) ? norm.size() : sep; + std::string_view chunk = std::string_view(norm).substr(pos, end - pos); + // Trim leading/trailing newlines. + while (!chunk.empty() && chunk.front() == '\n') { + chunk.remove_prefix(1); + } + while (!chunk.empty() && chunk.back() == '\n') { + chunk.remove_suffix(1); + } + if (!chunk.empty()) { + // Re-insert canonical CRLF line endings and terminator. + std::string block; + for (size_t i = 0; i < chunk.size(); ++i) { + if (chunk[i] == '\n') { + block += "\r\n"; + } else { + block += chunk[i]; + } + } + block += "\r\n\r\n"; + HeaderCase c; + c.label = path.filename().string() + "#" + std::to_string(idx++); + c.is_response = block.rfind("HTTP/", 0) == 0; // status line starts with "HTTP/" + c.data = std::move(block); + out.push_back(std::move(c)); + } + if (sep == std::string::npos) { + break; + } + pos = sep + 2; + } +} + +Corpus +build_corpus(const std::vector &files, const std::vector &dirs) +{ + Corpus corp; + auto add = [&](std::string_view label, std::string_view data, bool is_resp) { + corp.cases.push_back({std::string(label), std::string(data), is_resp}); + }; + + // Realistic. + add("req_realistic", REQ_REALISTIC, false); + add("req_modern", REQ_MODERN, false); + add("req_classic", REQ_CLASSIC, false); + add("resp_realistic", RESP_REALISTIC, true); + add("resp_304", RESP_304, true); + + // Adversarial. + add("adv_many_fields", gen_many_fields(100), false); + add("adv_long_value", gen_long_value(8192), false); + add("adv_wks_miss", gen_wks_miss(50), false); + add("adv_dup_fields", gen_dup_fields(50), true); + add("adv_long_uri", gen_long_uri(4000), false); + + // File-loaded. + for (const auto &f : files) { + load_file(f, corp.cases); + } + for (const auto &d : dirs) { + std::error_code ec; + for (auto it = std::filesystem::directory_iterator(d, ec); !ec && it != std::filesystem::directory_iterator(); ++it) { + if (it->is_regular_file()) { + load_file(it->path(), corp.cases); + } + } + } + + corp.urls = {std::string(URL_REALISTIC), "http://www.example.com/" + std::string(4000, 'x')}; + + for (const auto &c : corp.cases) { + collect_field_names(c.data, corp.wks); + } + + return corp; +} + +Corpus g_corpus; + +// ---------------------------------------------------------------------------- +// Parse drivers. Each does the minimal realistic setup, one parse, teardown, and +// returns {ParseResult, sink}. +// +// The primary production path is ZERO-COPY: the IOBuffer proxy path +// (HdrTSOnly.cc parse_req(IOBufferReader*)) attaches the socket block to the +// heap and parses with must_copy_strings=false, and it runs under +// strict_uri_parsing=2 (the config default). So the drivers default to +// copy=false and strict=2; pass copy=true to measure the copy path. With +// copy=false the parsed field pointers alias the input buffer, which is a stable +// static corpus string here, so this is safe for the lifetime of the parse. +// ---------------------------------------------------------------------------- + +constexpr bool PROD_COPY = false; // zero-copy IOBuffer proxy path +constexpr int PROD_STRICT = 2; // proxy.config.http.strict_uri_parsing default + +using ParseOutcome = std::pair; + +ParseOutcome +drive_request(std::string_view raw, bool copy = PROD_COPY, int strict = PROD_STRICT) +{ + HTTPParser parser; + http_parser_init(&parser); + HTTPHdr hdr; + HdrHeap *heap = new_HdrHeap(HdrHeap::DEFAULT_SIZE + 64); // +64 avoids proxy alloc + hdr.create(HTTPType::REQUEST, HTTP_1_1, heap); + const char *start = raw.data(); + ParseResult ret = http_parser_parse_req(&parser, hdr.m_heap, hdr.m_http, &start, raw.data() + raw.size(), copy, + /*eof*/ true, strict, UINT16_MAX, 131070); + uint64_t sink = static_cast(start - raw.data()); + hdr.destroy(); + return {ret, sink}; +} + +ParseOutcome +drive_response(std::string_view raw, bool copy = PROD_COPY) +{ + HTTPParser parser; + http_parser_init(&parser); + HTTPHdr hdr; + HdrHeap *heap = new_HdrHeap(HdrHeap::DEFAULT_SIZE + 64); + hdr.create(HTTPType::RESPONSE, HTTP_1_1, heap); + const char *start = raw.data(); + ParseResult ret = http_parser_parse_resp(&parser, hdr.m_heap, hdr.m_http, &start, raw.data() + raw.size(), copy, /*eof*/ true); + uint64_t sink = static_cast(start - raw.data()); + hdr.destroy(); + return {ret, sink}; +} + +ParseOutcome +drive_mime(std::string_view fields, bool copy = PROD_COPY) +{ + MIMEParser parser; + mime_parser_init(&parser); + MIMEHdr hdr; + HdrHeap *heap = new_HdrHeap(HdrHeap::DEFAULT_SIZE + 64); + hdr.create(heap); + const char *start = fields.data(); + ParseResult ret = mime_parser_parse(&parser, hdr.m_heap, hdr.m_mime, &start, fields.data() + fields.size(), copy, + /*eof*/ true, /*remove_ws_from_field_name*/ false); + uint64_t sink = static_cast(start - fields.data()); + hdr.destroy(); + return {ret, sink}; +} + +ParseOutcome +drive_url(std::string_view uri, bool copy = PROD_COPY, int strict = PROD_STRICT) +{ + URL url; + HdrHeap *heap = new_HdrHeap(HdrHeap::DEFAULT_SIZE + 64); + url.create(heap); + const char *start = uri.data(); + ParseResult ret = url_parse(heap, url.m_url_impl, &start, uri.data() + uri.size(), copy, strict, /*verify_host*/ true); + // Read back parsed state so a successful parse (ParseResult::DONE == 0) cannot + // be optimized away. + uint64_t sink = static_cast(ret) + static_cast(url.host_get().length()) + url.port_get(); + url.destroy(); + return {ret, sink}; +} + +// Isolates the per-field well-known-string hash + lookup (and the miss-path +// validation) without the surrounding MIME machinery. +uint64_t +drive_wks(const std::vector &names) +{ + uint64_t sink = 0; + for (const auto &n : names) { + const char *wks = nullptr; + int idx = hdrtoken_tokenize(n.data(), static_cast(n.size()), &wks); + sink += static_cast(idx + 1) + reinterpret_cast(wks); + } + return sink; +} + +// ---------------------------------------------------------------------------- +// Profiling mode +// ---------------------------------------------------------------------------- + +enum class Target { Request, Response, Mime, Url, Wks, Unknown }; + +Target +parse_target(std::string_view s) +{ + if (s == "request") { + return Target::Request; + } + if (s == "response") { + return Target::Response; + } + if (s == "mime") { + return Target::Mime; + } + if (s == "url") { + return Target::Url; + } + if (s == "wks") { + return Target::Wks; + } + return Target::Unknown; +} + +const char * +profile_target_name(Target t) +{ + switch (t) { + case Target::Request: + return "request"; + case Target::Response: + return "response"; + case Target::Mime: + return "mime"; + case Target::Url: + return "url"; + case Target::Wks: + return "wks"; + default: + return "unknown"; + } +} + +int +run_profile(Target target, uint64_t iters) +{ + // Assemble the input set and a per-iteration byte count for throughput. + std::vector inputs; + uint64_t bytes_per_pass = 0; + + auto add_input = [&](std::string_view v) { + inputs.push_back(v); + bytes_per_pass += v.size(); + }; + + switch (target) { + case Target::Request: + for (const auto &c : g_corpus.cases) { + if (!c.is_response) { + add_input(c.data); + } + } + break; + case Target::Response: + for (const auto &c : g_corpus.cases) { + if (c.is_response) { + add_input(c.data); + } + } + break; + case Target::Mime: + for (const auto &c : g_corpus.cases) { + add_input(strip_start_line(c.data)); + } + break; + case Target::Url: + for (const auto &u : g_corpus.urls) { + add_input(u); + } + break; + case Target::Wks: + // Handled below (uses the owned name list directly). + for (const auto &n : g_corpus.wks) { + bytes_per_pass += n.size(); + } + break; + default: + std::fprintf(stderr, "unknown --profile target\n"); + return 2; + } + + if (target != Target::Wks && inputs.empty()) { + std::fprintf(stderr, "no inputs for the requested target\n"); + return 2; + } + + volatile uint64_t sink = 0; + auto t0 = std::chrono::steady_clock::now(); + uint64_t count = 0; + + if (target == Target::Wks) { + for (uint64_t i = 0; i < iters; ++i) { + sink += drive_wks(g_corpus.wks); + } + count = iters; // one full pass over all names per iter + } else { + for (uint64_t i = 0; i < iters; ++i) { + std::string_view in = inputs[i % inputs.size()]; + ParseOutcome r; + switch (target) { + case Target::Request: + r = drive_request(in); + break; + case Target::Response: + r = drive_response(in); + break; + case Target::Mime: + r = drive_mime(in); + break; + case Target::Url: + r = drive_url(in); + break; + default: + break; + } + sink += r.second; + } + count = iters; + } + + auto t1 = std::chrono::steady_clock::now(); + double ns = std::chrono::duration(t1 - t0).count(); + double ns_per = ns / static_cast(count); + + // One "op" is one parse for the parse targets, or one full pass over the name + // list for wks. Throughput is over the header bytes actually processed. + double avg_in = inputs.empty() ? 0.0 : static_cast(bytes_per_pass) / static_cast(inputs.size()); + double total_bytes = (target == Target::Wks) ? static_cast(bytes_per_pass) * static_cast(iters) : + avg_in * static_cast(iters); + double mibps = (total_bytes / (ns / 1e9)) / (1024.0 * 1024.0); + + std::printf("target=%s iters=%llu %.2f ns/op %.2f Mops/s %.0f MiB/s sink=%llu\n", profile_target_name(target), + static_cast(count), ns_per, 1000.0 / ns_per, mibps, static_cast(sink)); + return 0; +} + +} // namespace + +// ---------------------------------------------------------------------------- +// A/B stats mode (Catch2 BENCHMARK) +// ---------------------------------------------------------------------------- + +namespace +{ +const HeaderCase & +find_case(std::string_view label) +{ + for (const auto &c : g_corpus.cases) { + if (c.label == label) { + return c; + } + } + FAIL("missing corpus case: " << label); + return g_corpus.cases.front(); +} +} // namespace + +// Built-in cases must fully parse (DONE + all bytes consumed); file-loaded cases +// (label carries a '#') only must not ERROR. +bool +is_file_case(const HeaderCase &c) +{ + return c.label.find('#') != std::string::npos; +} + +TEST_CASE("hdr parse: request", "[bench][request]") +{ + for (const auto &c : g_corpus.cases) { + if (c.is_response) { + continue; + } + CAPTURE(c.label); + auto [ret, consumed] = drive_request(c.data); + if (is_file_case(c)) { + REQUIRE(ret != ParseResult::ERROR); + } else { + REQUIRE(ret == ParseResult::DONE); + REQUIRE(consumed == c.data.size()); + } + } + + const auto &realistic = find_case("req_realistic"); + const auto &modern = find_case("req_modern"); + const auto &many = find_case("adv_many_fields"); + + BENCHMARK("request: realistic (zero-copy)") + { + return drive_request(realistic.data).second; + }; + BENCHMARK("request: modern browser (zero-copy)") + { + return drive_request(modern.data).second; + }; + BENCHMARK("request: realistic (copy)") + { + return drive_request(realistic.data, /*copy*/ true).second; + }; + BENCHMARK("request: 100 fields") + { + return drive_request(many.data).second; + }; +} + +TEST_CASE("hdr parse: response", "[bench][response]") +{ + for (const auto &c : g_corpus.cases) { + if (!c.is_response) { + continue; + } + CAPTURE(c.label); + auto [ret, consumed] = drive_response(c.data); + if (is_file_case(c)) { + REQUIRE(ret != ParseResult::ERROR); + } else { + REQUIRE(ret == ParseResult::DONE); + REQUIRE(consumed == c.data.size()); + } + } + + const auto &realistic = find_case("resp_realistic"); + const auto &dups = find_case("adv_dup_fields"); + + BENCHMARK("response: realistic") + { + return drive_response(realistic.data).second; + }; + BENCHMARK("response: 50 dup fields") + { + return drive_response(dups.data).second; + }; +} + +TEST_CASE("hdr parse: mime only", "[bench][mime]") +{ + const auto &realistic = find_case("req_realistic"); + const auto &wksmiss = find_case("adv_wks_miss"); + + BENCHMARK("mime: realistic fields") + { + return drive_mime(strip_start_line(realistic.data)).second; + }; + BENCHMARK("mime: 50 wks-miss fields") + { + return drive_mime(strip_start_line(wksmiss.data)).second; + }; +} + +TEST_CASE("hdr parse: url only", "[bench][url]") +{ + REQUIRE(drive_url(URL_REALISTIC).first != ParseResult::ERROR); + + BENCHMARK("url: realistic") + { + return drive_url(URL_REALISTIC).second; + }; +} + +TEST_CASE("hdr parse: wks tokenize", "[bench][wks]") +{ + REQUIRE(!g_corpus.wks.empty()); + + BENCHMARK("wks: tokenize all field names") + { + return drive_wks(g_corpus.wks); + }; +} + +// ---------------------------------------------------------------------------- +// Entry point (own main, like the hdrs unit-test stub, so we can init the WKS +// tables and branch to the profiling loop). +// ---------------------------------------------------------------------------- + +int +main(int argc, char *argv[]) +{ + // No thread setup, forbid thread-local allocators (mirrors unit_test_main.cc). + cmd_disable_pfreelist = true; + // Populate the HTTP well-known strings; parsing depends on them. + http_init(); + + // Pull out our own flags (--profile/--iters/--corpus-*) and pass the rest to + // Catch. --corpus-* feed both modes. + std::vector corpus_files, corpus_dirs; + std::string profile_target; + uint64_t iters = 5'000'000; + std::vector catch_args; + catch_args.push_back(argv[0]); + + for (int i = 1; i < argc; ++i) { + std::string_view a = argv[i]; + if (a == "--profile" && i + 1 < argc) { + profile_target = argv[++i]; + } else if (a == "--iters" && i + 1 < argc) { + iters = std::strtoull(argv[++i], nullptr, 10); + } else if (a == "--corpus-file" && i + 1 < argc) { + corpus_files.emplace_back(argv[++i]); + } else if (a == "--corpus-dir" && i + 1 < argc) { + corpus_dirs.emplace_back(argv[++i]); + } else { + catch_args.push_back(argv[i]); + } + } + + g_corpus = build_corpus(corpus_files, corpus_dirs); + + if (!profile_target.empty()) { + Target t = parse_target(profile_target); + if (t == Target::Unknown) { + std::fprintf(stderr, "unknown target '%s' (want: request|response|mime|url|wks)\n", profile_target.c_str()); + return 2; + } + return run_profile(t, iters); + } + + return Catch::Session().run(static_cast(catch_args.size()), catch_args.data()); +}