/*****************************************************************************/ /* HPACK.c In HTTP/1.1 header fields are not compressed, the redundant header fields in these requests unnecessarily consume bandwidth, measurably increasing latency. HPACK is a compressor that eliminates redundant header fields, limits vulnerability to known security attacks, and has a bounded memory requirement for use in constrained environments. https://tools.ietf.org/html/rfc7541 Note that in section 1.1: The format defined in this specification treats a list of header fields as an ordered collection of name-value pairs that can include duplicate pairs. *Names* and values are considered to be *opaque sequences of octets*, and the order of header fields is preserved after being compressed and decompressed. However value pair names have similar HTTP/1 constraints applied as done by RequestGet() and RequestFields(). VERSION HISTORY --------------- 19-DEC-2019 MGD apply similar HTTP/1 constraints to (value pair) names request header fields by RequestGet() and RequestFields() 16-JUN-2017 MGD bugfix; HpackHeadersFrame() use ":authority" pseudo-header for "Host:" header according to RFC7540 8.1.2.3 06-AUG-2016 MGD bugfix; HpackHeadersFrame() uncompressed header size 22-MAY-2016 MGD bugfix; HpackHeadersFrame() multiple to single cookie header 22-AUG-2015 MGD initial */ /*****************************************************************************/ #ifdef WASD_VMS_V7 # undef __VMS_VER # define __VMS_VER 70000000 # undef __CRTL_VER # define __CRTL_VER 70000000 #else # ifdef WASD_VMS_V7 # undef _VMS__V6__SOURCE # define _VMS__V6__SOURCE # undef __VMS_VER # define __VMS_VER 70000000 # undef __CRTL_VER # define __CRTL_VER 70000000 # endif #endif #include #include #include #include "wasd.h" #include "hpack_huffman_table.h" #define WASD_MODULE "HPACK" /******************/ /* global storage */ /******************/ HPACK_ENTRY_STRUCT HpackStaticTable [HPACK_STATIC_TABLE_COUNT+1] = { { NULL, NULL, 0, 0, 0, NULL, NULL }, { NULL, NULL, 0, 10, 0, ":authority", NULL }, { NULL, NULL, 0, 7, 3, ":method", "GET" }, { NULL, NULL, 0, 7, 4, ":method", "POST" }, { NULL, NULL, 0, 5, 1, ":path", "/" }, { NULL, NULL, 0, 5, 11, ":path", "/index.html" }, { NULL, NULL, 0, 7, 4, ":scheme", "http" }, { NULL, NULL, 0, 7, 5, ":scheme", "https" }, { NULL, NULL, 0, 7, 3, ":status", "200" }, { NULL, NULL, 0, 7, 3, ":status", "204" }, { NULL, NULL, 0, 7, 3, ":status", "206" }, { NULL, NULL, 0, 7, 3, ":status", "304" }, { NULL, NULL, 0, 7, 3, ":status", "400" }, { NULL, NULL, 0, 7, 3, ":status", "404" }, { NULL, NULL, 0, 7, 3, ":status", "500" }, { NULL, NULL, 0, 14, 0, "accept-charset", NULL }, { NULL, NULL, 0, 15, 13, "accept-encoding", "gzip, deflate" }, { NULL, NULL, 0, 15, 0, "accept-language", NULL }, { NULL, NULL, 0, 13, 0, "accept-ranges", NULL }, { NULL, NULL, 0, 6, 0, "accept", NULL }, { NULL, NULL, 0, 27, 0, "accept-control-allow-origin", NULL }, { NULL, NULL, 0, 3, 0, "age", NULL }, { NULL, NULL, 0, 5, 0, "allow", NULL }, { NULL, NULL, 0, 13, 0, "authorization", NULL }, { NULL, NULL, 0, 13, 0, "cache-control", NULL }, { NULL, NULL, 0, 19, 0, "content-disposition", NULL }, { NULL, NULL, 0, 16, 0, "content-encoding", NULL }, { NULL, NULL, 0, 16, 0, "content-language", NULL }, { NULL, NULL, 0, 14, 0, "content-length", NULL }, { NULL, NULL, 0, 16, 0, "content-location", NULL }, { NULL, NULL, 0, 13, 0, "content-range", NULL }, { NULL, NULL, 0, 12, 0, "content-type", NULL }, { NULL, NULL, 0, 6, 0, "cookie", NULL }, { NULL, NULL, 0, 4, 0, "date", NULL }, { NULL, NULL, 0, 4, 0, "etag", NULL }, { NULL, NULL, 0, 6, 0, "expect", NULL }, { NULL, NULL, 0, 7, 0, "expires", NULL }, { NULL, NULL, 0, 4, 0, "from", NULL }, { NULL, NULL, 0, 4, 0, "host", NULL }, { NULL, NULL, 0, 8, 0, "if-match", NULL }, { NULL, NULL, 0, 17, 0, "if-modified-since", NULL }, { NULL, NULL, 0, 13, 0, "if-none-match", NULL }, { NULL, NULL, 0, 8, 0, "if-range", NULL }, { NULL, NULL, 0, 19, 0, "if-unmodified-since", NULL }, { NULL, NULL, 0, 13, 0, "last-modified", NULL }, { NULL, NULL, 0, 4, 0, "link", NULL }, { NULL, NULL, 0, 8, 0, "location", NULL }, { NULL, NULL, 0, 12, 0, "max-forwards", NULL }, { NULL, NULL, 0, 18, 0, "proxy-authenticate", NULL }, { NULL, NULL, 0, 19, 0, "proxy-authorization", NULL }, { NULL, NULL, 0, 5, 0, "range", NULL }, { NULL, NULL, 0, 7, 0, "referer", NULL }, { NULL, NULL, 0, 7, 0, "refresh", NULL }, { NULL, NULL, 0, 11, 0, "retry-after", NULL }, { NULL, NULL, 0, 6, 0, "server", NULL }, { NULL, NULL, 0, 10, 0, "set-cookie", NULL }, { NULL, NULL, 0, 25, 0, "strict-transport-security", NULL }, { NULL, NULL, 0, 17, 0, "transfer-encoding", NULL }, { NULL, NULL, 0, 10, 0, "user-agent", NULL }, { NULL, NULL, 0, 4, 0, "vary", NULL }, { NULL, NULL, 0, 3, 0, "via", NULL }, { NULL, NULL, 0, 16, 0, "www-authenticate", NULL } }; /********************/ /* external storage */ /********************/ extern BOOL Http2Enabled; extern uint Http2MaxHeaderTableSize; extern int ToLowerCase[]; extern char ErrorSanityCheck[]; extern CONFIG_STRUCT Config; extern LIST_HEAD Http2List; extern WATCH_STRUCT Watch; /*****************************************************************************/ /* Build a request from the header block. This involves decoding and decompressing the various HTTP/2 header elements while reconstructing an HTTP/1.1 request line and populating the request dictionary with equivalent header fields. This is obviously closely integrated with the request structure and necessarily parallels processing in RequestGet() and RequestFields(). The continuation frame handling is elementary but (probably) sufficient to purpose. If |flags| is -1 then for development purposes provide a module WATCHing report for a *response* header. */ int HpackHeadersFrame ( HPACK_TABLE_STRUCT *tabptr, uint flags, uint ident, uchar *BlockPtr, uint BlockLength ) { #define BUFFER_INCREMENT 4096 #define COOKIE_INCREMENT 1024 static uint CookieSize; static uchar *CookiePtr; int nlen, retval, vlen, AuthorityLength, CompressedLength, CookieLength, /* equivalent of HTTP/1.1 request header */ HeaderLength, MethodLength, PathLength; uint blen, bytes, depend, endhead, endstream, index, length, padlen, weight; uchar byte; uchar *bptr, *bzptr, *cptr, *nptr, *sptr, *vptr, *zptr; uchar *AuthorityPtr, *MethodPtr, *PathPtr; DICT_ENTRY_STRUCT *denptr; HTTP2_STRUCT *h2ptr; REQUEST_STRUCT *rqptr; HTTP2_STREAM_STRUCT *s2ptr; #if WATCH_MOD uchar ResponseHeader [2048]; #endif /*********/ /* begin */ /*********/ h2ptr = tabptr->h2ptr; if (WATCHMOD (h2ptr, WATCH_MOD_HTTP2)) WatchThis (WATCHITM(h2ptr), WATCH_MOD_HTTP2, "HpackHeadersFrame() 0x!2XL !&X !UL", flags, BlockPtr, BlockLength); #if WATCH_MOD if (flags == (uint)-1) zptr = (sptr = ResponseHeader) + sizeof(ResponseHeader)-1; else #endif /* WATCH_MOD */ { /* locate the request corresponding to the stream (ident) */ for (s2ptr = LIST_GET_HEAD(&h2ptr->StreamList); s2ptr != NULL; s2ptr = LIST_GET_NEXT(s2ptr)) if (s2ptr->Ident == ident) break; if (s2ptr == NULL) { /* no stream (request) found with that ident so create one */ if ((rqptr = Http2RequestBegin (h2ptr, ident)) == NULL) return (-HTTP2_ERROR_REFUSED); s2ptr = &rqptr->Http2Stream; } else rqptr = (REQUEST_STRUCT*)s2ptr->RequestPtr; } bzptr = (bptr = BlockPtr) + BlockLength; #if WATCH_MOD if (flags == (uint)-1) endhead = endstream = depend = padlen = weight = 0; else #endif /* WATCH_MOD */ { /*********/ /* flags */ /*********/ if (flags & HTTP2_FLAG_HEAD_PADDED) HTTP2_GET_8 (bptr, padlen) else padlen = 0; if (flags & HTTP2_FLAG_HEAD_PRIORITY) { HTTP2_GET_32 (bptr, depend) HTTP2_GET_8 (bptr, weight) } else depend = weight = 0; endhead = (flags & HTTP2_FLAG_HEAD_END_HEAD); endstream = (flags & HTTP2_FLAG_HEAD_END_STR); if (WATCHMOD (h2ptr, WATCH_MOD_HTTP2)) WatchThis (WATCHITM(h2ptr), WATCH_MOD_HTTP2, "endhead:!&B endstream:!&B depend:!UL pad:!UL weight:!UL", endhead, endstream, depend, padlen, weight); if ((bptr -= padlen) >= bzptr) return (-HTTP2_ERROR_PROTOCOL); /* reset the original parameters against any padding, etc. */ BlockLength -= bptr - BlockPtr; BlockPtr = bptr; /* handle continuation frames */ if (!endhead || s2ptr->ContinPtr) { /****************/ /* continuation */ /****************/ /* impose a (somewhat ambit) limit on (compressed) header list size */ if (s2ptr->ContinSize >= h2ptr->ServerMaxHeaderListSize) { if (WATCHING (rqptr, WATCH_HTTP2)) WatchThis (WATCHITM(rqptr), WATCH_HTTP2, "HTTP/2 continuation !UL exceeds !UL bytes", s2ptr->ContinSize, h2ptr->ServerMaxHeaderListSize); return (-HTTP2_ERROR_PROTOCOL); } /* buffer this frame content for subsequent processing */ bytes = s2ptr->ContinSize + BlockLength; s2ptr->ContinPtr = VmReallocHeap (rqptr, s2ptr->ContinPtr, bytes, FI_LI); memcpy (s2ptr->ContinPtr + s2ptr->ContinSize, BlockPtr, BlockLength); s2ptr->ContinSize = bytes; /* if this is not the last of the headers data */ if (!endhead) return (0); /* use the buffered headers data */ BlockLength = rqptr->Http2Stream.ContinSize; BlockPtr = rqptr->Http2Stream.ContinPtr; } } AuthorityPtr = MethodPtr = PathPtr = NULL; AuthorityLength = CookieLength = HeaderLength = MethodLength = PathLength = 0; CompressedLength = BlockLength; h2ptr->HpackClientInputCount += BlockLength; bzptr = (bptr = BlockPtr) + BlockLength; while (bptr < bzptr) { nptr = vptr = NULL; nlen = vlen = 0; byte = *bptr; if (byte & 0x80) { /***********************/ /* indexed field (6.1) */ /***********************/ if (WATCHMOD (h2ptr, WATCH_MOD_HTTP2)) WatchThis (WATCHITM(h2ptr), WATCH_MOD_HTTP2, "6.1"); retval = HpackDecodeInt32 (h2ptr, &bptr, bzptr, 7, &index); if (retval < 0) return (retval); if (!index) return (-HTTP2_ERROR_PROTOCOL); retval = HpackGetIndex (tabptr, index, &nptr, &nlen, &vptr, &vlen); if (retval < 0) return (retval); if (WATCHMOD (h2ptr, WATCH_MOD_HTTP2)) WatchThis (WATCHITM(h2ptr), WATCH_MOD_HTTP2, "index:!UL \"!AZ\" \"!AZ\"", index, nptr, vptr); } else if (((byte & 0xc0) == 0x40) || ((byte & 0xf0) == 0x00) || ((byte & 0xf0) == 0x10)) { /************************/ /* literal header (6.2) */ /************************/ if (WATCHMOD (h2ptr, WATCH_MOD_HTTP2)) { if ((byte & 0xc0) == 0x40) cptr = "6.2.1"; else if ((byte & 0xf0) == 0x00) cptr = "6.2.2"; else cptr = "6.2.3"; WatchThis (WATCHITM(h2ptr), WATCH_MOD_HTTP2, "!AZ", cptr); } if ((byte & 0xc0) == 0x40) retval = HpackDecodeInt32 (h2ptr, &bptr, bzptr, 6, &index); else retval = HpackDecodeInt32 (h2ptr, &bptr, bzptr, 4, &index); if (retval < 0) return (retval); if (index) { /* non-zero index, value only */ retval = HpackGetIndex (tabptr, index, &nptr, &nlen, &vptr, &vlen); if (retval < 0) return (retval); retval = HpackDecodeString (h2ptr, &bptr, bzptr, &vptr, &vlen); if (retval < 0) return (retval); } else { /* zero index, name then value */ retval = HpackDecodeString (h2ptr, &bptr, bzptr, &nptr, &nlen); if (retval < 0) return (retval); retval = HpackDecodeString (h2ptr, &bptr, bzptr, &vptr, &vlen); if (retval < 0) return (retval); } if (WATCHMOD (h2ptr, WATCH_MOD_HTTP2)) WatchThis (WATCHITM(h2ptr), WATCH_MOD_HTTP2, "index:!UL \"!AZ\" \"!AZ\"", index, nptr, vptr); if ((byte & 0xc0) == 0x40) { /* similar field name constraints as RequestFields() */ if (nlen > MAX_REQUEST_FIELD_NAME_LEN) { if (WATCHING (rqptr, WATCH_HTTP2)) WatchThis (WATCHITM(rqptr), WATCH_HTTP2, "FIELD \"!32AZ..\" exceeds !UL bytes", nptr, MAX_REQUEST_FIELD_NAME_LEN); #if WATCH_MOD FaoToStdout ("!AZ:!UL FIELD \"!32AZ..\" exceeds !UL bytes", FI_LI, nptr, MAX_REQUEST_FIELD_NAME_LEN); #endif return (-HTTP2_ERROR_REFUSED); } zptr = (cptr = nptr) + nlen; /* e.g. ":authority" */ if (*cptr == ':') cptr++; while (cptr < zptr && NOTCTL(*cptr) && NOTSEP(*cptr)) cptr++; if (cptr < zptr) { if (WATCHING (rqptr, WATCH_HTTP2)) WatchThis (WATCHITM(rqptr), WATCH_HTTP2, "FIELD \"!#AZ\" char \\x!2XL at !UL", nlen, nptr, *cptr, cptr - nptr); #if WATCH_MOD FaoToStdout ("%HTTPD-W-NOTICED, !20%D, !AZ:!UL, " "FIELD \"!#AZ\" char \\x!2XL at !UL", 0, FI_LI, nlen, nptr, *cptr, cptr - nptr); #endif return (-HTTP2_ERROR_REFUSED); } retval = HpackAddToTable (tabptr, nptr, nlen, vptr, vlen); if (retval < 0) return (retval); } } else if ((byte & 0xe0) == 0x20) { /******************************/ /* dynamic table update (6.3) */ /******************************/ if (WATCHMOD (h2ptr, WATCH_MOD_HTTP2)) WatchThis (WATCHITM(h2ptr), WATCH_MOD_HTTP2, "6.3"); retval = HpackDecodeInt32 (h2ptr, &bptr, bzptr, 5, &length); if (retval < 0) return (retval); if (h2ptr->ClientMaxHeaderTableSize) { if (length <= h2ptr->ClientMaxHeaderTableSize) { h2ptr->HpackClientTable.max = length; HpackTrimTable (tabptr); } else return (-HTTP2_ERROR_PROTOCOL); } else { /* if client hasn't set a maximum then (almost) free-for-all */ if (length <= HTTP2_MAX_HEAD_TAB_SIZE) { h2ptr->HpackClientTable.max = length; HpackTrimTable (tabptr); } else return (-HTTP2_ERROR_INTERNAL); } } else { /********/ /* hmmm */ /********/ ErrorNoticed (NULL, SS$_BUGCHECK, ErrorSanityCheck, FI_LI); return (-HTTP2_ERROR_PROTOCOL); } /*****************/ /* build request */ /*****************/ if (nptr && vptr) { #if WATCH_MOD if (flags == (uint)-1 && MATCH7 (nptr, ":status")) { /* only when WATCHing the response header */ for (cptr = "HTTP/1.1 "; *cptr && sptr < zptr; *sptr++ = *cptr++); for (cptr = vptr; *cptr && sptr < zptr; *sptr++ = *cptr++); if (sptr < zptr) *sptr++ = '\r'; if (sptr < zptr) *sptr++ = '\n'; *sptr = '\0'; } else #endif /* WATCH_MOD */ if (MATCH10 (nptr, ":authority")) { /* RFC7540 8.1.2.3 ... analogue to 'Host:" */ denptr = DictInsert (rqptr->rqDictPtr, DICT_TYPE_INTERNAL, "request_authority", 17, vptr, vlen); AuthorityPtr = DICT_GET_VALUE(denptr); AuthorityLength = DICT_GET_VALUE_LEN(denptr); } else if (MATCH7 (nptr, ":method")) { denptr = DictInsert (rqptr->rqDictPtr, DICT_TYPE_INTERNAL, "request_method", 14, vptr, vlen); MethodPtr = DICT_GET_VALUE(denptr); MethodLength = DICT_GET_VALUE_LEN(denptr); } else if (MATCH5 (nptr, ":path")) { denptr = DictInsert (rqptr->rqDictPtr, DICT_TYPE_INTERNAL, "request_path", 12, vptr, vlen); PathPtr = DICT_GET_VALUE(denptr); PathLength = DICT_GET_VALUE_LEN(denptr); } else #if WATCH_MOD /* only when WATCHing the response header */ if (flags != (uint)-1 && MATCH7 (nptr, "cookie")) #else if (MATCH7 (nptr, "cookie")) #endif { /* RFC7540 8.1.2.5 multiple HTTP/2 into single HTTP/1.n cookie */ uchar *cptr, *sptr, *zptr; length = vlen; if (CookieLength) length += 2; if (CookieLength + length > CookieSize) { while (CookieLength + length > CookieSize) CookieSize += COOKIE_INCREMENT; CookiePtr = VmRealloc (CookiePtr, CookieSize, FI_LI); } zptr = (sptr = CookiePtr) + CookieSize; sptr += CookieLength; if (CookieLength) { /* length += 2; */ *sptr++ = ';'; *sptr++ = ' '; } for (cptr = vptr; *cptr && sptr < zptr; *sptr++ = *cptr++); *sptr = '\0'; CookieLength = sptr - CookiePtr; } else if (*nptr != ':') { #if WATCH_MOD if (flags == (uint)-1) { /* only when WATCHing the response header */ for (cptr = nptr; *cptr && sptr < zptr; *sptr++ = *cptr++); for (cptr = ": "; *cptr && sptr < zptr; *sptr++ = *cptr++); for (cptr = vptr; *cptr && sptr < zptr; *sptr++ = *cptr++); if (sptr < zptr) *sptr++ = '\r'; if (sptr < zptr) *sptr++ = '\n'; *sptr = '\0'; } else #endif /* WATCH_MOD */ { DictInsert (rqptr->rqDictPtr, DICT_TYPE_REQUEST, nptr, nlen, vptr, vlen); /* RFC7540 section 6.5.2 */ HeaderLength += nlen + vlen + 32; h2ptr->HpackClientOutputCount += nlen + vlen + 4; } } /* impose the limit on (uncompressed) header list size */ if (HeaderLength + CookieLength+32 >= h2ptr->ServerMaxHeaderListSize) { /* RFC7540 section 6.5.2 SETTINGS_MAX_HEADER_LIST_SIZE */ if (WATCHING (rqptr, WATCH_HTTP2)) WatchThis (WATCHITM(rqptr), WATCH_HTTP2, "HTTP/2 header !UL exceeds !UL bytes", HeaderLength + CookieLength+32, h2ptr->ServerMaxHeaderListSize); return (-HTTP2_ERROR_PROTOCOL); } } } if (CookieLength) { DictInsert (rqptr->rqDictPtr, DICT_TYPE_REQUEST, "cookie", 6, CookiePtr, CookieLength); HeaderLength += CookieLength + 2; h2ptr->HpackClientOutputCount += CookieLength + 2; } #if WATCH_MOD if (flags != (uint)-1) #endif { /* create a request line */ if (MethodPtr && PathPtr) { /* reserve space in the dictionary entry then populate */ bytes = MethodLength + PathLength + 10; denptr = DictInsert (rqptr->rqDictPtr, DICT_TYPE_INTERNAL, "request_line", 12, NULL, bytes); zptr = (sptr = bptr = DICT_GET_VALUE (denptr)) + bytes; rqptr->rqHeader.RequestLinePtr = bptr; for (cptr = MethodPtr; *cptr && sptr < zptr; *sptr++ = *cptr++); if (sptr < zptr) *sptr++ = ' '; for (cptr = PathPtr; *cptr && sptr < zptr; *sptr++ = *cptr++); for (cptr = " HTTP/1.1"; *cptr && sptr < zptr; *sptr++ = *cptr++); DictValueLength (denptr, sptr - bptr); rqptr->rqHeader.RequestLineLength = DICT_GET_VALUE_LEN(denptr); h2ptr->HpackClientOutputCount += bytes + 2; HeaderLength += bytes + 2; /* similar request line constraints as RequestGet() */ for (cptr = bptr; *cptr && NOTCTL(*cptr); cptr++); if (*cptr) { if (WATCHING (rqptr, WATCH_HTTP2)) WatchThis (WATCHITM(rqptr), WATCH_HTTP2, "CHAR \\x!2XL at !UL", *cptr, cptr - bptr); #if WATCH_MOD FaoToStdout ("%HTTPD-W-NOTICED, !20%D, !AZ:!UL, " "CHAR \\x!2XL at !UL", 0, FI_LI, *cptr, cptr - bptr); #endif return (-HTTP2_ERROR_REFUSED); } /* HTTP/1.1 mandates a host field */ if (!DictLookup (rqptr->rqDictPtr, DICT_TYPE_REQUEST, "host", 4)) { /* the request has not supplied an explicit "host" field */ if (AuthorityPtr) { /* use the ":authority" pseudo-header RFC7540 8.1.2.3 */ DictInsert (rqptr->rqDictPtr, DICT_TYPE_REQUEST, "host", 4, AuthorityPtr, AuthorityLength); HeaderLength += AuthorityLength + 6; h2ptr->HpackClientOutputCount += AuthorityLength + 6; } else { /* fall back to the current virtual host */ DictInsert (rqptr->rqDictPtr, DICT_TYPE_REQUEST, "host", 4, h2ptr->ServicePtr->ServerHostPort, h2ptr->ServicePtr->ServerHostPortLength); HeaderLength += h2ptr->ServicePtr->ServerHostPortLength + 6; h2ptr->HpackClientOutputCount += h2ptr->ServicePtr->ServerHostPortLength + 6; } } } else return (-HTTP2_ERROR_INTERNAL); } /**********/ /* voila! */ /**********/ #if WATCH_MOD if (flags != (uint)-1) #endif { /* as if terminating blank line */ HeaderLength += 2; if (WATCHING (h2ptr, WATCH_HTTP2) && !WATCHING1S(h2ptr) && !WATCHING2(h2ptr)) WatchThis (WATCHITM(h2ptr), WATCH_HTTP2, "REQUEST header !UL->!UL !UL%", CompressedLength, HeaderLength, (CompressedLength * 100) / HeaderLength); /* keep the accounting representative */ ADD_LONG_QUAD (1, rqptr->NetIoPtr->BlocksRawRx); ADD_LONG_QUAD (1, rqptr->NetIoPtr->BlocksTallyRx); ADD_LONG_QUAD (CompressedLength, rqptr->NetIoPtr->BytesRawRx); ADD_LONG_QUAD (CompressedLength, rqptr->NetIoPtr->BytesTallyRx); ADD_LONG_QUAD (CompressedLength, rqptr->BytesRx); if (s2ptr->ContinPtr) { VmFreeFromHeap (rqptr, s2ptr->ContinPtr, FI_LI); s2ptr->ContinSize = 0; s2ptr->ContinPtr = NULL; } Http2RequestProcess (rqptr); } return (0); } /*****************************************************************************/ /* Decode a 32 bit, unsigned integer. Return zero for success, or a negated error code. h2o decode_int() function reimagined. */ int HpackDecodeInt32 ( HTTP2_STRUCT *h2ptr, uchar **baddr, uchar *bzptr, uint prebits, uint *iptr ) { uint mult, premask, preval, value; uchar premax; uchar *bptr; /*********/ /* begin */ /*********/ bptr = *baddr; if (bptr >= bzptr) return (-HTTP2_ERROR_PROTOCOL); premax = (1 << prebits) - 1; value = (uint)*bptr++ & premax; if (value != premax) { if (WATCHMOD (h2ptr, WATCH_MOD_HTTP2)) WatchThis (WATCHITM(h2ptr), WATCH_MOD_HTTP2, "HpackDecodeInt32() !UL 1", value); *iptr = value; *baddr = bptr; return (0); } premask = 0x80000000; value = preval = premax; for (mult = 1;; mult *= 128) { if (bptr >= bzptr) return (0); value += (*bptr & 0x7f) * mult; if (!(*bptr++ & 0x80)) break; /* detect integer overflow (most significant bit reset) */ if ((preval & premask) && !(value & premask)) break; preval = value; } if ((preval & premask) && !(value & premask)) { if (WATCHING (h2ptr, WATCH_HTTP2)) WatchThis (WATCHITM(h2ptr), WATCH_HTTP2, "OVERFLOW integer"); *iptr = 0; *baddr = bptr; return (-HTTP2_ERROR_PROTOCOL); } if (WATCHMOD (h2ptr, WATCH_MOD_HTTP2)) WatchThis (WATCHITM(h2ptr), WATCH_MOD_HTTP2, "HpackDecodeInt32() !UL", value); *iptr = value; *baddr = bptr; return (0); } /*****************************************************************************/ /* Encode a 32 bit, unsigned integer. h2o encode_int() function reimagined. */ int HpackEncodeInt32 ( HTTP2_STRUCT *h2ptr, uchar **baddr, uchar *bzptr, uint prebits, uint value ) { uchar *bptr; /*********/ /* begin */ /*********/ if (WATCHMOD (h2ptr, WATCH_MOD_HTTP2)) WatchThis (WATCHITM(h2ptr), WATCH_MOD_HTTP2, "HpackEncodeInt32() !UL !UL", value, prebits); bptr = *baddr; if (bptr >= bzptr) return (-HTTP2_ERROR_INTERNAL); if (value < (1 << prebits) - 1) *bptr++ |= value; else { value -= (1 << prebits) - 1; *bptr++ |= (1 << prebits) - 1; if (bptr >= bzptr) return (-HTTP2_ERROR_INTERNAL); for (; value >= 128; value >>= 7) *bptr++ = 0x80 | value; *bptr++ = value; if (bptr >= bzptr) return (-HTTP2_ERROR_INTERNAL); } *baddr = bptr; return (0); } /*****************************************************************************/ /* Decode a header string pointed at by the address at |baddr|. Return the number of characters in that string (0..n), or a negated error code. The string is located in an internal buffer at the address contained by |raddr| and must be used in situ or copied to an alternate buffer. */ int HpackDecodeString ( HTTP2_STRUCT *h2ptr, uchar **baddr, uchar *bzptr, uchar **raddr, uint **rlenaddr ) { static uint idx, idxcnt; static uint buflen [2]; static uchar *bufptr [2]; BOOL ishenc; int eos = 1; uint length, retval; uchar state = 0; uchar *bptr, *sptr, *zptr; nghttp2_huff_decode *entry; /*********/ /* begin */ /*********/ if (WATCHMOD (h2ptr, WATCH_MOD_HTTP2)) WatchThis (WATCHITM(h2ptr), WATCH_MOD_HTTP2, "HpackDecodeString() ishenc:!&B", **baddr & 0x80); bptr = *baddr; /* is it huffman encoded? */ ishenc = *bptr & 0x80; retval = HpackDecodeInt32 (h2ptr, &bptr, bzptr, 7, &length); if (retval < 0) return (retval); /* ensure there's enough data available */ if (bptr + length > bzptr) { if (WATCHMOD (h2ptr, WATCH_MOD_HTTP2)) WatchThis (WATCHITM(h2ptr), WATCH_MOD_HTTP2, "ERROR_PROTOCOL"); return (-HTTP2_ERROR_PROTOCOL); } /* maintain two buffers (concurrent name and value) */ idx = idxcnt++ & 1; /* max huffman compression ratio is <= 0.5 */ if (length * 2 > buflen[idx]) { for (buflen[idx] = 256; buflen[idx] < length * 2; buflen[idx] *= 2); if (bufptr[idx]) VmFree (bufptr[idx], FI_LI); bufptr[idx] = VmGet (buflen[idx]); } sptr = bufptr[idx]; if (ishenc) { /* h2o decode_huffman() function reimagined */ for (zptr = bptr + length; bptr < zptr; bptr++) { /* h2o huffdecode4() function reimagined (and inlined) */ entry = huff_decode_table[state] + (*bptr >> 4); if ((entry->flags & NGHTTP2_HUFF_FAIL) != 0) { if (WATCHMOD (h2ptr, WATCH_MOD_HTTP2)) WatchThis (WATCHITM(h2ptr), WATCH_MOD_HTTP2, "ERROR_PROTOCOL"); return (-HTTP2_ERROR_PROTOCOL); } if ((entry->flags & NGHTTP2_HUFF_SYM) != 0) *sptr++ = entry->sym; state = entry->state; eos = (entry->flags & NGHTTP2_HUFF_ACCEPTED) != 0; entry = huff_decode_table[state] + (*bptr & 0xf); if ((entry->flags & NGHTTP2_HUFF_FAIL) != 0) { if (WATCHMOD (h2ptr, WATCH_MOD_HTTP2)) WatchThis (WATCHITM(h2ptr), WATCH_MOD_HTTP2, "ERROR_PROTOCOL"); return (-HTTP2_ERROR_PROTOCOL); } if ((entry->flags & NGHTTP2_HUFF_SYM) != 0) *sptr++ = entry->sym; state = entry->state; eos = (entry->flags & NGHTTP2_HUFF_ACCEPTED) != 0; } if (!eos) { if (WATCHMOD (h2ptr, WATCH_MOD_HTTP2)) WatchThis (WATCHITM(h2ptr), WATCH_MOD_HTTP2, "ERROR_PROTOCOL"); return (-HTTP2_ERROR_PROTOCOL); } } else for (zptr = bptr + length; bptr < zptr; *sptr++ = *bptr++); *sptr = '\0'; *baddr = bptr; *raddr = bufptr[idx]; *rlenaddr = sptr - bufptr[idx]; if (WATCHMOD (h2ptr, WATCH_MOD_HTTP2)) WatchThis (WATCHITM(h2ptr), WATCH_MOD_HTTP2, "!UL->!UL !UL% |!AZ", length, sptr-bufptr[idx], (sptr-bufptr[idx]) ? length * 100 / (sptr-bufptr[idx]) : 0, bufptr[idx]); return (0); } /*****************************************************************************/ /* Encode a string. */ int HpackEncodeString ( HTTP2_STRUCT *h2ptr, uchar **baddr, uchar *bzptr, uchar *sptr, uint slen ) { #define ENDODE_HUFFMAN 1 int retval; uchar *bptr; #if ENCODE_HUFFMAN int clen; uint bits, remain; uchar *cptr; nghttp2_huff_sym *hsptr; #endif /*********/ /* begin */ /*********/ if (WATCHMOD (h2ptr, WATCH_MOD_HTTP2)) WatchThis (WATCHITM(h2ptr), WATCH_MOD_HTTP2, "HpackEncodeString() !UL !AZ", slen, sptr); bptr = *baddr; if (bptr >= bzptr) return (-HTTP2_ERROR_INTERNAL); *bptr = 0; #if ENCODE_HUFFMAN /* h2o encode_huffman() function reimagined (and inlined) */ if (slen) { /* first calculate the length of the encoded string */ bits = 0; remain = 40; clen = slen; cptr = sptr; while (clen) { hsptr = huff_decode_table + *cptr++; bits |= (uint)hsptr->code << (remain - hsptr->nbits); remain -= hsptr->nbits; while (remain <= 32) { bptr++; bits <<= 8; remain += 8; if (--clen == 0) break; } } if (remain != 40) { bits |= (1 << remain) - 1; bptr++; } clen = bptr - *baddr; /* then insert that length as an integer */ bptr = *baddr; retval = HpackEncodeInt32 (h2ptr, baddr, bzptr, 7, clen); if (retval < 0) return (retval); /* and OR-in the huffman-encoded flag */ *bptr |= 0x80; bptr = *baddr; if (bptr >= bzptr) return (-HTTP2_ERROR_INTERNAL); /* now actually huffman encode the string */ bits = 0; remain = 40; clen = slen; cptr = sptr; while (clen) { hsptr = huff_decode_table + *cptr++; bits |= (uint)hsptr->code << (remain - hsptr->nbits); remain -= hsptr->nbits; while (remain <= 32) { *bptr++ = bits >> 32; bits <<= 8; remain += 8; if (--clen == 0) break; } if (bptr >= bzptr) return (-HTTP2_ERROR_INTERNAL); } if (remain != 40) { bits |= (1 << remain) - 1; *bptr++ = bits >> 32; if (bptr >= bzptr) return (-HTTP2_ERROR_INTERNAL); } } else { retval = HpackEncodeInt32 (h2ptr, &bptr, bzptr, 7, slen); if (retval < 0) return (retval); } #else /* ENCODE_HUFFMAN */ retval = HpackEncodeInt32 (h2ptr, &bptr, bzptr, 7, slen); if (retval < 0) return (retval); while (slen-- && bptr < bzptr) *bptr++ = *sptr++; #endif /* ENCODE_HUFFMAN */ if (bptr >= bzptr) return (-HTTP2_ERROR_INTERNAL); *baddr = bptr; return (0); } /*****************************************************************************/ /* Return the length of the value (>=0) if hit, negative error code if lookup failed. Name and value pointers are updated if hit, not modified if lookup failed. Used by both client (request headers) and server (response headers) dynamic tables. */ int HpackGetIndex ( HPACK_TABLE_STRUCT *tabptr, uint index, uchar **naddr, uint **nlenaddr, uchar **vaddr, uint **vlenaddr ) { int count; uchar *nptr, *vptr; HPACK_ENTRY_STRUCT *heptr; HTTP2_STRUCT *h2ptr; /*********/ /* begin */ /*********/ h2ptr = tabptr->h2ptr; if (WATCHMOD (h2ptr, WATCH_MOD_HTTP2)) WatchThis (WATCHITM(h2ptr), WATCH_MOD_HTTP2, "HpackGetIndex() table:!AZ index:!UL", tabptr == &h2ptr->HpackClientTable ? "CLIENT" : "SERVER", index); if (!index) return (-HTTP2_ERROR_PROTOCOL); if (index <= HPACK_STATIC_TABLE_COUNT) heptr = &HpackStaticTable[index]; else { if (WATCHMOD (h2ptr, WATCH_MOD_HTTP2)) HpackWatchTable (tabptr); if (heptr = tabptr->head) for (count = HPACK_STATIC_TABLE_COUNT + 1; heptr && count < index; count++) heptr = heptr->blink; } if (heptr) { if (WATCHMOD (h2ptr, WATCH_MOD_HTTP2)) WatchThis (WATCHITM(h2ptr), WATCH_MOD_HTTP2, "!&Z=!&Z", heptr->name, heptr->value); *naddr = heptr->name; *nlenaddr = heptr->nlen; *vaddr = heptr->value; *vlenaddr = heptr->vlen; heptr->hits++; return (0); } if (WATCHMOD (h2ptr, WATCH_MOD_HTTP2)) WatchThis (WATCHITM(h2ptr), WATCH_MOD_HTTP2, "HTTP2_ERROR_INTERNAL"); return (-HTTP2_ERROR_INTERNAL); } /*****************************************************************************/ /* Insert the supplied name/value pair at the head of the dynamic table as described in RFC7541 section 4. Used to maintain both client (request headers) and server (response headers) dynamic tables. */ int HpackAddToTable ( HPACK_TABLE_STRUCT *tabptr, uchar *nptr, uint nlen, uchar *vptr, uint vlen ) { int size; uchar *sptr; HPACK_ENTRY_STRUCT *blink, *heptr; HTTP2_STRUCT *h2ptr; /*********/ /* begin */ /*********/ h2ptr = tabptr->h2ptr; if (WATCHMOD (h2ptr, WATCH_MOD_HTTP2)) { WatchThis (WATCHITM(h2ptr), WATCH_MOD_HTTP2, "HpackAddToTable() table:!AZ !SL|!AZ| !SL|!AZ", tabptr == &h2ptr->HpackClientTable ? "CLIENT" : "SERVER", nlen, nptr, vlen, vptr); HpackWatchTable (tabptr); } if (!nlen) return (-HTTP2_ERROR_INTERNAL); /* RFC7541 4.1 specifies 32 bytes plus the two strings */ size = nlen + vlen + sizeof(HPACK_ENTRY_STRUCT); if (size > tabptr->max) { if (WATCHING (h2ptr, WATCH_HTTP2)) WatchThis (WATCHITM(h2ptr), WATCH_HTTP2, "entry !UL > !UL bytes", size, tabptr->max); return (-HTTP2_ERROR_INTERNAL); } tabptr->size += size; heptr = VmGet2Heap (h2ptr, size); heptr->nlen = nlen; heptr->vlen = vlen; for (heptr->name = sptr = heptr->buffer; nlen--; *sptr++ = *nptr++); /* null is accomodated by the |buffer[4]| space */ *sptr++ = '\0'; for (heptr->value = sptr; vlen--; *sptr++ = *vptr++); /* null is accomodated by the |buffer[4]| space */ *sptr = '\0'; /* insert at head of table */ blink = tabptr->head; tabptr->head = heptr; if (blink == NULL) tabptr->tail = heptr; else { heptr->flink = NULL; heptr->blink = blink; blink->flink = heptr; } tabptr->count++; if (WATCHMOD (h2ptr, WATCH_MOD_HTTP2)) HpackWatchTable (tabptr); if (tabptr->size > tabptr->max) HpackTrimTable (tabptr); return (0); } /*****************************************************************************/ /* Find the name and value combination, or simply name, in the static/dynamic table. Used to maintain server (response headers) dynamic table. Baseline the search of the table at the number of entries before beginning processing the current set of response headers. That is, do not use any entries added during processing of the current headers! Return the index number or zero to indicate not found. */ int HpackFindInTable ( HPACK_TABLE_STRUCT *tabptr, uchar *nptr, uint nlen, uchar *vptr, uint vlen ) { int index; HPACK_ENTRY_STRUCT *heptr; HTTP2_STRUCT *h2ptr; /*********/ /* begin */ /*********/ h2ptr = tabptr->h2ptr; if (WATCHMOD (h2ptr, WATCH_MOD_HTTP2)) WatchThis (WATCHITM(h2ptr), WATCH_MOD_HTTP2, "HpackFindInTable() table:!AZ !UL|!AZ| !UL|!AZ", tabptr == &h2ptr->HpackClientTable ? "CLIENT" : "SERVER", nlen, nptr, vlen, vptr); if (nptr == NULL) { /* set a baseline before the start of response header processing */ tabptr->baseline = HPACK_STATIC_TABLE_COUNT + tabptr->count; return (0); } /* search the static table for a corresponding entry */ for (index = 1; index <= HPACK_STATIC_TABLE_COUNT; index++) { heptr = &HpackStaticTable[index]; if (heptr->nlen != nlen) continue; if (heptr->vlen != vlen) continue; if (!MATCH4 (heptr->name, nptr)) continue; if (vptr && heptr->value && !MATCH4 (heptr->value, vptr)) continue; if (strcmp (heptr->name, nptr)) continue; if (vptr && heptr->value && strcmp (heptr->value, vptr)) continue; /* hit it! */ break; } if (index > HPACK_STATIC_TABLE_COUNT) { /* search the dynamic table for a corresponding entry */ if (WATCHMOD (h2ptr, WATCH_MOD_HTTP2)) HpackWatchTable (tabptr); for (heptr = tabptr->head; heptr != NULL; index++, heptr = heptr->blink) { if (index > tabptr->baseline) { /* limit search to entries made prior to this response */ heptr = NULL; break; } if (heptr->nlen != nlen) continue; if (heptr->vlen != vlen) continue; if (!MATCH4 (heptr->name, nptr)) continue; if (vptr && !MATCH4 (heptr->value, vptr)) continue; if (strcmp (heptr->name, nptr)) continue; if (vptr && strcmp (heptr->value, vptr)) continue; /* finally! */ heptr->hits++; break; } } if (WATCHMOD (h2ptr, WATCH_MOD_HTTP2)) WatchThis (WATCHITM(h2ptr), WATCH_MOD_HTTP2, "index:!UL", heptr ? index : 0); if (heptr) return (index); return (0); } /*****************************************************************************/ /* Beginning at the tail of the dynamic table, trim entries until the size is less than the table maximum size according to RFC7541 section 4. Used to maintain both client (request headers) and server (response headers) dynamic tables. */ void HpackTrimTable (HPACK_TABLE_STRUCT *tabptr) { int size; HPACK_ENTRY_STRUCT *flink, *heptr; HTTP2_STRUCT *h2ptr; /*********/ /* begin */ /*********/ h2ptr = tabptr->h2ptr; if (WATCHMOD (h2ptr, WATCH_MOD_HTTP2)) { WatchThis (WATCHITM(h2ptr), WATCH_MOD_HTTP2, "HpackTrimTable() table:!AZ size:!UL max!UL", tabptr == &h2ptr->HpackClientTable ? "CLIENT" : "SERVER", tabptr->size, tabptr->max); HpackWatchTable (tabptr); } if (tabptr->size <= tabptr->max) return; heptr = tabptr->tail; while (heptr != NULL) { if (tabptr->size <= tabptr->max) break; size = heptr->nlen + heptr->vlen + sizeof(HPACK_ENTRY_STRUCT); flink = heptr->flink; VmFreeFrom2Heap (h2ptr, heptr, FI_LI); /* if top of list */ if ((heptr = flink) == NULL) break; heptr->blink = NULL; tabptr->tail = heptr; tabptr->size -= size; tabptr->count--; if (HPACK_STATIC_TABLE_COUNT + tabptr->count < tabptr->baseline) tabptr->baseline = HPACK_STATIC_TABLE_COUNT + tabptr->count; } /* if an empty table (maximum size of zero) */ if (tabptr->tail == NULL) tabptr->head = NULL; if (WATCHMOD (h2ptr, WATCH_MOD_HTTP2)) HpackWatchTable (tabptr); } /*****************************************************************************/ /* */ #ifdef WATCH_MOD void HpackWatchTable (HPACK_TABLE_STRUCT *tabptr) { int count; HPACK_ENTRY_STRUCT *heptr; HTTP2_STRUCT *h2ptr; /*********/ /* begin */ /*********/ h2ptr = tabptr->h2ptr; WatchThis (WATCHITM(h2ptr), WATCH_MOD_HTTP2, "HPACK table:!AZ count:!UL baseline:!UL head:!&X tail:!&X size:!UL max:!UL", tabptr == &h2ptr->HpackClientTable ? "CLIENT" : "SERVER", tabptr->count, tabptr->baseline, tabptr->head, tabptr->tail, tabptr->size, tabptr->max); /* need the [x]detail module WATCHing checkbox for the content as well! */ if (!(WATCH_MODULE(WATCH_MOD__OTHER) && WATCH_MODULE(WATCH_MOD__DETAIL))) return; count = HPACK_STATIC_TABLE_COUNT + 1; for (heptr = tabptr->head; heptr; heptr = heptr->blink) WatchDataFormatted ("!3ZL this:!&X flink:!&X blink:!&X \ !UL|!AZ| !UL|!AZ|\n", count++, heptr, heptr->flink, heptr->blink, heptr->nlen, heptr->name, heptr->vlen, heptr->value); } #endif /* WATCH_MOD */ /*****************************************************************************/ /* */ void HpackReport ( REQUEST_STRUCT *rqptr, uchar *ConnectNumber ) { static char TableBeginFao [] = "\n\

\n\ \n
\n\ \n" #if WATCH_MOD "\ \n\ \ \ \n\ \ \n\ " #if WATCH_MOD "\ \ " #endif "\ \ \ \n"; static char TableEntryFao [] = "" #if WATCH_MOD "\ \ " #endif "\ \ \ \n"; static char TableEndFao [] = "
!AZ
" #else "
!AZ
" #endif "Count:!UL\ Size:!UL\ Maximum:!UL\ Compression:!UL%\
EntryFlinkBlinkHitsNameValue
!3ZL!&X!&X!&X!UL!AZ!AZ
\n
\n"; static char PageEndFao [] = "!AZ\ \n\ \n\ \n"; static char NotFoundFao [] = "

HTTP/2 connection !UL not found!!\n\ \n\ \n\ \n"; int count, percent, status; ulong FaoVector [32]; ulong *vecptr; uchar *cptr; HPACK_ENTRY_STRUCT *heptr; HPACK_TABLE_STRUCT *tabptr; HTTP2_STRUCT *h2ptr; /*********/ /* begin */ /*********/ if (WATCHMOD (rqptr, WATCH_MOD_HTTP2)) WatchThis (WATCHITM(rqptr), WATCH_MOD_HTTP2, "HpackReport() !AZ !&X", ConnectNumber, strtol (ConnectNumber, NULL, 16)); AdminPageTitle (rqptr, "HPACK Report"); /* find the specified HTTP2 entry */ count = atol (ConnectNumber); for (h2ptr = LIST_GET_HEAD(&Http2List); h2ptr != NULL; h2ptr = LIST_GET_NEXT(h2ptr)) if (h2ptr->ConnectNumber == count) break; if (h2ptr == NULL) { FaoToNet (rqptr, NotFoundFao, count); AdminEnd (rqptr); return; } for (tabptr = NULL;;) { if (tabptr == NULL) { tabptr = &h2ptr->HpackClientTable; cptr = "Client"; if (h2ptr->HpackClientOutputCount) percent = (h2ptr->HpackClientInputCount * 100) / h2ptr->HpackClientOutputCount; else percent = 0; } else if (tabptr == &h2ptr->HpackClientTable) { tabptr = &h2ptr->HpackServerTable; cptr = "Server"; if (h2ptr->HpackServerInputCount) percent = (h2ptr->HpackServerOutputCount * 100) / h2ptr->HpackServerInputCount; else percent = 0; } else break; vecptr = FaoVector; *vecptr++ = cptr; *vecptr++ = tabptr->count; *vecptr++ = tabptr->size; *vecptr++ = tabptr->max; *vecptr++ = percent; status = FaolToNet (rqptr, TableBeginFao, &FaoVector); if (VMSnok (status)) ErrorNoticed (rqptr, status, NULL, FI_LI); count = HPACK_STATIC_TABLE_COUNT + 1; for (heptr = tabptr->head; heptr != NULL; heptr = heptr->blink) { vecptr = FaoVector; *vecptr++ = count++; #if WATCH_MOD *vecptr++ = heptr; *vecptr++ = heptr->flink; *vecptr++ = heptr->blink; #endif *vecptr++ = heptr->hits; *vecptr++ = heptr->name; *vecptr++ = heptr->value; status = FaolToNet (rqptr, TableEntryFao, &FaoVector); if (VMSnok (status)) ErrorNoticed (rqptr, status, NULL, FI_LI); } FaolToNet (rqptr, TableEndFao, NULL); } FaoToNet (rqptr, PageEndFao, AdminRefresh()); rqptr->rqResponse.PreExpired = PRE_EXPIRE_ADMIN; ResponseHeader200 (rqptr, "text/html", NULL); AdminEnd (rqptr); } /*****************************************************************************/