Use SWAR in cow_http1 header and path parsers

src/cow_http1.erl

@@ -31,6 +31,7 @@
 -export_type([version/0]).
 
 -include("cow_inline.hrl").
+-include("cow_swar.hrl").
 
 %% @doc Parse the request line.
 
@@ -70,7 +71,7 @@
 		<<"content-type: text/plain\r\n">>,
 		<<0:80, "\r\n">>
 	],
 	[{V, fun() -> {'EXIT', _} = (catch parse_request_line(V)) end}
 		|| V <- Tests].
 
 horse_parse_request_line_get_path() ->
@@ -178,6 +179,14 @@
 parse_header(Data, Acc) ->
 	parse_hd_name(Data, Acc, <<>>).
 
+%% SWAR fast path: 7 bytes are ASCII (< 128) and none of them is
+%% the terminator $: (0x3A) nor the whitespace error sentinels $\s
+%% (0x20) and $\t (0x09). On a match the 7 bytes are lowercased in
+%% parallel via ?swar_lower and appended in one bit-string operation,
+%% preserving the per-byte ?LOWER semantics of the slow path.
+parse_hd_name(<< W:56, Rest/bits >>, Acc, SoFar)
+		when ?is_safe_ascii_swar(W, $:, $\s, $\t) ->
+	parse_hd_name(Rest, Acc, << SoFar/binary, (?swar_lower(W)):56 >>);
 parse_hd_name(<< C, Rest/bits >>, Acc, SoFar) ->
 	case C of
 		$: -> parse_hd_before_value(Rest, Acc, SoFar);
@@ -200,6 +209,11 @@
 parse_hd_before_value(Data, Acc, Name) ->
 	parse_hd_value(Data, Acc, Name, <<>>).
 
+%% SWAR fast path: 7 bytes that are ASCII (< 128) and none is $\r (0x0D).
+%% Carriage return is the only delimiter for header values in HTTP/1.
+parse_hd_value(<< W:56, Rest/bits >>, Acc, Name, SoFar)
+		when ?is_safe_ascii_swar(W, $\r) ->
+	parse_hd_value(Rest, Acc, Name, << SoFar/binary, W:56 >>);
 parse_hd_value(<< $\r, Rest/bits >>, Acc, Name, SoFar) ->
 	case Rest of
 		<< $\n, C, Rest2/bits >> when C =:= $\s; C =:= $\t ->
@@ -272,6 +286,26 @@
 			"Content-Type: text/plain\r\n"
 			"\r\nRest">>)
 	).
+
+horse_parse_headers_lower() ->
+	horse:repeat(50000,
+		parse_headers(<<"server: Erlang/R17\r\n"
+			"date: Sun, 23 Feb 2014 09:30:39 GMT\r\n"
+			"x-forwarded-for: 198.51.100.42, 203.0.113.7\r\n"
+			"content-length: 12\r\n"
+			"content-type: text/plain\r\n"
+			"\r\nRest">>)
+	).
+
+horse_parse_headers_mixed_case() ->
+	horse:repeat(50000,
+		parse_headers(<<"X-Forwarded-For-Original-Value: 1\r\n"
+			"X-Request-ID: abc\r\n"
+			"X-Amzn-Trace-Id: Root=1-5759e988-bd862e3fe1be46a994272793\r\n"
+			"Strict-Transport-Security: max-age=63072000\r\n"
+			"Cache-Control: no-cache, no-store, must-revalidate\r\n"
+			"\r\nRest">>)
+	).
 -endif.
 
 %% @doc Extract path and query string from a binary,
@@ -284,10 +318,17 @@
 parse_fullpath(<<>>, Path) -> {Path, <<>>};
 parse_fullpath(<< $#, _/bits >>, Path) -> {Path, <<>>};
 parse_fullpath(<< $?, Qs/bits >>, Path) -> parse_fullpath_query(Qs, Path, <<>>);
+%% SWAR fast path: 7 bytes that are ASCII (< 128) and contain
+%% neither $# nor $?, the only delimiters for the path segment.
+parse_fullpath(<< W:56, Rest/bits >>, SoFar) when ?is_safe_ascii_swar(W, $#, $?) ->
+	parse_fullpath(Rest, << SoFar/binary, W:56 >>);
 parse_fullpath(<< C, Rest/bits >>, SoFar) -> parse_fullpath(Rest, << SoFar/binary, C >>).
 
 parse_fullpath_query(<<>>, Path, Query) -> {Path, Query};
 parse_fullpath_query(<< $#, _/bits >>, Path, Query) -> {Path, Query};
+%% SWAR fast path: 7 bytes that are ASCII (< 128) and contain no $#.
+parse_fullpath_query(<< W:56, Rest/bits >>, Path, SoFar) when ?is_safe_ascii_swar(W, $#) ->
+	parse_fullpath_query(Rest, Path, << SoFar/binary, W:56 >>);
 parse_fullpath_query(<< C, Rest/bits >>, Path, SoFar) ->
 	parse_fullpath_query(Rest, Path, << SoFar/binary, C >>).
 
@@ -303,6 +344,26 @@
 	{<<"/path/to/resource">>, <<"q=cowboy">>}
 		= parse_fullpath(<<"/path/to/resource?q=cowboy">>),
 	ok.
+
+horse_parse_fullpath_short() ->
+	horse:repeat(200000,
+		parse_fullpath(<<"/api/v1/users/42">>)
+	).
+
+horse_parse_fullpath_long() ->
+	horse:repeat(200000,
+		parse_fullpath(<<"/path/to/some/deeper/resource/in/the/api">>)
+	).
+
+horse_parse_fullpath_query() ->
+	horse:repeat(200000,
+		parse_fullpath(<<"/search?q=erlang+cowboy&sort=updated&state=open">>)
+	).
+
+horse_parse_fullpath_fragment() ->
+	horse:repeat(200000,
+		parse_fullpath(<<"/path/to/resource#anchor-name-here">>)
+	).
 -endif.
 
 %% @doc Convert an HTTP version to atom.

src/cow_swar.hrl

@@ -0,0 +1,117 @@
+%% Copyright (c) Loïc Hoguin <essen@ninenines.eu>
+%%
+%% Permission to use, copy, modify, and/or distribute this software for any
+%% purpose with or without fee is hereby granted, provided that the above
+%% copyright notice and this permission notice appear in all copies.
+%%
+%% THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+%% WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+%% MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+%% ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+%% WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+%% ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+%% OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+%% SWAR (SIMD Within A Register) helpers for validating 7 bytes at once.
+%%
+%% We use 56 bits (7 bytes) because it is the largest value that fits
+%% in a BEAM small integer (59-bit on 64-bit). This ensures bitwise and
+%% arithmetic guard operations (band, bor, bxor, +, -) compile to fast
+%% native code without bignum fallback.
+%%
+%% Call sites must provide a byte-by-byte fallback when the SWAR check
+%% fails; the same fallback is also responsible for handling tails
+%% shorter than 7 bytes.
+%%
+%% Inspired by the technique introduced in erlang/otp#10938
+%% (json:escape_binary, string:length).
+
+-define(SWAR_MASK80, 16#80808080808080).
+-define(SWAR_MASK01, 16#01010101010101).
+
+%% Detect if any byte in a 56-bit word is zero (Mycroft's trick).
+%%
+%% Full formula: ((V - 0x01..01) band (bnot V) band 0x80..80) =/= 0.
+%%   - (V - 0x01..01) borrows into a byte's high bit when that byte
+%%     underflows: true for byte 0x00, but ALSO for any byte >= 0x80,
+%%     whose high bit the subtraction does not clear.
+%%   - band (bnot V) keeps only lanes whose original byte was < 0x80,
+%%     discarding the >= 0x80 false positives from the previous step.
+%%   - band 0x80..80 isolates the per-byte high bit; non-zero => a 0x00.
+%%
+%% We drop the (bnot V) term. This is safe ONLY because every caller
+%% gates this macro behind `W band ?SWAR_MASK80 =:= 0 andalso ...`, so
+%% ?no_zero_byte runs exclusively on all-ASCII words (every byte < 0x80,
+%% XORed with an ASCII constant that is also < 0x80). With no byte >= 0x80
+%% the high-byte false positives (bnot V) guards against cannot occur, so
+%% the reduced check is exact on these inputs.
+%%
+%% Dropping bnot also avoids a JIT cost: bnot has no always-small fast
+%% path and would emit bignum-fallback type checks even when the result
+%% fits in a small.
+%%
+%% Consequence: a 7-byte window containing any non-ASCII byte (>= 0x80)
+%% fails the SWAR_MASK80 guard, short-circuits the andalso, and is NOT
+%% taken by this fast path. Such windows fall through to the byte-by-byte
+%% slow path, which handles them correctly. That is a deliberate fast-path
+%% miss, not an error. The ?is_safe_ascii_swar predicate macro that wraps
+%% this trick is therefore intentionally conservative: it REQUIRES all
+%% 7 bytes to be ASCII (high bit clear) AND free of their delimiter(s),
+%% so a window with any non-ASCII byte takes the slow path on purpose
+%% even when it contains no delimiter. Per essen's review this is
+%% acceptable: such windows are handled correctly, just more slowly.
+-define(no_zero_byte(V),
+	((V) - ?SWAR_MASK01) band ?SWAR_MASK80 =:= 0
+).
+
+%% Generalized SWAR safe-ASCII predicate: true iff every one of the 7 bytes
+%% in W is ASCII (high bit clear) AND differs from each delimiter argument.
+%% Reusable across Cowlib, Cowboy and Gun (they include cow_swar.hrl, which
+%% owns ?SWAR_MASK80/?SWAR_MASK01/?no_zero_byte that this expands to).
+%%
+%% Each delimiter A is broadcast across all 7 lanes by ?SWAR_MASK01 * (A);
+%% because A is a literal char constant at every call site the compiler
+%% folds the multiply at COMPILE time (verified: no '*' BIF is emitted and
+%% the folded 56-bit constant, e.g. 16#3A3A3A3A3A3A3A for $:, appears
+%% directly in the .beam assembly). XOR zeroes any lane equal to A, and
+%% ?no_zero_byte detects it. Delimiter args MUST be compile-time literals
+%% (a $char or integer literal); a runtime variable would emit a real
+%% runtime multiply and lose the optimization (and is illegal in a guard).
+-define(is_safe_ascii_swar(W, A),
+	(W) band ?SWAR_MASK80 =:= 0 andalso
+	?no_zero_byte((W) bxor (?SWAR_MASK01 * (A)))
+).
+-define(is_safe_ascii_swar(W, A, B),
+	(W) band ?SWAR_MASK80 =:= 0 andalso
+	?no_zero_byte((W) bxor (?SWAR_MASK01 * (A))) andalso
+	?no_zero_byte((W) bxor (?SWAR_MASK01 * (B)))
+).
+-define(is_safe_ascii_swar(W, A, B, C),
+	(W) band ?SWAR_MASK80 =:= 0 andalso
+	?no_zero_byte((W) bxor (?SWAR_MASK01 * (A))) andalso
+	?no_zero_byte((W) bxor (?SWAR_MASK01 * (B))) andalso
+	?no_zero_byte((W) bxor (?SWAR_MASK01 * (C)))
+).
+
+%% ASCII lowercase 7 bytes in parallel.
+%%
+%% For each lane, set bit 5 if the byte is in [0x41..0x5A] (A-Z),
+%% leave it alone otherwise. The standard range trick: bit 7 of
+%% (b + 0x3F) is set when b >= 0x41; bit 7 of (b + 0x25) is set
+%% when b >= 0x5B. So uppercase lanes are exactly those with
+%% bit 7 set in the first and clear in the second. Flipping bit
+%% 7 of the second term (bxor 0x80..80) lets us combine the two
+%% with a single band; the result has bit 7 set per uppercase
+%% lane. Shifting right by 2 moves that flag into bit 5, the
+%% 0x20 position; bor with the original W lowercases.
+%%
+%% Callers must verify `W band ?SWAR_MASK80 =:= 0` first: the
+%% lane additions assume each byte is < 0x80, otherwise carries
+%% propagate between lanes and the formula breaks.
+-define(swar_lower(W),
+	((W) bor (
+		((((W) + 16#3F3F3F3F3F3F3F)
+			band (((W) + 16#25252525252525) bxor ?SWAR_MASK80)
+			band ?SWAR_MASK80) bsr 2)
+	))
+).