Normalize encoded URL paths. (#14)

Author: samuel-williams-shopify

guides/getting-started/readme.md

@@ -290,11 +290,14 @@ messy = Protocol::URL["https://example.com/a/b/../c/./d"]
 # Parsing preserves the original path until simplification is requested:
 messy.path.to_s  # => "/a/b/../c/./d"
 
+# Normalization is intentionally lossy and produces a canonical path:
 messy.normalize!
 messy.path.to_s  # => "/a/c/d"
 messy.to_s  # => "https://example.com/a/c/d"
 ```
 
+If the original path structure is significant, retain the parsed URL and do not call `normalize!`.
+
 ## Best Practices
 
 ### Choose the Right Class

lib/protocol/url/path.rb

@@ -20,7 +20,8 @@ class Path
 
 			EMPTY_SEGMENTS = [].freeze
 			ROOT_SEGMENTS = ["", ""].freeze
-			private_constant :EMPTY_SEGMENTS, :ROOT_SEGMENTS
+			NORMALIZATION_PATTERN = /%[0-9A-Fa-f]{2}|%|[^a-zA-Z0-9_.~!$&'()*+,;=:@-]/
+			private_constant :EMPTY_SEGMENTS, :ROOT_SEGMENTS, :NORMALIZATION_PATTERN
 
 			# Coerce an encoded string or encoded segment array into a path.
 			#
@@ -252,6 +253,42 @@ def local_path(root)
 			alias to_s encoded
 			alias to_str encoded
 
+			# Normalize the encoded spelling of this path.
+			#
+			# Percent-encoded unreserved characters are decoded, retained percent escapes
+			# use uppercase hexadecimal digits, and literal characters outside the path
+			# segment grammar are percent encoded. Reserved characters retain their
+			# encoded or literal form because those forms are not generally equivalent.
+			#
+			# This operation preserves the path structure. Use {simplify} separately when
+			# application semantics permit resolving dot segments or collapsing repeated separators.
+			#
+			# @returns [Path] The normalized path, or this path if already normalized.
+			# @raises [ArgumentError] If the path contains malformed percent encoding, NUL, or invalid string encoding.
+			def normalize
+				encoded = self.encoded
+				unless encoded.valid_encoding? && encoded.encoding.ascii_compatible?
+					raise ArgumentError, "Path segment has invalid encoding!"
+				end
+				
+				segments = self.segments
+				normalized_segments = nil
+				
+				segments.each_with_index do |segment, index|
+					next unless NORMALIZATION_PATTERN.match?(segment)
+					
+					normalized = normalize_segment(segment)
+					next if normalized == segment
+					
+					normalized_segments ||= segments.dup
+					normalized_segments[index] = normalized
+				end
+				
+				return self unless normalized_segments
+				
+				return self.class.new(nil, normalized_segments)
+			end
+			
 			# Simplify this path in place by resolving literal or percent-encoded dot segments and repeated separators.
 			#
 			# @returns [Path | Nil] This path when changed, otherwise `nil`.
@@ -342,6 +379,42 @@ def relative(from)
 
 			private
 
+			# Normalize one encoded path segment:
+			def normalize_segment(segment)
+				return segment.gsub(NORMALIZATION_PATTERN) do |character|
+					byte = character.getbyte(0)
+					
+					if byte == 0
+						raise ArgumentError, "Path segment contains NUL!"
+					elsif byte == 0x25
+						if character.bytesize == 1
+							raise ArgumentError, "String contains malformed percent encoding!"
+						end
+						
+						byte = character.byteslice(1, 2).to_i(16)
+						if byte == 0
+							raise ArgumentError, "Path segment contains NUL!"
+						elsif unreserved_byte?(byte)
+							byte.chr
+						else
+							character.upcase
+						end
+					else
+						Encoding.escape(character)
+					end
+				end
+			end
+			
+			# Whether the byte represents an unreserved URI character:
+			def unreserved_byte?(byte)
+				case byte
+				when 0x30..0x39, 0x41..0x5A, 0x61..0x7A, 0x2D, 0x2E, 0x5F, 0x7E
+					return true
+				else
+					return false
+				end
+			end
+			
 			# Identify dot segments, including percent-encoded spellings. RFC 3986 treats
 			# percent-encoded unreserved characters as equivalent to their literal forms;
 			# the WHATWG URL Standard explicitly recognizes `%2e`, `.%2e`, `%2e.`, and
@@ -415,9 +488,9 @@ def simplify_segments!(segments, start_index = nil)
 							offset += 1
 						end
 					elsif segment == "" && index != last_index
-						# Collapse repeated separators.
+						# Collapse repeated separators:
 					elsif dot == ".." && offset > 0 && dot_segment(segments[offset - 1]) != ".."
-						# Pop a component, but never pop the absolute-path root.
+						# Pop a component, but never pop the absolute-path root:
 						offset -= 1 if segments[offset - 1] != ""
 
 						# A trailing parent reference also denotes a directory.

lib/protocol/url/relative.rb

@@ -132,12 +132,17 @@ def with(path: nil, query: @query, fragment: @fragment, pop: true)
 				self.class.new(path || @path, query, fragment)
 			end
 
-			# Normalize the path by resolving "." and ".." segments and removing duplicate slashes.
+			# Normalize the encoded path and simplify its structure.
 			#
-			# This modifies the URL in-place by simplifying the path component:
+			# This modifies the URL in-place by normalizing and simplifying the path component:
+			# - Decodes percent-encoded unreserved characters
+			# - Uses uppercase hexadecimal digits for retained percent escapes
 			# - Removes "." segments (current directory)
 			# - Resolves ".." segments (parent directory)
-			# - Collapses multiple consecutive slashes to single slashes (except at start)
+			# - Collapses empty path segments represented by consecutive slashes
+			#
+			# Normalization is intentionally lossy. Callers that need to preserve the
+			# original path structure should retain the parsed URL and avoid this method.
 			#
 			# @returns [self] The normalized URL.
 			#
@@ -146,7 +151,7 @@ def with(path: nil, query: @query, fragment: @fragment, pop: true)
 			#   url.normalize!
 			#   url.path.to_s  # => "/foo/bar/qux"
 			def normalize!
-				@path = @path.simplify
+				@path = @path.normalize.simplify
 
 				return self
 			end

releases.md

@@ -1,5 +1,9 @@
 # Releases
 
+## Unreleased
+
+  - Add conservative normalization of encoded URL paths.
+
 ## v0.12.0
 
   - Allow unfrozen relative and absolute URLs to replace their components.

test/protocol/url/path.rb

@@ -309,6 +309,94 @@
 
 			expect(path.simplify.encoded).to be == "/a/c"
 		end
+		
+		it "resolves a parent following a repeated separator" do
+			path = Protocol::URL::Path["/a//../b"]
+			
+			expect(path.simplify.encoded).to be == "/b"
+		end
+	end
+	
+	with "#normalize" do
+		it "decodes percent-encoded unreserved characters" do
+			path = Protocol::URL::Path["/%41%7a%30%2d%2e%5f%7e"]
+			
+			expect(path.normalize.encoded).to be == "/Az0-._~"
+		end
+		
+		it "uses uppercase hexadecimal digits for retained percent escapes" do
+			path = Protocol::URL::Path["/a%2fb%3fc%ff"]
+			
+			expect(path.normalize.encoded).to be == "/a%2Fb%3Fc%FF"
+		end
+		
+		it "preserves literal path segment delimiters" do
+			path = Protocol::URL::Path["/!$&'()*+,;=:@"]
+			
+			expect(path.normalize).to be_equal(path)
+		end
+		
+		it "preserves the distinction between encoded and literal reserved characters" do
+			path = Protocol::URL::Path["/a%3Ab:a%2Fb"]
+			
+			expect(path.normalize).to be_equal(path)
+		end
+		
+		it "percent encodes characters outside the path segment grammar" do
+			path = Protocol::URL::Path["/hello world?[x]#"]
+			
+			expect(path.normalize.encoded).to be == "/hello%20world%3F%5Bx%5D%23"
+		end
+		
+		it "percent encodes literal unicode characters" do
+			path = Protocol::URL::Path["/❤️"]
+			
+			expect(path.normalize.encoded).to be == "/%E2%9D%A4%EF%B8%8F"
+		end
+		
+		it "preserves path structure" do
+			path = Protocol::URL::Path["//a/%2e%2e/%77elcome"]
+			
+			expect(path.normalize.encoded).to be == "//a/../welcome"
+		end
+		
+		it "returns itself when already normalized" do
+			path = Protocol::URL::Path["/welcome/a:b/%2F"]
+			
+			expect(path.normalize).to be_equal(path)
+		end
+		
+		it "rejects literal NUL" do
+			path = Protocol::URL::Path["/a\0b"]
+			
+			expect do
+				path.normalize
+			end.to raise_exception(ArgumentError, message: be == "Path segment contains NUL!")
+		end
+		
+		it "rejects percent-encoded NUL" do
+			path = Protocol::URL::Path["/a%00b"]
+			
+			expect do
+				path.normalize
+			end.to raise_exception(ArgumentError, message: be == "Path segment contains NUL!")
+		end
+		
+		it "rejects malformed percent encoding" do
+			["/a%", "/a%0", "/a%gg"].each do |encoded|
+				expect do
+					Protocol::URL::Path[encoded].normalize
+				end.to raise_exception(ArgumentError, message: be == "String contains malformed percent encoding!")
+			end
+		end
+		
+		it "rejects invalid string encoding" do
+			path = Protocol::URL::Path["/a\xFF".dup.force_encoding(::Encoding::UTF_8)]
+			
+			expect do
+				path.normalize
+			end.to raise_exception(ArgumentError, message: be == "Path segment has invalid encoding!")
+		end
 	end
 
 	with "#simplify!" do
@@ -444,6 +532,12 @@
 			expect(Protocol::URL::Path["documents/report.pdf"].local_path(root)).to be == expected
 		end
 
+		it "maps empty URL segments to the same local filesystem path" do
+			expected = File.join(root, "documents", "report.pdf")
+			
+			expect(Protocol::URL::Path["/documents//report.pdf"].local_path(root)).to be == expected
+		end
+		
 		it "unescapes percent-encoded and Unicode characters" do
 			expect(Protocol::URL::Path["/files/My%20Document.txt"].local_path(root)).to be == File.join(root, "files", "My Document.txt")
 			expect(Protocol::URL::Path["/files/%E2%9D%A4%EF%B8%8F.txt"].local_path(root)).to be == File.join(root, "files", "❤️.txt")

test/protocol/url/relative.rb

@@ -269,6 +269,18 @@
 	end
 
 	with "#normalize!" do
+		it "normalizes the encoded path" do
+			url = Protocol::URL::Relative.new("/%66oo/a%2fb")
+			url.normalize!
+			expect(url.path).to be == Protocol::URL::Path["/foo/a%2Fb"]
+		end
+		
+		it "simplifies normalized dot segments" do
+			url = Protocol::URL::Relative.new("/foo/%2e%2e/bar")
+			url.normalize!
+			expect(url.path).to be == Protocol::URL::Path["/bar"]
+		end
+		
 		it "removes dot segments" do
 			url = Protocol::URL::Relative.new("/foo/./bar")
 			url.normalize!
@@ -281,7 +293,7 @@
 			expect(url.path).to be == Protocol::URL::Path["/foo/baz"]
 		end
 
-		it "collapses multiple slashes" do
+		it "collapses empty path segments" do
 			url = Protocol::URL::Relative.new("/foo//bar///baz")
 			url.normalize!
 			expect(url.path).to be == Protocol::URL::Path["/foo/bar/baz"]