Support syntax-only SPDX expression parsing (#25)

* Support syntax-only SPDX expression parsing

* Validate SPDX reference identifiers

* Tighten SPDX identifier syntax validation

Author: abhinavgautam01

README.md

@@ -46,8 +46,17 @@ fmt.Println(expr.String())  // "MIT OR (GPL-2.0-only AND Apache-2.0)"
 // ParseStrict requires valid SPDX IDs (no fuzzy normalization)
 expr, err := spdx.ParseStrict("MIT OR Apache-2.0")  // succeeds
 expr, err := spdx.ParseStrict("Apache 2 OR MIT")    // fails
+
+// ParseSyntax validates expression grammar without requiring identifiers
+// to exist in the SPDX list bundled by this module
+expr, err := spdx.ParseSyntax("Future-License-1.0 OR MIT") // succeeds
 ```
 
+`ParseSyntax` is useful when the caller validates identifiers against another
+pinned data source. Known identifiers are returned in their canonical form,
+while well-formed unknown license and exception identifiers are preserved.
+`ParseStrict` continues to require identifiers from the bundled SPDX list.
+
 ### Rewrite expression identifiers
 
 Parse an expression once, then replace its identifiers while keeping its

parse.go

@@ -255,13 +255,17 @@ const (
 
 // parser parses SPDX expressions.
 type parser struct {
-	lexer   *lexer
-	current token
-	depth   int
+	lexer                   *lexer
+	current                 token
+	depth                   int
+	allowUnknownIdentifiers bool
 }
 
-func newParser(input string) (*parser, error) {
-	p := &parser{lexer: newLexer(input)}
+func newParser(input string, allowUnknownIdentifiers bool) (*parser, error) {
+	p := &parser{
+		lexer:                   newLexer(input),
+		allowUnknownIdentifiers: allowUnknownIdentifiers,
+	}
 	tok, err := p.lexer.next()
 	if err != nil {
 		return nil, err
@@ -306,21 +310,12 @@ func Parse(expression string) (Expression, error) {
 		return nil, err
 	}
 
-	p, err := newParser(normalized)
-	if err != nil {
-		return nil, err
-	}
-
-	expr, err := p.parseExpression()
+	p, err := newParser(normalized, false)
 	if err != nil {
 		return nil, err
 	}
 
-	if p.current.typ != tokenEOF {
-		return nil, fmt.Errorf("%w: %s", ErrUnexpectedToken, p.current.value)
-	}
-
-	return expr, nil
+	return p.parse()
 }
 
 // ParseStrict parses an SPDX expression requiring strict SPDX identifiers.
@@ -333,6 +328,30 @@ func Parse(expression string) (Expression, error) {
 //	ParseStrict("MIT OR Apache-2.0")  // succeeds
 //	ParseStrict("mit OR apache 2")    // fails - "apache 2" is not a valid SPDX ID
 func ParseStrict(expression string) (Expression, error) {
+	return parseWithoutNormalization(expression, false)
+}
+
+// ParseSyntax parses an SPDX expression without requiring bare license and
+// exception identifiers to exist in the bundled SPDX identifier list. It
+// validates identifier syntax, operators, modifiers, and grouping. Known
+// identifiers are returned in their canonical form; unknown identifiers are
+// preserved as written.
+//
+// Use ParseStrict when identifiers must also be present in the bundled SPDX
+// list.
+//
+// Example:
+//
+//	ParseSyntax("Future-License-1.0 OR MIT") // succeeds
+//	ParseStrict("Future-License-1.0 OR MIT") // fails
+func ParseSyntax(expression string) (Expression, error) {
+	return parseWithoutNormalization(expression, true)
+}
+
+func parseWithoutNormalization(
+	expression string,
+	allowUnknownIdentifiers bool,
+) (Expression, error) {
 	expression = strings.TrimSpace(expression)
 	if expression == "" {
 		return nil, ErrEmptyExpression
@@ -341,11 +360,15 @@ func ParseStrict(expression string) (Expression, error) {
 		return nil, ErrExpressionTooLarge
 	}
 
-	p, err := newParser(expression)
+	p, err := newParser(expression, allowUnknownIdentifiers)
 	if err != nil {
 		return nil, err
 	}
 
+	return p.parse()
+}
+
+func (p *parser) parse() (Expression, error) {
 	expr, err := p.parseExpression()
 	if err != nil {
 		return nil, err
@@ -426,9 +449,9 @@ func (p *parser) parseWith() (Expression, error) {
 			return nil, fmt.Errorf("%w: expected exception after WITH", ErrMissingOperand)
 		}
 
-		exception := lookupException(p.current.value)
-		if exception == "" {
-			return nil, fmt.Errorf("%w: %s", ErrInvalidException, p.current.value)
+		exception, err := p.resolveExceptionIdentifier(p.current.value)
+		if err != nil {
+			return nil, err
 		}
 
 		license.Exception = exception
@@ -471,59 +494,116 @@ func (p *parser) parseAtom() (Expression, error) {
 		return expr, nil
 
 	case tokenLicense:
-		value := p.current.value
-		upper := strings.ToUpper(value)
-
-		// Handle special values
-		if upper == "NONE" || upper == "NOASSERTION" {
-			if err := p.advance(); err != nil {
-				return nil, err
-			}
-			return &SpecialValue{Value: upper}, nil
-		}
+		return p.parseLicenseAtom()
 
-		// Look up the canonical license ID
-		id := lookupLicense(value)
-		if id == "" {
-			return nil, fmt.Errorf("%w: %s", ErrInvalidLicenseID, value)
-		}
-
-		license := &License{ID: id}
+	case tokenLicenseRef:
+		return p.parseLicenseReference(false)
 
-		if err := p.advance(); err != nil {
-			return nil, err
-		}
+	case tokenDocumentRef:
+		return p.parseLicenseReference(true)
 
-		// Check for +
-		if p.current.typ == tokenPlus {
-			license.Plus = true
-			if err := p.advance(); err != nil {
-				return nil, err
-			}
-		}
+	case tokenEOF:
+		return nil, ErrMissingOperand
 
-		return license, nil
+	default:
+		return nil, fmt.Errorf("%w: %s", ErrUnexpectedToken, p.current.value)
+	}
+}
 
-	case tokenLicenseRef:
-		ref := parseLicenseRef(p.current.value)
+func (p *parser) parseLicenseAtom() (Expression, error) {
+	value := p.current.value
+	upper := strings.ToUpper(value)
+	if upper == "NONE" || upper == "NOASSERTION" {
 		if err := p.advance(); err != nil {
 			return nil, err
 		}
-		return ref, nil
+		return &SpecialValue{Value: upper}, nil
+	}
 
-	case tokenDocumentRef:
-		ref := parseDocumentRef(p.current.value)
+	id, err := p.resolveLicenseIdentifier(value)
+	if err != nil {
+		return nil, err
+	}
+	license := &License{ID: id}
+	if err := p.advance(); err != nil {
+		return nil, err
+	}
+	if p.current.typ == tokenPlus {
+		license.Plus = true
 		if err := p.advance(); err != nil {
 			return nil, err
 		}
-		return ref, nil
+	}
+	return license, nil
+}
 
-	case tokenEOF:
-		return nil, ErrMissingOperand
+func (p *parser) resolveLicenseIdentifier(identifier string) (string, error) {
+	if id := lookupLicense(identifier); id != "" {
+		return id, nil
+	}
+	if p.allowUnknownIdentifiers && validIdentifier(identifier) {
+		return identifier, nil
+	}
+	return "", fmt.Errorf("%w: %s", ErrInvalidLicenseID, identifier)
+}
 
-	default:
-		return nil, fmt.Errorf("%w: %s", ErrUnexpectedToken, p.current.value)
+func (p *parser) resolveExceptionIdentifier(identifier string) (string, error) {
+	if exception := lookupException(identifier); exception != "" {
+		return exception, nil
 	}
+	if p.allowUnknownIdentifiers && validIdentifier(identifier) {
+		return identifier, nil
+	}
+	return "", fmt.Errorf("%w: %s", ErrInvalidException, identifier)
+}
+
+func (p *parser) parseLicenseReference(document bool) (Expression, error) {
+	value := p.current.value
+	var reference *LicenseRef
+	if document {
+		reference = parseDocumentRef(value)
+	} else {
+		reference = parseLicenseRef(value)
+	}
+	if !validLicenseReference(reference, document) {
+		return nil, fmt.Errorf("%w: %s", ErrInvalidLicenseID, value)
+	}
+	if err := p.advance(); err != nil {
+		return nil, err
+	}
+	return reference, nil
+}
+
+func validLicenseReference(reference *LicenseRef, document bool) bool {
+	if reference == nil || !validIdentifier(reference.LicenseRef) {
+		return false
+	}
+	if document {
+		return validIdentifier(reference.DocumentRef)
+	}
+	return reference.DocumentRef == ""
+}
+
+func validIdentifier(identifier string) bool {
+	if identifier == "" || !isIdentifierAlphanumeric(rune(identifier[0])) ||
+		!isIdentifierAlphanumeric(rune(identifier[len(identifier)-1])) {
+		return false
+	}
+	for _, character := range identifier {
+		switch {
+		case isIdentifierAlphanumeric(character):
+		case character == '-', character == '.':
+		default:
+			return false
+		}
+	}
+	return true
+}
+
+func isIdentifierAlphanumeric(character rune) bool {
+	return character >= 'a' && character <= 'z' ||
+		character >= 'A' && character <= 'Z' ||
+		character >= '0' && character <= '9'
 }
 
 // parseLicenseRef parses "LicenseRef-xxx" into a LicenseRef.