TagScope/config.yaml at main · JerushaGray/TagScope · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
# TagScope - Configuration File
# Copy and customize this file for your specific needs.

# Crawl Settings
crawl:
  max_pages: 100              # Maximum number of pages to crawl
  max_depth: 3                # Maximum depth from start URL
  rate_limit: 1.0             # Seconds between requests
  concurrent_pages: 3         # Number of pages to crawl simultaneously (1-10)
  timeout: 30                 # Page load timeout in seconds

# Retry Settings
retry:
  max_retries: 3              # Number of retries for failed requests
  retry_delay: 2.0            # Seconds to wait between retries
  retry_on_status: [500, 502, 503, 504, 429]

# Browser Settings
browser:
  headless: true              # Run browser in headless mode
  user_agent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
  viewport:
    width: 1920
    height: 1080
  javascript_enabled: true
  ignore_https_errors: false

# URL Filtering
filters:
  respect_robots: true
  exclude_patterns: []        # Regex patterns to exclude
  include_patterns: []        # Regex patterns to include (empty = include all)
  skip_extensions:
    - .pdf
    - .jpg
    - .jpeg
    - .png
    - .gif
    - .svg
    - .css
    - .js
    - .zip
    - .tar
    - .gz
    - .xml
    - .json
    - .mp4
    - .mp3
    - .avi
    - .mov

# Technology Detection
technology:
  detect_all: true            # Detect all 50 built-in technology patterns

# Tag Detection
tags:
  detect_all: true            # Detect all 73 configured marketing/analytics tags
  custom_patterns: {}         # Add custom tag patterns (same format as TAG_PATTERNS)

# DataLayer Analysis
datalayer:
  extract: true               # Extract dataLayer contents
  parse_events: true          # Parse and categorize GA4 events
  max_events: 100             # Maximum dataLayer events to capture per page

# Performance Metrics
performance:
  capture_metrics: true       # Capture page load metrics
  capture_screenshots: false  # Capture screenshots (increases storage/time)
  screenshot_format: "png"    # png or jpeg

# Output Settings
output:
  formats: ["json", "csv", "html"]
  prefix: "site-audit"        # Output file prefix
  save_progress: true         # Save progress periodically (enables resume)
  progress_interval: 10       # Save progress every N pages

# Logging
logging:
  level: "INFO"               # DEBUG, INFO, WARNING, ERROR
  log_file: "tagscope.log"  # Log file name (null for no file logging)
  console: true               # Log to console

# Resume Settings
resume:
  enabled: true               # Allow resuming interrupted crawls
  state_file: "crawl_state.json"