Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 34 additions & 0 deletions src/core/link_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,40 @@ def extract_links(self, soup, current_url, depth, should_crawl_callback):
self.all_discovered_urls.add(clean_url)
self.discovered_urls.append((clean_url, depth))

"""Extract images from HTML and add to discovery queue"""
imgs = soup.find_all('img', src=True)
for img in imgs:
src = img.get('src', '').strip()
if not src or src.startswith('data:'):
continue

try:
absolute_url = urljoin(current_url, src)
parsed = urlparse(absolute_url)

if parsed.scheme not in ('http', 'https'):
continue

clean_url = f"{parsed.scheme}://{parsed.netloc}{parsed.path}"
if parsed.query:
clean_url += f"?{parsed.query}"

with self.urls_lock:
if clean_url not in self.source_pages:
self.source_pages[clean_url] = []
if current_url not in self.source_pages[clean_url]:
self.source_pages[clean_url].append(current_url)

if (clean_url not in self.visited_urls and
clean_url not in self.all_discovered_urls and
clean_url != current_url):

if should_crawl_callback(clean_url):
self.all_discovered_urls.add(clean_url)
self.discovered_urls.append((clean_url, depth))
except Exception:
continue

def collect_all_links(self, soup, source_url, crawl_results):
"""Collect all links for the Links tab display"""
links = soup.find_all('a', href=True)
Expand Down
3 changes: 2 additions & 1 deletion src/crawler.py
Original file line number Diff line number Diff line change
Expand Up @@ -163,7 +163,8 @@ def _get_default_config(self):
'accept_language': 'en-US,en;q=0.9',
'respect_robots': True,
'allow_cookies': True,
'include_extensions': ['html', 'htm', 'php', 'asp', 'aspx', 'jsp'],
'include_extensions': ['html', 'htm', 'php', 'asp', 'aspx', 'jsp',
'jpg', 'jpeg', 'png', 'gif', 'webp', 'svg', 'ico', 'avif', 'bmp'],
'exclude_extensions': ['pdf', 'doc', 'docx', 'zip', 'exe', 'dmg'],
'include_patterns': [],
'exclude_patterns': [],
Expand Down
2 changes: 1 addition & 1 deletion src/settings_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ def _get_default_settings(self):
'googleApiKey': '',

# Filter settings
'includeExtensions': 'html,htm,php,asp,aspx,jsp',
'includeExtensions': 'html,htm,php,asp,aspx,jsp,jpg,jpeg,png,gif,webp,svg,ico,avif,bmp',
'excludeExtensions': 'pdf,doc,docx,zip,exe,dmg',
'includePatterns': '',
'excludePatterns': '',
Expand Down
2 changes: 1 addition & 1 deletion web/static/js/settings.js
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ let defaultSettings = {
googleApiKey: '',

// Filter settings
includeExtensions: 'html,htm,php,asp,aspx,jsp',
includeExtensions: 'html,htm,php,asp,aspx,jsp,jpg,jpeg,png,gif,webp,svg,ico,avif,bmp',
excludeExtensions: 'pdf,doc,docx,zip,exe,dmg',
includePatterns: '',
excludePatterns: '',
Expand Down
2 changes: 1 addition & 1 deletion web/templates/index.html
Original file line number Diff line number Diff line change
Expand Up @@ -635,7 +635,7 @@ <h3>Content Filters</h3>

<div class="setting-group">
<label for="includeExtensions">Include File Extensions</label>
<input type="text" id="includeExtensions" value="html,htm,php,asp,aspx,jsp">
<input type="text" id="includeExtensions" value="html,htm,php,asp,aspx,jsp,jpg,jpeg,png,gif,webp,svg,ico,avif,bmp">
<span class="setting-help">Comma-separated list of extensions to crawl</span>
</div>

Expand Down