diff --git a/src/core/link_manager.py b/src/core/link_manager.py index 611fa81..299033f 100644 --- a/src/core/link_manager.py +++ b/src/core/link_manager.py @@ -54,6 +54,40 @@ def extract_links(self, soup, current_url, depth, should_crawl_callback): self.all_discovered_urls.add(clean_url) self.discovered_urls.append((clean_url, depth)) + """Extract images from HTML and add to discovery queue""" + imgs = soup.find_all('img', src=True) + for img in imgs: + src = img.get('src', '').strip() + if not src or src.startswith('data:'): + continue + + try: + absolute_url = urljoin(current_url, src) + parsed = urlparse(absolute_url) + + if parsed.scheme not in ('http', 'https'): + continue + + clean_url = f"{parsed.scheme}://{parsed.netloc}{parsed.path}" + if parsed.query: + clean_url += f"?{parsed.query}" + + with self.urls_lock: + if clean_url not in self.source_pages: + self.source_pages[clean_url] = [] + if current_url not in self.source_pages[clean_url]: + self.source_pages[clean_url].append(current_url) + + if (clean_url not in self.visited_urls and + clean_url not in self.all_discovered_urls and + clean_url != current_url): + + if should_crawl_callback(clean_url): + self.all_discovered_urls.add(clean_url) + self.discovered_urls.append((clean_url, depth)) + except Exception: + continue + def collect_all_links(self, soup, source_url, crawl_results): """Collect all links for the Links tab display""" links = soup.find_all('a', href=True) diff --git a/src/crawler.py b/src/crawler.py index c8d12fa..c1f0b5e 100644 --- a/src/crawler.py +++ b/src/crawler.py @@ -163,7 +163,8 @@ def _get_default_config(self): 'accept_language': 'en-US,en;q=0.9', 'respect_robots': True, 'allow_cookies': True, - 'include_extensions': ['html', 'htm', 'php', 'asp', 'aspx', 'jsp'], + 'include_extensions': ['html', 'htm', 'php', 'asp', 'aspx', 'jsp', + 'jpg', 'jpeg', 'png', 'gif', 'webp', 'svg', 'ico', 'avif', 'bmp'], 'exclude_extensions': ['pdf', 'doc', 'docx', 'zip', 'exe', 'dmg'], 'include_patterns': [], 'exclude_patterns': [], diff --git a/src/settings_manager.py b/src/settings_manager.py index f1448e7..c0aaf6f 100644 --- a/src/settings_manager.py +++ b/src/settings_manager.py @@ -91,7 +91,7 @@ def _get_default_settings(self): 'googleApiKey': '', # Filter settings - 'includeExtensions': 'html,htm,php,asp,aspx,jsp', + 'includeExtensions': 'html,htm,php,asp,aspx,jsp,jpg,jpeg,png,gif,webp,svg,ico,avif,bmp', 'excludeExtensions': 'pdf,doc,docx,zip,exe,dmg', 'includePatterns': '', 'excludePatterns': '', diff --git a/web/static/js/settings.js b/web/static/js/settings.js index c4ee9c3..874c72b 100644 --- a/web/static/js/settings.js +++ b/web/static/js/settings.js @@ -20,7 +20,7 @@ let defaultSettings = { googleApiKey: '', // Filter settings - includeExtensions: 'html,htm,php,asp,aspx,jsp', + includeExtensions: 'html,htm,php,asp,aspx,jsp,jpg,jpeg,png,gif,webp,svg,ico,avif,bmp', excludeExtensions: 'pdf,doc,docx,zip,exe,dmg', includePatterns: '', excludePatterns: '', diff --git a/web/templates/index.html b/web/templates/index.html index f54a3c9..0a45e19 100644 --- a/web/templates/index.html +++ b/web/templates/index.html @@ -635,7 +635,7 @@

Content Filters

- + Comma-separated list of extensions to crawl