From 8049575895cb15117f4dc6ffff323df38abd8e9e Mon Sep 17 00:00:00 2001 From: Daniel Naab Date: Mon, 20 Apr 2020 13:43:51 -0500 Subject: [PATCH 01/17] Basic Lighthouse scan using CLI --- README.md | 1 + scanners/lighthouse.py | 134 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 135 insertions(+) create mode 100644 scanners/lighthouse.py diff --git a/README.md b/README.md index 8fc178ed..d8276ecc 100644 --- a/README.md +++ b/README.md @@ -74,6 +74,7 @@ Append columns to each row with metadata about the scan itself, such as how long * `trustymail`: The `trustymail` command, available from the [`trustymail`](https://github.com/dhs-ncats/trustymail) Python package from the [Department of Homeland Security's NCATS team](https://github.com/dhs-ncats). (Override path by setting the `TRUSTYMAIL_PATH` environment variable.) * `third_parties` - What third party web services are in use, using [headless Chrome](https://developers.google.com/web/updates/2017/04/headless-chrome) to trap outgoing requests. (See documentation for [using](#headless-chrome) or [writing](#developing-chrome-scanners) Chrome-based scanners.) * `a11y` - Accessibility issues, using [`pa11y`](https://github.com/pa11y/pa11y). +* `lighthouse` - Scanner that runs [`Google Lighthouse`](https://developers.google.com/web/tools/lighthouse). * `noop` - Test scanner (no-op) used for development and debugging. Does nothing. ### Parallelization diff --git a/scanners/lighthouse.py b/scanners/lighthouse.py new file mode 100644 index 00000000..b5d1daa6 --- /dev/null +++ b/scanners/lighthouse.py @@ -0,0 +1,134 @@ +""" +Implements a Google Lighthouse scan. + +https://developers.google.com/web/tools/lighthouse + +To use, set the `LIGHTHOUSE_PATH` environment variable to the Lighthouse path. +""" + + +import json +import logging +import os + +from utils import utils + + + +LIGHTHOUSE_PATH = os.environ.get('LIGHTHOUSE_PATH', 'lighthouse') +LIGHTHOUSE_AUDITS = [ + 'color-contrast', + 'tap-targets', + 'image-alt', + 'input-image-alt', + 'font-size', + 'unminified-css', + 'unminified-javascript', + 'uses-text-compression', + 'total-byte-weight', + 'performance-budget', + 'timing-budget', + 'viewport', +] + + +# Set a default number of workers for a particular scan type. +# Overridden by a --workers flag. +workers = 2 + + +# Optional one-time initialization for all scans. +# If defined, any data returned will be passed to every scan instance and used +# to update the environment dict for that instance +# Will halt scan execution if it returns False or raises an exception. +# +# Run locally. +# def init(environment: dict, options: dict) -> dict: +# logging.debug("Init function.") + +# #cache_dir = options.get('_', {}).get('cache_dir', './cache') + +# return {'constant': 12345} + + +# Optional one-time initialization per-scan. If defined, any data +# returned will be passed to the instance for that domain and used to update +# the environment dict for that particular domain. +# +# Run locally. +# def init_domain(domain: str, environment: dict, options: dict) -> dict: +# logging.debug("Init function for %s." % domain) +# return {'variable': domain} + + + +def _url_for_domain(domain: str, cache_dir: str): + if domain.startswith('http://') or domain.startswith('https://'): + return domain + + # If we have data from pshtt, use the canonical endpoint. + canonical = utils.domain_canonical(domain, cache_dir=cache_dir) + if canonical: + return canonical + + # Otherwise, well, whatever. + return 'http://' + domain + + +# Required scan function. This is the meat of the scanner, where things +# that use the network or are otherwise expensive would go. +# +# Runs locally or in the cloud (Lambda). +def scan(domain: str, environment: dict, options: dict) -> dict: + logging.debug('Scan function called with options: %s' % options) + + cache_dir = options.get('_', {}).get('cache_dir', './cache') + + logging.info('Running Lighthouse CLI...') + raw = utils.scan([ + LIGHTHOUSE_PATH, + _url_for_domain(domain, cache_dir), + '--quiet', + '--output=json', + '--chrome-flags="--headless"', + *(f'--only-audits={audit}' for audit in LIGHTHOUSE_AUDITS), + ]) + logging.info('Done running Lighthouse CLI') + + return json.loads(raw)['audits'] + + +# Required CSV row conversion function. Usually one row, can be more. +# +# Run locally. +def to_rows(data): + return [[ + audit['id'], + audit['description'], + audit['title'], + audit['score'], + audit['scoreDisplayMode'] + ] for audit in data.values()] + + +# CSV headers for each row of data. Referenced locally. +headers = ['ID', 'Description', 'Title', 'Score', 'Score Display Mode'] + + +# TODO: Add ability to override default LIGHTHOUSE_AUDITS +# Optional handler for custom CLI parameters. Takes the args (as a list of +# strings) and returns a dict of the options values and names that the scanner +# expects, and a list of the arguments it didn't know how to parse. +# +# Should return a dict of the options parsed by this parser (not a mutated form +# of the opts that are passed to it) and a list of the remaining args that it +# didn't recognize. +# def handle_scanner_args(args, opts) -> Tuple[dict, list]: +# parser = ArgumentParser(prefix_chars='--') +# parser.add_argument('--noop-delay', nargs=1) +# parsed, unknown = parser.parse_known_args(args) +# dicted = vars(parsed) +# should_be_single = ['noop_delay'] +# dicted = make_values_single(dicted, should_be_single) +# dicted['noop_delay'] = int(dicted['noop_delay'], 10) +# return dicted, unknown From 92e9e7591ce1d394093675952af12501f2638d6e Mon Sep 17 00:00:00 2001 From: Daniel Naab Date: Mon, 20 Apr 2020 13:57:49 -0500 Subject: [PATCH 02/17] Linting --- scanners/lighthouse.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/scanners/lighthouse.py b/scanners/lighthouse.py index b5d1daa6..fc40f479 100644 --- a/scanners/lighthouse.py +++ b/scanners/lighthouse.py @@ -14,7 +14,6 @@ from utils import utils - LIGHTHOUSE_PATH = os.environ.get('LIGHTHOUSE_PATH', 'lighthouse') LIGHTHOUSE_AUDITS = [ 'color-contrast', @@ -61,7 +60,6 @@ # return {'variable': domain} - def _url_for_domain(domain: str, cache_dir: str): if domain.startswith('http://') or domain.startswith('https://'): return domain From 5804d707e1ce5052a73d76433cfe89496547d841 Mon Sep 17 00:00:00 2001 From: Daniel Naab Date: Tue, 21 Apr 2020 10:30:12 -0500 Subject: [PATCH 03/17] Alphabetize audit list --- scanners/lighthouse.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scanners/lighthouse.py b/scanners/lighthouse.py index fc40f479..dd92e6b5 100644 --- a/scanners/lighthouse.py +++ b/scanners/lighthouse.py @@ -17,16 +17,16 @@ LIGHTHOUSE_PATH = os.environ.get('LIGHTHOUSE_PATH', 'lighthouse') LIGHTHOUSE_AUDITS = [ 'color-contrast', - 'tap-targets', + 'font-size', 'image-alt', 'input-image-alt', - 'font-size', + 'performance-budget', + 'tap-targets', + 'timing-budget', + 'total-byte-weight', 'unminified-css', 'unminified-javascript', 'uses-text-compression', - 'total-byte-weight', - 'performance-budget', - 'timing-budget', 'viewport', ] From 7bfa36c6f0bff7c4bc090deb5205441c6ef49355 Mon Sep 17 00:00:00 2001 From: Daniel Naab Date: Wed, 22 Apr 2020 21:57:29 -0500 Subject: [PATCH 04/17] Add Lighthouse npm dependency --- package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/package.json b/package.json index d5de209c..7bc1951b 100644 --- a/package.json +++ b/package.json @@ -2,6 +2,7 @@ "name": "domain-scan-headless-lambda", "description": "Dependencies for building Lambda containers for headless Chrome in domain-scan.", "dependencies": { + "lighthouse": "^5.6.0", "puppeteer": "^2.0.0", "tar": "^5.0.5" } From b38ddd55f1a60a56369edafe02570eed0819394a Mon Sep 17 00:00:00 2001 From: Daniel Naab Date: Wed, 22 Apr 2020 22:57:53 -0500 Subject: [PATCH 05/17] Use --no-sandbox for Docker root --- scanners/lighthouse.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scanners/lighthouse.py b/scanners/lighthouse.py index dd92e6b5..900a864c 100644 --- a/scanners/lighthouse.py +++ b/scanners/lighthouse.py @@ -88,7 +88,7 @@ def scan(domain: str, environment: dict, options: dict) -> dict: _url_for_domain(domain, cache_dir), '--quiet', '--output=json', - '--chrome-flags="--headless"', + '--chrome-flags="--headless --no-sandbox"', *(f'--only-audits={audit}' for audit in LIGHTHOUSE_AUDITS), ]) logging.info('Done running Lighthouse CLI') From 96dd098100d0b94c75b3d36b4479fe5060d1ce3c Mon Sep 17 00:00:00 2001 From: Daniel Naab Date: Sun, 26 Apr 2020 20:00:27 -0500 Subject: [PATCH 06/17] Use shell=True for lighthouse indirect popen call --- scanners/lighthouse.py | 5 +++-- utils/utils.py | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/scanners/lighthouse.py b/scanners/lighthouse.py index 900a864c..81cb82c3 100644 --- a/scanners/lighthouse.py +++ b/scanners/lighthouse.py @@ -29,11 +29,12 @@ 'uses-text-compression', 'viewport', ] +CHROME_PATH = os.environ.get('CHROME_PATH') # Set a default number of workers for a particular scan type. # Overridden by a --workers flag. -workers = 2 +workers = 3 # Optional one-time initialization for all scans. @@ -90,7 +91,7 @@ def scan(domain: str, environment: dict, options: dict) -> dict: '--output=json', '--chrome-flags="--headless --no-sandbox"', *(f'--only-audits={audit}' for audit in LIGHTHOUSE_AUDITS), - ]) + ], shell=True) logging.info('Done running Lighthouse CLI') return json.loads(raw)['audits'] diff --git a/utils/utils.py b/utils/utils.py index 3722df0b..07bc6583 100644 --- a/utils/utils.py +++ b/utils/utils.py @@ -416,12 +416,12 @@ def try_command(command): return False -def scan(command, env=None, allowed_return_codes=[]): +def scan(command, env=None, allowed_return_codes=[], shell=False): try: response = subprocess.check_output( command, stderr=subprocess.STDOUT, - shell=False, env=env + shell=shell, env=env ) return str(response, encoding='UTF-8') except subprocess.CalledProcessError as exc: From fbdd2ff5169205334c7b33f0030608a1f9fca161 Mon Sep 17 00:00:00 2001 From: Daniel Naab Date: Sun, 26 Apr 2020 20:28:22 -0500 Subject: [PATCH 07/17] Pass Lighthouse cmd as a string rather than arg list, to work with shell=True. --- scanners/lighthouse.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/scanners/lighthouse.py b/scanners/lighthouse.py index 81cb82c3..8c40d28a 100644 --- a/scanners/lighthouse.py +++ b/scanners/lighthouse.py @@ -83,15 +83,17 @@ def scan(domain: str, environment: dict, options: dict) -> dict: cache_dir = options.get('_', {}).get('cache_dir', './cache') - logging.info('Running Lighthouse CLI...') - raw = utils.scan([ + lighthouse_cmd = ' '.join([ LIGHTHOUSE_PATH, _url_for_domain(domain, cache_dir), '--quiet', '--output=json', '--chrome-flags="--headless --no-sandbox"', *(f'--only-audits={audit}' for audit in LIGHTHOUSE_AUDITS), - ], shell=True) + ]) + + logging.info('Running Lighthouse CLI...') + raw = utils.scan(lighthouse_cmd, shell=True) logging.info('Done running Lighthouse CLI') return json.loads(raw)['audits'] From a7345893472282315ddff3acbfdd9640a81a62d5 Mon Sep 17 00:00:00 2001 From: Daniel Naab Date: Mon, 27 Apr 2020 13:26:11 -0500 Subject: [PATCH 08/17] Set lighthouse workers to 1 - for now --- scanners/lighthouse.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scanners/lighthouse.py b/scanners/lighthouse.py index 8c40d28a..728ed052 100644 --- a/scanners/lighthouse.py +++ b/scanners/lighthouse.py @@ -34,7 +34,7 @@ # Set a default number of workers for a particular scan type. # Overridden by a --workers flag. -workers = 3 +workers = 1 # Optional one-time initialization for all scans. From e7091e5443064a2b5eee10ff5096ad60ce88ac97 Mon Sep 17 00:00:00 2001 From: Daniel Naab Date: Mon, 27 Apr 2020 17:02:31 -0500 Subject: [PATCH 09/17] Return empty dict on Lighthouse error --- scanners/lighthouse.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/scanners/lighthouse.py b/scanners/lighthouse.py index 728ed052..1021ba74 100644 --- a/scanners/lighthouse.py +++ b/scanners/lighthouse.py @@ -83,9 +83,10 @@ def scan(domain: str, environment: dict, options: dict) -> dict: cache_dir = options.get('_', {}).get('cache_dir', './cache') + url = _url_for_domain(domain, cache_dir) lighthouse_cmd = ' '.join([ LIGHTHOUSE_PATH, - _url_for_domain(domain, cache_dir), + url, '--quiet', '--output=json', '--chrome-flags="--headless --no-sandbox"', @@ -96,7 +97,11 @@ def scan(domain: str, environment: dict, options: dict) -> dict: raw = utils.scan(lighthouse_cmd, shell=True) logging.info('Done running Lighthouse CLI') - return json.loads(raw)['audits'] + try: + return json.loads(raw)['audits'] + except BaseException as e: + logging.exception(f'Error running Lighthouse scan for {url}') + return {} # Required CSV row conversion function. Usually one row, can be more. From 90c070e5edaa8797563bfb239272e26db4464b8f Mon Sep 17 00:00:00 2001 From: Daniel Naab Date: Wed, 29 Apr 2020 12:13:07 -0500 Subject: [PATCH 10/17] Prune how much is logged with Lighthouse errors. --- scanners/lighthouse.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/scanners/lighthouse.py b/scanners/lighthouse.py index 1021ba74..37dabd6a 100644 --- a/scanners/lighthouse.py +++ b/scanners/lighthouse.py @@ -10,6 +10,7 @@ import json import logging import os +import subprocess from utils import utils @@ -79,7 +80,7 @@ def _url_for_domain(domain: str, cache_dir: str): # # Runs locally or in the cloud (Lambda). def scan(domain: str, environment: dict, options: dict) -> dict: - logging.debug('Scan function called with options: %s' % options) + logging.debug('Scan function called with options: %s', options) cache_dir = options.get('_', {}).get('cache_dir', './cache') @@ -94,13 +95,18 @@ def scan(domain: str, environment: dict, options: dict) -> dict: ]) logging.info('Running Lighthouse CLI...') - raw = utils.scan(lighthouse_cmd, shell=True) - logging.info('Done running Lighthouse CLI') try: + response = subprocess.check_output( + lighthouse_cmd, + stderr=subprocess.STDOUT, + shell=True, env=None + ) + raw = str(response, encoding='UTF-8') + logging.info('Done running Lighthouse CLI') return json.loads(raw)['audits'] - except BaseException as e: - logging.exception(f'Error running Lighthouse scan for {url}') + except subprocess.CalledProcessError as exc: + logging.warning("Error running Lighthouse scan for URL %s." % url) return {} From 38dfb58b61ad34098047afc24a7dec1d583ce6a2 Mon Sep 17 00:00:00 2001 From: Daniel Naab Date: Tue, 5 May 2020 12:42:43 -0500 Subject: [PATCH 11/17] Add speed-index Lighthouse audit --- scanners/lighthouse.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scanners/lighthouse.py b/scanners/lighthouse.py index 37dabd6a..352c5de5 100644 --- a/scanners/lighthouse.py +++ b/scanners/lighthouse.py @@ -29,6 +29,7 @@ 'unminified-javascript', 'uses-text-compression', 'viewport', + 'speed-index', ] CHROME_PATH = os.environ.get('CHROME_PATH') From 25c96850ee8fb694afadaeeaa9d134a27616c65b Mon Sep 17 00:00:00 2001 From: Daniel Naab Date: Tue, 5 May 2020 13:34:52 -0500 Subject: [PATCH 12/17] Remove unused exception var --- scanners/lighthouse.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scanners/lighthouse.py b/scanners/lighthouse.py index 352c5de5..0eaa2bc9 100644 --- a/scanners/lighthouse.py +++ b/scanners/lighthouse.py @@ -106,7 +106,7 @@ def scan(domain: str, environment: dict, options: dict) -> dict: raw = str(response, encoding='UTF-8') logging.info('Done running Lighthouse CLI') return json.loads(raw)['audits'] - except subprocess.CalledProcessError as exc: + except subprocess.CalledProcessError: logging.warning("Error running Lighthouse scan for URL %s." % url) return {} From 63a195c185ba946ad0935044f900ee9d8da64a33 Mon Sep 17 00:00:00 2001 From: Daniel Naab Date: Tue, 12 May 2020 13:33:05 -0500 Subject: [PATCH 13/17] Upgrade Puppeteer version --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 7bc1951b..15068a53 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,7 @@ "description": "Dependencies for building Lambda containers for headless Chrome in domain-scan.", "dependencies": { "lighthouse": "^5.6.0", - "puppeteer": "^2.0.0", + "puppeteer": "^2.1.1", "tar": "^5.0.5" } } From 47aca99a0a04872fbca917723198ede7b43bc613 Mon Sep 17 00:00:00 2001 From: Daniel Naab Date: Thu, 14 May 2020 17:17:57 -0500 Subject: [PATCH 14/17] Add in a first-pass at a node.js-based scanner. --- .snyk | 4 ++ lighthouse.js | 65 ++++++++++++++++++++ scanners/lighthouse.js | 51 ++++++++++++++++ scanners/lighthouse.py | 135 ++++++++++++++++++----------------------- 4 files changed, 178 insertions(+), 77 deletions(-) create mode 100644 .snyk create mode 100644 lighthouse.js create mode 100644 scanners/lighthouse.js diff --git a/.snyk b/.snyk new file mode 100644 index 00000000..8cb541f9 --- /dev/null +++ b/.snyk @@ -0,0 +1,4 @@ +# Snyk (https://snyk.io) policy file, patches or ignores known vulnerabilities. +version: v1.14.1 +ignore: {} +patch: {} diff --git a/lighthouse.js b/lighthouse.js new file mode 100644 index 00000000..a4089ab8 --- /dev/null +++ b/lighthouse.js @@ -0,0 +1,65 @@ +#!/usr/bin/env node + +/** + * Lighthouse scanner + * This module orchestrates parallel Lighthouse scans over one headless Chrome + * instance. + */ + +const chromeLauncher = require('chrome-launcher'); +const lighthouse = require('lighthouse'); +const puppeteer = require('puppeteer'); + + +var getBrowser = async () => { + return await puppeteer.launch({ + // TODO: Let executable path be overrideable. + // executablePath: config.executablePath, + headless: true, + ignoreHTTPSErrors: true, + args: [ + '--no-sandbox', + '--disable-gpu', + '--single-process' + ] + }); +}; + +function launchChromeAndRunLighthouse(url, opts, config = null) { + return chromeLauncher.launch({chromeFlags: opts.chromeFlags}).then(chrome => { + opts.port = chrome.port; + return lighthouse(url, opts, config).then(results => { + return chrome.kill().then(() => results.lhr) + }); + }); +} + +const opts = { + chromeFlags: ['--headless', '--no-sandbox'] +}; + +/* +function configure() { + let port; + chromeLauncher.launch({ + chromeFlags: ['--headless', '--no-sandbox'] + }).then(chrome => { + port = chrome.port + }); +} + +launchChromeAndRunLighthouse('https://www.whitehouse.gov', opts).then(results => { + console.log(results); +}); +*/ + +getBrowser().then(async browser => { + const url = 'https://www.whitehouse.gov'; + const {lhr} = await lighthouse(url, { + port: (new URL(browser.wsEndpoint())).port, + output: 'json', + logLevel: 'info', + }); + console.log(lhr); + await browser.close(); +}); diff --git a/scanners/lighthouse.js b/scanners/lighthouse.js new file mode 100644 index 00000000..e1b1ee2a --- /dev/null +++ b/scanners/lighthouse.js @@ -0,0 +1,51 @@ +'use strict'; + +const lighthouse = require('lighthouse'); + + +const LIGHTHOUSE_AUDITS = [ + 'color-contrast', + 'font-size', + 'image-alt', + 'input-image-alt', + 'performance-budget', + 'speed-index', + 'tap-targets', + 'timing-budget', + 'total-byte-weight', + 'unminified-css', + 'unminified-javascript', + 'uses-text-compression', + 'viewport', +] + + +// JS entry point for Lighthouse scan. +module.exports = { + scan: async (domain, environment, options, browser, page) => { + const url = 'https://' + domain; + try { + const output = await lighthouse(url, { + port: (new URL(browser.wsEndpoint())).port, + onlyAudits: LIGHTHOUSE_AUDITS, + + disableStorageReset: false, + saveAssets: false, + listAllAudits: false, + listTraceCategories: false, + printConfig: false, + output: [ 'json' ], + chromeFlags: '', + enableErrorReporting: false, + logLevel: 'silent', + outputPath: 'stdout', + }); + return output.lhr.audits; + + } catch (exc) { + return { + error: exc.message + } + } + } +} diff --git a/scanners/lighthouse.py b/scanners/lighthouse.py index 0eaa2bc9..5d9e0274 100644 --- a/scanners/lighthouse.py +++ b/scanners/lighthouse.py @@ -6,14 +6,16 @@ To use, set the `LIGHTHOUSE_PATH` environment variable to the Lighthouse path. """ - -import json -import logging import os -import subprocess -from utils import utils +# Can also be run in Lambda. +# NOTE: untested +lambda_support = False + +# Signal that this is a JS-based scan using headless Chrome. +# The scan method will be defined in lighthouse.js instead. +scan_headless = True LIGHTHOUSE_PATH = os.environ.get('LIGHTHOUSE_PATH', 'lighthouse') LIGHTHOUSE_AUDITS = [ @@ -39,78 +41,6 @@ workers = 1 -# Optional one-time initialization for all scans. -# If defined, any data returned will be passed to every scan instance and used -# to update the environment dict for that instance -# Will halt scan execution if it returns False or raises an exception. -# -# Run locally. -# def init(environment: dict, options: dict) -> dict: -# logging.debug("Init function.") - -# #cache_dir = options.get('_', {}).get('cache_dir', './cache') - -# return {'constant': 12345} - - -# Optional one-time initialization per-scan. If defined, any data -# returned will be passed to the instance for that domain and used to update -# the environment dict for that particular domain. -# -# Run locally. -# def init_domain(domain: str, environment: dict, options: dict) -> dict: -# logging.debug("Init function for %s." % domain) -# return {'variable': domain} - - -def _url_for_domain(domain: str, cache_dir: str): - if domain.startswith('http://') or domain.startswith('https://'): - return domain - - # If we have data from pshtt, use the canonical endpoint. - canonical = utils.domain_canonical(domain, cache_dir=cache_dir) - if canonical: - return canonical - - # Otherwise, well, whatever. - return 'http://' + domain - - -# Required scan function. This is the meat of the scanner, where things -# that use the network or are otherwise expensive would go. -# -# Runs locally or in the cloud (Lambda). -def scan(domain: str, environment: dict, options: dict) -> dict: - logging.debug('Scan function called with options: %s', options) - - cache_dir = options.get('_', {}).get('cache_dir', './cache') - - url = _url_for_domain(domain, cache_dir) - lighthouse_cmd = ' '.join([ - LIGHTHOUSE_PATH, - url, - '--quiet', - '--output=json', - '--chrome-flags="--headless --no-sandbox"', - *(f'--only-audits={audit}' for audit in LIGHTHOUSE_AUDITS), - ]) - - logging.info('Running Lighthouse CLI...') - - try: - response = subprocess.check_output( - lighthouse_cmd, - stderr=subprocess.STDOUT, - shell=True, env=None - ) - raw = str(response, encoding='UTF-8') - logging.info('Done running Lighthouse CLI') - return json.loads(raw)['audits'] - except subprocess.CalledProcessError: - logging.warning("Error running Lighthouse scan for URL %s." % url) - return {} - - # Required CSV row conversion function. Usually one row, can be more. # # Run locally. @@ -128,6 +58,57 @@ def to_rows(data): headers = ['ID', 'Description', 'Title', 'Score', 'Score Display Mode'] +# +# Below is an implementation that will spawn Lighthouse via its cli rather than +# use a Puppeteer-managed headless Chrome. +# + +# def _url_for_domain(domain: str, cache_dir: str): +# if domain.startswith('http://') or domain.startswith('https://'): +# return domain + +# # If we have data from pshtt, use the canonical endpoint. +# canonical = utils.domain_canonical(domain, cache_dir=cache_dir) +# if canonical: +# return canonical + +# # Otherwise, well, whatever. +# return 'http://' + domain + +# Required scan function. This is the meat of the scanner, where things +# that use the network or are otherwise expensive would go. +# +# Runs locally or in the cloud (Lambda). +# def scan(domain: str, environment: dict, options: dict) -> dict: +# logging.debug('Scan function called with options: %s', options) + +# cache_dir = options.get('_', {}).get('cache_dir', './cache') + +# url = _url_for_domain(domain, cache_dir) +# lighthouse_cmd = ' '.join([ +# LIGHTHOUSE_PATH, +# url, +# '--quiet', +# '--output=json', +# '--chrome-flags="--headless --no-sandbox"', +# *(f'--only-audits={audit}' for audit in LIGHTHOUSE_AUDITS), +# ]) + +# logging.info('Running Lighthouse CLI...') + +# try: +# response = subprocess.check_output( +# lighthouse_cmd, +# stderr=subprocess.STDOUT, +# shell=True, env=None +# ) +# raw = str(response, encoding='UTF-8') +# logging.info('Done running Lighthouse CLI') +# return json.loads(raw)['audits'] +# except subprocess.CalledProcessError: +# logging.warning("Error running Lighthouse scan for URL %s." % url) +# return {} + # TODO: Add ability to override default LIGHTHOUSE_AUDITS # Optional handler for custom CLI parameters. Takes the args (as a list of # strings) and returns a dict of the options values and names that the scanner From 27fada53bd5ca5db1cf83125a8d1fb4ccb1197d2 Mon Sep 17 00:00:00 2001 From: Daniel Naab Date: Mon, 18 May 2020 14:42:35 -0500 Subject: [PATCH 15/17] Handle Lighthouse scans with errors more gracefully, by omitting them from CSV results and adding an empty error value to successful scans so its shape is consistent on each. --- scanners/lighthouse.js | 5 ++++- scanners/lighthouse.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/scanners/lighthouse.js b/scanners/lighthouse.js index e1b1ee2a..68ee65c7 100644 --- a/scanners/lighthouse.js +++ b/scanners/lighthouse.js @@ -40,7 +40,10 @@ module.exports = { logLevel: 'silent', outputPath: 'stdout', }); - return output.lhr.audits; + return { + error: '', + ...output.lhr.audits + }; } catch (exc) { return { diff --git a/scanners/lighthouse.py b/scanners/lighthouse.py index 5d9e0274..7be7c3ef 100644 --- a/scanners/lighthouse.py +++ b/scanners/lighthouse.py @@ -51,7 +51,7 @@ def to_rows(data): audit['title'], audit['score'], audit['scoreDisplayMode'] - ] for audit in data.values()] + ] for name, audit in data.items() if name != 'error'] # CSV headers for each row of data. Referenced locally. From 1d56f2079fa58fe296af03bd9fa8f82743d4e594 Mon Sep 17 00:00:00 2001 From: Daniel Naab Date: Mon, 18 May 2020 14:47:38 -0500 Subject: [PATCH 16/17] Linting --- utils/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils/utils.py b/utils/utils.py index 07bc6583..0fe73516 100644 --- a/utils/utils.py +++ b/utils/utils.py @@ -707,5 +707,5 @@ def suffix_pattern(suffixes): return re.compile("(?:%s)$" % center) -def flatten(l): - return list(chain.from_iterable(l)) +def flatten(lst): + return list(chain.from_iterable(lst)) From 3fd38bb361b4ef19df604e71c0d6256fb6cab207 Mon Sep 17 00:00:00 2001 From: Daniel Naab Date: Tue, 19 May 2020 13:50:55 -0500 Subject: [PATCH 17/17] Return "null" on failed scans, to defer on how to represent errors to the calling "scan" entrypoint script. --- scanners/lighthouse.js | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/scanners/lighthouse.js b/scanners/lighthouse.js index 68ee65c7..6174d0a6 100644 --- a/scanners/lighthouse.js +++ b/scanners/lighthouse.js @@ -40,15 +40,11 @@ module.exports = { logLevel: 'silent', outputPath: 'stdout', }); - return { - error: '', - ...output.lhr.audits - }; + return output.lhr.audits; } catch (exc) { - return { - error: exc.message - } + console.log('problem scanning ' + domain + ' ' + exc.message); + return null; } } }