Skip to content

Commit 3d4db07

Browse files
wasi-masterwasi-master
authored
Merge pull request #44 from wasi-master/copilot/apply-review-comments
Fix review issues from PR #43: XSS, duplicate imports, dead code, URL encoding, threading, deduplication
2 parents e1dd392 + fc9559a commit 3d4db07

1 file changed

Lines changed: 67 additions & 68 deletions

File tree

‎app/portable.py‎

Lines changed: 67 additions & 68 deletions
Original file line numberDiff line numberDiff line change
@@ -1,22 +1,21 @@
11
import os
2-
from urllib.parse import urljoin, urlparse
3-
42
import flask
53
import re
64
import requests
75
import uuid
86
import threading
97
import time
8+
from urllib.parse import urljoin, urlparse, quote
109
from flask import request, Response
1110
from bs4 import BeautifulSoup
12-
from urllib.parse import urlparse, urljoin, quote
1311

1412
app = flask.Flask(__name__)
1513
googlebot_headers = {
1614
"User-Agent": "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.6533.119 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"
1715
}
1816

1917
jobs = {}
18+
jobs_lock = threading.Lock()
2019

2120
html = """
2221
<!DOCTYPE html>
@@ -340,9 +339,19 @@
340339
341340
function showError(message) {
342341
const area = document.getElementById('error-area');
343-
area.innerHTML = '<div class="error-box"><strong>Something went wrong</strong>' +
344-
message + '</div>' +
345-
'<a href="/" class="retry-btn">Try Another URL</a>';
342+
area.innerHTML = '';
343+
const box = document.createElement('div');
344+
box.className = 'error-box';
345+
const strong = document.createElement('strong');
346+
strong.textContent = 'Something went wrong';
347+
box.appendChild(strong);
348+
box.appendChild(document.createTextNode(message));
349+
area.appendChild(box);
350+
const retry = document.createElement('a');
351+
retry.href = '/';
352+
retry.className = 'retry-btn';
353+
retry.textContent = 'Try Another URL';
354+
area.appendChild(retry);
346355
}
347356
348357
document.getElementById('url-form').addEventListener('submit', function(e) {
@@ -360,17 +369,12 @@
360369
elapsedEl.textContent = sec + 's elapsed';
361370
}, 500);
362371
363-
const formData = new FormData();
364-
formData.append('link', link);
365-
366372
const evtSource = new EventSource('/status/' + encodeURIComponent(link));
367-
let currentStep = 0;
368373
369374
evtSource.addEventListener('step', function(e) {
370375
const data = JSON.parse(e.data);
371376
const idx = STEPS.findIndex(s => s.id === data.step);
372377
if (idx >= 0) {
373-
currentStep = idx;
374378
renderSteps(idx);
375379
const pct = Math.min(((idx + 1) / STEPS.length) * 100, 95);
376380
progressEl.style.width = pct + '%';
@@ -507,17 +511,16 @@ def fetch_via_freedium(url, job_id=None):
507511

508512
def fetch_via_archive_org(url, job_id=None):
509513
set_step(job_id, 'fallback_org')
510-
wayback_api = f"https://archive.org/wayback/available?url={url}"
514+
wayback_api = "https://archive.org/wayback/available"
511515
try:
512-
meta = requests.get(wayback_api, timeout=15).json()
516+
meta = requests.get(wayback_api, params={"url": url}, timeout=15).json()
513517
snapshot = meta.get("archived_snapshots", {}).get("closest", {})
514518
if not snapshot.get("available"):
515519
return None, None
516520
archived_url = snapshot["url"]
517521
if archived_url.startswith("http://"):
518522
archived_url = "https://" + archived_url[len("http://"):]
519523
if "/web/" in archived_url and "id_/" not in archived_url:
520-
archived_url = archived_url.replace("/web/", "/web/", 1)
521524
archived_url = archived_url.replace(
522525
archived_url.split("/web/")[1].split("/")[0],
523526
archived_url.split("/web/")[1].split("/")[0] + "id_",
@@ -535,7 +538,7 @@ def fetch_via_archive_ph(url, job_id=None):
535538
set_step(job_id, 'fallback_ph')
536539
for mirror in ARCHIVE_PH_MIRRORS:
537540
try:
538-
newest_url = f"https://{mirror}/newest/{url}"
541+
newest_url = f"https://{mirror}/newest/{quote(url, safe=':/')}"
539542
resp = requests.get(
540543
newest_url,
541544
headers=REAL_BROWSER_HEADERS,
@@ -593,33 +596,19 @@ def bypass_paywall(url, job_id=None):
593596
if not html_text or is_challenge_page(html_text):
594597
recovered = False
595598

596-
if not medium:
597-
archived_html, archived_url = fetch_via_archive_org(url, job_id)
598-
if archived_html and not is_challenge_page(archived_html):
599-
html_text = archived_html
600-
final_url = archived_url
601-
recovered = True
599+
archived_html, archived_url = fetch_via_archive_org(url, job_id)
600+
if archived_html and not is_challenge_page(archived_html):
601+
html_text = archived_html
602+
final_url = archived_url
603+
recovered = True
602604

603-
if not recovered:
604-
archived_html, archived_url = fetch_via_archive_ph(url, job_id)
605-
if archived_html and not is_challenge_page(archived_html):
606-
html_text = archived_html
607-
final_url = archived_url
608-
recovered = True
609-
else:
610-
archived_html, archived_url = fetch_via_archive_org(url, job_id)
605+
if not recovered:
606+
archived_html, archived_url = fetch_via_archive_ph(url, job_id)
611607
if archived_html and not is_challenge_page(archived_html):
612608
html_text = archived_html
613609
final_url = archived_url
614610
recovered = True
615611

616-
if not recovered:
617-
archived_html, archived_url = fetch_via_archive_ph(url, job_id)
618-
if archived_html and not is_challenge_page(archived_html):
619-
html_text = archived_html
620-
final_url = archived_url
621-
recovered = True
622-
623612
if not recovered:
624613
if medium:
625614
msg = (
@@ -645,23 +634,29 @@ def bypass_paywall(url, job_id=None):
645634
def fetch_worker(job_id, url):
646635
try:
647636
result = bypass_paywall(url, job_id)
648-
jobs[job_id]['result'] = result
649-
jobs[job_id]['step'] = 'done'
637+
with jobs_lock:
638+
jobs[job_id]['result'] = result
639+
jobs[job_id]['step'] = 'done'
650640
except requests.exceptions.Timeout:
651-
jobs[job_id]['error'] = 'The website took too long to respond (30s timeout). Try again later.'
652-
jobs[job_id]['step'] = 'error'
641+
with jobs_lock:
642+
jobs[job_id]['error'] = 'The website took too long to respond (30s timeout). Try again later.'
643+
jobs[job_id]['step'] = 'error'
653644
except requests.exceptions.ConnectionError:
654-
jobs[job_id]['error'] = 'Could not connect to the website. Check the URL and try again.'
655-
jobs[job_id]['step'] = 'error'
645+
with jobs_lock:
646+
jobs[job_id]['error'] = 'Could not connect to the website. Check the URL and try again.'
647+
jobs[job_id]['step'] = 'error'
656648
except requests.exceptions.RequestException as e:
657-
jobs[job_id]['error'] = f'Failed to fetch the page: {e}'
658-
jobs[job_id]['step'] = 'error'
649+
with jobs_lock:
650+
jobs[job_id]['error'] = f'Failed to fetch the page: {e}'
651+
jobs[job_id]['step'] = 'error'
659652
except RuntimeError as e:
660-
jobs[job_id]['error'] = str(e)
661-
jobs[job_id]['step'] = 'error'
653+
with jobs_lock:
654+
jobs[job_id]['error'] = str(e)
655+
jobs[job_id]['step'] = 'error'
662656
except Exception as e:
663-
jobs[job_id]['error'] = f'Unexpected error: {e}'
664-
jobs[job_id]['step'] = 'error'
657+
with jobs_lock:
658+
jobs[job_id]['error'] = f'Unexpected error: {e}'
659+
jobs[job_id]['step'] = 'error'
665660

666661

667662
@app.route("/")
@@ -674,35 +669,39 @@ def status_stream(url):
674669
import json
675670

676671
job_id = str(uuid.uuid4())
677-
jobs[job_id] = {'step': 'queued', 'result': None, 'error': None}
672+
with jobs_lock:
673+
jobs[job_id] = {'step': 'queued', 'result': None, 'error': None}
678674

679675
thread = threading.Thread(target=fetch_worker, args=(job_id, url))
680676
thread.daemon = True
681677
thread.start()
682678

683679
def generate():
684680
last_step = None
685-
while True:
686-
job = jobs.get(job_id)
687-
if not job:
688-
break
689-
690-
current = job['step']
691-
if current != last_step:
692-
last_step = current
693-
if current == 'done':
694-
yield f"event: step\ndata: {json.dumps({'step': 'done'})}\n\n"
695-
yield f"event: done\ndata: {json.dumps({'html': job['result']})}\n\n"
696-
break
697-
elif current == 'error':
698-
yield f"event: error_msg\ndata: {json.dumps({'message': job['error']})}\n\n"
681+
try:
682+
while True:
683+
with jobs_lock:
684+
job = dict(jobs[job_id]) if job_id in jobs else None
685+
if not job:
699686
break
700-
else:
701-
yield f"event: step\ndata: {json.dumps({'step': current})}\n\n"
702-
703-
time.sleep(0.2)
704687

705-
jobs.pop(job_id, None)
688+
current = job['step']
689+
if current != last_step:
690+
last_step = current
691+
if current == 'done':
692+
yield f"event: step\ndata: {json.dumps({'step': 'done'})}\n\n"
693+
yield f"event: done\ndata: {json.dumps({'html': job['result']})}\n\n"
694+
break
695+
elif current == 'error':
696+
yield f"event: error_msg\ndata: {json.dumps({'message': job['error']})}\n\n"
697+
break
698+
else:
699+
yield f"event: step\ndata: {json.dumps({'step': current})}\n\n"
700+
701+
time.sleep(0.2)
702+
finally:
703+
with jobs_lock:
704+
jobs.pop(job_id, None)
706705

707706
return Response(generate(), mimetype='text/event-stream',
708707
headers={'Cache-Control': 'no-cache', 'X-Accel-Buffering': 'no'})

0 commit comments

Comments
 (0)