From e26665f4678b6551748a629f09138506d5716936 Mon Sep 17 00:00:00 2001 From: Shumin Liu Date: Fri, 14 Aug 2026 15:57:26 +1000 Subject: [PATCH 1/8] feat(web): describe each page for search engines MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every page sent a and nothing else, so a search engine built its own snippet out of the first text it found. On the home page that is the terminal banner ($ pip install ojhunt), which tells a reader nothing about the project. A shared link showed no preview either. base.html.jinja now carries a meta description, a canonical link, and the Open Graph tags. Each page declares its summary in a meta_description block; the home and About pages inherit the default, which summarises the project as a whole. Descriptions stay at 160 characters or shorter, where the snippet is cut. The easter-egg routes answer 200 on 40+ scanner paths (/admin, /.env, /wp-login.php). Those pages and the 404 page now send noindex and no canonical URL, so they stay out of the index. Page routes render through the new render_page() helper, which supplies the absolute base URL and the canonical URL from the request — the same request.base_url pattern robots.txt and sitemap.xml already use. Resolves #271 --- src/ojhunt/web/app.py | 10 +- src/ojhunt/web/pages.py | 69 ++++++--- src/ojhunt/web/templates/404.html.jinja | 2 + src/ojhunt/web/templates/base.html.jinja | 10 ++ src/ojhunt/web/templates/crawlers.html.jinja | 2 + .../web/templates/easter_egg.html.jinja | 2 + .../web/templates/easter_egg_rick.html.jinja | 2 + .../web/templates/pdf_legacy.html.jinja | 2 + src/ojhunt/web/templates/pdf_merge.html.jinja | 2 + tests/web/pages_test.py | 138 ++++++++++++++++++ 10 files changed, 213 insertions(+), 26 deletions(-) diff --git a/src/ojhunt/web/app.py b/src/ojhunt/web/app.py index b202404..5150798 100644 --- a/src/ojhunt/web/app.py +++ b/src/ojhunt/web/app.py @@ -24,7 +24,7 @@ from ojhunt.web.api import router as api_router from ojhunt.web.crawler_status import start_checker, stop_checker from ojhunt.web.http_client import close_http_client, get_http_client, init_http_client -from ojhunt.web.pages import jinja_env +from ojhunt.web.pages import render_page from ojhunt.web.pages import router as pages_router load_dotenv() @@ -145,9 +145,13 @@ async def http_exception_handler( ) -> Response: if exc.status_code == 404 and not request.url.path.startswith("/api"): image_filename = random.choice(["cat.jpg", "man.jpg", "metro.jpg"]) - template = jinja_env.get_template("404.html.jinja") return HTMLResponse( - template.render(image_filename=image_filename), + render_page( + "404.html.jinja", + request, + canonical=False, + image_filename=image_filename, + ), status_code=404, ) return await _default_http_exception_handler(request, exc) diff --git a/src/ojhunt/web/pages.py b/src/ojhunt/web/pages.py index 5ce9dde..6468567 100644 --- a/src/ojhunt/web/pages.py +++ b/src/ojhunt/web/pages.py @@ -83,8 +83,24 @@ jinja_env.globals["static_version"] = STATIC_VERSION +def render_page( + template_name: str, request: Request, *, canonical: bool = True, **context: object +) -> str: + """Render a page template with the metadata `base.html.jinja` needs. + + Pass `canonical=False` for a page that must stay out of the search index + (404, easter eggs) — a noindex page must not claim a canonical URL. + """ + base = str(request.base_url).rstrip("/") + return jinja_env.get_template(template_name).render( + site_base=base, + canonical_url=f"{base}{request.url.path}" if canonical else None, + **context, + ) + + @router.get("/", response_class=HTMLResponse) -async def index() -> str: +async def index(request: Request) -> str: crawler_data = { name: { "title": info.meta.title, @@ -93,13 +109,11 @@ async def index() -> str: } for name, info in sorted(crawler_registry.items()) } - template = jinja_env.get_template("index.html.jinja") - return template.render(crawlers=crawler_data) + return render_page("index.html.jinja", request, crawlers=crawler_data) @router.get("/about", response_class=HTMLResponse) -async def about() -> str: - template = jinja_env.get_template("about.html.jinja") +async def about(request: Request) -> str: build_time_str = None if BUILD_TIME: try: @@ -111,7 +125,9 @@ async def about() -> str: ) except ValueError: build_time_str = BUILD_TIME - return template.render( + return render_page( + "about.html.jinja", + request, build_time=build_time_str, git_commit_sha=GIT_COMMIT_SHA, ) @@ -155,22 +171,24 @@ async def statistics_redirect(): @router.get("/pdf/legacy", response_class=HTMLResponse) -async def pdf_legacy_get() -> str: - template = jinja_env.get_template("pdf_legacy.html.jinja") - return template.render( +async def pdf_legacy_get(request: Request) -> str: + return render_page( + "pdf_legacy.html.jinja", + request, active_page="legacy", legacy_available=Path("legacy.db").exists(), ) @router.post("/pdf/legacy") -async def pdf_legacy_post(username: str = Form(...)): - template = jinja_env.get_template("pdf_legacy.html.jinja") +async def pdf_legacy_post(request: Request, username: str = Form(...)): try: pdf_bytes = export_user_pdf(username.strip()) except FileNotFoundError: return HTMLResponse( - template.render( + render_page( + "pdf_legacy.html.jinja", + request, active_page="legacy", legacy_available=False, prefill_username=username, @@ -178,7 +196,9 @@ async def pdf_legacy_post(username: str = Form(...)): ) except ValueError as e: return HTMLResponse( - template.render( + render_page( + "pdf_legacy.html.jinja", + request, active_page="legacy", legacy_available=True, error=str(e), @@ -196,17 +216,16 @@ async def pdf_legacy_post(username: str = Form(...)): @router.get("/pdf/merge", response_class=HTMLResponse) -async def pdf_merge_get() -> str: - template = jinja_env.get_template("pdf_merge.html.jinja") - return template.render(active_page="merge") +async def pdf_merge_get(request: Request) -> str: + return render_page("pdf_merge.html.jinja", request, active_page="merge") @router.post("/pdf/merge") async def pdf_merge_post( + request: Request, pdf_a: UploadFile = File(...), pdf_b: UploadFile = File(...), ): - template = jinja_env.get_template("pdf_merge.html.jinja") try: bytes_a = await pdf_a.read() bytes_b = await pdf_b.read() @@ -223,7 +242,11 @@ async def pdf_merge_post( ) pdf_bytes = generate_pdf(data_a.settings, history, snapshot) except ValueError as e: - return HTMLResponse(template.render(active_page="merge", error=str(e))) + return HTMLResponse( + render_page( + "pdf_merge.html.jinja", request, active_page="merge", error=str(e) + ) + ) return Response( content=pdf_bytes, media_type="application/pdf", @@ -232,7 +255,7 @@ async def pdf_merge_post( @router.get("/crawlers", response_class=HTMLResponse) -async def crawlers_page(test_availability: str | None = None) -> str: +async def crawlers_page(request: Request, test_availability: str | None = None) -> str: import json if test_availability is not None: @@ -260,16 +283,16 @@ async def crawlers_page(test_availability: str | None = None) -> str: ), } ) - template = jinja_env.get_template("crawlers.html.jinja") - return template.render(crawlers=crawler_list) + return render_page("crawlers.html.jinja", request, crawlers=crawler_list) async def _easter_egg_handler(request: Request) -> HTMLResponse: easter_egg_path = random.choice( ["easter_egg.html.jinja", "easter_egg_rick.html.jinja"] ) - template = jinja_env.get_template(easter_egg_path) - return HTMLResponse(template.render(path=request.url.path)) + return HTMLResponse( + render_page(easter_egg_path, request, canonical=False, path=request.url.path) + ) for _path in _EASTER_EGG_PATHS: diff --git a/src/ojhunt/web/templates/404.html.jinja b/src/ojhunt/web/templates/404.html.jinja index d26e930..c3a5d69 100644 --- a/src/ojhunt/web/templates/404.html.jinja +++ b/src/ojhunt/web/templates/404.html.jinja @@ -2,6 +2,8 @@ {% block title %}???????? — OJHunt Lite{% endblock %} +{% block robots %}<meta name="robots" content="noindex">{% endblock %} + {% block extra_head %} <style> .error-page { diff --git a/src/ojhunt/web/templates/base.html.jinja b/src/ojhunt/web/templates/base.html.jinja index 143a63e..1232079 100644 --- a/src/ojhunt/web/templates/base.html.jinja +++ b/src/ojhunt/web/templates/base.html.jinja @@ -4,6 +4,16 @@ <meta charset="UTF-8"> <meta name="viewport" content="width=device-width, initial-scale=1.0"> <title>{% block title %}OJHunt Lite{% endblock %} + + {% block robots %}{% endblock %} + {% if canonical_url %}{% endif %} + + + + + {% if canonical_url %}{% endif %} + {% if site_base %}{% endif %} + diff --git a/src/ojhunt/web/templates/crawlers.html.jinja b/src/ojhunt/web/templates/crawlers.html.jinja index 4310223..f98edc2 100644 --- a/src/ojhunt/web/templates/crawlers.html.jinja +++ b/src/ojhunt/web/templates/crawlers.html.jinja @@ -60,6 +60,8 @@ {% block title %}Supported Crawlers - OJHunt Lite{% endblock %} +{% block meta_description %}All {{ crawlers | length }} online judges that OJHunt Lite supports, with live availability status — Codeforces, AtCoder, LeetCode, HDU, POJ, SPOJ, UVa and more.{% endblock %} + {% block page_id %}page-crawlers{% endblock %} {% block content %} diff --git a/src/ojhunt/web/templates/easter_egg.html.jinja b/src/ojhunt/web/templates/easter_egg.html.jinja index c3bb81c..c919120 100644 --- a/src/ojhunt/web/templates/easter_egg.html.jinja +++ b/src/ojhunt/web/templates/easter_egg.html.jinja @@ -2,6 +2,8 @@ {% block title %}你访问的不是 {{ path }} — OJHunt Lite{% endblock %} +{% block robots %}{% endblock %} + {% block extra_head %}