From b6d8dbebd60890fac81fefbe9311a885728f3107 Mon Sep 17 00:00:00 2001 From: David Pomerenke <46022183+davidpomerenke@users.noreply.github.com> Date: Wed, 17 Jun 2026 02:16:45 -0400 Subject: [PATCH] docs: update README and docs; add basic tests --- README.md | 232 ++++++++++++++----------- docs/TODO.md | 16 +- python/tests/__init__.py | 0 python/tests/test_metadata_cleaning.py | 70 ++++++++ src/__tests__/auth.test.ts | 72 ++++++++ src/__tests__/get-base-url.test.ts | 81 +++++++++ vitest.config.ts | 14 ++ 7 files changed, 385 insertions(+), 100 deletions(-) create mode 100644 python/tests/__init__.py create mode 100644 python/tests/test_metadata_cleaning.py create mode 100644 src/__tests__/auth.test.ts create mode 100644 src/__tests__/get-base-url.test.ts create mode 100644 vitest.config.ts diff --git a/README.md b/README.md index b275bc6..e75589d 100644 --- a/README.md +++ b/README.md @@ -1,27 +1,25 @@ -# UN Website Boilerplate (with Auth) +# UN80 SG Reports Survey -A Next.js template with UN branding and magic link authentication. - -Based on: https://github.com/kleinlennart/un-website-boilerplate +A web application for the UN-EOSG Analytics team to manage the UN80 review of Secretary-General mandatory reports. UN system staff review reports assigned to their entity, submit survey responses, and confirm entity assignments. Admins view aggregated analysis and export data. ## Features -- UN branding (logo, colors, Roboto font) -- Magic link authentication (configurable email domains via DB) -- Rate limiting on magic link requests (2 min cooldown) -- 30-day session duration -- PostgreSQL session/user storage -- Configurable database schema per app -- Entity selection on first login (with "Other" option) -- Entity change dialog (click entity badge in header) -- Public landing page (`/about`) + protected dashboard (`/`) +- **Magic-link authentication** — passwordless login; allowed email domains configured in database +- **Admin whitelist** — admin privileges granted via `admin_emails` DB table (separate from user accounts) +- **Entity dashboard** — each user sees reports suggested for their entity; they confirm the list and submit survey responses +- **Survey responses** — per-user, per-report responses: continue / merge / discontinue, with frequency and format preferences +- **Entity confirmation** — users confirm or adjust AI/DGACM/DRI-suggested entity assignments +- **Analysis page** (`/analysis`, admin only) — coverage metrics, per-entity progress, frequency-direction breakdown +- **Excel export** — survey responses and entity confirmations (`/api/export/survey`); entity progress table (`/api/export/entities`, admin only) +- **AI chat** — OpenAI-powered RAG chat over the reports database +- **Python data pipeline** — scrapes UN Digital Library, generates embeddings, calculates historical frequencies -## Setup +## Quick Start ### 1. Install dependencies ```bash -npm install +pnpm install ``` ### 2. Configure environment @@ -30,124 +28,162 @@ npm install cp .env.template .env.local ``` -Edit `.env.local`: -- `DATABASE_URL` - PostgreSQL connection string -- `DB_SCHEMA` - Schema for auth tables (e.g. `sg_reports_survey` → `sg_reports_survey.users`, `sg_reports_survey.magic_tokens`) -- `AUTH_SECRET` - Generate with `openssl rand -hex 32` -- `SMTP_*` - Mail server for magic links -- `BASE_URL` - Your app URL (for magic link emails) +Required variables in `.env.local`: + +| Variable | Description | +|---|---| +| `DATABASE_URL` | PostgreSQL connection string | +| `DB_SCHEMA` | Schema name (e.g. `sg_reports_survey`) | +| `AUTH_SECRET` | HMAC secret — generate with `openssl rand -hex 32` | +| `SMTP_HOST` / `SMTP_PORT` / `SMTP_USER` / `SMTP_PASS` | Mail server for magic links | +| `SMTP_FROM` | Sender address (falls back to `SMTP_USER`) | +| `BASE_URL` | Public app URL (for magic link emails, optional if behind a proxy) | +| `OPENAI_API_KEY` | OpenAI API key (for AI chat) | -### 3. Create database tables +### 3. Set up the database -Edit `sql/auth_tables.sql` and replace `sg_reports_survey` with your schema name (must match `DB_SCHEMA`), then: +Apply the SQL files **in this order**: ```bash +# Auth tables (users, magic_tokens, allowed_domains) psql $DATABASE_URL -f sql/auth_tables.sql -``` -The schema includes an `allowed_domains` table pre-populated with UN system domains (un.org, undp.org, unicef.org, who.int, etc.). Edit the SQL to remove domains you don't need, or add custom ones: +# Application tables (documents, survey_responses, etc.) +psql $DATABASE_URL -f sql/reports_tables.sql +psql $DATABASE_URL -f sql/survey_responses_table.sql +psql $DATABASE_URL -f sql/frequency_confirmations_table.sql +psql $DATABASE_URL -f sql/report_frequencies_table.sql + +# Migrations (apply in order) +psql $DATABASE_URL -f sql/migrations/002_add_normalized_body.sql +psql $DATABASE_URL -f sql/migrations/003_add_entity_role.sql +psql $DATABASE_URL -f sql/migrations/004_manual_reports_migration.sql +psql $DATABASE_URL -f sql/migrations/005_multi_user_responses_and_admin_role.sql +psql $DATABASE_URL -f sql/migrations/006_drop_response_email_audit_columns.sql +psql $DATABASE_URL -f sql/migrations/007_admin_emails_whitelist.sql +psql $DATABASE_URL -f sql/migrations/008_fix_frequency_check_constraint.sql + +# Views +psql $DATABASE_URL -f sql/views.sql +``` +To grant an admin: ```sql --- Add a custom domain -INSERT INTO myapp.allowed_domains (entity, domain) VALUES - ('*', 'example.org'), -- global: allow for all entities - ('PARTNER', 'partner.org'); -- entity-specific +INSERT INTO sg_reports_survey.admin_emails (email) VALUES ('name@un.org'); ``` ### 4. Run ```bash -npm run dev +pnpm dev ``` -Open [http://localhost:3000](http://localhost:3000) with your browser. +Open [http://localhost:3000](http://localhost:3000). -## Auth Flow +## Python Data Pipeline -1. User visits `/about` (public landing page) -2. User clicks "Sign In" → `/login` -3. User enters email, magic link sent (rate limited: 2 min cooldown) -4. User clicks link → `/verify?token=...` -5. First login: select entity (combobox with "Other" option); returning users: direct sign-in -6. Session cookie set (30 days) -7. Header shows user email, clickable entity badge (to change), and logout -8. Unauthenticated users accessing protected routes → redirect to `/about` +The `python/` directory contains numbered scripts that populate the database: -## Customization +| Script | What it does | +|---|---| +| `01_get_reports.py` | Scrapes UN Digital Library, downloads document metadata | +| `02_populate_reporting_entities.py` | Matches documents to UN entities (DGACM + DRI sources) | +| `03_generate_embeddings.py` | Generates OpenAI embeddings for semantic search | +| `04_extract_mandate_info.py` | Extracts mandate details from PDF documents | +| `05_ai_entity_suggestions.py` | AI-assisted entity matching | +| `06_calculate_frequencies.py` | Computes historical reporting frequencies | -- **Site title/subtitle**: Edit `SITE_TITLE` and `SITE_SUBTITLE` in `src/components/Header.tsx` -- **Allowed email domains**: Add to `allowed_domains` table in database -- **Entity list**: Query in `fetchEntities()` in `src/app/api/entities/route.ts` -- **Document search**: Query in `src/app/api/documents/search/route.ts` -- **Protected routes**: Edit `PUBLIC_PATHS` in `src/middleware.ts` -- **Auth schema**: Set `DB_SCHEMA` env var and update `sql/auth_tables.sql` +Run scripts in order. Each script loads `.env` via `python-dotenv`. Required env vars: `DATABASE_URL`, `DB_SCHEMA`, `OPENAI_API_KEY`. + +```bash +uv run python python/01_get_reports.py +# ... etc. +``` + +## Commands + +```bash +pnpm dev # Dev server → http://localhost:3000 +pnpm build # Production build +pnpm lint # ESLint +pnpm typecheck # TypeScript check (no emit) +pnpm format # Prettier +``` ## File Structure ``` src/ ├── app/ -│ ├── about/ # Public landing page +│ ├── about/ # Public landing page +│ ├── analysis/ # Admin analysis page (coverage, entity progress) │ ├── api/ -│ │ ├── auth/ # Auth API routes (backup, actions preferred) -│ │ ├── documents/search/ # Document search -│ │ └── entities/ # Entity list + fetchEntities() -│ ├── login/ # Login page + layout -│ ├── verify/ # Token verification + entity selection -│ └── page.tsx # Protected dashboard +│ │ ├── auth/ # Auth API routes (request, verify, logout) +│ │ ├── chat/ # AI chat (OpenAI RAG) +│ │ ├── documents/ # Document search +│ │ ├── entities/ # Entity list +│ │ ├── entity-confirmations/ # Entity assignment confirmations +│ │ ├── entity-suggestions/ # AI entity suggestions +│ │ ├── export/ # Excel export (survey + entities) +│ │ ├── frequency-confirmations/ # One-time report confirmations +│ │ ├── reports/ # Report data +│ │ ├── sg-reports/ # SG-specific report queries +│ │ ├── similar-reports/ # Embedding-based similarity +│ │ ├── stats/ # Survey statistics +│ │ └── survey-responses/ # Survey response CRUD +│ ├── login/ # Login page +│ ├── reports/ # Browse all reports (public) +│ ├── stats/ # Stats page +│ ├── verify/ # Token verification + entity selection +│ └── page.tsx # Protected dashboard ├── components/ -│ ├── DocumentSearch.tsx # Document autocomplete -│ ├── EntityChangeDialog.tsx # Dialog to change entity -│ ├── EntityCombobox.tsx # Entity dropdown with "Other" option -│ ├── EntitySearch.tsx # Entity autocomplete (for search) -│ ├── Footer.tsx # Site footer -│ ├── Header.tsx # Site header with maxWidth, hideAbout props -│ ├── LoginForm.tsx # Login form (uses server actions) -│ ├── UserMenu.tsx # Email + entity badge + logout -│ └── VerifyForm.tsx # Verify form with returning user detection +│ ├── EntityDashboard.tsx # Main entity view +│ ├── SGReportsList.tsx # Report list with survey UI +│ ├── ReportSidebar.tsx # Report detail sidebar +│ ├── SurveyExportButton.tsx +│ ├── EntityTableExport.tsx +│ └── ui/ # shadcn/ui primitives ├── lib/ -│ ├── actions.ts # Server actions for auth -│ ├── auth.ts # Auth logic (isAllowedDomain, sessions, etc.) -│ ├── config.ts # DB_SCHEMA config + table names -│ ├── db.ts # PostgreSQL pool -│ ├── mail.ts # Magic link emails -│ └── utils.ts # Tailwind cn() helper -└── middleware.ts # Route protection -sql/ -└── auth_tables.sql # Database schema (users, tokens, allowed_domains) +│ ├── actions.ts # Server actions (login, verify, logout) +│ ├── auth.ts # Auth logic (sessions, tokens, getCurrentUser) +│ ├── config.ts # DB_SCHEMA config + table names +│ ├── db.ts # PostgreSQL pool +│ ├── get-base-url.ts # Dynamic host detection +│ └── mail.ts # Magic link emails +└── proxy.ts # Next.js middleware (route protection) +python/ # Data pipeline scripts +sql/ # Database schema and migrations + ├── auth_tables.sql + ├── reports_tables.sql + ├── survey_responses_table.sql + ├── views.sql + └── migrations/ # Incremental migrations (apply in order) +docs/ # Analysis docs and notes ``` +## Auth Flow + +1. User visits `/about` (public landing page) +2. User clicks “Sign In” → `/login` +3. User enters email; magic link sent (rate-limited: 2-minute cooldown) +4. User clicks link → `/verify?token=...` +5. First login: select entity; returning users: sign in directly +6. Session cookie set (30 days, HMAC-signed) +7. Unauthenticated users accessing protected routes → redirect to `/about` + +Admin access: add email to `admin_emails` table directly in the database. + ## Maintenance -### Check for issues ```bash -npm audit # Security vulnerabilities -npm outdated # Outdated packages -npm run lint # ESLint errors -npx tsc --noEmit # TypeScript errors +pnpm audit # Security vulnerabilities +pnpm outdated # Outdated packages +pnpm lint # ESLint +pnpm typecheck # TypeScript errors ``` -### Update packages -```bash -npm update # Safe patch/minor updates -npm install next@latest eslint-config-next@latest # Update Next.js -``` +## Adding shadcn/ui components -### Clean install (if issues occur) ```bash -rm -rf node_modules .next && npm install +npx shadcn@latest add ``` - -## Good to know - -- use `npx shadcn@latest add ` when you need to add components. - -- https://nextjs.org/docs/app/api-reference/file-conventions/src-folder -- https://nextjs.org/docs/app/getting-started/project-structure - -- The `/public` directory should remain in the root of your project. -- Config files like `package.json`, `next.config.js` and `tsconfig.json` should remain in the root of your project. -- `.env.*` files should remain in the root of your project. - -- [Next.js Documentation](https://nextjs.org/docs) -- [Learn Next.js](https://nextjs.org/learn) diff --git a/docs/TODO.md b/docs/TODO.md index c15089d..692c787 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -1,2 +1,14 @@ -- Dashboard survey column (src/components/SGReportsList.tsx): still reads only my-responses → survey_responses. One-time confirmed reports still show "Go to survey" badge. -- Multi-entity edge case: if two entities are assigned the same report and both confirm it as 'one-time', only the last confirming entity gets per-entity credit (schema limitation of report_frequency_confirmations). +# Known Issues + +These are tracked open issues. File as GitHub issues when prioritising. + +## Open + +- **One-time confirmed reports show wrong badge** (`src/components/SGReportsList.tsx`): The dashboard survey column still reads only from `survey_responses`. Reports confirmed as `one-time` via `report_frequency_confirmations` still display a “Go to survey” badge rather than a completion badge. Fix: join `report_frequency_confirmations` in the dashboard query. + +- **Multi-entity one-time credit race condition**: If two entities are assigned the same report and both confirm it as `one-time`, only the last confirming entity gets per-entity credit. This is a schema limitation of `report_frequency_confirmations` (no entity column). Fix: add `entity` or `user_id` scope to `report_frequency_confirmations`. + +## Recently Fixed + +- Migration 007 replaced `users.role` column with `admin_emails` whitelist table (see `sql/migrations/007_admin_emails_whitelist.sql`). +- Migration 008 fixed the `frequency` check constraint to include all valid values. diff --git a/python/tests/__init__.py b/python/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/python/tests/test_metadata_cleaning.py b/python/tests/test_metadata_cleaning.py new file mode 100644 index 0000000..02d1045 --- /dev/null +++ b/python/tests/test_metadata_cleaning.py @@ -0,0 +1,70 @@ +""" +Basic unit tests for python/util/metadata_cleaning.py. + +Run with: uv run pytest python/tests/ +""" + +import sys +import os + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) + +from util.metadata_cleaning import ( + clean_symbol, + normalize_text, + extract_year_from_symbol, +) + + +class TestCleanSymbol: + def test_strips_leading_trailing_whitespace(self): + result = clean_symbol(" A/79/1 ") + assert result == "A/79/1" + + def test_returns_none_for_empty_string(self): + result = clean_symbol("") + assert result is None + + def test_returns_none_for_whitespace_only(self): + result = clean_symbol(" ") + assert result is None + + def test_returns_none_for_none_input(self): + result = clean_symbol(None) # type: ignore[arg-type] + assert result is None + + def test_preserves_valid_symbol(self): + result = clean_symbol("S/2024/123") + assert result == "S/2024/123" + + +class TestNormalizeText: + def test_strips_whitespace(self): + result = normalize_text(" hello world ") + assert result == "hello world" + + def test_collapses_internal_whitespace(self): + result = normalize_text("hello world") + assert result == "hello world" + + def test_returns_empty_string_for_none(self): + result = normalize_text(None) # type: ignore[arg-type] + assert result == "" + + def test_handles_empty_string(self): + result = normalize_text("") + assert result == "" + + +class TestExtractYearFromSymbol: + def test_extracts_year_from_sc_symbol_with_year(self): + result = extract_year_from_symbol("S/2024/123") + assert result == 2024 + + def test_returns_none_for_invalid_symbol(self): + result = extract_year_from_symbol("") + assert result is None + + def test_returns_none_for_none_input(self): + result = extract_year_from_symbol(None) # type: ignore[arg-type] + assert result is None diff --git a/src/__tests__/auth.test.ts b/src/__tests__/auth.test.ts new file mode 100644 index 0000000..828724f --- /dev/null +++ b/src/__tests__/auth.test.ts @@ -0,0 +1,72 @@ +/** + * Unit tests for src/lib/auth.ts pure/synchronous functions. + * + * Run with: npx vitest run src/__tests__/auth.test.ts + * + * These tests cover the session signing / verification logic which is + * self-contained and has no DB or network dependencies. + */ + +import { describe, it, expect, beforeAll } from "vitest"; + +// We test the exported functions directly, setting AUTH_SECRET via env before import. +beforeAll(() => { + process.env.AUTH_SECRET = "test-secret-do-not-use-in-production"; +}); + +// Dynamic import so the module reads the env var we set above. +const getModule = () => import("../lib/auth"); + +describe("verifySession", () => { + it("returns null for an empty string", async () => { + const { verifySession } = await getModule(); + expect(verifySession("")).toBeNull(); + }); + + it("returns null for a token with no dot separator", async () => { + const { verifySession } = await getModule(); + expect(verifySession("nodot")).toBeNull(); + }); + + it("returns null for a token with a tampered signature", async () => { + const { verifySession } = await getModule(); + const payload = Buffer.from( + JSON.stringify({ userId: "abc", exp: Date.now() + 1_000_000 }) + ).toString("base64"); + const fakeSig = "0".repeat(64); + expect(verifySession(`${payload}.${fakeSig}`)).toBeNull(); + }); + + it("returns null for an expired token", async () => { + const { verifySession } = await getModule(); + const payload = Buffer.from( + JSON.stringify({ userId: "abc", exp: Date.now() - 1 }) + ).toString("base64"); + const { createHmac } = await import("crypto"); + const payloadStr = Buffer.from(payload, "base64").toString(); + const sig = createHmac("sha256", process.env.AUTH_SECRET!) + .update(payloadStr) + .digest("hex"); + expect(verifySession(`${payload}.${sig}`)).toBeNull(); + }); +}); + +describe("generateToken", () => { + it("generates a 64-character hex string", async () => { + const { generateToken } = await getModule(); + const token = generateToken(); + expect(token).toMatch(/^[0-9a-f]{64}$/); + }); + + it("generates unique tokens on successive calls", async () => { + const { generateToken } = await getModule(); + expect(generateToken()).not.toBe(generateToken()); + }); +}); + +describe("isAllowedDomain (shape contract)", () => { + it("exports isAllowedDomain as a function", async () => { + const { isAllowedDomain } = await getModule(); + expect(typeof isAllowedDomain).toBe("function"); + }); +}); diff --git a/src/__tests__/get-base-url.test.ts b/src/__tests__/get-base-url.test.ts new file mode 100644 index 0000000..2c1384e --- /dev/null +++ b/src/__tests__/get-base-url.test.ts @@ -0,0 +1,81 @@ +/** + * Unit tests for src/lib/get-base-url.ts + * + * Run with: npx vitest run src/__tests__/get-base-url.test.ts + */ + +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; + +vi.mock("next/headers", () => ({ + headers: vi.fn(), +})); + +const mockHeaders = async (map: Record) => { + const { headers } = await import("next/headers"); + (headers as ReturnType).mockResolvedValue({ + get: (key: string) => map[key] ?? null, + }); +}; + +describe("getBaseUrl", () => { + beforeEach(() => { + vi.resetModules(); + delete process.env.BASE_URL; + delete process.env.VERCEL_PROJECT_PRODUCTION_URL; + delete process.env.VERCEL_URL; + delete process.env.PORT; + }); + + afterEach(() => { + vi.clearAllMocks(); + }); + + it("uses http for localhost host header", async () => { + await mockHeaders({ host: "localhost:3000", "x-forwarded-proto": "https" }); + const { getBaseUrl } = await import("../lib/get-base-url"); + const url = await getBaseUrl(); + expect(url).toBe("http://localhost:3000"); + }); + + it("uses https for non-localhost host header", async () => { + await mockHeaders({ + host: "app.example.un.org", + "x-forwarded-proto": "https", + }); + const { getBaseUrl } = await import("../lib/get-base-url"); + const url = await getBaseUrl(); + expect(url).toBe("https://app.example.un.org"); + }); + + it("falls back to BASE_URL env var when headers unavailable", async () => { + const { headers } = await import("next/headers"); + (headers as ReturnType).mockRejectedValue( + new Error("headers() not available") + ); + process.env.BASE_URL = "https://custom.domain.org"; + const { getBaseUrl } = await import("../lib/get-base-url"); + const url = await getBaseUrl(); + expect(url).toBe("https://custom.domain.org"); + }); + + it("falls back to localhost when nothing is configured", async () => { + const { headers } = await import("next/headers"); + (headers as ReturnType).mockRejectedValue( + new Error("headers() not available") + ); + const { getBaseUrl } = await import("../lib/get-base-url"); + const url = await getBaseUrl(); + expect(url).toBe("http://localhost:3000"); + }); + + it("strips trailing slash from BASE_URL", async () => { + const { headers } = await import("next/headers"); + (headers as ReturnType).mockRejectedValue( + new Error("headers() not available") + ); + process.env.BASE_URL = "https://custom.domain.org/"; + const { getBaseUrl } = await import("../lib/get-base-url"); + const url = await getBaseUrl(); + expect(url).toBe("https://custom.domain.org"); + }); +}); diff --git a/vitest.config.ts b/vitest.config.ts new file mode 100644 index 0000000..87e0d82 --- /dev/null +++ b/vitest.config.ts @@ -0,0 +1,14 @@ +import { defineConfig } from "vitest/config"; +import path from "path"; + +export default defineConfig({ + test: { + environment: "node", + globals: true, + }, + resolve: { + alias: { + "@": path.resolve(__dirname, "./src"), + }, + }, +});