From f1458066e27dca89d25a4d87bbd43404818d6022 Mon Sep 17 00:00:00 2001 From: carlos Date: Fri, 19 Jun 2026 08:50:06 -0500 Subject: [PATCH] feat: autogenerate and publish example docs --- .github/workflows/ci.yml | 20 ++- .github/workflows/deploy-docs.yml | 20 ++- .gitignore | 3 + build.gradle | 14 ++ src/catdata/cql/AqlTester.java | 121 +++++++-------- src/catdata/cql/CqlExamplesTest.java | 4 +- src/catdata/cql/exp/AqlExamples.java | 178 +++++++++++++++++++++++ src/catdata/cql/exp/AqlExamplesTest.java | 94 ++++++++++++ 8 files changed, 386 insertions(+), 68 deletions(-) create mode 100644 src/catdata/cql/exp/AqlExamples.java create mode 100644 src/catdata/cql/exp/AqlExamplesTest.java diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 19ea3376..5c8f5674 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -146,7 +146,7 @@ jobs: - run: ./gradlew spotlessCheck docs: - name: Docs (AqlHelp) + name: Docs (AqlHelp + AqlExamples) runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -174,3 +174,21 @@ jobs: name: cql-manual path: help/ retention-days: 14 + + - name: Generate CQL examples site + run: ./gradlew generateExamples + + - name: Verify examples generated properly + run: | + set -euo pipefail + test -d examples || { echo "::error::examples/ was not produced"; exit 1; } + test -s examples/index.html || { echo "::error::missing or empty examples/index.html"; exit 1; } + test -s examples/Employees.html || { echo "::error::missing or empty examples/Employees.html"; exit 1; } + + - name: Upload generated examples + if: always() + uses: actions/upload-artifact@v4 + with: + name: cql-examples + path: examples/ + retention-days: 14 diff --git a/.github/workflows/deploy-docs.yml b/.github/workflows/deploy-docs.yml index 8e810e9f..e9e44fd6 100644 --- a/.github/workflows/deploy-docs.yml +++ b/.github/workflows/deploy-docs.yml @@ -1,7 +1,8 @@ name: Deploy docs to website -# Regenerate the CQL HTML manual and push it to the website repo's help/ directory. -# Only help/ is touched, so the hand-authored pages on the site are never affected. +# Regenerate the CQL HTML manual and the standalone examples site, then push them to the website +# repo's help/ and examples/ directories. Only those two directories are touched, so the +# hand-authored pages on the site are never affected. on: @@ -14,7 +15,7 @@ permissions: jobs: deploy: - name: Generate and publish help/ + name: Generate and publish help/ and examples/ runs-on: ubuntu-latest if: github.repository == 'CategoricalData/CQL' steps: @@ -29,6 +30,10 @@ jobs: run: ./gradlew generateHelp timeout-minutes: 20 + - name: Generate the CQL examples site + run: ./gradlew generateExamples + timeout-minutes: 20 + - name: Check out the website repo (via deploy key) uses: actions/checkout@v4 with: @@ -36,18 +41,19 @@ jobs: ssh-key: ${{ secrets.PAGES_DEPLOY_KEY }} path: site - - name: Replace help/ and push if it changed + - name: Replace help/ and examples/ and push if they changed run: | set -euo pipefail - rm -rf site/help + rm -rf site/help site/examples cp -r help site/help + cp -r examples site/examples cd site git config user.name "cql-docs-bot" git config user.email "actions@users.noreply.github.com" - git add help + git add help examples if git diff --cached --quiet; then echo "No documentation changes — nothing to publish." else - git commit -m "docs: refresh CQL manual from CQL@${GITHUB_SHA::7}" + git commit -m "docs: refresh CQL manual and examples from CQL@${GITHUB_SHA::7}" git push fi diff --git a/.gitignore b/.gitignore index 5fa89f5a..3dd52cab 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,6 @@ lib/com.microsoft.z3.jar .classpath .project help/ +# Anchored to the repo root so it ignores the generated examples/ site WITHOUT touching the +# source examples under resources/open/examples/. +/examples/ diff --git a/build.gradle b/build.gradle index 96bc482e..4773142e 100644 --- a/build.gradle +++ b/build.gradle @@ -133,6 +133,20 @@ tasks.register('generateHelp', JavaExec) { jvmArgs '--add-opens', 'java.base/java.lang=ALL-UNNAMED' } +// Runs AqlExamples.main, which generates the standalone examples site (one self-contained page +// per self-tester example, showing its source and executed output) into the examples/ directory +// (relative to the project root). The forked JVM calls System.exit(0) on success. +tasks.register('generateExamples', JavaExec) { + group = 'documentation' + description = 'Generates the standalone CQL examples site into the examples/ directory.' + dependsOn 'classes' + mainClass = 'catdata.cql.exp.AqlExamples' + classpath = sourceSets.main.runtimeClasspath + // Examples may touch AWT; force headless so it runs on CI with no display. + systemProperty 'java.awt.headless', 'true' + jvmArgs '--add-opens', 'java.base/java.lang=ALL-UNNAMED' +} + // Git pre-commit hook: runs spotlessCheck before each commit tasks.register('installGitHook', Copy) { from 'config/hooks/pre-commit' diff --git a/src/catdata/cql/AqlTester.java b/src/catdata/cql/AqlTester.java index 17b0be1d..754015e5 100644 --- a/src/catdata/cql/AqlTester.java +++ b/src/catdata/cql/AqlTester.java @@ -1,18 +1,5 @@ package catdata.cql; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.io.IOException; -import java.io.OutputStream; -import java.io.PrintStream; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; - -import javax.swing.JOptionPane; -import javax.swing.JTabbedPane; - import catdata.Program; import catdata.Util; import catdata.cql.exp.AqlEnv; @@ -24,55 +11,77 @@ import catdata.ide.Examples; import catdata.ide.Language; import gnu.trove.map.hash.THashMap; +import java.io.IOException; +import java.io.OutputStream; +import java.io.PrintStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Map; +import java.util.Set; +import javax.swing.JOptionPane; +import javax.swing.JTabbedPane; public class AqlTester { - static String message = "This self-test will run all the built-in CQL examples and check for exceptions. This test cannot be interupted. This window will disappear for a while. Continue?"; + static String message = + "This self-test will run all the built-in CQL examples and check for exceptions. This test cannot be interupted. This window will disappear for a while. Continue?"; + + /** + * Built-in CQL examples the self-test does NOT run (they need external resources, are too slow, + * or are not standalone programs). The docs/examples generators share this set so that what is + * published can never drift from what the self-test actually verifies. + */ + public static final Set NON_RUNNABLE = + Set.of("TutorialTSP", "QuickSQL", "Stdlib", "Imports"); public static void doSelfTestGui() { - deleteFilesCreatedDuring(() -> { - int c = JOptionPane.showConfirmDialog(null, message, "Run Self-Test?", JOptionPane.YES_NO_OPTION); - if (c != JOptionPane.YES_OPTION) { - return null; - } - Map result = getTestResult(); - if (result.isEmpty()) { - JOptionPane.showMessageDialog(null, "OK: Tests Passed"); - return null; - } - JTabbedPane t = new JTabbedPane(); - for (String k : result.keySet()) { - t.addTab(k, new CodeTextPanel("Error", result.get(k).getMessage())); - } - JOptionPane.showMessageDialog(null, t); - return null; // ignored - }); + deleteFilesCreatedDuring( + () -> { + int c = + JOptionPane.showConfirmDialog( + null, message, "Run Self-Test?", JOptionPane.YES_NO_OPTION); + if (c != JOptionPane.YES_OPTION) { + return null; + } + Map result = getTestResult(); + if (result.isEmpty()) { + JOptionPane.showMessageDialog(null, "OK: Tests Passed"); + return null; + } + JTabbedPane t = new JTabbedPane(); + for (String k : result.keySet()) { + t.addTab(k, new CodeTextPanel("Error", result.get(k).getMessage())); + } + JOptionPane.showMessageDialog(null, t); + return null; // ignored + }); } - /** - * Return a map of test name to error on that test, empty if all tests are successful. - */ + /** Return a map of test name to error on that test, empty if all tests are successful. */ public static Map doSelfTestSilent() { - return deleteFilesCreatedDuring(() -> { - // silence output - PrintStream out = System.out, err = System.err; - PrintStream nul = new PrintStream(new OutputStream() { - @Override - public void write(int arg0) throws IOException { - } - }); - try { - System.setOut(nul); - System.setErr(nul); - - // run tests - return getTestResult(); - } finally { - // restore output - System.setOut(out); - System.setErr(nul); - } - }); + return deleteFilesCreatedDuring( + () -> { + // silence output + PrintStream out = System.out, err = System.err; + PrintStream nul = + new PrintStream( + new OutputStream() { + @Override + public void write(int arg0) throws IOException {} + }); + try { + System.setOut(nul); + System.setErr(nul); + + // run tests + return getTestResult(); + } finally { + // restore output + System.setOut(out); + System.setErr(nul); + } + }); } public static A deleteFilesCreatedDuring(java.util.function.Supplier closure) { @@ -109,8 +118,7 @@ private static Set walkPwd() throws IOException { private static Map getTestResult() { Map progs = new THashMap<>(); for (Example e : Examples.getExamples(Language.CQL)) { - if (e.getName().equals("TutorialTSP") || e.getName().equals("QuickSQL") - || e.getName().equals("Stdlib") || e.getName().equals("Imports")) { + if (NON_RUNNABLE.contains(e.getName())) { continue; } progs.put(e.getName(), e.getText()); @@ -135,5 +143,4 @@ private static Map getTestResult() { } return result; } - } diff --git a/src/catdata/cql/CqlExamplesTest.java b/src/catdata/cql/CqlExamplesTest.java index 15e97be8..bf6b1b39 100644 --- a/src/catdata/cql/CqlExamplesTest.java +++ b/src/catdata/cql/CqlExamplesTest.java @@ -23,9 +23,7 @@ class CqlExamplesTest { /** Names to skip (require external resources or are too slow). */ - private static final Set SKIP = Set.of( - "TutorialTSP", "QuickSQL", "Stdlib", "Imports" - ); + private static final Set SKIP = AqlTester.NON_RUNNABLE; @Test @Tag("integration") diff --git a/src/catdata/cql/exp/AqlExamples.java b/src/catdata/cql/exp/AqlExamples.java new file mode 100644 index 00000000..9fcb80c9 --- /dev/null +++ b/src/catdata/cql/exp/AqlExamples.java @@ -0,0 +1,178 @@ +package catdata.cql.exp; + +import catdata.Program; +import catdata.Util; +import catdata.cql.AqlTester; +import catdata.cql.Instance; +import catdata.cql.Pragma; +import catdata.ide.Example; +import catdata.ide.Examples; +import catdata.ide.Language; +import java.io.File; +import java.util.Collection; + +/** + * Generates a standalone, self-contained examples/ website from the CQL self-tester. + * + *

For every built-in CQL example that the self-test runs (see {@link AqlTester#NON_RUNNABLE} for + * the ones it skips), this writes one HTML page showing the example's source and its executed + * output (instances rendered as tables, command results). An index.html links them + * together. Because the pages are produced by parsing and running the examples on every build, they + * are always in sync with the language — this is the auto-generated counterpart to {@code + * AqlHelp}'s manual, but free of the help frameset so the top-level website can link straight into + * any page. + * + *

Run via the Gradle generateExamples task, which supplies headless mode and the + * {@code --add-opens} flag (the examples may touch AWT through {@code AqlMultiDriver}). + */ +public final class AqlExamples { + + private AqlExamples() {} + + /** Self-contained page styling; inlined so every page renders standalone with no external CSS. */ + static final String CSS = + ""; + + /** + * The CQL examples to document: every built-in example the self-test runs, i.e. all of them minus + * {@link AqlTester#NON_RUNNABLE}. + */ + public static Collection examplesToDocument() { + java.util.List out = new java.util.ArrayList<>(); + for (Example e : Examples.getExamples(Language.CQL)) { + if (!AqlTester.NON_RUNNABLE.contains(e.getName())) { + out.add(e); + } + } + return out; + } + + /** Builds the {@code index.html} listing every documented example alphabetically. */ + public static String renderIndex(Collection examples) { + StringBuilder sb = new StringBuilder(); + sb.append("CQL Examples") + .append(CSS) + .append(""); + sb.append("

CQL Examples

"); + sb.append( + "

These examples are generated automatically from the CQL self-tester. Each one is" + + " parsed and executed on every build, so the source and output shown are always" + + " current. Source: CategoricalData/CQL.

"); + sb.append("
    "); + for (Example ex : Util.alphabetical(examples)) { + String name = ex.getName().trim(); + sb.append("
  • ").append(name).append("
    • "); + } + sb.append("
"); + return sb.toString(); + } + + /** + * Builds the page for a single example: its source, and (when {@code run}) its executed output. + * Splitting {@code run} out keeps the page renderable without executing the program, which the + * unit tests rely on. + */ + public static String renderExamplePage(Example ex, boolean run) { + String name = ex.getName().trim(); + StringBuilder sb = new StringBuilder(); + sb.append("") + .append(name) + .append(" — CQL Example") + .append(CSS) + .append(""); + sb.append("

← All examples

"); + sb.append("

example ").append(name).append("

"); + sb.append("
\n").append(AqlInACan.strip(ex.getText().trim())).append("\n
"); + if (run) { + sb.append(renderExecutedOutput(ex)); + } + sb.append(""); + return sb.toString(); + } + + /** + * Parses and runs the example, rendering its resulting instances (as HTML tables) and commands. + * Any parse/run failure is rendered inline rather than thrown, so one bad example cannot abort + * generation of the whole site. + */ + @SuppressWarnings("unchecked") + static String renderExecutedOutput(Example ex) { + StringBuilder sb = new StringBuilder(); + try { + Program> prog = AqlParserFactory.getParser().parseProgram(ex.getText()); + AqlMultiDriver dr = new AqlMultiDriver(prog, null); + dr.start(); // blocks until finished + if (dr.env.exn != null) { + return errorSection(dr.env.exn.getMessage()); + } + for (String k : Util.alphabetical(dr.env.defs.insts.keySet())) { + Instance i = + (Instance) + dr.env.defs.insts.get(k); + sb.append("

instance ").append(k).append("

"); + sb.append(AqlInACan.toHtml(dr.env, i)); + } + for (String k : Util.alphabetical(dr.env.defs.ps.keySet())) { + Pragma p = dr.env.defs.ps.get(k); + sb.append("

command ").append(k).append("

");
+        sb.append(AqlInACan.strip(p.toString())).append("
"); + } + } catch (Exception e) { + return errorSection(e.getMessage()); + } + return sb.toString(); + } + + private static String errorSection(String message) { + return "

error

" + AqlInACan.strip(String.valueOf(message)) + "
"; + } + + /** Writes {@code index.html} plus one page per example into {@code dir}. */ + public static void generate(File dir, boolean run) throws java.io.IOException { + if (!dir.exists()) { + dir.mkdir(); + } + Collection examples = examplesToDocument(); + for (Example ex : examples) { + System.out.println(ex.getName()); + Util.writeFile( + renderExamplePage(ex, run), + new File(dir, ex.getName().trim() + ".html").getAbsolutePath()); + } + Util.writeFile(renderIndex(examples), new File(dir, "index.html").getAbsolutePath()); + } + + /** + * Generates the manual into a temp dir (wrapped so examples that write files on disk are cleaned + * up), then atomically swaps it in as the top-level {@code examples/} directory. + */ + public static void main(String[] args) throws java.io.IOException { + File tmp = java.nio.file.Files.createTempDirectory("aqlexamples").toFile(); + AqlTester.deleteFilesCreatedDuring( + () -> { + try { + generate(tmp, true); + } catch (java.io.IOException e) { + throw new RuntimeException(e); + } + return null; // not used + }); + File examples = new File("examples"); + if (examples.exists()) { + for (File f : examples.listFiles()) { + f.delete(); + } + examples.delete(); + } + tmp.renameTo(examples); + System.exit(0); // slay daemons (AWT/netty non-daemon threads otherwise hang the JVM) + } +} diff --git a/src/catdata/cql/exp/AqlExamplesTest.java b/src/catdata/cql/exp/AqlExamplesTest.java new file mode 100644 index 00000000..f9e3d5fb --- /dev/null +++ b/src/catdata/cql/exp/AqlExamplesTest.java @@ -0,0 +1,94 @@ +package catdata.cql.exp; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; + +import catdata.cql.AqlTester; +import catdata.ide.Example; +import catdata.ide.Examples; +import catdata.ide.Language; + +/** Unit tests for the standalone examples-site generator {@link AqlExamples}. */ +class AqlExamplesTest { + + /** A minimal in-memory {@link Example} so rendering can be tested without disk or execution. */ + private static Example example(String name, String text) { + return new Example() { + @Override + public String getName() { + return name; + } + + @Override + public String getText() { + return text; + } + }; + } + + @Test + void renderIndexListsEveryExampleAlphabetically() { + List examples = new ArrayList<>(); + examples.add(example("Bravo", "schema B = literal : ty {}")); + examples.add(example("Alpha", "schema A = literal : ty {}")); + + String html = AqlExamples.renderIndex(examples); + + assertTrue(html.contains("href=\"Alpha.html\""), "index should link to Alpha"); + assertTrue(html.contains("href=\"Bravo.html\""), "index should link to Bravo"); + assertTrue( + html.indexOf("Alpha.html") < html.indexOf("Bravo.html"), + "examples should be listed alphabetically"); + } + + @Test + void renderExamplePageShowsStrippedSourceAndTitle() { + // The '<' must be HTML-escaped by AqlInACan.strip so it renders literally in the 
 block.
+    Example ex = example("Demo", "-- a < b\nschema S = literal : ty {}");
+
+    String html = AqlExamples.renderExamplePage(ex, false);
+
+    assertTrue(html.contains("
example Demo
"), "page should carry the example title");
+    assertTrue(html.contains("a < b"), "source angle brackets should be escaped");
+    assertTrue(html.contains("href=\"index.html\""), "page should link back to the index");
+  }
+
+  @Test
+  void examplesToDocumentExcludesNonRunnableExamples() {
+    Collection docs = AqlExamples.examplesToDocument();
+
+    assertFalse(docs.isEmpty(), "there should be runnable examples to document");
+    for (Example ex : docs) {
+      assertFalse(
+          AqlTester.NON_RUNNABLE.contains(ex.getName()),
+          "skipped example should not be documented: " + ex.getName());
+    }
+  }
+
+  @Test
+  @Tag("integration")
+  void runningARealExampleRendersItsExecutedOutput() {
+    Example employees = findExample("Employees");
+    String html = AqlExamples.renderExamplePage(employees, true);
+    assertTrue(
+        html.contains("
instance "), "executed output should include at least one instance");
+  }
+
+  private static Example findExample(String name) {
+    for (Example e : Examples.getExamples(Language.CQL)) {
+      if (e.getName().equals(name)) {
+        return e;
+      }
+    }
+    fail("expected built-in example not found: " + name);
+    throw new AssertionError("unreachable");
+  }
+}