Support documenting globals

docs/advanced_stardoc_usage.md

@@ -34,6 +34,21 @@ def my_function(foo, bar):
 Markdown formatting constructs are handled appropriately by Stardoc's default
 output format ("markdown_tables"), even as part of a table.
 
+### Documenting global symbols
+
+With Bazel 9 or newer, global values without a built-in documentation mechanism,
+such as strings, lists, dictionaries, and structs, can be documented with `#:`
+comments:
+
+```starlark
+#: The default timeout used by this module.
+DEFAULT_TIMEOUT = 60
+```
+
+The extracted documentation includes the symbol's name, documentation, and
+Starlark type. Bazel does not currently include the symbol's value in the
+Stardoc output proto.
+
 
 <a name="custom-output"></a>
 ## Custom Output

docs/stardoc_rule.md

@@ -11,11 +11,11 @@ load("@stardoc//stardoc:stardoc.bzl", "stardoc")
 
 stardoc(*, <a href="#stardoc-name">name</a>, <a href="#stardoc-input">input</a>, <a href="#stardoc-out">out</a>, <a href="#stardoc-deps">deps</a>, <a href="#stardoc-format">format</a>, <a href="#stardoc-symbol_names">symbol_names</a>, <a href="#stardoc-renderer">renderer</a>, <a href="#stardoc-aspect_template">aspect_template</a>, <a href="#stardoc-func_template">func_template</a>,
         <a href="#stardoc-macro_template">macro_template</a>, <a href="#stardoc-header_template">header_template</a>, <a href="#stardoc-table_of_contents_template">table_of_contents_template</a>, <a href="#stardoc-provider_template">provider_template</a>, <a href="#stardoc-rule_template">rule_template</a>,
-        <a href="#stardoc-repository_rule_template">repository_rule_template</a>, <a href="#stardoc-module_extension_template">module_extension_template</a>, <a href="#stardoc-footer_template">footer_template</a>, <a href="#stardoc-render_main_repo_name">render_main_repo_name</a>,
-        <a href="#stardoc-stamp">stamp</a>, <a href="#stardoc-kwargs">**kwargs</a>)
+        <a href="#stardoc-repository_rule_template">repository_rule_template</a>, <a href="#stardoc-module_extension_template">module_extension_template</a>, <a href="#stardoc-starlark_other_symbol_template">starlark_other_symbol_template</a>,
+        <a href="#stardoc-footer_template">footer_template</a>, <a href="#stardoc-render_main_repo_name">render_main_repo_name</a>, <a href="#stardoc-stamp">stamp</a>, <a href="#stardoc-kwargs">**kwargs</a>)
 </pre>
 
-Generates documentation for exported starlark rule definitions in a target starlark file.
+Generates documentation for exported symbols in a target Starlark file.
 
 **PARAMETERS**
 
@@ -27,7 +27,7 @@ Generates documentation for exported starlark rule definitions in a target starl
 | <a id="stardoc-out"></a>out |  The file to which documentation will be output (mandatory).   |  none |
 | <a id="stardoc-deps"></a>deps |  A list of bzl_library dependencies which the input depends on.   |  `[]` |
 | <a id="stardoc-format"></a>format |  The format of the output file. Valid values: 'markdown' or 'proto'.   |  `"markdown"` |
-| <a id="stardoc-symbol_names"></a>symbol_names |  A list of symbol names to generate documentation for. These should correspond to the names of rule definitions in the input file. If this list is empty, then documentation for all exported rule definitions will be generated.   |  `[]` |
+| <a id="stardoc-symbol_names"></a>symbol_names |  A list of exported symbol names to generate documentation for. If this list is empty, documentation for all exported symbols will be generated.   |  `[]` |
 | <a id="stardoc-renderer"></a>renderer |  The location of the renderer tool.   |  `Label("@stardoc//stardoc:renderer")` |
 | <a id="stardoc-aspect_template"></a>aspect_template |  The input file template for generating documentation of aspects   |  `Label("@stardoc//stardoc:templates/markdown_tables/aspect.vm")` |
 | <a id="stardoc-func_template"></a>func_template |  The input file template for generating documentation of functions, including legacy macros.   |  `Label("@stardoc//stardoc:templates/markdown_tables/func.vm")` |
@@ -38,6 +38,7 @@ Generates documentation for exported starlark rule definitions in a target starl
 | <a id="stardoc-rule_template"></a>rule_template |  The input file template for generating documentation of rules.   |  `Label("@stardoc//stardoc:templates/markdown_tables/rule.vm")` |
 | <a id="stardoc-repository_rule_template"></a>repository_rule_template |  The input file template for generating documentation of repository rules.   |  `Label("@stardoc//stardoc:templates/markdown_tables/repository_rule.vm")` |
 | <a id="stardoc-module_extension_template"></a>module_extension_template |  The input file template for generating documentation of module extensions.   |  `Label("@stardoc//stardoc:templates/markdown_tables/module_extension.vm")` |
+| <a id="stardoc-starlark_other_symbol_template"></a>starlark_other_symbol_template |  The input file template for generating documentation of Starlark global symbols, such as constants, which use `#:` doc comments.   |  `Label("@stardoc//stardoc:templates/markdown_tables/starlark_other_symbol.vm")` |
 | <a id="stardoc-footer_template"></a>footer_template |  The input file template for generating the footer of the output documentation. Optional.   |  `None` |
 | <a id="stardoc-render_main_repo_name"></a>render_main_repo_name |  Render labels in the main repository with a repo component (either the module name or workspace name).   |  `True` |
 | <a id="stardoc-stamp"></a>stamp |  Whether to provide stamping information to templates, where it can be accessed via `$util.formatBuildTimestamp()` and`$stamping`. Example: <pre><code class="language-vm">Built on `$util.formatBuildTimestamp($stamping.volatile.BUILD_TIMESTAMP, "UTC", "yyyy-MM-dd HH:mm")`</code></pre> Possible values: <ul> <li>`stamp = 1`: Always provide stamping information, even in     [--nostamp](https://bazel.build/docs/user-manual#stamp) builds.     This setting should be avoided, since it potentially kills remote caching for the target     and any downstream actions that depend on it.</li> <li>`stamp = 0`: Do not provide stamping information.</li> <li>`stamp = -1`: Provide stamping information only if the      [--stamp](https://bazel.build/docs/user-manual#stamp) flag is set.</li> </ul> Stamped targets are not rebuilt unless their dependencies change.   |  `-1` |

src/main/java/com/google/devtools/build/stardoc/renderer/RendererMain.java

@@ -33,6 +33,7 @@
 import com.google.devtools.build.lib.starlarkdocextract.StardocOutputProtos.RepositoryRuleInfo;
 import com.google.devtools.build.lib.starlarkdocextract.StardocOutputProtos.RuleInfo;
 import com.google.devtools.build.lib.starlarkdocextract.StardocOutputProtos.StarlarkFunctionInfo;
+import com.google.devtools.build.lib.starlarkdocextract.StardocOutputProtos.StarlarkOtherSymbolInfo;
 import com.google.devtools.build.stardoc.rendering.MarkdownRenderer;
 import com.google.devtools.build.stardoc.rendering.MarkdownRenderer.Renderer;
 import com.google.devtools.build.stardoc.rendering.Stamping;
@@ -98,6 +99,7 @@ public void println() {
               rendererOptions.aspectTemplateFilePath,
               rendererOptions.repositoryRuleTemplateFilePath,
               rendererOptions.moduleExtensionTemplateFilePath,
+              rendererOptions.starlarkOtherSymbolTemplateFilePath,
               !moduleInfo.getFile().isEmpty()
                   ? Optional.of(moduleInfo.getFile())
                   : Optional.empty(),
@@ -154,6 +156,12 @@ public void println() {
               .sorted(comparing(ModuleExtensionInfo::getExtensionName))
               .collect(toImmutableList());
 
+      // Other documented Starlark symbols are printed sorted by their qualified name.
+      ImmutableList<StarlarkOtherSymbolInfo> sortedStarlarkOtherSymbolInfos =
+          ImmutableList.sortedCopyOf(
+              comparing(StarlarkOtherSymbolInfo::getName),
+              moduleInfo.getStarlarkOtherSymbolInfoList());
+
       printWriter.println(renderer.renderMarkdownHeader(moduleInfo));
       if (rendererOptions.tableOfContentsTemplateFilePath != null) {
         printWriter.println(
@@ -164,7 +172,8 @@ public void println() {
                 sortedStarlarkFunctions,
                 sortedAspectInfos,
                 sortedRepositoryRuleInfos,
-                sortedModuleExtensionInfos));
+                sortedModuleExtensionInfos,
+                sortedStarlarkOtherSymbolInfos));
       }
       print(printWriter, renderer::render, sortedRuleInfos);
       print(printWriter, renderer::render, sortedProviderInfos);
@@ -173,6 +182,7 @@ public void println() {
       print(printWriter, renderer::render, sortedAspectInfos);
       print(printWriter, renderer::render, sortedRepositoryRuleInfos);
       print(printWriter, renderer::render, sortedModuleExtensionInfos);
+      print(printWriter, renderer::render, sortedStarlarkOtherSymbolInfos);
       if (rendererOptions.footerTemplateFilePath != null) {
         printWriter.println(renderer.renderMarkdownFooter(moduleInfo));
       }

src/main/java/com/google/devtools/build/stardoc/renderer/RendererOptions.java

@@ -86,6 +86,12 @@ class RendererOptions {
       description = "The template for the documentation of a module extension")
   String moduleExtensionTemplateFilePath;
 
+  @Parameter(
+      names = "--starlark_other_symbol_template",
+      required = true,
+      description = "The template for the documentation of a Starlark global symbol")
+  String starlarkOtherSymbolTemplateFilePath;
+
   @Parameter(names = "--footer_template", description = "The template for the footer string")
   String footerTemplateFilePath;
 

src/main/java/com/google/devtools/build/stardoc/rendering/MarkdownRenderer.java

@@ -30,6 +30,7 @@
 import com.google.devtools.build.lib.starlarkdocextract.StardocOutputProtos.RepositoryRuleInfo;
 import com.google.devtools.build.lib.starlarkdocextract.StardocOutputProtos.RuleInfo;
 import com.google.devtools.build.lib.starlarkdocextract.StardocOutputProtos.StarlarkFunctionInfo;
+import com.google.devtools.build.lib.starlarkdocextract.StardocOutputProtos.StarlarkOtherSymbolInfo;
 import com.google.escapevelocity.EvaluationException;
 import com.google.escapevelocity.ParseException;
 import com.google.escapevelocity.Template;
@@ -61,6 +62,7 @@ public interface Renderer<T> {
   private final String aspectTemplateFilename;
   private final String repositoryRuleTemplateFilename;
   private final String moduleExtensionTemplateFilename;
+  private final String starlarkOtherSymbolTemplateFilename;
   private final Optional<String> entrypointBzlFile;
   private final String footerTemplateFilename;
   private final Stamping stamping;
@@ -75,6 +77,7 @@ public MarkdownRenderer(
       String aspectTemplate,
       String repositoryRuleTemplate,
       String moduleExtensionTemplate,
+      String starlarkOtherSymbolTemplate,
       Optional<String> entrypointBzlFile,
       String footerTemplate,
       Stamping stamping) {
@@ -87,6 +90,7 @@ public MarkdownRenderer(
     this.aspectTemplateFilename = aspectTemplate;
     this.repositoryRuleTemplateFilename = repositoryRuleTemplate;
     this.moduleExtensionTemplateFilename = moduleExtensionTemplate;
+    this.starlarkOtherSymbolTemplateFilename = starlarkOtherSymbolTemplate;
     this.entrypointBzlFile = entrypointBzlFile;
     this.footerTemplateFilename = footerTemplate;
     this.stamping = stamping;
@@ -124,7 +128,8 @@ public String renderTableOfContents(
       List<StarlarkFunctionInfo> starlarkFunctions,
       List<AspectInfo> aspectInfos,
       List<RepositoryRuleInfo> repositoryRuleInfos,
-      List<ModuleExtensionInfo> moduleExtensionInfos)
+      List<ModuleExtensionInfo> moduleExtensionInfos,
+      List<StarlarkOtherSymbolInfo> starlarkOtherSymbolInfos)
       throws IOException {
 
     ImmutableMap<String, Object> vars =
@@ -136,7 +141,8 @@ public String renderTableOfContents(
             "functionInfos", starlarkFunctions,
             "aspectInfos", aspectInfos,
             "repositoryRuleInfos", repositoryRuleInfos,
-            "moduleExtensionInfos", moduleExtensionInfos);
+            "moduleExtensionInfos", moduleExtensionInfos,
+            "starlarkOtherSymbolInfos", starlarkOtherSymbolInfos);
     Reader reader = readerFromPath(tableOfContentsTemplateFilename);
     try {
       return Template.parseFrom(reader).evaluate(vars);
@@ -348,6 +354,22 @@ public String render(ModuleExtensionInfo moduleExtensionInfo) throws IOException
     }
   }
 
+  /** Returns a markdown rendering of a documented Starlark global symbol. */
+  public String render(StarlarkOtherSymbolInfo starlarkOtherSymbolInfo) throws IOException {
+    ImmutableMap<String, Object> vars =
+        ImmutableMap.of(
+            "util",
+            new MarkdownUtil(entrypointBzlFile),
+            "symbolInfo",
+            starlarkOtherSymbolInfo);
+    Reader reader = readerFromPath(starlarkOtherSymbolTemplateFilename);
+    try {
+      return Template.parseFrom(reader).evaluate(vars);
+    } catch (ParseException | EvaluationException e) {
+      throw new IOException(e);
+    }
+  }
+
   /** Returns a markdown header string that should appear at the end of Stardoc's output. */
   public String renderMarkdownFooter(ModuleInfo moduleInfo) throws IOException {
     ImmutableMap<String, Object> vars =

stardoc/html_tables_stardoc.bzl

@@ -35,5 +35,6 @@ def html_tables_stardoc(name, **kwargs):
         rule_template = Label("//stardoc:templates/html_tables/rule.vm"),
         repository_rule_template = Label("//stardoc:templates/html_tables/repository_rule.vm"),
         module_extension_template = Label("//stardoc:templates/html_tables/module_extension.vm"),
+        starlark_other_symbol_template = Label("//stardoc:templates/html_tables/starlark_other_symbol.vm"),
         **kwargs
     )

stardoc/private/stardoc.bzl

@@ -42,6 +42,7 @@ def _renderer_action_run(ctx, out_file, proto_file):
     renderer_args.add("--rule_template=" + str(ctx.file.rule_template.path))
     renderer_args.add("--repository_rule_template=" + str(ctx.file.repository_rule_template.path))
     renderer_args.add("--module_extension_template=" + str(ctx.file.module_extension_template.path))
+    renderer_args.add("--starlark_other_symbol_template=" + str(ctx.file.starlark_other_symbol_template.path))
     if ctx.file.footer_template:
         renderer_args.add("--footer_template=" + str(ctx.file.footer_template.path))
     if stamp_enabled:
@@ -58,6 +59,7 @@ def _renderer_action_run(ctx, out_file, proto_file):
         ctx.file.rule_template,
         ctx.file.repository_rule_template,
         ctx.file.module_extension_template,
+        ctx.file.starlark_other_symbol_template,
     ]
     if ctx.attr.table_of_contents_template:
         inputs.append(ctx.file.table_of_contents_template)
@@ -138,6 +140,11 @@ _common_renderer_attrs = {
         allow_single_file = [".vm"],
         mandatory = True,
     ),
+    "starlark_other_symbol_template": attr.label(
+        doc = "The input file template for generating documentation of Starlark global symbols.",
+        allow_single_file = [".vm"],
+        mandatory = True,
+    ),
     "footer_template": attr.label(
         doc = "The input file template for generating the footer of the output documentation. Optional.",
         allow_single_file = [".vm"],
@@ -190,7 +197,7 @@ _stardoc_markdown_renderer_attrs = {
 stardoc_markdown_renderer = rule(
     _stardoc_markdown_renderer_impl,
     doc = """
-Generates markdown documentation for starlark rule definitions from the corresponding binary proto.
+Generates Markdown documentation for Starlark symbols from the corresponding binary proto.
 """,
     attrs = _stardoc_markdown_renderer_attrs,
 )

stardoc/stardoc.bzl

@@ -35,21 +35,21 @@ def stardoc(
         rule_template = Label("//stardoc:templates/markdown_tables/rule.vm"),
         repository_rule_template = Label("//stardoc:templates/markdown_tables/repository_rule.vm"),
         module_extension_template = Label("//stardoc:templates/markdown_tables/module_extension.vm"),
+        starlark_other_symbol_template = Label("//stardoc:templates/markdown_tables/starlark_other_symbol.vm"),
         footer_template = None,
         render_main_repo_name = True,
         stamp = -1,
         **kwargs):
-    """Generates documentation for exported starlark rule definitions in a target starlark file.
+    """Generates documentation for exported symbols in a target Starlark file.
 
     Args:
       name: The name of the stardoc target.
       input: The starlark file to generate documentation for (mandatory).
       out: The file to which documentation will be output (mandatory).
       deps: A list of bzl_library dependencies which the input depends on.
       format: The format of the output file. Valid values: 'markdown' or 'proto'.
-      symbol_names: A list of symbol names to generate documentation for. These should correspond to the names of rule
-        definitions in the input file. If this list is empty, then documentation for all exported rule definitions will
-        be generated.
+      symbol_names: A list of exported symbol names to generate documentation for. If this list is
+        empty, documentation for all exported symbols will be generated.
       renderer: The location of the renderer tool.
       aspect_template: The input file template for generating documentation of aspects
       header_template: The input file template for the header of the output documentation.
@@ -62,6 +62,8 @@ def stardoc(
       rule_template: The input file template for generating documentation of rules.
       repository_rule_template: The input file template for generating documentation of repository rules.
       module_extension_template: The input file template for generating documentation of module extensions.
+      starlark_other_symbol_template: The input file template for generating documentation of
+        Starlark global symbols, such as constants, which use `#:` doc comments.
       footer_template: The input file template for generating the footer of the output documentation. Optional.
       render_main_repo_name: Render labels in the main repository with a repo component (either
         the module name or workspace name).
@@ -132,6 +134,7 @@ def stardoc(
             rule_template = rule_template,
             repository_rule_template = repository_rule_template,
             module_extension_template = module_extension_template,
+            starlark_other_symbol_template = starlark_other_symbol_template,
             macro_template = macro_template,
             footer_template = footer_template,
             stamp = stamp,

stardoc/templates/html_tables/starlark_other_symbol.vm

@@ -0,0 +1,9 @@
+<a id="${symbolInfo.name}"></a>
+
+#[[##]]# ${symbolInfo.name} (`${symbolInfo.typeName}`)
+
+<pre>
+${util.loadStatement($symbolInfo.name)}
+</pre>
+
+${util.htmlEscape($symbolInfo.doc)}

stardoc/templates/markdown_tables/starlark_other_symbol.vm

@@ -0,0 +1,9 @@
+<a id="${symbolInfo.name}"></a>
+
+#[[##]]# ${symbolInfo.name} (`${symbolInfo.typeName}`)
+
+<pre>
+${util.loadStatement($symbolInfo.name)}
+</pre>
+
+${symbolInfo.doc}

stardoc/templates/markdown_tables/table_of_contents.vm

@@ -55,3 +55,11 @@
 #end
 
 #end
+#if (!$starlarkOtherSymbolInfos.isEmpty())
+#[[##]]# Globals
+
+#foreach ($symbolInfo in $starlarkOtherSymbolInfos)
+- [$symbolInfo.name](#$symbolInfo.name)
+#end
+
+#end

test/BUILD

@@ -54,6 +54,27 @@ stardoc_test(
     symbol_names = ["my_rule"],
 )
 
+stardoc_test(
+    name = "starlark_other_symbols_test",
+    golden_file = "testdata/starlark_other_symbols_test/golden.md",
+    input_file = "testdata/starlark_other_symbols_test/input.bzl",
+    tags = [
+        "bazel_9",
+        "manual",
+    ],
+)
+
+stardoc_test(
+    name = "starlark_other_symbols_html_tables_test",
+    golden_file = "testdata/starlark_other_symbols_test/html_tables_golden.md",
+    input_file = "testdata/starlark_other_symbols_test/input.bzl",
+    tags = [
+        "bazel_9",
+        "manual",
+    ],
+    test = "html_tables",
+)
+
 stardoc_test(
     name = "repo_rules_test",
     golden_file = "testdata/repo_rules_test/golden.md",

test/testdata/starlark_other_symbols_test/golden.md

@@ -0,0 +1,38 @@
+<!-- Generated with Stardoc: http://skydoc.bazel.build -->
+
+A golden test for documented Starlark global symbols.
+
+<a id="A_STRING"></a>
+
+## A_STRING (`string`)
+
+<pre>
+load("@stardoc//test:testdata/starlark_other_symbols_test/input.bzl", "A_STRING")
+</pre>
+
+A documented string constant.
+
+
+<a id="STRUCT_CONSTANT"></a>
+
+## STRUCT_CONSTANT (`struct`)
+
+<pre>
+load("@stardoc//test:testdata/starlark_other_symbols_test/input.bzl", "STRUCT_CONSTANT")
+</pre>
+
+A documented struct constant.
+
+
+<a id="Z_LIST"></a>
+
+## Z_LIST (`list`)
+
+<pre>
+load("@stardoc//test:testdata/starlark_other_symbols_test/input.bzl", "Z_LIST")
+</pre>
+
+The first line of documentation for a list.
+The second line of documentation for a list.
+
+