Update hyperlinks in documentation

doc/sphinx/Makefile

@@ -16,11 +16,6 @@ help:
 # Catch-all target: route all unknown targets to Sphinx using the new
 # "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
 %: Makefile MakeTheoryCSV
-	@if test $@ != "clean"; then \
-		sphinx-apidoc -f -o ./$(SOURCEDIR)/modules/validphys ../../validphys2/src/validphys/ ; \
-		sphinx-apidoc -f -o ./$(SOURCEDIR)/modules/n3fit-code ../../n3fit/src/n3fit/ ; \
-		sphinx-apidoc -f -o ./$(SOURCEDIR)/modules/n3fit-code ../../n3fit/src/evolven3fit/ ; \
-	fi
 	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
 
 .PHONY: rsync

doc/sphinx/source/api/index.rst

@@ -0,0 +1,12 @@
+Python API reference
+====================
+
+The API reference is generated from the Python source code and its docstrings.
+
+.. toctree::
+   :maxdepth: 1
+
+   validphys <../modules/validphys/validphys>
+   n3fit <../modules/n3fit/n3fit>
+   evolven3fit <../modules/evolven3fit/evolven3fit>
+   nnpdf_data <../modules/nnpdf_data/nnpdf_data>

doc/sphinx/source/conf.py

@@ -11,12 +11,24 @@
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
 #
-# import sys
-#
-#
+import sys
+from datetime import datetime
+from pathlib import Path
+
+REPO_ROOT = Path(__file__).resolve().parents[3]
+DOC_SOURCE_DIR = Path(__file__).resolve().parent
+PACKAGE_SOURCE_DIRS = [
+    REPO_ROOT / "validphys2" / "src",
+    REPO_ROOT / "n3fit" / "src",
+    REPO_ROOT / "nnpdf_data",
+]
+
+for source_dir in reversed(PACKAGE_SOURCE_DIRS):
+    sys.path.insert(0, str(source_dir))
+
+import validphys
 
 # -- Project information -----------------------------------------------------
-from datetime import datetime
 
 project = "NNPDF"
 copyright = f"{datetime.now().year}, NNPDF collaboration"
@@ -45,7 +57,7 @@
     "sphinx.ext.coverage",
     "sphinx.ext.mathjax",
     "sphinx.ext.ifconfig",
-    "sphinx.ext.viewcode",
+    "sphinx.ext.linkcode",
     "sphinx.ext.napoleon",
     "sphinxcontrib.bibtex",
     "sphinx.ext.autosectionlabel",
@@ -75,12 +87,13 @@
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
 # This pattern also affects html_static_path and html_extra_path.
-exclude_patterns = ['modules/**']
+# Ignore the output directory used by the former Makefile-based generator.
+# Current API pages are generated in one directory per package below.
+exclude_patterns = ["modules/n3fit-code/**", "modules/**/*.tests*"]
 
 # The name of the Pygments (syntax highlighting) style to use.
 pygments_style = None
 
-
 # -- Options for HTML output -------------------------------------------------
 
 # The theme to use for HTML and HTML Help pages.  See the documentation for
@@ -201,6 +214,112 @@
 # Example configuration for intersphinx: refer to the Python standard library.
 intersphinx_mapping = {"python": ("https://docs.python.org/", None)}
 
+
+# -- Source code links (analogous to numpy implementation) -------------------
+
+import inspect
+from os.path import relpath
+
+
+def linkcode_resolve(domain, info):
+    """
+    Determine the URL corresponding to Python object.
+    """
+    if domain != "py":
+        return None
+    modname = info["module"]
+    fullname = info["fullname"]
+    submod = sys.modules.get(modname)
+    if submod is None:
+        return None
+    obj = submod
+    for part in fullname.split("."):
+        try:
+            obj = getattr(obj, part)
+        except Exception:
+            return None
+
+    # strip decorators, which would resolve to the source of the decorator
+    # possibly an upstream bug in getsourcefile, bpo-1764286
+    try:
+        unwrap = inspect.unwrap
+    except AttributeError:
+        pass
+    else:
+        obj = unwrap(obj)
+
+    fn = None
+    lineno = None
+    if fn is None:
+        try:
+            fn = inspect.getsourcefile(obj)
+        except Exception:
+            fn = None
+    if not fn:
+        return None
+
+    # Ignore re-exports as their source files are external modules
+    module = inspect.getmodule(obj)
+    if module is not None and not module.__name__.startswith(
+        ("validphys", "n3fit", "evolven3fit", "nnpdf_data")
+    ):
+        return None
+
+    try:
+        source, lineno = inspect.getsourcelines(obj)
+    except Exception:
+        lineno = None
+
+    fn = relpath(fn, start=REPO_ROOT)
+
+    if lineno:
+        linespec = f"#L{lineno}-L{lineno + len(source) - 1}"
+    else:
+        linespec = ""
+
+    # Now define the nnpdf version. If it's a dev version, it goes to the master branch,
+    # otherwise to the published tag of the version.
+    if "dev" in validphys.__version__:
+        return f"https://github.com/NNPDF/nnpdf/blob/master/{fn}{linespec}"
+    return f"https://github.com/NNPDF/nnpdf/blob/{validphys.__version__}/{fn}{linespec}"
+
+
+# -- Automatic Python API reference generation ------------------------------
+
+API_PACKAGES = {
+    "validphys": REPO_ROOT / "validphys2" / "src" / "validphys",
+    "n3fit": REPO_ROOT / "n3fit" / "src" / "n3fit",
+    "evolven3fit": REPO_ROOT / "n3fit" / "src" / "evolven3fit",
+    "nnpdf_data": REPO_ROOT / "nnpdf_data" / "nnpdf_data",
+}
+
+
+def run_apidoc(_):
+    from sphinx.ext.apidoc import main
+
+    for package_name, package_path in API_PACKAGES.items():
+        output_path = DOC_SOURCE_DIR / "modules" / package_name
+        apidoc_arguments = [
+            "--force",
+            "--module-first",
+            "-o",
+            str(output_path),
+            str(package_path),
+        ]
+        tests_path = package_path / "tests"
+        if tests_path.is_dir():
+            apidoc_arguments.append(str(tests_path))
+        main(apidoc_arguments)
+        # Each package root is linked explicitly from api/index.rst, so the
+        # extra top-level file generated by apidoc is not needed.
+        (output_path / "modules.rst").unlink(missing_ok=True)
+
+
+def setup(app):
+    """Configure sphinx."""
+    app.connect("builder-inited", run_apidoc)
+
+
 # -- Options for todo extension ----------------------------------------------
 
 # If true, `todo` and `todoList` produce output, else they produce nothing.

doc/sphinx/source/external-code/pdf-codes.rst

@@ -16,14 +16,8 @@ release.
 PDF evolution
 -------------
 
-`APFEL <https://apfel.hepforge.org/>`_ ('A PDF Evolution Library') is the PDF evolution code currently
-used by the NNPDF Collaboration. In addition to its PDF evolution capabilities, it also produces
-predictions of deep-inelastic scattering structure functions. In recent years it has been developed
-alongside NNPDF, and so it therefore contains the features and settings required in an NNPDF fit.
-That is, it includes quark masses in the MSbar scheme, the various FONLL heavy quark schemes, scale
-variations up to NLO, etc. Note that at the time of writing, a more streamlined code is being
-written to replace APFEL, which is currently dubbed EKO ('Evolution Kernel Operator'). To find more
-general information about PDF evolution and the DGLAP equations, you can go to the :ref:`Theory section <theory>`.
+The evolution of PDFs is fully handled in the `EKO <https://github.com/nnpdf/eko>`_ code,
+and the way QCD evolution is calculated is described in great detail `here <https://eko.readthedocs.io/>`_.
 
 PDF compression
 ---------------
@@ -42,7 +36,11 @@ the subset that most truthfully reproduces the underlying probability distributi
 Other codes
 ~~~~~~~~~~~
 
-`Hoppet <https://hoppet.hepforge.org/>`_ ('Higher Order Perturbative Parton Evolution Toolkit') is an
-alternative PDF evolution code which is capable of evolving unpolarised PDFs to NNLO and linearly
-polarised PDFs to NLO. The unpolarised evolution includes heavy-quark thresholds in the MSbar
-scheme.
+`Hoppet <https://github.com/hoppet-code/hoppet>`_ ('Higher Order Perturbative Parton Evolution Toolkit') is an
+alternative PDF evolution code which is capable of evolving unpolarised PDFs.
+
+Pineline
+~~~~~~~~
+
+The full theory production pipeline used in the NNPDF framework is extensively described in the 
+`Pineline <https://nnpdf.github.io/pineline>`_ documentation.

doc/sphinx/source/index.rst

@@ -157,6 +157,7 @@ Contents
    data/index
    theory/index
    figuresofmerit/index
+   api/index
    contributing/index
    releases
    ci/index