docs: Adds CI tests for the doc building process.

Adds test_sphinx_idf_extensions.py and test_docs.py for testing the documentation building process.
Both are run as a part of the host_test stage.

Closes IDF-1648

docs/build_docs.py

@@ -78,6 +78,7 @@ def main():
     parser.add_argument("--language", "-l", choices=LANGUAGES, required=False)
     parser.add_argument("--target", "-t", choices=TARGETS, required=False)
     parser.add_argument("--build-dir", "-b", type=str, default="_build")
+    parser.add_argument("--source-dir", "-s", type=str, default="")
     parser.add_argument("--builders", "-bs", nargs='+', type=str, default=["html"],
                         help="List of builders for Sphinx, e.g. html or latex, for latex a PDF is also generated")
     parser.add_argument("--sphinx-parallel-builds", "-p", choices=["auto"] + [str(x) for x in range(8)],
@@ -155,7 +156,9 @@ def parallel_call(args, callback):
     for target in targets:
         for language in languages:
             build_dir = os.path.realpath(os.path.join(args.build_dir, language, target))
-            entries.append((language, target, build_dir, args.sphinx_parallel_jobs, args.builders, args.input_docs))
+            source_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), args.source_dir, language)
+
+            entries.append((language, target, build_dir, source_dir, args.sphinx_parallel_jobs, args.builders, args.input_docs))
 
     print(entries)
     errcodes = pool.map(callback, entries)
@@ -177,7 +180,7 @@ def parallel_call(args, callback):
         return 0
 
 
-def sphinx_call(language, target, build_dir, sphinx_parallel_jobs, buildername, input_docs):
+def sphinx_call(language, target, build_dir, src_dir, sphinx_parallel_jobs, buildername, input_docs):
     # Note: because this runs in a multiprocessing Process, everything which happens here should be isolated to a single process
     # (ie it doesn't matter if Sphinx is using global variables, as they're it's own copy of the global variables)
 
@@ -204,7 +207,7 @@ def sphinx_call(language, target, build_dir, sphinx_parallel_jobs, buildername,
             "-t", target,
             "-D", "idf_target={}".format(target),
             "-D", "docs_to_build={}".format(",". join(input_docs)),
-            os.path.join(os.path.abspath(os.path.dirname(__file__)), language),  # srcdir for this language
+            src_dir,
             os.path.join(build_dir, buildername)                    # build directory
             ]
 
@@ -240,9 +243,9 @@ def action_build(args):
 
 
 def call_build_docs(entry):
-    (language, target, build_dir, sphinx_parallel_jobs, builders, input_docs) = entry
+    (language, target, build_dir, src_dir, sphinx_parallel_jobs, builders, input_docs) = entry
     for buildername in builders:
-        ret = sphinx_call(language, target, build_dir, sphinx_parallel_jobs, buildername, input_docs)
+        ret = sphinx_call(language, target, build_dir, src_dir, sphinx_parallel_jobs, buildername, input_docs)
 
         # Warnings are checked after each builder as logs are overwritten
         # check Doxygen warnings:

docs/test/README.md

@@ -0,0 +1,25 @@
+# Documentation Test Folder
+
+This folder contains the files needed for running tests the ESP-IDF documentation building system.
+
+Tests are divided into two categories: unit tests and integration tests:
+
+## Unit Tests
+
+The Sphinx IDF extensions are unit-tested in [test_sphinx_idf_extensions.py](test_sphinx_idf_extensions.py)
+
+## Integration Tests
+Due to the thigh integration with Sphinx some functionality is difficult to test with simple unit tests.
+
+ To check that the output from the Sphinx build process is as expected [test_docs.py](test_docs.py) builds a test subset of the documentation, found in the [en](en/) folder. The HTML output is then checked to see that it contains the expected content.
+
+# Running Tests
+
+Both [test_sphinx_idf_extensions.py](test_sphinx_idf_extensions.py) and [test_docs.py](test_docs.py) are run as part of the `host_test` stage of the CI pipeline.
+
+It's also possible to run the tests locally by running the following commands from the test folder:
+
+* `./test_sphinx_idf_extensions.py`
+* `./test_docs.py`
+
+Note that [test_docs.py](test_docs.py) tries to build a test subset of the documentation, and thus requires your environment to be set up for building documents. See [Documenting Code](https://docs.espressif.com/projects/esp-idf/en/latest/contribute/documenting-code.html) for instructions on how to set up the `build_docs` environment.

docs/test/en/bt_page.rst

@@ -0,0 +1,7 @@
+Bluetooth
+=========
+Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed iaculis, est ut blandit faucibus, dolor libero luctus tortor, finibus luctus neque elit et lacus.
+
+Sed at enim sed felis vehicula vehicula. Etiam ex ante, lacinia non purus quis, luctus ornare nibh. Phasellus rhoncus massa vitae tincidunt semper.
+
+Ut dapibus iaculis metus, vel consectetur diam euismod placerat. Maecenas nibh mauris, maximus et accumsan sit amet, lacinia at felis. Curabitur commodo eu lacus gravida volutpat. In hac habitasse platea dictumst. Quisque et tellus pulvinar, convallis nunc nec, sollicitudin mi. Curabitur et purus justo. Fusce non turpis quis nisi eleifend placerat a vitae.

docs/test/en/conf.py

@@ -0,0 +1,33 @@
+# -*- coding: utf-8 -*-
+#
+# English Language RTD & Sphinx config file
+#
+# Uses ../conf_common.py for most non-language-specific settings.
+# Importing conf_common adds all the non-language-specific
+# parts to this conf module
+try:
+    from conf_common import *  # noqa: F403,F401
+except ImportError:
+    import sys
+    import os
+    sys.path.insert(0, os.path.abspath('../..'))
+    from conf_common import *  # noqa: F403,F401
+
+# General information about the project.
+project = u'ESP-IDF Programming Guide'
+copyright = u'2016 - 2020, Espressif Systems (Shanghai) CO., LTD'
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+language = 'en'
+html_copy_source = False
+
+
+html_logo = None
+latex_logo = None
+html_static_path = []
+
+conditional_include_dict = {'esp32':["esp32_page.rst"],
+                            'esp32s2':["esp32s2_page.rst"],
+                            'SOC_BT_SUPPORTED':["bt_page.rst"],
+                            }

docs/test/en/esp32_page.rst

@@ -0,0 +1,3 @@
+ESP32 Page
+============
+{ESP32_CONTENT}

docs/test/en/esp32s2_page.rst

@@ -0,0 +1,19 @@
+ESP32S2 Page
+============
+Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed iaculis, est ut blandit faucibus, dolor libero luctus tortor, finibus luctus neque elit et lacus.
+
+Sed at enim sed felis vehicula vehicula. Etiam ex ante, lacinia non purus quis, luctus ornare nibh. Phasellus rhoncus massa vitae tincidunt semper.
+
+Ut dapibus iaculis metus, vel consectetur diam euismod placerat. Maecenas nibh mauris, maximus et accumsan sit amet, lacinia at felis. Curabitur commodo eu lacus gravida volutpat. In hac habitasse platea dictumst. Quisque et tellus pulvinar, convallis nunc nec, sollicitudin mi. Curabitur et purus justo. Fusce non turpis quis nisi eleifend placerat a vitae.
+
+.. only:: esp32
+
+    ESP32 Content. !ESP32_CONTENT!
+
+.. only:: esp32s2
+
+    ESP32 S2 Content. !ESP32_S2_CONTENT!
+
+.. only:: SOC_BT_SUPPORTED
+
+    Bluetooth Content. !BT_CONTENT!

docs/test/en/idf_target_format.rst

@@ -0,0 +1,3 @@
+IDF Target Format
+=================
+

docs/test/en/index.rst

@@ -0,0 +1,10 @@
+Build docs test index
+=====================
+
+
+.. toctree::
+
+   :esp32s2: ESP32-S2 Page <esp32s2_page>
+   :esp32: ESP32 Page !ESP32_CONTENT! <esp32_page>
+   :SOC_BT_SUPPORTED: BT Page !BT_CONTENT! <bt_page.rst>
+   IDF Target Format <idf_target_format>

docs/test/page_redirects.txt

@@ -0,0 +1,7 @@
+# Redirects from "old URL" "new URL"
+#
+# Space delimited
+#
+# New URL should be relative to document root, only)
+#
+# Empty lines and lines starting with # are ignored

docs/test/test_docs.py

@@ -0,0 +1,98 @@
+#!/usr/bin/env python3
+
+import unittest
+import subprocess
+import sys
+import os
+
+CURRENT_DIR = os.path.dirname(os.path.realpath(__file__))
+ESP32_DOC = "esp32_page"
+ESP32_S2_DOC = "esp32s2_page"
+BT_DOC = "bt_page"
+LINK_ROLES_DOC = "link_roles"
+IDF_FORMAT_DOC = "idf_target_format"
+
+
+class DocBuilder():
+    build_docs_py_path = os.path.join(CURRENT_DIR, '..', 'build_docs.py')
+
+    def __init__(self, src_dir, build_dir, target, language):
+        self.language = language
+        self.target = target
+        self.src_dir = src_dir
+        self.build_dir = build_dir
+        self.html_out_dir = os.path.join(CURRENT_DIR, build_dir, language, target, 'html')
+
+    def build(self, opt_args=[]):
+        args = [sys.executable, self.build_docs_py_path, "-b", self.build_dir, "-s", self.src_dir, "-t", self.target, "-l", self.language]
+        args.extend(opt_args)
+        return subprocess.call(args)
+
+
+class TestDocs(unittest.TestCase):
+
+    @classmethod
+    def setUpClass(cls):
+        cls.builder = DocBuilder("test", "_build/test_docs", "esp32s2", "en")
+        cls.build_ret_flag = cls.builder.build()
+
+    def setUp(self):
+        if self.build_ret_flag:
+            self.fail("Build docs failed with return: {}".format(self.build_ret_flag))
+
+    def assert_str_not_in_doc(self, doc_name, str_to_find):
+        with open(os.path.join(self.builder.html_out_dir, doc_name)) as f:
+            content = f.read()
+            self.assertFalse(str_to_find in content, "Found {} in {}".format(str_to_find, doc_name))
+
+    def assert_str_in_doc(self, doc_name, str_to_find):
+        with open(os.path.join(self.builder.html_out_dir, doc_name)) as f:
+            content = f.read()
+            self.assertTrue(str_to_find in content, "Did not find {} in {}".format(str_to_find, doc_name))
+
+    def test_only_dir(self):
+        # Test that ESP32 content was excluded
+        self.assert_str_not_in_doc(ESP32_S2_DOC + ".html", "!ESP32_CONTENT!")
+
+        # Test that ESP32 S2 content was included
+        self.assert_str_in_doc(ESP32_S2_DOC + ".html", "!ESP32_S2_CONTENT!")
+
+        # Test that BT content was excluded
+        self.assert_str_not_in_doc(ESP32_S2_DOC + ".html", "!BT_CONTENT!")
+
+    def test_toctree_filter(self):
+        # ESP32 page should NOT be built
+        esp32_doc = os.path.join(self.builder.html_out_dir, ESP32_DOC + ".html")
+        self.assertFalse(os.path.isfile(esp32_doc), "Found {}".format(esp32_doc))
+        self.assert_str_not_in_doc('index.html', "!ESP32_CONTENT!")
+
+        esp32s2_doc = os.path.join(self.builder.html_out_dir, ESP32_S2_DOC + ".html")
+        self.assertTrue(os.path.isfile(esp32s2_doc), "{} not found".format(esp32s2_doc))
+
+        # Spot check a few other tags
+        # No Bluetooth on ESP32 S2
+        bt_doc = os.path.join(self.builder.html_out_dir, BT_DOC + ".html")
+        self.assertFalse(os.path.isfile(bt_doc), "Found {}".format(bt_doc))
+        self.assert_str_not_in_doc('index.html', "!BT_CONTENT!")
+
+    def test_link_roles(self):
+        print("test")
+
+
+class TestBuildSubset(unittest.TestCase):
+    def test_build_subset(self):
+        builder = DocBuilder("test", "_build/test_build_subset", "esp32", "en")
+
+        docs_to_build = "esp32_page.rst"
+
+        self.assertFalse(builder.build(["-i", docs_to_build]))
+
+        # Check that we only built the input docs
+        bt_doc = os.path.join(builder.html_out_dir, BT_DOC + ".html")
+        esp32_doc = os.path.join(builder.html_out_dir, ESP32_DOC + ".html")
+        self.assertFalse(os.path.isfile(bt_doc), "Found {}".format(bt_doc))
+        self.assertTrue(os.path.isfile(esp32_doc), "Found {}".format(esp32_doc))
+
+
+if __name__ == '__main__':
+    unittest.main()

docs/test/test_sphinx_idf_extensions.py

@@ -0,0 +1,119 @@
+#!/usr/bin/env python3
+
+import os
+import sys
+import unittest
+from unittest.mock import MagicMock
+from tempfile import TemporaryDirectory
+
+from sphinx.util import tags
+
+try:
+    from idf_extensions import exclude_docs
+except ImportError:
+    sys.path.append('..')
+    from idf_extensions import exclude_docs
+
+from idf_extensions import format_idf_target
+from idf_extensions import gen_idf_tools_links
+from idf_extensions import link_roles
+
+
+class TestFormatIdfTarget(unittest.TestCase):
+
+    def setUp(self):
+        self.str_sub = format_idf_target.StringSubstituter()
+
+        config = MagicMock()
+        config.idf_target = 'esp32'
+        self.str_sub.init_sub_strings(config)
+
+    def test_add_subs(self):
+
+        self.assertEqual(self.str_sub.substitute_strings['{IDF_TARGET_NAME}'], "ESP32")
+        self.assertEqual(self.str_sub.substitute_strings['{IDF_TARGET_PATH_NAME}'], "esp32")
+        self.assertEqual(self.str_sub.substitute_strings['{IDF_TARGET_TOOLCHAIN_NAME}'], "esp32")
+        self.assertEqual(self.str_sub.substitute_strings['{IDF_TARGET_CFG_PREFIX}'], "ESP32")
+        self.assertEqual(self.str_sub.substitute_strings['{IDF_TARGET_TRM_EN_URL}'],
+                         "https://www.espressif.com/sites/default/files/documentation/esp32_technical_reference_manual_en.pdf")
+        self.assertEqual(self.str_sub.substitute_strings['{IDF_TARGET_TRM_CN_URL}'],
+                         "https://www.espressif.com/sites/default/files/documentation/esp32_technical_reference_manual_cn.pdf")
+
+    def test_sub(self):
+        content = ('This is a {IDF_TARGET_NAME}, with {IDF_TARGET_PATH_NAME}/soc.c, compiled with '
+                   'xtensa-{IDF_TARGET_TOOLCHAIN_NAME}-elf-gcc with CONFIG_{IDF_TARGET_CFG_PREFIX}_MULTI_DOC. '
+                   'TRM can be found at {IDF_TARGET_TRM_EN_URL} or {IDF_TARGET_TRM_CN_URL}')
+
+        expected = ('This is a ESP32, with esp32/soc.c, compiled with xtensa-esp32-elf-gcc with CONFIG_ESP32_MULTI_DOC. '
+                    'TRM can be found at https://www.espressif.com/sites/default/files/documentation/esp32_technical_reference_manual_en.pdf '
+                    'or https://www.espressif.com/sites/default/files/documentation/esp32_technical_reference_manual_cn.pdf')
+
+        self.assertEqual(self.str_sub.substitute(content), expected)
+
+    def test_local_sub(self):
+        content = ('{IDF_TARGET_TX_PIN:default="IO3", esp32="IO4", esp32s2="IO5"}'
+                   'The {IDF_TARGET_NAME} UART {IDF_TARGET_TX_PIN} uses for TX')
+
+        expected = "The ESP32 UART IO4 uses for TX"
+        self.assertEqual(self.str_sub.substitute(content), expected)
+
+    def test_local_sub_default(self):
+        content = ('{IDF_TARGET_TX_PIN:default="IO3", esp32s2="IO5"}'
+                   'The {IDF_TARGET_NAME} UART {IDF_TARGET_TX_PIN} uses for TX')
+
+        expected = "The ESP32 UART IO3 uses for TX"
+        self.assertEqual(self.str_sub.substitute(content), expected)
+
+    def test_local_sub_no_default(self):
+        content = ('{IDF_TARGET_TX_PIN: esp32="IO4", esp32s2="IO5"}'
+                   'The {IDF_TARGET_NAME} UART {IDF_TARGET_TX_PIN} uses for TX')
+
+        self.assertRaises(ValueError, self.str_sub.substitute, content)
+
+
+class TestExclude(unittest.TestCase):
+
+    def setUp(self):
+        self.app = MagicMock()
+        self.app.tags = tags.Tags()
+        self.app.config.conditional_include_dict = {"esp32":["esp32.rst", "bt.rst"], "esp32s2":["esp32s2.rst"]}
+        self.app.config.docs_to_build = None
+        self.app.config.exclude_patterns = []
+
+    def test_update_exclude_pattern(self):
+        self.app.tags.add("esp32")
+        exclude_docs.update_exclude_patterns(self.app, self.app.config)
+        docs_to_build = set(self.app.config.conditional_include_dict['esp32'])
+
+        # Check that the set of docs to build and the set of docs to exclude do not overlap
+        self.assertFalse(docs_to_build & set(self.app.config.exclude_patterns))
+
+
+class TestGenIDFToolLinks(unittest.TestCase):
+    def setUp(self):
+        self.app = MagicMock()
+        self.app.config.build_dir = "_build"
+        self.app.config.idf_path = os.environ['IDF_PATH']
+
+    def test_gen_idf_tool_links(self):
+
+        with TemporaryDirectory() as temp_dir:
+            self.app.config.build_dir = temp_dir
+            gen_idf_tools_links.generate_idf_tools_links(self.app, None)
+            self.assertTrue(os.path.isfile(os.path.join(self.app.config.build_dir, 'inc', 'idf-tools-inc.rst')))
+
+
+class TestLinkRoles(unittest.TestCase):
+    def test_get_submodules(self):
+        submod_dict = link_roles.get_submodules()
+
+        # Test a known submodule to see if it's in the dict
+        test_submod_name = 'components/asio/asio'
+        self.assertIn(test_submod_name, submod_dict)
+        self.assertIsNotNone(submod_dict[test_submod_name].url)
+        self.assertIsNotNone(submod_dict[test_submod_name].rev)
+        self.assertIsNotNone(submod_dict[test_submod_name].url)
+
+
+if __name__ == '__main__':
+    unittest.main()

tools/ci/config/host-test.yml

@@ -313,3 +313,25 @@ test_mkdfu:
     - cd ${IDF_PATH}/tools/test_mkdfu
     - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./test_mkdfu.py
 
+test_docs:
+  stage: host_test
+  image: $ESP_IDF_DOC_ENV_IMAGE
+  tags:
+    - host_test
+  dependencies: []
+  only:
+    variables:
+      - $BOT_TRIGGER_WITH_LABEL == null
+      - $BOT_LABEL_HOST_TEST
+      - $BOT_LABEL_REGULAR_TEST
+  artifacts:
+    when: on_failure
+    paths:
+        - docs/test/_build/*/*/*/html/*
+    expire_in: 1 week
+  script:
+  - cd ${IDF_PATH}/docs/test
+  - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 pip install -r ${IDF_PATH}/docs/requirements.txt
+  - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 ./test_docs.py
+  - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 ./test_sphinx_idf_extensions.py
+

tools/ci/executable-list.txt

@@ -18,6 +18,8 @@ components/ulp/esp32ulp_mapgen.py
 docs/build_docs.py
 docs/check_lang_folder_sync.sh
 docs/idf_extensions/gen_version_specific_includes.py
+docs/test/test_docs.py
+docs/test/test_sphinx_idf_extensions.py
 examples/build_system/cmake/idf_as_lib/build-esp32.sh
 examples/build_system/cmake/idf_as_lib/build.sh
 examples/build_system/cmake/idf_as_lib/run-esp32.sh