docs: use new sphinx_idf_theme theme, add variables to allow theme to build version footer

docs/README.md

@@ -13,15 +13,8 @@ Use actual documentation generated within about 20 minutes on each commit:
 
 The above URLs are all for the master branch latest version. Click the drop-down in the bottom left to choose a stable version or to download a PDF.
 
-
 # Building Documentation
 
-* Documentation build requres Python 3 and will not work with Python 2
-* Install dependencies for ESP-IDF as per the Getting Started guide
-* Install documentation Python depdendencies, ie `pip install -r $IDF_PATH/docs/requirements.txt`
-* Run `./build_docs.py build` to build docs for all supported Language & Target combinations, or `./build_docs.py -t esp32 -l en build` to build docs for a single supported language & target combination only.
-
-See [Documenting Code](https://docs.espressif.com/projects/esp-idf/en/latest/contribute/documenting-code.rst) for more information.
-
-Available languages are `en` and `zh_CN`, targets are any target supported by ESP-IDF - for example `esp32` or `esp32s2`.
+The documentation build requirements Python 3 and running the wrapper `./build_docs.py` tool. Running `./build_docs.py --help` will give a summary of available options.
 
+See [Documenting Code](https://docs.espressif.com/projects/esp-idf/en/latest/contribute/documenting-code.html) in the ESP-IDF Programming Guide for full information about setting up to build the docs, and how to use the `./build_docs.py` tool.

docs/conf_common.py

@@ -36,14 +36,14 @@ suppress_warnings = ['image.nonlocal_uri']
 
 # If your documentation needs a minimal Sphinx version, state it here.
 # needs_sphinx = '1.0'
-idf_target = ''
+
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
 extensions = ['breathe',
 
               'sphinx.ext.todo',
-
+              'sphinx_idf_theme',
               'sphinxcontrib.blockdiag',
               'sphinxcontrib.seqdiag',
               'sphinxcontrib.actdiag',
@@ -80,7 +80,6 @@ todo_include_todos = False
 # Enabling this fixes cropping of blockdiag edge labels
 seqdiag_antialias = True
 
-
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']
 
@@ -102,16 +101,24 @@ master_doc = 'index'
 # built documents.
 #
 
-# Readthedocs largely ignores 'version' and 'release', and displays one of
-# 'latest', tag name, or branch name, depending on the build type.
-# Still, this is useful for non-RTD builds.
-# This is supposed to be "the short X.Y version", but it's the only version
+# This is the full exact version, canonical git version description
 # visible when you open index.html.
-# Display full version to make things less confusing.
 version = subprocess.check_output(['git', 'describe']).strip().decode('utf-8')
-# The full version, including alpha/beta/rc tags.
-# If needed, nearest tag is returned by 'git describe --abbrev=0'.
-release = version
+
+# The 'release' version is the same as version for non-CI builds, but for CI
+# builds on a branch then it's replaced with the branch name
+try:
+    # default to using the Gitlab CI branch or tag if one is present
+    release = os.environ['CI_COMMIT_REF_NAME']
+
+    # emulate RTD's naming scheme for branches, master->latest, etc
+    release = release.replace('/', '-')
+    if release == "master":
+        release = "latest"
+except KeyError:
+    # otherwise, fall back to the full git version (no branch info)
+    release = version
+
 print('Version: {0}  Release: {1}'.format(version, release))
 
 # There are two options for replacing |today|: either, you set today to some
@@ -189,6 +196,15 @@ pygments_style = 'sphinx'
 # keep_warnings = False
 
 
+# Extra options required by sphinx_idf_theme
+project_slug = 'esp-idf'
+versions_url = 'https://dl.espressif.com/dl/esp-idf/idf_versions.js'
+
+idf_targets = ['esp32', 'esp32s2']
+languages = ['en', 'zh_CN']
+
+project_homepage = "https://github.com/espressif/esp-idf"
+
 # -- Options for HTML output ----------------------------------------------
 
 # Custom added feature to allow redirecting old URLs
@@ -204,7 +220,8 @@ html_redirect_pages = [tuple(l.split(' ')) for l in lines]
 
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
-html_theme = 'sphinx_rtd_theme'
+
+html_theme = 'sphinx_idf_theme'
 
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the
@@ -368,7 +385,12 @@ texinfo_documents = [
 # https://github.com/rtfd/sphinx_rtd_theme/pull/432
 def setup(app):
     app.add_stylesheet('theme_overrides.css')
-    app.add_config_value('idf_target', '', 'env')
+
+    # these two must be pushed in by build_docs.py
+    if "idf_target" not in app.config:
+        app.add_config_value('idf_target', None, 'env')
+        app.add_config_value('idf_targets', None, 'env')
+
     # Breathe extension variables (depend on build_dir)
     # note: we generate into xml_in and then copy_if_modified to xml dir
     app.config.breathe_projects = {"esp32-idf": os.path.join(app.config.build_dir, "xml_in/")}

docs/en/COPYRIGHT.rst

@@ -77,6 +77,11 @@ This is the list of licenses for tools included in this repository, which are us
 * :idf_file:`Menuconfig of Kconfiglib <tools/kconfig_new/menuconfig.py>` is Copyright (C) 2018-2019, Nordic Semiconductor ASA and Ulf Magnusson, and is licensed under the ISC License.
 
 
+Documentation
+-------------
+
+* HTML version of the `ESP-IDF Programming Guide`_ uses the Sphinx theme `sphinx_idf_theme`_, which is Copyright (c) 2013-2020 Dave Snider, Read the Docs, Inc. & contributors, and Espressif Systems (Shanghai) CO., LTD. It is based on `sphinx_rtd_theme`_. Both are licensed under MIT license.
+
 ROM Source Code Copyrights
 ==========================
 
@@ -171,3 +176,7 @@ Copyright (C) 2011, ChaN, all right reserved.
 .. _zephyr: https://github.com/zephyrproject-rtos/zephyr
 .. _mynewt-nimble: https://github.com/apache/mynewt-nimble
 .. _TinyCBOR: https://github.com/intel/tinycbor
+.. _ESP-IDF Programming Guide: https://docs.espressif.com/projects/esp-idf/en/latest/
+.. _sphinx_idf_theme: https://github.com/espressif/sphinx_idf_theme
+.. _sphinx_rtd_theme: https://github.com/readthedocs/sphinx_rtd_theme
+

docs/en/contribute/documenting-code.rst

@@ -361,17 +361,20 @@ You can setup environment to build documentation locally on your PC by installin
 1. Doxygen - https://www.stack.nl/~dimitri/doxygen/
 2. Sphinx - https://github.com/sphinx-doc/sphinx/#readme-for-sphinx
 3. Breathe - https://github.com/michaeljones/breathe#breathe
-4. Document theme "sphinx_rtd_theme" - https://github.com/rtfd/sphinx_rtd_theme
+4. Document theme "sphinx_idf_theme" - https://github.com/rtfd/sphinx_idf_theme
 5. Custom 404 page "sphinx-notfound-page" - https://github.com/rtfd/sphinx-notfound-page
 6. Blockdiag - http://blockdiag.com/en/index.html
 7. Recommonmark - https://github.com/rtfd/recommonmark
 
-The package "sphinx_rtd_theme" is added to have the same "look and feel" of `ESP32 Programming Guide <https://docs.espressif.com/projects/esp-idf/en/latest/index.html>`_ documentation like on the "Read the Docs" hosting site.
+The package "sphinx_idf_theme" is added to have the same "look and feel" of `ESP32 Programming Guide <https://docs.espressif.com/projects/esp-idf/en/latest/index.html>`_.
 
-Do not worry about being confronted with several packages to install. Besides Doxygen, all remaining packages are written in Python. Therefore installation of all of them is combined into one simple step.
+Do not worry about being confronted with several packages to install. Besides Doxygen and sphinx_idf_theme, all remaining packages are written in pure Python. Therefore installation of all of them is combined into one simple step.
 
 .. important:: Docs building now supports Python 3 only. Python 2 installations will not work.
 
+Doxygen
+@@@@@@@
+
 Installation of Doxygen is OS dependent:
 
 **Linux**
@@ -414,6 +417,26 @@ Installation of Doxygen is OS dependent:
 
     A downside of Windows installation is that fonts of the `blockdiag pictures <add-illustrations>` do not render correctly, you will see some random characters instead. Until this issue is fixed, you can use the `interactive shell`_ to see how the complete picture looks like.
 
+sphinx_idf_theme
+@@@@@@@@@@@@@@@@
+
+The ``sphinx_idf_theme`` needs a mixture of Python and JavaScript in order to build. So currently it's necessary install `node.js <https://nodejs.org/en/download/>` in order to build it locally.
+
+::
+   cd ~/esp
+   git clone https://github.com/espressif/sphinx_idf_theme.git
+   cd sphinx_idf_theme
+   npm install
+   python setup.py build
+   python setup.py install
+
+This is a temporary inconvenience. We plan to provide a way to install ``sphinx_idf_theme`` as a prebuilt package, soon.
+
+
+Remaining applications
+@@@@@@@@@@@@@@@@@@@@@@
+
+
 All remaining applications are `Python <https://www.python.org/>`_ packages and you can install them in one step as follows:
 
 ::

docs/requirements.txt

@@ -3,7 +3,6 @@
 #
 sphinx==2.3.1
 breathe==4.14.1
-sphinx-rtd-theme
 sphinx-notfound-page
 sphinxcontrib-blockdiag==2.0.0
 sphinxcontrib-seqdiag==2.0.0
@@ -14,3 +13,12 @@ nwdiag==2.0.0
 recommonmark
 future>=0.16.0 # for ../tools/gen_esp_err_to_name.py
 sphinx_selective_exclude==1.0.3
+
+
+# note: this package is not available on pypi, install by
+# downloading from https://github.com/espressif/sphinx_idf_theme then build and
+# install (requires node.js also).
+#
+# See docs/en/contribute/documenting-code.rst, 'sphinx_idf_theme' section
+# for full setup instructions.
+sphinx_idf_theme==0.1

tools/ci/config/build.yml

@@ -250,6 +250,7 @@ build_test_apps_esp32s2:
   dependencies: []
   script:
     - cd docs
+    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 pip install -r requirements.txt
     - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 ./build_docs.py -l $DOCLANG -t $DOCTGT build
 
 build_docs_en_esp32:

tools/ci/config/deploy.yml

@@ -72,11 +72,12 @@ push_to_github:
 
 .upload_doc_archive: &upload_doc_archive |
   cd docs/_build/$DOCLANG/$DOCTGT
-  mv html $GIT_VER
-  tar czf $GIT_VER.tar.gz $GIT_VER
-  ssh $DOCS_SERVER -x "mkdir -p $DOCS_PATH/$DOCLANG/$DOCTGT"
-  scp $GIT_VER.tar.gz $DOCS_SERVER:$DOCS_PATH/$DOCLANG/$DOCTGT
-  ssh $DOCS_SERVER -x "cd $DOCS_PATH/$DOCLANG/$DOCTGT && tar xzf $GIT_VER.tar.gz && rm -f latest && ln -s $GIT_VER latest"
+  mv html $DOCTGT
+  tar czf $DOCTGT.tar.gz $DOCTGT
+  # arrange for URL format language/version/target
+  ssh $DOCS_SERVER -x "mkdir -p $DOCS_PATH/$DOCLANG/$GIT_VER"
+  scp $DOCTGT.tar.gz $DOCS_SERVER:$DOCS_PATH/$DOCLANG/$GIT_VER"
+  ssh $DOCS_SERVER -x "cd $DOCS_PATH/$DOCLANG/$GIT_VER && tar xzf ${DOCTGT}.tar.gz && rm -f ../latest && ln -s . ../latest"
   cd -
 
 deploy_docs:
@@ -121,11 +122,43 @@ deploy_docs:
     - DOCLANG=zh_CN; DOCTGT=esp32s2
     - *upload_doc_archive
 
-    # add link to preview doc
-    - echo "[document preview][en][esp32] https://$CI_DOCKER_REGISTRY/docs/esp-idf/en/esp32/${GIT_VER}/index.html"
-    - echo "[document preview][en][esp32s2] https://$CI_DOCKER_REGISTRY/docs/esp-idf/en/esp32s2/${GIT_VER}/index.html"
-    - echo "[document preview][zh_CN][esp32] https://$CI_DOCKER_REGISTRY/docs/esp-idf/zh_CN/esp32/${GIT_VER}/index.html"
-    - echo "[document preview][zh_CN][esp32s2] https://$CI_DOCKER_REGISTRY/docs/esp-idf/zh_CN/esp32s2/${GIT_VER}/index.html"
+    # log link to doc URLs
+    - echo "[document $TYPE][en][esp32] $DOCS_URL_BASE/en/${GIT_VER}/esp32/index.html"
+    - echo "[document $TYPE][en][esp32s2] $DOCS_URL_BASE/en/${GIT_VER}/esp32s2/index.html"
+    - echo "[document $TYPE][zh_CN][esp32] $DOCS_URL_BASE/zh_CN/${GIT_VER}/esp32/index.html"
+    - echo "[document $TYPE][zh_CN][esp32s2] $DOCS_URL_BASE/zh_CN/${GIT_VER}/esp32s2/index.html"
+
+# deploys docs to CI_DOCKER_REGISTRY webserver, for internal review
+deploy_docs_preview:
+  extends: .deploy_docs_template
+  only:
+    refs:
+      - triggers
+    variables:
+      - $BOT_TRIGGER_WITH_LABEL == null
+      - $BOT_LABEL_BUILD_DOCS
+  variables:
+    TYPE: "preview"
+    # older branches use DOCS_DEPLOY_KEY, DOCS_SERVER, DOCS_PATH for preview server so we keep them
+    DOCS_DEPLOY_KEY: "$DOCS_DEPLOY_KEY"
+    DOCS_SERVER: "$DOCS_SERVER"
+    DOCS_PATH: "$DOCS_PATH"
+    DOCS_URL_BASE: "https://$CI_DOCKER_REGISTRY/docs/esp-idf"
+
+# deploy docs to production webserver
+deploy_docs_production:
+  extends: .deploy_docs_template
+  only:
+    refs:
+      - master
+      - /^release\/v/
+      - /^v\d+\.\d+(\.\d+)?($|-)/
+  variables:
+    TYPE: "preview"
+    DOCS_DEPLOY_KEY: "WIP"
+    DOCS_SERVER: "WIP"
+    DOCS_PATH: "WIP"
+    DOCS_URL_BASE: "https://$CI_DOCKER_REGISTRY/docs/esp-idf"
 
 deploy_test_result:
   stage: deploy

tools/ci/config/post_deploy.yml

@@ -15,6 +15,7 @@
   dependencies: []
   script:
     - cd docs
+    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 pip install -r requirements.txt
     - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 ./build_docs.py -l $DOCLANG -t $DOCTGT linkcheck
 
 check_doc_links_en_esp32:

tools/ci/config/pre_check.yml

@@ -24,6 +24,7 @@ check_docs_gh_links:
     SUBMODULES_TO_FETCH: "none"
   script:
     - cd docs
+    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 pip install -r requirements.txt
     - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 ./build_docs.py gh-linkcheck
 
 check_version: