Merge branch 'feature/deploy_multi_target_docs' into 'master'

docs: deploy docs to production server from CI

Closes IDF-1215 and IDF-1218

See merge request espressif/esp-idf!7637

.gitlab-ci.yml

@@ -46,6 +46,8 @@ variables:
 # target test config file, used by assign test job
   CI_TARGET_TEST_CONFIG_FILE: "$CI_PROJECT_DIR/tools/ci/config/target-test.yml"
 
+  # Versioned esp-idf-doc env image to use for all document building jobs
+  ESP_IDF_DOC_ENV_IMAGE: "$CI_DOCKER_REGISTRY/esp-idf-doc-env:v3"
 
 
 # before each job, we need to check if this job is filtered by bot stage/job filter

docs/README.md

@@ -13,15 +13,8 @@ Use actual documentation generated within about 20 minutes on each commit:
 
 The above URLs are all for the master branch latest version. Click the drop-down in the bottom left to choose a stable version or to download a PDF.
 
-
 # Building Documentation
 
-* Documentation build requres Python 3 and will not work with Python 2
-* Install dependencies for ESP-IDF as per the Getting Started guide
-* Install documentation Python depdendencies, ie `pip install -r $IDF_PATH/docs/requirements.txt`
-* Run `./build_docs.py build` to build docs for all supported Language & Target combinations, or `./build_docs.py -t esp32 -l en build` to build docs for a single supported language & target combination only.
-
-See [Documenting Code](https://docs.espressif.com/projects/esp-idf/en/latest/contribute/documenting-code.rst) for more information.
-
-Available languages are `en` and `zh_CN`, targets are any target supported by ESP-IDF - for example `esp32` or `esp32s2`.
+The documentation build requirements Python 3 and running the wrapper `./build_docs.py` tool. Running `./build_docs.py --help` will give a summary of available options.
 
+See [Documenting Code](https://docs.espressif.com/projects/esp-idf/en/latest/contribute/documenting-code.html) in the ESP-IDF Programming Guide for full information about setting up to build the docs, and how to use the `./build_docs.py` tool.

docs/build_docs.py

@@ -9,6 +9,20 @@
 #
 # Specific custom docs functionality should be added in conf_common.py or in a Sphinx extension, not here.
 #
+# Copyright 2020 Espressif Systems (Shanghai) PTE LTD
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
 from __future__ import print_function
 import argparse
 import locale

docs/conf_common.py

@@ -21,6 +21,7 @@ import os
 import os.path
 import re
 import subprocess
+from sanitize_version import sanitize_version
 from idf_extensions.util import download_file_if_missing
 
 # build_docs on the CI server sometimes fails under Python3. This is a workaround:
@@ -36,14 +37,14 @@ suppress_warnings = ['image.nonlocal_uri']
 
 # If your documentation needs a minimal Sphinx version, state it here.
 # needs_sphinx = '1.0'
-idf_target = ''
+
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
 extensions = ['breathe',
 
               'sphinx.ext.todo',
-
+              'sphinx_idf_theme',
               'sphinxcontrib.blockdiag',
               'sphinxcontrib.seqdiag',
               'sphinxcontrib.actdiag',
@@ -80,7 +81,6 @@ todo_include_todos = False
 # Enabling this fixes cropping of blockdiag edge labels
 seqdiag_antialias = True
 
-
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']
 
@@ -102,16 +102,14 @@ master_doc = 'index'
 # built documents.
 #
 
-# Readthedocs largely ignores 'version' and 'release', and displays one of
-# 'latest', tag name, or branch name, depending on the build type.
-# Still, this is useful for non-RTD builds.
-# This is supposed to be "the short X.Y version", but it's the only version
+# This is the full exact version, canonical git version description
 # visible when you open index.html.
-# Display full version to make things less confusing.
 version = subprocess.check_output(['git', 'describe']).strip().decode('utf-8')
-# The full version, including alpha/beta/rc tags.
-# If needed, nearest tag is returned by 'git describe --abbrev=0'.
-release = version
+
+# The 'release' version is the same as version for non-CI builds, but for CI
+# builds on a branch then it's replaced with the branch name
+release = sanitize_version(version)
+
 print('Version: {0}  Release: {1}'.format(version, release))
 
 # There are two options for replacing |today|: either, you set today to some
@@ -156,9 +154,10 @@ def update_exclude_patterns(tags):
             exclude_patterns.append(e)
 
     if "esp32s2" not in tags:
-        # Exclude ESP32-only document pages so they aren't found in the initial search for .rst files
+        # Exclude ESP32-S2-only document pages so they aren't found in the initial search for .rst files
         # note: in toctrees, these also need to be marked with a :esp32: filter
-        for e in ['hw-reference/esp32s2/**',
+        for e in ['esp32s2.rst',
+                  'hw-reference/esp32s2/**',
                   'api-guides/ulps2_instruction_set.rst',
                   'api-reference/peripherals/temp_sensor.rst']:
             exclude_patterns.append(e)
@@ -189,6 +188,15 @@ pygments_style = 'sphinx'
 # keep_warnings = False
 
 
+# Extra options required by sphinx_idf_theme
+project_slug = 'esp-idf'
+versions_url = 'https://dl.espressif.com/dl/esp-idf/idf_versions.js'
+
+idf_targets = ['esp32', 'esp32s2']
+languages = ['en', 'zh_CN']
+
+project_homepage = "https://github.com/espressif/esp-idf"
+
 # -- Options for HTML output ----------------------------------------------
 
 # Custom added feature to allow redirecting old URLs
@@ -204,7 +212,8 @@ html_redirect_pages = [tuple(l.split(' ')) for l in lines]
 
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
-html_theme = 'sphinx_rtd_theme'
+
+html_theme = 'sphinx_idf_theme'
 
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the
@@ -368,7 +377,12 @@ texinfo_documents = [
 # https://github.com/rtfd/sphinx_rtd_theme/pull/432
 def setup(app):
     app.add_stylesheet('theme_overrides.css')
-    app.add_config_value('idf_target', '', 'env')
+
+    # these two must be pushed in by build_docs.py
+    if "idf_target" not in app.config:
+        app.add_config_value('idf_target', None, 'env')
+        app.add_config_value('idf_targets', None, 'env')
+
     # Breathe extension variables (depend on build_dir)
     # note: we generate into xml_in and then copy_if_modified to xml dir
     app.config.breathe_projects = {"esp32-idf": os.path.join(app.config.build_dir, "xml_in/")}

docs/en/COPYRIGHT.rst

@@ -77,6 +77,11 @@ This is the list of licenses for tools included in this repository, which are us
 * :idf_file:`Menuconfig of Kconfiglib <tools/kconfig_new/menuconfig.py>` is Copyright (C) 2018-2019, Nordic Semiconductor ASA and Ulf Magnusson, and is licensed under the ISC License.
 
 
+Documentation
+-------------
+
+* HTML version of the `ESP-IDF Programming Guide`_ uses the Sphinx theme `sphinx_idf_theme`_, which is Copyright (c) 2013-2020 Dave Snider, Read the Docs, Inc. & contributors, and Espressif Systems (Shanghai) CO., LTD. It is based on `sphinx_rtd_theme`_. Both are licensed under MIT license.
+
 ROM Source Code Copyrights
 ==========================
 
@@ -171,3 +176,7 @@ Copyright (C) 2011, ChaN, all right reserved.
 .. _zephyr: https://github.com/zephyrproject-rtos/zephyr
 .. _mynewt-nimble: https://github.com/apache/mynewt-nimble
 .. _TinyCBOR: https://github.com/intel/tinycbor
+.. _ESP-IDF Programming Guide: https://docs.espressif.com/projects/esp-idf/en/latest/
+.. _sphinx_idf_theme: https://github.com/espressif/sphinx_idf_theme
+.. _sphinx_rtd_theme: https://github.com/readthedocs/sphinx_rtd_theme
+

docs/en/api-guides/build-system.rst

@@ -960,7 +960,7 @@ Selecting the Target
 
 ESP-IDF supports multiple targets (chips). The identifiers used for each chip are as follows:
 
-* ``esp32`` — for ESP32-D0WD, ESP32-D2WD, ESP32-S0WD (ESP-SOLO), ESP32-U4WD, ESP32-PICO-D4
+* ``esp32`` — for ESP32-D0WD, ESP32-D2WD, ESP32-S0WD (ESP-SOLO), ESP32-U4WDH, ESP32-PICO-D4
 * ``esp32s2``— for ESP32-S2
 
 To select the target before building the project, use ``idf.py set-target <target>`` command, for example::

docs/en/conf.py

@@ -13,7 +13,7 @@ from conf_common import *  # noqa: F401, F403 - need to make available everythin
 
 # General information about the project.
 project = u'ESP-IDF Programming Guide'
-copyright = u'2016 - 2019, Espressif Systems (Shanghai) CO., LTD'
+copyright = u'2016 - 2020, Espressif Systems (Shanghai) CO., LTD'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

docs/en/contribute/documenting-code.rst

@@ -361,17 +361,20 @@ You can setup environment to build documentation locally on your PC by installin
 1. Doxygen - https://www.stack.nl/~dimitri/doxygen/
 2. Sphinx - https://github.com/sphinx-doc/sphinx/#readme-for-sphinx
 3. Breathe - https://github.com/michaeljones/breathe#breathe
-4. Document theme "sphinx_rtd_theme" - https://github.com/rtfd/sphinx_rtd_theme
+4. Document theme "sphinx_idf_theme" - https://github.com/rtfd/sphinx_idf_theme
 5. Custom 404 page "sphinx-notfound-page" - https://github.com/rtfd/sphinx-notfound-page
 6. Blockdiag - http://blockdiag.com/en/index.html
 7. Recommonmark - https://github.com/rtfd/recommonmark
 
-The package "sphinx_rtd_theme" is added to have the same "look and feel" of `ESP32 Programming Guide <https://docs.espressif.com/projects/esp-idf/en/latest/index.html>`_ documentation like on the "Read the Docs" hosting site.
+The package "sphinx_idf_theme" is added to have the same "look and feel" of `ESP32 Programming Guide <https://docs.espressif.com/projects/esp-idf/en/latest/index.html>`_.
 
-Do not worry about being confronted with several packages to install. Besides Doxygen, all remaining packages are written in Python. Therefore installation of all of them is combined into one simple step.
+Do not worry about being confronted with several packages to install. Besides Doxygen and sphinx_idf_theme, all remaining packages are written in pure Python. Therefore installation of all of them is combined into one simple step.
 
 .. important:: Docs building now supports Python 3 only. Python 2 installations will not work.
 
+Doxygen
+@@@@@@@
+
 Installation of Doxygen is OS dependent:
 
 **Linux**
@@ -414,6 +417,26 @@ Installation of Doxygen is OS dependent:
 
     A downside of Windows installation is that fonts of the `blockdiag pictures <add-illustrations>` do not render correctly, you will see some random characters instead. Until this issue is fixed, you can use the `interactive shell`_ to see how the complete picture looks like.
 
+sphinx_idf_theme
+@@@@@@@@@@@@@@@@
+
+The ``sphinx_idf_theme`` needs a mixture of Python and JavaScript in order to build. So currently it's necessary install `node.js <https://nodejs.org/en/download/>` in order to build it locally.
+
+::
+   cd ~/esp
+   git clone https://github.com/espressif/sphinx_idf_theme.git
+   cd sphinx_idf_theme
+   npm install
+   python setup.py build
+   python setup.py install
+
+This is a temporary inconvenience. We plan to provide a way to install ``sphinx_idf_theme`` as a prebuilt package, soon.
+
+
+Remaining applications
+@@@@@@@@@@@@@@@@@@@@@@
+
+
 All remaining applications are `Python <https://www.python.org/>`_ packages and you can install them in one step as follows:
 
 ::

docs/en/esp32s2.rst

@@ -1,14 +1,16 @@
 ESP32-S2 Preview Support
 ========================
 
-The current master branch of ESP-IDF (pre-V4.2) contains preview support for ESP32-S2.
+The current master branch of ESP-IDF (pre-v4.2) contains preview support for ESP32-S2.
 
-Software support is still being developed ahead of the final ESP32-S2 release (see :ref:`esp32s2-limitations` and :ref:`esp32s2-roadmap` sections below).
+..note: Support for the ESP32-S2beta chip is included in ESP-IDF v4.1 as a preview only. In ESP-IDF v4.2 (this version) it has been replaced by ESP32-S2 (non-beta) chip support.
+
+Software support and documentation for ESP32-S2 chip is still being developed. See :ref:`esp32s2-limitations` section below.
 
 Documentation
 ^^^^^^^^^^^^^
 
-Except where it specifically mentions ESP32-S2, this version of the ESP-IDF Programming Guide is written for ESP32 only. Most software APIs and components are the same or very similar for ESP32-S2, but differences may not be documented yet.
+You are viewing the version of the ESP-IDF Programming Guide that describes ESP32-S2 support. However, as this version is still under development then there may be some changes from ESP32 that are not yet correctly documented here. If you find documentation that needs updating, please use the "Provide feedback about this document" link at the bottom of the page to tell us about it.
 
 Setting up for ESP32-S2
 ^^^^^^^^^^^^^^^^^^^^^^^
@@ -22,12 +24,7 @@ The toolchain tuple is `xtensa-esp32s2-elf-`, for example the GCC compiler for t
 Manual toolchain URLs
 ---------------------
 
-If not using ``install.sh`` or ``install.bat``, you can download the manual toolchain manually from here:
-
-- `Windows <https://dl.espressif.com/dl/xtensa-esp32s2-elf-gcc8_2_0-esp-2019r2-win32.zip>`_
-- `macOS <https://dl.espressif.com/dl/xtensa-esp32s2-elf-gcc8_2_0-esp-2019r2-macos.tar.gz>`_
-- `Linux 64-bit x86 <https://dl.espressif.com/dl/xtensa-esp32s2-elf-gcc8_2_0-esp-2019r2-linux-amd64.tar.gz>`_
-- `Linux 32-bit x86 <https://dl.espressif.com/dl/xtensa-esp32s2-elf-gcc8_2_0-esp-2019r2-linux-i686.tar.gz>`_
+If not using ``install.sh`` or ``install.bat``, you can download the toolchain manually. Download links can be found in the :ref:`idf-tools-list`, under the name ``xtensa-esp32s2-elf``.
 
 If installing the toolchain manually, unpack it somewhere and add the ``bin`` subdirectory to your `PATH`.
 
@@ -48,12 +45,12 @@ Before building an ESP-IDF project, run the following command to switch the targ
 
 .. important:: Running ``set-target`` will clear the project configuration and create a new empty ``sdkconfig`` file for the project. The old configuration will be saved in ``sdkconfig.old``.
 
+See :ref:`selecting-idf-target` for more details about switching between different targets (esp32, esp32s2).
+
 Then configure the project::
 
   idf.py menuconfig
 
-.. important:: There are 3 versions of ESP32-S2 available with different Wi-Fi PHYs. They can be identified by the marking on the chip itself. The project must be configured to match the chip via the project setting "ESP32-S2 chip version" (``CONFIG_ESP32S2_CHIP_VERSION``). This can be found under Component Config -> ESP32S2-specific. If the project setting doesn't match the chip then Wi-Fi performance may be very poor, or the chip may crash when initializing Wi-Fi.
-
 Build and flash the project as usual, for example::
 
   idf.py -p PORT flash monitor
@@ -71,25 +68,15 @@ ESP32-S2 support is currently a preview and does not support all features. Espre
 
 - The integrated USB OTG peripheral is not supported
 - Documentation not updated for ESP32-S2 in most places
-- Peripheral drivers are a work in progress
-- No power saving modes
+- Peripheral drivers and power saving modes are a work in progress
 - No hardware security features or cryptographic accelerator support
 - Time-of-Flight (TOF) measurements with normal Wi-Fi packets is not supported
 
-.. _esp32s2-roadmap:
-
-Roadmap
-^^^^^^^
-
-- ESP-IDF V4.1 will support ESP32-S2beta chip as a preview with some limitations.
-- ESP-IDF V4.2 will support ESP32-S2 (the previous ESP32-S2beta chip can't work on V4.2).
-
-Support for ESP32-S2beta will be removed from ESP-IDF once ESP32-S2 is available.
-
+The up-to-date list of features supported on ESP32-S2 is maintained at `in this forum post <https://esp32.com/viewtopic.php?f=10&p=56361>`_.
 
 Issues
 ^^^^^^
 
 If you find issues which are not mentioned in :ref:`esp32s2-limitations`, please `open an issue on GitHub <https://github.com/espressif/esp-idf/issues>`_.
 
-Please make sure to mention that you are using the ESP32-S2 preview support and give your exact version of ESP-IDF.
+Please make sure to mention that you are using the ESP32-S2 chip and give your exact version of ESP-IDF.

docs/en/index.rst

@@ -2,7 +2,16 @@ ESP-IDF Programming Guide
 =========================
 :link_to_translation:`zh_CN:[中文]`
 
-This is the documentation for Espressif IoT Development Framework (`esp-idf <https://github.com/espressif/esp-idf>`_). ESP-IDF is the official development framework for the `ESP32 <https://espressif.com/en/products/hardware/esp32/overview>`_ chip.
+This is the documentation for Espressif IoT Development Framework (`esp-idf <https://github.com/espressif/esp-idf>`_). ESP-IDF is the official development framework for the `ESP32 <https://www.espressif.com/en/products/hardware/socs>`_ SoC family.
+
+.. only:: html
+
+    This document describes using ESP-IDF with the {IDF_TARGET_NAME} SoC. To switch to a different SoC target, click "Switch Version" in the bottom left.
+
+.. only:: latex
+
+    This document describes using ESP-IDF with the {IDF_TARGET_NAME} SoC.
+
 
 ==================  ==================  ==================
 |Get Started|_      |API Reference|_    |H/W Reference|_
@@ -38,10 +47,10 @@ This is the documentation for Espressif IoT Development Framework (`esp-idf <htt
 .. toctree::
    :hidden:
 
+   :esp32s2: ESP32-S2 Preview Support <esp32s2>
    Get Started <get-started/index>
    API Reference <api-reference/index>
    H/W Reference <hw-reference/index>
-   ESP32-S2 Preview Support <esp32s2>
    API Guides <api-guides/index>
    Libraries and Frameworks <libraries-and-frameworks/index>
    Contribute <contribute/index>

docs/idf_extensions/link_roles.py

@@ -36,17 +36,7 @@ def setup(app):
     app.add_role('example_raw', github_link('raw', rev, '/examples/', app.config))
 
     # link to the current documentation file in specific language version
-    on_rtd = os.environ.get('READTHEDOCS', None) == 'True'
-    if on_rtd:
-        # provide RTD specific commit identification to be included in the link
-        tag_rev = 'latest'
-        if (subprocess.check_output(['git','rev-parse', '--short', 'HEAD']).decode('utf-8').strip() != rev):
-            tag_rev = rev
-    else:
-        # if not on the RTD then provide generic identification
-        tag_rev = subprocess.check_output(['git', 'describe', '--always']).decode('utf-8').strip()
-
-    app.add_role('link_to_translation', crosslink('%s../../%s/{}/%s.html'.format(tag_rev)))
+    app.add_role('link_to_translation', link_to_translation(app.config))
 
     return {'parallel_read_safe': True, 'parallel_write_safe': True, 'version': '0.2'}
 
@@ -117,13 +107,15 @@ def github_link(link_type, rev, root_path, app_config):
     return role
 
 
-def crosslink(pattern):
+def link_to_translation(config):
     def role(name, rawtext, text, lineno, inliner, options={}, content=[]):
         (language, link_text) = text.split(':')
         docname = inliner.document.settings.env.docname
         doc_path = inliner.document.settings.env.doc2path(docname, None, None)
-        return_path = '../' * doc_path.count('/')
-        url = pattern % (return_path, language, docname)
+        return_path = '../' * doc_path.count('/')  # path back to the root from 'docname'
+        # then take off 3 more paths for language/release/targetname and build the new URL
+        url = "{}.html".format(os.path.join(return_path, '../../..', language, config.release,
+                                            config.idf_target, docname))
         node = nodes.reference(rawtext, link_text, refuri=url, **options)
         return [node], []
     return role

docs/requirements.txt

@@ -3,7 +3,6 @@
 #
 sphinx==2.3.1
 breathe==4.14.1
-sphinx-rtd-theme
 sphinx-notfound-page
 sphinxcontrib-blockdiag==2.0.0
 sphinxcontrib-seqdiag==2.0.0
@@ -14,3 +13,12 @@ nwdiag==2.0.0
 recommonmark
 future>=0.16.0 # for ../tools/gen_esp_err_to_name.py
 sphinx_selective_exclude==1.0.3
+
+
+# note: this package is not available on pypi, install by
+# downloading from https://github.com/espressif/sphinx_idf_theme then build and
+# install (requires node.js also).
+#
+# See docs/en/contribute/documenting-code.rst, 'sphinx_idf_theme' section
+# for full setup instructions.
+sphinx_idf_theme==0.1

docs/sanitize_version.py

@@ -0,0 +1,43 @@
+# Tiny Python module to sanitize a Git version into something that can be used in a URL
+#
+# (this is used in multiple places: conf_common.py and in tools/ci/docs_deploy
+#
+# Copyright 2020 Espressif Systems (Shanghai) PTE LTD
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import os
+
+
+def sanitize_version(original_version):
+    """ Given a version (probably output from 'git describe --always' or similar), return
+    a URL-safe sanitized version. (this is used as 'release' config variable when building
+    the docs.)
+
+    Will override the original version with the Gitlab CI CI_COMMIT_REF_NAME environment variable if
+    this is present.
+
+    Also follows the RTD-ism that master branch is named 'latest'
+
+    """
+
+    try:
+        version = os.environ['CI_COMMIT_REF_NAME']
+    except KeyError:
+        version = original_version
+
+    if version == "master":
+        return "latest"
+
+    version = version.replace('/', '-')
+
+    return version

docs/zh_CN/api-guides/build-system.rst

@@ -791,13 +791,37 @@ Flash 参数
 
 子项目通过 :idf_file:`/components/bootloader/project_include.cmake` 文件作为外部项目插入到项目的顶层，主构建进程会运行子项目的 CMake，包括查找组件（主项目使用的组件的子集），生成引导程序专用的配置文件（从主 ``sdkconfig`` 文件中派生）。
 
-选择硬件目标
-============
+.. _selecting-idf-target:
 
-当前 ESP-IDF 仅支持一个硬件目标，即 ``esp32``，这也是构建系统默认的硬件目标。开发人员可以按照如下方法来添加对新硬件目标的支持::
+Selecting the Target
+====================
 
-    rm sdkconfig
-    idf.py -DIDF_TARGET=new_target reconfigure
+ESP-IDF supports multiple targets (chips). The identifiers used for each chip are as follows:
+
+* ``esp32`` — for ESP32-D0WD, ESP32-D2WD, ESP32-S0WD (ESP-SOLO), ESP32-U4WDH, ESP32-PICO-D4
+* ``esp32s2``— for ESP32-S2
+
+To select the target before building the project, use ``idf.py set-target <target>`` command, for example::
+
+    idf.py set-target esp32s2
+
+.. important::
+
+    ``idf.py set-target`` will clear the build directory and re-generate the ``sdkconfig`` file from scratch. The old ``sdkconfig`` file will be saved as ``sdkconfig.old``.
+
+.. note::
+
+    The behavior of ``idf.py set-target`` command is equivalent to:
+
+    1. clearing the build directory (``idf.py fullclean``)
+    2. removing the sdkconfig file (``mv sdkconfig sdkconfig.old``)
+    3. configuring the project with the new target (``idf.py -DIDF_TARGET=esp32 reconfigure``)
+
+It is also possible to pass the desired ``IDF_TARGET`` as an environement variable (e.g. ``export IDF_TARGET=esp32s2``) or as a CMake variable (e.g. ``-DIDF_TARGET=esp32s2`` argument to CMake or idf.py). Setting the environment variable is a convenient method if you mostly work with one type of the chip.
+
+To specify the _default_ value of ``IDF_TARGET`` for a given project, add ``CONFIG_IDF_TARGET`` value to ``sdkconfig.defaults``. For example, ``CONFIG_IDF_TARGET="esp32s2"``. This value will be used if ``IDF_TARGET`` is not specified by other method: using an environment variable, CMake variable, or ``idf.py set-target`` command.
+
+If the target has not been set by any of these methods, the build system will default to ``esp32`` target.
 
 .. _write-pure-component:
 

docs/zh_CN/conf.py

@@ -13,7 +13,7 @@ from conf_common import *  # noqa: F401, F403 - need to make available everythin
 
 # General information about the project.
 project = u'ESP-IDF 编程指南'
-copyright = u'2016 - 2019 乐鑫信息科技（上海）股份有限公司'
+copyright = u'2016 - 2020 乐鑫信息科技（上海）股份有限公司'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

docs/zh_CN/get-started/index.rst

@@ -320,6 +320,8 @@ Windows 操作系统
     cd %userprofile%\esp\hello_world
     idf.py menuconfig
 
+Setting the target with ``idf.py set-target {IDF_TARGET}`` should be done once, after opening a new project. If the project contains some existing builds and configuration, they will be cleared and initialized. The target may be saved in environment variable to skip this step at all. See :ref:`selecting-idf-target` for additional information.
+
 如果之前的步骤都正确，则会显示下面的菜单：
 
 .. figure:: ../../_static/project-configuration.png

docs/zh_CN/index.rst

@@ -39,10 +39,10 @@ ESP-IDF 编程指南
 .. toctree::
    :hidden:
 
+   :esp32s2: ESP32-S2 预览 <esp32s2>
    快速入门 <get-started/index>
    API 参考 <api-reference/index>
    H/W 参考 <hw-reference/index>
-   ESP32-S2 预览 <esp32s2>
    API 指南 <api-guides/index>
    Libraries and Frameworks <libraries-and-frameworks/index>
    贡献代码 <contribute/index>

tools/ci/config/build.yml

@@ -232,7 +232,7 @@ build_test_apps_esp32s2:
 
 .build_docs_template: &build_docs_template
   stage: build
-  image: $CI_DOCKER_REGISTRY/esp-idf-doc-env:v2
+  image: $ESP_IDF_DOC_ENV_IMAGE
   tags:
     - build_docs
   artifacts:
@@ -250,6 +250,7 @@ build_test_apps_esp32s2:
   dependencies: []
   script:
     - cd docs
+    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 pip install -r requirements.txt
     - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 ./build_docs.py -l $DOCLANG -t $DOCTGT build
 
 build_docs_en_esp32:

tools/ci/config/deploy.yml

@@ -70,62 +70,67 @@ push_to_github:
     - git remote add github git@github.com:espressif/esp-idf.git
     - tools/ci/push_to_github.sh
 
-.upload_doc_archive: &upload_doc_archive |
-  cd docs/_build/$DOCLANG/$DOCTGT
-  mv html $GIT_VER
-  tar czf $GIT_VER.tar.gz $GIT_VER
-  ssh $DOCS_SERVER -x "mkdir -p $DOCS_PATH/$DOCLANG/$DOCTGT"
-  scp $GIT_VER.tar.gz $DOCS_SERVER:$DOCS_PATH/$DOCLANG/$DOCTGT
-  ssh $DOCS_SERVER -x "cd $DOCS_PATH/$DOCLANG/$DOCTGT && tar xzf $GIT_VER.tar.gz && rm -f latest && ln -s $GIT_VER latest"
-  cd -
-
-deploy_docs:
+.deploy_docs_template:
+  extends: .before_script_lesser
   stage: deploy
-  image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
+  image: $ESP_IDF_DOC_ENV_IMAGE
   tags:
     - deploy
     - shiny
-  only:
-    refs:
-      - master
-      - /^release\/v/
-      - /^v\d+\.\d+(\.\d+)?($|-)/
-      - triggers
-    variables:
-      - $BOT_TRIGGER_WITH_LABEL == null
-      - $BOT_LABEL_BUILD_DOCS
   dependencies:
     - build_docs_en_esp32
     - build_docs_en_esp32s2
     - build_docs_zh_CN_esp32
     - build_docs_zh_CN_esp32s2
-  extends: .before_script_lesser
+  variables:
+    DOCS_BUILD_DIR: "${IDF_PATH}/docs/_build/"
+    PYTHONUNBUFFERED: 1
   script:
     - mkdir -p ~/.ssh
     - chmod 700 ~/.ssh
-    - echo -n $DOCS_DEPLOY_KEY > ~/.ssh/id_rsa_base64
+    - echo -n $DOCS_DEPLOY_PRIVATEKEY > ~/.ssh/id_rsa_base64
     - base64 --decode --ignore-garbage ~/.ssh/id_rsa_base64 > ~/.ssh/id_rsa
     - chmod 600 ~/.ssh/id_rsa
-    - echo -e "Host $DOCS_SERVER\n\tStrictHostKeyChecking no\n\tUser $DOCS_SERVER_USER\n" >> ~/.ssh/config
+    - echo -e "Host $DOCS_DEPLOY_SERVER\n\tStrictHostKeyChecking no\n\tUser $DOCS_DEPLOY_SERVER_USER\n" >> ~/.ssh/config
     - export GIT_VER=$(git describe --always)
 
-    - DOCLANG=en; DOCTGT=esp32
-    - *upload_doc_archive
+    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 ${IDF_PATH}/tools/ci/deploy_docs.py
 
-    - DOCLANG=en; DOCTGT=esp32s2
-    - *upload_doc_archive
 
-    - DOCLANG=zh_CN; DOCTGT=esp32
-    - *upload_doc_archive
+# deploys docs to CI_DOCKER_REGISTRY webserver, for internal review
+deploy_docs_preview:
+  extends: .deploy_docs_template
+  only:
+    refs:
+      - triggers
+    variables:
+      - $BOT_TRIGGER_WITH_LABEL == null
+      - $BOT_LABEL_BUILD_DOCS
+  variables:
+    TYPE: "preview"
+    # older branches use DOCS_DEPLOY_KEY, DOCS_SERVER, DOCS_SERVER_USER, DOCS_PATH for preview server so we keep these names for 'preview'
+    DOCS_DEPLOY_PRIVATEKEY: "$DOCS_DEPLOY_KEY"
+    DOCS_DEPLOY_SERVER: "$DOCS_SERVER"
+    DOCS_DEPLOY_SERVER_USER: "$DOCS_SERVER_USER"
+    DOCS_DEPLOY_PATH: "$DOCS_PATH"
+    DOCS_DEPLOY_URL_BASE: "https://$CI_DOCKER_REGISTRY/docs/esp-idf"
 
-    - DOCLANG=zh_CN; DOCTGT=esp32s2
-    - *upload_doc_archive
-
-    # add link to preview doc
-    - echo "[document preview][en][esp32] https://$CI_DOCKER_REGISTRY/docs/esp-idf/en/esp32/${GIT_VER}/index.html"
-    - echo "[document preview][en][esp32s2] https://$CI_DOCKER_REGISTRY/docs/esp-idf/en/esp32s2/${GIT_VER}/index.html"
-    - echo "[document preview][zh_CN][esp32] https://$CI_DOCKER_REGISTRY/docs/esp-idf/zh_CN/esp32/${GIT_VER}/index.html"
-    - echo "[document preview][zh_CN][esp32s2] https://$CI_DOCKER_REGISTRY/docs/esp-idf/zh_CN/esp32s2/${GIT_VER}/index.html"
+# deploy docs to production webserver
+deploy_docs_production:
+  extends: .deploy_docs_template
+  only:
+    refs:
+      # The DOCS_PROD_* variables used by this job are "Protected" so these branches must all be marked "Protected" in Gitlab settings
+      - master
+      - /^release\/v/
+      - /^v\d+\.\d+(\.\d+)?($|-)/
+  variables:
+    TYPE: "preview"
+    DOCS_DEPLOY_PRIVATEKEY: "$DOCS_PROD_DEPLOY_KEY"
+    DOCS_DEPLOY_SERVER: "$DOCS_PROD_SERVER"
+    DOCS_DEPLOY_SERVER_USER: "$DOCS_PROD_SERVER_USER"
+    DOCS_DEPLOY_PATH: "$DOCS_PROD_PATH"
+    DOCS_DEPLOY_URL_BASE: "https://docs.espressif.com/projects/esp-idf"
 
 deploy_test_result:
   stage: deploy

tools/ci/config/post_deploy.yml

@@ -1,6 +1,6 @@
 .check_doc_links_template: &check_doc_links_template
   stage: post_deploy
-  image: $CI_DOCKER_REGISTRY/esp-idf-doc-env:v2
+  image: $ESP_IDF_DOC_ENV_IMAGE
   tags: [ "build", "amd64", "internet" ]
   only:
     - master
@@ -15,6 +15,7 @@
   dependencies: []
   script:
     - cd docs
+    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 pip install -r requirements.txt
     - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 ./build_docs.py -l $DOCLANG -t $DOCTGT linkcheck
 
 check_doc_links_en_esp32:

tools/ci/config/pre_check.yml

@@ -24,6 +24,7 @@ check_docs_gh_links:
     SUBMODULES_TO_FETCH: "none"
   script:
     - cd docs
+    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 pip install -r requirements.txt
     - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 ./build_docs.py gh-linkcheck
 
 check_version:

tools/ci/deploy_docs.py

@@ -0,0 +1,185 @@
+#!/usr/bin/env python3
+#
+# CI script to deploy docs to a webserver. Not useful outside of CI environment
+#
+#
+# Copyright 2020 Espressif Systems (Shanghai) PTE LTD
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import glob
+import os
+import os.path
+import re
+import stat
+import sys
+import subprocess
+import tarfile
+import packaging.version
+
+
+def env(variable, default=None):
+    """ Shortcut to return the expanded version of an environment variable """
+    return os.path.expandvars(os.environ.get(variable, default) if default else os.environ[variable])
+
+
+# import sanitize_version from the docs directory, shared with here
+sys.path.append(os.path.join(env("IDF_PATH"), "docs"))
+from sanitize_version import sanitize_version  # noqa
+
+
+def main():
+    # if you get KeyErrors on the following lines, it's probably because you're not running in Gitlab CI
+    git_ver = env("GIT_VER")  # output of git describe --always
+    ci_ver = env("CI_COMMIT_REF_NAME", git_ver)  # branch or tag we're building for (used for 'release' & URL)
+
+    version = sanitize_version(ci_ver)
+    print("Git version: {}".format(git_ver))
+    print("CI Version: {}".format(ci_ver))
+    print("Deployment version: {}".format(version))
+
+    if not version:
+        raise RuntimeError("A version is needed to deploy")
+
+    build_dir = env("DOCS_BUILD_DIR")  # top-level local build dir, where docs have already been built
+
+    if not build_dir:
+        raise RuntimeError("Valid DOCS_BUILD_DIR is needed to deploy")
+
+    url_base = env("DOCS_DEPLOY_URL_BASE")  # base for HTTP URLs, used to print the URL to the log after deploying
+
+    docs_server = env("DOCS_DEPLOY_SERVER")  # ssh server to deploy to
+    docs_user = env("DOCS_DEPLOY_SERVER_USER")
+    docs_path = env("DOCS_DEPLOY_PATH")  # filesystem path on DOCS_SERVER
+
+    if not docs_server:
+        raise RuntimeError("Valid DOCS_DEPLOY_SERVER is needed to deploy")
+
+    if not docs_user:
+        raise RuntimeError("Valid DOCS_DEPLOY_SERVER_USER is needed to deploy")
+
+    docs_server = "{}@{}".format(docs_user, docs_server)
+
+    if not docs_path:
+        raise RuntimeError("Valid DOCS_DEPLOY_PATH is needed to deploy")
+
+    print("DOCS_DEPLOY_SERVER {} DOCS_DEPLOY_PATH {}".format(docs_server, docs_path))
+
+    tarball_path, version_urls = build_doc_tarball(version, build_dir)
+
+    deploy(version, tarball_path, docs_path, docs_server)
+
+    print("Docs URLs:")
+    for vurl in version_urls:
+        url = "{}/{}/index.html".format(url_base, vurl)  # (index.html needed for the preview server)
+        url = re.sub(r"([^:])//", r"\1/", url)  # get rid of any // that isn't in the https:// part
+        print(url)
+
+    # note: it would be neater to use symlinks for stable, but because of the directory order
+    # (language first) it's kind of a pain to do on a remote server, so we just repeat the
+    # process but call the version 'stable' this time
+    if is_stable_version(version):
+        print("Deploying again as stable version...")
+        tarball_path, version_urls = build_doc_tarball("stable", build_dir)
+        deploy("stable", tarball_path, docs_path, docs_server)
+
+
+def deploy(version, tarball_path, docs_path, docs_server):
+
+    def run_ssh(commands):
+        """ Log into docs_server and run a sequence of commands using ssh """
+        print("Running ssh: {}".format(commands))
+        subprocess.run(["ssh", "-o", "BatchMode=yes", docs_server, "-x", " && ".join(commands)], check=True)
+
+    # copy the version tarball to the server
+    run_ssh(["mkdir -p {}".format(docs_path)])
+    print("Running scp {} to {}".format(tarball_path, "{}:{}".format(docs_server, docs_path)))
+    subprocess.run(["scp", "-B", tarball_path, "{}:{}".format(docs_server, docs_path)], check=True)
+
+    tarball_name = os.path.basename(tarball_path)
+
+    run_ssh(["cd {}".format(docs_path),
+             "rm -rf ./*/{}".format(version),   # remove any pre-existing docs matching this version
+             "tar -zxvf {}".format(tarball_name),  # untar the archive with the new docs
+             "rm {}".format(tarball_name)])
+
+    # Note: deleting and then extracting the archive is a bit awkward for updating stable/latest/etc
+    # as the version will be invalid for a window of time. Better to do it atomically, but this is
+    # another thing made much more complex by the directory structure putting language before version...
+
+
+def build_doc_tarball(version, build_dir):
+    """ Make a tar.gz archive of the docs, in the directory structure used to deploy as
+        the given version """
+    version_paths = []
+    tarball_path = "{}/{}.tar.gz".format(build_dir, version)
+
+    # find all the 'html/' directories under build_dir
+    html_dirs = glob.glob("{}/**/html/".format(build_dir), recursive=True)
+    print("Found %d html directories" % len(html_dirs))
+
+    def not_sources_dir(ti):
+        """ Filter the _sources directories out of the tarballs """
+        if ti.name.endswith("/_sources"):
+            return None
+
+        ti.mode |= stat.S_IWGRP  # make everything group-writeable
+        return ti
+
+    try:
+        os.remove(tarball_path)
+    except OSError:
+        pass
+
+    with tarfile.open(tarball_path, "w:gz") as tarball:
+        for html_dir in html_dirs:
+            # html_dir has the form '<ignored>/<language>/<target>/html/'
+            target_dirname = os.path.dirname(os.path.dirname(html_dir))
+            target = os.path.basename(target_dirname)
+            language = os.path.basename(os.path.dirname(target_dirname))
+
+            # when deploying, we want the top-level directory layout 'language/version/target'
+            archive_path = "{}/{}/{}".format(language, version, target)
+            print("Archiving '{}' as '{}'...".format(html_dir, archive_path))
+            tarball.add(html_dir, archive_path, filter=not_sources_dir)
+            version_paths.append(archive_path)
+
+    return (os.path.abspath(tarball_path), version_paths)
+
+
+def is_stable_version(version):
+    """ Heuristic for whether this is the latest stable release """
+    if not version.startswith("v"):
+        return False  # branch name
+    if "-" in version:
+        return False  # prerelease tag
+
+    git_out = subprocess.run(["git", "tag", "-l"], capture_output=True, check=True)
+
+    versions = [v.strip() for v in git_out.stdout.decode("utf-8").split("\n")]
+    versions = [v for v in versions if re.match(r"^v[\d\.]+$", v)]  # include vX.Y.Z only
+
+    versions = [packaging.version.parse(v) for v in versions]
+
+    max_version = max(versions)
+
+    if max_version.public != version[1:]:
+        print("Stable version is v{}. This version is {}.".format(max_version.public, version))
+        return False
+    else:
+        print("This version {} is the stable version".format(version))
+        return True
+
+
+if __name__ == "__main__":
+    main()

tools/ci/executable-list.txt

@@ -45,6 +45,7 @@ tools/ci/check_idf_version.sh
 tools/ci/check_public_headers.sh
 tools/ci/check_ut_cmake_make.sh
 tools/ci/checkout_project_ref.py
+tools/ci/deploy_docs.py
 tools/ci/envsubst.py
 tools/ci/fix_empty_prototypes.sh
 tools/ci/get-full-sources.sh