docs: self host docs

Move doc hosting from readthedocs to espressif servers

Update CI, Sphinx configs and add IDF Sphinx theme

.gitlab-ci.yml

@@ -60,6 +60,9 @@ variables:
   CI_AUTO_TEST_SCRIPT_REPO_URL: "https://gitlab-ci-token:${BOT_TOKEN}@${CI_SERVER_HOST}:${CI_SERVER_PORT}/qa/auto_test_script.git"
   CI_AUTO_TEST_SCRIPT_REPO_BRANCH: "ci/v3.1"
 
+# Versioned esp-idf-doc env image to use for all document building jobs
+  ESP_IDF_DOC_ENV_IMAGE: "$CI_DOCKER_REGISTRY/esp-idf-doc-env:v7"
+
 
 
 # before each job, we need to check if this job is filtered by bot stage/job filter

docs/Doxyfile

@@ -266,6 +266,7 @@ MACRO_EXPANSION        = YES
 EXPAND_ONLY_PREDEF     = YES
 PREDEFINED             = \
     __attribute__(x)= \
+    _Static_assert()= \
     IDF_DEPRECATED(X)= \
     IRAM_ATTR= \
     configSUPPORT_DYNAMIC_ALLOCATION=1 \

docs/conf_common.py

@@ -20,16 +20,21 @@ import sys
 import os
 import re
 import subprocess
+from sanitize_version import sanitize_version
+from get_github_rev import get_github_rev
+
 
 # Note: If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute
 
-from local_util import run_cmd_get_output, copy_if_modified
+from local_util import copy_if_modified
 
 # build_docs on the CI server sometimes fails under Python3. This is a workaround:
 sys.setrecursionlimit(3500)
 
+config_dir = os.path.abspath(os.path.dirname(__file__))
+
 try:
     builddir = os.environ['BUILDDIR']
 except KeyError:
@@ -146,6 +151,7 @@ suppress_warnings = ['image.nonlocal_uri']
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
 extensions = ['breathe',
+              'sphinx_idf_theme',
               'link-roles',
               'sphinxcontrib.blockdiag',
               'sphinxcontrib.seqdiag',
@@ -154,6 +160,7 @@ extensions = ['breathe',
               'sphinxcontrib.rackdiag',
               'sphinxcontrib.packetdiag',
               'html_redirects',
+              'latex_builder',
               ]
 
 # Enabling this fixes cropping of blockdiag edge labels
@@ -188,16 +195,14 @@ master_doc = 'index'
 # built documents.
 #
 
-# Readthedocs largely ignores 'version' and 'release', and displays one of
+# This is the full exact version, canonical git version description
-# 'latest', tag name, or branch name, depending on the build type.
-# Still, this is useful for non-RTD builds.
-# This is supposed to be "the short X.Y version", but it's the only version
 # visible when you open index.html.
-# Display full version to make things less confusing.
+version = subprocess.check_output(['git', 'describe']).strip().decode('utf-8')
-version = run_cmd_get_output('git describe')
+
-# The full version, including alpha/beta/rc tags.
+# The 'release' version is the same as version for non-CI builds, but for CI
-# If needed, nearest tag is returned by 'git describe --abbrev=0'.
+# builds on a branch then it's replaced with the branch name
-release = version
+release = sanitize_version(version)
+
 print('Version: {0}  Release: {1}'.format(version, release))
 
 # There are two options for replacing |today|: either, you set today to some
@@ -228,6 +233,13 @@ exclude_patterns = ['_build','README.md']
 # The name of the Pygments (syntax highlighting) style to use.
 pygments_style = 'sphinx'
 
+# Extra options required by sphinx_idf_theme
+project_slug = 'esp-idf'
+versions_url = 'https://dl.espressif.com/dl/esp-idf/idf_versions.js'
+
+languages = ['en', 'zh_CN']
+
+
 # A list of ignored prefixes for module index sorting.
 # modindex_common_prefix = []
 
@@ -250,7 +262,15 @@ html_redirect_pages = [tuple(line.split(' ')) for line in lines]
 
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
-html_theme = 'sphinx_rtd_theme'
+html_theme = 'sphinx_idf_theme'
+
+# context used by sphinx_idf_theme
+html_context = {
+    "display_github": True,  # Add 'Edit on Github' link instead of 'View page source'
+    "github_user": "espressif",
+    "github_repo": "esp-idf",
+    "github_version": get_github_rev(),
+}
 
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the
@@ -333,28 +353,41 @@ htmlhelp_basename = 'ReadtheDocsTemplatedoc'
 
 # -- Options for LaTeX output ---------------------------------------------
 
-latex_elements = {
+latex_template_dir = os.path.join(config_dir, 'latex_templates')
-    # The paper size ('letterpaper' or 'a4paper').
-    # 'papersize': 'letterpaper',
-    #
-    # The font size ('10pt', '11pt' or '12pt').
-    # 'pointsize': '10pt',
-    #
-    # Additional stuff for the LaTeX preamble.
-    # 'preamble': '',
-}
 
-# Grouping the document tree into LaTeX files. List of tuples
+preamble = ''
-# (source start file, target name, title,
+with open(os.path.join(latex_template_dir, 'preamble.tex')) as f:
-#  author, documentclass [howto, manual, or own class]).
+    preamble = f.read()
-latex_documents = [
+
-    ('index', 'ReadtheDocsTemplate.tex', u'Read the Docs Template Documentation',
+titlepage = ''
-     u'Read the Docs', 'manual'),
+with open(os.path.join(latex_template_dir, 'titlepage.tex')) as f:
-]
+    titlepage = f.read()
+
+
+latex_elements = {
+    'papersize': 'a4paper',
+
+    # Latex figure (float) alignment
+    'figure_align':'htbp',
+
+    'pointsize': '10pt',
+
+    # Additional stuff for the LaTeX preamble.
+    'fncychap': '\\usepackage[Sonny]{fncychap}',
+
+    'preamble': preamble,
+
+    'maketitle': titlepage,
+}
 
 # The name of an image file (relative to this directory) to place at the top of
 # the title page.
-# latex_logo = None
+
+# The name of an image file (relative to this directory) to place at the bottom of
+# the title page.
+latex_logo = "../_static/espressif2.pdf"
+latex_engine = 'xelatex'
+latex_use_xindy = False
 
 # For "manual" documents, if this is true, then toplevel headings are parts,
 # not chapters.
@@ -415,3 +448,24 @@ texinfo_documents = [
 def setup(app):
     app.add_stylesheet('theme_overrides.css')
     generate_version_specific_includes(app)
+
+    # Not all config variables are set when setup is called
+    app.connect('config-inited',  setup_config_values)
+    app.connect('config-inited',  setup_html_context)
+
+
+def setup_config_values(app, config):
+    # Sets up global config values needed by other extensions
+    idf_target_title_dict = {
+        'esp32': 'ESP32',
+    }
+
+    app.add_config_value('idf_target_title_dict', idf_target_title_dict, 'env')
+
+    pdf_name = "esp-idf-{}-{}-{}".format(app.config.language, app.config.version, "esp32")
+    app.add_config_value('pdf_file', pdf_name, 'env')
+
+
+def setup_html_context(app, config):
+    # Setup path for 'edit on github'-link
+    app.config.html_context['conf_py_path'] = "/docs/{}/".format(app.config.language)

docs/en/conf.py

@@ -17,7 +17,7 @@ except ImportError:
 
 # General information about the project.
 project = u'ESP-IDF Programming Guide'
-copyright = u'2016 - 2020, Espressif Systems (Shanghai) CO., LTD'
+copyright = u'2016 - 2021, Espressif Systems (Shanghai) CO., LTD'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

docs/gen-version-specific-includes.py

@@ -189,34 +189,29 @@ def write_version_note(template, out_dir, version, ver_type, is_stable):
 
 def get_version():
     """
-    Returns a tuple of (name of branch/tag, type branch/tag, is_stable)
+    Returns a tuple of (name of branch/tag/commit-id, type branch/tag/commit, is_stable)
     """
-    # Trust what RTD says our version is, if it is set
+    # Use git to look for a tag
-    version = os.environ.get("READTHEDOCS_VERSION", None)
-    if version == "latest":
-        return ("master", "branch", False)
-
-    # Otherwise, use git to look for a tag
     try:
-        tag = subprocess.check_output(["git", "describe", "--exact-match"]).strip()
+        tag = subprocess.check_output(["git", "describe", "--exact-match"]).strip().decode('utf-8')
         is_stable = re.match(r"v[0-9\.]+$", tag) is not None
         return (tag, "tag", is_stable)
     except subprocess.CalledProcessError:
         pass
 
-    # No tag, look for a branch
+    # No tag, look at branch name from CI, this will give the correct branch name even if the ref for the branch we
-    refs = subprocess.check_output(["git", "for-each-ref", "--points-at", "HEAD", "--format", "%(refname)"])
+    # merge into has moved forward before the pipeline runs
-    print("refs:\n%s" % refs)
+    branch = os.environ.get("CI_COMMIT_REF_NAME", None)
-    refs = refs.split(b"\n")
+    if branch is not None:
-    # Note: this looks for branches in 'origin' because GitLab CI doesn't check out a local branch
+        return (branch, "branch", False)
-    branches = [r.replace(b"refs/remotes/origin/",b"").strip() for r in refs if r.startswith(b"refs/remotes/origin/")]
+
-    if len(branches) == 0:
+    # Try to find the branch name even if docs are built locally
-        # last resort, return the commit (may happen on Gitlab CI sometimes, unclear why)
+    branch = subprocess.check_output(["git", "rev-parse", "--abbrev-ref", "HEAD"]).strip().decode('utf-8')
-        return (subprocess.check_output(["git", "rev-parse", "--short", "HEAD"]).strip(), "commit", False)
+    if branch != "HEAD":
-    if "master" in branches:
+        return (branch, "branch", False)
-        return ("master", "branch", False)
+
-    else:
+    # As a last resort we return commit SHA-1, should never happen in CI/docs that should be published
-        return (branches[0], "branch", False)  # take whatever the first branch is
+    return (subprocess.check_output(["git", "rev-parse", "--short", "HEAD"]).strip().decode('utf-8'), "commit", False)
 
 
 if __name__ == "__main__":

docs/get_github_rev.py

@@ -0,0 +1,15 @@
+import subprocess
+
+
+# Get revision used for constructing github URLs
+def get_github_rev():
+    path = subprocess.check_output(['git', 'rev-parse', '--short', 'HEAD']).strip().decode('utf-8')
+    try:
+        tag = subprocess.check_output(['git', 'describe', '--exact-match']).strip().decode('utf-8')
+    except subprocess.CalledProcessError:
+        tag = None
+    print('Git commit ID: ', path)
+    if tag:
+        print('Git tag: ', tag)
+        return tag
+    return path

docs/latex_builder.py

@@ -0,0 +1,55 @@
+from sphinx.builders.latex import LaTeXBuilder
+import os
+
+
+# Overrides the default Sphinx latex build
+class IdfLatexBuilder(LaTeXBuilder):
+
+    def __init__(self, app):
+
+        # Sets up the latex_documents config value, done here instead of conf.py since it depends on the runtime value 'idf_target'
+        self.init_latex_documents(app)
+
+        super().__init__(app)
+
+    def init_latex_documents(self, app):
+
+        file_name = app.config.pdf_file + '.tex'
+
+        if app.config.language == 'zh_CN':
+            latex_documents = [('index', file_name, u'ESP-IDF 编程指南', u'乐鑫信息科技', 'manual')]
+        else:
+            # Default to english naming
+            latex_documents = [('index', file_name, u'ESP-IDF Programming Guide', u'Espressif Systems', 'manual')]
+
+        app.config.latex_documents = latex_documents
+
+    def prepare_latex_macros(self, package_path, config):
+
+        PACKAGE_NAME = "espidf.sty"
+        latex_package = ''
+        with open(package_path, 'r') as template:
+
+            latex_package = template.read()
+
+        idf_target_title = "ESP32"
+        latex_package = latex_package.replace('<idf_target_title>', idf_target_title)
+
+        # Release name for the PDF front page, remove '_' as this is used for subscript in Latex
+        idf_release_name = "Release {}".format(config.version.replace('_', '-'))
+        latex_package = latex_package.replace('<idf_release_name>', idf_release_name)
+
+        with open(os.path.join(self.outdir, PACKAGE_NAME), 'w') as package_file:
+            package_file.write(latex_package)
+
+    def finish(self):
+        super().finish()
+
+        TEMPLATE_PATH = "../latex_templates/espidf.sty"
+        self.prepare_latex_macros(os.path.join(self.confdir,TEMPLATE_PATH), self.config)
+
+
+def setup(app):
+    app.add_builder(IdfLatexBuilder, override=True)
+
+    return {'parallel_read_safe': True, 'parallel_write_safe': True, 'version': '0.1'}

docs/latex_templates/espidf.sty

@@ -0,0 +1,7 @@
+\NeedsTeXFormat{LaTeX2e}[1995/12/01]
+\ProvidesPackage{espidf}[2020/03/25 v0.1.0 LaTeX package (ESP-IDF markup)]
+
+\newcommand{\idfTarget}{<idf_target_title>}
+\newcommand{\idfReleaseName}{<idf_release_name>}
+
+\endinput

docs/latex_templates/preamble.tex

@@ -0,0 +1,129 @@
+% package with esp-idf specific macros
+\usepackage{espidf}
+
+\setcounter{secnumdepth}{2}
+\setcounter{tocdepth}{2}
+
+\usepackage{amsmath,amsfonts,amssymb,amsthm}
+\usepackage{graphicx}
+%%% reduce spaces for Table of contents, figures and tables
+%%% it is used "\addtocontents{toc}{\vskip -1.2cm}" etc. in the document
+\usepackage[notlot,nottoc,notlof]{}
+
+\usepackage{color}
+\usepackage{transparent}
+\usepackage{eso-pic}
+\usepackage{lipsum}
+
+%%% Needed for displaying Chinese in English documentation
+\usepackage{xeCJK}
+
+\usepackage{footnotebackref} %%link at the footnote to go to the place of footnote in the text
+
+%% spacing between line
+\usepackage{setspace}
+\singlespacing
+
+
+\definecolor{myred}{RGB}{229, 32, 26}
+\definecolor{mygrayy}{RGB}{127, 127, 127}
+\definecolor{myblack}{RGB}{64, 64, 64}
+
+
+%%%%%%%%%%% datetime
+\usepackage{datetime}
+
+\newdateformat{MonthYearFormat}{%
+    \monthname[\THEMONTH], \THEYEAR}
+
+
+%% RO, LE will not work for 'oneside' layout.
+%% Change oneside to twoside in document class
+\usepackage{fancyhdr}
+\pagestyle{fancy}
+\fancyhf{}
+
+% Header and footer
+\makeatletter
+  \fancypagestyle{normal}{
+    \fancyhf{}
+    \fancyhead[L]{\nouppercase{\leftmark}}
+    \fancyfoot[C]{\py@HeaderFamily\thepage \\ \href{https://www.espressif.com/en/company/documents/documentation_feedback?docId=4287&sections=&version=\idfReleaseName}{Submit Document Feedback}}
+    \fancyfoot[L]{Espressif Systems}
+    \fancyfoot[R]{\idfReleaseName}
+    \renewcommand{\headrulewidth}{0.4pt}
+    \renewcommand{\footrulewidth}{0.4pt}
+  }
+\makeatother
+
+\renewcommand{\headrulewidth}{0.5pt}
+\renewcommand{\footrulewidth}{0.5pt}
+
+
+% Define a spacing for section, subsection and subsubsection
+% http://tex.stackexchange.com/questions/108684/spacing-before-and-after-section-titles
+
+\titlespacing*{\section}{0pt}{6pt plus 0pt minus 0pt}{6pt plus 0pt minus 0pt}
+\titlespacing*{\subsection}{0pt}{18pt plus 64pt minus 0pt}{0pt}
+\titlespacing*{\subsubsection}{0pt}{12pt plus 0pt minus 0pt}{0pt}
+\titlespacing*{\paragraph}    {0pt}{3.25ex plus 1ex minus .2ex}{1.5ex plus .2ex}
+\titlespacing*{\subparagraph} {0pt}{3.25ex plus 1ex minus .2ex}{1.5ex plus .2ex}
+
+% Define the colors of table of contents
+% This is helpful to understand http://tex.stackexchange.com/questions/110253/what-the-first-argument-for-lsubsection-actually-is
+\definecolor{LochmaraColor}{HTML}{1020A0}
+
+% Hyperlinks
+\hypersetup{
+    colorlinks = true,
+    allcolors = {LochmaraColor},
+}
+
+
+\RequirePackage{tocbibind} %%% comment this to remove page number for following
+\addto\captionsenglish{\renewcommand{\contentsname}{Table of contents}}
+\addto\captionsenglish{\renewcommand{\listfigurename}{List of figures}}
+\addto\captionsenglish{\renewcommand{\listtablename}{List of tables}}
+% \addto\captionsenglish{\renewcommand{\chaptername}{Chapter}}
+
+
+
+
+%%reduce spacing for itemize
+\usepackage{enumitem}
+\setlist{nosep}
+
+%%%%%%%%%%% Quote Styles at the top of chapter
+\usepackage{epigraph}
+\setlength{\epigraphwidth}{0.8\columnwidth}
+\newcommand{\chapterquote}[2]{\epigraphhead[60]{\epigraph{\textit{#1}}{\textbf {\textit{--#2}}}}}
+%%%%%%%%%%% Quote for all places except Chapter
+\newcommand{\sectionquote}[2]{{\quote{\textit{``#1''}}{\textbf {\textit{--#2}}}}}
+
+% Insert 22pt white space before roc title. \titlespacing at line 65 changes it by -22 later on.
+\renewcommand*\contentsname{\hspace{0pt}Contents}
+
+
+% Define section, subsection and subsubsection font size and color
+\usepackage{sectsty}
+\definecolor{AllportsColor}{HTML}{A02010}
+\allsectionsfont{\color{AllportsColor}}
+
+\usepackage{titlesec}
+\titleformat{\section}
+{\color{AllportsColor}\LARGE\bfseries}{\thesection.}{1em}{}
+
+\titleformat{\subsection}
+{\color{AllportsColor}\Large\bfseries}{\thesubsection.}{1em}{}
+
+\titleformat{\subsubsection}
+{\color{AllportsColor}\large\bfseries}{\thesubsubsection.}{1em}{}
+
+\titleformat{\paragraph}
+{\color{AllportsColor}\large\bfseries}{\theparagraph}{1em}{}
+
+\titleformat{\subparagraph}
+  {\normalfont\normalsize\bfseries}{\thesubparagraph}{1em}{}
+
+\titleformat{\subsubparagraph}
+  {\normalfont\normalsize\bfseries}{\thesubsubparagraph}{1em}{}

docs/latex_templates/titlepage.tex

@@ -0,0 +1,39 @@
+\makeatletter
+\newgeometry{left=0cm,right=0cm,bottom=2cm}
+
+
+\cfoot{www.espressif.com}
+
+\renewcommand{\headrulewidth}{0pt}
+
+{\color{myred}\rule{30pt}{2.1cm}}
+    \hspace{0.2cm}
+    \begin{minipage}[b]{18cm}
+    {\fontsize{36pt}{48pt}\textbf{\idfTarget}}\\
+
+    {\fontsize{28pt}{18pt}\textbf{\color{mygrayy}\@title}}
+    \end{minipage}
+    \hspace{\stretch{1}}
+
+\vspace{48em}
+
+
+\begin{flushright}
+ \setlength\parindent{8em}
+    \begin{minipage}[b]{2cm}
+    \sphinxlogo
+    \end{minipage}
+    \hspace{0.2cm}
+    \rule{3pt}{1.9cm}
+    \hspace{0.2cm}
+    \begin{minipage}[b]{7cm}
+       {\large{\idfReleaseName}}\smallskip\newline
+        {\large{\@author}}\smallskip\newline
+         {\large{\@date}}\smallskip
+    \end{minipage}
+    {\color{myred}\rule{30pt}{1.9cm}}
+\end{flushright}
+
+
+\restoregeometry
+\makeatother

docs/requirements.txt

@@ -1,17 +1,15 @@
 # This is a list of python packages used to generate documentation. This file is used with pip:
 # pip install --user -r requirements.txt
 #
-sphinx>=1.8.4
+sphinx==2.3.1
-breathe==4.11.1
+breathe==4.14.1
-sphinx-rtd-theme
 sphinx-notfound-page
-sphinxcontrib-blockdiag>=1.5.5, <2.0.0
+sphinxcontrib-blockdiag==2.0.0
-sphinxcontrib-seqdiag>=0.8.5, <2.0.0
+sphinxcontrib-seqdiag==2.0.0
-sphinxcontrib-actdiag>=0.8.5, <2.0.0
+sphinxcontrib-actdiag==2.0.0
-sphinxcontrib-nwdiag>=0.9.5, <2.0.0
+sphinxcontrib-nwdiag==2.0.0
-blockdiag>=1.5.4, <2.0.0
+sphinxcontrib-wavedrom==2.0.0
-seqdiag>=0.9.6, <2.0.0
+nwdiag==2.0.0
-actdiag>=0.5.4, <2.0.0
-nwdiag>=1.0.4, <2.0.0
 recommonmark
 future>=0.16.0 # for ../tools/gen_esp_err_to_name.py
+sphinx_idf_theme==0.2

docs/sanitize_version.py

@@ -0,0 +1,43 @@
+# Tiny Python module to sanitize a Git version into something that can be used in a URL
+#
+# (this is used in multiple places: conf_common.py and in tools/ci/docs_deploy
+#
+# Copyright 2020 Espressif Systems (Shanghai) PTE LTD
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import os
+
+
+def sanitize_version(original_version):
+    """ Given a version (probably output from 'git describe --always' or similar), return
+    a URL-safe sanitized version. (this is used as 'release' config variable when building
+    the docs.)
+
+    Will override the original version with the Gitlab CI CI_COMMIT_REF_NAME environment variable if
+    this is present.
+
+    Also follows the RTD-ism that master branch is named 'latest'
+
+    """
+
+    try:
+        version = os.environ['CI_COMMIT_REF_NAME']
+    except KeyError:
+        version = original_version
+
+    if version == "master":
+        return "latest"
+
+    version = version.replace('/', '-')
+
+    return version

docs/zh_CN/api-guides/unit-tests-legacy.rst

@@ -189,7 +189,7 @@ DUT2 终端::
 
 当单元测试应用程序空闲时，输入回车键，它会打印出测试菜单，其中包含所有的测试项目。
 
-.. code:: bash
+.. code::
 
    Here's the test menu, pick your combo:
    (1)     "esp_ota_begin() verifies arguments" [ota]
@@ -236,7 +236,7 @@ DUT2 终端::
 
 一旦选择了多设备测试用例，它会打印一个子菜单：
 
-.. code:: bash
+.. code::
 
    Running gpio master/slave test example...
    gpio master/slave test example
@@ -247,7 +247,7 @@ DUT2 终端::
 
 与多设备测试用例相似，多阶段测试用例也会打印子菜单：
 
-.. code:: bash
+.. code::
 
    Running reset reason check for deepsleep...
    reset reason check for deepsleep

docs/zh_CN/api-guides/unit-tests.rst

@@ -17,7 +17,7 @@ C 文件可以包含多个测试用例。测试文件的名字要以 “test”
 
 测试用例需要通过 C 文件中特定的函数来添加，如下所示：
 
-.. code:: c
+.. code-block:: c
 
    TEST_CASE("test name", "[module name]"
    {
@@ -179,7 +179,7 @@ DUT2（slave）终端：
 
 当单元测试应用程序空闲时，输入回车键，它会打印出测试菜单，其中包含所有的测试项目。
 
-.. code:: bash
+.. code::
 
    Here's the test menu, pick your combo:
    (1)     "esp_ota_begin() verifies arguments" [ota]
@@ -226,7 +226,7 @@ DUT2（slave）终端：
 
 一旦选择了多设备测试用例，它会打印一个子菜单：
 
-.. code:: bash
+.. code::
 
    Running gpio master/slave test example...
    gpio master/slave test example
@@ -237,7 +237,7 @@ DUT2（slave）终端：
 
 与多设备测试用例相似，多阶段测试用例也会打印子菜单：
 
-.. code:: bash
+.. code::
 
    Running reset reason check for deepsleep...
    reset reason check for deepsleep

docs/zh_CN/conf.py

@@ -17,7 +17,7 @@ except ImportError:
 
 # General information about the project.
 project = u'ESP-IDF 编程指南'
-copyright = u'2016 - 2020 乐鑫信息科技（上海）股份有限公司'
+copyright = u'2016 - 2021 乐鑫信息科技（上海）股份有限公司'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

tools/ci/config/build.yml

@@ -198,25 +198,17 @@ build_examples_cmake_esp32:
 
 # If you want to add new build example jobs, please add it into dependencies of `.example_test_template`
 
-build_docs:
+.build_docs_template: &build_docs_template
   stage: build
-  image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
+  image: $ESP_IDF_DOC_ENV_IMAGE
   tags:
     - build_docs
   artifacts:
     when: always
     paths:
-      # English version of documentation
+      - docs/*/*.txt
-      - docs/en/doxygen-warning-log.txt
+      - docs/_build/*/html/*
-      - docs/en/sphinx-warning-log.txt
+      - docs/_build/*/latex/*
-      - docs/en/sphinx-warning-log-sanitized.txt
-      - docs/en/_build/html
-      - docs/sphinx-err-*
-      # Chinese version of documentation
-      - docs/zh_CN/doxygen-warning-log.txt
-      - docs/zh_CN/sphinx-warning-log.txt
-      - docs/zh_CN/sphinx-warning-log-sanitized.txt
-      - docs/zh_CN/_build/html
     expire_in: 4 days
   only:
     variables:
@@ -224,17 +216,31 @@ build_docs:
       - $BOT_LABEL_BUILD
       - $BOT_LABEL_BUILD_DOCS
       - $BOT_LABEL_REGULAR_TEST
+  dependencies: []
   script:
+    # Active python 3.6.10 env as this is where Sphinx is installed
+    - source /opt/pyenv/activate && pyenv global 3.6.10
+      # Setup a build dir with both languages to simplify deployment
     - cd docs
+    - mkdir -p _build/$DOCLANG
     - ./check_lang_folder_sync.sh
-    - cd en
+    - cd $DOCLANG
     - make gh-linkcheck
     - make html
     - ../check_doc_warnings.sh
-    - cd ../zh_CN
+    - make latexpdf LATEXMKOPTS="--f --interaction=nonstopmode --quiet --outdir=build"
-    - make gh-linkcheck
-    - make html
     - ../check_doc_warnings.sh
+    - mv -f _build/* ../_build/$DOCLANG
+
+build_docs_en:
+  extends: .build_docs_template
+  variables:
+    DOCLANG: "en"
+
+build_docs_zh_CN:
+  extends: .build_docs_template
+  variables:
+    DOCLANG: "zh_CN"
 
 verify_cmake_style:
   extends: .check_job_template

tools/ci/config/deploy.yml

@@ -73,45 +73,67 @@ push_to_github:
     - git remote add github git@github.com:espressif/esp-idf.git
     - tools/ci/push_to_github.sh
 
-deploy_docs:
+.deploy_docs_template:
+  extends: .before_script_lesser
   stage: deploy
-  image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
+  image: $ESP_IDF_DOC_ENV_IMAGE
   tags:
     - deploy
     - shiny
+  dependencies:
+    - build_docs_en
+    - build_docs_zh_CN
+  variables:
+    DOCS_BUILD_DIR: "${IDF_PATH}/docs/_build/"
+    PYTHONUNBUFFERED: 1
+  script:
+    - mkdir -p ~/.ssh
+    - chmod 700 ~/.ssh
+    - echo -n $DOCS_DEPLOY_PRIVATEKEY > ~/.ssh/id_rsa_base64
+    - base64 --decode --ignore-garbage ~/.ssh/id_rsa_base64 > ~/.ssh/id_rsa
+    - chmod 600 ~/.ssh/id_rsa
+    - echo -e "Host $DOCS_DEPLOY_SERVER\n\tStrictHostKeyChecking no\n\tUser $DOCS_DEPLOY_SERVER_USER\n" >> ~/.ssh/config
+    - export GIT_VER=$(git describe --always)
+
+    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 ${IDF_PATH}/tools/ci/deploy_docs.py
+
+
+# deploys docs to CI_DOCKER_REGISTRY webserver, for internal review
+deploy_docs_preview:
+  extends: .deploy_docs_template
   only:
     refs:
-      - master
-      - /^release\/v/
-      - /^v\d+\.\d+(\.\d+)?($|-)/
       - triggers
     variables:
       - $BOT_TRIGGER_WITH_LABEL == null
       - $BOT_LABEL_BUILD_DOCS
-  dependencies:
+  variables:
-    - build_docs
+    TYPE: "preview"
-  extends: .before_script_lesser
+    # older branches use DOCS_DEPLOY_KEY, DOCS_SERVER, DOCS_SERVER_USER, DOCS_PATH for preview server so we keep these names for 'preview'
-  script:
+    DOCS_DEPLOY_PRIVATEKEY: "$DOCS_DEPLOY_KEY"
-    - mkdir -p ~/.ssh
+    DOCS_DEPLOY_SERVER: "$DOCS_SERVER"
-    - chmod 700 ~/.ssh
+    DOCS_DEPLOY_SERVER_USER: "$DOCS_SERVER_USER"
-    - echo -n $DOCS_DEPLOY_KEY > ~/.ssh/id_rsa_base64
+    DOCS_DEPLOY_PATH: "$DOCS_PATH"
-    - base64 --decode --ignore-garbage ~/.ssh/id_rsa_base64 > ~/.ssh/id_rsa
+    DOCS_DEPLOY_URL_BASE: "https://$CI_DOCKER_REGISTRY/docs/esp-idf"
-    - chmod 600 ~/.ssh/id_rsa
+
-    - echo -e "Host $DOCS_SERVER\n\tStrictHostKeyChecking no\n\tUser $DOCS_SERVER_USER\n" >> ~/.ssh/config
+# deploy docs to production webserver
-    - export GIT_VER=$(git describe --always)
+deploy_docs_production:
-    - cd docs/en/_build/
+  extends: .deploy_docs_template
-    - mv html $GIT_VER
+  only:
-    - tar czvf $GIT_VER.tar.gz $GIT_VER
+    refs:
-    - scp $GIT_VER.tar.gz $DOCS_SERVER:$DOCS_PATH/en
+      # The DOCS_PROD_* variables used by this job are "Protected" so these branches must all be marked "Protected" in Gitlab settings
-    - ssh $DOCS_SERVER -x "cd $DOCS_PATH/en && tar xzvf $GIT_VER.tar.gz && rm -f latest && ln -s $GIT_VER latest"
+      - master
-    - cd ../../zh_CN/_build/
+      - /^release\/v/
-    - mv html $GIT_VER
+      - /^v\d+\.\d+(\.\d+)?($|-)/
-    - tar czvf $GIT_VER.tar.gz $GIT_VER
+    variables:
-    - scp $GIT_VER.tar.gz $DOCS_SERVER:$DOCS_PATH/zh_CN
+      - $BOT_TRIGGER_WITH_LABEL == null
-    - ssh $DOCS_SERVER -x "cd $DOCS_PATH/zh_CN && tar xzvf $GIT_VER.tar.gz && rm -f latest && ln -s $GIT_VER latest"
+  variables:
-    # add link to preview doc
+    TYPE: "preview"
-    - echo "[document preview][en] $CI_DOCKER_REGISTRY/docs/esp-idf/en/${GIT_VER}/index.html"
+    DOCS_DEPLOY_PRIVATEKEY: "$DOCS_PROD_DEPLOY_KEY"
-    - echo "[document preview][zh_CN] $CI_DOCKER_REGISTRY/docs/esp-idf/zh_CN/${GIT_VER}/index.html"
+    DOCS_DEPLOY_SERVER: "$DOCS_PROD_SERVER"
+    DOCS_DEPLOY_SERVER_USER: "$DOCS_PROD_SERVER_USER"
+    DOCS_DEPLOY_PATH: "$DOCS_PROD_PATH"
+    DOCS_DEPLOY_URL_BASE: "https://docs.espressif.com/projects/esp-idf"
 
 deploy_test_result:
   stage: deploy

tools/ci/deploy_docs.py

@@ -0,0 +1,223 @@
+#!/usr/bin/env python3
+#
+# CI script to deploy docs to a webserver. Not useful outside of CI environment
+#
+#
+# Copyright 2020 Espressif Systems (Shanghai) PTE LTD
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import glob
+import os
+import os.path
+import re
+import stat
+import sys
+import subprocess
+import tarfile
+import packaging.version
+
+
+def env(variable, default=None):
+    """ Shortcut to return the expanded version of an environment variable """
+    return os.path.expandvars(os.environ.get(variable, default) if default else os.environ[variable])
+
+
+# import sanitize_version from the docs directory, shared with here
+sys.path.append(os.path.join(env("IDF_PATH"), "docs"))
+from sanitize_version import sanitize_version  # noqa
+
+
+def main():
+    # if you get KeyErrors on the following lines, it's probably because you're not running in Gitlab CI
+    git_ver = env("GIT_VER")  # output of git describe --always
+    ci_ver = env("CI_COMMIT_REF_NAME", git_ver)  # branch or tag we're building for (used for 'release' & URL)
+
+    version = sanitize_version(ci_ver)
+    print("Git version: {}".format(git_ver))
+    print("CI Version: {}".format(ci_ver))
+    print("Deployment version: {}".format(version))
+
+    if not version:
+        raise RuntimeError("A version is needed to deploy")
+
+    build_dir = env("DOCS_BUILD_DIR")  # top-level local build dir, where docs have already been built
+
+    if not build_dir:
+        raise RuntimeError("Valid DOCS_BUILD_DIR is needed to deploy")
+
+    url_base = env("DOCS_DEPLOY_URL_BASE")  # base for HTTP URLs, used to print the URL to the log after deploying
+
+    docs_server = env("DOCS_DEPLOY_SERVER")  # ssh server to deploy to
+    docs_user = env("DOCS_DEPLOY_SERVER_USER")
+    docs_path = env("DOCS_DEPLOY_PATH")  # filesystem path on DOCS_SERVER
+
+    if not docs_server:
+        raise RuntimeError("Valid DOCS_DEPLOY_SERVER is needed to deploy")
+
+    if not docs_user:
+        raise RuntimeError("Valid DOCS_DEPLOY_SERVER_USER is needed to deploy")
+
+    docs_server = "{}@{}".format(docs_user, docs_server)
+
+    if not docs_path:
+        raise RuntimeError("Valid DOCS_DEPLOY_PATH is needed to deploy")
+
+    print("DOCS_DEPLOY_SERVER {} DOCS_DEPLOY_PATH {}".format(docs_server, docs_path))
+
+    tarball_path, version_urls = build_doc_tarball(version, git_ver, build_dir)
+
+    deploy(version, tarball_path, docs_path, docs_server)
+
+    print("Docs URLs:")
+    doc_deploy_type = os.getenv('TYPE')
+    for vurl in version_urls:
+        language, _,  = vurl.split('/')
+        tag = '{}'.format(language)
+        url = "{}/{}/index.html".format(url_base, vurl)  # (index.html needed for the preview server)
+        url = re.sub(r"([^:])//", r"\1/", url)  # get rid of any // that isn't in the https:// part
+        print('[document {}][{}] {}'.format(doc_deploy_type, tag, url))
+
+    # note: it would be neater to use symlinks for stable, but because of the directory order
+    # (language first) it's kind of a pain to do on a remote server, so we just repeat the
+    # process but call the version 'stable' this time
+    if is_stable_version(version):
+        print("Deploying again as stable version...")
+        tarball_path, version_urls = build_doc_tarball("stable", git_ver, build_dir)
+        deploy("stable", tarball_path, docs_path, docs_server)
+
+
+def deploy(version, tarball_path, docs_path, docs_server):
+    def run_ssh(commands):
+        """ Log into docs_server and run a sequence of commands using ssh """
+        print("Running ssh: {}".format(commands))
+        subprocess.run(["ssh", "-o", "BatchMode=yes", docs_server, "-x", " && ".join(commands)], check=True)
+
+    # copy the version tarball to the server
+    run_ssh(["mkdir -p {}".format(docs_path)])
+    print("Running scp {} to {}".format(tarball_path, "{}:{}".format(docs_server, docs_path)))
+    subprocess.run(["scp", "-B", tarball_path, "{}:{}".format(docs_server, docs_path)], check=True)
+
+    tarball_name = os.path.basename(tarball_path)
+
+    run_ssh(["cd {}".format(docs_path),
+             "rm -rf ./*/{}".format(version),  # remove any pre-existing docs matching this version
+             "tar -zxvf {}".format(tarball_name),  # untar the archive with the new docs
+             "rm {}".format(tarball_name)])
+
+    # Note: deleting and then extracting the archive is a bit awkward for updating stable/latest/etc
+    # as the version will be invalid for a window of time. Better to do it atomically, but this is
+    # another thing made much more complex by the directory structure putting language before version...
+
+
+def build_doc_tarball(version, git_ver, build_dir):
+    """ Make a tar.gz archive of the docs, in the directory structure used to deploy as
+        the given version """
+    version_paths = []
+    tarball_path = "{}/{}.tar.gz".format(build_dir, version)
+
+    # find all the 'html/' directories under build_dir
+    html_dirs = glob.glob("{}/**/html/".format(build_dir), recursive=True)
+    print("Found %d html directories" % len(html_dirs))
+
+    pdfs = glob.glob("{}/**/latex/build/*.pdf".format(build_dir), recursive=True)
+    print("Found %d PDFs in latex directories" % len(pdfs))
+
+    # add symlink for stable and latest and adds them to PDF blob
+    symlinks = create_and_add_symlinks(version, git_ver, pdfs)
+
+    def not_sources_dir(ti):
+        """ Filter the _sources directories out of the tarballs """
+        if ti.name.endswith("/_sources"):
+            return None
+
+        ti.mode |= stat.S_IWGRP  # make everything group-writeable
+        return ti
+
+    try:
+        os.remove(tarball_path)
+    except OSError:
+        pass
+
+    with tarfile.open(tarball_path, "w:gz") as tarball:
+        for html_dir in html_dirs:
+            # html_dir has the form '<ignored>/<language>/html/'
+            language_dirname = os.path.dirname(os.path.dirname(html_dir))
+            language = os.path.basename(language_dirname)
+
+            # when deploying, we want the top-level directory layout 'language/version'
+            archive_path = "{}/{}".format(language, version)
+            print("Archiving '{}' as '{}'...".format(html_dir, archive_path))
+            tarball.add(html_dir, archive_path, filter=not_sources_dir)
+            version_paths.append(archive_path)
+
+        for pdf_path in pdfs:
+            # pdf_path has the form '<ignored>/<language>/<target>/latex/build'
+            latex_dirname = os.path.dirname(pdf_path)
+            pdf_filename = os.path.basename(pdf_path)
+            language_dirname = os.path.dirname(os.path.dirname(latex_dirname))
+            language = os.path.basename(language_dirname)
+
+            # when deploying, we want the layout 'language/version/pdf'
+            archive_path = "{}/{}/{}".format(language, version, pdf_filename)
+            print("Archiving '{}' as '{}'...".format(pdf_path, archive_path))
+            tarball.add(pdf_path, archive_path)
+
+    for symlink in symlinks:
+        os.unlink(symlink)
+
+    return (os.path.abspath(tarball_path), version_paths)
+
+
+def create_and_add_symlinks(version, git_ver, pdfs):
+    """ Create symbolic links for PDFs for 'latest' and 'stable' releases """
+
+    symlinks = []
+    if 'stable' in version or 'latest' in version:
+        for pdf_path in pdfs:
+            symlink_path = pdf_path.replace(git_ver, version)
+            os.symlink(pdf_path, symlink_path)
+            symlinks.append(symlink_path)
+
+        pdfs.extend(symlinks)
+        print("Found %d PDFs in latex directories after adding symlink" % len(pdfs))
+
+    return symlinks
+
+
+def is_stable_version(version):
+    """ Heuristic for whether this is the latest stable release """
+    if not version.startswith("v"):
+        return False  # branch name
+    if "-" in version:
+        return False  # prerelease tag
+
+    git_out = subprocess.check_output(["git", "tag", "-l"]).decode("utf-8")
+
+    versions = [v.strip() for v in git_out.split("\n")]
+    versions = [v for v in versions if re.match(r"^v[\d\.]+$", v)]  # include vX.Y.Z only
+
+    versions = [packaging.version.parse(v) for v in versions]
+
+    max_version = max(versions)
+
+    if max_version.public != version[1:]:
+        print("Stable version is v{}. This version is {}.".format(max_version.public, version))
+        return False
+    else:
+        print("This version {} is the stable version".format(version))
+        return True
+
+
+if __name__ == "__main__":
+    main()

tools/ci/executable-list.txt

@@ -40,6 +40,7 @@ tools/ci/check_examples_cmake_make.sh
 tools/ci/check_idf_version.sh
 tools/ci/check_ut_cmake_make.sh
 tools/ci/checkout_project_ref.py
+tools/ci/deploy_docs.py
 tools/ci/envsubst.py
 tools/ci/get-full-sources.sh
 tools/ci/get_supported_examples.sh