Merge branch 'feature/doc_fast_build' into 'master'

docs: add fast build feature and activate it in CI

See merge request espressif/esp-idf!13352

.gitlab/ci/dependencies/dependencies.yml

@@ -49,22 +49,6 @@
     - build_system
     - macos
 
-"build:docs":
-  labels:
-    - docs
-    - build_docs  # for backward compatibility
-  patterns:
-    - docs
-  deploy:
-    - production
-
-"build:docs:label-only":
-  labels:
-    - docs
-    - build_docs  # for backward compatibility
-  deploy:
-    - preview
-
 # ---------------
 # Build Test Jobs
 # ---------------

.gitlab/ci/docs.yml

@@ -1,8 +1,45 @@
+.patterns-docs: &patterns-docs
+  - ".gitlab/ci/docs.yml"
+  - "docs/**/*"
+  - "components/**/*.h"
+  - "components/**/Kconfig*"
+  - "components/**/CMakeList.txt"
+  - "components/**/sdkconfig*"
+  - "tools/kconfig_new/**/*"
+  - "CONTRIBUTING.rst"
+
+.if-protected: &if-protected
+  if: '($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_BRANCH =~ /^release\/v/ || $CI_COMMIT_TAG =~ /^v\d+\.\d+(\.\d+)?($|-)/)'
+
+.if-protected-no_label: &if-protected-no_label
+  if: '($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_BRANCH =~ /^release\/v/ || $CI_COMMIT_TAG =~ /^v\d+\.\d+(\.\d+)?($|-)/) && $BOT_TRIGGER_WITH_LABEL == null'
+
+.if-label-build_docs: &if-label-build_docs
+  if: '$BOT_LABEL_BUILD_DOCS || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*build_docs(?:,[^,\n\r]+)*$/i'
+
+.if-label-docs: &if-label-docs
+  if: '$BOT_LABEL_DOCS || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*docs(?:,[^,\n\r]+)*$/i'
+
+.if-label-docs_full: &if-label-docs_full
+  if: '$BOT_LABEL_DOCS_FULL || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*docs_full(?:,[^,\n\r]+)*$/i'
+
+.if-dev-push: &if-dev-push
+  if: '$CI_COMMIT_REF_NAME != "master" && $CI_COMMIT_BRANCH !~ /^release\/v/ && $CI_COMMIT_TAG !~ /^v\d+\.\d+(\.\d+)?($|-)/ && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "merge_request_event")'
+
+.doc-rules:build:docs:
+  rules:
+    - <<: *if-protected
+    - <<: *if-label-build_docs
+    - <<: *if-label-docs
+    - <<: *if-label-docs_full
+    - <<: *if-dev-push
+      changes: *patterns-docs
+
 # stage: pre_check
 check_readme_links:
   extends:
     - .pre_check_job_template
-    - .rules:build:docs
+    - .doc-rules:build:docs
   tags: ["build", "amd64", "internet"]
   allow_failure: true
   script:
@@ -11,7 +48,7 @@ check_readme_links:
 check_docs_lang_sync:
   extends:
     - .pre_check_job_template
-    - .rules:build:docs
+    - .doc-rules:build:docs
   script:
     - cd docs
     - ./check_lang_folder_sync.sh
@@ -34,7 +71,7 @@ check_docs_gh_links:
   extends:
     - .pre_check_job_template
     - .build_docs_template
-    - .rules:build:docs
+    - .doc-rules:build:docs
   script:
     - cd docs
     - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 pip install -r requirements.txt
@@ -45,7 +82,6 @@ check_docs_gh_links:
 .build_docs_build_stage_template:
   extends:
     - .build_docs_template
-    - .rules:build:docs
   stage: build_doc
   needs:
     - job: check_docs_lang_sync
@@ -53,9 +89,14 @@ check_docs_gh_links:
     - job: check_docs_gh_links
       artifacts: false
 
-build_docs_html:
+# Doc jobs have a lot of special cases, we specify rules here directly instead
+# in dependencies.yml to simplify things
+build_docs_html_full:
   extends:
     - .build_docs_build_stage_template
+  rules:
+    - <<: *if-protected
+    - <<: *if-label-docs_full
   artifacts:
     when: always
     paths:
@@ -65,10 +106,32 @@ build_docs_html:
   variables:
     DOC_BUILDERS: "html"
 
+build_docs_html_fast:
+  extends:
+    - .build_docs_build_stage_template
+  rules:
+    - <<: *if-label-docs_full
+      when: never
+    - <<: *if-label-build_docs
+    - <<: *if-label-docs
+    - <<: *if-dev-push
+      changes: *patterns-docs
+  artifacts:
+    when: always
+    paths:
+      - docs/_build/*/*/*.txt
+      - docs/_build/*/*/html/*
+    expire_in: 4 days
+  variables:
+    DOC_BUILDERS: "html"
+    DOCS_FAST_BUILD: "yes"
+
 build_docs_pdf:
   extends:
     - .build_docs_build_stage_template
-    - .rules:build:docs:label-only
+  rules:
+    - <<: *if-protected
+    - <<: *if-label-docs_full
   artifacts:
     when: always
     paths:
@@ -98,10 +161,12 @@ build_docs_pdf:
 deploy_docs_preview:
   extends:
     - .deploy_docs_template
-    - .rules:build:docs:label-only-preview
+  rules:
-  dependencies:  # set dependencies to null to avoid missing artifacts issue
+    - <<: *if-label-build_docs
-  needs:
+    - <<: *if-label-docs
-    - build_docs_html
+  dependencies:
+    - build_docs_html_fast
+    - build_docs_html_full
     - build_docs_pdf
   variables:
     TYPE: "preview"
@@ -117,11 +182,12 @@ deploy_docs_production:
   # The DOCS_PROD_* variables used by this job are "Protected" so these branches must all be marked "Protected" in Gitlab settings
   extends:
     - .deploy_docs_template
-    - .rules:build:docs-production
+  rules:
+    - <<: *if-protected-no_label
   stage: post_deploy
   dependencies:  # set dependencies to null to avoid missing artifacts issue
   needs: # ensure runs after push_to_github succeeded
-    - build_docs_html
+    - build_docs_html_full
     - build_docs_pdf
     - job: push_to_github
       artifacts: false

.gitlab/ci/rules.yml

@@ -1,16 +1,6 @@
 ############
 # Patterns #
 ############
-.patterns-docs: &patterns-docs
-  - ".gitlab/ci/docs.yml"
-  - "docs/**/*"
-  - "components/**/*.h"
-  - "components/**/Kconfig*"
-  - "components/**/CMakeList.txt"
-  - "components/**/sdkconfig*"
-  - "tools/kconfig_new/**/*"
-  - "CONTRIBUTING.rst"
-
 .patterns-c-files: &patterns-c-files
   - ".gitlab/ci/static-code-analysis.yml"
   - "tools/ci/static-analysis-rules.yml"
@@ -258,9 +248,6 @@
 .if-label-build: &if-label-build
   if: '$BOT_LABEL_BUILD || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*build(?:,[^,\n\r]+)*$/i'
 
-.if-label-build_docs: &if-label-build_docs
-  if: '$BOT_LABEL_BUILD_DOCS || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*build_docs(?:,[^,\n\r]+)*$/i'
-
 .if-label-component_ut: &if-label-component_ut
   if: '$BOT_LABEL_COMPONENT_UT || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*component_ut(?:,[^,\n\r]+)*$/i'
 
@@ -294,9 +281,6 @@
 .if-label-docker: &if-label-docker
   if: '$BOT_LABEL_DOCKER || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*docker(?:,[^,\n\r]+)*$/i'
 
-.if-label-docs: &if-label-docs
-  if: '$BOT_LABEL_DOCS || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*docs(?:,[^,\n\r]+)*$/i'
-
 .if-label-example_test: &if-label-example_test
   if: '$BOT_LABEL_EXAMPLE_TEST || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*example_test(?:,[^,\n\r]+)*$/i'
 
@@ -486,29 +470,6 @@
     - <<: *if-dev-push
       changes: *patterns-docker
 
-.rules:build:docs:
-  rules:
-    - <<: *if-protected
-    - <<: *if-label-build_docs
-    - <<: *if-label-docs
-    - <<: *if-dev-push
-      changes: *patterns-docs
-
-.rules:build:docs-production:
-  rules:
-    - <<: *if-protected-no_label
-
-.rules:build:docs:label-only:
-  rules:
-    - <<: *if-protected
-    - <<: *if-label-build_docs
-    - <<: *if-label-docs
-
-.rules:build:docs:label-only-preview:
-  rules:
-    - <<: *if-label-build_docs
-    - <<: *if-label-docs
-
 .rules:build:example_test-esp32:
   rules:
     - <<: *if-protected

docs/build_docs.py

@@ -100,6 +100,7 @@ def main():
 
     build_parser = action_parsers.add_parser('build', help='Build documentation')
     build_parser.add_argument('--check-warnings-only', '-w', action='store_true')
+    build_parser.add_argument('--fast-build', '-f', action='store_true', help='Skips including doxygen generated APIs into the Sphinx build')
 
     action_parsers.add_parser('linkcheck', help='Check links (a current IDF revision should be uploaded to GitHub)')
 
@@ -124,6 +125,11 @@ def main():
     if args.action == 'build' or args.action is None:
         if args.action is None:
             args.check_warnings_only = False
+            args.fast_build = False
+
+        if args.fast_build:
+            os.environ['DOCS_FAST_BUILD'] = 'y'
+
         sys.exit(action_build(args))
 
     if args.action == 'linkcheck':

docs/en/contribute/documenting-code.rst

@@ -513,6 +513,16 @@ As well as wildcards::
 
 Note that this is a feature intended to simply testing and debugging during writing of documentation. The HTML output won't be perfect, i.e. it will not build a proper index that lists all the documents, and any references to documents that are not built will result in warnings.
 
+Fast build
+""""""""""
+Another trick to speed up building is to skip including doxygen generated API documention into the Sphinx build process, skipping this drastically reduces build times.
+
+This can be achieved by adding the fast-build argument::
+
+    ./build_docs.py build -f
+
+or by setting the environment variable `DOCS_FAST_BUILD`. Note that the the `-f` argument is a subargument to `build` and thus must be listed after `build`.
+
 Building PDF
 """"""""""""
 

docs/en/index.rst

@@ -2,6 +2,12 @@ ESP-IDF Programming Guide
 =========================
 :link_to_translation:`zh_CN:[中文]`
 
+.. only:: fast_build
+
+    .. warning::
+
+        The function API documentation is excluded from the preview. Add docs_full label to the MR to generate the complete documentation.
+
 This is the documentation for Espressif IoT Development Framework (`esp-idf <https://github.com/espressif/esp-idf>`_). ESP-IDF is the official development framework for the `ESP32, ESP32-S and ESP32-C Series SoCs <https://www.espressif.com/en/products/hardware/socs>`_.
 
 .. only:: html

docs/idf_extensions/run_doxygen.py

@@ -73,6 +73,8 @@ def convert_api_xml_to_inc(app, doxyfiles):
     xml_directory_path = '{}/xml'.format(build_dir)
     inc_directory_path = '{}/inc'.format(build_dir)
 
+    fast_build = os.environ.get('DOCS_FAST_BUILD', None)
+
     if not os.path.isdir(xml_directory_path):
         raise RuntimeError('Directory {} does not exist!'.format(xml_directory_path))
 
@@ -96,6 +98,14 @@ def convert_api_xml_to_inc(app, doxyfiles):
             with open(inc_file_path, 'w', encoding='utf-8') as inc_file:
                 inc_file.write(rst_output)
 
+        # For fast builds we wipe the doxygen api documention.
+        # Parsing this output during the sphinx build process is
+        # what takes 95% of the build time
+        if fast_build:
+            with open(inc_file_path, 'w', encoding='utf-8') as inc_file:
+                inc_file.write('')
+            app.tags.add('fast_build')
+
 
 def get_doxyfile_input_paths(app, doxyfile_path):
     """Get contents of Doxyfile's INPUT statement.

docs/zh_CN/index.rst

@@ -2,6 +2,12 @@ ESP-IDF 编程指南
 =========================
 :link_to_translation:`en:[English]`
 
+.. only:: fast_build
+
+    .. warning::
+
+        The function API documentation is excluded from the preview. Add docs_full label to the MR to generate the complete documentation.
+
 这里是乐鑫 IoT 开发框架 (`esp-idf <https://github.com/espressif/esp-idf>`_) 的文档中心。ESP-IDF 是 `ESP32、ESP32-S 和 ESP32-C <https://www.espressif.com/en/products/hardware/socs>`_ 系列芯片的官方开发框架。
 
 .. only:: html