Merge branch 'feature/doc_fast_build' into 'master'

docs: add fast build feature and activate it in CI See merge request espressif/esp-idf!13352
2024-10-05 20:47:46 -04:00 · 2021-05-06 03:56:56 +00:00 · 2021-05-06 03:56:56 +00:00 · ca89608f22
commit ca89608f22
parent d7f219686f 41b3043cb4
8 changed files with 116 additions and 67 deletions
--- a/.gitlab/ci/dependencies/dependencies.yml
+++ b/.gitlab/ci/dependencies/dependencies.yml
@ -49,22 +49,6 @@
    - build_system
    - macos

-"build:docs":
-  labels:
-    - docs
-    - build_docs  # for backward compatibility
-  patterns:
-    - docs
-  deploy:
-    - production
-
-"build:docs:label-only":
-  labels:
-    - docs
-    - build_docs  # for backward compatibility
-  deploy:
-    - preview
-
 # ---------------
 # Build Test Jobs
 # ---------------
--- a/.gitlab/ci/docs.yml
+++ b/.gitlab/ci/docs.yml
@ -1,8 +1,45 @@
+.patterns-docs: &patterns-docs
+  - ".gitlab/ci/docs.yml"
+  - "docs/**/*"
+  - "components/**/*.h"
+  - "components/**/Kconfig*"
+  - "components/**/CMakeList.txt"
+  - "components/**/sdkconfig*"
+  - "tools/kconfig_new/**/*"
+  - "CONTRIBUTING.rst"
+
+.if-protected: &if-protected
+  if: '($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_BRANCH =~ /^release\/v/ || $CI_COMMIT_TAG =~ /^v\d+\.\d+(\.\d+)?($|-)/)'
+
+.if-protected-no_label: &if-protected-no_label
+  if: '($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_BRANCH =~ /^release\/v/ || $CI_COMMIT_TAG =~ /^v\d+\.\d+(\.\d+)?($|-)/) && $BOT_TRIGGER_WITH_LABEL == null'
+
+.if-label-build_docs: &if-label-build_docs
+  if: '$BOT_LABEL_BUILD_DOCS || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*build_docs(?:,[^,\n\r]+)*$/i'
+
+.if-label-docs: &if-label-docs
+  if: '$BOT_LABEL_DOCS || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*docs(?:,[^,\n\r]+)*$/i'
+
+.if-label-docs_full: &if-label-docs_full
+  if: '$BOT_LABEL_DOCS_FULL || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*docs_full(?:,[^,\n\r]+)*$/i'
+
+.if-dev-push: &if-dev-push
+  if: '$CI_COMMIT_REF_NAME != "master" && $CI_COMMIT_BRANCH !~ /^release\/v/ && $CI_COMMIT_TAG !~ /^v\d+\.\d+(\.\d+)?($|-)/ && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "merge_request_event")'
+
+.doc-rules:build:docs:
+  rules:
+    - <<: *if-protected
+    - <<: *if-label-build_docs
+    - <<: *if-label-docs
+    - <<: *if-label-docs_full
+    - <<: *if-dev-push
+      changes: *patterns-docs
+
 # stage: pre_check
 check_readme_links:
  extends:
    - .pre_check_job_template
-    - .rules:build:docs
+    - .doc-rules:build:docs
  tags: ["build", "amd64", "internet"]
  allow_failure: true
  script:
@ -11,7 +48,7 @@ check_readme_links:
 check_docs_lang_sync:
  extends:
    - .pre_check_job_template
-    - .rules:build:docs
+    - .doc-rules:build:docs
  script:
    - cd docs
    - ./check_lang_folder_sync.sh
@ -34,7 +71,7 @@ check_docs_gh_links:
  extends:
    - .pre_check_job_template
    - .build_docs_template
-    - .rules:build:docs
+    - .doc-rules:build:docs
  script:
    - cd docs
    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 pip install -r requirements.txt
@ -45,7 +82,6 @@ check_docs_gh_links:
 .build_docs_build_stage_template:
  extends:
    - .build_docs_template
-    - .rules:build:docs
  stage: build_doc
  needs:
    - job: check_docs_lang_sync
@ -53,9 +89,14 @@ check_docs_gh_links:
    - job: check_docs_gh_links
      artifacts: false

-build_docs_html:
+# Doc jobs have a lot of special cases, we specify rules here directly instead
+# in dependencies.yml to simplify things
+build_docs_html_full:
  extends:
    - .build_docs_build_stage_template
+  rules:
+    - <<: *if-protected
+    - <<: *if-label-docs_full
  artifacts:
    when: always
    paths:
@ -65,10 +106,32 @@ build_docs_html:
  variables:
    DOC_BUILDERS: "html"

+build_docs_html_fast:
+  extends:
+    - .build_docs_build_stage_template
+  rules:
+    - <<: *if-label-docs_full
+      when: never
+    - <<: *if-label-build_docs
+    - <<: *if-label-docs
+    - <<: *if-dev-push
+      changes: *patterns-docs
+  artifacts:
+    when: always
+    paths:
+      - docs/_build/*/*/*.txt
+      - docs/_build/*/*/html/*
+    expire_in: 4 days
+  variables:
+    DOC_BUILDERS: "html"
+    DOCS_FAST_BUILD: "yes"
+
 build_docs_pdf:
  extends:
    - .build_docs_build_stage_template
-    - .rules:build:docs:label-only
+  rules:
+    - <<: *if-protected
+    - <<: *if-label-docs_full
  artifacts:
    when: always
    paths:
@ -98,10 +161,12 @@ build_docs_pdf:
 deploy_docs_preview:
  extends:
    - .deploy_docs_template
-    - .rules:build:docs:label-only-preview
-  dependencies:  # set dependencies to null to avoid missing artifacts issue
-  needs:
-    - build_docs_html
+  rules:
+    - <<: *if-label-build_docs
+    - <<: *if-label-docs
+  dependencies:
+    - build_docs_html_fast
+    - build_docs_html_full
    - build_docs_pdf
  variables:
    TYPE: "preview"
@ -117,11 +182,12 @@ deploy_docs_production:
  # The DOCS_PROD_* variables used by this job are "Protected" so these branches must all be marked "Protected" in Gitlab settings
  extends:
    - .deploy_docs_template
-    - .rules:build:docs-production
+  rules:
+    - <<: *if-protected-no_label
  stage: post_deploy
  dependencies:  # set dependencies to null to avoid missing artifacts issue
  needs: # ensure runs after push_to_github succeeded
-    - build_docs_html
+    - build_docs_html_full
    - build_docs_pdf
    - job: push_to_github
      artifacts: false
--- a/.gitlab/ci/rules.yml
+++ b/.gitlab/ci/rules.yml
@ -1,16 +1,6 @@
 ############
 # Patterns #
 ############
-.patterns-docs: &patterns-docs
-  - ".gitlab/ci/docs.yml"
-  - "docs/**/*"
-  - "components/**/*.h"
-  - "components/**/Kconfig*"
-  - "components/**/CMakeList.txt"
-  - "components/**/sdkconfig*"
-  - "tools/kconfig_new/**/*"
-  - "CONTRIBUTING.rst"
-
 .patterns-c-files: &patterns-c-files
  - ".gitlab/ci/static-code-analysis.yml"
  - "tools/ci/static-analysis-rules.yml"
@ -258,9 +248,6 @@
 .if-label-build: &if-label-build
  if: '$BOT_LABEL_BUILD || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*build(?:,[^,\n\r]+)*$/i'

-.if-label-build_docs: &if-label-build_docs
-  if: '$BOT_LABEL_BUILD_DOCS || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*build_docs(?:,[^,\n\r]+)*$/i'
-
 .if-label-component_ut: &if-label-component_ut
  if: '$BOT_LABEL_COMPONENT_UT || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*component_ut(?:,[^,\n\r]+)*$/i'

@ -294,9 +281,6 @@
 .if-label-docker: &if-label-docker
  if: '$BOT_LABEL_DOCKER || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*docker(?:,[^,\n\r]+)*$/i'

-.if-label-docs: &if-label-docs
-  if: '$BOT_LABEL_DOCS || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*docs(?:,[^,\n\r]+)*$/i'
-
 .if-label-example_test: &if-label-example_test
  if: '$BOT_LABEL_EXAMPLE_TEST || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*example_test(?:,[^,\n\r]+)*$/i'

@ -486,29 +470,6 @@
    - <<: *if-dev-push
      changes: *patterns-docker

-.rules:build:docs:
-  rules:
-    - <<: *if-protected
-    - <<: *if-label-build_docs
-    - <<: *if-label-docs
-    - <<: *if-dev-push
-      changes: *patterns-docs
-
-.rules:build:docs-production:
-  rules:
-    - <<: *if-protected-no_label
-
-.rules:build:docs:label-only:
-  rules:
-    - <<: *if-protected
-    - <<: *if-label-build_docs
-    - <<: *if-label-docs
-
-.rules:build:docs:label-only-preview:
-  rules:
-    - <<: *if-label-build_docs
-    - <<: *if-label-docs
-
 .rules:build:example_test-esp32:
  rules:
    - <<: *if-protected
--- a/docs/build_docs.py
+++ b/docs/build_docs.py
@ -100,6 +100,7 @@ def main():

    build_parser = action_parsers.add_parser('build', help='Build documentation')
    build_parser.add_argument('--check-warnings-only', '-w', action='store_true')
+    build_parser.add_argument('--fast-build', '-f', action='store_true', help='Skips including doxygen generated APIs into the Sphinx build')

    action_parsers.add_parser('linkcheck', help='Check links (a current IDF revision should be uploaded to GitHub)')

@ -124,6 +125,11 @@ def main():
    if args.action == 'build' or args.action is None:
        if args.action is None:
            args.check_warnings_only = False
+            args.fast_build = False
+
+        if args.fast_build:
+            os.environ['DOCS_FAST_BUILD'] = 'y'
+
        sys.exit(action_build(args))

    if args.action == 'linkcheck':
--- a/docs/en/contribute/documenting-code.rst
+++ b/docs/en/contribute/documenting-code.rst
@ -513,6 +513,16 @@ As well as wildcards::

 Note that this is a feature intended to simply testing and debugging during writing of documentation. The HTML output won't be perfect, i.e. it will not build a proper index that lists all the documents, and any references to documents that are not built will result in warnings.

+Fast build
+""""""""""
+Another trick to speed up building is to skip including doxygen generated API documention into the Sphinx build process, skipping this drastically reduces build times.
+
+This can be achieved by adding the fast-build argument::
+
+    ./build_docs.py build -f
+
+or by setting the environment variable `DOCS_FAST_BUILD`. Note that the the `-f` argument is a subargument to `build` and thus must be listed after `build`.
+
 Building PDF
 """"""""""""

--- a/docs/en/index.rst
+++ b/docs/en/index.rst
@ -2,6 +2,12 @@ ESP-IDF Programming Guide
 =========================
 :link_to_translation:`zh_CN:[中文]`

+.. only:: fast_build
+
+    .. warning::
+
+        The function API documentation is excluded from the preview. Add docs_full label to the MR to generate the complete documentation.
+
 This is the documentation for Espressif IoT Development Framework (`esp-idf <https://github.com/espressif/esp-idf>`_). ESP-IDF is the official development framework for the `ESP32, ESP32-S and ESP32-C Series SoCs <https://www.espressif.com/en/products/hardware/socs>`_.

 .. only:: html
--- a/docs/idf_extensions/run_doxygen.py
+++ b/docs/idf_extensions/run_doxygen.py
@ -73,6 +73,8 @@ def convert_api_xml_to_inc(app, doxyfiles):
    xml_directory_path = '{}/xml'.format(build_dir)
    inc_directory_path = '{}/inc'.format(build_dir)

+    fast_build = os.environ.get('DOCS_FAST_BUILD', None)
+
    if not os.path.isdir(xml_directory_path):
        raise RuntimeError('Directory {} does not exist!'.format(xml_directory_path))

@ -96,6 +98,14 @@ def convert_api_xml_to_inc(app, doxyfiles):
            with open(inc_file_path, 'w', encoding='utf-8') as inc_file:
                inc_file.write(rst_output)

+        # For fast builds we wipe the doxygen api documention.
+        # Parsing this output during the sphinx build process is
+        # what takes 95% of the build time
+        if fast_build:
+            with open(inc_file_path, 'w', encoding='utf-8') as inc_file:
+                inc_file.write('')
+            app.tags.add('fast_build')
+

 def get_doxyfile_input_paths(app, doxyfile_path):
    """Get contents of Doxyfile's INPUT statement.
--- a/docs/zh_CN/index.rst
+++ b/docs/zh_CN/index.rst
@ -2,6 +2,12 @@ ESP-IDF 编程指南
 =========================
 :link_to_translation:`en:[English]`

+.. only:: fast_build
+
+    .. warning::
+
+        The function API documentation is excluded from the preview. Add docs_full label to the MR to generate the complete documentation.
+
 这里是乐鑫 IoT 开发框架 (`esp-idf <https://github.com/espressif/esp-idf>`_) 的文档中心。ESP-IDF 是 `ESP32、ESP32-S 和 ESP32-C <https://www.espressif.com/en/products/hardware/socs>`_ 系列芯片的官方开发框架。

 .. only:: html