alex/esp-idf
alex/esp-idf
1
0
Fork 0
mirror of https://github.com/espressif/esp-idf.git synced 2024-10-05 20:47:46 -04:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

ci: upload elf map file to s3 server to reduce artifacts size

Browse Source 
f
This commit is contained in:
Fu Hanxi 2022-08-26 12:24:19 +08:00
parent 33b51563dc
commit 1267e826ca
1 changed files with 105 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

169
.gitlab/ci/build.yml
View File

@ -4,7 +4,6 @@
tags: tags:
- build - build
variables: variables:
SIZE_INFO_LOCATION: "$CI_PROJECT_DIR/size_info.txt"
# Enable ccache for all build jobs. See configure_ci_environment.sh for more ccache related settings. # Enable ccache for all build jobs. See configure_ci_environment.sh for more ccache related settings.
IDF_CCACHE_ENABLE: "1" IDF_CCACHE_ENABLE: "1"
after_script: after_script:
@ -12,35 +11,101 @@
- test "$CI_CCACHE_STATS" == 1 && test -n "$(which ccache)" && ccache --show-stats || true - test "$CI_CCACHE_STATS" == 1 && test -n "$(which ccache)" && ccache --show-stats || true
dependencies: [] dependencies: []
.build_pytest_template: .build_cmake_template:
extends: extends:
- .build_template - .build_template
- .before_script_build_jobs - .before_script_build_jobs
dependencies: # set dependencies to null to avoid missing artifacts issue dependencies: # set dependencies to null to avoid missing artifacts issue
needs: needs:
- job: fast_template_app - job: fast_template_app
artifacts: false artifacts: false
variables:
LDGEN_CHECK_MAPPING: 1
artifacts: artifacts:
paths: paths:
- "**/build*/size.json" - "**/build*/size.json"
- "**/build*/build_log.txt" - "**/build*/build_log.txt"
- "**/build*/*.bin" - "**/build*/*.bin"
# upload to s3 server to save the artifacts size
# - "**/build*/*.map"
# ttfw tests require elf files
- "**/build*/*.elf" - "**/build*/*.elf"
- "**/build*/*.map" - "**/build*/flasher_args.json"
- "**/build*/flash_project_args"
- "**/build*/config/sdkconfig.json"
# ttfw tests require sdkconfig file
- "**/build*/sdkconfig"
- "**/build*/bootloader/*.bin"
- "**/build*/partition_table/*.bin"
- list_job_*.json
- size_info.txt
# unit test specific
- components/idf_test/unit_test/*.yml
when: always
expire_in: 4 days
after_script:
# Show ccache statistics if enabled globally
- test "$CI_CCACHE_STATS" == 1 && test -n "$(which ccache)" && ccache --show-stats || true
# upload the binary files to s3 server
- echo -e "\e[0Ksection_start:`date +%s`:upload_binaries_to_s3_server[collapsed=true]\r\e[0KUploading binaries to s3 Server"
- shopt -s globstar
# use || true to bypass the no-file error
- zip ${CI_JOB_ID}.zip **/build*/*.bin || true
- zip ${CI_JOB_ID}.zip **/build*/*.elf || true
- zip ${CI_JOB_ID}.zip **/build*/*.map || true
- zip ${CI_JOB_ID}.zip **/build*/flasher_args.json || true
- zip ${CI_JOB_ID}.zip **/build*/flash_project_args || true
- zip ${CI_JOB_ID}.zip **/build*/config/sdkconfig.json || true
- zip ${CI_JOB_ID}.zip **/build*/sdkconfig || true
- zip ${CI_JOB_ID}.zip **/build*/bootloader/*.bin || true
- zip ${CI_JOB_ID}.zip **/build*/partition_table/*.bin || true
- mc alias set shiny-s3 ${SHINY_S3_SERVER} ${SHINY_S3_ACCESS_KEY} ${SHINY_S3_SECRET_KEY}
- mc cp ${CI_JOB_ID}.zip shiny-s3/idf-artifacts/${CI_PIPELINE_ID}/${CI_JOB_ID}.zip
- echo -e "\e[0Ksection_end:`date +%s`:upload_binaries_to_s3_server\r\e[0K"
- echo "Please download the full binary files (including *.elf and *.map files) from the following share link"
# would be clean up after 4 days
- mc share download shiny-s3/idf-artifacts/${CI_PIPELINE_ID}/${CI_JOB_ID}.zip --expire=96h
script:
# CI specific options start from "--collect-size-info xxx". could ignore when running locally
- run_cmd python tools/ci/ci_build_apps.py $TEST_DIR -v
-t $IDF_TARGET
--copy-sdkconfig
--collect-size-info size_info.txt
--collect-app-info list_job_${CI_NODE_INDEX:-1}.json
--parallel-count ${CI_NODE_TOTAL:-1}
--parallel-index ${CI_NODE_INDEX:-1}
--extra-preserve-dirs
examples/bluetooth/esp_ble_mesh/ble_mesh_console
examples/bluetooth/hci/controller_hci_uart_esp32
examples/wifi/iperf
.build_pytest_template:
extends:
- .build_cmake_template
- .before_script_build_jobs
artifacts:
paths:
- "**/build*/size.json"
- "**/build*/build_log.txt"
- "**/build*/*.bin"
# upload to s3 server to save the artifacts size
# - "**/build*/*.map"
# - "**/build*/*.elf"
- "**/build*/flasher_args.json" - "**/build*/flasher_args.json"
- "**/build*/flash_project_args" - "**/build*/flash_project_args"
- "**/build*/config/sdkconfig.json" - "**/build*/config/sdkconfig.json"
- "**/build*/bootloader/*.bin" - "**/build*/bootloader/*.bin"
- "**/build*/partition_table/*.bin" - "**/build*/partition_table/*.bin"
- $SIZE_INFO_LOCATION - list_job_*.json
- size_info.txt
when: always when: always
expire_in: 3 days expire_in: 4 days
script: script:
# CI specific options start from "--collect-size-info xxx". could ignore when running locally # CI specific options start from "--collect-size-info xxx". could ignore when running locally
- run_cmd python tools/ci/ci_build_apps.py $TEST_DIR -v - run_cmd python tools/ci/ci_build_apps.py $TEST_DIR -v
-t $IDF_TARGET -t $IDF_TARGET
--pytest-apps --pytest-apps
--collect-size-info $SIZE_INFO_LOCATION --collect-size-info size_info.txt
--parallel-count ${CI_NODE_TOTAL:-1} --parallel-count ${CI_NODE_TOTAL:-1}
--parallel-index ${CI_NODE_INDEX:-1} --parallel-index ${CI_NODE_INDEX:-1}
@ -138,14 +203,35 @@ build_non_test_components_apps:
# CI specific options start from "--collect-size-info xxx". could ignore when running locally # CI specific options start from "--collect-size-info xxx". could ignore when running locally
- run_cmd python tools/ci/ci_build_apps.py $COMPONENT_UT_DIRS -v - run_cmd python tools/ci/ci_build_apps.py $COMPONENT_UT_DIRS -v
-t all -t all
--collect-size-info $SIZE_INFO_LOCATION --collect-size-info size_info.txt
--collect-app-info list_job_${CI_NODE_INDEX:-1}.json --collect-app-info list_job_${CI_NODE_INDEX:-1}.json
--parallel-count ${CI_NODE_TOTAL:-1} --parallel-count ${CI_NODE_TOTAL:-1}
--parallel-index ${CI_NODE_INDEX:-1} --parallel-index ${CI_NODE_INDEX:-1}
.build_pytest_test_apps_template:
extends: .build_pytest_template
artifacts:
paths:
- "**/build*/size.json"
- "**/build*/build_log.txt"
- "**/build*/*.bin"
# upload to s3 server to save the artifacts size
# - "**/build*/*.map"
# pytest test apps requires elf files for coredump tests
- "**/build*/*.elf"
- "**/build*/flasher_args.json"
- "**/build*/flash_project_args"
- "**/build*/config/sdkconfig.json"
- "**/build*/bootloader/*.bin"
- "**/build*/partition_table/*.bin"
- list_job_*.json
- size_info.txt
when: always
expire_in: 4 days
build_pytest_test_apps_esp32: build_pytest_test_apps_esp32:
extends: extends:
- .build_pytest_template - .build_pytest_test_apps_template
- .rules:build:custom_test-esp32 - .rules:build:custom_test-esp32
variables: variables:
IDF_TARGET: esp32 IDF_TARGET: esp32
@ -153,7 +239,7 @@ build_pytest_test_apps_esp32:
build_pytest_test_apps_esp32s2: build_pytest_test_apps_esp32s2:
extends: extends:
- .build_pytest_template - .build_pytest_test_apps_template
- .rules:build:custom_test-esp32s2 - .rules:build:custom_test-esp32s2
variables: variables:
IDF_TARGET: esp32s2 IDF_TARGET: esp32s2
@ -161,7 +247,7 @@ build_pytest_test_apps_esp32s2:
build_pytest_test_apps_esp32s3: build_pytest_test_apps_esp32s3:
extends: extends:
- .build_pytest_template - .build_pytest_test_apps_template
- .rules:build:custom_test-esp32s3 - .rules:build:custom_test-esp32s3
variables: variables:
IDF_TARGET: esp32s3 IDF_TARGET: esp32s3
@ -169,7 +255,7 @@ build_pytest_test_apps_esp32s3:
build_pytest_test_apps_esp32c3: build_pytest_test_apps_esp32c3:
extends: extends:
- .build_pytest_template - .build_pytest_test_apps_template
- .rules:build:custom_test-esp32c3 - .rules:build:custom_test-esp32c3
variables: variables:
IDF_TARGET: esp32c3 IDF_TARGET: esp32c3
@ -177,7 +263,7 @@ build_pytest_test_apps_esp32c3:
build_pytest_test_apps_esp32c2: build_pytest_test_apps_esp32c2:
extends: extends:
- .build_pytest_template - .build_pytest_test_apps_template
- .rules:build:custom_test-esp32c2 - .rules:build:custom_test-esp32c2
variables: variables:
IDF_TARGET: esp32c2 IDF_TARGET: esp32c2
@ -268,19 +354,17 @@ build_ssc_esp32s3:
.build_esp_idf_tests_cmake_template: .build_esp_idf_tests_cmake_template:
extends: extends:
- .build_template - .build_cmake_template
- .before_script_build_jobs - .before_script_build_jobs
dependencies: # set dependencies to null to avoid missing artifacts issue
needs:
- job: fast_template_app
artifacts: false
artifacts: artifacts:
paths: paths:
- "**/build*/size.json" - "**/build*/size.json"
- "**/build*/build_log.txt" - "**/build*/build_log.txt"
- "**/build*/*.bin" - "**/build*/*.bin"
# upload to s3 server to save the artifacts size
# - "**/build*/*.map"
# ttfw tests require elf files
- "**/build*/*.elf" - "**/build*/*.elf"
- "**/build*/*.map"
- "**/build*/flasher_args.json" - "**/build*/flasher_args.json"
- "**/build*/flash_project_args" - "**/build*/flash_project_args"
- "**/build*/config/sdkconfig.json" - "**/build*/config/sdkconfig.json"
@ -288,12 +372,10 @@ build_ssc_esp32s3:
- "**/build*/bootloader/*.bin" - "**/build*/bootloader/*.bin"
- "**/build*/partition_table/*.bin" - "**/build*/partition_table/*.bin"
- list_job_*.json - list_job_*.json
- $SIZE_INFO_LOCATION - size_info.txt
- components/idf_test/unit_test/*.yml - components/idf_test/unit_test/*.yml
when: always when: always
expire_in: 4 days expire_in: 4 days
variables:
LDGEN_CHECK_MAPPING: 1
script: script:
# CI specific options start from "--collect-size-info xxx". could ignore when running locally # CI specific options start from "--collect-size-info xxx". could ignore when running locally
- run_cmd python tools/ci/ci_build_apps.py tools/unit-test-app -v - run_cmd python tools/ci/ci_build_apps.py tools/unit-test-app -v
@ -301,7 +383,7 @@ build_ssc_esp32s3:
--config "configs/*=" --config "configs/*="
--copy-sdkconfig --copy-sdkconfig
--preserve-all --preserve-all
--collect-size-info $SIZE_INFO_LOCATION --collect-size-info size_info.txt
--collect-app-info list_job_${CI_NODE_INDEX:-1}.json --collect-app-info list_job_${CI_NODE_INDEX:-1}.json
--parallel-count ${CI_NODE_TOTAL:-1} --parallel-count ${CI_NODE_TOTAL:-1}
--parallel-index ${CI_NODE_INDEX:-1} --parallel-index ${CI_NODE_INDEX:-1}
@ -344,47 +426,6 @@ build_esp_idf_tests_cmake_esp32c3:
variables: variables:
IDF_TARGET: esp32c3 IDF_TARGET: esp32c3
.build_cmake_template:
extends:
- .build_template
- .before_script_build_jobs
dependencies: # set dependencies to null to avoid missing artifacts issue
needs:
- job: fast_template_app
artifacts: false
variables:
LDGEN_CHECK_MAPPING: 1
artifacts:
paths:
- "**/build*/size.json"
- "**/build*/build_log.txt"
- "**/build*/*.bin"
- "**/build*/*.elf"
- "**/build*/*.map"
- "**/build*/flasher_args.json"
- "**/build*/flash_project_args"
- "**/build*/config/sdkconfig.json"
- "**/build*/sdkconfig"
- "**/build*/bootloader/*.bin"
- "**/build*/partition_table/*.bin"
- list_job_*.json
- $SIZE_INFO_LOCATION
when: always
expire_in: 4 days
script:
# CI specific options start from "--collect-size-info xxx". could ignore when running locally
- run_cmd python tools/ci/ci_build_apps.py $TEST_DIR -v
-t $IDF_TARGET
--copy-sdkconfig
--collect-size-info $SIZE_INFO_LOCATION
--collect-app-info list_job_${CI_NODE_INDEX:-1}.json
--parallel-count ${CI_NODE_TOTAL:-1}
--parallel-index ${CI_NODE_INDEX:-1}
--extra-preserve-dirs
examples/bluetooth/esp_ble_mesh/ble_mesh_console
examples/bluetooth/hci/controller_hci_uart_esp32
examples/wifi/iperf
build_examples_cmake_esp32: build_examples_cmake_esp32:
extends: extends:
- .build_cmake_template - .build_cmake_template