build system: Add support for embedded arbitrary binary or text files in .rodata

Simplifies examples of embedding a certificate file or a root cert. This is a much cruder mechanism than the full flash filesystem we want eventually, but still sometimes useful.
2024-10-05 20:47:46 -04:00 · 2016-10-06 12:51:47 +11:00 · 2016-10-06 12:51:47 +11:00 · fce359b240
commit fce359b240
parent 98a0387854
6 changed files with 117 additions and 60 deletions
--- a/docs/build_system.rst
+++ b/docs/build_system.rst
@ -280,6 +280,28 @@ component and resides under the component path. Because logo.h is a
 generated file, it needs to be cleaned when make clean is called which
 why it is added to the COMPONENT_EXTRA_CLEAN variable.

+Embedding Binary Data
+=====================
+
+Sometimes you have a file with some binary or text data that you'd like to make available to your component - but you don't want to reformat the file as C source.
+
+You can set a variable COMPONENT_EMBED_FILES in component.mk, giving the names of the files to embed in this way::
+
+  COMPONENT_EMBED_FILES := server_root_cert.der
+
+Or if the file is a string, you can use the variable COMPONENT_EMBED_TXTFILES. This will embed the contents of the text file as a null-terminated string::
+
+  COMPONENT_EMBED_TXTFILES := server_root_cert.pem
+
+The file's contents will be added to the .rodata section in flash, and are available via symbol names as follows::
+
+  extern const uint8_t server_root_cert_pem_start[] asm("_binary_server_root_cert_pem_start");
+  extern const uint8_t server_root_cert_pem_end[]   asm("_binary_server_root_cert_pem_end");
+
+The names are generated from the full name of the file, as given in COMPONENT_EMBED_FILES. Characters /, ., etc. are replaced with underscores. The _binary prefix in the symbol name is added by objcopy and is the same for both text and binary files.
+
+For an example of using this technique, see examples/04_https_request - the certificate file contents are loaded from the text .pem file at compile time.
+
 Cosmetic Improvements
 =====================

--- a/examples/04_https_request/main/cert.c
+++ b/examples/04_https_request/main/cert.c
@ -1,44 +0,0 @@
-/* This is the CA certificate for the CA trust chain of
-   www.howsmyssl.com in PEM format, as dumped via:
-
-   openssl s_client -showcerts -connect www.howsmyssl.com:443 </dev/null
-
-   The CA cert is the last cert in the chain output by the server.
-*/
-#include <stdio.h>
-#include <stdint.h>
-#include <string.h>
-
-/*
- 1 s:/C=US/O=Let's Encrypt/CN=Let's Encrypt Authority X3
-   i:/O=Digital Signature Trust Co./CN=DST Root CA X3
- */
-const char *server_root_cert = "-----BEGIN CERTIFICATE-----\r\n"
-"MIIEkjCCA3qgAwIBAgIQCgFBQgAAAVOFc2oLheynCDANBgkqhkiG9w0BAQsFADA/\r\n"
-"MSQwIgYDVQQKExtEaWdpdGFsIFNpZ25hdHVyZSBUcnVzdCBDby4xFzAVBgNVBAMT\r\n"
-"DkRTVCBSb290IENBIFgzMB4XDTE2MDMxNzE2NDA0NloXDTIxMDMxNzE2NDA0Nlow\r\n"
-"SjELMAkGA1UEBhMCVVMxFjAUBgNVBAoTDUxldCdzIEVuY3J5cHQxIzAhBgNVBAMT\r\n"
-"GkxldCdzIEVuY3J5cHQgQXV0aG9yaXR5IFgzMIIBIjANBgkqhkiG9w0BAQEFAAOC\r\n"
-"AQ8AMIIBCgKCAQEAnNMM8FrlLke3cl03g7NoYzDq1zUmGSXhvb418XCSL7e4S0EF\r\n"
-"q6meNQhY7LEqxGiHC6PjdeTm86dicbp5gWAf15Gan/PQeGdxyGkOlZHP/uaZ6WA8\r\n"
-"SMx+yk13EiSdRxta67nsHjcAHJyse6cF6s5K671B5TaYucv9bTyWaN8jKkKQDIZ0\r\n"
-"Z8h/pZq4UmEUEz9l6YKHy9v6Dlb2honzhT+Xhq+w3Brvaw2VFn3EK6BlspkENnWA\r\n"
-"a6xK8xuQSXgvopZPKiAlKQTGdMDQMc2PMTiVFrqoM7hD8bEfwzB/onkxEz0tNvjj\r\n"
-"/PIzark5McWvxI0NHWQWM6r6hCm21AvA2H3DkwIDAQABo4IBfTCCAXkwEgYDVR0T\r\n"
-"AQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAYYwfwYIKwYBBQUHAQEEczBxMDIG\r\n"
-"CCsGAQUFBzABhiZodHRwOi8vaXNyZy50cnVzdGlkLm9jc3AuaWRlbnRydXN0LmNv\r\n"
-"bTA7BggrBgEFBQcwAoYvaHR0cDovL2FwcHMuaWRlbnRydXN0LmNvbS9yb290cy9k\r\n"
-"c3Ryb290Y2F4My5wN2MwHwYDVR0jBBgwFoAUxKexpHsscfrb4UuQdf/EFWCFiRAw\r\n"
-"VAYDVR0gBE0wSzAIBgZngQwBAgEwPwYLKwYBBAGC3xMBAQEwMDAuBggrBgEFBQcC\r\n"
-"ARYiaHR0cDovL2Nwcy5yb290LXgxLmxldHNlbmNyeXB0Lm9yZzA8BgNVHR8ENTAz\r\n"
-"MDGgL6AthitodHRwOi8vY3JsLmlkZW50cnVzdC5jb20vRFNUUk9PVENBWDNDUkwu\r\n"
-"Y3JsMB0GA1UdDgQWBBSoSmpjBH3duubRObemRWXv86jsoTANBgkqhkiG9w0BAQsF\r\n"
-"AAOCAQEA3TPXEfNjWDjdGBX7CVW+dla5cEilaUcne8IkCJLxWh9KEik3JHRRHGJo\r\n"
-"uM2VcGfl96S8TihRzZvoroed6ti6WqEBmtzw3Wodatg+VyOeph4EYpr/1wXKtx8/\r\n"
-"wApIvJSwtmVi4MFU5aMqrSDE6ea73Mj2tcMyo5jMd6jmeWUHK8so/joWUoHOUgwu\r\n"
-"X4Po1QYz+3dszkDqMp4fklxBwXRsW10KXzPMTZ+sOPAveyxindmjkW8lGy+QsRlG\r\n"
-"PfZ+G6Z6h7mjem0Y+iWlkYcV4PIWL1iwBi8saCbGS5jN2p8M+X+Q7UNKEkROb3N6\r\n"
-"KOqkqm57TH2H3eDJAkSnh6/DNFu0Qg==\r\n"
-"-----END CERTIFICATE-----\r\n";
-
-
--- a/examples/04_https_request/main/component.mk
+++ b/examples/04_https_request/main/component.mk
@ -1,10 +1,9 @@
 #
 # Main Makefile. This is basically the same as a component makefile.
 #
-# This Makefile should, at the very least, just include $(SDK_PATH)/make/component_common.mk. By default, 
-# this will take the sources in the src/ directory, compile them and link them into 
-# lib(subdirectory_name).a in the build directory. This behaviour is entirely configurable,
-# please read the ESP-IDF documents if you need to do this.
-#
+
+# embed files from the "certs" directory as binary data symbols
+# in the app
+COMPONENT_EMBED_TXTFILES := server_root_cert.pem

 include $(IDF_PATH)/make/component_common.mk
--- a/examples/04_https_request/main/https_request_main.c
+++ b/examples/04_https_request/main/https_request_main.c
@ -74,8 +74,18 @@ static const char *REQUEST = "GET " WEB_URL " HTTP/1.1\n"
    "User-Agent: esp-idf/1.0 esp32\n"
    "\n";

-/* Root cert for howsmyssl.com, found in cert.c */
-extern const char *server_root_cert;
+/* Root cert for howsmyssl.com, taken from server_root_cert.pem
+
+   The PEM file was extracted from the output of this command:
+   openssl s_client -showcerts -connect www.howsmyssl.com:443 </dev/null
+
+   The CA root cert is the last cert given in the chain of certs.
+
+   To embed it in the app binary, the PEM file is named
+   in the component.mk COMPONENT_EMBED_TXTFILES variable.
+*/
+extern const uint8_t server_root_cert_pem_start[] asm("_binary_server_root_cert_pem_start");
+extern const uint8_t server_root_cert_pem_end[]   asm("_binary_server_root_cert_pem_end");

 #ifdef MBEDTLS_DEBUG_C

@ -191,7 +201,9 @@ static void https_get_task(void *pvParameters)

    ESP_LOGI(TAG, "Loading the CA root certificate...");

-    ret = mbedtls_x509_crt_parse(&cacert, (uint8_t*)server_root_cert, strlen(server_root_cert)+1);
+    ret = mbedtls_x509_crt_parse(&cacert, server_root_cert_pem_start,
+                                 server_root_cert_pem_end-server_root_cert_pem_start);
+
    if(ret < 0)
    {
        ESP_LOGE(TAG, "mbedtls_x509_crt_parse returned -0x%x\n\n", -ret);
--- a/examples/04_https_request/main/server_root_cert.pem
+++ b/examples/04_https_request/main/server_root_cert.pem
@ -0,0 +1,27 @@
+-----BEGIN CERTIFICATE-----
+MIIEkjCCA3qgAwIBAgIQCgFBQgAAAVOFc2oLheynCDANBgkqhkiG9w0BAQsFADA/
+MSQwIgYDVQQKExtEaWdpdGFsIFNpZ25hdHVyZSBUcnVzdCBDby4xFzAVBgNVBAMT
+DkRTVCBSb290IENBIFgzMB4XDTE2MDMxNzE2NDA0NloXDTIxMDMxNzE2NDA0Nlow
+SjELMAkGA1UEBhMCVVMxFjAUBgNVBAoTDUxldCdzIEVuY3J5cHQxIzAhBgNVBAMT
+GkxldCdzIEVuY3J5cHQgQXV0aG9yaXR5IFgzMIIBIjANBgkqhkiG9w0BAQEFAAOC
+AQ8AMIIBCgKCAQEAnNMM8FrlLke3cl03g7NoYzDq1zUmGSXhvb418XCSL7e4S0EF
+q6meNQhY7LEqxGiHC6PjdeTm86dicbp5gWAf15Gan/PQeGdxyGkOlZHP/uaZ6WA8
+SMx+yk13EiSdRxta67nsHjcAHJyse6cF6s5K671B5TaYucv9bTyWaN8jKkKQDIZ0
+Z8h/pZq4UmEUEz9l6YKHy9v6Dlb2honzhT+Xhq+w3Brvaw2VFn3EK6BlspkENnWA
+a6xK8xuQSXgvopZPKiAlKQTGdMDQMc2PMTiVFrqoM7hD8bEfwzB/onkxEz0tNvjj
+/PIzark5McWvxI0NHWQWM6r6hCm21AvA2H3DkwIDAQABo4IBfTCCAXkwEgYDVR0T
+AQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAYYwfwYIKwYBBQUHAQEEczBxMDIG
+CCsGAQUFBzABhiZodHRwOi8vaXNyZy50cnVzdGlkLm9jc3AuaWRlbnRydXN0LmNv
+bTA7BggrBgEFBQcwAoYvaHR0cDovL2FwcHMuaWRlbnRydXN0LmNvbS9yb290cy9k
+c3Ryb290Y2F4My5wN2MwHwYDVR0jBBgwFoAUxKexpHsscfrb4UuQdf/EFWCFiRAw
+VAYDVR0gBE0wSzAIBgZngQwBAgEwPwYLKwYBBAGC3xMBAQEwMDAuBggrBgEFBQcC
+ARYiaHR0cDovL2Nwcy5yb290LXgxLmxldHNlbmNyeXB0Lm9yZzA8BgNVHR8ENTAz
+MDGgL6AthitodHRwOi8vY3JsLmlkZW50cnVzdC5jb20vRFNUUk9PVENBWDNDUkwu
+Y3JsMB0GA1UdDgQWBBSoSmpjBH3duubRObemRWXv86jsoTANBgkqhkiG9w0BAQsF
+AAOCAQEA3TPXEfNjWDjdGBX7CVW+dla5cEilaUcne8IkCJLxWh9KEik3JHRRHGJo
+uM2VcGfl96S8TihRzZvoroed6ti6WqEBmtzw3Wodatg+VyOeph4EYpr/1wXKtx8/
+wApIvJSwtmVi4MFU5aMqrSDE6ea73Mj2tcMyo5jMd6jmeWUHK8so/joWUoHOUgwu
+X4Po1QYz+3dszkDqMp4fklxBwXRsW10KXzPMTZ+sOPAveyxindmjkW8lGy+QsRlG
+PfZ+G6Z6h7mjem0Y+iWlkYcV4PIWL1iwBi8saCbGS5jN2p8M+X+Q7UNKEkROb3N6
+KOqkqm57TH2H3eDJAkSnh6/DNFu0Qg==
+-----END CERTIFICATE-----
--- a/make/component_common.mk
+++ b/make/component_common.mk
@ -36,7 +36,10 @@ COMPONENT_LIBRARY := lib$(COMPONENT_NAME).a
 #Source dirs a component has. Default to root directory of component.
 COMPONENT_SRCDIRS ?= .

-#Object files which need to be linked into the library
+#Names of binary files to embed as symbols in the component library
+COMPONENT_EMBED_FILES ?= 
+
+#Object files which need to be added to the library
 #By default we take all .c/.S files in the component directory.
 ifeq ("$(COMPONENT_OBJS)", "")
 #Find all source files in all COMPONENT_SRCDIRS
@ -47,6 +50,10 @@ COMPONENT_OBJS += $(foreach compsrcdir,$(COMPONENT_SRCDIRS),$(patsubst %.S,%.o,$
 COMPONENT_OBJS := $(patsubst $(COMPONENT_PATH)/%,%,$(COMPONENT_OBJS))
 endif

+#Object files with embedded binaries to add to the component library
+#Correspond to the files named in COMPONENT_EMBED_FILES & COMPONENT_EMBED_TXTFILES
+COMPONENT_EMBED_OBJS ?= $(addsuffix .bin.o,$(COMPONENT_EMBED_FILES)) $(addsuffix .txt.o,$(COMPONENT_EMBED_TXTFILES))
+
 #By default, include only the include/ dir.
 COMPONENT_ADD_INCLUDEDIRS ?= include
 COMPONENT_ADD_LDFLAGS ?= -l$(COMPONENT_NAME)
@ -71,16 +78,16 @@ build: $(COMPONENT_LIBRARY)

 #Build the archive. We remove the archive first, otherwise ar will get confused if we update
 #an archive when multiple filenames have the same name (src1/test.o and src2/test.o)
-$(COMPONENT_LIBRARY): $(COMPONENT_OBJS)
+$(COMPONENT_LIBRARY): $(COMPONENT_OBJS) $(COMPONENT_EMBED_OBJS)
 	$(summary) AR $@
 	$(Q) rm -f $@
-	$(Q) $(AR) cru $@ $(COMPONENT_OBJS)
+	$(Q) $(AR) cru $@ $^
 endif

 ifeq ("$(COMPONENT_OWNCLEANTARGET)", "")
 clean:
-	$(summary) RM $(COMPONENT_LIBRARY) $(COMPONENT_OBJS) $(COMPONENT_OBJS:.o=.d) $(COMPONENT_EXTRA_CLEAN)
-	$(Q) rm -f $(COMPONENT_LIBRARY) $(COMPONENT_OBJS) $(COMPONENT_OBJS:.o=.d) $(COMPONENT_EXTRA_CLEAN)
+	$(summary) RM $(COMPONENT_LIBRARY) $(COMPONENT_OBJS) $(COMPONENT_OBJS:.o=.d) $(COMPONENT_EMBED_OBJS) $(COMPONENT_EXTRA_CLEAN)
+	$(Q) rm -f $(COMPONENT_LIBRARY) $(COMPONENT_OBJS) $(COMPONENT_OBJS:.o=.d) $(COMPONENT_EMBED_OBJS) $(COMPONENT_EXTRA_CLEAN)
 endif

 #Include all dependency files already generated
@ -101,10 +108,44 @@ $(1)/%.o: $$(COMPONENT_PATH)/$(1)/%.S | $(1)
 	$$(summary) AS $$@
 	$$(Q) $$(CC) $$(CFLAGS) $(CPPFLAGS) $$(addprefix -I ,$$(COMPONENT_INCLUDES)) $$(addprefix -I ,$$(COMPONENT_EXTRA_INCLUDES)) -I$(1) -c $$< -o $$@

-# CWD is build dir, create the build subdirectory if it doesn't exist
+endef
+
+define GenerateBuildDirTarget
+# CWD is build dir, create the build subdirectory $(1) if it doesn't exist
 $(1):
 	@mkdir -p $(1)
 endef

-#Generate all the compile target recipes
-$(foreach srcdir,$(COMPONENT_SRCDIRS), $(eval $(call GenerateCompileTargets,$(srcdir))))
+#Generate all the compile target recipes & the build directory recipes
+$(foreach srcdir,$(COMPONENT_SRCDIRS), $(eval $(call GenerateCompileTargets,$(srcdir)))  $(eval $(call GenerateBuildDirTarget,$(srcdir))))
+
+## Support for embedding binary files into the ELF as symbols
+
+OBJCOPY_EMBED_ARGS := --input binary --output elf32-xtensa-le --binary-architecture xtensa --rename-section .data=.rodata.embedded
+
+# Generate pattern for embedding text or binary files into the app
+# $(1) is name of file (as relative path inside component)
+# $(2) is txt or bin depending on file contents
+#
+# txt files are null-terminated before being embedded (otherwise
+# identical behaviour.)
+#
+# Files are temporarily copied to the build directory before objcopy,
+# because objcopy generates the symbol name from the full command line
+# path to the input file.
+define GenerateEmbedTarget
+$(1).$(2).o: $$(COMPONENT_PATH)/$(1) | $$(dir $(1))
+	$$(summary) EMBED $$@
+	$$(Q) cp $$< $$(notdir $$<)
+	$$(Q) $(if $(subst bin,,$(2)),echo -ne '\0' >> $$(notdir $$<) )
+	$$(Q) $$(OBJCOPY) $(OBJCOPY_EMBED_ARGS) $$(notdir $$<) $$@
+	$$(Q) rm $$(notdir $$<)
+endef
+
+# generate targets to embed binary & text files
+$(foreach binfile,$(COMPONENT_EMBED_FILES), $(eval $(call GenerateEmbedTarget,$(binfile),bin)))
+
+$(foreach txtfile,$(COMPONENT_EMBED_TXTFILES), $(eval $(call GenerateEmbedTarget,$(txtfile),txt)))
+
+# generate targets to create binary embed directories
+$(foreach bindir,$(sort $(dir $(COMPONENT_EMBED_FILES))), $(eval $(call GenerateBuildDirTarget,$(bindir))))