Fixes issues with gen_crt_bundle.py for unicode chars

gen_crt_bundle.py could fail to parse the certificates if it contained a non-ascii character.

components/mbedtls/esp_crt_bundle/gen_crt_bundle.py

@@ -30,6 +30,7 @@ import struct
 import argparse
 import csv
 import re
+from io import open
 
 try:
     from cryptography import x509
@@ -80,7 +81,7 @@ class CertificateBundle:
         try:
             if file_path.endswith('.pem'):
                 status("Parsing certificates from %s" % file_path)
-                with open(file_path, 'r') as f:
+                with open(file_path, 'r', encoding='utf-8') as f:
                     crt_str = f.read()
                     self.add_from_pem(crt_str)
                     return True
@@ -153,7 +154,7 @@ class CertificateBundle:
     def add_with_filter(self, crts_path, filter_path):
 
         filter_set = set()
-        with open(filter_path, 'r') as f:
+        with open(filter_path, 'r', encoding='utf-8') as f:
             csv_reader = csv.reader(f, delimiter=',')
 
             # Skip header
@@ -163,7 +164,7 @@ class CertificateBundle:
 
         status("Parsing certificates from %s" % crts_path)
         crt_str = []
-        with open(crts_path, 'r') as f:
+        with open(crts_path, 'r', encoding='utf-8') as f:
             crt_str = f.read()
 
             # Split all certs into a list of (name, certificate string) tuples

components/mbedtls/esp_crt_bundle/test_gen_crt_bundle/non_ascii_crt.pem

@@ -0,0 +1,23 @@
+NetLock Arany (Class Gold) Főtanúsítvány
+========================================
+-----BEGIN CERTIFICATE-----
+MIIEFTCCAv2gAwIBAgIGSUEs5AAQMA0GCSqGSIb3DQEBCwUAMIGnMQswCQYDVQQGEwJIVTERMA8G
+A1UEBwwIQnVkYXBlc3QxFTATBgNVBAoMDE5ldExvY2sgS2Z0LjE3MDUGA1UECwwuVGFuw7pzw610
+dsOhbnlraWFkw7NrIChDZXJ0aWZpY2F0aW9uIFNlcnZpY2VzKTE1MDMGA1UEAwwsTmV0TG9jayBB
+cmFueSAoQ2xhc3MgR29sZCkgRsWRdGFuw7pzw610dsOhbnkwHhcNMDgxMjExMTUwODIxWhcNMjgx
+MjA2MTUwODIxWjCBpzELMAkGA1UEBhMCSFUxETAPBgNVBAcMCEJ1ZGFwZXN0MRUwEwYDVQQKDAxO
+ZXRMb2NrIEtmdC4xNzA1BgNVBAsMLlRhbsO6c8OtdHbDoW55a2lhZMOzayAoQ2VydGlmaWNhdGlv
+biBTZXJ2aWNlcykxNTAzBgNVBAMMLE5ldExvY2sgQXJhbnkgKENsYXNzIEdvbGQpIEbFkXRhbsO6
+c8OtdHbDoW55MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAxCRec75LbRTDofTjl5Bu
+0jBFHjzuZ9lk4BqKf8owyoPjIMHj9DrTlF8afFttvzBPhCf2nx9JvMaZCpDyD/V/Q4Q3Y1GLeqVw
+/HpYzY6b7cNGbIRwXdrzAZAj/E4wqX7hJ2Pn7WQ8oLjJM2P+FpD/sLj916jAwJRDC7bVWaaeVtAk
+H3B5r9s5VA1lddkVQZQBr17s9o3x/61k/iCa11zr/qYfCGSji3ZVrR47KGAuhyXoqq8fxmRGILdw
+fzzeSNuWU7c5d+Qa4scWhHaXWy+7GRWF+GmF9ZmnqfI0p6m2pgP8b4Y9VHx2BJtr+UBdADTHLpl1
+neWIA6pN+APSQnbAGwIDAKiLo0UwQzASBgNVHRMBAf8ECDAGAQH/AgEEMA4GA1UdDwEB/wQEAwIB
+BjAdBgNVHQ4EFgQUzPpnk/C2uNClwB7zU/2MU9+D15YwDQYJKoZIhvcNAQELBQADggEBAKt/7hwW
+qZw8UQCgwBEIBaeZ5m8BiFRhbvG5GK1Krf6BQCOUL/t1fC8oS2IkgYIL9WHxHG64YTjrgfpioTta
+YtOUZcTh5m2C+C8lcLIhJsFyUR+MLMOEkMNaj7rP9KdlpeuY0fsFskZ1FSNqb4VjMIDw1Z4fKRzC
+bLBQWV2QWzuoDTDPv31/zvGdg73JRm4gpvlhUbohL3u+pRVjodSVh/GeufOJ8z2FuLjbvrW5Kfna
+NwUASZQDhETnv0Mxz3WLJdH0pmT1kvarBes96aULNmLazAZfNou2XjG4Kvte9nHfRCaexOYNkbQu
+dZWAUWpLMKawYqGT8ZvYzsRjdT9ZR7E=
+-----END CERTIFICATE-----

components/mbedtls/esp_crt_bundle/test_gen_crt_bundle/test_gen_crt_bundle.py

@@ -22,6 +22,7 @@ pem_test_file       = 'entrust.pem'
 verified_der_bundle = 'baltimore_crt_bundle'
 verified_pem_bundle = 'entrust_crt_bundle'
 invalid_test_file   = 'invalid_crt.pem'
+non_ascii_file      = 'non_ascii_crt.pem'
 ca_crts_all_file    = 'cacrt_all.pem'
 
 
@@ -72,6 +73,12 @@ class GenCrtBundleTests(Py23TestCase):
         with self.assertRaisesRegex(gen_crt_bundle.InputError, "No certificate found"):
             bundle.add_from_pem("")
 
+    def test_non_ascii_crt_input(self):
+        bundle = gen_crt_bundle.CertificateBundle()
+
+        bundle.add_from_file(test_crts_path  + non_ascii_file)
+        self.assertTrue(len(bundle.certificates))
+
 
 if __name__ == "__main__":
     unittest.main()