alex/esp-idf
alex/esp-idf
1
0
Fork 0
mirror of https://github.com/espressif/esp-idf.git synced 2024-10-05 20:47:46 -04:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Merge branch 'bugfix/gen_esp32part' into 'master'

Browse Source 
Partition table gen esp32part fixes: App offset errors, Python 3 support

As mentioned on forums, gen_esp32part.py wasn't erroring correctly if a non-64KB-aligned app partition was referenced.

Also merges a PR to add Python 2&3 support (with some tweaks). Not very useful in IDF right now, but useful for Arduino.

See merge request !751
This commit is contained in:
Angus Gratton 2017-05-16 15:03:11 +08:00
commit 4dad7ab621
3 changed files with 128 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
.gitlab-ci.yml
View File

@ -151,6 +151,15 @@ test_nvs_on_host:
- cd components/nvs_flash/test_nvs_host - cd components/nvs_flash/test_nvs_host
- make test - make test
test_partition_table_on_host:
stage: test
image: $CI_DOCKER_REGISTRY/esp32-ci-env
tags:
- build
script:
- cd components/partition_table/test_gen_esp32part_host
- ./gen_esp32part_tests.py
test_wl_on_host: test_wl_on_host:
stage: test stage: test
image: $CI_DOCKER_REGISTRY/esp32-ci-env image: $CI_DOCKER_REGISTRY/esp32-ci-env

76
components/partition_table/gen_esp32part.py
View File

@ -4,8 +4,23 @@
# #
# Converts partition tables to/from CSV and binary formats. # Converts partition tables to/from CSV and binary formats.
# #
# See http://esp-idf.readthedocs.io/en/latest/partition-tables.html for explanation of # See http://esp-idf.readthedocs.io/en/latest/api-guides/partition-tables.html
# partition table structure and uses. # for explanation of partition table structure and uses.
#
# Copyright 2015-2016 Espressif Systems (Shanghai) PTE LTD
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http:#www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import print_function, division
import argparse import argparse
import os import os
import re import re
@ -80,7 +95,7 @@ class PartitionTable(list):
p.verify() p.verify()
# check for overlaps # check for overlaps
last = None last = None
for p in sorted(self): for p in sorted(self, key=lambda x:x.offset):
if p.offset < 0x5000: if p.offset < 0x5000:
raise InputError("Partition offset 0x%x is below 0x5000" % p.offset) raise InputError("Partition offset 0x%x is below 0x5000" % p.offset)
if last is not None and p.offset < last.offset + last.size: if last is not None and p.offset < last.offset + last.size:
@ -94,16 +109,16 @@ class PartitionTable(list):
data = b[o:o+32] data = b[o:o+32]
if len(data) != 32: if len(data) != 32:
raise InputError("Partition table length must be a multiple of 32 bytes") raise InputError("Partition table length must be a multiple of 32 bytes")
if data == '\xFF'*32: if data == b'\xFF'*32:
return result # got end marker return result # got end marker
result.append(PartitionDefinition.from_binary(data)) result.append(PartitionDefinition.from_binary(data))
raise InputError("Partition table is missing an end-of-table marker") raise InputError("Partition table is missing an end-of-table marker")
def to_binary(self): def to_binary(self):
result = "".join(e.to_binary() for e in self) result = b"".join(e.to_binary() for e in self)
if len(result )>= MAX_PARTITION_LENGTH: if len(result )>= MAX_PARTITION_LENGTH:
raise InputError("Binary partition table length (%d) longer than max" % len(result)) raise InputError("Binary partition table length (%d) longer than max" % len(result))
result += "\xFF" * (MAX_PARTITION_LENGTH - len(result)) # pad the sector, for signing result += b"\xFF" * (MAX_PARTITION_LENGTH - len(result)) # pad the sector, for signing
return result return result
def to_csv(self, simple_formatting=False): def to_csv(self, simple_formatting=False):
@ -137,10 +152,10 @@ class PartitionDefinition(object):
}, },
} }
MAGIC_BYTES = "\xAA\x50" MAGIC_BYTES = b"\xAA\x50"
ALIGNMENT = { ALIGNMENT = {
APP_TYPE : 0x1000, APP_TYPE : 0x10000,
DATA_TYPE : 0x04, DATA_TYPE : 0x04,
} }
@ -226,16 +241,16 @@ class PartitionDefinition(object):
def verify(self): def verify(self):
if self.type is None: if self.type is None:
raise ValidationError("Type field is not set") raise ValidationError(self, "Type field is not set")
if self.subtype is None: if self.subtype is None:
raise ValidationError("Subtype field is not set") raise ValidationError(self, "Subtype field is not set")
if self.offset is None: if self.offset is None:
raise ValidationError("Offset field is not set") raise ValidationError(self, "Offset field is not set")
align = self.ALIGNMENT.get(self.type, 4) align = self.ALIGNMENT.get(self.type, 4)
if self.offset % align: if self.offset % align:
raise ValidationError("%s offset 0x%x is not aligned to 0x%x" % (self.name, self.offset, align)) raise ValidationError(self, "Offset 0x%x is not aligned to 0x%x" % (self.offset, align))
if self.size is None: if self.size is None:
raise ValidationError("Size field is not set") raise ValidationError(self, "Size field is not set")
STRUCT_FORMAT = "<2sBBLL16sL" STRUCT_FORMAT = "<2sBBLL16sL"
@ -246,8 +261,9 @@ class PartitionDefinition(object):
res = cls() res = cls()
(magic, res.type, res.subtype, res.offset, (magic, res.type, res.subtype, res.offset,
res.size, res.name, flags) = struct.unpack(cls.STRUCT_FORMAT, b) res.size, res.name, flags) = struct.unpack(cls.STRUCT_FORMAT, b)
if "\x00" in res.name: # strip null byte padding from name string if b"\x00" in res.name: # strip null byte padding from name string
res.name = res.name[:res.name.index("\x00")] res.name = res.name[:res.name.index(b"\x00")]
res.name = res.name.decode()
if magic != cls.MAGIC_BYTES: if magic != cls.MAGIC_BYTES:
raise InputError("Invalid magic bytes (%r) for partition definition" % magic) raise InputError("Invalid magic bytes (%r) for partition definition" % magic)
for flag,bit in cls.FLAGS.items(): for flag,bit in cls.FLAGS.items():
@ -267,7 +283,7 @@ class PartitionDefinition(object):
self.MAGIC_BYTES, self.MAGIC_BYTES,
self.type, self.subtype, self.type, self.subtype,
self.offset, self.size, self.offset, self.size,
self.name, self.name.encode(),
flags) flags)
def to_csv(self, simple_formatting=False): def to_csv(self, simple_formatting=False):
@ -275,7 +291,7 @@ class PartitionDefinition(object):
if not simple_formatting and include_sizes: if not simple_formatting and include_sizes:
for (val, suffix) in [ (0x100000, "M"), (0x400, "K") ]: for (val, suffix) in [ (0x100000, "M"), (0x400, "K") ]:
if a % val == 0: if a % val == 0:
return "%d%s" % (a / val, suffix) return "%d%s" % (a // val, suffix)
return "0x%x" % a return "0x%x" % a
def lookup_keyword(t, keywords): def lookup_keyword(t, keywords):
@ -295,9 +311,6 @@ class PartitionDefinition(object):
addr_format(self.size, True), addr_format(self.size, True),
generate_text_flags()]) generate_text_flags()])
class InputError(RuntimeError):
def __init__(self, e):
super(InputError, self).__init__(e)
def parse_int(v, keywords={}): def parse_int(v, keywords={}):
"""Generic parser for integer fields - int(x,0) with provision for """Generic parser for integer fields - int(x,0) with provision for
@ -323,7 +336,7 @@ def main():
parser.add_argument('--verify', '-v', help='Verify partition table fields', default=True, action='store_false') parser.add_argument('--verify', '-v', help='Verify partition table fields', default=True, action='store_false')
parser.add_argument('--quiet', '-q', help="Don't print status messages to stderr", action='store_true') parser.add_argument('--quiet', '-q', help="Don't print status messages to stderr", action='store_true')
parser.add_argument('input', help='Path to CSV or binary file to parse. Will use stdin if omitted.', type=argparse.FileType('r'), default=sys.stdin) parser.add_argument('input', help='Path to CSV or binary file to parse. Will use stdin if omitted.', type=argparse.FileType('rb'), default=sys.stdin)
parser.add_argument('output', help='Path to output converted binary or CSV file. Will use stdout if omitted, unless the --display argument is also passed (in which case only the summary is printed.)', parser.add_argument('output', help='Path to output converted binary or CSV file. Will use stdout if omitted, unless the --display argument is also passed (in which case only the summary is printed.)',
nargs='?', nargs='?',
default='-') default='-')
@ -337,6 +350,7 @@ def main():
status("Parsing binary partition input...") status("Parsing binary partition input...")
table = PartitionTable.from_binary(input) table = PartitionTable.from_binary(input)
else: else:
input = input.decode()
status("Parsing CSV input...") status("Parsing CSV input...")
table = PartitionTable.from_csv(input) table = PartitionTable.from_csv(input)
@ -346,14 +360,28 @@ def main():
if input_is_binary: if input_is_binary:
output = table.to_csv() output = table.to_csv()
with sys.stdout if args.output == '-' else open(args.output, 'w') as f:
f.write(output)
else: else:
output = table.to_binary() output = table.to_binary()
with sys.stdout if args.output == '-' else open(args.output, 'w') as f: with sys.stdout.buffer if args.output == '-' else open(args.output, 'wb') as f:
f.write(output) f.write(output)
class InputError(RuntimeError):
def __init__(self, e):
super(InputError, self).__init__(e)
class ValidationError(InputError):
def __init__(self, partition, message):
super(ValidationError, self).__init__(
"Partition %s invalid: %s" % (partition.name, message))
if __name__ == '__main__': if __name__ == '__main__':
try: try:
main() main()
except InputError as e: except InputError as e:
print >>sys.stderr, e print(e, file=sys.stderr)
sys.exit(2) sys.exit(2)

115
components/partition_table/tests/gen_esp32part_tests.py → components/partition_table/test_gen_esp32part_host/gen_esp32part_tests.py
View File

@ -1,4 +1,5 @@
#!/usr/bin/env python #!/usr/bin/env python
from __future__ import print_function, division
import unittest import unittest
import struct import struct
import csv import csv
@ -14,38 +15,40 @@ SIMPLE_CSV = """
factory,0,2,65536,1048576, factory,0,2,65536,1048576,
""" """
LONGER_BINARY_TABLE = "" LONGER_BINARY_TABLE = b""
# type 0x00, subtype 0x00, # type 0x00, subtype 0x00,
# offset 64KB, size 1MB # offset 64KB, size 1MB
LONGER_BINARY_TABLE += "\xAA\x50\x00\x00" + \ LONGER_BINARY_TABLE += b"\xAA\x50\x00\x00" + \
"\x00\x00\x01\x00" + \ b"\x00\x00\x01\x00" + \
"\x00\x00\x10\x00" + \ b"\x00\x00\x10\x00" + \
"factory\0" + ("\0"*8) + \ b"factory\0" + (b"\0"*8) + \
"\x00\x00\x00\x00" b"\x00\x00\x00\x00"
# type 0x01, subtype 0x20, # type 0x01, subtype 0x20,
# offset 0x110000, size 128KB # offset 0x110000, size 128KB
LONGER_BINARY_TABLE += "\xAA\x50\x01\x20" + \ LONGER_BINARY_TABLE += b"\xAA\x50\x01\x20" + \
"\x00\x00\x11\x00" + \ b"\x00\x00\x11\x00" + \
"\x00\x02\x00\x00" + \ b"\x00\x02\x00\x00" + \
"data" + ("\0"*12) + \ b"data" + (b"\0"*12) + \
"\x00\x00\x00\x00" b"\x00\x00\x00\x00"
# type 0x10, subtype 0x00, # type 0x10, subtype 0x00,
# offset 0x150000, size 1MB # offset 0x150000, size 1MB
LONGER_BINARY_TABLE += "\xAA\x50\x10\x00" + \ LONGER_BINARY_TABLE += b"\xAA\x50\x10\x00" + \
"\x00\x00\x15\x00" + \ b"\x00\x00\x15\x00" + \
"\x00\x10\x00\x00" + \ b"\x00\x10\x00\x00" + \
"second" + ("\0"*10) + \ b"second" + (b"\0"*10) + \
"\x00\x00\x00\x00" b"\x00\x00\x00\x00"
LONGER_BINARY_TABLE += "\xFF" * 32 LONGER_BINARY_TABLE += b"\xFF" * 32
def _strip_trailing_ffs(binary_table): def _strip_trailing_ffs(binary_table):
""" """
Strip all FFs down to the last 32 bytes (terminating entry) Strip all FFs down to the last 32 bytes (terminating entry)
""" """
while binary_table.endswith("\xFF"*64): while binary_table.endswith(b"\xFF"*64):
binary_table = binary_table[0:len(binary_table)-32] binary_table = binary_table[0:len(binary_table)-32]
return binary_table return binary_table
class CSVParserTests(unittest.TestCase): class CSVParserTests(unittest.TestCase):
def test_simple_partition(self): def test_simple_partition(self):
@ -108,11 +111,11 @@ myota_status, data, ota,, 0x100000
def test_unit_suffixes(self): def test_unit_suffixes(self):
csv = """ csv = """
# Name, Type, Subtype, Offset, Size # Name, Type, Subtype, Offset, Size
one_megabyte, app, factory, 32k, 1M one_megabyte, app, factory, 64k, 1M
""" """
t = PartitionTable.from_csv(csv) t = PartitionTable.from_csv(csv)
t.verify() t.verify()
self.assertEqual(t[0].offset, 32*1024) self.assertEqual(t[0].offset, 64*1024)
self.assertEqual(t[0].size, 1*1024*1024) self.assertEqual(t[0].size, 1*1024*1024)
def test_default_offsets(self): def test_default_offsets(self):
@ -166,8 +169,8 @@ first, 0x30, 0xEE, 0x100400, 0x300000
t = PartitionTable.from_csv(csv) t = PartitionTable.from_csv(csv)
tb = _strip_trailing_ffs(t.to_binary()) tb = _strip_trailing_ffs(t.to_binary())
self.assertEqual(len(tb), 64) self.assertEqual(len(tb), 64)
self.assertEqual('\xAA\x50', tb[0:2]) # magic self.assertEqual(b'\xAA\x50', tb[0:2]) # magic
self.assertEqual('\x30\xee', tb[2:4]) # type, subtype self.assertEqual(b'\x30\xee', tb[2:4]) # type, subtype
eo, es = struct.unpack("<LL", tb[4:12]) eo, es = struct.unpack("<LL", tb[4:12])
self.assertEqual(eo, 0x100400) # offset self.assertEqual(eo, 0x100400) # offset
self.assertEqual(es, 0x300000) # size self.assertEqual(es, 0x300000) # size
@ -180,8 +183,8 @@ second,0x31, 0xEF, , 0x100000
t = PartitionTable.from_csv(csv) t = PartitionTable.from_csv(csv)
tb = _strip_trailing_ffs(t.to_binary()) tb = _strip_trailing_ffs(t.to_binary())
self.assertEqual(len(tb), 96) self.assertEqual(len(tb), 96)
self.assertEqual('\xAA\x50', tb[0:2]) self.assertEqual(b'\xAA\x50', tb[0:2])
self.assertEqual('\xAA\x50', tb[32:34]) self.assertEqual(b'\xAA\x50', tb[32:34])
def test_encrypted_flag(self): def test_encrypted_flag(self):
@ -200,12 +203,12 @@ class BinaryParserTests(unittest.TestCase):
def test_parse_one_entry(self): def test_parse_one_entry(self):
# type 0x30, subtype 0xee, # type 0x30, subtype 0xee,
# offset 1MB, size 2MB # offset 1MB, size 2MB
entry = "\xAA\x50\x30\xee" + \ entry = b"\xAA\x50\x30\xee" + \
"\x00\x00\x10\x00" + \ b"\x00\x00\x10\x00" + \
"\x00\x00\x20\x00" + \ b"\x00\x00\x20\x00" + \
"0123456789abc\0\0\0" + \ b"0123456789abc\0\0\0" + \
"\x00\x00\x00\x00" + \ b"\x00\x00\x00\x00" + \
"\xFF" * 32 b"\xFF" * 32
# verify that parsing 32 bytes as a table # verify that parsing 32 bytes as a table
# or as a single Definition are the same thing # or as a single Definition are the same thing
t = PartitionTable.from_binary(entry) t = PartitionTable.from_binary(entry)
@ -240,33 +243,36 @@ class BinaryParserTests(unittest.TestCase):
self.assertEqual(round_trip, LONGER_BINARY_TABLE) self.assertEqual(round_trip, LONGER_BINARY_TABLE)
def test_bad_magic(self): def test_bad_magic(self):
bad_magic = "OHAI" + \ bad_magic = b"OHAI" + \
"\x00\x00\x10\x00" + \ b"\x00\x00\x10\x00" + \
"\x00\x00\x20\x00" + \ b"\x00\x00\x20\x00" + \
"0123456789abc\0\0\0" + \ b"0123456789abc\0\0\0" + \
"\x00\x00\x00\x00" b"\x00\x00\x00\x00"
with self.assertRaisesRegexp(InputError, "Invalid magic bytes"): with self.assertRaisesRegexp(InputError, "Invalid magic bytes"):
PartitionTable.from_binary(bad_magic) PartitionTable.from_binary(bad_magic)
def test_bad_length(self): def test_bad_length(self):
bad_length = "OHAI" + \ bad_length = b"OHAI" + \
"\x00\x00\x10\x00" + \ b"\x00\x00\x10\x00" + \
"\x00\x00\x20\x00" + \ b"\x00\x00\x20\x00" + \
"0123456789" b"0123456789"
with self.assertRaisesRegexp(InputError, "32 bytes"): with self.assertRaisesRegexp(InputError, "32 bytes"):
PartitionTable.from_binary(bad_length) PartitionTable.from_binary(bad_length)
class CSVOutputTests(unittest.TestCase): class CSVOutputTests(unittest.TestCase):
def _readcsv(self, source_str):
return list(csv.reader(source_str.split("\n")))
def test_output_simple_formatting(self): def test_output_simple_formatting(self):
table = PartitionTable.from_csv(SIMPLE_CSV) table = PartitionTable.from_csv(SIMPLE_CSV)
as_csv = table.to_csv(True) as_csv = table.to_csv(True)
c = csv.reader(as_csv.split("\n")) c = self._readcsv(as_csv)
# first two lines should start with comments # first two lines should start with comments
self.assertEqual(c.next()[0][0], "#") self.assertEqual(c[0][0][0], "#")
self.assertEqual(c.next()[0][0], "#") self.assertEqual(c[1][0][0], "#")
row = c.next() row = c[2]
self.assertEqual(row[0], "factory") self.assertEqual(row[0], "factory")
self.assertEqual(row[1], "0") self.assertEqual(row[1], "0")
self.assertEqual(row[2], "2") self.assertEqual(row[2], "2")
@ -280,11 +286,11 @@ class CSVOutputTests(unittest.TestCase):
def test_output_smart_formatting(self): def test_output_smart_formatting(self):
table = PartitionTable.from_csv(SIMPLE_CSV) table = PartitionTable.from_csv(SIMPLE_CSV)
as_csv = table.to_csv(False) as_csv = table.to_csv(False)
c = csv.reader(as_csv.split("\n")) c = self._readcsv(as_csv)
# first two lines should start with comments # first two lines should start with comments
self.assertEqual(c.next()[0][0], "#") self.assertEqual(c[0][0][0], "#")
self.assertEqual(c.next()[0][0], "#") self.assertEqual(c[1][0][0], "#")
row = c.next() row = c[2]
self.assertEqual(row[0], "factory") self.assertEqual(row[0], "factory")
self.assertEqual(row[1], "app") self.assertEqual(row[1], "app")
self.assertEqual(row[2], "2") self.assertEqual(row[2], "2")
@ -303,7 +309,7 @@ class CommandLineTests(unittest.TestCase):
csvpath = tempfile.mktemp() csvpath = tempfile.mktemp()
# copy binary contents to temp file # copy binary contents to temp file
with open(binpath, 'w') as f: with open(binpath, 'wb') as f:
f.write(LONGER_BINARY_TABLE) f.write(LONGER_BINARY_TABLE)
# run gen_esp32part.py to convert binary file to CSV # run gen_esp32part.py to convert binary file to CSV
@ -331,5 +337,18 @@ class CommandLineTests(unittest.TestCase):
pass pass
class VerificationTests(unittest.TestCase):
def test_bad_alignment(self):
csv = """
# Name,Type, SubType,Offset,Size
app,app, factory, 32K, 1M
"""
with self.assertRaisesRegexp(ValidationError,
r"Offset.+not aligned"):
t = PartitionTable.from_csv(csv)
t.verify()
if __name__ =="__main__": if __name__ =="__main__":
unittest.main() unittest.main()