Merge branch 'contrib/github_pr_11821' into 'master'

feat(idf.py): Allow adding arguments from file via @filename.txt (#11783) (GitHub PR)

Closes IDFGH-10584 and IDFGH-10539

See merge request espressif/esp-idf!24955

tools/idf.py

@@ -18,6 +18,7 @@ import json
 import locale
 import os
 import os.path
+import shlex
 import subprocess
 import sys
 from collections import Counter, OrderedDict, _OrderedDictKeysView
@@ -695,7 +696,7 @@ def init_cli(verbose_output: List=None) -> Any:
     return CLI(help=cli_help, verbose_output=verbose_output, all_actions=all_actions)
 
 
-def main() -> None:
+def main(argv: List[Any] = None) -> None:
     # Check the environment only when idf.py is invoked regularly from command line.
     checks_output = None if SHELL_COMPLETE_RUN else check_environment()
 
@@ -713,7 +714,54 @@ def main() -> None:
         else:
             raise
     else:
-        cli(sys.argv[1:], prog_name=PROG, complete_var=SHELL_COMPLETE_VAR)
+        argv = expand_file_arguments(argv or sys.argv[1:])
+
+        cli(argv, prog_name=PROG, complete_var=SHELL_COMPLETE_VAR)
+
+
+def expand_file_arguments(argv: List[Any]) -> List[Any]:
+    """
+    Any argument starting with "@" gets replaced with all values read from a text file.
+    Text file arguments can be split by newline or by space.
+    Values are added "as-is", as if they were specified in this order
+    on the command line.
+    """
+    visited = set()
+    expanded = False
+
+    def expand_args(args: List[Any], parent_path: str, file_stack: List[str]) -> List[str]:
+        expanded_args = []
+        for arg in args:
+            if not arg.startswith('@'):
+                expanded_args.append(arg)
+            else:
+                nonlocal expanded, visited
+                expanded = True
+
+                file_name = arg[1:]
+                rel_path = os.path.normpath(os.path.join(parent_path, file_name))
+
+                if rel_path in visited:
+                    file_stack_str = ' -> '.join(['@' + f for f in file_stack + [file_name]])
+                    raise FatalError(f'Circular dependency in file argument expansion: {file_stack_str}')
+                visited.add(rel_path)
+
+                try:
+                    with open(rel_path, 'r') as f:
+                        for line in f:
+                            expanded_args.extend(expand_args(shlex.split(line), os.path.dirname(rel_path), file_stack + [file_name]))
+                except IOError:
+                    file_stack_str = ' -> '.join(['@' + f for f in file_stack + [file_name]])
+                    raise FatalError(f"File '{rel_path}' (expansion of {file_stack_str}) could not be opened. "
+                                     'Please ensure the file exists and you have the necessary permissions to read it.')
+        return expanded_args
+
+    argv = expand_args(argv, os.getcwd(), [])
+
+    if expanded:
+        print(f'Running: idf.py {" ".join(argv)}')
+
+    return argv
 
 
 def _valid_unicode_config() -> Union[codecs.CodecInfo, bool]:

tools/test_idf_py/file_args_expansion_inputs/args_a

@@ -0,0 +1 @@
+DAAA DBBB

tools/test_idf_py/file_args_expansion_inputs/args_b

@@ -0,0 +1 @@
+DCCC DDDD

tools/test_idf_py/file_args_expansion_inputs/args_circular_a

@@ -0,0 +1 @@
+DAAA @args_circular_b

tools/test_idf_py/file_args_expansion_inputs/args_circular_b

@@ -0,0 +1 @@
+DBBB @args_circular_a

tools/test_idf_py/file_args_expansion_inputs/args_recursive

@@ -0,0 +1 @@
+@args_a DEEE DFFF

tools/test_idf_py/test_idf_py.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 #
-# SPDX-FileCopyrightText: 2019-2022 Espressif Systems (Shanghai) CO LTD
+# SPDX-FileCopyrightText: 2019-2023 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
 
 import json
@@ -291,5 +291,58 @@ class TestROMs(TestWithoutExtensions):
                 self.assertTrue(build_date_str == k['build_date_str'])
 
 
+class TestFileArgumentExpansion(TestCase):
+    def test_file_expansion(self):
+        """Test @filename expansion functionality"""
+        try:
+            output = subprocess.check_output(
+                [sys.executable, idf_py_path, '--version', '@file_args_expansion_inputs/args_a'],
+                env=os.environ,
+                stderr=subprocess.STDOUT).decode('utf-8', 'ignore')
+            self.assertIn('Running: idf.py --version DAAA DBBB', output)
+        except subprocess.CalledProcessError as e:
+            self.fail(f'Process should have exited normally, but it exited with a return code of {e.returncode}')
+
+    def test_multiple_file_arguments(self):
+        """Test multiple @filename arguments"""
+        try:
+            output = subprocess.check_output(
+                [sys.executable, idf_py_path, '--version', '@file_args_expansion_inputs/args_a', '@file_args_expansion_inputs/args_b'],
+                env=os.environ,
+                stderr=subprocess.STDOUT).decode('utf-8', 'ignore')
+            self.assertIn('Running: idf.py --version DAAA DBBB DCCC DDDD', output)
+        except subprocess.CalledProcessError as e:
+            self.fail(f'Process should have exited normally, but it exited with a return code of {e.returncode}')
+
+    def test_recursive_expansion(self):
+        """Test recursive expansion of @filename arguments"""
+        try:
+            output = subprocess.check_output(
+                [sys.executable, idf_py_path, '--version', '@file_args_expansion_inputs/args_recursive'],
+                env=os.environ,
+                stderr=subprocess.STDOUT).decode('utf-8', 'ignore')
+            self.assertIn('Running: idf.py --version DAAA DBBB DEEE DFFF', output)
+        except subprocess.CalledProcessError as e:
+            self.fail(f'Process should have exited normally, but it exited with a return code of {e.returncode}')
+
+    def test_circular_dependency(self):
+        """Test circular dependency detection in file argument expansion"""
+        with self.assertRaises(subprocess.CalledProcessError) as cm:
+            subprocess.check_output(
+                [sys.executable, idf_py_path, '--version', '@file_args_expansion_inputs/args_circular_a'],
+                env=os.environ,
+                stderr=subprocess.STDOUT).decode('utf-8', 'ignore')
+        self.assertIn('Circular dependency in file argument expansion', cm.exception.output.decode('utf-8', 'ignore'))
+
+    def test_missing_file(self):
+        """Test missing file detection in file argument expansion"""
+        with self.assertRaises(subprocess.CalledProcessError) as cm:
+            subprocess.check_output(
+                [sys.executable, idf_py_path, '--version', '@args_non_existent'],
+                env=os.environ,
+                stderr=subprocess.STDOUT).decode('utf-8', 'ignore')
+        self.assertIn('(expansion of @args_non_existent) could not be opened', cm.exception.output.decode('utf-8', 'ignore'))
+
+
 if __name__ == '__main__':
     main()