diff --git a/tools/ci/idf_ci_utils.py b/tools/ci/idf_ci_utils.py
index 42e1673acd..d76468dee6 100644
--- a/tools/ci/idf_ci_utils.py
+++ b/tools/ci/idf_ci_utils.py
@@ -87,8 +87,6 @@ def is_in_directory(file_path: str, folder: str) -> bool:
 
 
 def get_pytest_dirs(folder: str) -> List[str]:
-    from io import StringIO
-
     import pytest
     from _pytest.nodes import Item
 
@@ -102,11 +100,11 @@ def get_pytest_dirs(folder: str) -> List[str]:
 
     collector = CollectPlugin()
 
-    sys_stdout = sys.stdout
-    sys.stdout = StringIO()  # swallow the output
-    pytest.main(['--collect-only', folder], plugins=[collector])
-    sys.stdout = sys_stdout  # restore sys.stdout
+    res = pytest.main(['--collect-only', '-q', folder], plugins=[collector])
+    if res.value != 0:
+        raise RuntimeError('pytest collection failed')
 
+    sys.stdout.flush()  # print instantly
     test_file_paths = set(node.fspath for node in collector.nodes)
 
     return [os.path.dirname(file) for file in test_file_paths]