ci: use more flexible assign for unit test cases

tools/tiny-test-fw/CIAssignUnitTest.py

@@ -47,6 +47,21 @@ class Group(CIAssignTest.Group):
             attr = Group.ATTR_CONVERT_TABLE[attr]
         return case[attr]
 
+    def add_extra_case(self, case):
+        """ If current group contains all tags required by case, then add succeed """
+        added = False
+        if self.accept_new_case():
+            for key in self.filters:
+                if self._get_case_attr(case, key) != self.filters[key]:
+                    if key == "tags":
+                        if self._get_case_attr(case, key).issubset(self.filters[key]):
+                            continue
+                    break
+            else:
+                self.case_list.append(case)
+                added = True
+        return added
+
     def _create_extra_data(self, test_cases, test_function):
         """
         For unit test case, we need to copy some attributes of test cases into config file.
@@ -142,6 +157,8 @@ class UnitTestAssignTest(CIAssignTest.AssignTest):
             with open(test_case_path, "r") as f:
                 raw_data = yaml.load(f, Loader=Loader)
             test_cases = raw_data["test cases"]
+            for case in test_cases:
+                case["tags"] = set(case["tags"])
         except IOError:
             print("Test case path is invalid. Should only happen when use @bot to skip unit test.")
             test_cases = []

tools/tiny-test-fw/Utility/CIAssignTest.py

@@ -105,6 +105,20 @@ class Group(object):
                 added = True
         return added
 
+    def add_extra_case(self, case):
+        """
+        By default (``add_case`` method), cases will only be added when have equal values of all filters with group.
+        But in some cases, we also want to add cases which are not best fit.
+        For example, one group has can run cases require (A, B). It can also accept cases require (A, ) and (B, ).
+        When assign failed by best fit, we will use this method to try if we can assign all failed cases.
+
+        If subclass want to retry, they need to overwrite this method.
+        Logic can be applied to handle such scenario could be different for different cases.
+
+        :return: True if accepted else False
+        """
+        pass
+
     def output(self):
         """
         output data for job configs
@@ -193,6 +207,26 @@ class AssignTest(object):
                 groups.append(self.case_group(case))
         return groups
 
+    def _assign_failed_cases(self, assigned_groups, failed_groups):
+        """ try to assign failed cases to already assigned test groups """
+        still_failed_groups = []
+        failed_cases = []
+        for group in failed_groups:
+            failed_cases.extend(group.case_list)
+        for case in failed_cases:
+            # first try to assign to already assigned groups
+            for group in assigned_groups:
+                if group.add_extra_case(case):
+                    break
+            else:
+                # if failed, group the failed cases
+                for group in still_failed_groups:
+                    if group.add_case(case):
+                        break
+                else:
+                    still_failed_groups.append(self.case_group(case))
+        return still_failed_groups
+
     @staticmethod
     def _apply_bot_filter():
         """
@@ -241,6 +275,7 @@ class AssignTest(object):
         :return: None
         """
         failed_to_assign = []
+        assigned_groups = []
         case_filter = self._apply_bot_filter()
         self.test_cases = self._search_cases(self.test_case_path, case_filter)
         self._apply_bot_test_count()
@@ -250,10 +285,14 @@ class AssignTest(object):
             for job in self.jobs:
                 if job.match_group(group):
                     job.assign_group(group)
+                    assigned_groups.append(group)
                     break
             else:
                 failed_to_assign.append(group)
 
+        if failed_to_assign:
+            failed_to_assign = self._assign_failed_cases(assigned_groups, failed_to_assign)
+
         # print debug info
         # total requirement of current pipeline
         required_group_count = self._count_groups_by_keys(test_groups)