MVP version of combi-search implemented (#13)

Author: RemyKroese

GenEC/core/analyze.py

@@ -28,6 +28,7 @@ class ConfigOptions(Enum):
 class TextFilterTypes(Enum):
     REGEX = 'Regex'
     POSITIONAL = 'Positional'
+    COMBI_SEARCH = 'Combi-search'
     KEYWORD = 'Keyword_UNSUPPORTED'
     SPLIT_KEYWORDS = 'Split-keywords_UNSUPPORTED'
 
@@ -121,6 +122,15 @@ def request_text_filter(self):
                                       'line': int(self.ask_open_question('Please provide the line number in the cluster: ')),
                                       'occurrence': int(self.ask_open_question('Please provide the occurrence number: '))}
             return positional_text_filter
+        elif self.config.get(ConfigOptions.TEXT_FILTER_TYPE.value) == TextFilterTypes.COMBI_SEARCH.value:
+            combi_search_filters = []
+            index = 1
+            while True:
+                combi_search_filters.append(self.ask_open_question('Please provide a regex filter for search {0}: '.format(index)))
+                index += 1
+                if self.ask_open_question('Do you wish to provide a next search parameter [yes/y]: ').lower() not in YES_INPUT:
+                    break
+            return combi_search_filters
         else:
             raise ValueError('Unsupported filter type: %s' % self.config.get(ConfigOptions.TEXT_FILTER_TYPE.value))
 
@@ -174,6 +184,8 @@ def extract_from_data(self, data, file):
             return self.extract_text_from_clusters_by_regex(clusters)
         elif self.config.get(ConfigOptions.TEXT_FILTER_TYPE.value) == TextFilterTypes.POSITIONAL.value:
             return self.extract_text_from_clusters_by_position(clusters)
+        elif self.config.get(ConfigOptions.TEXT_FILTER_TYPE.value) == TextFilterTypes.COMBI_SEARCH.value:
+            return self.extract_text_from_clusters_by_combi_search(clusters)
         else:
             raise ValueError("Unsupported filter type: %s" % self.config.get(ConfigOptions.TEXT_FILTER_TYPE.value))
 
@@ -205,10 +217,11 @@ def get_sliced_clusters(self, clusters, start_keyword='', end_keyword=''):
 
         return (clusters[start_cluster_index:end_cluster_index+1])
 
-    def extract_text_from_clusters_by_regex(self, clusters):
+    def extract_text_from_clusters_by_regex(self, clusters, regex_pattern=None):
         filtered_text = []
+        pattern = re.compile(regex_pattern if regex_pattern else self.config.get(ConfigOptions.TEXT_FILTER.value))
         for cluster in clusters:
-            search_result = re.search(self.config.get(ConfigOptions.TEXT_FILTER.value), cluster)
+            search_result = pattern.search(cluster)
             if search_result is not None:
                 filtered_text.append(search_result.group(1))
         return filtered_text
@@ -224,6 +237,14 @@ def extract_text_from_clusters_by_position(self, clusters):
                 continue
         return filtered_text
 
+    def extract_text_from_clusters_by_combi_search(self, clusters):
+        '''Combi-search executes multiple user-defined regex searches, isolating only the relevant clusters for the final search'''
+        filters = self.config.get(ConfigOptions.TEXT_FILTER.value)
+        for filter in filters[:-1]:
+            pattern = re.compile(filter)
+            clusters = [cluster for cluster in clusters if pattern.search(cluster)]
+        return self.extract_text_from_clusters_by_regex(clusters, filters[-1])
+
 
 class Comparer:
     def __init__(self, source, reference):

tests/unit_tests/test_extractor.py

@@ -67,6 +67,37 @@ def test_extract_from_data_by_position(mock_extract_text_from_clusters_by_positi
     assert extractor_instance.extract_from_data('', Files.SOURCE.value) == ['my_result']
 
 
+@patch.object(Extractor, 'extract_text_from_clusters_by_combi_search')
+def test_extract_from_data_by_combi_search(mock_extract_text_from_clusters_by_combi_search, extractor_instance):
+    mock_extract_text_from_clusters_by_combi_search.return_value = ['my_result']
+    extractor_instance.config[ConfigOptions.TEXT_FILTER_TYPE.value] = TextFilterTypes.COMBI_SEARCH.value
+    assert extractor_instance.extract_from_data('', Files.SOURCE.value) == ['my_result']
+
+
+@pytest.mark.parametrize(
+    'clusters, regex_filters, expected_filtered_clusters',
+    [
+        # Case 1: Some clusters remain after filtering
+        (['abc123', 'xyz89', 'test456', 'hello111', 'finalTest'],
+         [r'\d{3}', r'test'],
+         ['abc123', 'test456', 'hello111']),
+        # Case 2: More restrictive filtering, leaving only 'test456'
+        (['abc123', 'xyz89', 'test456', 'hello111', 'finalTest'],
+         [r'\d{3}', r'test', r'final'],
+         ['test456']),
+        # Case 3: All clusters are removed by the regex filters, leaving an empty list
+        (['abc123', 'xyz89', 'test456', 'hello111', 'finalTest'],
+         [r'^z.*$', r'test'],  # First filter keeps only words starting with 'z'
+         [])
+    ]
+)
+@patch.object(Extractor, 'extract_text_from_clusters_by_regex')
+def test_extract_text_from_clusters_by_combi_search(mock_extract_text, extractor_instance, clusters, regex_filters, expected_filtered_clusters):
+    extractor_instance.config[ConfigOptions.TEXT_FILTER.value] = regex_filters
+    extractor_instance.extract_text_from_clusters_by_combi_search(clusters)
+    mock_extract_text.assert_called_once_with(expected_filtered_clusters, regex_filters[-1])
+
+
 def test_extract_from_data_unsupported_filter_type(extractor_instance):
     data = 'saiucdjh1\ndusi2hiuw\n3134ferw\n4waijc\ndjhe56fk7\niuaijaudc'
     extractor_instance.config[ConfigOptions.CLUSTER_FILTER.value] = '\n'

tests/unit_tests/test_input_manager.py

@@ -242,6 +242,18 @@ def test_request_POSITIONAL_filter_type(mock_input, mock_side_effect, mock_outpu
     assert im_instance.request_text_filter() == {'separator': mock_output[0], 'line': mock_output[1], 'occurrence': mock_output[2]}
 
 
+@pytest.mark.parametrize('mock_side_effect, mock_output', [
+    (['regex_1', 'y', 'regex_2', 'done'], ['regex_1', 'regex_2']),
+    (['regex_1', 'n'], ['regex_1']),
+    (['regex_1', 'Y', 'regex_2', 'YeS', 'regex_3', ''], ['regex_1', 'regex_2', 'regex_3']),
+])
+@patch.object(InputManager, 'ask_open_question')
+def test_request_COMBI_SEARCH_filter_type(mock_input, mock_side_effect, mock_output, im_instance):
+    mock_input.side_effect = mock_side_effect
+    im_instance.config[ConfigOptions.TEXT_FILTER_TYPE.value] = TextFilterTypes.COMBI_SEARCH.value
+    assert im_instance.request_text_filter() == mock_output
+
+
 @pytest.mark.parametrize('filter_type', [
     (TextFilterTypes.KEYWORD.value),
     (TextFilterTypes.SPLIT_KEYWORDS.value)])