#347 add scroll api functionality to requesting logs from OS

Author: kirill-suspitsin-ext-sefe

src/Dorc.OpenSearchData/Sources/DeploymentLogService.cs

@@ -44,35 +44,57 @@ public void EnrichDeploymentResultsWithLogs(IEnumerable<DeploymentResultApiModel
         private IEnumerable<DeployOpenSearchLogModel> GetLogsFromOpenSearch(List<int> requestIds, List<int> deploymentResultIds)
         {
             var logs = new List<DeployOpenSearchLogModel>();
+            const string scrollTimeout = "1m";
 
-            for (int pageNumber = 1; ; pageNumber++)
+            var searchResponse = _openSearchClient.Search<DeployOpenSearchLogModel>(s => s
+                .Index(_deploymentResultIndex)
+                .Query(q => q
+                    .Bool(b => b
+                        .Must(must => must
+                            .Terms(t => t
+                                .Field(field => field.deployment_result_id)
+                                .Terms(deploymentResultIds)),
+                            must => must
+                            .Terms(t => t
+                                .Field(field => field.request_id)
+                                .Terms(requestIds)))))
+                .Size(_pageSize)
+                .Scroll(scrollTimeout)
+            );
+
+            if (!searchResponse.IsValid)
             {
+                _logger.Error($"OpenSearch query exception: {searchResponse.OriginalException?.Message}.{Environment.NewLine}Request information: {searchResponse.DebugInformation}");
+                return logs;
+            }
 
-                var searchResult = _openSearchClient.Search<DeployOpenSearchLogModel>(s => s
-                                    .Index(_deploymentResultIndex)
-                                    .Query(q => q
-                                        .Bool(b => b
-                                            .Must(must => must
-                                                .Terms(t => t
-                                                    .Field(field => field.deployment_result_id)
-                                                    .Terms(deploymentResultIds)),
-                                                must => must
-                                                .Terms(t => t
-                                                    .Field(field => field.request_id)
-                                                    .Terms(requestIds)))))
-                                    .From((pageNumber - 1) * _pageSize)
-                                    .Size(_pageSize));
-
-                if (!searchResult.IsValid)
+            logs.AddRange(searchResponse.Documents);
+
+            var scrollId = searchResponse.ScrollId;
+            while (!string.IsNullOrEmpty(scrollId))
+            {
+                var scrollResponse = _openSearchClient.Scroll<DeployOpenSearchLogModel>(scrollTimeout, scrollId);
+
+                if (!scrollResponse.IsValid)
                 {
-                    _logger.Error($"OpenSearch query exception: {searchResult.OriginalException?.Message}.{Environment.NewLine}Request information: {searchResult.DebugInformation}");
-                    return logs;
+                    _logger.Error($"OpenSearch scroll query exception: {scrollResponse.OriginalException?.Message}.{Environment.NewLine}Request information: {scrollResponse.DebugInformation}");
+                    break;
                 }
 
-                if (searchResult.Documents != null && searchResult.Documents.Any())
-                    logs.AddRange(searchResult.Documents);
+                if (scrollResponse.Documents != null && scrollResponse.Documents.Any())
+                {
+                    logs.AddRange(scrollResponse.Documents);
+                    scrollId = scrollResponse.ScrollId;
+                }
                 else
+                {
                     break;
+                }
+            }
+
+            if (!string.IsNullOrEmpty(scrollId))
+            {
+                _openSearchClient.ClearScroll(c => c.ScrollId(scrollId));
             }
 
             return logs;