zl程序教程

您现在的位置是:首页 >  工具

当前栏目

es 5.x 版本使用srcoll scan解决数据太大,分页报错的问题

ES数据 解决 报错 版本 分页 scan 问题
2023-09-27 14:22:12 时间
 @Override
    public Page<RealTimeMonitorDomain> querySimlarInfoList(int pageSize, int page, String groupId,String id) {
        //1.初始化条件
        NativeSearchQueryBuilder nsb = new NativeSearchQueryBuilder();//总的查询条件
        BoolQueryBuilder bools = QueryBuilders.boolQuery();
        //2.设置查询
        bools.must(QueryBuilders.termQuery("groupId",groupId));
        bools.mustNot(QueryBuilders.termQuery("id",id));
        nsb.withQuery(bools);
        //3.设置排序
        nsb.withSort(new FieldSortBuilder("pubDate").order(SortOrder.DESC));
        //4.设置分页
        nsb.withPageable(new PageRequest(page-1, pageSize));//pagenum是从0开始,坑爹啊!
        //5.查询的建立
        SearchQuery searchQuery = nsb.build();//查询建立
        logger.info("相似度列表查询sql:"+searchQuery.getQuery().toString());
       // Page<RealTimeMonitorDomain> pageable = elasticsearchTemplate.queryForPage(searchQuery,RealTimeMonitorDomain.class);
        Paging paging=new Paging();
        List<RealTimeMonitorDomain> rtmdList=new ArrayList<RealTimeMonitorDomain>();
        String scrollId = elasticsearchTemplate.scan(searchQuery,1000,false);
        boolean hasRecords = true;
        while (hasRecords) {
            Page<RealTimeMonitorDomain> singlePage = elasticsearchTemplate.scroll(scrollId, 5000L, new SearchResultMapper() {
                @Override
                public <T> AggregatedPage<T> mapResults(SearchResponse response, Class<T> clazz, Pageable pageable) {
                    List<RealTimeMonitorDomain> chunk = new ArrayList<RealTimeMonitorDomain>();
                    for (SearchHit searchHit : response.getHits()) {
                        if (response.getHits().getHits().length <= 0) {
                            return null;
                        }
                        RealTimeMonitorDomain user = new RealTimeMonitorDomain();
                        user.setId(searchHit.getId());
                        user.setTitle(searchHit.getSource().get("message").toString());
                        chunk.add(user);
                    }
                    if (chunk.size() > 0) {
                        return new AggregatedPageImpl<T>((List<T>) chunk);
                    }
                    return null;
                }
            });
            if (singlePage!= null) {
                rtmdList.addAll(singlePage.getContent());
            } else {
                hasRecords = false;
            }
            paging.setTotalCount(singlePage.getTotalElements());
            paging.setTotalPage(singlePage.getTotalPages());
        }
        elasticsearchTemplate.clearScroll(scrollId);
        paging.setPage(page);
        paging.setPageSize(pageSize);
        paging.setResult(rtmdList);
        return pageable;
    }