zl程序教程

您现在的位置是:首页 >  工具

当前栏目

es 大批量一次性查询所有数据和大批量分页查询数据(使用srcoll scan)

ES数据 查询 所有 分页 一次性 scan 大批量
2023-09-27 14:22:13 时间

一大批量分页查询数据:

  * 统计用户男女数量、总量、骑行次数等指标
     */
    public void userInfoIndicate(String adCode,Long orderStartTime, Long orderEndTime,Long userRegisterStartTime,Long userRegisterEnddTime, UserProfileBo userProfileBo){
        //1.定义存储的变量
    int maleNum = 0;//记录男性个数
    int femaleNum = 0;//记录女性个数
    int otherNum=0;//其他未知个数
    Map<String,Integer>   genderMap=new TreeMap<String,Integer>();//性别统计
    Map<Long, Integer> rideMap = new TreeMap<>();//骑行次数统计
    Map<String,Integer> ageMap=new TreeMap<String, Integer>();//用户年龄段统计
    Map<String,Integer> mobileMap=new TreeMap<String, Integer>();//用户手机品牌统计
    Map<String,Integer> appFromMap=new TreeMap<String, Integer>();//用户来源统计
    Set<Long> userIdSet=new TreeSet<Long>();//用户数量统计
    //2.查询统计数
        // 此为新增客标签的基本查询-----所有的查询依赖于此
    BoolQueryBuilder boolQueryBuilder = createBoolQueryBuilderByNewUser(adCode,orderStartTime,  orderEndTime, userRegisterStartTime,userRegisterEnddTime);
    NativeSearchQueryBuilder nativeSearchQueryBuilder= queryEsSql(boolQueryBuilder);
    SearchQuery searchQuery = nativeSearchQueryBuilder.build();
    int total = getRecordTotal(searchQuery);
    List<String> userList=new ArrayList<String>();
    int pageTotal = total % PAGE_SIZE == 0 ? total / PAGE_SIZE : total / PAGE_SIZE + 1;
    int count=0;
        //3.分页统计
        ScrolledPage<OrderDetailEntity> dataScrollList=null;
        for (int page = 1; page <= pageTotal; page++) {
            BoolQueryBuilder  boolQuery = createBoolQueryBuilderByNewUser(adCode, orderStartTime,  orderEndTime, userRegisterStartTime,userRegisterEnddTime);
            NativeSearchQueryBuilder nsql= queryEsSql(boolQuery);
            nsql.withPageable(PageRequest.of(page-1, PAGE_SIZE ));
            SearchQuery eSql = nsql.build();
            System.out.println("esql:"+eSql.getQuery().toString());
            if(page==1){
                dataScrollList  = (ScrolledPage<OrderDetailEntity>) elasticsearchTemplate.startScroll(SCROLL_TIMEOUT, eSql, OrderDetailEntity.class, searchResultMapper);
            }
            else{
                //取下一页,scrollId在es服务器上可能会发生变化,需要用最新的。发起continueScroll请求会重新刷新快照保留时间
                dataScrollList = (ScrolledPage<OrderDetailEntity>) elasticsearchTemplate.continueScroll(dataScrollList.getScrollId(), SCROLL_TIMEOUT, OrderDetailEntity.class, searchResultMapper);
            }
            List<OrderDetailEntity> dataList=dataScrollList.getContent();
            for(OrderDetailEntity orderDetailEntity:dataList){
              String orderNo=orderDetailEntity.getOrderNo();
                userList.add(orderNo);
                Long userId=orderDetailEntity.getUserId();
                count++;
                if(orderNo.equals("827702323110450273")||orderNo.equals("825030648671322231")){
                    System.out.println("orderNo:"+orderNo+" useriD:"+userId);

                }
                System.out.println("第"+count+"个订单号:"+orderNo);

            }
            System.out.println("第"+page+"页");

        }
        //4.总体封装获取的统计
        System.out.println("订单号:"+userList.size());
       if(dataScrollList!=null){
            //及时释放es服务器资源
            elasticsearchTemplate.clearScroll(dataScrollList.getScrollId());
        }

}

二 大批量一次性查询所有数据:

public List<OrderDetailEntity> queryDataByScrollScan( SearchQuery eSql){
    List<OrderDetailEntity> orderDetailEntitiesList=new ArrayList<OrderDetailEntity>();
    ScrolledPage<OrderDetailEntity> dataScrollList = (ScrolledPage<OrderDetailEntity>) elasticsearchTemplate.startScroll(SCROLL_TIMEOUT, eSql, OrderDetailEntity.class, searchResultMapper);
    System.out.println("查询总命中数:" + dataScrollList.getTotalElements()+">>"+dataScrollList.getContent().size());
    while (dataScrollList.hasContent()) {
        for (OrderDetailEntity dto : dataScrollList.getContent()) {
            //Do your work here
            orderDetailEntitiesList.add(dto);
        }
        //取下一页,scrollId在es服务器上可能会发生变化,需要用最新的。发起continueScroll请求会重新刷新快照保留时间
        dataScrollList = (ScrolledPage<OrderDetailEntity>) elasticsearchTemplate.continueScroll(dataScrollList.getScrollId(), SCROLL_TIMEOUT, OrderDetailEntity.class, searchResultMapper);
    }
    //及时释放es服务器资源
    elasticsearchTemplate.clearScroll(dataScrollList.getScrollId());

     return orderDetailEntitiesList;
}