新增数据清洗功能

dgtis-modules/dgtis-modules-data/src/main/java/com/dgtis/data/api/CustomerApiController.java

@@ -1277,4 +1277,93 @@ public class CustomerApiController {
 
     }
 
+    /**
+     * 导出数据清洗数据
+     * @param response
+     * @param request
+     */
+    @GetMapping("/dataCleaning")
+    public void dataCleaning(HttpServletResponse response, HttpServletRequest request,String sameType)
+    {
+        logger.info("导出数据清洗进时间："+DateUtils.getTime());
+
+        try{
+
+            Request scriptRequest = new Request("GET", "/shanglifeecif.samecustomer/_search?sort=name:asc&q=sametype:"+sameType+"&size=10000&scroll=5m");
+            Response responseData = restClient.performRequest(scriptRequest);
+            String result = EntityUtils.toString(responseData.getEntity());
+            JSONObject jsonObject = JSON.parseObject(result);
+            JSONObject hitsobject = (JSONObject) jsonObject.get("hits");
+            JSONArray array = JSON.parseArray(hitsobject.get("hits").toString());
+            List<Customer> customers = new ArrayList<Customer>();
+            String scroll_id = jsonObject.getString("_scroll_id");
+            HttpEntity entity = new NStringEntity("{\n" +
+                    "  \"scroll_id\": \""+scroll_id+"\",\n" +
+                    "  \"scroll\": \"5m\"\n" +
+                    "}", ContentType.APPLICATION_JSON);
+            while (array.size() != 0) {
+
+                // 这里填写你的业务逻辑，即对每一条数据的处理
+
+                for (int i = 0; i <array.size() ; i++) {
+                    JSONObject json = (JSONObject)array.get(i);
+                    Map mp = new HashMap();
+                    Customer customer = new Customer();
+                    json.getString("_source");
+                    mp.put("id",json.getString("_id"));
+                    JSONObject jsonOb = JSON.parseObject(json.getString("_source"));
+                    mp.putAll(JSON.parseObject(json.getString("_source"),Map.class));
+                    customer.setScustId("\t"+jsonOb.getString("scustid"));
+                    customer.setName(jsonOb.getString("name"));
+                    customer.setSex("0".equals(jsonOb.getString("gender"))?"男":"女");
+                    if(mp.containsKey("birthday")){
+                        String birthday = jsonOb.getString("birthday");
+                        mp.put("birthday", birthday);
+                        customer.setBirthDay("\t"+birthday);
+                        customer.setAge(DateUtils.getAge(DateUtils.dateTime(DateUtils.YYYY_MM_DD,jsonOb.getString("birthday"))));
+                    }
+
+                    if(mp.containsKey("idcard")){
+                        customer.setIdType("身份证");
+                        customer.setIdCard("\t"+jsonOb.getString("idcard"));
+                    }else if(mp.containsKey("passport")){
+                        customer.setIdType("护照");
+                        customer.setIdCard("\t"+jsonOb.getString("passport"));
+                    }else if(mp.containsKey("dlicense")){
+                        customer.setIdType("驾驶证");
+                        customer.setIdCard("\t"+jsonOb.getString("dlicense"));
+                    }else if(mp.containsKey("otheridnumber")){
+                        customer.setIdType("其它证件类型");
+                        customer.setIdCard("\t"+jsonOb.getString("otheridnumber"));
+                    }
+
+                    customers.add(customer);
+                }
+
+                // 继续向后查询
+                scriptRequest = new Request("GET", "/_search/scroll");
+                scriptRequest.setEntity(entity);
+                responseData = restClient.performRequest(scriptRequest);
+                result = EntityUtils.toString(responseData.getEntity());
+                jsonObject = JSON.parseObject(result);
+                hitsobject = (JSONObject) jsonObject.get("hits");
+                array = JSON.parseArray(hitsobject.get("hits").toString());
+            }
+            Map<String, String> header = new LinkedHashMap<>();
+            header.put("scustId", "核心客户号");
+            header.put("name", "姓名");
+            header.put("idType", "证件类型");
+            header.put("idCard", "证件号码");
+            header.put("sex", "性别");
+            header.put("birthDay", "出生日期");
+            CsvUtil.export(request, response, "customer", header, customers);
+
+            logger.info("导出数据清洗出时间："+DateUtils.getTime());
+        }catch (Exception e){
+            logger.info("导出数据清洗出错："+DateUtils.getTime());
+            e.printStackTrace();
+        }
+
+    }
+
 }