Skip to content

G
gf_back
Commit d01fe368 authored by tianchao's avatar tianchao
Browse files
Options

1天气数据抽取时，每个小时内的数据，进行递增

parent 61060199
Hide whitespace changes
Inline Side-by-side
Showing with 77 additions and 17 deletions
C09-base/pps-core-base/src/main/java/pps/core/base/service/BaseWatherCloudServiceImpl.java
View file @ d01fe368
...@@ -18,8 +18,10 @@ import pps.core.base.mapper.BasePhotovoltaicPlantViewMapper; ...@@ -18,8 +18,10 @@ import pps.core.base.mapper.BasePhotovoltaicPlantViewMapper;
import pps.core.base.mapper.PlantPredictedPowerDataMapper; import pps.core.base.mapper.PlantPredictedPowerDataMapper;
import pps.core.base.mapper.PlantPredictedPowerDataViewMapper; import pps.core.base.mapper.PlantPredictedPowerDataViewMapper;
import pps.core.base.mapper.WeatherDataMapper; import pps.core.base.mapper.WeatherDataMapper;
import pps.core.base.service.data.base_wather.GetBaseWatherInput;
import pps.core.base.utils.HttpUtils; import pps.core.base.utils.HttpUtils;
import pps.core.common.constant.BusinessConstant; import pps.core.common.constant.BusinessConstant;
import pps.core.common.session.MediaTypes;
import pps.core.common.utils.DateUtil; import pps.core.common.utils.DateUtil;
import xstartup.annotation.XService; import xstartup.annotation.XService;
import xstartup.base.XContext; import xstartup.base.XContext;
...@@ -37,6 +39,7 @@ import java.text.SimpleDateFormat; ...@@ -37,6 +39,7 @@ import java.text.SimpleDateFormat;
import java.time.LocalDate; import java.time.LocalDate;
import java.time.LocalTime; import java.time.LocalTime;
import java.util.*; import java.util.*;
import java.util.stream.Collectors;
@XService @XService
...@@ -120,7 +123,7 @@ public class BaseWatherCloudServiceImpl implements BaseWatherCloudService { ...@@ -120,7 +123,7 @@ public class BaseWatherCloudServiceImpl implements BaseWatherCloudService {
public static void insertData(XContext context,String plantId ,String provinceCode, String cityCode, int runCount){ public static void insertData(XContext context,String plantId ,String provinceCode, String cityCode, int runCount){
String url ="http://weather.cma.cn/web/weather/"+cityCode+".html"; String url ="http://weather.cma.cn/web/weather/"+cityCode+".html";
try{ try{
List<String> timeList = Arrays.asList("00:00" ,"15:00" , "30:00" , "45:00"); List<String> timeList = Arrays.asList("15:00" , "30:00" , "45:00");
PlantPredictedPowerDataMapper mapper= context.getBean(PlantPredictedPowerDataMapper.class); PlantPredictedPowerDataMapper mapper= context.getBean(PlantPredictedPowerDataMapper.class);
Document document = Jsoup.parse(new URL(url), 30000); Document document = Jsoup.parse(new URL(url), 30000);
Elements elements = document.getElementsByClass("mt15"); // 获取class name 是‘mt15’的都有元素对象 Elements elements = document.getElementsByClass("mt15"); // 获取class name 是‘mt15’的都有元素对象
...@@ -230,13 +233,17 @@ public class BaseWatherCloudServiceImpl implements BaseWatherCloudService { ...@@ -230,13 +233,17 @@ public class BaseWatherCloudServiceImpl implements BaseWatherCloudService {
.eq(PlantPredictedPowerDataEnt::getDayTime , dao.getDayTime()) .eq(PlantPredictedPowerDataEnt::getDayTime , dao.getDayTime())
.eq(PlantPredictedPowerDataEnt::getHourTime , dao.getHourTime()); .eq(PlantPredictedPowerDataEnt::getHourTime , dao.getHourTime());
mapper.delete(queryWrapper); mapper.delete(queryWrapper);
for (String time : timeList){ dao.setMinTime("00:00");
dao.setMinTime(time); dao.setDataDate(dao.getYearTime() + "-" + dao.getMonthTime() + "-" + dao.getDayTime() +" " + dao.getHourTime() + ":" + dao.getMinTime() );
dao.setDataDate(dao.getYearTime() + "-" + dao.getMonthTime() + "-" + dao.getDayTime() +" " + dao.getHourTime() + ":" + dao.getMinTime() ); PlantPredictedPowerDataEnt insertDao = XCopyUtils.copyNewObject(dao , PlantPredictedPowerDataEnt.class);
PlantPredictedPowerDataEnt insertDao = XCopyUtils.copyNewObject(dao , PlantPredictedPowerDataEnt.class); allList.add(insertDao);
// mapper.insert(insertDao); // for (String time : timeList){
allList.add(insertDao); // dao.setMinTime(time);
} // dao.setDataDate(dao.getYearTime() + "-" + dao.getMonthTime() + "-" + dao.getDayTime() +" " + dao.getHourTime() + ":" + dao.getMinTime() );
// PlantPredictedPowerDataEnt insertDao = XCopyUtils.copyNewObject(dao , PlantPredictedPowerDataEnt.class);
// // mapper.insert(insertDao);
// allList.add(insertDao);
// }
PlantPredictedPowerDataEnt isExit = objList.stream().filter(item->item.getYearTime().equals(dao.getYearTime())&& PlantPredictedPowerDataEnt isExit = objList.stream().filter(item->item.getYearTime().equals(dao.getYearTime())&&
item.getMonthTime().equals(dao.getMonthTime())&& item.getMonthTime().equals(dao.getMonthTime())&&
item.getDayTime().equals(dao.getDayTime())&& item.getDayTime().equals(dao.getDayTime())&&
...@@ -298,25 +305,71 @@ public class BaseWatherCloudServiceImpl implements BaseWatherCloudService { ...@@ -298,25 +305,71 @@ public class BaseWatherCloudServiceImpl implements BaseWatherCloudService {
.eq(PlantPredictedPowerDataEnt::getDayTime , dao.getDayTime()) .eq(PlantPredictedPowerDataEnt::getDayTime , dao.getDayTime())
.eq(PlantPredictedPowerDataEnt::getHourTime , dao.getHourTime()); .eq(PlantPredictedPowerDataEnt::getHourTime , dao.getHourTime());
mapper.delete(queryWrapper); mapper.delete(queryWrapper);
for (String time : timeList){ dao.setMinTime("00:00");
dao.setMinTime(time); dao.setDataDate(dao.getYearTime() + "-" + dao.getMonthTime() + "-" + dao.getDayTime() +" " + dao.getHourTime() + ":" + dao.getMinTime());
dao.setDataDate(dao.getYearTime() + "-" + dao.getMonthTime() + "-" + dao.getDayTime() +" " + dao.getHourTime() + ":" + dao.getMinTime()); PlantPredictedPowerDataEnt insertDao = XCopyUtils.copyNewObject(dao , PlantPredictedPowerDataEnt.class);
PlantPredictedPowerDataEnt insertDao = XCopyUtils.copyNewObject(dao , PlantPredictedPowerDataEnt.class); allList.add(insertDao);
// mapper.insert(insertDao); // for (String time : timeList){
allList.add(insertDao); // dao.setMinTime(time);
} // dao.setDataDate(dao.getYearTime() + "-" + dao.getMonthTime() + "-" + dao.getDayTime() +" " + dao.getHourTime() + ":" + dao.getMinTime());
// PlantPredictedPowerDataEnt insertDao = XCopyUtils.copyNewObject(dao , PlantPredictedPowerDataEnt.class);
// // mapper.insert(insertDao);
// allList.add(insertDao);
// }
} }
} }
List<PlantPredictedPowerDataEnt> batchList = new ArrayList<>(); List<PlantPredictedPowerDataEnt> batchList = new ArrayList<>();
PlantPredictedPowerDataViewMapper dataViewMapper = context.getBean(PlantPredictedPowerDataViewMapper.class); PlantPredictedPowerDataViewMapper dataViewMapper = context.getBean(PlantPredictedPowerDataViewMapper.class);
for (PlantPredictedPowerDataEnt item :allList ){ List<PlantPredictedPowerDataEnt> sortList = allList.stream().sorted(Comparator.comparing(PlantPredictedPowerDataEnt::getDataDate)).collect(Collectors.toList());
for(int i=0;i < sortList.size();i++){
PlantPredictedPowerDataEnt item = sortList.get(i);
batchList.add(item); batchList.add(item);
if(batchList.size() > 0){ if(i < sortList.size() - 1){
for(int y = 0; y < timeList.size() ; y++) {
PlantPredictedPowerDataEnt dao = XCopyUtils.copyNewObject(item);
dao.setMinTime(timeList.get(y));
dao.setDataDate(dao.getYearTime() + "-" + dao.getMonthTime() + "-" + dao.getDayTime() + " " + dao.getHourTime() + ":" + dao.getMinTime());
int I_i=i+1;
BigDecimal temperature_1 = sortList.get(i).getTemperature() == null ? new BigDecimal(0) : sortList.get(i).getTemperature();
BigDecimal temperature_2 = sortList.get(I_i).getTemperature() == null ? new BigDecimal(0) : sortList.get(I_i).getTemperature();
dao.setTemperature(temperature_1.add(temperature_2.subtract(temperature_1).multiply(new BigDecimal(0.3*(y+1)))));
BigDecimal humidity_1 = sortList.get(i).getHumidity() == null ? new BigDecimal(0) : sortList.get(i).getHumidity();
BigDecimal humidity_2 = sortList.get(I_i).getHumidity() == null ? new BigDecimal(0) : sortList.get(I_i).getHumidity();
dao.setHumidity(humidity_1.add(humidity_2.subtract(humidity_1).multiply(new BigDecimal(0.3*(y+1)))));
BigDecimal windSpeed_1 = sortList.get(i).getWindSpeed() == null ? new BigDecimal(0) : sortList.get(i).getWindSpeed();
BigDecimal windSpeed_2 = sortList.get(I_i).getWindSpeed() == null ? new BigDecimal(0) : sortList.get(I_i).getWindSpeed();
dao.setWindSpeed(windSpeed_1.add(windSpeed_2.subtract(windSpeed_1).multiply(new BigDecimal(0.3*(y+1)))));
BigDecimal pressure_1 = sortList.get(i).getPressure() == null ? new BigDecimal(0) : sortList.get(i).getPressure();
BigDecimal pressure_2 = sortList.get(I_i).getPressure() == null ? new BigDecimal(0) : sortList.get(I_i).getPressure();
dao.setPressure(pressure_1.add(pressure_2.subtract(pressure_1).multiply(new BigDecimal(0.3*(y+1)))));
BigDecimal planeIrradiance_1 = sortList.get(i).getPlaneIrradiance() == null ? new BigDecimal(0) : sortList.get(i).getPlaneIrradiance();
BigDecimal planeIrradiance_2 = sortList.get(I_i).getPlaneIrradiance() == null ? new BigDecimal(0) : sortList.get(I_i).getPlaneIrradiance();
dao.setPlaneIrradiance(planeIrradiance_1.add(planeIrradiance_2.subtract(planeIrradiance_1).multiply(new BigDecimal(0.3*(y+1)))));
BigDecimal horizontalIrradiance_1 = sortList.get(i).getHorizontalIrradiance() == null ? new BigDecimal(0) : sortList.get(i).getHorizontalIrradiance();
BigDecimal horizontalIrradiance_2 = sortList.get(I_i).getHorizontalIrradiance() == null ? new BigDecimal(0) : sortList.get(I_i).getHorizontalIrradiance();
dao.setHorizontalIrradiance(horizontalIrradiance_1.add(horizontalIrradiance_2.subtract(horizontalIrradiance_1).multiply(new BigDecimal(0.3*(y+1)))));
BigDecimal power_1 = sortList.get(i).getPower() == null ? new BigDecimal(0) : sortList.get(i).getPower();
BigDecimal power_2 = sortList.get(I_i).getPower() == null ? new BigDecimal(0) : sortList.get(I_i).getPower();
dao.setPower(power_1.add(power_2.subtract(power_1).multiply(new BigDecimal(0.3*(y+1)))));
batchList.add(dao);
}
}
if(batchList.size() > 500){
dataViewMapper.insertBatch(batchList); dataViewMapper.insertBatch(batchList);
batchList = new ArrayList<>(); batchList = new ArrayList<>();
} }
} }
if(batchList.size() > 0)
dataViewMapper.insertBatch(batchList);
// for (PlantPredictedPowerDataEnt item :allList ){
//
// batchList.add(item);
// if(batchList.size() > 0){
// dataViewMapper.insertBatch(batchList);
// batchList = new ArrayList<>();
// }
// }
//训练接口http://127.0.0.1:10098/aiprediction/xgbtrain?plantId=018a64635ac47cf58895147b0e1bf7e3 //训练接口http://127.0.0.1:10098/aiprediction/xgbtrain?plantId=018a64635ac47cf58895147b0e1bf7e3
//自动调用预测接口 //自动调用预测接口
HttpUtils.send2("http://127.0.0.1:10098/aiprediction/xgbreason?plantId=" + plantId, HttpUtils.send2("http://127.0.0.1:10098/aiprediction/xgbreason?plantId=" + plantId,
...@@ -519,4 +572,11 @@ public class BaseWatherCloudServiceImpl implements BaseWatherCloudService { ...@@ -519,4 +572,11 @@ public class BaseWatherCloudServiceImpl implements BaseWatherCloudService {
context.getLogger().error(e.getMessage()); context.getLogger().error(e.getMessage());
} }
} }
@XApiAnonymous
@XApiGet
public XServiceResult crawlPlantBaseWatherDataTest(XContext context){
crawlPlantBaseWatherData(context );
return XServiceResult.OK;
}
} }
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment