Skip to content

S
smart_gov_platform
Commit 6a0f4015 authored by 赵啸非's avatar 赵啸非
Browse files
Options

添加部门事项过滤

parent ee2a0be6
Hide whitespace changes
Inline Side-by-side
Showing with 104 additions and 30 deletions
base-manager/src/main/java/com/mortals/xhx/common/utils/MatterHtmlParseUtil.java
View file @ 6a0f4015
......@@ -82,6 +82,9 @@ public class MatterHtmlParseUtil {
String matterListExp = "//div[@class=\"sx_list\"]//span[1]";
String matterListLiExp = "//div[@class=\"sx_list\"]//li/a[1]";
List<MatterEntity> matterEntityList = new ArrayList<>();
String evaluationUrl = "";
String netApplyUrl = "";
String href ="";
try {
Document dom = Jsoup.connect(url).data(params).get();
//System.out.println(dom.html());
......@@ -93,21 +96,29 @@ public class MatterHtmlParseUtil {
continue;
}
String title = element.attr("title");
String href = element.firstElementChild().attr("href");
// String href = element.firstElementChild().attr("href");
//element.child()
if (href.equalsIgnoreCase("javascript:void(0)")) {
/* if (href.equalsIgnoreCase("javascript:void(0)")) {
continue;
}
}*/
//抓取申请与评价页面地址
Element nextElementSibling = element.nextElementSibling();
String evaluationUrl = "";
String netApplyUrl = "";
Elements elementsA = nextElementSibling.selectXpath("//a");
if (elementsA != null) {
for (Element tempElement : elementsA) {
if ("办事指南".equals(tempElement.text().trim())) {
String onclick = tempElement.attr("onclick");
href = StrUtil.subBetween(onclick, "ywblurl('", "')");
}
if ("办事指南".equals(tempElement.text().trim())) {
String onclick = tempElement.attr("onclick");
List<String> list = ReUtil.findAllGroup0("'(.*?)'", onclick);
if (list.size() > 1) {
href = StrUtil.subBetween(list.get(0), "'", "'");
}
}
if (tempElement.text().trim().equals("好差评")) {
String onclick = tempElement.attr("onclick");
evaluationUrl = StrUtil.subBetween(onclick, "evaluation('", "')");
......@@ -131,14 +142,18 @@ public class MatterHtmlParseUtil {
continue;
}
String title = element.attr("title");
String href = element.attr("href");
href = element.attr("href");
//抓取申请与评价页面地址
String evaluationUrl = "";
String netApplyUrl = "";
Element nextElementSibling = element.nextElementSibling();
if (nextElementSibling != null) {
Elements elementsA = nextElementSibling.selectXpath("//a");
for (Element tempElement : elementsA) {
if ("办事指南".equals(tempElement.text().trim())) {
String onclick = tempElement.attr("onclick");
href = StrUtil.subBetween(onclick, "ywblurl('", "')");
}
if ("好差评".equals(tempElement.text().trim())) {
String onclick = tempElement.attr("onclick");
evaluationUrl = StrUtil.subBetween(onclick, "evaluation('", "')");
......@@ -270,7 +285,7 @@ public class MatterHtmlParseUtil {
// System.out.println(JSON.toJSONString(allList));
String url = "http://www.sczwfw.gov.cn/jiq/interface/item/tags";
/* String url = "http://www.sczwfw.gov.cn/jiq/interface/item/tags";
HashMap<String, String> params = new HashMap<>();
params.put("areaCode", "511500000000");
params.put("dxType", "3");
......@@ -280,7 +295,7 @@ public class MatterHtmlParseUtil {
params.put("taskType", "");
// params.put("pageno", "1");
Rest<Map<String, Integer>> rest = MatterHtmlParseUtil.statSiteMatterDeptCount(params, url);
System.out.println(JSON.toJSONString(rest));
System.out.println(JSON.toJSONString(rest));*/
/*
......@@ -314,6 +329,12 @@ public class MatterHtmlParseUtil {
*/
}
}
base-manager/src/main/java/com/mortals/xhx/daemon/task/SyncUserTaskImpl.java
View file @ 6a0f4015
......@@ -48,8 +48,6 @@ public class SyncUserTaskImpl implements ITaskExcuteService {
userService.updateUserList(list.getData().getData());
//resourceService.updateUserList();
/* UserPdu userPdu = new UserPdu();
......
base-manager/src/main/java/com/mortals/xhx/module/site/service/impl/SiteServiceImpl.java
View file @ 6a0f4015
......@@ -368,14 +368,14 @@ public class SiteServiceImpl extends AbstractCRUDCacheServiceImpl<SiteDao, SiteE
private List<SiteTreeSelect> getSiteTreeSelects(String userId) {
String siteTreeSelectStr = cacheService.hget(USER_SITE_TREE, userId, String.class);
log.info("userId:{},siteTreeSelectStr:{}",userId,siteTreeSelectStr);
log.info("userId:{},siteTreeSelectStr:{}", userId, siteTreeSelectStr);
//反序列化树对象
if(ObjectUtils.isEmpty(siteTreeSelectStr)){
if (ObjectUtils.isEmpty(siteTreeSelectStr)) {
return new ArrayList<>();
}
JSONArray jsonArray = JSON.parseArray(siteTreeSelectStr);
if(ObjectUtils.isEmpty(jsonArray)){
if (ObjectUtils.isEmpty(jsonArray)) {
return new ArrayList<>();
}
List<SiteTreeSelect> collect = jsonArray.stream().map(item -> {
......@@ -554,13 +554,34 @@ public class SiteServiceImpl extends AbstractCRUDCacheServiceImpl<SiteDao, SiteE
return Rest.ok(allList);
}
private List<MatterEntity> getMatters(HashMap<String, String> params, Context context) {
String url = GlobalSysInfo.getParamValue(Constant.GOV_MATTER_PAGELIST_URL, "http://www.sczwfw.gov.cn/jiq/interface/item/tags");
Rest<Map<String, Integer>> restStat = MatterHtmlParseUtil.statSiteMatterDeptCount(params, url);
if (restStat.getCode() == YesNoEnum.YES.getValue()) {
Integer pageNum = restStat.getData().getOrDefault("pageNum", 0);
Integer total = restStat.getData().getOrDefault("total", 0);
//获取事项全列表
Rest<List<MatterEntity>> matterAllRest = this.getMatterAllListByGOV(params, pageNum, context);
if (matterAllRest.getCode() == YesNoEnum.YES.getValue()) {
return matterAllRest.getData();
}
}
return new ArrayList<MatterEntity>();
}
@Override
public Rest<String> syncMatterBySiteId(SiteEntity siteEntity, Context context) {
//根据站点中区域编码 查询当前是否已经存在事项
//根据站点区域编码查询政务网事项列表
AreaEntity areaEntity = areaService.getCache(siteEntity.getAreaCode());
String dxType = AreaLevelDxTypeEnum.getByValue(areaEntity == null ? 2 : areaEntity.getAreaLevel()).getDesc();
String url = GlobalSysInfo.getParamValue(Constant.GOV_MATTER_PAGELIST_URL, "http://www.sczwfw.gov.cn/jiq/interface/item/tags");
//String dxType = AreaLevelDxTypeEnum.getByValue(areaEntity == null ? 2 : areaEntity.getAreaLevel()).getDesc();
String dxType="2";
//String dxType = "3";
HashMap<String, String> params = new HashMap<>();
params.put("dxType", dxType);
params.put("areaCode", siteEntity.getAreaCode());
......@@ -568,18 +589,50 @@ public class SiteServiceImpl extends AbstractCRUDCacheServiceImpl<SiteDao, SiteE
params.put("searchtext", "");
params.put("taskType", "");
params.put("pageno", "1");
params.put("type", "2");
Rest<Map<String, Integer>> restStat = MatterHtmlParseUtil.statSiteMatterCount(params, url);
List<MatterEntity> govMatterList = this.getMatters(params, context);
dxType = "3";
params = new HashMap<>();
params.put("dxType", dxType);
params.put("areaCode", siteEntity.getAreaCode());
params.put("deptCode", "");
params.put("searchtext", "");
params.put("taskType", "");
params.put("pageno", "1");
params.put("type", "2");
List<MatterEntity> mattersTwo = this.getMatters(params, context);
govMatterList.addAll(mattersTwo);
List<MatterEntity> localMatterList = matterService.find(new MatterQuery().areaCode(siteEntity.getAreaCode()));
List<MatterEntity> subList = this.subList(govMatterList, localMatterList);
log.info("需要添加事项数量====" + subList.size());
//差集进行插入并更新详细数据
if (!ObjectUtils.isEmpty(subList)) {
for (MatterEntity matterEntity : subList) {
DeptEntity deptCache = deptService.getExtCache(matterEntity.getDeptCode());
matterEntity.setDeptName(deptCache == null ? "" : deptCache.getName());
matterService.save(matterEntity, context);
}
List<List<MatterEntity>> partition = ListUtil.partition(subList, 100);
for (List<MatterEntity> matterEntityList : partition) {
log.info("insert subList size:" + matterEntityList.size());
int count = matterService.save(matterEntityList, context);
log.info("insert subList size success:" + count);
}
}
/* Rest<Map<String, Integer>> restStat = MatterHtmlParseUtil.statSiteMatterDeptCount(params, url);
if (restStat.getCode() == YesNoEnum.YES.getValue()) {
Integer pageNum = restStat.getData().getOrDefault("pageNum", 0);
Integer total = restStat.getData().getOrDefault("total", 0);
//获取事项全列表
Rest<List<MatterEntity>> matterAllRest = this.getMatterAllListByGOV(params, pageNum, context);
if (total != matterAllRest.getData().size()) {
log.warn(String.format("抓取事项数量不一致，抓取计划:%d条 ,实际抓取:%d条", total, matterAllRest.getData().size()));
}
if (matterAllRest.getCode() == YesNoEnum.YES.getValue()) {
List<MatterEntity> govMatterList = matterAllRest.getData();
List<MatterEntity> localMatterList = matterService.find(new MatterQuery().areaCode(siteEntity.getAreaCode()));
......@@ -602,7 +655,7 @@ public class SiteServiceImpl extends AbstractCRUDCacheServiceImpl<SiteDao, SiteE
}
}
}
*/
return Rest.ok("同步事项条数成功！");
}
......
base-manager/src/main/java/com/mortals/xhx/module/site/service/impl/SiteThemeServiceImpl.java
View file @ 6a0f4015
......@@ -41,9 +41,6 @@ import java.util.regex.Pattern;
@Service("siteThemeService")
public class SiteThemeServiceImpl extends AbstractCRUDServiceImpl<SiteThemeDao, SiteThemeEntity, Long> implements SiteThemeService {
@Autowired
private SiteService siteService;
@Override
public Rest<String> syncThemeBySiteId(SiteEntity siteEntity, Context context) {
......@@ -151,16 +148,21 @@ public class SiteThemeServiceImpl extends AbstractCRUDServiceImpl<SiteThemeDao,
public static void main(String[] args) {
String str = "changeTheme('005', '1')";
String str = "ywblurl('http://www.sczwfw.gov.cn/jiq/front/transition/ywTransToDetail?areaCode=511500000000&amp;itemCode=511A0141700000-511500000000-000-125112007566044888-1-00&amp;taskType=1&amp;deptCode=511501-17','在公路增设或改造平面交叉道口审批')";
/*
String reg = "'.*?'";
Pattern compile = Pattern.compile(reg);
*/
List<String> allGroups = ReUtil.findAllGroup0(compile, str);
List<String> allGroups = ReUtil.findAllGroup0("'(.*?)'", str);
allGroups.forEach(item -> {
System.out.println(item);
// System.out.println(item);
});
System.out.println(allGroups.get(0));
}
}
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment