Skip to content

S
smart_gov_platform
Commit ae4978f6 authored by 赵啸非's avatar 赵啸非
Browse files
Options

修改重试机制

parent 5a240978
Hide whitespace changes
Inline Side-by-side
Showing with 139 additions and 149 deletions
base-manager/src/main/java/com/mortals/xhx/common/utils/MatterHtmlParseUtil.java
View file @ ae4978f6
......@@ -22,6 +22,8 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static com.mortals.xhx.common.key.Constant.MAX_RETRY_COUNT;
@Slf4j
public class MatterHtmlParseUtil {
......@@ -29,61 +31,39 @@ public class MatterHtmlParseUtil {
String matterTotalExp = "//input[@id=\"result_count\"]";
String matterPageExp = "//input[@id=\"pageNum\"]";
Map<String, Integer> resultMap = new HashMap<>();
try {
Document dom = Jsoup.connect(url)
.ignoreContentType(true)
.ignoreHttpErrors(true)
.data(params).get();
Elements elements = dom.selectXpath(matterTotalExp);
if (elements.size() > 0) {
Integer total = elements.get(0) == null ? 0 : DataUtil.converStr2Int(elements.get(0).attr("value"), 0);
resultMap.put("total", total);
}
int reCount = 0;
while (reCount < MAX_RETRY_COUNT) {
try {
Document dom = Jsoup.connect(url)
.ignoreContentType(true)
.ignoreHttpErrors(true)
.timeout(60 * 1000)
.data(params).get();
Elements elements = dom.selectXpath(matterTotalExp);
if (elements.size() > 0) {
Integer total = elements.get(0) == null ? 0 : DataUtil.converStr2Int(elements.get(0).attr("value"), 0);
resultMap.put("total", total);
}
elements = dom.selectXpath(matterPageExp);
if (elements.size() > 0) {
Integer pageNum = elements.get(0) == null ? 0 : DataUtil.converStr2Int(elements.get(0).attr("value"), 0);
resultMap.put("pageNum", pageNum);
elements = dom.selectXpath(matterPageExp);
if (elements.size() > 0) {
Integer pageNum = elements.get(0) == null ? 0 : DataUtil.converStr2Int(elements.get(0).attr("value"), 0);
resultMap.put("pageNum", pageNum);
}
} catch (Exception e) {
log.error("获取事项数量异常！params:{},重试:{}次", JSON.toJSONString(params), reCount, e);
reCount++;
try {
Thread.sleep(2000);
} catch (InterruptedException interruptedException) {
}
}
} catch (Exception e) {
log.error("获取事项数量异常！params:" + JSON.toJSONString(params), e);
return Rest.fail(e.getMessage());
}
return Rest.ok(resultMap);
}
public static Rest<Map<String, Integer>> statSiteMatterDeptCount(Map<String, String> params, String url) {
String matterTotalExp = "//input[@id=\"result_countDept\"]";
String matterPageExp = "//input[@id=\"pageNumDept\"]";
Map<String, Integer> resultMap = new HashMap<>();
try {
Document dom = Jsoup.connect(url)
.ignoreContentType(true)
.ignoreHttpErrors(true)
.data(params).get();
Elements elements = dom.selectXpath(matterTotalExp);
if (elements.size() > 0) {
Integer total = elements.get(0) == null ? 0 : DataUtil.converStr2Int(elements.get(0).attr("value"), 0);
resultMap.put("total", total);
}
elements = dom.selectXpath(matterPageExp);
if (elements.size() > 0) {
Integer pageNum = elements.get(0) == null ? 0 : DataUtil.converStr2Int(elements.get(0).attr("value"), 0);
resultMap.put("pageNum", pageNum);
}
} catch (Exception e) {
log.error("获取事项数量异常！params:" + JSON.toJSONString(params), e);
return Rest.fail(e.getMessage());
}
return Rest.ok(resultMap);
}
public static Rest<List<MatterEntity>> getMatterList(Map<String, String> params, String url) {
String matterListExp = "//div[@class=\"sx_list\"]//span[1]";
String matterListLiExp = "//div[@class=\"sx_list\"]//li/a[1]";
......@@ -91,107 +71,117 @@ public class MatterHtmlParseUtil {
String evaluationUrl = "";
String netApplyUrl = "";
String href = "";
try {
Document dom = Jsoup.connect(url).data(params).get();
//System.out.println(dom.html());
Elements elements = dom.selectXpath(matterListExp);
for (int i = 0; i < elements.size(); i++) {
Element element = elements.get(i);
if (element == null) {
continue;
}
String title = element.attr("title");
href = element.firstElementChild().attr("href");
int reCount = 0;
while (reCount < MAX_RETRY_COUNT) {
try {
Document dom = Jsoup.connect(url)
.ignoreContentType(true)
.ignoreHttpErrors(true)
.timeout(60 * 1000)
.data(params).get();
Elements elements = dom.selectXpath(matterListExp);
for (int i = 0; i < elements.size(); i++) {
Element element = elements.get(i);
if (element == null) {
continue;
}
String title = element.attr("title");
href = element.firstElementChild().attr("href");
//element.child()
if (href.equalsIgnoreCase("javascript:void(0)")) {
continue;
}
//抓取申请与评价页面地址
Element nextElementSibling = element.nextElementSibling();
Elements elementsA = nextElementSibling.children();
//Elements elementsA = nextElementSibling.selectXpath("//a");
if (elementsA != null) {
for (Element tempElement : elementsA) {
if ("办事指南".equals(tempElement.text().trim())) {
String onclick = tempElement.attr("onclick");
List<String> list = ReUtil.findAllGroup0("'(.*?)'", onclick);
if (list.size() > 1) {
href = StrUtil.subBetween(list.get(0), "'", "'");
//element.child()
if (href.equalsIgnoreCase("javascript:void(0)")) {
continue;
}
//抓取申请与评价页面地址
Element nextElementSibling = element.nextElementSibling();
Elements elementsA = nextElementSibling.children();
//Elements elementsA = nextElementSibling.selectXpath("//a");
if (elementsA != null) {
for (Element tempElement : elementsA) {
if ("办事指南".equals(tempElement.text().trim())) {
String onclick = tempElement.attr("onclick");
List<String> list = ReUtil.findAllGroup0("'(.*?)'", onclick);
if (list.size() > 1) {
href = StrUtil.subBetween(list.get(0), "'", "'");
}
}
}
if ("好差评".equals(tempElement.text().trim())) {
String onclick = tempElement.attr("onclick");
evaluationUrl = StrUtil.subBetween(onclick, "evaluation('", "')");
}
if ("申请".equals(tempElement.text().trim())) {
String onclick = tempElement.attr("onclick");
List<String> list = ReUtil.findAllGroup0("'(.*?)'", onclick);
if (list.size() > 4) {
netApplyUrl = StrUtil.subBetween(list.get(3), "'", "'");
if ("好差评".equals(tempElement.text().trim())) {
String onclick = tempElement.attr("onclick");
evaluationUrl = StrUtil.subBetween(onclick, "evaluation('", "')");
}
if ("申请".equals(tempElement.text().trim())) {
String onclick = tempElement.attr("onclick");
List<String> list = ReUtil.findAllGroup0("'(.*?)'", onclick);
if (list.size() > 4) {
netApplyUrl = StrUtil.subBetween(list.get(3), "'", "'");
}
}
}
}
}
if (ObjectUtils.isEmpty(href)) {
log.info("error href ,title:" + title);
}
buildMatter(matterEntityList, title, href, evaluationUrl, netApplyUrl);
}
if (ObjectUtils.isEmpty(href)) {
log.info("error href ,title:" + title);
}
elements = dom.selectXpath(matterListLiExp);
for (int i = 0; i < elements.size(); i++) {
Element element = elements.get(i);
if (element == null) {
continue;
buildMatter(matterEntityList, title, href, evaluationUrl, netApplyUrl);
}
String title = element.attr("title");
href = element.attr("href");
//抓取申请与评价页面地址
Element nextElementSibling = element.nextElementSibling();
if (nextElementSibling != null) {
Elements elementsA = nextElementSibling.children();
//Elements elementsA = nextElementSibling.selectXpath("//a");
for (Element tempElement : elementsA) {
if ("办事指南".equals(tempElement.text().trim())) {
String onclick = tempElement.attr("onclick");
if(ObjectUtils.isEmpty(onclick)) continue;
List<String> list = ReUtil.findAllGroup0("'(.*?)'", onclick);
if (list.size() > 1) {
href = StrUtil.subBetween(list.get(0), "'", "'");
elements = dom.selectXpath(matterListLiExp);
for (int i = 0; i < elements.size(); i++) {
Element element = elements.get(i);
if (element == null) {
continue;
}
String title = element.attr("title");
href = element.attr("href");
//抓取申请与评价页面地址
Element nextElementSibling = element.nextElementSibling();
if (nextElementSibling != null) {
Elements elementsA = nextElementSibling.children();
//Elements elementsA = nextElementSibling.selectXpath("//a");
for (Element tempElement : elementsA) {
if ("办事指南".equals(tempElement.text().trim())) {
String onclick = tempElement.attr("onclick");
if (ObjectUtils.isEmpty(onclick)) continue;
List<String> list = ReUtil.findAllGroup0("'(.*?)'", onclick);
if (list.size() > 1) {
href = StrUtil.subBetween(list.get(0), "'", "'");
}
}
}
if ("好差评".equals(tempElement.text().trim())) {
String onclick = tempElement.attr("onclick");
evaluationUrl = StrUtil.subBetween(onclick, "evaluation('", "')");
}
if ("申请".equals(tempElement.text().trim())) {
String onclick = tempElement.attr("onclick");
List<String> list = ReUtil.findAllGroup0("'(.*?)'", onclick);
if (list.size() > 4) {
netApplyUrl = StrUtil.subBetween(list.get(3), "'", "'");
if ("好差评".equals(tempElement.text().trim())) {
String onclick = tempElement.attr("onclick");
evaluationUrl = StrUtil.subBetween(onclick, "evaluation('", "')");
}
if ("申请".equals(tempElement.text().trim())) {
String onclick = tempElement.attr("onclick");
List<String> list = ReUtil.findAllGroup0("'(.*?)'", onclick);
if (list.size() > 4) {
netApplyUrl = StrUtil.subBetween(list.get(3), "'", "'");
}
}
}
if (ObjectUtils.isEmpty(href)) {
log.info("error href ,title:" + title);
}
if (ObjectUtils.isEmpty(href)) {
log.info("error href ,title:" + title);
}
}
}
buildMatter(matterEntityList, title, href, evaluationUrl, netApplyUrl);
}
buildMatter(matterEntityList, title, href, evaluationUrl, netApplyUrl);
}
} catch (Exception e) {
log.error("获取列表异常！params:" + JSON.toJSONString(params), e);
return Rest.fail(e.getMessage());
} catch (Exception e) {
log.error("获取列表异常！params:{},重试:{}次", JSON.toJSONString(params), reCount, e);
reCount++;
try {
Thread.sleep(2000);
} catch (InterruptedException interruptedException) {
}
}
}
return Rest.ok(matterEntityList);
}
......@@ -220,10 +210,13 @@ public class MatterHtmlParseUtil {
public static Rest<Map<String, String>> syncDeptBySiteId(Map<String, String> params, String url) {
String deptListExp = "//ul[@class='bm-list']//li";
Map<String, String> map = new HashMap<>();
try {
int reCount = 0;
while (reCount < MAX_RETRY_COUNT) {
try {
Document dom = Jsoup.connect(url)
.ignoreContentType(true)
.ignoreHttpErrors(true)
.timeout(60 * 1000)
.data(params).get();
Elements elements = dom.selectXpath(deptListExp);
for (int i = 0; i < elements.size(); i++) {
......@@ -237,8 +230,14 @@ public class MatterHtmlParseUtil {
String deptCode = builder.getQuery().get("deptCode").toString();
map.put(deptCode, deptName);
}
} catch (Exception e) {
return Rest.fail(e.getMessage());
} catch (Exception e) {
log.error("当前站点同步添加部门异常！params:{},重试:{}次", JSON.toJSONString(params), reCount, e);
reCount++;
try {
Thread.sleep(2000);
} catch (InterruptedException interruptedException) {
}
}
}
return Rest.ok("当前站点同步添加部门成功！", map);
......@@ -248,9 +247,9 @@ public class MatterHtmlParseUtil {
public static void main(String[] args) {
String url = "http://www.sczwfw.gov.cn/jiq/interface/item/tags";
HashMap<String, String> params = new HashMap<>();
params.put("dxType", "54");
params.put("areaCode", "511503003999");
params.put("deptCode", "");
params.put("dxType", "6");
params.put("areaCode", "511523000000");
params.put("deptCode", "2632");
params.put("searchtext", "");
params.put("pageno", "1");
params.put("taskType", "");
......@@ -258,6 +257,10 @@ public class MatterHtmlParseUtil {
System.out.println(rest.getData().size());
//{"searchtext":"","areaCode":"511523000000","taskType":"","dxType":"6","deptCode":"2632"}
/* HashMap<String, String> params = new HashMap<>();
params.put("areaCode", "510116000000");
......
base-manager/src/main/java/com/mortals/xhx/module/matter/service/impl/MatterExtServiceImpl.java
View file @ ae4978f6
......@@ -127,26 +127,13 @@ public class MatterExtServiceImpl extends AbstractCRUDServiceImpl<MatterExtDao,
matterList = matterService.getDao().getMatterListByAreaCode(matterQuery.areaCode(siteEntity.getAreaCode()));
}
log.info("开始更新事项详细！");
// List<MatterEntity> updateList = new ArrayList<>();
for (MatterEntity matterEntity : matterList) {
Rest<String> rest = matterService.buildMatterDetail(matterEntity, null);
if (rest.getCode() == YesNoEnum.YES.getValue()) {
matterEntity.setUpdateTime(new Date());
// updateList.add(matterEntity);
// log.info("id==>{} matterEditon==>{}",matterEntity.getId(),matterEntity.getMatterEdition());
matterService.update(matterEntity, null);
}
}
/* if (!ObjectUtils.isEmpty(updateList)) {
log.info("更新事项详细==》{}", updateList.size());
List<List<MatterEntity>> partition = Lists.partition(updateList, 50);
for (List<MatterEntity> matterEntityList : partition) {
matterService.update(matterEntityList, null);
}
}*/
return Rest.ok();
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment