Skip to content

S
smart_gov_platform
Commit 3bae4198 authored by 赵啸非's avatar 赵啸非
Browse files
Options

修改重试机制

parent 7f5e2976
Show whitespace changes
Inline Side-by-side
Showing with 109 additions and 124 deletions
base-manager/src/main/java/com/mortals/xhx/common/utils/MatterDetailHtmlParseUtil.java
View file @ 3bae4198
......@@ -26,50 +26,7 @@ public class MatterDetailHtmlParseUtil {
try {
document = Jsoup.connect(url).get();
return document;
/*Jsoup.connect(url).get();
html = Jsoup.connect(url).get().body().html();
// html = HttpUtil.get(url);
//System.out.println(html);
HtmlCleaner hc = new HtmlCleaner();
TagNode tn = hc.clean(html);
Object[] rs = tn.evaluateXPath("//div");
for (Integer i = 0; i < rs.length; i++) {
TagNode n = (TagNode) rs[i];
System.out.println(n.getText());
}*/
// System.out.println(tn.getText());
/* CleanerProperties cleanerProperties = new CleanerProperties();
cleanerProperties.setAdvancedXmlEscape(true);
cleanerProperties.setOmitXmlDeclaration(true);
cleanerProperties.setOmitDoctypeDeclaration(true);
cleanerProperties.setTranslateSpecialEntities(true);
cleanerProperties.setTransResCharsToNCR(true);
cleanerProperties.setRecognizeUnicodeChars(true);
cleanerProperties.setIgnoreQuestAndExclam(true);
cleanerProperties.setUseEmptyElementTags(false);
Document dom = new DomSerializer(cleanerProperties).createDOM(tn);*/
/*XPath xPath = XPathFactory.newInstance().newXPath();
String tklrExp = String.format("//div[@class=\"smltc9_bottom\"]//p");
Object result = xPath.evaluate(tklrExp, dom, XPathConstants.NODESET);
NodeList nodeList = (NodeList) result;
for (Integer i = 0; i < nodeList.getLength(); i++) {
System.out.println(nodeList.item(i).getTextContent());
System.out.println("===============");
}*/
//div[@class='section10']//table[1]//tr[%d]//td
//return dom;
} catch (Exception e) {
e.printStackTrace();
}
......
base-manager/src/main/java/com/mortals/xhx/common/utils/MatterHtmlParseUtil.java
View file @ 3bae4198
......@@ -50,6 +50,7 @@ public class MatterHtmlParseUtil {
Integer pageNum = elements.get(0) == null ? 0 : DataUtil.converStr2Int(elements.get(0).attr("value"), 0);
resultMap.put("pageNum", pageNum);
}
break;
} catch (Exception e) {
log.error("获取事项数量异常！params:{},重试:{}次", JSON.toJSONString(params), reCount, e);
reCount++;
......@@ -230,6 +231,7 @@ public class MatterHtmlParseUtil {
String deptCode = builder.getQuery().get("deptCode").toString();
map.put(deptCode, deptName);
}
break;
} catch (Exception e) {
log.error("当前站点同步添加部门异常！params:{},重试:{}次", JSON.toJSONString(params), reCount, e);
reCount++;
......
base-manager/src/main/java/com/mortals/xhx/common/utils/MatterTypeHtmlParseUtil.java
View file @ 3bae4198
......@@ -22,6 +22,8 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static com.mortals.xhx.common.key.Constant.MAX_RETRY_COUNT;
/**
* 解析事项类型，主题，热度等
*/
......@@ -40,10 +42,13 @@ public class MatterTypeHtmlParseUtil {
String matterTotalExp = "//input[@id=\"result_countTheme\"]";
String matterPageExp = "//input[@id=\"pageNumTheme\"]";
Map<String, Integer> resultMap = new HashMap<>();
int reCount = 0;
while (reCount < MAX_RETRY_COUNT) {
try {
Document dom = Jsoup.connect(url)
.ignoreContentType(true)
.ignoreHttpErrors(true)
.timeout(60 * 1000)
.data(params).get();
Elements elements = dom.selectXpath(matterTotalExp);
if (elements.size() > 0) {
......@@ -60,10 +65,15 @@ public class MatterTypeHtmlParseUtil {
} else {
resultMap.put("pageNum", 0);
}
break;
} catch (Exception e) {
log.error("获取事项数量异常！params:" + JSON.toJSONString(params), e);
return Rest.fail(e.getMessage());
reCount++;
try {
Thread.sleep(2000);
} catch (InterruptedException interruptedException) {
}
}
}
return Rest.ok(resultMap);
......@@ -80,10 +90,13 @@ public class MatterTypeHtmlParseUtil {
String matterTotalExp = "//input[@id=\"result_count\"]";
String matterPageExp = "//input[@id=\"pageNum\"]";
Map<String, Integer> resultMap = new HashMap<>();
int reCount = 0;
while (reCount < MAX_RETRY_COUNT) {
try {
Document dom = Jsoup.connect(url)
.ignoreContentType(true)
.ignoreHttpErrors(true)
.timeout(60 * 1000)
.data(params).get();
Elements elements = dom.selectXpath(matterTotalExp);
if (elements.size() > 0) {
......@@ -100,10 +113,15 @@ public class MatterTypeHtmlParseUtil {
} else {
resultMap.put("pageNum", 0);
}
break;
} catch (Exception e) {
log.error("获取事项数量异常！params:" + JSON.toJSONString(params), e);
return Rest.fail(e.getMessage());
reCount++;
try {
Thread.sleep(2000);
} catch (InterruptedException interruptedException) {
}
}
}
return Rest.ok(resultMap);
......@@ -120,10 +138,13 @@ public class MatterTypeHtmlParseUtil {
public static Rest<Map<String, String>> getThemeList(Map<String, String> params, String url) {
String ThemeListExp = "//ul[@class=\"zhuti_list\"]//li//a";
Map<String, String> map = new HashMap<>();
int reCount = 0;
while (reCount < MAX_RETRY_COUNT) {
try {
Document dom = Jsoup.connect(url)
.ignoreContentType(true)
.ignoreHttpErrors(true)
.timeout(60 * 1000)
.data(params).get();
Elements elements = dom.selectXpath(ThemeListExp);
......@@ -137,10 +158,15 @@ public class MatterTypeHtmlParseUtil {
String title = element.attr("title");
map.put(themeCode, title);
}
break;
} catch (Exception e) {
log.error("获取列表异常！params:" + JSON.toJSONString(params), e);
return Rest.fail(e.getMessage());
reCount++;
try {
Thread.sleep(2000);
} catch (InterruptedException interruptedException) {
}
}
}
return Rest.ok(map);
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment