接入第三方数据到mysql

1
2
3
4
5
6
7
8
9
10
package com.example.demo;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
@SpringBootApplication
public class DemoApplication {
public static void main(String[] args) {
SpringApplication.run(DemoApplication.class, args);
}
}

1
2
3
4
5
6
7
8
9
10
11
12
package com.example.demo;
import lombok.Data;
/**
* @Author shichaoran
* @Date 2020/4/18 23:35
* @Version
*/
@Data
public class GTResponseBO {
private String publishTime;
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
package demo;
/**
* @Author shichaoran
* @Date 2020/4/17 15:01
* @Version
*/
import java.util.HashMap;
import java.util.Map;
/**
* 请求头
*/
public class HttpHeader {
private Map<String, String> params = new HashMap<String, String>();
public HttpHeader addParam(String name, String value) {
this.params.put(name, value);
return this;
}
public Map<String, String> getParams() {
return this.params;
}
}

1
2
3
4
5
6
7
8
9
10
11
12
13
package demo;
import org.apache.commons.collections4.Get;
import javax.ws.rs.GET;
import javax.ws.rs.POST;
/**
* @Author shichaoran
* @Date 2020/4/17 15:27
* @Version
*/
public enum HttpMethod {
GET, POST;
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
package demo;
/**
* @Author shichaoran
* @Date 2020/4/17 14:57
* @Version
*/
import java.io.IOException;
import java.net.URLEncoder;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;
import com.alibaba.fastjson.JSON;
import org.springframework.http.HttpMethod;
/**
* 请求参数
*/
public class HttpParamers {
private Map<String, String> params = new HashMap<String, String>();
private HttpMethod httpMethod;
private String jsonParamer = "";
public HttpParamers(HttpMethod httpMethod) {
this.httpMethod = httpMethod;
}
public static HttpParamers httpPostParamers() {
return new HttpParamers(HttpMethod.POST);
}
public static HttpParamers httpGetParamers() {
return new HttpParamers(HttpMethod.GET);
}
public HttpParamers addParam(String name, String value) {
this.params.put(name, value);
return this;
}
public HttpMethod getHttpMethod() {
return this.httpMethod;
}
public String getQueryString(String charset) throws IOException {
if ((this.params == null) || (this.params.isEmpty())) {
return null;
}
StringBuilder query = new StringBuilder();
Set<Map.Entry<String, String>> entries = this.params.entrySet();
for (Map.Entry<String, String> entry : entries) {
String name = entry.getKey();
String value = entry.getValue();
query.append("&").append(name).append("=").append(URLEncoder.encode(value, charset));
}
return query.substring(1);
}
public boolean isJson() {
return !isEmpty(this.jsonParamer);
}
public Map<String, String> getParams() {
return this.params;
}
public String toString() {
return "HttpParamers " + JSON.toJSONString(this);
}
public String getJsonParamer() {
return this.jsonParamer;
}
public void setJsonParamer() {
this.jsonParamer = JSON.toJSONString(this.params);
}
private static boolean isEmpty(CharSequence cs) {
return (cs == null) || (cs.length() == 0);
}
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
*/
public class RemoteDataUtil {
public static String getData() throws Exception {
// 1根据token获取url
String response = "";
List<String> list =new ArrayList<>() ;
String uri = "";
HttpParamers paramers = HttpParamers.httpPostParamers();
paramers.addParam("time", String.valueOf(System.currentTimeMillis()));
HttpService httpService = new HttpService(uri);
response = httpService.service("/posts", paramers);
JSONObject jsonObject = JSONObject.parseObject(response);
Object markets = jsonObject.get("markets");
JSONArray jsonArray = JSON.parseArray(markets.toString());
jsonArray.forEach(json->{
JSONObject jsonObject1 = JSONObject.parseObject(json.toString());
Object urlObject = jsonObject1.get("url");
AtomicReference<String> response1 = new AtomicReference<>("");
HttpService httpService1 = new HttpService(urlObject.toString());
try {
response1.set(httpService1.service(urlObject.toString(),paramers));
} catch (Exception e) {
e.printStackTrace();
}
list.add(response1.get());
});
return list.toString();
}
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
package Service;
/**
* @Author shichaoran
* @Date 2020/4/17 15:03
* @Version
*/
import java.util.Map;
import Client.HttpClient;
import com.alibaba.fastjson.JSONObject;
import com.alibaba.fastjson.TypeReference;
import demo.HttpHeader;
import demo.HttpParamers;
import lombok.Data;
import lombok.ToString;
@Data
@ToString
public class HttpService {
private String serverUrl;
private int connectTimeout = 15000;
private int readTimeout = 30000;
public HttpService(String serverUrl) {
this.serverUrl = serverUrl.trim();
}
public Map<String, Object> commonService(String serviceUrl, HttpParamers paramers) throws Exception {
return commonService(serviceUrl, paramers, null);
}
public Map<String, Object> commonService(String serviceUrl, HttpParamers paramers, HttpHeader header) throws Exception {
String response = service(serviceUrl, paramers, header);
try {
Map<String, Object> result = JSONObject.parseObject(response, new TypeReference<Map<String, Object>>() {
});
if ((result == null) || (result.isEmpty())) {
throw new Exception("远程服务返回的数据无法解析");
}
Integer code = (Integer) result.get("code");
if ((code == null) || (code.intValue() != 0)) {
throw new Exception((String) result.get("message"));
}
return result;
} catch (Exception e) {
throw new Exception("返回结果异常,response:" + response, e);
}
}
public String service(String serviceUrl, HttpParamers paramers) throws Exception {
return service(serviceUrl, paramers, null);
}
public String service(String serviceUrl, HttpParamers paramers, HttpHeader header) throws Exception {
String url = this.serverUrl + serviceUrl;
String responseData = "";
try {
responseData = HttpClient.doService(url, paramers, header, this.connectTimeout, this.readTimeout);
} catch (Exception e) {
throw new Exception(e.getMessage(), e);
}
return responseData;
}
public String getServerUrl() {
return this.serverUrl;
}
public int getConnectTimeout() {
return this.connectTimeout;
}
public int getReadTimeout() {
return this.readTimeout;
}
public void setConnectTimeout(int connectTimeout) {
this.connectTimeout = connectTimeout;
}
public void setReadTimeout(int readTimeout) {
this.readTimeout = readTimeout;
}
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
package Util;
import org.jsoup.Connection;
import org.jsoup.Connection.Method;
import org.jsoup.Connection.Response;
import org.jsoup.Jsoup;
import java.io.*;
import java.security.SecureRandom;
import java.security.cert.CertificateException;
import java.security.cert.X509Certificate;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import javax.net.ssl.HostnameVerifier;
import javax.net.ssl.HttpsURLConnection;
import javax.net.ssl.SSLContext;
import javax.net.ssl.SSLSession;
import javax.net.ssl.X509TrustManager;
/**
* Http发送post请求工具，兼容http和https两种请求类型
*/
public class HttpUtils {
/**
* 请求超时时间
*/
private static final int TIME_OUT = 120000;
/**
* Https请求
*/
private static final String HTTPS = "https";
/**
* Content-Type
*/
private static final String CONTENT_TYPE = "Content-Type";
/**
* 表单提交方式Content-Type
*/
private static final String FORM_TYPE = "application/x-www-form-urlencoded;charset=UTF-8";
/**
* JSON提交方式Content-Type
*/
private static final String JSON_TYPE = "application/json;charset=UTF-8";
/**
* 发送Get请求
*
* @param url 请求URL
* @return HTTP响应对象
* @throws IOException 程序异常时抛出，由调用者处理
*/
public static Response get(String url) throws IOException {
return get(url, null);
}
/**
* 发送Get请求
*
* @param url 请求URL
* @param headers 请求头参数
* @return HTTP响应对象
* @throws IOException 程序异常时抛出，由调用者处理
*/
public static Response get(String url, Map<String, String> headers) throws IOException {
if (null == url || url.isEmpty()) {
throw new RuntimeException("The request URL is blank.");
}
// 如果是Https请求
if (url.startsWith(HTTPS)) {
getTrust();
}
Connection connection = Jsoup.connect(url);
connection.method(Connection.Method.GET);
connection.timeout(TIME_OUT);
connection.ignoreHttpErrors(true);
connection.ignoreContentType(true);
connection.maxBodySize(0);
if (null != headers) {
connection.headers(headers);
}
Response response = connection.execute();
return response;
}
/**
* 发送JSON格式参数POST请求
*
* @param url 请求路径
* @param params JSON格式请求参数
* @return HTTP响应对象
* @throws IOException 程序异常时抛出，由调用者处理
*/
public static Response post(String url, String params) throws IOException {
return doPostRequest(url, null, params);
}
/**
* 发送JSON格式参数POST请求
*
* @param url 请求路径
* @param headers 请求头中设置的参数
* @param params JSON格式请求参数
* @return HTTP响应对象
* @throws IOException 程序异常时抛出，由调用者处理
*/
public static Response post(String url, Map<String, String> headers, String params) throws IOException {
return doPostRequest(url, headers, params);
}
/**
* 字符串参数post请求
*
* @param url 请求URL地址
* @param paramMap 请求字符串参数集合
* @return HTTP响应对象
* @throws IOException 程序异常时抛出，由调用者处理
*/
public static Response post(String url, Map<String, String> paramMap) throws IOException {
return doPostRequest(url, null, paramMap, null);
}
/**
* 带请求头的普通表单提交方式post请求
*
* @param headers 请求头参数
* @param url 请求URL地址
* @param paramMap 请求字符串参数集合
* @return HTTP响应对象
* @throws IOException 程序异常时抛出，由调用者处理
*/
public static Response post(Map<String, String> headers, String url, Map<String, String> paramMap) throws IOException {
return doPostRequest(url, headers, paramMap, null);
}
/**
* 带上传文件的post请求
*
* @param url 请求URL地址
* @param paramMap 请求字符串参数集合
* @param fileMap 请求文件参数集合
* @return HTTP响应对象
* @throws IOException 程序异常时抛出，由调用者处理
*/
public static Response post(String url, Map<String, String> paramMap, Map<String, File> fileMap) throws IOException {
return doPostRequest(url, null, paramMap, fileMap);
}
/**
* 带请求头的上传文件post请求
*
* @param url 请求URL地址
* @param headers 请求头参数
* @param paramMap 请求字符串参数集合
* @param fileMap 请求文件参数集合
* @return HTTP响应对象
* @throws IOException 程序异常时抛出，由调用者处理
*/
public static Response post(String url, Map<String, String> headers, Map<String, String> paramMap, Map<String, File> fileMap) throws IOException {
return doPostRequest(url, headers, paramMap, fileMap);
}
/**
* 执行post请求
*
* @param url 请求URL地址
* @param headers 请求头
* @param jsonParams 请求JSON格式字符串参数
* @return HTTP响应对象
* @throws IOException 程序异常时抛出，由调用者处理
*/
private static Response doPostRequest(String url, Map<String, String> headers, String jsonParams) throws IOException {
if (null == url || url.isEmpty()) {
throw new RuntimeException("The request URL is blank.");
}
// 如果是Https请求
if (url.startsWith(HTTPS)) {
getTrust();
}
Connection connection = Jsoup.connect(url);
connection.method(Method.POST);
connection.timeout(TIME_OUT);
connection.ignoreHttpErrors(true);
connection.ignoreContentType(true);
connection.maxBodySize(0);
if (null != headers) {
connection.headers(headers);
}
connection.header(CONTENT_TYPE, JSON_TYPE);
connection.requestBody(jsonParams);
Response response = connection.method(Connection.Method.POST).execute();
/*Response response = connection.execute();*/
return response;
}
/**
* 普通表单方式发送POST请求
*
* @param url 请求URL地址
* @param headers 请求头
* @param paramMap 请求字符串参数集合
* @param fileMap 请求文件参数集合
* @return HTTP响应对象
* @throws IOException 程序异常时抛出，由调用者处理
*/
private static Response doPostRequest(String url, Map<String, String> headers, Map<String, String> paramMap, Map<String, File> fileMap) throws IOException {
if (null == url || url.isEmpty()) {
throw new RuntimeException("The request URL is blank.");
}
// 如果是Https请求
if (url.startsWith(HTTPS)) {
getTrust();
}
Connection connection = Jsoup.connect(url);
connection.method(Method.POST);
connection.timeout(TIME_OUT);
connection.ignoreHttpErrors(true);
connection.ignoreContentType(true);
connection.maxBodySize(0);
if (null != headers) {
connection.headers(headers);
}
// 收集上传文件输入流，最终全部关闭
List<InputStream> inputStreamList = null;
try {
// 添加文件参数
if (null != fileMap && !fileMap.isEmpty()) {
inputStreamList = new ArrayList<InputStream>();
InputStream in = null;
File file = null;
for (Entry<String, File> e : fileMap.entrySet()) {
file = e.getValue();
in = new FileInputStream(file);
inputStreamList.add(in);
connection.data(e.getKey(), file.getName(), in);
}
}
// 普通表单提交方式
else {
connection.header(CONTENT_TYPE, FORM_TYPE);
}
// 添加字符串类参数
if (null != paramMap && !paramMap.isEmpty()) {
connection.data(paramMap);
}
Response response = connection.execute();
return response;
}
// 关闭上传文件的输入流
finally {
closeStream(inputStreamList);
}
}
/**
* 关流
*
* @param streamList 流集合
*/
private static void closeStream(List<? extends Closeable> streamList) {
if (null != streamList) {
for (Closeable stream : streamList) {
try {
stream.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
}
/**
* 获取服务器信任
*/
private static void getTrust() {
try {
HttpsURLConnection.setDefaultHostnameVerifier(new HostnameVerifier() {
public boolean verify(String hostname, SSLSession session) {
return true;
}
});
SSLContext context = SSLContext.getInstance("TLS");
context.init(null, new X509TrustManager[] { new X509TrustManager() {
public void checkClientTrusted(X509Certificate[] chain, String authType) throws CertificateException {}
public void checkServerTrusted(X509Certificate[] chain, String authType) throws CertificateException {}
public X509Certificate[] getAcceptedIssuers() {
return new X509Certificate[0];
}
} }, new SecureRandom());
HttpsURLConnection.setDefaultSSLSocketFactory(context.getSocketFactory());
} catch (Exception e) {
e.printStackTrace();
}
}
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
package com.example.demo;
/**
* @Author shichaoran
* @Date 2020/4/17 15:04
* @Version
*/
import com.alibaba.fastjson.JSONObject;
import demo.HttpParamers;
import Service.HttpService;
import demo.RemoteDataUtil;
import org.junit.Ignore;
import org.junit.Test;
import java.util.List;
public class Test1 {
String uri = "http://open.api.mysteel.cn/zjsj/api_marketl_zjsjl.html?page=3&tableId=15608&pageSize=2&token=IgtdMORZvrU7BZ4PGBBPLK34U4cNuB5F&startTime=2020-01-01&endTime=2020-04-30";
//
//get方式请求数据
//
//请求地址：http://jsonplaceholder.typicode.com/posts
//
@Ignore("lue")
//
@Test
//
public void test1() {
//
System.out.print("n" + "test1---------------------------" + "n");
//
HttpParamers paramers = HttpParamers.httpGetParamers();
//
//
try {
//
HttpService httpService = new HttpService(uri);
//
String response = httpService.service("/posts", paramers);
//
JSONObject jsonObject = JSONObject.parseObject(response);
//
List<JSONObject> list = (List) jsonObject.get("markets");
//
for (JSONObject object : list) {
//
System.out.println(object.get("url"));
//
}

System.out.println("response: " + JSON.toJSONString(jsonObject));
//
} catch (Exception e) {
//
e.printStackTrace();
//
}
//
//
}
//
String uris = "http://open.api.mysteel.cn/zjsj/api_marketd_zjsjd.html?token=IgtdMORZvrU7BZ4PGBBPLK34U4cNuB5F&id=gn1vepj2he5di";
//
//
//get方式请求数据
//
//请求地址：http://jsonplaceholder.typicode.com/posts?userId=5
//
@Ignore("lue")
//
@Test
//
public void test2() {
//
System.out.print("n" + "test2---------------------------" + "n");
//
HttpParamers paramers = HttpParamers.httpGetParamers();
//
paramers.addParam("userId", "5");
//
String response = "";
//
try {
//
HttpService httpService = new HttpService(uris);
//
response = httpService.service("/posts", paramers);
//
} catch (Exception e) {
//
e.printStackTrace();
//
}
//
System.out.print(response);
//
}
//post方式请求数据
//请求地址：http://jsonplaceholder.typicode.com/posts
@Test
public void test3() throws Exception {
System.out.print("n" + "test3---------------------------" + "n");
String data = RemoteDataUtil.getData();
System.out.println(data);
}
public static void main(String[] args) {
System.out.println();
}
}