1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
import java.io.*;
import java.net.HttpURLConnection;
import java.net.URL;
import java.time.LocalDate;
import java.time.format.DateTimeFormatter;
import java.util.Scanner;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class HanFu {
    private static int startPage = 1; //开始页
    //    private int endPage = 125; //结束页
    private static int endPage = 3; //结束页 这里只处理3页
    private static String rootUrl = "https://www.aihanfu.com/zixun/tushang-";

    public static void main(String[] args) {
        everyPage(startPage, endPage);
    }

    public static void everyPage(int startPage, int endPage) {
        for (int page = startPage; page <= endPage; page++) {
            System.out.printf("处理第 [%d] 页的数据；n", page);

            String pageUrl = rootUrl + page;
//            System.out.println(pageUrl + "n");
            String html = getUrlHtml(pageUrl);
//            System.out.println(html);

            //<h3 class="yh"><a target="_blank" href="http://www.aihanfu.com/wen/8332/">【汉服美景】天地之间，你是另一种绝色</a></h3>
            String pattern = "<h3 class="yh"><a target="_blank" href="(.*?)">([^<]+)</a>";
//            Pattern patternCompile = Pattern.compile(pattern); //, Pattern.MULTILINE
            Pattern patternCompile = Pattern.compile(pattern); //, Pattern.MULTILINE

            Matcher matcher = patternCompile.matcher(html);
//            System.out.println(matcher.matches());//字符串中是否包含了
//            System.out.println(matcher.find());//字符串中是否找到数字串

            if (!matcher.find()) {
                System.out.printf("第 [%d] 页未匹配到数据，执行下一页n", page);
                continue;
            }

            while (matcher.find()) {
                //方法一：打印匹配到的整个表达式
//                System.out.println(matcher.group());
                //方法二：打印匹配到的整个表达式
//                System.out.println(matcher.group(0));

                //打印匹配到的结果内容
//                System.out.println(matcher.group(1));
//                System.out.println(matcher.group(2));
                String detailUrl = matcher.group(1); //每篇文章的地址
                String detailTitle = matcher.group(2); //每篇文章的标题

                evertArticle(detailUrl, detailTitle);

                //记录日志
                logger(detailTitle + "-" + detailUrl + "n");
            }

            /*if (matcher.find()) { //查找匹配到的第一条数据
                System.out.println(matcher.group(0) + "n");
                System.out.println(matcher.group(1) + "n");
                System.out.println(matcher.group(2) + "n");
            } else {
                System.out.println("没有匹配到数据");
            }*/

//            System.out.print(matcher.groupCount()); //表达式有多少个分组（结果集），（不包含特殊的组（group(0)）它总是代表整个表达式。该组不包括在 groupCount 的返回值中。）
        }
    }

    //处理每一篇文章
    public static Boolean evertArticle(String detailUrl, String detailTitle) {
        String html = getUrlHtml(detailUrl);

        //详情页一张图片的Html内容如下，我们可以写匹配的语句了
        //<figure class="image"><img src="https://static.aihanfu.net/uploadfile/2022/1007/20221007094025127.jpeg"></figure>
        String pattern = "<figure class="image"><img src="(.*?)"></figure>";
        Pattern patternCompile = Pattern.compile(pattern);
        Matcher matcher = patternCompile.matcher(html);

//        System.out.println(matcher.find());
//        System.exit(1);

        String dir = createDir(detailTitle); //根据标题名创建文件夹

        while (matcher.find()) {
            //方法一：打印匹配到的整个表达式
//                System.out.println(matcher.group());
            //方法二：打印匹配到的整个表达式
//                System.out.println(matcher.group(0));

            //打印匹配到的结果内容
//                System.out.println(matcher.group(1)); //图片的地址
            String imageUrl = matcher.group(1); //图片的地址
//            System.out.println(imageUrl);

            int lastEqIndex = imageUrl.lastIndexOf("/"); //匹配最右边的/位置
            String fileName = imageUrl.substring(lastEqIndex); //带/的原始文件名
            String savePath = dir + fileName; //图片保存路径

            //图片下载
            try {
                downLoadFile(imageUrl, savePath);
            } catch (IOException e) {
                e.printStackTrace();
            }

            System.out.printf("---- 图片 [ %s ] 下载 [ %s ] 成功n", imageUrl, savePath);
        }
        return true;
    }


    //获取url的html内容
    public static String getUrlHtml(String pageUrl) {
        String html = "";
        try {
            URL url = new URL(pageUrl);
            HttpURLConnection conn = (HttpURLConnection) url.openConnection();
            //设置超时时间为3秒
            conn.setConnectTimeout(3 * 1000);
            //防止屏蔽程序抓取而返回403错误
            conn.setRequestProperty("User-Agent", "Mozilla/4.0 (compatible; MSIE 5.0; Windows NT; DigExt)");
            conn.setRequestMethod("GET");
            Scanner scanner = new Scanner(conn.getInputStream());
            while (scanner.hasNextLine()) {
                html += scanner.nextLine();
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
        return html;
    }

    //写日志
    public static void logger(String content) {
        String dir = "logger/";

        //创建文件夹
        createDirectory(dir);

        String fileName = getTodayDate() + ".txt";

        try {
            BufferedWriter out = new BufferedWriter(new FileWriter(fileName, true)); //true，文件内容支持追加
            out.write(content);
            out.close();
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    //根据标题创建文件夹
    public static String createDir(String title) {
        //日期格式化
        String dateStr = getTodayDate();

        //目录拼接
        String dir = "image/" + dateStr + "/" + title;

        //创建文件夹
        createDirectory(dir);

        return dir;
    }

    //尝试递归创建文件夹
    public static void createDirectory(String filePath) {
        File file = new File(filePath);
        if (!file.exists() || !file.isDirectory()) { //文件不存在 或 不是文件夹
            boolean result = file.mkdirs();
            if (!result) {
                die("文件夹创建失败");
            }
        }
    }

    //获取今天的日期
    public static String getTodayDate() {
        LocalDate date = LocalDate.now();
        DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd"); //年-月-日
        String dateStr = date.format(formatter);
        return dateStr;
    }

    //下载文件
    public static void downLoadFile(String imageUrl, String savePath) throws IOException {
        URL url = new URL(imageUrl);
        BufferedInputStream bufferedInputStream = new BufferedInputStream(url.openStream());
        FileOutputStream fileOutputStream = new FileOutputStream(savePath);
        byte[] buffer = new byte[1024];
        int count = 0;
        while ((count = bufferedInputStream.read(buffer, 0, 1024)) != -1) {
            fileOutputStream.write(buffer, 0, count);
        }
        bufferedInputStream.close();
        fileOutputStream.close();
    }

    //执行程序结束操作
    public static void die(String msg) {
        if (msg != "") {
            System.out.println(msg);
        }
        System.exit(1);
    }
}