C# 爬虫简单教程

268 阅读 0 评论 177 点赞

我是靠谱客的博主怕孤独橘子，这篇文章主要介绍C# 爬虫简单教程，现在分享给大家，希望可以做个参考。

1、使用第三方类库 HtmlAgilityPack

官方网址：https://html-agility-pack.net/?z=codeplex、

复制代码

1
2
3
4
5
6
7
8
9
10
11
12
// From File 从文件获取html信息
var doc = new HtmlDocument();
doc.Load(filePath);

// From String 从字符串获取html信息
var doc = new HtmlDocument();
doc.LoadHtml(html);

// From Web  从网址获取html信息
var url = "http://html-agility-pack.net/";
var web = new HtmlWeb();
var doc = web.Load(url);

1.1、这里介绍一下最后一种用法

复制代码

1
2
var web = new HtmlWeb();
var doc = web.Load(url);

在 web 中我们还可以设置cookie、headers等信息，来处理一些特定的网站需求，比如需要登陆等。

1.2 用法解释

网页在你查看网页源代码之后只是一段字符串，而爬虫所做的就是在这堆字符串中，查询到我们想要的信息，挑选出来。
以往的筛选方法：正则（太麻烦了，写起来有些头疼）
HtmlAgilityPack 支持通过XPath来解析我们需要的信息。

1.2.1 在哪里找XPath？

网页右键检查

通过XPath就可以准确获取你想要元素的全部信息。

1.2.2 获取选中Html元素的信息？

获取选中元素

复制代码

1
2
3
var web = new HtmlWeb();
var doc = web.Load(url);
var htmlnode = doc?.DocumentNode?.SelectSingleNode("/html/body/header")

获取元素信息

复制代码

1
2
3
4
htmlnode.InnerText;
htmlnode.InnerHtml;
//根据属性取值
htmlnode?.GetAttributeValue("src", "未找到")

2、自己封装的类库

复制代码

/// <summary>
  /// 下载HTML帮助类
  /// </summary>
  public static class LoadHtmlHelper
  {
    /// <summary>
    /// 从Url地址下载页面
    /// </summary>
    /// <param name="url"></param>
    /// <returns></returns>
    public async static ValueTask<HtmlDocument> LoadHtmlFromUrlAsync(string url)
    {
      HtmlWeb web = new HtmlWeb();
       return await
         web?.LoadFromWebAsync(url);
    }

/// <summary>
    /// 获取单个节点扩展方法
    /// </summary>
    /// <param name="htmlDocument">文档对象</param>
    /// <param name="xPath">xPath路径</param>
    /// <returns></returns>
    public static HtmlNode GetSingleNode(this HtmlDocument htmlDocument, string xPath)
    {
     return htmlDocument?.DocumentNode?.SelectSingleNode(xPath);
    }

/// <summary>
    /// 获取多个节点扩展方法
    /// </summary>
    /// <param name="htmlDocument">文档对象</param>
    /// <param name="xPath">xPath路径</param>
    /// <returns></returns>
    public static HtmlNodeCollection GetNodes(this HtmlDocument htmlDocument, string xPath)
    {
      return htmlDocument?.DocumentNode?.SelectNodes(xPath);
    }

/// <summary>
    /// 获取多个节点扩展方法
    /// </summary>
    /// <param name="htmlDocument">文档对象</param>
    /// <param name="xPath">xPath路径</param>
    /// <returns></returns>
    public static HtmlNodeCollection GetNodes(this HtmlNode htmlNode, string xPath)
    {
      return htmlNode?.SelectNodes(xPath);
    }

/// <summary>
    /// 获取单个节点扩展方法
    /// </summary>
    /// <param name="htmlDocument">文档对象</param>
    /// <param name="xPath">xPath路径</param>
    /// <returns></returns>
    public static HtmlNode GetSingleNode(this HtmlNode htmlNode, string xPath)
    {
      return htmlNode?.SelectSingleNode(xPath);
    }

/// <summary>
    /// 下载图片
    /// </summary>
    /// <param name="url">地址</param>
    /// <param name="filpath">文件路径</param>
    /// <returns></returns>
    public async static ValueTask<bool> DownloadImg(string url ,string filpath)
    {
      HttpClient httpClient = new HttpClient();
      try
      {
        var bytes = await httpClient.GetByteArrayAsync(url);
        using (FileStream fs = File.Create(filpath))
        {
          fs.Write(bytes, 0, bytes.Length);
        }
        return File.Exists(filpath);
      }
      catch (Exception ex)
      {
       
        throw new Exception("下载图片异常", ex);
      }
      
    }
  }

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
/// <summary>
  /// 下载HTML帮助类
  /// </summary>
  public static class LoadHtmlHelper
  {
    /// <summary>
    /// 从Url地址下载页面
    /// </summary>
    /// <param name="url"></param>
    /// <returns></returns>
    public async static ValueTask<HtmlDocument> LoadHtmlFromUrlAsync(string url)
    {
      HtmlWeb web = new HtmlWeb();
       return await
         web?.LoadFromWebAsync(url);
    }

    /// <summary>
    /// 获取单个节点扩展方法
    /// </summary>
    /// <param name="htmlDocument">文档对象</param>
    /// <param name="xPath">xPath路径</param>
    /// <returns></returns>
    public static HtmlNode GetSingleNode(this HtmlDocument htmlDocument, string xPath)
    {
     return htmlDocument?.DocumentNode?.SelectSingleNode(xPath);
    }

    /// <summary>
    /// 获取多个节点扩展方法
    /// </summary>
    /// <param name="htmlDocument">文档对象</param>
    /// <param name="xPath">xPath路径</param>
    /// <returns></returns>
    public static HtmlNodeCollection GetNodes(this HtmlDocument htmlDocument, string xPath)
    {
      return htmlDocument?.DocumentNode?.SelectNodes(xPath);
    }

   

    /// <summary>
    /// 获取多个节点扩展方法
    /// </summary>
    /// <param name="htmlDocument">文档对象</param>
    /// <param name="xPath">xPath路径</param>
    /// <returns></returns>
    public static HtmlNodeCollection GetNodes(this HtmlNode htmlNode, string xPath)
    {
      return htmlNode?.SelectNodes(xPath);
    }


    /// <summary>
    /// 获取单个节点扩展方法
    /// </summary>
    /// <param name="htmlDocument">文档对象</param>
    /// <param name="xPath">xPath路径</param>
    /// <returns></returns>
    public static HtmlNode GetSingleNode(this HtmlNode htmlNode, string xPath)
    {
      return htmlNode?.SelectSingleNode(xPath);
    }

    /// <summary>
    /// 下载图片
    /// </summary>
    /// <param name="url">地址</param>
    /// <param name="filpath">文件路径</param>
    /// <returns></returns>
    public async static ValueTask<bool> DownloadImg(string url ,string filpath)
    {
      HttpClient httpClient = new HttpClient();
      try
      {
        var bytes = await httpClient.GetByteArrayAsync(url);
        using (FileStream fs = File.Create(filpath))
        {
          fs.Write(bytes, 0, bytes.Length);
        }
        return File.Exists(filpath);
      }
      catch (Exception ex)
      {
       
        throw new Exception("下载图片异常", ex);
      }
      
    }
  }