爬虫-07-反爬虫

 1 #middlewares.py文件
 2 from fake_useragent import UserAgent #这是一个随机UserAgent的包，里面有很多UserAgent
 3 class RandomUserAgentMiddleware(object):
 4     def __init__(self, crawler):
 5         super(RandomUserAgentMiddleware, self).__init__()
 6 
 7         self.ua = UserAgent()
 8         self.ua_type = crawler.settings.get('RANDOM_UA_TYPE', 'random') #从setting文件中读取RANDOM_UA_TYPE值
 9 
10     @classmethod
11     def from_crawler(cls, crawler):
12         return cls(crawler)
13 
14     def process_request(self, request, spider):
15         def get_ua():
16             '''Gets random UA based on the type setting (random, firefox…)'''
17             return getattr(self.ua, self.ua_type) 
18 
19         user_agent_random=get_ua()
20         request.headers.setdefault('User-Agent', user_agent_random) #这样就是实现了User-Agent的随即变换

1 #settings.py文件
2 DOWNLOADER_MIDDLEWARES = {
3    'Lagou.middlewares.RandomUserAgentMiddleware': 543,
4    'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware':None, #这里要设置原来的scrapy的useragent为None，否者会被覆盖掉
5 }
6 RANDOM_UA_TYPE='random'

 1 #middlewares.py文件
 2 class RandomProxyMiddleware(object):
 3     '''动态设置ip代理'''
 4     def process_request(self,request,spider):
 5         get_ip = GetIP() #这里的函数是传值ip的
 6         request.meta["proxy"] = get_ip
 7         #例如
 8         #get_ip = GetIP() #这里的函数是传值ip的
 9         #request.meta["proxy"] = 'http://110.73.54.0:8123'
10 
11 
12 #settings.py文件
13 DOWNLOADER_MIDDLEWARES = {
14    'Lagou.middlewares.RandomProxyMiddleware':542,
15    'Lagou.middlewares.RandomUserAgentMiddleware': 543,
16    'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware':None, #这里要设置原来的scrapy的useragent为None，否者会被覆盖掉
17 }

1 select ip,port from proxy_ip
2 order by rand()
3 limit 1

1 from scrapy.selector import Selector
2 html=requests.get(url)
3 Selector=Selector(text=html.text)
4 Selector.xpath()

1  if __name__ == "__main__":
2      get_ip=GetIp()
3      get_ip.get_random_ip()

1 #在spider.py文件中
2 custom_settings={
3      "COOKIES_ENABLED":True,
4       "":"",
5       "":"",
6 }