17.获取代理ip

代码语言：javascript复制

import redis
import telnetlib
import urllib.request
from bs4 import BeautifulSoup

r = redis.Redis(host='127.0.0.1', port=6379)

for d in range(1, 3):  # 采集1到2页
    scrapeUrl = 'http://www.xicidaili.com/nn/%d/' % d
    req = urllib.request.Request(scrapeUrl)
    req.add_header('User-Agent', 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)')
    response = urllib.request.urlopen(req)
    html = response.read()

    bsObj = BeautifulSoup(html, "html.parser")

    for i in range(100):
        speed = float(bsObj.select('td')[6   i * 10].div.get('title').replace('秒', ''))
        if speed < 0.2:  # 验证速度，只要速度在0.2秒之内的
            ip = bsObj.select('td')[1   i * 10].get_text()
            port = bsObj.select('td')[2   i * 10].get_text()
            ip_address = 'http://'   ip   ':'   port
            try:
                telnetlib.Telnet(ip, port=port, timeout=2)  # 用telnet对ip进行验证
            except:
                print('fail')
            else:
                print('sucess：'   ip_address)
                r.sadd('ippool', ip_address)  # 可用的ip导入到redis
                f = open('proxy_list.txt', 'a')
                f.write(ip_address   'n')
                f.close()

ip 代理

0 人点赞