[网络协议]python爬虫之有效代理多线程爬取并检测有效ip并存储

本案例只做学习，不做他用！

import random
import requests
from lxml import etree
import telnetlib
import threading

UA=[
    "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; AcooBrowser; .NET CLR 1.1.4322; .NET CLR 2.0.50727)",
    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; Acoo Browser; SLCC1; .NET CLR 2.0.50727; Media Center PC 5.0; .NET CLR 3.0.04506)",
    "Mozilla/4.0 (compatible; MSIE 7.0; AOL 9.5; AOLBuild 4337.35; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)",
    "Mozilla/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US)",
    "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 2.0.50727; Media Center PC 6.0)",
    "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 1.0.3705; .NET CLR 1.1.4322)",
    "Mozilla/4.0 (compatible; MSIE 7.0b; Windows NT 5.2; .NET CLR 1.1.4322; .NET CLR 2.0.50727; InfoPath.2; .NET CLR 3.0.04506.30)",
    "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN) AppleWebKit/523.15 (KHTML, like Gecko, Safari/419.3) Arora/0.3 (Change: 287 c9dfb30)",
    "Mozilla/5.0 (X11; U; Linux; en-US) AppleWebKit/527+ (KHTML, like Gecko, Safari/419.3) Arora/0.6",
    "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.2pre) Gecko/20070215 K-Ninja/2.1.1",
    "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9) Gecko/20080705 Firefox/3.0 Kapiko/3.0",
    "Mozilla/5.0 (X11; Linux i686; U;) Gecko/20070322 Kazehakase/0.4.5",
    "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.8) Gecko Fedora/1.9.0.8-1.fc10 Kazehakase/0.5.6",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/535.20 (KHTML, like Gecko) Chrome/19.0.1036.7 Safari/535.20",
    "Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; fr) Presto/2.9.168 Version/11.52",
]
ua=random.choice(UA)
headers={'User-Agent':ua}
m=1
def get_html(url):
    html=requests.get(url,headers=headers).text
    get_ip(html)

def get_ip(html):
    global m
    lst=[]
    #xpath获取ip并组成字典形式
    resp=etree.HTML(html)
    ip=resp.xpath('//tbody/tr/td[1]/text()')
    port=resp.xpath('//tbody/tr/td[2]/text()')
    ip_prot=zip(ip,port)
    print(f'待检测ip(第{m}页)')
    m+=1
    for ip1,port1 in ip_prot:
        print(ip1.strip(),':',port1.strip())
        item={'ip':ip1.strip(),'port':port1.strip()}
        lst.append(item)
    print('-' * 50)
    save_ip(lst)

def save_ip(lst):
    #判断ip是否可用
    # for lst1 in lst:
    #     url = "http://icanhazip.com/"
    #     proxies={
    #     'http':"http://{}:{}".format(lst1['ip'], lst1['port']),
    #     'https':"https://{}:{}".format(lst1['ip'], lst1['port'])
    #     }
    #     print(proxies)
    #     try:
    #         res = requests.get(url, proxies=proxies, timeout=3)
    #         if res.status_code == 200:
    #             # 存储ip到txt中
    #             with open('有效ip.txt', 'a', encoding='utf-8') as f:
    #                 f.write(lst1['ip']+':'+lst1['port']+'\n')
    #     except Exception as e:
    #         print(e)
    print('开始检测ip是否可用：')
    for item in lst:
        ip = item["ip"]
        port = item["port"]

        try:
            tn = telnetlib.Telnet(ip, port=port, timeout=3)
        except:
            print('[-] ip:{}:{}'.format(ip, port))
        else:
            print('[+] ip:{}:{}'.format(ip, port))
            with open('有效ip.txt', 'a') as f:
                f.write(ip+':'+port+'\n')
    print("阶段性检测完毕，并已经存储到有效ip文件中！")




if __name__ == '__main__':
    # url='https://www.89ip.cn/index_1.html'
    s=threading.BoundedSemaphore(5)
    for n in range(1,74):
        url=f'https://www.89ip.cn/index_{n}.html'
        t=threading.Thread(target=get_html,args=(url,))
        t.start()