python : Speed up Reverse DNS lookup

Question

我计划在 4700 万个 ip 上运行反向 DNS。这是我的代码

with open(file,'r') as f:
    with open ('./ip_ptr_new.txt','a') as w:

        for l in f:

            la = l.rstrip('\n')
            ip,countdomain = la.split('|')
            ips.append(ip)

           try:
                ais = socket.gethostbyaddr(ip)
                print ("%s|%s|%s" % (ip,ais[0],countdomain), file = w)    
           except:
                print ("%s|%s|%s" % (ip,"None",countdomain), file = w)

目前速度非常慢。有人有任何加快速度的建议吗？

Answer 1

尝试使用多处理模块。我对大约 8000 ips 的性能进行了计时，得到了以下结果:

#dns.py
real    0m2.864s
user    0m0.788s
sys     0m1.216s


#slowdns.py
real    0m17.841s
user    0m0.712s
sys     0m0.772s


# dns.py
from multiprocessing import Pool
import socket
def dns_lookup(ip):
    ip, countdomain = ip
    try:
        ais = socket.gethostbyaddr(ip)
        print ("%s|%s|%s" % (ip,ais[0],countdomain))
    except:
        print ("%s|%s|%s" % (ip,"None",countdomain))

if __name__ == '__main__':
    filename = "input.txt"
    ips = []
    with open(filename,'r') as f:
        with open ('./ip_ptr_new.txt','a') as w:
            for l in f:
                la = l.rstrip('\n')
                ip,countdomain = la.split('|')
                ips.append((ip, countdomain))
    p = Pool(5)
    p.map(dns_lookup, ips)





#slowdns.py
import socket
from multiprocessing import Pool

filename = "input.txt"
if __name__ == '__main__':
    ips = []
    with open(filename,'r') as f:
        with open ('./ip_ptr_new.txt','a') as w:
            for l in f:
               la = l.rstrip('\n')
               ip,countdomain = la.split('|')
               ips.append(ip)
               try:
                    ais = socket.gethostbyaddr(ip)
                    print ("%s|%s|%s" % (ip,ais[0],countdomain), file = w)
               except:
                    print ("%s|%s|%s" % (ip,"None",countdomain), file = w)