from ctypes.wintypes import DOUBLE
from multiprocessing import Pool
import os, time, random
from ftplib import FTP
import datetime
import os, time
import multiprocessing
import gc
import threading
import sys
import random, requests
SUFFIXES = {1000: ['KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB'],
1024 : [ 'KiB' , 'MiB' , 'GiB' , 'TiB' , 'PiB' , 'EiB' , 'ZiB' , 'YiB' ]}
def approximate_size ( size , a_kilobyte_is_1024_bytes = True ):
'''Convert a file size to human-readable form.
Keyword arguments:
size -- file size in bytes
a_kilobyte_is_1024_bytes -- if True (default), use multiples of 1024
if False, use multiples of 1000
Returns: string
'''
if size < 0 :
raise ValueError ( 'approximate_size: number must be non-negative' )
multiple = 1024 if a_kilobyte_is_1024_bytes else 1000
for suffix in SUFFIXES [ multiple ]:
size /= multiple
if size < multiple :
return '{0:.1f} {1}' . format ( size , suffix )
raise ValueError ( 'approximate_size: number too large' )
urllist=[]
def getUrlList():
with open('url.txt','r') as f:
for line in f:
urllist.append(list(line.strip('\n').split(',')))
print(urllist)
lastTime = datetime.datetime.now()
lasttotal =0
nump = 0
def getnump():
while True:
ten=input("请输入进程数量（1-99）:")
try:
x=eval(ten)
if (type(x)==int) and (x in range(0, 100)):
break
except:
pass
return x
def getcontent_length(content_length):
try:
x=eval(content_length)
if (type(x)==int) :
return x
except:
return 0
return 0
speed = 0
starttime = datetime.datetime.now()
def worker(urllist, i, l, totalOfTotal, nump):
def printdata(data, content_length, total, url, l, totalOfTotal, nump ):
lendata = len(data)
global lastTime, lasttotal, speed, starttime
l.acquire()
try:
content_length = max(content_length, 1)
thistime = datetime.datetime.now()
interval = int(max(nump/2, 1))
total_seconds = max(1, (thistime - lastTime ).total_seconds())
start_seconds = max(1, (thistime - starttime ).total_seconds())
if start_seconds < interval :
speed = (total- lasttotal)/start_seconds
if ( (thistime - lastTime ).total_seconds() > interval):
speed = (total- lasttotal)/interval
print(strpid, thistime," ", lendata, " ",
approximate_size(speed),'/S',start_seconds, 'seconds',
approximate_size(content_length), 'percent: {:.2%}'.format(total/content_length), url,
approximate_size(lasttotal), approximate_size(total), approximate_size(totalOfTotal.value))
if (speed > 600000):
print('=============================================')
lastTime = thistime
lasttotal = total
except Exception as e:
print('printdata:', url, str(e))
finally:
l.release()
total = 0
try:
while 1:
port = 21
timeout = 22
strpid = "进程号" + str(os.getpid()) + ' '+ str(i)+': '
rlist = random.randint(1,len(urllist))-1
rurl = urllist[rlist]
print(strpid, 'random:', rlist, ' ', rurl )
try:
filename = 'RETR ' + "1G.rar"
r = requests.get( url=rurl[0], stream=True )
print(strpid, rurl, r.headers, r.headers.get('content-length') )
content_length = 0
content_length = getcontent_length(r.headers.get('content-length'))
global lasttotal, starttime, uplimit
lasttotal = 0
starttime = datetime.datetime.now()
for chunk in r.iter_content(chunk_size=1024):
if chunk:
total = total + len(chunk)
totalOfTotal.value = totalOfTotal.value + len(chunk)
printdata(chunk,content_length, total, rurl, l, totalOfTotal, nump)
global speed
total_seconds = max((datetime.datetime.now()- starttime).total_seconds(), 1)
if (total_seconds>22) and(total>10000000) and (speed < 600000):
break
if (total_seconds>55) and (speed < 100000):
break
if total>content_length:
break
if totalOfTotal.value >uplimit:
break
except Exception as e:
print(e.args)
print(strpid, rurl, str(e))
print(repr(e))
time.sleep(3)
print("进程号:" + str(os.getpid()) + " done!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! " , 'speed:',
approximate_size(speed),'/S',approximate_size(total),total_seconds,'seconds', rurl)
except Exception as e:
print('printdata:', rurl, str(e))
f=open('error.txt', 'a+')
f.write(str(datetime.datetime.now())+"进程号:" + str(os.getpid())+' '+'speed:'+
approximate_size(speed)+'/S '+approximate_size(total)+str(total_seconds) +'seconds'+ rurl)
f.close
def puttotalOfTotal(totalOfTotal):
global totalOfTotalname
with open(totalOfTotalname,'w') as f:
f.write(str(totalOfTotal.value))
totalOfTotalname = 'totalOfTotal.txt'
def gettotalOfTotal():
global totalOfTotalname
if os.path.exists(totalOfTotalname):
with open(totalOfTotalname,'r') as f:
s = f.read()
print('s', s)
return(s)
return '0'
uplimit = 8400000000000
if __name__ == '__main__':
try:
multiprocessing.freeze_support()
getUrlList()
if os.path.exists(os.getcwd()):
s_list = os.listdir(os.getcwd())
for s_list_line in s_list:
if "CPE_D_" in s_list_line:
os.remove(os.getcwd() + "\\" + s_list_line)
nump = getnump()
plist = []
lock = multiprocessing.Lock()
totalOfTotal = multiprocessing.Value('d', 0)
totalOfTotal.value = float(gettotalOfTotal())
print('gettotalOfTotal()', totalOfTotal.value)
for i in range(nump):
p = multiprocessing.Process(target=worker, args=(urllist,i, lock, totalOfTotal, nump))
p.start()
plist.append(p)
gc.collect()
while 1:
time.sleep(11)
puttotalOfTotal(totalOfTotal)
active_children = len(multiprocessing.active_children())
print('multiprocessing.active_childrennnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn()', active_children)
if active_children < nump :
p = multiprocessing.Process(target=worker, args=(urllist,active_children, lock, totalOfTotal, nump))
p.start()
plist.append(p)
gc.collect()
po.close()
po.join()
except Exception as e:
print('__main__', str(e))
time.sleep(2)
|