Skip to content
Snippets Groups Projects
Commit 36e9998f authored by Adrien Béraud's avatar Adrien Béraud
Browse files

scanner: use urllib3

parent b2529c24
No related branches found
No related tags found
No related merge requests found
...@@ -18,7 +18,7 @@ ...@@ -18,7 +18,7 @@
import time, sys, os import time, sys, os
from pprint import pprint from pprint import pprint
from math import cos, sin, pi from math import cos, sin, pi
from urllib import request import urllib3
import gzip import gzip
import asyncio import asyncio
...@@ -35,6 +35,8 @@ from mpl_toolkits.basemap import Basemap ...@@ -35,6 +35,8 @@ from mpl_toolkits.basemap import Basemap
import GeoIP import GeoIP
http = urllib3.PoolManager()
done = 0 done = 0
all_nodes = NodeSet() all_nodes = NodeSet()
...@@ -72,7 +74,7 @@ def check_dl(fname, url): ...@@ -72,7 +74,7 @@ def check_dl(fname, url):
if os.path.isfile(fname): if os.path.isfile(fname):
return return
print('downloading', url) print('downloading', url)
ghandle = gzip.GzipFile(fileobj=request.urlopen(url)) ghandle = gzip.GzipFile(fileobj=http.request('GET', url, headers={'User-Agent': 'Mozilla/5.0'}))
with open(fname, 'wb') as out: with open(fname, 'wb') as out:
for line in ghandle: for line in ghandle:
out.write(line) out.write(line)
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment