From 36e9998fe2b1de4b49764f464b4d32e9c42b5c20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20B=C3=A9raud?= <adrien.beraud@savoirfairelinux.com> Date: Tue, 1 Nov 2016 02:07:55 -0400 Subject: [PATCH] scanner: use urllib3 --- python/tools/scanner.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/python/tools/scanner.py b/python/tools/scanner.py index 34628cfa..baa001a9 100755 --- a/python/tools/scanner.py +++ b/python/tools/scanner.py @@ -18,7 +18,7 @@ import time, sys, os from pprint import pprint from math import cos, sin, pi -from urllib import request +import urllib3 import gzip import asyncio @@ -35,6 +35,8 @@ from mpl_toolkits.basemap import Basemap import GeoIP +http = urllib3.PoolManager() + done = 0 all_nodes = NodeSet() @@ -72,7 +74,7 @@ def check_dl(fname, url): if os.path.isfile(fname): return print('downloading', url) - ghandle = gzip.GzipFile(fileobj=request.urlopen(url)) + ghandle = gzip.GzipFile(fileobj=http.request('GET', url, headers={'User-Agent': 'Mozilla/5.0'})) with open(fname, 'wb') as out: for line in ghandle: out.write(line) -- GitLab