How to Make Sure Python Urllib request urlretrieve Finishes Before Continuing
Python urllib.request() Examples
The following are 30 code examples of urllib.request() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may also want to check out all available functions/classes of the module urllib , or try the search function 
.
Example #1
def get_response(url): headers = {} headers['User-Agent'] = 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6' req = urllib.request.Request( url = url, headers = headers ) #try: response = urllib.request.urlopen(req) data = response.read() #except: # 抓取出错 # return None if response.info().get('Content-Encoding') == 'gzip': data = ungzip(data) elif response.info().get('Content-Encoding') == 'deflate': data = undeflate(data) response.data = data return response ## 抓取网页html Example #2
def get_filesize(url, timeout=15): ''' Fetches file's size of a file over HTTP. :param url: Url address. :type url: string :param timeout: Timeout in seconds. Default is 15. :type timeout: int :returns: Size in bytes. :rtype: int ''' try: urlObj = urllib.request.urlopen(url, timeout=timeout) file_size = int(urlObj.headers["Content-Length"]) except (IndexError, KeyError, TypeError, urllib.error.HTTPError, urllib.error.URLError): return 0 return file_size
Example #3
def request(self, method: str, url: str, body=None, headers=None, **client_options): client = self.create_client(**client_options) url, headers, body = client.sign(url, method, body, headers) request = urllib.request.Request(url, body, headers, method=method) try: return urllib.request.urlopen(request) except urllib.error.HTTPError as e: logger = logging.getLogger(__name__ + '.StashTeam.request') logger.exception( '[%s %s] %s\nrequest headers: %r\nrequest body: %r\n' 'client_options: %r\nresponse status: %r\n' 'response headers: %r\nresponse body: %r', method, url, e, headers, body, client_options, e.code, dict(e.headers), e.read() ) raise
Example #4
def register(self, identity: Identity, public_key: PKey) -> None: team = self.team if not (isinstance(team, identity.team_type) and cast(str, identity.identifier).startswith(team.server_url)): return data = json.dumps({ 'text': format_openssh_pubkey(public_key) }) try: self.request( identity, 'POST', self.REGISTER_URL.format(self.team), data, headers={'Content-Type': 'application/json'} ) except urllib.error.HTTPError as e: if e.code == 409: errors = json.loads(e.read().decode('utf-8'))['errors'] raise DuplicatePublicKeyError(errors[0]['message']) raise Example #5
def feature_popup_content(request): url = request.POST.get("url", None) if url is not None: host = "{uri.hostname}".format(uri=urlparse(url)) try: if host in settings.ALLOWED_POPUP_HOSTS: if url is not None: f = urllib.request.urlopen(url) return HttpResponse(f.read()) else: raise Exception() except: return HttpResponseNotFound() else: return HttpResponseNotFound() Example #6
def get(self, rest_query): """ Sends a GET request to the Alyx server. Will raise an exception on any status_code other than 200, 201. For the dictionary contents and list of endpoints, refer to: https://alyx.internationalbrainlab.org/docs :param rest_query: example: '/sessions?user=Hamish'. :type rest_query: str :return: (dict/list) json interpreted dictionary from response """ rep = self._generic_request(requests.get, rest_query) if isinstance(rep, dict) and list(rep.keys()) == ['count', 'next', 'previous', 'results']: if len(rep['results']) < rep['count']: rep = _PaginatedResponse(self, rep) else: rep = rep['results'] return rep
Example #7
def patch(self, rest_query, data=None): """ Sends a PATCH request to the Alyx server. For the dictionary contents, refer to: https://alyx.internationalbrainlab.org/docs :param rest_query: (required)the endpoint as full or relative URL :type rest_query: str :param data: json encoded string or dictionary :type data: None, dict or str :return: response object """ if isinstance(data, dict): data = json.dumps(data) return self._generic_request(requests.patch, rest_query, data=data)
Example #8
def getResource(self, url, uuid): """Provides the resource""" if self.__cacheDir is None: return None if url in self.__urlToFileName: return self.__urlToFileName[url] fName = self.__cacheDir + hashlib.md5(url.encode('utf-8')).hexdigest() if fName in self.__threads: # Reject double request return None thread = ResourceRetriever() self.__threads[fName] = thread self.__connectThread(thread) thread.get(url, fName, uuid) return None Example #9
def _url_status(url): parse_obj = urllib.parse.urlparse(url) timer = 1 for i in range(6): try: connection = http.client.HTTPConnection(parse_obj.netloc) connection.request('HEAD', parse_obj.path) break except Exception as e: print(url, e, 'sleep', timer) time.sleep(timer) timer *= 2 else: return e response = connection.getresponse() connection.close() return response.status Example #10
def get_url(url): req_headers = { 'User-Agent': 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/525.13 (KHTML, like Gecko) Chrome/0.A.B.C Safari/525.13', 'Referer': url } request = urllib.request.Request(url, headers=req_headers) opener = urllib.request.build_opener() timer = 1 for i in range(1): try: return opener.open(request).read() except: time.sleep(timer) timer *= 2 raise IOError("Unable to download `%s`."%url) Example #11
def download_files(self, urls, env, headers=None, dir=None): if dir is None: dir = env.get('cache-dir') dest_directory = os.path.join(dir, "tmp_" + str(uuid.uuid4())) if not os.path.exists(dest_directory): os.makedirs(dest_directory) for data_url in urls: if isinstance(data_url, tuple): data_url, download_file = data_url else: download_file = data_url.split('/')[-1] download_path = os.path.join(dest_directory, download_file) if headers: opener = urllib.request.build_opener() opener.addheaders = headers urllib.request.install_opener(opener) try: urllib.request.urlretrieve(data_url, filename=download_path, reporthook=self._report_hook(download_file), data=None) except Exception as e: raise VergeMLError("Could not download {}: {}".format(data_url, e)) finally: if headers: urllib.request.install_opener(urllib.request.build_opener()) return dest_directory Example #12
def read_file_function(self, filepath): if filepath.startswith('http://'): from urllib.request import urlopen s = urlopen(filepath, timeout=5).read().decode('utf-8') return re.sub('\r+\n*', '\n', s) if self.base_path: filepath = os.path.join(self.base_path, filepath) filepath = os.path.abspath(filepath) view = CompileKspThread.find_view_by_filename(filepath, self.base_path) if view is None: s = codecs.open(filepath, 'r', 'utf-8').read() return re.sub('\r+\n*', '\n', s) else: return view.substr(sublime.Region(0, view.size())) Example #13
def jsonload(url): '''Load a web page and return the resulting JSON object''' request = urllib.request.Request(url, headers={'User-Agent': 'Mozilla/5.0'}) with urllib.request.urlopen(request, timeout=TIMEOUT) as webpage: data = str(webpage.read(), 'UTF-8') data = json.loads(data) return data Example #14
def url_size(url): if isinstance(url, str): headers = urllib.request.urlopen(url).headers else: headers = url if 'Content-Length' in headers: return int(headers['Content-Length']) else: return False
Example #15
def stream_url(url: str, start_byte: Optional[int] = None, block_size: int = 32 * 1024, progress_bar: bool = True) -> Iterable: """Stream url by chunk Args: url (str): Url. start_byte (int, optional): Start streaming at that point (Default: ``None``). block_size (int, optional): Size of chunks to stream (Default: ``32 * 1024``). progress_bar (bool, optional): Display a progress bar (Default: ``True``). """ # If we already have the whole file, there is no need to download it again req = urllib.request.Request(url, method="HEAD") url_size = int(urllib.request.urlopen(req).info().get("Content-Length", -1)) if url_size == start_byte: return req = urllib.request.Request(url) if start_byte: req.headers["Range"] = "bytes={}-".format(start_byte) with urllib.request.urlopen(req) as upointer, tqdm( unit="B", unit_scale=True, unit_divisor=1024, total=url_size, disable=not progress_bar, ) as pbar: num_bytes = 0 while True: chunk = upointer.read(block_size) if not chunk: break yield chunk num_bytes += len(chunk) pbar.update(len(chunk)) Example #16
def save_picture(recipes_raw, url): recipe = recipes_raw[url] path_save = path.join( config.path_img, '{}.jpg'.format(URL_to_filename(url))) if not path.isfile(path_save): if 'picture_link' in recipe: link = recipe['picture_link'] if link is not None: try: if 'epicurious' in url: img_url = 'https://{}'.format(link[2:]) urllib.request.urlretrieve(img_url, path_save) else: urllib.request.urlretrieve(link, path_save) except: print('Could not download image from {}'.format(link)) Example #17
def test_gunicorn_worker(gunicorn_worker): with urllib.request.urlopen("http://localhost:1337/") as f: res = json.loads(f.read(100).decode()) assert res["test"] Example #18
def test_gunicorn_worker_no_logs(gunicorn_worker_with_env_var): """ if SANIC_ACCESS_LOG was set to False do not show access logs """ with urllib.request.urlopen("http://localhost:1339/") as _: gunicorn_worker_with_env_var.kill() assert not gunicorn_worker_with_env_var.stdout.read() Example #19
def test_gunicorn_worker_with_logs(gunicorn_worker_with_access_logs): """ default - show access logs """ with urllib.request.urlopen("http://localhost:1338/") as _: gunicorn_worker_with_access_logs.kill() assert ( b"(sanic.access)[INFO][127.0.0.1" in gunicorn_worker_with_access_logs.stdout.read() ) Example #20
def test_run_max_requests_exceeded(worker): loop = asyncio.new_event_loop() worker.ppid = 1 worker.alive = True sock = mock.Mock() sock.cfg_addr = ("localhost", 8080) worker.sockets = [sock] worker.wsgi = mock.Mock() worker.connections = set() worker.log = mock.Mock() worker.loop = loop worker.servers = { "server1": {"requests_count": 14}, "server2": {"requests_count": 15}, } worker.max_requests = 10 worker._run = mock.Mock(wraps=_a_noop) # exceeding request count _runner = asyncio.ensure_future(worker._check_alive(), loop=loop) loop.run_until_complete(_runner) assert not worker.alive worker.notify.assert_called_with() worker.log.info.assert_called_with( "Max requests exceeded, shutting " "down: %s", worker ) Example #21
def task_8( img_url: str = 'https://i.imgur.com/B75zq0x.jpg' ) -> object: ''' Task 8: Module Args: img_url: address of an image Returns: result_img: an PIL Image Hints: * Make sure you have installed the PIL package * Take a look at utils.py first * You could easily find answers with Google ''' from urllib import request result_img = None # TODO: download the image from img_url with the request module # and add your student ID on it with draw_text() in the utils module # under src/. # You are allowed to change the img_url to your own image URL. # Display the image: # result_img.show() # Note: please comment this line when hand in. # If you are running on a server, use # result.save('test.jpg') # and copy the file to local or use Jupyter Notebook to render. # End of TODO return result_img Example #22
def task_8( img_url: str = 'https://i.imgur.com/B75zq0x.jpg' ) -> object: ''' Task 8: Module Args: img_url: address of an image Returns: result_img: an PIL Image Hints: * Make sure you have installed the PIL package * Take a look at utils.py first * You could easily find answers with Google ''' import urllib.request result_img = None from utils import draw_text # TODO: download the image from img_url with the request module # and add your student ID on it with draw_name() in the utils module # under src/. from PIL import Image urllib.request.urlretrieve(img_url,'/mnt/c/users/attis/desktop/pythonhomework/src/b07902113.jpg') with Image.open('/mnt/c/users/attis/desktop/pythonhomework/src/b07902113.jpg')as result_img: result_img = draw_text(result_img,'b07902113') result_img.show() # result_img.show(req) # You are allowed to change the img_url to your own image URL. # Display the image. If you are running on a server, change this line to # result.save('test.jpg') # End of TODO return result_img Example #23
def task_8( img_url: str = 'https://i.imgur.com/B75zq0x.jpg' ) -> object: ''' Task 8: Module Args: img_url: address of an image Returns: result_img: an PIL Image Hints: * Make sure you have installed the PIL package * Take a look at utils.py first * You could easily find answers with Google ''' from urllib import request result_img = None # TODO: download the image from img_url with the request module # and add your student ID on it with draw_name() in the utils module # under src/. # You are allowed to change the img_url to your own image URL. # Display the image: # result_img.show() # Note: please comment this line when hand in. # If you are running on a server, use # result.save('test.jpg') # and copy the file to local or use Jupyter Notebook to render. # End of TODO return result_img Example #24
def task_8( img_url: str = 'https://i.imgur.com/B75zq0x.jpg' ) -> object: ''' Task 8: Module Args: img_url: address of an image Returns: result_img: an PIL Image Hints: * Make sure you have installed the PIL package * Take a look at utils.py first * You could easily find answers with Google ''' from urllib import request result_img = None # TODO: download the image from img_url with the request module # and add your student ID on it with draw_text() in the utils module # under src/. # You are allowed to change the img_url to your own image URL. # Display the image: # result_img.show() # Note: please comment this line when hand in. # If you are running on a server, use # result.save('test.jpg') # and copy the file to local or use Jupyter Notebook to render. # End of TODO return result_img Example #25
def get_plugins_info (plugins_id_list): """Ask api the info on each plugin(s) Will return: - error: - None if no error encounter - the error an error occur - an array of plugin with the all the data return by the steam api Return error(error), array(array_of_plugins) """ json_response = [] error = None data = const_data['file'] data['itemcount'] = len(plugins_id_list) for idx, plugin_id in enumerate(plugins_id_list): data['publishedfileids[' + str(idx) + ']'] = plugin_id encode_data = urllib.parse.urlencode(data).encode('ascii') try: response = urllib.request.urlopen(const_urls['file'], encode_data) except HTTPError as e: print("Server return " + str(e.code) + " error") error = e except URLError as e: print("Can't reach server: " + e.reason) error = e else: json_response = json.loads(response.read().decode('utf8')) json_response = json_response['response']['publishedfiledetails'] return error, json_response Example #26
def get_json(s): req = urllib.request.Request(penguin_url + s, None, headers) with urllib.request.urlopen(req, timeout=5) as response: return json.loads(response.read().decode())
Example #27
def request_data(url_stats, url_rules, save_path_stats, save_path_rules): """ To request probability and convertion rules from web resources and store at local. Args: url_stats: string. url to the dropping rate stats data. url_rules: string. url to the composing rules data. save_path_stats: string. local path for storing the stats data. save_path_rules: string. local path for storing the composing rules data. Returns: material_probs: dictionary. Content of the stats json file. convertion_rules: dictionary. Content of the rules json file. """ try: os.mkdir(os.path.dirname(save_path_stats)) except: pass try: os.mkdir(os.path.dirname(save_path_rules)) except: pass req = urllib.request.Request(url_stats, None, headers) with urllib.request.urlopen(req, timeout=5) as response: material_probs = json.loads(response.read().decode()) with open(save_path_stats, 'w') as outfile: json.dump(material_probs, outfile) req = urllib.request.Request(url_rules, None, headers) with urllib.request.urlopen(req, timeout=5) as response: response = urllib.request.urlopen(req) convertion_rules = json.loads(response.read().decode()) with open(save_path_rules, 'w') as outfile: json.dump(convertion_rules, outfile) return material_probs, convertion_rules
Example #28
def from_import(module_name, *symbol_names, **kwargs): """ Example use: >>> HTTPConnection = from_import('http.client', 'HTTPConnection') >>> HTTPServer = from_import('http.server', 'HTTPServer') >>> urlopen, urlparse = from_import('urllib.request', 'urlopen', 'urlparse') Equivalent to this on Py3: >>> from module_name import symbol_names[0], symbol_names[1], ... and this on Py2: >>> from future.moves.module_name import symbol_names[0], ... or: >>> from future.backports.module_name import symbol_names[0], ... except that it also handles dotted module names such as ``http.client``. """ if PY3: return __import__(module_name) else: if 'backport' in kwargs and bool(kwargs['backport']): prefix = 'future.backports' else: prefix = 'future.moves' parts = prefix.split('.') + module_name.split('.') module = importlib.import_module(prefix + '.' + module_name) output = [getattr(module, name) for name in symbol_names] if len(output) == 1: return output[0] else: return output Example #29
def download(url, fn=None, path=None): if path is not None: os.makedirs(path, exist_ok=True) else: path = '.' if fn == None: fn = url.split('/')[-1] dest_fn = os.path.join(path, fn) u = urllib.request.urlopen(url) meta = u.info() file_size = int(meta.get_all("Content-Length")[0]) print('Downloading: [{}] ({:.2f} MB)'.format(fn, file_size / 1024 ** 2)) print(' URL : {}'.format(url)) print(' Destination: {}'.format(dest_fn)) with open(dest_fn, 'wb') as f: downloaded = 0 block_size = 65536 while True: buffer = u.read(block_size) if not buffer: break f.write(buffer) downloaded += len(buffer) progress = ' {:.2f}MB [{:3.2f}%]'.format(downloaded / 1024 ** 2, downloaded * 100 / file_size) print(progress, end='\r') print() Example #30
def from_import(module_name, *symbol_names, **kwargs): """ Example use: >>> HTTPConnection = from_import('http.client', 'HTTPConnection') >>> HTTPServer = from_import('http.server', 'HTTPServer') >>> urlopen, urlparse = from_import('urllib.request', 'urlopen', 'urlparse') Equivalent to this on Py3: >>> from module_name import symbol_names[0], symbol_names[1], ... and this on Py2: >>> from future.moves.module_name import symbol_names[0], ... or: >>> from future.backports.module_name import symbol_names[0], ... except that it also handles dotted module names such as ``http.client``. """ if PY3: return __import__(module_name) else: if 'backport' in kwargs and bool(kwargs['backport']): prefix = 'future.backports' else: prefix = 'future.moves' parts = prefix.split('.') + module_name.split('.') module = importlib.import_module(prefix + '.' + module_name) output = [getattr(module, name) for name in symbol_names] if len(output) == 1: return output[0] else: return output Source: https://www.programcreek.com/python/example/81442/urllib.request
Post a Comment for "How to Make Sure Python Urllib request urlretrieve Finishes Before Continuing"