Option to dump intermediate pages

This commit is contained in:
Philipp Hagemeister 2013-04-11 18:31:35 +02:00
parent 927c8c4924
commit 855703e55e
2 changed files with 12 additions and 0 deletions

View file

@ -134,6 +134,14 @@ class InfoExtractor(object):
else:
encoding = 'utf-8'
webpage_bytes = urlh.read()
if self._downloader.params.get('dump_intermediate_pages', False):
try:
url = url_or_request.get_full_url()
except AttributeError:
url = url_or_request
self._downloader.to_screen(u'Dumping request to ' + url)
dump = base64.b64encode(webpage_bytes).decode('ascii')
self._downloader.to_screen(dump)
return webpage_bytes.decode(encoding, 'replace')