From cd61d3d36cf4f652c0908a86fd3bcf5da2f10b23 Mon Sep 17 00:00:00 2001 From: Matěj Cepl Date: Mon, 18 Apr 2016 00:16:58 +0200 Subject: More debugging --- gg_scraper.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'gg_scraper.py') diff --git a/gg_scraper.py b/gg_scraper.py index 2c24082..ebcd21a 100755 --- a/gg_scraper.py +++ b/gg_scraper.py @@ -65,7 +65,6 @@ MANGLED_ADDR_RE = re.compile( __version__ = '0.10.0' pyver = sys.version_info -py26 = pyver[:2] < (2, 7) py3k = pyver[0] == 3 log = logging.getLogger('gg_scraper') @@ -261,6 +260,7 @@ class Group(Page): def collect_group(self): self.topics = self.get_topics() len_topics = len(self.topics) + log.debug('self.topics = %s, len %d', self.topics, len_topics) jobs = [] with ThreadPoolExecutor(MAX_THREADS) as executor: for top in self.topics: @@ -271,8 +271,10 @@ class Group(Page): job = executor.submit(top.get_articles) jobs.append(job) + log.debug('jobs = %s', jobs) for job in as_completed(jobs): arts = job.result() + log.debug('arts = %s', arts) top.articles = arts msg_jobs = {} @@ -280,9 +282,12 @@ class Group(Page): m_job = executor.submit(a_job.collect_message) msg_jobs[m_job] = a_job + log.debug('msg_jobs = %s', msg_jobs) for m_job in as_completed(msg_jobs): a_job = msg_jobs[m_job] + log.debug('a_job = %s', a_job) msg = m_job.result() + log.debug('msg = %s', msg) if msg is not None: a_job.raw_message = msg -- cgit