path: root/gg_scraper.py

#!/usr/bin/python
# -*- coding: utf-8 -*-
"""
Download a Google Group to MBOX
Copyright (C) 2014 Matěj Cepl

This program is free software: you can redistribute it and/or modify it
under the terms of the GNU General Public License as published by the
Free Software Foundation, either version 3 of the License, or (at your
option) any later version.

This program is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
General Public License for more details.

You should have received a copy of the GNU General Public License along
with this program.  If not, see <http://www.gnu.org/licenses/>.'
"""
from __future__ import absolute_import, print_function, unicode_literals

import argparse
try:
    from collections import OrderedDict
except ImportError:
    from ordereddict import OrderedDict
try:
    from configparser import ConfigParser
except ImportError:
    from ConfigParser import ConfigParser
import logging
import mailbox
import operator
import os.path
import re
import shutil
import subprocess
import sys

import yaml
try:
    from urllib.error import HTTPError
    from urllib.request import HTTPHandler, HTTPRedirectHandler, \
        build_opener
except ImportError:
    from urllib2 import (HTTPError, HTTPHandler, HTTPRedirectHandler,
                         build_opener)

from bs4 import BeautifulSoup
from concurrent.futures import ThreadPoolExecutor, as_completed
try:
    from queue import Queue
except ImportError:
    from Queue import Queue  # noqa

logging.basicConfig(format='%(levelname)s:%(funcName)s:%(message)s',
                    level=logging.DEBUG)

ADDR_SEC_LABEL = 'addresses'
MAX_THREADS = 42
MANGLED_ADDR_RE = re.compile(
    r'([a-zA-Z0-9_.+-]+(\.)+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+)',
    re.IGNORECASE)

__version__ = '0.10.0'

pyver = sys.version_info
py26 = pyver[:2] < (2, 7)
py3k = pyver[0] == 3

log = logging.getLogger('gg_scraper')


class BadURLError(ValueError):
    pass


class Page(object):
    verb_handler = HTTPHandler()
    if logging.getLogger().getEffectiveLevel() == logging.DEBUG:
        verb_handler.set_http_debuglevel(2)
    redir_handler = HTTPRedirectHandler()
    opener = build_opener(verb_handler, redir_handler)

    def __init__(self):
        pass

    @staticmethod
    def unescape_google_bang_url(old_url):
        """
        See https://developers.google.com/webmasters\
                /ajax-crawling/docs/getting-started for more information
        """
        if old_url.find('#!') != -1:
            return old_url.replace('#!', '?_escaped_fragment_=')
        elif old_url.startswith('https://groups.google.com/d/topic/'):
            # DEBUG:get_one_topic:URL collected =
            #     https://groups.google.com/d/topic/jbrout/dreCkob3KSs
            # DEBUG:__init__:root_URL =
            #     https://groups.google.com/forum/\
            #        ?_escaped_fragment_=topic/jbrout/dreCkob3KSs
            return old_url.replace(
                'https://groups.google.com/d/',
                'https://groups.google.com/forum/?_escaped_fragment_='
            )
        else:
            return old_url

    def _get_page_bs(self, url):
        res = self.opener.open(self.unescape_google_bang_url(url))
        in_str = res.read()
        bs = BeautifulSoup(in_str)
        res.close()
        return bs


class Article(Page):
    def __init__(self, url):
        super(Article, self).__init__()
        self.root = url.replace('d/msg/', 'forum/message/raw?msg=')
        self.raw_message = ''

    def collect_message(self):
        log.debug('self.root = {0}'.format(self.root))
        result = None
        try:
            res = self.opener.open(self.root)
            if not py3k:
                raw_msg = res.read().decode('utf8')
            else:
                raw_msg = res.read()
            proc = subprocess.Popen(['/usr/bin/formail'],
                                    stdin=subprocess.PIPE,
                                    stdout=subprocess.PIPE)
            if not(py3k and isinstance(raw_msg, bytes)):
                raw_msg = raw_msg.encode('utf8')
            result = proc.communicate(raw_msg)[0]
            if not py3k:
                result = result.decode('utf8')
            res.close()
        except HTTPError as exc:
            logging.warning('Exception on downloading {0}:\n{1}'.format(
                self.root, exc))

        return result


class Topic(Page):
    def __init__(self, url, name):
        super(Topic, self).__init__()
        self.name = name
        root_url = self.unescape_google_bang_url(url)
        self.root = root_url
        self.articles = []

    def __unicode__(self):
        return "%s: %s" % (self.root, self.name)

    @staticmethod
    def get_one_article(elem):
        return elem

    def get_count_articles(self):
        '''Get total number of articles from the number on the page
        itself.
        '''
        bs = self._get_page_bs(self.root)
        i_elem = bs.find_all('i')
        if len(i_elem) <= 0:
            raise ValueError('Cannot find count of topics!')

        i_str = i_elem[0].string
        return int(re.match(r'\D+ \d+\D+\d+ \D+ (\d+) \D+$', i_str).group(1))

    def get_articles(self):
        out = []
        page = self._get_page_bs(self.root)
        for a_elem in page.find_all('a'):
            if 'href' in a_elem.attrs:
                a_href = a_elem['href']
                if re.match(r'https://groups.google.com/d/msg/',
                            a_href) is not None:
                    out.append(Article(a_href))

        return out


class Group(Page):
    good_url_re = re.compile(r'https://groups.google.com/forum/#!forum/(.+)')

    def __init__(self, url):
        super(Group, self).__init__()
        self.group_URL = url
        self.topics = []
        match = self.good_url_re.match(url)
        log.debug('match = %s', match)
        if match is None:
            raise BadURLError(
                "Required url in form " +
                "'https://groups.google.com/forum/#!forum/GROUPNAME'")

        self.name = match.group(1)

    @staticmethod
    def get_count_topics(bs):
        '''Get total number of topics from the number on the page
        itself.

        Which would be awesome for control, except it is wrong on all
        pages in various and different ways. :(
        '''
        i_elem = bs.find_all('i')
        if len(i_elem) <= 0:
            raise ValueError('Cannot find count of topics!')

        i_str = i_elem[0].string
        return int(re.match(r'\D+ \d+ - \d+ \D+ (\d+) \D+$', i_str).group(1))

    @staticmethod
    def get_one_topic(elem):
        sys.stdout.write('. ')
        sys.stdout.flush()
        # log.debug('URL collected = {0}'.format(elem['href']))
        if 'title' in elem.attrs:
            # filter out all-non-topic <a>s
            return True, Topic(elem['href'], elem['title'])
        else:
            log.debug('other = %s', elem)
            return False, elem

    def get_topics(self):
        '''Recursively[?] get all topic (as special objects)
        Also return (for error checking) number of topics from the head
        of the topic page.
        '''
        out = []
        target_stack = [self.group_URL]

        while target_stack:
            other = []
            bs = self._get_page_bs(target_stack.pop(0))
            for a_elem in bs.find_all('a'):
                is_topic, res = self.get_one_topic(a_elem)
                # Ignore link in welcome message, e.g. django-oscar group
                is_welcomemsg = a_elem.get('target') == 'welcomeMsg'
                if is_topic:
                    out.append(res)
                elif not is_welcomemsg:
                    other.append(res)

            if len(other) == 1:
                target_stack.append(other[0]['href'])
            elif len(other) != 0:
                raise ValueError(
                    'There must be either one or none link to the next page!')

        sys.stdout.write('\n')
        sys.stdout.flush()
        return out

    def collect_group(self):
        self.topics = self.get_topics()
        len_topics = len(self.topics)
        jobs = []
        with ThreadPoolExecutor(MAX_THREADS) as executor:
            for top in self.topics:
                # print('[%d/%d] downloading "%s"' % (self.topics.index(top),
                #      len_topics, top.name))
                print('[%d/%d] downloading' %
                      (self.topics.index(top), len_topics))
                job = executor.submit(top.get_articles)
                jobs.append(job)

            for job in as_completed(jobs):
                arts = job.result()
                top.articles = arts
                msg_jobs = {}

                for a_job in arts:
                    m_job = executor.submit(a_job.collect_message)
                    msg_jobs[m_job] = a_job

                for m_job in as_completed(msg_jobs):
                    a_job = msg_jobs[m_job]
                    msg = m_job.result()
                    if msg is not None:
                        a_job.raw_message = msg

    def all_messages(self):
        '''Iterate over all messages in the group'''
        for top in self.topics:
            for art in top.articles:
                yield art.raw_message

    def collect_mangled_addrs(self):
        addrs = {}
        for top in self.topics:
            for art in top.articles:
                msg_str = art.raw_message
                # see http://stackoverflow.com/questions/201323
                msg_matches = MANGLED_ADDR_RE.findall(msg_str)
                if msg_matches is not None:
                    for mtch in msg_matches:
                        if isinstance(mtch, tuple):
                            mtch = mtch[0]
                        if mtch in addrs:
                            addrs[mtch] += 1
                        else:
                            addrs[mtch] = 1

        addrs = OrderedDict(sorted(addrs.items(),
                                   key=operator.itemgetter(1),
                                   reverse=True))

        with open('{0}.cnf'.format(self.name), 'w') as cnf_f:
            cnf_p = ConfigParser(dict_type=OrderedDict)
            cnf_p.add_section(ADDR_SEC_LABEL)
            for addr in addrs:
                try:
                    cnf_p.set(ADDR_SEC_LABEL, addr, '')
                except TypeError:
                    # Failed with addr = ('richte...@gmail.com', '.')
                    logging.info('Failed with addr = {0}'.format(addr))
                    raise
            cnf_p.write(cnf_f)


class MBOX(mailbox.mbox):
    def __init__(self, filename):
        if os.path.exists(filename):
            shutil.move(filename, '{0}.bak'.format(filename))
        mailbox.mbox.__init__(self, filename)
        self.box_name = filename

        lockfile = '{0}.lock'.format(filename)
        if os.path.exists(lockfile):
            os.unlink(lockfile)

    def write_group(self, group_object):
        self.lock()
        for mbx_str in group_object.all_messages():
            try:
                self.add(mbx_str.encode('utf8'))
            except UnicodeDecodeError:
                logging.warning('mbx_str = type {0}'.format(type(mbx_str)))
        self.unlock()
        self.close()


def main(group_url):
    # Collect all messages to the internal variables
    if os.path.exists('group.yaml'):
        with open('group.yaml') as yf:
            log.debug('Loading state from group.yaml')
            grp = yaml.load(yf)
            log.debug('Done')
    else:
        grp = Group(group_url)
        grp.collect_group()

        # dump the state for debugging
        with open('group.yaml', 'w') as yf:
            yaml.dump(grp, yf)

    # Write MBOX
    mbx = MBOX("{0}.mbx".format(grp.name))
    mbx.write_group(grp)

    # generate list of addresses protected against spammers
    grp.collect_mangled_addrs()


def demangle(correct_list, orig_mbx, out_mbx):
    cnf_p = ConfigParser(dict_type=OrderedDict)
    cnf_p.read(correct_list)
    # pairs = dict(cnf_p.items(ADDR_SEC_LABEL))
    pairs = dict((k, {'repl': v, 'RE': re.compile(r'\b%s\b' % k,
                                                  re.IGNORECASE)})
                 for (k, v) in cnf_p.items(ADDR_SEC_LABEL)
                 if v is not None)
    counter = 0

    if os.path.exists(out_mbx):
        shutil.move(out_mbx, '{0}.bak'.format(out_mbx))

    in_mbx = mailbox.mbox(orig_mbx)
    out_mbx = mailbox.mbox(out_mbx)

    out_mbx.lock()
    for msg in in_mbx.itervalues():
        msg_str = str(msg)
        matches = MANGLED_ADDR_RE.search(msg_str)
        if matches is not None:
            u_from = msg.get_from()
            for orig, fixed in pairs.items():
                # if (orig is None) or (fixed is None):
                #    continue
                # msg_str = msg_str.replace(orig, fixed)
                msg_str = fixed['RE'].sub(fixed['repl'], msg_str)
                counter += 1  # This is wrong
            out_msg = mailbox.mboxMessage(msg_str)
            out_msg.set_from(u_from)

            out_mbx.add(out_msg)
        else:
            out_mbx.add(msg)
    out_mbx.close()
    in_mbx.close()
    logging.info('Change counter = {0}'.format(counter))


if __name__ == '__main__':
    parser = argparse.ArgumentParser(
        description='Scrape a Google Groups group.')
    parser.add_argument('group', metavar='URL', nargs='?',
                        help='URL of the group')
    parser.add_argument('-d', '--demangle', metavar='DEMANGLE_FILE', nargs=3,
                        help='Demangle mbox from stdin to stdout' +
                        'according to the values in the configuration' +
                        'file.')
    args = parser.parse_args()

    log.debug('args = {0}'.format(args))

    if args.demangle is not None:
        demangle(args.demangle[0], args.demangle[1], args.demangle[2])
    else:
        main(args.group)