aboutsummaryrefslogtreecommitdiffstats
path: root/dlpcvp.py
blob: 259a89165a6b55461797d2cca702234922425af9 (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
#!/usr/bin/python3
import argparse
import configparser
import json
import logging
import os.path as osp
import sqlite3
import ssl
import sys
import urllib.request
import xml.etree.ElementTree as ET
from distutils.version import LooseVersion
from typing import Iterable, List, Optional, Tuple, Union
from urllib.error import URLError, HTTPError
from urllib.request import Request, urlopen

# PyPI API documentation https://warehouse.readthedocs.io/api-reference/
PyPI_base = "https://pypi.org/pypi/{}/json"
# https://github.com/openSUSE/open-build-service/blob/master/docs/api/api/api.txt
# https://build.opensuse.org/apidocs/index
OBS_base = "https://api.opensuse.org"
OBS_realm = "Use your novell account"
ConfigRCs = [osp.expanduser('~/.oscrc'),
             osp.expanduser('~/.config/osc/oscrc')]
CUTCHARS = len('python-')

DBConnType = sqlite3.Connection
OStr = Optional[str]

config = configparser.ConfigParser()
config.read(ConfigRCs)

logging.basicConfig(format='%(levelname)s:%(funcName)s:%(message)s',
                    stream=sys.stdout, level=logging.INFO)
log = logging.getLogger()

# or HTTPPasswordMgrWithPriorAuth ?
password_mgr = urllib.request.HTTPPasswordMgrWithDefaultRealm()
user = config[OBS_base]['user']
passw = config[OBS_base]['pass']
password_mgr.add_password(OBS_realm, OBS_base, user, passw)

auth_handler = urllib.request.HTTPBasicAuthHandler(password_mgr)
ssl_handler = urllib.request.HTTPSHandler(
    context=ssl._create_unverified_context())
opener = urllib.request.build_opener(auth_handler, ssl_handler)
urllib.request.install_opener(opener)

TB_EXISTS = """SELECT name FROM sqlite_master
        WHERE type='table' AND name='etags'
        """
# FIXME we need to distinguish between PyPI and OpenSUSE records
TB_CREATE = """CREATE TABLE etags
               (pkg text NOT NULL PRIMARY KEY,
               suse_name_etag text, suse_spec_etag text, pypi_etag text)
            """
TB_CREATE_IDX = "CREATE UNIQUE INDEX etags_names ON etags(pkg)"


def suse_packages(proj: str) -> Iterable[str]:
    """
    Iterator returning names of all packages in the given proj

    ETag management won't work here, because I don't know about any way
    how to return it in iterator.
    """
    req = Request(url=OBS_base + f'/source/{proj}')
    with open('exceptions.json') as exc_f:
        exc_dict = json.load(exc_f)

    exc_list = []  # type: List[str]
    if proj in exc_dict:
        exc_list = exc_dict[proj]

    try:
        with opener.open(req) as resp:
            raw_xml_data = ET.parse(resp)
            root = raw_xml_data.getroot()
            for elem in root.iter('entry'):
                pkg_name = elem.get('name')
                # Invalid packages are skipped
                if pkg_name is None:
                    log.warning(f"Package {elem} doesn’t have a name!")
                    continue
                # We don't want -doc subpackages
                if pkg_name.endswith('-doc'):
                    continue
                # Nor we want packages which we specifically excluded
                if pkg_name in exc_list:
                    continue
                yield pkg_name
    except HTTPError as ex:
        if ex.getcode() == 404:
            log.warning(f'Cannot find packages for {proj}!')
        else:
            raise


def get_etags(con: DBConnType, pkg: str) -> \
        Tuple[OStr, OStr, OStr, OStr]:
    # pkg, suse_name_etag, suse_spec_etag, pypi_etag
    cur = con.execute("SELECT * FROM etags WHERE pkg=?", (pkg,))
    ret = cur.fetchone()
    log.debug(f'ret = {ret}')
    if ret is None:
        return None, None, None, None
    else:
        return ret


def update_etags(con: sqlite3.Connection, pkg: str,
                 e_suse_name: OStr, e_suse_spec: OStr, e_pypi: OStr):
    res = con.execute("SELECT * FROM etags WHERE pkg=?", (pkg,)).fetchone()
    if res:
        e_suse_name = res[1] if e_suse_name is None else e_suse_name
        e_suse_spec = res[2] if e_suse_spec is None else e_suse_spec
        e_pypi = res[3] if e_pypi is None else e_pypi
    con.execute('''REPLACE INTO etags
        (pkg, suse_name_etag, suse_spec_etag, pypi_etag)
        VALUES (?, ?, ?, ?)''', (pkg, e_suse_name, e_suse_spec, e_pypi))
    con.commit()


def is_develpackage(proj: str, pkg: str) -> bool:
    req = Request(url=OBS_base + f'/source/openSUSE:Factory/{pkg}/_meta')
    log.debug('Looking up URL: %s', req.full_url)
    try:
        with opener.open(req) as resp:
            xml_data = ET.parse(resp)
            for pel in xml_data.iter('package'):
                for delem in pel.iter('devel'):
                    if pel.attrib['name'] == pkg and \
                        delem.attrib['package'] == pkg and \
                            delem.attrib['project'] == proj:
                        return True
            return False
    except HTTPError as ex:
        if ex.getcode() == 404:
            log.warning(f'Cannot acquire _meta of {pkg}.')
            return None
        else:
            raise


def parse_spec(spec_file: Union[str, bytes]) -> LooseVersion:
    if isinstance(spec_file, bytes):
        spec_file = spec_file.decode()

    rest_of_line = ''
    for line in spec_file.splitlines():
        if line.startswith('Version:'):
            rest_of_line = line[len('Version:'):].strip()
            break

    return LooseVersion(rest_of_line)


def get_spec_name(req: Request, proj: str, pkg: str, etag: OStr = None) -> \
        OStr:
    """Acquire version from the listing of the project directory.
    """
    spec_files = []  # type: List[str]

    if etag is not None:
        req.add_header('ETag', etag)

    if not spec_files:
        IOError(f'Cannot find SPEC file for {pkg}.')

    try:
        with opener.open(req) as resp:
            etag = str(resp.info()['ETag'])
            raw_xml_data = ET.parse(resp)
            root = raw_xml_data.getroot()
            for elem in root.iter('entry'):
                name = elem.get('name')
                # Invalid package
                if name is None:
                    log.warning(f"Package {elem} doesn’t have a name!")
                    return None
                if name.endswith('.spec'):
                    spec_files.append(name)
                if name == '_aggregate':
                    log.warning(f'Package {pkg} is just aggregate, ignoring.')
                    return None

    except HTTPError as ex:
        if ex.getcode() in (400, 404):
            return None
        else:
            raise

    try:
        fname = sorted(spec_files, key=len)[0]
        return fname
    except IndexError:
        log.error(
                f'{pkg} is most likely not a branch, but a link: ' +
                f'use osc linktobranch {proj} {pkg} to convert it.')
        return None


def get_version_from_pypi(name: str, con: DBConnType = None) -> \
        Optional[LooseVersion]:
    """
    For the given name of module return the latest version available on PyPI.
    """
    # pkg, suse_name_etag, suse_spec_etag, pypi_etag
    if con is None:
        etag = None
    else:
        _, _, _, etag = get_etags(con, name)
    req = Request(url=PyPI_base.format(name))

    if etag is not None:
        req.add_header('ETag', etag)

    try:
        with urlopen(req) as resp:
            data = json.load(resp)
            info_dict = data['info']
            curr_etag = str(resp.info()['ETag'])
            if (con is not None) and curr_etag:
                update_etags(con, name, None, None, curr_etag)

            # Cleanup version
            version = info_dict['version'].replace('-', '.')

            return LooseVersion(version)
    except HTTPError as ex:
        if ex.getcode() == 404:
            log.warning(f'Cannot find {name} on PyPI')
        else:
            raise
        return None


def package_version(proj: str, pkgn: str,
                    con: DBConnType = None) -> Optional[LooseVersion]:
    """
    Return the version of the given package in the given proj.

    Downloads SPEC file from OBS and parses it.
    """
    # pkg, suse_name_etag, suse_spec_etag, pypi_etag
    if con:
        _, etag_fn, etag_spcf, _ = get_etags(con, pkgn)
    else:
        etag_fn, etag_spcf = None, None

    # Get listing of the package repository
    req_spc_name = Request(url=OBS_base + f'/source/{proj}/{pkgn}?expand=1')
    spc_fname = get_spec_name(req_spc_name, proj, pkgn, etag_fn)

    if spc_fname is None:
        return None

    req_spec = Request(url=OBS_base +
                       f'/source/{proj}/{pkgn}/{spc_fname}?expand=1')

    if etag_spcf is not None:
        req_spc_name.add_header('ETag', etag_spcf)

    try:
        with opener.open(req_spec) as resp:
            etag_spcf = str(resp.info()['ETag'])
            etag_spcf = None if not etag_spcf else etag_spcf
            spec_file_str = resp.read().decode()

            if (con is not None) and (etag_spcf or etag_fn):
                update_etags(con, pkgn, etag_fn, etag_spcf, None)
            return parse_spec(spec_file_str)
    except HTTPError as ex:
        if ex.getcode() == 404:
            log.warning(f'Cannot parse SPEC file {spc_fname} for {pkgn}')
        else:
            raise
        return None


def main(prj):
    db_name = osp.splitext(osp.basename(osp.realpath(__file__)))[0] + ".db"
    to_be_upgraded = []  # type: List[Tuple[str, LooseVersion, LooseVersion]]
    missing_on_PyPI = []  # type: List[str]
    with sqlite3.connect(db_name) as conn:
        if not conn.execute(TB_EXISTS).fetchone():
            conn.execute(TB_CREATE)
            conn.execute(TB_CREATE_IDX)

        for pkg in suse_packages(prj):
            log.debug(f'pkg = {pkg}')
            print(pkg[0], file=sys.stderr, end='', flush=True)
            if pkg.startswith('python-'):
                pypi_name = pkg[CUTCHARS:]
                try:
                    suse_ver = package_version(prj, pkg, conn)
                    if suse_ver is None:
                        raise RuntimeError('not in OBS')
                    if not is_develpackage(prj, pkg):
                        continue
                    pypi_ver = get_version_from_pypi(pypi_name, conn)
                    if pypi_ver is None:
                        raise RuntimeError('not in PyPI')
                except RuntimeError as ex:
                    log.warning(f'Package {pkg} cannot be found: {ex}')
                    continue
                except URLError as ex:
                    log.warning(
                        'Investigation of package %s caused network error: %s',
                        pkg, ex)
                    continue

                try:
                    if pypi_ver > suse_ver:
                        to_be_upgraded.append((pkg, suse_ver, pypi_ver))
                except TypeError:
                    log.warning(f'{pkg} pypi_ver = {pypi_ver}')
                    log.warning(f'{pkg} suse_ver = {suse_ver}')
                    continue
            else:
                missing_on_PyPI.append(pkg)

    sys.stdout.flush()
    if missing_on_PyPI:
        print("\nThese packages don't seem to be available on PyPI:")
        print("{}\n".format('\n'.join(missing_on_PyPI)))

    if to_be_upgraded:
        print('These packages need to be upgraded:')
        for pkg_upgr in to_be_upgraded:
            print(f'{pkg_upgr[0]} ({pkg_upgr[1]} -> {pkg_upgr[2]})')


if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Check available versions '
                                     'of the upstream packages on PyPI')
    parser.add_argument('project', nargs='?',
                        default='devel:languages:python:numeric',
                        help='The OpenBuildService project. Defaults '
                        'to %(default)s')

    args = parser.parse_args()
    sys.exit(main(args.project))