diff options
| -rwxr-xr-x | dlpcvp.py | 98 |
1 files changed, 52 insertions, 46 deletions
@@ -31,13 +31,16 @@ config = configparser.ConfigParser() config.read(ConfigRC) logging.basicConfig(format='%(levelname)s:%(funcName)s:%(message)s', - stream=sys.stdout, level=logging.DEBUG) + stream=sys.stdout, level=logging.INFO) log = logging.getLogger() # or HTTPPasswordMgrWithPriorAuth ? password_mgr = urllib.request.HTTPPasswordMgrWithDefaultRealm() user = config[OBS_base]['user'] passw = config[OBS_base]['pass'] +# MYPY: Argument 1 to "add_password" of +# "HTTPPasswordMgrWithDefaultRealm" has incompatible type "None"; +# expected "str" password_mgr.add_password(None, OBS_base, user, passw) handler = urllib.request.HTTPBasicAuthHandler(password_mgr) @@ -54,28 +57,6 @@ TB_CREATE = """CREATE TABLE etags TB_CREATE_IDX = "CREATE UNIQUE INDEX etags_names ON etags(pkg)" -def get_version_from_pypi(name: str, etag: str = None) -> Tuple[str, str, str]: - """ - For the given name of module return the latest version available on PyPI. - """ - req = Request(url=PyPI_base.format(name)) - - if etag is not None: - req.add_header('ETag', etag) - - try: - with urlopen(req) as resp: - data = json.load(resp) - info_dict = data['info'] - return info_dict['name'], info_dict['version'], resp.info()['ETag'] - except HTTPError as ex: - if ex.getcode() == 404: - log.error(f'Cannot find {name} on PyPI') - return None - else: - raise - - def suse_packages(proj: str) -> Iterable[str]: """ Iterator returning names of all packages in the given proj @@ -98,7 +79,7 @@ def suse_packages(proj: str) -> Iterable[str]: raise -def parse_spec_in_dev_null(spec_file, pkg, e_fn, e_spcf): +def parse_spec_in_dev_null(spec_file, pkg): # rpm library generates awful lot of nonsensical goo on # stderr with open(os.devnull, 'wb') as nullf: @@ -112,8 +93,7 @@ def parse_spec_in_dev_null(spec_file, pkg, e_fn, e_spcf): log.error("Cannot parse {}".format(pkg)) else: try: - return pkg, spc.packages[0].header['Version'].decode(), \ - e_fn, e_spcf + return pkg, spc.packages[0].header['Version'].decode() except IndexError: pass finally: @@ -121,7 +101,9 @@ def parse_spec_in_dev_null(spec_file, pkg, e_fn, e_spcf): sys.stdout = old_stdout -def get_etags(con, pkg): +def get_etags(con: sqlite3.Connection, pkg: str) -> \ + Tuple[Optional[str], Optional[str], Optional[str], Optional[str]]: + # pkg, suse_name_etag, suse_spec_etag, pypi_etag cur = con.execute("SELECT * FROM etags WHERE pkg=?", (pkg,)) ret = cur.fetchone() if ret is None: @@ -130,8 +112,12 @@ def get_etags(con, pkg): return ret -def update_etags(con, pkg, e_suse_name, e_suse_spec, e_pypi): - if con.execute("SELECT * FROM etags WHERE pkg=?", (pkg,)).fetchone(): +def update_etags(con: sqlite3.Connection, pkg: str, e_suse_name: Optional[str], e_suse_spec: Optional[str], e_pypi: Optional[str]): + res = con.execute("SELECT * FROM etags WHERE pkg=?", (pkg,)).fetchone() + if res: + e_suse_name = res[1] if e_suse_name is None else e_suse_name + e_suse_spec = res[2] if e_suse_spec is None else e_suse_spec + e_pypi = res[3] if e_pypi is None else e_pypi con.execute('''UPDATE etags SET suse_name_etag=?, suse_spec_etag=?, pypi_etag=? WHERE pkg=?''', (e_suse_name, e_suse_spec, e_pypi, pkg)) @@ -142,14 +128,42 @@ def update_etags(con, pkg, e_suse_name, e_suse_spec, e_pypi): con.commit() -def package_version(proj: str, pkg_name: str, - etag_fn: str = None, etag_spcf: str = None) \ - -> Optional[Tuple[str, str, str]]: +def get_version_from_pypi(name: str, con: sqlite3.Connection) \ + -> Optional[Tuple[str, str]]: + """ + For the given name of module return the latest version available on PyPI. + """ + # pkg, suse_name_etag, suse_spec_etag, pypi_etag + _, _, _, etag = get_etags(con, name) + req = Request(url=PyPI_base.format(name)) + + if etag is not None: + req.add_header('ETag', etag) + + try: + with urlopen(req) as resp: + data = json.load(resp) + info_dict = data['info'] + update_etags(con, name, None, None, resp.info()['ETag']) + return info_dict['name'], info_dict['version'] + except HTTPError as ex: + if ex.getcode() == 404: + log.error(f'Cannot find {name} on PyPI') + return None + else: + raise + + +def package_version(proj: str, pkg_name: str, con: sqlite3.Connection) \ + -> Optional[Tuple[str, str, str]]: """ Return the version of the given package in the given proj. Downloads SPEC file from OBS and parses it. """ + # pkg, suse_name_etag, suse_spec_etag, pypi_etag + _, etag_fn, etag_spcf, _ = get_etags(con, pkg_name) + req_spc_name = Request(url=OBS_base + f'/source/{proj}/{pkg_name}') spec_files = [] @@ -168,7 +182,7 @@ def package_version(proj: str, pkg_name: str, except HTTPError as ex: if ex.getcode() == 404: log.error(f'Cannot accquire version of {pkg_name} in {proj}') - return + return None else: raise @@ -197,8 +211,8 @@ def package_version(proj: str, pkg_name: str, spf.flush() os.fsync(spf.fileno()) - return parse_spec_in_dev_null(spec_file_name, pkg_name, - etag_fn, etag_spcf) + update_etags(con, pkg_name, etag_fn, etag_spcf, None) + return parse_spec_in_dev_null(spec_file_name, pkg_name) except HTTPError as ex: if ex.getcode() == 404: log.error(f'Cannot parse SPEC file {spc_fname} for {pkg_name}') @@ -217,21 +231,13 @@ def main(prj): log.debug(f'pkg = {pkg}') if pkg.startswith('python-'): pypi_name = pkg[CUTCHARS:] - _, et_suse_name, et_suse_spec, et_pypi = get_etags(conn, pkg) - try: - suse_ver_tup = package_version(prj, pkg, - et_suse_name, et_suse_spec) + suse_ver_tup = package_version(prj, pkg, conn) if suse_ver_tup is None: raise RuntimeError('not in OBS') - - pypi_ver_tup = get_version_from_pypi(pypi_name, et_pypi) + pypi_ver_tup = get_version_from_pypi(pypi_name, conn) if pypi_ver_tup is None: raise RuntimeError('not in PyPI') - - update_etags(conn, pkg, - suse_ver_tup[1], suse_ver_tup[2], - pypi_ver_tup[2]) except RuntimeError as ex: log.warning(f'Package {pkg} cannot be found: {ex}') continue @@ -252,7 +258,7 @@ if __name__ == '__main__': parser = argparse.ArgumentParser(description='Check available versions ' 'of the upstream packages on PyPI') parser.add_argument('--opensuse-project', - default='devel:languages:python:numeric', + default='devel:languages:python', help='The OpenBuildService project. Defaults ' 'to %(default)s') |