diff options
| -rwxr-xr-x | dlpcvp.py | 91 | ||||
| -rw-r--r-- | exceptions.json | 13 |
2 files changed, 61 insertions, 43 deletions
@@ -3,15 +3,14 @@ import argparse import configparser import json import logging -import os -import os.path +import os.path as osp import sqlite3 import sys import urllib.request from distutils.version import LooseVersion from urllib.error import HTTPError from urllib.request import Request, urlopen -from typing import Iterable, Optional, Tuple +from typing import Iterable, List, Optional, Tuple import xml.etree.ElementTree as ET @@ -20,12 +19,16 @@ PyPI_base = "https://pypi.org/pypi/{}/json" # https://github.com/openSUSE/open-build-service/blob/master/docs/api/api/api.txt # https://build.opensuse.org/apidocs/index OBS_base = "https://api.opensuse.org" -ConfigRC = [os.path.expanduser('~/.oscrc'), - os.path.expanduser('~/.config/osc/oscrc')] +OBS_realm = "Use your novell account" +ConfigRCs = [osp.expanduser('~/.oscrc'), + osp.expanduser('~/.config/osc/oscrc')] CUTCHARS = len('python-') +DBConnType = sqlite3.Connection +OStr = Optional[str] + config = configparser.ConfigParser() -config.read(ConfigRC) +config.read(ConfigRCs) logging.basicConfig(format='%(levelname)s:%(funcName)s:%(message)s', stream=sys.stdout, level=logging.INFO) @@ -35,10 +38,7 @@ log = logging.getLogger() password_mgr = urllib.request.HTTPPasswordMgrWithDefaultRealm() user = config[OBS_base]['user'] passw = config[OBS_base]['pass'] -# MYPY: Argument 1 to "add_password" of -# "HTTPPasswordMgrWithDefaultRealm" has incompatible type "None"; -# expected "str" -password_mgr.add_password(None, OBS_base, user, passw) +password_mgr.add_password(OBS_realm, OBS_base, user, passw) handler = urllib.request.HTTPBasicAuthHandler(password_mgr) opener = urllib.request.build_opener(handler) @@ -48,7 +48,7 @@ TB_EXISTS = """SELECT name FROM sqlite_master WHERE type='table' AND name='etags' """ TB_CREATE = """CREATE TABLE etags - (pkg text NOT NULL, + (pkg text NOT NULL PRIMARY KEY, suse_name_etag text, suse_spec_etag text, pypi_etag text) """ TB_CREATE_IDX = "CREATE UNIQUE INDEX etags_names ON etags(pkg)" @@ -62,22 +62,30 @@ def suse_packages(proj: str) -> Iterable[str]: how to return it in iterator. """ req = Request(url=OBS_base + f'/source/{proj}') + with open('exceptions.json') as exc_f: + exc_dict = json.load(exc_f) + + if proj in exc_dict: + exc_list = exc_dict[proj] try: with opener.open(req) as resp: raw_xml_data = ET.parse(resp) root = raw_xml_data.getroot() for elem in root.iter('entry'): - yield elem.get('name') + pkg_name = elem.get('name') + if pkg_name in exc_list: + continue + yield pkg_name except HTTPError as ex: if ex.getcode() == 404: - log.error(f'Cannot find packages for {proj}!') + log.warning(f'Cannot find packages for {proj}!') else: raise -def get_etags(con: sqlite3.Connection, pkg: str) -> \ - Tuple[Optional[str], Optional[str], Optional[str], Optional[str]]: +def get_etags(con: DBConnType, pkg: str) -> \ + Tuple[OStr, OStr, OStr, OStr]: # pkg, suse_name_etag, suse_spec_etag, pypi_etag cur = con.execute("SELECT * FROM etags WHERE pkg=?", (pkg,)) ret = cur.fetchone() @@ -87,19 +95,16 @@ def get_etags(con: sqlite3.Connection, pkg: str) -> \ return ret -def update_etags(con: sqlite3.Connection, pkg: str, e_suse_name: Optional[str], e_suse_spec: Optional[str], e_pypi: Optional[str]): +def update_etags(con: sqlite3.Connection, pkg: str, + e_suse_name: OStr, e_suse_spec: OStr, e_pypi: OStr): res = con.execute("SELECT * FROM etags WHERE pkg=?", (pkg,)).fetchone() if res: e_suse_name = res[1] if e_suse_name is None else e_suse_name e_suse_spec = res[2] if e_suse_spec is None else e_suse_spec e_pypi = res[3] if e_pypi is None else e_pypi - con.execute('''UPDATE etags - SET suse_name_etag=?, suse_spec_etag=?, pypi_etag=? - WHERE pkg=?''', (e_suse_name, e_suse_spec, e_pypi, pkg)) - else: - con.execute('''INSERT INTO etags - (pkg, suse_name_etag, suse_spec_etag, pypi_etag) - VALUES (?, ?, ?, ?)''', (pkg, e_suse_name, e_suse_spec, e_pypi)) + con.execute('''REPLACE INTO etags + (pkg, suse_name_etag, suse_spec_etag, pypi_etag) + VALUES (?, ?, ?, ?)''', (pkg, e_suse_name, e_suse_spec, e_pypi)) con.commit() @@ -116,10 +121,10 @@ def parse_spec(spec_file: str, pkg: str) -> LooseVersion: return LooseVersion(rest_of_line) -def get_spec_name(req: Request, pkg: str, etag: str = None) -> Optional[str]: +def get_spec_name(req: Request, pkg: str, etag: str = None) -> OStr: """Acquire version from the listing of the project directory. """ - spec_files = [] + spec_files = [] # type: List[str] if etag is not None: req.add_header('ETag', etag) @@ -129,7 +134,7 @@ def get_spec_name(req: Request, pkg: str, etag: str = None) -> Optional[str]: try: with opener.open(req) as resp: - etag = resp.info()['ETag'] + etag = str(resp.info()['ETag']) raw_xml_data = ET.parse(resp) root = raw_xml_data.getroot() for elem in root.iter('entry'): @@ -138,7 +143,7 @@ def get_spec_name(req: Request, pkg: str, etag: str = None) -> Optional[str]: spec_files.append(name) except HTTPError as ex: if ex.getcode() == 404: - log.error(f'Cannot acquire version of {pkg}.') + log.warning(f'Cannot acquire version of {pkg}.') return None else: raise @@ -147,11 +152,12 @@ def get_spec_name(req: Request, pkg: str, etag: str = None) -> Optional[str]: fname = sorted(spec_files, key=len)[0] return fname except IndexError: - log.exception(f'Cannot find correct spec_files: {spec_files} ({type(spec_files)}) for {pkg}') - return + log.exception( + f'Cannot find correct spec_files: {spec_files} ({type(spec_files)}) for {pkg}') + return None -def get_version_from_pypi(name: str, con: sqlite3.Connection = None) -> Optional[LooseVersion]: +def get_version_from_pypi(name: str, con: DBConnType = None) -> Optional[LooseVersion]: """ For the given name of module return the latest version available on PyPI. """ @@ -170,17 +176,18 @@ def get_version_from_pypi(name: str, con: sqlite3.Connection = None) -> Optional data = json.load(resp) info_dict = data['info'] if con: - update_etags(con, name, None, None, resp.info()['ETag']) + update_etags(con, name, None, None, str(resp.info()['ETag'])) return LooseVersion(info_dict['version']) except HTTPError as ex: if ex.getcode() == 404: - log.error(f'Cannot find {name} on PyPI') - return None + log.warning(f'Cannot find {name} on PyPI') else: raise + return None -def package_version(proj: str, pkg_name: str, con: sqlite3.Connection = None) -> LooseVersion: +def package_version(proj: str, pkg_name: str, + con: DBConnType = None) -> Optional[LooseVersion]: """ Return the version of the given package in the given proj. @@ -203,21 +210,22 @@ def package_version(proj: str, pkg_name: str, con: sqlite3.Connection = None) -> try: with opener.open(req_spec) as resp: - etag_spcf = resp.info()['ETag'] - spec_file_str = resp.read() + etag_spcf = str(resp.info()['ETag']) + spec_file_str = resp.read().decode() if con: update_etags(con, pkg_name, etag_fn, etag_spcf, None) return parse_spec(spec_file_str, pkg_name) except HTTPError as ex: if ex.getcode() == 404: - log.error(f'Cannot parse SPEC file {spc_fname} for {pkg_name}') + log.warning(f'Cannot parse SPEC file {spc_fname} for {pkg_name}') else: raise + return None def main(prj): - db_name = os.path.split(os.path.basename(sys.argv[0]))[0] + ".db" + db_name = osp.splitext(osp.basename(osp.realpath(__file__)))[0] + ".db" with sqlite3.connect(db_name) as conn: if not conn.execute(TB_EXISTS).fetchone(): conn.execute(TB_CREATE) @@ -243,8 +251,8 @@ def main(prj): print( f'We need to upgrade {pkg} ({suse_ver} -> {pypi_ver})') except TypeError: - log.error(f'{pkg} pypi_ver = {pypi_ver}') - log.error(f'{pkg} suse_ver = {suse_ver}') + log.warning(f'{pkg} pypi_ver = {pypi_ver}') + log.warning(f'{pkg} suse_ver = {suse_ver}') continue else: print(f'Is {pkg} on PyPI?') @@ -260,6 +268,3 @@ if __name__ == '__main__': args = parser.parse_args() sys.exit(main(args.opensuse_project)) - - - diff --git a/exceptions.json b/exceptions.json new file mode 100644 index 0000000..fa6f802 --- /dev/null +++ b/exceptions.json @@ -0,0 +1,13 @@ +{ + "devel:languages:python:numeric": [ + "mayavi", + "opensesame", + "python-espressopp", + "python-pyo", + "python2-matplotlib", + "python3-espressomd", + "spyder", + "spyder3", + "ViTables" + ] +} |