More improvements.

  * there is now JSON file exceptions.json which contains dict with
    names of packages for each project, which shouldn’t be searched for
    on PyPI.
  * remove PEP-484 type hints, we actually don’t need them
  * fix location of the SQLite3 database file with ETags

2 files changed, 61 insertions, 43 deletions

diff --git a/dlpcvp.py b/dlpcvp.py
index 02a0bb5..ee44474 100755
--- a/dlpcvp.py
+++ b/dlpcvp.py
@@ -3,15 +3,14 @@ import argparse
 import configparser
 import json
 import logging
-import os
-import os.path
+import os.path as osp
 import sqlite3
 import sys
 import urllib.request
 from distutils.version import LooseVersion
 from urllib.error import HTTPError
 from urllib.request import Request, urlopen
-from typing import Iterable, Optional, Tuple
+from typing import Iterable, List, Optional, Tuple
 import xml.etree.ElementTree as ET
 
 
@@ -20,12 +19,16 @@ PyPI_base = "https://pypi.org/pypi/{}/json"
 # https://github.com/openSUSE/open-build-service/blob/master/docs/api/api/api.txt
 # https://build.opensuse.org/apidocs/index
 OBS_base = "https://api.opensuse.org"
-ConfigRC = [os.path.expanduser('~/.oscrc'),
-            os.path.expanduser('~/.config/osc/oscrc')]
+OBS_realm = "Use your novell account"
+ConfigRCs = [osp.expanduser('~/.oscrc'),
+             osp.expanduser('~/.config/osc/oscrc')]
 CUTCHARS = len('python-')
 
+DBConnType = sqlite3.Connection
+OStr = Optional[str]
+
 config = configparser.ConfigParser()
-config.read(ConfigRC)
+config.read(ConfigRCs)
 
 logging.basicConfig(format='%(levelname)s:%(funcName)s:%(message)s',
                     stream=sys.stdout, level=logging.INFO)
@@ -35,10 +38,7 @@ log = logging.getLogger()
 password_mgr = urllib.request.HTTPPasswordMgrWithDefaultRealm()
 user = config[OBS_base]['user']
 passw = config[OBS_base]['pass']
-# MYPY: Argument 1 to "add_password" of
-# "HTTPPasswordMgrWithDefaultRealm" has incompatible type "None";
-# expected "str"
-password_mgr.add_password(None, OBS_base, user, passw)
+password_mgr.add_password(OBS_realm, OBS_base, user, passw)
 
 handler = urllib.request.HTTPBasicAuthHandler(password_mgr)
 opener = urllib.request.build_opener(handler)
@@ -48,7 +48,7 @@ TB_EXISTS = """SELECT name FROM sqlite_master
         WHERE type='table' AND name='etags'
         """
 TB_CREATE = """CREATE TABLE etags
-               (pkg text NOT NULL,
+               (pkg text NOT NULL PRIMARY KEY,
                suse_name_etag text, suse_spec_etag text, pypi_etag text)
             """
 TB_CREATE_IDX = "CREATE UNIQUE INDEX etags_names ON etags(pkg)"
@@ -62,22 +62,30 @@ def suse_packages(proj: str) -> Iterable[str]:
     how to return it in iterator.
     """
     req = Request(url=OBS_base + f'/source/{proj}')
+    with open('exceptions.json') as exc_f:
+        exc_dict = json.load(exc_f)
+
+    if proj in exc_dict:
+        exc_list = exc_dict[proj]
 
     try:
         with opener.open(req) as resp:
             raw_xml_data = ET.parse(resp)
             root = raw_xml_data.getroot()
             for elem in root.iter('entry'):
-                yield elem.get('name')
+                pkg_name = elem.get('name')
+                if pkg_name in exc_list:
+                    continue
+                yield pkg_name
     except HTTPError as ex:
         if ex.getcode() == 404:
-            log.error(f'Cannot find packages for {proj}!')
+            log.warning(f'Cannot find packages for {proj}!')
         else:
             raise
 
 
-def get_etags(con: sqlite3.Connection, pkg: str) -> \
-        Tuple[Optional[str], Optional[str], Optional[str], Optional[str]]:
+def get_etags(con: DBConnType, pkg: str) -> \
+        Tuple[OStr, OStr, OStr, OStr]:
     # pkg, suse_name_etag, suse_spec_etag, pypi_etag
     cur = con.execute("SELECT * FROM etags WHERE pkg=?", (pkg,))
     ret = cur.fetchone()
@@ -87,19 +95,16 @@ def get_etags(con: sqlite3.Connection, pkg: str) -> \
         return ret
 
 
-def update_etags(con: sqlite3.Connection, pkg: str, e_suse_name: Optional[str], e_suse_spec: Optional[str], e_pypi: Optional[str]):
+def update_etags(con: sqlite3.Connection, pkg: str,
+                 e_suse_name: OStr, e_suse_spec: OStr, e_pypi: OStr):
     res = con.execute("SELECT * FROM etags WHERE pkg=?", (pkg,)).fetchone()
     if res:
         e_suse_name = res[1] if e_suse_name is None else e_suse_name
         e_suse_spec = res[2] if e_suse_spec is None else e_suse_spec
         e_pypi = res[3] if e_pypi is None else e_pypi
-        con.execute('''UPDATE etags
-            SET suse_name_etag=?, suse_spec_etag=?, pypi_etag=?
-            WHERE pkg=?''', (e_suse_name, e_suse_spec, e_pypi, pkg))
-    else:
-        con.execute('''INSERT INTO etags
-            (pkg, suse_name_etag, suse_spec_etag, pypi_etag)
-            VALUES (?, ?, ?, ?)''', (pkg, e_suse_name, e_suse_spec, e_pypi))
+    con.execute('''REPLACE INTO etags
+        (pkg, suse_name_etag, suse_spec_etag, pypi_etag)
+        VALUES (?, ?, ?, ?)''', (pkg, e_suse_name, e_suse_spec, e_pypi))
     con.commit()
 
 
@@ -116,10 +121,10 @@ def parse_spec(spec_file: str, pkg: str) -> LooseVersion:
     return LooseVersion(rest_of_line)
 
 
-def get_spec_name(req: Request, pkg: str, etag: str = None) -> Optional[str]:
+def get_spec_name(req: Request, pkg: str, etag: str = None) -> OStr:
     """Acquire version from the listing of the project directory.
     """
-    spec_files = []
+    spec_files = []  # type: List[str]
 
     if etag is not None:
         req.add_header('ETag', etag)
@@ -129,7 +134,7 @@ def get_spec_name(req: Request, pkg: str, etag: str = None) -> Optional[str]:
 
     try:
         with opener.open(req) as resp:
-            etag = resp.info()['ETag']
+            etag = str(resp.info()['ETag'])
             raw_xml_data = ET.parse(resp)
             root = raw_xml_data.getroot()
             for elem in root.iter('entry'):
@@ -138,7 +143,7 @@ def get_spec_name(req: Request, pkg: str, etag: str = None) -> Optional[str]:
                     spec_files.append(name)
     except HTTPError as ex:
         if ex.getcode() == 404:
-            log.error(f'Cannot acquire version of {pkg}.')
+            log.warning(f'Cannot acquire version of {pkg}.')
             return None
         else:
             raise
@@ -147,11 +152,12 @@ def get_spec_name(req: Request, pkg: str, etag: str = None) -> Optional[str]:
         fname = sorted(spec_files, key=len)[0]
         return fname
     except IndexError:
-        log.exception(f'Cannot find correct spec_files: {spec_files} ({type(spec_files)}) for {pkg}')
-        return
+        log.exception(
+            f'Cannot find correct spec_files: {spec_files} ({type(spec_files)}) for {pkg}')
+        return None
 
 
-def get_version_from_pypi(name: str, con: sqlite3.Connection = None) -> Optional[LooseVersion]:
+def get_version_from_pypi(name: str, con: DBConnType = None) -> Optional[LooseVersion]:
     """
     For the given name of module return the latest version available on PyPI.
     """
@@ -170,17 +176,18 @@ def get_version_from_pypi(name: str, con: sqlite3.Connection = None) -> Optional
             data = json.load(resp)
             info_dict = data['info']
             if con:
-                update_etags(con, name, None, None, resp.info()['ETag'])
+                update_etags(con, name, None, None, str(resp.info()['ETag']))
             return LooseVersion(info_dict['version'])
     except HTTPError as ex:
         if ex.getcode() == 404:
-            log.error(f'Cannot find {name} on PyPI')
-            return None
+            log.warning(f'Cannot find {name} on PyPI')
         else:
             raise
+        return None
 
 
-def package_version(proj: str, pkg_name: str, con: sqlite3.Connection = None) -> LooseVersion:
+def package_version(proj: str, pkg_name: str,
+                    con: DBConnType = None) -> Optional[LooseVersion]:
     """
     Return the version of the given package in the given proj.
 
@@ -203,21 +210,22 @@ def package_version(proj: str, pkg_name: str, con: sqlite3.Connection = None) ->
 
     try:
         with opener.open(req_spec) as resp:
-            etag_spcf = resp.info()['ETag']
-            spec_file_str = resp.read()
+            etag_spcf = str(resp.info()['ETag'])
+            spec_file_str = resp.read().decode()
 
             if con:
                 update_etags(con, pkg_name, etag_fn, etag_spcf, None)
             return parse_spec(spec_file_str, pkg_name)
     except HTTPError as ex:
         if ex.getcode() == 404:
-            log.error(f'Cannot parse SPEC file {spc_fname} for {pkg_name}')
+            log.warning(f'Cannot parse SPEC file {spc_fname} for {pkg_name}')
         else:
             raise
+        return None
 
 
 def main(prj):
-    db_name = os.path.split(os.path.basename(sys.argv[0]))[0] + ".db"
+    db_name = osp.splitext(osp.basename(osp.realpath(__file__)))[0] + ".db"
     with sqlite3.connect(db_name) as conn:
         if not conn.execute(TB_EXISTS).fetchone():
             conn.execute(TB_CREATE)
@@ -243,8 +251,8 @@ def main(prj):
                         print(
                             f'We need to upgrade {pkg} ({suse_ver} -> {pypi_ver})')
                 except TypeError:
-                    log.error(f'{pkg} pypi_ver = {pypi_ver}')
-                    log.error(f'{pkg} suse_ver = {suse_ver}')
+                    log.warning(f'{pkg} pypi_ver = {pypi_ver}')
+                    log.warning(f'{pkg} suse_ver = {suse_ver}')
                     continue
             else:
                 print(f'Is {pkg} on PyPI?')
@@ -260,6 +268,3 @@ if __name__ == '__main__':
 
     args = parser.parse_args()
     sys.exit(main(args.opensuse_project))
-
-
-
diff --git a/exceptions.json b/exceptions.json
new file mode 100644
index 0000000..fa6f802
--- /dev/null
+++ b/exceptions.json
@@ -0,0 +1,13 @@
+{
+  "devel:languages:python:numeric": [
+    "mayavi",
+    "opensesame",
+    "python-espressopp",
+    "python-pyo",
+    "python2-matplotlib",
+    "python3-espressomd",
+    "spyder",
+    "spyder3",
+    "ViTables"
+  ]
+}