Various touch-ups now that we have hut support.HEADmasterdevel

1 files changed, 103 insertions, 70 deletions

diff --git a/import_issues.py b/import_issues.py
index 42b0a1f..106c613 100755
--- a/import_issues.py
+++ b/import_issues.py
@@ -136,7 +136,6 @@ from email.message import EmailMessage
 from email.utils import format_datetime, make_msgid
 from pathlib import Path
 from typing import Any, Optional
-from urllib.parse import urlparse
 
 logging.basicConfig(
     format="%(levelname)s:%(funcName)s:%(message)s",
@@ -150,8 +149,6 @@ ID_RE = re.compile(r"^[0-9]+$")
 # to be sure, we will do just 12k
 MAX_SIZE_COMMENT = 12 * 1024
 
-tickets_to_be_closed = []
-
 
 def split_long_str(in_str: str, max_len: int = MAX_SIZE_COMMENT) -> list[str]:
     out = []
@@ -167,6 +164,9 @@ def split_long_str(in_str: str, max_len: int = MAX_SIZE_COMMENT) -> list[str]:
     return out
 
 
+label_cache: dict[str, list[dict[str, str]]] = {}
+
+
 def get_labels(tracker: str) -> list[dict[str, str]]:
     """
     collects labels for your named tracker
@@ -174,26 +174,36 @@ def get_labels(tracker: str) -> list[dict[str, str]]:
     param: tracker: name of the tracker
     return: list of all labels in the tracker
     """
-    query = (
-        'query { me { tracker(name: "'
-        + tracker
-        + '") { labels { results { id, name, foregroundColor, backgroundColor, created } } } }}'
-    )
+    global label_cache
 
-    try:
-        ret = subprocess.run(
-            ["hut", "graphql", "todo", "--stdin"],
-            input=query,
-            text=True,
-            check=True,
-            capture_output=True,
+    result: list[dict[str, str]]
+
+    if tracker in label_cache:
+        result = label_cache[tracker]
+    else:
+        query = (
+            'query { me { tracker(name: "'
+            + tracker
+            + '") { labels { results { id, name, foregroundColor, backgroundColor, created } } } }}'
         )
-    except subprocess.CalledProcessError as ex:
-        raise RuntimeError(
-            f"hut failed with excitcode {ex.returncode}\n\nstdout:\n{ex.stdout}\n\nand stderr:\n{ex.stderr}"
-        ) from ex
-    data = json.loads(ret.stdout)
-    return data["me"]["tracker"]["labels"]["results"]
+
+        try:
+            ret = subprocess.run(
+                ["hut", "graphql", "todo", "--stdin"],
+                input=query,
+                text=True,
+                check=True,
+                capture_output=True,
+            )
+        except subprocess.CalledProcessError as ex:
+            raise RuntimeError(
+                f"hut failed with exitcode {ex.returncode}\n\nstdout:\n{ex.stdout}\n\nand stderr:\n{ex.stderr}"
+            ) from ex
+        data = json.loads(ret.stdout)
+        result = data["me"]["tracker"]["labels"]["results"]
+        label_cache[tracker] = result
+
+    return result
 
 
 log = logging.getLogger()
@@ -284,7 +294,9 @@ def do_mail(
         raise RuntimeError(f"Unknown mode: {mode!r}")
 
 
-def run_hut(cmds, tracker, msg, args=None, delay=None):
+def run_hut(
+    cmds, tracker, msg, args=[], *, delay: float
+) -> subprocess.CompletedProcess:
     log.debug(
         "cmds = %s, tracker = %s, args = %s\n\nmsg:\n%s", cmds, tracker, args, msg
     )
@@ -310,7 +322,7 @@ def run_hut(cmds, tracker, msg, args=None, delay=None):
             )
     except subprocess.CalledProcessError as ex:
         raise RuntimeError(
-            f"hut failed with excitcode {ex.returncode}\n\nstdout:\n{ex.stdout}\n\nand stderr:\n{ex.stderr}"
+            f"hut failed with exitcode {ex.returncode}\n\nstdout:\n{ex.stdout}\n\nand stderr:\n{ex.stderr}"
         ) from ex
 
     time.sleep(delay)
@@ -319,6 +331,7 @@ def run_hut(cmds, tracker, msg, args=None, delay=None):
 
 
 def open_ticket_by_email(
+    *,
     smtp,
     delay: float,
     mode: str,
@@ -335,6 +348,15 @@ def open_ticket_by_email(
     milestone_name: Optional[str],
     gitlab_ticket_url: str,
 ):
+    # This is presently the ID of the issue we are going to create, following
+    # along with e.g. any skipped confidential issues.  Unfortunately, unlike
+    # calling 'hut', we don't get a response after sending the email to create
+    # the ticket that tells us what the ticket ID actually is, so we have to
+    # assume it's the next one in sequence.  It should be, unless ticket
+    # creation fails for some reason (again, we don't know if that happens, we
+    # can't see email responses).
+    global issue_count
+
     lines = []
     pheaders = []
 
@@ -385,24 +407,21 @@ def open_ticket_by_email(
 
     # then add remaining parts of the description as comments
     if len(body_split) > 1:
-        url_split = urlparse(gitlab_ticket_url)
-        issue_id = int(os.path.basename(url_split.path))
-
         for part in body_split[1:]:
             do_mail(
                 smtp=smtp,
                 delay=delay,
                 mode=mode,
                 frm=frm,
-                to=f"{tracker}/{issue_id}@todo.sr.ht",
+                to=f"{tracker}/{issue_count}@todo.sr.ht",
                 body=part,
             )
 
 
 def open_ticket_by_hut(
+    *,
     delay: float,
     tracker: str,
-    frm: str,
     title: str,
     body: str,
     created_by: Optional[str],
@@ -413,8 +432,7 @@ def open_ticket_by_hut(
     labels: list[dict[str, Any]],
     milestone_name: Optional[str],
     gitlab_ticket_url: str,
-) -> str:
-
+):
     lines = []
     pheaders = []
 
@@ -445,7 +463,12 @@ def open_ticket_by_hut(
     msg_split = split_long_str(msg)
 
     out = run_hut(["ticket", "create"], tracker, msg_split[0], delay=delay)
-    out.issue_id = int(out.stderr.strip()[len("Created new ticket #") :])
+    stderr_msg = out.stderr.strip()
+    expected_prefix = "Created new ticket #"
+    assert stderr_msg.startswith(
+        expected_prefix
+    ), f"Expected stderr to start with {expected_prefix!r}, stderr = {stderr_msg!r}"
+    issue_id = int(stderr_msg[len(expected_prefix) :])
 
     for label in sorted(labels, key=lambda x: x["label"]["title"]):
         # {"target_type":"Issue",
@@ -468,18 +491,14 @@ def open_ticket_by_hut(
             ["ticket", "label"],
             tracker,
             None,
-            args=[str(out.issue_id), "-l", label_name],
+            [str(issue_id), "-l", label_name],
             delay=delay,
         )
 
     # then add remaining parts of the description as comments
     if len(msg_split) > 1:
         for part in msg_split[1:]:
-            run_hut(
-                ["ticket", "comment"], tracker, part, [str(out.issue_id)], delay=delay
-            )
-
-    return out
+            run_hut(["ticket", "comment"], tracker, part, [str(issue_id)], delay=delay)
 
 
 def open_ticket(
@@ -502,42 +521,44 @@ def open_ticket(
 ) -> int:
     global issue_count
 
+    # Proactively increment this so that open_ticket_by_email() can use it for
+    # the current issue ID.
+    issue_count += 1
+
     if mode in ("send", "print"):
         open_ticket_by_email(
-            smtp,
-            delay,
-            mode,
-            tracker,
-            frm,
-            title,
-            body,
-            created_by,
-            created_at,
-            closed_at,
-            is_closed,
-            is_confidential,
-            labels,
-            milestone_name,
-            gitlab_ticket_url,
+            smtp=smtp,
+            delay=delay,
+            mode=mode,
+            tracker=tracker,
+            frm=frm,
+            title=title,
+            body=body,
+            created_by=created_by,
+            created_at=created_at,
+            closed_at=closed_at,
+            is_closed=is_closed,
+            is_confidential=is_confidential,
+            labels=labels,
+            milestone_name=milestone_name,
+            gitlab_ticket_url=gitlab_ticket_url,
         )
     elif mode == "hut":
         open_ticket_by_hut(
-            delay,
-            tracker,
-            frm,
-            title,
-            body,
-            created_by,
-            created_at,
-            closed_at,
-            is_closed,
-            is_confidential,
-            labels,
-            milestone_name,
-            gitlab_ticket_url,
+            delay=delay,
+            tracker=tracker,
+            title=title,
+            body=body,
+            created_by=created_by,
+            created_at=created_at,
+            closed_at=closed_at,
+            is_closed=is_closed,
+            is_confidential=is_confidential,
+            labels=labels,
+            milestone_name=milestone_name,
+            gitlab_ticket_url=gitlab_ticket_url,
         )
 
-    issue_count += 1
     return issue_count
 
 
@@ -549,6 +570,7 @@ def file_missing_ticket(
     tracker: str,
     frm: str,
     issue_id: int,
+    tickets_to_be_closed: list[tuple[Any, float, str, str, str, int]],
 ):
     global issue_count
 
@@ -680,10 +702,17 @@ def close_ticket(
 
 
 def ensure_label(
-    tracker: str, name: str, bg_color: str, fg_color: str = "#FFFFFF", delay=None
+    tracker: str,
+    name: str,
+    bg_color: str,
+    fg_color: str = "#FFFFFF",
+    *,
+    delay: float,
 ):
+    global label_cache
+
     labels = get_labels(tracker.split("/", 1)[1])
-    if not ([x for x in labels if x["name"] == name]):
+    if not any(x["name"] == name for x in labels):
         run_hut(
             ["label", "create"],
             tracker,
@@ -692,6 +721,8 @@ def ensure_label(
             delay=delay,
         )
 
+        label_cache.pop(tracker, None)  # The label cache is now invalid.
+
 
 def run(
     *,
@@ -790,6 +821,7 @@ def run(
     log.info("Creating tickets.")
 
     issue_id_map: dict[int, int] = {}
+    tickets_to_be_closed: list[tuple[Any, float, str, str, str, int]] = []
 
     # While we're creating tickets, we can't just loop over the sorted
     # issue_jsons.  We have to loop over potential issue IDs and handle any that
@@ -804,6 +836,7 @@ def run(
                     tracker=tracker,
                     frm=frm,
                     issue_id=gitlab_issue_id,
+                    tickets_to_be_closed=tickets_to_be_closed,
                 )
             elif not skip_missing_issues:
                 raise RuntimeError(
@@ -926,7 +959,7 @@ def run(
                 is_closed=(issue_json["state"] == "closed"),
             )
 
-    log.info("Delayed closing issues.")
+    log.info("Performing delayed closing of issues.")
     for ticket in tickets_to_be_closed:
         close_ticket(
             smtp=ticket[0],
@@ -936,7 +969,7 @@ def run(
             frm=ticket[4],
             issue_id=ticket[5],
             closed_at=None,
-            is_closed=False,
+            is_closed=False,  # Save one line of text.
         )