aboutsummaryrefslogtreecommitdiffstats
path: root/import_issues.py
diff options
context:
space:
mode:
Diffstat (limited to 'import_issues.py')
-rwxr-xr-ximport_issues.py170
1 files changed, 123 insertions, 47 deletions
diff --git a/import_issues.py b/import_issues.py
index f487211..4b1722e 100755
--- a/import_issues.py
+++ b/import_issues.py
@@ -46,9 +46,14 @@
#
# 3. Gitlab project exports are missing some crucial information, in particular
# they don't include ticket author names or label IDs. For best results,
-# appropriate mappings for your project can be filled in manually in the LABELS
-# and USERS dicts below, if desired, or these features can be disabled. See the
-# documentation for these variables.
+# appropriate mappings for your project can be provided manually in CSV files to
+# --labels-file and --users-file. These CSV files should be headerless, and
+# each row should contain a label or user ID, followed by the name for that
+# entity. If you want to skip these, then --skip-labels and --skip-users must
+# be passed. Some label and user info will still be included, but label
+# references in comments and issue creator names will be missing. You can run
+# with incomplete files by passing --skip-unknown-labels or
+# --skip-unknown-users.
#
# 4. If your project has confidential issues or comments in it, then you will
# need to decide to exclude them with --skip-confidential, or include them all
@@ -69,20 +74,34 @@
# output to make sure things look right, and ensure that the command completes
# without error:
#
+# touch labels.csv users.csv # First create these empty files.
+#
# ./import_issues.py \
# --srht-owner=MY_SRHT_USER \
# --srht-tracker=MY_SRHT_TRACKER \
# --gitlab-project-url=https://gitlab.com/ME/PROJECT/ \
# --from='Moi <me@email.com>' \
+# --labels-file=labels.csv \
+# --users-file=users.csv \
# .../gitlab-export/tree/project \
# >issue-emails.txt
#
# You may get errors if you are missing label or user mappings, and you haven't
-# disabled these; see the LABELS and USERS variables below.
+# disabled these; add them to the labels.csv or users.csv until you get no more
+# errors:
+#
+# labels.csv:
+# 123456,Bug
+# 232323,Feature
+# ...
+#
+# users.csv:
+# 1234000,John Joe (@jdoe)
+# ...
#
-# If this file looks correct, then you can proceed with sending emails.
-# Double-check that your tracket is empty to start with, then rerun the command
-# with "--mode=send" and with your SMTP parameters. SMTP options can be
+# If the issue-emails.txt file looks correct, then you can proceed with sending
+# emails. Double-check that your tracket is empty to start with, then rerun the
+# command with "--mode=send" and with your SMTP parameters. SMTP options can be
# specified either via parameters --smtp-{host,port,user,password} or the
# equivalent SMTP_{HOST,PORT,USER,PASSWORD} environment variables. Pass
# --smtp-ssl to enable SSL. Also by default there is a five-second delay
@@ -93,6 +112,8 @@
# --srht-tracker=MY_SRHT_TRACKER \
# --gitlab-project-url=https://gitlab.com/ME/PROJECT/ \
# --from='Moi <me@email.com>' \
+# --labels-file=labels.csv \
+# --users-file=users.csv \
# --smtp-host=SMTP_HOSTNAME \
# --smtp-ssl \
# --smtp-user=SMTP_USERNAME \
@@ -101,6 +122,7 @@
import argparse
+import csv
import json
import os
import re
@@ -113,44 +135,32 @@ from pathlib import Path
from typing import Dict, List, Optional
-# Mapping from label IDs to names for the project. This info is unfortunately
-# not included in the Gitlab project export, and it's needed to transform raw
-# label IDs into label names in issue notes.
-#
-# Any missing labels that are referenced from issues will cause an exception to
-# be thrown. Run with --mode=print first to make sure no labels are missing,
-# before using --mode=send.
-#
-# Alternatively, set this to None to disable translation of label IDs to names.
-LABELS: Optional[Dict[int, str]] = {
- # 123456: "Bug",
- # 232323: "Feature",
- # ...
-}
-
-
-# Mapping from user IDs to strings to use for their names when recording who
-# created each ticket. Gitlab exports user full names (but not necessarily
-# IDs) names (but not necessarily IDs) for each note on an issue, but for the
-# creator of an issue, only exports the user ID, no name.
-#
-# Any missing users that created issues will cause an exception to be thrown.
-# Run with --mode=print first to make sure no users are missing, before using
-# --mode=send.
-#
-# Alternatively, set this to None to disable recording issue creators.
-#
-# TODO Might be able to automatically map note.events.author_id to note.author.name.
-USERS: Optional[Dict[int, str]] = {
- # 1234000: "John Joe (@jdoe)",
- # ...
-}
+ID_RE = re.compile(r'^[0-9]+$')
email_count = 0
issue_count = 0
+def read_id_map_file(file_path: Path) -> Dict[int, str]:
+ """Reads a CSV file with ID,NAME mappings and returns the resulting dict."""
+ result: Dict[int, str] = {}
+
+ with open(file_path, newline='') as fh:
+ reader = csv.reader(fh)
+ line_num = 0
+ for row in reader:
+ line_num += 1
+ assert len(row) == 2 and ID_RE.search(row[0]) and row[1], \
+ f"Row {line_num} of {file_path} is not in the form <ID>,<NAME>: {row!r}"
+ new_id = int(row[0])
+ assert new_id not in result, \
+ f"ID {new_id} appears multiple times in {file_path}."
+ result[new_id] = row[1]
+
+ return result
+
+
def do_mail(
*,
smtp,
@@ -396,11 +406,22 @@ def run(
frm: str,
export_dir_path: Path,
gitlab_project_url: str,
+ labels_file_path: Optional[Path],
+ skip_unknown_labels: bool,
+ users_file_path: Optional[Path],
+ skip_unknown_users: bool,
skip_missing_issues: bool,
create_missing_issues: bool,
include_confidential: bool,
skip_confidential: bool,
):
+ label_ids_to_names: Optional[Dict[int, str]] = \
+ read_id_map_file(labels_file_path) if labels_file_path else None
+ user_ids_to_names: Optional[Dict[int, str]] = \
+ read_id_map_file(users_file_path) if users_file_path else None
+ # TODO Might be able to automatically map note.events.author_id to
+ # note.author.name for a subset of relevant users.
+
milestone_jsons = []
with open(export_dir_path / 'milestones.ndjson') as milestones_file:
for line in milestones_file:
@@ -501,12 +522,15 @@ def run(
author_id = issue_json['author_id']
created_by: Optional[str]
- if USERS is None:
+ if user_ids_to_names is None:
created_by = None
+ elif author_id in user_ids_to_names:
+ created_by = user_ids_to_names[author_id]
else:
- assert author_id in USERS, \
- f"Unknown author #{author_id} of ticket #{gitlab_issue_id}, please add to USERS."
- created_by = USERS[author_id]
+ assert skip_unknown_users, \
+ f"Unknown author #{author_id} of ticket #{gitlab_issue_id}, " \
+ f"please add to the users file."
+ created_by = None
srht_issue_id = open_ticket(
smtp=smtp,
@@ -545,14 +569,16 @@ def run(
body = note_json['note']
# The "Removed" part is a guess here, don't know if that actually shows up.
- if LABELS is not None and (
+ if label_ids_to_names is not None and (
system_action == 'label' or re.search(r'^(Added|Removed) ~[0-9]+ label', body)
):
def expand_label(ref):
ref_num = int(ref.group(1))
- assert ref_num in LABELS, \
- f"Unknown label #{ref_num}, please add to LABELS."
- return LABELS[ref_num]
+ if ref_num in label_ids_to_names:
+ return label_ids_to_names[ref_num]
+ assert skip_unknown_labels, \
+ f"Unknown label #{ref_num}, please add to the labels file."
+ return ref.group(0) # Return the original "~id" string.
body = re.sub(r'~([0-9]+)', expand_label, body)
@@ -673,6 +699,40 @@ def main():
)
parser.add_argument(
+ '--labels-file',
+ help="CSV file mapping label IDs to names.",
+ )
+
+ parser.add_argument(
+ '--skip-labels',
+ action='store_true',
+ help="Skip mapping label IDs to names.",
+ )
+
+ parser.add_argument(
+ '--skip-unknown-labels',
+ action='store_true',
+ help="Skip mapping labels that aren't in the labels file.",
+ )
+
+ parser.add_argument(
+ '--users-file',
+ help="CSV file mapping user IDs to names.",
+ )
+
+ parser.add_argument(
+ '--skip-users',
+ action='store_true',
+ help="Skip mapping user IDs to names.",
+ )
+
+ parser.add_argument(
+ '--skip-unknown-users',
+ action='store_true',
+ help="Skip mapping users that aren't in the users file.",
+ )
+
+ parser.add_argument(
'--skip-missing-issues',
action='store_true',
help="Skip missing Gitlab issue IDs; GL and sr.ht IDs will not match.",
@@ -712,6 +772,18 @@ def main():
mode = args['mode']
frm = args['from']
+ labels_file = args['labels_file']
+ skip_labels = args['skip_labels']
+ skip_unknown_labels = args['skip_unknown_labels']
+ assert labels_file or skip_labels, \
+ f"One of --labels-file or --skip-labels must be provided."
+
+ users_file = args['users_file']
+ skip_users = args['skip_users']
+ skip_unknown_users = args['skip_unknown_users']
+ assert skip_users or users_file, \
+ f"One of --users-file or --skip-users must be provided."
+
skip_missing_issues = args['skip_missing_issues']
create_missing_issues = args['create_missing_issues']
assert not (skip_missing_issues and create_missing_issues), \
@@ -760,6 +832,10 @@ def main():
frm=frm,
export_dir_path=export_dir_path,
gitlab_project_url=args['gitlab_project_url'].rstrip('/'),
+ labels_file_path=None if skip_labels else Path(labels_file),
+ skip_unknown_labels=skip_unknown_labels,
+ users_file_path=None if skip_users else Path(users_file),
+ skip_unknown_users=skip_unknown_users,
skip_missing_issues=skip_missing_issues,
create_missing_issues=create_missing_issues,
include_confidential=include_confidential,