diff options
| -rwxr-xr-x | import_issues.py | 170 |
1 files changed, 123 insertions, 47 deletions
diff --git a/import_issues.py b/import_issues.py index f487211..4b1722e 100755 --- a/import_issues.py +++ b/import_issues.py @@ -46,9 +46,14 @@ # # 3. Gitlab project exports are missing some crucial information, in particular # they don't include ticket author names or label IDs. For best results, -# appropriate mappings for your project can be filled in manually in the LABELS -# and USERS dicts below, if desired, or these features can be disabled. See the -# documentation for these variables. +# appropriate mappings for your project can be provided manually in CSV files to +# --labels-file and --users-file. These CSV files should be headerless, and +# each row should contain a label or user ID, followed by the name for that +# entity. If you want to skip these, then --skip-labels and --skip-users must +# be passed. Some label and user info will still be included, but label +# references in comments and issue creator names will be missing. You can run +# with incomplete files by passing --skip-unknown-labels or +# --skip-unknown-users. # # 4. If your project has confidential issues or comments in it, then you will # need to decide to exclude them with --skip-confidential, or include them all @@ -69,20 +74,34 @@ # output to make sure things look right, and ensure that the command completes # without error: # +# touch labels.csv users.csv # First create these empty files. +# # ./import_issues.py \ # --srht-owner=MY_SRHT_USER \ # --srht-tracker=MY_SRHT_TRACKER \ # --gitlab-project-url=https://gitlab.com/ME/PROJECT/ \ # --from='Moi <me@email.com>' \ +# --labels-file=labels.csv \ +# --users-file=users.csv \ # .../gitlab-export/tree/project \ # >issue-emails.txt # # You may get errors if you are missing label or user mappings, and you haven't -# disabled these; see the LABELS and USERS variables below. +# disabled these; add them to the labels.csv or users.csv until you get no more +# errors: +# +# labels.csv: +# 123456,Bug +# 232323,Feature +# ... +# +# users.csv: +# 1234000,John Joe (@jdoe) +# ... # -# If this file looks correct, then you can proceed with sending emails. -# Double-check that your tracket is empty to start with, then rerun the command -# with "--mode=send" and with your SMTP parameters. SMTP options can be +# If the issue-emails.txt file looks correct, then you can proceed with sending +# emails. Double-check that your tracket is empty to start with, then rerun the +# command with "--mode=send" and with your SMTP parameters. SMTP options can be # specified either via parameters --smtp-{host,port,user,password} or the # equivalent SMTP_{HOST,PORT,USER,PASSWORD} environment variables. Pass # --smtp-ssl to enable SSL. Also by default there is a five-second delay @@ -93,6 +112,8 @@ # --srht-tracker=MY_SRHT_TRACKER \ # --gitlab-project-url=https://gitlab.com/ME/PROJECT/ \ # --from='Moi <me@email.com>' \ +# --labels-file=labels.csv \ +# --users-file=users.csv \ # --smtp-host=SMTP_HOSTNAME \ # --smtp-ssl \ # --smtp-user=SMTP_USERNAME \ @@ -101,6 +122,7 @@ import argparse +import csv import json import os import re @@ -113,44 +135,32 @@ from pathlib import Path from typing import Dict, List, Optional -# Mapping from label IDs to names for the project. This info is unfortunately -# not included in the Gitlab project export, and it's needed to transform raw -# label IDs into label names in issue notes. -# -# Any missing labels that are referenced from issues will cause an exception to -# be thrown. Run with --mode=print first to make sure no labels are missing, -# before using --mode=send. -# -# Alternatively, set this to None to disable translation of label IDs to names. -LABELS: Optional[Dict[int, str]] = { - # 123456: "Bug", - # 232323: "Feature", - # ... -} - - -# Mapping from user IDs to strings to use for their names when recording who -# created each ticket. Gitlab exports user full names (but not necessarily -# IDs) names (but not necessarily IDs) for each note on an issue, but for the -# creator of an issue, only exports the user ID, no name. -# -# Any missing users that created issues will cause an exception to be thrown. -# Run with --mode=print first to make sure no users are missing, before using -# --mode=send. -# -# Alternatively, set this to None to disable recording issue creators. -# -# TODO Might be able to automatically map note.events.author_id to note.author.name. -USERS: Optional[Dict[int, str]] = { - # 1234000: "John Joe (@jdoe)", - # ... -} +ID_RE = re.compile(r'^[0-9]+$') email_count = 0 issue_count = 0 +def read_id_map_file(file_path: Path) -> Dict[int, str]: + """Reads a CSV file with ID,NAME mappings and returns the resulting dict.""" + result: Dict[int, str] = {} + + with open(file_path, newline='') as fh: + reader = csv.reader(fh) + line_num = 0 + for row in reader: + line_num += 1 + assert len(row) == 2 and ID_RE.search(row[0]) and row[1], \ + f"Row {line_num} of {file_path} is not in the form <ID>,<NAME>: {row!r}" + new_id = int(row[0]) + assert new_id not in result, \ + f"ID {new_id} appears multiple times in {file_path}." + result[new_id] = row[1] + + return result + + def do_mail( *, smtp, @@ -396,11 +406,22 @@ def run( frm: str, export_dir_path: Path, gitlab_project_url: str, + labels_file_path: Optional[Path], + skip_unknown_labels: bool, + users_file_path: Optional[Path], + skip_unknown_users: bool, skip_missing_issues: bool, create_missing_issues: bool, include_confidential: bool, skip_confidential: bool, ): + label_ids_to_names: Optional[Dict[int, str]] = \ + read_id_map_file(labels_file_path) if labels_file_path else None + user_ids_to_names: Optional[Dict[int, str]] = \ + read_id_map_file(users_file_path) if users_file_path else None + # TODO Might be able to automatically map note.events.author_id to + # note.author.name for a subset of relevant users. + milestone_jsons = [] with open(export_dir_path / 'milestones.ndjson') as milestones_file: for line in milestones_file: @@ -501,12 +522,15 @@ def run( author_id = issue_json['author_id'] created_by: Optional[str] - if USERS is None: + if user_ids_to_names is None: created_by = None + elif author_id in user_ids_to_names: + created_by = user_ids_to_names[author_id] else: - assert author_id in USERS, \ - f"Unknown author #{author_id} of ticket #{gitlab_issue_id}, please add to USERS." - created_by = USERS[author_id] + assert skip_unknown_users, \ + f"Unknown author #{author_id} of ticket #{gitlab_issue_id}, " \ + f"please add to the users file." + created_by = None srht_issue_id = open_ticket( smtp=smtp, @@ -545,14 +569,16 @@ def run( body = note_json['note'] # The "Removed" part is a guess here, don't know if that actually shows up. - if LABELS is not None and ( + if label_ids_to_names is not None and ( system_action == 'label' or re.search(r'^(Added|Removed) ~[0-9]+ label', body) ): def expand_label(ref): ref_num = int(ref.group(1)) - assert ref_num in LABELS, \ - f"Unknown label #{ref_num}, please add to LABELS." - return LABELS[ref_num] + if ref_num in label_ids_to_names: + return label_ids_to_names[ref_num] + assert skip_unknown_labels, \ + f"Unknown label #{ref_num}, please add to the labels file." + return ref.group(0) # Return the original "~id" string. body = re.sub(r'~([0-9]+)', expand_label, body) @@ -673,6 +699,40 @@ def main(): ) parser.add_argument( + '--labels-file', + help="CSV file mapping label IDs to names.", + ) + + parser.add_argument( + '--skip-labels', + action='store_true', + help="Skip mapping label IDs to names.", + ) + + parser.add_argument( + '--skip-unknown-labels', + action='store_true', + help="Skip mapping labels that aren't in the labels file.", + ) + + parser.add_argument( + '--users-file', + help="CSV file mapping user IDs to names.", + ) + + parser.add_argument( + '--skip-users', + action='store_true', + help="Skip mapping user IDs to names.", + ) + + parser.add_argument( + '--skip-unknown-users', + action='store_true', + help="Skip mapping users that aren't in the users file.", + ) + + parser.add_argument( '--skip-missing-issues', action='store_true', help="Skip missing Gitlab issue IDs; GL and sr.ht IDs will not match.", @@ -712,6 +772,18 @@ def main(): mode = args['mode'] frm = args['from'] + labels_file = args['labels_file'] + skip_labels = args['skip_labels'] + skip_unknown_labels = args['skip_unknown_labels'] + assert labels_file or skip_labels, \ + f"One of --labels-file or --skip-labels must be provided." + + users_file = args['users_file'] + skip_users = args['skip_users'] + skip_unknown_users = args['skip_unknown_users'] + assert skip_users or users_file, \ + f"One of --users-file or --skip-users must be provided." + skip_missing_issues = args['skip_missing_issues'] create_missing_issues = args['create_missing_issues'] assert not (skip_missing_issues and create_missing_issues), \ @@ -760,6 +832,10 @@ def main(): frm=frm, export_dir_path=export_dir_path, gitlab_project_url=args['gitlab_project_url'].rstrip('/'), + labels_file_path=None if skip_labels else Path(labels_file), + skip_unknown_labels=skip_unknown_labels, + users_file_path=None if skip_users else Path(users_file), + skip_unknown_users=skip_unknown_users, skip_missing_issues=skip_missing_issues, create_missing_issues=create_missing_issues, include_confidential=include_confidential, |