1#!/usr/bin/python3
2
3# Used to generate changelogs from the repository.
4
5from __future__ import print_function
6
7import subprocess
8import re
9import sys
10import os
11import os.path
12
13# Holds each changelog entry indexed by SHA
14ENTRIES = {}
15# Links SHAs together, if we have a "X cherry-picked from Y" situation, those
16# two commits will be linked, and this will be used in cases where we have
17# reverted a commit.
18LINKED_SHAS = {}
19# A map of shas to a list of bugtracker numbers, as extracted from the commit
20# messages.
21SHA_TO_TRACKER = {}
22
23# more relaxed regexp to find JIRA issues anywhere in commit message
24JIRA_REGEX = r"(?:Jira:? *)?(?:https?://tracker.mender.io/browse/)?((?:CFE|ENT|INF|ARCHIVE|MEN|QA)-[0-9]+)"
25# more strict regexp to find JIRA issues only in the beginning of title
26JIRA_TITLE_REGEX = r"^(?:CFE|ENT|INF|ARCHIVE|MEN|QA)-[0-9]+"
27TRACKER_REGEX = r"\(?(?:Ref:? *)?%s\)?:? *" % (JIRA_REGEX)
28
29POSSIBLE_MISSED_TICKETS = {}
30
31# Only for testing.
32SORT_CHANGELOG = True
33
34# Type of log to generate, this is bitwise.
35LOG_TYPE = 0
36# Values for the above.
37LOG_REPO = 1
38LOG_COMMUNITY = 2
39LOG_ENTERPRISE = 4
40LOG_MASTERFILES = 8
41
42def add_entry(sha, msg):
43    if msg.lower().strip() == "none":
44        return
45
46    sha_list = ENTRIES.get(sha)
47    if sha_list is None:
48        sha_list = []
49    sha_list.append(msg)
50    ENTRIES[sha] = sha_list
51
52
53if len(sys.argv) < 2 or sys.argv[1] == "-h" or sys.argv[1] == "--help":
54    sys.stderr.write('''Usage:
55changelog-generator [options] <commit-range>
56
57The command accepts all the same options that git-log does.
58
59Options:
60  --community   Automatically includes all repositories for community builds.
61  --enterprise  Automatically includes Enterprise specific repositories.
62  --masterfiles Automatically includes masterfiles repository.
63  --repo        Includes only the current repository.
64
65--community and --enterprise can be given together to generate one master log
66for both.
67''')
68    sys.exit(1)
69
70while True:
71    if sys.argv[1] == "--sort-changelog":
72        SORT_CHANGELOG = True
73        sys.argv[1:] = sys.argv[2:]
74    elif sys.argv[1] == "--community":
75        LOG_TYPE |= LOG_COMMUNITY
76        sys.argv[1:] = sys.argv[2:]
77    elif sys.argv[1] == "--enterprise":
78        LOG_TYPE |= LOG_ENTERPRISE
79        sys.argv[1:] = sys.argv[2:]
80    elif sys.argv[1] == "--masterfiles":
81        LOG_TYPE |= LOG_MASTERFILES
82        sys.argv[1:] = sys.argv[2:]
83    elif sys.argv[1] == "--repo":
84        LOG_TYPE |= LOG_REPO
85        sys.argv[1:] = sys.argv[2:]
86    else:
87        break
88
89if LOG_TYPE == 0:
90    sys.stderr.write("Must give one of --community, --enterprise, --masterfiles or --repo\n")
91    sys.exit(1)
92
93repos = []
94base_path = os.path.dirname(sys.argv[0])
95if LOG_TYPE & LOG_COMMUNITY != 0:
96    repos.append("../buildscripts")
97    repos.append("../core")
98    repos.append("../masterfiles")
99    repos.append("../design-center")
100if LOG_TYPE & LOG_ENTERPRISE != 0:
101    repos.append("../enterprise")
102    repos.append("../nova")
103    repos.append("../mission-portal")
104if LOG_TYPE & LOG_MASTERFILES != 0:
105    repos.append("../masterfiles")
106if LOG_TYPE == LOG_REPO:
107    repos.append(".")
108else:
109    os.chdir(base_path + "/../..")
110
111for repo in repos:
112    os.chdir(repo)
113    sha_list = subprocess.Popen(
114            ["git", "rev-list", "--no-merges", "--reverse"] + sys.argv[1:],
115            stdout=subprocess.PIPE)
116    for sha in sha_list.stdout:
117        sha = sha.decode().rstrip('\n')
118        blob = subprocess.Popen(
119                ["git", "log", "--format=%B", "-n", "1", sha],
120                stdout=subprocess.PIPE)
121
122        title_fetched = False
123        title = ""
124        commit_msg = ""
125        log_entry_title = False
126        log_entry_commit = False
127        log_entry_local = False
128        log_entry = ""
129        for line in blob.stdout:
130            line = line.decode().rstrip('\r\n')
131
132            if line == "" and log_entry:
133                add_entry(sha, log_entry)
134                log_entry = ""
135                log_entry_local = False
136
137            # Tracker reference, remove from string.
138            for match in re.finditer(TRACKER_REGEX, line, re.IGNORECASE):
139                if not SHA_TO_TRACKER.get(sha):
140                    SHA_TO_TRACKER[sha] = set()
141                SHA_TO_TRACKER[sha].add("".join(match.groups("")))
142                tracker_removed = re.sub(TRACKER_REGEX, "", line, flags=re.IGNORECASE)
143                tracker_removed = tracker_removed.strip(' ')
144                if re.match(JIRA_TITLE_REGEX, line) and not title_fetched:
145                    log_entry_title = True
146                line = tracker_removed
147
148            if not title_fetched:
149                title = line
150                title_fetched = True
151
152            match = re.match("^ *Changelog: *(.*)", line, re.IGNORECASE)
153            if match:
154                log_entry_title = False
155                if log_entry:
156                    add_entry(sha, log_entry)
157                    log_entry = ""
158                log_entry_local = False
159
160                if re.match("^Title[ .]*$", match.group(1), re.IGNORECASE):
161                    log_entry = title
162                elif re.match("^Commit[ .]*$", match.group(1), re.IGNORECASE):
163                    log_entry_commit = True
164                elif re.match("^None[ .]*$", match.group(1), re.IGNORECASE):
165                    pass
166                else:
167                    log_entry_local = True
168                    log_entry = match.group(1)
169                continue
170
171            for cancel_expr in ["^ *Cancel-Changelog: *([0-9a-f]+).*",
172                                "^This reverts commit ([0-9a-f]+).*"]:
173                match = re.match(cancel_expr, line, re.IGNORECASE)
174                if match:
175                    if log_entry:
176                        add_entry(sha, log_entry)
177                        log_entry = ""
178                    log_entry_local = False
179
180                    linked_shas = [match.group(1)]
181                    if LINKED_SHAS.get(match.group(1)):
182                        for linked_sha in LINKED_SHAS.get(match.group(1)):
183                            linked_shas.append(linked_sha)
184                    for linked_sha in linked_shas:
185                        if LINKED_SHAS.get(linked_sha):
186                            del LINKED_SHAS[linked_sha]
187                        if ENTRIES.get(linked_sha):
188                            del ENTRIES[linked_sha]
189                    continue
190
191            match = re.match("^\(cherry picked from commit ([0-9a-f]+)\)", line, re.IGNORECASE)
192            if match:
193                if log_entry:
194                    add_entry(sha, log_entry)
195                    log_entry = ""
196                log_entry_local = False
197
198                if not LINKED_SHAS.get(sha):
199                    LINKED_SHAS[sha] = []
200                LINKED_SHAS[sha].append(match.group(1))
201                if not LINKED_SHAS.get(match.group(1)):
202                    LINKED_SHAS[match.group(1)] = []
203                LINKED_SHAS[match.group(1)].append(sha)
204                continue
205
206            match = re.match("^Signed-off-by:.*", line, re.IGNORECASE)
207            if match:
208                # Ignore such lines.
209                continue
210
211            if log_entry_local:
212                log_entry += "\n" + line
213            else:
214                if commit_msg:
215                    commit_msg += "\n"
216                commit_msg += line
217
218        blob.wait()
219
220        if log_entry_title:
221            add_entry(sha, title)
222        if log_entry_commit:
223            add_entry(sha, commit_msg)
224        if log_entry:
225            add_entry(sha, log_entry)
226
227    sha_list.wait()
228
229entry_list = []
230for sha_entry in ENTRIES:
231    tracker = ""
232    if SHA_TO_TRACKER.get(sha_entry):
233        jiras = [ticket.upper() for ticket in SHA_TO_TRACKER[sha_entry]]
234        tracker = ""
235        if len(jiras) > 0:
236            tracker += "(" + ", ".join(sorted(jiras)) + ")"
237    for entry in ENTRIES[sha_entry]:
238        # Safety check. See if there are still numbers at least four digits long in
239        # the output, and if so, warn about it. This may be ticket references that
240        # we missed.
241        match = re.search("[0-9]{4,}", entry)
242        if match:
243            POSSIBLE_MISSED_TICKETS[sha_entry] = match.group(0)
244        entry = entry.strip("\n")
245        if tracker:
246            if len(entry) - entry.rfind("\n") + len(tracker) >= 70:
247                entry += "\n"
248            else:
249                entry += " "
250            entry += tracker
251        entry_list.append(entry)
252
253if SORT_CHANGELOG:
254    entry_list.sort()
255for entry in entry_list:
256    entry = "\t- " + entry
257    # Blank lines look bad in changelog because entries don't have blank lines
258    # between them, so remove that from commit messages.
259    entry = re.sub("\n\n+", "\n", entry)
260    # Indent all lines.
261    entry = entry.replace("\n", "\n\t  ")
262    print(entry)
263
264for missed in POSSIBLE_MISSED_TICKETS:
265    sys.stderr.write("*** Commit %s had a number %s which may be a ticket reference we missed. Should be manually checked.\n"
266                     % (missed, POSSIBLE_MISSED_TICKETS[missed]))
267
268sys.exit(0)
269