xref: /reactos/sdk/tools/winesync/winesync.py (revision 53cc9261)
1#!/usr/bin/env python3
2
3import sys
4import os
5import posixpath
6import string
7import argparse
8import subprocess
9import fnmatch
10import pygit2
11import yaml
12
13def string_to_valid_file_name(to_convert):
14    valid_chars = '-_.()' + string.ascii_letters + string.digits
15    result = ''
16    for c in to_convert:
17        if c in valid_chars:
18            result += c
19        else:
20            result += '_'
21    # strip final dot, if any
22    if result.endswith('.'):
23        return result[:-1]
24    return result
25
26class wine_sync:
27    def __init__(self, module):
28        if os.path.isfile('winesync.cfg'):
29            with open('winesync.cfg', 'r') as file_input:
30                config = yaml.safe_load(file_input)
31            self.reactos_src = config['repos']['reactos']
32            self.wine_src = config['repos']['wine']
33            self.wine_staging_src = config['repos']['wine-staging']
34        else:
35            config = { }
36            self.reactos_src = input('Please enter the path to the reactos git tree: ')
37            self.wine_src = input('Please enter the path to the wine git tree: ')
38            self.wine_staging_src = input('Please enter the path to the wine-staging git tree: ')
39            config['repos'] = { 'reactos': self.reactos_src,
40                                'wine': self.wine_src,
41                                'wine-staging': self.wine_staging_src }
42            with open('winesync.cfg', 'w') as file_output:
43                yaml.dump(config, file_output)
44
45        self.wine_repo = pygit2.Repository(self.wine_src)
46        self.wine_staging_repo = pygit2.Repository(self.wine_staging_src)
47        self.reactos_repo = pygit2.Repository(self.reactos_src)
48
49        # the standard author signature we will use
50        self.winesync_author_signature = pygit2.Signature('winesync', 'ros-dev@reactos.org')
51
52        # read the index from the reactos tree
53        self.reactos_index = self.reactos_repo.index
54        self.reactos_index.read()
55
56        # get the actual state for the asked module
57        self.module = module
58        with open(module + '.cfg', 'r') as file_input:
59            self.module_cfg = yaml.safe_load(file_input)
60
61        self.staged_patch_dir = posixpath.join('sdk', 'tools', 'winesync', self.module + '_staging')
62
63    def create_or_checkout_wine_branch(self, wine_tag, wine_staging_tag):
64        # build the wine branch name
65        wine_branch_name = 'winesync-' + wine_tag
66        if wine_staging_tag:
67            wine_branch_name += '-' + wine_staging_tag
68
69        branch = self.wine_repo.lookup_branch(wine_branch_name)
70        if branch is None:
71            # get our target commits
72            wine_target_commit = self.wine_repo.revparse_single(wine_tag)
73            if isinstance(wine_target_commit, pygit2.Tag):
74                wine_target_commit = wine_target_commit.target
75            if isinstance(wine_target_commit, pygit2.Commit):
76                wine_target_commit = wine_target_commit.id
77
78            # do the same for the wine-staging tree
79            if wine_staging_tag:
80                wine_staging_target_commit = self.wine_staging_repo.revparse_single(wine_staging_tag)
81                if isinstance(wine_staging_target_commit, pygit2.Tag):
82                    wine_staging_target_commit = wine_staging_target_commit.target
83                if isinstance(wine_staging_target_commit, pygit2.Commit):
84                    wine_staging_target_commit = wine_staging_target_commit.id
85
86            self.wine_repo.branches.local.create(wine_branch_name, self.wine_repo.revparse_single('HEAD'))
87            self.wine_repo.checkout(self.wine_repo.lookup_branch(wine_branch_name))
88            self.wine_repo.reset(wine_target_commit, pygit2.GIT_RESET_HARD)
89
90            # do the same for the wine-staging tree
91            if wine_staging_tag:
92                self.wine_staging_repo.branches.local.create(wine_branch_name, self.wine_staging_repo.revparse_single('HEAD'))
93                self.wine_staging_repo.checkout(self.wine_staging_repo.lookup_branch(wine_branch_name))
94                self.wine_staging_repo.reset(wine_staging_target_commit, pygit2.GIT_RESET_HARD)
95
96                # run the wine-staging script
97                if subprocess.call(['python', self.wine_staging_src + '/staging/patchinstall.py', 'DESTDIR=' + self.wine_src, '--all', '--backend=git-am']):
98                    # the new script failed (it doesn't exist?), try the old one
99                    subprocess.call(['bash', '-c', self.wine_staging_src + '/patches/patchinstall.sh DESTDIR=' + self.wine_src + ' --all --backend=git-am'])
100
101                # delete the branch we created
102                self.wine_staging_repo.checkout(self.wine_staging_repo.lookup_branch('master'))
103                self.wine_staging_repo.branches.delete(wine_branch_name)
104        else:
105            self.wine_repo.checkout(self.wine_repo.lookup_branch(wine_branch_name))
106
107        return wine_branch_name
108
109    # Helper function for resolving wine tree path to reactos one
110    # Note: it doesn't care about the fact that the file actually exists or not
111    def wine_to_reactos_path(self, wine_path):
112        if self.module_cfg['files'] and (wine_path in self.module_cfg['files']):
113            # we have a direct mapping
114            return self.module_cfg['files'][wine_path]
115
116        if not '/' in wine_path:
117            # root files should have a direct mapping
118            return None
119
120        wine_dir, wine_file = os.path.split(wine_path)
121        if self.module_cfg['directories'] and (wine_dir in self.module_cfg['directories']):
122            # we have a mapping for the directory
123            return posixpath.join(self.module_cfg['directories'][wine_dir], wine_file)
124
125        # no match
126        return None
127
128    def sync_wine_commit(self, wine_commit, in_staging, staging_patch_index):
129        # Get the diff object
130        diff = self.wine_repo.diff(wine_commit.parents[0], wine_commit)
131
132        modified_files = False
133        ignored_files = []
134        warning_message = ''
135        complete_patch = ''
136
137        if in_staging:
138            # see if we already applied this patch
139            patch_file_name = f'{staging_patch_index:04}-{string_to_valid_file_name(wine_commit.message.splitlines()[0])}.diff'
140            patch_dir = os.path.join(self.reactos_src, self.staged_patch_dir)
141            patch_path = os.path.join(patch_dir, patch_file_name)
142            if os.path.isfile(patch_path):
143                print(f'Skipping patch as {patch_path} already exists')
144                return True, ''
145
146        for delta in diff.deltas:
147            if delta.status == pygit2.GIT_DELTA_ADDED:
148                # check if we should care
149                new_reactos_path = self.wine_to_reactos_path(delta.new_file.path)
150                if not new_reactos_path is None:
151                    warning_message += 'file ' + delta.new_file.path + ' is added to the wine tree!\n'
152                    old_reactos_path = '/dev/null'
153                else:
154                    old_reactos_path = None
155            elif delta.status == pygit2.GIT_DELTA_DELETED:
156                # check if we should care
157                old_reactos_path = self.wine_to_reactos_path(delta.old_file.path)
158                if not old_reactos_path is None:
159                    warning_message += 'file ' + delta.old_file.path + ' is removed from the wine tree!\n'
160                    new_reactos_path = '/dev/null'
161                else:
162                    new_reactos_path = None
163            elif delta.new_file.path.endswith('Makefile.in'):
164                warning_message += 'file ' + delta.new_file.path + ' was modified!\n'
165                # no need to warn that those are ignored, we just did.
166                continue
167            else:
168                new_reactos_path = self.wine_to_reactos_path(delta.new_file.path)
169                old_reactos_path = self.wine_to_reactos_path(delta.old_file.path)
170
171            if (new_reactos_path is not None) or (old_reactos_path is not None):
172                # print('Must apply diff: ' + old_reactos_path + ' --> ' + new_reactos_path)
173                if delta.status == pygit2.GIT_DELTA_ADDED:
174                    new_blob = self.wine_repo.get(delta.new_file.id)
175                    blob_patch = pygit2.Patch.create_from(
176                        old=None,
177                        new=new_blob,
178                        new_as_path=new_reactos_path)
179                elif delta.status == pygit2.GIT_DELTA_DELETED:
180                    old_blob = self.wine_repo.get(delta.old_file.id)
181                    blob_patch = pygit2.Patch.create_from(
182                        old=old_blob,
183                        new=None,
184                        old_as_path=old_reactos_path)
185                else:
186                    new_blob = self.wine_repo.get(delta.new_file.id)
187                    old_blob = self.wine_repo.get(delta.old_file.id)
188
189                    blob_patch = pygit2.Patch.create_from(
190                        old=old_blob,
191                        new=new_blob,
192                        old_as_path=old_reactos_path,
193                        new_as_path=new_reactos_path)
194
195                # print(str(wine_commit.id))
196                # print(blob_patch.text)
197
198                # this doesn't work
199                # reactos_diff = pygit2.Diff.parse_diff(blob_patch.text)
200                # reactos_repo.apply(reactos_diff)
201                try:
202                    subprocess.run(['git', '-C', self.reactos_src, 'apply', '--reject'], input=blob_patch.data, check=True)
203                except subprocess.CalledProcessError as err:
204                    warning_message += 'Error while applying patch to ' + new_reactos_path + '\n'
205
206                if delta.status == pygit2.GIT_DELTA_DELETED:
207                    try:
208                        self.reactos_index.remove(old_reactos_path)
209                    except IOError as err:
210                        warning_message += 'Error while removing file ' + old_reactos_path + '\n'
211                # here we check if the file exists. We don't complain, because applying the patch already failed anyway
212                elif os.path.isfile(os.path.join(self.reactos_src, new_reactos_path)):
213                    self.reactos_index.add(new_reactos_path)
214
215                complete_patch += blob_patch.text
216
217                modified_files = True
218            else:
219                ignored_files += [delta.old_file.path, delta.new_file.path]
220
221        if not modified_files:
222            # We applied nothing
223            return False, ''
224
225        print('Applied patches from wine commit ' + str(wine_commit.id))
226
227        if ignored_files:
228            warning_message += 'WARNING: some files were ignored: ' + ' '.join(ignored_files) + '\n'
229
230        if not in_staging:
231            self.module_cfg['tags']['wine'] = str(wine_commit.id)
232            with open(self.module + '.cfg', 'w') as file_output:
233                yaml.dump(self.module_cfg, file_output)
234            self.reactos_index.add(f'sdk/tools/winesync/{self.module}.cfg')
235        else:
236            # Add the staging patch
237            # do not save the wine commit ID in <module>.cfg, as it's a local one for staging patches
238            if not os.path.isdir(patch_dir):
239                os.mkdir(patch_dir)
240            with open(patch_path, 'w') as file_output:
241                file_output.write(complete_patch)
242            self.reactos_index.add(posixpath.join(self.staged_patch_dir, patch_file_name))
243
244        self.reactos_index.write()
245
246        commit_msg = f'[WINESYNC] {wine_commit.message}\n'
247        if (in_staging):
248            commit_msg += f'wine-staging patch by {wine_commit.author.name} <{wine_commit.author.email}>'
249        else:
250            commit_msg += f'wine commit id {str(wine_commit.id)} by {wine_commit.author.name} <{wine_commit.author.email}>'
251
252        self.reactos_repo.create_commit(
253            'HEAD',
254            self.winesync_author_signature,
255            self.reactos_repo.default_signature,
256            commit_msg,
257            self.reactos_index.write_tree(),
258            [self.reactos_repo.head.target])
259
260        if (warning_message != ''):
261            warning_message += 'If needed, amend the current commit in your reactos tree and start this script again'
262
263            if not in_staging:
264                warning_message += f'\n' \
265                    f'You can see the details of the wine commit here:\n' \
266                    f'    https://source.winehq.org/git/wine.git/commit/{str(wine_commit.id)}\n'
267            else:
268                patch_file_path = posixpath.join(self.staged_patch_dir, patch_file_name)
269                warning_message += f'\n' \
270                    f'Do not forget to run\n' \
271                    f'    git diff HEAD^ \':(exclude){patch_file_path}\' > {patch_file_path}\n' \
272                    f'after your correction and then\n' \
273                    f'    git add {patch_file_path}\n' \
274                    f'before running "git commit --amend"'
275
276        return True, warning_message
277
278    def revert_staged_patchset(self):
279        # revert all of this in one commit
280        staged_patch_dir_path = posixpath.join(self.reactos_src, self.staged_patch_dir)
281        if not os.path.isdir(staged_patch_dir_path):
282            return True
283
284        has_patches = False
285
286        for patch_file_name in sorted(os.listdir(staged_patch_dir_path), reverse=True):
287            patch_path = os.path.join(staged_patch_dir_path, patch_file_name)
288            if not os.path.isfile(patch_path):
289                continue
290
291            has_patches = True
292
293            with open(patch_path, 'rb') as patch_file:
294                try:
295                    subprocess.run(['git', '-C', self.reactos_src, 'apply', '-R', '--ignore-whitespace', '--reject'], stdin=patch_file, check=True)
296                except subprocess.CalledProcessError as err:
297                    print(f'Error while reverting patch {patch_file_name}')
298                    print('Please check, remove the offending patch with git rm, and relaunch this script')
299                    return False
300
301            self.reactos_index.remove(posixpath.join(self.staged_patch_dir, patch_file_name))
302            self.reactos_index.write()
303            os.remove(patch_path)
304
305        if not has_patches:
306            return True
307
308        # Note: these path lists may be empty or None, in which case
309        # we should not call index.add_all(), otherwise we would add
310        # any untracked file present in the repository.
311        if self.module_cfg['files']:
312            self.reactos_index.add_all([f for f in self.module_cfg['files'].values()])
313        if self.module_cfg['directories']:
314            self.reactos_index.add_all([f'{d}/*.*' for d in self.module_cfg['directories'].values()])
315        self.reactos_index.write()
316
317        self.reactos_repo.create_commit(
318            'HEAD',
319            self.winesync_author_signature,
320            self.reactos_repo.default_signature,
321            f'[WINESYNC]: revert wine-staging patchset for {self.module}',
322            self.reactos_index.write_tree(),
323            [self.reactos_repo.head.target])
324        return True
325
326    def sync_to_wine(self, wine_tag, wine_staging_tag):
327        # Get our target commit
328        wine_target_commit = self.wine_repo.revparse_single(wine_tag)
329        if isinstance(wine_target_commit, pygit2.Tag):
330            wine_target_commit = wine_target_commit.target
331        if isinstance(wine_target_commit, pygit2.Commit):
332            wine_target_commit = wine_target_commit.id
333        # print(f'wine target commit is {wine_target_commit}')
334
335        # get the wine commit id where we left
336        in_staging = False
337        wine_last_sync = self.wine_repo.revparse_single(self.module_cfg['tags']['wine'])
338        if isinstance(wine_last_sync, pygit2.Tag):
339            if not self.revert_staged_patchset():
340                return
341            wine_last_sync = wine_last_sync.target
342        if isinstance(wine_last_sync, pygit2.Commit):
343            wine_last_sync = wine_last_sync.id
344
345        # create a branch to keep things clean
346        wine_branch_name = self.create_or_checkout_wine_branch(wine_tag, wine_staging_tag)
347
348        finished_sync = True
349        staging_patch_index = 1
350
351        # walk each commit between last sync and the asked tag/revision
352        wine_commit_walker = self.wine_repo.walk(self.wine_repo.head.target, pygit2.GIT_SORT_TOPOLOGICAL | pygit2.GIT_SORT_REVERSE)
353        wine_commit_walker.hide(wine_last_sync)
354        for wine_commit in wine_commit_walker:
355            applied_patch, warning_message = self.sync_wine_commit(wine_commit, in_staging, staging_patch_index)
356
357            if str(wine_commit.id) == str(wine_target_commit):
358                print('We are now in staging territory')
359                in_staging = True
360
361            if not applied_patch:
362                continue
363
364            if in_staging:
365                staging_patch_index += 1
366
367            if warning_message != '':
368                print("THERE WERE SOME ISSUES WHEN APPLYING THE PATCH\n\n")
369                print(warning_message)
370                print("\n")
371                finished_sync = False
372                break
373
374        # we're done without error
375        if finished_sync:
376            # update wine tag and commit
377            self.module_cfg['tags']['wine'] = wine_tag
378            with open(self.module + '.cfg', 'w') as file_output:
379                yaml.dump(self.module_cfg, file_output)
380
381            self.reactos_index.add(f'sdk/tools/winesync/{self.module}.cfg')
382            self.reactos_index.write()
383            self.reactos_repo.create_commit(
384                'HEAD',
385                self.winesync_author_signature,
386                self.reactos_repo.default_signature,
387                f'[WINESYNC]: {self.module} is now in sync with wine-staging {wine_tag}',
388                self.reactos_index.write_tree(),
389                [self.reactos_repo.head.target])
390
391            print('The branch ' + wine_branch_name + ' was created in your wine repository. You might want to delete it, but you should keep it in case you want to sync more module up to this wine version')
392
393def main():
394    parser = argparse.ArgumentParser()
395    parser.add_argument('module', help='The module you want to sync. <module>.cfg must exist in the current directory.')
396    parser.add_argument('wine_tag', help='The wine tag or commit id to sync to.')
397    parser.add_argument('wine_staging_tag', nargs='?', default=None, help='The optional wine staging tag or commit id to pick wine staged patches from.')
398
399    args = parser.parse_args()
400
401    syncator = wine_sync(args.module)
402
403    return syncator.sync_to_wine(args.wine_tag, args.wine_staging_tag)
404
405
406if __name__ == '__main__':
407    main()
408