1// Copyright 2019 The Gitea Authors. All rights reserved. 2// Use of this source code is governed by a MIT-style 3// license that can be found in the LICENSE file. 4 5package repository 6 7import ( 8 "context" 9 "fmt" 10 "io" 11 "net/http" 12 "path" 13 "strings" 14 "time" 15 16 "code.gitea.io/gitea/models" 17 "code.gitea.io/gitea/models/db" 18 repo_model "code.gitea.io/gitea/models/repo" 19 user_model "code.gitea.io/gitea/models/user" 20 "code.gitea.io/gitea/modules/git" 21 "code.gitea.io/gitea/modules/lfs" 22 "code.gitea.io/gitea/modules/log" 23 "code.gitea.io/gitea/modules/migration" 24 "code.gitea.io/gitea/modules/setting" 25 "code.gitea.io/gitea/modules/timeutil" 26 "code.gitea.io/gitea/modules/util" 27 28 "gopkg.in/ini.v1" 29) 30 31/* 32 GitHub, GitLab, Gogs: *.wiki.git 33 BitBucket: *.git/wiki 34*/ 35var commonWikiURLSuffixes = []string{".wiki.git", ".git/wiki"} 36 37// WikiRemoteURL returns accessible repository URL for wiki if exists. 38// Otherwise, it returns an empty string. 39func WikiRemoteURL(remote string) string { 40 remote = strings.TrimSuffix(remote, ".git") 41 for _, suffix := range commonWikiURLSuffixes { 42 wikiURL := remote + suffix 43 if git.IsRepoURLAccessible(wikiURL) { 44 return wikiURL 45 } 46 } 47 return "" 48} 49 50// MigrateRepositoryGitData starts migrating git related data after created migrating repository 51func MigrateRepositoryGitData(ctx context.Context, u *user_model.User, 52 repo *repo_model.Repository, opts migration.MigrateOptions, 53 httpTransport *http.Transport, 54) (*repo_model.Repository, error) { 55 repoPath := repo_model.RepoPath(u.Name, opts.RepoName) 56 57 if u.IsOrganization() { 58 t, err := models.OrgFromUser(u).GetOwnerTeam() 59 if err != nil { 60 return nil, err 61 } 62 repo.NumWatches = t.NumMembers 63 } else { 64 repo.NumWatches = 1 65 } 66 67 migrateTimeout := time.Duration(setting.Git.Timeout.Migrate) * time.Second 68 69 var err error 70 if err = util.RemoveAll(repoPath); err != nil { 71 return repo, fmt.Errorf("Failed to remove %s: %v", repoPath, err) 72 } 73 74 if err = git.CloneWithContext(ctx, opts.CloneAddr, repoPath, git.CloneRepoOptions{ 75 Mirror: true, 76 Quiet: true, 77 Timeout: migrateTimeout, 78 SkipTLSVerify: setting.Migrations.SkipTLSVerify, 79 }); err != nil { 80 return repo, fmt.Errorf("Clone: %v", err) 81 } 82 83 if opts.Wiki { 84 wikiPath := repo_model.WikiPath(u.Name, opts.RepoName) 85 wikiRemotePath := WikiRemoteURL(opts.CloneAddr) 86 if len(wikiRemotePath) > 0 { 87 if err := util.RemoveAll(wikiPath); err != nil { 88 return repo, fmt.Errorf("Failed to remove %s: %v", wikiPath, err) 89 } 90 91 if err = git.CloneWithContext(ctx, wikiRemotePath, wikiPath, git.CloneRepoOptions{ 92 Mirror: true, 93 Quiet: true, 94 Timeout: migrateTimeout, 95 Branch: "master", 96 SkipTLSVerify: setting.Migrations.SkipTLSVerify, 97 }); err != nil { 98 log.Warn("Clone wiki: %v", err) 99 if err := util.RemoveAll(wikiPath); err != nil { 100 return repo, fmt.Errorf("Failed to remove %s: %v", wikiPath, err) 101 } 102 } 103 } 104 } 105 106 if repo.OwnerID == u.ID { 107 repo.Owner = u 108 } 109 110 if err = models.CheckDaemonExportOK(ctx, repo); err != nil { 111 return repo, fmt.Errorf("checkDaemonExportOK: %v", err) 112 } 113 114 if stdout, err := git.NewCommandContext(ctx, "update-server-info"). 115 SetDescription(fmt.Sprintf("MigrateRepositoryGitData(git update-server-info): %s", repoPath)). 116 RunInDir(repoPath); err != nil { 117 log.Error("MigrateRepositoryGitData(git update-server-info) in %v: Stdout: %s\nError: %v", repo, stdout, err) 118 return repo, fmt.Errorf("error in MigrateRepositoryGitData(git update-server-info): %v", err) 119 } 120 121 gitRepo, err := git.OpenRepository(repoPath) 122 if err != nil { 123 return repo, fmt.Errorf("OpenRepository: %v", err) 124 } 125 defer gitRepo.Close() 126 127 repo.IsEmpty, err = gitRepo.IsEmpty() 128 if err != nil { 129 return repo, fmt.Errorf("git.IsEmpty: %v", err) 130 } 131 132 if !repo.IsEmpty { 133 if len(repo.DefaultBranch) == 0 { 134 // Try to get HEAD branch and set it as default branch. 135 headBranch, err := gitRepo.GetHEADBranch() 136 if err != nil { 137 return repo, fmt.Errorf("GetHEADBranch: %v", err) 138 } 139 if headBranch != nil { 140 repo.DefaultBranch = headBranch.Name 141 } 142 } 143 144 if !opts.Releases { 145 if err = SyncReleasesWithTags(repo, gitRepo); err != nil { 146 log.Error("Failed to synchronize tags to releases for repository: %v", err) 147 } 148 } 149 150 if opts.LFS { 151 endpoint := lfs.DetermineEndpoint(opts.CloneAddr, opts.LFSEndpoint) 152 lfsClient := lfs.NewClient(endpoint, httpTransport) 153 if err = StoreMissingLfsObjectsInRepository(ctx, repo, gitRepo, lfsClient); err != nil { 154 log.Error("Failed to store missing LFS objects for repository: %v", err) 155 } 156 } 157 } 158 159 if err = models.UpdateRepoSize(db.DefaultContext, repo); err != nil { 160 log.Error("Failed to update size for repository: %v", err) 161 } 162 163 if opts.Mirror { 164 mirrorModel := repo_model.Mirror{ 165 RepoID: repo.ID, 166 Interval: setting.Mirror.DefaultInterval, 167 EnablePrune: true, 168 NextUpdateUnix: timeutil.TimeStampNow().AddDuration(setting.Mirror.DefaultInterval), 169 LFS: opts.LFS, 170 } 171 if opts.LFS { 172 mirrorModel.LFSEndpoint = opts.LFSEndpoint 173 } 174 175 if opts.MirrorInterval != "" { 176 parsedInterval, err := time.ParseDuration(opts.MirrorInterval) 177 if err != nil { 178 log.Error("Failed to set Interval: %v", err) 179 return repo, err 180 } 181 if parsedInterval == 0 { 182 mirrorModel.Interval = 0 183 mirrorModel.NextUpdateUnix = 0 184 } else if parsedInterval < setting.Mirror.MinInterval { 185 err := fmt.Errorf("Interval %s is set below Minimum Interval of %s", parsedInterval, setting.Mirror.MinInterval) 186 log.Error("Interval: %s is too frequent", opts.MirrorInterval) 187 return repo, err 188 } else { 189 mirrorModel.Interval = parsedInterval 190 mirrorModel.NextUpdateUnix = timeutil.TimeStampNow().AddDuration(parsedInterval) 191 } 192 } 193 194 if err = repo_model.InsertMirror(&mirrorModel); err != nil { 195 return repo, fmt.Errorf("InsertOne: %v", err) 196 } 197 198 repo.IsMirror = true 199 err = models.UpdateRepository(repo, false) 200 } else { 201 repo, err = CleanUpMigrateInfo(repo) 202 } 203 204 return repo, err 205} 206 207// cleanUpMigrateGitConfig removes mirror info which prevents "push --all". 208// This also removes possible user credentials. 209func cleanUpMigrateGitConfig(configPath string) error { 210 cfg, err := ini.Load(configPath) 211 if err != nil { 212 return fmt.Errorf("open config file: %v", err) 213 } 214 cfg.DeleteSection("remote \"origin\"") 215 if err = cfg.SaveToIndent(configPath, "\t"); err != nil { 216 return fmt.Errorf("save config file: %v", err) 217 } 218 return nil 219} 220 221// CleanUpMigrateInfo finishes migrating repository and/or wiki with things that don't need to be done for mirrors. 222func CleanUpMigrateInfo(repo *repo_model.Repository) (*repo_model.Repository, error) { 223 repoPath := repo.RepoPath() 224 if err := createDelegateHooks(repoPath); err != nil { 225 return repo, fmt.Errorf("createDelegateHooks: %v", err) 226 } 227 if repo.HasWiki() { 228 if err := createDelegateHooks(repo.WikiPath()); err != nil { 229 return repo, fmt.Errorf("createDelegateHooks.(wiki): %v", err) 230 } 231 } 232 233 _, err := git.NewCommand("remote", "rm", "origin").RunInDir(repoPath) 234 if err != nil && !strings.HasPrefix(err.Error(), "exit status 128 - fatal: No such remote ") { 235 return repo, fmt.Errorf("CleanUpMigrateInfo: %v", err) 236 } 237 238 if repo.HasWiki() { 239 if err := cleanUpMigrateGitConfig(path.Join(repo.WikiPath(), "config")); err != nil { 240 return repo, fmt.Errorf("cleanUpMigrateGitConfig (wiki): %v", err) 241 } 242 } 243 244 return repo, models.UpdateRepository(repo, false) 245} 246 247// SyncReleasesWithTags synchronizes release table with repository tags 248func SyncReleasesWithTags(repo *repo_model.Repository, gitRepo *git.Repository) error { 249 existingRelTags := make(map[string]struct{}) 250 opts := models.FindReleasesOptions{ 251 IncludeDrafts: true, 252 IncludeTags: true, 253 ListOptions: db.ListOptions{PageSize: 50}, 254 } 255 for page := 1; ; page++ { 256 opts.Page = page 257 rels, err := models.GetReleasesByRepoID(repo.ID, opts) 258 if err != nil { 259 return fmt.Errorf("unable to GetReleasesByRepoID in Repo[%d:%s/%s]: %w", repo.ID, repo.OwnerName, repo.Name, err) 260 } 261 if len(rels) == 0 { 262 break 263 } 264 for _, rel := range rels { 265 if rel.IsDraft { 266 continue 267 } 268 commitID, err := gitRepo.GetTagCommitID(rel.TagName) 269 if err != nil && !git.IsErrNotExist(err) { 270 return fmt.Errorf("unable to GetTagCommitID for %q in Repo[%d:%s/%s]: %w", rel.TagName, repo.ID, repo.OwnerName, repo.Name, err) 271 } 272 if git.IsErrNotExist(err) || commitID != rel.Sha1 { 273 if err := models.PushUpdateDeleteTag(repo, rel.TagName); err != nil { 274 return fmt.Errorf("unable to PushUpdateDeleteTag: %q in Repo[%d:%s/%s]: %w", rel.TagName, repo.ID, repo.OwnerName, repo.Name, err) 275 } 276 } else { 277 existingRelTags[strings.ToLower(rel.TagName)] = struct{}{} 278 } 279 } 280 } 281 tags, err := gitRepo.GetTags(0, 0) 282 if err != nil { 283 return fmt.Errorf("unable to GetTags in Repo[%d:%s/%s]: %w", repo.ID, repo.OwnerName, repo.Name, err) 284 } 285 for _, tagName := range tags { 286 if _, ok := existingRelTags[strings.ToLower(tagName)]; !ok { 287 if err := PushUpdateAddTag(repo, gitRepo, tagName); err != nil { 288 return fmt.Errorf("unable to PushUpdateAddTag: %q to Repo[%d:%s/%s]: %w", tagName, repo.ID, repo.OwnerName, repo.Name, err) 289 } 290 } 291 } 292 return nil 293} 294 295// PushUpdateAddTag must be called for any push actions to add tag 296func PushUpdateAddTag(repo *repo_model.Repository, gitRepo *git.Repository, tagName string) error { 297 tag, err := gitRepo.GetTag(tagName) 298 if err != nil { 299 return fmt.Errorf("unable to GetTag: %w", err) 300 } 301 commit, err := tag.Commit(gitRepo) 302 if err != nil { 303 return fmt.Errorf("unable to get tag Commit: %w", err) 304 } 305 306 sig := tag.Tagger 307 if sig == nil { 308 sig = commit.Author 309 } 310 if sig == nil { 311 sig = commit.Committer 312 } 313 314 var author *user_model.User 315 createdAt := time.Unix(1, 0) 316 317 if sig != nil { 318 author, err = user_model.GetUserByEmail(sig.Email) 319 if err != nil && !user_model.IsErrUserNotExist(err) { 320 return fmt.Errorf("unable to GetUserByEmail for %q: %w", sig.Email, err) 321 } 322 createdAt = sig.When 323 } 324 325 commitsCount, err := commit.CommitsCount() 326 if err != nil { 327 return fmt.Errorf("unable to get CommitsCount: %w", err) 328 } 329 330 rel := models.Release{ 331 RepoID: repo.ID, 332 TagName: tagName, 333 LowerTagName: strings.ToLower(tagName), 334 Sha1: commit.ID.String(), 335 NumCommits: commitsCount, 336 CreatedUnix: timeutil.TimeStamp(createdAt.Unix()), 337 IsTag: true, 338 } 339 if author != nil { 340 rel.PublisherID = author.ID 341 } 342 343 return models.SaveOrUpdateTag(repo, &rel) 344} 345 346// StoreMissingLfsObjectsInRepository downloads missing LFS objects 347func StoreMissingLfsObjectsInRepository(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository, lfsClient lfs.Client) error { 348 contentStore := lfs.NewContentStore() 349 350 pointerChan := make(chan lfs.PointerBlob) 351 errChan := make(chan error, 1) 352 go lfs.SearchPointerBlobs(ctx, gitRepo, pointerChan, errChan) 353 354 downloadObjects := func(pointers []lfs.Pointer) error { 355 err := lfsClient.Download(ctx, pointers, func(p lfs.Pointer, content io.ReadCloser, objectError error) error { 356 if objectError != nil { 357 return objectError 358 } 359 360 defer content.Close() 361 362 _, err := models.NewLFSMetaObject(&models.LFSMetaObject{Pointer: p, RepositoryID: repo.ID}) 363 if err != nil { 364 log.Error("Repo[%-v]: Error creating LFS meta object %-v: %v", repo, p, err) 365 return err 366 } 367 368 if err := contentStore.Put(p, content); err != nil { 369 log.Error("Repo[%-v]: Error storing content for LFS meta object %-v: %v", repo, p, err) 370 if _, err2 := models.RemoveLFSMetaObjectByOid(repo.ID, p.Oid); err2 != nil { 371 log.Error("Repo[%-v]: Error removing LFS meta object %-v: %v", repo, p, err2) 372 } 373 return err 374 } 375 return nil 376 }) 377 if err != nil { 378 select { 379 case <-ctx.Done(): 380 return nil 381 default: 382 } 383 } 384 return err 385 } 386 387 var batch []lfs.Pointer 388 for pointerBlob := range pointerChan { 389 meta, err := models.GetLFSMetaObjectByOid(repo.ID, pointerBlob.Oid) 390 if err != nil && err != models.ErrLFSObjectNotExist { 391 log.Error("Repo[%-v]: Error querying LFS meta object %-v: %v", repo, pointerBlob.Pointer, err) 392 return err 393 } 394 if meta != nil { 395 log.Trace("Repo[%-v]: Skipping unknown LFS meta object %-v", repo, pointerBlob.Pointer) 396 continue 397 } 398 399 log.Trace("Repo[%-v]: LFS object %-v not present in repository", repo, pointerBlob.Pointer) 400 401 exist, err := contentStore.Exists(pointerBlob.Pointer) 402 if err != nil { 403 log.Error("Repo[%-v]: Error checking if LFS object %-v exists: %v", repo, pointerBlob.Pointer, err) 404 return err 405 } 406 407 if exist { 408 log.Trace("Repo[%-v]: LFS object %-v already present; creating meta object", repo, pointerBlob.Pointer) 409 _, err := models.NewLFSMetaObject(&models.LFSMetaObject{Pointer: pointerBlob.Pointer, RepositoryID: repo.ID}) 410 if err != nil { 411 log.Error("Repo[%-v]: Error creating LFS meta object %-v: %v", repo, pointerBlob.Pointer, err) 412 return err 413 } 414 } else { 415 if setting.LFS.MaxFileSize > 0 && pointerBlob.Size > setting.LFS.MaxFileSize { 416 log.Info("Repo[%-v]: LFS object %-v download denied because of LFS_MAX_FILE_SIZE=%d < size %d", repo, pointerBlob.Pointer, setting.LFS.MaxFileSize, pointerBlob.Size) 417 continue 418 } 419 420 batch = append(batch, pointerBlob.Pointer) 421 if len(batch) >= lfsClient.BatchSize() { 422 if err := downloadObjects(batch); err != nil { 423 return err 424 } 425 batch = nil 426 } 427 } 428 } 429 if len(batch) > 0 { 430 if err := downloadObjects(batch); err != nil { 431 return err 432 } 433 } 434 435 err, has := <-errChan 436 if has { 437 log.Error("Repo[%-v]: Error enumerating LFS objects for repository: %v", repo, err) 438 return err 439 } 440 441 return nil 442} 443