1// Copyright 2019 The Gitea Authors. All rights reserved. 2// Copyright 2018 Jonas Franz. All rights reserved. 3// Use of this source code is governed by a MIT-style 4// license that can be found in the LICENSE file. 5 6package migrations 7 8import ( 9 "context" 10 "fmt" 11 "net" 12 "net/url" 13 "path/filepath" 14 "strings" 15 16 "code.gitea.io/gitea/models" 17 admin_model "code.gitea.io/gitea/models/admin" 18 repo_model "code.gitea.io/gitea/models/repo" 19 user_model "code.gitea.io/gitea/models/user" 20 "code.gitea.io/gitea/modules/hostmatcher" 21 "code.gitea.io/gitea/modules/log" 22 base "code.gitea.io/gitea/modules/migration" 23 "code.gitea.io/gitea/modules/setting" 24 "code.gitea.io/gitea/modules/util" 25) 26 27// MigrateOptions is equal to base.MigrateOptions 28type MigrateOptions = base.MigrateOptions 29 30var ( 31 factories []base.DownloaderFactory 32 33 allowList *hostmatcher.HostMatchList 34 blockList *hostmatcher.HostMatchList 35) 36 37// RegisterDownloaderFactory registers a downloader factory 38func RegisterDownloaderFactory(factory base.DownloaderFactory) { 39 factories = append(factories, factory) 40} 41 42// IsMigrateURLAllowed checks if an URL is allowed to be migrated from 43func IsMigrateURLAllowed(remoteURL string, doer *user_model.User) error { 44 // Remote address can be HTTP/HTTPS/Git URL or local path. 45 u, err := url.Parse(remoteURL) 46 if err != nil { 47 return &models.ErrInvalidCloneAddr{IsURLError: true} 48 } 49 50 if u.Scheme == "file" || u.Scheme == "" { 51 if !doer.CanImportLocal() { 52 return &models.ErrInvalidCloneAddr{Host: "<LOCAL_FILESYSTEM>", IsPermissionDenied: true, LocalPath: true} 53 } 54 isAbs := filepath.IsAbs(u.Host + u.Path) 55 if !isAbs { 56 return &models.ErrInvalidCloneAddr{Host: "<LOCAL_FILESYSTEM>", IsInvalidPath: true, LocalPath: true} 57 } 58 isDir, err := util.IsDir(u.Host + u.Path) 59 if err != nil { 60 log.Error("Unable to check if %s is a directory: %v", u.Host+u.Path, err) 61 return err 62 } 63 if !isDir { 64 return &models.ErrInvalidCloneAddr{Host: "<LOCAL_FILESYSTEM>", IsInvalidPath: true, LocalPath: true} 65 } 66 67 return nil 68 } 69 70 if u.Scheme == "git" && u.Port() != "" && (strings.Contains(remoteURL, "%0d") || strings.Contains(remoteURL, "%0a")) { 71 return &models.ErrInvalidCloneAddr{Host: u.Host, IsURLError: true} 72 } 73 74 if u.Opaque != "" || u.Scheme != "" && u.Scheme != "http" && u.Scheme != "https" && u.Scheme != "git" { 75 return &models.ErrInvalidCloneAddr{Host: u.Host, IsProtocolInvalid: true, IsPermissionDenied: true, IsURLError: true} 76 } 77 78 hostName, _, err := net.SplitHostPort(u.Host) 79 if err != nil { 80 // u.Host can be "host" or "host:port" 81 err = nil //nolint 82 hostName = u.Host 83 } 84 addrList, err := net.LookupIP(hostName) 85 if err != nil { 86 return &models.ErrInvalidCloneAddr{Host: u.Host, NotResolvedIP: true} 87 } 88 89 var ipAllowed bool 90 var ipBlocked bool 91 for _, addr := range addrList { 92 ipAllowed = ipAllowed || allowList.MatchIPAddr(addr) 93 ipBlocked = ipBlocked || blockList.MatchIPAddr(addr) 94 } 95 var blockedError error 96 if blockList.MatchHostName(hostName) || ipBlocked { 97 blockedError = &models.ErrInvalidCloneAddr{Host: u.Host, IsPermissionDenied: true} 98 } 99 // if we have an allow-list, check the allow-list first 100 if !allowList.IsEmpty() { 101 if !allowList.MatchHostName(hostName) && !ipAllowed { 102 return &models.ErrInvalidCloneAddr{Host: u.Host, IsPermissionDenied: true} 103 } 104 } 105 // otherwise, we always follow the blocked list 106 return blockedError 107} 108 109// MigrateRepository migrate repository according MigrateOptions 110func MigrateRepository(ctx context.Context, doer *user_model.User, ownerName string, opts base.MigrateOptions, messenger base.Messenger) (*repo_model.Repository, error) { 111 err := IsMigrateURLAllowed(opts.CloneAddr, doer) 112 if err != nil { 113 return nil, err 114 } 115 if opts.LFS && len(opts.LFSEndpoint) > 0 { 116 err := IsMigrateURLAllowed(opts.LFSEndpoint, doer) 117 if err != nil { 118 return nil, err 119 } 120 } 121 downloader, err := newDownloader(ctx, ownerName, opts) 122 if err != nil { 123 return nil, err 124 } 125 126 var uploader = NewGiteaLocalUploader(ctx, doer, ownerName, opts.RepoName) 127 uploader.gitServiceType = opts.GitServiceType 128 129 if err := migrateRepository(downloader, uploader, opts, messenger); err != nil { 130 if err1 := uploader.Rollback(); err1 != nil { 131 log.Error("rollback failed: %v", err1) 132 } 133 if err2 := admin_model.CreateRepositoryNotice(fmt.Sprintf("Migrate repository from %s failed: %v", opts.OriginalURL, err)); err2 != nil { 134 log.Error("create respotiry notice failed: ", err2) 135 } 136 return nil, err 137 } 138 return uploader.repo, nil 139} 140 141func newDownloader(ctx context.Context, ownerName string, opts base.MigrateOptions) (base.Downloader, error) { 142 var ( 143 downloader base.Downloader 144 err error 145 ) 146 147 for _, factory := range factories { 148 if factory.GitServiceType() == opts.GitServiceType { 149 downloader, err = factory.New(ctx, opts) 150 if err != nil { 151 return nil, err 152 } 153 break 154 } 155 } 156 157 if downloader == nil { 158 opts.Wiki = true 159 opts.Milestones = false 160 opts.Labels = false 161 opts.Releases = false 162 opts.Comments = false 163 opts.Issues = false 164 opts.PullRequests = false 165 downloader = NewPlainGitDownloader(ownerName, opts.RepoName, opts.CloneAddr) 166 log.Trace("Will migrate from git: %s", opts.OriginalURL) 167 } 168 169 if setting.Migrations.MaxAttempts > 1 { 170 downloader = base.NewRetryDownloader(ctx, downloader, setting.Migrations.MaxAttempts, setting.Migrations.RetryBackoff) 171 } 172 return downloader, nil 173} 174 175// migrateRepository will download information and then upload it to Uploader, this is a simple 176// process for small repository. For a big repository, save all the data to disk 177// before upload is better 178func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts base.MigrateOptions, messenger base.Messenger) error { 179 if messenger == nil { 180 messenger = base.NilMessenger 181 } 182 183 repo, err := downloader.GetRepoInfo() 184 if err != nil { 185 if !base.IsErrNotSupported(err) { 186 return err 187 } 188 log.Info("migrating repo infos is not supported, ignored") 189 } 190 repo.IsPrivate = opts.Private 191 repo.IsMirror = opts.Mirror 192 if opts.Description != "" { 193 repo.Description = opts.Description 194 } 195 if repo.CloneURL, err = downloader.FormatCloneURL(opts, repo.CloneURL); err != nil { 196 return err 197 } 198 199 log.Trace("migrating git data from %s", repo.CloneURL) 200 messenger("repo.migrate.migrating_git") 201 if err = uploader.CreateRepo(repo, opts); err != nil { 202 return err 203 } 204 defer uploader.Close() 205 206 log.Trace("migrating topics") 207 messenger("repo.migrate.migrating_topics") 208 topics, err := downloader.GetTopics() 209 if err != nil { 210 if !base.IsErrNotSupported(err) { 211 return err 212 } 213 log.Warn("migrating topics is not supported, ignored") 214 } 215 if len(topics) != 0 { 216 if err = uploader.CreateTopics(topics...); err != nil { 217 return err 218 } 219 } 220 221 if opts.Milestones { 222 log.Trace("migrating milestones") 223 messenger("repo.migrate.migrating_milestones") 224 milestones, err := downloader.GetMilestones() 225 if err != nil { 226 if !base.IsErrNotSupported(err) { 227 return err 228 } 229 log.Warn("migrating milestones is not supported, ignored") 230 } 231 232 msBatchSize := uploader.MaxBatchInsertSize("milestone") 233 for len(milestones) > 0 { 234 if len(milestones) < msBatchSize { 235 msBatchSize = len(milestones) 236 } 237 238 if err := uploader.CreateMilestones(milestones...); err != nil { 239 return err 240 } 241 milestones = milestones[msBatchSize:] 242 } 243 } 244 245 if opts.Labels { 246 log.Trace("migrating labels") 247 messenger("repo.migrate.migrating_labels") 248 labels, err := downloader.GetLabels() 249 if err != nil { 250 if !base.IsErrNotSupported(err) { 251 return err 252 } 253 log.Warn("migrating labels is not supported, ignored") 254 } 255 256 lbBatchSize := uploader.MaxBatchInsertSize("label") 257 for len(labels) > 0 { 258 if len(labels) < lbBatchSize { 259 lbBatchSize = len(labels) 260 } 261 262 if err := uploader.CreateLabels(labels...); err != nil { 263 return err 264 } 265 labels = labels[lbBatchSize:] 266 } 267 } 268 269 if opts.Releases { 270 log.Trace("migrating releases") 271 messenger("repo.migrate.migrating_releases") 272 releases, err := downloader.GetReleases() 273 if err != nil { 274 if !base.IsErrNotSupported(err) { 275 return err 276 } 277 log.Warn("migrating releases is not supported, ignored") 278 } 279 280 relBatchSize := uploader.MaxBatchInsertSize("release") 281 for len(releases) > 0 { 282 if len(releases) < relBatchSize { 283 relBatchSize = len(releases) 284 } 285 286 if err = uploader.CreateReleases(releases[:relBatchSize]...); err != nil { 287 return err 288 } 289 releases = releases[relBatchSize:] 290 } 291 292 // Once all releases (if any) are inserted, sync any remaining non-release tags 293 if err = uploader.SyncTags(); err != nil { 294 return err 295 } 296 } 297 298 var ( 299 commentBatchSize = uploader.MaxBatchInsertSize("comment") 300 reviewBatchSize = uploader.MaxBatchInsertSize("review") 301 ) 302 303 supportAllComments := downloader.SupportGetRepoComments() 304 305 if opts.Issues { 306 log.Trace("migrating issues and comments") 307 messenger("repo.migrate.migrating_issues") 308 var issueBatchSize = uploader.MaxBatchInsertSize("issue") 309 310 for i := 1; ; i++ { 311 issues, isEnd, err := downloader.GetIssues(i, issueBatchSize) 312 if err != nil { 313 if !base.IsErrNotSupported(err) { 314 return err 315 } 316 log.Warn("migrating issues is not supported, ignored") 317 break 318 } 319 320 if err := uploader.CreateIssues(issues...); err != nil { 321 return err 322 } 323 324 if opts.Comments && !supportAllComments { 325 var allComments = make([]*base.Comment, 0, commentBatchSize) 326 for _, issue := range issues { 327 log.Trace("migrating issue %d's comments", issue.Number) 328 comments, _, err := downloader.GetComments(base.GetCommentOptions{ 329 Context: issue.Context, 330 }) 331 if err != nil { 332 if !base.IsErrNotSupported(err) { 333 return err 334 } 335 log.Warn("migrating comments is not supported, ignored") 336 } 337 338 allComments = append(allComments, comments...) 339 340 if len(allComments) >= commentBatchSize { 341 if err = uploader.CreateComments(allComments[:commentBatchSize]...); err != nil { 342 return err 343 } 344 345 allComments = allComments[commentBatchSize:] 346 } 347 } 348 349 if len(allComments) > 0 { 350 if err = uploader.CreateComments(allComments...); err != nil { 351 return err 352 } 353 } 354 } 355 356 if isEnd { 357 break 358 } 359 } 360 } 361 362 if opts.PullRequests { 363 log.Trace("migrating pull requests and comments") 364 messenger("repo.migrate.migrating_pulls") 365 var prBatchSize = uploader.MaxBatchInsertSize("pullrequest") 366 for i := 1; ; i++ { 367 prs, isEnd, err := downloader.GetPullRequests(i, prBatchSize) 368 if err != nil { 369 if !base.IsErrNotSupported(err) { 370 return err 371 } 372 log.Warn("migrating pull requests is not supported, ignored") 373 break 374 } 375 376 if err := uploader.CreatePullRequests(prs...); err != nil { 377 return err 378 } 379 380 if opts.Comments { 381 if !supportAllComments { 382 // plain comments 383 var allComments = make([]*base.Comment, 0, commentBatchSize) 384 for _, pr := range prs { 385 log.Trace("migrating pull request %d's comments", pr.Number) 386 comments, _, err := downloader.GetComments(base.GetCommentOptions{ 387 Context: pr.Context, 388 }) 389 if err != nil { 390 if !base.IsErrNotSupported(err) { 391 return err 392 } 393 log.Warn("migrating comments is not supported, ignored") 394 } 395 396 allComments = append(allComments, comments...) 397 398 if len(allComments) >= commentBatchSize { 399 if err = uploader.CreateComments(allComments[:commentBatchSize]...); err != nil { 400 return err 401 } 402 allComments = allComments[commentBatchSize:] 403 } 404 } 405 if len(allComments) > 0 { 406 if err = uploader.CreateComments(allComments...); err != nil { 407 return err 408 } 409 } 410 } 411 412 // migrate reviews 413 var allReviews = make([]*base.Review, 0, reviewBatchSize) 414 for _, pr := range prs { 415 reviews, err := downloader.GetReviews(pr.Context) 416 if err != nil { 417 if !base.IsErrNotSupported(err) { 418 return err 419 } 420 log.Warn("migrating reviews is not supported, ignored") 421 break 422 } 423 424 allReviews = append(allReviews, reviews...) 425 426 if len(allReviews) >= reviewBatchSize { 427 if err = uploader.CreateReviews(allReviews[:reviewBatchSize]...); err != nil { 428 return err 429 } 430 allReviews = allReviews[reviewBatchSize:] 431 } 432 } 433 if len(allReviews) > 0 { 434 if err = uploader.CreateReviews(allReviews...); err != nil { 435 return err 436 } 437 } 438 } 439 440 if isEnd { 441 break 442 } 443 } 444 } 445 446 if opts.Comments && supportAllComments { 447 log.Trace("migrating comments") 448 for i := 1; ; i++ { 449 comments, isEnd, err := downloader.GetComments(base.GetCommentOptions{ 450 Page: i, 451 PageSize: commentBatchSize, 452 }) 453 if err != nil { 454 return err 455 } 456 457 if err := uploader.CreateComments(comments...); err != nil { 458 return err 459 } 460 461 if isEnd { 462 break 463 } 464 } 465 } 466 467 return uploader.Finish() 468} 469 470// Init migrations service 471func Init() error { 472 // TODO: maybe we can deprecate these legacy ALLOWED_DOMAINS/ALLOW_LOCALNETWORKS/BLOCKED_DOMAINS, use ALLOWED_HOST_LIST/BLOCKED_HOST_LIST instead 473 474 blockList = hostmatcher.ParseSimpleMatchList("migrations.BLOCKED_DOMAINS", setting.Migrations.BlockedDomains) 475 476 allowList = hostmatcher.ParseSimpleMatchList("migrations.ALLOWED_DOMAINS/ALLOW_LOCALNETWORKS", setting.Migrations.AllowedDomains) 477 if allowList.IsEmpty() { 478 // the default policy is that migration module can access external hosts 479 allowList.AppendBuiltin(hostmatcher.MatchBuiltinExternal) 480 } 481 if setting.Migrations.AllowLocalNetworks { 482 allowList.AppendBuiltin(hostmatcher.MatchBuiltinPrivate) 483 allowList.AppendBuiltin(hostmatcher.MatchBuiltinLoopback) 484 } 485 return nil 486} 487