1package operations 2 3import ( 4 "bytes" 5 "context" 6 "encoding/base64" 7 "errors" 8 "fmt" 9 "io" 10 "strings" 11 12 "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus/ctxlogrus" 13 "github.com/sirupsen/logrus" 14 "gitlab.com/gitlab-org/gitaly/v14/internal/git" 15 "gitlab.com/gitlab-org/gitaly/v14/internal/git/localrepo" 16 "gitlab.com/gitlab-org/gitaly/v14/internal/git/remoterepo" 17 "gitlab.com/gitlab-org/gitaly/v14/internal/git/updateref" 18 "gitlab.com/gitlab-org/gitaly/v14/internal/git2go" 19 "gitlab.com/gitlab-org/gitaly/v14/internal/gitaly/storage" 20 "gitlab.com/gitlab-org/gitaly/v14/internal/helper" 21 "gitlab.com/gitlab-org/gitaly/v14/proto/go/gitalypb" 22 "google.golang.org/grpc/codes" 23 "google.golang.org/grpc/status" 24) 25 26// UserCommitFiles allows for committing from a set of actions. See the protobuf documentation 27// for details. 28func (s *Server) UserCommitFiles(stream gitalypb.OperationService_UserCommitFilesServer) error { 29 firstRequest, err := stream.Recv() 30 if err != nil { 31 return err 32 } 33 34 header := firstRequest.GetHeader() 35 if header == nil { 36 return status.Errorf(codes.InvalidArgument, "UserCommitFiles: empty UserCommitFilesRequestHeader") 37 } 38 39 if err = validateUserCommitFilesHeader(header); err != nil { 40 return status.Errorf(codes.InvalidArgument, "UserCommitFiles: %v", err) 41 } 42 43 ctx := stream.Context() 44 45 if err := s.userCommitFiles(ctx, header, stream); err != nil { 46 ctxlogrus.AddFields(ctx, logrus.Fields{ 47 "repository_storage": header.Repository.StorageName, 48 "repository_relative_path": header.Repository.RelativePath, 49 "branch_name": header.BranchName, 50 "start_branch_name": header.StartBranchName, 51 "start_sha": header.StartSha, 52 "force": header.Force, 53 }) 54 55 if startRepo := header.GetStartRepository(); startRepo != nil { 56 ctxlogrus.AddFields(ctx, logrus.Fields{ 57 "start_repository_storage": startRepo.StorageName, 58 "start_repository_relative_path": startRepo.RelativePath, 59 }) 60 } 61 62 var ( 63 response gitalypb.UserCommitFilesResponse 64 indexError git2go.IndexError 65 hookError updateref.HookError 66 ) 67 68 switch { 69 case errors.As(err, &indexError): 70 response = gitalypb.UserCommitFilesResponse{IndexError: indexError.Error()} 71 case errors.As(err, new(git2go.DirectoryExistsError)): 72 response = gitalypb.UserCommitFilesResponse{IndexError: "A directory with this name already exists"} 73 case errors.As(err, new(git2go.FileExistsError)): 74 response = gitalypb.UserCommitFilesResponse{IndexError: "A file with this name already exists"} 75 case errors.As(err, new(git2go.FileNotFoundError)): 76 response = gitalypb.UserCommitFilesResponse{IndexError: "A file with this name doesn't exist"} 77 case errors.As(err, &hookError): 78 response = gitalypb.UserCommitFilesResponse{PreReceiveError: hookError.Error()} 79 case errors.As(err, new(git2go.InvalidArgumentError)): 80 return helper.ErrInvalidArgument(err) 81 default: 82 return err 83 } 84 85 ctxlogrus.Extract(ctx).WithError(err).Error("user commit files failed") 86 return stream.SendAndClose(&response) 87 } 88 89 return nil 90} 91 92func validatePath(rootPath, relPath string) (string, error) { 93 if relPath == "" { 94 return "", git2go.IndexError("You must provide a file path") 95 } else if strings.Contains(relPath, "//") { 96 // This is a workaround to address a quirk in porting the RPC from Ruby to Go. 97 // GitLab's QA pipeline runs tests with filepath 'invalid://file/name/here'. 98 // Go's filepath.Clean returns 'invalid:/file/name/here'. The Ruby implementation's 99 // filepath normalization accepted the path as is. Adding a file with this path to the 100 // index via Rugged failed with an invalid path error. As Go's cleaning resulted a valid 101 // filepath, adding the file succeeded, which made the QA pipeline's specs fail. 102 // 103 // The Rails code expects to receive an error prefixed with 'invalid path', which is done 104 // here to retain compatibility. 105 return "", git2go.IndexError(fmt.Sprintf("invalid path: '%s'", relPath)) 106 } 107 108 path, err := storage.ValidateRelativePath(rootPath, relPath) 109 if err != nil { 110 if errors.Is(err, storage.ErrRelativePathEscapesRoot) { 111 return "", git2go.IndexError("Path cannot include directory traversal") 112 } 113 114 return "", err 115 } 116 117 return path, nil 118} 119 120func (s *Server) userCommitFiles(ctx context.Context, header *gitalypb.UserCommitFilesRequestHeader, stream gitalypb.OperationService_UserCommitFilesServer) error { 121 quarantineDir, quarantineRepo, err := s.quarantinedRepo(ctx, header.GetRepository()) 122 if err != nil { 123 return err 124 } 125 126 repoPath, err := quarantineRepo.Path() 127 if err != nil { 128 return fmt.Errorf("get repo path: %w", err) 129 } 130 131 remoteRepo := header.GetStartRepository() 132 if sameRepository(header.GetRepository(), remoteRepo) { 133 // Some requests set a StartRepository that refers to the same repository as the target repository. 134 // This check never works behind Praefect. See: https://gitlab.com/gitlab-org/gitaly/-/issues/3294 135 // Plain Gitalies still benefit from identifying the case and avoiding unnecessary RPC to resolve the 136 // branch. 137 remoteRepo = nil 138 } 139 140 targetBranchName := git.NewReferenceNameFromBranchName(string(header.BranchName)) 141 targetBranchCommit, err := quarantineRepo.ResolveRevision(ctx, targetBranchName.Revision()+"^{commit}") 142 if err != nil { 143 if !errors.Is(err, git.ErrReferenceNotFound) { 144 return fmt.Errorf("resolve target branch commit: %w", err) 145 } 146 147 // the branch is being created 148 } 149 150 var parentCommitOID git.ObjectID 151 if header.StartSha == "" { 152 parentCommitOID, err = s.resolveParentCommit( 153 ctx, 154 quarantineRepo, 155 remoteRepo, 156 targetBranchName, 157 targetBranchCommit, 158 string(header.StartBranchName), 159 ) 160 if err != nil { 161 return fmt.Errorf("resolve parent commit: %w", err) 162 } 163 } else { 164 parentCommitOID, err = git.NewObjectIDFromHex(header.StartSha) 165 if err != nil { 166 return helper.ErrInvalidArgumentf("cannot resolve parent commit: %w", err) 167 } 168 } 169 170 if parentCommitOID != targetBranchCommit { 171 if err := s.fetchMissingCommit(ctx, quarantineRepo, remoteRepo, parentCommitOID); err != nil { 172 return fmt.Errorf("fetch missing commit: %w", err) 173 } 174 } 175 176 type action struct { 177 header *gitalypb.UserCommitFilesActionHeader 178 content []byte 179 } 180 181 var pbActions []action 182 183 for { 184 req, err := stream.Recv() 185 if err != nil { 186 if errors.Is(err, io.EOF) { 187 break 188 } 189 190 return fmt.Errorf("receive request: %w", err) 191 } 192 193 switch payload := req.GetAction().GetUserCommitFilesActionPayload().(type) { 194 case *gitalypb.UserCommitFilesAction_Header: 195 pbActions = append(pbActions, action{header: payload.Header}) 196 case *gitalypb.UserCommitFilesAction_Content: 197 if len(pbActions) == 0 { 198 return errors.New("content sent before action") 199 } 200 201 // append the content to the previous action 202 content := &pbActions[len(pbActions)-1].content 203 *content = append(*content, payload.Content...) 204 default: 205 return fmt.Errorf("unhandled action payload type: %T", payload) 206 } 207 } 208 209 actions := make([]git2go.Action, 0, len(pbActions)) 210 for _, pbAction := range pbActions { 211 if _, ok := gitalypb.UserCommitFilesActionHeader_ActionType_name[int32(pbAction.header.Action)]; !ok { 212 return fmt.Errorf("NoMethodError: undefined method `downcase' for %d:Integer", pbAction.header.Action) 213 } 214 215 path, err := validatePath(repoPath, string(pbAction.header.FilePath)) 216 if err != nil { 217 return fmt.Errorf("validate path: %w", err) 218 } 219 220 content := io.Reader(bytes.NewReader(pbAction.content)) 221 if pbAction.header.Base64Content { 222 content = base64.NewDecoder(base64.StdEncoding, content) 223 } 224 225 switch pbAction.header.Action { 226 case gitalypb.UserCommitFilesActionHeader_CREATE: 227 blobID, err := quarantineRepo.WriteBlob(ctx, path, content) 228 if err != nil { 229 return fmt.Errorf("write created blob: %w", err) 230 } 231 232 actions = append(actions, git2go.CreateFile{ 233 OID: blobID.String(), 234 Path: path, 235 ExecutableMode: pbAction.header.ExecuteFilemode, 236 }) 237 case gitalypb.UserCommitFilesActionHeader_CHMOD: 238 actions = append(actions, git2go.ChangeFileMode{ 239 Path: path, 240 ExecutableMode: pbAction.header.ExecuteFilemode, 241 }) 242 case gitalypb.UserCommitFilesActionHeader_MOVE: 243 prevPath, err := validatePath(repoPath, string(pbAction.header.PreviousPath)) 244 if err != nil { 245 return fmt.Errorf("validate previous path: %w", err) 246 } 247 248 var oid git.ObjectID 249 if !pbAction.header.InferContent { 250 var err error 251 oid, err = quarantineRepo.WriteBlob(ctx, path, content) 252 if err != nil { 253 return err 254 } 255 } 256 257 actions = append(actions, git2go.MoveFile{ 258 Path: prevPath, 259 NewPath: path, 260 OID: oid.String(), 261 }) 262 case gitalypb.UserCommitFilesActionHeader_UPDATE: 263 oid, err := quarantineRepo.WriteBlob(ctx, path, content) 264 if err != nil { 265 return fmt.Errorf("write updated blob: %w", err) 266 } 267 268 actions = append(actions, git2go.UpdateFile{ 269 Path: path, 270 OID: oid.String(), 271 }) 272 case gitalypb.UserCommitFilesActionHeader_DELETE: 273 actions = append(actions, git2go.DeleteFile{ 274 Path: path, 275 }) 276 case gitalypb.UserCommitFilesActionHeader_CREATE_DIR: 277 actions = append(actions, git2go.CreateDirectory{ 278 Path: path, 279 }) 280 } 281 } 282 283 now, err := dateFromProto(header) 284 if err != nil { 285 return helper.ErrInvalidArgument(err) 286 } 287 288 committer := git2go.NewSignature(string(header.User.Name), string(header.User.Email), now) 289 author := committer 290 if len(header.CommitAuthorName) > 0 && len(header.CommitAuthorEmail) > 0 { 291 author = git2go.NewSignature(string(header.CommitAuthorName), string(header.CommitAuthorEmail), now) 292 } 293 294 commitID, err := s.git2go.Commit(ctx, quarantineRepo, git2go.CommitParams{ 295 Repository: repoPath, 296 Author: author, 297 Committer: committer, 298 Message: string(header.CommitMessage), 299 Parent: parentCommitOID.String(), 300 Actions: actions, 301 }) 302 if err != nil { 303 return fmt.Errorf("commit: %w", err) 304 } 305 306 hasBranches, err := quarantineRepo.HasBranches(ctx) 307 if err != nil { 308 return fmt.Errorf("was repo created: %w", err) 309 } 310 311 oldRevision := parentCommitOID 312 if targetBranchCommit == "" { 313 oldRevision = git.ZeroOID 314 } else if header.Force { 315 oldRevision = targetBranchCommit 316 } 317 318 if err := s.updateReferenceWithHooks(ctx, header.GetRepository(), header.User, quarantineDir, targetBranchName, commitID, oldRevision); err != nil { 319 if errors.As(err, &updateref.Error{}) { 320 return status.Errorf(codes.FailedPrecondition, err.Error()) 321 } 322 323 return fmt.Errorf("update reference: %w", err) 324 } 325 326 return stream.SendAndClose(&gitalypb.UserCommitFilesResponse{BranchUpdate: &gitalypb.OperationBranchUpdate{ 327 CommitId: commitID.String(), 328 RepoCreated: !hasBranches, 329 BranchCreated: oldRevision.IsZeroOID(), 330 }}) 331} 332 333func sameRepository(repoA, repoB *gitalypb.Repository) bool { 334 return repoA.GetStorageName() == repoB.GetStorageName() && 335 repoA.GetRelativePath() == repoB.GetRelativePath() 336} 337 338func (s *Server) resolveParentCommit( 339 ctx context.Context, 340 local git.Repository, 341 remote *gitalypb.Repository, 342 targetBranch git.ReferenceName, 343 targetBranchCommit git.ObjectID, 344 startBranch string, 345) (git.ObjectID, error) { 346 if remote == nil && startBranch == "" { 347 return targetBranchCommit, nil 348 } 349 350 repo := local 351 if remote != nil { 352 var err error 353 repo, err = remoterepo.New(ctx, remote, s.conns) 354 if err != nil { 355 return "", fmt.Errorf("remote repository: %w", err) 356 } 357 } 358 359 if hasBranches, err := repo.HasBranches(ctx); err != nil { 360 return "", fmt.Errorf("has branches: %w", err) 361 } else if !hasBranches { 362 // GitLab sends requests to UserCommitFiles where target repository 363 // and start repository are the same. If the request hits Gitaly directly, 364 // Gitaly could check if the repos are the same by comparing their storages 365 // and relative paths and simply resolve the branch locally. When request is proxied 366 // through Praefect, the start repository's storage is not rewritten, thus Gitaly can't 367 // identify the repos as being the same. 368 // 369 // If the start repository is set, we have to resolve the branch there as it 370 // might be on a different commit than the local repository. As Gitaly can't identify 371 // the repositories are the same behind Praefect, it has to perform an RPC to resolve 372 // the branch. The resolving would fail as the branch does not yet exist in the start 373 // repository, which is actually the local repository. 374 // 375 // Due to this, we check if the remote has any branches. If not, we likely hit this case 376 // and we're creating the first branch. If so, we'll just return the commit that was 377 // already resolved locally. 378 // 379 // See: https://gitlab.com/gitlab-org/gitaly/-/issues/3294 380 return targetBranchCommit, nil 381 } 382 383 branch := targetBranch 384 if startBranch != "" { 385 branch = git.NewReferenceNameFromBranchName(startBranch) 386 } 387 refish := branch + "^{commit}" 388 389 commit, err := repo.ResolveRevision(ctx, git.Revision(refish)) 390 if err != nil { 391 return "", fmt.Errorf("resolving refish %q in %T: %w", refish, repo, err) 392 } 393 394 return commit, nil 395} 396 397func (s *Server) fetchMissingCommit( 398 ctx context.Context, 399 localRepo *localrepo.Repo, 400 remoteRepo *gitalypb.Repository, 401 commit git.ObjectID, 402) error { 403 if _, err := localRepo.ResolveRevision(ctx, commit.Revision()+"^{commit}"); err != nil { 404 if !errors.Is(err, git.ErrReferenceNotFound) || remoteRepo == nil { 405 return fmt.Errorf("lookup parent commit: %w", err) 406 } 407 408 if err := localRepo.FetchInternal( 409 ctx, 410 remoteRepo, 411 []string{commit.String()}, 412 localrepo.FetchOpts{Tags: localrepo.FetchOptsTagsNone}, 413 ); err != nil { 414 return fmt.Errorf("fetch parent commit: %w", err) 415 } 416 } 417 418 return nil 419} 420 421func validateUserCommitFilesHeader(header *gitalypb.UserCommitFilesRequestHeader) error { 422 if header.GetRepository() == nil { 423 return fmt.Errorf("empty Repository") 424 } 425 if header.GetUser() == nil { 426 return fmt.Errorf("empty User") 427 } 428 if len(header.GetCommitMessage()) == 0 { 429 return fmt.Errorf("empty CommitMessage") 430 } 431 if len(header.GetBranchName()) == 0 { 432 return fmt.Errorf("empty BranchName") 433 } 434 435 startSha := header.GetStartSha() 436 if len(startSha) > 0 { 437 err := git.ValidateObjectID(startSha) 438 if err != nil { 439 return err 440 } 441 } 442 443 return nil 444} 445