1 //------------------------------------------------------------------------------ 2 // <copyright file="CoordinatorScratchpad.cs" company="Microsoft"> 3 // Copyright (c) Microsoft Corporation. All rights reserved. 4 // </copyright> 5 // <owner current="true" primary="true">Microsoft</owner> 6 // <owner current="true" primary="false">Microsoft</owner> 7 //------------------------------------------------------------------------------ 8 9 using System.Collections.Generic; 10 using System.Data.Objects.ELinq; 11 using System.Data.Query.InternalTrees; 12 using System.Diagnostics; 13 using System.Linq; 14 using System.Linq.Expressions; 15 using System.Reflection; 16 using System.Runtime.CompilerServices; 17 using System.Security; 18 using System.Security.Permissions; 19 20 namespace System.Data.Common.Internal.Materialization 21 { 22 /// <summary> 23 /// Used in the Translator to aggregate information about a (nested) reader 24 /// coordinator. After the translator visits the columnMaps, it will compile 25 /// the coordinator(s) which produces an immutable CoordinatorFactory that 26 /// can be shared amongst many query instances. 27 /// </summary> 28 internal class CoordinatorScratchpad 29 { 30 #region private state 31 32 private readonly Type _elementType; 33 private CoordinatorScratchpad _parent; 34 private readonly List<CoordinatorScratchpad> _nestedCoordinatorScratchpads; 35 /// <summary> 36 /// Map from original expressions to expressions with detailed error handling. 37 /// </summary> 38 private readonly Dictionary<Expression, Expression> _expressionWithErrorHandlingMap; 39 /// <summary> 40 /// Expressions that should be precompiled (i.e. reduced to constants in 41 /// compiled delegates. 42 /// </summary> 43 private readonly HashSet<LambdaExpression> _inlineDelegates; 44 45 #endregion 46 47 #region constructor 48 CoordinatorScratchpad(Type elementType)49 internal CoordinatorScratchpad(Type elementType) 50 { 51 _elementType = elementType; 52 _nestedCoordinatorScratchpads = new List<CoordinatorScratchpad>(); 53 _expressionWithErrorHandlingMap = new Dictionary<Expression, Expression>(); 54 _inlineDelegates = new HashSet<LambdaExpression>(); 55 } 56 57 #endregion 58 59 #region "public" surface area 60 61 /// <summary> 62 /// For nested collections, returns the parent coordinator. 63 /// </summary> 64 internal CoordinatorScratchpad Parent 65 { 66 get { return _parent; } 67 } 68 69 /// <summary> 70 /// Gets or sets an Expression setting key values (these keys are used 71 /// to determine when a collection has entered a new chapter) from the 72 /// underlying store data reader. 73 /// </summary> 74 internal Expression SetKeys { get; set; } 75 76 /// <summary> 77 /// Gets or sets an Expression returning 'true' when the key values for 78 /// the current nested result (see SetKeys) are equal to the current key 79 /// values on the underlying data reader. 80 /// </summary> 81 internal Expression CheckKeys { get; set; } 82 83 /// <summary> 84 /// Gets or sets an expression returning 'true' if the current row in 85 /// the underlying data reader contains an element of the collection. 86 /// </summary> 87 internal Expression HasData { get; set; } 88 89 /// <summary> 90 /// Gets or sets an Expression yielding an element of the current collection 91 /// given values in the underlying data reader. 92 /// </summary> 93 internal Expression Element { get; set; } 94 95 /// <summary> 96 /// Gets or sets an Expression initializing the collection storing results from this coordinator. 97 /// </summary> 98 internal Expression InitializeCollection { get; set; } 99 100 /// <summary> 101 /// Indicates which Shaper.State slot is home for this collection's coordinator. 102 /// Used by Parent to pull out nested collection aggregators/streamers. 103 /// </summary> 104 internal int StateSlotNumber { get; set; } 105 106 /// <summary> 107 /// Gets or sets the depth of the current coordinator. A root collection has depth 0. 108 /// </summary> 109 internal int Depth { get; set; } 110 111 /// <summary> 112 /// List of all record types that we can return at this level in the query. 113 /// </summary> 114 private List<RecordStateScratchpad> _recordStateScratchpads; 115 116 /// <summary> 117 /// Allows sub-expressions to register an 'interest' in exceptions thrown when reading elements 118 /// for this coordinator. When an exception is thrown, we rerun the delegate using the slower 119 /// but more error-friendly versions of expressions (e.g. reader.GetValue + type check instead 120 /// of reader.GetInt32()) 121 /// </summary> 122 /// <param name="expression">The lean and mean raw version of the expression</param> 123 /// <param name="expressionWithErrorHandling">The slower version of the same expression with better 124 /// error handling</param> AddExpressionWithErrorHandling(Expression expression, Expression expressionWithErrorHandling)125 internal void AddExpressionWithErrorHandling(Expression expression, Expression expressionWithErrorHandling) 126 { 127 _expressionWithErrorHandlingMap[expression] = expressionWithErrorHandling; 128 } 129 130 /// <summary> 131 /// Registers a lambda expression for pre-compilation (i.e. reduction to a constant expression) 132 /// within materialization expression. Otherwise, the expression will be compiled every time 133 /// the enclosing delegate is invoked. 134 /// </summary> 135 /// <param name="expression">Lambda expression to register.</param> AddInlineDelegate(LambdaExpression expression)136 internal void AddInlineDelegate(LambdaExpression expression) 137 { 138 _inlineDelegates.Add(expression); 139 } 140 141 /// <summary> 142 /// Registers a coordinator for a nested collection contained in elements of this collection. 143 /// </summary> AddNestedCoordinator(CoordinatorScratchpad nested)144 internal void AddNestedCoordinator(CoordinatorScratchpad nested) 145 { 146 Debug.Assert(nested.Depth == this.Depth + 1, "can only nest depth + 1"); 147 nested._parent = this; 148 _nestedCoordinatorScratchpads.Add(nested); 149 } 150 151 /// <summary> 152 /// Use the information stored on the scratchpad to compile an immutable factory used 153 /// to construct the coordinators used at runtime when materializing results. 154 /// </summary> 155 [MethodImpl(MethodImplOptions.NoInlining | MethodImplOptions.NoOptimization)] Compile()156 internal CoordinatorFactory Compile() 157 { 158 RecordStateFactory[] recordStateFactories; 159 if (null != _recordStateScratchpads) 160 { 161 recordStateFactories = new RecordStateFactory[_recordStateScratchpads.Count]; 162 for (int i = 0; i < recordStateFactories.Length; i++) 163 { 164 recordStateFactories[i] = _recordStateScratchpads[i].Compile(); 165 } 166 } 167 else 168 { 169 recordStateFactories = new RecordStateFactory[0]; 170 } 171 172 CoordinatorFactory[] nestedCoordinators = new CoordinatorFactory[_nestedCoordinatorScratchpads.Count]; 173 for (int i = 0; i < nestedCoordinators.Length; i++) 174 { 175 nestedCoordinators[i] = _nestedCoordinatorScratchpads[i].Compile(); 176 } 177 178 // compile inline delegates 179 ReplacementExpressionVisitor replacementVisitor = new ReplacementExpressionVisitor(null, this._inlineDelegates); 180 Expression element = new SecurityBoundaryExpressionVisitor().Visit(replacementVisitor.Visit(this.Element)); 181 182 // substitute expressions that have error handlers into a new expression (used 183 // when a more detailed exception message is needed) 184 replacementVisitor = new ReplacementExpressionVisitor(this._expressionWithErrorHandlingMap, this._inlineDelegates); 185 Expression elementWithErrorHandling = new SecurityBoundaryExpressionVisitor().Visit(replacementVisitor.Visit(this.Element)); 186 187 CoordinatorFactory result = (CoordinatorFactory)Activator.CreateInstance(typeof(CoordinatorFactory<>).MakeGenericType(_elementType), new object[] { 188 this.Depth, 189 this.StateSlotNumber, 190 this.HasData, 191 this.SetKeys, 192 this.CheckKeys, 193 nestedCoordinators, 194 element, 195 elementWithErrorHandling, 196 this.InitializeCollection, 197 recordStateFactories 198 }); 199 return result; 200 } 201 202 /// <summary> 203 /// Allocates a new RecordStateScratchpad and adds it to the list of the ones we're 204 /// responsible for; will create the list if it hasn't alread been created. 205 /// </summary> CreateRecordStateScratchpad()206 internal RecordStateScratchpad CreateRecordStateScratchpad() 207 { 208 RecordStateScratchpad recordStateScratchpad = new RecordStateScratchpad(); 209 210 if (null == _recordStateScratchpads) 211 { 212 _recordStateScratchpads = new List<RecordStateScratchpad>(); 213 } 214 _recordStateScratchpads.Add(recordStateScratchpad); 215 return recordStateScratchpad; 216 } 217 #endregion 218 219 #region Nested types 220 221 /// <summary> 222 /// Visitor supporting (non-recursive) replacement of LINQ sub-expressions and 223 /// compilation of inline delegates. 224 /// </summary> 225 private class ReplacementExpressionVisitor : EntityExpressionVisitor 226 { 227 // Map from original expressions to replacement expressions. 228 private readonly Dictionary<Expression, Expression> _replacementDictionary; 229 private readonly HashSet<LambdaExpression> _inlineDelegates; 230 ReplacementExpressionVisitor(Dictionary<Expression, Expression> replacementDictionary, HashSet<LambdaExpression> inlineDelegates)231 internal ReplacementExpressionVisitor(Dictionary<Expression, Expression> replacementDictionary, 232 HashSet<LambdaExpression> inlineDelegates) 233 { 234 this._replacementDictionary = replacementDictionary; 235 this._inlineDelegates = inlineDelegates; 236 } 237 Visit(Expression expression)238 internal override Expression Visit(Expression expression) 239 { 240 if (null == expression) 241 { 242 return expression; 243 } 244 245 Expression result; 246 247 // check to see if a substitution has been provided for this expression 248 Expression replacement; 249 if (null != this._replacementDictionary && this._replacementDictionary.TryGetValue(expression, out replacement)) 250 { 251 // once a substitution is found, we stop walking the sub-expression and 252 // return immediately (since recursive replacement is not needed or wanted) 253 result = replacement; 254 } 255 else 256 { 257 // check if we need to precompile an inline delegate 258 bool preCompile = false; 259 LambdaExpression lambda = null; 260 261 if (expression.NodeType == ExpressionType.Lambda && 262 null != _inlineDelegates) 263 { 264 lambda = (LambdaExpression)expression; 265 preCompile = _inlineDelegates.Contains(lambda); 266 } 267 268 if (preCompile) 269 { 270 // do replacement in the body of the lambda expression 271 Expression body = Visit(lambda.Body); 272 273 // compile to a delegate 274 result = Expression.Constant(Translator.Compile(body.Type, body)); 275 } 276 else 277 { 278 result = base.Visit(expression); 279 } 280 } 281 282 return result; 283 } 284 } 285 286 /// <summary> 287 /// Used to replace references to user expressions with compiled delegates 288 /// which represent those expressions. 289 /// </summary> 290 /// <remarks> 291 /// The materialization delegate used to be one big function, which included 292 /// user-provided expressions in various places in the tree. Due to security reasons 293 /// (Dev11 311339), we need to separate this delegate into two pieces: trusted code, 294 /// run under a security assert, and untrusted code, run under the current AppDomain's 295 /// permission set. 296 /// 297 /// This visitor does that separation by compiling the untrusted code into delegates 298 /// and re-inserting them back into the expression tree. When the untrusted code is 299 /// run, it will run in another stack frame that does not have a security assert 300 /// associated with it; therefore, any attempt to take advantage of MemberAccess 301 /// reflection permissions will be blocked by the CLR. 302 /// 303 /// The compiled user delegates accept two parameters, one of type DbDataReader 304 /// to speed up access to the current reader, and the other of type object[], 305 /// which contains all other values that they might require to correctly materialize an object. Most of these 306 /// objects require the <see cref="Shaper"/>, so they must be run inside of trusted code. 307 /// </remarks> 308 private sealed class SecurityBoundaryExpressionVisitor : EntityExpressionVisitor 309 { 310 private static readonly MethodInfo s_userMaterializationFuncInvokeMethod = typeof(Func<DbDataReader, object[], object>).GetMethod("Invoke"); 311 private ParameterExpression _values = Expression.Parameter(typeof(object[]), "values"); 312 private ParameterExpression _reader = Expression.Parameter(typeof(DbDataReader), "reader"); 313 private List<Expression> _initializationArguments = new List<Expression>(); 314 private int _userExpressionDepth; 315 316 /// <summary> 317 /// Used to track the type of a constructor argument or member assignment 318 /// when it could be a special type we create (e.g., CompensatingCollection{T} 319 /// for collections and Grouping{K,V} for groups). 320 /// </summary> 321 private Type _userArgumentType; 322 Visit(Expression exp)323 internal override Expression Visit(Expression exp) 324 { 325 if (exp == null) 326 { 327 return exp; 328 } 329 330 var nex = exp as NewExpression; 331 if (nex != null && _userExpressionDepth >= 1) 332 { 333 // We are creating an internal type like CompensatingCollection<T> or Grouping<K, V> 334 // and at this particular point we are sure that the user isn't creating these 335 // since this.userArgumentType is not null. 336 if (_userArgumentType != null && !nex.Type.IsPublic && nex.Type.Assembly == typeof(SecurityBoundaryExpressionVisitor).Assembly) 337 { 338 return this.CreateInitializationArgumentReplacement(nex, _userArgumentType); 339 } 340 341 var constructorParameters = nex.Constructor.GetParameters(); 342 var arguments = nex.Arguments; 343 var newArguments = new List<Expression>(); 344 for (var i = 0; i < arguments.Count; ++i) 345 { 346 var argument = arguments[i]; 347 348 // Visit this argument because it itself could be a user expression e.g. 349 // new { Argument = new SecureString { m_length = 32 } } 350 _userArgumentType = constructorParameters[i].ParameterType; 351 var visitedArgument = this.Visit(argument); 352 353 // If it hasn't changed, it's trusted code. (Untrusted code would have its 354 // Convert and MarkAsUserExpression expressions removed.) 355 if (visitedArgument == argument) 356 { 357 var convert = this.CreateInitializationArgumentReplacement(argument); 358 359 // Change the argument to access the values array. 360 newArguments.Add(convert); 361 } 362 else 363 { 364 newArguments.Add(visitedArgument); 365 } 366 } 367 368 nex = Expression.New(nex.Constructor, newArguments); 369 370 if (_userExpressionDepth == 1) 371 { 372 var userMaterializationFunc = Expression.Lambda<Func<DbDataReader, object[], object>>(nex, _reader, _values).Compile(); 373 374 // Convert the constructor invocation into a func that runs without elevated permissions. 375 return Expression.Convert( 376 Expression.Call( 377 Expression.Constant(userMaterializationFunc), 378 s_userMaterializationFuncInvokeMethod, 379 Translator.Shaper_Reader, 380 Expression.NewArrayInit(typeof(object), _initializationArguments)), 381 nex.Type); 382 } 383 384 return nex; 385 } 386 387 return base.Visit(exp); 388 } 389 VisitConditional(ConditionalExpression c)390 internal override Expression VisitConditional(ConditionalExpression c) 391 { 392 if (_userExpressionDepth >= 1 && _userArgumentType != null) 393 { 394 var test = c.Test as MethodCallExpression; 395 var ifFalse = c.IfFalse as MethodCallExpression; 396 397 // We can optimize the path that checks for DbNull and then 398 // reads a value directly off the reader or invokes another user expression. 399 if (test != null && test.Object != null 400 && typeof(DbDataReader).IsAssignableFrom(test.Object.Type) 401 && test.Method.Name == "IsDBNull") 402 { 403 if (ifFalse != null && (ifFalse.Object != null && typeof(DbDataReader).IsAssignableFrom(ifFalse.Object.Type) || IsUserExpressionMethod(ifFalse.Method))) 404 { 405 return base.VisitConditional(c); 406 } 407 } 408 409 // If there's something more complicated then we have to replace it all. 410 // We can't just replace the false expression because it may not be evaluated 411 // if the test returns true. 412 return this.CreateInitializationArgumentReplacement(c); 413 } 414 415 return base.VisitConditional(c); 416 } 417 VisitMemberAccess(MemberExpression m)418 internal override Expression VisitMemberAccess(MemberExpression m) 419 { 420 if (_userExpressionDepth >= 1) 421 { 422 // Sometimes we will add expressions inside of a user expression that is actually 423 // our code, but we need to rewrite it since it accesses the shaper's reader to check if a column is null. 424 // e.g. Select(x => new { Y = new Entity { Name = x.Name } }) 425 // -> new f<>__AnonymousType`1(IIF($shaper.Reader.IsDbNull(0), null, new Entity { Name = $shaper.Reader.GetString(0) })) 426 if (typeof(DbDataReader).IsAssignableFrom(m.Type)) 427 { 428 var shaper = m.Expression as ParameterExpression; 429 if (shaper != null && shaper == Translator.Shaper_Parameter) 430 { 431 return _reader; 432 } 433 } 434 } 435 436 return base.VisitMemberAccess(m); 437 } 438 VisitMemberInit(MemberInitExpression init)439 internal override Expression VisitMemberInit(MemberInitExpression init) 440 { 441 if (_userExpressionDepth >= 1) 442 { 443 var newMemberInit = base.VisitMemberInit(init); 444 445 // Only compile into a delegate if this is the top-level user expression. 446 if (newMemberInit != init && _userExpressionDepth == 1) 447 { 448 var userMaterializationFunc = Expression.Lambda<Func<DbDataReader, object[], object>>(newMemberInit, _reader, _values).Compile(); 449 450 // Convert the object initializer into a func that runs without elevated permissions. 451 return Expression.Convert( 452 Expression.Call( 453 Expression.Constant(userMaterializationFunc), 454 s_userMaterializationFuncInvokeMethod, 455 Translator.Shaper_Reader, 456 Expression.NewArrayInit(typeof(object), _initializationArguments)), 457 init.Type); 458 } 459 else 460 { 461 return newMemberInit; 462 } 463 } 464 465 return base.VisitMemberInit(init); 466 } 467 VisitMemberAssignment(MemberAssignment assignment)468 internal override MemberAssignment VisitMemberAssignment(MemberAssignment assignment) 469 { 470 if (_userExpressionDepth >= 1) 471 { 472 var fieldInfo = assignment.Member as FieldInfo; 473 var propertyInfo = assignment.Member as PropertyInfo; 474 if (fieldInfo != null) 475 { 476 _userArgumentType = fieldInfo.FieldType; 477 } 478 else if (propertyInfo != null) 479 { 480 _userArgumentType = propertyInfo.PropertyType; 481 } 482 } 483 484 return base.VisitMemberAssignment(assignment); 485 } 486 VisitMethodCall(MethodCallExpression m)487 internal override Expression VisitMethodCall(MethodCallExpression m) 488 { 489 var method = m.Method; 490 if (IsUserExpressionMethod(method)) 491 { 492 Debug.Assert( 493 m.Arguments.Count == 1, 494 "m.Arguments.Count == 1", 495 "There should be one expression argument provided to the user expression marker."); 496 497 try 498 { 499 // Clear this type because we are about to process a user expression 500 _userArgumentType = null; 501 502 _userExpressionDepth++; 503 return this.Visit(m.Arguments[0]); 504 } 505 finally 506 { 507 _userExpressionDepth--; 508 } 509 } 510 else if (_userExpressionDepth >= 1) 511 { 512 // If this method call is on a DbDataReader then we can replace it; otherwise, 513 // assume it's something on the shaper and extract the value into the values array. 514 if (m.Object != null && typeof(DbDataReader).IsAssignableFrom(m.Object.Type)) 515 { 516 return base.VisitMethodCall(m); 517 } 518 519 return this.CreateInitializationArgumentReplacement(m); 520 } 521 522 return base.VisitMethodCall(m); 523 } 524 CreateInitializationArgumentReplacement(Expression original)525 private Expression CreateInitializationArgumentReplacement(Expression original) 526 { 527 return this.CreateInitializationArgumentReplacement(original, original.Type); 528 } 529 CreateInitializationArgumentReplacement(Expression original, Type expressionType)530 private Expression CreateInitializationArgumentReplacement(Expression original, Type expressionType) 531 { 532 _initializationArguments.Add(Expression.Convert(original, typeof(object))); 533 534 return Expression.Convert( 535 Expression.MakeBinary(ExpressionType.ArrayIndex, _values, Expression.Constant(_initializationArguments.Count - 1)), 536 expressionType); 537 } 538 IsUserExpressionMethod(MethodInfo method)539 private static bool IsUserExpressionMethod(MethodInfo method) 540 { 541 return method.IsGenericMethod && method.GetGenericMethodDefinition() == InitializerMetadata.UserExpressionMarker; 542 } 543 } 544 #endregion 545 } 546 } 547