1// This file and its contents are licensed under the Apache License 2.0. 2// Please see the included NOTICE for copyright information and 3// LICENSE for a copy of the license. 4 5package end_to_end_tests 6 7import ( 8 "fmt" 9 "net/http" 10 "sort" 11 "testing" 12 13 "github.com/jackc/pgx/v4/pgxpool" 14 "github.com/stretchr/testify/require" 15 "github.com/timescale/promscale/pkg/clockcache" 16 "github.com/timescale/promscale/pkg/pgclient" 17 "github.com/timescale/promscale/pkg/pgmodel/cache" 18 "github.com/timescale/promscale/pkg/pgmodel/lreader" 19 "github.com/timescale/promscale/pkg/pgmodel/model" 20 "github.com/timescale/promscale/pkg/pgmodel/querier" 21 "github.com/timescale/promscale/pkg/pgxconn" 22 "github.com/timescale/promscale/pkg/prompb" 23 "github.com/timescale/promscale/pkg/tenancy" 24) 25 26func TestMultiTenancyWithoutValidTenants(t *testing.T) { 27 ts, tenants := generateSmallMultiTenantTimeseries() 28 withDB(t, *testDatabase, func(db *pgxpool.Pool, t testing.TB) { 29 // Without valid tenants. 30 cfg := tenancy.NewAllowAllTenantsConfig(false) 31 mt, err := tenancy.NewAuthorizer(cfg) 32 require.NoError(t, err) 33 34 // Ingestion. 35 client, err := pgclient.NewClientWithPool(&pgclient.Config{}, 1, db, mt, false) 36 require.NoError(t, err) 37 defer client.Close() 38 39 for _, tenant := range tenants { 40 request := newWriteRequestWithTs(copyMetrics(ts)) 41 // Pre-processing. 42 wauth := mt.WriteAuthorizer() 43 err = wauth.Process(requestWithHeaderTenant(tenant), request) 44 require.NoError(t, err) 45 _, _, err = client.Ingest(request) 46 require.NoError(t, err) 47 } 48 49 // Querying. 50 mCache := &cache.MetricNameCache{Metrics: clockcache.WithMax(cache.DefaultMetricCacheSize)} 51 lCache := clockcache.WithMax(100) 52 dbConn := pgxconn.NewPgxConn(db) 53 labelsReader := lreader.NewLabelsReader(dbConn, lCache) 54 qr := querier.NewQuerier(dbConn, mCache, labelsReader, nil, mt.ReadAuthorizer()) 55 56 // ----- query-test: querying a single tenant (tenant-a) ----- 57 expectedResult := []prompb.TimeSeries{ 58 { 59 Labels: []prompb.Label{ 60 {Name: model.MetricNameLabelName, Value: "firstMetric"}, 61 {Name: "foo", Value: "bar"}, 62 {Name: "common", Value: "tag"}, 63 {Name: "empty", Value: ""}, 64 {Name: tenancy.TenantLabelKey, Value: "tenant-a"}, 65 }, 66 Samples: []prompb.Sample{ 67 {Timestamp: 2, Value: 0.2}, 68 {Timestamp: 3, Value: 0.3}, 69 {Timestamp: 4, Value: 0.4}, 70 }, 71 }, 72 } 73 74 result, err := qr.Query(&prompb.Query{ 75 Matchers: []*prompb.LabelMatcher{ 76 { 77 Type: prompb.LabelMatcher_EQ, 78 Name: model.MetricNameLabelName, 79 Value: "firstMetric", 80 }, 81 { 82 Type: prompb.LabelMatcher_EQ, 83 Name: tenancy.TenantLabelKey, 84 Value: "tenant-a", 85 }, 86 }, 87 StartTimestampMs: 2, 88 EndTimestampMs: 4, 89 }) 90 require.NoError(t, err) 91 92 // Verifying result. 93 verifyResults(t, expectedResult, result) 94 95 // ----- query-test: querying across multiple tenants (tenant-a & tenant-c) ----- 96 expectedResult = []prompb.TimeSeries{ 97 { 98 Labels: []prompb.Label{ 99 {Name: model.MetricNameLabelName, Value: "secondMetric"}, 100 {Name: "job", Value: "baz"}, 101 {Name: "ins", Value: "tag"}, 102 {Name: tenancy.TenantLabelKey, Value: "tenant-a"}, 103 }, 104 Samples: []prompb.Sample{ 105 {Timestamp: 2, Value: 2.2}, 106 {Timestamp: 3, Value: 2.3}, 107 {Timestamp: 4, Value: 2.4}, 108 }, 109 }, 110 { 111 Labels: []prompb.Label{ 112 {Name: model.MetricNameLabelName, Value: "secondMetric"}, 113 {Name: "job", Value: "baz"}, 114 {Name: "ins", Value: "tag"}, 115 {Name: tenancy.TenantLabelKey, Value: "tenant-c"}, 116 }, 117 Samples: []prompb.Sample{ 118 {Timestamp: 2, Value: 2.2}, 119 {Timestamp: 3, Value: 2.3}, 120 {Timestamp: 4, Value: 2.4}, 121 }, 122 }, 123 } 124 125 result, err = qr.Query(&prompb.Query{ 126 Matchers: []*prompb.LabelMatcher{ 127 { 128 Type: prompb.LabelMatcher_EQ, 129 Name: model.MetricNameLabelName, 130 Value: "secondMetric", 131 }, 132 { 133 Type: prompb.LabelMatcher_RE, 134 Name: tenancy.TenantLabelKey, 135 Value: "tenant-a|tenant-c", 136 }, 137 }, 138 StartTimestampMs: 2, 139 EndTimestampMs: 4, 140 }) 141 require.NoError(t, err) 142 143 // Verifying result. 144 verifyResults(t, expectedResult, result) 145 146 // ----- query-test: querying without tenant matcher ----- 147 expectedResult = []prompb.TimeSeries{ 148 { 149 Labels: []prompb.Label{ 150 {Name: model.MetricNameLabelName, Value: "secondMetric"}, 151 {Name: "job", Value: "baz"}, 152 {Name: "ins", Value: "tag"}, 153 {Name: tenancy.TenantLabelKey, Value: "tenant-a"}, 154 }, 155 Samples: []prompb.Sample{ 156 {Timestamp: 2, Value: 2.2}, 157 {Timestamp: 3, Value: 2.3}, 158 {Timestamp: 4, Value: 2.4}, 159 }, 160 }, 161 { 162 Labels: []prompb.Label{ 163 {Name: model.MetricNameLabelName, Value: "secondMetric"}, 164 {Name: "job", Value: "baz"}, 165 {Name: "ins", Value: "tag"}, 166 {Name: tenancy.TenantLabelKey, Value: "tenant-b"}, 167 }, 168 Samples: []prompb.Sample{ 169 {Timestamp: 2, Value: 2.2}, 170 {Timestamp: 3, Value: 2.3}, 171 {Timestamp: 4, Value: 2.4}, 172 }, 173 }, 174 { 175 Labels: []prompb.Label{ 176 {Name: model.MetricNameLabelName, Value: "secondMetric"}, 177 {Name: "job", Value: "baz"}, 178 {Name: "ins", Value: "tag"}, 179 {Name: tenancy.TenantLabelKey, Value: "tenant-c"}, 180 }, 181 Samples: []prompb.Sample{ 182 {Timestamp: 2, Value: 2.2}, 183 {Timestamp: 3, Value: 2.3}, 184 {Timestamp: 4, Value: 2.4}, 185 }, 186 }, 187 } 188 189 result, err = qr.Query(&prompb.Query{ 190 Matchers: []*prompb.LabelMatcher{ 191 { 192 Type: prompb.LabelMatcher_EQ, 193 Name: model.MetricNameLabelName, 194 Value: "secondMetric", 195 }, 196 }, 197 StartTimestampMs: 2, 198 EndTimestampMs: 4, 199 }) 200 require.NoError(t, err) 201 202 // Verifying result. 203 verifyResults(t, expectedResult, result) 204 }) 205} 206 207func TestMultiTenancyWithValidTenants(t *testing.T) { 208 ts, tenants := generateSmallMultiTenantTimeseries() 209 withDB(t, *testDatabase, func(db *pgxpool.Pool, t testing.TB) { 210 // With valid tenants. 211 cfg := tenancy.NewSelectiveTenancyConfig(tenants[:2], false) // valid tenant-a & tenant-b. 212 mt, err := tenancy.NewAuthorizer(cfg) 213 require.NoError(t, err) 214 215 // Ingestion. 216 client, err := pgclient.NewClientWithPool(&pgclient.Config{}, 1, db, mt, false) 217 require.NoError(t, err) 218 defer client.Close() 219 220 wauth := mt.WriteAuthorizer() 221 // Ingest tenant-a. 222 request := newWriteRequestWithTs(copyMetrics(ts)) 223 err = wauth.Process(requestWithHeaderTenant(tenants[0]), request) 224 require.NoError(t, err) 225 _, _, err = client.Ingest(request) 226 require.NoError(t, err) 227 228 // Ingest tenant-b. 229 request = newWriteRequestWithTs(copyMetrics(ts)) 230 err = wauth.Process(requestWithHeaderTenant(tenants[1]), request) 231 require.NoError(t, err) 232 _, _, err = client.Ingest(request) 233 require.NoError(t, err) 234 require.NoError(t, err) 235 236 // Ingest tenant-c. 237 request = newWriteRequestWithTs(copyMetrics(ts)) 238 err = wauth.Process(requestWithHeaderTenant(tenants[2]), request) 239 require.Error(t, err) 240 require.Equal(t, err.Error(), "write-authorizer process: authorization error for tenant tenant-c: unauthorized or invalid tenant") 241 242 // Querying. 243 mCache := &cache.MetricNameCache{Metrics: clockcache.WithMax(cache.DefaultMetricCacheSize)} 244 lCache := clockcache.WithMax(100) 245 dbConn := pgxconn.NewPgxConn(db) 246 labelsReader := lreader.NewLabelsReader(dbConn, lCache) 247 qr := querier.NewQuerier(dbConn, mCache, labelsReader, nil, mt.ReadAuthorizer()) 248 249 // ----- query-test: querying a valid tenant (tenant-a) ----- 250 expectedResult := []prompb.TimeSeries{ 251 { 252 Labels: []prompb.Label{ 253 {Name: model.MetricNameLabelName, Value: "secondMetric"}, 254 {Name: "job", Value: "baz"}, 255 {Name: "ins", Value: "tag"}, 256 {Name: tenancy.TenantLabelKey, Value: "tenant-a"}, 257 }, 258 Samples: []prompb.Sample{ 259 {Timestamp: 2, Value: 2.2}, 260 {Timestamp: 3, Value: 2.3}, 261 {Timestamp: 4, Value: 2.4}, 262 }, 263 }, 264 } 265 result, err := qr.Query(&prompb.Query{ 266 Matchers: []*prompb.LabelMatcher{ 267 { 268 Type: prompb.LabelMatcher_EQ, 269 Name: model.MetricNameLabelName, 270 Value: "secondMetric", 271 }, 272 { 273 Type: prompb.LabelMatcher_RE, 274 Name: tenancy.TenantLabelKey, 275 Value: "tenant-a", 276 }, 277 }, 278 StartTimestampMs: 2, 279 EndTimestampMs: 4, 280 }) 281 require.NoError(t, err) 282 283 // Verifying result. 284 verifyResults(t, expectedResult, result) 285 286 // ----- query-test: querying an invalid tenant (tenant-c) ----- 287 expectedResult = []prompb.TimeSeries{} 288 289 result, err = qr.Query(&prompb.Query{ 290 Matchers: []*prompb.LabelMatcher{ 291 { 292 Type: prompb.LabelMatcher_EQ, 293 Name: model.MetricNameLabelName, 294 Value: "firstMetric", 295 }, 296 { 297 Type: prompb.LabelMatcher_RE, 298 Name: tenancy.TenantLabelKey, 299 Value: "tenant-c", 300 }, 301 }, 302 StartTimestampMs: 2, 303 EndTimestampMs: 4, 304 }) 305 require.NoError(t, err) 306 307 // Verifying result. 308 verifyResults(t, expectedResult, result) 309 310 // ----- query-test: querying across multiple tenants (tenant-a & tenant-b) ----- 311 expectedResult = []prompb.TimeSeries{ 312 { 313 Labels: []prompb.Label{ 314 {Name: model.MetricNameLabelName, Value: "secondMetric"}, 315 {Name: "job", Value: "baz"}, 316 {Name: "ins", Value: "tag"}, 317 {Name: tenancy.TenantLabelKey, Value: "tenant-a"}, 318 }, 319 Samples: []prompb.Sample{ 320 {Timestamp: 2, Value: 2.2}, 321 {Timestamp: 3, Value: 2.3}, 322 {Timestamp: 4, Value: 2.4}, 323 }, 324 }, 325 { 326 Labels: []prompb.Label{ 327 {Name: model.MetricNameLabelName, Value: "secondMetric"}, 328 {Name: "job", Value: "baz"}, 329 {Name: "ins", Value: "tag"}, 330 {Name: tenancy.TenantLabelKey, Value: "tenant-b"}, 331 }, 332 Samples: []prompb.Sample{ 333 {Timestamp: 2, Value: 2.2}, 334 {Timestamp: 3, Value: 2.3}, 335 {Timestamp: 4, Value: 2.4}, 336 }, 337 }, 338 } 339 340 result, err = qr.Query(&prompb.Query{ 341 Matchers: []*prompb.LabelMatcher{ 342 { 343 Type: prompb.LabelMatcher_EQ, 344 Name: model.MetricNameLabelName, 345 Value: "secondMetric", 346 }, 347 { 348 Type: prompb.LabelMatcher_RE, 349 Name: tenancy.TenantLabelKey, 350 Value: "tenant-a|tenant-b", 351 }, 352 }, 353 StartTimestampMs: 2, 354 EndTimestampMs: 4, 355 }) 356 require.NoError(t, err) 357 358 // Verifying result. 359 verifyResults(t, expectedResult, result) 360 361 // query-test: ingested by one org, and being queried by some other org, so no result should happen. 362 // 363 // eg: tenant-a and tenant-b is ingested. Now, a reader who is just authorized to read tenant-a, 364 // tries tenant-b should get empty result. 365 cfg = tenancy.NewSelectiveTenancyConfig(tenants[:1], false) // valid tenant-a only. 366 mt, err = tenancy.NewAuthorizer(cfg) 367 require.NoError(t, err) 368 369 labelsReader = lreader.NewLabelsReader(dbConn, lCache) 370 qr = querier.NewQuerier(dbConn, mCache, labelsReader, nil, mt.ReadAuthorizer()) 371 372 expectedResult = []prompb.TimeSeries{} 373 374 result, err = qr.Query(&prompb.Query{ 375 Matchers: []*prompb.LabelMatcher{ 376 { 377 Type: prompb.LabelMatcher_RE, 378 Name: tenancy.TenantLabelKey, 379 Value: "tenant-b", 380 }, 381 }, 382 StartTimestampMs: 2, 383 EndTimestampMs: 4, 384 }) 385 require.NoError(t, err) 386 387 // Verifying result. 388 verifyResults(t, expectedResult, result) 389 }) 390} 391 392func TestMultiTenancyWithValidTenantsAndNonTenantOps(t *testing.T) { 393 ts, tenants := generateSmallMultiTenantTimeseries() 394 withDB(t, *testDatabase, func(db *pgxpool.Pool, t testing.TB) { 395 // With valid tenants and non-tenant operations are allowed. 396 cfg := tenancy.NewSelectiveTenancyConfig(tenants[:2], true) // valid tenant-a & tenant-b. 397 mt, err := tenancy.NewAuthorizer(cfg) 398 require.NoError(t, err) 399 400 // Ingestion. 401 client, err := pgclient.NewClientWithPool(&pgclient.Config{}, 1, db, mt, false) 402 require.NoError(t, err) 403 defer client.Close() 404 405 wauth := mt.WriteAuthorizer() 406 // Ingest tenant-a. 407 request := newWriteRequestWithTs(copyMetrics(ts)) 408 err = wauth.Process(requestWithHeaderTenant(tenants[0]), request) 409 require.NoError(t, err) 410 _, _, err = client.Ingest(request) 411 require.NoError(t, err) 412 413 // Ingest tenant-b. 414 request = newWriteRequestWithTs(copyMetrics(ts)) 415 err = wauth.Process(requestWithHeaderTenant(tenants[1]), request) 416 require.NoError(t, err) 417 _, _, err = client.Ingest(request) 418 require.NoError(t, err) 419 420 ts = []prompb.TimeSeries{ 421 { 422 Labels: []prompb.Label{ 423 {Name: model.MetricNameLabelName, Value: "thirdMetric"}, 424 {Name: "foo", Value: "bar"}, 425 {Name: "common", Value: "tag"}, 426 }, 427 Samples: []prompb.Sample{ 428 {Timestamp: 1, Value: 0.1}, 429 {Timestamp: 2, Value: 0.2}, 430 {Timestamp: 3, Value: 0.3}, 431 {Timestamp: 4, Value: 0.4}, 432 {Timestamp: 5, Value: 0.5}, 433 }, 434 }, 435 } 436 // Ingest without tenants. 437 request = newWriteRequestWithTs(copyMetrics(ts)) 438 err = wauth.Process(&http.Request{}, request) // Ingest without tenants. 439 require.NoError(t, err) 440 _, _, err = client.Ingest(request) // Non-MT write. 441 require.NoError(t, err) 442 443 // Querying. 444 mCache := &cache.MetricNameCache{Metrics: clockcache.WithMax(cache.DefaultMetricCacheSize)} 445 lCache := clockcache.WithMax(100) 446 dbConn := pgxconn.NewPgxConn(db) 447 labelsReader := lreader.NewLabelsReader(dbConn, lCache) 448 qr := querier.NewQuerier(dbConn, mCache, labelsReader, nil, mt.ReadAuthorizer()) 449 450 // ----- query-test: querying a non-tenant ----- 451 expectedResult := []prompb.TimeSeries{ 452 { 453 Labels: []prompb.Label{ 454 {Name: model.MetricNameLabelName, Value: "thirdMetric"}, 455 {Name: "foo", Value: "bar"}, 456 {Name: "common", Value: "tag"}, 457 }, 458 Samples: []prompb.Sample{ 459 {Timestamp: 2, Value: 0.2}, 460 {Timestamp: 3, Value: 0.3}, 461 {Timestamp: 4, Value: 0.4}, 462 }, 463 }, 464 } 465 466 result, err := qr.Query(&prompb.Query{ 467 Matchers: []*prompb.LabelMatcher{ 468 { 469 Type: prompb.LabelMatcher_EQ, 470 Name: model.MetricNameLabelName, 471 Value: "thirdMetric", 472 }, 473 }, 474 StartTimestampMs: 2, 475 EndTimestampMs: 4, 476 }) 477 require.NoError(t, err) 478 479 // Verifying result. 480 verifyResults(t, expectedResult, result) 481 482 // ----- query-tests: querying across multiple tenants (tenant-a & tenant-b) ----- 483 expectedResult = []prompb.TimeSeries{ 484 { 485 Labels: []prompb.Label{ 486 {Name: model.MetricNameLabelName, Value: "secondMetric"}, 487 {Name: "job", Value: "baz"}, 488 {Name: "ins", Value: "tag"}, 489 {Name: tenancy.TenantLabelKey, Value: "tenant-a"}, 490 }, 491 Samples: []prompb.Sample{ 492 {Timestamp: 2, Value: 2.2}, 493 {Timestamp: 3, Value: 2.3}, 494 {Timestamp: 4, Value: 2.4}, 495 }, 496 }, 497 { 498 Labels: []prompb.Label{ 499 {Name: model.MetricNameLabelName, Value: "secondMetric"}, 500 {Name: "job", Value: "baz"}, 501 {Name: "ins", Value: "tag"}, 502 {Name: tenancy.TenantLabelKey, Value: "tenant-b"}, 503 }, 504 Samples: []prompb.Sample{ 505 {Timestamp: 2, Value: 2.2}, 506 {Timestamp: 3, Value: 2.3}, 507 {Timestamp: 4, Value: 2.4}, 508 }, 509 }, 510 } 511 512 result, err = qr.Query(&prompb.Query{ 513 Matchers: []*prompb.LabelMatcher{ 514 { 515 Type: prompb.LabelMatcher_EQ, 516 Name: "job", 517 Value: "baz", 518 }, 519 }, 520 StartTimestampMs: 2, 521 EndTimestampMs: 4, 522 }) 523 require.NoError(t, err) 524 525 // Verifying result. 526 verifyResults(t, expectedResult, result) 527 528 // query-test: ingested by one org with NonMT true, and being queried by some other org with NonMT false, 529 // so result should contain MT writes of valid tenants by the later org. 530 cfg = tenancy.NewSelectiveTenancyConfig(tenants[:2], false) // valid tenant-a & tenant-b. 531 mt, err = tenancy.NewAuthorizer(cfg) 532 require.NoError(t, err) 533 534 labelsReader = lreader.NewLabelsReader(dbConn, lCache) 535 qr = querier.NewQuerier(dbConn, mCache, labelsReader, nil, mt.ReadAuthorizer()) 536 537 expectedResult = []prompb.TimeSeries{ 538 { 539 Labels: []prompb.Label{ 540 {Name: model.MetricNameLabelName, Value: "secondMetric"}, 541 {Name: tenancy.TenantLabelKey, Value: "tenant-a"}, 542 {Name: "job", Value: "baz"}, 543 {Name: "ins", Value: "tag"}, 544 }, 545 Samples: []prompb.Sample{ 546 {Timestamp: 2, Value: 2.2}, 547 {Timestamp: 3, Value: 2.3}, 548 {Timestamp: 4, Value: 2.4}, 549 }, 550 }, 551 { 552 Labels: []prompb.Label{ 553 {Name: model.MetricNameLabelName, Value: "secondMetric"}, 554 {Name: tenancy.TenantLabelKey, Value: "tenant-b"}, 555 {Name: "job", Value: "baz"}, 556 {Name: "ins", Value: "tag"}, 557 }, 558 Samples: []prompb.Sample{ 559 {Timestamp: 2, Value: 2.2}, 560 {Timestamp: 3, Value: 2.3}, 561 {Timestamp: 4, Value: 2.4}, 562 }, 563 }, 564 } 565 566 result, err = qr.Query(&prompb.Query{ 567 Matchers: []*prompb.LabelMatcher{ 568 { 569 Type: prompb.LabelMatcher_EQ, 570 Name: model.MetricNameLabelName, 571 Value: "secondMetric", 572 }, 573 }, 574 StartTimestampMs: 2, 575 EndTimestampMs: 4, 576 }) 577 require.NoError(t, err) 578 579 // Verifying result. 580 verifyResults(t, expectedResult, result) 581 582 expectedResult = []prompb.TimeSeries{} 583 result, err = qr.Query(&prompb.Query{ 584 Matchers: []*prompb.LabelMatcher{ 585 { 586 Type: prompb.LabelMatcher_EQ, 587 Name: model.MetricNameLabelName, 588 Value: "thirdMetric", 589 }, 590 }, 591 StartTimestampMs: 2, 592 EndTimestampMs: 4, 593 }) 594 require.NoError(t, err) 595 596 // Verifying result. 597 verifyResults(t, expectedResult, result) 598 }) 599} 600 601func TestMultiTenancyWithValidTenantsAsLabels(t *testing.T) { 602 ts, tenants := generateSmallMultiTenantTimeseries() 603 withDB(t, *testDatabase, func(db *pgxpool.Pool, t testing.TB) { 604 // With valid tenants. 605 cfg := tenancy.NewSelectiveTenancyConfig(tenants[:2], false) // valid tenant-a & tenant-b. 606 mt, err := tenancy.NewAuthorizer(cfg) 607 require.NoError(t, err) 608 609 // Ingestion. 610 client, err := pgclient.NewClientWithPool(&pgclient.Config{}, 1, db, mt, false) 611 require.NoError(t, err) 612 defer client.Close() 613 614 wauth := mt.WriteAuthorizer() 615 // Ingest tenant-a. 616 request := newWriteRequestWithTs(applyTenantInLabels(tenants[0], copyMetrics(ts))) 617 err = wauth.Process(&http.Request{}, request) 618 require.NoError(t, err) 619 _, _, err = client.Ingest(request) 620 require.NoError(t, err) 621 622 // Ingest tenant-b. 623 request = newWriteRequestWithTs(applyTenantInLabels(tenants[1], copyMetrics(ts))) 624 err = wauth.Process(&http.Request{}, request) 625 require.NoError(t, err) 626 _, _, err = client.Ingest(request) 627 require.NoError(t, err) 628 require.NoError(t, err) 629 630 // Ingest tenant-c. 631 request = newWriteRequestWithTs(applyTenantInLabels(tenants[2], copyMetrics(ts))) 632 err = wauth.Process(&http.Request{}, request) 633 require.Error(t, err) 634 require.Equal(t, err.Error(), "write-authorizer process: authorization error for tenant tenant-c: unauthorized or invalid tenant") 635 636 // Querying. 637 mCache := &cache.MetricNameCache{Metrics: clockcache.WithMax(cache.DefaultMetricCacheSize)} 638 lCache := clockcache.WithMax(100) 639 dbConn := pgxconn.NewPgxConn(db) 640 labelsReader := lreader.NewLabelsReader(dbConn, lCache) 641 qr := querier.NewQuerier(dbConn, mCache, labelsReader, nil, mt.ReadAuthorizer()) 642 643 // ----- query-test: querying a single tenant (tenant-b) ----- 644 expectedResult := []prompb.TimeSeries{ 645 { 646 Labels: []prompb.Label{ 647 {Name: model.MetricNameLabelName, Value: "secondMetric"}, 648 {Name: tenancy.TenantLabelKey, Value: "tenant-b"}, 649 {Name: "job", Value: "baz"}, 650 {Name: "ins", Value: "tag"}, 651 }, 652 Samples: []prompb.Sample{ 653 {Timestamp: 2, Value: 2.2}, 654 {Timestamp: 3, Value: 2.3}, 655 {Timestamp: 4, Value: 2.4}, 656 }, 657 }, 658 } 659 660 result, err := qr.Query(&prompb.Query{ 661 Matchers: []*prompb.LabelMatcher{ 662 { 663 Type: prompb.LabelMatcher_EQ, 664 Name: model.MetricNameLabelName, 665 Value: "secondMetric", 666 }, 667 { 668 Type: prompb.LabelMatcher_RE, 669 Name: tenancy.TenantLabelKey, 670 Value: "tenant-b", 671 }, 672 }, 673 StartTimestampMs: 2, 674 EndTimestampMs: 4, 675 }) 676 require.NoError(t, err) 677 678 // Verifying result. 679 verifyResults(t, expectedResult, result) 680 681 // ----- query-test: querying across multiple tenants (tenant-a & tenant-b) ----- 682 expectedResult = []prompb.TimeSeries{ 683 { 684 Labels: []prompb.Label{ 685 {Name: model.MetricNameLabelName, Value: "secondMetric"}, 686 {Name: "job", Value: "baz"}, 687 {Name: "ins", Value: "tag"}, 688 {Name: tenancy.TenantLabelKey, Value: "tenant-a"}, 689 }, 690 Samples: []prompb.Sample{ 691 {Timestamp: 2, Value: 2.2}, 692 {Timestamp: 3, Value: 2.3}, 693 {Timestamp: 4, Value: 2.4}, 694 }, 695 }, 696 { 697 Labels: []prompb.Label{ 698 {Name: model.MetricNameLabelName, Value: "secondMetric"}, 699 {Name: "job", Value: "baz"}, 700 {Name: "ins", Value: "tag"}, 701 {Name: tenancy.TenantLabelKey, Value: "tenant-b"}, 702 }, 703 Samples: []prompb.Sample{ 704 {Timestamp: 2, Value: 2.2}, 705 {Timestamp: 3, Value: 2.3}, 706 {Timestamp: 4, Value: 2.4}, 707 }, 708 }, 709 } 710 711 result, err = qr.Query(&prompb.Query{ 712 Matchers: []*prompb.LabelMatcher{ 713 { 714 Type: prompb.LabelMatcher_EQ, 715 Name: model.MetricNameLabelName, 716 Value: "secondMetric", 717 }, 718 { 719 Type: prompb.LabelMatcher_RE, 720 Name: tenancy.TenantLabelKey, 721 Value: "tenant-a|tenant-b", 722 }, 723 }, 724 StartTimestampMs: 2, 725 EndTimestampMs: 4, 726 }) 727 require.NoError(t, err) 728 729 // Verifying result. 730 verifyResults(t, expectedResult, result) 731 }) 732} 733 734func verifyResults(t testing.TB, expectedResult []prompb.TimeSeries, receivedResult []*prompb.TimeSeries) { 735 if len(receivedResult) != len(expectedResult) { 736 require.Fail(t, fmt.Sprintf("lengths of result (%d) and expectedResult (%d) does not match", len(receivedResult), len(expectedResult))) 737 } 738 for k := 0; k < len(receivedResult); k++ { 739 sort.SliceStable(receivedResult[k].Labels, func(i, j int) bool { 740 return receivedResult[k].Labels[i].Name < receivedResult[k].Labels[j].Name 741 }) 742 sort.SliceStable(expectedResult[k].Labels, func(i, j int) bool { 743 return expectedResult[k].Labels[i].Name < expectedResult[k].Labels[j].Name 744 }) 745 require.Equal(t, expectedResult[k].Labels, receivedResult[k].Labels) 746 require.Equal(t, expectedResult[k].Samples, receivedResult[k].Samples) 747 } 748} 749 750func requestWithHeaderTenant(tenant string) *http.Request { 751 header := make(http.Header) 752 header.Add("TENANT", tenant) 753 return &http.Request{Header: header} 754} 755 756func applyTenantInLabels(tenant string, ts []prompb.TimeSeries) []prompb.TimeSeries { 757 for i := 0; i < len(ts); i++ { 758 ts[i].Labels = append(ts[i].Labels, prompb.Label{Name: tenancy.TenantLabelKey, Value: tenant}) 759 } 760 return ts 761} 762