1 //
2 // Copyright 2016 Pixar
3 //
4 // Licensed under the Apache License, Version 2.0 (the "Apache License")
5 // with the following modification; you may not use this file except in
6 // compliance with the Apache License and the following modification to it:
7 // Section 6. Trademarks. is deleted and replaced with:
8 //
9 // 6. Trademarks. This License does not grant permission to use the trade
10 // names, trademarks, service marks, or product names of the Licensor
11 // and its affiliates, except as required to comply with Section 4(c) of
12 // the License and to reproduce the content of the NOTICE file.
13 //
14 // You may obtain a copy of the Apache License at
15 //
16 // http://www.apache.org/licenses/LICENSE-2.0
17 //
18 // Unless required by applicable law or agreed to in writing, software
19 // distributed under the Apache License with the above modification is
20 // distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
21 // KIND, either express or implied. See the Apache License for the specific
22 // language governing permissions and limitations under the Apache License.
23 //
24 // threadLimits.cpp
25 //
26
27 #include "pxr/pxr.h"
28 #include "pxr/base/work/threadLimits.h"
29
30 #include "pxr/base/tf/envSetting.h"
31
32 #include <tbb/task_scheduler_init.h>
33 #include <tbb/task_arena.h>
34
35 #include <algorithm>
36 #include <atomic>
37
38 PXR_NAMESPACE_USING_DIRECTIVE
39
40 // The environment variable used to limit the number of threads the application
41 // may spawn:
42 // 0 - no change, i.e. defaults to maximum physical concurrency
43 // 1 - single-threaded mode
44 // positive n - limit to n threads
45 // negative n - limit to all but n machine cores (minimum 1).
46 //
47 // Note that the environment variable value always wins over any value passed to
48 // the API calls below. If PXR_WORK_THREAD_LIMIT is set to a non-zero value, the
49 // concurrency limit cannot be changed at runtime.
50 //
51 TF_DEFINE_ENV_SETTING(
52 PXR_WORK_THREAD_LIMIT, 0,
53 "Limits the number of threads the application may spawn. 0 (default) "
54 "allows for maximum concurrency as determined by the number of physical "
55 "cores, or the process's affinity mask, whichever is smaller. Note that "
56 "the environment variable (if set to a non-zero value) will override any "
57 "value passed to Work thread-limiting API calls.");
58
59 PXR_NAMESPACE_OPEN_SCOPE
60
61 // We create a task_scheduler_init instance at static initialization time if
62 // PXR_WORK_THREAD_LIMIT is set to a nonzero value. Otherwise this stays NULL.
63 static tbb::task_scheduler_init *_tbbTaskSchedInit;
64
65 unsigned
WorkGetPhysicalConcurrencyLimit()66 WorkGetPhysicalConcurrencyLimit()
67 {
68 // Use TBB here, since it pays attention to the affinity mask on Linux and
69 // Windows.
70 return tbb::task_scheduler_init::default_num_threads();
71 }
72
73 // This function always returns an actual thread count >= 1.
74 static unsigned
Work_NormalizeThreadCount(const int n)75 Work_NormalizeThreadCount(const int n)
76 {
77 // Zero means "no change", and n >= 1 means exactly n threads, so simply
78 // pass those values through unchanged.
79 // For negative integers, subtract the absolute value from the total number
80 // of available cores (denoting all but n cores). If n == number of cores,
81 // clamp to 1 to set single-threaded mode.
82 return n >= 0 ? n : std::max<int>(1, n + WorkGetPhysicalConcurrencyLimit());
83 }
84
85 // Returns the normalized thread limit value from the environment setting. Note
86 // that 0 means "no change", i.e. the environment setting does not apply.
87 static unsigned
Work_GetConcurrencyLimitSetting()88 Work_GetConcurrencyLimitSetting()
89 {
90 return Work_NormalizeThreadCount(TfGetEnvSetting(PXR_WORK_THREAD_LIMIT));
91 }
92
93 // Overrides weakValue with strongValue if strongValue is non-zero, and returns
94 // the resulting thread limit.
95 static unsigned
Work_OverrideConcurrencyLimit(unsigned weakValue,unsigned strongValue)96 Work_OverrideConcurrencyLimit(unsigned weakValue, unsigned strongValue)
97 {
98 // If the new limit is 0, i.e. "no change", simply pass the weakValue
99 // through unchanged. Otherwise, the new value wins.
100 return strongValue ? strongValue : weakValue;
101 }
102
103 static void
Work_InitializeThreading()104 Work_InitializeThreading()
105 {
106 // Get the thread limit from the environment setting. Note that this value
107 // can be 0, i.e. the environment setting does not apply.
108 const unsigned settingVal = Work_GetConcurrencyLimitSetting();
109
110 // Threading is initialized with maximum physical concurrency.
111 const unsigned physicalLimit = WorkGetPhysicalConcurrencyLimit();
112
113 // To assign the thread limit, override the initial limit with the
114 // environment setting. The environment setting always wins over the initial
115 // limit, unless it has been set to 0 (default). Semantically, 0 means
116 // "no change".
117 unsigned threadLimit =
118 Work_OverrideConcurrencyLimit(physicalLimit, settingVal);
119
120 // Only eagerly grab TBB if the PXR_WORK_THREAD_LIMIT setting was set to
121 // some non-zero value. Otherwise, the scheduler will be default initialized
122 // with maximum physical concurrency, or will be left untouched if
123 // previously initialized by the hosting environment (e.g. if we are running
124 // as a plugin to another application.)
125 if (settingVal) {
126 _tbbTaskSchedInit = new tbb::task_scheduler_init(threadLimit);
127 }
128 }
129 static int _forceInitialization = (Work_InitializeThreading(), 0);
130
131 void
WorkSetConcurrencyLimit(unsigned n)132 WorkSetConcurrencyLimit(unsigned n)
133 {
134 // We only assign a new concurrency limit if n is non-zero, since 0 means
135 // "no change". Note that we need to re-initialize the TBB
136 // task_scheduler_init instance in either case, because if the client
137 // explicitly requests a concurrency limit through this library, we need to
138 // attempt to take control of the TBB scheduler if we can, i.e. if the host
139 // environment has not already done so.
140 unsigned threadLimit = 0;
141 if (n) {
142 // Get the thread limit from the environment setting. Note this value
143 // may be 0 (default).
144 const unsigned settingVal = Work_GetConcurrencyLimitSetting();
145
146 // Override n with the environment setting. This will make sure that the
147 // setting always wins over the specified value n, but only if the
148 // setting has been set to a non-zero value.
149 threadLimit = Work_OverrideConcurrencyLimit(n, settingVal);
150 }
151 else {
152 // Use the current thread limit.
153 threadLimit = WorkGetConcurrencyLimit();
154 }
155
156 // Note that we need to do some performance testing and decide if it's
157 // better here to simply delete the task_scheduler_init object instead
158 // of re-initializing it. If we decide that it's better to re-initialize
159 // it, then we have to make sure that when this library is opened in
160 // an application (e.g., Maya) that already has initialized its own
161 // task_scheduler_init object, that the limits of those are respected.
162 // According to the documentation that should be the case, but we should
163 // make sure. If we do decide to delete it, we have to make sure to
164 // note that it has already been initialized.
165 if (_tbbTaskSchedInit) {
166 _tbbTaskSchedInit->terminate();
167 _tbbTaskSchedInit->initialize(threadLimit);
168 } else {
169 _tbbTaskSchedInit = new tbb::task_scheduler_init(threadLimit);
170 }
171 }
172
173 void
WorkSetMaximumConcurrencyLimit()174 WorkSetMaximumConcurrencyLimit()
175 {
176 WorkSetConcurrencyLimit(WorkGetPhysicalConcurrencyLimit());
177 }
178
179 void
WorkSetConcurrencyLimitArgument(int n)180 WorkSetConcurrencyLimitArgument(int n)
181 {
182 WorkSetConcurrencyLimit(Work_NormalizeThreadCount(n));
183 }
184
185 unsigned
WorkGetConcurrencyLimit()186 WorkGetConcurrencyLimit()
187 {
188 return tbb::this_task_arena::max_concurrency();
189 }
190
191 bool
WorkHasConcurrency()192 WorkHasConcurrency()
193 {
194 return WorkGetConcurrencyLimit() > 1;
195 }
196
197 PXR_NAMESPACE_CLOSE_SCOPE
198