1# frozen_string_literal: true 2 3# An InternalId is a strictly monotone sequence of integers 4# generated for a given scope and usage. 5# 6# The monotone sequence may be broken if an ID is explicitly provided 7# to `#track_greatest`. 8# 9# For example, issues use their project to scope internal ids: 10# In that sense, scope is "project" and usage is "issues". 11# Generated internal ids for an issue are unique per project. 12# 13# See InternalId#usage enum for available usages. 14# 15# In order to leverage InternalId for other usages, the idea is to 16# * Add `usage` value to enum 17# * (Optionally) add columns to `internal_ids` if needed for scope. 18class InternalId < ApplicationRecord 19 extend Gitlab::Utils::StrongMemoize 20 21 belongs_to :project 22 belongs_to :namespace 23 24 enum usage: Enums::InternalId.usage_resources 25 26 validates :usage, presence: true 27 28 scope :filter_by, -> (scope, usage) do 29 where(**scope, usage: usage) 30 end 31 32 class << self 33 def track_greatest(subject, scope, usage, new_value, init) 34 build_generator(subject, scope, usage, init).track_greatest(new_value) 35 end 36 37 def generate_next(subject, scope, usage, init) 38 build_generator(subject, scope, usage, init).generate 39 end 40 41 def reset(subject, scope, usage, value) 42 build_generator(subject, scope, usage).reset(value) 43 end 44 45 # Flushing records is generally safe in a sense that those 46 # records are going to be re-created when needed. 47 # 48 # A filter condition has to be provided to not accidentally flush 49 # records for all projects. 50 def flush_records!(filter) 51 raise ArgumentError, "filter cannot be empty" if filter.blank? 52 53 where(filter).delete_all 54 end 55 56 def internal_id_transactions_increment(operation:, usage:) 57 self.internal_id_transactions_total.increment( 58 operation: operation, 59 usage: usage.to_s, 60 in_transaction: ActiveRecord::Base.connection.transaction_open?.to_s # rubocop: disable Database/MultipleDatabases 61 ) 62 end 63 64 def internal_id_transactions_total 65 strong_memoize(:internal_id_transactions_total) do 66 name = :gitlab_internal_id_transactions_total 67 comment = 'Counts all the internal ids happening within transaction' 68 69 Gitlab::Metrics.counter(name, comment) 70 end 71 end 72 73 private 74 75 def build_generator(subject, scope, usage, init = nil) 76 ImplicitlyLockingInternalIdGenerator.new(subject, scope, usage, init) 77 end 78 end 79 80 class ImplicitlyLockingInternalIdGenerator 81 # Generate next internal id for a given scope and usage. 82 # 83 # For currently supported usages, see #usage enum. 84 # 85 # The method implements a locking scheme that has the following properties: 86 # 1) Generated sequence of internal ids is unique per (scope and usage) 87 # 2) The method is thread-safe and may be used in concurrent threads/processes. 88 # 3) The generated sequence is gapless. 89 # 4) In the absence of a record in the internal_ids table, one will be created 90 # and last_value will be calculated on the fly. 91 # 92 # subject: The instance or class we're generating an internal id for. 93 # scope: Attributes that define the scope for id generation. 94 # Valid keys are `project/project_id` and `namespace/namespace_id`. 95 # usage: Symbol to define the usage of the internal id, see InternalId.usages 96 # init: Proc that accepts the subject and the scope and returns Integer|NilClass 97 attr_reader :subject, :scope, :scope_attrs, :usage, :init 98 99 RecordAlreadyExists = Class.new(StandardError) 100 101 def initialize(subject, scope, usage, init = nil) 102 @subject = subject 103 @scope = scope 104 @usage = usage 105 @init = init 106 107 raise ArgumentError, 'Scope is not well-defined, need at least one column for scope (given: 0)' if scope.empty? 108 109 unless InternalId.usages.has_key?(usage.to_s) 110 raise ArgumentError, "Usage '#{usage}' is unknown. Supported values are #{InternalId.usages.keys} from InternalId.usages" 111 end 112 end 113 114 # Generates next internal id and returns it 115 # init: Block that gets called to initialize InternalId record if not present 116 # Make sure to not throw exceptions in the absence of records (if this is expected). 117 def generate 118 InternalId.internal_id_transactions_increment(operation: :generate, usage: usage) 119 120 next_iid = update_record!(subject, scope, usage, arel_table[:last_value] + 1) 121 122 return next_iid if next_iid 123 124 create_record!(subject, scope, usage, initial_value(subject, scope) + 1) 125 rescue RecordAlreadyExists 126 retry 127 end 128 129 # Reset tries to rewind to `value-1`. This will only succeed, 130 # if `value` stored in database is equal to `last_value`. 131 # value: The expected last_value to decrement 132 def reset(value) 133 return false unless value 134 135 InternalId.internal_id_transactions_increment(operation: :reset, usage: usage) 136 137 iid = update_record!(subject, scope.merge(last_value: value), usage, arel_table[:last_value] - 1) 138 iid == value - 1 139 end 140 141 # Create a record in internal_ids if one does not yet exist 142 # and set its new_value if it is higher than the current last_value 143 def track_greatest(new_value) 144 InternalId.internal_id_transactions_increment(operation: :track_greatest, usage: usage) 145 146 function = Arel::Nodes::NamedFunction.new('GREATEST', [ 147 arel_table[:last_value], 148 new_value.to_i 149 ]) 150 151 next_iid = update_record!(subject, scope, usage, function) 152 return next_iid if next_iid 153 154 create_record!(subject, scope, usage, [initial_value(subject, scope), new_value].max) 155 rescue RecordAlreadyExists 156 retry 157 end 158 159 private 160 161 def update_record!(subject, scope, usage, new_value) 162 stmt = Arel::UpdateManager.new 163 stmt.table(arel_table) 164 stmt.set(arel_table[:last_value] => new_value) 165 stmt.wheres = InternalId.filter_by(scope, usage).arel.constraints 166 167 InternalId.connection.insert(stmt, 'Update InternalId', 'last_value') 168 end 169 170 def create_record!(subject, scope, usage, value) 171 scope[:project].save! if scope[:project] && !scope[:project].persisted? 172 scope[:namespace].save! if scope[:namespace] && !scope[:namespace].persisted? 173 174 attributes = { 175 project_id: scope[:project]&.id || scope[:project_id], 176 namespace_id: scope[:namespace]&.id || scope[:namespace_id], 177 usage: usage_value, 178 last_value: value 179 } 180 181 result = InternalId.insert(attributes) 182 183 raise RecordAlreadyExists if result.empty? 184 185 value 186 end 187 188 def arel_table 189 InternalId.arel_table 190 end 191 192 def initial_value(subject, scope) 193 raise ArgumentError, 'Cannot initialize without init!' unless init 194 195 # `init` computes the maximum based on actual records. We use the 196 # primary to make sure we have up to date results 197 Gitlab::Database::LoadBalancing::Session.current.use_primary do 198 instance = subject.is_a?(::Class) ? nil : subject 199 200 init.call(instance, scope) || 0 201 end 202 end 203 204 def usage_value 205 @usage_value ||= InternalId.usages[usage.to_s] 206 end 207 end 208end 209