1# frozen_string_literal: true
2
3# An InternalId is a strictly monotone sequence of integers
4# generated for a given scope and usage.
5#
6# The monotone sequence may be broken if an ID is explicitly provided
7# to `#track_greatest`.
8#
9# For example, issues use their project to scope internal ids:
10# In that sense, scope is "project" and usage is "issues".
11# Generated internal ids for an issue are unique per project.
12#
13# See InternalId#usage enum for available usages.
14#
15# In order to leverage InternalId for other usages, the idea is to
16# * Add `usage` value to enum
17# * (Optionally) add columns to `internal_ids` if needed for scope.
18class InternalId < ApplicationRecord
19  extend Gitlab::Utils::StrongMemoize
20
21  belongs_to :project
22  belongs_to :namespace
23
24  enum usage: Enums::InternalId.usage_resources
25
26  validates :usage, presence: true
27
28  scope :filter_by, -> (scope, usage) do
29    where(**scope, usage: usage)
30  end
31
32  class << self
33    def track_greatest(subject, scope, usage, new_value, init)
34      build_generator(subject, scope, usage, init).track_greatest(new_value)
35    end
36
37    def generate_next(subject, scope, usage, init)
38      build_generator(subject, scope, usage, init).generate
39    end
40
41    def reset(subject, scope, usage, value)
42      build_generator(subject, scope, usage).reset(value)
43    end
44
45    # Flushing records is generally safe in a sense that those
46    # records are going to be re-created when needed.
47    #
48    # A filter condition has to be provided to not accidentally flush
49    # records for all projects.
50    def flush_records!(filter)
51      raise ArgumentError, "filter cannot be empty" if filter.blank?
52
53      where(filter).delete_all
54    end
55
56    def internal_id_transactions_increment(operation:, usage:)
57      self.internal_id_transactions_total.increment(
58        operation: operation,
59        usage: usage.to_s,
60        in_transaction: ActiveRecord::Base.connection.transaction_open?.to_s # rubocop: disable Database/MultipleDatabases
61      )
62    end
63
64    def internal_id_transactions_total
65      strong_memoize(:internal_id_transactions_total) do
66        name = :gitlab_internal_id_transactions_total
67        comment = 'Counts all the internal ids happening within transaction'
68
69        Gitlab::Metrics.counter(name, comment)
70      end
71    end
72
73    private
74
75    def build_generator(subject, scope, usage, init = nil)
76      ImplicitlyLockingInternalIdGenerator.new(subject, scope, usage, init)
77    end
78  end
79
80  class ImplicitlyLockingInternalIdGenerator
81    # Generate next internal id for a given scope and usage.
82    #
83    # For currently supported usages, see #usage enum.
84    #
85    # The method implements a locking scheme that has the following properties:
86    # 1) Generated sequence of internal ids is unique per (scope and usage)
87    # 2) The method is thread-safe and may be used in concurrent threads/processes.
88    # 3) The generated sequence is gapless.
89    # 4) In the absence of a record in the internal_ids table, one will be created
90    #    and last_value will be calculated on the fly.
91    #
92    # subject: The instance or class we're generating an internal id for.
93    # scope: Attributes that define the scope for id generation.
94    #        Valid keys are `project/project_id` and `namespace/namespace_id`.
95    # usage: Symbol to define the usage of the internal id, see InternalId.usages
96    # init: Proc that accepts the subject and the scope and returns Integer|NilClass
97    attr_reader :subject, :scope, :scope_attrs, :usage, :init
98
99    RecordAlreadyExists = Class.new(StandardError)
100
101    def initialize(subject, scope, usage, init = nil)
102      @subject = subject
103      @scope = scope
104      @usage = usage
105      @init = init
106
107      raise ArgumentError, 'Scope is not well-defined, need at least one column for scope (given: 0)' if scope.empty?
108
109      unless InternalId.usages.has_key?(usage.to_s)
110        raise ArgumentError, "Usage '#{usage}' is unknown. Supported values are #{InternalId.usages.keys} from InternalId.usages"
111      end
112    end
113
114    # Generates next internal id and returns it
115    # init: Block that gets called to initialize InternalId record if not present
116    #       Make sure to not throw exceptions in the absence of records (if this is expected).
117    def generate
118      InternalId.internal_id_transactions_increment(operation: :generate, usage: usage)
119
120      next_iid = update_record!(subject, scope, usage, arel_table[:last_value] + 1)
121
122      return next_iid if next_iid
123
124      create_record!(subject, scope, usage, initial_value(subject, scope) + 1)
125    rescue RecordAlreadyExists
126      retry
127    end
128
129    # Reset tries to rewind to `value-1`. This will only succeed,
130    # if `value` stored in database is equal to `last_value`.
131    # value: The expected last_value to decrement
132    def reset(value)
133      return false unless value
134
135      InternalId.internal_id_transactions_increment(operation: :reset, usage: usage)
136
137      iid = update_record!(subject, scope.merge(last_value: value), usage, arel_table[:last_value] - 1)
138      iid == value - 1
139    end
140
141    # Create a record in internal_ids if one does not yet exist
142    # and set its new_value if it is higher than the current last_value
143    def track_greatest(new_value)
144      InternalId.internal_id_transactions_increment(operation: :track_greatest, usage: usage)
145
146      function = Arel::Nodes::NamedFunction.new('GREATEST', [
147        arel_table[:last_value],
148        new_value.to_i
149      ])
150
151      next_iid = update_record!(subject, scope, usage, function)
152      return next_iid if next_iid
153
154      create_record!(subject, scope, usage, [initial_value(subject, scope), new_value].max)
155    rescue RecordAlreadyExists
156      retry
157    end
158
159    private
160
161    def update_record!(subject, scope, usage, new_value)
162      stmt = Arel::UpdateManager.new
163      stmt.table(arel_table)
164      stmt.set(arel_table[:last_value] => new_value)
165      stmt.wheres = InternalId.filter_by(scope, usage).arel.constraints
166
167      InternalId.connection.insert(stmt, 'Update InternalId', 'last_value')
168    end
169
170    def create_record!(subject, scope, usage, value)
171      scope[:project].save! if scope[:project] && !scope[:project].persisted?
172      scope[:namespace].save! if scope[:namespace] && !scope[:namespace].persisted?
173
174      attributes = {
175        project_id: scope[:project]&.id || scope[:project_id],
176        namespace_id: scope[:namespace]&.id || scope[:namespace_id],
177        usage: usage_value,
178        last_value: value
179      }
180
181      result = InternalId.insert(attributes)
182
183      raise RecordAlreadyExists if result.empty?
184
185      value
186    end
187
188    def arel_table
189      InternalId.arel_table
190    end
191
192    def initial_value(subject, scope)
193      raise ArgumentError, 'Cannot initialize without init!' unless init
194
195      # `init` computes the maximum based on actual records. We use the
196      # primary to make sure we have up to date results
197      Gitlab::Database::LoadBalancing::Session.current.use_primary do
198        instance = subject.is_a?(::Class) ? nil : subject
199
200        init.call(instance, scope) || 0
201      end
202    end
203
204    def usage_value
205      @usage_value ||= InternalId.usages[usage.to_s]
206    end
207  end
208end
209