1# frozen_string_literal: true
2
3# Keep separate directories for separate processes
4def prometheus_default_multiproc_dir
5  return unless Rails.env.development? || Rails.env.test?
6
7  if Gitlab::Runtime.sidekiq?
8    Rails.root.join('tmp/prometheus_multiproc_dir/sidekiq')
9  elsif Gitlab::Runtime.puma?
10    Rails.root.join('tmp/prometheus_multiproc_dir/puma')
11  else
12    Rails.root.join('tmp/prometheus_multiproc_dir')
13  end
14end
15
16::Prometheus::Client.configure do |config|
17  config.logger = Gitlab::AppLogger
18
19  config.initial_mmap_file_size = 4 * 1024
20
21  config.multiprocess_files_dir = ENV['prometheus_multiproc_dir'] || prometheus_default_multiproc_dir
22
23  config.pid_provider = ::Prometheus::PidProvider.method(:worker_id)
24end
25
26Gitlab::Application.configure do |config|
27  # 0 should be Sentry to catch errors in this middleware
28  config.middleware.insert_after(Labkit::Middleware::Rack, Gitlab::Metrics::RequestsRackMiddleware)
29end
30
31if Gitlab::Runtime.sidekiq? && (!ENV['SIDEKIQ_WORKER_ID'] || ENV['SIDEKIQ_WORKER_ID'] == '0')
32  # The single worker outside of a sidekiq-cluster, or the first worker (sidekiq_0)
33  # in a cluster of processes, is responsible for serving health checks.
34  #
35  # Do not clean the metrics directory here - the supervisor script should
36  # have already taken care of that.
37  Sidekiq.configure_server do |config|
38    config.on(:startup) do
39      # In https://gitlab.com/gitlab-org/gitlab/-/issues/345804 we are looking to
40      # only serve health-checks from a worker process; for backwards compatibility
41      # we still go through the metrics exporter server, but start to configure it
42      # with the new settings keys.
43      exporter_settings = Settings.monitoring.sidekiq_health_checks
44      Gitlab::Metrics::Exporter::SidekiqExporter.instance(exporter_settings).start
45    end
46  end
47end
48
49if !Rails.env.test? && Gitlab::Metrics.prometheus_metrics_enabled?
50  # When running Puma in a Single mode, `on_master_start` and `on_worker_start` are the same.
51  # Thus, we order these events to run `reinitialize_on_pid_change` with `force: true` first.
52  Gitlab::Cluster::LifecycleEvents.on_master_start do
53    ::Prometheus::Client.reinitialize_on_pid_change(force: true)
54
55    if Gitlab::Runtime.puma?
56      Gitlab::Metrics::Samplers::PumaSampler.instance.start
57    end
58
59    Gitlab::Metrics.gauge(:deployments, 'GitLab Version', {}, :max).set({ version: Gitlab::VERSION, revision: Gitlab.revision }, 1)
60
61    if Gitlab::Runtime.web_server?
62      Gitlab::Metrics::RequestsRackMiddleware.initialize_metrics
63    end
64
65    Gitlab::Ci::Parsers.instrument!
66  rescue IOError => e
67    Gitlab::ErrorTracking.track_exception(e)
68    Gitlab::Metrics.error_detected!
69  end
70
71  Gitlab::Cluster::LifecycleEvents.on_worker_start do
72    defined?(::Prometheus::Client.reinitialize_on_pid_change) && ::Prometheus::Client.reinitialize_on_pid_change
73
74    Gitlab::Metrics::Samplers::RubySampler.initialize_instance.start
75    Gitlab::Metrics::Samplers::DatabaseSampler.initialize_instance.start
76    Gitlab::Metrics::Samplers::ThreadsSampler.initialize_instance.start
77
78    if Gitlab::Runtime.web_server?
79      Gitlab::Metrics::Samplers::ActionCableSampler.instance.start
80    end
81
82    if Gitlab.ee? && Gitlab::Runtime.sidekiq?
83      Gitlab::Metrics::Samplers::GlobalSearchSampler.instance.start
84    end
85
86    Gitlab::Ci::Parsers.instrument!
87  rescue IOError => e
88    Gitlab::ErrorTracking.track_exception(e)
89    Gitlab::Metrics.error_detected!
90  end
91end
92
93if Gitlab::Runtime.web_server?
94  Gitlab::Cluster::LifecycleEvents.on_master_start do
95    Gitlab::Metrics::Exporter::WebExporter.instance.start
96  end
97
98  # DEPRECATED: TO BE REMOVED
99  # This is needed to implement blackout period of `web_exporter`
100  # https://gitlab.com/gitlab-org/gitlab/issues/35343#note_238479057
101  Gitlab::Cluster::LifecycleEvents.on_before_blackout_period do
102    Gitlab::Metrics::Exporter::WebExporter.instance.mark_as_not_running!
103  end
104
105  Gitlab::Cluster::LifecycleEvents.on_before_graceful_shutdown do
106    # We need to ensure that before we re-exec or shutdown server
107    # we do stop the exporter
108    Gitlab::Metrics::Exporter::WebExporter.instance.stop
109  end
110
111  Gitlab::Cluster::LifecycleEvents.on_before_master_restart do
112    # We need to ensure that before we re-exec server
113    # we do stop the exporter
114    #
115    # We do it again, for being extra safe,
116    # but it should not be needed
117    Gitlab::Metrics::Exporter::WebExporter.instance.stop
118  end
119
120  Gitlab::Cluster::LifecycleEvents.on_worker_start do
121    # The `#close_on_exec=` takes effect only on `execve`
122    # but this does not happen for Ruby fork
123    #
124    # This does stop server, as it is running on master.
125    Gitlab::Metrics::Exporter::WebExporter.instance.stop
126  end
127end
128