Merge branch 'toggle-points-per-packet' into 'master'

Configuring of points per UDP packet

Related to https://gitlab.com/gitlab-com/operations/issues/195. This option would allow us to experiment with finding a good balance between points-per-packet and the UDP packet size.

cc @pcarranza 

See merge request !3816

CHANGELOG

@@ -3,6 +3,7 @@ Please view this file on the master branch, on stable branches it's out of date.
 v 8.8.0 (unreleased)
 
 v 8.7.0 (unreleased)
+  - The number of InfluxDB points stored per UDP packet can now be configured
   - Transactions for /internal/allowed now have an "action" tag set
   - Method instrumentation now uses Module#prepend instead of aliasing methods
   - Repository.clean_old_archives is now instrumented

app/controllers/admin/application_settings_controller.rb

@@ -93,6 +93,7 @@ class Admin::ApplicationSettingsController < Admin::ApplicationController
       :akismet_api_key,
       :email_author_in_body,
       :repository_checks_enabled,
+      :metrics_packet_size,
       restricted_visibility_levels: [],
       import_sources: []
     )

app/views/admin/application_settings/_form.html.haml

@@ -218,6 +218,13 @@
         .help-block
           The sampling interval in seconds. Sampled data includes memory usage,
           retained Ruby objects, file descriptors and so on.
+    .form-group
+      = f.label :metrics_packet_size, 'Metrics per packet', class: 'control-label col-sm-2'
+      .col-sm-10
+        = f.number_field :metrics_packet_size, class: 'form-control'
+        .help-block
+          The amount of points to store in a single UDP packet. More points
+          results in fewer but larger UDP packets being sent.
 
   %fieldset
     %legend Spam and Anti-bot Protection
@@ -286,7 +293,7 @@
             = f.check_box :repository_checks_enabled
             Enable Repository Checks
           .help-block
-            GitLab will periodically run 
+            GitLab will periodically run
             %a{ href: 'https://www.kernel.org/pub/software/scm/git/docs/git-fsck.html', target: 'blank' } 'git fsck'
             in all project and wiki repositories to look for silent disk corruption issues.
     .form-group
@@ -294,7 +301,7 @@
         = link_to 'Clear all repository checks', clear_repository_check_states_admin_application_settings_path, data: { confirm: 'This will clear repository check states for ALL projects in the database. This cannot be undone. Are you sure?' }, method: :put, class: "btn btn-sm btn-remove"
         .help-block
           If you got a lot of false alarms from repository checks you can choose to clear all repository check information from the database.
- 
+
 
   .form-actions
     = f.submit 'Save', class: 'btn btn-save'

db/migrate/20160419120017_add_metrics_packet_size.rb

+class AddMetricsPacketSize < ActiveRecord::Migration
+  def change
+    add_column :application_settings, :metrics_packet_size, :integer, default: 1
+  end
+end

db/schema.rb

@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20160415133440) do
+ActiveRecord::Schema.define(version: 20160419120017) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -78,6 +78,7 @@ ActiveRecord::Schema.define(version: 20160415133440) do
     t.boolean  "email_author_in_body",              default: false
     t.integer  "default_group_visibility"
     t.boolean  "repository_checks_enabled",         default: true
+    t.integer  "metrics_packet_size",               default: 1
     t.text     "shared_runners_text"
   end
 
@@ -717,37 +718,37 @@ ActiveRecord::Schema.define(version: 20160415133440) do
     t.datetime "created_at"
     t.datetime "updated_at"
     t.integer  "creator_id"
-    t.boolean  "issues_enabled",         default: true,     null: false
-    t.boolean  "wall_enabled",           default: true,     null: false
-    t.boolean  "merge_requests_enabled", default: true,     null: false
-    t.boolean  "wiki_enabled",           default: true,     null: false
+    t.boolean  "issues_enabled",               default: true,     null: false
+    t.boolean  "wall_enabled",                 default: true,     null: false
+    t.boolean  "merge_requests_enabled",       default: true,     null: false
+    t.boolean  "wiki_enabled",                 default: true,     null: false
     t.integer  "namespace_id"
-    t.string   "issues_tracker",         default: "gitlab", null: false
+    t.string   "issues_tracker",               default: "gitlab", null: false
     t.string   "issues_tracker_id"
-    t.boolean  "snippets_enabled",       default: true,     null: false
+    t.boolean  "snippets_enabled",             default: true,     null: false
     t.datetime "last_activity_at"
     t.string   "import_url"
-    t.integer  "visibility_level",       default: 0,        null: false
-    t.boolean  "archived",               default: false,    null: false
+    t.integer  "visibility_level",             default: 0,        null: false
+    t.boolean  "archived",                     default: false,    null: false
     t.string   "avatar"
     t.string   "import_status"
-    t.float    "repository_size",        default: 0.0
-    t.integer  "star_count",             default: 0,        null: false
+    t.float    "repository_size",              default: 0.0
+    t.integer  "star_count",                   default: 0,        null: false
     t.string   "import_type"
     t.string   "import_source"
-    t.integer  "commit_count",           default: 0
+    t.integer  "commit_count",                 default: 0
     t.text     "import_error"
     t.integer  "ci_id"
-    t.boolean  "builds_enabled",         default: true,     null: false
-    t.boolean  "shared_runners_enabled", default: true,     null: false
+    t.boolean  "builds_enabled",               default: true,     null: false
+    t.boolean  "shared_runners_enabled",       default: true,     null: false
     t.string   "runners_token"
     t.string   "build_coverage_regex"
-    t.boolean  "build_allow_git_fetch",  default: true,     null: false
-    t.integer  "build_timeout",          default: 3600,     null: false
-    t.boolean  "pending_delete",         default: false
-    t.boolean  "public_builds",          default: true,     null: false
+    t.boolean  "build_allow_git_fetch",        default: true,     null: false
+    t.integer  "build_timeout",                default: 3600,     null: false
+    t.boolean  "pending_delete",               default: false
+    t.boolean  "public_builds",                default: true,     null: false
     t.string   "main_language"
-    t.integer  "pushes_since_gc",        default: 0
+    t.integer  "pushes_since_gc",              default: 0
     t.boolean  "last_repository_check_failed"
     t.datetime "last_repository_check_at"
   end

lib/gitlab/metrics.rb

@@ -14,7 +14,8 @@ module Gitlab
         method_call_threshold: current_application_settings[:metrics_method_call_threshold],
         host:                  current_application_settings[:metrics_host],
         port:                  current_application_settings[:metrics_port],
-        sample_interval:       current_application_settings[:metrics_sample_interval] || 15
+        sample_interval:       current_application_settings[:metrics_sample_interval] || 15,
+        packet_size:           current_application_settings[:metrics_packet_size] || 1
       }
     end
 
@@ -41,9 +42,9 @@ module Gitlab
       prepared = prepare_metrics(metrics)
 
       pool.with do |connection|
-        prepared.each do |metric|
+        prepared.each_slice(settings[:packet_size]) do |slice|
           begin
-            connection.write_points([metric])
+            connection.write_points(slice)
           rescue StandardError
           end
         end