DataDog · gmmeyer · Aug 12, 2016 · Aug 3, 2016
@@ -41,6 +41,7 @@ install:
 script:
   - bundle exec rake lint
   - rake ci:run[default]
+  - rake ci:run[mysql]
   - bundle exec rake requirements
 
 after_script:

@@ -24,7 +24,7 @@ dependencies:
         - echo "$HOME/dd-agent/" > ~/virtualenvs/venv-2.7.3/lib/python2.7/site-packages/datadog-agent.pth
         - pip install pip --upgrade
         - pip install docker-compose
-        - pip install pylint 
+        - pip install pylint
         - if [ -e ~/$CIRCLE_PROJECT_REPONAME/requirements.txt ]; then pip install -r ~/$CIRCLE_PROJECT_REPONAME/requirements.txt; fi
         - if [ -e ~/$CIRCLE_PROJECT_REPONAME/requirements-test.txt ]; then pip install -r ~/$CIRCLE_PROJECT_REPONAME/requirements-test.txt; fi
         - if [ -e ~/$CIRCLE_PROJECT_REPONAME/requirements-opt.txt ]; then pip install -r ~/$CIRCLE_PROJECT_REPONAME/requirements-opt.txt; fi
@@ -35,6 +35,7 @@ test:
     override:
         - bundle exec rake lint
         - rake ci:run[default]
+        - rake ci:run[mysql]
         - bundle exec rake requirements
     post:
         - if [[ $(docker ps -a -q) ]]; then docker stop $(docker ps -a -q); fi
@@ -11,5 +11,55 @@ instances:
     #   - optional_tag1
     #   - optional_tag2
     # options:               # Optional
-    #   replication: 0
-    #   galera_cluster: 1
+    #   replication: false
+    #   galera_cluster: false
+    #   extra_status_metrics: true
+    #   extra_innodb_metrics: true
+    #   extra_performance_metrics: true
+    #   schema_size_metrics: false
+    #   disable_innodb_metrics: false
+    #
+    #     NOTE: disable_innodb_metrics should only be used by users with older (unsupported) versions of
+    #           MySQL who do not run/have innodb engine support and may experiment issue otherwise.
+    #           Should this flag be enabled you will only receive a small subset of metrics.
+    #
+    #     NOTE: extra_performance_metrics will only be reported if `performance_schema` is enabled
+    #           in the MySQL instance and if the version for that instance is >= 5.6.0
+    #
+    #           extra_performance_metrics and schema_size_metrics will run two heavier queries
+    #           against your DB to compute the relevant metrics for all your existing schemas.
+    #           Due to the nature of these calls, if you have a high number of tables and/or schemas,
+    #           you may be subject to some negative impact in the performance of your DB.
+    #           Please bear that in mind when enabling them.
+    #           Metrics provided by the options:
+    #                     - mysql.info.schema.size (per schame)
+    #                     - mysql.performance.query_run_time.avg (per schema)
+    #                     - mysql.performance.digest_95th_percentile.avg_us
+    #
+    #           With the addition of new metrics to the MySQL catalog starting with agent >=5.7.0, because
+    #           we query additional schemas to get this full set of metrics. Some of these require the user
+    #           defined for the instance to have PROCESS and SELECT privileges. Please take a look at the
+    #           MySQL integration tile in the Datadog WebUI for further instructions.
+    #
+    # ssl:               # Optional
+    #   key: /path/to/my/key.file
+    #   cert: /path/to/my/cert.file
+    #   ca: /path/to/my/ca.file
+
+    # queries:               # Optional
+    #  - # Sample Custom metric
+    #    query: SELECT TIMESTAMPDIFF(second,MAX(create_time),NOW()) as last_accessed FROM requests
+    #    metric: app.seconds_since_last_request
+    #    tags:               # Optional - only applied to this custom metric query, will not affect default mysql metrics
+    #        - custom_tag1
+    #        - custom_tag2
+    #    type: gauge
+    #    field: last_accessed
+    #  - # Sample Custom metric
+    #    query: SELECT TIMESTAMPDIFF(second,MAX(create_time),NOW()) as last_user FROM users
+    #    metric: app.seconds_since_new_user
+    #    tags:               # Optional - only applied to this custom metric query, will not affect default mysql metrics
+    #        - custom_tag1
+    #        - custom_tag2
+    #    type: gauge
+    #    field: last_user
@@ -0,0 +1,59 @@
+require 'ci/common'
+
+def mysql_version
+  ENV['FLAVOR_VERSION'] || '2.4.12'
+end
+
+def mysql_rootdir
+  "#{ENV['INTEGRATIONS_DIR']}/mysql_#{mysql_version}"
+end
+
+namespace :ci do
+  namespace :mysql do |flavor|
+    task before_install: ['ci:common:before_install']
+
+    task install: ['ci:common:install'] do
+      use_venv = in_venv
+      install_requirements('mysql/requirements.txt',
+                           "--cache-dir #{ENV['PIP_CACHE']}",
+                           "#{ENV['VOLATILE_DIR']}/ci.log", use_venv)
+      sh %(bash mysql/ci/start-docker.sh)
+    end
+
+    task before_script: ['ci:common:before_script']
+
+    task script: ['ci:common:script'] do |_, attr|
+      this_provides = [
+        'mysql'
+      ]
+      Rake::Task['ci:common:run_tests'].invoke(this_provides)
+    end
+
+    task before_cache: ['ci:common:before_cache']
+
+    task cleanup: ['ci:common:cleanup'] do
+      sh %(bash mysql/ci/stop-docker.sh)
+    end
+
+    task :execute do
+      exception = nil
+      begin
+        %w(before_install install before_script).each do |u|
+          Rake::Task["#{flavor.scope.path}:#{u}"].invoke
+        end
+        Rake::Task["#{flavor.scope.path}:script"].invoke
+        Rake::Task["#{flavor.scope.path}:before_cache"].invoke
+      rescue => e
+        exception = e
+        puts "Failed task: #{e.class} #{e.message}".red
+      end
+      if ENV['SKIP_CLEANUP']
+        puts 'Skipping cleanup, disposable environments are great'.yellow
+      else
+        puts 'Cleaning up'
+        Rake::Task["#{flavor.scope.path}:cleanup"].invoke
+      end
+      raise exception if exception
+    end
+  end
+end
@@ -0,0 +1,35 @@
+#!/bin/bash
+
+# A script to create the MySQL containers.
+
+set -e
+
+NAME='dd-test-mysql'
+PORT=13306
+
+if docker ps | grep dd-test-mysql >/dev/null; then
+  echo 'the containers already exist, we have to remove them'
+  bash mysql/ci/stop-docker.sh
+fi
+
+MYSQL00_ID=$(docker run -p 3306:3306 --name $NAME -e MYSQL_ROOT_PASSWORD=datadog -d mysql:5.7)
+MYSQL00_IP=$(docker inspect ${MYSQL00_ID} | grep '"IPAddress"' | cut -d':' -f2 | cut -d'"' -f2)
+MYSQL00_IP=$(echo $MYSQL00_IP | cut -d " " -f2)
+echo 'running'
+
+# Wait for mysql to be ready to accept connections (otherwise the tests will fail).
+# Making the whole process wait for n seconds is not a good option:
+# With a large variety of infrastructure, such an option will inevitably cause test flakiness and failures
+# So, we grep the logs instead.
+until [[ `docker logs dd-test-mysql 2>&1` =~ .*"MySQL init process done. Ready for start up" ]];
+do
+  sleep 2
+done
+
+echo 'ready'
+
+# This is infrastructure setup, e.g. creating databases, adding users and assigning user permissions.
+# Any adding of data should go in the test files.
+docker run -it --link dd-test-mysql:mysql --rm mysql:5.7 sh -c 'exec mysql -h"$MYSQL_PORT_3306_TCP_ADDR" -P"MYSQL_PORT_3306_TCP_PORT" -uroot -p"$MYSQL_ENV_MYSQL_ROOT_PASSWORD" -e "create user \"dog\"@\"%\" identified by \"dog\"; GRANT PROCESS, REPLICATION CLIENT ON *.* TO \"dog\"@\"%\" WITH MAX_USER_CONNECTIONS 5; CREATE DATABASE testdb; CREATE TABLE testdb.users (name VARCHAR(20), age INT); GRANT SELECT ON testdb.users TO \"dog\"@\"%\"; INSERT INTO testdb.users (name,age) VALUES(\"Alice\",25); INSERT INTO testdb.users (name,age) VALUES(\"Bob\",20); GRANT SELECT ON performance_schema.* TO \"dog\"@\"%\"; USE testdb; SELECT * FROM users ORDER BY name; "'
+
+echo 'Docker setup finished'
@@ -0,0 +1,13 @@
+#!/bin/bash
+
+# A script to tear down the MySQL containers.
+
+set -e
+
+if docker ps -a | grep dd-test-mysql >/dev/null; then
+  containers=$(docker ps --format '{{.Names}}' --filter name=dd-test-mysql)
+  stopped_containers=$(docker ps -a --format '{{.Names}}' --filter name=dd-test-mysql)
+
+  docker kill $containers
+  docker rm $stopped_containers
+fi
@@ -4,6 +4,11 @@
   "display_name": "MySQL",
   "version": "0.1",
   "support": "core",
+  "manifest_version": "0.1.0",
+  "max_agent_version": "6.0.0",
+  "min_agent_version": "5.6.3",
+
+  "maintainer": "help@datadoghq.com",
 
   "short_description": "The MySQL integration helps to collect performance and avaialbility metrics from MySQL server instances.",
   "description": "The MySQL integration helps to collect performance and avaialbility metrics from MySQL server instances.",

@@ -0,0 +1,42 @@
+metric_name,metric_type,interval,unit_name,per_unit_name,description,orientation,integration,short_name
+mysql.galera.wsrep_cluster_size,gauge,,node,,The current number of nodes in the Galera cluster.,0,mysql,galera cluster size
+mysql.innodb.buffer_pool_free,gauge,,page,,The number of free pages in the InnoDB Buffer Pool.,1,mysql,innodb buf pool free
+mysql.innodb.buffer_pool_total,gauge,,page,,The total number of pages in the InnoDB Buffer Pool.,0,mysql,innodb buf pool total
+mysql.innodb.buffer_pool_used,gauge,,page,,The number of used pages in the InnoDB Buffer Pool.,-1,mysql,innodb buf pool used
+mysql.innodb.buffer_pool_utilization,gauge,,fraction,,The utilization of the InnoDB Buffer Pool.,-1,mysql,innodb buf pool usage
+mysql.innodb.current_row_locks,gauge,,lock,,The number of current row locks.,-1,mysql,current row locks
+mysql.innodb.data_reads,gauge,,read,second,The rate of data reads.,0,mysql,data reads
+mysql.innodb.data_writes,gauge,,write,second,The rate of data writes.,0,mysql,data writes
+mysql.innodb.mutex_os_waits,gauge,,event,second,The rate of mutex OS waits.,0,mysql,mutex os waits
+mysql.innodb.mutex_spin_rounds,gauge,,event,second,The rate of mutex spin rounds.,0,mysql,mutex spin rounds
+mysql.innodb.mutex_spin_waits,gauge,,event,second,The rate of mutex spin waits.,0,mysql,mutex spin waits
+mysql.innodb.os_log_fsyncs,gauge,,write,second,The rate of fsync writes to the log file.,0,mysql,log fsyncs
+mysql.innodb.row_lock_time,gauge,,fraction,,Fraction of time spent (ms/s) acquring row locks.,-1,mysql,row lock time
+mysql.innodb.row_lock_waits,gauge,,event,second,The number of times per second a row lock had to be waited for.,0,mysql,innodb row lock waits
+mysql.net.connections,gauge,,connection,second,The rate of connections to the server.,0,mysql,conns per s
+mysql.net.max_connections,gauge,,connection,,The maximum number of connections that have been in use simultaneously since the server started.,-1,mysql,max used conn
+mysql.performance.com_delete,gauge,,query,second,The rate of delete statements.,0,mysql,deletes
+mysql.performance.com_delete_multi,gauge,,query,second,The rate of delete-multi statements.,0,mysql,delete multi
+mysql.performance.com_insert,gauge,,query,second,The rate of insert statements.,0,mysql,inserts
+mysql.performance.com_insert_select,gauge,,query,second,The rate of insert-select statements.,0,mysql,insrt select
+mysql.performance.com_replace_select,gauge,,query,second,The rate of replace-select statements.,0,mysql,replc select
+mysql.performance.com_select,gauge,,query,second,The rate of select statements.,0,mysql,selects
+mysql.performance.com_update,gauge,,query,second,The rate of update statements.,0,mysql,updates
+mysql.performance.com_update_multi,gauge,,query,second,The rate of update-multi.,0,mysql,update multi
+mysql.performance.created_tmp_disk_tables,gauge,,table,second,The rate of internal on-disk temporary tables created by second by the server while executing statements.,-1,mysql,disk tmp tables created
+mysql.performance.created_tmp_files,gauge,,file,second,The rate of temporary files created by second.,-1,mysql,tmp files created
+mysql.performance.created_tmp_tables,gauge,,table,second,The rate of internal temporary tables created by second by the server while executing statements.,0,mysql,tmp tables created
+mysql.performance.kernel_time,gauge,,percent,,Percentage of CPU time spent in kernel space by MySQL.,-1,mysql,cpu kernel
+mysql.performance.key_cache_utilization,gauge,,fraction,,The key cache utilization ratio.,1,mysql,key cache utilization
+mysql.performance.open_files,gauge,,file,,The number of open files.,0,mysql,open files
+mysql.performance.open_tables,gauge,,table,,The number of of tables that are open.,0,mysql,open tables
+mysql.performance.qcache_hits,gauge,,hit,second,The rate of query cache hits.,0,mysql,query cache hits
+mysql.performance.questions,gauge,,query,second,The rate of statements executed by the server.,0,mysql,questions
+mysql.performance.slow_queries,gauge,,query,second,The rate of slow queries.,-1,mysql,slow queries
+mysql.performance.table_locks_waited,gauge,,,,The total number of times that a request for a table lock could not be granted immediately and a wait was needed.,-1,mysql,table locks waited
+mysql.performance.threads_connected,gauge,,connection,,The number of currently open connections.,0,mysql,threads connected
+mysql.performance.threads_running,gauge,,thread,,The number of threads that are not sleeping.,0,mysql,threads running
+mysql.performance.user_time,gauge,,percent,,Percentage of CPU time spent in user space by MySQL.,-1,mysql,cpu user
+mysql.replication.seconds_behind_master,gauge,,second,,The lag in seconds between the master and the slave.,-1,mysql,replication lag
+mysql.replication.slave_running,gauge,,,,A boolean showing if this server is a replication slave that is connected to a replication master.,0,mysql,slave running
+mysql.performance.queries,gauge,,query,second,The rate of queries.,0,mysql,queries
@@ -1 +1,2 @@
+# integration pip requirements
 pymysql==0.6.6