Use docker-setup.sh from upstream .ci repo (#1227) (#1229)

* Use docker-setup.sh from upstream .ci repo (#1227)

The custom docker-setup.sh script here is not materially different from the
shared script in the `.ci` repository. The only difference were pulling and
starting the elasticsearch image. This should not affect CI materially and the
consistency gain we get using the same `docker-setup.sh` file everywhere is
preferrable to having a slightly different script here.

* fix integration test targets

* Fix metrics spec

This commit backports the part of https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1220/files#diff-4ac1f3b518168b2e30c23d2c20e6eb3c4fa19c45a560bf481bca5641d41e968a that accounts for ES main/9.x. See #1229 (comment) for detailed breakdown.

Author: donoghuc

.ci/docker-setup.sh

@@ -11,8 +11,7 @@ jobs:
     - env: INTEGRATION=true SNAPSHOT=true LOG_LEVEL=info ELASTIC_STACK_VERSION=7.current
     - env: INTEGRATION=true SNAPSHOT=true LOG_LEVEL=info ELASTIC_STACK_VERSION=8.previous
     - env: INTEGRATION=true SNAPSHOT=true LOG_LEVEL=info ELASTIC_STACK_VERSION=8.current
-    - env: INTEGRATION=true SNAPSHOT=true LOG_LEVEL=info ELASTIC_STACK_VERSION=8.next
-    - env: INTEGRATION=true SNAPSHOT=true LOG_LEVEL=info ELASTIC_STACK_VERSION=9.next
+    - env: INTEGRATION=true SNAPSHOT=true LOG_LEVEL=info ELASTIC_STACK_VERSION=9.current
     - env: INTEGRATION=true SNAPSHOT=true LOG_LEVEL=info ELASTIC_STACK_VERSION=main
     - stage: "Secure Integration Tests"
       env: SECURE_INTEGRATION=true INTEGRATION=true LOG_LEVEL=info ELASTIC_STACK_VERSION=7.current
@@ -22,5 +21,6 @@ jobs:
     - env: SECURE_INTEGRATION=true INTEGRATION=true LOG_LEVEL=info ELASTIC_STACK_VERSION=9.current
     - env: SECURE_INTEGRATION=true INTEGRATION=true LOG_LEVEL=info ELASTIC_STACK_VERSION=9.current ES_SSL_KEY_INVALID=true
     - env: SECURE_INTEGRATION=true INTEGRATION=true LOG_LEVEL=info ELASTIC_STACK_VERSION=9.current ES_SSL_SUPPORTED_PROTOCOLS=TLSv1.3
-    - env: SECURE_INTEGRATION=true INTEGRATION=true LOG_LEVEL=info SNAPSHOT=true ELASTIC_STACK_VERSION=8.next
-    - env: SECURE_INTEGRATION=true INTEGRATION=true LOG_LEVEL=info SNAPSHOT=true ELASTIC_STACK_VERSION=9.next
+    - env: SECURE_INTEGRATION=true INTEGRATION=true LOG_LEVEL=info SNAPSHOT=true ELASTIC_STACK_VERSION=8.current
+    - env: SECURE_INTEGRATION=true INTEGRATION=true LOG_LEVEL=info SNAPSHOT=true ELASTIC_STACK_VERSION=9.current
+    - env: SECURE_INTEGRATION=true INTEGRATION=true LOG_LEVEL=info SNAPSHOT=true ELASTIC_STACK_VERSION=main

.travis.yml

@@ -5,7 +5,10 @@
     require "logstash/outputs/elasticsearch"
     settings = {
       "manage_template" => false,
-      "hosts" => "#{get_host_port()}"
+      "hosts" => "#{get_host_port()}",
+      # write data to a random non templated index to ensure the bulk partially fails
+      # don't use streams like "logs-*" as those have failure stores enabled, causing the bulk to succeed instead
+      "index" => "custom_index_#{rand(10000)}"
     }
     plugin = LogStash::Outputs::ElasticSearch.new(settings)
   end