Intellipaat Back

Explore Courses Blog Tutorials Interview Questions

Logstash conf error - amazon_es

0 votes
2 views
in AWS by (19.1k points)

I am trying to configure for the first time my logstash.conf file with output to amazon_es.

My whole logstash.conf file is here:

input {

  jdbc { 

    jdbc_connection_string => "jdbc:mysql://localhost:3306/testdb"

    # The user we wish to execute our statement as

    jdbc_user => "root"

    jdbc_password => "root"

    # The path to our downloaded jdbc driver

    jdbc_driver_library => "/mnt/c/Users/xxxxxxxx/mysql-connector-java-5.1.45/mysql-connector-java-5.1.45-bin.jar"

    jdbc_driver_class => "com.mysql.jdbc.Driver"

    # our query

    statement => "SELECT * FROM testtable"

    }

  }

output {

  amazon_es {

    hosts => ["search-xxxxx.eu-west-3.es.amazonaws.com"]

    region => "eu-west-3"

    aws_access_key_id => 'xxxxxxxxxxxxxxxxxxxxxx'

    aws_secret_access_key => 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxx'

    index => "test-migrate"

    document_type => "data"

    }

}

I have 3 elements selected from my database, but the first time i run the script, only the first element is indexed in elastic search. The second time i run it, all 3 elements are indexed. I get the error each time i run logstash with this conf file.


 

    [2018-02-08T14:31:18,270][INFO ][logstash.modules.scaffold] Initializing module {:module_name=>"fb_apache", :directory=>"/mnt/c/Users/anthony.maffert/l/logstash-6.2.0/modules/fb_apache/configuration"}

    [2018-02-08T14:31:18,279][DEBUG][logstash.plugins.registry] Adding plugin to the registry {:name=>"fb_apache", :type=>:modules, :class=>#<LogStash::Modules::Scaffold:0x47c515a1 @module_name="fb_apache", @directory="/mnt/c/Users/anthony.maffert/l/logstash-6.2.0/modules/fb_apache/configuration", @kibana_version_parts=["6", "0", "0"]>}

    [2018-02-08T14:31:18,286][INFO ][logstash.modules.scaffold] Initializing module {:module_name=>"netflow", :directory=>"/mnt/c/Users/anthony.maffert/l/logstash-6.2.0/modules/netflow/configuration"}

    [2018-02-08T14:31:18,287][DEBUG][logstash.plugins.registry] Adding plugin to the registry {:name=>"netflow", :type=>:modules, :class=>#<LogStash::Modules::Scaffold:0x6f1a5910 @module_name="netflow", @directory="/mnt/c/Users/anthony.maffert/l/logstash-6.2.0/modules/netflow/configuration", @kibana_version_parts=["6", "0", "0"]>}

    [2018-02-08T14:31:18,765][DEBUG][logstash.runner          ] -------- Logstash Settings (* means modified) ---------

    [2018-02-08T14:31:18,765][DEBUG][logstash.runner          ] node.name: "DEVFE-AMT"

    [2018-02-08T14:31:18,766][DEBUG][logstash.runner          ] *path.config: "logstash.conf"

    [2018-02-08T14:31:18,766][DEBUG][logstash.runner          ] path.data: "/mnt/c/Users/anthony.maffert/l/logstash-6.2.0/data"

    [2018-02-08T14:31:18,767][DEBUG][logstash.runner          ] modules.cli: []

    [2018-02-08T14:31:18,768][DEBUG][logstash.runner          ] modules: []

    [2018-02-08T14:31:18,768][DEBUG][logstash.runner          ] modules_setup: false

    [2018-02-08T14:31:18,768][DEBUG][logstash.runner          ] config.test_and_exit: false

    [2018-02-08T14:31:18,769][DEBUG][logstash.runner          ] config.reload.automatic: false

    [2018-02-08T14:31:18,769][DEBUG][logstash.runner          ] config.reload.interval: 3000000000

    [2018-02-08T14:31:18,769][DEBUG][logstash.runner          ] config.support_escapes: false

    [2018-02-08T14:31:18,770][DEBUG][logstash.runner          ] metric.collect: true

    [2018-02-08T14:31:18,770][DEBUG][logstash.runner          ] pipeline.id: "main"

    [2018-02-08T14:31:18,771][DEBUG][logstash.runner          ] pipeline.system: false

    [2018-02-08T14:31:18,771][DEBUG][logstash.runner          ] pipeline.workers: 8

    [2018-02-08T14:31:18,771][DEBUG][logstash.runner          ] pipeline.output.workers: 1

    [2018-02-08T14:31:18,772][DEBUG][logstash.runner          ] pipeline.batch.size: 125

    [2018-02-08T14:31:18,772][DEBUG][logstash.runner          ] pipeline.batch.delay: 50

    [2018-02-08T14:31:18,772][DEBUG][logstash.runner          ] pipeline.unsafe_shutdown: false

    [2018-02-08T14:31:18,772][DEBUG][logstash.runner          ] pipeline.java_execution: false

    [2018-02-08T14:31:18,773][DEBUG][logstash.runner          ] pipeline.reloadable: true

    [2018-02-08T14:31:18,773][DEBUG][logstash.runner          ] path.plugins: []

    [2018-02-08T14:31:18,773][DEBUG][logstash.runner          ] config.debug: false

    [2018-02-08T14:31:18,776][DEBUG][logstash.runner          ] *log.level: "debug" (default: "info")

    [2018-02-08T14:31:18,783][DEBUG][logstash.runner          ] version: false

    [2018-02-08T14:31:18,784][DEBUG][logstash.runner          ] help: false

    [2018-02-08T14:31:18,784][DEBUG][logstash.runner          ] log.format: "plain"

    [2018-02-08T14:31:18,786][DEBUG][logstash.runner          ] http.host: "127.0.0.1"

    [2018-02-08T14:31:18,793][DEBUG][logstash.runner          ] http.port: 9600..9700

    [2018-02-08T14:31:18,793][DEBUG][logstash.runner          ] http.environment: "production"

    [2018-02-08T14:31:18,794][DEBUG][logstash.runner          ] queue.type: "memory"

    [2018-02-08T14:31:18,796][DEBUG][logstash.runner          ] queue.drain: false

    [2018-02-08T14:31:18,804][DEBUG][logstash.runner          ] queue.page_capacity: 67108864

    [2018-02-08T14:31:18,809][DEBUG][logstash.runner          ] queue.max_bytes: 1073741824

    [2018-02-08T14:31:18,822][DEBUG][logstash.runner          ] queue.max_events: 0

    [2018-02-08T14:31:18,823][DEBUG][logstash.runner          ] queue.checkpoint.acks: 1024

    [2018-02-08T14:31:18,836][DEBUG][logstash.runner          ] queue.checkpoint.writes: 1024

    [2018-02-08T14:31:18,837][DEBUG][logstash.runner          ] queue.checkpoint.interval: 1000

    [2018-02-08T14:31:18,846][DEBUG][logstash.runner          ] dead_letter_queue.enable: false

    [2018-02-08T14:31:18,854][DEBUG][logstash.runner          ] dead_letter_queue.max_bytes: 1073741824

    [2018-02-08T14:31:18,859][DEBUG][logstash.runner          ] slowlog.threshold.warn: -1

    [2018-02-08T14:31:18,868][DEBUG][logstash.runner          ] slowlog.threshold.info: -1

    [2018-02-08T14:31:18,873][DEBUG][logstash.runner          ] slowlog.threshold.debug: -1

    [2018-02-08T14:31:18,885][DEBUG][logstash.runner          ] slowlog.threshold.trace: -1

    [2018-02-08T14:31:18,887][DEBUG][logstash.runner          ] keystore.classname: "org.logstash.secret.store.backend.JavaKeyStore"

    [2018-02-08T14:31:18,896][DEBUG][logstash.runner          ] keystore.file: "/mnt/c/Users/anthony.maffert/l/logstash-6.2.0/config/logstash.keystore"

    [2018-02-08T14:31:18,896][DEBUG][logstash.runner          ] path.queue: "/mnt/c/Users/anthony.maffert/l/logstash-6.2.0/data/queue"

    [2018-02-08T14:31:18,911][DEBUG][logstash.runner          ] path.dead_letter_queue: "/mnt/c/Users/anthony.maffert/l/logstash-6.2.0/data/dead_letter_queue"

    [2018-02-08T14:31:18,911][DEBUG][logstash.runner          ] path.settings: "/mnt/c/Users/anthony.maffert/l/logstash-6.2.0/config"

    [2018-02-08T14:31:18,926][DEBUG][logstash.runner          ] path.logs: "/mnt/c/Users/anthony.maffert/l/logstash-6.2.0/logs"

    [2018-02-08T14:31:18,926][DEBUG][logstash.runner          ] --------------- Logstash Settings -------------------

    [2018-02-08T14:31:18,998][WARN ][logstash.config.source.multilocal] Ignoring the 'pipelines.yml' file because modules or command line options are specified

    [2018-02-08T14:31:19,067][DEBUG][logstash.agent           ] Setting up metric collection

    [2018-02-08T14:31:19,147][DEBUG][logstash.instrument.periodicpoller.os] Starting {:polling_interval=>5, :polling_timeout=>120}

    [2018-02-08T14:31:19,293][DEBUG][logstash.instrument.periodicpoller.jvm] Starting {:polling_interval=>5, :polling_timeout=>120}

    [2018-02-08T14:31:19,422][DEBUG][logstash.instrument.periodicpoller.jvm] collector name {:name=>"ParNew"}

    [2018-02-08T14:31:19,429][DEBUG][logstash.instrument.periodicpoller.jvm] collector name {:name=>"ConcurrentMarkSweep"}

    [2018-02-08T14:31:19,453][DEBUG][logstash.instrument.periodicpoller.persistentqueue] Starting {:polling_interval=>5, :polling_timeout=>120}

    [2018-02-08T14:31:19,464][DEBUG][logstash.instrument.periodicpoller.deadletterqueue] Starting {:polling_interval=>5, :polling_timeout=>120}

    [2018-02-08T14:31:19,519][INFO ][logstash.runner          ] Starting Logstash {"logstash.version"=>"6.2.0"}

    [2018-02-08T14:31:19,537][DEBUG][logstash.agent           ] Starting agent

    [2018-02-08T14:31:19,565][DEBUG][logstash.agent           ] Starting puma

    [2018-02-08T14:31:19,580][DEBUG][logstash.agent           ] Trying to start WebServer {:port=>9600}

    [2018-02-08T14:31:19,654][DEBUG][logstash.config.source.local.configpathloader] Skipping the following files while reading config since they don't match the specified glob pattern {:files=>["/mnt/c/Users/anthony.maffert/l/logstash-6.2.0/CONTRIBUTORS", "/mnt/c/Users/anthony.maffert/l/logstash-6.2.0/Gemfile", "/mnt/c/Users/anthony.maffert/l/logstash-6.2.0/Gemfile.lock", "/mnt/c/Users/anthony.maffert/l/logstash-6.2.0/LICENSE", "/mnt/c/Users/anthony.maffert/l/logstash-6.2.0/NOTICE.TXT", "/mnt/c/Users/anthony.maffert/l/logstash-6.2.0/bin", "/mnt/c/Users/anthony.maffert/l/logstash-6.2.0/config", "/mnt/c/Users/anthony.maffert/l/logstash-6.2.0/data", "/mnt/c/Users/anthony.maffert/l/logstash-6.2.0/lib", "/mnt/c/Users/anthony.maffert/l/logstash-6.2.0/logs", "/mnt/c/Users/anthony.maffert/l/logstash-6.2.0/logstash-core", "/mnt/c/Users/anthony.maffert/l/logstash-6.2.0/logstash-core-plugin-api", "/mnt/c/Users/anthony.maffert/l/logstash-6.2.0/modules", "/mnt/c/Users/anthony.maffert/l/logstash-6.2.0/tools", "/mnt/c/Users/anthony.maffert/l/logstash-6.2.0/vendor"]}

    [2018-02-08T14:31:19,658][DEBUG][logstash.api.service     ] [api-service] start

1 Answer

0 votes
by (44.4k points)

Copy this ES 6.x template on your local system. Then add the following template setting to your amazon_es:

 amazon_es {

    hosts => ["search-xxxxx.eu-west-3.es.amazonaws.com"]

    region => "eu-west-3"

    aws_access_key_id => 'xxxxxxxxxxxxxxxxxxxxxx'

    aws_secret_access_key => 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxx'

    index => "test-migrate"

    document_type => "data"

    template => '/path/to/template.json'

  }

Related questions

Want to get 50% Hike on your Salary?

Learn how we helped 50,000+ professionals like you !

31k questions

32.8k answers

501 comments

693 users

Browse Categories

© COPYRIGHT 2011-2024 INTELLIPAAT.COM. ALL RIGHTS RESERVED.

...