· 7 years ago · Apr 24, 2018, 03:26 AM
1# cat /etc/logstash/conf.d/twitter_logstash.conf
2input {
3 twitter {
4 consumer_key => "XXXXXXX"
5 consumer_secret => "XXXXXXXX"
6 oauth_token => "XXXXXXXXXXXXXXd"
7 oauth_token_secret => "XXXXXXXXXX"
8 keywords => [ "bush", "trump", "obama", "thor", "spiderman", "wolverine", "ironman", "hulk" ]
9 full_tweet => true
10 }
11}
12
13filter { }
14
15output {
16 stdout {
17 codec => dots
18 }
19 elasticsearch {
20 hosts => "localhost:9200"
21 index => "twitter_elastic_example"
22 document_type => "tweets"
23 template => "./twitter_template.json"
24 template_name => "twitter_elastic_example"
25 template_overwrite => true
26 }
27}
28
29
30
31# cat /etc/logstash/conf.d/twitter_template.json | head -50
32{
33 "index_patterns": "twitter_elastic_example",
34 "settings": {
35 "number_of_shards": 1,
36 "number_of_replicas": 0,
37 "mapping": {
38 "total_fields": {
39 "limit": "10000"
40 }
41 }
42 },
43 "mappings": {
44 "tweets": {
45 "properties": {
46 "@timestamp": {
47 "type": "date",
48 "format": "dateOptionalTime"
49 },
50 "text": {
51 "type": "text"
52 },
53 "user": {
54 "type": "object",
55 "properties": {
56 "description": {
57 "type": "text"
58 }
59 }
60 },
61 "coordinates": {
62 "type": "object",
63 "properties": {
64 "coordinates": {
65 "type": "geo_point"
66 }
67 }
68 },
69 "entities": {
70 "type": "object",
71 "properties": {
72 "hashtags": {
73 "type": "object",
74 "properties": {
75 "text": {
76 "type": "text",
77 "fielddata": true
78 }
79 }
80 }
81 }
82 "retweeted_status": {
83 "type": "object",
84 "properties": {
85 "text": {
86 "type": "text"
87 }
88 }
89 }
90 },
91 "dynamic_templates": [
92 {
93 "string_template": {
94 "match": "*",
95 "match_mapping_type": "string",
96 "mapping": {
97 "type": "keyword"
98 }
99 }
100 }
101 }
102 }
103 }
104 ]
105 }
106 }
107}