· 8 years ago · Oct 07, 2017, 04:00 AM
1{
2 "configurations" : [
3 {
4 "hdfs-logsearch-conf" : {
5 "properties_attributes" : { },
6 "properties" : {
7 "component_mappings" : "NAMENODE:hdfs_namenode;DATANODE:hdfs_datanode;SECONDARY_NAMENODE:hdfs_secondarynamenode;JOURNALNODE:hdfs_journalnode;ZKFC:hdfs_zkfc;NFS_GATEWAY:hdfs_nfs3",
8 "content" : "\n{\n \"input\":[\n {\n \"type\":\"hdfs_datanode\",\n \"rowtype\":\"service\",\n \"path\":\"{{default('/configurations/hadoop-env/hdfs_log_dir_prefix', '/var/log/hadoop')}}/{{default('configurations/hadoop-env/hdfs_user', 'hdfs')}}/hadoop-{{default('configurations/hadoop-env/hdfs_user', 'hdfs')}}-datanode-*.log\"\n },\n {\n \"type\":\"hdfs_namenode\",\n \"rowtype\":\"service\",\n \"path\":\"{{default('/configurations/hadoop-env/hdfs_log_dir_prefix', '/var/log/hadoop')}}/{{default('configurations/hadoop-env/hdfs_user', 'hdfs')}}/hadoop-{{default('configurations/hadoop-env/hdfs_user', 'hdfs')}}-namenode-*.log\"\n },\n {\n \"type\":\"hdfs_journalnode\",\n \"rowtype\":\"service\",\n \"path\":\"{{default('/configurations/hadoop-env/hdfs_log_dir_prefix', '/var/log/hadoop')}}/{{default('configurations/hadoop-env/hdfs_user', 'hdfs')}}/hadoop-{{default('configurations/hadoop-env/hdfs_user', 'hdfs')}}-journalnode-*.log\"\n },\n {\n \"type\":\"hdfs_secondarynamenode\",\n \"rowtype\":\"service\",\n \"path\":\"{{default('/configurations/hadoop-env/hdfs_log_dir_prefix', '/var/log/hadoop')}}/{{default('configurations/hadoop-env/hdfs_user', 'hdfs')}}/hadoop-{{default('configurations/hadoop-env/hdfs_user', 'hdfs')}}-secondarynamenode-*.log\"\n },\n {\n \"type\":\"hdfs_zkfc\",\n \"rowtype\":\"service\",\n \"path\":\"{{default('/configurations/hadoop-env/hdfs_log_dir_prefix', '/var/log/hadoop')}}/{{default('configurations/hadoop-env/hdfs_user', 'hdfs')}}/hadoop-{{default('configurations/hadoop-env/hdfs_user', 'hdfs')}}-zkfc-*.log\"\n },\n {\n \"type\":\"hdfs_nfs3\",\n \"rowtype\":\"service\",\n \"path\":\"{{default('/configurations/hadoop-env/hdfs_log_dir_prefix', '/var/log/hadoop')}}/{{default('configurations/hadoop-env/hdfs_user', 'hdfs')}}/hadoop-{{default('configurations/hadoop-env/hdfs_user', 'hdfs')}}-nfs3-*.log\"\n },\n {\n \"type\":\"hdfs_audit\",\n \"rowtype\":\"audit\",\n \"is_enabled\":\"true\",\n \"add_fields\":{\n \"logType\":\"HDFSAudit\",\n \"enforcer\":\"hadoop-acl\",\n \"repoType\":\"1\",\n \"repo\":\"hdfs\"\n },\n \"path\":\"{{default('/configurations/hadoop-env/hdfs_log_dir_prefix', '/var/log/hadoop')}}/{{default('configurations/hadoop-env/hdfs_user', 'hdfs')}}/hdfs-audit.log\"\n }\n ],\n \"filter\":[\n {\n \"filter\":\"grok\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"hdfs_datanode\",\n \"hdfs_journalnode\",\n \"hdfs_secondarynamenode\",\n \"hdfs_namenode\",\n \"hdfs_zkfc\",\n \"hdfs_nfs3\"\n ]\n }\n },\n \"log4j_format\":\"%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n\",\n \"multiline_pattern\":\"^(%{TIMESTAMP_ISO8601:logtime})\",\n \"message_pattern\":\"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\\\(%{JAVAFILE:file}:%{JAVAMETHOD:method}\\\\(%{INT:line_number}\\\\)\\\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}\",\n \"post_map_values\":{\n \"logtime\":{\n \"map_date\":{\n \"target_date_pattern\":\"yyyy-MM-dd HH:mm:ss,SSS\"\n }\n }\n }\n },\n {\n \"filter\":\"grok\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"hdfs_audit\"\n ]\n }\n },\n \"log4j_format\":\"%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n\",\n \"multiline_pattern\":\"^(%{TIMESTAMP_ISO8601:evtTime})\",\n \"message_pattern\":\"(?m)^%{TIMESTAMP_ISO8601:evtTime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}:%{SPACE}%{GREEDYDATA:log_message}\",\n \"post_map_values\":{\n \"evtTime\":{\n \"map_date\":{\n \"target_date_pattern\":\"yyyy-MM-dd HH:mm:ss,SSS\"\n }\n }\n }\n },\n {\n \"filter\":\"keyvalue\",\n \"sort_order\":1,\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"hdfs_audit\"\n ]\n }\n },\n \"source_field\":\"log_message\",\n \"value_split\":\"=\",\n \"field_split\":\"\\t\",\n \"post_map_values\":{\n \"src\":{\n \"map_fieldname\":{\n \"new_fieldname\":\"resource\"\n }\n },\n \"ip\":{\n \"map_fieldname\":{\n \"new_fieldname\":\"cliIP\"\n }\n },\n \"allowed\":[\n {\n \"map_fieldvalue\":{\n \"pre_value\":\"true\",\n \"post_value\":\"1\"\n }\n },\n {\n \"map_fieldvalue\":{\n \"pre_value\":\"false\",\n \"post_value\":\"0\"\n }\n },\n {\n \"map_fieldname\":{\n \"new_fieldname\":\"result\"\n }\n }\n ],\n \"cmd\":{\n \"map_fieldname\":{\n \"new_fieldname\":\"action\"\n }\n },\n \"proto\":{\n \"map_fieldname\":{\n \"new_fieldname\":\"cliType\"\n }\n },\n \"callerContext\":{\n \"map_fieldname\":{\n \"new_fieldname\":\"req_caller_id\"\n }\n }\n }\n },\n {\n \"filter\":\"grok\",\n \"sort_order\":2,\n \"source_field\":\"ugi\",\n \"remove_source_field\":\"false\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"hdfs_audit\"\n ]\n }\n },\n \"message_pattern\":\"%{USERNAME:p_user}.+auth:%{USERNAME:p_authType}.+via %{USERNAME:k_user}.+auth:%{USERNAME:k_authType}|%{USERNAME:user}.+auth:%{USERNAME:authType}|%{USERNAME:x_user}\",\n \"post_map_values\":{\n \"user\":{\n \"map_fieldname\":{\n \"new_fieldname\":\"reqUser\"\n }\n },\n \"x_user\":{\n \"map_fieldname\":{\n \"new_fieldname\":\"reqUser\"\n }\n },\n \"p_user\":{\n \"map_fieldname\":{\n \"new_fieldname\":\"reqUser\"\n }\n },\n \"k_user\":{\n \"map_fieldname\":{\n \"new_fieldname\":\"proxyUsers\"\n }\n },\n \"p_authType\":{\n \"map_fieldname\":{\n \"new_fieldname\":\"authType\"\n }\n },\n \"k_authType\":{\n \"map_fieldname\":{\n \"new_fieldname\":\"proxyAuthType\"\n }\n }\n }\n }\n ]\n }",
9 "service_name" : "HDFS"
10 }
11 }
12 },
13 {
14 "streamline-env" : {
15 "properties_attributes" : { },
16 "properties" : {
17 "streamline_pid_dir" : "/var/run/streamline",
18 "streamline_user_nproc_limit" : "65536",
19 "streamline_principal_name" : "streamline",
20 "streamline_user" : "streamline",
21 "streamline_user_nofile_limit" : "65536",
22 "streamline_log_dir" : "/var/log/streamline",
23 "content" : "\n#!/bin/bash\n\n# Set STREAMLINE specific environment variables here.\n\n# The java implementation to use.\nexport JAVA_HOME={{java64_home}}\nexport PATH=$PATH:$JAVA_HOME/bin\nexport PID_DIR={{streamline_pid_dir}}\nexport LOG_DIR={{streamline_log_dir}}\nexport STREAMLINE_KERBEROS_PARAMS={{streamline_kerberos_params}}"
24 }
25 }
26 },
27 {
28 "ranger-nifi-policymgr-ssl" : {
29 "properties_attributes" : { },
30 "properties" : { }
31 }
32 },
33 {
34 "zoo.cfg" : {
35 "properties_attributes" : { },
36 "properties" : {
37 "autopurge.purgeInterval" : "24",
38 "dataDir" : "/hadoop/zookeeper",
39 "autopurge.snapRetainCount" : "30",
40 "clientPort" : "2181",
41 "initLimit" : "10",
42 "tickTime" : "3000",
43 "syncLimit" : "5"
44 }
45 }
46 },
47 {
48 "druid-broker" : {
49 "properties_attributes" : { },
50 "properties" : {
51 "druid.cache.type" : "local",
52 "druid.server.http.numThreads" : "40",
53 "druid.broker.http.numConnections" : "20",
54 "druid.cache.sizeInBytes" : "10000000",
55 "druid.broker.cache.populateCache" : "true",
56 "druid.broker.cache.useCache" : "true",
57 "druid.processing.numThreads" : "3",
58 "druid.processing.buffer.sizeBytes" : "1073741824",
59 "druid.service" : "druid/broker",
60 "druid.port" : "8082"
61 }
62 }
63 },
64 {
65 "zookeeper-log4j" : {
66 "properties_attributes" : { },
67 "properties" : {
68 "zookeeper_log_max_backup_size" : "10",
69 "zookeeper_log_number_of_backup_files" : "10",
70 "content" : "\n#\n#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership. The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License. You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied. See the License for the\n# specific language governing permissions and limitations\n# under the License.\n#\n#\n#\n\n#\n# ZooKeeper Logging Configuration\n#\n\n# DEFAULT: console appender only\nlog4j.rootLogger=INFO, CONSOLE, ROLLINGFILE\n\n# Example with rolling log file\n#log4j.rootLogger=DEBUG, CONSOLE, ROLLINGFILE\n\n# Example with rolling log file and tracing\n#log4j.rootLogger=TRACE, CONSOLE, ROLLINGFILE, TRACEFILE\n\n#\n# Log INFO level and above messages to the console\n#\nlog4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender\nlog4j.appender.CONSOLE.Threshold=INFO\nlog4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout\nlog4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n\n\n#\n# Add ROLLINGFILE to rootLogger to get log file output\n# Log DEBUG level and above messages to a log file\nlog4j.appender.ROLLINGFILE=org.apache.log4j.RollingFileAppender\nlog4j.appender.ROLLINGFILE.Threshold=DEBUG\nlog4j.appender.ROLLINGFILE.File={{zk_log_dir}}/zookeeper.log\n\n# Max log file size of 10MB\nlog4j.appender.ROLLINGFILE.MaxFileSize={{zookeeper_log_max_backup_size}}MB\n# uncomment the next line to limit number of backup files\n#log4j.appender.ROLLINGFILE.MaxBackupIndex={{zookeeper_log_number_of_backup_files}}\n\nlog4j.appender.ROLLINGFILE.layout=org.apache.log4j.PatternLayout\nlog4j.appender.ROLLINGFILE.layout.ConversionPattern=%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n\n\n\n#\n# Add TRACEFILE to rootLogger to get log file output\n# Log DEBUG level and above messages to a log file\nlog4j.appender.TRACEFILE=org.apache.log4j.FileAppender\nlog4j.appender.TRACEFILE.Threshold=TRACE\nlog4j.appender.TRACEFILE.File=zookeeper_trace.log\n\nlog4j.appender.TRACEFILE.layout=org.apache.log4j.PatternLayout\n### Notice we are including log4j's NDC here (%x)\nlog4j.appender.TRACEFILE.layout.ConversionPattern=%d{ISO8601} - %-5p [%t:%C{1}@%L][%x] - %m%n"
71 }
72 }
73 },
74 {
75 "hbase-log4j" : {
76 "properties_attributes" : { },
77 "properties" : {
78 "hbase_log_maxbackupindex" : "20",
79 "hbase_security_log_maxfilesize" : "256",
80 "hbase_log_maxfilesize" : "256",
81 "hbase_security_log_maxbackupindex" : "20",
82 "content" : "\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership. The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License. You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\n# Define some default values that can be overridden by system properties\nhbase.root.logger=INFO,console\nhbase.security.logger=INFO,console\nhbase.log.dir=.\nhbase.log.file=hbase.log\n\n# Define the root logger to the system property \"hbase.root.logger\".\nlog4j.rootLogger=${hbase.root.logger}\n\n# Logging Threshold\nlog4j.threshold=ALL\n\n#\n# Daily Rolling File Appender\n#\nlog4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender\nlog4j.appender.DRFA.File=${hbase.log.dir}/${hbase.log.file}\n\n# Rollver at midnight\nlog4j.appender.DRFA.DatePattern=.yyyy-MM-dd\n\n# 30-day backup\n#log4j.appender.DRFA.MaxBackupIndex=30\nlog4j.appender.DRFA.layout=org.apache.log4j.PatternLayout\n\n# Pattern format: Date LogLevel LoggerName LogMessage\nlog4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}: %m%n\n\n# Rolling File Appender properties\nhbase.log.maxfilesize={{hbase_log_maxfilesize}}MB\nhbase.log.maxbackupindex={{hbase_log_maxbackupindex}}\n\n# Rolling File Appender\nlog4j.appender.RFA=org.apache.log4j.RollingFileAppender\nlog4j.appender.RFA.File=${hbase.log.dir}/${hbase.log.file}\n\nlog4j.appender.RFA.MaxFileSize=${hbase.log.maxfilesize}\nlog4j.appender.RFA.MaxBackupIndex=${hbase.log.maxbackupindex}\n\nlog4j.appender.RFA.layout=org.apache.log4j.PatternLayout\nlog4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}: %m%n\n\n#\n# Security audit appender\n#\nhbase.security.log.file=SecurityAuth.audit\nhbase.security.log.maxfilesize={{hbase_security_log_maxfilesize}}MB\nhbase.security.log.maxbackupindex={{hbase_security_log_maxbackupindex}}\nlog4j.appender.RFAS=org.apache.log4j.RollingFileAppender\nlog4j.appender.RFAS.File=${hbase.log.dir}/${hbase.security.log.file}\nlog4j.appender.RFAS.MaxFileSize=${hbase.security.log.maxfilesize}\nlog4j.appender.RFAS.MaxBackupIndex=${hbase.security.log.maxbackupindex}\nlog4j.appender.RFAS.layout=org.apache.log4j.PatternLayout\nlog4j.appender.RFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n\nlog4j.category.SecurityLogger=${hbase.security.logger}\nlog4j.additivity.SecurityLogger=false\n#log4j.logger.SecurityLogger.org.apache.hadoop.hbase.security.access.AccessController=TRACE\n\n#\n# Null Appender\n#\nlog4j.appender.NullAppender=org.apache.log4j.varia.NullAppender\n\n#\n# console\n# Add \"console\" to rootlogger above if you want to use this\n#\nlog4j.appender.console=org.apache.log4j.ConsoleAppender\nlog4j.appender.console.target=System.err\nlog4j.appender.console.layout=org.apache.log4j.PatternLayout\nlog4j.appender.console.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}: %m%n\n\n# Custom Logging levels\n\nlog4j.logger.org.apache.zookeeper=INFO\n#log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG\nlog4j.logger.org.apache.hadoop.hbase=INFO\n# Make these two classes INFO-level. Make them DEBUG to see more zk debug.\nlog4j.logger.org.apache.hadoop.hbase.zookeeper.ZKUtil=INFO\nlog4j.logger.org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher=INFO\n#log4j.logger.org.apache.hadoop.dfs=DEBUG\n# Set this class to log INFO only otherwise its OTT\n# Enable this to get detailed connection error/retry logging.\n# log4j.logger.org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation=TRACE\n\n\n# Uncomment this line to enable tracing on _every_ RPC call (this can be a lot of output)\n#log4j.logger.org.apache.hadoop.ipc.HBaseServer.trace=DEBUG\n\n# Uncomment the below if you want to remove logging of client region caching'\n# and scan of .META. messages\n# log4j.logger.org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation=INFO\n# log4j.logger.org.apache.hadoop.hbase.client.MetaScanner=INFO"
83 }
84 }
85 },
86 {
87 "kafka_jaas_conf" : {
88 "properties_attributes" : { },
89 "properties" : {
90 "content" : "\nKafkaServer {\ncom.sun.security.auth.module.Krb5LoginModule required\nuseKeyTab=true\nkeyTab=\"{{kafka_keytab_path}}\"\nstoreKey=true\nuseTicketCache=false\nserviceName=\"{{kafka_bare_jaas_principal}}\"\nprincipal=\"{{kafka_jaas_principal}}\";\n};\nKafkaClient {\ncom.sun.security.auth.module.Krb5LoginModule required\nuseTicketCache=true\nrenewTicket=true\nserviceName=\"{{kafka_bare_jaas_principal}}\";\n};\nClient {\ncom.sun.security.auth.module.Krb5LoginModule required\nuseKeyTab=true\nkeyTab=\"{{kafka_keytab_path}}\"\nstoreKey=true\nuseTicketCache=false\nserviceName=\"zookeeper\"\nprincipal=\"{{kafka_jaas_principal}}\";\n};\ncom.sun.security.jgss.krb5.initiate {\n com.sun.security.auth.module.Krb5LoginModule required\n renewTGT=false\n doNotPrompt=true\n useKeyTab=true\n keyTab=\"{{kafka_keytab_path}}\"\n storeKey=true\n useTicketCache=false\n serviceName=\"{{kafka_bare_jaas_principal}}\"\n principal=\"{{kafka_jaas_principal}}\";\n};"
91 }
92 }
93 },
94 {
95 "ranger-hdfs-policymgr-ssl" : {
96 "properties_attributes" : { },
97 "properties" : { }
98 }
99 },
100 {
101 "druid-superset" : {
102 "properties_attributes" : { },
103 "properties" : {
104 "ROW_LIMIT" : "5000",
105 "SUPERSET_DATABASE_USER" : "superset",
106 "SUPERSET_WORKERS" : "4",
107 "SECRET_KEY" : "hortonworks1",
108 "SUPERSET_DATABASE_NAME" : "superset",
109 "SUPERSET_DATABASE_PORT" : "",
110 "SUPERSET_WEBSERVER_PORT" : "9088",
111 "SUPERSET_TIMEOUT" : "60",
112 "SUPERSET_DATABASE_HOSTNAME" : "localhost",
113 "MAPBOX_API_KEY" : "",
114 "SUPERSET_WEBSERVER_ADDRESS" : "0.0.0.0",
115 "ENABLE_PROXY_FIX" : "True",
116 "SUPERSET_DATABASE_TYPE" : "sqlite",
117 "WTF_CSRF_ENABLED" : "True"
118 }
119 }
120 },
121 {
122 "nifi-login-identity-providers-env" : {
123 "properties_attributes" : { },
124 "properties" : {
125 "content" : "\n\n <!--\n Licensed to the Apache Software Foundation (ASF) under one or more\n contributor license agreements. See the NOTICE file distributed with\n this work for additional information regarding copyright ownership.\n The ASF licenses this file to You under the Apache License, Version 2.0\n (the \"License\"); you may not use this file except in compliance with\n the License. You may obtain a copy of the License at\n http://www.apache.org/licenses/LICENSE-2.0\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n -->\n <!--\n This file lists the login identity providers to use when running securely. In order\n to use a specific provider it must be configured here and it's identifier\n must be specified in the nifi.properties file.\n -->\n <loginIdentityProviders>\n <!--\n Identity Provider for users logging in with username/password against an LDAP server.\n\n 'Authentication Strategy' - How the connection to the LDAP server is authenticated. Possible\n values are ANONYMOUS, SIMPLE, or START_TLS.\n\n 'Manager DN' - The DN of the manager that is used to bind to the LDAP server to search for users.\n 'Manager Password' - The password of the manager that is used to bind to the LDAP server to\n search for users.\n\n 'TLS - Keystore' - Path to the Keystore that is used when connecting to LDAP using START_TLS.\n 'TLS - Keystore Password' - Password for the Keystore that is used when connecting to LDAP\n using START_TLS.\n 'TLS - Keystore Type' - Type of the Keystore that is used when connecting to LDAP using\n START_TLS (i.e. JKS or PKCS12).\n 'TLS - Truststore' - Path to the Truststore that is used when connecting to LDAP using START_TLS.\n 'TLS - Truststore Password' - Password for the Truststore that is used when connecting to\n LDAP using START_TLS.\n 'TLS - Truststore Type' - Type of the Truststore that is used when connecting to LDAP using\n START_TLS (i.e. JKS or PKCS12).\n 'TLS - Client Auth' - Client authentication policy when connecting to LDAP using START_TLS.\n Possible values are REQUIRED, WANT, NONE.\n 'TLS - Protocol' - Protocol to use when connecting to LDAP using START_TLS. (i.e. TLS,\n TLSv1.1, TLSv1.2, etc).\n 'TLS - Shutdown Gracefully' - Specifies whether the TLS should be shut down gracefully\n before the target context is closed. Defaults to false.\n\n 'Referral Strategy' - Strategy for handling referrals. Possible values are FOLLOW, IGNORE, THROW.\n 'Connect Timeout' - Duration of connect timeout. (i.e. 10 secs).\n 'Read Timeout' - Duration of read timeout. (i.e. 10 secs).\n\n 'Url' - Url of the LDAP servier (i.e. ldap://<hostname>:<port>).\n 'User Search Base' - Base DN for searching for users (i.e. CN=Users,DC=example,DC=com).\n 'User Search Filter' - Filter for searching for users against the 'User Search Base'.\n (i.e. sAMAccountName={0}). The user specified name is inserted into '{0}'.\n\n 'Authentication Expiration' - The duration of how long the user authentication is valid\n for. If the user never logs out, they will be required to log back in following\n this duration.\n -->\n <!-- To enable the ldap-provider remove 2 lines. This is 1 of 2.\n <provider>\n <identifier>ldap-provider</identifier>\n <class>org.apache.nifi.ldap.LdapProvider</class>\n <property name=\"Identity Strategy\">USE_DN</property>\n <property name=\"Authentication Strategy\">START_TLS</property>\n <property name=\"Manager DN\"></property>\n <property name=\"Manager Password\"></property>\n <property name=\"TLS - Keystore\"></property>\n <property name=\"TLS - Keystore Password\"></property>\n <property name=\"TLS - Keystore Type\"></property>\n <property name=\"TLS - Truststore\"></property>\n <property name=\"TLS - Truststore Password\"></property>\n <property name=\"TLS - Truststore Type\"></property>\n <property name=\"TLS - Client Auth\"></property>\n <property name=\"TLS - Protocol\"></property>\n <property name=\"TLS - Shutdown Gracefully\"></property>\n <property name=\"Referral Strategy\">FOLLOW</property>\n <property name=\"Connect Timeout\">10 secs</property>\n <property name=\"Read Timeout\">10 secs</property>\n <property name=\"Url\"></property>\n <property name=\"User Search Base\"></property>\n <property name=\"User Search Filter\"></property>\n <property name=\"Authentication Expiration\">12 hours</property>\n </provider>\n To enable the ldap-provider remove 2 lines. This is 2 of 2. -->\n <!--\n Identity Provider for users logging in with username/password against a Kerberos KDC server.\n\n 'Default Realm' - Default realm to provide when user enters incomplete user principal (i.e. NIFI.APACHE.ORG).\n 'Authentication Expiration' - The duration of how long the user authentication is valid for. If the user never logs out, they will be required to log back in following this duration.\n -->\n {% if not security_enabled %}\n <!-- To enable the kerberos-provider remove 2 lines. This is 1 of 2.\n {% endif %}\n <provider>\n <identifier>kerberos-provider</identifier>\n <class>org.apache.nifi.kerberos.KerberosProvider</class>\n <property name=\"Default Realm\">{{nifi_kerberos_realm}}</property>\n <property name=\"Authentication Expiration\">{{nifi_kerberos_authentication_expiration}}</property>\n </provider>\n {% if not security_enabled %}\n To enable the kerberos-provider remove 2 lines. This is 2 of 2. -->\n {% endif %}\n\n </loginIdentityProviders>"
126 }
127 }
128 },
129 {
130 "ranger-hbase-security" : {
131 "properties_attributes" : { },
132 "properties" : { }
133 }
134 },
135 {
136 "ranger-nifi-audit" : {
137 "properties_attributes" : { },
138 "properties" : { }
139 }
140 },
141 {
142 "nifi-bootstrap-notification-services-env" : {
143 "properties_attributes" : { },
144 "properties" : {
145 "content" : "\n <!--\n Licensed to the Apache Software Foundation (ASF) under one or more\n contributor license agreements. See the NOTICE file distributed with\n this work for additional information regarding copyright ownership.\n The ASF licenses this file to You under the Apache License, Version 2.0\n (the \"License\"); you may not use this file except in compliance with\n the License. You may obtain a copy of the License at\n http://www.apache.org/licenses/LICENSE-2.0\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n -->\n <services>\n <!-- This file is used to define how interested parties are notified when events in NiFi's lifecycle occur. -->\n <!-- The format of this file is:\n <services>\n <service>\n <id>service-identifier</id>\n <class>org.apache.nifi.notifications.DesiredNotificationService</class>\n <property name=\"property name\">property value</property>\n <property name=\"another property\">another property value</property>\n </service>\n </services>\n\n This file can contain 0 to many different service definitions.\n The id can then be referenced from the bootstrap.conf file in order to configure the notification service\n to be used when particular lifecycle events occur.\n -->\n\n <!--\n <service>\n <id>email-notification</id>\n <class>org.apache.nifi.bootstrap.notification.email.EmailNotificationService</class>\n <property name=\"SMTP Hostname\"></property>\n <property name=\"SMTP Port\"></property>\n <property name=\"SMTP Username\"></property>\n <property name=\"SMTP Password\"></property>\n <property name=\"SMTP TLS\"></property>\n <property name=\"From\"></property>\n <property name=\"To\"></property>\n </service>\n -->\n\n <!--\n <service>\n <id>http-notification</id>\n <class>org.apache.nifi.bootstrap.notification.http.HttpNotificationService</class>\n <property name=\"URL\"></property>\n </service>\n -->\n\n </services>"
146 }
147 }
148 },
149 {
150 "storm-worker-log4j" : {
151 "properties_attributes" : { },
152 "properties" : {
153 "storm_wrkr_a1_maxbackupindex" : "9",
154 "storm_wrkr_a1_maxfilesize" : "100",
155 "storm_wrkr_err_maxbackupindex" : "4",
156 "storm_wrkr_out_maxfilesize" : "100",
157 "storm_wrkr_out_maxbackupindex" : "4",
158 "storm_wrkr_err_maxfilesize" : "100",
159 "content" : "\n<!--\n Licensed to the Apache Software Foundation (ASF) under one or more\n contributor license agreements. See the NOTICE file distributed with\n this work for additional information regarding copyright ownership.\n The ASF licenses this file to You under the Apache License, Version 2.0\n (the \"License\"); you may not use this file except in compliance with\n the License. You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n-->\n\n<configuration monitorInterval=\"60\">\n<properties>\n <property name=\"pattern\">%d{yyyy-MM-dd HH:mm:ss.SSS} %c{1.} %t [%p] %msg%n</property>\n <property name=\"patternNoTime\">%msg%n</property>\n <property name=\"patternMetrics\">%d %-8r %m%n</property>\n</properties>\n<appenders>\n <RollingFile name=\"A1\"\n\t\tfileName=\"${sys:workers.artifacts}/${sys:storm.id}/${sys:worker.port}/${sys:logfile.name}\"\n\t\tfilePattern=\"${sys:workers.artifacts}/${sys:storm.id}/${sys:worker.port}/${sys:logfile.name}.%i.gz\">\n <PatternLayout>\n <pattern>${pattern}</pattern>\n </PatternLayout>\n <Policies>\n <SizeBasedTriggeringPolicy size=\"{{storm_wrkr_a1_maxfilesize}} MB\"/> <!-- Or every 100 MB -->\n </Policies>\n <DefaultRolloverStrategy max=\"{{storm_wrkr_a1_maxbackupindex}}\"/>\n </RollingFile>\n <RollingFile name=\"STDOUT\"\n\t\tfileName=\"${sys:workers.artifacts}/${sys:storm.id}/${sys:worker.port}/${sys:logfile.name}.out\"\n\t\tfilePattern=\"${sys:workers.artifacts}/${sys:storm.id}/${sys:worker.port}/${sys:logfile.name}.out.%i.gz\">\n <PatternLayout>\n <pattern>${patternNoTime}</pattern>\n </PatternLayout>\n <Policies>\n <SizeBasedTriggeringPolicy size=\"{{storm_wrkr_out_maxfilesize}} MB\"/> <!-- Or every 100 MB -->\n </Policies>\n <DefaultRolloverStrategy max=\"{{storm_wrkr_out_maxbackupindex}}\"/>\n </RollingFile>\n <RollingFile name=\"STDERR\"\n\t\tfileName=\"${sys:workers.artifacts}/${sys:storm.id}/${sys:worker.port}/${sys:logfile.name}.err\"\n\t\tfilePattern=\"${sys:workers.artifacts}/${sys:storm.id}/${sys:worker.port}/${sys:logfile.name}.err.%i.gz\">\n <PatternLayout>\n <pattern>${patternNoTime}</pattern>\n </PatternLayout>\n <Policies>\n <SizeBasedTriggeringPolicy size=\"{{storm_wrkr_err_maxfilesize}} MB\"/> <!-- Or every 100 MB -->\n </Policies>\n <DefaultRolloverStrategy max=\"{{storm_wrkr_err_maxbackupindex}}\"/>\n </RollingFile>\n <RollingFile name=\"METRICS\"\n\t\tfileName=\"${sys:workers.artifacts}/${sys:storm.id}/${sys:worker.port}/${sys:logfile.name}.metrics\"\n\t\tfilePattern=\"${sys:workers.artifacts}/${sys:storm.id}/${sys:worker.port}/${sys:logfile.name}.metrics.%i.gz\">\n <PatternLayout>\n <pattern>${patternMetrics}</pattern>\n </PatternLayout>\n <Policies>\n <SizeBasedTriggeringPolicy size=\"2 MB\"/>\n </Policies>\n <DefaultRolloverStrategy max=\"9\"/>\n </RollingFile>\n <Syslog name=\"syslog\" format=\"RFC5424\" charset=\"UTF-8\" host=\"localhost\" port=\"514\"\n protocol=\"UDP\" appName=\"[${sys:storm.id}:${sys:worker.port}]\" mdcId=\"mdc\" includeMDC=\"true\"\n facility=\"LOCAL5\" enterpriseNumber=\"18060\" newLine=\"true\" exceptionPattern=\"%rEx{full}\"\n messageId=\"[${sys:user.name}:${sys:logging.sensitivity}]\" id=\"storm\" immediateFail=\"true\" immediateFlush=\"true\"/>\n</appenders>\n<loggers>\n <root level=\"info\"> <!-- We log everything -->\n <appender-ref ref=\"A1\"/>\n <appender-ref ref=\"syslog\"/>\n </root>\n <Logger name=\"org.apache.storm.metric.LoggingMetricsConsumer\" level=\"info\" additivity=\"false\">\n <appender-ref ref=\"METRICS\"/>\n </Logger>\n <Logger name=\"STDERR\" level=\"INFO\">\n <appender-ref ref=\"STDERR\"/>\n <appender-ref ref=\"syslog\"/>\n </Logger>\n <Logger name=\"STDOUT\" level=\"INFO\">\n <appender-ref ref=\"STDOUT\"/>\n <appender-ref ref=\"syslog\"/>\n </Logger>\n</loggers>\n</configuration>"
160 }
161 }
162 },
163 {
164 "hbase-site" : {
165 "properties_attributes" : { },
166 "properties" : {
167 "hbase.coprocessor.master.classes" : "",
168 "hfile.block.cache.size" : "0.4",
169 "hbase.master.port" : "16000",
170 "hbase.hstore.blockingStoreFiles" : "10",
171 "hbase.zookeeper.useMulti" : "true",
172 "dfs.domain.socket.path" : "/var/lib/hadoop-hdfs/dn_socket",
173 "zookeeper.session.timeout" : "90000",
174 "hbase.master.info.bindAddress" : "0.0.0.0",
175 "hbase.master.ui.readonly" : "false",
176 "hbase.custom-extensions.root" : "/hdp/ext/{{major_stack_version}}/hbase",
177 "hbase.local.dir" : "${hbase.tmp.dir}/local",
178 "phoenix.query.timeoutMs" : "60000",
179 "hbase.hregion.majorcompaction.jitter" : "0.50",
180 "hbase.hregion.memstore.flush.size" : "134217728",
181 "hbase.regionserver.wal.codec" : "org.apache.hadoop.hbase.regionserver.wal.WALCellCodec",
182 "hbase.client.retries.number" : "35",
183 "hbase.hregion.max.filesize" : "10737418240",
184 "zookeeper.recovery.retry" : "6",
185 "hbase.hstore.compaction.max" : "10",
186 "hbase.regionserver.global.memstore.size" : "0.4",
187 "hbase.zookeeper.quorum" : "%HOSTGROUP::host_group_1%",
188 "hbase.regionserver.info.port" : "16030",
189 "hbase.client.keyvalue.maxsize" : "1048576",
190 "hbase.tmp.dir" : "/tmp/hbase-${user.name}",
191 "hbase.rpc.protection" : "authentication",
192 "zookeeper.znode.parent" : "/hbase-unsecure",
193 "hbase.master.info.port" : "16010",
194 "hbase.rootdir" : "hdfs://%HOSTGROUP::host_group_1%:8020/apps/hbase/data",
195 "hbase.client.scanner.caching" : "100",
196 "hbase.cluster.distributed" : "true",
197 "hbase.hregion.majorcompaction" : "604800000",
198 "hbase.hstore.compactionThreshold" : "3",
199 "hbase.master.wait.on.regionservers.timeout" : "30000",
200 "hbase.hregion.memstore.mslab.enabled" : "true",
201 "hbase.master.namespace.init.timeout" : "2400000",
202 "hbase.rpc.timeout" : "90000",
203 "hbase.superuser" : "hbase",
204 "hbase.coprocessor.region.classes" : "org.apache.hadoop.hbase.security.access.SecureBulkLoadEndpoint",
205 "hbase.security.authentication" : "simple",
206 "hbase.regionserver.handler.count" : "30",
207 "hbase.bulkload.staging.dir" : "/apps/hbase/staging",
208 "hbase.regionserver.port" : "16020",
209 "hbase.security.authorization" : "false",
210 "hbase.zookeeper.property.clientPort" : "2181",
211 "hbase.regionserver.executor.openregion.threads" : "20",
212 "hbase.defaults.for.version.skip" : "true",
213 "hbase.hregion.memstore.block.multiplier" : "4"
214 }
215 }
216 },
217 {
218 "ranger-yarn-policymgr-ssl" : {
219 "properties_attributes" : { },
220 "properties" : { }
221 }
222 },
223 {
224 "mapred-logsearch-conf" : {
225 "properties_attributes" : { },
226 "properties" : {
227 "component_mappings" : "HISTORYSERVER:mapred_historyserver",
228 "content" : "\n{\n \"input\":[\n {\n \"type\":\"mapred_historyserver\",\n \"rowtype\":\"service\",\n \"path\":\"{{default('/configurations/mapred-env/mapred_log_dir_prefix', '/var/log/hadoop')}}/{{default('configurations/mapred-env/mapred_user', 'mapred')}}/mapred-{{default('configurations/mapred-env/mapred_user', 'mapred')}}-historyserver*.log\"\n }\n ],\n \"filter\":[\n {\n \"filter\":\"grok\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"mapred_historyserver\"\n ]\n }\n },\n \"log4j_format\":\"%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n\",\n \"multiline_pattern\":\"^(%{TIMESTAMP_ISO8601:logtime})\",\n \"message_pattern\":\"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\\\(%{JAVAFILE:file}:%{JAVAMETHOD:method}\\\\(%{INT:line_number}\\\\)\\\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}\",\n \"post_map_values\":{\n \"logtime\":{\n \"map_date\":{\n \"target_date_pattern\":\"yyyy-MM-dd HH:mm:ss,SSS\"\n }\n }\n }\n }\n ]\n }",
229 "service_name" : "MapReduce"
230 }
231 }
232 },
233 {
234 "nifi-ambari-ssl-config" : {
235 "properties_attributes" : { },
236 "properties" : {
237 "nifi.toolkit.dn.prefix" : "CN=",
238 "nifi.toolkit.tls.helper.days" : "1095",
239 "nifi.security.keystoreType" : "",
240 "nifi.security.keystore" : "{{nifi_config_dir}}/keystore.jks",
241 "nifi.node.ssl.isenabled" : "false",
242 "content" : "\n<!-- Provide the identity (typically a DN) of each node when clustered (see tool tip for detailed description of Node Identity). Must be specified when Ranger Nifi plugin will not be used for authorization. -->\n\n<!--\n<property name=\"Node Identity 1\"></property>\n<property name=\"Node Identity 2\"></property>\n<property name=\"Node Identity 3\"></property>\n<property name=\"Node Identity 4\"></property>\n-->",
243 "nifi.toolkit.tls.regenerate" : "false",
244 "nifi.toolkit.dn.suffix" : ", OU=NIFI",
245 "nifi.initial.admin.identity" : "",
246 "nifi.security.truststore" : "{{nifi_config_dir}}/truststore.jks",
247 "nifi.security.needClientAuth" : "false",
248 "nifi.security.truststoreType" : "",
249 "nifi.toolkit.tls.port" : "10443"
250 }
251 }
252 },
253 {
254 "zookeeper-logsearch-conf" : {
255 "properties_attributes" : { },
256 "properties" : {
257 "component_mappings" : "ZOOKEEPER_SERVER:zookeeper",
258 "content" : "\n{\n \"input\":[\n {\n \"type\":\"zookeeper\",\n \"rowtype\":\"service\",\n \"path\":\"{{default('/configurations/zookeeper-env/zk_log_dir', '/var/log/zookeeper')}}/zookeeper*.log\"\n }\n ],\n \"filter\":[\n {\n \"filter\":\"grok\",\n \"conditions\":{\n \"fields\":{\"type\":[\"zookeeper\"]}\n },\n \"log4j_format\":\"%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n\",\n \"multiline_pattern\":\"^(%{TIMESTAMP_ISO8601:logtime})\",\n \"message_pattern\":\"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}-%{SPACE}%{LOGLEVEL:level}%{SPACE}\\\\[%{DATA:thread_name}\\\\@%{INT:line_number}\\\\]%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}\",\n \"post_map_values\": {\n \"logtime\": {\n \"map_date\":{\n \"target_date_pattern\":\"yyyy-MM-dd HH:mm:ss,SSS\"\n }\n }\n }\n }\n ]\n}",
259 "service_name" : "Zookeeper"
260 }
261 }
262 },
263 {
264 "ssl-server" : {
265 "properties_attributes" : { },
266 "properties" : {
267 "ssl.server.truststore.location" : "/etc/security/serverKeys/all.jks",
268 "ssl.server.truststore.reload.interval" : "10000",
269 "ssl.server.truststore.type" : "jks",
270 "ssl.server.keystore.location" : "/etc/security/serverKeys/keystore.jks",
271 "ssl.server.keystore.type" : "jks"
272 }
273 }
274 },
275 {
276 "streamline-common" : {
277 "properties_attributes" : { },
278 "properties" : {
279 "jar.storage.type" : "local",
280 "database_name" : "streamline",
281 "streamline.storage.connector.user" : "streamline",
282 "streamline.storage.type" : "mysql",
283 "registry.url" : "http://localhost:7788/api/v1",
284 "streamline.storage.connector.connectURI" : "jdbc:mysql://localhost:3306/streamline",
285 "storm.client.home" : "/usr/hdf/current/storm-client",
286 "jar.storage" : "/hdf/streamline/jars",
287 "maven.repo.url" : "hwx-public^http://repo.hortonworks.com/content/groups/public/,hwx-private^http://nexus-private.hortonworks.com/nexus/content/groups/public/",
288 "jar.storage.hdfs.url" : "hdfs://localhost:9090",
289 "httpProxyServer" : "",
290 "httpProxyUsername" : "",
291 "streamline.storage.query.timeout" : "30",
292 "port" : "7777",
293 "streamline.dashboard.url" : "http://localhost:9088",
294 "adminPort" : "7778",
295 "topology.test.results.dir" : "/hdf/streamline/topology_test_results"
296 }
297 }
298 },
299 {
300 "nifi-ambari-config" : {
301 "properties_attributes" : { },
302 "properties" : {
303 "nifi.node.ssl.port" : "9091",
304 "nifi.ambari_reporting_frequency" : "1 mins",
305 "nifi.internal.dir" : "/var/lib/nifi",
306 "nifi.max_mem" : "512m",
307 "nifi.content.repository.dir.default" : "/var/lib/nifi/content_repository",
308 "nifi.node.protocol.port" : "9088",
309 "nifi.state.dir" : "{nifi_internal_dir}/state/local",
310 "nifi.flow.config.dir" : "{nifi_internal_dir}/conf",
311 "nifi.ambari_reporting_enabled" : "true",
312 "nifi.config.dir" : "{nifi_install_dir}/conf",
313 "nifi.initial_mem" : "512m",
314 "nifi.node.port" : "9090",
315 "nifi.flowfile.repository.dir" : "/var/lib/nifi/flowfile_repository",
316 "nifi.nifi_znode" : "/nifi",
317 "nifi.provenance.repository.dir.default" : "/var/lib/nifi/provenance_repository",
318 "nifi.database.dir" : "/var/lib/nifi/database_repository"
319 }
320 }
321 },
322 {
323 "nifi-authorizers-env" : {
324 "properties_attributes" : { },
325 "properties" : {
326 "content" : "\n <!--\n Licensed to the Apache Software Foundation (ASF) under one or more\n contributor license agreements. See the NOTICE file distributed with\n this work for additional information regarding copyright ownership.\n The ASF licenses this file to You under the Apache License, Version 2.0\n (the \"License\"); you may not use this file except in compliance with\n the License. You may obtain a copy of the License at\n http://www.apache.org/licenses/LICENSE-2.0\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n -->\n <!--\n This file lists the authority providers to use when running securely. In order\n to use a specific provider it must be configured here and it's identifier\n must be specified in the nifi.properties file.\n -->\n <authorizers>\n \n <!--\n The FileAuthorizer is NiFi\"s provided authorizer and has the following properties:\n - Authorizations File - The file where the FileAuthorizer will store policies.\n - Users File - The file where the FileAuthorizer will store users and groups.\n - Initial Admin Identity - The identity of an initial admin user that will be granted access to the UI and\n given the ability to create additional users, groups, and policies. The value of this property could be\n a DN when using certificates or LDAP, or a Kerberos principal. This property will only be used when there\n are no other users, groups, and policies defined. If this property is specified then a Legacy Authorized\n Users File can not be specified.\n NOTE: Any identity mapping rules specified in nifi.properties will also be applied to the initial admin identity,\n so the value should be the unmapped identity.\n - Legacy Authorized Users File - The full path to an existing authorized-users.xml that will be automatically\n converted to the new authorizations model. If this property is specified then an Initial Admin Identity can\n not be specified, and this property will only be used when there are no other users, groups, and policies defined.\n - Node Identity [unique key] - The identity of a NiFi cluster node. When clustered, a property for each node\n should be defined, so that every node knows about every other node. If not clustered these properties can be ignored.\n The name of each property must be unique, for example for a three node cluster:\n \"Node Identity A\", \"Node Identity B\", \"Node Identity C\" or \"Node Identity 1\", \"Node Identity 2\", \"Node Identity 3\"\n NOTE: Any identity mapping rules specified in nifi.properties will also be applied to the node identities,\n so the values should be the unmapped identities (i.e. full DN from a certificate). \n -->\n <authorizer>\n <identifier>{{nifi_authorizer}}</identifier>\n {% if has_ranger_admin and enable_ranger_nifi %}\n <class>org.apache.nifi.ranger.authorization.RangerNiFiAuthorizer</class>\n <property name=\"Ranger Audit Config Path\">{{nifi_config_dir}}/ranger-nifi-audit.xml</property>\n <property name=\"Ranger Security Config Path\">{{nifi_config_dir}}/ranger-nifi-security.xml</property>\n <property name=\"Ranger Service Type\">nifi</property>\n <property name=\"Ranger Application Id\">nifi</property>\n <property name=\"Allow Anonymous\">true</property>\n <property name=\"Ranger Admin Identity\">{{ranger_admin_identity}}</property>\n {% if security_enabled %}\n <property name=\"Ranger Kerberos Enabled\">true</property>\n {% else %}\n <property name=\"Ranger Kerberos Enabled\">false</property>\n {% endif %}\n {% else %}\n <class>org.apache.nifi.authorization.FileAuthorizer</class>\n <property name=\"Authorizations File\">{{nifi_flow_config_dir}}/authorizations.xml</property>\n <property name=\"Users File\">{{nifi_flow_config_dir}}/users.xml</property>\n <property name=\"Initial Admin Identity\">{{nifi_initial_admin_id}}</property>\n <property name=\"Legacy Authorized Users File\"></property>\n {% endif %}\n\n {{nifi_ssl_config_content}}\n\n </authorizer>\n </authorizers>"
327 }
328 }
329 },
330 {
331 "ranger-yarn-security" : {
332 "properties_attributes" : { },
333 "properties" : { }
334 }
335 },
336 {
337 "registry-common" : {
338 "properties_attributes" : { },
339 "properties" : {
340 "jar.storage.type" : "local",
341 "database_name" : "registry",
342 "registry.storage.query.timeout" : "30",
343 "registry.schema.cache.expiry.interval" : "3600",
344 "registry.storage.connector.connectURI" : "jdbc:mysql://localhost:3306/registry",
345 "jar.storage" : "/hdf/registry",
346 "registry.storage.connector.user" : "registry",
347 "jar.storage.hdfs.url" : "hdfs://localhost:9090",
348 "port" : "7788",
349 "adminPort" : "7789",
350 "registry.schema.cache.size" : "10000",
351 "registry.storage.type" : "mysql"
352 }
353 }
354 },
355 {
356 "hadoop-policy" : {
357 "properties_attributes" : { },
358 "properties" : {
359 "security.inter.datanode.protocol.acl" : "*",
360 "security.refresh.usertogroups.mappings.protocol.acl" : "hadoop",
361 "security.admin.operations.protocol.acl" : "hadoop",
362 "security.client.datanode.protocol.acl" : "*",
363 "security.datanode.protocol.acl" : "*",
364 "security.inter.tracker.protocol.acl" : "*",
365 "security.job.client.protocol.acl" : "*",
366 "security.client.protocol.acl" : "*",
367 "security.job.task.protocol.acl" : "*",
368 "security.refresh.policy.protocol.acl" : "hadoop",
369 "security.namenode.protocol.acl" : "*"
370 }
371 }
372 },
373 {
374 "ranger-yarn-audit" : {
375 "properties_attributes" : { },
376 "properties" : { }
377 }
378 },
379 {
380 "nifi-bootstrap-env" : {
381 "properties_attributes" : { },
382 "properties" : {
383 "content" : "\n#\n# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements. See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License. You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\n# Java command to use when running NiFi\njava=java\n\n# Username to use when running NiFi. This value will be ignored on Windows.\nrun.as={{nifi_user}}\n##run.as=root\n\n# Configure where NiFi's lib and conf directories live\nlib.dir={{nifi_install_dir}}/lib\nconf.dir={{nifi_config_dir}}\n\n# How long to wait after telling NiFi to shutdown before explicitly killing the Process\ngraceful.shutdown.seconds=20\n\n{% if security_enabled %}\njava.arg.0=-Djava.security.auth.login.config={{nifi_jaas_conf}}\n{% endif %}\n\n# Disable JSR 199 so that we can use JSP's without running a JDK\njava.arg.1=-Dorg.apache.jasper.compiler.disablejsr199=true\n\n# JVM memory settings\njava.arg.2=-Xms{{nifi_initial_mem}}\njava.arg.3=-Xmx{{nifi_max_mem}}\n\n# Enable Remote Debugging\n#java.arg.debug=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=8000\n\njava.arg.4=-Djava.net.preferIPv4Stack=true\n\n# allowRestrictedHeaders is required for Cluster/Node communications to work properly\njava.arg.5=-Dsun.net.http.allowRestrictedHeaders=true\njava.arg.6=-Djava.protocol.handler.pkgs=sun.net.www.protocol\n\n# The G1GC is still considered experimental but has proven to be very advantageous in providing great\n# performance without significant \"stop-the-world\" delays.\njava.arg.13=-XX:+UseG1GC\n\n#Set headless mode by default\njava.arg.14=-Djava.awt.headless=true\n\n#Ambari Metrics Collector URL - passed in to flow.xml for AmbariReportingTask\njava.arg.15=-Dambari.metrics.collector.url=http://{{metrics_collector_host}}:{{metrics_collector_port}}/ws/v1/timeline/metrics\n\n#Application ID - used in flow.xml - passed into flow.xml for AmbariReportingTask\njava.arg.16=-Dambari.application.id=nifi\n\n#Sets the provider of SecureRandom to /dev/urandom to prevent blocking on VMs\njava.arg.17=-Djava.security.egd=file:/dev/urandom\n\n###\n# Notification Services for notifying interested parties when NiFi is stopped, started, dies\n###\n\n# XML File that contains the definitions of the notification services\nnotification.services.file={{nifi_config_dir}}/bootstrap-notification-services.xml\n\n# In the case that we are unable to send a notification for an event, how many times should we retry?\nnotification.max.attempts=5\n\n# Comma-separated list of identifiers that are present in the notification.services.file; which services should be used to notify when NiFi is started?\n#nifi.start.notification.services=email-notification\n\n# Comma-separated list of identifiers that are present in the notification.services.file; which services should be used to notify when NiFi is stopped?\n#nifi.stop.notification.services=email-notification\n\n# Comma-separated list of identifiers that are present in the notification.services.file; which services should be used to notify when NiFi dies?\n#nifi.dead.notification.services=email-notification"
384 }
385 }
386 },
387 {
388 "ranger-nifi-plugin-properties" : {
389 "properties_attributes" : { },
390 "properties" : { }
391 }
392 },
393 {
394 "storm-cluster-log4j" : {
395 "properties_attributes" : { },
396 "properties" : {
397 "storm_a1_maxbackupindex" : "9",
398 "content" : "\n<!--\n Licensed to the Apache Software Foundation (ASF) under one or more\n contributor license agreements. See the NOTICE file distributed with\n this work for additional information regarding copyright ownership.\n The ASF licenses this file to You under the Apache License, Version 2.0\n (the \"License\"); you may not use this file except in compliance with\n the License. You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n-->\n<configuration monitorInterval=\"60\">\n<properties>\n <property name=\"pattern\">%d{yyyy-MM-dd HH:mm:ss.SSS} %c{1.} %t [%p] %msg%n</property>\n</properties>\n<appenders>\n <RollingFile name=\"A1\" immediateFlush=\"false\"\n fileName=\"${sys:storm.log.dir}/${sys:logfile.name}\"\n filePattern=\"${sys:storm.log.dir}/${sys:logfile.name}.%i.gz\">\n <PatternLayout>\n <pattern>${pattern}</pattern>\n </PatternLayout>\n <Policies>\n <SizeBasedTriggeringPolicy size=\"{{storm_a1_maxfilesize}} MB\"/> <!-- Or every 100 MB -->\n </Policies>\n <DefaultRolloverStrategy max=\"{{storm_a1_maxbackupindex}}\"/>\n </RollingFile>\n <RollingFile name=\"WEB-ACCESS\" immediateFlush=\"false\"\n fileName=\"${sys:storm.log.dir}/access-web-${sys:daemon.name}.log\"\n filePattern=\"${sys:storm.log.dir}/access-web-${sys:daemon.name}.log.%i.gz\">\n <PatternLayout>\n <pattern>${pattern}</pattern>\n </PatternLayout>\n <Policies>\n <SizeBasedTriggeringPolicy size=\"100 MB\"/> <!-- Or every 100 MB -->\n </Policies>\n <DefaultRolloverStrategy max=\"9\"/>\n </RollingFile>\n <RollingFile name=\"THRIFT-ACCESS\" immediateFlush=\"false\"\n fileName=\"${sys:storm.log.dir}/access-${sys:logfile.name}\"\n filePattern=\"${sys:storm.log.dir}/access-${sys:logfile.name}.%i.gz\">\n <PatternLayout>\n <pattern>${pattern}</pattern>\n </PatternLayout>\n <Policies>\n <SizeBasedTriggeringPolicy size=\"100 MB\"/> <!-- Or every 100 MB -->\n </Policies>\n <DefaultRolloverStrategy max=\"9\"/>\n </RollingFile>\n <Syslog name=\"syslog\" format=\"RFC5424\" charset=\"UTF-8\" host=\"localhost\" port=\"514\"\n protocol=\"UDP\" appName=\"[${sys:daemon.name}]\" mdcId=\"mdc\" includeMDC=\"true\"\n facility=\"LOCAL5\" enterpriseNumber=\"18060\" newLine=\"true\" exceptionPattern=\"%rEx{full}\"\n messageId=\"[${sys:user.name}:S0]\" id=\"storm\" immediateFlush=\"true\" immediateFail=\"true\"/>\n</appenders>\n<loggers>\n\n <Logger name=\"org.apache.storm.logging.filters.AccessLoggingFilter\" level=\"info\" additivity=\"false\">\n <AppenderRef ref=\"WEB-ACCESS\"/>\n <AppenderRef ref=\"syslog\"/>\n </Logger>\n <Logger name=\"org.apache.storm.logging.ThriftAccessLogger\" level=\"info\" additivity=\"false\">\n <AppenderRef ref=\"THRIFT-ACCESS\"/>\n <AppenderRef ref=\"syslog\"/>\n </Logger>\n <root level=\"info\"> <!-- We log everything -->\n <appender-ref ref=\"A1\"/>\n <appender-ref ref=\"syslog\"/>\n </root>\n</loggers>\n</configuration>",
399 "storm_a1_maxfilesize" : "100"
400 }
401 }
402 },
403 {
404 "core-site" : {
405 "properties_attributes" : {
406 "final" : {
407 "fs.defaultFS" : "true"
408 }
409 },
410 "properties" : {
411 "fs.defaultFS" : "hdfs://%HOSTGROUP::host_group_1%:8020",
412 "ha.failover-controller.active-standby-elector.zk.op.retries" : "120",
413 "hadoop.security.authentication" : "simple",
414 "ipc.server.tcpnodelay" : "true",
415 "hadoop.proxyuser.hdfs.hosts" : "*",
416 "mapreduce.jobtracker.webinterface.trusted" : "false",
417 "hadoop.security.auth_to_local" : "DEFAULT",
418 "ipc.client.connect.max.retries" : "50",
419 "hadoop.proxyuser.root.groups" : "*",
420 "io.file.buffer.size" : "131072",
421 "ipc.client.idlethreshold" : "8000",
422 "hadoop.proxyuser.hdfs.groups" : "*",
423 "fs.trash.interval" : "360",
424 "hadoop.http.authentication.simple.anonymous.allowed" : "true",
425 "hadoop.security.authorization" : "false",
426 "net.topology.script.file.name" : "/etc/hadoop/conf/topology_script.py",
427 "io.compression.codecs" : "org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.SnappyCodec",
428 "ipc.client.connection.maxidletime" : "30000",
429 "hadoop.custom-extensions.root" : "/hdp/ext/{{major_stack_version}}/hadoop",
430 "hadoop.proxyuser.root.hosts" : "sandbox-hdf.hortonworks.com",
431 "io.serializations" : "org.apache.hadoop.io.serializer.WritableSerialization"
432 }
433 }
434 },
435 {
436 "druid-common" : {
437 "properties_attributes" : { },
438 "properties" : {
439 "druid.storage.storageDirectory" : "/user/druid/data",
440 "druid.metadata.storage.connector.user" : "druid",
441 "druid.discovery.curator.path" : "/druid/discovery",
442 "druid.monitoring.monitors" : "[\"com.metamx.metrics.JvmMonitor\"]",
443 "druid.metadata.storage.connector.port" : "1527",
444 "druid.emitter.ambari-metrics.eventConverter" : "{\"type\":\"whiteList\"}",
445 "druid.indexer.logs.type" : "hdfs",
446 "druid.extensions.loadList" : "[\"druid-datasketches\", \"druid-hdfs-storage\", \"druid-kafka-indexing-service\"]",
447 "druid.security.extensions.loadList" : "[]",
448 "druid.zk.service.host" : "sandbox-hdf.hortonworks.com:2181",
449 "druid.extensions.pullList" : "[]",
450 "druid.emitter.ambari-metrics.trustStorePath" : "{{metric_truststore_path}}",
451 "druid.emitter.ambari-metrics.hostname" : "{{metric_collector_host}}",
452 "druid.extensions.repositoryList" : "[]",
453 "database_name" : "druid",
454 "druid.zk.paths.base" : "/druid",
455 "metastore_hostname" : "localhost",
456 "druid.emitter.ambari-metrics.protocol" : "{{metric_collector_protocol}}",
457 "druid.metadata.storage.connector.connectURI" : "jdbc:derby://localhost:1527/druid;create=true",
458 "druid.metadata.storage.type" : "derby",
459 "druid.indexer.logs.directory" : "/user/druid/logs",
460 "druid.emitter" : "{{metric_emitter_type}}",
461 "druid.emitter.ambari-metrics.port" : "{{metric_collector_port}}",
462 "druid.storage.type" : "hdfs",
463 "druid.emitter.ambari-metrics.trustStoreType" : "{{metric_truststore_type}}"
464 }
465 }
466 },
467 {
468 "ranger-nifi-security" : {
469 "properties_attributes" : { },
470 "properties" : { }
471 }
472 },
473 {
474 "hdfs-site" : {
475 "properties_attributes" : {
476 "final" : {
477 "dfs.webhdfs.enabled" : "true",
478 "dfs.namenode.http-address" : "true",
479 "dfs.support.append" : "true",
480 "dfs.namenode.name.dir" : "true",
481 "dfs.datanode.failed.volumes.tolerated" : "true",
482 "dfs.datanode.data.dir" : "true"
483 }
484 },
485 "properties" : {
486 "dfs.replication" : "3",
487 "dfs.namenode.audit.log.async" : "true",
488 "dfs.namenode.checkpoint.dir" : "/hadoop/hdfs/namesecondary",
489 "dfs.namenode.avoid.read.stale.datanode" : "true",
490 "manage.include.files" : "false",
491 "dfs.journalnode.http-address" : "0.0.0.0:8480",
492 "nfs.file.dump.dir" : "/tmp/.hdfs-nfs",
493 "dfs.namenode.rpc-address" : "%HOSTGROUP::host_group_1%:8020",
494 "dfs.namenode.https-address" : "%HOSTGROUP::host_group_1%:50470",
495 "dfs.encrypt.data.transfer.cipher.suites" : "AES/CTR/NoPadding",
496 "dfs.client.read.shortcircuit.streams.cache.size" : "4096",
497 "dfs.hosts.exclude" : "/etc/hadoop/conf/dfs.exclude",
498 "dfs.namenode.accesstime.precision" : "0",
499 "dfs.namenode.fslock.fair" : "false",
500 "dfs.permissions.enabled" : "true",
501 "dfs.datanode.balance.bandwidthPerSec" : "6250000",
502 "dfs.namenode.stale.datanode.interval" : "30000",
503 "dfs.content-summary.limit" : "5000",
504 "dfs.http.policy" : "HTTP_ONLY",
505 "dfs.journalnode.https-address" : "0.0.0.0:8481",
506 "dfs.datanode.du.reserved" : "8421447168",
507 "dfs.domain.socket.path" : "/var/lib/hadoop-hdfs/dn_socket",
508 "dfs.datanode.ipc.address" : "0.0.0.0:8010",
509 "dfs.cluster.administrators" : " hdfs",
510 "dfs.datanode.max.transfer.threads" : "16384",
511 "dfs.namenode.handler.count" : "100",
512 "dfs.https.port" : "50470",
513 "dfs.replication.max" : "50",
514 "dfs.client.read.shortcircuit" : "true",
515 "dfs.webhdfs.enabled" : "true",
516 "dfs.namenode.http-address" : "%HOSTGROUP::host_group_1%:50070",
517 "dfs.namenode.name.dir" : "/hadoop/hdfs/namenode",
518 "dfs.namenode.avoid.write.stale.datanode" : "true",
519 "dfs.datanode.https.address" : "0.0.0.0:50475",
520 "dfs.datanode.failed.volumes.tolerated" : "0",
521 "dfs.client.retry.policy.enabled" : "false",
522 "hadoop.caller.context.enabled" : "true",
523 "dfs.namenode.startup.delay.block.deletion.sec" : "3600",
524 "dfs.block.access.token.enable" : "true",
525 "dfs.datanode.data.dir" : "/hadoop/hdfs/data",
526 "dfs.permissions.superusergroup" : "hdfs",
527 "dfs.blocksize" : "134217728",
528 "dfs.namenode.checkpoint.edits.dir" : "${dfs.namenode.checkpoint.dir}",
529 "nfs.exports.allowed.hosts" : "* rw",
530 "dfs.datanode.address" : "0.0.0.0:50010",
531 "dfs.blockreport.initialDelay" : "120",
532 "dfs.datanode.data.dir.perm" : "750",
533 "dfs.namenode.write.stale.datanode.ratio" : "1.0f",
534 "dfs.namenode.name.dir.restore" : "true",
535 "dfs.heartbeat.interval" : "3",
536 "dfs.namenode.secondary.http-address" : "%HOSTGROUP::host_group_1%:50090",
537 "dfs.namenode.checkpoint.txns" : "1000000",
538 "dfs.journalnode.edits.dir" : "/hadoop/hdfs/journalnode",
539 "dfs.support.append" : "true",
540 "fs.permissions.umask-mode" : "022",
541 "dfs.namenode.safemode.threshold-pct" : "1",
542 "dfs.namenode.checkpoint.period" : "21600",
543 "dfs.datanode.http.address" : "0.0.0.0:50075"
544 }
545 }
546 },
547 {
548 "storm-logsearch-conf" : {
549 "properties_attributes" : { },
550 "properties" : {
551 "component_mappings" : "NIMBUS:storm_nimbus;SUPERVISOR:storm_supervisor,storm_worker,storm_logviewer;STORM_UI_SERVER:storm_ui;DRPC_SERVER:storm_drpc",
552 "content" : "\n{\n \"input\":[\n {\n \"type\":\"storm_drpc\",\n \"rowtype\":\"service\",\n \"path\":\"{{default('/configurations/storm-env/storm_log_dir', '/var/log/storm')}}/drpc.log\"\n },\n {\n \"type\":\"storm_logviewer\",\n \"rowtype\":\"service\",\n \"path\":\"{{default('/configurations/storm-env/storm_log_dir', '/var/log/storm')}}/logviewer.log\"\n },\n {\n \"type\":\"storm_nimbus\",\n \"rowtype\":\"service\",\n \"path\":\"{{default('/configurations/storm-env/storm_log_dir', '/var/log/storm')}}/nimbus.log\"\n },\n {\n \"type\":\"storm_supervisor\",\n \"rowtype\":\"service\",\n \"path\":\"{{default('/configurations/storm-env/storm_log_dir', '/var/log/storm')}}/supervisor.log\"\n },\n {\n \"type\":\"storm_ui\",\n \"rowtype\":\"service\",\n \"path\":\"{{default('/configurations/storm-env/storm_log_dir', '/var/log/storm')}}/ui.log\"\n },\n {\n \"type\":\"storm_worker\",\n \"rowtype\":\"service\",\n \"path\":\"{{default('/configurations/storm-env/storm_log_dir', '/var/log/storm')}}/*worker*.log\"\n }\n ],\n \"filter\":[\n {\n \"filter\":\"grok\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"storm_drpc\",\n \"storm_logviewer\",\n \"storm_nimbus\",\n \"storm_supervisor\",\n \"storm_ui\",\n \"storm_worker\"\n ]\n }\n },\n \"log4j_format\":\"\",\n \"multiline_pattern\":\"^(%{TIMESTAMP_ISO8601:logtime})\",\n \"message_pattern\":\"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\\\[%{LOGLEVEL:level}\\\\]%{SPACE}%{SPACE}%{GREEDYDATA:log_message}\",\n \"post_map_values\":{\n \"logtime\":{\n \"map_date\":{\n \"target_date_pattern\":\"yyyy-MM-dd HH:mm:ss.SSS\"\n }\n }\n }\n }\n ]\n }",
553 "service_name" : "Storm"
554 }
555 }
556 },
557 {
558 "kafka-broker" : {
559 "properties_attributes" : { },
560 "properties" : {
561 "replica.high.watermark.checkpoint.interval.ms" : "5000",
562 "offsets.topic.num.partitions" : "50",
563 "auto.create.topics.enable" : "true",
564 "controller.socket.timeout.ms" : "30000",
565 "external.kafka.metrics.include.prefix" : "kafka.network.RequestMetrics.ResponseQueueTimeMs.request.OffsetCommit.98percentile,kafka.network.RequestMetrics.ResponseQueueTimeMs.request.Offsets.95percentile,kafka.network.RequestMetrics.ResponseSendTimeMs.request.Fetch.95percentile,kafka.network.RequestMetrics.RequestsPerSec.request",
566 "replica.socket.receive.buffer.bytes" : "65536",
567 "min.insync.replicas" : "1",
568 "kafka.timeline.metrics.maxRowCacheSize" : "10000",
569 "zookeeper.connection.timeout.ms" : "25000",
570 "controlled.shutdown.retry.backoff.ms" : "5000",
571 "replica.fetch.wait.max.ms" : "500",
572 "num.recovery.threads.per.data.dir" : "1",
573 "kafka.timeline.metrics.reporter.sendInterval" : "5900",
574 "log.roll.hours" : "168",
575 "default.replication.factor" : "1",
576 "kafka.timeline.metrics.port" : "{{metric_collector_port}}",
577 "offsets.topic.segment.bytes" : "104857600",
578 "fetch.purgatory.purge.interval.requests" : "10000",
579 "replica.socket.timeout.ms" : "30000",
580 "message.max.bytes" : "1000000",
581 "replica.lag.max.messages" : "4000",
582 "kafka.ganglia.metrics.port" : "8671",
583 "num.io.threads" : "8",
584 "offsets.commit.required.acks" : "-1",
585 "delete.topic.enable" : "false",
586 "log.index.interval.bytes" : "4096",
587 "offsets.commit.timeout.ms" : "5000",
588 "log.segment.bytes" : "1073741824",
589 "offset.metadata.max.bytes" : "4096",
590 "zookeeper.connect" : "%HOSTGROUP::host_group_1%:2181",
591 "port" : "6667",
592 "zookeeper.sync.time.ms" : "2000",
593 "num.replica.fetchers" : "1",
594 "log.dirs" : "/kafka-logs",
595 "controlled.shutdown.enable" : "true",
596 "compression.type" : "producer",
597 "offsets.load.buffer.size" : "5242880",
598 "controlled.shutdown.max.retries" : "3",
599 "offsets.topic.replication.factor" : "3",
600 "queued.max.requests" : "500",
601 "auto.leader.rebalance.enable" : "true",
602 "leader.imbalance.check.interval.seconds" : "300",
603 "replica.lag.time.max.ms" : "10000",
604 "socket.request.max.bytes" : "104857600",
605 "zookeeper.session.timeout.ms" : "30000",
606 "kafka.timeline.metrics.truststore.path" : "{{metric_truststore_path}}",
607 "log.retention.bytes" : "-1",
608 "num.network.threads" : "3",
609 "kafka.timeline.metrics.protocol" : "{{metric_collector_protocol}}",
610 "offsets.retention.minutes" : "86400000",
611 "kafka.metrics.reporters" : "",
612 "log.cleanup.interval.mins" : "10",
613 "socket.send.buffer.bytes" : "102400",
614 "log.retention.hours" : "168",
615 "kafka.timeline.metrics.reporter.enabled" : "true",
616 "num.partitions" : "1",
617 "listeners" : "PLAINTEXT://localhost:6667",
618 "socket.receive.buffer.bytes" : "102400",
619 "replica.fetch.min.bytes" : "1",
620 "kafka.timeline.metrics.hosts" : "{{ams_collector_hosts}}",
621 "external.kafka.metrics.exclude.prefix" : "kafka.network.RequestMetrics,kafka.server.DelayedOperationPurgatory,kafka.server.BrokerTopicMetrics.BytesRejectedPerSec,kafka.server.KafkaServer.ClusterId",
622 "controller.message.queue.size" : "10",
623 "kafka.ganglia.metrics.reporter.enabled" : "true",
624 "log.index.size.max.bytes" : "10485760",
625 "offsets.retention.check.interval.ms" : "600000",
626 "producer.purgatory.purge.interval.requests" : "10000",
627 "kafka.ganglia.metrics.group" : "kafka",
628 "offsets.topic.compression.codec" : "0",
629 "replica.fetch.max.bytes" : "1048576",
630 "leader.imbalance.per.broker.percentage" : "10",
631 "kafka.timeline.metrics.truststore.type" : "{{metric_truststore_type}}"
632 }
633 }
634 },
635 {
636 "yarn-site" : {
637 "properties_attributes" : { },
638 "properties" : {
639 "yarn.resourcemanager.zk-address" : "%HOSTGROUP::host_group_1%:2181",
640 "yarn.nodemanager.container-monitor.interval-ms" : "3000",
641 "yarn.timeline-service.entity-group-fs-store.active-dir" : "/ats/active/",
642 "yarn.log.server.web-service.url" : "http://%HOSTGROUP::host_group_1%:8188/ws/v1/applicationhistory",
643 "yarn.nodemanager.linux-container-executor.group" : "hadoop",
644 "yarn.client.failover-proxy-provider" : "org.apache.hadoop.yarn.client.RequestHedgingRMFailoverProxyProvider",
645 "yarn.application.classpath" : "/etc/hadoop/conf,/usr/hdp/current/hadoop-client/*,/usr/hdp/current/hadoop-client/lib/*,/usr/hdp/current/hadoop-hdfs-client/*,/usr/hdp/current/hadoop-hdfs-client/lib/*,/usr/hdp/current/hadoop-yarn-client/*,/usr/hdp/current/hadoop-yarn-client/lib/*,/usr/hdp/current/ext/hadoop/*",
646 "yarn.nodemanager.kill-escape.launch-command-line" : "slider-agent,LLAP",
647 "yarn.admin.acl" : "yarn",
648 "yarn.timeline-service.entity-group-fs-store.cleaner-interval-seconds" : "3600",
649 "yarn.nodemanager.remote-app-log-dir-suffix" : "logs",
650 "yarn.nodemanager.address" : "0.0.0.0:45454",
651 "yarn.resourcemanager.webapp.address" : "%HOSTGROUP::host_group_1%:8088",
652 "yarn.timeline-service.leveldb-timeline-store.ttl-interval-ms" : "300000",
653 "yarn.scheduler.maximum-allocation-vcores" : "3",
654 "yarn.timeline-service.address" : "%HOSTGROUP::host_group_1%:10200",
655 "yarn.timeline-service.enabled" : "true",
656 "yarn.nodemanager.aux-services" : "mapreduce_shuffle,spark_shuffle,spark2_shuffle",
657 "yarn.nodemanager.resource.cpu-vcores" : "3",
658 "yarn.resourcemanager.webapp.delegation-token-auth-filter.enabled" : "false",
659 "yarn.nodemanager.aux-services.spark_shuffle.class" : "org.apache.spark.network.yarn.YarnShuffleService",
660 "yarn.nodemanager.log-aggregation.debug-enabled" : "false",
661 "yarn.resourcemanager.zk-acl" : "world:anyone:rwcda",
662 "yarn.client.nodemanager-connect.max-wait-ms" : "60000",
663 "yarn.http.policy" : "HTTP_ONLY",
664 "yarn.timeline-service.http-authentication.simple.anonymous.allowed" : "true",
665 "yarn.resourcemanager.zk-state-store.parent-path" : "/rmstore",
666 "yarn.timeline-service.leveldb-timeline-store.start-time-read-cache-size" : "10000",
667 "yarn.resourcemanager.admin.address" : "%HOSTGROUP::host_group_1%:8141",
668 "yarn.nodemanager.linux-container-executor.cgroups.strict-resource-usage" : "false",
669 "yarn.resourcemanager.resource-tracker.address" : "%HOSTGROUP::host_group_1%:8025",
670 "yarn.nodemanager.delete.debug-delay-sec" : "0",
671 "yarn.resourcemanager.fs.state-store.retry-policy-spec" : "2000, 500",
672 "yarn.timeline-service.http-authentication.type" : "simple",
673 "yarn.timeline-service.ttl-enable" : "true",
674 "yarn.timeline-service.entity-group-fs-store.retain-seconds" : "604800",
675 "yarn.log-aggregation.retain-seconds" : "2592000",
676 "yarn.nodemanager.resource.memory-mb" : "3072",
677 "yarn.nodemanager.disk-health-checker.min-healthy-disks" : "0.25",
678 "yarn.node-labels.enabled" : "false",
679 "yarn.nodemanager.kill-escape.user" : "hive",
680 "yarn.resourcemanager.connect.max-wait.ms" : "-1",
681 "yarn.resourcemanager.zk-num-retries" : "1000",
682 "yarn.scheduler.minimum-allocation-vcores" : "1",
683 "yarn.nodemanager.aux-services.spark2_shuffle.classpath" : "{{stack_root}}/${hdp.version}/spark2/aux/*",
684 "yarn.resourcemanager.scheduler.class" : "org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler",
685 "yarn.resourcemanager.system-metrics-publisher.enabled" : "true",
686 "yarn.timeline-service.client.max-retries" : "30",
687 "yarn.timeline-service.client.retry-interval-ms" : "1000",
688 "yarn.timeline-service.store-class" : "org.apache.hadoop.yarn.server.timeline.EntityGroupFSTimelineStore",
689 "yarn.timeline-service.bind-host" : "0.0.0.0",
690 "yarn.nodemanager.container-metrics.unregister-delay-ms" : "60000",
691 "yarn.timeline-service.generic-application-history.store-class" : "org.apache.hadoop.yarn.server.applicationhistoryservice.NullApplicationHistoryStore",
692 "yarn.resourcemanager.fs.state-store.uri" : " ",
693 "yarn.resourcemanager.recovery.enabled" : "true",
694 "yarn.timeline-service.entity-group-fs-store.group-id-plugin-classpath" : "",
695 "yarn.resourcemanager.scheduler.address" : "%HOSTGROUP::host_group_1%:8030",
696 "yarn.timeline-service.client.fd-flush-interval-secs" : "5",
697 "yarn.timeline-service.leveldb-timeline-store.start-time-write-cache-size" : "10000",
698 "yarn.timeline-service.ttl-ms" : "2678400000",
699 "yarn.nodemanager.health-checker.script.timeout-ms" : "60000",
700 "yarn.nodemanager.aux-services.spark2_shuffle.class" : "org.apache.spark.network.yarn.YarnShuffleService",
701 "yarn.log.server.url" : "http://%HOSTGROUP::host_group_1%:19888/jobhistory/logs",
702 "yarn.timeline-service.entity-group-fs-store.group-id-plugin-classes" : "",
703 "yarn.resourcemanager.address" : "%HOSTGROUP::host_group_1%:8050",
704 "yarn.timeline-service.recovery.enabled" : "true",
705 "yarn.nodemanager.recovery.dir" : "{{yarn_log_dir_prefix}}/nodemanager/recovery-state",
706 "yarn.resourcemanager.scheduler.monitor.enable" : "false",
707 "hadoop.registry.rm.enabled" : "false",
708 "yarn.resourcemanager.bind-host" : "0.0.0.0",
709 "yarn.nodemanager.log-aggregation.num-log-files-per-app" : "336",
710 "yarn.nodemanager.admin-env" : "MALLOC_ARENA_MAX=$MALLOC_ARENA_MAX",
711 "yarn.resourcemanager.system-metrics-publisher.dispatcher.pool-size" : "10",
712 "yarn.acl.enable" : "false",
713 "manage.include.files" : "false",
714 "yarn.resourcemanager.ha.enabled" : "false",
715 "yarn.timeline-service.entity-group-fs-store.app-cache-size" : "10",
716 "yarn.client.nodemanager-connect.retry-interval-ms" : "10000",
717 "yarn.timeline-service.version" : "1.5",
718 "yarn.timeline-service.leveldb-timeline-store.read-cache-size" : "104857600",
719 "yarn.resourcemanager.monitor.capacity.preemption.natural_termination_factor" : "1",
720 "yarn.nodemanager.local-dirs" : "/hadoop/yarn/local",
721 "yarn.timeline-service.leveldb-timeline-store.path" : "/hadoop/yarn/timeline",
722 "yarn.nodemanager.recovery.enabled" : "true",
723 "yarn.resourcemanager.zk-timeout-ms" : "10000",
724 "yarn.resourcemanager.am.max-attempts" : "2",
725 "yarn.resourcemanager.state-store.max-completed-applications" : "${yarn.resourcemanager.max-completed-applications}",
726 "yarn.node-labels.fs-store.root-dir" : "/system/yarn/node-labels",
727 "yarn.node-labels.fs-store.retry-policy-spec" : "2000, 500",
728 "yarn.nodemanager.aux-services.mapreduce_shuffle.class" : "org.apache.hadoop.mapred.ShuffleHandler",
729 "yarn.log-aggregation-enable" : "true",
730 "yarn.resourcemanager.work-preserving-recovery.enabled" : "true",
731 "yarn.nodemanager.aux-services.spark_shuffle.classpath" : "{{stack_root}}/${hdp.version}/spark/aux/*",
732 "yarn.resourcemanager.store.class" : "org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore",
733 "yarn.timeline-service.leveldb-state-store.path" : "/hadoop/yarn/timeline",
734 "yarn.nodemanager.log.retain-seconds" : "604800",
735 "yarn.timeline-service.entity-group-fs-store.done-dir" : "/ats/done/",
736 "yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage" : "90",
737 "hadoop.registry.zk.quorum" : "%HOSTGROUP::host_group_1%:2181",
738 "yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds" : "3600",
739 "yarn.nodemanager.container-executor.class" : "org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor",
740 "yarn.resourcemanager.nodes.exclude-path" : "/etc/hadoop/conf/yarn.exclude",
741 "yarn.timeline-service.state-store-class" : "org.apache.hadoop.yarn.server.timeline.recovery.LeveldbTimelineStateStore",
742 "yarn.nodemanager.bind-host" : "0.0.0.0",
743 "yarn.resourcemanager.hostname" : "%HOSTGROUP::host_group_1%",
744 "yarn.resourcemanager.monitor.capacity.preemption.total_preemption_per_round" : "1",
745 "yarn.resourcemanager.connect.retry-interval.ms" : "15000",
746 "yarn.timeline-service.webapp.address" : "%HOSTGROUP::host_group_1%:8188",
747 "yarn.scheduler.minimum-allocation-mb" : "1024",
748 "yarn.timeline-service.entity-group-fs-store.summary-store" : "org.apache.hadoop.yarn.server.timeline.RollingLevelDBTimelineStore",
749 "yarn.nodemanager.health-checker.interval-ms" : "135000",
750 "yarn.nodemanager.disk-health-checker.min-free-space-per-disk-mb" : "1000",
751 "yarn.resourcemanager.zk-retry-interval-ms" : "1000",
752 "yarn.scheduler.capacity.ordering-policy.priority-utilization.underutilized-preemption.enabled" : "false",
753 "yarn.nodemanager.remote-app-log-dir" : "/app-logs",
754 "yarn.scheduler.maximum-allocation-mb" : "3072",
755 "yarn.nodemanager.vmem-check-enabled" : "false",
756 "yarn.timeline-service.entity-group-fs-store.scan-interval-seconds" : "15",
757 "yarn.resourcemanager.webapp.https.address" : "%HOSTGROUP::host_group_1%:8090",
758 "yarn.timeline-service.webapp.https.address" : "%HOSTGROUP::host_group_1%:8190",
759 "yarn.nodemanager.resource.percentage-physical-cpu-limit" : "80",
760 "yarn.nodemanager.vmem-pmem-ratio" : "2.1",
761 "yarn.nodemanager.log-aggregation.compression-type" : "gz",
762 "yarn.resourcemanager.work-preserving-recovery.scheduling-wait-ms" : "10000",
763 "yarn.nodemanager.log-dirs" : "/hadoop/yarn/log"
764 }
765 }
766 },
767 {
768 "nifi-flow-env" : {
769 "properties_attributes" : { },
770 "properties" : {
771 "content" : "\n<flowController>\n <maxTimerDrivenThreadCount>10</maxTimerDrivenThreadCount>\n <maxEventDrivenThreadCount>5</maxEventDrivenThreadCount>\n <rootGroup>\n <id>7c84501d-d10c-407c-b9f3-1d80e38fe36a</id>\n <name>NiFi Flow</name>\n <position x=\"0.0\" y=\"0.0\"/>\n <comment/>\n </rootGroup>\n <controllerServices/>\n <reportingTasks>\n <reportingTask>\n <id>3b80ba0f-a6c0-48db-b721-4dbc04cef28e</id>\n <name>AmbariReportingTask</name>\n <comment/>\n <class>org.apache.nifi.reporting.ambari.AmbariReportingTask</class>\n <schedulingPeriod>{{nifi_ambari_reporting_frequency}}</schedulingPeriod>\n <scheduledState>RUNNING</scheduledState>\n <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>\n <property>\n <name>Metrics Collector URL</name>\n <value>${ambari.metrics.collector.url}</value>\n </property>\n <property>\n <name>Application ID</name>\n <value>${ambari.application.id}</value>\n </property>\n <property>\n <name>Hostname</name>\n <value>${hostname(true)}</value>\n </property>\n </reportingTask>\n </reportingTasks>\n</flowController>"
772 }
773 }
774 },
775 {
776 "nifi-jaas-conf" : {
777 "properties_attributes" : { },
778 "properties" : {
779 "content" : "\nNiFiClient {\n com.sun.security.auth.module.Krb5LoginModule required\n useKeyTab=true\n keyTab=\"{{nifi_service_keytab}}\"\n storeKey=true\n useTicketCache=false\n principal=\"{{nifi_service_principal}}\";\n};\nRegistryClient {\n com.sun.security.auth.module.Krb5LoginModule required\n useKeyTab=true\n keyTab=\"{{nifi_service_keytab}}\"\n storeKey=true\n useTicketCache=false\n principal=\"{{nifi_service_principal}}\";\n};"
780 }
781 }
782 },
783 {
784 "registry-log4j" : {
785 "properties_attributes" : { },
786 "properties" : {
787 "registry_log_maxbackupindex" : "20",
788 "registry_log_maxfilesize" : "256",
789 "content" : "\n # The default level of all loggers. Can be OFF, ERROR, WARN, INFO, DEBUG, TRACE, or ALL.\n level: INFO\n # Logger-specific levels.\n loggers:\n # Sets the level for 'com.example.app' to DEBUG.\n com.hortonworks.registry: INFO\n appenders:\n - type: file\n threshold: DEBUG\n logFormat: \"%-6level [%d{HH:mm:ss.SSS}] [%t] %logger{5} - %X{code} %msg %n\"\n currentLogFilename: {{registry_log_dir}}/registry.log\n archivedLogFilenamePattern: {{registry_log_dir}}/registry-%d{yyyy-MM-dd}-%i.log.gz\n archivedFileCount: {{registry_log_maxbackupindex}}\n timeZone: UTC\n maxFileSize: {{registry_log_maxfilesize}}"
790 }
791 }
792 },
793 {
794 "yarn-logsearch-conf" : {
795 "properties_attributes" : { },
796 "properties" : {
797 "component_mappings" : "RESOURCEMANAGER:yarn_resourcemanager,yarn_historyserver,yarn_jobsummary;NODEMANAGER:yarn_nodemanager;APP_TIMELINE_SERVER:yarn_timelineserver",
798 "content" : "\n{\n \"input\":[\n {\n \"type\":\"yarn_nodemanager\",\n \"rowtype\":\"service\",\n \"path\":\"{{default('/configurations/yarn-env/yarn_log_dir_prefix', '/var/log/hadoop')}}/{{default('configurations/yarn-env/yarn_user', 'yarn')}}/yarn-{{default('configurations/yarn-env/yarn_user', 'yarn')}}-nodemanager-*.log\"\n },\n {\n \"type\":\"yarn_resourcemanager\",\n \"rowtype\":\"service\",\n \"path\":\"{{default('/configurations/yarn-env/yarn_log_dir_prefix', '/var/log/hadoop')}}/{{default('configurations/yarn-env/yarn_user', 'yarn')}}/yarn-{{default('configurations/yarn-env/yarn_user', 'yarn')}}-resourcemanager-*.log\"\n },\n {\n \"type\":\"yarn_timelineserver\",\n \"rowtype\":\"service\",\n \"path\":\"{{default('/configurations/yarn-env/yarn_log_dir_prefix', '/var/log/hadoop')}}/{{default('configurations/yarn-env/yarn_user', 'yarn')}}/yarn-{{default('configurations/yarn-env/yarn_user', 'yarn')}}-timelineserver-*.log\"\n },\n {\n \"type\":\"yarn_historyserver\",\n \"rowtype\":\"service\",\n \"path\":\"{{default('/configurations/yarn-env/yarn_log_dir_prefix', '/var/log/hadoop')}}/{{default('configurations/yarn-env/yarn_user', 'yarn')}}/yarn-{{default('configurations/yarn-env/yarn_user', 'yarn')}}-historyserver-*.log\"\n },\n {\n \"type\":\"yarn_jobsummary\",\n \"rowtype\":\"service\",\n \"path\":\"{{default('/configurations/yarn-env/yarn_log_dir_prefix', '/var/log/hadoop')}}/{{default('configurations/yarn-env/yarn_user', 'yarn')}}/hadoop-mapreduce.jobsummary.log\"\n }\n ],\n \"filter\":[\n {\n \"filter\":\"grok\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"yarn_historyserver\",\n \"yarn_jobsummary\",\n \"yarn_nodemanager\",\n \"yarn_resourcemanager\",\n \"yarn_timelineserver\"\n ]\n }\n },\n \"log4j_format\":\"%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n\",\n \"multiline_pattern\":\"^(%{TIMESTAMP_ISO8601:logtime})\",\n \"message_pattern\":\"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\\\(%{JAVAFILE:file}:%{JAVAMETHOD:method}\\\\(%{INT:line_number}\\\\)\\\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}\",\n \"post_map_values\":{\n \"logtime\":{\n \"map_date\":{\n \"target_date_pattern\":\"yyyy-MM-dd HH:mm:ss,SSS\"\n }\n }\n }\n }\n ]\n}",
799 "service_name" : "YARN"
800 }
801 }
802 },
803 {
804 "ranger-yarn-plugin-properties" : {
805 "properties_attributes" : { },
806 "properties" : { }
807 }
808 },
809 {
810 "storm-env" : {
811 "properties_attributes" : { },
812 "properties" : {
813 "storm_user_nofile_limit" : "128000",
814 "jmxremote_port" : "56431",
815 "storm.atlas.hook" : "false",
816 "storm_log_dir" : "/var/log/storm",
817 "storm_pid_dir" : "/var/run/storm",
818 "nimbus_seeds_supported" : "true",
819 "storm_logs_supported" : "true",
820 "storm_user" : "storm",
821 "storm_user_nproc_limit" : "65536",
822 "content" : "\n#!/bin/bash\n\n# Set Storm specific environment variables here.\n\n# The java implementation to use.\nexport JAVA_HOME={{java64_home}}\n\nexport STORM_CONF_DIR={{conf_dir}}\nexport STORM_HOME={{storm_component_home_dir}}\nexport STORM_JAR_JVM_OPTS={{jar_jvm_opts}}\n\n#set storm-auto creds\n# check if storm_jaas.conf in config , only enable storm_auto_creds in secure mode.\nSTORM_JAAS_CONF=$STORM_HOME/conf/storm_jaas.conf\nSTORM_AUTOCREDS_LIB_DIR=$STORM_HOME/external/storm-autocreds\n\nif [ -f $STORM_JAAS_CONF ] && [ -d $STORM_AUTOCREDS_LIB_DIR ]; then\n export STORM_EXT_CLASSPATH=$STORM_AUTOCREDS_LIB_DIR\nfi"
823 }
824 }
825 },
826 {
827 "yarn-env" : {
828 "properties_attributes" : { },
829 "properties" : {
830 "yarn_cgroups_enabled" : "false",
831 "yarn_user_nproc_limit" : "65536",
832 "yarn_log_dir_prefix" : "/var/log/hadoop-yarn",
833 "apptimelineserver_heapsize" : "8072",
834 "yarn_user_nofile_limit" : "32768",
835 "content" : "\n export HADOOP_YARN_HOME={{hadoop_yarn_home}}\n export YARN_LOG_DIR={{yarn_log_dir_prefix}}/$USER\n export YARN_PID_DIR={{yarn_pid_dir_prefix}}/$USER\n export HADOOP_LIBEXEC_DIR={{hadoop_libexec_dir}}\n export JAVA_HOME={{java64_home}}\n export JAVA_LIBRARY_PATH=\"${JAVA_LIBRARY_PATH}:{{hadoop_java_io_tmpdir}}\"\n\n # We need to add the EWMA appender for the yarn daemons only;\n # however, YARN_ROOT_LOGGER is shared by the yarn client and the\n # daemons. This is restrict the EWMA appender to daemons only.\n INVOKER=\"${0##*/}\"\n if [ \"$INVOKER\" == \"yarn-daemon.sh\" ]; then\n export YARN_ROOT_LOGGER=${YARN_ROOT_LOGGER:-INFO,EWMA,RFA}\n fi\n\n # User for YARN daemons\n export HADOOP_YARN_USER=${HADOOP_YARN_USER:-yarn}\n\n # resolve links - $0 may be a softlink\n export YARN_CONF_DIR=\"${YARN_CONF_DIR:-$HADOOP_YARN_HOME/conf}\"\n\n # some Java parameters\n # export JAVA_HOME=/home/y/libexec/jdk1.6.0/\n if [ \"$JAVA_HOME\" != \"\" ]; then\n #echo \"run java in $JAVA_HOME\"\n JAVA_HOME=$JAVA_HOME\n fi\n\n if [ \"$JAVA_HOME\" = \"\" ]; then\n echo \"Error: JAVA_HOME is not set.\"\n exit 1\n fi\n\n JAVA=$JAVA_HOME/bin/java\n JAVA_HEAP_MAX=-Xmx1000m\n\n # For setting YARN specific HEAP sizes please use this\n # Parameter and set appropriately\n YARN_HEAPSIZE={{yarn_heapsize}}\n\n # check envvars which might override default args\n if [ \"$YARN_HEAPSIZE\" != \"\" ]; then\n JAVA_HEAP_MAX=\"-Xmx\"\"$YARN_HEAPSIZE\"\"m\"\n fi\n\n # Resource Manager specific parameters\n\n # Specify the max Heapsize for the ResourceManager using a numerical value\n # in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set\n # the value to 1000.\n # This value will be overridden by an Xmx setting specified in either YARN_OPTS\n # and/or YARN_RESOURCEMANAGER_OPTS.\n # If not specified, the default value will be picked from either YARN_HEAPMAX\n # or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two.\n export YARN_RESOURCEMANAGER_HEAPSIZE={{resourcemanager_heapsize}}\n\n # Specify the JVM options to be used when starting the ResourceManager.\n # These options will be appended to the options specified as YARN_OPTS\n # and therefore may override any similar flags set in YARN_OPTS\n #export YARN_RESOURCEMANAGER_OPTS=\n\n # Node Manager specific parameters\n\n # Specify the max Heapsize for the NodeManager using a numerical value\n # in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set\n # the value to 1000.\n # This value will be overridden by an Xmx setting specified in either YARN_OPTS\n # and/or YARN_NODEMANAGER_OPTS.\n # If not specified, the default value will be picked from either YARN_HEAPMAX\n # or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two.\n export YARN_NODEMANAGER_HEAPSIZE={{nodemanager_heapsize}}\n\n # Specify the max Heapsize for the timeline server using a numerical value\n # in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set\n # the value to 1024.\n # This value will be overridden by an Xmx setting specified in either YARN_OPTS\n # and/or YARN_TIMELINESERVER_OPTS.\n # If not specified, the default value will be picked from either YARN_HEAPMAX\n # or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two.\n export YARN_TIMELINESERVER_HEAPSIZE={{apptimelineserver_heapsize}}\n\n # Specify the JVM options to be used when starting the NodeManager.\n # These options will be appended to the options specified as YARN_OPTS\n # and therefore may override any similar flags set in YARN_OPTS\n #export YARN_NODEMANAGER_OPTS=\n\n # so that filenames w/ spaces are handled correctly in loops below\n IFS=\n\n\n # default log directory and file\n if [ \"$YARN_LOG_DIR\" = \"\" ]; then\n YARN_LOG_DIR=\"$HADOOP_YARN_HOME/logs\"\n fi\n if [ \"$YARN_LOGFILE\" = \"\" ]; then\n YARN_LOGFILE='yarn.log'\n fi\n\n # default policy file for service-level authorization\n if [ \"$YARN_POLICYFILE\" = \"\" ]; then\n YARN_POLICYFILE=\"hadoop-policy.xml\"\n fi\n\n # restore ordinary behaviour\n unset IFS\n\n\n YARN_OPTS=\"$YARN_OPTS -Dhadoop.log.dir=$YARN_LOG_DIR\"\n YARN_OPTS=\"$YARN_OPTS -Dyarn.log.dir=$YARN_LOG_DIR\"\n YARN_OPTS=\"$YARN_OPTS -Dhadoop.log.file=$YARN_LOGFILE\"\n YARN_OPTS=\"$YARN_OPTS -Dyarn.log.file=$YARN_LOGFILE\"\n YARN_OPTS=\"$YARN_OPTS -Dyarn.home.dir=$YARN_COMMON_HOME\"\n YARN_OPTS=\"$YARN_OPTS -Dyarn.id.str=$YARN_IDENT_STRING\"\n YARN_OPTS=\"$YARN_OPTS -Dhadoop.root.logger=${YARN_ROOT_LOGGER:-INFO,console}\"\n YARN_OPTS=\"$YARN_OPTS -Dyarn.root.logger=${YARN_ROOT_LOGGER:-INFO,console}\"\n export YARN_NODEMANAGER_OPTS=\"$YARN_NODEMANAGER_OPTS -Dnm.audit.logger=INFO,NMAUDIT\"\n export YARN_RESOURCEMANAGER_OPTS=\"$YARN_RESOURCEMANAGER_OPTS -Drm.audit.logger=INFO,RMAUDIT\"\n if [ \"x$JAVA_LIBRARY_PATH\" != \"x\" ]; then\n YARN_OPTS=\"$YARN_OPTS -Djava.library.path=$JAVA_LIBRARY_PATH\"\n fi\n YARN_OPTS=\"$YARN_OPTS -Dyarn.policy.file=$YARN_POLICYFILE\"\n YARN_OPTS=\"$YARN_OPTS -Djava.io.tmpdir={{hadoop_java_io_tmpdir}}\"\n {% if rm_security_opts is defined %}\n YARN_OPTS=\"{{rm_security_opts}} $YARN_OPTS\"\n {% endif %}",
836 "is_supported_yarn_ranger" : "true",
837 "service_check.queue.name" : "default",
838 "resourcemanager_heapsize" : "1024",
839 "yarn_pid_dir_prefix" : "/var/run/hadoop-yarn",
840 "yarn_user" : "yarn",
841 "min_user_id" : "1000",
842 "yarn_heapsize" : "1024",
843 "nodemanager_heapsize" : "1024"
844 }
845 }
846 },
847 {
848 "ranger-hdfs-audit" : {
849 "properties_attributes" : { },
850 "properties" : { }
851 }
852 },
853 {
854 "ranger-hdfs-plugin-properties" : {
855 "properties_attributes" : { },
856 "properties" : { }
857 }
858 },
859 {
860 "ranger-storm-plugin-properties" : {
861 "properties_attributes" : { },
862 "properties" : { }
863 }
864 },
865 {
866 "druid-middlemanager" : {
867 "properties_attributes" : { },
868 "properties" : {
869 "druid.indexer.task.hadoopWorkingPath" : "/tmp/druid-indexing",
870 "druid.server.http.numThreads" : "50",
871 "druid.indexer.runner.startPort" : "8100",
872 "druid.worker.capacity" : "3",
873 "druid.processing.numThreads" : "2",
874 "druid.indexer.runner.javaOpts" : "-server -Xmx2g -Duser.timezone=UTC -Dfile.encoding=UTF-8 -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager -Dhdp.version={{stack_version}} -Dhadoop.mapreduce.job.classloader=true",
875 "druid.indexer.task.baseTaskDir" : "/apps/druid/tasks",
876 "druid.processing.buffer.sizeBytes" : "256000000",
877 "druid.service" : "druid/middlemanager",
878 "druid.port" : "8091"
879 }
880 }
881 },
882 {
883 "ranger-storm-policymgr-ssl" : {
884 "properties_attributes" : { },
885 "properties" : { }
886 }
887 },
888 {
889 "ranger-storm-security" : {
890 "properties_attributes" : { },
891 "properties" : { }
892 }
893 },
894 {
895 "kafka_client_jaas_conf" : {
896 "properties_attributes" : { },
897 "properties" : {
898 "content" : "\nKafkaClient {\ncom.sun.security.auth.module.Krb5LoginModule required\nuseTicketCache=true\nrenewTicket=true\nserviceName=\"{{kafka_bare_jaas_principal}}\";\n};"
899 }
900 }
901 },
902 {
903 "hbase-policy" : {
904 "properties_attributes" : { },
905 "properties" : {
906 "security.masterregion.protocol.acl" : "*",
907 "security.admin.protocol.acl" : "*",
908 "security.client.protocol.acl" : "*"
909 }
910 }
911 },
912 {
913 "ranger-kafka-audit" : {
914 "properties_attributes" : { },
915 "properties" : { }
916 }
917 },
918 {
919 "capacity-scheduler" : {
920 "properties_attributes" : { },
921 "properties" : {
922 "capacity-scheduler" : "null",
923 "yarn.scheduler.capacity.root.accessible-node-labels" : "*",
924 "yarn.scheduler.capacity.default.minimum-user-limit-percent" : "100",
925 "yarn.scheduler.capacity.maximum-am-resource-percent" : "0.2",
926 "yarn.scheduler.capacity.root.acl_administer_queue" : "*",
927 "yarn.scheduler.capacity.root.default.acl_administer_jobs" : "*",
928 "yarn.scheduler.capacity.resource-calculator" : "org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator",
929 "yarn.scheduler.capacity.root.default.capacity" : "100",
930 "yarn.scheduler.capacity.root.default.user-limit-factor" : "1",
931 "yarn.scheduler.capacity.root.queues" : "default",
932 "yarn.scheduler.capacity.root.capacity" : "100",
933 "yarn.scheduler.capacity.root.default.acl_submit_applications" : "*",
934 "yarn.scheduler.capacity.root.default.maximum-capacity" : "100",
935 "yarn.scheduler.capacity.node-locality-delay" : "40",
936 "yarn.scheduler.capacity.maximum-applications" : "10000",
937 "yarn.scheduler.capacity.root.default.state" : "RUNNING"
938 }
939 }
940 },
941 {
942 "nifi-env" : {
943 "properties_attributes" : { },
944 "properties" : {
945 "nifi_node_log_dir" : "/var/log/nifi",
946 "nifi_user" : "nifi",
947 "nifi_pid_dir" : "/var/run/nifi",
948 "nifi_user_nproc_limit" : "10000",
949 "nifi_user_nofile_limit" : "50000",
950 "nifi_group" : "nifi",
951 "content" : "\n#!/bin/sh\n#\n# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements. See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License. You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\n# The java implementation to use.\nexport JAVA_HOME={{jdk64_home}}\n\nexport NIFI_HOME={{nifi_install_dir}}\n\n#The directory for the NiFi pid file\nexport NIFI_PID_DIR=\"{{nifi_pid_dir}}\"\n\n#The directory for NiFi log files\nexport NIFI_LOG_DIR=\"{{nifi_node_log_dir}}\""
952 }
953 }
954 },
955 {
956 "hadoop-metrics2.properties" : {
957 "properties_attributes" : { },
958 "properties" : {
959 "content" : "\n# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements. See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License. You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n# syntax: [prefix].[source|sink|jmx].[instance].[options]\n# See package.html for org.apache.hadoop.metrics2 for details\n\n{% if has_ganglia_server %}\n*.period=60\n\n*.sink.ganglia.class=org.apache.hadoop.metrics2.sink.ganglia.GangliaSink31\n*.sink.ganglia.period=10\n\n# default for supportsparse is false\n*.sink.ganglia.supportsparse=true\n\n.sink.ganglia.slope=jvm.metrics.gcCount=zero,jvm.metrics.memHeapUsedM=both\n.sink.ganglia.dmax=jvm.metrics.threadsBlocked=70,jvm.metrics.memHeapUsedM=40\n\n# Hook up to the server\nnamenode.sink.ganglia.servers={{ganglia_server_host}}:8661\ndatanode.sink.ganglia.servers={{ganglia_server_host}}:8659\njobtracker.sink.ganglia.servers={{ganglia_server_host}}:8662\ntasktracker.sink.ganglia.servers={{ganglia_server_host}}:8658\nmaptask.sink.ganglia.servers={{ganglia_server_host}}:8660\nreducetask.sink.ganglia.servers={{ganglia_server_host}}:8660\nresourcemanager.sink.ganglia.servers={{ganglia_server_host}}:8664\nnodemanager.sink.ganglia.servers={{ganglia_server_host}}:8657\nhistoryserver.sink.ganglia.servers={{ganglia_server_host}}:8666\njournalnode.sink.ganglia.servers={{ganglia_server_host}}:8654\nnimbus.sink.ganglia.servers={{ganglia_server_host}}:8649\nsupervisor.sink.ganglia.servers={{ganglia_server_host}}:8650\n\nresourcemanager.sink.ganglia.tagsForPrefix.yarn=Queue\n\n{% endif %}\n\n{% if has_metric_collector %}\n\n*.period={{metrics_collection_period}}\n*.sink.timeline.plugin.urls=file:///usr/lib/ambari-metrics-hadoop-sink/ambari-metrics-hadoop-sink.jar\n*.sink.timeline.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink\n*.sink.timeline.period={{metrics_collection_period}}\n*.sink.timeline.sendInterval={{metrics_report_interval}}000\n*.sink.timeline.slave.host.name={{hostname}}\n*.sink.timeline.zookeeper.quorum={{zookeeper_quorum}}\n*.sink.timeline.protocol={{metric_collector_protocol}}\n*.sink.timeline.port={{metric_collector_port}}\n\n# HTTPS properties\n*.sink.timeline.truststore.path = {{metric_truststore_path}}\n*.sink.timeline.truststore.type = {{metric_truststore_type}}\n*.sink.timeline.truststore.password = {{metric_truststore_password}}\n\ndatanode.sink.timeline.collector.hosts={{ams_collector_hosts}}\nnamenode.sink.timeline.collector.hosts={{ams_collector_hosts}}\nresourcemanager.sink.timeline.collector.hosts={{ams_collector_hosts}}\nnodemanager.sink.timeline.collector.hosts={{ams_collector_hosts}}\njobhistoryserver.sink.timeline.collector.hosts={{ams_collector_hosts}}\njournalnode.sink.timeline.collector.hosts={{ams_collector_hosts}}\nmaptask.sink.timeline.collector.hosts={{ams_collector_hosts}}\nreducetask.sink.timeline.collector.hosts={{ams_collector_hosts}}\napplicationhistoryserver.sink.timeline.collector.hosts={{ams_collector_hosts}}\n\nresourcemanager.sink.timeline.tagsForPrefix.yarn=Queue\n\n{% if is_nn_client_port_configured %}\n# Namenode rpc ports customization\nnamenode.sink.timeline.metric.rpc.client.port={{nn_rpc_client_port}}\n{% endif %}\n{% if is_nn_dn_port_configured %}\nnamenode.sink.timeline.metric.rpc.datanode.port={{nn_rpc_dn_port}}\n{% endif %}\n{% if is_nn_healthcheck_port_configured %}\nnamenode.sink.timeline.metric.rpc.healthcheck.port={{nn_rpc_healthcheck_port}}\n{% endif %}\n\n{% endif %}"
960 }
961 }
962 },
963 {
964 "hbase-env" : {
965 "properties_attributes" : { },
966 "properties" : {
967 "hbase_regionserver_xmn_max" : "4000",
968 "hbase_user" : "hbase",
969 "hbase_master_heapsize" : "1024",
970 "hbase_regionserver_xmn_ratio" : "0.2",
971 "content" : "\n# Set environment variables here.\n\n# The java implementation to use. Java 1.6 required.\nexport JAVA_HOME={{java64_home}}\n\n# HBase Configuration directory\nexport HBASE_CONF_DIR=${HBASE_CONF_DIR:-{{hbase_conf_dir}}}\n\n# Extra Java CLASSPATH elements. Optional.\nexport HBASE_CLASSPATH=${HBASE_CLASSPATH}:{{stack_root}}/current/ext/hbase/*\n\n\n# The maximum amount of heap to use, in MB. Default is 1000.\n# export HBASE_HEAPSIZE=1000\n\n# Extra Java runtime options.\n# Below are what we set by default. May only work with SUN JVM.\n# For more on why as well as other possible settings,\n# see http://wiki.apache.org/hadoop/PerformanceTuning\nexport SERVER_GC_OPTS=\"-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:{{log_dir}}/gc.log-`date +'%Y%m%d%H%M'`\"\n# Uncomment below to enable java garbage collection logging.\n# export HBASE_OPTS=\"$HBASE_OPTS -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:$HBASE_HOME/logs/gc-hbase.log\"\n\n# Uncomment and adjust to enable JMX exporting\n# See jmxremote.password and jmxremote.access in $JRE_HOME/lib/management to configure remote password access.\n# More details at: http://java.sun.com/javase/6/docs/technotes/guides/management/agent.html\n#\n# export HBASE_JMX_BASE=\"-Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false\"\n# If you want to configure BucketCache, specify '-XX: MaxDirectMemorySize=' with proper direct memory size\n# export HBASE_THRIFT_OPTS=\"$HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10103\"\n# export HBASE_ZOOKEEPER_OPTS=\"$HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10104\"\n\n# File naming hosts on which HRegionServers will run. $HBASE_HOME/conf/regionservers by default.\nexport HBASE_REGIONSERVERS=${HBASE_CONF_DIR}/regionservers\n\n# Extra ssh options. Empty by default.\n# export HBASE_SSH_OPTS=\"-o ConnectTimeout=1 -o SendEnv=HBASE_CONF_DIR\"\n\n# Where log files are stored. $HBASE_HOME/logs by default.\nexport HBASE_LOG_DIR={{log_dir}}\n\n# A string representing this instance of hbase. $USER by default.\n# export HBASE_IDENT_STRING=$USER\n\n# The scheduling priority for daemon processes. See 'man nice'.\n# export HBASE_NICENESS=10\n\n# The directory where pid files are stored. /tmp by default.\nexport HBASE_PID_DIR={{pid_dir}}\n\n# Seconds to sleep between slave commands. Unset by default. This\n# can be useful in large clusters, where, e.g., slave rsyncs can\n# otherwise arrive faster than the master can service them.\n# export HBASE_SLAVE_SLEEP=0.1\n\n# Tell HBase whether it should manage it's own instance of Zookeeper or not.\nexport HBASE_MANAGES_ZK=false\n\n{% if java_version < 8 %}\nJDK_DEPENDED_OPTS=\"-XX:PermSize=128m -XX:MaxPermSize=128m\"\n{% endif %}\n\n{% if security_enabled %}\nexport HBASE_OPTS=\"$HBASE_OPTS -XX:+UseConcMarkSweepGC -XX:ErrorFile={{log_dir}}/hs_err_pid%p.log -Djava.security.auth.login.config={{client_jaas_config_file}} -Djava.io.tmpdir={{java_io_tmpdir}}\"\nexport HBASE_MASTER_OPTS=\"$HBASE_MASTER_OPTS -Xmx{{master_heapsize}} -Djava.security.auth.login.config={{master_jaas_config_file}} -Djavax.security.auth.useSubjectCredsOnly=false $JDK_DEPENDED_OPTS\"\nexport HBASE_REGIONSERVER_OPTS=\"$HBASE_REGIONSERVER_OPTS -Xmn{{regionserver_xmn_size}} -XX:CMSInitiatingOccupancyFraction=70 -Xms{{regionserver_heapsize}} -Xmx{{regionserver_heapsize}} -Djava.security.auth.login.config={{regionserver_jaas_config_file}} -Djavax.security.auth.useSubjectCredsOnly=false $JDK_DEPENDED_OPTS\"\nexport PHOENIX_QUERYSERVER_OPTS=\"$PHOENIX_QUERYSERVER_OPTS -Djava.security.auth.login.config={{queryserver_jaas_config_file}}\"\n{% else %}\nexport HBASE_OPTS=\"$HBASE_OPTS -XX:+UseConcMarkSweepGC -XX:ErrorFile={{log_dir}}/hs_err_pid%p.log -Djava.io.tmpdir={{java_io_tmpdir}}\"\nexport HBASE_MASTER_OPTS=\"$HBASE_MASTER_OPTS -Xmx{{master_heapsize}} $JDK_DEPENDED_OPTS\"\nexport HBASE_REGIONSERVER_OPTS=\"$HBASE_REGIONSERVER_OPTS -Xmn{{regionserver_xmn_size}} -XX:CMSInitiatingOccupancyFraction=70 -Xms{{regionserver_heapsize}} -Xmx{{regionserver_heapsize}} $JDK_DEPENDED_OPTS\"\n{% endif %}\n\n# HBase off-heap MaxDirectMemorySize\nexport HBASE_REGIONSERVER_OPTS=\"$HBASE_REGIONSERVER_OPTS {% if hbase_max_direct_memory_size %} -XX:MaxDirectMemorySize={{hbase_max_direct_memory_size}}m {% endif %}\"\nexport HBASE_MASTER_OPTS=\"$HBASE_MASTER_OPTS {% if hbase_max_direct_memory_size %} -XX:MaxDirectMemorySize={{hbase_max_direct_memory_size}}m {% endif %}\"",
972 "hbase_java_io_tmpdir" : "/tmp",
973 "hbase_user_nproc_limit" : "16000",
974 "hbase_pid_dir" : "/var/run/hbase",
975 "hbase_regionserver_shutdown_timeout" : "30",
976 "hbase_log_dir" : "/var/log/hbase",
977 "hbase_regionserver_heapsize" : "1024",
978 "hbase_user_nofile_limit" : "32000",
979 "phoenix_sql_enabled" : "false"
980 }
981 }
982 },
983 {
984 "streamline-log4j" : {
985 "properties_attributes" : { },
986 "properties" : {
987 "streamline_log_maxfilesize" : "256",
988 "content" : "\n # The default level of all loggers. Can be OFF, ERROR, WARN, INFO, DEBUG, TRACE, or ALL.\n level: INFO\n # Logger-specific levels.\n loggers:\n # Sets the level for 'com.example.app' to DEBUG.\n com.hortonworks.streamline: INFO\n appenders:\n - type: file\n threshold: DEBUG\n logFormat: \"%-6level [%d{HH:mm:ss.SSS}] [%t] %logger{5} - %X{code} %msg %n\"\n currentLogFilename: {{streamline_log_dir}}/streamline.log\n archivedLogFilenamePattern: {{streamline_log_dir}}/streamline-%d{yyyy-MM-dd}-%i.log.gz\n archivedFileCount: {{streamline_log_maxbackupindex}}\n timeZone: UTC\n maxFileSize: {{streamline_log_maxfilesize}}",
989 "streamline_log_maxbackupindex" : "20"
990 }
991 }
992 },
993 {
994 "mapred-site" : {
995 "properties_attributes" : { },
996 "properties" : {
997 "mapreduce.map.speculative" : "false",
998 "mapreduce.jobhistory.recovery.store.class" : "org.apache.hadoop.mapreduce.v2.hs.HistoryServerLeveldbStateStoreService",
999 "mapreduce.job.counters.max" : "130",
1000 "mapreduce.reduce.log.level" : "INFO",
1001 "mapreduce.shuffle.port" : "13562",
1002 "yarn.app.mapreduce.am.admin-command-opts" : "-Dhdp.version=${hdp.version}",
1003 "mapreduce.job.emit-timeline-data" : "false",
1004 "mapreduce.jobhistory.recovery.enable" : "true",
1005 "mapreduce.map.log.level" : "INFO",
1006 "yarn.app.mapreduce.am.staging-dir" : "/user",
1007 "mapreduce.reduce.shuffle.merge.percent" : "0.66",
1008 "mapreduce.output.fileoutputformat.compress" : "false",
1009 "mapreduce.admin.map.child.java.opts" : "-server -XX:NewRatio=8 -Djava.net.preferIPv4Stack=true -Dhdp.version=${hdp.version}",
1010 "mapreduce.job.reduce.slowstart.completedmaps" : "0.05",
1011 "mapreduce.jobhistory.http.policy" : "HTTP_ONLY",
1012 "mapreduce.job.queuename" : "default",
1013 "mapreduce.application.framework.path" : "/hdp/apps/${hdp.version}/mapreduce/mapreduce.tar.gz#mr-framework",
1014 "mapreduce.application.classpath" : "$PWD/mr-framework/hadoop/share/hadoop/mapreduce/*:$PWD/mr-framework/hadoop/share/hadoop/mapreduce/lib/*:$PWD/mr-framework/hadoop/share/hadoop/common/*:$PWD/mr-framework/hadoop/share/hadoop/common/lib/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/lib/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/lib/*:$PWD/mr-framework/hadoop/share/hadoop/tools/lib/*:/usr/hdp/${hdp.version}/hadoop/lib/hadoop-lzo-0.6.0.${hdp.version}.jar:/etc/hadoop/conf/secure:/usr/hdp/current/ext/hadoop/*",
1015 "mapreduce.reduce.java.opts" : "-Xmx1638m",
1016 "mapreduce.map.output.compress" : "false",
1017 "mapreduce.reduce.input.buffer.percent" : "0.0",
1018 "mapreduce.jobhistory.intermediate-done-dir" : "/mr-history/tmp",
1019 "yarn.app.mapreduce.am.log.level" : "INFO",
1020 "mapreduce.reduce.shuffle.fetch.retry.interval-ms" : "1000",
1021 "mapreduce.reduce.shuffle.input.buffer.percent" : "0.7",
1022 "mapreduce.reduce.speculative" : "false",
1023 "mapreduce.output.fileoutputformat.compress.type" : "BLOCK",
1024 "mapreduce.task.io.sort.mb" : "573",
1025 "mapreduce.cluster.administrators" : " hadoop",
1026 "mapreduce.reduce.shuffle.fetch.retry.timeout-ms" : "30000",
1027 "mapreduce.jobhistory.address" : "%HOSTGROUP::host_group_1%:10020",
1028 "mapreduce.map.sort.spill.percent" : "0.7",
1029 "mapreduce.map.memory.mb" : "1024",
1030 "mapreduce.admin.reduce.child.java.opts" : "-server -XX:NewRatio=8 -Djava.net.preferIPv4Stack=true -Dhdp.version=${hdp.version}",
1031 "mapreduce.jobhistory.bind-host" : "0.0.0.0",
1032 "mapreduce.task.timeout" : "300000",
1033 "mapreduce.admin.user.env" : "LD_LIBRARY_PATH=/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-{{architecture}}-64",
1034 "mapreduce.am.max-attempts" : "2",
1035 "mapreduce.reduce.memory.mb" : "2048",
1036 "mapreduce.framework.name" : "yarn",
1037 "mapreduce.reduce.shuffle.parallelcopies" : "30",
1038 "mapreduce.jobhistory.done-dir" : "/mr-history/done",
1039 "mapreduce.jobhistory.webapp.address" : "%HOSTGROUP::host_group_1%:19888",
1040 "mapreduce.reduce.shuffle.fetch.retry.enabled" : "1",
1041 "mapreduce.task.io.sort.factor" : "100",
1042 "mapreduce.jobhistory.recovery.store.leveldb.path" : "/hadoop/mapreduce/jhs",
1043 "yarn.app.mapreduce.client.job.max-retries" : "60",
1044 "yarn.app.mapreduce.am.command-opts" : "-Xmx819m -Dhdp.version=${hdp.version}",
1045 "mapreduce.map.java.opts" : "-Xmx819m",
1046 "yarn.app.mapreduce.am.resource.mb" : "1024"
1047 }
1048 }
1049 },
1050 {
1051 "ranger-kafka-policymgr-ssl" : {
1052 "properties_attributes" : { },
1053 "properties" : { }
1054 }
1055 },
1056 {
1057 "ranger-storm-audit" : {
1058 "properties_attributes" : { },
1059 "properties" : { }
1060 }
1061 },
1062 {
1063 "druid-historical" : {
1064 "properties_attributes" : { },
1065 "properties" : {
1066 "druid.segmentCache.infoDir" : "/apps/druid/segmentCache/info_dir",
1067 "druid.server.http.numThreads" : "40",
1068 "druid.segmentCache.locations" : "[{\"path\":\"/apps/druid/segmentCache\",\"maxSize\":300000000000}]",
1069 "druid.processing.numThreads" : "3",
1070 "druid.processing.buffer.sizeBytes" : "1073741824",
1071 "druid.service" : "druid/historical",
1072 "druid.port" : "8083",
1073 "druid.server.maxSize" : "300000000000"
1074 }
1075 }
1076 },
1077 {
1078 "ranger-hbase-policymgr-ssl" : {
1079 "properties_attributes" : { },
1080 "properties" : { }
1081 }
1082 },
1083 {
1084 "streamline_jaas_conf" : {
1085 "properties_attributes" : { },
1086 "properties" : {
1087 "content" : "\nStreamlineServer {\ncom.sun.security.auth.module.Krb5LoginModule required\nuseKeyTab=true\nkeyTab=\"{{streamline_keytab_path}}\"\nstoreKey=true\nuseTicketCache=false\nserviceName=\"{{streamline_bare_principal}}\"\nprincipal=\"{{streamline_jaas_principal}}\";\n};\nStormClient {\ncom.sun.security.auth.module.Krb5LoginModule required\nuseKeyTab=true\nkeyTab=\"{{streamline_keytab_path}}\"\nstoreKey=true\nuseTicketCache=false\nserviceName=\"nimbus\"\nprincipal=\"{{streamline_jaas_principal}}\";\n};\nRegistryClient {\ncom.sun.security.auth.module.Krb5LoginModule required\nuseKeyTab=true\nkeyTab=\"{{streamline_keytab_path}}\"\nstoreKey=true\nuseTicketCache=false\nprincipal=\"{{streamline_jaas_principal}}\";\n};"
1088 }
1089 }
1090 },
1091 {
1092 "druid-router" : {
1093 "properties_attributes" : { },
1094 "properties" : {
1095 "druid.router.http.numConnections" : "20",
1096 "druid.server.http.numThreads" : "50",
1097 "druid.router.tierToBrokerMap" : "{\"_default_tier\":\"druid/broker\"}",
1098 "druid.service" : "druid/router",
1099 "druid.port" : "8888"
1100 }
1101 }
1102 },
1103 {
1104 "hdfs-log4j" : {
1105 "properties_attributes" : { },
1106 "properties" : {
1107 "hadoop_security_log_max_backup_size" : "256",
1108 "hadoop_security_log_number_of_backup_files" : "20",
1109 "hadoop_log_number_of_backup_files" : "10",
1110 "content" : "\n#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership. The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License. You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied. See the License for the\n# specific language governing permissions and limitations\n# under the License.\n#\n\n\n# Define some default values that can be overridden by system properties\n# To change daemon root logger use hadoop_root_logger in hadoop-env\nhadoop.root.logger=INFO,console\nhadoop.log.dir=.\nhadoop.log.file=hadoop.log\n\n\n# Define the root logger to the system property \"hadoop.root.logger\".\nlog4j.rootLogger=${hadoop.root.logger}, EventCounter\n\n# Logging Threshold\nlog4j.threshhold=ALL\n\n#\n# Daily Rolling File Appender\n#\n\nlog4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender\nlog4j.appender.DRFA.File=${hadoop.log.dir}/${hadoop.log.file}\n\n# Rollver at midnight\nlog4j.appender.DRFA.DatePattern=.yyyy-MM-dd\n\n# 30-day backup\n#log4j.appender.DRFA.MaxBackupIndex=30\nlog4j.appender.DRFA.layout=org.apache.log4j.PatternLayout\n\n# Pattern format: Date LogLevel LoggerName LogMessage\nlog4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n\n# Debugging Pattern format\n#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n\n\n\n#\n# console\n# Add \"console\" to rootlogger above if you want to use this\n#\n\nlog4j.appender.console=org.apache.log4j.ConsoleAppender\nlog4j.appender.console.target=System.err\nlog4j.appender.console.layout=org.apache.log4j.PatternLayout\nlog4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n\n\n#\n# TaskLog Appender\n#\n\n#Default values\nhadoop.tasklog.taskid=null\nhadoop.tasklog.iscleanup=false\nhadoop.tasklog.noKeepSplits=4\nhadoop.tasklog.totalLogFileSize=100\nhadoop.tasklog.purgeLogSplits=true\nhadoop.tasklog.logsRetainHours=12\n\nlog4j.appender.TLA=org.apache.hadoop.mapred.TaskLogAppender\nlog4j.appender.TLA.taskId=${hadoop.tasklog.taskid}\nlog4j.appender.TLA.isCleanup=${hadoop.tasklog.iscleanup}\nlog4j.appender.TLA.totalLogFileSize=${hadoop.tasklog.totalLogFileSize}\n\nlog4j.appender.TLA.layout=org.apache.log4j.PatternLayout\nlog4j.appender.TLA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n\n\n#\n#Security audit appender\n#\nhadoop.security.logger=INFO,console\nhadoop.security.log.maxfilesize={{hadoop_security_log_max_backup_size}}MB\nhadoop.security.log.maxbackupindex={{hadoop_security_log_number_of_backup_files}}\nlog4j.category.SecurityLogger=${hadoop.security.logger}\nhadoop.security.log.file=SecurityAuth.audit\nlog4j.additivity.SecurityLogger=false\nlog4j.appender.DRFAS=org.apache.log4j.DailyRollingFileAppender\nlog4j.appender.DRFAS.File=${hadoop.log.dir}/${hadoop.security.log.file}\nlog4j.appender.DRFAS.layout=org.apache.log4j.PatternLayout\nlog4j.appender.DRFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n\nlog4j.appender.DRFAS.DatePattern=.yyyy-MM-dd\n\nlog4j.appender.RFAS=org.apache.log4j.RollingFileAppender\nlog4j.appender.RFAS.File=${hadoop.log.dir}/${hadoop.security.log.file}\nlog4j.appender.RFAS.layout=org.apache.log4j.PatternLayout\nlog4j.appender.RFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n\nlog4j.appender.RFAS.MaxFileSize=${hadoop.security.log.maxfilesize}\nlog4j.appender.RFAS.MaxBackupIndex=${hadoop.security.log.maxbackupindex}\n\n#\n# hdfs audit logging\n#\nhdfs.audit.logger=INFO,console\nlog4j.logger.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=${hdfs.audit.logger}\nlog4j.additivity.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=false\nlog4j.appender.DRFAAUDIT=org.apache.log4j.DailyRollingFileAppender\nlog4j.appender.DRFAAUDIT.File=${hadoop.log.dir}/hdfs-audit.log\nlog4j.appender.DRFAAUDIT.layout=org.apache.log4j.PatternLayout\nlog4j.appender.DRFAAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n\nlog4j.appender.DRFAAUDIT.DatePattern=.yyyy-MM-dd\n\n#\n# NameNode metrics logging.\n# The default is to retain two namenode-metrics.log files up to 64MB each.\n#\nnamenode.metrics.logger=INFO,NullAppender\nlog4j.logger.NameNodeMetricsLog=${namenode.metrics.logger}\nlog4j.additivity.NameNodeMetricsLog=false\nlog4j.appender.NNMETRICSRFA=org.apache.log4j.RollingFileAppender\nlog4j.appender.NNMETRICSRFA.File=${hadoop.log.dir}/namenode-metrics.log\nlog4j.appender.NNMETRICSRFA.layout=org.apache.log4j.PatternLayout\nlog4j.appender.NNMETRICSRFA.layout.ConversionPattern=%d{ISO8601} %m%n\nlog4j.appender.NNMETRICSRFA.MaxBackupIndex=1\nlog4j.appender.NNMETRICSRFA.MaxFileSize=64MB\n\n#\n# mapred audit logging\n#\nmapred.audit.logger=INFO,console\nlog4j.logger.org.apache.hadoop.mapred.AuditLogger=${mapred.audit.logger}\nlog4j.additivity.org.apache.hadoop.mapred.AuditLogger=false\nlog4j.appender.MRAUDIT=org.apache.log4j.DailyRollingFileAppender\nlog4j.appender.MRAUDIT.File=${hadoop.log.dir}/mapred-audit.log\nlog4j.appender.MRAUDIT.layout=org.apache.log4j.PatternLayout\nlog4j.appender.MRAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n\nlog4j.appender.MRAUDIT.DatePattern=.yyyy-MM-dd\n\n#\n# Rolling File Appender\n#\n\nlog4j.appender.RFA=org.apache.log4j.RollingFileAppender\nlog4j.appender.RFA.File=${hadoop.log.dir}/${hadoop.log.file}\n\n# Logfile size and and 30-day backups\nlog4j.appender.RFA.MaxFileSize={{hadoop_log_max_backup_size}}MB\nlog4j.appender.RFA.MaxBackupIndex={{hadoop_log_number_of_backup_files}}\n\nlog4j.appender.RFA.layout=org.apache.log4j.PatternLayout\nlog4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} - %m%n\nlog4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n\n\n\n# Custom Logging levels\n\nhadoop.metrics.log.level=INFO\n#log4j.logger.org.apache.hadoop.mapred.JobTracker=DEBUG\n#log4j.logger.org.apache.hadoop.mapred.TaskTracker=DEBUG\n#log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG\nlog4j.logger.org.apache.hadoop.metrics2=${hadoop.metrics.log.level}\n\n# Jets3t library\nlog4j.logger.org.jets3t.service.impl.rest.httpclient.RestS3Service=ERROR\n\n#\n# Null Appender\n# Trap security logger on the hadoop client side\n#\nlog4j.appender.NullAppender=org.apache.log4j.varia.NullAppender\n\n#\n# Event Counter Appender\n# Sends counts of logging messages at different severity levels to Hadoop Metrics.\n#\nlog4j.appender.EventCounter=org.apache.hadoop.log.metrics.EventCounter\n\n# Removes \"deprecated\" messages\nlog4j.logger.org.apache.hadoop.conf.Configuration.deprecation=WARN\n\n#\n# HDFS block state change log from block manager\n#\n# Uncomment the following to suppress normal block state change\n# messages from BlockManager in NameNode.\n#log4j.logger.BlockStateChange=WARN",
1111 "hadoop_log_max_backup_size" : "256"
1112 }
1113 }
1114 },
1115 {
1116 "nifi-logsearch-conf" : {
1117 "properties_attributes" : { },
1118 "properties" : {
1119 "component_mappings" : "NIFI_MASTER:nifi_app,nifi_bootstrap,nifi_setup,nifi_user;NIFI_CA:nifi_ca",
1120 "content" : "\n{\n \"input\":[\n {\n \"type\":\"nifi_app\",\n \"rowtype\":\"service\",\n \"path\":\"{{nifi_log_dir}}/nifi-app.log\"\n },\n {\n \"type\":\"nifi_bootstrap\",\n \"rowtype\":\"service\",\n \"path\":\"{{nifi_log_dir}}/nifi-bootstrap.log\"\n },\n {\n \"type\":\"nifi_setup\",\n \"rowtype\":\"service\",\n \"path\":\"{{nifi_log_dir}}/nifi-setup.log\"\n },\n {\n \"type\":\"nifi_user\",\n \"rowtype\":\"service\",\n \"path\":\"{{nifi_log_dir}}/nifi-user.log\"\n },\n {\n \"type\":\"nifi_ca\",\n \"rowtype\":\"service\",\n \"path\":\"{{nifi_log_dir}}/nifi-ca.stdout\"\n }\n ],\n \"filter\":[\n {\n \"filter\":\"grok\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"nifi_app\",\n \"nifi_bootstrap\",\n \"nifi_setup\",\n \"nifi_user\",\n \"nifi_ca\"\n ]\n\n }\n\n },\n \"log4j_format\":\"\",\n \"multiline_pattern\":\"^(%{TIMESTAMP_ISO8601:logtime})\",\n \"message_pattern\":\"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}\\\\[%{DATA:thread_name}\\\\]%{SPACE}%{GREEDYDATA:log_message}\",\n \"post_map_values\":{\n \"logtime\":{\n \"map_date\":{\n \"target_date_pattern\":\"yyyy-MM-dd HH:mm:ss,SSS\"\n }\n\n }\n\n }\n\n }\n\n ]\n\n}",
1121 "service_name" : "NiFi"
1122 }
1123 }
1124 },
1125 {
1126 "druid-env" : {
1127 "properties_attributes" : { },
1128 "properties" : {
1129 "druid.historical.jvm.direct.memory" : "1048576",
1130 "druid.coordinator.jvm.opts" : "-Duser.timezone=UTC -Dfile.encoding=UTF-8",
1131 "druid.coordinator.jvm.heap.memory" : "512",
1132 "druid.overlord.jvm.direct.memory" : "1048576",
1133 "druid.overlord.jvm.heap.memory" : "512",
1134 "druid.middlemanager.jvm.opts" : "-Duser.timezone=UTC -Dfile.encoding=UTF-8",
1135 "druid.router.jvm.heap.memory" : "512",
1136 "druid.router.jvm.direct.memory" : "1048576",
1137 "druid_log_dir" : "/var/log/druid",
1138 "druid.broker.jvm.opts" : "-Duser.timezone=UTC -Dfile.encoding=UTF-8",
1139 "druid.overlord.jvm.opts" : "-Duser.timezone=UTC -Dfile.encoding=UTF-8",
1140 "druid.router.jvm.opts" : "-Duser.timezone=UTC -Dfile.encoding=UTF-8",
1141 "druid.historical.jvm.heap.memory" : "2048",
1142 "content" : "\n #!/bin/bash\n\n # Set DRUID specific environment variables here.\n\n # The java implementation to use.\n export JAVA_HOME={{java8_home}}\n export PATH=$JAVA_HOME/bin:$PATH\n export DRUID_PID_DIR={{druid_pid_dir}}\n export DRUID_LOG_DIR={{druid_log_dir}}\n export DRUID_CONF_DIR={{druid_conf_dir}}\n export DRUID_LIB_DIR={{druid_home}}/lib\n export HADOOP_CONF_DIR={{hadoop_conf_dir}}",
1143 "druid.coordinator.jvm.direct.memory" : "1048576",
1144 "druid.broker.jvm.heap.memory" : "2048",
1145 "druid.broker.jvm.direct.memory" : "1048576",
1146 "druid_pid_dir" : "/var/run/druid",
1147 "druid.middlemanager.jvm.heap.memory" : "256",
1148 "druid.historical.jvm.opts" : "-Duser.timezone=UTC -Dfile.encoding=UTF-8",
1149 "druid.middlemanager.jvm.direct.memory" : "1048576",
1150 "druid_user" : "druid"
1151 }
1152 }
1153 },
1154 {
1155 "druid-logrotate" : {
1156 "properties_attributes" : { },
1157 "properties" : {
1158 "druid_log_maxbackupindex" : "7",
1159 "content" : "\n {{druid_log_dir}}/*.log {\n copytruncate\n rotate {{druid_log_maxbackupindex}}\n daily\n nocompress\n missingok\n notifempty\n create 660 druid users\n dateext\n dateformat -%Y-%m-%d-%s\n size {{druid_log_maxfilesize}}MB\n }",
1160 "druid_log_maxfilesize" : "256"
1161 }
1162 }
1163 },
1164 {
1165 "hbase-logsearch-conf" : {
1166 "properties_attributes" : { },
1167 "properties" : {
1168 "component_mappings" : "HBASE_MASTER:hbase_master;HBASE_REGIONSERVER:hbase_regionserver;PHOENIX_QUERY_SERVER:hbase_phoenix_server",
1169 "content" : "\n{\n \"input\":[\n {\n \"type\":\"hbase_master\",\n \"rowtype\":\"service\",\n \"path\":\"{{default('/configurations/hbase-env/hbase_log_dir', '/var/log/hbase')}}/hbase-*-master-*.log\"\n },\n {\n \"type\":\"hbase_regionserver\",\n \"rowtype\":\"service\",\n \"path\":\"{{default('/configurations/hbase-env/hbase_log_dir', '/var/log/hbase')}}/hbase-*-regionserver-*.log\"\n },\n {\n \"type\":\"hbase_phoenix_server\",\n \"rowtype\":\"service\",\n \"path\":\"{{default('/configurations/hbase-env/hbase_log_dir', '/var/log/hbase')}}/phoenix-*-server.log\"\n }\n ],\n \"filter\":[\n {\n \"filter\":\"grok\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"hbase_master\",\n \"hbase_regionserver\"\n ]\n }\n },\n \"log4j_format\":\"%d{ISO8601} %-5p [%t] %c{2}: %m%n\",\n \"multiline_pattern\":\"^(%{TIMESTAMP_ISO8601:logtime})\",\n \"message_pattern\":\"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}\\\\[%{DATA:thread_name}\\\\]%{SPACE}%{JAVACLASS:logger_name}:%{SPACE}%{GREEDYDATA:log_message}\",\n \"post_map_values\":{\n \"logtime\":{\n \"map_date\":{\n \"target_date_pattern\":\"yyyy-MM-dd HH:mm:ss,SSS\"\n }\n }\n }\n },\n {\n \"filter\":\"grok\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"hbase_phoenix_server\"\n ]\n }\n },\n \"log4j_format\":\"%d{ISO8601} %-5p [%t] %c{2}: %m%n\",\n \"multiline_pattern\":\"^(%{TIMESTAMP_ISO8601:logtime})\",\n \"message_pattern\":\"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}:%{SPACE}%{GREEDYDATA:log_message}\",\n \"post_map_values\":{\n \"logtime\":{\n \"map_date\":{\n \"target_date_pattern\":\"yyyy-MM-dd HH:mm:ss,SSS\"\n }\n }\n }\n }\n ]\n }",
1170 "service_name" : "HBase"
1171 }
1172 }
1173 },
1174 {
1175 "druid-log4j" : {
1176 "properties_attributes" : { },
1177 "properties" : {
1178 "metamx_log_level" : "info",
1179 "root_log_level" : "WARN",
1180 "druid_log_level" : "info",
1181 "content" : "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n<!--\n ~ Licensed to the Apache Software Foundation (ASF) under one\n ~ or more contributor license agreements. See the NOTICE file\n ~ distributed with this work for additional information\n ~ regarding copyright ownership. The ASF licenses this file\n ~ to you under the Apache License, Version 2.0 (the\n ~ \"License\"); you may not use this file except in compliance\n ~ with the License. You may obtain a copy of the License at\n ~\n ~ http://www.apache.org/licenses/LICENSE-2.0\n ~\n ~ Unless required by applicable law or agreed to in writing, software\n ~ distributed under the License is distributed on an \"AS IS\" BASIS,\n ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n ~ See the License for the specific language governing permissions and\n ~ limitations under the License.\n -->\n <Configuration>\n <Appenders>\n <Console name=\"Console\" target=\"SYSTEM_OUT\">\n <PatternLayout pattern=\"%d{ISO8601} %p [%t] %c - %m%n\"/>\n </Console>\n </Appenders>\n <Loggers>\n <Logger name=\"com.metamx\" level=\"{{metamx_log_level}}\"/>\n <Logger name=\"io.druid\" level=\"{{druid_log_level}}\"/>\n <Root level=\"{{root_log_level}}\">\n <AppenderRef ref=\"Console\"/>\n </Root>\n </Loggers>\n </Configuration>"
1182 }
1183 }
1184 },
1185 {
1186 "nifi-state-management-env" : {
1187 "properties_attributes" : { },
1188 "properties" : {
1189 "content" : "\n <!--\n Licensed to the Apache Software Foundation (ASF) under one or more\n contributor license agreements. See the NOTICE file distributed with\n this work for additional information regarding copyright ownership.\n The ASF licenses this file to You under the Apache License, Version 2.0\n (the \"License\"); you may not use this file except in compliance with\n the License. You may obtain a copy of the License at\n http://www.apache.org/licenses/LICENSE-2.0\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n -->\n\n <!--\n This file provides a mechanism for defining and configuring the State Providers\n that should be used for storing state locally and across a NiFi cluster. In order\n to use a specific provider, it must be configured here and its identifier\n must be specified in the nifi.properties file.\n -->\n <stateManagement>\n <!--\n State Provider that stores state locally in a configurable directory. This Provider requires the following properties:\n\n Directory - the directory to store components' state in. If the directory being used is a sub-directory of the NiFi installation, it\n is important that the directory be copied over to the new version when upgrading NiFi.\n -->\n <local-provider>\n <id>local-provider</id>\n <class>org.apache.nifi.controller.state.providers.local.WriteAheadLocalStateProvider</class>\n <property name=\"Directory\">{{nifi_state_dir}}</property>\n <property name=\"Always Sync\">false</property>\n <property name=\"Partitions\">16</property>\n <property name=\"Checkpoint Interval\">2 mins</property>\n </local-provider>\n\n <!--\n State Provider that is used to store state in ZooKeeper. This Provider requires the following properties:\n\n Root Node - the root node in ZooKeeper where state should be stored. The default is '/nifi', but it is advisable to change this to a different value if not using\n the embedded ZooKeeper server and if multiple NiFi instances may all be using the same ZooKeeper Server.\n\n Connect String - A comma-separated list of host:port pairs to connect to ZooKeeper. For example, myhost.mydomain:2181,host2.mydomain:5555,host3:6666\n\n Session Timeout - Specifies how long this instance of NiFi is allowed to be disconnected from ZooKeeper before creating a new ZooKeeper Session. Default value is \"30 seconds\"\n\n Access Control - Specifies which Access Controls will be applied to the ZooKeeper ZNodes that are created by this State Provider. This value must be set to one of:\n - Open : ZNodes will be open to any ZooKeeper client.\n - CreatorOnly : ZNodes will be accessible only by the creator. The creator will have full access to create children, read, write, delete, and administer the ZNodes.\n This option is available only if access to ZooKeeper is secured via Kerberos or if a Username and Password are set.\n -->\n\n\n <cluster-provider>\n <id>zk-provider</id>\n <class>org.apache.nifi.controller.state.providers.zookeeper.ZooKeeperStateProvider</class>\n <property name=\"Connect String\">{{zookeeper_quorum}}</property>\n <property name=\"Root Node\">{{nifi_znode}}</property>\n <property name=\"Session Timeout\">30 seconds</property>\n <property name=\"Access Control\">Open</property>\n </cluster-provider>\n </stateManagement>"
1190 }
1191 }
1192 },
1193 {
1194 "registry-env" : {
1195 "properties_attributes" : { },
1196 "properties" : {
1197 "registry_user" : "registry",
1198 "registry_pid_dir" : "/var/run/registry",
1199 "registry_user_nofile_limit" : "65536",
1200 "registry_log_dir" : "/var/log/registry",
1201 "content" : "\n#!/bin/bash\n\n# Set REGISTRY specific environment variables here.\n\n# The java implementation to use.\nexport JAVA_HOME={{java64_home}}\nexport PATH=$PATH:$JAVA_HOME/bin\nexport PID_DIR={{registry_pid_dir}}\nexport LOG_DIR={{registry_log_dir}}\nexport REGISTRY_KERBEROS_PARAMS={{registry_kerberos_params}}",
1202 "registry_user_nproc_limit" : "65536"
1203 }
1204 }
1205 },
1206 {
1207 "ssl-client" : {
1208 "properties_attributes" : { },
1209 "properties" : {
1210 "ssl.client.truststore.reload.interval" : "10000",
1211 "ssl.client.keystore.location" : "/etc/security/clientKeys/keystore.jks",
1212 "ssl.client.truststore.location" : "/etc/security/clientKeys/all.jks",
1213 "ssl.client.truststore.type" : "jks",
1214 "ssl.client.keystore.type" : "jks"
1215 }
1216 }
1217 },
1218 {
1219 "kafka-log4j" : {
1220 "properties_attributes" : { },
1221 "properties" : {
1222 "controller_log_maxbackupindex" : "20",
1223 "controller_log_maxfilesize" : "256",
1224 "kafka_log_maxfilesize" : "256",
1225 "content" : "\n#\n#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership. The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License. You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied. See the License for the\n# specific language governing permissions and limitations\n# under the License.\n#\n#\n#\nkafka.logs.dir=logs\n\nlog4j.rootLogger=INFO, stdout\n\nlog4j.appender.stdout=org.apache.log4j.ConsoleAppender\nlog4j.appender.stdout.layout=org.apache.log4j.PatternLayout\nlog4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c)%n\n\nlog4j.appender.kafkaAppender=org.apache.log4j.DailyRollingFileAppender\nlog4j.appender.kafkaAppender.DatePattern='.'yyyy-MM-dd-HH\nlog4j.appender.kafkaAppender.File=${kafka.logs.dir}/server.log\nlog4j.appender.kafkaAppender.layout=org.apache.log4j.PatternLayout\nlog4j.appender.kafkaAppender.layout.ConversionPattern=[%d] %p %m (%c)%n\nlog4j.appender.kafkaAppender.MaxFileSize = {{kafka_log_maxfilesize}}MB\nlog4j.appender.kafkaAppender.MaxBackupIndex = {{kafka_log_maxbackupindex}}\n\nlog4j.appender.stateChangeAppender=org.apache.log4j.DailyRollingFileAppender\nlog4j.appender.stateChangeAppender.DatePattern='.'yyyy-MM-dd-HH\nlog4j.appender.stateChangeAppender.File=${kafka.logs.dir}/state-change.log\nlog4j.appender.stateChangeAppender.layout=org.apache.log4j.PatternLayout\nlog4j.appender.stateChangeAppender.layout.ConversionPattern=[%d] %p %m (%c)%n\n\nlog4j.appender.requestAppender=org.apache.log4j.DailyRollingFileAppender\nlog4j.appender.requestAppender.DatePattern='.'yyyy-MM-dd-HH\nlog4j.appender.requestAppender.File=${kafka.logs.dir}/kafka-request.log\nlog4j.appender.requestAppender.layout=org.apache.log4j.PatternLayout\nlog4j.appender.requestAppender.layout.ConversionPattern=[%d] %p %m (%c)%n\n\nlog4j.appender.cleanerAppender=org.apache.log4j.DailyRollingFileAppender\nlog4j.appender.cleanerAppender.DatePattern='.'yyyy-MM-dd-HH\nlog4j.appender.cleanerAppender.File=${kafka.logs.dir}/log-cleaner.log\nlog4j.appender.cleanerAppender.layout=org.apache.log4j.PatternLayout\nlog4j.appender.cleanerAppender.layout.ConversionPattern=[%d] %p %m (%c)%n\n\nlog4j.appender.controllerAppender=org.apache.log4j.DailyRollingFileAppender\nlog4j.appender.controllerAppender.DatePattern='.'yyyy-MM-dd-HH\nlog4j.appender.controllerAppender.File=${kafka.logs.dir}/controller.log\nlog4j.appender.controllerAppender.layout=org.apache.log4j.PatternLayout\nlog4j.appender.controllerAppender.layout.ConversionPattern=[%d] %p %m (%c)%n\nlog4j.appender.controllerAppender.MaxFileSize = {{controller_log_maxfilesize}}MB\nlog4j.appender.controllerAppender.MaxBackupIndex = {{controller_log_maxbackupindex}}\n# Turn on all our debugging info\n#log4j.logger.kafka.producer.async.DefaultEventHandler=DEBUG, kafkaAppender\n#log4j.logger.kafka.client.ClientUtils=DEBUG, kafkaAppender\n#log4j.logger.kafka.perf=DEBUG, kafkaAppender\n#log4j.logger.kafka.perf.ProducerPerformance$ProducerThread=DEBUG, kafkaAppender\n#log4j.logger.org.I0Itec.zkclient.ZkClient=DEBUG\nlog4j.logger.kafka=INFO, kafkaAppender\nlog4j.logger.kafka.network.RequestChannel$=WARN, requestAppender\nlog4j.additivity.kafka.network.RequestChannel$=false\n\n#log4j.logger.kafka.network.Processor=TRACE, requestAppender\n#log4j.logger.kafka.server.KafkaApis=TRACE, requestAppender\n#log4j.additivity.kafka.server.KafkaApis=false\nlog4j.logger.kafka.request.logger=WARN, requestAppender\nlog4j.additivity.kafka.request.logger=false\n\nlog4j.logger.kafka.controller=TRACE, controllerAppender\nlog4j.additivity.kafka.controller=false\n\nlog4j.logger.kafka.log.LogCleaner=INFO, cleanerAppender\nlog4j.additivity.kafka.log.LogCleaner=false\n\nlog4j.logger.state.change.logger=TRACE, stateChangeAppender\nlog4j.additivity.state.change.logger=false",
1226 "kafka_log_maxbackupindex" : "20"
1227 }
1228 }
1229 },
1230 {
1231 "ranger-hbase-audit" : {
1232 "properties_attributes" : { },
1233 "properties" : { }
1234 }
1235 },
1236 {
1237 "nifi-node-logback-env" : {
1238 "properties_attributes" : { },
1239 "properties" : {
1240 "content" : "\n <!--\n Licensed to the Apache Software Foundation (ASF) under one or more\n contributor license agreements. See the NOTICE file distributed with\n this work for additional information regarding copyright ownership.\n The ASF licenses this file to You under the Apache License, Version 2.0\n (the \"License\"); you may not use this file except in compliance with\n the License. You may obtain a copy of the License at\n http://www.apache.org/licenses/LICENSE-2.0\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n -->\n\n <configuration scan=\"true\" scanPeriod=\"30 seconds\">\n <contextListener class=\"ch.qos.logback.classic.jul.LevelChangePropagator\">\n <resetJUL>true</resetJUL>\n </contextListener>\n\n <appender name=\"APP_FILE\" class=\"ch.qos.logback.core.rolling.RollingFileAppender\">\n <file>${org.apache.nifi.bootstrap.config.log.dir}/nifi-app.log</file>\n <rollingPolicy class=\"ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy\">\n <!--\n For daily rollover, use 'app_%d.log'.\n For hourly rollover, use 'app_%d{yyyy-MM-dd_HH}.log'.\n To GZIP rolled files, replace '.log' with '.log.gz'.\n To ZIP rolled files, replace '.log' with '.log.zip'.\n -->\n <fileNamePattern>${org.apache.nifi.bootstrap.config.log.dir}/nifi-app_%d{yyyy-MM-dd_HH}.%i.log</fileNamePattern>\n <maxFileSize>100MB</maxFileSize>\n <!-- keep 30 log files worth of history -->\n <maxHistory>30</maxHistory>\n\n <!-- optional setting for keeping 10GB total of log files\n <totalSizeCap>10GB</totalSizeCap>\n -->\n </rollingPolicy>\n <immediateFlush>true</immediateFlush>\n <encoder class=\"ch.qos.logback.classic.encoder.PatternLayoutEncoder\">\n <pattern>%date %level [%thread] %logger{40} %msg%n</pattern>\n </encoder>\n </appender>\n\n <appender name=\"USER_FILE\" class=\"ch.qos.logback.core.rolling.RollingFileAppender\">\n <file>${org.apache.nifi.bootstrap.config.log.dir}/nifi-user.log</file>\n <rollingPolicy class=\"ch.qos.logback.core.rolling.TimeBasedRollingPolicy\">\n <!--\n For daily rollover, use 'user_%d.log'.\n For hourly rollover, use 'user_%d{yyyy-MM-dd_HH}.log'.\n To GZIP rolled files, replace '.log' with '.log.gz'.\n To ZIP rolled files, replace '.log' with '.log.zip'.\n -->\n <fileNamePattern>${org.apache.nifi.bootstrap.config.log.dir}/nifi-user_%d.log</fileNamePattern>\n <!-- keep 30 log files worth of history -->\n <maxHistory>30</maxHistory>\n </rollingPolicy>\n <encoder class=\"ch.qos.logback.classic.encoder.PatternLayoutEncoder\">\n <pattern>%date %level [%thread] %logger{40} %msg%n</pattern>\n </encoder>\n </appender>\n\n <appender name=\"BOOTSTRAP_FILE\" class=\"ch.qos.logback.core.rolling.RollingFileAppender\">\n <file>${org.apache.nifi.bootstrap.config.log.dir}/nifi-bootstrap.log</file>\n <rollingPolicy class=\"ch.qos.logback.core.rolling.TimeBasedRollingPolicy\">\n <!--\n For daily rollover, use 'user_%d.log'.\n For hourly rollover, use 'user_%d{yyyy-MM-dd_HH}.log'.\n To GZIP rolled files, replace '.log' with '.log.gz'.\n To ZIP rolled files, replace '.log' with '.log.zip'.\n -->\n <fileNamePattern>${org.apache.nifi.bootstrap.config.log.dir}/nifi-bootstrap_%d.log</fileNamePattern>\n <!-- keep 5 log files worth of history -->\n <maxHistory>5</maxHistory>\n </rollingPolicy>\n <encoder class=\"ch.qos.logback.classic.encoder.PatternLayoutEncoder\">\n <pattern>%date %level [%thread] %logger{40} %msg%n</pattern>\n </encoder>\n </appender>\n\n <appender name=\"CONSOLE\" class=\"ch.qos.logback.core.ConsoleAppender\">\n <encoder class=\"ch.qos.logback.classic.encoder.PatternLayoutEncoder\">\n <pattern>%date %level [%thread] %logger{40} %msg%n</pattern>\n </encoder>\n </appender>\n\n <!-- valid logging levels: TRACE, DEBUG, INFO, WARN, ERROR -->\n\n <logger name=\"org.apache.nifi\" level=\"INFO\"/>\n <logger name=\"org.apache.nifi.processors\" level=\"WARN\"/>\n <logger name=\"org.apache.nifi.processors.standard.LogAttribute\" level=\"INFO\"/>\n <logger name=\"org.apache.nifi.controller.repository.StandardProcessSession\" level=\"WARN\" />\n\n\n <logger name=\"org.apache.zookeeper.ClientCnxn\" level=\"ERROR\" />\n <logger name=\"org.apache.zookeeper.server.NIOServerCnxn\" level=\"ERROR\" />\n <logger name=\"org.apache.zookeeper.server.NIOServerCnxnFactory\" level=\"ERROR\" />\n <logger name=\"org.apache.zookeeper.server.quorum\" level=\"ERROR\" />\n <logger name=\"org.apache.zookeeper.ZooKeeper\" level=\"ERROR\" />\n <logger name=\"org.apache.zookeeper.server.PrepRequestProcessor\" level=\"ERROR\" />\n\n <logger name=\"org.apache.curator.framework.recipes.leader.LeaderSelector\" level=\"OFF\" />\n <logger name=\"org.apache.curator.ConnectionState\" level=\"OFF\" />\n\n <!-- Logger for managing logging statements for nifi clusters. -->\n <logger name=\"org.apache.nifi.cluster\" level=\"INFO\"/>\n\n <!-- Logger for logging HTTP requests received by the web server. -->\n <logger name=\"org.apache.nifi.server.JettyServer\" level=\"INFO\"/>\n\n <!-- Logger for managing logging statements for jetty -->\n <logger name=\"org.eclipse.jetty\" level=\"INFO\"/>\n\n <!-- Suppress non-error messages due to excessive logging by class or library -->\n <logger name=\"com.sun.jersey.spi.container.servlet.WebComponent\" level=\"ERROR\"/>\n <logger name=\"com.sun.jersey.spi.spring\" level=\"ERROR\"/>\n <logger name=\"org.springframework\" level=\"ERROR\"/>\n\n <!-- Suppress non-error messages due to known warning about redundant path annotation (NIFI-574) -->\n <logger name=\"com.sun.jersey.spi.inject.Errors\" level=\"ERROR\"/>\n\n <!--\n Logger for capturing user events. We do not want to propagate these\n log events to the root logger. These messages are only sent to the\n user-log appender.\n -->\n <logger name=\"org.apache.nifi.web.security\" level=\"INFO\" additivity=\"false\">\n <appender-ref ref=\"USER_FILE\"/>\n </logger>\n <logger name=\"org.apache.nifi.web.api.config\" level=\"INFO\" additivity=\"false\">\n <appender-ref ref=\"USER_FILE\"/>\n </logger>\n <logger name=\"org.apache.nifi.authorization\" level=\"INFO\" additivity=\"false\">\n <appender-ref ref=\"USER_FILE\"/>\n </logger>\n <logger name=\"org.apache.nifi.cluster.authorization\" level=\"INFO\" additivity=\"false\">\n <appender-ref ref=\"USER_FILE\"/>\n </logger>\n <logger name=\"org.apache.nifi.web.filter.RequestLogger\" level=\"INFO\" additivity=\"false\">\n <appender-ref ref=\"USER_FILE\"/>\n </logger>\n\n\n <!--\n Logger for capturing Bootstrap logs and NiFi's standard error and standard out.\n -->\n <logger name=\"org.apache.nifi.bootstrap\" level=\"INFO\" additivity=\"false\">\n <appender-ref ref=\"BOOTSTRAP_FILE\" />\n </logger>\n <logger name=\"org.apache.nifi.bootstrap.Command\" level=\"INFO\" additivity=\"false\">\n <appender-ref ref=\"CONSOLE\" />\n <appender-ref ref=\"BOOTSTRAP_FILE\" />\n </logger>\n\n <!-- Everything written to NiFi's Standard Out will be logged with the logger org.apache.nifi.StdOut at INFO level -->\n <logger name=\"org.apache.nifi.StdOut\" level=\"INFO\" additivity=\"false\">\n <appender-ref ref=\"BOOTSTRAP_FILE\" />\n </logger>\n\n <!-- Everything written to NiFi's Standard Error will be logged with the logger org.apache.nifi.StdErr at ERROR level -->\n <logger name=\"org.apache.nifi.StdErr\" level=\"ERROR\" additivity=\"false\">\n <appender-ref ref=\"BOOTSTRAP_FILE\" />\n </logger>\n\n <!-- Turn off Calcite Exception logging -->\n <logger name=\"org.apache.calcite.runtime.CalciteException\" level=\"OFF\" />\n\n\n <root level=\"INFO\">\n <appender-ref ref=\"APP_FILE\"/>\n </root>\n\n </configuration>"
1241 }
1242 }
1243 },
1244 {
1245 "ranger-kafka-plugin-properties" : {
1246 "properties_attributes" : { },
1247 "properties" : { }
1248 }
1249 },
1250 {
1251 "kafka-env" : {
1252 "properties_attributes" : { },
1253 "properties" : {
1254 "kafka_pid_dir" : "/var/run/kafka",
1255 "kafka_user_nofile_limit" : "128000",
1256 "kafka_log_dir" : "/var/log/kafka",
1257 "kafka_user_nproc_limit" : "65536",
1258 "content" : "\n#!/bin/bash\n\n# Set KAFKA specific environment variables here.\n\n# The java implementation to use.\nexport JAVA_HOME={{java64_home}}\nexport PATH=$PATH:$JAVA_HOME/bin\nexport PID_DIR={{kafka_pid_dir}}\nexport LOG_DIR={{kafka_log_dir}}\n{% if security_enabled %}\nexport KAFKA_KERBEROS_PARAMS=\"-Djavax.security.auth.useSubjectCredsOnly=false {{kafka_kerberos_params}}\"\n{% else %}\nexport KAFKA_KERBEROS_PARAMS={{kafka_kerberos_params}}\n{% endif %}\n# Add kafka sink to classpath and related depenencies\nif [ -e \"/usr/lib/ambari-metrics-kafka-sink/ambari-metrics-kafka-sink.jar\" ]; then\n export CLASSPATH=$CLASSPATH:/usr/lib/ambari-metrics-kafka-sink/ambari-metrics-kafka-sink.jar\n export CLASSPATH=$CLASSPATH:/usr/lib/ambari-metrics-kafka-sink/lib/*\nfi\nif [ -f /etc/kafka/conf/kafka-ranger-env.sh ]; then\n. /etc/kafka/conf/kafka-ranger-env.sh\nfi",
1259 "is_supported_kafka_ranger" : "true",
1260 "kafka_user" : "kafka"
1261 }
1262 }
1263 },
1264 {
1265 "druid-coordinator" : {
1266 "properties_attributes" : { },
1267 "properties" : {
1268 "druid.coordinator.merge.on" : "false",
1269 "druid.service" : "druid/coordinator",
1270 "druid.port" : "8081"
1271 }
1272 }
1273 },
1274 {
1275 "ranger-hbase-plugin-properties" : {
1276 "properties_attributes" : { },
1277 "properties" : { }
1278 }
1279 },
1280 {
1281 "ranger-hdfs-security" : {
1282 "properties_attributes" : { },
1283 "properties" : { }
1284 }
1285 },
1286 {
1287 "druid-overlord" : {
1288 "properties_attributes" : { },
1289 "properties" : {
1290 "druid.indexer.storage.type" : "metadata",
1291 "druid.indexer.runner.type" : "remote",
1292 "druid.service" : "druid/overlord",
1293 "druid.port" : "8090"
1294 }
1295 }
1296 },
1297 {
1298 "hadoop-env" : {
1299 "properties_attributes" : { },
1300 "properties" : {
1301 "hadoop_heapsize" : "1024",
1302 "proxyuser_group" : "users",
1303 "hadoop_root_logger" : "INFO,RFA",
1304 "dtnode_heapsize" : "1024m",
1305 "namenode_backup_dir" : "/tmp/upgrades",
1306 "hdfs_user" : "hdfs",
1307 "hadoop_pid_dir_prefix" : "/var/run/hadoop",
1308 "content" : "\n # Set Hadoop-specific environment variables here.\n\n # The only required environment variable is JAVA_HOME. All others are\n # optional. When running a distributed configuration it is best to\n # set JAVA_HOME in this file, so that it is correctly defined on\n # remote nodes.\n\n # The java implementation to use. Required.\n export JAVA_HOME={{java_home}}\n export HADOOP_HOME_WARN_SUPPRESS=1\n\n # Hadoop home directory\n export HADOOP_HOME=${HADOOP_HOME:-{{hadoop_home}}}\n\n # Hadoop Configuration Directory\n\n {# this is different for HDP1 #}\n # Path to jsvc required by secure HDP 2.0 datanode\n export JSVC_HOME={{jsvc_path}}\n\n\n # The maximum amount of heap to use, in MB. Default is 1000.\n export HADOOP_HEAPSIZE=\"{{hadoop_heapsize}}\"\n\n export HADOOP_NAMENODE_INIT_HEAPSIZE=\"-Xms{{namenode_heapsize}}\"\n\n # Extra Java runtime options. Empty by default.\n export HADOOP_OPTS=\"-Djava.net.preferIPv4Stack=true ${HADOOP_OPTS}\"\n\n # Command specific options appended to HADOOP_OPTS when specified\n HADOOP_JOBTRACKER_OPTS=\"-server -XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC -XX:ErrorFile={{hdfs_log_dir_prefix}}/$USER/hs_err_pid%p.log -XX:NewSize={{jtnode_opt_newsize}} -XX:MaxNewSize={{jtnode_opt_maxnewsize}} -Xloggc:{{hdfs_log_dir_prefix}}/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xmx{{jtnode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dmapred.audit.logger=INFO,MRAUDIT -Dhadoop.mapreduce.jobsummary.logger=INFO,JSA ${HADOOP_JOBTRACKER_OPTS}\"\n\n HADOOP_TASKTRACKER_OPTS=\"-server -Xmx{{ttnode_heapsize}} -Dhadoop.security.logger=ERROR,console -Dmapred.audit.logger=ERROR,console ${HADOOP_TASKTRACKER_OPTS}\"\n\n {% if java_version < 8 %}\n SHARED_HADOOP_NAMENODE_OPTS=\"-server -XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC -XX:ErrorFile={{hdfs_log_dir_prefix}}/$USER/hs_err_pid%p.log -XX:NewSize={{namenode_opt_newsize}} -XX:MaxNewSize={{namenode_opt_maxnewsize}} -XX:PermSize={{namenode_opt_permsize}} -XX:MaxPermSize={{namenode_opt_maxpermsize}} -Xloggc:{{hdfs_log_dir_prefix}}/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -XX:CMSInitiatingOccupancyFraction=70 -XX:+UseCMSInitiatingOccupancyOnly -Xms{{namenode_heapsize}} -Xmx{{namenode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT\"\n export HADOOP_NAMENODE_OPTS=\"${SHARED_HADOOP_NAMENODE_OPTS} -XX:OnOutOfMemoryError=\\\"/usr/hdp/current/hadoop-hdfs-namenode/bin/kill-name-node\\\" -Dorg.mortbay.jetty.Request.maxFormContentSize=-1 ${HADOOP_NAMENODE_OPTS}\"\n export HADOOP_DATANODE_OPTS=\"-server -XX:ParallelGCThreads=4 -XX:+UseConcMarkSweepGC -XX:ErrorFile=/var/log/hadoop/$USER/hs_err_pid%p.log -XX:NewSize=200m -XX:MaxNewSize=200m -XX:PermSize=128m -XX:MaxPermSize=256m -Xloggc:/var/log/hadoop/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xms{{dtnode_heapsize}} -Xmx{{dtnode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT ${HADOOP_DATANODE_OPTS} -XX:CMSInitiatingOccupancyFraction=70 -XX:+UseCMSInitiatingOccupancyOnly\"\n\n export HADOOP_SECONDARYNAMENODE_OPTS=\"${SHARED_HADOOP_NAMENODE_OPTS} -XX:OnOutOfMemoryError=\\\"/usr/hdp/current/hadoop-hdfs-secondarynamenode/bin/kill-secondary-name-node\\\" ${HADOOP_SECONDARYNAMENODE_OPTS}\"\n\n # The following applies to multiple commands (fs, dfs, fsck, distcp etc)\n export HADOOP_CLIENT_OPTS=\"-Xmx${HADOOP_HEAPSIZE}m -XX:MaxPermSize=512m $HADOOP_CLIENT_OPTS\"\n\n {% else %}\n SHARED_HADOOP_NAMENODE_OPTS=\"-server -XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC -XX:ErrorFile={{hdfs_log_dir_prefix}}/$USER/hs_err_pid%p.log -XX:NewSize={{namenode_opt_newsize}} -XX:MaxNewSize={{namenode_opt_maxnewsize}} -Xloggc:{{hdfs_log_dir_prefix}}/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -XX:CMSInitiatingOccupancyFraction=70 -XX:+UseCMSInitiatingOccupancyOnly -Xms{{namenode_heapsize}} -Xmx{{namenode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT\"\n export HADOOP_NAMENODE_OPTS=\"${SHARED_HADOOP_NAMENODE_OPTS} -XX:OnOutOfMemoryError=\\\"/usr/hdp/current/hadoop-hdfs-namenode/bin/kill-name-node\\\" -Dorg.mortbay.jetty.Request.maxFormContentSize=-1 ${HADOOP_NAMENODE_OPTS}\"\n export HADOOP_DATANODE_OPTS=\"-server -XX:ParallelGCThreads=4 -XX:+UseConcMarkSweepGC -XX:ErrorFile=/var/log/hadoop/$USER/hs_err_pid%p.log -XX:NewSize=200m -XX:MaxNewSize=200m -Xloggc:/var/log/hadoop/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xms{{dtnode_heapsize}} -Xmx{{dtnode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT ${HADOOP_DATANODE_OPTS} -XX:CMSInitiatingOccupancyFraction=70 -XX:+UseCMSInitiatingOccupancyOnly\"\n\n export HADOOP_SECONDARYNAMENODE_OPTS=\"${SHARED_HADOOP_NAMENODE_OPTS} -XX:OnOutOfMemoryError=\\\"/usr/hdp/current/hadoop-hdfs-secondarynamenode/bin/kill-secondary-name-node\\\" ${HADOOP_SECONDARYNAMENODE_OPTS}\"\n\n # The following applies to multiple commands (fs, dfs, fsck, distcp etc)\n export HADOOP_CLIENT_OPTS=\"-Xmx${HADOOP_HEAPSIZE}m $HADOOP_CLIENT_OPTS\"\n {% endif %}\n\n HADOOP_NFS3_OPTS=\"-Xmx{{nfsgateway_heapsize}}m -Dhadoop.security.logger=ERROR,DRFAS ${HADOOP_NFS3_OPTS}\"\n HADOOP_BALANCER_OPTS=\"-server -Xmx{{hadoop_heapsize}}m ${HADOOP_BALANCER_OPTS}\"\n\n\n # On secure datanodes, user to run the datanode as after dropping privileges\n export HADOOP_SECURE_DN_USER=${HADOOP_SECURE_DN_USER:-{{hadoop_secure_dn_user}}}\n\n # Extra ssh options. Empty by default.\n export HADOOP_SSH_OPTS=\"-o ConnectTimeout=5 -o SendEnv=HADOOP_CONF_DIR\"\n\n # Where log files are stored. $HADOOP_HOME/logs by default.\n export HADOOP_LOG_DIR={{hdfs_log_dir_prefix}}/$USER\n\n # History server logs\n export HADOOP_MAPRED_LOG_DIR={{mapred_log_dir_prefix}}/$USER\n\n # Where log files are stored in the secure data environment.\n export HADOOP_SECURE_DN_LOG_DIR={{hdfs_log_dir_prefix}}/$HADOOP_SECURE_DN_USER\n\n # File naming remote slave hosts. $HADOOP_HOME/conf/slaves by default.\n # export HADOOP_SLAVES=${HADOOP_HOME}/conf/slaves\n\n # host:path where hadoop code should be rsync'd from. Unset by default.\n # export HADOOP_MASTER=master:/home/$USER/src/hadoop\n\n # Seconds to sleep between slave commands. Unset by default. This\n # can be useful in large clusters, where, e.g., slave rsyncs can\n # otherwise arrive faster than the master can service them.\n # export HADOOP_SLAVE_SLEEP=0.1\n\n # The directory where pid files are stored. /tmp by default.\n export HADOOP_PID_DIR={{hadoop_pid_dir_prefix}}/$USER\n export HADOOP_SECURE_DN_PID_DIR={{hadoop_pid_dir_prefix}}/$HADOOP_SECURE_DN_USER\n\n # History server pid\n export HADOOP_MAPRED_PID_DIR={{mapred_pid_dir_prefix}}/$USER\n\n YARN_RESOURCEMANAGER_OPTS=\"-Dyarn.server.resourcemanager.appsummary.logger=INFO,RMSUMMARY\"\n\n # A string representing this instance of hadoop. $USER by default.\n export HADOOP_IDENT_STRING=$USER\n\n # The scheduling priority for daemon processes. See 'man nice'.\n\n # export HADOOP_NICENESS=10\n\n # Add database libraries\n JAVA_JDBC_LIBS=\"\"\n if [ -d \"/usr/share/java\" ]; then\n for jarFile in `ls /usr/share/java | grep -E \"(mysql|ojdbc|postgresql|sqljdbc)\" 2>/dev/null`\n do\n JAVA_JDBC_LIBS=${JAVA_JDBC_LIBS}:$jarFile\n done\n fi\n\n # Add libraries to the hadoop classpath - some may not need a colon as they already include it\n export HADOOP_CLASSPATH=${HADOOP_CLASSPATH}${JAVA_JDBC_LIBS}\n\n # Setting path to hdfs command line\n export HADOOP_LIBEXEC_DIR={{hadoop_libexec_dir}}\n\n # Mostly required for hadoop 2.0\n export JAVA_LIBRARY_PATH=${JAVA_LIBRARY_PATH}:{{hadoop_lib_home}}/native/Linux-{{architecture}}-64\n\n export HADOOP_OPTS=\"-Dhdp.version=$HDP_VERSION $HADOOP_OPTS\"\n\n\n # Fix temporary bug, when ulimit from conf files is not picked up, without full relogin.\n # Makes sense to fix only when runing DN as root\n if [ \"$command\" == \"datanode\" ] && [ \"$EUID\" -eq 0 ] && [ -n \"$HADOOP_SECURE_DN_USER\" ]; then\n {% if is_datanode_max_locked_memory_set %}\n ulimit -l {{datanode_max_locked_memory}}\n {% endif %}\n ulimit -n {{hdfs_user_nofile_limit}}\n fi\n\n {% if hadoop_custom_extensions_enabled %}\n #Enable custom extensions\n export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:{{stack_root}}/current/ext/hadoop/*\n {% endif %}\n\n # Enable ACLs on zookeper znodes if required\n {% if hadoop_zkfc_opts is defined %}\n export HADOOP_ZKFC_OPTS=\"{{hadoop_zkfc_opts}} $HADOOP_ZKFC_OPTS\"\n {% endif %}",
1309 "hdfs_log_dir_prefix" : "/var/log/hadoop",
1310 "namenode_opt_newsize" : "128m",
1311 "namenode_heapsize" : "1024m",
1312 "hdfs_tmp_dir" : "/tmp",
1313 "namenode_opt_maxpermsize" : "256m",
1314 "nfsgateway_heapsize" : "1024",
1315 "hdfs_user_nofile_limit" : "128000",
1316 "keyserver_host" : " ",
1317 "keyserver_port" : "",
1318 "hdfs_user_nproc_limit" : "65536",
1319 "namenode_opt_maxnewsize" : "128m",
1320 "namenode_opt_permsize" : "128m"
1321 }
1322 }
1323 },
1324 {
1325 "storm-site" : {
1326 "properties_attributes" : { },
1327 "properties" : {
1328 "storm.messaging.netty.min_wait_ms" : "100",
1329 "topology.enable.message.timeouts" : "true",
1330 "storm.messaging.netty.buffer_size" : "5242880",
1331 "client.jartransformer.class" : "org.apache.storm.hack.StormShadeTransformer",
1332 "ui.filter" : "null",
1333 "storm.messaging.netty.client_worker_threads" : "1",
1334 "transactional.zookeeper.root" : "/transactional",
1335 "topology.metrics.aggregate.metric.evict.secs" : "5",
1336 "topology.tuple.serializer" : "org.apache.storm.serialization.types.ListDelegateSerializer",
1337 "topology.spout.wait.strategy" : "org.apache.storm.spout.SleepSpoutWaitStrategy",
1338 "drpc.port" : "3772",
1339 "topology.max.spout.pending" : "1000",
1340 "topology.transfer.buffer.size" : "1024",
1341 "supervisor.worker.timeout.secs" : "30",
1342 "logviewer.port" : "8000",
1343 "nimbus.file.copy.expiration.secs" : "600",
1344 "drpc.childopts" : "-Xmx768m _JAAS_PLACEHOLDER",
1345 "logviewer.childopts" : "-Xmx128m _JAAS_PLACEHOLDER",
1346 "nimbus.task.launch.secs" : "120",
1347 "storm.zookeeper.servers" : "['%HOSTGROUP::host_group_1%']",
1348 "storm.messaging.transport" : "org.apache.storm.messaging.netty.Context",
1349 "topology.workers" : "1",
1350 "drpc.invocations.port" : "3773",
1351 "topology.kryo.factory" : "org.apache.storm.serialization.DefaultKryoFactory",
1352 "nimbus.cleanup.inbox.freq.secs" : "600",
1353 "_storm.thrift.secure.transport" : "org.apache.storm.security.auth.kerberos.KerberosSaslTransportPlugin",
1354 "topology.fall.back.on.java.serialization" : "true",
1355 "logviewer.appender.name" : "A1",
1356 "storm.messaging.netty.server_worker_threads" : "1",
1357 "supervisor.slots.ports" : "[6700, 6701]",
1358 "storm.zookeeper.connection.timeout" : "30000",
1359 "topology.tick.tuple.freq.secs" : "null",
1360 "storm.cluster.metrics.consumer.register" : "null",
1361 "topology.stats.sample.rate" : "0.05",
1362 "storm.local.dir" : "/hadoop/storm",
1363 "nimbus.inbox.jar.expiration.secs" : "3600",
1364 "topology.max.replication.wait.time.sec.default" : "60",
1365 "storm.messaging.netty.max_retries" : "30",
1366 "topology.debug" : "false",
1367 "storm.zookeeper.retry.interval" : "1000",
1368 "topology.receiver.buffer.size" : "8",
1369 "storm.zookeeper.retry.times" : "5",
1370 "java.library.path" : "/usr/local/lib:/opt/local/lib:/usr/lib:/usr/hdp/current/storm-client/lib",
1371 "zmq.linger.millis" : "5000",
1372 "drpc.request.timeout.secs" : "600",
1373 "zmq.threads" : "1",
1374 "topology.state.synchronization.timeout.secs" : "60",
1375 "topology.worker.shared.thread.pool.size" : "4",
1376 "topology.executor.receive.buffer.size" : "1024",
1377 "supervisor.monitor.frequency.secs" : "3",
1378 "topology.optimize" : "true",
1379 "transactional.zookeeper.port" : "null",
1380 "storm.zookeeper.port" : "2181",
1381 "storm.zookeeper.retry.intervalceiling.millis" : "30000",
1382 "nimbus.thrift.port" : "6627",
1383 "nimbus.thrift.threads" : "196",
1384 "_storm.thrift.nonsecure.transport" : "org.apache.storm.security.auth.SimpleTransportPlugin",
1385 "nimbus.seeds" : "['%HOSTGROUP::host_group_1%']",
1386 "topology.min.replication.count" : "{{actual_topology_min_replication_count}}",
1387 "topology.worker.childopts" : "null",
1388 "topology.max.replication.wait.time.sec" : "{{actual_topology_max_replication_wait_time_sec}}",
1389 "topology.min.replication.count.default" : "1",
1390 "topology.max.task.parallelism" : "null",
1391 "topology.acker.executors" : "null",
1392 "supervisor.worker.start.timeout.secs" : "120",
1393 "_storm.min.ruid" : "null",
1394 "nimbus.reassign" : "true",
1395 "supervisor.heartbeat.frequency.secs" : "5",
1396 "topology.metrics.aggregate.per.worker" : "true",
1397 "topology.trident.batch.emit.interval.millis" : "500",
1398 "task.heartbeat.frequency.secs" : "3",
1399 "storm.thrift.transport" : "{{storm_thrift_transport}}",
1400 "drpc.worker.threads" : "64",
1401 "ui.port" : "8744",
1402 "drpc.queue.size" : "128",
1403 "topology.message.timeout.secs" : "30",
1404 "topology.error.throttle.interval.secs" : "10",
1405 "storm.log.dir" : "{{log_dir}}",
1406 "topology.builtin.metrics.bucket.size.secs" : "60",
1407 "nimbus.monitor.freq.secs" : "10",
1408 "topology.executor.send.buffer.size" : "1024",
1409 "storm.local.mode.zmq" : "false",
1410 "transactional.zookeeper.servers" : "null",
1411 "nimbus.task.timeout.secs" : "30",
1412 "topology.metrics.consumer.register" : "null",
1413 "dev.zookeeper.path" : "/tmp/dev-storm-zookeeper",
1414 "topology.metrics.expand.map.type" : "true",
1415 "nimbus.supervisor.timeout.secs" : "60",
1416 "topology.skip.missing.kryo.registrations" : "false",
1417 "storm.zookeeper.root" : "/storm",
1418 "storm.zookeeper.session.timeout" : "30000",
1419 "topology.disruptor.wait.strategy" : "com.lmax.disruptor.BlockingWaitStrategy",
1420 "zmq.hwm" : "0",
1421 "topology.sleep.spout.wait.strategy.time.ms" : "1",
1422 "nimbus.topology.validator" : "org.apache.storm.nimbus.DefaultTopologyValidator",
1423 "storm.cluster.mode" : "distributed",
1424 "task.refresh.poll.secs" : "10",
1425 "ui.childopts" : "-Xmx768m _JAAS_PLACEHOLDER",
1426 "storm.messaging.netty.max_wait_ms" : "1000",
1427 "worker.heartbeat.frequency.secs" : "1",
1428 "topology.max.error.report.per.interval" : "5",
1429 "nimbus.thrift.max_buffer_size" : "1048576",
1430 "topology.metrics.metric.name.separator" : "."
1431 }
1432 }
1433 },
1434 {
1435 "nifi-properties" : {
1436 "properties_attributes" : { },
1437 "properties" : {
1438 "nifi.remote.input.http.transaction.ttl" : "30 sec",
1439 "nifi.flow.configuration.archive.max.storage" : "500 MB",
1440 "nifi.content.claim.max.appendable.size" : "10 MB",
1441 "nifi.sensitive.props.provider" : "BC",
1442 "nifi.cluster.node.protocol.max.threads" : "",
1443 "nifi.content.repository.archive.enabled" : "true",
1444 "nifi.flow.configuration.archive.dir" : "{{nifi_internal_dir}}/archive/",
1445 "nifi.security.keystore" : "{{nifi_keystore}}",
1446 "nifi.web.http.host" : "{{nifi_node_nonssl_host}}",
1447 "nifi.flowservice.writedelay.interval" : "500 ms",
1448 "nifi.remote.input.secure" : "{{nifi_ssl_enabled}}",
1449 "nifi.provenance.repository.max.storage.time" : "24 hours",
1450 "nifi.swap.in.period" : "5 sec",
1451 "nifi.web.https.network.interface.default" : "",
1452 "nifi.bored.yield.duration" : "10 millis",
1453 "nifi.variable.registry.properties" : "",
1454 "nifi.provenance.repository.query.threads" : "2",
1455 "nifi.swap.in.threads" : "1",
1456 "nifi.cluster.node.protocol.threads" : "10",
1457 "nifi.ui.banner.text" : "",
1458 "nifi.flow.configuration.archive.max.time" : "30 days",
1459 "nifi.security.keyPasswd" : "{{nifi_keyPasswd}}",
1460 "nifi.security.ocsp.responder.url" : "",
1461 "nifi.provenance.repository.compress.on.rollover" : "true",
1462 "nifi.kerberos.spnego.principal" : "",
1463 "nifi.flowcontroller.graceful.shutdown.period" : "10 sec",
1464 "nifi.cluster.node.event.history.size" : "25",
1465 "nifi.provenance.repository.implementation" : "org.apache.nifi.provenance.PersistentProvenanceRepository",
1466 "nifi.provenance.repository.rollover.size" : "100 MB",
1467 "nifi.cluster.flow.election.max.wait.time" : "5 mins",
1468 "nifi.components.status.repository.implementation" : "org.apache.nifi.controller.status.history.VolatileComponentStatusRepository",
1469 "nifi.content.repository.archive.max.retention.period" : "12 hours",
1470 "nifi.flow.configuration.archive.max.count" : "",
1471 "nifi.cluster.flow.election.max.candidates" : "{{nifi_num_nodes}}",
1472 "nifi.provenance.repository.encryption.key.provider.implementation" : "",
1473 "nifi.web.jetty.threads" : "200",
1474 "nifi.provenance.repository.rollover.time" : "30 secs",
1475 "nifi.cluster.protocol.heartbeat.interval" : "5 sec",
1476 "nifi.provenance.repository.max.storage.size" : "1 GB",
1477 "nifi.h2.url.append" : ";LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE",
1478 "nifi.provenance.repository.debug.frequency" : "1_000_000",
1479 "nifi.sensitive.props.additional.keys" : "",
1480 "nifi.security.ocsp.responder.certificate" : "",
1481 "nifi.version" : "{{service_version}}.{{stack_version_buildnum}}",
1482 "nifi.cluster.node.connection.timeout" : "5 sec",
1483 "nifi.components.status.snapshot.frequency" : "1 min",
1484 "nifi.security.identity.mapping.pattern.kerb" : "",
1485 "nifi.web.http.network.interface.default" : "",
1486 "nifi.cluster.is.node" : "{{nifi_is_node}}",
1487 "nifi.security.needClientAuth" : "{{nifi_needClientAuth}}",
1488 "nifi.flowfile.repository.partitions" : "256",
1489 "nifi.provenance.repository.indexed.attributes" : "",
1490 "nifi.provenance.repository.index.shard.size" : "500 MB",
1491 "nifi.state.management.embedded.zookeeper.start" : "false",
1492 "nifi.security.keystoreType" : "{{nifi_keystoreType}}",
1493 "nifi.state.management.provider.local" : "local-provider",
1494 "nifi.provenance.repository.encryption.key.provider.location" : "",
1495 "nifi.flow.configuration.file" : "{{nifi_flow_config_dir}}/flow.xml.gz",
1496 "nifi.nar.working.directory" : "{{nifi_internal_dir}}/work/nar",
1497 "nifi.authorizer.configuration.file" : "{{nifi_config_dir}}/authorizers.xml",
1498 "nifi.templates.directory" : "{{nifi_internal_dir}}/templates",
1499 "nifi.state.management.provider.cluster" : "zk-provider",
1500 "nifi.zookeeper.root.node" : "{{nifi_znode}}",
1501 "nifi.security.keystorePasswd" : "{{nifi_keystorePasswd}}",
1502 "nifi.security.truststoreType" : "{{nifi_truststoreType}}",
1503 "nifi.content.viewer.url" : "/nifi-content-viewer/",
1504 "nifi.provenance.repository.encryption.key.id" : "",
1505 "nifi.zookeeper.session.timeout" : "3 secs",
1506 "nifi.content.repository.archive.max.usage.percentage" : "50%",
1507 "nifi.nar.library.directory" : "{{nifi_install_dir}}/lib",
1508 "nifi.swap.manager.implementation" : "org.apache.nifi.controller.FileSystemSwapManager",
1509 "nifi.cluster.node.address" : "{{nifi_node_host}}",
1510 "nifi.flow.configuration.archive.enabled" : "true",
1511 "nifi.provenance.repository.journal.count" : "16",
1512 "nifi.provenance.repository.buffer.size" : "100000",
1513 "nifi.kerberos.krb5.file" : "",
1514 "nifi.queue.swap.threshold" : "20000",
1515 "nifi.sensitive.props.key" : "{{nifi_sensitive_props_key}}",
1516 "nifi.components.status.repository.buffer.size" : "1440",
1517 "nifi.security.identity.mapping.value.kerb" : "",
1518 "nifi.content.repository.directory.default" : "{{nifi_content_repo_dir_default}}",
1519 "nifi.kerberos.service.principal" : "",
1520 "nifi.flowfile.repository.implementation" : "org.apache.nifi.controller.repository.WriteAheadFlowFileRepository",
1521 "nifi.state.management.embedded.zookeeper.properties" : "{{nifi_config_dir}}/zookeeper.properties",
1522 "nifi.web.war.directory" : "{{nifi_install_dir}}/lib",
1523 "nifi.swap.out.period" : "5 sec",
1524 "nifi.kerberos.spnego.authentication.expiration" : "12 hours",
1525 "nifi.documentation.working.directory" : "{{nifi_internal_dir}}/work/docs/components",
1526 "nifi.remote.input.socket.port" : "",
1527 "nifi.web.https.host" : "{{nifi_node_ssl_host}}",
1528 "nifi.web.jetty.working.directory" : "{{nifi_internal_dir}}/work/jetty",
1529 "nifi.security.identity.mapping.pattern.dn" : "",
1530 "nifi.zookeeper.connect.timeout" : "3 secs",
1531 "nifi.flowfile.repository.checkpoint.interval" : "2 mins",
1532 "nifi.cluster.node.read.timeout" : "5 sec",
1533 "nifi.administrative.yield.duration" : "30 sec",
1534 "nifi.sensitive.props.algorithm" : "PBEWITHMD5AND256BITAES-CBC-OPENSSL",
1535 "nifi.content.repository.always.sync" : "false",
1536 "nifi.provenance.repository.indexed.fields" : "EventType, FlowFileUUID, Filename, ProcessorID, Relationship",
1537 "nifi.provenance.repository.index.threads" : "1",
1538 "nifi.provenance.repository.encryption.key" : "",
1539 "nifi.swap.out.threads" : "4",
1540 "nifi.cluster.protocol.is.secure" : "{{nifi_ssl_enabled}}",
1541 "nifi.web.http.port" : "{{nifi_node_port}}",
1542 "nifi.remote.input.host" : "",
1543 "nifi.content.claim.max.flow.files" : "100",
1544 "nifi.flowcontroller.autoResumeState" : "true",
1545 "nifi.provenance.repository.max.attribute.length" : "65536",
1546 "nifi.security.user.authorizer" : "{{nifi_authorizer}}",
1547 "nifi.kerberos.spnego.keytab.location" : "",
1548 "nifi.zookeeper.connect.string" : "{{zookeeper_quorum}}",
1549 "nifi.flowfile.repository.always.sync" : "false",
1550 "nifi.provenance.repository.directory.default" : "{{nifi_provenance_repo_dir_default}}",
1551 "nifi.state.management.configuration.file" : "{{nifi_config_dir}}/state-management.xml",
1552 "nifi.web.https.port" : "{{nifi_node_ssl_port}}",
1553 "nifi.security.user.login.identity.provider" : "",
1554 "nifi.content.repository.implementation" : "org.apache.nifi.controller.repository.FileSystemRepository",
1555 "nifi.flowfile.repository.directory" : "{{nifi_flowfile_repo_dir}}",
1556 "nifi.login.identity.provider.configuration.file" : "{{nifi_config_dir}}/login-identity-providers.xml",
1557 "nifi.provenance.repository.always.sync" : "false",
1558 "nifi.kerberos.service.keytab.location" : "",
1559 "nifi.cluster.node.protocol.port" : "{{nifi_node_protocol_port}}",
1560 "nifi.security.truststorePasswd" : "{{nifi_truststorePasswd}}",
1561 "nifi.ui.autorefresh.interval" : "30 sec",
1562 "nifi.security.identity.mapping.value.dn" : "",
1563 "nifi.security.truststore" : "{{nifi_truststore}}",
1564 "nifi.remote.input.http.enabled" : "true",
1565 "nifi.database.directory" : "{{nifi_database_dir}}"
1566 }
1567 }
1568 },
1569 {
1570 "kafka-logsearch-conf" : {
1571 "properties_attributes" : { },
1572 "properties" : {
1573 "component_mappings" : "KAFKA_BROKER:kafka_server,kafka_request,kafka_logcleaner,kafka_controller,kafka_statechange",
1574 "content" : "\n{\n \"input\":[\n {\n \"type\":\"kafka_controller\",\n \"rowtype\":\"service\",\n \"path\":\"{{default('/configurations/kafka-env/kafka_log_dir', '/var/log/kafka')}}/controller.log\"\n },\n {\n \"type\":\"kafka_request\",\n \"rowtype\":\"service\",\n \"path\":\"{{default('/configurations/kafka-env/kafka_log_dir', '/var/log/kafka')}}/kafka-request.log\"\n },\n {\n \"type\":\"kafka_logcleaner\",\n \"rowtype\":\"service\",\n \"path\":\"{{default('/configurations/kafka-env/kafka_log_dir', '/var/log/kafka')}}/log-cleaner.log\"\n },\n {\n \"type\":\"kafka_server\",\n \"rowtype\":\"service\",\n \"path\":\"{{default('/configurations/kafka-env/kafka_log_dir', '/var/log/kafka')}}/server.log\"\n },\n {\n \"type\":\"kafka_statechange\",\n \"rowtype\":\"service\",\n \"path\":\"{{kafka_log_dir}}/state-change.log\"\n }\n ],\n \"filter\":[\n {\n \"filter\":\"grok\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"kafka_controller\",\n \"kafka_request\",\n \"kafka_logcleaner\"\n ]\n }\n },\n \"log4j_format\":\"[%d] %p %m (%c)%n\",\n \"multiline_pattern\":\"^(\\\\[%{TIMESTAMP_ISO8601:logtime}\\\\])\",\n \"message_pattern\":\"(?m)^\\\\[%{TIMESTAMP_ISO8601:logtime}\\\\]%{SPACE}%{LOGLEVEL:level}%{SPACE}\\\\[%{DATA:thread_name}\\\\]%{SPACE}%{GREEDYDATA:log_message}\",\n \"post_map_values\":{\n \"logtime\":{\n \"map_date\":{\n \"target_date_pattern\":\"yyyy-MM-dd HH:mm:ss,SSS\"\n }\n }\n }\n },\n {\n \"filter\":\"grok\",\n \"comment\":\"Suppose to be same log4j pattern as other kafka processes, but some reason thread is not printed\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"kafka_server\",\n \"kafka_statechange\"\n ]\n }\n },\n \"log4j_format\":\"[%d] %p %m (%c)%n\",\n \"multiline_pattern\":\"^(\\\\[%{TIMESTAMP_ISO8601:logtime}\\\\])\",\n \"message_pattern\":\"(?m)^\\\\[%{TIMESTAMP_ISO8601:logtime}\\\\]%{SPACE}%{LOGLEVEL:level}%{SPACE}%{GREEDYDATA:log_message}\",\n \"post_map_values\":{\n \"logtime\":{\n \"map_date\":{\n \"target_date_pattern\":\"yyyy-MM-dd HH:mm:ss,SSS\"\n }\n }\n }\n }\n ]\n }",
1575 "service_name" : "Kafka"
1576 }
1577 }
1578 },
1579 {
1580 "storm-atlas-application.properties" : {
1581 "properties_attributes" : { },
1582 "properties" : {
1583 "atlas.hook.storm.numRetries" : "3"
1584 }
1585 }
1586 },
1587 {
1588 "zookeeper-env" : {
1589 "properties_attributes" : { },
1590 "properties" : {
1591 "zk_log_dir" : "/var/log/zookeeper",
1592 "zk_user" : "zookeeper",
1593 "zk_pid_dir" : "/var/run/zookeeper",
1594 "content" : "\nexport JAVA_HOME={{java64_home}}\nexport ZOOKEEPER_HOME={{zk_home}}\nexport ZOO_LOG_DIR={{zk_log_dir}}\nexport ZOOPIDFILE={{zk_pid_file}}\nexport SERVER_JVMFLAGS={{zk_server_heapsize}}\nexport JAVA=$JAVA_HOME/bin/java\nexport CLASSPATH=$CLASSPATH:/usr/share/zookeeper/*\n\n{% if security_enabled %}\nexport SERVER_JVMFLAGS=\"$SERVER_JVMFLAGS -Djava.security.auth.login.config={{zk_server_jaas_file}}\"\nexport CLIENT_JVMFLAGS=\"$CLIENT_JVMFLAGS -Djava.security.auth.login.config={{zk_client_jaas_file}}\"\n{% endif %}"
1595 }
1596 }
1597 },
1598 {
1599 "yarn-log4j" : {
1600 "properties_attributes" : { },
1601 "properties" : {
1602 "yarn_rm_summary_log_max_backup_size" : "256",
1603 "yarn_rm_summary_log_number_of_backup_files" : "20",
1604 "content" : "\n#Relative to Yarn Log Dir Prefix\nyarn.log.dir=.\n#\n# Job Summary Appender\n#\n# Use following logger to send summary to separate file defined by\n# hadoop.mapreduce.jobsummary.log.file rolled daily:\n# hadoop.mapreduce.jobsummary.logger=INFO,JSA\n#\nhadoop.mapreduce.jobsummary.logger=${hadoop.root.logger}\nhadoop.mapreduce.jobsummary.log.file=hadoop-mapreduce.jobsummary.log\nlog4j.appender.JSA=org.apache.log4j.DailyRollingFileAppender\n# Set the ResourceManager summary log filename\nyarn.server.resourcemanager.appsummary.log.file=hadoop-mapreduce.jobsummary.log\n# Set the ResourceManager summary log level and appender\nyarn.server.resourcemanager.appsummary.logger=${hadoop.root.logger}\n#yarn.server.resourcemanager.appsummary.logger=INFO,RMSUMMARY\n\n# To enable AppSummaryLogging for the RM,\n# set yarn.server.resourcemanager.appsummary.logger to\n# LEVEL,RMSUMMARY in hadoop-env.sh\n\n# Appender for ResourceManager Application Summary Log\n# Requires the following properties to be set\n# - hadoop.log.dir (Hadoop Log directory)\n# - yarn.server.resourcemanager.appsummary.log.file (resource manager app summary log filename)\n# - yarn.server.resourcemanager.appsummary.logger (resource manager app summary log level and appender)\nlog4j.appender.RMSUMMARY=org.apache.log4j.RollingFileAppender\nlog4j.appender.RMSUMMARY.File=${yarn.log.dir}/${yarn.server.resourcemanager.appsummary.log.file}\nlog4j.appender.RMSUMMARY.MaxFileSize={{yarn_rm_summary_log_max_backup_size}}MB\nlog4j.appender.RMSUMMARY.MaxBackupIndex={{yarn_rm_summary_log_number_of_backup_files}}\nlog4j.appender.RMSUMMARY.layout=org.apache.log4j.PatternLayout\nlog4j.appender.RMSUMMARY.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n\nlog4j.appender.JSA.layout=org.apache.log4j.PatternLayout\nlog4j.appender.JSA.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n\nlog4j.appender.JSA.DatePattern=.yyyy-MM-dd\nlog4j.appender.JSA.layout=org.apache.log4j.PatternLayout\nlog4j.logger.org.apache.hadoop.yarn.server.resourcemanager.RMAppManager$ApplicationSummary=${yarn.server.resourcemanager.appsummary.logger}\nlog4j.additivity.org.apache.hadoop.yarn.server.resourcemanager.RMAppManager$ApplicationSummary=false\n\n# Appender for viewing information for errors and warnings\nyarn.ewma.cleanupInterval=300\nyarn.ewma.messageAgeLimitSeconds=86400\nyarn.ewma.maxUniqueMessages=250\nlog4j.appender.EWMA=org.apache.hadoop.yarn.util.Log4jWarningErrorMetricsAppender\nlog4j.appender.EWMA.cleanupInterval=${yarn.ewma.cleanupInterval}\nlog4j.appender.EWMA.messageAgeLimitSeconds=${yarn.ewma.messageAgeLimitSeconds}\nlog4j.appender.EWMA.maxUniqueMessages=${yarn.ewma.maxUniqueMessages}\n\n# Audit logging for ResourceManager\nrm.audit.logger=${hadoop.root.logger}\nlog4j.logger.org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger=${rm.audit.logger}\nlog4j.additivity.org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger=false\nlog4j.appender.RMAUDIT=org.apache.log4j.DailyRollingFileAppender\nlog4j.appender.RMAUDIT.File=${yarn.log.dir}/rm-audit.log\nlog4j.appender.RMAUDIT.layout=org.apache.log4j.PatternLayout\nlog4j.appender.RMAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n\nlog4j.appender.RMAUDIT.DatePattern=.yyyy-MM-dd\n\n# Audit logging for NodeManager\nnm.audit.logger=${hadoop.root.logger}\nlog4j.logger.org.apache.hadoop.yarn.server.nodemanager.NMAuditLogger=${nm.audit.logger}\nlog4j.additivity.org.apache.hadoop.yarn.server.nodemanager.NMAuditLogger=false\nlog4j.appender.NMAUDIT=org.apache.log4j.DailyRollingFileAppender\nlog4j.appender.NMAUDIT.File=${yarn.log.dir}/nm-audit.log\nlog4j.appender.NMAUDIT.layout=org.apache.log4j.PatternLayout\nlog4j.appender.NMAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n\nlog4j.appender.NMAUDIT.DatePattern=.yyyy-MM-dd"
1605 }
1606 }
1607 },
1608 {
1609 "mapred-env" : {
1610 "properties_attributes" : { },
1611 "properties" : {
1612 "jobhistory_heapsize" : "900",
1613 "mapred_user_nofile_limit" : "32768",
1614 "mapred_user" : "mapred",
1615 "mapred_user_nproc_limit" : "65536",
1616 "mapred_pid_dir_prefix" : "/var/run/hadoop-mapreduce",
1617 "mapred_log_dir_prefix" : "/var/log/hadoop-mapreduce",
1618 "content" : "\n# export JAVA_HOME=/home/y/libexec/jdk1.6.0/\n\nexport HADOOP_JOB_HISTORYSERVER_HEAPSIZE={{jobhistory_heapsize}}\n\nexport HADOOP_MAPRED_ROOT_LOGGER=INFO,RFA\n\n{% if security_enabled %}\nexport HADOOP_JOB_HISTORYSERVER_OPTS=\"-Djava.security.auth.login.config={{mapred_jaas_file}} -Djavax.security.auth.useSubjectCredsOnly=false\"\n{% endif %}\n#export HADOOP_MAPRED_LOG_DIR=\"\" # Where log files are stored. $HADOOP_MAPRED_HOME/logs by default.\n#export HADOOP_JHS_LOGGER=INFO,RFA # Hadoop JobSummary logger.\n#export HADOOP_MAPRED_PID_DIR= # The pid files are stored. /tmp by default.\n#export HADOOP_MAPRED_IDENT_STRING= #A string representing this instance of hadoop. $USER by default\n#export HADOOP_MAPRED_NICENESS= #The scheduling priority for daemons. Defaults to 0.\nexport HADOOP_OPTS=\"-Dhdp.version=$HDP_VERSION $HADOOP_OPTS\"\nexport HADOOP_OPTS=\"-Djava.io.tmpdir={{hadoop_java_io_tmpdir}} $HADOOP_OPTS\"\nexport JAVA_LIBRARY_PATH=\"${JAVA_LIBRARY_PATH}:{{hadoop_java_io_tmpdir}}\""
1619 }
1620 }
1621 },
1622 {
1623 "ranger-kafka-security" : {
1624 "properties_attributes" : { },
1625 "properties" : { }
1626 }
1627 },
1628 {
1629 "cluster-env" : {
1630 "properties_attributes" : { },
1631 "properties" : {
1632 "hide_yarn_memory_widget" : "false",
1633 "enable_external_ranger" : "false",
1634 "recovery_max_count" : "6",
1635 "manage_dirs_on_root" : "true",
1636 "managed_hdfs_resource_property_names" : "",
1637 "recovery_retry_interval" : "5",
1638 "stack_features" : "{\n \"HDP\": {\n \"stack_features\": [\n {\n \"name\": \"snappy\",\n \"description\": \"Snappy compressor/decompressor support\",\n \"min_version\": \"2.0.0.0\",\n \"max_version\": \"2.2.0.0\"\n },\n {\n \"name\": \"lzo\",\n \"description\": \"LZO libraries support\",\n \"min_version\": \"2.2.1.0\"\n },\n {\n \"name\": \"express_upgrade\",\n \"description\": \"Express upgrade support\",\n \"min_version\": \"2.1.0.0\"\n },\n {\n \"name\": \"rolling_upgrade\",\n \"description\": \"Rolling upgrade support\",\n \"min_version\": \"2.2.0.0\"\n },\n {\n \"name\": \"kafka_acl_migration_support\",\n \"description\": \"ACL migration support\",\n \"min_version\": \"2.3.4.0\"\n },\n {\n \"name\": \"secure_zookeeper\",\n \"description\": \"Protect ZNodes with SASL acl in secure clusters\",\n \"min_version\": \"2.6.0.0\"\n },\n {\n \"name\": \"config_versioning\",\n \"description\": \"Configurable versions support\",\n \"min_version\": \"2.3.0.0\"\n },\n {\n \"name\": \"datanode_non_root\",\n \"description\": \"DataNode running as non-root support (AMBARI-7615)\",\n \"min_version\": \"2.2.0.0\"\n },\n {\n \"name\": \"remove_ranger_hdfs_plugin_env\",\n \"description\": \"HDFS removes Ranger env files (AMBARI-14299)\",\n \"min_version\": \"2.3.0.0\"\n },\n {\n \"name\": \"ranger\",\n \"description\": \"Ranger Service support\",\n \"min_version\": \"2.2.0.0\"\n },\n {\n \"name\": \"ranger_tagsync_component\",\n \"description\": \"Ranger Tagsync component support (AMBARI-14383)\",\n \"min_version\": \"2.5.0.0\"\n },\n {\n \"name\": \"phoenix\",\n \"description\": \"Phoenix Service support\",\n \"min_version\": \"2.3.0.0\"\n },\n {\n \"name\": \"nfs\",\n \"description\": \"NFS support\",\n \"min_version\": \"2.3.0.0\"\n },\n {\n \"name\": \"tez_for_spark\",\n \"description\": \"Tez dependency for Spark\",\n \"min_version\": \"2.2.0.0\",\n \"max_version\": \"2.3.0.0\"\n },\n {\n \"name\": \"timeline_state_store\",\n \"description\": \"Yarn application timeline-service supports state store property (AMBARI-11442)\",\n \"min_version\": \"2.2.0.0\"\n },\n {\n \"name\": \"copy_tarball_to_hdfs\",\n \"description\": \"Copy tarball to HDFS support (AMBARI-12113)\",\n \"min_version\": \"2.2.0.0\"\n },\n {\n \"name\": \"spark_16plus\",\n \"description\": \"Spark 1.6+\",\n \"min_version\": \"2.4.0.0\"\n },\n {\n \"name\": \"spark_thriftserver\",\n \"description\": \"Spark Thrift Server\",\n \"min_version\": \"2.3.2.0\"\n },\n {\n \"name\": \"storm_kerberos\",\n \"description\": \"Storm Kerberos support (AMBARI-7570)\",\n \"min_version\": \"2.2.0.0\"\n },\n {\n \"name\": \"storm_ams\",\n \"description\": \"Storm AMS integration (AMBARI-10710)\",\n \"min_version\": \"2.2.0.0\"\n },\n {\n \"name\": \"create_kafka_broker_id\",\n \"description\": \"Ambari should create Kafka Broker Id (AMBARI-12678)\",\n \"min_version\": \"2.2.0.0\",\n \"max_version\": \"2.3.0.0\"\n },\n {\n \"name\": \"kafka_listeners\",\n \"description\": \"Kafka listeners (AMBARI-10984)\",\n \"min_version\": \"2.3.0.0\"\n },\n {\n \"name\": \"kafka_kerberos\",\n \"description\": \"Kafka Kerberos support (AMBARI-10984)\",\n \"min_version\": \"2.3.0.0\"\n },\n {\n \"name\": \"pig_on_tez\",\n \"description\": \"Pig on Tez support (AMBARI-7863)\",\n \"min_version\": \"2.2.0.0\"\n },\n {\n \"name\": \"ranger_usersync_non_root\",\n \"description\": \"Ranger Usersync as non-root user (AMBARI-10416)\",\n \"min_version\": \"2.3.0.0\"\n },\n {\n \"name\": \"ranger_audit_db_support\",\n \"description\": \"Ranger Audit to DB support\",\n \"min_version\": \"2.2.0.0\",\n \"max_version\": \"2.4.99.99\"\n },\n {\n \"name\": \"accumulo_kerberos_user_auth\",\n \"description\": \"Accumulo Kerberos User Auth (AMBARI-10163)\",\n \"min_version\": \"2.3.0.0\"\n },\n {\n \"name\": \"knox_versioned_data_dir\",\n \"description\": \"Use versioned data dir for Knox (AMBARI-13164)\",\n \"min_version\": \"2.3.2.0\"\n },\n {\n \"name\": \"knox_sso_topology\",\n \"description\": \"Knox SSO Topology support (AMBARI-13975)\",\n \"min_version\": \"2.3.8.0\"\n },\n {\n \"name\": \"atlas_rolling_upgrade\",\n \"description\": \"Rolling upgrade support for Atlas\",\n \"min_version\": \"2.3.0.0\"\n },\n {\n \"name\": \"oozie_admin_user\",\n \"description\": \"Oozie install user as an Oozie admin user (AMBARI-7976)\",\n \"min_version\": \"2.2.0.0\"\n },\n {\n \"name\": \"oozie_create_hive_tez_configs\",\n \"description\": \"Oozie create configs for Ambari Hive and Tez deployments (AMBARI-8074)\",\n \"min_version\": \"2.2.0.0\"\n },\n {\n \"name\": \"oozie_setup_shared_lib\",\n \"description\": \"Oozie setup tools used to shared Oozie lib to HDFS (AMBARI-7240)\",\n \"min_version\": \"2.2.0.0\"\n },\n {\n \"name\": \"oozie_host_kerberos\",\n \"description\": \"Oozie in secured clusters uses _HOST in Kerberos principal (AMBARI-9775)\",\n \"min_version\": \"2.0.0.0\"\n },\n {\n \"name\": \"falcon_extensions\",\n \"description\": \"Falcon Extension\",\n \"min_version\": \"2.5.0.0\"\n },\n {\n \"name\": \"hive_metastore_upgrade_schema\",\n \"description\": \"Hive metastore upgrade schema support (AMBARI-11176)\",\n \"min_version\": \"2.3.0.0\"\n },\n {\n \"name\": \"hive_server_interactive\",\n \"description\": \"Hive server interactive support (AMBARI-15573)\",\n \"min_version\": \"2.5.0.0\"\n },\n {\n \"name\": \"hive_webhcat_specific_configs\",\n \"description\": \"Hive webhcat specific configurations support (AMBARI-12364)\",\n \"min_version\": \"2.3.0.0\"\n },\n {\n \"name\": \"hive_purge_table\",\n \"description\": \"Hive purge table support (AMBARI-12260)\",\n \"min_version\": \"2.3.0.0\"\n },\n {\n \"name\": \"hive_server2_kerberized_env\",\n \"description\": \"Hive server2 working on kerberized environment (AMBARI-13749)\",\n \"min_version\": \"2.2.3.0\",\n \"max_version\": \"2.2.5.0\"\n },\n {\n \"name\": \"hive_env_heapsize\",\n \"description\": \"Hive heapsize property defined in hive-env (AMBARI-12801)\",\n \"min_version\": \"2.2.0.0\"\n },\n {\n \"name\": \"ranger_kms_hsm_support\",\n \"description\": \"Ranger KMS HSM support (AMBARI-15752)\",\n \"min_version\": \"2.5.0.0\"\n },\n {\n \"name\": \"ranger_log4j_support\",\n \"description\": \"Ranger supporting log-4j properties (AMBARI-15681)\",\n \"min_version\": \"2.5.0.0\"\n },\n {\n \"name\": \"ranger_kerberos_support\",\n \"description\": \"Ranger Kerberos support\",\n \"min_version\": \"2.5.0.0\"\n },\n {\n \"name\": \"hive_metastore_site_support\",\n \"description\": \"Hive Metastore site support\",\n \"min_version\": \"2.5.0.0\"\n },\n {\n \"name\": \"ranger_usersync_password_jceks\",\n \"description\": \"Saving Ranger Usersync credentials in jceks\",\n \"min_version\": \"2.5.0.0\"\n },\n {\n \"name\": \"ranger_install_infra_client\",\n \"description\": \"Ambari Infra Service support\",\n \"min_version\": \"2.5.0.0\"\n },\n {\n \"name\": \"falcon_atlas_support_2_3\",\n \"description\": \"Falcon Atlas integration support for 2.3 stack\",\n \"min_version\": \"2.3.99.0\",\n \"max_version\": \"2.4.0.0\"\n },\n {\n \"name\": \"falcon_atlas_support\",\n \"description\": \"Falcon Atlas integration\",\n \"min_version\": \"2.5.0.0\"\n },\n {\n \"name\": \"hbase_home_directory\",\n \"description\": \"Hbase home directory in HDFS needed for HBASE backup\",\n \"min_version\": \"2.5.0.0\"\n },\n {\n \"name\": \"spark_livy\",\n \"description\": \"Livy as slave component of spark\",\n \"min_version\": \"2.5.0.0\"\n },\n {\n \"name\": \"spark_livy2\",\n \"description\": \"Livy as slave component of spark\",\n \"min_version\": \"2.6.0.0\"\n },\n {\n \"name\": \"atlas_ranger_plugin_support\",\n \"description\": \"Atlas Ranger plugin support\",\n \"min_version\": \"2.5.0.0\"\n },\n {\n \"name\": \"atlas_conf_dir_in_path\",\n \"description\": \"Prepend the Atlas conf dir (/etc/atlas/conf) to the classpath of Storm and Falcon\",\n \"min_version\": \"2.3.0.0\",\n \"max_version\": \"2.4.99.99\"\n },\n {\n \"name\": \"atlas_upgrade_support\",\n \"description\": \"Atlas supports express and rolling upgrades\",\n \"min_version\": \"2.5.0.0\"\n },\n {\n \"name\": \"atlas_hook_support\",\n \"description\": \"Atlas support for hooks in Hive, Storm, Falcon, and Sqoop\",\n \"min_version\": \"2.5.0.0\"\n },\n {\n \"name\": \"ranger_pid_support\",\n \"description\": \"Ranger Service support pid generation AMBARI-16756\",\n \"min_version\": \"2.5.0.0\"\n },\n {\n \"name\": \"ranger_kms_pid_support\",\n \"description\": \"Ranger KMS Service support pid generation\",\n \"min_version\": \"2.5.0.0\"\n },\n {\n \"name\": \"ranger_admin_password_change\",\n \"description\": \"Allow ranger admin credentials to be specified during cluster creation (AMBARI-17000)\",\n \"min_version\": \"2.5.0.0\"\n },\n {\n \"name\": \"ranger_setup_db_on_start\",\n \"description\": \"Allows setup of ranger db and java patches to be called multiple times on each START\",\n \"min_version\": \"2.6.0.0\"\n },\n {\n \"name\": \"storm_metrics_apache_classes\",\n \"description\": \"Metrics sink for Storm that uses Apache class names\",\n \"min_version\": \"2.5.0.0\"\n },\n {\n \"name\": \"spark_java_opts_support\",\n \"description\": \"Allow Spark to generate java-opts file\",\n \"min_version\": \"2.2.0.0\",\n \"max_version\": \"2.4.0.0\"\n },\n {\n \"name\": \"atlas_hbase_setup\",\n \"description\": \"Use script to create Atlas tables in Hbase and set permissions for Atlas user.\",\n \"min_version\": \"2.5.0.0\"\n },\n {\n \"name\": \"ranger_hive_plugin_jdbc_url\",\n \"description\": \"Handle Ranger hive repo config jdbc url change for stack 2.5 (AMBARI-18386)\",\n \"min_version\": \"2.5.0.0\"\n },\n {\n \"name\": \"zkfc_version_advertised\",\n \"description\": \"ZKFC advertise version\",\n \"min_version\": \"2.5.0.0\"\n },\n {\n \"name\": \"phoenix_core_hdfs_site_required\",\n \"description\": \"HDFS and CORE site required for Phoenix\",\n \"max_version\": \"2.5.9.9\"\n },\n {\n \"name\": \"ranger_tagsync_ssl_xml_support\",\n \"description\": \"Ranger Tagsync ssl xml support.\",\n \"min_version\": \"2.6.0.0\"\n },\n {\n \"name\": \"ranger_xml_configuration\",\n \"description\": \"Ranger code base support xml configurations\",\n \"min_version\": \"2.3.0.0\"\n },\n {\n \"name\": \"kafka_ranger_plugin_support\",\n \"description\": \"Ambari stack changes for Ranger Kafka Plugin (AMBARI-11299)\",\n \"min_version\": \"2.3.0.0\"\n },\n {\n \"name\": \"yarn_ranger_plugin_support\",\n \"description\": \"Implement Stack changes for Ranger Yarn Plugin integration (AMBARI-10866)\",\n \"min_version\": \"2.3.0.0\"\n },\n {\n \"name\": \"ranger_solr_config_support\",\n \"description\": \"Showing Ranger solrconfig.xml on UI\",\n \"min_version\": \"2.6.0.0\"\n },\n {\n \"name\": \"hive_interactive_atlas_hook_required\",\n \"description\": \"Registering Atlas Hook for Hive Interactive.\",\n \"min_version\": \"2.6.0.0\"\n },\n {\n \"name\": \"core_site_for_ranger_plugins\",\n \"description\": \"Adding core-site.xml in when Ranger plugin is enabled for Storm, Kafka, and Knox.\",\n \"min_version\": \"2.6.0.0\"\n },\n {\n \"name\": \"atlas_install_hook_package_support\",\n \"description\": \"Stop installing packages from 2.6\",\n \"max_version\": \"2.5.9.9\"\n },\n {\n \"name\": \"atlas_hdfs_site_on_namenode_ha\",\n \"description\": \"Need to create hdfs-site under atlas-conf dir when Namenode-HA is enabled.\",\n \"min_version\": \"2.6.0.0\"\n },\n {\n \"name\": \"hive_interactive_ga\",\n \"description\": \"Hive Interactive GA support\",\n \"min_version\": \"2.6.0.0\"\n },\n {\n \"name\": \"secure_ranger_ssl_password\",\n \"description\": \"Securing Ranger Admin and Usersync SSL and Trustore related passwords in jceks\",\n \"min_version\": \"2.6.0.0\"\n },\n {\n \"name\": \"ranger_kms_ssl\",\n \"description\": \"Ranger KMS SSL properties in ambari stack\",\n \"min_version\": \"2.6.0.0\"\n },\n {\n \"name\": \"nifi_encrypt_config\",\n \"description\": \"Encrypt sensitive properties written to nifi property file\",\n \"min_version\": \"2.6.0.0\"\n },\n {\n \"name\": \"toolkit_config_update\",\n \"description\": \"Support separate input and output for toolkit configuration\",\n \"min_version\": \"2.6.0.0\"\n },\n {\n \"name\": \"admin_toolkit_support\",\n \"description\": \"Supports the nifi admin toolkit\",\n \"min_version\": \"2.6.0.0\"\n },\n {\n \"name\": \"tls_toolkit_san\",\n \"description\": \"Support subject alternative name flag\",\n \"min_version\": \"2.6.0.0\"\n },\n {\n \"name\": \"nifi_jaas_conf_create\",\n \"description\": \"Create NIFI jaas configuration when kerberos is enabled\",\n \"min_version\": \"2.6.0.0\"\n },\n {\n \"name\": \"atlas_core_site_support\",\n \"description\": \"Need to create core-site under Atlas conf directory.\",\n \"min_version\": \"2.6.0.0\"\n },\n {\n \"name\": \"hadoop_custom_extensions\",\n \"description\": \"Support hadoop custom extensions\",\n \"min_version\": \"2.6.0.0\"\n }\n ]\n }\n}",
1639 "stack_tools" : "{\n \"HDP\": {\n \"stack_selector\": [\n \"hdp-select\",\n \"/usr/bin/hdp-select\",\n \"hdp-select\"\n ],\n \"conf_selector\": [\n \"conf-select\",\n \"/usr/bin/conf-select\",\n \"conf-select\"\n ]\n }\n}",
1640 "repo_ubuntu_template" : "{{package_type}} {{base_url}} {{components}}",
1641 "stack_root" : "{\"HDP\":\"/usr/hdp\"}",
1642 "fetch_nonlocal_groups" : "true",
1643 "one_dir_per_partition" : "false",
1644 "kerberos_domain" : "EXAMPLE.COM",
1645 "ignore_bad_mounts" : "false",
1646 "user_group" : "hadoop",
1647 "sysprep_skip_setup_jce" : "false",
1648 "override_uid" : "true",
1649 "security_enabled" : "false",
1650 "sysprep_skip_copy_fast_jar_hdfs" : "false",
1651 "agent_mounts_ignore_list" : "",
1652 "recovery_lifetime_max_count" : "1024",
1653 "recovery_type" : "AUTO_START",
1654 "repo_suse_rhel_template" : "[{{repo_id}}]\nname={{repo_id}}\n{% if mirror_list %}mirrorlist={{mirror_list}}{% else %}baseurl={{base_url}}{% endif %}\n\npath=/\nenabled=1\ngpgcheck=0",
1655 "sysprep_skip_create_users_and_groups" : "false",
1656 "sysprep_skip_copy_tarballs_hdfs" : "false",
1657 "sysprep_skip_copy_oozie_share_lib_to_hdfs" : "false",
1658 "recovery_enabled" : "true",
1659 "smokeuser" : "ambari-qa",
1660 "smokeuser_keytab" : "/etc/security/keytabs/smokeuser.headless.keytab",
1661 "alerts_repeat_tolerance" : "1",
1662 "recovery_window_in_minutes" : "60",
1663 "stack_name" : "HDP",
1664 "ignore_groupsusers_create" : "false"
1665 }
1666 }
1667 },
1668 {
1669 "druid-superset-env" : {
1670 "properties_attributes" : { },
1671 "properties" : {
1672 "superset_admin_lastname" : "Works",
1673 "superset_pid_dir" : "/var/run/superset",
1674 "superset_log_dir" : "/var/log/superset",
1675 "superset_admin_email" : "admin@example.com",
1676 "superset_admin_firstname" : "Horton",
1677 "content" : "\n # Set environment variables here.\n \n export LC_ALL=en_US.UTF-8\n export LANG=en_US.UTF-8\n\n # Superset Home Dir\n export SUPERSET_CONFIG_DIR={{superset_config_dir}}\n\n # Superset Log Dir\n export SUPERSET_LOG_DIR={{superset_log_dir}}\n\n # Superset Log Dir\n export SUPERSET_BIN_DIR={{superset_bin_dir}}\n\n # Superset PID Dir\n export SUPERSET_PID_DIR={{superset_pid_dir}}\n\n # Add superset_config.py to PYTHONPATH\n export PYTHONPATH=$PYTHONPATH:{{superset_config_dir}}\n\n export SUPERSET_WEBSERVER_PORT={{superset_webserver_port}}\n export SUPERSET_WEBSERVER_ADDRESS={{superset_webserver_address}}\n export SUPERSET_TIMEOUT={{superset_timeout}}\n export SUPERSET_WORKERS={{superset_workers}}",
1678 "superset_admin_user" : "admin"
1679 }
1680 }
1681 }
1682 ],
1683 "host_groups" : [
1684 {
1685 "components" : [
1686 {
1687 "name" : "NODEMANAGER"
1688 },
1689 {
1690 "name" : "DRUID_MIDDLEMANAGER"
1691 },
1692 {
1693 "name" : "DRUID_ROUTER"
1694 },
1695 {
1696 "name" : "REGISTRY_SERVER"
1697 },
1698 {
1699 "name" : "DRUID_HISTORICAL"
1700 },
1701 {
1702 "name" : "NFS_GATEWAY"
1703 },
1704 {
1705 "name" : "DRUID_BROKER"
1706 },
1707 {
1708 "name" : "HBASE_MASTER"
1709 },
1710 {
1711 "name" : "NIFI_CA"
1712 },
1713 {
1714 "name" : "ZOOKEEPER_CLIENT"
1715 },
1716 {
1717 "name" : "KAFKA_BROKER"
1718 },
1719 {
1720 "name" : "SECONDARY_NAMENODE"
1721 },
1722 {
1723 "name" : "ZOOKEEPER_SERVER"
1724 },
1725 {
1726 "name" : "HBASE_REGIONSERVER"
1727 },
1728 {
1729 "name" : "NIFI_MASTER"
1730 },
1731 {
1732 "name" : "DRPC_SERVER"
1733 },
1734 {
1735 "name" : "PHOENIX_QUERY_SERVER"
1736 },
1737 {
1738 "name" : "DRUID_SUPERSET"
1739 },
1740 {
1741 "name" : "DRUID_COORDINATOR"
1742 },
1743 {
1744 "name" : "YARN_CLIENT"
1745 },
1746 {
1747 "name" : "HDFS_CLIENT"
1748 },
1749 {
1750 "name" : "STORM_UI_SERVER"
1751 },
1752 {
1753 "name" : "HISTORYSERVER"
1754 },
1755 {
1756 "name" : "SUPERVISOR"
1757 },
1758 {
1759 "name" : "NAMENODE"
1760 },
1761 {
1762 "name" : "NIMBUS"
1763 },
1764 {
1765 "name" : "DRUID_OVERLORD"
1766 },
1767 {
1768 "name" : "MAPREDUCE2_CLIENT"
1769 },
1770 {
1771 "name" : "AMBARI_SERVER"
1772 },
1773 {
1774 "name" : "STREAMLINE_SERVER"
1775 },
1776 {
1777 "name" : "DATANODE"
1778 },
1779 {
1780 "name" : "HBASE_CLIENT"
1781 },
1782 {
1783 "name" : "APP_TIMELINE_SERVER"
1784 },
1785 {
1786 "name" : "RESOURCEMANAGER"
1787 }
1788 ],
1789 "configurations" : [ ],
1790 "name" : "host_group_1",
1791 "cardinality" : "1"
1792 }
1793 ],
1794 "settings" : [
1795 {
1796 "recovery_settings" : [
1797 {
1798 "recovery_enabled" : "false"
1799 }
1800 ]
1801 },
1802 {
1803 "service_settings" : [ ]
1804 },
1805 {
1806 "component_settings" : [ ]
1807 }
1808 ],
1809 "Blueprints" : {
1810 "stack_name" : "HDP",
1811 "stack_version" : "2.6"
1812 }
1813}