============================================================
INSTALL APACHE KAFKA 2.13-3.4.1

OS: Linux
Platform: Rocky 9
Hadoop Version: 3.3.6
============================================================

Step 1: Download and Extract
============================

# Download and extract the package (perform as root)

  wget -P /tmp https://archive.apache.org/dist/kafka/3.4.1/kafka_2.13-3.4.1.tgz

  tar xvzf /tmp/kafka_2.13-3.4.1.tgz -C /opt

# Set Kafka profile and path

  echo 'export PATH=$PATH:/opt/kafka_2.13-3.4.1/bin; export KAFKA_HOME=/opt/kafka_2.13-3.4.1; export ZOOKEEPER_HOME=/opt/kafka_2.13-3.4.1/bin' >/etc/profile.d/kafka.sh

Step 2: Copy Config Files, Set up Kafka User, and Data Directory
================================================================

# copy config files, make copy of original files first

  cp /opt/kafka_2.13-3.4.1/config/server.properties  /opt/kafka_2.13-3.4.1/config/server.properties.orig
  cp /opt/kafka_2.13-3.4.1/config/zookeeper.properties /opt/kafka_2.13-3.4.1/config/zookeeper.properties.orig

# Do the copy from files directory

  cp files/server.properties  /opt/kafka_2.13-3.4.1/config
  cp files/zookeeper.properties /opt/kafka_2.13-3.4.1/config

# create a Kafka user and change ownership (do as root)

  useradd -g hadoop kafka
  chown -R kafka:hadoop /opt/kafka_2.13-3.4.1/

# Make Kafka Log (for user data) directory (under /var)

  mkdir /var/data/kafka
  mkdir /var/data/kafka/zookeeper
  chown -R kafka:hadoop /var/data/kafka

# Make Kafka SERVER log (not user data) directory (under /var/log)

  mkdir /var/log/kafka
  chown kafka:hadoop /var/log/kafka/

# Create Kafka user in HDFS (perform as root, become user hdfs)

  su - hdfs
  hdfs dfs -mkdir -p /user/kafka
  hdfs dfs -chown -R kafka:hadoop  /user/kafka
  exit

Step 3: Setup Systemd
=====================

# Copy Kafka start and stop scripts to /opt/services (perform as root)
# Starts/Stops a single instance of zookeeper and a Kafka server

  cp scripts/*-kafka.sh /opt/services/
  chown kafka:hadoop /opt/services/*-kafka.sh
  chmod 744 /opt/services/*-kafka.sh

# Copy the systemd service script

  cp files/kafka.service /etc/systemd/system
  systemctl enable kafka.service

# To Start Kafka
  
  systemctl start kafka.service

# To Stop Kafka
 
  systemctl stop kafka.service

Step 4: Testing Kafka
=====================

# From FIRST Terminal window
# If any of these scripts time out, restart Kafka (systemctl start kafka.service)

# Create topic "test"

  kafka-topics.sh --create --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1 --topic test

# List the topics

  kafka-topics.sh --list --bootstrap-server localhost:9092

#  start a producer to grab input from the terminal

  kafka-console-producer.sh --bootstrap-server localhost:9092 --topic test
  > ENTER TEXT HERE

# From SECOND terminal start a consumer
  
  kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic test --from-beginning
  
# Text entered in FIRST producer window should show up in consumer window


Step 5: Install the KafkaEsque GUI (X86_64 only)

================================================

# Get rpm and install:

  wget https://github.com/patschuh/KafkaEsque/releases/download/v2.9.6/kafkaesque-2.9.6-1.x86_64.rpm
  rpm -i kafkaesque-2.9.6-1.x86_64.rpm

# Add to kafka profile (Note: use ">>")

  echo 'export PATH=$PATH:/opt/kafkaesque/bin;' >>/etc/profile.d/kafka.sh

# copy wrapper script (hides output) and make sure it is executable

  cp scripts/kafkaesque /opt/kafkaesque/bin/
  chmod ugo+x /opt/kafkaesque/bin/kafkaesque

# Start as follows. Note: The wrapper starts KafkaEsque and places it in
# the backgound. It also redirects all run output (INFO messages)
# to the user: ~/.kafkaesque/kafkaesque.run Also, the previous
# kafkaesque.run is overwritten on a new invocation.
#
# Make sure the SSH session has been launched with -X or -XY for Linux and Mac systems.
# On windows, MobaXterm sets  this automatically

  $ kafkaesque
  Run output is in /home/hands-on/.kafkaesque/kafkaesque.run
  To stop KafkaEsque, close the application window.

