ElasticSearch

Installation

CentOS

1. Insatall java
   sudo dnf install java-1.8.0-openjdk.x86_64
2. Confirm java installed
   java -version
3. Add repo
   

cd yum.repos.d/
vi elasticsearch.repo

4. Add in code below

[Elasticsearch-7]
name=Elasticsearch repository for 7.x packages
baseurl=https://artifacts.elastic.co/packages/7.x/yum
gpgcheck=1
gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch
enabled=1
autorefresh=1
type=rpm-md

5. Install
   sudo yum install --enablerepo=elasticsearch elasticsearch
6. Configure /etc/elasticsearch/elasticsearch.yml

cluster.name: cluster-name
node.name: node-1
cluster.initial_master_nodes: ["node-1"]

6. Enable service

sudo /bin/systemctl daemon-reload
sudo /bin/systemctl enable elasticsearch.service
sudo systemctl start elasticsearch.service
sudo systemctl status elasticsearch.service
sudo netstat -pnltu | grep elastic
sudo netstat -pnltu | grep elasticsearch
sudo netstat -pnltu

7. Enable firewall

sudo firewall-cmd --add-port=9200/tcp --permament
sudo firewall-cmd --add-port=9200/tcp --permanent
sudo firewall-cmd --add-port=9
sudo firewall-cmd --reload

10. Test
    curl -X GET "http://localhost:9200/"

Nodejs example

const elasticsearch = require('@elastic/elasticsearch')

async function search() {
  const client = new elasticsearch.Client(
    {
      cloud: {
        id: '<your cloud id>'
      },
      log: 'trace',
      auth: {
        username: '<username>',
        password: '<password>',
      },
    }
  )
  const result = await client.search(
    {
      index: '<index>',
      from: 1,
      size: 10,
    }, {
      ignore: [404],
      maxRetries: 3
    }
  )
  console.log(JSON.stringify(result))
}

search()

Multi node setup

Create instances

Create instances.yaml,

instances:
  - name: es-1
    ip:
      - 10.0.1.10
    dns:
      - es-1
  - name: es-2
    ip:
      - 10.0.2.10
    dns:
      - es-2

Create certificate

To create certificate,

bin/elasticsearch-certutil ca --pem --out ./path/ca.zip
unzip -o ./path/ca.zip -d ./path/ca
bin/elasticsearch-certutil cert --pem --in ./path/instances.yaml --ca-cert ./path/ca/ca.crt --ca-key ./path/ca/ca.key --out ./part/certs.zip

S3 snapshots

Create IAM policy for EC2:

• ListBucket
• GetBucketLocation
• GetObject
• PutObject
• DeleteObject

{
    "Version": "2012-10-17",
    "Statement": [
        {
            "Sid": "VisualEditor0",
            "Effect": "Allow",
            "Action": [
                "s3:PutObject",
                "s3:GetObject",
                "s3:ListBucket",
                "s3:DeleteObject",
                "s3:GetBucketLocation"
            ],
            "Resource": [
                "arn:aws:s3:::es-snapshots/*",
                "arn:aws:s3:::es-snapshots"
            ]
        }
    ]
}

To install s3 plugin. For version 8.x, this is no longer a plugin and not required.

bin/elasticsearch-plugin install repository-s3

Health check

# Check shards
curl -u elastic:PASSWORD http://localhost:9200/_cat/shards?v

# Check node
curl -u elastic:PASSWORD http://localhost:9200/_cat/nodes?v

# Check cluster
curl -u elastic:PASSWORD http://localhost:9200/_cluster/health?pretty

To check if the module/plugin available:

# Check modules
curl -u username:password http://localhost:9200/_nodes?filter_path=nodes.*.modules.name

# To register
curl -u username:password -X PUT "http://localhost:9200/_snapshot/s3_backup" \
  -H "Content-Type: application/json" \
  -d '{
    "type": "s3",
    "settings": {
      "bucket": "bucket-name",
      "region": "ap-southeast-1",
      "base_path": "path",
      "compress": true
    }
  }' 

# To check
curl -u elastic:password http://localhost:9200/_snapshot/_all?pretty

# List indices
curl -u elastic:password http://localhost:9200/_cat/indices?v

# Snapshot
curl -u elastic:password -X PUT \
  "http://localhost:9200/_snapshot/s3_backup/snap-$(date +%Y%m%d-%H%M)" \
  -H "Content-Type: application/json" \
  -d '{
    "indices": "*",
    "ignore_unavailable": true,
    "include_global_state": true
  }'

# List Snapshot
curl -u elastic:PASSWORD \
  "http://<ES_HOST>:9200/_snapshot/s3_backup/_all?ignore_unavailable=true&pretty"
  
# Restore Snapshot
curl -u elastic:PASSWORD -X POST \
"http://<NEW_MASTER_IP>:9200/_snapshot/<repo>/<snapshot_name>/_restore" \
-H "Content-Type: application/json" \
-d '{
  "indices": "*",
  "include_global_state": true
}'
  
# Add template
curl -u elastic:PASSWORD -X PUT "http://MASTER_IP:9200/_index_template/default-durable" \
  -H "Content-Type: application/json" \
  -d '{
    "index_patterns": ["*"],
    "template": { "settings": { "number_of_replicas": 1 } },
    "priority": 0
  }'
  

# Update replicas
curl -u elastic:PASSWORD -X PUT "http://MASTER_IP:9200/*/_settings" \
  -H "Content-Type: application/json" \
  -d '{ "index": { "number_of_replicas": 1 } }'
  
# Add schedule Snapshot
curl -u elastic:password -X PUT \
  "http://localhost:9200/_slm/policy/daily-s3-7d" \
  -H "Content-Type: application/json" \
  -d '{
    "schedule": "0 30 1 * * ?",
    "name": "<snap-{now/d}>",
    "repository": "s3_backup",
    "config": {
      "indices": ["*"],
      "ignore_unavailable": true,
      "include_global_state": true
    },
    "retention": {
      "expire_after": "7d",
      "min_count": 3,
      "max_count": 10
    }
  }'

OS Configure

sudo sysctl -w vm.max_map_count=262144

echo "vm.max_map_count=262144" | sudo tee /etc/sysctl.d/99-elasticsearch.conf
sudo sysctl --system

Docker Compose files

Master + Data Node

services:
  elasticsearch:
    image: docker.elastic.co/elasticsearch/elasticsearch:8.17.2
    container_name: es-1
    network_mode: host
    environment:
      TZ: Asia/Kuala_Lumpur

      cluster.name: es-cluster
      node.name: es-1
      node.roles: master,data

      network.host: 0.0.0.0
      http.port: 9200
      transport.port: 9300

      # discovery (multi-host)
      discovery.seed_hosts: 10.0.1.10,10.0.2.10

      # bootstrap ONLY the first time you create this new 2-node cluster
      cluster.initial_master_nodes: es-1

      # auth
      ELASTIC_PASSWORD: some_password
      xpack.security.enabled: "true"
      xpack.security.enrollment.enabled: "false"

      # transport TLS (node-to-node)
      xpack.security.transport.ssl.enabled: "true"
      xpack.security.transport.ssl.verification_mode: certificate
      xpack.security.transport.ssl.certificate_authorities: /usr/share/elasticsearch/config/certs/ca/ca.crt
      xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/certs/nodes/es-1/es-1.crt
      xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/certs/nodes/es-1/es-1.key

      # HTTP TLS off (ok inside VPC; enable later if you want)
      xpack.security.http.ssl.enabled: "false"

      ES_JAVA_OPTS: -Xms1g -Xmx1g

    volumes:
      - ./data:/usr/share/elasticsearch/data
      - ./logs:/usr/share/elasticsearch/logs
      - ./certs:/usr/share/elasticsearch/config/certs
    ulimits:
      memlock:
        soft: -1
        hard: -1
      nofile:
        soft: 65535
        hard: 65535
    restart: unless-stopped

Data node

services:
  elasticsearch:
    image: docker.elastic.co/elasticsearch/elasticsearch:8.17.2
    container_name: es-2
    network_mode: host
    environment:
      TZ: Asia/Kuala_Lumpur

      cluster.name: es-cluster
      node.name: es-2
      node.roles: data

      network.host: 0.0.0.0
      http.port: 9200
      transport.port: 9300

      discovery.seed_hosts: 10.0.1.10,10.0.2.10

      ELASTIC_PASSWORD: some_password
      xpack.security.enabled: "true"
      xpack.security.enrollment.enabled: "false"

      xpack.security.transport.ssl.enabled: "true"
      xpack.security.transport.ssl.verification_mode: certificate
      xpack.security.transport.ssl.certificate_authorities: /usr/share/elasticsearch/config/certs/ca/ca.crt
      xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/certs/nodes/es-2/es-2.crt
      xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/certs/nodes/es-2/es-2.key

      xpack.security.http.ssl.enabled: "false"

      ES_JAVA_OPTS: -Xms1g -Xmx1g

    volumes:
      - ./data:/usr/share/elasticsearch/data
      - ./logs:/usr/share/elasticsearch/logs
      - ./certs:/usr/share/elasticsearch/config/certs
    ulimits:
      memlock:
        soft: -1
        hard: -1
      nofile:
        soft: 65535
        hard: 65535
    restart: unless-stopped