QPod · haobibo · Sep 21, 2022 · Sep 20, 2022 · Sep 21, 2022 · Sep 21, 2022
diff --git a/.gitattributes b/.gitattributes
@@ -0,0 +1,25 @@
+# Ignore all differences in line endings
+*               text=auto eol=lf
+*.md            text eol=lf
+*.py            text eol=lf
+*.sh            text eol=lf
+*.yml           text eol=lf
+*.yaml          text eol=lf
+*.json          text eol=lf
+*.properties    text eol=lf
+*.conf          text eol=lf
+*.ipynb         text eol=lf
+Dockerfile*     text eol=lf
+.gitattributes  text eol=lf
+.gitignore      text eol=lf
+.dockerignore   text eol=lf
+
+# Files using LFS to track
+*.tgz   filter=lfs diff=lfs merge=lfs -text
+*.h5    filter=lfs diff=lfs merge=lfs -text
+*.jsonl filter=lfs diff=lfs merge=lfs -text
+*.xlsx  filter=lfs diff=lfs merge=lfs -text
+*.bin   filter=lfs diff=lfs merge=lfs -text
+*.png   filter=lfs diff=lfs merge=lfs -text
+*.jpg   filter=lfs diff=lfs merge=lfs -text
+*.jpeg  filter=lfs diff=lfs merge=lfs -text
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
@@ -0,0 +1,34 @@
+name: qpod-data-lab
+
+on:
+  push:
+    branches: [ main ]
+    paths-ignore:
+      - "*.md"
+
+  pull_request:
    branches: [ main ]
+    paths-ignore:
+      - "*.md"
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+env:
+  DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }}
+  DOCKER_REGISTRY_PASSWORD: ${{ secrets.DOCKER_REGISTRY_PASSWORD }}
+
+jobs:
+  qpod_pyflink:
+    name: qpod/pyflink
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - run: source ./tool.sh && build_image pyflink latest         docker_pyflink/Dockerfile       && push_image
+
+  qpod_elasticsearch:
+    name: qpod/elasticsearch
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - run: source ./tool.sh && build_image elasticsearch latest   docker_elasticsearch/Dockerfile && push_image
diff --git a/.gitignore b/.gitignore
@@ -0,0 +1,67 @@
+# Byte-compiled / optimized / DLL files
+
+*~
+
+__pycache__/
+*.py[cod]
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*,cover
+
+# IDE
+.vscode/
+.idea/
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Mac OS X
+.DS_Store
+
+dockerspawner
+dockerspawner.tar.gz
+*.orig
+.ipynb_checkpoints/
+.vscode/
+.pytest_cache/
diff --git a/docker_elasticsearch/Dockerfile b/docker_elasticsearch/Dockerfile
@@ -0,0 +1,75 @@
+ARG BASE_NAMESPACE
+ARG BASE_IMG="base"
+FROM ${BASE_NAMESPACE:+$BASE_NAMESPACE/}${BASE_IMG} as builder
+
+ARG ES_VERSION="7.11.2"
+
+
+RUN source /opt/utils/script-utils.sh \
+ && ES_SOURCECODE="https://github.com/elastic/elasticsearch/archive/v${ES_VERSION}.tar.gz" \
+ && ES_ARTIFACT="https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-${ES_VERSION}-linux-x86_64.tar.gz" \
+ && install_tar_gz ${ES_SOURCECODE} && mv /opt/elasticsearch-* /tmp/elasticsearch \
+ && install_tar_gz ${ES_ARTIFACT}   && mv /opt/elasticsearch-* /opt/elasticsearch \
+ && mv /opt/elasticsearch/jdk /opt/ \
+ && ln -s /opt/jdk /opt/elasticsearch/ \
+ && ln -s /opt/jdk/bin/* /usr/bin/ \
+ && echo "@ Version of Java (java/javac):" && java -version && javac -version \
+ && sed -i -e 's/ES_DISTRIBUTION_TYPE=tar/ES_DISTRIBUTION_TYPE=docker/' /opt/elasticsearch/bin/elasticsearch-env \
+ && mkdir -pv config/jvm.options.d
+
+RUN set -ex \
+ && mkdir -pv /tmp/build/src    && cd /tmp/build \
+ && ln -s  /opt/elasticsearch/lib     /tmp/build/ \
+ && ln -s  /opt/elasticsearch/modules /tmp/build/ \
+ && find   /tmp/elasticsearch -name "License.java" | xargs -r -I {} cp {} . \
+ && sed -i 's#this.type = type;#this.type = "platinum";#g' License.java \
+ && sed -i 's#validate();#// validate();#g' License.java \
+ && javac -cp "`ls lib/elasticsearch-${ES_VERSION}.jar`:`ls lib/elasticsearch-x-content-*.jar`:`ls lib/lucene-core-*.jar`:`ls modules/x-pack-core/x-pack-core-*.jar`" License.java \
+ && cd /tmp/build/src \
+ && find /opt/elasticsearch/ -name "x-pack-core-*.jar" | xargs -r -I {} cp {} . \
+ && jar xf x-pack-core-${ES_VERSION}.jar \
+ && cp -f ../License*.class org/elasticsearch/license/ \
+ && jar cf x-pack-core-${ES_VERSION}.jar . \
+ && cp -r /tmp/build/src/x-pack-core-*.jar /opt/elasticsearch/modules/x-pack-core/ \
+ && rm -rf /tmp/*
+
+COPY --chown=1000:0                 docker-entrypoint.sh    /opt/elasticsearch/docker-entrypoint.sh
+COPY --chown=1000:0                 elasticsearch.yml       /opt/elasticsearch/config/
+COPY --chown=1000:0                 log4j2.properties       /opt/elasticsearch/config/
+
+# Second build stage
+FROM ${BASE_NAMESPACE:+$BASE_NAMESPACE/}${BASE_IMG}
+
+LABEL maintainer="haobibo@gmail.com"
+
+COPY --from=builder                 /opt/jdk                /opt/jdk
+COPY --from=builder --chown=1000:0  /opt/elasticsearch      /opt/elasticsearch
+
+RUN source /opt/utils/script-setup.sh \
+ && setup_tini \
+ && ln -s /opt/jdk/bin/*     /usr/bin/ \
+ && ln -s /opt/elasticsearch /usr/share/ \
+ && groupadd --gid 1000 elasticsearch \
+ && adduser --system -q --uid 1000 --gid 1000 --gid 0 --home /opt/elasticsearch elasticsearch \
+ && chmod -R 0775 /opt/elasticsearch    && chgrp 0 /opt/elasticsearch \
+ && chmod g=u /etc/passwd \
+ && find / -xdev -perm -4000 -exec chmod ug-s {} + \
+ && mkdir -p /data/elasticsearch/ && cd /data/elasticsearch && mkdir data logs \
+ && cd /opt/elasticsearch && rm -rf ./data ./logs ./plugins && ln -s /data/elasticsearch/* ./ \
+ && echo "elastic" | bin/elasticsearch-keystore add -xf bootstrap.password \
+ && echo "" | bin/elasticsearch-certutil ca   -s --out elastic-stack-ca.p12 --pass \
+ && echo "" | bin/elasticsearch-certutil cert -s --ca elastic-stack-ca.p12 --ca-pass "" --out elastic-certificates.p12 --pass \
+ && mv elastic-*.p12 config/ && chown 1000:0 config/elastic-*.p12 config/elasticsearch.keystore \
+ && echo "vm.swappiness=0"          >> /etc/sysctl.conf \
+ && echo "vm.max_map_count=655360 " >> /etc/sysctl.conf \
+ && echo "* soft memlock unlimited" >> /etc/security/limits.conf \
+ && echo "* hard memlock unlimited" >> /etc/security/limits.conf
+
+USER elasticsearch
+EXPOSE 9200 9300
+WORKDIR /opt/elasticsearch
+VOLUME /data/elasticsearch
+
+ENTRYPOINT ["tini", "--", "/opt/elasticsearch/docker-entrypoint.sh"]
+
+CMD ["eswrapper"]
diff --git a/docker_elasticsearch/docker-entrypoint.sh b/docker_elasticsearch/docker-entrypoint.sh
@@ -0,0 +1,83 @@
+#!/bin/bash
+set -e
+
+# Files created by Elasticsearch should always be group writable too
+umask 0002
+
+run_as_other_user_if_needed() {
+  if [[ "$(id -u)" == "0" ]]; then
+    # If running as root, drop to specified UID and run command
+    exec chroot --userspec=1000 / "${@}"
+  else
+    # Either we are running in Openshift with random uid and are a member of the root group
+    # or with a custom --user
+    exec "${@}"
+  fi
+}
+
+# Allow user specify custom CMD, maybe bin/elasticsearch itself
+# for example to directly specify `-E` style parameters for elasticsearch on k8s
+# or simply to run /bin/bash to check the image
+if [[ "$1" != "eswrapper" ]]; then
+  if [[ "$(id -u)" == "0" && $(basename "$1") == "elasticsearch" ]]; then
+    # centos:7 chroot doesn't have the `--skip-chdir` option and
+    # changes our CWD.
+    # Rewrite CMD args to replace $1 with `elasticsearch` explicitly,
+    # so that we are backwards compatible with the docs
+    # from the previous Elasticsearch versions<6
+    # and configuration option D:
+    # https://www.elastic.co/guide/en/elasticsearch/reference/5.6/docker.html#_d_override_the_image_8217_s_default_ulink_url_https_docs_docker_com_engine_reference_run_cmd_default_command_or_options_cmd_ulink
+    # Without this, user could specify `elasticsearch -E x.y=z` but
+    # `bin/elasticsearch -E x.y=z` would not work.
+    set -- "elasticsearch" "${@:2}"
+    # Use chroot to switch to UID 1000
+    exec chroot --userspec=1000 / "$@"
+  else
+    # User probably wants to run something else, like /bin/bash, with another uid forced (Openshift?)
+    exec "$@"
+  fi
+fi
+
+# Allow environment variables to be set by creating a file with the
+# contents, and setting an environment variable with the suffix _FILE to
+# point to it. This can be used to provide secrets to a container, without
+# the values being specified explicitly when running the container.
+#
+# This is also sourced in elasticsearch-env, and is only needed here
+# as well because we use ELASTIC_PASSWORD below. Sourcing this script
+# is idempotent.
+source /usr/share/elasticsearch/bin/elasticsearch-env-from-file
+
+if [[ -f bin/elasticsearch-users ]]; then
+  # Check for the ELASTIC_PASSWORD environment variable to set the
+  # bootstrap password for Security.
+  #
+  # This is only required for the first node in a cluster with Security
+  # enabled, but we have no way of knowing which node we are yet. We'll just
+  # honor the variable if it's present.
+      if [[ -n "$ELASTIC_PASSWORD" ]]; then
+    [[ -f /usr/share/elasticsearch/config/elasticsearch.keystore ]] || (run_as_other_user_if_needed elasticsearch-keystore create)
+    if ! (run_as_other_user_if_needed elasticsearch-keystore has-passwd --silent) ; then
+      # keystore is unencrypted
+      if ! (run_as_other_user_if_needed elasticsearch-keystore list | grep -q '^bootstrap.password$'); then
+        (run_as_other_user_if_needed echo "$ELASTIC_PASSWORD" | elasticsearch-keystore add -x 'bootstrap.password')
+      fi
+    else
+      # keystore requires password
+      if ! (run_as_other_user_if_needed echo "$KEYSTORE_PASSWORD" \
+          | elasticsearch-keystore list | grep -q '^bootstrap.password$') ; then
+        COMMANDS="$(printf "%s\n%s" "$KEYSTORE_PASSWORD" "$ELASTIC_PASSWORD")"
+        (run_as_other_user_if_needed echo "$COMMANDS" | elasticsearch-keystore add -x 'bootstrap.password')
+      fi
+    fi
+  fi
+fi
+
+if [[ "$(id -u)" == "0" ]]; then
+  # If requested and running as root, mutate the ownership of bind-mounts
+  if [[ -n "$TAKE_FILE_OWNERSHIP" ]]; then
+    chown -R 1000:0 /usr/share/elasticsearch/{data,logs}
+  fi
+fi
+
+run_as_other_user_if_needed /usr/share/elasticsearch/bin/elasticsearch <<<"$KEYSTORE_PASSWORD"
diff --git a/docker_elasticsearch/elasticsearch.yml b/docker_elasticsearch/elasticsearch.yml
@@ -0,0 +1,30 @@
+cluster.name: "es-cluster-docker"
+network.host: 0.0.0.0
+bootstrap.memory_lock: false
+path.data: /data/elasticsearch/data
+path.logs: /data/elasticsearch/logs
+node.name: ${HOSTNAME}
+discovery.seed_hosts: ["127.0.0.1", "[::1]"]
+cluster.initial_master_nodes: ["${HOSTNAME}"]
+
+http.cors.enabled: true
+http.cors.allow-origin: "*"
+# http.cors.allow-headers: Authorization
+
+xpack.security.enabled: true
+
+xpack.security.transport.ssl.enabled: true
+
+# To use certificate, run the command below and enable the settings
+# bin/elasticsearch-certutil ca -s && bin/elasticsearch-certutil cert -s --ca elastic-stack-ca.p12
+# mv elastic-*.p12 config/
+xpack.security.transport.ssl.verification_mode: certificate
+xpack.security.transport.ssl.keystore.path: elastic-certificates.p12
+xpack.security.transport.ssl.truststore.path: elastic-certificates.p12
+
+# xpack.security.http.ssl.enabled: true
+# xpack.security.http.ssl.keystore.path: elastic-certificates.p12
+# xpack.security.http.ssl.truststore.path: elastic-certificates.p12
+# xpack.security.http.ssl.client_authentication: none
+# xpack.ssl.verification_mode: none
+