From f2e30a3340c906ff0520c42f27273bd64f705915 Mon Sep 17 00:00:00 2001 From: yyuuttaaoo Date: Fri, 8 Jul 2022 12:10:43 +0800 Subject: [PATCH 1/2] add script to build ilogtail production image --- .gitignore | 23 ++--- Makefile | 4 +- core/app_config/AppConfigBase.cpp | 2 +- core/logtail.cpp | 14 +++ core/logtail_windows.cpp | 12 +++ ...gtail_part => Dockerfile_development_part} | 0 docker/Dockerfile_production_part | 42 +++++++++ .../quick_start/ilogtail_config.json | 4 +- .../start_with_container/ilogtail_config.json | 2 + .../user_yaml_config.d/file_simple.yaml | 37 ++++++++ .../user_yaml_config.d/stdout_simple.yaml | 38 ++++++++ scripts/docker_build.sh | 12 ++- scripts/gen_build_scripts.sh | 2 +- scripts/plugin_build.sh | 2 +- scripts/run_ilogtail.sh | 90 +++++++++++++++++++ 15 files changed, 261 insertions(+), 23 deletions(-) rename docker/{Dockerfile_ilogtail_part => Dockerfile_development_part} (100%) create mode 100644 docker/Dockerfile_production_part create mode 100644 example_config/start_with_container/ilogtail_config.json create mode 100644 example_config/start_with_container/user_yaml_config.d/file_simple.yaml create mode 100644 example_config/start_with_container/user_yaml_config.d/stdout_simple.yaml create mode 100755 scripts/run_ilogtail.sh diff --git a/.gitignore b/.gitignore index ed7f85c81b..153b9c8271 100644 --- a/.gitignore +++ b/.gitignore @@ -8,10 +8,10 @@ *.bak # go -plugin_main/plugin_main -pluginmanager/checkpoint/ +/plugin_main/plugin_main +/pluginmanager/checkpoint/ *.prof -plugin_main/checkpoint/* +/plugin_main/checkpoint/* *.LOG *.LOG.* *.test @@ -25,10 +25,10 @@ CMakeCache.txt Makefile *.cbp *.a -core/log_pb/*.pb.* -core/build/ -core/common/Version.cpp -core/installer/public/ +/core/log_pb/*.pb.* +/core/build/ +/core/common/Version.cpp +/core/installer/public/ !/Makefile !core/dependencies.cmake !core/utils.cmakels @@ -36,10 +36,11 @@ core/installer/public/ # output e2e-test -bin/ -build/ -find_licenses/ +/bin/ +/generated_files/ +/build/ +/find_licenses/ # vscode .vscode -!.vscode/settings.json \ No newline at end of file +!.vscode/settings.json diff --git a/Makefile b/Makefile index d089d6afa3..a234db7742 100644 --- a/Makefile +++ b/Makefile @@ -102,12 +102,12 @@ plugin_main: clean .PHONY: docker docker: clean ./scripts/gen_build_scripts.sh all $(GENERATED_HOME) $(VERSION) $(DOCKER_REPOSITORY) - ./scripts/docker_build.sh default $(GENERATED_HOME) $(VERSION) $(DOCKER_REPOSITORY) $(DOCKER_PUSH) + ./scripts/docker_build.sh production $(GENERATED_HOME) $(VERSION) $(DOCKER_REPOSITORY) $(DOCKER_PUSH) .PHONY: e2edocker e2edocker: clean ./scripts/gen_build_scripts.sh e2e $(GENERATED_HOME) $(VERSION) $(DOCKER_REPOSITORY) - ./scripts/docker_build.sh default $(GENERATED_HOME) $(VERSION) $(DOCKER_REPOSITORY) false + ./scripts/docker_build.sh development $(GENERATED_HOME) $(VERSION) $(DOCKER_REPOSITORY) false # provide a goc server for e2e testing .PHONY: gocdocker diff --git a/core/app_config/AppConfigBase.cpp b/core/app_config/AppConfigBase.cpp index fdda6db7e2..9f8e9f767b 100644 --- a/core/app_config/AppConfigBase.cpp +++ b/core/app_config/AppConfigBase.cpp @@ -361,7 +361,7 @@ void LoadEnvValueIfExisting(const char* envKey, T& cfgValue) { void AppConfigBase::LoadEnvResourceLimit() { LoadSingleValueEnvConfig("cpu_usage_limit", mCpuUsageUpLimit, (float)0.4); - LoadSingleValueEnvConfig("mem_usage_limit", mMemUsageUpLimit, (int64_t)100); + LoadSingleValueEnvConfig("mem_usage_limit", mMemUsageUpLimit, (int64_t)384); LoadSingleValueEnvConfig("max_bytes_per_sec", mMaxBytePerSec, (int32_t)(1024 * 1024)); LoadSingleValueEnvConfig("process_thread_count", mProcessThreadCount, (int32_t)1); LoadSingleValueEnvConfig("send_request_concurrency", mSendRequestConcurrency, (int32_t)2); diff --git a/core/logtail.cpp b/core/logtail.cpp index c11521a0ac..fd9ec9afc9 100644 --- a/core/logtail.cpp +++ b/core/logtail.cpp @@ -62,6 +62,10 @@ DEFINE_FLAG_INT32(fork_interval, "fork dispatcher process interval", 10); DECLARE_FLAG_INT32(max_open_files_limit); DECLARE_FLAG_INT32(max_reader_open_files); DECLARE_FLAG_STRING(ilogtail_config_env_name); +DECLARE_FLAG_STRING(logtail_sys_conf_dir); +DECLARE_FLAG_STRING(check_point_filename); +DECLARE_FLAG_STRING(default_buffer_file_path); +DECLARE_FLAG_STRING(ilogtail_docker_file_path_config); void HandleSighupSignal(int signum, siginfo_t* info, void* context) { ConfigManager::GetInstance()->SetMappingPathsChanged(); @@ -90,6 +94,14 @@ void enable_core(void) { } } +static void overwrite_community_edition_flags() { + // support run in installation dir on default + STRING_FLAG(logtail_sys_conf_dir) = "."; + STRING_FLAG(check_point_filename) = "checkpoint/logtail_check_point"; + STRING_FLAG(default_buffer_file_path) = "checkpoint"; + STRING_FLAG(ilogtail_docker_file_path_config) = "checkpoint/docker_path_config.json"; +} + // Main routine of worker process. void do_worker_process() { Logger::Instance().InitGlobalLoggers(); @@ -144,6 +156,8 @@ void do_worker_process() { AppConfig::GetInstance()->SetWorkingDir(GetProcessExecutionDir()); } + overwrite_community_edition_flags(); + char* configEnv = getenv(STRING_FLAG(ilogtail_config_env_name).c_str()); if (configEnv == NULL || strlen(configEnv) == 0) { AppConfig::GetInstance()->LoadAppConfig(STRING_FLAG(ilogtail_config)); diff --git a/core/logtail_windows.cpp b/core/logtail_windows.cpp index 687411cfef..b24d731b76 100644 --- a/core/logtail_windows.cpp +++ b/core/logtail_windows.cpp @@ -47,6 +47,18 @@ using namespace logtail; DEFINE_FLAG_STRING(ilogtail_daemon_startup_hints, "hints passed from daemon during startup", ""); DECLARE_FLAG_STRING(ilogtail_config_env_name); +DECLARE_FLAG_STRING(logtail_sys_conf_dir); +DECLARE_FLAG_STRING(check_point_filename); +DECLARE_FLAG_STRING(default_buffer_file_path); +DECLARE_FLAG_STRING(ilogtail_docker_file_path_config); + +static void overwrite_community_edition_flags() { + // support run in installation dir on default + STRING_FLAG(logtail_sys_conf_dir) = "."; + STRING_FLAG(check_point_filename) = "checkpoint/logtail_check_point"; + STRING_FLAG(default_buffer_file_path) = "checkpoint"; + STRING_FLAG(ilogtail_docker_file_path_config) = "checkpoint/docker_path_config.json"; +} void do_worker_process() { Logger::Instance().InitGlobalLoggers(); diff --git a/docker/Dockerfile_ilogtail_part b/docker/Dockerfile_development_part similarity index 100% rename from docker/Dockerfile_ilogtail_part rename to docker/Dockerfile_development_part diff --git a/docker/Dockerfile_production_part b/docker/Dockerfile_production_part new file mode 100644 index 0000000000..57f0466184 --- /dev/null +++ b/docker/Dockerfile_production_part @@ -0,0 +1,42 @@ +# Copyright 2021 iLogtail Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM centos:centos7.9.2009 +ENV container docker +RUN (cd /lib/systemd/system/sysinit.target.wants/; for i in *; do [ $i == \ +systemd-tmpfiles-setup.service ] || rm -f $i; done); \ +rm -f /lib/systemd/system/multi-user.target.wants/*;\ +rm -f /etc/systemd/system/*.wants/*;\ +rm -f /lib/systemd/system/local-fs.target.wants/*; \ +rm -f /lib/systemd/system/sockets.target.wants/*udev*; \ +rm -f /lib/systemd/system/sockets.target.wants/*initctl*; \ +rm -f /lib/systemd/system/basic.target.wants/*;\ +rm -f /lib/systemd/system/anaconda.target.wants/*; +VOLUME [ "/sys/fs/cgroup" ] + +ARG HOST_OS=Linux +ARG VERSION=1.1.0 + +WORKDIR /usr/local/ilogtail + +COPY --from=build /src/bin/libPluginBase.so /usr/local/ilogtail/ +COPY --from=build /src/example_config/start_with_container/ilogtail_config.json /usr/local/ilogtail/ +COPY --from=build /src/core/build/ilogtail /usr/local/ilogtail/ +COPY --from=build /src/core/build/plugin/libPluginAdapter.so /usr/local/ilogtail/ +COPY --from=build /src/scripts/run_ilogtail.sh /usr/local/ilogtail/ + +RUN chmod -R 755 /usr/local/ilogtail/ +RUN mkdir /usr/local/ilogtail/checkpoint + +ENTRYPOINT ["/usr/local/ilogtail/run_ilogtail.sh"] diff --git a/example_config/quick_start/ilogtail_config.json b/example_config/quick_start/ilogtail_config.json index 44b202a0dd..66f599c20e 100644 --- a/example_config/quick_start/ilogtail_config.json +++ b/example_config/quick_start/ilogtail_config.json @@ -1,6 +1,4 @@ { "default_access_key_id": "", - "default_access_key": "", - "logtail_sys_conf_dir": ".", - "check_point_filename": "checkpoint/logtail_check_point" + "default_access_key": "" } diff --git a/example_config/start_with_container/ilogtail_config.json b/example_config/start_with_container/ilogtail_config.json new file mode 100644 index 0000000000..7a73a41bfd --- /dev/null +++ b/example_config/start_with_container/ilogtail_config.json @@ -0,0 +1,2 @@ +{ +} \ No newline at end of file diff --git a/example_config/start_with_container/user_yaml_config.d/file_simple.yaml b/example_config/start_with_container/user_yaml_config.d/file_simple.yaml new file mode 100644 index 0000000000..ddb0876a82 --- /dev/null +++ b/example_config/start_with_container/user_yaml_config.d/file_simple.yaml @@ -0,0 +1,37 @@ +# Copyright 2022 iLogtail Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Copyright 2022 iLogtail Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +enable: true +inputs: + - Type: file_log + LogPath: /root + FilePattern: simple.log + DockerFile: true +flushers: + - Type: flusher_stdout + OnlyStdout: true \ No newline at end of file diff --git a/example_config/start_with_container/user_yaml_config.d/stdout_simple.yaml b/example_config/start_with_container/user_yaml_config.d/stdout_simple.yaml new file mode 100644 index 0000000000..853f74f0d9 --- /dev/null +++ b/example_config/start_with_container/user_yaml_config.d/stdout_simple.yaml @@ -0,0 +1,38 @@ +# Copyright 2022 iLogtail Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Copyright 2022 iLogtail Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +enable: true +inputs: + - Type: service_docker_stdout + BeginLineCheckLength: 10 + BeginLineRegex: ".*" + Stderr: false + Stdout: true +flushers: + - Type: flusher_stdout + FileName: simple.stdout \ No newline at end of file diff --git a/scripts/docker_build.sh b/scripts/docker_build.sh index 25a34d0204..91c601d1b8 100755 --- a/scripts/docker_build.sh +++ b/scripts/docker_build.sh @@ -15,11 +15,12 @@ # limitations under the License. -# Currently, there are 3 supported docker categories, which are goc, build, and default. +# Currently, there are 4 supported docker categories, which are goc, build, development and production. # # goc: build goc server with Dockerfile_doc # build: build core or plugin binary with Dockerfile_build -# default: build ilogtail images. +# development: build ilogtail development images. +# production: build ilogtail production images. CATEGORY=$1 GENERATED_HOME=$2 VERSION=${3:-1.1.0} @@ -36,9 +37,12 @@ touch $GEN_DOCKERFILE if [[ $CATEGORY = "goc" || $CATEGORY = "build" ]]; then cat $ROOTDIR/docker/Dockerfile_$CATEGORY|grep -v "#" > $GEN_DOCKERFILE; -elif [[ $CATEGORY = "default" ]]; then +elif [[ $CATEGORY = "development" ]]; then cat $ROOTDIR/docker/Dockerfile_build |grep -v "#" > $GEN_DOCKERFILE; - cat $ROOTDIR/docker/Dockerfile_ilogtail_part |grep -v "#">> $GEN_DOCKERFILE; + cat $ROOTDIR/docker/Dockerfile_development_part |grep -v "#">> $GEN_DOCKERFILE; +elif [[ $CATEGORY = "production" ]]; then + cat $ROOTDIR/docker/Dockerfile_build |grep -v "#" > $GEN_DOCKERFILE; + cat $ROOTDIR/docker/Dockerfile_production_part |grep -v "#">> $GEN_DOCKERFILE; fi echo "=============DOCKERFILE==================" diff --git a/scripts/gen_build_scripts.sh b/scripts/gen_build_scripts.sh index 39e8d96d52..fecacbbb29 100755 --- a/scripts/gen_build_scripts.sh +++ b/scripts/gen_build_scripts.sh @@ -52,7 +52,7 @@ function generateCopyScript() { echo 'docker cp "$id":/src/core/build/ilogtail $BINDIR' >> $COPY_SCRIPT_FILE; echo 'docker cp "$id":/src/core/build/plugin/libPluginAdapter.so $BINDIR' >> $COPY_SCRIPT_FILE; else - echo ' docker cp "$id":/src/bin/libPluginBase.so $BINDIR' >> $COPY_SCRIPT_FILE; + echo 'docker cp "$id":/src/bin/libPluginBase.so $BINDIR' >> $COPY_SCRIPT_FILE; echo 'docker cp "$id":/src/core/build/ilogtail $BINDIR' >> $COPY_SCRIPT_FILE; echo 'docker cp "$id":/src/core/build/plugin/libPluginAdapter.so $BINDIR' >> $COPY_SCRIPT_FILE; fi diff --git a/scripts/plugin_build.sh b/scripts/plugin_build.sh index 4540ec788e..2333e41850 100755 --- a/scripts/plugin_build.sh +++ b/scripts/plugin_build.sh @@ -34,7 +34,7 @@ os OS_FLAG=$? ROOTDIR=$(cd $(dirname "${BASH_SOURCE[0]}") && cd .. && pwd) -mkdir "$ROOTDIR"/bin +mkdir -p "$ROOTDIR"/bin if [ $OS_FLAG = 1 ]; then IDFLAGS='-extldflags "-Wl,--wrap=memcpy"' diff --git a/scripts/run_ilogtail.sh b/scripts/run_ilogtail.sh new file mode 100755 index 0000000000..d736ee06bf --- /dev/null +++ b/scripts/run_ilogtail.sh @@ -0,0 +1,90 @@ +#!/bin/bash +set -ue +set -o pipefail + +caller_dir="$PWD" +ilogtail_dir=$(cd $(dirname "${BASH_SOURCE[0]}") && pwd) +ilogtail_pid= +delaySec=0 +exit_flag=0 +exit_timeout=10 + +gen_config() { + : +} + +start_ilogtail() { + ./ilogtail & + ilogtail_pid=$! + echo "ilogtail started. pid: $ilogtail_pid" +} + +check_liveness() { + # check if process exists + [[ -d /proc/$ilogtail_pid ]] || { + echo "ilogtail process exited." + return 1 + } + pid_status=`head /proc/$ilogtail_pid/status | grep "State:*"` + # check if process is zombie + [[ "$pid_status" =~ .*"zombie"*. ]] && \ + echo "ilogtail process became zombie" && \ + return 2 || : + # ilogtail is healthy + return 0 +} + +stop_ilogtail () { + # just return if ilogtail has not started + [[ -z $ilogtail_pid ]] && echo stop ilogtail done || : + + echo 'delay stop ilogtail, sleep ' $delaySec + sleep $delaySec + echo stop ilogtail + # try to kill with SIGTERM, and wait for process to exit + kill $ilogtail_pid + exit_time=0 + exit_result="force killed" + while [[ $exit_time -lt $exit_timeout ]] + do + check_liveness || { + [[ $? -eq 1 ]] && exit_result="exited normally" || exit_result="became zombie" + break + } + sleep 1 + done + # force kill with SIGKILL if timed out + [[ $exit_time -ge $exit_timeout ]] && kill -9 ilogtail_pid || : + echo stop ilogtail done, result ${exit_result}. +} + +exit_handler() { + echo 'receive stop signal' + exit_flag=1 +} + + +# main procedure +if [ $# -gt 0 ];then + delaySec=$1 +fi +echo 'delay stop seconds: ' $delaySec + +trap 'exit_handler' SIGTERM +trap 'exit_handler' SIGINT + +gen_config + +start_ilogtail + +while [[ $exit_flag -eq 0 ]] +do + sleep 1 + check_liveness || { + echo "ilogtail exited unexpectedly" + exit 1 + } +done + +stop_ilogtail +cd "$caller_dir" From 607219984d319b2aaa091863a60c8d9fdfd8b523 Mon Sep 17 00:00:00 2001 From: yyuuttaaoo Date: Fri, 8 Jul 2022 13:11:52 +0800 Subject: [PATCH 2/2] add license to run_ilogtail.sh --- scripts/run_ilogtail.sh | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/scripts/run_ilogtail.sh b/scripts/run_ilogtail.sh index d736ee06bf..8318f2845b 100755 --- a/scripts/run_ilogtail.sh +++ b/scripts/run_ilogtail.sh @@ -1,4 +1,19 @@ -#!/bin/bash +#!/usr/bin/env bash + +# Copyright 2021 iLogtail Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + set -ue set -o pipefail