diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 8dff54a..9f500a3 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -49,7 +49,7 @@ jobs: --build-arg "ARG_PROFILE_NODEJS=base" \ --build-arg "ARG_PROFILE_JAVA=base,maven" \ --build-arg "ARG_PROFILE_LATEX=base,cjk" - push_image + push_image core qpod_py-data: name: qpod/py-data @@ -318,7 +318,22 @@ jobs: --build-arg "ARG_PROFILE_NODEJS=base" \ --build-arg "ARG_PROFILE_JAVA=base,maven" \ --build-arg "ARG_PROFILE_LATEX=base,cjk" - alias_image full-cuda-11.8 latest core-cuda latest && push_image + alias_image full-cuda-11.8 latest core-cuda latest && push_image cuda + + + qpod_base-dev: + name: qpod/base-dev + needs: qpod_py-data + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - run: | + source ./tool.sh && free_diskspace + build_image base-dev latest docker_dev/Dockerfile \ + --build-arg "BASE_IMG=py-data" \ + --build-arg "ARG_PROFILE_JUPYTER=base,kernels,extensions" \ + --build-arg "ARG_PROFILE_VSCODE=base" + alias_image base-dev latest developer latest && push_image dev qpod_core-dev: name: qpod/core-dev @@ -327,8 +342,9 @@ jobs: steps: - uses: actions/checkout@v4 - run: | - source ./tool.sh + source ./tool.sh && free_diskspace build_image core-dev latest docker_dev/Dockerfile \ + --build-arg "BASE_IMG=core" \ --build-arg "ARG_PROFILE_JUPYTER=base,kernels,extensions" \ --build-arg "ARG_PROFILE_VSCODE=base" alias_image core-dev latest full latest && push_image diff --git a/docker_atom/Dockerfile b/docker_atom/Dockerfile index dc27afe..bbb3778 100644 --- a/docker_atom/Dockerfile +++ b/docker_atom/Dockerfile @@ -19,10 +19,14 @@ ENV SHELL=/bin/bash \ LANGUAGE="en_US.UTF-8" \ HOME_DIR=/root +# '-c' option make bash commands are read from string. +# If there are arguments after the string, they are assigned to the positional parameters, starting with $0. +# '--login': make bash first reads and executes commands from the file /etc/profile, if that file exists. +# After that, it looks for ~/.bash_profile, ~/.bash_login, and ~/.profile, in that order, and reads and executes commands from the first one that exists and is readable. SHELL ["/bin/bash", "--login", "-c"] # --> Install OS libraries and setup some configurations -RUN cd /tmp \ +RUN set -x && cd /tmp \ && apt-get -qq update --fix-missing && apt-get -y -qq upgrade \ && apt-get -qq install -y --no-install-recommends \ apt-utils apt-transport-https ca-certificates gnupg2 dirmngr locales sudo lsb-release curl \ diff --git a/docker_atom/work/script-setup-db-clients.sh b/docker_atom/work/script-setup-db-clients.sh index 889cdf3..21458e0 100644 --- a/docker_atom/work/script-setup-db-clients.sh +++ b/docker_atom/work/script-setup-db-clients.sh @@ -5,7 +5,7 @@ setup_postgresql_client() { local VER_PG=${VERSION_PG:-"14"} # from: https://www.postgresql.org/download/linux/ubuntu/ echo "deb https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" | sudo tee /etc/apt/sources.list.d/pgdg.list - curl "https://www.postgresql.org/media/keys/ACCC4CF8.asc" | sudo apt-key add - + curl "https://www.postgresql.org/media/keys/ACCC4CF8.asc" | sudo sudo tee /etc/apt/trusted.gpg.d/postgresql.asc sudo apt-get update # will download ~9MB files and use ~55MB disk after installation sudo apt-get -y install "postgresql-client-${VER_PG}" @@ -24,7 +24,7 @@ setup_mysql_client() { setup_mongosh_client() { # from: https://www.mongodb.com/docs/mongodb-shell/install/ echo "deb [ arch=amd64,arm64 ] https://repo.mongodb.org/apt/ubuntu $(lsb_release -cs)/mongodb-org/6.0 multiverse" > /etc/apt/sources.list.d/mongodb-org-6.0.list - curl -sL https://www.mongodb.org/static/pgp/server-6.0.asc | sudo apt-key add - + curl -sL https://www.mongodb.org/static/pgp/server-6.0.asc | sudo sudo tee /etc/apt/trusted.gpg.d/mongodb.asc sudo apt-get update # will download ~38MB files and use ~218MB disk after installation sudo apt-get -y install mongodb-mongosh diff --git a/docker_atom/work/script-setup.sh b/docker_atom/work/script-setup.sh index 696aea7..c14d693 100644 --- a/docker_atom/work/script-setup.sh +++ b/docker_atom/work/script-setup.sh @@ -141,15 +141,15 @@ setup_node() { && mv /opt/node* /opt/node \ && echo "PATH=/opt/node/bin:$PATH" >> /etc/bash.bashrc \ && export PATH=/opt/node/bin:$PATH \ + && corepack enable \ && npm install -g npm yarn \ - && ln -sf /opt/node/bin/* /usr/bin/ \ && echo "@ Version of Node, npm, and yarn: $(node -v) $(npm -v)" \ && echo "@ Version of Yarn: $(yarn -v)" } setup_R_base() { - apt-key adv --keyserver keyserver.ubuntu.com --recv-keys E298A3A825C0D65DFD57CBB651716619E084DAB9 \ + curl -sL https://cloud.r-project.org/bin/linux/ubuntu/marutter_pubkey.asc | sudo tee -a /etc/apt/trusted.gpg.d/cran_ubuntu_key.asc \ && echo "deb https://cloud.r-project.org/bin/linux/ubuntu $(lsb_release -cs)-cran40/" > /etc/apt/sources.list.d/cran.list \ && install_apt /opt/utils/install_list_R_base.apt \ && echo "options(repos=structure(c(CRAN=\"https://cloud.r-project.org\")))" >> /etc/R/Rprofile.site \ diff --git a/docker_atom/work/script-utils.sh b/docker_atom/work/script-utils.sh index a4bfe92..e621484 100644 --- a/docker_atom/work/script-utils.sh +++ b/docker_atom/work/script-utils.sh @@ -1,4 +1,4 @@ -set -x +# shell util functions # function to debug, resolve package names from a text file and display. install_echo() { cat $1 | cut -d "%" -f 1 | sed '/^$/d' | xargs -r -n1 printf '%s\n' ; } @@ -11,7 +11,7 @@ install_conda() { cat $1 | cut -d "%" -f 1 | sed '/^$/d' | xargs -r -n1 conda install_mamba() { cat $1 | cut -d "%" -f 1 | sed '/^$/d' | xargs -r -n1 mamba install -yq --root-prefix="${CONDA_PREFIX}" --prefix="${CONDA_PREFIX}" ; } # function to install python packages with pip from a text file which lists package names (add comments with % char) -install_pip() { cat $1 | cut -d "%" -f 1 | sed '/^$/d' | xargs -r -n1 pip install -U --root-user-action=ignore --pre ; } +install_pip() { cat $1 | cut -d "%" -f 1 | sed '/^$/d' | xargs -r -n1 pip install --no-cache-dir --root-user-action=ignore -U --pre ; } # function to install R packages from a text file which lists package names (add comments with % char, use quiet=T to be less verbose) install_R() { R -e "options(Ncpus=4);lapply(scan('$1','c',comment.char='%'),function(x){cat(x,system.time(install.packages(x,clean=T,quiet=T)),'\n')})"; } diff --git a/docker_base/Dockerfile b/docker_base/Dockerfile index a21692d..9343002 100644 --- a/docker_base/Dockerfile +++ b/docker_base/Dockerfile @@ -10,9 +10,9 @@ ARG PYTHON_VERSION="3.11" ENV CONDA_PREFIX=/opt/conda ENV PATH=$PATH:${CONDA_PREFIX}/bin -RUN source /opt/utils/script-setup.sh \ +RUN set -x && source /opt/utils/script-setup.sh \ # update source for CMake - && curl -sL https://apt.kitware.com/keys/kitware-archive-latest.asc | sudo apt-key add - \ + && curl -sL https://apt.kitware.com/keys/kitware-archive-latest.asc | sudo sudo tee /etc/apt/trusted.gpg.d/kitware.asc \ && echo "deb https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" > /etc/apt/sources.list.d/kitware.list \ && install_apt /opt/utils/install_list_base.apt \ && echo "Install tini:" && setup_tini \ diff --git a/docker_cuda/Dockerfile b/docker_cuda/Dockerfile index 6001520..2bb2003 100644 --- a/docker_cuda/Dockerfile +++ b/docker_cuda/Dockerfile @@ -11,6 +11,6 @@ ENV NVIDIA_DISABLE_REQUIRE=1 # For cuda version 10.0, the image is solely serverd for legacy tensorflow 1.15, which requires python 3.7 # For tensorflow 2.x or torch, python>=3.9 is supported. -RUN echo ${CUDA_VERSION} && nvcc --version \ +RUN set -x && echo ${CUDA_VERSION} && nvcc --version \ && source /opt/utils/script-setup.sh && setup_nvtop \ && install__clean diff --git a/docker_dev/Dockerfile b/docker_dev/Dockerfile index 676533d..d32657b 100644 --- a/docker_dev/Dockerfile +++ b/docker_dev/Dockerfile @@ -18,7 +18,7 @@ COPY work /opt/utils/ # Setup Jupyter: Basic Configurations and Extensions... RUN mkdir -pv /opt/conda/etc/jupyter/ \ - && mv /opt/utils/jupyter_notebook_config.json /opt/conda/etc/jupyter/ \ + && mv /opt/utils/etc_jupyter/* /opt/conda/etc/jupyter/ && rm -rf /opt/utils/etc_jupyter \ && mv /opt/utils/start-*.sh /usr/local/bin/ && chmod +x /usr/local/bin/start-*.sh \ && source /opt/utils/script-extend.sh \ && for profile in $(echo $ARG_PROFILE_JUPYTER | tr "," "\n") ; do ( setup_jupyter_${profile} || true ) ; done @@ -31,10 +31,18 @@ RUN source /opt/utils/script-extend.sh \ && for profile in $(echo $ARG_PROFILE_VSCODE | tr "," "\n") ; do ( setup_vscode_${profile} || true ) ; done # Clean up and display components version information... -RUN source /opt/utils/script-utils.sh && install__clean +RUN source /opt/utils/script-utils.sh && install__clean && list_installed_packages WORKDIR $HOME_DIR EXPOSE 8888 ENTRYPOINT ["tini", "-g", "--"] CMD ["start-notebook.sh"] + +# '-c' option make bash commands are read from string. +# If there are arguments after the string, they are assigned to the positional parameters, starting with $0. +# '-o pipefail' prevents errors in a pipeline from being masked. +# If any command in a pipeline fails, that return code will be used as the return code of the whole pipeline. +# '--login': make bash first reads and executes commands from the file /etc/profile, if that file exists. +# After that, it looks for ~/.bash_profile, ~/.bash_login, and ~/.profile, in that order, and reads and executes commands from the first one that exists and is readable. +SHELL ["/bin/bash", "--login", "-o", "pipefail", "-c"] diff --git a/docker_dev/work/jupyter_notebook_config.json b/docker_dev/work/etc_jupyter/jupyter_server_config.json similarity index 100% rename from docker_dev/work/jupyter_notebook_config.json rename to docker_dev/work/etc_jupyter/jupyter_server_config.json diff --git a/docker_dev/work/install_list_JPY_extend.pip b/docker_dev/work/install_list_JPY_extend.pip index a58d911..9c49303 100644 --- a/docker_dev/work/install_list_JPY_extend.pip +++ b/docker_dev/work/install_list_JPY_extend.pip @@ -3,11 +3,9 @@ ipyparallel % https://github.com/ipython/ipyparallel ipynb % https://github.com/ipython/ipynb + jupyter-resource-usage % https://github.com/jupyter-server/jupyter-resource-usage jupyter-collaboration % https://github.com/jupyterlab/jupyter-collaboration +jupyterlab_server[openapi] % https://github.com/jupyterlab/jupyterlab_server jupyterlab-git % https://github.com/jupyterlab/jupyterlab-git -jupyterlab-latex % https://github.com/jupyterlab/jupyterlab-latex -jupyterlab_server % https://github.com/jupyterlab/jupyterlab_server -# jupyter_nbextensions_configurator % Jupyter Notebook extension configurator -# jupyter_contrib_nbextensions % Jupyter Notebook third-party extensions -# qpod_hub % https://github.com/QPod/qpod-hub +% jupyterlab-latex % bug on pypi version: https://github.com/jupyterlab/jupyterlab-latex diff --git a/docker_dev/work/script-extend.sh b/docker_dev/work/script-extend.sh index 2035cea..8e45bc0 100644 --- a/docker_dev/work/script-extend.sh +++ b/docker_dev/work/script-extend.sh @@ -1,9 +1,11 @@ source /opt/utils/script-utils.sh setup_jupyter_base() { - pip install -Uq --pre jupyterhub jupyterlab notebook ipywidgets \ - && echo "@ Version of Jupyter Notebook/JupyterLab: $(jupyter notebook --version)" \ - && echo "@ Version of Jupyter Notebook/JupyterLab: $(jupyter lab --version)" + pip install -Uq --pre jupyterhub jupyterlab notebook nbclassic ipywidgets \ + && echo "@ Version of Jupyter Server: $(jupyter server --version)" \ + && echo "@ Version of Jupyter Lab: $(jupyter lab --version)" \ + && echo "@ Version of Jupyter Notebook: $(jupyter notebook --version)" \ + && echo "@ Version of JupyterHub: $(jupyterhub --version)" } @@ -49,23 +51,9 @@ setup_jupyter_extensions() { install_apt /opt/utils/install_list_JPY_extend.apt \ && install_pip /opt/utils/install_list_JPY_extend.pip - # jupyter nbextensions_configurator enable --sys-prefix - # jupyter contrib nbextension install --sys-prefix - # jupyter serverextension enable --sys-prefix --py jupyterlab_git - - # && jupyter nbextension enable --py widgetsnbextension \ - # && jupyter labextension install @jupyter-widgets/jupyterlab-manager \ - - # jupyter labextension install --no-build \ - # @jupyterlab/toc @jupyterlab/shortcutui @jupyterlab/git @jupyterlab/latex \ - # @jupyterlab/mathjax3-extension @jupyterlab/fasta-extension @jupyterlab/geojson-extension \ - # @jupyterlab/vega3-extension @jupyterlab/commenting-extension - # TEMP fix: (not compatible with JupyterLab 2.0) @jupyterlab/metadata-extension @jupyterlab/dataregistry-extension - - jupyter lab build && echo "@ Jupyter Extension list:" \ - && jupyter nbextension list \ - && jupyter serverextension list \ - && jupyter labextension list + echo "@ Jupyter Server Extension list: " && jupyter server extension list \ + && echo "@ Jupyter Lab Extension list: " && jupyter labextension list \ + && echo "@ Jupyter Notebook Extension list: " && jupyter notebook extension list } diff --git a/tool.sh b/tool.sh index 10ff96f..d601607 100755 --- a/tool.sh +++ b/tool.sh @@ -19,12 +19,11 @@ echo "--------> CI_PROJECT_NAMESPACE=${CI_PROJECT_NAMESPACE}" echo "--------> DOCKER_REGISTRY_NAMESPACE=${NAMESPACE}" if [ -f /etc/docker/daemon.json ]; then - jq '.experimental=true' /etc/docker/daemon.json > /tmp/daemon.json && sudo mv /tmp/daemon.json /etc/docker/ \ + jq '.experimental=true | ."data-root"="/mnt/docker"' /etc/docker/daemon.json > /tmp/daemon.json && sudo mv /tmp/daemon.json /etc/docker/ \ && ( sudo service docker restart || true ) -else - docker info fi - +cat /etc/docker/daemon.json +docker info build_image() { echo "$@" ;