QPod · haobibo · Jan 5, 2024 · Jan 4, 2024 · Jan 4, 2024 · Jan 5, 2024
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
@@ -49,7 +49,7 @@ jobs:
             --build-arg "ARG_PROFILE_NODEJS=base" \
             --build-arg "ARG_PROFILE_JAVA=base,maven" \
             --build-arg "ARG_PROFILE_LATEX=base,cjk"
-          push_image
+          push_image core
 
   qpod_py-data:
     name: qpod/py-data
@@ -318,7 +318,22 @@ jobs:
             --build-arg "ARG_PROFILE_NODEJS=base" \
             --build-arg "ARG_PROFILE_JAVA=base,maven" \
             --build-arg "ARG_PROFILE_LATEX=base,cjk"
-          alias_image full-cuda-11.8 latest core-cuda latest && push_image
+          alias_image full-cuda-11.8 latest core-cuda latest && push_image cuda
+
+
+  qpod_base-dev:
+    name: qpod/base-dev
+    needs: qpod_py-data
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - run: |
+          source ./tool.sh && free_diskspace
+          build_image base-dev latest docker_dev/Dockerfile \
+            --build-arg "BASE_IMG=py-data" \
+            --build-arg "ARG_PROFILE_JUPYTER=base,kernels,extensions" \
+            --build-arg "ARG_PROFILE_VSCODE=base"
+            alias_image base-dev latest developer latest && push_image dev
 
   qpod_core-dev:
     name: qpod/core-dev
@@ -327,8 +342,9 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - run: |
-          source ./tool.sh
+          source ./tool.sh && free_diskspace
           build_image core-dev latest docker_dev/Dockerfile \
+            --build-arg "BASE_IMG=core" \
             --build-arg "ARG_PROFILE_JUPYTER=base,kernels,extensions" \
             --build-arg "ARG_PROFILE_VSCODE=base"
           alias_image core-dev latest full latest && push_image

diff --git a/docker_atom/Dockerfile b/docker_atom/Dockerfile
@@ -19,10 +19,14 @@ ENV SHELL=/bin/bash \
     LANGUAGE="en_US.UTF-8" \
     HOME_DIR=/root
 
+# '-c' option make bash commands are read from string.
+#   If there are arguments after the string, they are assigned to the positional parameters, starting with $0.
+# '--login': make bash first reads and executes commands from  the file /etc/profile, if that file exists.
+#   After that, it looks for ~/.bash_profile, ~/.bash_login, and ~/.profile, in that order, and reads and executes commands from the first one that exists and is readable.
 SHELL ["/bin/bash", "--login", "-c"]
 
 # --> Install OS libraries and setup some configurations
-RUN cd /tmp \
+RUN set -x && cd /tmp \
  && apt-get -qq update --fix-missing && apt-get -y -qq upgrade \
  && apt-get -qq install -y --no-install-recommends \
      apt-utils apt-transport-https ca-certificates gnupg2 dirmngr locales sudo lsb-release curl \

diff --git a/docker_atom/work/script-setup-db-clients.sh b/docker_atom/work/script-setup-db-clients.sh
@@ -5,7 +5,7 @@ setup_postgresql_client() {
   local VER_PG=${VERSION_PG:-"14"}
   # from: https://www.postgresql.org/download/linux/ubuntu/
   echo "deb https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" | sudo tee /etc/apt/sources.list.d/pgdg.list
-  curl "https://www.postgresql.org/media/keys/ACCC4CF8.asc" | sudo apt-key add -
+  curl "https://www.postgresql.org/media/keys/ACCC4CF8.asc" | sudo sudo tee /etc/apt/trusted.gpg.d/postgresql.asc
   sudo apt-get update
   # will download ~9MB files and use ~55MB disk after installation
   sudo apt-get -y install "postgresql-client-${VER_PG}"
@@ -24,7 +24,7 @@ setup_mysql_client() {
 setup_mongosh_client() {
   # from: https://www.mongodb.com/docs/mongodb-shell/install/
   echo "deb [ arch=amd64,arm64 ] https://repo.mongodb.org/apt/ubuntu $(lsb_release -cs)/mongodb-org/6.0 multiverse" > /etc/apt/sources.list.d/mongodb-org-6.0.list
-  curl -sL https://www.mongodb.org/static/pgp/server-6.0.asc | sudo apt-key add -
+  curl -sL https://www.mongodb.org/static/pgp/server-6.0.asc | sudo sudo tee /etc/apt/trusted.gpg.d/mongodb.asc
   sudo apt-get update
   # will download ~38MB files and use ~218MB disk after installation
   sudo apt-get -y install mongodb-mongosh

diff --git a/docker_atom/work/script-setup.sh b/docker_atom/work/script-setup.sh
@@ -141,15 +141,15 @@ setup_node() {
   && mv /opt/node* /opt/node \
   && echo  "PATH=/opt/node/bin:$PATH" >> /etc/bash.bashrc \
   && export PATH=/opt/node/bin:$PATH \
+  && corepack enable \
   && npm install -g npm yarn \
-  && ln -sf /opt/node/bin/* /usr/bin/ \
   && echo "@ Version of Node, npm, and yarn: $(node -v) $(npm -v)" \
   && echo "@ Version of Yarn: $(yarn -v)"
 }
 
 
 setup_R_base() {
-     apt-key adv --keyserver keyserver.ubuntu.com --recv-keys E298A3A825C0D65DFD57CBB651716619E084DAB9 \
+     curl -sL https://cloud.r-project.org/bin/linux/ubuntu/marutter_pubkey.asc | sudo tee -a /etc/apt/trusted.gpg.d/cran_ubuntu_key.asc \
   && echo "deb https://cloud.r-project.org/bin/linux/ubuntu $(lsb_release -cs)-cran40/" > /etc/apt/sources.list.d/cran.list \
   && install_apt  /opt/utils/install_list_R_base.apt \
   && echo "options(repos=structure(c(CRAN=\"https://cloud.r-project.org\")))" >> /etc/R/Rprofile.site \

diff --git a/docker_atom/work/script-utils.sh b/docker_atom/work/script-utils.sh
@@ -1,4 +1,4 @@
-set -x
+# shell util functions
 
 # function to debug, resolve package names from a text file and display.
 install_echo()    { cat $1 | cut -d "%" -f 1 | sed '/^$/d' | xargs -r -n1 printf '%s\n' ; }
@@ -11,7 +11,7 @@ install_conda()   { cat $1 | cut -d "%" -f 1 | sed '/^$/d' | xargs -r -n1 conda
 install_mamba()   { cat $1 | cut -d "%" -f 1 | sed '/^$/d' | xargs -r -n1 mamba install -yq --root-prefix="${CONDA_PREFIX}" --prefix="${CONDA_PREFIX}" ; }
 
 # function to install python packages with pip from a text file which lists package names (add comments with % char)
-install_pip()     { cat $1 | cut -d "%" -f 1 | sed '/^$/d' | xargs -r -n1 pip install -U --root-user-action=ignore --pre ; }
+install_pip()     { cat $1 | cut -d "%" -f 1 | sed '/^$/d' | xargs -r -n1 pip install --no-cache-dir --root-user-action=ignore -U --pre ; }
 
 # function to install R packages from a text file which lists package names (add comments with % char, use quiet=T to be less verbose)
 install_R()       { R -e "options(Ncpus=4);lapply(scan('$1','c',comment.char='%'),function(x){cat(x,system.time(install.packages(x,clean=T,quiet=T)),'\n')})"; }

diff --git a/docker_base/Dockerfile b/docker_base/Dockerfile
@@ -10,9 +10,9 @@ ARG PYTHON_VERSION="3.11"
 ENV CONDA_PREFIX=/opt/conda
 ENV PATH=$PATH:${CONDA_PREFIX}/bin
 
-RUN source /opt/utils/script-setup.sh \
+RUN set -x && source /opt/utils/script-setup.sh \
  # update source for CMake
- && curl -sL https://apt.kitware.com/keys/kitware-archive-latest.asc | sudo apt-key add - \
+ && curl -sL https://apt.kitware.com/keys/kitware-archive-latest.asc | sudo sudo tee /etc/apt/trusted.gpg.d/kitware.asc \
  && echo "deb https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" > /etc/apt/sources.list.d/kitware.list \
  && install_apt /opt/utils/install_list_base.apt \
  && echo "Install tini:" && setup_tini \

diff --git a/docker_cuda/Dockerfile b/docker_cuda/Dockerfile
@@ -11,6 +11,6 @@ ENV NVIDIA_DISABLE_REQUIRE=1
 
 # For cuda version 10.0, the image is solely serverd for legacy tensorflow 1.15, which requires python 3.7
 # For tensorflow 2.x or torch, python>=3.9 is supported.
-RUN echo ${CUDA_VERSION} && nvcc --version \
+RUN set -x && echo ${CUDA_VERSION} && nvcc --version \
  && source /opt/utils/script-setup.sh && setup_nvtop \
  && install__clean
diff --git a/docker_dev/Dockerfile b/docker_dev/Dockerfile
@@ -18,7 +18,7 @@ COPY work /opt/utils/
 
 # Setup Jupyter: Basic Configurations and Extensions...
 RUN mkdir -pv /opt/conda/etc/jupyter/ \
- && mv /opt/utils/jupyter_notebook_config.json /opt/conda/etc/jupyter/ \
+ && mv /opt/utils/etc_jupyter/* /opt/conda/etc/jupyter/ && rm -rf /opt/utils/etc_jupyter \
  && mv /opt/utils/start-*.sh /usr/local/bin/ && chmod +x /usr/local/bin/start-*.sh \
  && source /opt/utils/script-extend.sh \
  && for profile in $(echo $ARG_PROFILE_JUPYTER | tr "," "\n") ; do ( setup_jupyter_${profile} || true ) ; done
@@ -31,10 +31,18 @@ RUN source /opt/utils/script-extend.sh \
  && for profile in $(echo $ARG_PROFILE_VSCODE | tr "," "\n") ; do ( setup_vscode_${profile} || true ) ; done
 
 # Clean up and display components version information...
-RUN source /opt/utils/script-utils.sh  && install__clean
+RUN source /opt/utils/script-utils.sh && install__clean && list_installed_packages
 
 WORKDIR $HOME_DIR
 EXPOSE 8888
 
 ENTRYPOINT ["tini", "-g", "--"]
 CMD ["start-notebook.sh"]
+
+# '-c' option make bash commands are read from string.
+#   If there are arguments after the string, they are assigned to the positional parameters, starting with $0.
+# '-o pipefail'  prevents errors in a pipeline from being masked.
+#   If any command in a pipeline fails, that return code will be used as the return code of the whole pipeline.
+# '--login': make bash first reads and executes commands from  the file /etc/profile, if that file exists.
+#   After that, it looks for ~/.bash_profile, ~/.bash_login, and ~/.profile, in that order, and reads and executes commands from the first one that exists and is readable.
+SHELL ["/bin/bash", "--login", "-o", "pipefail", "-c"]
diff --git a/docker_dev/work/jupyter_notebook_config.json → ...rk/etc_jupyter/jupyter_server_config.json b/docker_dev/work/jupyter_notebook_config.json → ...rk/etc_jupyter/jupyter_server_config.json
diff --git a/docker_dev/work/install_list_JPY_extend.pip b/docker_dev/work/install_list_JPY_extend.pip
@@ -3,11 +3,9 @@
 
 ipyparallel                             % https://github.com/ipython/ipyparallel
 ipynb                                   % https://github.com/ipython/ipynb
+
 jupyter-resource-usage                  % https://github.com/jupyter-server/jupyter-resource-usage
 jupyter-collaboration                   % https://github.com/jupyterlab/jupyter-collaboration
+jupyterlab_server[openapi]              % https://github.com/jupyterlab/jupyterlab_server
 jupyterlab-git                          % https://github.com/jupyterlab/jupyterlab-git
-jupyterlab-latex                        % https://github.com/jupyterlab/jupyterlab-latex
-jupyterlab_server                       % https://github.com/jupyterlab/jupyterlab_server
-# jupyter_nbextensions_configurator       % Jupyter Notebook extension configurator
-# jupyter_contrib_nbextensions            % Jupyter Notebook third-party extensions
-# qpod_hub                                % https://github.com/QPod/qpod-hub
+% jupyterlab-latex                      % bug on pypi version: https://github.com/jupyterlab/jupyterlab-latex
diff --git a/docker_dev/work/script-extend.sh b/docker_dev/work/script-extend.sh
@@ -1,9 +1,11 @@
 source /opt/utils/script-utils.sh
 
 setup_jupyter_base() {
-     pip install -Uq --pre jupyterhub jupyterlab notebook ipywidgets \
-  && echo "@ Version of Jupyter Notebook/JupyterLab: $(jupyter notebook --version)" \
-  && echo "@ Version of Jupyter Notebook/JupyterLab: $(jupyter lab --version)"
+     pip install -Uq --pre jupyterhub jupyterlab notebook nbclassic ipywidgets \
+  && echo "@ Version of Jupyter Server: $(jupyter server --version)" \
+  && echo "@ Version of Jupyter Lab: $(jupyter lab --version)" \
+  && echo "@ Version of Jupyter Notebook: $(jupyter notebook --version)" \
+  && echo "@ Version of JupyterHub: $(jupyterhub --version)"
 }
 
 
@@ -49,23 +51,9 @@ setup_jupyter_extensions() {
      install_apt /opt/utils/install_list_JPY_extend.apt \
   && install_pip /opt/utils/install_list_JPY_extend.pip
 
-  # jupyter nbextensions_configurator enable --sys-prefix
-  # jupyter contrib nbextension install --sys-prefix
-  # jupyter serverextension enable  --sys-prefix --py jupyterlab_git
-
-  # && jupyter nbextension  enable --py widgetsnbextension \
-  # && jupyter labextension install @jupyter-widgets/jupyterlab-manager \
-
-  # jupyter labextension install --no-build \
-  #   @jupyterlab/toc @jupyterlab/shortcutui @jupyterlab/git @jupyterlab/latex \
-  #   @jupyterlab/mathjax3-extension @jupyterlab/fasta-extension @jupyterlab/geojson-extension \
-  #   @jupyterlab/vega3-extension @jupyterlab/commenting-extension
-    # TEMP fix: (not compatible with JupyterLab 2.0)  @jupyterlab/metadata-extension @jupyterlab/dataregistry-extension
-
-  jupyter lab build && echo "@ Jupyter Extension list:" \
-    && jupyter nbextension list \
-    && jupyter serverextension list \
-    && jupyter labextension list
+     echo "@ Jupyter Server Extension list: " && jupyter server extension list \
+  && echo "@ Jupyter Lab Extension list: " && jupyter labextension list \
+  && echo "@ Jupyter Notebook Extension list: " && jupyter notebook extension list
 }
 
 

diff --git a/tool.sh b/tool.sh
@@ -19,12 +19,11 @@ echo "--------> CI_PROJECT_NAMESPACE=${CI_PROJECT_NAMESPACE}"
 echo "--------> DOCKER_REGISTRY_NAMESPACE=${NAMESPACE}"
 
 if [ -f /etc/docker/daemon.json ]; then
-       jq '.experimental=true' /etc/docker/daemon.json > /tmp/daemon.json && sudo mv /tmp/daemon.json /etc/docker/ \
+       jq '.experimental=true | ."data-root"="/mnt/docker"' /etc/docker/daemon.json > /tmp/daemon.json && sudo mv /tmp/daemon.json /etc/docker/ \
     && ( sudo service docker restart || true )
-else
-    docker info
 fi
-
+cat /etc/docker/daemon.json
+docker info
 
 build_image() {
     echo "$@" ;