mteodor · mteodor · Jun 10, 2021 · Jun 10, 2021 · Jun 10, 2021 · Jun 10, 2021
diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml
@@ -39,14 +39,16 @@ services:
     command: /entrypoint.sh
 
   nginx:
-    image: nginx:1.20.0-alpine
+    image: mainflux-nginx:latest
     container_name: mainflux-nginx
     restart: on-failure
     volumes:
-      - ./nginx/nginx-${AUTH-key}.conf:/etc/nginx/nginx.conf.template
+      - ./nginx/nginx-${AUTH-key}.conf:/usr/local/openresty/nginx/conf/nginx.conf.template
+      - ./nginx/mqtt-${AUTH-key}.conf:/usr/local/openresty/nginx/conf/mqtt.conf.template
+      - ./nginx/mfx-${AUTH-key}.conf:/usr/local/openresty/nginx/conf/nginx.conf
       - ./nginx/entrypoint.sh:/entrypoint.sh
-      - ./nginx/snippets:/etc/nginx/snippets
-      - ./ssl/authorization.js:/etc/nginx/authorization.js
+      - ./nginx/snippets:/usr/local/openresty/nginx/conf/snippets
+      - ./ssl/authorization.js:/usr/local/openresty/nginx/conf/snippets/authorization.js
       - ./ssl/certs/mainflux-server.crt:/etc/ssl/certs/mainflux-server.crt
       - ./ssl/certs/ca.crt:/etc/ssl/certs/ca.crt
       - ./ssl/certs/mainflux-server.key:/etc/ssl/private/mainflux-server.key

diff --git a/docker/nginx/Dockerfile b/docker/nginx/Dockerfile
@@ -0,0 +1,14 @@
+FROM openresty/openresty:alpine-fat
+
+RUN mkdir -p /var/log/nginx && \
+    ln -sf /dev/stdout /var/log/nginx/access.log && \
+    ln -sf /dev/stderr /var/log/nginx/error.log
+
+RUN /usr/local/openresty/luajit/bin/luarocks install lua-resty-http
+RUN /usr/local/openresty/luajit/bin/luarocks install lua-cjson
+RUN /usr/local/openresty/luajit/bin/luarocks install lua-resty-session
+RUN /usr/local/openresty/luajit/bin/luarocks install lua-resty-openidc
+
+RUN mkdir -p /data/nginx/cache/ui
+
+ENTRYPOINT ["/entrypoint.sh"]
diff --git a/docker/nginx/README.md b/docker/nginx/README.md
@@ -0,0 +1,15 @@
+# Openresty configuration
+
+Prior to the starting the composition `mainflux-nginx` image must be built using [Dockerfile](Dockerfile)
+
+```
+pwd
+../ui/docker
+docker build --tag=mainflux-nginx -f Dockerfile .
+```
+
+When starting `openresty` image we must mount our own `nginx.conf` as a volume, we cannot rely on `entrypoint.sh` to parse template configuration and create `nginx.conf`.
+So we mount `nginx.conf` which will include configuration files that should be parsed and adjusted according to environment variables.
+
+- `nginx-key.conf` will be mounted as volume in case of plain HTTP, it will include `mfx.conf` which will be result of parsing `mfx-key.conf`. Similar it will include `mqtt.conf` which is result of parsing `mqtt-key.conf` in `entrypoint.sh`.
+- `nginx-x509.conf` will be mounted as volume in case of HTTPS it will include `mfx.conf` which will be result of parsing `mfx-x509.conf`. Additionaly it includes `mqtt.conf` which is result of parsing `mqtt-x509.conf` in `entrypoint.sh`
diff --git a/docker/nginx/entrypoint.sh b/docker/nginx/entrypoint.sh
@@ -2,11 +2,11 @@
 
 if [ -z "$MF_MQTT_CLUSTER" ]
 then
-      envsubst '${MF_MQTT_ADAPTER_MQTT_PORT}' < /etc/nginx/snippets/mqtt-upstream-single.conf > /etc/nginx/snippets/mqtt-upstream.conf
-      envsubst '${MF_MQTT_ADAPTER_WS_PORT}' < /etc/nginx/snippets/mqtt-ws-upstream-single.conf > /etc/nginx/snippets/mqtt-ws-upstream.conf
+      envsubst '${MF_MQTT_ADAPTER_MQTT_PORT}' < /usr/local/openresty/nginx/conf/snippets/mqtt-upstream-single.conf > /usr/local/openresty/nginx/conf/snippets/mqtt-upstream.conf
+      envsubst '${MF_MQTT_ADAPTER_WS_PORT}' < /usr/local/openresty/nginx/conf/snippets/mqtt-ws-upstream-single.conf > /usr/local/openresty/nginx/conf/snippets/mqtt-ws-upstream.conf
 else
-      envsubst '${MF_MQTT_ADAPTER_MQTT_PORT}' < /etc/nginx/snippets/mqtt-upstream-cluster.conf > /etc/nginx/snippets/mqtt-upstream.conf
-      envsubst '${MF_MQTT_ADAPTER_WS_PORT}' < /etc/nginx/snippets/mqtt-ws-upstream-cluster.conf > /etc/nginx/snippets/mqtt-ws-upstream.conf
+      envsubst '${MF_MQTT_ADAPTER_MQTT_PORT}' < /usr/local/openresty/nginx/conf/snippets/mqtt-upstream-cluster.conf > /usr/local/openresty/nginx/conf/snippets/mqtt-upstream.conf
+      envsubst '${MF_MQTT_ADAPTER_WS_PORT}' < /usr/local/openresty/nginx/conf/snippets/mqtt-ws-upstream-cluster.conf > /usr/local/openresty/nginx/conf/snippets/mqtt-ws-upstream.conf
 fi
 
 envsubst '
@@ -22,6 +22,10 @@ envsubst '
     ${MF_INFLUX_READER_PORT}
     ${MF_BOOTSTRAP_PORT}
     ${MF_TWINS_HTTP_PORT}
-    ${MF_OPCUA_ADAPTER_HTTP_PORT}' < /etc/nginx/nginx.conf.template > /etc/nginx/nginx.conf
+    ${MF_OPCUA_ADAPTER_HTTP_PORT}' < /usr/local/openresty/nginx/conf/nginx.conf.template > /usr/local/openresty/nginx/conf/mfx-nginx.conf
 
-exec nginx -g "daemon off;"
+envsubst '
+    ${MF_NGINX_MQTT_PORT}
+    ${MF_NGINX_MQTTS_PORT}' < /usr/local/openresty/nginx/conf/mqtt.conf.template > /usr/local/openresty/nginx/conf/mqtt.conf
+
+exec /usr/local/openresty/bin/openresty -g "daemon off;" 
diff --git a/docker/nginx/mfx-key.conf b/docker/nginx/mfx-key.conf
@@ -0,0 +1,66 @@
+worker_processes  auto;
+
+error_log  /var/log/nginx/error.log debug;
+pid        /var/run/nginx.pid;
+
+events {
+    worker_connections  1024;
+}
+
+http {
+    sendfile on;
+    tcp_nopush on;
+    tcp_nodelay on;
+    keepalive_timeout 65;
+    types_hash_max_size 2048;
+
+
+    ssl_protocols TLSv1.2 TLSv1.3;
+    ssl_prefer_server_ciphers on;
+
+    # Include single-node or multiple-node (cluster) upstream
+    include snippets/mqtt-ws-upstream.conf;
+
+    include       /usr/local/openresty/nginx/conf/mime.types;
+    default_type  application/octet-stream;
+
+    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+                      '$status $body_bytes_sent "$http_referer" '
+                      '"$http_user_agent" "$http_x_forwarded_for" '
+                      'request ID "$upstream_http_x_men_requestid" '
+                      '$request_time';
+
+    log_format access_log_json    '{ "timestamp": "$msec", '
+                                '"nginx": { '
+                                '"remote_addr": "$remote_addr", '
+                                '"remote_user": "$remote_user", '
+                                '"body_bytes_sent": "$body_bytes_sent", '
+                                '"status": "$status", '
+                                '"request_method": "$request_method", '
+                                '"request_uri": "$request_uri", '
+                                '"uri": "$uri", '
+                                '"request_time": "$request_time", '
+                                '"request_id": "$upstream_http_x_men_requestid", '
+                                '"http_referrer": "$http_referer", '
+                                '"http_x_forwarded_for": "$http_x_forwarded_for", '
+                                '"user": "$http_x_webauth_user", '
+                                '"http_user_agent": "$http_user_agent" } }';
+
+    access_log  /var/log/nginx/access.log  access_log_json;
+
+
+    proxy_cache_path /data/nginx/cache/ui levels=1:2 keys_zone=ui_cache:10m max_size=100m
+                     inactive=1h use_temp_path=off;
+
+    # limit_req_zone $binary_remote_addr zone=mylimit:10m rate=@RATE_LIMIT_GLOBAL_RATE@;
+    # limit_req zone=mylimit @RATE_LIMIT_GLOBAL_BURST@ nodelay;
+    # limit_req_status 429;
+
+    #gzip  on;
+    include /usr/local/openresty/nginx/conf/mfx-nginx.conf;
+
+}
+
+# MQTT
+include /usr/local/openresty/nginx/conf/mqtt.conf;
+
diff --git a/docker/nginx/mfx-x509.conf b/docker/nginx/mfx-x509.conf
@@ -0,0 +1,79 @@
+# Copyright (c) Mainflux
+# SPDX-License-Identifier: Apache-2.0
+user nginx;
+worker_processes auto;
+worker_cpu_affinity auto;
+pid /run/nginx.pid;
+load_module /etc/nginx/modules/ngx_stream_js_module.so;
+load_module /etc/nginx/modules/ngx_http_js_module.so;
+include /etc/nginx/modules-enabled/*.conf;
+
+events {
+    # Explanation: https://serverfault.com/questions/787919/optimal-value-for-nginx-worker-connections
+    # We'll keep 10k connections per core (assuming one worker per core)
+    worker_connections 10000;
+}
+
+http {
+    js_include authorization.js;
+    js_set $auth_key setKey;
+
+    sendfile on;
+    tcp_nopush on;
+    tcp_nodelay on;
+    keepalive_timeout 65;
+    types_hash_max_size 2048;
+
+
+    ssl_protocols TLSv1.2 TLSv1.3;
+    ssl_prefer_server_ciphers on;
+
+    # Include single-node or multiple-node (cluster) upstream
+    include snippets/mqtt-ws-upstream.conf;
+
+    include       /usr/local/openresty/nginx/conf/mime.types;
+    default_type  application/octet-stream;
+
+    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+                      '$status $body_bytes_sent "$http_referer" '
+                      '"$http_user_agent" "$http_x_forwarded_for" '
+                      'request ID "$upstream_http_x_men_requestid" '
+                      '$request_time';
+
+    log_format access_log_json    '{ "timestamp": "$msec", '
+                                '"nginx": { '
+                                '"remote_addr": "$remote_addr", '
                                '"remote_user": "$remote_user", '
+                                '"body_bytes_sent": "$body_bytes_sent", '
+                                '"status": "$status", '
+                                '"request_method": "$request_method", '
+                                '"request_uri": "$request_uri", '
+                                '"uri": "$uri", '
+                                '"request_time": "$request_time", '
+                                '"request_id": "$upstream_http_x_men_requestid", '
+                                '"http_referrer": "$http_referer", '
+                                '"http_x_forwarded_for": "$http_x_forwarded_for", '
+                                '"user": "$http_x_webauth_user", '
+                                '"http_user_agent": "$http_user_agent" } }';
+
+    access_log  /var/log/nginx/access.log  access_log_json;
+
+
+    proxy_cache_path /data/nginx/cache/ui levels=1:2 keys_zone=ui_cache:10m max_size=100m
+                     inactive=1h use_temp_path=off;
+
+    # limit_req_zone $binary_remote_addr zone=mylimit:10m rate=@RATE_LIMIT_GLOBAL_RATE@;
+    # limit_req zone=mylimit @RATE_LIMIT_GLOBAL_BURST@ nodelay;
+    # limit_req_status 429;
+
+    #gzip  on;
+
+    include /usr/local/openresty/nginx/conf/mfx-nginx.conf;
+}
+
+# MQTT
+include /usr/local/openresty/nginx/conf/mqtt.conf;
+
+}
+
+error_log  info.log info;
diff --git a/docker/nginx/mqtt-key.conf b/docker/nginx/mqtt-key.conf
@@ -0,0 +1,17 @@
+stream {
+   include snippets/stream_access_log.conf;
+
+    # Include single-node or multiple-node (cluster) upstream
+    include snippets/mqtt-upstream.conf;
+
+    server {
+        listen ${MF_NGINX_MQTT_PORT};
+        listen [::]:${MF_NGINX_MQTT_PORT};
+        listen ${MF_NGINX_MQTTS_PORT} ssl;
+        listen [::]:${MF_NGINX_MQTTS_PORT} ssl;
+
+        include snippets/ssl.conf;
+
+        proxy_pass mqtt_cluster;
+    }
+}
diff --git a/docker/nginx/mqtt-x509.conf b/docker/nginx/mqtt-x509.conf
@@ -0,0 +1,26 @@
+# Copyright (c) Mainflux
+# SPDX-License-Identifier: Apache-2.0
+
+stream {
+   include snippets/stream_access_log.conf;
+
+    # Include JS script for mTLS
+    js_include authorization.js;
+
+    # Include single-node or multiple-node (cluster) upstream
+    include snippets/mqtt-upstream.conf;
+    ssl_verify_client on;
+    include snippets/ssl-client.conf;
+
+    server {
+        listen ${MF_NGINX_MQTT_PORT};
+        listen [::]:${MF_NGINX_MQTT_PORT};
+        listen ${MF_NGINX_MQTTS_PORT} ssl;
+        listen [::]:${MF_NGINX_MQTTS_PORT} ssl;
+
+        include snippets/ssl.conf;
+        js_preread authenticate;
+
+        proxy_pass mqtt_cluster;
+    }
+}
diff --git a/docker/nginx/nginx-key.conf b/docker/nginx/nginx-key.conf
@@ -3,35 +3,6 @@
 
 # This is the default Mainflux NGINX configuration.
 
-user nginx;
-worker_processes auto;
-worker_cpu_affinity auto;
-pid /run/nginx.pid;
-include /etc/nginx/modules-enabled/*.conf;
-
-events {
-    # Explanation: https://serverfault.com/questions/787919/optimal-value-for-nginx-worker-connections
-    # We'll keep 10k connections per core (assuming one worker per core)
-    worker_connections 10000;
-}
-
-http {
-    include snippets/http_access_log.conf;
-
-    sendfile on;
-    tcp_nopush on;
-    tcp_nodelay on;
-    keepalive_timeout 65;
-    types_hash_max_size 2048;
-
-    include /etc/nginx/mime.types;
-    default_type application/octet-stream;
-
-    ssl_protocols TLSv1.2 TLSv1.3;
-    ssl_prefer_server_ciphers on;
-
-    # Include single-node or multiple-node (cluster) upstream
-    include snippets/mqtt-ws-upstream.conf;
 
     server {
         listen 80 default_server;
@@ -140,25 +111,4 @@ http {
             proxy_pass http://ui:${MF_UI_PORT};
         }
     }
-}
-
-# MQTT
-stream {
-   include snippets/stream_access_log.conf;
-
-    # Include single-node or multiple-node (cluster) upstream
-    include snippets/mqtt-upstream.conf;
-
-    server {
-        listen 1883;
-        listen [::]:1883;
-        listen 8883 ssl;
-        listen [::]:8883 ssl;
-
-        include snippets/ssl.conf;
-
-        proxy_pass mqtt_cluster;
-    }
-}
 
-error_log info.log info;