appwrite · TorstenDittmann · Jan 24, 2024 · Oct 25, 2023 · Oct 25, 2023 · Oct 25, 2023
diff --git a/.env b/.env
@@ -4,12 +4,13 @@ _APP_WORKER_PER_CORE=6
 _APP_CONSOLE_WHITELIST_ROOT=disabled
 _APP_CONSOLE_WHITELIST_EMAILS=
 _APP_CONSOLE_WHITELIST_IPS=
+_APP_CONSOLE_HOSTNAMES=localhost,appwrite.io,*.appwrite.io
 _APP_SYSTEM_EMAIL_NAME=Appwrite
 _APP_SYSTEM_EMAIL_ADDRESS=team@appwrite.io
 _APP_SYSTEM_SECURITY_EMAIL_ADDRESS=security@appwrite.io
 _APP_SYSTEM_RESPONSE_FORMAT=
 _APP_OPTIONS_ABUSE=disabled
-_APP_OPTIONS_ROUTER_PROTECTION=disbled
+_APP_OPTIONS_ROUTER_PROTECTION=disabled
 _APP_OPTIONS_FORCE_HTTPS=disabled
 _APP_OPTIONS_FUNCTIONS_FORCE_HTTPS=disabled
 _APP_OPENSSL_KEY_V1=your-secret-key
@@ -50,10 +51,6 @@ _APP_STORAGE_WASABI_BUCKET=
 _APP_STORAGE_ANTIVIRUS=disabled
 _APP_STORAGE_ANTIVIRUS_HOST=clamav
 _APP_STORAGE_ANTIVIRUS_PORT=3310
-_APP_INFLUXDB_HOST=influxdb
-_APP_INFLUXDB_PORT=8086
-_APP_STATSD_HOST=telegraf
-_APP_STATSD_PORT=8125
 _APP_SMTP_HOST=maildev
 _APP_SMTP_PORT=1025
 _APP_SMTP_SECURE=
@@ -79,7 +76,7 @@ _APP_MAINTENANCE_RETENTION_CACHE=2592000
 _APP_MAINTENANCE_RETENTION_EXECUTION=1209600
 _APP_MAINTENANCE_RETENTION_ABUSE=86400
 _APP_MAINTENANCE_RETENTION_AUDIT=1209600
-_APP_USAGE_AGGREGATION_INTERVAL=5
+_APP_USAGE_AGGREGATION_INTERVAL=60000
 _APP_MAINTENANCE_RETENTION_USAGE_HOURLY=8640000
 _APP_MAINTENANCE_RETENTION_SCHEDULES=86400
 _APP_USAGE_STATS=enabled

diff --git a/CHANGES.md b/CHANGES.md
@@ -1,3 +1,14 @@
+# Version 1.4.14
+
+## Changes
+- New usage metrics collection flow [#7005](https://github.com/appwrite/appwrite/pull/7005)
+  - Deprecated influxdb, telegraf containers and removed all of their occurrences from the code.
+  - Removed _APP_INFLUXDB_HOST, _APP_INFLUXDB_PORT, _APP_STATSD_HOST, _APP_STATSD_PORT env variables.
+  - Removed usage labels dependency.
+  - Dropped type attribute from stats collection.
+  - Usage metrics are processed via new usage worker.
+  - updated Metric names.
+
 # Version 1.4.13
 
 ## Notable changes
@@ -49,6 +60,7 @@
 * Use getQueueSize() in the Health service's get X queue endpoints [#7073](https://github.com/appwrite/appwrite/pull/7073)
 * Delete linked VCS repos and comments [#7066](https://github.com/appwrite/appwrite/pull/7066)
 
+
 # Version 1.4.9
 
 ## Bug fixes

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -210,7 +210,6 @@ Appwrite's current structure is a combination of both [Monolithic](https://en.wi
 │       ├── Task
 │       ├── Template
 │       ├── URL
-│       ├── Usage
 │       └── Utopia
 └── tests # End to end & unit tests
     ├── e2e

diff --git a/Dockerfile b/Dockerfile
@@ -74,7 +74,6 @@ RUN mkdir -p /storage/uploads && \
 # Executables
 RUN chmod +x /usr/local/bin/doctor && \
     chmod +x /usr/local/bin/maintenance &&  \
-    chmod +x /usr/local/bin/usage && \
     chmod +x /usr/local/bin/install && \
     chmod +x /usr/local/bin/upgrade && \
     chmod +x /usr/local/bin/migrate && \
@@ -95,7 +94,9 @@ RUN chmod +x /usr/local/bin/doctor && \
     chmod +x /usr/local/bin/worker-messaging && \
     chmod +x /usr/local/bin/worker-webhooks && \
     chmod +x /usr/local/bin/worker-migrations && \
-    chmod +x /usr/local/bin/worker-hamster
+    chmod +x /usr/local/bin/worker-hamster && \
+    chmod +x /usr/local/bin/worker-usage
+
 
 # Cloud Executabless
 RUN chmod +x /usr/local/bin/hamster && \
@@ -107,7 +108,8 @@ RUN chmod +x /usr/local/bin/hamster && \
     chmod +x /usr/local/bin/clear-card-cache && \
     chmod +x /usr/local/bin/calc-users-stats && \
     chmod +x /usr/local/bin/calc-tier-stats && \
-    chmod +x /usr/local/bin/get-migration-stats
+    chmod +x /usr/local/bin/get-migration-stats && \
+    chmod +x /usr/local/bin/create-inf-metric
 
 # Letsencrypt Permissions
 RUN mkdir -p /etc/letsencrypt/live/ && chmod -Rf 755 /etc/letsencrypt/live/

diff --git a/app/cli.php b/app/cli.php
@@ -125,30 +125,6 @@
     };
 }, ['pools', 'dbForConsole', 'cache']);
 
-CLI::setResource('influxdb', function (Registry $register) {
-    $client = $register->get('influxdb'); /** @var InfluxDB\Client $client */
-    $attempts = 0;
-    $max = 10;
-    $sleep = 1;
-
-    do { // check if telegraf database is ready
-        try {
-            $attempts++;
-            $database = $client->selectDB('telegraf');
-            if (in_array('telegraf', $client->listDatabases())) {
-                break; // leave the do-while if successful
-            }
-        } catch (\Throwable $th) {
-            Console::warning("InfluxDB not ready. Retrying connection ({$attempts})...");
-            if ($attempts >= $max) {
-                throw new \Exception('InfluxDB database not ready yet');
-            }
-            sleep($sleep);
-        }
-    } while ($attempts < $max);
-    return $database;
-}, ['register']);
-
 CLI::setResource('queue', function (Group $pools) {
     return $pools->get('queue')->pop()->getResource();
 }, ['pools']);

diff --git a/app/config/collections.php b/app/config/collections.php
@@ -18,6 +18,63 @@
  */
 
 $commonCollections = [
+    'cache' => [
+        '$collection' => Database::METADATA,
+        '$id' => 'cache',
+        'name' => 'Cache',
+        'attributes' => [
+            [
+                '$id' => 'resource',
+                'type' => Database::VAR_STRING,
+                'format' => '',
+                'size' => 255,
+                'signed' => true,
+                'required' => false,
+                'default' => null,
+                'array' => false,
+                'filters' => [],
+            ],
+            [
+                '$id' => 'accessedAt',
+                'type' => Database::VAR_DATETIME,
+                'format' => '',
+                'size' => 0,
+                'signed' => false,
+                'required' => false,
+                'default' => null,
+                'array' => false,
+                'filters' => ['datetime'],
+            ],
+            [
+                '$id' => 'signature',
+                'type' => Database::VAR_STRING,
+                'format' => '',
+                'size' => 255,
+                'signed' => true,
+                'required' => false,
+                'default' => null,
+                'array' => false,
+                'filters' => [],
+            ],
+        ],
+        'indexes' => [
+            [
+                '$id' => '_key_accessedAt',
+                'type' => Database::INDEX_KEY,
+                'attributes' => ['accessedAt'],
+                'lengths' => [],
+                'orders' => [],
+            ],
+            [
+                '$id' => '_key_resource',
+                'type' => Database::INDEX_KEY,
+                'attributes' => ['resource'],
+                'lengths' => [],
+                'orders' => [],
+            ],
+        ],
+    ],
+
     'users' => [
         '$collection' => ID::custom(Database::METADATA),
         '$id' => ID::custom('users'),
@@ -1270,10 +1327,10 @@
         ]
     ],
 
-    'stats' => [
+    'stats_v2' => [
         '$collection' => ID::custom(Database::METADATA),
-        '$id' => ID::custom('stats'),
-        'name' => 'Stats',
+        '$id' => ID::custom('stats_v2'),
+        'name' => 'stats_v2',
         'attributes' => [
             [
                 '$id' => ID::custom('metric'),
@@ -1302,7 +1359,7 @@
                 'type' => Database::VAR_INTEGER,
                 'format' => '',
                 'size' => 8,
-                'signed' => false,
+                'signed' => true,
                 'required' => true,
                 'default' => null,
                 'array' => false,
@@ -1330,17 +1387,6 @@
                 'array' => false,
                 'filters' => [],
             ],
-            [
-                '$id' => ID::custom('type'),
-                'type' => Database::VAR_INTEGER,
-                'format' => '',
-                'size' => 1,
-                'signed' => false,
-                'required' => true,
-                'default' => 0, // 0 -> count, 1 -> sum
-                'array' => false,
-                'filters' => [],
-            ],
         ],
         'indexes' => [
             [
@@ -1359,7 +1405,7 @@
             ],
             [
                 '$id' => ID::custom('_key_metric_period_time'),
-                'type' => Database::INDEX_KEY,
+                'type' => Database::INDEX_UNIQUE,
                 'attributes' => ['metric', 'period', 'time'],
                 'lengths' => [],
                 'orders' => [Database::ORDER_DESC],
@@ -2883,63 +2929,6 @@
         ],
     ],
 
-    'cache' => [
-        '$collection' => Database::METADATA,
-        '$id' => 'cache',
-        'name' => 'Cache',
-        'attributes' => [
-            [
-                '$id' => 'resource',
-                'type' => Database::VAR_STRING,
-                'format' => '',
-                'size' => 255,
-                'signed' => true,
-                'required' => false,
-                'default' => null,
-                'array' => false,
-                'filters' => [],
-            ],
-            [
-                '$id' => 'accessedAt',
-                'type' => Database::VAR_DATETIME,
-                'format' => '',
-                'size' => 0,
-                'signed' => false,
-                'required' => false,
-                'default' => null,
-                'array' => false,
-                'filters' => ['datetime'],
-            ],
-            [
-                '$id' => 'signature',
-                'type' => Database::VAR_STRING,
-                'format' => '',
-                'size' => 255,
-                'signed' => true,
-                'required' => false,
-                'default' => null,
-                'array' => false,
-                'filters' => [],
-            ],
-        ],
-        'indexes' => [
-            [
-                '$id' => '_key_accessedAt',
-                'type' => Database::INDEX_KEY,
-                'attributes' => ['accessedAt'],
-                'lengths' => [],
-                'orders' => [],
-            ],
-            [
-                '$id' => '_key_resource',
-                'type' => Database::INDEX_KEY,
-                'attributes' => ['resource'],
-                'lengths' => [],
-                'orders' => [],
-            ],
-        ],
-    ],
-
     'variables' => [
         '$collection' => Database::METADATA,
         '$id' => 'variables',

diff --git a/app/config/locale/templates/email-inner-base.tpl b/app/config/locale/templates/email-inner-base.tpl
@@ -1,15 +1,9 @@
-<p>{{hello}}</p>
-
-<br>
-
+<p>{{hello}},</p>
 <p>{{body}}</p>
-
-<a href="{{redirect}}" target="_blank">{{redirect}}</a>
-
 <p>{{footer}}</p>
-
-<br>
-
-<p>{{thanks}}</p>
-
-<p>{{signature}}</p>
+<p style="margin-bottom: 32px">
+    {{thanks}},
+    <br/>
+    {{signature}}
+</p>
diff --git a/app/config/platforms.php b/app/config/platforms.php
@@ -185,7 +185,7 @@
             [
                 'key' => 'web',
                 'name' => 'Console',
-                'version' => '0.3.0',
+                'version' => '0.5.0',
                 'url' => 'https://github.com/appwrite/sdk-for-console',
                 'package' => '',
                 'enabled' => true,
@@ -195,8 +195,8 @@
                 'family' => APP_PLATFORM_CONSOLE,
                 'prism' => 'javascript',
                 'source' => \realpath(__DIR__ . '/../sdks/console-web'),
-                'gitUrl' => 'git@github.com:appwrite/sdk-for-console.git',
-                'gitBranch' => 'dev',
+                'gitUrl' => 'https://github.com/appwrite/sdk-for-console.git',
+                'gitBranch' => 'main',
                 'gitRepoName' => 'sdk-for-console',
                 'gitUserName' => 'appwrite',
             ],

diff --git a/app/config/specs/open-api3-1.4.x-client.json b/app/config/specs/open-api3-1.4.x-client.json
diff --git a/app/config/specs/open-api3-1.4.x-console.json b/app/config/specs/open-api3-1.4.x-console.json
diff --git a/app/config/specs/open-api3-1.4.x-server.json b/app/config/specs/open-api3-1.4.x-server.json
diff --git a/app/config/specs/open-api3-latest-client.json b/app/config/specs/open-api3-latest-client.json
diff --git a/app/config/specs/open-api3-latest-console.json b/app/config/specs/open-api3-latest-console.json
diff --git a/app/config/specs/open-api3-latest-server.json b/app/config/specs/open-api3-latest-server.json
diff --git a/app/config/specs/swagger2-1.4.x-client.json b/app/config/specs/swagger2-1.4.x-client.json
diff --git a/app/config/specs/swagger2-1.4.x-console.json b/app/config/specs/swagger2-1.4.x-console.json
diff --git a/app/config/specs/swagger2-1.4.x-server.json b/app/config/specs/swagger2-1.4.x-server.json
diff --git a/app/config/specs/swagger2-latest-client.json b/app/config/specs/swagger2-latest-client.json
diff --git a/app/config/specs/swagger2-latest-console.json b/app/config/specs/swagger2-latest-console.json
diff --git a/app/config/specs/swagger2-latest-server.json b/app/config/specs/swagger2-latest-server.json
diff --git a/app/config/variables.php b/app/config/variables.php
@@ -115,14 +115,6 @@
                 'question' => '',
                 'filter' => ''
             ],
-            // [
-            //     'name' => '_APP_CONSOLE_WHITELIST_DOMAINS',
-            //     'description' => 'This option allows you to limit creation of users to Appwrite console for users sharing the same email domains. This option is very useful for team working with company emails domain.\n\nTo enable this option, pass a list of allowed email domains separated by a comma.',
-            //     'introduction' => '',
-            //     'default' => '',
-            //     'required' => false,
-            //     'question' => '',
-            // ],
             [
                 'name' => '_APP_CONSOLE_WHITELIST_IPS',
                 'description' => "This last option allows you to limit creation of users in Appwrite console for users sharing the same set of IP addresses. This option is very useful for team working with a VPN service or a company IP.\n\nTo enable/activate this option, pass a list of allowed IP addresses separated by a comma.",
@@ -132,6 +124,15 @@
                 'question' => '',
                 'filter' => ''
             ],
+            [
+                'name' => '_APP_CONSOLE_HOSTNAMES',
+                'description' => 'This option allows you to add additional hostnames to your Appwrite console. This option is very useful for allowing access to the console project from additional domains. To enable it, pass a list of allowed hostnames separated by a comma.',
+                'introduction' => '1.5.0',
+                'default' => '',
+                'required' => false,
+                'question' => '',
+                'filter' => ''
+            ],
             [
                 'name' => '_APP_SYSTEM_EMAIL_NAME',
                 'description' => 'This is the sender name value that will appear on email messages sent to developers from the Appwrite console. The default value is: \'Appwrite\'. You can use url encoded strings for spaces and special chars.',
@@ -337,7 +338,7 @@
     ],
     [
         'category' => 'InfluxDB',
-        'description' => 'Appwrite uses an InfluxDB server for managing time-series data and server stats. The InfluxDB env vars are used to allow Appwrite server to connect to the InfluxDB container.',
+        'description' => 'Deprecated since 1.4.8.',
         'variables' => [
             [
                 'name' => '_APP_INFLUXDB_HOST',
@@ -361,7 +362,7 @@
     ],
     [
         'category' => 'StatsD',
-        'description' => 'Appwrite uses a StatsD server for aggregating and sending stats data over a fast UDP connection. The StatsD env vars are used to allow Appwrite server to connect to the StatsD container.',
+        'description' => 'Deprecated since 1.4.8.',
         'variables' => [
             [
                 'name' => '_APP_STATSD_HOST',