8000 Allow compressed CSV by ItzNotABug · Pull Request #9673 · appwrite/appwrite · GitHub
[go: up one dir, main page]
More Web Proxy on the site http://driver.im/
Skip to content

Allow compressed CSV #9673

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Apr 21, 2025
Merged

Allow compressed CSV #9673

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
873b384
update: allow compressed files.
ItzNotABug Apr 21, 2025
21be819
fix: test message!
ItzNotABug Apr 21, 2025
0accc49
update: tests.
ItzNotABug Apr 21, 2025
927489b
update: handle decryption as well.
ItzNotABug Apr 21, 2025
53a0424
update: tests.
ItzNotABug Apr 21, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 42 additions & 6 deletions app/controllers/api/migrations.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
use Appwrite\Event\Event;
use Appwrite\Event\Migration;
use Appwrite\Extend\Exception;
use Appwrite\OpenSSL\OpenSSL;
use Appwrite\SDK\AuthType;
use Appwrite\SDK\ContentType;
use Appwrite\SDK\Method;
Expand All @@ -27,8 +28,11 @@
use Utopia\Migration\Sources\NHost;
use Utopia\Migration\Sources\Supabase;
use Utopia\Migration\Transfer;
use Utopia\Storage\Compression\Algorithms\GZIP;
use Utopia\Storage\Compression\Algorithms\Zstd;
use Utopia\Storage\Compression\Compression;
use Utopia\Storage\Device;
use Utopia\System\System;
use Utopia\Validator\ArrayList;
use Utopia\Validator\Integer;
use Utopia\Validator\Text;
Expand Down Expand Up @@ -345,18 +349,50 @@
throw new Exception(Exception::STORAGE_FILE_NOT_FOUND, 'File not found in ' . $path);
}

if (!empty($file->getAttribute('openSSLCipher')) || $file->getAttribute('algorithm', Compression::NONE) !== Compression::NONE) {
throw new Exception(Exception::STORAGE_FILE_TYPE_UNSUPPORTED, "Only uncompressed, unencrypted CSV files can be used for document import.");
}
// no encryption, compression on files above 20MB.
$hasEncryption = !empty($file->getAttribute('openSSLCipher'));
$compression = $file->getAttribute('algorithm', Compression::NONE);
$hasCompression = $compression !== Compression::NONE;

// copy to temporary folder
$migrationId = ID::unique();
$newPath = $deviceForImports->getPath('/' . $migrationId . '_' . $fileId . '.csv');
if (!$deviceForFiles->transfer($path, $newPath, $deviceForImports)) {

if ($hasEncryption || $hasCompression) {
$source = $deviceForFiles->read($path);

// 1. decrypt
if ($hasEncryption) {
$source = OpenSSL::decrypt(
$source,
$file->getAttribute('openSSLCipher'),
System::getEnv('_APP_OPENSSL_KEY_V' . $file->getAttribute('openSSLVersion')),
0,
hex2bin($file->getAttribute('openSSLIV')),
hex2bin($file->getAttribute('openSSLTag'))
);
}

// 2. decompress
if ($hasCompression) {
switch ($compression) {
case Compression::ZSTD:
$source = (new Zstd())->decompress($source);
break;
case Compression::GZIP:
$source = (new GZIP())->decompress($source);
break;
}
}

// manual write after decryption and/or decompression
if (! $deviceForImports->write($newPath, $source, 'text/csv')) {
throw new \Exception("Unable to copy file");
}
} elseif (! $deviceForFiles->transfer($path, $newPath, $deviceForImports)) {
throw new \Exception("Unable to copy file");
}

$fileSize = $deviceForImports->getFileSize($path);
$fileSize = $deviceForImports->getFileSize($newPath);
$resources = Transfer::extractServices([Transfer::GROUP_DATABASES]);

$migration = $dbForProject->createDocument('migrations', new Document([
Expand Down
51 changes: 7 additions & 44 deletions tests/e2e/Services/Migrations/MigrationsBase.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -971,7 +971,6 @@ public function testCreateCsvMigration(): array
$this->assertEquals($response['body']['required'], true);

// make a bucket, upload a file to it!
// 1. enable compression, encryption
$bucketOne = $this->client->call(Client::METHOD_POST, '/storage/buckets', [
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
Expand All @@ -989,33 +988,11 @@ public function testCreateCsvMigration(): array

$bucketOneId = $bucketOne['body']['$id'];

// 2. no compression and encryption
$bucketTwo = $this->client->call(Client::METHOD_POST, '/storage/buckets', [
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
'x-appwrite-key' => $this->getProject()['apiKey'],
], [
'bucketId' => ID::unique(),
'name' => 'Test Bucket 2',
'maximumFileSize' => 2000000, //2MB
'allowedFileExtensions' => ['csv'],
'compression' => 'none',
'encryption' => false
]);

$this->assertNotEmpty($bucketTwo['body']['$id']);
$this->assertEquals(201, $bucketTwo['headers']['status-code']);

$bucketTwoId = $bucketTwo['body']['$id'];

$bucketIds = [
'compressed' => $bucketOneId,
'uncompressed' => $bucketTwoId,

// in uncompressed buckets!
'missing-row' => $bucketTwoId,
'missing-column' => $bucketTwoId,
'irrelevant-column' => $bucketTwoId,
'default' => $bucketOneId,
'missing-row' => $bucketOneId,
'missing-column' => $bucketOneId,
'irrelevant-column' => $bucketOneId,
];

$fileIds = [];
Expand Down Expand Up @@ -1049,20 +1026,6 @@ public function testCreateCsvMigration(): array
$fileIds[$label] = $response['body']['$id'];
}

// compressed, fail.
$compressed = $this->performCsvMigration(
[
'fileId' => $fileIds['compressed'],
'bucketId' => $bucketIds['compressed'],
'resourceId' => $databaseId . ':' . $collectionId,
]
);

// fail on compressed, encrypted buckets!
$this->assertEquals(400, $compressed['body']['code']);
$this->assertEquals('storage_file_type_unsupported', $compressed['body']['type']);
$this->assertEquals('Only uncompressed, unencrypted CSV files can be used for document import.', $compressed['body']['message']);

// missing attribute, fail in worker.
$missingColumn = $this->performCsvMigration(
[
Expand Down Expand Up @@ -1150,12 +1113,12 @@ public function testCreateCsvMigration(): array
);
}, 60000, 500);

// no compression, no encryption, pass.
// all data exists, pass/
$migration = $this->performCsvMigration(
[
'endpoint' => 'http://localhost/v1',
'fileId' => $fileIds['uncompressed'],
'bucketId' => $bucketIds['uncompressed'],
'fileId' => $fileIds['default'],
'bucketId' => $bucketIds['default'],
'resourceId' => $databaseId . ':' . $collectionId,
]
);
Expand Down
0