8000 feat: remove package version files by coolyuantao · Pull Request #781 · cnpm/cnpmcore · GitHub
[go: up one dir, main page]
More Web Proxy on the site http://driver.im/
Skip to content

feat: remove package version files #781

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 37 additions & 0 deletions app/core/service/PackageManagerService.ts
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ import type {
PackageManifestType,
PackageRepository,
} from '../../repository/PackageRepository.js';
import type { PackageVersionFileRepository } from '../../repository/PackageVersionFileRepository.js';
import type { PackageVersionBlockRepository } from '../../repository/PackageVersionBlockRepository.js';
import type { PackageVersionDownloadRepository } from '../../repository/PackageVersionDownloadRepository.js';
import type { DistRepository } from '../../repository/DistRepository.js';
Expand Down Expand Up @@ -101,6 +102,8 @@ export class PackageManagerService extends AbstractService {
@Inject()
private readonly packageRepository: PackageRepository;
@Inject()
private readonly packageVersionFileRepository: PackageVersionFileRepository;
@Inject()
private readonly packageVersionBlockRepository: PackageVersionBlockRepository;
@Inject()
private readonly packageVersionDownloadRepository: PackageVersionDownloadRepository;
Expand Down Expand Up @@ -726,6 +729,40 @@ export class PackageManagerService extends AbstractService {
]);
// remove from repository
await this.packageRepository.removePackageVersion(pkgVersion);

// remove package version files
await this.removePackageVersionFileAndDist(pkgVersion);
}

public async removePackageVersionFileAndDist(pkgVersion: PackageVersion) {
let fileDists: Dist[] = [];
if (!this.config.cnpmcore.enableUnpkg) return fileDists;
if (!this.config.cnpmcore.enableSyncUnpkgFiles) return fileDists;

fileDists =
await this.packageVersionFileRepository.findPackageVersionFileDists(
pkgVersion.packageVersionId
);

// remove nfs dists
await pMap(
fileDists,
async dist => {
await this.distRepository.destroyDist(dist);
},
{
concurrency: 50,
stopOnError: false,
}
);

// remove package version files from repository
await this.packageVersionFileRepository.removePackageVersionFiles(
pkgVersion.packageVersionId,
fileDists.map(dists => dists.distId)
);

return fileDists;
}

public async unpublishPackage(pkg: Package) {
Expand Down
27 changes: 27 additions & 0 deletions app/core/service/PackageVersionFileService.ts
Original file line number Diff line number Diff line change
Expand Up @@ -335,6 +335,33 @@ export class PackageVersionFileService extends AbstractService {
}
}

async removePackageVersionFiles(pkgVersion: PackageVersion) {
const files: PackageVersionFile[] = [];
if (!this.config.cnpmcore.enableUnpkg) return files;
if (!this.config.cnpmcore.enableSyncUnpkgFiles) return files;
const pkg = await this.packageRepository.findPackageByPackageId(
pkgVersion.packageId
);
if (!pkg) return files;

const fileDists =
await this.packageManagerService.removePackageVersionFileAndDist(
pkgVersion
);

return fileDists.map(dist => {
const { directory, name } = this.#getDirectoryAndName(dist.path);
return PackageVersionFile.create({
packageVersionId: pkgVersion.packageVersionId,
directory,
name,
dist,
contentType: mimeLookup(dist.path),
mtime: pkgVersion.publishTime,
});
});
}

async #savePackageVersionFile(
pkg: Package,
pkgVersion: PackageVersion,
Expand Down
30 changes: 30 additions & 0 deletions app/port/controller/PackageVersionFileController.ts
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,36 @@ export class PackageVersionFileController extends AbstractController {
return files.map(file => formatFileItem(file));
}

@HTTPMethod({
// DELETE /:fullname/:versionSpec/files
path: `/:fullname(${FULLNAME_REG_STRING})/:versionSpec/files`,
method: HTTPMethodEnum.DELETE,
})
@Middleware(AdminAccess)
async removeFiles(
@Context() ctx: EggContext,
@HTTPParam() fullname: string,
@HTTPParam() versionSpec: string
) {
ctx.tValidate(Spec, `${fullname}@${versionSpec}`);
this.#requireUnpkgEnable();
const [scope, name] = getScopeAndName(fullname);
const { packageVersion } =
await this.packageManagerService.showPackageVersionByVersionOrTag(
scope,
name,
versionSpec
);
if (!packageVersion) {
throw new NotFoundError(`${fullname}@${versionSpec} not found`);
}
const files =
await this.packageVersionFileService.removePackageVersionFiles(
packageVersion
);
return files.map(file => formatFileItem(file));
}

@HTTPMethod({
// GET /:fullname/:versionSpec/files => /:fullname/:versionSpec/files/${pkg.main}
// GET /:fullname/:versionSpec/files?meta
Expand Down
59 changes: 59 additions & 0 deletions app/repository/PackageVersionFileRepository.ts
Original file line number Diff line number Diff line change
Expand Up @@ -50 6D4E ,6 +50,24 @@
);
}

async findPackageVersionFileDists(packageVersionId: string) {
const fileDists: DistEntity[] = [];
const queue = ['/'];
while (queue.length > 0) {
const directory = queue.shift();
if (!directory) {
continue;
}

Check warning on line 60 in app/repository/PackageVersionFileRepository.ts

View check run for this annotation

Codecov / codecov/patch

app/repository/PackageVersionFileRepository.ts#L59-L60

Added lines #L59 - L60 were not covered by tests
const { files, directories } = await this.listPackageVersionFiles(
packageVersionId,
directory
);
fileDists.push(...files.map(file => file.dist));
queue.push(...directories);
}
return fileDists;
}

async listPackageVersionFiles(packageVersionId: string, directory: string) {
const isRoot = directory === '/';
const where = isRoot
Expand Down Expand Up @@ -96,6 +114,47 @@
return { files, directories: Array.from(subDirectories) };
}

async removePackageVersionFiles(
packageVersionId: string,
distIds?: string[]
) {
if (!distIds) {
const fileDists =
await this.findPackageVersionFileDists(packageVersionId);
distIds = fileDists.map(dist => dist.distId);
}

Check warning on line 125 in app/repository/PackageVersionFileRepository.ts

View check run for this annotation

Codecov / codecov/patch

app/repository/PackageVersionFileRepository.ts#L122-L125

Added lines #L122 - L125 were not covered by tests

await this.PackageVersionFile.transaction(async transaction => {
const removeCount = await this.PackageVersionFile.remove(
{ packageVersionId },
true,
transaction
);
this.logger.info(
'[PackageVersionFileRepository:removePackageVersionFiles:remove] %d rows in PackageVersionFile, packageVersionId: %s',
removeCount,
packageVersionId
);

if (distIds.length > 0) {
const distCount = await this.Dist.remove(
{
distId: distIds,
},
true,
transaction
);
this.logger.info(
'[PackageVersionFileRepository:removePackageVersionFiles:remove] %d rows in Dist, packageVersionId: %s',
distCount,
packageVersionId
);
}
});

return distIds;
}

async hasPackageVersionFiles(packageVersionId: string) {
const model = await this.PackageVersionFile.findOne({ packageVersionId });
return !!model;
Expand Down
145 changes: 145 additions & 0 deletions test/port/controller/PackageVersionFileController/removeFiles.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,145 @@
import assert from 'node:assert/strict';
import { app, mock } from '@eggjs/mock/bootstrap';

import { TestUtil, type TestUser } from '../../../../test/TestUtil.js';

describe('test/port/controller/PackageVersionFileController/removeFiles.test.ts', () => {
let publisher: TestUser;
let adminUser: TestUser;
beforeEach(async () => {
publisher = await TestUtil.createUser();
adminUser = await TestUtil.createAdmin();
});

describe('[DELETE /:fullname/:versionSpec/files] removeFiles()', () => {
it('should 404 when enableUnpkg = false', async () => {
mock(app.config.cnpmcore, 'allowPublishNonScopePackage', true);
mock(app.config.cnpmcore, 'enableUnpkg', false);
const pkg = await TestUtil.getFullPackage({
name: 'foo',
version: '1.0.0',
versionObject: {
description: 'work with utf8mb4 💩, 𝌆 utf8_unicode_ci, foo𝌆bar 🍻',
},
});
await app
.httpRequest()
.put(`/${pkg.name}`)
.set('authorization', publisher.authorization)
.set('user-agent', publisher.ua)
.send(pkg)
.expect(201);
const res = await app
.httpRequest()
.delete('/foo/1.0.0/files')
.set('authorization', adminUser.authorization)
.expect(404)
.expect('content-type', 'application/json; charset=utf-8');
assert.equal(res.body.error, '[NOT_FOUND] Not Found');
});

it('should work', async () => {
mock(app.config.cnpmcore, 'allowPublishNonScopePackage', true);
mock(app.config.cnpmcore, 'enableUnpkg', true);
mock(app.config.cnpmcore, 'enableSyncUnpkgFiles', true);
app.mockLog();

const pkg = await TestUtil.getFullPackage({
name: 'foo',
version: '1.0.0',
versionObject: {
description: 'work with utf8mb4 💩, 𝌆 utf8_unicode_ci, foo𝌆bar 🍻',
},
});

// publish package
await app
.httpRequest()
.put(`/${pkg.name}`)
.set('authorization', publisher.authorization)
.set('user-agent', publisher.ua)
.send(pkg)
.expect(201);
const pkgResponse = await app
.httpRequest()
.get(`/${pkg.name}/1.0.0`)
.expect(200);
const publishTime = new Date(pkgResponse.body.publish_time).toISOString();

// sync package version files
await app
.httpRequest()
.put(`/${pkg.name}/1.0.0/files`)
.set('authorization', adminUser.authorization)
.expect(200)
.expect('content-type', 'application/json; charset=utf-8');
await app
.httpRequest()
.get(`/${pkg.name}/1.0.0/files/package.json`)
.expect(200);

// remove package version files
const packageVersionFilesDeleteResponse = await app
.httpRequest()
.delete(`/${pkg.name}/1.0.0/files`)
.set('authorization', adminUser.authorization)
.expect(200)
.expect('content-type', 'application/json; charset=utf-8');
assert.deepEqual(packageVersionFilesDeleteResponse.body, [
{
path: `/packages/${pkg.name}/1.0.0/files/package.json`,
type: 'file',
contentType: 'application/json',
integrity:
'sha512-yTg/L7tUtFK54aNH3iwgIp7sF3PiAcUrIEUo06bSNq3haIKRnagy6qOwxiEmtfAtNarbjmEpl31ZymySsECi3Q==',
lastModified: publishTime,
size: 209,
},
]);
app.expectLog(
`DELETE /${pkg.name}/1.0.0/files] [nfsAdapter:remove] key: /packages/foo/1.0.0/files/package.json`
);
app.expectLog(
`DELETE /${pkg.name}/1.0.0/files] [PackageVersionFileRepository:removePackageVersionFiles:remove] 1 rows in PackageVersionFile`
);
app.expectLog(
`DELETE /${pkg.name}/1.0.0/files] [PackageVersionFileRepository:removePackageVersionFiles:remove] 1 rows in Dist`
);

// remove again
const packageVersionFilesDeleteResponse2 = await app
.httpRequest()
.delete(`/${pkg.name}/1.0.0/files`)
.set('authorization', adminUser.authorization)
.expect(200)
.expect('content-type', 'application/json; charset=utf-8');
assert.deepEqual(packageVersionFilesDeleteResponse2.body, []);
app.expectLog(
`DELETE /${pkg.name}/1.0.0/files] [PackageVersionFileRepository:removePackageVersionFiles:remove] 0 rows in PackageVersionFile`
);
app.notExpectLog(
`DELETE /${pkg.name}/1.0.0/files] [PackageVersionFileRepository:removePackageVersionFiles:remove] 0 rows in Dist`
);
});

it('should 404 when package not exists', async () => {
const res = await app
.httpRequest()
.delete('/@cnpm/foonot-exists/1.0.40000404/files')
.set('authorization', adminUser.authorization)
.expect(404);
assert.equal(
res.body.error,
'[NOT_FOUND] @cnpm/foonot-exists@1.0.40000404 not found'
);
});

it('should 403 when non-admin request', async () => {
const res = await app
.httpRequest()
.delete('/@cnpm/foonot-exists/1.0.40000404/files')
.expect(403);
assert.equal(res.body.error, '[FORBIDDEN] Not allow to access');
});
});
});
Loading
Loading
0