8000 feat(cloudflare): determine DO class migrations using server-side sta… · sam-goodwin/alchemy@d7c0d2c · GitHub
[go: up one dir, main page]
More Web Proxy on the site http://driver.im/
Skip to content

Commit d7c0d2c

Browse files
sam-goodwinsam-goodwin
authored
feat(cloudflare): determine DO class migrations using server-side state (#222)
1 parent 0543977 commit d7c0d2c

File tree

4 files changed

+217
-62
lines changed

4 files changed

+217
-62
lines changed

‎alchemy/src/cloudflare/worker-metadata.ts

Lines changed: 129 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,18 @@
11
import type { Context } from "../context.js";
22
import { slugify } from "../util/slugify.js";
3-
import { Self, type Bindings, type WorkerBindingSpec } from "./bindings.js";
4-
import type { DurableObjectNamespace } from "./durable-object-namespace.js";
3+
import {
4+
Self,
5+
type Bindings,
6+
type WorkerBindingDurableObjectNamespace,
7+
type WorkerBindingSpec,
8+
} from "./bindings.js";
9+
import {
10+
isDurableObjectNamespace,
11+
type DurableObjectNamespace,
12+
} from "./durable-object-namespace.js";
513
import { createAssetConfig, type AssetUploadResult } from "./worker-assets.js";
614
import type { SingleStepMigration } from "./worker-migration.js";
715
import type { AssetsConfig, Worker, WorkerProps } from "./worker.js";
8-
import type { Workflow } from "./workflow.js";
916

1017
/**
1118
* Metadata returned by Cloudflare API for a worker script
@@ -186,28 +193,84 @@ export interface WorkerMetadata {
186193

187194
export async function prepareWorkerMetadata<B extends Bindings>(
188195
ctx: Context<Worker<B>>,
189-
oldBindings: Bindings | undefined,
196+
oldBindings: WorkerBindingSpec[] | undefined,
197+
oldTags: string[] | undefined,
190198
props: WorkerProps & {
191199
compatibilityDate: string;
192200
compatibilityFlags: string[];
193201
workerName: string;
194202
},
195203
assetUploadResult?: AssetUploadResult,
196204
): Promise<WorkerMetadata> {
197-
const deletedClasses = Object.entries(oldBindings ?? {})
198-
.filter(([key]) => !props.bindings?.[key])
199-
.flatMap(([_, binding]) => {
200-
if (
201-
binding &&
202-
typeof binding === "object" &&
203-
binding.type === "durable_object_namespace" &&
204-
(binding.scriptName === undefined ||
205-
binding.scriptName === props.workerName)
206-
) {
207-
return [binding.className];
205+
// we use Cloudflare Worker tags to store a mapping between Alchemy's stable identifier and the binding name
206+
// e.g.
207+
// {
208+
// BINDING_NAME: new DurableObjectNamespace("stable-id")
209+
// }
210+
// will be stored as alchemy:do:stable-id:BINDING_NAME
211+
// TODO(sam): should we base64 encode to ensure no `:` collision risk?
212+
const bindingNameToStableId = Object.fromEntries(
213+
oldTags?.flatMap((tag) => {
214+
// alchemy:do:{stableId}:{bindingName}
215+
if (tag.startsWith("alchemy:do:")) {
216+
const [, , stableId, bindingName] = tag.split(":");
217+
return [[bindingName, stableId]];
208218
}
209219
return [];
210-
});
220+
}) ?? [],
221+
);
222+
223+
const deletedClasses = oldBindings?.flatMap((oldBinding) => {
224+
if (
225+
oldBinding.type === "durable_object_namespace" &&
226+
(oldBinding.script_name === undefined ||
227+
// if this a cross-script binding, we don't need to do migrations in the remote worker
228+
oldBinding.script_name === props.workerName)
229+
) {
230+
// reverse the stableId from our tag-encoded metadata
231+
const stableId = bindingNameToStableId[oldBinding.name];
232+
if (stableId) {
233+
if (props.bindings === undefined) {
234+
// all classes are deleted
235+
return [oldBinding.class_name];
236+
}
237+
// we created this worker on latest version, we can now intelligently determine migrations
238+
239+
// try and find the DO binding by stable id
240+
const object = Object.values(props.bindings).find(
241+
(binding): binding is DurableObjectNamespace<any> =>
242+
isDurableObjectNamespace(binding) && binding.id === stableId,
243+
);
244+
if (object) {
245+
// we found the corresponding object, it should not be deleted
246+
return [];
247+
} else {
248+
// it was not found, we will now delete it
249+
return [oldBinding.class_name];
250+
}
251+
} else {
252+
// ok, we were unable to find the stableId, this worker must have been created by an old alchemy or outside of alchemy
253+
// let's now apply a herusitic based on binding name (assume binding name is consistent)
254+
// TODO(sam): this has a chance of being wrong, is that OK? Users should be encouraged to upgrade alchemy version and re-deploy
255+
const object = props.bindings?.[oldBinding.name];
256+
if (object && isDurableObjectNamespace(object)) {
257+
if (object.className === oldBinding.class_name) {
258+
// this is relatively safe to assume is the right match, do not delete
259+
return [];
260+
} else {
261+
// the class name has changed, this could indicate one of:
262+
// 1. the user has changed the class name and we should migrate it
263+
// 2. the user deleted the DO a long time ago and this is unrelated (we should just create a new one)
264+
return [oldBinding.class_name];
265+
}
266+
} else {
267+
// we didn't find it, so delete it
268+
return [oldBinding.class_name];
269+
}
270+
}
271+
}
272+
return [];
273+
});
211274

212275
// Prepare metadata with bindings
213276
const meta: WorkerMetadata = {
@@ -218,11 +281,22 @@ export async function prepareWorkerMetadata<B extends Bindings>(
218281
enabled: props.observability?.enabled !== false,
219282
},
220283
// TODO(sam): base64 encode instead? 0 collision risk vs readability.
221-
tags: [`alchemy:id:${slugify(ctx.fqn)}`],
284+
tags: [
285+
`alchemy:id:${slugify(ctx.fqn)}`,
286+
// encode a mapping table of Durable Object stable ID -> binding name
287+
// we use this to reliably compute class migrations based on server-side state
288+
...Object.entries(props.bindings ?? {}).flatMap(
289+
([bindingName, binding]) =>
290+
isDurableObjectNamespace(binding)
291+
? // TODO(sam): base64 encode if contains `:`?
292+
[`alchemy:do:${binding.id}:${bindingName}`]
293+
: [],
294+
),
295+
],
222296
migrations: {
223297
new_classes: props.migrations?.new_classes ?? [],
224298
deleted_classes: [
225-
...deletedClasses,
299+
...(deletedClasses ?? []),
226300
...(props.migrations?.deleted_classes ?? []),
227301
],
228302
renamed_classes: props.migrations?.renamed_classes ?? [],
@@ -303,7 +377,7 @@ export async function prepareWorkerMetadata<B extends Bindings>(
303377
binding.scriptName === props.workerName
304378
) {
305379
// we do not need configure class migrations for cross-script bindings
306-
configureClassMigration(binding, binding.id, binding.className);
380+
configureClassMigration(bindingName, binding);
307381
}
308382
} else if (binding.type === "r2_bucket") {
309383
meta.bindings.push({
@@ -393,29 +467,46 @@ export async function prepareWorkerMetadata<B extends Bindings>(
393467
}
394468

395469
function configureClassMigration(
396-
binding: DurableObjectNamespace<any> | Workflow,
397-
stableId: string,
398-
className: string,
470+
bindingName: string,
471+
newBinding: DurableObjectNamespace<any>,
399472
) {
400-
const oldBinding: DurableObjectNamespace<any> | Workflow | undefined =
401-
Object.values(oldBindings ?? {})
402-
?.filter(
403-
(b) =>
404-
typeof b === "object" &&
405-
(b.type === "durable_object_namespace" || b.type === "workflow"),
406-
)
407-
?.find((b) => b.id === stableId);
408-
409-
if (!oldBinding) {
410-
if (binding.type === "durable_object_namespace" && binding.sqlite) {
411-
meta.migrations!.new_sqlite_classes!.push(className);
473+
let prevBinding: WorkerBindingDurableObjectNamespace | undefined;
474+
if (oldBindings) {
475+
// try and find the prev binding for this
476+
for (const oldBinding of oldBindings) {
477+
if (oldBinding.type === "durable_object_namespace") {
478+
const stableId = bindingNameToStableId[oldBinding.name];
479+
if (stableId) {
480+
// (happy case)
481+
// great, this Worker was created with Alchemy and we can map stable ids
482+
if (stableId === newBinding.id) {
483+
prevBinding = oldBinding;
484+
break;
485+
}
486+
} else {
487+
// (heuristic case)
488+
// we were unable to find the stableId, this Worker must not have been created with Alchemy
489+
// now, try and resolve by assuming 1:1 binding name correspondence
490+
// WARNING: this is an imperfect assumption. Users are advised to upgrade alchemy and re-deploy
491+
if (oldBinding.name === bindingName) {
492+
prevBinding = oldBinding;
493+
break;
494+
}
495+
}
496+
}
497+
}
498+
}
499+
500+
if (!prevBinding) {
501+
if (newBinding.sqlite) {
502+
meta.migrations!.new_sqlite_classes!.push(newBinding.className);
412503
} else {
413-
meta.migrations!.new_classes!.push(className);
504+
meta.migrations!.new_classes!.push(newBinding.className);
414505
}
415-
} else if (oldBinding.className !== className) {
506+
} else if (prevBinding.class_name !== newBinding.className) {
416507
meta.migrations!.renamed_classes!.push({
417-
from: oldBinding.className,
418-
to: className,
508+
from: prevBinding.class_name,
509+
to: newBinding.className,
419510
});
420511
}
421512
}

‎alchemy/src/cloudflare/worker.ts

Lines changed: 46 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,12 @@ import {
1717
createCloudflareApi,
1818
} from "./api.js";
1919
import type { Assets } from "./assets.js";
20-
import { type Binding, type Bindings, Json } from "./bindings.js";
20+
import {
21+
type Binding,
22+
type Bindings,
23+
Json,
24+
type WorkerBindingSpec,
25+
} from "./bindings.js";
2126
import type { Bound } from "./bound.js";
2227
import { isBucket } from "./bucket.js";
2328
import { bundleWorkerScript } from "./bundle/bundle-worker.js";
@@ -678,7 +683,11 @@ export const _Worker = Resource(
678683
const compatibilityFlags = props.compatibilityFlags ?? [];
679684

680685
const uploadWorkerScript = async (props: WorkerProps<B>) => {
681-
const oldBindings = await this.get<Bindings>("bindings");
686+
const [oldBindings, oldMetadata] = await Promise.all([
687+
getWorkerBindings(api, workerName),
688+
getWorkerScriptMetadata(api, workerName),
689+
]);
690+
const oldTags = oldMetadata?.default_environment?.script?.tags;
682691

683692
// Get the script content - either from props.script, or by bundling
684693
const scriptContent =
@@ -725,6 +734,7 @@ export const _Worker = Resource(
725734
const scriptMetadata = await prepareWorkerMetadata(
726735
this,
727736
oldBindings,
737+
oldTags,
728738
{
729739
...props,
730740
compatibilityDate,
@@ -736,9 +746,6 @@ export const _Worker = Resource(
736746

737747
await putWorker(api, workerName, scriptContent, scriptMetadata);
738748

739-
// TODO: it is less than ideal that this can fail, resulting in state problem
740-
await this.set("bindings", props.bindings);
741-
742749
for (const workflow of workflowsBindings) {
743750
await upsertWorkflow(api, {
744751
workflowName: workflow.workflowName,
@@ -1106,34 +1113,49 @@ export async function getWorkerScriptMetadata(
11061113
return ((await response.json()) as any).result as WorkerScriptMetadata;
11071114
}
11081115

1109-
export async function _getWorkerBindings(
1110-
api: CloudflareApi,
1111-
workerName: string,
1112-
environment = "production",
1113-
) {
1116+
async function getWorkerBindings(api: CloudflareApi, workerName: string) {
1117+
// Fetch the bindings for a worker by calling the Cloudflare API endpoint:
1118+
// GET /accounts/:account_id/workers/scripts/:script_name/bindings
1119+
// See: https://developers.cloudflare.com/api/resources/workers/subresources/scripts/subresources/script_and_version_settings/methods/get/
11141120
const response = await api.get(
1115-
`/accounts/${api.accountId}/workers/services/${workerName}/environments/${environment}/bindings`,
1116-
{
1117-
headers: {
1118-
Authorization: `Bearer ${process.env.CLOUDFLARE_API_TOKEN}`,
1119-
"Content-Type": "application/json",
1120-
},
1121-
},
1121+
`/accounts/${api.accountId}/workers/scripts/${workerName}/settings`,
11221122
);
1123-
11241123
if (response.status === 404) {
11251124
return undefined;
11261125
}
1127-
11281126
if (!response.ok) {
11291127
throw new Error(
1130-
`Failed to fetch bindings: ${response.status} ${response.statusText}`,
1128+
`Error getting worker bindings: ${response.status} ${response.statusText}`,
11311129
);
11321130
}
1133-
1134-
const data: any = await response.json();
1135-
1136-
return data.result;
1131+
// The result is an object with a "result" property containing the bindings array
1132+
const { result, success, errors } = (await response.json()) as {
1133+
result: {
1134+
bindings: WorkerBindingSpec[];
1135+
compatibility_date: string;
1136+
compatibility_flags: string[];
1137+
[key: string]: any;
1138+
};
1139+
success: boolean;
1140+
errors: Array<{
1141+
code: number;
1142+
message: string;
1143+
documentation_url: string;
1144+
[key: string]: any;
1145+
}>;
1146+
messages: Array<{
1147+
code: number;
1148+
message: string;
1149+
documentation_url: string;
1150+
[key: string]: any;
1151+
}>;
1152+
};
1153+
if (!success) {
1154+
throw new Error(
1155+
`Error getting worker bindings: ${response.status} ${response.statusText}\nErrors:\n${errors.map((e) => `- [${e.code}] ${e.message} (${e.documentation_url})`).join("\n")}`,
1156+
);
1157+
}
1158+
return result.bindings;
11371159
}
11381160

11391161
/**

‎alchemy/src/cloudflare/workflow.ts

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import { handleApiError } from "./api-error.js";
22
import type { CloudflareApi } from "./api.js";
3+
import type { Binding } from "./bindings.js";
34

45
export interface WorkflowProps {
56
/**
@@ -20,6 +21,10 @@ export interface WorkflowProps {
2021
className?: string;
2122
}
2223

24+
export function isWorkflow(binding: Binding): binding is Workflow {
25+
return typeof binding === "object" && binding.type === "workflow";
26+
}
27+
2328
export class Workflow<PARAMS = unknown> {
2429
public readonly type = "workflow";
2530
/**

‎alchemy/test/cloudflare/worker.test.ts

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2569,4 +2569,41 @@ describe("Worker Resource", () => {
25692569
await assertWorkerDoesNotExist(clientWorkerName);
25702570
}
25712571
}, 120000); // Increased timeout for cross-script DO operations
2572+
2573+
test("adopting a Worker should use server-side state to migrate classes", async (scope) => {
2574+
try {
2575+
const workerName = `${BRANCH_PREFIX}-test-worker-adoption-migrate`;
2576+
await Worker("worker-1", {
2577+
name: workerName,
2578+
script: `
2579+
export class Counter {}
2580+
export default { fetch() {} }
2581+
`,
2582+
bindings: {
2583+
DO: new DurableObjectNamespace("DO", {
2584+
className: "Counter",
2585+
}),
2586+
},
2587+
});
2588+
2589+
await Worker("worker-2", {
2590+
name: workerName,
2591+
// adopt the worker since it already exists
2592+
adopt: true,
2593+
script: `
2594+
export class Counter2 {}
2595+
export default { fetch() {} }
2596+
`,
2597+
bindings: {
2598+
// mapped by stable ID "DO"
2599+
DO_1: new DurableObjectNamespace("DO", {
2600+
// should migrate to Counter 2
2601+
className: "Counter2",
2602+
}),
2603+
},
2604+
});
2605+
} finally {
2606+
await destroy(scope);
2607+
}
2608+
});
25722609
});

0 commit comments

Comments
 (0)
0