From d455481dca7da6a7cf16bae80e995e536458d9b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karel=20Kl=C3=ADma?= Date: Sat, 10 May 2025 18:45:49 +0200 Subject: [PATCH 1/4] Moved DBpedia scenarios to a dedicated folder --- performance/dbpedia/README.md | 38 +++++++++++++++++ performance/dbpedia/common.ts | 40 ++++++++++++++++++ performance/dbpedia/deno.json | 12 ++++++ .../dbpedia}/scenario1.ts | 6 +-- .../dbpedia}/scenario2.ts | 11 +++-- performance/dbpedia/scenario3.ts | 31 ++++++++++++++ tests/performance/common.ts | 41 ------------------- tests/performance/scenario3.ts | 20 --------- 8 files changed, 129 insertions(+), 70 deletions(-) create mode 100644 performance/dbpedia/README.md create mode 100644 performance/dbpedia/common.ts create mode 100644 performance/dbpedia/deno.json rename {tests/performance => performance/dbpedia}/scenario1.ts (67%) rename {tests/performance => performance/dbpedia}/scenario2.ts (66%) create mode 100644 performance/dbpedia/scenario3.ts delete mode 100644 tests/performance/common.ts delete mode 100644 tests/performance/scenario3.ts diff --git a/performance/dbpedia/README.md b/performance/dbpedia/README.md new file mode 100644 index 0000000..1969e50 --- /dev/null +++ b/performance/dbpedia/README.md @@ -0,0 +1,38 @@ +# DBpedia performance suite + +This test suite include a set of tests that query data from DBpedia SPARQL +endpoint. There are three scenarios, ranging from the most simple one to the +most complex. + +Please be aware that the test scenarios may timeout unexpectedly because of rate +limiting of the public endpoint. + +Deno is required to run the test scenarios. + +## 1) Install Deno + +Windows: + +``` +irm https://deno.land/install.ps1 | iex +``` + +MacOS/Linux: + +``` +curl -fsSL https://deno.land/install.sh | sh +``` + +## 2) Run test scenarios + +``` +deno task scenario1 +``` + +``` +deno task scenario2 +``` + +``` +deno task scenario3 +``` diff --git a/performance/dbpedia/common.ts b/performance/dbpedia/common.ts new file mode 100644 index 0000000..ec1ddb2 --- /dev/null +++ b/performance/dbpedia/common.ts @@ -0,0 +1,40 @@ +import { type Options, type QueryContext, QueryEngine } from "ldkit"; +import { createNamespace } from "ldkit/namespaces"; + +export { createLens } from "ldkit"; +export { dbo } from "ldkit/namespaces"; + +export const dbp = createNamespace( + { + iri: "http://dbpedia.org/property/", + prefix: "dbp:", + terms: [ + "title", + "author", + "country", + "language", + "genre", + ], + } as const, +); + +class LoggingQueryEngine extends QueryEngine { + override async query( + body: string, + requestType: "application/sparql-query" | "application/sparql-update", + // deno-lint-ignore no-explicit-any + responseType: any, + context?: QueryContext, + ) { + console.log("Query: ", body); + console.time("DBpedia first response time"); + const result = await super.query(body, requestType, responseType, context); + console.timeEnd("DBpedia first response time"); + return result; + } +} + +export const options: Options = { + engine: new LoggingQueryEngine(), + sources: ["https://dbpedia.org/sparql"], // SPARQL endpoint +}; diff --git a/performance/dbpedia/deno.json b/performance/dbpedia/deno.json new file mode 100644 index 0000000..6c27c6b --- /dev/null +++ b/performance/dbpedia/deno.json @@ -0,0 +1,12 @@ +{ + "imports": { + "ldkit": "../../mod.ts", + "ldkit/namespaces": "../../namespaces.ts" + }, + "tasks": { + "scenario1": "deno run --allow-net --allow-env ./scenario1.ts", + "scenario2": "deno run --allow-net --allow-env ./scenario2.ts", + "scenario3": "deno run --allow-net --allow-env ./scenario3.ts" + }, + "lock": false +} diff --git a/tests/performance/scenario1.ts b/performance/dbpedia/scenario1.ts similarity index 67% rename from tests/performance/scenario1.ts rename to performance/dbpedia/scenario1.ts index 67716bc..04892bd 100644 --- a/tests/performance/scenario1.ts +++ b/performance/dbpedia/scenario1.ts @@ -1,4 +1,4 @@ -import { context, createLens, dbo, dbp, engine } from "./common.ts"; +import { createLens, dbo, dbp, options } from "./common.ts"; // Create a schema export const BookSchema = { @@ -7,11 +7,11 @@ export const BookSchema = { } as const; // Create a resource using the data schema and context above -const Lens = createLens(BookSchema, context, engine); +const Lens = createLens(BookSchema, options); // Fetch 1000 resources console.time("LDkit total time"); -const entities = await Lens.find(undefined, 1000); +const entities = await Lens.find({ take: 1000 }); console.timeEnd("LDkit total time"); console.log("Number of results: ", entities.length); console.log("Sample entity", entities[0]); diff --git a/tests/performance/scenario2.ts b/performance/dbpedia/scenario2.ts similarity index 66% rename from tests/performance/scenario2.ts rename to performance/dbpedia/scenario2.ts index f45c921..e3bc7a4 100644 --- a/tests/performance/scenario2.ts +++ b/performance/dbpedia/scenario2.ts @@ -1,6 +1,4 @@ -import { context, createLens, dbo, dbp, engine } from "./common.ts"; - -import { BookSchema } from "./scenario1.ts"; +import { createLens, dbo, dbp, options } from "./common.ts"; // Create a schema export const PersonSchema = { @@ -9,7 +7,8 @@ export const PersonSchema = { } as const; export const ExtendedBookSchema = { - ...BookSchema, + "@type": dbo.Book, + title: dbp.title, author: { "@id": dbp.author, "@schema": PersonSchema, @@ -17,11 +16,11 @@ export const ExtendedBookSchema = { } as const; // Create a resource using the data schema and context above -const Lens = createLens(ExtendedBookSchema, context, engine); +const Lens = createLens(ExtendedBookSchema, options); // Fetch 1000 resources console.time("LDkit total time"); -const entities = await Lens.find(undefined, 1000); +const entities = await Lens.find({ take: 1000 }); console.timeEnd("LDkit total time"); console.log("Number of results: ", entities.length); console.log("Sample entity", entities[0]); diff --git a/performance/dbpedia/scenario3.ts b/performance/dbpedia/scenario3.ts new file mode 100644 index 0000000..c4a4792 --- /dev/null +++ b/performance/dbpedia/scenario3.ts @@ -0,0 +1,31 @@ +import { createLens, dbo, dbp, options } from "./common.ts"; + +// Note: This test will likely fail because of DBpedia rate limiting + +// Create a schema +export const PersonSchema = { + "@type": dbo.Person, + name: dbo.birthName, +} as const; + +const DoubleExtendedBookSchema = { + "@type": dbo.Book, + title: dbp.title, + author: { + "@id": dbp.author, + "@schema": PersonSchema, + }, + country: dbp.country, + language: dbp.language, + genre: dbo.literaryGenre, +} as const; + +// Create a resource using the data schema and context above +const Lens = createLens(DoubleExtendedBookSchema, options); + +// Fetch 1000 resources +console.time("LDkit total time"); +const entities = await Lens.find({ take: 1000 }); +console.timeEnd("LDkit total time"); +console.log("Number of results: ", entities.length); +console.log("Sample entity", entities[0]); diff --git a/tests/performance/common.ts b/tests/performance/common.ts deleted file mode 100644 index 1830f2c..0000000 --- a/tests/performance/common.ts +++ /dev/null @@ -1,41 +0,0 @@ -import { type QueryContext, QueryEngine } from "../../mod.ts"; -import { createNamespace } from "../../namespaces.ts"; - -export { createLens } from "../../mod.ts"; -export { dbo } from "../../namespaces.ts"; - -export const dbp = createNamespace( - { - iri: "http://dbpedia.org/property/", - prefix: "dbp:", - terms: [ - "title", - "author", - "country", - "language", - "genre", - ], - } as const, -); - -class LoggingQueryEngine extends QueryEngine { - async query( - query: string, - // deno-lint-ignore no-explicit-any - responseType: any, - context?: QueryContext, - ) { - console.log("Query: ", query); - console.time("Query time"); - const result = await super.query(query, responseType, context); - console.timeEnd("Query time"); - return result; - } -} - -export const engine = new LoggingQueryEngine(); - -// Create a context for query engine -export const context: QueryContext = { - sources: ["https://dbpedia.org/sparql"], // SPARQL endpoint -}; diff --git a/tests/performance/scenario3.ts b/tests/performance/scenario3.ts deleted file mode 100644 index 2412dfa..0000000 --- a/tests/performance/scenario3.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { context, createLens, dbp, engine } from "./common.ts"; - -import { ExtendedBookSchema } from "./scenario2.ts"; - -const DoubleExtendedBookSchema = { - ...ExtendedBookSchema, - country: dbp.country, - language: dbp.language, - genre: dbp.genre, -} as const; - -// Create a resource using the data schema and context above -const Lens = createLens(DoubleExtendedBookSchema, context, engine); - -// Fetch 1000 resources -console.time("LDkit total time"); -const entities = await Lens.find(undefined, 1000); -console.timeEnd("LDkit total time"); -console.log("Number of results: ", entities.length); -console.log("Sample entity", entities[0]); From 16a9a8f8966f9f2d8a8cc4433341465c207d233d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karel=20Kl=C3=ADma?= Date: Sun, 11 May 2025 13:19:30 +0200 Subject: [PATCH 2/4] Added performance benchmark scripts --- performance/benchmark/common.ts | 17 +++ performance/benchmark/deno.json | 10 ++ performance/benchmark/mocks.ts | 61 ++++++++++ performance/benchmark/populate.ts | 44 +++++++ performance/benchmark/test.ts | 194 ++++++++++++++++++++++++++++++ 5 files changed, 326 insertions(+) create mode 100644 performance/benchmark/common.ts create mode 100644 performance/benchmark/deno.json create mode 100644 performance/benchmark/mocks.ts create mode 100644 performance/benchmark/populate.ts create mode 100644 performance/benchmark/test.ts diff --git a/performance/benchmark/common.ts b/performance/benchmark/common.ts new file mode 100644 index 0000000..9951eeb --- /dev/null +++ b/performance/benchmark/common.ts @@ -0,0 +1,17 @@ +export const QUERY_SOURCE = "http://localhost:7200/repositories/bench"; +export const UPDATE_SOURCE = `${QUERY_SOURCE}/statements`; + +export async function directQuery( + body: string, +): Promise { + const response = await fetch(QUERY_SOURCE, { + method: "POST", + headers: { + "accept": "application/n-triples", + "content-type": `application/sparql-query; charset=UTF-8`, + }, + body, + }); + const text = await response.text(); + return text; +} diff --git a/performance/benchmark/deno.json b/performance/benchmark/deno.json new file mode 100644 index 0000000..377d772 --- /dev/null +++ b/performance/benchmark/deno.json @@ -0,0 +1,10 @@ +{ + "imports": { + "ldkit": "../../mod.ts" + }, + "tasks": { + "test": "deno run --allow-net --allow-env --allow-write ./test.ts", + "populate": "deno run --allow-net --allow-env ./populate.ts" + }, + "lock": false +} diff --git a/performance/benchmark/mocks.ts b/performance/benchmark/mocks.ts new file mode 100644 index 0000000..8de65f9 --- /dev/null +++ b/performance/benchmark/mocks.ts @@ -0,0 +1,61 @@ +import { Schema } from "ldkit"; + +const x = (value: string) => `https://x/${value}`; + +const uuid = () => crypto.randomUUID(); + +const id = () => x(uuid()); + +export type Entity = { [key: string]: string | string[] | Entity }; + +export function createEntity( + simpleProperties: number, + arrayProperties: number, + objectProperties: number, +) { + const entity: Entity = { + $id: id(), + }; + + for (let i = 1; i <= simpleProperties; i++) { + entity[`simpleProperty${i}`] = uuid(); + } + + for (let i = 1; i <= arrayProperties; i++) { + entity[`arrayProperty${i}`] = [uuid(), uuid(), uuid()]; + } + + for (let i = 1; i <= objectProperties; i++) { + entity[`objectProperty${i}`] = createEntity(3, 0, 0); + } + + return entity; +} + +export function createSchema( + simpleProperties: number, + arrayProperties: number, + objectProperties: number, + type: string = "Entity", +) { + const schema: Schema = { + "@type": x(type), + }; + + for (let i = 1; i <= simpleProperties; i++) { + schema[`simpleProperty${i}`] = x(`simpleProperty${i}`); + } + for (let i = 1; i <= arrayProperties; i++) { + schema[`arrayProperty${i}`] = { + "@id": x(`arrayProperty${i}`), + "@array": true, + }; + } + for (let i = 1; i <= objectProperties; i++) { + schema[`objectProperty${i}`] = { + "@id": x(`objectProperty${i}`), + "@schema": createSchema(3, 0, 0, `SubEntity`), + }; + } + return schema; +} diff --git a/performance/benchmark/populate.ts b/performance/benchmark/populate.ts new file mode 100644 index 0000000..e208323 --- /dev/null +++ b/performance/benchmark/populate.ts @@ -0,0 +1,44 @@ +import { createLens } from "ldkit"; +import { UPDATE_SOURCE } from "./common.ts"; +import { createEntity, createSchema, Entity } from "./mocks.ts"; + +console.log( + "This script will generate performance test data in the repository.", +); +console.log( + "It will create 10,000 entities with 10 simple properties, 10 array properties, and 10 object properties.", +); + +console.log(`Repository URL: ${UPDATE_SOURCE}`); + +const shouldProceed = confirm( + "Do you want to proceed?", +); + +if (!shouldProceed) { + console.log("Aborted."); + Deno.exit(0); +} + +console.log("Should proceed:", shouldProceed); + +const EntitySchema = createSchema(10, 10, 10); + +const UpdateEntities = createLens(EntitySchema, { + sources: [UPDATE_SOURCE], +}); + +const iterations = 10; +for (let x = 1; x <= iterations; x++) { + console.log(`Creating 1000 entities, iteration ${x}/${iterations}`); + console.log("Generating entities..."); + const entities: Entity[] = []; + for (let i = 0; i < 1000; i++) { + entities.push(createEntity(10, 10, 10)); + } + console.log("Inserting entities..."); + // deno-lint-ignore no-explicit-any + await UpdateEntities.insert(...entities as any); +} + +console.log("DONE"); diff --git a/performance/benchmark/test.ts b/performance/benchmark/test.ts new file mode 100644 index 0000000..a43e7ec --- /dev/null +++ b/performance/benchmark/test.ts @@ -0,0 +1,194 @@ +import { createLens, type Schema } from "ldkit"; + +import { directQuery, QUERY_SOURCE } from "./common.ts"; +import { createSchema } from "./mocks.ts"; + +console.log( + "This script will execute performance test against the repository.", +); +console.log( + "Make sure the repository is populated with the data before running this script.", +); + +console.log(`Repository URL: ${QUERY_SOURCE}`); + +const shouldProceed = confirm( + "Do you want to proceed?", +); + +if (!shouldProceed) { + console.log("Aborted."); + Deno.exit(0); +} + +type TestSetup = [number, number, number]; + +type TestResult = { + ok: boolean; + ldkit: number; + query: number; +}; + +const results: Record = {}; + +console.log("Running simple properties test..."); +await test([1, 0, 0], [1, 0, 0]); + +console.log("Running object properties test..."); +await test([0, 0, 1], [0, 0, 1]); + +console.log("Running array properties test..."); +await test([0, 1, 0], [0, 1, 0]); + +console.log("Running simple and object properties test..."); +await test([1, 0, 1], [1, 0, 1]); + +console.log("Running simple and object and array mixed properties test..."); +await test([10, 1, 10], [0, 1, 0]); + +console.log("RESULTS", results); +Deno.writeTextFileSync("results.json", JSON.stringify(results, null, 2)); +console.log("DONE"); +console.log("Results saved to results.json"); + +async function test( + initialSetup: TestSetup, + increment: TestSetup, +): Promise { + let setup = initialSetup; + let skipRest = false; + for (let i = 0; i < 10; i++) { + const key = `${setup[0]} ${setup[1]} ${setup[2]}`; + + if (!skipRest) { + console.log(`Testing ${key}`); + const schema = createSchema(setup[0], setup[1], setup[2]); + const res = await testSchema(schema); + results[key] = res; + if (!res.ok || res.query > 10000) { + skipRest = true; + } + } else { + console.log(`Skipping ${key} due to maximum timeout reached`); + results[key] = { + ok: false, + ldkit: -1, + query: -1, + }; + } + + setup = [ + setup[0] + increment[0], + setup[1] + increment[1], + setup[2] + increment[2], + ]; + } +} + +/** + * Calculate the average of an array of numbers, excluding the max and min values. + * This is useful for benchmarking to avoid outliers. + * @param arr values to calculate the average of + * @returns average of the array + */ +function avg(arr: number[]): number { + // Remove the max and min values from the array + const max = Math.max(...arr); + const min = Math.min(...arr); + const filtered = arr.filter((v) => v !== max && v !== min); + return filtered.reduce((acc, val) => acc + val, 0) / filtered.length; +} + +/** + * This function will execute the performance test against the repository + * and return the average times for ldkit and query. + * It will run 10 iterations of the test, querying 1000 entities each time. + * @param schema schema to test + * @returns test result with average times for ldkit and query + */ +async function testSchema(schema: Schema): Promise { + const results: TestResult[] = []; + for (let i = 0; i < 10; i++) { + const result = await testQuery(schema, 1000, i * 1000); + results.push(result); + } + + if (results.find((r) => !r.ok)) { + return { + ok: false, + ldkit: -1, + query: -1, + }; + } + + return { + ok: true, + ldkit: avg(results.map((r) => r.ldkit)), + query: avg(results.map((r) => r.query)), + }; +} + +/** + * This function will execute the performance test against the repository + * and return the times for ldkit and query. + * It will query entities using LDkit and directly using the query string. + * @param schema schema to test + * @param take number of entities to take + * @param skip number of entities to skip + * @returns test result with ldkit and query times + */ +async function testQuery( + schema: Schema, + take: number, + skip: number, +): Promise { + performance.clearMarks(); + performance.clearMeasures(); + let query = ""; + let LDkitOK = false; + let QueryOK = false; + + const Entities = createLens(schema, { + sources: [QUERY_SOURCE], + logQuery: (q: string) => { + query = q; + }, + }); + + try { + performance.mark("ldkit-start"); + await Entities.find({ take, skip }); + performance.mark("ldkit-end"); + performance.measure("ldkit", "ldkit-start", "ldkit-end"); + LDkitOK = true; + } catch (e) { + console.error("Error", e); + } + + try { + performance.mark("query-start"); + await directQuery(query); + performance.mark("query-end"); + performance.measure("query", "query-start", "query-end"); + QueryOK = true; + } catch (e) { + console.error("Error", e); + } + + if (!LDkitOK || !QueryOK) { + return { + ok: false, + ldkit: -1, + query: -1, + }; + } + + const ldkitMeasure = performance.getEntriesByName("ldkit")[0]; + const queryMeasure = performance.getEntriesByName("query")[0]; + + return { + ok: true, + ldkit: ldkitMeasure.duration, + query: queryMeasure.duration, + }; +} From 0b67367bffcc620b5e782529258ef99de953b11d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karel=20Kl=C3=ADma?= Date: Sun, 11 May 2025 13:19:50 +0200 Subject: [PATCH 3/4] Added performance test results --- performance/benchmark/results.json | 252 +++++++++++++++++++++++++++++ 1 file changed, 252 insertions(+) create mode 100644 performance/benchmark/results.json diff --git a/performance/benchmark/results.json b/performance/benchmark/results.json new file mode 100644 index 0000000..cd583e2 --- /dev/null +++ b/performance/benchmark/results.json @@ -0,0 +1,252 @@ +{ + "1 0 0": { + "ok": true, + "ldkit": 125.3561125, + "query": 92.76656249999985 + }, + "2 0 0": { + "ok": true, + "ldkit": 146.30707500000028, + "query": 108.9394125 + }, + "3 0 0": { + "ok": true, + "ldkit": 165.2793125000003, + "query": 137.64444999999978 + }, + "4 0 0": { + "ok": true, + "ldkit": 192.05189999999993, + "query": 174.5225375 + }, + "5 0 0": { + "ok": true, + "ldkit": 228.6790875000006, + "query": 208.39051249999966 + }, + "6 0 0": { + "ok": true, + "ldkit": 261.6477249999998, + "query": 237.08709999999974 + }, + "7 0 0": { + "ok": true, + "ldkit": 278.5881124999996, + "query": 268.1921624999991 + }, + "8 0 0": { + "ok": true, + "ldkit": 318.9909750000006, + "query": 288.76227500000095 + }, + "9 0 0": { + "ok": true, + "ldkit": 346.7820750000001, + "query": 315.7693374999999 + }, + "10 0 0": { + "ok": true, + "ldkit": 373.42453749999913, + "query": 338.20577499999945 + }, + "0 0 1": { + "ok": true, + "ldkit": 266.2802124999989, + "query": 238.46059999999943 + }, + "0 0 2": { + "ok": true, + "ldkit": 362.11783749999995, + "query": 358.16975 + }, + "0 0 3": { + "ok": true, + "ldkit": 496.5346999999965, + "query": 495.09622500000023 + }, + "0 0 4": { + "ok": true, + "ldkit": 658.0126875000005, + "query": 617.3534624999993 + }, + "0 0 5": { + "ok": true, + "ldkit": 768.0673125000012, + "query": 739.1817625000003 + }, + "0 0 6": { + "ok": true, + "ldkit": 915.2503749999996, + "query": 865.769774999997 + }, + "0 0 7": { + "ok": true, + "ldkit": 1057.093037499997, + "query": 1007.5307499999963 + }, + "0 0 8": { + "ok": true, + "ldkit": 1207.0080624999973, + "query": 1159.2273000000023 + }, + "0 0 9": { + "ok": true, + "ldkit": 1354.0509250000068, + "query": 1263.027837499998 + }, + "0 0 10": { + "ok": true, + "ldkit": 1498.5836499999969, + "query": 1414.8765875000063 + }, + "0 1 0": { + "ok": true, + "ldkit": 175.94309999999678, + "query": 163.17731249999633 + }, + "0 2 0": { + "ok": true, + "ldkit": 330.8431999999957, + "query": 324.65451250000115 + }, + "0 3 0": { + "ok": true, + "ldkit": 588.3618624999981, + "query": 568.2088499999991 + }, + "0 4 0": { + "ok": true, + "ldkit": 1281.1556500000079, + "query": 1252.358612499993 + }, + "0 5 0": { + "ok": true, + "ldkit": 3134.912574999995, + "query": 3106.9908625000025 + }, + "0 6 0": { + "ok": true, + "ldkit": 9097.18801249998, + "query": 9104.659262500005 + }, + "0 7 0": { + "ok": false, + "ldkit": -1, + "query": -1 + }, + "0 8 0": { + "ok": false, + "ldkit": -1, + "query": -1 + }, + "0 9 0": { + "ok": false, + "ldkit": -1, + "query": -1 + }, + "0 10 0": { + "ok": false, + "ldkit": -1, + "query": -1 + }, + "1 0 1": { + "ok": true, + "ldkit": 306.6827624999569, + "query": 256.7828249999875 + }, + "2 0 2": { + "ok": true, + "ldkit": 468.5906249999971, + "query": 429.0295000000042 + }, + "3 0 3": { + "ok": true, + "ldkit": 648.0601374999824, + "query": 625.3284750000021 + }, + "4 0 4": { + "ok": true, + "ldkit": 842.2756999999838, + "query": 792.0479374999995 + }, + "5 0 5": { + "ok": true, + "ldkit": 1004.1485625000059, + "query": 946.4888625000167 + }, + "6 0 6": { + "ok": true, + "ldkit": 1193.7576999999874, + "query": 1132.2487874999933 + }, + "7 0 7": { + "ok": true, + "ldkit": 1382.4604125000187, + "query": 1298.9434125000116 + }, + "8 0 8": { + "ok": true, + "ldkit": 1615.4087, + "query": 1476.0199125000072 + }, + "9 0 9": { + "ok": true, + "ldkit": 1747.911737500006, + "query": 1638.4784249999648 + }, + "10 0 10": { + "ok": true, + "ldkit": 1947.042775000009, + "query": 1815.4263249999785 + }, + "10 1 10": { + "ok": true, + "ldkit": 2769.823974999992, + "query": 2624.2905500000197 + }, + "10 2 10": { + "ok": true, + "ldkit": 3183.08308750001, + "query": 3015.684212500011 + }, + "10 3 10": { + "ok": true, + "ldkit": 4254.884612499998, + "query": 4137.4510375000245 + }, + "10 4 10": { + "ok": true, + "ldkit": 7391.460950000008, + "query": 7208.183950000064 + }, + "10 5 10": { + "ok": false, + "ldkit": -1, + "query": -1 + }, + "10 6 10": { + "ok": false, + "ldkit": -1, + "query": -1 + }, + "10 7 10": { + "ok": false, + "ldkit": -1, + "query": -1 + }, + "10 8 10": { + "ok": false, + "ldkit": -1, + "query": -1 + }, + "10 9 10": { + "ok": false, + "ldkit": -1, + "query": -1 + }, + "10 10 10": { + "ok": false, + "ldkit": -1, + "query": -1 + } +} \ No newline at end of file From 788598f2acba26206648a9b9e1dcd24c302d6ed4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karel=20Kl=C3=ADma?= Date: Sun, 11 May 2025 13:36:07 +0200 Subject: [PATCH 4/4] Added README --- performance/benchmark/README.md | 64 +++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 performance/benchmark/README.md diff --git a/performance/benchmark/README.md b/performance/benchmark/README.md new file mode 100644 index 0000000..d91be78 --- /dev/null +++ b/performance/benchmark/README.md @@ -0,0 +1,64 @@ +# Performance benchmark + +This test suite runs a series of performance tests against local triplestore and +publishes results into the `results.json` file. + +A test specification consists of a key with three numbers, e.g. `10 0 5`, which +represent the complexity of LDkit schema that is being used for the performance +test. The first number represents the number of simple properties in the schema, +the second represents a number of array properties (each array has three +elements), and the third represents a number of object properties in the schema +(each object has three properties on its own). For example, the `10 0 5` test +includes a LDkit schema with 10 simple properties and 5 object properties, +having 15 properties in total. + +Each test case measures the execution time of querying 1000 entities according +to the schema for both LDkit and direct query of the triplestore, so that the +two times can be compared. + +## Test results + +The rest results contained in the `results.json` show average execution time for +various test cases. The test was performed on a PC with Intel Core i7-5500U CPU +@ 2.40 GHz and 8 GB RAM running Windows 10. Local installation of GraphDB +version 11.0 was used as the triplestore. + +## Running the benchmark + +Deno and GraphDB are required to run the test scenarios. Instead of GraphDB, +another triplestore may be used. + +### 1) Install Deno + +Windows: + +``` +irm https://deno.land/install.ps1 | iex +``` + +MacOS/Linux: + +``` +curl -fsSL https://deno.land/install.sh | sh +``` + +### 2) Install GraphDB + +[GraphDB Desktop installation](https://graphdb.ontotext.com/documentation/11.0/graphdb-desktop-installation.html) + +### 3) Populate the triplestore with data + +``` +deno task populate +``` + +The script will create 10000 entities that will be used for the performance +test. + +### 4) Run the performance tests + +``` +deno task test +``` + +The results of the test are saved into `results.json` file.