8000 feat(forward-headers): handle forwarding of generic headers and cooki… · algolia/renderscript@af45367 · GitHub
[go: up one dir, main page]
More Web Proxy on the site http://driver.im/
Skip to content

Commit af45367

Browse files
JonathanMontaneJonathanMontane
authored
feat(forward-headers): handle forwarding of generic headers and cookie (#14)
* feat(forward-headers): handle forwarding of generic headers and cookie * feat(forward-headers): restrict header forwarding to nav requests and use named params * chore(best-practices): remove unnecessary console.log
1 parent a4702a3 commit af45367

File tree

3 files changed

+80
-20
lines changed

3 files changed
+80
-20
lines changed

‎README.md

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -59,14 +59,16 @@ open http://localhost:3000/render?url=https%3A%2F%2Fwww.algolia.com
5959

6060
See `.env.prod` to see which ones are installed by default (they still need to be provided to `docker run` to be enabled).
6161

62-
- `ADBLOCK_LISTS`: Comma separated list of adblocking lists download link
62+
- `ADBLOCK_LISTS`: Comma-separated list of adblocking lists download link
6363
Example: `https://easylist.to/easylist/easylist.txt`
64-
- `EXTENSIONS`: Comma separated list of extensions download link (expects a `.zip` file)
64+
- `EXTENSIONS`: Comma-separated list of extensions download link (expects a `.zip` file)
6565
Example: `https://github.com/gorhill/uBlock/releases/download/1.19.6/uBlock0_1.19.6.chromium.zip`
6666
- `ALLOW_LOCALHOST`: Allow calls on localhost IPs
6767
Example: `ALLOW_LOCALHOST=true`
68-
- `IP_PREFIXES_WHITELIST`: a comma-separated list of prefixes to whitelist when `ALLOW_LOCALHOST` is set to true.
68+
- `IP_PREFIXES_WHITELIST`: Comma-separated list of prefixes to whitelist when `ALLOW_LOCALHOST` is set to true.
6969
Example: `IP_PREFIXES_WHITELIST=127.,0.,::1` (these are the default values used when the variable is not provided alongside `ALLOW_LOCALHOST`)
70+
- `HEADERS_TO_FORWARD`: Comma-separated list of headers to forward on navigation request
71+
Example: `HEADERS_TO_FORWARD=Cookie,Authorization` (default value)
7072

7173
## Credits
7274

‎src/api/routes/render.ts

Lines changed: 21 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,22 @@ import renderer from "lib/rendererSingleton";
44
import { badRequest } from "api/helpers/errors";
55
import buildUrl from 'api/helpers/buildUrl';
66

7+
const HEADERS_TO_FORWARD = process.env.HEADERS_TO_FORWARD
8+
? process.env.HEADERS_TO_FORWARD.split(',')
9+
: ['Cookie', 'Authorization']
10+
11+
export function getForwardedHeadersFromRequest(req: express.Request) {
12+
const headersToForward = HEADERS_TO_FORWARD.reduce((partial, headerName) => {
13+
const name = headerName.toLowerCase();
14+
if (req.headers[name]) {
15+
return { ...partial, [name]: req.headers[name] }
16+
}
17+
return partial;
18+
}, {});
19+
20+
return headersToForward;
21+
}
22+
723
export function getURLFromQuery(
824
req: express.Request,
925
res: express.Response,
@@ -61,8 +77,10 @@ export async function render(
6177
next: express.NextFunction
6278
) {
6379
const { url } = res.locals;
80+
const headersToForward = getForwardedHeadersFromRequest(req);
81+
6482
try {
65-
const { error, statusCode, body } = await renderer.task({ url });
83+
const { error, statusCode, body } = await renderer.task({ url, headersToForward });
6684
if (error) {
6785
res.status(400).json({ error });
6886
return;
@@ -95,8 +113,9 @@ export async function renderJSON(
95113
next: express.NextFunction
96114
) {
97115
const { url } = res.locals;
116+
const headersToForward = getForwardedHeadersFromRequest(req);
98117
try {
99-
const { error, statusCode, headers, body, timeout, resolvedUrl } = await renderer.task({ url });
118+
const { error, statusCode, headers, body, timeout, resolvedUrl } = await renderer.task({ url, headersToForward });
100119
if (error) {
101120
res.status(400).json({ error });
102121
return;

‎src/lib/Renderer.ts

Lines changed: 54 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,9 @@ const TIMEOUT = 10000;
2626

2727
export interface taskParams {
2828
url: URL;
29+
headersToForward: {
30+
[s: string]: string;
31+
}
2932
}
3033

3134
export interface taskResult {
@@ -264,22 +267,33 @@ class Renderer {
264267
await extensionsPage.close();
265268
}
266269

267-
private async _createNewPage() {
268-
if (this._stopping) {
269-
throw new Error("Called _createNewPage on a stopping Renderer");
270-
}
271-
272-
const browser = await this._getBrowser();
273-
const context = await browser.createIncognitoBrowserContext();
274-
const page = await context.newPage();
275-
276-
await page.setUserAgent("Algolia Crawler Renderscript");
277-
await page.setCacheEnabled(false);
278-
await page.setViewport({ width: WIDTH, height: HEIGHT });
270+
private async _defineRequestContextForPage({
271+
page,
272+
task
273+
}: {
274+
page: puppeteer.Page,
275+
task: taskParams
276+
}) {
277+
const { url, headersToForward } = task;
279278

280-
/* Ignore useless resources */
281279
await page.setRequestInterception(true);
282-
page.on("request", async req => {
280+
if (headersToForward.cookie) {
281+
const cookies = headersToForward.cookie.split('; ').map(c => {
282+
const [ key, ...v ] = c.split('=');
283+
// url attribute is required because it is not possible set cookies on a blank page
284+
// so page.setCookie would crash if no url is provided, since we start with a blank page
285+
return { url: url.href, name: key, value: v.join('=') };
286+
});
287+
try {
288+
await page.setCookie(...cookies)
289+
}
290+
catch (e) {
291+
console.error('failed to set cookie on page', url);
292+
}
293+
}
294+
295+
/* Ignore useless/dangerous resources */
296+
page.on('request', async (req: puppeteer.Request) => {
283297
// check for ssrf attempts
284298
try {
285299
await validateURL({
@@ -311,12 +325,34 @@ class Renderer {
311325
return;
312326
}
313327
// console.log(req.resourceType(), req.url());
328+
if (req.isNavigationRequest()) {
329+
const headers = req.headers();
330+
await req.continue({
331+
// headers ignore values set for `Cookie`, relies to page.setCookie instead
332+
headers: { ...headers, ...headersToForward }
333+
});
334+
return;
335+
}
314336
await req.continue();
315337
} catch (e) {
316338
if (!e.message.match(/Request is already handled/)) throw e;
317339
// Ignore Request is already handled error
318340
}
319341
});
342+
}
343+
344+
private async _createNewPage() {
345+
if (this._stopping) {
346+
throw new Error("Called _createNewPage on a stopping Renderer");
347+
}
348+
349+
const browser = await this._getBrowser();
350+
const context = await browser.createIncognitoBrowserContext();
351+
const page = await context.newPage();
352+
353+
await page.setUserAgent("Algolia Crawler Renderscript");
354+
await page.setCacheEnabled(false);
355+
await page.setViewport({ width: WIDTH, height: HEIGHT });
320356

321357
return { page, context };
322358
}
@@ -326,10 +362,13 @@ class Renderer {
326362
return await this._pageBuffer.shift()!;
327363
}
328364

329-
private async _processPage({ url }: taskParams, taskId: string) {
365+
private async _processPage(task: taskParams, taskId: string) {
330366
/* Setup */
367+
const { url } = task;
331368
const { context, page } = await this._newPage();
332369

370+
await this._defineRequestContextForPage({ page, task });
371+
333372
let response: puppeteer.Response | null = null;
334373
let timeout = false;
335374
page.addListener("response", (r: puppeteer.Response) => {

0 commit comments

Comments
 (0)
0