adobecom · milo-pr-merge · Jun 3, 2025 · Jun 2, 2025 · Jun 2, 2025 · Jun 2, 2025
diff --git a/.github/workflows/import/LOCAL_DEBUG_ENTRIES.js b/.github/workflows/import/LOCAL_DEBUG_ENTRIES.js
@@ -0,0 +1,6 @@
+export default [
+  // {
+  //  "path": "/africa/fragments/products/cards/headless-cms",
+  //  "route": "preview"
+  // },
+]
diff --git a/.github/workflows/import/constants.js b/.github/workflows/import/constants.js
@@ -0,0 +1,15 @@
+export const AEM_ORIGIN = 'https://admin.hlx.page';
+
+export const SUPPORTED_FILES = {
+  html: 'text/html',
+  jpeg: 'image/jpeg',
+  json: 'application/json',
+  jpg: 'image/jpeg',
+  png: 'image/png',
+  gif: 'image/gif',
+  mp4: 'video/mp4',
+  pdf: 'application/pdf',
+  svg: 'image/svg+xml',
+};
+
+export const DA_ORIGIN = 'https://admin.da.live';
diff --git a/.github/workflows/import/converters.js b/.github/workflows/import/converters.js
@@ -0,0 +1,168 @@
+import { unified } from 'unified';
+import remarkParse from 'remark-parse';
+import remarkGridTable from '@adobe/remark-gridtables';
+import { toHast as mdast2hast, defaultHandlers } from 'mdast-util-to-hast';
+import { raw } from 'hast-util-raw';
+import { mdast2hastGridTablesHandler } from '@adobe/mdast-util-gridtables';
+import { toHtml } from 'hast-util-to-html';
+
+import { JSDOM } from 'jsdom';
+
+function toBlockCSSClassNames(text) {
+  if (!text) return [];
+  const names = [];
+  const idx = text.lastIndexOf('(');
+  if (idx >= 0) {
+    names.push(text.substring(0, idx));
+    names.push(...text.substring(idx + 1).split(','));
+  } else {
+    names.push(text);
+  }
+
+  return names
+    .map((name) =>
+      name
+        .toLowerCase()
+        .replace(/[^0-9a-z]+/g, '-')
+        .replace(/^-+/, '')
+        .replace(/-+$/, '')
+    )
+    .filter((name) => !!name);
+}
+
+function convertBlocks(dom) {
+  const tables = dom.window.document.querySelectorAll('body > table');
+
+  tables.forEach((table) => {
+    const rows = [
+      ...table.querySelectorAll(':scope > tbody > tr, :scope > thead > tr'),
+    ];
+    const nameRow = rows.shift();
+    const divs = rows.map((row) => {
+      const cols = row.querySelectorAll(':scope > td, :scope > th');
+      // eslint-disable-next-line no-shadow
+      const divs = [...cols].map((col) => {
+        const { innerHTML } = col;
+        const div = dom.window.document.createElement('div');
+        div.innerHTML = innerHTML;
+        return div;
+      });
+      const div = dom.window.document.createElement('div');
+      div.append(...divs);
+      return div;
+    });
+
+    const div = dom.window.document.createElement('div');
+    div.className = toBlockCSSClassNames(nameRow.textContent).join(' ');
+    div.append(...divs);
+    table.parentElement.replaceChild(div, table);
+  });
+}
+
+function makePictures(dom) {
+  const imgs = dom.window.document.querySelectorAll('img');
+  imgs.forEach((img) => {
+    const clone = img.cloneNode(true);
+    clone.setAttribute('loading', 'lazy');
+    // clone.src = `${clone.src}?optimize=medium`;
+
+    let pic = dom.window.document.createElement('picture');
+
+    const srcMobile = dom.window.document.createElement('source');
+    srcMobile.srcset = clone.src;
+
+    const srcTablet = dom.window.document.createElement('source');
+    srcTablet.srcset = clone.src;
+    srcTablet.media = '(min-width: 600px)';
+
+    pic.append(srcMobile, srcTablet, clone);
+
+    const hrefAttr = img.getAttribute('href');
+    if (hrefAttr) {
+      const a = dom.window.document.createElement('a');
+      a.href = hrefAttr;
+      const titleAttr = img.getAttribute('title');
+      if (titleAttr) {
+        a.title = titleAttr;
+      }
+      a.append(pic);
+      pic = a;
+    }
+
+    // Determine what to replace
+    const imgParent = img.parentElement;
+    const imgGrandparent = imgParent.parentElement;
+    if (imgParent.nodeName === 'P' && imgGrandparent?.childElementCount === 1) {
+      imgGrandparent.replaceChild(pic, imgParent);
+    } else {
+      imgParent.replaceChild(pic, img);
+    }
+  });
+}
+
+function makeSections(dom) {
+  const children = dom.window.document.body.querySelectorAll(':scope > *');
+
+  const section = dom.window.document.createElement('div');
+  const sections = [...children].reduce(
+    (acc, child) => {
+      if (child.nodeName === 'HR') {
+        child.remove();
+        acc.push(dom.window.document.createElement('div'));
+      } else {
+        acc[acc.length - 1].append(child);
+      }
+      return acc;
+    },
+    [section]
+  );
+
+  dom.window.document.body.append(...sections);
+}
+
+// Generic docs have table blocks and HRs, but not ProseMirror decorations
+export function docDomToAemHtml(dom) {
+  convertBlocks(dom);
+  makePictures(dom);
+  makeSections(dom);
+
+  return dom.window.document.body.innerHTML;
+}
+
+function makeHast(mdast) {
+  const handlers = {
+    ...defaultHandlers,
+    gridTable: mdast2hastGridTablesHandler(),
+  };
+  const hast = mdast2hast(mdast, { handlers, allowDangerousHtml: true });
+  return raw(hast);
+}
+
+function removeImageSizeHash(dom) {
+  const imgs = dom.window.document.querySelectorAll('[src*="#width"]');
+  imgs.forEach((img) => {
+    img.setAttribute('src', img.src.split('#width')[0]);
+  });
+}
+
+export function mdToDocDom(md) {
+  // convert linebreaks
+  const converted = md.replace(/(\r\n|\n|\r)/gm, '\n');
+
+  // convert to mdast
+  const mdast = unified()
+    .use(remarkParse)
+    .use(remarkGridTable)
+    .parse(converted);
+
+  const hast = makeHast(mdast);
+
+  let htmlText = toHtml(hast);
+  htmlText = htmlText.replaceAll('.hlx.page', '.hlx.live');
+  htmlText = htmlText.replaceAll('.aem.page', '.aem.live');
+
+  const dom = new JSDOM(htmlText);
+  removeImageSizeHash(dom);
+
+  return dom;
+}
diff --git a/.github/workflows/import/daFetch.js b/.github/workflows/import/daFetch.js
@@ -0,0 +1,107 @@
+import { DA_ORIGIN } from './constants.js';
+
+let token;
+export async function getImsToken() {
+  console.log('Fetching IMS token');
+  const params = new URLSearchParams();
+  params.append('client_id', process.env.ROLLING_IMPORT_CLIENT_ID);
+  params.append('client_secret', process.env.ROLLING_IMPORT_CLIENT_SECRET);
+  params.append('code', process.env.ROLLING_IMPORT_CODE);
+  params.append('grant_type', process.env.ROLLING_IMPORT_GRANT_TYPE);
+
+  const response = await fetch(process.env.ROLLING_IMPORT_IMS_URL, {
+    method: 'POST',
+    headers: {
+      'Content-Type': 'application/x-www-form-urlencoded',
+    },
+    body: params,
+  });
+
+  if (!response.ok) {
+    throw new Error('Failed to retrieve IMS token');
+  }
+
+  const data = await response.json();
+  token = data.access_token;
+  console.log('Fetched IMS token');
+}
+
+export const daFetch = async (url, opts = {}) => {
+  opts.headers ||= {};
+  opts.headers.Authorization = `Bearer ${token}`;
+  const resp = await fetch(url, opts);
+  if (!resp.ok) throw new Error('DA import failed');
+  return resp;
+};
+
+export function replaceHtml(text, fromOrg, fromRepo) {
+  let inner = text;
+  if (fromOrg && fromRepo) {
+    const fromOrigin = `https://main--${fromRepo}--${fromOrg}.aem.live`;
+    inner = text
+      .replaceAll('./media', `${fromOrigin}/media`)
+      .replaceAll('href="/', `href="${fromOrigin}/`);
+  }
+
+  return `
+    <body>
+      <header></header>
+      <main>${inner}</main>
+      <footer></footer>
+    </body>
+  `;
+}
+
+export async function saveToDa(text, url) {
+  const daPath = `/${url.org}/${url.repo}${url.pathname}`;
+  const daHref = `https://da.live/edit#${daPath}`;
+  const { org, repo } = url;
+
+  const body = replaceHtml(text, org, repo);
+
+  const blob = new Blob([body], { type: 'text/html' });
+  const formData = new FormData();
+  formData.append('data', blob);
+  const opts = { method: 'PUT', body: formData };
+  try {
+    const daResp = await daFetch(`${DA_ORIGIN}/source${daPath}.html`, opts);
+    return { daHref, daStatus: daResp.status, daResp, ok: daResp.ok };
+  } catch (e) {
+    console.log(`Couldn't save ${url.daUrl} `);
+    throw e;
+  }
+}
+
+function getBlob(url, content) {
+  const body =
+    url.type === 'json'
+      ? content
+      : replaceHtml(content, url.fromOrg, url.fromRepo);
+
+  const type = url.type === 'json' ? 'application/json' : 'text/html';
+
+  return new Blob([body], { type });
+}
+
+export async function saveAllToDa(url, content) {
+  const { toOrg, toRepo, destPath, editPath, type } = url;
+
+  const route = type === 'json' ? '/sheet' : '/edit';
+  url.daHref = `https://da.live${route}#/${toOrg}/${toRepo}${editPath}`;
+
+  const blob = getBlob(url, content);
+  const body = new FormData();
+  body.append('data', blob);
+  const opts = { method: 'PUT', body };
+
+  try {
+    const resp = await daFetch(
+      `${DA_ORIGIN}/source/${toOrg}/${toRepo}${destPath}`,
+      opts
+    );
+    return resp.status;
+  } catch {
+    console.log(`Couldn't save ${destPath}`);
+    return 500;
+  }
+}