Feature/sync api

package.json

@@ -50,6 +50,7 @@
     "lerna": "^4.0.0",
     "lint-staged": "^12.1.2",
     "prettier": "^2.4.1",
+    "supertest": "^6.2.3",
     "ts-jest": "^27.0.7",
     "ts-node": "^10.4.0",
     "typescript": "^4.5.2"

packages/contentful-ssg/README.md

@@ -47,7 +47,7 @@ npx cssg init --typescript
 | spaceId            | `string`                                                              | `undefined`   | Contentful Space id                                                                                                                                                                                                                                                                                                                                                                                                                                                        |
 | environmentId      | `string`                                                              | `'master'`    | Contentful Environment id                                                                                                                                                                                                                                                                                                                                                                                                                                                  |
 | format             | `string`\|`function`\|`object`                                        | `'yaml'`      | File format ( `yaml`, `toml`, `md`, `json`) You can add a function returning the format or you can add a mapping object like `{yaml: [glob pattern]}` ([pattern](https://github.com/micromatch/micromatch) should match the directory)                                                                                                                                                                                                                                     |
-| plugins            | `[string]`\|`[[string, options]]`\|`[{resolve:'string', options:{}}]` | `[]`          | Add plugins to contentful-ssg. See [Plugins](#plugins) |
+| plugins            | `[string]`\|`[[string, options]]`\|`[{resolve:'string', options:{}}]` | `[]`          | Add plugins to contentful-ssg. See [Plugins](#plugins)                                                                                                                                                                                                                                                                                                                                                                                                                     |
 | directory          | `string`                                                              | `'./content'` | Base directory for content files.                                                                                                                                                                                                                                                                                                                                                                                                                                          |
 | validate           | `function`                                                            | `undefined`   | Pass `function(transformContext, runtimeContext){...}` to validate an entry. Return `false` to skip the entry completely. Without a validate function entries with a missing required field are skipped.                                                                                                                                                                                                                                                                   |
 | transform          | `function`                                                            | `undefined`   | Pass `function(transformContext, runtimeContext){...}` to modify the stored object. Return `undefined` to skip the entry completely. (no file will be written)                                                                                                                                                                                                                                                                                                             |
@@ -71,13 +71,15 @@ plugins: ['my-plugin-package', './plugins/my-local-plugin]
 All plugins can have options specified by wrapping the name and an options object in an array inside your config or by using a more verbose object notation.
 
 For specifying no options, these are all equivalent:
+
 ```js
 {
   "plugins": ["my-plugin", ["my-plugin"], ["my-plugin", {}], {resolve: "my-plugin", options: {}}]
 }
 ```
 
 To specify an option, pass an object with the keys as the option names.
+
 ```js
 {
   "plugins": [
@@ -225,7 +227,6 @@ itself is waiting for the current entry to be transformed.
 
       // Do something usefull with the transformed data
       // which you can't do with context.entryMap.get('<contentful-id>')
-
     } catch (error) {
       // Entry isn't available, the transform method for the entry throws an error
       // or we encountered a cyclic dependency
@@ -245,10 +246,25 @@ npx cssg fetch
 ```
 
 To see all available command line options call
+
 ```bash
 npx cssg help fetch
 ```
 
+### watch
+
+Same as `fetch` but also starts a webserver listening for changes and registers a contentful webhook
+
+```bash
+npx cssg watch
+```
+
+To see all available command line options call
+
+```bash
+npx cssg help watch
+```
+
 ## Example configuration
 
 ### Grow

packages/contentful-ssg/package.json

@@ -105,6 +105,7 @@
     "@endemolshinegroup/cosmiconfig-typescript-loader": "^3.0.2",
     "@iarna/toml": "^2.2.5",
     "@swc-node/register": "^1.4.0",
+    "async-exit-hook": "^2.0.1",
     "chalk": "^4.1.2",
     "commander": "^8.3.0",
     "contentful": "^9.1.4",
@@ -114,25 +115,30 @@
     "dotenv": "^10.0.0",
     "dotenv-expand": "^5.1.0",
     "esbuild": "^0.13.13",
+    "express": "^4.18.1",
     "find-up": "^6.2.0",
     "fs-extra": "^10.0.0",
+    "get-port": "^6.1.2",
     "globby": "^12.0.2",
     "gray-matter": "^4.0.3",
     "inquirer": "^8.2.0",
     "js-yaml": "^4.1.0",
     "listr": "^0.14.3",
     "merge-options": "^3.0.4",
     "micromatch": "^4.0.4",
+    "ngrok": "^4.3.1",
     "prettier": "^2.4.1",
     "read-pkg-up": "^9.0.0",
     "rxjs": "^7.5.5",
     "slash": "^4.0.0",
     "snake-case": "^3.0.4",
     "tempy": "^2.0.0",
-    "type-fest": "^2.5.3"
+    "type-fest": "^2.5.3",
+    "uuid": "^8.3.2"
   },
   "devDependencies": {
     "@types/dlv": "^1.1.2",
+    "@types/express": "^4.17.13",
     "@types/fs-extra": "^9.0.13",
     "@types/inquirer": "^8.1.3",
     "@types/jest": "^27.0.3",
@@ -151,6 +157,7 @@
     "eslint-plugin-prettier": "^4.0.0",
     "jest": "^27.3.0",
     "rimraf": "^3.0.2",
+    "supertest": "^6.2.3",
     "ts-jest": "^27.0.7",
     "ts-node": "^10.4.0",
     "typescript": "^4.5.2"

packages/contentful-ssg/src/__test__/mock.ts

@@ -8,7 +8,15 @@ import {
 } from '../lib/contentful.js';
 import { FileManager } from '../lib/file-manager.js';
 import { HookManager } from '../lib/hook-manager.js';
-import type { Config, RuntimeContext, TransformContext } from '../types.js';
+import type {
+  ContentType,
+  Locale,
+  Asset,
+  Entry,
+  Config,
+  RuntimeContext,
+  TransformContext,
+} from '../types.js';
 
 const cache = new Map();
 
@@ -31,10 +39,10 @@ export const readFixtureSync = (file) => {
 };
 
 export const getContent = async () => {
-  const assets = await readFixture('assets.json');
-  const entries = await readFixture('entries.json');
-  const locales = await readFixture('locales.json');
-  const contentTypes = await readFixture('content_types.json');
+  const assets = (await readFixture('assets.json')) as Asset[];
+  const entries = (await readFixture('entries.json')) as Entry[];
+  const locales = (await readFixture('locales.json')) as Locale[];
+  const contentTypes = (await readFixture('content_types.json')) as ContentType[];
 
   const [entry] = entries;
   const [asset] = assets;
@@ -54,7 +62,21 @@ export const getContent = async () => {
     },
   };
 
-  return { entries, assets, contentTypes, locales, assetLink, entryLink, entry, asset };
+  const assetMap = new Map(assets.map((asset) => [asset.sys.id, asset]));
+  const entryMap = new Map(entries.map((entry) => [entry.sys.id, entry]));
+
+  return {
+    entries,
+    assets,
+    contentTypes,
+    locales,
+    assetLink,
+    entryLink,
+    entry,
+    asset,
+    assetMap,
+    entryMap,
+  };
 };
 
 export const getConfig = (fixture: Partial<Config> = {}): Config => ({
@@ -64,10 +86,10 @@ export const getConfig = (fixture: Partial<Config> = {}): Config => ({
 });
 
 export const getRuntimeContext = (fixture: Partial<RuntimeContext> = {}): RuntimeContext => {
-  const assets = readFixtureSync('assets.json');
-  const entries = readFixtureSync('entries.json');
-  const locales = readFixtureSync('locales.json');
-  const contentTypes = readFixtureSync('content_types.json');
+  const assets = readFixtureSync('assets.json') as Asset[];
+  const entries = readFixtureSync('entries.json') as Entry[];
+  const locales = readFixtureSync('locales.json') as Locale[];
+  const contentTypes = readFixtureSync('content_types.json') as ContentType[];
 
   const fieldSettings = getFieldSettings(contentTypes);
   const { code: defaultLocale } = locales.find((locale) => locale.default) || locales[0];

packages/contentful-ssg/src/cli.ts

@@ -1,8 +1,12 @@
 #!/usr/bin/env node
+/* eslint-disable @typescript-eslint/no-unsafe-call */
 
 /* eslint-env node */
 import path from 'path';
 import chalk from 'chalk';
+import ngrok from 'ngrok';
+import getPort from 'get-port';
+import exitHook from 'async-exit-hook';
 import { existsSync } from 'fs';
 import { readFile } from 'fs/promises';
 import { outputFile } from 'fs-extra';
@@ -12,10 +16,12 @@ import dotenv from 'dotenv';
 import dotenvExpand from 'dotenv-expand';
 import { logError, confirm, askAll, askMissing } from './lib/ui.js';
 import { omitKeys } from './lib/object.js';
+import { getApp } from './server/index.js';
 
 import { getConfig, getEnvironmentConfig } from './lib/config.js';
 import { run } from './index.js';
 import { Config, ContentfulConfig } from './types.js';
+import { addWatchWebhook, resetSync } from './lib/contentful.js';
 
 const env = dotenv.config();
 dotenvExpand(env);
@@ -44,7 +50,7 @@ const errorHandler = (error: CommandError, silence: boolean) => {
 const actionRunner =
   (fn, log = true) =>
   (...args) =>
-    // eslint-disable-next-line @typescript-eslint/no-unsafe-return, @typescript-eslint/no-unsafe-call
+    // eslint-disable-next-line @typescript-eslint/no-unsafe-return
     fn(...args).catch((error) => errorHandler(error, !log));
 const program = new Command();
 program
@@ -151,11 +157,63 @@ program
   .option('--ignore-errors', 'No error return code when transform has errors')
   .action(
     actionRunner(async (cmd) => {
+      await resetSync();
       const config = await getConfig(parseFetchArgs(cmd || {}));
       const verified = await askMissing(config);
 
       return run(verified);
     })
   );
 
+program
+  .command('watch')
+  .description('Fetch content objects && watch for changes')
+  .option('-p, --preview', 'Fetch with preview mode')
+  .option('-v, --verbose', 'Verbose output')
+  .option('--url <url>', 'Url where the the server is reachable from the outside')
+  .option('--ignore-errors', 'No error return code when transform has errors')
+  .action(
+    actionRunner(async (cmd) => {
+      await resetSync();
+      const config = await getConfig(parseFetchArgs(cmd || {}));
+      const verified = await askMissing(config);
+
+      let prev = await run({ ...verified, sync: true });
+
+      let port = await getPort({ port: 1314 });
+      if (cmd.url) {
+        const url = new URL(cmd.url);
+        port = url.port || url.protocol === 'https:' ? 443 : 80;
+      }
+
+      const app = getApp(async () => {
+        prev = await run({ ...verified, sync: true }, prev);
+      });
+
+      const server = app.listen(port);
+
+      const stopServer = async () =>
+        new Promise((resolve, reject) => {
+          server.close((err) => {
+            if (err) {
+              reject(err);
+            } else {
+              resolve(true);
+            }
+          });
+        });
+
+      const url = (cmd.url as string) || (await ngrok.connect(port));
+      console.log(`\n  Listening for hooks on ${chalk.cyan(url)}\n`);
+      const webhook = await addWatchWebhook(verified as ContentfulConfig, url);
+
+      exitHook(async (cb) => {
+        await webhook.delete();
+        await stopServer();
+        await resetSync();
+        cb();
+      });
+    })
+  );
+
 program.parse(process.argv);

packages/contentful-ssg/src/index.test.ts

@@ -1,6 +1,8 @@
 import chalk from 'chalk';
-import { run } from './index.js';
+import { run, cleanupPrevData } from './index.js';
 import { write } from './tasks/write.js';
+import { RunResult, RuntimeContext } from './types.js';
+import { getContent } from './__test__/mock.js';
 
 jest.mock('./lib/contentful.js', () => {
   const originalModule = jest.requireActual('./lib/contentful.js');
@@ -144,4 +146,66 @@ describe('Run', () => {
     expect(mockExit).toBeCalledTimes(0);
     mockExit.mockRestore();
   });
+
+  test('does not fail on transform exception in sync mode', async () => {
+    console.log = jest.fn();
+    const mockExit = jest.spyOn(process, 'exit').mockImplementation((number) => {
+      throw new Error('process.exit: ' + number);
+    });
+
+    await run({
+      directory: 'test',
+      sync: true,
+      transform: async () => {
+        throw new Error();
+      },
+    });
+
+    expect(mockExit).toBeCalledTimes(0);
+    mockExit.mockRestore();
+  });
+
+  test('cleanupPrevData', async () => {
+    const mockData = await getContent();
+    const [entry] = mockData.entries;
+    const [asset] = mockData.assets;
+    const [locale] = mockData.locales;
+
+    const prev: RunResult = {
+      observables: {},
+      localized: {
+        [locale.code]: {
+          assets: mockData.assets,
+          entries: mockData.entries,
+          assetMap: mockData.assetMap,
+          entryMap: mockData.entryMap,
+        },
+      },
+    };
+
+    const deletedEntry = { sys: { id: entry.sys.id, type: 'DeletedEntry' } };
+    const deletedAsset = { sys: { id: asset.sys.id, type: 'DeletedAsset' } };
+
+    const context: RuntimeContext = {
+      defaultLocale: locale.code,
+      data: {
+        deletedEntries: [deletedEntry],
+        deletedAssets: [deletedAsset],
+        locales: [locale],
+      },
+    } as RuntimeContext;
+
+    cleanupPrevData(context, prev);
+
+    for (let node of context.data.deletedEntries) {
+      expect(Object.keys(node)).toEqual(expect.arrayContaining(['sys', 'fields']));
+      expect(Object.keys(node.sys)).toEqual(expect.arrayContaining(['contentType']));
+      expect(node.sys.type).toEqual('DeletedEntry');
+    }
+
+    expect(prev.localized[locale.code].entries.length).toBe(mockData.entries.length - 1);
+    expect(prev.localized[locale.code].assets.length).toBe(mockData.assets.length - 1);
+    expect(prev.localized[locale.code].entryMap.has(deletedEntry.sys.id)).toBe(false);
+    expect(prev.localized[locale.code].assetMap.has(deletedAsset.sys.id)).toBe(false);
+  });
 });