Skip to content

Commit

Permalink
Relax schamas, start refactor of cache and tree structures
Browse files Browse the repository at this point in the history 
This moves towards storing more information about categories in the data/* files
(re: #4906)
  • Loading branch information
@bhousel
bhousel committed Feb 17, 2021
1 parent f1a0292 commit 953b553
Show file tree
Hide file tree
Showing 10 changed files with 575 additions and 294 deletions.
534 changes: 383 additions & 151 deletions config/trees.json
View file

Large diffs are not rendered by default.

218 changes: 124 additions & 94 deletions lib/file_tree.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,20 @@ const itemsSchema = require('../schema/items.json');
// - validates data on read, generating any missing data
// - cleans data on write, sorting and lowercasing all the keys and arrays

// cache: {
// id: {}, // Map of item id -> item
// category: {}, // Map of tkv -> category data
// path: {
// items: [],
// templates: []
// }
// }

exports.read = (cache, loco) => {
cache = cache || {};
cache.id = cache.id || {};
cache.category = cache.category || {};
cache.path = cache.path || {};
cache.template = cache.template || {};

Object.keys(trees).forEach(t => {
const tree = trees[t];
Expand Down Expand Up @@ -52,83 +60,93 @@ exports.read = (cache, loco) => {
validate(file, input, itemsSchema);

let seenkv = {};
Object.keys(input).forEach(tkv => {
const parts = tkv.split('/', 3); // tkv = "tree/key/value"
const k = parts[1];
const v = parts[2];
const kv = `${k}/${v}`;

// make sure each k/v pair appears in only one tree
const other = seenkv[kv];
if (other && other !== t) {
console.error(colors.red(`Error - '${kv}' found in multiple trees: ${other} and ${t}.`));
console.error(' ' + colors.yellow(file));
process.exit(1);
} else {
seenkv[kv] = t;
}
// new : old
const tkv = input.path || Object.keys(input)[0];
const parts = tkv.split('/', 3); // tkv = "tree/key/value"
const k = parts[1];
const v = parts[2];
const kv = `${k}/${v}`;

// make sure t/k/v is unique
if (cache.category[tkv]) {
console.error(colors.red(`Error - '${tkv}' found in multiple files.`));
console.error(' ' + colors.yellow(file));
process.exit(1);
} else {
cache.category[tkv] = input;
cache.path[tkv] = { items: [], templates: [] };
}

// make sure each k/v pair appears in only one tree
const other = seenkv[kv];
if (other && other !== t) {
console.error(colors.red(`Error - '${kv}' found in multiple trees: ${other} and ${t}.`));
console.error(' ' + colors.yellow(file));
process.exit(1);
} else {
seenkv[kv] = t;
}

// check and merge each item
let seenName = {};
input[tkv].forEach(item => {
itemCount++;
// check and merge each item
let seenName = {};

if (item.templateSource) { // It's a template item
if (!cache.template[tkv]) cache.template[tkv] = [];
cache.template[tkv].push(item);
return;
}
// new : old
let items = input.items || input[tkv];
items.forEach(item => {
itemCount++;

// check displayName for uniqueness within this category
if (seenName[item.displayName]) {
console.error(colors.red(`Error - duplicate displayName '${item.displayName}' in:`));
console.error(' ' + colors.yellow(file));
process.exit(1);
} else {
seenName[item.displayName] = true;
}
if (item.templateSource) { // It's a template item
cache.path[tkv].templates.push(item);
return;
}

// check locationSet
let locationID;
try {
const resolved = loco.resolveLocationSet(item.locationSet);
locationID = resolved.id;
if (!resolved.feature.geometry.coordinates.length || !resolved.feature.properties.area) {
throw new Error(`locationSet ${locationID} resolves to an empty feature.`);
}
} catch (err) {
console.error(colors.red(`Error - ${err.message} in:`));
console.error(' ' + colors.yellow(item.displayName));
console.error(' ' + colors.yellow(file));
process.exit(1);
// check displayName for uniqueness within this category
if (seenName[item.displayName]) {
console.error(colors.red(`Error - duplicate displayName '${item.displayName}' in:`));
console.error(' ' + colors.yellow(file));
process.exit(1);
} else {
seenName[item.displayName] = true;
}

// check locationSet
let locationID;
try {
const resolved = loco.resolveLocationSet(item.locationSet);
locationID = resolved.id;
if (!resolved.feature.geometry.coordinates.length || !resolved.feature.properties.area) {
throw new Error(`locationSet ${locationID} resolves to an empty feature.`);
}
} catch (err) {
console.error(colors.red(`Error - ${err.message} in:`));
console.error(' ' + colors.yellow(item.displayName));
console.error(' ' + colors.yellow(file));
process.exit(1);
}

// check tags
item.tags[k] = v; // sanity check: `k=v` must exist as a tag.
// check tags
item.tags[k] = v; // sanity check: `k=v` must exist as a tag.

// generate id
item.id = idgen(item, tkv, locationID);
if (!item.id) {
console.error(colors.red(`Error - Couldn't generate an id for:`));
console.error(' ' + colors.yellow(item.displayName));
console.error(' ' + colors.yellow(file));
process.exit(1);
}
// generate id
item.id = idgen(item, tkv, locationID);
if (!item.id) {
console.error(colors.red(`Error - Couldn't generate an id for:`));
console.error(' ' + colors.yellow(item.displayName));
console.error(' ' + colors.yellow(file));
process.exit(1);
}

// merge into caches
const existing = cache.id[item.id];
if (existing) {
console.error(colors.red(`Error - Duplicate id '${item.id}' in:`));
console.error(' ' + colors.yellow(item.displayName));
console.error(' ' + colors.yellow(file));
process.exit(1);
} else {
if (!cache.path[tkv]) cache.path[tkv] = [];
cache.path[tkv].push(item);
cache.id[item.id] = item;
}
});
// merge into caches
const existing = cache.id[item.id];
if (existing) {
console.error(colors.red(`Error - Duplicate id '${item.id}' in:`));
console.error(' ' + colors.yellow(item.displayName));
console.error(' ' + colors.yellow(file));
process.exit(1);
} else {
cache.path[tkv].items.push(item);
cache.id[item.id] = item;
}
});
});

Expand All @@ -141,34 +159,29 @@ exports.read = (cache, loco) => {

exports.write = (cache) => {
cache = cache || {};
cache.id = cache.id || {};
cache.category = cache.category || {};
cache.path = cache.path || {};
cache.template = cache.template || {};

Object.keys(trees).forEach(t => {
const tree = trees[t];
let itemCount = 0;
let fileCount = 0;

// gather paths used by normal items and template items..
const paths = new Set();
Object.keys(cache.path).forEach(tkv => { if (tkv.split('/')[0] === t) paths.add(tkv); });
Object.keys(cache.template).forEach(tkv => { if (tkv.split('/')[0] === t) paths.add(tkv); });
Object.keys(cache.path).forEach(tkv => {
if (tkv.split('/')[0] !== t) return;

if (!paths.size) {
console.error(colors.red(`Error - No data to write for ${t}`));
process.exit(1);
}
const parts = tkv.split('/', 3); // tkv = "tree/key/value"
const k = parts[1];
const v = parts[2];
const kv = `${k}/${v}`;

paths.forEach(tkv => {
const file = `./data/${tkv}.json`;
fileCount++;

let templateItems = cache.template[tkv] || [];
let normalItems = cache.path[tkv] || [];
let templateItems = cache.path[tkv].templates || [];
let normalItems = cache.path[tkv].items || [];
if (!templateItems.length && !normalItems.length) return; // nothing to do

const file = `./data/${tkv}.json`;

templateItems = templateItems
.sort((a, b) => withLocale(a.templateSource, b.templateSource)) // sort templateItems by templateSource
.map(item => {
Expand Down Expand Up @@ -237,8 +250,30 @@ exports.write = (cache) => {
});


//old
// let output = {};
// output[tkv] = templateItems.concat(normalItems);
//new
let output = {};
output[tkv] = templateItems.concat(normalItems);
output.path = tkv;


//new
let contains = tree.contains[kv];
let excludeNames = contains && contains.exclude;

// add properties for this category
let category = cache.category[tkv] || {};
if (category.exclude || excludeNames) {
output.exclude = {};
// if (category.exclude.named) output.exclude.named = category.exclude.named;
// if (category.exclude.generic) output.exclude.generic = category.exclude.generic;

if (excludeNames) output.exclude.named = excludeNames;
}

// add items
output.items = templateItems.concat(normalItems);

try {
fs.ensureFileSync(file);
Expand Down Expand Up @@ -270,14 +305,10 @@ exports.expandTemplates = (cache, loco) => {
cache = cache || {};
cache.id = cache.id || {};
cache.path = cache.path || {};
cache.template = cache.template || {};

// gather paths used by template items..
const paths = Object.keys(cache.template) || [];

paths.forEach(tkv => {
Object.keys(cache.path).forEach(tkv => {
const file = `./data/${tkv}.json`;
let templateItems = cache.template[tkv] || [];
const templateItems = cache.path[tkv].templates || [];

// expand each template item into real items..
templateItems.forEach(templateItem => {
Expand All @@ -286,7 +317,7 @@ exports.expandTemplates = (cache, loco) => {
const templateSource = templateItem.templateSource;
const templateTags = templateItem.templateTags;

const sourceItems = cache.path[templateSource];
const sourceItems = cache.path[templateSource].items;
if (!Array.isArray(sourceItems)) {
console.error(colors.red(`Error - template item references invalid source path '${templateSource}' in:`));
console.error(' ' + colors.yellow(file));
Expand Down Expand Up @@ -360,8 +391,7 @@ exports.expandTemplates = (cache, loco) => {
console.error(' ' + colors.yellow(file));
process.exit(1);
} else {
if (!cache.path[tkv]) cache.path[tkv] = [];
cache.path[tkv].push(item);
cache.path[tkv].items.push(item);
cache.id[item.id] = item;
}
});
Expand Down
8 changes: 4 additions & 4 deletions lib/matcher.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,8 +88,8 @@ module.exports = () => {
// `all` needs to be an Object indexed on a 'tree/key/value' path.
// (The cache in `file_tree.js` makes this)
// {
// 'brands/amenity/bank': [ {}, {}, … ],
// 'brands/amenity/bar': [ {}, {}, … ],
// 'brands/amenity/bank': { items: [ {}, {}, … ] },
// 'brands/amenity/bar': { items: [ {}, {}, … ] },
// …
// }
//
Expand All @@ -98,7 +98,7 @@ module.exports = () => {
_matchIndex = { primary: {}, alternate: {} };

Object.keys(all).forEach(tkv => {
let items = all[tkv];
let items = all[tkv].items;
if (!Array.isArray(items) || !items.length) return;

const parts = tkv.split('/', 3); // tkv = "tree/key/value"
Expand Down Expand Up @@ -260,7 +260,7 @@ module.exports = () => {
_locationSets = {};

Object.keys(all).forEach(tkv => {
let items = all[tkv];
let items = all[tkv].items;
if (!Array.isArray(items) || !items.length) return;

items.forEach(item => {
Expand Down
2 changes: 2 additions & 0 deletions lib/sort_object.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@ const withLocale = require('locale-compare')('en-US');
// Returns an object with sorted keys and sorted values.
// (This is useful for file diffing)
module.exports = (obj) => {
if (!obj) return null;

let sorted = {};
Object.keys(obj).sort(keyCompare).forEach(k => {
sorted[k] = Array.isArray(obj[k]) ? obj[k].sort(withLocale) : obj[k];
Expand Down
14 changes: 11 additions & 3 deletions schema/items.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,18 @@
"title": "items.json",
"description": "JSON Schema for name-suggestion-index items",
"type": "object",
"additionalProperties": false,
"patternProperties": {
"required": [],
"additionalProperties": true,
"properties": {

"^\\S+/\\S+/\\S+$": {
"path": {
"description": "(required) The path for this category in the form tree/key/value",
"type": "string",
"pattern": "^\\S+/\\S+/\\S+$"
},

"items": {
"description": "(required) The items in this category",
"type": "array",
"uniqueItems": true,
"items": {
Expand Down
Loading

0 comments on commit 953b553

Please sign in to comment.