From 50c66e397218574f8af88daebf030c54ac14cd85 Mon Sep 17 00:00:00 2001 From: Matthias Mohr Date: Thu, 10 Aug 2023 14:38:59 +0200 Subject: [PATCH 01/36] Rewritten the code for more features and to extract validation, v2.0.0-beta.1 --- README.md | 4 +- bin/cli.js | 3 +- index.js | 417 ------------------------------------------ package.json | 9 +- src/cli.js | 96 ++++++++++ src/config.js | 94 ++++++++++ src/index.js | 263 ++++++++++++++++++++++++++ iri.js => src/iri.js | 0 src/lint.js | 58 ++++++ src/loader/default.js | 8 + src/loader/node.js | 15 ++ src/nodeUtils.js | 244 ++++++++++++++++++++++++ src/utils.js | 92 ++++++++++ tests/cli.test.js | 2 +- 14 files changed, 879 insertions(+), 426 deletions(-) delete mode 100644 index.js create mode 100644 src/cli.js create mode 100644 src/config.js create mode 100644 src/index.js rename iri.js => src/iri.js (100%) create mode 100644 src/lint.js create mode 100644 src/loader/default.js create mode 100644 src/loader/node.js create mode 100644 src/nodeUtils.js create mode 100644 src/utils.js diff --git a/README.md b/README.md index a0c9347..84b8f57 100644 --- a/README.md +++ b/README.md @@ -6,11 +6,11 @@ See the [STAC Validator Comparison](COMPARISON.md) for the features supported by ## Versions -**Current version: 1.3.0** +**Current version: 2.0.0-beta.1** | STAC Node Validator Version | Supported STAC Versions | | --------------------------- | ----------------------- | -| 1.1.0 / 1.2.x | >= 1.0.0-rc.1 | +| 1.1.0 / 1.2.x / 2.x.x | >= 1.0.0-rc.1 | | 0.4.x / 1.0.x | >= 1.0.0-beta.2 and < 1.0.0-rc.3 | | 0.3.0 | 1.0.0-beta.2 | | 0.2.1 | 1.0.0-beta.1 | diff --git a/bin/cli.js b/bin/cli.js index 642c81e..e234b07 100755 --- a/bin/cli.js +++ b/bin/cli.js @@ -1,2 +1,3 @@ #!/usr/bin/env node -require('../index.js')() +const runCLI = require('../src/cli.js'); +runCLI(); diff --git a/index.js b/index.js deleted file mode 100644 index fe082de..0000000 --- a/index.js +++ /dev/null @@ -1,417 +0,0 @@ -const Ajv = require('ajv'); -const axios = require('axios'); -const addFormats = require('ajv-formats'); -const iriFormats = require('./iri.js'); -const fs = require('fs-extra'); -const klaw = require('klaw'); -const path = require('path') -const minimist = require('minimist'); -const versions = require('compare-versions'); -const {diffStringsUnified} = require('jest-diff'); -const { version } = require('./package.json'); - -let DEBUG = false; - -let ajv = new Ajv({ - formats: iriFormats, - allErrors: true, - strict: false, - logger: DEBUG ? console : false, - loadSchema: loadJsonFromUri -}); -addFormats(ajv); - -let verbose = false; -let schemaMap = {}; -let schemaFolder = null; - -let booleanArgs = ['verbose', 'ignoreCerts', 'lint', 'format', 'version', 'strict', 'all']; - -async function run(config) { - try { - let args = config || minimist(process.argv.slice(2)); - - if (args.version) { - console.log(version); - process.exit(0); - } - else { - console.log(`STAC Node Validator v${version}\n`); - } - - // Show minimal help output - if (args.help) { - console.log("For more information on using this command line tool, please visit"); - console.log("https://github.com/stac-utils/stac-node-validator/blob/master/README.md#usage"); - - process.exit(0); - } - - // Read config from file - if (typeof args.config === 'string') { - let configFile; - try { - configFile = await fs.readFile(args.config, "utf8"); - } catch (error) { - throw new Error('Config file does not exist.'); - } - try { - config = JSON.parse(configFile); - } catch (error) { - throw new Error('Config file is invalid JSON.'); - } - } - - // Merge CLI parameters into config - if (!config) { - config = {}; - } - for(let key in args) { - let value = args[key]; - if (key === '_' && Array.isArray(value) && value.length > 0) { - config.files = value; - } - else if (booleanArgs.includes(key)) { - if (typeof value === 'string' && value.toLowerCase() === 'false') { - config[key] = false; - } - else { - config[key] = Boolean(value); - } - } - else { - config[key] = value; - } - } - - verbose = Boolean(config.verbose); - - let files = Array.isArray(config.files) ? config.files : []; - if (files.length === 0) { - throw new Error('No path or URL specified.'); - } - else if (files.length === 1 && !isUrl(files[0])) { - // Special handling for reading directories - let stat = await fs.lstat(files[0]); - if (stat.isDirectory()) { - if (config.all) { - files = await readFolder(files[0], /.+\.json$/i); - } - else { - files = await readFolder(files[0], /(^|\/|\\)examples(\/|\\).+\.json$/i); - } - } - } - - if (config.ignoreCerts) { - process.env["NODE_TLS_REJECT_UNAUTHORIZED"] = 0; - } - if (config.strict) { - ajv.opts.strictSchema = true; - ajv.opts.strictNumbers = true; - ajv.opts.strictTuples = true; - } - - if (typeof config.schemas === 'string') { - let stat = await fs.lstat(config.schemas); - if (stat.isDirectory()) { - schemaFolder = normalizePath(config.schemas); - } - else { - throw new Error('Schema folder is not a valid STAC directory'); - } - } - - let schemaMapArgs = []; - if (config.schemaMap && typeof config.schemaMap === 'object') { - // Recommended way - schemaMapArgs = config.schemaMap; - } - else if (typeof config.schemaMap === 'string') { - // Backward compliance - schemaMapArgs = config.schemaMap.split(';'); - } - for(let url in schemaMapArgs) { - let path = schemaMapArgs[url]; - if (typeof url === 'string') { // from CLI - [url, path] = path.split("="); - } - let stat = await fs.lstat(path); - if (stat.isFile()) { - schemaMap[url] = path; - } - else { - console.error(`Schema mapping for ${url} is not a valid file: ${normalizePath(path)}`); - } - } - - const doLint = Boolean(config.lint); - const doFormat = Boolean(config.format); - - let stats = { - files: files.length, - invalid: 0, - valid: 0, - malformed: 0 - } - for(let file of files) { - // Read STAC file - let json; - console.log(`- ${normalizePath(file)}`); - try { - let fileIsUrl = isUrl(file); - if (!fileIsUrl && (doLint || doFormat)) { - let fileContent = await fs.readFile(file, "utf8"); - json = JSON.parse(fileContent); - const expectedContent = JSON.stringify(json, null, 2); - if (!matchFile(fileContent, expectedContent)) { - stats.malformed++; - if (doLint) { - console.warn("-- Lint: File is malformed -> use `--format` to fix the issue"); - if (verbose) { - console.log(diffStringsUnified(fileContent, expectedContent)); - } - } - if (doFormat) { - console.warn("-- Format: File was malformed -> fixed the issue"); - await fs.writeFile(file, expectedContent); - } - } - else if (doLint && verbose) { - console.warn("-- Lint: File is well-formed"); - } - } - else { - json = await loadJsonFromUri(file); - if (fileIsUrl && (doLint || doFormat)) { - let what = []; - doLint && what.push('Linting'); - doFormat && what.push('Formatting'); - console.warn(`-- ${what.join(' and ')} not supported for remote files`); - } - } - } - catch(error) { - stats.invalid++; - stats.malformed++; - console.error("-- " + error.message + "\n"); - continue; - } - - let isApiList = false; - let entries; - if (Array.isArray(json.collections)) { - entries = json.collections; - isApiList = true; - if (verbose) { - console.log(`-- The file is a /collections endpoint. Validating all ${entries.length} collections, but ignoring the other parts of the response.`); - if (entries.length > 1) { - console.log(''); - } - } - } - else if (Array.isArray(json.features)) { - entries = json.features; - isApiList = true; - if (verbose) { - console.log(`-- The file is a /collections/:id/items endpoint. Validating all ${entries.length} items, but ignoring the other parts of the response.`); - if (entries.length > 1) { - console.log(''); - } - } - } - else { - entries = [json]; - } - - let fileValid = true; - for(let data of entries) { - let id = ''; - if (isApiList) { - id = `${data.id}: `; - } - if (typeof data.stac_version !== 'string') { - console.error(`-- ${id}Skipping; No STAC version found\n`); - fileValid = false; - continue; - } - else if (versions.compare(data.stac_version, '1.0.0-rc.1', '<')) { - console.error(`-- ${id}Skipping; Can only validate STAC version >= 1.0.0-rc.1\n`); - continue; - } - else if (verbose) { - console.log(`-- ${id}STAC Version: ${data.stac_version}`); - } - - switch(data.type) { - case 'FeatureCollection': - console.warn(`-- ${id}Skipping; STAC ItemCollections not supported yet\n`); - continue; - case 'Catalog': - case 'Collection': - case 'Feature': - break; - default: - console.error(`-- ${id}Invalid; Can't detect type of the STAC object. Is the 'type' field missing or invalid?\n`); - fileValid = false; - continue; - } - - // Get all schema to validate against - let schemas = [data.type]; - if (Array.isArray(data.stac_extensions)) { - schemas = schemas.concat(data.stac_extensions); - // Convert shortcuts supported in 1.0.0 RC1 into schema URLs - if (versions.compare(data.stac_version, '1.0.0-rc.1', '=')) { - schemas = schemas.map(ext => ext.replace(/^(eo|projection|scientific|view)$/, 'https://schemas.stacspec.org/v1.0.0-rc.1/extensions/$1/json-schema/schema.json')); - } - } - - for(let schema of schemas) { - try { - let schemaId; - let core = false; - switch(schema) { - case 'Feature': - schema = 'Item'; - case 'Catalog': - case 'Collection': - let type = schema.toLowerCase(); - schemaId = `https://schemas.stacspec.org/v${data.stac_version}/${type}-spec/json-schema/${type}.json`; - core = true; - break; - default: // extension - if (isUrl(schema)) { - schemaId = schema; - } - else { - throw new Error("'stac_extensions' must contain a valid schema URL, not a shortcut."); - } - } - let validate = await loadSchema(schemaId); - let valid = validate(data); - if (!valid) { - console.log(`--- ${schema}: invalid`); - console.warn(validate.errors); - console.log("\n"); - fileValid = false; - if (core && !DEBUG) { - if (verbose) { - console.warn("-- Validation error in core, skipping extension validation"); - } - break; - } - } - else if (verbose) { - console.log(`--- ${schema}: valid`); - } - } catch (error) { - fileValid = false; - console.error(`--- ${schema}: ${error.message}`); - if (DEBUG) { - console.trace(error); - } - } - } - if (!fileValid || verbose) { - console.log(''); - } - } - fileValid ? stats.valid++ : stats.invalid++; - } - console.info("Files: " + stats.files); - console.info("Valid: " + stats.valid); - console.info("Invalid: " + stats.invalid); - if (doLint || doFormat) { - console.info("Malformed: " + stats.malformed); - } - let errored = (stats.invalid > 0 || (doLint && !doFormat && stats.malformed > 0)) ? 1 : 0; - process.exit(errored); - } - catch(error) { - console.error(error); - process.exit(1); - } -} - -const SUPPORTED_PROTOCOLS = ['http', 'https']; - -function matchFile(given, expected) { - return normalizeNewline(given) === normalizeNewline(expected); -} - -function normalizePath(path) { - return path.replace(/\\/g, '/').replace(/\/$/, ""); -} - -function normalizeNewline(str) { - // 2 spaces, *nix newlines, newline at end of file - return str.trimRight().replace(/(\r\n|\r)/g, "\n") + "\n"; -} - -function isUrl(uri) { - if (typeof uri === 'string') { - let part = uri.match(/^(\w+):\/\//i); - if(part) { - if (!SUPPORTED_PROTOCOLS.includes(part[1].toLowerCase())) { - throw new Error(`Given protocol "${part[1]}" is not supported.`); - } - return true; - } - } - return false; -} - -async function readFolder(folder, pattern) { - var files = []; - for await (let file of klaw(folder, {depthLimit: -1})) { - let relPath = path.relative(folder, file.path); - if (relPath.match(pattern)) { - files.push(file.path); - } - } - return files; -} - -async function loadJsonFromUri(uri) { - if (schemaMap[uri]) { - uri = schemaMap[uri]; - } - else if (schemaFolder) { - uri = uri.replace(/^https:\/\/schemas\.stacspec\.org\/v[^\/]+/, schemaFolder); - } - if (isUrl(uri)) { - let response = await axios.get(uri); - return response.data; - } - else { - return JSON.parse(await fs.readFile(uri, "utf8")); - } -} - -async function loadSchema(schemaId) { - let schema = ajv.getSchema(schemaId); - if (schema) { - return schema; - } - - try { - json = await loadJsonFromUri(schemaId); - } catch (error) { - if (DEBUG) { - console.trace(error); - } - throw new Error(`-- Schema at '${schemaId}' not found. Please ensure all entries in 'stac_extensions' are valid.`); - } - - schema = ajv.getSchema(json.$id); - if (schema) { - return schema; - } - - return await ajv.compileAsync(json); -} - -module.exports = async config => { - await run(config); -}; diff --git a/package.json b/package.json index 09c7696..135c03d 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "stac-node-validator", - "version": "1.3.0", + "version": "2.0.0-beta.1", "description": "STAC Validator for NodeJS", "author": "Matthias Mohr", "license": "Apache-2.0", @@ -16,14 +16,13 @@ "type": "git", "url": "https://github.com/stac-utils/stac-node-validator.git" }, - "main": "index.js", + "main": "src/index.js", "bin": { "stac-node-validator": "./bin/cli.js" }, "files": [ "bin/cli.js", - "iri.js", - "index.js" + "src/" ], "dependencies": { "ajv": "^8.8.2", @@ -33,7 +32,7 @@ "fs-extra": "^10.0.0", "jest-diff": "^29.0.1", "klaw": "^4.0.1", - "minimist": "^1.2.5" + "yargs": "^17.7.2" }, "devDependencies": { "jest": "^29.0.1" diff --git a/src/cli.js b/src/cli.js new file mode 100644 index 0000000..3db0c2a --- /dev/null +++ b/src/cli.js @@ -0,0 +1,96 @@ +const fs = require('fs-extra'); +const { version } = require('../package.json'); +const ConfigSource = require('./config.js'); +const validate = require('../src/index.js'); +const { printConfig, printSummary, resolveFiles, printReport } = require('./nodeUtils'); +const nodeLoader = require('./loader/node'); +const { getSummary } = require('./utils'); +const lint = require('./lint'); + + +async function run() { + console.log(`STAC Node Validator v${version}`); + console.log(); + + // Read config from CLI and config file (if any) + const cliConfig = ConfigSource.fromCLI(); + let config = {}; + if (typeof cliConfig.config === 'string') { + config = ConfigSource.fromFile(config.config); + } + Object.assign(config, cliConfig); + if (!config.loader) { + config.loader = nodeLoader; + } + + // Handle ignoreCerts option in Node + if (config.ignoreCerts) { + process.env["NODE_TLS_REJECT_UNAUTHORIZED"] = 0; + } + + // Abort if no files have been provided + if (config.files.length === 0) { + abort('No path or URL specified.'); + } + + config.depth = config.depth >= 0 ? config.depth : -1; + + // Verify files exist / read folders + let data = await resolveFiles(config.files, config.depth); + delete config.files; + if (data.length === 1) { + data = data[0]; + } + + // Resolve schema folder + if (config.schemas) { + let stat = await fs.lstat(config.schemas); + if (stat.isDirectory()) { + config.schemas = normalizePath(config.schemas); + } + else { + abort('Schema folder is not a directory'); + } + } + + // Print config + if (config.verbose) { + printConfig(config); + console.log(); + } + + if (config.lint || config.format) { + config.lintFn = async (data, report, config) => { + if (!report.apiList) { + report.lint = await lint(data, config); + if (report.lint && !report.lint.valid) { + report.valid = false; + } + } + return report; + } + } + + // Finally run validation + const result = await validate(data, config); + + // Print not supported error once for API lists + if (result.apiList) { + printLint(null, config); + } + + // Print report and summary + printReport(result, config); + if (config.verbose || !result.valid) { + console.log(); + } + + const summary = getSummary(result, config); + printSummary(summary); + + // Exit with error code or report success + let errored = (summary.invalid > 0 || (config.lint && !config.format && summary.malformed > 0)) ? 1 : 0; + process.exit(errored); +} + +module.exports = run; diff --git a/src/config.js b/src/config.js new file mode 100644 index 0000000..c6c997d --- /dev/null +++ b/src/config.js @@ -0,0 +1,94 @@ +const yargs = require('yargs/yargs'); +const { hideBin } = require('yargs/helpers'); +const fs = require('fs-extra'); + +const { strArrayToObject } = require('./nodeUtils'); + +function fromCLI() { + let config = yargs(hideBin(process.argv)) + .parserConfiguration({ + 'camel-case-expansion': false, + 'boolean-negation': false, + 'strip-aliased': true + }) + .option('lint', { + alias: 'l', + type: 'boolean', + default: false, + description: 'Check whether the JSON files are well-formatted, based on the JavaScript implementation with a 2-space indentation.' + }) + .option('format', { + alias: 'f', + type: 'boolean', + default: false, + description: 'Writes the JSON files according to the linting rules.\nATTENTION: Overrides the source files!' + }) + .option('schemas', { + alias: 's', + type: 'string', + default: null, + requiresArg: true, + description: 'Validate against schemas in a local or remote STAC folder.' + }) + .option('schemaMap', { + type: 'array', + default: [], + requiresArg: true, + description: 'Validate against a specific local schema (e.g. an external extension). Provide the schema URI and the local path separated by an equal sign.\nExample: https://stac-extensions.github.io/foobar/v1.0.0/schema.json=./json-schema/schema.json', + coerce: strArrayToObject + }) + .option('ignoreCerts', { + type: 'boolean', + default: false, + description: 'Disable verification of SSL/TLS certificates.' + }) + .option('depth', { + type: 'integer', + default: -1, + description: 'The number of levels to recurse into when looking for files in folders. 0 = no subfolders, -1 = unlimited' + }) + .option('strict', { + type: 'boolean', + default: false, + description: 'Enable strict mode in validation for schemas and numbers (as defined by ajv for options `strictSchema`, `strictNumbers` and `strictTuples`.' + }) + .option('verbose', { + alias: 'v', + type: 'boolean', + default: false, + description: 'Run with verbose logging and a diff for linting.' + }) + .option('config', { + alias: 'c', + type: 'string', + default: null, + description: 'Load the options from a config file. CLI Options override config options.' + }) + .version() + .parse() + + delete config.$0; + config.files = config._; + delete config._; + + return config; +} + +async function fromFile(path) { + let configFile; + try { + configFile = await fs.readFile(path, "utf8"); + } catch (error) { + throw new Error('Config file does not exist.'); + } + try { + return JSON.parse(configFile); + } catch (error) { + throw new Error('Config file is invalid JSON.'); + } +} + +module.exports = { + fromCLI, + fromFile +}; diff --git a/src/index.js b/src/index.js new file mode 100644 index 0000000..3d2e18c --- /dev/null +++ b/src/index.js @@ -0,0 +1,263 @@ +const versions = require('compare-versions'); + +const { createAjv, isUrl, loadSchemaFromUri, normalizePath, isObject } = require('./utils'); +const defaultLoader = require('./loader/default'); + +/** + * @typedef Config + * @type {Object} + * @property {function|null} [loader=null] A function that loads the JSON from the given files. + * @property {string|null} [schemas=null] Validate against schemas in a local or remote STAC folder. + * @property {Object.} [schemaMap={}] Validate against a specific local schema (e.g. an external extension). Provide the schema URI as key and the local path as value. + * @property {boolean} [strict=false] Enable strict mode in validation for schemas and numbers (as defined by ajv for options `strictSchema`, `strictNumbers` and `strictTuples + */ + +/** + * @typedef Report + * @type {Object} + * @property {string} id + * @property {string} type + * @property {string} version + * @property {boolean} valid + * @property {Array.} messages + * @property {Array.<*>} results + * @property {Array.} children + * @property {Extensions.} extensions + * @property {boolean} apiList + */ + +/** + * @returns {Report} + */ +function createReport() { + let result = { + id: null, + type: null, + version: null, + valid: null, + skipped: false, + messages: [], + children: [], + results: { + core: [], + extensions: {} + }, + apiList: false + }; + return result; +} + +/** + * @param {Array.|Array.|Object|string} data The data to validate + * @param {Config} config The configuration object + * @returns {Report|null} + */ +async function validate(data, config) { + const defaultConfig = { + loader: defaultLoader, + schemas: null, + schemaMap: {}, + strict: false + }; + config = Object.assign({}, defaultConfig, config); + config.ajv = createAjv(config); + + let report = createReport(); + if (typeof data === 'string') { + report.id = normalizePath(data); + data = await config.loader(data); + } + + if (isObject(data)) { + report.id = report.id || data.id; + report.version = data.stac_version; + report.type = data.type; + + if (Array.isArray(data.collections)) { + data = data.collections; + report.apiList = true; + if (config.verbose) { + report.messages.push(`The file is a CollectionCollection. Validating all ${entries.length} collections, but ignoring the other parts of the response.`); + } + } + else if (Array.isArray(data.features)) { + data = data.features; + report.apiList = true; + if (config.verbose) { + report.messages.push(`The file is a ItemCollection. Validating all ${entries.length} items, but ignoring the other parts of the response.`); + } + } + else { + return validateOne(data, config, report); + } + } + + if (Array.isArray(data) && data.length > 0) { + for(const obj of data) { + const subreport = await validateOne(obj, config); + report.children.push(subreport); + } + return summarizeResults(report); + } + else { + return null; + } +} + + +/** + * @param {Object|string} source The data to validate + * @param {Config} config The configuration object + * @param {Report} report Parent report + * @returns {Report} + */ +async function validateOne(source, config, report = null) { + if (!report) { + report = createReport(); + } + + let data = source; + if (!report.id) { + if (typeof data === 'string') { + report.id = normalizePath(data); + data = await config.loader(data); + } + else { + report.id = data.id; + } + } + report.version = data.stac_version; + report.type = data.type; + + if (typeof config.lintFn === 'function') { + report = await config.lintFn(source, report, config); + } + + // Check stac_version + if (typeof data.stac_version !== 'string') { + report.skipped = true; + report.messages.push('No STAC version found'); + return report; + } + else if (versions.compare(data.stac_version, '1.0.0-rc.1', '<')) { + report.skipped = true; + report.messages.push('Can only validate STAC version >= 1.0.0-rc.1'); + return report; + } + + // Check type field + switch(data.type) { + case 'FeatureCollection': + report.skipped = true; + report.messages.push('STAC ItemCollections not supported yet'); + return report; + case 'Catalog': + case 'Collection': + case 'Feature': + // pass + break; + default: + report.valid = false; + report.results.core.push({ + instancePath: "/type", + message: "Can't detect type of the STAC object. Is the 'type' field missing or invalid?" + }); + return report; + } + + // Validate against tzhe core schemas + await validateSchema('core', data.type, data, report, config); + + // Get all extension schemas to validate against + let schemas = []; + if (Array.isArray(data.stac_extensions)) { + schemas = schemas.concat(data.stac_extensions); + // Convert shortcuts supported in 1.0.0 RC1 into schema URLs + if (versions.compare(data.stac_version, '1.0.0-rc.1', '=')) { + schemas = schemas.map(ext => ext.replace(/^(eo|projection|scientific|view)$/, 'https://schemas.stacspec.org/v1.0.0-rc.1/extensions/$1/json-schema/schema.json')); + } + } + for(const schema of schemas) { + await validateSchema('extensions', schema, data, report, config); + } + + return report; +} + +async function validateSchema(key, schema, data, report, config) { + // Get schema identifier/uri + let schemaId; + switch(schema) { + case 'Feature': + schema = 'Item'; + case 'Catalog': + case 'Collection': + let type = schema.toLowerCase(); + schemaId = `https://schemas.stacspec.org/v${report.version}/${type}-spec/json-schema/${type}.json`; + break; + default: // extension + if (isUrl(schema)) { + schemaId = schema; + } + } + + // Validate + const setValidity = (schema, errors = []) => { + if (report.valid !== false) { + report.valid = errors.length === 0; + } + if (key === 'core') { + report.results.core = errors; + } + else { + report.results.extensions[schema] = errors; + } + }; + try { + if (key !== 'core' && !schemaId) { + throw new Error("'stac_extensions' must contain a valid schema URL, not a shortcut."); + } + const validate = await loadSchema(config, schemaId); + const valid = validate(data); + if (valid) { + setValidity(schema); + } + else { + setValidity(schema, validate.errors); + } + } catch (error) { + setValidity(schema, [{ + message: error.message + }]); + } +} + +function summarizeResults(report) { + if (report.children.length > 0) { + report.valid = Boolean(report.children.every(result => result.valid)); + } + return report; +} + +async function loadSchema(config, schemaId) { + let schema = config.ajv.getSchema(schemaId); + if (schema) { + return schema; + } + + try { + json = await loadSchemaFromUri(schemaId, config); + } catch (error) { + console.trace(error); + throw new Error(`Schema at '${schemaId}' not found. Please ensure all entries in 'stac_extensions' are valid.`); + } + + schema = config.ajv.getSchema(json.$id); + if (schema) { + return schema; + } + + return await config.ajv.compileAsync(json); +} + +module.exports = validate; diff --git a/iri.js b/src/iri.js similarity index 100% rename from iri.js rename to src/iri.js diff --git a/src/lint.js b/src/lint.js new file mode 100644 index 0000000..7f92342 --- /dev/null +++ b/src/lint.js @@ -0,0 +1,58 @@ +const fs = require('fs-extra'); +const { diffStringsUnified } = require('jest-diff'); +const { isUrl, isObject } = require('./utils'); + +/** + * @typedef LintResult + * @type {Object} + * @property {boolean} valid + * @property {boolean} fixed + * @property {Error|null} error + * @property {string|null} diff + */ + +/** + * @param {string} file + * @param {Object} config + * @returns {LintResult} + */ +async function lint(file, config) { + if (isObject(file)) { + return null; + } + else if (isUrl(file)) { + return null; + } + + let result = { + valid: false, + fixed: false, + error: null, + diff: null + }; + try { + const fileContent = await fs.readFile(file, "utf8"); + const expectedContent = JSON.stringify(JSON.parse(fileContent), null, 2); + result.valid = normalizeNewline(fileContent) === normalizeNewline(expectedContent); + + if (!result.valid) { + if (config.verbose) { + result.diff = diffStringsUnified(fileContent, expectedContent); + } + if (config.format) { + await fs.writeFile(file, expectedContent); + result.fixed = true; + } + } + } catch (error) { + result.error = error; + } + return result; +} + +function normalizeNewline(str) { + // 2 spaces, *nix newlines, newline at end of file + return str.trimRight().replace(/(\r\n|\r)/g, "\n") + "\n"; +} + +module.exports = lint; diff --git a/src/loader/default.js b/src/loader/default.js new file mode 100644 index 0000000..2a95417 --- /dev/null +++ b/src/loader/default.js @@ -0,0 +1,8 @@ +const axios = require('axios'); + +async function loader(uri) { + let response = await axios.get(uri); + return response.data; +} + +module.exports = loader; diff --git a/src/loader/node.js b/src/loader/node.js new file mode 100644 index 0000000..c16a6bd --- /dev/null +++ b/src/loader/node.js @@ -0,0 +1,15 @@ +const axios = require('axios'); +const fs = require('fs-extra'); +const { isUrl } = require("../utils"); + +async function loader(uri) { + if (isUrl(uri)) { + let response = await axios.get(uri); + return response.data; + } + else { + return JSON.parse(await fs.readFile(uri, "utf8")); + } +} + +module.exports = loader; diff --git a/src/nodeUtils.js b/src/nodeUtils.js new file mode 100644 index 0000000..040edab --- /dev/null +++ b/src/nodeUtils.js @@ -0,0 +1,244 @@ +const klaw = require('klaw'); +const fs = require('fs-extra'); +const path = require('path'); + +const { isUrl } = require('./utils'); + +const SCHEMA_CHOICE = ['anyOf', 'oneOf']; + +function abort(message) { + console.error(message); + process.exit(1); +} + +function printConfig(config) { + console.group("Config"); + console.dir(config); + console.groupEnd(); +} + +function printSummary(summary) { + console.group(`Summary (${summary.total})`); + console.info("Valid: " + summary.valid); + console.info("Invalid: " + summary.invalid); + if (summary.malformed !== null) { + console.info("Malformed: " + summary.malformed); + } + console.info("Skipped: " + summary.skipped); + console.groupEnd(); +} + +function printLint(lint, config) { + const what = []; + config.lint && what.push('Linting'); + config.format && what.push('Formatting'); + const title = what.join(' and '); + + if (!lint) { + if (config.lint || config.format) { + console.group(title); + console.warn('Not supported for remote files'); + console.groupEnd(); + } + return; + } + + if (config.verbose) { + console.group(title); + if (lint.valid) { + console.info('File is well-formed'); + } + else { + if (lint.fixed) { + console.info('File was malformed -> fixed the issue'); + } + else { + console.error('File is malformed -> use `--format` to fix the issue'); + } + } + if (lint.error) { + console.error(lint.error); + } + if (lint.diff) { + console.groupCollapsed("File Diff"); + console.log(lint.diff); + console.groupEnd(); + } + console.groupEnd(); + } + else if (!lint.valid && !lint.fixed) { + console.group(title); + console.error('File is malformed -> use `--format` to fix the issue'); + if (lint.error) { + console.error(lint.error); + } + console.groupEnd(); + } +} + +function printReport(report, config) { + if (report.valid && !config.verbose) { + return; + } + + console.group(report.id || "Report"); + + if (config.verbose && report.version) { + console.log(`STAC Version: ${report.version}`); + } + + if (report.messages) { + report.messages.forEach(str => console.info(str)); + } + + if (!report.apiList) { + printLint(report.lint, config); + } + + if (!report.valid || config.verbose) { + printAjvValidationResult(report.results.core, report.type, report.valid, config); + if (report.type) { + const count = Object.keys(report.results.extensions).length; + if (count > 0) { + console.group("Extensions"); + Object.entries(report.results.extensions) + .forEach(([ext, result]) => printAjvValidationResult(result, ext, report.valid, config)); + console.groupEnd(); + } + else { + console.info("Extensions: None"); + } + } + } + + report.children.forEach(child => printReport(child, config)); + + console.groupEnd(); +} + +function printAjvValidationResult(result, category, reportValid, config) { + if (!category) { + return; + } + if (!config.verbose && isUrl(category)) { + const match = category.match(/^https?:\/\/stac-extensions\.github\.io\/([^/]+)\/v?([^/]+)(?:\/([^/.]+))?\/schema/); + if (match) { + let title = match[1]; + if (match[3]) { + title += ' - ' + formatKey(match[3]); + } + category = `${title} (${match[2]})`; + } + } + if (result.length > 0) { + console.group(category); + if (config.verbose) { + console.dir(result); + } + else { + result + .filter(error => result.length === 1 || !SCHEMA_CHOICE.includes(error.keyword)) // Remove messages that are usually not so important (anyOf/oneOf) + .sort((a,b) => { + // Sort so that anyOf/oneOf related messages come last, these are usually not so important + let aa = isSchemaChoice(a.schemaPath); + let bb = isSchemaChoice(b.schemaPath); + if (aa && bb) { + return 0; + } + else if (aa) { + return 1; + } + else if (bb) { + return -1; + } + else { + return 0; + } + }) + .map(error => makeAjvErrorMessage(error)) // Convert to string + .filter((value, i, array) => array.indexOf(value) === i) // Remove duplicates + .forEach((msg, i) => console.error(`${i+1}. ${msg}`)); // Print it as list + } + console.groupEnd(); + } + else if (!reportValid || config.verbose) { + console.log(`${category}: valid`); + } +} + +function isSchemaChoice(schemaPath) { + return typeof schemaPath === 'string' && schemaPath.match(/\/(one|any)Of\/\d+\//); +} + +function makeAjvErrorMessage(error) { + let message = error.message; + if (Object.keys(error.params).length > 0) { + let params = Object.entries(error.params) + .map(([key, value]) => { + let label = key.replace(/([^A-Z]+)([A-Z])/g, "$1 $2").toLowerCase(); + return `${label}: ${value}`; + }) + .join(', ') + message += ` (${params})`; + } + if (error.instancePath) { + return `${error.instancePath} ${message}`; + } + else if (error.schemaPath) { + return `${message}, for schema ${error.schemaPath}`; + } + else if (message) { + return message; + } + else { + return String(error); + } +} + +async function resolveFiles(files, depth = -1) { + const resolved = []; + const extensions = [".geojson", ".json"]; + const klawOptions = { + depthLimit: depth + } + for (const file of files) { + if (isUrl(file)) { + resolved.push(file); + continue; + } + + // Special handling for reading directories + const stat = await fs.lstat(file); + if (stat.isDirectory()) { + ; + for await (const child of klaw(file, klawOptions)) { + const ext = path.extname(child.path).toLowerCase(); + if (extensions.includes(ext)) { + resolved.push(child.path); + } + } + } + else { + resolved.push(file); + } + } + return resolved; +} + +function strArrayToObject(list, sep = "=") { + let map = {}; + for (let str of list) { + let [key, value] = str.split(sep, 2); + map[key] = value; + } + return map; +} + +module.exports = { + abort, + printConfig, + printReport, + printSummary, + resolveFiles, + strArrayToObject +}; diff --git a/src/utils.js b/src/utils.js new file mode 100644 index 0000000..f79c230 --- /dev/null +++ b/src/utils.js @@ -0,0 +1,92 @@ +const Ajv = require('ajv'); +const addFormats = require('ajv-formats'); +const iriFormats = require('./iri'); + +const SUPPORTED_PROTOCOLS = ['http', 'https']; + +function isObject(obj) { + return (typeof obj === 'object' && obj === Object(obj) && !Array.isArray(obj)); +} + +function isUrl(uri) { + if (typeof uri === 'string') { + let part = uri.match(/^(\w+):\/\//i); + if (part) { + if (!SUPPORTED_PROTOCOLS.includes(part[1].toLowerCase())) { + throw new Error(`Given protocol "${part[1]}" is not supported.`); + } + return true; + } + } + return false; +} + +function createAjv(config) { + let instance = new Ajv({ + formats: iriFormats, + allErrors: config.verbose, + strict: false, + logger: config.verbose ? console : false, + loadSchema: async (uri) => await loadSchemaFromUri(uri, config) + }); + addFormats(instance); + if (config.strict) { + instance.opts.strictSchema = true; + instance.opts.strictNumbers = true; + instance.opts.strictTuples = true; + } + return instance; +} + +async function loadSchemaFromUri(uri, config) { + if (isObject(config.schemaMap) && config.schemaMap[uri]) { + uri = config.schemaMap[uri]; + } + else if (config.schemas) { + uri = uri.replace(/^https:\/\/schemas\.stacspec\.org\/v[^\/]+/, config.schemas); + } + return await config.loader(uri); +} + +function normalizePath(path) { + return path.replace(/\\/g, '/').replace(/\/$/, ""); +} + +function getSummary(result, config) { + let summary = { + total: 0, + valid: 0, + invalid: 0, + malformed: null, + skipped: 0 + }; + if (result.children.length > 0) { + // todo: speed this up by computing in one iteration + summary.total = result.children.length; + summary.valid = result.children.filter(c => c.valid === true).length; + summary.invalid = result.children.filter(c => c.valid === false).length; + if (config.lint || config.format) { + summary.malformed = result.children.filter(c => c.lint && c.lint.valid).length; + } + summary.skipped = result.children.filter(c => c.skipped).length; + } + else { + summary.total = 1; + summary.valid = result.valid === true ? 1 : 0; + summary.invalid = result.valid === false ? 1 : 0; + if (result.lint) { + summary.malformed = result.lint.valid ? 0 : 1; + } + summary.skipped = result.skipped ? 1 : 0; + } + return summary; +} + +module.exports = { + createAjv, + getSummary, + isObject, + isUrl, + loadSchemaFromUri, + normalizePath +}; diff --git a/tests/cli.test.js b/tests/cli.test.js index 7a21677..f542559 100644 --- a/tests/cli.test.js +++ b/tests/cli.test.js @@ -1,4 +1,4 @@ -const app = require('../index'); +const app = require('../src/index'); const { version } = require('../package.json'); const fs = require('fs/promises'); const { exec } = require("child_process"); From a74e9fe392b9edaebc241271c9a7ce58a3630bda Mon Sep 17 00:00:00 2001 From: Matthias Mohr Date: Thu, 7 Sep 2023 13:12:19 +0200 Subject: [PATCH 02/36] Implement custom validation + doc improvements --- README.md | 4 +- custom.example.js | 25 +++++++ package.json | 2 + src/baseValidator.js | 41 ++++++++++++ src/cli.js | 7 ++ src/config.js | 5 ++ src/index.js | 39 ++++++++++- src/nodeUtils.js | 7 +- src/test.js | 155 +++++++++++++++++++++++++++++++++++++++++++ 9 files changed, 279 insertions(+), 6 deletions(-) create mode 100644 custom.example.js create mode 100644 src/baseValidator.js create mode 100644 src/test.js diff --git a/README.md b/README.md index 84b8f57..223bcd8 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,7 @@ Further options to add to the commands above: - Add `--strict` to enable strict mode in validation for schemas and numbers (as defined by [ajv](https://ajv.js.org/strict-mode.html) for options `strictSchema`, `strictNumbers` and `strictTuples`) - To lint local JSON files: `--lint` (add `--verbose` to get a diff with the changes required) - To format / pretty-print local JSON files: `--format` (Attention: this will override the source files without warning!) +- To run custom validation code: `--custom ./path/to/validation.js` - The validation.js needs to contain a class that implements the `BaseValidator` interface. See [custom.example.js](./custom.example.js) for an example. **Note on API support:** Validating lists of STAC items/collections (i.e. `GET /collections` and `GET /collections/:id/items`) is partially supported. It only checks the contained items/collections, but not the other parts of the response (e.g. `links`). @@ -74,7 +75,8 @@ The schema map is an object instead of string separated with a `=` character. "lint": true, "format": false, "strict": true, - "all": false + "all": false, + "custom": null } ``` diff --git a/custom.example.js b/custom.example.js new file mode 100644 index 0000000..dbd5482 --- /dev/null +++ b/custom.example.js @@ -0,0 +1,25 @@ +const BaseValidator = require('./src/baseValidator.js'); + +class CustomValidator extends BaseValidator { + + /** + * Any custom validation routines you want to run. + * + * You can either create a list of errors using the test interface + * or just throw on the first error. + * + * @param {STAC} data + * @param {Test} test + * @param {import('.').Report} report + * @param {import('.').Config} config + * @throws {Error} + */ + async afterValidation(data, test, report, config) { + if (data.id === 'solid-earth') { + test.deepEqual(data.example, [1,2,3]); + } + } + +} + +module.exports = CustomValidator; diff --git a/package.json b/package.json index 135c03d..bb0b3d0 100644 --- a/package.json +++ b/package.json @@ -27,11 +27,13 @@ "dependencies": { "ajv": "^8.8.2", "ajv-formats": "^2.1.1", + "assert": "^2.0.0", "axios": "^1.1.3", "compare-versions": "^6.1.0", "fs-extra": "^10.0.0", "jest-diff": "^29.0.1", "klaw": "^4.0.1", + "stac-js": "^0.0.8", "yargs": "^17.7.2" }, "devDependencies": { diff --git a/src/baseValidator.js b/src/baseValidator.js new file mode 100644 index 0000000..91a0835 --- /dev/null +++ b/src/baseValidator.js @@ -0,0 +1,41 @@ +const { STAC } = import('stac-js'); + +class BaseValidator { + + /** + * + */ + constructor() { + } + + /** + * Any preprocessing work you want to do on the data. + * + * @param {Object} data + * @param {import('.').Report} report + * @param {import('.').Config} config + * @returns {Object} + */ + async afterLoading(data, report, config) { + return data; + } + + /** + * Any custom validation routines you want to run. + * + * You can either create a list of errors using the test interface + * or just throw on the first error. + * + * @param {STAC} data + * @param {Test} test + * @param {import('.').Report} report + * @param {import('.').Config} config + * @throws {Error} + */ + async afterValidation(data, test, report, config) { + + } + +} + +module.exports = BaseValidator; diff --git a/src/cli.js b/src/cli.js index 3db0c2a..5e16b26 100644 --- a/src/cli.js +++ b/src/cli.js @@ -1,4 +1,5 @@ const fs = require('fs-extra'); +const path = require('path'); const { version } = require('../package.json'); const ConfigSource = require('./config.js'); const validate = require('../src/index.js'); @@ -71,6 +72,12 @@ async function run() { } } + if (config.custom) { + const absPath = path.resolve(process.cwd(), config.custom); + const validator = require(absPath); + config.customValidator = new validator(); + } + // Finally run validation const result = await validate(data, config); diff --git a/src/config.js b/src/config.js index c6c997d..9a0fafa 100644 --- a/src/config.js +++ b/src/config.js @@ -37,6 +37,11 @@ function fromCLI() { description: 'Validate against a specific local schema (e.g. an external extension). Provide the schema URI and the local path separated by an equal sign.\nExample: https://stac-extensions.github.io/foobar/v1.0.0/schema.json=./json-schema/schema.json', coerce: strArrayToObject }) + .option('custom', { + type: 'string', + default: null, + description: 'Load a custom validation routine from a JavaScript file.' + }) .option('ignoreCerts', { type: 'boolean', default: false, diff --git a/src/index.js b/src/index.js index 3d2e18c..73e0b4b 100644 --- a/src/index.js +++ b/src/index.js @@ -2,6 +2,8 @@ const versions = require('compare-versions'); const { createAjv, isUrl, loadSchemaFromUri, normalizePath, isObject } = require('./utils'); const defaultLoader = require('./loader/default'); +const BaseValidator = require('./baseValidator'); +const Test = require('./test'); /** * @typedef Config @@ -10,6 +12,7 @@ const defaultLoader = require('./loader/default'); * @property {string|null} [schemas=null] Validate against schemas in a local or remote STAC folder. * @property {Object.} [schemaMap={}] Validate against a specific local schema (e.g. an external extension). Provide the schema URI as key and the local path as value. * @property {boolean} [strict=false] Enable strict mode in validation for schemas and numbers (as defined by ajv for options `strictSchema`, `strictNumbers` and `strictTuples + * @property {BaseValidator} [customValidator=null] A validator with custom rules. */ /** @@ -20,12 +23,19 @@ const defaultLoader = require('./loader/default'); * @property {string} version * @property {boolean} valid * @property {Array.} messages - * @property {Array.<*>} results * @property {Array.} children - * @property {Extensions.} extensions + * @property {Results} results * @property {boolean} apiList */ +/** + * @typedef Results + * @type {Object} + * @property {OArray.} core + * @property {Object.>} extensions + * @property {Array.} custom + */ + /** * @returns {Report} */ @@ -40,7 +50,8 @@ function createReport() { children: [], results: { core: [], - extensions: {} + extensions: {}, + custom: [] }, apiList: false }; @@ -129,6 +140,10 @@ async function validateOne(source, config, report = null) { report.version = data.stac_version; report.type = data.type; + if (config.customValidator) { + data = await config.customValidator.afterLoading(data, report, config); + } + if (typeof config.lintFn === 'function') { report = await config.lintFn(source, report, config); } @@ -181,6 +196,24 @@ async function validateOne(source, config, report = null) { await validateSchema('extensions', schema, data, report, config); } + if (config.customValidator) { + const { default: create } = await import('stac-js'); + const stac = create(data, false, false); + try { + const test = new Test(); + await config.customValidator.afterValidation(stac, test, report, config); + report.results.custom = test.errors; + } catch (error) { + report.results.custom = [ + error + ]; + } finally { + if (report.results.custom.length > 0) { + report.valid = false; + } + } + } + return report; } diff --git a/src/nodeUtils.js b/src/nodeUtils.js index 040edab..dc866cf 100644 --- a/src/nodeUtils.js +++ b/src/nodeUtils.js @@ -2,7 +2,7 @@ const klaw = require('klaw'); const fs = require('fs-extra'); const path = require('path'); -const { isUrl } = require('./utils'); +const { isUrl, isObject } = require('./utils'); const SCHEMA_CHOICE = ['anyOf', 'oneOf']; @@ -109,6 +109,9 @@ function printReport(report, config) { console.info("Extensions: None"); } } + if (config.custom) { + printAjvValidationResult(report.results.custom, 'Custom', report.valid, config); + } } report.children.forEach(child => printReport(child, config)); @@ -172,7 +175,7 @@ function isSchemaChoice(schemaPath) { function makeAjvErrorMessage(error) { let message = error.message; - if (Object.keys(error.params).length > 0) { + if (isObject(error.params) && Object.keys(error.params).length > 0) { let params = Object.entries(error.params) .map(([key, value]) => { let label = key.replace(/([^A-Z]+)([A-Z])/g, "$1 $2").toLowerCase(); diff --git a/src/test.js b/src/test.js new file mode 100644 index 0000000..c17d01b --- /dev/null +++ b/src/test.js @@ -0,0 +1,155 @@ +const assert = require('assert'); + +class Test { + + constructor() { + this.errors = []; + } + + truthy(...args) { + try { + assert(...args); + } catch (error) { + this.errors.push(error); + } + } + + deepEqual(...args) { + try { + assert.deepEqual(...args); + } catch (error) { + this.errors.push(error); + } + } + + deepStrictEqual(...args) { + try { + assert.deepStrictEqual(...args); + } catch (error) { + this.errors.push(error); + } + } + + doesNotMatch(...args) { + try { + assert.doesNotMatch(...args); + } catch (error) { + this.errors.push(error); + } + } + + async doesNotReject(...args) { + try { + await assert.doesNotReject(...args); + } catch (error) { + this.errors.push(error); + } + } + + doesNotThrow(...args) { + try { + assert.doesNotThrow(...args); + } catch (error) { + this.errors.push(error); + } + } + + equal(...args) { + try { + assert.equal(...args); + } catch (error) { + this.errors.push(error); + } + } + + fail(...args) { + try { + assert.fail(...args); + } catch (error) { + this.errors.push(error); + } + } + + ifError(...args) { + try { + assert.ifError(...args); + } catch (error) { + this.errors.push(error); + } + } + + match(...args) { + try { + assert.match(...args); + } catch (error) { + this.errors.push(error); + } + } + + notDeepEqual(...args) { + try { + assert.notDeepEqual(...args); + } catch (error) { + this.errors.push(error); + } + } + + notDeepStrictEqual(...args) { + try { + assert.notDeepStrictEqual(...args); + } catch (error) { + this.errors.push(error); + } + } + + notEqual(...args) { + try { + assert.notEqual(...args); + } catch (error) { + this.errors.push(error); + } + } + + notStrictEqual(...args) { + try { + assert.notStrictEqual(...args); + } catch (error) { + this.errors.push(error); + } + } + + ok(...args) { + try { + assert.ok(...args); + } catch (error) { + this.errors.push(error); + } + } + + async rejects(...args) { + try { + await assert.rejects(...args); + } catch (error) { + this.errors.push(error); + } + } + + strictEqual(...args) { + try { + assert.strictEqual(...args); + } catch (error) { + this.errors.push(error); + } + } + + throws(...args) { + try { + assert.throws(...args); + } catch (error) { + this.errors.push(error); + } + } + +} + +module.exports = Test; From 043bbcb54d8c7d90664c2619a0ca609c300e97d5 Mon Sep 17 00:00:00 2001 From: Matthias Mohr Date: Thu, 7 Sep 2023 17:48:37 +0200 Subject: [PATCH 03/36] 2.0.0-beta.2 --- COMPARISON.md | 8 ++++---- README.md | 2 +- package.json | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/COMPARISON.md b/COMPARISON.md index f6403f0..e98c6ce 100644 --- a/COMPARISON.md +++ b/COMPARISON.md @@ -14,13 +14,13 @@ Here I'd like to give an overview of what the validators are capable of and what | | Python Validator | PySTAC | STAC Node Validator | | :------------------------- | ------------------------------------------ | ------------------- | ------------------- | -| Validator Version | 1.0.1 | 0.5.2 | 1.3.0 | +| Validator Version | 1.0.1 | 0.5.2 | 2.0.0 | | Language | Python 3.6 | Python 3 | NodeJS | | CLI | Yes | No | Yes | -| Programmatic | Yes | Yes | Planned | -| Online | Yes, [staclint.com](https://staclint.com/) | No | Planned | +| Programmatic | Yes | Yes | Yes | +| Online | Yes, [staclint.com](https://staclint.com/) | No | Yes | | Protocols supported (Read) | HTTP(S), Filesystem | HTTP(S), Filesystem | HTTP(S), Filesystem | -| Gives | HTML / CLI / Python Output | Python Dict | CLI output | +| Gives | HTML / CLI / Python | Python | CLI / JavaScript | ## Specifications supported diff --git a/README.md b/README.md index 223bcd8..3ae48fa 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ See the [STAC Validator Comparison](COMPARISON.md) for the features supported by ## Versions -**Current version: 2.0.0-beta.1** +**Current version: 2.0.0-beta.2** | STAC Node Validator Version | Supported STAC Versions | | --------------------------- | ----------------------- | diff --git a/package.json b/package.json index bb0b3d0..d661ee0 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "stac-node-validator", - "version": "2.0.0-beta.1", + "version": "2.0.0-beta.2", "description": "STAC Validator for NodeJS", "author": "Matthias Mohr", "license": "Apache-2.0", From 0f04671d4171cd626ca470d2f277b5d33b2ab662 Mon Sep 17 00:00:00 2001 From: Matthias Mohr Date: Thu, 7 Sep 2023 18:33:31 +0200 Subject: [PATCH 04/36] fix various issues, v2.0.0-beta.3 --- README.md | 28 +--------- custom.example.js | 2 +- package.json | 2 +- src/cli.js | 14 ++--- src/config.js | 130 +++++++++++++++++++++++----------------------- 5 files changed, 76 insertions(+), 100 deletions(-) diff --git a/README.md b/README.md index 3ae48fa..c2c9d92 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ See the [STAC Validator Comparison](COMPARISON.md) for the features supported by ## Versions -**Current version: 2.0.0-beta.2** +**Current version: 2.0.0-beta.3** | STAC Node Validator Version | Supported STAC Versions | | --------------------------- | ----------------------- | @@ -52,36 +52,12 @@ It only checks the contained items/collections, but not the other parts of the r ### Config file You can also pass a config file via the `--config` option. Simply pass a file path as value. -Parameters set via CLI will override the corresponding setting in the config file. -Make sure to use the value `false` to override boolean flags that are set to `true` in the config file. +Parameters set via CLI will not override the corresponding setting in the config file. The config file uses the same option names as above. To specify the files to be validated, add an array with paths. The schema map is an object instead of string separated with a `=` character. -**Example:** -```json -{ - "files": [ - "/path/to/your/catalog.json", - "/path/to/your/item.json" - ], - "schemas": "/path/to/stac/folder", - "schemaMap": { - "https://stac-extensions.github.io/foobar/v1.0.0/schema.json": "./json-schema/schema.json" - }, - "ignoreCerts": false, - "verbose": false, - "lint": true, - "format": false, - "strict": true, - "all": false, - "custom": null -} -``` - -You could now override some options as follows in CLI: `stac-node-validator example.json --config /path/to/config.json --lint false` - ### Development 1. `git clone https://github.com/stac-utils/stac-node-validator` to clone the repo diff --git a/custom.example.js b/custom.example.js index dbd5482..9d6c6a2 100644 --- a/custom.example.js +++ b/custom.example.js @@ -1,4 +1,4 @@ -const BaseValidator = require('./src/baseValidator.js'); +const BaseValidator = require('stac-node-validator/src/baseValidator.js'); class CustomValidator extends BaseValidator { diff --git a/package.json b/package.json index d661ee0..da42f89 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "stac-node-validator", - "version": "2.0.0-beta.2", + "version": "2.0.0-beta.3", "description": "STAC Validator for NodeJS", "author": "Matthias Mohr", "license": "Apache-2.0", diff --git a/src/cli.js b/src/cli.js index 5e16b26..196d44c 100644 --- a/src/cli.js +++ b/src/cli.js @@ -14,12 +14,10 @@ async function run() { console.log(); // Read config from CLI and config file (if any) - const cliConfig = ConfigSource.fromCLI(); - let config = {}; - if (typeof cliConfig.config === 'string') { - config = ConfigSource.fromFile(config.config); + let config = ConfigSource.fromCLI(); + if (typeof config.config === 'string') { + Object.assign(config, await ConfigSource.fromFile(config.config)); } - Object.assign(config, cliConfig); if (!config.loader) { config.loader = nodeLoader; } @@ -31,7 +29,8 @@ async function run() { // Abort if no files have been provided if (config.files.length === 0) { - abort('No path or URL specified.'); + console.error('No path or URL specified.'); + process.exit(1); } config.depth = config.depth >= 0 ? config.depth : -1; @@ -50,7 +49,8 @@ async function run() { config.schemas = normalizePath(config.schemas); } else { - abort('Schema folder is not a directory'); + console.error('Schema folder is not a directory'); + process.exit(1); } } diff --git a/src/config.js b/src/config.js index 9a0fafa..a30fc88 100644 --- a/src/config.js +++ b/src/config.js @@ -6,71 +6,71 @@ const { strArrayToObject } = require('./nodeUtils'); function fromCLI() { let config = yargs(hideBin(process.argv)) - .parserConfiguration({ - 'camel-case-expansion': false, - 'boolean-negation': false, - 'strip-aliased': true - }) - .option('lint', { - alias: 'l', - type: 'boolean', - default: false, - description: 'Check whether the JSON files are well-formatted, based on the JavaScript implementation with a 2-space indentation.' - }) - .option('format', { - alias: 'f', - type: 'boolean', - default: false, - description: 'Writes the JSON files according to the linting rules.\nATTENTION: Overrides the source files!' - }) - .option('schemas', { - alias: 's', - type: 'string', - default: null, - requiresArg: true, - description: 'Validate against schemas in a local or remote STAC folder.' - }) - .option('schemaMap', { - type: 'array', - default: [], - requiresArg: true, - description: 'Validate against a specific local schema (e.g. an external extension). Provide the schema URI and the local path separated by an equal sign.\nExample: https://stac-extensions.github.io/foobar/v1.0.0/schema.json=./json-schema/schema.json', - coerce: strArrayToObject - }) - .option('custom', { - type: 'string', - default: null, - description: 'Load a custom validation routine from a JavaScript file.' - }) - .option('ignoreCerts', { - type: 'boolean', - default: false, - description: 'Disable verification of SSL/TLS certificates.' - }) - .option('depth', { - type: 'integer', - default: -1, - description: 'The number of levels to recurse into when looking for files in folders. 0 = no subfolders, -1 = unlimited' - }) - .option('strict', { - type: 'boolean', - default: false, - description: 'Enable strict mode in validation for schemas and numbers (as defined by ajv for options `strictSchema`, `strictNumbers` and `strictTuples`.' - }) - .option('verbose', { - alias: 'v', - type: 'boolean', - default: false, - description: 'Run with verbose logging and a diff for linting.' - }) - .option('config', { - alias: 'c', - type: 'string', - default: null, - description: 'Load the options from a config file. CLI Options override config options.' - }) - .version() - .parse() + .parserConfiguration({ + 'camel-case-expansion': false, + 'boolean-negation': false, + 'strip-aliased': true + }) + .option('lint', { + alias: 'l', + type: 'boolean', + default: false, + description: 'Check whether the JSON files are well-formatted, based on the JavaScript implementation with a 2-space indentation.' + }) + .option('format', { + alias: 'f', + type: 'boolean', + default: false, + description: 'Writes the JSON files according to the linting rules.\nATTENTION: Overrides the source files!' + }) + .option('schemas', { + alias: 's', + type: 'string', + default: null, + requiresArg: true, + description: 'Validate against schemas in a local or remote STAC folder.' + }) + .option('schemaMap', { + type: 'array', + default: [], + requiresArg: true, + description: 'Validate against a specific local schema (e.g. an external extension). Provide the schema URI and the local path separated by an equal sign.\nExample: https://stac-extensions.github.io/foobar/v1.0.0/schema.json=./json-schema/schema.json', + coerce: strArrayToObject + }) + .option('custom', { + type: 'string', + default: null, + description: 'Load a custom validation routine from a JavaScript file.' + }) + .option('ignoreCerts', { + type: 'boolean', + default: false, + description: 'Disable verification of SSL/TLS certificates.' + }) + .option('depth', { + type: 'integer', + default: -1, + description: 'The number of levels to recurse into when looking for files in folders. 0 = no subfolders, -1 = unlimited' + }) + .option('strict', { + type: 'boolean', + default: false, + description: 'Enable strict mode in validation for schemas and numbers (as defined by ajv for options `strictSchema`, `strictNumbers` and `strictTuples`.' + }) + .option('verbose', { + alias: 'v', + type: 'boolean', + default: false, + description: 'Run with verbose logging and a diff for linting.' + }) + .option('config', { + alias: 'c', + type: 'string', + default: null, + description: 'Load the options from a config file. CLI Options override config options.' + }) + .version() + .parse() delete config.$0; config.files = config._; From 2491e0d9e419c51ad65eb7d6315448640d5b9e23 Mon Sep 17 00:00:00 2001 From: Matthias Mohr Date: Fri, 6 Oct 2023 17:27:21 +0200 Subject: [PATCH 05/36] fix CLI output issues, v2.0.0-beta.4 --- README.md | 2 +- package.json | 2 +- src/cli.js | 2 +- src/nodeUtils.js | 10 +++++----- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index c2c9d92..33faee1 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ See the [STAC Validator Comparison](COMPARISON.md) for the features supported by ## Versions -**Current version: 2.0.0-beta.3** +**Current version: 2.0.0-beta.4** | STAC Node Validator Version | Supported STAC Versions | | --------------------------- | ----------------------- | diff --git a/package.json b/package.json index da42f89..7e63bf9 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "stac-node-validator", - "version": "2.0.0-beta.3", + "version": "2.0.0-beta.4", "description": "STAC Validator for NodeJS", "author": "Matthias Mohr", "license": "Apache-2.0", diff --git a/src/cli.js b/src/cli.js index 196d44c..3ff973a 100644 --- a/src/cli.js +++ b/src/cli.js @@ -81,7 +81,7 @@ async function run() { // Finally run validation const result = await validate(data, config); - // Print not supported error once for API lists + // Print not a "supported error" once for API lists if (result.apiList) { printLint(null, config); } diff --git a/src/nodeUtils.js b/src/nodeUtils.js index dc866cf..fbc6ad9 100644 --- a/src/nodeUtils.js +++ b/src/nodeUtils.js @@ -53,11 +53,11 @@ function printLint(lint, config) { console.info('File was malformed -> fixed the issue'); } else { - console.error('File is malformed -> use `--format` to fix the issue'); + console.warn('File is malformed -> use `--format` to fix the issue'); } } if (lint.error) { - console.error(lint.error); + console.warn(lint.error); } if (lint.diff) { console.groupCollapsed("File Diff"); @@ -68,9 +68,9 @@ function printLint(lint, config) { } else if (!lint.valid && !lint.fixed) { console.group(title); - console.error('File is malformed -> use `--format` to fix the issue'); + console.warn('File is malformed -> use `--format` to fix the issue'); if (lint.error) { - console.error(lint.error); + console.warn(lint.error); } console.groupEnd(); } @@ -160,7 +160,7 @@ function printAjvValidationResult(result, category, reportValid, config) { }) .map(error => makeAjvErrorMessage(error)) // Convert to string .filter((value, i, array) => array.indexOf(value) === i) // Remove duplicates - .forEach((msg, i) => console.error(`${i+1}. ${msg}`)); // Print it as list + .forEach((msg, i) => console.warn(`${i+1}. ${msg}`)); // Print it as list } console.groupEnd(); } From ef062ff68ce6d639085085e9b49d889032ff7f98 Mon Sep 17 00:00:00 2001 From: Matthias Mohr Date: Fri, 6 Oct 2023 17:38:55 +0200 Subject: [PATCH 06/36] fix CLI output issues, v2.0.0-beta.5 --- README.md | 2 +- package.json | 2 +- src/cli.js | 8 +++----- src/nodeUtils.js | 28 ++++++++++++++-------------- 4 files changed, 19 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 33faee1..a9c2a68 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ See the [STAC Validator Comparison](COMPARISON.md) for the features supported by ## Versions -**Current version: 2.0.0-beta.4** +**Current version: 2.0.0-beta.5** | STAC Node Validator Version | Supported STAC Versions | | --------------------------- | ----------------------- | diff --git a/package.json b/package.json index 7e63bf9..2cc8b32 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "stac-node-validator", - "version": "2.0.0-beta.4", + "version": "2.0.0-beta.5", "description": "STAC Validator for NodeJS", "author": "Matthias Mohr", "license": "Apache-2.0", diff --git a/src/cli.js b/src/cli.js index 3ff973a..2dba184 100644 --- a/src/cli.js +++ b/src/cli.js @@ -3,7 +3,7 @@ const path = require('path'); const { version } = require('../package.json'); const ConfigSource = require('./config.js'); const validate = require('../src/index.js'); -const { printConfig, printSummary, resolveFiles, printReport } = require('./nodeUtils'); +const { printConfig, printSummary, resolveFiles, printReport, abort } = require('./nodeUtils'); const nodeLoader = require('./loader/node'); const { getSummary } = require('./utils'); const lint = require('./lint'); @@ -29,8 +29,7 @@ async function run() { // Abort if no files have been provided if (config.files.length === 0) { - console.error('No path or URL specified.'); - process.exit(1); + abort('No path or URL specified.'); } config.depth = config.depth >= 0 ? config.depth : -1; @@ -49,8 +48,7 @@ async function run() { config.schemas = normalizePath(config.schemas); } else { - console.error('Schema folder is not a directory'); - process.exit(1); + abort('Schema folder is not a directory'); } } diff --git a/src/nodeUtils.js b/src/nodeUtils.js index fbc6ad9..1c5c5f7 100644 --- a/src/nodeUtils.js +++ b/src/nodeUtils.js @@ -19,12 +19,12 @@ function printConfig(config) { function printSummary(summary) { console.group(`Summary (${summary.total})`); - console.info("Valid: " + summary.valid); - console.info("Invalid: " + summary.invalid); + console.log("Valid: " + summary.valid); + console.log("Invalid: " + summary.invalid); if (summary.malformed !== null) { - console.info("Malformed: " + summary.malformed); + console.log("Malformed: " + summary.malformed); } - console.info("Skipped: " + summary.skipped); + console.log("Skipped: " + summary.skipped); console.groupEnd(); } @@ -37,7 +37,7 @@ function printLint(lint, config) { if (!lint) { if (config.lint || config.format) { console.group(title); - console.warn('Not supported for remote files'); + console.log('Not supported for remote files'); console.groupEnd(); } return; @@ -46,18 +46,18 @@ function printLint(lint, config) { if (config.verbose) { console.group(title); if (lint.valid) { - console.info('File is well-formed'); + console.log('File is well-formed'); } else { if (lint.fixed) { - console.info('File was malformed -> fixed the issue'); + console.log('File was malformed -> fixed the issue'); } else { - console.warn('File is malformed -> use `--format` to fix the issue'); + console.log('File is malformed -> use `--format` to fix the issue'); } } if (lint.error) { - console.warn(lint.error); + console.log(lint.error); } if (lint.diff) { console.groupCollapsed("File Diff"); @@ -68,9 +68,9 @@ function printLint(lint, config) { } else if (!lint.valid && !lint.fixed) { console.group(title); - console.warn('File is malformed -> use `--format` to fix the issue'); + console.log('File is malformed -> use `--format` to fix the issue'); if (lint.error) { - console.warn(lint.error); + console.log(lint.error); } console.groupEnd(); } @@ -88,7 +88,7 @@ function printReport(report, config) { } if (report.messages) { - report.messages.forEach(str => console.info(str)); + report.messages.forEach(str => console.log(str)); } if (!report.apiList) { @@ -106,7 +106,7 @@ function printReport(report, config) { console.groupEnd(); } else { - console.info("Extensions: None"); + console.log("Extensions: None"); } } if (config.custom) { @@ -160,7 +160,7 @@ function printAjvValidationResult(result, category, reportValid, config) { }) .map(error => makeAjvErrorMessage(error)) // Convert to string .filter((value, i, array) => array.indexOf(value) === i) // Remove duplicates - .forEach((msg, i) => console.warn(`${i+1}. ${msg}`)); // Print it as list + .forEach((msg, i) => console.log(`${i+1}. ${msg}`)); // Print it as list } console.groupEnd(); } From 2d9f06f90b4e2032a32205ffca72b92145b57006 Mon Sep 17 00:00:00 2001 From: Matthias Mohr Date: Wed, 14 Feb 2024 13:03:42 +0100 Subject: [PATCH 07/36] Add funding info --- package.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/package.json b/package.json index 2cc8b32..ea4fca9 100644 --- a/package.json +++ b/package.json @@ -16,6 +16,10 @@ "type": "git", "url": "https://github.com/stac-utils/stac-node-validator.git" }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/m-mohr" + }, "main": "src/index.js", "bin": { "stac-node-validator": "./bin/cli.js" From 798731f9b1d92b39612a9dab2a4ade3b90ea0044 Mon Sep 17 00:00:00 2001 From: Matthias Mohr Date: Fri, 15 Mar 2024 01:33:15 +0100 Subject: [PATCH 08/36] Restructure --- src/nodeUtils.js | 27 +-------------------------- src/utils.js | 26 ++++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 26 deletions(-) diff --git a/src/nodeUtils.js b/src/nodeUtils.js index 1c5c5f7..af88f8a 100644 --- a/src/nodeUtils.js +++ b/src/nodeUtils.js @@ -2,7 +2,7 @@ const klaw = require('klaw'); const fs = require('fs-extra'); const path = require('path'); -const { isUrl, isObject } = require('./utils'); +const { isUrl, makeAjvErrorMessage } = require('./utils'); const SCHEMA_CHOICE = ['anyOf', 'oneOf']; @@ -173,31 +173,6 @@ function isSchemaChoice(schemaPath) { return typeof schemaPath === 'string' && schemaPath.match(/\/(one|any)Of\/\d+\//); } -function makeAjvErrorMessage(error) { - let message = error.message; - if (isObject(error.params) && Object.keys(error.params).length > 0) { - let params = Object.entries(error.params) - .map(([key, value]) => { - let label = key.replace(/([^A-Z]+)([A-Z])/g, "$1 $2").toLowerCase(); - return `${label}: ${value}`; - }) - .join(', ') - message += ` (${params})`; - } - if (error.instancePath) { - return `${error.instancePath} ${message}`; - } - else if (error.schemaPath) { - return `${message}, for schema ${error.schemaPath}`; - } - else if (message) { - return message; - } - else { - return String(error); - } -} - async function resolveFiles(files, depth = -1) { const resolved = []; const extensions = [".geojson", ".json"]; diff --git a/src/utils.js b/src/utils.js index f79c230..a74a11c 100644 --- a/src/utils.js +++ b/src/utils.js @@ -82,11 +82,37 @@ function getSummary(result, config) { return summary; } +function makeAjvErrorMessage(error) { + let message = error.message; + if (isObject(error.params) && Object.keys(error.params).length > 0) { + let params = Object.entries(error.params) + .map(([key, value]) => { + let label = key.replace(/([^A-Z]+)([A-Z])/g, "$1 $2").toLowerCase(); + return `${label}: ${value}`; + }) + .join(', ') + message += ` (${params})`; + } + if (error.instancePath) { + return `${error.instancePath} ${message}`; + } + else if (error.schemaPath) { + return `${message}, for schema ${error.schemaPath}`; + } + else if (message) { + return message; + } + else { + return String(error); + } +} + module.exports = { createAjv, getSummary, isObject, isUrl, loadSchemaFromUri, + makeAjvErrorMessage, normalizePath }; From 776de0be46693071e9f1640e8f20b870cada52d3 Mon Sep 17 00:00:00 2001 From: Matthias Mohr Date: Fri, 15 Mar 2024 10:56:42 +0100 Subject: [PATCH 09/36] Implement temporary workaround for https://github.com/OSGeo/PROJ/issues/4088 --- src/loader/default.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/loader/default.js b/src/loader/default.js index 2a95417..0a748ac 100644 --- a/src/loader/default.js +++ b/src/loader/default.js @@ -1,6 +1,12 @@ const axios = require('axios'); async function loader(uri) { + // Todo: Temporary workaround for https://github.com/OSGeo/PROJ/issues/4088 + const projjson = uri.startsWith('https://proj.org/schemas/'); + if (projjson) { + uri = uri.replace('https://proj.org/schemas/', 'https://proj.org/en/latest/schemas/'); + } + let response = await axios.get(uri); return response.data; } From bf48fe0e39a582975c7cbe8b24402c10196d1a73 Mon Sep 17 00:00:00 2001 From: Matthias Mohr Date: Fri, 15 Mar 2024 12:28:17 +0100 Subject: [PATCH 10/36] v2.0.0-beta.6 --- README.md | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index a9c2a68..0374e52 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ See the [STAC Validator Comparison](COMPARISON.md) for the features supported by ## Versions -**Current version: 2.0.0-beta.5** +**Current version: 2.0.0-beta.6** | STAC Node Validator Version | Supported STAC Versions | | --------------------------- | ----------------------- | diff --git a/package.json b/package.json index ea4fca9..0628aea 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "stac-node-validator", - "version": "2.0.0-beta.5", + "version": "2.0.0-beta.6", "description": "STAC Validator for NodeJS", "author": "Matthias Mohr", "license": "Apache-2.0", From d2d441b7928ca005f252642d136e4551730d2577 Mon Sep 17 00:00:00 2001 From: Matthias Mohr Date: Fri, 15 Mar 2024 14:37:25 +0100 Subject: [PATCH 11/36] clean-up --- src/index.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/index.js b/src/index.js index 73e0b4b..b417518 100644 --- a/src/index.js +++ b/src/index.js @@ -180,7 +180,7 @@ async function validateOne(source, config, report = null) { return report; } - // Validate against tzhe core schemas + // Validate against the core schemas await validateSchema('core', data.type, data, report, config); // Get all extension schemas to validate against @@ -235,7 +235,7 @@ async function validateSchema(key, schema, data, report, config) { } // Validate - const setValidity = (schema, errors = []) => { + const setValidity = (errors = []) => { if (report.valid !== false) { report.valid = errors.length === 0; } @@ -253,13 +253,13 @@ async function validateSchema(key, schema, data, report, config) { const validate = await loadSchema(config, schemaId); const valid = validate(data); if (valid) { - setValidity(schema); + setValidity(); } else { - setValidity(schema, validate.errors); + setValidity(validate.errors); } } catch (error) { - setValidity(schema, [{ + setValidity([{ message: error.message }]); } From 8b364b4dbda5d3f844b7d22091e9a09c0f0f2944 Mon Sep 17 00:00:00 2001 From: Matthias Mohr Date: Wed, 20 Mar 2024 14:55:26 +0100 Subject: [PATCH 12/36] Fix validation of iri without a host (i.e. localhost) --- src/iri.js | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/iri.js b/src/iri.js index 6f4ca4e..ef69cd3 100644 --- a/src/iri.js +++ b/src/iri.js @@ -1,14 +1,14 @@ const { parse } = require('uri-js'); // We don't allow empty URIs, same-document and mailto here -module.exports = { +const IRI = { 'iri': value => { if (typeof value !== 'string' || value.length === 0) { - return; + return false; } const iri = parse(value); - if ((iri.reference === 'absolute' || iri.reference === 'uri') && iri.scheme && iri.host) { + if ((iri.reference === 'absolute' || iri.reference === 'uri') && iri.scheme && (iri.host || iri.path)) { return true; } @@ -16,14 +16,16 @@ module.exports = { }, 'iri-reference': value => { if (typeof value !== 'string' || value.length === 0) { - return; + return false; } const iri = parse(value); - if ((iri.reference === 'absolute' || iri.reference === 'uri') && iri.scheme && iri.host) { + if ((iri.reference === 'absolute' || iri.reference === 'uri') && iri.scheme && (iri.host || iri.path)) { return true; } return (iri.path && (iri.reference === 'relative' || iri.reference === 'uri')); } }; + +module.exports = IRI; From 48faec6020074d0d723f96fc986aae7d4defd429 Mon Sep 17 00:00:00 2001 From: Matthias Mohr Date: Wed, 20 Mar 2024 14:56:47 +0100 Subject: [PATCH 13/36] v2.0.0-beta.7 --- README.md | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 0374e52..9a995cf 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ See the [STAC Validator Comparison](COMPARISON.md) for the features supported by ## Versions -**Current version: 2.0.0-beta.6** +**Current version: 2.0.0-beta.7** | STAC Node Validator Version | Supported STAC Versions | | --------------------------- | ----------------------- | diff --git a/package.json b/package.json index 0628aea..df0a506 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "stac-node-validator", - "version": "2.0.0-beta.6", + "version": "2.0.0-beta.7", "description": "STAC Validator for NodeJS", "author": "Matthias Mohr", "license": "Apache-2.0", From f848077a1f4d18fed077b53c39a15492cf97b070 Mon Sep 17 00:00:00 2001 From: Matthias Mohr Date: Tue, 26 Mar 2024 15:12:50 +0100 Subject: [PATCH 14/36] Revert "Implement temporary workaround for https://github.com/OSGeo/PROJ/issues/4088" This reverts commit 776de0be46693071e9f1640e8f20b870cada52d3. --- src/loader/default.js | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/loader/default.js b/src/loader/default.js index 0a748ac..2a95417 100644 --- a/src/loader/default.js +++ b/src/loader/default.js @@ -1,12 +1,6 @@ const axios = require('axios'); async function loader(uri) { - // Todo: Temporary workaround for https://github.com/OSGeo/PROJ/issues/4088 - const projjson = uri.startsWith('https://proj.org/schemas/'); - if (projjson) { - uri = uri.replace('https://proj.org/schemas/', 'https://proj.org/en/latest/schemas/'); - } - let response = await axios.get(uri); return response.data; } From 98ede5ff08c593abfd349ede4d6d638f9e6fd580 Mon Sep 17 00:00:00 2001 From: Matthias Mohr Date: Fri, 19 Jul 2024 16:21:38 +0200 Subject: [PATCH 15/36] Handle schemas that have no $id defined --- src/index.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/index.js b/src/index.js index b417518..cbb05d1 100644 --- a/src/index.js +++ b/src/index.js @@ -284,6 +284,9 @@ async function loadSchema(config, schemaId) { console.trace(error); throw new Error(`Schema at '${schemaId}' not found. Please ensure all entries in 'stac_extensions' are valid.`); } + if (!json.$id) { + json.$id = schemaId; + } schema = config.ajv.getSchema(json.$id); if (schema) { From 0f805003f1525342b90dda095008ca0875939cc0 Mon Sep 17 00:00:00 2001 From: Matthias Mohr Date: Fri, 19 Jul 2024 16:56:53 +0200 Subject: [PATCH 16/36] Import doesn't work --- src/baseValidator.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/baseValidator.js b/src/baseValidator.js index 91a0835..cd7c394 100644 --- a/src/baseValidator.js +++ b/src/baseValidator.js @@ -1,4 +1,4 @@ -const { STAC } = import('stac-js'); +// const { STAC } = require('stac-js'); class BaseValidator { From 445af987b2d0fc7ae072fadcf35f9de82cc5a652 Mon Sep 17 00:00:00 2001 From: Matthias Mohr Date: Fri, 19 Jul 2024 17:08:41 +0200 Subject: [PATCH 17/36] v2.0.0-beta.8 --- README.md | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 9a995cf..416a833 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ See the [STAC Validator Comparison](COMPARISON.md) for the features supported by ## Versions -**Current version: 2.0.0-beta.7** +**Current version: 2.0.0-beta.8** | STAC Node Validator Version | Supported STAC Versions | | --------------------------- | ----------------------- | diff --git a/package.json b/package.json index df0a506..71d8439 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "stac-node-validator", - "version": "2.0.0-beta.7", + "version": "2.0.0-beta.8", "description": "STAC Validator for NodeJS", "author": "Matthias Mohr", "license": "Apache-2.0", From fd598cf6870e8363fa332305a2afa2a09bb1614e Mon Sep 17 00:00:00 2001 From: Matthias Mohr Date: Mon, 5 Aug 2024 10:00:16 +0200 Subject: [PATCH 18/36] Report data loading issues --- src/index.js | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/index.js b/src/index.js index cbb05d1..1d7d8b4 100644 --- a/src/index.js +++ b/src/index.js @@ -131,7 +131,16 @@ async function validateOne(source, config, report = null) { if (!report.id) { if (typeof data === 'string') { report.id = normalizePath(data); - data = await config.loader(data); + try { + data = await config.loader(data); + } catch (error) { + report.valid = false; + report.results.core.push({ + instancePath: "", + message: error.message + }); + return report; + } } else { report.id = data.id; @@ -281,7 +290,6 @@ async function loadSchema(config, schemaId) { try { json = await loadSchemaFromUri(schemaId, config); } catch (error) { - console.trace(error); throw new Error(`Schema at '${schemaId}' not found. Please ensure all entries in 'stac_extensions' are valid.`); } if (!json.$id) { From 57c0922e76813ed9dde708f937522b931b39f4e5 Mon Sep 17 00:00:00 2001 From: Matthias Mohr Date: Mon, 5 Aug 2024 10:00:31 +0200 Subject: [PATCH 19/36] Add missing uri-js dependency, v2.0.0-beta.9 --- README.md | 2 +- package.json | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 416a833..900e23f 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ See the [STAC Validator Comparison](COMPARISON.md) for the features supported by ## Versions -**Current version: 2.0.0-beta.8** +**Current version: 2.0.0-beta.9** | STAC Node Validator Version | Supported STAC Versions | | --------------------------- | ----------------------- | diff --git a/package.json b/package.json index 71d8439..dbddabb 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "stac-node-validator", - "version": "2.0.0-beta.8", + "version": "2.0.0-beta.9", "description": "STAC Validator for NodeJS", "author": "Matthias Mohr", "license": "Apache-2.0", @@ -38,6 +38,7 @@ "jest-diff": "^29.0.1", "klaw": "^4.0.1", "stac-js": "^0.0.8", + "uri-js": "^4.4.1", "yargs": "^17.7.2" }, "devDependencies": { From 35a29defa1a103398ffe04868cb9b144b949c6a4 Mon Sep 17 00:00:00 2001 From: Matthias Mohr Date: Mon, 5 Aug 2024 10:04:13 +0200 Subject: [PATCH 20/36] Fix missing import, v2.0.0-beta.10 --- README.md | 2 +- package.json | 2 +- src/cli.js | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 900e23f..5590344 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ See the [STAC Validator Comparison](COMPARISON.md) for the features supported by ## Versions -**Current version: 2.0.0-beta.9** +**Current version:** 2.0.0-beta.10 | STAC Node Validator Version | Supported STAC Versions | | --------------------------- | ----------------------- | diff --git a/package.json b/package.json index dbddabb..46e0f7e 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "stac-node-validator", - "version": "2.0.0-beta.9", + "version": "2.0.0-beta.10", "description": "STAC Validator for NodeJS", "author": "Matthias Mohr", "license": "Apache-2.0", diff --git a/src/cli.js b/src/cli.js index 2dba184..12abc71 100644 --- a/src/cli.js +++ b/src/cli.js @@ -5,7 +5,7 @@ const ConfigSource = require('./config.js'); const validate = require('../src/index.js'); const { printConfig, printSummary, resolveFiles, printReport, abort } = require('./nodeUtils'); const nodeLoader = require('./loader/node'); -const { getSummary } = require('./utils'); +const { getSummary, normalizePath } = require('./utils'); const lint = require('./lint'); From c3cffc3d6dee7407b04784ca0464a93c248d088e Mon Sep 17 00:00:00 2001 From: Matthias Mohr Date: Mon, 5 Aug 2024 10:11:48 +0200 Subject: [PATCH 21/36] Fix linting report, v2.0.0-beta.11 --- README.md | 2 +- package.json | 2 +- src/utils.js | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 5590344..ba330fb 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ See the [STAC Validator Comparison](COMPARISON.md) for the features supported by ## Versions -**Current version:** 2.0.0-beta.10 +**Current version:** 2.0.0-beta.11 | STAC Node Validator Version | Supported STAC Versions | | --------------------------- | ----------------------- | diff --git a/package.json b/package.json index 46e0f7e..36b46f6 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "stac-node-validator", - "version": "2.0.0-beta.10", + "version": "2.0.0-beta.11", "description": "STAC Validator for NodeJS", "author": "Matthias Mohr", "license": "Apache-2.0", diff --git a/src/utils.js b/src/utils.js index a74a11c..85735d0 100644 --- a/src/utils.js +++ b/src/utils.js @@ -66,7 +66,7 @@ function getSummary(result, config) { summary.valid = result.children.filter(c => c.valid === true).length; summary.invalid = result.children.filter(c => c.valid === false).length; if (config.lint || config.format) { - summary.malformed = result.children.filter(c => c.lint && c.lint.valid).length; + summary.malformed = result.children.filter(c => c.lint && !c.lint.valid).length; } summary.skipped = result.children.filter(c => c.skipped).length; } From a9a99a3c437936e2ceb7f3f60adaf5f8a3d38302 Mon Sep 17 00:00:00 2001 From: Matthias Mohr Date: Wed, 7 Aug 2024 11:26:41 +0200 Subject: [PATCH 22/36] Allow to customize Ajv via custom validator, v2.0.0-beta.12 --- README.md | 2 +- package.json | 2 +- src/baseValidator.js | 4 ++++ src/index.js | 3 +++ 4 files changed, 9 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index ba330fb..e3fe514 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ See the [STAC Validator Comparison](COMPARISON.md) for the features supported by ## Versions -**Current version:** 2.0.0-beta.11 +**Current version:** 2.0.0-beta.12 | STAC Node Validator Version | Supported STAC Versions | | --------------------------- | ----------------------- | diff --git a/package.json b/package.json index 36b46f6..826a682 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "stac-node-validator", - "version": "2.0.0-beta.11", + "version": "2.0.0-beta.12", "description": "STAC Validator for NodeJS", "author": "Matthias Mohr", "license": "Apache-2.0", diff --git a/src/baseValidator.js b/src/baseValidator.js index cd7c394..cdd4624 100644 --- a/src/baseValidator.js +++ b/src/baseValidator.js @@ -8,6 +8,10 @@ class BaseValidator { constructor() { } + async createAjv(ajv) { + return ajv; + } + /** * Any preprocessing work you want to do on the data. * diff --git a/src/index.js b/src/index.js index 1d7d8b4..6c86eac 100644 --- a/src/index.js +++ b/src/index.js @@ -72,6 +72,9 @@ async function validate(data, config) { }; config = Object.assign({}, defaultConfig, config); config.ajv = createAjv(config); + if (config.customValidator) { + config.ajv = await config.customValidator.createAjv(config.ajv); + } let report = createReport(); if (typeof data === 'string') { From 1894549510ea389d0ab7e82061d7d2bc00858116 Mon Sep 17 00:00:00 2001 From: Matthias Mohr Date: Thu, 31 Oct 2024 18:33:20 +0100 Subject: [PATCH 23/36] Don't run MacOS 11 CI anylonger --- .github/workflows/test.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 1d934db..b2f0ade 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -11,7 +11,6 @@ jobs: fail-fast: false matrix: runner: [ - 'macos-11', 'macos-12', 'macos-13', 'macos-latest', From 7a84f420e410fb81d5f22bab957eefa51bf904a9 Mon Sep 17 00:00:00 2001 From: Matthias Mohr Date: Thu, 31 Oct 2024 18:44:18 +0100 Subject: [PATCH 24/36] Check against more current node versions --- .github/workflows/test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index ed9a394..458a50e 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -21,7 +21,7 @@ jobs: 'windows-2022', 'windows-latest', ] - node: [ '16', '18', '20', '22', 'lts/*' ] + node: [ '18', '20', '22', '23', 'lts/*' ] runs-on: ${{ matrix.runner }} name: ${{ matrix.runner }} runner with Node.js ${{ matrix.node }} steps: From 80fef01aa210a4d33faf3c861d805ee2ca610841 Mon Sep 17 00:00:00 2001 From: Matthias Mohr Date: Thu, 31 Oct 2024 19:12:54 +0100 Subject: [PATCH 25/36] Add possibility to bypass validator --- src/baseValidator.js | 16 ++++++++++++++++ src/index.js | 7 +++++++ 2 files changed, 23 insertions(+) diff --git a/src/baseValidator.js b/src/baseValidator.js index cdd4624..792e165 100644 --- a/src/baseValidator.js +++ b/src/baseValidator.js @@ -24,6 +24,22 @@ class BaseValidator { return data; } + /** + * Bypass the STAC validation, do something different but still return a report. + * + * Could be used to validate against a different schema, e.g. OGC API - Records. + * + * Return a Report to bypass validation, or null to continue with STAC validation. + * + * @param {Object} data + * @param {import('.').Report} report + * @param {import('.').Config} config + * @returns {import('.').Report|null} + */ + async bypassValidation(data, report, config) { + return null; + } + /** * Any custom validation routines you want to run. * diff --git a/src/index.js b/src/index.js index 6c86eac..cb46448 100644 --- a/src/index.js +++ b/src/index.js @@ -160,6 +160,13 @@ async function validateOne(source, config, report = null) { report = await config.lintFn(source, report, config); } + if (config.customValidator) { + const bypass = await config.customValidator.bypassValidation(data, report, config); + if (bypass) { + return bypass; + } + } + // Check stac_version if (typeof data.stac_version !== 'string') { report.skipped = true; From c5e8dd8d2a396def20dc2503be5681cf7d0468e1 Mon Sep 17 00:00:00 2001 From: Matthias Mohr Date: Thu, 31 Oct 2024 23:49:12 +0100 Subject: [PATCH 26/36] v2.0.0-beta.13 --- README.md | 2 +- package.json | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 8580e5d..fb0a00b 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ See the [STAC Validator Comparison](COMPARISON.md) for the features supported by ## Versions -**Current version:** 2.0.0-beta.12 +**Current version:** 2.0.0-beta.13 | STAC Node Validator Version | Supported STAC Versions | | --------------------------- | ----------------------- | diff --git a/package.json b/package.json index 826a682..7be01df 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "stac-node-validator", - "version": "2.0.0-beta.12", + "version": "2.0.0-beta.13", "description": "STAC Validator for NodeJS", "author": "Matthias Mohr", "license": "Apache-2.0", @@ -32,7 +32,7 @@ "ajv": "^8.8.2", "ajv-formats": "^2.1.1", "assert": "^2.0.0", - "axios": "^1.1.3", + "axios": "^1.7.4", "compare-versions": "^6.1.0", "fs-extra": "^10.0.0", "jest-diff": "^29.0.1", From d88b9ed520a470ff1465390588f3ae8123826582 Mon Sep 17 00:00:00 2001 From: Matthias Mohr Date: Thu, 30 Jan 2025 15:15:05 +0100 Subject: [PATCH 27/36] Support for partial URLs in schema maps --- src/utils.js | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/utils.js b/src/utils.js index 85735d0..c02bd76 100644 --- a/src/utils.js +++ b/src/utils.js @@ -1,6 +1,7 @@ const Ajv = require('ajv'); const addFormats = require('ajv-formats'); const iriFormats = require('./iri'); +const path = require('path'); const SUPPORTED_PROTOCOLS = ['http', 'https']; @@ -39,10 +40,15 @@ function createAjv(config) { } async function loadSchemaFromUri(uri, config) { - if (isObject(config.schemaMap) && config.schemaMap[uri]) { - uri = config.schemaMap[uri]; + if (isObject(config.schemaMap)) { + const patterns = Object.entries(config.schemaMap); + const match = patterns.find(map => uri.startsWith(map[0])); + if (match) { + const [pattern, target] = match; + uri = path.join(target, uri.substring(pattern.length)); + } } - else if (config.schemas) { + if (config.schemas) { uri = uri.replace(/^https:\/\/schemas\.stacspec\.org\/v[^\/]+/, config.schemas); } return await config.loader(uri); From 3628d0a85ac479a1a33260d8679ea5b0c5462583 Mon Sep 17 00:00:00 2001 From: Matthias Mohr Date: Thu, 30 Jan 2025 15:15:47 +0100 Subject: [PATCH 28/36] Support for JS config files --- src/config.js | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/src/config.js b/src/config.js index a30fc88..ec52ae9 100644 --- a/src/config.js +++ b/src/config.js @@ -1,6 +1,7 @@ const yargs = require('yargs/yargs'); const { hideBin } = require('yargs/helpers'); const fs = require('fs-extra'); +const path = require('path'); const { strArrayToObject } = require('./nodeUtils'); @@ -34,7 +35,7 @@ function fromCLI() { type: 'array', default: [], requiresArg: true, - description: 'Validate against a specific local schema (e.g. an external extension). Provide the schema URI and the local path separated by an equal sign.\nExample: https://stac-extensions.github.io/foobar/v1.0.0/schema.json=./json-schema/schema.json', + description: 'Validate against a specific local schema (e.g. an external extension). Provide the schema URI and the local path separated by an equal sign.\nExample: https://stac-extensions.github.io/foobar/v1.0.0/schema.json=./json-schema/schema.json\nThis can also be a partial URL and path so that all children are also mapped.\nExample: https://stac-extensions.github.io/foobar/=./json-schema/', coerce: strArrayToObject }) .option('custom', { @@ -67,7 +68,7 @@ function fromCLI() { alias: 'c', type: 'string', default: null, - description: 'Load the options from a config file. CLI Options override config options.' + description: 'Load the options from a config file (.js or .json). CLI options override config options.' }) .version() .parse() @@ -79,17 +80,23 @@ function fromCLI() { return config; } -async function fromFile(path) { - let configFile; - try { - configFile = await fs.readFile(path, "utf8"); - } catch (error) { - throw new Error('Config file does not exist.'); +async function fromFile(filepath) { + filepath = path.resolve(filepath); + if (filepath.endsWith('.js')) { + return require(filepath); } - try { - return JSON.parse(configFile); - } catch (error) { - throw new Error('Config file is invalid JSON.'); + else { + let configFile; + try { + configFile = await fs.readFile(filepath, "utf8"); + } catch (error) { + throw new Error('Config file does not exist.'); + } + try { + return JSON.parse(configFile); + } catch (error) { + throw new Error('Config file is invalid JSON.'); + } } } From 1f4423147cb35ceb3b069146a489ec8d43fafa16 Mon Sep 17 00:00:00 2001 From: Matthias Mohr Date: Thu, 30 Jan 2025 15:18:38 +0100 Subject: [PATCH 29/36] Handle invalid input paths better --- src/cli.js | 14 ++++++++++++-- src/nodeUtils.js | 48 ++++++++++++++++++++++++++++++------------------ 2 files changed, 42 insertions(+), 20 deletions(-) diff --git a/src/cli.js b/src/cli.js index 12abc71..bd1e2b4 100644 --- a/src/cli.js +++ b/src/cli.js @@ -35,11 +35,21 @@ async function run() { config.depth = config.depth >= 0 ? config.depth : -1; // Verify files exist / read folders - let data = await resolveFiles(config.files, config.depth); + const files = await resolveFiles(config.files, config.depth); delete config.files; - if (data.length === 1) { + for (let file in files.error) { + const error = files.error[file]; + console.warn(`${file}: Can't be validated for the following reason: ${error}`); + } + if (files.files.length === 0) { + abort('No files found that are suitable for validation.'); + } + else if (files.files.length === 1) { data = data[0]; } + else { + data = files.files; + } // Resolve schema folder if (config.schemas) { diff --git a/src/nodeUtils.js b/src/nodeUtils.js index af88f8a..a49125e 100644 --- a/src/nodeUtils.js +++ b/src/nodeUtils.js @@ -174,33 +174,45 @@ function isSchemaChoice(schemaPath) { } async function resolveFiles(files, depth = -1) { - const resolved = []; + const result = { + files: [], + error: {} + }; const extensions = [".geojson", ".json"]; const klawOptions = { depthLimit: depth } for (const file of files) { - if (isUrl(file)) { - resolved.push(file); - continue; - } - - // Special handling for reading directories - const stat = await fs.lstat(file); - if (stat.isDirectory()) { - ; - for await (const child of klaw(file, klawOptions)) { - const ext = path.extname(child.path).toLowerCase(); - if (extensions.includes(ext)) { - resolved.push(child.path); + const url = URL.parse(file); + if (url && url.protocol !== 'file:') { + if (['https:', 'http:'].includes(url.protocol)) { + result.files.push(file); + continue; + } + else { + result.error[file] = new Error(`Protocol "${url.protocol}" is not supported.`); + } + } else { + try { + const stat = await fs.lstat(file); + if (stat.isDirectory()) { + // Special handling for reading directories + for await (const child of klaw(file, klawOptions)) { + const ext = path.extname(child.path).toLowerCase(); + if (extensions.includes(ext)) { + result.files.push(child.path); + } + } + } + else { + result.files.push(file); } + } catch (error) { + result.error[file] = error; } } - else { - resolved.push(file); - } } - return resolved; + return result; } function strArrayToObject(list, sep = "=") { From 7202d531d6ef10267b0c5b4261797b0b7873f92b Mon Sep 17 00:00:00 2001 From: Matthias Mohr Date: Thu, 30 Jan 2025 15:59:43 +0100 Subject: [PATCH 30/36] Better URL parsing and handling of unsupported protocols --- src/index.js | 6 +++--- src/lint.js | 4 ++-- src/loader/node.js | 19 +++++++++++++++---- src/nodeUtils.js | 8 ++++---- src/utils.js | 20 ++++++++------------ 5 files changed, 32 insertions(+), 25 deletions(-) diff --git a/src/index.js b/src/index.js index cb46448..239451e 100644 --- a/src/index.js +++ b/src/index.js @@ -1,6 +1,6 @@ const versions = require('compare-versions'); -const { createAjv, isUrl, loadSchemaFromUri, normalizePath, isObject } = require('./utils'); +const { createAjv, isHttpUrl, loadSchemaFromUri, normalizePath, isObject } = require('./utils'); const defaultLoader = require('./loader/default'); const BaseValidator = require('./baseValidator'); const Test = require('./test'); @@ -248,7 +248,7 @@ async function validateSchema(key, schema, data, report, config) { schemaId = `https://schemas.stacspec.org/v${report.version}/${type}-spec/json-schema/${type}.json`; break; default: // extension - if (isUrl(schema)) { + if (isHttpUrl(schema)) { schemaId = schema; } } @@ -267,7 +267,7 @@ async function validateSchema(key, schema, data, report, config) { }; try { if (key !== 'core' && !schemaId) { - throw new Error("'stac_extensions' must contain a valid schema URL, not a shortcut."); + throw new Error("'stac_extensions' must contain a valid HTTP(S) URL to a schema."); } const validate = await loadSchema(config, schemaId); const valid = validate(data); diff --git a/src/lint.js b/src/lint.js index 7f92342..e2930e3 100644 --- a/src/lint.js +++ b/src/lint.js @@ -1,6 +1,6 @@ const fs = require('fs-extra'); const { diffStringsUnified } = require('jest-diff'); -const { isUrl, isObject } = require('./utils'); +const { isHttpUrl, isObject } = require('./utils'); /** * @typedef LintResult @@ -20,7 +20,7 @@ async function lint(file, config) { if (isObject(file)) { return null; } - else if (isUrl(file)) { + else if (isHttpUrl(file)) { return null; } diff --git a/src/loader/node.js b/src/loader/node.js index c16a6bd..bb5048e 100644 --- a/src/loader/node.js +++ b/src/loader/node.js @@ -1,14 +1,25 @@ const axios = require('axios'); const fs = require('fs-extra'); -const { isUrl } = require("../utils"); +const { isHttpUrl } = require('../utils'); async function loader(uri) { - if (isUrl(uri)) { - let response = await axios.get(uri); + if (isHttpUrl(uri)) { + const response = await axios.get(uri); return response.data; } else { - return JSON.parse(await fs.readFile(uri, "utf8")); + if (await fs.exists(uri)) { + return JSON.parse(await fs.readFile(uri, "utf8")); + } + else { + const url = URL.parse(uri); + if (url.protocol && url.protocol !== 'file:' && url.protocol.length > 1) { + throw new Error(`Protocol not supported: ${url.protocol}`); + } + else { + throw new Error(`File not found: ${uri}`); + } + } } } diff --git a/src/nodeUtils.js b/src/nodeUtils.js index a49125e..1172254 100644 --- a/src/nodeUtils.js +++ b/src/nodeUtils.js @@ -2,7 +2,7 @@ const klaw = require('klaw'); const fs = require('fs-extra'); const path = require('path'); -const { isUrl, makeAjvErrorMessage } = require('./utils'); +const { makeAjvErrorMessage, isHttpUrl, SUPPORTED_PROTOCOLS } = require('./utils'); const SCHEMA_CHOICE = ['anyOf', 'oneOf']; @@ -123,7 +123,7 @@ function printAjvValidationResult(result, category, reportValid, config) { if (!category) { return; } - if (!config.verbose && isUrl(category)) { + if (!config.verbose && isHttpUrl(category)) { const match = category.match(/^https?:\/\/stac-extensions\.github\.io\/([^/]+)\/v?([^/]+)(?:\/([^/.]+))?\/schema/); if (match) { let title = match[1]; @@ -184,8 +184,8 @@ async function resolveFiles(files, depth = -1) { } for (const file of files) { const url = URL.parse(file); - if (url && url.protocol !== 'file:') { - if (['https:', 'http:'].includes(url.protocol)) { + if (url && url.protocol && url.protocol.length > 1 && url.protocol !== 'file:') { + if (SUPPORTED_PROTOCOLS.includes(url.protocol)) { result.files.push(file); continue; } diff --git a/src/utils.js b/src/utils.js index c02bd76..1fc34c5 100644 --- a/src/utils.js +++ b/src/utils.js @@ -3,21 +3,16 @@ const addFormats = require('ajv-formats'); const iriFormats = require('./iri'); const path = require('path'); -const SUPPORTED_PROTOCOLS = ['http', 'https']; +const SUPPORTED_PROTOCOLS = ['http:', 'https:']; function isObject(obj) { return (typeof obj === 'object' && obj === Object(obj) && !Array.isArray(obj)); } -function isUrl(uri) { - if (typeof uri === 'string') { - let part = uri.match(/^(\w+):\/\//i); - if (part) { - if (!SUPPORTED_PROTOCOLS.includes(part[1].toLowerCase())) { - throw new Error(`Given protocol "${part[1]}" is not supported.`); - } - return true; - } +function isHttpUrl(url) { + const parsed = URL.parse(url); + if (parsed && SUPPORTED_PROTOCOLS.includes(parsed.protocol)) { + return true; } return false; } @@ -117,8 +112,9 @@ module.exports = { createAjv, getSummary, isObject, - isUrl, + isHttpUrl, loadSchemaFromUri, makeAjvErrorMessage, - normalizePath + normalizePath, + SUPPORTED_PROTOCOLS }; From 7e39c5e63cabd74acc3f8c208636863acdd26108 Mon Sep 17 00:00:00 2001 From: Matthias Mohr Date: Thu, 30 Jan 2025 16:02:52 +0100 Subject: [PATCH 31/36] v2.0.0-beta.14 --- README.md | 2 +- package.json | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index fb0a00b..f021a99 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ See the [STAC Validator Comparison](COMPARISON.md) for the features supported by ## Versions -**Current version:** 2.0.0-beta.13 +**Current version:** 2.0.0-beta.14 | STAC Node Validator Version | Supported STAC Versions | | --------------------------- | ----------------------- | diff --git a/package.json b/package.json index 7be01df..8ce9cc8 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "stac-node-validator", - "version": "2.0.0-beta.13", + "version": "2.0.0-beta.14", "description": "STAC Validator for NodeJS", "author": "Matthias Mohr", "license": "Apache-2.0", @@ -14,7 +14,7 @@ }, "repository": { "type": "git", - "url": "https://github.com/stac-utils/stac-node-validator.git" + "url": "git+https://github.com/stac-utils/stac-node-validator.git" }, "funding": { "type": "github", @@ -22,7 +22,7 @@ }, "main": "src/index.js", "bin": { - "stac-node-validator": "./bin/cli.js" + "stac-node-validator": "bin/cli.js" }, "files": [ "bin/cli.js", From 8611efb3cd26c50dfbd845273231449e4c3906b2 Mon Sep 17 00:00:00 2001 From: Matthias Mohr Date: Thu, 30 Jan 2025 16:48:39 +0100 Subject: [PATCH 32/36] Fix reference error --- src/loader/node.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/loader/node.js b/src/loader/node.js index bb5048e..5eba9ba 100644 --- a/src/loader/node.js +++ b/src/loader/node.js @@ -13,7 +13,7 @@ async function loader(uri) { } else { const url = URL.parse(uri); - if (url.protocol && url.protocol !== 'file:' && url.protocol.length > 1) { + if (url && url.protocol && url.protocol !== 'file:' && url.protocol.length > 1) { throw new Error(`Protocol not supported: ${url.protocol}`); } else { From f1289993a54dc19c33bb4035951da52be30f2361 Mon Sep 17 00:00:00 2001 From: Matthias Mohr Date: Thu, 30 Jan 2025 18:42:04 +0100 Subject: [PATCH 33/36] Fix various issues of the last release --- src/index.js | 25 +------------------------ src/loader/node.js | 4 +++- src/nodeUtils.js | 3 ++- src/utils.js | 32 +++++++++++++++++++++++++++++--- 4 files changed, 35 insertions(+), 29 deletions(-) diff --git a/src/index.js b/src/index.js index 239451e..221b9e2 100644 --- a/src/index.js +++ b/src/index.js @@ -1,6 +1,6 @@ const versions = require('compare-versions'); -const { createAjv, isHttpUrl, loadSchemaFromUri, normalizePath, isObject } = require('./utils'); +const { createAjv, isHttpUrl, loadSchema, normalizePath, isObject } = require('./utils'); const defaultLoader = require('./loader/default'); const BaseValidator = require('./baseValidator'); const Test = require('./test'); @@ -291,27 +291,4 @@ function summarizeResults(report) { return report; } -async function loadSchema(config, schemaId) { - let schema = config.ajv.getSchema(schemaId); - if (schema) { - return schema; - } - - try { - json = await loadSchemaFromUri(schemaId, config); - } catch (error) { - throw new Error(`Schema at '${schemaId}' not found. Please ensure all entries in 'stac_extensions' are valid.`); - } - if (!json.$id) { - json.$id = schemaId; - } - - schema = config.ajv.getSchema(json.$id); - if (schema) { - return schema; - } - - return await config.ajv.compileAsync(json); -} - module.exports = validate; diff --git a/src/loader/node.js b/src/loader/node.js index 5eba9ba..379caf5 100644 --- a/src/loader/node.js +++ b/src/loader/node.js @@ -13,10 +13,12 @@ async function loader(uri) { } else { const url = URL.parse(uri); - if (url && url.protocol && url.protocol !== 'file:' && url.protocol.length > 1) { + // url.protocol.length > 2 check that it's not a Windows path, e.g. c: as in c://foo/bar + if (url && url.protocol && url.protocol.length > 2 && url.protocol !== 'file:') { throw new Error(`Protocol not supported: ${url.protocol}`); } else { + const path = require('path'); throw new Error(`File not found: ${uri}`); } } diff --git a/src/nodeUtils.js b/src/nodeUtils.js index 1172254..7b6dc05 100644 --- a/src/nodeUtils.js +++ b/src/nodeUtils.js @@ -184,7 +184,8 @@ async function resolveFiles(files, depth = -1) { } for (const file of files) { const url = URL.parse(file); - if (url && url.protocol && url.protocol.length > 1 && url.protocol !== 'file:') { + // url.protocol.length > 2 check that it's not a Windows path, e.g. c: as in c://foo/bar + if (url && url.protocol && url.protocol.length > 2 && url.protocol !== 'file:') { if (SUPPORTED_PROTOCOLS.includes(url.protocol)) { result.files.push(file); continue; diff --git a/src/utils.js b/src/utils.js index 1fc34c5..7ecfbb6 100644 --- a/src/utils.js +++ b/src/utils.js @@ -34,19 +34,44 @@ function createAjv(config) { return instance; } +async function loadSchema(config, schemaId) { + let schema = config.ajv.getSchema(schemaId); + if (schema) { + return schema; + } + + try { + json = await loadSchemaFromUri(schemaId, config); + } catch (error) { + throw new Error(`Schema at '${schemaId}' not found. Please ensure all entries in 'stac_extensions' are valid.`); + } + + schema = config.ajv.getSchema(json.$id); + if (schema) { + return schema; + } + + return await config.ajv.compileAsync(json); +} + async function loadSchemaFromUri(uri, config) { + let newUri = uri; if (isObject(config.schemaMap)) { const patterns = Object.entries(config.schemaMap); const match = patterns.find(map => uri.startsWith(map[0])); if (match) { const [pattern, target] = match; - uri = path.join(target, uri.substring(pattern.length)); + newUri = path.join(target, uri.substring(pattern.length)); } } if (config.schemas) { - uri = uri.replace(/^https:\/\/schemas\.stacspec\.org\/v[^\/]+/, config.schemas); + newUri = newUri.replace(/^https:\/\/schemas\.stacspec\.org\/v[^\/]+/, config.schemas); + } + const schema = await config.loader(newUri); + if (!schema.$id) { + schema.$id = uri; } - return await config.loader(uri); + return schema; } function normalizePath(path) { @@ -113,6 +138,7 @@ module.exports = { getSummary, isObject, isHttpUrl, + loadSchema, loadSchemaFromUri, makeAjvErrorMessage, normalizePath, From f6d7cdb6cab71c3cc1e996b8bf3300edfa936903 Mon Sep 17 00:00:00 2001 From: Matthias Mohr Date: Thu, 30 Jan 2025 18:43:15 +0100 Subject: [PATCH 34/36] v2.0.0-beta.15 --- README.md | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index f021a99..47e7089 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ See the [STAC Validator Comparison](COMPARISON.md) for the features supported by ## Versions -**Current version:** 2.0.0-beta.14 +**Current version:** 2.0.0-beta.15 | STAC Node Validator Version | Supported STAC Versions | | --------------------------- | ----------------------- | diff --git a/package.json b/package.json index 8ce9cc8..5517c3b 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "stac-node-validator", - "version": "2.0.0-beta.14", + "version": "2.0.0-beta.15", "description": "STAC Validator for NodeJS", "author": "Matthias Mohr", "license": "Apache-2.0", From 75b6529b4bac8a378aada48a362a87e41c064d24 Mon Sep 17 00:00:00 2001 From: Matthias Mohr Date: Thu, 30 Jan 2025 19:11:59 +0100 Subject: [PATCH 35/36] Update stac-js --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 5517c3b..573bdef 100644 --- a/package.json +++ b/package.json @@ -37,7 +37,7 @@ "fs-extra": "^10.0.0", "jest-diff": "^29.0.1", "klaw": "^4.0.1", - "stac-js": "^0.0.8", + "stac-js": "^0.1.4", "uri-js": "^4.4.1", "yargs": "^17.7.2" }, From 9553f84ff25ff222801782173bef34794289f524 Mon Sep 17 00:00:00 2001 From: Matthias Mohr Date: Thu, 30 Jan 2025 21:51:13 +0100 Subject: [PATCH 36/36] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 47e7089..946648c 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ See the [STAC Validator Comparison](COMPARISON.md) for the features supported by ## Setup -1. Install [node and npm](https://nodejs.org) - should run with any version >= 16. Older versions may still work, but no guarantee. +1. Install [node and npm](https://nodejs.org) - should run with any version >= 22.1.0. 2. `npm install -g stac-node-validator` to install the library permanently ## Usage