diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index a2ee37b..458a50e 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -11,7 +11,6 @@ jobs: fail-fast: false matrix: runner: [ - 'macos-11', 'macos-12', 'macos-13', 'macos-latest', @@ -22,7 +21,7 @@ jobs: 'windows-2022', 'windows-latest', ] - node: [ '16', '18', '20', '22', 'lts/*' ] + node: [ '18', '20', '22', '23', 'lts/*' ] runs-on: ${{ matrix.runner }} name: ${{ matrix.runner }} runner with Node.js ${{ matrix.node }} steps: diff --git a/COMPARISON.md b/COMPARISON.md index 94f2343..e98c6ce 100644 --- a/COMPARISON.md +++ b/COMPARISON.md @@ -14,13 +14,13 @@ Here I'd like to give an overview of what the validators are capable of and what | | Python Validator | PySTAC | STAC Node Validator | | :------------------------- | ------------------------------------------ | ------------------- | ------------------- | -| Validator Version | 1.0.1 | 0.5.2 | 1.3.1 | +| Validator Version | 1.0.1 | 0.5.2 | 2.0.0 | | Language | Python 3.6 | Python 3 | NodeJS | | CLI | Yes | No | Yes | -| Programmatic | Yes | Yes | Planned | -| Online | Yes, [staclint.com](https://staclint.com/) | No | Planned | +| Programmatic | Yes | Yes | Yes | +| Online | Yes, [staclint.com](https://staclint.com/) | No | Yes | | Protocols supported (Read) | HTTP(S), Filesystem | HTTP(S), Filesystem | HTTP(S), Filesystem | -| Gives | HTML / CLI / Python Output | Python Dict | CLI output | +| Gives | HTML / CLI / Python | Python | CLI / JavaScript | ## Specifications supported diff --git a/README.md b/README.md index 364c027..946648c 100644 --- a/README.md +++ b/README.md @@ -6,11 +6,11 @@ See the [STAC Validator Comparison](COMPARISON.md) for the features supported by ## Versions -**Current version: 1.3.1** +**Current version:** 2.0.0-beta.15 | STAC Node Validator Version | Supported STAC Versions | | --------------------------- | ----------------------- | -| 1.1.0 / 1.2.x | >= 1.0.0-rc.1 | +| 1.1.0 / 1.2.x / 2.x.x | >= 1.0.0-rc.1 | | 0.4.x / 1.0.x | >= 1.0.0-beta.2 and < 1.0.0-rc.3 | | 0.3.0 | 1.0.0-beta.2 | | 0.2.1 | 1.0.0-beta.1 | @@ -22,7 +22,7 @@ See the [STAC Validator Comparison](COMPARISON.md) for the features supported by ## Setup -1. Install [node and npm](https://nodejs.org) - should run with any version >= 16. Older versions may still work, but no guarantee. +1. Install [node and npm](https://nodejs.org) - should run with any version >= 22.1.0. 2. `npm install -g stac-node-validator` to install the library permanently ## Usage @@ -44,6 +44,7 @@ Further options to add to the commands above: - Add `--strict` to enable strict mode in validation for schemas and numbers (as defined by [ajv](https://ajv.js.org/strict-mode.html) for options `strictSchema`, `strictNumbers` and `strictTuples`) - To lint local JSON files: `--lint` (add `--verbose` to get a diff with the changes required) - To format / pretty-print local JSON files: `--format` (Attention: this will override the source files without warning!) +- To run custom validation code: `--custom ./path/to/validation.js` - The validation.js needs to contain a class that implements the `BaseValidator` interface. See [custom.example.js](./custom.example.js) for an example. **Note on API support:** Validating lists of STAC items/collections (i.e. `GET /collections` and `GET /collections/:id/items`) is partially supported. It only checks the contained items/collections, but not the other parts of the response (e.g. `links`). @@ -51,35 +52,12 @@ It only checks the contained items/collections, but not the other parts of the r ### Config file You can also pass a config file via the `--config` option. Simply pass a file path as value. -Parameters set via CLI will override the corresponding setting in the config file. -Make sure to use the value `false` to override boolean flags that are set to `true` in the config file. +Parameters set via CLI will not override the corresponding setting in the config file. The config file uses the same option names as above. To specify the files to be validated, add an array with paths. The schema map is an object instead of string separated with a `=` character. -**Example:** -```json -{ - "files": [ - "/path/to/your/catalog.json", - "/path/to/your/item.json" - ], - "schemas": "/path/to/stac/folder", - "schemaMap": { - "https://stac-extensions.github.io/foobar/v1.0.0/schema.json": "./json-schema/schema.json" - }, - "ignoreCerts": false, - "verbose": false, - "lint": true, - "format": false, - "strict": true, - "all": false -} -``` - -You could now override some options as follows in CLI: `stac-node-validator example.json --config /path/to/config.json --lint false` - ### Development 1. `git clone https://github.com/stac-utils/stac-node-validator` to clone the repo diff --git a/bin/cli.js b/bin/cli.js index 642c81e..e234b07 100755 --- a/bin/cli.js +++ b/bin/cli.js @@ -1,2 +1,3 @@ #!/usr/bin/env node -require('../index.js')() +const runCLI = require('../src/cli.js'); +runCLI(); diff --git a/custom.example.js b/custom.example.js new file mode 100644 index 0000000..9d6c6a2 --- /dev/null +++ b/custom.example.js @@ -0,0 +1,25 @@ +const BaseValidator = require('stac-node-validator/src/baseValidator.js'); + +class CustomValidator extends BaseValidator { + + /** + * Any custom validation routines you want to run. + * + * You can either create a list of errors using the test interface + * or just throw on the first error. + * + * @param {STAC} data + * @param {Test} test + * @param {import('.').Report} report + * @param {import('.').Config} config + * @throws {Error} + */ + async afterValidation(data, test, report, config) { + if (data.id === 'solid-earth') { + test.deepEqual(data.example, [1,2,3]); + } + } + +} + +module.exports = CustomValidator; diff --git a/index.js b/index.js deleted file mode 100644 index fe082de..0000000 --- a/index.js +++ /dev/null @@ -1,417 +0,0 @@ -const Ajv = require('ajv'); -const axios = require('axios'); -const addFormats = require('ajv-formats'); -const iriFormats = require('./iri.js'); -const fs = require('fs-extra'); -const klaw = require('klaw'); -const path = require('path') -const minimist = require('minimist'); -const versions = require('compare-versions'); -const {diffStringsUnified} = require('jest-diff'); -const { version } = require('./package.json'); - -let DEBUG = false; - -let ajv = new Ajv({ - formats: iriFormats, - allErrors: true, - strict: false, - logger: DEBUG ? console : false, - loadSchema: loadJsonFromUri -}); -addFormats(ajv); - -let verbose = false; -let schemaMap = {}; -let schemaFolder = null; - -let booleanArgs = ['verbose', 'ignoreCerts', 'lint', 'format', 'version', 'strict', 'all']; - -async function run(config) { - try { - let args = config || minimist(process.argv.slice(2)); - - if (args.version) { - console.log(version); - process.exit(0); - } - else { - console.log(`STAC Node Validator v${version}\n`); - } - - // Show minimal help output - if (args.help) { - console.log("For more information on using this command line tool, please visit"); - console.log("https://github.com/stac-utils/stac-node-validator/blob/master/README.md#usage"); - - process.exit(0); - } - - // Read config from file - if (typeof args.config === 'string') { - let configFile; - try { - configFile = await fs.readFile(args.config, "utf8"); - } catch (error) { - throw new Error('Config file does not exist.'); - } - try { - config = JSON.parse(configFile); - } catch (error) { - throw new Error('Config file is invalid JSON.'); - } - } - - // Merge CLI parameters into config - if (!config) { - config = {}; - } - for(let key in args) { - let value = args[key]; - if (key === '_' && Array.isArray(value) && value.length > 0) { - config.files = value; - } - else if (booleanArgs.includes(key)) { - if (typeof value === 'string' && value.toLowerCase() === 'false') { - config[key] = false; - } - else { - config[key] = Boolean(value); - } - } - else { - config[key] = value; - } - } - - verbose = Boolean(config.verbose); - - let files = Array.isArray(config.files) ? config.files : []; - if (files.length === 0) { - throw new Error('No path or URL specified.'); - } - else if (files.length === 1 && !isUrl(files[0])) { - // Special handling for reading directories - let stat = await fs.lstat(files[0]); - if (stat.isDirectory()) { - if (config.all) { - files = await readFolder(files[0], /.+\.json$/i); - } - else { - files = await readFolder(files[0], /(^|\/|\\)examples(\/|\\).+\.json$/i); - } - } - } - - if (config.ignoreCerts) { - process.env["NODE_TLS_REJECT_UNAUTHORIZED"] = 0; - } - if (config.strict) { - ajv.opts.strictSchema = true; - ajv.opts.strictNumbers = true; - ajv.opts.strictTuples = true; - } - - if (typeof config.schemas === 'string') { - let stat = await fs.lstat(config.schemas); - if (stat.isDirectory()) { - schemaFolder = normalizePath(config.schemas); - } - else { - throw new Error('Schema folder is not a valid STAC directory'); - } - } - - let schemaMapArgs = []; - if (config.schemaMap && typeof config.schemaMap === 'object') { - // Recommended way - schemaMapArgs = config.schemaMap; - } - else if (typeof config.schemaMap === 'string') { - // Backward compliance - schemaMapArgs = config.schemaMap.split(';'); - } - for(let url in schemaMapArgs) { - let path = schemaMapArgs[url]; - if (typeof url === 'string') { // from CLI - [url, path] = path.split("="); - } - let stat = await fs.lstat(path); - if (stat.isFile()) { - schemaMap[url] = path; - } - else { - console.error(`Schema mapping for ${url} is not a valid file: ${normalizePath(path)}`); - } - } - - const doLint = Boolean(config.lint); - const doFormat = Boolean(config.format); - - let stats = { - files: files.length, - invalid: 0, - valid: 0, - malformed: 0 - } - for(let file of files) { - // Read STAC file - let json; - console.log(`- ${normalizePath(file)}`); - try { - let fileIsUrl = isUrl(file); - if (!fileIsUrl && (doLint || doFormat)) { - let fileContent = await fs.readFile(file, "utf8"); - json = JSON.parse(fileContent); - const expectedContent = JSON.stringify(json, null, 2); - if (!matchFile(fileContent, expectedContent)) { - stats.malformed++; - if (doLint) { - console.warn("-- Lint: File is malformed -> use `--format` to fix the issue"); - if (verbose) { - console.log(diffStringsUnified(fileContent, expectedContent)); - } - } - if (doFormat) { - console.warn("-- Format: File was malformed -> fixed the issue"); - await fs.writeFile(file, expectedContent); - } - } - else if (doLint && verbose) { - console.warn("-- Lint: File is well-formed"); - } - } - else { - json = await loadJsonFromUri(file); - if (fileIsUrl && (doLint || doFormat)) { - let what = []; - doLint && what.push('Linting'); - doFormat && what.push('Formatting'); - console.warn(`-- ${what.join(' and ')} not supported for remote files`); - } - } - } - catch(error) { - stats.invalid++; - stats.malformed++; - console.error("-- " + error.message + "\n"); - continue; - } - - let isApiList = false; - let entries; - if (Array.isArray(json.collections)) { - entries = json.collections; - isApiList = true; - if (verbose) { - console.log(`-- The file is a /collections endpoint. Validating all ${entries.length} collections, but ignoring the other parts of the response.`); - if (entries.length > 1) { - console.log(''); - } - } - } - else if (Array.isArray(json.features)) { - entries = json.features; - isApiList = true; - if (verbose) { - console.log(`-- The file is a /collections/:id/items endpoint. Validating all ${entries.length} items, but ignoring the other parts of the response.`); - if (entries.length > 1) { - console.log(''); - } - } - } - else { - entries = [json]; - } - - let fileValid = true; - for(let data of entries) { - let id = ''; - if (isApiList) { - id = `${data.id}: `; - } - if (typeof data.stac_version !== 'string') { - console.error(`-- ${id}Skipping; No STAC version found\n`); - fileValid = false; - continue; - } - else if (versions.compare(data.stac_version, '1.0.0-rc.1', '<')) { - console.error(`-- ${id}Skipping; Can only validate STAC version >= 1.0.0-rc.1\n`); - continue; - } - else if (verbose) { - console.log(`-- ${id}STAC Version: ${data.stac_version}`); - } - - switch(data.type) { - case 'FeatureCollection': - console.warn(`-- ${id}Skipping; STAC ItemCollections not supported yet\n`); - continue; - case 'Catalog': - case 'Collection': - case 'Feature': - break; - default: - console.error(`-- ${id}Invalid; Can't detect type of the STAC object. Is the 'type' field missing or invalid?\n`); - fileValid = false; - continue; - } - - // Get all schema to validate against - let schemas = [data.type]; - if (Array.isArray(data.stac_extensions)) { - schemas = schemas.concat(data.stac_extensions); - // Convert shortcuts supported in 1.0.0 RC1 into schema URLs - if (versions.compare(data.stac_version, '1.0.0-rc.1', '=')) { - schemas = schemas.map(ext => ext.replace(/^(eo|projection|scientific|view)$/, 'https://schemas.stacspec.org/v1.0.0-rc.1/extensions/$1/json-schema/schema.json')); - } - } - - for(let schema of schemas) { - try { - let schemaId; - let core = false; - switch(schema) { - case 'Feature': - schema = 'Item'; - case 'Catalog': - case 'Collection': - let type = schema.toLowerCase(); - schemaId = `https://schemas.stacspec.org/v${data.stac_version}/${type}-spec/json-schema/${type}.json`; - core = true; - break; - default: // extension - if (isUrl(schema)) { - schemaId = schema; - } - else { - throw new Error("'stac_extensions' must contain a valid schema URL, not a shortcut."); - } - } - let validate = await loadSchema(schemaId); - let valid = validate(data); - if (!valid) { - console.log(`--- ${schema}: invalid`); - console.warn(validate.errors); - console.log("\n"); - fileValid = false; - if (core && !DEBUG) { - if (verbose) { - console.warn("-- Validation error in core, skipping extension validation"); - } - break; - } - } - else if (verbose) { - console.log(`--- ${schema}: valid`); - } - } catch (error) { - fileValid = false; - console.error(`--- ${schema}: ${error.message}`); - if (DEBUG) { - console.trace(error); - } - } - } - if (!fileValid || verbose) { - console.log(''); - } - } - fileValid ? stats.valid++ : stats.invalid++; - } - console.info("Files: " + stats.files); - console.info("Valid: " + stats.valid); - console.info("Invalid: " + stats.invalid); - if (doLint || doFormat) { - console.info("Malformed: " + stats.malformed); - } - let errored = (stats.invalid > 0 || (doLint && !doFormat && stats.malformed > 0)) ? 1 : 0; - process.exit(errored); - } - catch(error) { - console.error(error); - process.exit(1); - } -} - -const SUPPORTED_PROTOCOLS = ['http', 'https']; - -function matchFile(given, expected) { - return normalizeNewline(given) === normalizeNewline(expected); -} - -function normalizePath(path) { - return path.replace(/\\/g, '/').replace(/\/$/, ""); -} - -function normalizeNewline(str) { - // 2 spaces, *nix newlines, newline at end of file - return str.trimRight().replace(/(\r\n|\r)/g, "\n") + "\n"; -} - -function isUrl(uri) { - if (typeof uri === 'string') { - let part = uri.match(/^(\w+):\/\//i); - if(part) { - if (!SUPPORTED_PROTOCOLS.includes(part[1].toLowerCase())) { - throw new Error(`Given protocol "${part[1]}" is not supported.`); - } - return true; - } - } - return false; -} - -async function readFolder(folder, pattern) { - var files = []; - for await (let file of klaw(folder, {depthLimit: -1})) { - let relPath = path.relative(folder, file.path); - if (relPath.match(pattern)) { - files.push(file.path); - } - } - return files; -} - -async function loadJsonFromUri(uri) { - if (schemaMap[uri]) { - uri = schemaMap[uri]; - } - else if (schemaFolder) { - uri = uri.replace(/^https:\/\/schemas\.stacspec\.org\/v[^\/]+/, schemaFolder); - } - if (isUrl(uri)) { - let response = await axios.get(uri); - return response.data; - } - else { - return JSON.parse(await fs.readFile(uri, "utf8")); - } -} - -async function loadSchema(schemaId) { - let schema = ajv.getSchema(schemaId); - if (schema) { - return schema; - } - - try { - json = await loadJsonFromUri(schemaId); - } catch (error) { - if (DEBUG) { - console.trace(error); - } - throw new Error(`-- Schema at '${schemaId}' not found. Please ensure all entries in 'stac_extensions' are valid.`); - } - - schema = ajv.getSchema(json.$id); - if (schema) { - return schema; - } - - return await ajv.compileAsync(json); -} - -module.exports = async config => { - await run(config); -}; diff --git a/package.json b/package.json index 2273507..573bdef 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "stac-node-validator", - "version": "1.3.1", + "version": "2.0.0-beta.15", "description": "STAC Validator for NodeJS", "author": "Matthias Mohr", "license": "Apache-2.0", @@ -14,27 +14,32 @@ }, "repository": { "type": "git", - "url": "https://github.com/stac-utils/stac-node-validator.git" + "url": "git+https://github.com/stac-utils/stac-node-validator.git" }, - "main": "index.js", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/m-mohr" + }, + "main": "src/index.js", "bin": { - "stac-node-validator": "./bin/cli.js" + "stac-node-validator": "bin/cli.js" }, "files": [ "bin/cli.js", - "iri.js", - "index.js" + "src/" ], "dependencies": { "ajv": "^8.8.2", "ajv-formats": "^2.1.1", - "axios": "^1.1.3", + "assert": "^2.0.0", + "axios": "^1.7.4", "compare-versions": "^6.1.0", "fs-extra": "^10.0.0", "jest-diff": "^29.0.1", "klaw": "^4.0.1", - "minimist": "^1.2.5", - "uri-js": "^4.4.1" + "stac-js": "^0.1.4", + "uri-js": "^4.4.1", + "yargs": "^17.7.2" }, "devDependencies": { "jest": "^29.0.1" diff --git a/src/baseValidator.js b/src/baseValidator.js new file mode 100644 index 0000000..792e165 --- /dev/null +++ b/src/baseValidator.js @@ -0,0 +1,61 @@ +// const { STAC } = require('stac-js'); + +class BaseValidator { + + /** + * + */ + constructor() { + } + + async createAjv(ajv) { + return ajv; + } + + /** + * Any preprocessing work you want to do on the data. + * + * @param {Object} data + * @param {import('.').Report} report + * @param {import('.').Config} config + * @returns {Object} + */ + async afterLoading(data, report, config) { + return data; + } + + /** + * Bypass the STAC validation, do something different but still return a report. + * + * Could be used to validate against a different schema, e.g. OGC API - Records. + * + * Return a Report to bypass validation, or null to continue with STAC validation. + * + * @param {Object} data + * @param {import('.').Report} report + * @param {import('.').Config} config + * @returns {import('.').Report|null} + */ + async bypassValidation(data, report, config) { + return null; + } + + /** + * Any custom validation routines you want to run. + * + * You can either create a list of errors using the test interface + * or just throw on the first error. + * + * @param {STAC} data + * @param {Test} test + * @param {import('.').Report} report + * @param {import('.').Config} config + * @throws {Error} + */ + async afterValidation(data, test, report, config) { + + } + +} + +module.exports = BaseValidator; diff --git a/src/cli.js b/src/cli.js new file mode 100644 index 0000000..bd1e2b4 --- /dev/null +++ b/src/cli.js @@ -0,0 +1,111 @@ +const fs = require('fs-extra'); +const path = require('path'); +const { version } = require('../package.json'); +const ConfigSource = require('./config.js'); +const validate = require('../src/index.js'); +const { printConfig, printSummary, resolveFiles, printReport, abort } = require('./nodeUtils'); +const nodeLoader = require('./loader/node'); +const { getSummary, normalizePath } = require('./utils'); +const lint = require('./lint'); + + +async function run() { + console.log(`STAC Node Validator v${version}`); + console.log(); + + // Read config from CLI and config file (if any) + let config = ConfigSource.fromCLI(); + if (typeof config.config === 'string') { + Object.assign(config, await ConfigSource.fromFile(config.config)); + } + if (!config.loader) { + config.loader = nodeLoader; + } + + // Handle ignoreCerts option in Node + if (config.ignoreCerts) { + process.env["NODE_TLS_REJECT_UNAUTHORIZED"] = 0; + } + + // Abort if no files have been provided + if (config.files.length === 0) { + abort('No path or URL specified.'); + } + + config.depth = config.depth >= 0 ? config.depth : -1; + + // Verify files exist / read folders + const files = await resolveFiles(config.files, config.depth); + delete config.files; + for (let file in files.error) { + const error = files.error[file]; + console.warn(`${file}: Can't be validated for the following reason: ${error}`); + } + if (files.files.length === 0) { + abort('No files found that are suitable for validation.'); + } + else if (files.files.length === 1) { + data = data[0]; + } + else { + data = files.files; + } + + // Resolve schema folder + if (config.schemas) { + let stat = await fs.lstat(config.schemas); + if (stat.isDirectory()) { + config.schemas = normalizePath(config.schemas); + } + else { + abort('Schema folder is not a directory'); + } + } + + // Print config + if (config.verbose) { + printConfig(config); + console.log(); + } + + if (config.lint || config.format) { + config.lintFn = async (data, report, config) => { + if (!report.apiList) { + report.lint = await lint(data, config); + if (report.lint && !report.lint.valid) { + report.valid = false; + } + } + return report; + } + } + + if (config.custom) { + const absPath = path.resolve(process.cwd(), config.custom); + const validator = require(absPath); + config.customValidator = new validator(); + } + + // Finally run validation + const result = await validate(data, config); + + // Print not a "supported error" once for API lists + if (result.apiList) { + printLint(null, config); + } + + // Print report and summary + printReport(result, config); + if (config.verbose || !result.valid) { + console.log(); + } + + const summary = getSummary(result, config); + printSummary(summary); + + // Exit with error code or report success + let errored = (summary.invalid > 0 || (config.lint && !config.format && summary.malformed > 0)) ? 1 : 0; + process.exit(errored); +} + +module.exports = run; diff --git a/src/config.js b/src/config.js new file mode 100644 index 0000000..ec52ae9 --- /dev/null +++ b/src/config.js @@ -0,0 +1,106 @@ +const yargs = require('yargs/yargs'); +const { hideBin } = require('yargs/helpers'); +const fs = require('fs-extra'); +const path = require('path'); + +const { strArrayToObject } = require('./nodeUtils'); + +function fromCLI() { + let config = yargs(hideBin(process.argv)) + .parserConfiguration({ + 'camel-case-expansion': false, + 'boolean-negation': false, + 'strip-aliased': true + }) + .option('lint', { + alias: 'l', + type: 'boolean', + default: false, + description: 'Check whether the JSON files are well-formatted, based on the JavaScript implementation with a 2-space indentation.' + }) + .option('format', { + alias: 'f', + type: 'boolean', + default: false, + description: 'Writes the JSON files according to the linting rules.\nATTENTION: Overrides the source files!' + }) + .option('schemas', { + alias: 's', + type: 'string', + default: null, + requiresArg: true, + description: 'Validate against schemas in a local or remote STAC folder.' + }) + .option('schemaMap', { + type: 'array', + default: [], + requiresArg: true, + description: 'Validate against a specific local schema (e.g. an external extension). Provide the schema URI and the local path separated by an equal sign.\nExample: https://stac-extensions.github.io/foobar/v1.0.0/schema.json=./json-schema/schema.json\nThis can also be a partial URL and path so that all children are also mapped.\nExample: https://stac-extensions.github.io/foobar/=./json-schema/', + coerce: strArrayToObject + }) + .option('custom', { + type: 'string', + default: null, + description: 'Load a custom validation routine from a JavaScript file.' + }) + .option('ignoreCerts', { + type: 'boolean', + default: false, + description: 'Disable verification of SSL/TLS certificates.' + }) + .option('depth', { + type: 'integer', + default: -1, + description: 'The number of levels to recurse into when looking for files in folders. 0 = no subfolders, -1 = unlimited' + }) + .option('strict', { + type: 'boolean', + default: false, + description: 'Enable strict mode in validation for schemas and numbers (as defined by ajv for options `strictSchema`, `strictNumbers` and `strictTuples`.' + }) + .option('verbose', { + alias: 'v', + type: 'boolean', + default: false, + description: 'Run with verbose logging and a diff for linting.' + }) + .option('config', { + alias: 'c', + type: 'string', + default: null, + description: 'Load the options from a config file (.js or .json). CLI options override config options.' + }) + .version() + .parse() + + delete config.$0; + config.files = config._; + delete config._; + + return config; +} + +async function fromFile(filepath) { + filepath = path.resolve(filepath); + if (filepath.endsWith('.js')) { + return require(filepath); + } + else { + let configFile; + try { + configFile = await fs.readFile(filepath, "utf8"); + } catch (error) { + throw new Error('Config file does not exist.'); + } + try { + return JSON.parse(configFile); + } catch (error) { + throw new Error('Config file is invalid JSON.'); + } + } +} + +module.exports = { + fromCLI, + fromFile +}; diff --git a/src/index.js b/src/index.js new file mode 100644 index 0000000..221b9e2 --- /dev/null +++ b/src/index.js @@ -0,0 +1,294 @@ +const versions = require('compare-versions'); + +const { createAjv, isHttpUrl, loadSchema, normalizePath, isObject } = require('./utils'); +const defaultLoader = require('./loader/default'); +const BaseValidator = require('./baseValidator'); +const Test = require('./test'); + +/** + * @typedef Config + * @type {Object} + * @property {function|null} [loader=null] A function that loads the JSON from the given files. + * @property {string|null} [schemas=null] Validate against schemas in a local or remote STAC folder. + * @property {Object.} [schemaMap={}] Validate against a specific local schema (e.g. an external extension). Provide the schema URI as key and the local path as value. + * @property {boolean} [strict=false] Enable strict mode in validation for schemas and numbers (as defined by ajv for options `strictSchema`, `strictNumbers` and `strictTuples + * @property {BaseValidator} [customValidator=null] A validator with custom rules. + */ + +/** + * @typedef Report + * @type {Object} + * @property {string} id + * @property {string} type + * @property {string} version + * @property {boolean} valid + * @property {Array.} messages + * @property {Array.} children + * @property {Results} results + * @property {boolean} apiList + */ + +/** + * @typedef Results + * @type {Object} + * @property {OArray.} core + * @property {Object.>} extensions + * @property {Array.} custom + */ + +/** + * @returns {Report} + */ +function createReport() { + let result = { + id: null, + type: null, + version: null, + valid: null, + skipped: false, + messages: [], + children: [], + results: { + core: [], + extensions: {}, + custom: [] + }, + apiList: false + }; + return result; +} + +/** + * @param {Array.|Array.|Object|string} data The data to validate + * @param {Config} config The configuration object + * @returns {Report|null} + */ +async function validate(data, config) { + const defaultConfig = { + loader: defaultLoader, + schemas: null, + schemaMap: {}, + strict: false + }; + config = Object.assign({}, defaultConfig, config); + config.ajv = createAjv(config); + if (config.customValidator) { + config.ajv = await config.customValidator.createAjv(config.ajv); + } + + let report = createReport(); + if (typeof data === 'string') { + report.id = normalizePath(data); + data = await config.loader(data); + } + + if (isObject(data)) { + report.id = report.id || data.id; + report.version = data.stac_version; + report.type = data.type; + + if (Array.isArray(data.collections)) { + data = data.collections; + report.apiList = true; + if (config.verbose) { + report.messages.push(`The file is a CollectionCollection. Validating all ${entries.length} collections, but ignoring the other parts of the response.`); + } + } + else if (Array.isArray(data.features)) { + data = data.features; + report.apiList = true; + if (config.verbose) { + report.messages.push(`The file is a ItemCollection. Validating all ${entries.length} items, but ignoring the other parts of the response.`); + } + } + else { + return validateOne(data, config, report); + } + } + + if (Array.isArray(data) && data.length > 0) { + for(const obj of data) { + const subreport = await validateOne(obj, config); + report.children.push(subreport); + } + return summarizeResults(report); + } + else { + return null; + } +} + + +/** + * @param {Object|string} source The data to validate + * @param {Config} config The configuration object + * @param {Report} report Parent report + * @returns {Report} + */ +async function validateOne(source, config, report = null) { + if (!report) { + report = createReport(); + } + + let data = source; + if (!report.id) { + if (typeof data === 'string') { + report.id = normalizePath(data); + try { + data = await config.loader(data); + } catch (error) { + report.valid = false; + report.results.core.push({ + instancePath: "", + message: error.message + }); + return report; + } + } + else { + report.id = data.id; + } + } + report.version = data.stac_version; + report.type = data.type; + + if (config.customValidator) { + data = await config.customValidator.afterLoading(data, report, config); + } + + if (typeof config.lintFn === 'function') { + report = await config.lintFn(source, report, config); + } + + if (config.customValidator) { + const bypass = await config.customValidator.bypassValidation(data, report, config); + if (bypass) { + return bypass; + } + } + + // Check stac_version + if (typeof data.stac_version !== 'string') { + report.skipped = true; + report.messages.push('No STAC version found'); + return report; + } + else if (versions.compare(data.stac_version, '1.0.0-rc.1', '<')) { + report.skipped = true; + report.messages.push('Can only validate STAC version >= 1.0.0-rc.1'); + return report; + } + + // Check type field + switch(data.type) { + case 'FeatureCollection': + report.skipped = true; + report.messages.push('STAC ItemCollections not supported yet'); + return report; + case 'Catalog': + case 'Collection': + case 'Feature': + // pass + break; + default: + report.valid = false; + report.results.core.push({ + instancePath: "/type", + message: "Can't detect type of the STAC object. Is the 'type' field missing or invalid?" + }); + return report; + } + + // Validate against the core schemas + await validateSchema('core', data.type, data, report, config); + + // Get all extension schemas to validate against + let schemas = []; + if (Array.isArray(data.stac_extensions)) { + schemas = schemas.concat(data.stac_extensions); + // Convert shortcuts supported in 1.0.0 RC1 into schema URLs + if (versions.compare(data.stac_version, '1.0.0-rc.1', '=')) { + schemas = schemas.map(ext => ext.replace(/^(eo|projection|scientific|view)$/, 'https://schemas.stacspec.org/v1.0.0-rc.1/extensions/$1/json-schema/schema.json')); + } + } + for(const schema of schemas) { + await validateSchema('extensions', schema, data, report, config); + } + + if (config.customValidator) { + const { default: create } = await import('stac-js'); + const stac = create(data, false, false); + try { + const test = new Test(); + await config.customValidator.afterValidation(stac, test, report, config); + report.results.custom = test.errors; + } catch (error) { + report.results.custom = [ + error + ]; + } finally { + if (report.results.custom.length > 0) { + report.valid = false; + } + } + } + + return report; +} + +async function validateSchema(key, schema, data, report, config) { + // Get schema identifier/uri + let schemaId; + switch(schema) { + case 'Feature': + schema = 'Item'; + case 'Catalog': + case 'Collection': + let type = schema.toLowerCase(); + schemaId = `https://schemas.stacspec.org/v${report.version}/${type}-spec/json-schema/${type}.json`; + break; + default: // extension + if (isHttpUrl(schema)) { + schemaId = schema; + } + } + + // Validate + const setValidity = (errors = []) => { + if (report.valid !== false) { + report.valid = errors.length === 0; + } + if (key === 'core') { + report.results.core = errors; + } + else { + report.results.extensions[schema] = errors; + } + }; + try { + if (key !== 'core' && !schemaId) { + throw new Error("'stac_extensions' must contain a valid HTTP(S) URL to a schema."); + } + const validate = await loadSchema(config, schemaId); + const valid = validate(data); + if (valid) { + setValidity(); + } + else { + setValidity(validate.errors); + } + } catch (error) { + setValidity([{ + message: error.message + }]); + } +} + +function summarizeResults(report) { + if (report.children.length > 0) { + report.valid = Boolean(report.children.every(result => result.valid)); + } + return report; +} + +module.exports = validate; diff --git a/iri.js b/src/iri.js similarity index 79% rename from iri.js rename to src/iri.js index 6f4ca4e..ef69cd3 100644 --- a/iri.js +++ b/src/iri.js @@ -1,14 +1,14 @@ const { parse } = require('uri-js'); // We don't allow empty URIs, same-document and mailto here -module.exports = { +const IRI = { 'iri': value => { if (typeof value !== 'string' || value.length === 0) { - return; + return false; } const iri = parse(value); - if ((iri.reference === 'absolute' || iri.reference === 'uri') && iri.scheme && iri.host) { + if ((iri.reference === 'absolute' || iri.reference === 'uri') && iri.scheme && (iri.host || iri.path)) { return true; } @@ -16,14 +16,16 @@ module.exports = { }, 'iri-reference': value => { if (typeof value !== 'string' || value.length === 0) { - return; + return false; } const iri = parse(value); - if ((iri.reference === 'absolute' || iri.reference === 'uri') && iri.scheme && iri.host) { + if ((iri.reference === 'absolute' || iri.reference === 'uri') && iri.scheme && (iri.host || iri.path)) { return true; } return (iri.path && (iri.reference === 'relative' || iri.reference === 'uri')); } }; + +module.exports = IRI; diff --git a/src/lint.js b/src/lint.js new file mode 100644 index 0000000..e2930e3 --- /dev/null +++ b/src/lint.js @@ -0,0 +1,58 @@ +const fs = require('fs-extra'); +const { diffStringsUnified } = require('jest-diff'); +const { isHttpUrl, isObject } = require('./utils'); + +/** + * @typedef LintResult + * @type {Object} + * @property {boolean} valid + * @property {boolean} fixed + * @property {Error|null} error + * @property {string|null} diff + */ + +/** + * @param {string} file + * @param {Object} config + * @returns {LintResult} + */ +async function lint(file, config) { + if (isObject(file)) { + return null; + } + else if (isHttpUrl(file)) { + return null; + } + + let result = { + valid: false, + fixed: false, + error: null, + diff: null + }; + try { + const fileContent = await fs.readFile(file, "utf8"); + const expectedContent = JSON.stringify(JSON.parse(fileContent), null, 2); + result.valid = normalizeNewline(fileContent) === normalizeNewline(expectedContent); + + if (!result.valid) { + if (config.verbose) { + result.diff = diffStringsUnified(fileContent, expectedContent); + } + if (config.format) { + await fs.writeFile(file, expectedContent); + result.fixed = true; + } + } + } catch (error) { + result.error = error; + } + return result; +} + +function normalizeNewline(str) { + // 2 spaces, *nix newlines, newline at end of file + return str.trimRight().replace(/(\r\n|\r)/g, "\n") + "\n"; +} + +module.exports = lint; diff --git a/src/loader/default.js b/src/loader/default.js new file mode 100644 index 0000000..2a95417 --- /dev/null +++ b/src/loader/default.js @@ -0,0 +1,8 @@ +const axios = require('axios'); + +async function loader(uri) { + let response = await axios.get(uri); + return response.data; +} + +module.exports = loader; diff --git a/src/loader/node.js b/src/loader/node.js new file mode 100644 index 0000000..379caf5 --- /dev/null +++ b/src/loader/node.js @@ -0,0 +1,28 @@ +const axios = require('axios'); +const fs = require('fs-extra'); +const { isHttpUrl } = require('../utils'); + +async function loader(uri) { + if (isHttpUrl(uri)) { + const response = await axios.get(uri); + return response.data; + } + else { + if (await fs.exists(uri)) { + return JSON.parse(await fs.readFile(uri, "utf8")); + } + else { + const url = URL.parse(uri); + // url.protocol.length > 2 check that it's not a Windows path, e.g. c: as in c://foo/bar + if (url && url.protocol && url.protocol.length > 2 && url.protocol !== 'file:') { + throw new Error(`Protocol not supported: ${url.protocol}`); + } + else { + const path = require('path'); + throw new Error(`File not found: ${uri}`); + } + } + } +} + +module.exports = loader; diff --git a/src/nodeUtils.js b/src/nodeUtils.js new file mode 100644 index 0000000..7b6dc05 --- /dev/null +++ b/src/nodeUtils.js @@ -0,0 +1,235 @@ +const klaw = require('klaw'); +const fs = require('fs-extra'); +const path = require('path'); + +const { makeAjvErrorMessage, isHttpUrl, SUPPORTED_PROTOCOLS } = require('./utils'); + +const SCHEMA_CHOICE = ['anyOf', 'oneOf']; + +function abort(message) { + console.error(message); + process.exit(1); +} + +function printConfig(config) { + console.group("Config"); + console.dir(config); + console.groupEnd(); +} + +function printSummary(summary) { + console.group(`Summary (${summary.total})`); + console.log("Valid: " + summary.valid); + console.log("Invalid: " + summary.invalid); + if (summary.malformed !== null) { + console.log("Malformed: " + summary.malformed); + } + console.log("Skipped: " + summary.skipped); + console.groupEnd(); +} + +function printLint(lint, config) { + const what = []; + config.lint && what.push('Linting'); + config.format && what.push('Formatting'); + const title = what.join(' and '); + + if (!lint) { + if (config.lint || config.format) { + console.group(title); + console.log('Not supported for remote files'); + console.groupEnd(); + } + return; + } + + if (config.verbose) { + console.group(title); + if (lint.valid) { + console.log('File is well-formed'); + } + else { + if (lint.fixed) { + console.log('File was malformed -> fixed the issue'); + } + else { + console.log('File is malformed -> use `--format` to fix the issue'); + } + } + if (lint.error) { + console.log(lint.error); + } + if (lint.diff) { + console.groupCollapsed("File Diff"); + console.log(lint.diff); + console.groupEnd(); + } + console.groupEnd(); + } + else if (!lint.valid && !lint.fixed) { + console.group(title); + console.log('File is malformed -> use `--format` to fix the issue'); + if (lint.error) { + console.log(lint.error); + } + console.groupEnd(); + } +} + +function printReport(report, config) { + if (report.valid && !config.verbose) { + return; + } + + console.group(report.id || "Report"); + + if (config.verbose && report.version) { + console.log(`STAC Version: ${report.version}`); + } + + if (report.messages) { + report.messages.forEach(str => console.log(str)); + } + + if (!report.apiList) { + printLint(report.lint, config); + } + + if (!report.valid || config.verbose) { + printAjvValidationResult(report.results.core, report.type, report.valid, config); + if (report.type) { + const count = Object.keys(report.results.extensions).length; + if (count > 0) { + console.group("Extensions"); + Object.entries(report.results.extensions) + .forEach(([ext, result]) => printAjvValidationResult(result, ext, report.valid, config)); + console.groupEnd(); + } + else { + console.log("Extensions: None"); + } + } + if (config.custom) { + printAjvValidationResult(report.results.custom, 'Custom', report.valid, config); + } + } + + report.children.forEach(child => printReport(child, config)); + + console.groupEnd(); +} + +function printAjvValidationResult(result, category, reportValid, config) { + if (!category) { + return; + } + if (!config.verbose && isHttpUrl(category)) { + const match = category.match(/^https?:\/\/stac-extensions\.github\.io\/([^/]+)\/v?([^/]+)(?:\/([^/.]+))?\/schema/); + if (match) { + let title = match[1]; + if (match[3]) { + title += ' - ' + formatKey(match[3]); + } + category = `${title} (${match[2]})`; + } + } + if (result.length > 0) { + console.group(category); + if (config.verbose) { + console.dir(result); + } + else { + result + .filter(error => result.length === 1 || !SCHEMA_CHOICE.includes(error.keyword)) // Remove messages that are usually not so important (anyOf/oneOf) + .sort((a,b) => { + // Sort so that anyOf/oneOf related messages come last, these are usually not so important + let aa = isSchemaChoice(a.schemaPath); + let bb = isSchemaChoice(b.schemaPath); + if (aa && bb) { + return 0; + } + else if (aa) { + return 1; + } + else if (bb) { + return -1; + } + else { + return 0; + } + }) + .map(error => makeAjvErrorMessage(error)) // Convert to string + .filter((value, i, array) => array.indexOf(value) === i) // Remove duplicates + .forEach((msg, i) => console.log(`${i+1}. ${msg}`)); // Print it as list + } + console.groupEnd(); + } + else if (!reportValid || config.verbose) { + console.log(`${category}: valid`); + } +} + +function isSchemaChoice(schemaPath) { + return typeof schemaPath === 'string' && schemaPath.match(/\/(one|any)Of\/\d+\//); +} + +async function resolveFiles(files, depth = -1) { + const result = { + files: [], + error: {} + }; + const extensions = [".geojson", ".json"]; + const klawOptions = { + depthLimit: depth + } + for (const file of files) { + const url = URL.parse(file); + // url.protocol.length > 2 check that it's not a Windows path, e.g. c: as in c://foo/bar + if (url && url.protocol && url.protocol.length > 2 && url.protocol !== 'file:') { + if (SUPPORTED_PROTOCOLS.includes(url.protocol)) { + result.files.push(file); + continue; + } + else { + result.error[file] = new Error(`Protocol "${url.protocol}" is not supported.`); + } + } else { + try { + const stat = await fs.lstat(file); + if (stat.isDirectory()) { + // Special handling for reading directories + for await (const child of klaw(file, klawOptions)) { + const ext = path.extname(child.path).toLowerCase(); + if (extensions.includes(ext)) { + result.files.push(child.path); + } + } + } + else { + result.files.push(file); + } + } catch (error) { + result.error[file] = error; + } + } + } + return result; +} + +function strArrayToObject(list, sep = "=") { + let map = {}; + for (let str of list) { + let [key, value] = str.split(sep, 2); + map[key] = value; + } + return map; +} + +module.exports = { + abort, + printConfig, + printReport, + printSummary, + resolveFiles, + strArrayToObject +}; diff --git a/src/test.js b/src/test.js new file mode 100644 index 0000000..c17d01b --- /dev/null +++ b/src/test.js @@ -0,0 +1,155 @@ +const assert = require('assert'); + +class Test { + + constructor() { + this.errors = []; + } + + truthy(...args) { + try { + assert(...args); + } catch (error) { + this.errors.push(error); + } + } + + deepEqual(...args) { + try { + assert.deepEqual(...args); + } catch (error) { + this.errors.push(error); + } + } + + deepStrictEqual(...args) { + try { + assert.deepStrictEqual(...args); + } catch (error) { + this.errors.push(error); + } + } + + doesNotMatch(...args) { + try { + assert.doesNotMatch(...args); + } catch (error) { + this.errors.push(error); + } + } + + async doesNotReject(...args) { + try { + await assert.doesNotReject(...args); + } catch (error) { + this.errors.push(error); + } + } + + doesNotThrow(...args) { + try { + assert.doesNotThrow(...args); + } catch (error) { + this.errors.push(error); + } + } + + equal(...args) { + try { + assert.equal(...args); + } catch (error) { + this.errors.push(error); + } + } + + fail(...args) { + try { + assert.fail(...args); + } catch (error) { + this.errors.push(error); + } + } + + ifError(...args) { + try { + assert.ifError(...args); + } catch (error) { + this.errors.push(error); + } + } + + match(...args) { + try { + assert.match(...args); + } catch (error) { + this.errors.push(error); + } + } + + notDeepEqual(...args) { + try { + assert.notDeepEqual(...args); + } catch (error) { + this.errors.push(error); + } + } + + notDeepStrictEqual(...args) { + try { + assert.notDeepStrictEqual(...args); + } catch (error) { + this.errors.push(error); + } + } + + notEqual(...args) { + try { + assert.notEqual(...args); + } catch (error) { + this.errors.push(error); + } + } + + notStrictEqual(...args) { + try { + assert.notStrictEqual(...args); + } catch (error) { + this.errors.push(error); + } + } + + ok(...args) { + try { + assert.ok(...args); + } catch (error) { + this.errors.push(error); + } + } + + async rejects(...args) { + try { + await assert.rejects(...args); + } catch (error) { + this.errors.push(error); + } + } + + strictEqual(...args) { + try { + assert.strictEqual(...args); + } catch (error) { + this.errors.push(error); + } + } + + throws(...args) { + try { + assert.throws(...args); + } catch (error) { + this.errors.push(error); + } + } + +} + +module.exports = Test; diff --git a/src/utils.js b/src/utils.js new file mode 100644 index 0000000..7ecfbb6 --- /dev/null +++ b/src/utils.js @@ -0,0 +1,146 @@ +const Ajv = require('ajv'); +const addFormats = require('ajv-formats'); +const iriFormats = require('./iri'); +const path = require('path'); + +const SUPPORTED_PROTOCOLS = ['http:', 'https:']; + +function isObject(obj) { + return (typeof obj === 'object' && obj === Object(obj) && !Array.isArray(obj)); +} + +function isHttpUrl(url) { + const parsed = URL.parse(url); + if (parsed && SUPPORTED_PROTOCOLS.includes(parsed.protocol)) { + return true; + } + return false; +} + +function createAjv(config) { + let instance = new Ajv({ + formats: iriFormats, + allErrors: config.verbose, + strict: false, + logger: config.verbose ? console : false, + loadSchema: async (uri) => await loadSchemaFromUri(uri, config) + }); + addFormats(instance); + if (config.strict) { + instance.opts.strictSchema = true; + instance.opts.strictNumbers = true; + instance.opts.strictTuples = true; + } + return instance; +} + +async function loadSchema(config, schemaId) { + let schema = config.ajv.getSchema(schemaId); + if (schema) { + return schema; + } + + try { + json = await loadSchemaFromUri(schemaId, config); + } catch (error) { + throw new Error(`Schema at '${schemaId}' not found. Please ensure all entries in 'stac_extensions' are valid.`); + } + + schema = config.ajv.getSchema(json.$id); + if (schema) { + return schema; + } + + return await config.ajv.compileAsync(json); +} + +async function loadSchemaFromUri(uri, config) { + let newUri = uri; + if (isObject(config.schemaMap)) { + const patterns = Object.entries(config.schemaMap); + const match = patterns.find(map => uri.startsWith(map[0])); + if (match) { + const [pattern, target] = match; + newUri = path.join(target, uri.substring(pattern.length)); + } + } + if (config.schemas) { + newUri = newUri.replace(/^https:\/\/schemas\.stacspec\.org\/v[^\/]+/, config.schemas); + } + const schema = await config.loader(newUri); + if (!schema.$id) { + schema.$id = uri; + } + return schema; +} + +function normalizePath(path) { + return path.replace(/\\/g, '/').replace(/\/$/, ""); +} + +function getSummary(result, config) { + let summary = { + total: 0, + valid: 0, + invalid: 0, + malformed: null, + skipped: 0 + }; + if (result.children.length > 0) { + // todo: speed this up by computing in one iteration + summary.total = result.children.length; + summary.valid = result.children.filter(c => c.valid === true).length; + summary.invalid = result.children.filter(c => c.valid === false).length; + if (config.lint || config.format) { + summary.malformed = result.children.filter(c => c.lint && !c.lint.valid).length; + } + summary.skipped = result.children.filter(c => c.skipped).length; + } + else { + summary.total = 1; + summary.valid = result.valid === true ? 1 : 0; + summary.invalid = result.valid === false ? 1 : 0; + if (result.lint) { + summary.malformed = result.lint.valid ? 0 : 1; + } + summary.skipped = result.skipped ? 1 : 0; + } + return summary; +} + +function makeAjvErrorMessage(error) { + let message = error.message; + if (isObject(error.params) && Object.keys(error.params).length > 0) { + let params = Object.entries(error.params) + .map(([key, value]) => { + let label = key.replace(/([^A-Z]+)([A-Z])/g, "$1 $2").toLowerCase(); + return `${label}: ${value}`; + }) + .join(', ') + message += ` (${params})`; + } + if (error.instancePath) { + return `${error.instancePath} ${message}`; + } + else if (error.schemaPath) { + return `${message}, for schema ${error.schemaPath}`; + } + else if (message) { + return message; + } + else { + return String(error); + } +} + +module.exports = { + createAjv, + getSummary, + isObject, + isHttpUrl, + loadSchema, + loadSchemaFromUri, + makeAjvErrorMessage, + normalizePath, + SUPPORTED_PROTOCOLS +}; diff --git a/tests/cli.test.js b/tests/cli.test.js index 7a21677..f542559 100644 --- a/tests/cli.test.js +++ b/tests/cli.test.js @@ -1,4 +1,4 @@ -const app = require('../index'); +const app = require('../src/index'); const { version } = require('../package.json'); const fs = require('fs/promises'); const { exec } = require("child_process");