diff --git a/README.md b/README.md index da70527..4864591 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,17 @@ # couchwarehouse -*couchwarehouse* is a command-line tool that turns your [Apache CouchDB](http://couchdb.apache.org/) database(s) into a local data warehouse. It works by: +*couchwarehouse* is a command-line tool that turns your [Apache CouchDB](http://couchdb.apache.org/) database(s) into a local data warehouse. The target database can be either be [SQLite](https://www.sqlite.org/index.html), [PostgreSQL](https://www.postgresql.org/), [MySQL](https://www.mysql.com/) or [Elasticsearch](https://www.elastic.co/products/elasticsearch). -- discovering the "schema" of your CouchDB database. -- creating a new [SQLite](https://www.sqlite.org/index.html), [PostgreSQL](https://www.postgresql.org/) or [MySQL](https://www.mysql.com/) table to match the schema. -- downloading all the documents (except design documents) and inserting one row per document into the SQL database. +It works by: + +- discovering the "schema" of your CouchDB database (for the relational databases). +- creating a new [SQLite](https://www.sqlite.org/index.html), [PostgreSQL](https://www.postgresql.org/) or [MySQL](https://www.mysql.com/) table to match the schema, or in the case of Elasticsearch simply moving the JSON over. +- downloading all the documents (except design documents) and inserting one row per document into the target database. - continuously monitoring CouchDB for new documents, updates to existing documents and deletions. ![](img/couchwarehouse.png) -Once downloaded your database can be queried using SQL. +Once downloaded your database can be queried using SQL or the target database's API. ## Installation @@ -19,7 +21,7 @@ Once downloaded your database can be queried using SQL. npm install -g couchwarehouse ``` -## Usage +## Usage with SQLite By default, your CouchDB installation is expected to be on "http://localhost:5984". Override this with the `--url`/`-u` parameter and specify the database name with `--database`/`-db`: @@ -42,7 +44,7 @@ After downloading is complete, *couchwarehouse* will continuously poll the sourc Press "Ctrl-C" to exit. -## Accessing the data warehouse +## Accessing the SQLite data warehouse In another terminal, simply run the `sqlite3` command-line tool (which may be pre-installed on your computer, otherwise [download here](https://www.sqlite.org/download.html)). @@ -67,7 +69,7 @@ SQLite has an [extensive query language](https://www.sqlite.org/lang.html) inclu N.B if your database name has a `-` character in it, it will be removed from the subsequent SQL table e.g "month-54" becomes "month54". -## Using with PostgreSQL instead of SQLite +## Using with PostgreSQL as the target database The PostgreSQL connection details are gleaned from [environment variables](https://www.postgresql.org/docs/9.3/libpq-envars.html). If you're [running PostgreSQL locally](https://www.postgresql.org/docs/11/tutorial-install.html) without password protection, you need only worry about the `PGDATABASE` environment variable which defines the name of the database the `couchwarehouse` tables will be created. If left undefined, a database matching your current username will be assumed (e.g. `glynnb`). I had to create this database first: @@ -96,7 +98,7 @@ glynnb=# select * from mydb limit 5; (5 rows) ``` -## Using with MySQL instead of SQLite +## Using with MySQL as the target database The MySQL connection string is taken from the `MYSQLCONFIG` environment variable, or if absent `mysql://root:@localhost:3306/couchwarehouse` is used. connection details are gleaned from [environment variables]. You will need to create the `couchwarehouse` database first: @@ -129,11 +131,28 @@ mysql> select * from mydb limit 5; 5 rows in set (0.00 sec) ``` +## Using with Elasticsearch as the target database + +The MySQL connection string is taken from the `ESCONFIG` environment variable, or if absent `http://localhost:9200` is used. + +Run `couchwarehouse` specifyinhg the `--databaseType` parameter: + +```sh +$ couchwarehouse --url https://U:P@host.cloudant.com --db mydb --databaseType elasticsearch +``` + +You can then access your datawarehouse using the Elasticsearch API: + +``` +$ curl 'http://localhost:9200/couchwarehouse/_search?q=name:"York"' +{"took":3,"timed_out":false,"_shards":{"total":5,"successful":5,"skipped":0,"failed":0},"hits":{"total":6,"max_score":7.998925,"hits":[{"_index":"couchwarehouse","_type":"default","_id":"4562407","_score":7.998925,"_source":{"name":"York","latitude":39.9626,"longitude":-76.72774,"country":"US","population":43718,"timezone":"America/New_York"}},{"_index":"couchwarehouse","_type":"default","_id":"2633352","_score":7.998925,"_source":{"name":"York","latitude":53.95763,"longitude":-1.08271,"country":"GB","population":144202,"timezone":"Europe/London"}},{"_index":"couchwarehouse","_type":"default","_id":"6091104","_score":5.9267497,"_source":{"name":"North York","latitude":43.76681,"longitude":-79.4163,"country":"CA","population":636000,"timezone":"America/Toronto"}},{"_index":"couchwarehouse","_type":"default","_id":"7870925","_score":5.9267497,"_source":{"name":"East York","latitude":43.69053,"longitude":-79.32794,"country":"CA","population":115365,"timezone":"America/Toronto"}},{"_index":"couchwarehouse","_type":"default","_id":"5128581","_score":5.283532,"_source":{"name":"New York City","latitude":40.71427,"longitude":-74.00597,"country":"US","population":8175133,"timezone":"America/New_York"}},{"_index":"couchwarehouse","_type":"default","_id":"5106292","_score":4.734778,"_source":{"name":"West New York","latitude":40.78788,"longitude":-74.01431,"country":"US","population":49708,"timezone":"America/New_York"}}]} +``` + ## Command-line parameter reference - `--url`/`-u` - the URL of the CouchDB instance e.g. `http://localhost:5984` - `--database`/`--db`/`-d` - the name of the CouchDB database to work with -= `--databaseType`/`-dt` - the type of database - `sqlite` or z`postgresql` (default: sqlite) += `--databaseType`/`-dt` - the type of database - `sqlite`, `mysql`,`postgresql` or `elasticsearch` (default: sqlite) - `--verbose` - whether to show progress on the terminal (default: true) - `--reset`/`-r` - reset the data. Delete existing data and start from scratch (default: false) - `--transform`/`-t` - transform each document with a supplied JavaScript function (default: null) diff --git a/bin/couchwarehouse.bin.js b/bin/couchwarehouse.bin.js index 5060972..92c00de 100755 --- a/bin/couchwarehouse.bin.js +++ b/bin/couchwarehouse.bin.js @@ -6,7 +6,7 @@ const db = process.env.COUCH_DATABASE const args = require('yargs') .option('url', { alias: 'u', describe: 'CouchDB URL', default: url }) .option('database', { alias: ['db', 'd'], describe: 'CouchDB database name', demandOption: !db, default: db }) - .option('databaseType', { alias: ['dt'], describe: 'SQL database type', default: 'sqlite' }) + .option('databaseType', { alias: ['dt'], describe: 'Target database type (postgresql,mysql,sqlite,elasticsearch)', default: 'sqlite' }) .option('verbose', { describe: 'Show instructions and progress in the output', default: true }) .option('reset', { alias: 'r', describe: 'Ignore previously downloaded data and start again', default: false }) .option('transform', { alias: 't', describe: 'Path to a JavaScript transformation function', default: process.env.COUCH_TRANSFORM ? process.env.COUCH_TRANSFORM : null }) diff --git a/img/couchwarehouse.png b/img/couchwarehouse.png index 24b775b..3265905 100644 Binary files a/img/couchwarehouse.png and b/img/couchwarehouse.png differ diff --git a/index.js b/index.js index 7aea555..40986a0 100644 --- a/index.js +++ b/index.js @@ -17,6 +17,9 @@ const loadDatabaseDriver = (opts) => { case 'mysql': sqldb = require('./lib/mysql.js') break + case 'elasticsearch': + sqldb = require('./lib/elasticsearch.js') + break case 'sqlite': default: sqldb = require('./lib/sqlite.js') @@ -110,6 +113,9 @@ const spoolChanges = async (opts, theSchema, maxChange) => { bar.tick(b.length) } + // write checkpoint + await sqldb.writeCheckpoint(opts.database, lastSeq) + // call the done callback if provided if (typeof done === 'function') { done() @@ -218,7 +224,6 @@ const start = async (opts) => { debug('Spooling changes') if (opts.verbose) { opts.usableDbName = util.calculateUsableDbName(opts, opts.database, null) - console.log(opts.database, opts.usableDbName) sqldb.message(opts) } const lastSeq = await spoolChanges(opts, theSchema, maxChange) diff --git a/lib/elasticsearch.js b/lib/elasticsearch.js new file mode 100644 index 0000000..0b29b68 --- /dev/null +++ b/lib/elasticsearch.js @@ -0,0 +1,92 @@ +const elasticsearch = require('elasticsearch') +let db + +const initialise = async (reset) => { + const config = process.env.ESCONFIG || 'http://localhost:9200' + db = new elasticsearch.Client({ + hosts: [config] + }) +} + +const generateCreateTableSQL = (opts, docType, dbName, schema, reset) => { + return [] +} + +// insert an array of changes into the database +const insertBulk = async (opts, createSQL, dbName, theSchema, batch) => { + const actions = [] + + batch.forEach(async b => { + // ignore design docs + if (!b.id.match(/^_design/)) { + // get the schema we're working with + const docType = opts.split ? b.doc[opts.split] : 'default' + + // if this is a deletion + if (b.deleted) { + // delete from Elasticsearch + actions.push({ delete: { _index: 'couchwarehouse', _type: docType, _id: b.doc._id } }) + } else { + actions.push({ index: { _index: 'couchwarehouse', _type: docType, _id: b.doc._id } }) + delete b.doc._id + delete b.doc._rev + delete b.doc._attachments + actions.push(b.doc) + } + } + }) + + if (actions.length > 0) { + return db.bulk({ body: actions }) + } else { + return null + } +} + +const query = async (sql, substitutions) => { + return null +} + +// write a checkpoint to keep track of where we got to +// with each table +const writeCheckpoint = async (tablename, seq) => { + const actions = [] + actions.push({ + index: { + _index: 'couchwarehousemeta', + _type: tablename, + _id: 'checkpoint' + } + }) + actions.push({ seq: seq }) + return db.bulk({ body: actions }) +} + +const getCheckpoint = async (tablename) => { + try { + const doc = await db.get({ + index: 'couchwarehousemeta', + type: tablename, + id: 'checkpoint' + }) + return (doc && doc._source && doc._source.seq) ? doc._source.seq : null + } catch (e) { + return null + } +} + +const message = (opts) => { + console.log('Spooling data into Elasticsearch') + console.log('Use the API to access the data: https://www.elastic.co/guide/en/elasticsearch/reference/5.5/_the_search_api.html') + console.log('p.s Press ctrl-C to stop monitoring for further changes') +} + +module.exports = { + initialise, + generateCreateTableSQL, + insertBulk, + query: query, + writeCheckpoint: writeCheckpoint, + getCheckpoint: getCheckpoint, + message: message +} diff --git a/package-lock.json b/package-lock.json index 34681bf..c4d0523 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "couchwarehouse", - "version": "1.5.0", + "version": "1.6.0", "lockfileVersion": 1, "requires": true, "dependencies": { @@ -58,6 +58,14 @@ "acorn": "^5.0.3" } }, + "agentkeepalive": { + "version": "3.5.2", + "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-3.5.2.tgz", + "integrity": "sha512-e0L/HNe6qkQ7H19kTlRRqUibEAwDK5AFk6y3PtMsuut2VAH6+Q4xZml1tNDJD7kSAyqmbG/K08K5WEJYtUrSlQ==", + "requires": { + "humanize-ms": "^1.2.1" + } + }, "ajv": { "version": "6.7.0", "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.7.0.tgz", @@ -89,8 +97,7 @@ "ansi-styles": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz", - "integrity": "sha1-tDLdM1i2NM914eRmQ2gkBTPB3b4=", - "dev": true + "integrity": "sha1-tDLdM1i2NM914eRmQ2gkBTPB3b4=" }, "aproba": { "version": "1.2.0", @@ -619,6 +626,56 @@ "safer-buffer": "^2.1.0" } }, + "elasticsearch": { + "version": "15.4.1", + "resolved": "https://registry.npmjs.org/elasticsearch/-/elasticsearch-15.4.1.tgz", + "integrity": "sha512-IL46Sv9krCKtpvlI37/vQVQrWx6QeT1OJhfWW6L3fIXzR1Vv5utO+DHYz8AosUI6vlkxShoq+y6sUIBhTF1OIg==", + "requires": { + "agentkeepalive": "^3.4.1", + "chalk": "^1.0.0", + "lodash": "^4.17.10" + }, + "dependencies": { + "ansi-regex": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", + "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=" + }, + "chalk": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", + "integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=", + "requires": { + "ansi-styles": "^2.2.1", + "escape-string-regexp": "^1.0.2", + "has-ansi": "^2.0.0", + "strip-ansi": "^3.0.0", + "supports-color": "^2.0.0" + } + }, + "strip-ansi": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", + "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=", + "requires": { + "ansi-regex": "^2.0.0" + } + } + } + }, + "emoji-regex": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-7.0.3.tgz", + "integrity": "sha512-CwBLREIQ7LvYFB0WyRvwhq5N5qPhc6PMjD6bYggFlI5YyDgl+0vxq5VHbMOFqLg7hfWzmu8T5Z1QofhmTIhItA==" + }, + "end-of-stream": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.1.tgz", + "integrity": "sha512-1MkrZNvWTKCaigbn+W15elq2BB/L22nqrSY5DKlo3X6+vclJm8Bb5djXJBmEX6fS3+zCh/F4VBK5Z2KxJt4s2Q==", + "requires": { + "once": "^1.4.0" + } + }, "error-ex": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.2.tgz", @@ -660,8 +717,7 @@ "escape-string-regexp": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", - "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=", - "dev": true + "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=" }, "eslint": { "version": "5.4.0", @@ -973,12 +1029,12 @@ "dev": true }, "execa": { - "version": "0.10.0", - "resolved": "https://registry.npmjs.org/execa/-/execa-0.10.0.tgz", - "integrity": "sha512-7XOMnz8Ynx1gGo/3hyV9loYNPWM94jG3+3T3Y8tsfSstFmETmENCMU/A/zj8Lyaj1lkgEepKepvd6240tBRvlw==", + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/execa/-/execa-1.0.0.tgz", + "integrity": "sha512-adbxcyWV46qiHyvSp50TKt05tB4tK3HcmF7/nxfAdhnox83seTDbwnaqKO4sXRy7roHAIFqJP/Rw/AuEbX61LA==", "requires": { "cross-spawn": "^6.0.0", - "get-stream": "^3.0.0", + "get-stream": "^4.0.0", "is-stream": "^1.1.0", "npm-run-path": "^2.0.0", "p-finally": "^1.0.0", @@ -1148,7 +1204,7 @@ }, "strip-ansi": { "version": "3.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", + "resolved": "http://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=", "requires": { "ansi-regex": "^2.0.0" @@ -1157,9 +1213,9 @@ } }, "get-caller-file": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-1.0.3.tgz", - "integrity": "sha512-3t6rVToeoZfYSGd8YoLFR2DJkiQrIiUrGcjvFX2mDw3bn6k2OtwHN0TNCLbBO+w8qTvimhDkv+LSscbJY1vE6w==" + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.1.tgz", + "integrity": "sha512-SpOZHfz845AH0wJYVuZk2jWDqFmu7Xubsx+ldIpwzy5pDUpu7OJHK7QYNSA2NPlDSKQwM1GFaAkciOWjjW92Sg==" }, "get-func-name": { "version": "2.0.0", @@ -1174,9 +1230,12 @@ "dev": true }, "get-stream": { - "version": "3.0.0", - "resolved": "http://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz", - "integrity": "sha1-jpQ9E1jcN1VQVOy+LtsFqhdO3hQ=" + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-4.1.0.tgz", + "integrity": "sha512-GMat4EJ5161kIy2HevLlr4luNjBgvmj413KaQA7jt4V8B4RDsfpHk7WQ9GVqfYyyx8OS/L66Kox+rJRNklLK7w==", + "requires": { + "pump": "^3.0.0" + } }, "getpass": { "version": "0.1.7", @@ -1258,7 +1317,6 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/has-ansi/-/has-ansi-2.0.0.tgz", "integrity": "sha1-NPUEnOHs3ysGSa8+8k5F7TVBbZE=", - "dev": true, "requires": { "ansi-regex": "^2.0.0" }, @@ -1266,8 +1324,7 @@ "ansi-regex": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", - "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=", - "dev": true + "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=" } } }, @@ -1310,6 +1367,14 @@ "sshpk": "^1.7.0" } }, + "humanize-ms": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz", + "integrity": "sha1-xG4xWaKT9riW2ikxbYtv6Lt5u+0=", + "requires": { + "ms": "^2.0.0" + } + }, "iconv-lite": { "version": "0.4.24", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", @@ -1629,13 +1694,13 @@ } }, "mem": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/mem/-/mem-4.0.0.tgz", - "integrity": "sha512-WQxG/5xYc3tMbYLXoXPm81ET2WDULiU5FxbuIoNbJqLOOI8zehXFdZuiUEgfdrU2mVB1pxBZUGlYORSrpuJreA==", + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/mem/-/mem-4.1.0.tgz", + "integrity": "sha512-I5u6Q1x7wxO0kdOpYBB28xueHADYps5uty/zg936CiG8NTe5sJL8EjrCuLneuDW3PlMdZBGDIn8BirEVdovZvg==", "requires": { "map-age-cleaner": "^0.1.1", "mimic-fn": "^1.0.0", - "p-is-promise": "^1.1.0" + "p-is-promise": "^2.0.0" } }, "mime-db": { @@ -1777,7 +1842,7 @@ }, "nan": { "version": "2.10.0", - "resolved": "https://registry.npmjs.org/nan/-/nan-2.10.0.tgz", + "resolved": "http://registry.npmjs.org/nan/-/nan-2.10.0.tgz", "integrity": "sha512-bAdJv7fBLhWC+/Bls0Oza+mvTaNQtP+1RyhhhvD95pgUJz6XM5IzgmxOkItJ9tkoCiplvAnXI1tNmmUD/eScyA==" }, "nano": { @@ -1990,11 +2055,11 @@ "integrity": "sha1-/7xJiDNuDoM94MFox+8VISGqf7M=" }, "os-locale": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/os-locale/-/os-locale-3.0.1.tgz", - "integrity": "sha512-7g5e7dmXPtzcP4bgsZ8ixDVqA7oWYuEz4lOSujeWyliPai4gfVDiFIcwBg3aGCPnmSGfzOKTK3ccPn0CKv3DBw==", + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/os-locale/-/os-locale-3.1.0.tgz", + "integrity": "sha512-Z8l3R4wYWM40/52Z+S265okfFj8Kt2cC2MKY+xNi3kFs+XGI7WXu/I309QQQYbRW4ijiZ+yxs9pqEhJh0DqW3Q==", "requires": { - "execa": "^0.10.0", + "execa": "^1.0.0", "lcid": "^2.0.0", "mem": "^4.0.0" } @@ -2024,14 +2089,14 @@ "integrity": "sha1-P7z7FbiZpEEjs0ttzBi3JDNqLK4=" }, "p-is-promise": { - "version": "1.1.0", - "resolved": "http://registry.npmjs.org/p-is-promise/-/p-is-promise-1.1.0.tgz", - "integrity": "sha1-nJRWmJ6fZYgBewQ01WCXZ1w9oF4=" + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/p-is-promise/-/p-is-promise-2.0.0.tgz", + "integrity": "sha512-pzQPhYMCAgLAKPWD2jC3Se9fEfrD9npNos0y150EeqZll7akhEgGhTW/slB6lHku8AvYGiJ+YJ5hfHKePPgFWg==" }, "p-limit": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.0.0.tgz", - "integrity": "sha512-fl5s52lI5ahKCernzzIyAP0QAZbGIovtVHGwpcu1Jr/EpzLVDI2myISHwGqK7m8uQFugVWSrbxH7XnhGtvEc+A==", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.1.0.tgz", + "integrity": "sha512-NhURkNcrVB+8hNfLuysU8enY5xn2KXphsHBaC2YmRNTZRc7RWusw6apSpdEj3jo4CMb6W9nrF6tTnsJsJeyu6g==", "requires": { "p-try": "^2.0.0" } @@ -2050,9 +2115,9 @@ "integrity": "sha512-hMp0onDKIajHfIkdRk3P4CdCmErkYAxxDtP3Wx/4nZ3aGlau2VKh3mZpcuFkH27WQkL/3WBCPOktzA9ZOAnMQQ==" }, "packet-reader": { - "version": "0.3.1", - "resolved": "https://registry.npmjs.org/packet-reader/-/packet-reader-0.3.1.tgz", - "integrity": "sha1-zWLmCvjX/qinBexP+ZCHHEaHHyc=" + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/packet-reader/-/packet-reader-1.0.0.tgz", + "integrity": "sha512-HAKu/fG3HpHFO0AA8WE8q2g+gBJaZ9MG7fcKk+IJPLTGAD6Psw4443l+9DGRbOIh3/aXr7Phy0TjilYivJo5XQ==" }, "parse-json": { "version": "2.2.0", @@ -2111,12 +2176,12 @@ "integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=" }, "pg": { - "version": "7.8.0", - "resolved": "https://registry.npmjs.org/pg/-/pg-7.8.0.tgz", - "integrity": "sha512-yS3C9YD+ft0H7G47uU0eKajgTieggCXdA+Fxhm5G+wionY6kPBa8BEVDwPLMxQvkRkv3/LXiFEqjZm9gfxdW+g==", + "version": "7.8.1", + "resolved": "https://registry.npmjs.org/pg/-/pg-7.8.1.tgz", + "integrity": "sha512-m9aIrOV4mgfo+1Ze+eNoJwaWZDvpeBz8Kzwi0zzqLC+tQBsQgIuu+FGPqzyRv9HFlS7tHO1I33LKp9gP5g7U4Q==", "requires": { "buffer-writer": "2.0.0", - "packet-reader": "0.3.1", + "packet-reader": "1.0.0", "pg-connection-string": "0.1.3", "pg-pool": "^2.0.4", "pg-types": "~2.0.0", @@ -2333,9 +2398,9 @@ "integrity": "sha1-4tiXAu/bJY/52c7g/pG9BpdSV6g=" }, "postgres-interval": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.1.2.tgz", - "integrity": "sha512-fC3xNHeTskCxL1dC8KOtxXt7YeFmlbTYtn7ul8MkVERuTmf7pI4DrkAxcw3kh1fQ9uz4wQmd03a1mRiXUZChfQ==", + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz", + "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==", "requires": { "xtend": "^4.0.0" } @@ -2377,6 +2442,15 @@ "resolved": "https://registry.npmjs.org/psl/-/psl-1.1.31.tgz", "integrity": "sha512-/6pt4+C+T+wZUieKR620OpzN/LlnNKuWjy1iFLQ/UG35JqHlR/89MP1d96dUfkf6Dne3TuLQzOYEYshJ+Hx8mw==" }, + "pump": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", + "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==", + "requires": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, "punycode": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz", @@ -2400,7 +2474,7 @@ "dependencies": { "minimist": { "version": "1.2.0", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz", + "resolved": "http://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz", "integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ=" } } @@ -2524,9 +2598,9 @@ "integrity": "sha1-jGStX9MNqxyXbiNE/+f3kqam30I=" }, "require-main-filename": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/require-main-filename/-/require-main-filename-1.0.1.tgz", - "integrity": "sha1-l/cXtp1IeE9fUmpsWqj/3aBVpNE=" + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/require-main-filename/-/require-main-filename-2.0.0.tgz", + "integrity": "sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg==" }, "require-uncached": { "version": "1.0.3", @@ -2805,8 +2879,7 @@ "supports-color": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz", - "integrity": "sha1-U10EXOa2Nj+kARcIRimZXp3zJMc=", - "dev": true + "integrity": "sha1-U10EXOa2Nj+kARcIRimZXp3zJMc=" }, "symbol-observable": { "version": "1.0.1", @@ -3005,7 +3078,7 @@ }, "wrap-ansi": { "version": "2.1.0", - "resolved": "http://registry.npmjs.org/wrap-ansi/-/wrap-ansi-2.1.0.tgz", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-2.1.0.tgz", "integrity": "sha1-2Pw9KE3QV5T+hJc8rs3Rz4JP3YU=", "requires": { "string-width": "^1.0.1", @@ -3037,7 +3110,7 @@ }, "strip-ansi": { "version": "3.0.1", - "resolved": "http://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=", "requires": { "ansi-regex": "^2.0.0" @@ -3075,28 +3148,52 @@ "integrity": "sha512-S+Zk8DEWE6oKpV+vI3qWkaK+jSbIK86pCwe2IF/xwIpQ8jEuxpw9NyaGjmp9+BoJv5FV2piqCDcoCtStppiq2A==" }, "yargs": { - "version": "12.0.5", - "resolved": "https://registry.npmjs.org/yargs/-/yargs-12.0.5.tgz", - "integrity": "sha512-Lhz8TLaYnxq/2ObqHDql8dX8CJi97oHxrjUcYtzKbbykPtVW9WB+poxI+NM2UIzsMgNCZTIf0AQwsjK5yMAqZw==", + "version": "13.2.1", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-13.2.1.tgz", + "integrity": "sha512-HgY0xHGmPPakg6kEDufqxZuXVtvPZcipORC8O7S44iEnwsUmP+qnhReHc6d1dyeIZkrPmYFblh45Z2oeDn++fQ==", "requires": { "cliui": "^4.0.0", - "decamelize": "^1.2.0", "find-up": "^3.0.0", - "get-caller-file": "^1.0.1", - "os-locale": "^3.0.0", + "get-caller-file": "^2.0.1", + "os-locale": "^3.1.0", "require-directory": "^2.1.1", - "require-main-filename": "^1.0.1", + "require-main-filename": "^2.0.0", "set-blocking": "^2.0.0", - "string-width": "^2.0.0", + "string-width": "^3.0.0", "which-module": "^2.0.0", - "y18n": "^3.2.1 || ^4.0.0", - "yargs-parser": "^11.1.1" + "y18n": "^4.0.0", + "yargs-parser": "^13.0.0" + }, + "dependencies": { + "ansi-regex": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.0.0.tgz", + "integrity": "sha512-iB5Dda8t/UqpPI/IjsejXu5jOGDrzn41wJyljwPH65VCIbk6+1BzFIMJGFwTNrYXT1CrD+B4l19U7awiQ8rk7w==" + }, + "string-width": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-3.0.0.tgz", + "integrity": "sha512-rr8CUxBbvOZDUvc5lNIJ+OC1nPVpz+Siw9VBtUjB9b6jZehZLFt0JMCZzShFHIsI8cbhm0EsNIfWJMFV3cu3Ew==", + "requires": { + "emoji-regex": "^7.0.1", + "is-fullwidth-code-point": "^2.0.0", + "strip-ansi": "^5.0.0" + } + }, + "strip-ansi": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.0.0.tgz", + "integrity": "sha512-Uu7gQyZI7J7gn5qLn1Np3G9vcYGTVqB+lFTytnDJv83dd8T22aGH451P3jueT2/QemInJDfxHB5Tde5OzgG1Ow==", + "requires": { + "ansi-regex": "^4.0.0" + } + } } }, "yargs-parser": { - "version": "11.1.1", - "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-11.1.1.tgz", - "integrity": "sha512-C6kB/WJDiaxONLJQnF8ccx9SEeoTTLek8RVbaOIsrAUS8VrBEXfmeSnCZxygc+XC2sNMBIwOOnfcxiynjHsVSQ==", + "version": "13.0.0", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-13.0.0.tgz", + "integrity": "sha512-w2LXjoL8oRdRQN+hOyppuXs+V/fVAYtpcrRxZuF7Kt/Oc+Jr2uAcVntaUTNT6w5ihoWfFDpNY8CPx1QskxZ/pw==", "requires": { "camelcase": "^5.0.0", "decamelize": "^1.2.0" diff --git a/package.json b/package.json index 4f9312d..994d486 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "couchwarehouse", - "version": "1.5.0", + "version": "1.6.0", "description": "Data Warehouse PoC for CouchDB", "main": "index.js", "scripts": { @@ -25,12 +25,13 @@ "dependencies": { "changesreader": "^1.1.3", "debug": "^4.1.1", + "elasticsearch": "^15.4.1", "mysql": "^2.16.0", "nano": "^8.0.0", - "pg": "^7.8.0", + "pg": "^7.8.1", "progress": "^2.0.3", "sqlite3": "^4.0.6", - "yargs": "^12.0.5" + "yargs": "^13.2.1" }, "bin": { "couchwarehouse": "bin/couchwarehouse.bin.js"