From f57444a3d12494d09164d8eddbe64f1f85c5d347 Mon Sep 17 00:00:00 2001 From: Timo Hocker Date: Tue, 25 Aug 2020 15:24:20 +0200 Subject: [PATCH] reduction job, chart duplicates --- lib/db/index.js | 14 ++++-- lib/db/job.js | 85 ++++++++++++++++++++++++++++++++ lib/defaults.js | 23 +++++++++ migrations/00001-reduction.js | 22 +++++++++ package.json | 6 ++- seeds/fake.js | 21 ++------ src/components/ChartView.vue | 23 +++++++-- src/components/ViewComponent.vue | 1 + src/default.js | 9 ++-- src/template.js | 5 ++ yarn.lock | 5 ++ 11 files changed, 185 insertions(+), 29 deletions(-) create mode 100644 lib/db/job.js create mode 100644 lib/defaults.js create mode 100644 migrations/00001-reduction.js diff --git a/lib/db/index.js b/lib/db/index.js index 73c9e99..87c7f85 100644 --- a/lib/db/index.js +++ b/lib/db/index.js @@ -11,6 +11,7 @@ const knex = require ('knex'); const fs = require ('fs'); +const job = require ('./job'); let db = null; @@ -18,8 +19,10 @@ async function init (use_fake_seed) { if (!fs.existsSync ('data')) fs.mkdirSync ('data'); - if (use_fake_seed && fs.existsSync ('data/db.sqlite')) + if (use_fake_seed && fs.existsSync ('data/db.sqlite')) { fs.unlinkSync ('data/db.sqlite'); + fs.copyFileSync ('data/test.sqlite', 'data/db.sqlite'); + } db = knex ({ client: 'sqlite', @@ -30,10 +33,11 @@ async function init (use_fake_seed) { }); await db.migrate.latest (); - if (use_fake_seed) - await db.seed.run ({ specific: 'fake.js' }); - else - await db.seed.run ({ specific: 'prod.js' }); + + await db.seed.run ({ specific: 'prod.js' }); + + // await db.seed.run ({ specific: 'fake.js' }); + await job (db); } function get_db () { diff --git a/lib/db/job.js b/lib/db/job.js new file mode 100644 index 0000000..8733883 --- /dev/null +++ b/lib/db/job.js @@ -0,0 +1,85 @@ +/* eslint-disable no-console */ +/* eslint-disable no-await-in-loop */ +'use strict'; + +const chunk = require ('lodash.chunk'); + +function get_targets (knex, app_id, duration, reduction = null) { + return knex.from ('log') + .where ({ app_id }) + .andWhere ('timestamp', '<', Number (new Date) - (duration * 1000)) + .andWhere ((builder) => { + if (reduction === null) { + builder.whereNotNull ('id'); + } + else { + builder.where ('reduction', '<', reduction) + .orWhere ('reduction', null); + } + }); +} + +async function batch_delete (knex, ids) { + for (const c of chunk (ids, 100)) { + await knex ('log') + .whereIn ('id', c) + .del (); + } + + return ids.length; +} + +async function batch_update (knex, ids, data) { + for (const c of chunk (ids, 100)) { + await knex ('log') + .whereIn ('id', c) + .update (data); + } + + return ids.length; +} + +module.exports = async (knex) => { + const apps = await knex ('app') + .select ('id', 'name', 'reduction'); + + for (const app of apps) { + const reduction = JSON.parse (app.reduction); + const duplicates = reduction.shift (); + const end = reduction.pop (); + + // delete anything older than now - end + const deleted_old = await get_targets (knex, app.id, end) + .del (); + + console.log (`deleted ${deleted_old} old datasets`); + + for (const r of reduction) { + const targets = (await get_targets (knex, app.id, r, r) + .orderBy ('timestamp') + .select ('id')) + .map ((v) => v.id); + + const even = targets.filter ((v, i) => (i % 2 === 0)); + const odd = targets.filter ((v, i) => (i % 2 !== 0)); + + const deleted_reduction = await batch_delete (knex, even); + + console.log (`reduction ${r} deleted ${deleted_reduction}`); + + await batch_update (knex, odd, { reduction: r }); + } + + const deleted_duplicates = await get_targets (knex, app.id, duplicates) + .andWhere ((builder) => { + builder.whereNotIn ('id', (inBuilder) => { + get_targets (inBuilder, app.id, duplicates) + .groupBy ('message', 'data') + .min ({ id: 'id' }); + }); + }) + .del (); + + console.log (`deleted ${deleted_duplicates} duplicates`); + } +}; diff --git a/lib/defaults.js b/lib/defaults.js new file mode 100644 index 0000000..23afc6f --- /dev/null +++ b/lib/defaults.js @@ -0,0 +1,23 @@ +'use strict'; + +module.exports = { + app: { + + /** + * reduce data after given time durations + * 1. delete duplicates (1 day) + * 2. divide by 2 (1 week) + * ... + * 7. delete all (6 weeks) + */ + reduction: JSON.stringify ([ + 86400, + 604800, + 1209600, + 1814400, + 2419200, + 3024000, + 3628800 + ]) + } +}; diff --git a/migrations/00001-reduction.js b/migrations/00001-reduction.js new file mode 100644 index 0000000..fe31ab7 --- /dev/null +++ b/migrations/00001-reduction.js @@ -0,0 +1,22 @@ +'use strict'; + +const defaults = require ('../lib/defaults'); + +async function up (knex) { + await knex.schema.table ('app', (t) => { + t.string ('reduction'); + }); + + await knex.schema.table ('log', (t) => { + t.integer ('reduction'); + }); + + await knex ('app') + .update ({ reduction: defaults.app.reduction }); +} + +function down () { + // noop +} + +module.exports = { up, down }; diff --git a/package.json b/package.json index 1b3db04..e49172e 100644 --- a/package.json +++ b/package.json @@ -20,6 +20,7 @@ "express-http-proxy": "^1.6.2", "faker": "^4.1.0", "knex": "^0.21.2", + "lodash.chunk": "^4.2.0", "sqlite3": "^5.0.0", "vue": "^2.6.11", "vue-chartjs": "^3.5.0", @@ -45,5 +46,8 @@ "author": { "name": "Timo Hocker", "email": "timo@scode.ovh" + }, + "engines": { + "node": ">=10.0.0" } -} \ No newline at end of file +} diff --git a/seeds/fake.js b/seeds/fake.js index c8410b7..8a1a70b 100644 --- a/seeds/fake.js +++ b/seeds/fake.js @@ -9,16 +9,6 @@ const faker = require ('faker'); -const apps = []; - -async function create_app (knex) { - const [ id ] = await knex ('app') - .insert ( - { name: faker.random.word () } - ); - apps.push (id); -} - let last_t = 0; let last_h = 0; @@ -32,7 +22,7 @@ function create_log (timestamp) { humidity: last_h }; return { - app_id: faker.random.arrayElement (apps), + app_id: 1, message: faker.random.words (), data: JSON.stringify (data), timestamp @@ -40,15 +30,14 @@ function create_log (timestamp) { } async function seed (knex) { + await knex ('log') + .del (); // eslint-disable-next-line no-console console.log ('creating seeds'); - for (let i = 0; i < 5; i++) - // eslint-disable-next-line no-await-in-loop - await create_app (knex); - const log = (Array (1000)) + const log = (Array (10000)) .fill (() => null) - .map (() => faker.date.recent (30)) + .map (() => faker.date.recent (60)) .sort () .map ((t) => create_log (t)); diff --git a/src/components/ChartView.vue b/src/components/ChartView.vue index a90b0ee..c06f75f 100644 --- a/src/components/ChartView.vue +++ b/src/components/ChartView.vue @@ -17,6 +17,10 @@ export default { yaxis: { type: Array, required: true + }, + remove_duplicates: { + type: Boolean, + default: false } }, computed: { @@ -28,10 +32,23 @@ export default { data: [], yAxisID: index, borderColor: y.color, - backgroundColor: y.fill + backgroundColor: y.fill, + spanGaps: true }; - for (const data of this.data) - res.data.push (resolve_data (data, y.field)); + let last = null; + for (let i = 0; i < this.data.length; i++) { + const data = this.data[i]; + const val = resolve_data (data, y.field); + if ( + !this.remove_duplicates + || last !== val + || this.data.length - 1 === i + ) + res.data.push (val); + else + res.data.push (null); + last = val; + } return res; }); return { datasets, labels }; diff --git a/src/components/ViewComponent.vue b/src/components/ViewComponent.vue index 7aa4d59..756bf67 100644 --- a/src/components/ViewComponent.vue +++ b/src/components/ViewComponent.vue @@ -4,6 +4,7 @@ :data="[...data].reverse()" :xaxis="config.x" :yaxis="config.y" + :remove_duplicates="config.remove_duplicates" />