/* eslint-disable no-console */ /* eslint-disable no-await-in-loop */ 'use strict'; const chunk = require ('lodash.chunk'); function get_targets (knex, app_id, duration, reduction = null) { return knex.from ('log') .where ({ app_id }) .andWhere ('timestamp', '<', Number (new Date) - (duration * 1000)) .andWhere ((builder) => { if (reduction === null) { builder.whereNotNull ('id'); } else { builder.where ('reduction', '<', reduction) .orWhere ('reduction', null); } }); } async function batch_delete (knex, ids) { for (const c of chunk (ids, 100)) { await knex ('log') .whereIn ('id', c) .del (); } return ids.length; } async function batch_update (knex, ids, data) { for (const c of chunk (ids, 100)) { await knex ('log') .whereIn ('id', c) .update (data); } return ids.length; } module.exports = async (knex) => { console.log ('-- running database cleanup --'); const apps = await knex ('app') .select ('id', 'name', 'reduction'); for (const app of apps) { const reduction = JSON.parse (app.reduction); const duplicates = reduction.shift (); const end = reduction.pop (); // delete anything older than now - end const deleted_old = await get_targets (knex, app.id, end) .del (); console.log (`deleted ${deleted_old} old datasets`); for (const r of reduction) { const targets = (await get_targets (knex, app.id, r, r) .orderBy ('timestamp') .select ('id')) .map ((v) => v.id); const even = targets.filter ((v, i) => (i % 2 === 0)); const odd = targets.filter ((v, i) => (i % 2 !== 0)); const deleted_reduction = await batch_delete (knex, even); console.log (`reduction ${r} deleted ${deleted_reduction}`); await batch_update (knex, odd, { reduction: r }); } const deleted_duplicates = await get_targets (knex, app.id, duplicates) .andWhere ((builder) => { builder.whereNotIn ('id', (inBuilder) => { get_targets (inBuilder, app.id, duplicates) .groupBy ('message', 'data') .min ({ id: 'id' }); }); }) .del (); console.log (`deleted ${deleted_duplicates} duplicates`); } };