88 lines
2.2 KiB
JavaScript
88 lines
2.2 KiB
JavaScript
/* eslint-disable no-console */
|
|
/* eslint-disable no-await-in-loop */
|
|
'use strict';
|
|
|
|
const chunk = require ('lodash.chunk');
|
|
|
|
function get_targets (knex, app_id, duration, reduction = null) {
|
|
return knex.from ('log')
|
|
.where ({ app_id })
|
|
.andWhere ('timestamp', '<', Number (new Date) - (duration * 1000))
|
|
.andWhere ((builder) => {
|
|
if (reduction === null) {
|
|
builder.whereNotNull ('id');
|
|
}
|
|
else {
|
|
builder.where ('reduction', '<', reduction)
|
|
.orWhere ('reduction', null);
|
|
}
|
|
});
|
|
}
|
|
|
|
async function batch_delete (knex, ids) {
|
|
for (const c of chunk (ids, 100)) {
|
|
await knex ('log')
|
|
.whereIn ('id', c)
|
|
.del ();
|
|
}
|
|
|
|
return ids.length;
|
|
}
|
|
|
|
async function batch_update (knex, ids, data) {
|
|
for (const c of chunk (ids, 100)) {
|
|
await knex ('log')
|
|
.whereIn ('id', c)
|
|
.update (data);
|
|
}
|
|
|
|
return ids.length;
|
|
}
|
|
|
|
module.exports = async (knex) => {
|
|
console.log ('-- running database cleanup --');
|
|
|
|
const apps = await knex ('app')
|
|
.select ('id', 'name', 'reduction');
|
|
|
|
for (const app of apps) {
|
|
const reduction = JSON.parse (app.reduction);
|
|
const duplicates = reduction.shift ();
|
|
const end = reduction.pop ();
|
|
|
|
// delete anything older than now - end
|
|
const deleted_old = await get_targets (knex, app.id, end)
|
|
.del ();
|
|
|
|
console.log (`deleted ${deleted_old} old datasets`);
|
|
|
|
for (const r of reduction) {
|
|
const targets = (await get_targets (knex, app.id, r, r)
|
|
.orderBy ('timestamp')
|
|
.select ('id'))
|
|
.map ((v) => v.id);
|
|
|
|
const even = targets.filter ((v, i) => (i % 2 === 0));
|
|
const odd = targets.filter ((v, i) => (i % 2 !== 0));
|
|
|
|
const deleted_reduction = await batch_delete (knex, even);
|
|
|
|
console.log (`reduction ${r} deleted ${deleted_reduction}`);
|
|
|
|
await batch_update (knex, odd, { reduction: r });
|
|
}
|
|
|
|
const deleted_duplicates = await get_targets (knex, app.id, duplicates)
|
|
.andWhere ((builder) => {
|
|
builder.whereNotIn ('id', (inBuilder) => {
|
|
get_targets (inBuilder, app.id, duplicates)
|
|
.groupBy ('message', 'data')
|
|
.min ({ id: 'id' });
|
|
});
|
|
})
|
|
.del ();
|
|
|
|
console.log (`deleted ${deleted_duplicates} duplicates`);
|
|
}
|
|
};
|