add csv ingress to knex migrations
This commit is contained in:
parent
b79bfdde43
commit
615171e409
1
.eslintignore
Normal file
1
.eslintignore
Normal file
@ -0,0 +1 @@
|
|||||||
|
**/*.template.js
|
2
Jenkinsfile
vendored
2
Jenkinsfile
vendored
@ -5,7 +5,7 @@ pipeline {
|
|||||||
VERSION = VersionNumber([
|
VERSION = VersionNumber([
|
||||||
versionNumberString:
|
versionNumberString:
|
||||||
'${BUILDS_ALL_TIME}',
|
'${BUILDS_ALL_TIME}',
|
||||||
versionPrefix: '1.0.',
|
versionPrefix: '1.1.',
|
||||||
worstResultForIncrement: 'SUCCESS'
|
worstResultForIncrement: 'SUCCESS'
|
||||||
])
|
])
|
||||||
}
|
}
|
||||||
|
58
snippets/migration/csv.template.js
Normal file
58
snippets/migration/csv.template.js
Normal file
@ -0,0 +1,58 @@
|
|||||||
|
'use strict';
|
||||||
|
|
||||||
|
const csv = require ('csv-parse');
|
||||||
|
const fs = require ('fs-extra');
|
||||||
|
const path = require ('path');
|
||||||
|
|
||||||
|
/**
|
||||||
|
* get all csv data that should be loaded
|
||||||
|
*
|
||||||
|
* @returns {Promise<Array<object>>} csv data
|
||||||
|
*/
|
||||||
|
async function get_csv () {
|
||||||
|
const csv_list = await Promise.all ((
|
||||||
|
await fs.readdir ('migrations/csv'))
|
||||||
|
.map (
|
||||||
|
async (file) => {
|
||||||
|
if (path.extname (file) !== '.csv')
|
||||||
|
return null;
|
||||||
|
const name = path.basename (file, '.csv');
|
||||||
|
|
||||||
|
const data = [];
|
||||||
|
const parser = csv (await fs.readFile (
|
||||||
|
path.join ('migrations/csv', file)
|
||||||
|
), { columns: true });
|
||||||
|
|
||||||
|
for await (const entry of parser)
|
||||||
|
data.push (entry);
|
||||||
|
|
||||||
|
return { name, data };
|
||||||
|
}
|
||||||
|
));
|
||||||
|
|
||||||
|
return csv_list.filter ((val) => val !== null);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* execute migration
|
||||||
|
*
|
||||||
|
* @param {any} knex database connection
|
||||||
|
*/
|
||||||
|
async function up (knex) {
|
||||||
|
const data = await get_csv ();
|
||||||
|
await Promise.all (
|
||||||
|
data.map (
|
||||||
|
(table) => knex (table.name)
|
||||||
|
.insert (table.data)
|
||||||
|
)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* revert migration
|
||||||
|
*/
|
||||||
|
function down () {
|
||||||
|
// noop
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = { up, down };
|
@ -20,8 +20,12 @@ const path = require ('path');
|
|||||||
* @param {Array} args function arguments
|
* @param {Array} args function arguments
|
||||||
*/
|
*/
|
||||||
async function run (folder, args) {
|
async function run (folder, args) {
|
||||||
const template = path.join (__dirname, 'template.js');
|
const template = path.join (
|
||||||
const dest = path.join (folder, `${args[0]}.js`);
|
__dirname,
|
||||||
|
args[0] === 'csv' ? 'csv.template.js' : 'template.js'
|
||||||
|
);
|
||||||
|
const target_name = args[0] === 'csv' ? '000001_csv_ingress' : args[0];
|
||||||
|
const dest = path.join (folder, `${target_name}.js`);
|
||||||
await fs.writeFile (dest, await fs.readFile (template));
|
await fs.writeFile (dest, await fs.readFile (template));
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -43,7 +47,7 @@ function assert (folder, args) {
|
|||||||
reason: 'name is not a string'
|
reason: 'name is not a string'
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
f: () => ((/^[0-9]+-[a-z]+$/iu).test (args[0])),
|
f: () => ((/^(?:[0-9]+-[a-z]+|csv)$/iu).test (args[0])),
|
||||||
reason: 'name has to match /^[0-9]+-[a-z]+$/'
|
reason: 'name has to match /^[0-9]+-[a-z]+$/'
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
|
Loading…
x
Reference in New Issue
Block a user