prevent huge data amounts, separate sources

lib/api/get-log.js

@@ -11,15 +11,19 @@ const db = require ('../db');
 const { http } = require ('@sapphirecode/consts');
 
 module.exports = async (req, res) => {
-  if (
+  const limit = parseInt (req.headers.limit);
-    typeof req.headers.app_id === 'undefined'
+  const offset = parseInt (req.headers.offset);
-    || isNaN (parseInt (req.headers.app_id))
+  const app_id = parseInt (req.headers.app_id);
-  ) {
+
+  if (isNaN (app_id)) {
     res.status (http.status_bad_request)
-      .end ();
+      .end ('app id not specified');
-  }
-  else {
-    res.status (http.status_ok)
-      .json (await db.log.get_all (parseInt (req.headers.app_id)));
   }
+
+  res.status (http.status_ok)
+    .json (await db.log.get_all (
+      parseInt (req.headers.app_id),
+      isNaN (limit) ? 100 : limit,
+      isNaN (offset) ? 0 : offset
+    ));
 };

lib/db/log.js

@@ -8,7 +8,7 @@
 'use strict';
 
 module.exports = (get_db) => ({
-  get_all (app_id) {
+  get_all (app_id, limit = 100, offset = 0) {
     const knex = get_db ();
     return knex.select (
       'id',
@@ -17,7 +17,10 @@ module.exports = (get_db) => ({
       'timestamp'
     )
       .from ('log')
-      .where ({ app_id });
+      .where ({ app_id })
+      .orderBy ('timestamp')
+      .limit (Math.min (limit, 10000))
+      .offset (offset);
   },
   insert (app_id, message, data = '{}', timestamp = (new Date)) {
     const knex = get_db ();

package.json

@@ -10,6 +10,7 @@
   },
   "dependencies": {
     "@sapphirecode/consts": "^1.1.28",
+    "@sapphirecode/crypto-helper": "^1.1.57",
     "@sapphirecode/utilities": "^1.8.5",
     "body-parser": "^1.19.0",
     "chart.js": "^2.9.3",

src/components/ConfigEditor.vue

@@ -7,6 +7,14 @@
       class="label"
       v-text="template.name"
     />
+    <button
+      v-if="removable"
+      class="array_remove"
+      type="button"
+      @click="$emit('remove')"
+    >
+      X
+    </button>
     <!-- editor -->
     <div
       v-if="template.type === 'object'"
@@ -26,10 +34,11 @@
     >
       <ConfigEditor
         v-for="(child,key) of config"
-        :key="key"
+        :key="salt (key)"
         v-model="config[key]"
         :template="template.child"
-        :enabled="is_enabled(template.child.if)"
+        :removable="true"
+        @remove="config.splice(key, 1)"
       />
       <button
         type="button"
@@ -83,8 +92,10 @@
 </template>
 
 <script>
+import { create_salt } from '@sapphirecode/crypto-helper';
 import { resolve_data } from '../helper';
 
+
 export default {
   name:  'ConfigEditor',
   props: {
@@ -98,6 +109,10 @@ export default {
     enabled: {
       type:    Boolean,
       default: true
+    },
+    removable: {
+      type:     Boolean,
+      default:  false
     }
   },
   data () {
@@ -155,6 +170,9 @@ export default {
         return res;
       }
       return null;
+    },
+    salt () {
+      return create_salt ();
     }
   }
 };
@@ -177,4 +195,11 @@ export default {
   display: inline-block;
   grid-column: label;
 }
+
+.array_remove {
+  display: inline-block;
+  width: 20px;
+  height: 20px;
+  padding: 0;
+}
 </style>

src/default.js

@@ -5,38 +5,49 @@
  * Created by Timo Hocker <timo@scode.ovh>, August 2020
  */
 
-export default [
+export default {
-  {
+  sources:  [
-    type:    'chart',
+    {
-    x:    'timestamp',
+      name:   'default',
-    y:    [
+      limit:  10,
-      {
+      offset: 0
-        label: 'temperature',
+    }
-        field: 'data/temperature',
+  ],
-        color: '#ff0000',
+  displays: [
-        fill:  '#0000'
+    {
-      },
+      source: 'default',
-      {
+      type:    'chart',
-        label: 'humidity',
+      x:      'timestamp',
-        field: 'data/humidity',
+      y:      [
-        color: '#0000ff',
+        {
-        fill:  '#0000'
+          label: 'temperature',
-      },
+          field: 'data/temperature',
-      {
+          color: '#ff0000',
-        label: 'light',
+          fill:  '#0000'
-        field: 'data/light',
+        },
-        color: '#ffff00',
+        {
-        fill:  '#0000'
+          label: 'humidity',
-      }
+          field: 'data/humidity',
-    ]
+          color: '#0000ff',
-  },
+          fill:  '#0000'
-  {
+        },
-    type:    'table',
+        {
-    columns: [
+          label: 'light',
-      'id',
+          field: 'data/light',
-      'message',
+          color: '#ffff00',
-      'data',
+          fill:  '#0000'
-      'timestamp'
+        }
-    ]
+      ]
-  }
+    },
-];
+    {
+      source:  'default',
+      type:    'table',
+      columns: [
+        'id',
+        'message',
+        'data',
+        'timestamp'
+      ]
+    }
+  ]
+};

src/store/index.js

@@ -11,17 +11,42 @@ import Vuex from 'vuex';
 Vue.use (Vuex);
 
 export default new Vuex.Store ({
-  state:     { log: [] },
+  state:     { log: {} },
   mutations: {
     set_log (state, log) {
       state.log = log;
     }
   },
+  getters: {
+    log (state) {
+      return (source) => {
+        if (typeof state.log[source] === 'undefined')
+          return [];
+        return state.log[source];
+      };
+    }
+  },
   actions:   {
-    async get_log ({ commit }, { app_id }) {
+    async get_log ({ commit }, { app_id, sources }) {
-      const log = await fetch ('/log', { headers: { app_id } })
+      const logs = {};
-        .then ((res) => res.json ());
+      for (const source of sources) {
-      commit ('set_log', log);
+        // eslint-disable-next-line no-await-in-loop
+        const log = await fetch ('/log', {
+          headers: {
+            app_id,
+            offset: source.offset,
+            limit:  source.limit
+          }
+        })
+          .then ((res) => res.json ())
+          .then ((json) => json.map ((entry) => {
+            entry.data = JSON.parse (entry.data);
+            return entry;
+          }));
+        logs[source.name] = log;
+      }
+
+      commit ('set_log', logs);
     }
   },
   modules: {}

src/template.js

@@ -6,43 +6,74 @@
  */
 
 export default {
-  type:  'array',
+  type:       'object',
-  child: {
+  properties: [
-    type:       'object',
+    {
-    properties: [
+      type:  'array',
-      {
+      name:  'sources',
-        type:    'string',
+      child: {
-        name:    'type',
+        type:       'object',
-        choices: [
+        properties: [
-          'table',
+          {
-          'chart'
+            type: 'string',
+            name: 'name'
+          },
+          {
+            type: 'number',
+            name: 'limit'
+          },
+          {
+            type: 'number',
+            name: 'offset'
+          }
         ]
-      },
-      {
-        name:  'columns',
-        type:  'array',
-        child: { type: 'string' },
-        if:    { prop: 'type', op: '=', val: 'table' }
-      },
-      {
-        name: 'x',
-        type: 'string',
-        if:    { prop: 'type', op: '=', val: 'chart' }
-      },
-      {
-        name:  'y',
-        type:  'array',
-        if:    { prop: 'type', op: '=', val: 'chart' },
-        child: {
-          type:       'object',
-          properties: [
-            { type: 'string', name: 'label' },
-            { type: 'string', name: 'field' },
-            { type: 'string', name: 'color' },
-            { type: 'string', name: 'fill' }
-          ]
-        }
       }
-    ]
+    },
-  }
+    {
+      type:  'array',
+      name:  'displays',
+      child: {
+        type:       'object',
+        properties: [
+          {
+            type: 'string',
+            name: 'source'
+          },
+          {
+            type:    'string',
+            name:    'type',
+            choices: [
+              'table',
+              'chart'
+            ]
+          },
+          {
+            name:  'columns',
+            type:  'array',
+            child: { type: 'string' },
+            if:    { prop: 'type', op: '=', val: 'table' }
+          },
+          {
+            name: 'x',
+            type: 'string',
+            if:    { prop: 'type', op: '=', val: 'chart' }
+          },
+          {
+            name:  'y',
+            type:  'array',
+            if:    { prop: 'type', op: '=', val: 'chart' },
+            child: {
+              type:       'object',
+              properties: [
+                { type: 'string', name: 'label' },
+                { type: 'string', name: 'field' },
+                { type: 'string', name: 'color' },
+                { type: 'string', name: 'fill' }
+              ]
+            }
+          }
+        ]
+      }
+    }
+  ]
 };

src/views/AppView.vue

@@ -7,10 +7,10 @@
       :template="template"
     />
     <ViewComponent
-      v-for="(item,key) of saved_config"
+      v-for="(item,key) of saved_config.displays"
       :key="key"
       :config="item"
-      :data="parsed_log"
+      :data="log(item.source)"
     />
   </div>
 </template>
@@ -32,17 +32,9 @@ export default {
       template:     default_template
     };
   },
-  computed:   {
+  computed: { ...Vuex.mapGetters ({ log: 'log' }) },
-    parsed_log () {
-      return this.log.map ((l) => {
-        l.data = JSON.parse (l.data);
-        return l;
-      });
-    },
-    ...Vuex.mapState ({ log: (state) => state.log })
-  },
   mounted () {
-    this.get_log ({ app_id: this.$route.params.id });
+    this.fetch_log ();
     document.body.addEventListener ('keydown', (ev) => {
       if (ev.key === 's' && ev.ctrlKey) {
         this.save_config ();
@@ -59,6 +51,13 @@ export default {
         body:   JSON.stringify (this.config)
       });
       this.saved_config = copy_object (this.config);
+      this.fetch_log ();
+    },
+    fetch_log () {
+      this.get_log ({
+        app_id:  this.$route.params.id,
+        sources: this.saved_config.sources
+      });
     },
     ...Vuex.mapActions ({ get_log: 'get_log' })
   }

yarn.lock

@@ -941,6 +941,18 @@
   resolved "https://registry.yarnpkg.com/@sapphirecode/consts/-/consts-1.1.28.tgz#4f9400a80666c3e41b55acada999c877d32eb55c"
   integrity sha512-OVZpkhOJtdzf379GNVPBb3D1iqTTSger8/LF/hmq9dqvrcfHm+rara2jChg9qOqk840EEzZCLEFLjyEMz+aDeA==
 
+"@sapphirecode/crypto-helper@^1.1.57":
+  version "1.1.57"
+  resolved "https://registry.yarnpkg.com/@sapphirecode/crypto-helper/-/crypto-helper-1.1.57.tgz#cfa7d7fefd417e875c2b080816b63edf699d79bf"
+  integrity sha512-ReKGCFOMq+S8y/XEY2bjG5BQFoYzoebiJi/1BYuu6WXLLTIzGO2JkZ7rsFZV2FEW1dQGVdbjpGmovMwtZjJhtw==
+  dependencies:
+    "@sapphirecode/encoding-helper" "^1.0.38"
+
+"@sapphirecode/encoding-helper@^1.0.38":
+  version "1.0.49"
+  resolved "https://registry.yarnpkg.com/@sapphirecode/encoding-helper/-/encoding-helper-1.0.49.tgz#cb5389ff3b469910b4067b5b0487e99c55320424"
+  integrity sha512-iMgBXnXPFDuPlWJdECap+VIAwrkN0wAUK/ah4V5xJM1jmUhCHjqORv75bhkmPY+92DeusZaxBnQkC1hboeTW7A==
+
 "@sapphirecode/eslint-config-es6@^1.1.1":
   version "1.1.17"
   resolved "https://registry.yarnpkg.com/@sapphirecode/eslint-config-es6/-/eslint-config-es6-1.1.17.tgz#52e554d1fa9870cc813d2a39bb84ae85f9db612d"