prevent huge data amounts, separate sources

lib/api/get-log.js

@@ -11,15 +11,19 @@ const db = require ('../db');
 const { http } = require ('@sapphirecode/consts');
 
 module.exports = async (req, res) => {
-  if (
+  const limit = parseInt (req.headers.limit);
-    typeof req.headers.app_id === 'undefined'
+  const offset = parseInt (req.headers.offset);
-    || isNaN (parseInt (req.headers.app_id))
+  const app_id = parseInt (req.headers.app_id);
-  ) {
+
+  if (isNaN (app_id)) {
     res.status (http.status_bad_request)
-      .end ();
+      .end ('app id not specified');
   }
-  else {
+
   res.status (http.status_ok)
-      .json (await db.log.get_all (parseInt (req.headers.app_id)));
+    .json (await db.log.get_all (
-  }
+      parseInt (req.headers.app_id),
+      isNaN (limit) ? 100 : limit,
+      isNaN (offset) ? 0 : offset
+    ));
 };

lib/db/log.js

@@ -8,7 +8,7 @@
 'use strict';
 
 module.exports = (get_db) => ({
-  get_all (app_id) {
+  get_all (app_id, limit = 100, offset = 0) {
     const knex = get_db ();
     return knex.select (
       'id',
@@ -17,7 +17,10 @@ module.exports = (get_db) => ({
       'timestamp'
     )
       .from ('log')
-      .where ({ app_id });
+      .where ({ app_id })
+      .orderBy ('timestamp')
+      .limit (Math.min (limit, 10000))
+      .offset (offset);
   },
   insert (app_id, message, data = '{}', timestamp = (new Date)) {
     const knex = get_db ();

package.json

@@ -10,6 +10,7 @@
   },
   "dependencies": {
     "@sapphirecode/consts": "^1.1.28",
+    "@sapphirecode/crypto-helper": "^1.1.57",
     "@sapphirecode/utilities": "^1.8.5",
     "body-parser": "^1.19.0",
     "chart.js": "^2.9.3",

src/components/ConfigEditor.vue

@@ -7,6 +7,14 @@
       class="label"
       v-text="template.name"
     />
+    <button
+      v-if="removable"
+      class="array_remove"
+      type="button"
+      @click="$emit('remove')"
+    >
+      X
+    </button>
     <!-- editor -->
     <div
       v-if="template.type === 'object'"
@@ -26,10 +34,11 @@
     >
       <ConfigEditor
         v-for="(child,key) of config"
-        :key="key"
+        :key="salt (key)"
         v-model="config[key]"
         :template="template.child"
-        :enabled="is_enabled(template.child.if)"
+        :removable="true"
+        @remove="config.splice(key, 1)"
       />
       <button
         type="button"
@@ -83,8 +92,10 @@
 </template>
 
 <script>
+import { create_salt } from '@sapphirecode/crypto-helper';
 import { resolve_data } from '../helper';
 
+
 export default {
   name:  'ConfigEditor',
   props: {
@@ -98,6 +109,10 @@ export default {
     enabled: {
       type:    Boolean,
       default: true
+    },
+    removable: {
+      type:     Boolean,
+      default:  false
     }
   },
   data () {
@@ -155,6 +170,9 @@ export default {
         return res;
       }
       return null;
+    },
+    salt () {
+      return create_salt ();
     }
   }
 };
@@ -177,4 +195,11 @@ export default {
   display: inline-block;
   grid-column: label;
 }
+
+.array_remove {
+  display: inline-block;
+  width: 20px;
+  height: 20px;
+  padding: 0;
+}
 </style>

src/default.js

@@ -5,8 +5,17 @@
  * Created by Timo Hocker <timo@scode.ovh>, August 2020
  */
 
-export default [
+export default {
+  sources:  [
     {
+      name:   'default',
+      limit:  10,
+      offset: 0
+    }
+  ],
+  displays: [
+    {
+      source: 'default',
       type:    'chart',
       x:      'timestamp',
       y:      [
@@ -31,6 +40,7 @@ export default [
       ]
     },
     {
+      source:  'default',
       type:    'table',
       columns: [
         'id',
@@ -39,4 +49,5 @@ export default [
         'timestamp'
       ]
     }
-];
+  ]
+};

src/store/index.js

@@ -11,17 +11,42 @@ import Vuex from 'vuex';
 Vue.use (Vuex);
 
 export default new Vuex.Store ({
-  state:     { log: [] },
+  state:     { log: {} },
   mutations: {
     set_log (state, log) {
       state.log = log;
     }
   },
+  getters: {
+    log (state) {
+      return (source) => {
+        if (typeof state.log[source] === 'undefined')
+          return [];
+        return state.log[source];
+      };
+    }
+  },
   actions:   {
-    async get_log ({ commit }, { app_id }) {
+    async get_log ({ commit }, { app_id, sources }) {
-      const log = await fetch ('/log', { headers: { app_id } })
+      const logs = {};
-        .then ((res) => res.json ());
+      for (const source of sources) {
-      commit ('set_log', log);
+        // eslint-disable-next-line no-await-in-loop
+        const log = await fetch ('/log', {
+          headers: {
+            app_id,
+            offset: source.offset,
+            limit:  source.limit
+          }
+        })
+          .then ((res) => res.json ())
+          .then ((json) => json.map ((entry) => {
+            entry.data = JSON.parse (entry.data);
+            return entry;
+          }));
+        logs[source.name] = log;
+      }
+
+      commit ('set_log', logs);
     }
   },
   modules: {}

src/template.js

@@ -6,10 +6,39 @@
  */
 
 export default {
+  type:       'object',
+  properties: [
+    {
       type:  'array',
+      name:  'sources',
       child: {
         type:       'object',
         properties: [
+          {
+            type: 'string',
+            name: 'name'
+          },
+          {
+            type: 'number',
+            name: 'limit'
+          },
+          {
+            type: 'number',
+            name: 'offset'
+          }
+        ]
+      }
+    },
+    {
+      type:  'array',
+      name:  'displays',
+      child: {
+        type:       'object',
+        properties: [
+          {
+            type: 'string',
+            name: 'source'
+          },
           {
             type:    'string',
             name:    'type',
@@ -45,4 +74,6 @@ export default {
           }
         ]
       }
+    }
+  ]
 };

src/views/AppView.vue

@@ -7,10 +7,10 @@
       :template="template"
     />
     <ViewComponent
-      v-for="(item,key) of saved_config"
+      v-for="(item,key) of saved_config.displays"
       :key="key"
       :config="item"
-      :data="parsed_log"
+      :data="log(item.source)"
     />
   </div>
 </template>
@@ -32,17 +32,9 @@ export default {
       template:     default_template
     };
   },
-  computed:   {
+  computed: { ...Vuex.mapGetters ({ log: 'log' }) },
-    parsed_log () {
-      return this.log.map ((l) => {
-        l.data = JSON.parse (l.data);
-        return l;
-      });
-    },
-    ...Vuex.mapState ({ log: (state) => state.log })
-  },
   mounted () {
-    this.get_log ({ app_id: this.$route.params.id });
+    this.fetch_log ();
     document.body.addEventListener ('keydown', (ev) => {
       if (ev.key === 's' && ev.ctrlKey) {
         this.save_config ();
@@ -59,6 +51,13 @@ export default {
         body:   JSON.stringify (this.config)
       });
       this.saved_config = copy_object (this.config);
+      this.fetch_log ();
+    },
+    fetch_log () {
+      this.get_log ({
+        app_id:  this.$route.params.id,
+        sources: this.saved_config.sources
+      });
     },
     ...Vuex.mapActions ({ get_log: 'get_log' })
   }

yarn.lock

@@ -941,6 +941,18 @@
   resolved "https://registry.yarnpkg.com/@sapphirecode/consts/-/consts-1.1.28.tgz#4f9400a80666c3e41b55acada999c877d32eb55c"
   integrity sha512-OVZpkhOJtdzf379GNVPBb3D1iqTTSger8/LF/hmq9dqvrcfHm+rara2jChg9qOqk840EEzZCLEFLjyEMz+aDeA==
 
+"@sapphirecode/crypto-helper@^1.1.57":
+  version "1.1.57"
+  resolved "https://registry.yarnpkg.com/@sapphirecode/crypto-helper/-/crypto-helper-1.1.57.tgz#cfa7d7fefd417e875c2b080816b63edf699d79bf"
+  integrity sha512-ReKGCFOMq+S8y/XEY2bjG5BQFoYzoebiJi/1BYuu6WXLLTIzGO2JkZ7rsFZV2FEW1dQGVdbjpGmovMwtZjJhtw==
+  dependencies:
+    "@sapphirecode/encoding-helper" "^1.0.38"
+
+"@sapphirecode/encoding-helper@^1.0.38":
+  version "1.0.49"
+  resolved "https://registry.yarnpkg.com/@sapphirecode/encoding-helper/-/encoding-helper-1.0.49.tgz#cb5389ff3b469910b4067b5b0487e99c55320424"
+  integrity sha512-iMgBXnXPFDuPlWJdECap+VIAwrkN0wAUK/ah4V5xJM1jmUhCHjqORv75bhkmPY+92DeusZaxBnQkC1hboeTW7A==
+
 "@sapphirecode/eslint-config-es6@^1.1.1":
   version "1.1.17"
   resolved "https://registry.yarnpkg.com/@sapphirecode/eslint-config-es6/-/eslint-config-es6-1.1.17.tgz#52e554d1fa9870cc813d2a39bb84ae85f9db612d"