Copy utils from cytube-common and remove dep

The `cytube-common` module was created as part of a now-defunct experiment and since then has just remained a crufty container for a few utils. Moved the utils to the main repo and removed the dependency.
calzoneman · Jul 20, 2017 · ff3ecec · ff3ecec
1 parent e780e7d
commit ff3ecec
Show file tree

Hide file tree

Showing 13 changed files with 358 additions and 9 deletions.
diff --git a/integration_test/testutil/config.js b/integration_test/testutil/config.js
@@ -1,4 +1,4 @@
-const loadFromToml = require('cytube-common/lib/configuration/configloader').loadFromToml;
+const loadFromToml = require('../../lib/configuration/configloader').loadFromToml;
 const path = require('path');
 
 class IntegrationTestConfig {

diff --git a/package.json b/package.json
@@ -18,7 +18,6 @@
     "cookie-parser": "^1.4.0",
     "create-error": "^0.3.1",
     "csrf": "^3.0.0",
-    "cytube-common": "git://github.com/CyTube/cytube-common",
     "cytube-mediaquery": "git://github.com/CyTube/mediaquery",
     "cytubefilters": "git://github.com/calzoneman/cytubefilters#67c7c69a",
     "express": "^4.13.3",

diff --git a/src/config.js b/src/config.js
@@ -4,7 +4,7 @@ var nodemailer = require("nodemailer");
 var net = require("net");
 var YAML = require("yamljs");
 
-import { loadFromToml } from 'cytube-common/lib/configuration/configloader';
+import { loadFromToml } from './configuration/configloader';
 import { CamoConfig } from './configuration/camoconfig';
 import { PrometheusConfig } from './configuration/prometheusconfig';
 

diff --git a/src/configuration/configloader.js b/src/configuration/configloader.js
@@ -0,0 +1,19 @@
+import toml from 'toml';
+import fs from 'fs';
+
+/** @module cytube-common/configuration/configloader */
+
+/**
+ * Load a toml file and pass the results to a configuration
+ * constructor.
+ *
+ * @param {function} constructor Constructor to call with the loaded data
+ * @param {string} filename Path to the toml file to load
+ * @returns {Object} Configuration object constructed from the provided constructor
+ * @throws {SyntaxError} Errors propagated from toml.parse()
+ */
+export function loadFromToml(constructor, filename) {
+    const rawContents = fs.readFileSync(filename).toString('utf8');
+    const configData = toml.parse(rawContents);
+    return new (constructor)(configData);
+}
diff --git a/src/counters.js b/src/counters.js
@@ -1,7 +1,7 @@
 import io from 'socket.io';
 import Socket from 'socket.io/lib/socket';
-import * as Metrics from 'cytube-common/lib/metrics/metrics';
-import { JSONFileMetricsReporter } from 'cytube-common/lib/metrics/jsonfilemetricsreporter';
+import * as Metrics from './metrics/metrics';
+import { JSONFileMetricsReporter } from './metrics/jsonfilemetricsreporter';
 
 const LOGGER = require('@calzoneman/jsli')('counters');
 

diff --git a/src/database.js b/src/database.js
@@ -4,7 +4,7 @@ var Config = require("./config");
 var tables = require("./database/tables");
 var net = require("net");
 var util = require("./utilities");
-import * as Metrics from 'cytube-common/lib/metrics/metrics';
+import * as Metrics from './metrics/metrics';
 import knex from 'knex';
 import { GlobalBanDB } from './db/globalban';
 

diff --git a/src/metrics/jsonfilemetricsreporter.js b/src/metrics/jsonfilemetricsreporter.js
@@ -0,0 +1,73 @@
+import fs from 'fs';
+
+/** MetricsReporter that records metrics as JSON objects in a file, one per line */
+class JSONFileMetricsReporter {
+    /**
+     * Create a new JSONFileMetricsReporter that writes to the given file path.
+     *
+     * @param {string} filename file path to write to
+     */
+    constructor(filename) {
+        this.writeStream = fs.createWriteStream(filename, { flags: 'a' });
+        this.metrics = {};
+        this.timers = {};
+    }
+
+    /**
+     * @see {@link module:cytube-common/metrics/metrics.incCounter}
+     */
+    incCounter(counter, value) {
+        if (!this.metrics.hasOwnProperty(counter)) {
+            this.metrics[counter] = 0;
+        }
+
+        this.metrics[counter] += value;
+    }
+
+    /**
+     * Add a time metric
+     *
+     * @param {string} timer name of the timer
+     * @param {number} ms milliseconds to record
+     */
+    addTime(timer, ms) {
+        if (!this.timers.hasOwnProperty(timer)) {
+            this.timers[timer] = {
+                totalTime: 0,
+                count: 0,
+                p100: 0
+            };
+        }
+
+        this.timers[timer].totalTime += ms;
+        this.timers[timer].count++;
+        if (ms > this.timers[timer].p100) {
+            this.timers[timer].p100 = ms;
+        }
+    }
+
+    /**
+     * @see {@link module:cytube-common/metrics/metrics.addProperty}
+     */
+    addProperty(property, value) {
+        this.metrics[property] = value;
+    }
+
+    report() {
+        for (const timer in this.timers) {
+            this.metrics[timer+':avg'] = this.timers[timer].totalTime / this.timers[timer].count;
+            this.metrics[timer+':count'] = this.timers[timer].count;
+            this.metrics[timer+':p100'] = this.timers[timer].p100;
+        }
+
+        const line = JSON.stringify(this.metrics) + '\n';
+        try {
+            this.writeStream.write(line);
+        } finally {
+            this.metrics = {};
+            this.timers = {};
+        }
+    }
+}
+
+export { JSONFileMetricsReporter };
diff --git a/src/metrics/metrics.js b/src/metrics/metrics.js
@@ -0,0 +1,132 @@
+import os from 'os';
+
+/** @module cytube-common/metrics/metrics */
+
+const MEM_RSS = 'memory:rss';
+const LOAD_1MIN = 'load:1min';
+const TIMESTAMP = 'time';
+const logger = require('@calzoneman/jsli')('metrics');
+
+var delegate = null;
+var reportInterval = null;
+var reportHooks = [];
+let warnedNoReporter = false;
+
+function warnNoReporter() {
+    if (!warnedNoReporter) {
+        warnedNoReporter = true;
+        logger.warn('No metrics reporter configured.  Metrics will not be recorded.');
+    }
+}
+
+/**
+ * Increment a metrics counter by the specified amount.
+ *
+ * @param {string} counter name of the counter to increment
+ * @param {number} value optional value to increment by (default 1)
+ */
+export function incCounter(counter, amount = 1) {
+    if (delegate === null) {
+        warnNoReporter();
+    } else {
+        delegate.incCounter(counter, amount);
+    }
+}
+
+/**
+ * Start a timer.  Returns a handle to use to end the timer.
+ *
+ * @param {string} timer name
+ * @return {object} timer handle
+ */
+export function startTimer(timer) {
+    return {
+        timer: timer,
+        hrtime: process.hrtime()
+    };
+}
+
+/**
+ * Stop a timer and record the time (as an average)
+ *
+ * @param {object} handle timer handle to Stop
+ */
+export function stopTimer(handle) {
+    if (delegate === null) {
+        warnNoReporter();
+        return;
+    }
+    const [seconds, ns] = process.hrtime(handle.hrtime);
+    delegate.addTime(handle.timer, seconds*1e3 + ns/1e6);
+}
+
+/**
+ * Add a property to the current metrics period.
+ *
+ * @param {string} property property name to add
+ * @param {any} property value
+ */
+export function addProperty(property, value) {
+    if (delegate === null) {
+        warnNoReporter();
+    } else {
+        delegate.addProperty(property, value);
+    }
+}
+
+/**
+ * Set the metrics reporter to record to.
+ *
+ * @param {MetricsReporter} reporter reporter to record metrics to
+ */
+export function setReporter(reporter) {
+    delegate = reporter;
+}
+
+/**
+ * Set the interval at which to report metrics.
+ *
+ * @param {number} interval time in milliseconds between successive reports
+ */
+export function setReportInterval(interval) {
+    clearInterval(reportInterval);
+    if (!isNaN(interval) && interval >= 0) {
+        reportInterval = setInterval(reportLoop, interval);
+    }
+}
+
+/**
+ * Add a callback to add additional metrics before reporting.
+ *
+ * @param {function(metricsReporter)} hook callback to be invoked before reporting
+ */
+export function addReportHook(hook) {
+    reportHooks.push(hook);
+}
+
+/**
+ * Force metrics to be reported right now.
+ */
+export function flush() {
+    reportLoop();
+}
+
+function addDefaults() {
+    addProperty(MEM_RSS, process.memoryUsage().rss / 1048576);
+    addProperty(LOAD_1MIN, os.loadavg()[0]);
+    addProperty(TIMESTAMP, new Date());
+}
+
+function reportLoop() {
+    if (delegate !== null) {
+        try {
+            addDefaults();
+            reportHooks.forEach(hook => {
+                hook(delegate);
+            });
+            delegate.report();
+        } catch (error) {
+            logger.error(error.stack);
+        }
+    }
+}
diff --git a/src/partition/partitionchannelindex.js b/src/partition/partitionchannelindex.js
@@ -1,6 +1,6 @@
 import Promise from 'bluebird';
 import uuid from 'uuid';
-import { runLuaScript } from 'cytube-common/lib/redis/lualoader';
+import { runLuaScript } from '../redis/lualoader';
 import path from 'path';
 
 const LOGGER = require('@calzoneman/jsli')('partitionchannelindex');

diff --git a/src/partition/partitionmodule.js b/src/partition/partitionmodule.js
@@ -1,8 +1,8 @@
-import { loadFromToml } from 'cytube-common/lib/configuration/configloader';
+import { loadFromToml } from '../configuration/configloader';
 import { PartitionConfig } from './partitionconfig';
 import { PartitionDecider } from './partitiondecider';
 import { PartitionClusterClient } from '../io/cluster/partitionclusterclient';
-import RedisClientProvider from 'cytube-common/lib/redis/redisclientprovider';
+import RedisClientProvider from '../redis/redisclientprovider';
 import LegacyConfig from '../config';
 import path from 'path';
 import { AnnouncementRefresher } from './announcementrefresher';

diff --git a/src/redis/lualoader.js b/src/redis/lualoader.js
@@ -0,0 +1,44 @@
+import fs from 'fs';
+import logger from '../logger';
+
+const CACHE = {};
+const EVALSHA_CACHE = {};
+
+export function loadLuaScript(filename) {
+    if (CACHE.hasOwnProperty(filename)) {
+        return CACHE[filename];
+    }
+
+    CACHE[filename] = fs.readFileSync(filename).toString('utf8');
+    return CACHE[filename];
+}
+
+function loadAndExecuteScript(redisClient, filename, args) {
+    return redisClient.scriptAsync('load', loadLuaScript(filename))
+            .then(sha => {
+        EVALSHA_CACHE[filename] = sha;
+        logger.debug(`Cached ${filename} as ${sha}`);
+        return runEvalSha(redisClient, filename, args);
+    });
+}
+
+function runEvalSha(redisClient, filename, args) {
+    const evalInput = args.slice();
+    evalInput.unshift(EVALSHA_CACHE[filename])
+    return redisClient.evalshaAsync.apply(redisClient, evalInput);
+}
+
+export function runLuaScript(redisClient, filename, args) {
+    if (EVALSHA_CACHE.hasOwnProperty(filename)) {
+        return runEvalSha(redisClient, filename, args).catch(error => {
+            if (error.code === 'NOSCRIPT') {
+                logger.warn(`Got NOSCRIPT error for ${filename}, reloading script`);
+                return loadAndExecuteScript(redisClient, filename, args);
+            } else {
+                throw error;
+            }
+        });
+    } else {
+        return loadAndExecuteScript(redisClient, filename, args);
+    }
+}
diff --git a/src/redis/redisclientprovider.js b/src/redis/redisclientprovider.js
@@ -0,0 +1,49 @@
+import clone from 'clone';
+import redis from 'redis';
+import Promise from 'bluebird';
+Promise.promisifyAll(redis.RedisClient.prototype);
+Promise.promisifyAll(redis.Multi.prototype);
+
+/**
+ * Provider for RedisClients.
+ */
+class RedisClientProvider {
+    /**
+     * Create a new RedisClientProvider.
+     *
+     * @param {Object} redisConfig default configuration to use
+     * @see {@link https://www.npmjs.com/package/redis}
+     */
+    constructor(redisConfig) {
+        this.redisConfig = redisConfig;
+    }
+
+    /**
+     * Get a RedisClient.
+     *
+     * @param {Object} options optional override configuration for the RedisClient
+     * @return {RedisClient} redis client using the provided configuration
+     */
+    get(options = {}) {
+        const config = clone(this.redisConfig);
+        for (const key in options) {
+            config[key] = options[key];
+        }
+
+        const client = redis.createClient(config);
+        client.on('error', this._defaultErrorHandler);
+
+        return client;
+    }
+
+    /**
+     * Handle an <code>'error'</code> event from a provided client.
+     *
+     * @param {Error} err error from the client
+     * @private
+     */
+    _defaultErrorHandler(err) {
+    }
+}
+
+export default RedisClientProvider