TerribleCodeClub
/
ButlerBin
12
0
Fork 0
Code Pull Requests Releases Activity

Simplify for Butlersaurus
TerribleCodeClub/ButlerBin/pipeline/head This commit looks good Details

This commit is contained in:
Jack Hadrill 2020-06-08 20:56:17 +01:00
parent 9b0a5ff0a3
commit bea448306d
21 changed files with 33 additions and 645 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
Dockerfile Normal file
View File

@ -0,0 +1,5 @@
FROM node:alpine
WORKDIR /app
COPY . /app
RUN npm install
CMD [ "npm", "start" ]

0
Jenkinsfile vendored Normal file
View File

20
config.js
View File

@ -1,16 +1,10 @@
{
"host": "0.0.0.0",
"port": 7777,
"keyLength": 10,
"maxLength": 400000,
"staticMaxAge": 86400,
"recompressStaticAssets": true,
"logging": [
{
"level": "verbose",
@ -18,11 +12,6 @@
"colorize": true
}
],
"keyGenerator": {
"type": "phonetic"
},
"rateLimits": {
"categories": {
"normal": {
@ -31,16 +20,7 @@
}
}
},
"storage": {
"type": "memcached",
"host": "127.0.0.1",
"port": 11211,
"expire": 2592000
},
"documents": {
"about": "./about.md"
}
}

26
lib/document_handler.js
View File

@ -1,4 +1,4 @@
var winston = require('winston');
var Winston = require('winston');
var Busboy = require('busboy');
// For handling serving stored documents
@ -16,35 +16,35 @@ var DocumentHandler = function(options) {
DocumentHandler.defaultKeyLength = 10;
// Handle retrieving a document
DocumentHandler.prototype.handleGet = function(key, response, skipExpire) {
DocumentHandler.prototype.handleGet = function(key, response) {
this.store.get(key, function(ret) {
if (ret) {
winston.verbose('retrieved document', { key: key });
Winston.verbose('retrieved document', { key: key });
response.writeHead(200, { 'content-type': 'application/json' });
response.end(JSON.stringify({ data: ret, key: key }));
}
else {
winston.warn('document not found', { key: key });
Winston.warn('document not found', { key: key });
response.writeHead(404, { 'content-type': 'application/json' });
response.end(JSON.stringify({ message: 'Document not found.' }));
}
}, skipExpire);
});
};
// Handle retrieving the raw version of a document
DocumentHandler.prototype.handleRawGet = function(key, response, skipExpire) {
DocumentHandler.prototype.handleRawGet = function(key, response) {
this.store.get(key, function(ret) {
if (ret) {
winston.verbose('retrieved raw document', { key: key });
Winston.verbose('retrieved raw document', { key: key });
response.writeHead(200, { 'content-type': 'text/plain; charset=UTF-8' });
response.end(ret);
}
else {
winston.warn('raw document not found', { key: key });
Winston.warn('raw document not found', { key: key });
response.writeHead(404, { 'content-type': 'application/json' });
response.end(JSON.stringify({ message: 'Document not found.' }));
}
}, skipExpire);
});
};
// Handle adding a new Document
@ -58,7 +58,7 @@ DocumentHandler.prototype.handlePost = function (request, response) {
// Check length
if (_this.maxLength && buffer.length > _this.maxLength) {
cancelled = true;
winston.warn('document >maxLength', { maxLength: _this.maxLength });
Winston.warn('document >maxLength', { maxLength: _this.maxLength });
response.writeHead(400, { 'content-type': 'application/json' });
response.end(
JSON.stringify({ message: 'Document exceeds maximum length.' })
@ -69,12 +69,12 @@ DocumentHandler.prototype.handlePost = function (request, response) {
_this.chooseKey(function (key) {
_this.store.set(key, buffer, function (res) {
if (res) {
winston.verbose('added document', { key: key });
Winston.verbose('added document', { key: key });
response.writeHead(200, { 'content-type': 'application/json' });
response.end(JSON.stringify({ key: key }));
}
else {
winston.verbose('error adding document');
Winston.verbose('error adding document');
response.writeHead(500, { 'content-type': 'application/json' });
response.end(JSON.stringify({ message: 'Error adding document.' }));
}
@ -105,7 +105,7 @@ DocumentHandler.prototype.handlePost = function (request, response) {
onSuccess();
});
request.on('error', function (error) {
winston.error('connection error: ' + error.message);
Winston.error('connection error: ' + error.message);
response.writeHead(500, { 'content-type': 'application/json' });
response.end(JSON.stringify({ message: 'Connection error.' }));
cancelled = true;

16
lib/document_stores/file.js → lib/document_store.js
View File

@ -7,9 +7,9 @@ var winston = require('winston');
// options[type] = file
// options[path] - Where to store
var FileDocumentStore = function(options) {
this.basePath = options.path || './data';
this.expire = options.expire;
var FileDocumentStore = function() {
this.basePath = '/data';
this.expire = null;
};
// Generate md5 of a string
@ -21,7 +21,7 @@ FileDocumentStore.md5 = function(str) {
// Save data in a file, key as md5 - since we don't know what we could
// be passed here
FileDocumentStore.prototype.set = function(key, data, callback, skipExpire) {
FileDocumentStore.prototype.set = function(key, data, callback) {
try {
var _this = this;
fs.mkdir(this.basePath, '700', function() {
@ -32,9 +32,6 @@ FileDocumentStore.prototype.set = function(key, data, callback, skipExpire) {
}
else {
callback(true);
if (_this.expire && !skipExpire) {
winston.warn('file store cannot set expirations on keys');
}
}
});
});
@ -44,7 +41,7 @@ FileDocumentStore.prototype.set = function(key, data, callback, skipExpire) {
};
// Get data from a file from key
FileDocumentStore.prototype.get = function(key, callback, skipExpire) {
FileDocumentStore.prototype.get = function(key, callback) {
var _this = this;
var fn = this.basePath + '/' + FileDocumentStore.md5(key);
fs.readFile(fn, 'utf8', function(err, data) {
@ -53,9 +50,6 @@ FileDocumentStore.prototype.get = function(key, callback, skipExpire) {
}
else {
callback(data);
if (_this.expire && !skipExpire) {
winston.warn('file store cannot set expirations on keys');
}
}
});
};

56
lib/document_stores/amazon-s3.js
View File

@ -1,56 +0,0 @@
/*global require,module,process*/
var AWS = require('aws-sdk');
var winston = require('winston');
var AmazonS3DocumentStore = function(options) {
this.expire = options.expire;
this.bucket = options.bucket;
this.client = new AWS.S3({region: options.region});
};
AmazonS3DocumentStore.prototype.get = function(key, callback, skipExpire) {
var _this = this;
var req = {
Bucket: _this.bucket,
Key: key
};
_this.client.getObject(req, function(err, data) {
if(err) {
callback(false);
}
else {
callback(data.Body.toString('utf-8'));
if (_this.expire && !skipExpire) {
winston.warn('amazon s3 store cannot set expirations on keys');
}
}
});
}
AmazonS3DocumentStore.prototype.set = function(key, data, callback, skipExpire) {
var _this = this;
var req = {
Bucket: _this.bucket,
Key: key,
Body: data,
ContentType: 'text/plain'
};
_this.client.putObject(req, function(err, data) {
if (err) {
callback(false);
}
else {
callback(true);
if (_this.expire && !skipExpire) {
winston.warn('amazon s3 store cannot set expirations on keys');
}
}
});
}
module.exports = AmazonS3DocumentStore;

52
lib/document_stores/memcached.js
View File

@ -1,52 +0,0 @@
const memcached = require('memcached');
const winston = require('winston');
class MemcachedDocumentStore {
// Create a new store with options
constructor(options) {
this.expire = options.expire;
const host = options.host || '127.0.0.1';
const port = options.port || 11211;
const url = `${host}:${port}`;
this.connect(url);
}
// Create a connection
connect(url) {
this.client = new memcached(url);
winston.info(`connecting to memcached on ${url}`);
this.client.on('failure', function(error) {
winston.info('error connecting to memcached', {error});
});
}
// Save file in a key
set(key, data, callback, skipExpire) {
this.client.set(key, data, skipExpire ? 0 : this.expire, (error) => {
callback(!error);
});
}
// Get a file from a key
get(key, callback, skipExpire) {
this.client.get(key, (error, data) => {
callback(error ? false : data);
// Update the key so that the expiration is pushed forward
if (!skipExpire) {
this.set(key, data, (updateSucceeded) => {
if (!updateSucceeded) {
winston.error('failed to update expiration on GET', {key});
}
}, skipExpire);
}
});
}
}
module.exports = MemcachedDocumentStore;

79
lib/document_stores/postgres.js
View File

@ -1,79 +0,0 @@
/*global require,module,process*/
var postgres = require('pg');
var winston = require('winston');
// create table entries (id serial primary key, key varchar(255) not null, value text not null, expiration int, unique(key));
// A postgres document store
var PostgresDocumentStore = function (options) {
this.expireJS = options.expire;
this.connectionUrl = process.env.DATABASE_URL || options.connectionUrl;
};
PostgresDocumentStore.prototype = {
// Set a given key
set: function (key, data, callback, skipExpire) {
var now = Math.floor(new Date().getTime() / 1000);
var that = this;
this.safeConnect(function (err, client, done) {
if (err) { return callback(false); }
client.query('INSERT INTO entries (key, value, expiration) VALUES ($1, $2, $3)', [
key,
data,
that.expireJS && !skipExpire ? that.expireJS + now : null
], function (err) {
if (err) {
winston.error('error persisting value to postgres', { error: err });
return callback(false);
}
callback(true);
done();
});
});
},
// Get a given key's data
get: function (key, callback, skipExpire) {
var now = Math.floor(new Date().getTime() / 1000);
var that = this;
this.safeConnect(function (err, client, done) {
if (err) { return callback(false); }
client.query('SELECT id,value,expiration from entries where KEY = $1 and (expiration IS NULL or expiration > $2)', [key, now], function (err, result) {
if (err) {
winston.error('error retrieving value from postgres', { error: err });
return callback(false);
}
callback(result.rows.length ? result.rows[0].value : false);
if (result.rows.length && that.expireJS && !skipExpire) {
client.query('UPDATE entries SET expiration = $1 WHERE ID = $2', [
that.expireJS + now,
result.rows[0].id
], function (err) {
if (!err) {
done();
}
});
} else {
done();
}
});
});
},
// A connection wrapper
safeConnect: function (callback) {
postgres.connect(this.connectionUrl, function (err, client, done) {
if (err) {
winston.error('error connecting to postgres', { error: err });
callback(err);
} else {
callback(undefined, client, done);
}
});
}
};
module.exports = PostgresDocumentStore;

89
lib/document_stores/redis.js
View File

@ -1,89 +0,0 @@
var redis = require('redis');
var winston = require('winston');
// For storing in redis
// options[type] = redis
// options[host] - The host to connect to (default localhost)
// options[port] - The port to connect to (default 5379)
// options[db] - The db to use (default 0)
// options[expire] - The time to live for each key set (default never)
var RedisDocumentStore = function(options, client) {
this.expire = options.expire;
if (client) {
winston.info('using predefined redis client');
RedisDocumentStore.client = client;
} else if (!RedisDocumentStore.client) {
winston.info('configuring redis');
RedisDocumentStore.connect(options);
}
};
// Create a connection according to config
RedisDocumentStore.connect = function(options) {
var host = options.host || '127.0.0.1';
var port = options.port || 6379;
var index = options.db || 0;
RedisDocumentStore.client = redis.createClient(port, host);
// authenticate if password is provided
if (options.password) {
RedisDocumentStore.client.auth(options.password);
}
RedisDocumentStore.client.on('error', function(err) {
winston.error('redis disconnected', err);
});
RedisDocumentStore.client.select(index, function(err) {
if (err) {
winston.error(
'error connecting to redis index ' + index,
{ error: err }
);
process.exit(1);
}
else {
winston.info('connected to redis on ' + host + ':' + port + '/' + index);
}
});
};
// Save file in a key
RedisDocumentStore.prototype.set = function(key, data, callback, skipExpire) {
var _this = this;
RedisDocumentStore.client.set(key, data, function(err) {
if (err) {
callback(false);
}
else {
if (!skipExpire) {
_this.setExpiration(key);
}
callback(true);
}
});
};
// Expire a key in expire time if set
RedisDocumentStore.prototype.setExpiration = function(key) {
if (this.expire) {
RedisDocumentStore.client.expire(key, this.expire, function(err) {
if (err) {
winston.error('failed to set expiry on key: ' + key);
}
});
}
};
// Get a file from a key
RedisDocumentStore.prototype.get = function(key, callback, skipExpire) {
var _this = this;
RedisDocumentStore.client.get(key, function(err, reply) {
if (!err && !skipExpire) {
_this.setExpiration(key);
}
callback(err ? false : reply);
});
};
module.exports = RedisDocumentStore;

46
lib/document_stores/rethinkdb.js
View File

@ -1,46 +0,0 @@
const crypto = require('crypto');
const rethink = require('rethinkdbdash');
const winston = require('winston');
const md5 = (str) => {
const md5sum = crypto.createHash('md5');
md5sum.update(str);
return md5sum.digest('hex');
};
class RethinkDBStore {
constructor(options) {
this.client = rethink({
silent: true,
host: options.host || '127.0.0.1',
port: options.port || 28015,
db: options.db || 'haste',
user: options.user || 'admin',
password: options.password || ''
});
}
set(key, data, callback) {
this.client.table('uploads').insert({ id: md5(key), data: data }).run((error) => {
if (error) {
callback(false);
winston.error('failed to insert to table', error);
return;
}
callback(true);
});
}
get(key, callback) {
this.client.table('uploads').get(md5(key)).run((error, result) => {
if (error || !result) {
callback(false);
if (error) winston.error('failed to insert to table', error);
return;
}
callback(result.data);
});
}
}
module.exports = RethinkDBStore;

6
lib/key_generators/random.js → lib/key_generator.js
View File

@ -1,8 +1,8 @@
module.exports = class RandomKeyGenerator {
module.exports = class KeyGenerator {
// Initialize a new generator with the given keySpace
constructor(options = {}) {
this.keyspace = options.keyspace || 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789';
constructor() {
this.keyspace = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789';
}
// Generate a key of the given length

32
lib/key_generators/dictionary.js
View File

@ -1,32 +0,0 @@
const fs = require('fs');
module.exports = class DictionaryGenerator {
constructor(options, readyCallback) {
// Check options format
if (!options) throw Error('No options passed to generator');
if (!options.path) throw Error('No dictionary path specified in options');
// Load dictionary
fs.readFile(options.path, 'utf8', (err, data) => {
if (err) throw err;
this.dictionary = data.split(/[\n\r]+/);
if (readyCallback) readyCallback();
});
}
// Generates a dictionary-based key, of keyLength words
createKey(keyLength) {
let text = '';
for (let i = 0; i < keyLength; i++) {
const index = Math.floor(Math.random() * this.dictionary.length);
text += this.dictionary[index];
}
return text;
}
};

27
lib/key_generators/phonetic.js
View File

@ -1,27 +0,0 @@
// Draws inspiration from pwgen and http://tools.arantius.com/password
const randOf = (collection) => {
return () => {
return collection[Math.floor(Math.random() * collection.length)];
};
};
// Helper methods to get an random vowel or consonant
const randVowel = randOf('aeiou');
const randConsonant = randOf('bcdfghjklmnpqrstvwxyz');
module.exports = class PhoneticKeyGenerator {
// Generate a phonetic key of alternating consonant & vowel
createKey(keyLength) {
let text = '';
const start = Math.round(Math.random());
for (let i = 0; i < keyLength; i++) {
text += (i % 2 == start) ? randConsonant() : randVowel();
}
return text;
}
};

5
package-lock.json generated
View File

@ -112,11 +112,6 @@
"utils-merge": "1.0.0"
}
},
"connect-ratelimit": {
"version": "0.0.7",
"resolved": "https://registry.npmjs.org/connect-ratelimit/-/connect-ratelimit-0.0.7.tgz",
"integrity": "sha1-5uCclQZJ6ElJnKsYcKQVoH9zFWg="
},
"connect-route": {
"version": "0.1.5",
"resolved": "https://registry.npmjs.org/connect-route/-/connect-route-0.1.5.tgz",

1
package.json
View File

@ -14,7 +14,6 @@
},
"main": "haste",
"dependencies": {
"connect-ratelimit": "0.0.7",
"connect-route": "0.1.5",
"connect": "3.4.1",
"st": "1.1.0",

59
server.js
View File

@ -6,7 +6,6 @@ var winston = require('winston');
var connect = require('connect');
var route = require('connect-route');
var connect_st = require('st');
var connect_rate_limit = require('connect-ratelimit');
var DocumentHandler = require('./lib/document_handler');
@ -32,26 +31,8 @@ if (config.logging) {
}
}
// build the store from the config on-demand - so that we don't load it
// for statics
if (!config.storage) {
config.storage = { type: 'file' };
}
if (!config.storage.type) {
config.storage.type = 'file';
}
var Store, preferredStore;
if (process.env.REDISTOGO_URL && config.storage.type === 'redis') {
var redisClient = require('redis-url').connect(process.env.REDISTOGO_URL);
Store = require('./lib/document_stores/redis');
preferredStore = new Store(config.storage, redisClient);
}
else {
Store = require('./lib/document_stores/' + config.storage.type);
preferredStore = new Store(config.storage);
}
var Store = require('./lib/document_store');
store = new Store();
// Compress the static javascript assets
if (config.recompressStaticAssets) {
@ -68,31 +49,13 @@ if (config.recompressStaticAssets) {
}
}
// Send the static documents into the preferred store, skipping expirations
var path, data;
for (var name in config.documents) {
path = config.documents[name];
data = fs.readFileSync(path, 'utf8');
winston.info('loading static document', { name: name, path: path });
if (data) {
preferredStore.set(name, data, function(cb) {
winston.debug('loaded static document', { success: cb });
}, true);
}
else {
winston.warn('failed to load static document', { name: name, path: path });
}
}
// Pick up a key generator
var pwOptions = config.keyGenerator || {};
pwOptions.type = pwOptions.type || 'random';
var gen = require('./lib/key_generators/' + pwOptions.type);
var keyGenerator = new gen(pwOptions);
var KeyGenerator = require('./lib/key_generator');
var keyGenerator = new KeyGenerator();
// Configure the document handler
var documentHandler = new DocumentHandler({
store: preferredStore,
store: store,
maxLength: config.maxLength,
keyLength: config.keyLength,
keyGenerator: keyGenerator
@ -100,19 +63,12 @@ var documentHandler = new DocumentHandler({
var app = connect();
// Rate limit all requests
if (config.rateLimits) {
config.rateLimits.end = true;
app.use(connect_rate_limit(config.rateLimits));
}
// first look at API calls
app.use(route(function(router) {
// get raw documents - support getting with extension
router.get('/raw/:id', function(request, response) {
var key = request.params.id.split('.')[0];
var skipExpire = !!config.documents[key];
return documentHandler.handleRawGet(key, response, skipExpire);
return documentHandler.handleRawGet(key, response);
});
// add documents
router.post('/documents', function(request, response) {
@ -121,8 +77,7 @@ app.use(route(function(router) {
// get documents
router.get('/documents/:id', function(request, response) {
var key = request.params.id.split('.')[0];
var skipExpire = !!config.documents[key];
return documentHandler.handleGet(key, response, skipExpire);
return documentHandler.handleGet(key, response);
});
}));

26
test/document_handler_spec.js
View File

@ -1,26 +0,0 @@
/* global describe, it */
var assert = require('assert');
var DocumentHandler = require('../lib/document_handler');
var Generator = require('../lib/key_generators/random');
describe('document_handler', function() {
describe('randomKey', function() {
it('should choose a key of the proper length', function() {
var gen = new Generator();
var dh = new DocumentHandler({ keyLength: 6, keyGenerator: gen });
assert.equal(6, dh.acceptableKey().length);
});
it('should choose a default key length', function() {
var gen = new Generator();
var dh = new DocumentHandler({ keyGenerator: gen });
assert.equal(dh.keyLength, DocumentHandler.defaultKeyLength);
});
});
});

33
test/key_generators/dictionary_spec.js
View File

@ -1,33 +0,0 @@
/* global describe, it */
const assert = require('assert');
const fs = require('fs');
const Generator = require('../../lib/key_generators/dictionary');
describe('RandomKeyGenerator', function() {
describe('randomKey', function() {
it('should throw an error if given no options', () => {
assert.throws(() => {
new Generator();
}, Error);
});
it('should throw an error if given no path', () => {
assert.throws(() => {
new Generator({});
}, Error);
});
it('should return a key of the proper number of words from the given dictionary', () => {
const path = '/tmp/haste-server-test-dictionary';
const words = ['cat'];
fs.writeFileSync(path, words.join('\n'));
const gen = new Generator({path}, () => {
assert.equal('catcatcat', gen.createKey(3));
});
});
});
});

27
test/key_generators/phonetic_spec.js
View File

@ -1,27 +0,0 @@
/* global describe, it */
const assert = require('assert');
const Generator = require('../../lib/key_generators/phonetic');
const vowels = 'aeiou';
const consonants = 'bcdfghjklmnpqrstvwxyz';
describe('RandomKeyGenerator', () => {
describe('randomKey', () => {
it('should return a key of the proper length', () => {
const gen = new Generator();
assert.equal(6, gen.createKey(6).length);
});
it('should alternate consonants and vowels', () => {
const gen = new Generator();
const key = gen.createKey(3);
assert.ok(consonants.includes(key[0]));
assert.ok(consonants.includes(key[2]));
assert.ok(vowels.includes(key[1]));
});
});
});

19
test/key_generators/random_spec.js
View File

@ -1,19 +0,0 @@
/* global describe, it */
const assert = require('assert');
const Generator = require('../../lib/key_generators/random');
describe('RandomKeyGenerator', () => {
describe('randomKey', () => {
it('should return a key of the proper length', () => {
const gen = new Generator();
assert.equal(6, gen.createKey(6).length);
});
it('should use a key from the given keyset if given', () => {
const gen = new Generator({keyspace: 'A'});
assert.equal('AAAAAA', gen.createKey(6));
});
});
});

54
test/redis_document_store_spec.js
View File

@ -1,54 +0,0 @@
/* global it, describe, afterEach */
var assert = require('assert');
var winston = require('winston');
winston.remove(winston.transports.Console);
var RedisDocumentStore = require('../lib/document_stores/redis');
describe('redis_document_store', function() {
/* reconnect to redis on each test */
afterEach(function() {
if (RedisDocumentStore.client) {
RedisDocumentStore.client.quit();
RedisDocumentStore.client = false;
}
});
describe('set', function() {
it('should be able to set a key and have an expiration set', function(done) {
var store = new RedisDocumentStore({ expire: 10 });
store.set('hello1', 'world', function() {
RedisDocumentStore.client.ttl('hello1', function(err, res) {
assert.ok(res > 1);
done();
});
});
});
it('should not set an expiration when told not to', function(done) {
var store = new RedisDocumentStore({ expire: 10 });
store.set('hello2', 'world', function() {
RedisDocumentStore.client.ttl('hello2', function(err, res) {
assert.equal(-1, res);
done();
});
}, true);
});
it('should not set an expiration when expiration is off', function(done) {
var store = new RedisDocumentStore({ expire: false });
store.set('hello3', 'world', function() {
RedisDocumentStore.client.ttl('hello3', function(err, res) {
assert.equal(-1, res);
done();
});
});
});
});
});