From 1615925ae51cd3177d292f7bfd001951d7e4a6d9 Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Mon, 13 Jul 2015 15:31:52 -0300 Subject: [PATCH 1/4] better access log with morgan --- app/controllers/addresses.js | 7 +++++-- config/express.js | 3 ++- package.json | 2 +- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/app/controllers/addresses.js b/app/controllers/addresses.js index 7ac39fa3b..0c3ef7d03 100644 --- a/app/controllers/addresses.js +++ b/app/controllers/addresses.js @@ -139,9 +139,12 @@ exports.multitxs = function(req, res, next) { txIndex[tx.txid] = tx; }); - async.each(txs, function(tx, callback) { + async.eachLimit(txs, 5, function(tx, callback) { tDb.fromIdWithInfo(tx.txid, function(err, tx) { - if (err) console.log(err); + if (err) { + console.log(err); + return common.handleErrors(err, res); + } if (tx && tx.info) { txIndex[tx.txid].info = tx.info; } diff --git a/config/express.js b/config/express.js index 737958bac..4689b6c0a 100644 --- a/config/express.js +++ b/config/express.js @@ -28,7 +28,8 @@ module.exports = function(app, historicSync, peerSync) { app.enable('jsonp callback'); app.use(config.apiPrefix, setHistoric); app.use(config.apiPrefix, setPeer); - app.use(express.logger('dev')); + app.use(require('morgan')(':remote-addr :date[iso] ":method :url" :status :res[content-length] :response-time ":user-agent" ')); + app.use(express.json()); app.use(express.urlencoded()); app.use(express.methodOverride()); diff --git a/package.json b/package.json index 3b18e22a2..08cd8f307 100644 --- a/package.json +++ b/package.json @@ -51,7 +51,7 @@ "async": "*", "base58-native": "0.1.2", "bignum": "*", - "bitauth": "^0.1.1", + "morgan": "*", "bitcore": "git://github.com/bitpay/bitcore.git#aa41c70cff2583d810664c073a324376c39c8b36", "bufferput": "git://github.com/bitpay/node-bufferput.git", "buffertools": "*", From 71e3c1ba9ce1b4b29cc0900d551600ebda8c5e13 Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Mon, 13 Jul 2015 16:39:05 -0300 Subject: [PATCH 2/4] enforce limit at POST /txs requests --- app/controllers/addresses.js | 53 ++++++++++++++++++++++-------------- lib/TransactionDb.js | 9 +++++- 2 files changed, 40 insertions(+), 22 deletions(-) diff --git a/app/controllers/addresses.js b/app/controllers/addresses.js index 0c3ef7d03..16d5d7ca5 100644 --- a/app/controllers/addresses.js +++ b/app/controllers/addresses.js @@ -9,6 +9,9 @@ var Address = require('../models/Address'); var common = require('./common'); var async = require('async'); +var MAX_BATCH_SIZE = 100; +var RPC_CONCURRENCY = 5; + var tDb = require('../../lib/TransactionDb').default(); var checkSync = function(req, res) { @@ -50,7 +53,7 @@ var getAddrs = function(req, res, next) { } } catch (e) { common.handleErrors({ - message: 'Invalid address:' + e.message, + message: 'Invalid addrs param:' + e.message, code: 1 }, res, next); return null; @@ -101,7 +104,7 @@ exports.multiutxo = function(req, res, next) { var as = getAddrs(req, res, next); if (as) { var utxos = []; - async.each(as, function(a, callback) { + async.eachLimit(as, RPC_CONCURRENCY, function(a, callback) { a.update(function(err) { if (err) callback(err); utxos = utxos.concat(a.unspent); @@ -123,23 +126,31 @@ exports.multitxs = function(req, res, next) { function processTxs(txs, from, to, cb) { txs = _.uniq(_.flatten(txs), 'txid'); var nbTxs = txs.length; - var paginated = !_.isUndefined(from) || !_.isUndefined(to); - if (paginated) { - txs.sort(function(a, b) { - return (b.ts || b.ts) - (a.ts || a.ts); - }); - var start = Math.max(from || 0, 0); - var end = Math.min(to || txs.length, txs.length); - txs = txs.slice(start, end); + if ( _.isUndefined(from) && _.isUndefined(to)) { + from = 0; + to = MAX_BATCH_SIZE; } + if ( ! _.isUndefined(from) && _.isUndefined(to)) + to = from + MAX_BATCH_SIZE; + + if ( ! _.isUndefined(from) && ! _.isUndefined(to) && to - from > MAX_BATCH_SIZE) + to = from + MAX_BATCH_SIZE; + + txs.sort(function(a, b) { + return (b.ts || b.ts) - (a.ts || a.ts); + }); + var start = Math.max(from || 0, 0); + var end = Math.min(to || nbTxs, nbTxs); + txs = txs.slice(start, end); + var txIndex = {}; _.each(txs, function(tx) { txIndex[tx.txid] = tx; }); - async.eachLimit(txs, 5, function(tx, callback) { + async.eachLimit(txs, RPC_CONCURRENCY, function(tx, callback) { tDb.fromIdWithInfo(tx.txid, function(err, tx) { if (err) { console.log(err); @@ -154,14 +165,12 @@ exports.multitxs = function(req, res, next) { if (err) return cb(err); var transactions = _.pluck(txs, 'info'); - if (paginated) { - transactions = { - totalItems: nbTxs, - from: +from, - to: +to, - items: transactions, - }; - } + transactions = { + totalItems: nbTxs, + from: +from, + to: +to, + items: transactions, + }; return cb(null, transactions); }); }; @@ -172,17 +181,19 @@ exports.multitxs = function(req, res, next) { var as = getAddrs(req, res, next); if (as) { var txs = []; - async.eachLimit(as, 10, function(a, callback) { + async.eachLimit(as, RPC_CONCURRENCY, function(a, callback) { a.update(function(err) { if (err) callback(err); txs.push(a.transactions); callback(); }, { ignoreCache: req.param('noCache'), - includeTxInfo: true + includeTxInfo: true, + dontFillSpent: true, }); }, function(err) { // finished callback if (err) return common.handleErrors(err, res); + processTxs(txs, from, to, function(err, transactions) { if (err) return common.handleErrors(err, res); res.jsonp(transactions); diff --git a/lib/TransactionDb.js b/lib/TransactionDb.js index 35bada647..a3e9580bb 100644 --- a/lib/TransactionDb.js +++ b/lib/TransactionDb.js @@ -17,7 +17,7 @@ var ADDR_PREFIX = 'txa2-'; //txa---- // TODO: use bitcore networks module var genesisTXID = '4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b'; -var CONCURRENCY = 10; +var CONCURRENCY = 5; var DEFAULT_SAFE_CONFIRMATIONS = 6; var MAX_OPEN_FILES = 500; @@ -448,6 +448,9 @@ TransactionDb.prototype._parseAddrData = function(k, data, ignoreCache) { return item; }; + +// opts.dontFillSpent + TransactionDb.prototype.fromAddr = function(addr, opts, cb) { opts = opts || {}; var self = this; @@ -470,6 +473,10 @@ TransactionDb.prototype.fromAddr = function(addr, opts, cb) { }) .on('error', cb) .on('end', function() { + if (opts.dontFillSpent) { + return cb(null, ret) + } + async.eachLimit(ret.filter(function(x) { return !x.spentIsConfirmed; }), CONCURRENCY, function(o, e_c) { From ef296c5ebfa4a6294c484c60360059ec036813d5 Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Mon, 13 Jul 2015 17:14:02 -0300 Subject: [PATCH 3/4] rm start/end --- app/controllers/addresses.js | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/app/controllers/addresses.js b/app/controllers/addresses.js index 16d5d7ca5..9e03a933a 100644 --- a/app/controllers/addresses.js +++ b/app/controllers/addresses.js @@ -127,23 +127,24 @@ exports.multitxs = function(req, res, next) { txs = _.uniq(_.flatten(txs), 'txid'); var nbTxs = txs.length; - if ( _.isUndefined(from) && _.isUndefined(to)) { + if (_.isUndefined(from) && _.isUndefined(to)) { from = 0; to = MAX_BATCH_SIZE; } - - if ( ! _.isUndefined(from) && _.isUndefined(to)) + if (!_.isUndefined(from) && _.isUndefined(to)) to = from + MAX_BATCH_SIZE; - if ( ! _.isUndefined(from) && ! _.isUndefined(to) && to - from > MAX_BATCH_SIZE) + if (!_.isUndefined(from) && !_.isUndefined(to) && to - from > MAX_BATCH_SIZE) to = from + MAX_BATCH_SIZE; + if (from < 0) from = 0; + if (to > nbTxs) tx = nbTxs; + txs.sort(function(a, b) { return (b.ts || b.ts) - (a.ts || a.ts); }); - var start = Math.max(from || 0, 0); - var end = Math.min(to || nbTxs, nbTxs); - txs = txs.slice(start, end); + + txs = txs.slice(from, to); var txIndex = {}; _.each(txs, function(tx) { From 671ef9a5bb0b79e8fa7ffb10b6e0c8365a6f5f3b Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Mon, 13 Jul 2015 17:18:07 -0300 Subject: [PATCH 4/4] rm start/end 2 --- app/controllers/addresses.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/controllers/addresses.js b/app/controllers/addresses.js index 9e03a933a..b217edb37 100644 --- a/app/controllers/addresses.js +++ b/app/controllers/addresses.js @@ -138,7 +138,9 @@ exports.multitxs = function(req, res, next) { to = from + MAX_BATCH_SIZE; if (from < 0) from = 0; - if (to > nbTxs) tx = nbTxs; + if (to < 0) to = 0; + if (from > nbTxs) from = nbTxs; + if (to > nbTxs) to = nbTxs; txs.sort(function(a, b) { return (b.ts || b.ts) - (a.ts || a.ts);