diff --git a/.gitignore b/.gitignore
index 5bfc897..2d1a24e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -26,6 +26,7 @@ build/Release
# https://www.npmjs.org/doc/misc/npm-faq.html#should-i-check-my-node_modules-folder-into-git
node_modules
-config.env
+*.env
dump.rdb
npm-debug.log
+data/
diff --git a/README.md b/README.md
index 07cbdf7..8888cee 100644
--- a/README.md
+++ b/README.md
@@ -1,78 +1,168 @@
# hits
-What if there was a *simple+easy* way to see how many people have viewed your GitHub Repository?
+A _simple & easy_ way to see how many people have _viewed_ your GitHub Repository.
-[](https://travis-ci.org/dwyl/hits)
-[](https://github.com/nelsonic/hits)
-[](https://codeclimate.com/github/dwyl/hits)
-[](http://codecov.io/github/dwyl/hits?branch=master)
-[](https://david-dm.org/dwyl/hits)
-[](https://david-dm.org/dwyl/hits#info=devDependencies)
+[](https://travis-ci.org/dwyl/hits)
+[](https://github.com/dwyl/hits)
+[](http://codecov.io/github/dwyl/hits?branch=master)
+[](https://david-dm.org/dwyl/hits)
+[](https://david-dm.org/dwyl/hits#info=devDependencies)
## Why?
-We have a few repos on GitHub ... but sadly, we have no idea how many people
-are looking at the repos unless they star/watch them; GitHub does not share
-any stats with people using their site.
+We have a _few_ projects on GitHub ...
+_Sadly_, we ~~have~~ _had_ no idea how many people
+are _reading/using_ the projects because GitHub only shares "[traffic](https://github.com/blog/1672-introducing-github-traffic-analytics)" stats
+for the [_past 14 days_](https://github.com/dwyl/hits/issues/49) and **not** in "***real time***".
+(_unless people star/watch the repo_) Also, _manually_ checking who has viewed a
+project is _exceptionally_ tedious when you have more than a handful of projects.
-We would like to *know* the popularity of each of our repos
-to know where we need to be investing our time.
+We want to *know* the popularity of _each_ of our repos
+to know what people are finding _useful_ and help us
+decide where we need to be investing our time.
## What?
A simple way to add (*very basic*) analytics to your GitHub repos.
-There are already *many* "Badges" available which people put in their repos: https://github.com/dwyl/repo-badges
+There are already *many* "badges" that people use in their repos.
+See: [github.com/dwyl/**repo-badges**](https://github.com/dwyl/repo-badges)
But we haven't seen one that gives a "***hit counter***"
-of the number of times a page has been viewed ...
+of the number of times a GitHub page has been viewed ...
+So, in today's mini project we're going to _create_ a _basic **Web Counter**_.
-## How?
-
-Place a badge (*image*) in your repo `README.md` so others can
-can see how popular the page is and you can track it.
+https://en.wikipedia.org/wiki/Web_counter
-### Implementation
+### What Data to Capture/Store?
-What is the ***minimum possible*** amount of data we can store?
+The _first_ question we asked ourselves was:
+What is the ***minimum possible*** amount of (_useful/unique_)
+**info** we can store ***per visit*** (_to one of our projects_)?
-+ **date+time** the person visited the site.
+1. **date + time** (_timestamp_) ***when***
+the person visited the site/page.
https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Date/now
-+ **user-agent** the browser or crawler visiting the page
+
+2. **url** being visited. i.e. which project was viewed.
+
+3. **user-agent** the browser/device (_or "crawler"_) visiting the site/page
https://en.wikipedia.org/wiki/User_agent
-+ **referer** url of the page where the image is requested from?
-https://en.wikipedia.org/wiki/HTTP_referer
-Log entries are stored as a `String` which can be parsed and re-formatted into
-any other format:
+4. IP Address of the client. (_for checking uniqueness_)
+
+5. **Language** of the person's web browser.
+_Note: While not "essential", we added **Browser Language**
+as the **5th** piece of data (when it is set/sent by the browser/device)
+because it's **insightful** to know what language people are using
+so that we can determine if we should be **translating**/"**localising**"
+our content._
+
+### "Common Log Format" (CLF) ?
+
+We initially _considered_ using the "Common Log Format" (CLF)
+because it's well-known/understood.
+see: https://en.wikipedia.org/wiki/Common_Log_Format
+
+An example log entry:
+```
+127.0.0.1 user-identifier frank [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326
+```
+
+Real example:
+```
+84.91.136.21 Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) 007 [05/Aug/2017:16:50:51 -0000] "GET github.com/dwyl/phase-two HTTP/1.0" 200 42247
+```
+
+The data makes sense when viewed as a table:
+
+| IP Address of Client | User Identifier | User ID | Date+Imte of Request | Request "Verb" and URL of Request | HTTP Status Code | Size of Response |
+| -------------|:-----------|:--|:------------:|:--------:|:--|--|--|
+| 84.91.136.21 | Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) | 007 | [05/Aug/2017:16:50:51 -0000] | "GET github.com/dwyl/phase-two HTTP/1.0" | 200 | 42247 |
+
+On further reflection, we think the "Common Log Format" is _inneficient_
+as it contains a lot of _duplicate_ and some _useless_ data.
+
+We can do better.
+
+### Alternative Log Format ("ALF")
+
+From the CLF we can remove:
+
++ **IP Address**, **User Identifier** and **User ID** can be condensed into a single hash (_see below_).
++ "**GET**"" - the word is implied by the service we are running (_we only accept GET requests_)
++ **Response size** is _irrelevant_ and will be the same for most requests.
+
+| Timestamp | URL | User Agent | IP Address | Language | Hit Count |
+| ------------- |:------------|:------------|:------------:|:--------:|
+| 1436570536950 | github.com/dwyl/the-book | Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) | 84.91.136.21 | EN-GB | 42 |
+
+
+In the log entry (_example_) described above the first 3 bits of data will
+identify the "user" requesting the page/resource, so rather than duplicating the data in an inefficient string, we can _hash_ it!
+
+Any repeating user-identifying data should be concactenated
+
+Log entries are stored as a (_"pipe" delimited_) `String`
+which can be parsed and re-formatted into any other format:
+
```sh
-1436570536950 x7uapo9 84.91.136.21
+1436570536950|github.com/dwyl/phase-two|Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5)|88.88.88.88|EN-US|42
```
-| Timestamp | User Agent | IP Address |
-| ------------- |:------------|:------------:|
-| 1436570536950 | x7uapo9 | 84.91.136.21 |
-We then have a user-agent hash where we can lookup the by id:
+### Reducing Storage (_Costs_)
+
+If a person views _multiple_ pages, _three_ pieces of data are duplicated:
+User Agent, IP Address and Language.
+Rather than storing this data multiple times, we _hash_ the data
+and store the hash as a lookup.
+
+#### Hash Long Repeating (Identical) Data
+
+If we run the following `Browser|IP|Language` `String`:
+```sh
+'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5)|84.91.136.21|EN-US'
+```
+through a **SHA** hash function we get: `8HKg3NB5Cf` (_always_)1.
+
+_Sample_ code:
```js
-{
- "x7uapo9":"Mozilla/5.0 (iPad; U; CPU OS 3_2_1 like Mac OS X; en-us) AppleWebKit/531.21.10",
- "N03v1lz":"Googlebot/2.1 (+http://www.google.com/bot.html)"
-}
+var hash = require('./lib/hash.js');
+var user_agent_string = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5)|88.88.88.88|EN-US';
+var agent_hash = hash(user_agent_string, 10); // 8HKg3NB5Cf
+```
+
+1Note: SHA hash is _always_ 40 characters,
+but we _truncate_ it because 10 alphanumeric characters (_selected from a set of 26 letters + 10 digits_)
+means there are 3610 = [3,656,158,440,062,976](http://www.wolframalpha.com/input/?i=36%5E10)
+(_three and a half [**Quadrillion**](http://www.wolframalpha.com/input/?i=3,656,158,440,062,976+in+english)_)
+possible strings which we consider "_enough_" entropy.
+(_if you disagree, tell us why in an
+ [issue](https://github.com/dwyl/hits/issues)_!)
+
+#### Hit Data With Hash
+
```
+1436570536950|github.com/dwyl/the-book|8HKg3NB5Cf|42
+```
+
+
+## How?
+
+Place a badge (*image*) in your repo `README.md` so others can
+can see how popular the page is and you can track it.
-### Fetch SVG from shields.io and serve it just-in-time
-Given that shields.io has a badge creation service,
-and it has acceptable latency, we are proxying the their service.
-## Run it!
+## _Run_ it Your_self_!
Download (clone) the code to your local machine:
+
```sh
git clone https://github.com/dwyl/hits.git && cd hits
```
-> Note: you will need to have Redis running on your localhost,
-> if you are new to Redis see: https://github.com/dwyl/learn-redis
+
+> Note: you will need to have Node.js running on your localhost.
Install dependencies:
```sh
@@ -85,6 +175,20 @@ npm run dev
Visit: http://localhost:8000/any/url/count.svg
+# Data Storage
+
+Recording the "hit" data is _essential_
+for this app to _work_ and be _useful_.
+
+We have built it to work with _two_ "data stores":
+Filesystem and Redis
+> _**Note**: you only need **one** storage option to be available_.
+
+## Filesystem
+
+
+
+
## Research
### User Agents
@@ -108,3 +212,12 @@ http://www.monitorware.com/en/logsamples/apache.php
### Node.js http module headers
https://nodejs.org/api/http.html#http_message_rawheaders
+
+## Running the Test Suite locally
+
+The test suite includes tests for 3 databases
+therefore running the tests on your `localhost`
+requires all 3 to be running.
+
+Deploying and _using_ the app only requires _one_
+of the databases to be available.
diff --git a/lib/client.js b/lib/client.js
index c77deb0..964588d 100644
--- a/lib/client.js
+++ b/lib/client.js
@@ -1,9 +1,52 @@
-// connect to websocket server
-$( document ).ready(function() {
- console.log('Ready!', window.location.host);
+var root = document.getElementById("hits");
+console.log('Ready!', window.location.host);
+
+setTimeout(function(){
var socket = io(window.location.host);
socket.on('news', function (data) {
console.log(data);
- socket.emit('my other event', { my: 'data' });
+ socket.emit('hello', { msg: 'Hi!' });
});
-});
+
+ socket.on('hit', function (data) {
+ var previous = root.childNodes[0];
+ root.insertBefore(div(Date.now(), data.hit), previous);
+ });
+
+ // borrowed from: https://git.io/v536m
+ function div(divid, text) {
+ var div = document.createElement('div');
+ div.id = divid;
+ div.className = divid;
+ if(text !== undefined) { // if text is passed in render it in a "Text Node"
+ var txt = document.createTextNode(text);
+ div.appendChild(txt);
+ }
+ return div;
+ }
+ document.getElementById("how").classList.remove('dn'); // show form if JS available (progressive enhancement)
+ document.getElementById("nojs").classList.add('dn'); // show form if JS available (progressive enhancement)
+ display_badge_markdown(); // render initial markdown template
+}, 500);
+
+// Markdown Template
+var mt = '[](http://hits.dwyl.io/{user}/{repo})';
+
+function generate_markdown () {
+ var user = document.getElementById("username").value || '{username}';
+ var repo = document.getElementById("repo").value || '{project}';
+ // console.log('user: ', user, 'repo: ', repo);
+ return mt.replace(/{user}/g, user).replace(/{repo}/g, repo);
+}
+
+function display_badge_markdown() {
+ var md = generate_markdown()
+ var pre = document.getElementById("badge").innerHTML = md;
+}
+
+var get = document.getElementsByTagName('input');
+ for (i = 0; i < get.length; i++) {
+ get[i].addEventListener('keyup', display_badge_markdown, false);
+ get[i].addEventListener('keyup', display_badge_markdown, false);
+
+ }
diff --git a/lib/climate.svg b/lib/climate.svg
deleted file mode 100644
index 8e5f2fb..0000000
--- a/lib/climate.svg
+++ /dev/null
@@ -1,21 +0,0 @@
-
-
diff --git a/lib/db_filesystem.js b/lib/db_filesystem.js
new file mode 100644
index 0000000..b9b759d
--- /dev/null
+++ b/lib/db_filesystem.js
@@ -0,0 +1,68 @@
+var fs = require('fs');
+var rl = require('readline');
+var path = require('path');
+var hash = require('./hash.js');
+var error_log = require('../lib/error_logger.js');
+var EOL = require('os').EOL; // https://stackoverflow.com/a/14063413/1148249
+var LOG_DIR = path.resolve(__dirname, '../logs');
+
+if (!fs.existsSync(LOG_DIR)) { // ignored if already exists
+ fs.mkdirSync(LOG_DIR);
+}
+
+var LOG_FILE = path.resolve(LOG_DIR, 'access.log');
+fs.appendFileSync(LOG_FILE, '- - - New Logs:');
+
+var AGENTS_DIR = path.join(LOG_DIR, '/agents');
+if (!fs.existsSync(AGENTS_DIR)) { // ignored if already exists
+ fs.mkdirSync(AGENTS_DIR);
+}
+
+/**
+ * add - adds an entry into the List for a given url
+ * @param {Object} hit - the hit we just received
+ * @param {Function} callback - call this once redis responds
+ */
+module.exports = function file_save_hit (hit, callback) {
+ var h = hit.split('|'); // See README.md#How secton for sample data
+ var url = h[1];
+ var LOG_FILE = path.join(LOG_DIR,
+ url.split('/').join('_').replace(':', '') + '.log');
+ var count = 1; // hit count starts at 1
+ // save unique hash of browser data to avoid duplication
+ var unique_browser_string = [ h[2], h[3], h[4] ].join('|');
+ var hashed_agent = hash(unique_browser_string, 10);
+ // save unique data in file to reduce duplication in logs
+ var agent_path = path.join(path.resolve(AGENTS_DIR, hashed_agent))
+ fs.writeFile(agent_path, unique_browser_string, function (err, data) {
+ error_log(err, 'unable to save agent data: ' + agent_path);
+ lineReader = rl.createInterface({
+ input: require('fs').createReadStream(LOG_FILE)
+ .on('error', function (err) {
+ console.log('Error!', err);
+ error_log(err, 'unable to save agent data: ' + LOG_FILE);
+ var entry = [h[0], h[1], hashed_agent, count].join('|') + EOL;
+ fs.appendFile(LOG_FILE, entry, function (err) {
+ error_log(err, 'unable to APPEND to file:' + LOG_FILE);
+ callback(err, count);
+ });
+ })
+ });
+ var lines = [];
+ lineReader.on('line', function (line) {
+ lines.push(line);
+ });
+
+ lineReader.on('close', function() {
+ var last_line = lines[lines.length - 1];
+ var parts = last_line.split('|'); // parse and incremnt count:
+ count = parseInt(parts[parts.length - 1], 10) + 1;
+
+ var entry = [h[0], h[1], hashed_agent, count].join('|') + EOL;
+ fs.appendFile(LOG_FILE, entry, function (err) {
+ error_log(err, 'unable to APPEND to file:' + LOG_FILE);
+ callback(err, count);
+ }); // if slow, optimise: https://stackoverflow.com/questions/12453057
+ });
+ });
+}
diff --git a/lib/db_redis.js b/lib/db_redis.js
new file mode 100644
index 0000000..57d596d
--- /dev/null
+++ b/lib/db_redis.js
@@ -0,0 +1,22 @@
+var redisClient = require('redis-connection')();
+var hash = require('./hash.js');
+/**
+ * add - adds an entry into the List for a given url
+ * @param {Object} hit - the hit we just received
+ * @param {Function} callback - call this once redis responds
+ */
+module.exports = function redis_save_hit (hit, callback) {
+ var h = hit.split('|'); // See README.md#How secton for sample data
+ var url = h[1];
+
+ // save unique hash of browser data to avoid duplication
+ var unique_browser_string = [ h[2], h[3], h[4] ].join('|');
+ var hashed_agent = hash(unique_browser_string, 10);
+ redisClient.hset('agents', hashed_agent, unique_browser_string);
+
+ // save hit data with hashed browser data:
+ var entry = h[0] + '|' + hashed_agent;
+ redisClient.rpush(url, entry, function (err, data) {
+ callback(err, data)
+ });
+}
diff --git a/lib/error_logger.js b/lib/error_logger.js
new file mode 100644
index 0000000..b590812
--- /dev/null
+++ b/lib/error_logger.js
@@ -0,0 +1,30 @@
+var fs = require('fs');
+var path = require('path');
+var EOL = require('os').EOL; // https://stackoverflow.com/a/14063413/1148249
+var LOG_DIR = path.resolve(__dirname, '../logs');
+
+/**
+ * add - adds an entry into the List for a given url
+ * @param {Object|null} error - the error we just received
+ * @param {String} [message=none] - the user-defined error message
+ & @returns {Boolean} - returns false if not error and true if error logged
+ */
+module.exports = function error_logger (error, message) {
+ if (error) {
+ var _error;
+ var filepath = path.resolve(LOG_DIR, 'error.log');
+ if (typeof error !== 'object') {
+ _error = { error: error };
+ } else {
+ _error = error;
+ }
+ _error._message = message || 'none';
+ console.log('- - - - - - - - - - - - Error: - - - - - - - - - - - - ');
+ console.error(_error);
+ console.log('- - - - - - - - - - - - - - - - - - - - - - - - - - - - ');
+ // we expect errors to be infrequent so Sync is "OK" here!
+ fs.writeFileSync(filepath, JSON.stringify(_error) + EOL);
+ return true;
+ }
+ return false;
+}
diff --git a/lib/extract_request_data.js b/lib/extract_request_data.js
new file mode 100644
index 0000000..da7a29e
--- /dev/null
+++ b/lib/extract_request_data.js
@@ -0,0 +1,29 @@
+/**
+ * This file/module's only job is to extract the request data from http headers
+ * @param {Object} request - the standard nodejs http request object.
+ * @returns {string} hit - see readme for format.
+ */
+module.exports = function extract (request) {
+ var h = request.headers || {}; // shortcut to headers reduces typing
+ var lang; // the browser language
+
+ // get the user's IP addres from headers or connection object:
+ var ip = h['x-forwarded-for'] ||
+ request.connection && request.connection.remoteAddress;
+
+ // get url the client requested:
+ var url = request.url.replace('.svg', '')
+ .replace('.png', '')
+ .replace('https://github.com/', ''); // strip to save storage
+
+ if(h['accept-language']) { // Language for: github.com/dwyl/hits/issues/43
+ if (h['accept-language'].indexOf(',') > -1) { // e.g: en-GB,en;q=0.5
+ lang = h['accept-language'].split(',')[0].toUpperCase(); // e.g: EN-GB
+ } else {
+ lang = h['accept-language'].toUpperCase();
+ }
+ }
+
+ return [ Math.floor(Date.now()/1000),
+ url, h['user-agent'], ip, lang ].join('|');
+}
diff --git a/lib/format_hit.js b/lib/format_hit.js
new file mode 100644
index 0000000..5b9855c
--- /dev/null
+++ b/lib/format_hit.js
@@ -0,0 +1,23 @@
+var hash = require('./hash.js');
+/**
+ * This file/module's only job is to format the Hit data for human-friendly UI
+ * @param {String} hit - the standard nodejs http request object.
+ * @param {Number} count - the count for the given url
+ * @returns {string} hit - human-friendly hit data for display in UI
+ */
+module.exports = function format_hit_for_ui (hit, count) {
+ var h = hit.split('|'); // See README.md#How secton for sample data
+ var url = h[1];
+ var date = format_date_time_from_timestamp(h[0])
+
+ // save unique hash of browser data to avoid duplication
+ var unique_browser_string = [ h[2], h[3], h[4] ].join('|');
+ var hashed_agent = hash(unique_browser_string, 10);
+ return [date, url, count, hashed_agent].join(' ');
+}
+
+function format_date_time_from_timestamp (timestamp) {
+ var date = new Date(timestamp * 1000).toJSON();
+ var len = date.length;
+ return date.substring(0, len -5).replace('T', ' ');
+}
diff --git a/lib/hash.js b/lib/hash.js
new file mode 100644
index 0000000..26c84a9
--- /dev/null
+++ b/lib/hash.js
@@ -0,0 +1,9 @@
+var crypto = require('crypto');
+
+// generate a hash of a specific length
+module.exports = function hash (str, length) {
+ return crypto.createHash('sha512') // crypto.stackexchange.com/questions/26336
+ .update(str.toString()).digest('base64')
+ .replace('/','').replace(/[Il0oO=\/\+]/g,'') // remove ambiguous chars
+ .substring(0, length || 12);
+}
diff --git a/lib/headers.json b/lib/headers.json
new file mode 100644
index 0000000..333e49f
--- /dev/null
+++ b/lib/headers.json
@@ -0,0 +1,6 @@
+{
+ "Cache-Control": "no-cache, no-store, must-revalidate",
+ "Pragma": "no-cache",
+ "Expires": "0",
+ "Content-Type":"image/svg+xml"
+}
diff --git a/lib/hits.js b/lib/hits.js
index 7da52be..96a8d30 100644
--- a/lib/hits.js
+++ b/lib/hits.js
@@ -1,41 +1,13 @@
-var redisClient = require('redis-connection')();
-var uniki = require('uniki');
/**
* add - adds an entry into the List for a given url
- * @param {String} url - the url for the hit
* @param {Object} hit - the hit we just received
* @param {Function} callback - call this once redis responds
*/
-module.exports.add = function add (hit, callback) {
- // console.log(hit);
- var url = hit.url.replace('https://github.com', ''); // don't waste RAM!
- var now = Date.now();
- var agent = uniki(hit['user-agent'],7);
- if(hit['accept-language'] && hit['accept-language'].indexOf(',') > -1){
- hit.lang = hit['accept-language'].split(',')[0];
- } else {
- hit.lang = '';
+module.exports = function add (hit, callback) {
+ if(process.env.REDISCLOUD_URL) {
+ return require('./db_redis.js')(hit, callback);
+ }
+ else {
+ return require('./db_filesystem.js')(hit, callback);
}
- // console.log('agent',agent);
- redisClient.hset('agents', agent, hit['user-agent']);
- var entry = now + ' ' + agent + ' ' +hit.lang + ' ' + hit.ip
- redisClient.rpush(url, entry, function (err, data) {
- callback(err, data)
- });
-}
-
-/**
- * count - counts the number of hits for a given url
- * @param {String} url - the url for the hit
- * @param {Function} callback - call this once redis responds
- */
-module.exports.count = function count (url, callback) {
- console.log(url);
- url = url.replace('https://github.com', ''); // don't waste space in Redis
- redisClient.llen(url, function(err, data){
- callback(err, data);
- });
}
-
-
-module.exports.redisClient = redisClient;
diff --git a/lib/index.html b/lib/index.html
index e00c1da..6563e33 100644
--- a/lib/index.html
+++ b/lib/index.html
@@ -1,20 +1,87 @@
- Stats
+ Hits!
-
-
+
+
+
-
-
+
+