Barton's Picturelinux counter image.

How It Works Page


The application is written for node.js using express.js and some other modules. Express is a framework used to create a server and router. The express directory hierarchy looks like this:

In the project root is the app.js file which looks like this:

// BLP 2017-03-07 -- This is a nodjs experment. It creates the
// www.bartonlp.org:7000 webpage.

const express = require('express');
const path = require('path');
const logger = require('morgan');
const cookieParser = require('cookie-parser');
const bodyParser = require('body-parser');
const favicon = require('serve-favicon');
const count = require('./routes/utilfunctions.js').count;

// get the routing unit

const routes = require('./routes/index');
const app = express();

// view engine setup

// NOTE we are using 'views.old' which is the 'jade' implementation not
// 'view' which uses 'pug'!!

app.set('views', path.join(__dirname, 'views.old'));
app.set('view engine', 'jade');

//app.use(logger(...)); // Logs info to the console.log in 'development' mode

app.use(logger('[:date[clf]] :remote-addr :remote-user :method :url :status :res[content-length] ":user-agent"'));

// Catch all

app.use(function(req, res, next) {
  if(req.hostname != 'www.bartonlp.org') {
    console.log("headers.host: ", req.headers.host);
    console.log("Hostname: ", req.hostname);
    console.log("Org Url: %s, Url: %s", req.originalUrl, req.url);
    console.log("ERROR: Return");
    next(new Error('Bad Route'));
  }
  next();
});

/*
 * This goes before any router methods. This does the
 * counter and bots logic.
 */

app.use(function(req, res, next) {
  // Get the originalUrl and grab what is after the first / which will
  // be the name of the site jade file.

  count(req, function(err, cnt) {
    if(err) {
      next(err);
      return;
    }

    // Put the count into the req property as cnt.

    req.cnt = cnt;
    next();
  });
});

app.use(bodyParser.json());
app.use(bodyParser.urlencoded({ extended: false }));
app.use(cookieParser());

// Now override this with the 'routes' to .../routes/index.js

app.use('/', routes);

// Set the document root to 'public'

app.use(express.static(path.join(__dirname, 'public')));
app.use(favicon(path.join(__dirname, 'public', 'favicon.png'))); // favicon.png is a sprocket icon.

// catch 404 and forward to error handler

app.use(function(req, res, next) {
  var err = new Error('Not Found');
  err.status = 404;
  next(err);
});

// development error handler
// will print stacktrace

// To remove development mode set the 'env' to blank. Uncomment to
// disable development.
//app.set('env', '');

if(app.get('env') === 'development') {
  app.use(function(err, req, res, next) {
    res.status(err.status || 500);
    if(err.status != 404) {
      req.url = null;
    }

    res.render('error', {
      message: err.message,
      url: req.url,
      status: err.status,
      error: err
    });
  });
}

// production error handler
// no stacktraces leaked to user

app.use(function(err, req, res, next) {
  res.status(err.status || 500);
  if(err.status != 404) {
    req.url = null;
  }
  console.log("MSG: %s", err.message);
  console.log("URL: %s", req.url);
  console.log("ERROR: ", err);
  
  res.render('error', {
    message: err.message,
    url: req.url,
    status: err.status,
    error: {}
  });
});

module.exports = app;

We load ./routes/index.js' and then we app.use('/', routes).

In the routes directory the file index.js does the routing for the application.

/* index.js */

const express = require('express');
const router = express.Router();
const fs = require('fs');
const dns = require('dns');
const request = require('request');

const utilfunctions = require("./utilfunctions.js");
const query = utilfunctions.query;
const count = utilfunctions.count;
const mtime = utilfunctions.mtime;
const run = utilfunctions.run;
const robots = utilfunctions.robots;

var args = {
  copyright: "2017 Barton Phillips",
  author: "Barton Phillips http://www.bartonphillips.com",
  address: "2701 Amhurst Blvd. #12A, New Bern, North Carolina 28562",
  desc: "node.js example",
  msg: "Counter Reset: Feb. 4, 2017",
  site: "Node",
};

/* GET home page. */

router.get(['/','/index(\.(html|php))?'], function(req, res, next) {
  const port = req.headers.host.match(/:(.*)/)[1];
  args.footer = req.cnt;

  return run(function *(resume) {
    try {
      var address = yield   dns.lookup('bartonphillips.dyndns.org', resume);
      console.log("ADDRESS:", address);
    
      //var admin = yield request.get('http://www.bartonlp.com/adminsites.txt', resume);
      //args.adminStuff = admin.body;
      //console.log("adminStuff: ", args.adminStuff);
    } catch(err) {
      console.log("NOT bartonphillips.dyndns.org");
    }
  
    var sql = "select filename, site, ip, agent, count, concat(date(lasttime), ' ', time(lasttime)) as lasttime "+
              "from barton.counter where lasttime>current_date() order by lasttime desc";

    query(sql, function(err, result) {
      if(err) {
        next(err); //new Error('Error: '+err));
        return;
      };

      args.title = 'Node.js';
      args.banner = "Node.js Page";
      args.port = port;
      args.mtime = mtime("index");
      //console.log("mtime: "+args.mtime);
      
      res.render('index', {
        result: result,
        args: args,
      });
    });
  });
});
    
/* GET howitworks */

router.get('/howitworks', function(req, res, next) {
  args.footer = req.cnt;
  return run(function *(resume) {
    try {
      var app = yield fs.readFile("app.js", resume);
      var index = yield fs.readFile("routes/index.js", resume);
      var views = yield fs.readFile("views.old/index.jade", resume);
      var how = yield fs.readFile("views.old/howitworks.jade", resume);
      var layout = yield fs.readFile("views.old/layout.jade", resume);
      var utils = yield fs.readFile("routes/utilfunctions.js", resume);
    } catch(err) {
      return next(err); //new Error("Error: "+err));
    }
    args.title = 'How It Works';
    args.banner = 'How It Works Page';
    args.mtime = mtime("howitworks");

    res.render('howitworks', {
      args: args,
      code1: app,
      code2: index,
      code3: utils,
      code4: views,
      code5: how,
      code6: layout,
    });
  });
});

// Robots.txt

router.get("/robots.txt", function(req, res, next) {
  console.log("Robots Node");
  robots(args.site, req, function(err, robottxt) {
    res.send("<pre>" + robottxt + "</pre>");
    res.end();
  });
});

// About website

router.get('/aboutwebsite', function(req, res, next) {
  args.mtime = mtime("aboutwebsite");
  args.footer = req.cnt;

  res.render('aboutwebsite', {
    args: args,
  });
});

module.exports = router;

The utilfunctions.js has almost all the rest of the logic needed. The actual connection to the database is in a seperate file called createpool.js which only does a

const mysql = require('mysql');
const pool = mysql.createPool({
  host: "localhost",
  user: "USERNAME",  
  password: "PASSWORD",  
  database: "DATABASENAME",  
});

Replace the UPPERCASE values with real mysql database information.

utilfunctions.js
/* Utility Functions */

const mysql = require('mysql');
const fs = require('fs');
const pool = require('../routes/createpool.js').pool;

// Do a mysql query

const query = function(sql, value, cb) {
  if(typeof cb === 'undefined') {
    cb = value;
    value = null;
  }

  pool.getConnection(function(err, con) {
    if(err){
      console.log('Error connecting to Db');
      return cb(err);
    }

    con.query(sql, value, function(err, result) {
      if(err) {
        console.log("SQLERROR: ", err);
      }
      con.release();
      return cb(err, result);
    });
  });
}

exports.query = query;

// run is a little helper

function run(gen, iter) {
  (iter = gen((err, data) => (err && iter.throw(err)) || iter.next(data))).next();
}

// Count the number of hits. Bots and Bots2

exports.count = function(req, cb) {
  var site = req.originalUrl.match(/^\/applitec/i) == null ? 'Node' : 'Applitec';
  var file = req.originalUrl.match(/^.*?(\/.*)$/)[1];

  return run(function *(resume) {
    try {
      var agent = req.headers['user-agent'];
      var ip = req._remoteAddress.replace(/::.*?:/, '');

      var sql = "insert into counter (filename, site, ip, agent, count, lasttime) "+
                "values(?, ?, ?, ?, 1, now()) "+
                "on duplicate key update site=?, ip=?, agent=?, count=count+1, lasttime=now()";

      var value = [ file, site, ip, agent, site, ip, agent ];

      var result = yield query(sql, value, resume);
      
      sql = "select count from barton.counter where site=? and filename=?";
      value = [ site, file ];
      var result = yield query(sql, value, resume);

      var cnt = result[0].count;

      sql = "select count(*) as one from information_schema.tables "+
            "where (table_schema = 'barton') and (table_name = 'bots')";
      
      var ok = (yield query(sql, resume))[0].one;
      //ok = ok[0].one;
      
      if(ok == 1) {
        var x = yield query("select ip from barton.bots where ip='"+ip+"'", resume);

        var isBot = x.length == 0 ? false:  true;

        var isBot = agent.match(/\+*http:\/\/|Googlebot-Image|python|java|wget|nutch|perl|libwww|lwp-trivial|curl|PHP\/|urllib|GT::WWW|Snoopy|MFC_Tear_Sample|HTTP::Lite|PHPCrawl|URI::Fetch|Zend_Http_Client|http client|PECL::HTTP/i)
                    || isBot;

        if(isBot) {
          try {
            // Bots has primary key(ip, agent)
            var result = yield query("insert into barton.bots (ip, agent, count, robots, who, creation_time, lasttime) "+
                                     "values(?, ?, 1, 4, ?, now(), now())", [ip, agent, site], resume);
          } catch(err) {
            if(err.errno == 1062) { // duplicate key
              var who = (yield query("select who from barton.bots where ip=? and agent=?", [ip, agent], resume))[0].who;
              
              if(!who) {
                who = site;
              }
              
              if(who.indexOf(site) == -1) {
                who += ', ' + site;
              }
              
              result = yield query("update barton.bots set robots=robots | 8, who=?, "+
                                   "count=count+1, lasttime=now() "+
                                   "where ip=? and agent=?", [who, ip, agent], resume);
            }
          }
        }

        sql = "select count(*) as one from information_schema.tables "+
              "where (table_schema = 'barton') and (table_name = 'bots2')";

        var ok = (yield query(sql, resume))[0].one;

        if(ok == 1) {
          if(isBot) {
            // Bots2 has primary key (ip, agent, date, site, which)
            result = yield query("insert into barton.bots2 (ip, agent, date, site, which, count, lasttime) "+
                                 "values(?, ?, current_date(), ?, 2, 1, now()) "+
                                 "on duplicate key update count=count+1, lasttime=now()",
                                 [ip, agent, site], resume);
          }
        }
      }

      sql = "select count(*) as one from information_schema.tables "+
            "where (table_schema = 'barton') and (table_name = 'tracker')";

      var ok = (yield query(sql, resume))[0].one;
      //ok = ok[0].one;

      if(ok == 1) {
        var java = 0;

        if(isBot) {
          java = 0x2000; // This is the robots tag
        }

        // tracker key is id
        
        result = yield query("insert into barton.tracker (site, page, ip, agent, starttime, isJavaScript, lasttime) "+
                             "values(?, ?, ?, ?, now(), ?, now())",
                            [site, file, ip, agent, java], resume);

        //$this->LAST_ID = $this->getLastInsertId();
      }

      sql = "select count(*) as one from information_schema.tables "+
            "where (table_schema = 'barton') and (table_name = 'daycounts')";

      var ok = (yield query(sql, resume))[0].one;
      //ok = ok[0].one;

      if(ok == 1) {
        if(isBot) {
          var bots = 1, real = 0;
        } else {
          real = 1, bots = 0;
        }
        sql = "insert into barton.daycounts (site, `date`, `real`, bots, members, visits, lasttime) "+
              "values(?, now(), ?, ?, 0, 1, now()) " +
              "on duplicate key update `real`=`real`+?, bots=bots+?, visits=visits+1, lasttime=now()";

        result = yield query(sql, [site, real, bots, real, bots], resume);
      }

      return cb(null, cnt);
    } catch(err) {
      return cb(err);
    }
  });
}

// Get last modified time

exports.mtime = function(pugfile) {
  // we are using the views.old which has .jade not .pug files
/*
return String(new Date(fs.statSync("/var/www/mynode/views/"+pugfile+".pug").mtime)).replace(/GMT.*? /, '');
*/

  // Note: mtime has '<date> GMT-nnnn (PDT)' and we want just the (PDT)
  // after the GMT-nnnn
  
  var stat = String(new Date(fs.statSync("/var/www/mynode/views.old/"+pugfile+".jade").mtime));
  //console.log("statSync: ", stat);
  return stat.replace(/GMT.*? /, '');
}

exports.run = function(gen, iter) {
  (iter = gen((err, data) => (err && iter.throw(err)) || iter.next(data))).next();
}

exports.robots = function(site, req, cb) {
  var agent = req.headers['user-agent'];
  var ip = req._remoteAddress.replace(/::.*?:/, '');
  
  return run(function *(resume) {
    try {
      var result = yield query("insert into barton.bots (ip, agent, count, robots, who, creation_time, lasttime) "+
                               "values(?, ?, 1, 1, ?, now(), now())", [ip, agent, site], resume);
    } catch(err) {
      if(err.errno == 1062) { // dup key
        var who = (yield query("select who from barton.bots where ip=? and agent=?", [ip, agent], resume))[0].who;
        if(!who) {
          who = site;
        }

        if(who.indexOf(site) == -1) {
          who += ', ' + site;
        }

        result = yield query("update barton.bots set robots=robots | 2, who=?, "+
                             "count=count+1, lasttime=now() "+
                             "where ip=? and agent=?", [who, ip, agent], resume);
      }
    }

    try {
      var robotsFile = site+".robots.txt";
      var robottxt = yield fs.readFile("/var/www/mynode/routes/"+robotsFile, resume);
      return cb(null, robottxt);
    } catch(err) {
      return cb(err);
    }
  });
}

The views directory has the main page and the howitworks.jade page

index.jade:
extends layout.jade

append stylesheets
  style.
    #info { border: 1px solid black; width: 100%; max-height: 20rem; overflow: auto; }
    #info table { widht: 100%; }
    #info td, #info th { border: 1px solid black; padding: .5rem; }

block content 
  p.
    This page is on port #{args.port} and is written using <b>node.js</b> and <b>express.js</b>.
  p This is the root or '/'.

  p. 
    While this may not seem like a wonderful thing it is interesting. I have used <b>express</b>
    as the framework.
    For details on the implementation 
    <a href="http://www.bartonlp.org:#{args.port}/howitworks">How It Works</a>.

  h3 The following is from my mysql 'barton.counter' table
  p You should see your IP and Agent in the list.

  #info
    if result[0] == null
      h3 Counter Table Empty
    else
      table
        thead
          tr
          each item, key in result[0]
            th= key
        tbody
          each c in result
            tr
            each item in c
              td= item
howitworks.jade:
extends layout.jade

append jscripts
  script(src="javascripts/syntaxhighlighter.js")

  script.
    jQuery(document).ready(function($) {
      $("pre").addClass("brush: js");
      $(".nopre").removeClass("brush: js");
    });

append stylesheets
  link(rel="stylesheet" href="stylesheets/theme.css")
  style.
    .syntaxhighlighter {
      height: 10rem;
      font-size: .8rem !important;
    }
    .nopre {
      font-size: .8rem;
      border-left: .5rem solid gray;
      padding-left: .5rem;
    }

block content 
  p.
    The application is written for <b>node.js</b> using <b>express.js</b> and some other modules. 
    Express is a framework used to create a server and router. 
    The <b>express</b> directory hierarchy looks like this:
  ul
    li Project root
    ul
      li bin
      li node_modules
      li public
      li routes
      li views

  p.
    In the project root is the <b>app.js</b> file which looks like this:
  pre #{code1}

  p.
    We load <b>./routes/index.js'</b> and then we <b>app.use('/', routes)</b>.
  p.
    In the <b>routes</b> directory the file <b>index.js</b> does the routing for the application.
  pre #{code2}
  p. 
    The <b>utilfunctions.js</b> has almost all the rest of the logic needed. The actual connection
    to the database is in a seperate file called <b>createpool.js</b> which only does a <br>  
  pre(class='nopre').
    const mysql = require('mysql');
    const pool = mysql.createPool({
      host: "localhost",
      user: "USERNAME",  
      password: "PASSWORD",  
      database: "DATABASENAME",  
    });
  p.
    Replace the UPPERCASE values with real mysql database information.
  b utilfunctions.js
  pre #{code3}
  p.
    The <b>views</b> directory has the main page and the <b>howitworks.jade</b> page
  b index.jade:
  pre #{code4}
  b howitworks.jade:
  pre #{code5}
  p.
    Finally <b>layout.jade</b> looks like this
  b layout.jade:
  pre #{code6}

block footer


Finally layout.jade looks like this

layout.jade:
doctype html

html
  head
    title #{args.title}
    meta(name=viewport content="width=device-width, initial-scale=1")
    meta(charset='utf-8')
    meta(name="copyright" content="#{args.copyright}")
    meta(name="Author" content="#{args.author}")
    meta(name="description" content="#{args.desc}")
    meta(name="keywords" content="Barton Phillips")
    
    link(rel="stylesheet" href="https://bartonphillips.net/css/blp.css" title="blp default")

    style.
      .red {color: red;}
      .center {text-align: center;}
      #hitCounter { margin-left: auto; margin-right: auto; width: 50%; text-align: center; }
      #hitCountertbl { font-size: 1em; width: 0; border: 8px ridge yellow; margin-left: auto; margin-right: auto; background-color: #F5DEB3 }      #hitCountertr { width: 0; border: 8px ridge yellow; margin-left: auto; margin-right: auto; background-color: #F5DEB3 }
      #hitCounterth { color: rgb(123, 16, 66); }

    block stylesheets

    script(src="https://ajax.googleapis.com/ajax/libs/jquery/1/jquery.min.js")

    block jscripts

  body
    header
      div
        a(href="/")
          img(id='blpimg' src="https://bartonphillips.net/images/blp-image.png" width='90' height-'120' alt="Barton's Picture")
        a(href="http://linuxcounter.net/")
          img(id='linuxcounter' src="https://bartonphillips.net/images/146624.png" width='190' height='110' alt="linux counter image.")

      h1.center #{args.banner}

      hr
  
    block content

    footer
      hr
      block footer

      |Return to 
      a.underline(href="/") Home Page

      h2.center: a(href="/aboutwebsite") About This Site

      #counter
        div #{args.msg}

        #hitCounter
          table#hitCountertbl
            tr#hitCountertr
              th#hitCounterth #{args.footer}
        #address
          address.
            Copyright &copy; #{args.copyright}
            <br>
            Barton Phillips, #{args.address}
            <br>

        Last Modified: #{args.mtime}