LowPowerLab Forum

Software support => Pi Gateway => Topic started by: rd909 on July 04, 2019, 02:59:54 PM

Title: gateway v9 httpendpoint api
Post by: rd909 on July 04, 2019, 02:59:54 PM
I was able to modify the "httpEndPointHandler" function in gateway.js to accept commands, which could be used by some other app (homebridge in my case) to execute commands. Below is the code for the function that has my modifications in it. It's pretty hack-ish, but is a POC that hopefully someone else might find useful and expand upon.

Code: [Select]
function httpEndPointHandler(req, res) {
  var queryString = url.parse(req.url, true).query; //parse query string
  var ip = req.headers['x-forwarded-for'] /*|| req.connection.remoteAddress*/ ; //appended by nginx proxy
  var id = queryString.id || ip;
  //homebridge var
  var cmd = queryString.cmd;

  if (metricsDef.isValidNodeId(id)) {
    if (metricsDef.isNumeric(id)) id = parseInt(id);
    db.find({
      _id: id
    }, function(err, entries) {
      var existingNode = {};
      var matchedMetrics = 0;
      if (entries.length == 1) existingNode = entries[0]; //update
      existingNode._id = id;
      if (metricsDef.isNumeric(id)) existingNode._ip = ip; //add/override IP address for HTTP requests, if node ID was specified as a number (so we know what IP to send requests back to)
      existingNode.updated = Date.now(); //update timestamp we last heard from this node, regardless of any matches
      if (existingNode.metrics == undefined) existingNode.metrics = {};

      for (var queryStringKey in queryString) {
        var matchingMetric;
        var token;
        for (var metric in metricsDef.metrics) //try to match a metric definition
        {
          token = queryStringKey.trim() + ':' + queryString[queryStringKey].trim();
          if (metricsDef.metrics[metric].regexp.test(token)) {
            var tokenMatch = metricsDef.metrics[metric].regexp.exec(queryStringKey + ':' + queryString[queryStringKey]);
            matchingMetric = metricsDef.metrics[metric];
            if (existingNode.metrics[matchingMetric.name] == null) existingNode.metrics[matchingMetric.name] = {};
            existingNode.metrics[matchingMetric.name].label = existingNode.metrics[matchingMetric.name].label || matchingMetric.name;
            existingNode.metrics[matchingMetric.name].descr = existingNode.metrics[matchingMetric.name].descr || matchingMetric.descr || undefined;
            existingNode.metrics[matchingMetric.name].value = matchingMetric.value || metricsDef.determineValue(matchingMetric, tokenMatch);
            existingNode.metrics[matchingMetric.name].unit = matchingMetric.unit || undefined;
            existingNode.metrics[matchingMetric.name].updated = existingNode.updated;
            existingNode.metrics[matchingMetric.name].pin = existingNode.metrics[matchingMetric.name].pin != undefined ? existingNode.metrics[matchingMetric.name].pin : matchingMetric.pin;
            existingNode.metrics[matchingMetric.name].graph = existingNode.metrics[matchingMetric.name].graph != undefined ? existingNode.metrics[matchingMetric.name].graph : matchingMetric.graph;

            //log data for graphing purposes, keep labels as short as possible since this log will grow indefinitely and is not compacted like the node database
            if (existingNode.metrics[matchingMetric.name].graph == 1) {
              var graphValue = metricsDef.isNumeric(matchingMetric.logValue) ? matchingMetric.logValue : metricsDef.determineGraphValue(matchingMetric, tokenMatch); //existingNode.metrics[matchingMetric.name].value;
              if (metricsDef.isNumeric(graphValue)) {
                var ts = Math.floor(Date.now() / 1000); //get timestamp in whole seconds
                var logfile = path.join(__dirname, dbDir, dbLog.getLogName(id, matchingMetric.name));
                try {
                  console.log('post: ' + logfile + '[' + ts + ',' + graphValue + ']');
                  dbLog.postData(logfile, ts, graphValue, matchingMetric.duplicateInterval || null);
                } catch (err) {
                  console.error('   POST ERROR: ' + err.message); /*console.log('   POST ERROR STACK TRACE: ' + err.stack); */
                } //because this is a callback concurrent calls to the same log, milliseconds apart, can cause a file handle concurrency exception
              } else console.log('   METRIC NOT NUMERIC, logging skipped... (extracted value:' + graphValue + ')');
            }

            if (matchingMetric.name != 'RSSI') matchedMetrics++; //excluding RSSI because noise can produce a packet with a valid RSSI reading
            break; //--> this stops matching as soon as 1 metric definition regex is matched on the data. You could keep trying to match more definitions and that would create multiple metrics from the same data token, but generally this is not desired behavior.
          }
        }
      }

      //prepare entry to save to DB, undefined values will not be saved, hence saving space
      var entry = {
        _id: id,
        _ip: existingNode._ip,
        updated: existingNode.updated,
        type: existingNode.type || undefined,
        label: existingNode.label || undefined,
        descr: existingNode.descr || undefined,
        hidden: existingNode.hidden || undefined,
        rssi: existingNode.rssi,
        metrics: Object.keys(existingNode.metrics).length > 0 ? existingNode.metrics : {},
        events: existingNode.events,
        settings: existingNode.settings,
        multiGraphs: existingNode.multiGraphs,
        icon: existingNode.icon || undefined
      };

      //console.log('HTTP REQUEST MATCH from: ' + id + ' : ' + JSON.stringify(entry));

      //save to DB
      if (cmd != 'sts') //don't write to the log or update the db for homebridge status update requests
      {
        db.findOne({
          _id: id
        }, function(err, doc) {
          if (doc == null) {
            if (settings.general.genNodeIfNoMatch.value == true || settings.general.genNodeIfNoMatch.value == 'true' || matchedMetrics) {
              db.insert(entry);
              console.log('   [' + id + '] DB-Insert new _id:' + id);
            } else return;
          } else
            db.update({
              _id: id
            }, {
              $set: entry
            }, {}, function(err, numReplaced) {
              console.log('   [' + id + '] DB-Updates:' + numReplaced);
            });

          //publish updated node to clients
          io.sockets.emit('UPDATENODE', entry);
          //handle any server side events (email, sms, custom actions)
          handleNodeEvents(entry);
        });
      }

      res.writeHead(200, {
        'Content-Type': 'application/json'
      });
      if (cmd == 'sts' || cmd == 'lck' || cmd == 'ulk' || cmd == 'cls' || cmd == 'opn') { //custom returns for homebridge commands
        if (cmd == 'sts') { //this is a garagedoor status check
          res.write(JSON.stringify({
            status: entry.metrics.Status.value,
            LockStatus: entry.metrics.LockStatus.value
          }));
        } else { //actually do something
          sendMessageToNode({
            nodeId: id,
            action: cmd
          })
          res.write(JSON.stringify({
            status: entry.metrics.Status.value,
            LockStatus: entry.metrics.LockStatus.value
          }));
        }
      } else
        res.write(JSON.stringify({
          status: 'success',
          message: 'SUCCESS!',
          matchedMetrics: matchedMetrics
        }));
      res.end();
    });
  } else {
    res.writeHead(406, {
      'Content-Type': 'application/json'
    });
    res.write(JSON.stringify({
      status: 'error',
      message: 'FAIL, invalid id:' + id
    }));
    res.end();
  }
}
Title: Re: gateway v9 httpendpoint api
Post by: Lukapple on July 12, 2019, 10:19:34 AM
Hi,
I'm already using it with HomeBridge, using custom api.js for reading data from database.

@Felix httpendpoint is probably used just for posting data to gateway or is there also a way to read current node data from DB file?

I need this to read a current temperature data of specific node from DB file. I'm currently using custom api.js script to achieve this, but it would be really great if Gateway officially supports this.
I think that extending httpendpoint funcionality with "read node data" shouldn't be problem, using:
db.findOne({_id:parseInt(id)}, function (err, doc) {...

Title: Re: gateway v9 httpendpoint api
Post by: Felix on July 12, 2019, 03:39:16 PM
Yes that would be a good addition, thanks.
I will consider doing it but I'm not sure when I get to it.
Maybe you could try it on your own given the pattern that is in place already.
Title: Re: gateway v9 httpendpoint api
Post by: Lukapple on July 18, 2019, 01:53:26 AM
Yes that would be a good addition, thanks.
I will consider doing it but I'm not sure when I get to it.
Maybe you could try it on your own given the pattern that is in place already.

I’ll do a branch on github. I think that the right way of doing it is that you change requests that add/update nodes to “POST” method and “GET” method should return existing data.
Title: Re: gateway v9 httpendpoint api
Post by: Felix on July 18, 2019, 09:05:09 AM
Awesome that you are taking the task to contribute  ;)
You can post updates and progress here.
Thanks!
Title: Re: gateway v9 httpendpoint api
Post by: HeneryH on February 18, 2020, 03:44:07 PM
This might be useful to me.  I have a Twilio account that can receive SMS messages and send web callback requests to any web server.  The goal is to grant access to clients to a space by listening for SMS messages and then pressing a virtual 'open' button.
Title: Re: gateway v9 httpendpoint api
Post by: gigawatts on May 29, 2020, 10:54:19 PM
So I see that making a GET request to, for example:
Code: [Select]
/httpendpoint/?id=42&TEMP=68.7
will record a value that can be graphed on the Gateway. Cool!

What I'm trying to figure out, is how to use this feature to activate a SwitchMote relay. I thought that making a GET request to something like:
Code: [Select]
/httpendpoint/?id=2&BTN1=0
would turn off the relay programmed to BTN1 on wireless SwitchMote Node # 2.

It does not. It does. however, make the gateway interface believe that relay is now off, as an off event is recorded and the web UI power button for that light will update accordingly, but that message is never sent over the RF network to the switchmote, so the light does not turn off.

Is there a built-in means to do this? Or would a custom event trigger need to be written to listen for such an event, and trigger a "send RF message to SwitchMote relay" command?
Title: Re: gateway v9 httpendpoint api
Post by: gigawatts on May 30, 2020, 12:28:30 PM
I also tried
Code: [Select]
/httpendpoint/?id=2&SSR=1
It also changes the state of the switch on the gateway, but not on the SwitchMote.
Title: Re: gateway v9 httpendpoint api
Post by: gigawatts on May 30, 2020, 02:45:55 PM
Taking a deeper look at rd909's above code, I just got this working with the following changes to gateway.js:

Code: [Select]
function httpEndPointHandler(req, res) {
  var queryString = url.parse(req.url, true).query; //parse query string
  var ip = req.headers['x-forwarded-for'] /*|| req.connection.remoteAddress*/; //appended by nginx proxy
  var id = queryString.id || ip;
  var cmd = queryString.cmd;  // store custom command variable

...

      res.writeHead(200, {'Content-Type': 'application/json'});
// my changes start here -------------------------------------------------
      if ( cmd && cmd.substring(0,3) == 'BTN' ) { //custom returns for BTN commands
        sendMessageToNode({ nodeId: id, action: cmd })
        res.write(JSON.stringify({
          status: 'success',
          message: 'Command sent to nodeId ' + id,
          command: cmd,
          matchedMetrics: matchedMetrics
        }));
      } else {
        res.write(JSON.stringify({
          status: 'success',
          message: 'SUCCESS!',
          matchedMetrics: matchedMetrics
        }));
      }
//  my changes stop here -------------------------------------------------
      res.end();

Now making a GET request such as this works great!
Code: [Select]
/httpendpoint/?id=2&cmd=BTN1:0
Title: Re: gateway v9 httpendpoint api
Post by: gigawatts on May 30, 2020, 04:11:25 PM
@Felix I can create a PR for these changes, if you are interested in merging them into the main project
Title: Re: gateway v9 httpendpoint api
Post by: Felix on June 02, 2020, 08:16:02 AM
gigawatts,
Thanks for the updates. The way you implemented it works, but it is hardcoded. This is a new feature that would allow any action from one node (network node) to be sent off to another node (RF).
The current httpendpoint implementation, as you observed, is just a one way. Currently you can make a switchmote turn on, for example, when a motionmote detects and reports motion, via an EVENT that can be turned ON/OFF. That's the same pattern or a similar pattern needs to be followed. Otherwise every time there is a new request for another command, we need to change the core app which is not maintainable.

So keep what you have and I will add this to my TODO for a coming release. It's threads like these that help me understand how you folks want to use the app and what features you need. I will do my best to accomodate and find the time to implement the best features as time permits.
Title: Re: gateway v9 httpendpoint api
Post by: gigawatts on June 02, 2020, 06:33:16 PM
Ah, that makes sense, using events properly :)

For now, this hack works great for my uses.

I'll look forward to this as a full feature! Feel free to ping me if you want a beta tester!

Side note: is there a way to remove the IP I now have attached to several of my nodes, while experimenting with that endpoint? It's not harming anything, but just curious.
Title: Re: gateway v9 httpendpoint api
Post by: Felix on June 03, 2020, 07:41:29 PM
Side note: is there a way to remove the IP I now have attached to several of my nodes, while experimenting with that endpoint? It's not harming anything, but just curious.
Not sure I understand this, can you detail a bit? You mean the IP that is assigned to a node when it first receives data through the endpoint?
Title: Re: gateway v9 httpendpoint api
Post by: gigawatts on June 03, 2020, 10:42:28 PM
Not sure I understand this, can you detail a bit? You mean the IP that is assigned to a node when it first receives data through the endpoint?

Sorta. I had existing RF only nodes, 2 - 5. While experimenting with using the httpendpoint and trying to get it to "trigger" buttons on that remote RF node (before I fully understood that is not what this feature was for), the gateway interface added the IP of my laptop I used to play with the endpoint to each of those nodes I was trying to trigger.

Here is an example of one of my SwitchMote RF nodes, displaying the IP of my laptop (the SwitchMote, obviously, does not have an IP):
(https://i.imgur.com/loY1jgd.png)

It is not harming anything, but I would just like to know if I can remove that IP from that node.
Title: Re: gateway v9 httpendpoint api
Post by: Felix on June 05, 2020, 08:38:29 AM
Ok so here's the rationale behind the IP field.
Essentially a node could be both RF as well as hooked up to the LAN. In which case it has ... 2 IDs. This is intentional and not an overlook.

Now to answer your Q, to remove the IP right now, you'd have to sudo systemctl stop gateway, remove that property manually from the DB (do a COMPACT before you stop), then sudo systemctl restart gateway.

So it's really more about making sure you tailor the incoming packets the way you want it.
But to make it removable through the UI, it would be an enhancement. I could add that in but I have to think about it if it doesn't cause any issues in some other ways. If it's OK, then it would likely just be a "trash" button in the same control, like you see on the events, clicking that would remove that property from the node.
Further packets from that same IP to that same node would add the IP property back obviously.
Title: Re: gateway v9 httpendpoint api
Post by: gigawatts on June 05, 2020, 01:08:47 PM
Good info, I'll try that. Thanks!