EzDevInfo.com

How to create database within nodejs application code in node-webkit?

I'm trying to use NeDB database for persistence of application data. I'm trying to use the following approach to connect to my database like the following:

var Datastore = require('nedb')
  , path = require('path')
  , db = new Datastore({ filename: path.join(require('nw.gui').App.dataPath, 'something.db') }

But unfortunatly it fails because this works only client code withit <script></script> tags in html file. How could I do same thing on server side?

Express.js database and validation logic. Where?

Nothing important
My first question here on stackoverflow. I've used it for years to find answers, but now I need a bit of guidance. I'm new to node and express and the async way of structuring an app.

Goal - A REST interface with validation and neDB database
I got the following code working. POST a new user is the only route. It's based on many answers and tuts mixed together. I find it hard to scaffold out the logic, to get a structure you can build on.

I'm not sure at all whether this structure is crap or not. Any advice would be appreciated.

Main file is initializing the database, middleware validator, and starting the app.

// rest.js
var express    = require('express'),
    bodyParser = require('body-parser'),
    validator  = require('express-validator'),

    db         = require('./database/db'),
    userRouter = require('./routers/users');

db.init();

var app = express();

app.use(bodyParser.urlencoded({ extended: true }));
app.use(bodyParser.json());
app.use(validator());

app.use('/api/users', userRouter);

var port = process.env.PORT || 8080; 
app.listen(port);

Database
This question/answer made me create the small database module with alterations. How do you pass objects around node express application?
It doesn't have much attention. Maybe because it's very obvious or maybe not a good answer.

The idea is that whem i get multiple collections, they all get initialized on startup, but I can request a single collection if that's all the module needs, or I can get the entire db object back if another module would require that.

// database/db.js

var nedb       = require('nedb');

var db = {};

db.init = function() {
    db.users = new nedb({ filename: './database/data/users', autoload: true });
    db.users.ensureIndex({ fieldName: 'username', unique: true }, function (err) {});
    db.users.ensureIndex({ fieldName: 'email', unique: true }, function (err) {});
};

db.get = function(collection) {
    if (collection && db[collection])
        return db[collection];
    return db;
}

module.exports = db;

Router
I require the User Model here and use the express-validator and sanitizes the request before passing it on to the model, based on a minimalist key schema in the model. I don't have any controllers. If I had (or when I do), I would put the validation there. The router is supposed to send the response and status right?

// routers/users.js

var express    = require('express'),
    _          = require('lodash'),
    User       = require('../models/user');

var userRouter = express.Router();

userRouter.route('/')
    .post(function(req, res) {

        req.checkBody('username', 'Username must be 3-20 chars').len(3,20);
        req.checkBody('email', 'Not valid email').isEmail();
        req.checkBody('password', 'Password must be 6-20 chars').len(6,20);

        var err = req.validationErrors();
        if (err) {
            res.status(422).send(err);
            return;
        }

        var data = _.pick(req.body, _.keys(User.schema));

        User.create(data, function (err, newData) {
            if (err) {
                res.status(409).send(err);
            } else {
                res.status(201).send(newData);
            }
        });

    });

module.exports = userRouter;

Model
The model requires the database module and gets the "connection". Is this OK?

// models/user.js
var db          = require('../database/db');

var User = function (data) {
    this.data = data;
};

User.schema = {
    _id: null,
    username: null,
    email: null,
    password: null
};

User.create = function (data, callback) {
    db.get('users').insert(data, callback);
};

module.exports = User;

Thanks for reading this far. Now, my question is: Is there something fundamentally wrong with this setup, concerning the database usage and the validation logic. I know the model looks stupid :)

Node-webkit use the data from nedb with reactjs

I'm trying to create desktop app with database support. Since nedb is asynchronous I use this code with React:

db.find({}, function (err, docs) {
    React.render(
        React.createElement(TaskList, {data: docs}),
        document.getElementById('content')
    );
});

Question: is this correct way to fetch the data and use it with React? Is there any better way?

Thank you in advance.

Read variables from a parent function

This is a javascript code from my node-webkit app, working with jquery and nedb for managing the databases.

librodb.find({_id: docs[i].libro}, function (err, bookdoc) {
window.titulo = bookdoc[0].titulo;
window.ISBN = bookdoc[0].ISBN;
});

That reads the entries from the db and returns them into an array (bookdoc).

for (var i = 0; i < docs.length; i++) {
  librodb.find({_id: docs[i].libro}, function (err, bookdoc) {
  window.titulo = bookdoc[0].titulo;
  window.ISBN = bookdoc[0].ISBN;
  });
  switch(docs[i].razon){
    case 1:
      $(".listed").append('<li><i class="fa fa-institution"></i><i class="fa fa-sign-in"></i>El '+docs[i].fecha+' '+docs[i].cantidad+' Libros ("'+window.titulo+'", ISBN: '+window.ISBN+') producidos.</li>');
      break;
      case 2:
        libreriadb.find({_id: docs[i].libreria}, function (err, librarydoc) {
          window.nombre = librarydoc[0].nombre;
        });
        $(".listed").append('<li><i class="fa fa-institution"></i><i class="fa fa-sign-in"></i>El '+docs[i].fecha+' '+docs[i].cantidad+' Libros ("'+window.titulo+'", ISBN: '+window.ISBN+') devueltos por Libreria ("'+window.nombre+'"), recibo '+docs[i].documento+'.</li>');
      break;
      case 3:
        $(".listed").append('<li><i class="fa fa-question"></i><i class="fa fa-sign-in"></i>El '+docs[i].fecha+' '+docs[i].cantidad+' Libros ("'+window.titulo+'", ISBN: '+window.ISBN+') en stock ingresaron por "'+docs[i].descripcion+'".</li>');
      break;
    }
}

The issue is that the variables window.titulo and window.ISBN are defined inside the reading database function, but outside there arent.

If i use

window.variablename=

When i call the variables after de librodb.find function both return "undefined".

if i use

var variablename=

or

variablename=

The execution stops with the following error: "ReferenceError: titulo is not defined" (in the place where i try to call it from the switch).

In all the three cases an alert inside the librodb.find function returns the value that is supossed to return.

How do i have to define or call the variables?

Nedb Multiple Collection Single Datastore

I am a new to nedb. Its a kinda what sqlite is for sql community but for the node.js community. [https://github.com/louischatriot/nedb]

I wanted to ask is possible to have multiple collections in a single database file (datastore). If there is, could please show me some code sample on how to go about it? I have tried this:

var Datastore = require('nedb'),
    databaseURL="tudls.db",
    db = new Datastore({filename: databaseURL, autoload: true});

This creates a single datastore called db. From the documentation, I saw that nedb is mongo-like. So to insert a record I tried this:

 app.post('/todos', function(req, res){
        var task = req.body.text;
        db.todols.insert({text: task, done: false}, function(err, saved){
            if(err||!saved){
                res.send("Task not saved...");
            }
            res.send("Task saved...");});
    });

However, I get a 'cannot call method insert of undefined.' I thought that if I call the collection name (todols) when inserting a record it would work so that I can proceed to add another collection to the datastore (db.user) but I was mistaken.

Hence, is it possible to have multiple collections in a single datastore or am I to have a datastore for each collection? If it is possible, does anyone know how to achieve this? Thank you...

nedb method update and delete creates a new entry instead updating existing one

I'm using nedb and I'm trying to update an existing record by matching it's ID, and changing a title property. What happens is that a new record gets created, and the old one is still there. I've tried several combinations, and tried googling for it, but the search results are scarce.

var Datastore = require('nedb');
var db = {
    files: new Datastore({ filename: './db/files.db', autoload: true })
};

db.files.update(
  {_id: id}, 
  {$set: {title: title}}, 
  {}, 
  callback
);

What's even crazier when performing a delete, a new record gets added again, but this time the record has a weird property: {"$$deleted":true,"_id":"WFZaMYRx51UzxBs7"}

This is the code that I'm using:

db.files.remove({_id: id}, callback);

Correct way to pass data from nodejs to angular in nwjs (node-webkit)

Solved: The below code now works for anyone who needs it.

1. Create an Angular factory that queries a database and returns query results.
2. Pass those results of the query into the $scope of my controller
3. Render the results of the $scope variable in my view (html)

Challenge: Because this is for a node-webkit (nwjs) app, I am trying to do this without using express, setting up api endpoints, and using the $http service to return the query results. I feel like there should be a more eloquent way to do this by directly passing the data from nodejs to an angular controller. Below is what I've attempted but it hasn't worked.

Database: Below code is for a nedb database.

My Updated Controllers

app.controller('homeCtrl', function($scope,homeFactory){

    homeFactory.getTitles().then(function(data){
        $scope.movieTitles = data;
    });
});

My Updated Factory:

app.factory('homeFactory', function($http,$q) {

return {

    getTitles: function () {
        var deferred = $q.defer();
        var allTitles = [];

        db.find({}, function (err, data) {
            for (var i = 0, len = data.length; i < len; i++) {
                allTitles.push(data[i].title)
            }
            deferred.resolve(allTitles);
        });

        return deferred.promise;
    }
}

});

HTML

   <script>
        var Datastore = require('nedb');
        var db = new Datastore({ filename: './model/movies.db', autoload: true });
    </script>

<--shows up as an empty array-->
<p ng-repeat="movie in movieTitles">{{movie}}</p>

How to build models (ODM) in nodejs to use with nedb

I am looking for a ODM module that can be used with Nedb in NodeJs-Express. Is it somehow possible to use Mongoose or Waterline schemas in combination with nedb for validation? Mongoose and Waterline do not provide an official adapter for nedb.

At the moment have very low requirements for performance and i am working on a project for the raspberry pi therefore i would like to stick with nedb (or similar file-based nosql database).

Electron + NeDB getting error: async.queue is not a function

I'm trying to use NeDB in my Electron app (in render process). I'm getting the following error when trying to create a DB:

Uncaught TypeError: async.queue is not a function (executor.js:13)

I've noticed that var async = require('async') in executor.js returns an empty object. Any ideas how to fix that? By the way, both NeDB and async modules are installed.

NeDB didn't update the records

Hi I don't know why this function didn't do anything, even don't show anything on the console, and the callback function didn't work , Sorry if I made an obvious mistake I'm new with node js and NeDb.

here is my update function:

var Datastore = require('nedb'),
    db = {
        games: new Datastore({filename: './backend/data/games.db', autoload: true})
    };

var models = { 
     games : {
              update: function (query, update, options, callback) {
                           db.games.update(query, update, options,callback)
             }
            }
           }
module.exports = models;

and I called with:

var models = require('./models');

models.games.update({_id = game_id}, {$set: {fen: game.fen(), pgn: game.pgn()}}, {}, function(err,numDocs,docs){
      if (err){
         console.log(err);
      } else {
         console.log(numDocs);
         console.log(docs);
      }
    });

mi database looks like:

{"_id":"Egw17uRnAd5sdaKXVlOfxRQ6zr4VnSvFghiXkXHyCi9oiDMqDS","startpos":"rnbqkbnr/pppppppp/8/8/8/8/PPPPPPPP/RNBQKBNR w KQkq - 0 1","fen":"rnbqkbnr/pppppppp/8/8/8/8/PPPPPPPP/RNBQKBNR w KQkq - 0 1","pgn":"","w_id":"","b_id":""}
{"_id":"6ZlTTQgk2hEhorTGeTV45kkRsUvrxfmROiCJMqLRQoTnvabhqK","startpos":"rnbqkbnr/pppppppp/8/8/8/8/PPPPPPPP/RNBQKBNR w KQkq - 0 1","fen":"rnbqkbnr/pppppppp/8/8/8/8/PPPPPPPP/RNBQKBNR w KQkq - 0 1","pgn":"","w_id":"","b_id":""} 

and the variables for the query

game_id = '6ZlTTQgk2hEhorTGeTV45kkRsUvrxfmROiCJMqLRQoTnvabhqK'
game.pgn() = '1. e4 e5 2. Nf3'
game.fen() = 'rnbqkbnr/pppp1ppp/8/4p3/4P3/5N2/PPPP1PPP/RNBQKB1R b KQkq - 1 2'

UPDATE 1

if I put all the code together ,not in separated module, works perfectly

var Datastore = require('nedb'),
    db = {
          games: new Datastore({filename: './backend/data/games.db', autoload: true})
          };
db.games.update({_id : game_id}, {$set: {fen: game.fen(), pgn: game.pgn()}}, {}, function(err,numDocs,docs){
   if (err){
       console.log(err);
   } else {
       console.log(numDocs);
       console.log(docs);
   }
});

maybe is something with been working in separated module

UPDATE 2 for simplicity

The models.js file

    var Datastore = require('nedb'),
        db = {
            games: new Datastore({filename: 'database.db', autoload: true})
        };


    var models = {
        games: {
            create: function (id, startpos, fen, pgn, w_id, b_id, callback) {
                db.games.insert({_id: id, startpos: startpos, fen: fen, pgn: pgn, w_id: w_id, b_id: b_id}, callback);
            },
            update: function (query, update, options, callback) {
                db.games.update(query, update, options, callback)
            }
        }
    };

    module.exports = models;

the runupdate.js file

    var models = require('./models');

    models.games.create('6ZlTTQgk2hEhorTGeTV45kkRsUvrxfmROiCJMqLRQoTnvabhqK', 'rnbqkbnr/pppppppp/8/8/8/8/PPPPPPPP/RNBQKBNR w KQkq - 0 1', 'rnbqkbnr/pppppppp/8/8/8/8/PPPPPPPP/RNBQKBNR w KQkq - 0 1', '', '', '');

    models.games.update({_id: '6ZlTTQgk2hEhorTGeTV45kkRsUvrxfmROiCJMqLRQoTnvabhqK'}, {$set: {fen: 'r1bqk1nr/pppp1ppp/2n5/2b1p3/2B1P3/5N2/PPPP1PPP/RNBQ1RK1 b kq - 5 4', pgn: '1. e4 e5 2. Nf3 Nc6 3. Bc4 Bc5 4. O-O'}}, {}, function(err,numDocs,docs){
        if (err){
            console.log(err);
        } else {
            console.log(numDocs);
        }
    });

Updating an array element in nedb

I know this has been answered before, but for some reason none of the solutions has helped me. So I'd like to ask for some help and see if you can spot what I'm doing wrong. I have nedb running and a .db file that looks something like this (removed a few attributs to make it clearer):

{
    "name":"Haj",
    "members": [{"id":"x","name":"x"}],
    "shops": [{"id":"123","name":"shopname","category":"xyz"}],
    "_id":"XXXXX"
}

What I want to do is edit one attribute in one shop, for example the name. I've tried this that seems to be the solutions I see a lot. But for some reason it doesn't work for me.

router.put('/vote', function(req, res) {
    //hardcoded testing
    req.db.groups.update(
      {"_id": "XXXXX", "shops.id": "123"},
      {$set:{"shops.$.name":"why wont you work"}}, 
      function(err, items){
        res.contentType('application/json');
        res.send('{"success":true}');
    });
});

It works if I change an attribute that's not within the shops array (like "name":"Haj"). Using:

{$set:{"name":"this works"}}

I hope someone is able to help me :)

Promise not resolved in karma when testing a service

I have a service 'dbService' written in angular.

devlog.service('dbService', ['$q', function($q) {

    this.getLogs = function() {
        var deferred = $q.defer();

        db.find({}, function(err, docs) {
            if(!err) {
                deferred.resolve(docs);
            } else {
                deferred.reject(err);
            }
        });

        return deferred.promise;
    }
}

The above code fetches data from database(nedb). Its just a wrapper around nedb's api, which returns a promise.

The idea of unit testing, is to test the logic without making any xhr or db calls.

How do i do that here, in my case ?

I've tried testing in the following way, assuming we are breaking the above law of separation of concerns.

describe("DevLog Services", function() {
    var dbService;
    var $rootScope;

    beforeEach(module('devLog'));

    beforeEach(inject(function(_$rootScope_, _dbService_) {
        $rootScope = _$rootScope_;
        dbService = _dbService_;
    }));

    it('should work', function(done) {
        var promise = dbService.getLogs();

        promise.then(function(logs) {
            console.log('yipee' + logs);
            expect(logs).toBeDefined();
        });

        $rootScope.$apply();
        done();
    });
}

The above code doesn't resolve the promise.

I have tried out, adding an afterEach which does $rootScope.$apply();

    afterEach(function() {
        $rootScope.$apply();
    }

    it('should work', function(done) {
        var promise = dbService.getLogs();
        var allLogs;

        promise.then(function(logs) {
            allLogs = logs;
            console.log(allLogs);
            done();
        });

        $rootScope.$apply();
        expect(allLogs).toBeDefined();

    });

In this case, i am getting this error Error: Timeout - Async callback was not invoked within timeout specified by jasmine.DEFAULT_TIMEOUT_INTERVAL..

At least in this case, promise is getting resolved. But it is resolved after that test case is complete, which is causing the test case to fail.

Any ideas, how to resolve this

Solution:

I found the solution for this. The trick is to use $rootScope.$apply() in the service itself.

 devlog.service('dbService', ['$q', '$rootScope', function($q, $rootScope) {

    this.getLogs = function() {
        var deferred = $q.defer();

        db.find({}, function(err, docs) {
            if(!err) {
                deferred.resolve(docs);
            } else {
                deferred.reject(err);
            }

            $rootScope.$apply();
        });

        return deferred.promise;
    }
}

and in tests

it('should work', function(done) {
    var promise = dbService.getLogs();
    var allLogs;
    promise.then(function(logs) {
        expect(logs).toBeDefined();
        expect(logs.length).toBeGreaterThan(0);
        done();
    });

});

Update only works sometimes

When trying to update data in a NeDB database I'm getting Field names cannot begin with the $ character however it only happens after the second or third time updating the DB.

Here's the code I'm using to update/insert data:

addNew: function(data) {
  var deferred = $q.defer(),
      query = {
        _id: data._id,
        title: data.title,
        year: data.year,
        genres: data.genres,
        poster: data.poster
      };

  db.update(query, { $addToSet: { episodes: data.episodes[0] } }, { upsert: true }, function(err, numReplaced, upsert) {
    if (!err) {
      var data = numReplaced + '/' + upsert;
      deferred.resolve(data);
    } else {
      deferred.reject(err);
    }
  });

  return deferred.promise;
}

data.episodes[0] looks like this:

{
  "number": 5,
  "season": 3,
  "title": "Kissed by Fire",
  "overview": "The Hound is judged by the gods. Jaime is judged by men. Jon proves himself. Robb is betrayed. Tyrion learns the cost of weddings.",
  "screen": "http://slurm.trakt.us/images/episodes/1395-3-5.80.jpg"
}

The data in the DB I'm trying to update looks like this:

{
  "_id": "tt0944947",
  "title": "Game of Thrones",
  "year": 2011,
  "genres": "Drama, Fantasy, Adventure",
  "poster": "http://slurm.trakt.us/images/posters/1395-300.jpg",
  "episodes": [
    {
      "number": 10,
      "season": 4,
      "title": "The Children",
      "overview": "Circumstances change after an unexpected arrival from north of the Wall. Daenerys must face harsh realities. Bran learns more about his destiny. Tyrion sees the truth about his situation.",
      "screen": "http://slurm.trakt.us/images/episodes/1395-4-10.80.jpg"
    }
  ]
}

Suspected Javascript Async issue with for loop

I have a pretty simple node script which parses a json file from an external url. I'm attempting to have the script loop over each record returned and make a decision to add it to the DB (using nedb) if we haven't previously got it. At the moment my script below looks to be only processing the last record in the json file.

var Datastore = require('nedb')
  , db = new Datastore({ filename: 'foo.db', autoload: true });


var request = require('request');
;

var cgrecentsalesurl = "http://ffo.com.json";

request({
    url: cgrecentsalesurl,
    headers: {
        'User-Agent': 'cgapistats'
    },
    json: true
}, function (error, response, body) {

    if (!error && response.statusCode === 200) {

        var cgrecentsales = body["recent-sales"];

        for (var i in cgrecentsales) {
            console.log( "processing record " + i );
            (function() {
            var query = { saletimestamp : cgrecentsales[i].sold_at };
            db.find( query , function( err, docs ) {
                 console.log( docs.length );
                 if ( docs.length == 0 ) {
                    db.insert( { item: cgrecentsales[i].item, saletimestamp: cgrecentsales[i].sold_at, amount: cgrecentsales[i].amount } );
                    console.log( "db record inserted" );
                 } else {
                    console.log( "record exists!" )
                 };
            });
            })();
        }
    } else {
        console.log(response);
    }
})

Any ideas what I'm doing wrong? Thanks!

Creating an array of tables with an asyncronous call

I'm having a fairly complicated problem and I was wondering if any of you could point me in the right direction.

I'm trying to draw a few bar graphs using Google Charts and NeDB, within AngularJS. I have a controller which retrieves the necessary data from a backend. This data is used in multiple sub-pages, for each of which I am writing controllers. The following controller controls the page with the bar graphs:

controlmodule.controller('StatisticsController', ['$scope', function($scope){
  $scope.dataTables = [];
  function createDataTables(){
    if(!($scope.dataLoaded[0] && $scope.dataLoaded[1] && $scope.dataLoaded[2])){
      return;
    }

    for(var question in $scope.questions){
      var dataTable = new google.visualization.DataTable();
      $scope.dataTables.push(dataTable);
      dataTable.addColumn('string', 'ID');
      dataTable.addColumn('number', 'Survey');
      var answers = $scope.answers[question.toString()];
      for(var answer in answers){
        for(var id in answers[answer]){
          var query = {};
          query[question] = +id;
          $scope.responses.find(query, (function(answer, index){
            return function(err, docs){
              $scope.dataTables[index].addRow([
                answer,
                docs.length 
              ]);
            }
          }(answers[answer][id.toString()], $scope.dataTables.length-1)));
        }
      }
    }
  }
  $scope.$watch('dataLoaded', createDataTables, true);
  createDataTables();
}]);

$scope.responses is a NeDB database, which has a find function to do queries. The problems I'm experiencing however is that this call is asynchronous. This makes everything very difficult, especially because of the nested for loop. I have a watch directive on the dataTables variable, but this does not get executed because of this asynchronicity. A solution is to call $scope.$digest() everytime I add a new row to a datatable, but this is killing for performance.

Ideally, what I think I'd like to do, is to call $scope.$digest ONCE, when the for-loop is done and all the find-calls have returned. Is there a clever way to do this?

Here's a sample of what the data looks like:

$scope.responses:

[
   {
      "144":833,
      "147":872,
      "survey_id":"39",
      "id":894,
      "150":893,
      "submission_date":"05/13/14",
      "148":882,
      "141":852,
      "149":887
   },
   {
      "144":860,
      "147":872,
      "survey_id":"39",
      "id":895,
      "150":892,
      "submission_date":"05/13/14",
      "148":882,
      "141":851,
      "149":884
   }
]

$scope.questions:

{
   "144":"What is your highest (expected) educational degree?",
   "147":"What is your age?",
   "148":"What is your gender",
   "149":"What do you do in daily life?",
   "150":"Where do you work or study?",
   "141":"What is your field of study?"
}

$scope.answers:

{
   "144":[
      {
         "830":"None"
      },
      {
         "831":"PhD"
      },
      {
         "832":"MBA"
      },
      {
         "833":"Master (non-MBA)"
      },
      {
         "860":"WO Bachelor"
      },
      {
         "861":"HBO Master"
      },
      {
         "862":"HBO Bachelor"
      },
      {
         "864":"Higher Vocational Education"
      },
      {
         "865":"High School"
      }
   ],
   "147":[
      {
         "871":"20 or younger"
      },
      {
         "875":"33 - 36"
      },
      {
         "879":"51 - 60"
      },
      {
         "872":"21 - 24"
      },
      {
         "876":"37 - 40"
      },
      {
         "880":"61 or older"
      },
      {
         "873":"25 - 28"
      },
      {
         "877":"41 - 45"
      },
      {
         "874":"29 - 32"
      },
      {
         "878":"46 - 50"
      }
   ]
}