node.js - Looping mysql queries in nodejs -

node.js - Looping mysql queries in nodejs -

i scraping tables on several webpages , updating mysql database items in tables. using mysql module , running sorts of errors when looping through rows. here 1 iteration of function:

var connection = mysql.createconnection(opts); connection.connect(); function updatedb(o,lang){ var tbl_name = "news_"+lang; o.foreach(function(entry,index,arr){ var sql1 = "select * "+tbl_name+" url = '"+htmlencode.htmlencode(entry.url)+"' , omit = '1'"; connection.query(sql1, function(err, rows, fields) { connection.end(); if (err){ console.log("can't run query=" + sql1 +"\n error="+err); } else{ console.log('rows:',rows); // if rows.length = 0 update table new info here... } }); }); };

this works if comment out connection.end() line connection never ends , process stays live forever. running batch script need end gracefully. connection.end() in place getting "cannot enqueue quit after invoking quit."

i tried putting connection.end in final callback using async.series still not work. i've tried using pools , different iterations of code seek work. i'm @ wits end using mysql module , wondering if there 1 work better.

looks have 1 connection , you're using .foreach not asynchronous friendly.

you should utilize pool of connections , async.eachlimit command concurrency.

var pool = mysql.createpool({ host: process.env.mysql_host, user: process.env.mysql_user, password: process.env.mysql_pass, database: process.env.mysql_name, connectionlimit: 10 }); function updatedb(o,lang){ // submit sub-function `lang` closure // organized different way, fits construction function submit(entry, callback) { var tbl_name = "news_"+lang; var sql = [ 'select * ' + tbl_name, ' url="?" , omit="1"' ].join(''); pool.getconnection(function(err, connection) { if(err) homecoming callback(err); connection.query(sql, [entry.url], function(err, results) { connection.release(); if(err) homecoming callback(err); console.log('rows:',results); }); }); } async.eachlimit(o, 10, submit, function(err) { if(err) throw err; console.log('done processing'); }); }

you should utilize pool, if don't need more 1 connection. pool recreate connections needed. handy connection can die mysql timeouts, network issues , number of other reasons. when utilize pool, code more robust.

if want 1 query @ time, alter sec parameter async.eachlimit 1.

mysql node.js