diff --git a/examples/bi.html b/examples/bi.html new file mode 100644 index 00000000..ff7bba55 --- /dev/null +++ b/examples/bi.html @@ -0,0 +1,88 @@ + + + + + +
+
+ + + + + + + + + diff --git a/lib/torque/leaflet/torque.js b/lib/torque/leaflet/torque.js index 1c267194..b2f2ad98 100644 --- a/lib/torque/leaflet/torque.js +++ b/lib/torque/leaflet/torque.js @@ -10,6 +10,7 @@ L.TorqueLayer = L.CanvasLayer.extend({ providers: { 'sql_api': torque.providers.json, + 'filterable_sql_api': torque.providers.filterableJson, 'url_template': torque.providers.JsonArray, 'windshaft': torque.providers.windshaft, 'tileJSON': torque.providers.tileJSON @@ -28,10 +29,16 @@ L.TorqueLayer = L.CanvasLayer.extend({ options.tileLoader = true; this.key = 0; this.prevRenderedKey = 0; + this._filters = {} + if (options.cartocss) { torque.extend(options, torque.common.TorqueLayer.optionsFromCartoCSS(options.cartocss)); } + + options.columns = options.countby.split(";") + + options.resolution = options.resolution || 2; options.steps = options.steps || 100; options.visible = options.visible === undefined ? true: options.visible; @@ -98,11 +105,48 @@ L.TorqueLayer = L.CanvasLayer.extend({ self.redraw(); } self.fire('tileLoaded'); + self.fire('dataUpdate') }); }, this); }, + setFilters: function() { + this.provider.setFilters(this._filters); + this._reloadTiles(); + return this; + }, + + filterByRange: function(variableName, start, end) { + this._filters[variableName] = {type: 'range', range: {start: start, end: end} } + this._filtersChanged() + this.fire('dataUpdate') + return this + }, + + filterByCat: function(variableName, categories) { + this._filters[variableName] = {type: 'cat', categories: categories} + this._filtersChanged() + return this + }, + + clearFilter: function(name){ + if(name) { + delete this._filters[name] + } + else { + this._filters = {} + } + this._filtersChanged() + return this + }, + + _filtersChanged:function(){ + this.provider._filters = this._filters; + this._clearTileCaches() + this._render() + }, + _clearTileCaches: function() { var t, tile; for(t in this._tiles) { @@ -118,6 +162,96 @@ L.TorqueLayer = L.CanvasLayer.extend({ this._clearTileCaches(); }, + valuesForRangeVariable:function(variable){ + var t, tile; + + var variable_id = this.provider.idForRange(variable) + var values = [ ] + + + for(t in this._tiles){ + tile = this._tiles[t] + var noPoints = tile.x.length; + for(var i=0; i < tile.x.length; i++){ + if(tile.renderFlags[i] ){ + value = tile.renderData[variable_id*noPoints + i] + values.push(value) + } + } + } + return values; + }, + + valuesForCatVariable:function(variable){ + var t, tile; + + var categories = this.provider.idsForCategory(variable) + var result = [ ] + + + for(t in this._tiles){ + tile = this._tiles[t] + var noPoints = tile.x.length; + for(var i=0; i < tile.x.length; i++){ + if(tile.renderFlags[i] ){ + var vals={} + Object.keys(categories).forEach(function(categoryName){ + var variable_id = categories[categoryName] + value = tile.renderData[variable_id*noPoints + i] + vals[categoryName] = value + }.bind(this)) + result.push(vals) + } + } + } + return result; + + }, + getValues:function(variable,callback){ + var type= this.provider._mapping[variable].type + if(type=='float'){ + callback(this.valuesForRangeVariable(variable)) + } + else{ + callback(this.valuesForCatVariable(variable)) + } + }, + getHistogram:function(variable,callback,noBins){ + var type= this.provider._mapping[variable].type + if(type=='float'){ + callback(this.histogramForRangeVariable(variable,noBins)) + } + else if(type='cat'){ + callback(this.histogramForCatVariable(variable)) + } + return this + }, + histogramForCatVariable:function(variable){ + var result = {} + this.valuesForCatVariable(variable).forEach(function(point){ + Object.keys(point).forEach(function(key){ + result[key] = result[key] || 0 + result[key] += point[key] + }) + }) + return result + }, + histogramForRangeVariable:function(variable,noBins){ + noBins = noBins || 10 + + var vals = this.valuesForRangeVariable(variable) + + var min = Math.min.apply(null, vals) + var max = Math.max.apply(null, vals) + var binSize = (max-min)/noBins + var result = [] + vals.forEach(function(val){ + var bin = (val -min)/binSize + result[bin]= result[bin] || 0 + result[bin] += val + }) + return result + }, onAdd: function (map) { map.on({ 'zoomend': this._clearCaches, @@ -208,6 +342,40 @@ L.TorqueLayer = L.CanvasLayer.extend({ canvas.width = canvas.width; }, + /* + _filterTile:function(tile){ + var noPoints = tile.x.length + + var renderFlags = [] + for(var i =0; i < noPoints; i++){ + var includePoint = true + Object.keys(this._filters).forEach(function(key){ + var filter = this._filters[key] + var variableId = this.provider.idForRange(key) + + var value = tile.renderData[variableId*noPoints+i] + if(filter.type=='range'){ + if(value < filter.range.start || value > filter.range.end){ + includePoint = false; + } + } + else if (filter.type=='cat'){ + var ids = this.provider.idsForCategory(key); + filter.categories.forEach(function(key){ + var catId = ids[key] + var value = tile.renderData[catId*noPoints + i] + if(value==0){ + includePoint= false + } + }.bind(this)) + } + }.bind(this)) + renderFlags[i] = includePoint + } + return renderFlags + }, + */ + /** * render the selectef key * don't call this function directly, it's called by @@ -236,10 +404,13 @@ L.TorqueLayer = L.CanvasLayer.extend({ // all the points this.renderer._ctx.drawImage(tile._tileCache, 0, 0); } else { + + //tile.renderFlags = this._filterTile(tile) this.renderer.renderTile(tile, this.key); } } } + this.renderer.applyFilters(); // prepare caches if the animation is not running diff --git a/lib/torque/provider/filterableJson.js b/lib/torque/provider/filterableJson.js new file mode 100644 index 00000000..ad2ce24f --- /dev/null +++ b/lib/torque/provider/filterableJson.js @@ -0,0 +1,521 @@ +var torque = require('../'); +var Profiler = require('../profiler'); + + var Uint8Array = torque.types.Uint8Array; + var Int32Array = torque.types.Int32Array; + var Uint32Array = torque.types.Uint32Array; + + // format('hello, {0}', 'rambo') -> "hello, rambo" + function format(str) { + for(var i = 1; i < arguments.length; ++i) { + var attrs = arguments[i]; + for(var attr in attrs) { + str = str.replace(RegExp('\\{' + attr + '\\}', 'g'), attrs[attr]); + } + } + return str; + } + + var filterableJson = function (options) { + this._ready = false; + this._tileQueue = []; + this.options = options; + this._filters = {}; + this._mapping = { + no:{ + type: 'float', + col_id:6 + }, + passenger_count:{ + type: 'float', + col_id: 0 + }, + tip_amount:{ + type:'float', + col_id:1 + }, + payment_type:{ + type:'cat', + col_ids:{ + "Credit card": 2, + "Cash": 3, + "No charge": 4, + "Dispute": 5 + } + }, + }; + + this.options.is_time = this.options.is_time === undefined ? true: this.options.is_time; + this.options.tiler_protocol = options.tiler_protocol || 'http'; + this.options.tiler_domain = options.tiler_domain || 'cartodb.com'; + this.options.tiler_port = options.tiler_port || 80; + + if (this.options.data_aggregation) { + this.options.cumulative = this.options.data_aggregation === 'cumulative'; + } + + // check options + if (options.resolution === undefined ) throw new Error("resolution should be provided"); + if (options.steps === undefined ) throw new Error("steps should be provided"); + if(options.start === undefined) { + this._fetchKeySpan(); + } else { + this._setReady(true); + } + }; + + + + filterableJson.prototype = { + + /** + * return the torque tile encoded in an efficient javascript + * structure: + * { + * x:Uint8Array x coordinates in tile reference system, normally from 0-255 + * y:Uint8Array y coordinates in tile reference system + * Index: Array index to the properties + * } + */ + proccessTile: function(rows, coord, zoom) { + var r; + var x = new Uint8Array(rows.length); + var y = new Uint8Array(rows.length); + + var prof_mem = Profiler.metric('ProviderJSON:mem'); + var prof_point_count = Profiler.metric('ProviderJSON:point_count'); + var prof_process_time = Profiler.metric('ProviderJSON:process_time').start() + + // count number of steps + var steps = 0; + var maxDateSlots = -1; + for (r = 0; r < rows.length; ++r) { + var row = rows[r]; + dates += row.steps.length; + for(var d = 0; d < row.steps.length; ++d) { + maxDateSlots = Math.max(maxDateSlots, row.steps[d]); + } + } + + if(this.options.cumulative) { + steps = (1 + maxDateSlots) * rows.length; + } + + + + // var type = Uint8Array; + + + // reserve memory for all the dates + // var timeIndex = new Int32Array(maxDateSlots + 1); //index-size + // var timeCount = new Int32Array(maxDateSlots + 1); + var renderData = new Array() + // var renderDataPos = new Uint32Array(dates); + + prof_mem.inc( + 4 * maxDateSlots + // timeIndex + 4 * maxDateSlots + // timeCount + steps + //renderData + steps * 4 + ); //renderDataPos + + prof_point_count.inc(rows.length); + + var rowsPerSlot = {}; + + // precache pixel positions + for (var r = 0; r < rows.length; ++r) { + var row = rows[r]; + x[r] = row.x //* this.options.resolution; + // fix value when it's in the tile EDGE + // TODO: this should be fixed in SQL query + if (row.y === -1) { + y[r] = 0; + } else { + y[r] = row.y //* this.options.resolution; + } + + for(var i=0; i < row.steps.length; i++){ + renderData[r*row.steps.length + i] = row.vals[i] + } + /*var lastDateSlot = dates[dates.length - 1]; + + y[r] = row.y * this.options.resolution; + } + + var steps = row.steps; + var vals = row.vals; + if (!this.options.cumulative) { + for (var j = 0, len = steps.length; j < len; ++j) { + var rr = rowsPerSlot[steps[j]] || (rowsPerSlot[steps[j]] = []); + if(this.options.cumulative) { + vals[j] += prev_val; + } + prev_val = vals[j]; + rr.push([r, vals[j]]); + } + } else { + var valByDate = {} + for (var j = 0, len = steps.length; j < len; ++j) { + valByDate[steps[j]] = vals[j]; + } + var accum = 0; + + // extend the latest to the end + for (var j = steps[0]; j <= maxDateSlots; ++j) { + var rr = rowsPerSlot[j] || (rowsPerSlot[j] = []); + var v = valByDate[j]; + if (v) { + accum += v; + } + rr.push([r, accum]); + } + + /*var lastDateSlot = steps[steps.length - 1]; +>>>>>>> 4ad0dba547e2a57300064c484d42e196eb846356 + for (var j = lastDateSlot + 1; j <= maxDateSlots; ++j) { + var rr = rowsPerSlot[j] || (rowsPerSlot[j] = []); + rr.push([r, prev_val]); + } + */ + + } + + // for each timeslot search active buckets + + + prof_process_time.end(); + + return { + x: x, + y: y, + z: zoom, + coord: { + x: coord.x, + y: coord.y, + z: zoom + }, + // timeCount: timeCount, + // timeIndex: timeIndex, + // renderDataPos: renderDataPos, + renderData: renderData, + // maxDate: maxDateSlots + }; + }, + + _generateFilterSQLForCat:function(name,categories){ + return name+" in "+categories; + }, + + _generateFilterSQLForRange:function(name,range){ + var result = "" + if (range.start) { + result += " " + name + " > " + range.start; + } + if (range.end) { + if (range.start) { + result += " and " + } + result += " " + name + " < " + range.end; + } + return result + }, + + _setFilters:function(filters){ + this.filters = filters + }, + + _generateFiltersSQL: function() { + var self = this; + + return Object.keys(this._filters).map(function(filterName){ + var filter = self._filters[filterName] + if (filter) { + if (filter.type == 'range') { + return self._generateFilterSQLForRange(filterName, filter.range) + } + else if (filter.type == 'cat') { + return self._generateFilterSQLForCat(filterName, filter.categories) + } + else { + return "" + } + } + else{ + return "" + } + }).join(" and ") + }, + + _host: function() { + var opts = this.options; + var port = opts.sql_api_port; + var domain = ((opts.user_name || opts.user) + '.' + (opts.sql_api_domain || 'cartodb.com')) + (port ? ':' + port: ''); + var protocol = opts.sql_api_protocol || 'http'; + return this.options.url || protocol + '://' + domain + '/api/v2/sql'; + }, + + url: function(subhost) { + var opts = this.options; + var protocol = opts.sql_api_protocol || 'http'; + if (!this.options.cdn_url) { + return this._host(); + } + var h = protocol+ "://"; + if (subhost) { + h += subhost + "."; + } + var cdn_host = opts.cdn_url; + if(!cdn_host.http && !cdn_host.https) { + throw new Error("cdn_host should contain http and/or https entries"); + } + h += cdn_host[protocol] + "/" + (opts.user_name || opts.user) + '/api/v2/sql'; + return h; + }, + + _hash: function(str) { + var hash = 0; + if (!str || str.length == 0) return hash; + for (var i = 0, l = str.length; i < l; ++i) { + hash = (( (hash << 5 ) - hash ) + str.charCodeAt(i)) | 0; + } + return hash; + }, + + _extraParams: function() { + if (this.options.extra_params) { + var p = []; + for(var k in this.options.extra_params) { + var v = this.options.extra_params[k]; + if (v) { + p.push(k + "=" + encodeURIComponent(v)); + } + } + return p.join('&'); + } + return null; + }, + + isHttps: function() { + return this.options.sql_api_protocol && this.options.sql_api_protocol === 'https'; + }, + + // execute actual query + sql: function(sql, callback, options) { + options = options || {}; + var subdomains = this.options.subdomains || '0123'; + if(this.isHttps()) { + subdomains = [null]; // no subdomain + } + + + var url; + if (options.no_cdn) { + url = this._host(); + } else { + url = this.url(subdomains[Math.abs(this._hash(sql))%subdomains.length]); + } + var extra = this._extraParams(); + torque.net.get( url + "?q=" + encodeURIComponent(sql) + (extra ? "&" + extra: ''), function (data) { + if(options.parseJSON) { + data = JSON.parse(data && data.responseText); + } + callback && callback(data); + }); + }, + + getTileData: function(coord, zoom, callback) { + if(!this._ready) { + this._tileQueue.push([coord, zoom, callback]); + } else { + this._getTileData(coord, zoom, callback); + } + }, + + _setReady: function(ready) { + this._ready = true; + this._processQueue(); + this.options.ready && this.options.ready(); + }, + + _processQueue: function() { + var item; + while (item = this._tileQueue.pop()) { + this._getTileData.apply(this, item); + } + }, + + /** + * `coord` object like {x : tilex, y: tiley } + * `zoom` quadtree zoom level + */ + + _getTileData: function(coord, zoom, callback) { + var prof_fetch_time = Profiler.metric('ProviderJSON:tile_fetch_time').start() + this.table = this.options.table; + var numTiles = 1 << zoom; + + var column_conv = this.options.column; + // + // if(this.options.is_time) { + // column_conv = format("date_part('epoch', {column})", this.options); + // } + + // var sql = "" + + // "WITH " + + // "par AS (" + + // " SELECT CDB_XYZ_Resolution({zoom})*{resolution} as res" + + // ", 256/{resolution} as tile_size" + + // ", CDB_XYZ_Extent({x}, {y}, {zoom}) as ext " + + // ")," + + // "cte AS ( "+ + // " SELECT ST_SnapToGrid(i.the_geom_webmercator, p.res) g" + + // ", avg(passenger_count) c1, avg(tip_amount) c2, sum( case when payment_type=1 then 1 else 0 end) c3, sum(case when payment_type=2 then 1 else 0 end ) c4, sum(case when payment_type=3 then 1 else 0 end ) c5,sum(case when payment_type=4 then 1 else 0 end ) c6, count(cartodb_id) c7 " + + // " FROM ({_sql}) i, par p " + + // " WHERE i.the_geom_webmercator && p.ext " + + // " GROUP BY g" + + // ") " + + // "" + + // "SELECT (st_x(g)-st_xmin(p.ext))/p.res x__uint8, " + + // " (st_y(g)-st_ymin(p.ext))/p.res y__uint8," + + // " Array[c1,c2,c3,c4,c5,c6,c7] vals__uint8," + + // " Array[0,1,2,3,4,5,6] dates__uint16" + + // // the tile_size where are needed because the overlaps query in cte subquery includes the points + // // in the left and bottom borders of the tile + // " FROM cte, par p where (st_y(g)-st_ymin(p.ext))/p.res < tile_size and (st_x(g)-st_xmin(p.ext))/p.res < tile_size "; + this.options.filters = this._generateFiltersSQL() + var sql = ""+ + "SELECT * FROM torque_tile(x:={x}, y:={y}, z:={zoom}, aggr:=ARRAY['{columns}'], table_name:='{table}', where_clause:='{filters}');" + + + var query = format(sql, this.options, { + zoom: zoom, + x: coord.x, + y: coord.y, + column: column_conv, + table: this.options.table, + filters: this._generateFiltersSQL() + }); + + + var self = this; + console.log("query is ", query) + this.sql(query, function (data) { + + if (data) { + var rows = JSON.parse(data.responseText).rows; + callback(self.proccessTile(rows, coord, zoom)); + } else { + callback(null); + } + prof_fetch_time.end(); + }); + }, + + getKeySpan: function() { + return { + start: this.options.start * 1000, + end: this.options.end * 1000, + step: this.options.step, + steps: this.options.steps, + columnType: this.options.is_time ? 'date': 'number' + }; + }, + + setColumn: function(column, isTime) { + this.options.column = column; + this.options.is_time = isTime === undefined ? true: false; + this.reload(); + }, + + setResolution: function(res) { + this.options.resolution = res; + }, + + // return true if tiles has been changed + setOptions: function(opt) { + var refresh = false; + + if(opt.resolution !== undefined && opt.resolution !== this.options.resolution) { + this.options.resolution = opt.resolution; + refresh = true; + } + + if(opt.steps !== undefined && opt.steps !== this.options.steps) { + this.setSteps(opt.steps, { silent: true }); + refresh = true; + } + + if(opt.column !== undefined && opt.column !== this.options.column) { + this.options.column = opt.column; + refresh = true; + } + + if(opt.countby !== undefined && opt.countby !== this.options.countby) { + this.options.countby = opt.countby; + refresh = true; + } + + if(opt.data_aggregation !== undefined) { + var c = opt.data_aggregation === 'cumulative'; + if (this.options.cumulative !== c) { + this.options.cumulative = c; + refresh = true; + } + } + + if (refresh) this.reload(); + return refresh; + + }, + + reload: function() { + this._ready = false; + this._fetchKeySpan(); + }, + + setSQL: function(sql) { + if (this.options.sql != sql) { + this.options.sql = sql; + this.reload(); + } + }, + + getSteps: function() { + return Math.min(this.options.steps, this.options.data_steps); + }, + + setSteps: function(steps, opt) { + opt = opt || {}; + if (this.options.steps !== steps) { + this.options.steps = steps; + this.options.step = (this.options.end - this.options.start)/this.getSteps(); + this.options.step = this.options.step || 1; + if (!opt.silent) this.reload(); + } + }, + + getBounds: function() { + return this.options.bounds; + }, + + getSQL: function() { + return this.options.sql || "select * from " + this.options.table; + }, + + _tilerHost: function() { + var opts = this.options; + var user = (opts.user_name || opts.user); + return opts.tiler_protocol + + "://" + (user ? user + "." : "") + + opts.tiler_domain + + ((opts.tiler_port != "") ? (":" + opts.tiler_port) : ""); + }, + + _fetchKeySpan: function() { + this._setReady(true); + } + + }; + +module.exports = filterableJson; diff --git a/lib/torque/provider/index.js b/lib/torque/provider/index.js index 7073e03a..43c2848e 100644 --- a/lib/torque/provider/index.js +++ b/lib/torque/provider/index.js @@ -1,5 +1,6 @@ module.exports = { json: require('./json'), + filterableJson: require('./filterableJson'), JsonArray: require('./jsonarray'), windshaft: require('./windshaft'), tileJSON: require('./tilejson') diff --git a/lib/torque/renderer/point.js b/lib/torque/renderer/point.js index 4b07d149..26b42444 100644 --- a/lib/torque/renderer/point.js +++ b/lib/torque/renderer/point.js @@ -57,7 +57,7 @@ var Filters = require('./torque_filters'); this.TILE_SIZE = 256; this._style = null; this._gradients = {}; - + this._forcePoints = false; } @@ -165,14 +165,14 @@ var Filters = require('./torque_filters'); i.src = canvas.toDataURL(); return i; } - + return canvas; }, // // renders all the layers (and frames for each layer) from cartocss // - renderTile: function(tile, key, callback) { + renderTile: function(tile, key, renderFlags, callback) { if (this._iconsToLoad > 0) { this.on('allIconsLoaded', function() { this.renderTile.apply(this, [tile, key, callback]); @@ -193,7 +193,7 @@ var Filters = require('./torque_filters'); } } } - + prof.end(true); return callback && callback(null); @@ -237,12 +237,12 @@ var Filters = require('./torque_filters'); }, // - // renders a tile in the canvas for key defined in + // renders a tile in the canvas for key defined in // the torque tile // _renderTile: function(tile, key, frame_offset, sprites, shader, shaderVars) { - if (!this._canvas) return; + if (!this._canvas) return; var prof = Profiler.metric('torque.renderer.point.renderTile').start(); var ctx = this._ctx; var blendMode = compop2canvas(shader.eval('comp-op')) || this.options.blendmode; @@ -254,27 +254,29 @@ var Filters = require('./torque_filters'); key = tile.maxDate; } var tileMax = this.options.resolution * (this.TILE_SIZE/this.options.resolution - 1) - var activePixels = tile.timeCount[key]; + var activePixels = tile.x.length; var anchor = this.options.resolution/2; if (activePixels) { - var pixelIndex = tile.timeIndex[key]; + var pixelIndex = 0;//tile.timeIndex[key]; for(var p = 0; p < activePixels; ++p) { - var posIdx = tile.renderDataPos[pixelIndex + p]; - var c = tile.renderData[pixelIndex + p]; - if (c) { - var sp = sprites[c]; - if (sp === undefined) { - sp = sprites[c] = this.generateSprite(shader, c, torque.extend({ zoom: tile.z, 'frame-offset': frame_offset }, shaderVars)); - } - if (sp) { - var x = tile.x[posIdx]- (sp.width >> 1) + anchor; - var y = tileMax - tile.y[posIdx] + anchor; // flip mercator - ctx.drawImage(sp, x, y - (sp.height >> 1)); - } - } + if(tile.renderFlags[p]){ + var posIdx = p// tile.renderDataPos[pixelIndex + p]; + + var posIdx = tile.renderDataPos[pixelIndex + p]; + + var c = tile.renderData[pixelIndex + p]; + var sp = sprites[c]; + if (sp === undefined) { + sp = sprites[c] = this.generateSprite(shader, c, torque.extend({ zoom: tile.z, 'frame-offset': frame_offset }, shaderVars)); + } + if (sp) { + var x = tile.x[posIdx]- (sp.width >> 1) + anchor; + var y = tileMax - tile.y[posIdx] + anchor; // flip mercator + ctx.drawImage(sp, x, y - (sp.height >> 1)); + } } } - + prof.end(true); }, @@ -442,7 +444,7 @@ var Filters = require('./torque_filters'); } gradient = {}; var colorize = this._style['image-filters'].args; - + var increment = 1/colorize.length; for (var i = 0; i < colorize.length; i++){ var key = increment * i + increment;