The Web map stack on Django

Paul Smithhttp://www.pauladamsmith.com/ @paulsmith

EveryBlockEuroDjangoCon ‘09

Data types

11 metros

● Boston

● Charlotte

● Chicago

● Los Angeles

● Miami

● New York

● Philadelphia

● San Francisco

● San Jose

● Seattle

● Washington, DC

… and growing

Open sourceThis summer

Why?

Control design

Prioritize visualizations

The Web map stack

The Web map stack

The Web map stack

The Web map stack

The Web map stack

GeoDjango + Mapnikexample app

“Your Political Footprint”

Mapnik overview

# models.py

from django.contrib.gis.db import models

class CongressionalDistrict(models.Model): state = models.ForeignKey(State) name = models.CharField(max_length=32) # ex. 1st, 25th, at-large number = models.IntegerField() # 0 if at-large district = models.MultiPolygonField(srid=4326) objects = models.GeoManager()

def __unicode__(self): return '%s %s' % (self.state.name, self.name)

class Footprint(models.Model): location = models.CharField(max_length=200) point = models.PointField(srid=4326) cong_dist = models.ForeignKey(CongressionalDistrict) objects = models.GeoManager()

def __unicode__(self): return '%s in %s' % (self.location, self.cong_dist)

GET /tile/?bbox=-112.5,22.5,-90,45

# urls.py

from django.conf import settingsfrom django.conf.urls.defaults import *from edc_demo.footprint import views

urlpatterns = patterns('', (r'^footprint/', views.political_footprint), (r'^tile/', views.map_tile))

# views.py

from mapnik import *from django.http import HttpResponse, Http404from django.conf import settingsfrom edc_demo.footprint.models import CongressionalDistrict

TILE_WIDTH = TILE_HEIGHT = 256TILE_MIMETYPE = 'image/png'LIGHT_GREY = '#C0CCC4'PGIS_DB_CONN = dict( host=settings.DATABASE_HOST, dbname=settings.DATABASE_NAME, user=settings.DATABASE_USER, password=settings.DATABASE_PASSWORD)

def map_tile(request): if request.GET.has_key('bbox'): bbox = [float(x) for x in request.GET['bbox'].split(',')] tile = Map(TILE_WIDTH, TILE_HEIGHT) rule = Rule() rule.symbols.append(LineSymbolizer(Color(LIGHT_GREY), 1.0)) style = Style() style.rules.append(rule) tile.append_style('cong_dist', style) layer = Layer('cong_dists') db_table = CongressionalDistrict._meta.db_table layer.datasource = PostGIS(table=db_table, **PGIS_DB_CONN) layer.styles.append('cong_dist') tile.layers.append(layer) tile.zoom_to_box(Envelope(*bbox)) img = Image(tile.width, tile.height) render(tile, img) img_bytes = img.tostring(TILE_MIMETYPE.split('/')[1]) return HttpResponse(img_bytes, mimetype=TILE_MIMETYPE) else: raise Http404()

# views.py cont'd

from django.shortcuts import render_to_responsefrom edc_demo.footprint.geocoder import geocode

def political_footprint(request): context = {} if request.GET.has_key('location'): point = geocode(request.GET['location']) cd = CongressionalDistrict.objects.get(district__contains=point) footprint = Footprint.objects.create( location = request.GET['location'], point = point, cong_dist = cd ) context['footprint'] = footprint context['cd_bbox'] = cong_dist.district.extent return render_to_response('footprint.html', context)

// footprint.html<script type="text/javascript">var map;var TileLayerClass = OpenLayers.Class(OpenLayers.Layer.TMS, { initialize: function(footprint_id) {

var name = "tiles";var url = "http://127.0.0.1:8000/tile/";

var args = [];args.push(name, url, {}, {});

OpenLayers.Layer.Grid.prototype.initialize.apply(this, args);this.footprint_id = footprint_id;

},

getURL: function(bounds) {var url = this.url + "?bbox=" + bounds.toBBOX();if (this.footprint_id)

url += "&fp_id=" + this.footprint_id; return url; }});function onload() { var options = {

minScale: 19660800,numZoomLevels: 14,units: "degrees"

}; map = new OpenLayers.Map("map"); {% if not footprint %}

var bbox = new OpenLayers.Bounds(-126.298828, 17.578125, -64.775391, 57.128906); var tileLayer = new TileLayerClass(); {% else %} var bbox = new OpenLayers.Bounds({{ cd_bbox|join:", " }}); var tileLayer = new TileLayerClass({{ footprint.id }}); {% endif %} map.addLayer(tileLayer); map.zoomToExtent(bbox);}

# views.py

from edc_demo.footprint.models import Footprint

def map_tile(request): if request.GET.has_key('bbox'): bbox = [float(x) for x in request.GET['bbox'].split(',')] tile = Map(TILE_WIDTH, TILE_HEIGHT) rule = Rule() rule.symbols.append(LineSymbolizer(Color(LIGHT_GREY), 1.0)) style = Style() style.rules.append(rule) if request.GET.has_key('fp_id'): footprint = Footprint.objects.get(pk=request.GET['fp_id']) rule = Rule() rule.symbols.append(LineSymbolizer(Color(GREEN), 1.0)) rule.symbols.append(PolygonSymbolizer(Color(LIGHT_GREEN))) rule.filter = Filter('[id] = ' + str(footprint.cong_dist.id)) style.rules.append(rule) tile.append_style('cong_dist', style) layer = Layer('cong_dists') db_table = CongressionalDistrict._meta.db_table layer.datasource = PostGIS(table=db_table, **PGIS_DB_CONN) layer.styles.append('cong_dist') tile.layers.append(layer) if request.GET.has_key('fp_id'): add_footprint_layer(tile, footprint) tile.zoom_to_box(Envelope(*bbox)) img = Image(tile.width, tile.height) render(tile, img) img_bytes = img.tostring(TILE_MIMETYPE.split('/')[1]) return HttpResponse(img_bytes, mimetype=TILE_MIMETYPE) else: raise Http404()

# views.py cont'd

def add_footprint_layer(tile, footprint): rule = Rule() rule.symbols.append(

PointSymbolizer(os.path.join(settings.STATIC_MEDIA_DIR, 'img', 'footprint.png'),'png', 46, 46)

) rule.filter = Filter('[id] = ' + str(footprint.id)) style = Style() style.rules.append(rule) tile.append_style('footprint', style) layer = Layer('footprint') layer.datasource = PostGIS(table=Footprint._meta.db_table, **PGIS_DB_CONN) layer.styles.append('footprint') tile.layers.append(layer)

Serving tiles

Zoom levels

Tile example

z: 5, x: 2384, y: 1352

TileCache

pro

● Cache population integrated with request/response cycle

● Flexible storage

con

● Python overhead (rendering, serving)

Pre-render + custom nginx mod

pro

● Fast responses

● Parallelizable, offline rendering

con

● Render everything in advance

● C module inflexibility (esp. storage backends)

Tile rendering

for each zoom level z:

for each column x:

for each row y:

render tile (x, y, z)

Tile rendering

for each zoom level z:

for each column x:

for each row y:

render tile (x, y, z)

Tile rendering

for each zoom level z:

for each column x:

for each row y:

render tile (x, y, z)

Tile rendering

for each zoom level z:

for each column x:

for each row y:

render tile (x, y, z)

# nginx.conf

server { server_name tile.example.com root /var/www/maptiles; expires max; location ~* ^/[^/]+/\w+/\d+/\d+,\d+\.(jpg|gif|png)$ { tilecache; }}

// ngx_tilecache_mod.c

/* * This struct holds the attributes that uniquely identify a map tile. */typedef struct { u_char *version; u_char *name; int x; int y; int z; u_char *ext;} tilecache_tile_t;

/* * The following regex pattern matches the request URI for a tile and * creates capture groups for the tile attributes. Example request URI: * * /1.0/main/8/654,23.png * * would map to the following attributes: * * version: 1.0 * name: main * z: 8 * x: 654 * y: 23 * extension: png */static ngx_str_t tile_request_pat = ngx_string("^/([^/]+)/([^/]+)/([0-9]+)/([0-9]+),([0-9]+)\\.([a-z]+)$");

// ngx_tilecache_mod.c

u_char *get_disk_key(u_char *s, u_char *name, int x, int y, int z, u_char *ext){ u_int a, b, c, d, e, f;

a = x / 100000; b = (x / 1000) % 1000; c = x % 1000; d = y / 100000; e = (y / 1000) % 1000; f = y % 1000;

return ngx_sprintf(s, "/%s/%02d/%03d/%03d/%03d/%03d/%03d/%03d.%s", name, z, a, b, c, d, e, f, ext);}

static ngx_int_tngx_tilecache_handler(ngx_http_request_t *r){ // ... snip ... sub_uri.data = ngx_pcalloc(r->pool, len + 1); if (sub_uri.data == NULL) { return NGX_ERROR; }

get_disk_key(sub_uri.data, tile->name, tile->x, tile->y, tile->z, tile->ext); sub_uri.len = ngx_strlen(sub_uri.data);

return ngx_http_internal_redirect(r, &sub_uri, &r->args);}

Custom tile cache

technique

● Far-future expiry header expires max;

responsibility

● Tile versions for cache invalidation

// everyblock.js

eb.TileLayer = OpenLayers.Class(OpenLayers.Layer.TMS, { version: null, // see eb.TILE_VERSION layername: null, // lower-cased: "main", "locator" type: null, // i.e., mime-type extension: "png", "jpg", "gif"

initialize: function(name, url, options) { var args = []; args.push(name, url, {}, options); OpenLayers.Layer.TMS.prototype.initialize.apply(this, args); },

// Returns an object with the x, y, and z of a tile for a given bounds getCoordinate: function(bounds) { bounds = this.adjustBounds(bounds);

var res = this.map.getResolution();var x = Math.round((bounds.left - this.tileOrigin.lon) / (res * this.tileSize.w));var y = Math.round((bounds.bottom - this.tileOrigin.lat) / (res * this.tileSize.h));var z = this.map.getZoom();return {x: x, y: y, z: z};

},

getPath: function(x, y, z) { return this.version + "/" + this.layername + "/" + z + "/" + x + "," + y + "." +

this.type; },

getURL: function(bounds) {var coord = this.getCoordinate(bounds);

var path = this.getPath(coord.x, coord.y, coord.z); var url = this.url; if (url instanceof Array) url = this.selectUrl(path, url); return url + path; },

CLASS_NAME: "eb.TileLayer"});

Clustering

# cluster.py

import mathfrom everyblock.maps.clustering.models import Bunch

def euclidean_distance(a, b): return math.hypot(a[0] - b[0], a[1] - b[1])

def buffer_cluster(objects, radius, dist_fn=euclidean_distance): bunches = [] buffer_ = radius for key, point in objects.iteritems(): bunched = False for bunch in bunches: if dist_fn(point, bunch.center) <= buffer_: bunch.add_obj(key, point) bunched = True break if not bunched: bunches.append(Bunch(key, point)) return bunches

# bunch.py

class Bunch(object): def __init__(self, obj, point): self.objects = [] self.points = [] self.center = (0, 0) self.add_obj(obj, point)

def add_obj(self, obj, point): self.objects.append(obj) self.points.append(point) self.update_center(point)

def update_center(self, point): xs = [p[0] for p in self.points] ys = [p[1] for p in self.points] self.center = (sum(xs) * 1.0 / len(self.objects), sum(ys) * 1.0 / len(self.objects))

# cluster_scale.py

from everyblock.maps import utilsfrom everyblock.maps.clustering import cluster

def cluster_by_scale(objs, radius, scale, extent=(-180, -90, 180, 90)): resolution = utils.get_resolution(scale) # Translate from lng/lat into coordinate system of the display. objs = dict([(k, utils.px_from_lnglat(v, resolution, extent)) for k, v in objs.iteritems()]) bunches = [] for bunch in cluster.buffer_cluster(objs, radius): # Translate back into lng/lat. bunch.center = utils.lnglat_from_px(bunch.center, resolution, extent) bunches.append(bunch) return bunches

Sneak peek

Sneak peek

Sneak peek

Thank you

http://www.pauladamsmith.com/@paulsmith

paulsmith@gmail.com

Further exploration:“How to Lie with Maps”

Mark Monmonier