Managing AppEngine from IPython

When developing a Google Appengine application, it can be a pain to manipulate the data in it, as you don’t have a console where to write python code (like you have in the dev appserver). Fortunately, a remote api exists.

I wanted a bit more, though:

integration with IPython, to benefit from code completion and other goodies
ability to use my .netrc as a credentials provider

So here is an IPython profile that does that (in addition, it wraps the functionality of appcfg). Except for the bulkloader functionality, which is kind of ugly, the original code from appengine was generic enough to allow that kind of manipulation in a relatively clean way.

	import IPython.ipapi

	from getpass import getpass
	from netrc import netrc
	from optparse import OptionParser

	from google.appengine.ext.remote_api import remote_api_stub
	from google.appengine.tools.appcfg import AppCfgApp, StatusUpdate
	from google.appengine.tools.bulkloader import RequestManager

	ip = IPython.ipapi.get()

	class ConnectParser(OptionParser):

	def __init__(self):
	OptionParser.__init__(self, usage="usage: %connect [options] <app_id>")
	self.add_option("-d", "--development",
	action="store_true", dest="development", default=False,
	help="connect to a dev server")
	self.add_option("-a", "--address",
	dest="address", default=None,
	help="server:port to connect to")
	self.add_option("-u", "--username",
	dest="username", default=None,
	help="username to connect as")
	self.add_option("-p", "--password",
	dest="password", default=None,
	help="password")

	class AppCfgParser(OptionParser):

	def parse_args(self, args):
	options, args = OptionParser.parse_args(self, args)
	host = options.server
	u,_,p = netrc().authenticators(host) or (None, None, None)
	if u is not None:
	options.email = options.email or u
	options._known_password = p
	return options, args

	class MyAppCfgApp(AppCfgApp):

	def __init__(self, argv):
	AppCfgApp.__init__(self, argv, parser_class=AppCfgParser)
	if self.options._known_password is not None:
	self.password_input_fn = (lambda prompt: self.options._known_password)

	def patchBulkLoader():
	def AuthFunction(this):
	host = this.host
	u,_,p = netrc().authenticators(host) or (None, None, None)
	this.email = this.email or u
	def inputPassword(prompt):
	return p or getpass(prompt)
	return this.AuthFunction_orig(password_input_fn=inputPassword)

	RequestManager.AuthFunction_orig = RequestManager.AuthFunction
	RequestManager.AuthFunction = AuthFunction

	patchBulkLoader()
	connect_parser = ConnectParser()

	def _connect(self, arg):
	args = arg.split()
	try:
	options, args = connect_parser.parse_args(args)
	except SystemExit:
	return

	app_id = args[0]
	if options.address is not None:
	address = options.address
	else:
	if options.development:
	address = 'localhost:8080'
	else:
	address = '%s.appspot.com' % (app_id)

	if options.development:
	# no need for valid credentials with dev_appserver
	user, pwd = "foo", "bar"
	else:
	u,_,p = netrc().authenticators(address) or (None, None, None)
	user = options.username or u or raw_input('Username: ')
	pwd = options.password or p or getpass('Password: ')

	def auth_func():
	return user, pwd

	remote_api_stub.ConfigureRemoteDatastore(app_id, '/remote_api', auth_func, address)

	def _appcfg(self, arg):
	argv = ['%appcfg'] + arg.split()
	try:
	return MyAppCfgApp(argv).Run()
	except KeyboardInterrupt:
	StatusUpdate('Interrupted.')
	except SystemExit:
	return

	def defineHelpers():
	ip.expose_magic('connect', _connect)
	ip.expose_magic('appcfg', _appcfg)

	def main():
	o = ip.options
	o.system_verbose = 0

	o.banner = "GAE console v0.1"
	o.prompt_in1= r'\C_Green\|\#> '
	o.prompt_in2= r'\C_Green\|\C_LightGreen\D\C_Green> '
	o.prompt_out= '<\#> '

	o.confirm_exit = False

	try:
	ip.ex("import os, sys, datetime, google")

	defineHelpers()

	except Exception, e:
	print "Exception: %s" % str(e)

	main()

view raw ipy_profile_gae.py hosted with ❤ by GitHub

And this is what it allows:

	$ ipython -p gae
	GAE console v0.1
	[git/my-app]\|1> %connect -d my-app # validate first on development server

	[git/my-app]\|2> %run bootstrap.py # tweak the datastore

	[git/my-app]\|3> %connect my-app # now replicate on production server

	[git/my-app]\|4> %run bootstrap.py # tweak the same

	[git/my-app]\|5> %appcfg --filename /tmp/my-app.data download_data .
	Application: my-app; version: 1.
	Downloading data records.
	[INFO ] Logging to bulkloader-log-20100916.143830
	[INFO ] Throttling transfers:
	[INFO ] Bandwidth: 250000 bytes/second
	[INFO ] HTTP connections: 8/second
	[INFO ] Entities inserted/fetched/modified: 20/second
	[INFO ] Batch Size: 10
	[INFO ] Opening database: bulkloader-progress-20100916.143830.sql3
	[INFO ] Opening database: bulkloader-results-20100916.143830.sql3
	[INFO ] Connecting to my-app.appspot.com/remote_api
	[INFO ] Downloading kinds: [u'Foo', u'Bar', u'Baz']
	...
	[INFO ] Have 5 entities, 0 previously transferred
	[INFO ] 5 entities (6533 bytes) transferred in 3.9 seconds

	[git/my-app]\|6>

view raw ipython_session hosted with ❤ by GitHub

Just in case, my ~/.netrc looks like:

	machine my-app.appspot.com
	login me@example.com
	password xxxxxxxxxxx
	machine appengine.google.com
	login me@example.com
	password xxxxxxxxxxx

view raw netrc hosted with ❤ by GitHub

And of course, all those appengine libraries have to be somewhere in your python path. Here is the .pth file for my dedicated virtualenv

	/home/yann/apps/google_appengine
	/home/yann/apps/google_appengine/lib/yaml/lib
	/home/yann/apps/google_appengine/lib/fancy_urllib
	/home/yann/apps/google_appengine/lib/antlr3
	/home/yann/apps/google_appengine/lib/cacerts
	/home/yann/apps/google_appengine/lib/django
	/home/yann/apps/google_appengine/lib/ipaddr
	/home/yann/apps/google_appengine/lib/webob

view raw virtualenv_path_extensions.pth hosted with ❤ by GitHub