python--memcached---activemq

1.memcachedをインストールするpythonクライアント
 wget ftp://ftp.tummy.com/pub/python-memcached/python-memcached-latest.tar.gz   python2.x setup.py install
2.activemqのpythonクライアントをインストールする
  wget http://stomppy.googlecode.com/files/stomp.py-3.0.3.tar.gz   python2.x setup.py install --prefix=/usr/local/similarlib/
stomp公式サイトのサポート:http://code.google.com/p/stomppy/
3.python-activemqのdemoプログラム
mq_send:

#!/usr/bin/python
#
# Usage: stomp_send.py <msize> <nmsgs>
#

import sys
import os
import logging
import stomp
import time
import json

logging.basicConfig()

nmsgs = int(sys.argv[1])
msg = {'pid':1,'id':1,'start_at':"time",'type':'parse_title','articleid':0, 'from':1, 'to':361}
dest = '/queue/worker'
start = time.time()

conn = stomp.Connection([('172.16.1.217', 61612)])
conn.start()
conn.connect(wait=True)

for i in range(nmsgs):
	msg['articleid'] = i
	conn.send(json.write(msg), destination=dest)
	print "send one"

conn.disconnect()
print "OK Finished msgs %d time %f" % (nmsgs, (time.time()-start))

mq_receive:

import sys
import os
import logging
import stomp
import json
import time

class MyListener(object):
	def on_error(self, headers, message):
		print 'received an error %s' % message

	def on_message(self, headers, message):
		print 'received a message %s' % message
#		print headers['message-id']
#		sys.exit(0)
		conn.ack({'message-id':headers['message-id']})

dest = '/queue/test1'
logging.basicConfig()

conn = stomp.Connection([('172.16.1.217', 61612)])
conn.set_listener('', MyListener())
conn.start()
conn.connect(wait=True)
conn.subscribe(destination=dest, ack='client')

while True:
	try:
		time.sleep(1)
	except:
		break

4.python-memcachedのdemoプログラム

import time,memcache,pickle,cPickle,sys #,msgpack

f = file("/tmp/ids", 'r')
mc = memcache.Client(["172.16.1.217:11211"])
#count = 0
#for i in range(1,10000):
#	if mc.get("content-%s"%str(i)) != None:
#		count += 1
#		f.write("%s
"%str(i))
#	if count >= 1000:
#		break
#sys.exit(0)
id_list = [int(x.strip()) for x in f]
print id_list
start = time.time()
results = mc.get_multi(id_list, key_prefix="content-")
#for k,v in results.iteritems():
#	results[k] = cPickle.loads(v)
#	print v 
#	time.sleep(1)
#mc.set_multi(results,key_prefix="content-")
print "eats %s s"%str(time.time()-start)
print len(results)

の下には、mqとmemcachedを同時に使用するdemoがあります.

#!/usr/bin/python
#
#

import sys
import os
import logging
import stomp
import time
import json,memcache

mc = memcache.Client(["172.16.1.217:11211"])

t = time.time()
tstart = time.strftime("%Y-%m-%d %H:%M:%S",time.localtime(t))
tstart += ".%03d" % ((t-int(t))*1000)

start = time.time()
maxaid = 10000
conn = stomp.Connection([('172.16.1.217', 61612)])
dest = '/queue/test_worker_2'
conn.start()
conn.connect(wait=True)

for item in ['title','content','tag','cate','baike']:
	for i in range(1,25666):
		if mc.get(item+"-"+str(i)) == None:
			task = {'id':str(time.time()),'pid':'','start_at':tstart,
				'type':"parse_"+item,'from':i,'to':i}
			conn.send(json.write(task), destination=dest)

conn.disconnect()