DJANGOによるGAEでの小型ネットワーク爬虫類の実現

Google App EngineとDjangoは、WSGI標準を使用してアプリケーションを実行できます.したがって、ミドルウェアを含むDjangoスタックのほぼ全体をGoogle App Engineで使用できます.開発者として、Djangoデータモデルを変更して、Google App EngineデータストレージAPIと高速で拡張性のあるGoogle App Engineデータストレージを組み合わせて使用する必要があります.DjangoはGoogle App Engineのモデルコンセプトと似ており、Django開発者としてGAEのデータストレージを使用するためにアプリケーションを迅速に調整することができます.
Google App Engineのデフォルトバージョンは0.96です.もちろん、1.0にアップグレードすることもできます.GAEを使うためにDJANGOのMODELはもちろん使えないので、GOOGLEにしますが、VIEWやURLは役に立ちます.以下は私のコードと説明の一部です.
app.yaml:

application: mydjango-test
version: 2
runtime: python
api_version: 1
handlers:
  - url: /.*
    script: bootstrap.py

注意、アプリケーションはあなたのGAEの名前と同じように、同期できないようにしなければなりません.
次はbootstrapですpyは、私たちは彼がどのように働いているのかを知る必要はありません.ウェブサイトに入ってから、URLを通じて、彼は自動的にDJANGOにマッピングされます.他のURLはここで配合する必要はありません.この时、私たちのDJANGOのURLSが威張っています.
私のVIEWSモジュールコード:

#-*- coding: utf8 -*-
from google.appengine.ext import db
from django import http
from django import shortcuts
from datetime import datetime 
from google.appengine.api import urlfetch
import re
from BeautifulSoup import BeautifulSoup
dict = {u' ':1,u' ':2,u' ':3,u' ':4,u' ':5}

class Sources(db.Model):
    #id = db.IntegerProperty(required = True)
    name = db.StringProperty(required =True)

class Articles(db.Model):
    #id = db.IntegerPropeirty()
    titles = db.StringProperty()#required = True)
    link = db.LinkProperty()#required = True)
    source_id = db.IntegerProperty()
    content = db.TextProperty()
    price = db.FloatProperty()
    region = db.StringProperty()
    subregion = db.StringProperty()
    area = db.IntegerProperty()
    area_name = db.StringProperty()
    community = db.StringProperty()
    room_layout = db.IntegerProperty()
    pub_date = db.DateTimeProperty()
    crawl_date = db.DateTimeProperty()
    original_id = db.IntegerProperty()

def ReturnText(element):
    text =""
    try:
        elements = element.contents
        for e in elements:
            text += ReturnText(e).strip()
    except:
        text = element
    return (text)

def parse(data):
    data = data.decode("utf8")
    re_s = u"""<tr>\s+<td class="center" style="width:85px"><b>(\d+)</b></td>\s+<td class="center" style="width:85px">(\d) (\d) (\d) </td>\s+<td class="t"><a href="(.{1,300}?)" target="_blank" class="t">(.{1,300}?)</a>.{1,300}?<td class="center" style="width:100px">(.{1,300}?)</td>\s+</tr>"""
    items = re.compile(re_s,re.M|re.S).findall(data)
    return items

def sub_parse(data):
    soup = BeautifulSoup(data)
    sTag = soup.find('span',{'class':'updatedate'})
    p = soup.find('span',{'class':'rednum'})
    if p:
        p = p.string.strip()
        try:
            price = float(p)
        except:
            price = 0

    str_date = sTag.string.strip().split(u"：")[-1]
    pub_date = datetime.strptime(str_date,'%Y-%m-%d %H:%M')
    cTag = soup.find('code')
    content = ReturnText(cTag)
    table = soup.find('table',{'class':'boxleft'})
    trs = table.findAll('tr')
    aTag = trs[0].findAll('a')
    if len(aTag) == 1:
        subregion = aTag[0].string.strip()
        area_name = ''
    else:
        subregion = aTag[0].string.strip()
        area_name = aTag[1].string.strip()
    try:
        community = trs[1].find('a').string.strip()
    except:
        dTag = trs[1].findAll('td')[-1]
        community = dTag.string.strip()
    
    for tr in trs[2:]:
        txt = tr.find('td').string.strip()
        if txt == u'  ：' or txt == u'  ：':
            tmp = tr.findAll('td')[-1].string.strip()
            num = tmp.split(u' ')[0]
            li = dict.keys()
            for key in dict.keys():
                if key == num:
                    value = dict[key]
                    break
            n = re.sub('\D','',tmp)
            room_layout = int(str(value) + str(n))
            
            break
    for tr in trs[2:]:
        txt = tr.find('td').string.strip()
        if txt == u'    ：' or txt == u'    ：':
            tmp = tr.findAll('td')[-1].string.strip()
            area = re.sub('\D','',tmp)
            if area == u'':
                area = 0
            else:
                area = int(area)
            break
        else:
            area = 0



    return (price,pub_date,content,subregion,area_name,community,area,room_layout)

def deal_func(item):
    try:
        price = float(item[0])
    except:
        price = 0
    link = item[4]
    tmp = link.split('/')[-1]
    original = re.sub('\D','',tmp)
    original = int(original)
    title = item[5]

    data = open_page(link)
    result = sub_parse(data)

    source_id = 1
    article = Articles()
    article.price = price
    article.titles = title
    article.link = link
    article.pub_date = result[1]
    article.source_id = source_id
    article.content = result[2]
    article.region = 'Beijing'
    article.subregion = result[3]
    article.area = result[6]
    article.area_name = result[4]
    article.community = result[5]
    article.room_layout = result[7]
    article.crawl_date = datetime.now()
    article.original_id = original
    article.put()

def open_page(url):
    res = urlfetch.fetch(url)
    data =res.content
    return data

def home_page(request):
    url = "http://bj.58.com/haidian/zufang/0/"
    data = open_page(url)
    items = parse(data)
    for item in items:
        deal_func(item)
    variable = 'If you want to check the data,please click the link'
    return shortcuts.render_to_response('index.html',{'variable':variable})

def result_page(request):
    temp = db.GqlQuery("SELECT * FROM Articles")
    results = temp.fetch(1000)    
    return shortcuts.render_to_response('result.html',{'results':results})

URLSモジュール:

from django.conf.urls.defaults import *
from views import *
urlpatterns = patterns(
    (r'^$',home_page),
    (r'^result$',result_page),
)

SETTINGSモジュール:

import os
DEBUG = True
TEMPLATE_DEBUG = DEBUG

ADMINS = (
    # ('Your Name', '[email protected]'),
)

MANAGERS = ADMINS

DATABASE_NAME = ''             # Or path to database file if using sqlite3.
DATABASE_USER = ''             # Not used with sqlite3.
DATABASE_PASSWORD = ''         # Not used with sqlite3.
DATABASE_HOST = ''             # Set to empty string for localhost. Not used with sqlite3.
DATABASE_PORT = ''             # Set to empty string for default. Not used with sqlite3.

# Local time zone for this installation. Choices can be found here:
# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
# although not all choices may be available on all operating systems.
# If running in a Windows environment this must be set to the same as your
# system time zone.
TIME_ZONE = 'Asia/Chongqing'

# Language code for this installation. All choices can be found here:
# http://www.i18nguy.com/unicode/language-identifiers.html

LANGUAGE_CODE = 'en-US'
FILE_CHARSET = 'utf-8'
DEFAULT_CHARSET = 'utf-8'
SITE_ID = 1

# If you set this to False, Django will make some optimizations so as not
# to load the internationalization machinery.
USE_I18N = True

# Absolute path to the directory that holds media.
# Example: "/home/media/media.lawrence.com/"
MEDIA_ROOT = ''

# URL that handles the media served from MEDIA_ROOT. Make sure to use a
# trailing slash if there is a path component (optional in other cases).
# Examples: "http://media.lawrence.com", "http://example.com/media/"
MEDIA_URL = ''

# URL prefix for admin media -- CSS, JavaScript and images. Make sure to use a
# trailing slash.
# Examples: "http://foo.com/media/", "/media/".
ADMIN_MEDIA_PREFIX = '/media/'

# Make this unique, and don't share it with anybody.
SECRET_KEY = '^ab3&g(clwew7@#=*&-$4i$09kirn&q$1$h6)^*8^9pdblpm&f'

# List of callables that know how to import templates from various sources.
TEMPLATE_LOADERS = (
    'django.template.loaders.filesystem.load_template_source',
    #'django.template.loaders.app_directories.load_template_source',
#     'django.template.loaders.eggs.load_template_source',
)

MIDDLEWARE_CLASSES = (
    #'django.middleware.common.CommonMiddleware',
    #'django.contrib.sessions.middleware.SessionMiddleware',
    #'django.contrib.auth.middleware.AuthenticationMiddleware',
)

ROOT_URLCONF = 'urls'

TEMPLATE_DIRS = (
        os.path.join(os.path.dirname(__file__),'templates')
    # Put strings here, like "/home/html/django_templates" or "C:/www/django/templates".
    # Always use forward slashes, even on Windows.
    # Don't forget to use absolute paths, not relative paths.
)

INSTALLED_APPS = (
    #'django.contrib.auth',
    'django.contrib.contenttypes',
    #'django.contrib.sessions',
    'django.contrib.sites',
    #'website.mywebsite'
)

注意、GAEで使用されているので、多くのADMINのコンポーネントや一部のミドルウェアは使用できないので、コメントしました.
だから細かいことを書くと、当初は純粋な正則で書くつもりだったのですが、面倒なことに気づきました.その後、BeautifulSoupが使えることに気づき、仕事量がすぐに軽減しました.私の添付ファイルにはすべてのソースコードが含まれています.comの爬虫類は、好きなものをダウンロードしてゆっくり閲覧しましょう.BEAUTIFULSOUPに詳しくない方は
http://www.crummy.com/software/BeautifulSoup/documentation.html関連ドキュメントを参照してください.