python接続HBAse
6771 ワード
環境
hadoop 2.7.0 hbase 1.2.1 Thrift 0.9.0
hbaseのThrift RPCを起動
pythonを生成するThriftモジュール
hbase_client.py
hadoop 2.7.0 hbase 1.2.1 Thrift 0.9.0
hbaseのThrift RPCを起動
./hbase-daemon.sh start thrift
pythonを生成するThriftモジュール
cd hbase-1.2.1/hbase-thrift/src/main/resources/org/apache/hadoop/hbase/thrift
thrift --gen py Hbase.thrift
# gen-py
.
├── gen-py
│ ├── hbase
│ │ ├── constants.py
│ │ ├── Hbase.py
│ │ ├── Hbase-remote
│ │ ├── __init__.py
│ │ └── ttypes.py
│ └── __init__.py
└── Hbase.thrift
# gen-py/hbase
.
├── hbase
│ ├── constants.py
│ ├── Hbase.py
│ ├── Hbase.pyc
│ ├── Hbase-remote
│ ├── __init__.py
│ ├── __init__.pyc
│ ├── ttypes.py
│ └── ttypes.pyc
└── hbase_client.py
hbase_client.py
# # -*- coding: utf-8 -*-
from thrift.transport import TSocket
from thrift.transport import TTransport
from thrift.protocol import TBinaryProtocol
from hbase import Hbase
from hbase.ttypes import ColumnDescriptor, Mutation
class HbaseClient(object):
def __init__(self, host='localhost', port=9090):
transport = TTransport.TBufferedTransport(TSocket.TSocket(host, port))
protocol = TBinaryProtocol.TBinaryProtocol(transport)
self.client = Hbase.Client(protocol)
transport.open()
def get_tables(self):
"""
"""
return self.client.getTableNames()
def create_table(self, table, *columns):
"""
"""
self.client.createTable(table, map(lambda column: ColumnDescriptor(column), columns))
def put(self, table, row, columns, attributes=None):
"""
@:param columns {"k:1":"11"}
"""
self.client.mutateRow(table, row, map(lambda (k,v): Mutation(column=k, value=v), columns.items()), attributes)
def scan(self, table, start_row="", columns=None, attributes=None):
"""
"""
scanner = self.client.scannerOpen(table, start_row, columns, attributes)
while True:
r = self.client.scannerGet(scanner)
if not r:
break
yield dict(map(lambda (k, v): (k, v.value),r[0].columns.items()))
if __name__ == "__main__":
client = HbaseClient("192.168.19.74", 9090)
client.create_table("student", "name", "coruse")
print(client.get_tables())
client.put("student", "1", {"name:":"zhangsan", "coruse:art": "88", "coruse:math": "12"})
client.put("student", "2", {"name:":"lisi", "coruse:art": "90", "coruse:math": "100"})
client.put("student", "3", {"name:":"lisi2"})
for v in client.scan("student", columns=["name"]):
print(v)
for v in client.scan("student"):
print(v)