[メモ]hdfs namenode FSImage分析1

メタデータファイルfsimageの分析
fsimageはメタデータミラーファイルであり、ある時点でNameNodeメモリメタデータ情報を格納し、すべてのINode情報、書き込み中のファイル情報、その他のステータス情報などを含む.
参照
 /**
   * Save current image and empty journal into {@code current} directory.
   */
  protected void saveCurrent(StorageDirectory sd) throws IOException
StorageDirectoryのタイプがName NodeFileである場合.IMAGEの場合、FSImageを../に永続化current/fsimageファイル.
fsimageの部分構造分析は以下の通りである.
int
int
long
long
...
LAYOUT_VERSION
namespaceID
numItemsInTree
timestamp
inodes
inodesがディレクトリの場合の構造
short
binary
short
long
long
long
int
long
long
permission
namelen
name
0
modificationTime
0
0
-1
nsquota
dsquota
permission
inodesがファイルの場合の構造
short
binary
short
long
long
long
int
...
permission
namelen
name
replication
modificationTime
accessTime
preferredBlockSize
numBlocksInFile
blocks
permission
permissionの構造
Text
Text
short
username
groupname
permission
blocksの構造
long
long
long
blockId
numBytes
timestamp
あるfsimageの結果を解析する
LAYOUT_VERSION:-41
namespaceID:1167023747
numItemsInTree:19
generationStamp:1012
node:{name:, modtime:1377679793059, nsquota:2147483647, dsquota:-1, permission:hue supergroup 755}
node:{name:/test, modtime:1377679792720, nsquota:-1, dsquota:-1, permission:hue supergroup 755}
node:{name:/tmp, modtime:1377679793059, nsquota:-1, dsquota:-1, permission:hue supergroup 755}
node:{name:/test/input, modtime:1377679736766, nsquota:-1, dsquota:-1, permission:hue supergroup 755}
node:{name:/test/output, modtime:1377679792720, nsquota:-1, dsquota:-1, permission:hue supergroup 755}
node:{name:/test/input/chapter1.txt, copys:2, blksize:67108864, blk:[{3939378972164311396,8976,1002}], modtime:1377679736766, acctime:1377679736766, permission:hue supergroup 644}
node:{name:/test/output/_SUCCESS, copys:2, blksize:67108864, blk:[], modtime:1377679792720, acctime:1377679792720, permission:hue supergroup 644}
node:{name:/test/output/_logs, modtime:1377679773673, nsquota:-1, dsquota:-1, permission:hue supergroup 755}
node:{name:/test/output/part-r-00000, copys:2, blksize:67108864, blk:[{5401467920515552367,6128,1011}], modtime:1377679787973, acctime:1377679787973, permission:hue supergroup 644}
node:{name:/test/output/_logs/history, modtime:1377679773673, nsquota:-1, dsquota:-1, permission:hue supergroup 755}
node:{name:/test/output/_logs/history/job_201308281640_0001_1377679773095_hue_word+count, copys:2, blksize:67108864, blk:[{-7275418827867675308,13833,1012}], modtime:1377679773322, acctime:1377679773322, permission:hue supergroup 644}
node:{name:/test/output/_logs/history/job_201308281640_0001_conf.xml, copys:2, blksize:67108864, blk:[{-4116164172580388309,47105,1010}], modtime:1377679773673, acctime:1377679773673, permission:hue supergroup 644}
node:{name:/tmp/hadoop-hue, modtime:1377679793059, nsquota:-1, dsquota:-1, permission:hue supergroup 755}
node:{name:/tmp/hadoop-hue/mapred, modtime:1377679793059, nsquota:-1, dsquota:-1, permission:hue supergroup 755}
node:{name:/tmp/hadoop-hue/mapred/staging, modtime:1377679793059, nsquota:-1, dsquota:-1, permission:hue supergroup 755}
node:{name:/tmp/hadoop-hue/mapred/system, modtime:1377679793049, nsquota:-1, dsquota:-1, permission:hue supergroup 700}
node:{name:/tmp/hadoop-hue/mapred/staging/hue, modtime:1377679793059, nsquota:-1, dsquota:-1, permission:hue supergroup 755}
node:{name:/tmp/hadoop-hue/mapred/staging/hue/.staging, modtime:1377679793059, nsquota:-1, dsquota:-1, permission:hue supergroup 700}
node:{name:/tmp/hadoop-hue/mapred/system/jobtracker.info, copys:2, blksize:67108864, blk:[{-7415943897866965603,4,1001}], modtime:1377679253906, acctime:1377679253906, permission:hue supergroup 600}
解析コードは次のとおりです.

try (DataInputStream in = new DataInputStream(new BufferedInputStream(
				new FileInputStream(fileName)))) {
	System.out.printf("LAYOUT_VERSION:%s
", in.readInt());
	System.out.printf("namespaceID:%s
", in.readInt());
	long nodes = in.readLong();
	System.out.printf("numItemsInTree:%s
", nodes);
	System.out.printf("generationStamp:%s

", in.readLong());

	byte[] byteStore = new byte[4 * FSConstants.MAX_PATH_LENGTH];
	for (long i = 0; i < nodes; i++) {
		printINode(in, byteStore);
		Arrays.fill(byteStore, (byte) 0x00);
	}
}

	void printINode(DataInputStream in, byte[] byteStore)
			throws IOException {
		boolean isDir = true;
		short namelen = in.readShort();
		in.read(byteStore, 0, namelen);
		short replication = in.readShort();
		long modTime = in.readLong();
		long accessTime = in.readLong(); // access time
		long blockSize = in.readLong(); // preferred block size
		int blocks = in.readInt(); // # of blocks
		StringBuilder sb = new StringBuilder();
		long nsQuota = 0;
		long dsQuota = 0;
		if (blocks >= 0) {
			isDir = false;
			sb.append("[");
			for (int i = 0; i < blocks; i++) {
				sb.append("{").append(in.readLong()).append(",")
						.append(in.readLong()).append(",")
						.append(in.readLong()).append("}");
				if (i + 1 != blocks) {
					sb.append(",");
				}
			}
			sb.append("]");
		} else {
			nsQuota = in.readLong();
			dsQuota = in.readLong();
		}
		String userName = Text.readString(in);
		String groupName = Text.readString(in);
		short permit = in.readShort();
		if (isDir) {
			System.out
					.printf("node:{name:%s, modtime:%s, nsquota:%s, dsquota:%s, permission:%s %s %s}
",
							new String(byteStore, 0, namelen), modTime,
							nsQuota, dsQuota, userName, groupName,
							Integer.toOctalString(permit));
		} else {
			System.out
					.printf("node:{name:%s, copys:%s, blksize:%s, blk:%s, modtime:%s, acctime:%s, permission:%s %s %s}
",
							new String(byteStore, 0, namelen), replication,
							blockSize, sb.toString(), modTime, accessTime,
							userName, groupName, Integer.toOctalString(permit));
		}

	}