Hadoop处理HDF文件

1、前言

HDF文件是遥感应用中一种常见的数据格式，因为其高度结构化的特点，笔者曾被怎样使用Hadoop处理HDF文件这个问题困扰过相当长的一段时间。于是Google各种解决方式，但都没有找到一种理想的处理办法。也曾參考过HDFGroup官方发的一篇帖子（网址在这里），里面提供了使用Hadoop针对大、中、小HDF文件的处理思路。尽管依据他提供的解决的方法，按图索骥，肯定能解决怎样使用Hadoop处理HDF文件这个问题，但个人感觉方法偏复杂且须要对HDF的数据格式有较深的理解，实现起来不太easy。于是乎，笔者又继续寻找解决方式，最终发现了一种办法，以下将对该方法进行详细说明。

2、MapReduce主程序

这里主要使用到了netcdf的库进行hdf数据流的反序列化工作（netcdf库的下载地址）。与HDF官方提供的Java库不同，netcdf仅利用Java进行HDF文件的读写操作，且这个库支持多种科学数据，包含HDF4、HDF5等多种格式。而HDF的官方Java库中，底层实际仍是用C进行HDF文件的操作。以下贴出MapReduce的Mapper函数代码：

package example;

import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.net.URI;
import java.util.List;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

import ucar.ma2.ArrayShort;
import ucar.nc2.Dimension;
import ucar.nc2.Group;
import ucar.nc2.NetcdfFile;
import ucar.nc2.Variable;

public class ReadMapper extends
		Mapper<Text, BytesWritable, Text, BytesWritable> {

	public void map(Text key, BytesWritable value, Context context)
			throws IOException, InterruptedException {
		String fileName = key.toString();
		NetcdfFile file = NetcdfFile.openInMemory("hdf4", value.get());
		Group dataGroup = (file.findGroup("MOD_Grid_monthly_1km_VI")).findGroup("Data_Fields");
		//读取到1_km_monthly_red_reflectance的变量
		Variable redVar = dataGroup.findVariable("1_km_monthly_red_reflectance");
		short[][] data = new short[1200][1200];
		if(dataGroup != null){
			ArrayShort.D2 dataArray;
			//读取redVar中的影像数据
			dataArray = (ArrayShort.D2) redVar.read();
			List<Dimension> dimList = file.getDimensions();
			//获取影像的y方向像元个数
			Dimension ydim = dimList.get(0);
			//获取影像的x方向像元个数
			Dimension xdim = dimList.get(1);
			//遍历整个影像，读取出像元的值
			for(int i=0;i<xdim.getLength();i++){
				for(int j=0;j<ydim.getLength();j++){
					data[i][j] = dataArray.get(i, j);
				}
			}
		}
		System.out.print(file.getDetailInfo());
	}
}

注意程序中的NetcdfFile.openInMemory方法，该静态方法支持从byte[]中构造HDF文件，从而实现了HDF文件的反序列化操作。以下贴出主程序的演示样例代码：

package example;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat;

import example.WholeFileInputFormat;

public class ReadMain {
	public boolean runJob(String[] args) throws IOException,
			ClassNotFoundException, InterruptedException {
		Configuration conf = new Configuration();
		// conf.set("mapred.job.tracker", Utils.JOBTRACKER);
		String rootPath= "/opt/hadoop-2.3.0/etc/hadoop";
		//String rootPath="/opt/hadoop-2.3.0/etc/hadoop/";
		conf.addResource(new Path(rootPath+"yarn-site.xml"));
		conf.addResource(new Path(rootPath+"core-site.xml"));
		conf.addResource(new Path(rootPath+"hdfs-site.xml"));
		conf.addResource(new Path(rootPath+"mapred-site.xml"));
		Job job = new Job(conf);

		job.setJobName("Job name:" + args[0]);
		job.setJarByClass(ReadMain.class);

		job.setMapperClass(ReadMapper.class);
		job.setMapOutputKeyClass(Text.class);
		job.setMapOutputValueClass(BytesWritable.class);

		job.setInputFormatClass(WholeFileInputFormat.class);
		job.setOutputFormatClass(NullOutputFormat.class);
		FileInputFormat.addInputPath(job, new Path(args[1]));
		FileOutputFormat.setOutputPath(job, new Path(args[2]));
		boolean flag = job.waitForCompletion(true);
		return flag;
	}

	public static void main(String[] args) throws ClassNotFoundException,
			IOException, InterruptedException {
		String[] inputPaths = new String[] { "normalizeJob",
				"hdfs://192.168.168.101:9000/user/hduser/hdf/MOD13A3.A2005274.h00v10.005.2008079143041.hdf",
				"hdfs://192.168.168.101:9000/user/hduser/test/" };
		ReadMain test = new ReadMain();
		test.runJob(inputPaths);
	}

}

关于MapReduce主程序有几点值得说明一下：

1、MapReduce数据的输入格式为WholeFileInputFormat.class，即不正确数据进行切分。关于该格式，能够參考另外一篇博客：怎样通过Java程序提交Yarn的计算任务，这里不再赘述。

2、本人用的是Yarn2.3.0来运行计算任务，假设用老版本号的hadoop，如1.2.0，则把以上主程序中的conf.addResource部分的代码删掉就可以。

3、以上MapReduce程序中，仅仅用到了Map函数，未设置Reduce函数。

4、以上程序用到的为HDF4格式的数据，按理说，HDF5格式的数据应该也是支持的。

3、HDF数据的格式

因为HDF数据高度结构化，因此在netcdf库的使用中，须要使用类似于"标签"的方式来訪问HDF中的详细数据。以下贴出netcdf中读出来的HDF数据的详细格式信息（即使用file.getDetailInfo()函数，打印出来的信息）：

注意，ReadMapper函数中出现的类似于“MOD_Grid_monthly_1km_VI”、"Data_Fields"等信息，即依据下面HDF数据的格式信息得到的。

netcdf D:/2005-274/MOD13A3.A2005274.h00v08.005.2008079142757.hdf {
  variables:
    char StructMetadata.0(32000);

    char CoreMetadata.0(40874);

    char ArchiveMetadata.0(6530);

  group: MOD_Grid_monthly_1km_VI {
    variables:
      short _HDFEOS_CRS;
        :Projection = "GCTP_SNSOID";
        :UpperLeftPointMtrs = -2.0015109354E7, 1111950.519667; // double
        :LowerRightMtrs = -1.8903158834333E7, -0.0; // double
        :ProjParams = 6371007.181, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0; // double
        :SphereCode = "-1";

    group: Data_Fields {
      dimensions:
        YDim = 1200;
        XDim = 1200;
      variables:
        short 1_km_monthly_NDVI(YDim=1200, XDim=1200);
          :long_name = "1 km monthly NDVI";
          :units = "NDVI";
          :valid_range = -2000S, 10000S; // short
          :_FillValue = -3000S; // short
          :scale_factor = 10000.0; // double
          :scale_factor_err = 0.0; // double
          :add_offset = 0.0; // double
          :add_offset_err = 0.0; // double
          :calibrated_nt = 5; // int

        short 1_km_monthly_EVI(YDim=1200, XDim=1200);
          :long_name = "1 km monthly EVI";
          :units = "EVI";
          :valid_range = -2000S, 10000S; // short
          :_FillValue = -3000S; // short
          :scale_factor = 10000.0; // double
          :scale_factor_err = 0.0; // double
          :add_offset = 0.0; // double
          :add_offset_err = 0.0; // double
          :calibrated_nt = 5; // int

        short 1_km_monthly_VI_Quality(YDim=1200, XDim=1200);
          :_Unsigned = "true";
          :long_name = "1 km monthly VI Quality";
          :units = "bit field";
          :valid_range = 0S, -2S; // short
          :_FillValue = -1S; // short
          :Legend = "\n\t Bit Fields Description (Right to Left): \n\t[0-1] : MODLAND_QA [2 bit range]\n\t\t 00: VI produced, good quality \n\t\t 01: VI produced, but check other QA \n\t\t 10: Pixel produced, but most probably cloudy \n\t\t 11: Pixel not produced due to other reasons than clouds \n\t[2-5] : VI usefulness [4 bit range]  \n\t\t 0000: Highest quality  \n\t\t 0001: Lower quality  \n\t\t 0010..1010: Decreasing quality  \n\t\t 1100: Lowest quality  \n\t\t 1101: Quality so low that it is not useful \n\t\t 1110: L1B data faulty \n\t\t 1111: Not useful for any other reason/not processed \n\t[6-7] : Aerosol quantity [2 bit range] \n\t\t 00: Climatology \n\t\t 01: Low \n\t\t 10: Average \n\t\t 11: High (11) \n\t[8] : Adjacent cloud detected; [1 bit range] \n\t\t 1: Yes \n\t\t 0: No \n\t[9] : Atmosphere BRDF correction performed [1 bit range] \n\t\t 1: Yes \n\t\t 0: No \n\t[10] : Mixed clouds  [1 bit range] \n\t\t 1: Yes \n\t\t 0: No \n\t[11-13] : Land/Water Flag [3 bit range]   \n\t\t 000: Shallow ocean \n\t\t 001: Land (Nothing else but land) \n\t\t 010: Ocean coastlines and lake shorelines \n\t\t 011: Shallow inland water \n\t\t 100: Ephemeral water \n\t\t 101: Deep inland water \n\t\t 110: Moderate or continental ocean \n\t\t 111: Deep ocean \n\t[14] : Possible snow/ice [1 bit range] \n\t\t 1: Yes \n\t\t 0: No \n\t[15] : Possible shadow [1 bit range] \n\t\t 1: Yes \n\t\t 0: No \n";

        short 1_km_monthly_red_reflectance(YDim=1200, XDim=1200);
          :long_name = "1 km monthly red reflectance";
          :units = "reflectance";
          :valid_range = 0S, 10000S; // short
          :_FillValue = -1000S; // short
          :scale_factor = 10000.0; // double
          :scale_factor_err = 0.0; // double
          :add_offset = 0.0; // double
          :add_offset_err = 0.0; // double
          :calibrated_nt = 5; // int

        short 1_km_monthly_NIR_reflectance(YDim=1200, XDim=1200);
          :long_name = "1 km monthly NIR reflectance";
          :units = "reflectance";
          :valid_range = 0S, 10000S; // short
          :_FillValue = -1000S; // short
          :scale_factor = 10000.0; // double
          :scale_factor_err = 0.0; // double
          :add_offset = 0.0; // double
          :add_offset_err = 0.0; // double
          :calibrated_nt = 5; // int

        short 1_km_monthly_blue_reflectance(YDim=1200, XDim=1200);
          :long_name = "1 km monthly blue reflectance";
          :units = "reflectance";
          :valid_range = 0S, 10000S; // short
          :_FillValue = -1000S; // short
          :scale_factor = 10000.0; // double
          :scale_factor_err = 0.0; // double
          :add_offset = 0.0; // double
          :add_offset_err = 0.0; // double
          :calibrated_nt = 5; // int

        short 1_km_monthly_MIR_reflectance(YDim=1200, XDim=1200);
          :long_name = "1 km monthly MIR reflectance";
          :units = "reflectance";
          :valid_range = 0S, 10000S; // short
          :_FillValue = -1000S; // short
          :Legend = "\n\t The MIR band saved in the VI product is MODIS band 7 \n\t\t Bandwidth : 2105-2155 nm \n\t\t Band center: 2130 nm \n";
          :scale_factor = 10000.0; // double
          :scale_factor_err = 0.0; // double
          :add_offset = 0.0; // double
          :add_offset_err = 0.0; // double
          :calibrated_nt = 5; // int

        short 1_km_monthly_view_zenith_angle(YDim=1200, XDim=1200);
          :long_name = "1 km monthly view zenith angle";
          :units = "degrees";
          :valid_range = -9000S, 9000S; // short
          :_FillValue = -10000S; // short
          :scale_factor = 100.0; // double
          :scale_factor_err = 0.0; // double
          :add_offset = 0.0; // double
          :add_offset_err = 0.0; // double
          :calibrated_nt = 5; // int

        short 1_km_monthly_sun_zenith_angle(YDim=1200, XDim=1200);
          :long_name = "1 km monthly sun zenith angle";
          :units = "degrees";
          :valid_range = -9000S, 9000S; // short
          :_FillValue = -10000S; // short
          :scale_factor = 100.0; // double
          :scale_factor_err = 0.0; // double
          :add_offset = 0.0; // double
          :add_offset_err = 0.0; // double
          :calibrated_nt = 5; // int

        short 1_km_monthly_relative_azimuth_angle(YDim=1200, XDim=1200);
          :long_name = "1 km monthly relative azimuth angle";
          :units = "degrees";
          :valid_range = -3600S, 3600S; // short
          :_FillValue = -4000S; // short
          :scale_factor = 10.0; // double
          :scale_factor_err = 0.0; // double
          :add_offset = 0.0; // double
          :add_offset_err = 0.0; // double
          :calibrated_nt = 5; // int

        byte 1_km_monthly_pixel_raliability(YDim=1200, XDim=1200);
          :long_name = "1 km monthly pixel raliability";
          :units = "rank";
          :valid_range = 0B, 3B; // byte
          :_FillValue = -1B; // byte
          :Legend = "\n\t Rank Keys: \n\t\t[-1]:  Fill/No Data-Not Processed. \n\t\t [0]:  Good data     - Use with confidence \n\t\t [1]:  Marginal data - Useful, but look at other QA information \n\t\t [2]:  Snow/Ice      - Target covered with snow/ice\n\t\t [3]:  Cloudy        - Target not visible, covered with cloud \n";

    }
  }
  // global attributes:
  :HDFEOSVersion = "HDFEOS_V2.9";
  :_History = "Direct read of HDF4 file through CDM library; HDF-EOS StructMetadata information was read";
  :HDF4_Version = "4.2.1 (NCSA HDF Version 4.2 Release 1-post3, January 27, 2006)";
  :featureType = "GRID";
}