一、安装环境

1，前期准备：官网下载“NuGet Package Manager”，按自己已有的VS环境下载对应版本；

2，利用NuGet下载Hadoop For .NET SDK,地址“http://hadoopsdk.codeplex.com/”

3，安装。

4，通过HDInsight，安装Windows Azure，目前是预览版本。

5，参照网址“http://blogs.msdn.com/b/data_otaku/archive/2013/08/14/hadoop-for-net-developers.aspx” 系统学习API

二、测试DEMO

代码

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Microsoft.Hadoop;
using Microsoft.Hadoop.MapReduce;
using Microsoft.Hadoop.WebClient.WebHCatClient;
using System.Diagnostics;
using System.IO;
using System.IO.MemoryMappedFiles;
namespace HadoopConsol
{
class Program
{
static void Main(string[] args)
{
Stopwatch sw = new Stopwatch();
long hadoopTime=0;
long normalTime=0;
sw.Start();
//start hadoop
Console.WriteLine(" Hadoop Process Strating ....");
#region Hadoop time
#region hadoopconnet
Console.WriteLine(" Hadoop Connect Strating ....");
//establish job configuration
HadoopJobConfiguration myConfig = new HadoopJobConfiguration();
myConfig.InputPath = "/demo/simple/in";
myConfig.OutputFolder = "/demo/simple/out";
//connect to cluster
Uri myUri = new Uri("http://localhost");
string userName = "hadoop";
string passWord = null;
IHadoop myCluster = Hadoop.Connect(myUri, userName, passWord);
hadoopTime += sw.ElapsedMilliseconds;
Console.WriteLine(" Hadoop Connect End.");
Console.WriteLine(" Hadoop Connect time:" + sw.ElapsedMilliseconds);
#endregion
#region hadoopmapreduce
sw.Reset();
sw.Start();
Console.WriteLine(" Hadoop MapReduce Strating ....");
//execute mapreduce job
MapReduceResult jobResult =
myCluster.MapReduceJob.Execute<MySimpleMapper, MySimpleReducer>(myConfig);
hadoopTime += sw.ElapsedMilliseconds;
Console.WriteLine(" Hadoop MapReduce End.");
Console.WriteLine(" Hadoop MapReduce Time:"+sw.ElapsedMilliseconds);
#endregion
#region Hadoop End
sw.Reset();
sw.Start();
Console.WriteLine(" Hadoop Endprocess Strating ....");
//write job result to console
int exitCode = jobResult.Info.ExitCode;
string exitStatus = "Failure";
if (exitCode == 0) exitStatus = "Success";
exitStatus = exitCode + " (" + exitStatus + ")";
Console.WriteLine();
Console.Write("Exit Code = " + exitStatus);
Console.WriteLine(" Hadoop Endprocess End.");
hadoopTime += sw.ElapsedMilliseconds;
Console.WriteLine(" Hadoop Exit Time:" + sw.ElapsedMilliseconds);
Console.WriteLine(" Hadoop Process All Time:" + hadoopTime);
#endregion
#endregion
#region Normal time
//start Normal
Console.WriteLine(" Normal Process Strating ....");
sw.Reset();
sw.Start();
//normal process
#region Normal Process
int myevenCount = 0;
int myeventSum = 0;
int myoddCount = 0;
int myoddSum = 0;
StreamReader fs = new StreamReader(@"c:\TEMP\integers.txt");
while (fs.Peek() >= 0)
{
string strTemp = fs.ReadLine();
if (Int32.Parse(strTemp) % 2 == 0)
{
myevenCount++;
myeventSum += Int32.Parse(strTemp);
}
else
{
myoddCount++;
myoddSum += Int32.Parse(strTemp);
}
}
//MemoryMappedFile m = MemoryMappedFile.
Console.WriteLine("even:" + "\t" + myevenCount + "\t" + myeventSum);
Console.WriteLine("odd:" + "\t" + myoddCount + "\t" + myoddSum);
#endregion
Console.WriteLine(" Normal Process End.");
normalTime += sw.ElapsedMilliseconds;
Console.WriteLine(" Normal Exit Time:" + sw.ElapsedMilliseconds);
Console.WriteLine(" Normal Process All Time:" + normalTime);
#endregion
sw.Stop();
Console.Read();
}
}
public class MySimpleMapper : MapperBase
{
public override void Map(string inputLine, MapperContext context)
{
//interpret the incoming line as an integer value
int value = int.Parse(inputLine);
//determine whether value is even or odd
string key = (value % 2 == 0) ? "even" : "odd";
//output key assignment with value
context.EmitKeyValue(key, value.ToString());
}
}
public class MySimpleReducer : ReducerCombinerBase
{
public override void Reduce(
string key, IEnumerable<string> values, ReducerCombinerContext context
)
{
//initialize counters
int myCount = 0;
int mySum = 0;
//count and sum incoming values
foreach (string value in values)
{
mySum += int.Parse(value);
myCount++;
}
//output results
context.EmitKeyValue(key, myCount + "\t" + mySum);
}
}
}

三、测试结果

本版精华
热门帖子

操作引用/回复

总帖数

每页帖数

1/1页

返回列表

用户登录

一、安装环境

二、测试DEMO

Weblogic中间件技术论坛

Tuxedo中间件技术论坛

数据库论坛

Java论坛

Linux/unix论坛

网站地图