最近跑hadoop遇到的一些问题

一、

[#|2013-09-16T18:19:02.663+0800|INFO|glassfish3.1.2|javax.enterprise.system.std.com.sun.enterprise.server.logging|_ThreadID=23364;_ThreadName=Thread-2;|2013-09-1618:19:02,663 WARN  DataStreamerException: org.apache.hadoop.ipc.RemoteException: java.io.IOException:
File:xxx could only be replicatedto 0 nodes, instead of 1

atorg.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAdditionalBlock(FSNamesystem.java:1558)

atorg.apache.hadoop.hdfs.server.namenode.NameNode.addBlock(NameNode.java:696)

atsun.reflect.GeneratedMethodAccessor6.invoke(Unknown Source)

atsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)

atjava.lang.reflect.Method.invoke(Method.java:597)

atorg.apache.hadoop.ipc.RPC$Server.call(RPC.java:563)

atorg.apache.hadoop.ipc.Server$Handler$1.run(Server.java:1388)

atorg.apache.hadoop.ipc.Server$Handler$1.run(Server.java:1384)

atjava.security.AccessController.doPrivileged(Native Method)

atjavax.security.auth.Subject.doAs(Subject.java:396)

atorg.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1121)

atorg.apache.hadoop.ipc.Server$Handler.run(Server.java:1382)

atorg.apache.hadoop.ipc.Client.call(Client.java:1070)

atorg.apache.hadoop.ipc.RPC$Invoker.invoke(RPC.java:225)

at$Proxy140.addBlock(Unknown Source)

atsun.reflect.GeneratedMethodAccessor84.invoke(Unknown Source)

atsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)

atjava.lang.reflect.Method.invoke(Method.java:597)

atorg.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:82)

atorg.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:59)

at$Proxy140.addBlock(Unknown Source)

atorg.apache.hadoop.hdfs.DFSClient$DFSOutputStream.locateFollowingBlock(DFSClient.java:3510)

atorg.apache.hadoop.hdfs.DFSClient$DFSOutputStream.nextBlockOutputStream(DFSClient.java:3373)

atorg.apache.hadoop.hdfs.DFSClient$DFSOutputStream.access$2600(DFSClient.java:2589)

atorg.apache.hadoop.hdfs.DFSClient$DFSOutputStream$DataStreamer.run(DFSClient.java:2829)

|#]

二、

[#|2013-09-16T18:19:02.664+0800|INFO|glassfish3.1.2|javax.enterprise.system.std.com.sun.enterprise.server.logging|_ThreadID=23367;_ThreadName=Thread-2;|2013-09-1618:19:02,664 WARN  Error Recovery forblock null bad datanode[0] nodes
== null

|#]

[#|2013-09-16T18:19:02.664+0800|INFO|glassfish3.1.2|javax.enterprise.system.std.com.sun.enterprise.server.logging|_ThreadID=23367;_ThreadName=Thread-2;|2013-09-1618:19:02,664 WARN  Could not get blocklocations. Source file "xxx"-
Aborting...

|#]

三、

[#|2013-09-16T18:20:13.134+0800|SEVERE|glassfish3.1.2|javax.enterprise.system.std.com.sun.enterprise.server.logging|_ThreadID=23467;_ThreadName=Thread-2;|java.util.concurrent.TimeoutException:Timed out(5000 milliseconds) waiting for
operation while connected to xxx:11211

atnet.rubyeye.xmemcached.XMemcachedClient.latchWait(XMemcachedClient.java:2617)

atnet.rubyeye.xmemcached.XMemcachedClient.fetch0(XMemcachedClient.java:617)

atnet.rubyeye.xmemcached.XMemcachedClient.get0(XMemcachedClient.java:1030)

atnet.rubyeye.xmemcached.XMemcachedClient.gets(XMemcachedClient.java:1043)

atnet.rubyeye.xmemcached.XMemcachedClient.gets(XMemcachedClient.java:1065)

at net.rubyeye.xmemcached.XMemcachedClient.gets(XMemcachedClient.java:1054)

atcom.haierpip.service.impl.MemcachedServiceImpl.gets(MemcachedServiceImpl.java:104)

atcom.haierpip.timer.UpdateImgQueueMapJob.updateList(UpdateImgQueueMapJob.java:41)

at com.haierpip.timer.UpdateImgQueueMapJob.executeInternal(UpdateImgQueueMapJob.java:26)

at sun.reflect.GeneratedMethodAccessor64.invoke(UnknownSource)

atsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)

at java.lang.reflect.Method.invoke(Method.java:597)

atorg.springframework.util.MethodInvoker.invoke(MethodInvoker.java:273)

atorg.springframework.scheduling.quartz.MethodInvokingJobDetailFactoryBean$MethodInvokingJob.executeInternal(MethodInvokingJobDetailFactoryBean.java:311)

atorg.springframework.scheduling.quartz.QuartzJobBean.execute(QuartzJobBean.java:113)

at org.quartz.core.JobRunShell.run(JobRunShell.java:202)

atorg.quartz.simpl.SimpleThreadPool$WorkerThread.run(SimpleThreadPool.java:525)

|#]

四、

[#|2013-09-16T18:05:12.983+0800|INFO|glassfish3.1.2|javax.enterprise.system.std.com.sun.enterprise.server.logging|_ThreadID=1595;_ThreadName=Thread-2;|2013-09-1618:05:12,983 INFO  Exception increateBlockOutputStream 10.255.254.6:50010
java.net.ConnectException:Connection timed out

|#]

[#|2013-09-16T18:05:12.983+0800|INFO|glassfish3.1.2|javax.enterprise.system.std.com.sun.enterprise.server.logging|_ThreadID=1595;_ThreadName=Thread-2;|2013-09-1618:05:12,983 INFO  Abandoning blockblk_2971499765308711932_1671045

|#]

[#|2013-09-16T18:05:12.984+0800|INFO|glassfish3.1.2|javax.enterprise.system.std.com.sun.enterprise.server.logging|_ThreadID=1595;_ThreadName=Thread-2;|2013-09-1618:05:12,984 INFO  Excluding datanode10.255.254.6:50010

|#]

五、

2013-09-16 17:56:04,034WARN org.apache.hadoop.hdfs.server.datanode.DataNode: Block report is due, andbeen waiting for it for 708 seconds...

六、

[#|2013-09-16T18:05:40.699+0800|INFO|glassfish3.1.2|javax.enterprise.system.std.com.sun.enterprise.server.logging|_ThreadID=1571;_ThreadName=Thread-2;|2013-09-16 18:05:40,699 INFO  Exception in createBlockOutputStream xxx:50010 java.io.IOException: Bad connect ack with firstBadLink as
xxx.6:50010

 |#]

页面

WARNING : There are about146 missing blocks. Please check the log or run fsck.