Coherence对象压缩以及对象大小计算

1.通过util.zip带的gzip压缩程序 

Coherence对象压缩程序如下

package coherencetest;

import com.tangosol.net.CacheFactory;

import java.util.zip.*;
import java.io.*;
import com.tangosol.net.CacheFactory;
import com.tangosol.net.ConfigurableCacheFactory;
import com.tangosol.net.DefaultConfigurableCacheFactory;
import com.tangosol.net.NamedCache;

public class CompressObject {
public CompressObject() {
super();
}

public static byte[] writeCompressObject(Person object)
{
byte[] data_=null;
try
{
//建立字节数组输出流
ByteArrayOutputStream o = new ByteArrayOutputStream();
//建立gzip压缩输出流
GZIPOutputStream gzout=new GZIPOutputStream(o);
//建立对象序列化输出流
ObjectOutputStream out = new ObjectOutputStream(gzout);
out.writeObject(object);
out.flush();
out.close();
gzout.close();
//返回压缩字节流
data_=o.toByteArray();
o.close();
}catch(IOException e)
{
System.out.println(e);
}
return(data_);
}

public static Person readCompressObject(byte[] data_)
{
Person object_=null;
try
{
//建立字节数组输入流
ByteArrayInputStream i = new ByteArrayInputStream(data_);
//建立gzip解压输入流
GZIPInputStream gzin=new GZIPInputStream(i);
//建立对象序列化输入流
ObjectInputStream in = new ObjectInputStream(gzin);
//按制定类型还原对象
object_=(Person)in.readObject();
i.close();
gzin.close();
in.close();
}catch(ClassNotFoundException e)
{
System.out.println(e);
}catch(IOException e)
{
System.out.println(e);
}

return(object_);
}

public static void main (String [] args) {
CompressObject co = new CompressObject();

Person person = new Person();

writeObjectToFile("zipcompress_before",person);

//System.out.println(person);
byte[] compressperson = writeCompressObject(person);

writeBytesToFile("zipcompress_after",compressperson);

String personstr = new String(compressperson);

NamedCache cache = CacheFactory.getCache("POFSample");
for (int i=0;i<10000;i++) {
// cache.put (i,compressperson);
cache.put (Integer.toString(i),compressperson);

}
System.out.println("put success");

byte[] a = (byte[]) cache.get("1");
Person person1 = readCompressObject(a);
System.out.println(person1.getFirstname());
System.out.println(person1.getLastname());
System.out.println(person1.getAddress());

//cache.retrieveCache();
}

public static void writeObjectToFile(String Filename, Object obj) {

try {

FileOutputStream outStream = new FileOutputStream("c:/"+Filename);
ObjectOutputStream objectOutputStream = new ObjectOutputStream(outStream);
objectOutputStream.writeObject(obj);
outStream.close();
System.out.println("successful");

// if file doesnt exists, then create it
} catch (FileNotFoundException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
}

public static void writeBytesToFile(String Filename, byte[] objs) {

try {

FileOutputStream outStream = new FileOutputStream("c:/"+Filename);
ObjectOutputStream objectOutputStream = new ObjectOutputStream(outStream);
objectOutputStream.write(objs);
outStream.close();
System.out.println("successful");

// if file doesnt exists, then create it
} catch (FileNotFoundException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
}

}

测试了一下,原来的对象3.96kb,然后经过压缩以后2.01K,还是有减少。

但通过Coherence的visualvm看了一下，发现平均对象大小是1byte.难道coherence只存放对象的指针吗？

2.通过kryo的序列化方法

package coherencetest;

import com.esotericsoftware.kryo.Kryo;
import com.esotericsoftware.kryo.io.Input;
import com.esotericsoftware.kryo.io.Output;

import com.tangosol.dev.assembler.New;
import com.tangosol.net.CacheFactory;
import com.tangosol.net.NamedCache;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;

import org.objenesis.strategy.SerializingInstantiatorStrategy;
import org.objenesis.strategy.StdInstantiatorStrategy;

import java.io.IOException;
import java.io.ObjectOutputStream;

public class kryoCompress {
static int BYTES_LENTH = 20000;

public kryoCompress() {
super();
}

public static void main(String[] args) {
// TODO Auto-generated method stub

Person person = new Person();

writeObjectToFile("person2",person);

Kryo kryo = new Kryo();
kryo.setInstantiatorStrategy(new StdInstantiatorStrategy());

byte[] scbytes = null;
try {
scbytes= kryocompress(person,kryo);
//System.out.println(ObjectSizeFetcher.getObjectSize(scbytes));
writeBytesToFile("personcompress",scbytes);
} catch (Exception e) {
System.out.println("kryocompress:"+e.getMessage());
}

Person backsc =kryodeserialize(scbytes,kryo);
System.out.println("kryodeserialize:["+backsc.getAddress()+"]");

}

public static byte[] kryocompress(Person object, Kryo kryo) throws IOException {

byte[] buffer = new byte[BYTES_LENTH];

Output out = new Output(buffer);

kryo.writeObject(out, object);

//System.out.println("kryocompress====total:"+out.total());
//System.out.println("kryocompress====position:"+out.position());

return out.toBytes();

}

public static Person kryodeserialize(byte[] bytes,Kryo kryo) {
Input input = null;

try {

input = new Input(bytes);

return (Person)kryo.readObject(input, Person.class);
} catch (Exception e) {
System.out.println("kryodeserialize==#==["+e.getMessage()+"]");
//System.out.println(e.getMessage());
}
return null;

}

public static void writeObjectToFile(String Filename, Object obj) {

try {

FileOutputStream outStream = new FileOutputStream("c:/"+Filename);
ObjectOutputStream objectOutputStream = new ObjectOutputStream(outStream);
objectOutputStream.writeObject(obj);
outStream.close();
System.out.println("successful");

// if file doesnt exists, then create it
} catch (FileNotFoundException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
}

public static void writeBytesToFile(String Filename, byte[] objs) {

try {

FileOutputStream outStream = new FileOutputStream("c:/"+Filename);
ObjectOutputStream objectOutputStream = new ObjectOutputStream(outStream);
objectOutputStream.write(objs);
outStream.close();
System.out.println("successful");

// if file doesnt exists, then create it
} catch (FileNotFoundException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
}

}

测试了一下，原来标准的3.96K的对象，经过kryo序列化后变成了3.01K.可见应该只是序列化上的优化，压缩率比较小.

计算Coherence对象大小的程序

另外附上一个计算coherence对象大小的程序

package coherencetest;

import java.text.DecimalFormat;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;

import javax.management.MBeanServer;
import javax.management.MBeanServerFactory;
import javax.management.ObjectName;

import com.tangosol.net.CacheFactory;

import java.io.IOException;

import javax.management.MBeanServerConnection;
import javax.management.remote.JMXConnector;
import javax.management.remote.JMXConnectorFactory;
import javax.management.remote.JMXServiceURL;

public class CalculateTheSizeOfPeopleCache {

@SuppressWarnings({ "unchecked", "rawtypes" })
private void run() throws Exception {

// Enable JMX support in this Coherence data grid session...
System.setProperty("tangosol.coherence.management", "all");

// Create a sample cache just to access the data grid...
CacheFactory.getCache(MBeanServerFactory.class.getName());

// Gets the JMX server from Coherence data grid...
MBeanServer jmxServer = getJMXServer();
System.out.println(jmxServer.toString());
if (jmxServer != null)
System.out.println("can not get jmxServer");

//MBeanServerConnection jmxServer = getJMXServer();

// Creates a internal data structure that would maintain
// the statistics from each cache in the data grid...
Map cacheList = new TreeMap();
Set jmxObjectList = jmxServer.queryNames(new ObjectName("Coherence:type=Cache,*"), null);
if (jmxObjectList !=null) {
System.out.println("can not get jmxOBjectList");
System.out.println(jmxObjectList.size());
}
for (Object jmxObject : jmxObjectList) {
System.out.println("Enter");
ObjectName jmxObjectName = (ObjectName) jmxObject;
String cacheName = jmxObjectName.getKeyProperty("name");
if (cacheName.equals(MBeanServerFactory.class.getName())) {
continue;
} else {
cacheList.put(cacheName, new Statistics(cacheName));
}
}

// Updates the internal data structure with statistic data
// retrieved from caches inside the in-memory data grid...
Set<String> cacheNames = cacheList.keySet();
for (String cacheName : cacheNames) {
Set resultSet = jmxServer.queryNames(
new ObjectName("Coherence:type=Cache,name=" + cacheName + ",*"), null);
for (Object resultSetRef : resultSet) {
ObjectName objectName = (ObjectName) resultSetRef;
if (objectName.getKeyProperty("tier").equals("back")) {
int unit = (Integer) jmxServer.getAttribute(objectName, "Units");
int size = (Integer) jmxServer.getAttribute(objectName, "Size");
Statistics statistics = (Statistics) cacheList.get(cacheName);
statistics.incrementUnit(unit);
statistics.incrementSize(size);
cacheList.put(cacheName, statistics);
}
}
}

// Finally... print the objects from the internal data
// structure that represents the statistics from caches...
cacheNames = cacheList.keySet();
for (String cacheName : cacheNames) {
Statistics estatisticas = (Statistics) cacheList.get(cacheName);
System.out.println(estatisticas);
}

}
/*
public static MBeanServerConnection getJMXServer() throws IOException {
JMXServiceURL url = new JMXServiceURL("service://...");
JMXConnector jmxc = JMXConnectorFactory.connect(url, null);
return jmxc.getMBeanServerConnection();
}
*/
public MBeanServer getJMXServer() {
MBeanServer jmxServer = null;
for (Object jmxServerRef : MBeanServerFactory.findMBeanServer(null)) {
jmxServer = (MBeanServer) jmxServerRef;
System.out.println(jmxServer.getDefaultDomain().toString());
if (jmxServer.getDefaultDomain().equals(DEFAULT_DOMAIN) || DEFAULT_DOMAIN.length() == 0) {
break;
}
jmxServer = null;
}
if (jmxServer == null) {
jmxServer = MBeanServerFactory.createMBeanServer(DEFAULT_DOMAIN);
}
return jmxServer;
}

private class Statistics {

private long unit;
private long size;
private String cacheName;

public Statistics(String cacheName) {
this.cacheName = cacheName;
}

public void incrementUnit(long unit) {
this.unit += unit;
}

public void incrementSize(long size) {
this.size += size;
}

public long getUnit() {
return unit;
}

public long getSize() {
return size;
}

public double getUnitInMB() {
return unit / (1024.0 * 1024.0);
}

public double getAverageSize() {
return size == 0 ? 0 : unit / size;
}

public String toString() {
StringBuffer sb = new StringBuffer();
sb.append("\nCache Statistics of '").append(cacheName).append("':\n");
sb.append(" - Total Entries of Cache -----> " + getSize()).append("\n");
sb.append(" - Used Memory (Bytes) --------> " + getUnit()).append("\n");
sb.append(" - Used Memory (MB) -----------> " + FORMAT.format(getUnitInMB())).append("\n");
sb.append(" - Object Average Size --------> " + FORMAT.format(getAverageSize())).append("\n");
return sb.toString();
}

}

public static void main(String[] args) throws Exception {
new CalculateTheSizeOfPeopleCache().run();
}

public static final DecimalFormat FORMAT = new DecimalFormat("###.###");
public static final String DEFAULT_DOMAIN = "DefaultDomain";
public static final String DOMAIN_NAME = "Coherence";
//public static final String DOMAIN_NAME = "enie's cluster";

}

需要的条件是:

• 不能以Coherence Extend Client的方式连入集群
• 需要开启-Dtangosol.coherence.management.remote=true  -Dtangosol.coherence.management=all 参数
• 需要和cluster环境保持一直,生产就是生产,开发就是开发,-Dtangosol.coherence.mode=prod

输出如下:

Cache Statistics of 'POFSample':
- Total Entries of Cache -----> 10001
- Used Memory (Bytes) --------> 10001
- Used Memory (MB) -----------> 0.01
- Object Average Size --------> 1