Commit 8577c51d authored by Tigran Mkrtchyan's avatar Tigran Mkrtchyan
Browse files

nfsv41: refactor NFSv41DeviceManager interface

Return a file layout instead of device. This will allow
to return different layout types and striping patters
depending on NFSv41DeviceManager implementations.

Match method names to NFS spec. This makes current code
incompatible with any existing implementations
( dCache is the only one which I aware about :) )
The NFSv4Client is included into NFSv41DeviceManager calls
to enable behavior change depending on client.

Acked-By: Gerd
Patch: http://rb.dcache.org/r/1753/
parent 901ffff8
package org.dcache.chimera.nfs.v4;
import java.util.Arrays;
import org.dcache.chimera.nfs.v4.xdr.deviceid4;
import org.dcache.chimera.nfs.v4.xdr.nfs4_prot;
/**
* Helper ( wrapper ) class for byte[] based deviceid4.
* required to be able to use byte[] as a key in the Collections
*
*/
public class DeviceID {
private final byte[] _id;
public DeviceID(byte[] id) {
_id = new byte[id.length];
System.arraycopy(id, 0, _id, 0, id.length);
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + Arrays.hashCode(_id);
return result;
}
@Override
public boolean equals(Object obj) {
if (this == obj) return true;
if( !(obj instanceof DeviceID)) return false;
final DeviceID other = (DeviceID) obj;
return Arrays.equals(_id, other._id);
}
public byte[] getId() {
byte[] id = new byte[_id.length];
System.arraycopy(_id, 0, id, 0, _id.length);
return id;
}
public static DeviceID valueOf(deviceid4 id) {
return new DeviceID(id.value);
}
public static DeviceID valueOf(int id) {
return new DeviceID(id2deviceid(id));
}
private static byte[] id2deviceid(int id) {
byte[] buf = Integer.toString(id).getBytes();
byte[] devData = new byte[nfs4_prot.NFS4_DEVICEID4_SIZE];
int len = Math.min(buf.length, nfs4_prot.NFS4_DEVICEID4_SIZE);
System.arraycopy(buf, 0, devData, 0, len);
return devData;
}
public deviceid4 toDeviceid4() {
return new deviceid4(_id);
}
@Override
public String toString() {
return Arrays.toString(_id);
}
}
......@@ -3,31 +3,24 @@
*/
package org.dcache.chimera.nfs.v4;
import org.dcache.chimera.nfs.v4.xdr.uint32_t;
import org.dcache.chimera.nfs.v4.xdr.layouttype4;
import org.dcache.chimera.nfs.v4.xdr.stateid4;
import org.dcache.chimera.nfs.v4.xdr.netaddr4;
import org.dcache.chimera.nfs.v4.xdr.nfsv4_1_file_layout_ds_addr4;
import org.dcache.chimera.nfs.v4.xdr.device_addr4;
import org.dcache.chimera.nfs.v4.xdr.multipath_list4;
import org.dcache.chimera.FsInode;
import org.dcache.chimera.FsInodeType;
import org.dcache.chimera.nfs.v4.xdr.*;
import org.dcache.xdr.OncRpcException;
import org.dcache.xdr.XdrBuffer;
import org.dcache.xdr.XdrEncodingStream;
import java.io.IOException;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.UnknownHostException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.concurrent.ConcurrentHashMap;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.dcache.xdr.OncRpcException;
import org.dcache.xdr.XdrEncodingStream;
import org.dcache.chimera.FsInode;
import org.dcache.xdr.XdrBuffer;
/**
*
......@@ -38,14 +31,18 @@ import org.dcache.xdr.XdrBuffer;
public class DeviceManager implements NFSv41DeviceManager {
/*
* reserved device for IO through MDS (for pnfs dot files)
*/
private static final deviceid4 MDS_ID = deviceidOf(0);
private static final Logger _log = Logger.getLogger(DeviceManager.class.getName());
/* hack for multiple pools */
Random _poolManager = new Random();
private final Random _deviceIdGenerator = new Random();
private final Map<DeviceID, NFS4IoDevice> _deviceMap =
new ConcurrentHashMap<DeviceID, NFS4IoDevice>();
private final Map<deviceid4, device_addr4> _deviceMap =
new ConcurrentHashMap<deviceid4, device_addr4>();
/*
* (non-Javadoc)
......@@ -53,56 +50,51 @@ public class DeviceManager implements NFSv41DeviceManager {
* @see org.dcache.chimera.nfsv4.NFSv41DeviceManager#getIoDeviceId(org.dcache.chimera.FsInode,
* int, java.net.InetAddress)
*/
public NFS4IoDevice getIoDevice(FsInode inode, int ioMode, InetAddress clientIp, stateid4 stateid) {
public Layout layoutGet(FsInode inode, int ioMode, NFS4Client client, stateid4 stateid)
throws IOException {
int id = _poolManager.nextInt(256);
++id; /* 0 is reserved */
DeviceID deviceId = DeviceID.valueOf(id);
device_addr4 deviceAddr;
deviceid4 deviceId;
_log.log(Level.FINEST, "generating new device: {0} ({1}) for stateid {2}",
new Object[] {deviceId, id, stateid}
);
if (inode.type() != FsInodeType.INODE) {
deviceId = MDS_ID;
} else {
InetAddress addr = null;
int id = _deviceIdGenerator.nextInt(256);
++id; /* 0 is reserved */
deviceId = deviceidOf(id);
try {
addr = InetAddress.getByName("127.0.0.1");
}catch(UnknownHostException e) {
// fine with me
}
_log.log(Level.FINEST, "generating new device: {0} ({1}) for stateid {2}",
new Object[]{deviceId, id, stateid});
//hard coded for now
device_addr4 deviceAddr = deviceAddrOf( new InetSocketAddress(addr, 2052) );
//hard coded for now
InetSocketAddress addr = new InetSocketAddress(InetAddress.getByName("127.0.0.1") , 2052 );
deviceAddr = deviceAddrOf(addr);
NFS4IoDevice newDevice = new NFS4IoDevice(deviceId , deviceAddr);
_deviceMap.put(deviceId, deviceAddr);
}
addIoDevice(newDevice, ioMode);
nfs_fh4 fh = new nfs_fh4(inode.toFullString().getBytes());
// -1 is special value, which means entire file
layout4 layout = Layout.getLayoutSegment(deviceId, fh, ioMode, 0, nfs4_prot.NFS4_UINT64_MAX);
return newDevice;
return new Layout(true, stateid, new layout4[]{layout});
}
/**
* Create IO dive and maps to given id
*
* @param device
* id to be mapped
* @param ioMode
*/
public void addIoDevice(NFS4IoDevice device, int ioMode ) {
_log.log(Level.FINEST, "add device: {0}", device.getDeviceId());
_deviceMap.put(device.getDeviceId(), device);
}
/*
* (non-Javadoc)
*
* @see org.dcache.chimera.nfsv4.NFSv41DeviceManager#getIoDevice(int)
* @see org.dcache.chimera.nfsv4.NFSv41DeviceManager#layoutGet(int)
*/
public NFS4IoDevice getIoDevice(DeviceID deviceId) {
public device_addr4 getDeviceInfo(NFS4Client client, deviceid4 deviceId) {
_log.log(Level.FINEST, "lookup for device: {0}", deviceId );
/* in case of MDS access we return the same interface which client already connected to */
if(deviceId.equals(MDS_ID)) {
return deviceAddrOf(client.getLocalAddress());
}
return _deviceMap.get(deviceId);
}
......@@ -110,25 +102,18 @@ public class DeviceManager implements NFSv41DeviceManager {
/*
* (non-Javadoc)
*
* @see org.dcache.chimera.nfsv4.NFSv41DeviceManager#getIoDeviceList()
* @see org.dcache.chimera.nfsv4.NFSv41DeviceManager#getDeviceList()
*/
public List<NFS4IoDevice> getIoDeviceList() {
List<NFS4IoDevice> deviceList = new ArrayList<NFS4IoDevice>();
deviceList.addAll(_deviceMap.values());
for(NFS4IoDevice device : deviceList ) {
_log.log(Level.FINEST, "known device: {0}", device.getDeviceId() );
}
return deviceList;
public List<deviceid4> getDeviceList(NFS4Client client) {
return new ArrayList<deviceid4>(_deviceMap.keySet());
}
/*
* (non-Javadoc)
*
* @see org.dcache.chimera.nfsv4.NFSv41DeviceManager#releaseDevice()
* @see org.dcache.chimera.nfsv4.NFSv41DeviceManager#layoutReturn()
*/
public void releaseDevice(stateid4 stateid) {
public void layoutReturn(NFS4Client client, stateid4 stateid) {
// I'am fine
_log.log(Level.FINEST, "release device for stateid {0}", stateid );
}
......@@ -168,11 +153,11 @@ public class DeviceManager implements NFSv41DeviceManager {
file_type.xdrEncode(xdr);
xdr.endEncoding();
}catch(OncRpcException e) {
// should never happen
// TODO: in case of - panic!
/* forced by interface, should never happen. */
throw new RuntimeException("Unexpected OncRpcException:", e);
}catch(IOException e) {
// should never happen
// TODO: in case of - panic!
/* forced by interface, should never happen. */
throw new RuntimeException("Unexpected IOException:", e);
}
ByteBuffer body = xdr.body();
......@@ -186,4 +171,14 @@ public class DeviceManager implements NFSv41DeviceManager {
return addr;
}
private static deviceid4 deviceidOf(int id) {
return new deviceid4(id2deviceid(id));
}
private static byte[] id2deviceid(int id) {
byte[] buf = Integer.toString(id).getBytes();
return buf;
}
}
/*
* This library is free software; you can redistribute it and/or modify
* it under the terms of the GNU Library General Public License as
* published by the Free Software Foundation; either version 2 of the
* License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this program (see the file COPYING.LIB for more
* details); if not, write to the Free Software Foundation, Inc.,
* 675 Mass Ave, Cambridge, MA 02139, USA.
*/
package org.dcache.chimera.nfs.v4;
import java.io.IOException;
import java.nio.ByteBuffer;
import org.dcache.chimera.nfs.ChimeraNFSException;
import org.dcache.chimera.nfs.v4.xdr.deviceid4;
import org.dcache.chimera.nfs.v4.xdr.stateid4;
import org.dcache.chimera.nfs.v4.xdr.layout4;
import org.dcache.chimera.nfs.v4.xdr.layout_content4;
import org.dcache.chimera.nfs.v4.xdr.layouttype4;
import org.dcache.chimera.nfs.v4.xdr.length4;
import org.dcache.chimera.nfs.v4.xdr.nfl_util4;
import org.dcache.chimera.nfs.v4.xdr.nfs4_prot;
import org.dcache.chimera.nfs.v4.xdr.nfs_fh4;
import org.dcache.chimera.nfs.v4.xdr.nfsstat4;
import org.dcache.chimera.nfs.v4.xdr.nfsv4_1_file_layout4;
import org.dcache.chimera.nfs.v4.xdr.offset4;
import org.dcache.chimera.nfs.v4.xdr.uint32_t;
import org.dcache.chimera.nfs.v4.xdr.uint64_t;
import org.dcache.xdr.OncRpcException;
import org.dcache.xdr.XdrBuffer;
import org.dcache.xdr.XdrEncodingStream;
/**
* A Layout defines how a file's data is organized on one or more storage devices.
* There are three layout types are defined in rfc 5661:
* <pre>
* NFSV4.1 FILE (rfc 5661)
* BLOCK VOLUME (rfc 5663)
* ODS OBJECT (rfc 5664)
* </pre>
*
* A Layout is expressed as an array of layout segments. The elements of the array
* MUST be sorted in ascending order of the value of the offset field of each element.
*
* <pre>
* segment1: offset 0, len 256K
* segment2: offset 256, len 256K
* ....
* segmentN: offset M, len 256K
* </pre>
*
* There MUST be no gaps or overlaps in the range between two successive elements.
*
* There are a two ways to stripe a file: across different devices or/and across
* multiple data servers defined as a single device. Striping within a single
* device is the one which supported by all clients (as of kernel 2.6.34 linux
* client does not support multiple layout segments and uses the first entry one only).
*/
public class Layout {
private final boolean _returnOnClose;
private final stateid4 _stateid;
private final layout4[] _layoutSegments;
public Layout(boolean returnOnClose, stateid4 stateid, layout4[] layoutSegments) {
_returnOnClose = returnOnClose;
_stateid = stateid;
_layoutSegments = layoutSegments;
}
/**
* Should the client return the layout prior close.
* @return <code>true</code> if a client should returns the layout prior close.
*/
public boolean returnOnClose() {
return _returnOnClose;
}
/**
* Get stateid associated with layout.
* @return stateid
*/
public stateid4 getStateid() {
return _stateid;
}
/**
* Get array of layout segments.
* @return layout segments.
*/
public layout4[] getLayoutSegments() {
return _layoutSegments;
}
private static layout_content4 getSegmentContent(deviceid4 deviceid, nfs_fh4 fh) throws ChimeraNFSException {
nfsv4_1_file_layout4 layout = new nfsv4_1_file_layout4();
layout.nfl_deviceid = deviceid;
/*
* The number of elements in nfl_fh_list MUST be one of three values:
*
* Zero. This means that filehandles used for each data
* server are the same as the filehandle returned by the OPEN
* operation from the metadata server.
*
* One. This means that every data server uses the same
* filehandle: what is specified in nfl_fh_list[0].
*
* The same number of elements in nflda_multipath_ds_list.
* Thus, in this case, when sending an I/O operation to any
* data server in nflda_multipath_ds_list[X], the filehandle
* in nfl_fh_list[X] MUST be used.
*/
layout.nfl_fh_list = new nfs_fh4[1];
layout.nfl_fh_list[0] = fh;
layout.nfl_first_stripe_index = new uint32_t(0);
layout.nfl_util = new nfl_util4(new uint32_t(NFSv4Defaults.NFS4_STRIPE_SIZE
& nfs4_prot.NFL4_UFLG_STRIPE_UNIT_SIZE_MASK));
layout.nfl_util = new nfl_util4(new uint32_t(layout.nfl_util.value.value
| nfs4_prot.NFL4_UFLG_DENSE));
//where the striping pattern starts
layout.nfl_pattern_offset = new offset4(new uint64_t(0));
XdrEncodingStream xdr = new XdrBuffer(512);
xdr.beginEncoding();
try {
layout.xdrEncode(xdr);
} catch (IOException e) {
throw new ChimeraNFSException(nfsstat4.NFS4ERR_SERVERFAULT,
"failed to encode layout body");
} catch (OncRpcException e) {
throw new ChimeraNFSException(nfsstat4.NFS4ERR_SERVERFAULT,
"failed to encode layout body");
}
xdr.endEncoding();
ByteBuffer xdrBody = xdr.body();
byte[] body = new byte[xdrBody.limit()];
xdrBody.get(body);
layout_content4 content = new layout_content4();
content.loc_type = layouttype4.LAYOUT4_NFSV4_1_FILES;
content.loc_body = body;
return content;
}
/**
* Create a layout segment for a given io mode, offset and length.
* The special value of length NFS4_UINT64_MAX corresponds to up to EOF.
* The valid values for <code>iomode</code> are LAYOUTIOMODE4_READ
* or LAYOUTIOMODE4_RW.
*
* @param deviceid on which segment is available.
* @param fh file handle to be used on data servers.
* @param iomode io mode for the segment.
* @param offset where segment starts.
* @param length segment length.
* @return layout segment
* @throws IOException
*/
public static layout4 getLayoutSegment(deviceid4 deviceid, nfs_fh4 fh, int iomode, long offset, long length)
throws IOException {
layout4 segment = new layout4();
segment.lo_offset = new offset4(new uint64_t(offset));
segment.lo_length = new length4(new uint64_t(length));
segment.lo_iomode = iomode;
segment.lo_content = new layout_content4();
segment.lo_content = getSegmentContent(deviceid, fh);
return segment;
}
}
/*
* $Id:NFS4IoDevice.java 140 2007-06-07 13:44:55Z tigran $
*/
package org.dcache.chimera.nfs.v4;
import org.dcache.chimera.nfs.v4.xdr.device_addr4;
/** Immutable */
public class NFS4IoDevice {
private final DeviceID _deviceId;
private final device_addr4 _addr;
public NFS4IoDevice(DeviceID id, device_addr4 addr) {
_deviceId = id;
_addr = addr;
}
public DeviceID getDeviceId() {
return _deviceId;
}
public int getDeviceType() {
return _addr.da_layout_type;
}
public device_addr4 getDeviceAddr() {
return _addr;
}
}
/*
* $Log: NFS4IoDevice.java,v $
* Revision 1.2 2006/11/16 21:00:48 tigran
* code cleanup
*
* Revision 1.1 2006/09/13 16:12:36 tigran
* added dummy device manager
*
*/
/*
* This library is free software; you can redistribute it and/or modify
* it under the terms of the GNU Library General Public License as
* published by the Free Software Foundation; either version 2 of the
* License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this program (see the file COPYING.LIB for more
* details); if not, write to the Free Software Foundation, Inc.,
* 675 Mass Ave, Cambridge, MA 02139, USA.
*/
package org.dcache.chimera.nfs.v4;
import org.dcache.chimera.FsInode;
import org.dcache.chimera.nfs.v4.xdr.device_addr4;
import org.dcache.chimera.nfs.v4.xdr.deviceid4;
import org.dcache.chimera.nfs.v4.xdr.stateid4;
import java.io.IOException;
import java.net.InetAddress;
import java.util.List;
import org.dcache.chimera.FsInode;
public interface NFSv41DeviceManager {
/**
* Get device address for a request. An state is associated with the returned device.
* To discard the state, <i>releaseDevice</i> have to be called.
* Get a file {@link Layout}. An state is associated with the returned layout.
* The layout is considered to be in use until <code>layoutReturn</code>
* method is not called.
*
* @param inode
* @param ioMode the value of the returned layout iomode. A ioMode of either
* LAYOUTIOMODE4_READ or LAYOUTIOMODE4_RW MUST be returned.
* @param clientIp
* @param stateid reflecting the correspondingly valid open, byte-range lock,
* @param client associated with LAYOUTGET call
* @param stateid reflecting the corresponding valid open, byte-range lock,
* or delegation stateid.
* @return
* @throws IOException
*/
public NFS4IoDevice getIoDevice(FsInode inode, int ioMode, InetAddress clientIp, stateid4 stateid)
public Layout layoutGet(FsInode inode, int ioMode, NFS4Client client, stateid4 stateid)
throws IOException;
/**
* Get storage device address information for the specified device {@link deviceid4}.
*
* Add a new device into list of known devices.
*
* @param device
* @param ioMode
*/
public void addIoDevice(NFS4IoDevice device, int ioMode);
/**
* Get device by deviceId.
*
* @param client
* @param deviceId
* @return NFS io device address
* @return device address
*/
public NFS4IoDevice getIoDevice(DeviceID deviceId);
public device_addr4 getDeviceInfo(NFS4Client client, deviceid4 deviceId) throws IOException;
/**
* Get list of all registered devices.
* Get list of all device IDs used bu server.
*
* @param client
* @return list of devices
*/
public List<NFS4IoDevice> getIoDeviceList();
public List<deviceid4> getDeviceList(NFS4Client client) throws IOException;
/**
* Release layout associated by state id.
* Release layout associated with state id.
*
* @param layout stateid
* @param client
* @param stateid