Commit 5fc372b7 authored by Tigran Mkrtchyan's avatar Tigran Mkrtchyan
Browse files

nfs: add Hazelcast-based distributed lock manager

Motivation:
lock management across multiple nfs servers.

Modification:
add Hazelcaset-based subclass of AbstractLockManager. Classes which are
used in lock implementations are updated to implement java.io.Serializible.

NOTICE, that we do not include any (except of hazelcast's own default
config) configuration. Thus end-application can itself decide how
Hazelcast is instantiated and configured.

Result:
locks are honored across set of nfs servers, if configured.

Acked-by: Paul Millar
Target: master
parent 638e6b06
......@@ -45,6 +45,14 @@
</manifestEntries>
</archive>
</configuration>
<executions>
<!-- produce jar for unit test that is used by other mudules -->
<execution>
<goals>
<goal>test-jar</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
......
......@@ -20,6 +20,7 @@
package org.dcache.nfs.v4;
import com.google.common.base.MoreObjects;
import java.io.Serializable;
import org.dcache.nfs.status.BadSeqidException;
import org.dcache.nfs.v4.xdr.seqid4;
import org.dcache.nfs.v4.xdr.state_owner4;
......@@ -27,7 +28,9 @@ import org.dcache.nfs.v4.xdr.state_owner4;
/**
* Owner associated with the open/lock operations.
*/
public class StateOwner {
public class StateOwner implements Serializable {
private static final long serialVersionUID = -4712959403595550903L;
/**
* Per owner sequence to serialize opens with nfsv4.0
......
......@@ -20,11 +20,14 @@
package org.dcache.nfs.v4.nlm;
import com.google.common.base.MoreObjects;
import java.io.Serializable;
import org.dcache.nfs.v4.StateOwner;
import org.dcache.nfs.v4.xdr.nfs4_prot;
import org.dcache.nfs.v4.xdr.nfs_lock_type4;
public class NlmLock {
public class NlmLock implements Serializable{
private static final long serialVersionUID = -839338915510175006L;
/**
* Opaque object that identifies the host or process that is holding the
......
......@@ -20,8 +20,11 @@
package org.dcache.nfs.v4.xdr;
import org.dcache.xdr.*;
import java.io.IOException;
import java.io.Serializable;
public class clientid4 extends uint64_t {
public class clientid4 extends uint64_t implements Serializable {
private static final long serialVersionUID = -1680592874479756784L;
public clientid4() {
}
......
......@@ -25,13 +25,16 @@ import java.util.Objects;
import com.google.common.base.MoreObjects;
import com.google.common.io.BaseEncoding;
import java.io.Serializable;
import org.dcache.xdr.OncRpcException;
import org.dcache.xdr.XdrAble;
import org.dcache.xdr.XdrDecodingStream;
import org.dcache.xdr.XdrEncodingStream;
public class state_owner4 implements XdrAble {
public class state_owner4 implements XdrAble, Serializable {
private static final long serialVersionUID = -7146149387990285155L;
public clientid4 clientid;
public byte [] owner;
......
......@@ -20,8 +20,11 @@
package org.dcache.nfs.v4.xdr;
import org.dcache.xdr.*;
import java.io.IOException;
import java.io.Serializable;
public class uint64_t implements XdrAble {
public class uint64_t implements XdrAble, Serializable {
private static final long serialVersionUID = -8761508414722135859L;
public long value;
......
/*
* Copyright (c) 2009 - 2012 Deutsches Elektronen-Synchroton,
* Copyright (c) 2009 - 2017 Deutsches Elektronen-Synchroton,
* Member of the Helmholtz Association, (DESY), HAMBURG, GERMANY
*
* This library is free software; you can redistribute it and/or modify
......@@ -19,13 +19,16 @@
*/
package org.dcache.utils;
import java.io.Serializable;
import java.util.Arrays;
/**
* A helper class for opaque data manipulations.
* Enabled opaque date to be used as a key in {@link java.util.Collection}
*/
public class Opaque {
public class Opaque implements Serializable {
private static final long serialVersionUID = 1532238396149112674L;
private final byte[] _opaque;
......
......@@ -319,7 +319,7 @@ public class SimpleLmTest {
nlm.test(file1, lock3);
}
private static class LockBuilder {
public static class LockBuilder {
private long offset;
private long length;
......
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.dcache</groupId>
<artifactId>nfs4j</artifactId>
<version>0.16.0-SNAPSHOT</version>
</parent>
<name>Distributed lock manager</name>
<groupId>org.dcache</groupId>
<artifactId>nfs4j-dlm</artifactId>
<packaging>jar</packaging>
<dependencies>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<!-- to get console logging during development/debug -->
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.hazelcast</groupId>
<artifactId>hazelcast</artifactId>
</dependency>
<dependency>
<groupId>com.hazelcast</groupId>
<artifactId>hazelcast-client</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</dependency>
<dependency>
<groupId>org.dcache</groupId>
<artifactId>nfs4j-core</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.dcache</groupId>
<artifactId>nfs4j-core</artifactId>
<version>${project.version}</version>
<type>test-jar</type>
<scope>test</scope>
</dependency>
</dependencies>
</project>
/*
* Copyright (c) 2017 - 2018 Deutsches Elektronen-Synchroton,
* Member of the Helmholtz Association, (DESY), HAMBURG, GERMANY
*
* This library is free software; you can redistribute it and/or modify
* it under the terms of the GNU Library General Public License as
* published by the Free Software Foundation; either version 2 of the
* License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this program (see the file COPYING.LIB for more
* details); if not, write to the Free Software Foundation, Inc.,
* 675 Mass Ave, Cambridge, MA 02139, USA.
*/
package org.dcache.nfs.v4.nlm;
import com.google.common.io.BaseEncoding;
import com.hazelcast.core.HazelcastInstance;
import com.hazelcast.core.MultiMap;
import java.util.Collection;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.Lock;
/**
* An implementation of {@link LockManager} which uses Hazelcast's distributed
* {@link MultiMap} to store locks.
*
* <p>
* Example:
*
* <pre>
* HazelcastInstance hz = ...;
* LockManager lm1 = new DistributedLockManager(hz, "distributed-byte-range-lock");
* LockManager lm2 = new DistributedLockManager(hz, "distributed-byte-range-lock");
* </pre>
*
* The {@code lm1} and {@code lm2} will share the same set of locks as long as they
* connected to the same Hazelcast cluster.
*
* @since 0.16
*/
public class DistributedLockManager extends AbstractLockManager {
private final MultiMap<String, NlmLock> locks;
/**
* Create a new {@code DistributedLockManager} with a given {@code name}.
* The other instances with the same name will share the same back-end store and,
* as a result, will see the same set of locks.
*
* @param hz reference to Haselcast instance.
* @param name name of the lock manager.
*/
public DistributedLockManager(HazelcastInstance hz, String name) {
locks = hz.getMultiMap(name);
}
@Override
protected Lock getObjectLock(byte[] objId) {
String key = BaseEncoding.base16().upperCase().encode(objId);
return new Lock() {
@Override
public void lock() {
locks.lock(key);
}
@Override
public void lockInterruptibly() throws InterruptedException {
locks.tryLock(key, Long.MAX_VALUE, TimeUnit.DAYS);
}
@Override
public boolean tryLock() {
return locks.tryLock(key);
}
@Override
public boolean tryLock(long time, TimeUnit unit) throws InterruptedException {
return locks.tryLock(key, time, unit);
}
@Override
public void unlock() {
locks.unlock(key);
}
@Override
public Condition newCondition() {
throw new UnsupportedOperationException("Not supported yet.");
}
};
}
/**
* Get collection of currently used active locks on the object.
* @param objId object id.
* @return collection of active locks.
*/
@Override
protected Collection<NlmLock> getActiveLocks(byte[] objId) {
String key = objIdToKey(objId);
return locks.get(key);
}
@Override
protected void add(byte[] objId, NlmLock lock) {
String key = objIdToKey(objId);
locks.put(key, lock);
}
@Override
protected boolean remove(byte[] objId, NlmLock lock) {
String key = objIdToKey(objId);
return locks.remove(key, lock);
}
@Override
protected void addAll(byte[] objId, Collection<NlmLock> locks) {
String key = objIdToKey(objId);
locks.forEach(l -> this.locks.put(key, l));
}
@Override
protected void removeAll(byte[] objId, Collection<NlmLock> locks) {
String key = objIdToKey(objId);
locks.forEach(l -> this.locks.remove(key, l));
}
private static String objIdToKey(byte[] objId) {
return BaseEncoding.base64().omitPadding().encode(objId);
}
}
package org.dcache.nfs.v4.nlm;
import com.hazelcast.client.HazelcastClient;
import com.hazelcast.client.config.ClientConfig;
import com.hazelcast.core.Hazelcast;
import com.hazelcast.core.HazelcastInstance;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import static org.junit.Assert.fail;
public class DistributedLockManagerTest {
private HazelcastInstance hzSerrver;
private HazelcastInstance hzClient;
private LockManager lm1;
private LockManager lm2;
private byte[] file1;
@Before
public void setUp() throws Exception {
file1 = "file1".getBytes(StandardCharsets.UTF_8);
hzSerrver = Hazelcast.newHazelcastInstance();
ClientConfig config = new ClientConfig();
config.getNetworkConfig().setAddresses(Arrays.asList(
hzSerrver.getCluster().getLocalMember().getAddress().getHost() +
":" + hzSerrver.getCluster().getLocalMember().getAddress().getPort()));
hzClient = HazelcastClient.newHazelcastClient(config);
lm1 = new DistributedLockManager(hzClient, "distributed-byte-range-lock");
lm2 = new DistributedLockManager(hzClient, "distributed-byte-range-lock");
}
@Test(expected = LockDeniedException.class)
public void testConflictingLockDifferentOwner() throws LockException {
NlmLock lock1 = new SimpleLmTest.LockBuilder()
.withOwner("owner1")
.from(0)
.length(1)
.forRead()
.build();
lm1.lock(file1, lock1);
NlmLock lock2 = new SimpleLmTest.LockBuilder()
.withOwner("owner2")
.from(0)
.length(1)
.forWrite()
.build();
lm2.lock(file1, lock2);
}
@Test
public void testConflictingLockSameOwner() throws LockException {
NlmLock lock1 = new SimpleLmTest.LockBuilder()
.withOwner("owner1")
.from(0)
.length(1)
.forRead()
.build();
lm1.lock(file1, lock1);
NlmLock lock2 = new SimpleLmTest.LockBuilder()
.withOwner("owner1")
.from(0)
.length(1)
.forWrite()
.build();
lm2.lock(file1, lock2);
try {
NlmLock lock3 = new SimpleLmTest.LockBuilder()
.withOwner("owner2")
.from(0)
.length(1)
.forWrite()
.build();
lm1.test(file1, lock3);
fail("Lock on lm1 is lost");
} catch (LockDeniedException e) {
// pass
}
}
@After
public void tearDown() {
if (hzClient != null) {
hzClient.shutdown();
}
if (hzSerrver != null) {
hzSerrver.shutdown();
}
}
}
......@@ -39,6 +39,7 @@
<module>core</module>
<module>basic-client</module>
<module>spring</module>
<module>dlm</module>
</modules>
<build>
......@@ -135,6 +136,16 @@
<artifactId>oncrpc4j-core</artifactId>
<version>2.7.0</version>
</dependency>
<dependency>
<groupId>com.hazelcast</groupId>
<artifactId>hazelcast</artifactId>
<version>3.9.2</version>
</dependency>
<dependency>
<groupId>com.hazelcast</groupId>
<artifactId>hazelcast-client</artifactId>
<version>3.9.2</version>
</dependency>
</dependencies>
</dependencyManagement>
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment