573 lines
22 KiB
Plaintext
573 lines
22 KiB
Plaintext
|
///////////////////////////////////////////////////////////////////////////////
|
||
|
|
//
|
||
|
|
// Microsoft Research Singularity
|
||
|
|
//
|
||
|
|
// Copyright (c) Microsoft Corporation. All rights reserved.
|
||
|
|
//
|
||
|
|
// File: Libraries\Resiliency\DirectoryServiceProxy.sg
|
||
|
|
//
|
||
|
|
// Note: Intermediary of DirectoryServiceContract
|
||
|
|
//
|
||
|
|
using System;
|
||
|
|
using System.Collections;
|
||
|
|
using System.Threading;
|
||
|
|
using Microsoft.SingSharp;
|
||
|
|
using Microsoft.Singularity;
|
||
|
|
using Microsoft.Singularity.Channels;
|
||
|
|
using Microsoft.Singularity.Directory;
|
||
|
|
using Microsoft.Singularity.ServiceManager;
|
||
|
|
using Microsoft.Singularity.Services;
|
||
|
|
|
||
|
|
namespace Microsoft.Singularity.Resiliency
|
||
|
|
{
|
||
|
|
internal sealed class DirectoryServiceProxy : IRunnable, IRecoveryAware
|
||
|
|
{
|
||
|
|
private bool recovery;
|
||
|
|
private Object! recoveryLock;
|
||
|
|
private ServiceProxy! manager;
|
||
|
|
private IList! providerList;
|
||
|
|
private JournaletFactory! factory;
|
||
|
|
private PersistentMemory! objectStore;
|
||
|
|
private TRef<DirectoryServiceContract.Imp:Ready> rootDsRef;
|
||
|
|
private TRef<DirectoryServiceContract.Exp:Ready> myDsRef;
|
||
|
|
private TRef<ServiceProxyContract.Exp:Ready> proxyRef;
|
||
|
|
private TRef<ThreadTerminationContract.Exp:Start> signalRef;
|
||
|
|
private TRef<SignalContract.Imp:Start> senderRef;
|
||
|
|
private TRef<SignalContract.Exp:Start> receiverRef;
|
||
|
|
|
||
|
|
/// dep Channel to the (root) directory service. This channel is
|
||
|
|
/// necessary to intercept the connection request from a client
|
||
|
|
/// to the server.
|
||
|
|
/// fep Channel from the target server. This channel acts as the
|
||
|
|
/// directory service for the server.
|
||
|
|
internal DirectoryServiceProxy(ServiceProxy! proxy,
|
||
|
|
JournaletFactory! factory,
|
||
|
|
[Claims]DirectoryServiceContract.Imp:Ready! dep,
|
||
|
|
[Claims]DirectoryServiceContract.Exp:Ready! fep,
|
||
|
|
[Claims]ServiceProxyContract.Exp:Ready! xep)
|
||
|
|
{
|
||
|
|
SignalContract.Imp! imp;
|
||
|
|
SignalContract.Exp! exp;
|
||
|
|
|
||
|
|
this.manager = proxy;
|
||
|
|
this.factory = factory;
|
||
|
|
this.providerList = new ArrayList();
|
||
|
|
this.objectStore = new PersistentMemory();
|
||
|
|
this.recoveryLock = new Object();
|
||
|
|
|
||
|
|
this.myDsRef = new TRef<DirectoryServiceContract.Exp:Ready>(fep);
|
||
|
|
this.proxyRef = new TRef<ServiceProxyContract.Exp:Ready>(xep);
|
||
|
|
this.rootDsRef = new TRef<DirectoryServiceContract.Imp:Ready>(dep);
|
||
|
|
|
||
|
|
SignalContract.NewChannel(out imp, out exp);
|
||
|
|
senderRef = new TRef<SignalContract.Imp:Start>(imp);
|
||
|
|
receiverRef = new TRef<SignalContract.Exp:Start>(exp);
|
||
|
|
}
|
||
|
|
|
||
|
|
public void Signal([Claims]ThreadTerminationContract.Exp:Start! ep)
|
||
|
|
{
|
||
|
|
signalRef = new TRef<ThreadTerminationContract.Exp:Start>(ep);
|
||
|
|
}
|
||
|
|
|
||
|
|
// Proxy recovers from server failure by itself.
|
||
|
|
public void Run()
|
||
|
|
{
|
||
|
|
DirectoryServiceContract.Exp:Ready newEP;
|
||
|
|
ServiceProxyContract.Exp:Ready newPx;
|
||
|
|
SignalContract.Imp:Start! signal;
|
||
|
|
|
||
|
|
for (;;) {
|
||
|
|
DebugStub.Print("DPx: HandleProxyService\n");
|
||
|
|
if (!HandleProxyService()) {
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
|
||
|
|
manager.RecoverService(out newEP, out newPx);
|
||
|
|
|
||
|
|
if (newEP == null || newPx == null) {
|
||
|
|
delete newEP;
|
||
|
|
delete newPx;
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
DebugStub.Print("DPx: Replace DS\n");
|
||
|
|
// replace with the new endpoint
|
||
|
|
delete myDsRef.Acquire();
|
||
|
|
myDsRef.Release(newEP);
|
||
|
|
DebugStub.Print("DPx: Replace proxy\n");
|
||
|
|
delete proxyRef.Acquire();
|
||
|
|
proxyRef.Release(newPx);
|
||
|
|
}
|
||
|
|
|
||
|
|
delete signalRef.Acquire();
|
||
|
|
}
|
||
|
|
|
||
|
|
///
|
||
|
|
/// Handles DirectoryService and ServiceProvider
|
||
|
|
///
|
||
|
|
private bool HandleProxyService()
|
||
|
|
{
|
||
|
|
DirectoryServiceContract.Exp:Ready! myDS;
|
||
|
|
ServiceProxyContract.Exp:Ready! proxy;
|
||
|
|
DirectoryServiceContract.Imp:Ready! rootDS;
|
||
|
|
ThreadTerminationContract.Exp:Start! signalFromManager;
|
||
|
|
SignalContract.Exp:Start! signalInternal;
|
||
|
|
int currentGeneration = -1;
|
||
|
|
int previousGeneration = 0;
|
||
|
|
Random random = new Random();
|
||
|
|
|
||
|
|
if (signalRef == null) {
|
||
|
|
DebugStub.Print("DPx: signal is not set!\n");
|
||
|
|
return recovery;
|
||
|
|
}
|
||
|
|
|
||
|
|
myDS = myDsRef.Acquire();
|
||
|
|
rootDS = rootDsRef.Acquire();
|
||
|
|
proxy = proxyRef.Acquire();
|
||
|
|
signalFromManager = signalRef.Acquire();
|
||
|
|
signalInternal = receiverRef.Acquire();
|
||
|
|
recovery = false;
|
||
|
|
|
||
|
|
for (;;) {
|
||
|
|
switch receive {
|
||
|
|
// BEGIN SWR LEVEL0
|
||
|
|
// From the service process
|
||
|
|
case myDS.Register(path, imp):
|
||
|
|
{
|
||
|
|
string! pathName;
|
||
|
|
ServiceProviderProxy provider;
|
||
|
|
ServiceProviderContract.Imp! client;
|
||
|
|
ServiceProviderContract.Exp! server;
|
||
|
|
ServiceProviderContract.Imp:Start! oldEp;
|
||
|
|
|
||
|
|
pathName = Bitter.ToString2(path);
|
||
|
|
DebugStub.Print("DPx: Register '{0}'\n",
|
||
|
|
__arglist(pathName));
|
||
|
|
|
||
|
|
provider = LookupProviderProxy(pathName);
|
||
|
|
if (provider != null) { // We already have the one.
|
||
|
|
provider.Stop(out oldEp);
|
||
|
|
delete oldEp;
|
||
|
|
provider.Start(imp);
|
||
|
|
myDS.SendAckRegister();
|
||
|
|
delete path;
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
|
||
|
|
// Register the JP's endpoint to the NS.
|
||
|
|
ServiceProviderContract.NewChannel(out client,
|
||
|
|
out server);
|
||
|
|
rootDS.SendRegister(path, client);
|
||
|
|
|
||
|
|
// BEGIN SWR LEVEL1
|
||
|
|
switch receive {
|
||
|
|
// From the root DS
|
||
|
|
case rootDS.AckRegister():
|
||
|
|
{
|
||
|
|
myDS.SendAckRegister();
|
||
|
|
CreateProviderProxy(pathName, imp, server);
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
// From the root DS
|
||
|
|
case rootDS.NakRegister(rejected, error):
|
||
|
|
{
|
||
|
|
myDS.SendNakRegister(imp, error);
|
||
|
|
|
||
|
|
delete rejected;
|
||
|
|
delete server;
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
// From the root DS
|
||
|
|
case rootDS.NakRegisterReparse(npath, rest,
|
||
|
|
link, rejected):
|
||
|
|
{
|
||
|
|
DebugStub.Print("DPx: NakRegisterReparse\n");
|
||
|
|
myDS.SendNakRegisterReparse(npath, rest,
|
||
|
|
link, imp);
|
||
|
|
delete rejected;
|
||
|
|
delete server;
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
// internal communication
|
||
|
|
case signalFromManager.Stop():
|
||
|
|
{
|
||
|
|
signalFromManager.SendAckStop();
|
||
|
|
delete server;
|
||
|
|
delete imp;
|
||
|
|
goto exit;
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
// internal communication
|
||
|
|
case signalFromManager.ChannelClosed():
|
||
|
|
{
|
||
|
|
delete server;
|
||
|
|
delete imp;
|
||
|
|
goto exit;
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
case unsatisfiable:
|
||
|
|
{
|
||
|
|
// protocol mismatch
|
||
|
|
delete server;
|
||
|
|
delete imp;
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
} // END SWR LEVEL1
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
// From the service process
|
||
|
|
case myDS.Deregister(path):
|
||
|
|
{
|
||
|
|
string! pathName;
|
||
|
|
ServiceProviderProxy provider;
|
||
|
|
|
||
|
|
pathName = Bitter.ToString2(path);
|
||
|
|
DebugStub.Print("DPx: Deregister '{0}'\n",
|
||
|
|
__arglist(pathName));
|
||
|
|
rootDS.SendDeregister(path);
|
||
|
|
|
||
|
|
switch receive {
|
||
|
|
// From the root DS
|
||
|
|
case rootDS.AckDeregister(ep):
|
||
|
|
{
|
||
|
|
provider = LookupProviderProxy(pathName);
|
||
|
|
if (provider != null) {
|
||
|
|
ServiceProviderContract.Imp:Start! endpoint;
|
||
|
|
provider.Terminate(out endpoint);
|
||
|
|
myDS.SendAckDeregister(endpoint);
|
||
|
|
RemoveProviderProxy(provider);
|
||
|
|
}
|
||
|
|
else {
|
||
|
|
myDS.SendNakDeregister(ErrorCode.NotFound);
|
||
|
|
}
|
||
|
|
delete ep;
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
// From the root DS
|
||
|
|
case rootDS.NakDeregister(error):
|
||
|
|
{
|
||
|
|
myDS.SendNakDeregister(error);
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
case rootDS.NakDeregisterReparse(npath, rest, link):
|
||
|
|
{
|
||
|
|
myDS.SendNakDeregisterReparse(npath, rest,
|
||
|
|
link);
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
} // END SWR LEVEL1
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
|
||
|
|
//
|
||
|
|
// PRef style
|
||
|
|
//NOTE: Not tested yet
|
||
|
|
case proxy.Allocate(obj):
|
||
|
|
{
|
||
|
|
proxy.SendNakAllocate(obj);
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
case proxy.Deallocate(id):
|
||
|
|
{
|
||
|
|
proxy.SendNakDeallocate();
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
case proxy.AcquireObject(id):
|
||
|
|
{
|
||
|
|
proxy.SendNakAcquireObject();
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
case proxy.ReleaseObject(obj):
|
||
|
|
{
|
||
|
|
proxy.SendNakReleaseObject();
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
|
||
|
|
//
|
||
|
|
// Serialization style
|
||
|
|
//
|
||
|
|
case proxy.Upload(data):
|
||
|
|
{
|
||
|
|
proxy.SendAckUpload();
|
||
|
|
objectStore.Store(data);
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
case proxy.Download():
|
||
|
|
{
|
||
|
|
byte[] in ExHeap buffer = objectStore.Restore();
|
||
|
|
if (buffer != null) {
|
||
|
|
proxy.SendAckDownload(buffer);
|
||
|
|
}
|
||
|
|
else {
|
||
|
|
proxy.SendNakDownload();
|
||
|
|
}
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
|
||
|
|
//
|
||
|
|
// Checkpoint update protocol
|
||
|
|
//NOTE: not tested yet
|
||
|
|
case proxy.Suspend(/* journalet id */):
|
||
|
|
{
|
||
|
|
proxy.SendAckSuspend(currentGeneration);
|
||
|
|
SuspendProviderProxies();
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
case proxy.Resume(generation):
|
||
|
|
{
|
||
|
|
proxy.SendAckResume();
|
||
|
|
if (generation == currentGeneration) {
|
||
|
|
ResumeProviderProxies();
|
||
|
|
}
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
case proxy.Update(generation):
|
||
|
|
{
|
||
|
|
proxy.SendAckUpdate();
|
||
|
|
if (generation == currentGeneration) {
|
||
|
|
objectStore.Flush();
|
||
|
|
UpdateProviderProxies();
|
||
|
|
previousGeneration = currentGeneration;
|
||
|
|
do {
|
||
|
|
currentGeneration = random.Next();
|
||
|
|
} while (currentGeneration == previousGeneration);
|
||
|
|
}
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
|
||
|
|
case proxy.ChannelClosed():
|
||
|
|
{
|
||
|
|
DebugStub.Print("DPx: Server channel closed. " +
|
||
|
|
"Recover.\n");
|
||
|
|
objectStore.Undo();
|
||
|
|
recovery = true;
|
||
|
|
goto exit;
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
// From the service process
|
||
|
|
case myDS.ChannelClosed():
|
||
|
|
{
|
||
|
|
DebugStub.Print("DPx: Server channel closed. " +
|
||
|
|
"Recover.\n");
|
||
|
|
objectStore.Undo();
|
||
|
|
recovery = true;
|
||
|
|
goto exit;
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
// From the root DS
|
||
|
|
case rootDS.ChannelClosed():
|
||
|
|
{
|
||
|
|
DebugStub.Print("DPx: DS channel closed. Exit.\n");
|
||
|
|
goto exit;
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
case signalFromManager.Stop():
|
||
|
|
{
|
||
|
|
TerminateProviderProxies();
|
||
|
|
signalFromManager.SendAckStop();
|
||
|
|
goto exit;
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
case signalFromManager.ChannelClosed():
|
||
|
|
{
|
||
|
|
TerminateProviderProxies();
|
||
|
|
goto exit;
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
case signalInternal.Stop():
|
||
|
|
{
|
||
|
|
recovery = true;
|
||
|
|
signalInternal.SendAckStop();
|
||
|
|
goto exit;
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
// this works with the proxy initiation style
|
||
|
|
case signalInternal.Update():
|
||
|
|
{
|
||
|
|
DebugStub.Print("DPx: Update Master\n");
|
||
|
|
objectStore.Flush();
|
||
|
|
signalInternal.SendAckUpdate();
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
case signalInternal.ChannelClosed():
|
||
|
|
{
|
||
|
|
TerminateProviderProxies();
|
||
|
|
goto exit;
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
} // END SWR LEVEL0
|
||
|
|
} // END OF LOOP
|
||
|
|
exit:
|
||
|
|
signalRef.Release(signalFromManager);
|
||
|
|
receiverRef.Release(signalInternal);
|
||
|
|
|
||
|
|
myDsRef.Release(myDS);
|
||
|
|
rootDsRef.Release(rootDS);
|
||
|
|
proxyRef.Release(proxy);
|
||
|
|
|
||
|
|
return recovery;
|
||
|
|
} // HandleDirectoryService
|
||
|
|
|
||
|
|
private void CreateProviderProxy(string! path,
|
||
|
|
[Claims]ServiceProviderContract.Imp:Start! provider,
|
||
|
|
[Claims]ServiceProviderContract.Exp:Start! myProvider)
|
||
|
|
{
|
||
|
|
ServiceProviderProxy pp = new ServiceProviderProxy(this, path,
|
||
|
|
factory,
|
||
|
|
myProvider);
|
||
|
|
lock (providerList) {
|
||
|
|
providerList.Add(pp);
|
||
|
|
}
|
||
|
|
pp.Start(provider);
|
||
|
|
}
|
||
|
|
|
||
|
|
private ServiceProviderProxy LookupProviderProxy(string! path)
|
||
|
|
{
|
||
|
|
ServiceProviderProxy pp;
|
||
|
|
|
||
|
|
lock (providerList) {
|
||
|
|
foreach (Object obj in providerList) {
|
||
|
|
if (obj == null) {
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
pp = obj as ServiceProviderProxy;
|
||
|
|
if (pp == null) {
|
||
|
|
continue;
|
||
|
|
}
|
||
|
|
if (pp.Path == path) {
|
||
|
|
return pp;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return null;
|
||
|
|
}
|
||
|
|
|
||
|
|
private void RemoveProviderProxy(ServiceProviderProxy! sp)
|
||
|
|
{
|
||
|
|
lock (providerList) {
|
||
|
|
providerList.Remove(sp);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
private void TerminateProviderProxies()
|
||
|
|
{
|
||
|
|
ServiceProviderProxy pp;
|
||
|
|
ServiceProviderContract.Imp:Start! ep;
|
||
|
|
|
||
|
|
//DebugStub.Print("DPx: Enter TerminateProviderProxies\n");
|
||
|
|
lock (providerList) {
|
||
|
|
foreach (Object obj in providerList) {
|
||
|
|
if (obj == null) {
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
pp = obj as ServiceProviderProxy;
|
||
|
|
if (pp == null) {
|
||
|
|
continue;
|
||
|
|
}
|
||
|
|
pp.Terminate(out ep);
|
||
|
|
delete ep;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
//DebugStub.Print("DPx: Exit TerminateProviderProxies\n");
|
||
|
|
}
|
||
|
|
|
||
|
|
private void SuspendProviderProxies()
|
||
|
|
{
|
||
|
|
ServiceProviderProxy pp;
|
||
|
|
|
||
|
|
lock (providerList) {
|
||
|
|
foreach (Object obj in providerList) {
|
||
|
|
if (obj == null) {
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
pp = obj as ServiceProviderProxy;
|
||
|
|
if (pp == null) {
|
||
|
|
continue;
|
||
|
|
}
|
||
|
|
pp.Suspend();
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
private void ResumeProviderProxies()
|
||
|
|
{
|
||
|
|
ServiceProviderProxy pp;
|
||
|
|
|
||
|
|
lock (providerList) {
|
||
|
|
foreach (Object obj in providerList) {
|
||
|
|
if (obj == null) {
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
pp = obj as ServiceProviderProxy;
|
||
|
|
if (pp == null) {
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
pp.Resume();
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
private void UpdateProviderProxies()
|
||
|
|
{
|
||
|
|
ServiceProviderProxy pp;
|
||
|
|
|
||
|
|
lock (providerList) {
|
||
|
|
foreach (Object obj in providerList) {
|
||
|
|
if (obj == null) {
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
pp = obj as ServiceProviderProxy;
|
||
|
|
if (pp == null) {
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
pp.Flush();
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
/// <summary>
|
||
|
|
/// Signal to the main loop for recovering the server.
|
||
|
|
/// Invoked by ServiceProviderProxy.
|
||
|
|
/// </summary>
|
||
|
|
public void NotifyRecovery()
|
||
|
|
{
|
||
|
|
SignalContract.Imp:Start! signal;
|
||
|
|
|
||
|
|
if (!Monitor.TryEnter(recoveryLock)) {
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
|
||
|
|
if (!recovery) {
|
||
|
|
signal = senderRef.Acquire();
|
||
|
|
signal.SendStop();
|
||
|
|
switch receive {
|
||
|
|
case signal.AckStop():
|
||
|
|
break;
|
||
|
|
case unsatisfiable:
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
senderRef.Release(signal);
|
||
|
|
}
|
||
|
|
|
||
|
|
Monitor.Exit(recoveryLock);
|
||
|
|
}
|
||
|
|
|
||
|
|
public void UpdateCheckpoint()
|
||
|
|
{
|
||
|
|
SignalContract.Imp:Start! signal;
|
||
|
|
|
||
|
|
//DebugStub.Print("DPx: Enter UpdateCheckpoint\n");
|
||
|
|
signal = senderRef.Acquire();
|
||
|
|
SuspendProviderProxies();
|
||
|
|
signal.SendUpdate();
|
||
|
|
switch receive {
|
||
|
|
case signal.AckUpdate():
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
UpdateProviderProxies();
|
||
|
|
ResumeProviderProxies();
|
||
|
|
senderRef.Release(signal);
|
||
|
|
//DebugStub.Print("DPx: Exit UpdateCheckpoint\n");
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|