/////////////////////////////////////////////////////////////////////////////// // // Microsoft Research Singularity // // Copyright (c) Microsoft Corporation. All rights reserved. // // File: Libraries\Resiliency\JournalProducer.sg // // Note: Logging Worker Creator Template // using System; using System.Collections; using System.Collections.Specialized; using System.Threading; using Microsoft.SingSharp; using Microsoft.SingSharp.Reflection; using Microsoft.Singularity; using Microsoft.Singularity.Channels; using Microsoft.Singularity.Configuration; using Microsoft.Singularity.Directory; using Microsoft.Singularity.Services; using Microsoft.Singularity.ServiceManager; namespace Microsoft.Singularity.Resiliency { public abstract class JournalProducer { protected Thread dsThread; protected Thread managerThread; protected Thread providerThread; protected TRef rootDsRef; protected TRef myDsRef; protected TRef providerRef; protected TRef myProviderRef; protected TRef serviceRef; protected TRef controlRef; protected TRef managerRef; protected TRef dsSignalSenderRef; protected TRef dsSignalReceiverRef; protected TRef mgSignalSenderRef; protected TRef mgSignalReceiverRef; protected TRef spSignalSenderRef; protected TRef spSignalReceiverRef; protected TRef recoverySignalSenderRef; protected TRef recoverySignalReceiverRef; protected IList! journaletList; protected Object! journaletListLock; public JournalProducer() { this.journaletList = new ArrayList(); this.journaletListLock = new Object(); } /// /// Checks the type of this endpoint is that of the subclass can handle. /// protected abstract bool Accept(ServiceContract.Exp:Start! ep); /// /// Substitutes the ServiceContract with a new one in order to /// intercept the messages exchanged through the given ServiceContract. /// protected abstract void Substitute([Claims]ServiceContract.Exp:Start! ep, out ServiceContract.Exp! newEp); public void RegisterJournalet(Journalet j) { if (j != null) { lock (journaletListLock) { try { journaletList.Add(j); } catch (Exception) {} } } } public void DeregisterJournalet(Journalet j) { if (j != null) { lock (journaletListLock) { try { journaletList.Remove(j); } catch (Exception) {} } } } /// /// This actually launches the JournalProducer. The management thread /// is started here. /// /// JournalProducer is initialized with three channels. /// mep Channel from the service manager. JournalProducer is a /// kind of service, so it has to be controlled by the service /// manager. /// dep Channel to the (root) directory service. This channel is /// necessary to intercept the connection request from a client /// to the server that JournalProducer supports. /// fep Channel from the target service. This channel pretends the /// root directory service. /// /// NOTE: x is an temporal solution so that JournalProducer receives a /// new DS channel during the service recovery. /// public void Start([Claims]ManagedServiceContract.Exp:Start! mep, [Claims]DirectoryServiceContract.Imp:Ready! dep, [Claims]DirectoryServiceContract.Exp:Start! fep, [Claims]ManagedProxyContract.Imp:Start! pep) { ThreadTerminationContract.Imp! sender; ThreadTerminationContract.Exp! receiver; // // The root directory service // rootDsRef = new TRef(dep); // // Initialize connection to the service manager // mep.SendSuccess(); serviceRef = new TRef(mep); ThreadTerminationContract.NewChannel(out sender, out receiver); mgSignalSenderRef = new TRef(sender); mgSignalReceiverRef = new TRef(receiver); // // Initialize my directory service // fep.SendSuccess(); myDsRef = new TRef(fep); // // Initialize the channel to the service process // // xep.SendSuccess(); // proxyRef = new TRef(xep); // // Set up another connection for recovery to the service manager // switch receive { case pep.Success(): break; case unsatisfiable: DebugStub.Break(); break; } managerRef = new TRef(pep); ThreadTerminationContract.NewChannel(out sender, out receiver); recoverySignalSenderRef = new TRef(sender); recoverySignalReceiverRef = new TRef(receiver); // // Set up the service controller // /* cep.RecvSuccess(); controlRef = new TRef(cep); */ // // Set up the thread terminator for the substitute DS thread // ThreadTerminationContract.NewChannel(out sender, out receiver); dsSignalSenderRef = new TRef(sender); dsSignalReceiverRef = new TRef(receiver); // // Set up the thread terminator for the producer thread // ThreadTerminationContract.NewChannel(out sender, out receiver); spSignalSenderRef = new TRef(sender); spSignalReceiverRef = new TRef(receiver); // // Create the substitute DS thread // dsThread = new Thread(new ThreadStart(DirectoryServiceThread)); // // Create the journalet producer thread // providerThread = new Thread(new ThreadStart(ProviderThread)); // // Create and start the service management thread // managerThread = new Thread(new ThreadStart(ManagerThread)); managerThread.Start(); } /// /// JournalProducer provides a directory service on behalf of the /// root DirectoryService so that it intercepts all outputs of /// a service process. /// /// Here you will see the first logic of the directory service. It /// transits back and forth between the normal state and the recovery /// state. This seems simple but it's a bit tricky. The normal operation /// state normally deals with DirectoryServiceContract, but it also /// creates or deletes logging workers. /// /// The recovery operation state recovers the service registration /// state. A service process registers its ServiceProviderContract /// to the DirectoryService. JournalProducer intercepts this /// registration and registers another ServiceProviderContract to the /// DirectoryService. The recovery operation doesn't create and /// register a new ServiceProvider because it is already registered /// to the DirectoryService. /// /// NOTE: Current implementation doesn't support all protocols of the /// DirectoryServiceContract. protected void DirectoryServiceThread() { ThreadTerminationContract.Imp:Start! signalToManager; signalToManager = mgSignalSenderRef.Acquire(); for (;;) { if (!HandleDirectoryService()) { break; } // Trigger the recovery signalToManager.SendStop(); switch receive { case signalToManager.AckStop(): // Transit to the recovery mode if (!RecoverDirectoryService()) { goto exit; } break; case signalToManager.ChannelClosed(): // Recovery failed goto exit; break; } } exit: mgSignalSenderRef.Release(signalToManager); } /// /// Emulates a directory service. /// protected bool HandleDirectoryService() { // Flag for the recovery mode outside this method bool recovery = false; DirectoryServiceContract.Exp:Ready! myDS; DirectoryServiceContract.Imp:Ready! rootDS; ThreadTerminationContract.Exp:Start! signalFromManager; ThreadTerminationContract.Exp:Start! signalFromProvider; ThreadTerminationContract.Imp:Start! signalToProvider; myDS = myDsRef.Acquire(); rootDS = rootDsRef.Acquire(); signalToProvider = spSignalSenderRef.Acquire(); signalFromManager = dsSignalReceiverRef.Acquire(); signalFromProvider = recoverySignalReceiverRef.Acquire(); for (;;) { switch receive { // // The following 4 cases are message from the service // process. // case myDS.Bind(path, exp): { DebugStub.Print("JP DS: Bind, break\n"); DebugStub.Break(); delete path; delete exp; break; } case myDS.Register(path, imp): { ServiceProviderContract.Imp! client; ServiceProviderContract.Exp! server; // HI: Current implementation can handle only one name // per service. DebugStub.Print("JP DS: Register @ '{0}'\n", __arglist(Bitter.ToString2(path))); providerRef = new TRef(imp); ServiceProviderContract.NewChannel(out client, out server); myProviderRef = new TRef(server); // Now register the JP's endpoint to the NS. rootDS.SendRegister(path, client); break; } case myDS.Deregister(path): { DebugStub.Print("JP DS: Deregister\n"); rootDS.SendDeregister(path); break; } case myDS.ChannelClosed(): { DebugStub.Print("JP DS: Server channel closed." + " Waits for provider's signal.\n"); switch receive { case signalFromProvider.Stop(): { signalFromProvider.SendAckStop(); recovery = true; break; } case signalFromProvider.ChannelClosed(): { recovery = true; break; } case signalFromManager.Stop(): { signalToProvider.SendStop(); signalToProvider.RecvAckStop(); delete myProviderRef.Acquire(); delete providerRef.Acquire(); providerRef = null; signalFromManager.SendAckStop(); goto exit; break; } case signalFromManager.ChannelClosed(): { goto exit; break; } } DebugStub.Print("JP DS: Provider's signal received.\n"); goto exit; break; } // // The following messages are delivered from DS. // case rootDS.AckRegister(): { DebugStub.Print("JP DS: AckRegister\n"); myDS.SendAckRegister(); // // HI: At this point producer thread is allowed to // start. // providerThread.Start(); break; } case rootDS.NakRegister(ep, error): { DebugStub.Print("JP DS: NakRegister\n"); delete ep; delete myProviderRef.Acquire(); myDS.SendNakRegister(providerRef.Acquire(), error); providerRef = null; break; } case rootDS.NakRegisterReparse(path, rest, link, ep): { DebugStub.Print("JP DS: NakRegisterReparse\n"); delete ep; delete myProviderRef.Acquire(); myDS.SendNakRegisterReparse(path, rest, link, providerRef.Acquire()); providerRef = null; break; } case rootDS.AckDeregister(ep): { DebugStub.Print("JP DS: AckDeregister\n"); delete ep; signalToProvider.SendStop(); signalToProvider.RecvAckStop(); delete myProviderRef.Acquire(); myDS.SendAckDeregister(providerRef.Acquire()); providerRef = null; break; } case rootDS.NakDeregister(error): { DebugStub.Print("JP DS: NakDeregister\n"); myDS.SendNakDeregister(error); break; } case rootDS.NakDeregisterReparse(path, rest, link): { DebugStub.Print("JP DS: NakDeregisterReparse\n"); myDS.SendNakDeregisterReparse(path, rest, link); break; } case rootDS.ChannelClosed(): { DebugStub.Print("JP DS: DS channel closed. Break.\n"); // Emulate channel closing delete providerRef.Acquire(); providerRef = null; DebugStub.Break(); break; } case signalFromManager.Stop(): { signalToProvider.SendStop(); signalToProvider.RecvAckStop(); delete myProviderRef.Acquire(); delete providerRef.Acquire(); providerRef = null; signalFromManager.SendAckStop(); goto exit; break; } // // Signal from ProviderThread in this object. This is the // trigger to start recovery. // case signalFromProvider.Stop(): DebugStub.Print("JP DS: Received signal." + " Start recovery.\n"); recovery = true; signalFromProvider.SendAckStop(); goto exit; break; } } exit: recoverySignalReceiverRef.Release(signalFromProvider); spSignalSenderRef.Release(signalToProvider); dsSignalReceiverRef.Release(signalFromManager); myDsRef.Release(myDS); rootDsRef.Release(rootDS); return recovery; } // HandleDirectoryService protected bool RecoverDirectoryService() { DirectoryServiceContract.Exp:Ready! substitute; DebugStub.Print("JP: ENTER RecoveryOperation\n"); // // get a new Directory Service endpoint // substitute = myDsRef.Acquire(); DebugStub.Print("JP: Get new substitute."); switch receive { case substitute.Register(path, imp): { DebugStub.Print("JP DS Recovery: Re-register '{0}'\n", __arglist(Bitter.ToString2(path))); // // Refresh the server-side ServiceProviderContract // DebugStub.Print("JP: Replacing the server-side channel... "); delete providerRef.Acquire(); providerRef.Release(imp); DebugStub.Print("done\n"); // // This time, we immediately send back the ack. // substitute.SendAckRegister(); delete path; break; } } myDsRef.Release(substitute); new Thread(new ThreadStart(RecoveryThread)).Start(); //DebugStub.Print("JP: EXIT RecoveryOperation\n"); return true; } /// /// This thread communicates with SMS and deals with general stuff on /// the service management. /// // Communicates with DirectoryServiceThread protected void ManagerThread() { ManagedServiceContract.Exp:Ready! manager; ManagedProxyContract.Imp:Ready! managerRecovery; ThreadTerminationContract.Imp:Start! signalToDs; ThreadTerminationContract.Exp:Start! signalFromDs; manager = serviceRef.Acquire(); managerRecovery = managerRef.Acquire(); signalToDs = dsSignalSenderRef.Acquire(); signalFromDs = mgSignalReceiverRef.Acquire(); for (;;) { //DebugStub.Print("JP Manager th: Set\n"); switch receive { case manager.StartService(): { dsThread.Start(); manager.SendAckStartService(); //else { // manager.SendNakStartService(); //} break; } case manager.StopService(): { //DebugStub.Print("JPMan: StopService\n"); //providerThread.Stop(); manager.SendAckStopService(); break; } case manager.RestartService(): { DebugStub.Print("JPMan: RestartService\n"); DebugStub.Break(); //providerThread.Stop(); //providerThread.Start(); manager.SendAckRestartService(); break; } case manager.Knock(): { manager.SendAlive(); break; } case manager.Stop(): { signalToDs.SendStop(); break; } case manager.Restart(): { DebugStub.Break(); break; } case signalFromDs.Stop(): { delete myDsRef.Acquire(); managerRecovery.SendRequestDSRecovery(); break; } // Receives a new DirectoryServiceContract for the // restarted service process. JP provides the directory // service again with this endpoint. case managerRecovery.AckDSRecovery(directory): { directory.SendSuccess(); myDsRef.Release(directory); signalFromDs.SendAckStop(); break; } case managerRecovery.NakDSRecovery(): { goto exit; } case managerRecovery.ChannelClosed(): { goto exit; } case signalToDs.AckStop(): { manager.SendAckStop(); goto exit; break; } } } exit: delete manager; delete managerRecovery; delete signalFromDs; delete signalToDs; } /// /// Creates a Journalet for each C-S connection. /// protected void ProviderThread() { bool release = false; bool signal = false; bool recovery = false; // From the root directory service ServiceProviderContract.Exp:Start! providerFromRootDs; // To the resilient service ServiceProviderContract.Imp:Start! provider; ThreadTerminationContract.Imp:Start! dsSignal; ThreadTerminationContract.Exp:Start! signalFromDs; ThreadTerminationContract.Imp:Start! signalToDs; providerFromRootDs = myProviderRef.Acquire(); signalFromDs = spSignalReceiverRef.Acquire(); signalToDs = recoverySignalSenderRef.Acquire(); // // lock the server // provider = providerRef.Acquire(); for (;;) { switch receive { case providerFromRootDs.Connect(ep): { //DebugStub.Print("JP Producer: received" + // " client connection request ... "); if (Accept(ep)) { //DebugStub.Print("accepted.\n"); ServiceContract.Exp! newExp; Substitute(ep, out newExp); provider.SendConnect(newExp); switch receive { case provider.AckConnect(): providerFromRootDs.SendAckConnect(); break; case provider.NackConnect(rejected): delete rejected; providerFromRootDs.SendNackConnect(null); break; case provider.ChannelClosed(): // // Try to recover // providerFromRootDs.SendNackConnect(null); release = true; signal = false; recovery = true; goto exit; break; } } else { //DebugStub.Print("denied\n"); providerFromRootDs.SendNackConnect(ep); } break; } case providerFromRootDs.ChannelClosed(): { goto exit; break; } case signalFromDs.Stop(): { release = false; signal = true; goto exit; break; } case signalFromDs.ChannelClosed(): { goto exit; release = false; break; } case provider.ChannelClosed(): { // Recovery DebugStub.Print("JP: Service provider lost. " + "Recover.\n"); release = true; signal = false; recovery = true; goto exit; break; } } } exit: // Unlock the server // This allows the DirectoryServiceThread to replace the // ServiceProviderContract with a new one. providerRef.Release(provider); if (recovery) { // Notify the internal DS to restart signalToDs.SendStop(); signalToDs.RecvAckStop(); } // No matter if it recovers or not, sends a signal to get the // DirectoryServiceThread out of the loop. recoverySignalSenderRef.Release(signalToDs); if (release) { myProviderRef.Release(providerFromRootDs); } else { // Disconnect from the root DS. delete providerFromRootDs; } if (signal) { // It's over. Get the ManagerThread out of loop. signalFromDs.SendAckStop(); } spSignalReceiverRef.Release(signalFromDs); } /// /// Runs Journalets to replay their logs to the restarted and refreshed /// service process. Each Journalet emulates normal operations: create /// a new pair of ServiceContract, deliver an endpoint through the /// ServiceProviderContract, then interacts with the service process. /// /// Each Journalet has its own thread to replay the log, so they are /// running concurrently. /// protected virtual void RecoveryThread() { Journalet journalet; ServiceContract.Exp:Start! ep; ServiceProviderContract.Imp:Start! provider; //DebugStub.Print("JP: ENTER Recovery Thread\n"); // Lock the server. This blocks new clients to connect the service // process. provider = providerRef.Acquire(); lock (journaletListLock) { foreach (Object obj in journaletList) { if (obj == null) { break; } journalet = obj as Journalet; if (journalet == null) { continue; } journalet.CreateServerEndpoint(out ep); provider.SendConnect(ep); switch receive { case provider.AckConnect(): break; case provider.NackConnect(rejected): delete rejected; DebugStub.Print("JP Recovery Th: " + "Provider connection rejected\n"); break; case provider.ChannelClosed(): DebugStub.Print("Server lost during recovery." + " Break,\n"); DebugStub.Break(); break; } } } // unlock the server providerRef.Release(provider); providerThread = new Thread(new ThreadStart(ProviderThread)); providerThread.Start(); //DebugStub.Print("JP: EXIT Recovery Thread\n"); } } }