singrdk/base/Applications/Benchmarks/SpecWeb99/webfiles/webfiles.cs

// ----------------------------------------------------------------------------
//
//  Copyright (c) Microsoft Corporation.  All rights reserved.
//
// ----------------------------------------------------------------------------

//-----------------------------------------------------------------------
// Simulate the file reading portion of the specweb99 test suite
//    randomly pick files from the content generated by wafgen99
//    read them from disk
//-----------------------------------------------------------------------

using System;
using System.Text;
using System.Collections;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Threading;

#if SINGULARITY
using Microsoft.Singularity;
using Microsoft.Singularity.Directory;
using Microsoft.Singularity.V1.Services;
using Microsoft.Singularity.Io;
using Microsoft.Singularity.Channels;
using Microsoft.SingSharp;
using FileSystem.Utils;
using Singularity.Application.SPECWeb99;
using Microsoft.Contracts;
using Microsoft.Contracts;
using Microsoft.SingSharp.Reflection;
using Microsoft.Singularity.Applications;
using Microsoft.Singularity.Io;
using Microsoft.Singularity.Configuration;
[assembly: Transform(typeof(ApplicationResourceTransform))]

#endif

namespace Singularity.Application.SPECWeb99.WebFiles
{
    class Defaults
    {
        internal const string Root          = "/fs";
        internal const string RelativePath  = "file_set";
        internal const int    BlockSize     = 16384;
        internal const int    Connections   = 10;
        internal const int    RunSeconds    = 10;
        internal const int    WarmUpSeconds = 2;
    };

#if SINGULARITY
    [ConsoleCategory(HelpMessage="Null Disk r/w performance test", DefaultAction=true)]
    internal class Parameters
    {
        [InputEndpoint("data")]
        public readonly TRef<UnicodePipeContract.Exp:READY> Stdin;

        [OutputEndpoint("data")]
        public readonly TRef<UnicodePipeContract.Imp:READY> Stdout;

        [LongParameter( "b", Default=Defaults.BlockSize, HelpMessage="Block Size to be read.")]
        internal long blockSize;

        [LongParameter( "c", Default=Defaults.Connections, HelpMessage="Set number of connections to simulate.")]
        internal long connections;

        [LongParameter( "f", Default=0, HelpMessage="Forced iterations.")]
        internal long forcedIterations;

        [LongParameter( "k", Default=-1, HelpMessage="Restrict files to this class number.")]
        internal long classSizeNum;

        [LongParameter( "r", Default=Defaults.RunSeconds, HelpMessage="Run time in seconds.")]
        internal long runTime;

        [LongParameter( "t", Default=1, HelpMessage="Number of threads to use/")]
        internal long numThreads;

        [LongParameter( "v", Default=0, HelpMessage="Verbosity level")]
        internal long verbose;

        [LongParameter( "w", Default=Defaults.WarmUpSeconds, HelpMessage="Warmup time in seconds.")]
        internal long rampTime;

        [BoolParameter( "x", Default=false, HelpMessage="no read -- just bind")]
        internal bool testMode;

        [StringParameter("directory", Mandatory=true, Position = 0, Default = Defaults.Root, HelpMessage="path to find wafgen99 data")]
        internal string contentPrefix;

        reflective internal Parameters();

        internal int AppMain() {
            return SpecMain.AppMain(this);
          }
    }
#endif
#if !SINGULARITY
    class FileReader
    {
        //const uint FILE_FLAG_NO_BUFFERING = 0x20000000;
        const uint FILE_FLAG_NO_BUFFERING = 0;
        const uint GENERIC_READ = 0x80000000;
        const uint OPEN_EXISTING = 3;
        IntPtr handle;

        [DllImport("kernel32", SetLastError=true)]
        private static extern unsafe IntPtr CreateFile(
            string FileName,                    // file name
            uint DesiredAccess,                 // access mode
            uint ShareMode,                     // share mode
            uint SecurityAttributes,            // Security Attributes
            uint CreationDisposition,           // how to create
            uint FlagsAndAttributes,            // file attributes
            int hTemplateFile                   // handle to template file
            );

        [DllImport("kernel32", SetLastError=true)]
        private static extern unsafe bool ReadFile(
            IntPtr hFile,                       // handle to file
            void* pBuffer,                      // data buffer
            int NumberOfBytesToRead,            // number of bytes to read
            int* pNumberOfBytesRead,            // number of bytes read
            int Overlapped                      // overlapped buffer
            );

        [DllImport("kernel32", SetLastError=true)]
        private static extern unsafe bool CloseHandle(
            IntPtr hObject                      // handle to object
            );

        [DllImport("kernel32", SetLastError=true)]
        private static extern unsafe uint GetFileSize(
            IntPtr hObject,                     // handle to object
            uint *pFileSizeHigh                 // receives high 32-bits of file size.
            );

        public bool Open(string FileName) {
            // open the existing file for reading
            handle = CreateFile(
                FileName,
                GENERIC_READ,
                0,
                0,
                OPEN_EXISTING,
                FILE_FLAG_NO_BUFFERING,
                0);

            if (handle != IntPtr.Zero && handle != ((IntPtr) (-1))) {
                return true;
            }
            else {
                return false;
            }
        }

        public unsafe int Read(byte[] buffer, int index, int count) {
            int n = 0;
            fixed (byte* p = buffer) {
                if (!ReadFile(handle, p + index, count, &n, 0)) {
                    return 0;
                }
            }
            return n;
        }

        public bool Close() {
            // close file handle
            return CloseHandle(handle);
        }

        public unsafe int Size() {
            return (int)GetFileSize(handle, null);
        }
    }
#endif

    public class FileSpec
    {
        public int dirNum, classNum, fileNum;
        public long size;
    }

    public class UrlSet
    {
        public int pos;
        public FileSpec[]/*!*/ urls;

        public UrlSet(int size)
        {
            pos = 0;
            urls = new FileSpec[size];
        }
    }

    public class Client
    {
        //////////////////////////////////
        // Constants
        private const int    FILES_PER_CLASS       = 9;
        private const int    MAX_PATH              = 256;
        private const int    LINESIZE              = 64;
        private const int    MARKER_FREQ           = 4096;
        private const int    BUF_SIZE              = 1024*1024; // 1MB
        private  const int   MAX_DECK              = 1000;
        //////////////////////////////////
        // Pseudo-constants
        private static readonly int[] FILE_ORDER = {4,3,5,2,6,1,7,8,0};
        private static readonly int[] CLASS_BASE_SIZE = {1024, 10240, 102400, 1024000};
        private static readonly double[] CLASS_PROB = {0.35, 0.50, 0.14, 0.01};
        private static readonly int MAX_CLASS_COUNT = CLASS_BASE_SIZE.Length;

        //////////////////////////////////
        // static variables
        public static string pathPrefix   = "";
        public static int [] fileBucket;

        //////////////////////////////////
        // Instance variables
        public Random.RandomState randomState;
        public bool     generatingUrls;
        public Mutex/*!*/   globalLock;
        public Mutex/*!*/   syncLock;
        public UrlSet/*!*/   urlDeck;
        public UrlSet/*!*/   freeDeck;
        public UrlSet/*!*/   spareDeck;
#if SINGULARITY
        public static TRef<DirectoryServiceContract.Imp:Ready> namespaceRootT;
#else
      /// <summary>
      ///  will not work for multiple threads!!!
      /// </summary>
       private static FileReader reader = new FileReader();
#endif
        private long    dirCount     = 0;
        private long    fileCount    = 0;
        private long    repeatCount  = 0;
        private static long    blockSize    = 0;
        private long    totalSize    = 0;
        private long    fileSizeNum  = 0;
        private long    classSizeNum = 0;
        private long    threadNum    = 1;
        private long    verbose      = 0;
        private bool    dumpTable    = false;
        private bool    testMode = false;
        private int     runTime;
        private int     rampTime;
        private int     forcedIterations;

        private LocalState[] workerLocalStates;
        private StringBuilder sb;
        public static WorkerState     currentState;

        Random.ZipfState zipfDir;
        Random.ZipfState zipfFile;
        Random rand;

        public enum WorkerState
        {
            Start = 0,
            Rampup = 1,
            Run = 2,
            Done = 3,
        }

        // per-thread state
        private class LocalState
        {
            public WorkerState State;
            public DateTime StartTime;
            public long bindCount;
            public TimeSpan bindTime;
            public long readCount;
            public TimeSpan readTime;
            public TimeSpan totalTime;
            public long bytesRead;
            public int forcedIterations;

            public LocalState(int forcedIterations)
            {
                bytesRead = 0;
                bindTime = new TimeSpan(0);
                readTime = new TimeSpan(0);
                totalTime = new TimeSpan(0);

                readCount = 0;
                bindCount = 0;
                this.forcedIterations = forcedIterations;
            }
        }

        [NotDelayed]
        public Client(string/*!*/ prefix, long dir,
                      long bSize, long fSize,
                      long threads, long verbose,
                      long cSize, bool test,
                      int rampTime, int runTime,
                      int forcedIterations)
        {
            dirCount        = dir;
            pathPrefix      = prefix;
            fileSizeNum     = fSize;
            threadNum       = threads;
            classSizeNum    = cSize;
            blockSize       = bSize;
            this.verbose    = verbose;
            testMode        = test;
            this.rampTime   = rampTime;
            this.runTime    = runTime;
            this.forcedIterations = forcedIterations;

            globalLock      = new Mutex();
            syncLock        = new Mutex();
            urlDeck         = new UrlSet(MAX_DECK);
            freeDeck        = new UrlSet(MAX_DECK);
            spareDeck       = new UrlSet(MAX_DECK);
            fileBucket      = new int [FILES_PER_CLASS];
            sb              = new StringBuilder(512);

#if SINGULARITY
            namespaceRootT = new TRef<DirectoryServiceContract.Imp:Ready>(
                DirectoryService.NewClientEndpoint());
#else
#endif
            workerLocalStates = new LocalState[threads];

#if SINGULARITY
            base();
#endif // SINGULARITY
        }

        //------------------------------------------------------------------
        // ReadFile
        //  open and read its content of fileName
        //  throw the content away (no copying, no converting etc...)
        //------------------------------------------------------------------

        // TODO: The way this is called is rather bizarre. Fix it.
        // a Client spawns LocalWorker objects that call back to this
        // function. Should be moved into the LoadWorker class

 #if SINGULARITY
        private byte[]/*!*/ in ExHeap ReadFile(string/*!*/ fileName,
                [Claims] byte[]/*!*/ in ExHeap inBuf,
                ref LocalState local
                )
        {
            if (local == null) {
                Console.WriteLine("ReadFile:  local state is null!");
                return inBuf;
            }

            FileContract.Imp conn = null;
            DateTime start;
            DateTime beginTotal;
            TimeSpan btime = new TimeSpan(0);

            beginTotal = DateTime.Now;

            // only perform timing while in the run state

            DirectoryServiceContract.Imp:Ready! ds = ((!)Client.namespaceRootT).Acquire();
            ErrorCode errorCode = ErrorCode.Unknown;
            try {
                if (currentState == WorkerState.Run) {
                    start = DateTime.Now;
                    conn = FileUtils.OpenFile(fileName, ds, out errorCode);
                    btime =  DateTime.Now - start;
                    local.bindTime += btime;
                    local.bindCount++;
                }
                else {
                    conn = FileUtils.OpenFile(fileName, ds, out errorCode);
                }
            }
            finally {
                Client.namespaceRootT.Release(ds);
            }

            if (conn == null) {
                Console.WriteLine("Failed to open file '{0}' code={1}",fileName, SdsUtils.ErrorCodeToString(errorCode));
                throw new Exception(" file missing");
                return inBuf;
            }

            try {
                int error  = 0;
                long filePos  = 0;
                long  bytesRead = 0;

                start = DateTime.Now;

                while (!testMode) {
                    conn.SendRead(inBuf, 0, filePos, inBuf.Length);
                    conn.RecvAckRead(out inBuf, out bytesRead, out error);
                    local.readCount++;
                    if (error != 0) {
                        throw new Exception(" Encountered error while reading " + fileName);
                        break;
                    }
                    if (bytesRead == 0) break;
                    filePos += bytesRead;
                }

                if (currentState == WorkerState.Run) {
                    local.bytesRead += filePos;
                    TimeSpan x =  DateTime.Now - start;
                    local.readTime += x;
                }
            }
            finally {
                delete conn;
            }

            if (currentState == WorkerState.Run) {
                DateTime end = DateTime.Now;
                local.totalTime += end - beginTotal;
            }
            return inBuf;
        }
#endif

        ///
        /// generate a new "deck" of random files
        ///
        public void GenerateFileSpecs()
        {
            FileSpec fs;
            long size, linkCount;
            uint flags;
            double totalProb = 0.0;


            UrlSet/*!*/ fr = freeDeck;
            UrlSet/*!*/ sp = spareDeck;

            //
            // Ensure the spare deck is fully allocated
            //     if there are any elements in the free pool use them
            //     otherwise we need a new allocations
            //
            globalLock.WaitOne();
#if verbose
            Console.WriteLine("gendeck: {0} freed slots spare={1},",
                fr.pos,sp.pos,urlDeck.pos);
            //DebugStub.Break();
#endif
            // fill up the spare deck.  Start with using
            // items from the free deck. If none are available
            // allocate them.

            // NOTE allocation should mostly happen during the initialization phase

            for (int i = 0; i < fr.urls.Length; i++) {
                if (fr.pos == 0) {
                    //if ( sp.urls[i] != null) DebugStub.Break();
                    sp.urls[i] = new FileSpec();
                    if (sp.urls[i] == null) {
                        throw new Exception ("no memory!");
                    }
                    // spit out diagnostics if we are allocating during the run phase
                    if (currentState == WorkerState.Run)  Console.Write(" @{0}",i);
                }
                else {
                    sp.urls[i] = fr.urls[--fr.pos];
                }
            }
            globalLock.ReleaseMutex();

            // Now that we have a spare deck with the needed number of items
            // fill in their values according to spec.

            // The loop immediately below will generate s set of files conforming to the
            // requested probabilities but will not be random.

            for (int classNo = 0; classNo < MAX_CLASS_COUNT; classNo++) {
                totalProb += CLASS_PROB[classNo];
                while (sp.pos < totalProb * sp.urls.Length) {
                    fs = sp.urls[sp.pos++];
                    if (fs == null) {
                        throw new Exception(" spare[" + sp.pos + "] is null!");
                    }
                    fs.classNum = classNo;
                    fs.fileNum  = FILE_ORDER[rand.spec_zipf(zipfFile)];
                    fs.dirNum   = rand.spec_zipf(zipfDir);

                    fs.size = CLASS_BASE_SIZE[fs.classNum] * (fs.fileNum +1) / 10;

                    if (verbose > 4) {
                        sb.Length = 0; //reset the buffer
                        sb.Append(pathPrefix);
                        sb.Append("/dir");
                        sb.Append(fs.dirNum.ToString("d05"));
                        sb.Append("/class");
                        sb.Append(fs.classNum);
                        sb.Append("_");
                        sb.Append(fs.fileNum);
                        Console.WriteLine("size {0} file={1}",fs.size,sb.ToString());
                    }
                }
                if (verbose > 1) {
                    Console.WriteLine("Generating class[{0}] prob={1:f2} urlCount={2}",
                        classNo, CLASS_PROB[classNo], sp.pos);
                }
            }

            // Randomize deck
            for (int i = 0; i < sp.pos; i++) {
                 /*^ assert this.randomState != null; ^*/
                 /*^ assert this.rand != null; ^*/
                int pos = rand.Spec_nrandom(randomState) % (sp.pos - i);
                FileSpec obj  = sp.urls[sp.pos- i - 1];
                sp.urls[sp.pos - i - 1] = sp.urls[pos];
                sp.urls[pos] = obj;
            }
        }

        //------------------------------------------------------------------
        // Generate Load
        //  creates the random list of files to be done every iteration
        //  Reads contents of entire file for every file in the list
        //  This list is generated so we can see if each loop has identical timings
        //------------------------------------------------------------------
        private class LoadWorker
        {
            private Client hostClient;
            private LocalState localState;
            private readonly ArrayList fs;
            private StringBuilder sb;
            private int readSize;
            private Cache cache;
            private int id; // local thread index

            public LoadWorker(Client parent, ref LocalState localState, int readSize, int id)
            {
                hostClient      = parent;
                this.localState = localState;
                sb              = new StringBuilder(512);
                this.readSize   = readSize;
                cache           = new Cache(20, 20 *2);
                this.id         = id;
            }
#if SINGULARITY
            public byte []/*!*/ in ExHeap ProcessUrl(
                                       FileSpec fileSpec,
                                       [Claims] byte []/*!*/ in ExHeap buf) {
#else
            public void ProcessUrl(FileSpec fileSpec, byte[] buf, ref LocalState local ) {
#endif
                if (fileSpec == null) throw new Exception ("Attempting to read from null fileSpec");
#if SINGULARITY
                sb.Length = 0; //reset the buffer
                sb.Append(pathPrefix);
                sb.Append("/dir");
                sb.Append(fileSpec.dirNum.ToString("d05"));
                sb.Append("/class");
                sb.Append(fileSpec.classNum);
                sb.Append("_");
                sb.Append(fileSpec.fileNum);
                string path = sb.ToString();

                if (hostClient.verbose > 2) {
                    Console.WriteLine("reading {0}", path);
                }
                buf = hostClient.ReadFile(path, buf, ref localState);
#else
                sb.Length = 0; //reset the buffer
                sb.Append(pathPrefix);
                sb.Append("\\dir");
                sb.Append(fileSpec.dirNum.ToString("d05"));
                sb.Append("\\class");
                sb.Append(fileSpec.classNum);
                sb.Append("_");
                sb.Append(fileSpec.fileNum);
                string path = sb.ToString();

                int error = 0;
                //Console.Write(".");
                DateTime beginTotal = DateTime.Now;
                if (!reader.Open(path)) {
                    Console.WriteLine("open failed: {0}", path);
                    throw new Exception("open failure: "+path);
                }

                DateTime beginRead = DateTime.Now;
                for (long pos = 0; pos < fileSpec.size;) {
                    int bytes = reader.Read(buf, 0, buf.Length);
                    //Console.Write(".");
                    if (bytes == 0) {
                        Console.WriteLine("bytes read={0}, error={1} pos={2}",
                            bytes, error, pos);
                        break;
                    }
                    pos += bytes;
                    //if  (pos != fileSpec.size) throw new Exception (" wrong size pos=" + pos + " bytes=" + bytes);
                }
                reader.Close();
                if (currentState == WorkerState.Run) {
                    DateTime end = DateTime.Now;
                    local.totalTime += end - beginTotal;
                    local.bindTime += beginRead - beginTotal;
                    local.readTime += end - beginRead ;
                    local.bytesRead += fileSpec.size;
                }
#endif
                Client.fileBucket[fileSpec.fileNum]++;
                path = null;
#if SINGULARITY
                return buf;
#endif
            }

            public void RunLoad()
            {
#if SINGULARITY
                byte [] in ExHeap buf = new[ExHeap] byte[readSize];
#else
                // Allocate one one buffer for all operations
                byte[] buf = new byte [blockSize];
#endif
                long size;

                // if loop we've been told to quit, but run at least the forcedIterations.
                for (int iterations = 0;; iterations++) {
                    if (iterations >= localState.forcedIterations &&
                        localState.State == WorkerState.Done) {
                        break;
                    }

                    FileSpec fs  = cache.GetUrl(hostClient,id);
#if SINGULARITY
                    buf = ProcessUrl(fs, buf);
#else
                    ProcessUrl(fs,buf, ref localState);
#endif
                    cache.FreeUrl(fs);
                }
#if SINGULARITY
                delete buf;
#endif
            }
        }

        public void InitializeGlobalState()
        {
            // the following initialization is taken from  specweb99/load_generate and client.c
            rand = new Random();
            int first_seed = 2;
            randomState = new Random.RandomState(first_seed);
            zipfDir = rand.spec_zipf_setup(randomState, (int)dirCount, 1.0);
            zipfFile = rand.spec_zipf_setup(randomState, FILES_PER_CLASS, 1.0);
            generatingUrls = false;
        }

        public long GatherStats(out long totalFiles)
        {
            long total = 0;
            totalFiles = 0;
            Console.WriteLine("Thread stats:");
            for (int i = 0; i < threadNum; i++) {
                LocalState temp = workerLocalStates[i];
                if (temp != null) {
                    double  t1 = (double)temp.totalTime.Ticks / TimeSpan.TicksPerSecond;
                    double  t2 = (double)temp.bindTime.Ticks / TimeSpan.TicksPerSecond;
                    double  t3 = (double)temp.readTime.Ticks / TimeSpan.TicksPerSecond;
                    string summ = String.Format(
                        " [{0}] Total: {1:f3} bind: {2:f3}, read: {3:f3} bytes: {4}",
                        i, t1, t2, t3, temp.bytesRead);
                    Console.WriteLine(summ);
#if SINGULARITY
                    DebugStub.Print(summ);
#endif // SINGULARITY
                    //Console.WriteLine(" bind count ={0}, read count ={1}",temp.bindCount, temp.readCount);
                   total += temp.bytesRead;
                   totalFiles += temp.bindCount;
                }
            }
            return total;
        }

        ///
        /// Main load generation routine
        /// manages Global state machine  via timers etc.
        public void GenerateLoad()
        {
            ArrayList/*!*/ urlList;
            totalSize = 0;
            double megaBytes;
            InitializeGlobalState();

            //if (testMode) return;
            totalSize *= threadNum;
            //megaBytes = ( (double) (totalSize) / (double) (1024*1024) );

#if SINGULARITY
            long startIrqCount;
            long startSwitchCount;
            long startKernelGcCount;
#endif
            int  startGcCount;
            long startGcMillis;
            long startGcBytes;
            DateTime startTime;

            long endGcMillis;
            long endGcBytes;
            DateTime endTime;
            //DateTime globalStartTime = DateTime.Now;

            // Create all the workers
            LoadWorker[] workers = new LoadWorker[threadNum];
            Thread[] threads = new Thread[threadNum];

            for (int i = 0; i < threadNum; i++) {
                workerLocalStates[i] = new LocalState(forcedIterations);
                workers[i] = new LoadWorker(this, ref workerLocalStates[i],(int) blockSize,i);
            }

            // Create the threads
            for (int i = 0; i < threadNum; i++) {
                LocalState temp = workerLocalStates[i];
                if (temp != null) temp.State = WorkerState.Rampup;
                threads[i] = new Thread(new ThreadStart(workers[i].RunLoad));
            }

            if (rampTime != 0) {
                // ramp-up time isn't part of benchmark.

                Console.WriteLine("Entering Rampup state");
                // Run all the threads
                currentState = WorkerState.Rampup;
                for (int i = 0; i < threadNum; i++) {
                    Thread t = threads[i];
                     /*^ assume t!= null; ^*/
                    t.Start();
                }

                // sleep for some time and change states from Rampup to Run
                Thread.Sleep(rampTime*1000);
                Console.WriteLine("\nEntering Run state");
                globalLock.WaitOne();

                // start perf counting after we have the lock
#if SINGULARITY
                GC.PerformanceCounters(out startGcCount,
                                       out startGcMillis,
                                       out startGcBytes);

                startTime = DateTime.Now;
                startIrqCount = ProcessService.GetKernelInterruptCount();
                startSwitchCount = ProcessService.GetContextSwitchCount();
                startKernelGcCount = ProcessService.GetKernelGcCount();
#else
                startTime = DateTime.Now;
#endif
            }
            else {
                // ramp-up time is part of benchmark.
#if SINGULARITY
                GC.PerformanceCounters(out startGcCount,
                                       out startGcMillis,
                                       out startGcBytes);

                startTime = DateTime.Now;
                startIrqCount = ProcessService.GetKernelInterruptCount();
                startSwitchCount = ProcessService.GetContextSwitchCount();
                startKernelGcCount = ProcessService.GetKernelGcCount();
#else
                startTime = DateTime.Now;
#endif

                Console.WriteLine("Starting threads");
                // Run all the threads
                currentState = WorkerState.Rampup;
                for (int i = 0; i < threadNum; i++) {
                    Thread t = threads[i];
                     /*^ assume t!= null; ^*/
                    t.Start();
                }

                // sleep for some time and change states from Rampup to Run
                Thread.Sleep(rampTime*1000);
                Console.WriteLine("\nEntering Run state");
                globalLock.WaitOne();
            }

            // Workers are ready to run.
            currentState = WorkerState.Run;

            for (int i = 0; i < threadNum; i++) {
                LocalState temp = workerLocalStates[i];
                if (temp != null) temp.State = WorkerState.Run;
            }
            globalLock.ReleaseMutex();


            // sleep for some time and change states from Run to Done
            Thread.Sleep(runTime*1000);
            globalLock.WaitOne();
            currentState = WorkerState.Done;
            for (int i = 0; i < threadNum; i++) {
                LocalState temp = workerLocalStates[i];
                if (temp != null) temp.State = WorkerState.Done;
            }
            globalLock.ReleaseMutex();

#if SINGULARITY
            bool wasActive = Monitoring.isActive();
            if  (wasActive) Console.WriteLine("May want to reset Monitoring state -- was active");
            Monitoring.setActive(false);
            // Wait for all the threads to complete
            for (int i = 0; i < threadNum; i++) {
                Thread t = threads[i];
                 /*^ assume t!= null; ^*/
                t.Join();
            }
#endif

            // REVIEW: Should stats happen before or after thread join?

            long totalFiles;
            long totalBytes = GatherStats(out totalFiles);

            megaBytes = ( (double) (totalBytes) / (double) (1024*1024) );

            int endGcCount;
#if SINGULARITY
            endTime = DateTime.Now;
            GC.PerformanceCounters(out endGcCount,
                                    out endGcMillis,
                                    out endGcBytes);
#else
           endTime = DateTime.Now;
#endif


            TimeSpan elapsed = endTime - startTime;
            double delta = (double)elapsed.Ticks / TimeSpan.TicksPerSecond;
            string summ =
                String.Format(
                    "Class:{0:d} Block:{1:d} Elapsed: {2:f2} MBps: {3:f3}, Files/s: {4:f3}",
                    classSizeNum,
                    blockSize,
                    delta,
                    (double)(megaBytes / delta),
                    (double)(totalFiles / delta));

#if SINGULARITY
            string info =
                String.Format(
                    "[AppGC:- cnt:{0} bytes:{1} Kern:- ints:{2} swi:{3} gcs:{4}]",
                    endGcCount - startGcCount,
                    endGcBytes - startGcBytes,
                    ProcessService.GetKernelInterruptCount() - startIrqCount,
                    ProcessService.GetContextSwitchCount() - startSwitchCount,
                    ProcessService.GetKernelGcCount() - startKernelGcCount);
#else
            string info = "not implemented";
#endif
            if (verbose > 0) {
                Console.WriteLine();
                Console.WriteLine(summ);
                Console.WriteLine(info);
            }


            if (verbose > 1) {
                long ftotal = 0;
                for (int i = 0; i < 9; i++) {
                    ftotal += fileBucket[i];
                }

                Console.WriteLine("Actual Zipf file distribution:");
                for (int i = 0; i < 9; i++) {
                    double foo = (double) (fileBucket[i] * 100) / (double) ftotal;
                    Console.WriteLine("   FileBucket[{0}] = {1:d6}. %={2:f2}",
                        i, fileBucket[i], foo);
                }
            }
#if SINGULARITY
            DebugStub.WriteLine(summ);
            DebugStub.WriteLine(info);
#endif
       }
    } // client

    public class SpecMain
    {
        //--------------------------------------------------
        //--------------------------------------------------
        private static int ConnectionsToDirectories(int conns)
        {
            return ((int) (25 + (((400000.0 / 122000.0) * conns)/5.0)));
        }

        static bool ParseNumber(string/*!*/ arg,
                                string/*!*/ name,
                                out long value)
        {
            // arg should look like "[-][A-z]:[0-9]*"
            if (arg.Length >= 4) {
                try {
                    value = Int64.Parse(arg.Substring(3));
                    return true;
                }
                catch (FormatException) {
                    Console.WriteLine("format ex");
                }
                catch (OverflowException) {
                    Console.WriteLine("overflow ex");
                }
            }
            Console.WriteLine("Could not parse {0}, {1}", name,arg.Substring(3));
            value = 0;
            return false;
        }

        private static void Usage()
        {
            Console.WriteLine("Usage: specweb99 client [options] content-root");
            Console.WriteLine("  randomly read files generated by wafgen99");
            Console.WriteLine("Options:");
            Console.WriteLine(
                "  /b:<n>      Block Size to be read [{0}].",
                Defaults.WarmUpSeconds
                );
            Console.WriteLine(
                "  /c:<n>      Set number of connections to simulate [{0}].",
                Defaults.Connections
                );
            Console.WriteLine("  /k:<n>      Restrict files to this class number.");
            Console.WriteLine("  /f:<n>      Forced iterations [0].");
            Console.WriteLine("  /r:<n>      run time in seconds [{0}].",
                              Defaults.RunSeconds);
            Console.WriteLine("  /t:<n>      Number of threads to use [1].");
            Console.WriteLine("  /x          no read -- just bind");
            Console.WriteLine("  /w:<n>      warmup time in seconds[5.]");
        }

#if SINGULARITY
        internal static int AppMain(Parameters! config)
        {

            long dirCount           = (long) (ConnectionsToDirectories((int)config.connections) -1 ); //zero based
            long blockSize          = config.blockSize;
            long fileSizeNum        = -1;
            long classSizeNum       = config.classSizeNum;
            long verbose            = config.verbose;
            long numThreads         = config.numThreads;
            bool testMode           = config.testMode;
            long runTime            = config.runTime;
            long rampTime           = config.rampTime;
            long forcedIterations   = config.forcedIterations;
            string contentPrefix    = config.contentPrefix + "/" + Defaults.RelativePath;
            if (forcedIterations != 0) {
                rampTime = 0;
                runTime = 0;
            }

            Console.WriteLine(" args: content=" + contentPrefix +" dir=" + dirCount + " block size="+blockSize);
            Console.WriteLine("       verbose="+verbose+" FileSizeNum="+fileSizeNum+" ClassSizeNum="+classSizeNum);
            Console.WriteLine("       threads="+numThreads+" forcedIterations="+forcedIterations);
            //DateTime start = DateTime.Now;

            Client c = new Client(contentPrefix, dirCount,  blockSize, fileSizeNum,
                                  numThreads, verbose, classSizeNum,
                                  testMode, (int) rampTime, (int)runTime,
                                  (int)forcedIterations);

            if (c != null) {
                c.GenerateLoad();
            }

            //TimeSpan elapsed = DateTime.Now - beginRunTime;
            //Console.WriteLine("Elapsed time: "+elapsed);
            return 0;

        }
#else
        //--------------------------------------------------
        // Windows Main routine
        //--------------------------------------------------
        public static int Main(string[]/*!*/ args)
        {
            string contentPrefix = Defaults.Root;
            long dirCount     = (long) (ConnectionsToDirectories(Defaults.Connections) -1); //zero based
            long blockSize    = Defaults.Connections;
            long fileSizeNum  = -1;
            long classSizeNum  = -1;
            long verbose = 1;
            long numThreads = 1;
            bool testMode = false;
            long runTime = Defaults.RunSeconds;
            long rampTime  = Defaults.WarmUpSeconds;
            long forcedIterations = 0;

            // Temporaries for command-line parsing
            bool  needHelp = args.Length == 0;

            int i;
            for (i = 1; i < args.Length; i++) {
                string/*!*/ arg = (string /*!*/) args[i];

                if (arg.Length < 2 || (arg[0] != '-' && arg[0] != '/')) {
                    Console.WriteLine("Invalid argument: {0}", arg);
                    Usage();
                    return -1;
                }

                long temp;
                switch (Char.ToLower(arg[1])) {
                    case '?':
                    case 'h':
                        needHelp = true;
                        break;
                    case 'b':
                        ParseNumber(arg, "block size", out blockSize);
                        break;
                    case 'c':
                        long connections;
                        ParseNumber(arg, "# of connections", out connections);
                        dirCount = (long) (ConnectionsToDirectories((int)connections) -1 ); //zero based

                        break;
                    case 'f':
                        ParseNumber(arg, "# of forced iterations", out forcedIterations);
                        rampTime = 0;
                        runTime = 0;
                        break;
                    case 'k':
                        ParseNumber(arg, "class number (size) to be read", out classSizeNum);
                        break;
                    case 'r':
                        ParseNumber(arg, "Run time", out runTime);
                        break;
                    case 't':
#if SINGULARITY
                        ParseNumber(arg, "number of threads to use", out numThreads);
#else
                        Console.WriteLine("Multi-thread not spported on NT -- ignoring");
#endif
                        break;
                    case 'v':
                        ParseNumber(arg, "verbosity level", out verbose);
                        break;
                    case 'w':
                        ParseNumber(arg, "Warmup time", out rampTime);
                        break;
                    case 'x':
                        testMode = true;
                        break;
                    default:
                        contentPrefix = arg;
                        break;
                }
            } // for args

            contentPrefix += "/" + Defaults.RelativePath;

            if (needHelp) {
                Usage();
                return -1;
            }

            Console.WriteLine(" args: content=" + contentPrefix +" dir=" + dirCount + " block size="+blockSize);
            Console.WriteLine("       verbose="+verbose+" FileSizeNum="+fileSizeNum+" ClassSizeNum="+classSizeNum);
            Console.WriteLine("       threads="+numThreads+" forcedIterations="+forcedIterations);
            //DateTime start = DateTime.Now;

            Client c = new Client(contentPrefix, dirCount,  blockSize, fileSizeNum,
                                  numThreads, verbose, classSizeNum,
                                  testMode, (int) rampTime, (int)runTime,
                                  (int)forcedIterations);

            if (c != null) {
                c.GenerateLoad();
            }

            //TimeSpan elapsed = DateTime.Now - beginRunTime;
            //Console.WriteLine("Elapsed time: "+elapsed);
            return 0;
        } // main
#endif
    } //specmain class
} //namespace