1089 lines
41 KiB
C#
1089 lines
41 KiB
C#
|
///////////////////////////////////////////////////////////////////////////////
|
||
|
//
|
||
|
// Microsoft Research Singularity
|
||
|
//
|
||
|
// Copyright (c) Microsoft Corporation. All rights reserved.
|
||
|
//
|
||
|
///////////////////////////////////////////////////////////////////////////////
|
||
|
|
||
|
//-----------------------------------------------------------------------
|
||
|
// Simulate the file reading portion of the specweb99 test suite
|
||
|
// randomly pick files from the content generated by wafgen99
|
||
|
// read them from disk
|
||
|
//-----------------------------------------------------------------------
|
||
|
|
||
|
using System;
|
||
|
using System.Text;
|
||
|
using System.Collections;
|
||
|
using System.Runtime.CompilerServices;
|
||
|
using System.Runtime.InteropServices;
|
||
|
using System.Threading;
|
||
|
|
||
|
#if SINGULARITY
|
||
|
using Microsoft.Singularity;
|
||
|
using Microsoft.Singularity.Directory;
|
||
|
using Microsoft.Singularity.V1.Services;
|
||
|
using Microsoft.Singularity.Io;
|
||
|
using Microsoft.Singularity.Channels;
|
||
|
using Microsoft.SingSharp;
|
||
|
using FileSystem.Utils;
|
||
|
using Singularity.Application.SPECWeb99;
|
||
|
using Microsoft.Contracts;
|
||
|
using Microsoft.Contracts;
|
||
|
using Microsoft.SingSharp.Reflection;
|
||
|
using Microsoft.Singularity.Applications;
|
||
|
using Microsoft.Singularity.Io;
|
||
|
using Microsoft.Singularity.Configuration;
|
||
|
[assembly: Transform(typeof(ApplicationResourceTransform))]
|
||
|
|
||
|
#endif
|
||
|
|
||
|
namespace Singularity.Application.SPECWeb99.WebFiles
|
||
|
{
|
||
|
class Defaults
|
||
|
{
|
||
|
internal const string Root = "/fs";
|
||
|
internal const string RelativePath = "file_set";
|
||
|
internal const int BlockSize = 16384;
|
||
|
internal const int Connections = 10;
|
||
|
internal const int RunSeconds = 10;
|
||
|
internal const int WarmUpSeconds = 2;
|
||
|
};
|
||
|
|
||
|
#if SINGULARITY
|
||
|
[ConsoleCategory(HelpMessage="Null Disk r/w performance test", DefaultAction=true)]
|
||
|
internal class Parameters
|
||
|
{
|
||
|
[InputEndpoint("data")]
|
||
|
public readonly TRef<UnicodePipeContract.Exp:READY> Stdin;
|
||
|
|
||
|
[OutputEndpoint("data")]
|
||
|
public readonly TRef<UnicodePipeContract.Imp:READY> Stdout;
|
||
|
|
||
|
[LongParameter( "b", Default=Defaults.BlockSize, HelpMessage="Block Size to be read.")]
|
||
|
internal long blockSize;
|
||
|
|
||
|
[LongParameter( "c", Default=Defaults.Connections, HelpMessage="Set number of connections to simulate.")]
|
||
|
internal long connections;
|
||
|
|
||
|
[LongParameter( "f", Default=0, HelpMessage="Forced iterations.")]
|
||
|
internal long forcedIterations;
|
||
|
|
||
|
[LongParameter( "k", Default=-1, HelpMessage="Restrict files to this class number.")]
|
||
|
internal long classSizeNum;
|
||
|
|
||
|
[LongParameter( "r", Default=Defaults.RunSeconds, HelpMessage="Run time in seconds.")]
|
||
|
internal long runTime;
|
||
|
|
||
|
[LongParameter( "t", Default=1, HelpMessage="Number of threads to use/")]
|
||
|
internal long numThreads;
|
||
|
|
||
|
[LongParameter( "v", Default=0, HelpMessage="Verbosity level")]
|
||
|
internal long verbose;
|
||
|
|
||
|
[LongParameter( "w", Default=Defaults.WarmUpSeconds, HelpMessage="Warmup time in seconds.")]
|
||
|
internal long rampTime;
|
||
|
|
||
|
[BoolParameter( "x", Default=false, HelpMessage="no read -- just bind")]
|
||
|
internal bool testMode;
|
||
|
|
||
|
[StringParameter("directory", Mandatory=true, Position = 0, Default = Defaults.Root, HelpMessage="path to find wafgen99 data")]
|
||
|
internal string contentPrefix;
|
||
|
|
||
|
reflective internal Parameters();
|
||
|
|
||
|
internal int AppMain() {
|
||
|
return SpecMain.AppMain(this);
|
||
|
}
|
||
|
}
|
||
|
#endif
|
||
|
#if !SINGULARITY
|
||
|
class FileReader {
|
||
|
//const uint FILE_FLAG_NO_BUFFERING = 0x20000000;
|
||
|
const uint FILE_FLAG_NO_BUFFERING = 0;
|
||
|
const uint GENERIC_READ = 0x80000000;
|
||
|
const uint OPEN_EXISTING = 3;
|
||
|
IntPtr handle;
|
||
|
|
||
|
[DllImport("kernel32", SetLastError=true)]
|
||
|
private static extern unsafe IntPtr CreateFile(
|
||
|
string FileName, // file name
|
||
|
uint DesiredAccess, // access mode
|
||
|
uint ShareMode, // share mode
|
||
|
uint SecurityAttributes, // Security Attributes
|
||
|
uint CreationDisposition, // how to create
|
||
|
uint FlagsAndAttributes, // file attributes
|
||
|
int hTemplateFile // handle to template file
|
||
|
);
|
||
|
|
||
|
[DllImport("kernel32", SetLastError=true)]
|
||
|
private static extern unsafe bool ReadFile(
|
||
|
IntPtr hFile, // handle to file
|
||
|
void* pBuffer, // data buffer
|
||
|
int NumberOfBytesToRead, // number of bytes to read
|
||
|
int* pNumberOfBytesRead, // number of bytes read
|
||
|
int Overlapped // overlapped buffer
|
||
|
);
|
||
|
|
||
|
[DllImport("kernel32", SetLastError=true)]
|
||
|
private static extern unsafe bool CloseHandle(
|
||
|
IntPtr hObject // handle to object
|
||
|
);
|
||
|
|
||
|
[DllImport("kernel32", SetLastError=true)]
|
||
|
private static extern unsafe uint GetFileSize(
|
||
|
IntPtr hObject, // handle to object
|
||
|
uint *pFileSizeHigh // receives high 32-bits of file size.
|
||
|
);
|
||
|
|
||
|
public bool Open(string FileName) {
|
||
|
// open the existing file for reading
|
||
|
handle = CreateFile(
|
||
|
FileName,
|
||
|
GENERIC_READ,
|
||
|
0,
|
||
|
0,
|
||
|
OPEN_EXISTING,
|
||
|
FILE_FLAG_NO_BUFFERING,
|
||
|
0);
|
||
|
|
||
|
if (handle != IntPtr.Zero && handle != ((IntPtr) (-1)) ) {
|
||
|
return true;
|
||
|
}
|
||
|
else {
|
||
|
return false;
|
||
|
}
|
||
|
}
|
||
|
|
||
|
public unsafe int Read(byte[] buffer, int index, int count) {
|
||
|
int n = 0;
|
||
|
fixed (byte* p = buffer) {
|
||
|
if (!ReadFile(handle, p + index, count, &n, 0)) {
|
||
|
return 0;
|
||
|
}
|
||
|
}
|
||
|
return n;
|
||
|
}
|
||
|
|
||
|
public bool Close() {
|
||
|
// close file handle
|
||
|
return CloseHandle(handle);
|
||
|
}
|
||
|
|
||
|
public unsafe int Size() {
|
||
|
return (int)GetFileSize(handle, null);
|
||
|
}
|
||
|
}
|
||
|
#endif
|
||
|
|
||
|
public class FileSpec
|
||
|
{
|
||
|
public int dirNum, classNum, fileNum;
|
||
|
public long size;
|
||
|
}
|
||
|
|
||
|
public class UrlSet
|
||
|
{
|
||
|
public int pos;
|
||
|
public FileSpec[] urls;
|
||
|
|
||
|
public UrlSet(int size)
|
||
|
{
|
||
|
pos = 0;
|
||
|
urls = new FileSpec[size];
|
||
|
}
|
||
|
}
|
||
|
|
||
|
public class Client
|
||
|
{
|
||
|
//////////////////////////////////
|
||
|
// Constants
|
||
|
private const int FILES_PER_CLASS = 9;
|
||
|
private const int MAX_PATH = 256;
|
||
|
private const int LINESIZE = 64;
|
||
|
private const int MARKER_FREQ = 4096;
|
||
|
private const int BUF_SIZE = 1024*1024; /* 1MB */
|
||
|
private const int MAX_DECK = 1000;
|
||
|
//////////////////////////////////
|
||
|
// Pseudo-constants
|
||
|
private static readonly int[] FILE_ORDER = {4,3,5,2,6,1,7,8,0};
|
||
|
private static readonly int[] CLASS_BASE_SIZE = {1024, 10240, 102400, 1024000};
|
||
|
private static readonly double[] CLASS_PROB = {0.35, 0.50, 0.14, 0.01};
|
||
|
private static readonly int MAX_CLASS_COUNT = CLASS_BASE_SIZE.Length;
|
||
|
|
||
|
//////////////////////////////////
|
||
|
// static variables
|
||
|
public static string pathPrefix = "";
|
||
|
public static int [] fileBucket;
|
||
|
|
||
|
//////////////////////////////////
|
||
|
// Instance variables
|
||
|
public Random.RandomState randomState;
|
||
|
public bool generatingUrls;
|
||
|
public Mutex globalLock;
|
||
|
public Mutex syncLock;
|
||
|
public UrlSet urlDeck;
|
||
|
public UrlSet freeDeck;
|
||
|
public UrlSet spareDeck;
|
||
|
#if SINGULARITY
|
||
|
public static TRef<DirectoryServiceContract.Imp:Ready> namespaceRootT;
|
||
|
private long fileCount = 0;
|
||
|
private long repeatCount = 0;
|
||
|
private bool dumpTable = false;
|
||
|
#else
|
||
|
/// <summary>
|
||
|
/// will not work for multiple threads!!!
|
||
|
/// </summary>
|
||
|
private static FileReader reader = new FileReader();
|
||
|
#endif
|
||
|
private long dirCount = 0;
|
||
|
private static long blockSize = 0;
|
||
|
private long totalSize = 0;
|
||
|
private long fileSizeNum = 0;
|
||
|
private long classSizeNum = 0;
|
||
|
private long threadNum = 1;
|
||
|
private long verbose = 0;
|
||
|
private bool testMode = false;
|
||
|
private int runTime;
|
||
|
private int rampTime;
|
||
|
private int forcedIterations;
|
||
|
|
||
|
private LocalState[] workerLocalStates;
|
||
|
private StringBuilder sb;
|
||
|
public static WorkerState currentState;
|
||
|
|
||
|
Random.ZipfState zipfDir;
|
||
|
Random.ZipfState zipfFile;
|
||
|
Random rand;
|
||
|
|
||
|
public enum WorkerState
|
||
|
{
|
||
|
Start = 0,
|
||
|
Rampup = 1,
|
||
|
Run = 2,
|
||
|
Done = 3,
|
||
|
}
|
||
|
|
||
|
// per-thread state
|
||
|
private class LocalState
|
||
|
{
|
||
|
public WorkerState State;
|
||
|
#if SINGULARITY
|
||
|
public DateTime StartTime;
|
||
|
#endif
|
||
|
public long bindCount;
|
||
|
public TimeSpan bindTime;
|
||
|
public long readCount;
|
||
|
public TimeSpan readTime;
|
||
|
public TimeSpan totalTime;
|
||
|
public long bytesRead;
|
||
|
public int forcedIterations;
|
||
|
|
||
|
public LocalState(int forcedIterations)
|
||
|
{
|
||
|
bytesRead = 0;
|
||
|
bindTime = new TimeSpan(0);
|
||
|
readTime = new TimeSpan(0);
|
||
|
totalTime = new TimeSpan(0);
|
||
|
|
||
|
readCount = 0;
|
||
|
bindCount = 0;
|
||
|
this.forcedIterations = forcedIterations;
|
||
|
}
|
||
|
}
|
||
|
|
||
|
#if SINGULARITY
|
||
|
[NotDelayed]
|
||
|
#endif
|
||
|
public Client(string/*!*/ prefix, long dir,
|
||
|
long bSize, long fSize,
|
||
|
long threads, long verbose,
|
||
|
long cSize, bool test,
|
||
|
int rampTime, int runTime,
|
||
|
int forcedIterations)
|
||
|
{
|
||
|
dirCount = dir;
|
||
|
pathPrefix = prefix;
|
||
|
fileSizeNum = fSize;
|
||
|
threadNum = threads;
|
||
|
classSizeNum = cSize;
|
||
|
blockSize = bSize;
|
||
|
this.verbose = verbose;
|
||
|
testMode = test;
|
||
|
this.rampTime = rampTime;
|
||
|
this.runTime = runTime;
|
||
|
this.forcedIterations = forcedIterations;
|
||
|
|
||
|
globalLock = new Mutex();
|
||
|
syncLock = new Mutex();
|
||
|
urlDeck = new UrlSet(MAX_DECK);
|
||
|
freeDeck = new UrlSet(MAX_DECK);
|
||
|
spareDeck = new UrlSet(MAX_DECK);
|
||
|
fileBucket = new int [FILES_PER_CLASS];
|
||
|
sb = new StringBuilder(512);
|
||
|
|
||
|
if (sb == null) throw new Exception (" could not allocate memory for string builder");
|
||
|
#if SINGULARITY
|
||
|
namespaceRootT = new TRef<DirectoryServiceContract.Imp:Ready>(
|
||
|
DirectoryService.NewClientEndpoint());
|
||
|
#else
|
||
|
#endif
|
||
|
workerLocalStates = new LocalState[threads];
|
||
|
|
||
|
if (workerLocalStates == null) {
|
||
|
throw new Exception ("unable to alloc workerstats");
|
||
|
}
|
||
|
}
|
||
|
|
||
|
//------------------------------------------------------------------
|
||
|
// ReadFile
|
||
|
// open and read its content of fileName
|
||
|
// throw the content away (no copying, no converting etc...)
|
||
|
//------------------------------------------------------------------
|
||
|
|
||
|
// FIXFIX way this is called is rather bizarre.
|
||
|
// a Client spawns LocalWorker objects that call back to this
|
||
|
// function. Should be moved into the LoadWorker class
|
||
|
|
||
|
#if SINGULARITY
|
||
|
private byte[]/*!*/ in ExHeap ReadFile(string/*!*/ fileName,
|
||
|
[Claims] byte[]/*!*/ in ExHeap inBuf,
|
||
|
ref LocalState local
|
||
|
)
|
||
|
{
|
||
|
if (local == null) {
|
||
|
Console.WriteLine("ReadFile: local state is null!");
|
||
|
return inBuf;
|
||
|
}
|
||
|
|
||
|
FileContract.Imp conn = null;
|
||
|
DateTime start;
|
||
|
DateTime beginTotal;
|
||
|
TimeSpan btime = new TimeSpan(0);
|
||
|
|
||
|
beginTotal = DateTime.Now;
|
||
|
|
||
|
// only perform timing while in the run state
|
||
|
|
||
|
DirectoryServiceContract.Imp:Ready! ds = Client.namespaceRootT.Acquire();
|
||
|
ErrorCode errorCode = ErrorCode.Unknown;
|
||
|
try {
|
||
|
if (currentState == WorkerState.Run) {
|
||
|
start = DateTime.Now;
|
||
|
conn = FileUtils.OpenFile(fileName, ds, out errorCode);
|
||
|
btime = DateTime.Now - start;
|
||
|
local.bindTime += btime;
|
||
|
local.bindCount++;
|
||
|
} else {
|
||
|
conn = FileUtils.OpenFile(fileName, ds, out errorCode);
|
||
|
}
|
||
|
}
|
||
|
finally {
|
||
|
Client.namespaceRootT.Release(ds);
|
||
|
}
|
||
|
|
||
|
if (conn == null) {
|
||
|
Console.WriteLine("Failed to open file '{0}' code={1}",fileName, SdsUtils.ErrorCodeToString(errorCode));
|
||
|
throw new Exception(" file missing");
|
||
|
return inBuf;
|
||
|
}
|
||
|
|
||
|
try {
|
||
|
int error = 0;
|
||
|
long filePos = 0;
|
||
|
long bytesRead = 0;
|
||
|
|
||
|
start = DateTime.Now;
|
||
|
|
||
|
while (!testMode) {
|
||
|
conn.SendRead(inBuf, 0, filePos, inBuf.Length);
|
||
|
conn.RecvAckRead(out inBuf, out bytesRead, out error);
|
||
|
local.readCount++;
|
||
|
if (error != 0) {
|
||
|
throw new Exception(" Encountered error while reading " + fileName);
|
||
|
break;
|
||
|
}
|
||
|
if (bytesRead == 0) break;
|
||
|
filePos += bytesRead;
|
||
|
}
|
||
|
|
||
|
if (currentState == WorkerState.Run) {
|
||
|
local.bytesRead += filePos;
|
||
|
TimeSpan x = DateTime.Now - start;
|
||
|
local.readTime += x;
|
||
|
}
|
||
|
}
|
||
|
finally {
|
||
|
delete conn;
|
||
|
}
|
||
|
|
||
|
if (currentState == WorkerState.Run) {
|
||
|
local.totalTime += DateTime.Now - beginTotal;
|
||
|
}
|
||
|
return inBuf;
|
||
|
}
|
||
|
#endif
|
||
|
|
||
|
///
|
||
|
/// generate a new "deck" of random files
|
||
|
///
|
||
|
public void GenerateFileSpecs()
|
||
|
{
|
||
|
FileSpec fs;
|
||
|
|
||
|
double totalProb = 0.0;
|
||
|
|
||
|
|
||
|
UrlSet fr = freeDeck;
|
||
|
UrlSet sp = spareDeck;
|
||
|
|
||
|
//
|
||
|
// Ensure the spare deck is fully allocated
|
||
|
// if there are any elements in the free pool use them
|
||
|
// otherwise we need a new allocations
|
||
|
//
|
||
|
globalLock.WaitOne();
|
||
|
#if verbose
|
||
|
Console.WriteLine("gendeck: {0} freed slots spare={1},",
|
||
|
fr.pos,sp.pos,urlDeck.pos);
|
||
|
//DebugStub.Break();
|
||
|
#endif
|
||
|
// fill up the spare deck. Start with using
|
||
|
// items from the free deck. If none are available
|
||
|
// allocate them.
|
||
|
|
||
|
// NOTE allocation should mostly happen during the initialization phase
|
||
|
|
||
|
for (int i=0; i < fr.urls.Length; i++){
|
||
|
if (fr.pos == 0 ) {
|
||
|
//if ( sp.urls[i] != null) DebugStub.Break();
|
||
|
sp.urls[i] = new FileSpec();
|
||
|
if (sp.urls[i] == null) {
|
||
|
throw new Exception ("no memory!");
|
||
|
}
|
||
|
// spit out diagnostics if we are allocating during the run phase
|
||
|
if (currentState == WorkerState.Run) Console.Write(" @{0}",i);
|
||
|
}
|
||
|
else {
|
||
|
sp.urls[i] = fr.urls[--fr.pos];
|
||
|
}
|
||
|
}
|
||
|
globalLock.ReleaseMutex();
|
||
|
|
||
|
// Now that we have a spare deck with the needed number of items
|
||
|
// fill in their values according to spec.
|
||
|
|
||
|
// The loop immediately below will generate s set of files conforming to the
|
||
|
// requested probabilities but will not be random.
|
||
|
|
||
|
for (int classNo = 0; classNo < MAX_CLASS_COUNT; classNo++) {
|
||
|
totalProb += CLASS_PROB[classNo];
|
||
|
while (sp.pos < totalProb * sp.urls.Length) {
|
||
|
fs = sp.urls[sp.pos++];
|
||
|
if (fs == null) {
|
||
|
throw new Exception(" spare[" + sp.pos + "] is null!");
|
||
|
}
|
||
|
fs.classNum = classNo;
|
||
|
fs.fileNum = FILE_ORDER[rand.spec_zipf(zipfFile)];
|
||
|
fs.dirNum = rand.spec_zipf(zipfDir);
|
||
|
|
||
|
fs.size = CLASS_BASE_SIZE[fs.classNum] * (fs.fileNum +1) / 10;
|
||
|
|
||
|
if (verbose > 4) {
|
||
|
sb.Length = 0; //reset the buffer
|
||
|
sb.Append(pathPrefix);
|
||
|
sb.Append("/dir");
|
||
|
sb.Append(fs.dirNum.ToString("d05"));
|
||
|
sb.Append("/class");
|
||
|
sb.Append(fs.classNum);
|
||
|
sb.Append("_");
|
||
|
sb.Append(fs.fileNum);
|
||
|
Console.WriteLine("size {0} file={1}",fs.size,sb.ToString());
|
||
|
}
|
||
|
}
|
||
|
if (verbose > 1) {
|
||
|
Console.WriteLine("Generating class[{0}] prob={1:f2} urlCount={2}",
|
||
|
classNo, CLASS_PROB[classNo], sp.pos);
|
||
|
}
|
||
|
}
|
||
|
|
||
|
/* Randomize deck */
|
||
|
for (int i = 0; i < sp.pos ; i++) {
|
||
|
int pos = rand.Spec_nrandom(randomState) % (sp.pos - i);
|
||
|
FileSpec obj = sp.urls[sp.pos- i - 1];
|
||
|
sp.urls[sp.pos - i - 1] = sp.urls[pos];
|
||
|
sp.urls[pos] = obj;
|
||
|
}
|
||
|
}
|
||
|
|
||
|
//------------------------------------------------------------------
|
||
|
// Generate Load
|
||
|
// creates the random list of files to be done every iteration
|
||
|
// Reads contents of entire file for every file in the list
|
||
|
// This list is generated so we can see if each loop has identical timings
|
||
|
//------------------------------------------------------------------
|
||
|
private class LoadWorker
|
||
|
{
|
||
|
private Client hostClient;
|
||
|
private LocalState localState;
|
||
|
#if SINGULARITY
|
||
|
private readonly ArrayList fs;
|
||
|
#endif
|
||
|
private StringBuilder sb;
|
||
|
private int readSize;
|
||
|
private Cache cache;
|
||
|
private int id; // local thread index
|
||
|
|
||
|
public LoadWorker(Client parent, ref LocalState localState, int readSize, int id)
|
||
|
{
|
||
|
hostClient = parent;
|
||
|
this.localState = localState;
|
||
|
sb = new StringBuilder(512);
|
||
|
this.readSize = readSize;
|
||
|
cache = new Cache(20, 20 *2);
|
||
|
this.id = id;
|
||
|
}
|
||
|
#if SINGULARITY
|
||
|
public byte []/*!*/ in ExHeap ProcessUrl(
|
||
|
FileSpec fileSpec,
|
||
|
[Claims] byte []/*!*/ in ExHeap buf) {
|
||
|
#else
|
||
|
public void ProcessUrl(FileSpec fileSpec, byte[] buf, ref LocalState local ) {
|
||
|
#endif
|
||
|
if (fileSpec == null) throw new Exception ("Attempting to read from null fileSpec");
|
||
|
#if SINGULARITY
|
||
|
sb.Length = 0; //reset the buffer
|
||
|
sb.Append(pathPrefix);
|
||
|
sb.Append("/dir");
|
||
|
sb.Append(fileSpec.dirNum.ToString("d05"));
|
||
|
sb.Append("/class");
|
||
|
sb.Append(fileSpec.classNum);
|
||
|
sb.Append("_");
|
||
|
sb.Append(fileSpec.fileNum);
|
||
|
string path = sb.ToString();
|
||
|
|
||
|
if (hostClient.verbose > 2) {
|
||
|
Console.WriteLine("reading {0}", path);
|
||
|
}
|
||
|
buf = hostClient.ReadFile(path, buf, ref localState);
|
||
|
#else
|
||
|
sb.Length = 0; //reset the buffer
|
||
|
sb.Append(pathPrefix);
|
||
|
sb.Append("\\dir");
|
||
|
sb.Append(fileSpec.dirNum.ToString("d05"));
|
||
|
sb.Append("\\class");
|
||
|
sb.Append(fileSpec.classNum);
|
||
|
sb.Append("_");
|
||
|
sb.Append(fileSpec.fileNum);
|
||
|
string path = sb.ToString();
|
||
|
|
||
|
int error = 0;
|
||
|
//Console.Write(".");
|
||
|
DateTime beginTotal = DateTime.Now;
|
||
|
if (!reader.Open(path)) {
|
||
|
Console.WriteLine("open failed: {0}", path);
|
||
|
throw new Exception("open failure: "+path);
|
||
|
}
|
||
|
|
||
|
DateTime beginRead = DateTime.Now;
|
||
|
for (long pos = 0; pos < fileSpec.size;) {
|
||
|
int bytes = reader.Read(buf, 0, buf.Length);
|
||
|
//Console.Write(".");
|
||
|
if (bytes == 0) {
|
||
|
Console.WriteLine("bytes read={0}, error={1} pos={2}",
|
||
|
bytes, error, pos);
|
||
|
break;
|
||
|
}
|
||
|
pos += bytes;
|
||
|
//if (pos != fileSpec.size) throw new Exception (" wrong size pos=" + pos + " bytes=" + bytes);
|
||
|
}
|
||
|
reader.Close();
|
||
|
if (currentState == WorkerState.Run) {
|
||
|
DateTime end = DateTime.Now;
|
||
|
local.totalTime += end - beginTotal;
|
||
|
local.bindTime += beginRead - beginTotal;
|
||
|
local.readTime += end - beginRead ;
|
||
|
local.bytesRead += fileSpec.size;
|
||
|
}
|
||
|
#endif
|
||
|
Client.fileBucket[fileSpec.fileNum]++;
|
||
|
path = null;
|
||
|
#if SINGULARITY
|
||
|
return buf;
|
||
|
#endif
|
||
|
}
|
||
|
|
||
|
public void RunLoad()
|
||
|
{
|
||
|
#if SINGULARITY
|
||
|
byte [] in ExHeap buf = new[ExHeap] byte[readSize];
|
||
|
#else
|
||
|
// Allocate one one buffer for all operations
|
||
|
byte[] buf = new byte [blockSize];
|
||
|
#endif
|
||
|
// if loop we've been told to quit, but run at least the forcedIterations.
|
||
|
for (int iterations = 0;; iterations++) {
|
||
|
if (iterations >= localState.forcedIterations &&
|
||
|
localState.State == WorkerState.Done) {
|
||
|
break;
|
||
|
}
|
||
|
|
||
|
FileSpec fs = cache.GetUrl(hostClient,id);
|
||
|
#if SINGULARITY
|
||
|
buf = ProcessUrl(fs, buf);
|
||
|
#else
|
||
|
ProcessUrl(fs,buf, ref localState);
|
||
|
#endif
|
||
|
cache.FreeUrl(fs);
|
||
|
}
|
||
|
#if SINGULARITY
|
||
|
delete buf;
|
||
|
#endif
|
||
|
}
|
||
|
}
|
||
|
|
||
|
public void InitializeGlobalState()
|
||
|
{
|
||
|
// the following initialization is taken from specweb99/load_generate and client.c
|
||
|
rand = new Random();
|
||
|
int first_seed = 2;
|
||
|
randomState = new Random.RandomState(first_seed);
|
||
|
zipfDir = rand.spec_zipf_setup(randomState, (int)dirCount, 1.0);
|
||
|
zipfFile = rand.spec_zipf_setup(randomState, FILES_PER_CLASS, 1.0);
|
||
|
generatingUrls = false;
|
||
|
}
|
||
|
|
||
|
public long GatherStats(out long totalFiles)
|
||
|
{
|
||
|
long total = 0;
|
||
|
totalFiles = 0;
|
||
|
Console.WriteLine("Thread stats:");
|
||
|
for (int i = 0; i < threadNum; i++) {
|
||
|
LocalState temp = workerLocalStates[i];
|
||
|
if (temp != null) {
|
||
|
double t1 = (double)temp.totalTime.Ticks / TimeSpan.TicksPerSecond;
|
||
|
double t2 = (double)temp.bindTime.Ticks / TimeSpan.TicksPerSecond;
|
||
|
double t3 = (double)temp.readTime.Ticks / TimeSpan.TicksPerSecond;
|
||
|
string summ = String.Format(
|
||
|
" [{0}] Total: {1:f3} bind: {2:f3}, read: {3:f3} bytes: {4}",
|
||
|
i, t1, t2, t3, temp.bytesRead);
|
||
|
Console.WriteLine(summ);
|
||
|
#if SINGULARITY
|
||
|
DebugStub.Print(summ);
|
||
|
#endif // SINGULARITY
|
||
|
//Console.WriteLine(" bind count ={0}, read count ={1}",temp.bindCount, temp.readCount);
|
||
|
total += temp.bytesRead;
|
||
|
totalFiles += temp.bindCount;
|
||
|
}
|
||
|
}
|
||
|
return total;
|
||
|
}
|
||
|
|
||
|
///
|
||
|
/// Main load generation routine
|
||
|
/// manages Global state machine via timers etc.
|
||
|
public void GenerateLoad()
|
||
|
{
|
||
|
totalSize = 0;
|
||
|
double megaBytes;
|
||
|
InitializeGlobalState();
|
||
|
|
||
|
//if (testMode) return;
|
||
|
totalSize *= threadNum;
|
||
|
//megaBytes = ( (double) (totalSize) / (double) (1024*1024) );
|
||
|
|
||
|
#if SINGULARITY
|
||
|
long startIrqCount;
|
||
|
long startSwitchCount;
|
||
|
long startKernelGcCount;
|
||
|
int startGcCount;
|
||
|
long startGcMillis;
|
||
|
long startGcBytes;
|
||
|
long endGcMillis;
|
||
|
long endGcBytes;
|
||
|
TimeSpan startTime;
|
||
|
TimeSpan endTime;
|
||
|
#else
|
||
|
DateTime startTime;
|
||
|
DateTime endTime;
|
||
|
#endif
|
||
|
|
||
|
//DateTime globalStartTime = ProcessService.GetUpTime();
|
||
|
|
||
|
// Create all the workers
|
||
|
LoadWorker[] workers = new LoadWorker[threadNum];
|
||
|
Thread[] threads = new Thread[threadNum];
|
||
|
|
||
|
for (int i = 0; i < threadNum; i++) {
|
||
|
workerLocalStates[i] = new LocalState(forcedIterations);
|
||
|
workers[i] = new LoadWorker(this, ref workerLocalStates[i],(int) blockSize,i);
|
||
|
}
|
||
|
|
||
|
// Create the threads
|
||
|
for (int i = 0; i < threadNum; i++) {
|
||
|
LocalState temp = workerLocalStates[i];
|
||
|
if (temp != null) temp.State = WorkerState.Rampup;
|
||
|
threads[i] = new Thread(new ThreadStart(workers[i].RunLoad));
|
||
|
}
|
||
|
|
||
|
if (rampTime != 0) {
|
||
|
// ramp-up time isn't part of benchmark.
|
||
|
|
||
|
Console.WriteLine("Entering Rampup state");
|
||
|
// Run all the threads
|
||
|
currentState = WorkerState.Rampup;
|
||
|
for (int i = 0; i < threadNum; i++) {
|
||
|
Thread t = threads[i];
|
||
|
/*^ assume t!= null; ^*/
|
||
|
t.Start();
|
||
|
}
|
||
|
|
||
|
// sleep for some time and change states from Rampup to Run
|
||
|
Thread.Sleep(rampTime*1000);
|
||
|
Console.WriteLine("\nEntering Run state");
|
||
|
globalLock.WaitOne();
|
||
|
|
||
|
// start perf counting after we have the lock
|
||
|
#if SINGULARITY
|
||
|
GC.PerformanceCounters(out startGcCount,
|
||
|
out startGcMillis,
|
||
|
out startGcBytes);
|
||
|
|
||
|
startTime = ProcessService.GetUpTime();
|
||
|
startIrqCount = ProcessService.GetKernelInterruptCount();
|
||
|
startSwitchCount = ProcessService.GetContextSwitchCount();
|
||
|
startKernelGcCount = ProcessService.GetKernelGcCount();
|
||
|
#else
|
||
|
startTime = DateTime.Now;
|
||
|
#endif
|
||
|
}
|
||
|
else {
|
||
|
// ramp-up time is part of benchmark.
|
||
|
#if SINGULARITY
|
||
|
GC.PerformanceCounters(out startGcCount,
|
||
|
out startGcMillis,
|
||
|
out startGcBytes);
|
||
|
|
||
|
startTime = ProcessService.GetUpTime();
|
||
|
startIrqCount = ProcessService.GetKernelInterruptCount();
|
||
|
startSwitchCount = ProcessService.GetContextSwitchCount();
|
||
|
startKernelGcCount = ProcessService.GetKernelGcCount();
|
||
|
#else
|
||
|
startTime = DateTime.Now;
|
||
|
#endif
|
||
|
|
||
|
Console.WriteLine("Starting threads");
|
||
|
// Run all the threads
|
||
|
currentState = WorkerState.Rampup;
|
||
|
for (int i = 0; i < threadNum; i++) {
|
||
|
Thread t = threads[i];
|
||
|
/*^ assume t!= null; ^*/
|
||
|
t.Start();
|
||
|
}
|
||
|
|
||
|
// sleep for some time and change states from Rampup to Run
|
||
|
Thread.Sleep(rampTime*1000);
|
||
|
Console.WriteLine("\nEntering Run state");
|
||
|
globalLock.WaitOne();
|
||
|
}
|
||
|
|
||
|
// Workers are ready to run.
|
||
|
currentState = WorkerState.Run;
|
||
|
|
||
|
for (int i = 0; i < threadNum; i++) {
|
||
|
LocalState temp = workerLocalStates[i];
|
||
|
if (temp != null) temp.State = WorkerState.Run;
|
||
|
}
|
||
|
globalLock.ReleaseMutex();
|
||
|
|
||
|
|
||
|
// sleep for some time and change states from Run to Done
|
||
|
Thread.Sleep(runTime*1000);
|
||
|
globalLock.WaitOne();
|
||
|
currentState = WorkerState.Done;
|
||
|
for (int i = 0; i < threadNum; i++) {
|
||
|
LocalState temp = workerLocalStates[i];
|
||
|
if (temp != null) temp.State = WorkerState.Done;
|
||
|
}
|
||
|
globalLock.ReleaseMutex();
|
||
|
|
||
|
#if SINGULARITY
|
||
|
bool wasActive = Monitoring.isActive();
|
||
|
if (wasActive) Console.WriteLine("May want to reset Monitoring state -- was active");
|
||
|
Monitoring.setActive(false);
|
||
|
// Wait for all the threads to complete
|
||
|
for (int i = 0 ; i < threadNum; i++) {
|
||
|
Thread t = threads[i];
|
||
|
/*^ assume t!= null; ^*/
|
||
|
t.Join();
|
||
|
}
|
||
|
#endif
|
||
|
|
||
|
// should stats happen before or after thread join?
|
||
|
long totalFiles;
|
||
|
long totalBytes = GatherStats(out totalFiles);
|
||
|
|
||
|
megaBytes = ( (double) (totalBytes) / (double) (1024*1024) );
|
||
|
|
||
|
#if SINGULARITY
|
||
|
int endGcCount;
|
||
|
endTime = ProcessService.GetUpTime();
|
||
|
GC.PerformanceCounters(out endGcCount,
|
||
|
out endGcMillis,
|
||
|
out endGcBytes);
|
||
|
#else
|
||
|
endTime = DateTime.Now;
|
||
|
#endif
|
||
|
|
||
|
|
||
|
TimeSpan elapsed = endTime - startTime;
|
||
|
double delta = (double)elapsed.Ticks / TimeSpan.TicksPerSecond;
|
||
|
string summ =
|
||
|
String.Format(
|
||
|
"Class:{0:d} Block:{1:d} Elapsed: {2:f2} MBps: {3:f3}, Files/s: {4:f3}",
|
||
|
classSizeNum,
|
||
|
blockSize,
|
||
|
delta,
|
||
|
(double)(megaBytes / delta),
|
||
|
(double)(totalFiles / delta));
|
||
|
|
||
|
#if SINGULARITY
|
||
|
string info =
|
||
|
String.Format(
|
||
|
"[AppGC:- cnt:{0} bytes:{1} Kern:- ints:{2} swi:{3} gcs:{4}]",
|
||
|
endGcCount - startGcCount,
|
||
|
endGcBytes - startGcBytes,
|
||
|
ProcessService.GetKernelInterruptCount() - startIrqCount,
|
||
|
ProcessService.GetContextSwitchCount() - startSwitchCount,
|
||
|
ProcessService.GetKernelGcCount() - startKernelGcCount);
|
||
|
#else
|
||
|
string info = "not implemented";
|
||
|
#endif
|
||
|
if (verbose > 0 ) {
|
||
|
Console.WriteLine();
|
||
|
Console.WriteLine(summ);
|
||
|
Console.WriteLine(info);
|
||
|
}
|
||
|
|
||
|
|
||
|
if (verbose > 1 ) {
|
||
|
long ftotal = 0;
|
||
|
for (int i=0; i < 9; i++) {
|
||
|
ftotal += fileBucket[i];
|
||
|
}
|
||
|
|
||
|
Console.WriteLine("Actual Zipf file distribution:");
|
||
|
for (int i=0; i < 9; i++) {
|
||
|
double foo = (double) (fileBucket[i] * 100) / (double) ftotal;
|
||
|
Console.WriteLine(" FileBucket[{0}] = {1:d6}. %={2:f2}",
|
||
|
i, fileBucket[i], foo);
|
||
|
}
|
||
|
}
|
||
|
#if SINGULARITY
|
||
|
DebugStub.WriteLine(summ);
|
||
|
DebugStub.WriteLine(info);
|
||
|
#endif
|
||
|
}
|
||
|
} // client
|
||
|
|
||
|
public class SpecMain
|
||
|
{
|
||
|
//--------------------------------------------------
|
||
|
//--------------------------------------------------
|
||
|
private static int ConnectionsToDirectories(int conns)
|
||
|
{
|
||
|
return ((int) (25 + (((400000.0 / 122000.0) * conns)/5.0)));
|
||
|
}
|
||
|
|
||
|
static bool ParseNumber(string/*!*/ arg,
|
||
|
string/*!*/ name,
|
||
|
out long value)
|
||
|
{
|
||
|
// arg should look like "[-][A-z]:[0-9]*"
|
||
|
if (arg.Length >= 4)
|
||
|
{
|
||
|
try
|
||
|
{
|
||
|
value = Int64.Parse(arg.Substring(3));
|
||
|
return true;
|
||
|
}
|
||
|
catch (FormatException)
|
||
|
{Console.WriteLine("format ex"); }
|
||
|
catch (OverflowException)
|
||
|
{Console.WriteLine("overflow ex");}
|
||
|
}
|
||
|
Console.WriteLine("Could not parse {0}, {1}", name,arg.Substring(3));
|
||
|
value = 0;
|
||
|
return false;
|
||
|
}
|
||
|
|
||
|
private static void Usage()
|
||
|
{
|
||
|
Console.WriteLine("Usage: specweb99 client [options] content-root");
|
||
|
Console.WriteLine(" randomly read files generated by wafgen99");
|
||
|
Console.WriteLine("Options:");
|
||
|
Console.WriteLine(
|
||
|
" /b:<n> Block Size to be read [{0}].",
|
||
|
Defaults.WarmUpSeconds
|
||
|
);
|
||
|
Console.WriteLine(
|
||
|
" /c:<n> Set number of connections to simulate [{0}].",
|
||
|
Defaults.Connections
|
||
|
);
|
||
|
Console.WriteLine(" /k:<n> Restrict files to this class number.");
|
||
|
Console.WriteLine(" /f:<n> Forced iterations [0].");
|
||
|
Console.WriteLine(" /r:<n> run time in seconds [{0}].",
|
||
|
Defaults.RunSeconds);
|
||
|
Console.WriteLine(" /t:<n> Number of threads to use [1].");
|
||
|
Console.WriteLine(" /x no read -- just bind");
|
||
|
Console.WriteLine(" /w:<n> warmup time in seconds[5.]");
|
||
|
}
|
||
|
|
||
|
#if SINGULARITY
|
||
|
internal static int AppMain(Parameters! config)
|
||
|
{
|
||
|
|
||
|
long dirCount = (long) (ConnectionsToDirectories((int)config.connections) -1 ); //zero based
|
||
|
long blockSize = config.blockSize;
|
||
|
long fileSizeNum = -1;
|
||
|
long classSizeNum = config.classSizeNum;
|
||
|
long verbose = config.verbose;
|
||
|
long numThreads = config.numThreads;
|
||
|
bool testMode = config.testMode;
|
||
|
long runTime = config.runTime;
|
||
|
long rampTime = config.rampTime;
|
||
|
long forcedIterations = config.forcedIterations;
|
||
|
string contentPrefix = config.contentPrefix + "/" + Defaults.RelativePath;
|
||
|
if (forcedIterations != 0) {
|
||
|
rampTime = 0;
|
||
|
runTime = 0;
|
||
|
}
|
||
|
Console.WriteLine(" args: content=" + contentPrefix +" dir=" + dirCount + " block size="+blockSize);
|
||
|
Console.WriteLine(" verbose="+verbose+" FileSizeNum="+fileSizeNum+" ClassSizeNum="+classSizeNum);
|
||
|
Console.WriteLine(" threads="+numThreads+" forcedIterations="+forcedIterations);
|
||
|
//DateTime start = DateTime.Now;
|
||
|
|
||
|
Client c = new Client(contentPrefix, dirCount, blockSize, fileSizeNum,
|
||
|
numThreads, verbose, classSizeNum,
|
||
|
testMode, (int) rampTime, (int)runTime,
|
||
|
(int)forcedIterations);
|
||
|
|
||
|
if (c != null ) {
|
||
|
c.GenerateLoad();
|
||
|
}
|
||
|
|
||
|
//TimeSpan elapsed = DateTime.Now - beginRunTime;
|
||
|
//Console.WriteLine("Elapsed time: "+elapsed);
|
||
|
return 0;
|
||
|
|
||
|
}
|
||
|
#else
|
||
|
//--------------------------------------------------
|
||
|
// Windows Main routine
|
||
|
//--------------------------------------------------
|
||
|
public static int Main(string[]/*!*/ args)
|
||
|
{
|
||
|
string contentPrefix = Defaults.Root;
|
||
|
long dirCount = (long) (ConnectionsToDirectories(Defaults.Connections) -1); //zero based
|
||
|
long blockSize = Defaults.Connections;
|
||
|
long fileSizeNum = -1;
|
||
|
long classSizeNum = -1;
|
||
|
long verbose = 1;
|
||
|
long numThreads = 1;
|
||
|
bool testMode = false;
|
||
|
long runTime = Defaults.RunSeconds;
|
||
|
long rampTime = Defaults.WarmUpSeconds;
|
||
|
long forcedIterations = 0;
|
||
|
|
||
|
// Temporaries for command-line parsing
|
||
|
bool needHelp = args.Length == 0;
|
||
|
|
||
|
int i;
|
||
|
for (i = 1; i < args.Length; i++)
|
||
|
{
|
||
|
string/*!*/ arg = (string /*!*/) args[i];
|
||
|
|
||
|
if (arg.Length < 2 || (arg[0] != '-' && arg[0] != '/') )
|
||
|
{
|
||
|
Console.WriteLine("Invalid argument: {0}", arg);
|
||
|
Usage();
|
||
|
return -1;
|
||
|
}
|
||
|
|
||
|
switch (Char.ToLower(arg[1]))
|
||
|
{
|
||
|
case '?':
|
||
|
case 'h':
|
||
|
needHelp = true;
|
||
|
break;
|
||
|
case 'b':
|
||
|
ParseNumber(arg, "block size", out blockSize);
|
||
|
break;
|
||
|
case 'c':
|
||
|
long connections;
|
||
|
ParseNumber(arg, "# of connections", out connections);
|
||
|
dirCount = (long) (ConnectionsToDirectories((int)connections) -1 ); //zero based
|
||
|
|
||
|
break;
|
||
|
case 'f':
|
||
|
ParseNumber(arg, "# of forced iterations", out forcedIterations);
|
||
|
rampTime = 0;
|
||
|
runTime = 0;
|
||
|
break;
|
||
|
case 'k':
|
||
|
ParseNumber(arg, "class number (size) to be read", out classSizeNum);
|
||
|
break;
|
||
|
case 'r':
|
||
|
ParseNumber(arg, "Run time", out runTime);
|
||
|
break;
|
||
|
case 't':
|
||
|
#if SINGULARITY
|
||
|
ParseNumber(arg, "number of threads to use", out numThreads);
|
||
|
#else
|
||
|
Console.WriteLine("Multi-thread not spported on NT -- ignoring");
|
||
|
#endif
|
||
|
break;
|
||
|
case 'v':
|
||
|
ParseNumber(arg, "verbosity level", out verbose);
|
||
|
break;
|
||
|
case 'w':
|
||
|
ParseNumber(arg, "Warmup time", out rampTime);
|
||
|
break;
|
||
|
case 'x':
|
||
|
testMode = true;
|
||
|
break;
|
||
|
default:
|
||
|
contentPrefix = arg;
|
||
|
break;
|
||
|
}
|
||
|
} // for args
|
||
|
|
||
|
contentPrefix += "/" + Defaults.RelativePath;
|
||
|
|
||
|
if (needHelp) {
|
||
|
Usage();
|
||
|
return -1;
|
||
|
}
|
||
|
|
||
|
Console.WriteLine(" args: content=" + contentPrefix +" dir=" + dirCount + " block size="+blockSize);
|
||
|
Console.WriteLine(" verbose="+verbose+" FileSizeNum="+fileSizeNum+" ClassSizeNum="+classSizeNum);
|
||
|
Console.WriteLine(" threads="+numThreads+" forcedIterations="+forcedIterations);
|
||
|
//DateTime start = DateTime.Now;
|
||
|
|
||
|
Client c = new Client(contentPrefix, dirCount, blockSize, fileSizeNum,
|
||
|
numThreads, verbose, classSizeNum,
|
||
|
testMode, (int) rampTime, (int)runTime,
|
||
|
(int)forcedIterations);
|
||
|
|
||
|
if (c != null ) {
|
||
|
c.GenerateLoad();
|
||
|
}
|
||
|
|
||
|
//TimeSpan elapsed = DateTime.Now - beginRunTime;
|
||
|
//Console.WriteLine("Elapsed time: "+elapsed);
|
||
|
return 0;
|
||
|
} // main
|
||
|
#endif
|
||
|
} //specmain class
|
||
|
} //namespace
|