// ---------------------------------------------------------------------------- // // Copyright (c) Microsoft Corporation. All rights reserved. // // ---------------------------------------------------------------------------- // // Ported from SpecWeb99 source code // using FileSystem.Utils; using Microsoft.SingSharp; using Microsoft.SingSharp.Runtime; using Microsoft.Singularity; using Microsoft.Singularity.Channels; using Microsoft.Singularity.Directory; using System; using System.Diagnostics; using System.IO; using System.Text; /////////////////////////////////////// // Quick and dirty abstraction of file operations ////////////////////////////////////// using Microsoft.Contracts; using Microsoft.SingSharp.Reflection; using Microsoft.Singularity.Applications; using Microsoft.Singularity.Io; using Microsoft.Singularity.Configuration; [assembly: Transform(typeof(ApplicationResourceTransform))] namespace Microsoft.Singularity.Applications { [ConsoleCategory(HelpMessage="Generate files for Specweb", DefaultAction=true)] internal class Parameters { [InputEndpoint("data")] public readonly TRef Stdin; [OutputEndpoint("data")] public readonly TRef Stdout; [Endpoint] public readonly TRef nsRef; [BoolParameter( "v", Default=false, HelpMessage="verbose")] internal bool doVerbose; [BoolParameter( "V", Default=false, HelpMessage="validate only")] internal bool doValidateOnly; [BoolParameter( "t", Default=false, HelpMessage="test mode")] internal bool doTestMode; [BoolParameter( "p", Default=false, HelpMessage="Purge files on exit")] internal bool doCleanup; [LongParameter( "n", Default=9, HelpMessage="Files per class")] internal long filesPerClass; [LongParameter( "c", Default=10, HelpMessage="Number of connections")] internal long connections; [StringParameter( "directory", Mandatory=true, Position=0, HelpMessage="directory.")] internal string directory; reflective internal Parameters(); internal int AppMain() { WafGen99.AppMain(this); return 0; } } public class FILE { static private VContainer! workBuffer; private TRef! connRef; private long filePos; private string! path; public static FILE CreateFile(string! path) { if (FileUtils.CreateFile(path) != 0) { return null; } return OpenFile(path); } public static FILE OpenFile(string! path) { FileContract.Imp conn = FileUtils.OpenFile(path); if (conn != null) { return new FILE(conn, path); } else { return null; } } static FILE() { workBuffer = new VContainer(new [ExHeap] byte[2 * 1024 * 1024]); } private FILE([Claims] FileContract.Imp:Ready! conn, string! thePath) { connRef = new TRef(conn); filePos = 0; path = thePath; base(); } public int Write(string! str) { byte[] bytes = Encoding.ASCII.GetBytes(str); return Write(bytes, 0, bytes.Length); } public int Write(byte[]! buf, int offset, int length) { FileContract.Imp conn = connRef.Acquire(); byte[]! in ExHeap wbuf = workBuffer.Acquire(); if (length > wbuf.Length) { DebugStub.Print("workbuf resize {0} -> {1} (write)\n", __arglist(wbuf.Length, length)); delete wbuf; wbuf = new [ExHeap] byte [length]; } Bitter.FromByteArray(wbuf, 0, length, buf, offset); try { long bytesWritten; int error; conn.SendWrite(wbuf, 0, filePos, length); conn.RecvAckWrite(out wbuf, out bytesWritten, out error); workBuffer.Release(wbuf); filePos += bytesWritten; if (error == 0) { return (int)bytesWritten; } else { throw new Exception("Write failed"); } } finally { connRef.Release(conn); } return 0; } public int Read(byte[]! buf, int offset, int length) { FileContract.Imp conn = connRef.Acquire(); byte[]! in ExHeap bytes = workBuffer.Acquire(); if (length > bytes.Length) { DebugStub.Print("workbuf resize {0} -> {1} (read)\n", __arglist(bytes.Length, length)); delete bytes; bytes = new [ExHeap] byte [length]; } try { long bytesRead; int error; conn.SendRead(bytes, 0, filePos, length); conn.RecvAckRead(out bytes, out bytesRead, out error); filePos += bytesRead; if (error == 0) { Bitter.ToByteArray(bytes, 0, length, buf, offset); workBuffer.Release(bytes); return (int)bytesRead; } else { workBuffer.Release(bytes); throw new Exception("Read failed"); return 0; } } finally { connRef.Release(conn); } } public int Size { // TODO get { FileAttributesRecord attr; ErrorCode error; bool ok = FileUtils.GetAttributes(path, out attr, out error); if (!ok) { throw new Exception("Failed to get attributes"); } return unchecked((int)attr.FileSize); } } public void Close() { FileContract.Imp conn = connRef.Acquire(); try { conn.SendClose(); } finally { connRef.Release(conn); } } } /////////////////////////////////////// // The content generator / verifier ////////////////////////////////////// public class WafGen99 { ////////////////////////////////// // Constants private const int MAX_PATH = 256; private const int LINESIZE = 64; private const int MARKER_FREQ = 4096; private const int BUF_SIZE = 1024*1024; // 1MB private const string topDirName = "file_set"; ////////////////////////////////// // Pseudo-constants private static readonly int[]! CLASS_BASE_SIZE = {1024, 10240, 102400, 1024000}; private static readonly int MAX_CLASS_COUNT = CLASS_BASE_SIZE.Length; ////////////////////////////////// // Instance variables private bool verbose = false; private bool testMode = false; private byte[]! buf; private string baseDir; private readonly int filesPerClass; public WafGen99(bool verb, bool test, string theBase, int filesPerClass) { buf = new byte[BUF_SIZE]; base(); verbose = verb; testMode = test; this.filesPerClass = filesPerClass; baseDir = theBase; } private static byte GetTestValue(int index) { string pattern = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; return (byte) pattern[index % pattern.Length]; } // Returns false on error private bool CreateFile(int classNum, int num, int size, string! name, string! path) { FILE fp; int initialSize = size; int[] incLocations = new int[MAX_CLASS_COUNT * filesPerClass * 10]; int markerCt; int includes; int includeCt; int markersPerInclude; if (verbose) { Console.WriteLine("Creating {0} [ {1} bytes ]", name, size); } // Try to open the file fp = FILE.CreateFile(path); if (fp == null) { Console.WriteLine("Can't open file '" + path + "'"); return false; } // Write a header line size -= fp.Write(String.Format("{0,9:d} {1}\n", size, name)); markerCt = size / MARKER_FREQ + 1; includeCt = size / (4096*4) + 1; if (classNum == 0) { includeCt = 0; } else if (classNum == 1) { includeCt = 1; } else if (classNum == 2) { includeCt = 2; } else { includeCt = 0; } Random rand = new Random(); if (includeCt > 0) { markersPerInclude = markerCt / includeCt; for (int i = 0; i < includeCt; i++) { // The replacements can't cover up the implanted validation //characters -- they will always just after a marker incLocations[i] = rand.Next(markersPerInclude * i, markersPerInclude * (i+1) - 1); if (incLocations[i] > markerCt) { incLocations[i] = markerCt; } } } includes = 0; markerCt = 0; int pos = 0; int nameOffset = 0; for (int i = 0; i < size; i++) { if (!testMode && (i % MARKER_FREQ == 0)) { buf[pos++] = ((byte)name[nameOffset++]); if ((includes < includeCt) && (incLocations[includes] == markerCt)) { includes++; string markerStr = String.Format("\n"); int encodedBytes = Encoding.ASCII.GetBytes(markerStr, 0, markerStr.Length, buf, pos); i += encodedBytes; pos += encodedBytes; } markerCt++; if (nameOffset == name.Length) { nameOffset = 0; } } else if (i % LINESIZE == LINESIZE - 1) { buf[pos++] = (byte)'\n'; } else if (testMode) { buf[pos++] = GetTestValue(i); } else { buf[pos++] = (byte)rand.Next(33,125); } if (pos >= BUF_SIZE) { fp.Write(buf, 0, pos); pos = 0; } } if (pos > 0) { fp.Write(buf, 0, pos); } fp.Close(); if (!ValidateFile(classNum, num, initialSize, name, path)) { DebugStub.Break(); ValidateFile(classNum, num, initialSize, name, path); return false; } return true; } // returns false on failure bool ValidateFile(int classNum, int fileNum, int size, string! name, string! path) { FILE fp; if (verbose) { Console.WriteLine("Verifying {0} [ {1} bytes ]", name, size); } fp = FILE.OpenFile(path); if (fp == null) { Console.WriteLine("Couldn't open file '" + path + "'"); return false; } if (fp.Size != size) { Console.WriteLine("{0}: Incorrect file size (expected {1} bytes, have {2} bytes).", name, size, fp.Size); return false; } string valString = String.Format("{0,9:d} {1}\n", size, name); int headerSize = valString.Length; // Read the file header fp.Read(buf, 0, headerSize); string readString = Encoding.ASCII.GetString(buf, 0, headerSize); if (valString != readString) { Console.WriteLine("{0}: Header for file is incorrect", name); return false; } int curBufSize = fp.Read(buf, 0, BUF_SIZE); int offset = 0, nameOffset = 0; // Examine every byte in the file in this loop for (int i = 0; i < size - headerSize; i++) { if (!testMode && (i % MARKER_FREQ == 0)) { if (buf[i - offset] != name[nameOffset++]) { Console.WriteLine("{0}:Validation data incorrect at offset {1}", name, headerSize + offset + i); fp.Close(); return false; } if (nameOffset == name.Length) { nameOffset = 0; } } else if (i % LINESIZE == LINESIZE - 1) { if (buf[i - offset] != '\n') { Console.WriteLine("{0}:Missing newline at offset {1}", name, headerSize + offset + i); fp.Close(); return false; } } else if (testMode) { byte expected = GetTestValue(i); if (buf[i - offset] != expected) { Console.WriteLine("{0}:Bad byte value at offset {1} ({2} != {3})", name, headerSize + offset + i, buf[i - offset], expected); fp.Close(); return false; } } if (i >= curBufSize) { offset += curBufSize; curBufSize = fp.Read(buf, 0, BUF_SIZE); if (curBufSize <= 0) { Console.WriteLine("{0}: Error reading from file", path); fp.Close(); return false; } } } fp.Close(); return true; } private struct FileSpec { public string name, fullPath; public int size, classNum, fileNum; } private string[]! MakeDirList(int startDir, int endDir) { int numDirs = endDir - startDir; string[] retval = new string[numDirs]; for (int dir = startDir; dir < endDir; dir++) { retval[dir - startDir] = String.Format("{0}/dir{1}", topDirName, dir.ToString("d05")); } return retval; } private FileSpec[]! MakeFileList(string[]! dirs) { FileSpec[] retval = new FileSpec[MAX_CLASS_COUNT * filesPerClass * dirs.Length]; int idx = 0; foreach (string dirName in dirs) { for (int classNum = 0; classNum < MAX_CLASS_COUNT; classNum++) { for (int num = 0; num < filesPerClass; num++) { int size = (CLASS_BASE_SIZE[classNum] * (num+1)) / 10; retval[idx] = new FileSpec(); retval[idx].name = String.Format("{0}/class{1,1:d}_{2,1:d}", dirName, classNum, num); retval[idx].fullPath = String.Format("{0}/{1}", baseDir, retval[idx].name); retval[idx].classNum = classNum; retval[idx].fileNum = num; retval[idx].size = size; idx++; } } } return retval; } // Returns false on failure private bool CreateFiles(int startDir, int endDir) { string topDirectory = String.Format("{0}/{1}", baseDir, topDirName); if (DirectoryUtils.CreateDirectory(topDirectory) != 0) { Console.WriteLine("Couldn't mkdir '" + topDirectory + "'"); return false; } string[]! dirs = MakeDirList(startDir, endDir); foreach (string relPath in dirs) { string fullPath = String.Format("{0}/{1}", baseDir, relPath); if (DirectoryUtils.CreateDirectory(fullPath) != 0) { Console.WriteLine("Couldn't mkdir '" + fullPath + "'"); return false; } } FileSpec[]! files = MakeFileList(dirs); foreach (FileSpec spec in files) { if (!CreateFile(spec.classNum, spec.fileNum, spec.size, (!)spec.name, (!)spec.fullPath)) { Console.WriteLine("Couldn't make file '" + spec.fullPath + "'"); return false; } } return true; } // Returns false on failure private bool ValidateFiles(int startDir, int endDir) { string[]! dirs = MakeDirList(startDir, endDir); FileSpec[]! files = MakeFileList(dirs); foreach (FileSpec spec in files) { if (!ValidateFile(spec.classNum, spec.fileNum, spec.size, (!)spec.name, (!)spec.fullPath)) { Console.WriteLine("Validation failed for file '" + spec.fullPath + "'"); return false; } } return true; } private bool Cleanup(int startDir, int endDir) { bool success = true; string[]! dirs = MakeDirList(startDir, endDir); FileSpec[]! files = MakeFileList(dirs); foreach (FileSpec spec in files) { int error = FileUtils.DeleteFile((!)spec.fullPath); if (error != 0) { Console.WriteLine("Failed to delete {0} (error {1})", spec.fullPath, error); success = false; } } foreach (string! relPath in dirs) { string fullPath = baseDir + "/" + relPath; int error = DirectoryUtils.DeleteDirectory(fullPath); if (error != 0) { Console.WriteLine("Failed to delete {0} (error {1})", fullPath, error); success = false; } } do { string topDirectory = String.Format("{0}/{1}", baseDir, topDirName); int error = DirectoryUtils.DeleteDirectory(topDirectory); if (error != 0) { Console.WriteLine("Failed to delete {0} (error {1})", topDirectory, error); success = false; } } while (false); return success; } private static int ConnectionsToDirectories(int conns) { return ((int) (25 + (((400000.0 / 122000.0) * conns)/5.0))); } private static void Usage() { Console.WriteLine("wafgen99: A static-content fileset generator for SpecWEB99"); Console.WriteLine(""); Console.WriteLine("wafgen99 [-v] [-V] [-t] [connectionCount] targetDir"); Console.WriteLine(" -v : Verbose output"); Console.WriteLine(" -V : Validate files only (no generation)"); Console.WriteLine(" -t : Test mode; no random characters in generated files"); Console.WriteLine(" targetDir : Location to hold the fileset"); } internal static void AppMain(Parameters! config) { bool verbose = config.doVerbose; bool testmode = config.doTestMode; bool validateOnly = config.doValidateOnly; string basePath = config.directory; int connectionCount = (int)config.connections; int filesPerClass = (int)config.filesPerClass; if (testmode && validateOnly) { Console.WriteLine("Test mode is incompatible with validate-only mode"); Usage(); return; } if (basePath == null) { Console.WriteLine("No target path specified"); Usage(); return; } WafGen99 waf = new WafGen99(verbose, testmode, basePath, filesPerClass); int directoryCount = ConnectionsToDirectories(connectionCount); if (!validateOnly) { if (!waf.CreateFiles(0, directoryCount)) { Console.WriteLine("Failed to create all files."); return; } } if (!waf.ValidateFiles(0, directoryCount)) { Console.WriteLine("Validation failed."); return; } if (config.doCleanup && !waf.Cleanup(0, directoryCount)) { Console.WriteLine("Clean up failed."); return; } Console.WriteLine("Success."); } } }