879 lines
29 KiB
C#
879 lines
29 KiB
C#
|
///////////////////////////////////////////////////////////////////////////////
|
||
|
|
//
|
||
|
|
// Microsoft Research Singularity
|
||
|
|
//
|
||
|
|
// Copyright (c) Microsoft Corporation. All rights reserved.
|
||
|
|
//
|
||
|
|
///////////////////////////////////////////////////////////////////////////////
|
||
|
|
|
||
|
|
/*
|
||
|
|
|
||
|
|
This program is a simple parallel task runner. It reads a set of tasks from input files
|
||
|
|
(or from stdin), builds a queue of tasks, starts N threads, and each thread pulls tasks
|
||
|
|
from the queue and runs them. Each worker thread is affinitized to a specific processor.
|
||
|
|
|
||
|
|
The input files are flat text files, one command per line. Blank lines are ignored.
|
||
|
|
If a line consists of "+DRAIN", then the scheduler will wait for all previous tasks
|
||
|
|
to finish before continuing.
|
||
|
|
|
||
|
|
The program is designed to be run from command-line scripts (automated builds). It does
|
||
|
|
support a GUI for displaying status, but the GUI never prompts the user for anything,
|
||
|
|
never blocks, etc. and so is at worst benign during automated builds. The GUI can be
|
||
|
|
enabled with the /gui switch.
|
||
|
|
|
||
|
|
|
||
|
|
Arlie Davis
|
||
|
|
July 2006
|
||
|
|
|
||
|
|
*/
|
||
|
|
|
||
|
|
using System;
|
||
|
|
using System.IO;
|
||
|
|
using System.Threading;
|
||
|
|
using System.Runtime.InteropServices;
|
||
|
|
using System.Text;
|
||
|
|
using System.Diagnostics;
|
||
|
|
using Windows;
|
||
|
|
|
||
|
|
#if ENABLE_PROGRESS_FORM
|
||
|
|
using System.Windows.Forms;
|
||
|
|
#endif
|
||
|
|
|
||
|
|
namespace RunParallel
|
||
|
|
{
|
||
|
|
sealed class Program
|
||
|
|
{
|
||
|
|
static uint _availableProcessorMask;
|
||
|
|
static uint _threadMask;
|
||
|
|
|
||
|
|
/// <summary>
|
||
|
|
/// The shell to use when starting child processes; this comes from the COMSPEC
|
||
|
|
/// variable in the environment.
|
||
|
|
/// </summary>
|
||
|
|
static string _shell;
|
||
|
|
|
||
|
|
static int _errorCount;
|
||
|
|
|
||
|
|
/// <summary>
|
||
|
|
/// Handle to the NT kernel "job" object, created using the Win32 CreateJobObject()
|
||
|
|
/// function. All processes that this tool starts are put into the job, so that we
|
||
|
|
/// can query the total CPU usage at the end.
|
||
|
|
/// </summary>
|
||
|
|
internal static IntPtr _JobObjectHandle;
|
||
|
|
|
||
|
|
|
||
|
|
static bool _verbose;
|
||
|
|
|
||
|
|
public static string ShellName
|
||
|
|
{
|
||
|
|
get { return _shell; }
|
||
|
|
}
|
||
|
|
|
||
|
|
public static object SyncLock
|
||
|
|
{
|
||
|
|
get { return _lock; }
|
||
|
|
}
|
||
|
|
|
||
|
|
const string DrainBarrierMarker = "+DRAIN";
|
||
|
|
|
||
|
|
#if ENABLE_PROGRESS_FORM
|
||
|
|
/// <summary>
|
||
|
|
/// If true, the GUI progress form will be displayed.
|
||
|
|
/// </summary>
|
||
|
|
static bool _enableProgressForm;
|
||
|
|
#endif
|
||
|
|
|
||
|
|
/// <summary>
|
||
|
|
/// This is the queue of work items that the main thread uses.
|
||
|
|
/// Only the main thread ever touched this queue. The main thread
|
||
|
|
/// dequeues items from this queue, and moves them to the _workerTaskQueue.
|
||
|
|
/// When it moves a job, the main thread also increments _workerActiveTaskCount.
|
||
|
|
///
|
||
|
|
/// This allows the main thread to implement synchronize/drain. When the main
|
||
|
|
/// thread needs to wait for all threads to finish processing, it just dequeues
|
||
|
|
/// items from _workerFinishedQueue until _workerActiveTaskCount reaches zero.
|
||
|
|
/// </summary>
|
||
|
|
static readonly Queue_of_Task _taskQueue = new Queue_of_Task();
|
||
|
|
|
||
|
|
/// <summary>
|
||
|
|
/// The synchronized queue of tasks to execute. Worker threads pull items
|
||
|
|
/// from this queue, process them, and then enqueue those items to _workerFinishedQueue.
|
||
|
|
/// </summary>
|
||
|
|
static internal readonly SyncQueue_of_Task _workerTaskQueue = new SyncQueue_of_Task();
|
||
|
|
|
||
|
|
/// <summary>
|
||
|
|
/// The synchronized queue of tasks that have finished. Worker threads enqueue
|
||
|
|
/// items here, and the main thread dequeues items.
|
||
|
|
/// </summary>
|
||
|
|
static internal readonly SyncQueue_of_Task _workerFinishedQueue = new SyncQueue_of_Task();
|
||
|
|
|
||
|
|
// number of items that the main thread has submitted to the worker threads
|
||
|
|
static int _workerActiveTaskCount;
|
||
|
|
|
||
|
|
internal static int _tasksCompletedCount;
|
||
|
|
static int _totalTaskCount;
|
||
|
|
|
||
|
|
static IntPtr ThisProcessHandle { get { return (IntPtr)(-1); } }
|
||
|
|
|
||
|
|
static int Main(string[] args)
|
||
|
|
{
|
||
|
|
try
|
||
|
|
{
|
||
|
|
_shell = Environment.GetEnvironmentVariable("COMSPEC");
|
||
|
|
if (Util.StringIsNullOrEmpty(_shell))
|
||
|
|
_shell = "cmd.exe";
|
||
|
|
|
||
|
|
uint systemAffinityMask;
|
||
|
|
if (!GetProcessAffinityMask(ThisProcessHandle, out _availableProcessorMask, out systemAffinityMask))
|
||
|
|
{
|
||
|
|
WriteLine("Failed to get process affinity mask?!");
|
||
|
|
_availableProcessorMask = 1;
|
||
|
|
systemAffinityMask = 1;
|
||
|
|
}
|
||
|
|
|
||
|
|
_threadMask = _availableProcessorMask;
|
||
|
|
|
||
|
|
if (Kernel32.IsCurrentProcessInJob())
|
||
|
|
{
|
||
|
|
WriteLine("This process is already in a job; job accounting is disabled.");
|
||
|
|
_JobObjectHandle = IntPtr.Zero;
|
||
|
|
}
|
||
|
|
else
|
||
|
|
{
|
||
|
|
try
|
||
|
|
{
|
||
|
|
_JobObjectHandle = Kernel32.CreateJobObject();
|
||
|
|
}
|
||
|
|
catch (Exception ex)
|
||
|
|
{
|
||
|
|
WriteLine("WARNING: Failed to create job for processes.");
|
||
|
|
ShowException(ex);
|
||
|
|
_JobObjectHandle = IntPtr.Zero;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
ParseArgs(args);
|
||
|
|
|
||
|
|
_totalTaskCount = _taskQueue.Count;
|
||
|
|
|
||
|
|
DateTime timeStarted = DateTime.Now;
|
||
|
|
|
||
|
|
RunTasks();
|
||
|
|
|
||
|
|
|
||
|
|
DateTime timeEnded = DateTime.Now;
|
||
|
|
|
||
|
|
TimeSpan elapsed = timeEnded - timeStarted;
|
||
|
|
|
||
|
|
const string times_format = " {0,-32} : {1}";
|
||
|
|
|
||
|
|
if (_JobObjectHandle != IntPtr.Zero)
|
||
|
|
{
|
||
|
|
try
|
||
|
|
{
|
||
|
|
JOBOBJECT_BASIC_ACCOUNTING_INFORMATION job_info;
|
||
|
|
job_info = Kernel32.QueryJobObjectBasicAccountingInformation(_JobObjectHandle);
|
||
|
|
|
||
|
|
TimeSpan total_job_time = job_info.TotalUserTime + job_info.TotalKernelTime;
|
||
|
|
|
||
|
|
WriteLine(
|
||
|
|
times_format,
|
||
|
|
"CPU time consumed by jobs",
|
||
|
|
total_job_time);
|
||
|
|
|
||
|
|
WriteLine(
|
||
|
|
times_format,
|
||
|
|
" in user mode",
|
||
|
|
job_info.TotalUserTime);
|
||
|
|
|
||
|
|
WriteLine(
|
||
|
|
times_format,
|
||
|
|
" in kernel mode",
|
||
|
|
job_info.TotalKernelTime);
|
||
|
|
|
||
|
|
WriteLine(
|
||
|
|
times_format,
|
||
|
|
"Elapsed time (wall time)",
|
||
|
|
elapsed);
|
||
|
|
|
||
|
|
WriteLine("");
|
||
|
|
if (elapsed.TotalMilliseconds > 0)
|
||
|
|
{
|
||
|
|
double elapsed_ms = elapsed.TotalMilliseconds;
|
||
|
|
double total_job_ms = total_job_time.TotalMilliseconds;
|
||
|
|
double throughput_ratio = total_job_ms / elapsed_ms;
|
||
|
|
double throughput_percent = Math.Round(throughput_ratio * 100.0);
|
||
|
|
WriteLine(" Parallel throughput ratio: {0}%", throughput_percent);
|
||
|
|
}
|
||
|
|
else
|
||
|
|
{
|
||
|
|
WriteLine(" Too little time elapsed to compute ratio.");
|
||
|
|
}
|
||
|
|
}
|
||
|
|
catch (Exception ex)
|
||
|
|
{
|
||
|
|
WriteLine("FAILED to query job info!");
|
||
|
|
ShowException(ex);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
else
|
||
|
|
{
|
||
|
|
WriteLine("Child process times not available.");
|
||
|
|
}
|
||
|
|
|
||
|
|
if (_errorCount != 0)
|
||
|
|
{
|
||
|
|
WriteLine("ERROR: {0} {1} completed unsuccessfully!",
|
||
|
|
_errorCount, _errorCount != 1 ? "tasks" : "task");
|
||
|
|
return 2;
|
||
|
|
}
|
||
|
|
|
||
|
|
return 0;
|
||
|
|
}
|
||
|
|
catch (Exception ex)
|
||
|
|
{
|
||
|
|
WriteLine("A top-level exception occurred.");
|
||
|
|
ShowException(ex);
|
||
|
|
return 1;
|
||
|
|
}
|
||
|
|
finally
|
||
|
|
{
|
||
|
|
if (_JobObjectHandle != IntPtr.Zero)
|
||
|
|
{
|
||
|
|
Kernel32.CloseHandle(_JobObjectHandle);
|
||
|
|
_JobObjectHandle = IntPtr.Zero;
|
||
|
|
}
|
||
|
|
|
||
|
|
StopProgressForm();
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
#region Progress Thread Support
|
||
|
|
#if ENABLE_PROGRESS_FORM
|
||
|
|
|
||
|
|
static AutoResetEvent _progressThreadReady;
|
||
|
|
|
||
|
|
static void CreateProgressThread()
|
||
|
|
{
|
||
|
|
using (AutoResetEvent ready = new AutoResetEvent(false))
|
||
|
|
{
|
||
|
|
_progressThreadReady = ready;
|
||
|
|
|
||
|
|
_progressThread = new Thread(new ThreadStart(ProgressThreadRoutine));
|
||
|
|
_progressThread.Start();
|
||
|
|
|
||
|
|
if (!ready.WaitOne(30000, false))
|
||
|
|
{
|
||
|
|
_progressThread.Abort();
|
||
|
|
_progressThread.Join();
|
||
|
|
throw new Exception("The GUI thread failed to start.");
|
||
|
|
}
|
||
|
|
|
||
|
|
_progressThreadReady = null;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
static void StopProgressForm()
|
||
|
|
{
|
||
|
|
if (_progressForm != null)
|
||
|
|
{
|
||
|
|
Debug.WriteLine("telling gui thread to quit");
|
||
|
|
MethodInvoker close = new MethodInvoker(_progressForm.PublicClose);
|
||
|
|
_progressForm.Invoke(close);
|
||
|
|
Debug.WriteLine("waiting for GUI thread to quit");
|
||
|
|
_progressForm = null;
|
||
|
|
}
|
||
|
|
|
||
|
|
if (_progressThread != null)
|
||
|
|
{
|
||
|
|
if (!_progressThread.Join(TimeSpan.FromSeconds(3)))
|
||
|
|
{
|
||
|
|
_progressThread.Abort();
|
||
|
|
_progressThread.Join();
|
||
|
|
}
|
||
|
|
_progressThread = null;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
static void ProgressThreadRoutine()
|
||
|
|
{
|
||
|
|
try
|
||
|
|
{
|
||
|
|
Thread.CurrentThread.Priority = ThreadPriority.Highest;
|
||
|
|
|
||
|
|
Debug.Assert(_progressThreadReady != null);
|
||
|
|
|
||
|
|
using (ProgressForm form = new ProgressForm())
|
||
|
|
{
|
||
|
|
_progressForm = form;
|
||
|
|
|
||
|
|
lock (_lock)
|
||
|
|
{
|
||
|
|
foreach (WorkerThread worker in _workers)
|
||
|
|
{
|
||
|
|
ListViewItem item = new ListViewItem(worker.ThreadIndex.ToString());
|
||
|
|
item.SubItems.Add("");
|
||
|
|
item.SubItems.Add("");
|
||
|
|
_progressForm.WorkerListView.Items.Add(item);
|
||
|
|
worker.ProgressListItem = item;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
_progressThreadReady.Set();
|
||
|
|
Application.Run(form);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
catch (Exception ex)
|
||
|
|
{
|
||
|
|
Debug.WriteLine("Exception on GUI thread: " + ex.Message);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
static Thread _progressThread;
|
||
|
|
#else
|
||
|
|
static void StopProgressForm()
|
||
|
|
{
|
||
|
|
}
|
||
|
|
#endif
|
||
|
|
|
||
|
|
#endregion
|
||
|
|
|
||
|
|
[DllImport("KERNEL32.DLL")]
|
||
|
|
static extern bool GetProcessAffinityMask(IntPtr process, out uint processAffinityMask, out uint systemAffinityMask);
|
||
|
|
|
||
|
|
[DllImport("KERNEL32.DLL")]
|
||
|
|
static extern uint SetThreadIdealProcessor(IntPtr thread, uint processor);
|
||
|
|
|
||
|
|
static uint ComputeProcessorAffinity(int maxThreads)
|
||
|
|
{
|
||
|
|
uint available_mask = _availableProcessorMask;
|
||
|
|
uint enabled_mask = 0;
|
||
|
|
int bit = 0;
|
||
|
|
int enabled_count = 0;
|
||
|
|
|
||
|
|
while (enabled_count < maxThreads)
|
||
|
|
{
|
||
|
|
for (; ; )
|
||
|
|
{
|
||
|
|
if (bit < 32)
|
||
|
|
{
|
||
|
|
uint mask = (1u << bit);
|
||
|
|
if ((available_mask & mask) != 0)
|
||
|
|
{
|
||
|
|
enabled_mask |= mask;
|
||
|
|
bit++;
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
else
|
||
|
|
{
|
||
|
|
bit++;
|
||
|
|
continue;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
else
|
||
|
|
{
|
||
|
|
// No more processors!
|
||
|
|
Print("Max task count specified is higher than number of processors.");
|
||
|
|
Print("Capping at {0} concurrent tasks.", enabled_count);
|
||
|
|
return enabled_mask;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
return enabled_mask;
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
static char[] _commandSeparators = { ':', '=' };
|
||
|
|
|
||
|
|
const string Prefix = "RUNPARALLEL: ";
|
||
|
|
|
||
|
|
static void Print(string text)
|
||
|
|
{
|
||
|
|
WriteLine(Prefix + text);
|
||
|
|
}
|
||
|
|
|
||
|
|
static void Print(string format, params object[] args)
|
||
|
|
{
|
||
|
|
Print(String.Format(format, args));
|
||
|
|
}
|
||
|
|
|
||
|
|
/// <summary>
|
||
|
|
/// Actually, this does more than just parse the args.
|
||
|
|
/// It also loads the tasks.
|
||
|
|
/// </summary>
|
||
|
|
/// <param name="args"></param>
|
||
|
|
static void ParseArgs(string[] args)
|
||
|
|
{
|
||
|
|
bool foundInputFiles = false;
|
||
|
|
|
||
|
|
for (int i = 0; i < args.Length; i++)
|
||
|
|
{
|
||
|
|
string arg = args[i];
|
||
|
|
|
||
|
|
if (arg.StartsWith("/") || arg.StartsWith("-"))
|
||
|
|
{
|
||
|
|
int index = arg.IndexOfAny(_commandSeparators, 1);
|
||
|
|
string name;
|
||
|
|
string value;
|
||
|
|
if (index >= 0)
|
||
|
|
{
|
||
|
|
name = arg.Substring(1, index - 1);
|
||
|
|
value = arg.Substring(index + 1);
|
||
|
|
}
|
||
|
|
else
|
||
|
|
{
|
||
|
|
name = arg.Substring(1);
|
||
|
|
value = "";
|
||
|
|
}
|
||
|
|
|
||
|
|
name = name.ToLower();
|
||
|
|
switch (name)
|
||
|
|
{
|
||
|
|
case "help":
|
||
|
|
case "?":
|
||
|
|
Usage();
|
||
|
|
break;
|
||
|
|
|
||
|
|
case "stdin":
|
||
|
|
ReadTasksText(Console.In);
|
||
|
|
foundInputFiles = true;
|
||
|
|
break;
|
||
|
|
|
||
|
|
case "max":
|
||
|
|
int maxThreads;
|
||
|
|
try
|
||
|
|
{
|
||
|
|
maxThreads = Int32.Parse(name);
|
||
|
|
}
|
||
|
|
catch
|
||
|
|
{
|
||
|
|
WriteLine("Invalid value for -max");
|
||
|
|
Usage();
|
||
|
|
maxThreads = 0; // satisfy compiler
|
||
|
|
}
|
||
|
|
_threadMask = ComputeProcessorAffinity(maxThreads);
|
||
|
|
break;
|
||
|
|
|
||
|
|
case "v":
|
||
|
|
case "verbose":
|
||
|
|
_verbose = true;
|
||
|
|
break;
|
||
|
|
|
||
|
|
case "gui":
|
||
|
|
#if ENABLE_PROGRESS_FORM
|
||
|
|
_enableProgressForm = true;
|
||
|
|
#endif
|
||
|
|
break;
|
||
|
|
|
||
|
|
case "tasks":
|
||
|
|
using (StreamReader reader = File.OpenText(arg))
|
||
|
|
{
|
||
|
|
ReadTasksText(reader);
|
||
|
|
foundInputFiles = true;
|
||
|
|
}
|
||
|
|
break;
|
||
|
|
|
||
|
|
default:
|
||
|
|
WriteLine("Unrecognized switch: " + name);
|
||
|
|
Usage();
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
else
|
||
|
|
{
|
||
|
|
#if false
|
||
|
|
// It appears to be an inline repeat, sort of like xargs.
|
||
|
|
int start = i;
|
||
|
|
i++;
|
||
|
|
while (i < args.Length && args[i] != "--")
|
||
|
|
i++;
|
||
|
|
if (i == args.Length)
|
||
|
|
{
|
||
|
|
WriteLine("No tasks were specified. Please use -- to separate the fixed portion");
|
||
|
|
WriteLine("of the command line from the list of tasks.");
|
||
|
|
ShortUsage();
|
||
|
|
}
|
||
|
|
|
||
|
|
string command_base = ArgsToString(args, start, i - start);
|
||
|
|
const string insertion_marker = "$=";
|
||
|
|
bool has_insertion_marker = command_base.IndexOf(insertion_marker) != -1;
|
||
|
|
|
||
|
|
// For each remaining argument, build a task.
|
||
|
|
i++;
|
||
|
|
bool drain_required = false;
|
||
|
|
for (int j = i; j < args.Length; j++)
|
||
|
|
{
|
||
|
|
string taskarg = args[j];
|
||
|
|
|
||
|
|
if (Util.StringCompareCaseInsensitive(taskarg, DrainBarrierMarker) == 0)
|
||
|
|
{
|
||
|
|
// A drain marker indicates that all tasks in the current sequence must
|
||
|
|
// finish before any tasks after the marker begin.
|
||
|
|
drain_required = true;
|
||
|
|
continue;
|
||
|
|
}
|
||
|
|
|
||
|
|
string command = command_base;
|
||
|
|
|
||
|
|
if (has_insertion_marker)
|
||
|
|
command = command_base.Replace(insertion_marker, taskarg);
|
||
|
|
else
|
||
|
|
command = command_base + " " + taskarg;
|
||
|
|
|
||
|
|
Task task = new Task();
|
||
|
|
task.Name = taskarg;
|
||
|
|
task.CommandLine = command;
|
||
|
|
task.DrainRequired = drain_required;
|
||
|
|
_taskQueue.Enqueue(task);
|
||
|
|
|
||
|
|
if (drain_required)
|
||
|
|
{
|
||
|
|
// WriteLine("{0}: task will cause drain", task.CommandLine);
|
||
|
|
}
|
||
|
|
|
||
|
|
drain_required = false;
|
||
|
|
}
|
||
|
|
foundInputFiles = true;
|
||
|
|
break;
|
||
|
|
#else
|
||
|
|
ReadTasksText(arg);
|
||
|
|
foundInputFiles = true;
|
||
|
|
#endif
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
if (!foundInputFiles)
|
||
|
|
Usage();
|
||
|
|
}
|
||
|
|
|
||
|
|
static string ArgsToString(string[] args, int offset, int count)
|
||
|
|
{
|
||
|
|
StringBuilder buffer = new StringBuilder();
|
||
|
|
|
||
|
|
for (int i = 0; i < count; i++)
|
||
|
|
{
|
||
|
|
if (i > 0)
|
||
|
|
buffer.Append(" ");
|
||
|
|
|
||
|
|
string arg = args[offset + i];
|
||
|
|
|
||
|
|
bool quotes = false;
|
||
|
|
if (arg.IndexOf(" ") != -1)
|
||
|
|
quotes = true;
|
||
|
|
|
||
|
|
if (quotes)
|
||
|
|
buffer.Append("\"");
|
||
|
|
|
||
|
|
buffer.Append(arg);
|
||
|
|
|
||
|
|
if (quotes)
|
||
|
|
buffer.Append("\"");
|
||
|
|
}
|
||
|
|
|
||
|
|
return buffer.ToString();
|
||
|
|
}
|
||
|
|
|
||
|
|
static void ShortUsage()
|
||
|
|
{
|
||
|
|
WriteLine("Use -? or -help for a list of full arguments.");
|
||
|
|
Environment.Exit(1);
|
||
|
|
}
|
||
|
|
|
||
|
|
static void Usage()
|
||
|
|
{
|
||
|
|
WriteLine(
|
||
|
|
@"
|
||
|
|
RUNPARALLEL: A simple parallel task runner.
|
||
|
|
|
||
|
|
Usage: RUNPARALLEL [options] /tasks:foo.txt /tasks:bar.txt [/stdin]
|
||
|
|
|
||
|
|
Or:
|
||
|
|
RUNPARALLEL [options] foo.exe arg1 ... $= ... -- task1 task2 task3
|
||
|
|
|
||
|
|
where [options] can be:
|
||
|
|
|
||
|
|
/max:nn Maximum of nn active tasks
|
||
|
|
"
|
||
|
|
#if ENABLE_PROGRESS_FORM
|
||
|
|
+ @" /gui Show the GUI progress form (non-blocking)\r\n"
|
||
|
|
#endif
|
||
|
|
+ @" /stdin Read tasks from stdin
|
||
|
|
/v Verbose info about job scheduling
|
||
|
|
|
||
|
|
Each input file contains commands to run in parallel. Each command is on a
|
||
|
|
separate line. The +DRAIN marker will cause all previous commands to complete
|
||
|
|
before continuing. You can use /stdin to read tasks from standard input.
|
||
|
|
|
||
|
|
If you use the second form, then no input file is necessary. The first arguments
|
||
|
|
form the command-line for the program to run. The ""--"" marker separates the
|
||
|
|
command-line to run from the tasks to run. If the command-line to run has a
|
||
|
|
'$=' marker in it, then that marker will be replaced with each of the task values.
|
||
|
|
If there is no marker, then the task value will be appended to the end of the
|
||
|
|
command-line. For example:
|
||
|
|
|
||
|
|
runparallel cd $= ^&^& nmake /nologo -- app1 app2 app3
|
||
|
|
|
||
|
|
will result in these commands being run in parallel:
|
||
|
|
|
||
|
|
cd app1 && nmake /nologo
|
||
|
|
cd app2 && nmake /nologo
|
||
|
|
cd app3 && nmake /nologo
|
||
|
|
|
||
|
|
The special task-name +DRAIN will cause the scheduler to wait for all previous
|
||
|
|
tasks to complete before scheduling more tasks.
|
||
|
|
|
||
|
|
Contact Arlie Davis (arlied) with questions.
|
||
|
|
");
|
||
|
|
Environment.Exit(1);
|
||
|
|
|
||
|
|
}
|
||
|
|
|
||
|
|
/// <summary>
|
||
|
|
/// This method reads a text file and creates work items. Each line in the file
|
||
|
|
/// contains a work item. There is no comment character. Blank lines are ignored.
|
||
|
|
/// </summary>
|
||
|
|
/// <param name="reader"></param>
|
||
|
|
static void ReadTasksText(TextReader reader)
|
||
|
|
{
|
||
|
|
bool drain_required = false;
|
||
|
|
|
||
|
|
for (; ; )
|
||
|
|
{
|
||
|
|
string line = reader.ReadLine();
|
||
|
|
if (line == null)
|
||
|
|
break;
|
||
|
|
|
||
|
|
if (Util.IsBlank(line))
|
||
|
|
continue;
|
||
|
|
|
||
|
|
line = line.Trim();
|
||
|
|
if (Util.StringCompareCaseInsensitive(line, DrainBarrierMarker) == 0)
|
||
|
|
{
|
||
|
|
drain_required = true;
|
||
|
|
continue;
|
||
|
|
}
|
||
|
|
|
||
|
|
Task task = new Task();
|
||
|
|
|
||
|
|
task.Name = line;
|
||
|
|
task.CommandLine = line;
|
||
|
|
task.DrainRequired = drain_required;
|
||
|
|
|
||
|
|
_taskQueue.Enqueue(task);
|
||
|
|
|
||
|
|
drain_required = false;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
static void ReadTasksText(string filename)
|
||
|
|
{
|
||
|
|
using (StreamReader reader = File.OpenText(filename))
|
||
|
|
{
|
||
|
|
ReadTasksText(reader);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
static int GetSetBitCount(uint value)
|
||
|
|
{
|
||
|
|
uint result = ((value & 0xaaaaaaaau) >> 1) + (value & 0x55555555u);
|
||
|
|
result = ((result & 0xccccccccu) >> 2) + (result & 0x33333333u);
|
||
|
|
result = ((result & 0xf0f0f0f0u) >> 4) + (result & 0x0f0f0f0fu);
|
||
|
|
result = ((result & 0xff00ff00u) >> 8) + (result & 0x00ff00ffu);
|
||
|
|
result = ((result & 0xffff0000u) >> 16) + (result & 0x0000ffffu);
|
||
|
|
return checked((int)result);
|
||
|
|
}
|
||
|
|
|
||
|
|
static void RunTasks()
|
||
|
|
{
|
||
|
|
int threadcount = GetSetBitCount(_threadMask);
|
||
|
|
|
||
|
|
WriteLine("Running {0} tasks on {1} thread(s) in directory: {2}",
|
||
|
|
_taskQueue.Count,
|
||
|
|
threadcount,
|
||
|
|
Environment.CurrentDirectory);
|
||
|
|
|
||
|
|
List_of_WorkerThread slotlist = new List_of_WorkerThread();
|
||
|
|
|
||
|
|
uint runMask = _threadMask;
|
||
|
|
|
||
|
|
for (int i = 0; i < 32; i++)
|
||
|
|
{
|
||
|
|
uint mask = (1u << i);
|
||
|
|
if ((runMask & mask) != 0)
|
||
|
|
{
|
||
|
|
WorkerThread worker = new WorkerThread();
|
||
|
|
worker.AffinityMask = mask;
|
||
|
|
worker.Thread = new Thread(new ThreadStart(worker.ThreadRoutine));
|
||
|
|
worker.ThreadIndex = i;
|
||
|
|
slotlist.Add(worker);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
#if CLR2
|
||
|
|
_workers = slotlist.ToArray();
|
||
|
|
#else
|
||
|
|
_workers = (WorkerThread[])slotlist.ToArray(typeof(WorkerThread));
|
||
|
|
#endif
|
||
|
|
|
||
|
|
#if ENABLE_PROGRESS_FORM
|
||
|
|
if (_enableProgressForm)
|
||
|
|
{
|
||
|
|
CreateProgressThread();
|
||
|
|
}
|
||
|
|
#endif
|
||
|
|
|
||
|
|
foreach (WorkerThread worker in _workers)
|
||
|
|
{
|
||
|
|
worker.Thread.Start();
|
||
|
|
}
|
||
|
|
|
||
|
|
// WriteLine("Dispatching work items...");
|
||
|
|
|
||
|
|
_workerActiveTaskCount = 0;
|
||
|
|
|
||
|
|
for (; ; )
|
||
|
|
{
|
||
|
|
if (_taskQueue.Count != 0)
|
||
|
|
{
|
||
|
|
Task task = (Task)_taskQueue.Dequeue();
|
||
|
|
|
||
|
|
if (task.DrainRequired)
|
||
|
|
{
|
||
|
|
if (_workerActiveTaskCount != 0)
|
||
|
|
WriteLine("Performing synchronization drain.");
|
||
|
|
WaitActiveTasks();
|
||
|
|
}
|
||
|
|
|
||
|
|
_workerTaskQueue.Enqueue(task);
|
||
|
|
_workerActiveTaskCount++;
|
||
|
|
}
|
||
|
|
else
|
||
|
|
{
|
||
|
|
// WriteLine("Main thread has reached end of task queue. Closing worker queue.");
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
_workerTaskQueue.Close();
|
||
|
|
WaitActiveTasks();
|
||
|
|
|
||
|
|
foreach (WorkerThread worker in _workers)
|
||
|
|
{
|
||
|
|
if (worker.Thread != null)
|
||
|
|
{
|
||
|
|
worker.Thread.Join();
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
WriteLine("Done.");
|
||
|
|
}
|
||
|
|
|
||
|
|
static void WaitActiveTasks()
|
||
|
|
{
|
||
|
|
Debug.Assert(_workerActiveTaskCount >= 0);
|
||
|
|
|
||
|
|
if (_verbose)
|
||
|
|
WriteLine("waiting for {0} active task(s) to complete.", _workerActiveTaskCount);
|
||
|
|
|
||
|
|
if (_workerActiveTaskCount == 0)
|
||
|
|
{
|
||
|
|
// WriteLine("no need to wait; no active tasks");
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
while (_workerActiveTaskCount > 0)
|
||
|
|
{
|
||
|
|
Task task;
|
||
|
|
if (_workerFinishedQueue.Dequeue(out task))
|
||
|
|
{
|
||
|
|
_workerActiveTaskCount--;
|
||
|
|
if (!task.Succeeded)
|
||
|
|
{
|
||
|
|
_errorCount++;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
else
|
||
|
|
{
|
||
|
|
Debug.Fail("_workerFinishedQueue.Dequeue did not return an item!!");
|
||
|
|
WriteLine("Internal error! _workerFinishedQueue did not return an item!");
|
||
|
|
throw new Exception("Internal error! _workerFinishedQueue did not return an item!");
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
static WorkerThread[] _workers;
|
||
|
|
|
||
|
|
static readonly object _lock = new Object();
|
||
|
|
|
||
|
|
#if ENABLE_PROGRESS_FORM
|
||
|
|
static ProgressForm _progressForm;
|
||
|
|
|
||
|
|
static void UpdateProgressAnyThread()
|
||
|
|
{
|
||
|
|
if (_progressForm == null)
|
||
|
|
return;
|
||
|
|
|
||
|
|
_progressForm.Invoke(new MethodInvoker(UpdateProgressGuiThread), null);
|
||
|
|
}
|
||
|
|
|
||
|
|
internal static void UpdateProgressGuiThread()
|
||
|
|
{
|
||
|
|
// This runs on the GUI thread.
|
||
|
|
|
||
|
|
lock (_lock)
|
||
|
|
{
|
||
|
|
if (_totalTaskCount != 0)
|
||
|
|
{
|
||
|
|
int percent = (100 * _tasksCompletedCount) / _totalTaskCount;
|
||
|
|
_progressForm.OverallProgressBar.Value = percent;
|
||
|
|
}
|
||
|
|
|
||
|
|
_progressForm._TotalTasksLabel.Text = _totalTaskCount.ToString();
|
||
|
|
int tasksRemaining = _taskQueue.Count;
|
||
|
|
_progressForm._TasksRemainingLabel.Text = tasksRemaining.ToString();
|
||
|
|
|
||
|
|
|
||
|
|
foreach (WorkerThread slot in _workers)
|
||
|
|
{
|
||
|
|
ListViewItem item = slot.ProgressListItem;
|
||
|
|
|
||
|
|
if (slot.CurrentTask != null)
|
||
|
|
{
|
||
|
|
item.SubItems[1].Text = slot.CurrentTask.Name;
|
||
|
|
item.SubItems[2].Text = slot.CurrentTask.CommandLine;
|
||
|
|
}
|
||
|
|
else
|
||
|
|
{
|
||
|
|
item.SubItems[1].Text = "(none)";
|
||
|
|
item.SubItems[2].Text = "(none)";
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
#endif
|
||
|
|
|
||
|
|
static public void ShowException(Exception chain)
|
||
|
|
{
|
||
|
|
for (Exception current = chain; current != null; current = current.InnerException)
|
||
|
|
{
|
||
|
|
WriteLine("{0}: {1}", current.GetType().FullName, current.Message);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
static void WriteLine(string line)
|
||
|
|
{
|
||
|
|
Console.WriteLine("main> " + line);
|
||
|
|
}
|
||
|
|
|
||
|
|
static void WriteLine(string format, params object[] args)
|
||
|
|
{
|
||
|
|
WriteLine(String.Format(format, args));
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
}
|