1 using System;
2 using System.Collections;
3 using System.Threading;
4 using cmsx.machine;
5 using cmsx.util;
6
7 public const ulong ERROR_IO_PENDING = 997u;
8
9 namespace cmsx.kernel
10 {
11 public const ulong ddExitCompletionKey = 0xFFFFFFFFFFFFFFFFu;
12 public const ulong ddRequestCompletionKey = 0xFFFFFFFFFFFFFFFEu;
13
14 public enum IOOperation
15 {
16 read, write
17 }
18
19 public nothrow string IOOperationStr(IOOperation op)
20 {
21 if (op == IOOperation.read)
22 {
23 return "read";
24 }
25 else if (op == IOOperation.write)
26 {
27 return "write";
28 }
29 return string();
30 }
31
32 public class Overlapped
33 {
34 public nothrow Overlapped(ulong offset)
35 {
36 overlapped = OsCreateOverlapped(offset);
37 }
38 public ~Overlapped()
39 {
40 OsDestroyOverlapped(overlapped);
41 }
42 public inline nothrow void* Get() const
43 {
44 return overlapped;
45 }
46 private void* overlapped;
47 }
48
49 public class Buffer
50 {
51 public nothrow Buffer() : buffer(RtMemAlloc(blockSize))
52 {
53 RtMemZero(buffer, blockSize);
54 }
55 public ~Buffer()
56 {
57 RtMemFree(buffer);
58 }
59 public inline nothrow void* Get() const
60 {
61 return buffer;
62 }
63 private void* buffer;
64 }
65
66 public class IORequest
67 {
68 public nothrow IORequest(int number_, Block* block_, ulong offset_, cmsx.kernel.Process* process_, IOOperation op_, bool deliver_) :
69 number(number_), block(block_), overlapped(offset_), process(process_), op(op_), deliver(deliver_)
70 {
71 }
72 public string ToString()
73 {
74 string s = "number=";
75 s.Append(ToString(number));
76 s.Append(".block=").Append(block->ToString());
77 s.Append(".overlapped=").Append(ToHexString(cast<ulong>(overlapped.Get())));
78 s.Append(".process=").Append(ProcessName(process));
79 s.Append(".op=").Append(IOOperationStr(op));
80 s.Append(".count=").Append(ToString(numberOfBytesTransferred));
81 s.Append(".deliver=").Append(ToString(deliver));
82 return s;
83 }
84 public int number;
85 public Block* block;
86 public Overlapped overlapped;
87 public Buffer buffer;
88 public cmsx.kernel.Process* process;
89 public IOOperation op;
90 public ulong numberOfBytesTransferred;
91 public bool deliver;
92 }
93
94 public class IOCompletion
95 {
96 public nothrow IOCompletion(IORequest* request_) : request(request_)
97 {
98 }
99 public IORequest* request;
100 }
101
102 public abstract class DDRequest
103 {
104 public abstract void Execute(DiskDriver* dd);
105 }
106
107 public class AddFileDDRequest : DDRequest
108 {
109 public nothrow AddFileDDRequest(HostFile* hostFile_) : hostFile(hostFile_)
110 {
111 }
112 public override void Execute(DiskDriver* dd)
113 {
114 dd->AddFile(hostFile);
115 }
116 private HostFile* hostFile;
117 }
118
119 public class DiskDriver
120 {
121 static DiskDriver() : instance(new DiskDriver())
122 {
123 }
124 public static nothrow DiskDriver& Instance()
125 {
126 return *instance;
127 }
128 private DiskDriver() : exit(false), completionPort(OsCreateIoCompletionPort()), nextHostFileKey(0u), ddRequestProcessedEvent(OsCreateEvent())
129 {
130 if (completionPort == null)
131 {
132 ulong lastError = OsGetLastError();
133 Buffer messageBuf;
134 OsFormatMessage(lastError, cast<char*>(messageBuf.Get()));
135 const char* msg = cast<const char*>(messageBuf.Get());
136 string message = "error opening disk driver: creating I/O completion port failed: ";
137 message.Append(msg);
138 Panic(message);
139 }
140 }
141 public ~DiskDriver()
142 {
143 OsCloseIoCompletionPort(completionPort);
144 OsCloseEvent(ddRequestProcessedEvent);
145 }
146 public HostFile* GetOrInsertHostFile(const string& hostFilePath, bool randomAccess)
147 {
148 if (Log())
149 {
150 LogMessage("fs.dd.GetOrInsertHostFile", "begin");
151 }
152 HostFile* hostFile = null;
153 {
154 LockGuard<Mutex> lock(mtx);
155 HashMap<string, HostFile*>.ConstIterator it = pathHostFileMap.CFind(hostFilePath);
156 if (it != pathHostFileMap.CEnd())
157 {
158 return it->second;
159 }
160 else
161 {
162 hostFile = new HostFile(hostFilePath, nextHostFileKey++, randomAccess);
163 pathHostFileMap[hostFilePath] = hostFile;
164 keyHostFileMap[hostFile->Key()] = hostFile;
165 hostFiles.Add(UniquePtr<HostFile>(hostFile));
166 if (Log())
167 {
168 LogMessage("fs.dd.GetOrInsertHostFile", "add:key=" + ToString(hostFile->Key()) + ".path=" + hostFile->GetHostFilePath() + ".randomAccess=" + ToString(randomAccess));
169 }
170 }
171 }
172 if (Log())
173 {
174 LogMessage("fs.dd.GetOrInsertHostFile", "end");
175 }
176 SendAddFileRequest(hostFile);
177 return hostFile;
178 }
179 public void SendAddFileRequest(HostFile* hostFile)
180 {
181 {
182 LockGuard<Mutex> lock(mtx);
183 ddRequestQueue.Put(UniquePtr<DDRequest>(new AddFileDDRequest(hostFile)));
184 if (!OsPostQueuedCompletionStatus(completionPort, 0u, ddRequestCompletionKey))
185 {
186 ulong lastError = OsGetLastError();
187 Buffer messageBuf;
188 OsFormatMessage(lastError, cast<char*>(messageBuf.Get()));
189 const char* msg = cast<const char*>(messageBuf.Get());
190 string message = "error operating disk driver: post queued completion status failed: ";
191 message.Append(msg);
192 Panic(message);
193 }
194 }
195 OsWaitEvent(ddRequestProcessedEvent);
196 }
197 public void AddFile(HostFile* hostFile)
198 {
199 if (Log())
200 {
201 LogMessage("fs.dd.AddFile", "begin.key=" + ToString(hostFile->Key()) + ".path=" + hostFile->GetHostFilePath());
202 }
203 if (OsAssociateFileWithCompletionPort(hostFile->GetFileHandle(), completionPort, hostFile->Key()) == null)
204 {
205 ulong lastError = OsGetLastError();
206 Buffer messageBuf;
207 OsFormatMessage(lastError, cast<char*>(messageBuf.Get()));
208 const char* msg = cast<const char*>(messageBuf.Get());
209 string message = "error operating disk driver: associating file with completion port failed: ";
210 message.Append(msg);
211 throw Exception(message);
212 }
213 if (Log())
214 {
215 LogMessage("fs.dd.AddFile", "end");
216 }
217 }
218 public void RemoveHostFile(const string& hostFilePath)
219 {
220 if (Log())
221 {
222 LogMessage("fs.dd.RemoveHostFile", "begin");
223 }
224 {
225 LockGuard<Mutex> lock(mtx);
226 HashMap<string, HostFile*>.ConstIterator it = pathHostFileMap.CFind(hostFilePath);
227 if (it != pathHostFileMap.CEnd())
228 {
229 HostFile* hostFile = it->second;
230 long n = hostFiles.Count();
231 for (long i = 0; i < n; ++i;)
232 {
233 if (hostFile == hostFiles[i].Get())
234 {
235 if (Log())
236 {
237 LogMessage("fs.dd.RemoveHostFile", "remove:key=" + ToString(hostFile->Key()) + ".path=" + hostFile->GetHostFilePath());
238 }
239 keyHostFileMap.Remove(hostFile->Key());
240 pathHostFileMap.Remove(hostFile->GetHostFilePath());
241 hostFiles.Remove(hostFiles.Begin() + i);
242 if (Log())
243 {
244 LogMessage("fs.dd.RemoveHostFile", "end");
245 }
246 return;
247 }
248 }
249 }
250 else
251 {
252 throw SystemError(EFAIL, "host file '" + hostFilePath + "' not found in disk driver");
253 }
254 }
255 if (Log())
256 {
257 LogMessage("fs.dd.RemoveHostFile", "end");
258 }
259 }
260 public void Start()
261 {
262 ThreadStartMethod runMethod = Run;
263 thread = Thread.StartMethod(runMethod);
264 }
265 public void Stop()
266 {
267 Exit();
268 thread.Join();
269 }
270 public void Exit()
271 {
272 if (Log())
273 {
274 LogMessage("fs.dd", "exit");
275 }
276 exit = true;
277 if (!OsPostQueuedCompletionStatus(completionPort, 0u, ddExitCompletionKey))
278 {
279 if (Log())
280 {
281 LogMessage("fs.dd", "exit:postQueuedCompletionStatus.failed");
282 }
283 }
284 }
285 public long Read(Block* block)
286 {
287 if (Log())
288 {
289 LogMessage("fs.dd.read", "begin");
290 }
291 FileSystem* fs = GetMountTable().GetFileSystem(block->Key().fsNumber);
292 HostFile* hostFile = fs->GetHostFile(block->Key().fsNumber);
293 ulong offset = cast<ulong>(block->Key().blockNumber * blockSize);
294 void* fiberData = OsGetFiberData();
295 cmsx.kernel.Process* process = cast<cmsx.kernel.Process*>(fiberData);
296 UniquePtr<IORequest> request;
297 {
298 LockGuard<Mutex> lock(mtx);
299 request.Reset(new IORequest(nextRequestNumber++, block, offset, process, IOOperation.read, true));
300 requestMap[request->overlapped.Get()] = request.Get();
301 }
302 if (Log())
303 {
304 LogMessage("fs.dd.read", block->ToString() + ".request:" + request->ToString());
305 }
306 bool result = OsReadFile(hostFile->GetFileHandle(), request->buffer.Get(), cast<uint>(blockSize), request->overlapped.Get());
307 if (result)
308 {
309 if (Log())
310 {
311 LogMessage("fs.dd.read", "immediate=" + request->ToString());
312 }
313 MemoryWriter writer(cast<byte*>(request->buffer.Get()), blockSize);
314 block->Write(writer);
315 {
316 LockGuard<Mutex> lock(mtx);
317 requestMap.Remove(request->overlapped.Get());
318 }
319 long res = cast<long>(request->numberOfBytesTransferred);
320 if (Log())
321 {
322 LogMessage("fs.dd.read", "end.immediate");
323 }
324 return res;
325 }
326 ulong lastError = OsGetLastError();
327 if (lastError == ERROR_IO_PENDING)
328 {
329 lastError = 0u;
330 }
331 if (lastError == 0u)
332 {
333 if (Log())
334 {
335 LogMessage("fs.dd.read", "sleep");
336 }
337 SleepProcess(process, blockIOEvent, cast<ulong>(cast<void*>(block)), 0u);
338 if (Log())
339 {
340 LogMessage("fs.dd.read", "wake up");
341 }
342 MemoryReader reader(cast<byte*>(request->buffer.Get()), blockSize);
343 block->Read(reader);
344 {
345 LockGuard<Mutex> lock(mtx);
346 requestMap.Remove(request->overlapped.Get());
347 }
348 if (Log())
349 {
350 LogMessage("fs.dd.read", "end.deliver=true");
351 }
352 long res = cast<long>(request->numberOfBytesTransferred);
353 return res;
354 }
355 else
356 {
357 {
358 LockGuard<Mutex> lock(mtx);
359 requestMap.Remove(request->overlapped.Get());
360 }
361 Buffer messageBuf;
362 OsFormatMessage(lastError, cast<char*>(messageBuf.Get()));
363 const char* msg = cast<const char*>(messageBuf.Get());
364 string message = "Could not read from file '";
365 message.Append(hostFile->GetHostFilePath()).Append("': ").Append(msg);
366 if (Log())
367 {
368 LogMessage("fs.dd.read", "end.error");
369 }
370 throw SystemError(EIO, message);
371 }
372 }
373 public long Write(Block* block)
374 {
375 if (Log())
376 {
377 LogMessage("fs.dd.write", "begin");
378 }
379 FileSystem* fs = GetMountTable().GetFileSystem(block->Key().fsNumber);
380 HostFile* hostFile = fs->GetHostFile(block->Key().fsNumber);
381 ulong offset = cast<ulong>(block->Key().blockNumber * blockSize);
382 void* fiberData = OsGetFiberData();
383 cmsx.kernel.Process* process = cast<cmsx.kernel.Process*>(fiberData);
384 bool deliver = false;
385 if (Machine.GetFlag(Machine.Flags.sleepingWrite) || process == null)
386 {
387 deliver = true;
388 }
389 UniquePtr<IORequest> request;
390 {
391 LockGuard<Mutex> lock(mtx);
392 request.Reset(new IORequest(nextRequestNumber++, block, offset, process, IOOperation.write, deliver));
393 requestMap[request->overlapped.Get()] = request.Get();
394 }
395 MemoryWriter writer(cast<byte*>(request->buffer.Get()), blockSize);
396 block->Write(writer);
397 if (Log())
398 {
399 LogMessage("fs.dd.write", block->ToString() + ".request:" + request->ToString());
400 }
401 bool result = OsWriteFile(hostFile->GetFileHandle(), request->buffer.Get(), cast<uint>(blockSize), request->overlapped.Get());
402 if (result)
403 {
404 if (Log())
405 {
406 LogMessage("fs.dd.write", "immediate=" + request->ToString());
407 }
408 {
409 LockGuard<Mutex> lock(mtx);
410 requestMap.Remove(request->overlapped.Get());
411 }
412 long res = cast<long>(request->numberOfBytesTransferred);
413 if (Log())
414 {
415 LogMessage("fs.dd.write", "end.immediate");
416 }
417 return res;
418 }
419 ulong lastError = OsGetLastError();
420 if (lastError == ERROR_IO_PENDING)
421 {
422 lastError = 0u;
423 }
424 if (lastError == 0u)
425 {
426 if (request->deliver)
427 {
428 if (Log())
429 {
430 LogMessage("fs.dd.write", "sleep");
431 }
432 SleepProcess(process, blockIOEvent, cast<ulong>(cast<void*>(block)), 0u);
433 if (Log())
434 {
435 LogMessage("fs.dd.write", "wake up");
436 }
437 {
438 LockGuard<Mutex> lock(mtx);
439 requestMap.Remove(request->overlapped.Get());
440 }
441 long res = cast<long>(request->numberOfBytesTransferred);
442 if (Log())
443 {
444 LogMessage("fs.dd.write", "end.deliver=true");
445 }
446 return res;
447 }
448 else
449 {
450 if (Log())
451 {
452 LogMessage("fs.dd.write", "end.deliver=false");
453 }
454 request.Release();
455 return blockSize;
456 }
457 }
458 else
459 {
460 {
461 LockGuard<Mutex> lock(mtx);
462 requestMap.Remove(request->overlapped.Get());
463 }
464 Buffer messageBuf;
465 OsFormatMessage(lastError, cast<char*>(messageBuf.Get()));
466 const char* msg = cast<const char*>(messageBuf.Get());
467 string message = "Could not write to file '";
468 message.Append(hostFile->GetHostFilePath()).Append("': ").Append(msg);
469 if (Log())
470 {
471 LogMessage("fs.dd.write", "end.error");
472 }
473 throw SystemError(EIO, message);
474 }
475 }
476 public void Run()
477 {
478 try
479 {
480 OsConvertThreadToFiber(null);
481 while (!exit)
482 {
483 if (Log())
484 {
485 LogMessage("fs.dd.run", "begin");
486 }
487 ulong numberOfBytesTransferred = 0u;
488 ulong completionKey = 0u;
489 void* overlapped = null;
490 if (Log())
491 {
492 LogMessage("fs.dd.run", ">getQueuedCompletionStatus");
493 }
494 bool retval = OsGetQueuedCompletionStatus(completionPort, &numberOfBytesTransferred, &completionKey, &overlapped);
495 if (Log())
496 {
497 LogMessage("fs.dd.run", "<getQueuedCompletionStatus:completionKey=" + ToHexString(completionKey));
498 }
499 if (retval)
500 {
501 HostFile* hostFile = null;
502 if (completionKey == ddExitCompletionKey)
503 {
504 if (Log())
505 {
506 LogMessage("fs.dd.run", "exit.received");
507 LogMessage("fs.dd.run", "end");
508 }
509 return;
510 }
511 else if (completionKey == ddRequestCompletionKey)
512 {
513 if (Log())
514 {
515 LogMessage("fs.dd.run", "request.received");
516 }
517 ProcessRequests();
518 if (Log())
519 {
520 LogMessage("fs.dd.run", "end");
521 }
522 continue;
523 }
524 else
525 {
526 LockGuard<Mutex> lock(mtx);
527 HashMap<ulong, HostFile*>.ConstIterator it = keyHostFileMap.CFind(completionKey);
528 if (it != keyHostFileMap.CEnd())
529 {
530 hostFile = it->second;
531 }
532 }
533 if (hostFile != null)
534 {
535 if (Log())
536 {
537 LogMessage("fs.dd.run", "completion.received:hostFile=" + ToString(completionKey));
538 }
539 bool handleCompletions = false;
540 UniquePtr<IORequest> ioRequest;
541 {
542 LockGuard<Mutex> lock(mtx);
543 ioRequest.Reset(requestMap[overlapped]);
544 }
545 if (ioRequest.IsNull())
546 {
547 Panic("error from disk driver run: I/O request for overlapped " + ToHexString(cast<ulong>(overlapped)) + " not found");
548 }
549 ioRequest->numberOfBytesTransferred = numberOfBytesTransferred;
550 if (Log())
551 {
552 LogMessage("fs.dd.run.ioready", "request:" + ioRequest->ToString());
553 }
554 if (ioRequest->deliver)
555 {
556 {
557 LockGuard<Mutex> lock(mtx);
558 completionQueue.Put(IOCompletion(ioRequest.Release()));
559 }
560 if (GetKernel().Booting())
561 {
562 if (Log())
563 {
564 LogMessage("fs.dd.run", "deliver.handleCompletions");
565 }
566 handleCompletions = true;
567 }
568 else
569 {
570 if (Log())
571 {
572 LogMessage("fs.dd.run", "deliver.setInterrupt");
573 }
574 GetMachine().GetRegisters().SetInterrupt(DISK_BIT);
575 }
576 }
577 else
578 {
579 if (Log())
580 {
581 LogMessage("fs.dd.run", "deliver.remove");
582 }
583 LockGuard<Mutex> lock(mtx);
584 requestMap.Remove(overlapped);
585 }
586 if (handleCompletions)
587 {
588 InterruptService();
589 }
590 }
591 else
592 {
593 Panic("error from disk driver run: unknown completion key received");
594 }
595 }
596 else
597 {
598 Panic("error from disk driver run: get queued completion status failed");
599 }
600 if (Log())
601 {
602 LogMessage("fs.dd.run", "end");
603 }
604 }
605 }
606 catch (const Exception& ex)
607 {
608 Console.Error() << ex.Message() << endl();
609 ExceptionPtr exceptionPtr = CaptureCurrentException();
610 Machine& machine = GetMachine();
611 machine.SetException(exceptionPtr);
612 machine.Exit();
613 }
614 }
615 public void ProcessRequests()
616 {
617 if (Log())
618 {
619 LogMessage("fs.dd.ProcessRequests", "begin");
620 }
621 LockGuard<Mutex> lock(mtx);
622 while (!ddRequestQueue.IsEmpty())
623 {
624 UniquePtr<DDRequest> request = ddRequestQueue.Get();
625 request->Execute(this);
626 OsSetEvent(ddRequestProcessedEvent);
627 }
628 if (Log())
629 {
630 LogMessage("fs.dd.ProcessRequests", "end");
631 }
632 }
633 public void InterruptService()
634 {
635 if (Log())
636 {
637 LogMessage("fs.dd.interrupt", "begin");
638 }
639 Kernel& kernel = GetKernel();
640 ProcessTable& processTable = kernel.GetProcessTable();
641 {
642 LockGuard<Mutex> lock(mtx);
643 while (!completionQueue.IsEmpty())
644 {
645 IOCompletion ioCompletion = completionQueue.Get();
646 WakeUpProcess(processTable, ioCompletion.request->process);
647 }
648 }
649 if (processTable.GetRunning() == processTable.GetIdle())
650 {
651 if (Log())
652 {
653 LogMessage("fs.dd.interrupt", "schedule");
654 }
655 GetMachine().GetRegisters().SetInterrupt(CLOCK_BIT);
656 }
657 if (Log())
658 {
659 LogMessage("fs.dd.interrupt", "end");
660 }
661 }
662 private static UniquePtr<DiskDriver> instance;
663 private bool exit;
664 private void* completionPort;
665 private ulong nextHostFileKey;
666 private List<UniquePtr<HostFile>> hostFiles;
667 private HashMap<string, HostFile*> pathHostFileMap;
668 private HashMap<ulong, HostFile*> keyHostFileMap;
669 private List<void*> eventHandles;
670 private Queue<IOCompletion> completionQueue;
671 private Queue<UniquePtr<DDRequest>> ddRequestQueue;
672 private HashMap<void*, IORequest*> requestMap;
673 private Thread thread;
674 private int nextRequestNumber;
675 private Mutex mtx;
676 private void* ddRequestProcessedEvent;
677 }
678
679 public DiskDriver& GetDiskDriver()
680 {
681 return DiskDriver.Instance();
682 }
683 }