VirtualBox

Ticket #5501: timer-r0drv-linux.c

File timer-r0drv-linux.c, 29.9 KB (added by Frank Mehnert, 15 years ago)

testcase (hrtimers enabled)

Line 
1/* $Id: timer-r0drv-linux.c 56075 2009-12-16 13:44:58Z fmehnert $ */
2/** @file
3 * IPRT - Timers, Ring-0 Driver, Linux.
4 */
5
6/*
7 * Copyright (C) 2006-2008 Sun Microsystems, Inc.
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 *
17 * The contents of this file may alternatively be used under the terms
18 * of the Common Development and Distribution License Version 1.0
19 * (CDDL) only, as it comes in the "COPYING.CDDL" file of the
20 * VirtualBox OSE distribution, in which case the provisions of the
21 * CDDL are applicable instead of those of the GPL.
22 *
23 * You may elect to license modified versions of this file under the
24 * terms and conditions of either the GPL or the CDDL or both.
25 *
26 * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa
27 * Clara, CA 95054 USA or visit http://www.sun.com if you need
28 * additional information or have any questions.
29 */
30
31
32/*******************************************************************************
33* Header Files *
34*******************************************************************************/
35#include "the-linux-kernel.h"
36#include "internal/iprt.h"
37
38#include <iprt/timer.h>
39#include <iprt/time.h>
40#include <iprt/mp.h>
41#include <iprt/cpuset.h>
42#include <iprt/spinlock.h>
43#include <iprt/err.h>
44#include <iprt/asm.h>
45#include <iprt/assert.h>
46#include <iprt/alloc.h>
47
48#include "internal/magics.h"
49
50/* We use the API of Linux 2.6.28+ (hrtimer_add_expires_ns()) */
51#if !defined(RT_USE_LINUX_HRTIMER) \
52 && LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 28)
53# define RT_USE_LINUX_HRTIMER
54#endif
55
56/* This check must match the ktime usage in rtTimeGetSystemNanoTS() / time-r0drv-linux.c. */
57#if defined(RT_USE_LINUX_HRTIMER) \
58 && LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 28)
59# error "RT_USE_LINUX_HRTIMER requires 2.6.28 or later, sorry."
60#endif
61
62
63/*******************************************************************************
64* Structures and Typedefs *
65*******************************************************************************/
66/**
67 * Timer state machine.
68 *
69 * This is used to try handle the issues with MP events and
70 * timers that runs on all CPUs. It's relatively nasty :-/
71 */
72typedef enum RTTIMERLNXSTATE
73{
74 /** Stopped. */
75 RTTIMERLNXSTATE_STOPPED = 0,
76 /** Transient state; next ACTIVE. */
77 RTTIMERLNXSTATE_STARTING,
78 /** Transient state; next ACTIVE. (not really necessary) */
79 RTTIMERLNXSTATE_MP_STARTING,
80 /** Active. */
81 RTTIMERLNXSTATE_ACTIVE,
82 /** Transient state; next STOPPED. */
83 RTTIMERLNXSTATE_STOPPING,
84 /** Transient state; next STOPPED. */
85 RTTIMERLNXSTATE_MP_STOPPING,
86 /** The usual 32-bit hack. */
87 RTTIMERLNXSTATE_32BIT_HACK = 0x7fffffff
88} RTTIMERLNXSTATE;
89
90
91/**
92 * A Linux sub-timer.
93 */
94typedef struct RTTIMERLNXSUBTIMER
95{
96 /** The linux timer structure. */
97#ifdef RT_USE_LINUX_HRTIMER
98 struct hrtimer LnxTimer;
99#else
100 struct timer_list LnxTimer;
101 /** The start of the current run (ns).
102 * This is used to calculate when the timer ought to fire the next time. */
103 uint64_t u64StartTS;
104 /** The start of the current run (ns).
105 * This is used to calculate when the timer ought to fire the next time. */
106 uint64_t u64NextTS;
107#endif
108 /** The current tick number (since u64StartTS). */
109 uint64_t iTick;
110 /** Pointer to the parent timer. */
111 PRTTIMER pParent;
112#ifndef RT_USE_LINUX_HRTIMER
113 /** The u64NextTS in jiffies. */
114 unsigned long ulNextJiffies;
115#endif
116 /** The current sub-timer state. */
117 RTTIMERLNXSTATE volatile enmState;
118} RTTIMERLNXSUBTIMER;
119/** Pointer to a linux sub-timer. */
120typedef RTTIMERLNXSUBTIMER *PRTTIMERLNXSUBTIMER;
121AssertCompileMemberOffset(RTTIMERLNXSUBTIMER, LnxTimer, 0);
122
123
124/**
125 * The internal representation of an Linux timer handle.
126 */
127typedef struct RTTIMER
128{
129 /** Magic.
130 * This is RTTIMER_MAGIC, but changes to something else before the timer
131 * is destroyed to indicate clearly that thread should exit. */
132 uint32_t volatile u32Magic;
133 /** Spinlock synchronizing the fSuspended and MP event handling.
134 * This is NIL_RTSPINLOCK if cCpus == 1. */
135 RTSPINLOCK hSpinlock;
136 /** Flag indicating that the timer is suspended. */
137 bool volatile fSuspended;
138 /** Whether the timer must run on one specific CPU or not. */
139 bool fSpecificCpu;
140#ifdef CONFIG_SMP
141 /** Whether the timer must run on all CPUs or not. */
142 bool fAllCpus;
143#endif /* else: All -> specific on non-SMP kernels */
144 /** The CPU it must run on if fSpecificCpu is set. */
145 RTCPUID idCpu;
146 /** The number of CPUs this timer should run on. */
147 RTCPUID cCpus;
148 /** Callback. */
149 PFNRTTIMER pfnTimer;
150 /** User argument. */
151 void *pvUser;
152 /** The timer interval. 0 if one-shot. */
153 uint64_t u64NanoInterval;
154#ifndef RT_USE_LINUX_HRTIMER
155 /** This is set to the number of jiffies between ticks if the interval is
156 * an exact number of jiffies. */
157 unsigned long cJiffies;
158#endif
159 /** Sub-timers.
160 * Normally there is just one, but for RTTIMER_FLAGS_CPU_ALL this will contain
161 * an entry for all possible cpus. In that case the index will be the same as
162 * for the RTCpuSet. */
163 RTTIMERLNXSUBTIMER aSubTimers[1];
164} RTTIMER;
165
166
167/**
168 * A rtTimerLinuxStartOnCpu and rtTimerLinuxStartOnCpu argument package.
169 */
170typedef struct RTTIMERLINUXSTARTONCPUARGS
171{
172 /** The current time (RTTimeNanoTS). */
173 uint64_t u64Now;
174 /** When to start firing (delta). */
175 uint64_t u64First;
176} RTTIMERLINUXSTARTONCPUARGS;
177/** Pointer to a rtTimerLinuxStartOnCpu argument package. */
178typedef RTTIMERLINUXSTARTONCPUARGS *PRTTIMERLINUXSTARTONCPUARGS;
179
180
181/**
182 * Sets the state.
183 */
184DECLINLINE(void) rtTimerLnxSetState(RTTIMERLNXSTATE volatile *penmState, RTTIMERLNXSTATE enmNewState)
185{
186 ASMAtomicWriteU32((uint32_t volatile *)penmState, enmNewState);
187}
188
189
190/**
191 * Sets the state if it has a certain value.
192 *
193 * @return true if xchg was done.
194 * @return false if xchg wasn't done.
195 */
196DECLINLINE(bool) rtTimerLnxCmpXchgState(RTTIMERLNXSTATE volatile *penmState, RTTIMERLNXSTATE enmNewState, RTTIMERLNXSTATE enmCurState)
197{
198 return ASMAtomicCmpXchgU32((uint32_t volatile *)penmState, enmNewState, enmCurState);
199}
200
201
202/**
203 * Gets the state.
204 */
205DECLINLINE(RTTIMERLNXSTATE) rtTimerLnxGetState(RTTIMERLNXSTATE volatile *penmState)
206{
207 return (RTTIMERLNXSTATE)ASMAtomicUoReadU32((uint32_t volatile *)penmState);
208}
209
210
211#ifdef RT_USE_LINUX_HRTIMER
212/**
213 * Converts a nano second time stamp to ktime_t.
214 *
215 * ASSUMES RTTimeNanoTS() is implemented using ktime_get_ts().
216 *
217 * @returns ktime_t.
218 * @param cNanoSecs Nanoseconds.
219 */
220DECLINLINE(ktime_t) rtTimerLnxNanoToKt(uint64_t cNanoSecs)
221{
222 /* With some luck the compiler optimizes the division out of this... (Bet it doesn't.) */
223 return ktime_set(cNanoSecs / 1000000000, cNanoSecs % 1000000000);
224}
225
226/**
227 * Converts ktime_t to a nano second time stamp.
228 *
229 * ASSUMES RTTimeNanoTS() is implemented using ktime_get_ts().
230 *
231 * @returns nano second time stamp.
232 * @param Kt ktime_t.
233 */
234DECLINLINE(uint64_t) rtTimerLnxKtToNano(ktime_t Kt)
235{
236 return ktime_to_ns(Kt);
237}
238
239#else /* ! RT_USE_LINUX_HRTIMER */
240
241/**
242 * Converts a nano second interval to jiffies.
243 *
244 * @returns Jiffies.
245 * @param cNanoSecs Nanoseconds.
246 */
247DECLINLINE(unsigned long) rtTimerLnxNanoToJiffies(uint64_t cNanoSecs)
248{
249 /* this can be made even better... */
250 if (cNanoSecs > (uint64_t)TICK_NSEC * MAX_JIFFY_OFFSET)
251 return MAX_JIFFY_OFFSET;
252# if ARCH_BITS == 32
253 if (RT_LIKELY(cNanoSecs <= UINT32_MAX))
254 return ((uint32_t)cNanoSecs + (TICK_NSEC-1)) / TICK_NSEC;
255# endif
256 return (cNanoSecs + (TICK_NSEC-1)) / TICK_NSEC;
257}
258#endif /* ! RT_USE_LINUX_HRTIMER */
259
260
261/**
262 * Starts a sub-timer (RTTimerStart).
263 *
264 * @param pSubTimer The sub-timer to start.
265 * @param u64Now The current timestamp (RTTimeNanoTS()).
266 * @param u64First The interval from u64Now to the first time the timer should fire.
267 */
268static void rtTimerLnxStartSubTimer(PRTTIMERLNXSUBTIMER pSubTimer, uint64_t u64Now, uint64_t u64First)
269{
270 /*
271 * Calc when it should start firing.
272 */
273 uint64_t u64NextTS = u64Now + u64First;
274#ifndef RT_USE_LINUX_HRTIMER
275 pSubTimer->u64StartTS = u64NextTS;
276 pSubTimer->u64NextTS = u64NextTS;
277#endif
278
279 pSubTimer->iTick = 0;
280
281#ifdef RT_USE_LINUX_HRTIMER
282 hrtimer_start(&pSubTimer->LnxTimer, rtTimerLnxNanoToKt(u64NextTS), HRTIMER_MODE_ABS);
283#else
284 {
285 unsigned long cJiffies = !u64First ? 0 : rtTimerLnxNanoToJiffies(u64First);
286 pSubTimer->ulNextJiffies = jiffies + cJiffies;
287 mod_timer(&pSubTimer->LnxTimer, pSubTimer->ulNextJiffies);
288 }
289#endif
290
291 rtTimerLnxSetState(&pSubTimer->enmState, RTTIMERLNXSTATE_ACTIVE);
292}
293
294
295/**
296 * Stops a sub-timer (RTTimerStart and rtTimerLinuxMpEvent()).
297 *
298 * @param pSubTimer The sub-timer.
299 */
300static void rtTimerLnxStopSubTimer(PRTTIMERLNXSUBTIMER pSubTimer)
301{
302#ifdef RT_USE_LINUX_HRTIMER
303 hrtimer_cancel(&pSubTimer->LnxTimer);
304#else
305 if (timer_pending(&pSubTimer->LnxTimer))
306 del_timer_sync(&pSubTimer->LnxTimer);
307#endif
308
309 rtTimerLnxSetState(&pSubTimer->enmState, RTTIMERLNXSTATE_STOPPED);
310}
311
312
313#ifdef RT_USE_LINUX_HRTIMER
314/**
315 * Timer callback function.
316 * @returns HRTIMER_NORESTART or HRTIMER_RESTART depending on whether it's a one-shot or interval timer.
317 * @param pHrTimer Pointer to the sub-timer structure.
318 */
319static enum hrtimer_restart rtTimerLinuxCallback(struct hrtimer *pHrTimer)
320#else
321/**
322 * Timer callback function.
323 * @param ulUser Address of the sub-timer structure.
324 */
325static void rtTimerLinuxCallback(unsigned long ulUser)
326#endif
327{
328#ifdef RT_USE_LINUX_HRTIMER
329 enum hrtimer_restart rc;
330 PRTTIMERLNXSUBTIMER pSubTimer = (PRTTIMERLNXSUBTIMER)pHrTimer;
331#else
332 PRTTIMERLNXSUBTIMER pSubTimer = (PRTTIMERLNXSUBTIMER)ulUser;
333#endif
334 PRTTIMER pTimer = pSubTimer->pParent;
335
336 /*
337 * Don't call the handler if the timer has been suspended.
338 * Also, when running on all CPUS, make sure we don't call out twice
339 * on a CPU because of timer migration.
340 *
341 * For the specific cpu case, we're just ignoring timer migration for now... (bad)
342 */
343 if ( ASMAtomicUoReadBool(&pTimer->fSuspended)
344#ifdef CONFIG_SMP
345 || ( pTimer->fAllCpus
346 && (RTCPUID)(pSubTimer - &pTimer->aSubTimers[0]) != RTMpCpuId())
347#endif
348 )
349 {
350 rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_STOPPED, RTTIMERLNXSTATE_ACTIVE);
351# ifdef RT_USE_LINUX_HRTIMER
352 rc = HRTIMER_NORESTART;
353# endif
354 }
355 else if (!pTimer->u64NanoInterval)
356 {
357 /*
358 * One shot timer, stop it before dispatching it.
359 */
360 if (pTimer->cCpus == 1)
361 ASMAtomicWriteBool(&pTimer->fSuspended, true);
362 rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_STOPPED, RTTIMERLNXSTATE_ACTIVE);
363#ifdef RT_USE_LINUX_HRTIMER
364 rc = HRTIMER_NORESTART;
365#else
366 /* detached before we're called, nothing to do for this case. */
367#endif
368
369 pTimer->pfnTimer(pTimer, pTimer->pvUser, ++pSubTimer->iTick);
370 }
371 else
372 {
373 const uint64_t iTick = ++pSubTimer->iTick;
374
375#ifdef RT_USE_LINUX_HRTIMER
376 hrtimer_add_expires_ns(&pSubTimer->LnxTimer, pTimer->u64NanoInterval);
377 rc = HRTIMER_RESTART;
378#else
379 const uint64_t u64NanoTS = RTTimeNanoTS();
380
381 /*
382 * Interval timer, calculate the next timeout and re-arm it.
383 *
384 * The first time around, we'll re-adjust the u64StartTS to
385 * try prevent some jittering if we were started at a bad time.
386 * This may of course backfire with highres timers...
387 */
388 if (RT_UNLIKELY(iTick == 1))
389 {
390 pSubTimer->u64StartTS = pSubTimer->u64NextTS = u64NanoTS;
391 pSubTimer->ulNextJiffies = jiffies;
392 }
393
394 pSubTimer->u64NextTS += pTimer->u64NanoInterval;
395 if (pTimer->cJiffies)
396 {
397 pSubTimer->ulNextJiffies += pTimer->cJiffies;
398 /* Prevent overflows when the jiffies counter wraps around.
399 * Special thanks to Ken Preslan for helping debugging! */
400 while (time_before(pSubTimer->ulNextJiffies, jiffies))
401 {
402 pSubTimer->ulNextJiffies += pTimer->cJiffies;
403 pSubTimer->u64NextTS += pTimer->u64NanoInterval;
404 }
405 }
406 else
407 {
408 while (pSubTimer->u64NextTS < u64NanoTS)
409 pSubTimer->u64NextTS += pTimer->u64NanoInterval;
410 pSubTimer->ulNextJiffies = jiffies + rtTimerLnxNanoToJiffies(pSubTimer->u64NextTS - u64NanoTS);
411 }
412
413 mod_timer(&pSubTimer->LnxTimer, pSubTimer->ulNextJiffies);
414#endif
415
416 /*
417 * Run the timer.
418 */
419 pTimer->pfnTimer(pTimer, pTimer->pvUser, iTick);
420 }
421
422#ifdef RT_USE_LINUX_HRTIMER
423 return rc;
424#endif
425}
426
427
428#ifdef CONFIG_SMP
429
430/**
431 * Per-cpu callback function (RTMpOnAll/RTMpOnSpecific).
432 *
433 * @param idCpu The current CPU.
434 * @param pvUser1 Pointer to the timer.
435 * @param pvUser2 Pointer to the argument structure.
436 */
437static DECLCALLBACK(void) rtTimerLnxStartAllOnCpu(RTCPUID idCpu, void *pvUser1, void *pvUser2)
438{
439 PRTTIMERLINUXSTARTONCPUARGS pArgs = (PRTTIMERLINUXSTARTONCPUARGS)pvUser2;
440 PRTTIMER pTimer = (PRTTIMER)pvUser1;
441 Assert(idCpu < pTimer->cCpus);
442 rtTimerLnxStartSubTimer(&pTimer->aSubTimers[idCpu], pArgs->u64Now, pArgs->u64First);
443}
444
445
446/**
447 * Worker for RTTimerStart() that takes care of the ugly bit.s
448 *
449 * @returns RTTimerStart() return value.
450 * @param pTimer The timer.
451 * @param pArgs The argument structure.
452 */
453static int rtTimerLnxStartAll(PRTTIMER pTimer, PRTTIMERLINUXSTARTONCPUARGS pArgs)
454{
455 RTSPINLOCKTMP Tmp = RTSPINLOCKTMP_INITIALIZER;
456 RTCPUID iCpu;
457 RTCPUSET OnlineSet;
458 RTCPUSET OnlineSet2;
459 int rc2;
460
461 /*
462 * Prepare all the sub-timers for the startup and then flag the timer
463 * as a whole as non-suspended, make sure we get them all before
464 * clearing fSuspended as the MP handler will be waiting on this
465 * should something happen while we're looping.
466 */
467 RTSpinlockAcquire(pTimer->hSpinlock, &Tmp);
468
469 do
470 {
471 RTMpGetOnlineSet(&OnlineSet);
472 for (iCpu = 0; iCpu < pTimer->cCpus; iCpu++)
473 {
474 Assert(pTimer->aSubTimers[iCpu].enmState != RTTIMERLNXSTATE_MP_STOPPING);
475 rtTimerLnxSetState(&pTimer->aSubTimers[iCpu].enmState,
476 RTCpuSetIsMember(&OnlineSet, iCpu)
477 ? RTTIMERLNXSTATE_STARTING
478 : RTTIMERLNXSTATE_STOPPED);
479 }
480 } while (!RTCpuSetIsEqual(&OnlineSet, RTMpGetOnlineSet(&OnlineSet2)));
481
482 ASMAtomicWriteBool(&pTimer->fSuspended, false);
483
484 RTSpinlockRelease(pTimer->hSpinlock, &Tmp);
485
486 /*
487 * Start them (can't find any exported function that allows me to
488 * do this without the cross calls).
489 */
490 pArgs->u64Now = RTTimeNanoTS();
491 rc2 = RTMpOnAll(rtTimerLnxStartAllOnCpu, pTimer, pArgs);
492 AssertRC(rc2); /* screw this if it fails. */
493
494 /*
495 * Reset the sub-timers who didn't start up (ALL CPUs case).
496 * CPUs that comes online between the
497 */
498 RTSpinlockAcquire(pTimer->hSpinlock, &Tmp);
499
500 for (iCpu = 0; iCpu < pTimer->cCpus; iCpu++)
501 if (rtTimerLnxCmpXchgState(&pTimer->aSubTimers[iCpu].enmState, RTTIMERLNXSTATE_STOPPED, RTTIMERLNXSTATE_STARTING))
502 {
503 /** @todo very odd case for a rainy day. Cpus that temporarily went offline while
504 * we were between calls needs to nudged as the MP handler will ignore events for
505 * them because of the STARTING state. This is an extremely unlikely case - not that
506 * that means anything in my experience... ;-) */
507 }
508
509 RTSpinlockRelease(pTimer->hSpinlock, &Tmp);
510
511 return VINF_SUCCESS;
512}
513
514
515/**
516 * Worker for RTTimerStop() that takes care of the ugly SMP bits.
517 *
518 * @returns RTTimerStop() return value.
519 * @param pTimer The timer (valid).
520 */
521static int rtTimerLnxStopAll(PRTTIMER pTimer)
522{
523 RTSPINLOCKTMP Tmp = RTSPINLOCKTMP_INITIALIZER;
524 RTCPUID iCpu;
525
526
527 /*
528 * Mark the timer as suspended and flag all timers as stopping, except
529 * for those being stopped by an MP event.
530 */
531 RTSpinlockAcquire(pTimer->hSpinlock, &Tmp);
532
533 ASMAtomicWriteBool(&pTimer->fSuspended, true);
534 for (iCpu = 0; iCpu < pTimer->cCpus; iCpu++)
535 {
536 RTTIMERLNXSTATE enmState;
537 do
538 {
539 enmState = rtTimerLnxGetState(&pTimer->aSubTimers[iCpu].enmState);
540 if ( enmState == RTTIMERLNXSTATE_STOPPED
541 || enmState == RTTIMERLNXSTATE_MP_STOPPING)
542 break;
543 Assert(enmState == RTTIMERLNXSTATE_ACTIVE);
544 } while (!rtTimerLnxCmpXchgState(&pTimer->aSubTimers[iCpu].enmState, RTTIMERLNXSTATE_STOPPING, enmState));
545 }
546
547 RTSpinlockRelease(pTimer->hSpinlock, &Tmp);
548
549 /*
550 * Do the actual stopping. Fortunately, this doesn't require any IPIs.
551 * Unfortunately it cannot be done synchronously from within the spinlock,
552 * because we might end up in an active waiting for a handler to complete.
553 */
554 for (iCpu = 0; iCpu < pTimer->cCpus; iCpu++)
555 if (rtTimerLnxGetState(&pTimer->aSubTimers[iCpu].enmState) == RTTIMERLNXSTATE_STOPPING)
556 rtTimerLnxStopSubTimer(&pTimer->aSubTimers[iCpu]);
557
558 return VINF_SUCCESS;
559}
560
561
562/**
563 * Per-cpu callback function (RTMpOnSpecific) used by rtTimerLinuxMpEvent()
564 * to start a sub-timer on a cpu that just have come online.
565 *
566 * @param idCpu The current CPU.
567 * @param pvUser1 Pointer to the timer.
568 * @param pvUser2 Pointer to the argument structure.
569 */
570static DECLCALLBACK(void) rtTimerLinuxMpStartOnCpu(RTCPUID idCpu, void *pvUser1, void *pvUser2)
571{
572 PRTTIMERLINUXSTARTONCPUARGS pArgs = (PRTTIMERLINUXSTARTONCPUARGS)pvUser2;
573 PRTTIMER pTimer = (PRTTIMER)pvUser1;
574 RTSPINLOCK hSpinlock;
575 Assert(idCpu < pTimer->cCpus);
576
577 /*
578 * We have to be kind of careful here as we might be racing RTTimerStop
579 * (and/or RTTimerDestroy, thus the paranoia.
580 */
581 hSpinlock = pTimer->hSpinlock;
582 if ( hSpinlock != NIL_RTSPINLOCK
583 && pTimer->u32Magic == RTTIMER_MAGIC)
584 {
585 RTSPINLOCKTMP Tmp = RTSPINLOCKTMP_INITIALIZER;
586 RTSpinlockAcquire(hSpinlock, &Tmp);
587
588 if ( !ASMAtomicUoReadBool(&pTimer->fSuspended)
589 && pTimer->u32Magic == RTTIMER_MAGIC)
590 {
591 /* We're sane and the timer is not suspended yet. */
592 PRTTIMERLNXSUBTIMER pSubTimer = &pTimer->aSubTimers[idCpu];
593 if (rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_MP_STARTING, RTTIMERLNXSTATE_STOPPED))
594 rtTimerLnxStartSubTimer(pSubTimer, pArgs->u64Now, pArgs->u64First);
595 }
596
597 RTSpinlockRelease(hSpinlock, &Tmp);
598 }
599}
600
601
602/**
603 * MP event notification callback.
604 *
605 * @param enmEvent The event.
606 * @param idCpu The cpu it applies to.
607 * @param pvUser The timer.
608 */
609static DECLCALLBACK(void) rtTimerLinuxMpEvent(RTMPEVENT enmEvent, RTCPUID idCpu, void *pvUser)
610{
611 PRTTIMER pTimer = (PRTTIMER)pvUser;
612 PRTTIMERLNXSUBTIMER pSubTimer = &pTimer->aSubTimers[idCpu];
613 RTSPINLOCK hSpinlock;
614 RTSPINLOCKTMP Tmp = RTSPINLOCKTMP_INITIALIZER;
615
616 Assert(idCpu < pTimer->cCpus);
617
618 /*
619 * Some initial paranoia.
620 */
621 if (pTimer->u32Magic != RTTIMER_MAGIC)
622 return;
623 hSpinlock = pTimer->hSpinlock;
624 if (hSpinlock == NIL_RTSPINLOCK)
625 return;
626
627 RTSpinlockAcquire(hSpinlock, &Tmp);
628
629 /* Is it active? */
630 if ( !ASMAtomicUoReadBool(&pTimer->fSuspended)
631 && pTimer->u32Magic == RTTIMER_MAGIC)
632 {
633 switch (enmEvent)
634 {
635 /*
636 * Try do it without leaving the spin lock, but if we have to, retake it
637 * when we're on the right cpu.
638 */
639 case RTMPEVENT_ONLINE:
640 if (rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_MP_STARTING, RTTIMERLNXSTATE_STOPPED))
641 {
642 RTTIMERLINUXSTARTONCPUARGS Args;
643 Args.u64Now = RTTimeNanoTS();
644 Args.u64First = 0;
645
646 if (RTMpCpuId() == idCpu)
647 rtTimerLnxStartSubTimer(pSubTimer, Args.u64Now, Args.u64First);
648 else
649 {
650 rtTimerLnxSetState(&pSubTimer->enmState, RTTIMERLNXSTATE_STOPPED); /* we'll recheck it. */
651 RTSpinlockRelease(hSpinlock, &Tmp);
652
653 RTMpOnSpecific(idCpu, rtTimerLinuxMpStartOnCpu, pTimer, &Args);
654 return; /* we've left the spinlock */
655 }
656 }
657 break;
658
659 /*
660 * The CPU is (going) offline, make sure the sub-timer is stopped.
661 *
662 * Linux will migrate it to a different CPU, but we don't want this. The
663 * timer function is checking for this.
664 */
665 case RTMPEVENT_OFFLINE:
666 if (rtTimerLnxCmpXchgState(&pSubTimer->enmState, RTTIMERLNXSTATE_MP_STOPPING, RTTIMERLNXSTATE_ACTIVE))
667 {
668 RTSpinlockRelease(hSpinlock, &Tmp);
669
670 rtTimerLnxStopSubTimer(pSubTimer);
671 return; /* we've left the spinlock */
672 }
673 break;
674 }
675 }
676
677 RTSpinlockRelease(hSpinlock, &Tmp);
678}
679
680#endif /* CONFIG_SMP */
681
682
683/**
684 * Callback function use by RTTimerStart via RTMpOnSpecific to start
685 * a timer running on a specific CPU.
686 *
687 * @param idCpu The current CPU.
688 * @param pvUser1 Pointer to the timer.
689 * @param pvUser2 Pointer to the argument structure.
690 */
691static DECLCALLBACK(void) rtTimerLnxStartOnSpecificCpu(RTCPUID idCpu, void *pvUser1, void *pvUser2)
692{
693 PRTTIMERLINUXSTARTONCPUARGS pArgs = (PRTTIMERLINUXSTARTONCPUARGS)pvUser2;
694 PRTTIMER pTimer = (PRTTIMER)pvUser1;
695 rtTimerLnxStartSubTimer(&pTimer->aSubTimers[0], pArgs->u64Now, pArgs->u64First);
696}
697
698
699RTDECL(int) RTTimerStart(PRTTIMER pTimer, uint64_t u64First)
700{
701 RTTIMERLINUXSTARTONCPUARGS Args;
702 int rc2;
703
704 /*
705 * Validate.
706 */
707 AssertPtrReturn(pTimer, VERR_INVALID_HANDLE);
708 AssertReturn(pTimer->u32Magic == RTTIMER_MAGIC, VERR_INVALID_HANDLE);
709
710 if (!ASMAtomicUoReadBool(&pTimer->fSuspended))
711 return VERR_TIMER_ACTIVE;
712
713 Args.u64First = u64First;
714#ifdef CONFIG_SMP
715 /*
716 * Omnit timer?
717 */
718 if (pTimer->fAllCpus)
719 return rtTimerLnxStartAll(pTimer, &Args);
720#endif
721
722 /*
723 * Simple timer - Pretty straight forward.
724 */
725 Args.u64Now = RTTimeNanoTS();
726 rtTimerLnxSetState(&pTimer->aSubTimers[0].enmState, RTTIMERLNXSTATE_STARTING);
727 ASMAtomicWriteBool(&pTimer->fSuspended, false);
728 if (!pTimer->fSpecificCpu)
729 rtTimerLnxStartSubTimer(&pTimer->aSubTimers[0], Args.u64Now, Args.u64First);
730 else
731 {
732 rc2 = RTMpOnSpecific(pTimer->idCpu, rtTimerLnxStartOnSpecificCpu, pTimer, &Args);
733 if (RT_FAILURE(rc2))
734 {
735 /* Suspend it, the cpu id is probably invalid or offline. */
736 ASMAtomicWriteBool(&pTimer->fSuspended, true);
737 rtTimerLnxSetState(&pTimer->aSubTimers[0].enmState, RTTIMERLNXSTATE_STOPPED);
738 return rc2;
739 }
740 }
741
742 return VINF_SUCCESS;
743}
744RT_EXPORT_SYMBOL(RTTimerStart);
745
746
747RTDECL(int) RTTimerStop(PRTTIMER pTimer)
748{
749
750 /*
751 * Validate.
752 */
753 AssertPtrReturn(pTimer, VERR_INVALID_HANDLE);
754 AssertReturn(pTimer->u32Magic == RTTIMER_MAGIC, VERR_INVALID_HANDLE);
755
756 if (ASMAtomicUoReadBool(&pTimer->fSuspended))
757 return VERR_TIMER_SUSPENDED;
758
759#ifdef CONFIG_SMP
760 /*
761 * Omni timer?
762 */
763 if (pTimer->fAllCpus)
764 return rtTimerLnxStopAll(pTimer);
765#endif
766
767 /*
768 * Simple timer.
769 */
770 ASMAtomicWriteBool(&pTimer->fSuspended, true);
771 rtTimerLnxSetState(&pTimer->aSubTimers[0].enmState, RTTIMERLNXSTATE_STOPPING);
772 rtTimerLnxStopSubTimer(&pTimer->aSubTimers[0]);
773
774 return VINF_SUCCESS;
775}
776RT_EXPORT_SYMBOL(RTTimerStop);
777
778
779RTDECL(int) RTTimerDestroy(PRTTIMER pTimer)
780{
781 RTSPINLOCK hSpinlock;
782
783 /* It's ok to pass NULL pointer. */
784 if (pTimer == /*NIL_RTTIMER*/ NULL)
785 return VINF_SUCCESS;
786 AssertPtrReturn(pTimer, VERR_INVALID_HANDLE);
787 AssertReturn(pTimer->u32Magic == RTTIMER_MAGIC, VERR_INVALID_HANDLE);
788
789 /*
790 * Remove the MP notifications first because it'll reduce the risk of
791 * us overtaking any MP event that might theoretically be racing us here.
792 */
793 hSpinlock = pTimer->hSpinlock;
794#ifdef CONFIG_SMP
795 if ( pTimer->cCpus > 1
796 && hSpinlock != NIL_RTSPINLOCK)
797 {
798 int rc = RTMpNotificationDeregister(rtTimerLinuxMpEvent, pTimer);
799 AssertRC(rc);
800 }
801#endif /* CONFIG_SMP */
802
803 /*
804 * Stop the timer if it's running.
805 */
806 if (!ASMAtomicUoReadBool(&pTimer->fSuspended))
807 RTTimerStop(pTimer);
808
809 /*
810 * Uninitialize the structure and free the associated resources.
811 * The spinlock goes last.
812 */
813 ASMAtomicWriteU32(&pTimer->u32Magic, ~RTTIMER_MAGIC);
814 RTMemFree(pTimer);
815 if (hSpinlock != NIL_RTSPINLOCK)
816 RTSpinlockDestroy(hSpinlock);
817
818 return VINF_SUCCESS;
819}
820RT_EXPORT_SYMBOL(RTTimerDestroy);
821
822
823RTDECL(int) RTTimerCreateEx(PRTTIMER *ppTimer, uint64_t u64NanoInterval, unsigned fFlags, PFNRTTIMER pfnTimer, void *pvUser)
824{
825 PRTTIMER pTimer;
826 RTCPUID iCpu;
827 unsigned cCpus;
828
829 *ppTimer = NULL;
830
831 /*
832 * Validate flags.
833 */
834 if (!RTTIMER_FLAGS_ARE_VALID(fFlags))
835 return VERR_INVALID_PARAMETER;
836 if ( (fFlags & RTTIMER_FLAGS_CPU_SPECIFIC)
837 && (fFlags & RTTIMER_FLAGS_CPU_ALL) != RTTIMER_FLAGS_CPU_ALL
838 && !RTMpIsCpuOnline(fFlags & RTTIMER_FLAGS_CPU_MASK))
839 return (fFlags & RTTIMER_FLAGS_CPU_MASK) > RTMpGetMaxCpuId()
840 ? VERR_CPU_NOT_FOUND
841 : VERR_CPU_OFFLINE;
842
843 /*
844 * Allocate the timer handler.
845 */
846 cCpus = 1;
847#ifdef CONFIG_SMP
848 if ((fFlags & RTTIMER_FLAGS_CPU_ALL) == RTTIMER_FLAGS_CPU_ALL)
849 {
850 cCpus = RTMpGetMaxCpuId() + 1;
851 Assert(cCpus <= RTCPUSET_MAX_CPUS); /* On linux we have a 1:1 relationship between cpuid and set index. */
852 AssertReturn(u64NanoInterval, VERR_NOT_IMPLEMENTED); /* We don't implement single shot on all cpus, sorry. */
853 }
854#endif
855
856 pTimer = (PRTTIMER)RTMemAllocZ(RT_OFFSETOF(RTTIMER, aSubTimers[cCpus]));
857 if (!pTimer)
858 return VERR_NO_MEMORY;
859
860 /*
861 * Initialize it.
862 */
863 pTimer->u32Magic = RTTIMER_MAGIC;
864 pTimer->hSpinlock = NIL_RTSPINLOCK;
865 pTimer->fSuspended = true;
866#ifdef CONFIG_SMP
867 pTimer->fSpecificCpu = (fFlags & RTTIMER_FLAGS_CPU_SPECIFIC) && (fFlags & RTTIMER_FLAGS_CPU_ALL) != RTTIMER_FLAGS_CPU_ALL;
868 pTimer->fAllCpus = (fFlags & RTTIMER_FLAGS_CPU_ALL) == RTTIMER_FLAGS_CPU_ALL;
869 pTimer->idCpu = fFlags & RTTIMER_FLAGS_CPU_MASK;
870#else
871 pTimer->fSpecificCpu = !!(fFlags & RTTIMER_FLAGS_CPU_SPECIFIC);
872 pTimer->idCpu = RTMpCpuId();
873#endif
874 pTimer->cCpus = cCpus;
875 pTimer->pfnTimer = pfnTimer;
876 pTimer->pvUser = pvUser;
877 pTimer->u64NanoInterval = u64NanoInterval;
878#ifndef RT_USE_LINUX_HRTIMER
879 pTimer->cJiffies = u64NanoInterval / RTTimerGetSystemGranularity();
880 if (pTimer->cJiffies * RTTimerGetSystemGranularity() != u64NanoInterval)
881 pTimer->cJiffies = 0;
882#endif
883
884 for (iCpu = 0; iCpu < cCpus; iCpu++)
885 {
886#ifdef RT_USE_LINUX_HRTIMER
887 hrtimer_init(&pTimer->aSubTimers[iCpu].LnxTimer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
888 pTimer->aSubTimers[iCpu].LnxTimer.function = rtTimerLinuxCallback;
889#else
890 init_timer(&pTimer->aSubTimers[iCpu].LnxTimer);
891 pTimer->aSubTimers[iCpu].LnxTimer.data = (unsigned long)&pTimer->aSubTimers[iCpu];
892 pTimer->aSubTimers[iCpu].LnxTimer.function = rtTimerLinuxCallback;
893 pTimer->aSubTimers[iCpu].LnxTimer.expires = jiffies;
894 pTimer->aSubTimers[iCpu].u64StartTS = 0;
895 pTimer->aSubTimers[iCpu].u64NextTS = 0;
896#endif
897 pTimer->aSubTimers[iCpu].iTick = 0;
898 pTimer->aSubTimers[iCpu].pParent = pTimer;
899 pTimer->aSubTimers[iCpu].enmState = RTTIMERLNXSTATE_STOPPED;
900 }
901
902#ifdef CONFIG_SMP
903 /*
904 * If this is running on ALL cpus, we'll have to register a callback
905 * for MP events (so timers can be started/stopped on cpus going
906 * online/offline). We also create the spinlock for syncrhonizing
907 * stop/start/mp-event.
908 */
909 if (cCpus > 1)
910 {
911 int rc = RTSpinlockCreate(&pTimer->hSpinlock);
912 if (RT_SUCCESS(rc))
913 rc = RTMpNotificationRegister(rtTimerLinuxMpEvent, pTimer);
914 else
915 pTimer->hSpinlock = NIL_RTSPINLOCK;
916 if (RT_FAILURE(rc))
917 {
918 RTTimerDestroy(pTimer);
919 return rc;
920 }
921 }
922#endif /* CONFIG_SMP */
923
924 *ppTimer = pTimer;
925 return VINF_SUCCESS;
926}
927RT_EXPORT_SYMBOL(RTTimerCreateEx);
928
929
930RTDECL(uint32_t) RTTimerGetSystemGranularity(void)
931{
932#ifdef RT_USE_LINUX_HRTIMER
933 struct timespec Ts;
934 int rc = hrtimer_get_res(CLOCK_MONOTONIC, &Ts);
935 if (!rc)
936 {
937 Assert(!Ts.tv_sec);
938 return Ts.tv_nsec;
939 }
940#endif
941 return 1000000000 / HZ; /* ns */
942}
943RT_EXPORT_SYMBOL(RTTimerGetSystemGranularity);
944
945
946RTDECL(int) RTTimerRequestSystemGranularity(uint32_t u32Request, uint32_t *pu32Granted)
947{
948 return VERR_NOT_SUPPORTED;
949}
950RT_EXPORT_SYMBOL(RTTimerRequestSystemGranularity);
951
952
953RTDECL(int) RTTimerReleaseSystemGranularity(uint32_t u32Granted)
954{
955 return VERR_NOT_SUPPORTED;
956}
957RT_EXPORT_SYMBOL(RTTimerReleaseSystemGranularity);
958

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy