timekeeping.c source code [Linux/kernel/time/timekeeping.c]

1	// SPDX-License-Identifier: GPL-2.0
2	/*
3	* Kernel timekeeping code and accessor functions. Based on code from
4	* timer.c, moved in commit 8524070b7982.
5	*/
6	#include <linux/timekeeper_internal.h>
7	#include <linux/module.h>
8	#include <linux/interrupt.h>
9	#include <linux/kobject.h>
10	#include <linux/percpu.h>
11	#include <linux/init.h>
12	#include <linux/mm.h>
13	#include <linux/nmi.h>
14	#include <linux/sched.h>
15	#include <linux/sched/loadavg.h>
16	#include <linux/sched/clock.h>
17	#include <linux/syscore_ops.h>
18	#include <linux/clocksource.h>
19	#include <linux/jiffies.h>
20	#include <linux/time.h>
21	#include <linux/timex.h>
22	#include <linux/tick.h>
23	#include <linux/stop_machine.h>
24	#include <linux/pvclock_gtod.h>
25	#include <linux/compiler.h>
26	#include <linux/audit.h>
27	#include <linux/random.h>
28
29	#include <vdso/auxclock.h>
30
31	#include "tick-internal.h"
32	#include "ntp_internal.h"
33	#include "timekeeping_internal.h"
34
35	#define TK_CLEAR_NTP (1 << 0)
36	#define TK_CLOCK_WAS_SET (1 << 1)
37
38	#define TK_UPDATE_ALL (TK_CLEAR_NTP \| TK_CLOCK_WAS_SET)
39
40	enum timekeeping_adv_mode {
41	/ Update timekeeper when a tick has passed /
42	TK_ADV_TICK,
43
44	/ Update timekeeper on a direct frequency change /
45	TK_ADV_FREQ
46	};
47
48	/*
49	* The most important data for readout fits into a single 64 byte
50	* cache line.
51	*/
52	struct tk_data {
53	seqcount_raw_spinlock_t seq;
54	struct timekeeper timekeeper;
55	struct timekeeper shadow_timekeeper;
56	raw_spinlock_t lock;
57	} ____cacheline_aligned;
58
59	static struct tk_data timekeeper_data[TIMEKEEPERS_MAX];
60
61	/ The core timekeeper /
62	#define tk_core (timekeeper_data[TIMEKEEPER_CORE])
63
64	#ifdef CONFIG_POSIX_AUX_CLOCKS
65	static inline bool tk_get_aux_ts64(unsigned int tkid, struct timespec64 *ts)
66	{
67	return ktime_get_aux_ts64(CLOCK_AUX + tkid - TIMEKEEPER_AUX_FIRST, ts);
68	}
69
70	static inline bool tk_is_aux(const struct timekeeper *tk)
71	{
72	return tk->id >= TIMEKEEPER_AUX_FIRST && tk->id <= TIMEKEEPER_AUX_LAST;
73	}
74	#else
75	static inline bool tk_get_aux_ts64(unsigned int tkid, struct timespec64 *ts)
76	{
77	return false;
78	}
79
80	static inline bool tk_is_aux(const struct timekeeper *tk)
81	{
82	return false;
83	}
84	#endif
85
86	static inline void tk_update_aux_offs(struct timekeeper *tk, ktime_t offs)
87	{
88	tk->offs_aux = offs;
89	tk->monotonic_to_aux = ktime_to_timespec64(offs);
90	}
91
92	/ flag for if timekeeping is suspended /
93	int __read_mostly timekeeping_suspended;
94
95	/**
96	* struct tk_fast - NMI safe timekeeper
97	* @seq: Sequence counter for protecting updates. The lowest bit
98	* is the index for the tk_read_base array
99	* @base: tk_read_base array. Access is indexed by the lowest bit of
100	* @seq.
101	*
102	* See @update_fast_timekeeper() below.
103	*/
104	struct tk_fast {
105	seqcount_latch_t seq;
106	struct tk_read_base base[`2`];
107	};
108
109	/ Suspend-time cycles value for halted fast timekeeper. /
110	static u64 cycles_at_suspend;
111
112	static u64 dummy_clock_read(struct clocksource *cs)
113	{
114	if (timekeeping_suspended)
115	return cycles_at_suspend;
116	return local_clock();
117	}
118
119	static struct clocksource dummy_clock = {
120	.read = dummy_clock_read,
121	};
122
123	/*
124	* Boot time initialization which allows local_clock() to be utilized
125	* during early boot when clocksources are not available. local_clock()
126	* returns nanoseconds already so no conversion is required, hence mult=1
127	* and shift=0. When the first proper clocksource is installed then
128	* the fast time keepers are updated with the correct values.
129	*/
130	#define FAST_TK_INIT \
131	{ \
132	.clock = &dummy_clock, \
133	.mask = CLOCKSOURCE_MASK(64), \
134	.mult = 1, \
135	.shift = 0, \
136	}
137
138	static struct tk_fast tk_fast_mono ____cacheline_aligned = {
139	.seq = SEQCNT_LATCH_ZERO(tk_fast_mono.seq),
140	.base[`0`] = FAST_TK_INIT,
141	.base[`1`] = FAST_TK_INIT,
142	};
143
144	static struct tk_fast tk_fast_raw ____cacheline_aligned = {
145	.seq = SEQCNT_LATCH_ZERO(tk_fast_raw.seq),
146	.base[`0`] = FAST_TK_INIT,
147	.base[`1`] = FAST_TK_INIT,
148	};
149
150	#ifdef CONFIG_POSIX_AUX_CLOCKS
151	static __init void tk_aux_setup(void);
152	static void tk_aux_update_clocksource(void);
153	static void tk_aux_advance(void);
154	#else
155	static inline void tk_aux_setup(void) { }
156	static inline void tk_aux_update_clocksource(void) { }
157	static inline void tk_aux_advance(void) { }
158	#endif
159
160	unsigned long timekeeper_lock_irqsave(void)
161	{
162	unsigned long flags;
163
164	raw_spin_lock_irqsave(&tk_core.lock, flags);
165	return flags;
166	}
167
168	void timekeeper_unlock_irqrestore(unsigned long flags)
169	{
170	raw_spin_unlock_irqrestore(&tk_core.lock, flags);
171	}
172
173	/*
174	* Multigrain timestamps require tracking the latest fine-grained timestamp
175	* that has been issued, and never returning a coarse-grained timestamp that is
176	* earlier than that value.
177	*
178	* mg_floor represents the latest fine-grained time that has been handed out as
179	* a file timestamp on the system. This is tracked as a monotonic ktime_t, and
180	* converted to a realtime clock value on an as-needed basis.
181	*
182	* Maintaining mg_floor ensures the multigrain interfaces never issue a
183	* timestamp earlier than one that has been previously issued.
184	*
185	* The exception to this rule is when there is a backward realtime clock jump. If
186	* such an event occurs, a timestamp can appear to be earlier than a previous one.
187	*/
188	static __cacheline_aligned_in_smp atomic64_t mg_floor;
189
190	static inline void tk_normalize_xtime(struct timekeeper *tk)
191	{
192	while (tk->tkr_mono.xtime_nsec >= ((u64)NSEC_PER_SEC << tk->tkr_mono.shift)) {
193	tk->tkr_mono.xtime_nsec -= (u64)NSEC_PER_SEC << tk->tkr_mono.shift;
194	tk->xtime_sec++;
195	}
196	while (tk->tkr_raw.xtime_nsec >= ((u64)NSEC_PER_SEC << tk->tkr_raw.shift)) {
197	tk->tkr_raw.xtime_nsec -= (u64)NSEC_PER_SEC << tk->tkr_raw.shift;
198	tk->raw_sec++;
199	}
200	}
201
202	static inline struct timespec64 tk_xtime(const struct timekeeper *tk)
203	{
204	struct timespec64 ts;
205
206	ts.tv_sec = tk->xtime_sec;
207	ts.tv_nsec = (long)(tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift);
208	return ts;
209	}
210
211	static inline struct timespec64 tk_xtime_coarse(const struct timekeeper *tk)
212	{
213	struct timespec64 ts;
214
215	ts.tv_sec = tk->xtime_sec;
216	ts.tv_nsec = tk->coarse_nsec;
217	return ts;
218	}
219
220	/*
221	* Update the nanoseconds part for the coarse time keepers. They can't rely
222	* on xtime_nsec because xtime_nsec could be adjusted by a small negative
223	* amount when the multiplication factor of the clock is adjusted, which
224	* could cause the coarse clocks to go slightly backwards. See
225	* timekeeping_apply_adjustment(). Thus we keep a separate copy for the coarse
226	* clockids which only is updated when the clock has been set or we have
227	* accumulated time.
228	*/
229	static inline void tk_update_coarse_nsecs(struct timekeeper *tk)
230	{
231	tk->coarse_nsec = tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift;
232	}
233
234	static void tk_set_xtime(struct timekeeper tk, const* struct timespec64 *ts)
235	{
236	tk->xtime_sec = ts->tv_sec;
237	tk->tkr_mono.xtime_nsec = (u64)ts->tv_nsec << tk->tkr_mono.shift;
238	tk_update_coarse_nsecs(tk);
239	}
240
241	static void tk_xtime_add(struct timekeeper tk, const* struct timespec64 *ts)
242	{
243	tk->xtime_sec += ts->tv_sec;
244	tk->tkr_mono.xtime_nsec += (u64)ts->tv_nsec << tk->tkr_mono.shift;
245	tk_normalize_xtime(tk);
246	tk_update_coarse_nsecs(tk);
247	}
248
249	static void tk_set_wall_to_mono(struct timekeeper tk, struct* timespec64 wtm)
250	{
251	struct timespec64 tmp;
252
253	/*
254	* Verify consistency of: offset_real = -wall_to_monotonic
255	* before modifying anything
256	*/
257	set_normalized_timespec64(ts: &tmp, sec: -tk->wall_to_monotonic.tv_sec,
258	nsec: -tk->wall_to_monotonic.tv_nsec);
259	WARN_ON_ONCE(tk->offs_real != timespec64_to_ktime(tmp));
260	tk->wall_to_monotonic = wtm;
261	set_normalized_timespec64(ts: &tmp, sec: -wtm.tv_sec, nsec: -wtm.tv_nsec);
262	/ Paired with READ_ONCE() in ktime_mono_to_any() /
263	WRITE_ONCE(tk->offs_real, timespec64_to_ktime(tmp));
264	WRITE_ONCE(tk->offs_tai, ktime_add(tk->offs_real, ktime_set(tk->tai_offset, `0`)));
265	}
266
267	static inline void tk_update_sleep_time(struct timekeeper *tk, ktime_t delta)
268	{
269	/ Paired with READ_ONCE() in ktime_mono_to_any() /
270	WRITE_ONCE(tk->offs_boot, ktime_add(tk->offs_boot, delta));
271	/*
272	* Timespec representation for VDSO update to avoid 64bit division
273	* on every update.
274	*/
275	tk->monotonic_to_boot = ktime_to_timespec64(tk->offs_boot);
276	}
277
278	/*
279	* tk_clock_read - atomic clocksource read() helper
280	*
281	* This helper is necessary to use in the read paths because, while the
282	* seqcount ensures we don't return a bad value while structures are updated,
283	* it doesn't protect from potential crashes. There is the possibility that
284	* the tkr's clocksource may change between the read reference, and the
285	* clock reference passed to the read function. This can cause crashes if
286	* the wrong clocksource is passed to the wrong read function.
287	* This isn't necessary to use when holding the tk_core.lock or doing
288	* a read of the fast-timekeeper tkrs (which is protected by its own locking
289	* and update logic).
290	*/
291	static inline u64 tk_clock_read(const struct tk_read_base *tkr)
292	{
293	struct clocksource *clock = READ_ONCE(tkr->clock);
294
295	return clock->read(clock);
296	}
297
298	/**
299	* tk_setup_internals - Set up internals to use clocksource clock.
300	*
301	* @tk: The target timekeeper to setup.
302	* @clock: Pointer to clocksource.
303	*
304	* Calculates a fixed cycle/nsec interval for a given clocksource/adjustment
305	* pair and interval request.
306	*
307	* Unless you're the timekeeping code, you should not be using this!
308	*/
309	static void tk_setup_internals(struct timekeeper tk, struct* clocksource *clock)
310	{
311	u64 interval;
312	u64 tmp, ntpinterval;
313	struct clocksource *old_clock;
314
315	++tk->cs_was_changed_seq;
316	old_clock = tk->tkr_mono.clock;
317	tk->tkr_mono.clock = clock;
318	tk->tkr_mono.mask = clock->mask;
319	tk->tkr_mono.cycle_last = tk_clock_read(tkr: &tk->tkr_mono);
320
321	tk->tkr_raw.clock = clock;
322	tk->tkr_raw.mask = clock->mask;
323	tk->tkr_raw.cycle_last = tk->tkr_mono.cycle_last;
324
325	/ Do the ns -> cycle conversion first, using original mult /
326	tmp = NTP_INTERVAL_LENGTH;
327	tmp <<= clock->shift;
328	ntpinterval = tmp;
329	tmp += clock->mult/`2`;
330	do_div(tmp, clock->mult);
331	if (tmp == `0`)
332	tmp = `1`;
333
334	interval = (u64) tmp;
335	tk->cycle_interval = interval;
336
337	/ Go back from cycles -> shifted ns /
338	tk->xtime_interval = interval * clock->mult;
339	tk->xtime_remainder = ntpinterval - tk->xtime_interval;
340	tk->raw_interval = interval * clock->mult;
341
342	/ if changing clocks, convert xtime_nsec shift units /
343	if (old_clock) {
344	int shift_change = clock->shift - old_clock->shift;
345	if (shift_change < `0`) {
346	tk->tkr_mono.xtime_nsec >>= -shift_change;
347	tk->tkr_raw.xtime_nsec >>= -shift_change;
348	} else {
349	tk->tkr_mono.xtime_nsec <<= shift_change;
350	tk->tkr_raw.xtime_nsec <<= shift_change;
351	}
352	}
353
354	tk->tkr_mono.shift = clock->shift;
355	tk->tkr_raw.shift = clock->shift;
356
357	tk->ntp_error = `0`;
358	tk->ntp_error_shift = NTP_SCALE_SHIFT - clock->shift;
359	tk->ntp_tick = ntpinterval << tk->ntp_error_shift;
360
361	/*
362	* The timekeeper keeps its own mult values for the currently
363	* active clocksource. These value will be adjusted via NTP
364	* to counteract clock drifting.
365	*/
366	tk->tkr_mono.mult = clock->mult;
367	tk->tkr_raw.mult = clock->mult;
368	tk->ntp_err_mult = `0`;
369	tk->skip_second_overflow = `0`;
370	}
371
372	/ Timekeeper helper functions. /
373	static noinline u64 delta_to_ns_safe(const struct tk_read_base *tkr, u64 delta)
374	{
375	return mul_u64_u32_add_u64_shr(a: delta, mul: tkr->mult, b: tkr->xtime_nsec, shift: tkr->shift);
376	}
377
378	static inline u64 timekeeping_cycles_to_ns(const struct tk_read_base *tkr, u64 cycles)
379	{
380	/ Calculate the delta since the last update_wall_time() /
381	u64 mask = tkr->mask, delta = (cycles - tkr->cycle_last) & mask;
382
383	/*
384	* This detects both negative motion and the case where the delta
385	* overflows the multiplication with tkr->mult.
386	*/
387	if (unlikely(delta > tkr->clock->max_cycles)) {
388	/*
389	* Handle clocksource inconsistency between CPUs to prevent
390	* time from going backwards by checking for the MSB of the
391	* mask being set in the delta.
392	*/
393	if (delta & ~(mask >> `1`))
394	return tkr->xtime_nsec >> tkr->shift;
395
396	return delta_to_ns_safe(tkr, delta);
397	}
398
399	return ((delta * tkr->mult) + tkr->xtime_nsec) >> tkr->shift;
400	}
401
402	static __always_inline u64 timekeeping_get_ns(const struct tk_read_base *tkr)
403	{
404	return timekeeping_cycles_to_ns(tkr, cycles: tk_clock_read(tkr));
405	}
406
407	/**
408	* update_fast_timekeeper - Update the fast and NMI safe monotonic timekeeper.
409	* @tkr: Timekeeping readout base from which we take the update
410	* @tkf: Pointer to NMI safe timekeeper
411	*
412	* We want to use this from any context including NMI and tracing /
413	* instrumenting the timekeeping code itself.
414	*
415	* Employ the latch technique; see @write_seqcount_latch.
416	*
417	* So if a NMI hits the update of base[0] then it will use base[1]
418	* which is still consistent. In the worst case this can result is a
419	* slightly wrong timestamp (a few nanoseconds). See
420	* @ktime_get_mono_fast_ns.
421	*/
422	static void update_fast_timekeeper(const struct tk_read_base *tkr,
423	struct tk_fast *tkf)
424	{
425	struct tk_read_base *base = tkf->base;
426
427	/ Force readers off to base[1] /
428	write_seqcount_latch_begin(s: &tkf->seq);
429
430	/ Update base[0] /
431	memcpy(to: base, from: tkr, len: sizeof(*base));
432
433	/ Force readers back to base[0] /
434	write_seqcount_latch(s: &tkf->seq);
435
436	/ Update base[1] /
437	memcpy(to: base + `1`, from: base, len: sizeof(*base));
438
439	write_seqcount_latch_end(s: &tkf->seq);
440	}
441
442	static __always_inline u64 __ktime_get_fast_ns(struct tk_fast *tkf)
443	{
444	struct tk_read_base *tkr;
445	unsigned int seq;
446	u64 now;
447
448	do {
449	seq = read_seqcount_latch(s: &tkf->seq);
450	tkr = tkf->base + (seq & `0x01`);
451	now = ktime_to_ns(kt: tkr->base);
452	now += timekeeping_get_ns(tkr);
453	} while (read_seqcount_latch_retry(s: &tkf->seq, start: seq));
454
455	return now;
456	}
457
458	/**
459	* ktime_get_mono_fast_ns - Fast NMI safe access to clock monotonic
460	*
461	* This timestamp is not guaranteed to be monotonic across an update.
462	* The timestamp is calculated by:
463	*
464	* now = base_mono + clock_delta * slope
465	*
466	* So if the update lowers the slope, readers who are forced to the
467	* not yet updated second array are still using the old steeper slope.
468	*
469	* tmono
470	* ^
471	* \| o n
472	* \| o n
473	* \| u
474	* \| o
475	* \|o
476	* \|12345678---> reader order
477	*
478	* o = old slope
479	* u = update
480	* n = new slope
481	*
482	* So reader 6 will observe time going backwards versus reader 5.
483	*
484	* While other CPUs are likely to be able to observe that, the only way
485	* for a CPU local observation is when an NMI hits in the middle of
486	* the update. Timestamps taken from that NMI context might be ahead
487	* of the following timestamps. Callers need to be aware of that and
488	* deal with it.
489	*/
490	u64 notrace ktime_get_mono_fast_ns(void)
491	{
492	return __ktime_get_fast_ns(tkf: &tk_fast_mono);
493	}
494	EXPORT_SYMBOL_GPL(ktime_get_mono_fast_ns);
495
496	/**
497	* ktime_get_raw_fast_ns - Fast NMI safe access to clock monotonic raw
498	*
499	* Contrary to ktime_get_mono_fast_ns() this is always correct because the
500	* conversion factor is not affected by NTP/PTP correction.
501	*/
502	u64 notrace ktime_get_raw_fast_ns(void)
503	{
504	return __ktime_get_fast_ns(tkf: &tk_fast_raw);
505	}
506	EXPORT_SYMBOL_GPL(ktime_get_raw_fast_ns);
507
508	/**
509	* ktime_get_boot_fast_ns - NMI safe and fast access to boot clock.
510	*
511	* To keep it NMI safe since we're accessing from tracing, we're not using a
512	* separate timekeeper with updates to monotonic clock and boot offset
513	* protected with seqcounts. This has the following minor side effects:
514	*
515	* (1) Its possible that a timestamp be taken after the boot offset is updated
516	* but before the timekeeper is updated. If this happens, the new boot offset
517	* is added to the old timekeeping making the clock appear to update slightly
518	* earlier:
519	* CPU 0 CPU 1
520	* timekeeping_inject_sleeptime64()
521	* __timekeeping_inject_sleeptime(tk, delta);
522	* timestamp();
523	* timekeeping_update_staged(tkd, TK_CLEAR_NTP...);
524	*
525	* (2) On 32-bit systems, the 64-bit boot offset (tk->offs_boot) may be
526	* partially updated. Since the tk->offs_boot update is a rare event, this
527	* should be a rare occurrence which postprocessing should be able to handle.
528	*
529	* The caveats vs. timestamp ordering as documented for ktime_get_mono_fast_ns()
530	* apply as well.
531	*/
532	u64 notrace ktime_get_boot_fast_ns(void)
533	{
534	struct timekeeper *tk = &tk_core.timekeeper;
535
536	return (ktime_get_mono_fast_ns() + ktime_to_ns(data_race(tk->offs_boot)));
537	}
538	EXPORT_SYMBOL_GPL(ktime_get_boot_fast_ns);
539
540	/**
541	* ktime_get_tai_fast_ns - NMI safe and fast access to tai clock.
542	*
543	* The same limitations as described for ktime_get_boot_fast_ns() apply. The
544	* mono time and the TAI offset are not read atomically which may yield wrong
545	* readouts. However, an update of the TAI offset is an rare event e.g., caused
546	* by settime or adjtimex with an offset. The user of this function has to deal
547	* with the possibility of wrong timestamps in post processing.
548	*/
549	u64 notrace ktime_get_tai_fast_ns(void)
550	{
551	struct timekeeper *tk = &tk_core.timekeeper;
552
553	return (ktime_get_mono_fast_ns() + ktime_to_ns(data_race(tk->offs_tai)));
554	}
555	EXPORT_SYMBOL_GPL(ktime_get_tai_fast_ns);
556
557	/**
558	* ktime_get_real_fast_ns: - NMI safe and fast access to clock realtime.
559	*
560	* See ktime_get_mono_fast_ns() for documentation of the time stamp ordering.
561	*/
562	u64 ktime_get_real_fast_ns(void)
563	{
564	struct tk_fast *tkf = &tk_fast_mono;
565	struct tk_read_base *tkr;
566	u64 baser, delta;
567	unsigned int seq;
568
569	do {
570	seq = raw_read_seqcount_latch(s: &tkf->seq);
571	tkr = tkf->base + (seq & `0x01`);
572	baser = ktime_to_ns(kt: tkr->base_real);
573	delta = timekeeping_get_ns(tkr);
574	} while (raw_read_seqcount_latch_retry(s: &tkf->seq, start: seq));
575
576	return baser + delta;
577	}
578	EXPORT_SYMBOL_GPL(ktime_get_real_fast_ns);
579
580	/**
581	* halt_fast_timekeeper - Prevent fast timekeeper from accessing clocksource.
582	* @tk: Timekeeper to snapshot.
583	*
584	* It generally is unsafe to access the clocksource after timekeeping has been
585	* suspended, so take a snapshot of the readout base of @tk and use it as the
586	* fast timekeeper's readout base while suspended. It will return the same
587	* number of cycles every time until timekeeping is resumed at which time the
588	* proper readout base for the fast timekeeper will be restored automatically.
589	*/
590	static void halt_fast_timekeeper(const struct timekeeper *tk)
591	{
592	static struct tk_read_base tkr_dummy;
593	const struct tk_read_base *tkr = &tk->tkr_mono;
594
595	memcpy(to: &tkr_dummy, from: tkr, len: sizeof(tkr_dummy));
596	cycles_at_suspend = tk_clock_read(tkr);
597	tkr_dummy.clock = &dummy_clock;
598	tkr_dummy.base_real = tkr->base + tk->offs_real;
599	update_fast_timekeeper(tkr: &tkr_dummy, tkf: &tk_fast_mono);
600
601	tkr = &tk->tkr_raw;
602	memcpy(to: &tkr_dummy, from: tkr, len: sizeof(tkr_dummy));
603	tkr_dummy.clock = &dummy_clock;
604	update_fast_timekeeper(tkr: &tkr_dummy, tkf: &tk_fast_raw);
605	}
606
607	static RAW_NOTIFIER_HEAD(pvclock_gtod_chain);
608
609	static void update_pvclock_gtod(struct timekeeper *tk, bool was_set)
610	{
611	raw_notifier_call_chain(nh: &pvclock_gtod_chain, val: was_set, v: tk);
612	}
613
614	/**
615	* pvclock_gtod_register_notifier - register a pvclock timedata update listener
616	* @nb: Pointer to the notifier block to register
617	*/
618	int pvclock_gtod_register_notifier(struct notifier_block *nb)
619	{
620	struct timekeeper *tk = &tk_core.timekeeper;
621	int ret;
622
623	guard(raw_spinlock_irqsave)(l: &tk_core.lock);
624	ret = raw_notifier_chain_register(nh: &pvclock_gtod_chain, nb);
625	update_pvclock_gtod(tk, was_set: true);
626
627	return ret;
628	}
629	EXPORT_SYMBOL_GPL(pvclock_gtod_register_notifier);
630
631	/**
632	* pvclock_gtod_unregister_notifier - unregister a pvclock
633	* timedata update listener
634	* @nb: Pointer to the notifier block to unregister
635	*/
636	int pvclock_gtod_unregister_notifier(struct notifier_block *nb)
637	{
638	guard(raw_spinlock_irqsave)(l: &tk_core.lock);
639	return raw_notifier_chain_unregister(nh: &pvclock_gtod_chain, nb);
640	}
641	EXPORT_SYMBOL_GPL(pvclock_gtod_unregister_notifier);
642
643	/*
644	* tk_update_leap_state - helper to update the next_leap_ktime
645	*/
646	static inline void tk_update_leap_state(struct timekeeper *tk)
647	{
648	tk->next_leap_ktime = ntp_get_next_leap(tkid: tk->id);
649	if (tk->next_leap_ktime != KTIME_MAX)
650	/ Convert to monotonic time /
651	tk->next_leap_ktime = ktime_sub(tk->next_leap_ktime, tk->offs_real);
652	}
653
654	/*
655	* Leap state update for both shadow and the real timekeeper
656	* Separate to spare a full memcpy() of the timekeeper.
657	*/
658	static void tk_update_leap_state_all(struct tk_data *tkd)
659	{
660	write_seqcount_begin(&tkd->seq);
661	tk_update_leap_state(tk: &tkd->shadow_timekeeper);
662	tkd->timekeeper.next_leap_ktime = tkd->shadow_timekeeper.next_leap_ktime;
663	write_seqcount_end(&tkd->seq);
664	}
665
666	/*
667	* Update the ktime_t based scalar nsec members of the timekeeper
668	*/
669	static inline void tk_update_ktime_data(struct timekeeper *tk)
670	{
671	u64 seconds;
672	u32 nsec;
673
674	/*
675	* The xtime based monotonic readout is:
676	* nsec = (xtime_sec + wtm_sec) * 1e9 + wtm_nsec + now();
677	* The ktime based monotonic readout is:
678	* nsec = base_mono + now();
679	* ==> base_mono = (xtime_sec + wtm_sec) * 1e9 + wtm_nsec
680	*/
681	seconds = (u64)(tk->xtime_sec + tk->wall_to_monotonic.tv_sec);
682	nsec = (u32) tk->wall_to_monotonic.tv_nsec;
683	tk->tkr_mono.base = ns_to_ktime(ns: seconds * NSEC_PER_SEC + nsec);
684
685	/*
686	* The sum of the nanoseconds portions of xtime and
687	* wall_to_monotonic can be greater/equal one second. Take
688	* this into account before updating tk->ktime_sec.
689	*/
690	nsec += (u32)(tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift);
691	if (nsec >= NSEC_PER_SEC)
692	seconds++;
693	tk->ktime_sec = seconds;
694
695	/ Update the monotonic raw base /
696	tk->tkr_raw.base = ns_to_ktime(ns: tk->raw_sec * NSEC_PER_SEC);
697	}
698
699	/*
700	* Restore the shadow timekeeper from the real timekeeper.
701	*/
702	static void timekeeping_restore_shadow(struct tk_data *tkd)
703	{
704	lockdep_assert_held(&tkd->lock);
705	memcpy(to: &tkd->shadow_timekeeper, from: &tkd->timekeeper, len: sizeof(tkd->timekeeper));
706	}
707
708	static void timekeeping_update_from_shadow(struct tk_data tkd, unsigned* int action)
709	{
710	struct timekeeper *tk = &tkd->shadow_timekeeper;
711
712	lockdep_assert_held(&tkd->lock);
713
714	/*
715	* Block out readers before running the updates below because that
716	* updates VDSO and other time related infrastructure. Not blocking
717	* the readers might let a reader see time going backwards when
718	* reading from the VDSO after the VDSO update and then reading in
719	* the kernel from the timekeeper before that got updated.
720	*/
721	write_seqcount_begin(&tkd->seq);
722
723	if (action & TK_CLEAR_NTP) {
724	tk->ntp_error = `0`;
725	ntp_clear(tkid: tk->id);
726	}
727
728	tk_update_leap_state(tk);
729	tk_update_ktime_data(tk);
730	tk->tkr_mono.base_real = tk->tkr_mono.base + tk->offs_real;
731
732	if (tk->id == TIMEKEEPER_CORE) {
733	update_vsyscall(tk);
734	update_pvclock_gtod(tk, was_set: action & TK_CLOCK_WAS_SET);
735
736	update_fast_timekeeper(tkr: &tk->tkr_mono, tkf: &tk_fast_mono);
737	update_fast_timekeeper(tkr: &tk->tkr_raw, tkf: &tk_fast_raw);
738	} else if (tk_is_aux(tk)) {
739	vdso_time_update_aux(tk);
740	}
741
742	if (action & TK_CLOCK_WAS_SET)
743	tk->clock_was_set_seq++;
744
745	/*
746	* Update the real timekeeper.
747	*
748	* We could avoid this memcpy() by switching pointers, but that has
749	* the downside that the reader side does not longer benefit from
750	* the cacheline optimized data layout of the timekeeper and requires
751	* another indirection.
752	*/
753	memcpy(to: &tkd->timekeeper, from: tk, len: sizeof(*tk));
754	write_seqcount_end(&tkd->seq);
755	}
756
757	/**
758	* timekeeping_forward_now - update clock to the current time
759	* @tk: Pointer to the timekeeper to update
760	*
761	* Forward the current clock to update its state since the last call to
762	* update_wall_time(). This is useful before significant clock changes,
763	* as it avoids having to deal with this time offset explicitly.
764	*/
765	static void timekeeping_forward_now(struct timekeeper *tk)
766	{
767	u64 cycle_now, delta;
768
769	cycle_now = tk_clock_read(tkr: &tk->tkr_mono);
770	delta = clocksource_delta(now: cycle_now, last: tk->tkr_mono.cycle_last, mask: tk->tkr_mono.mask,
771	max_delta: tk->tkr_mono.clock->max_raw_delta);
772	tk->tkr_mono.cycle_last = cycle_now;
773	tk->tkr_raw.cycle_last = cycle_now;
774
775	while (delta > `0`) {
776	u64 max = tk->tkr_mono.clock->max_cycles;
777	u64 incr = delta < max ? delta : max;
778
779	tk->tkr_mono.xtime_nsec += incr * tk->tkr_mono.mult;
780	tk->tkr_raw.xtime_nsec += incr * tk->tkr_raw.mult;
781	tk_normalize_xtime(tk);
782	delta -= incr;
783	}
784	tk_update_coarse_nsecs(tk);
785	}
786
787	/**
788	* ktime_get_real_ts64 - Returns the time of day in a timespec64.
789	* @ts: pointer to the timespec to be set
790	*
791	* Returns the time of day in a timespec64 (WARN if suspended).
792	*/
793	void ktime_get_real_ts64(struct timespec64 *ts)
794	{
795	struct timekeeper *tk = &tk_core.timekeeper;
796	unsigned int seq;
797	u64 nsecs;
798
799	WARN_ON(timekeeping_suspended);
800
801	do {
802	seq = read_seqcount_begin(&tk_core.seq);
803
804	ts->tv_sec = tk->xtime_sec;
805	nsecs = timekeeping_get_ns(tkr: &tk->tkr_mono);
806
807	} while (read_seqcount_retry(&tk_core.seq, seq));
808
809	ts->tv_nsec = `0`;
810	timespec64_add_ns(a: ts, ns: nsecs);
811	}
812	EXPORT_SYMBOL(ktime_get_real_ts64);
813
814	ktime_t ktime_get(void)
815	{
816	struct timekeeper *tk = &tk_core.timekeeper;
817	unsigned int seq;
818	ktime_t base;
819	u64 nsecs;
820
821	WARN_ON(timekeeping_suspended);
822
823	do {
824	seq = read_seqcount_begin(&tk_core.seq);
825	base = tk->tkr_mono.base;
826	nsecs = timekeeping_get_ns(tkr: &tk->tkr_mono);
827
828	} while (read_seqcount_retry(&tk_core.seq, seq));
829
830	return ktime_add_ns(base, nsecs);
831	}
832	EXPORT_SYMBOL_GPL(ktime_get);
833
834	u32 ktime_get_resolution_ns(void)
835	{
836	struct timekeeper *tk = &tk_core.timekeeper;
837	unsigned int seq;
838	u32 nsecs;
839
840	WARN_ON(timekeeping_suspended);
841
842	do {
843	seq = read_seqcount_begin(&tk_core.seq);
844	nsecs = tk->tkr_mono.mult >> tk->tkr_mono.shift;
845	} while (read_seqcount_retry(&tk_core.seq, seq));
846
847	return nsecs;
848	}
849	EXPORT_SYMBOL_GPL(ktime_get_resolution_ns);
850
851	static ktime_t *offsets[TK_OFFS_MAX] = {
852	[TK_OFFS_REAL] = &tk_core.timekeeper.offs_real,
853	[TK_OFFS_BOOT] = &tk_core.timekeeper.offs_boot,
854	[TK_OFFS_TAI] = &tk_core.timekeeper.offs_tai,
855	};
856
857	ktime_t ktime_get_with_offset(enum tk_offsets offs)
858	{
859	struct timekeeper *tk = &tk_core.timekeeper;
860	unsigned int seq;
861	ktime_t base, *offset = offsets[offs];
862	u64 nsecs;
863
864	WARN_ON(timekeeping_suspended);
865
866	do {
867	seq = read_seqcount_begin(&tk_core.seq);
868	base = ktime_add(tk->tkr_mono.base, *offset);
869	nsecs = timekeeping_get_ns(tkr: &tk->tkr_mono);
870
871	} while (read_seqcount_retry(&tk_core.seq, seq));
872
873	return ktime_add_ns(base, nsecs);
874
875	}
876	EXPORT_SYMBOL_GPL(ktime_get_with_offset);
877
878	ktime_t ktime_get_coarse_with_offset(enum tk_offsets offs)
879	{
880	struct timekeeper *tk = &tk_core.timekeeper;
881	ktime_t base, *offset = offsets[offs];
882	unsigned int seq;
883	u64 nsecs;
884
885	WARN_ON(timekeeping_suspended);
886
887	do {
888	seq = read_seqcount_begin(&tk_core.seq);
889	base = ktime_add(tk->tkr_mono.base, *offset);
890	nsecs = tk->coarse_nsec;
891
892	} while (read_seqcount_retry(&tk_core.seq, seq));
893
894	return ktime_add_ns(base, nsecs);
895	}
896	EXPORT_SYMBOL_GPL(ktime_get_coarse_with_offset);
897
898	/**
899	* ktime_mono_to_any() - convert monotonic time to any other time
900	* @tmono: time to convert.
901	* @offs: which offset to use
902	*/
903	ktime_t ktime_mono_to_any(ktime_t tmono, enum tk_offsets offs)
904	{
905	ktime_t *offset = offsets[offs];
906	unsigned int seq;
907	ktime_t tconv;
908
909	if (IS_ENABLED(CONFIG_64BIT)) {
910	/*
911	* Paired with WRITE_ONCE()s in tk_set_wall_to_mono() and
912	* tk_update_sleep_time().
913	*/
914	return ktime_add(tmono, READ_ONCE(*offset));
915	}
916
917	do {
918	seq = read_seqcount_begin(&tk_core.seq);
919	tconv = ktime_add(tmono, *offset);
920	} while (read_seqcount_retry(&tk_core.seq, seq));
921
922	return tconv;
923	}
924	EXPORT_SYMBOL_GPL(ktime_mono_to_any);
925
926	/**
927	* ktime_get_raw - Returns the raw monotonic time in ktime_t format
928	*/
929	ktime_t ktime_get_raw(void)
930	{
931	struct timekeeper *tk = &tk_core.timekeeper;
932	unsigned int seq;
933	ktime_t base;
934	u64 nsecs;
935
936	do {
937	seq = read_seqcount_begin(&tk_core.seq);
938	base = tk->tkr_raw.base;
939	nsecs = timekeeping_get_ns(tkr: &tk->tkr_raw);
940
941	} while (read_seqcount_retry(&tk_core.seq, seq));
942
943	return ktime_add_ns(base, nsecs);
944	}
945	EXPORT_SYMBOL_GPL(ktime_get_raw);
946
947	/**
948	* ktime_get_ts64 - get the monotonic clock in timespec64 format
949	* @ts: pointer to timespec variable
950	*
951	* The function calculates the monotonic clock from the realtime
952	* clock and the wall_to_monotonic offset and stores the result
953	* in normalized timespec64 format in the variable pointed to by @ts.
954	*/
955	void ktime_get_ts64(struct timespec64 *ts)
956	{
957	struct timekeeper *tk = &tk_core.timekeeper;
958	struct timespec64 tomono;
959	unsigned int seq;
960	u64 nsec;
961
962	WARN_ON(timekeeping_suspended);
963
964	do {
965	seq = read_seqcount_begin(&tk_core.seq);
966	ts->tv_sec = tk->xtime_sec;
967	nsec = timekeeping_get_ns(tkr: &tk->tkr_mono);
968	tomono = tk->wall_to_monotonic;
969
970	} while (read_seqcount_retry(&tk_core.seq, seq));
971
972	ts->tv_sec += tomono.tv_sec;
973	ts->tv_nsec = `0`;
974	timespec64_add_ns(a: ts, ns: nsec + tomono.tv_nsec);
975	}
976	EXPORT_SYMBOL_GPL(ktime_get_ts64);
977
978	/**
979	* ktime_get_seconds - Get the seconds portion of CLOCK_MONOTONIC
980	*
981	* Returns the seconds portion of CLOCK_MONOTONIC with a single non
982	* serialized read. tk->ktime_sec is of type 'unsigned long' so this
983	* works on both 32 and 64 bit systems. On 32 bit systems the readout
984	* covers ~136 years of uptime which should be enough to prevent
985	* premature wrap arounds.
986	*/
987	time64_t ktime_get_seconds(void)
988	{
989	struct timekeeper *tk = &tk_core.timekeeper;
990
991	WARN_ON(timekeeping_suspended);
992	return tk->ktime_sec;
993	}
994	EXPORT_SYMBOL_GPL(ktime_get_seconds);
995
996	/**
997	* ktime_get_real_seconds - Get the seconds portion of CLOCK_REALTIME
998	*
999	* Returns the wall clock seconds since 1970.
1000	*
1001	* For 64bit systems the fast access to tk->xtime_sec is preserved. On
1002	* 32bit systems the access must be protected with the sequence
1003	* counter to provide "atomic" access to the 64bit tk->xtime_sec
1004	* value.
1005	*/
1006	time64_t ktime_get_real_seconds(void)
1007	{
1008	struct timekeeper *tk = &tk_core.timekeeper;
1009	time64_t seconds;
1010	unsigned int seq;
1011
1012	if (IS_ENABLED(CONFIG_64BIT))
1013	return tk->xtime_sec;
1014
1015	do {
1016	seq = read_seqcount_begin(&tk_core.seq);
1017	seconds = tk->xtime_sec;
1018
1019	} while (read_seqcount_retry(&tk_core.seq, seq));
1020
1021	return seconds;
1022	}
1023	EXPORT_SYMBOL_GPL(ktime_get_real_seconds);
1024
1025	/**
1026	* __ktime_get_real_seconds - Unprotected access to CLOCK_REALTIME seconds
1027	*
1028	* The same as ktime_get_real_seconds() but without the sequence counter
1029	* protection. This function is used in restricted contexts like the x86 MCE
1030	* handler and in KGDB. It's unprotected on 32-bit vs. concurrent half
1031	* completed modification and only to be used for such critical contexts.
1032	*
1033	* Returns: Racy snapshot of the CLOCK_REALTIME seconds value
1034	*/
1035	noinstr time64_t __ktime_get_real_seconds(void)
1036	{
1037	struct timekeeper *tk = &tk_core.timekeeper;
1038
1039	return tk->xtime_sec;
1040	}
1041
1042	/**
1043	* ktime_get_snapshot - snapshots the realtime/monotonic raw clocks with counter
1044	* @systime_snapshot: pointer to struct receiving the system time snapshot
1045	*/
1046	void ktime_get_snapshot(struct system_time_snapshot *systime_snapshot)
1047	{
1048	struct timekeeper *tk = &tk_core.timekeeper;
1049	unsigned int seq;
1050	ktime_t base_raw;
1051	ktime_t base_real;
1052	ktime_t base_boot;
1053	u64 nsec_raw;
1054	u64 nsec_real;
1055	u64 now;
1056
1057	WARN_ON_ONCE(timekeeping_suspended);
1058
1059	do {
1060	seq = read_seqcount_begin(&tk_core.seq);
1061	now = tk_clock_read(tkr: &tk->tkr_mono);
1062	systime_snapshot->cs_id = tk->tkr_mono.clock->id;
1063	systime_snapshot->cs_was_changed_seq = tk->cs_was_changed_seq;
1064	systime_snapshot->clock_was_set_seq = tk->clock_was_set_seq;
1065	base_real = ktime_add(tk->tkr_mono.base,
1066	tk_core.timekeeper.offs_real);
1067	base_boot = ktime_add(tk->tkr_mono.base,
1068	tk_core.timekeeper.offs_boot);
1069	base_raw = tk->tkr_raw.base;
1070	nsec_real = timekeeping_cycles_to_ns(tkr: &tk->tkr_mono, cycles: now);
1071	nsec_raw = timekeeping_cycles_to_ns(tkr: &tk->tkr_raw, cycles: now);
1072	} while (read_seqcount_retry(&tk_core.seq, seq));
1073
1074	systime_snapshot->cycles = now;
1075	systime_snapshot->real = ktime_add_ns(base_real, nsec_real);
1076	systime_snapshot->boot = ktime_add_ns(base_boot, nsec_real);
1077	systime_snapshot->raw = ktime_add_ns(base_raw, nsec_raw);
1078	}
1079	EXPORT_SYMBOL_GPL(ktime_get_snapshot);
1080
1081	/ Scale base by mult/div checking for overflow /
1082	static int scale64_check_overflow(u64 mult, u64 div, u64 *base)
1083	{
1084	u64 tmp, rem;
1085
1086	tmp = div64_u64_rem(dividend: *base, divisor: div, remainder: &rem);
1087
1088	if (((int)sizeof(u64)*`8` - fls64(x: mult) < fls64(x: tmp)) \|\|
1089	((int)sizeof(u64)*`8` - fls64(x: mult) < fls64(x: rem)))
1090	return -EOVERFLOW;
1091	tmp *= mult;
1092
1093	rem = div64_u64(dividend: rem * mult, divisor: div);
1094	*base = tmp + rem;
1095	return `0`;
1096	}
1097
1098	/**
1099	* adjust_historical_crosststamp - adjust crosstimestamp previous to current interval
1100	* @history: Snapshot representing start of history
1101	* @partial_history_cycles: Cycle offset into history (fractional part)
1102	* @total_history_cycles: Total history length in cycles
1103	* @discontinuity: True indicates clock was set on history period
1104	* @ts: Cross timestamp that should be adjusted using
1105	* partial/total ratio
1106	*
1107	* Helper function used by get_device_system_crosststamp() to correct the
1108	* crosstimestamp corresponding to the start of the current interval to the
1109	* system counter value (timestamp point) provided by the driver. The
1110	* total_history_* quantities are the total history starting at the provided
1111	* reference point and ending at the start of the current interval. The cycle
1112	* count between the driver timestamp point and the start of the current
1113	* interval is partial_history_cycles.
1114	*/
1115	static int adjust_historical_crosststamp(struct system_time_snapshot *history,
1116	u64 partial_history_cycles,
1117	u64 total_history_cycles,
1118	bool discontinuity,
1119	struct system_device_crosststamp *ts)
1120	{
1121	struct timekeeper *tk = &tk_core.timekeeper;
1122	u64 corr_raw, corr_real;
1123	bool interp_forward;
1124	int ret;
1125
1126	if (total_history_cycles == `0` \|\| partial_history_cycles == `0`)
1127	return `0`;
1128
1129	/ Interpolate shortest distance from beginning or end of history /
1130	interp_forward = partial_history_cycles > total_history_cycles / `2`;
1131	partial_history_cycles = interp_forward ?
1132	total_history_cycles - partial_history_cycles :
1133	partial_history_cycles;
1134
1135	/*
1136	* Scale the monotonic raw time delta by:
1137	* partial_history_cycles / total_history_cycles
1138	*/
1139	corr_raw = (u64)ktime_to_ns(
1140	ktime_sub(ts->sys_monoraw, history->raw));
1141	ret = scale64_check_overflow(mult: partial_history_cycles,
1142	div: total_history_cycles, base: &corr_raw);
1143	if (ret)
1144	return ret;
1145
1146	/*
1147	* If there is a discontinuity in the history, scale monotonic raw
1148	* correction by:
1149	* mult(real)/mult(raw) yielding the realtime correction
1150	* Otherwise, calculate the realtime correction similar to monotonic
1151	* raw calculation
1152	*/
1153	if (discontinuity) {
1154	corr_real = mul_u64_u32_div
1155	(a: corr_raw, mul: tk->tkr_mono.mult, div: tk->tkr_raw.mult);
1156	} else {
1157	corr_real = (u64)ktime_to_ns(
1158	ktime_sub(ts->sys_realtime, history->real));
1159	ret = scale64_check_overflow(mult: partial_history_cycles,
1160	div: total_history_cycles, base: &corr_real);
1161	if (ret)
1162	return ret;
1163	}
1164
1165	/ Fixup monotonic raw and real time time values /
1166	if (interp_forward) {
1167	ts->sys_monoraw = ktime_add_ns(history->raw, corr_raw);
1168	ts->sys_realtime = ktime_add_ns(history->real, corr_real);
1169	} else {
1170	ts->sys_monoraw = ktime_sub_ns(ts->sys_monoraw, corr_raw);
1171	ts->sys_realtime = ktime_sub_ns(ts->sys_realtime, corr_real);
1172	}
1173
1174	return `0`;
1175	}
1176
1177	/*
1178	* timestamp_in_interval - true if ts is chronologically in [start, end]
1179	*
1180	* True if ts occurs chronologically at or after start, and before or at end.
1181	*/
1182	static bool timestamp_in_interval(u64 start, u64 end, u64 ts)
1183	{
1184	if (ts >= start && ts <= end)
1185	return true;
1186	if (start > end && (ts >= start \|\| ts <= end))
1187	return true;
1188	return false;
1189	}
1190
1191	static bool convert_clock(u64 *val, u32 numerator, u32 denominator)
1192	{
1193	u64 rem, res;
1194
1195	if (!numerator \|\| !denominator)
1196	return false;
1197
1198	res = div64_u64_rem(dividend: val, divisor: denominator, remainder: &rem) numerator;
1199	val = res + div_u64(dividend: rem numerator, divisor: denominator);
1200	return true;
1201	}
1202
1203	static bool convert_base_to_cs(struct system_counterval_t *scv)
1204	{
1205	struct clocksource *cs = tk_core.timekeeper.tkr_mono.clock;
1206	struct clocksource_base *base;
1207	u32 num, den;
1208
1209	/ The timestamp was taken from the time keeper clock source /
1210	if (cs->id == scv->cs_id)
1211	return true;
1212
1213	/*
1214	* Check whether cs_id matches the base clock. Prevent the compiler from
1215	* re-evaluating @base as the clocksource might change concurrently.
1216	*/
1217	base = READ_ONCE(cs->base);
1218	if (!base \|\| base->id != scv->cs_id)
1219	return false;
1220
1221	num = scv->use_nsecs ? cs->freq_khz : base->numerator;
1222	den = scv->use_nsecs ? USEC_PER_SEC : base->denominator;
1223
1224	if (!convert_clock(val: &scv->cycles, numerator: num, denominator: den))
1225	return false;
1226
1227	scv->cycles += base->offset;
1228	return true;
1229	}
1230
1231	static bool convert_cs_to_base(u64 cycles, enum* clocksource_ids base_id)
1232	{
1233	struct clocksource *cs = tk_core.timekeeper.tkr_mono.clock;
1234	struct clocksource_base *base;
1235
1236	/*
1237	* Check whether base_id matches the base clock. Prevent the compiler from
1238	* re-evaluating @base as the clocksource might change concurrently.
1239	*/
1240	base = READ_ONCE(cs->base);
1241	if (!base \|\| base->id != base_id)
1242	return false;
1243
1244	*cycles -= base->offset;
1245	if (!convert_clock(val: cycles, numerator: base->denominator, denominator: base->numerator))
1246	return false;
1247	return true;
1248	}
1249
1250	static bool convert_ns_to_cs(u64 *delta)
1251	{
1252	struct tk_read_base *tkr = &tk_core.timekeeper.tkr_mono;
1253
1254	if (BITS_TO_BYTES(fls64(delta) + tkr->shift) >= sizeof(delta))
1255	return false;
1256
1257	delta = div_u64(dividend: (delta << tkr->shift) - tkr->xtime_nsec, divisor: tkr->mult);
1258	return true;
1259	}
1260
1261	/**
1262	* ktime_real_to_base_clock() - Convert CLOCK_REALTIME timestamp to a base clock timestamp
1263	* @treal: CLOCK_REALTIME timestamp to convert
1264	* @base_id: base clocksource id
1265	* @cycles: pointer to store the converted base clock timestamp
1266	*
1267	* Converts a supplied, future realtime clock value to the corresponding base clock value.
1268	*
1269	* Return: true if the conversion is successful, false otherwise.
1270	*/
1271	bool ktime_real_to_base_clock(ktime_t treal, enum clocksource_ids base_id, u64 *cycles)
1272	{
1273	struct timekeeper *tk = &tk_core.timekeeper;
1274	unsigned int seq;
1275	u64 delta;
1276
1277	do {
1278	seq = read_seqcount_begin(&tk_core.seq);
1279	if ((u64)treal < tk->tkr_mono.base_real)
1280	return false;
1281	delta = (u64)treal - tk->tkr_mono.base_real;
1282	if (!convert_ns_to_cs(delta: &delta))
1283	return false;
1284	*cycles = tk->tkr_mono.cycle_last + delta;
1285	if (!convert_cs_to_base(cycles, base_id))
1286	return false;
1287	} while (read_seqcount_retry(&tk_core.seq, seq));
1288
1289	return true;
1290	}
1291	EXPORT_SYMBOL_GPL(ktime_real_to_base_clock);
1292
1293	/**
1294	* get_device_system_crosststamp - Synchronously capture system/device timestamp
1295	* @get_time_fn: Callback to get simultaneous device time and
1296	* system counter from the device driver
1297	* @ctx: Context passed to get_time_fn()
1298	* @history_begin: Historical reference point used to interpolate system
1299	* time when counter provided by the driver is before the current interval
1300	* @xtstamp: Receives simultaneously captured system and device time
1301	*
1302	* Reads a timestamp from a device and correlates it to system time
1303	*/
1304	int get_device_system_crosststamp(int (*get_time_fn)
1305	(ktime_t *device_time,
1306	struct system_counterval_t *sys_counterval,
1307	void *ctx),
1308	void *ctx,
1309	struct system_time_snapshot *history_begin,
1310	struct system_device_crosststamp *xtstamp)
1311	{
1312	struct system_counterval_t system_counterval = {};
1313	struct timekeeper *tk = &tk_core.timekeeper;
1314	u64 cycles, now, interval_start;
1315	unsigned int clock_was_set_seq = `0`;
1316	ktime_t base_real, base_raw;
1317	u64 nsec_real, nsec_raw;
1318	u8 cs_was_changed_seq;
1319	unsigned int seq;
1320	bool do_interp;
1321	int ret;
1322
1323	do {
1324	seq = read_seqcount_begin(&tk_core.seq);
1325	/*
1326	* Try to synchronously capture device time and a system
1327	* counter value calling back into the device driver
1328	*/
1329	ret = get_time_fn(&xtstamp->device, &system_counterval, ctx);
1330	if (ret)
1331	return ret;
1332
1333	/*
1334	* Verify that the clocksource ID associated with the captured
1335	* system counter value is the same as for the currently
1336	* installed timekeeper clocksource
1337	*/
1338	if (system_counterval.cs_id == CSID_GENERIC \|\|
1339	!convert_base_to_cs(scv: &system_counterval))
1340	return -ENODEV;
1341	cycles = system_counterval.cycles;
1342
1343	/*
1344	* Check whether the system counter value provided by the
1345	* device driver is on the current timekeeping interval.
1346	*/
1347	now = tk_clock_read(tkr: &tk->tkr_mono);
1348	interval_start = tk->tkr_mono.cycle_last;
1349	if (!timestamp_in_interval(start: interval_start, end: now, ts: cycles)) {
1350	clock_was_set_seq = tk->clock_was_set_seq;
1351	cs_was_changed_seq = tk->cs_was_changed_seq;
1352	cycles = interval_start;
1353	do_interp = true;
1354	} else {
1355	do_interp = false;
1356	}
1357
1358	base_real = ktime_add(tk->tkr_mono.base,
1359	tk_core.timekeeper.offs_real);
1360	base_raw = tk->tkr_raw.base;
1361
1362	nsec_real = timekeeping_cycles_to_ns(tkr: &tk->tkr_mono, cycles);
1363	nsec_raw = timekeeping_cycles_to_ns(tkr: &tk->tkr_raw, cycles);
1364	} while (read_seqcount_retry(&tk_core.seq, seq));
1365
1366	xtstamp->sys_realtime = ktime_add_ns(base_real, nsec_real);
1367	xtstamp->sys_monoraw = ktime_add_ns(base_raw, nsec_raw);
1368
1369	/*
1370	* Interpolate if necessary, adjusting back from the start of the
1371	* current interval
1372	*/
1373	if (do_interp) {
1374	u64 partial_history_cycles, total_history_cycles;
1375	bool discontinuity;
1376
1377	/*
1378	* Check that the counter value is not before the provided
1379	* history reference and that the history doesn't cross a
1380	* clocksource change
1381	*/
1382	if (!history_begin \|\|
1383	!timestamp_in_interval(start: history_begin->cycles,
1384	end: cycles, ts: system_counterval.cycles) \|\|
1385	history_begin->cs_was_changed_seq != cs_was_changed_seq)
1386	return -EINVAL;
1387	partial_history_cycles = cycles - system_counterval.cycles;
1388	total_history_cycles = cycles - history_begin->cycles;
1389	discontinuity =
1390	history_begin->clock_was_set_seq != clock_was_set_seq;
1391
1392	ret = adjust_historical_crosststamp(history: history_begin,
1393	partial_history_cycles,
1394	total_history_cycles,
1395	discontinuity, ts: xtstamp);
1396	if (ret)
1397	return ret;
1398	}
1399
1400	return `0`;
1401	}
1402	EXPORT_SYMBOL_GPL(get_device_system_crosststamp);
1403
1404	/**
1405	* timekeeping_clocksource_has_base - Check whether the current clocksource
1406	* is based on given a base clock
1407	* @id: base clocksource ID
1408	*
1409	* Note: The return value is a snapshot which can become invalid right
1410	* after the function returns.
1411	*
1412	* Return: true if the timekeeper clocksource has a base clock with @id,
1413	* false otherwise
1414	*/
1415	bool timekeeping_clocksource_has_base(enum clocksource_ids id)
1416	{
1417	/*
1418	* This is a snapshot, so no point in using the sequence
1419	* count. Just prevent the compiler from re-evaluating @base as the
1420	* clocksource might change concurrently.
1421	*/
1422	struct clocksource_base *base = READ_ONCE(tk_core.timekeeper.tkr_mono.clock->base);
1423
1424	return base ? base->id == id : false;
1425	}
1426	EXPORT_SYMBOL_GPL(timekeeping_clocksource_has_base);
1427
1428	/**
1429	* do_settimeofday64 - Sets the time of day.
1430	* @ts: pointer to the timespec64 variable containing the new time
1431	*
1432	* Sets the time of day to the new time and update NTP and notify hrtimers
1433	*/
1434	int do_settimeofday64(const struct timespec64 *ts)
1435	{
1436	struct timespec64 ts_delta, xt;
1437
1438	if (!timespec64_valid_settod(ts))
1439	return -EINVAL;
1440
1441	scoped_guard (raw_spinlock_irqsave, &tk_core.lock) {
1442	struct timekeeper *tks = &tk_core.shadow_timekeeper;
1443
1444	timekeeping_forward_now(tk: tks);
1445
1446	xt = tk_xtime(tk: tks);
1447	ts_delta = timespec64_sub(lhs: *ts, rhs: xt);
1448
1449	if (timespec64_compare(lhs: &tks->wall_to_monotonic, rhs: &ts_delta) > `0`) {
1450	timekeeping_restore_shadow(tkd: &tk_core);
1451	return -EINVAL;
1452	}
1453
1454	tk_set_wall_to_mono(tk: tks, wtm: timespec64_sub(lhs: tks->wall_to_monotonic, rhs: ts_delta));
1455	tk_set_xtime(tk: tks, ts);
1456	timekeeping_update_from_shadow(tkd: &tk_core, TK_UPDATE_ALL);
1457	}
1458
1459	/ Signal hrtimers about time change /
1460	clock_was_set(CLOCK_SET_WALL);
1461
1462	audit_tk_injoffset(offset: ts_delta);
1463	add_device_randomness(buf: ts, len: sizeof(*ts));
1464	return `0`;
1465	}
1466	EXPORT_SYMBOL(do_settimeofday64);
1467
1468	static inline bool timekeeper_is_core_tk(struct timekeeper *tk)
1469	{
1470	return !IS_ENABLED(CONFIG_POSIX_AUX_CLOCKS) \|\| tk->id == TIMEKEEPER_CORE;
1471	}
1472
1473	/**
1474	* __timekeeping_inject_offset - Adds or subtracts from the current time.
1475	* @tkd: Pointer to the timekeeper to modify
1476	* @ts: Pointer to the timespec variable containing the offset
1477	*
1478	* Adds or subtracts an offset value from the current time.
1479	*/
1480	static int __timekeeping_inject_offset(struct tk_data tkd, const* struct timespec64 *ts)
1481	{
1482	struct timekeeper *tks = &tkd->shadow_timekeeper;
1483	struct timespec64 tmp;
1484
1485	if (ts->tv_nsec < `0` \|\| ts->tv_nsec >= NSEC_PER_SEC)
1486	return -EINVAL;
1487
1488	timekeeping_forward_now(tk: tks);
1489
1490	if (timekeeper_is_core_tk(tk: tks)) {
1491	/ Make sure the proposed value is valid /
1492	tmp = timespec64_add(lhs: tk_xtime(tk: tks), rhs: *ts);
1493	if (timespec64_compare(lhs: &tks->wall_to_monotonic, rhs: ts) > `0` \|\|
1494	!timespec64_valid_settod(ts: &tmp)) {
1495	timekeeping_restore_shadow(tkd);
1496	return -EINVAL;
1497	}
1498
1499	tk_xtime_add(tk: tks, ts);
1500	tk_set_wall_to_mono(tk: tks, wtm: timespec64_sub(lhs: tks->wall_to_monotonic, rhs: *ts));
1501	} else {
1502	struct tk_read_base *tkr_mono = &tks->tkr_mono;
1503	ktime_t now, offs;
1504
1505	/ Get the current time /
1506	now = ktime_add_ns(tkr_mono->base, timekeeping_get_ns(tkr_mono));
1507	/ Add the relative offset change /
1508	offs = ktime_add(tks->offs_aux, timespec64_to_ktime(*ts));
1509
1510	/ Prevent that the resulting time becomes negative /
1511	if (ktime_add(now, offs) < `0`) {
1512	timekeeping_restore_shadow(tkd);
1513	return -EINVAL;
1514	}
1515	tk_update_aux_offs(tk: tks, offs);
1516	}
1517
1518	timekeeping_update_from_shadow(tkd, TK_UPDATE_ALL);
1519	return `0`;
1520	}
1521
1522	static int timekeeping_inject_offset(const struct timespec64 *ts)
1523	{
1524	int ret;
1525
1526	scoped_guard (raw_spinlock_irqsave, &tk_core.lock)
1527	ret = __timekeeping_inject_offset(tkd: &tk_core, ts);
1528
1529	/ Signal hrtimers about time change /
1530	if (!ret)
1531	clock_was_set(CLOCK_SET_WALL);
1532	return ret;
1533	}
1534
1535	/*
1536	* Indicates if there is an offset between the system clock and the hardware
1537	* clock/persistent clock/rtc.
1538	*/
1539	int persistent_clock_is_local;
1540
1541	/*
1542	* Adjust the time obtained from the CMOS to be UTC time instead of
1543	* local time.
1544	*
1545	* This is ugly, but preferable to the alternatives. Otherwise we
1546	* would either need to write a program to do it in /etc/rc (and risk
1547	* confusion if the program gets run more than once; it would also be
1548	* hard to make the program warp the clock precisely n hours) or
1549	* compile in the timezone information into the kernel. Bad, bad....
1550	*
1551	* - TYT, 1992-01-01
1552	*
1553	* The best thing to do is to keep the CMOS clock in universal time (UTC)
1554	* as real UNIX machines always do it. This avoids all headaches about
1555	* daylight saving times and warping kernel clocks.
1556	*/
1557	void timekeeping_warp_clock(void)
1558	{
1559	if (sys_tz.tz_minuteswest != `0`) {
1560	struct timespec64 adjust;
1561
1562	persistent_clock_is_local = `1`;
1563	adjust.tv_sec = sys_tz.tz_minuteswest * `60`;
1564	adjust.tv_nsec = `0`;
1565	timekeeping_inject_offset(ts: &adjust);
1566	}
1567	}
1568
1569	/*
1570	* __timekeeping_set_tai_offset - Sets the TAI offset from UTC and monotonic
1571	*/
1572	static void __timekeeping_set_tai_offset(struct timekeeper *tk, s32 tai_offset)
1573	{
1574	tk->tai_offset = tai_offset;
1575	tk->offs_tai = ktime_add(tk->offs_real, ktime_set(tai_offset, `0`));
1576	}
1577
1578	/*
1579	* change_clocksource - Swaps clocksources if a new one is available
1580	*
1581	* Accumulates current time interval and initializes new clocksource
1582	*/
1583	static int change_clocksource(void *data)
1584	{
1585	struct clocksource new = data, old = NULL;
1586
1587	/*
1588	* If the clocksource is in a module, get a module reference.
1589	* Succeeds for built-in code (owner == NULL) as well. Abort if the
1590	* reference can't be acquired.
1591	*/
1592	if (!try_module_get(module: new->owner))
1593	return `0`;
1594
1595	/ Abort if the device can't be enabled /
1596	if (new->enable && new->enable(new) != `0`) {
1597	module_put(module: new->owner);
1598	return `0`;
1599	}
1600
1601	scoped_guard (raw_spinlock_irqsave, &tk_core.lock) {
1602	struct timekeeper *tks = &tk_core.shadow_timekeeper;
1603
1604	timekeeping_forward_now(tk: tks);
1605	old = tks->tkr_mono.clock;
1606	tk_setup_internals(tk: tks, clock: new);
1607	timekeeping_update_from_shadow(tkd: &tk_core, TK_UPDATE_ALL);
1608	}
1609
1610	tk_aux_update_clocksource();
1611
1612	if (old) {
1613	if (old->disable)
1614	old->disable(old);
1615	module_put(module: old->owner);
1616	}
1617
1618	return `0`;
1619	}
1620
1621	/**
1622	* timekeeping_notify - Install a new clock source
1623	* @clock: pointer to the clock source
1624	*
1625	* This function is called from clocksource.c after a new, better clock
1626	* source has been registered. The caller holds the clocksource_mutex.
1627	*/
1628	int timekeeping_notify(struct clocksource *clock)
1629	{
1630	struct timekeeper *tk = &tk_core.timekeeper;
1631
1632	if (tk->tkr_mono.clock == clock)
1633	return `0`;
1634	stop_machine(fn: change_clocksource, data: clock, NULL);
1635	tick_clock_notify();
1636	return tk->tkr_mono.clock == clock ? `0` : -`1`;
1637	}
1638
1639	/**
1640	* ktime_get_raw_ts64 - Returns the raw monotonic time in a timespec
1641	* @ts: pointer to the timespec64 to be set
1642	*
1643	* Returns the raw monotonic time (completely un-modified by ntp)
1644	*/
1645	void ktime_get_raw_ts64(struct timespec64 *ts)
1646	{
1647	struct timekeeper *tk = &tk_core.timekeeper;
1648	unsigned int seq;
1649	u64 nsecs;
1650
1651	do {
1652	seq = read_seqcount_begin(&tk_core.seq);
1653	ts->tv_sec = tk->raw_sec;
1654	nsecs = timekeeping_get_ns(tkr: &tk->tkr_raw);
1655
1656	} while (read_seqcount_retry(&tk_core.seq, seq));
1657
1658	ts->tv_nsec = `0`;
1659	timespec64_add_ns(a: ts, ns: nsecs);
1660	}
1661	EXPORT_SYMBOL(ktime_get_raw_ts64);
1662
1663	/**
1664	* ktime_get_clock_ts64 - Returns time of a clock in a timespec
1665	* @id: POSIX clock ID of the clock to read
1666	* @ts: Pointer to the timespec64 to be set
1667	*
1668	* The timestamp is invalidated (@ts->sec is set to -1) if the
1669	* clock @id is not available.
1670	*/
1671	void ktime_get_clock_ts64(clockid_t id, struct timespec64 *ts)
1672	{
1673	/ Invalidate time stamp /
1674	ts->tv_sec = -`1`;
1675	ts->tv_nsec = `0`;
1676
1677	switch (id) {
1678	case CLOCK_REALTIME:
1679	ktime_get_real_ts64(ts);
1680	return;
1681	case CLOCK_MONOTONIC:
1682	ktime_get_ts64(ts);
1683	return;
1684	case CLOCK_MONOTONIC_RAW:
1685	ktime_get_raw_ts64(ts);
1686	return;
1687	case CLOCK_AUX ... CLOCK_AUX_LAST:
1688	if (IS_ENABLED(CONFIG_POSIX_AUX_CLOCKS))
1689	ktime_get_aux_ts64(id, kt: ts);
1690	return;
1691	default:
1692	WARN_ON_ONCE(`1`);
1693	}
1694	}
1695	EXPORT_SYMBOL_GPL(ktime_get_clock_ts64);
1696
1697	/**
1698	* timekeeping_valid_for_hres - Check if timekeeping is suitable for hres
1699	*/
1700	int timekeeping_valid_for_hres(void)
1701	{
1702	struct timekeeper *tk = &tk_core.timekeeper;
1703	unsigned int seq;
1704	int ret;
1705
1706	do {
1707	seq = read_seqcount_begin(&tk_core.seq);
1708
1709	ret = tk->tkr_mono.clock->flags & CLOCK_SOURCE_VALID_FOR_HRES;
1710
1711	} while (read_seqcount_retry(&tk_core.seq, seq));
1712
1713	return ret;
1714	}
1715
1716	/**
1717	* timekeeping_max_deferment - Returns max time the clocksource can be deferred
1718	*/
1719	u64 timekeeping_max_deferment(void)
1720	{
1721	struct timekeeper *tk = &tk_core.timekeeper;
1722	unsigned int seq;
1723	u64 ret;
1724
1725	do {
1726	seq = read_seqcount_begin(&tk_core.seq);
1727
1728	ret = tk->tkr_mono.clock->max_idle_ns;
1729
1730	} while (read_seqcount_retry(&tk_core.seq, seq));
1731
1732	return ret;
1733	}
1734
1735	/**
1736	* read_persistent_clock64 - Return time from the persistent clock.
1737	* @ts: Pointer to the storage for the readout value
1738	*
1739	* Weak dummy function for arches that do not yet support it.
1740	* Reads the time from the battery backed persistent clock.
1741	* Returns a timespec with tv_sec=0 and tv_nsec=0 if unsupported.
1742	*
1743	* XXX - Do be sure to remove it once all arches implement it.
1744	*/
1745	void __weak read_persistent_clock64(struct timespec64 *ts)
1746	{
1747	ts->tv_sec = `0`;
1748	ts->tv_nsec = `0`;
1749	}
1750
1751	/**
1752	* read_persistent_wall_and_boot_offset - Read persistent clock, and also offset
1753	* from the boot.
1754	* @wall_time: current time as returned by persistent clock
1755	* @boot_offset: offset that is defined as wall_time - boot_time
1756	*
1757	* Weak dummy function for arches that do not yet support it.
1758	*
1759	* The default function calculates offset based on the current value of
1760	* local_clock(). This way architectures that support sched_clock() but don't
1761	* support dedicated boot time clock will provide the best estimate of the
1762	* boot time.
1763	*/
1764	void __weak __init
1765	read_persistent_wall_and_boot_offset(struct timespec64 *wall_time,
1766	struct timespec64 *boot_offset)
1767	{
1768	read_persistent_clock64(ts: wall_time);
1769	*boot_offset = ns_to_timespec64(nsec: local_clock());
1770	}
1771
1772	static __init void tkd_basic_setup(struct tk_data tkd, enum* timekeeper_ids tk_id, bool valid)
1773	{
1774	raw_spin_lock_init(&tkd->lock);
1775	seqcount_raw_spinlock_init(&tkd->seq, &tkd->lock);
1776	tkd->timekeeper.id = tkd->shadow_timekeeper.id = tk_id;
1777	tkd->timekeeper.clock_valid = tkd->shadow_timekeeper.clock_valid = valid;
1778	}
1779
1780	/*
1781	* Flag reflecting whether timekeeping_resume() has injected sleeptime.
1782	*
1783	* The flag starts of false and is only set when a suspend reaches
1784	* timekeeping_suspend(), timekeeping_resume() sets it to false when the
1785	* timekeeper clocksource is not stopping across suspend and has been
1786	* used to update sleep time. If the timekeeper clocksource has stopped
1787	* then the flag stays true and is used by the RTC resume code to decide
1788	* whether sleeptime must be injected and if so the flag gets false then.
1789	*
1790	* If a suspend fails before reaching timekeeping_resume() then the flag
1791	* stays false and prevents erroneous sleeptime injection.
1792	*/
1793	static bool suspend_timing_needed;
1794
1795	/ Flag for if there is a persistent clock on this platform /
1796	static bool persistent_clock_exists;
1797
1798	/*
1799	* timekeeping_init - Initializes the clocksource and common timekeeping values
1800	*/
1801	void __init timekeeping_init(void)
1802	{
1803	struct timespec64 wall_time, boot_offset, wall_to_mono;
1804	struct timekeeper *tks = &tk_core.shadow_timekeeper;
1805	struct clocksource *clock;
1806
1807	tkd_basic_setup(tkd: &tk_core, tk_id: TIMEKEEPER_CORE, valid: true);
1808	tk_aux_setup();
1809
1810	read_persistent_wall_and_boot_offset(wall_time: &wall_time, boot_offset: &boot_offset);
1811	if (timespec64_valid_settod(ts: &wall_time) &&
1812	timespec64_to_ns(ts: &wall_time) > `0`) {
1813	persistent_clock_exists = true;
1814	} else if (timespec64_to_ns(ts: &wall_time) != `0`) {
1815	pr_warn("Persistent clock returned invalid value");
1816	wall_time = (struct timespec64){`0`};
1817	}
1818
1819	if (timespec64_compare(&wall_time, &boot_offset) < `0`)
1820	boot_offset = (struct timespec64){`0`};
1821
1822	/*
1823	* We want set wall_to_mono, so the following is true:
1824	* wall time + wall_to_mono = boot time
1825	*/
1826	wall_to_mono = timespec64_sub(boot_offset, wall_time);
1827
1828	guard(raw_spinlock_irqsave)(&tk_core.lock);
1829
1830	ntp_init();
1831
1832	clock = clocksource_default_clock();
1833	if (clock->enable)
1834	clock->enable(clock);
1835	tk_setup_internals(tks, clock);
1836
1837	tk_set_xtime(tks, &wall_time);
1838	tks->raw_sec = `0`;
1839
1840	tk_set_wall_to_mono(tks, wall_to_mono);
1841
1842	timekeeping_update_from_shadow(&tk_core, TK_CLOCK_WAS_SET);
1843	}
1844
1845	/ time in seconds when suspend began for persistent clock /
1846	static struct timespec64 timekeeping_suspend_time;
1847
1848	/**
1849	* __timekeeping_inject_sleeptime - Internal function to add sleep interval
1850	* @tk: Pointer to the timekeeper to be updated
1851	* @delta: Pointer to the delta value in timespec64 format
1852	*
1853	* Takes a timespec offset measuring a suspend interval and properly
1854	* adds the sleep offset to the timekeeping variables.
1855	*/
1856	static void __timekeeping_inject_sleeptime(struct timekeeper *tk,
1857	const struct timespec64 *delta)
1858	{
1859	if (!timespec64_valid_strict(ts: delta)) {
1860	printk_deferred(KERN_WARNING
1861	"__timekeeping_inject_sleeptime: Invalid "
1862	"sleep delta value!\n");
1863	return;
1864	}
1865	tk_xtime_add(tk, ts: delta);
1866	tk_set_wall_to_mono(tk, wtm: timespec64_sub(lhs: tk->wall_to_monotonic, rhs: *delta));
1867	tk_update_sleep_time(tk, delta: timespec64_to_ktime(ts: *delta));
1868	tk_debug_account_sleep_time(t: delta);
1869	}
1870
1871	#if defined(CONFIG_PM_SLEEP) && defined(CONFIG_RTC_HCTOSYS_DEVICE)
1872	/*
1873	* We have three kinds of time sources to use for sleep time
1874	* injection, the preference order is:
1875	* 1) non-stop clocksource
1876	* 2) persistent clock (ie: RTC accessible when irqs are off)
1877	* 3) RTC
1878	*
1879	* 1) and 2) are used by timekeeping, 3) by RTC subsystem.
1880	* If system has neither 1) nor 2), 3) will be used finally.
1881	*
1882	*
1883	* If timekeeping has injected sleeptime via either 1) or 2),
1884	* 3) becomes needless, so in this case we don't need to call
1885	* rtc_resume(), and this is what timekeeping_rtc_skipresume()
1886	* means.
1887	*/
1888	bool timekeeping_rtc_skipresume(void)
1889	{
1890	return !suspend_timing_needed;
1891	}
1892
1893	/*
1894	* 1) can be determined whether to use or not only when doing
1895	* timekeeping_resume() which is invoked after rtc_suspend(),
1896	* so we can't skip rtc_suspend() surely if system has 1).
1897	*
1898	* But if system has 2), 2) will definitely be used, so in this
1899	* case we don't need to call rtc_suspend(), and this is what
1900	* timekeeping_rtc_skipsuspend() means.
1901	*/
1902	bool timekeeping_rtc_skipsuspend(void)
1903	{
1904	return persistent_clock_exists;
1905	}
1906
1907	/**
1908	* timekeeping_inject_sleeptime64 - Adds suspend interval to timeekeeping values
1909	* @delta: pointer to a timespec64 delta value
1910	*
1911	* This hook is for architectures that cannot support read_persistent_clock64
1912	* because their RTC/persistent clock is only accessible when irqs are enabled.
1913	* and also don't have an effective nonstop clocksource.
1914	*
1915	* This function should only be called by rtc_resume(), and allows
1916	* a suspend offset to be injected into the timekeeping values.
1917	*/
1918	void timekeeping_inject_sleeptime64(const struct timespec64 *delta)
1919	{
1920	scoped_guard(raw_spinlock_irqsave, &tk_core.lock) {
1921	struct timekeeper *tks = &tk_core.shadow_timekeeper;
1922
1923	suspend_timing_needed = false;
1924	timekeeping_forward_now(tks);
1925	__timekeeping_inject_sleeptime(tks, delta);
1926	timekeeping_update_from_shadow(&tk_core, TK_UPDATE_ALL);
1927	}
1928
1929	/ Signal hrtimers about time change /
1930	clock_was_set(CLOCK_SET_WALL \| CLOCK_SET_BOOT);
1931	}
1932	#endif
1933
1934	/**
1935	* timekeeping_resume - Resumes the generic timekeeping subsystem.
1936	*/
1937	void timekeeping_resume(void)
1938	{
1939	struct timekeeper *tks = &tk_core.shadow_timekeeper;
1940	struct clocksource *clock = tks->tkr_mono.clock;
1941	struct timespec64 ts_new, ts_delta;
1942	bool inject_sleeptime = false;
1943	u64 cycle_now, nsec;
1944	unsigned long flags;
1945
1946	read_persistent_clock64(ts: &ts_new);
1947
1948	clockevents_resume();
1949	clocksource_resume();
1950
1951	raw_spin_lock_irqsave(&tk_core.lock, flags);
1952
1953	/*
1954	* After system resumes, we need to calculate the suspended time and
1955	* compensate it for the OS time. There are 3 sources that could be
1956	* used: Nonstop clocksource during suspend, persistent clock and rtc
1957	* device.
1958	*
1959	* One specific platform may have 1 or 2 or all of them, and the
1960	* preference will be:
1961	* suspend-nonstop clocksource -> persistent clock -> rtc
1962	* The less preferred source will only be tried if there is no better
1963	* usable source. The rtc part is handled separately in rtc core code.
1964	*/
1965	cycle_now = tk_clock_read(tkr: &tks->tkr_mono);
1966	nsec = clocksource_stop_suspend_timing(cs: clock, now: cycle_now);
1967	if (nsec > `0`) {
1968	ts_delta = ns_to_timespec64(nsec);
1969	inject_sleeptime = true;
1970	} else if (timespec64_compare(lhs: &ts_new, rhs: &timekeeping_suspend_time) > `0`) {
1971	ts_delta = timespec64_sub(lhs: ts_new, rhs: timekeeping_suspend_time);
1972	inject_sleeptime = true;
1973	}
1974
1975	if (inject_sleeptime) {
1976	suspend_timing_needed = false;
1977	__timekeeping_inject_sleeptime(tk: tks, delta: &ts_delta);
1978	}
1979
1980	/ Re-base the last cycle value /
1981	tks->tkr_mono.cycle_last = cycle_now;
1982	tks->tkr_raw.cycle_last = cycle_now;
1983
1984	tks->ntp_error = `0`;
1985	timekeeping_suspended = `0`;
1986	timekeeping_update_from_shadow(tkd: &tk_core, TK_CLOCK_WAS_SET);
1987	raw_spin_unlock_irqrestore(&tk_core.lock, flags);
1988
1989	touch_softlockup_watchdog();
1990
1991	/ Resume the clockevent device(s) and hrtimers /
1992	tick_resume();
1993	/ Notify timerfd as resume is equivalent to clock_was_set() /
1994	timerfd_resume();
1995	}
1996
1997	int timekeeping_suspend(void)
1998	{
1999	struct timekeeper *tks = &tk_core.shadow_timekeeper;
2000	struct timespec64 delta, delta_delta;
2001	static struct timespec64 old_delta;
2002	struct clocksource *curr_clock;
2003	unsigned long flags;
2004	u64 cycle_now;
2005
2006	read_persistent_clock64(ts: &timekeeping_suspend_time);
2007
2008	/*
2009	* On some systems the persistent_clock can not be detected at
2010	* timekeeping_init by its return value, so if we see a valid
2011	* value returned, update the persistent_clock_exists flag.
2012	*/
2013	if (timekeeping_suspend_time.tv_sec \|\| timekeeping_suspend_time.tv_nsec)
2014	persistent_clock_exists = true;
2015
2016	suspend_timing_needed = true;
2017
2018	raw_spin_lock_irqsave(&tk_core.lock, flags);
2019	timekeeping_forward_now(tk: tks);
2020	timekeeping_suspended = `1`;
2021
2022	/*
2023	* Since we've called forward_now, cycle_last stores the value
2024	* just read from the current clocksource. Save this to potentially
2025	* use in suspend timing.
2026	*/
2027	curr_clock = tks->tkr_mono.clock;
2028	cycle_now = tks->tkr_mono.cycle_last;
2029	clocksource_start_suspend_timing(cs: curr_clock, start_cycles: cycle_now);
2030
2031	if (persistent_clock_exists) {
2032	/*
2033	* To avoid drift caused by repeated suspend/resumes,
2034	* which each can add ~1 second drift error,
2035	* try to compensate so the difference in system time
2036	* and persistent_clock time stays close to constant.
2037	*/
2038	delta = timespec64_sub(lhs: tk_xtime(tk: tks), rhs: timekeeping_suspend_time);
2039	delta_delta = timespec64_sub(lhs: delta, rhs: old_delta);
2040	if (abs(delta_delta.tv_sec) >= `2`) {
2041	/*
2042	* if delta_delta is too large, assume time correction
2043	* has occurred and set old_delta to the current delta.
2044	*/
2045	old_delta = delta;
2046	} else {
2047	/ Otherwise try to adjust old_system to compensate /
2048	timekeeping_suspend_time =
2049	timespec64_add(lhs: timekeeping_suspend_time, rhs: delta_delta);
2050	}
2051	}
2052
2053	timekeeping_update_from_shadow(tkd: &tk_core, action: `0`);
2054	halt_fast_timekeeper(tk: tks);
2055	raw_spin_unlock_irqrestore(&tk_core.lock, flags);
2056
2057	tick_suspend();
2058	clocksource_suspend();
2059	clockevents_suspend();
2060
2061	return `0`;
2062	}
2063
2064	/ sysfs resume/suspend bits for timekeeping /
2065	static struct syscore_ops timekeeping_syscore_ops = {
2066	.resume = timekeeping_resume,
2067	.suspend = timekeeping_suspend,
2068	};
2069
2070	static int __init timekeeping_init_ops(void)
2071	{
2072	register_syscore_ops(ops: &timekeeping_syscore_ops);
2073	return `0`;
2074	}
2075	device_initcall(timekeeping_init_ops);
2076
2077	/*
2078	* Apply a multiplier adjustment to the timekeeper
2079	*/
2080	static __always_inline void timekeeping_apply_adjustment(struct timekeeper *tk,
2081	s64 offset,
2082	s32 mult_adj)
2083	{
2084	s64 interval = tk->cycle_interval;
2085
2086	if (mult_adj == `0`) {
2087	return;
2088	} else if (mult_adj == -`1`) {
2089	interval = -interval;
2090	offset = -offset;
2091	} else if (mult_adj != `1`) {
2092	interval *= mult_adj;
2093	offset *= mult_adj;
2094	}
2095
2096	/*
2097	* So the following can be confusing.
2098	*
2099	* To keep things simple, lets assume mult_adj == 1 for now.
2100	*
2101	* When mult_adj != 1, remember that the interval and offset values
2102	* have been appropriately scaled so the math is the same.
2103	*
2104	* The basic idea here is that we're increasing the multiplier
2105	* by one, this causes the xtime_interval to be incremented by
2106	* one cycle_interval. This is because:
2107	* xtime_interval = cycle_interval * mult
2108	* So if mult is being incremented by one:
2109	* xtime_interval = cycle_interval * (mult + 1)
2110	* Its the same as:
2111	* xtime_interval = (cycle_interval * mult) + cycle_interval
2112	* Which can be shortened to:
2113	* xtime_interval += cycle_interval
2114	*
2115	* So offset stores the non-accumulated cycles. Thus the current
2116	* time (in shifted nanoseconds) is:
2117	* now = (offset * adj) + xtime_nsec
2118	* Now, even though we're adjusting the clock frequency, we have
2119	* to keep time consistent. In other words, we can't jump back
2120	* in time, and we also want to avoid jumping forward in time.
2121	*
2122	* So given the same offset value, we need the time to be the same
2123	* both before and after the freq adjustment.
2124	* now = (offset * adj_1) + xtime_nsec_1
2125	* now = (offset * adj_2) + xtime_nsec_2
2126	* So:
2127	* (offset * adj_1) + xtime_nsec_1 =
2128	* (offset * adj_2) + xtime_nsec_2
2129	* And we know:
2130	* adj_2 = adj_1 + 1
2131	* So:
2132	* (offset * adj_1) + xtime_nsec_1 =
2133	* (offset * (adj_1+1)) + xtime_nsec_2
2134	* (offset * adj_1) + xtime_nsec_1 =
2135	* (offset * adj_1) + offset + xtime_nsec_2
2136	* Canceling the sides:
2137	* xtime_nsec_1 = offset + xtime_nsec_2
2138	* Which gives us:
2139	* xtime_nsec_2 = xtime_nsec_1 - offset
2140	* Which simplifies to:
2141	* xtime_nsec -= offset
2142	*/
2143	if ((mult_adj > `0`) && (tk->tkr_mono.mult + mult_adj < mult_adj)) {
2144	/ NTP adjustment caused clocksource mult overflow /
2145	WARN_ON_ONCE(`1`);
2146	return;
2147	}
2148
2149	tk->tkr_mono.mult += mult_adj;
2150	tk->xtime_interval += interval;
2151	tk->tkr_mono.xtime_nsec -= offset;
2152	}
2153
2154	/*
2155	* Adjust the timekeeper's multiplier to the correct frequency
2156	* and also to reduce the accumulated error value.
2157	*/
2158	static void timekeeping_adjust(struct timekeeper *tk, s64 offset)
2159	{
2160	u64 ntp_tl = ntp_tick_length(tkid: tk->id);
2161	u32 mult;
2162
2163	/*
2164	* Determine the multiplier from the current NTP tick length.
2165	* Avoid expensive division when the tick length doesn't change.
2166	*/
2167	if (likely(tk->ntp_tick == ntp_tl)) {
2168	mult = tk->tkr_mono.mult - tk->ntp_err_mult;
2169	} else {
2170	tk->ntp_tick = ntp_tl;
2171	mult = div64_u64(dividend: (tk->ntp_tick >> tk->ntp_error_shift) -
2172	tk->xtime_remainder, divisor: tk->cycle_interval);
2173	}
2174
2175	/*
2176	* If the clock is behind the NTP time, increase the multiplier by 1
2177	* to catch up with it. If it's ahead and there was a remainder in the
2178	* tick division, the clock will slow down. Otherwise it will stay
2179	* ahead until the tick length changes to a non-divisible value.
2180	*/
2181	tk->ntp_err_mult = tk->ntp_error > `0` ? `1` : `0`;
2182	mult += tk->ntp_err_mult;
2183
2184	timekeeping_apply_adjustment(tk, offset, mult_adj: mult - tk->tkr_mono.mult);
2185
2186	if (unlikely(tk->tkr_mono.clock->maxadj &&
2187	(abs(tk->tkr_mono.mult - tk->tkr_mono.clock->mult)
2188	> tk->tkr_mono.clock->maxadj))) {
2189	printk_once(KERN_WARNING
2190	"Adjusting %s more than 11%% (%ld vs %ld)\n",
2191	tk->tkr_mono.clock->name, (long)tk->tkr_mono.mult,
2192	(long)tk->tkr_mono.clock->mult + tk->tkr_mono.clock->maxadj);
2193	}
2194
2195	/*
2196	* It may be possible that when we entered this function, xtime_nsec
2197	* was very small. Further, if we're slightly speeding the clocksource
2198	* in the code above, its possible the required corrective factor to
2199	* xtime_nsec could cause it to underflow.
2200	*
2201	* Now, since we have already accumulated the second and the NTP
2202	* subsystem has been notified via second_overflow(), we need to skip
2203	* the next update.
2204	*/
2205	if (unlikely((s64)tk->tkr_mono.xtime_nsec < `0`)) {
2206	tk->tkr_mono.xtime_nsec += (u64)NSEC_PER_SEC <<
2207	tk->tkr_mono.shift;
2208	tk->xtime_sec--;
2209	tk->skip_second_overflow = `1`;
2210	}
2211	}
2212
2213	/*
2214	* accumulate_nsecs_to_secs - Accumulates nsecs into secs
2215	*
2216	* Helper function that accumulates the nsecs greater than a second
2217	* from the xtime_nsec field to the xtime_secs field.
2218	* It also calls into the NTP code to handle leapsecond processing.
2219	*/
2220	static inline unsigned int accumulate_nsecs_to_secs(struct timekeeper *tk)
2221	{
2222	u64 nsecps = (u64)NSEC_PER_SEC << tk->tkr_mono.shift;
2223	unsigned int clock_set = `0`;
2224
2225	while (tk->tkr_mono.xtime_nsec >= nsecps) {
2226	int leap;
2227
2228	tk->tkr_mono.xtime_nsec -= nsecps;
2229	tk->xtime_sec++;
2230
2231	/*
2232	* Skip NTP update if this second was accumulated before,
2233	* i.e. xtime_nsec underflowed in timekeeping_adjust()
2234	*/
2235	if (unlikely(tk->skip_second_overflow)) {
2236	tk->skip_second_overflow = `0`;
2237	continue;
2238	}
2239
2240	/ Figure out if its a leap sec and apply if needed /
2241	leap = second_overflow(tkid: tk->id, secs: tk->xtime_sec);
2242	if (unlikely(leap)) {
2243	struct timespec64 ts;
2244
2245	tk->xtime_sec += leap;
2246
2247	ts.tv_sec = leap;
2248	ts.tv_nsec = `0`;
2249	tk_set_wall_to_mono(tk,
2250	wtm: timespec64_sub(lhs: tk->wall_to_monotonic, rhs: ts));
2251
2252	__timekeeping_set_tai_offset(tk, tai_offset: tk->tai_offset - leap);
2253
2254	clock_set = TK_CLOCK_WAS_SET;
2255	}
2256	}
2257	return clock_set;
2258	}
2259
2260	/*
2261	* logarithmic_accumulation - shifted accumulation of cycles
2262	*
2263	* This functions accumulates a shifted interval of cycles into
2264	* a shifted interval nanoseconds. Allows for O(log) accumulation
2265	* loop.
2266	*
2267	* Returns the unconsumed cycles.
2268	*/
2269	static u64 logarithmic_accumulation(struct timekeeper *tk, u64 offset,
2270	u32 shift, unsigned int *clock_set)
2271	{
2272	u64 interval = tk->cycle_interval << shift;
2273	u64 snsec_per_sec;
2274
2275	/ If the offset is smaller than a shifted interval, do nothing /
2276	if (offset < interval)
2277	return offset;
2278
2279	/ Accumulate one shifted interval /
2280	offset -= interval;
2281	tk->tkr_mono.cycle_last += interval;
2282	tk->tkr_raw.cycle_last += interval;
2283
2284	tk->tkr_mono.xtime_nsec += tk->xtime_interval << shift;
2285	*clock_set \|= accumulate_nsecs_to_secs(tk);
2286
2287	/ Accumulate raw time /
2288	tk->tkr_raw.xtime_nsec += tk->raw_interval << shift;
2289	snsec_per_sec = (u64)NSEC_PER_SEC << tk->tkr_raw.shift;
2290	while (tk->tkr_raw.xtime_nsec >= snsec_per_sec) {
2291	tk->tkr_raw.xtime_nsec -= snsec_per_sec;
2292	tk->raw_sec++;
2293	}
2294
2295	/ Accumulate error between NTP and clock interval /
2296	tk->ntp_error += tk->ntp_tick << shift;
2297	tk->ntp_error -= (tk->xtime_interval + tk->xtime_remainder) <<
2298	(tk->ntp_error_shift + shift);
2299
2300	return offset;
2301	}
2302
2303	/*
2304	* timekeeping_advance - Updates the timekeeper to the current time and
2305	* current NTP tick length
2306	*/
2307	static bool __timekeeping_advance(struct tk_data tkd, enum* timekeeping_adv_mode mode)
2308	{
2309	struct timekeeper *tk = &tkd->shadow_timekeeper;
2310	struct timekeeper *real_tk = &tkd->timekeeper;
2311	unsigned int clock_set = `0`;
2312	int shift = `0`, maxshift;
2313	u64 offset, orig_offset;
2314
2315	/ Make sure we're fully resumed: /
2316	if (unlikely(timekeeping_suspended))
2317	return false;
2318
2319	offset = clocksource_delta(now: tk_clock_read(tkr: &tk->tkr_mono),
2320	last: tk->tkr_mono.cycle_last, mask: tk->tkr_mono.mask,
2321	max_delta: tk->tkr_mono.clock->max_raw_delta);
2322	orig_offset = offset;
2323	/ Check if there's really nothing to do /
2324	if (offset < real_tk->cycle_interval && mode == TK_ADV_TICK)
2325	return false;
2326
2327	/*
2328	* With NO_HZ we may have to accumulate many cycle_intervals
2329	* (think "ticks") worth of time at once. To do this efficiently,
2330	* we calculate the largest doubling multiple of cycle_intervals
2331	* that is smaller than the offset. We then accumulate that
2332	* chunk in one go, and then try to consume the next smaller
2333	* doubled multiple.
2334	*/
2335	shift = ilog2(offset) - ilog2(tk->cycle_interval);
2336	shift = max(`0`, shift);
2337	/ Bound shift to one less than what overflows tick_length /
2338	maxshift = (`64` - (ilog2(ntp_tick_length(tk->id)) + `1`)) - `1`;
2339	shift = min(shift, maxshift);
2340	while (offset >= tk->cycle_interval) {
2341	offset = logarithmic_accumulation(tk, offset, shift, clock_set: &clock_set);
2342	if (offset < tk->cycle_interval<<shift)
2343	shift--;
2344	}
2345
2346	/ Adjust the multiplier to correct NTP error /
2347	timekeeping_adjust(tk, offset);
2348
2349	/*
2350	* Finally, make sure that after the rounding
2351	* xtime_nsec isn't larger than NSEC_PER_SEC
2352	*/
2353	clock_set \|= accumulate_nsecs_to_secs(tk);
2354
2355	/*
2356	* To avoid inconsistencies caused adjtimex TK_ADV_FREQ calls
2357	* making small negative adjustments to the base xtime_nsec
2358	* value, only update the coarse clocks if we accumulated time
2359	*/
2360	if (orig_offset != offset)
2361	tk_update_coarse_nsecs(tk);
2362
2363	timekeeping_update_from_shadow(tkd, action: clock_set);
2364
2365	return !!clock_set;
2366	}
2367
2368	static bool timekeeping_advance(enum timekeeping_adv_mode mode)
2369	{
2370	guard(raw_spinlock_irqsave)(l: &tk_core.lock);
2371	return __timekeeping_advance(tkd: &tk_core, mode);
2372	}
2373
2374	/**
2375	* update_wall_time - Uses the current clocksource to increment the wall time
2376	*
2377	* It also updates the enabled auxiliary clock timekeepers
2378	*/
2379	void update_wall_time(void)
2380	{
2381	if (timekeeping_advance(mode: TK_ADV_TICK))
2382	clock_was_set_delayed();
2383	tk_aux_advance();
2384	}
2385
2386	/**
2387	* getboottime64 - Return the real time of system boot.
2388	* @ts: pointer to the timespec64 to be set
2389	*
2390	* Returns the wall-time of boot in a timespec64.
2391	*
2392	* This is based on the wall_to_monotonic offset and the total suspend
2393	* time. Calls to settimeofday will affect the value returned (which
2394	* basically means that however wrong your real time clock is at boot time,
2395	* you get the right time here).
2396	*/
2397	void getboottime64(struct timespec64 *ts)
2398	{
2399	struct timekeeper *tk = &tk_core.timekeeper;
2400	ktime_t t = ktime_sub(tk->offs_real, tk->offs_boot);
2401
2402	*ts = ktime_to_timespec64(t);
2403	}
2404	EXPORT_SYMBOL_GPL(getboottime64);
2405
2406	void ktime_get_coarse_real_ts64(struct timespec64 *ts)
2407	{
2408	struct timekeeper *tk = &tk_core.timekeeper;
2409	unsigned int seq;
2410
2411	do {
2412	seq = read_seqcount_begin(&tk_core.seq);
2413
2414	*ts = tk_xtime_coarse(tk);
2415	} while (read_seqcount_retry(&tk_core.seq, seq));
2416	}
2417	EXPORT_SYMBOL(ktime_get_coarse_real_ts64);
2418
2419	/**
2420	* ktime_get_coarse_real_ts64_mg - return latter of coarse grained time or floor
2421	* @ts: timespec64 to be filled
2422	*
2423	* Fetch the global mg_floor value, convert it to realtime and compare it
2424	* to the current coarse-grained time. Fill @ts with whichever is
2425	* latest. Note that this is a filesystem-specific interface and should be
2426	* avoided outside of that context.
2427	*/
2428	void ktime_get_coarse_real_ts64_mg(struct timespec64 *ts)
2429	{
2430	struct timekeeper *tk = &tk_core.timekeeper;
2431	u64 floor = atomic64_read(v: &mg_floor);
2432	ktime_t f_real, offset, coarse;
2433	unsigned int seq;
2434
2435	do {
2436	seq = read_seqcount_begin(&tk_core.seq);
2437	*ts = tk_xtime_coarse(tk);
2438	offset = tk_core.timekeeper.offs_real;
2439	} while (read_seqcount_retry(&tk_core.seq, seq));
2440
2441	coarse = timespec64_to_ktime(ts: *ts);
2442	f_real = ktime_add(floor, offset);
2443	if (ktime_after(cmp1: f_real, cmp2: coarse))
2444	*ts = ktime_to_timespec64(f_real);
2445	}
2446
2447	/**
2448	* ktime_get_real_ts64_mg - attempt to update floor value and return result
2449	* @ts: pointer to the timespec to be set
2450	*
2451	* Get a monotonic fine-grained time value and attempt to swap it into
2452	* mg_floor. If that succeeds then accept the new floor value. If it fails
2453	* then another task raced in during the interim time and updated the
2454	* floor. Since any update to the floor must be later than the previous
2455	* floor, either outcome is acceptable.
2456	*
2457	* Typically this will be called after calling ktime_get_coarse_real_ts64_mg(),
2458	* and determining that the resulting coarse-grained timestamp did not effect
2459	* a change in ctime. Any more recent floor value would effect a change to
2460	* ctime, so there is no need to retry the atomic64_try_cmpxchg() on failure.
2461	*
2462	* @ts will be filled with the latest floor value, regardless of the outcome of
2463	* the cmpxchg. Note that this is a filesystem specific interface and should be
2464	* avoided outside of that context.
2465	*/
2466	void ktime_get_real_ts64_mg(struct timespec64 *ts)
2467	{
2468	struct timekeeper *tk = &tk_core.timekeeper;
2469	ktime_t old = atomic64_read(v: &mg_floor);
2470	ktime_t offset, mono;
2471	unsigned int seq;
2472	u64 nsecs;
2473
2474	do {
2475	seq = read_seqcount_begin(&tk_core.seq);
2476
2477	ts->tv_sec = tk->xtime_sec;
2478	mono = tk->tkr_mono.base;
2479	nsecs = timekeeping_get_ns(tkr: &tk->tkr_mono);
2480	offset = tk_core.timekeeper.offs_real;
2481	} while (read_seqcount_retry(&tk_core.seq, seq));
2482
2483	mono = ktime_add_ns(mono, nsecs);
2484
2485	/*
2486	* Attempt to update the floor with the new time value. As any
2487	* update must be later then the existing floor, and would effect
2488	* a change to ctime from the perspective of the current task,
2489	* accept the resulting floor value regardless of the outcome of
2490	* the swap.
2491	*/
2492	if (atomic64_try_cmpxchg(v: &mg_floor, old: &old, new: mono)) {
2493	ts->tv_nsec = `0`;
2494	timespec64_add_ns(a: ts, ns: nsecs);
2495	timekeeping_inc_mg_floor_swaps();
2496	} else {
2497	/*
2498	* Another task changed mg_floor since "old" was fetched.
2499	* "old" has been updated with the latest value of "mg_floor".
2500	* That value is newer than the previous floor value, which
2501	* is enough to effect a change to ctime. Accept it.
2502	*/
2503	*ts = ktime_to_timespec64(ktime_add(old, offset));
2504	}
2505	}
2506
2507	void ktime_get_coarse_ts64(struct timespec64 *ts)
2508	{
2509	struct timekeeper *tk = &tk_core.timekeeper;
2510	struct timespec64 now, mono;
2511	unsigned int seq;
2512
2513	do {
2514	seq = read_seqcount_begin(&tk_core.seq);
2515
2516	now = tk_xtime_coarse(tk);
2517	mono = tk->wall_to_monotonic;
2518	} while (read_seqcount_retry(&tk_core.seq, seq));
2519
2520	set_normalized_timespec64(ts, sec: now.tv_sec + mono.tv_sec,
2521	nsec: now.tv_nsec + mono.tv_nsec);
2522	}
2523	EXPORT_SYMBOL(ktime_get_coarse_ts64);
2524
2525	/*
2526	* Must hold jiffies_lock
2527	*/
2528	void do_timer(unsigned long ticks)
2529	{
2530	jiffies_64 += ticks;
2531	calc_global_load();
2532	}
2533
2534	/**
2535	* ktime_get_update_offsets_now - hrtimer helper
2536	* @cwsseq: pointer to check and store the clock was set sequence number
2537	* @offs_real: pointer to storage for monotonic -> realtime offset
2538	* @offs_boot: pointer to storage for monotonic -> boottime offset
2539	* @offs_tai: pointer to storage for monotonic -> clock tai offset
2540	*
2541	* Returns current monotonic time and updates the offsets if the
2542	* sequence number in @cwsseq and timekeeper.clock_was_set_seq are
2543	* different.
2544	*
2545	* Called from hrtimer_interrupt() or retrigger_next_event()
2546	*/
2547	ktime_t ktime_get_update_offsets_now(unsigned int cwsseq, ktime_t offs_real,
2548	ktime_t offs_boot, ktime_t offs_tai)
2549	{
2550	struct timekeeper *tk = &tk_core.timekeeper;
2551	unsigned int seq;
2552	ktime_t base;
2553	u64 nsecs;
2554
2555	do {
2556	seq = read_seqcount_begin(&tk_core.seq);
2557
2558	base = tk->tkr_mono.base;
2559	nsecs = timekeeping_get_ns(tkr: &tk->tkr_mono);
2560	base = ktime_add_ns(base, nsecs);
2561
2562	if (*cwsseq != tk->clock_was_set_seq) {
2563	*cwsseq = tk->clock_was_set_seq;
2564	*offs_real = tk->offs_real;
2565	*offs_boot = tk->offs_boot;
2566	*offs_tai = tk->offs_tai;
2567	}
2568
2569	/ Handle leapsecond insertion adjustments /
2570	if (unlikely(base >= tk->next_leap_ktime))
2571	*offs_real = ktime_sub(tk->offs_real, ktime_set(`1`, `0`));
2572
2573	} while (read_seqcount_retry(&tk_core.seq, seq));
2574
2575	return base;
2576	}
2577
2578	/*
2579	* timekeeping_validate_timex - Ensures the timex is ok for use in do_adjtimex
2580	*/
2581	static int timekeeping_validate_timex(const struct __kernel_timex *txc, bool aux_clock)
2582	{
2583	if (txc->modes & ADJ_ADJTIME) {
2584	/ singleshot must not be used with any other mode bits /
2585	if (!(txc->modes & ADJ_OFFSET_SINGLESHOT))
2586	return -EINVAL;
2587	if (!(txc->modes & ADJ_OFFSET_READONLY) &&
2588	!capable(CAP_SYS_TIME))
2589	return -EPERM;
2590	} else {
2591	/ In order to modify anything, you gotta be super-user! /
2592	if (txc->modes && !capable(CAP_SYS_TIME))
2593	return -EPERM;
2594	/*
2595	* if the quartz is off by more than 10% then
2596	* something is VERY wrong!
2597	*/
2598	if (txc->modes & ADJ_TICK &&
2599	(txc->tick < `900000`/USER_HZ \|\|
2600	txc->tick > `1100000`/USER_HZ))
2601	return -EINVAL;
2602	}
2603
2604	if (txc->modes & ADJ_SETOFFSET) {
2605	/ In order to inject time, you gotta be super-user! /
2606	if (!capable(CAP_SYS_TIME))
2607	return -EPERM;
2608
2609	/*
2610	* Validate if a timespec/timeval used to inject a time
2611	* offset is valid. Offsets can be positive or negative, so
2612	* we don't check tv_sec. The value of the timeval/timespec
2613	* is the sum of its fields,but NOTE:
2614	* The field tv_usec/tv_nsec must always be non-negative and
2615	* we can't have more nanoseconds/microseconds than a second.
2616	*/
2617	if (txc->time.tv_usec < `0`)
2618	return -EINVAL;
2619
2620	if (txc->modes & ADJ_NANO) {
2621	if (txc->time.tv_usec >= NSEC_PER_SEC)
2622	return -EINVAL;
2623	} else {
2624	if (txc->time.tv_usec >= USEC_PER_SEC)
2625	return -EINVAL;
2626	}
2627	}
2628
2629	/*
2630	* Check for potential multiplication overflows that can
2631	* only happen on 64-bit systems:
2632	*/
2633	if ((txc->modes & ADJ_FREQUENCY) && (BITS_PER_LONG == `64`)) {
2634	if (LLONG_MIN / PPM_SCALE > txc->freq)
2635	return -EINVAL;
2636	if (LLONG_MAX / PPM_SCALE < txc->freq)
2637	return -EINVAL;
2638	}
2639
2640	if (aux_clock) {
2641	/ Auxiliary clocks are similar to TAI and do not have leap seconds /
2642	if (txc->status & (STA_INS \| STA_DEL))
2643	return -EINVAL;
2644
2645	/ No TAI offset setting /
2646	if (txc->modes & ADJ_TAI)
2647	return -EINVAL;
2648
2649	/ No PPS support either /
2650	if (txc->status & (STA_PPSFREQ \| STA_PPSTIME))
2651	return -EINVAL;
2652	}
2653
2654	return `0`;
2655	}
2656
2657	/**
2658	* random_get_entropy_fallback - Returns the raw clock source value,
2659	* used by random.c for platforms with no valid random_get_entropy().
2660	*/
2661	unsigned long random_get_entropy_fallback(void)
2662	{
2663	struct tk_read_base *tkr = &tk_core.timekeeper.tkr_mono;
2664	struct clocksource *clock = READ_ONCE(tkr->clock);
2665
2666	if (unlikely(timekeeping_suspended \|\| !clock))
2667	return `0`;
2668	return clock->read(clock);
2669	}
2670	EXPORT_SYMBOL_GPL(random_get_entropy_fallback);
2671
2672	struct adjtimex_result {
2673	struct audit_ntp_data ad;
2674	struct timespec64 delta;
2675	bool clock_set;
2676	};
2677
2678	static int __do_adjtimex(struct tk_data tkd, struct* __kernel_timex *txc,
2679	struct adjtimex_result *result)
2680	{
2681	struct timekeeper *tks = &tkd->shadow_timekeeper;
2682	bool aux_clock = !timekeeper_is_core_tk(tk: tks);
2683	struct timespec64 ts;
2684	s32 orig_tai, tai;
2685	int ret;
2686
2687	/ Validate the data before disabling interrupts /
2688	ret = timekeeping_validate_timex(txc, aux_clock);
2689	if (ret)
2690	return ret;
2691	add_device_randomness(buf: txc, len: sizeof(*txc));
2692
2693	if (!aux_clock)
2694	ktime_get_real_ts64(&ts);
2695	else
2696	tk_get_aux_ts64(tkid: tkd->timekeeper.id, ts: &ts);
2697
2698	add_device_randomness(buf: &ts, len: sizeof(ts));
2699
2700	guard(raw_spinlock_irqsave)(l: &tkd->lock);
2701
2702	if (!tks->clock_valid)
2703	return -ENODEV;
2704
2705	if (txc->modes & ADJ_SETOFFSET) {
2706	result->delta.tv_sec = txc->time.tv_sec;
2707	result->delta.tv_nsec = txc->time.tv_usec;
2708	if (!(txc->modes & ADJ_NANO))
2709	result->delta.tv_nsec *= `1000`;
2710	ret = __timekeeping_inject_offset(tkd, ts: &result->delta);
2711	if (ret)
2712	return ret;
2713	result->clock_set = true;
2714	}
2715
2716	orig_tai = tai = tks->tai_offset;
2717	ret = ntp_adjtimex(tkid: tks->id, txc, ts: &ts, time_tai: &tai, ad: &result->ad);
2718
2719	if (tai != orig_tai) {
2720	__timekeeping_set_tai_offset(tk: tks, tai_offset: tai);
2721	timekeeping_update_from_shadow(tkd, TK_CLOCK_WAS_SET);
2722	result->clock_set = true;
2723	} else {
2724	tk_update_leap_state_all(tkd: &tk_core);
2725	}
2726
2727	/ Update the multiplier immediately if frequency was set directly /
2728	if (txc->modes & (ADJ_FREQUENCY \| ADJ_TICK))
2729	result->clock_set \|= __timekeeping_advance(tkd, mode: TK_ADV_FREQ);
2730
2731	return ret;
2732	}
2733
2734	/**
2735	* do_adjtimex() - Accessor function to NTP __do_adjtimex function
2736	* @txc: Pointer to kernel_timex structure containing NTP parameters
2737	*/
2738	int do_adjtimex(struct __kernel_timex *txc)
2739	{
2740	struct adjtimex_result result = { };
2741	int ret;
2742
2743	ret = __do_adjtimex(tkd: &tk_core, txc, result: &result);
2744	if (ret < `0`)
2745	return ret;
2746
2747	if (txc->modes & ADJ_SETOFFSET)
2748	audit_tk_injoffset(offset: result.delta);
2749
2750	audit_ntp_log(ad: &result.ad);
2751
2752	if (result.clock_set)
2753	clock_was_set(CLOCK_SET_WALL);
2754
2755	ntp_notify_cmos_timer(offset_set: result.delta.tv_sec != `0`);
2756
2757	return ret;
2758	}
2759
2760	/*
2761	* Invoked from NTP with the time keeper lock held, so lockless access is
2762	* fine.
2763	*/
2764	long ktime_get_ntp_seconds(unsigned int id)
2765	{
2766	return timekeeper_data[id].timekeeper.xtime_sec;
2767	}
2768
2769	#ifdef CONFIG_NTP_PPS
2770	/**
2771	* hardpps() - Accessor function to NTP __hardpps function
2772	* @phase_ts: Pointer to timespec64 structure representing phase timestamp
2773	* @raw_ts: Pointer to timespec64 structure representing raw timestamp
2774	*/
2775	void hardpps(const struct timespec64 phase_ts, const* struct timespec64 *raw_ts)
2776	{
2777	guard(raw_spinlock_irqsave)(&tk_core.lock);
2778	__hardpps(phase_ts, raw_ts);
2779	}
2780	EXPORT_SYMBOL(hardpps);
2781	#endif /* CONFIG_NTP_PPS */
2782
2783	#ifdef CONFIG_POSIX_AUX_CLOCKS
2784	#include "posix-timers.h"
2785
2786	/*
2787	* Bitmap for the activated auxiliary timekeepers to allow lockless quick
2788	* checks in the hot paths without touching extra cache lines. If set, then
2789	* the state of the corresponding timekeeper has to be re-checked under
2790	* timekeeper::lock.
2791	*/
2792	static unsigned long aux_timekeepers;
2793
2794	static inline unsigned int clockid_to_tkid(unsigned int id)
2795	{
2796	return TIMEKEEPER_AUX_FIRST + id - CLOCK_AUX;
2797	}
2798
2799	static inline struct tk_data *aux_get_tk_data(clockid_t id)
2800	{
2801	if (!clockid_aux_valid(id))
2802	return NULL;
2803	return &timekeeper_data[clockid_to_tkid(id)];
2804	}
2805
2806	/ Invoked from timekeeping after a clocksource change /
2807	static void tk_aux_update_clocksource(void)
2808	{
2809	unsigned long active = READ_ONCE(aux_timekeepers);
2810	unsigned int id;
2811
2812	for_each_set_bit(id, &active, BITS_PER_LONG) {
2813	struct tk_data *tkd = &timekeeper_data[id + TIMEKEEPER_AUX_FIRST];
2814	struct timekeeper *tks = &tkd->shadow_timekeeper;
2815
2816	guard(raw_spinlock_irqsave)(&tkd->lock);
2817	if (!tks->clock_valid)
2818	continue;
2819
2820	timekeeping_forward_now(tks);
2821	tk_setup_internals(tks, tk_core.timekeeper.tkr_mono.clock);
2822	timekeeping_update_from_shadow(tkd, TK_UPDATE_ALL);
2823	}
2824	}
2825
2826	static void tk_aux_advance(void)
2827	{
2828	unsigned long active = READ_ONCE(aux_timekeepers);
2829	unsigned int id;
2830
2831	/ Lockless quick check to avoid extra cache lines /
2832	for_each_set_bit(id, &active, BITS_PER_LONG) {
2833	struct tk_data *aux_tkd = &timekeeper_data[id + TIMEKEEPER_AUX_FIRST];
2834
2835	guard(raw_spinlock)(&aux_tkd->lock);
2836	if (aux_tkd->shadow_timekeeper.clock_valid)
2837	__timekeeping_advance(aux_tkd, TK_ADV_TICK);
2838	}
2839	}
2840
2841	/**
2842	* ktime_get_aux - Get time for a AUX clock
2843	* @id: ID of the clock to read (CLOCK_AUX...)
2844	* @kt: Pointer to ktime_t to store the time stamp
2845	*
2846	* Returns: True if the timestamp is valid, false otherwise
2847	*/
2848	bool ktime_get_aux(clockid_t id, ktime_t *kt)
2849	{
2850	struct tk_data *aux_tkd = aux_get_tk_data(id);
2851	struct timekeeper *aux_tk;
2852	unsigned int seq;
2853	ktime_t base;
2854	u64 nsecs;
2855
2856	WARN_ON(timekeeping_suspended);
2857
2858	if (!aux_tkd)
2859	return false;
2860
2861	aux_tk = &aux_tkd->timekeeper;
2862	do {
2863	seq = read_seqcount_begin(&aux_tkd->seq);
2864	if (!aux_tk->clock_valid)
2865	return false;
2866
2867	base = ktime_add(aux_tk->tkr_mono.base, aux_tk->offs_aux);
2868	nsecs = timekeeping_get_ns(&aux_tk->tkr_mono);
2869	} while (read_seqcount_retry(&aux_tkd->seq, seq));
2870
2871	*kt = ktime_add_ns(base, nsecs);
2872	return true;
2873	}
2874	EXPORT_SYMBOL_GPL(ktime_get_aux);
2875
2876	/**
2877	* ktime_get_aux_ts64 - Get time for a AUX clock
2878	* @id: ID of the clock to read (CLOCK_AUX...)
2879	* @ts: Pointer to timespec64 to store the time stamp
2880	*
2881	* Returns: True if the timestamp is valid, false otherwise
2882	*/
2883	bool ktime_get_aux_ts64(clockid_t id, struct timespec64 *ts)
2884	{
2885	ktime_t now;
2886
2887	if (!ktime_get_aux(id, &now))
2888	return false;
2889	*ts = ktime_to_timespec64(now);
2890	return true;
2891	}
2892	EXPORT_SYMBOL_GPL(ktime_get_aux_ts64);
2893
2894	static int aux_get_res(clockid_t id, struct timespec64 *tp)
2895	{
2896	if (!clockid_aux_valid(id))
2897	return -ENODEV;
2898
2899	tp->tv_sec = aux_clock_resolution_ns() / NSEC_PER_SEC;
2900	tp->tv_nsec = aux_clock_resolution_ns() % NSEC_PER_SEC;
2901	return `0`;
2902	}
2903
2904	static int aux_get_timespec(clockid_t id, struct timespec64 *tp)
2905	{
2906	return ktime_get_aux_ts64(id, tp) ? `0` : -ENODEV;
2907	}
2908
2909	static int aux_clock_set(const clockid_t id, const struct timespec64 *tnew)
2910	{
2911	struct tk_data *aux_tkd = aux_get_tk_data(id);
2912	struct timekeeper *aux_tks;
2913	ktime_t tnow, nsecs;
2914
2915	if (!timespec64_valid_settod(tnew))
2916	return -EINVAL;
2917	if (!aux_tkd)
2918	return -ENODEV;
2919
2920	aux_tks = &aux_tkd->shadow_timekeeper;
2921
2922	guard(raw_spinlock_irq)(&aux_tkd->lock);
2923	if (!aux_tks->clock_valid)
2924	return -ENODEV;
2925
2926	/ Forward the timekeeper base time /
2927	timekeeping_forward_now(aux_tks);
2928	/*
2929	* Get the updated base time. tkr_mono.base has not been
2930	* updated yet, so do that first. That makes the update
2931	* in timekeeping_update_from_shadow() redundant, but
2932	* that's harmless. After that @tnow can be calculated
2933	* by using tkr_mono::cycle_last, which has been set
2934	* by timekeeping_forward_now().
2935	*/
2936	tk_update_ktime_data(aux_tks);
2937	nsecs = timekeeping_cycles_to_ns(&aux_tks->tkr_mono, aux_tks->tkr_mono.cycle_last);
2938	tnow = ktime_add(aux_tks->tkr_mono.base, nsecs);
2939
2940	/*
2941	* Calculate the new AUX offset as delta to @tnow ("monotonic").
2942	* That avoids all the tk::xtime back and forth conversions as
2943	* xtime ("realtime") is not applicable for auxiliary clocks and
2944	* kept in sync with "monotonic".
2945	*/
2946	tk_update_aux_offs(aux_tks, ktime_sub(timespec64_to_ktime(*tnew), tnow));
2947
2948	timekeeping_update_from_shadow(aux_tkd, TK_UPDATE_ALL);
2949	return `0`;
2950	}
2951
2952	static int aux_clock_adj(const clockid_t id, struct __kernel_timex *txc)
2953	{
2954	struct tk_data *aux_tkd = aux_get_tk_data(id);
2955	struct adjtimex_result result = { };
2956
2957	if (!aux_tkd)
2958	return -ENODEV;
2959
2960	/*
2961	* @result is ignored for now as there are neither hrtimers nor a
2962	* RTC related to auxiliary clocks for now.
2963	*/
2964	return __do_adjtimex(aux_tkd, txc, &result);
2965	}
2966
2967	const struct k_clock clock_aux = {
2968	.clock_getres = aux_get_res,
2969	.clock_get_timespec = aux_get_timespec,
2970	.clock_set = aux_clock_set,
2971	.clock_adj = aux_clock_adj,
2972	};
2973
2974	static void aux_clock_enable(clockid_t id)
2975	{
2976	struct tk_read_base *tkr_raw = &tk_core.timekeeper.tkr_raw;
2977	struct tk_data *aux_tkd = aux_get_tk_data(id);
2978	struct timekeeper *aux_tks = &aux_tkd->shadow_timekeeper;
2979
2980	/ Prevent the core timekeeper from changing. /
2981	guard(raw_spinlock_irq)(&tk_core.lock);
2982
2983	/*
2984	* Setup the auxiliary clock assuming that the raw core timekeeper
2985	* clock frequency conversion is close enough. Userspace has to
2986	* adjust for the deviation via clock_adjtime(2).
2987	*/
2988	guard(raw_spinlock_nested)(&aux_tkd->lock);
2989
2990	/ Remove leftovers of a previous registration /
2991	memset(aux_tks, `0`, sizeof(*aux_tks));
2992	/ Restore the timekeeper id /
2993	aux_tks->id = aux_tkd->timekeeper.id;
2994	/ Setup the timekeeper based on the current system clocksource /
2995	tk_setup_internals(aux_tks, tkr_raw->clock);
2996
2997	/ Mark it valid and set it live /
2998	aux_tks->clock_valid = true;
2999	timekeeping_update_from_shadow(aux_tkd, TK_UPDATE_ALL);
3000	}
3001
3002	static void aux_clock_disable(clockid_t id)
3003	{
3004	struct tk_data *aux_tkd = aux_get_tk_data(id);
3005
3006	guard(raw_spinlock_irq)(&aux_tkd->lock);
3007	aux_tkd->shadow_timekeeper.clock_valid = false;
3008	timekeeping_update_from_shadow(aux_tkd, TK_UPDATE_ALL);
3009	}
3010
3011	static DEFINE_MUTEX(aux_clock_mutex);
3012
3013	static ssize_t aux_clock_enable_store(struct kobject kobj, struct* kobj_attribute *attr,
3014	const char *buf, size_t count)
3015	{
3016	/ Lazy atoi() as name is "0..7" /
3017	int id = kobj->name[`0`] & `0x7`;
3018	bool enable;
3019
3020	if (!capable(CAP_SYS_TIME))
3021	return -EPERM;
3022
3023	if (kstrtobool(buf, &enable) < `0`)
3024	return -EINVAL;
3025
3026	guard(mutex)(&aux_clock_mutex);
3027	if (enable == test_bit(id, &aux_timekeepers))
3028	return count;
3029
3030	if (enable) {
3031	aux_clock_enable(CLOCK_AUX + id);
3032	set_bit(id, &aux_timekeepers);
3033	} else {
3034	aux_clock_disable(CLOCK_AUX + id);
3035	clear_bit(id, &aux_timekeepers);
3036	}
3037	return count;
3038	}
3039
3040	static ssize_t aux_clock_enable_show(struct kobject kobj, struct* kobj_attribute attr, char* *buf)
3041	{
3042	unsigned long active = READ_ONCE(aux_timekeepers);
3043	/ Lazy atoi() as name is "0..7" /
3044	int id = kobj->name[`0`] & `0x7`;
3045
3046	return sysfs_emit(buf, "%d\n", test_bit(id, &active));
3047	}
3048
3049	static struct kobj_attribute aux_clock_enable_attr = __ATTR_RW(aux_clock_enable);
3050
3051	static struct attribute *aux_clock_enable_attrs[] = {
3052	&aux_clock_enable_attr.attr,
3053	NULL
3054	};
3055
3056	static const struct attribute_group aux_clock_enable_attr_group = {
3057	.attrs = aux_clock_enable_attrs,
3058	};
3059
3060	static int __init tk_aux_sysfs_init(void)
3061	{
3062	struct kobject auxo, tko = kobject_create_and_add("time", kernel_kobj);
3063
3064	if (!tko)
3065	return -ENOMEM;
3066
3067	auxo = kobject_create_and_add("aux_clocks", tko);
3068	if (!auxo) {
3069	kobject_put(tko);
3070	return -ENOMEM;
3071	}
3072
3073	for (int i = `0`; i <= MAX_AUX_CLOCKS; i++) {
3074	char id[`2`] = { [`0`] = `'0'` + i, };
3075	struct kobject *clk = kobject_create_and_add(id, auxo);
3076
3077	if (!clk)
3078	return -ENOMEM;
3079
3080	int ret = sysfs_create_group(clk, &aux_clock_enable_attr_group);
3081
3082	if (ret)
3083	return ret;
3084	}
3085	return `0`;
3086	}
3087	late_initcall(tk_aux_sysfs_init);
3088
3089	static __init void tk_aux_setup(void)
3090	{
3091	for (int i = TIMEKEEPER_AUX_FIRST; i <= TIMEKEEPER_AUX_LAST; i++)
3092	tkd_basic_setup(&timekeeper_data[i], i, false);
3093	}
3094	#endif /* CONFIG_POSIX_AUX_CLOCKS */
3095

Browse the source code of Linux/kernel/time/timekeeping.c