1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright 2019 ARM Ltd.
4 *
5 * Generic implementation of update_vsyscall and update_vsyscall_tz.
6 *
7 * Based on the x86 specific implementation.
8 */
9
10#include <linux/hrtimer.h>
11#include <linux/timekeeper_internal.h>
12#include <vdso/datapage.h>
13#include <vdso/helpers.h>
14#include <vdso/vsyscall.h>
15
16#include "timekeeping_internal.h"
17
18static inline void fill_clock_configuration(struct vdso_clock *vc, const struct tk_read_base *base)
19{
20 vc->cycle_last = base->cycle_last;
21#ifdef CONFIG_GENERIC_VDSO_OVERFLOW_PROTECT
22 vc->max_cycles = base->clock->max_cycles;
23#endif
24 vc->mask = base->mask;
25 vc->mult = base->mult;
26 vc->shift = base->shift;
27}
28
29static inline void update_vdso_time_data(struct vdso_time_data *vdata, struct timekeeper *tk)
30{
31 struct vdso_clock *vc = vdata->clock_data;
32 struct vdso_timestamp *vdso_ts;
33 u64 nsec, sec;
34
35 fill_clock_configuration(vc: &vc[CS_HRES_COARSE], base: &tk->tkr_mono);
36 fill_clock_configuration(vc: &vc[CS_RAW], base: &tk->tkr_raw);
37
38 /* CLOCK_MONOTONIC */
39 vdso_ts = &vc[CS_HRES_COARSE].basetime[CLOCK_MONOTONIC];
40 vdso_ts->sec = tk->xtime_sec + tk->wall_to_monotonic.tv_sec;
41
42 nsec = tk->tkr_mono.xtime_nsec;
43 nsec += ((u64)tk->wall_to_monotonic.tv_nsec << tk->tkr_mono.shift);
44 while (nsec >= (((u64)NSEC_PER_SEC) << tk->tkr_mono.shift)) {
45 nsec -= (((u64)NSEC_PER_SEC) << tk->tkr_mono.shift);
46 vdso_ts->sec++;
47 }
48 vdso_ts->nsec = nsec;
49
50 /* Copy MONOTONIC time for BOOTTIME */
51 sec = vdso_ts->sec;
52 /* Add the boot offset */
53 sec += tk->monotonic_to_boot.tv_sec;
54 nsec += (u64)tk->monotonic_to_boot.tv_nsec << tk->tkr_mono.shift;
55
56 /* CLOCK_BOOTTIME */
57 vdso_ts = &vc[CS_HRES_COARSE].basetime[CLOCK_BOOTTIME];
58 vdso_ts->sec = sec;
59
60 while (nsec >= (((u64)NSEC_PER_SEC) << tk->tkr_mono.shift)) {
61 nsec -= (((u64)NSEC_PER_SEC) << tk->tkr_mono.shift);
62 vdso_ts->sec++;
63 }
64 vdso_ts->nsec = nsec;
65
66 /* CLOCK_MONOTONIC_RAW */
67 vdso_ts = &vc[CS_RAW].basetime[CLOCK_MONOTONIC_RAW];
68 vdso_ts->sec = tk->raw_sec;
69 vdso_ts->nsec = tk->tkr_raw.xtime_nsec;
70
71 /* CLOCK_TAI */
72 vdso_ts = &vc[CS_HRES_COARSE].basetime[CLOCK_TAI];
73 vdso_ts->sec = tk->xtime_sec + (s64)tk->tai_offset;
74 vdso_ts->nsec = tk->tkr_mono.xtime_nsec;
75}
76
77void update_vsyscall(struct timekeeper *tk)
78{
79 struct vdso_time_data *vdata = vdso_k_time_data;
80 struct vdso_clock *vc = vdata->clock_data;
81 struct vdso_timestamp *vdso_ts;
82 s32 clock_mode;
83 u64 nsec;
84
85 /* copy vsyscall data */
86 vdso_write_begin(vd: vdata);
87
88 clock_mode = tk->tkr_mono.clock->vdso_clock_mode;
89 vc[CS_HRES_COARSE].clock_mode = clock_mode;
90 vc[CS_RAW].clock_mode = clock_mode;
91
92 /* CLOCK_REALTIME also required for time() */
93 vdso_ts = &vc[CS_HRES_COARSE].basetime[CLOCK_REALTIME];
94 vdso_ts->sec = tk->xtime_sec;
95 vdso_ts->nsec = tk->tkr_mono.xtime_nsec;
96
97 /* CLOCK_REALTIME_COARSE */
98 vdso_ts = &vc[CS_HRES_COARSE].basetime[CLOCK_REALTIME_COARSE];
99 vdso_ts->sec = tk->xtime_sec;
100 vdso_ts->nsec = tk->coarse_nsec;
101
102 /* CLOCK_MONOTONIC_COARSE */
103 vdso_ts = &vc[CS_HRES_COARSE].basetime[CLOCK_MONOTONIC_COARSE];
104 vdso_ts->sec = tk->xtime_sec + tk->wall_to_monotonic.tv_sec;
105 nsec = tk->coarse_nsec;
106 nsec = nsec + tk->wall_to_monotonic.tv_nsec;
107 vdso_ts->sec += __iter_div_u64_rem(dividend: nsec, NSEC_PER_SEC, remainder: &vdso_ts->nsec);
108
109 /*
110 * Read without the seqlock held by clock_getres().
111 */
112 WRITE_ONCE(vdata->hrtimer_res, hrtimer_resolution);
113
114 /*
115 * If the current clocksource is not VDSO capable, then spare the
116 * update of the high resolution parts.
117 */
118 if (clock_mode != VDSO_CLOCKMODE_NONE)
119 update_vdso_time_data(vdata, tk);
120
121 __arch_update_vdso_clock(vc: &vc[CS_HRES_COARSE]);
122 __arch_update_vdso_clock(vc: &vc[CS_RAW]);
123
124 vdso_write_end(vd: vdata);
125
126 __arch_sync_vdso_time_data(vdata);
127}
128
129void update_vsyscall_tz(void)
130{
131 struct vdso_time_data *vdata = vdso_k_time_data;
132
133 vdata->tz_minuteswest = sys_tz.tz_minuteswest;
134 vdata->tz_dsttime = sys_tz.tz_dsttime;
135
136 __arch_sync_vdso_time_data(vdata);
137}
138
139#ifdef CONFIG_POSIX_AUX_CLOCKS
140void vdso_time_update_aux(struct timekeeper *tk)
141{
142 struct vdso_time_data *vdata = vdso_k_time_data;
143 struct vdso_timestamp *vdso_ts;
144 struct vdso_clock *vc;
145 s32 clock_mode;
146 u64 nsec;
147
148 vc = &vdata->aux_clock_data[tk->id - TIMEKEEPER_AUX_FIRST];
149 vdso_ts = &vc->basetime[VDSO_BASE_AUX];
150 clock_mode = tk->tkr_mono.clock->vdso_clock_mode;
151 if (!tk->clock_valid)
152 clock_mode = VDSO_CLOCKMODE_NONE;
153
154 /* copy vsyscall data */
155 vdso_write_begin_clock(vc);
156
157 vc->clock_mode = clock_mode;
158
159 if (clock_mode != VDSO_CLOCKMODE_NONE) {
160 fill_clock_configuration(vc, &tk->tkr_mono);
161
162 vdso_ts->sec = tk->xtime_sec + tk->monotonic_to_aux.tv_sec;
163
164 nsec = tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift;
165 nsec += tk->monotonic_to_aux.tv_nsec;
166 vdso_ts->sec += __iter_div_u64_rem(nsec, NSEC_PER_SEC, &nsec);
167 nsec = nsec << tk->tkr_mono.shift;
168 vdso_ts->nsec = nsec;
169 }
170
171 __arch_update_vdso_clock(vc);
172
173 vdso_write_end_clock(vc);
174
175 __arch_sync_vdso_time_data(vdata);
176}
177#endif
178
179/**
180 * vdso_update_begin - Start of a VDSO update section
181 *
182 * Allows architecture code to safely update the architecture specific VDSO
183 * data. Disables interrupts, acquires timekeeper lock to serialize against
184 * concurrent updates from timekeeping and invalidates the VDSO data
185 * sequence counter to prevent concurrent readers from accessing
186 * inconsistent data.
187 *
188 * Returns: Saved interrupt flags which need to be handed in to
189 * vdso_update_end().
190 */
191unsigned long vdso_update_begin(void)
192{
193 struct vdso_time_data *vdata = vdso_k_time_data;
194 unsigned long flags = timekeeper_lock_irqsave();
195
196 vdso_write_begin(vd: vdata);
197 return flags;
198}
199
200/**
201 * vdso_update_end - End of a VDSO update section
202 * @flags: Interrupt flags as returned from vdso_update_begin()
203 *
204 * Pairs with vdso_update_begin(). Marks vdso data consistent, invokes data
205 * synchronization if the architecture requires it, drops timekeeper lock
206 * and restores interrupt flags.
207 */
208void vdso_update_end(unsigned long flags)
209{
210 struct vdso_time_data *vdata = vdso_k_time_data;
211
212 vdso_write_end(vd: vdata);
213 __arch_sync_vdso_time_data(vdata);
214 timekeeper_unlock_irqrestore(flags);
215}
216