1
2
3
4
5
6#include <linux/bitops.h>
7#include <linux/bsearch.h>
8#include <linux/interrupt.h>
9#include <linux/irq.h>
10#include <linux/kvm.h>
11#include <linux/kvm_host.h>
12#include <kvm/iodev.h>
13#include <kvm/arm_arch_timer.h>
14#include <kvm/arm_vgic.h>
15
16#include "vgic.h"
17#include "vgic-mmio.h"
18
19unsigned long vgic_mmio_read_raz(struct kvm_vcpu *vcpu,
20 gpa_t addr, unsigned int len)
21{
22 return 0;
23}
24
25unsigned long vgic_mmio_read_rao(struct kvm_vcpu *vcpu,
26 gpa_t addr, unsigned int len)
27{
28 return -1UL;
29}
30
31void vgic_mmio_write_wi(struct kvm_vcpu *vcpu, gpa_t addr,
32 unsigned int len, unsigned long val)
33{
34
35}
36
37int vgic_mmio_uaccess_write_wi(struct kvm_vcpu *vcpu, gpa_t addr,
38 unsigned int len, unsigned long val)
39{
40
41 return 0;
42}
43
44unsigned long vgic_mmio_read_group(struct kvm_vcpu *vcpu,
45 gpa_t addr, unsigned int len)
46{
47 u32 intid = VGIC_ADDR_TO_INTID(addr, 1);
48 u32 value = 0;
49 int i;
50
51
52 for (i = 0; i < len * 8; i++) {
53 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
54
55 if (irq->group)
56 value |= BIT(i);
57
58 vgic_put_irq(vcpu->kvm, irq);
59 }
60
61 return value;
62}
63
64static void vgic_update_vsgi(struct vgic_irq *irq)
65{
66 WARN_ON(its_prop_update_vsgi(irq->host_irq, irq->priority, irq->group));
67}
68
69void vgic_mmio_write_group(struct kvm_vcpu *vcpu, gpa_t addr,
70 unsigned int len, unsigned long val)
71{
72 u32 intid = VGIC_ADDR_TO_INTID(addr, 1);
73 int i;
74 unsigned long flags;
75
76 for (i = 0; i < len * 8; i++) {
77 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
78
79 raw_spin_lock_irqsave(&irq->irq_lock, flags);
80 irq->group = !!(val & BIT(i));
81 if (irq->hw && vgic_irq_is_sgi(irq->intid)) {
82 vgic_update_vsgi(irq);
83 raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
84 } else {
85 vgic_queue_irq_unlock(vcpu->kvm, irq, flags);
86 }
87
88 vgic_put_irq(vcpu->kvm, irq);
89 }
90}
91
92
93
94
95
96unsigned long vgic_mmio_read_enable(struct kvm_vcpu *vcpu,
97 gpa_t addr, unsigned int len)
98{
99 u32 intid = VGIC_ADDR_TO_INTID(addr, 1);
100 u32 value = 0;
101 int i;
102
103
104 for (i = 0; i < len * 8; i++) {
105 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
106
107 if (irq->enabled)
108 value |= (1U << i);
109
110 vgic_put_irq(vcpu->kvm, irq);
111 }
112
113 return value;
114}
115
116void vgic_mmio_write_senable(struct kvm_vcpu *vcpu,
117 gpa_t addr, unsigned int len,
118 unsigned long val)
119{
120 u32 intid = VGIC_ADDR_TO_INTID(addr, 1);
121 int i;
122 unsigned long flags;
123
124 for_each_set_bit(i, &val, len * 8) {
125 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
126
127 raw_spin_lock_irqsave(&irq->irq_lock, flags);
128 if (irq->hw && vgic_irq_is_sgi(irq->intid)) {
129 if (!irq->enabled) {
130 struct irq_data *data;
131
132 irq->enabled = true;
133 data = &irq_to_desc(irq->host_irq)->irq_data;
134 while (irqd_irq_disabled(data))
135 enable_irq(irq->host_irq);
136 }
137
138 raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
139 vgic_put_irq(vcpu->kvm, irq);
140
141 continue;
142 } else if (vgic_irq_is_mapped_level(irq)) {
143 bool was_high = irq->line_level;
144
145
146
147
148
149
150 irq->line_level = vgic_get_phys_line_level(irq);
151
152
153
154
155 if (!irq->active && was_high && !irq->line_level)
156 vgic_irq_set_phys_active(irq, false);
157 }
158 irq->enabled = true;
159 vgic_queue_irq_unlock(vcpu->kvm, irq, flags);
160
161 vgic_put_irq(vcpu->kvm, irq);
162 }
163}
164
165void vgic_mmio_write_cenable(struct kvm_vcpu *vcpu,
166 gpa_t addr, unsigned int len,
167 unsigned long val)
168{
169 u32 intid = VGIC_ADDR_TO_INTID(addr, 1);
170 int i;
171 unsigned long flags;
172
173 for_each_set_bit(i, &val, len * 8) {
174 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
175
176 raw_spin_lock_irqsave(&irq->irq_lock, flags);
177 if (irq->hw && vgic_irq_is_sgi(irq->intid) && irq->enabled)
178 disable_irq_nosync(irq->host_irq);
179
180 irq->enabled = false;
181
182 raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
183 vgic_put_irq(vcpu->kvm, irq);
184 }
185}
186
187int vgic_uaccess_write_senable(struct kvm_vcpu *vcpu,
188 gpa_t addr, unsigned int len,
189 unsigned long val)
190{
191 u32 intid = VGIC_ADDR_TO_INTID(addr, 1);
192 int i;
193 unsigned long flags;
194
195 for_each_set_bit(i, &val, len * 8) {
196 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
197
198 raw_spin_lock_irqsave(&irq->irq_lock, flags);
199 irq->enabled = true;
200 vgic_queue_irq_unlock(vcpu->kvm, irq, flags);
201
202 vgic_put_irq(vcpu->kvm, irq);
203 }
204
205 return 0;
206}
207
208int vgic_uaccess_write_cenable(struct kvm_vcpu *vcpu,
209 gpa_t addr, unsigned int len,
210 unsigned long val)
211{
212 u32 intid = VGIC_ADDR_TO_INTID(addr, 1);
213 int i;
214 unsigned long flags;
215
216 for_each_set_bit(i, &val, len * 8) {
217 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
218
219 raw_spin_lock_irqsave(&irq->irq_lock, flags);
220 irq->enabled = false;
221 raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
222
223 vgic_put_irq(vcpu->kvm, irq);
224 }
225
226 return 0;
227}
228
229unsigned long vgic_mmio_read_pending(struct kvm_vcpu *vcpu,
230 gpa_t addr, unsigned int len)
231{
232 u32 intid = VGIC_ADDR_TO_INTID(addr, 1);
233 u32 value = 0;
234 int i;
235
236
237 for (i = 0; i < len * 8; i++) {
238 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
239 unsigned long flags;
240 bool val;
241
242 raw_spin_lock_irqsave(&irq->irq_lock, flags);
243 if (irq->hw && vgic_irq_is_sgi(irq->intid)) {
244 int err;
245
246 val = false;
247 err = irq_get_irqchip_state(irq->host_irq,
248 IRQCHIP_STATE_PENDING,
249 &val);
250 WARN_RATELIMIT(err, "IRQ %d", irq->host_irq);
251 } else {
252 val = irq_is_pending(irq);
253 }
254
255 value |= ((u32)val << i);
256 raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
257
258 vgic_put_irq(vcpu->kvm, irq);
259 }
260
261 return value;
262}
263
264static bool is_vgic_v2_sgi(struct kvm_vcpu *vcpu, struct vgic_irq *irq)
265{
266 return (vgic_irq_is_sgi(irq->intid) &&
267 vcpu->kvm->arch.vgic.vgic_model == KVM_DEV_TYPE_ARM_VGIC_V2);
268}
269
270void vgic_mmio_write_spending(struct kvm_vcpu *vcpu,
271 gpa_t addr, unsigned int len,
272 unsigned long val)
273{
274 u32 intid = VGIC_ADDR_TO_INTID(addr, 1);
275 int i;
276 unsigned long flags;
277
278 for_each_set_bit(i, &val, len * 8) {
279 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
280
281
282 if (is_vgic_v2_sgi(vcpu, irq)) {
283 vgic_put_irq(vcpu->kvm, irq);
284 continue;
285 }
286
287 raw_spin_lock_irqsave(&irq->irq_lock, flags);
288
289 if (irq->hw && vgic_irq_is_sgi(irq->intid)) {
290
291 int err;
292 err = irq_set_irqchip_state(irq->host_irq,
293 IRQCHIP_STATE_PENDING,
294 true);
295 WARN_RATELIMIT(err, "IRQ %d", irq->host_irq);
296
297 raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
298 vgic_put_irq(vcpu->kvm, irq);
299
300 continue;
301 }
302
303 irq->pending_latch = true;
304 if (irq->hw)
305 vgic_irq_set_phys_active(irq, true);
306
307 vgic_queue_irq_unlock(vcpu->kvm, irq, flags);
308 vgic_put_irq(vcpu->kvm, irq);
309 }
310}
311
312int vgic_uaccess_write_spending(struct kvm_vcpu *vcpu,
313 gpa_t addr, unsigned int len,
314 unsigned long val)
315{
316 u32 intid = VGIC_ADDR_TO_INTID(addr, 1);
317 int i;
318 unsigned long flags;
319
320 for_each_set_bit(i, &val, len * 8) {
321 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
322
323 raw_spin_lock_irqsave(&irq->irq_lock, flags);
324 irq->pending_latch = true;
325
326
327
328
329
330
331 if (is_vgic_v2_sgi(vcpu, irq))
332 irq->source |= BIT(vcpu->vcpu_id);
333
334 vgic_queue_irq_unlock(vcpu->kvm, irq, flags);
335
336 vgic_put_irq(vcpu->kvm, irq);
337 }
338
339 return 0;
340}
341
342
343static void vgic_hw_irq_cpending(struct kvm_vcpu *vcpu, struct vgic_irq *irq)
344{
345 irq->pending_latch = false;
346
347
348
349
350
351
352
353
354
355
356
357
358 vgic_irq_set_phys_pending(irq, false);
359 if (!irq->active)
360 vgic_irq_set_phys_active(irq, false);
361}
362
363void vgic_mmio_write_cpending(struct kvm_vcpu *vcpu,
364 gpa_t addr, unsigned int len,
365 unsigned long val)
366{
367 u32 intid = VGIC_ADDR_TO_INTID(addr, 1);
368 int i;
369 unsigned long flags;
370
371 for_each_set_bit(i, &val, len * 8) {
372 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
373
374
375 if (is_vgic_v2_sgi(vcpu, irq)) {
376 vgic_put_irq(vcpu->kvm, irq);
377 continue;
378 }
379
380 raw_spin_lock_irqsave(&irq->irq_lock, flags);
381
382 if (irq->hw && vgic_irq_is_sgi(irq->intid)) {
383
384 int err;
385 err = irq_set_irqchip_state(irq->host_irq,
386 IRQCHIP_STATE_PENDING,
387 false);
388 WARN_RATELIMIT(err, "IRQ %d", irq->host_irq);
389
390 raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
391 vgic_put_irq(vcpu->kvm, irq);
392
393 continue;
394 }
395
396 if (irq->hw)
397 vgic_hw_irq_cpending(vcpu, irq);
398 else
399 irq->pending_latch = false;
400
401 raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
402 vgic_put_irq(vcpu->kvm, irq);
403 }
404}
405
406int vgic_uaccess_write_cpending(struct kvm_vcpu *vcpu,
407 gpa_t addr, unsigned int len,
408 unsigned long val)
409{
410 u32 intid = VGIC_ADDR_TO_INTID(addr, 1);
411 int i;
412 unsigned long flags;
413
414 for_each_set_bit(i, &val, len * 8) {
415 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
416
417 raw_spin_lock_irqsave(&irq->irq_lock, flags);
418
419
420
421
422
423 if (is_vgic_v2_sgi(vcpu, irq))
424 irq->source = 0;
425
426 irq->pending_latch = false;
427
428 raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
429
430 vgic_put_irq(vcpu->kvm, irq);
431 }
432
433 return 0;
434}
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451static void vgic_access_active_prepare(struct kvm_vcpu *vcpu, u32 intid)
452{
453 if (vcpu->kvm->arch.vgic.vgic_model == KVM_DEV_TYPE_ARM_VGIC_V3 ||
454 intid >= VGIC_NR_PRIVATE_IRQS)
455 kvm_arm_halt_guest(vcpu->kvm);
456}
457
458
459static void vgic_access_active_finish(struct kvm_vcpu *vcpu, u32 intid)
460{
461 if (vcpu->kvm->arch.vgic.vgic_model == KVM_DEV_TYPE_ARM_VGIC_V3 ||
462 intid >= VGIC_NR_PRIVATE_IRQS)
463 kvm_arm_resume_guest(vcpu->kvm);
464}
465
466static unsigned long __vgic_mmio_read_active(struct kvm_vcpu *vcpu,
467 gpa_t addr, unsigned int len)
468{
469 u32 intid = VGIC_ADDR_TO_INTID(addr, 1);
470 u32 value = 0;
471 int i;
472
473
474 for (i = 0; i < len * 8; i++) {
475 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
476
477
478
479
480
481 if (irq->active)
482 value |= (1U << i);
483
484 vgic_put_irq(vcpu->kvm, irq);
485 }
486
487 return value;
488}
489
490unsigned long vgic_mmio_read_active(struct kvm_vcpu *vcpu,
491 gpa_t addr, unsigned int len)
492{
493 u32 intid = VGIC_ADDR_TO_INTID(addr, 1);
494 u32 val;
495
496 mutex_lock(&vcpu->kvm->lock);
497 vgic_access_active_prepare(vcpu, intid);
498
499 val = __vgic_mmio_read_active(vcpu, addr, len);
500
501 vgic_access_active_finish(vcpu, intid);
502 mutex_unlock(&vcpu->kvm->lock);
503
504 return val;
505}
506
507unsigned long vgic_uaccess_read_active(struct kvm_vcpu *vcpu,
508 gpa_t addr, unsigned int len)
509{
510 return __vgic_mmio_read_active(vcpu, addr, len);
511}
512
513
514static void vgic_hw_irq_change_active(struct kvm_vcpu *vcpu, struct vgic_irq *irq,
515 bool active, bool is_uaccess)
516{
517 if (is_uaccess)
518 return;
519
520 irq->active = active;
521 vgic_irq_set_phys_active(irq, active);
522}
523
524static void vgic_mmio_change_active(struct kvm_vcpu *vcpu, struct vgic_irq *irq,
525 bool active)
526{
527 unsigned long flags;
528 struct kvm_vcpu *requester_vcpu = kvm_get_running_vcpu();
529
530 raw_spin_lock_irqsave(&irq->irq_lock, flags);
531
532 if (irq->hw && !vgic_irq_is_sgi(irq->intid)) {
533 vgic_hw_irq_change_active(vcpu, irq, active, !requester_vcpu);
534 } else if (irq->hw && vgic_irq_is_sgi(irq->intid)) {
535
536
537
538
539
540 irq->active = false;
541 } else {
542 u32 model = vcpu->kvm->arch.vgic.vgic_model;
543 u8 active_source;
544
545 irq->active = active;
546
547
548
549
550
551
552
553
554
555
556
557
558 active_source = (requester_vcpu) ? requester_vcpu->vcpu_id : 0;
559
560 if (model == KVM_DEV_TYPE_ARM_VGIC_V2 &&
561 active && vgic_irq_is_sgi(irq->intid))
562 irq->active_source = active_source;
563 }
564
565 if (irq->active)
566 vgic_queue_irq_unlock(vcpu->kvm, irq, flags);
567 else
568 raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
569}
570
571static void __vgic_mmio_write_cactive(struct kvm_vcpu *vcpu,
572 gpa_t addr, unsigned int len,
573 unsigned long val)
574{
575 u32 intid = VGIC_ADDR_TO_INTID(addr, 1);
576 int i;
577
578 for_each_set_bit(i, &val, len * 8) {
579 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
580 vgic_mmio_change_active(vcpu, irq, false);
581 vgic_put_irq(vcpu->kvm, irq);
582 }
583}
584
585void vgic_mmio_write_cactive(struct kvm_vcpu *vcpu,
586 gpa_t addr, unsigned int len,
587 unsigned long val)
588{
589 u32 intid = VGIC_ADDR_TO_INTID(addr, 1);
590
591 mutex_lock(&vcpu->kvm->lock);
592 vgic_access_active_prepare(vcpu, intid);
593
594 __vgic_mmio_write_cactive(vcpu, addr, len, val);
595
596 vgic_access_active_finish(vcpu, intid);
597 mutex_unlock(&vcpu->kvm->lock);
598}
599
600int vgic_mmio_uaccess_write_cactive(struct kvm_vcpu *vcpu,
601 gpa_t addr, unsigned int len,
602 unsigned long val)
603{
604 __vgic_mmio_write_cactive(vcpu, addr, len, val);
605 return 0;
606}
607
608static void __vgic_mmio_write_sactive(struct kvm_vcpu *vcpu,
609 gpa_t addr, unsigned int len,
610 unsigned long val)
611{
612 u32 intid = VGIC_ADDR_TO_INTID(addr, 1);
613 int i;
614
615 for_each_set_bit(i, &val, len * 8) {
616 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
617 vgic_mmio_change_active(vcpu, irq, true);
618 vgic_put_irq(vcpu->kvm, irq);
619 }
620}
621
622void vgic_mmio_write_sactive(struct kvm_vcpu *vcpu,
623 gpa_t addr, unsigned int len,
624 unsigned long val)
625{
626 u32 intid = VGIC_ADDR_TO_INTID(addr, 1);
627
628 mutex_lock(&vcpu->kvm->lock);
629 vgic_access_active_prepare(vcpu, intid);
630
631 __vgic_mmio_write_sactive(vcpu, addr, len, val);
632
633 vgic_access_active_finish(vcpu, intid);
634 mutex_unlock(&vcpu->kvm->lock);
635}
636
637int vgic_mmio_uaccess_write_sactive(struct kvm_vcpu *vcpu,
638 gpa_t addr, unsigned int len,
639 unsigned long val)
640{
641 __vgic_mmio_write_sactive(vcpu, addr, len, val);
642 return 0;
643}
644
645unsigned long vgic_mmio_read_priority(struct kvm_vcpu *vcpu,
646 gpa_t addr, unsigned int len)
647{
648 u32 intid = VGIC_ADDR_TO_INTID(addr, 8);
649 int i;
650 u64 val = 0;
651
652 for (i = 0; i < len; i++) {
653 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
654
655 val |= (u64)irq->priority << (i * 8);
656
657 vgic_put_irq(vcpu->kvm, irq);
658 }
659
660 return val;
661}
662
663
664
665
666
667
668
669
670void vgic_mmio_write_priority(struct kvm_vcpu *vcpu,
671 gpa_t addr, unsigned int len,
672 unsigned long val)
673{
674 u32 intid = VGIC_ADDR_TO_INTID(addr, 8);
675 int i;
676 unsigned long flags;
677
678 for (i = 0; i < len; i++) {
679 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
680
681 raw_spin_lock_irqsave(&irq->irq_lock, flags);
682
683 irq->priority = (val >> (i * 8)) & GENMASK(7, 8 - VGIC_PRI_BITS);
684 if (irq->hw && vgic_irq_is_sgi(irq->intid))
685 vgic_update_vsgi(irq);
686 raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
687
688 vgic_put_irq(vcpu->kvm, irq);
689 }
690}
691
692unsigned long vgic_mmio_read_config(struct kvm_vcpu *vcpu,
693 gpa_t addr, unsigned int len)
694{
695 u32 intid = VGIC_ADDR_TO_INTID(addr, 2);
696 u32 value = 0;
697 int i;
698
699 for (i = 0; i < len * 4; i++) {
700 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
701
702 if (irq->config == VGIC_CONFIG_EDGE)
703 value |= (2U << (i * 2));
704
705 vgic_put_irq(vcpu->kvm, irq);
706 }
707
708 return value;
709}
710
711void vgic_mmio_write_config(struct kvm_vcpu *vcpu,
712 gpa_t addr, unsigned int len,
713 unsigned long val)
714{
715 u32 intid = VGIC_ADDR_TO_INTID(addr, 2);
716 int i;
717 unsigned long flags;
718
719 for (i = 0; i < len * 4; i++) {
720 struct vgic_irq *irq;
721
722
723
724
725
726
727
728 if (intid + i < VGIC_NR_PRIVATE_IRQS)
729 continue;
730
731 irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
732 raw_spin_lock_irqsave(&irq->irq_lock, flags);
733
734 if (test_bit(i * 2 + 1, &val))
735 irq->config = VGIC_CONFIG_EDGE;
736 else
737 irq->config = VGIC_CONFIG_LEVEL;
738
739 raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
740 vgic_put_irq(vcpu->kvm, irq);
741 }
742}
743
744u64 vgic_read_irq_line_level_info(struct kvm_vcpu *vcpu, u32 intid)
745{
746 int i;
747 u64 val = 0;
748 int nr_irqs = vcpu->kvm->arch.vgic.nr_spis + VGIC_NR_PRIVATE_IRQS;
749
750 for (i = 0; i < 32; i++) {
751 struct vgic_irq *irq;
752
753 if ((intid + i) < VGIC_NR_SGIS || (intid + i) >= nr_irqs)
754 continue;
755
756 irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
757 if (irq->config == VGIC_CONFIG_LEVEL && irq->line_level)
758 val |= (1U << i);
759
760 vgic_put_irq(vcpu->kvm, irq);
761 }
762
763 return val;
764}
765
766void vgic_write_irq_line_level_info(struct kvm_vcpu *vcpu, u32 intid,
767 const u64 val)
768{
769 int i;
770 int nr_irqs = vcpu->kvm->arch.vgic.nr_spis + VGIC_NR_PRIVATE_IRQS;
771 unsigned long flags;
772
773 for (i = 0; i < 32; i++) {
774 struct vgic_irq *irq;
775 bool new_level;
776
777 if ((intid + i) < VGIC_NR_SGIS || (intid + i) >= nr_irqs)
778 continue;
779
780 irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
781
782
783
784
785
786
787 new_level = !!(val & (1U << i));
788 raw_spin_lock_irqsave(&irq->irq_lock, flags);
789 irq->line_level = new_level;
790 if (new_level)
791 vgic_queue_irq_unlock(vcpu->kvm, irq, flags);
792 else
793 raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
794
795 vgic_put_irq(vcpu->kvm, irq);
796 }
797}
798
799static int match_region(const void *key, const void *elt)
800{
801 const unsigned int offset = (unsigned long)key;
802 const struct vgic_register_region *region = elt;
803
804 if (offset < region->reg_offset)
805 return -1;
806
807 if (offset >= region->reg_offset + region->len)
808 return 1;
809
810 return 0;
811}
812
813const struct vgic_register_region *
814vgic_find_mmio_region(const struct vgic_register_region *regions,
815 int nr_regions, unsigned int offset)
816{
817 return bsearch((void *)(uintptr_t)offset, regions, nr_regions,
818 sizeof(regions[0]), match_region);
819}
820
821void vgic_set_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcr)
822{
823 if (kvm_vgic_global_state.type == VGIC_V2)
824 vgic_v2_set_vmcr(vcpu, vmcr);
825 else
826 vgic_v3_set_vmcr(vcpu, vmcr);
827}
828
829void vgic_get_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcr)
830{
831 if (kvm_vgic_global_state.type == VGIC_V2)
832 vgic_v2_get_vmcr(vcpu, vmcr);
833 else
834 vgic_v3_get_vmcr(vcpu, vmcr);
835}
836
837
838
839
840
841
842
843
844
845
846unsigned long vgic_data_mmio_bus_to_host(const void *val, unsigned int len)
847{
848 unsigned long data = kvm_mmio_read_buf(val, len);
849
850 switch (len) {
851 case 1:
852 return data;
853 case 2:
854 return le16_to_cpu(data);
855 case 4:
856 return le32_to_cpu(data);
857 default:
858 return le64_to_cpu(data);
859 }
860}
861
862
863
864
865
866
867
868
869
870
871void vgic_data_host_to_mmio_bus(void *buf, unsigned int len,
872 unsigned long data)
873{
874 switch (len) {
875 case 1:
876 break;
877 case 2:
878 data = cpu_to_le16(data);
879 break;
880 case 4:
881 data = cpu_to_le32(data);
882 break;
883 default:
884 data = cpu_to_le64(data);
885 }
886
887 kvm_mmio_write_buf(buf, len, data);
888}
889
890static
891struct vgic_io_device *kvm_to_vgic_iodev(const struct kvm_io_device *dev)
892{
893 return container_of(dev, struct vgic_io_device, dev);
894}
895
896static bool check_region(const struct kvm *kvm,
897 const struct vgic_register_region *region,
898 gpa_t addr, int len)
899{
900 int flags, nr_irqs = kvm->arch.vgic.nr_spis + VGIC_NR_PRIVATE_IRQS;
901
902 switch (len) {
903 case sizeof(u8):
904 flags = VGIC_ACCESS_8bit;
905 break;
906 case sizeof(u32):
907 flags = VGIC_ACCESS_32bit;
908 break;
909 case sizeof(u64):
910 flags = VGIC_ACCESS_64bit;
911 break;
912 default:
913 return false;
914 }
915
916 if ((region->access_flags & flags) && IS_ALIGNED(addr, len)) {
917 if (!region->bits_per_irq)
918 return true;
919
920
921 return VGIC_ADDR_TO_INTID(addr, region->bits_per_irq) < nr_irqs;
922 }
923
924 return false;
925}
926
927const struct vgic_register_region *
928vgic_get_mmio_region(struct kvm_vcpu *vcpu, struct vgic_io_device *iodev,
929 gpa_t addr, int len)
930{
931 const struct vgic_register_region *region;
932
933 region = vgic_find_mmio_region(iodev->regions, iodev->nr_regions,
934 addr - iodev->base_addr);
935 if (!region || !check_region(vcpu->kvm, region, addr, len))
936 return NULL;
937
938 return region;
939}
940
941static int vgic_uaccess_read(struct kvm_vcpu *vcpu, struct vgic_io_device *iodev,
942 gpa_t addr, u32 *val)
943{
944 const struct vgic_register_region *region;
945 struct kvm_vcpu *r_vcpu;
946
947 region = vgic_get_mmio_region(vcpu, iodev, addr, sizeof(u32));
948 if (!region) {
949 *val = 0;
950 return 0;
951 }
952
953 r_vcpu = iodev->redist_vcpu ? iodev->redist_vcpu : vcpu;
954 if (region->uaccess_read)
955 *val = region->uaccess_read(r_vcpu, addr, sizeof(u32));
956 else
957 *val = region->read(r_vcpu, addr, sizeof(u32));
958
959 return 0;
960}
961
962static int vgic_uaccess_write(struct kvm_vcpu *vcpu, struct vgic_io_device *iodev,
963 gpa_t addr, const u32 *val)
964{
965 const struct vgic_register_region *region;
966 struct kvm_vcpu *r_vcpu;
967
968 region = vgic_get_mmio_region(vcpu, iodev, addr, sizeof(u32));
969 if (!region)
970 return 0;
971
972 r_vcpu = iodev->redist_vcpu ? iodev->redist_vcpu : vcpu;
973 if (region->uaccess_write)
974 return region->uaccess_write(r_vcpu, addr, sizeof(u32), *val);
975
976 region->write(r_vcpu, addr, sizeof(u32), *val);
977 return 0;
978}
979
980
981
982
983int vgic_uaccess(struct kvm_vcpu *vcpu, struct vgic_io_device *dev,
984 bool is_write, int offset, u32 *val)
985{
986 if (is_write)
987 return vgic_uaccess_write(vcpu, dev, offset, val);
988 else
989 return vgic_uaccess_read(vcpu, dev, offset, val);
990}
991
992static int dispatch_mmio_read(struct kvm_vcpu *vcpu, struct kvm_io_device *dev,
993 gpa_t addr, int len, void *val)
994{
995 struct vgic_io_device *iodev = kvm_to_vgic_iodev(dev);
996 const struct vgic_register_region *region;
997 unsigned long data = 0;
998
999 region = vgic_get_mmio_region(vcpu, iodev, addr, len);
1000 if (!region) {
1001 memset(val, 0, len);
1002 return 0;
1003 }
1004
1005 switch (iodev->iodev_type) {
1006 case IODEV_CPUIF:
1007 data = region->read(vcpu, addr, len);
1008 break;
1009 case IODEV_DIST:
1010 data = region->read(vcpu, addr, len);
1011 break;
1012 case IODEV_REDIST:
1013 data = region->read(iodev->redist_vcpu, addr, len);
1014 break;
1015 case IODEV_ITS:
1016 data = region->its_read(vcpu->kvm, iodev->its, addr, len);
1017 break;
1018 }
1019
1020 vgic_data_host_to_mmio_bus(val, len, data);
1021 return 0;
1022}
1023
1024static int dispatch_mmio_write(struct kvm_vcpu *vcpu, struct kvm_io_device *dev,
1025 gpa_t addr, int len, const void *val)
1026{
1027 struct vgic_io_device *iodev = kvm_to_vgic_iodev(dev);
1028 const struct vgic_register_region *region;
1029 unsigned long data = vgic_data_mmio_bus_to_host(val, len);
1030
1031 region = vgic_get_mmio_region(vcpu, iodev, addr, len);
1032 if (!region)
1033 return 0;
1034
1035 switch (iodev->iodev_type) {
1036 case IODEV_CPUIF:
1037 region->write(vcpu, addr, len, data);
1038 break;
1039 case IODEV_DIST:
1040 region->write(vcpu, addr, len, data);
1041 break;
1042 case IODEV_REDIST:
1043 region->write(iodev->redist_vcpu, addr, len, data);
1044 break;
1045 case IODEV_ITS:
1046 region->its_write(vcpu->kvm, iodev->its, addr, len, data);
1047 break;
1048 }
1049
1050 return 0;
1051}
1052
1053struct kvm_io_device_ops kvm_io_gic_ops = {
1054 .read = dispatch_mmio_read,
1055 .write = dispatch_mmio_write,
1056};
1057
1058int vgic_register_dist_iodev(struct kvm *kvm, gpa_t dist_base_address,
1059 enum vgic_type type)
1060{
1061 struct vgic_io_device *io_device = &kvm->arch.vgic.dist_iodev;
1062 int ret = 0;
1063 unsigned int len;
1064
1065 switch (type) {
1066 case VGIC_V2:
1067 len = vgic_v2_init_dist_iodev(io_device);
1068 break;
1069 case VGIC_V3:
1070 len = vgic_v3_init_dist_iodev(io_device);
1071 break;
1072 default:
1073 BUG_ON(1);
1074 }
1075
1076 io_device->base_addr = dist_base_address;
1077 io_device->iodev_type = IODEV_DIST;
1078 io_device->redist_vcpu = NULL;
1079
1080 mutex_lock(&kvm->slots_lock);
1081 ret = kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS, dist_base_address,
1082 len, &io_device->dev);
1083 mutex_unlock(&kvm->slots_lock);
1084
1085 return ret;
1086}
1087