1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
#![no_std]
#![no_main]
#![feature(panic_info_message)]
#![deny(clippy::undocumented_unsafe_blocks)]

#[allow(unused_macros)]
macro_rules! dbg {
    ($value:expr) => {{
        let value = $value;
        log::debug!("{:?}", value);
        value
    }};
}

macro_rules! read_special_reg {
    ($special:literal) => {{
        let result: u64;
        ::core::arch::asm!(concat!("mrs {}, ", $special), out(reg) result);
        result
    }};
}

macro_rules! write_special_reg {
    ($special:literal, $value:expr) => {
        ::core::arch::asm!(concat!("msr ", $special, ", {:x}"), in(reg) $value);
    };
}

mod a53;
mod gicv2;
mod logging;
mod reg;
mod scheduler;
mod sync;
mod task;
mod tt;

use core::arch::{asm, global_asm};
use core::fmt::Write;
use core::panic::PanicInfo;
use core::ptr::null;

use allocator::Allocator;
use scheduler::Scheduler;
use task::Context;

use crate::gicv2::InterruptId;
use crate::logging::Pl011Writer;
use crate::sync::OnceCell;
use crate::tt::page::PageBox;
use crate::tt::table::TranslationTable;
use crate::tt::Level0;
// use crate::tt::{PageBox, TranslationTable};

global_asm!(include_str!("entry.s"), options(raw));

extern "C" {
    static VECTORS: [u8; 0x800];
}

// TODO starting with the incorrect values seems bad, is this bad?
static mut TIMER_INTERRUPT: InterruptId = InterruptId::spurious();
static mut GICD: gicv2::Distributor = gicv2::Distributor::new(null());
static mut GICC: gicv2::CpuInterface = gicv2::CpuInterface::new(null());
static mut SCHEDULER: OnceCell<Scheduler> = OnceCell::new();
static mut ALLOCATOR: OnceCell<Allocator> = OnceCell::new();

#[no_mangle]
unsafe extern "C" fn vector_el1_sp0_synchronous() {
    log::trace!("vector_el1_sp0_synchronous");
    panic_on_synchronous_or_serror(b'A');
}

#[no_mangle]
unsafe extern "C" fn vector_el1_sp0_irq() {
    log::trace!("vector_el1_sp0_irq");
}

#[no_mangle]
unsafe extern "C" fn vector_el1_sp0_fiq() {
    log::trace!("vector_el1_sp0_fiq");
}

#[no_mangle]
unsafe extern "C" fn vector_el1_sp0_serror() {
    log::trace!("vector_el1_sp0_serror");
    panic_on_synchronous_or_serror(b'D');
}

#[no_mangle]
unsafe extern "C" fn vector_el1_sp1_synchronous() {
    log::trace!("vector_el1_sp1_synchronous");
    panic_on_synchronous_or_serror(b'E');
}

#[no_mangle]
unsafe extern "C" fn vector_el1_sp1_irq() {
    log::trace!("vector_el1_sp1_irq");
}

#[no_mangle]
unsafe extern "C" fn vector_el1_sp1_fiq() {
    log::trace!("vector_el1_sp1_fiq");
}

#[no_mangle]
unsafe extern "C" fn vector_el1_sp1_serror(_context: *const Context) -> *const Context {
    log::trace!("vector_el1_sp1_serror");
    panic_on_synchronous_or_serror(b'H');
}

#[no_mangle]
unsafe extern "C" fn vector_el0_a64_synchronous(_context: *const Context) -> *const Context {
    log::trace!("vector_el0_a64_synchronous");
    panic_on_synchronous_or_serror(b'I');
}

#[no_mangle]
unsafe extern "C" fn vector_el0_a64_irq(mut context: *const Context) -> *const Context {
    log::trace!("vector_el0_a64_irq");
    log::debug!("{:?}", *context);

    GICC.handle(|cpuid, interrupt_id| {
        log::trace!("elx_irq cpuid = {cpuid}, interrupt_id = {interrupt_id:?}");
        match interrupt_id {
            x if x == TIMER_INTERRUPT => {
                write_special_reg!("CNTP_TVAL_EL0", read_special_reg!("CNTFRQ_EL0") / 10);

                if let Some(scheduler) = SCHEDULER.get_mut() {
                    context = scheduler.schedule().context();
                }
            }
            _ => {}
        }
    });

    context
}

#[no_mangle]
unsafe extern "C" fn vector_el0_a64_fiq(context: *const Context) -> *const Context {
    log::trace!("vector_el0_a64_fiq");

    context
}

#[no_mangle]
unsafe extern "C" fn vector_el0_a64_serror(_context: *const Context) -> *const Context {
    log::trace!("vector_el0_a64_serror");
    panic_on_synchronous_or_serror(b'L');
}

#[no_mangle]
unsafe extern "C" fn vector_el0_a32_synchronous() {
    log::trace!("vector_el0_a32_synchronous");
    panic_on_synchronous_or_serror(b'M');
}

#[no_mangle]
unsafe extern "C" fn vector_el0_a32_irq() {
    log::trace!("vector_el0_a32_irq");
}

#[no_mangle]
unsafe extern "C" fn vector_el0_a32_fiq() {
    log::trace!("vector_el0_a32_fiq");
}

#[no_mangle]
unsafe extern "C" fn vector_el0_a32_serror() {
    log::trace!("vector_el0_a32_serror");
    panic_on_synchronous_or_serror(b'P');
}

fn panic_on_synchronous_or_serror(kind: u8) -> ! {
    // TODO get rid of these kind codes, no need to call this from asm
    let kind = match kind {
        b'A' => "synchronous, SP_EL0",
        b'D' => "SError, SP_EL0",
        b'E' => "synchronous, SP_ELx",
        b'H' => "SError, SP_ELx",
        b'I' => "synchronous, lower64",
        b'L' => "SError, lower64",
        b'M' => "synchronous, lower32",
        b'P' => "SError, lower32",
        _ => unreachable!(),
    };
    // TODO migrate to SystemRegister api
    let syndrome = unsafe { read_special_reg!("ESR_EL1") };
    let exception_class = syndrome >> 26 & 0x3F;
    let reason = match exception_class {
        0x00 => Some("Unknown reason"),
        0x15 => Some("SVC instruction execution in AArch64 state"),
        _ => None,
    };
    if let Some(reason) = reason {
        panic!(
            "Exception ({}): {:016X}h\n    reason {:02X}h = {}",
            kind, syndrome, exception_class, reason
        );
    } else {
        panic!(
            "Exception ({}): {:016X}h\n    reason {:02X}h",
            kind, syndrome, exception_class
        );
    }
}

#[panic_handler]
fn on_panic(info: &PanicInfo) -> ! {
    // We've already panicked, so this is our last ditch effort to communicate to the user any
    // relevant information that could be used to debug the issue. As such, if writing fails, we
    // can't do much about it.
    trait ResultExt {
        fn ignore(self);
    }

    impl<T, E> ResultExt for Result<T, E> {
        fn ignore(self) {
            // do nothing
        }
    }

    const RED_BOLD: &str = "\x1b[31m\x1b[1m";
    const BRIGHT_BLACK: &str = "\x1b[38;5;240m";
    const SGR0: &str = "\x1b[0m";

    if let Some(writer) = unsafe { &mut logging::WRITER } {
        write!(writer, "\n\n💣 💥 🐶 {RED_BOLD}panicked{SGR0} 🐶 💥 💣").ignore();
        if let Some(location) = info.location() {
            write!(writer, " {BRIGHT_BLACK}at {location}{SGR0}").ignore();
        }
        writeln!(writer).ignore();

        if let Some(message) = info.message() {
            write!(writer, "{message}").ignore();
        } else if let Some(payload) = info.payload().downcast_ref::<&'static str>() {
            write!(writer, "{payload}").ignore();
        } else {
            write!(writer, "<no message>").ignore();
        }
        write!(writer, "\n\n").ignore();
    }

    loop {}
}

#[no_mangle]
pub extern "C" fn kernel_main() {
    // SAFETY: QEMU loads a FDT at the base of memory (0x4000_0000) for non-Linux images (e.g. ELFs)
    // passed to -kernel, provided that the image leaves enough space at the base of RAM for the
    // FDT.
    //
    // This does mean that there may not be an FDT at this location in memory. In this case, the
    // pointer is still valid to read from (avoiding UB) but Fdt::from_ptr will fail as the memory
    // (hopefully) does not the FDT magic value.
    //
    // See https://qemu-project.gitlab.io/qemu/system/arm/virt.html#hardware-configuration-information-for-bare-metal-programming.
    let fdt = unsafe { fdt::Fdt::from_ptr(0x4000_0000 as *const u8).unwrap() };

    let uart0 = fdt.find_compatible(&["arm,pl011"]).unwrap();
    let uart0 = uart0.reg().unwrap().next().unwrap();
    let uart0 = Pl011Writer::new(uart0.starting_address);
    logging::init(uart0, log::LevelFilter::Trace);

    extern "C" {
        static _kernel_va: u8;
        static _kernel_pa: u8;
        static _ekernel_va: u8;
    }

    // TODO: PageBox
    let mut tt = PageBox::new(TranslationTable::<Level0>::new());

    // annoying: relocation fails (out of range) when we try and use the PA like we do the VAs below
    let pa: usize;
    unsafe { asm!("ldr {}, =_kernel_pa", out(reg) pa) };

    tt.map_contiguous(
        unsafe { &_kernel_va } as *const _ as usize,
        unsafe { &_ekernel_va } as *const _ as usize,
        pa,
        "rx",
    );

    unsafe {
        asm!("msr TTBR1_EL1, {:x}", "dsb sy", in(reg) tt.addr().addr());
    }

    log::error!("error woof");
    log::warn!("warn woof");
    log::info!("info woof");
    log::debug!("debug woof");
    log::trace!("trace woof");

    log::debug!("woof!!!! wraaaooo!!");

    // enable timer interrupts
    unsafe {
        log::debug!("CNTFRQ_EL0 = {:016X}h", read_special_reg!("CNTFRQ_EL0"));
        write_special_reg!("CNTP_CTL_EL0", 1u64);
    }

    let timer = fdt.find_compatible(&["arm,armv8-timer"]).unwrap();
    let timer_interrupts = timer.property("interrupts").unwrap().value;
    let mut timer_interrupts = gicv2::InterruptSpecifier::interrupts_iter(timer_interrupts);
    unsafe { TIMER_INTERRUPT = timer_interrupts.nth(1).unwrap().interrupt_id().unwrap() };

    let gic = fdt.find_compatible(&["arm,cortex-a15-gic"]).unwrap();
    let mut gic = gic.reg().unwrap();
    unsafe {
        GICD = gicv2::Distributor::new(gic.next().unwrap().starting_address);
        GICD.enable();

        // TODO document this, is it the virt or the non-secure phys?
        // https://github.com/torvalds/linux/blob/90b0c2b2edd1adff742c621e246562fbefa11b70/Documentation/devicetree/bindings/timer/arm%2Carch_timer.yaml#L44-L58
        GICD.enable_interrupt(TIMER_INTERRUPT);

        GICC = gicv2::CpuInterface::new(gic.next().unwrap().starting_address);
        GICC.enable();
    }

    unsafe {
        // set up vector table base address
        asm!("msr VBAR_EL1, {}", in(reg) &VECTORS);

        SCHEDULER.get_or_init(|| Scheduler::new());
    }

    extern "C" {
        // FIXME relocation R_AARCH64_ADR_PREL_PG_HI21 out of range:
        // 281476054814720 is not in [-4294967296, 4294967295]; references '_buddy_alloc_tree_pa'
        // static _buddy_alloc_tree_pa: u8;
        // static _kernel_pa: u8;
        static _buddy_alloc_tree_va: u8;
    }
    let ram = fdt.memory().regions().next().unwrap();
    let allocator_start = unsafe { &_buddy_alloc_tree_va } as *const u8;
    let allocator_start_pa = unsafe { allocator_start.sub(0xffff000000000000 - 0x40000000) };
    let allocator_len = unsafe {
        ram.size.unwrap() - allocator_start_pa.offset_from(ram.starting_address) as usize
    };
    let allocator_end = unsafe { (&_buddy_alloc_tree_va as *const u8).add(allocator_len) };
    unsafe {
        dbg!(ALLOCATOR.get_or_init(|| Allocator::new(allocator_start, allocator_end)));
    }

    // Permanently transfer control to the scheduler.
    // We don’t need to explicitly clear DAIF.I, because the initial task_restore (entry.s) will
    // clear it when ERET copies the task’s SPSR to PSTATE.
    unsafe { SCHEDULER.get_mut() }.unwrap().start();
}