const bootloader = @import("bootloader"); const cpu = @import("cpu"); const lib = @import("lib"); const privileged = @import("privileged"); const birth = @import("birth"); const Allocator = lib.Allocator; const assert = lib.assert; const ELF = lib.ELF(64); const log = lib.log.scoped(.INIT); const PhysicalAddress = lib.PhysicalAddress; const PhysicalMemoryRegion = lib.PhysicalMemoryRegion; const VirtualAddress = lib.VirtualAddress; const VirtualMemoryRegion = lib.VirtualMemoryRegion; const Leaf = cpu.interface.Leaf; const PageTable = cpu.interface.PageTable; const panic = cpu.panic; const RegionList = cpu.RegionList; const x86_64 = cpu.arch.current; const paging = privileged.arch.paging; const APIC = privileged.arch.x86_64.APIC; const cr0 = privileged.arch.x86_64.registers.cr0; const cr3 = privileged.arch.x86_64.registers.cr3; const cr4 = privileged.arch.x86_64.registers.cr4; const XCR0 = privileged.arch.x86_64.registers.XCR0; const IA32_APIC_BASE = privileged.arch.x86_64.registers.IA32_APIC_BASE; const IA32_EFER = privileged.arch.x86_64.registers.IA32_EFER; const IA32_FS_BASE = privileged.arch.x86_64.registers.IA32_FS_BASE; const IA32_FSTAR = privileged.arch.x86_64.registers.IA32_FSTAR; const IA32_FMASK = privileged.arch.x86_64.registers.IA32_FMASK; const IA32_LSTAR = privileged.arch.x86_64.registers.IA32_LSTAR; const IA32_STAR = privileged.arch.x86_64.registers.IA32_STAR; pub fn entryPoint() callconv(.Naked) noreturn { asm volatile ( \\lea stack(%rip), %rsp \\add %[stack_len], %rsp \\pushq $0 \\mov %rsp, %rbp \\jmp *%[main] : : [stack_len] "i" (cpu.stack.len), [main] "{rax}" (&main), : "rsp", "rbp" ); } noinline fn main(bootloader_information: *bootloader.Information) callconv(.C) noreturn { log.info("Initializing...\n\n\t[BUILD MODE] {s}\n\t[BOOTLOADER] {s}\n\t[BOOT PROTOCOL] {s}\n", .{ @tagName(lib.build_mode), @tagName(bootloader_information.bootloader), @tagName(bootloader_information.protocol) }); cpu.init.initialize(bootloader_information) catch |err| { cpu.panicWithStackTrace(@errorReturnTrace(), "Failed to initialize CPU: {}", .{err}); }; } pub inline fn initialize() !void { const cpuid = lib.arch.x86_64.cpuid; if (x86_64.pcid) { if (cpuid(1).ecx & (1 << 17) == 0) return error.feature_requested_and_not_available; } if (x86_64.invariant_tsc) { if (cpuid(0x80000007).edx & (1 << 8) == 0) return error.feature_requested_and_not_available; } // Initialize GDT const gdt_descriptor = x86_64.GDT.Descriptor{ .limit = @sizeOf(x86_64.GDT) - 1, .address = @intFromPtr(&gdt), }; asm volatile ( \\lgdt %[gdt] \\mov %[ds], %rax \\movq %rax, %ds \\movq %rax, %es \\movq %rax, %fs \\movq %rax, %gs \\movq %rax, %ss \\pushq %[cs] \\lea 1f(%rip), %rax \\pushq %rax \\lretq \\1: : : [gdt] "*p" (&gdt_descriptor), [ds] "i" (x86_64.data_64), [cs] "i" (x86_64.code_64), : "memory" ); const tss_address = @intFromPtr(&tss); gdt.tss_descriptor = .{ .limit_low = @as(u16, @truncate(@sizeOf(x86_64.TSS))), .base_low = @as(u16, @truncate(tss_address)), .base_mid_low = @as(u8, @truncate(tss_address >> 16)), .access = .{ .type = .tss_available, .dpl = 0, .present = true, }, .attributes = .{ .limit = @as(u4, @truncate(@sizeOf(x86_64.TSS) >> 16)), .available_for_system_software = false, .granularity = false, }, .base_mid_high = @as(u8, @truncate(tss_address >> 24)), .base_high = @as(u32, @truncate(tss_address >> 32)), }; tss.rsp[0] = @intFromPtr(&interrupt_stack) + interrupt_stack.len; asm volatile ( \\ltr %[tss_selector] : : [tss_selector] "r" (@as(u16, x86_64.tss_selector)), : "memory" ); // Initialize IDT for (&idt.descriptors, interrupt_handlers, 0..) |*descriptor, interrupt_handler, i| { const interrupt_address = @intFromPtr(interrupt_handler); descriptor.* = .{ .offset_low = @as(u16, @truncate(interrupt_address)), .segment_selector = x86_64.code_64, .flags = .{ .ist = 0, .type = if (i < 32) .trap_gate else .interrupt_gate, // TODO: I think this is not correct .dpl = 0, .present = true, }, .offset_mid = @as(u16, @truncate(interrupt_address >> 16)), .offset_high = @as(u32, @truncate(interrupt_address >> 32)), }; } const idt_descriptor = x86_64.IDT.Descriptor{ .limit = @sizeOf(x86_64.IDT) - 1, .address = @intFromPtr(&idt), }; asm volatile ( \\lidt %[idt_descriptor] : : [idt_descriptor] "*p" (&idt_descriptor), : "memory" ); // Mask PIC privileged.arch.io.write(u8, 0xa1, 0xff); privileged.arch.io.write(u8, 0x21, 0xff); asm volatile ("sti" ::: "memory"); const star = IA32_STAR{ .kernel_cs = x86_64.code_64, .user_cs_anchor = x86_64.data_64, }; comptime { assert(x86_64.data_64 == star.kernel_cs + 8); assert(star.user_cs_anchor == x86_64.user_data_64 - 8); assert(star.user_cs_anchor == x86_64.user_code_64 - 16); } star.write(); IA32_LSTAR.write(@intFromPtr(&cpu.arch.x86_64.syscall.entryPoint)); const syscall_mask = privileged.arch.x86_64.registers.syscall_mask; IA32_FMASK.write(syscall_mask); // Enable syscall extensions var efer = IA32_EFER.read(); efer.SCE = true; efer.write(); const avx_xsave_cpuid = cpuid(1, 0); const xsave_support = avx_xsave_cpuid.ecx & (1 << 26) != 0; // TODO: AVX var my_cr4 = cr4.read(); my_cr4.OSFXSR = true; my_cr4.OSXMMEXCPT = true; if (xsave_support) { // my_cr4.OSXSAVE = true; } my_cr4.page_global_enable = true; my_cr4.performance_monitoring_counter_enable = true; my_cr4.write(); var my_cr0 = cr0.read(); my_cr0.monitor_coprocessor = true; my_cr0.emulation = false; my_cr0.numeric_error = true; my_cr0.task_switched = false; my_cr0.write(); const avx_support = avx_xsave_cpuid.ecx & (1 << 28) != 0; // const avx2_support = cpuid(7).ebx & (1 << 5) != 0; log.debug("AVX: {}. AVX2: {}. XSAVE: {}. Can't enable them yet", .{ avx_support, false, xsave_support }); comptime { assert(lib.arch.valid_page_sizes[0] == 0x1000); } // The bootloader already mapped APIC, so it's not necessary to map it here var ia32_apic_base = IA32_APIC_BASE.read(); cpu.bsp = ia32_apic_base.bsp; ia32_apic_base.global_enable = true; const spurious_vector: u8 = 0xFF; APIC.write(.spurious, @as(u32, 0x100) | spurious_vector); const tpr = APIC.TaskPriorityRegister{}; tpr.write(); const lvt_timer = APIC.LVTTimer{}; lvt_timer.write(); ia32_apic_base.write(); x86_64.ticks_per_ms = APIC.calibrateTimer(); cpu.core_id = APIC.read(.id); asm volatile ( \\fninit // TODO: figure out why this crashes with KVM //\\ldmxcsr %[mxcsr] :: //[mxcsr] "m" (@as(u32, 0x1f80)), : "memory"); // TODO: configure PAT } // TODO: // Write user TLS base address export var interrupt_stack: [0x1000]u8 align(lib.arch.stack_alignment) = undefined; export var gdt = x86_64.GDT{}; export var tss = x86_64.TSS{}; export var idt = x86_64.IDT{}; export var user_stack: u64 = 0; comptime { assert(birth.arch.user_code_selector == x86_64.user_code_selector); assert(birth.arch.user_data_selector == x86_64.user_data_selector); } pub fn InterruptHandler(comptime interrupt_number: u64, comptime has_error_code: bool) fn () callconv(.Naked) noreturn { return struct { fn handler() callconv(.Naked) noreturn { asm volatile ( \\endbr64 ::: "memory"); if (x86_64.smap) { // TODO: Investigate why this is Exception #6 asm volatile ( \\clac ::: "memory"); } asm volatile ( \\cld ::: "memory"); if (!has_error_code) { asm volatile ("pushq $0" ::: "memory"); } asm volatile ( \\push %rdi \\push %rsi \\push %rdx \\push %rcx \\push %rax \\push %r8 \\push %r9 \\push %r10 \\push %r11 \\push %rbx \\push %rbp \\push %r12 \\push %r13 \\push %r14 \\push %r15 \\mov %rsp, %rdi \\mov %[interrupt_number], %rsi \\call interruptHandler \\pop %r15 \\pop %r14 \\pop %r13 \\pop %r12 \\pop %rbp \\pop %rbx \\pop %r11 \\pop %r10 \\pop %r9 \\pop %r8 \\pop %rax \\pop %rcx \\pop %rdx \\pop %rsi \\pop %rdi : : [interrupt_number] "i" (interrupt_number), : "memory" ); if (!has_error_code) { asm volatile ( \\add $0x8, %rsp ::: "memory"); } asm volatile ( \\iretq \\int3 ::: "memory"); } }.handler; } const Interrupt = enum(u5) { DE = 0x00, DB = 0x01, NMI = 0x02, BP = 0x03, OF = 0x04, BR = 0x05, UD = 0x06, NM = 0x07, DF = 0x08, CSO = 0x09, // Not used anymore TS = 0x0A, NP = 0x0B, SS = 0x0C, GP = 0x0D, PF = 0x0E, MF = 0x10, AC = 0x11, MC = 0x12, XM = 0x13, VE = 0x14, CP = 0x15, _, }; const interrupt_handlers = [256]*const fn () callconv(.Naked) noreturn{ InterruptHandler(@intFromEnum(Interrupt.DE), false), InterruptHandler(@intFromEnum(Interrupt.DB), false), InterruptHandler(@intFromEnum(Interrupt.NMI), false), InterruptHandler(@intFromEnum(Interrupt.BP), false), InterruptHandler(@intFromEnum(Interrupt.OF), false), InterruptHandler(@intFromEnum(Interrupt.BR), false), InterruptHandler(@intFromEnum(Interrupt.UD), false), InterruptHandler(@intFromEnum(Interrupt.NM), false), InterruptHandler(@intFromEnum(Interrupt.DF), true), InterruptHandler(@intFromEnum(Interrupt.CSO), false), InterruptHandler(@intFromEnum(Interrupt.TS), true), InterruptHandler(@intFromEnum(Interrupt.NP), true), InterruptHandler(@intFromEnum(Interrupt.SS), true), InterruptHandler(@intFromEnum(Interrupt.GP), true), InterruptHandler(@intFromEnum(Interrupt.PF), true), InterruptHandler(0x0f, false), InterruptHandler(@intFromEnum(Interrupt.MF), false), InterruptHandler(@intFromEnum(Interrupt.AC), true), InterruptHandler(@intFromEnum(Interrupt.MC), false), InterruptHandler(@intFromEnum(Interrupt.XM), false), InterruptHandler(@intFromEnum(Interrupt.VE), false), InterruptHandler(@intFromEnum(Interrupt.CP), true), InterruptHandler(0x16, false), InterruptHandler(0x17, false), InterruptHandler(0x18, false), InterruptHandler(0x19, false), InterruptHandler(0x1a, false), InterruptHandler(0x1b, false), InterruptHandler(0x1c, false), InterruptHandler(0x1d, false), InterruptHandler(0x1e, false), InterruptHandler(0x1f, false), InterruptHandler(0x20, false), InterruptHandler(0x21, false), InterruptHandler(0x22, false), InterruptHandler(0x23, false), InterruptHandler(0x24, false), InterruptHandler(0x25, false), InterruptHandler(0x26, false), InterruptHandler(0x27, false), InterruptHandler(0x28, false), InterruptHandler(0x29, false), InterruptHandler(0x2a, false), InterruptHandler(0x2b, false), InterruptHandler(0x2c, false), InterruptHandler(0x2d, false), InterruptHandler(0x2e, false), InterruptHandler(0x2f, false), InterruptHandler(0x30, false), InterruptHandler(0x31, false), InterruptHandler(0x32, false), InterruptHandler(0x33, false), InterruptHandler(0x34, false), InterruptHandler(0x35, false), InterruptHandler(0x36, false), InterruptHandler(0x37, false), InterruptHandler(0x38, false), InterruptHandler(0x39, false), InterruptHandler(0x3a, false), InterruptHandler(0x3b, false), InterruptHandler(0x3c, false), InterruptHandler(0x3d, false), InterruptHandler(0x3e, false), InterruptHandler(0x3f, false), InterruptHandler(0x40, false), InterruptHandler(0x41, false), InterruptHandler(0x42, false), InterruptHandler(0x43, false), InterruptHandler(0x44, false), InterruptHandler(0x45, false), InterruptHandler(0x46, false), InterruptHandler(0x47, false), InterruptHandler(0x48, false), InterruptHandler(0x49, false), InterruptHandler(0x4a, false), InterruptHandler(0x4b, false), InterruptHandler(0x4c, false), InterruptHandler(0x4d, false), InterruptHandler(0x4e, false), InterruptHandler(0x4f, false), InterruptHandler(0x50, false), InterruptHandler(0x51, false), InterruptHandler(0x52, false), InterruptHandler(0x53, false), InterruptHandler(0x54, false), InterruptHandler(0x55, false), InterruptHandler(0x56, false), InterruptHandler(0x57, false), InterruptHandler(0x58, false), InterruptHandler(0x59, false), InterruptHandler(0x5a, false), InterruptHandler(0x5b, false), InterruptHandler(0x5c, false), InterruptHandler(0x5d, false), InterruptHandler(0x5e, false), InterruptHandler(0x5f, false), InterruptHandler(0x60, false), InterruptHandler(0x61, false), InterruptHandler(0x62, false), InterruptHandler(0x63, false), InterruptHandler(0x64, false), InterruptHandler(0x65, false), InterruptHandler(0x66, false), InterruptHandler(0x67, false), InterruptHandler(0x68, false), InterruptHandler(0x69, false), InterruptHandler(0x6a, false), InterruptHandler(0x6b, false), InterruptHandler(0x6c, false), InterruptHandler(0x6d, false), InterruptHandler(0x6e, false), InterruptHandler(0x6f, false), InterruptHandler(0x70, false), InterruptHandler(0x71, false), InterruptHandler(0x72, false), InterruptHandler(0x73, false), InterruptHandler(0x74, false), InterruptHandler(0x75, false), InterruptHandler(0x76, false), InterruptHandler(0x77, false), InterruptHandler(0x78, false), InterruptHandler(0x79, false), InterruptHandler(0x7a, false), InterruptHandler(0x7b, false), InterruptHandler(0x7c, false), InterruptHandler(0x7d, false), InterruptHandler(0x7e, false), InterruptHandler(0x7f, false), InterruptHandler(0x80, false), InterruptHandler(0x81, false), InterruptHandler(0x82, false), InterruptHandler(0x83, false), InterruptHandler(0x84, false), InterruptHandler(0x85, false), InterruptHandler(0x86, false), InterruptHandler(0x87, false), InterruptHandler(0x88, false), InterruptHandler(0x89, false), InterruptHandler(0x8a, false), InterruptHandler(0x8b, false), InterruptHandler(0x8c, false), InterruptHandler(0x8d, false), InterruptHandler(0x8e, false), InterruptHandler(0x8f, false), InterruptHandler(0x90, false), InterruptHandler(0x91, false), InterruptHandler(0x92, false), InterruptHandler(0x93, false), InterruptHandler(0x94, false), InterruptHandler(0x95, false), InterruptHandler(0x96, false), InterruptHandler(0x97, false), InterruptHandler(0x98, false), InterruptHandler(0x99, false), InterruptHandler(0x9a, false), InterruptHandler(0x9b, false), InterruptHandler(0x9c, false), InterruptHandler(0x9d, false), InterruptHandler(0x9e, false), InterruptHandler(0x9f, false), InterruptHandler(0xa0, false), InterruptHandler(0xa1, false), InterruptHandler(0xa2, false), InterruptHandler(0xa3, false), InterruptHandler(0xa4, false), InterruptHandler(0xa5, false), InterruptHandler(0xa6, false), InterruptHandler(0xa7, false), InterruptHandler(0xa8, false), InterruptHandler(0xa9, false), InterruptHandler(0xaa, false), InterruptHandler(0xab, false), InterruptHandler(0xac, false), InterruptHandler(0xad, false), InterruptHandler(0xae, false), InterruptHandler(0xaf, false), InterruptHandler(0xb0, false), InterruptHandler(0xb1, false), InterruptHandler(0xb2, false), InterruptHandler(0xb3, false), InterruptHandler(0xb4, false), InterruptHandler(0xb5, false), InterruptHandler(0xb6, false), InterruptHandler(0xb7, false), InterruptHandler(0xb8, false), InterruptHandler(0xb9, false), InterruptHandler(0xba, false), InterruptHandler(0xbb, false), InterruptHandler(0xbc, false), InterruptHandler(0xbd, false), InterruptHandler(0xbe, false), InterruptHandler(0xbf, false), InterruptHandler(0xc0, false), InterruptHandler(0xc1, false), InterruptHandler(0xc2, false), InterruptHandler(0xc3, false), InterruptHandler(0xc4, false), InterruptHandler(0xc5, false), InterruptHandler(0xc6, false), InterruptHandler(0xc7, false), InterruptHandler(0xc8, false), InterruptHandler(0xc9, false), InterruptHandler(0xca, false), InterruptHandler(0xcb, false), InterruptHandler(0xcc, false), InterruptHandler(0xcd, false), InterruptHandler(0xce, false), InterruptHandler(0xcf, false), InterruptHandler(0xd0, false), InterruptHandler(0xd1, false), InterruptHandler(0xd2, false), InterruptHandler(0xd3, false), InterruptHandler(0xd4, false), InterruptHandler(0xd5, false), InterruptHandler(0xd6, false), InterruptHandler(0xd7, false), InterruptHandler(0xd8, false), InterruptHandler(0xd9, false), InterruptHandler(0xda, false), InterruptHandler(0xdb, false), InterruptHandler(0xdc, false), InterruptHandler(0xdd, false), InterruptHandler(0xde, false), InterruptHandler(0xdf, false), InterruptHandler(0xe0, false), InterruptHandler(0xe1, false), InterruptHandler(0xe2, false), InterruptHandler(0xe3, false), InterruptHandler(0xe4, false), InterruptHandler(0xe5, false), InterruptHandler(0xe6, false), InterruptHandler(0xe7, false), InterruptHandler(0xe8, false), InterruptHandler(0xe9, false), InterruptHandler(0xea, false), InterruptHandler(0xeb, false), InterruptHandler(0xec, false), InterruptHandler(0xed, false), InterruptHandler(0xee, false), InterruptHandler(0xef, false), InterruptHandler(0xf0, false), InterruptHandler(0xf1, false), InterruptHandler(0xf2, false), InterruptHandler(0xf3, false), InterruptHandler(0xf4, false), InterruptHandler(0xf5, false), InterruptHandler(0xf6, false), InterruptHandler(0xf7, false), InterruptHandler(0xf8, false), InterruptHandler(0xf9, false), InterruptHandler(0xfa, false), InterruptHandler(0xfb, false), InterruptHandler(0xfc, false), InterruptHandler(0xfd, false), InterruptHandler(0xfe, false), InterruptHandler(0xff, false), }; const half_page_table_entry_count = @divExact(paging.page_table_entry_count, 2); var once: bool = false; fn map(address_space: paging.Specific, virtual: VirtualAddress, physical: PhysicalAddress, size: usize, flags: privileged.Mapping.Flags) !void { try address_space.map(physical, virtual, size, flags, cpu.page_allocator.getPageTableAllocatorInterface()); if (flags.user) { const indexed: paging.IndexedVirtualAddress = @bitCast(virtual.value()); const indices = indexed.toIndices(); const top_indexed: paging.IndexedVirtualAddress = @bitCast(virtual.offset(size).value() - lib.arch.valid_page_sizes[0]); const top_indices = top_indexed.toIndices(); _ = top_indices; // TODO: make this fast or not care, depending on how many times this is going to be executed // const user_page_tables = &cpu.user_scheduler.s.capability_root_node.dynamic.page_table; var page_table_ref = user_page_tables.user; for (0..paging.Level.count - 1) |level_index| { const page_table = user_page_tables.getPageTable(page_table_ref) catch |err| { log.err("Error {s} at level {} when trying to map 0x{x} to 0x{x}", .{ @errorName(err), level_index, virtual.value(), physical.value() }); const physical_address = address_space.translateAddress(virtual, .{ .execute_disable = !flags.execute, .write = flags.write, .user = flags.user, }) catch @panic("Could not translate address"); if (physical_address.value() != physical.value()) { @panic("Address mismatch"); } else { @panic("Address match"); } }; page_table_ref = page_table.children[indices[level_index]]; } assert(indexed.PML4 == top_indexed.PML4); assert(indexed.PDP == top_indexed.PDP); assert(indexed.PD == top_indexed.PD); assert(indexed.PT <= top_indexed.PT); const page_table = try user_page_tables.getPageTable(page_table_ref); var index: u10 = indexed.PT; while (index <= top_indexed.PT) : (index += 1) { const leaf = Leaf{ .physical = physical.offset(index - indexed.PT), .flags = .{ .size = .@"4KB", }, }; const leaf_ref = try user_page_tables.appendLeaf(&cpu.user_scheduler.s.capability_root_node.heap.allocator, leaf); page_table.children[index] = leaf_ref; } } } const CPUPageTables = privileged.arch.CPUPageTables; // TODO: construct scheduler virtual memory tree pub fn setupMapping(scheduler: *cpu.UserScheduler, user_virtual_region: VirtualMemoryRegion, cpu_page_tables: CPUPageTables, init_file: cpu.init.InitFile, regions: extern struct { scheduler: cpu.init.MappingArgument, heap: cpu.init.MappingArgument, }) !void { // INFO: Need this hack for page table allocation callback to work cpu.user_scheduler = scheduler; _ = user_virtual_region; const page_tables = &scheduler.s.capability_root_node.dynamic.page_table; const heap_allocator = &scheduler.s.capability_root_node.heap.allocator; const page_table_size = paging.page_table_entry_count * paging.page_table_entry_size; log.debug("Root page table allocation", .{}); const root_page_table_allocation = try cpu.page_allocator.allocateAligned(2 * page_table_size, cpu.arch.user_root_page_table_alignment, .{ .reason = .user_protected }); const root_page_tables = root_page_table_allocation.split(2); log.debug("R priv: 0x{x}. R user: 0x{x}", .{ root_page_tables[0].address.value(), root_page_tables[1].address.value() }); page_tables.privileged = .{ .region = root_page_tables[0], .mapping = root_page_tables[0].address.toHigherHalfVirtualAddress(), .flags = .{ .level = .PML4, }, }; const root_user_page_table = cpu.interface.PageTable{ .region = root_page_tables[1], .mapping = root_page_tables[1].address.toHigherHalfVirtualAddress(), .flags = .{ .level = .PML4, }, }; log.debug("Appending user root page table", .{}); page_tables.user = try page_tables.appendPageTable(heap_allocator, root_user_page_table); log.debug("Copying higher half", .{}); { // Copy the higher half into the user protected address space const current_address_space = paging.Specific{ .cr3 = cr3.read() }; const src_half = (try current_address_space.getPML4TableUnchecked())[half_page_table_entry_count..][0..half_page_table_entry_count]; const dst_half = page_tables.privileged.region.toHigherHalfVirtualAddress().access(paging.PML4TE)[half_page_table_entry_count..][0..half_page_table_entry_count]; @memcpy(dst_half, src_half); // Map CPU driver into the CPU page table const cpu_pte_count = paging.page_table_entry_count - paging.CPUPageTables.left_ptables; const cpu_support_page_table_size = (paging.Level.count - 1) * paging.page_table_size; const cpu_support_page_table_allocation = try cpu.page_allocator.allocate(cpu_support_page_table_size, .{ .reason = .user_protected }); var cpu_support_page_table_allocator = cpu_support_page_table_allocation; const pdp = try cpu_support_page_table_allocator.takeSlice(paging.page_table_size); const pd = try cpu_support_page_table_allocator.takeSlice(paging.page_table_size); const pt = try cpu_support_page_table_allocator.takeSlice(paging.page_table_size); assert(cpu_support_page_table_allocator.size == 0); // Copy CPU driver PTEs to user protected address space const cpu_ptes = cpu_page_tables.p_table.toHigherHalfVirtualAddress().access(*paging.PTable)[0..cpu_pte_count]; const user_mapped_cpu_ptes = pt.toHigherHalfVirtualAddress().access(paging.PTE)[0..cpu_pte_count]; @memcpy(user_mapped_cpu_ptes, cpu_ptes); // Fill the PML4 entry root_user_page_table.region.toHigherHalfVirtualAddress().access(paging.PML4TE)[paging.CPUPageTables.pml4_index] = paging.PML4TE{ .present = true, .write = true, .execute_disable = false, .address = paging.packAddress(paging.PML4TE, pdp.address.value()), }; // Fill the PDP entry pdp.toHigherHalfVirtualAddress().access(paging.PDPTE)[paging.CPUPageTables.pdp_index] = paging.PDPTE{ .present = true, .write = true, .execute_disable = false, .address = paging.packAddress(paging.PDPTE, pd.address.value()), }; // Fill the PD entry pd.toHigherHalfVirtualAddress().access(paging.PDTE)[paging.CPUPageTables.pd_index] = paging.PDTE{ .present = true, .write = true, .execute_disable = false, .address = paging.packAddress(paging.PDTE, pt.address.value()), }; } const privileged_address_space = paging.Specific.fromPhysicalRegion(root_page_tables[0]); const user_address_space = paging.Specific.fromPhysicalRegion(root_page_tables[1]); const scheduler_memory_map_flags = .{ .write = true, .user = true, }; try map(user_address_space, regions.scheduler.virtual, regions.scheduler.physical, regions.scheduler.size, scheduler_memory_map_flags); try map(user_address_space, regions.heap.virtual, regions.heap.physical, regions.heap.size, scheduler_memory_map_flags); for (init_file.segments) |segment| { try map(user_address_space, segment.virtual, segment.physical, segment.memory_size, segment.flags); } // Map protected stack const privileged_stack_physical_region = try cpu.page_allocator.allocate(x86_64.capability_address_space_stack_size, .{ .reason = .user_protected }); try map(privileged_address_space, x86_64.capability_address_space_stack_address, privileged_stack_physical_region.address, x86_64.capability_address_space_stack_size, .{ .write = true, .execute = false, .user = false, }); const cpu_pml4 = try privileged_address_space.getPML4TableUnchecked(); const user_pml4 = try user_address_space.getPML4TableUnchecked(); @memcpy(cpu_pml4[0..cpu.arch.init.half_page_table_entry_count], user_pml4[0..cpu.arch.init.half_page_table_entry_count]); scheduler.s.capability_root_node.dynamic.page_table.switchPrivileged(); } pub fn setupSchedulerCommon(scheduler_common: *birth.Scheduler.Common, entry_point: usize) void { const user_scheduler_virtual_address = @intFromPtr(scheduler_common); IA32_FS_BASE.write(user_scheduler_virtual_address); // Set arguments // First argument scheduler_common.disabled_save_area.registers.rdi = user_scheduler_virtual_address; // Second argument const is_init = true; scheduler_common.disabled_save_area.registers.rsi = @intFromBool(is_init); scheduler_common.disabled_save_area.registers.rip = entry_point; scheduler_common.disabled_save_area.registers.rsp = user_scheduler_virtual_address + @offsetOf(birth.Scheduler.Common, "setup_stack") + scheduler_common.setup_stack.len; scheduler_common.setup_stack_lock.value = true; scheduler_common.disabled_save_area.registers.rflags = .{ .IF = true }; // Set RFLAGS scheduler_common.disabled_save_area.fpu.fcw = 0x037f; // Set FPU scheduler_common.disabled_save_area.fpu.mxcsr = 0x1f80; }