lazy stack: do nothing in kernel threads and on populated stack

This patch annotates the relevant call sites with the invariant assert
expressions to validate assumptions that let us do "nothing" in all these
cases. We also reorganize some code in the scheduler to help differentiate
between cases when given function/method is called with interrupts or preemption
disabled or from kernel thread or by interrupt handler.

Following methods get added to scheduler code with names describing state of interrupts
or preemption or kernel caller:
- timer_base::set_with_irq_disabled(osv::clock::uptime::time_point time)
- timer_base::set_with_irq_disabled(std::chrono::duration<Rep, Period> duration)
- thread::wake_with_irq_disabled()
- thread::wake_with_irq_or_preemption_disabled(Action action)
- thread_handle::wake_from_kernel_or_with_irq_disabled()

In general:
- we modify all interrupt handlers (those that are executed on interrupt stack)
  to call one of the 3 new wake_...() methods (mostly wake_with_irq_disabled())
  to indicate we do not need/should not pre-fault the stack; most of those are in
  device drivers code
- we modify all code executed on kernel threads that disables preemption or interrupts
  by adding relevant invariant - assert(!sched::thread::current()->is_app()); we do
  not need to pre-fault because the stack is populated
- we also modify the code whhich is indirectly called from kernel threads like
  classifier::post_packet() in net channels
- finally we also modify the scheduler code to use timer_bas::set_with_irq_disabled()
  mostly around preemption_timer to indicate that we should not pre-fault the stack
  downstream

Signed-off-by: Waldemar Kozaczuk <[email protected]>

Author: wkozaczuk

arch/aarch64/exceptions.cc

@@ -122,6 +122,9 @@ void interrupt_table::unregister_interrupt(interrupt *interrupt)
 
 bool interrupt_table::invoke_interrupt(unsigned int id)
 {
+#if CONF_lazy_stack_invariant
+    assert(!arch::irq_enabled());
+#endif
     WITH_LOCK(osv::rcu_read_lock) {
         assert(id < this->nr_irqs);
         interrupt_desc *desc = this->irq_desc[id].read();

arch/aarch64/interrupt.cc

@@ -22,6 +22,9 @@ sgi_interrupt::~sgi_interrupt()
 
 void sgi_interrupt::send(sched::cpu* cpu)
 {
+#if CONF_lazy_stack_invariant
+    assert(!arch::irq_enabled() || !sched::preemptable());
+#endif
     gic::gic->send_sgi(gic::sgi_filter::SGI_TARGET_LIST,
                        cpu->arch.smp_idx, get_id());
 }

arch/x64/exceptions.cc

@@ -220,6 +220,9 @@ void interrupt_descriptor_table::unregister_interrupt(gsi_level_interrupt *inter
 
 void interrupt_descriptor_table::invoke_interrupt(unsigned vector)
 {
+#if CONF_lazy_stack_invariant
+    assert(!arch::irq_enabled());
+#endif
     WITH_LOCK(osv::rcu_read_lock) {
         unsigned i, nr_shared;
         bool handled = false;

arch/x64/mmu.cc

@@ -64,7 +64,7 @@ std::atomic<int> tlb_flush_pendingconfirms;
 inter_processor_interrupt tlb_flush_ipi{IPI_TLB_FLUSH, [] {
         mmu::flush_tlb_local();
         if (tlb_flush_pendingconfirms.fetch_add(-1) == 1) {
-            tlb_flush_waiter.wake();
+            tlb_flush_waiter.wake_from_kernel_or_with_irq_disabled();
         }
 }};
 

arch/x64/msi.cc

@@ -126,7 +126,7 @@ static inline void set_affinity_and_wake(
         v->msix_unmask_entries();
     }
 
-    t->wake();
+    t->wake_with_irq_disabled();
 }
 
 bool interrupt_manager::easy_register(std::initializer_list<msix_binding> bindings)

core/async.cc

@@ -160,6 +160,9 @@ class async_worker {
                 return _timer.expired() || !_queue.empty();
             });
 
+#if CONF_lazy_stack_invariant
+            assert(!sched::thread::current()->is_app());
+#endif
             WITH_LOCK(preempt_lock) {
                 _timer.cancel();
 
@@ -224,6 +227,9 @@ class async_worker {
         }
 
         auto& master = *task.master;
+#if CONF_lazy_stack_invariant
+        assert(!sched::thread::current()->is_app());
+#endif
         DROP_LOCK(preempt_lock) {
             master.fire(task);
         }

core/epoll.cc

@@ -245,7 +245,7 @@ class epoll_file final : public special_file {
         if (!_activity_ring.push(key)) {
             _activity_ring_overflow.store(true, std::memory_order_relaxed);
         }
-        _activity_ring_owner.wake();
+        _activity_ring_owner.wake_from_kernel_or_with_irq_disabled();
     }
 };
 

core/mempool.cc

@@ -154,6 +154,9 @@ void pool::collect_garbage()
 
 static void garbage_collector_fn()
 {
+#if CONF_lazy_stack_invariant
+    assert(!sched::thread::current()->is_app());
+#endif
     WITH_LOCK(preempt_lock) {
         pool::collect_garbage();
     }
@@ -1352,9 +1355,12 @@ void l1::fill_thread()
     auto& pbuf = get_l1();
     for (;;) {
         sched::thread::wait_until([&] {
-                WITH_LOCK(preempt_lock) {
-                    return pbuf.nr < pbuf.watermark_lo || pbuf.nr > pbuf.watermark_hi;
-                }
+#if CONF_lazy_stack_invariant
+            assert(!sched::thread::current()->is_app());
+#endif
+            WITH_LOCK(preempt_lock) {
+                return pbuf.nr < pbuf.watermark_lo || pbuf.nr > pbuf.watermark_hi;
+            }
         });
         if (pbuf.nr < pbuf.watermark_lo) {
             while (pbuf.nr + page_batch::nr_pages < pbuf.max / 2) {

core/net_channel.cc

@@ -40,13 +40,16 @@ void net_channel::process_queue()
 
 void net_channel::wake_pollers()
 {
+#if CONF_lazy_stack_invariant
+    assert(!sched::thread::current()->is_app());
+#endif
     WITH_LOCK(osv::rcu_read_lock) {
         auto pl = _pollers.read();
         if (pl) {
             for (pollreq* pr : *pl) {
                 // net_channel is self synchronizing
                 pr->_awake.store(true, std::memory_order_relaxed);
-                pr->_poll_thread.wake();
+                pr->_poll_thread.wake_from_kernel_or_with_irq_disabled();
             }
         }
         // can't call epoll_wake from rcu, so copy the data
@@ -128,6 +131,9 @@ void classifier::remove(ipv4_tcp_conn_id id)
 
 bool classifier::post_packet(mbuf* m)
 {
+#if CONF_lazy_stack_invariant
+    assert(!sched::thread::current()->is_app());
+#endif
     WITH_LOCK(osv::rcu_read_lock) {
         if (auto nc = classify_ipv4_tcp(m)) {
             log_packet_in(m, NETISR_ETHER);

core/rcu.cc

@@ -118,6 +118,9 @@ void cpu_quiescent_state_thread::do_work()
 {
     while (true) {
         bool toclean = false;
+#if CONF_lazy_stack_invariant
+        assert(!sched::thread::current()->is_app());
+#endif
         WITH_LOCK(preempt_lock) {
             auto p = &*percpu_callbacks;
             if (p->ncallbacks[p->buf]) {
@@ -242,7 +245,7 @@ void rcu_flush()
             rcu_defer([&] { s.post(); });
             // rcu_defer() might not wake the cleanup thread until enough deferred
             // callbacks have accumulated, so wake it up now.
-            percpu_quiescent_state_thread->wake();
+            percpu_quiescent_state_thread->wake_from_kernel_or_with_irq_disabled();
         }, sched::thread::attr().pin(c)));
         t->start();
         t->join();