- missing <linux/mm.h> include (broke build on alpha)

[packages/kernel.git] / sched-2.4.19-rc2-A4

--- linux/fs/proc/array.c.orig	Thu Jul 18 15:22:23 2002
+++ linux/fs/proc/array.c	Thu Jul 18 15:22:33 2002
@@ -338,9 +338,8 @@
 
 	/* scale priority and nice values from timeslices to -20..20 */
 	/* to make it look like a "normal" Unix priority/nice value  */
-	priority = task->counter;
-	priority = 20 - (priority * 10 + DEF_COUNTER / 2) / DEF_COUNTER;
-	nice = task->nice;
+	priority = task_prio(task);
+	nice = task_nice(task);
 
 	read_lock(&tasklist_lock);
 	ppid = task->pid ? task->p_opptr->pid : 0;
@@ -390,7 +389,7 @@
 		task->nswap,
 		task->cnswap,
 		task->exit_signal,
-		task->processor);
+		task->cpu);
 	if(mm)
 		mmput(mm);
 	return res;
--- linux/fs/proc/proc_misc.c.orig	Thu Jul 18 15:22:23 2002
+++ linux/fs/proc/proc_misc.c	Thu Jul 18 15:22:33 2002
@@ -107,11 +107,11 @@
 	a = avenrun[0] + (FIXED_1/200);
 	b = avenrun[1] + (FIXED_1/200);
 	c = avenrun[2] + (FIXED_1/200);
-	len = sprintf(page,"%d.%02d %d.%02d %d.%02d %d/%d %d\n",
+	len = sprintf(page,"%d.%02d %d.%02d %d.%02d %ld/%d %d\n",
 		LOAD_INT(a), LOAD_FRAC(a),
 		LOAD_INT(b), LOAD_FRAC(b),
 		LOAD_INT(c), LOAD_FRAC(c),
-		nr_running, nr_threads, last_pid);
+		nr_running(), nr_threads, last_pid);
 	return proc_calc_metrics(page, start, off, count, eof, len);
 }
 
@@ -123,7 +123,7 @@
 	int len;
 
 	uptime = jiffies;
-	idle = init_tasks[0]->times.tms_utime + init_tasks[0]->times.tms_stime;
+	idle = init_task.times.tms_utime + init_task.times.tms_stime;
 
 	/* The formula for the fraction parts really is ((t * 100) / HZ) % 100, but
 	   that would overflow about every five days at HZ == 100.
@@ -360,10 +360,10 @@
 	}
 
 	proc_sprintf(page, &off, &len,
-		"\nctxt %u\n"
+		"\nctxt %lu\n"
 		"btime %lu\n"
 		"processes %lu\n",
-		kstat.context_swtch,
+		nr_context_switches(),
 		xtime.tv_sec - jif / HZ,
 		total_forks);
 
--- linux/fs/nfs/pagelist.c.orig	Thu Jul 18 15:22:23 2002
+++ linux/fs/nfs/pagelist.c	Thu Jul 18 15:22:33 2002
@@ -96,8 +96,7 @@
 			continue;
 		if (signalled() && (server->flags & NFS_MOUNT_INTR))
 			return ERR_PTR(-ERESTARTSYS);
-		current->policy |= SCHED_YIELD;
-		schedule();
+		yield();
 	}
 
 	/* Initialize the request struct. Initially, we assume a
--- linux/fs/ufs/truncate.c.orig	Thu Jul 18 15:22:16 2002
+++ linux/fs/ufs/truncate.c	Thu Jul 18 15:22:33 2002
@@ -448,10 +448,7 @@
 		if (IS_SYNC(inode) && (inode->i_state & I_DIRTY))
 			ufs_sync_inode (inode);
 		run_task_queue(&tq_disk);
-		current->policy |= SCHED_YIELD;
-		schedule ();
-
-
+		yield();
 	}
 	offset = inode->i_size & uspi->s_fshift;
 	if (offset) {
--- linux/fs/reiserfs/buffer2.c.orig	Thu Jul 18 15:22:11 2002
+++ linux/fs/reiserfs/buffer2.c	Thu Jul 18 15:22:33 2002
@@ -33,8 +33,7 @@
 			buffer_journal_dirty(bh) ? ' ' : '!');
     }
     run_task_queue(&tq_disk);
-    current->policy |= SCHED_YIELD;
-    schedule();
+    yield();
   }
   if (repeat_counter > 30000000) {
     reiserfs_warning("vs-3051: done waiting, ignore vs-3050 messages for (%b)\n", bh) ;
@@ -52,11 +51,11 @@
 struct buffer_head  * reiserfs_bread (struct super_block *super, int n_block, int n_size) 
 {
     struct buffer_head  *result;
-    PROC_EXP( unsigned int ctx_switches = kstat.context_swtch );
+    PROC_EXP( unsigned int ctx_switches = nr_context_switches(); );
 
     result = bread (super -> s_dev, n_block, n_size);
     PROC_INFO_INC( super, breads );
-    PROC_EXP( if( kstat.context_swtch != ctx_switches ) 
+    PROC_EXP( if( nr_context_switches() != ctx_switches ) 
 	      PROC_INFO_INC( super, bread_miss ) );
     return result;
 }
--- linux/fs/reiserfs/journal.c.orig	Thu Jul 18 15:22:23 2002
+++ linux/fs/reiserfs/journal.c	Thu Jul 18 15:22:33 2002
@@ -151,8 +151,7 @@
   }
   bn = allocate_bitmap_node(p_s_sb) ;
   if (!bn) {
-    current->policy |= SCHED_YIELD ;
-    schedule() ;
+    yield();
     goto repeat ;
   }
   return bn ;
--- linux/fs/jffs2/background.c.orig	Thu Jul 18 15:22:02 2002
+++ linux/fs/jffs2/background.c	Thu Jul 18 15:22:33 2002
@@ -106,9 +106,6 @@
 
         sprintf(current->comm, "jffs2_gcd_mtd%d", c->mtd->index);
 
-	/* FIXME in the 2.2 backport */
-	current->nice = 10;
-
 	for (;;) {
 		spin_lock_irq(&current->sigmask_lock);
 		siginitsetinv (&current->blocked, sigmask(SIGHUP) | sigmask(SIGKILL) | sigmask(SIGSTOP) | sigmask(SIGCONT));
--- linux/fs/jbd/journal.c.orig	Thu Jul 18 15:22:23 2002
+++ linux/fs/jbd/journal.c	Thu Jul 18 15:22:33 2002
@@ -462,8 +462,7 @@
 			printk (KERN_NOTICE __FUNCTION__
 				": ENOMEM at get_unused_buffer_head, "
 				"trying again.\n");
-			current->policy |= SCHED_YIELD;
-			schedule();
+			yield();
 		}
 	} while (!new_bh);
 	/* keep subsequent assertions sane */
@@ -1543,8 +1542,7 @@
 			last_warning = jiffies;
 		}
 		
-		current->policy |= SCHED_YIELD;
-		schedule();
+		yield();
 	}
 }
 
@@ -1602,8 +1600,7 @@
 			last_warning = jiffies;
 		}
 		while (ret == 0) {
-			current->policy |= SCHED_YIELD;
-			schedule();
+			yield();
 			ret = kmem_cache_alloc(journal_head_cache, GFP_NOFS);
 		}
 	}
--- linux/fs/jbd/revoke.c.orig	Thu Jul 18 15:22:16 2002
+++ linux/fs/jbd/revoke.c	Thu Jul 18 15:22:33 2002
@@ -137,8 +137,7 @@
 	if (!journal_oom_retry)
 		return -ENOMEM;
 	jbd_debug(1, "ENOMEM in " __FUNCTION__ ", retrying.\n");
-	current->policy |= SCHED_YIELD;
-	schedule();
+	yield();
 	goto repeat;
 }
 
--- linux/fs/jbd/transaction.c.orig	Thu Jul 18 15:22:16 2002
+++ linux/fs/jbd/transaction.c	Thu Jul 18 15:22:33 2002
@@ -1379,8 +1379,7 @@
 		do {
 			old_handle_count = transaction->t_handle_count;
 			set_current_state(TASK_RUNNING);
-			current->policy |= SCHED_YIELD;
-			schedule();
+			yield();
 		} while (old_handle_count != transaction->t_handle_count);
 	}
 
--- linux/fs/binfmt_elf.c.orig	Thu Jul 18 15:22:23 2002
+++ linux/fs/binfmt_elf.c	Thu Jul 18 15:22:33 2002
@@ -1143,7 +1143,7 @@
 	psinfo.pr_state = i;
 	psinfo.pr_sname = (i < 0 || i > 5) ? '.' : "RSDZTD"[i];
 	psinfo.pr_zomb = psinfo.pr_sname == 'Z';
-	psinfo.pr_nice = current->nice;
+	psinfo.pr_nice = task_nice(current);
 	psinfo.pr_flag = current->flags;
 	psinfo.pr_uid = NEW_TO_OLD_UID(current->uid);
 	psinfo.pr_gid = NEW_TO_OLD_GID(current->gid);
--- linux/fs/buffer.c.orig	Thu Jul 18 15:22:23 2002
+++ linux/fs/buffer.c	Thu Jul 18 15:22:33 2002
@@ -736,9 +736,8 @@
 	wakeup_bdflush();
 	try_to_free_pages(zone, GFP_NOFS, 0);
 	run_task_queue(&tq_disk);
-	current->policy |= SCHED_YIELD;
 	__set_current_state(TASK_RUNNING);
-	schedule();
+	sys_sched_yield();
 }
 
 void init_buffer(struct buffer_head *bh, bh_end_io_t *handler, void *private)
--- linux/fs/locks.c.orig	Thu Jul 18 15:22:00 2002
+++ linux/fs/locks.c	Thu Jul 18 15:22:33 2002
@@ -445,8 +445,7 @@
 			/* Let the blocked process remove waiter from the
 			 * block list when it gets scheduled.
 			 */
-			current->policy |= SCHED_YIELD;
-			schedule();
+			yield();
 		} else {
 			/* Remove waiter from the block list, because by the
 			 * time it wakes up blocker won't exist any more.
--- linux/init/do_mounts.c.orig	Thu Jul 18 15:22:25 2002
+++ linux/init/do_mounts.c	Thu Jul 18 15:22:33 2002
@@ -776,10 +776,8 @@
 
 	pid = kernel_thread(do_linuxrc, "/linuxrc", SIGCHLD);
 	if (pid > 0) {
-		while (pid != wait(&i)) {
-			current->policy |= SCHED_YIELD;
-			schedule();
-		}
+		while (pid != wait(&i))
+			yield();
 	}
 
 	sys_mount("..", ".", NULL, MS_MOVE, NULL);
--- linux/init/main.c.orig	Thu Jul 18 15:22:25 2002
+++ linux/init/main.c	Thu Jul 18 15:22:33 2002
@@ -288,8 +288,6 @@
 extern void setup_arch(char **);
 extern void cpu_idle(void);
 
-unsigned long wait_init_idle;
-
 #ifndef CONFIG_SMP
 
 #ifdef CONFIG_X86_LOCAL_APIC
@@ -303,29 +301,19 @@
 
 #else
 
-
 /* Called by boot processor to activate the rest. */
 static void __init smp_init(void)
 {
 	/* Get other processors into their bootup holding patterns. */
 	smp_boot_cpus();
-	wait_init_idle = cpu_online_map;
-	clear_bit(current->processor, &wait_init_idle); /* Don't wait on me! */
 
 	smp_threads_ready=1;
 	smp_commence();
-
-	/* Wait for the other cpus to set up their idle processes */
-	printk("Waiting on wait_init_idle (map = 0x%lx)\n", wait_init_idle);
-	while (wait_init_idle) {
-		cpu_relax();
-		barrier();
-	}
-	printk("All processors have done init_idle\n");
 }
 
 #endif
 
+
 /*
  * We need to finalize in a non-__init function or else race conditions
  * between the root thread and the init thread may cause start_kernel to
@@ -337,9 +325,8 @@
 {
 	kernel_thread(init, NULL, CLONE_FS | CLONE_FILES | CLONE_SIGNAL);
 	unlock_kernel();
-	current->need_resched = 1;
- 	cpu_idle();
-} 
+	cpu_idle();
+}
 
 /*
  *	Activate the first processor.
@@ -424,14 +411,18 @@
 	ipc_init();
 #endif
 	check_bugs();
+
 	printk("POSIX conformance testing by UNIFIX\n");
 
-	/* 
-	 *	We count on the initial thread going ok 
-	 *	Like idlers init is an unlocked kernel thread, which will
-	 *	make syscalls (and thus be locked).
+	init_idle(current, smp_processor_id());
+	/*
+	 *      We count on the initial thread going ok
+	 *      Like idlers init is an unlocked kernel thread, which will
+	 *      make syscalls (and thus be locked).
 	 */
 	smp_init();
+
+	/* Do the rest non-__init'ed, we're now alive */
 	rest_init();
 }
 
@@ -460,6 +451,10 @@
  */
 static void __init do_basic_setup(void)
 {
+	/* Start the per-CPU migration threads */
+#if CONFIG_SMP
+	migration_init();
+#endif
 
 	/*
 	 * Tell the world that we're going to be the grim
--- linux/kernel/capability.c.orig	Sat Jun 24 06:06:37 2000
+++ linux/kernel/capability.c	Thu Jul 18 15:22:33 2002
@@ -8,6 +8,8 @@
 #include <linux/mm.h>
 #include <asm/uaccess.h>
 
+unsigned securebits = SECUREBITS_DEFAULT; /* systemwide security settings */
+
 kernel_cap_t cap_bset = CAP_INIT_EFF_SET;
 
 /* Note: never hold tasklist_lock while spinning for this one */
--- linux/kernel/exit.c.orig	Thu Jul 18 15:22:25 2002
+++ linux/kernel/exit.c	Thu Jul 18 15:22:33 2002
@@ -28,49 +28,22 @@
 
 static void release_task(struct task_struct * p)
 {
-	if (p != current) {
+	if (p == current)
+		BUG();
 #ifdef CONFIG_SMP
-		/*
-		 * Wait to make sure the process isn't on the
-		 * runqueue (active on some other CPU still)
-		 */
-		for (;;) {
-			task_lock(p);
-			if (!task_has_cpu(p))
-				break;
-			task_unlock(p);
-			do {
-				cpu_relax();
-				barrier();
-			} while (task_has_cpu(p));
-		}
-		task_unlock(p);
+	wait_task_inactive(p);
 #endif
-		atomic_dec(&p->user->processes);
-		free_uid(p->user);
-		unhash_process(p);
-
-		release_thread(p);
-		current->cmin_flt += p->min_flt + p->cmin_flt;
-		current->cmaj_flt += p->maj_flt + p->cmaj_flt;
-		current->cnswap += p->nswap + p->cnswap;
-		/*
-		 * Potentially available timeslices are retrieved
-		 * here - this way the parent does not get penalized
-		 * for creating too many processes.
-		 *
-		 * (this cannot be used to artificially 'generate'
-		 * timeslices, because any timeslice recovered here
-		 * was given away by the parent in the first place.)
-		 */
-		current->counter += p->counter;
-		if (current->counter >= MAX_COUNTER)
-			current->counter = MAX_COUNTER;
-		p->pid = 0;
-		free_task_struct(p);
-	} else {
-		printk("task releasing itself\n");
-	}
+	atomic_dec(&p->user->processes);
+	free_uid(p->user);
+	unhash_process(p);
+
+	release_thread(p);
+	current->cmin_flt += p->min_flt + p->cmin_flt;
+	current->cmaj_flt += p->maj_flt + p->cmaj_flt;
+	current->cnswap += p->nswap + p->cnswap;
+	sched_exit(p);
+	p->pid = 0;
+	free_task_struct(p);
 }
 
 /*
@@ -148,6 +121,79 @@
 	}
 	read_unlock(&tasklist_lock);
 	return retval;
+}
+
+/**
+ * reparent_to_init() - Reparent the calling kernel thread to the init task.
+ *
+ * If a kernel thread is launched as a result of a system call, or if
+ * it ever exits, it should generally reparent itself to init so that
+ * it is correctly cleaned up on exit.
+ *
+ * The various task state such as scheduling policy and priority may have
+ * been inherited from a user process, so we reset them to sane values here.
+ *
+ * NOTE that reparent_to_init() gives the caller full capabilities.
+ */
+void reparent_to_init(void)
+{
+	write_lock_irq(&tasklist_lock);
+
+	/* Reparent to init */
+	REMOVE_LINKS(current);
+	current->p_pptr = child_reaper;
+	current->p_opptr = child_reaper;
+	SET_LINKS(current);
+
+	/* Set the exit signal to SIGCHLD so we signal init on exit */
+	current->exit_signal = SIGCHLD;
+
+	current->ptrace = 0;
+	if ((current->policy == SCHED_OTHER) && (task_nice(current) < 0))
+		set_user_nice(current, 0);
+	/* cpus_allowed? */
+	/* rt_priority? */
+	/* signals? */
+	current->cap_effective = CAP_INIT_EFF_SET;
+	current->cap_inheritable = CAP_INIT_INH_SET;
+	current->cap_permitted = CAP_FULL_SET;
+	current->keep_capabilities = 0;
+	memcpy(current->rlim, init_task.rlim, sizeof(*(current->rlim)));
+	current->user = INIT_USER;
+
+	write_unlock_irq(&tasklist_lock);
+}
+
+/*
+ *	Put all the gunge required to become a kernel thread without
+ *	attached user resources in one place where it belongs.
+ */
+
+void daemonize(void)
+{
+	struct fs_struct *fs;
+
+
+	/*
+	 * If we were started as result of loading a module, close all of the
+	 * user space pages.  We don't need them, and if we didn't close them
+	 * they would be locked into memory.
+	 */
+	exit_mm(current);
+
+	current->session = 1;
+	current->pgrp = 1;
+	current->tty = NULL;
+
+	/* Become as one with the init task */
+
+	exit_fs(current);	/* current->fs->count--; */
+	fs = init_task.fs;
+	current->fs = fs;
+	atomic_inc(&fs->count);
+ 	exit_files(current);
+	current->files = init_task.files;
+	atomic_inc(&current->files->count);
 }
 
 /*
--- linux/kernel/fork.c.orig	Thu Jul 18 15:22:25 2002
+++ linux/kernel/fork.c	Fri Jul 19 15:01:41 2002
@@ -30,7 +30,6 @@
 
 /* The idle threads do not count.. */
 int nr_threads;
-int nr_running;
 
 int max_threads;
 unsigned long total_forks;	/* Handle normal Linux uptimes. */
@@ -38,6 +37,8 @@
 
 struct task_struct *pidhash[PIDHASH_SZ];
 
+rwlock_t tasklist_lock __cacheline_aligned = RW_LOCK_UNLOCKED;  /* outer */
+
 void add_wait_queue(wait_queue_head_t *q, wait_queue_t * wait)
 {
 	unsigned long flags;
@@ -638,8 +639,7 @@
 	if (p->pid == 0 && current->pid != 0)
 		goto bad_fork_cleanup;
 
-	p->run_list.next = NULL;
-	p->run_list.prev = NULL;
+	INIT_LIST_HEAD(&p->run_list);
 
 	p->p_cptr = NULL;
 	init_waitqueue_head(&p->wait_chldexit);
@@ -665,14 +665,15 @@
 #ifdef CONFIG_SMP
 	{
 		int i;
-		p->cpus_runnable = ~0UL;
-		p->processor = current->processor;
+
 		/* ?? should we just memset this ?? */
 		for(i = 0; i < smp_num_cpus; i++)
-			p->per_cpu_utime[i] = p->per_cpu_stime[i] = 0;
+			p->per_cpu_utime[cpu_logical_map(i)] =
+				p->per_cpu_stime[cpu_logical_map(i)] = 0;
 		spin_lock_init(&p->sigmask_lock);
 	}
 #endif
+	p->array = NULL;
 	p->lock_depth = -1;		/* -1 = no lock */
 	p->start_time = jiffies;
 
@@ -706,15 +707,27 @@
 	p->pdeath_signal = 0;
 
 	/*
-	 * "share" dynamic priority between parent and child, thus the
-	 * total amount of dynamic priorities in the system doesnt change,
-	 * more scheduling fairness. This is only important in the first
-	 * timeslice, on the long run the scheduling behaviour is unchanged.
-	 */
-	p->counter = (current->counter + 1) >> 1;
-	current->counter >>= 1;
-	if (!current->counter)
-		current->need_resched = 1;
+	 * Share the timeslice between parent and child, thus the
+	 * total amount of pending timeslices in the system doesnt change,
+	 * resulting in more scheduling fairness.
+	 */
+	__cli();
+	if (!current->time_slice)
+		BUG();
+	p->time_slice = (current->time_slice + 1) >> 1;
+	p->first_time_slice = 1;
+	current->time_slice >>= 1;
+	if (!current->time_slice) {
+		/*
+		 * This case is rare, it happens when the parent has only
+		 * a single jiffy left from its timeslice. Taking the
+		 * runqueue lock is not a problem.
+		 */
+		current->time_slice = 1;
+		scheduler_tick(0,0);
+	}
+	p->sleep_timestamp = jiffies;
+	__sti();
 
 	/*
 	 * Ok, add it to the run-queues and make it
@@ -750,11 +763,16 @@
 
 	if (p->ptrace & PT_PTRACED)
 		send_sig(SIGSTOP, p, 1);
-
-	wake_up_process(p);		/* do this last */
+	wake_up_forked_process(p);	/* do this last */
 	++total_forks;
 	if (clone_flags & CLONE_VFORK)
 		wait_for_completion(&vfork);
+	else
+		/*
+		 * Let the child process run first, to avoid most of the
+		 * COW overhead when the child exec()s afterwards.
+		 */
+		current->need_resched = 1;
 
 fork_out:
 	return retval;
--- linux/kernel/ksyms.c.orig	Thu Jul 18 15:22:25 2002
+++ linux/kernel/ksyms.c	Thu Jul 18 15:22:33 2002
@@ -439,7 +439,9 @@
 /* process management */
 EXPORT_SYMBOL(complete_and_exit);
 EXPORT_SYMBOL(__wake_up);
-EXPORT_SYMBOL(__wake_up_sync);
+#if CONFIG_SMP
+EXPORT_SYMBOL_GPL(__wake_up_sync); /* internal use only */
+#endif
 EXPORT_SYMBOL(wake_up_process);
 EXPORT_SYMBOL(sleep_on);
 EXPORT_SYMBOL(sleep_on_timeout);
@@ -448,6 +450,12 @@
 EXPORT_SYMBOL(schedule);
 EXPORT_SYMBOL(schedule_timeout);
 EXPORT_SYMBOL(sys_sched_yield);
+EXPORT_SYMBOL(set_user_nice);
+EXPORT_SYMBOL(task_nice);
+EXPORT_SYMBOL_GPL(idle_cpu);
+#ifdef CONFIG_SMP
+EXPORT_SYMBOL_GPL(set_cpus_allowed);
+#endif
 EXPORT_SYMBOL(jiffies);
 EXPORT_SYMBOL(xtime);
 EXPORT_SYMBOL(do_gettimeofday);
@@ -459,6 +467,7 @@
 
 EXPORT_SYMBOL(kstat);
 EXPORT_SYMBOL(nr_running);
+EXPORT_SYMBOL(nr_context_switches);
 
 /* misc */
 EXPORT_SYMBOL(panic);
--- linux/kernel/ptrace.c.orig	Thu Jul 18 15:22:25 2002
+++ linux/kernel/ptrace.c	Thu Jul 18 15:22:33 2002
@@ -31,20 +31,7 @@
 		if (child->state != TASK_STOPPED)
 			return -ESRCH;
 #ifdef CONFIG_SMP
-		/* Make sure the child gets off its CPU.. */
-		for (;;) {
-			task_lock(child);
-			if (!task_has_cpu(child))
-				break;
-			task_unlock(child);
-			do {
-				if (child->state != TASK_STOPPED)
-					return -ESRCH;
-				barrier();
-				cpu_relax();
-			} while (task_has_cpu(child));
-		}
-		task_unlock(child);
+		wait_task_inactive(child);
 #endif		
 	}
 
--- linux/kernel/sched.c.orig	Thu Jul 18 15:22:25 2002
+++ linux/kernel/sched.c	Fri Jul 19 14:59:03 2002
@@ -1,343 +1,421 @@
 /*
- *  linux/kernel/sched.c
+ *  kernel/sched.c
  *
  *  Kernel scheduler and related syscalls
  *
- *  Copyright (C) 1991, 1992  Linus Torvalds
+ *  Copyright (C) 1991-2002  Linus Torvalds
  *
  *  1996-12-23  Modified by Dave Grothe to fix bugs in semaphores and
- *              make semaphores SMP safe
+ *		make semaphores SMP safe
  *  1998-11-19	Implemented schedule_timeout() and related stuff
  *		by Andrea Arcangeli
- *  1998-12-28  Implemented better SMP scheduling by Ingo Molnar
+ *  2002-01-04	New ultra-scalable O(1) scheduler by Ingo Molnar:
+ *		hybrid priority-list and round-robin design with
+ *		an array-switch method of distributing timeslices
+ *		and per-CPU runqueues.  Additional code by Davide
+ *		Libenzi, Robert Love, and Rusty Russell.
  */
 
-/*
- * 'sched.c' is the main kernel file. It contains scheduling primitives
- * (sleep_on, wakeup, schedule etc) as well as a number of simple system
- * call functions (type getpid()), which just extract a field from
- * current-task
- */
-
-#include <linux/config.h>
 #include <linux/mm.h>
+#include <linux/nmi.h>
 #include <linux/init.h>
+#include <asm/uaccess.h>
+#include <linux/highmem.h>
 #include <linux/smp_lock.h>
-#include <linux/nmi.h>
+#include <asm/mmu_context.h>
 #include <linux/interrupt.h>
-#include <linux/kernel_stat.h>
 #include <linux/completion.h>
-#include <linux/prefetch.h>
-#include <linux/compiler.h>
+#include <linux/kernel_stat.h>
 
-#include <asm/uaccess.h>
-#include <asm/mmu_context.h>
+/*
+ * Convert user-nice values [ -20 ... 0 ... 19 ]
+ * to static priority [ MAX_RT_PRIO..MAX_PRIO-1 ],
+ * and back.
+ */
+#define NICE_TO_PRIO(nice)	(MAX_RT_PRIO + (nice) + 20)
+#define PRIO_TO_NICE(prio)	((prio) - MAX_RT_PRIO - 20)
+#define TASK_NICE(p)		PRIO_TO_NICE((p)->static_prio)
 
-extern void timer_bh(void);
-extern void tqueue_bh(void);
-extern void immediate_bh(void);
+/*
+ * 'User priority' is the nice value converted to something we
+ * can work with better when scaling various scheduler parameters,
+ * it's a [ 0 ... 39 ] range.
+ */
+#define USER_PRIO(p)		((p)-MAX_RT_PRIO)
+#define TASK_USER_PRIO(p)	USER_PRIO((p)->static_prio)
+#define MAX_USER_PRIO		(USER_PRIO(MAX_PRIO))
+
+/*
+ * These are the 'tuning knobs' of the scheduler:
+ *
+ * Minimum timeslice is 10 msecs, default timeslice is 150 msecs,
+ * maximum timeslice is 300 msecs. Timeslices get refilled after
+ * they expire.
+ */
+#define MIN_TIMESLICE		( 10 * HZ / 1000)
+#define MAX_TIMESLICE		(300 * HZ / 1000)
+#define CHILD_PENALTY		95
+#define PARENT_PENALTY		100
+#define EXIT_WEIGHT		3
+#define PRIO_BONUS_RATIO	25
+#define INTERACTIVE_DELTA	2
+#define MAX_SLEEP_AVG		(2*HZ)
+#define STARVATION_LIMIT	(2*HZ)
 
 /*
- * scheduler variables
+ * If a task is 'interactive' then we reinsert it in the active
+ * array after it has expired its current timeslice. (it will not
+ * continue to run immediately, it will still roundrobin with
+ * other interactive tasks.)
+ *
+ * This part scales the interactivity limit depending on niceness.
+ *
+ * We scale it linearly, offset by the INTERACTIVE_DELTA delta.
+ * Here are a few examples of different nice levels:
+ *
+ *  TASK_INTERACTIVE(-20): [1,1,1,1,1,1,1,1,1,0,0]
+ *  TASK_INTERACTIVE(-10): [1,1,1,1,1,1,1,0,0,0,0]
+ *  TASK_INTERACTIVE(  0): [1,1,1,1,0,0,0,0,0,0,0]
+ *  TASK_INTERACTIVE( 10): [1,1,0,0,0,0,0,0,0,0,0]
+ *  TASK_INTERACTIVE( 19): [0,0,0,0,0,0,0,0,0,0,0]
+ *
+ * (the X axis represents the possible -5 ... 0 ... +5 dynamic
+ *  priority range a task can explore, a value of '1' means the
+ *  task is rated interactive.)
+ *
+ * Ie. nice +19 tasks can never get 'interactive' enough to be
+ * reinserted into the active array. And only heavily CPU-hog nice -20
+ * tasks will be expired. Default nice 0 tasks are somewhere between,
+ * it takes some effort for them to get interactive, but it's not
+ * too hard.
  */
 
-unsigned securebits = SECUREBITS_DEFAULT; /* systemwide security settings */
+#define SCALE(v1,v1_max,v2_max) \
+	(v1) * (v2_max) / (v1_max)
 
-extern void mem_use(void);
+#define DELTA(p) \
+	(SCALE(TASK_NICE(p), 40, MAX_USER_PRIO*PRIO_BONUS_RATIO/100) + \
+		INTERACTIVE_DELTA)
+
+#define TASK_INTERACTIVE(p) \
+	((p)->prio <= (p)->static_prio - DELTA(p))
 
 /*
- * Scheduling quanta.
- *
- * NOTE! The unix "nice" value influences how long a process
- * gets. The nice value ranges from -20 to +19, where a -20
- * is a "high-priority" task, and a "+10" is a low-priority
- * task.
+ * BASE_TIMESLICE scales user-nice values [ -20 ... 19 ]
+ * to time slice values.
  *
- * We want the time-slice to be around 50ms or so, so this
- * calculation depends on the value of HZ.
+ * The higher a process's priority, the bigger timeslices
+ * it gets during one round of execution. But even the lowest
+ * priority process gets MIN_TIMESLICE worth of execution time.
+ *
+ * task_timeslice() is the interface that is used by the scheduler.
+ * SCHED_BATCH tasks get longer timeslices to make use of better
+ * caching. They are inherently noninteractive and they are
+ * immediately preempted by SCHED_NORMAL tasks so there is no
+ * downside in using shorter timeslices.
  */
-#if HZ < 200
-#define TICK_SCALE(x)	((x) >> 2)
-#elif HZ < 400
-#define TICK_SCALE(x)	((x) >> 1)
-#elif HZ < 800
-#define TICK_SCALE(x)	(x)
-#elif HZ < 1600
-#define TICK_SCALE(x)	((x) << 1)
-#else
-#define TICK_SCALE(x)	((x) << 2)
-#endif
 
-#define NICE_TO_TICKS(nice)	(TICK_SCALE(20-(nice))+1)
+#define BASE_TIMESLICE(p) (MIN_TIMESLICE + \
+	((MAX_TIMESLICE - MIN_TIMESLICE) * (MAX_PRIO-1-(p)->static_prio)/(MAX_USER_PRIO - 1)))
 
+static inline unsigned int task_timeslice(task_t *p)
+{
+	if (p->policy == SCHED_BATCH)
+		return BASE_TIMESLICE(p) * 10;
+	else
+		return BASE_TIMESLICE(p);
+}
 
 /*
- *	Init task must be ok at boot for the ix86 as we will check its signals
- *	via the SMP irq return path.
+ * These are the runqueue data structures:
  */
- 
-struct task_struct * init_tasks[NR_CPUS] = {&init_task, };
+
+#define BITMAP_SIZE ((((MAX_PRIO+1+7)/8)+sizeof(long)-1)/sizeof(long))
+
+typedef struct runqueue runqueue_t;
+
+struct prio_array {
+	int nr_active;
+	unsigned long bitmap[BITMAP_SIZE];
+	list_t queue[MAX_PRIO];
+};
 
 /*
- * The tasklist_lock protects the linked list of processes.
- *
- * The runqueue_lock locks the parts that actually access
- * and change the run-queues, and have to be interrupt-safe.
+ * This is the main, per-CPU runqueue data structure.
  *
- * If both locks are to be concurrently held, the runqueue_lock
- * nests inside the tasklist_lock.
- *
- * task->alloc_lock nests inside tasklist_lock.
+ * Locking rule: those places that want to lock multiple runqueues
+ * (such as the load balancing or the process migration code), lock
+ * acquire operations must be ordered by ascending &runqueue.
  */
-spinlock_t runqueue_lock __cacheline_aligned = SPIN_LOCK_UNLOCKED;  /* inner */
-rwlock_t tasklist_lock __cacheline_aligned = RW_LOCK_UNLOCKED;	/* outer */
+struct runqueue {
+	spinlock_t lock;
+	unsigned long nr_running, nr_switches, expired_timestamp,
+			nr_uninterruptible;
+	task_t *curr, *idle;
+	prio_array_t *active, *expired, arrays[2];
+	int prev_nr_running[NR_CPUS];
+
+	task_t *migration_thread;
+	list_t migration_queue;
 
-static LIST_HEAD(runqueue_head);
+	/*
+	 * The batch queue is a secondary ready-queue:
+	 */
+	unsigned long nr_batch;
+	list_t batch_queue;
+
+	/*
+	 * Per-CPU idle CPU time tracking:
+	 *
+	 * - idle_ticks_left counts back from HZ to 0.
+	 * - idle_count is the number of idle ticks in the last second.
+	 * - once it reaches 0, a new idle_avg is calculated.
+	 */
+	#define IDLE_TICKS (HZ)
+
+	unsigned int idle_ticks_left, idle_count, idle_avg;
+
+} ____cacheline_aligned;
+
+static struct runqueue runqueues[NR_CPUS] __cacheline_aligned;
+
+#define cpu_rq(cpu)		(runqueues + (cpu))
+#define this_rq()		cpu_rq(smp_processor_id())
+#define task_rq(p)		cpu_rq(task_cpu(p))
+#define cpu_curr(cpu)		(cpu_rq(cpu)->curr)
+#define rt_task(p)		((p)->prio < MAX_RT_PRIO)
 
 /*
- * We align per-CPU scheduling data on cacheline boundaries,
- * to prevent cacheline ping-pong.
+ * Default context-switch locking:
  */
-static union {
-	struct schedule_data {
-		struct task_struct * curr;
-		cycles_t last_schedule;
-	} schedule_data;
-	char __pad [SMP_CACHE_BYTES];
-} aligned_data [NR_CPUS] __cacheline_aligned = { {{&init_task,0}}};
-
-#define cpu_curr(cpu) aligned_data[(cpu)].schedule_data.curr
-#define last_schedule(cpu) aligned_data[(cpu)].schedule_data.last_schedule
+#ifndef prepare_arch_switch
+# define prepare_arch_switch(rq, next) do { } while(0)
+# define finish_arch_switch(rq, next)  spin_unlock_irq(&(rq)->lock)
+# define task_running(rq, p)           ((rq)->curr == (p))
+#endif
 
-struct kernel_stat kstat;
-extern struct task_struct *child_reaper;
+/*
+ * task_rq_lock - lock the runqueue a given task resides on and disable
+ * interrupts.  Note the ordering: we can safely lookup the task_rq without
+ * explicitly disabling preemption.
+ */
+static inline runqueue_t *task_rq_lock(task_t *p, unsigned long *flags)
+{
+	struct runqueue *rq;
 
-#ifdef CONFIG_SMP
+repeat_lock_task:
+	local_irq_save(*flags);
+	rq = task_rq(p);
+	spin_lock(&rq->lock);
+	if (unlikely(rq != task_rq(p))) {
+		spin_unlock_irqrestore(&rq->lock, *flags);
+		goto repeat_lock_task;
+	}
+	return rq;
+}
 
-#define idle_task(cpu) (init_tasks[cpu_number_map(cpu)])
-#define can_schedule(p,cpu) \
-	((p)->cpus_runnable & (p)->cpus_allowed & (1 << cpu))
+static inline void task_rq_unlock(runqueue_t *rq, unsigned long *flags)
+{
+	spin_unlock_irqrestore(&rq->lock, *flags);
+}
 
-#else
+/*
+ * rq_lock - lock a given runqueue and disable interrupts.
+ */
+static inline runqueue_t *this_rq_lock(void)
+{
+	runqueue_t *rq;
 
-#define idle_task(cpu) (&init_task)
-#define can_schedule(p,cpu) (1)
+	local_irq_disable();
+	rq = this_rq();
+	spin_lock(&rq->lock);
 
-#endif
+	return rq;
+}
 
-void scheduling_functions_start_here(void) { }
+static inline void rq_unlock(runqueue_t *rq)
+{
+	spin_unlock(&rq->lock);
+	local_irq_enable();
+}
 
 /*
- * This is the function that decides how desirable a process is..
- * You can weigh different processes against each other depending
- * on what CPU they've run on lately etc to try to handle cache
- * and TLB miss penalties.
- *
- * Return values:
- *	 -1000: never select this
- *	     0: out of time, recalculate counters (but it might still be
- *		selected)
- *	   +ve: "goodness" value (the larger, the better)
- *	 +1000: realtime process, select this.
+ * Adding/removing a task to/from a priority array:
  */
+static inline void dequeue_task(struct task_struct *p, prio_array_t *array)
+{
+	array->nr_active--;
+	list_del(&p->run_list);
+	if (list_empty(array->queue + p->prio))
+		__clear_bit(p->prio, array->bitmap);
+}
 
-static inline int goodness(struct task_struct * p, int this_cpu, struct mm_struct *this_mm)
+static inline void enqueue_task(struct task_struct *p, prio_array_t *array)
 {
-	int weight;
+	list_add_tail(&p->run_list, array->queue + p->prio);
+	__set_bit(p->prio, array->bitmap);
+	array->nr_active++;
+	p->array = array;
+}
 
-	/*
-	 * select the current process after every other
-	 * runnable process, but before the idle thread.
-	 * Also, dont trigger a counter recalculation.
-	 */
-	weight = -1;
-	if (p->policy & SCHED_YIELD)
-		goto out;
+static inline int effective_prio(task_t *p)
+{
+	int bonus, prio;
 
 	/*
-	 * Non-RT process - normal case first.
+	 * Here we scale the actual sleep average [0 .... MAX_SLEEP_AVG]
+	 * into the -5 ... 0 ... +5 bonus/penalty range.
+	 *
+	 * We use 25% of the full 0...39 priority range so that:
+	 *
+	 * 1) nice +19 interactive tasks do not preempt nice 0 CPU hogs.
+	 * 2) nice -20 CPU hogs do not get preempted by nice 0 tasks.
+	 *
+	 * Both properties are important to certain workloads.
 	 */
-	if (p->policy == SCHED_OTHER) {
+	bonus = MAX_USER_PRIO*PRIO_BONUS_RATIO*p->sleep_avg/MAX_SLEEP_AVG/100 -
+			MAX_USER_PRIO*PRIO_BONUS_RATIO/100/2;
+
+	prio = p->static_prio - bonus;
+	if (prio < MAX_RT_PRIO)
+		prio = MAX_RT_PRIO;
+	if (prio > MAX_PRIO-1)
+		prio = MAX_PRIO-1;
+	return prio;
+}
+
+static inline void activate_task(task_t *p, runqueue_t *rq)
+{
+	unsigned long sleep_time = jiffies - p->sleep_timestamp;
+	prio_array_t *array = rq->active;
+
+	if (!rt_task(p) && sleep_time) {
 		/*
-		 * Give the process a first-approximation goodness value
-		 * according to the number of clock-ticks it has left.
-		 *
-		 * Don't do any other calculations if the time slice is
-		 * over..
+		 * This code gives a bonus to interactive tasks. We update
+		 * an 'average sleep time' value here, based on
+		 * sleep_timestamp. The more time a task spends sleeping,
+		 * the higher the average gets - and the higher the priority
+		 * boost gets as well.
 		 */
-		weight = p->counter;
-		if (!weight)
-			goto out;
-			
-#ifdef CONFIG_SMP
-		/* Give a largish advantage to the same processor...   */
-		/* (this is equivalent to penalizing other processors) */
-		if (p->processor == this_cpu)
-			weight += PROC_CHANGE_PENALTY;
-#endif
-
-		/* .. and a slight advantage to the current MM */
-		if (p->mm == this_mm || !p->mm)
-			weight += 1;
-		weight += 20 - p->nice;
-		goto out;
+		p->sleep_avg += sleep_time;
+		if (p->sleep_avg > MAX_SLEEP_AVG)
+			p->sleep_avg = MAX_SLEEP_AVG;
+		p->prio = effective_prio(p);
 	}
+	enqueue_task(p, array);
+	rq->nr_running++;
+}
+
+static inline void activate_batch_task(task_t *p, runqueue_t *rq)
+{
+	rq->nr_batch--;
+	list_del(&p->run_list);
+	activate_task(p, rq);
+	p->flags &= ~PF_BATCH;
+}
+
+static inline void deactivate_task(struct task_struct *p, runqueue_t *rq)
+{
+	rq->nr_running--;
+	if (p->state == TASK_UNINTERRUPTIBLE)
+		rq->nr_uninterruptible++;
+	dequeue_task(p, p->array);
+	p->array = NULL;
+}
 
+static inline void deactivate_batch_task(task_t *p, runqueue_t *rq)
+{
+	prio_array_t *array = p->array;
+
+	deactivate_task(p, rq);
+	rq->nr_batch++;
+	if (array == rq->expired)
+		list_add_tail(&p->run_list, &rq->batch_queue);
+	else
+		list_add(&p->run_list, &rq->batch_queue);
 	/*
-	 * Realtime process, select the first one on the
-	 * runqueue (taking priorities within processes
-	 * into account).
+	 * Via this bit we can tell whether a task is in the batchqueue,
+	 * this information is not available in any other cheap way.
 	 */
-	weight = 1000 + p->rt_priority;
-out:
-	return weight;
+	p->flags |= PF_BATCH;
 }
 
-/*
- * the 'goodness value' of replacing a process on a given CPU.
- * positive value means 'replace', zero or negative means 'dont'.
- */
-static inline int preemption_goodness(struct task_struct * prev, struct task_struct * p, int cpu)
+static inline void resched_task(task_t *p)
 {
-	return goodness(p, cpu, prev->active_mm) - goodness(prev, cpu, prev->active_mm);
+#if CONFIG_SMP
+	int need_resched;
+
+	need_resched = p->need_resched;
+	wmb();
+	set_tsk_need_resched(p);
+	if (!need_resched && (p->cpu != smp_processor_id()))
+		smp_send_reschedule(p->cpu);
+#else
+	set_tsk_need_resched(p);
+#endif
 }
 
+#ifdef CONFIG_SMP
+
 /*
- * This is ugly, but reschedule_idle() is very timing-critical.
- * We are called with the runqueue spinlock held and we must
- * not claim the tasklist_lock.
+ * Wait for a process to unschedule. This is used by the exit() and
+ * ptrace() code.
  */
-static FASTCALL(void reschedule_idle(struct task_struct * p));
-
-static void reschedule_idle(struct task_struct * p)
+void wait_task_inactive(task_t * p)
 {
-#ifdef CONFIG_SMP
-	int this_cpu = smp_processor_id();
-	struct task_struct *tsk, *target_tsk;
-	int cpu, best_cpu, i, max_prio;
-	cycles_t oldest_idle;
-
-	/*
-	 * shortcut if the woken up task's last CPU is
-	 * idle now.
-	 */
-	best_cpu = p->processor;
-	if (can_schedule(p, best_cpu)) {
-		tsk = idle_task(best_cpu);
-		if (cpu_curr(best_cpu) == tsk) {
-			int need_resched;
-send_now_idle:
-			/*
-			 * If need_resched == -1 then we can skip sending
-			 * the IPI altogether, tsk->need_resched is
-			 * actively watched by the idle thread.
-			 */
-			need_resched = tsk->need_resched;
-			tsk->need_resched = 1;
-			if ((best_cpu != this_cpu) && !need_resched)
-				smp_send_reschedule(best_cpu);
-			return;
-		}
-	}
+	unsigned long flags;
+	runqueue_t *rq;
 
-	/*
-	 * We know that the preferred CPU has a cache-affine current
-	 * process, lets try to find a new idle CPU for the woken-up
-	 * process. Select the least recently active idle CPU. (that
-	 * one will have the least active cache context.) Also find
-	 * the executing process which has the least priority.
-	 */
-	oldest_idle = (cycles_t) -1;
-	target_tsk = NULL;
-	max_prio = 0;
-
-	for (i = 0; i < smp_num_cpus; i++) {
-		cpu = cpu_logical_map(i);
-		if (!can_schedule(p, cpu))
-			continue;
-		tsk = cpu_curr(cpu);
+repeat:
+	rq = task_rq(p);
+	if (unlikely(task_running(rq, p))) {
+		cpu_relax();
 		/*
-		 * We use the first available idle CPU. This creates
-		 * a priority list between idle CPUs, but this is not
-		 * a problem.
+		 * enable/disable preemption just to make this
+		 * a preemption point - we are busy-waiting
+		 * anyway.
 		 */
-		if (tsk == idle_task(cpu)) {
-#if defined(__i386__) && defined(CONFIG_SMP)
-                        /*
-			 * Check if two siblings are idle in the same
-			 * physical package. Use them if found.
-			 */
-			if (smp_num_siblings == 2) {
-				if (cpu_curr(cpu_sibling_map[cpu]) == 
-			            idle_task(cpu_sibling_map[cpu])) {
-					oldest_idle = last_schedule(cpu);
-					target_tsk = tsk;
-					break;
-				}
-				
-                        }
-#endif		
-			if (last_schedule(cpu) < oldest_idle) {
-				oldest_idle = last_schedule(cpu);
-				target_tsk = tsk;
-			}
-		} else {
-			if (oldest_idle == -1ULL) {
-				int prio = preemption_goodness(tsk, p, cpu);
-
-				if (prio > max_prio) {
-					max_prio = prio;
-					target_tsk = tsk;
-				}
-			}
-		}
+		goto repeat;
 	}
-	tsk = target_tsk;
-	if (tsk) {
-		if (oldest_idle != -1ULL) {
-			best_cpu = tsk->processor;
-			goto send_now_idle;
-		}
-		tsk->need_resched = 1;
-		if (tsk->processor != this_cpu)
-			smp_send_reschedule(tsk->processor);
+	rq = task_rq_lock(p, &flags);
+	if (unlikely(task_running(rq, p))) {
+		task_rq_unlock(rq, &flags);
+		goto repeat;
 	}
-	return;
-		
-
-#else /* UP */
-	int this_cpu = smp_processor_id();
-	struct task_struct *tsk;
-
-	tsk = cpu_curr(this_cpu);
-	if (preemption_goodness(tsk, p, this_cpu) > 0)
-		tsk->need_resched = 1;
-#endif
+	task_rq_unlock(rq, &flags);
 }
+#endif
 
 /*
- * Careful!
+ * Kick the remote CPU if the task is running currently,
+ * this code is used by the signal code to signal tasks
+ * which are in user-mode as quickly as possible.
+ *
+ * (Note that we do this lockless - if the task does anything
+ * while the message is in flight then it will notice the
+ * sigpending condition anyway.)
  *
- * This has to add the process to the _beginning_ of the
- * run-queue, not the end. See the comment about "This is
- * subtle" in the scheduler proper..
+ * this code also activates batch processes if they get a signal.
  */
-static inline void add_to_runqueue(struct task_struct * p)
-{
-	list_add(&p->run_list, &runqueue_head);
-	nr_running++;
-}
-
-static inline void move_last_runqueue(struct task_struct * p)
+void kick_if_running(task_t * p)
 {
-	list_del(&p->run_list);
-	list_add_tail(&p->run_list, &runqueue_head);
-}
+	if (task_running(task_rq(p), p) && (p->cpu != smp_processor_id()))
+		resched_task(p);
+	/*
+	 * If batch processes get signals but are not running currently
+	 * then give them a chance to handle the signal. (the kernel
+	 * side signal handling code will run for sure, the userspace
+	 * part depends on system load and might be delayed indefinitely.)
+	 */
+	if (p->policy == SCHED_BATCH) {
+		unsigned long flags;
+		runqueue_t *rq;
 
-static inline void move_first_runqueue(struct task_struct * p)
-{
-	list_del(&p->run_list);
-	list_add(&p->run_list, &runqueue_head);
+		rq = task_rq_lock(p, &flags);
+		if (p->flags & PF_BATCH)
+			activate_batch_task(p, rq);
+		task_rq_unlock(rq, &flags);
+	}
 }
 
 /*
@@ -347,416 +425,793 @@
  * progress), and as such you're allowed to do the simpler
  * "current->state = TASK_RUNNING" to mark yourself runnable
  * without the overhead of this.
+ *
+ * returns failure only if the task is already active.
  */
-static inline int try_to_wake_up(struct task_struct * p, int synchronous)
+static int try_to_wake_up(task_t * p, int sync)
 {
 	unsigned long flags;
 	int success = 0;
+	long old_state;
+	runqueue_t *rq;
 
-	/*
-	 * We want the common case fall through straight, thus the goto.
-	 */
-	spin_lock_irqsave(&runqueue_lock, flags);
+repeat_lock_task:
+	rq = task_rq_lock(p, &flags);
+	old_state = p->state;
+	if (!p->array) {
+		/*
+		 * Fast-migrate the task if it's not running or runnable
+		 * currently. Do not violate hard affinity.
+		 */
+		if (unlikely(sync && !task_running(rq, p) &&
+			(task_cpu(p) != smp_processor_id()) &&
+			(p->cpus_allowed & (1UL << smp_processor_id())))) {
+
+			set_task_cpu(p, smp_processor_id());
+
+			task_rq_unlock(rq, &flags);
+			goto repeat_lock_task;
+		}
+		if (old_state == TASK_UNINTERRUPTIBLE)
+			rq->nr_uninterruptible--;
+		activate_task(p, rq);
+
+		if (p->prio < rq->curr->prio || rq->curr->policy == SCHED_BATCH)
+			resched_task(rq->curr);
+		success = 1;
+	}
 	p->state = TASK_RUNNING;
-	if (task_on_runqueue(p))
-		goto out;
-	add_to_runqueue(p);
-	if (!synchronous || !(p->cpus_allowed & (1 << smp_processor_id())))
-		reschedule_idle(p);
-	success = 1;
-out:
-	spin_unlock_irqrestore(&runqueue_lock, flags);
+	task_rq_unlock(rq, &flags);
+
 	return success;
 }
 
-inline int wake_up_process(struct task_struct * p)
+int wake_up_process(task_t * p)
 {
 	return try_to_wake_up(p, 0);
 }
 
-static void process_timeout(unsigned long __data)
+void wake_up_forked_process(task_t * p)
 {
-	struct task_struct * p = (struct task_struct *) __data;
+	runqueue_t *rq = this_rq_lock();
+
+	p->state = TASK_RUNNING;
+	if (!rt_task(p)) {
+		/*
+		 * We decrease the sleep average of forking parents
+		 * and children as well, to keep max-interactive tasks
+		 * from forking tasks that are max-interactive.
+		 */
+		current->sleep_avg = current->sleep_avg * PARENT_PENALTY / 100;
+		p->sleep_avg = p->sleep_avg * CHILD_PENALTY / 100;
+		p->prio = effective_prio(p);
+	}
+	set_task_cpu(p, smp_processor_id());
+	activate_task(p, rq);
 
-	wake_up_process(p);
+	rq_unlock(rq);
 }
 
-/**
- * schedule_timeout - sleep until timeout
- * @timeout: timeout value in jiffies
- *
- * Make the current task sleep until @timeout jiffies have
- * elapsed. The routine will return immediately unless
- * the current task state has been set (see set_current_state()).
- *
- * You can set the task state as follows -
- *
- * %TASK_UNINTERRUPTIBLE - at least @timeout jiffies are guaranteed to
- * pass before the routine returns. The routine will return 0
- *
- * %TASK_INTERRUPTIBLE - the routine may return early if a signal is
- * delivered to the current task. In this case the remaining time
- * in jiffies will be returned, or 0 if the timer expired in time
- *
- * The current task state is guaranteed to be TASK_RUNNING when this 
- * routine returns.
- *
- * Specifying a @timeout value of %MAX_SCHEDULE_TIMEOUT will schedule
- * the CPU away without a bound on the timeout. In this case the return
- * value will be %MAX_SCHEDULE_TIMEOUT.
- *
- * In all cases the return value is guaranteed to be non-negative.
+/*
+ * Potentially available exiting-child timeslices are
+ * retrieved here - this way the parent does not get
+ * penalized for creating too many processes.
+ *
+ * (this cannot be used to 'generate' timeslices
+ * artificially, because any timeslice recovered here
+ * was given away by the parent in the first place.)
  */
-signed long schedule_timeout(signed long timeout)
+void sched_exit(task_t * p)
 {
-	struct timer_list timer;
-	unsigned long expire;
+	__cli();
+	if (p->first_time_slice) {
+		current->time_slice += p->time_slice;
+		if (unlikely(current->time_slice > MAX_TIMESLICE))
+			current->time_slice = MAX_TIMESLICE;
+	}
+	__sti();
+	/*
+	 * If the child was a (relative-) CPU hog then decrease
+	 * the sleep_avg of the parent as well.
+	 */
+	if (p->sleep_avg < current->sleep_avg)
+		current->sleep_avg = (current->sleep_avg * EXIT_WEIGHT +
+			p->sleep_avg) / (EXIT_WEIGHT + 1);
+}
 
-	switch (timeout)
-	{
-	case MAX_SCHEDULE_TIMEOUT:
-		/*
-		 * These two special cases are useful to be comfortable
-		 * in the caller. Nothing more. We could take
-		 * MAX_SCHEDULE_TIMEOUT from one of the negative value
-		 * but I' d like to return a valid offset (>=0) to allow
-		 * the caller to do everything it want with the retval.
-		 */
-		schedule();
-		goto out;
-	default:
-		/*
-		 * Another bit of PARANOID. Note that the retval will be
-		 * 0 since no piece of kernel is supposed to do a check
-		 * for a negative retval of schedule_timeout() (since it
-		 * should never happens anyway). You just have the printk()
-		 * that will tell you if something is gone wrong and where.
-		 */
-		if (timeout < 0)
-		{
-			printk(KERN_ERR "schedule_timeout: wrong timeout "
-			       "value %lx from %p\n", timeout,
-			       __builtin_return_address(0));
-			current->state = TASK_RUNNING;
-			goto out;
-		}
+#if CONFIG_SMP
+asmlinkage void schedule_tail(task_t *prev)
+{
+	finish_arch_switch(this_rq(), prev);
+}
+#endif
+
+static inline task_t * context_switch(task_t *prev, task_t *next)
+{
+	struct mm_struct *mm = next->mm;
+	struct mm_struct *oldmm = prev->active_mm;
+
+	if (unlikely(!mm)) {
+		next->active_mm = oldmm;
+		atomic_inc(&oldmm->mm_count);
+		enter_lazy_tlb(oldmm, next, smp_processor_id());
+	} else
+		switch_mm(oldmm, mm, next, smp_processor_id());
+
+	if (unlikely(!prev->mm)) {
+		prev->active_mm = NULL;
+		mmdrop(oldmm);
 	}
 
-	expire = timeout + jiffies;
+	/* Here we just switch the register state and the stack. */
+	switch_to(prev, next, prev);
 
-	init_timer(&timer);
-	timer.expires = expire;
-	timer.data = (unsigned long) current;
-	timer.function = process_timeout;
+	return prev;
+}
 
-	add_timer(&timer);
-	schedule();
-	del_timer_sync(&timer);
+unsigned long nr_running(void)
+{
+	unsigned long i, sum = 0;
 
-	timeout = expire - jiffies;
+	for (i = 0; i < NR_CPUS; i++)
+		sum += cpu_rq(i)->nr_running;
 
- out:
-	return timeout < 0 ? 0 : timeout;
+	return sum;
+}
+
+unsigned long nr_uninterruptible(void)
+{
+	unsigned long i, sum = 0;
+
+	for (i = 0; i < NR_CPUS; i++)
+		sum += cpu_rq(i)->nr_uninterruptible;
+
+	return sum;
+}
+
+unsigned long nr_context_switches(void)
+{
+	unsigned long i, sum = 0;
+
+	for (i = 0; i < NR_CPUS; i++)
+		sum += cpu_rq(i)->nr_switches;
+
+	return sum;
 }
 
 /*
- * schedule_tail() is getting called from the fork return path. This
- * cleans up all remaining scheduler things, without impacting the
- * common case.
+ * double_rq_lock - safely lock two runqueues
+ *
+ * Note this does not disable interrupts like task_rq_lock,
+ * you need to do so manually before calling.
  */
-static inline void __schedule_tail(struct task_struct *prev)
+static inline void double_rq_lock(runqueue_t *rq1, runqueue_t *rq2)
 {
-#ifdef CONFIG_SMP
-	int policy;
+	if (rq1 == rq2)
+		spin_lock(&rq1->lock);
+	else {
+		if (rq1 < rq2) {
+			spin_lock(&rq1->lock);
+			spin_lock(&rq2->lock);
+		} else {
+			spin_lock(&rq2->lock);
+			spin_lock(&rq1->lock);
+		}
+	}
+}
+
+/*
+ * double_rq_unlock - safely unlock two runqueues
+ *
+ * Note this does not restore interrupts like task_rq_unlock,
+ * you need to do so manually after calling.
+ */
+static inline void double_rq_unlock(runqueue_t *rq1, runqueue_t *rq2)
+{
+	spin_unlock(&rq1->lock);
+	if (rq1 != rq2)
+		spin_unlock(&rq2->lock);
+}
+
+#if CONFIG_SMP
+
+/*
+ * Batch balancing is much simpler since it's optimized for
+ * CPU-intensive workloads. The balancer keeps the batch-queue
+ * length as close to the average length as possible. It weighs
+ * runqueue distribution based on the idle percentage of each
+ * CPU - this way statistical fairness of timeslice distribution
+ * is preserved, in the long run it does not matter whether a
+ * batch task is queued to a busy CPU or not, it will get an
+ * equal share of all available idle CPU time.
+ *
+ * CPU-intensive SCHED_BATCH processes have a much lower
+ * fork()/exit() flux, so the balancing does not have to
+ * be prepared for high statistical fluctuations in queue
+ * length.
+ */
+static inline void load_balance_batch(runqueue_t *this_rq, int this_cpu)
+{
+	int i, nr_batch, nr_idle, goal, rq_goal;
+	runqueue_t *rq_src;
 
 	/*
-	 * prev->policy can be written from here only before `prev'
-	 * can be scheduled (before setting prev->cpus_runnable to ~0UL).
-	 * Of course it must also be read before allowing prev
-	 * to be rescheduled, but since the write depends on the read
-	 * to complete, wmb() is enough. (the spin_lock() acquired
-	 * before setting cpus_runnable is not enough because the spin_lock()
-	 * common code semantics allows code outside the critical section
-	 * to enter inside the critical section)
+	 * First the unlocked fastpath - is there any work to do?
+	 * fastpath #1: no batch processes in the system,
+	 * fastpath #2: no idle time available in the system.
+	 * fastpath #3: no balancing needed for the current queue.
 	 */
-	policy = prev->policy;
-	prev->policy = policy & ~SCHED_YIELD;
-	wmb();
+	nr_batch = 0;
+	nr_idle = 0;
 
-	/*
-	 * fast path falls through. We have to clear cpus_runnable before
-	 * checking prev->state to avoid a wakeup race. Protect against
-	 * the task exiting early.
-	 */
-	task_lock(prev);
-	task_release_cpu(prev);
-	mb();
-	if (prev->state == TASK_RUNNING)
-		goto needs_resched;
+	for (i = 0; i < NR_CPUS; i++) {
+		if (!cpu_online(i))
+			continue;
 
-out_unlock:
-	task_unlock(prev);	/* Synchronise here with release_task() if prev is TASK_ZOMBIE */
-	return;
+		nr_batch += cpu_rq(i)->nr_batch;
+		nr_idle += cpu_rq(i)->idle_avg;
+	}
+	if (!nr_batch || !nr_idle)
+		return;
+
+	goal = this_rq->idle_avg * nr_batch / nr_idle;
+	if (this_rq->nr_batch >= goal)
+		return;
 
 	/*
-	 * Slow path - we 'push' the previous process and
-	 * reschedule_idle() will attempt to find a new
-	 * processor for it. (but it might preempt the
-	 * current process as well.) We must take the runqueue
-	 * lock and re-check prev->state to be correct. It might
-	 * still happen that this process has a preemption
-	 * 'in progress' already - but this is not a problem and
-	 * might happen in other circumstances as well.
+	 * The slow path - the local batch-queue is too short and
+	 * needs balancing. We unlock the runqueue (but keep
+	 * interrupts disabled) to simplify locking. (It does not
+	 * matter if the runqueues change meanwhile - this is all
+	 * statistical balancing so only the long run effects matter.)
 	 */
-needs_resched:
-	{
-		unsigned long flags;
+	spin_unlock(&this_rq->lock);
 
-		/*
-		 * Avoid taking the runqueue lock in cases where
-		 * no preemption-check is necessery:
-		 */
-		if ((prev == idle_task(smp_processor_id())) ||
-						(policy & SCHED_YIELD))
-			goto out_unlock;
+	for (i = 0; i < NR_CPUS; i++) {
+		if (!cpu_online(i) || (i == this_cpu))
+			continue;
 
-		spin_lock_irqsave(&runqueue_lock, flags);
-		if ((prev->state == TASK_RUNNING) && !task_has_cpu(prev))
-			reschedule_idle(prev);
-		spin_unlock_irqrestore(&runqueue_lock, flags);
-		goto out_unlock;
+		rq_src = cpu_rq(i);
+		double_rq_lock(this_rq, rq_src);
+
+		rq_goal = rq_src->idle_avg * nr_batch / nr_idle;
+
+		if (rq_src->nr_batch > rq_goal) {
+			/*
+			 * Migrate a single batch-process.
+			 */
+			list_t *tmp = rq_src->batch_queue.prev;
+
+			list_del(tmp);
+			list_add_tail(tmp, &this_rq->batch_queue);
+			rq_src->nr_batch--;
+			this_rq->nr_batch++;
+			set_task_cpu(list_entry(tmp, task_t, run_list), this_cpu);
+		}
+
+		double_rq_unlock(this_rq, rq_src);
+		if (this_rq->nr_batch >= goal)
+			break;
 	}
-#else
-	prev->policy &= ~SCHED_YIELD;
-#endif /* CONFIG_SMP */
+	spin_lock(&this_rq->lock);
 }
+/*
+ * Lock the busiest runqueue as well, this_rq is locked already.
+ * Recalculate nr_running if we have to drop the runqueue lock.
+ */
+static inline unsigned int double_lock_balance(runqueue_t *this_rq,
+	runqueue_t *busiest, int this_cpu, int idle, unsigned int nr_running)
+{
+	if (unlikely(!spin_trylock(&busiest->lock))) {
+		if (busiest < this_rq) {
+			spin_unlock(&this_rq->lock);
+			spin_lock(&busiest->lock);
+			spin_lock(&this_rq->lock);
+			/* Need to recalculate nr_running */
+			if (idle || (this_rq->nr_running > this_rq->prev_nr_running[this_cpu]))
+				nr_running = this_rq->nr_running;
+			else
+				nr_running = this_rq->prev_nr_running[this_cpu];
+		} else
+			spin_lock(&busiest->lock);
+	}
+	return nr_running;
+}
+
+static inline runqueue_t *find_busiest_queue(runqueue_t *this_rq, int this_cpu, int idle, int *imbalance)
+{
+	int nr_running, load, max_load, i;
+	runqueue_t *busiest, *rq_src;
+
+	/*
+	 * We search all runqueues to find the most busy one.
+	 * We do this lockless to reduce cache-bouncing overhead,
+	 * we re-check the 'best' source CPU later on again, with
+	 * the lock held.
+	 *
+	 * We fend off statistical fluctuations in runqueue lengths by
+	 * saving the runqueue length during the previous load-balancing
+	 * operation and using the smaller one the current and saved lengths.
+	 * If a runqueue is long enough for a longer amount of time then
+	 * we recognize it and pull tasks from it.
+	 *
+	 * The 'current runqueue length' is a statistical maximum variable,
+	 * for that one we take the longer one - to avoid fluctuations in
+	 * the other direction. So for a load-balance to happen it needs
+	 * stable long runqueue on the target CPU and stable short runqueue
+	 * on the local runqueue.
+	 *
+	 * We make an exception if this CPU is about to become idle - in
+	 * that case we are less picky about moving a task across CPUs and
+	 * take what can be taken.
+	 */
+	if (idle || (this_rq->nr_running > this_rq->prev_nr_running[this_cpu]))
+		nr_running = this_rq->nr_running;
+	else
+		nr_running = this_rq->prev_nr_running[this_cpu];
+
+	busiest = NULL;
+	max_load = 1;
+	for (i = 0; i < NR_CPUS; i++) {
+		if (!cpu_online(i))
+			continue;
+
+		rq_src = cpu_rq(i);
+		if (idle || (rq_src->nr_running < this_rq->prev_nr_running[i]))
+			load = rq_src->nr_running;
+		else
+			load = this_rq->prev_nr_running[i];
+		this_rq->prev_nr_running[i] = rq_src->nr_running;
+
+		if ((load > max_load) && (rq_src != this_rq)) {
+			busiest = rq_src;
+			max_load = load;
+		}
+	}
+
+	if (likely(!busiest))
+		goto out;
 
-asmlinkage void schedule_tail(struct task_struct *prev)
+	*imbalance = (max_load - nr_running) / 2;
+
+	/* It needs an at least ~25% imbalance to trigger balancing. */
+	if (!idle && (*imbalance < (max_load + 3)/4)) {
+		busiest = NULL;
+		goto out;
+	}
+
+	nr_running = double_lock_balance(this_rq, busiest, this_cpu, idle, nr_running);
+	/*
+	 * Make sure nothing changed since we checked the
+	 * runqueue length.
+	 */
+	if (busiest->nr_running <= nr_running + 1) {
+		spin_unlock(&busiest->lock);
+		busiest = NULL;
+	}
+out:
+	return busiest;
+}
+
+/*
+ * Move a task from a remote runqueue to the local runqueue.
+ * Both runqueues must be locked.
+ */
+static inline void pull_task(runqueue_t *src_rq, prio_array_t *src_array, task_t *p, runqueue_t *this_rq, int this_cpu)
 {
-	__schedule_tail(prev);
+	dequeue_task(p, src_array);
+	src_rq->nr_running--;
+	set_task_cpu(p, this_cpu);
+	this_rq->nr_running++;
+	enqueue_task(p, this_rq->active);
+	/*
+	 * Note that idle threads have a prio of MAX_PRIO, for this test
+	 * to be always true for them.
+	 */
+	if (p->prio < this_rq->curr->prio)
+		set_need_resched();
 }
 
 /*
- *  'schedule()' is the scheduler function. It's a very simple and nice
- * scheduler: it's not perfect, but certainly works for most things.
+ * Current runqueue is empty, or rebalance tick: if there is an
+ * inbalance (current runqueue is too short) then pull from
+ * busiest runqueue(s).
  *
- * The goto is "interesting".
+ * We call this with the current runqueue locked,
+ * irqs disabled.
+ */
+static void load_balance(runqueue_t *this_rq, int idle)
+{
+	int imbalance, idx, this_cpu = smp_processor_id();
+	runqueue_t *busiest;
+	prio_array_t *array;
+	list_t *head, *curr;
+	task_t *tmp;
+
+	busiest = find_busiest_queue(this_rq, this_cpu, idle, &imbalance);
+	if (!busiest)
+		goto balance_batch;
+
+	/*
+	 * We first consider expired tasks. Those will likely not be
+	 * executed in the near future, and they are most likely to
+	 * be cache-cold, thus switching CPUs has the least effect
+	 * on them.
+	 */
+	if (busiest->expired->nr_active)
+		array = busiest->expired;
+	else
+		array = busiest->active;
+
+new_array:
+	/* Start searching at priority 0: */
+	idx = 0;
+skip_bitmap:
+	if (!idx)
+		idx = sched_find_first_bit(array->bitmap);
+	else
+		idx = find_next_bit(array->bitmap, MAX_PRIO, idx);
+	if (idx == MAX_PRIO) {
+		if (array == busiest->expired) {
+			array = busiest->active;
+			goto new_array;
+		}
+		goto out_unlock;
+	}
+
+	head = array->queue + idx;
+	curr = head->prev;
+skip_queue:
+	tmp = list_entry(curr, task_t, run_list);
+
+	/*
+	 * We do not migrate tasks that are:
+	 * 1) running (obviously), or
+	 * 2) cannot be migrated to this CPU due to cpus_allowed, or
+	 * 3) are cache-hot on their current CPU.
+	 */
+
+#define CAN_MIGRATE_TASK(p,rq,this_cpu)					\
+	((jiffies - (p)->sleep_timestamp > cache_decay_ticks) &&	\
+		!task_running(rq, p) &&					\
+			((p)->cpus_allowed & (1UL << (this_cpu))))
+
+	curr = curr->prev;
+
+	if (!CAN_MIGRATE_TASK(tmp, busiest, this_cpu)) {
+		if (curr != head)
+			goto skip_queue;
+		idx++;
+		goto skip_bitmap;
+	}
+	pull_task(busiest, array, tmp, this_rq, this_cpu);
+	if (!idle && --imbalance) {
+		if (curr != head)
+			goto skip_queue;
+		idx++;
+		goto skip_bitmap;
+	}
+out_unlock:
+	spin_unlock(&busiest->lock);
+balance_batch:
+	load_balance_batch(this_rq, this_cpu);
+}
+
+/*
+ * One of the idle_cpu_tick() or the busy_cpu_tick() function will
+ * gets called every timer tick, on every CPU. Our balancing action
+ * frequency and balancing agressivity depends on whether the CPU is
+ * idle or not.
  *
- *   NOTE!!  Task 0 is the 'idle' task, which gets called when no other
- * tasks can run. It can not be killed, and it cannot sleep. The 'state'
- * information in task[0] is never used.
+ * busy-rebalance every 250 msecs. idle-rebalance every 1 msec. (or on
+ * systems with HZ=100, every 10 msecs.)
  */
-asmlinkage void schedule(void)
+#define BUSY_REBALANCE_TICK (HZ/4 ?: 1)
+#define IDLE_REBALANCE_TICK (HZ/1000 ?: 1)
+
+static inline void idle_tick(runqueue_t *rq)
 {
-	struct schedule_data * sched_data;
-	struct task_struct *prev, *next, *p;
-	struct list_head *tmp;
-	int this_cpu, c;
+	if (jiffies % IDLE_REBALANCE_TICK)
+		return;
+	spin_lock(&rq->lock);
+	load_balance(rq, 1);
+	spin_unlock(&rq->lock);
+}
+
+#endif
+
+/*
+ * We place interactive tasks back into the active array, if possible.
+ *
+ * To guarantee that this does not starve expired tasks we ignore the
+ * interactivity of a task if the first expired task had to wait more
+ * than a 'reasonable' amount of time. This deadline timeout is
+ * load-dependent, as the frequency of array switched decreases with
+ * increasing number of running tasks:
+ */
+#define EXPIRED_STARVING(rq) \
+		((rq)->expired_timestamp && \
+		(jiffies - (rq)->expired_timestamp >= \
+			STARVATION_LIMIT * ((rq)->nr_running) + 1))
 
+/*
+ * This function gets called by the timer code, with HZ frequency.
+ * We call it with interrupts disabled.
+ */
+void scheduler_tick(int user_ticks, int sys_ticks)
+{
+	int cpu = smp_processor_id();
+	runqueue_t *rq = this_rq();
+	task_t *p = current;
 
-	spin_lock_prefetch(&runqueue_lock);
+#if CONFIG_SMP
+	if (user_ticks || sys_ticks) {
+		/*
+		 * This code is rare, triggered only once per second:
+		 */
+		if (--rq->idle_ticks_left <= 0) {
+			/*
+			 * Maintain a simple running average:
+			 */
+			rq->idle_avg += rq->idle_count;
+			rq->idle_avg >>= 1;
 
-	BUG_ON(!current->active_mm);
-need_resched_back:
-	prev = current;
-	this_cpu = prev->processor;
+			rq->idle_ticks_left = IDLE_TICKS;
+			rq->idle_count = 0;
 
-	if (unlikely(in_interrupt())) {
-		printk("Scheduling in interrupt\n");
-		BUG();
+		}
 	}
+	if (p == rq->idle || p->policy == SCHED_BATCH)
+		rq->idle_count++;
+#endif
+	if (p == rq->idle) {
+		if (local_bh_count(cpu) || local_irq_count(cpu) > 1)
+			kstat.per_cpu_system[cpu] += sys_ticks;
+#if CONFIG_SMP
+		idle_tick(rq);
+#endif
+		return;
+	}
+	if (TASK_NICE(p) > 0 || p->policy == SCHED_BATCH)
+		kstat.per_cpu_nice[cpu] += user_ticks;
+	else
+		kstat.per_cpu_user[cpu] += user_ticks;
+	kstat.per_cpu_system[cpu] += sys_ticks;
 
-	release_kernel_lock(prev, this_cpu);
-
+	/* Task might have expired already, but not scheduled off yet */
+	if (p->array != rq->active) {
+		set_tsk_need_resched(p);
+		return;
+	}
+	spin_lock(&rq->lock);
+	if (unlikely(rt_task(p))) {
+		/*
+		 * RR tasks need a special form of timeslice management.
+		 * FIFO tasks have no timeslices.
+		 */
+		if ((p->policy == SCHED_RR) && !--p->time_slice) {
+			p->time_slice = task_timeslice(p);
+			p->first_time_slice = 0;
+			set_tsk_need_resched(p);
+
+			/* put it at the end of the queue: */
+			dequeue_task(p, rq->active);
+			enqueue_task(p, rq->active);
+		}
+		goto out;
+	}
 	/*
-	 * 'sched_data' is protected by the fact that we can run
-	 * only one process per CPU.
-	 */
-	sched_data = & aligned_data[this_cpu].schedule_data;
+	 * The task was running during this tick - update the
+	 * time slice counter and the sleep average. Note: we
+	 * do not update a process's priority until it either
+	 * goes to sleep or uses up its timeslice. This makes
+	 * it possible for interactive tasks to use up their
+	 * timeslices at their highest priority levels.
+	 */
+	if (p->sleep_avg)
+		p->sleep_avg--;
+	if (!--p->time_slice) {
+		dequeue_task(p, rq->active);
+		set_tsk_need_resched(p);
+		p->prio = effective_prio(p);
+		p->time_slice = task_timeslice(p);
+		p->first_time_slice = 0;
+
+		if (!TASK_INTERACTIVE(p) || EXPIRED_STARVING(rq)) {
+			if (!rq->expired_timestamp)
+				rq->expired_timestamp = jiffies;
+			enqueue_task(p, rq->expired);
+		} else
+			enqueue_task(p, rq->active);
+	}
+out:
+#if CONFIG_SMP
+	if (!(jiffies % BUSY_REBALANCE_TICK))
+		load_balance(rq, 0);
+#endif
+	spin_unlock(&rq->lock);
+}
 
-	spin_lock_irq(&runqueue_lock);
+void scheduling_functions_start_here(void) { }
 
-	/* move an exhausted RR process to be last.. */
-	if (unlikely(prev->policy == SCHED_RR))
-		if (!prev->counter) {
-			prev->counter = NICE_TO_TICKS(prev->nice);
-			move_last_runqueue(prev);
-		}
+/*
+ * This function is called by the lowlevel kernel entry code if
+ * pure userspace code is preempted. Such processes, if SCHED_BATCH,
+ * are candidates for batch scheduling. Every other process (including
+ * kernel-mode SCHED_BATCH processes) is scheduled in a non-batch way.
+ */
+asmlinkage void schedule_userspace(void)
+{
+	runqueue_t *rq;
 
-	switch (prev->state) {
-		case TASK_INTERRUPTIBLE:
-			if (signal_pending(prev)) {
-				prev->state = TASK_RUNNING;
-				break;
-			}
-		default:
-			del_from_runqueue(prev);
-		case TASK_RUNNING:;
+	if (current->policy != SCHED_BATCH) {
+		schedule();
+		return;
 	}
-	prev->need_resched = 0;
 
 	/*
-	 * this is the scheduler proper:
+	 * Only handle batch tasks that are runnable.
 	 */
+	if (current->state == TASK_RUNNING) {
+		rq = this_rq_lock();
+		deactivate_batch_task(current, rq);
 
-repeat_schedule:
-	/*
-	 * Default process to select..
-	 */
-	next = idle_task(this_cpu);
-	c = -1000;
-	list_for_each(tmp, &runqueue_head) {
-		p = list_entry(tmp, struct task_struct, run_list);
-		if (can_schedule(p, this_cpu)) {
-			int weight = goodness(p, this_cpu, prev->active_mm);
-			if (weight > c)
-				c = weight, next = p;
-		}
-	}
-
-	/* Do we need to re-calculate counters? */
-	if (unlikely(!c)) {
-		struct task_struct *p;
-
-		spin_unlock_irq(&runqueue_lock);
-		read_lock(&tasklist_lock);
-		for_each_task(p)
-			p->counter = (p->counter >> 1) + NICE_TO_TICKS(p->nice);
-		read_unlock(&tasklist_lock);
-		spin_lock_irq(&runqueue_lock);
-		goto repeat_schedule;
+		// we can keep irqs disabled:
+		spin_unlock(&rq->lock);
 	}
 
-	/*
-	 * from this point on nothing can prevent us from
-	 * switching to the next task, save this fact in
-	 * sched_data.
-	 */
-	sched_data->curr = next;
-	task_set_cpu(next, this_cpu);
-	spin_unlock_irq(&runqueue_lock);
+	schedule();
+}
 
-	if (unlikely(prev == next)) {
-		/* We won't go through the normal tail, so do this by hand */
-		prev->policy &= ~SCHED_YIELD;
-		goto same_process;
-	}
+/*
+ * 'schedule()' is the main scheduler function.
+ */
+asmlinkage void schedule(void)
+{
+	task_t *prev, *next;
+	runqueue_t *rq;
+	prio_array_t *array;
+	list_t *queue;
+	int idx;
 
-#ifdef CONFIG_SMP
- 	/*
- 	 * maintain the per-process 'last schedule' value.
- 	 * (this has to be recalculated even if we reschedule to
- 	 * the same process) Currently this is only used on SMP,
-	 * and it's approximate, so we do not have to maintain
-	 * it while holding the runqueue spinlock.
- 	 */
- 	sched_data->last_schedule = get_cycles();
+	if (unlikely(in_interrupt()))
+		BUG();
 
-	/*
-	 * We drop the scheduler lock early (it's a global spinlock),
-	 * thus we have to lock the previous process from getting
-	 * rescheduled during switch_to().
-	 */
+need_resched:
+	prev = current;
+	rq = this_rq();
 
-#endif /* CONFIG_SMP */
+	release_kernel_lock(prev, smp_processor_id());
+	prev->sleep_timestamp = jiffies;
+	spin_lock_irq(&rq->lock);
 
-	kstat.context_swtch++;
-	/*
-	 * there are 3 processes which are affected by a context switch:
-	 *
-	 * prev == .... ==> (last => next)
-	 *
-	 * It's the 'much more previous' 'prev' that is on next's stack,
-	 * but prev is set to (the just run) 'last' process by switch_to().
-	 * This might sound slightly confusing but makes tons of sense.
-	 */
-	prepare_to_switch();
-	{
-		struct mm_struct *mm = next->mm;
-		struct mm_struct *oldmm = prev->active_mm;
-		if (!mm) {
-			BUG_ON(next->active_mm);
-			next->active_mm = oldmm;
-			atomic_inc(&oldmm->mm_count);
-			enter_lazy_tlb(oldmm, next, this_cpu);
-		} else {
-			BUG_ON(next->active_mm != mm);
-			switch_mm(oldmm, mm, next, this_cpu);
+	switch (prev->state) {
+	case TASK_INTERRUPTIBLE:
+		if (unlikely(signal_pending(prev))) {
+			prev->state = TASK_RUNNING;
+			break;
 		}
+	default:
+		deactivate_task(prev, rq);
+	case TASK_RUNNING:
+		;
+	}
+#if CONFIG_SMP
+pick_next_task:
+#endif
+	if (unlikely(!rq->nr_running)) {
+#if CONFIG_SMP
+		load_balance(rq, 1);
+		if (rq->nr_running)
+			goto pick_next_task;
+#endif
+		/*
+		 * Pick a task from the batch queue if available.
+		 */
+		if (rq->nr_batch) {
+			list_t *tmp = rq->batch_queue.next;
 
-		if (!prev->mm) {
-			prev->active_mm = NULL;
-			mmdrop(oldmm);
-		}
+			next = list_entry(tmp, task_t, run_list);
+			activate_batch_task(next, rq);
+		} else
+			next = rq->idle;
+		rq->expired_timestamp = 0;
+		goto switch_tasks;
 	}
 
-	/*
-	 * This just switches the register state and the
-	 * stack.
-	 */
-	switch_to(prev, next, prev);
-	__schedule_tail(prev);
+	array = rq->active;
+	if (unlikely(!array->nr_active)) {
+		/*
+		 * Switch the active and expired arrays.
+		 */
+		rq->active = rq->expired;
+		rq->expired = array;
+		array = rq->active;
+		rq->expired_timestamp = 0;
+	}
+
+	idx = sched_find_first_bit(array->bitmap);
+	queue = array->queue + idx;
+	next = list_entry(queue->next, task_t, run_list);
+
+switch_tasks:
+	prefetch(next);
+	clear_tsk_need_resched(prev);
+
+	if (likely(prev != next)) {
+		rq->nr_switches++;
+		rq->curr = next;
+	
+		prepare_arch_switch(rq, next);
+		prev = context_switch(prev, next);
+		barrier();
+		rq = this_rq();
+		finish_arch_switch(rq, prev);
+	} else
+		spin_unlock_irq(&rq->lock);
 
-same_process:
 	reacquire_kernel_lock(current);
-	if (current->need_resched)
-		goto need_resched_back;
-	return;
+	if (need_resched())
+		goto need_resched;
 }
 
 /*
- * The core wakeup function.  Non-exclusive wakeups (nr_exclusive == 0) just wake everything
- * up.  If it's an exclusive wakeup (nr_exclusive == small +ve number) then we wake all the
- * non-exclusive tasks and one exclusive task.
+ * The core wakeup function.  Non-exclusive wakeups (nr_exclusive == 0) just
+ * wake everything up.  If it's an exclusive wakeup (nr_exclusive == small +ve
+ * number) then we wake all the non-exclusive tasks and one exclusive task.
  *
  * There are circumstances in which we can try to wake a task which has already
- * started to run but is not in state TASK_RUNNING.  try_to_wake_up() returns zero
- * in this (rare) case, and we handle it by contonuing to scan the queue.
+ * started to run but is not in state TASK_RUNNING.  try_to_wake_up() returns
+ * zero in this (rare) case, and we handle it by continuing to scan the queue.
  */
-static inline void __wake_up_common (wait_queue_head_t *q, unsigned int mode,
-			 	     int nr_exclusive, const int sync)
+static inline void __wake_up_common(wait_queue_head_t *q, unsigned int mode, int nr_exclusive, int sync)
 {
 	struct list_head *tmp;
-	struct task_struct *p;
-
-	CHECK_MAGIC_WQHEAD(q);
-	WQ_CHECK_LIST_HEAD(&q->task_list);
-	
-	list_for_each(tmp,&q->task_list) {
-		unsigned int state;
-                wait_queue_t *curr = list_entry(tmp, wait_queue_t, task_list);
+	unsigned int state;
+	wait_queue_t *curr;
+	task_t *p;
 
-		CHECK_MAGIC(curr->__magic);
+	list_for_each(tmp, &q->task_list) {
+		curr = list_entry(tmp, wait_queue_t, task_list);
 		p = curr->task;
 		state = p->state;
-		if (state & mode) {
-			WQ_NOTE_WAKER(curr);
-			if (try_to_wake_up(p, sync) && (curr->flags&WQ_FLAG_EXCLUSIVE) && !--nr_exclusive)
+		if ((state & mode) && try_to_wake_up(p, sync) &&
+			((curr->flags & WQ_FLAG_EXCLUSIVE) && !--nr_exclusive))
 				break;
-		}
 	}
 }
 
-void __wake_up(wait_queue_head_t *q, unsigned int mode, int nr)
+void __wake_up(wait_queue_head_t *q, unsigned int mode, int nr_exclusive)
 {
-	if (q) {
-		unsigned long flags;
-		wq_read_lock_irqsave(&q->lock, flags);
-		__wake_up_common(q, mode, nr, 0);
-		wq_read_unlock_irqrestore(&q->lock, flags);
-	}
+	unsigned long flags;
+
+	if (unlikely(!q))
+		return;
+
+	spin_lock_irqsave(&q->lock, flags);
+	__wake_up_common(q, mode, nr_exclusive, 0);
+	spin_unlock_irqrestore(&q->lock, flags);
 }
 
-void __wake_up_sync(wait_queue_head_t *q, unsigned int mode, int nr)
+#if CONFIG_SMP
+
+void __wake_up_sync(wait_queue_head_t *q, unsigned int mode, int nr_exclusive)
 {
-	if (q) {
-		unsigned long flags;
-		wq_read_lock_irqsave(&q->lock, flags);
-		__wake_up_common(q, mode, nr, 1);
-		wq_read_unlock_irqrestore(&q->lock, flags);
-	}
+	unsigned long flags;
+
+	if (unlikely(!q))
+		return;
+
+	spin_lock_irqsave(&q->lock, flags);
+	if (likely(nr_exclusive))
+		__wake_up_common(q, mode, nr_exclusive, 1);
+	else
+		__wake_up_common(q, mode, nr_exclusive, 0);
+	spin_unlock_irqrestore(&q->lock, flags);
 }
 
+#endif
+ 
 void complete(struct completion *x)
 {
 	unsigned long flags;
@@ -793,14 +1248,14 @@
 	init_waitqueue_entry(&wait, current);
 
 #define	SLEEP_ON_HEAD					\
-	wq_write_lock_irqsave(&q->lock,flags);		\
+	spin_lock_irqsave(&q->lock,flags);		\
 	__add_wait_queue(q, &wait);			\
-	wq_write_unlock(&q->lock);
+	spin_unlock(&q->lock);
 
 #define	SLEEP_ON_TAIL						\
-	wq_write_lock_irq(&q->lock);				\
+	spin_lock_irq(&q->lock);				\
 	__remove_wait_queue(q, &wait);				\
-	wq_write_unlock_irqrestore(&q->lock,flags);
+	spin_unlock_irqrestore(&q->lock, flags);
 
 void interruptible_sleep_on(wait_queue_head_t *q)
 {
@@ -852,6 +1307,42 @@
 
 void scheduling_functions_end_here(void) { }
 
+void set_user_nice(task_t *p, long nice)
+{
+	unsigned long flags;
+	prio_array_t *array;
+	runqueue_t *rq;
+
+	if (TASK_NICE(p) == nice || nice < -20 || nice > 19)
+		return;
+	/*
+	 * We have to be careful, if called from sys_setpriority(),
+	 * the task might be in the middle of scheduling on another CPU.
+	 */
+	rq = task_rq_lock(p, &flags);
+	if (rt_task(p)) {
+		p->static_prio = NICE_TO_PRIO(nice);
+		goto out_unlock;
+	}
+	array = p->array;
+	if (array)
+		dequeue_task(p, array);
+	p->static_prio = NICE_TO_PRIO(nice);
+	p->prio = NICE_TO_PRIO(nice);
+	if (array) {
+		enqueue_task(p, array);
+		/*
+		 * If the task is running and lowered its priority,
+		 * or increased its priority then reschedule its CPU:
+		 */
+		if ((NICE_TO_PRIO(nice) < p->static_prio) ||
+							task_running(rq, p))
+			resched_task(rq->curr);
+	}
+out_unlock:
+	task_rq_unlock(rq, &flags);
+}
+
 #ifndef __alpha__
 
 /*
@@ -862,7 +1353,7 @@
 
 asmlinkage long sys_nice(int increment)
 {
-	long newprio;
+	long nice;
 
 	/*
 	 *	Setpriority might change our priority at the same moment.
@@ -878,34 +1369,52 @@
 	if (increment > 40)
 		increment = 40;
 
-	newprio = current->nice + increment;
-	if (newprio < -20)
-		newprio = -20;
-	if (newprio > 19)
-		newprio = 19;
-	current->nice = newprio;
+	nice = PRIO_TO_NICE(current->static_prio) + increment;
+	if (nice < -20)
+		nice = -20;
+	if (nice > 19)
+		nice = 19;
+	set_user_nice(current, nice);
 	return 0;
 }
 
 #endif
 
-static inline struct task_struct *find_process_by_pid(pid_t pid)
+/*
+ * This is the priority value as seen by users in /proc
+ *
+ * RT tasks are offset by -200. Normal tasks are centered
+ * around 0, value goes from -16 to +15.
+ */
+int task_prio(task_t *p)
+{
+	return p->prio - MAX_USER_RT_PRIO;
+}
+
+int task_nice(task_t *p)
 {
-	struct task_struct *tsk = current;
+	return TASK_NICE(p);
+}
+
+int idle_cpu(int cpu)
+{
+	return cpu_curr(cpu) == cpu_rq(cpu)->idle;
+}
 
-	if (pid)
-		tsk = find_task_by_pid(pid);
-	return tsk;
+static inline task_t *find_process_by_pid(pid_t pid)
+{
+	return pid ? find_task_by_pid(pid) : current;
 }
 
-static int setscheduler(pid_t pid, int policy, 
-			struct sched_param *param)
+static int setscheduler(pid_t pid, int policy, struct sched_param *param)
 {
 	struct sched_param lp;
-	struct task_struct *p;
-	int retval;
+	int retval = -EINVAL;
+	prio_array_t *array;
+	unsigned long flags;
+	runqueue_t *rq;
+	task_t *p;
 
-	retval = -EINVAL;
 	if (!param || pid < 0)
 		goto out_nounlock;
 
@@ -917,58 +1426,73 @@
 	 * We play safe to avoid deadlocks.
 	 */
 	read_lock_irq(&tasklist_lock);
-	spin_lock(&runqueue_lock);
 
 	p = find_process_by_pid(pid);
 
 	retval = -ESRCH;
 	if (!p)
-		goto out_unlock;
-			
+		goto out_unlock_tasklist;
+
+	/*
+	 * To be able to change p->policy safely, the apropriate
+	 * runqueue lock must be held.
+	 */
+	rq = task_rq_lock(p, &flags);
+
 	if (policy < 0)
 		policy = p->policy;
 	else {
 		retval = -EINVAL;
 		if (policy != SCHED_FIFO && policy != SCHED_RR &&
-				policy != SCHED_OTHER)
+				policy != SCHED_NORMAL && policy != SCHED_BATCH)
 			goto out_unlock;
 	}
-	
+
 	/*
-	 * Valid priorities for SCHED_FIFO and SCHED_RR are 1..99, valid
-	 * priority for SCHED_OTHER is 0.
+	 * Valid priorities for SCHED_FIFO and SCHED_RR are
+	 * 1..MAX_USER_RT_PRIO-1, valid priority for SCHED_NORMAL and
+	 * SCHED_BATCH is 0.
 	 */
 	retval = -EINVAL;
-	if (lp.sched_priority < 0 || lp.sched_priority > 99)
+	if (lp.sched_priority < 0 || lp.sched_priority > MAX_USER_RT_PRIO-1)
 		goto out_unlock;
-	if ((policy == SCHED_OTHER) != (lp.sched_priority == 0))
+	if ((policy == SCHED_NORMAL || policy == SCHED_BATCH) !=
+						(lp.sched_priority == 0))
 		goto out_unlock;
 
 	retval = -EPERM;
-	if ((policy == SCHED_FIFO || policy == SCHED_RR) && 
+	if ((policy == SCHED_FIFO || policy == SCHED_RR) &&
 	    !capable(CAP_SYS_NICE))
 		goto out_unlock;
 	if ((current->euid != p->euid) && (current->euid != p->uid) &&
 	    !capable(CAP_SYS_NICE))
 		goto out_unlock;
 
+	if (p->flags & PF_BATCH)
+		activate_batch_task(p, rq);
+	array = p->array;
+	if (array)
+		deactivate_task(p, task_rq(p));
 	retval = 0;
 	p->policy = policy;
 	p->rt_priority = lp.sched_priority;
-	if (task_on_runqueue(p))
-		move_first_runqueue(p);
-
-	current->need_resched = 1;
+	if (policy != SCHED_NORMAL && policy != SCHED_BATCH)
+		p->prio = MAX_USER_RT_PRIO-1 - p->rt_priority;
+	else
+		p->prio = p->static_prio;
+	if (array)
+		activate_task(p, task_rq(p));
 
 out_unlock:
-	spin_unlock(&runqueue_lock);
+	task_rq_unlock(rq, &flags);
+out_unlock_tasklist:
 	read_unlock_irq(&tasklist_lock);
 
 out_nounlock:
 	return retval;
 }
 
-asmlinkage long sys_sched_setscheduler(pid_t pid, int policy, 
+asmlinkage long sys_sched_setscheduler(pid_t pid, int policy,
 				      struct sched_param *param)
 {
 	return setscheduler(pid, policy, param);
@@ -981,10 +1505,9 @@
 
 asmlinkage long sys_sched_getscheduler(pid_t pid)
 {
-	struct task_struct *p;
-	int retval;
+	int retval = -EINVAL;
+	task_t *p;
 
-	retval = -EINVAL;
 	if (pid < 0)
 		goto out_nounlock;
 
@@ -992,7 +1515,7 @@
 	read_lock(&tasklist_lock);
 	p = find_process_by_pid(pid);
 	if (p)
-		retval = p->policy & ~SCHED_YIELD;
+		retval = p->policy;
 	read_unlock(&tasklist_lock);
 
 out_nounlock:
@@ -1001,11 +1524,10 @@
 
 asmlinkage long sys_sched_getparam(pid_t pid, struct sched_param *param)
 {
-	struct task_struct *p;
 	struct sched_param lp;
-	int retval;
+	int retval = -EINVAL;
+	task_t *p;
 
-	retval = -EINVAL;
 	if (!param || pid < 0)
 		goto out_nounlock;
 
@@ -1030,44 +1552,118 @@
 	return retval;
 }
 
-asmlinkage long sys_sched_yield(void)
+/**
+ * sys_sched_setaffinity - set the cpu affinity of a process
+ * @pid: pid of the process
+ * @len: length in bytes of the bitmask pointed to by user_mask_ptr
+ * @user_mask_ptr: user-space pointer to the new cpu mask
+ */
+asmlinkage int sys_sched_setaffinity(pid_t pid, unsigned int len,
+				      unsigned long *user_mask_ptr)
 {
+	unsigned long new_mask;
+	int retval;
+	task_t *p;
+
+	if (len < sizeof(new_mask))
+		return -EINVAL;
+
+	if (copy_from_user(&new_mask, user_mask_ptr, sizeof(new_mask)))
+		return -EFAULT;
+
+	new_mask &= cpu_online_map;
+	if (!new_mask)
+		return -EINVAL;
+
+	read_lock(&tasklist_lock);
+
+	p = find_process_by_pid(pid);
+	if (!p) {
+		read_unlock(&tasklist_lock);
+		return -ESRCH;
+	}
+
 	/*
-	 * Trick. sched_yield() first counts the number of truly 
-	 * 'pending' runnable processes, then returns if it's
-	 * only the current processes. (This test does not have
-	 * to be atomic.) In threaded applications this optimization
-	 * gets triggered quite often.
+	 * It is not safe to call set_cpus_allowed with the
+	 * tasklist_lock held.  We will bump the task_struct's
+	 * usage count and then drop tasklist_lock.
 	 */
+	get_task_struct(p);
+	read_unlock(&tasklist_lock);
 
-	int nr_pending = nr_running;
+	retval = -EPERM;
+	if ((current->euid != p->euid) && (current->euid != p->uid) &&
+			!capable(CAP_SYS_NICE))
+		goto out_unlock;
 
-#if CONFIG_SMP
-	int i;
+	retval = 0;
+	set_cpus_allowed(p, new_mask);
 
-	// Subtract non-idle processes running on other CPUs.
-	for (i = 0; i < smp_num_cpus; i++) {
-		int cpu = cpu_logical_map(i);
-		if (aligned_data[cpu].schedule_data.curr != idle_task(cpu))
-			nr_pending--;
-	}
-#else
-	// on UP this process is on the runqueue as well
-	nr_pending--;
-#endif
-	if (nr_pending) {
-		/*
-		 * This process can only be rescheduled by us,
-		 * so this is safe without any locking.
-		 */
-		if (current->policy == SCHED_OTHER)
-			current->policy |= SCHED_YIELD;
-		current->need_resched = 1;
-
-		spin_lock_irq(&runqueue_lock);
-		move_last_runqueue(current);
-		spin_unlock_irq(&runqueue_lock);
+out_unlock:
+	free_task_struct(p);
+	return retval;
+}
+
+/**
+ * sys_sched_getaffinity - get the cpu affinity of a process
+ * @pid: pid of the process
+ * @len: length in bytes of the bitmask pointed to by user_mask_ptr
+ * @user_mask_ptr: user-space pointer to hold the current cpu mask
+ */
+asmlinkage int sys_sched_getaffinity(pid_t pid, unsigned int len,
+				      unsigned long *user_mask_ptr)
+{
+	unsigned int real_len;
+	unsigned long mask;
+	int retval;
+	task_t *p;
+
+	real_len = sizeof(mask);
+	if (len < real_len)
+		return -EINVAL;
+
+	read_lock(&tasklist_lock);
+
+	retval = -ESRCH;
+	p = find_process_by_pid(pid);
+	if (!p)
+		goto out_unlock;
+
+	retval = 0;
+	mask = p->cpus_allowed & cpu_online_map;
+
+out_unlock:
+	read_unlock(&tasklist_lock);
+	if (retval)
+		return retval;
+	if (copy_to_user(user_mask_ptr, &mask, real_len))
+		return -EFAULT;
+	return real_len;
+}
+
+asmlinkage long sys_sched_yield(void)
+{
+	runqueue_t *rq = this_rq_lock();
+	prio_array_t *array = current->array;
+
+	/*
+	 * We implement yielding by moving the task into the expired
+	 * queue.
+	 *
+	 * (special rule: RT tasks will just roundrobin in the active
+	 *  array.)
+	 */
+	if (likely(!rt_task(current))) {
+		dequeue_task(current, array);
+		enqueue_task(current, rq->expired);
+	} else {
+		list_del(&current->run_list);
+		list_add_tail(&current->run_list, array->queue + current->prio);
 	}
+	spin_unlock(&rq->lock);
+
+	schedule();
+
 	return 0;
 }
 
@@ -1078,9 +1674,10 @@
 	switch (policy) {
 	case SCHED_FIFO:
 	case SCHED_RR:
-		ret = 99;
+		ret = MAX_USER_RT_PRIO-1;
 		break;
-	case SCHED_OTHER:
+	case SCHED_NORMAL:
+	case SCHED_BATCH:
 		ret = 0;
 		break;
 	}
@@ -1096,7 +1693,8 @@
 	case SCHED_RR:
 		ret = 1;
 		break;
-	case SCHED_OTHER:
+	case SCHED_NORMAL:
+	case SCHED_BATCH:
 		ret = 0;
 	}
 	return ret;
@@ -1104,9 +1702,9 @@
 
 asmlinkage long sys_sched_rr_get_interval(pid_t pid, struct timespec *interval)
 {
-	struct timespec t;
-	struct task_struct *p;
 	int retval = -EINVAL;
+	struct timespec t;
+	task_t *p;
 
 	if (pid < 0)
 		goto out_nounlock;
@@ -1115,8 +1713,8 @@
 	read_lock(&tasklist_lock);
 	p = find_process_by_pid(pid);
 	if (p)
-		jiffies_to_timespec(p->policy & SCHED_FIFO ? 0 : NICE_TO_TICKS(p->nice),
-				    &t);
+		jiffies_to_timespec(p->policy & SCHED_FIFO ?
+					 0 : task_timeslice(p), &t);
 	read_unlock(&tasklist_lock);
 	if (p)
 		retval = copy_to_user(interval, &t, sizeof(t)) ? -EFAULT : 0;
@@ -1124,14 +1722,14 @@
 	return retval;
 }
 
-static void show_task(struct task_struct * p)
+static void show_task(task_t * p)
 {
 	unsigned long free = 0;
 	int state;
 	static const char * stat_nam[] = { "R", "S", "D", "Z", "T", "W" };
 
 	printk("%-13.13s ", p->comm);
-	state = p->state ? ffz(~p->state) + 1 : 0;
+	state = p->state ? __ffs(p->state) + 1 : 0;
 	if (((unsigned) state) < sizeof(stat_nam)/sizeof(char *))
 		printk(stat_nam[state]);
 	else
@@ -1172,7 +1770,7 @@
 		printk(" (NOTLB)\n");
 
 	{
-		extern void show_trace_task(struct task_struct *tsk);
+		extern void show_trace_task(task_t *tsk);
 		show_trace_task(p);
 	}
 }
@@ -1194,7 +1792,7 @@
 
 void show_state(void)
 {
-	struct task_struct *p;
+	task_t *p;
 
 #if (BITS_PER_LONG == 32)
 	printk("\n"
@@ -1217,121 +1815,244 @@
 	read_unlock(&tasklist_lock);
 }
 
-/**
- * reparent_to_init() - Reparent the calling kernel thread to the init task.
- *
- * If a kernel thread is launched as a result of a system call, or if
- * it ever exits, it should generally reparent itself to init so that
- * it is correctly cleaned up on exit.
- *
- * The various task state such as scheduling policy and priority may have
- * been inherited fro a user process, so we reset them to sane values here.
- *
- * NOTE that reparent_to_init() gives the caller full capabilities.
- */
-void reparent_to_init(void)
+void __init init_idle(task_t *idle, int cpu)
 {
-	struct task_struct *this_task = current;
+	runqueue_t *idle_rq = cpu_rq(cpu), *rq = cpu_rq(task_cpu(idle));
+	unsigned long flags;
+
+	__save_flags(flags);
+	__cli();
+	double_rq_lock(idle_rq, rq);
+
+	idle_rq->curr = idle_rq->idle = idle;
+	deactivate_task(idle, rq);
+	idle->array = NULL;
+	idle->prio = MAX_PRIO;
+	idle->state = TASK_RUNNING;
+	set_task_cpu(idle, cpu);
+	double_rq_unlock(idle_rq, rq);
+	set_tsk_need_resched(idle);
+	__restore_flags(flags);
+}
+
+#if CONFIG_SMP
 
-	write_lock_irq(&tasklist_lock);
+/*
+ * This is how migration works:
+ *
+ * 1) we queue a migration_req_t structure in the source CPU's
+ *    runqueue and wake up that CPU's migration thread.
+ * 2) we down() the locked semaphore => thread blocks.
+ * 3) migration thread wakes up (implicitly it forces the migrated
+ *    thread off the CPU)
+ * 4) it gets the migration request and checks whether the migrated
+ *    task is still in the wrong runqueue.
+ * 5) if it's in the wrong runqueue then the migration thread removes
+ *    it and puts it into the right queue.
+ * 6) migration thread up()s the semaphore.
+ * 7) we wake up and the migration is done.
+ */
 
-	/* Reparent to init */
-	REMOVE_LINKS(this_task);
-	this_task->p_pptr = child_reaper;
-	this_task->p_opptr = child_reaper;
-	SET_LINKS(this_task);
+typedef struct {
+	list_t list;
+	task_t *task;
+	struct semaphore sem;
+} migration_req_t;
 
-	/* Set the exit signal to SIGCHLD so we signal init on exit */
-	this_task->exit_signal = SIGCHLD;
+/*
+ * Change a given task's CPU affinity. Migrate the process to a
+ * proper CPU and schedule it away if the CPU it's executing on
+ * is removed from the allowed bitmask.
+ *
+ * NOTE: the caller must have a valid reference to the task, the
+ * task must not exit() & deallocate itself prematurely.  The
+ * call is not atomic; no spinlocks may be held.
+ */
+void set_cpus_allowed(task_t *p, unsigned long new_mask)
+{
+	unsigned long flags;
+	migration_req_t req;
+	runqueue_t *rq;
 
-	/* We also take the runqueue_lock while altering task fields
-	 * which affect scheduling decisions */
-	spin_lock(&runqueue_lock);
+	new_mask &= cpu_online_map;
+	if (!new_mask)
+		BUG();
 
-	this_task->ptrace = 0;
-	this_task->nice = DEF_NICE;
-	this_task->policy = SCHED_OTHER;
-	/* cpus_allowed? */
-	/* rt_priority? */
-	/* signals? */
-	this_task->cap_effective = CAP_INIT_EFF_SET;
-	this_task->cap_inheritable = CAP_INIT_INH_SET;
-	this_task->cap_permitted = CAP_FULL_SET;
-	this_task->keep_capabilities = 0;
-	memcpy(this_task->rlim, init_task.rlim, sizeof(*(this_task->rlim)));
-	this_task->user = INIT_USER;
+	rq = task_rq_lock(p, &flags);
+	p->cpus_allowed = new_mask;
+	/*
+	 * Can the task run on the task's current CPU? If not then
+	 * migrate the process off to a proper CPU.
+	 */
+	if (new_mask & (1UL << task_cpu(p))) {
+		task_rq_unlock(rq, &flags);
+		goto out;
+	}
+	/*
+	 * If the task is not on a runqueue (and not running), then
+	 * it is sufficient to simply update the task's cpu field.
+	 */
+	if (!p->array && !task_running(rq, p)) {
+		set_task_cpu(p, __ffs(p->cpus_allowed));
+		task_rq_unlock(rq, &flags);
+		goto out;
+	}
+	init_MUTEX_LOCKED(&req.sem);
+	req.task = p;
+	list_add(&req.list, &rq->migration_queue);
+	task_rq_unlock(rq, &flags);
+	wake_up_process(rq->migration_thread);
 
-	spin_unlock(&runqueue_lock);
-	write_unlock_irq(&tasklist_lock);
+	down(&req.sem);
+out:
+	return;
 }
 
-/*
- *	Put all the gunge required to become a kernel thread without
- *	attached user resources in one place where it belongs.
- */
+static __initdata int master_migration_thread;
 
-void daemonize(void)
+static int migration_thread(void * bind_cpu)
 {
-	struct fs_struct *fs;
+	int cpu = (int) (long) bind_cpu;
+	struct sched_param param = { sched_priority: MAX_RT_PRIO-1 };
+	runqueue_t *rq;
+	int ret;
 
+	daemonize();
+	sigfillset(&current->blocked);
+	set_fs(KERNEL_DS);
 
 	/*
-	 * If we were started as result of loading a module, close all of the
-	 * user space pages.  We don't need them, and if we didn't close them
-	 * they would be locked into memory.
+	 * The first migration thread is started on the boot CPU, it
+	 * migrates the other migration threads to their destination CPUs.
 	 */
-	exit_mm(current);
+	if (cpu != master_migration_thread) {
+		while (!cpu_rq(master_migration_thread)->migration_thread)
+			yield();
+		set_cpus_allowed(current, 1UL << cpu);
+	}
+	printk("migration_task %d on cpu=%d\n", cpu, smp_processor_id());
+	ret = setscheduler(0, SCHED_FIFO, &param);
 
-	current->session = 1;
-	current->pgrp = 1;
-	current->tty = NULL;
+	rq = this_rq();
+	rq->migration_thread = current;
 
-	/* Become as one with the init task */
+	sprintf(current->comm, "migration_CPU%d", smp_processor_id());
 
-	exit_fs(current);	/* current->fs->count--; */
-	fs = init_task.fs;
-	current->fs = fs;
-	atomic_inc(&fs->count);
- 	exit_files(current);
-	current->files = init_task.files;
-	atomic_inc(&current->files->count);
-}
+	for (;;) {
+		runqueue_t *rq_src, *rq_dest;
+		struct list_head *head;
+		int cpu_src, cpu_dest;
+		migration_req_t *req;
+		unsigned long flags;
+		task_t *p;
+
+		spin_lock_irqsave(&rq->lock, flags);
+		head = &rq->migration_queue;
+		current->state = TASK_INTERRUPTIBLE;
+		if (list_empty(head)) {
+			spin_unlock_irqrestore(&rq->lock, flags);
+			schedule();
+			continue;
+		}
+		req = list_entry(head->next, migration_req_t, list);
+		list_del_init(head->next);
+		spin_unlock_irqrestore(&rq->lock, flags);
+
+		p = req->task;
+		cpu_dest = __ffs(p->cpus_allowed);
+		rq_dest = cpu_rq(cpu_dest);
+repeat:
+		cpu_src = task_cpu(p);
+		rq_src = cpu_rq(cpu_src);
+
+		local_irq_save(flags);
+		double_rq_lock(rq_src, rq_dest);
+		if (task_cpu(p) != cpu_src) {
+			double_rq_unlock(rq_src, rq_dest);
+			local_irq_restore(flags);
+			goto repeat;
+		}
+		if (rq_src == rq) {
+			set_task_cpu(p, cpu_dest);
+			if (p->array) {
+				deactivate_task(p, rq_src);
+				activate_task(p, rq_dest);
+			}
+		}
+		double_rq_unlock(rq_src, rq_dest);
+		local_irq_restore(flags);
 
-extern unsigned long wait_init_idle;
+		up(&req->sem);
+	}
+}
 
-void __init init_idle(void)
+void __init migration_init(void)
 {
-	struct schedule_data * sched_data;
-	sched_data = &aligned_data[smp_processor_id()].schedule_data;
+	int cpu;
+
+	master_migration_thread = smp_processor_id();
+	current->cpus_allowed = 1UL << master_migration_thread;
+	
+	for (cpu = 0; cpu < NR_CPUS; cpu++) {
+		if (!cpu_online(cpu))
+			continue;
+		if (kernel_thread(migration_thread, (void *) (long) cpu,
+				CLONE_FS | CLONE_FILES | CLONE_SIGNAL) < 0)
+			BUG();
+	}
+	current->cpus_allowed = -1L;
 
-	if (current != &init_task && task_on_runqueue(current)) {
-		printk("UGH! (%d:%d) was on the runqueue, removing.\n",
-			smp_processor_id(), current->pid);
-		del_from_runqueue(current);
+	for (cpu = 0; cpu < NR_CPUS; cpu++) {
+		if (!cpu_online(cpu))
+			continue;
+		while (!cpu_rq(cpu)->migration_thread)
+			schedule_timeout(2);
 	}
-	sched_data->curr = current;
-	sched_data->last_schedule = get_cycles();
-	clear_bit(current->processor, &wait_init_idle);
 }
+#endif
 
-extern void init_timervecs (void);
+extern void init_timervecs(void);
+extern void timer_bh(void);
+extern void tqueue_bh(void);
+extern void immediate_bh(void);
 
 void __init sched_init(void)
 {
+	runqueue_t *rq;
+	int i, j, k;
+
+	for (i = 0; i < NR_CPUS; i++) {
+		prio_array_t *array;
+
+		rq = cpu_rq(i);
+		rq->active = rq->arrays;
+		rq->expired = rq->arrays + 1;
+		spin_lock_init(&rq->lock);
+		INIT_LIST_HEAD(&rq->migration_queue);
+		INIT_LIST_HEAD(&rq->batch_queue);
+		rq->idle_ticks_left = IDLE_TICKS;
+
+		for (j = 0; j < 2; j++) {
+			array = rq->arrays + j;
+			for (k = 0; k < MAX_PRIO; k++) {
+				INIT_LIST_HEAD(array->queue + k);
+				__clear_bit(k, array->bitmap);
+			}
+			// delimiter for bitsearch
+			__set_bit(MAX_PRIO, array->bitmap);
+		}
+	}
 	/*
 	 * We have to do a little magic to get the first
 	 * process right in SMP mode.
 	 */
-	int cpu = smp_processor_id();
-	int nr;
-
-	init_task.processor = cpu;
-
-	for(nr = 0; nr < PIDHASH_SZ; nr++)
-		pidhash[nr] = NULL;
+	rq = this_rq();
+	rq->curr = current;
+	rq->idle = current;
+	set_task_cpu(current, smp_processor_id());
+	wake_up_process(current);
 
 	init_timervecs();
-
 	init_bh(TIMER_BH, timer_bh);
 	init_bh(TQUEUE_BH, tqueue_bh);
 	init_bh(IMMEDIATE_BH, immediate_bh);
@@ -1340,5 +2061,6 @@
 	 * The boot idle thread does lazy MMU switching as well:
 	 */
 	atomic_inc(&init_mm.mm_count);
-	enter_lazy_tlb(&init_mm, current, cpu);
+	enter_lazy_tlb(&init_mm, current, smp_processor_id());
 }
+
--- linux/kernel/signal.c.orig	Thu Jul 18 15:22:25 2002
+++ linux/kernel/signal.c	Thu Jul 18 15:22:34 2002
@@ -507,12 +507,9 @@
 	 * process of changing - but no harm is done by that
 	 * other than doing an extra (lightweight) IPI interrupt.
 	 */
-	spin_lock(&runqueue_lock);
-	if (task_has_cpu(t) && t->processor != smp_processor_id())
-		smp_send_reschedule(t->processor);
-	spin_unlock(&runqueue_lock);
-#endif /* CONFIG_SMP */
-
+	if ((t->state == TASK_RUNNING) && (t->cpu != cpu()))
+		kick_if_running(t);
+#endif
 	if (t->state & TASK_INTERRUPTIBLE) {
 		wake_up_process(t);
 		return;
--- linux/kernel/softirq.c.orig	Thu Jul 18 15:22:25 2002
+++ linux/kernel/softirq.c	Thu Jul 18 15:22:34 2002
@@ -259,10 +259,9 @@
 
 	while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
 		current->state = TASK_RUNNING;
-		do {
-			current->policy |= SCHED_YIELD;
-			schedule();
-		} while (test_bit(TASKLET_STATE_SCHED, &t->state));
+		do
+			sys_sched_yield();
+		while (test_bit(TASKLET_STATE_SCHED, &t->state));
 	}
 	tasklet_unlock_wait(t);
 	clear_bit(TASKLET_STATE_SCHED, &t->state);
@@ -365,13 +364,13 @@
 	int cpu = cpu_logical_map(bind_cpu);
 
 	daemonize();
-	current->nice = 19;
+	set_user_nice(current, 19);
 	sigfillset(&current->blocked);
 
 	/* Migrate to the right CPU */
-	current->cpus_allowed = 1UL << cpu;
-	while (smp_processor_id() != cpu)
-		schedule();
+	set_cpus_allowed(current, 1UL << cpu);
+	if (cpu() != cpu)
+		BUG();
 
 	sprintf(current->comm, "ksoftirqd_CPU%d", bind_cpu);
 
@@ -396,7 +395,7 @@
 	}
 }
 
-static __init int spawn_ksoftirqd(void)
+__init int spawn_ksoftirqd(void)
 {
 	int cpu;
 
@@ -405,10 +404,8 @@
 				  CLONE_FS | CLONE_FILES | CLONE_SIGNAL) < 0)
 			printk("spawn_ksoftirqd() failed for cpu %d\n", cpu);
 		else {
-			while (!ksoftirqd_task(cpu_logical_map(cpu))) {
-				current->policy |= SCHED_YIELD;
-				schedule();
-			}
+			while (!ksoftirqd_task(cpu_logical_map(cpu)))
+				sys_sched_yield();
 		}
 	}
 
--- linux/kernel/sys.c.orig	Thu Jul 18 15:22:25 2002
+++ linux/kernel/sys.c	Thu Jul 18 15:22:34 2002
@@ -220,10 +220,10 @@
 		}
 		if (error == -ESRCH)
 			error = 0;
-		if (niceval < p->nice && !capable(CAP_SYS_NICE))
+		if (niceval < task_nice(p) && !capable(CAP_SYS_NICE))
 			error = -EACCES;
 		else
-			p->nice = niceval;
+			set_user_nice(p, niceval);
 	}
 	read_unlock(&tasklist_lock);
 
@@ -249,7 +249,7 @@
 		long niceval;
 		if (!proc_sel(p, which, who))
 			continue;
-		niceval = 20 - p->nice;
+		niceval = 20 - task_nice(p);
 		if (niceval > retval)
 			retval = niceval;
 	}
--- linux/kernel/timer.c.orig	Thu Jul 18 15:22:25 2002
+++ linux/kernel/timer.c	Thu Jul 18 15:22:34 2002
@@ -25,6 +25,8 @@
 
 #include <asm/uaccess.h>
 
+struct kernel_stat kstat;
+
 /*
  * Timekeeping variables
  */
@@ -598,18 +600,7 @@
 	int cpu = smp_processor_id(), system = user_tick ^ 1;
 
 	update_one_process(p, user_tick, system, cpu);
-	if (p->pid) {
-		if (--p->counter <= 0) {
-			p->counter = 0;
-			p->need_resched = 1;
-		}
-		if (p->nice > 0)
-			kstat.per_cpu_nice[cpu] += user_tick;
-		else
-			kstat.per_cpu_user[cpu] += user_tick;
-		kstat.per_cpu_system[cpu] += system;
-	} else if (local_bh_count(cpu) || local_irq_count(cpu) > 1)
-		kstat.per_cpu_system[cpu] += system;
+	scheduler_tick(user_tick, system);
 }
 
 /*
@@ -617,17 +608,7 @@
  */
 static unsigned long count_active_tasks(void)
 {
-	struct task_struct *p;
-	unsigned long nr = 0;
-
-	read_lock(&tasklist_lock);
-	for_each_task(p) {
-		if ((p->state == TASK_RUNNING ||
-		     (p->state & TASK_UNINTERRUPTIBLE)))
-			nr += FIXED_1;
-	}
-	read_unlock(&tasklist_lock);
-	return nr;
+	return (nr_running() + nr_uninterruptible()) * FIXED_1;
 }
 
 /*
@@ -810,6 +791,89 @@
 
 #endif
 
+static void process_timeout(unsigned long __data)
+{
+	wake_up_process((task_t *)__data);
+}
+
+/**
+ * schedule_timeout - sleep until timeout
+ * @timeout: timeout value in jiffies
+ *
+ * Make the current task sleep until @timeout jiffies have
+ * elapsed. The routine will return immediately unless
+ * the current task state has been set (see set_current_state()).
+ *
+ * You can set the task state as follows -
+ *
+ * %TASK_UNINTERRUPTIBLE - at least @timeout jiffies are guaranteed to
+ * pass before the routine returns. The routine will return 0
+ *
+ * %TASK_INTERRUPTIBLE - the routine may return early if a signal is
+ * delivered to the current task. In this case the remaining time
+ * in jiffies will be returned, or 0 if the timer expired in time
+ *
+ * The current task state is guaranteed to be TASK_RUNNING when this 
+ * routine returns.
+ *
+ * Specifying a @timeout value of %MAX_SCHEDULE_TIMEOUT will schedule
+ * the CPU away without a bound on the timeout. In this case the return
+ * value will be %MAX_SCHEDULE_TIMEOUT.
+ *
+ * In all cases the return value is guaranteed to be non-negative.
+ */
+signed long schedule_timeout(signed long timeout)
+{
+	struct timer_list timer;
+	unsigned long expire;
+
+	switch (timeout)
+	{
+	case MAX_SCHEDULE_TIMEOUT:
+		/*
+		 * These two special cases are useful to be comfortable
+		 * in the caller. Nothing more. We could take
+		 * MAX_SCHEDULE_TIMEOUT from one of the negative value
+		 * but I' d like to return a valid offset (>=0) to allow
+		 * the caller to do everything it want with the retval.
+		 */
+		schedule();
+		goto out;
+	default:
+		/*
+		 * Another bit of PARANOID. Note that the retval will be
+		 * 0 since no piece of kernel is supposed to do a check
+		 * for a negative retval of schedule_timeout() (since it
+		 * should never happens anyway). You just have the printk()
+		 * that will tell you if something is gone wrong and where.
+		 */
+		if (timeout < 0)
+		{
+			printk(KERN_ERR "schedule_timeout: wrong timeout "
+			       "value %lx from %p\n", timeout,
+			       __builtin_return_address(0));
+			current->state = TASK_RUNNING;
+			goto out;
+		}
+	}
+
+	expire = timeout + jiffies;
+
+	init_timer(&timer);
+	timer.expires = expire;
+	timer.data = (unsigned long) current;
+	timer.function = process_timeout;
+
+	add_timer(&timer);
+	schedule();
+	del_timer_sync(&timer);
+
+	timeout = expire - jiffies;
+
+ out:
+	return timeout < 0 ? 0 : timeout;
+}
+
 /* Thread ID - the internal kernel "pid" */
 asmlinkage long sys_gettid(void)
 {
@@ -856,4 +920,3 @@
 	}
 	return 0;
 }
-
--- linux/mm/highmem.c.orig	Thu Jul 18 15:22:25 2002
+++ linux/mm/highmem.c	Thu Jul 18 15:22:34 2002
@@ -355,9 +355,8 @@
 	/* we need to wait I/O completion */
 	run_task_queue(&tq_disk);
 
-	current->policy |= SCHED_YIELD;
 	__set_current_state(TASK_RUNNING);
-	schedule();
+	yield();
 	goto repeat_alloc;
 }
 
@@ -393,9 +392,8 @@
 	/* we need to wait I/O completion */
 	run_task_queue(&tq_disk);
 
-	current->policy |= SCHED_YIELD;
 	__set_current_state(TASK_RUNNING);
-	schedule();
+	yield();
 	goto repeat_alloc;
 }
 
--- linux/mm/oom_kill.c.orig	Thu Jul 18 15:22:25 2002
+++ linux/mm/oom_kill.c	Thu Jul 18 15:22:34 2002
@@ -82,7 +82,7 @@
 	 * Niced processes are most likely less important, so double
 	 * their badness points.
 	 */
-	if (p->nice > 0)
+	if (task_nice(p) > 0)
 		points *= 2;
 
 	/*
@@ -146,7 +146,7 @@
 	 * all the memory it needs. That way it should be able to
 	 * exit() and clear out its resources quickly...
 	 */
-	p->counter = 5 * HZ;
+	p->time_slice = HZ;
 	p->flags |= PF_MEMALLOC | PF_MEMDIE;
 
 	/* This process has hardware access, be more careful. */
@@ -188,8 +188,7 @@
 	 * killing itself before someone else gets the chance to ask
 	 * for more memory.
 	 */
-	current->policy |= SCHED_YIELD;
-	schedule();
+	yield();
 	return;
 }
 
--- linux/mm/page_alloc.c.orig	Thu Jul 18 15:22:25 2002
+++ linux/mm/page_alloc.c	Thu Jul 18 15:22:34 2002
@@ -407,9 +407,8 @@
 		return NULL;
 
 	/* Yield for kswapd, and try again */
-	current->policy |= SCHED_YIELD;
 	__set_current_state(TASK_RUNNING);
-	schedule();
+	yield();
 	goto rebalance;
 }
 
--- linux/include/linux/list.h.orig	Thu Jul 18 15:22:11 2002
+++ linux/include/linux/list.h	Fri Jul 19 15:03:33 2002
@@ -19,6 +19,8 @@
 	struct list_head *next, *prev;
 };
 
+typedef struct list_head list_t;
+
 #define LIST_HEAD_INIT(name) { &(name), &(name) }
 
 #define LIST_HEAD(name) \
--- linux/include/linux/kernel_stat.h.orig	Thu Jul 18 15:22:25 2002
+++ linux/include/linux/kernel_stat.h	Fri Jul 19 15:03:33 2002
@@ -29,8 +29,9 @@
 #if !defined(CONFIG_ARCH_S390)
 	unsigned int irqs[NR_CPUS][NR_IRQS];
 #endif
-	unsigned int context_swtch;
 };
+
+extern unsigned long nr_context_switches(void);
 
 extern struct kernel_stat kstat;
 
--- linux/include/linux/sched.h.orig	Thu Jul 18 15:22:25 2002
+++ linux/include/linux/sched.h	Fri Jul 19 15:03:33 2002
@@ -73,16 +73,16 @@
 #define CT_TO_SECS(x)	((x) / HZ)
 #define CT_TO_USECS(x)	(((x) % HZ) * 1000000/HZ)
 
-extern int nr_running, nr_threads;
+extern int nr_threads;
 extern int last_pid;
+extern unsigned long nr_running(void);
+extern unsigned long nr_uninterruptible(void);
 
 #include <linux/fs.h>
 #include <linux/time.h>
 #include <linux/param.h>
 #include <linux/resource.h>
-#ifdef __KERNEL__
 #include <linux/timer.h>
-#endif
 
 #include <asm/processor.h>
 
@@ -92,6 +92,10 @@
 #define TASK_ZOMBIE		4
 #define TASK_STOPPED		8
 
+#define task_cpu(p) ((p)->cpu)
+#define set_task_cpu(p, c) do { (p)->cpu = (c); } while (0)
+#define cpu_online(i) ((i) < smp_num_cpus)
+
 #define __set_task_state(tsk, state_value)		\
 	do { (tsk)->state = (state_value); } while (0)
 #ifdef CONFIG_SMP
@@ -115,15 +119,13 @@
 /*
  * Scheduling policies
  */
-#define SCHED_OTHER		0
+#define SCHED_NORMAL		0
 #define SCHED_FIFO		1
 #define SCHED_RR		2
+#define SCHED_BATCH		3
 
-/*
- * This is an additional bit set when we want to
- * yield the CPU for one re-schedule..
- */
-#define SCHED_YIELD		0x10
+/* compatibility */
+#define SCHED_OTHER		SCHED_NORMAL
 
 struct sched_param {
 	int sched_priority;
@@ -142,21 +144,27 @@
  * a separate lock).
  */
 extern rwlock_t tasklist_lock;
-extern spinlock_t runqueue_lock;
 extern spinlock_t mmlist_lock;
 
+typedef struct task_struct task_t;
+
 extern void sched_init(void);
-extern void init_idle(void);
+extern void init_idle(task_t *idle, int cpu);
 extern void show_state(void);
 extern void cpu_init (void);
 extern void trap_init(void);
 extern void update_process_times(int user);
-extern void update_one_process(struct task_struct *p, unsigned long user,
+extern void update_one_process(task_t *p, unsigned long user,
 			       unsigned long system, int cpu);
+extern void scheduler_tick(int user_tick, int system);
+extern void migration_init(void);
+extern unsigned long cache_decay_ticks;
+extern int set_user(uid_t new_ruid, int dumpclear);
 
 #define	MAX_SCHEDULE_TIMEOUT	LONG_MAX
 extern signed long FASTCALL(schedule_timeout(signed long timeout));
 asmlinkage void schedule(void);
+asmlinkage void schedule_userspace(void);
 
 extern int schedule_task(struct tq_struct *task);
 extern void flush_scheduled_tasks(void);
@@ -164,6 +172,36 @@
 extern int current_is_keventd(void);
 
 /*
+ * Priority of a process goes from 0..MAX_PRIO-1, valid RT
+ * priority is 0..MAX_RT_PRIO-1, and SCHED_NORMAL tasks are
+ * in the range MAX_RT_PRIO..MAX_PRIO-1. Priority values
+ * are inverted: lower p->prio value means higher priority.
+ *
+ * The MAX_RT_USER_PRIO value allows the actual maximum
+ * RT priority to be separate from the value exported to
+ * user-space.  This allows kernel threads to set their
+ * priority to a value higher than any user task. Note:
+ * MAX_RT_PRIO must not be smaller than MAX_USER_RT_PRIO.
+ */
+
+#define MAX_USER_RT_PRIO	100
+#define MAX_RT_PRIO		MAX_USER_RT_PRIO
+
+#define MAX_PRIO		(MAX_RT_PRIO + 40)
+
+/*
+ * The maximum RT priority is configurable.  If the resulting
+ * bitmap is 160-bits , we can use a hand-coded routine which
+ * is optimal.  Otherwise, we fall back on a generic routine for
+ * finding the first set bit from an arbitrarily-sized bitmap.
+ */
+#if MAX_PRIO < 160 && MAX_PRIO > 127
+#define sched_find_first_bit(map)	_sched_find_first_bit(map)
+#else
+#define sched_find_first_bit(map)	find_first_bit(map, MAX_PRIO)
+#endif
+
+/*
  * The default fd array needs to be at least BITS_PER_LONG,
  * as this is the granularity returned by copy_fdset().
  */
@@ -284,6 +322,8 @@
 extern struct user_struct root_user;
 #define INIT_USER (&root_user)
 
+typedef struct prio_array prio_array_t;
+
 struct task_struct {
 	/*
 	 * offsets of these are hardcoded elsewhere - touch with care
@@ -301,34 +341,25 @@
 
 	int lock_depth;		/* Lock depth */
 
-/*
- * offset 32 begins here on 32-bit platforms. We keep
- * all fields in a single cacheline that are needed for
- * the goodness() loop in schedule().
- */
-	long counter;
-	long nice;
-	unsigned long policy;
-	struct mm_struct *mm;
-	int processor;
 	/*
-	 * cpus_runnable is ~0 if the process is not running on any
-	 * CPU. It's (1 << cpu) if it's running on a CPU. This mask
-	 * is updated under the runqueue lock.
-	 *
-	 * To determine whether a process might run on a CPU, this
-	 * mask is AND-ed with cpus_allowed.
+	 * offset 32 begins here on 32-bit platforms.
 	 */
-	unsigned long cpus_runnable, cpus_allowed;
-	/*
-	 * (only the 'next' pointer fits into the cacheline, but
-	 * that's just fine.)
-	 */
-	struct list_head run_list;
-	unsigned long sleep_time;
+	unsigned int cpu;
+	int prio, static_prio;
+	list_t run_list;
+	prio_array_t *array;
+
+	unsigned long sleep_avg;
+	unsigned long sleep_timestamp;
+
+	unsigned long policy;
+	unsigned long cpus_allowed;
+	unsigned int time_slice, first_time_slice;
+
+	task_t *next_task, *prev_task;
+
+	struct mm_struct *mm, *active_mm;
 
-	struct task_struct *next_task, *prev_task;
-	struct mm_struct *active_mm;
 	struct list_head local_pages;
 	unsigned int allocation_order, nr_local_pages;
 
@@ -351,12 +382,12 @@
 	 * older sibling, respectively.  (p->father can be replaced with 
 	 * p->p_pptr->pid)
 	 */
-	struct task_struct *p_opptr, *p_pptr, *p_cptr, *p_ysptr, *p_osptr;
+	task_t *p_opptr, *p_pptr, *p_cptr, *p_ysptr, *p_osptr;
 	struct list_head thread_group;
 
 	/* PID hash table linkage. */
-	struct task_struct *pidhash_next;
-	struct task_struct **pidhash_pprev;
+	task_t *pidhash_next;
+	task_t **pidhash_pprev;
 
 	wait_queue_head_t wait_chldexit;	/* for wait4() */
 	struct completion *vfork_done;		/* for vfork() */
@@ -435,6 +466,7 @@
 #define PF_MEMDIE	0x00001000	/* Killed for out-of-memory */
 #define PF_FREE_PAGES	0x00002000	/* per process page freeing */
 #define PF_NOIO		0x00004000	/* avoid generating further I/O */
+#define PF_BATCH	0x00080000	/* batch-priority process */
 
 #define PF_USEDFPU	0x00100000	/* task used FPU this quantum (SMP) */
 
@@ -454,12 +486,19 @@
  */
 #define _STK_LIM	(8*1024*1024)
 
-#define DEF_COUNTER	(10*HZ/100)	/* 100 ms time slice */
-#define MAX_COUNTER	(20*HZ/100)
-#define DEF_NICE	(0)
+#if CONFIG_SMP
+extern void set_cpus_allowed(task_t *p, unsigned long new_mask);
+#else
+#define set_cpus_allowed(p, new_mask)	do { } while (0)
+#endif
+
+extern void set_user_nice(task_t *p, long nice);
+extern int task_prio(task_t *p);
+extern int task_nice(task_t *p);
+extern int idle_cpu(int cpu);
 
 asmlinkage long sys_sched_yield(void);
-#define yield()	sys_sched_yield()
+#define yield() sys_sched_yield()
 
 /*
  * The default (Linux) execution domain.
@@ -478,14 +517,14 @@
     addr_limit:		KERNEL_DS,					\
     exec_domain:	&default_exec_domain,				\
     lock_depth:		-1,						\
-    counter:		DEF_COUNTER,					\
-    nice:		DEF_NICE,					\
-    policy:		SCHED_OTHER,					\
+    prio:		MAX_PRIO-20,					\
+    static_prio:	MAX_PRIO-20,					\
+    policy:		SCHED_NORMAL,					\
+    cpus_allowed:	-1,						\
     mm:			NULL,						\
     active_mm:		&init_mm,					\
-    cpus_runnable:	-1,						\
-    cpus_allowed:	-1,						\
     run_list:		LIST_HEAD_INIT(tsk.run_list),			\
+    time_slice:		HZ,						\
     next_task:		&tsk,						\
     prev_task:		&tsk,						\
     p_opptr:		&tsk,						\
@@ -519,24 +558,24 @@
 #endif
 
 union task_union {
-	struct task_struct task;
+	task_t task;
 	unsigned long stack[INIT_TASK_SIZE/sizeof(long)];
 };
 
 extern union task_union init_task_union;
 
 extern struct   mm_struct init_mm;
-extern struct task_struct *init_tasks[NR_CPUS];
+extern task_t *init_tasks[NR_CPUS];
 
 /* PID hashing. (shouldnt this be dynamic?) */
 #define PIDHASH_SZ (4096 >> 2)
-extern struct task_struct *pidhash[PIDHASH_SZ];
+extern task_t *pidhash[PIDHASH_SZ];
 
 #define pid_hashfn(x)	((((x) >> 8) ^ (x)) & (PIDHASH_SZ - 1))
 
-static inline void hash_pid(struct task_struct *p)
+static inline void hash_pid(task_t *p)
 {
-	struct task_struct **htable = &pidhash[pid_hashfn(p->pid)];
+	task_t **htable = &pidhash[pid_hashfn(p->pid)];
 
 	if((p->pidhash_next = *htable) != NULL)
 		(*htable)->pidhash_pprev = &p->pidhash_next;
@@ -544,16 +583,16 @@
 	p->pidhash_pprev = htable;
 }
 
-static inline void unhash_pid(struct task_struct *p)
+static inline void unhash_pid(task_t *p)
 {
 	if(p->pidhash_next)
 		p->pidhash_next->pidhash_pprev = p->pidhash_pprev;
 	*p->pidhash_pprev = p->pidhash_next;
 }
 
-static inline struct task_struct *find_task_by_pid(int pid)
+static inline task_t *find_task_by_pid(int pid)
 {
-	struct task_struct *p, **htable = &pidhash[pid_hashfn(pid)];
+	task_t *p, **htable = &pidhash[pid_hashfn(pid)];
 
 	for(p = *htable; p && p->pid != pid; p = p->pidhash_next)
 		;
@@ -561,19 +600,6 @@
 	return p;
 }
 
-#define task_has_cpu(tsk) ((tsk)->cpus_runnable != ~0UL)
-
-static inline void task_set_cpu(struct task_struct *tsk, unsigned int cpu)
-{
-	tsk->processor = cpu;
-	tsk->cpus_runnable = 1UL << cpu;
-}
-
-static inline void task_release_cpu(struct task_struct *tsk)
-{
-	tsk->cpus_runnable = ~0UL;
-}
-
 /* per-UID process charging. */
 extern struct user_struct * alloc_uid(uid_t);
 extern void free_uid(struct user_struct *);
@@ -600,47 +626,51 @@
 extern void FASTCALL(interruptible_sleep_on(wait_queue_head_t *q));
 extern long FASTCALL(interruptible_sleep_on_timeout(wait_queue_head_t *q,
 						    signed long timeout));
-extern int FASTCALL(wake_up_process(struct task_struct * tsk));
+extern int FASTCALL(wake_up_process(task_t * tsk));
+extern void FASTCALL(wake_up_forked_process(task_t * tsk));
+extern void FASTCALL(sched_exit(task_t * p));
 
 #define wake_up(x)			__wake_up((x),TASK_UNINTERRUPTIBLE | TASK_INTERRUPTIBLE, 1)
 #define wake_up_nr(x, nr)		__wake_up((x),TASK_UNINTERRUPTIBLE | TASK_INTERRUPTIBLE, nr)
 #define wake_up_all(x)			__wake_up((x),TASK_UNINTERRUPTIBLE | TASK_INTERRUPTIBLE, 0)
-#define wake_up_sync(x)			__wake_up_sync((x),TASK_UNINTERRUPTIBLE | TASK_INTERRUPTIBLE, 1)
-#define wake_up_sync_nr(x, nr)		__wake_up_sync((x),TASK_UNINTERRUPTIBLE | TASK_INTERRUPTIBLE, nr)
 #define wake_up_interruptible(x)	__wake_up((x),TASK_INTERRUPTIBLE, 1)
 #define wake_up_interruptible_nr(x, nr)	__wake_up((x),TASK_INTERRUPTIBLE, nr)
 #define wake_up_interruptible_all(x)	__wake_up((x),TASK_INTERRUPTIBLE, 0)
-#define wake_up_interruptible_sync(x)	__wake_up_sync((x),TASK_INTERRUPTIBLE, 1)
-#define wake_up_interruptible_sync_nr(x) __wake_up_sync((x),TASK_INTERRUPTIBLE,  nr)
+#ifdef CONFIG_SMP
+#define wake_up_interruptible_sync(x)   __wake_up_sync((x),TASK_INTERRUPTIBLE, 1)
+#else
+#define wake_up_interruptible_sync(x)   __wake_up((x),TASK_INTERRUPTIBLE, 1)
+#endif
+
 asmlinkage long sys_wait4(pid_t pid,unsigned int * stat_addr, int options, struct rusage * ru);
 
 extern int in_group_p(gid_t);
 extern int in_egroup_p(gid_t);
 
 extern void proc_caches_init(void);
-extern void flush_signals(struct task_struct *);
-extern void flush_signal_handlers(struct task_struct *);
+extern void flush_signals(task_t *);
+extern void flush_signal_handlers(task_t *);
 extern void sig_exit(int, int, struct siginfo *);
 extern int dequeue_signal(sigset_t *, siginfo_t *);
 extern void block_all_signals(int (*notifier)(void *priv), void *priv,
 			      sigset_t *mask);
 extern void unblock_all_signals(void);
-extern int send_sig_info(int, struct siginfo *, struct task_struct *);
-extern int force_sig_info(int, struct siginfo *, struct task_struct *);
+extern int send_sig_info(int, struct siginfo *, task_t *);
+extern int force_sig_info(int, struct siginfo *, task_t *);
 extern int kill_pg_info(int, struct siginfo *, pid_t);
 extern int kill_sl_info(int, struct siginfo *, pid_t);
 extern int kill_proc_info(int, struct siginfo *, pid_t);
-extern void notify_parent(struct task_struct *, int);
-extern void do_notify_parent(struct task_struct *, int);
-extern void force_sig(int, struct task_struct *);
-extern int send_sig(int, struct task_struct *, int);
+extern void notify_parent(task_t *, int);
+extern void do_notify_parent(task_t *, int);
+extern void force_sig(int, task_t *);
+extern int send_sig(int, task_t *, int);
 extern int kill_pg(pid_t, int, int);
 extern int kill_sl(pid_t, int, int);
 extern int kill_proc(pid_t, int, int);
 extern int do_sigaction(int, const struct k_sigaction *, struct k_sigaction *);
 extern int do_sigaltstack(const stack_t *, stack_t *, unsigned long);
 
-static inline int signal_pending(struct task_struct *p)
+static inline int signal_pending(task_t *p)
 {
 	return (p->sigpending != 0);
 }
@@ -679,7 +709,7 @@
    This is required every time the blocked sigset_t changes.
    All callers should have t->sigmask_lock.  */
 
-static inline void recalc_sigpending(struct task_struct *t)
+static inline void recalc_sigpending(task_t *t)
 {
 	t->sigpending = has_pending_signals(&t->pending.signal, &t->blocked);
 }
@@ -786,16 +816,17 @@
 extern int expand_fdset(struct files_struct *, int nr);
 extern void free_fdset(fd_set *, int);
 
-extern int  copy_thread(int, unsigned long, unsigned long, unsigned long, struct task_struct *, struct pt_regs *);
+extern int  copy_thread(int, unsigned long, unsigned long, unsigned long, task_t *, struct pt_regs *);
 extern void flush_thread(void);
 extern void exit_thread(void);
 
-extern void exit_mm(struct task_struct *);
-extern void exit_files(struct task_struct *);
-extern void exit_sighand(struct task_struct *);
+extern void exit_mm(task_t *);
+extern void exit_files(task_t *);
+extern void exit_sighand(task_t *);
 
 extern void reparent_to_init(void);
 extern void daemonize(void);
+extern task_t *child_reaper;
 
 extern int do_execve(char *, char **, char **, struct pt_regs *);
 extern int do_fork(unsigned long, unsigned long, struct pt_regs *, unsigned long);
@@ -804,6 +835,9 @@
 extern void FASTCALL(add_wait_queue_exclusive(wait_queue_head_t *q, wait_queue_t * wait));
 extern void FASTCALL(remove_wait_queue(wait_queue_head_t *q, wait_queue_t * wait));
 
+extern void wait_task_inactive(task_t * p);
+extern void kick_if_running(task_t * p);
+
 #define __wait_event(wq, condition) 					\
 do {									\
 	wait_queue_t __wait;						\
@@ -885,27 +919,12 @@
 	for (task = next_thread(current) ; task != current ; task = next_thread(task))
 
 #define next_thread(p) \
-	list_entry((p)->thread_group.next, struct task_struct, thread_group)
+	list_entry((p)->thread_group.next, task_t, thread_group)
 
 #define thread_group_leader(p)	(p->pid == p->tgid)
 
-static inline void del_from_runqueue(struct task_struct * p)
-{
-	nr_running--;
-	p->sleep_time = jiffies;
-	list_del(&p->run_list);
-	p->run_list.next = NULL;
-}
-
-static inline int task_on_runqueue(struct task_struct *p)
+static inline void unhash_process(task_t *p)
 {
-	return (p->run_list.next != NULL);
-}
-
-static inline void unhash_process(struct task_struct *p)
-{
-	if (task_on_runqueue(p))
-		out_of_line_bug();
 	write_lock_irq(&tasklist_lock);
 	nr_threads--;
 	unhash_pid(p);
@@ -915,12 +934,12 @@
 }
 
 /* Protects ->fs, ->files, ->mm, and synchronises with wait4().  Nests inside tasklist_lock */
-static inline void task_lock(struct task_struct *p)
+static inline void task_lock(task_t *p)
 {
 	spin_lock(&p->alloc_lock);
 }
 
-static inline void task_unlock(struct task_struct *p)
+static inline void task_unlock(task_t *p)
 {
 	spin_unlock(&p->alloc_lock);
 }
@@ -942,6 +961,31 @@
 	dput(root);
 	mntput(rootmnt);
 	return res;
+}
+
+static inline void set_need_resched(void)
+{
+	current->need_resched = 1;
+}
+
+static inline void clear_need_resched(void)
+{
+	current->need_resched = 0;
+}
+
+static inline void set_tsk_need_resched(struct task_struct *tsk)
+{
+	tsk->need_resched = 1;
+}
+
+static inline void clear_tsk_need_resched(struct task_struct *tsk)
+{
+	tsk->need_resched = 0;
+}
+
+static inline int need_resched(void)
+{
+	return unlikely(current->need_resched);
 }
 
 #endif /* __KERNEL__ */
--- linux/include/linux/smp.h.orig	Sun Dec 31 20:10:17 2000
+++ linux/include/linux/smp.h	Fri Jul 19 15:03:33 2002
@@ -86,6 +86,14 @@
 #define cpu_number_map(cpu)			0
 #define smp_call_function(func,info,retry,wait)	({ 0; })
 #define cpu_online_map				1
+static inline void smp_send_reschedule(int cpu) { }
+static inline void smp_send_reschedule_all(void) { }
 
 #endif
+
+/*
+ * Common definitions:
+ */
+#define cpu()					smp_processor_id()
+
 #endif
--- linux/include/asm-generic/bitops.h.orig	Tue Nov 28 02:47:38 2000
+++ linux/include/asm-generic/bitops.h	Thu Jul 18 15:22:34 2002
@@ -51,6 +51,12 @@
 	return ((mask & *addr) != 0);
 }
 
+/*
+ * fls: find last bit set.
+ */
+
+#define fls(x) generic_fls(x)
+
 #ifdef __KERNEL__
 
 /*
--- linux/include/asm-i386/bitops.h.orig	Tue Aug 21 14:26:16 2001
+++ linux/include/asm-i386/bitops.h	Fri Jul 19 15:03:33 2002
@@ -6,6 +6,7 @@
  */
 
 #include <linux/config.h>
+#include <linux/compiler.h>
 
 /*
  * These have to be done with inline assembly: that way the bit-setting
@@ -75,6 +76,14 @@
 		:"=m" (ADDR)
 		:"Ir" (nr));
 }
+
+static __inline__ void __clear_bit(int nr, volatile void * addr)
+{
+	__asm__ __volatile__(
+		"btrl %1,%0"
+		:"=m" (ADDR)
+		:"Ir" (nr));
+}
 #define smp_mb__before_clear_bit()	barrier()
 #define smp_mb__after_clear_bit()	barrier()
 
@@ -284,6 +293,34 @@
 }
 
 /**
+ * find_first_bit - find the first set bit in a memory region
+ * @addr: The address to start the search at
+ * @size: The maximum size to search
+ *
+ * Returns the bit-number of the first set bit, not the number of the byte
+ * containing a bit.
+ */
+static __inline__ int find_first_bit(void * addr, unsigned size)
+{
+	int d0, d1;
+	int res;
+
+	/* This looks at memory. Mark it volatile to tell gcc not to move it around */
+	__asm__ __volatile__(
+		"xorl %%eax,%%eax\n\t"
+		"repe; scasl\n\t"
+		"jz 1f\n\t"
+		"leal -4(%%edi),%%edi\n\t"
+		"bsfl (%%edi),%%eax\n"
+		"1:\tsubl %%ebx,%%edi\n\t"
+		"shll $3,%%edi\n\t"
+		"addl %%edi,%%eax"
+		:"=a" (res), "=&c" (d0), "=&D" (d1)
+		:"1" ((size + 31) >> 5), "2" (addr), "b" (addr));
+	return res;
+}
+
+/**
  * find_next_zero_bit - find the first zero bit in a memory region
  * @addr: The address to base the search on
  * @offset: The bitnumber to start searching at
@@ -296,7 +333,7 @@
 	
 	if (bit) {
 		/*
-		 * Look for zero in first byte
+		 * Look for zero in the first 32 bits.
 		 */
 		__asm__("bsfl %1,%0\n\t"
 			"jne 1f\n\t"
@@ -317,6 +354,39 @@
 }
 
 /**
+ * find_next_bit - find the first set bit in a memory region
+ * @addr: The address to base the search on
+ * @offset: The bitnumber to start searching at
+ * @size: The maximum size to search
+ */
+static __inline__ int find_next_bit (void * addr, int size, int offset)
+{
+	unsigned long * p = ((unsigned long *) addr) + (offset >> 5);
+	int set = 0, bit = offset & 31, res;
+	
+	if (bit) {
+		/*
+		 * Look for nonzero in the first 32 bits:
+		 */
+		__asm__("bsfl %1,%0\n\t"
+			"jne 1f\n\t"
+			"movl $32, %0\n"
+			"1:"
+			: "=r" (set)
+			: "r" (*p >> bit));
+		if (set < (32 - bit))
+			return set + offset;
+		set = 32 - bit;
+		p++;
+	}
+	/*
+	 * No set bit yet, search remaining full words for a bit
+	 */
+	res = find_first_bit (p, size - 32 * (p - (unsigned long *) addr));
+	return (offset + set + res);
+}
+
+/**
  * ffz - find first zero in word.
  * @word: The word to search
  *
@@ -330,7 +400,40 @@
 	return word;
 }
 
+/**
+ * __ffs - find first bit in word.
+ * @word: The word to search
+ *
+ * Undefined if no bit exists, so code should check against 0 first.
+ */
+static __inline__ unsigned long __ffs(unsigned long word)
+{
+	__asm__("bsfl %1,%0"
+		:"=r" (word)
+		:"rm" (word));
+	return word;
+}
+
 #ifdef __KERNEL__
+
+/*
+ * Every architecture must define this function. It's the fastest
+ * way of searching a 140-bit bitmap where the first 100 bits are
+ * unlikely to be set. It's guaranteed that at least one of the 140
+ * bits is cleared.
+ */
+static inline int _sched_find_first_bit(unsigned long *b)
+{
+	if (unlikely(b[0]))
+		return __ffs(b[0]);
+	if (unlikely(b[1]))
+		return __ffs(b[1]) + 32;
+	if (unlikely(b[2]))
+		return __ffs(b[2]) + 64;
+	if (b[3])
+		return __ffs(b[3]) + 96;
+	return __ffs(b[4]) + 128;
+}
 
 /**
  * ffs - find first bit set
--- linux/include/asm-i386/mmu_context.h.orig	Thu Jul 18 15:22:23 2002
+++ linux/include/asm-i386/mmu_context.h	Fri Jul 19 15:03:33 2002
@@ -27,13 +27,13 @@
 
 static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next, struct task_struct *tsk, unsigned cpu)
 {
-	if (prev != next) {
+	if (likely(prev != next)) {
 		/* stop flush ipis for the previous mm */
 		clear_bit(cpu, &prev->cpu_vm_mask);
 		/*
 		 * Re-load LDT if necessary
 		 */
-		if (prev->context.segments != next->context.segments)
+		if (unlikely(prev->context.segments != next->context.segments))
 			load_LDT(next);
 #ifdef CONFIG_SMP
 		cpu_tlbstate[cpu].state = TLBSTATE_OK;
--- linux/include/asm-i386/smp.h.orig	Thu Jul 18 15:22:11 2002
+++ linux/include/asm-i386/smp.h	Fri Jul 19 15:03:33 2002
@@ -63,6 +63,7 @@
 extern void smp_flush_tlb(void);
 extern void smp_message_irq(int cpl, void *dev_id, struct pt_regs *regs);
 extern void smp_send_reschedule(int cpu);
+extern void smp_send_reschedule_all(void);
 extern void smp_invalidate_rcv(void);		/* Process an NMI */
 extern void (*mtrr_hook) (void);
 extern void zap_low_mappings (void);
@@ -104,7 +105,7 @@
  * so this is correct in the x86 case.
  */
 
-#define smp_processor_id() (current->processor)
+#define smp_processor_id() (current->cpu)
 
 static __inline int hard_smp_processor_id(void)
 {
@@ -121,18 +122,6 @@
 #endif /* !__ASSEMBLY__ */
 
 #define NO_PROC_ID		0xFF		/* No processor magic marker */
-
-/*
- *	This magic constant controls our willingness to transfer
- *	a process across CPUs. Such a transfer incurs misses on the L1
- *	cache, and on a P6 or P5 with multiple L2 caches L2 hits. My
- *	gut feeling is this will vary by board in value. For a board
- *	with separate L2 cache it probably depends also on the RSS, and
- *	for a board with shared L2 cache it ought to decay fast as other
- *	processes are run.
- */
- 
-#define PROC_CHANGE_PENALTY	15		/* Schedule penalty */
 
 #endif
 #endif
--- linux/include/asm-i386/system.h.orig	Thu Jul 18 15:22:09 2002
+++ linux/include/asm-i386/system.h	Fri Jul 19 15:03:33 2002
@@ -12,25 +12,22 @@
 struct task_struct;	/* one of the stranger aspects of C forward declarations.. */
 extern void FASTCALL(__switch_to(struct task_struct *prev, struct task_struct *next));
 
-#define prepare_to_switch()	do { } while(0)
 #define switch_to(prev,next,last) do {					\
 	asm volatile("pushl %%esi\n\t"					\
 		     "pushl %%edi\n\t"					\
 		     "pushl %%ebp\n\t"					\
 		     "movl %%esp,%0\n\t"	/* save ESP */		\
-		     "movl %3,%%esp\n\t"	/* restore ESP */	\
+		     "movl %2,%%esp\n\t"	/* restore ESP */	\
 		     "movl $1f,%1\n\t"		/* save EIP */		\
-		     "pushl %4\n\t"		/* restore EIP */	\
+		     "pushl %3\n\t"		/* restore EIP */	\
 		     "jmp __switch_to\n"				\
 		     "1:\t"						\
 		     "popl %%ebp\n\t"					\
 		     "popl %%edi\n\t"					\
 		     "popl %%esi\n\t"					\
-		     :"=m" (prev->thread.esp),"=m" (prev->thread.eip),	\
-		      "=b" (last)					\
+		     :"=m" (prev->thread.esp),"=m" (prev->thread.eip)	\
 		     :"m" (next->thread.esp),"m" (next->thread.eip),	\
-		      "a" (prev), "d" (next),				\
-		      "b" (prev));					\
+		      "a" (prev), "d" (next));				\
 } while (0)
 
 #define _set_base(addr,base) do { unsigned long __pr; \
--- linux/include/asm-mips/bitops.h.orig	Thu Jul 18 15:22:24 2002
+++ linux/include/asm-mips/bitops.h	Thu Jul 18 15:22:34 2002
@@ -43,6 +43,8 @@
 
 #ifdef CONFIG_CPU_HAS_LLSC
 
+#include <asm/mipsregs.h>
+
 /*
  * These functions for MIPS ISA > 1 are interrupt and SMP proof and
  * interrupt friendly
@@ -628,7 +630,8 @@
 		"2:"
 		: "=r" (res), "=r" (dummy), "=r" (addr)
 		: "0" ((signed int) 0), "1" ((unsigned int) 0xffffffff),
-		  "2" (addr), "r" (size));
+		  "2" (addr), "r" (size)
+		: "$1");
 
 	return res;
 }
@@ -663,7 +666,8 @@
 			".set\treorder\n"
 			"1:"
 			: "=r" (set), "=r" (dummy)
-			: "0" (0), "1" (1 << bit), "r" (*p));
+			: "0" (0), "1" (1 << bit), "r" (*p)
+			: "$1");
 		if (set < (32 - bit))
 			return set + offset;
 		set = 32 - bit;
@@ -684,20 +688,29 @@
  *
  * Undefined if no zero exists, so code should check against ~0UL first.
  */
-static __inline__ unsigned long ffz(unsigned long word)
+extern __inline__ unsigned long ffz(unsigned long word)
 {
-	int b = 0, s;
+	unsigned int	__res;
+	unsigned int	mask = 1;
 
-	word = ~word;
-	s = 16; if (word << 16 != 0) s = 0; b += s; word >>= s;
-	s =  8; if (word << 24 != 0) s = 0; b += s; word >>= s;
-	s =  4; if (word << 28 != 0) s = 0; b += s; word >>= s;
-	s =  2; if (word << 30 != 0) s = 0; b += s; word >>= s;
-	s =  1; if (word << 31 != 0) s = 0; b += s;
+	__asm__ (
+		".set\tnoreorder\n\t"
+		".set\tnoat\n\t"
+		"move\t%0,$0\n"
+		"1:\tand\t$1,%2,%1\n\t"
+		"beqz\t$1,2f\n\t"
+		"sll\t%1,1\n\t"
+		"bnez\t%1,1b\n\t"
+		"addiu\t%0,1\n\t"
+		".set\tat\n\t"
+		".set\treorder\n"
+		"2:\n\t"
+		: "=&r" (__res), "=r" (mask)
+		: "r" (word), "1" (mask)
+		: "$1");
 
-	return b;
+	return __res;
 }
-
 
 #ifdef __KERNEL__
 
--- linux/include/asm-alpha/bitops.h.orig	Thu Jul 18 15:22:00 2002
+++ linux/include/asm-alpha/bitops.h	Thu Jul 18 15:22:34 2002
@@ -3,6 +3,7 @@
 
 #include <linux/config.h>
 #include <linux/kernel.h>
+#include <asm/compiler.h>
 
 /*
  * Copyright 1994, Linus Torvalds.
@@ -60,25 +61,25 @@
 
 	__asm__ __volatile__(
 	"1:	ldl_l %0,%3\n"
-	"	and %0,%2,%0\n"
+	"	bic %0,%2,%0\n"
 	"	stl_c %0,%1\n"
 	"	beq %0,2f\n"
 	".subsection 2\n"
 	"2:	br 1b\n"
 	".previous"
 	:"=&r" (temp), "=m" (*m)
-	:"Ir" (~(1UL << (nr & 31))), "m" (*m));
+	:"Ir" (1UL << (nr & 31)), "m" (*m));
 }
 
 /*
  * WARNING: non atomic version.
  */
 static __inline__ void
-__change_bit(unsigned long nr, volatile void * addr)
+__clear_bit(unsigned long nr, volatile void * addr)
 {
 	int *m = ((int *) addr) + (nr >> 5);
 
-	*m ^= 1 << (nr & 31);
+	*m &= ~(1 << (nr & 31));
 }
 
 static inline void
@@ -99,6 +100,17 @@
 	:"Ir" (1UL << (nr & 31)), "m" (*m));
 }
 
+/*
+ * WARNING: non atomic version.
+ */
+static __inline__ void
+__change_bit(unsigned long nr, volatile void * addr)
+{
+	int *m = ((int *) addr) + (nr >> 5);
+
+	*m ^= 1 << (nr & 31);
+}
+
 static inline int
 test_and_set_bit(unsigned long nr, volatile void *addr)
 {
@@ -181,20 +193,6 @@
 	return (old & mask) != 0;
 }
 
-/*
- * WARNING: non atomic version.
- */
-static __inline__ int
-__test_and_change_bit(unsigned long nr, volatile void * addr)
-{
-	unsigned long mask = 1 << (nr & 0x1f);
-	int *m = ((int *) addr) + (nr >> 5);
-	int old = *m;
-
-	*m = old ^ mask;
-	return (old & mask) != 0;
-}
-
 static inline int
 test_and_change_bit(unsigned long nr, volatile void * addr)
 {
@@ -220,6 +218,20 @@
 	return oldbit != 0;
 }
 
+/*
+ * WARNING: non atomic version.
+ */
+static __inline__ int
+__test_and_change_bit(unsigned long nr, volatile void * addr)
+{
+	unsigned long mask = 1 << (nr & 0x1f);
+	int *m = ((int *) addr) + (nr >> 5);
+	int old = *m;
+
+	*m = old ^ mask;
+	return (old & mask) != 0;
+}
+
 static inline int
 test_bit(int nr, volatile void * addr)
 {
@@ -235,12 +247,15 @@
  */
 static inline unsigned long ffz_b(unsigned long x)
 {
-	unsigned long sum = 0;
+	unsigned long sum, x1, x2, x4;
 
 	x = ~x & -~x;		/* set first 0 bit, clear others */
-	if (x & 0xF0) sum += 4;
-	if (x & 0xCC) sum += 2;
-	if (x & 0xAA) sum += 1;
+	x1 = x & 0xAA;
+	x2 = x & 0xCC;
+	x4 = x & 0xF0;
+	sum = x2 ? 2 : 0;
+	sum += (x4 != 0) * 4;
+	sum += (x1 != 0);
 
 	return sum;
 }
@@ -257,24 +272,46 @@
 
 	__asm__("cmpbge %1,%2,%0" : "=r"(bits) : "r"(word), "r"(~0UL));
 	qofs = ffz_b(bits);
-	__asm__("extbl %1,%2,%0" : "=r"(bits) : "r"(word), "r"(qofs));
+	bits = __kernel_extbl(word, qofs);
 	bofs = ffz_b(bits);
 
 	return qofs*8 + bofs;
 #endif
 }
 
+/*
+ * __ffs = Find First set bit in word.  Undefined if no set bit exists.
+ */
+static inline unsigned long __ffs(unsigned long word)
+{
+#if defined(__alpha_cix__) && defined(__alpha_fix__)
+	/* Whee.  EV67 can calculate it directly.  */
+	unsigned long result;
+	__asm__("cttz %1,%0" : "=r"(result) : "r"(word));
+	return result;
+#else
+	unsigned long bits, qofs, bofs;
+
+	__asm__("cmpbge $31,%1,%0" : "=r"(bits) : "r"(word));
+	qofs = ffz_b(bits);
+	bits = __kernel_extbl(word, qofs);
+	bofs = ffz_b(~bits);
+
+	return qofs*8 + bofs;
+#endif
+}
+
 #ifdef __KERNEL__
 
 /*
  * ffs: find first bit set. This is defined the same way as
  * the libc and compiler builtin ffs routines, therefore
- * differs in spirit from the above ffz (man ffs).
+ * differs in spirit from the above __ffs.
  */
 
 static inline int ffs(int word)
 {
-	int result = ffz(~word);
+	int result = __ffs(word);
 	return word ? result+1 : 0;
 }
 
@@ -316,6 +353,14 @@
 #define hweight16(x) hweight64((x) & 0xfffful)
 #define hweight8(x)  hweight64((x) & 0xfful)
 #else
+static inline unsigned long hweight64(unsigned long w)
+{
+	unsigned long result;
+	for (result = 0; w ; w >>= 1)
+		result += (w & 1);
+	return result;
+}
+
 #define hweight32(x) generic_hweight32(x)
 #define hweight16(x) generic_hweight16(x)
 #define hweight8(x)  generic_hweight8(x)
@@ -365,12 +410,76 @@
 }
 
 /*
- * The optimizer actually does good code for this case..
+ * Find next one bit in a bitmap reasonably efficiently.
+ */
+static inline unsigned long
+find_next_bit(void * addr, unsigned long size, unsigned long offset)
+{
+	unsigned long * p = ((unsigned long *) addr) + (offset >> 6);
+	unsigned long result = offset & ~63UL;
+	unsigned long tmp;
+
+	if (offset >= size)
+		return size;
+	size -= result;
+	offset &= 63UL;
+	if (offset) {
+		tmp = *(p++);
+		tmp &= ~0UL << offset;
+		if (size < 64)
+			goto found_first;
+		if (tmp)
+			goto found_middle;
+		size -= 64;
+		result += 64;
+	}
+	while (size & ~63UL) {
+		if ((tmp = *(p++)))
+			goto found_middle;
+		result += 64;
+		size -= 64;
+	}
+	if (!size)
+		return result;
+	tmp = *p;
+found_first:
+	tmp &= ~0UL >> (64 - size);
+	if (!tmp)
+		return result + size;
+found_middle:
+	return result + __ffs(tmp);
+}
+
+/*
+ * The optimizer actually does good code for this case.
  */
 #define find_first_zero_bit(addr, size) \
 	find_next_zero_bit((addr), (size), 0)
+#define find_first_bit(addr, size) \
+	find_next_bit((addr), (size), 0)
 
 #ifdef __KERNEL__
+
+/*
+ * Every architecture must define this function. It's the fastest
+ * way of searching a 140-bit bitmap where the first 100 bits are
+ * unlikely to be set. It's guaranteed that at least one of the 140
+ * bits is set.
+ */
+static inline unsigned long
+sched_find_first_bit(unsigned long b[3])
+{
+	unsigned long b0 = b[0], b1 = b[1], b2 = b[2];
+	unsigned long ofs;
+
+	ofs = (b1 ? 64 : 128);
+	b1 = (b1 ? b1 : b2);
+	ofs = (b0 ? 0 : ofs);
+	b0 = (b0 ? b0 : b1);
+
+	return __ffs(b0) + ofs;
+}
+
 
 #define ext2_set_bit                 __test_and_set_bit
 #define ext2_clear_bit               __test_and_clear_bit
--- linux/include/asm-m68k/bitops.h.orig	Thu Jul 18 15:22:02 2002
+++ linux/include/asm-m68k/bitops.h	Thu Jul 18 15:22:34 2002
@@ -97,6 +97,7 @@
   (__builtin_constant_p(nr) ? \
    __constant_clear_bit(nr, vaddr) : \
    __generic_clear_bit(nr, vaddr))
+#define __clear_bit(nr,vaddr) clear_bit(nr,vaddr)
 
 extern __inline__ void __constant_clear_bit(int nr, volatile void * vaddr)
 {
@@ -239,6 +240,28 @@
 
 	return 32 - cnt;
 }
+#define __ffs(x) (ffs(x) - 1)
+
+
+/*
+ * Every architecture must define this function. It's the fastest
+ * way of searching a 140-bit bitmap where the first 100 bits are
+ * unlikely to be set. It's guaranteed that at least one of the 140
+ * bits is cleared.
+ */
+static inline int sched_find_first_bit(unsigned long *b)
+{
+	if (unlikely(b[0]))
+		return __ffs(b[0]);
+	if (unlikely(b[1]))
+		return __ffs(b[1]) + 32;
+	if (unlikely(b[2]))
+		return __ffs(b[2]) + 64;
+	if (b[3])
+		return __ffs(b[3]) + 96;
+	return __ffs(b[4]) + 128;
+}
+
 
 /*
  * hweightN: returns the hamming weight (i.e. the number
--- linux/include/asm-ppc/bitops.h.orig	Tue Aug 21 14:26:19 2001
+++ linux/include/asm-ppc/bitops.h	Thu Jul 18 15:22:34 2002
@@ -1,5 +1,5 @@
 /*
- * BK Id: SCCS/s.bitops.h 1.9 05/26/01 14:48:14 paulus
+ * BK Id: %F% %I% %G% %U% %#%
  */
 /*
  * bitops.h: Bit string operations on the ppc
@@ -10,7 +10,9 @@
 #define _PPC_BITOPS_H
 
 #include <linux/config.h>
+#include <linux/compiler.h>
 #include <asm/byteorder.h>
+#include <asm/atomic.h>
 
 /*
  * The test_and_*_bit operations are taken to imply a memory barrier
@@ -28,7 +30,7 @@
  * These used to be if'd out here because using : "cc" as a constraint
  * resulted in errors from egcs.  Things appear to be OK with gcc-2.95.
  */
-static __inline__ void set_bit(int nr, volatile void * addr)
+static __inline__ void set_bit(int nr, volatile unsigned long * addr)
 {
 	unsigned long old;
 	unsigned long mask = 1 << (nr & 0x1f);
@@ -36,8 +38,9 @@
 	
 	__asm__ __volatile__("\n\
 1:	lwarx	%0,0,%3 \n\
-	or	%0,%0,%2 \n\
-	stwcx.	%0,0,%3 \n\
+	or	%0,%0,%2 \n"
+	PPC405_ERR77(0,%3)
+"	stwcx.	%0,0,%3 \n\
 	bne-	1b"
 	: "=&r" (old), "=m" (*p)
 	: "r" (mask), "r" (p), "m" (*p)
@@ -47,7 +50,7 @@
 /*
  * non-atomic version
  */
-static __inline__ void __set_bit(int nr, volatile void *addr)
+static __inline__ void __set_bit(int nr, volatile unsigned long *addr)
 {
 	unsigned long mask = 1 << (nr & 0x1f);
 	unsigned long *p = ((unsigned long *)addr) + (nr >> 5);
@@ -61,7 +64,7 @@
 #define smp_mb__before_clear_bit()	smp_mb()
 #define smp_mb__after_clear_bit()	smp_mb()
 
-static __inline__ void clear_bit(int nr, volatile void *addr)
+static __inline__ void clear_bit(int nr, volatile unsigned long *addr)
 {
 	unsigned long old;
 	unsigned long mask = 1 << (nr & 0x1f);
@@ -69,8 +72,9 @@
 
 	__asm__ __volatile__("\n\
 1:	lwarx	%0,0,%3 \n\
-	andc	%0,%0,%2 \n\
-	stwcx.	%0,0,%3 \n\
+	andc	%0,%0,%2 \n"
+	PPC405_ERR77(0,%3)
+"	stwcx.	%0,0,%3 \n\
 	bne-	1b"
 	: "=&r" (old), "=m" (*p)
 	: "r" (mask), "r" (p), "m" (*p)
@@ -80,7 +84,7 @@
 /*
  * non-atomic version
  */
-static __inline__ void __clear_bit(int nr, volatile void *addr)
+static __inline__ void __clear_bit(int nr, volatile unsigned long *addr)
 {
 	unsigned long mask = 1 << (nr & 0x1f);
 	unsigned long *p = ((unsigned long *)addr) + (nr >> 5);
@@ -88,7 +92,7 @@
 	*p &= ~mask;
 }
 
-static __inline__ void change_bit(int nr, volatile void *addr)
+static __inline__ void change_bit(int nr, volatile unsigned long *addr)
 {
 	unsigned long old;
 	unsigned long mask = 1 << (nr & 0x1f);
@@ -96,8 +100,9 @@
 
 	__asm__ __volatile__("\n\
 1:	lwarx	%0,0,%3 \n\
-	xor	%0,%0,%2 \n\
-	stwcx.	%0,0,%3 \n\
+	xor	%0,%0,%2 \n"
+	PPC405_ERR77(0,%3)
+"	stwcx.	%0,0,%3 \n\
 	bne-	1b"
 	: "=&r" (old), "=m" (*p)
 	: "r" (mask), "r" (p), "m" (*p)
@@ -107,7 +112,7 @@
 /*
  * non-atomic version
  */
-static __inline__ void __change_bit(int nr, volatile void *addr)
+static __inline__ void __change_bit(int nr, volatile unsigned long *addr)
 {
 	unsigned long mask = 1 << (nr & 0x1f);
 	unsigned long *p = ((unsigned long *)addr) + (nr >> 5);
@@ -118,7 +123,7 @@
 /*
  * test_and_*_bit do imply a memory barrier (?)
  */
-static __inline__ int test_and_set_bit(int nr, volatile void *addr)
+static __inline__ int test_and_set_bit(int nr, volatile unsigned long *addr)
 {
 	unsigned int old, t;
 	unsigned int mask = 1 << (nr & 0x1f);
@@ -126,8 +131,9 @@
 
 	__asm__ __volatile__(SMP_WMB "\n\
 1:	lwarx	%0,0,%4 \n\
-	or	%1,%0,%3 \n\
-	stwcx.	%1,0,%4 \n\
+	or	%1,%0,%3 \n"
+	PPC405_ERR77(0,%4)
+"	stwcx.	%1,0,%4 \n\
 	bne	1b"
 	SMP_MB
 	: "=&r" (old), "=&r" (t), "=m" (*p)
@@ -140,7 +146,7 @@
 /*
  * non-atomic version
  */
-static __inline__ int __test_and_set_bit(int nr, volatile void *addr)
+static __inline__ int __test_and_set_bit(int nr, volatile unsigned long *addr)
 {
 	unsigned long mask = 1 << (nr & 0x1f);
 	unsigned long *p = ((unsigned long *)addr) + (nr >> 5);
@@ -150,7 +156,7 @@
 	return (old & mask) != 0;
 }
 
-static __inline__ int test_and_clear_bit(int nr, volatile void *addr)
+static __inline__ int test_and_clear_bit(int nr, volatile unsigned long *addr)
 {
 	unsigned int old, t;
 	unsigned int mask = 1 << (nr & 0x1f);
@@ -158,8 +164,9 @@
 
 	__asm__ __volatile__(SMP_WMB "\n\
 1:	lwarx	%0,0,%4 \n\
-	andc	%1,%0,%3 \n\
-	stwcx.	%1,0,%4 \n\
+	andc	%1,%0,%3 \n"
+	PPC405_ERR77(0,%4)
+"	stwcx.	%1,0,%4 \n\
 	bne	1b"
 	SMP_MB
 	: "=&r" (old), "=&r" (t), "=m" (*p)
@@ -172,7 +179,7 @@
 /*
  * non-atomic version
  */
-static __inline__ int __test_and_clear_bit(int nr, volatile void *addr)
+static __inline__ int __test_and_clear_bit(int nr, volatile unsigned long *addr)
 {
 	unsigned long mask = 1 << (nr & 0x1f);
 	unsigned long *p = ((unsigned long *)addr) + (nr >> 5);
@@ -182,7 +189,7 @@
 	return (old & mask) != 0;
 }
 
-static __inline__ int test_and_change_bit(int nr, volatile void *addr)
+static __inline__ int test_and_change_bit(int nr, volatile unsigned long *addr)
 {
 	unsigned int old, t;
 	unsigned int mask = 1 << (nr & 0x1f);
@@ -190,8 +197,9 @@
 
 	__asm__ __volatile__(SMP_WMB "\n\
 1:	lwarx	%0,0,%4 \n\
-	xor	%1,%0,%3 \n\
-	stwcx.	%1,0,%4 \n\
+	xor	%1,%0,%3 \n"
+	PPC405_ERR77(0,%4)
+"	stwcx.	%1,0,%4 \n\
 	bne	1b"
 	SMP_MB
 	: "=&r" (old), "=&r" (t), "=m" (*p)
@@ -204,7 +212,7 @@
 /*
  * non-atomic version
  */
-static __inline__ int __test_and_change_bit(int nr, volatile void *addr)
+static __inline__ int __test_and_change_bit(int nr, volatile unsigned long *addr)
 {
 	unsigned long mask = 1 << (nr & 0x1f);
 	unsigned long *p = ((unsigned long *)addr) + (nr >> 5);
@@ -214,7 +222,7 @@
 	return (old & mask) != 0;
 }
 
-static __inline__ int test_bit(int nr, __const__ volatile void *addr)
+static __inline__ int test_bit(int nr, __const__ volatile unsigned long *addr)
 {
 	__const__ unsigned int *p = (__const__ unsigned int *) addr;
 
@@ -222,7 +230,7 @@
 }
 
 /* Return the bit position of the most significant 1 bit in a word */
-static __inline__ int __ilog2(unsigned int x)
+static __inline__ int __ilog2(unsigned long x)
 {
 	int lz;
 
@@ -230,7 +238,7 @@
 	return 31 - lz;
 }
 
-static __inline__ int ffz(unsigned int x)
+static __inline__ int ffz(unsigned long x)
 {
 	if ((x = ~x) == 0)
 		return 32;
@@ -239,6 +247,11 @@
 
 #ifdef __KERNEL__
 
+static inline int __ffs(unsigned long x)
+{
+	return __ilog2(x & -x);
+}
+
 /*
  * ffs: find first bit set. This is defined the same way as
  * the libc and compiler builtin ffs routines, therefore
@@ -250,6 +263,18 @@
 }
 
 /*
+ * fls: find last (most-significant) bit set.
+ * Note fls(0) = 0, fls(1) = 1, fls(0x80000000) = 32.
+ */
+static __inline__ int fls(unsigned int x)
+{
+	int lz;
+
+	asm ("cntlzw %0,%1" : "=r" (lz) : "r" (x));
+	return 32 - lz;
+}
+
+/*
  * hweightN: returns the hamming weight (i.e. the number
  * of bits set) of a N-bit word
  */
@@ -261,13 +286,86 @@
 #endif /* __KERNEL__ */
 
 /*
+ * Find the first bit set in a 140-bit bitmap.
+ * The first 100 bits are unlikely to be set.
+ */
+static inline int sched_find_first_bit(unsigned long *b)
+{
+	if (unlikely(b[0]))
+		return __ffs(b[0]);
+	if (unlikely(b[1]))
+		return __ffs(b[1]) + 32;
+	if (unlikely(b[2]))
+		return __ffs(b[2]) + 64;
+	if (b[3])
+		return __ffs(b[3]) + 96;
+	return __ffs(b[4]) + 128;
+}
+
+/**
+ * find_next_bit - find the next set bit in a memory region
+ * @addr: The address to base the search on
+ * @offset: The bitnumber to start searching at
+ * @size: The maximum size to search
+ */
+static __inline__ unsigned long find_next_bit(unsigned long *addr,
+	unsigned long size, unsigned long offset)
+{
+	unsigned int *p = ((unsigned int *) addr) + (offset >> 5);
+	unsigned int result = offset & ~31UL;
+	unsigned int tmp;
+
+	if (offset >= size)
+		return size;
+	size -= result;
+	offset &= 31UL;
+	if (offset) {
+		tmp = *p++;
+		tmp &= ~0UL << offset;
+		if (size < 32)
+			goto found_first;
+		if (tmp)
+			goto found_middle;
+		size -= 32;
+		result += 32;
+	}
+	while (size >= 32) {
+		if ((tmp = *p++) != 0)
+			goto found_middle;
+		result += 32;
+		size -= 32;
+	}
+	if (!size)
+		return result;
+	tmp = *p;
+
+found_first:
+	tmp &= ~0UL >> (32 - size);
+	if (tmp == 0UL)        /* Are any bits set? */
+		return result + size; /* Nope. */
+found_middle:
+	return result + __ffs(tmp);
+}
+
+/**
+ * find_first_bit - find the first set bit in a memory region
+ * @addr: The address to start the search at
+ * @size: The maximum size to search
+ *
+ * Returns the bit-number of the first set bit, not the number of the byte
+ * containing a bit.
+ */
+#define find_first_bit(addr, size) \
+	find_next_bit((addr), (size), 0)
+
+/*
  * This implementation of find_{first,next}_zero_bit was stolen from
  * Linus' asm-alpha/bitops.h.
  */
 #define find_first_zero_bit(addr, size) \
 	find_next_zero_bit((addr), (size), 0)
 
-static __inline__ unsigned long find_next_zero_bit(void * addr,
+static __inline__ unsigned long find_next_zero_bit(unsigned long * addr,
 	unsigned long size, unsigned long offset)
 {
 	unsigned int * p = ((unsigned int *) addr) + (offset >> 5);
@@ -308,8 +406,8 @@
 
 #ifdef __KERNEL__
 
-#define ext2_set_bit(nr, addr)		__test_and_set_bit((nr) ^ 0x18, addr)
-#define ext2_clear_bit(nr, addr)	__test_and_clear_bit((nr) ^ 0x18, addr)
+#define ext2_set_bit(nr, addr)	__test_and_set_bit((nr) ^ 0x18, (unsigned long *)(addr))
+#define ext2_clear_bit(nr, addr) __test_and_clear_bit((nr) ^ 0x18, (unsigned long *)(addr))
 
 static __inline__ int ext2_test_bit(int nr, __const__ void * addr)
 {
--- linux/include/asm-sparc64/system.h.orig	Thu Jul 18 15:22:25 2002
+++ linux/include/asm-sparc64/system.h	Thu Jul 18 15:22:34 2002
@@ -143,7 +143,11 @@
 
 #define flush_user_windows flushw_user
 #define flush_register_windows flushw_all
-#define prepare_to_switch flushw_all
+
+#define prepare_arch_schedule(prev)		task_lock(prev)
+#define finish_arch_schedule(prev)		task_unlock(prev)
+#define prepare_arch_switch(rq)			do { spin_unlock(&(rq)->lock); flushw_all(); }
+#define finish_arch_switch(rq)			__sti()
 
 #ifndef CONFIG_DEBUG_SPINLOCK
 #define CHECK_LOCKS(PREV)	do { } while(0)
--- linux/include/asm-sparc64/bitops.h.orig	Thu Jul 18 15:22:11 2002
+++ linux/include/asm-sparc64/bitops.h	Thu Jul 18 15:22:34 2002
@@ -7,11 +7,12 @@
 #ifndef _SPARC64_BITOPS_H
 #define _SPARC64_BITOPS_H
 
+#include <linux/compiler.h>
 #include <asm/byteorder.h>
 
-extern long ___test_and_set_bit(unsigned long nr, volatile void *addr);
-extern long ___test_and_clear_bit(unsigned long nr, volatile void *addr);
-extern long ___test_and_change_bit(unsigned long nr, volatile void *addr);
+extern long ___test_and_set_bit(unsigned long nr, volatile unsigned long *addr);
+extern long ___test_and_clear_bit(unsigned long nr, volatile unsigned long *addr);
+extern long ___test_and_change_bit(unsigned long nr, volatile unsigned long *addr);
 
 #define test_and_set_bit(nr,addr)	({___test_and_set_bit(nr,addr)!=0;})
 #define test_and_clear_bit(nr,addr)	({___test_and_clear_bit(nr,addr)!=0;})
@@ -21,109 +22,132 @@
 #define change_bit(nr,addr)		((void)___test_and_change_bit(nr,addr))
 
 /* "non-atomic" versions... */
-#define __set_bit(X,Y)					\
-do {	unsigned long __nr = (X);			\
-	long *__m = ((long *) (Y)) + (__nr >> 6);	\
-	*__m |= (1UL << (__nr & 63));			\
-} while (0)
-#define __clear_bit(X,Y)				\
-do {	unsigned long __nr = (X);			\
-	long *__m = ((long *) (Y)) + (__nr >> 6);	\
-	*__m &= ~(1UL << (__nr & 63));			\
-} while (0)
-#define __change_bit(X,Y)				\
-do {	unsigned long __nr = (X);			\
-	long *__m = ((long *) (Y)) + (__nr >> 6);	\
-	*__m ^= (1UL << (__nr & 63));			\
-} while (0)
-#define __test_and_set_bit(X,Y)				\
-({	unsigned long __nr = (X);			\
-	long *__m = ((long *) (Y)) + (__nr >> 6);	\
-	long __old = *__m;				\
-	long __mask = (1UL << (__nr & 63));		\
-	*__m = (__old | __mask);			\
-	((__old & __mask) != 0);			\
-})
-#define __test_and_clear_bit(X,Y)			\
-({	unsigned long __nr = (X);			\
-	long *__m = ((long *) (Y)) + (__nr >> 6);	\
-	long __old = *__m;				\
-	long __mask = (1UL << (__nr & 63));		\
-	*__m = (__old & ~__mask);			\
-	((__old & __mask) != 0);			\
-})
-#define __test_and_change_bit(X,Y)			\
-({	unsigned long __nr = (X);			\
-	long *__m = ((long *) (Y)) + (__nr >> 6);	\
-	long __old = *__m;				\
-	long __mask = (1UL << (__nr & 63));		\
-	*__m = (__old ^ __mask);			\
-	((__old & __mask) != 0);			\
-})
+
+static __inline__ void __set_bit(int nr, volatile unsigned long *addr)
+{
+	volatile unsigned long *m = addr + (nr >> 6);
+
+	*m |= (1UL << (nr & 63));
+}
+
+static __inline__ void __clear_bit(int nr, volatile unsigned long *addr)
+{
+	volatile unsigned long *m = addr + (nr >> 6);
+
+	*m &= ~(1UL << (nr & 63));
+}
+
+static __inline__ void __change_bit(int nr, volatile unsigned long *addr)
+{
+	volatile unsigned long *m = addr + (nr >> 6);
+
+	*m ^= (1UL << (nr & 63));
+}
+
+static __inline__ int __test_and_set_bit(int nr, volatile unsigned long *addr)
+{
+	volatile unsigned long *m = addr + (nr >> 6);
+	long old = *m;
+	long mask = (1UL << (nr & 63));
+
+	*m = (old | mask);
+	return ((old & mask) != 0);
+}
+
+static __inline__ int __test_and_clear_bit(int nr, volatile unsigned long *addr)
+{
+	volatile unsigned long *m = addr + (nr >> 6);
+	long old = *m;
+	long mask = (1UL << (nr & 63));
+
+	*m = (old & ~mask);
+	return ((old & mask) != 0);
+}
+
+static __inline__ int __test_and_change_bit(int nr, volatile unsigned long *addr)
+{
+	volatile unsigned long *m = addr + (nr >> 6);
+	long old = *m;
+	long mask = (1UL << (nr & 63));
+
+	*m = (old ^ mask);
+	return ((old & mask) != 0);
+}
 
 #define smp_mb__before_clear_bit()	do { } while(0)
 #define smp_mb__after_clear_bit()	do { } while(0)
 
-extern __inline__ int test_bit(int nr, __const__ void *addr)
+static __inline__ int test_bit(int nr, __const__ volatile unsigned long *addr)
 {
-	return (1UL & (((__const__ long *) addr)[nr >> 6] >> (nr & 63))) != 0UL;
+	return (1UL & ((addr)[nr >> 6] >> (nr & 63))) != 0UL;
 }
 
 /* The easy/cheese version for now. */
-extern __inline__ unsigned long ffz(unsigned long word)
+static __inline__ unsigned long ffz(unsigned long word)
 {
 	unsigned long result;
 
-#ifdef ULTRA_HAS_POPULATION_COUNT	/* Thanks for nothing Sun... */
-	__asm__ __volatile__(
-"	brz,pn	%0, 1f\n"
-"	 neg	%0, %%g1\n"
-"	xnor	%0, %%g1, %%g2\n"
-"	popc	%%g2, %0\n"
-"1:	" : "=&r" (result)
-	  : "0" (word)
-	  : "g1", "g2");
-#else
-#if 1 /* def EASY_CHEESE_VERSION */
 	result = 0;
 	while(word & 1) {
 		result++;
 		word >>= 1;
 	}
-#else
-	unsigned long tmp;
+	return result;
+}
 
-	result = 0;	
-	tmp = ~word & -~word;
-	if (!(unsigned)tmp) {
-		tmp >>= 32;
-		result = 32;
-	}
-	if (!(unsigned short)tmp) {
-		tmp >>= 16;
-		result += 16;
-	}
-	if (!(unsigned char)tmp) {
-		tmp >>= 8;
-		result += 8;
+/**
+ * __ffs - find first bit in word.
+ * @word: The word to search
+ *
+ * Undefined if no bit exists, so code should check against 0 first.
+ */
+static __inline__ unsigned long __ffs(unsigned long word)
+{
+	unsigned long result = 0;
+
+	while (!(word & 1UL)) {
+		result++;
+		word >>= 1;
 	}
-	if (tmp & 0xf0) result += 4;
-	if (tmp & 0xcc) result += 2;
-	if (tmp & 0xaa) result ++;
-#endif
-#endif
 	return result;
 }
 
+/*
+ * fls: find last bit set.
+ */
+
+#define fls(x) generic_fls(x)
+
 #ifdef __KERNEL__
 
 /*
+ * Every architecture must define this function. It's the fastest
+ * way of searching a 140-bit bitmap where the first 100 bits are
+ * unlikely to be set. It's guaranteed that at least one of the 140
+ * bits is cleared.
+ */
+static inline int sched_find_first_bit(unsigned long *b)
+{
+	if (unlikely(b[0]))
+		return __ffs(b[0]);
+	if (unlikely(((unsigned int)b[1])))
+		return __ffs(b[1]) + 64;
+	if (b[1] >> 32)
+		return __ffs(b[1] >> 32) + 96;
+	return __ffs(b[2]) + 128;
+}
+
+/*
  * ffs: find first bit set. This is defined the same way as
  * the libc and compiler builtin ffs routines, therefore
  * differs in spirit from the above ffz (man ffs).
  */
-
-#define ffs(x) generic_ffs(x)
+static __inline__ int ffs(int x)
+{
+	if (!x)
+		return 0;
+	return __ffs((unsigned long)x);
+}
 
 /*
  * hweightN: returns the hamming weight (i.e. the number
@@ -132,7 +156,7 @@
 
 #ifdef ULTRA_HAS_POPULATION_COUNT
 
-extern __inline__ unsigned int hweight32(unsigned int w)
+static __inline__ unsigned int hweight32(unsigned int w)
 {
 	unsigned int res;
 
@@ -140,7 +164,7 @@
 	return res;
 }
 
-extern __inline__ unsigned int hweight16(unsigned int w)
+static __inline__ unsigned int hweight16(unsigned int w)
 {
 	unsigned int res;
 
@@ -148,7 +172,7 @@
 	return res;
 }
 
-extern __inline__ unsigned int hweight8(unsigned int w)
+static __inline__ unsigned int hweight8(unsigned int w)
 {
 	unsigned int res;
 
@@ -165,14 +189,69 @@
 #endif
 #endif /* __KERNEL__ */
 
+/**
+ * find_next_bit - find the next set bit in a memory region
+ * @addr: The address to base the search on
+ * @offset: The bitnumber to start searching at
+ * @size: The maximum size to search
+ */
+static __inline__ unsigned long find_next_bit(unsigned long *addr, unsigned long size, unsigned long offset)
+{
+	unsigned long *p = addr + (offset >> 6);
+	unsigned long result = offset & ~63UL;
+	unsigned long tmp;
+
+	if (offset >= size)
+		return size;
+	size -= result;
+	offset &= 63UL;
+	if (offset) {
+		tmp = *(p++);
+		tmp &= (~0UL << offset);
+		if (size < 64)
+			goto found_first;
+		if (tmp)
+			goto found_middle;
+		size -= 64;
+		result += 64;
+	}
+	while (size & ~63UL) {
+		if ((tmp = *(p++)))
+			goto found_middle;
+		result += 64;
+		size -= 64;
+	}
+	if (!size)
+		return result;
+	tmp = *p;
+
+found_first:
+	tmp &= (~0UL >> (64 - size));
+	if (tmp == 0UL)        /* Are any bits set? */
+		return result + size; /* Nope. */
+found_middle:
+	return result + __ffs(tmp);
+}
+
+/**
+ * find_first_bit - find the first set bit in a memory region
+ * @addr: The address to start the search at
+ * @size: The maximum size to search
+ *
+ * Returns the bit-number of the first set bit, not the number of the byte
+ * containing a bit.
+ */
+#define find_first_bit(addr, size) \
+	find_next_bit((addr), (size), 0)
+
 /* find_next_zero_bit() finds the first zero bit in a bit string of length
  * 'size' bits, starting the search at bit 'offset'. This is largely based
  * on Linus's ALPHA routines, which are pretty portable BTW.
  */
 
-extern __inline__ unsigned long find_next_zero_bit(void *addr, unsigned long size, unsigned long offset)
+static __inline__ unsigned long find_next_zero_bit(unsigned long *addr, unsigned long size, unsigned long offset)
 {
-	unsigned long *p = ((unsigned long *) addr) + (offset >> 6);
+	unsigned long *p = addr + (offset >> 6);
 	unsigned long result = offset & ~63UL;
 	unsigned long tmp;
 
@@ -211,15 +290,15 @@
 #define find_first_zero_bit(addr, size) \
         find_next_zero_bit((addr), (size), 0)
 
-extern long ___test_and_set_le_bit(int nr, volatile void *addr);
-extern long ___test_and_clear_le_bit(int nr, volatile void *addr);
+extern long ___test_and_set_le_bit(int nr, volatile unsigned long *addr);
+extern long ___test_and_clear_le_bit(int nr, volatile unsigned long *addr);
 
 #define test_and_set_le_bit(nr,addr)	({___test_and_set_le_bit(nr,addr)!=0;})
 #define test_and_clear_le_bit(nr,addr)	({___test_and_clear_le_bit(nr,addr)!=0;})
 #define set_le_bit(nr,addr)		((void)___test_and_set_le_bit(nr,addr))
 #define clear_le_bit(nr,addr)		((void)___test_and_clear_le_bit(nr,addr))
 
-extern __inline__ int test_le_bit(int nr, __const__ void * addr)
+static __inline__ int test_le_bit(int nr, __const__ unsigned long * addr)
 {
 	int			mask;
 	__const__ unsigned char	*ADDR = (__const__ unsigned char *) addr;
@@ -232,9 +311,9 @@
 #define find_first_zero_le_bit(addr, size) \
         find_next_zero_le_bit((addr), (size), 0)
 
-extern __inline__ unsigned long find_next_zero_le_bit(void *addr, unsigned long size, unsigned long offset)
+static __inline__ unsigned long find_next_zero_le_bit(unsigned long *addr, unsigned long size, unsigned long offset)
 {
-	unsigned long *p = ((unsigned long *) addr) + (offset >> 6);
+	unsigned long *p = addr + (offset >> 6);
 	unsigned long result = offset & ~63UL;
 	unsigned long tmp;
 
@@ -271,18 +350,22 @@
 
 #ifdef __KERNEL__
 
-#define ext2_set_bit			test_and_set_le_bit
-#define ext2_clear_bit			test_and_clear_le_bit
-#define ext2_test_bit  			test_le_bit
-#define ext2_find_first_zero_bit	find_first_zero_le_bit
-#define ext2_find_next_zero_bit		find_next_zero_le_bit
+#define ext2_set_bit(nr,addr)		test_and_set_le_bit((nr),(unsigned long *)(addr))
+#define ext2_clear_bit(nr,addr)		test_and_clear_le_bit((nr),(unsigned long *)(addr))
+#define ext2_test_bit(nr,addr)		test_le_bit((nr),(unsigned long *)(addr))
+#define ext2_find_first_zero_bit(addr, size) \
+	find_first_zero_le_bit((unsigned long *)(addr), (size))
+#define ext2_find_next_zero_bit(addr, size, off) \
+	find_next_zero_le_bit((unsigned long *)(addr), (size), (off))
 
 /* Bitmap functions for the minix filesystem.  */
-#define minix_test_and_set_bit(nr,addr) test_and_set_bit(nr,addr)
-#define minix_set_bit(nr,addr) set_bit(nr,addr)
-#define minix_test_and_clear_bit(nr,addr) test_and_clear_bit(nr,addr)
-#define minix_test_bit(nr,addr) test_bit(nr,addr)
-#define minix_find_first_zero_bit(addr,size) find_first_zero_bit(addr,size)
+#define minix_test_and_set_bit(nr,addr)	test_and_set_bit((nr),(unsigned long *)(addr))
+#define minix_set_bit(nr,addr)		set_bit((nr),(unsigned long *)(addr))
+#define minix_test_and_clear_bit(nr,addr) \
+	test_and_clear_bit((nr),(unsigned long *)(addr))
+#define minix_test_bit(nr,addr)		test_bit((nr),(unsigned long *)(addr))
+#define minix_find_first_zero_bit(addr,size) \
+	find_first_zero_bit((unsigned long *)(addr),(size))
 
 #endif /* __KERNEL__ */
 
--- linux/include/asm-arm/bitops.h.orig	Thu Jul 18 15:21:50 2002
+++ linux/include/asm-arm/bitops.h	Thu Jul 18 15:22:34 2002
@@ -2,6 +2,8 @@
  * Copyright 1995, Russell King.
  * Various bits and pieces copyrights include:
  *  Linus Torvalds (test_bit).
+ * Big endian support: Copyright 2001, Nicolas Pitre
+ *  reworked by rmk.
  *
  * bit 0 is the LSB of addr; bit 32 is the LSB of (addr+1).
  *
@@ -17,81 +19,271 @@
 
 #ifdef __KERNEL__
 
+#include <asm/system.h>
+
 #define smp_mb__before_clear_bit()	do { } while (0)
 #define smp_mb__after_clear_bit()	do { } while (0)
 
 /*
- * Function prototypes to keep gcc -Wall happy.
+ * These functions are the basis of our bit ops.
+ * First, the atomic bitops.
+ *
+ * The endian issue for these functions is handled by the macros below.
  */
-extern void set_bit(int nr, volatile void * addr);
+static inline void
+____atomic_set_bit_mask(unsigned int mask, volatile unsigned char *p)
+{
+	unsigned long flags;
+
+	local_irq_save(flags);
+	*p |= mask;
+	local_irq_restore(flags);
+}
+
+static inline void
+____atomic_clear_bit_mask(unsigned int mask, volatile unsigned char *p)
+{
+	unsigned long flags;
+
+	local_irq_save(flags);
+	*p &= ~mask;
+	local_irq_restore(flags);
+}
+
+static inline void
+____atomic_change_bit_mask(unsigned int mask, volatile unsigned char *p)
+{
+	unsigned long flags;
+
+	local_irq_save(flags);
+	*p ^= mask;
+	local_irq_restore(flags);
+}
 
-static inline void __set_bit(int nr, volatile void *addr)
+static inline int
+____atomic_test_and_set_bit_mask(unsigned int mask, volatile unsigned char *p)
 {
-	((unsigned char *) addr)[nr >> 3] |= (1U << (nr & 7));
+	unsigned long flags;
+	unsigned int res;
+
+	local_irq_save(flags);
+	res = *p;
+	*p = res | mask;
+	local_irq_restore(flags);
+
+	return res & mask;
 }
 
-extern void clear_bit(int nr, volatile void * addr);
+static inline int
+____atomic_test_and_clear_bit_mask(unsigned int mask, volatile unsigned char *p)
+{
+	unsigned long flags;
+	unsigned int res;
+
+	local_irq_save(flags);
+	res = *p;
+	*p = res & ~mask;
+	local_irq_restore(flags);
+
+	return res & mask;
+}
 
-static inline void __clear_bit(int nr, volatile void *addr)
+static inline int
+____atomic_test_and_change_bit_mask(unsigned int mask, volatile unsigned char *p)
 {
-	((unsigned char *) addr)[nr >> 3] &= ~(1U << (nr & 7));
+	unsigned long flags;
+	unsigned int res;
+
+	local_irq_save(flags);
+	res = *p;
+	*p = res ^ mask;
+	local_irq_restore(flags);
+
+	return res & mask;
 }
 
-extern void change_bit(int nr, volatile void * addr);
+/*
+ * Now the non-atomic variants.  We let the compiler handle all optimisations
+ * for these.
+ */
+static inline void ____nonatomic_set_bit(int nr, volatile void *p)
+{
+	((unsigned char *) p)[nr >> 3] |= (1U << (nr & 7));
+}
 
-static inline void __change_bit(int nr, volatile void *addr)
+static inline void ____nonatomic_clear_bit(int nr, volatile void *p)
 {
-	((unsigned char *) addr)[nr >> 3] ^= (1U << (nr & 7));
+	((unsigned char *) p)[nr >> 3] &= ~(1U << (nr & 7));
 }
 
-extern int test_and_set_bit(int nr, volatile void * addr);
+static inline void ____nonatomic_change_bit(int nr, volatile void *p)
+{
+	((unsigned char *) p)[nr >> 3] ^= (1U << (nr & 7));
+}
 
-static inline int __test_and_set_bit(int nr, volatile void *addr)
+static inline int ____nonatomic_test_and_set_bit(int nr, volatile void *p)
 {
 	unsigned int mask = 1 << (nr & 7);
 	unsigned int oldval;
 
-	oldval = ((unsigned char *) addr)[nr >> 3];
-	((unsigned char *) addr)[nr >> 3] = oldval | mask;
+	oldval = ((unsigned char *) p)[nr >> 3];
+	((unsigned char *) p)[nr >> 3] = oldval | mask;
 	return oldval & mask;
 }
 
-extern int test_and_clear_bit(int nr, volatile void * addr);
-
-static inline int __test_and_clear_bit(int nr, volatile void *addr)
+static inline int ____nonatomic_test_and_clear_bit(int nr, volatile void *p)
 {
 	unsigned int mask = 1 << (nr & 7);
 	unsigned int oldval;
 
-	oldval = ((unsigned char *) addr)[nr >> 3];
-	((unsigned char *) addr)[nr >> 3] = oldval & ~mask;
+	oldval = ((unsigned char *) p)[nr >> 3];
+	((unsigned char *) p)[nr >> 3] = oldval & ~mask;
 	return oldval & mask;
 }
 
-extern int test_and_change_bit(int nr, volatile void * addr);
-
-static inline int __test_and_change_bit(int nr, volatile void *addr)
+static inline int ____nonatomic_test_and_change_bit(int nr, volatile void *p)
 {
 	unsigned int mask = 1 << (nr & 7);
 	unsigned int oldval;
 
-	oldval = ((unsigned char *) addr)[nr >> 3];
-	((unsigned char *) addr)[nr >> 3] = oldval ^ mask;
+	oldval = ((unsigned char *) p)[nr >> 3];
+	((unsigned char *) p)[nr >> 3] = oldval ^ mask;
 	return oldval & mask;
 }
 
-extern int find_first_zero_bit(void * addr, unsigned size);
-extern int find_next_zero_bit(void * addr, int size, int offset);
-
 /*
  * This routine doesn't need to be atomic.
  */
-static inline int test_bit(int nr, const void * addr)
+static inline int ____test_bit(int nr, const void * p)
 {
-    return ((unsigned char *) addr)[nr >> 3] & (1U << (nr & 7));
+    return ((volatile unsigned char *) p)[nr >> 3] & (1U << (nr & 7));
 }	
 
 /*
+ *  A note about Endian-ness.
+ *  -------------------------
+ *
+ * When the ARM is put into big endian mode via CR15, the processor
+ * merely swaps the order of bytes within words, thus:
+ *
+ *          ------------ physical data bus bits -----------
+ *          D31 ... D24  D23 ... D16  D15 ... D8  D7 ... D0
+ * little     byte 3       byte 2       byte 1      byte 0
+ * big        byte 0       byte 1       byte 2      byte 3
+ *
+ * This means that reading a 32-bit word at address 0 returns the same
+ * value irrespective of the endian mode bit.
+ *
+ * Peripheral devices should be connected with the data bus reversed in
+ * "Big Endian" mode.  ARM Application Note 61 is applicable, and is
+ * available from http://www.arm.com/.
+ *
+ * The following assumes that the data bus connectivity for big endian
+ * mode has been followed.
+ *
+ * Note that bit 0 is defined to be 32-bit word bit 0, not byte 0 bit 0.
+ */
+
+/*
+ * Little endian assembly bitops.  nr = 0 -> byte 0 bit 0.
+ */
+extern void _set_bit_le(int nr, volatile void * p);
+extern void _clear_bit_le(int nr, volatile void * p);
+extern void _change_bit_le(int nr, volatile void * p);
+extern int _test_and_set_bit_le(int nr, volatile void * p);
+extern int _test_and_clear_bit_le(int nr, volatile void * p);
+extern int _test_and_change_bit_le(int nr, volatile void * p);
+extern int _find_first_zero_bit_le(void * p, unsigned size);
+extern int _find_next_zero_bit_le(void * p, int size, int offset);
+
+/*
+ * Big endian assembly bitops.  nr = 0 -> byte 3 bit 0.
+ */
+extern void _set_bit_be(int nr, volatile void * p);
+extern void _clear_bit_be(int nr, volatile void * p);
+extern void _change_bit_be(int nr, volatile void * p);
+extern int _test_and_set_bit_be(int nr, volatile void * p);
+extern int _test_and_clear_bit_be(int nr, volatile void * p);
+extern int _test_and_change_bit_be(int nr, volatile void * p);
+extern int _find_first_zero_bit_be(void * p, unsigned size);
+extern int _find_next_zero_bit_be(void * p, int size, int offset);
+
+
+/*
+ * The __* form of bitops are non-atomic and may be reordered.
+ */
+#define	ATOMIC_BITOP_LE(name,nr,p)		\
+	(__builtin_constant_p(nr) ?		\
+	 ____atomic_##name##_mask(1 << ((nr) & 7), \
+			((unsigned char *)(p)) + ((nr) >> 3)) : \
+	 _##name##_le(nr,p))
+
+#define	ATOMIC_BITOP_BE(name,nr,p)		\
+	(__builtin_constant_p(nr) ?		\
+	 ____atomic_##name##_mask(1 << ((nr) & 7), \
+			((unsigned char *)(p)) + (((nr) >> 3) ^ 3)) : \
+	 _##name##_be(nr,p))
+
+#define NONATOMIC_BITOP_LE(name,nr,p)	\
+	(____nonatomic_##name(nr, p))
+
+#define NONATOMIC_BITOP_BE(name,nr,p)	\
+	(____nonatomic_##name(nr ^ 0x18, p))
+
+#ifndef __ARMEB__
+/*
+ * These are the little endian, atomic definitions.
+ */
+#define set_bit(nr,p)			ATOMIC_BITOP_LE(set_bit,nr,p)
+#define clear_bit(nr,p)			ATOMIC_BITOP_LE(clear_bit,nr,p)
+#define change_bit(nr,p)		ATOMIC_BITOP_LE(change_bit,nr,p)
+#define test_and_set_bit(nr,p)		ATOMIC_BITOP_LE(test_and_set_bit,nr,p)
+#define test_and_clear_bit(nr,p)	ATOMIC_BITOP_LE(test_and_clear_bit,nr,p)
+#define test_and_change_bit(nr,p)	ATOMIC_BITOP_LE(test_and_change_bit,nr,p)
+#define test_bit(nr,p)			____test_bit(nr,p)
+#define find_first_zero_bit(p,sz)	_find_first_zero_bit_le(p,sz)
+#define find_next_zero_bit(p,sz,off)	_find_next_zero_bit_le(p,sz,off)
+
+/*
+ * These are the little endian, non-atomic definitions.
+ */
+#define __set_bit(nr,p)			NONATOMIC_BITOP_LE(set_bit,nr,p)
+#define __clear_bit(nr,p)		NONATOMIC_BITOP_LE(clear_bit,nr,p)
+#define __change_bit(nr,p)		NONATOMIC_BITOP_LE(change_bit,nr,p)
+#define __test_and_set_bit(nr,p)	NONATOMIC_BITOP_LE(test_and_set_bit,nr,p)
+#define __test_and_clear_bit(nr,p)	NONATOMIC_BITOP_LE(test_and_clear_bit,nr,p)
+#define __test_and_change_bit(nr,p)	NONATOMIC_BITOP_LE(test_and_change_bit,nr,p)
+#define __test_bit(nr,p)		____test_bit(nr,p)
+
+#else
+
+/*
+ * These are the big endian, atomic definitions.
+ */
+#define set_bit(nr,p)			ATOMIC_BITOP_BE(set_bit,nr,p)
+#define clear_bit(nr,p)			ATOMIC_BITOP_BE(clear_bit,nr,p)
+#define change_bit(nr,p)		ATOMIC_BITOP_BE(change_bit,nr,p)
+#define test_and_set_bit(nr,p)		ATOMIC_BITOP_BE(test_and_set_bit,nr,p)
+#define test_and_clear_bit(nr,p)	ATOMIC_BITOP_BE(test_and_clear_bit,nr,p)
+#define test_and_change_bit(nr,p)	ATOMIC_BITOP_BE(test_and_change_bit,nr,p)
+#define test_bit(nr,p)			____test_bit((nr) ^ 0x18, p)
+#define find_first_zero_bit(p,sz)	_find_first_zero_bit_be(p,sz)
+#define find_next_zero_bit(p,sz,off)	_find_next_zero_bit_be(p,sz,off)
+
+/*
+ * These are the big endian, non-atomic definitions.
+ */
+#define __set_bit(nr,p)			NONATOMIC_BITOP_BE(set_bit,nr,p)
+#define __clear_bit(nr,p)		NONATOMIC_BITOP_BE(clear_bit,nr,p)
+#define __change_bit(nr,p)		NONATOMIC_BITOP_BE(change_bit,nr,p)
+#define __test_and_set_bit(nr,p)	NONATOMIC_BITOP_BE(test_and_set_bit,nr,p)
+#define __test_and_clear_bit(nr,p)	NONATOMIC_BITOP_BE(test_and_clear_bit,nr,p)
+#define __test_and_change_bit(nr,p)	NONATOMIC_BITOP_BE(test_and_change_bit,nr,p)
+#define __test_bit(nr,p)		____test_bit((nr) ^ 0x18, p)
+
+#endif
+
+/*
  * ffz = Find First Zero in word. Undefined if no zero exists,
  * so code should check against ~0UL first..
  */
@@ -110,6 +302,29 @@
 }
 
 /*
+ * ffz = Find First Zero in word. Undefined if no zero exists,
+ * so code should check against ~0UL first..
+ */
+static inline unsigned long __ffs(unsigned long word)
+{
+	int k;
+
+	k = 31;
+	if (word & 0x0000ffff) { k -= 16; word <<= 16; }
+	if (word & 0x00ff0000) { k -= 8;  word <<= 8;  }
+	if (word & 0x0f000000) { k -= 4;  word <<= 4;  }
+	if (word & 0x30000000) { k -= 2;  word <<= 2;  }
+	if (word & 0x40000000) { k -= 1; }
+        return k;
+}
+
+/*
+ * fls: find last bit set.
+ */
+
+#define fls(x) generic_fls(x)
+
+/*
  * ffs: find first bit set. This is defined the same way as
  * the libc and compiler builtin ffs routines, therefore
  * differs in spirit from the above ffz (man ffs).
@@ -118,6 +333,22 @@
 #define ffs(x) generic_ffs(x)
 
 /*
+ * Find first bit set in a 168-bit bitmap, where the first
+ * 128 bits are unlikely to be set.
+ */
+static inline int sched_find_first_bit(unsigned long *b)
+{
+	unsigned long v;
+	unsigned int off;
+
+	for (off = 0; v = b[off], off < 4; off++) {
+		if (unlikely(v))
+			break;
+	}
+	return __ffs(v) + off * 32;
+}
+
+/*
  * hweightN: returns the hamming weight (i.e. the number
  * of bits set) of a N-bit word
  */
@@ -126,18 +357,25 @@
 #define hweight16(x) generic_hweight16(x)
 #define hweight8(x) generic_hweight8(x)
 
-#define ext2_set_bit			test_and_set_bit
-#define ext2_clear_bit			test_and_clear_bit
-#define ext2_test_bit			test_bit
-#define ext2_find_first_zero_bit	find_first_zero_bit
-#define ext2_find_next_zero_bit		find_next_zero_bit
-
-/* Bitmap functions for the minix filesystem. */
-#define minix_test_and_set_bit(nr,addr)	test_and_set_bit(nr,addr)
-#define minix_set_bit(nr,addr)		set_bit(nr,addr)
-#define minix_test_and_clear_bit(nr,addr)	test_and_clear_bit(nr,addr)
-#define minix_test_bit(nr,addr)		test_bit(nr,addr)
-#define minix_find_first_zero_bit(addr,size)	find_first_zero_bit(addr,size)
+/*
+ * Ext2 is defined to use little-endian byte ordering.
+ * These do not need to be atomic.
+ */
+#define ext2_set_bit(nr,p)			NONATOMIC_BITOP_LE(test_and_set_bit,nr,p)
+#define ext2_clear_bit(nr,p)			NONATOMIC_BITOP_LE(test_and_clear_bit,nr,p)
+#define ext2_test_bit(nr,p)			__test_bit(nr,p)
+#define ext2_find_first_zero_bit(p,sz)		_find_first_zero_bit_le(p,sz)
+#define ext2_find_next_zero_bit(p,sz,off)	_find_next_zero_bit_le(p,sz,off)
+
+/*
+ * Minix is defined to use little-endian byte ordering.
+ * These do not need to be atomic.
+ */
+#define minix_set_bit(nr,p)			NONATOMIC_BITOP_LE(set_bit,nr,p)
+#define minix_test_bit(nr,p)			__test_bit(nr,p)
+#define minix_test_and_set_bit(nr,p)		NONATOMIC_BITOP_LE(test_and_set_bit,nr,p)
+#define minix_test_and_clear_bit(nr,p)		NONATOMIC_BITOP_LE(test_and_clear_bit,nr,p)
+#define minix_find_first_zero_bit(p,sz)		_find_first_zero_bit_le(p,sz)
 
 #endif /* __KERNEL__ */
 
--- linux/include/asm-ia64/bitops.h.orig	Thu Jul 18 15:22:24 2002
+++ linux/include/asm-ia64/bitops.h	Thu Jul 18 15:22:34 2002
@@ -2,10 +2,15 @@
 #define _ASM_IA64_BITOPS_H
 
 /*
- * Copyright (C) 1998-2001 Hewlett-Packard Co
- * Copyright (C) 1998-2001 David Mosberger-Tang <davidm@hpl.hp.com>
+ * Copyright (C) 1998-2002 Hewlett-Packard Co
+ *	David Mosberger-Tang <davidm@hpl.hp.com>
+ *
+ * 02/06/02 find_next_bit() and find_first_bit() added from Erich Focht's ia64 O(1)
+ *	    scheduler patch
  */
 
+#include <linux/types.h>
+
 #include <asm/system.h>
 
 /**
@@ -89,6 +94,17 @@
 }
 
 /**
+ * __clear_bit - Clears a bit in memory (non-atomic version)
+ */
+static __inline__ void
+__clear_bit (int nr, volatile void *addr)
+{
+	volatile __u32 *p = (__u32 *) addr + (nr >> 5);
+	__u32 m = 1 << (nr & 31);
+	*p &= ~m;
+}
+
+/**
  * change_bit - Toggle a bit in memory
  * @nr: Bit to clear
  * @addr: Address to start counting from
@@ -264,12 +280,11 @@
 }
 
 /**
- * ffz - find the first zero bit in a memory region
- * @x: The address to start the search at
+ * ffz - find the first zero bit in a long word
+ * @x: The long word to find the bit in
  *
- * Returns the bit-number (0..63) of the first (least significant) zero bit, not
- * the number of the byte containing a bit.  Undefined if no zero exists, so
- * code should check against ~0UL first...
+ * Returns the bit-number (0..63) of the first (least significant) zero bit.  Undefined if
+ * no zero exists, so code should check against ~0UL first...
  */
 static inline unsigned long
 ffz (unsigned long x)
@@ -280,6 +295,21 @@
 	return result;
 }
 
+/**
+ * __ffs - find first bit in word.
+ * @x: The word to search
+ *
+ * Undefined if no bit exists, so code should check against 0 first.
+ */
+static __inline__ unsigned long
+__ffs (unsigned long x)
+{
+	unsigned long result;
+
+	__asm__ ("popcnt %0=%1" : "=r" (result) : "r" ((x - 1) & ~x));
+	return result;
+}
+
 #ifdef __KERNEL__
 
 /*
@@ -296,6 +326,12 @@
 	return exp - 0xffff;
 }
 
+static int
+fls (int x)
+{
+	return ia64_fls((unsigned int) x);
+}
+
 /*
  * ffs: find first bit set. This is defined the same way as the libc and compiler builtin
  * ffs routines, therefore differs in spirit from the above ffz (man ffs): it operates on
@@ -368,8 +404,53 @@
  */
 #define find_first_zero_bit(addr, size) find_next_zero_bit((addr), (size), 0)
 
+/*
+ * Find next bit in a bitmap reasonably efficiently..
+ */
+static inline int
+find_next_bit (void *addr, unsigned long size, unsigned long offset)
+{
+	unsigned long *p = ((unsigned long *) addr) + (offset >> 6);
+	unsigned long result = offset & ~63UL;
+	unsigned long tmp;
+
+	if (offset >= size)
+		return size;
+	size -= result;
+	offset &= 63UL;
+	if (offset) {
+		tmp = *(p++);
+		tmp &= ~0UL << offset;
+		if (size < 64)
+			goto found_first;
+		if (tmp)
+			goto found_middle;
+		size -= 64;
+		result += 64;
+	}
+	while (size & ~63UL) {
+		if ((tmp = *(p++)))
+			goto found_middle;
+		result += 64;
+		size -= 64;
+	}
+	if (!size)
+		return result;
+	tmp = *p;
+  found_first:
+	tmp &= ~0UL >> (64-size);
+	if (tmp == 0UL)		/* Are any bits set? */
+		return result + size; /* Nope. */
+  found_middle:
+	return result + __ffs(tmp);
+}
+
+#define find_first_bit(addr, size) find_next_bit((addr), (size), 0)
+
 #ifdef __KERNEL__
 
+#define __clear_bit(nr, addr)        clear_bit(nr, addr)
+
 #define ext2_set_bit                 test_and_set_bit
 #define ext2_clear_bit               test_and_clear_bit
 #define ext2_test_bit                test_bit
@@ -382,6 +463,16 @@
 #define minix_test_and_clear_bit(nr,addr)	test_and_clear_bit(nr,addr)
 #define minix_test_bit(nr,addr)			test_bit(nr,addr)
 #define minix_find_first_zero_bit(addr,size)	find_first_zero_bit(addr,size)
+
+static inline int
+sched_find_first_bit (unsigned long *b)
+{
+	if (unlikely(b[0]))
+		return __ffs(b[0]);
+	if (unlikely(b[1]))
+		return 64 + __ffs(b[1]);
+	return __ffs(b[2]) + 128;
+}
 
 #endif /* __KERNEL__ */
 
--- linux/include/asm-mips64/bitops.h.orig	Thu Jul 18 15:22:24 2002
+++ linux/include/asm-mips64/bitops.h	Thu Jul 18 15:22:34 2002
@@ -19,6 +19,7 @@
 
 #include <asm/system.h>
 #include <asm/sgidefs.h>
+#include <asm/mipsregs.h>
 
 /*
  * set_bit - Atomically set a bit in memory
@@ -30,7 +31,8 @@
  * Note that @nr may be almost arbitrarily large; this function is not
  * restricted to acting on a single-word quantity.
  */
-static inline void set_bit(unsigned long nr, volatile void *addr)
+extern __inline__ void
+set_bit(unsigned long nr, volatile void *addr)
 {
 	unsigned long *m = ((unsigned long *) addr) + (nr >> 6);
 	unsigned long temp;
@@ -54,7 +56,7 @@
  * If it's called on the same region of memory simultaneously, the effect
  * may be that only one operation succeeds.
  */
-static inline void __set_bit(int nr, volatile void * addr)
+extern __inline__ void __set_bit(int nr, volatile void * addr)
 {
 	unsigned long * m = ((unsigned long *) addr) + (nr >> 6);
 
@@ -71,7 +73,8 @@
  * you should call smp_mb__before_clear_bit() and/or smp_mb__after_clear_bit()
  * in order to ensure changes are visible on other processors.
  */
-static inline void clear_bit(unsigned long nr, volatile void *addr)
+extern __inline__ void
+clear_bit(unsigned long nr, volatile void *addr)
 {
 	unsigned long *m = ((unsigned long *) addr) + (nr >> 6);
 	unsigned long temp;
@@ -97,7 +100,8 @@
  * Note that @nr may be almost arbitrarily large; this function is not
  * restricted to acting on a single-word quantity.
  */
-static inline void change_bit(unsigned long nr, volatile void *addr)
+extern __inline__ void
+change_bit(unsigned long nr, volatile void *addr)
 {
 	unsigned long *m = ((unsigned long *) addr) + (nr >> 6);
 	unsigned long temp;
@@ -120,7 +124,7 @@
  * If it's called on the same region of memory simultaneously, the effect
  * may be that only one operation succeeds.
  */
-static inline void __change_bit(int nr, volatile void * addr)
+extern __inline__ void __change_bit(int nr, volatile void * addr)
 {
 	unsigned long * m = ((unsigned long *) addr) + (nr >> 6);
 
@@ -135,8 +139,8 @@
  * This operation is atomic and cannot be reordered.  
  * It also implies a memory barrier.
  */
-static inline unsigned long test_and_set_bit(unsigned long nr,
-					     volatile void *addr)
+extern __inline__ unsigned long
+test_and_set_bit(unsigned long nr, volatile void *addr)
 {
 	unsigned long *m = ((unsigned long *) addr) + (nr >> 6);
 	unsigned long temp, res;
@@ -168,7 +172,8 @@
  * If two examples of this operation race, one can appear to succeed
  * but actually fail.  You must protect multiple accesses with a lock.
  */
-static inline int __test_and_set_bit(int nr, volatile void *addr)
+extern __inline__ int
+__test_and_set_bit(int nr, volatile void * addr)
 {
 	unsigned long mask, retval;
 	long *a = (unsigned long *) addr;
@@ -189,8 +194,8 @@
  * This operation is atomic and cannot be reordered.  
  * It also implies a memory barrier.
  */
-static inline unsigned long test_and_clear_bit(unsigned long nr,
-					       volatile void *addr)
+extern __inline__ unsigned long
+test_and_clear_bit(unsigned long nr, volatile void *addr)
 {
 	unsigned long *m = ((unsigned long *) addr) + (nr >> 6);
 	unsigned long temp, res;
@@ -223,7 +228,8 @@
  * If two examples of this operation race, one can appear to succeed
  * but actually fail.  You must protect multiple accesses with a lock.
  */
-static inline int __test_and_clear_bit(int nr, volatile void * addr)
+extern __inline__ int
+__test_and_clear_bit(int nr, volatile void * addr)
 {
 	unsigned long mask, retval;
 	unsigned long *a = (unsigned long *) addr;
@@ -244,8 +250,8 @@
  * This operation is atomic and cannot be reordered.  
  * It also implies a memory barrier.
  */
-static inline unsigned long test_and_change_bit(unsigned long nr,
-						volatile void *addr)
+extern __inline__ unsigned long
+test_and_change_bit(unsigned long nr, volatile void *addr)
 {
 	unsigned long *m = ((unsigned long *) addr) + (nr >> 6);
 	unsigned long temp, res;
@@ -277,7 +283,8 @@
  * If two examples of this operation race, one can appear to succeed
  * but actually fail.  You must protect multiple accesses with a lock.
  */
-static inline int __test_and_change_bit(int nr, volatile void *addr)
+extern __inline__ int
+__test_and_change_bit(int nr, volatile void * addr)
 {
 	unsigned long mask, retval;
 	unsigned long *a = (unsigned long *) addr;
@@ -294,7 +301,8 @@
  * @nr: bit number to test
  * @addr: Address to start counting from
  */
-static inline unsigned long test_bit(int nr, volatile void * addr)
+extern __inline__ unsigned long
+test_bit(int nr, volatile void * addr)
 {
 	return 1UL & (((volatile unsigned long *) addr)[nr >> 6] >> (nr & 0x3f));
 }
@@ -311,7 +319,8 @@
  * Returns the bit-number of the first zero bit, not the number of the byte
  * containing a bit.
  */
-static inline int find_first_zero_bit (void *addr, unsigned size)
+extern __inline__ int
+find_first_zero_bit (void *addr, unsigned size)
 {
 	unsigned long dummy;
 	int res;
@@ -347,7 +356,8 @@
 		"2:"
 		: "=r" (res), "=r" (dummy), "=r" (addr)
 		: "0" ((signed int) 0), "1" ((unsigned int) 0xffffffff),
-		  "2" (addr), "r" (size));
+		  "2" (addr), "r" (size)
+		: "$1");
 
 	return res;
 }
@@ -358,7 +368,8 @@
  * @offset: The bitnumber to start searching at
  * @size: The maximum size to search
  */
-static inline int find_next_zero_bit (void * addr, int size, int offset)
+extern __inline__ int
+find_next_zero_bit (void * addr, int size, int offset)
 {
 	unsigned int *p = ((unsigned int *) addr) + (offset >> 5);
 	int set = 0, bit = offset & 31, res;
@@ -379,7 +390,8 @@
 			".set\treorder\n"
 			"1:"
 			: "=r" (set), "=r" (dummy)
-			: "0" (0), "1" (1 << bit), "r" (*p));
+			: "0" (0), "1" (1 << bit), "r" (*p)
+			: "$1");
 		if (set < (32 - bit))
 			return set + offset;
 		set = 32 - bit;
@@ -400,19 +412,20 @@
  *
  * Undefined if no zero exists, so code should check against ~0UL first.
  */
-static __inline__ unsigned long ffz(unsigned long word)
+extern __inline__ unsigned long ffz(unsigned long word)
 {
-	int b = 0, s;
+	unsigned long k;
 
 	word = ~word;
-        s = 32; if (word << 32 != 0) s = 0; b += s; word >>= s;
-        s = 16; if (word << 48 != 0) s = 0; b += s; word >>= s;
-        s =  8; if (word << 56 != 0) s = 0; b += s; word >>= s;
-        s =  4; if (word << 60 != 0) s = 0; b += s; word >>= s;
-        s =  2; if (word << 62 != 0) s = 0; b += s; word >>= s;
-        s =  1; if (word << 63 != 0) s = 0; b += s;
+	k = 63;
+	if (word & 0x00000000ffffffffUL) { k -= 32; word <<= 32; }
+	if (word & 0x0000ffff00000000UL) { k -= 16; word <<= 16; }
+	if (word & 0x00ff000000000000UL) { k -= 8;  word <<= 8;  }
+	if (word & 0x0f00000000000000UL) { k -= 4;  word <<= 4;  }
+	if (word & 0x3000000000000000UL) { k -= 2;  word <<= 2;  }
+	if (word & 0x4000000000000000UL) { k -= 1; }
 
-	return b;
+	return k;
 }
 
 #ifdef __KERNEL__
@@ -450,8 +463,8 @@
  * @offset: The bitnumber to start searching at
  * @size: The maximum size to search
  */
-static inline unsigned long find_next_zero_bit(void *addr, unsigned long size,
-					       unsigned long offset)
+extern __inline__ unsigned long
+find_next_zero_bit(void *addr, unsigned long size, unsigned long offset)
 {
 	unsigned long *p = ((unsigned long *) addr) + (offset >> 6);
 	unsigned long result = offset & ~63UL;
@@ -498,7 +511,8 @@
 
 #ifdef __MIPSEB__
 
-static inline int ext2_set_bit(int nr,void * addr)
+extern inline int
+ext2_set_bit(int nr,void * addr)
 {
 	int		mask, retval, flags;
 	unsigned char	*ADDR = (unsigned char *) addr;
@@ -512,7 +526,8 @@
 	return retval;
 }
 
-static inline int ext2_clear_bit(int nr, void * addr)
+extern inline int
+ext2_clear_bit(int nr, void * addr)
 {
 	int		mask, retval, flags;
 	unsigned char	*ADDR = (unsigned char *) addr;
@@ -526,7 +541,8 @@
 	return retval;
 }
 
-static inline int ext2_test_bit(int nr, const void * addr)
+extern inline int
+ext2_test_bit(int nr, const void * addr)
 {
 	int			mask;
 	const unsigned char	*ADDR = (const unsigned char *) addr;
@@ -539,9 +555,8 @@
 #define ext2_find_first_zero_bit(addr, size) \
         ext2_find_next_zero_bit((addr), (size), 0)
 
-static inline unsigned int ext2_find_next_zero_bit(void *addr,
-						   unsigned long size,
-						   unsigned long offset)
+extern inline unsigned int
+ext2_find_next_zero_bit(void *addr, unsigned long size, unsigned long offset)
 {
 	unsigned int *p = ((unsigned int *) addr) + (offset >> 5);
 	unsigned int result = offset & ~31UL;
--- linux/include/asm-s390/bitops.h.orig	Thu Jul 18 15:22:25 2002
+++ linux/include/asm-s390/bitops.h	Thu Jul 18 15:22:35 2002
@@ -47,272 +47,217 @@
 extern const char _oi_bitmap[];
 extern const char _ni_bitmap[];
 extern const char _zb_findmap[];
+extern const char _sb_findmap[];
 
 #ifdef CONFIG_SMP
 /*
  * SMP save set_bit routine based on compare and swap (CS)
  */
-static __inline__ void set_bit_cs(int nr, volatile void * addr)
+static inline void set_bit_cs(int nr, volatile void *ptr)
 {
-	unsigned long bits, mask;
-        __asm__ __volatile__(
+        unsigned long addr, old, new, mask;
+
+	addr = (unsigned long) ptr;
 #if ALIGN_CS == 1
-             "   lhi   %2,3\n"         /* CS must be aligned on 4 byte b. */
-             "   nr    %2,%1\n"        /* isolate last 2 bits of address */
-             "   xr    %1,%2\n"        /* make addr % 4 == 0 */
-             "   sll   %2,3\n"
-             "   ar    %0,%2\n"        /* add alignement to bitnr */
+	addr ^= addr & 3;		/* align address to 4 */
+	nr += (addr & 3) << 3;		/* add alignment to bit number */
 #endif
-             "   lhi   %2,31\n"
-             "   nr    %2,%0\n"        /* make shift value */
-             "   xr    %0,%2\n"
-             "   srl   %0,3\n"
-             "   lhi   %3,1\n"
-             "   la    %1,0(%0,%1)\n"  /* calc. address for CS */
-             "   sll   %3,0(%2)\n"       /* make OR mask */
-             "   l     %0,0(%1)\n"
-             "0: lr    %2,%0\n"         /* CS loop starts here */
-             "   or    %2,%3\n"          /* set bit */
-             "   cs    %0,%2,0(%1)\n"
-             "   jl    0b"
-             : "+a" (nr), "+a" (addr), "=&a" (bits), "=&d" (mask) :
-             : "cc", "memory" );
+	addr += (nr ^ (nr & 31)) >> 3;	/* calculate address for CS */
+	mask = 1UL << (nr & 31);	/* make OR mask */
+	asm volatile(
+		"   l   %0,0(%4)\n"
+		"0: lr  %1,%0\n"
+		"   or  %1,%3\n"
+		"   cs  %0,%1,0(%4)\n"
+		"   jl  0b"
+		: "=&d" (old), "=&d" (new), "+m" (*(unsigned int *) addr)
+		: "d" (mask), "a" (addr) 
+		: "cc" );
 }
 
 /*
  * SMP save clear_bit routine based on compare and swap (CS)
  */
-static __inline__ void clear_bit_cs(int nr, volatile void * addr)
+static inline void clear_bit_cs(int nr, volatile void *ptr)
 {
-        static const int minusone = -1;
-	unsigned long bits, mask;
-        __asm__ __volatile__(
+        unsigned long addr, old, new, mask;
+
+	addr = (unsigned long) ptr;
 #if ALIGN_CS == 1
-             "   lhi   %2,3\n"         /* CS must be aligned on 4 byte b. */
-             "   nr    %2,%1\n"        /* isolate last 2 bits of address */
-             "   xr    %1,%2\n"        /* make addr % 4 == 0 */
-             "   sll   %2,3\n"
-             "   ar    %0,%2\n"        /* add alignement to bitnr */
+	addr ^= addr & 3;		/* align address to 4 */
+	nr += (addr & 3) << 3;		/* add alignment to bit number */
 #endif
-             "   lhi   %2,31\n"
-             "   nr    %2,%0\n"        /* make shift value */
-             "   xr    %0,%2\n"
-             "   srl   %0,3\n"
-             "   lhi   %3,1\n"
-             "   la    %1,0(%0,%1)\n"  /* calc. address for CS */
-             "   sll   %3,0(%2)\n"
-             "   x     %3,%4\n"        /* make AND mask */
-             "   l     %0,0(%1)\n"
-             "0: lr    %2,%0\n"        /* CS loop starts here */
-             "   nr    %2,%3\n"        /* clear bit */
-             "   cs    %0,%2,0(%1)\n"
-             "   jl    0b"
-             : "+a" (nr), "+a" (addr), "=&a" (bits), "=&d" (mask)
-             : "m" (minusone) : "cc", "memory" );
+	addr += (nr ^ (nr & 31)) >> 3;	/* calculate address for CS */
+	mask = ~(1UL << (nr & 31));	/* make AND mask */
+	asm volatile(
+		"   l   %0,0(%4)\n"
+		"0: lr  %1,%0\n"
+		"   nr  %1,%3\n"
+		"   cs  %0,%1,0(%4)\n"
+		"   jl  0b"
+		: "=&d" (old), "=&d" (new), "+m" (*(unsigned int *) addr)
+		: "d" (mask), "a" (addr) 
+		: "cc" );
 }
 
 /*
  * SMP save change_bit routine based on compare and swap (CS)
  */
-static __inline__ void change_bit_cs(int nr, volatile void * addr)
+static inline void change_bit_cs(int nr, volatile void *ptr)
 {
-	unsigned long bits, mask;
-        __asm__ __volatile__(
+        unsigned long addr, old, new, mask;
+
+	addr = (unsigned long) ptr;
 #if ALIGN_CS == 1
-             "   lhi   %2,3\n"         /* CS must be aligned on 4 byte b. */
-             "   nr    %2,%1\n"        /* isolate last 2 bits of address */
-             "   xr    %1,%2\n"        /* make addr % 4 == 0 */
-             "   sll   %2,3\n"
-             "   ar    %0,%2\n"        /* add alignement to bitnr */
+	addr ^= addr & 3;		/* align address to 4 */
+	nr += (addr & 3) << 3;		/* add alignment to bit number */
 #endif
-             "   lhi   %2,31\n"
-             "   nr    %2,%0\n"        /* make shift value */
-             "   xr    %0,%2\n"
-             "   srl   %0,3\n"
-             "   lhi   %3,1\n"
-             "   la    %1,0(%0,%1)\n"  /* calc. address for CS */
-             "   sll   %3,0(%2)\n"     /* make XR mask */
-             "   l     %0,0(%1)\n"
-             "0: lr    %2,%0\n"        /* CS loop starts here */
-             "   xr    %2,%3\n"        /* change bit */
-             "   cs    %0,%2,0(%1)\n"
-             "   jl    0b"
-             : "+a" (nr), "+a" (addr), "=&a" (bits), "=&d" (mask) : 
-             : "cc", "memory" );
+	addr += (nr ^ (nr & 31)) >> 3;	/* calculate address for CS */
+	mask = 1UL << (nr & 31);	/* make XOR mask */
+	asm volatile(
+		"   l   %0,0(%4)\n"
+		"0: lr  %1,%0\n"
+		"   xr  %1,%3\n"
+		"   cs  %0,%1,0(%4)\n"
+		"   jl  0b"
+		: "=&d" (old), "=&d" (new), "+m" (*(unsigned int *) addr)
+		: "d" (mask), "a" (addr) 
+		: "cc" );
 }
 
 /*
  * SMP save test_and_set_bit routine based on compare and swap (CS)
  */
-static __inline__ int test_and_set_bit_cs(int nr, volatile void * addr)
+static inline int test_and_set_bit_cs(int nr, volatile void *ptr)
 {
-	unsigned long bits, mask;
-        __asm__ __volatile__(
+        unsigned long addr, old, new, mask;
+
+	addr = (unsigned long) ptr;
 #if ALIGN_CS == 1
-             "   lhi   %2,3\n"         /* CS must be aligned on 4 byte b. */
-             "   nr    %2,%1\n"        /* isolate last 2 bits of address */
-             "   xr    %1,%2\n"        /* make addr % 4 == 0 */
-             "   sll   %2,3\n"
-             "   ar    %0,%2\n"        /* add alignement to bitnr */
+	addr ^= addr & 3;		/* align address to 4 */
+	nr += (addr & 3) << 3;		/* add alignment to bit number */
 #endif
-             "   lhi   %2,31\n"
-             "   nr    %2,%0\n"        /* make shift value */
-             "   xr    %0,%2\n"
-             "   srl   %0,3\n"
-             "   lhi   %3,1\n"
-             "   la    %1,0(%0,%1)\n"  /* calc. address for CS */
-             "   sll   %3,0(%2)\n"     /* make OR mask */
-             "   l     %0,0(%1)\n"
-             "0: lr    %2,%0\n"        /* CS loop starts here */
-             "   or    %2,%3\n"        /* set bit */
-             "   cs    %0,%2,0(%1)\n"
-             "   jl    0b\n"
-             "   nr    %0,%3\n"        /* isolate old bit */
-             : "+a" (nr), "+a" (addr), "=&a" (bits), "=&d" (mask) :
-             : "cc", "memory" );
-        return nr != 0;
+	addr += (nr ^ (nr & 31)) >> 3;	/* calculate address for CS */
+	mask = 1UL << (nr & 31);	/* make OR/test mask */
+	asm volatile(
+		"   l   %0,0(%4)\n"
+		"0: lr  %1,%0\n"
+		"   or  %1,%3\n"
+		"   cs  %0,%1,0(%4)\n"
+		"   jl  0b"
+		: "=&d" (old), "=&d" (new), "+m" (*(unsigned int *) addr)
+		: "d" (mask), "a" (addr) 
+		: "cc" );
+	return (old & mask) != 0;
 }
 
 /*
  * SMP save test_and_clear_bit routine based on compare and swap (CS)
  */
-static __inline__ int test_and_clear_bit_cs(int nr, volatile void * addr)
+static inline int test_and_clear_bit_cs(int nr, volatile void *ptr)
 {
-        static const int minusone = -1;
-	unsigned long bits, mask;
-        __asm__ __volatile__(
+        unsigned long addr, old, new, mask;
+
+	addr = (unsigned long) ptr;
 #if ALIGN_CS == 1
-             "   lhi   %2,3\n"         /* CS must be aligned on 4 byte b. */
-             "   nr    %2,%1\n"        /* isolate last 2 bits of address */
-             "   xr    %1,%2\n"        /* make addr % 4 == 0 */
-             "   sll   %2,3\n"
-             "   ar    %0,%2\n"        /* add alignement to bitnr */
+	addr ^= addr & 3;		/* align address to 4 */
+	nr += (addr & 3) << 3;		/* add alignment to bit number */
 #endif
-             "   lhi   %2,31\n"
-             "   nr    %2,%0\n"        /* make shift value */
-             "   xr    %0,%2\n"
-             "   srl   %0,3\n"
-             "   lhi   %3,1\n"
-             "   la    %1,0(%0,%1)\n"  /* calc. address for CS */
-             "   sll   %3,0(%2)\n"
-             "   l     %0,0(%1)\n"
-             "   x     %3,%4\n"        /* make AND mask */
-             "0: lr    %2,%0\n"        /* CS loop starts here */
-             "   nr    %2,%3\n"        /* clear bit */
-             "   cs    %0,%2,0(%1)\n"
-             "   jl    0b\n"
-             "   x     %3,%4\n"
-             "   nr    %0,%3\n"         /* isolate old bit */
-             : "+a" (nr), "+a" (addr), "=&a" (bits), "=&d" (mask)
-             : "m" (minusone) : "cc", "memory" );
-        return nr;
+	addr += (nr ^ (nr & 31)) >> 3;	/* calculate address for CS */
+	mask = ~(1UL << (nr & 31));	/* make AND mask */
+	asm volatile(
+		"   l   %0,0(%4)\n"
+		"0: lr  %1,%0\n"
+		"   nr  %1,%3\n"
+		"   cs  %0,%1,0(%4)\n"
+		"   jl  0b"
+		: "=&d" (old), "=&d" (new), "+m" (*(unsigned int *) addr)
+		: "d" (mask), "a" (addr) 
+		: "cc" );
+	return (old ^ new) != 0;
 }
 
 /*
  * SMP save test_and_change_bit routine based on compare and swap (CS) 
  */
-static __inline__ int test_and_change_bit_cs(int nr, volatile void * addr)
+static inline int test_and_change_bit_cs(int nr, volatile void *ptr)
 {
-	unsigned long bits, mask;
-        __asm__ __volatile__(
+        unsigned long addr, old, new, mask;
+
+	addr = (unsigned long) ptr;
 #if ALIGN_CS == 1
-             "   lhi   %2,3\n"         /* CS must be aligned on 4 byte b. */
-             "   nr    %2,%1\n"        /* isolate last 2 bits of address */
-             "   xr    %1,%2\n"        /* make addr % 4 == 0 */
-             "   sll   %2,3\n"
-             "   ar    %0,%2\n"        /* add alignement to bitnr */
+	addr ^= addr & 3;		/* align address to 4 */
+	nr += (addr & 3) << 3;		/* add alignment to bit number */
 #endif
-             "   lhi   %2,31\n"
-             "   nr    %2,%0\n"        /* make shift value */
-             "   xr    %0,%2\n"
-             "   srl   %0,3\n"
-             "   lhi   %3,1\n"
-             "   la    %1,0(%0,%1)\n"  /* calc. address for CS */
-             "   sll   %3,0(%2)\n"     /* make OR mask */
-             "   l     %0,0(%1)\n"
-             "0: lr    %2,%0\n"        /* CS loop starts here */
-             "   xr    %2,%3\n"        /* change bit */
-             "   cs    %0,%2,0(%1)\n"
-             "   jl    0b\n"
-             "   nr    %0,%3\n"        /* isolate old bit */
-             : "+a" (nr), "+a" (addr), "=&a" (bits), "=&d" (mask) :
-             : "cc", "memory" );
-        return nr != 0;
+	addr += (nr ^ (nr & 31)) >> 3;	/* calculate address for CS */
+	mask = 1UL << (nr & 31);	/* make XOR mask */
+	asm volatile(
+		"   l   %0,0(%4)\n"
+		"0: lr  %1,%0\n"
+		"   xr  %1,%3\n"
+		"   cs  %0,%1,0(%4)\n"
+		"   jl  0b"
+		: "=&d" (old), "=&d" (new), "+m" (*(unsigned int *) addr)
+		: "d" (mask), "a" (addr) 
+		: "cc" );
+	return (old & mask) != 0;
 }
 #endif /* CONFIG_SMP */
 
 /*
  * fast, non-SMP set_bit routine
  */
-static __inline__ void __set_bit(int nr, volatile void * addr)
+static inline void __set_bit(int nr, volatile void *ptr)
 {
-	unsigned long reg1, reg2;
-        __asm__ __volatile__(
-             "   lhi   %1,24\n"
-             "   lhi   %0,7\n"
-             "   xr    %1,%2\n"
-             "   nr    %0,%2\n"
-             "   srl   %1,3\n"
-             "   la    %1,0(%1,%3)\n"
-             "   la    %0,0(%0,%4)\n"
-             "   oc    0(1,%1),0(%0)"
-             : "=&a" (reg1), "=&a" (reg2)
-             : "r" (nr), "a" (addr), "a" (&_oi_bitmap) : "cc", "memory" );
-}
-
-static __inline__ void 
-__constant_set_bit(const int nr, volatile void * addr)
-{
-  switch (nr&7) {
-  case 0:
-    __asm__ __volatile__ ("la 1,%0\n\t"
-                          "oi 0(1),0x01"
-                          : "=m" (*((volatile char *) addr + ((nr>>3)^3))) 
-                          : : "1", "cc", "memory");
-    break;
-  case 1:
-    __asm__ __volatile__ ("la 1,%0\n\t"
-                          "oi 0(1),0x02"
-                          : "=m" (*((volatile char *) addr + ((nr>>3)^3)))
-                          : : "1", "cc", "memory" );
-    break;
-  case 2:
-    __asm__ __volatile__ ("la 1,%0\n\t"
-                          "oi 0(1),0x04"
-                          : "=m" (*((volatile char *) addr + ((nr>>3)^3)))
-                          : : "1", "cc", "memory" );
-    break;
-  case 3:
-    __asm__ __volatile__ ("la 1,%0\n\t"
-                          "oi 0(1),0x08"
-                          : "=m" (*((volatile char *) addr + ((nr>>3)^3)))
-                          : : "1", "cc", "memory" );
-    break;
-  case 4:
-    __asm__ __volatile__ ("la 1,%0\n\t"
-                          "oi 0(1),0x10"
-                          : "=m" (*((volatile char *) addr + ((nr>>3)^3)))
-                          : : "1", "cc", "memory" );
-    break;
-  case 5:
-    __asm__ __volatile__ ("la 1,%0\n\t"
-                          "oi 0(1),0x20"
-                          : "=m" (*((volatile char *) addr + ((nr>>3)^3)))
-                          : : "1", "cc", "memory" );
-    break;
-  case 6:
-    __asm__ __volatile__ ("la 1,%0\n\t"
-                          "oi 0(1),0x40"
-                          : "=m" (*((volatile char *) addr + ((nr>>3)^3)))
-                          : : "1", "cc", "memory" );
-    break;
-  case 7:
-    __asm__ __volatile__ ("la 1,%0\n\t"
-                          "oi 0(1),0x80"
-                          : "=m" (*((volatile char *) addr + ((nr>>3)^3)))
-                          : : "1", "cc", "memory" );
-    break;
-  }
+	unsigned long addr;
+
+	addr = (unsigned long) ptr + ((nr ^ 24) >> 3);
+        asm volatile("oc 0(1,%1),0(%2)"
+		     : "+m" (*(char *) addr)
+		     : "a" (addr), "a" (_oi_bitmap + (nr & 7))
+		     : "cc" );
+}
+
+static inline void 
+__constant_set_bit(const int nr, volatile void *ptr)
+{
+	unsigned long addr;
+
+	addr = ((unsigned long) ptr) + ((nr >> 3) ^ 3);
+	switch (nr&7) {
+	case 0:
+		asm volatile ("oi 0(%1),0x01"
+			      : "+m" (*(char *) addr) : "a" (addr) : "cc" );
+		break;
+	case 1:
+		asm volatile ("oi 0(%1),0x02"
+			      : "+m" (*(char *) addr) : "a" (addr) : "cc" );
+		break;
+	case 2:
+		asm volatile ("oi 0(%1),0x04" 
+			      : "+m" (*(char *) addr) : "a" (addr) : "cc" );
+		break;
+	case 3:
+		asm volatile ("oi 0(%1),0x08" 
+			      : "+m" (*(char *) addr) : "a" (addr) : "cc" );
+		break;
+	case 4:
+		asm volatile ("oi 0(%1),0x10" 
+			      : "+m" (*(char *) addr) : "a" (addr) : "cc" );
+		break;
+	case 5:
+		asm volatile ("oi 0(%1),0x20" 
+			      : "+m" (*(char *) addr) : "a" (addr) : "cc" );
+		break;
+	case 6:
+		asm volatile ("oi 0(%1),0x40" 
+			      : "+m" (*(char *) addr) : "a" (addr) : "cc" );
+		break;
+	case 7:
+		asm volatile ("oi 0(%1),0x80" 
+			      : "+m" (*(char *) addr) : "a" (addr) : "cc" );
+		break;
+	}
 }
 
 #define set_bit_simple(nr,addr) \
@@ -323,76 +268,58 @@
 /*
  * fast, non-SMP clear_bit routine
  */
-static __inline__ void 
-__clear_bit(int nr, volatile void * addr)
+static inline void 
+__clear_bit(int nr, volatile void *ptr)
 {
-	unsigned long reg1, reg2;
-        __asm__ __volatile__(
-             "   lhi   %1,24\n"
-             "   lhi   %0,7\n"
-             "   xr    %1,%2\n"
-             "   nr    %0,%2\n"
-             "   srl   %1,3\n"
-             "   la    %1,0(%1,%3)\n"
-             "   la    %0,0(%0,%4)\n"
-             "   nc    0(1,%1),0(%0)"
-             : "=&a" (reg1), "=&a" (reg2)
-             : "r" (nr), "a" (addr), "a" (&_ni_bitmap) : "cc", "memory" );
-}
-
-static __inline__ void 
-__constant_clear_bit(const int nr, volatile void * addr)
-{
-  switch (nr&7) {
-  case 0:
-    __asm__ __volatile__ ("la 1,%0\n\t"
-                          "ni 0(1),0xFE"
-                          : "=m" (*((volatile char *) addr + ((nr>>3)^3)))
-                          : : "1", "cc", "memory" );
-    break;
-  case 1:
-    __asm__ __volatile__ ("la 1,%0\n\t"
-                          "ni 0(1),0xFD"
-                          : "=m" (*((volatile char *) addr + ((nr>>3)^3)))
-                          : : "1", "cc", "memory" );
-    break;
-  case 2:
-    __asm__ __volatile__ ("la 1,%0\n\t"
-                          "ni 0(1),0xFB"
-                          : "=m" (*((volatile char *) addr + ((nr>>3)^3)))
-                          : : "1", "cc", "memory" );
-    break;
-  case 3:
-    __asm__ __volatile__ ("la 1,%0\n\t"
-                          "ni 0(1),0xF7"
-                          : "=m" (*((volatile char *) addr + ((nr>>3)^3)))
-                          : : "1", "cc", "memory" );
-    break;
-  case 4:
-    __asm__ __volatile__ ("la 1,%0\n\t"
-                          "ni 0(1),0xEF"
-                          : "=m" (*((volatile char *) addr + ((nr>>3)^3)))
-                          : : "cc", "memory" );
-    break;
-  case 5:
-    __asm__ __volatile__ ("la 1,%0\n\t"
-                          "ni 0(1),0xDF"
-                          : "=m" (*((volatile char *) addr + ((nr>>3)^3)))
-                          : : "1", "cc", "memory" );
-    break;
-  case 6:
-    __asm__ __volatile__ ("la 1,%0\n\t"
-                          "ni 0(1),0xBF"
-                          : "=m" (*((volatile char *) addr + ((nr>>3)^3)))
-                          : : "1", "cc", "memory" );
-    break;
-  case 7:
-    __asm__ __volatile__ ("la 1,%0\n\t"
-                          "ni 0(1),0x7F"
-                          : "=m" (*((volatile char *) addr + ((nr>>3)^3)))
-                          : : "1", "cc", "memory" );
-    break;
-  }
+	unsigned long addr;
+
+	addr = (unsigned long) ptr + ((nr ^ 24) >> 3);
+        asm volatile("nc 0(1,%1),0(%2)"
+		     : "+m" (*(char *) addr)
+		     : "a" (addr), "a" (_ni_bitmap + (nr & 7))
+		     : "cc" );
+}
+
+static inline void 
+__constant_clear_bit(const int nr, volatile void *ptr)
+{
+	unsigned long addr;
+
+	addr = ((unsigned long) ptr) + ((nr >> 3) ^ 3);
+	switch (nr&7) {
+	case 0:
+		asm volatile ("ni 0(%1),0xFE"
+			      : "+m" (*(char *) addr) : "a" (addr) : "cc" );
+		break;
+	case 1:
+		asm volatile ("ni 0(%1),0xFD" 
+			      : "+m" (*(char *) addr) : "a" (addr) : "cc" );
+		break;
+	case 2:
+		asm volatile ("ni 0(%1),0xFB" 
+			      : "+m" (*(char *) addr) : "a" (addr) : "cc" );
+		break;
+	case 3:
+		asm volatile ("ni 0(%1),0xF7" 
+			      : "+m" (*(char *) addr) : "a" (addr) : "cc" );
+		break;
+	case 4:
+		asm volatile ("ni 0(%1),0xEF" 
+			      : "+m" (*(char *) addr) : "a" (addr) : "cc" );
+		break;
+	case 5:
+		asm volatile ("ni 0(%1),0xDF" 
+			      : "+m" (*(char *) addr) : "a" (addr) : "cc" );
+		break;
+	case 6:
+		asm volatile ("ni 0(%1),0xBF" 
+			      : "+m" (*(char *) addr) : "a" (addr) : "cc" );
+		break;
+	case 7:
+		asm volatile ("ni 0(%1),0x7F" 
+			      : "+m" (*(char *) addr) : "a" (addr) : "cc" );
+		break;
+	}
 }
 
 #define clear_bit_simple(nr,addr) \
@@ -403,75 +330,57 @@
 /* 
  * fast, non-SMP change_bit routine 
  */
-static __inline__ void __change_bit(int nr, volatile void * addr)
+static inline void __change_bit(int nr, volatile void *ptr)
 {
-	unsigned long reg1, reg2;
-        __asm__ __volatile__(
-             "   lhi   %1,24\n"
-             "   lhi   %0,7\n"
-             "   xr    %1,%2\n"
-             "   nr    %0,%2\n"
-             "   srl   %1,3\n"
-             "   la    %1,0(%1,%3)\n"
-             "   la    %0,0(%0,%4)\n"
-             "   xc    0(1,%1),0(%0)"
-             : "=&a" (reg1), "=&a" (reg2)
-             : "r" (nr), "a" (addr), "a" (&_oi_bitmap) : "cc", "memory" );
-}
-
-static __inline__ void 
-__constant_change_bit(const int nr, volatile void * addr) 
-{
-  switch (nr&7) {
-  case 0:
-    __asm__ __volatile__ ("la 1,%0\n\t"
-                          "xi 0(1),0x01"
-                          : "=m" (*((volatile char *) addr + ((nr>>3)^3)))
-                          : : "cc", "memory" );
-    break;
-  case 1:
-    __asm__ __volatile__ ("la 1,%0\n\t"
-                          "xi 0(1),0x02"
-                          : "=m" (*((volatile char *) addr + ((nr>>3)^3)))
-                          : : "cc", "memory" );
-    break;
-  case 2:
-    __asm__ __volatile__ ("la 1,%0\n\t"
-                          "xi 0(1),0x04"
-                          : "=m" (*((volatile char *) addr + ((nr>>3)^3)))
-                          : : "cc", "memory" );
-    break;
-  case 3:
-    __asm__ __volatile__ ("la 1,%0\n\t"
-                          "xi 0(1),0x08"
-                          : "=m" (*((volatile char *) addr + ((nr>>3)^3)))
-                          : : "cc", "memory" );
-    break;
-  case 4:
-    __asm__ __volatile__ ("la 1,%0\n\t"
-                          "xi 0(1),0x10"
-                          : "=m" (*((volatile char *) addr + ((nr>>3)^3)))
-                          : : "cc", "memory" );
-    break;
-  case 5:
-    __asm__ __volatile__ ("la 1,%0\n\t"
-                          "xi 0(1),0x20"
-                          : "=m" (*((volatile char *) addr + ((nr>>3)^3)))
-                          : : "1", "cc", "memory" );
-    break;
-  case 6:
-    __asm__ __volatile__ ("la 1,%0\n\t"
-                          "xi 0(1),0x40"
-                          : "=m" (*((volatile char *) addr + ((nr>>3)^3)))
-                          : : "1", "cc", "memory" );
-    break;
-  case 7:
-    __asm__ __volatile__ ("la 1,%0\n\t"
-                          "xi 0(1),0x80"
-                          : "=m" (*((volatile char *) addr + ((nr>>3)^3)))
-                          : : "1", "cc", "memory" );
-    break;
-  }
+	unsigned long addr;
+
+	addr = (unsigned long) ptr + ((nr ^ 24) >> 3);
+        asm volatile("xc 0(1,%1),0(%2)"
+		     : "+m" (*(char *) addr)
+		     : "a" (addr), "a" (_oi_bitmap + (nr & 7))
+		     : "cc" );
+}
+
+static inline void 
+__constant_change_bit(const int nr, volatile void *ptr) 
+{
+	unsigned long addr;
+
+	addr = ((unsigned long) ptr) + ((nr >> 3) ^ 3);
+	switch (nr&7) {
+	case 0:
+		asm volatile ("xi 0(%1),0x01" 
+			      : "+m" (*(char *) addr) : "a" (addr) : "cc" );
+		break;
+	case 1:
+		asm volatile ("xi 0(%1),0x02" 
+			      : "+m" (*(char *) addr) : "a" (addr) : "cc" );
+		break;
+	case 2:
+		asm volatile ("xi 0(%1),0x04" 
+			      : "+m" (*(char *) addr) : "a" (addr) : "cc" );
+		break;
+	case 3:
+		asm volatile ("xi 0(%1),0x08" 
+			      : "+m" (*(char *) addr) : "a" (addr) : "cc" );
+		break;
+	case 4:
+		asm volatile ("xi 0(%1),0x10" 
+			      : "+m" (*(char *) addr) : "a" (addr) : "cc" );
+		break;
+	case 5:
+		asm volatile ("xi 0(%1),0x20" 
+			      : "+m" (*(char *) addr) : "a" (addr) : "cc" );
+		break;
+	case 6:
+		asm volatile ("xi 0(%1),0x40" 
+			      : "+m" (*(char *) addr) : "a" (addr) : "cc" );
+		break;
+	case 7:
+		asm volatile ("xi 0(%1),0x80" 
+			      : "+m" (*(char *) addr) : "a" (addr) : "cc" );
+		break;
+	}
 }
 
 #define change_bit_simple(nr,addr) \
@@ -482,74 +391,54 @@
 /*
  * fast, non-SMP test_and_set_bit routine
  */
-static __inline__ int test_and_set_bit_simple(int nr, volatile void * addr)
+static inline int test_and_set_bit_simple(int nr, volatile void *ptr)
 {
-	unsigned long reg1, reg2;
-        int oldbit;
-        __asm__ __volatile__(
-             "   lhi   %1,24\n"
-             "   lhi   %2,7\n"
-             "   xr    %1,%3\n"
-             "   nr    %2,%3\n"
-             "   srl   %1,3\n"
-             "   la    %1,0(%1,%4)\n"
-             "   ic    %0,0(%1)\n"
-             "   srl   %0,0(%2)\n"
-             "   la    %2,0(%2,%5)\n"
-             "   oc    0(1,%1),0(%2)"
-             : "=d&" (oldbit), "=&a" (reg1), "=&a" (reg2)
-             : "r" (nr), "a" (addr), "a" (&_oi_bitmap) : "cc", "memory" );
-        return oldbit & 1;
+	unsigned long addr;
+	unsigned char ch;
+
+	addr = (unsigned long) ptr + ((nr ^ 24) >> 3);
+	ch = *(unsigned char *) addr;
+        asm volatile("oc 0(1,%1),0(%2)"
+		     : "+m" (*(char *) addr)
+		     : "a" (addr), "a" (_oi_bitmap + (nr & 7))
+		     : "cc" );
+	return (ch >> (nr & 7)) & 1;
 }
 #define __test_and_set_bit(X,Y)		test_and_set_bit_simple(X,Y)
 
 /*
  * fast, non-SMP test_and_clear_bit routine
  */
-static __inline__ int test_and_clear_bit_simple(int nr, volatile void * addr)
+static inline int test_and_clear_bit_simple(int nr, volatile void *ptr)
 {
-	unsigned long reg1, reg2;
-        int oldbit;
+	unsigned long addr;
+	unsigned char ch;
 
-        __asm__ __volatile__(
-             "   lhi   %1,24\n"
-             "   lhi   %2,7\n"
-             "   xr    %1,%3\n"
-             "   nr    %2,%3\n"
-             "   srl   %1,3\n"
-             "   la    %1,0(%1,%4)\n"
-             "   ic    %0,0(%1)\n"
-             "   srl   %0,0(%2)\n"
-             "   la    %2,0(%2,%5)\n"
-             "   nc    0(1,%1),0(%2)"
-             : "=d&" (oldbit), "=&a" (reg1), "=&a" (reg2)
-             : "r" (nr), "a" (addr), "a" (&_ni_bitmap) : "cc", "memory" );
-        return oldbit & 1;
+	addr = (unsigned long) ptr + ((nr ^ 24) >> 3);
+	ch = *(unsigned char *) addr;
+        asm volatile("nc 0(1,%1),0(%2)"
+		     : "+m" (*(char *) addr)
+		     : "a" (addr), "a" (_ni_bitmap + (nr & 7))
+		     : "cc" );
+	return (ch >> (nr & 7)) & 1;
 }
 #define __test_and_clear_bit(X,Y)	test_and_clear_bit_simple(X,Y)
 
 /*
  * fast, non-SMP test_and_change_bit routine
  */
-static __inline__ int test_and_change_bit_simple(int nr, volatile void * addr)
+static inline int test_and_change_bit_simple(int nr, volatile void *ptr)
 {
-	unsigned long reg1, reg2;
-        int oldbit;
+	unsigned long addr;
+	unsigned char ch;
 
-        __asm__ __volatile__(
-             "   lhi   %1,24\n"
-             "   lhi   %2,7\n"
-             "   xr    %1,%3\n"
-             "   nr    %2,%1\n"
-             "   srl   %1,3\n"
-             "   la    %1,0(%1,%4)\n"
-             "   ic    %0,0(%1)\n"
-             "   srl   %0,0(%2)\n"
-             "   la    %2,0(%2,%5)\n"
-             "   xc    0(1,%1),0(%2)"
-             : "=d&" (oldbit), "=&a" (reg1), "=&a" (reg2)
-             : "r" (nr), "a" (addr), "a" (&_oi_bitmap) : "cc", "memory" );
-        return oldbit & 1;
+	addr = (unsigned long) ptr + ((nr ^ 24) >> 3);
+	ch = *(unsigned char *) addr;
+        asm volatile("xc 0(1,%1),0(%2)"
+		     : "+m" (*(char *) addr)
+		     : "a" (addr), "a" (_oi_bitmap + (nr & 7))
+		     : "cc" );
+	return (ch >> (nr & 7)) & 1;
 }
 #define __test_and_change_bit(X,Y)	test_and_change_bit_simple(X,Y)
 
@@ -574,25 +463,17 @@
  * This routine doesn't need to be atomic.
  */
 
-static __inline__ int __test_bit(int nr, volatile void * addr)
+static inline int __test_bit(int nr, volatile void *ptr)
 {
-	unsigned long reg1, reg2;
-        int oldbit;
+	unsigned long addr;
+	unsigned char ch;
 
-        __asm__ __volatile__(
-             "   lhi   %2,24\n"
-             "   lhi   %1,7\n"
-             "   xr    %2,%3\n"
-             "   nr    %1,%3\n"
-             "   srl   %2,3\n"
-             "   ic    %0,0(%2,%4)\n"
-             "   srl   %0,0(%1)"
-             : "=d&" (oldbit), "=&a" (reg1), "=&a" (reg2)
-             : "r" (nr), "a" (addr) : "cc" );
-        return oldbit & 1;
+	addr = (unsigned long) ptr + ((nr ^ 24) >> 3);
+	ch = *(unsigned char *) addr;
+	return (ch >> (nr & 7)) & 1;
 }
 
-static __inline__ int __constant_test_bit(int nr, volatile void * addr) {
+static inline int __constant_test_bit(int nr, volatile void * addr) {
     return (((volatile char *) addr)[(nr>>3)^3] & (1<<(nr&7))) != 0;
 }
 
@@ -604,7 +485,7 @@
 /*
  * Find-bit routines..
  */
-static __inline__ int find_first_zero_bit(void * addr, unsigned size)
+static inline int find_first_zero_bit(void * addr, unsigned size)
 {
 	unsigned long cmp, count;
         int res;
@@ -642,7 +523,45 @@
         return (res < size) ? res : size;
 }
 
-static __inline__ int find_next_zero_bit (void * addr, int size, int offset)
+static inline int find_first_bit(void * addr, unsigned size)
+{
+	unsigned long cmp, count;
+        int res;
+
+        if (!size)
+                return 0;
+        __asm__("   slr  %1,%1\n"
+                "   lr   %2,%3\n"
+                "   slr  %0,%0\n"
+                "   ahi  %2,31\n"
+                "   srl  %2,5\n"
+                "0: c    %1,0(%0,%4)\n"
+                "   jne  1f\n"
+                "   ahi  %0,4\n"
+                "   brct %2,0b\n"
+                "   lr   %0,%3\n"
+                "   j    4f\n"
+                "1: l    %2,0(%0,%4)\n"
+                "   sll  %0,3\n"
+                "   lhi  %1,0xff\n"
+                "   tml  %2,0xffff\n"
+                "   jnz  2f\n"
+                "   ahi  %0,16\n"
+                "   srl  %2,16\n"
+                "2: tml  %2,0x00ff\n"
+                "   jnz  3f\n"
+                "   ahi  %0,8\n"
+                "   srl  %2,8\n"
+                "3: nr   %2,%1\n"
+                "   ic   %2,0(%2,%5)\n"
+                "   alr  %0,%2\n"
+                "4:"
+                : "=&a" (res), "=&d" (cmp), "=&a" (count)
+                : "a" (size), "a" (addr), "a" (&_sb_findmap) : "cc" );
+        return (res < size) ? res : size;
+}
+
+static inline int find_next_zero_bit (void * addr, int size, int offset)
 {
         unsigned long * p = ((unsigned long *) addr) + (offset >> 5);
         unsigned long bitvec, reg;
@@ -680,11 +599,49 @@
         return (offset + res);
 }
 
+static inline int find_next_bit (void * addr, int size, int offset)
+{
+        unsigned long * p = ((unsigned long *) addr) + (offset >> 5);
+        unsigned long bitvec, reg;
+        int set, bit = offset & 31, res;
+
+        if (bit) {
+                /*
+                 * Look for set bit in first word
+                 */
+                bitvec = (*p) >> bit;
+                __asm__("   slr  %0,%0\n"
+                        "   lhi  %2,0xff\n"
+                        "   tml  %1,0xffff\n"
+                        "   jnz  0f\n"
+                        "   ahi  %0,16\n"
+                        "   srl  %1,16\n"
+                        "0: tml  %1,0x00ff\n"
+                        "   jnz  1f\n"
+                        "   ahi  %0,8\n"
+                        "   srl  %1,8\n"
+                        "1: nr   %1,%2\n"
+                        "   ic   %1,0(%1,%3)\n"
+                        "   alr  %0,%1"
+                        : "=&d" (set), "+a" (bitvec), "=&d" (reg)
+                        : "a" (&_sb_findmap) : "cc" );
+                if (set < (32 - bit))
+                        return set + offset;
+                offset += 32 - bit;
+                p++;
+        }
+        /*
+         * No set bit yet, search remaining full words for a bit
+         */
+        res = find_first_bit (p, size - 32 * (p - (unsigned long *) addr));
+        return (offset + res);
+}
+
 /*
  * ffz = Find First Zero in word. Undefined if no zero exists,
  * so code should check against ~0UL first..
  */
-static __inline__ unsigned long ffz(unsigned long word)
+static inline unsigned long ffz(unsigned long word)
 {
 	unsigned long reg;
         int result;
@@ -708,40 +665,109 @@
 }
 
 /*
+ * __ffs = find first bit in word. Undefined if no bit exists,
+ * so code should check against 0UL first..
+ */
+static inline unsigned long __ffs(unsigned long word)
+{
+	unsigned long reg, result;
+
+        __asm__("   slr  %0,%0\n"
+                "   lhi  %2,0xff\n"
+                "   tml  %1,0xffff\n"
+                "   jnz  0f\n"
+                "   ahi  %0,16\n"
+                "   srl  %1,16\n"
+                "0: tml  %1,0x00ff\n"
+                "   jnz  1f\n"
+                "   ahi  %0,8\n"
+                "   srl  %1,8\n"
+                "1: nr   %1,%2\n"
+                "   ic   %1,0(%1,%3)\n"
+                "   alr  %0,%1"
+                : "=&d" (result), "+a" (word), "=&d" (reg)
+                : "a" (&_sb_findmap) : "cc" );
+        return result;
+}
+
+/*
+ * Every architecture must define this function. It's the fastest
+ * way of searching a 140-bit bitmap where the first 100 bits are
+ * unlikely to be set. It's guaranteed that at least one of the 140
+ * bits is cleared.
+ */
+static inline int sched_find_first_bit(unsigned long *b)
+{
+	return find_first_bit(b, 140);
+}
+
+/*
  * ffs: find first bit set. This is defined the same way as
  * the libc and compiler builtin ffs routines, therefore
  * differs in spirit from the above ffz (man ffs).
  */
 
-extern int __inline__ ffs (int x)
+extern int inline ffs (int x)
 {
-        int r;
+        int r = 1;
 
         if (x == 0)
-          return 0;
-        __asm__("    slr  %0,%0\n"
-                "    tml  %1,0xffff\n"
+		return 0;
+        __asm__("    tml  %1,0xffff\n"
                 "    jnz  0f\n"
-                "    ahi  %0,16\n"
                 "    srl  %1,16\n"
+                "    ahi  %0,16\n"
                 "0:  tml  %1,0x00ff\n"
                 "    jnz  1f\n"
-                "    ahi  %0,8\n"
                 "    srl  %1,8\n"
+                "    ahi  %0,8\n"
                 "1:  tml  %1,0x000f\n"
                 "    jnz  2f\n"
-                "    ahi  %0,4\n"
                 "    srl  %1,4\n"
+                "    ahi  %0,4\n"
                 "2:  tml  %1,0x0003\n"
                 "    jnz  3f\n"
-                "    ahi  %0,2\n"
                 "    srl  %1,2\n"
+                "    ahi  %0,2\n"
                 "3:  tml  %1,0x0001\n"
                 "    jnz  4f\n"
                 "    ahi  %0,1\n"
                 "4:"
                 : "=&d" (r), "+d" (x) : : "cc" );
-        return r+1;
+        return r;
+}
+
+/*
+ * fls: find last bit set.
+ */
+extern __inline__ int fls(int x)
+{
+	int r = 32;
+
+	if (x == 0)
+		return 0;
+	__asm__("    tmh  %1,0xffff\n"
+		"    jz   0f\n"
+		"    sll  %1,16\n"
+		"    ahi  %0,-16\n"
+		"0:  tmh  %1,0xff00\n"
+		"    jz   1f\n"
+		"    sll  %1,8\n"
+		"    ahi  %0,-8\n"
+		"1:  tmh  %1,0xf000\n"
+		"    jz   2f\n"
+		"    sll  %1,4\n"
+		"    ahi  %0,-4\n"
+		"2:  tmh  %1,0xc000\n"
+		"    jz   3f\n"
+		"    sll  %1,2\n"
+		"    ahi  %0,-2\n"
+		"3:  tmh  %1,0x8000\n"
+		"    jz   4f\n"
+		"    ahi  %0,-1\n"
+		"4:"
+		: "+d" (r), "+d" (x) : : "cc" );
+	return r;
 }
 
 /*
@@ -769,7 +795,7 @@
 #define ext2_set_bit(nr, addr)       test_and_set_bit((nr)^24, addr)
 #define ext2_clear_bit(nr, addr)     test_and_clear_bit((nr)^24, addr)
 #define ext2_test_bit(nr, addr)      test_bit((nr)^24, addr)
-static __inline__ int ext2_find_first_zero_bit(void *vaddr, unsigned size)
+static inline int ext2_find_first_zero_bit(void *vaddr, unsigned size)
 {
 	unsigned long cmp, count;
         int res;
@@ -808,7 +834,7 @@
         return (res < size) ? res : size;
 }
 
-static __inline__ int 
+static inline int 
 ext2_find_next_zero_bit(void *vaddr, unsigned size, unsigned offset)
 {
         unsigned long *addr = vaddr;
--- linux/include/asm-cris/bitops.h.orig	Thu Jul 18 15:22:16 2002
+++ linux/include/asm-cris/bitops.h	Thu Jul 18 15:22:35 2002
@@ -22,6 +22,7 @@
 /* We use generic_ffs so get it; include guards resolve the possible
    mutually inclusion.  */
 #include <linux/bitops.h>
+#include <linux/compiler.h>
 
 /*
  * Some hacks to defeat gcc over-optimizations..
@@ -43,6 +44,8 @@
 
 #define set_bit(nr, addr)    (void)test_and_set_bit(nr, addr)
 
+#define __set_bit(nr, addr)    (void)__test_and_set_bit(nr, addr)
+
 /*
  * clear_bit - Clears a bit in memory
  * @nr: Bit to clear
@@ -56,6 +59,8 @@
 
 #define clear_bit(nr, addr)  (void)test_and_clear_bit(nr, addr)
 
+#define __clear_bit(nr, addr)  (void)__test_and_clear_bit(nr, addr)
+
 /*
  * change_bit - Toggle a bit in memory
  * @nr: Bit to clear
@@ -89,7 +94,7 @@
  * It also implies a memory barrier.
  */
 
-static __inline__ int test_and_set_bit(int nr, void *addr)
+static inline int test_and_set_bit(int nr, void *addr)
 {
 	unsigned int mask, retval;
 	unsigned long flags;
@@ -105,6 +110,18 @@
 	return retval;
 }
 
+static inline int __test_and_set_bit(int nr, void *addr)
+{
+	unsigned int mask, retval;
+	unsigned int *adr = (unsigned int *)addr;
+	
+	adr += nr >> 5;
+	mask = 1 << (nr & 0x1f);
+	retval = (mask & *adr) != 0;
+	*adr |= mask;
+	return retval;
+}
+
 /*
  * clear_bit() doesn't provide any barrier for the compiler.
  */
@@ -120,7 +137,7 @@
  * It also implies a memory barrier.
  */
 
-static __inline__ int test_and_clear_bit(int nr, void *addr)
+static inline int test_and_clear_bit(int nr, void *addr)
 {
 	unsigned int mask, retval;
 	unsigned long flags;
@@ -146,7 +163,7 @@
  * but actually fail.  You must protect multiple accesses with a lock.
  */
 
-static __inline__ int __test_and_clear_bit(int nr, void *addr)
+static inline int __test_and_clear_bit(int nr, void *addr)
 {
 	unsigned int mask, retval;
 	unsigned int *adr = (unsigned int *)addr;
@@ -166,7 +183,7 @@
  * It also implies a memory barrier.
  */
 
-static __inline__ int test_and_change_bit(int nr, void *addr)
+static inline int test_and_change_bit(int nr, void *addr)
 {
 	unsigned int mask, retval;
 	unsigned long flags;
@@ -183,7 +200,7 @@
 
 /* WARNING: non atomic and it can be reordered! */
 
-static __inline__ int __test_and_change_bit(int nr, void *addr)
+static inline int __test_and_change_bit(int nr, void *addr)
 {
 	unsigned int mask, retval;
 	unsigned int *adr = (unsigned int *)addr;
@@ -204,7 +221,7 @@
  * This routine doesn't need to be atomic.
  */
 
-static __inline__ int test_bit(int nr, const void *addr)
+static inline int test_bit(int nr, const void *addr)
 {
 	unsigned int mask;
 	unsigned int *adr = (unsigned int *)addr;
@@ -225,7 +242,7 @@
  * number.  They differ in that the first function also inverts all bits
  * in the input.
  */
-static __inline__ unsigned long cris_swapnwbrlz(unsigned long w)
+static inline unsigned long cris_swapnwbrlz(unsigned long w)
 {
 	/* Let's just say we return the result in the same register as the
 	   input.  Saying we clobber the input but can return the result
@@ -241,7 +258,7 @@
 	return res;
 }
 
-static __inline__ unsigned long cris_swapwbrlz(unsigned long w)
+static inline unsigned long cris_swapwbrlz(unsigned long w)
 {
 	unsigned res;
 	__asm__ ("swapwbr %0 \n\t"
@@ -255,7 +272,7 @@
  * ffz = Find First Zero in word. Undefined if no zero exists,
  * so code should check against ~0UL first..
  */
-static __inline__ unsigned long ffz(unsigned long w)
+static inline unsigned long ffz(unsigned long w)
 {
 	/* The generic_ffs function is used to avoid the asm when the
 	   argument is a constant.  */
@@ -268,7 +285,7 @@
  * Somewhat like ffz but the equivalent of generic_ffs: in contrast to
  * ffz we return the first one-bit *plus one*.
  */
-static __inline__ unsigned long ffs(unsigned long w)
+static inline unsigned long ffs(unsigned long w)
 {
 	/* The generic_ffs function is used to avoid the asm when the
 	   argument is a constant.  */
@@ -283,7 +300,7 @@
  * @offset: The bitnumber to start searching at
  * @size: The maximum size to search
  */
-static __inline__ int find_next_zero_bit (void * addr, int size, int offset)
+static inline int find_next_zero_bit (void * addr, int size, int offset)
 {
 	unsigned long *p = ((unsigned long *) addr) + (offset >> 5);
 	unsigned long result = offset & ~31UL;
@@ -354,7 +371,45 @@
 #define minix_test_bit(nr,addr) test_bit(nr,addr)
 #define minix_find_first_zero_bit(addr,size) find_first_zero_bit(addr,size)
 
-#endif /* __KERNEL__ */
+#if 0
+/* TODO: see below */
+#define sched_find_first_zero_bit(addr) find_first_zero_bit(addr, 168)
+
+#else
+/* TODO: left out pending where to put it.. (there are .h dependencies) */
+
+ /*
+ * Every architecture must define this function. It's the fastest
+ * way of searching a 168-bit bitmap where the first 128 bits are
+ * unlikely to be set. It's guaranteed that at least one of the 168
+ * bits is cleared.
+ */
+#if 0
+#if MAX_RT_PRIO != 128 || MAX_PRIO != 168
+# error update this function.
+#endif
+#else
+#define MAX_RT_PRIO 128
+#define MAX_PRIO 168
+#endif
+
+static inline int sched_find_first_zero_bit(char *bitmap)
+{
+	unsigned int *b = (unsigned int *)bitmap;
+	unsigned int rt;
+
+	rt = b[0] & b[1] & b[2] & b[3];
+	if (unlikely(rt != 0xffffffff))
+		return find_first_zero_bit(bitmap, MAX_RT_PRIO);
+
+	if (b[4] != ~0)
+		return ffz(b[4]) + MAX_RT_PRIO;
+	return ffz(b[5]) + 32 + MAX_RT_PRIO;
+}
+#undef MAX_PRIO
+#undef MAX_RT_PRIO
+#endif
 
+#endif /* __KERNEL__ */
 
 #endif /* _CRIS_BITOPS_H */
--- linux/include/asm-s390x/bitops.h.orig	Thu Jul 18 15:22:25 2002
+++ linux/include/asm-s390x/bitops.h	Thu Jul 18 15:22:35 2002
@@ -51,271 +51,220 @@
 extern const char _oi_bitmap[];
 extern const char _ni_bitmap[];
 extern const char _zb_findmap[];
+extern const char _sb_findmap[];
 
 #ifdef CONFIG_SMP
 /*
  * SMP save set_bit routine based on compare and swap (CS)
  */
-static __inline__ void set_bit_cs(unsigned long nr, volatile void * addr)
+static inline void set_bit_cs(unsigned long nr, volatile void *ptr)
 {
-        unsigned long bits, mask;
-        __asm__ __volatile__(
+        unsigned long addr, old, new, mask;
+
+	addr = (unsigned long) ptr;
 #if ALIGN_CS == 1
-             "   lghi  %2,7\n"         /* CS must be aligned on 4 byte b. */
-             "   ngr   %2,%1\n"        /* isolate last 2 bits of address */
-             "   xgr   %1,%2\n"        /* make addr % 4 == 0 */
-             "   sllg  %2,%2,3\n"
-             "   agr   %0,%2\n"        /* add alignement to bitnr */
+	addr ^= addr & 7;		/* align address to 8 */
+	nr += (addr & 7) << 3;		/* add alignment to bit number */
 #endif
-             "   lghi  %2,63\n"
-             "   nr    %2,%0\n"        /* make shift value */
-             "   xr    %0,%2\n"
-             "   srlg  %0,%0,3\n"
-             "   lghi  %3,1\n"
-             "   la    %1,0(%0,%1)\n"  /* calc. address for CS */
-             "   sllg  %3,%3,0(%2)\n"  /* make OR mask */
-             "   lg    %0,0(%1)\n"
-             "0: lgr   %2,%0\n"        /* CS loop starts here */
-             "   ogr   %2,%3\n"        /* set bit */
-             "   csg   %0,%2,0(%1)\n"
-             "   jl    0b"
-             : "+a" (nr), "+a" (addr), "=&a" (bits), "=&d" (mask) :
-             : "cc", "memory" );
+	addr += (nr ^ (nr & 63)) >> 3;	/* calculate address for CS */
+	mask = 1UL << (nr & 63);	/* make OR mask */
+	asm volatile(
+		"   lg   %0,0(%4)\n"
+		"0: lgr  %1,%0\n"
+		"   ogr  %1,%3\n"
+		"   csg  %0,%1,0(%4)\n"
+		"   jl   0b"
+		: "=&d" (old), "=&d" (new), "+m" (*(unsigned long *) addr)
+		: "d" (mask), "a" (addr) 
+		: "cc" );
 }
 
 /*
  * SMP save clear_bit routine based on compare and swap (CS)
  */
-static __inline__ void clear_bit_cs(unsigned long nr, volatile void * addr)
+static inline void clear_bit_cs(unsigned long nr, volatile void *ptr)
 {
-        unsigned long bits, mask;
-        __asm__ __volatile__(
+        unsigned long addr, old, new, mask;
+
+	addr = (unsigned long) ptr;
 #if ALIGN_CS == 1
-             "   lghi  %2,7\n"         /* CS must be aligned on 4 byte b. */
-             "   ngr   %2,%1\n"        /* isolate last 2 bits of address */
-             "   xgr   %1,%2\n"        /* make addr % 4 == 0 */
-             "   sllg  %2,%2,3\n"
-             "   agr   %0,%2\n"        /* add alignement to bitnr */
+	addr ^= addr & 7;		/* align address to 8 */
+	nr += (addr & 7) << 3;		/* add alignment to bit number */
 #endif
-             "   lghi  %2,63\n"
-             "   nr    %2,%0\n"        /* make shift value */
-             "   xr    %0,%2\n"
-             "   srlg  %0,%0,3\n"
-             "   lghi  %3,-2\n"
-             "   la    %1,0(%0,%1)\n"  /* calc. address for CS */
-             "   lghi  %3,-2\n"
-             "   rllg  %3,%3,0(%2)\n"  /* make AND mask */
-             "   lg    %0,0(%1)\n"
-             "0: lgr   %2,%0\n"        /* CS loop starts here */
-             "   ngr   %2,%3\n"        /* clear bit */
-             "   csg   %0,%2,0(%1)\n"
-             "   jl    0b"
-             : "+a" (nr), "+a" (addr), "=&a" (bits), "=&d" (mask) :
-             : "cc", "memory" );
+	addr += (nr ^ (nr & 63)) >> 3;	/* calculate address for CS */
+	mask = ~(1UL << (nr & 63));	/* make AND mask */
+	asm volatile(
+		"   lg   %0,0(%4)\n"
+		"0: lgr  %1,%0\n"
+		"   ngr  %1,%3\n"
+		"   csg  %0,%1,0(%4)\n"
+		"   jl   0b"
+		: "=&d" (old), "=&d" (new), "+m" (*(unsigned long *) addr)
+		: "d" (mask), "a" (addr) 
+		: "cc" );
 }
 
 /*
  * SMP save change_bit routine based on compare and swap (CS)
  */
-static __inline__ void change_bit_cs(unsigned long nr, volatile void * addr)
+static inline void change_bit_cs(unsigned long nr, volatile void *ptr)
 {
-        unsigned long bits, mask;
-        __asm__ __volatile__(
+        unsigned long addr, old, new, mask;
+
+	addr = (unsigned long) ptr;
 #if ALIGN_CS == 1
-             "   lghi  %2,7\n"         /* CS must be aligned on 4 byte b. */
-             "   ngr   %2,%1\n"        /* isolate last 2 bits of address */
-             "   xgr   %1,%2\n"        /* make addr % 4 == 0 */
-             "   sllg  %2,%2,3\n"
-             "   agr   %0,%2\n"        /* add alignement to bitnr */
+	addr ^= addr & 7;		/* align address to 8 */
+	nr += (addr & 7) << 3;		/* add alignment to bit number */
 #endif
-             "   lghi  %2,63\n"
-             "   nr    %2,%0\n"        /* make shift value */
-             "   xr    %0,%2\n"
-             "   srlg  %0,%0,3\n"
-             "   lghi  %3,1\n"
-             "   la    %1,0(%0,%1)\n"  /* calc. address for CS */
-             "   sllg  %3,%3,0(%2)\n"  /* make XR mask */
-             "   lg    %0,0(%1)\n"
-             "0: lgr   %2,%0\n"        /* CS loop starts here */
-             "   xgr   %2,%3\n"        /* change bit */
-             "   csg   %0,%2,0(%1)\n"
-             "   jl    0b"
-             : "+a" (nr), "+a" (addr), "=&a" (bits), "=&d" (mask) : 
-             : "cc", "memory" );
+	addr += (nr ^ (nr & 63)) >> 3;	/* calculate address for CS */
+	mask = 1UL << (nr & 63);	/* make XOR mask */
+	asm volatile(
+		"   lg   %0,0(%4)\n"
+		"0: lgr  %1,%0\n"
+		"   xgr  %1,%3\n"
+		"   csg  %0,%1,0(%4)\n"
+		"   jl   0b"
+		: "=&d" (old), "=&d" (new), "+m" (*(unsigned long *) addr)
+		: "d" (mask), "a" (addr) 
+		: "cc" );
 }
 
 /*
  * SMP save test_and_set_bit routine based on compare and swap (CS)
  */
-static __inline__ int 
-test_and_set_bit_cs(unsigned long nr, volatile void * addr)
+static inline int 
+test_and_set_bit_cs(unsigned long nr, volatile void *ptr)
 {
-        unsigned long bits, mask;
-        __asm__ __volatile__(
+        unsigned long addr, old, new, mask;
+
+	addr = (unsigned long) ptr;
 #if ALIGN_CS == 1
-             "   lghi  %2,7\n"         /* CS must be aligned on 4 byte b. */
-             "   ngr   %2,%1\n"        /* isolate last 2 bits of address */
-             "   xgr   %1,%2\n"        /* make addr % 4 == 0 */
-             "   sllg  %2,%2,3\n"
-             "   agr   %0,%2\n"        /* add alignement to bitnr */
+	addr ^= addr & 7;		/* align address to 8 */
+	nr += (addr & 7) << 3;		/* add alignment to bit number */
 #endif
-             "   lghi  %2,63\n"
-             "   nr    %2,%0\n"        /* make shift value */
-             "   xr    %0,%2\n"
-             "   srlg  %0,%0,3\n"
-             "   lghi  %3,1\n"
-             "   la    %1,0(%0,%1)\n"  /* calc. address for CS */
-             "   sllg  %3,%3,0(%2)\n"  /* make OR mask */
-             "   lg    %0,0(%1)\n"
-             "0: lgr   %2,%0\n"        /* CS loop starts here */
-             "   ogr   %2,%3\n"        /* set bit */
-             "   csg   %0,%2,0(%1)\n"
-             "   jl    0b\n"
-             "   ngr   %0,%3\n"        /* isolate old bit */
-             : "+a" (nr), "+a" (addr), "=&a" (bits), "=&d" (mask) :
-             : "cc", "memory" );
-        return nr != 0;
+	addr += (nr ^ (nr & 63)) >> 3;	/* calculate address for CS */
+	mask = 1UL << (nr & 63);	/* make OR/test mask */
+	asm volatile(
+		"   lg   %0,0(%4)\n"
+		"0: lgr  %1,%0\n"
+		"   ogr  %1,%3\n"
+		"   csg  %0,%1,0(%4)\n"
+		"   jl   0b"
+		: "=&d" (old), "=&d" (new), "+m" (*(unsigned long *) addr)
+		: "d" (mask), "a" (addr) 
+		: "cc" );
+	return (old & mask) != 0;
 }
 
 /*
  * SMP save test_and_clear_bit routine based on compare and swap (CS)
  */
-static __inline__ int
-test_and_clear_bit_cs(unsigned long nr, volatile void * addr)
+static inline int
+test_and_clear_bit_cs(unsigned long nr, volatile void *ptr)
 {
-        unsigned long bits, mask;
-        __asm__ __volatile__(
+        unsigned long addr, old, new, mask;
+
+	addr = (unsigned long) ptr;
 #if ALIGN_CS == 1
-             "   lghi  %2,7\n"         /* CS must be aligned on 4 byte b. */
-             "   ngr   %2,%1\n"        /* isolate last 2 bits of address */
-             "   xgr   %1,%2\n"        /* make addr % 4 == 0 */
-             "   sllg  %2,%2,3\n"
-             "   agr   %0,%2\n"        /* add alignement to bitnr */
+	addr ^= addr & 7;		/* align address to 8 */
+	nr += (addr & 7) << 3;		/* add alignment to bit number */
 #endif
-             "   lghi  %2,63\n"
-             "   nr    %2,%0\n"        /* make shift value */
-             "   xr    %0,%2\n"
-             "   srlg  %0,%0,3\n"
-             "   lghi  %3,-2\n"
-             "   la    %1,0(%0,%1)\n"  /* calc. address for CS */
-             "   rllg  %3,%3,0(%2)\n"  /* make AND mask */
-             "   lg    %0,0(%1)\n"
-             "0: lgr   %2,%0\n"        /* CS loop starts here */
-             "   ngr   %2,%3\n"        /* clear bit */
-             "   csg   %0,%2,0(%1)\n"
-             "   jl    0b\n"
-             "   xgr   %0,%2\n"        /* isolate old bit */
-             : "+a" (nr), "+a" (addr), "=&a" (bits), "=&d" (mask) :
-             : "cc", "memory" );
-        return nr != 0;
+	addr += (nr ^ (nr & 63)) >> 3;	/* calculate address for CS */
+	mask = ~(1UL << (nr & 63));	/* make AND mask */
+	asm volatile(
+		"   lg   %0,0(%4)\n"
+		"0: lgr  %1,%0\n"
+		"   ngr  %1,%3\n"
+		"   csg  %0,%1,0(%4)\n"
+		"   jl   0b"
+		: "=&d" (old), "=&d" (new), "+m" (*(unsigned long *) addr)
+		: "d" (mask), "a" (addr) 
+		: "cc" );
+	return (old ^ new) != 0;
 }
 
 /*
  * SMP save test_and_change_bit routine based on compare and swap (CS) 
  */
-static __inline__ int
-test_and_change_bit_cs(unsigned long nr, volatile void * addr)
+static inline int
+test_and_change_bit_cs(unsigned long nr, volatile void *ptr)
 {
-        unsigned long bits, mask;
-        __asm__ __volatile__(
+        unsigned long addr, old, new, mask;
+
+	addr = (unsigned long) ptr;
 #if ALIGN_CS == 1
-             "   lghi  %2,7\n"         /* CS must be aligned on 4 byte b. */
-             "   ngr   %2,%1\n"        /* isolate last 2 bits of address */
-             "   xgr   %1,%2\n"        /* make addr % 4 == 0 */
-             "   sllg  %2,%2,3\n"
-             "   agr   %0,%2\n"        /* add alignement to bitnr */
+	addr ^= addr & 7;		/* align address to 8 */
+	nr += (addr & 7) << 3;		/* add alignment to bit number */
 #endif
-             "   lghi  %2,63\n"
-             "   nr    %2,%0\n"        /* make shift value */
-             "   xr    %0,%2\n"
-             "   srlg  %0,%0,3\n"
-             "   lghi  %3,1\n"
-             "   la    %1,0(%0,%1)\n"  /* calc. address for CS */
-             "   sllg  %3,%3,0(%2)\n"  /* make OR mask */
-             "   lg    %0,0(%1)\n"
-             "0: lgr   %2,%0\n"        /* CS loop starts here */
-             "   xgr   %2,%3\n"        /* change bit */
-             "   csg   %0,%2,0(%1)\n"
-             "   jl    0b\n"
-             "   ngr   %0,%3\n"        /* isolate old bit */
-             : "+a" (nr), "+a" (addr), "=&a" (bits), "=&d" (mask) :
-             : "cc", "memory" );
-        return nr != 0;
+	addr += (nr ^ (nr & 63)) >> 3;	/* calculate address for CS */
+	mask = 1UL << (nr & 63);	/* make XOR mask */
+	asm volatile(
+		"   lg   %0,0(%4)\n"
+		"0: lgr  %1,%0\n"
+		"   xgr  %1,%3\n"
+		"   csg  %0,%1,0(%4)\n"
+		"   jl   0b"
+		: "=&d" (old), "=&d" (new), "+m" (*(unsigned long *) addr)
+		: "d" (mask), "a" (addr) 
+		: "cc" );
+	return (old & mask) != 0;
 }
 #endif /* CONFIG_SMP */
 
 /*
  * fast, non-SMP set_bit routine
  */
-static __inline__ void __set_bit(unsigned long nr, volatile void * addr)
+static inline void __set_bit(unsigned long nr, volatile void *ptr)
 {
-	unsigned long reg1, reg2;
-        __asm__ __volatile__(
-             "   lghi  %1,56\n"
-             "   lghi  %0,7\n"
-             "   xgr   %1,%2\n"
-             "   nr    %0,%2\n"
-             "   srlg  %1,%1,3\n"
-             "   la    %1,0(%1,%3)\n"
-             "   la    %0,0(%0,%4)\n"
-             "   oc    0(1,%1),0(%0)"
-             : "=&a" (reg1), "=&a" (reg2)
-             : "a" (nr), "a" (addr), "a" (&_oi_bitmap) : "cc", "memory" );
-}
-
-static __inline__ void 
-__constant_set_bit(const unsigned long nr, volatile void * addr)
-{
-  switch (nr&7) {
-  case 0:
-    __asm__ __volatile__ ("la 1,%0\n\t"
-                          "oi 0(1),0x01"
-                          : "=m" (*((volatile char *) addr + ((nr>>3)^7))) 
-                          : : "1", "cc", "memory");
-    break;
-  case 1:
-    __asm__ __volatile__ ("la 1,%0\n\t"
-                          "oi 0(1),0x02"
-                          : "=m" (*((volatile char *) addr + ((nr>>3)^7)))
-                          : : "1", "cc", "memory" );
-    break;
-  case 2:
-    __asm__ __volatile__ ("la 1,%0\n\t"
-                          "oi 0(1),0x04"
-                          : "=m" (*((volatile char *) addr + ((nr>>3)^7)))
-                          : : "1", "cc", "memory" );
-    break;
-  case 3:
-    __asm__ __volatile__ ("la 1,%0\n\t"
-                          "oi 0(1),0x08"
-                          : "=m" (*((volatile char *) addr + ((nr>>3)^7)))
-                          : : "1", "cc", "memory" );
-    break;
-  case 4:
-    __asm__ __volatile__ ("la 1,%0\n\t"
-                          "oi 0(1),0x10"
-                          : "=m" (*((volatile char *) addr + ((nr>>3)^7)))
-                          : : "1", "cc", "memory" );
-    break;
-  case 5:
-    __asm__ __volatile__ ("la 1,%0\n\t"
-                          "oi 0(1),0x20"
-                          : "=m" (*((volatile char *) addr + ((nr>>3)^7)))
-                          : : "1", "cc", "memory" );
-    break;
-  case 6:
-    __asm__ __volatile__ ("la 1,%0\n\t"
-                          "oi 0(1),0x40"
-                          : "=m" (*((volatile char *) addr + ((nr>>3)^7)))
-                          : : "1", "cc", "memory" );
-    break;
-  case 7:
-    __asm__ __volatile__ ("la 1,%0\n\t"
-                          "oi 0(1),0x80"
-                          : "=m" (*((volatile char *) addr + ((nr>>3)^7)))
-                          : : "1", "cc", "memory" );
-    break;
-  }
+	unsigned long addr;
+
+	addr = (unsigned long) ptr + ((nr ^ 56) >> 3);
+        asm volatile("oc 0(1,%1),0(%2)"
+		     : "+m" (*(char *) addr)
+		     : "a" (addr), "a" (_oi_bitmap + (nr & 7))
+		     : "cc" );
+}
+
+static inline void 
+__constant_set_bit(const unsigned long nr, volatile void *ptr)
+{
+	unsigned long addr;
+
+	addr = ((unsigned long) ptr) + ((nr >> 3) ^ 7);
+	switch (nr&7) {
+	case 0:
+		asm volatile ("oi 0(%1),0x01"
+			      : "+m" (*(char *) addr) : "a" (addr) : "cc" );
+		break;
+	case 1:
+		asm volatile ("oi 0(%1),0x02"
+			      : "+m" (*(char *) addr) : "a" (addr) : "cc" );
+		break;
+	case 2:
+		asm volatile ("oi 0(%1),0x04" 
+			      : "+m" (*(char *) addr) : "a" (addr) : "cc" );
+		break;
+	case 3:
+		asm volatile ("oi 0(%1),0x08" 
+			      : "+m" (*(char *) addr) : "a" (addr) : "cc" );
+		break;
+	case 4:
+		asm volatile ("oi 0(%1),0x10" 
+			      : "+m" (*(char *) addr) : "a" (addr) : "cc" );
+		break;
+	case 5:
+		asm volatile ("oi 0(%1),0x20" 
+			      : "+m" (*(char *) addr) : "a" (addr) : "cc" );
+		break;
+	case 6:
+		asm volatile ("oi 0(%1),0x40" 
+			      : "+m" (*(char *) addr) : "a" (addr) : "cc" );
+		break;
+	case 7:
+		asm volatile ("oi 0(%1),0x80" 
+			      : "+m" (*(char *) addr) : "a" (addr) : "cc" );
+		break;
+	}
 }
 
 #define set_bit_simple(nr,addr) \
@@ -326,76 +275,58 @@
 /*
  * fast, non-SMP clear_bit routine
  */
-static __inline__ void 
-__clear_bit(unsigned long nr, volatile void * addr)
+static inline void 
+__clear_bit(unsigned long nr, volatile void *ptr)
 {
-	unsigned long reg1, reg2;
-        __asm__ __volatile__(
-             "   lghi  %1,56\n"
-             "   lghi  %0,7\n"
-             "   xgr   %1,%2\n"
-             "   nr    %0,%2\n"
-             "   srlg  %1,%1,3\n"
-             "   la    %1,0(%1,%3)\n"
-             "   la    %0,0(%0,%4)\n"
-             "   nc    0(1,%1),0(%0)"
-             : "=&a" (reg1), "=&a" (reg2)
-	     : "d" (nr), "a" (addr), "a" (&_ni_bitmap) : "cc", "memory" );
-}
-
-static __inline__ void 
-__constant_clear_bit(const unsigned long nr, volatile void * addr)
-{
-  switch (nr&7) {
-  case 0:
-    __asm__ __volatile__ ("la 1,%0\n\t"
-                          "ni 0(1),0xFE"
-                          : "=m" (*((volatile char *) addr + ((nr>>3)^7)))
-                          : : "1", "cc", "memory" );
-    break;
-  case 1:
-    __asm__ __volatile__ ("la 1,%0\n\t"
-                          "ni 0(1),0xFD"
-                          : "=m" (*((volatile char *) addr + ((nr>>3)^7)))
-                          : : "1", "cc", "memory" );
-    break;
-  case 2:
-    __asm__ __volatile__ ("la 1,%0\n\t"
-                          "ni 0(1),0xFB"
-                          : "=m" (*((volatile char *) addr + ((nr>>3)^7)))
-                          : : "1", "cc", "memory" );
-    break;
-  case 3:
-    __asm__ __volatile__ ("la 1,%0\n\t"
-                          "ni 0(1),0xF7"
-                          : "=m" (*((volatile char *) addr + ((nr>>3)^7)))
-                          : : "1", "cc", "memory" );
-    break;
-  case 4:
-    __asm__ __volatile__ ("la 1,%0\n\t"
-                          "ni 0(1),0xEF"
-                          : "=m" (*((volatile char *) addr + ((nr>>3)^7)))
-                          : : "cc", "memory" );
-    break;
-  case 5:
-    __asm__ __volatile__ ("la 1,%0\n\t"
-                          "ni 0(1),0xDF"
-                          : "=m" (*((volatile char *) addr + ((nr>>3)^7)))
-                          : : "1", "cc", "memory" );
-    break;
-  case 6:
-    __asm__ __volatile__ ("la 1,%0\n\t"
-                          "ni 0(1),0xBF"
-                          : "=m" (*((volatile char *) addr + ((nr>>3)^7)))
-                          : : "1", "cc", "memory" );
-    break;
-  case 7:
-    __asm__ __volatile__ ("la 1,%0\n\t"
-                          "ni 0(1),0x7F"
-                          : "=m" (*((volatile char *) addr + ((nr>>3)^7)))
-                          : : "1", "cc", "memory" );
-    break;
-  }
+	unsigned long addr;
+
+	addr = (unsigned long) ptr + ((nr ^ 56) >> 3);
+        asm volatile("nc 0(1,%1),0(%2)"
+		     : "+m" (*(char *) addr)
+		     : "a" (addr), "a" (_ni_bitmap + (nr & 7))
+		     : "cc" );
+}
+
+static inline void 
+__constant_clear_bit(const unsigned long nr, volatile void *ptr)
+{
+	unsigned long addr;
+
+	addr = ((unsigned long) ptr) + ((nr >> 3) ^ 7);
+	switch (nr&7) {
+	case 0:
+		asm volatile ("ni 0(%1),0xFE" 
+			      : "+m" (*(char *) addr) : "a" (addr) : "cc" );
+		break;
+	case 1:
+		asm volatile ("ni 0(%1),0xFD"
+			      : "+m" (*(char *) addr) : "a" (addr) : "cc" );
+		break;
+	case 2:
+		asm volatile ("ni 0(%1),0xFB" 
+			      : "+m" (*(char *) addr) : "a" (addr) : "cc" );
+		break;
+	case 3:
+		asm volatile ("ni 0(%1),0xF7" 
+			      : "+m" (*(char *) addr) : "a" (addr) : "cc" );
+		break;
+	case 4:
+		asm volatile ("ni 0(%1),0xEF" 
+			      : "+m" (*(char *) addr) : "a" (addr) : "cc" );
+		break;
+	case 5:
+		asm volatile ("ni 0(%1),0xDF" 
+			      : "+m" (*(char *) addr) : "a" (addr) : "cc" );
+		break;
+	case 6:
+		asm volatile ("ni 0(%1),0xBF" 
+			      : "+m" (*(char *) addr) : "a" (addr) : "cc" );
+		break;
+	case 7:
+		asm volatile ("ni 0(%1),0x7F" 
+			      : "+m" (*(char *) addr) : "a" (addr) : "cc" );
+		break;
+	}
 }
 
 #define clear_bit_simple(nr,addr) \
@@ -406,75 +337,57 @@
 /* 
  * fast, non-SMP change_bit routine 
  */
-static __inline__ void __change_bit(unsigned long nr, volatile void * addr)
+static inline void __change_bit(unsigned long nr, volatile void *ptr)
 {
-	unsigned long reg1, reg2;
-        __asm__ __volatile__(
-             "   lghi  %1,56\n"
-             "   lghi  %0,7\n"
-             "   xgr   %1,%2\n"
-             "   nr    %0,%2\n"
-             "   srlg  %1,%1,3\n"
-             "   la    %1,0(%1,%3)\n"
-             "   la    %0,0(%0,%4)\n"
-             "   xc    0(1,%1),0(%0)"
-             : "=&a" (reg1), "=&a" (reg2)
-	     : "d" (nr), "a" (addr), "a" (&_oi_bitmap) : "cc", "memory" );
-}
-
-static __inline__ void 
-__constant_change_bit(const unsigned long nr, volatile void * addr) 
-{
-  switch (nr&7) {
-  case 0:
-    __asm__ __volatile__ ("la 1,%0\n\t"
-                          "xi 0(1),0x01"
-                          : "=m" (*((volatile char *) addr + ((nr>>3)^7)))
-                          : : "cc", "memory" );
-    break;
-  case 1:
-    __asm__ __volatile__ ("la 1,%0\n\t"
-                          "xi 0(1),0x02"
-                          : "=m" (*((volatile char *) addr + ((nr>>3)^7)))
-                          : : "cc", "memory" );
-    break;
-  case 2:
-    __asm__ __volatile__ ("la 1,%0\n\t"
-                          "xi 0(1),0x04"
-                          : "=m" (*((volatile char *) addr + ((nr>>3)^7)))
-                          : : "cc", "memory" );
-    break;
-  case 3:
-    __asm__ __volatile__ ("la 1,%0\n\t"
-                          "xi 0(1),0x08"
-                          : "=m" (*((volatile char *) addr + ((nr>>3)^7)))
-                          : : "cc", "memory" );
-    break;
-  case 4:
-    __asm__ __volatile__ ("la 1,%0\n\t"
-                          "xi 0(1),0x10"
-                          : "=m" (*((volatile char *) addr + ((nr>>3)^7)))
-                          : : "cc", "memory" );
-    break;
-  case 5:
-    __asm__ __volatile__ ("la 1,%0\n\t"
-                          "xi 0(1),0x20"
-                          : "=m" (*((volatile char *) addr + ((nr>>3)^7)))
-                          : : "1", "cc", "memory" );
-    break;
-  case 6:
-    __asm__ __volatile__ ("la 1,%0\n\t"
-                          "xi 0(1),0x40"
-                          : "=m" (*((volatile char *) addr + ((nr>>3)^7)))
-                          : : "1", "cc", "memory" );
-    break;
-  case 7:
-    __asm__ __volatile__ ("la 1,%0\n\t"
-                          "xi 0(1),0x80"
-                          : "=m" (*((volatile char *) addr + ((nr>>3)^7)))
-                          : : "1", "cc", "memory" );
-    break;
-  }
+	unsigned long addr;
+
+	addr = (unsigned long) ptr + ((nr ^ 56) >> 3);
+        asm volatile("xc 0(1,%1),0(%2)"
+		     :  "+m" (*(char *) addr)
+		     : "a" (addr), "a" (_oi_bitmap + (nr & 7))
+		     : "cc" );
+}
+
+static inline void 
+__constant_change_bit(const unsigned long nr, volatile void *ptr) 
+{
+	unsigned long addr;
+
+	addr = ((unsigned long) ptr) + ((nr >> 3) ^ 7);
+	switch (nr&7) {
+	case 0:
+		asm volatile ("xi 0(%1),0x01" 
+			      : "+m" (*(char *) addr) : "a" (addr) : "cc" );
+		break;
+	case 1:
+		asm volatile ("xi 0(%1),0x02" 
+			      : "+m" (*(char *) addr) : "a" (addr) : "cc" );
+		break;
+	case 2:
+		asm volatile ("xi 0(%1),0x04" 
+			      : "+m" (*(char *) addr) : "a" (addr) : "cc" );
+		break;
+	case 3:
+		asm volatile ("xi 0(%1),0x08" 
+			      : "+m" (*(char *) addr) : "a" (addr) : "cc" );
+		break;
+	case 4:
+		asm volatile ("xi 0(%1),0x10" 
+			      : "+m" (*(char *) addr) : "a" (addr) : "cc" );
+		break;
+	case 5:
+		asm volatile ("xi 0(%1),0x20" 
+			      : "+m" (*(char *) addr) : "a" (addr) : "cc" );
+		break;
+	case 6:
+		asm volatile ("xi 0(%1),0x40" 
+			      : "+m" (*(char *) addr) : "a" (addr) : "cc" );
+		break;
+	case 7:
+		asm volatile ("xi 0(%1),0x80"
+			      : "+m" (*(char *) addr) : "a" (addr) : "cc" );
+		break;
+	}
 }
 
 #define change_bit_simple(nr,addr) \
@@ -485,77 +398,57 @@
 /*
  * fast, non-SMP test_and_set_bit routine
  */
-static __inline__ int
-test_and_set_bit_simple(unsigned long nr, volatile void * addr)
+static inline int
+test_and_set_bit_simple(unsigned long nr, volatile void *ptr)
 {
-	unsigned long reg1, reg2;
-        int oldbit;
-        __asm__ __volatile__(
-             "   lghi  %1,56\n"
-             "   lghi  %2,7\n"
-             "   xgr   %1,%3\n"
-             "   nr    %2,%3\n"
-             "   srlg  %1,%1,3\n"
-             "   la    %1,0(%1,%4)\n"
-             "   ic    %0,0(%1)\n"
-             "   srl   %0,0(%2)\n"
-             "   la    %2,0(%2,%5)\n"
-             "   oc    0(1,%1),0(%2)"
-             : "=&d" (oldbit), "=&a" (reg1), "=&a" (reg2)
-	     : "d" (nr), "a" (addr), "a" (&_oi_bitmap) : "cc", "memory" );
-        return oldbit & 1;
+	unsigned long addr;
+	unsigned char ch;
+
+	addr = (unsigned long) ptr + ((nr ^ 56) >> 3);
+	ch = *(unsigned char *) addr;
+        asm volatile("oc 0(1,%1),0(%2)"
+		     : "+m" (*(char *) addr)
+		     : "a" (addr), "a" (_oi_bitmap + (nr & 7))
+		     : "cc" );
+	return (ch >> (nr & 7)) & 1;
 }
 #define __test_and_set_bit(X,Y)		test_and_set_bit_simple(X,Y)
 
 /*
  * fast, non-SMP test_and_clear_bit routine
  */
-static __inline__ int
-test_and_clear_bit_simple(unsigned long nr, volatile void * addr)
+static inline int
+test_and_clear_bit_simple(unsigned long nr, volatile void *ptr)
 {
-	unsigned long reg1, reg2;
-        int oldbit;
+	unsigned long addr;
+	unsigned char ch;
 
-        __asm__ __volatile__(
-             "   lghi  %1,56\n"
-             "   lghi  %2,7\n"
-             "   xgr   %1,%3\n"
-             "   nr    %2,%3\n"
-             "   srlg  %1,%1,3\n"
-             "   la    %1,0(%1,%4)\n"
-             "   ic    %0,0(%1)\n"
-             "   srl   %0,0(%2)\n"
-             "   la    %2,0(%2,%5)\n"
-             "   nc    0(1,%1),0(%2)"
-             : "=&d" (oldbit), "=&a" (reg1), "=&a" (reg2)
-	     : "d" (nr), "a" (addr), "a" (&_ni_bitmap) : "cc", "memory" );
-        return oldbit & 1;
+	addr = (unsigned long) ptr + ((nr ^ 56) >> 3);
+	ch = *(unsigned char *) addr;
+        asm volatile("nc 0(1,%1),0(%2)"
+		     : "+m" (*(char *) addr)
+		     : "a" (addr), "a" (_ni_bitmap + (nr & 7))
+		     : "cc" );
+	return (ch >> (nr & 7)) & 1;
 }
 #define __test_and_clear_bit(X,Y)	test_and_clear_bit_simple(X,Y)
 
 /*
  * fast, non-SMP test_and_change_bit routine
  */
-static __inline__ int
-test_and_change_bit_simple(unsigned long nr, volatile void * addr)
+static inline int
+test_and_change_bit_simple(unsigned long nr, volatile void *ptr)
 {
-	unsigned long reg1, reg2;
-        int oldbit;
+	unsigned long addr;
+	unsigned char ch;
 
-        __asm__ __volatile__(
-             "   lghi  %1,56\n"
-             "   lghi  %2,7\n"
-             "   xgr   %1,%3\n"
-             "   nr    %2,%3\n"
-             "   srlg  %1,%1,3\n"
-             "   la    %1,0(%1,%4)\n"
-             "   ic    %0,0(%1)\n"
-             "   srl   %0,0(%2)\n"
-             "   la    %2,0(%2,%5)\n"
-             "   xc    0(1,%1),0(%2)"
-             : "=&d" (oldbit), "=&a" (reg1), "=&a" (reg2)
-	     : "d" (nr), "a" (addr), "a" (&_oi_bitmap) : "cc", "memory" );
-        return oldbit & 1;
+	addr = (unsigned long) ptr + ((nr ^ 56) >> 3);
+	ch = *(unsigned char *) addr;
+        asm volatile("xc 0(1,%1),0(%2)"
+		     : "+m" (*(char *) addr)
+		     : "a" (addr), "a" (_oi_bitmap + (nr & 7))
+		     : "cc" );
+	return (ch >> (nr & 7)) & 1;
 }
 #define __test_and_change_bit(X,Y)	test_and_change_bit_simple(X,Y)
 
@@ -580,26 +473,18 @@
  * This routine doesn't need to be atomic.
  */
 
-static __inline__ int __test_bit(unsigned long nr, volatile void * addr)
+static inline int __test_bit(unsigned long nr, volatile void *ptr)
 {
-	unsigned long reg1, reg2;
-        int oldbit;
+	unsigned long addr;
+	unsigned char ch;
 
-        __asm__ __volatile__(
-             "   lghi  %2,56\n"
-             "   lghi  %1,7\n"
-             "   xgr   %2,%3\n"
-             "   nr    %1,%3\n"
-             "   srlg  %2,%2,3\n"
-             "   ic    %0,0(%2,%4)\n"
-             "   srl   %0,0(%1)\n"
-             : "=&d" (oldbit), "=&a" (reg1), "=&a" (reg2)
-	     : "d" (nr), "a" (addr) : "cc" );
-        return oldbit & 1;
+	addr = (unsigned long) ptr + ((nr ^ 56) >> 3);
+	ch = *(unsigned char *) addr;
+	return (ch >> (nr & 7)) & 1;
 }
 
-static __inline__ int 
-__constant_test_bit(unsigned long nr, volatile void * addr) {
+static inline int 
+__constant_test_bit(unsigned long nr, volatile void *addr) {
     return (((volatile char *) addr)[(nr>>3)^7] & (1<<(nr&7))) != 0;
 }
 
@@ -611,7 +496,7 @@
 /*
  * Find-bit routines..
  */
-static __inline__ unsigned long
+static inline unsigned long
 find_first_zero_bit(void * addr, unsigned long size)
 {
         unsigned long res, cmp, count;
@@ -653,7 +538,49 @@
         return (res < size) ? res : size;
 }
 
-static __inline__ unsigned long
+static inline unsigned long
+find_first_bit(void * addr, unsigned long size)
+{
+        unsigned long res, cmp, count;
+
+        if (!size)
+                return 0;
+        __asm__("   slgr  %1,%1\n"
+                "   lgr   %2,%3\n"
+                "   slgr  %0,%0\n"
+                "   aghi  %2,63\n"
+                "   srlg  %2,%2,6\n"
+                "0: cg    %1,0(%0,%4)\n"
+                "   jne   1f\n"
+                "   aghi  %0,8\n"
+                "   brct  %2,0b\n"
+                "   lgr   %0,%3\n"
+                "   j     5f\n"
+                "1: lg    %2,0(%0,%4)\n"
+                "   sllg  %0,%0,3\n"
+                "   clr   %2,%1\n"
+		"   jne   2f\n"
+		"   aghi  %0,32\n"
+                "   srlg  %2,%2,32\n"
+		"2: lghi  %1,0xff\n"
+                "   tmll  %2,0xffff\n"
+                "   jnz   3f\n"
+                "   aghi  %0,16\n"
+                "   srl   %2,16\n"
+                "3: tmll  %2,0x00ff\n"
+                "   jnz   4f\n"
+                "   aghi  %0,8\n"
+                "   srl   %2,8\n"
+                "4: ngr   %2,%1\n"
+                "   ic    %2,0(%2,%5)\n"
+                "   algr  %0,%2\n"
+                "5:"
+                : "=&a" (res), "=&d" (cmp), "=&a" (count)
+		: "a" (size), "a" (addr), "a" (&_sb_findmap) : "cc" );
+        return (res < size) ? res : size;
+}
+
+static inline unsigned long
 find_next_zero_bit (void * addr, unsigned long size, unsigned long offset)
 {
         unsigned long * p = ((unsigned long *) addr) + (offset >> 6);
@@ -697,14 +624,56 @@
         return (offset + res);
 }
 
+static inline unsigned long
+find_next_bit (void * addr, unsigned long size, unsigned long offset)
+{
+        unsigned long * p = ((unsigned long *) addr) + (offset >> 6);
+        unsigned long bitvec, reg;
+        unsigned long set, bit = offset & 63, res;
+
+        if (bit) {
+                /*
+                 * Look for zero in first word
+                 */
+                bitvec = (*p) >> bit;
+                __asm__("   slgr %0,%0\n"
+                        "   ltr  %1,%1\n"
+                        "   jnz  0f\n"
+                        "   aghi %0,32\n"
+                        "   srlg %1,%1,32\n"
+			"0: lghi %2,0xff\n"
+                        "   tmll %1,0xffff\n"
+                        "   jnz  1f\n"
+                        "   aghi %0,16\n"
+                        "   srlg %1,%1,16\n"
+                        "1: tmll %1,0x00ff\n"
+                        "   jnz  2f\n"
+                        "   aghi %0,8\n"
+                        "   srlg %1,%1,8\n"
+                        "2: ngr  %1,%2\n"
+                        "   ic   %1,0(%1,%3)\n"
+                        "   algr %0,%1"
+                        : "=&d" (set), "+a" (bitvec), "=&d" (reg)
+                        : "a" (&_sb_findmap) : "cc" );
+                if (set < (64 - bit))
+                        return set + offset;
+                offset += 64 - bit;
+                p++;
+        }
+        /*
+         * No set bit yet, search remaining full words for a bit
+         */
+        res = find_first_bit (p, size - 64 * (p - (unsigned long *) addr));
+        return (offset + res);
+}
+
 /*
  * ffz = Find First Zero in word. Undefined if no zero exists,
  * so code should check against ~0UL first..
  */
-static __inline__ unsigned long ffz(unsigned long word)
+static inline unsigned long ffz(unsigned long word)
 {
-	unsigned long reg;
-        int result;
+	unsigned long reg, result;
 
         __asm__("   lhi  %2,-1\n"
                 "   slgr %0,%0\n"
@@ -730,40 +699,112 @@
 }
 
 /*
+ * __ffs = find first bit in word. Undefined if no bit exists,
+ * so code should check against 0UL first..
+ */
+static inline unsigned long __ffs (unsigned long word)
+{
+        unsigned long reg, result;
+
+        __asm__("   slgr %0,%0\n"
+                "   ltr  %1,%1\n"
+                "   jnz  0f\n"
+                "   aghi %0,32\n"
+                "   srlg %1,%1,32\n"
+                "0: lghi %2,0xff\n"
+                "   tmll %1,0xffff\n"
+                "   jnz  1f\n"
+                "   aghi %0,16\n"
+                "   srlg %1,%1,16\n"
+                "1: tmll %1,0x00ff\n"
+                "   jnz  2f\n"
+                "   aghi %0,8\n"
+                "   srlg %1,%1,8\n"
+                "2: ngr  %1,%2\n"
+                "   ic   %1,0(%1,%3)\n"
+                "   algr %0,%1"
+                : "=&d" (result), "+a" (word), "=&d" (reg)
+                : "a" (&_sb_findmap) : "cc" );
+        return result;
+}
+
+/*
+ * Every architecture must define this function. It's the fastest
+ * way of searching a 140-bit bitmap where the first 100 bits are
+ * unlikely to be set. It's guaranteed that at least one of the 140
+ * bits is cleared.
+ */
+static inline int sched_find_first_bit(unsigned long *b)
+{
+	return find_first_bit(b, 140);
+}
+
+/*
  * ffs: find first bit set. This is defined the same way as
  * the libc and compiler builtin ffs routines, therefore
  * differs in spirit from the above ffz (man ffs).
  */
-
-extern int __inline__ ffs (int x)
+extern int inline ffs (int x)
 {
-        int r;
+        int r = 1;
 
         if (x == 0)
-          return 0;
-        __asm__("    slr  %0,%0\n"
-                "    tml  %1,0xffff\n"
+		return 0;
+        __asm__("    tml  %1,0xffff\n"
                 "    jnz  0f\n"
-                "    ahi  %0,16\n"
                 "    srl  %1,16\n"
+                "    ahi  %0,16\n"
                 "0:  tml  %1,0x00ff\n"
                 "    jnz  1f\n"
-                "    ahi  %0,8\n"
                 "    srl  %1,8\n"
+                "    ahi  %0,8\n"
                 "1:  tml  %1,0x000f\n"
                 "    jnz  2f\n"
-                "    ahi  %0,4\n"
                 "    srl  %1,4\n"
+                "    ahi  %0,4\n"
                 "2:  tml  %1,0x0003\n"
                 "    jnz  3f\n"
-                "    ahi  %0,2\n"
                 "    srl  %1,2\n"
+                "    ahi  %0,2\n"
                 "3:  tml  %1,0x0001\n"
                 "    jnz  4f\n"
                 "    ahi  %0,1\n"
                 "4:"
                 : "=&d" (r), "+d" (x) : : "cc" );
-        return r+1;
+        return r;
+}
+
+/*
+ * fls: find last bit set.
+ */
+extern __inline__ int fls(int x)
+{
+	int r = 32;
+
+	if (x == 0)
+		return 0;
+	__asm__("    tmh  %1,0xffff\n"
+		"    jz   0f\n"
+		"    sll  %1,16\n"
+		"    ahi  %0,-16\n"
+		"0:  tmh  %1,0xff00\n"
+		"    jz   1f\n"
+		"    sll  %1,8\n"
+		"    ahi  %0,-8\n"
+		"1:  tmh  %1,0xf000\n"
+		"    jz   2f\n"
+		"    sll  %1,4\n"
+		"    ahi  %0,-4\n"
+		"2:  tmh  %1,0xc000\n"
+		"    jz   3f\n"
+		"    sll  %1,2\n"
+		"    ahi  %0,-2\n"
+		"3:  tmh  %1,0x8000\n"
+		"    jz   4f\n"
+		"    ahi  %0,-1\n"
+		"4:"
+		: "+d" (r), "+d" (x) : : "cc" );
+	return r;
 }
 
 /*
@@ -791,7 +832,7 @@
 #define ext2_set_bit(nr, addr)       test_and_set_bit((nr)^56, addr)
 #define ext2_clear_bit(nr, addr)     test_and_clear_bit((nr)^56, addr)
 #define ext2_test_bit(nr, addr)      test_bit((nr)^56, addr)
-static __inline__ unsigned long
+static inline unsigned long
 ext2_find_first_zero_bit(void *vaddr, unsigned long size)
 {
         unsigned long res, cmp, count;
@@ -833,7 +874,7 @@
         return (res < size) ? res : size;
 }
 
-static __inline__ unsigned long
+static inline unsigned long
 ext2_find_next_zero_bit(void *vaddr, unsigned long size, unsigned long offset)
 {
         unsigned long *addr = vaddr;
--- linux/include/asm-ppc64/bitops.h.orig	Thu Jul 18 15:22:25 2002
+++ linux/include/asm-ppc64/bitops.h	Thu Jul 18 15:22:35 2002
@@ -33,7 +33,6 @@
 
 #ifdef __KERNEL__
 
-#include <asm/byteorder.h>
 #include <asm/memory.h>
 
 /*
@@ -42,12 +41,12 @@
 #define smp_mb__before_clear_bit()	smp_mb()
 #define smp_mb__after_clear_bit()	smp_mb()
 
-static __inline__ int test_bit(unsigned long nr, __const__ volatile void *addr)
+static __inline__ int test_bit(unsigned long nr, __const__ volatile unsigned long *addr)
 {
 	return (1UL & (((__const__ long *) addr)[nr >> 6] >> (nr & 63)));
 }
 
-static __inline__ void set_bit(unsigned long nr, volatile void *addr)
+static __inline__ void set_bit(unsigned long nr, volatile unsigned long *addr)
 {
 	unsigned long old;
 	unsigned long mask = 1UL << (nr & 0x3f);
@@ -63,7 +62,7 @@
 	: "cc");
 }
 
-static __inline__ void clear_bit(unsigned long nr, volatile void *addr)
+static __inline__ void clear_bit(unsigned long nr, volatile unsigned long *addr)
 {
 	unsigned long old;
 	unsigned long mask = 1UL << (nr & 0x3f);
@@ -79,7 +78,7 @@
 	: "cc");
 }
 
-static __inline__ void change_bit(unsigned long nr, volatile void *addr)
+static __inline__ void change_bit(unsigned long nr, volatile unsigned long *addr)
 {
 	unsigned long old;
 	unsigned long mask = 1UL << (nr & 0x3f);
@@ -95,7 +94,7 @@
 	: "cc");
 }
 
-static __inline__ int test_and_set_bit(unsigned long nr, volatile void *addr)
+static __inline__ int test_and_set_bit(unsigned long nr, volatile unsigned long *addr)
 {
 	unsigned long old, t;
 	unsigned long mask = 1UL << (nr & 0x3f);
@@ -115,7 +114,7 @@
 	return (old & mask) != 0;
 }
 
-static __inline__ int test_and_clear_bit(unsigned long nr, volatile void *addr)
+static __inline__ int test_and_clear_bit(unsigned long nr, volatile unsigned long *addr)
 {
 	unsigned long old, t;
 	unsigned long mask = 1UL << (nr & 0x3f);
@@ -135,7 +134,7 @@
 	return (old & mask) != 0;
 }
 
-static __inline__ int test_and_change_bit(unsigned long nr, volatile void *addr)
+static __inline__ int test_and_change_bit(unsigned long nr, volatile unsigned long *addr)
 {
 	unsigned long old, t;
 	unsigned long mask = 1UL << (nr & 0x3f);
@@ -158,7 +157,7 @@
 /*
  * non-atomic versions
  */
-static __inline__ void __set_bit(unsigned long nr, volatile void *addr)
+static __inline__ void __set_bit(unsigned long nr, volatile unsigned long *addr)
 {
 	unsigned long mask = 1UL << (nr & 0x3f);
 	unsigned long *p = ((unsigned long *)addr) + (nr >> 6);
@@ -166,7 +165,7 @@
 	*p |= mask;
 }
 
-static __inline__ void __clear_bit(unsigned long nr, volatile void *addr)
+static __inline__ void __clear_bit(unsigned long nr, volatile unsigned long *addr)
 {
 	unsigned long mask = 1UL << (nr & 0x3f);
 	unsigned long *p = ((unsigned long *)addr) + (nr >> 6);
@@ -174,7 +173,7 @@
 	*p &= ~mask;
 }
 
-static __inline__ void __change_bit(unsigned long nr, volatile void *addr)
+static __inline__ void __change_bit(unsigned long nr, volatile unsigned long *addr)
 {
 	unsigned long mask = 1UL << (nr & 0x3f);
 	unsigned long *p = ((unsigned long *)addr) + (nr >> 6);
@@ -182,7 +181,7 @@
 	*p ^= mask;
 }
 
-static __inline__ int __test_and_set_bit(unsigned long nr, volatile void *addr)
+static __inline__ int __test_and_set_bit(unsigned long nr, volatile unsigned long *addr)
 {
 	unsigned long mask = 1UL << (nr & 0x3f);
 	unsigned long *p = ((unsigned long *)addr) + (nr >> 6);
@@ -192,7 +191,7 @@
 	return (old & mask) != 0;
 }
 
-static __inline__ int __test_and_clear_bit(unsigned long nr, volatile void *addr)
+static __inline__ int __test_and_clear_bit(unsigned long nr, volatile unsigned long *addr)
 {
 	unsigned long mask = 1UL << (nr & 0x3f);
 	unsigned long *p = ((unsigned long *)addr) + (nr >> 6);
@@ -202,7 +201,7 @@
 	return (old & mask) != 0;
 }
 
-static __inline__ int __test_and_change_bit(unsigned long nr, volatile void *addr)
+static __inline__ int __test_and_change_bit(unsigned long nr, volatile unsigned long *addr)
 {
 	unsigned long mask = 1UL << (nr & 0x3f);
 	unsigned long *p = ((unsigned long *)addr) + (nr >> 6);
@@ -224,54 +223,29 @@
 	return 63 - lz;
 }
 
-/* Return the zero-based bit position
- *  from RIGHT TO LEFT  63 --> 0
- *   of the most significant (left-most) 1-bit in an 8-byte area.
- */
-static __inline__ long cnt_trailing_zeros(unsigned long mask)
-{
-        long cnt;
-
-	asm(
-"	addi	%0,%1,-1	\n\
-	andc	%0,%0,%1	\n\
-	cntlzd	%0,%0		\n\
-	subfic	%0,%0,64"
-	: "=r" (cnt)
-	: "r" (mask));
-	return cnt;
-}
-
-
-
 /*
- * ffz = Find First Zero in word. Undefined if no zero exists,
- *    Determines the bit position of the LEAST significant
- *    (rightmost) 0 bit in the specified DOUBLE-WORD.
- *    The returned bit position will be zero-based, starting
- *    from the right side (63 - 0).
- *    the code should check against ~0UL first..
+ * Determines the bit position of the least significant (rightmost) 0 bit
+ * in the specified double word. The returned bit position will be zero-based,
+ * starting from the right side (63 - 0).
  */
 static __inline__ unsigned long ffz(unsigned long x)
 {
-	u32  tempRC;
-
-	/* Change all of x's 1s to 0s and 0s to 1s in x.
-	 * And insure at least 1 zero exists in the 8 byte area.
-	 */
+	/* no zero exists anywhere in the 8 byte area. */
 	if ((x = ~x) == 0)
-		/* no zero exists anywhere in the 8 byte area. */
 		return 64;
 
-	/* Calculate the bit position of the least significant '1' bit in x
-	 * (since x has been changed this will actually be the least
-	 * significant '0' bit in the original x).
-	 * Note: (x & -x) gives us a mask that is the LEAST significant
-	 * (RIGHT-most) 1-bit of the value in x.
+	/*
+	 * Calculate the bit position of the least signficant '1' bit in x
+	 * (since x has been changed this will actually be the least signficant
+	 * '0' bit in * the original x).  Note: (x & -x) gives us a mask that
+	 * is the least significant * (RIGHT-most) 1-bit of the value in x.
 	 */
-	tempRC = __ilog2(x & -x);
+	return __ilog2(x & -x);
+}
 
-	return tempRC;
+static __inline__ int __ffs(unsigned long x)
+{
+	return __ilog2(x & -x);
 }
 
 /*
@@ -281,8 +255,8 @@
  */
 static __inline__ int ffs(int x)
 {
-	int result = ffz(~x);
-	return x ? result+1 : 0;
+	unsigned long i = (unsigned long)x;
+	return __ilog2(i & -i) + 1;
 }
 
 /*
@@ -293,139 +267,82 @@
 #define hweight16(x) generic_hweight16(x)
 #define hweight8(x) generic_hweight8(x)
 
-extern unsigned long find_next_zero_bit(void * addr, unsigned long size,
-					unsigned long offset);
-/*
- * The optimizer actually does good code for this case..
- */
-#define find_first_zero_bit(addr, size) find_next_zero_bit((addr), (size), 0)
+extern unsigned long find_next_zero_bit(unsigned long *addr, unsigned long size, unsigned long offset);
+#define find_first_zero_bit(addr, size) \
+	find_next_zero_bit((addr), (size), 0)
+
+extern unsigned long find_next_bit(unsigned long *addr, unsigned long size, unsigned long offset);
+#define find_first_bit(addr, size) \
+	find_next_bit((addr), (size), 0)
+
+extern unsigned long find_next_zero_le_bit(unsigned long *addr, unsigned long size, unsigned long offset);
+#define find_first_zero_le_bit(addr, size) \
+	find_next_zero_le_bit((addr), (size), 0)
 
-/* Bitmap functions for the ext2 filesystem. */
-#define _EXT2_HAVE_ASM_BITOPS_
-
-static __inline__ int ext2_set_bit(int nr, void* addr)
+static __inline__ int test_le_bit(unsigned long nr, __const__ unsigned long * addr)
 {
-	/* This method needs to take into account the fact that the ext2 file system represents
-	 *  it's bitmaps as "little endian" unsigned integers.
-	 * Note: this method is not atomic, but ext2 does not need it to be.
-	 */
-	int mask;
-	int oldbit;
-	unsigned char* ADDR = (unsigned char*) addr;
-
-	/* Determine the BYTE containing the specified bit
-	 * (nr) - important as if we go to a byte there are no
-	 * little endian concerns.
-	 */
-	ADDR += nr >> 3;
-	mask = 1 << (nr & 0x07);  /* Create a mask to the bit within this byte. */
-	oldbit = *ADDR & mask;  /* Save the bit's previous value. */
-	*ADDR |= mask;  /* Turn the bit on. */
-	return oldbit;  /* Return the bit's previous value. */
+	__const__ unsigned char	*ADDR = (__const__ unsigned char *) addr;
+	return (ADDR[nr >> 3] >> (nr & 7)) & 1;
 }
 
-static __inline__ int ext2_clear_bit(int nr, void* addr)
+/*
+ * non-atomic versions
+ */
+static __inline__ void __set_le_bit(unsigned long nr, unsigned long *addr)
 {
-	/* This method needs to take into account the fact that the ext2 file system represents
-	 * | it's bitmaps as "little endian" unsigned integers.
-	 * Note: this method is not atomic, but ext2 does not need it to be.
-	 */
-        int     mask;
-        int oldbit;
-        unsigned char* ADDR = (unsigned char*) addr;
-
-	/* Determine the BYTE containing the specified bit (nr)
-	 *  - important as if we go to a byte there are no little endian concerns.
-	 */
-        ADDR += nr >> 3;
-        mask = 1 << (nr & 0x07);  /* Create a mask to the bit within this byte. */
-        oldbit = *ADDR & mask;  /* Save the bit's previous value. */
-        *ADDR = *ADDR & ~mask;  /* Turn the bit off. */
-        return oldbit;  /* Return the bit's previous value. */
-}
+	unsigned char *ADDR = (unsigned char *)addr;
 
-static __inline__ int ext2_test_bit(int nr, __const__ void * addr)
-{
-	/* This method needs to take into account the fact that the ext2 file system represents
-	 * | it's bitmaps as "little endian" unsigned integers.
-	 * Determine the BYTE containing the specified bit (nr),
-	 *   then shift to the right the correct number of bits and return that bit's value.
-	 */
-	__const__ unsigned char	*ADDR = (__const__ unsigned char *) addr;
-	return (ADDR[nr >> 3] >> (nr & 7)) & 1;
+	ADDR += nr >> 3;
+	*ADDR |= 1 << (nr & 0x07);
 }
 
-/* Returns the bit position of the most significant 1 bit in a WORD. */
-static __inline__ int ext2_ilog2(unsigned int x)
+static __inline__ void __clear_le_bit(unsigned long nr, unsigned long *addr)
 {
-        int lz;
+	unsigned char *ADDR = (unsigned char *)addr;
 
-        asm ("cntlzw %0,%1" : "=r" (lz) : "r" (x));
-        return 31 - lz;
+	ADDR += nr >> 3;
+	*ADDR &= ~(1 << (nr & 0x07));
 }
 
-/* ext2_ffz = ext2's Find First Zero.
- *    Determines the bit position of the LEAST significant (rightmost) 0 bit in the specified WORD.
- *    The returned bit position will be zero-based, starting from the right side (31 - 0).
- */
-static __inline__ int ext2_ffz(unsigned int x)
+static __inline__ int __test_and_set_le_bit(unsigned long nr, unsigned long *addr)
 {
-	u32  tempRC;
-	/* Change all of x's 1s to 0s and 0s to 1s in x.  And insure at least 1 zero exists in the word. */
-	if ((x = ~x) == 0)
-		/* no zero exists anywhere in the 4 byte area. */
-		return 32;
-	/* Calculate the bit position of the least significant '1' bit in x
-	 * (since x has been changed this will actually be the least
-	 * significant '0' bit in the original x).
-	 * Note: (x & -x) gives us a mask that is the LEAST significant
-	 * (RIGHT-most) 1-bit of the value in x.
-	 */
-	tempRC = ext2_ilog2(x & -x);
-	return tempRC;
+	int mask, retval;
+	unsigned char *ADDR = (unsigned char *)addr;
+
+	ADDR += nr >> 3;
+	mask = 1 << (nr & 0x07);
+	retval = (mask & *ADDR) != 0;
+	*ADDR |= mask;
+	return retval;
 }
 
-static __inline__ u32 ext2_find_next_zero_bit(void* addr, u32 size, u32 offset)
+static __inline__ int __test_and_clear_le_bit(unsigned long nr, unsigned long *addr)
 {
-	/* This method needs to take into account the fact that the ext2 file system represents
-	 * | it's bitmaps as "little endian" unsigned integers.
-	 */
-        unsigned int *p = ((unsigned int *) addr) + (offset >> 5);
-        unsigned int result = offset & ~31;
-        unsigned int tmp;
-
-        if (offset >= size)
-                return size;
-        size -= result;
-        offset &= 31;
-        if (offset) {
-                tmp = cpu_to_le32p(p++);
-                tmp |= ~0U >> (32-offset); /* bug or feature ? */
-                if (size < 32)
-                        goto found_first;
-                if (tmp != ~0)
-                        goto found_middle;
-                size -= 32;
-                result += 32;
-        }
-        while (size >= 32) {
-                if ((tmp = cpu_to_le32p(p++)) != ~0)
-                        goto found_middle;
-                result += 32;
-                size -= 32;
-        }
-        if (!size)
-                return result;
-        tmp = cpu_to_le32p(p);
-found_first:
-        tmp |= ~0 << size;
-        if (tmp == ~0)          /* Are any bits zero? */
-                return result + size; /* Nope. */
-found_middle:
-        return result + ext2_ffz(tmp);
-}
+	int mask, retval;
+	unsigned char *ADDR = (unsigned char *)addr;
 
-#define ext2_find_first_zero_bit(addr, size) ext2_find_next_zero_bit((addr), (size), 0)
+	ADDR += nr >> 3;
+	mask = 1 << (nr & 0x07);
+	retval = (mask & *ADDR) != 0;
+	*ADDR &= ~mask;
+	return retval;
+}
+
+#define ext2_set_bit(nr,addr) \
+	__test_and_set_le_bit((nr),(unsigned long*)addr)
+#define ext2_clear_bit(nr, addr) \
+	__test_and_clear_le_bit((nr),(unsigned long*)addr)
+#define ext2_test_bit(nr, addr)      test_le_bit((nr),(unsigned long*)addr)
+#define ext2_find_first_zero_bit(addr, size) \
+	find_first_zero_le_bit((unsigned long*)addr, size)
+#define ext2_find_next_zero_bit(addr, size, off) \
+	find_next_zero_le_bit((unsigned long*)addr, size, off)
+
+#define minix_test_and_set_bit(nr,addr)		test_and_set_bit(nr,addr)
+#define minix_set_bit(nr,addr)			set_bit(nr,addr)
+#define minix_test_and_clear_bit(nr,addr)	test_and_clear_bit(nr,addr)
+#define minix_test_bit(nr,addr)			test_bit(nr,addr)
+#define minix_find_first_zero_bit(addr,size)	find_first_zero_bit(addr,size)
 
 #endif /* __KERNEL__ */
 #endif /* _PPC64_BITOPS_H */
--- linux/net/unix/af_unix.c.orig	Thu Jul 18 15:22:25 2002
+++ linux/net/unix/af_unix.c	Thu Jul 18 15:22:35 2002
@@ -565,10 +565,8 @@
 				      addr->hash)) {
 		write_unlock(&unix_table_lock);
 		/* Sanity yield. It is unusual case, but yet... */
-		if (!(ordernum&0xFF)) {
-			current->policy |= SCHED_YIELD;
-			schedule();
-		}
+		if (!(ordernum&0xFF))
+			yield();
 		goto retry;
 	}
 	addr->hash ^= sk->type;
--- linux/net/ipv4/tcp_output.c.orig	Thu Jul 18 15:22:25 2002
+++ linux/net/ipv4/tcp_output.c	Thu Jul 18 15:22:35 2002
@@ -1010,8 +1010,7 @@
 			skb = alloc_skb(MAX_TCP_HEADER, GFP_KERNEL);
 			if (skb)
 				break;
-			current->policy |= SCHED_YIELD;
-			schedule();
+			yield();
 		}
 
 		/* Reserve space for headers and prepare control bits. */
--- linux/net/sunrpc/sched.c.orig	Thu Jul 18 15:22:25 2002
+++ linux/net/sunrpc/sched.c	Thu Jul 18 15:22:35 2002
@@ -770,8 +770,7 @@
 		}
 		if (flags & RPC_TASK_ASYNC)
 			return NULL;
-		current->policy |= SCHED_YIELD;
-		schedule();
+		yield();
 	} while (!signalled());
 
 	return NULL;
@@ -1112,8 +1111,7 @@
 		__rpc_schedule();
 		if (all_tasks) {
 			dprintk("rpciod_killall: waiting for tasks to exit\n");
-			current->policy |= SCHED_YIELD;
-			schedule();
+			yield();
 		}
 	}
 
@@ -1183,8 +1181,7 @@
 	 * wait briefly before checking the process id.
 	 */
 	current->sigpending = 0;
-	current->policy |= SCHED_YIELD;
-	schedule();
+	yield();
 	/*
 	 * Display a message if we're going to wait longer.
 	 */
--- linux/net/sched/sch_generic.c.orig	Thu Jul 18 15:22:25 2002
+++ linux/net/sched/sch_generic.c	Thu Jul 18 15:22:35 2002
@@ -475,10 +475,8 @@
 
 	dev_watchdog_down(dev);
 
-	while (test_bit(__LINK_STATE_SCHED, &dev->state)) {
-		current->policy |= SCHED_YIELD;
-		schedule();
-	}
+	while (test_bit(__LINK_STATE_SCHED, &dev->state))
+		yield();
 
 	spin_unlock_wait(&dev->xmit_lock);
 }
--- linux/net/socket.c.orig	Thu Jul 18 15:22:25 2002
+++ linux/net/socket.c	Thu Jul 18 15:22:35 2002
@@ -147,8 +147,7 @@
 	while (atomic_read(&net_family_lockct) != 0) {
 		spin_unlock(&net_family_lock);
 
-		current->policy |= SCHED_YIELD;
-		schedule();
+		yield();
 
 		spin_lock(&net_family_lock);
 	}
--- linux/drivers/net/slip.c.orig	Thu Jul 18 15:22:12 2002
+++ linux/drivers/net/slip.c	Thu Jul 18 15:22:35 2002
@@ -1393,10 +1393,8 @@
 		/* First of all: check for active disciplines and hangup them.
 		 */
 		do {
-			if (busy) {
-				current->counter = 0;
-				schedule();
-			}
+			if (busy)
+				sys_sched_yield();
 
 			busy = 0;
 			local_bh_disable();
--- linux/drivers/block/loop.c.orig	Thu Jul 18 15:22:20 2002
+++ linux/drivers/block/loop.c	Thu Jul 18 15:22:35 2002
@@ -573,9 +573,6 @@
 	flush_signals(current);
 	spin_unlock_irq(&current->sigmask_lock);
 
-	current->policy = SCHED_OTHER;
-	current->nice = -20;
-
 	spin_lock_irq(&lo->lo_lock);
 	lo->lo_state = Lo_bound;
 	atomic_inc(&lo->lo_pending);
--- linux/drivers/char/mwave/mwavedd.c.orig	Thu Jul 18 15:22:12 2002
+++ linux/drivers/char/mwave/mwavedd.c	Thu Jul 18 15:22:35 2002
@@ -279,7 +279,6 @@
 			pDrvData->IPCs[ipcnum].bIsHere = FALSE;
 			pDrvData->IPCs[ipcnum].bIsEnabled = TRUE;
 	#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,4,0)
-			current->nice = -20;	/* boost to provide priority timing */
 	#else
 			current->priority = 0x28;	/* boost to provide priority timing */
 	#endif
--- linux/drivers/char/drm-4.0/ffb_drv.c.orig	Thu Jul 18 15:22:12 2002
+++ linux/drivers/char/drm-4.0/ffb_drv.c	Thu Jul 18 15:22:35 2002
@@ -710,8 +710,7 @@
 		/* Contention */
 		atomic_inc(&dev->total_sleeps);
 		current->state = TASK_INTERRUPTIBLE;
-		current->policy |= SCHED_YIELD;
-		schedule();
+		yield();
 		if (signal_pending(current)) {
 			ret = -ERESTARTSYS;
 			break;
--- linux/drivers/char/drm-4.0/tdfx_drv.c.orig	Thu Jul 18 15:22:12 2002
+++ linux/drivers/char/drm-4.0/tdfx_drv.c	Thu Jul 18 15:22:35 2002
@@ -554,7 +554,6 @@
 					lock.context, current->pid, j,
 					dev->lock.lock_time, jiffies);
                                 current->state = TASK_INTERRUPTIBLE;
-				current->policy |= SCHED_YIELD;
                                 schedule_timeout(DRM_LOCK_SLICE-j);
 				DRM_DEBUG("jiffies=%d\n", jiffies);
                         }
@@ -578,10 +577,7 @@
 
                                 /* Contention */
                         atomic_inc(&dev->total_sleeps);
-#if 1
-			current->policy |= SCHED_YIELD;
-#endif
-                        schedule();
+			yield();
                         if (signal_pending(current)) {
                                 ret = -ERESTARTSYS;
                                 break;
@@ -604,8 +600,7 @@
                    when dev->last_context == lock.context
                    NOTE WE HOLD THE LOCK THROUGHOUT THIS
                    TIME! */
-		current->policy |= SCHED_YIELD;
-	        schedule();
+		yield();
 	        current->state = TASK_RUNNING;
 	        remove_wait_queue(&dev->context_wait, &entry);
 	        if (signal_pending(current)) {
--- linux/drivers/char/serial_txx927.c.orig	Thu Jul 18 15:22:20 2002
+++ linux/drivers/char/serial_txx927.c	Thu Jul 18 15:22:35 2002
@@ -1533,7 +1533,6 @@
 		printk("cisr = %d (jiff=%lu)...", cisr, jiffies);
 #endif
 		current->state = TASK_INTERRUPTIBLE;
-		current->counter = 0;	/* make us low-priority */
 		schedule_timeout(char_time);
 		if (signal_pending(current))
 			break;
--- linux/drivers/ide/ataraid.c.orig	Thu Jul 18 15:22:01 2002
+++ linux/drivers/ide/ataraid.c	Thu Jul 18 15:22:35 2002
@@ -123,8 +123,7 @@
 		ptr=kmalloc(sizeof(struct buffer_head),GFP_NOIO);
 		if (!ptr) {
 			__set_current_state(TASK_RUNNING);
-	                current->policy |= SCHED_YIELD;
-	                schedule();             
+			yield();
 		}
 	}
 	return ptr;
@@ -139,8 +138,7 @@
 		ptr=kmalloc(sizeof(struct ataraid_bh_private),GFP_NOIO);
 		if (!ptr) {
 			__set_current_state(TASK_RUNNING);
-	                current->policy |= SCHED_YIELD;
-	                schedule();             
+			yield();
 		}
 	}
 	return ptr;
--- linux/drivers/md/md.c.orig	Thu Jul 18 15:22:21 2002
+++ linux/drivers/md/md.c	Thu Jul 18 15:22:35 2002
@@ -2936,8 +2936,6 @@
 	 * bdflush, otherwise bdflush will deadlock if there are too
 	 * many dirty RAID5 blocks.
 	 */
-	current->policy = SCHED_OTHER;
-	current->nice = -20;
 	md_unlock_kernel();
 
 	complete(thread->event);
@@ -3391,11 +3389,6 @@
 	       "(but not more than %d KB/sec) for reconstruction.\n",
 	       sysctl_speed_limit_max);
 
-	/*
-	 * Resync has low priority.
-	 */
-	current->nice = 19;
-
 	is_mddev_idle(mddev); /* this also initializes IO event counters */
 	for (m = 0; m < SYNC_MARKS; m++) {
 		mark[m] = jiffies;
@@ -3473,16 +3466,13 @@
 		currspeed = (j-mddev->resync_mark_cnt)/2/((jiffies-mddev->resync_mark)/HZ +1) +1;
 
 		if (currspeed > sysctl_speed_limit_min) {
-			current->nice = 19;
-
 			if ((currspeed > sysctl_speed_limit_max) ||
 					!is_mddev_idle(mddev)) {
 				current->state = TASK_INTERRUPTIBLE;
 				md_schedule_timeout(HZ/4);
 				goto repeat;
 			}
-		} else
-			current->nice = -20;
+		}
 	}
 	printk(KERN_INFO "md: md%d: sync done.\n",mdidx(mddev));
 	err = 0;
--- linux/arch/i386/mm/fault.c.orig	Thu Jul 18 15:22:17 2002
+++ linux/arch/i386/mm/fault.c	Thu Jul 18 15:22:35 2002
@@ -86,8 +86,7 @@
 
 out_of_memory:
 	if (current->pid == 1) {
-		current->policy |= SCHED_YIELD;
-		schedule();
+		yield();
 		goto survive;
 	}
 	goto bad_area;
@@ -336,8 +335,7 @@
 out_of_memory:
 	up_read(&mm->mmap_sem);
 	if (tsk->pid == 1) {
-		tsk->policy |= SCHED_YIELD;
-		schedule();
+		yield();
 		down_read(&mm->mmap_sem);
 		goto survive;
 	}
--- linux/arch/i386/kernel/entry.S.orig	Thu Jul 18 15:22:17 2002
+++ linux/arch/i386/kernel/entry.S	Thu Jul 18 15:22:35 2002
@@ -77,7 +77,7 @@
 exec_domain	= 16
 need_resched	= 20
 tsk_ptrace	= 24
-processor	= 52
+cpu		= 32
 
 ENOSYS = 38
 
@@ -176,9 +176,11 @@
 
 
 ENTRY(ret_from_fork)
+#if CONFIG_SMP
 	pushl %ebx
 	call SYMBOL_NAME(schedule_tail)
 	addl $4, %esp
+#endif
 	GET_CURRENT(%ebx)
 	testb $0x02,tsk_ptrace(%ebx)	# PT_TRACESYS
 	jne tracesys_exit
@@ -256,7 +258,16 @@
 
 	ALIGN
 reschedule:
-	call SYMBOL_NAME(schedule)    # test
+	movl EFLAGS(%esp),%eax		# mix EFLAGS and CS
+	movb CS(%esp),%al
+	testl $(VM_MASK | 3),%eax	# return to VM86 mode or non-supervisor?
+	jne userspace_resched
+
+	call SYMBOL_NAME(schedule)
+	jmp ret_from_sys_call
+
+userspace_resched:
+	call SYMBOL_NAME(schedule_userspace)
 	jmp ret_from_sys_call
 
 ENTRY(divide_error)
@@ -637,8 +648,8 @@
  	.long SYMBOL_NAME(sys_tkill)
 	.long SYMBOL_NAME(sys_ni_syscall)	/* reserved for sendfile64 */
 	.long SYMBOL_NAME(sys_ni_syscall)	/* 240 reserved for futex */
-	.long SYMBOL_NAME(sys_ni_syscall)	/* reserved for sched_setaffinity */
-	.long SYMBOL_NAME(sys_ni_syscall)	/* reserved for sched_getaffinity */
+	.long SYMBOL_NAME(sys_sched_setaffinity)
+	.long SYMBOL_NAME(sys_sched_getaffinity)
 
 	.rept NR_syscalls-(.-sys_call_table)/4
 		.long SYMBOL_NAME(sys_ni_syscall)
--- linux/arch/i386/kernel/process.c.orig	Thu Jul 18 15:22:17 2002
+++ linux/arch/i386/kernel/process.c	Thu Jul 18 15:22:35 2002
@@ -124,15 +124,12 @@
 void cpu_idle (void)
 {
 	/* endless idle loop with no priority at all */
-	init_idle();
-	current->nice = 20;
-	current->counter = -100;
 
 	while (1) {
 		void (*idle)(void) = pm_idle;
 		if (!idle)
 			idle = default_idle;
-		while (!current->need_resched)
+		if (!current->need_resched)
 			idle();
 		schedule();
 		check_pgt_cache();
@@ -697,15 +694,17 @@
 	asm volatile("movl %%gs,%0":"=m" (*(int *)&prev->gs));
 
 	/*
-	 * Restore %fs and %gs.
+	 * Restore %fs and %gs if needed.
 	 */
-	loadsegment(fs, next->fs);
-	loadsegment(gs, next->gs);
+	if (unlikely(prev->fs | prev->gs | next->fs | next->gs)) {
+		loadsegment(fs, next->fs);
+		loadsegment(gs, next->gs);
+	}
 
 	/*
 	 * Now maybe reload the debug registers
 	 */
-	if (next->debugreg[7]){
+	if (unlikely(next->debugreg[7])) {
 		loaddebug(next, 0);
 		loaddebug(next, 1);
 		loaddebug(next, 2);
@@ -715,7 +714,7 @@
 		loaddebug(next, 7);
 	}
 
-	if (prev->ioperm || next->ioperm) {
+	if (unlikely(prev->ioperm || next->ioperm)) {
 		if (next->ioperm) {
 			/*
 			 * 4 cachelines copy ... not good, but not that
--- linux/arch/i386/kernel/setup.c.orig	Thu Jul 18 15:22:17 2002
+++ linux/arch/i386/kernel/setup.c	Thu Jul 18 15:22:35 2002
@@ -3131,9 +3131,10 @@
 	load_TR(nr);
 	load_LDT(&init_mm);
 
-	/*
-	 * Clear all 6 debug registers:
-	 */
+	/* Clear %fs and %gs. */
+	asm volatile ("xorl %eax, %eax; movl %eax, %fs; movl %eax, %gs");
+
+	/* Clear all 6 debug registers: */
 
 #define CD(register) __asm__("movl %0,%%db" #register ::"r"(0) );
 
--- linux/arch/i386/kernel/smp.c.orig	Thu Jul 18 15:22:17 2002
+++ linux/arch/i386/kernel/smp.c	Thu Jul 18 15:22:35 2002
@@ -493,10 +493,20 @@
  * it goes straight through and wastes no time serializing
  * anything. Worst case is that we lose a reschedule ...
  */
-
 void smp_send_reschedule(int cpu)
 {
 	send_IPI_mask(1 << cpu, RESCHEDULE_VECTOR);
+}
+
+/*
+ * this function sends a reschedule IPI to all (other) CPUs.
+ * This should only be used if some 'global' task became runnable,
+ * such as a RT task, that must be handled now. The first CPU
+ * that manages to grab the task will run it.
+ */
+void smp_send_reschedule_all(void)
+{
+	send_IPI_allbutself(RESCHEDULE_VECTOR);
 }
 
 /*
--- linux/arch/i386/kernel/smpboot.c.orig	Thu Jul 18 15:22:17 2002
+++ linux/arch/i386/kernel/smpboot.c	Thu Jul 18 15:22:35 2002
@@ -308,14 +308,14 @@
 			if (tsc_values[i] < avg)
 				realdelta = -realdelta;
 
-			printk("BIOS BUG: CPU#%d improperly initialized, has %ld usecs TSC skew! FIXED.\n",
-				i, realdelta);
+			printk("BIOS BUG: CPU#%d improperly initialized, has %ld usecs TSC skew! FIXED.\n", i, realdelta);
 		}
 
 		sum += delta;
 	}
 	if (!buggy)
 		printk("passed.\n");
+		;
 }
 
 static void __init synchronize_tsc_ap (void)
@@ -365,7 +365,7 @@
 	 * (This works even if the APIC is not enabled.)
 	 */
 	phys_id = GET_APIC_ID(apic_read(APIC_ID));
-	cpuid = current->processor;
+	cpuid = cpu();
 	if (test_and_set_bit(cpuid, &cpu_online_map)) {
 		printk("huh, phys CPU#%d, CPU#%d already present??\n",
 					phys_id, cpuid);
@@ -435,6 +435,7 @@
 	 */
  	smp_store_cpu_info(cpuid);
 
+	disable_APIC_timer();
 	/*
 	 * Allow the master to continue.
 	 */
@@ -465,6 +466,7 @@
 	smp_callin();
 	while (!atomic_read(&smp_commenced))
 		rep_nop();
+	enable_APIC_timer();
 	/*
 	 * low-memory mappings have been cleared, flush them from
 	 * the local TLBs too.
@@ -803,16 +805,13 @@
 	if (!idle)
 		panic("No idle process for CPU %d", cpu);
 
-	idle->processor = cpu;
-	idle->cpus_runnable = 1 << cpu; /* we schedule the first task manually */
+	init_idle(idle, cpu);
 
 	map_cpu_to_boot_apicid(cpu, apicid);
 
 	idle->thread.eip = (unsigned long) start_secondary;
 
-	del_from_runqueue(idle);
 	unhash_process(idle);
-	init_tasks[cpu] = idle;
 
 	/* start_eip had better be page-aligned! */
 	start_eip = setup_trampoline();
@@ -925,6 +924,7 @@
 }
 
 cycles_t cacheflush_time;
+unsigned long cache_decay_ticks;
 
 static void smp_tune_scheduling (void)
 {
@@ -958,9 +958,13 @@
 		cacheflush_time = (cpu_khz>>10) * (cachesize<<10) / bandwidth;
 	}
 
+	cache_decay_ticks = (long)cacheflush_time/cpu_khz * HZ / 1000;
+
 	printk("per-CPU timeslice cutoff: %ld.%02ld usecs.\n",
 		(long)cacheflush_time/(cpu_khz/1000),
 		((long)cacheflush_time*100/(cpu_khz/1000)) % 100);
+	printk("task migration cache decay timeout: %ld msecs.\n",
+		(cache_decay_ticks + 1) * 1000 / HZ);
 }
 
 /*
@@ -1023,8 +1027,7 @@
 	map_cpu_to_boot_apicid(0, boot_cpu_apicid);
 
 	global_irq_holder = 0;
-	current->processor = 0;
-	init_idle();
+	current->cpu = 0;
 	smp_tune_scheduling();
 
 	/*
--- linux/arch/alpha/mm/fault.c.orig	Thu Jul 18 15:21:51 2002
+++ linux/arch/alpha/mm/fault.c	Thu Jul 18 15:22:35 2002
@@ -196,8 +196,7 @@
  */
 out_of_memory:
 	if (current->pid == 1) {
-		current->policy |= SCHED_YIELD;
-		schedule();
+		yield();
 		down_read(&mm->mmap_sem);
 		goto survive;
 	}
--- linux/arch/alpha/kernel/process.c.orig	Thu Jul 18 15:21:55 2002
+++ linux/arch/alpha/kernel/process.c	Thu Jul 18 15:22:35 2002
@@ -74,8 +74,7 @@
 cpu_idle(void)
 {
 	/* An endless idle loop with no priority at all.  */
-	current->nice = 20;
-	current->counter = -100;
+	init_idle();
 
 	while (1) {
 		/* FIXME -- EV6 and LCA45 know how to power down
--- linux/arch/sparc/kernel/process.c.orig	Thu Jul 18 15:22:19 2002
+++ linux/arch/sparc/kernel/process.c	Thu Jul 18 15:22:35 2002
@@ -74,8 +74,6 @@
 		goto out;
 
 	/* endless idle loop with no priority at all */
-	current->nice = 20;
-	current->counter = -100;
 	init_idle();
 
 	for (;;) {
@@ -128,8 +126,6 @@
 int cpu_idle(void)
 {
 	/* endless idle loop with no priority at all */
-	current->nice = 20;
-	current->counter = -100;
 	init_idle();
 
 	while(1) {
--- linux/arch/mips/mm/fault.c.orig	Thu Jul 18 15:22:18 2002
+++ linux/arch/mips/mm/fault.c	Thu Jul 18 15:22:35 2002
@@ -213,8 +213,7 @@
 out_of_memory:
 	up_read(&mm->mmap_sem);
 	if (tsk->pid == 1) {
-		tsk->policy |= SCHED_YIELD;
-		schedule();
+		yield();
 		down_read(&mm->mmap_sem);
 		goto survive;
 	}
--- linux/arch/mips/kernel/process.c.orig	Thu Jul 18 15:22:18 2002
+++ linux/arch/mips/kernel/process.c	Thu Jul 18 15:22:35 2002
@@ -34,8 +34,6 @@
 ATTRIB_NORET void cpu_idle(void)
 {
 	/* endless idle loop with no priority at all */
-	current->nice = 20;
-	current->counter = -100;
 	init_idle();
 
 	while (1) {
--- linux/arch/ppc/mm/fault.c.orig	Thu Jul 18 15:21:56 2002
+++ linux/arch/ppc/mm/fault.c	Thu Jul 18 15:22:35 2002
@@ -197,8 +197,7 @@
 out_of_memory:
 	up_read(&mm->mmap_sem);
 	if (current->pid == 1) {
-		current->policy |= SCHED_YIELD;
-		schedule();
+		yield();
 		down_read(&mm->mmap_sem);
 		goto survive;
 	}
--- linux/arch/ppc/kernel/idle.c.orig	Thu Jul 18 15:22:19 2002
+++ linux/arch/ppc/kernel/idle.c	Thu Jul 18 15:22:35 2002
@@ -55,9 +55,8 @@
 		do_power_save = 1;
 
 	/* endless loop with no priority at all */
-	current->nice = 20;
-	current->counter = -100;
 	init_idle();
+
 	for (;;) {
 #ifdef CONFIG_SMP
 		if (!do_power_save) {
--- linux/arch/ppc/8xx_io/uart.c.orig	Thu Jul 18 15:22:19 2002
+++ linux/arch/ppc/8xx_io/uart.c	Thu Jul 18 15:22:35 2002
@@ -1796,7 +1796,6 @@
 		printk("lsr = %d (jiff=%lu)...", lsr, jiffies);
 #endif
 		current->state = TASK_INTERRUPTIBLE;
-/*		current->counter = 0;	 make us low-priority */
 		schedule_timeout(char_time);
 		if (signal_pending(current))
 			break;
--- linux/arch/ppc/8260_io/uart.c.orig	Thu Jul 18 15:22:19 2002
+++ linux/arch/ppc/8260_io/uart.c	Thu Jul 18 15:22:35 2002
@@ -1732,7 +1732,6 @@
 		printk("lsr = %d (jiff=%lu)...", lsr, jiffies);
 #endif
 		current->state = TASK_INTERRUPTIBLE;
-/*		current->counter = 0;	 make us low-priority */
 		schedule_timeout(char_time);
 		if (signal_pending(current))
 			break;
--- linux/arch/m68k/kernel/process.c.orig	Thu Jul 18 15:22:18 2002
+++ linux/arch/m68k/kernel/process.c	Thu Jul 18 15:22:35 2002
@@ -80,8 +80,6 @@
 {
 	/* endless idle loop with no priority at all */
 	init_idle();
-	current->nice = 20;
-	current->counter = -100;
 	idle();
 }
 
--- linux/arch/m68k/mm/fault.c.orig	Thu Jul 18 15:22:18 2002
+++ linux/arch/m68k/mm/fault.c	Thu Jul 18 15:22:35 2002
@@ -181,8 +181,7 @@
 out_of_memory:
 	up_read(&mm->mmap_sem);
 	if (current->pid == 1) {
-		current->policy |= SCHED_YIELD;
-		schedule();
+		yield();
 		down_read(&mm->mmap_sem);
 		goto survive;
 	}
--- linux/arch/sparc64/kernel/process.c.orig	Thu Jul 18 15:22:19 2002
+++ linux/arch/sparc64/kernel/process.c	Thu Jul 18 15:22:35 2002
@@ -53,8 +53,6 @@
 		return -EPERM;
 
 	/* endless idle loop with no priority at all */
-	current->nice = 20;
-	current->counter = -100;
 	init_idle();
 
 	for (;;) {
@@ -83,8 +81,6 @@
 #define unidle_me()		(cpu_data[current->processor].idle_volume = 0)
 int cpu_idle(void)
 {
-	current->nice = 20;
-	current->counter = -100;
 	init_idle();
 
 	while(1) {
--- linux/arch/arm/kernel/process.c.orig	Thu Jul 18 15:22:17 2002
+++ linux/arch/arm/kernel/process.c	Thu Jul 18 15:22:35 2002
@@ -83,8 +83,6 @@
 {
 	/* endless idle loop with no priority at all */
 	init_idle();
-	current->nice = 20;
-	current->counter = -100;
 
 	while (1) {
 		void (*idle)(void) = pm_idle;
--- linux/arch/arm/mm/fault-common.c.orig	Thu Jul 18 15:22:17 2002
+++ linux/arch/arm/mm/fault-common.c	Thu Jul 18 15:22:35 2002
@@ -225,8 +225,7 @@
 	 * If we are out of memory for pid1,
 	 * sleep for a while and retry
 	 */
-	tsk->policy |= SCHED_YIELD;
-	schedule();
+	yield();
 	goto survive;
 
 check_stack:
--- linux/arch/sh/kernel/process.c.orig	Thu Jul 18 15:21:59 2002
+++ linux/arch/sh/kernel/process.c	Thu Jul 18 15:22:35 2002
@@ -40,8 +40,6 @@
 {
 	/* endless idle loop with no priority at all */
 	init_idle();
-	current->nice = 20;
-	current->counter = -100;
 
 	while (1) {
 		if (hlt_counter) {
--- linux/arch/sh/mm/fault.c.orig	Thu Jul 18 15:21:59 2002
+++ linux/arch/sh/mm/fault.c	Thu Jul 18 15:22:35 2002
@@ -207,8 +207,7 @@
 out_of_memory:
 	up_read(&mm->mmap_sem);
 	if (current->pid == 1) {
-		current->policy |= SCHED_YIELD;
-		schedule();
+		yield();
 		down_read(&mm->mmap_sem);
 		goto survive;
 	}
--- linux/arch/ia64/kernel/process.c.orig	Thu Jul 18 15:22:17 2002
+++ linux/arch/ia64/kernel/process.c	Thu Jul 18 15:22:35 2002
@@ -127,9 +127,6 @@
 {
 	/* endless idle loop with no priority at all */
 	init_idle();
-	current->nice = 20;
-	current->counter = -100;
-
 
 	while (1) {
 #ifdef CONFIG_SMP
--- linux/arch/ia64/mm/fault.c.orig	Thu Jul 18 15:22:17 2002
+++ linux/arch/ia64/mm/fault.c	Thu Jul 18 15:22:35 2002
@@ -196,8 +196,7 @@
   out_of_memory:
 	up_read(&mm->mmap_sem);
 	if (current->pid == 1) {
-		current->policy |= SCHED_YIELD;
-		schedule();
+		yield();
 		down_read(&mm->mmap_sem);
 		goto survive;
 	}
--- linux/arch/mips64/kernel/process.c.orig	Thu Jul 18 15:22:18 2002
+++ linux/arch/mips64/kernel/process.c	Thu Jul 18 15:22:35 2002
@@ -34,8 +34,7 @@
 {
 	/* endless idle loop with no priority at all */
 	init_idle();
-	current->nice = 20;
-	current->counter = -100;
+
 	while (1) {
 		while (!current->need_resched)
 			if (cpu_wait)
--- linux/arch/mips64/mm/fault.c.orig	Thu Jul 18 15:22:18 2002
+++ linux/arch/mips64/mm/fault.c	Thu Jul 18 15:22:35 2002
@@ -238,8 +238,7 @@
 out_of_memory:
 	up_read(&mm->mmap_sem);
 	if (tsk->pid == 1) {
-		tsk->policy |= SCHED_YIELD;
-		schedule();
+		yield();
 		down_read(&mm->mmap_sem);
 		goto survive;
 	}
--- linux/arch/s390/kernel/process.c.orig	Thu Jul 18 15:22:19 2002
+++ linux/arch/s390/kernel/process.c	Thu Jul 18 15:22:35 2002
@@ -57,8 +57,7 @@
 
 	/* endless idle loop with no priority at all */
         init_idle();
-	current->nice = 20;
-	current->counter = -100;
+
 	while (1) {
 		if (current->need_resched) {
 			schedule();
--- linux/arch/s390/mm/fault.c.orig	Thu Jul 18 15:22:19 2002
+++ linux/arch/s390/mm/fault.c	Thu Jul 18 15:22:35 2002
@@ -293,8 +293,7 @@
 out_of_memory:
 	up_read(&mm->mmap_sem);
 	if (tsk->pid == 1) {
-		tsk->policy |= SCHED_YIELD;
-		schedule();
+		yield();
 		down_read(&mm->mmap_sem);
 		goto survive;
 	}
--- linux/arch/parisc/kernel/process.c.orig	Tue Aug 21 14:26:04 2001
+++ linux/arch/parisc/kernel/process.c	Thu Jul 18 15:22:35 2002
@@ -70,8 +70,6 @@
 {
 	/* endless idle loop with no priority at all */
 	init_idle();
-	current->nice = 20;
-	current->counter = -100;
 
 	while (1) {
 		while (!current->need_resched) {
--- linux/arch/cris/kernel/process.c.orig	Thu Jul 18 15:22:11 2002
+++ linux/arch/cris/kernel/process.c	Thu Jul 18 15:22:35 2002
@@ -124,10 +124,10 @@
  
 int cpu_idle(void *unused)
 {
-	while(1) {
-		current->counter = -100;
+	init_idle();
+
+	while(1)
 		schedule();
-	}
 }
 
 /* if the watchdog is enabled, we can simply disable interrupts and go
--- linux/arch/s390x/kernel/process.c.orig	Thu Jul 18 15:22:19 2002
+++ linux/arch/s390x/kernel/process.c	Thu Jul 18 15:22:35 2002
@@ -57,8 +57,7 @@
 
 	/* endless idle loop with no priority at all */
         init_idle();
-	current->nice = 20;
-	current->counter = -100;
+
 	while (1) {
 		if (current->need_resched) {
 			schedule();
--- linux/arch/s390x/mm/fault.c.orig	Thu Jul 18 15:22:19 2002
+++ linux/arch/s390x/mm/fault.c	Thu Jul 18 15:22:35 2002
@@ -293,8 +293,7 @@
 out_of_memory:
 	up_read(&mm->mmap_sem);
 	if (tsk->pid == 1) {
-		tsk->policy |= SCHED_YIELD;
-		schedule();
+		yield();
 		down_read(&mm->mmap_sem);
 		goto survive;
 	}
--- linux/arch/ppc64/kernel/idle.c.orig	Thu Jul 18 15:22:19 2002
+++ linux/arch/ppc64/kernel/idle.c	Thu Jul 18 15:22:35 2002
@@ -76,9 +76,6 @@
 	unsigned long CTRL;
 #endif
 
-	/* endless loop with no priority at all */
-	current->nice = 20;
-	current->counter = -100;
 #ifdef CONFIG_PPC_ISERIES
 	/* ensure iSeries run light will be out when idle */
 	current->thread.flags &= ~PPC_FLAG_RUN_LIGHT;
@@ -86,6 +83,7 @@
 	CTRL &= ~RUNLATCH;
 	mtspr(CTRLT, CTRL);
 #endif
+	/* endless loop with no priority at all */
 	init_idle();	
 
 	lpaca = get_paca();