Author: tjansen Date: Mon Oct 19 16:24:00 2009 New Revision: 1233 Log: Made scheduler reentrant. Moved all static variables from scheduler.c to a separate data structure pisa_scheduler. This required a lot of changes as a pointer to pisa_scheduler is now passed to every function in scheduler.c. Note that it's no longer possible for conmgr.c to remove tasks, as it doesn't know which pisa_scheduler to use. Instead, use the callback function you pass to pisa_conmgr_init for that purpose. Modified: trunk/include/scheduler.h trunk/libpisa/conmgr.c trunk/libpisa/scheduler.c trunk/pisacd/cdctx.c trunk/pisacd/cdctx.h trunk/pisacd/cdderegister.c trunk/pisacd/cdheartbeat.c trunk/pisacd/cdmain.c trunk/pisacd/cdpending.c trunk/pisacd/cdregister.c trunk/pisacd/cdtimeout.c trunk/pisasd/sdctx.c trunk/pisasd/sdctx.h trunk/pisasd/sdmain.c Modified: trunk/include/scheduler.h ============================================================================== --- trunk/include/scheduler.h Mon Oct 19 15:07:26 2009 (r1232) +++ trunk/include/scheduler.h Mon Oct 19 16:24:00 2009 (r1233) @@ -27,18 +27,35 @@ struct pisa_sched_task *next; } pisa_sched_task; -void pisa_sched_init(int *p); -void pisa_sched_cleanup(void); -void pisa_sched_run(void); +typedef struct pisa_scheduler { + /** Our thread ID */ + pthread_t thread; -pisa_sched_task *pisa_sched_add(pisa_sched_func func, struct timeval *delay, void *data); -void pisa_sched_remove(pisa_sched_task *task); + /** Mutex for the task list */ + pthread_mutex_t mutex_list; -static inline pisa_sched_task *pisa_sched_add_now(pisa_sched_func func, void *data) + /** Head of the sorted task list */ + pisa_sched_task *head; + + /** Internal pipe for waking up the scheduler */ + int pipe_sched[2]; + + /** External pipe for waking up the main select loop */ + int pipe_main[2]; +} pisa_scheduler; + +void pisa_sched_init(pisa_scheduler *sched); +void pisa_sched_cleanup(pisa_scheduler *sched); +void pisa_sched_run(pisa_scheduler *sched); + +pisa_sched_task *pisa_sched_add(pisa_scheduler *sched, pisa_sched_func func, struct timeval *delay, void *data); +void pisa_sched_remove(pisa_scheduler *sched, pisa_sched_task *task); + +static inline pisa_sched_task *pisa_sched_add_now(pisa_scheduler *sched, pisa_sched_func func, void *data) { struct timeval delay; timerclear(&delay); - return pisa_sched_add(func, &delay, data); + return pisa_sched_add(sched, func, &delay, data); } #endif /* PISA_SCHEDULER_H */ Modified: trunk/libpisa/conmgr.c ============================================================================== --- trunk/libpisa/conmgr.c Mon Oct 19 15:07:26 2009 (r1232) +++ trunk/libpisa/conmgr.c Mon Oct 19 16:24:00 2009 (r1233) @@ -207,8 +207,6 @@ HASH_DELETE(hh_ipv4, conlist->hash_ipv4, entry); if (conlist->connection_remove_callback != NULL) conlist->connection_remove_callback(entry); - if (entry->timeout_task) - pisa_sched_remove(entry->timeout_task); pisa_conmgr_free(entry); } Modified: trunk/libpisa/scheduler.c ============================================================================== --- trunk/libpisa/scheduler.c Mon Oct 19 15:07:26 2009 (r1232) +++ trunk/libpisa/scheduler.c Mon Oct 19 16:24:00 2009 (r1233) @@ -17,69 +17,44 @@ #include "util.h" /** - * Our way to tell the main loop that we want to be called is writing a byte - * to the pipe. - */ -static int pipe_main = 0; - -/** - * Our thread ID for the scheduler thread - */ -static pthread_t thread; - -/** - * Mutex for locking the task list - */ -static pthread_mutex_t mutex_list; - -/** - * Head of the task list. - */ -static pisa_sched_task *head = NULL; - -/** - * Pipe indicates that the scheduler should wake up before the first task - */ -static int pipe_sched[2]; - -/** * Main function of the scheduler thread */ static void *pisa_sched_main(void *arg) { fd_set fds; struct timeval timeout, now; + pisa_scheduler *sched = (pisa_scheduler *)arg; timerclear(&timeout); while (1) { FD_ZERO(&fds); - FD_SET(pipe_sched[0], &fds); + FD_SET(sched->pipe_sched[0], &fds); - select(pipe_sched[0] + 1, &fds, NULL, NULL, &timeout); - if (FD_ISSET(pipe_sched[0], &fds)) { + select(sched->pipe_sched[0] + 1, &fds, NULL, NULL, &timeout); + if (FD_ISSET(sched->pipe_sched[0], &fds)) { /* Someone woke us up to reschedule, read the bytes */ char c; - read(pipe_sched[0], &c, 1); + read(sched->pipe_sched[0], &c, 1); } - pthread_mutex_lock(&mutex_list); + pthread_mutex_lock(&sched->mutex_list); gettimeofday(&now, NULL); - if (head && pisa_time_before(&head->due, &now)) { + if (sched->head && pisa_time_before(&sched->head->due, &now)) { /* At least the first action is due now, raise the * flag. As we have to reschedule afterwards, sleep * for a _long_ time. We don't care about tv_usec. */ - write(pipe_main, "1", 1); + write(sched->pipe_main[1], "1", 1); timeout.tv_sec = 1000000; } else { /* No action is due now, sleep until the head is due * or we are woken up for rescheduling. */ - if (head) - timersub(&head->due, &now, &timeout); + if (sched->head) + timersub(&sched->head->due, &now, &timeout); else timeout.tv_sec = 1000000; } - pthread_mutex_unlock(&mutex_list); + pthread_mutex_unlock(&sched->mutex_list); } return NULL; @@ -91,24 +66,23 @@ * * @param p pipe to the main thread */ -void pisa_sched_init(int *p) +void pisa_sched_init(pisa_scheduler *sched) { int result; - result = pipe(p); + result = pipe(sched->pipe_main); if (result != 0) PISA_ERROR("Could not create external pipe for scheduler.\n"); - pipe_main = p[1]; - result = pipe(pipe_sched); + result = pipe(sched->pipe_sched); if (result != 0) PISA_ERROR("Could not create internal pipe for scheduler.\n"); - result = pthread_mutex_init(&mutex_list, NULL); + result = pthread_mutex_init(&sched->mutex_list, NULL); if (result != 0) PISA_ERROR("Could not create mutex for scheduler.\n"); - result = pthread_create(&thread, NULL, pisa_sched_main, NULL); + result = pthread_create(&sched->thread, NULL, pisa_sched_main, sched); if (result != 0) PISA_ERROR("Could not create maintenance thread.\n"); } @@ -116,18 +90,20 @@ /** * Clean up the scheduler. Free the task list and kill the thread. */ -void pisa_sched_cleanup(void) +void pisa_sched_cleanup(pisa_scheduler *sched) { - pthread_cancel(thread); - pthread_join(thread, NULL); + pthread_cancel(sched->thread); + pthread_join(sched->thread, NULL); - close(pipe_sched[0]); - close(pipe_sched[1]); + close(sched->pipe_sched[0]); + close(sched->pipe_sched[1]); + close(sched->pipe_main[0]); + close(sched->pipe_main[1]); - pthread_mutex_destroy(&mutex_list); + pthread_mutex_destroy(&sched->mutex_list); - while (head) - pisa_sched_remove(head); + while (sched->head) + pisa_sched_remove(sched, sched->head); } /** @@ -136,12 +112,12 @@ * * @param task task to be removed from the schedule */ -static void pisa_sched_remove_internal(pisa_sched_task *task) +static void pisa_sched_remove_internal(pisa_scheduler *sched, pisa_sched_task *task) { - pisa_sched_task *cur = head; + pisa_sched_task *cur = sched->head; - if (task == head) { - head = head->next; + if (task == sched->head) { + sched->head = sched->head->next; } else { while (task != cur->next) { cur = cur->next; @@ -158,7 +134,7 @@ * Run scheduled actions. Called by the main thread eventually if we set the * shared variable to a value != 0. Handle all actions that are due. */ -void pisa_sched_run(void) +void pisa_sched_run(pisa_scheduler *sched) { struct timeval now; struct pisa_sched_task *run = NULL, *run_tail = NULL; @@ -167,20 +143,20 @@ * list. The runlist is executed after the mutex has been released to * allow addition of new tasks in a task handler (useful for * periodically recurring tasks). */ - pthread_mutex_lock(&mutex_list); + pthread_mutex_lock(&sched->mutex_list); gettimeofday(&now, NULL); - while (head && pisa_time_before(&head->due, &now)) { + while (sched->head && pisa_time_before(&sched->head->due, &now)) { if (run_tail == NULL) - run = run_tail = head; + run = run_tail = sched->head; else { - run_tail->next = head; - run_tail = head; + run_tail->next = sched->head; + run_tail = sched->head; } - pisa_sched_remove_internal(head); + pisa_sched_remove_internal(sched, sched->head); } - pthread_mutex_unlock(&mutex_list); + pthread_mutex_unlock(&sched->mutex_list); /* Execute the run list */ while (run) { @@ -191,7 +167,7 @@ free(tmp); } - write(pipe_sched[1], "0", 1); + write(sched->pipe_sched[1], "0", 1); } /** @@ -201,12 +177,12 @@ * @param delay delay until the function is run * @param data opaque payload */ -pisa_sched_task *pisa_sched_add(pisa_sched_func func, struct timeval *delay, void *data) +pisa_sched_task *pisa_sched_add(pisa_scheduler *sched, pisa_sched_func func, struct timeval *delay, void *data) { - pisa_sched_task *cur = head, *task = malloc(sizeof(pisa_sched_task)); + pisa_sched_task *cur = sched->head, *task = malloc(sizeof(pisa_sched_task)); struct timeval now; - pthread_mutex_lock(&mutex_list); + pthread_mutex_lock(&sched->mutex_list); gettimeofday(&now, NULL); @@ -214,9 +190,9 @@ task->data = data; timeradd(&now, delay, &task->due); - if (head == NULL || pisa_time_before(&task->due, &head->due)) { - task->next = head; - head = task; + if (sched->head == NULL || pisa_time_before(&task->due, &sched->head->due)) { + task->next = sched->head; + sched->head = task; } else { while (cur->next && pisa_time_before(&cur->next->due, &task->due)) cur = cur->next; @@ -224,9 +200,9 @@ cur->next = task; } - write(pipe_sched[1], "0", 1); + write(sched->pipe_sched[1], "0", 1); - pthread_mutex_unlock(&mutex_list); + pthread_mutex_unlock(&sched->mutex_list); return task; } @@ -236,14 +212,14 @@ * * @param task task to be removed from the schedule */ -void pisa_sched_remove(pisa_sched_task *task) +void pisa_sched_remove(pisa_scheduler *sched, pisa_sched_task *task) { - pthread_mutex_unlock(&mutex_list); + pthread_mutex_lock(&sched->mutex_list); - pisa_sched_remove_internal(task); + pisa_sched_remove_internal(sched, task); free(task); - write(pipe_sched[1], "0", 1); + write(sched->pipe_sched[1], "0", 1); - pthread_mutex_unlock(&mutex_list); + pthread_mutex_unlock(&sched->mutex_list); } Modified: trunk/pisacd/cdctx.c ============================================================================== --- trunk/pisacd/cdctx.c Mon Oct 19 15:07:26 2009 (r1232) +++ trunk/pisacd/cdctx.c Mon Oct 19 16:24:00 2009 (r1233) @@ -80,8 +80,7 @@ cdctx->natlist = NULL; cdctx->conlist = NULL; - cdctx->fd_scheduler[0] = -1; - cdctx->fd_scheduler[1] = -1; + memset(&cdctx->scheduler, 0, sizeof(pisa_scheduler)); } /** Modified: trunk/pisacd/cdctx.h ============================================================================== --- trunk/pisacd/cdctx.h Mon Oct 19 15:07:26 2009 (r1232) +++ trunk/pisacd/cdctx.h Mon Oct 19 16:24:00 2009 (r1233) @@ -19,6 +19,7 @@ #include "ctrlhandler.h" #include "linkedlist.h" #include "nat.h" +#include "scheduler.h" /** * data structure containing all of the necessary information about the @@ -138,9 +139,9 @@ pisa_conmgr_list *conlist; /** - * Pipe used by the scheduler to signal that an action is ready. + * The scheduler for our asynchronous tasks. */ - int fd_scheduler[2]; + pisa_scheduler scheduler; } cd_context; extern cd_context cd_ctx; Modified: trunk/pisacd/cdderegister.c ============================================================================== --- trunk/pisacd/cdderegister.c Mon Oct 19 15:07:26 2009 (r1232) +++ trunk/pisacd/cdderegister.c Mon Oct 19 16:24:00 2009 (r1233) @@ -29,7 +29,8 @@ pisa_remove_default_route(); entry->status = PISA_CON_DISCONNECTED; - pisa_sched_remove(entry->timeout_task); + if (entry->timeout_task) + pisa_sched_remove(&cd_ctx.scheduler, entry->timeout_task); entry->timeout_task = NULL; } Modified: trunk/pisacd/cdheartbeat.c ============================================================================== --- trunk/pisacd/cdheartbeat.c Mon Oct 19 15:07:26 2009 (r1232) +++ trunk/pisacd/cdheartbeat.c Mon Oct 19 16:24:00 2009 (r1233) @@ -111,5 +111,5 @@ time(&t); pisa_conmgr_iterate_connected(cd_ctx.conlist, pisa_client_send_heartbeat, &t); - pisa_sched_add(pisa_task_heartbeat, &delay, NULL); + pisa_sched_add(&cd_ctx.scheduler, pisa_task_heartbeat, &delay, NULL); } Modified: trunk/pisacd/cdmain.c ============================================================================== --- trunk/pisacd/cdmain.c Mon Oct 19 15:07:26 2009 (r1232) +++ trunk/pisacd/cdmain.c Mon Oct 19 16:24:00 2009 (r1233) @@ -143,13 +143,16 @@ } /** - * Remove all pending messages sent to this connection + * A connection is removed permanently from the list. Clean up all associated + * state. * * @param entry The entry associated with this connection */ void pisacd_cleanup_after_removed_connection(pisa_conmgr_entry* entry) { pisa_pending_remove_by_entry(entry); + if (entry->timeout_task) + pisa_sched_remove(&cd_ctx.scheduler, entry->timeout_task); } /** @@ -276,9 +279,9 @@ pisa_tunnel_configure_main(cd_ctx.ifname_tunnel, &cd_cfg.local_ipv4, &cd_cfg.local_netmask, MTU_TUN); /* Initialize the scheduler and start some maintenance tasks */ - pisa_sched_init(cd_ctx.fd_scheduler); - pisa_sched_add_now(pisa_task_heartbeat, NULL); - pisa_sched_add_now(pisa_task_pending, NULL); + pisa_sched_init(&cd_ctx.scheduler); + pisa_sched_add_now(&cd_ctx.scheduler, pisa_task_heartbeat, NULL); + pisa_sched_add_now(&cd_ctx.scheduler, pisa_task_pending, NULL); } static void cd_deinit(void) @@ -295,7 +298,7 @@ cdctx_destroy(&cd_ctx); cdconf_destroy(&cd_cfg); pisa_arp_cleanup(); - pisa_sched_cleanup(); + pisa_sched_cleanup(&cd_ctx.scheduler); #ifdef REMOVE_PREAUTH_CODE close(cd_ctx.fd_pacli); @@ -305,8 +308,6 @@ close(cd_ctx.tunc); close(cd_ctx.tund); close(cd_ctx.tunnel); - close(cd_ctx.fd_scheduler[0]); - close(cd_ctx.fd_scheduler[1]); pisa_ctrlhandler_cleanup(&cd_ctx.ctrlhandlers); @@ -325,8 +326,8 @@ static inline void pisa_handle_scheduler(void) { char c; - read(cd_ctx.fd_scheduler[0], &c, 1); - pisa_sched_run(); + read(cd_ctx.scheduler.pipe_main[0], &c, 1); + pisa_sched_run(&cd_ctx.scheduler); } /** @@ -367,7 +368,7 @@ FD_SET(cd_ctx.fd_pasrv, &readfds); #endif /* REMOVE_PREAUTH_CODE */ FD_SET(cd_ctx.fd_pisaconf, &readfds); - FD_SET(cd_ctx.fd_scheduler[0], &readfds); + FD_SET(cd_ctx.scheduler.pipe_main[0], &readfds); #ifdef REMOVE_PREAUTH_CODE /* Send periodical neighbor exchange requests, @@ -443,7 +444,7 @@ if (FD_ISSET(cd_ctx.fd_pisaconf, &readfds)) pisa_conf_handle_packet(cd_ctx.fd_pisaconf); - if (FD_ISSET(cd_ctx.fd_scheduler[0], &readfds)) + if (FD_ISSET(cd_ctx.scheduler.pipe_main[0], &readfds)) pisa_handle_scheduler(); } @@ -933,16 +934,16 @@ #endif /* REMOVE_PREAUTH_CODE */ tmfd = pisa_maxInt(tmfd, cd_ctx.tunc); tmfd = pisa_maxInt(tmfd, cd_ctx.tund); - tmfd = pisa_maxInt(tmfd, cd_ctx.fd_scheduler[0]); + tmfd = pisa_maxInt(tmfd, cd_ctx.scheduler.pipe_main[0]); maxfd = tmfd + 1; #else #ifdef REMOVE_PREAUTH_CODE maxfd = pisa_maxof(7, cd_ctx.tunnel, cd_ctx.fd_pisaconf, cd_ctx.fd_pacli, cd_ctx.fd_pasrv, - cd_ctx.tunc, cd_ctx.tund, cd_ctx.fd_scheduler[0]) + 1; + cd_ctx.tunc, cd_ctx.tund, cd_ctx.scheduler.pipe_main[0]) + 1; #else maxfd = pisa_maxof(5, cd_ctx.tunnel, cd_ctx.fd_pisaconf, - cd_ctx.tunc, cd_ctx.tund, cd_ctx.fd_scheduler[0]) + 1; + cd_ctx.tunc, cd_ctx.tund, cd_ctx.scheduler.pipe_main[0]) + 1; #endif /* REMOVE_PREAUTH_CODE */ #endif Modified: trunk/pisacd/cdpending.c ============================================================================== --- trunk/pisacd/cdpending.c Mon Oct 19 15:07:26 2009 (r1232) +++ trunk/pisacd/cdpending.c Mon Oct 19 16:24:00 2009 (r1233) @@ -13,6 +13,7 @@ #include <stdlib.h> #include "debug.h" +#include "cdctx.h" #include "cdpending.h" #include "util.h" #include "scheduler.h" @@ -190,7 +191,7 @@ } } - pisa_sched_add(pisa_task_pending, &delay, NULL); + pisa_sched_add(&cd_ctx.scheduler, pisa_task_pending, &delay, NULL); /* return hash_pending ? 1 : 0; */ } Modified: trunk/pisacd/cdregister.c ============================================================================== --- trunk/pisacd/cdregister.c Mon Oct 19 15:07:26 2009 (r1232) +++ trunk/pisacd/cdregister.c Mon Oct 19 16:24:00 2009 (r1233) @@ -160,7 +160,7 @@ entry->timeout_flag = 0; delay.tv_sec = cd_cfg.idle_disconnect_delay; delay.tv_usec = 0; - entry->timeout_task = pisa_sched_add(pisa_task_timeout, &delay, entry); + entry->timeout_task = pisa_sched_add(&cd_ctx.scheduler, pisa_task_timeout, &delay, entry); inet_ntop(AF_INET6, &addr->sin6_addr, buffer, sizeof(buffer)); PISA_DEBUG(PL_REGISTER, "Connected to PISA server %s\n", buffer); Modified: trunk/pisacd/cdtimeout.c ============================================================================== --- trunk/pisacd/cdtimeout.c Mon Oct 19 15:07:26 2009 (r1232) +++ trunk/pisacd/cdtimeout.c Mon Oct 19 16:24:00 2009 (r1233) @@ -56,5 +56,5 @@ /* Reset the timeout flag and reschedule */ e->timeout_flag = 0; - e->timeout_task = pisa_sched_add(pisa_task_timeout, &delay, e); + e->timeout_task = pisa_sched_add(&cd_ctx.scheduler, pisa_task_timeout, &delay, e); } Modified: trunk/pisasd/sdctx.c ============================================================================== --- trunk/pisasd/sdctx.c Mon Oct 19 15:07:26 2009 (r1232) +++ trunk/pisasd/sdctx.c Mon Oct 19 16:24:00 2009 (r1233) @@ -67,8 +67,7 @@ sdctx->conlist = NULL; sdctx->disable_ip4_forward=0; - sdctx->fd_scheduler[0] = -1; - sdctx->fd_scheduler[1] = -1; + memset(&sdctx->scheduler, 0, sizeof(pisa_scheduler)); } /** Modified: trunk/pisasd/sdctx.h ============================================================================== --- trunk/pisasd/sdctx.h Mon Oct 19 15:07:26 2009 (r1232) +++ trunk/pisasd/sdctx.h Mon Oct 19 16:24:00 2009 (r1233) @@ -126,9 +126,9 @@ int disable_ip4_forward; /** - * Pipe used by the scheduler to signal that an action is ready. + * The scheduler for our asynchronous tasks. */ - int fd_scheduler[2]; + pisa_scheduler scheduler; } sd_context; extern sd_context sd_ctx; Modified: trunk/pisasd/sdmain.c ============================================================================== --- trunk/pisasd/sdmain.c Mon Oct 19 15:07:26 2009 (r1232) +++ trunk/pisasd/sdmain.c Mon Oct 19 16:24:00 2009 (r1233) @@ -250,7 +250,7 @@ pisa_tunnel_configure_main(sd_ctx.fd_pisa_tunnel_name, &sd_cfg.ipaddr, &netmask, MTU_TUN); /* Initialize the scheduler */ - pisa_sched_init(sd_ctx.fd_scheduler); + pisa_sched_init(&sd_ctx.scheduler); } /** @@ -273,7 +273,7 @@ sdctx_destroy(&sd_ctx); sdconf_destroy(&sd_cfg); pisa_arp_cleanup(); - pisa_sched_cleanup(); + pisa_sched_cleanup(&sd_ctx.scheduler); /* finish all the remaining jobs */ close(sd_ctx.tunc); @@ -284,8 +284,6 @@ #endif /* REMOVE_PREAUTH_CODE */ close(sd_ctx.tunnel); close(sd_ctx.fd_pisaconf); - close(sd_ctx.fd_scheduler[0]); - close(sd_ctx.fd_scheduler[1]); /* Disable IPv4 forwarding */ pisa_forwarding_stop(); @@ -309,8 +307,8 @@ static inline void pisa_handle_scheduler(void) { char c; - read(sd_ctx.fd_scheduler[0], &c, 1); - pisa_sched_run(); + read(sd_ctx.scheduler.pipe_main[0], &c, 1); + pisa_sched_run(&sd_ctx.scheduler); } /** @@ -345,20 +343,20 @@ #endif /* REMOVE_PREAUTH_CODE */ FD_SET(sd_ctx.tunnel, &readfds); FD_SET(sd_ctx.fd_pisaconf, &readfds); - FD_SET(sd_ctx.fd_scheduler[0], &readfds); + FD_SET(sd_ctx.scheduler.pipe_main[0], &readfds); #ifdef REMOVE_PREAUTH_CODE maxfd = 1 + pisa_maxof(7, sd_ctx.tunc, sd_ctx.tund, sd_ctx.fd_pstunc, sd_ctx.fd_pstuns, sd_ctx.tunnel,sd_ctx.tunnel, - sd_ctx.fd_scheduler[0]); + sd_ctx.scheduler.pipe_main[0]); #else /* Performance optimization: Does this really have to be * recalcutated every time? Check again after PREAUTH is * removed. -- Thomas */ maxfd = 1 + pisa_maxof(5, sd_ctx.tunc, sd_ctx.tund, sd_ctx.tunnel,sd_ctx.tunnel, - sd_ctx.fd_scheduler[0]); + sd_ctx.scheduler.pipe_main[0]); #endif /* REMOVE_PREAUTH_CODE */ if (select(maxfd + 1, &readfds, NULL, NULL, NULL) > 0) { @@ -379,7 +377,7 @@ if (FD_ISSET(sd_ctx.fd_pisaconf, &readfds)) pisa_conf_handle_packet(sd_ctx.fd_pisaconf); - if (FD_ISSET(sd_ctx.fd_scheduler[0], &readfds)) + if (FD_ISSET(sd_ctx.scheduler.pipe_main[0], &readfds)) pisa_handle_scheduler(); }