[packages/autofs.git] / autofs-5.0.4-always-read-file-maps.patch

autofs-5.0.4 - always read file maps

From: Ian Kent <raven@themaw.net>

autofs tries to not load an entire map into the internal cache unless it
has to. For maps that do get loaded into the cache it relies on checks to
work out if a map is up to date in order to trigger a map read. This is
fine for maps that can do direct key lookups but file maps need to do a
linear search through the file when locating an entry for a key. For large
maps this can be a huge overhead. This patch make autofs always load file
based maps at start and makes use of the map file mtime to discover if the
cache needs to be refreshed.
---

 CHANGELOG             |    1 +
 daemon/lookup.c       |    9 +++++--
 modules/lookup_file.c |   65 ++++++++++++++++---------------------------------
 3 files changed, 28 insertions(+), 47 deletions(-)


diff --git a/CHANGELOG b/CHANGELOG
index d4dd70b..afd1335 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -14,6 +14,7 @@
 - check for stale SASL credentials upon connect fail.
 - add "forcestart" and "forcerestart" init script options to allow
   use of 5.0.3 strartup behavior if required.
+- always read entire file map into cache to speed lookups.
 
 4/11/2008 autofs-5.0.4
 -----------------------
diff --git a/daemon/lookup.c b/daemon/lookup.c
index 741d846..e034348 100644
--- a/daemon/lookup.c
+++ b/daemon/lookup.c
@@ -283,10 +283,13 @@ static int do_read_map(struct autofs_point *ap, struct map_source *map, time_t a
 	 * for the fail cases to function correctly and to cache the
 	 * lookup handle.
 	 *
-	 * We always need to whole map for direct mounts in order to
-	 * mount the triggers.
+	 * We always need to read the whole map for direct mounts in
+	 * order to mount the triggers. We also want to read the whole
+	 * map if it's a file map to avoid potentially lengthy linear
+	 * file scanning.
 	 */
-	if (!(ap->flags & MOUNT_FLAG_GHOST) && ap->type != LKP_DIRECT)
+	if (strcmp(map->type, "file") &&
+	    !(ap->flags & MOUNT_FLAG_GHOST) && ap->type != LKP_DIRECT)
 		return NSS_STATUS_SUCCESS;
 
 	if (!map->stale)
diff --git a/modules/lookup_file.c b/modules/lookup_file.c
index 95b9f6f..aafeb8b 100644
--- a/modules/lookup_file.c
+++ b/modules/lookup_file.c
@@ -44,7 +44,6 @@ typedef enum { esc_none, esc_char, esc_val, esc_all } ESCAPES;
 
 struct lookup_context {
 	const char *mapname;
-	time_t mtime;
 	struct parse_mod *parse;
 };
 
@@ -54,7 +53,6 @@ int lookup_init(const char *mapfmt, int argc, const char *const *argv, void **co
 {
 	struct lookup_context *ctxt;
 	char buf[MAX_ERR_BUF];
-	struct stat st;
 
 	*context = NULL;
 
@@ -87,15 +85,6 @@ int lookup_init(const char *mapfmt, int argc, const char *const *argv, void **co
 		return 1;
 	}
 
-	if (stat(ctxt->mapname, &st)) {
-		free(ctxt);
-		logmsg(MODPREFIX "file map %s, could not stat",
-		     argv[0]);
-		return 1;
-	}
-		
-	ctxt->mtime = st.st_mtime;
-
 	if (!mapfmt)
 		mapfmt = MAPFMT_DEFAULT;
 
@@ -391,9 +380,7 @@ int lookup_read_master(struct master *master, time_t age, void *context)
 	int blen;
 	char *path;
 	char *ent;
-	struct stat st;
 	FILE *f;
-	int fd;
 	unsigned int path_len, ent_len;
 	int entry, cur_state;
 
@@ -428,8 +415,6 @@ int lookup_read_master(struct master *master, time_t age, void *context)
 		return NSS_STATUS_UNAVAIL;
 	}
 
-	fd = fileno(f);
-
 	while(1) {
 		entry = read_one(logopt, f, path, &path_len, ent, &ent_len);
 		if (!entry) {
@@ -504,13 +489,6 @@ int lookup_read_master(struct master *master, time_t age, void *context)
 			break;
 	}
 
-	if (fstat(fd, &st)) {
-		crit(logopt, MODPREFIX "file map %s, could not stat",
-		       ctxt->mapname);
-		return NSS_STATUS_UNAVAIL;
-	}
-	ctxt->mtime = st.st_mtime;
-
 	fclose(f);
 
 	return NSS_STATUS_SUCCESS;
@@ -642,9 +620,7 @@ int lookup_read_map(struct autofs_point *ap, time_t age, void *context)
 	struct mapent_cache *mc;
 	char *key;
 	char *mapent;
-	struct stat st;
 	FILE *f;
-	int fd;
 	unsigned int k_len, m_len;
 	int entry;
 
@@ -684,8 +660,6 @@ int lookup_read_map(struct autofs_point *ap, time_t age, void *context)
 		return NSS_STATUS_UNAVAIL;
 	}
 
-	fd = fileno(f);
-
 	while(1) {
 		entry = read_one(ap->logopt, f, key, &k_len, mapent, &m_len);
 		if (!entry) {
@@ -748,13 +722,6 @@ int lookup_read_map(struct autofs_point *ap, time_t age, void *context)
 			break;
 	}
 
-	if (fstat(fd, &st)) {
-		crit(ap->logopt,
-		     MODPREFIX "file map %s, could not stat",
-		     ctxt->mapname);
-		return NSS_STATUS_UNAVAIL;
-	}
-	ctxt->mtime = st.st_mtime;
 	source->age = age;
 
 	fclose(f);
@@ -951,9 +918,6 @@ static int check_map_indirect(struct autofs_point *ap,
 	if (ret == CHE_FAIL)
 		return NSS_STATUS_NOTFOUND;
 
-	if (ret & CHE_UPDATED)
-		source->stale = 1;
-
 	pthread_cleanup_push(cache_lock_cleanup, mc);
 	cache_writelock(mc);
 	exists = cache_lookup_distinct(mc, key);
@@ -963,7 +927,6 @@ static int check_map_indirect(struct autofs_point *ap,
 			free(exists->mapent);
 			exists->mapent = NULL;
 			exists->status = 0;
-			source->stale = 1;
 		}
 	}
 	pthread_cleanup_pop(1);
@@ -985,14 +948,8 @@ static int check_map_indirect(struct autofs_point *ap,
 		we = cache_lookup_distinct(mc, "*");
 		if (we) {
 			/* Wildcard entry existed and is now gone */
-			if (we->source == source && (wild & CHE_MISSING)) {
+			if (we->source == source && (wild & CHE_MISSING))
 				cache_delete(mc, "*");
-				source->stale = 1;
-			}
-		} else {
-			/* Wildcard not in map but now is */
-			if (wild & (CHE_OK | CHE_UPDATED))
-				source->stale = 1;
 		}
 		pthread_cleanup_pop(1);
 
@@ -1062,9 +1019,28 @@ int lookup_mount(struct autofs_point *ap, const char *name, int name_len, void *
 	 * we never know about it.
 	 */
 	if (ap->type == LKP_INDIRECT && *key != '/') {
+		struct stat st;
 		char *lkp_key;
 
+		/*
+		 * We can skip the map lookup and cache update altogether
+		 * if we know the map hasn't been modified since it was
+		 * last read. If it has then we can mark the map stale
+		 * so a re-read is triggered following the lookup.
+		 */
+		if (stat(ctxt->mapname, &st)) {
+			error(ap->logopt, MODPREFIX
+			      "file map %s, could not stat", ctxt->mapname);
+			return NSS_STATUS_UNAVAIL;
+		}
+
 		cache_readlock(mc);
+		me = cache_lookup_first(mc);
+		if (me && st.st_mtime <= me->age)
+			goto do_cache_lookup;
+		else
+			source->stale = 1;
+
 		me = cache_lookup_distinct(mc, key);
 		if (me && me->multi)
 			lkp_key = strdup(me->multi->key);
@@ -1088,6 +1064,7 @@ int lookup_mount(struct autofs_point *ap, const char *name, int name_len, void *
 	}
 
 	cache_readlock(mc);
+do_cache_lookup:
 	me = cache_lookup(mc, key);
 	/* Stale mapent => check for entry in alternate source or wildcard */
 	if (me && !me->mapent) {
Commit	Line	Data
e5fd101c PS	1	autofs-5.0.4 - always read file maps
	2
	3	From: Ian Kent <raven@themaw.net>
	4
	5	autofs tries to not load an entire map into the internal cache unless it
	6	has to. For maps that do get loaded into the cache it relies on checks to
	7	work out if a map is up to date in order to trigger a map read. This is
	8	fine for maps that can do direct key lookups but file maps need to do a
	9	linear search through the file when locating an entry for a key. For large
	10	maps this can be a huge overhead. This patch make autofs always load file
	11	based maps at start and makes use of the map file mtime to discover if the
	12	cache needs to be refreshed.
	13	---
	14
	15	CHANGELOG \| 1 +
	16	daemon/lookup.c \| 9 +++++--
	17	modules/lookup_file.c \| 65 ++++++++++++++++---------------------------------
	18	3 files changed, 28 insertions(+), 47 deletions(-)
	19
	20
	21	diff --git a/CHANGELOG b/CHANGELOG
	22	index d4dd70b..afd1335 100644
	23	--- a/CHANGELOG
	24	+++ b/CHANGELOG
	25	@@ -14,6 +14,7 @@
	26	- check for stale SASL credentials upon connect fail.
	27	- add "forcestart" and "forcerestart" init script options to allow
	28	use of 5.0.3 strartup behavior if required.
	29	+- always read entire file map into cache to speed lookups.
	30
	31	4/11/2008 autofs-5.0.4
	32	-----------------------
	33	diff --git a/daemon/lookup.c b/daemon/lookup.c
	34	index 741d846..e034348 100644
	35	--- a/daemon/lookup.c
	36	+++ b/daemon/lookup.c
	37	@@ -283,10 +283,13 @@ static int do_read_map(struct autofs_point ap, struct map_source map, time_t a
	38	* for the fail cases to function correctly and to cache the
	39	* lookup handle.
	40	*
	41	- * We always need to whole map for direct mounts in order to
	42	- * mount the triggers.
	43	+ * We always need to read the whole map for direct mounts in
	44	+ * order to mount the triggers. We also want to read the whole
	45	+ * map if it's a file map to avoid potentially lengthy linear
	46	+ * file scanning.
	47	*/
	48	- if (!(ap->flags & MOUNT_FLAG_GHOST) && ap->type != LKP_DIRECT)
	49	+ if (strcmp(map->type, "file") &&
	50	+ !(ap->flags & MOUNT_FLAG_GHOST) && ap->type != LKP_DIRECT)
	51	return NSS_STATUS_SUCCESS;
	52
	53	if (!map->stale)
	54	diff --git a/modules/lookup_file.c b/modules/lookup_file.c
	55	index 95b9f6f..aafeb8b 100644
	56	--- a/modules/lookup_file.c
	57	+++ b/modules/lookup_file.c
	58	@@ -44,7 +44,6 @@ typedef enum { esc_none, esc_char, esc_val, esc_all } ESCAPES;
	59
	60	struct lookup_context {
	61	const char *mapname;
	62	- time_t mtime;
	63	struct parse_mod *parse;
	64	};
65
66	@@ -54,7 +53,6 @@ int lookup_init(const char mapfmt, int argc, const char const argv, void *co
67	{
68	struct lookup_context *ctxt;
69	char buf[MAX_ERR_BUF];
70	- struct stat st;
71
72	*context = NULL;
73
74	@@ -87,15 +85,6 @@ int lookup_init(const char mapfmt, int argc, const char const argv, void *co
75	return 1;
76	}
77
78	- if (stat(ctxt->mapname, &st)) {
79	- free(ctxt);
80	- logmsg(MODPREFIX "file map %s, could not stat",
81	- argv[0]);
82	- return 1;
83	- }
84	-
85	- ctxt->mtime = st.st_mtime;
86	-
87	if (!mapfmt)
88	mapfmt = MAPFMT_DEFAULT;
89
90	@@ -391,9 +380,7 @@ int lookup_read_master(struct master master, time_t age, void context)
91	int blen;
92	char *path;
93	char *ent;
94	- struct stat st;
95	FILE *f;
96	- int fd;
97	unsigned int path_len, ent_len;
98	int entry, cur_state;
99
100	@@ -428,8 +415,6 @@ int lookup_read_master(struct master master, time_t age, void context)
101	return NSS_STATUS_UNAVAIL;
102	}
103
104	- fd = fileno(f);
105	-
106	while(1) {
107	entry = read_one(logopt, f, path, &path_len, ent, &ent_len);
108	if (!entry) {
109	@@ -504,13 +489,6 @@ int lookup_read_master(struct master master, time_t age, void context)
110	break;
111	}
112
113	- if (fstat(fd, &st)) {
114	- crit(logopt, MODPREFIX "file map %s, could not stat",
115	- ctxt->mapname);
116	- return NSS_STATUS_UNAVAIL;
117	- }
118	- ctxt->mtime = st.st_mtime;
119	-
120	fclose(f);
121
122	return NSS_STATUS_SUCCESS;
123	@@ -642,9 +620,7 @@ int lookup_read_map(struct autofs_point ap, time_t age, void context)
124	struct mapent_cache *mc;
125	char *key;
126	char *mapent;
127	- struct stat st;
128	FILE *f;
129	- int fd;
130	unsigned int k_len, m_len;
131	int entry;
132
133	@@ -684,8 +660,6 @@ int lookup_read_map(struct autofs_point ap, time_t age, void context)
134	return NSS_STATUS_UNAVAIL;
135	}
136
137	- fd = fileno(f);
138	-
139	while(1) {
140	entry = read_one(ap->logopt, f, key, &k_len, mapent, &m_len);
141	if (!entry) {
142	@@ -748,13 +722,6 @@ int lookup_read_map(struct autofs_point ap, time_t age, void context)
143	break;
144	}
145
146	- if (fstat(fd, &st)) {
147	- crit(ap->logopt,
148	- MODPREFIX "file map %s, could not stat",
149	- ctxt->mapname);
150	- return NSS_STATUS_UNAVAIL;
151	- }
152	- ctxt->mtime = st.st_mtime;
153	source->age = age;
154
155	fclose(f);
156	@@ -951,9 +918,6 @@ static int check_map_indirect(struct autofs_point *ap,
157	if (ret == CHE_FAIL)
158	return NSS_STATUS_NOTFOUND;
159
160	- if (ret & CHE_UPDATED)
161	- source->stale = 1;
162	-
163	pthread_cleanup_push(cache_lock_cleanup, mc);
164	cache_writelock(mc);
165	exists = cache_lookup_distinct(mc, key);
166	@@ -963,7 +927,6 @@ static int check_map_indirect(struct autofs_point *ap,
167	free(exists->mapent);
168	exists->mapent = NULL;
169	exists->status = 0;
170	- source->stale = 1;
171	}
172	}
173	pthread_cleanup_pop(1);
174	@@ -985,14 +948,8 @@ static int check_map_indirect(struct autofs_point *ap,
175	we = cache_lookup_distinct(mc, "*");
176	if (we) {
177	/* Wildcard entry existed and is now gone */
178	- if (we->source == source && (wild & CHE_MISSING)) {
179	+ if (we->source == source && (wild & CHE_MISSING))
180	cache_delete(mc, "*");
181	- source->stale = 1;
182	- }
183	- } else {
184	- /* Wildcard not in map but now is */
185	- if (wild & (CHE_OK \| CHE_UPDATED))
186	- source->stale = 1;
187	}
188	pthread_cleanup_pop(1);
189
190	@@ -1062,9 +1019,28 @@ int lookup_mount(struct autofs_point ap, const char name, int name_len, void *
191	* we never know about it.
192	*/
193	if (ap->type == LKP_INDIRECT && *key != '/') {
194	+ struct stat st;
195	char *lkp_key;
196
197	+ /*
198	+ * We can skip the map lookup and cache update altogether
199	+ * if we know the map hasn't been modified since it was
200	+ * last read. If it has then we can mark the map stale
201	+ * so a re-read is triggered following the lookup.
202	+ */
203	+ if (stat(ctxt->mapname, &st)) {
204	+ error(ap->logopt, MODPREFIX
205	+ "file map %s, could not stat", ctxt->mapname);
206	+ return NSS_STATUS_UNAVAIL;
207	+ }
208	+
209	cache_readlock(mc);
210	+ me = cache_lookup_first(mc);
211	+ if (me && st.st_mtime <= me->age)
212	+ goto do_cache_lookup;
213	+ else
214	+ source->stale = 1;
215	+
216	me = cache_lookup_distinct(mc, key);
217	if (me && me->multi)
218	lkp_key = strdup(me->multi->key);
219	@@ -1088,6 +1064,7 @@ int lookup_mount(struct autofs_point ap, const char name, int name_len, void *
220	}
221
222	cache_readlock(mc);
223	+do_cache_lookup:
224	me = cache_lookup(mc, key);
225	/* Stale mapent => check for entry in alternate source or wildcard */
226	if (me && !me->mapent) {