From 5e5e320d361afcb4d9503354b3912b4c7a672197 Mon Sep 17 00:00:00 2001 From: Jeremy Allison Date: Thu, 27 Aug 1998 20:38:53 +0000 Subject: This is the stat cache code - seems to work fine (needs heavy NetBench testing though.... :-). Attempts to efficiently reduce the number of stat() calls Samba does. Jeremy. (This used to be commit d0e48a2d8072c3e77a57ac6a2fb5044c05f03b41) --- source3/smbd/filename.c | 290 +++++++++++++++++++++++++++++++++++++++++++++--- source3/smbd/nttrans.c | 4 +- source3/smbd/password.c | 4 - source3/smbd/process.c | 4 + source3/smbd/reply.c | 44 ++++---- source3/smbd/server.c | 2 +- source3/smbd/trans2.c | 14 +-- 7 files changed, 312 insertions(+), 50 deletions(-) (limited to 'source3/smbd') diff --git a/source3/smbd/filename.c b/source3/smbd/filename.c index ab5851fb5e..8ef2eef96a 100644 --- a/source3/smbd/filename.c +++ b/source3/smbd/filename.c @@ -29,10 +29,10 @@ extern fstring remote_machine; extern BOOL use_mangled_map; /**************************************************************************** -check if two filenames are equal - -this needs to be careful about whether we are case sensitive + Check if two filenames are equal. + This needs to be careful about whether we are case sensitive. ****************************************************************************/ + BOOL fname_equal(char *name1, char *name2) { int l1 = strlen(name1); @@ -66,8 +66,9 @@ BOOL fname_equal(char *name1, char *name2) /**************************************************************************** -mangle the 2nd name and check if it is then equal to the first name + Mangle the 2nd name and check if it is then equal to the first name. ****************************************************************************/ + BOOL mangled_equal(char *name1, char *name2) { pstring tmpname; @@ -81,6 +82,215 @@ BOOL mangled_equal(char *name1, char *name2) return(strequal(name1,tmpname)); } +/**************************************************************************** + Stat cache code used in unix_convert. +*****************************************************************************/ + +static int global_stat_cache_lookups; +static int global_stat_cache_misses; +static int global_stat_cache_hits; + +/**************************************************************************** + Stat cache statistics code. +*****************************************************************************/ + +void print_stat_cache_statistics(void) +{ + double eff = ((double)global_stat_cache_lookups/100.0)*(double)global_stat_cache_hits; + + DEBUG(0,("stat cache stats: lookups = %d, hits = %d, misses = %d, \ +stat cache was %f%% effective.\n", global_stat_cache_lookups, + global_stat_cache_hits, global_stat_cache_misses, eff )); +} + +typedef struct { + ubi_dlNode link; + int name_len; + pstring orig_name; + pstring translated_name; +} stat_cache_entry; + +#define MAX_STAT_CACHE_SIZE 50 + +static ubi_dlList stat_cache = { NULL, (ubi_dlNodePtr)&stat_cache, 0}; + +/**************************************************************************** + Compare two names in the stat cache. +*****************************************************************************/ + +static BOOL stat_name_equal( char *s1, char *s2) +{ + return (case_sensitive ? (strcmp( s1, s2) == 0) : (StrCaseCmp(s1, s2) == 0)); +} + +/**************************************************************************** + Compare two names in the stat cache. +*****************************************************************************/ + +static BOOL stat_name_equal_len( char *s1, char *s2, int len) +{ + return (case_sensitive ? (strncmp( s1, s2, len) == 0) : + (StrnCaseCmp(s1, s2, len) == 0)); +} + +/**************************************************************************** + Add an entry into the stat cache. +*****************************************************************************/ + +static void stat_cache_add( char *full_orig_name, char *orig_translated_path) +{ + stat_cache_entry *scp; + pstring orig_name; + pstring translated_path; + int namelen = strlen(orig_translated_path); + + /* + * Don't cache trivial valid directory entries. + */ + if(strequal(full_orig_name, ".") || strequal(full_orig_name, "..")) + return; + + /* + * If we are in case insentive mode, we need to + * store names that need no translation - else, it + * would be a waste. + */ + + if(case_sensitive && (strcmp(full_orig_name, orig_translated_path) == 0)) + return; + + /* + * Remove any trailing '/' characters from the + * translated path. + */ + + pstrcpy(translated_path, orig_translated_path); + if(translated_path[namelen-1] == '/') { + translated_path[namelen-1] = '\0'; + namelen--; + } + + /* + * We will only replace namelen characters + * of full_orig_name. + * StrnCpy always null terminates. + */ + + StrnCpy(orig_name, full_orig_name, namelen); + + /* + * Check this name doesn't exist in the cache before we + * add it. + */ + + for( scp = (stat_cache_entry *)ubi_dlFirst( &stat_cache); scp; + scp = (stat_cache_entry *)ubi_dlNext( scp )) { + if(stat_name_equal( scp->orig_name, orig_name) && + (strcmp( scp->translated_name, translated_path) == 0)) { + /* + * Name does exist - promote it. + */ + if( (stat_cache_entry *)ubi_dlFirst( &stat_cache) != scp ) { + ubi_dlRemThis( &stat_cache, scp); + ubi_dlAddHead( &stat_cache, scp); + } + return; + } + } + + if((scp = (stat_cache_entry *)malloc(sizeof(stat_cache_entry))) == NULL) { + DEBUG(0,("stat_cache_add: Out of memory !\n")); + return; + } + + pstrcpy(scp->orig_name, orig_name); + pstrcpy(scp->translated_name, translated_path); + scp->name_len = namelen; + + ubi_dlAddHead( &stat_cache, scp); + + DEBUG(10,("stat_cache_add: Added entry %s -> %s\n", scp->orig_name, scp->translated_name )); + + if(ubi_dlCount(&stat_cache) > MAX_STAT_CACHE_SIZE) { + scp = (stat_cache_entry *)ubi_dlRemTail( &stat_cache ); + free((char *)scp); + return; + } +} + +/**************************************************************************** + Look through the stat cache for an entry - promote it to the top if found. + Return True if we translated (and did a scuccessful stat on) the entire name. +*****************************************************************************/ + +static BOOL stat_cache_lookup( char *name, char *dirpath, char **start, struct stat *pst) +{ + stat_cache_entry *scp; + stat_cache_entry *longest_hit = NULL; + int namelen = strlen(name); + + *start = name; + global_stat_cache_lookups++; + + /* + * Don't lookup trivial valid directory entries. + */ + if(strequal(name, ".") || strequal(name, "..")) { + global_stat_cache_misses++; + return False; + } + + for( scp = (stat_cache_entry *)ubi_dlFirst( &stat_cache); scp; + scp = (stat_cache_entry *)ubi_dlNext( scp )) { + if(scp->name_len <= namelen) { + if(stat_name_equal_len(scp->orig_name, name, scp->name_len)) { + if((longest_hit == NULL) || (longest_hit->name_len <= scp->name_len)) + longest_hit = scp; + } + } + } + + if(longest_hit == NULL) { + DEBUG(10,("stat_cache_lookup: cache miss on %s\n", name)); + global_stat_cache_misses++; + return False; + } + + global_stat_cache_hits++; + + DEBUG(10,("stat_cache_lookup: cache hit for name %s. %s -> %s\n", + name, longest_hit->orig_name, longest_hit->translated_name )); + + /* + * longest_hit is the longest match we got in the list. + * Check it exists - if so, overwrite the original name + * and then promote it to the top. + */ + + if(sys_stat( longest_hit->translated_name, pst) != 0) { + /* + * Discard this entry. + */ + ubi_dlRemThis( &stat_cache, longest_hit); + free((char *)longest_hit); + return False; + } + + memcpy(name, longest_hit->translated_name, longest_hit->name_len); + if( (stat_cache_entry *)ubi_dlFirst( &stat_cache) != longest_hit ) { + ubi_dlRemThis( &stat_cache, longest_hit); + ubi_dlAddHead( &stat_cache, longest_hit); + } + + *start = &name[longest_hit->name_len]; + if(**start == '/') + ++*start; + + StrnCpy( dirpath, longest_hit->translated_name, name - (*start)); + + return (namelen == longest_hit->name_len); +} + /**************************************************************************** This routine is called to convert names from the dos namespace to unix namespace. It needs to handle any case conversions, mangling, format @@ -103,15 +313,21 @@ as Windows applications depend on ERRbadpath being returned if a component of a pathname does not exist. ****************************************************************************/ -BOOL unix_convert(char *name,connection_struct *conn,char *saved_last_component, BOOL *bad_path) +BOOL unix_convert(char *name,connection_struct *conn,char *saved_last_component, + BOOL *bad_path, struct stat *pst) { struct stat st; - char *start, *end; + char *start, *end, *orig_start; pstring dirpath; + pstring orig_path; int saved_errno; + BOOL component_was_mangled = False; + BOOL name_has_wildcard = False; *dirpath = 0; *bad_path = False; + if(pst) + memset( (char *)pst, '\0', sizeof(struct stat)); if(saved_last_component) *saved_last_component = 0; @@ -166,16 +382,34 @@ BOOL unix_convert(char *name,connection_struct *conn,char *saved_last_component, return(True); } + start = name; + while (strncmp(start,"./",2) == 0) + start += 2; + + pstrcpy(orig_path, name); + + if(stat_cache_lookup( name, dirpath, &start, &st)) { + if(pst) + *pst = st; + return True; + } + /* * stat the name - if it exists then we are all done! */ - if (sys_stat(name,&st) == 0) + if (sys_stat(name,&st) == 0) { + stat_cache_add(orig_path, name); + DEBUG(5,("conversion finished %s -> %s\n",orig_path, name)); + if(pst) + *pst = st; return(True); + } saved_errno = errno; - DEBUG(5,("unix_convert(%s)\n",name)); + DEBUG(5,("unix_convert begin: name = %s, dirpath = %s, start = %s\n", + name, dirpath, start)); /* * A special case - if we don't have any mangling chars and are case @@ -186,21 +420,20 @@ BOOL unix_convert(char *name,connection_struct *conn,char *saved_last_component, !lp_strip_dot() && !use_mangled_map && (saved_errno != ENOENT)) return(False); + if(strchr(start,'?') || strchr(start,'*')) + name_has_wildcard = True; + /* * Now we need to recursively match the name against the real * directory structure. */ - start = name; - while (strncmp(start,"./",2) == 0) - start += 2; - /* * Match each part of the path name separately, trying the names * as is first, then trying to scan the directory for matching names. */ - for (;start;start = (end?end+1:(char *)NULL)) { + for (orig_start = start; start ; start = (end?end+1:(char *)NULL)) { /* * Pinpoint the end of this section of the filename. */ @@ -231,6 +464,7 @@ BOOL unix_convert(char *name,connection_struct *conn,char *saved_last_component, *end = '/'; return(False); } + } else { pstring rest; @@ -247,6 +481,7 @@ BOOL unix_convert(char *name,connection_struct *conn,char *saved_last_component, /* * Try to find this part of the path in the directory. */ + if (strchr(start,'?') || strchr(start,'*') || !scan_directory(dirpath, start, conn, end?True:False)) { if (end) { @@ -283,8 +518,10 @@ BOOL unix_convert(char *name,connection_struct *conn,char *saved_last_component, * base of the filename. */ - if (is_mangled(start)) + if (is_mangled(start)) { + component_was_mangled = True; check_mangled_cache( start ); + } DEBUG(5,("New file %s\n",start)); return(True); @@ -302,8 +539,18 @@ BOOL unix_convert(char *name,connection_struct *conn,char *saved_last_component, /* * Add to the dirpath that we have resolved so far. */ - if (*dirpath) pstrcat(dirpath,"/"); - pstrcat(dirpath,start); + if (*dirpath) + pstrcat(dirpath,"/"); + + pstrcat(dirpath,start); + + /* + * Don't cache a name with mangled or wildcard components + * as this can change the size. + */ + + if(!component_was_mangled && !name_has_wildcard) + stat_cache_add(orig_path, dirpath); /* * Restore the / that we wiped out earlier. @@ -312,10 +559,19 @@ BOOL unix_convert(char *name,connection_struct *conn,char *saved_last_component, *end = '/'; } + /* + * Don't cache a name with mangled or wildcard components + * as this can change the size. + */ + + if(!component_was_mangled && !name_has_wildcard) + stat_cache_add(orig_path, name); + /* * The name has been resolved. */ - DEBUG(5,("conversion finished %s\n",name)); + + DEBUG(5,("conversion finished %s -> %s\n",orig_path, name)); return(True); } diff --git a/source3/smbd/nttrans.c b/source3/smbd/nttrans.c index 596d15c2c7..fbcc19e77d 100644 --- a/source3/smbd/nttrans.c +++ b/source3/smbd/nttrans.c @@ -490,7 +490,7 @@ int reply_ntcreate_and_X(connection_struct *conn, set_posix_case_semantics(file_attributes); - unix_convert(fname,conn,0,&bad_path); + unix_convert(fname,conn,0,&bad_path,NULL); fsp = file_new(); if (!fsp) { @@ -743,7 +743,7 @@ static int call_nt_transact_create(connection_struct *conn, set_posix_case_semantics(file_attributes); - unix_convert(fname,conn,0,&bad_path); + unix_convert(fname,conn,0,&bad_path,NULL); fsp = file_new(); if (!fsp) { diff --git a/source3/smbd/password.c b/source3/smbd/password.c index ecc86f29ec..4ee9e8705d 100644 --- a/source3/smbd/password.c +++ b/source3/smbd/password.c @@ -21,10 +21,6 @@ #include "includes.h" -#if (defined(HAVE_NETGROUP) && defined (WITH_AUTOMOUNT)) -#include "rpcsvc/ypclnt.h" -#endif - extern int DEBUGLEVEL; extern int Protocol; diff --git a/source3/smbd/process.c b/source3/smbd/process.c index 5bf8cdb2b1..63e51dc242 100644 --- a/source3/smbd/process.c +++ b/source3/smbd/process.c @@ -723,6 +723,10 @@ void smbd_process(void) DEBUG(0,("Reloading services after SIGHUP\n")); reload_services(False); reload_after_sighup = False; + /* + * Use this as an excuse to print some stats. + */ + print_stat_cache_statistics(); } /* automatic timeout if all connections are closed */ diff --git a/source3/smbd/reply.c b/source3/smbd/reply.c index 3e59e7dbd0..3052bd730c 100644 --- a/source3/smbd/reply.c +++ b/source3/smbd/reply.c @@ -751,14 +751,19 @@ int reply_chkpth(connection_struct *conn, char *inbuf,char *outbuf, int dum_size pstring name; BOOL ok = False; BOOL bad_path = False; + struct stat st; pstrcpy(name,smb_buf(inbuf) + 1); - unix_convert(name,conn,0,&bad_path); + unix_convert(name,conn,0,&bad_path,&st); mode = SVAL(inbuf,smb_vwv0); - if (check_name(name,conn)) - ok = directory_exist(name,NULL); + if (check_name(name,conn)) { + if(VALID_STAT(st)) + ok = S_ISDIR(st.st_mode); + else + ok = directory_exist(name,NULL); + } if (!ok) { @@ -809,7 +814,7 @@ int reply_getatr(connection_struct *conn, char *inbuf,char *outbuf, int dum_size BOOL bad_path = False; pstrcpy(fname,smb_buf(inbuf) + 1); - unix_convert(fname,conn,0,&bad_path); + unix_convert(fname,conn,0,&bad_path,&sbuf); /* dos smetimes asks for a stat of "" - it returns a "hidden directory" under WfWg - weird! */ @@ -824,7 +829,7 @@ int reply_getatr(connection_struct *conn, char *inbuf,char *outbuf, int dum_size else if (check_name(fname,conn)) { - if (sys_stat(fname,&sbuf) == 0) + if (VALID_STAT(sbuf) || sys_stat(fname,&sbuf) == 0) { mode = dos_mode(conn,fname,&sbuf); size = sbuf.st_size; @@ -881,15 +886,16 @@ int reply_setatr(connection_struct *conn, char *inbuf,char *outbuf, int dum_size BOOL ok=False; int mode; time_t mtime; + struct stat st; BOOL bad_path = False; pstrcpy(fname,smb_buf(inbuf) + 1); - unix_convert(fname,conn,0,&bad_path); + unix_convert(fname,conn,0,&bad_path,&st); mode = SVAL(inbuf,smb_vwv0); mtime = make_unix_date3(inbuf+smb_vwv1); - if (directory_exist(fname,NULL)) + if (VALID_STAT_OF_DIR(st) || directory_exist(fname,NULL)) mode |= aDIR; if (check_name(fname,conn)) ok = (dos_chmod(conn,fname,mode,NULL) == 0); @@ -990,7 +996,7 @@ int reply_search(connection_struct *conn, char *inbuf,char *outbuf, int dum_size pstrcpy(directory,smb_buf(inbuf)+1); pstrcpy(dir2,smb_buf(inbuf)+1); - unix_convert(directory,conn,0,&bad_path); + unix_convert(directory,conn,0,&bad_path,NULL); unix_format(dir2); if (!check_name(directory,conn)) @@ -1250,7 +1256,7 @@ int reply_open(connection_struct *conn, char *inbuf,char *outbuf, int dum_size, share_mode = SVAL(inbuf,smb_vwv0); pstrcpy(fname,smb_buf(inbuf)+1); - unix_convert(fname,conn,0,&bad_path); + unix_convert(fname,conn,0,&bad_path,NULL); fsp = file_new(); if (!fsp) @@ -1351,7 +1357,7 @@ int reply_open_and_X(connection_struct *conn, char *inbuf,char *outbuf,int lengt /* XXXX we need to handle passed times, sattr and flags */ pstrcpy(fname,smb_buf(inbuf)); - unix_convert(fname,conn,0,&bad_path); + unix_convert(fname,conn,0,&bad_path,NULL); fsp = file_new(); if (!fsp) @@ -1485,7 +1491,7 @@ int reply_mknew(connection_struct *conn, char *inbuf,char *outbuf, int dum_size, createmode = SVAL(inbuf,smb_vwv0); pstrcpy(fname,smb_buf(inbuf)+1); - unix_convert(fname,conn,0,&bad_path); + unix_convert(fname,conn,0,&bad_path,NULL); if (createmode & aVOLID) { @@ -1570,7 +1576,7 @@ int reply_ctemp(connection_struct *conn, char *inbuf,char *outbuf, int dum_size, createmode = SVAL(inbuf,smb_vwv0); pstrcpy(fname,smb_buf(inbuf)+1); pstrcat(fname,"/TMXXXXXX"); - unix_convert(fname,conn,0,&bad_path); + unix_convert(fname,conn,0,&bad_path,NULL); unixmode = unix_mode(conn,createmode); @@ -1674,7 +1680,7 @@ int reply_unlink(connection_struct *conn, char *inbuf,char *outbuf, int dum_size DEBUG(3,("reply_unlink : %s\n",name)); - unix_convert(name,conn,0,&bad_path); + unix_convert(name,conn,0,&bad_path,NULL); p = strrchr(name,'/'); if (!p) { @@ -2778,7 +2784,7 @@ int reply_mkdir(connection_struct *conn, char *inbuf,char *outbuf, int dum_size, BOOL bad_path = False; pstrcpy(directory,smb_buf(inbuf) + 1); - unix_convert(directory,conn,0,&bad_path); + unix_convert(directory,conn,0,&bad_path,NULL); if (check_name(directory, conn)) ret = sys_mkdir(directory,unix_mode(conn,aDIR)); @@ -2872,7 +2878,7 @@ int reply_rmdir(connection_struct *conn, char *inbuf,char *outbuf, int dum_size, BOOL bad_path = False; pstrcpy(directory,smb_buf(inbuf) + 1); - unix_convert(directory,conn, NULL,&bad_path); + unix_convert(directory,conn, NULL,&bad_path,NULL); if (check_name(directory,conn)) { @@ -3072,8 +3078,8 @@ int rename_internals(connection_struct *conn, *directory = *mask = 0; - unix_convert(name,conn,0,&bad_path1); - unix_convert(newname,conn,newname_last_component,&bad_path2); + unix_convert(name,conn,0,&bad_path1,NULL); + unix_convert(newname,conn,newname_last_component,&bad_path2,NULL); /* * Split the old name into directory and last component @@ -3373,8 +3379,8 @@ int reply_copy(connection_struct *conn, char *inbuf,char *outbuf, int dum_size, return(ERROR(ERRSRV,ERRinvdevice)); } - unix_convert(name,conn,0,&bad_path1); - unix_convert(newname,conn,0,&bad_path2); + unix_convert(name,conn,0,&bad_path1,NULL); + unix_convert(newname,conn,0,&bad_path2,NULL); target_is_directory = directory_exist(newname,NULL); diff --git a/source3/smbd/server.c b/source3/smbd/server.c index a08ff8184e..4c38fb5f4b 100644 --- a/source3/smbd/server.c +++ b/source3/smbd/server.c @@ -287,7 +287,7 @@ max can be %d\n", } /* end for num */ } /* end while 1 */ - return True; +/* NOTREACHED return True; */ } /**************************************************************************** diff --git a/source3/smbd/trans2.c b/source3/smbd/trans2.c index fe22a21ca1..da11dbcb29 100644 --- a/source3/smbd/trans2.c +++ b/source3/smbd/trans2.c @@ -214,7 +214,7 @@ static int call_trans2open(connection_struct *conn, char *inbuf, char *outbuf, /* XXXX we need to handle passed times, sattr and flags */ - unix_convert(fname,conn,0,&bad_path); + unix_convert(fname,conn,0,&bad_path,NULL); fsp = file_new(); if (!fsp) @@ -659,7 +659,7 @@ static int call_trans2findfirst(connection_struct *conn, DEBUG(5,("path=%s\n",directory)); - unix_convert(directory,conn,0,&bad_path); + unix_convert(directory,conn,0,&bad_path,NULL); if(!check_name(directory,conn)) { if((errno == ENOENT) && bad_path) { @@ -1225,8 +1225,8 @@ static int call_trans2qfilepathinfo(connection_struct *conn, info_level = SVAL(params,0); fname = &fname1[0]; pstrcpy(fname,¶ms[6]); - unix_convert(fname,conn,0,&bad_path); - if (!check_name(fname,conn) || sys_stat(fname,&sbuf)) { + unix_convert(fname,conn,0,&bad_path,&sbuf); + if (!check_name(fname,conn) || (!VALID_STAT(sbuf) && sys_stat(fname,&sbuf))) { DEBUG(3,("fileinfo of %s failed (%s)\n",fname,strerror(errno))); if((errno == ENOENT) && bad_path) { @@ -1459,7 +1459,7 @@ static int call_trans2setfilepathinfo(connection_struct *conn, info_level = SVAL(params,0); fname = fname1; pstrcpy(fname,¶ms[6]); - unix_convert(fname,conn,0,&bad_path); + unix_convert(fname,conn,0,&bad_path,&st); if(!check_name(fname, conn)) { if((errno == ENOENT) && bad_path) @@ -1470,7 +1470,7 @@ static int call_trans2setfilepathinfo(connection_struct *conn, return(UNIXERROR(ERRDOS,ERRbadpath)); } - if(sys_stat(fname,&st)!=0) { + if(!VALID_STAT(st) && sys_stat(fname,&st)!=0) { DEBUG(3,("stat of %s failed (%s)\n", fname, strerror(errno))); if((errno == ENOENT) && bad_path) { @@ -1648,7 +1648,7 @@ static int call_trans2mkdir(connection_struct *conn, DEBUG(3,("call_trans2mkdir : name = %s\n", directory)); - unix_convert(directory,conn,0,&bad_path); + unix_convert(directory,conn,0,&bad_path,NULL); if (check_name(directory,conn)) ret = sys_mkdir(directory,unix_mode(conn,aDIR)); -- cgit