Cachepath work broken out into functional commits

config.h.in

@@ -6,6 +6,9 @@
 /* Whether we are using a broken srun */
 #undef BROKEN_SRUN
 
+/* Colon-separated list of potential back-end cache directories */
+#undef CACHEPATHS
+
 /* Define if were using biter for client/server communication */
 #undef COMM_BITER
 

configure

@@ -846,6 +846,7 @@ enable_libtool_lock
 enable_maintainer_mode
 with_default_port
 with_default_num_ports
+with_cachepaths
 with_localstorage
 with_default_local_prefix
 with_testrm
@@ -1589,6 +1590,8 @@ Optional Packages:
   --with-default-numports=NUM
                           Number of TCP/IP ports to scan for Spindle server
                           communication
+  --with-cachepaths=DIR   Colon-separated list of potential back-end cache
+                          directories
   --with-localstorage=DIR Directory on back-ends for storing relocated files
   --with-default-local-prefix=DIRS
                           Colon-seperated list of directories that Spindle
@@ -16615,6 +16618,14 @@ else
 fi
 
 
+# Check whether --with-cachepaths was given.
+if test "${with_cachepaths+set}" = set; then :
+  withval=$with_cachepaths; CACHEPATHS=${withval}
+else
+  CACHEPATHS=$DEFAULT_LOC
+fi
+
+
 # Check whether --with-localstorage was given.
 if test "${with_localstorage+set}" = set; then :
   withval=$with_localstorage; SPINDLE_LOC=${withval}
@@ -16651,6 +16662,11 @@ cat >>confdefs.h <<_ACEOF
 _ACEOF
 
 
+cat >>confdefs.h <<_ACEOF
+#define CACHEPATHS "$CACHEPATHS"
+_ACEOF
+
+
 cat >>confdefs.h <<_ACEOF
 #define SPINDLE_LOCAL_PREFIX "$SPINDLE_LOCAL_PREFIX"
 _ACEOF

configure.common.ac

@@ -17,6 +17,10 @@ AC_ARG_WITH(default-num-ports,
             [AS_HELP_STRING([--with-default-numports=NUM],[Number of TCP/IP ports to scan for Spindle server communication])],
             [NUM_COBO_PORTS=${withval}],
             [NUM_COBO_PORTS=$DEFAULT_NUM_COBO_PORTS])
+AC_ARG_WITH(cachepaths,
+            [AS_HELP_STRING([--with-cachepaths=DIR],[Colon-separated list of potential back-end cache directories])],
+            [CACHEPATHS=${withval}],
+            [CACHEPATHS=$DEFAULT_LOC])
 AC_ARG_WITH(localstorage,
             [AS_HELP_STRING([--with-localstorage=DIR],[Directory on back-ends for storing relocated files])],
             [SPINDLE_LOC=${withval}],
@@ -29,6 +33,7 @@ AC_DEFINE_UNQUOTED([SPINDLE_PORT],[$SPINDLE_PORT],[The default port for Spindle]
 AC_DEFINE_UNQUOTED([NUM_COBO_PORTS],[$NUM_COBO_PORTS],[Number of ports for COBO to search for an open port])
 AC_DEFINE_UNQUOTED([SPINDLE_MAX_PORT],[$(($SPINDLE_PORT + $NUM_COBO_PORTS - 1))],[The maximum port value])
 AC_DEFINE_UNQUOTED([SPINDLE_LOC],"[$SPINDLE_LOC]",[The default local directory for Spindle])
+AC_DEFINE_UNQUOTED([CACHEPATHS],"[$CACHEPATHS]",[Colon-separated list of potential back-end cache directories])
 AC_DEFINE_UNQUOTED([SPINDLE_LOCAL_PREFIX],"[$SPINDLE_LOCAL_PREFIX]",[The default colon-separated list of directories that Spindle will not cache files out of])
 
 TESTRM=unknown

src/client/beboot/spindle_bootstrap.c

@@ -114,7 +114,6 @@ static void setup_environment()
 
    setenv("LD_AUDIT", client_lib, 1);
    setenv("LDCS_LOCATION", location, 1);
-   setenv("LDCS_ORIG_LOCATION", orig_location, 1);
    setenv("LDCS_NUMBER", number_s, 1);
    setenv("LDCS_RANKINFO", rankinfo_str, 1);
    if (connection_str)
@@ -161,6 +160,7 @@ static int parse_cmdline(int argc, char *argv[])
    }
 
    symbolic_location = argv[i++];
+   i++; // Skip over candidate_cachepaths.
    number_s = argv[i++];
    number = (number_t) strtoul(number_s, NULL, 0);
    opts_s = argv[i++];

src/client/client/client.c

@@ -40,6 +40,7 @@ Place, Suite 330, Boston, MA 02111-1307 USA
 #include "ccwarns.h"
 #include "exec_util.h"
 #include "intercept.h"
+#include "should_intercept.h"
 
 errno_location_t app_errno_location;
 
@@ -69,11 +70,8 @@ static const ElfW(Phdr) *libc_phdrs, *interp_phdrs;
 static int num_libc_phdrs, num_interp_phdrs;
 ElfW(Addr) libc_loadoffset, interp_loadoffset;
 
-/* location has the realize'd path to the local file cache. orig_location is not realized and
- * may contain symlinks
- */
-char *location;
-char *orig_location;
+static char *location;
+char *chosen_realized_cachepath, *chosen_parsed_cachepath;
 number_t number;
 static int have_stat_patches;
 
@@ -198,7 +196,6 @@ static int init_server_connection()
       return 0;
 
    location = getenv("LDCS_LOCATION");
-   orig_location = getenv("LDCS_ORIG_LOCATION");
    number = (number_t) strtoul(getenv("LDCS_NUMBER"), NULL, 0);
    connection = getenv("LDCS_CONNECTION");
    rankinfo_s = getenv("LDCS_RANKINFO");
@@ -218,7 +215,6 @@ static int init_server_connection()
       debug_printf("Disabling environment variables because we're not following forks\n");
       unsetenv("LD_AUDIT");
       unsetenv("LDCS_LOCATION");
-      unsetenv("LDCS_ORIG_LOCATION");
       unsetenv("LDCS_NUMBER");
       unsetenv("LDCS_CONNECTION");
       unsetenv("LDCS_RANKINFO");
@@ -262,7 +258,7 @@ static int init_server_connection()
          send_cpu(ldcsid, get_cur_cpu());
 #endif
    }
-
+   send_cachepath_query( ldcsid, &chosen_realized_cachepath, &chosen_parsed_cachepath );
    snprintf(debugging_name, 32, "Client.%d", rankinfo[0]);
    LOGGING_INIT(debugging_name);
 
@@ -467,7 +463,7 @@ char *client_library_load(const char *name)
 
    char *orig_file_name = (char *) name;
    if (is_in_spindle_cache(name)) {
-      debug_printf2("Library %s is in spindle cache (%s). Translating request\n", name, location);
+      debug_printf2("Library %s is in spindle cache (%s). Translating request\n", name, chosen_realized_cachepath);
       memset(fixed_name, 0, MAX_PATH_LEN+1);
       send_orig_path_request(ldcsid, orig_file_name, fixed_name);
       orig_file_name = fixed_name;

src/client/client/intercept_exec.c

@@ -142,7 +142,6 @@ static char **removeEnvironmentStrs(char **envp)
       if (strIsPrefix("LD", envp[i])) {
          if (strIsPrefix("LD_AUDIT=", envp[i]) ||
              strIsPrefix("LDCS_LOCATION=", envp[i]) ||
-             strIsPrefix("LDCS_ORIG_LOCATION=", envp[i]) ||
              strIsPrefix("LDCS_CONNECTION=", envp[i]) ||
              strIsPrefix("LDCS_RANKINFO=", envp[i]) ||
              strIsPrefix("LDCS_OPTIONS=", envp[i]) ||
@@ -177,7 +176,6 @@ static char **updateEnvironment(char **envp, int *num_modified, int propogate_sp
          unsetf("SPINDLE");
          unsetf("LD_AUDIT");
          unsetf("LDCS_LOCATION");
-         unsetf("LDCS_ORIG_LOCATION");
          unsetf("LDCS_CONNECTION");
          unsetf("LDCS_RANKINFO");
          unsetf("LDCS_OPTIONS");
@@ -198,13 +196,12 @@ static char **updateEnvironment(char **envp, int *num_modified, int propogate_sp
    if (envp) {
       debug_printf2("Propogating spindle environment by copying it to new envp list\n");
       for (cur = (char **) envp; *cur; cur++, orig_size++);
-      new_size = orig_size + 10;
+      new_size = orig_size + 9;
       newenv = (char **) malloc(new_size * sizeof(char*));
 
       propogateEnvironmentStr(envp, newenv, &pos, "SPINDLE");   
       propogateEnvironmentStr(envp, newenv, &pos, "LD_AUDIT");
       propogateEnvironmentStr(envp, newenv, &pos, "LDCS_LOCATION");
-      propogateEnvironmentStr(envp, newenv, &pos, "LDCS_ORIG_LOCATION");
       propogateEnvironmentStr(envp, newenv, &pos, "LDCS_CONNECTION");
       propogateEnvironmentStr(envp, newenv, &pos, "LDCS_RANKINFO");
       propogateEnvironmentStr(envp, newenv, &pos, "LDCS_OPTIONS");

src/client/client/intercept_readlink.c

@@ -31,19 +31,19 @@ Place, Suite 330, Boston, MA 02111-1307 USA
 ssize_t (*orig_readlink)(const char *path, char *buf, size_t bufsiz);
 ssize_t (*orig_readlinkat)(int dirfd, const char *pathname, char *buf, size_t bufsiz);
 
-extern char *location;
 
 static int fix_local_readlink(char *buf, size_t bufsiz)
 {
    char spindle_id[32];
-   int location_len, result;
+   int cachepath_len, result;
    char tmp[MAX_PATH_LEN+1];
+   extern char *chosen_realized_cachepath;
 
-   location_len = strlen(location);   
+   cachepath_len = strlen(chosen_realized_cachepath);   
    snprintf(spindle_id, sizeof(spindle_id), "spindle.%lx", number);
-   if (strstr(buf, spindle_id) && strncmp(location, buf, location_len) == 0) {
+   if (strstr(buf, spindle_id) && strncmp(chosen_realized_cachepath, buf, cachepath_len) == 0) {
       debug_printf2("readlink received spindle cache path %s. Translating\n", buf);
-      result = send_orig_path_request(ldcsid, buf+location_len+1, tmp);
+      result = send_orig_path_request(ldcsid, buf+cachepath_len+1, tmp);
       if (result == -1)
          return -1;
       debug_printf2("readlink translated spindle local path %s to %s\n", buf, tmp);

src/client/client/should_intercept.c

@@ -30,21 +30,19 @@
 
 extern int relocate_spindleapi();
 
-extern char *location;
-extern char *orig_location;
-
 int is_in_spindle_cache(const char *pathname)
 {
-   static int location_size = 0;
-   static int orig_location_size = 0;
-   if (!location_size) {
-      location_size = strlen(location);
+   static int cachepath_size = 0;
+   static int orig_cachepath_size = 0;
+   extern char *chosen_realized_cachepath, *chosen_parsed_cachepath;
+   if (!cachepath_size) {
+      cachepath_size = strlen(chosen_realized_cachepath);
    }
-   if (!orig_location_size) {
-      orig_location_size = strlen(orig_location);
+   if (!orig_cachepath_size) {
+      orig_cachepath_size = strlen(chosen_parsed_cachepath);
    }
-   return ((strncmp(pathname, location, location_size) == 0) ||
-           (strncmp(pathname, orig_location, orig_location_size) == 0));
+   return ((strncmp(pathname, chosen_realized_cachepath, cachepath_size) == 0) ||
+           (strncmp(pathname, chosen_parsed_cachepath, orig_cachepath_size) == 0));
 }
 
 extern int is_local_prefix(const char *path, char **cached_local_prefixes);

src/client/client_comlib/client_api.c

@@ -36,7 +36,41 @@ static struct lock_t comm_lock;
 
 #define COMM_LOCK do { if (lock(&comm_lock) == -1) return -1; } while (0)
 #define COMM_UNLOCK unlock(&comm_lock)
-
+
+
+int send_cachepath_query( int fd, char **chosen_realized_cachepath, char **chosen_parsed_cachepath){
+   ldcs_message_t message;
+   char buffer[MAX_PATH_LEN+1];
+   buffer[MAX_PATH_LEN] = '\0';
+
+   message.header.type = LDCS_MSG_CHOSEN_CACHEPATH_REQUEST;
+   message.header.len = MAX_PATH_LEN;
+   message.data = buffer;
+
+   COMM_LOCK;
+
+   debug_printf3("sending message of type: request_location_path.\n" );
+   client_send_msg(fd, &message);
+   client_recv_msg_static(fd, &message, LDCS_READ_BLOCK);
+
+   COMM_UNLOCK;
+
+   if (message.header.type != LDCS_MSG_CHOSEN_CACHEPATH || message.header.len > MAX_PATH_LEN) {
+      err_printf("Got unexpected message of type %d\n", (int) message.header.type);
+      return -1;
+   }
+   char *local_crc = strdup( buffer );
+   char *local_cpc = strdup( &buffer[ strlen(local_crc) + 1 ] );
+   if( chosen_realized_cachepath ){
+       *chosen_realized_cachepath = local_crc;
+   }
+   if( chosen_parsed_cachepath ){
+       *chosen_parsed_cachepath = local_cpc;
+   }
+
+   return 0;
+}
+
 int send_file_query(int fd, char* path, int dso, char** newpath, int *errcode) {
    ldcs_message_t message;
    char buffer[MAX_PATH_LEN+1+sizeof(int)];
@@ -68,7 +102,7 @@ int send_file_query(int fd, char* path, int dso, char** newpath, int *errcode) {
 
    if (message.header.type != LDCS_MSG_FILE_QUERY_ANSWER) {
       err_printf("Got unexpected message of type %d\n", (int) message.header.type);
-      assert(0);
+      return -1;
    }
 
    if (message.header.len > sizeof(int)) {
@@ -161,7 +195,7 @@ int send_existance_test(int fd, char *path, int *exists)
 
    if (message.header.type != LDCS_MSG_EXISTS_ANSWER || message.header.len != sizeof(uint32_t)) {
       err_printf("Got unexpected message after existance test: %d\n", (int) message.header.type);
-      assert(0);
+      return -1;
    }
 
    memcpy(exists, buffer, sizeof(*exists));
@@ -198,7 +232,7 @@ int send_orig_path_request(int fd, const char *path, char *newpath)
 
    if (message.header.type != LDCS_MSG_ORIGPATH_ANSWER || message.header.len > MAX_PATH_LEN) {
       err_printf("Got unexpected message after existance test: %d\n", (int) message.header.type);
-      assert(0);
+      return -1;
    }
    strncpy(newpath, buffer, MAX_PATH_LEN+1);
 
@@ -346,7 +380,7 @@ int send_ldso_info_request(int fd, const char *ldso_path, char *result_path)
 
    if (message.header.type != LDCS_MSG_LOADER_DATA_RESP) {
       err_printf("Got unexpected message after ldso req: %d\n", (int) message.header.type);
-      assert(0);
+      return -1;
    }
    return 0;
 }
@@ -388,7 +422,7 @@ int send_rankinfo_query(int fd, int *mylrank, int *mylsize, int *mymdrank, int *
    if (message.header.type != LDCS_MSG_MYRANKINFO_QUERY_ANSWER || message.header.len != 4*sizeof(int)) {
       err_printf("Received incorrect response to rankinfo query %d\n", message.header.type);
       *mylrank = *mylsize = *mymdrank = *mymdsize = -1;
-      assert(0);
+      return -1;
    }
 
    p = (int *) message.data;
@@ -423,7 +457,7 @@ int send_procmaps_query(int fd, int pid, char *result)
 
    if (message.header.type != LDCS_MSG_PROCMAPS_RESP) {
       err_printf("Received incorrect response to procmaps query %d\n", message.header.type);
-      assert(0);
+      return -1;
    }
 
    memcpy(result, buffer, MAX_PATH_LEN);
@@ -454,7 +488,7 @@ int send_pickone_query(int fd, char *key, int *result)
 
    if (message.header.type != LDCS_MSG_PICKONE_RESP) {
       err_printf("Received incorrect response to procmaps query %d\n", message.header.type);
-      assert(0);
+      return -1;
    }
 
    *result = *((int *) message.data);

src/client/client_comlib/client_api.h

@@ -42,6 +42,7 @@ int send_orig_path_request(int fd, const char *path, char *newpath);
 int send_dirlists_request(int fd, char **local_result, char **exece_result, char **to_free);
 int send_procmaps_query(int fd, int pid, char *result);
 int send_pickone_query(int fd, char *key, int *result);
+int send_cachepath_query( int fd, char **chosen_symbolic_cachepath, char **chosen_parsed_cachepath);
 
 int get_python_prefix(int fd, char **prefix);
 

src/client/config.h.in

@@ -6,6 +6,9 @@
 /* Whether we are using a broken srun */
 #undef BROKEN_SRUN
 
+/* Colon-separated list of potential back-end cache directories */
+#undef CACHEPATHS
+
 /* Define if were using biter for client/server communication */
 #undef COMM_BITER
 

src/client/configure

@@ -810,6 +810,7 @@ enable_libtool_lock
 enable_maintainer_mode
 with_default_port
 with_default_num_ports
+with_cachepaths
 with_localstorage
 with_default_local_prefix
 with_testrm
@@ -1532,6 +1533,8 @@ Optional Packages:
   --with-default-numports=NUM
                           Number of TCP/IP ports to scan for Spindle server
                           communication
+  --with-cachepaths=DIR   Colon-separated list of potential back-end cache
+                          directories
   --with-localstorage=DIR Directory on back-ends for storing relocated files
   --with-default-local-prefix=DIRS
                           Colon-seperated list of directories that Spindle
@@ -12587,6 +12590,14 @@ else
 fi
 
 
+# Check whether --with-cachepaths was given.
+if test "${with_cachepaths+set}" = set; then :
+  withval=$with_cachepaths; CACHEPATHS=${withval}
+else
+  CACHEPATHS=$DEFAULT_LOC
+fi
+
+
 # Check whether --with-localstorage was given.
 if test "${with_localstorage+set}" = set; then :
   withval=$with_localstorage; SPINDLE_LOC=${withval}
@@ -12623,6 +12634,11 @@ cat >>confdefs.h <<_ACEOF
 _ACEOF
 
 
+cat >>confdefs.h <<_ACEOF
+#define CACHEPATHS "$CACHEPATHS"
+_ACEOF
+
+
 cat >>confdefs.h <<_ACEOF
 #define SPINDLE_LOCAL_PREFIX "$SPINDLE_LOCAL_PREFIX"
 _ACEOF