rationalize -memlimit implementation: use long long to protect agains…

…t overflows, and also count static dag storage against the limit; also fix accounting inaccuracies in unpacking memory usage reported to [incr tsdb()]

git-svn-id: https://pet.opendfki.de/repos/pet/main@920 4200e16c-5112-0410-ac55-d7fb557a720a

cheap/fs.h

@@ -411,7 +411,7 @@ class fs_alloc_state
      * or temporary fs's that have been created so far (as if memory was not
      * reused). The value can be reset by clear_stats().
      */
-    long dynamic_usage()
+    long long dynamic_usage()
     {
         return dag_alloc_dynamic_mem();
     }
@@ -421,7 +421,7 @@ class fs_alloc_state
      * temporary fs's. The value can be reset by clear_stats(), which
      * has only a visible effect if allocated memory may shrink.
      */
-    long static_usage()
+    long long static_usage()
     {
         return dag_alloc_static_mem();
     }

cheap/item.cpp

@@ -864,7 +864,7 @@ int unpacking_level;
 
 inline bool unpacking_resources_exhausted(long memlimit) {
   // TODO add other limits
-  return memlimit > 0 && t_alloc.max_usage() >= memlimit;
+  return memlimit > 0 && (p_alloc.max_usage_mb() + t_alloc.max_usage_mb()) >= memlimit;
 }
 
 list<tItem *>
@@ -1020,7 +1020,7 @@ tPhrasalItem::unpack_cross(vector<list<tItem *> > &dtrs,
 // should be factored out.
 tItem *
 tPhrasalItem::unpack_combine(vector<tItem *> &daughters) {
-  long memlimit = get_opt_int("opt_memlimit") * 1024 * 1024;
+  long memlimit = get_opt_int("opt_memlimit");
 
   fs_alloc_state FSAS(false);
 
@@ -1047,8 +1047,7 @@ tPhrasalItem::unpack_combine(vector<tItem *> &daughters) {
 
   if (unpacking_resources_exhausted(memlimit)) {
     ostringstream s;
-    s << "memory limit exhausted (" << memlimit / (1024 * 1024)
-      << " MB)";
+    s << "memory limit exhausted (" << memlimit << " MB)";
     throw tError(s.str());
   }
 
@@ -1514,8 +1513,7 @@ tPhrasalItem::instantiate_hypothesis(list<tItem*> path, tHypothesis * hypo, int
 
     if (unpacking_resources_exhausted(memlimit)) {
       ostringstream s;
-      s << "memory limit exhausted (" << memlimit / (1024 * 1024)
-        << " MB)";
+      s << "memory limit exhausted (" << memlimit << " MB)";
       throw tError(s.str());
     }
 

cheap/parse.cpp

@@ -414,10 +414,11 @@ bool add_item(tItem *it) {
 }
 
 inline bool
-resources_exhausted(int pedgelimit, long memlimit, int timeout, int timestamp)
+resources_exhausted(int pedgelimit, long memlimit, 
+                    int timeout, int timestamp)
 {
   return (pedgelimit > 0 && Chart->pedges() >= pedgelimit) ||
-    (memlimit > 0 && t_alloc.max_usage() >= memlimit) ||
+    (memlimit > 0 && (p_alloc.max_usage_mb() + t_alloc.max_usage_mb()) >= memlimit) ||
     (timeout > 0 && timestamp >= timeout );
 }
 
@@ -451,11 +452,11 @@ test_resource_limits(std::string &message)
   } //if
 
   static long int memory = -1;
-  if(memory == -1) memory = get_opt_int("opt_memlimit") * 1024 * 1024;
-  if(memory > 0 && t_alloc.max_usage() >= memory) {
+  if(memory == -1) memory = get_opt_int("opt_memlimit");
+  if(memory > 0 && (p_alloc.max_usage_mb() + t_alloc.max_usage_mb()) >= memory) {
     ostringstream buffer;
     buffer << "resource limit exhausted (" 
-           << t_alloc.max_usage() << " bytes)";
+           << memory << " MB)";
     message = buffer.str();
     return true;
   } // if
@@ -478,7 +479,7 @@ test_resource_limits(std::string &message)
 
 void
 parse_loop(fs_alloc_state &FSAS, list<tError> &errors, clock_t timeout) {
-  long memlimit = get_opt_int("opt_memlimit") * 1024 * 1024;
+  long memlimit = get_opt_int("opt_memlimit");
   int pedgelimit = get_opt_int("opt_pedgelimit");
 
   //
@@ -511,7 +512,7 @@ int unpack_selectively(std::vector<tItem*> &trees, int upedgelimit,
                        long memlimit, int nsolutions,
                        timer *UnpackTime , vector<tItem *> &readings) {
   int nres = 0;
-  if (memlimit > 0 && t_alloc.max_usage() >= memlimit)
+  if (memlimit > 0 && (p_alloc.max_usage_mb() + t_alloc.max_usage_mb()) >= memlimit)
     //
     // _fix_me_
     // for all i can tell, the actual selective unpacking code does not always
@@ -649,8 +650,8 @@ collect_readings(fs_alloc_state &FSAS, list<tError> &errors,
       }
 
       stats.p_utcpu = UnpackTime->convert2ms(UnpackTime->elapsed());
-      stats.p_dyn_bytes = FSAS.dynamic_usage();
-      stats.p_stat_bytes = FSAS.static_usage();
+      stats.p_dyn_bytes = FSAS.dynamic_usage() - stats.dyn_bytes;
+      stats.p_stat_bytes = FSAS.static_usage() - stats.stat_bytes;
       FSAS.clear_stats();
       delete UnpackTime;
     }
@@ -668,7 +669,7 @@ collect_readings(fs_alloc_state &FSAS, list<tError> &errors,
 
 void
 parse_finish(fs_alloc_state &FSAS, list<tError> &errors, clock_t timeout) {
-  long memlimit = get_opt_int("opt_memlimit") * 1024 * 1024;
+  long memlimit = get_opt_int("opt_memlimit");
   int pedgelimit = get_opt_int("opt_pedgelimit");
   clock_t timestamp = (timeout > 0 ? times(NULL) : 0);
 
@@ -682,8 +683,9 @@ parse_finish(fs_alloc_state &FSAS, list<tError> &errors, clock_t timeout) {
 
   if(resources_exhausted(pedgelimit, memlimit, timeout, timestamp)) {
     ostringstream s;
-    if (memlimit > 0 && t_alloc.max_usage() >= memlimit) {
-      s << "memory limit exhausted (" << memlimit / (1024 * 1024) << " MB)";
+    if (memlimit > 0 
+        && (p_alloc.max_usage_mb() + t_alloc.max_usage_mb()) >= memlimit) {
+      s << "memory limit exhausted (" << memlimit << " MB)";
     }
     else if (pedgelimit > 0 && Chart->pedges() >= pedgelimit) {
       s << "edge limit exhausted (" << pedgelimit << " pedges)";

cheap/pcfg.cpp

@@ -79,9 +79,9 @@ static bool initialized = init();
 inline bool
 pcfg_resources_exhausted(int pedgelimit, long memlimit, int timeout, int timestamp)
 {
-  return (pedgelimit > 0 && Chart->pedges() >= pedgelimit) ||
-    (memlimit > 0 && t_alloc.max_usage() >= memlimit) ||
-    (timeout > 0 && timestamp >= timeout );
+  return (pedgelimit > 0 && Chart->pedges() >= pedgelimit) 
+    || (memlimit > 0 && (p_alloc.max_usage_mb() + t_alloc.max_usage_mb()) >= memlimit)
+    || (timeout > 0 && timestamp >= timeout );
 }
 
 
@@ -322,7 +322,7 @@ add_item_pcfg(tItem *it) {
 
 void
 parse_loop_pcfg() {
-  long memlimit = get_opt_int("opt_memlimit") * 1024 * 1024;
+  long memlimit = get_opt_int("opt_memlimit");
   int pedgelimit = get_opt_int("opt_pedgelimit");
   while (!Agenda->empty() &&
          ! pcfg_resources_exhausted(pedgelimit, memlimit, timeout, timestamp)) {
@@ -428,7 +428,7 @@ int unpack_selectively_pcfg(vector<tItem*> &trees, int upedgelimit,
 
 void parse_finish_pcfg(fs_alloc_state &FSAS, list<tError> &errors) {
   // _todo_ modify so that proper unpacking routines are invoked
-  long memlimit = get_opt_int("opt_memlimit") * 1024 * 1024;
+  long memlimit = get_opt_int("opt_memlimit");
   int pedgelimit = get_opt_int("opt_pedgelimit");
   clock_t timestamp = (timeout > 0 ? times(NULL) : 0);
 
@@ -437,9 +437,8 @@ void parse_finish_pcfg(fs_alloc_state &FSAS, list<tError> &errors) {
   if(pcfg_resources_exhausted(pedgelimit, memlimit, timeout, timestamp)) {
     ostringstream s;
 
-    if (memlimit > 0 && t_alloc.max_usage() >= memlimit)
-      s << "memory limit exhausted (" << memlimit / (1024 * 1024)
-        << " MB)";
+    if (memlimit > 0 && (p_alloc.max_usage_mb() + t_alloc.max_usage_mb()) >= memlimit)
+      s << "memory limit exhausted (" << memlimit << " MB)";
     else if (pedgelimit > 0 && Chart->pedges() >= pedgelimit)
       s << "edge limit exhausted (" << pedgelimit
         << " pedges)";

cheap/tsdb++.cpp

@@ -133,8 +133,8 @@ statistics::print(FILE *f)
            "medges: %d\n"
            "unifications_succ: %d\nunifications_fail: %d\n"
            "subsumptions_succ: %d\nsubsumptions_fail: %d\ncopies: %d\n"
-           "dyn_bytes: %ld\nstat_bytes: %ld\n"
-           "p_dyn_bytes: %ld\np_nstat_bytes: %ld\n"
+           "dyn_bytes: %lld\nstat_bytes: %lld\n"
+           "p_dyn_bytes: %lld\np_stat_bytes: %lld\n"
            "cycles: %d\nfssize: %d\n"
            "unify_cost_succ: %d\nunify_cost_fail: %d\n"
            "equivalent: %d\nproactive: %d\nretroactive: %d\n"
@@ -516,7 +516,7 @@ tsdb_parse::capi_print()
         capi_printf("(:treal . %d) ", treal);
 
     if(others != -1)
-        capi_printf("(:others . %ld) ", others);
+        capi_printf("(:others . %lld) ", others);
 
     if(p_ftasks != -1)
         capi_printf("(:p-ftasks . %d) ", p_ftasks);
@@ -792,7 +792,7 @@ cheap_tsdb_summarize_error(list<tError> &conditions, int treal, tsdb_parse &T)
     T.tgc = 0;
     T.treal = treal;
 
-    T.others = stats.stat_bytes;
+    T.others = stats.stat_bytes + stats.p_stat_bytes;
 
     T.p_ftasks = stats.ftasks_fi + stats.ftasks_qc;
     T.p_etasks = stats.etasks;

cheap/tsdb++.h

@@ -119,9 +119,9 @@ class statistics
   /** nr of copies */
   int copies;
   /** total dynamic memory in bytes */
-  long dyn_bytes;
+  long long dyn_bytes;
   /** total static memory in bytes */
-  long stat_bytes;
+  long long stat_bytes;
   /** cycles found */
   int cycles;
   /** avg size of all passive edges */
@@ -152,9 +152,9 @@ class statistics
   /** hypotheses in unpacking */
   int p_hypotheses;
   /** total dynamic memory in bytes in unpacking */
-  long p_dyn_bytes;
+  long long p_dyn_bytes;
   /** total static memory in bytes in unpacking */
-  long p_stat_bytes;
+  long long p_stat_bytes;
   /*@}*/
 
   void reset();
@@ -369,7 +369,7 @@ class tsdb_parse
   /** symbols allocated */
   int symbols;
   /** bytes of memory allocated */
-  long int others;
+  long long int others;
   /** number of garbage collections */
   int gcs;
   /** initial load (start of parse) */

common/chunk-alloc.h

@@ -90,8 +90,8 @@ class chunk_allocator
   inline int chunksize() { return _chunk_size; }
 
   /** The amount of memory currently allocated */
-  inline long int allocated()
-    { return _curr_chunk * _chunk_size + _chunk_pos; }
+  inline long long int allocated()
+    { return (long long)_curr_chunk * (long long)_chunk_size + _chunk_pos; }
 
   /** Pointer to the next free memory address */
   inline void *current()
@@ -126,8 +126,10 @@ class chunk_allocator
   void reset();
 
   /** The maximum amount of memory (in bytes) allocated so far */
-  inline long int max_usage()
+  inline long long int max_usage()
     { return _max; }
+  inline long long int max_usage_mb()
+    { return _max / (1024 * 1024); }
 
   /** Reset maximum allocated size counter.
    * Calling this method only makes sense after the total allocated memory
@@ -156,7 +158,7 @@ class chunk_allocator
   char **_chunk;
 
   /** max nr of bytes allocated so far */
-  long int _max;
+  long long int _max;
 
 
   void _overflow(int n);

common/dag-alloc.cpp

@@ -24,12 +24,12 @@
 
 int allocated_nodes = 0, allocated_arcs = 0;
 
-long dag_alloc_dynamic_mem()
+long long dag_alloc_dynamic_mem()
 {
   return sizeof(dag_node) * allocated_nodes + sizeof(dag_arc) * allocated_arcs;
 }
 
-long dag_alloc_static_mem()
+long long dag_alloc_static_mem()
 {
   return t_alloc.max_usage();
 }

common/dag-alloc.h

@@ -46,14 +46,14 @@ struct dag_alloc_state
  * reused). The value can be reset by dag_alloc_clear_stats().
  * \todo is documentation this correct?? 
  */
-long dag_alloc_dynamic_mem();
+long long dag_alloc_dynamic_mem();
 /**
  * Reports how much memory (in bytes) is currently allocated for storing
  * temporary dags. The value can be reset by dag_alloc_clear_stats(), which
  * has only a visible effect if allocated memory may shrink.
  * \todo is documentation this correct?? 
  */
-long dag_alloc_static_mem();
+long long dag_alloc_static_mem();
 /** Reset the statistics of static and dynamic memory usage */
 void dag_alloc_clear_stats();
 /*@}*/