Add cross-platform randomizer module.

The code uses two mechanisms for generating random numbers: srand()/rand(),
which is not thread-safe, and srandom()/random(), which is POSIX-specific.

Here I add a util/random.cc module that centralizes these calls, and unifies
some common usage patterns.  If the implementation is not good enough, we can
now change it in a single place.

To keep things simple, this uses the portable srand()/rand() but protects them
with a lock to avoid concurrency problems.

The hard part was to keep the regression tests passing: they rely on fixed
sequences of random numbers, so a small code change could break them very
thoroughly.  Util::rand(), for wide types like size_t, calls std::rand() not
once but twice.  This behaviour was generalized into utils::wide_rand() and
friends.

Author: Jeroen Vermeulen

contrib/mira/Main.h

@@ -25,6 +25,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 #include "moses/Word.h"
 #include "moses/FF/FeatureFunction.h"
 #include "Decoder.h"
+#include "util/random.hh"
 
 typedef std::map<const Moses::FeatureFunction*, std::vector< float > > ProducerWeightMap;
 typedef std::pair<const Moses::FeatureFunction*, std::vector< float > > ProducerWeightPair;
@@ -37,8 +38,11 @@ template <class T> bool from_string(T& t, const std::string& s, std::ios_base& (
 
 struct RandomIndex {
   ptrdiff_t operator()(ptrdiff_t max) {
-    srand(time(0));  // Initialize random number generator with current time.
-    return static_cast<ptrdiff_t> (rand() % max);
+    // TODO: Don't seed the randomizer here.  If this function gets called
+    // multiple times in the same second, it will return the same value on
+    // each of those calls.
+    util::rand_init();
+    return util::rand_excl(max);
   }
 };
 

contrib/relent-filter/src/Main.cpp

@@ -42,6 +42,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 #include "RelativeEntropyCalc.h"
 #include "LexicalReordering.h"
 #include "LexicalReorderingState.h"
+#include "util/random.hh"
 
 #ifdef HAVE_PROTOBUF
 #include "hypergraph.pb.h"
@@ -205,7 +206,7 @@ int main(int argc, char** argv)
 
 
     //initialise random numbers
-    srand(time(NULL));
+    rand_init();
 
     // set up read/writing class
     IOWrapper* ioWrapper = GetIOWrapper(staticData);

mert/Data.cpp

@@ -287,7 +287,7 @@ void Data::createShards(size_t shard_count, float shard_size, const string& scor
     } else {
       //create shards by randomly sampling
       for (size_t i = 0; i < floor(shard_size+0.5); ++i) {
-        shard_contents.push_back(util::rand_int() % data_size);
+        shard_contents.push_back(util::rand_excl(data_size));
       }
     }
 

mert/Point.cpp

@@ -58,10 +58,8 @@ void Point::Randomize()
   UTIL_THROW_IF(m_min.size() != Point::m_dim, util::Exception, "Error");
   UTIL_THROW_IF(m_max.size() != Point::m_dim, util::Exception, "Error");
 
-  for (unsigned int i = 0; i < size(); i++) {
-    const float scale = (m_max[i] - m_min[i]) / float(RAND_MAX);
-    operator[](i) = m_min[i] + util::rand_int() * scale;
-  }
+  for (unsigned int i = 0; i < size(); i++)
+    operator[](i) = util::rand_incl(m_min[i], m_max[i]);
 }
 
 double Point::operator*(const FeatureStats& F) const

mert/TODO

@@ -5,11 +5,8 @@
 
 - check that --pairwise-ranked is compatible with all optimization metrics
 
-- Replace the standard rand() currently used in MERT and PRO with better
-  random generators such as Boost's random generators (e.g., boost::mt19937).
-  - create a Random class to hide the details, i.e., how to generate
-    random numbers, which allows us to use custom random generators more
-    easily.
+- Use better random generators in util/random.cc, e.g. boost::mt19937.
+  - Support plugging of custom random generators.
 
   Pros:
   - In MERT, you might want to use the random restarting technique to avoid

mert/evaluator.cpp

@@ -95,7 +95,7 @@ void EvaluatorUtil::evaluate(const string& candFile, int bootstrap, bool nbest_i
     for (int i = 0; i < bootstrap; ++i) {
       ScoreData scoredata(g_scorer);
       for (int j = 0; j < n; ++j) {
-        int randomIndex = util::rand_int() % n;
+        const int randomIndex = util::rand_excl(n);
         scoredata.add(entries[randomIndex], j);
       }
       g_scorer->setScoreData(&scoredata);
@@ -285,10 +285,10 @@ void InitSeed(const ProgramOption *opt)
 {
   if (opt->has_seed) {
     cerr << "Seeding random numbers with " << opt->seed << endl;
-    util::rand_int_init(opt->seed);
+    util::rand_init(opt->seed);
   } else {
     cerr << "Seeding random numbers with system clock " << endl;
-    util::rand_int_init();
+    util::rand_init();
   }
 }
 

mert/kbmira.cpp

@@ -40,6 +40,7 @@ de recherches du Canada
 #include <boost/scoped_ptr.hpp>
 
 #include "util/exception.hh"
+#include "util/random.hh"
 
 #include "BleuScorer.h"
 #include "HopeFearDecoder.h"
@@ -122,10 +123,10 @@ int main(int argc, char** argv)
 
   if (vm.count("random-seed")) {
     cerr << "Initialising random seed to " << seed << endl;
-    srand(seed);
+    util::rand_init(seed);
   } else {
     cerr << "Initialising random seed from system clock" << endl;
-    srand(time(NULL));
+    util::rand_init();
   }
 
   // Initialize weights

mert/mert.cpp

@@ -290,10 +290,10 @@ int main(int argc, char **argv)
 
   if (option.has_seed) {
     cerr << "Seeding random numbers with " << option.seed << endl;
-    util::rand_int_init(option.seed);
+    util::rand_init(option.seed);
   } else {
     cerr << "Seeding random numbers with system clock " << endl;
-    util::rand_int_init();
+    util::rand_init();
   }
 
   if (option.sparse_weights_file.size()) ++option.pdim;

mert/pro.cpp

@@ -43,6 +43,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 #include "ScoreDataIterator.h"
 #include "BleuScorer.h"
 #include "Util.h"
+#include "util/random.hh"
 
 using namespace std;
 using namespace MosesTuning;
@@ -141,10 +142,10 @@ int main(int argc, char** argv)
 
   if (vm.count("random-seed")) {
     cerr << "Initialising random seed to " << seed << endl;
-    srand(seed);
+    util::rand_init(seed);
   } else {
     cerr << "Initialising random seed from system clock" << endl;
-    srand(time(NULL));
+    util::rand_init();
   }
 
   if (scoreFiles.size() == 0 || featureFiles.size() == 0) {
@@ -211,11 +212,11 @@ int main(int argc, char** argv)
     vector<float> scores;
     size_t n_translations = hypotheses.size();
     for(size_t  i=0; i<n_candidates; i++) {
-      size_t rand1 = rand() % n_translations;
+      size_t rand1 = util::rand_excl(n_translations);
       pair<size_t,size_t> translation1 = hypotheses[rand1];
       float bleu1 = smoothedSentenceBleu(scoreDataIters[translation1.first]->operator[](translation1.second), bleuSmoothing, smoothBP);
 
-      size_t rand2 = rand() % n_translations;
+      size_t rand2 = util::rand_excl(n_translations);
       pair<size_t,size_t> translation2 = hypotheses[rand2];
       float bleu2 = smoothedSentenceBleu(scoreDataIters[translation2.first]->operator[](translation2.second), bleuSmoothing, smoothBP);
 

moses-cmd/MainVW.cpp

@@ -45,6 +45,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 #include "moses/FF/StatefulFeatureFunction.h"
 #include "moses/FF/StatelessFeatureFunction.h"
 #include "moses/TrainingTask.h"
+#include "util/random.hh"
 
 #ifdef HAVE_PROTOBUF
 #include "hypergraph.pb.h"
@@ -117,7 +118,7 @@ int main(int argc, char** argv)
 
 
     //initialise random numbers
-    srand(time(NULL));
+    util::rand_init();
 
     // set up read/writing class
     IFVERBOSE(1) {

moses/ExportInterface.cpp

@@ -27,6 +27,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 #include <sstream>
 #include <vector>
 
+#include "util/random.hh"
 #include "util/usage.hh"
 
 #ifdef WIN32
@@ -91,7 +92,7 @@ SimpleTranslationInterface::SimpleTranslationInterface(const string &mosesIni):
       exit(1);
     }
 
-    srand(time(NULL));
+    util::rand_init();
 
 }
 
@@ -185,7 +186,7 @@ batch_run()
   const StaticData& staticData = StaticData::Instance();
 
   //initialise random numbers
-  srand(time(NULL));
+  util::rand_init();
 
   IFVERBOSE(1) PrintUserTime("Created input-output object");
 

moses/Manager.cpp

@@ -54,6 +54,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 #endif
 
 #include "util/exception.hh"
+#include "util/random.hh"
 
 using namespace std;
 
@@ -418,7 +419,7 @@ void Manager::CalcLatticeSamples(size_t count, TrellisPathList &ret) const
       //cerr << endl;
 
       //draw the sample
-      float frandom = log((float)rand()/RAND_MAX);
+      const float frandom = log(util::rand_incl(0.0f, 1.0f));
       size_t position = 1;
       float sum = candidateScores[0];
       for (; position < candidateScores.size() && sum < frandom; ++position) {

moses/Parameter.cpp

@@ -31,6 +31,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 #include "InputFileStream.h"
 #include "StaticData.h"
 #include "util/exception.hh"
+#include "util/random.hh"
 #include <boost/program_options.hpp>
 
 
@@ -1392,7 +1393,7 @@ struct Credit {
     this->contact							= contact						;
     this->currentPursuits			= currentPursuits		;
     this->areaResponsibility	= areaResponsibility;
-    this->sortId							= rand() % 1000;
+    this->sortId							= util::rand_excl(1000);
   }
 
   bool operator<(const Credit &other) const {

moses/TranslationModel/DynSAInclude/hash.h

@@ -6,6 +6,7 @@
 #include "utils.h"
 #include "FileHandler.h"
 #include "util/exception.hh"
+#include "util/random.hh"
 
 using namespace Moses;
 typedef uint64_t P;   // largest input range is 2^64
@@ -162,7 +163,7 @@ void Hash_shiftAddXOR<T>::initSeeds()
 {
   v_ = new T[this->H_];
   for(count_t i=0; i < this->H_; i++)
-    v_[i] = Utils::rand<T>() + 1;
+    v_[i] = util::wide_rand<T>() + 1;
 }
 template <typename T>
 T Hash_shiftAddXOR<T>::hash(const char* s, count_t h)
@@ -187,9 +188,8 @@ void UnivHash_tableXOR<T>::initSeeds()
   // fill with random values
   for(count_t j=0; j < this->H_; j++) {
     table_[j] = new T[tblLen_];
-    for(count_t i=0; i < tblLen_; i++) {
-      table_[j][i] = Utils::rand<T>(this->m_-1);
-    }
+    for(count_t i=0; i < tblLen_; i++)
+      table_[j][i] = util::wide_rand_excl(this->m_-1);
   }
 }
 template <typename T>
@@ -218,7 +218,7 @@ void UnivHash_noPrimes<T>::initSeeds()
 {
   a_ = new P[this->H_];
   for(T i=0; i < this->H_; i++) {
-    a_[i] = Utils::rand<P>();
+    a_[i] = util::wide_rand<P>();
     if(a_[i] % 2 == 0) a_[i]++;  // a must be odd
   }
 }
@@ -284,8 +284,8 @@ void UnivHash_linear<T>::initSeeds()
     a_[i] = new T[MAX_NGRAM_ORDER];
     b_[i] = new T[MAX_NGRAM_ORDER];
     for(count_t j=0; j < MAX_NGRAM_ORDER; j++) {
-      a_[i][j] = 1 + Utils::rand<T>();
-      b_[i][j] = Utils::rand<T>();
+      a_[i][j] = 1 + util::wide_rand<T>();
+      b_[i][j] = util::wide_rand<T>();
     }
   }
 }

moses/TranslationModel/DynSAInclude/utils.h

@@ -62,22 +62,6 @@ class Utils
       str[i] = tolower(str[i]);
     }
   }
-  // TODO: interface with decent PRG
-  template<typename T>
-  static T rand(T mod_bnd = 0) {
-    T random = 0;
-    if(sizeof(T) <= 4) {
-      random = static_cast<T>(std::rand());
-    } else if(sizeof(T) == 8) {
-      random = static_cast<T>(std::rand());
-      random <<= 31;
-      random <<= 1;
-      random |= static_cast<T>(std::rand());
-    }
-    if(mod_bnd != 0)
-      return random % mod_bnd;
-    else return random;
-  }
 };
 
 #endif

moses/TranslationModel/DynSuffixArray.cpp

@@ -1,4 +1,6 @@
 #include "DynSuffixArray.h"
+#include "util/random.hh"
+
 #include <iostream>
 #include <boost/foreach.hpp>
 
@@ -315,33 +317,31 @@ int DynSuffixArray::Compare(int pos1, int pos2, int max)
   return 0;
 }
 
+namespace
+{
+/// Helper: swap two entries in an int array.
+inline void swap_ints(int array[], int one, int other)
+{
+  const int tmp = array[one];
+  array[one] = array[other];
+  array[other] = tmp;
+}
+}
+
 void DynSuffixArray::Qsort(int* array, int begin, int end)
 {
   if(end > begin) {
-    int index;
+    int index = util::rand_incl(begin, end);
     {
-      index = begin + (rand() % (end - begin + 1));
-      int pivot = array[index];
-      {
-        int tmp = array[index];
-        array[index] = array[end];
-        array[end] = tmp;
-      }
+      const int pivot = array[index];
+      swap_ints(array, index, end);
       for(int i=index=begin; i < end; ++i) {
         if (Compare(array[i], pivot, 20) <= 0) {
-          {
-            int tmp = array[index];
-            array[index] = array[i];
-            array[i] = tmp;
-            index++;
-          }
+          swap_ints(array, index, i);
+          index++;
         }
       }
-      {
-        int tmp = array[index];
-        array[index] = array[end];
-        array[end] = tmp;
-      }
+      swap_ints(array, index, end);
     }
     Qsort(array, begin, index - 1);
     Qsort(array, index + 1,  end);

moses/TranslationModel/RuleTable/PhraseDictionaryFuzzyMatch.cpp

@@ -45,6 +45,7 @@
 #include "moses/TranslationModel/fuzzy-match/SentenceAlignment.h"
 #include "util/file.hh"
 #include "util/exception.hh"
+#include "util/random.hh"
 
 using namespace std;
 
@@ -62,8 +63,8 @@ char *mkdtemp(char *tempbuf)
     return NULL;
   }
 
-  srand((unsigned)time(0));
-  rand_value = (int)((rand() / ((double)RAND_MAX+1.0)) * 1e6);
+  util::rand_init();
+  rand_value = rand_excl(1e6);
   tempbase = strrchr(tempbuf, '/');
   tempbase = tempbase ? tempbase+1 : tempbuf;
   strcpy(tempbasebuf, tempbase);