Speed up find-german-comments

Instead of recreating a text_cat sub-process for every string, start a
single text-cat process (with the -s flag, "per line") and reuse that.

Testing on svl/source shows a speed-up of 7.5 x, down from 45.6s to 6.1s!

To avoid blocking on the pipes set autoflush in text-cat, and use
read_line instead of read on the find-german-comments side.

Change-Id: Ic6c00fc3c1e7cbcbbfff65512bf54ad11f900697
Reviewed-on: https://gerrit.libreoffice.org/9226
Reviewed-by: Michael Meeks <michael.meeks@collabora.com>
Tested-by: Michael Meeks <michael.meeks@collabora.com>

bin/find-german-comments

@@ -36,6 +36,7 @@ class Parser:
     """
     def __init__(self):
         self.strip = string.punctuation + " \n"
+        self.text_cat = self.start_text_cat();
         op = optparse.OptionParser()
         op.set_usage("%prog [options] <rootdir>\n\n" +
             "Searches for german comments in cxx/hxx source files inside a given root\n" +
@@ -117,19 +118,23 @@ class Parser:
                 yield (count, re.sub(".*/\*(.*)\*/.*", r"\1", i).strip(self.strip))
             count += 1
 
+    def start_text_cat(self):
+        cwd = os.getcwd()
+        # change to our directory
+        os.chdir(os.path.split(os.path.abspath(sys.argv[0]))[0])
+        sock = subprocess.Popen(["text_cat/text_cat", "-s", "-d", "text_cat/LM"], stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+        os.chdir(cwd)
+        return sock
+
     def get_lang(self, s):
         """ the output is 'german' or 'english' or 'german or english'. when
         unsure, just don't warn, there are strings where you just can't
         teremine the results reliably, like '#110680#' """
-        cwd = os.getcwd()
-        # change to our directory
-        os.chdir(os.path.split(os.path.abspath(sys.argv[0]))[0])
-        sock = subprocess.Popen(["text_cat/text_cat", "-d", "text_cat/LM"], stdin=subprocess.PIPE, stdout=subprocess.PIPE)
-        sock.stdin.write(s)
-        sock.stdin.close()
-        lang = sock.stdout.read().strip()
-        sock.stdout.close()
-        os.chdir(cwd)
+
+        self.text_cat.stdin.write(s)
+        self.text_cat.stdin.write("\n")
+        self.text_cat.stdin.flush()
+        lang = self.text_cat.stdout.readline().strip()
         return lang
 
     def is_german(self, s):

bin/text_cat/text_cat

@@ -19,6 +19,8 @@ $opt_f ||= 0;
 $opt_t ||= 400;
 $opt_u ||= 1.05;
 
+$| = 1; # auto-flush stdout
+
 sub help {
     print <<HELP
 Text Categorization. Typically used to determine the language of a